Commit 3687eada6a4c7a2f9444b090f49094c9755cb364 | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-07-23 14:20:58-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2025-07-25 14:15:33-07:00
log	3687eada6a4c7a2f9444b090f49094c9755cb364
tree	84de51657e54833d4b258962cb1361b45c4ef18a
parent	869ef00602328f97ea8c88358310cbd34d4391ab

std: rework zstd for new I/O API

This passes tests but it doesn't provide as big a window size as is
required to decompress larger streams.

The next commit in this branch will work towards that, without
introducing an additional buffer.

17 files changed, 2078 insertions(+), 3046 deletions(-)

lib
- std
  - Io
    DeprecatedReader.zig
    Reader.zig
  - compress.zig
  - compress
    xz.zig
    xz
    block.zig
    zstandard.zig
    zstandard
    decode
    block.zig
    fse.zig
    huffman.zig
    decompress.zig
    readers.zig
    types.zig
    zstd.zig
    zstd
    Decompress.zig
  - http
    Server.zig
src
- Package
  - Fetch.zig
  - Fetch
    git.zig

lib/std/Io/DeprecatedReader.zig+2-2

...	...	@@ -373,11 +373,11 @@ pub fn discard(self: Self) anyerror!u64 {
373	373	}
374	374
375	375	/// Helper for bridging to the new `Reader` API while upgrading.
376		pub fn adaptToNewApi(self: *const Self) Adapter {
	376	pub fn adaptToNewApi(self: *const Self, buffer: []u8) Adapter {
377	377	return .{
378	378	.derp_reader = self.*,
379	379	.new_interface = .{
380		.buffer = &.{},
	380	.buffer = buffer,
381	381	.vtable = &.{ .stream = Adapter.stream },
382	382	.seek = 0,
383	383	.end = 0,

lib/std/Io/Reader.zig+2-2

...	...	@@ -240,7 +240,7 @@ pub fn allocRemaining(r: *Reader, gpa: Allocator, limit: Limit) LimitedAllocErro
240	240	/// such case, the next byte that would be read will be the first one to exceed
241	241	/// `limit`, and all preceeding bytes have been appended to `list`.
242	242	///
243		/// Asserts `buffer` has nonzero capacity.
	243	/// If `limit` is not `Limit.unlimited`, asserts `buffer` has nonzero capacity.
244	244	///
245	245	/// See also:
246	246	/// * `allocRemaining`
...	...	@@ -251,7 +251,7 @@ pub fn appendRemaining(
251	251	list: *std.ArrayListAlignedUnmanaged(u8, alignment),
252	252	limit: Limit,
253	253	) LimitedAllocError!void {
254		assert(r.buffer.len != 0); // Needed to detect limit exceeded without losing data.
	254	if (limit != .unlimited) assert(r.buffer.len != 0); // Needed to detect limit exceeded without losing data.
255	255	const buffer_contents = r.buffer[r.seek..r.end];
256	256	const copy_len = limit.minInt(buffer_contents.len);
257	257	try list.appendSlice(gpa, r.buffer[0..copy_len]);

lib/std/compress.zig+2-58

...	...	@@ -1,75 +1,19 @@
1	1	//! Compression algorithms.
2	2
3		const std = @import("std.zig");
4
5	3	pub const flate = @import("compress/flate.zig");
6	4	pub const gzip = @import("compress/gzip.zig");
7	5	pub const zlib = @import("compress/zlib.zig");
8	6	pub const lzma = @import("compress/lzma.zig");
9	7	pub const lzma2 = @import("compress/lzma2.zig");
10	8	pub const xz = @import("compress/xz.zig");
11		pub const zstd = @import("compress/zstandard.zig");
12
13		pub fn HashedReader(ReaderType: type, HasherType: type) type {
14		return struct {
15		child_reader: ReaderType,
16		hasher: HasherType,
17
18		pub const Error = ReaderType.Error;
19		pub const Reader = std.io.GenericReader(*@This(), Error, read);
20
21		pub fn read(self: *@This(), buf: []u8) Error!usize {
22		const amt = try self.child_reader.read(buf);
23		self.hasher.update(buf[0..amt]);
24		return amt;
25		}
26
27		pub fn reader(self: *@This()) Reader {
28		return .{ .context = self };
29		}
30		};
31		}
32
33		pub fn hashedReader(
34		reader: anytype,
35		hasher: anytype,
36		) HashedReader(@TypeOf(reader), @TypeOf(hasher)) {
37		return .{ .child_reader = reader, .hasher = hasher };
38		}
39
40		pub fn HashedWriter(WriterType: type, HasherType: type) type {
41		return struct {
42		child_writer: WriterType,
43		hasher: HasherType,
44
45		pub const Error = WriterType.Error;
46		pub const Writer = std.io.GenericWriter(*@This(), Error, write);
47
48		pub fn write(self: *@This(), buf: []const u8) Error!usize {
49		const amt = try self.child_writer.write(buf);
50		self.hasher.update(buf[0..amt]);
51		return amt;
52		}
53
54		pub fn writer(self: *@This()) Writer {
55		return .{ .context = self };
56		}
57		};
58		}
59
60		pub fn hashedWriter(
61		writer: anytype,
62		hasher: anytype,
63		) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) {
64		return .{ .child_writer = writer, .hasher = hasher };
65		}
	9	pub const zstd = @import("compress/zstd.zig");
66	10
67	11	test {
	12	_ = flate;
68	13	_ = lzma;
69	14	_ = lzma2;
70	15	_ = xz;
71	16	_ = zstd;
72		_ = flate;
73	17	_ = gzip;
74	18	_ = zlib;
75	19	}

lib/std/compress/xz.zig+30-3

...	...	@@ -47,7 +47,7 @@ pub fn Decompress(comptime ReaderType: type) type {
47	47
48	48	var check: Check = undefined;
49	49	const hash_a = blk: {
50		var hasher = std.compress.hashedReader(source, Crc32.init());
	50	var hasher = hashedReader(source, Crc32.init());
51	51	try readStreamFlags(hasher.reader(), &check);
52	52	break :blk hasher.hasher.final();
53	53	};
...	...	@@ -80,7 +80,7 @@ pub fn Decompress(comptime ReaderType: type) type {
80	80	return r;
81	81
82	82	const index_size = blk: {
83		var hasher = std.compress.hashedReader(self.in_reader, Crc32.init());
	83	var hasher = hashedReader(self.in_reader, Crc32.init());
84	84	hasher.hasher.update(&[1]u8{0x00});
85	85
86	86	var counter = std.io.countingReader(hasher.reader());
...	...	@@ -115,7 +115,7 @@ pub fn Decompress(comptime ReaderType: type) type {
115	115	const hash_a = try self.in_reader.readInt(u32, .little);
116	116
117	117	const hash_b = blk: {
118		var hasher = std.compress.hashedReader(self.in_reader, Crc32.init());
	118	var hasher = hashedReader(self.in_reader, Crc32.init());
119	119	const hashed_reader = hasher.reader();
120	120
121	121	const backward_size = (@as(u64, try hashed_reader.readInt(u32, .little)) + 1) * 4;
...	...	@@ -140,6 +140,33 @@ pub fn Decompress(comptime ReaderType: type) type {
140	140	};
141	141	}
142	142
	143	pub fn HashedReader(ReaderType: type, HasherType: type) type {
	144	return struct {
	145	child_reader: ReaderType,
	146	hasher: HasherType,
	147
	148	pub const Error = ReaderType.Error;
	149	pub const Reader = std.io.GenericReader(*@This(), Error, read);
	150
	151	pub fn read(self: *@This(), buf: []u8) Error!usize {
	152	const amt = try self.child_reader.read(buf);
	153	self.hasher.update(buf[0..amt]);
	154	return amt;
	155	}
	156
	157	pub fn reader(self: *@This()) Reader {
	158	return .{ .context = self };
	159	}
	160	};
	161	}
	162
	163	pub fn hashedReader(
	164	reader: anytype,
	165	hasher: anytype,
	166	) HashedReader(@TypeOf(reader), @TypeOf(hasher)) {
	167	return .{ .child_reader = reader, .hasher = hasher };
	168	}
	169
143	170	test {
144	171	_ = @import("xz/test.zig");
145	172	}

lib/std/compress/xz/block.zig+1-1

...	...	@@ -91,7 +91,7 @@ pub fn Decoder(comptime ReaderType: type) type {
91	91
92	92	// Block Header
93	93	{
94		var header_hasher = std.compress.hashedReader(block_reader, Crc32.init());
	94	var header_hasher = xz.hashedReader(block_reader, Crc32.init());
95	95	const header_reader = header_hasher.reader();
96	96
97	97	const header_size = @as(u64, try header_reader.readByte()) * 4;

lib/std/compress/zstandard.zig deleted-310

...	...	@@ -1,310 +0,0 @@
1		const std = @import("std");
2		const RingBuffer = std.RingBuffer;
3
4		const types = @import("zstandard/types.zig");
5		pub const frame = types.frame;
6		pub const compressed_block = types.compressed_block;
7
8		pub const decompress = @import("zstandard/decompress.zig");
9
10		pub const DecompressorOptions = struct {
11		verify_checksum: bool = true,
12		window_buffer: []u8,
13
14		/// Recommended amount by the standard. Lower than this may result
15		/// in inability to decompress common streams.
16		pub const default_window_buffer_len = 8 * 1024 * 1024;
17		};
18
19		pub fn Decompressor(comptime ReaderType: type) type {
20		return struct {
21		const Self = @This();
22
23		const table_size_max = types.compressed_block.table_size_max;
24
25		source: std.io.CountingReader(ReaderType),
26		state: enum { NewFrame, InFrame, LastBlock },
27		decode_state: decompress.block.DecodeState,
28		frame_context: decompress.FrameContext,
29		buffer: WindowBuffer,
30		literal_fse_buffer: [table_size_max.literal]types.compressed_block.Table.Fse,
31		match_fse_buffer: [table_size_max.match]types.compressed_block.Table.Fse,
32		offset_fse_buffer: [table_size_max.offset]types.compressed_block.Table.Fse,
33		literals_buffer: [types.block_size_max]u8,
34		sequence_buffer: [types.block_size_max]u8,
35		verify_checksum: bool,
36		checksum: ?u32,
37		current_frame_decompressed_size: usize,
38
39		const WindowBuffer = struct {
40		data: []u8 = undefined,
41		read_index: usize = 0,
42		write_index: usize = 0,
43		};
44
45		pub const Error = ReaderType.Error \|\| error{
46		ChecksumFailure,
47		DictionaryIdFlagUnsupported,
48		MalformedBlock,
49		MalformedFrame,
50		OutOfMemory,
51		};
52
53		pub const Reader = std.io.GenericReader(*Self, Error, read);
54
55		pub fn init(source: ReaderType, options: DecompressorOptions) Self {
56		return .{
57		.source = std.io.countingReader(source),
58		.state = .NewFrame,
59		.decode_state = undefined,
60		.frame_context = undefined,
61		.buffer = .{ .data = options.window_buffer },
62		.literal_fse_buffer = undefined,
63		.match_fse_buffer = undefined,
64		.offset_fse_buffer = undefined,
65		.literals_buffer = undefined,
66		.sequence_buffer = undefined,
67		.verify_checksum = options.verify_checksum,
68		.checksum = undefined,
69		.current_frame_decompressed_size = undefined,
70		};
71		}
72
73		fn frameInit(self: *Self) !void {
74		const source_reader = self.source.reader();
75		switch (try decompress.decodeFrameHeader(source_reader)) {
76		.skippable => \|header\| {
77		try source_reader.skipBytes(header.frame_size, .{});
78		self.state = .NewFrame;
79		},
80		.zstandard => \|header\| {
81		const frame_context = try decompress.FrameContext.init(
82		header,
83		self.buffer.data.len,
84		self.verify_checksum,
85		);
86
87		const decode_state = decompress.block.DecodeState.init(
88		&self.literal_fse_buffer,
89		&self.match_fse_buffer,
90		&self.offset_fse_buffer,
91		);
92
93		self.decode_state = decode_state;
94		self.frame_context = frame_context;
95
96		self.checksum = null;
97		self.current_frame_decompressed_size = 0;
98
99		self.state = .InFrame;
100		},
101		}
102		}
103
104		pub fn reader(self: *Self) Reader {
105		return .{ .context = self };
106		}
107
108		pub fn read(self: *Self, buffer: []u8) Error!usize {
109		if (buffer.len == 0) return 0;
110
111		var size: usize = 0;
112		while (size == 0) {
113		while (self.state == .NewFrame) {
114		const initial_count = self.source.bytes_read;
115		self.frameInit() catch \|err\| switch (err) {
116		error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
117		error.EndOfStream => return if (self.source.bytes_read == initial_count)
118		0
119		else
120		error.MalformedFrame,
121		else => return error.MalformedFrame,
122		};
123		}
124		size = try self.readInner(buffer);
125		}
126		return size;
127		}
128
129		fn readInner(self: *Self, buffer: []u8) Error!usize {
130		std.debug.assert(self.state != .NewFrame);
131
132		var ring_buffer = RingBuffer{
133		.data = self.buffer.data,
134		.read_index = self.buffer.read_index,
135		.write_index = self.buffer.write_index,
136		};
137		defer {
138		self.buffer.read_index = ring_buffer.read_index;
139		self.buffer.write_index = ring_buffer.write_index;
140		}
141
142		const source_reader = self.source.reader();
143		while (ring_buffer.isEmpty() and self.state != .LastBlock) {
144		const header_bytes = source_reader.readBytesNoEof(3) catch
145		return error.MalformedFrame;
146		const block_header = decompress.block.decodeBlockHeader(&header_bytes);
147
148		decompress.block.decodeBlockReader(
149		&ring_buffer,
150		source_reader,
151		block_header,
152		&self.decode_state,
153		self.frame_context.block_size_max,
154		&self.literals_buffer,
155		&self.sequence_buffer,
156		) catch
157		return error.MalformedBlock;
158
159		if (self.frame_context.content_size) \|size\| {
160		if (self.current_frame_decompressed_size > size) return error.MalformedFrame;
161		}
162
163		const size = ring_buffer.len();
164		self.current_frame_decompressed_size += size;
165
166		if (self.frame_context.hasher_opt) \|*hasher\| {
167		if (size > 0) {
168		const written_slice = ring_buffer.sliceLast(size);
169		hasher.update(written_slice.first);
170		hasher.update(written_slice.second);
171		}
172		}
173		if (block_header.last_block) {
174		self.state = .LastBlock;
175		if (self.frame_context.has_checksum) {
176		const checksum = source_reader.readInt(u32, .little) catch
177		return error.MalformedFrame;
178		if (self.verify_checksum) {
179		if (self.frame_context.hasher_opt) \|*hasher\| {
180		if (checksum != decompress.computeChecksum(hasher))
181		return error.ChecksumFailure;
182		}
183		}
184		}
185		if (self.frame_context.content_size) \|content_size\| {
186		if (content_size != self.current_frame_decompressed_size) {
187		return error.MalformedFrame;
188		}
189		}
190		}
191		}
192
193		const size = @min(ring_buffer.len(), buffer.len);
194		if (size > 0) {
195		ring_buffer.readFirstAssumeLength(buffer, size);
196		}
197		if (self.state == .LastBlock and ring_buffer.len() == 0) {
198		self.state = .NewFrame;
199		}
200		return size;
201		}
202		};
203		}
204
205		pub fn decompressor(reader: anytype, options: DecompressorOptions) Decompressor(@TypeOf(reader)) {
206		return Decompressor(@TypeOf(reader)).init(reader, options);
207		}
208
209		fn testDecompress(data: []const u8) ![]u8 {
210		const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
211		defer std.testing.allocator.free(window_buffer);
212
213		var in_stream = std.io.fixedBufferStream(data);
214		var zstd_stream = decompressor(in_stream.reader(), .{ .window_buffer = window_buffer });
215		const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
216		return result;
217		}
218
219		fn testReader(data: []const u8, comptime expected: []const u8) !void {
220		const buf = try testDecompress(data);
221		defer std.testing.allocator.free(buf);
222		try std.testing.expectEqualSlices(u8, expected, buf);
223		}
224
225		test "decompression" {
226		const uncompressed = @embedFile("testdata/rfc8478.txt");
227		const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
228		const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
229
230		const buffer = try std.testing.allocator.alloc(u8, uncompressed.len);
231		defer std.testing.allocator.free(buffer);
232
233		const res3 = try decompress.decode(buffer, compressed3, true);
234		try std.testing.expectEqual(uncompressed.len, res3);
235		try std.testing.expectEqualSlices(u8, uncompressed, buffer);
236
237		@memset(buffer, undefined);
238		const res19 = try decompress.decode(buffer, compressed19, true);
239		try std.testing.expectEqual(uncompressed.len, res19);
240		try std.testing.expectEqualSlices(u8, uncompressed, buffer);
241
242		try testReader(compressed3, uncompressed);
243		try testReader(compressed19, uncompressed);
244		}
245
246		fn expectEqualDecoded(expected: []const u8, input: []const u8) !void {
247		{
248		const result = try decompress.decodeAlloc(std.testing.allocator, input, false, 1 << 23);
249		defer std.testing.allocator.free(result);
250		try std.testing.expectEqualStrings(expected, result);
251		}
252
253		{
254		var buffer = try std.testing.allocator.alloc(u8, 2 * expected.len);
255		defer std.testing.allocator.free(buffer);
256
257		const size = try decompress.decode(buffer, input, false);
258		try std.testing.expectEqualStrings(expected, buffer[0..size]);
259		}
260		}
261
262		fn expectEqualDecodedStreaming(expected: []const u8, input: []const u8) !void {
263		const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
264		defer std.testing.allocator.free(window_buffer);
265
266		var in_stream = std.io.fixedBufferStream(input);
267		var stream = decompressor(in_stream.reader(), .{ .window_buffer = window_buffer });
268
269		const result = try stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
270		defer std.testing.allocator.free(result);
271
272		try std.testing.expectEqualStrings(expected, result);
273		}
274
275		test "zero sized block" {
276		const input_raw =
277		"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
278		"\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
279		"\x01\x00\x00"; // block header with: last_block set, block_type raw, block_size zero
280
281		const input_rle =
282		"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
283		"\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
284		"\x03\x00\x00" ++ // block header with: last_block set, block_type rle, block_size zero
285		"\xaa"; // block_content
286
287		try expectEqualDecoded("", input_raw);
288		try expectEqualDecoded("", input_rle);
289		try expectEqualDecodedStreaming("", input_raw);
290		try expectEqualDecodedStreaming("", input_rle);
291		}
292
293		test "declared raw literals size too large" {
294		const input_raw =
295		"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
296		"\x00\x00" ++ // frame header: everything unset, window descriptor zero
297		"\x95\x00\x00" ++ // block header with: last_block set, block_type compressed, block_size 18
298		"\xbc\xf3\xae" ++ // literals section header with: type raw, size_format 3, regenerated_size 716603
299		"\xa5\x9f\xe3"; // some bytes of literal content - the content is shorter than regenerated_size
300
301		// Note that the regenerated_size in the above input is larger than block maximum size, so the
302		// block can't be valid as it is a raw literals block.
303
304		var fbs = std.io.fixedBufferStream(input_raw);
305		var window: [1024]u8 = undefined;
306		var stream = decompressor(fbs.reader(), .{ .window_buffer = &window });
307
308		var buf: [1024]u8 = undefined;
309		try std.testing.expectError(error.MalformedBlock, stream.read(&buf));
310		}

lib/std/compress/zstandard/decode/block.zig deleted-1149

...	...	@@ -1,1149 +0,0 @@
1		const std = @import("std");
2		const assert = std.debug.assert;
3		const RingBuffer = std.RingBuffer;
4
5		const types = @import("../types.zig");
6		const frame = types.frame;
7		const Table = types.compressed_block.Table;
8		const LiteralsSection = types.compressed_block.LiteralsSection;
9		const SequencesSection = types.compressed_block.SequencesSection;
10
11		const huffman = @import("huffman.zig");
12		const readers = @import("../readers.zig");
13
14		const decodeFseTable = @import("fse.zig").decodeFseTable;
15
16		pub const Error = error{
17		BlockSizeOverMaximum,
18		MalformedBlockSize,
19		ReservedBlock,
20		MalformedRleBlock,
21		MalformedCompressedBlock,
22		};
23
24		pub const DecodeState = struct {
25		repeat_offsets: [3]u32,
26
27		offset: StateData(8),
28		match: StateData(9),
29		literal: StateData(9),
30
31		offset_fse_buffer: []Table.Fse,
32		match_fse_buffer: []Table.Fse,
33		literal_fse_buffer: []Table.Fse,
34
35		fse_tables_undefined: bool,
36
37		literal_stream_reader: readers.ReverseBitReader,
38		literal_stream_index: usize,
39		literal_streams: LiteralsSection.Streams,
40		literal_header: LiteralsSection.Header,
41		huffman_tree: ?LiteralsSection.HuffmanTree,
42
43		literal_written_count: usize,
44		written_count: usize = 0,
45
46		fn StateData(comptime max_accuracy_log: comptime_int) type {
47		return struct {
48		state: State,
49		table: Table,
50		accuracy_log: u8,
51
52		const State = std.meta.Int(.unsigned, max_accuracy_log);
53		};
54		}
55
56		pub fn init(
57		literal_fse_buffer: []Table.Fse,
58		match_fse_buffer: []Table.Fse,
59		offset_fse_buffer: []Table.Fse,
60		) DecodeState {
61		return DecodeState{
62		.repeat_offsets = .{
63		types.compressed_block.start_repeated_offset_1,
64		types.compressed_block.start_repeated_offset_2,
65		types.compressed_block.start_repeated_offset_3,
66		},
67
68		.offset = undefined,
69		.match = undefined,
70		.literal = undefined,
71
72		.literal_fse_buffer = literal_fse_buffer,
73		.match_fse_buffer = match_fse_buffer,
74		.offset_fse_buffer = offset_fse_buffer,
75
76		.fse_tables_undefined = true,
77
78		.literal_written_count = 0,
79		.literal_header = undefined,
80		.literal_streams = undefined,
81		.literal_stream_reader = undefined,
82		.literal_stream_index = undefined,
83		.huffman_tree = null,
84
85		.written_count = 0,
86		};
87		}
88
89		/// Prepare the decoder to decode a compressed block. Loads the literals
90		/// stream and Huffman tree from `literals` and reads the FSE tables from
91		/// `source`.
92		///
93		/// Errors returned:
94		/// - `error.BitStreamHasNoStartBit` if the (reversed) literal bitstream's
95		/// first byte does not have any bits set
96		/// - `error.TreelessLiteralsFirst` `literals` is a treeless literals
97		/// section and the decode state does not have a Huffman tree from a
98		/// previous block
99		/// - `error.RepeatModeFirst` on the first call if one of the sequence FSE
100		/// tables is set to repeat mode
101		/// - `error.MalformedAccuracyLog` if an FSE table has an invalid accuracy
102		/// - `error.MalformedFseTable` if there are errors decoding an FSE table
103		/// - `error.EndOfStream` if `source` ends before all FSE tables are read
104		pub fn prepare(
105		self: *DecodeState,
106		source: anytype,
107		literals: LiteralsSection,
108		sequences_header: SequencesSection.Header,
109		) !void {
110		self.literal_written_count = 0;
111		self.literal_header = literals.header;
112		self.literal_streams = literals.streams;
113
114		if (literals.huffman_tree) \|tree\| {
115		self.huffman_tree = tree;
116		} else if (literals.header.block_type == .treeless and self.huffman_tree == null) {
117		return error.TreelessLiteralsFirst;
118		}
119
120		switch (literals.header.block_type) {
121		.raw, .rle => {},
122		.compressed, .treeless => {
123		self.literal_stream_index = 0;
124		switch (literals.streams) {
125		.one => \|slice\| try self.initLiteralStream(slice),
126		.four => \|streams\| try self.initLiteralStream(streams[0]),
127		}
128		},
129		}
130
131		if (sequences_header.sequence_count > 0) {
132		try self.updateFseTable(source, .literal, sequences_header.literal_lengths);
133		try self.updateFseTable(source, .offset, sequences_header.offsets);
134		try self.updateFseTable(source, .match, sequences_header.match_lengths);
135		self.fse_tables_undefined = false;
136		}
137		}
138
139		/// Read initial FSE states for sequence decoding.
140		///
141		/// Errors returned:
142		/// - `error.EndOfStream` if `bit_reader` does not contain enough bits.
143		pub fn readInitialFseState(self: DecodeState, bit_reader: readers.ReverseBitReader) error{EndOfStream}!void {
144		self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
145		self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
146		self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log);
147		}
148
149		fn updateRepeatOffset(self: *DecodeState, offset: u32) void {
150		self.repeat_offsets[2] = self.repeat_offsets[1];
151		self.repeat_offsets[1] = self.repeat_offsets[0];
152		self.repeat_offsets[0] = offset;
153		}
154
155		fn useRepeatOffset(self: *DecodeState, index: usize) u32 {
156		if (index == 1)
157		std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[1])
158		else if (index == 2) {
159		std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[2]);
160		std.mem.swap(u32, &self.repeat_offsets[1], &self.repeat_offsets[2]);
161		}
162		return self.repeat_offsets[0];
163		}
164
165		const DataType = enum { offset, match, literal };
166
167		fn updateState(
168		self: *DecodeState,
169		comptime choice: DataType,
170		bit_reader: *readers.ReverseBitReader,
171		) error{ MalformedFseBits, EndOfStream }!void {
172		switch (@field(self, @tagName(choice)).table) {
173		.rle => {},
174		.fse => \|table\| {
175		const data = table[@field(self, @tagName(choice)).state];
176		const T = @TypeOf(@field(self, @tagName(choice))).State;
177		const bits_summand = try bit_reader.readBitsNoEof(T, data.bits);
178		const next_state = std.math.cast(
179		@TypeOf(@field(self, @tagName(choice))).State,
180		data.baseline + bits_summand,
181		) orelse return error.MalformedFseBits;
182		@field(self, @tagName(choice)).state = next_state;
183		},
184		}
185		}
186
187		const FseTableError = error{
188		MalformedFseTable,
189		MalformedAccuracyLog,
190		RepeatModeFirst,
191		EndOfStream,
192		};
193
194		fn updateFseTable(
195		self: *DecodeState,
196		source: anytype,
197		comptime choice: DataType,
198		mode: SequencesSection.Header.Mode,
199		) !void {
200		const field_name = @tagName(choice);
201		switch (mode) {
202		.predefined => {
203		@field(self, field_name).accuracy_log =
204		@field(types.compressed_block.default_accuracy_log, field_name);
205
206		@field(self, field_name).table =
207		@field(types.compressed_block, "predefined_" ++ field_name ++ "_fse_table");
208		},
209		.rle => {
210		@field(self, field_name).accuracy_log = 0;
211		@field(self, field_name).table = .{ .rle = try source.readByte() };
212		},
213		.fse => {
214		var bit_reader = readers.bitReader(source);
215
216		const table_size = try decodeFseTable(
217		&bit_reader,
218		@field(types.compressed_block.table_symbol_count_max, field_name),
219		@field(types.compressed_block.table_accuracy_log_max, field_name),
220		@field(self, field_name ++ "_fse_buffer"),
221		);
222		@field(self, field_name).table = .{
223		.fse = @field(self, field_name ++ "_fse_buffer")[0..table_size],
224		};
225		@field(self, field_name).accuracy_log = std.math.log2_int_ceil(usize, table_size);
226		},
227		.repeat => if (self.fse_tables_undefined) return error.RepeatModeFirst,
228		}
229		}
230
231		const Sequence = struct {
232		literal_length: u32,
233		match_length: u32,
234		offset: u32,
235		};
236
237		fn nextSequence(
238		self: *DecodeState,
239		bit_reader: *readers.ReverseBitReader,
240		) error{ InvalidBitStream, EndOfStream }!Sequence {
241		const raw_code = self.getCode(.offset);
242		const offset_code = std.math.cast(u5, raw_code) orelse {
243		return error.InvalidBitStream;
244		};
245		const offset_value = (@as(u32, 1) << offset_code) + try bit_reader.readBitsNoEof(u32, offset_code);
246
247		const match_code = self.getCode(.match);
248		if (match_code >= types.compressed_block.match_length_code_table.len)
249		return error.InvalidBitStream;
250		const match = types.compressed_block.match_length_code_table[match_code];
251		const match_length = match[0] + try bit_reader.readBitsNoEof(u32, match[1]);
252
253		const literal_code = self.getCode(.literal);
254		if (literal_code >= types.compressed_block.literals_length_code_table.len)
255		return error.InvalidBitStream;
256		const literal = types.compressed_block.literals_length_code_table[literal_code];
257		const literal_length = literal[0] + try bit_reader.readBitsNoEof(u32, literal[1]);
258
259		const offset = if (offset_value > 3) offset: {
260		const offset = offset_value - 3;
261		self.updateRepeatOffset(offset);
262		break :offset offset;
263		} else offset: {
264		if (literal_length == 0) {
265		if (offset_value == 3) {
266		const offset = self.repeat_offsets[0] - 1;
267		self.updateRepeatOffset(offset);
268		break :offset offset;
269		}
270		break :offset self.useRepeatOffset(offset_value);
271		}
272		break :offset self.useRepeatOffset(offset_value - 1);
273		};
274
275		if (offset == 0) return error.InvalidBitStream;
276
277		return .{
278		.literal_length = literal_length,
279		.match_length = match_length,
280		.offset = offset,
281		};
282		}
283
284		fn executeSequenceSlice(
285		self: *DecodeState,
286		dest: []u8,
287		write_pos: usize,
288		sequence: Sequence,
289		) (error{MalformedSequence} \|\| DecodeLiteralsError)!void {
290		if (sequence.offset > write_pos + sequence.literal_length) return error.MalformedSequence;
291
292		try self.decodeLiteralsSlice(dest[write_pos..], sequence.literal_length);
293		const copy_start = write_pos + sequence.literal_length - sequence.offset;
294		for (
295		dest[write_pos + sequence.literal_length ..][0..sequence.match_length],
296		dest[copy_start..][0..sequence.match_length],
297		) \|d, s\| d. = s;
298		self.written_count += sequence.match_length;
299		}
300
301		fn executeSequenceRingBuffer(
302		self: *DecodeState,
303		dest: *RingBuffer,
304		sequence: Sequence,
305		) (error{MalformedSequence} \|\| DecodeLiteralsError)!void {
306		if (sequence.offset > @min(dest.data.len, self.written_count + sequence.literal_length))
307		return error.MalformedSequence;
308
309		try self.decodeLiteralsRingBuffer(dest, sequence.literal_length);
310		const copy_start = dest.write_index + dest.data.len - sequence.offset;
311		const copy_slice = dest.sliceAt(copy_start, sequence.match_length);
312		dest.writeSliceForwardsAssumeCapacity(copy_slice.first);
313		dest.writeSliceForwardsAssumeCapacity(copy_slice.second);
314		self.written_count += sequence.match_length;
315		}
316
317		const DecodeSequenceError = error{
318		InvalidBitStream,
319		EndOfStream,
320		MalformedSequence,
321		MalformedFseBits,
322		} \|\| DecodeLiteralsError;
323
324		/// Decode one sequence from `bit_reader` into `dest`, written starting at
325		/// `write_pos` and update FSE states if `last_sequence` is `false`.
326		/// `prepare()` must be called for the block before attempting to decode
327		/// sequences.
328		///
329		/// Errors returned:
330		/// - `error.MalformedSequence` if the decompressed sequence would be
331		/// longer than `sequence_size_limit` or the sequence's offset is too
332		/// large
333		/// - `error.UnexpectedEndOfLiteralStream` if the decoder state's literal
334		/// streams do not contain enough literals for the sequence (this may
335		/// mean the literal stream or the sequence is malformed).
336		/// - `error.InvalidBitStream` if the FSE sequence bitstream is malformed
337		/// - `error.EndOfStream` if `bit_reader` does not contain enough bits
338		/// - `error.DestTooSmall` if `dest` is not large enough to holde the
339		/// decompressed sequence
340		pub fn decodeSequenceSlice(
341		self: *DecodeState,
342		dest: []u8,
343		write_pos: usize,
344		bit_reader: *readers.ReverseBitReader,
345		sequence_size_limit: usize,
346		last_sequence: bool,
347		) (error{DestTooSmall} \|\| DecodeSequenceError)!usize {
348		const sequence = try self.nextSequence(bit_reader);
349		const sequence_length = @as(usize, sequence.literal_length) + sequence.match_length;
350		if (sequence_length > sequence_size_limit) return error.MalformedSequence;
351		if (sequence_length > dest[write_pos..].len) return error.DestTooSmall;
352
353		try self.executeSequenceSlice(dest, write_pos, sequence);
354		if (!last_sequence) {
355		try self.updateState(.literal, bit_reader);
356		try self.updateState(.match, bit_reader);
357		try self.updateState(.offset, bit_reader);
358		}
359		return sequence_length;
360		}
361
362		/// Decode one sequence from `bit_reader` into `dest`; see
363		/// `decodeSequenceSlice`.
364		pub fn decodeSequenceRingBuffer(
365		self: *DecodeState,
366		dest: *RingBuffer,
367		bit_reader: anytype,
368		sequence_size_limit: usize,
369		last_sequence: bool,
370		) DecodeSequenceError!usize {
371		const sequence = try self.nextSequence(bit_reader);
372		const sequence_length = @as(usize, sequence.literal_length) + sequence.match_length;
373		if (sequence_length > sequence_size_limit) return error.MalformedSequence;
374
375		try self.executeSequenceRingBuffer(dest, sequence);
376		if (!last_sequence) {
377		try self.updateState(.literal, bit_reader);
378		try self.updateState(.match, bit_reader);
379		try self.updateState(.offset, bit_reader);
380		}
381		return sequence_length;
382		}
383
384		fn nextLiteralMultiStream(
385		self: *DecodeState,
386		) error{BitStreamHasNoStartBit}!void {
387		self.literal_stream_index += 1;
388		try self.initLiteralStream(self.literal_streams.four[self.literal_stream_index]);
389		}
390
391		fn initLiteralStream(self: *DecodeState, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
392		try self.literal_stream_reader.init(bytes);
393		}
394
395		fn isLiteralStreamEmpty(self: *DecodeState) bool {
396		switch (self.literal_streams) {
397		.one => return self.literal_stream_reader.isEmpty(),
398		.four => return self.literal_stream_index == 3 and self.literal_stream_reader.isEmpty(),
399		}
400		}
401
402		const LiteralBitsError = error{
403		BitStreamHasNoStartBit,
404		UnexpectedEndOfLiteralStream,
405		};
406		fn readLiteralsBits(
407		self: *DecodeState,
408		bit_count_to_read: u16,
409		) LiteralBitsError!u16 {
410		return self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch bits: {
411		if (self.literal_streams == .four and self.literal_stream_index < 3) {
412		try self.nextLiteralMultiStream();
413		break :bits self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch
414		return error.UnexpectedEndOfLiteralStream;
415		} else {
416		return error.UnexpectedEndOfLiteralStream;
417		}
418		};
419		}
420
421		const DecodeLiteralsError = error{
422		MalformedLiteralsLength,
423		NotFound,
424		} \|\| LiteralBitsError;
425
426		/// Decode `len` bytes of literals into `dest`.
427		///
428		/// Errors returned:
429		/// - `error.MalformedLiteralsLength` if the number of literal bytes
430		/// decoded by `self` plus `len` is greater than the regenerated size of
431		/// `literals`
432		/// - `error.UnexpectedEndOfLiteralStream` and `error.NotFound` if there
433		/// are problems decoding Huffman compressed literals
434		pub fn decodeLiteralsSlice(
435		self: *DecodeState,
436		dest: []u8,
437		len: usize,
438		) DecodeLiteralsError!void {
439		if (self.literal_written_count + len > self.literal_header.regenerated_size)
440		return error.MalformedLiteralsLength;
441
442		switch (self.literal_header.block_type) {
443		.raw => {
444		const literal_data = self.literal_streams.one[self.literal_written_count..][0..len];
445		@memcpy(dest[0..len], literal_data);
446		self.literal_written_count += len;
447		self.written_count += len;
448		},
449		.rle => {
450		for (0..len) \|i\| {
451		dest[i] = self.literal_streams.one[0];
452		}
453		self.literal_written_count += len;
454		self.written_count += len;
455		},
456		.compressed, .treeless => {
457		// const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
458		const huffman_tree = self.huffman_tree orelse unreachable;
459		const max_bit_count = huffman_tree.max_bit_count;
460		const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
461		huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
462		max_bit_count,
463		);
464		var bits_read: u4 = 0;
465		var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
466		var bit_count_to_read: u4 = starting_bit_count;
467		for (0..len) \|i\| {
468		var prefix: u16 = 0;
469		while (true) {
470		const new_bits = self.readLiteralsBits(bit_count_to_read) catch \|err\| {
471		return err;
472		};
473		prefix <<= bit_count_to_read;
474		prefix \|= new_bits;
475		bits_read += bit_count_to_read;
476		const result = huffman_tree.query(huffman_tree_index, prefix) catch \|err\| {
477		return err;
478		};
479
480		switch (result) {
481		.symbol => \|sym\| {
482		dest[i] = sym;
483		bit_count_to_read = starting_bit_count;
484		bits_read = 0;
485		huffman_tree_index = huffman_tree.symbol_count_minus_one;
486		break;
487		},
488		.index => \|index\| {
489		huffman_tree_index = index;
490		const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
491		huffman_tree.nodes[index].weight,
492		max_bit_count,
493		);
494		bit_count_to_read = bit_count - bits_read;
495		},
496		}
497		}
498		}
499		self.literal_written_count += len;
500		self.written_count += len;
501		},
502		}
503		}
504
505		/// Decode literals into `dest`; see `decodeLiteralsSlice()`.
506		pub fn decodeLiteralsRingBuffer(
507		self: *DecodeState,
508		dest: *RingBuffer,
509		len: usize,
510		) DecodeLiteralsError!void {
511		if (self.literal_written_count + len > self.literal_header.regenerated_size)
512		return error.MalformedLiteralsLength;
513
514		switch (self.literal_header.block_type) {
515		.raw => {
516		const literals_end = self.literal_written_count + len;
517		const literal_data = self.literal_streams.one[self.literal_written_count..literals_end];
518		dest.writeSliceAssumeCapacity(literal_data);
519		self.literal_written_count += len;
520		self.written_count += len;
521		},
522		.rle => {
523		for (0..len) \|_\| {
524		dest.writeAssumeCapacity(self.literal_streams.one[0]);
525		}
526		self.literal_written_count += len;
527		self.written_count += len;
528		},
529		.compressed, .treeless => {
530		// const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
531		const huffman_tree = self.huffman_tree orelse unreachable;
532		const max_bit_count = huffman_tree.max_bit_count;
533		const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
534		huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
535		max_bit_count,
536		);
537		var bits_read: u4 = 0;
538		var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
539		var bit_count_to_read: u4 = starting_bit_count;
540		for (0..len) \|_\| {
541		var prefix: u16 = 0;
542		while (true) {
543		const new_bits = try self.readLiteralsBits(bit_count_to_read);
544		prefix <<= bit_count_to_read;
545		prefix \|= new_bits;
546		bits_read += bit_count_to_read;
547		const result = try huffman_tree.query(huffman_tree_index, prefix);
548
549		switch (result) {
550		.symbol => \|sym\| {
551		dest.writeAssumeCapacity(sym);
552		bit_count_to_read = starting_bit_count;
553		bits_read = 0;
554		huffman_tree_index = huffman_tree.symbol_count_minus_one;
555		break;
556		},
557		.index => \|index\| {
558		huffman_tree_index = index;
559		const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
560		huffman_tree.nodes[index].weight,
561		max_bit_count,
562		);
563		bit_count_to_read = bit_count - bits_read;
564		},
565		}
566		}
567		}
568		self.literal_written_count += len;
569		self.written_count += len;
570		},
571		}
572		}
573
574		fn getCode(self: *DecodeState, comptime choice: DataType) u32 {
575		return switch (@field(self, @tagName(choice)).table) {
576		.rle => \|value\| value,
577		.fse => \|table\| table[@field(self, @tagName(choice)).state].symbol,
578		};
579		}
580		};
581
582		/// Decode a single block from `src` into `dest`. The beginning of `src` must be
583		/// the start of the block content (i.e. directly after the block header).
584		/// Increments `consumed_count` by the number of bytes read from `src` to decode
585		/// the block and returns the decompressed size of the block.
586		///
587		/// Errors returned:
588		///
589		/// - `error.BlockSizeOverMaximum` if block's size is larger than 1 << 17 or
590		/// `dest[written_count..].len`
591		/// - `error.MalformedBlockSize` if `src.len` is smaller than the block size
592		/// and the block is a raw or compressed block
593		/// - `error.ReservedBlock` if the block is a reserved block
594		/// - `error.MalformedRleBlock` if the block is an RLE block and `src.len < 1`
595		/// - `error.MalformedCompressedBlock` if there are errors decoding a
596		/// compressed block
597		/// - `error.DestTooSmall` is `dest` is not large enough to hold the
598		/// decompressed block
599		pub fn decodeBlock(
600		dest: []u8,
601		src: []const u8,
602		block_header: frame.Zstandard.Block.Header,
603		decode_state: *DecodeState,
604		consumed_count: *usize,
605		block_size_max: usize,
606		written_count: usize,
607		) (error{DestTooSmall} \|\| Error)!usize {
608		const block_size = block_header.block_size;
609		if (block_size_max < block_size) return error.BlockSizeOverMaximum;
610		switch (block_header.block_type) {
611		.raw => {
612		if (src.len < block_size) return error.MalformedBlockSize;
613		if (dest[written_count..].len < block_size) return error.DestTooSmall;
614		@memcpy(dest[written_count..][0..block_size], src[0..block_size]);
615		consumed_count.* += block_size;
616		decode_state.written_count += block_size;
617		return block_size;
618		},
619		.rle => {
620		if (src.len < 1) return error.MalformedRleBlock;
621		if (dest[written_count..].len < block_size) return error.DestTooSmall;
622		for (written_count..block_size + written_count) \|write_pos\| {
623		dest[write_pos] = src[0];
624		}
625		consumed_count.* += 1;
626		decode_state.written_count += block_size;
627		return block_size;
628		},
629		.compressed => {
630		if (src.len < block_size) return error.MalformedBlockSize;
631		var bytes_read: usize = 0;
632		const literals = decodeLiteralsSectionSlice(src[0..block_size], &bytes_read) catch
633		return error.MalformedCompressedBlock;
634		var fbs = std.io.fixedBufferStream(src[bytes_read..block_size]);
635		const fbs_reader = fbs.reader();
636		const sequences_header = decodeSequencesHeader(fbs_reader) catch
637		return error.MalformedCompressedBlock;
638
639		decode_state.prepare(fbs_reader, literals, sequences_header) catch
640		return error.MalformedCompressedBlock;
641
642		bytes_read += fbs.pos;
643
644		var bytes_written: usize = 0;
645		{
646		const bit_stream_bytes = src[bytes_read..block_size];
647		var bit_stream: readers.ReverseBitReader = undefined;
648		bit_stream.init(bit_stream_bytes) catch return error.MalformedCompressedBlock;
649
650		if (sequences_header.sequence_count > 0) {
651		decode_state.readInitialFseState(&bit_stream) catch
652		return error.MalformedCompressedBlock;
653
654		var sequence_size_limit = block_size_max;
655		for (0..sequences_header.sequence_count) \|i\| {
656		const write_pos = written_count + bytes_written;
657		const decompressed_size = decode_state.decodeSequenceSlice(
658		dest,
659		write_pos,
660		&bit_stream,
661		sequence_size_limit,
662		i == sequences_header.sequence_count - 1,
663		) catch \|err\| switch (err) {
664		error.DestTooSmall => return error.DestTooSmall,
665		else => return error.MalformedCompressedBlock,
666		};
667		bytes_written += decompressed_size;
668		sequence_size_limit -= decompressed_size;
669		}
670		}
671
672		if (!bit_stream.isEmpty()) {
673		return error.MalformedCompressedBlock;
674		}
675		}
676
677		if (decode_state.literal_written_count < literals.header.regenerated_size) {
678		const len = literals.header.regenerated_size - decode_state.literal_written_count;
679		if (len > dest[written_count + bytes_written ..].len) return error.DestTooSmall;
680		decode_state.decodeLiteralsSlice(dest[written_count + bytes_written ..], len) catch
681		return error.MalformedCompressedBlock;
682		bytes_written += len;
683		}
684
685		switch (decode_state.literal_header.block_type) {
686		.treeless, .compressed => {
687		if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
688		},
689		.raw, .rle => {},
690		}
691
692		consumed_count.* += block_size;
693		return bytes_written;
694		},
695		.reserved => return error.ReservedBlock,
696		}
697		}
698
699		/// Decode a single block from `src` into `dest`; see `decodeBlock()`. Returns
700		/// the size of the decompressed block, which can be used with `dest.sliceLast()`
701		/// to get the decompressed bytes. `error.BlockSizeOverMaximum` is returned if
702		/// the block's compressed or decompressed size is larger than `block_size_max`.
703		pub fn decodeBlockRingBuffer(
704		dest: *RingBuffer,
705		src: []const u8,
706		block_header: frame.Zstandard.Block.Header,
707		decode_state: *DecodeState,
708		consumed_count: *usize,
709		block_size_max: usize,
710		) Error!usize {
711		const block_size = block_header.block_size;
712		if (block_size_max < block_size) return error.BlockSizeOverMaximum;
713		switch (block_header.block_type) {
714		.raw => {
715		if (src.len < block_size) return error.MalformedBlockSize;
716		// dest may have length zero if block_size == 0, causing division by zero in
717		// writeSliceAssumeCapacity()
718		if (block_size > 0) {
719		const data = src[0..block_size];
720		dest.writeSliceAssumeCapacity(data);
721		consumed_count.* += block_size;
722		decode_state.written_count += block_size;
723		}
724		return block_size;
725		},
726		.rle => {
727		if (src.len < 1) return error.MalformedRleBlock;
728		for (0..block_size) \|_\| {
729		dest.writeAssumeCapacity(src[0]);
730		}
731		consumed_count.* += 1;
732		decode_state.written_count += block_size;
733		return block_size;
734		},
735		.compressed => {
736		if (src.len < block_size) return error.MalformedBlockSize;
737		var bytes_read: usize = 0;
738		const literals = decodeLiteralsSectionSlice(src[0..block_size], &bytes_read) catch
739		return error.MalformedCompressedBlock;
740		var fbs = std.io.fixedBufferStream(src[bytes_read..block_size]);
741		const fbs_reader = fbs.reader();
742		const sequences_header = decodeSequencesHeader(fbs_reader) catch
743		return error.MalformedCompressedBlock;
744
745		decode_state.prepare(fbs_reader, literals, sequences_header) catch
746		return error.MalformedCompressedBlock;
747
748		bytes_read += fbs.pos;
749
750		var bytes_written: usize = 0;
751		{
752		const bit_stream_bytes = src[bytes_read..block_size];
753		var bit_stream: readers.ReverseBitReader = undefined;
754		bit_stream.init(bit_stream_bytes) catch return error.MalformedCompressedBlock;
755
756		if (sequences_header.sequence_count > 0) {
757		decode_state.readInitialFseState(&bit_stream) catch
758		return error.MalformedCompressedBlock;
759
760		var sequence_size_limit = block_size_max;
761		for (0..sequences_header.sequence_count) \|i\| {
762		const decompressed_size = decode_state.decodeSequenceRingBuffer(
763		dest,
764		&bit_stream,
765		sequence_size_limit,
766		i == sequences_header.sequence_count - 1,
767		) catch return error.MalformedCompressedBlock;
768		bytes_written += decompressed_size;
769		sequence_size_limit -= decompressed_size;
770		}
771		}
772
773		if (!bit_stream.isEmpty()) {
774		return error.MalformedCompressedBlock;
775		}
776		}
777
778		if (decode_state.literal_written_count < literals.header.regenerated_size) {
779		const len = literals.header.regenerated_size - decode_state.literal_written_count;
780		decode_state.decodeLiteralsRingBuffer(dest, len) catch
781		return error.MalformedCompressedBlock;
782		bytes_written += len;
783		}
784
785		switch (decode_state.literal_header.block_type) {
786		.treeless, .compressed => {
787		if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
788		},
789		.raw, .rle => {},
790		}
791
792		consumed_count.* += block_size;
793		if (bytes_written > block_size_max) return error.BlockSizeOverMaximum;
794		return bytes_written;
795		},
796		.reserved => return error.ReservedBlock,
797		}
798		}
799
800		/// Decode a single block from `source` into `dest`. Literal and sequence data
801		/// from the block is copied into `literals_buffer` and `sequence_buffer`, which
802		/// must be large enough or `error.LiteralsBufferTooSmall` and
803		/// `error.SequenceBufferTooSmall` are returned (the maximum block size is an
804		/// upper bound for the size of both buffers). See `decodeBlock`
805		/// and `decodeBlockRingBuffer` for function that can decode a block without
806		/// these extra copies. `error.EndOfStream` is returned if `source` does not
807		/// contain enough bytes.
808		pub fn decodeBlockReader(
809		dest: *RingBuffer,
810		source: anytype,
811		block_header: frame.Zstandard.Block.Header,
812		decode_state: *DecodeState,
813		block_size_max: usize,
814		literals_buffer: []u8,
815		sequence_buffer: []u8,
816		) !void {
817		const block_size = block_header.block_size;
818		var block_reader_limited = std.io.limitedReader(source, block_size);
819		const block_reader = block_reader_limited.reader();
820		if (block_size_max < block_size) return error.BlockSizeOverMaximum;
821		switch (block_header.block_type) {
822		.raw => {
823		if (block_size == 0) return;
824		const slice = dest.sliceAt(dest.write_index, block_size);
825		try source.readNoEof(slice.first);
826		try source.readNoEof(slice.second);
827		dest.write_index = dest.mask2(dest.write_index + block_size);
828		decode_state.written_count += block_size;
829		},
830		.rle => {
831		const byte = try source.readByte();
832		for (0..block_size) \|_\| {
833		dest.writeAssumeCapacity(byte);
834		}
835		decode_state.written_count += block_size;
836		},
837		.compressed => {
838		const literals = try decodeLiteralsSection(block_reader, literals_buffer);
839		const sequences_header = try decodeSequencesHeader(block_reader);
840
841		try decode_state.prepare(block_reader, literals, sequences_header);
842
843		var bytes_written: usize = 0;
844		{
845		const size = try block_reader.readAll(sequence_buffer);
846		var bit_stream: readers.ReverseBitReader = undefined;
847		try bit_stream.init(sequence_buffer[0..size]);
848
849		if (sequences_header.sequence_count > 0) {
850		if (sequence_buffer.len < block_reader_limited.bytes_left)
851		return error.SequenceBufferTooSmall;
852
853		decode_state.readInitialFseState(&bit_stream) catch
854		return error.MalformedCompressedBlock;
855
856		var sequence_size_limit = block_size_max;
857		for (0..sequences_header.sequence_count) \|i\| {
858		const decompressed_size = decode_state.decodeSequenceRingBuffer(
859		dest,
860		&bit_stream,
861		sequence_size_limit,
862		i == sequences_header.sequence_count - 1,
863		) catch return error.MalformedCompressedBlock;
864		sequence_size_limit -= decompressed_size;
865		bytes_written += decompressed_size;
866		}
867		}
868
869		if (!bit_stream.isEmpty()) {
870		return error.MalformedCompressedBlock;
871		}
872		}
873
874		if (decode_state.literal_written_count < literals.header.regenerated_size) {
875		const len = literals.header.regenerated_size - decode_state.literal_written_count;
876		decode_state.decodeLiteralsRingBuffer(dest, len) catch
877		return error.MalformedCompressedBlock;
878		bytes_written += len;
879		}
880
881		switch (decode_state.literal_header.block_type) {
882		.treeless, .compressed => {
883		if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
884		},
885		.raw, .rle => {},
886		}
887
888		if (bytes_written > block_size_max) return error.BlockSizeOverMaximum;
889		if (block_reader_limited.bytes_left != 0) return error.MalformedCompressedBlock;
890		decode_state.literal_written_count = 0;
891		},
892		.reserved => return error.ReservedBlock,
893		}
894		}
895
896		/// Decode the header of a block.
897		pub fn decodeBlockHeader(src: *const [3]u8) frame.Zstandard.Block.Header {
898		const last_block = src[0] & 1 == 1;
899		const block_type = @as(frame.Zstandard.Block.Type, @enumFromInt((src[0] & 0b110) >> 1));
900		const block_size = ((src[0] & 0b11111000) >> 3) + (@as(u21, src[1]) << 5) + (@as(u21, src[2]) << 13);
901		return .{
902		.last_block = last_block,
903		.block_type = block_type,
904		.block_size = block_size,
905		};
906		}
907
908		/// Decode the header of a block.
909		///
910		/// Errors returned:
911		/// - `error.EndOfStream` if `src.len < 3`
912		pub fn decodeBlockHeaderSlice(src: []const u8) error{EndOfStream}!frame.Zstandard.Block.Header {
913		if (src.len < 3) return error.EndOfStream;
914		return decodeBlockHeader(src[0..3]);
915		}
916
917		/// Decode a `LiteralsSection` from `src`, incrementing `consumed_count` by the
918		/// number of bytes the section uses.
919		///
920		/// Errors returned:
921		/// - `error.MalformedLiteralsHeader` if the header is invalid
922		/// - `error.MalformedLiteralsSection` if there are decoding errors
923		/// - `error.MalformedAccuracyLog` if compressed literals have invalid
924		/// accuracy
925		/// - `error.MalformedFseTable` if compressed literals have invalid FSE table
926		/// - `error.MalformedHuffmanTree` if there are errors decoding a Huffamn tree
927		/// - `error.EndOfStream` if there are not enough bytes in `src`
928		pub fn decodeLiteralsSectionSlice(
929		src: []const u8,
930		consumed_count: *usize,
931		) (error{ MalformedLiteralsHeader, MalformedLiteralsSection, EndOfStream } \|\| huffman.Error)!LiteralsSection {
932		var bytes_read: usize = 0;
933		const header = header: {
934		var fbs = std.io.fixedBufferStream(src);
935		defer bytes_read = fbs.pos;
936		break :header decodeLiteralsHeader(fbs.reader()) catch return error.MalformedLiteralsHeader;
937		};
938		switch (header.block_type) {
939		.raw => {
940		if (src.len < bytes_read + header.regenerated_size) return error.MalformedLiteralsSection;
941		const stream = src[bytes_read..][0..header.regenerated_size];
942		consumed_count.* += header.regenerated_size + bytes_read;
943		return LiteralsSection{
944		.header = header,
945		.huffman_tree = null,
946		.streams = .{ .one = stream },
947		};
948		},
949		.rle => {
950		if (src.len < bytes_read + 1) return error.MalformedLiteralsSection;
951		const stream = src[bytes_read..][0..1];
952		consumed_count.* += 1 + bytes_read;
953		return LiteralsSection{
954		.header = header,
955		.huffman_tree = null,
956		.streams = .{ .one = stream },
957		};
958		},
959		.compressed, .treeless => {
960		const huffman_tree_start = bytes_read;
961		const huffman_tree = if (header.block_type == .compressed)
962		try huffman.decodeHuffmanTreeSlice(src[bytes_read..], &bytes_read)
963		else
964		null;
965		const huffman_tree_size = bytes_read - huffman_tree_start;
966		const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
967		return error.MalformedLiteralsSection;
968
969		if (src.len < bytes_read + total_streams_size) return error.MalformedLiteralsSection;
970		const stream_data = src[bytes_read .. bytes_read + total_streams_size];
971
972		const streams = try decodeStreams(header.size_format, stream_data);
973		consumed_count.* += bytes_read + total_streams_size;
974		return LiteralsSection{
975		.header = header,
976		.huffman_tree = huffman_tree,
977		.streams = streams,
978		};
979		},
980		}
981		}
982
983		/// Decode a `LiteralsSection` from `src`, incrementing `consumed_count` by the
984		/// number of bytes the section uses. See `decodeLiterasSectionSlice()`.
985		pub fn decodeLiteralsSection(
986		source: anytype,
987		buffer: []u8,
988		) !LiteralsSection {
989		const header = try decodeLiteralsHeader(source);
990		switch (header.block_type) {
991		.raw => {
992		if (buffer.len < header.regenerated_size) return error.LiteralsBufferTooSmall;
993		try source.readNoEof(buffer[0..header.regenerated_size]);
994		return LiteralsSection{
995		.header = header,
996		.huffman_tree = null,
997		.streams = .{ .one = buffer },
998		};
999		},
1000		.rle => {
1001		buffer[0] = try source.readByte();
1002		return LiteralsSection{
1003		.header = header,
1004		.huffman_tree = null,
1005		.streams = .{ .one = buffer[0..1] },
1006		};
1007		},
1008		.compressed, .treeless => {
1009		var counting_reader = std.io.countingReader(source);
1010		const huffman_tree = if (header.block_type == .compressed)
1011		try huffman.decodeHuffmanTree(counting_reader.reader(), buffer)
1012		else
1013		null;
1014		const huffman_tree_size = @as(usize, @intCast(counting_reader.bytes_read));
1015		const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
1016		return error.MalformedLiteralsSection;
1017
1018		if (total_streams_size > buffer.len) return error.LiteralsBufferTooSmall;
1019		try source.readNoEof(buffer[0..total_streams_size]);
1020		const stream_data = buffer[0..total_streams_size];
1021
1022		const streams = try decodeStreams(header.size_format, stream_data);
1023		return LiteralsSection{
1024		.header = header,
1025		.huffman_tree = huffman_tree,
1026		.streams = streams,
1027		};
1028		},
1029		}
1030		}
1031
1032		fn decodeStreams(size_format: u2, stream_data: []const u8) !LiteralsSection.Streams {
1033		if (size_format == 0) {
1034		return .{ .one = stream_data };
1035		}
1036
1037		if (stream_data.len < 6) return error.MalformedLiteralsSection;
1038
1039		const stream_1_length: usize = std.mem.readInt(u16, stream_data[0..2], .little);
1040		const stream_2_length: usize = std.mem.readInt(u16, stream_data[2..4], .little);
1041		const stream_3_length: usize = std.mem.readInt(u16, stream_data[4..6], .little);
1042
1043		const stream_1_start = 6;
1044		const stream_2_start = stream_1_start + stream_1_length;
1045		const stream_3_start = stream_2_start + stream_2_length;
1046		const stream_4_start = stream_3_start + stream_3_length;
1047
1048		if (stream_data.len < stream_4_start) return error.MalformedLiteralsSection;
1049
1050		return .{ .four = .{
1051		stream_data[stream_1_start .. stream_1_start + stream_1_length],
1052		stream_data[stream_2_start .. stream_2_start + stream_2_length],
1053		stream_data[stream_3_start .. stream_3_start + stream_3_length],
1054		stream_data[stream_4_start..],
1055		} };
1056		}
1057
1058		/// Decode a literals section header.
1059		///
1060		/// Errors returned:
1061		/// - `error.EndOfStream` if there are not enough bytes in `source`
1062		pub fn decodeLiteralsHeader(source: anytype) !LiteralsSection.Header {
1063		const byte0 = try source.readByte();
1064		const block_type = @as(LiteralsSection.BlockType, @enumFromInt(byte0 & 0b11));
1065		const size_format = @as(u2, @intCast((byte0 & 0b1100) >> 2));
1066		var regenerated_size: u20 = undefined;
1067		var compressed_size: ?u18 = null;
1068		switch (block_type) {
1069		.raw, .rle => {
1070		switch (size_format) {
1071		0, 2 => {
1072		regenerated_size = byte0 >> 3;
1073		},
1074		1 => regenerated_size = (byte0 >> 4) + (@as(u20, try source.readByte()) << 4),
1075		3 => regenerated_size = (byte0 >> 4) +
1076		(@as(u20, try source.readByte()) << 4) +
1077		(@as(u20, try source.readByte()) << 12),
1078		}
1079		},
1080		.compressed, .treeless => {
1081		const byte1 = try source.readByte();
1082		const byte2 = try source.readByte();
1083		switch (size_format) {
1084		0, 1 => {
1085		regenerated_size = (byte0 >> 4) + ((@as(u20, byte1) & 0b00111111) << 4);
1086		compressed_size = ((byte1 & 0b11000000) >> 6) + (@as(u18, byte2) << 2);
1087		},
1088		2 => {
1089		const byte3 = try source.readByte();
1090		regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00000011) << 12);
1091		compressed_size = ((byte2 & 0b11111100) >> 2) + (@as(u18, byte3) << 6);
1092		},
1093		3 => {
1094		const byte3 = try source.readByte();
1095		const byte4 = try source.readByte();
1096		regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00111111) << 12);
1097		compressed_size = ((byte2 & 0b11000000) >> 6) + (@as(u18, byte3) << 2) + (@as(u18, byte4) << 10);
1098		},
1099		}
1100		},
1101		}
1102		return LiteralsSection.Header{
1103		.block_type = block_type,
1104		.size_format = size_format,
1105		.regenerated_size = regenerated_size,
1106		.compressed_size = compressed_size,
1107		};
1108		}
1109
1110		/// Decode a sequences section header.
1111		///
1112		/// Errors returned:
1113		/// - `error.ReservedBitSet` if the reserved bit is set
1114		/// - `error.EndOfStream` if there are not enough bytes in `source`
1115		pub fn decodeSequencesHeader(
1116		source: anytype,
1117		) !SequencesSection.Header {
1118		var sequence_count: u24 = undefined;
1119
1120		const byte0 = try source.readByte();
1121		if (byte0 == 0) {
1122		return SequencesSection.Header{
1123		.sequence_count = 0,
1124		.offsets = undefined,
1125		.match_lengths = undefined,
1126		.literal_lengths = undefined,
1127		};
1128		} else if (byte0 < 128) {
1129		sequence_count = byte0;
1130		} else if (byte0 < 255) {
1131		sequence_count = (@as(u24, (byte0 - 128)) << 8) + try source.readByte();
1132		} else {
1133		sequence_count = (try source.readByte()) + (@as(u24, try source.readByte()) << 8) + 0x7F00;
1134		}
1135
1136		const compression_modes = try source.readByte();
1137
1138		const matches_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00001100) >> 2));
1139		const offsets_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00110000) >> 4));
1140		const literal_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b11000000) >> 6));
1141		if (compression_modes & 0b11 != 0) return error.ReservedBitSet;
1142
1143		return SequencesSection.Header{
1144		.sequence_count = sequence_count,
1145		.offsets = offsets_mode,
1146		.match_lengths = matches_mode,
1147		.literal_lengths = literal_mode,
1148		};
1149		}

lib/std/compress/zstandard/decode/fse.zig deleted-153

...	...	@@ -1,153 +0,0 @@
1		const std = @import("std");
2		const assert = std.debug.assert;
3
4		const types = @import("../types.zig");
5		const Table = types.compressed_block.Table;
6
7		pub fn decodeFseTable(
8		bit_reader: anytype,
9		expected_symbol_count: usize,
10		max_accuracy_log: u4,
11		entries: []Table.Fse,
12		) !usize {
13		const accuracy_log_biased = try bit_reader.readBitsNoEof(u4, 4);
14		if (accuracy_log_biased > max_accuracy_log -\| 5) return error.MalformedAccuracyLog;
15		const accuracy_log = accuracy_log_biased + 5;
16
17		var values: [256]u16 = undefined;
18		var value_count: usize = 0;
19
20		const total_probability = @as(u16, 1) << accuracy_log;
21		var accumulated_probability: u16 = 0;
22
23		while (accumulated_probability < total_probability) {
24		// WARNING: The RFC is poorly worded, and would suggest std.math.log2_int_ceil is correct here,
25		// but power of two (remaining probabilities + 1) need max bits set to 1 more.
26		const max_bits = std.math.log2_int(u16, total_probability - accumulated_probability + 1) + 1;
27		const small = try bit_reader.readBitsNoEof(u16, max_bits - 1);
28
29		const cutoff = (@as(u16, 1) << max_bits) - 1 - (total_probability - accumulated_probability + 1);
30
31		const value = if (small < cutoff)
32		small
33		else value: {
34		const value_read = small + (try bit_reader.readBitsNoEof(u16, 1) << (max_bits - 1));
35		break :value if (value_read < @as(u16, 1) << (max_bits - 1))
36		value_read
37		else
38		value_read - cutoff;
39		};
40
41		accumulated_probability += if (value != 0) value - 1 else 1;
42
43		values[value_count] = value;
44		value_count += 1;
45
46		if (value == 1) {
47		while (true) {
48		const repeat_flag = try bit_reader.readBitsNoEof(u2, 2);
49		if (repeat_flag + value_count > 256) return error.MalformedFseTable;
50		for (0..repeat_flag) \|_\| {
51		values[value_count] = 1;
52		value_count += 1;
53		}
54		if (repeat_flag < 3) break;
55		}
56		}
57		if (value_count == 256) break;
58		}
59		bit_reader.alignToByte();
60
61		if (value_count < 2) return error.MalformedFseTable;
62		if (accumulated_probability != total_probability) return error.MalformedFseTable;
63		if (value_count > expected_symbol_count) return error.MalformedFseTable;
64
65		const table_size = total_probability;
66
67		try buildFseTable(values[0..value_count], entries[0..table_size]);
68		return table_size;
69		}
70
71		fn buildFseTable(values: []const u16, entries: []Table.Fse) !void {
72		const total_probability = @as(u16, @intCast(entries.len));
73		const accuracy_log = std.math.log2_int(u16, total_probability);
74		assert(total_probability <= 1 << 9);
75
76		var less_than_one_count: usize = 0;
77		for (values, 0..) \|value, i\| {
78		if (value == 0) {
79		entries[entries.len - 1 - less_than_one_count] = Table.Fse{
80		.symbol = @as(u8, @intCast(i)),
81		.baseline = 0,
82		.bits = accuracy_log,
83		};
84		less_than_one_count += 1;
85		}
86		}
87
88		var position: usize = 0;
89		var temp_states: [1 << 9]u16 = undefined;
90		for (values, 0..) \|value, symbol\| {
91		if (value == 0 or value == 1) continue;
92		const probability = value - 1;
93
94		const state_share_dividend = std.math.ceilPowerOfTwo(u16, probability) catch
95		return error.MalformedFseTable;
96		const share_size = @divExact(total_probability, state_share_dividend);
97		const double_state_count = state_share_dividend - probability;
98		const single_state_count = probability - double_state_count;
99		const share_size_log = std.math.log2_int(u16, share_size);
100
101		for (0..probability) \|i\| {
102		temp_states[i] = @as(u16, @intCast(position));
103		position += (entries.len >> 1) + (entries.len >> 3) + 3;
104		position &= entries.len - 1;
105		while (position >= entries.len - less_than_one_count) {
106		position += (entries.len >> 1) + (entries.len >> 3) + 3;
107		position &= entries.len - 1;
108		}
109		}
110		std.mem.sort(u16, temp_states[0..probability], {}, std.sort.asc(u16));
111		for (0..probability) \|i\| {
112		entries[temp_states[i]] = if (i < double_state_count) Table.Fse{
113		.symbol = @as(u8, @intCast(symbol)),
114		.bits = share_size_log + 1,
115		.baseline = single_state_count * share_size + @as(u16, @intCast(i)) * 2 * share_size,
116		} else Table.Fse{
117		.symbol = @as(u8, @intCast(symbol)),
118		.bits = share_size_log,
119		.baseline = (@as(u16, @intCast(i)) - double_state_count) * share_size,
120		};
121		}
122		}
123		}
124
125		test buildFseTable {
126		const literals_length_default_values = [36]u16{
127		5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2,
128		3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 2, 2, 2, 2, 2,
129		0, 0, 0, 0,
130		};
131
132		const match_lengths_default_values = [53]u16{
133		2, 5, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
134		2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
135		2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0,
136		0, 0, 0, 0, 0,
137		};
138
139		const offset_codes_default_values = [29]u16{
140		2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
141		2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0,
142		};
143
144		var entries: [64]Table.Fse = undefined;
145		try buildFseTable(&literals_length_default_values, &entries);
146		try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_literal_fse_table.fse, &entries);
147
148		try buildFseTable(&match_lengths_default_values, &entries);
149		try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_match_fse_table.fse, &entries);
150
151		try buildFseTable(&offset_codes_default_values, entries[0..32]);
152		try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_offset_fse_table.fse, entries[0..32]);
153		}

lib/std/compress/zstandard/decode/huffman.zig deleted-234

...	...	@@ -1,234 +0,0 @@
1		const std = @import("std");
2
3		const types = @import("../types.zig");
4		const LiteralsSection = types.compressed_block.LiteralsSection;
5		const Table = types.compressed_block.Table;
6
7		const readers = @import("../readers.zig");
8
9		const decodeFseTable = @import("fse.zig").decodeFseTable;
10
11		pub const Error = error{
12		MalformedHuffmanTree,
13		MalformedFseTable,
14		MalformedAccuracyLog,
15		EndOfStream,
16		};
17
18		fn decodeFseHuffmanTree(
19		source: anytype,
20		compressed_size: usize,
21		buffer: []u8,
22		weights: *[256]u4,
23		) !usize {
24		var stream = std.io.limitedReader(source, compressed_size);
25		var bit_reader = readers.bitReader(stream.reader());
26
27		var entries: [1 << 6]Table.Fse = undefined;
28		const table_size = decodeFseTable(&bit_reader, 256, 6, &entries) catch \|err\| switch (err) {
29		error.MalformedAccuracyLog, error.MalformedFseTable => \|e\| return e,
30		error.EndOfStream => return error.MalformedFseTable,
31		else => \|e\| return e,
32		};
33		const accuracy_log = std.math.log2_int_ceil(usize, table_size);
34
35		const amount = try stream.reader().readAll(buffer);
36		var huff_bits: readers.ReverseBitReader = undefined;
37		huff_bits.init(buffer[0..amount]) catch return error.MalformedHuffmanTree;
38
39		return assignWeights(&huff_bits, accuracy_log, &entries, weights);
40		}
41
42		fn decodeFseHuffmanTreeSlice(src: []const u8, compressed_size: usize, weights: *[256]u4) !usize {
43		if (src.len < compressed_size) return error.MalformedHuffmanTree;
44		var stream = std.io.fixedBufferStream(src[0..compressed_size]);
45		var counting_reader = std.io.countingReader(stream.reader());
46		var bit_reader = readers.bitReader(counting_reader.reader());
47
48		var entries: [1 << 6]Table.Fse = undefined;
49		const table_size = decodeFseTable(&bit_reader, 256, 6, &entries) catch \|err\| switch (err) {
50		error.MalformedAccuracyLog, error.MalformedFseTable => \|e\| return e,
51		error.EndOfStream => return error.MalformedFseTable,
52		};
53		const accuracy_log = std.math.log2_int_ceil(usize, table_size);
54
55		const start_index = std.math.cast(usize, counting_reader.bytes_read) orelse
56		return error.MalformedHuffmanTree;
57		const huff_data = src[start_index..compressed_size];
58		var huff_bits: readers.ReverseBitReader = undefined;
59		huff_bits.init(huff_data) catch return error.MalformedHuffmanTree;
60
61		return assignWeights(&huff_bits, accuracy_log, &entries, weights);
62		}
63
64		fn assignWeights(
65		huff_bits: *readers.ReverseBitReader,
66		accuracy_log: u16,
67		entries: *[1 << 6]Table.Fse,
68		weights: *[256]u4,
69		) !usize {
70		var i: usize = 0;
71		var even_state: u32 = huff_bits.readBitsNoEof(u32, accuracy_log) catch return error.MalformedHuffmanTree;
72		var odd_state: u32 = huff_bits.readBitsNoEof(u32, accuracy_log) catch return error.MalformedHuffmanTree;
73
74		while (i < 254) {
75		const even_data = entries[even_state];
76		var read_bits: u16 = 0;
77		const even_bits = huff_bits.readBits(u32, even_data.bits, &read_bits) catch unreachable;
78		weights[i] = std.math.cast(u4, even_data.symbol) orelse return error.MalformedHuffmanTree;
79		i += 1;
80		if (read_bits < even_data.bits) {
81		weights[i] = std.math.cast(u4, entries[odd_state].symbol) orelse return error.MalformedHuffmanTree;
82		i += 1;
83		break;
84		}
85		even_state = even_data.baseline + even_bits;
86
87		read_bits = 0;
88		const odd_data = entries[odd_state];
89		const odd_bits = huff_bits.readBits(u32, odd_data.bits, &read_bits) catch unreachable;
90		weights[i] = std.math.cast(u4, odd_data.symbol) orelse return error.MalformedHuffmanTree;
91		i += 1;
92		if (read_bits < odd_data.bits) {
93		if (i == 255) return error.MalformedHuffmanTree;
94		weights[i] = std.math.cast(u4, entries[even_state].symbol) orelse return error.MalformedHuffmanTree;
95		i += 1;
96		break;
97		}
98		odd_state = odd_data.baseline + odd_bits;
99		} else return error.MalformedHuffmanTree;
100
101		if (!huff_bits.isEmpty()) {
102		return error.MalformedHuffmanTree;
103		}
104
105		return i + 1; // stream contains all but the last symbol
106		}
107
108		fn decodeDirectHuffmanTree(source: anytype, encoded_symbol_count: usize, weights: *[256]u4) !usize {
109		const weights_byte_count = (encoded_symbol_count + 1) / 2;
110		for (0..weights_byte_count) \|i\| {
111		const byte = try source.readByte();
112		weights[2 * i] = @as(u4, @intCast(byte >> 4));
113		weights[2 * i + 1] = @as(u4, @intCast(byte & 0xF));
114		}
115		return encoded_symbol_count + 1;
116		}
117
118		fn assignSymbols(weight_sorted_prefixed_symbols: []LiteralsSection.HuffmanTree.PrefixedSymbol, weights: [256]u4) usize {
119		for (0..weight_sorted_prefixed_symbols.len) \|i\| {
120		weight_sorted_prefixed_symbols[i] = .{
121		.symbol = @as(u8, @intCast(i)),
122		.weight = undefined,
123		.prefix = undefined,
124		};
125		}
126
127		std.mem.sort(
128		LiteralsSection.HuffmanTree.PrefixedSymbol,
129		weight_sorted_prefixed_symbols,
130		weights,
131		lessThanByWeight,
132		);
133
134		var prefix: u16 = 0;
135		var prefixed_symbol_count: usize = 0;
136		var sorted_index: usize = 0;
137		const symbol_count = weight_sorted_prefixed_symbols.len;
138		while (sorted_index < symbol_count) {
139		var symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
140		const weight = weights[symbol];
141		if (weight == 0) {
142		sorted_index += 1;
143		continue;
144		}
145
146		while (sorted_index < symbol_count) : ({
147		sorted_index += 1;
148		prefixed_symbol_count += 1;
149		prefix += 1;
150		}) {
151		symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
152		if (weights[symbol] != weight) {
153		prefix = ((prefix - 1) >> (weights[symbol] - weight)) + 1;
154		break;
155		}
156		weight_sorted_prefixed_symbols[prefixed_symbol_count].symbol = symbol;
157		weight_sorted_prefixed_symbols[prefixed_symbol_count].prefix = prefix;
158		weight_sorted_prefixed_symbols[prefixed_symbol_count].weight = weight;
159		}
160		}
161		return prefixed_symbol_count;
162		}
163
164		fn buildHuffmanTree(weights: *[256]u4, symbol_count: usize) error{MalformedHuffmanTree}!LiteralsSection.HuffmanTree {
165		var weight_power_sum_big: u32 = 0;
166		for (weights[0 .. symbol_count - 1]) \|value\| {
167		weight_power_sum_big += (@as(u16, 1) << value) >> 1;
168		}
169		if (weight_power_sum_big >= 1 << 11) return error.MalformedHuffmanTree;
170		const weight_power_sum = @as(u16, @intCast(weight_power_sum_big));
171
172		// advance to next power of two (even if weight_power_sum is a power of 2)
173		// TODO: is it valid to have weight_power_sum == 0?
174		const max_number_of_bits = if (weight_power_sum == 0) 1 else std.math.log2_int(u16, weight_power_sum) + 1;
175		const next_power_of_two = @as(u16, 1) << max_number_of_bits;
176		weights[symbol_count - 1] = std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1;
177
178		var weight_sorted_prefixed_symbols: [256]LiteralsSection.HuffmanTree.PrefixedSymbol = undefined;
179		const prefixed_symbol_count = assignSymbols(weight_sorted_prefixed_symbols[0..symbol_count], weights.*);
180		const tree = LiteralsSection.HuffmanTree{
181		.max_bit_count = max_number_of_bits,
182		.symbol_count_minus_one = @as(u8, @intCast(prefixed_symbol_count - 1)),
183		.nodes = weight_sorted_prefixed_symbols,
184		};
185		return tree;
186		}
187
188		pub fn decodeHuffmanTree(
189		source: anytype,
190		buffer: []u8,
191		) (@TypeOf(source).Error \|\| Error)!LiteralsSection.HuffmanTree {
192		const header = try source.readByte();
193		var weights: [256]u4 = undefined;
194		const symbol_count = if (header < 128)
195		// FSE compressed weights
196		try decodeFseHuffmanTree(source, header, buffer, &weights)
197		else
198		try decodeDirectHuffmanTree(source, header - 127, &weights);
199
200		return buildHuffmanTree(&weights, symbol_count);
201		}
202
203		pub fn decodeHuffmanTreeSlice(
204		src: []const u8,
205		consumed_count: *usize,
206		) Error!LiteralsSection.HuffmanTree {
207		if (src.len == 0) return error.MalformedHuffmanTree;
208		const header = src[0];
209		var bytes_read: usize = 1;
210		var weights: [256]u4 = undefined;
211		const symbol_count = if (header < 128) count: {
212		// FSE compressed weights
213		bytes_read += header;
214		break :count try decodeFseHuffmanTreeSlice(src[1..], header, &weights);
215		} else count: {
216		var fbs = std.io.fixedBufferStream(src[1..]);
217		defer bytes_read += fbs.pos;
218		break :count try decodeDirectHuffmanTree(fbs.reader(), header - 127, &weights);
219		};
220
221		consumed_count.* += bytes_read;
222		return buildHuffmanTree(&weights, symbol_count);
223		}
224
225		fn lessThanByWeight(
226		weights: [256]u4,
227		lhs: LiteralsSection.HuffmanTree.PrefixedSymbol,
228		rhs: LiteralsSection.HuffmanTree.PrefixedSymbol,
229		) bool {
230		// NOTE: this function relies on the use of a stable sorting algorithm,
231		// otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs;
232		// should be added
233		return weights[lhs.symbol] < weights[rhs.symbol];
234		}

lib/std/compress/zstandard/decompress.zig deleted-633

...	...	@@ -1,633 +0,0 @@
1		const std = @import("std");
2		const assert = std.debug.assert;
3		const Allocator = std.mem.Allocator;
4		const RingBuffer = std.RingBuffer;
5
6		const types = @import("types.zig");
7		const frame = types.frame;
8		const LiteralsSection = types.compressed_block.LiteralsSection;
9		const SequencesSection = types.compressed_block.SequencesSection;
10		const SkippableHeader = types.frame.Skippable.Header;
11		const ZstandardHeader = types.frame.Zstandard.Header;
12		const Table = types.compressed_block.Table;
13
14		pub const block = @import("decode/block.zig");
15
16		const readers = @import("readers.zig");
17
18		/// Returns `true` is `magic` is a valid magic number for a skippable frame
19		pub fn isSkippableMagic(magic: u32) bool {
20		return frame.Skippable.magic_number_min <= magic and magic <= frame.Skippable.magic_number_max;
21		}
22
23		/// Returns the kind of frame at the beginning of `source`.
24		///
25		/// Errors returned:
26		/// - `error.BadMagic` if `source` begins with bytes not equal to the
27		/// Zstandard frame magic number, or outside the range of magic numbers for
28		/// skippable frames.
29		/// - `error.EndOfStream` if `source` contains fewer than 4 bytes
30		pub fn decodeFrameType(source: anytype) error{ BadMagic, EndOfStream }!frame.Kind {
31		const magic = try source.readInt(u32, .little);
32		return frameType(magic);
33		}
34
35		/// Returns the kind of frame associated to `magic`.
36		///
37		/// Errors returned:
38		/// - `error.BadMagic` if `magic` is not a valid magic number.
39		pub fn frameType(magic: u32) error{BadMagic}!frame.Kind {
40		return if (magic == frame.Zstandard.magic_number)
41		.zstandard
42		else if (isSkippableMagic(magic))
43		.skippable
44		else
45		error.BadMagic;
46		}
47
48		pub const FrameHeader = union(enum) {
49		zstandard: ZstandardHeader,
50		skippable: SkippableHeader,
51		};
52
53		pub const HeaderError = error{ BadMagic, EndOfStream, ReservedBitSet };
54
55		/// Returns the header of the frame at the beginning of `source`.
56		///
57		/// Errors returned:
58		/// - `error.BadMagic` if `source` begins with bytes not equal to the
59		/// Zstandard frame magic number, or outside the range of magic numbers for
60		/// skippable frames.
61		/// - `error.EndOfStream` if `source` contains fewer than 4 bytes
62		/// - `error.ReservedBitSet` if the frame is a Zstandard frame and any of the
63		/// reserved bits are set
64		pub fn decodeFrameHeader(source: anytype) (@TypeOf(source).Error \|\| HeaderError)!FrameHeader {
65		const magic = try source.readInt(u32, .little);
66		const frame_type = try frameType(magic);
67		switch (frame_type) {
68		.zstandard => return FrameHeader{ .zstandard = try decodeZstandardHeader(source) },
69		.skippable => return FrameHeader{
70		.skippable = .{
71		.magic_number = magic,
72		.frame_size = try source.readInt(u32, .little),
73		},
74		},
75		}
76		}
77
78		pub const ReadWriteCount = struct {
79		read_count: usize,
80		write_count: usize,
81		};
82
83		/// Decodes frames from `src` into `dest`; returns the length of the result.
84		/// The stream should not have extra trailing bytes - either all bytes in `src`
85		/// will be decoded, or an error will be returned. An error will be returned if
86		/// a Zstandard frame in `src` does not declare its content size.
87		///
88		/// Errors returned:
89		/// - `error.DictionaryIdFlagUnsupported` if a `src` contains a frame that
90		/// uses a dictionary
91		/// - `error.MalformedFrame` if a frame in `src` is invalid
92		/// - `error.UnknownContentSizeUnsupported` if a frame in `src` does not
93		/// declare its content size
94		pub fn decode(dest: []u8, src: []const u8, verify_checksum: bool) error{
95		MalformedFrame,
96		UnknownContentSizeUnsupported,
97		DictionaryIdFlagUnsupported,
98		}!usize {
99		var write_count: usize = 0;
100		var read_count: usize = 0;
101		while (read_count < src.len) {
102		const counts = decodeFrame(dest, src[read_count..], verify_checksum) catch \|err\| {
103		switch (err) {
104		error.UnknownContentSizeUnsupported => return error.UnknownContentSizeUnsupported,
105		error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
106		else => return error.MalformedFrame,
107		}
108		};
109		read_count += counts.read_count;
110		write_count += counts.write_count;
111		}
112		return write_count;
113		}
114
115		/// Decodes a stream of frames from `src`; returns the decoded bytes. The stream
116		/// should not have extra trailing bytes - either all bytes in `src` will be
117		/// decoded, or an error will be returned.
118		///
119		/// Errors returned:
120		/// - `error.DictionaryIdFlagUnsupported` if a `src` contains a frame that
121		/// uses a dictionary
122		/// - `error.MalformedFrame` if a frame in `src` is invalid
123		/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
124		pub fn decodeAlloc(
125		allocator: Allocator,
126		src: []const u8,
127		verify_checksum: bool,
128		window_size_max: usize,
129		) error{ DictionaryIdFlagUnsupported, MalformedFrame, OutOfMemory }![]u8 {
130		var result = std.ArrayList(u8).init(allocator);
131		errdefer result.deinit();
132
133		var read_count: usize = 0;
134		while (read_count < src.len) {
135		read_count += decodeFrameArrayList(
136		allocator,
137		&result,
138		src[read_count..],
139		verify_checksum,
140		window_size_max,
141		) catch \|err\| switch (err) {
142		error.OutOfMemory => return error.OutOfMemory,
143		error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
144		else => return error.MalformedFrame,
145		};
146		}
147		return result.toOwnedSlice();
148		}
149
150		/// Decodes the frame at the start of `src` into `dest`. Returns the number of
151		/// bytes read from `src` and written to `dest`. This function can only decode
152		/// frames that declare the decompressed content size.
153		///
154		/// Errors returned:
155		/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic
156		/// number for a Zstandard or skippable frame
157		/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
158		/// uncompressed content size
159		/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
160		/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
161		/// size declared by the frame header
162		/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
163		/// that is larger than `std.math.maxInt(usize)`
164		/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
165		/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
166		/// contains a checksum that does not match the checksum of the decompressed
167		/// data
168		/// - `error.ReservedBitSet` if any of the reserved bits of the frame header
169		/// are set
170		/// - `error.EndOfStream` if `src` does not contain a complete frame
171		/// - `error.BadContentSize` if the content size declared by the frame does
172		/// not equal the actual size of decompressed data
173		/// - an error in `block.Error` if there are errors decoding a block
174		/// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a
175		/// size greater than `src.len`
176		pub fn decodeFrame(
177		dest: []u8,
178		src: []const u8,
179		verify_checksum: bool,
180		) (error{
181		BadMagic,
182		UnknownContentSizeUnsupported,
183		ContentTooLarge,
184		ContentSizeTooLarge,
185		WindowSizeUnknown,
186		DictionaryIdFlagUnsupported,
187		SkippableSizeTooLarge,
188		} \|\| FrameError)!ReadWriteCount {
189		var fbs = std.io.fixedBufferStream(src);
190		switch (try decodeFrameType(fbs.reader())) {
191		.zstandard => return decodeZstandardFrame(dest, src, verify_checksum),
192		.skippable => {
193		const content_size = try fbs.reader().readInt(u32, .little);
194		if (content_size > std.math.maxInt(usize) - 8) return error.SkippableSizeTooLarge;
195		const read_count = @as(usize, content_size) + 8;
196		if (read_count > src.len) return error.SkippableSizeTooLarge;
197		return ReadWriteCount{
198		.read_count = read_count,
199		.write_count = 0,
200		};
201		},
202		}
203		}
204
205		/// Decodes the frame at the start of `src` into `dest`. Returns the number of
206		/// bytes read from `src`.
207		///
208		/// Errors returned:
209		/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic
210		/// number for a Zstandard or skippable frame
211		/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
212		/// - `error.WindowTooLarge` if the window size is larger than
213		/// `window_size_max`
214		/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
215		/// that is larger than `std.math.maxInt(usize)`
216		/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
217		/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
218		/// contains a checksum that does not match the checksum of the decompressed
219		/// data
220		/// - `error.ReservedBitSet` if any of the reserved bits of the frame header
221		/// are set
222		/// - `error.EndOfStream` if `src` does not contain a complete frame
223		/// - `error.BadContentSize` if the content size declared by the frame does
224		/// not equal the actual size of decompressed data
225		/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
226		/// - an error in `block.Error` if there are errors decoding a block
227		/// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a
228		/// size greater than `src.len`
229		pub fn decodeFrameArrayList(
230		allocator: Allocator,
231		dest: *std.ArrayList(u8),
232		src: []const u8,
233		verify_checksum: bool,
234		window_size_max: usize,
235		) (error{ BadMagic, OutOfMemory, SkippableSizeTooLarge } \|\| FrameContext.Error \|\| FrameError)!usize {
236		var fbs = std.io.fixedBufferStream(src);
237		const reader = fbs.reader();
238		const magic = try reader.readInt(u32, .little);
239		switch (try frameType(magic)) {
240		.zstandard => return decodeZstandardFrameArrayList(
241		allocator,
242		dest,
243		src,
244		verify_checksum,
245		window_size_max,
246		),
247		.skippable => {
248		const content_size = try fbs.reader().readInt(u32, .little);
249		if (content_size > std.math.maxInt(usize) - 8) return error.SkippableSizeTooLarge;
250		const read_count = @as(usize, content_size) + 8;
251		if (read_count > src.len) return error.SkippableSizeTooLarge;
252		return read_count;
253		},
254		}
255		}
256
257		/// Returns the frame checksum corresponding to the data fed into `hasher`
258		pub fn computeChecksum(hasher: *std.hash.XxHash64) u32 {
259		const hash = hasher.final();
260		return @as(u32, @intCast(hash & 0xFFFFFFFF));
261		}
262
263		const FrameError = error{
264		ChecksumFailure,
265		BadContentSize,
266		EndOfStream,
267		ReservedBitSet,
268		} \|\| block.Error;
269
270		/// Decode a Zstandard frame from `src` into `dest`, returning the number of
271		/// bytes read from `src` and written to `dest`. The first four bytes of `src`
272		/// must be the magic number for a Zstandard frame.
273		///
274		/// Error returned:
275		/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
276		/// uncompressed content size
277		/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
278		/// size declared by the frame header
279		/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
280		/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
281		/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
282		/// that is larger than `std.math.maxInt(usize)`
283		/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
284		/// contains a checksum that does not match the checksum of the decompressed
285		/// data
286		/// - `error.ReservedBitSet` if the reserved bit of the frame header is set
287		/// - `error.EndOfStream` if `src` does not contain a complete frame
288		/// - an error in `block.Error` if there are errors decoding a block
289		/// - `error.BadContentSize` if the content size declared by the frame does
290		/// not equal the actual size of decompressed data
291		pub fn decodeZstandardFrame(
292		dest: []u8,
293		src: []const u8,
294		verify_checksum: bool,
295		) (error{
296		UnknownContentSizeUnsupported,
297		ContentTooLarge,
298		ContentSizeTooLarge,
299		WindowSizeUnknown,
300		DictionaryIdFlagUnsupported,
301		} \|\| FrameError)!ReadWriteCount {
302		assert(std.mem.readInt(u32, src[0..4], .little) == frame.Zstandard.magic_number);
303		var consumed_count: usize = 4;
304
305		var frame_context = context: {
306		var fbs = std.io.fixedBufferStream(src[consumed_count..]);
307		const source = fbs.reader();
308		const frame_header = try decodeZstandardHeader(source);
309		consumed_count += fbs.pos;
310		break :context FrameContext.init(
311		frame_header,
312		std.math.maxInt(usize),
313		verify_checksum,
314		) catch \|err\| switch (err) {
315		error.WindowTooLarge => unreachable,
316		inline else => \|e\| return e,
317		};
318		};
319		const counts = try decodeZStandardFrameBlocks(
320		dest,
321		src[consumed_count..],
322		&frame_context,
323		);
324		return ReadWriteCount{
325		.read_count = counts.read_count + consumed_count,
326		.write_count = counts.write_count,
327		};
328		}
329
330		pub fn decodeZStandardFrameBlocks(
331		dest: []u8,
332		src: []const u8,
333		frame_context: *FrameContext,
334		) (error{ ContentTooLarge, UnknownContentSizeUnsupported } \|\| FrameError)!ReadWriteCount {
335		const content_size = frame_context.content_size orelse
336		return error.UnknownContentSizeUnsupported;
337		if (dest.len < content_size) return error.ContentTooLarge;
338
339		var consumed_count: usize = 0;
340		const written_count = decodeFrameBlocksInner(
341		dest[0..content_size],
342		src[consumed_count..],
343		&consumed_count,
344		if (frame_context.hasher_opt) \|*hasher\| hasher else null,
345		frame_context.block_size_max,
346		) catch \|err\| switch (err) {
347		error.DestTooSmall => return error.BadContentSize,
348		inline else => \|e\| return e,
349		};
350
351		if (written_count != content_size) return error.BadContentSize;
352		if (frame_context.has_checksum) {
353		if (src.len < consumed_count + 4) return error.EndOfStream;
354		const checksum = std.mem.readInt(u32, src[consumed_count..][0..4], .little);
355		consumed_count += 4;
356		if (frame_context.hasher_opt) \|*hasher\| {
357		if (checksum != computeChecksum(hasher)) return error.ChecksumFailure;
358		}
359		}
360		return ReadWriteCount{ .read_count = consumed_count, .write_count = written_count };
361		}
362
363		pub const FrameContext = struct {
364		hasher_opt: ?std.hash.XxHash64,
365		window_size: usize,
366		has_checksum: bool,
367		block_size_max: usize,
368		content_size: ?usize,
369
370		const Error = error{
371		DictionaryIdFlagUnsupported,
372		WindowSizeUnknown,
373		WindowTooLarge,
374		ContentSizeTooLarge,
375		};
376		/// Validates `frame_header` and returns the associated `FrameContext`.
377		///
378		/// Errors returned:
379		/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
380		/// - `error.WindowSizeUnknown` if the frame does not have a valid window
381		/// size
382		/// - `error.WindowTooLarge` if the window size is larger than
383		/// `window_size_max` or `std.math.intMax(usize)`
384		/// - `error.ContentSizeTooLarge` if the frame header indicates a content
385		/// size larger than `std.math.maxInt(usize)`
386		pub fn init(
387		frame_header: ZstandardHeader,
388		window_size_max: usize,
389		verify_checksum: bool,
390		) Error!FrameContext {
391		if (frame_header.descriptor.dictionary_id_flag != 0)
392		return error.DictionaryIdFlagUnsupported;
393
394		const window_size_raw = frameWindowSize(frame_header) orelse return error.WindowSizeUnknown;
395		const window_size = if (window_size_raw > window_size_max)
396		return error.WindowTooLarge
397		else
398		std.math.cast(usize, window_size_raw) orelse return error.WindowTooLarge;
399
400		const should_compute_checksum =
401		frame_header.descriptor.content_checksum_flag and verify_checksum;
402
403		const content_size = if (frame_header.content_size) \|size\|
404		std.math.cast(usize, size) orelse return error.ContentSizeTooLarge
405		else
406		null;
407
408		return .{
409		.hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,
410		.window_size = window_size,
411		.has_checksum = frame_header.descriptor.content_checksum_flag,
412		.block_size_max = @min(types.block_size_max, window_size),
413		.content_size = content_size,
414		};
415		}
416		};
417
418		/// Decode a Zstandard from from `src` and return number of bytes read; see
419		/// `decodeZstandardFrame()`. The first four bytes of `src` must be the magic
420		/// number for a Zstandard frame.
421		///
422		/// Errors returned:
423		/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
424		/// - `error.WindowTooLarge` if the window size is larger than
425		/// `window_size_max`
426		/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
427		/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
428		/// that is larger than `std.math.maxInt(usize)`
429		/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
430		/// contains a checksum that does not match the checksum of the decompressed
431		/// data
432		/// - `error.ReservedBitSet` if the reserved bit of the frame header is set
433		/// - `error.EndOfStream` if `src` does not contain a complete frame
434		/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
435		/// - an error in `block.Error` if there are errors decoding a block
436		/// - `error.BadContentSize` if the content size declared by the frame does
437		/// not equal the size of decompressed data
438		pub fn decodeZstandardFrameArrayList(
439		allocator: Allocator,
440		dest: *std.ArrayList(u8),
441		src: []const u8,
442		verify_checksum: bool,
443		window_size_max: usize,
444		) (error{OutOfMemory} \|\| FrameContext.Error \|\| FrameError)!usize {
445		assert(std.mem.readInt(u32, src[0..4], .little) == frame.Zstandard.magic_number);
446		var consumed_count: usize = 4;
447
448		var frame_context = context: {
449		var fbs = std.io.fixedBufferStream(src[consumed_count..]);
450		const source = fbs.reader();
451		const frame_header = try decodeZstandardHeader(source);
452		consumed_count += fbs.pos;
453		break :context try FrameContext.init(frame_header, window_size_max, verify_checksum);
454		};
455
456		consumed_count += try decodeZstandardFrameBlocksArrayList(
457		allocator,
458		dest,
459		src[consumed_count..],
460		&frame_context,
461		);
462		return consumed_count;
463		}
464
465		pub fn decodeZstandardFrameBlocksArrayList(
466		allocator: Allocator,
467		dest: *std.ArrayList(u8),
468		src: []const u8,
469		frame_context: *FrameContext,
470		) (error{OutOfMemory} \|\| FrameError)!usize {
471		const initial_len = dest.items.len;
472
473		var ring_buffer = try RingBuffer.init(allocator, frame_context.window_size);
474		defer ring_buffer.deinit(allocator);
475
476		// These tables take 7680 bytes
477		var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined;
478		var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined;
479		var offset_fse_data: [types.compressed_block.table_size_max.offset]Table.Fse = undefined;
480
481		var block_header = try block.decodeBlockHeaderSlice(src);
482		var consumed_count: usize = 3;
483		var decode_state = block.DecodeState.init(&literal_fse_data, &match_fse_data, &offset_fse_data);
484		while (true) : ({
485		block_header = try block.decodeBlockHeaderSlice(src[consumed_count..]);
486		consumed_count += 3;
487		}) {
488		const written_size = try block.decodeBlockRingBuffer(
489		&ring_buffer,
490		src[consumed_count..],
491		block_header,
492		&decode_state,
493		&consumed_count,
494		frame_context.block_size_max,
495		);
496		if (frame_context.content_size) \|size\| {
497		if (dest.items.len - initial_len > size) {
498		return error.BadContentSize;
499		}
500		}
501		if (written_size > 0) {
502		const written_slice = ring_buffer.sliceLast(written_size);
503		try dest.appendSlice(written_slice.first);
504		try dest.appendSlice(written_slice.second);
505		if (frame_context.hasher_opt) \|*hasher\| {
506		hasher.update(written_slice.first);
507		hasher.update(written_slice.second);
508		}
509		}
510		if (block_header.last_block) break;
511		}
512		if (frame_context.content_size) \|size\| {
513		if (dest.items.len - initial_len != size) {
514		return error.BadContentSize;
515		}
516		}
517
518		if (frame_context.has_checksum) {
519		if (src.len < consumed_count + 4) return error.EndOfStream;
520		const checksum = std.mem.readInt(u32, src[consumed_count..][0..4], .little);
521		consumed_count += 4;
522		if (frame_context.hasher_opt) \|*hasher\| {
523		if (checksum != computeChecksum(hasher)) return error.ChecksumFailure;
524		}
525		}
526		return consumed_count;
527		}
528
529		fn decodeFrameBlocksInner(
530		dest: []u8,
531		src: []const u8,
532		consumed_count: *usize,
533		hash: ?*std.hash.XxHash64,
534		block_size_max: usize,
535		) (error{ EndOfStream, DestTooSmall } \|\| block.Error)!usize {
536		// These tables take 7680 bytes
537		var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined;
538		var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined;
539		var offset_fse_data: [types.compressed_block.table_size_max.offset]Table.Fse = undefined;
540
541		var block_header = try block.decodeBlockHeaderSlice(src);
542		var bytes_read: usize = 3;
543		defer consumed_count.* += bytes_read;
544		var decode_state = block.DecodeState.init(&literal_fse_data, &match_fse_data, &offset_fse_data);
545		var count: usize = 0;
546		while (true) : ({
547		block_header = try block.decodeBlockHeaderSlice(src[bytes_read..]);
548		bytes_read += 3;
549		}) {
550		const written_size = try block.decodeBlock(
551		dest,
552		src[bytes_read..],
553		block_header,
554		&decode_state,
555		&bytes_read,
556		block_size_max,
557		count,
558		);
559		if (hash) \|hash_state\| hash_state.update(dest[count .. count + written_size]);
560		count += written_size;
561		if (block_header.last_block) break;
562		}
563		return count;
564		}
565
566		/// Decode the header of a skippable frame. The first four bytes of `src` must
567		/// be a valid magic number for a skippable frame.
568		pub fn decodeSkippableHeader(src: *const [8]u8) SkippableHeader {
569		const magic = std.mem.readInt(u32, src[0..4], .little);
570		assert(isSkippableMagic(magic));
571		const frame_size = std.mem.readInt(u32, src[4..8], .little);
572		return .{
573		.magic_number = magic,
574		.frame_size = frame_size,
575		};
576		}
577
578		/// Returns the window size required to decompress a frame, or `null` if it
579		/// cannot be determined (which indicates a malformed frame header).
580		pub fn frameWindowSize(header: ZstandardHeader) ?u64 {
581		if (header.window_descriptor) \|descriptor\| {
582		const exponent = (descriptor & 0b11111000) >> 3;
583		const mantissa = descriptor & 0b00000111;
584		const window_log = 10 + exponent;
585		const window_base = @as(u64, 1) << @as(u6, @intCast(window_log));
586		const window_add = (window_base / 8) * mantissa;
587		return window_base + window_add;
588		} else return header.content_size;
589		}
590
591		/// Decode the header of a Zstandard frame.
592		///
593		/// Errors returned:
594		/// - `error.ReservedBitSet` if any of the reserved bits of the header are set
595		/// - `error.EndOfStream` if `source` does not contain a complete header
596		pub fn decodeZstandardHeader(
597		source: anytype,
598		) (@TypeOf(source).Error \|\| error{ EndOfStream, ReservedBitSet })!ZstandardHeader {
599		const descriptor = @as(ZstandardHeader.Descriptor, @bitCast(try source.readByte()));
600
601		if (descriptor.reserved) return error.ReservedBitSet;
602
603		var window_descriptor: ?u8 = null;
604		if (!descriptor.single_segment_flag) {
605		window_descriptor = try source.readByte();
606		}
607
608		var dictionary_id: ?u32 = null;
609		if (descriptor.dictionary_id_flag > 0) {
610		// if flag is 3 then field_size = 4, else field_size = flag
611		const field_size = (@as(u4, 1) << descriptor.dictionary_id_flag) >> 1;
612		dictionary_id = try source.readVarInt(u32, .little, field_size);
613		}
614
615		var content_size: ?u64 = null;
616		if (descriptor.single_segment_flag or descriptor.content_size_flag > 0) {
617		const field_size = @as(u4, 1) << descriptor.content_size_flag;
618		content_size = try source.readVarInt(u64, .little, field_size);
619		if (field_size == 2) content_size.? += 256;
620		}
621
622		const header = ZstandardHeader{
623		.descriptor = descriptor,
624		.window_descriptor = window_descriptor,
625		.dictionary_id = dictionary_id,
626		.content_size = content_size,
627		};
628		return header;
629		}
630
631		test {
632		std.testing.refAllDecls(@This());
633		}

lib/std/compress/zstandard/readers.zig deleted-82

...	...	@@ -1,82 +0,0 @@
1		const std = @import("std");
2
3		pub const ReversedByteReader = struct {
4		remaining_bytes: usize,
5		bytes: []const u8,
6
7		const Reader = std.io.GenericReader(*ReversedByteReader, error{}, readFn);
8
9		pub fn init(bytes: []const u8) ReversedByteReader {
10		return .{
11		.bytes = bytes,
12		.remaining_bytes = bytes.len,
13		};
14		}
15
16		pub fn reader(self: *ReversedByteReader) Reader {
17		return .{ .context = self };
18		}
19
20		fn readFn(ctx: *ReversedByteReader, buffer: []u8) !usize {
21		if (ctx.remaining_bytes == 0) return 0;
22		const byte_index = ctx.remaining_bytes - 1;
23		buffer[0] = ctx.bytes[byte_index];
24		// buffer[0] = @bitReverse(ctx.bytes[byte_index]);
25		ctx.remaining_bytes = byte_index;
26		return 1;
27		}
28		};
29
30		/// A bit reader for reading the reversed bit streams used to encode
31		/// FSE compressed data.
32		pub const ReverseBitReader = struct {
33		byte_reader: ReversedByteReader,
34		bit_reader: std.io.BitReader(.big, ReversedByteReader.Reader),
35
36		pub fn init(self: *ReverseBitReader, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
37		self.byte_reader = ReversedByteReader.init(bytes);
38		self.bit_reader = std.io.bitReader(.big, self.byte_reader.reader());
39		if (bytes.len == 0) return;
40		var i: usize = 0;
41		while (i < 8 and 0 == self.readBitsNoEof(u1, 1) catch unreachable) : (i += 1) {}
42		if (i == 8) return error.BitStreamHasNoStartBit;
43		}
44
45		pub fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: u16) error{EndOfStream}!U {
46		return self.bit_reader.readBitsNoEof(U, num_bits);
47		}
48
49		pub fn readBits(self: @This(), comptime U: type, num_bits: u16, out_bits: u16) error{}!U {
50		return try self.bit_reader.readBits(U, num_bits, out_bits);
51		}
52
53		pub fn alignToByte(self: *@This()) void {
54		self.bit_reader.alignToByte();
55		}
56
57		pub fn isEmpty(self: ReverseBitReader) bool {
58		return self.byte_reader.remaining_bytes == 0 and self.bit_reader.count == 0;
59		}
60		};
61
62		pub fn BitReader(comptime Reader: type) type {
63		return struct {
64		underlying: std.io.BitReader(.little, Reader),
65
66		pub fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: u16) !U {
67		return self.underlying.readBitsNoEof(U, num_bits);
68		}
69
70		pub fn readBits(self: @This(), comptime U: type, num_bits: u16, out_bits: u16) !U {
71		return self.underlying.readBits(U, num_bits, out_bits);
72		}
73
74		pub fn alignToByte(self: *@This()) void {
75		self.underlying.alignToByte();
76		}
77		};
78		}
79
80		pub fn bitReader(reader: anytype) BitReader(@TypeOf(reader)) {
81		return .{ .underlying = std.io.bitReader(.little, reader) };
82		}

lib/std/compress/zstandard/types.zig deleted-403

...	...	@@ -1,403 +0,0 @@
1		pub const block_size_max = 1 << 17;
2
3		pub const frame = struct {
4		pub const Kind = enum { zstandard, skippable };
5
6		pub const Zstandard = struct {
7		pub const magic_number = 0xFD2FB528;
8
9		header: Header,
10		data_blocks: []Block,
11		checksum: ?u32,
12
13		pub const Header = struct {
14		descriptor: Descriptor,
15		window_descriptor: ?u8,
16		dictionary_id: ?u32,
17		content_size: ?u64,
18
19		pub const Descriptor = packed struct {
20		dictionary_id_flag: u2,
21		content_checksum_flag: bool,
22		reserved: bool,
23		unused: bool,
24		single_segment_flag: bool,
25		content_size_flag: u2,
26		};
27		};
28
29		pub const Block = struct {
30		pub const Header = struct {
31		last_block: bool,
32		block_type: Block.Type,
33		block_size: u21,
34		};
35
36		pub const Type = enum(u2) {
37		raw,
38		rle,
39		compressed,
40		reserved,
41		};
42		};
43		};
44
45		pub const Skippable = struct {
46		pub const magic_number_min = 0x184D2A50;
47		pub const magic_number_max = 0x184D2A5F;
48
49		pub const Header = struct {
50		magic_number: u32,
51		frame_size: u32,
52		};
53		};
54		};
55
56		pub const compressed_block = struct {
57		pub const LiteralsSection = struct {
58		header: Header,
59		huffman_tree: ?HuffmanTree,
60		streams: Streams,
61
62		pub const Streams = union(enum) {
63		one: []const u8,
64		four: [4][]const u8,
65		};
66
67		pub const Header = struct {
68		block_type: BlockType,
69		size_format: u2,
70		regenerated_size: u20,
71		compressed_size: ?u18,
72		};
73
74		pub const BlockType = enum(u2) {
75		raw,
76		rle,
77		compressed,
78		treeless,
79		};
80
81		pub const HuffmanTree = struct {
82		max_bit_count: u4,
83		symbol_count_minus_one: u8,
84		nodes: [256]PrefixedSymbol,
85
86		pub const PrefixedSymbol = struct {
87		symbol: u8,
88		prefix: u16,
89		weight: u4,
90		};
91
92		pub const Result = union(enum) {
93		symbol: u8,
94		index: usize,
95		};
96
97		pub fn query(self: HuffmanTree, index: usize, prefix: u16) error{NotFound}!Result {
98		var node = self.nodes[index];
99		const weight = node.weight;
100		var i: usize = index;
101		while (node.weight == weight) {
102		if (node.prefix == prefix) return Result{ .symbol = node.symbol };
103		if (i == 0) return error.NotFound;
104		i -= 1;
105		node = self.nodes[i];
106		}
107		return Result{ .index = i };
108		}
109
110		pub fn weightToBitCount(weight: u4, max_bit_count: u4) u4 {
111		return if (weight == 0) 0 else ((max_bit_count + 1) - weight);
112		}
113		};
114
115		pub const StreamCount = enum { one, four };
116		pub fn streamCount(size_format: u2, block_type: BlockType) StreamCount {
117		return switch (block_type) {
118		.raw, .rle => .one,
119		.compressed, .treeless => if (size_format == 0) .one else .four,
120		};
121		}
122		};
123
124		pub const SequencesSection = struct {
125		header: SequencesSection.Header,
126		literals_length_table: Table,
127		offset_table: Table,
128		match_length_table: Table,
129
130		pub const Header = struct {
131		sequence_count: u24,
132		match_lengths: Mode,
133		offsets: Mode,
134		literal_lengths: Mode,
135
136		pub const Mode = enum(u2) {
137		predefined,
138		rle,
139		fse,
140		repeat,
141		};
142		};
143		};
144
145		pub const Table = union(enum) {
146		fse: []const Fse,
147		rle: u8,
148
149		pub const Fse = struct {
150		symbol: u8,
151		baseline: u16,
152		bits: u8,
153		};
154		};
155
156		pub const literals_length_code_table = [36]struct { u32, u5 }{
157		.{ 0, 0 }, .{ 1, 0 }, .{ 2, 0 }, .{ 3, 0 },
158		.{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 },
159		.{ 8, 0 }, .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 },
160		.{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 }, .{ 15, 0 },
161		.{ 16, 1 }, .{ 18, 1 }, .{ 20, 1 }, .{ 22, 1 },
162		.{ 24, 2 }, .{ 28, 2 }, .{ 32, 3 }, .{ 40, 3 },
163		.{ 48, 4 }, .{ 64, 6 }, .{ 128, 7 }, .{ 256, 8 },
164		.{ 512, 9 }, .{ 1024, 10 }, .{ 2048, 11 }, .{ 4096, 12 },
165		.{ 8192, 13 }, .{ 16384, 14 }, .{ 32768, 15 }, .{ 65536, 16 },
166		};
167
168		pub const match_length_code_table = [53]struct { u32, u5 }{
169		.{ 3, 0 }, .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 }, .{ 8, 0 },
170		.{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 }, .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 },
171		.{ 15, 0 }, .{ 16, 0 }, .{ 17, 0 }, .{ 18, 0 }, .{ 19, 0 }, .{ 20, 0 },
172		.{ 21, 0 }, .{ 22, 0 }, .{ 23, 0 }, .{ 24, 0 }, .{ 25, 0 }, .{ 26, 0 },
173		.{ 27, 0 }, .{ 28, 0 }, .{ 29, 0 }, .{ 30, 0 }, .{ 31, 0 }, .{ 32, 0 },
174		.{ 33, 0 }, .{ 34, 0 }, .{ 35, 1 }, .{ 37, 1 }, .{ 39, 1 }, .{ 41, 1 },
175		.{ 43, 2 }, .{ 47, 2 }, .{ 51, 3 }, .{ 59, 3 }, .{ 67, 4 }, .{ 83, 4 },
176		.{ 99, 5 }, .{ 131, 7 }, .{ 259, 8 }, .{ 515, 9 }, .{ 1027, 10 }, .{ 2051, 11 },
177		.{ 4099, 12 }, .{ 8195, 13 }, .{ 16387, 14 }, .{ 32771, 15 }, .{ 65539, 16 },
178		};
179
180		pub const literals_length_default_distribution = [36]i16{
181		4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
182		2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
183		-1, -1, -1, -1,
184		};
185
186		pub const match_lengths_default_distribution = [53]i16{
187		1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
188		1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
189		1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
190		-1, -1, -1, -1, -1,
191		};
192
193		pub const offset_codes_default_distribution = [29]i16{
194		1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
195		1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1,
196		};
197
198		pub const predefined_literal_fse_table = Table{
199		.fse = &[64]Table.Fse{
200		.{ .symbol = 0, .bits = 4, .baseline = 0 },
201		.{ .symbol = 0, .bits = 4, .baseline = 16 },
202		.{ .symbol = 1, .bits = 5, .baseline = 32 },
203		.{ .symbol = 3, .bits = 5, .baseline = 0 },
204		.{ .symbol = 4, .bits = 5, .baseline = 0 },
205		.{ .symbol = 6, .bits = 5, .baseline = 0 },
206		.{ .symbol = 7, .bits = 5, .baseline = 0 },
207		.{ .symbol = 9, .bits = 5, .baseline = 0 },
208		.{ .symbol = 10, .bits = 5, .baseline = 0 },
209		.{ .symbol = 12, .bits = 5, .baseline = 0 },
210		.{ .symbol = 14, .bits = 6, .baseline = 0 },
211		.{ .symbol = 16, .bits = 5, .baseline = 0 },
212		.{ .symbol = 18, .bits = 5, .baseline = 0 },
213		.{ .symbol = 19, .bits = 5, .baseline = 0 },
214		.{ .symbol = 21, .bits = 5, .baseline = 0 },
215		.{ .symbol = 22, .bits = 5, .baseline = 0 },
216		.{ .symbol = 24, .bits = 5, .baseline = 0 },
217		.{ .symbol = 25, .bits = 5, .baseline = 32 },
218		.{ .symbol = 26, .bits = 5, .baseline = 0 },
219		.{ .symbol = 27, .bits = 6, .baseline = 0 },
220		.{ .symbol = 29, .bits = 6, .baseline = 0 },
221		.{ .symbol = 31, .bits = 6, .baseline = 0 },
222		.{ .symbol = 0, .bits = 4, .baseline = 32 },
223		.{ .symbol = 1, .bits = 4, .baseline = 0 },
224		.{ .symbol = 2, .bits = 5, .baseline = 0 },
225		.{ .symbol = 4, .bits = 5, .baseline = 32 },
226		.{ .symbol = 5, .bits = 5, .baseline = 0 },
227		.{ .symbol = 7, .bits = 5, .baseline = 32 },
228		.{ .symbol = 8, .bits = 5, .baseline = 0 },
229		.{ .symbol = 10, .bits = 5, .baseline = 32 },
230		.{ .symbol = 11, .bits = 5, .baseline = 0 },
231		.{ .symbol = 13, .bits = 6, .baseline = 0 },
232		.{ .symbol = 16, .bits = 5, .baseline = 32 },
233		.{ .symbol = 17, .bits = 5, .baseline = 0 },
234		.{ .symbol = 19, .bits = 5, .baseline = 32 },
235		.{ .symbol = 20, .bits = 5, .baseline = 0 },
236		.{ .symbol = 22, .bits = 5, .baseline = 32 },
237		.{ .symbol = 23, .bits = 5, .baseline = 0 },
238		.{ .symbol = 25, .bits = 4, .baseline = 0 },
239		.{ .symbol = 25, .bits = 4, .baseline = 16 },
240		.{ .symbol = 26, .bits = 5, .baseline = 32 },
241		.{ .symbol = 28, .bits = 6, .baseline = 0 },
242		.{ .symbol = 30, .bits = 6, .baseline = 0 },
243		.{ .symbol = 0, .bits = 4, .baseline = 48 },
244		.{ .symbol = 1, .bits = 4, .baseline = 16 },
245		.{ .symbol = 2, .bits = 5, .baseline = 32 },
246		.{ .symbol = 3, .bits = 5, .baseline = 32 },
247		.{ .symbol = 5, .bits = 5, .baseline = 32 },
248		.{ .symbol = 6, .bits = 5, .baseline = 32 },
249		.{ .symbol = 8, .bits = 5, .baseline = 32 },
250		.{ .symbol = 9, .bits = 5, .baseline = 32 },
251		.{ .symbol = 11, .bits = 5, .baseline = 32 },
252		.{ .symbol = 12, .bits = 5, .baseline = 32 },
253		.{ .symbol = 15, .bits = 6, .baseline = 0 },
254		.{ .symbol = 17, .bits = 5, .baseline = 32 },
255		.{ .symbol = 18, .bits = 5, .baseline = 32 },
256		.{ .symbol = 20, .bits = 5, .baseline = 32 },
257		.{ .symbol = 21, .bits = 5, .baseline = 32 },
258		.{ .symbol = 23, .bits = 5, .baseline = 32 },
259		.{ .symbol = 24, .bits = 5, .baseline = 32 },
260		.{ .symbol = 35, .bits = 6, .baseline = 0 },
261		.{ .symbol = 34, .bits = 6, .baseline = 0 },
262		.{ .symbol = 33, .bits = 6, .baseline = 0 },
263		.{ .symbol = 32, .bits = 6, .baseline = 0 },
264		},
265		};
266
267		pub const predefined_match_fse_table = Table{
268		.fse = &[64]Table.Fse{
269		.{ .symbol = 0, .bits = 6, .baseline = 0 },
270		.{ .symbol = 1, .bits = 4, .baseline = 0 },
271		.{ .symbol = 2, .bits = 5, .baseline = 32 },
272		.{ .symbol = 3, .bits = 5, .baseline = 0 },
273		.{ .symbol = 5, .bits = 5, .baseline = 0 },
274		.{ .symbol = 6, .bits = 5, .baseline = 0 },
275		.{ .symbol = 8, .bits = 5, .baseline = 0 },
276		.{ .symbol = 10, .bits = 6, .baseline = 0 },
277		.{ .symbol = 13, .bits = 6, .baseline = 0 },
278		.{ .symbol = 16, .bits = 6, .baseline = 0 },
279		.{ .symbol = 19, .bits = 6, .baseline = 0 },
280		.{ .symbol = 22, .bits = 6, .baseline = 0 },
281		.{ .symbol = 25, .bits = 6, .baseline = 0 },
282		.{ .symbol = 28, .bits = 6, .baseline = 0 },
283		.{ .symbol = 31, .bits = 6, .baseline = 0 },
284		.{ .symbol = 33, .bits = 6, .baseline = 0 },
285		.{ .symbol = 35, .bits = 6, .baseline = 0 },
286		.{ .symbol = 37, .bits = 6, .baseline = 0 },
287		.{ .symbol = 39, .bits = 6, .baseline = 0 },
288		.{ .symbol = 41, .bits = 6, .baseline = 0 },
289		.{ .symbol = 43, .bits = 6, .baseline = 0 },
290		.{ .symbol = 45, .bits = 6, .baseline = 0 },
291		.{ .symbol = 1, .bits = 4, .baseline = 16 },
292		.{ .symbol = 2, .bits = 4, .baseline = 0 },
293		.{ .symbol = 3, .bits = 5, .baseline = 32 },
294		.{ .symbol = 4, .bits = 5, .baseline = 0 },
295		.{ .symbol = 6, .bits = 5, .baseline = 32 },
296		.{ .symbol = 7, .bits = 5, .baseline = 0 },
297		.{ .symbol = 9, .bits = 6, .baseline = 0 },
298		.{ .symbol = 12, .bits = 6, .baseline = 0 },
299		.{ .symbol = 15, .bits = 6, .baseline = 0 },
300		.{ .symbol = 18, .bits = 6, .baseline = 0 },
301		.{ .symbol = 21, .bits = 6, .baseline = 0 },
302		.{ .symbol = 24, .bits = 6, .baseline = 0 },
303		.{ .symbol = 27, .bits = 6, .baseline = 0 },
304		.{ .symbol = 30, .bits = 6, .baseline = 0 },
305		.{ .symbol = 32, .bits = 6, .baseline = 0 },
306		.{ .symbol = 34, .bits = 6, .baseline = 0 },
307		.{ .symbol = 36, .bits = 6, .baseline = 0 },
308		.{ .symbol = 38, .bits = 6, .baseline = 0 },
309		.{ .symbol = 40, .bits = 6, .baseline = 0 },
310		.{ .symbol = 42, .bits = 6, .baseline = 0 },
311		.{ .symbol = 44, .bits = 6, .baseline = 0 },
312		.{ .symbol = 1, .bits = 4, .baseline = 32 },
313		.{ .symbol = 1, .bits = 4, .baseline = 48 },
314		.{ .symbol = 2, .bits = 4, .baseline = 16 },
315		.{ .symbol = 4, .bits = 5, .baseline = 32 },
316		.{ .symbol = 5, .bits = 5, .baseline = 32 },
317		.{ .symbol = 7, .bits = 5, .baseline = 32 },
318		.{ .symbol = 8, .bits = 5, .baseline = 32 },
319		.{ .symbol = 11, .bits = 6, .baseline = 0 },
320		.{ .symbol = 14, .bits = 6, .baseline = 0 },
321		.{ .symbol = 17, .bits = 6, .baseline = 0 },
322		.{ .symbol = 20, .bits = 6, .baseline = 0 },
323		.{ .symbol = 23, .bits = 6, .baseline = 0 },
324		.{ .symbol = 26, .bits = 6, .baseline = 0 },
325		.{ .symbol = 29, .bits = 6, .baseline = 0 },
326		.{ .symbol = 52, .bits = 6, .baseline = 0 },
327		.{ .symbol = 51, .bits = 6, .baseline = 0 },
328		.{ .symbol = 50, .bits = 6, .baseline = 0 },
329		.{ .symbol = 49, .bits = 6, .baseline = 0 },
330		.{ .symbol = 48, .bits = 6, .baseline = 0 },
331		.{ .symbol = 47, .bits = 6, .baseline = 0 },
332		.{ .symbol = 46, .bits = 6, .baseline = 0 },
333		},
334		};
335
336		pub const predefined_offset_fse_table = Table{
337		.fse = &[32]Table.Fse{
338		.{ .symbol = 0, .bits = 5, .baseline = 0 },
339		.{ .symbol = 6, .bits = 4, .baseline = 0 },
340		.{ .symbol = 9, .bits = 5, .baseline = 0 },
341		.{ .symbol = 15, .bits = 5, .baseline = 0 },
342		.{ .symbol = 21, .bits = 5, .baseline = 0 },
343		.{ .symbol = 3, .bits = 5, .baseline = 0 },
344		.{ .symbol = 7, .bits = 4, .baseline = 0 },
345		.{ .symbol = 12, .bits = 5, .baseline = 0 },
346		.{ .symbol = 18, .bits = 5, .baseline = 0 },
347		.{ .symbol = 23, .bits = 5, .baseline = 0 },
348		.{ .symbol = 5, .bits = 5, .baseline = 0 },
349		.{ .symbol = 8, .bits = 4, .baseline = 0 },
350		.{ .symbol = 14, .bits = 5, .baseline = 0 },
351		.{ .symbol = 20, .bits = 5, .baseline = 0 },
352		.{ .symbol = 2, .bits = 5, .baseline = 0 },
353		.{ .symbol = 7, .bits = 4, .baseline = 16 },
354		.{ .symbol = 11, .bits = 5, .baseline = 0 },
355		.{ .symbol = 17, .bits = 5, .baseline = 0 },
356		.{ .symbol = 22, .bits = 5, .baseline = 0 },
357		.{ .symbol = 4, .bits = 5, .baseline = 0 },
358		.{ .symbol = 8, .bits = 4, .baseline = 16 },
359		.{ .symbol = 13, .bits = 5, .baseline = 0 },
360		.{ .symbol = 19, .bits = 5, .baseline = 0 },
361		.{ .symbol = 1, .bits = 5, .baseline = 0 },
362		.{ .symbol = 6, .bits = 4, .baseline = 16 },
363		.{ .symbol = 10, .bits = 5, .baseline = 0 },
364		.{ .symbol = 16, .bits = 5, .baseline = 0 },
365		.{ .symbol = 28, .bits = 5, .baseline = 0 },
366		.{ .symbol = 27, .bits = 5, .baseline = 0 },
367		.{ .symbol = 26, .bits = 5, .baseline = 0 },
368		.{ .symbol = 25, .bits = 5, .baseline = 0 },
369		.{ .symbol = 24, .bits = 5, .baseline = 0 },
370		},
371		};
372		pub const start_repeated_offset_1 = 1;
373		pub const start_repeated_offset_2 = 4;
374		pub const start_repeated_offset_3 = 8;
375
376		pub const table_accuracy_log_max = struct {
377		pub const literal = 9;
378		pub const match = 9;
379		pub const offset = 8;
380		};
381
382		pub const table_symbol_count_max = struct {
383		pub const literal = 36;
384		pub const match = 53;
385		pub const offset = 32;
386		};
387
388		pub const default_accuracy_log = struct {
389		pub const literal = 6;
390		pub const match = 6;
391		pub const offset = 5;
392		};
393		pub const table_size_max = struct {
394		pub const literal = 1 << table_accuracy_log_max.literal;
395		pub const match = 1 << table_accuracy_log_max.match;
396		pub const offset = 1 << table_accuracy_log_max.offset;
397		};
398		};
399
400		test {
401		const testing = @import("std").testing;
402		testing.refAllDeclsRecursive(@This());
403		}

lib/std/compress/zstd.zig created+153

...	...	@@ -0,0 +1,153 @@
	1	const std = @import("../std.zig");
	2	const assert = std.debug.assert;
	3
	4	/// Recommended amount by the standard. Lower than this may result in inability
	5	/// to decompress common streams.
	6	pub const default_window_len = 8 * 1024 * 1024;
	7
	8	pub const Decompress = @import("zstd/Decompress.zig");
	9
	10	pub const block_size_max = 1 << 17;
	11
	12	pub const literals_length_default_distribution = [36]i16{
	13	4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
	14	2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
	15	-1, -1, -1, -1,
	16	};
	17
	18	pub const match_lengths_default_distribution = [53]i16{
	19	1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
	20	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	21	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
	22	-1, -1, -1, -1, -1,
	23	};
	24
	25	pub const offset_codes_default_distribution = [29]i16{
	26	1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
	27	1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1,
	28	};
	29
	30	pub const start_repeated_offset_1 = 1;
	31	pub const start_repeated_offset_2 = 4;
	32	pub const start_repeated_offset_3 = 8;
	33
	34	pub const literals_length_code_table = [36]struct { u32, u5 }{
	35	.{ 0, 0 }, .{ 1, 0 }, .{ 2, 0 }, .{ 3, 0 },
	36	.{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 },
	37	.{ 8, 0 }, .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 },
	38	.{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 }, .{ 15, 0 },
	39	.{ 16, 1 }, .{ 18, 1 }, .{ 20, 1 }, .{ 22, 1 },
	40	.{ 24, 2 }, .{ 28, 2 }, .{ 32, 3 }, .{ 40, 3 },
	41	.{ 48, 4 }, .{ 64, 6 }, .{ 128, 7 }, .{ 256, 8 },
	42	.{ 512, 9 }, .{ 1024, 10 }, .{ 2048, 11 }, .{ 4096, 12 },
	43	.{ 8192, 13 }, .{ 16384, 14 }, .{ 32768, 15 }, .{ 65536, 16 },
	44	};
	45
	46	pub const match_length_code_table = [53]struct { u32, u5 }{
	47	.{ 3, 0 }, .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 }, .{ 8, 0 },
	48	.{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 }, .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 },
	49	.{ 15, 0 }, .{ 16, 0 }, .{ 17, 0 }, .{ 18, 0 }, .{ 19, 0 }, .{ 20, 0 },
	50	.{ 21, 0 }, .{ 22, 0 }, .{ 23, 0 }, .{ 24, 0 }, .{ 25, 0 }, .{ 26, 0 },
	51	.{ 27, 0 }, .{ 28, 0 }, .{ 29, 0 }, .{ 30, 0 }, .{ 31, 0 }, .{ 32, 0 },
	52	.{ 33, 0 }, .{ 34, 0 }, .{ 35, 1 }, .{ 37, 1 }, .{ 39, 1 }, .{ 41, 1 },
	53	.{ 43, 2 }, .{ 47, 2 }, .{ 51, 3 }, .{ 59, 3 }, .{ 67, 4 }, .{ 83, 4 },
	54	.{ 99, 5 }, .{ 131, 7 }, .{ 259, 8 }, .{ 515, 9 }, .{ 1027, 10 }, .{ 2051, 11 },
	55	.{ 4099, 12 }, .{ 8195, 13 }, .{ 16387, 14 }, .{ 32771, 15 }, .{ 65539, 16 },
	56	};
	57
	58	pub const table_accuracy_log_max = struct {
	59	pub const literal = 9;
	60	pub const match = 9;
	61	pub const offset = 8;
	62	};
	63
	64	pub const table_symbol_count_max = struct {
	65	pub const literal = 36;
	66	pub const match = 53;
	67	pub const offset = 32;
	68	};
	69
	70	pub const default_accuracy_log = struct {
	71	pub const literal = 6;
	72	pub const match = 6;
	73	pub const offset = 5;
	74	};
	75	pub const table_size_max = struct {
	76	pub const literal = 1 << table_accuracy_log_max.literal;
	77	pub const match = 1 << table_accuracy_log_max.match;
	78	pub const offset = 1 << table_accuracy_log_max.offset;
	79	};
	80
	81	fn testDecompress(gpa: std.mem.Allocator, compressed: []const u8) ![]u8 {
	82	var out: std.ArrayListUnmanaged(u8) = .empty;
	83	defer out.deinit(gpa);
	84	try out.ensureUnusedCapacity(gpa, default_window_len);
	85
	86	var in: std.io.Reader = .fixed(compressed);
	87	var zstd_stream: Decompress = .init(&in, &.{}, .{});
	88	try zstd_stream.reader.appendRemaining(gpa, null, &out, .unlimited);
	89
	90	return out.toOwnedSlice(gpa);
	91	}
	92
	93	fn testExpectDecompress(uncompressed: []const u8, compressed: []const u8) !void {
	94	const gpa = std.testing.allocator;
	95	const result = try testDecompress(gpa, compressed);
	96	defer gpa.free(result);
	97	try std.testing.expectEqualSlices(u8, uncompressed, result);
	98	}
	99
	100	fn testExpectDecompressError(err: anyerror, compressed: []const u8) !void {
	101	const gpa = std.testing.allocator;
	102
	103	var out: std.ArrayListUnmanaged(u8) = .empty;
	104	defer out.deinit(gpa);
	105	try out.ensureUnusedCapacity(gpa, default_window_len);
	106
	107	var in: std.io.Reader = .fixed(compressed);
	108	var zstd_stream: Decompress = .init(&in, &.{}, .{});
	109	try std.testing.expectError(
	110	error.ReadFailed,
	111	zstd_stream.reader.appendRemaining(gpa, null, &out, .unlimited),
	112	);
	113	try std.testing.expectError(err, zstd_stream.err orelse {});
	114	}
	115
	116	test Decompress {
	117	const uncompressed = @embedFile("testdata/rfc8478.txt");
	118	const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
	119	const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
	120
	121	try testExpectDecompress(uncompressed, compressed3);
	122	try testExpectDecompress(uncompressed, compressed19);
	123	}
	124
	125	test "zero sized raw block" {
	126	const input_raw =
	127	"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
	128	"\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
	129	"\x01\x00\x00"; // block header with: last_block set, block_type raw, block_size zero
	130	try testExpectDecompress("", input_raw);
	131	}
	132
	133	test "zero sized rle block" {
	134	const input_rle =
	135	"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
	136	"\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
	137	"\x03\x00\x00" ++ // block header with: last_block set, block_type rle, block_size zero
	138	"\xaa"; // block_content
	139	try testExpectDecompress("", input_rle);
	140	}
	141
	142	test "declared raw literals size too large" {
	143	const input_raw =
	144	"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
	145	"\x00\x00" ++ // frame header: everything unset, window descriptor zero
	146	"\x95\x00\x00" ++ // block header with: last_block set, block_type compressed, block_size 18
	147	"\xbc\xf3\xae" ++ // literals section header with: type raw, size_format 3, regenerated_size 716603
	148	"\xa5\x9f\xe3"; // some bytes of literal content - the content is shorter than regenerated_size
	149
	150	// Note that the regenerated_size in the above input is larger than block maximum size, so the
	151	// block can't be valid as it is a raw literals block.
	152	try testExpectDecompressError(error.MalformedLiteralsSection, input_raw);
	153	}

lib/std/compress/zstd/Decompress.zig created+1819

...	...	@@ -0,0 +1,1819 @@
	1	const Decompress = @This();
	2	const std = @import("std");
	3	const assert = std.debug.assert;
	4	const Reader = std.io.Reader;
	5	const Limit = std.io.Limit;
	6	const zstd = @import("../zstd.zig");
	7	const Writer = std.io.Writer;
	8
	9	input: *Reader,
	10	reader: Reader,
	11	state: State,
	12	verify_checksum: bool,
	13	err: ?Error = null,
	14
	15	const State = union(enum) {
	16	new_frame,
	17	in_frame: InFrame,
	18	skipping_frame: usize,
	19	end,
	20
	21	const InFrame = struct {
	22	frame: Frame,
	23	checksum: ?u32,
	24	decompressed_size: usize,
	25	};
	26	};
	27
	28	pub const Options = struct {
	29	/// Verifying checksums is not implemented yet and will cause a panic if
	30	/// you set this to true.
	31	verify_checksum: bool = false,
	32	};
	33
	34	pub const Error = error{
	35	BadMagic,
	36	BlockOversize,
	37	ChecksumFailure,
	38	ContentOversize,
	39	DictionaryIdFlagUnsupported,
	40	EndOfStream,
	41	HuffmanTreeIncomplete,
	42	InvalidBitStream,
	43	MalformedAccuracyLog,
	44	MalformedBlock,
	45	MalformedCompressedBlock,
	46	MalformedFrame,
	47	MalformedFseBits,
	48	MalformedFseTable,
	49	MalformedHuffmanTree,
	50	MalformedLiteralsHeader,
	51	MalformedLiteralsLength,
	52	MalformedLiteralsSection,
	53	MalformedSequence,
	54	MissingStartBit,
	55	OutputBufferUndersize,
	56	InputBufferUndersize,
	57	ReadFailed,
	58	RepeatModeFirst,
	59	ReservedBitSet,
	60	ReservedBlock,
	61	SequenceBufferUndersize,
	62	TreelessLiteralsFirst,
	63	UnexpectedEndOfLiteralStream,
	64	WindowOversize,
	65	WindowSizeUnknown,
	66	};
	67
	68	pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress {
	69	return .{
	70	.input = input,
	71	.state = .new_frame,
	72	.verify_checksum = options.verify_checksum,
	73	.reader = .{
	74	.vtable = &.{ .stream = stream },
	75	.buffer = buffer,
	76	.seek = 0,
	77	.end = 0,
	78	},
	79	};
	80	}
	81
	82	fn stream(r: Reader, w: Writer, limit: Limit) Reader.StreamError!usize {
	83	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
	84	const in = d.input;
	85
	86	switch (d.state) {
	87	.new_frame => {
	88	// Allow error.EndOfStream only on the frame magic.
	89	const magic = try in.takeEnumNonexhaustive(Frame.Magic, .little);
	90	initFrame(d, w.buffer.len, magic) catch \|err\| {
	91	d.err = err;
	92	return error.ReadFailed;
	93	};
	94	return readInFrame(d, w, limit, &d.state.in_frame) catch \|err\| switch (err) {
	95	error.ReadFailed => return error.ReadFailed,
	96	error.WriteFailed => return error.WriteFailed,
	97	else => \|e\| {
	98	d.err = e;
	99	return error.ReadFailed;
	100	},
	101	};
	102	},
	103	.in_frame => \|*in_frame\| {
	104	return readInFrame(d, w, limit, in_frame) catch \|err\| switch (err) {
	105	error.ReadFailed => return error.ReadFailed,
	106	error.WriteFailed => return error.WriteFailed,
	107	else => \|e\| {
	108	d.err = e;
	109	return error.ReadFailed;
	110	},
	111	};
	112	},
	113	.skipping_frame => \|*remaining\| {
	114	const n = in.discard(.limited(remaining.*)) catch \|err\| {
	115	d.err = err;
	116	return error.ReadFailed;
	117	};
	118	remaining.* -= n;
	119	if (remaining.* == 0) d.state = .new_frame;
	120	return 0;
	121	},
	122	.end => return error.EndOfStream,
	123	}
	124	}
	125
	126	fn initFrame(d: *Decompress, window_size_max: usize, magic: Frame.Magic) !void {
	127	const in = d.input;
	128	switch (magic.kind() orelse return error.BadMagic) {
	129	.zstandard => {
	130	const header = try Frame.Zstandard.Header.decode(in);
	131	d.state = .{ .in_frame = .{
	132	.frame = try Frame.init(header, window_size_max, d.verify_checksum),
	133	.checksum = null,
	134	.decompressed_size = 0,
	135	} };
	136	},
	137	.skippable => {
	138	const frame_size = try in.takeInt(u32, .little);
	139	d.state = .{ .skipping_frame = frame_size };
	140	},
	141	}
	142	}
	143
	144	fn readInFrame(d: Decompress, w: Writer, limit: Limit, state: *State.InFrame) !usize {
	145	const in = d.input;
	146
	147	const header_bytes = try in.takeArray(3);
	148	const block_header: Frame.Zstandard.Block.Header = @bitCast(header_bytes.*);
	149	const block_size = block_header.size;
	150	const frame_block_size_max = state.frame.block_size_max;
	151	if (frame_block_size_max < block_size) return error.BlockOversize;
	152	if (@intFromEnum(limit) < block_size) return error.OutputBufferUndersize;
	153	var bytes_written: usize = 0;
	154	switch (block_header.type) {
	155	.raw => {
	156	try in.streamExact(w, block_size);
	157	bytes_written = block_size;
	158	},
	159	.rle => {
	160	const byte = try in.takeByte();
	161	try w.splatByteAll(byte, block_size);
	162	bytes_written = block_size;
	163	},
	164	.compressed => {
	165	var literal_fse_buffer: [zstd.table_size_max.literal]Table.Fse = undefined;
	166	var match_fse_buffer: [zstd.table_size_max.match]Table.Fse = undefined;
	167	var offset_fse_buffer: [zstd.table_size_max.offset]Table.Fse = undefined;
	168	var literals_buffer: [zstd.block_size_max]u8 = undefined;
	169	var sequence_buffer: [zstd.block_size_max]u8 = undefined;
	170	var decode: Frame.Zstandard.Decode = .init(&literal_fse_buffer, &match_fse_buffer, &offset_fse_buffer);
	171	var remaining: Limit = .limited(block_size);
	172	const literals = try LiteralsSection.decode(in, &remaining, &literals_buffer);
	173	const sequences_header = try SequencesSection.Header.decode(in, &remaining);
	174
	175	try decode.prepare(in, &remaining, literals, sequences_header);
	176
	177	{
	178	if (sequence_buffer.len < @intFromEnum(remaining))
	179	return error.SequenceBufferUndersize;
	180	const seq_slice = remaining.slice(&sequence_buffer);
	181	try in.readSliceAll(seq_slice);
	182	var bit_stream = try ReverseBitReader.init(seq_slice);
	183
	184	if (sequences_header.sequence_count > 0) {
	185	try decode.readInitialFseState(&bit_stream);
	186
	187	// Ensures the following calls to `decodeSequence` will not flush.
	188	if (frame_block_size_max > w.buffer.len) return error.OutputBufferUndersize;
	189	const dest = (try w.writableSliceGreedy(frame_block_size_max))[0..frame_block_size_max];
	190	for (0..sequences_header.sequence_count - 1) \|_\| {
	191	bytes_written += try decode.decodeSequence(dest, bytes_written, &bit_stream);
	192	try decode.updateState(.literal, &bit_stream);
	193	try decode.updateState(.match, &bit_stream);
	194	try decode.updateState(.offset, &bit_stream);
	195	}
	196	bytes_written += try decode.decodeSequence(dest, bytes_written, &bit_stream);
	197	if (bytes_written > dest.len) return error.MalformedSequence;
	198	w.advance(bytes_written);
	199	}
	200
	201	if (!bit_stream.isEmpty()) {
	202	return error.MalformedCompressedBlock;
	203	}
	204	}
	205
	206	if (decode.literal_written_count < literals.header.regenerated_size) {
	207	const len = literals.header.regenerated_size - decode.literal_written_count;
	208	try decode.decodeLiterals(w, len);
	209	decode.literal_written_count += len;
	210	bytes_written += len;
	211	}
	212
	213	switch (decode.literal_header.block_type) {
	214	.treeless, .compressed => {
	215	if (!decode.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
	216	},
	217	.raw, .rle => {},
	218	}
	219
	220	if (bytes_written > frame_block_size_max) return error.BlockOversize;
	221
	222	state.decompressed_size += bytes_written;
	223	if (state.frame.content_size) \|size\| {
	224	if (state.decompressed_size > size) return error.MalformedFrame;
	225	}
	226	},
	227	.reserved => return error.ReservedBlock,
	228	}
	229
	230	if (state.frame.hasher_opt) \|*hasher\| {
	231	if (bytes_written > 0) {
	232	_ = hasher;
	233	@panic("TODO all those bytes written needed to go through the hasher too");
	234	}
	235	}
	236
	237	if (block_header.last) {
	238	if (state.frame.has_checksum) {
	239	const expected_checksum = try in.takeInt(u32, .little);
	240	if (state.frame.hasher_opt) \|*hasher\| {
	241	const actual_checksum: u32 = @truncate(hasher.final());
	242	if (expected_checksum != actual_checksum) return error.ChecksumFailure;
	243	}
	244	}
	245	if (state.frame.content_size) \|content_size\| {
	246	if (content_size != state.decompressed_size) {
	247	return error.MalformedFrame;
	248	}
	249	}
	250	d.state = .new_frame;
	251	}
	252
	253	return bytes_written;
	254	}
	255
	256	pub const Frame = struct {
	257	hasher_opt: ?std.hash.XxHash64,
	258	window_size: usize,
	259	has_checksum: bool,
	260	block_size_max: usize,
	261	content_size: ?usize,
	262
	263	pub const Magic = enum(u32) {
	264	zstandard = 0xFD2FB528,
	265	_,
	266
	267	pub fn kind(m: Magic) ?Kind {
	268	return switch (@intFromEnum(m)) {
	269	@intFromEnum(Magic.zstandard) => .zstandard,
	270	@intFromEnum(Skippable.magic_min)...@intFromEnum(Skippable.magic_max) => .skippable,
	271	else => null,
	272	};
	273	}
	274
	275	pub fn isSkippable(m: Magic) bool {
	276	return switch (@intFromEnum(m)) {
	277	@intFromEnum(Skippable.magic_min)...@intFromEnum(Skippable.magic_max) => true,
	278	else => false,
	279	};
	280	}
	281	};
	282
	283	pub const Kind = enum { zstandard, skippable };
	284
	285	pub const Zstandard = struct {
	286	pub const magic: Magic = .zstandard;
	287
	288	header: Header,
	289	data_blocks: []Block,
	290	checksum: ?u32,
	291
	292	pub const Header = struct {
	293	descriptor: Descriptor,
	294	window_descriptor: ?u8,
	295	dictionary_id: ?u32,
	296	content_size: ?u64,
	297
	298	pub const Descriptor = packed struct {
	299	dictionary_id_flag: u2,
	300	content_checksum_flag: bool,
	301	reserved: bool,
	302	unused: bool,
	303	single_segment_flag: bool,
	304	content_size_flag: u2,
	305	};
	306
	307	pub const DecodeError = Reader.Error \|\| error{ReservedBitSet};
	308
	309	pub fn decode(in: *Reader) DecodeError!Header {
	310	const descriptor: Descriptor = @bitCast(try in.takeByte());
	311
	312	if (descriptor.reserved) return error.ReservedBitSet;
	313
	314	const window_descriptor: ?u8 = if (descriptor.single_segment_flag) null else try in.takeByte();
	315
	316	const dictionary_id: ?u32 = if (descriptor.dictionary_id_flag > 0) d: {
	317	// if flag is 3 then field_size = 4, else field_size = flag
	318	const field_size = (@as(u4, 1) << descriptor.dictionary_id_flag) >> 1;
	319	break :d try in.takeVarInt(u32, .little, field_size);
	320	} else null;
	321
	322	const content_size: ?u64 = if (descriptor.single_segment_flag or descriptor.content_size_flag > 0) c: {
	323	const field_size = @as(u4, 1) << descriptor.content_size_flag;
	324	const content_size = try in.takeVarInt(u64, .little, field_size);
	325	break :c if (field_size == 2) content_size + 256 else content_size;
	326	} else null;
	327
	328	return .{
	329	.descriptor = descriptor,
	330	.window_descriptor = window_descriptor,
	331	.dictionary_id = dictionary_id,
	332	.content_size = content_size,
	333	};
	334	}
	335
	336	/// Returns the window size required to decompress a frame, or `null` if it
	337	/// cannot be determined (which indicates a malformed frame header).
	338	pub fn windowSize(header: Header) ?u64 {
	339	if (header.window_descriptor) \|descriptor\| {
	340	const exponent = (descriptor & 0b11111000) >> 3;
	341	const mantissa = descriptor & 0b00000111;
	342	const window_log = 10 + exponent;
	343	const window_base = @as(u64, 1) << @as(u6, @intCast(window_log));
	344	const window_add = (window_base / 8) * mantissa;
	345	return window_base + window_add;
	346	} else return header.content_size;
	347	}
	348	};
	349
	350	pub const Block = struct {
	351	pub const Header = packed struct(u24) {
	352	last: bool,
	353	type: Type,
	354	size: u21,
	355	};
	356
	357	pub const Type = enum(u2) {
	358	raw,
	359	rle,
	360	compressed,
	361	reserved,
	362	};
	363	};
	364
	365	pub const Decode = struct {
	366	repeat_offsets: [3]u32,
	367
	368	offset: StateData(8),
	369	match: StateData(9),
	370	literal: StateData(9),
	371
	372	offset_fse_buffer: []Table.Fse,
	373	match_fse_buffer: []Table.Fse,
	374	literal_fse_buffer: []Table.Fse,
	375
	376	fse_tables_undefined: bool,
	377
	378	literal_stream_reader: ReverseBitReader,
	379	literal_stream_index: usize,
	380	literal_streams: LiteralsSection.Streams,
	381	literal_header: LiteralsSection.Header,
	382	huffman_tree: ?LiteralsSection.HuffmanTree,
	383
	384	literal_written_count: usize,
	385
	386	fn StateData(comptime max_accuracy_log: comptime_int) type {
	387	return struct {
	388	state: @This().State,
	389	table: Table,
	390	accuracy_log: u8,
	391
	392	const State = std.meta.Int(.unsigned, max_accuracy_log);
	393	};
	394	}
	395
	396	pub fn init(
	397	literal_fse_buffer: []Table.Fse,
	398	match_fse_buffer: []Table.Fse,
	399	offset_fse_buffer: []Table.Fse,
	400	) Decode {
	401	return .{
	402	.repeat_offsets = .{
	403	zstd.start_repeated_offset_1,
	404	zstd.start_repeated_offset_2,
	405	zstd.start_repeated_offset_3,
	406	},
	407
	408	.offset = undefined,
	409	.match = undefined,
	410	.literal = undefined,
	411
	412	.literal_fse_buffer = literal_fse_buffer,
	413	.match_fse_buffer = match_fse_buffer,
	414	.offset_fse_buffer = offset_fse_buffer,
	415
	416	.fse_tables_undefined = true,
	417
	418	.literal_written_count = 0,
	419	.literal_header = undefined,
	420	.literal_streams = undefined,
	421	.literal_stream_reader = undefined,
	422	.literal_stream_index = undefined,
	423	.huffman_tree = null,
	424	};
	425	}
	426
	427	pub const PrepareError = error{
	428	/// the (reversed) literal bitstream's first byte does not have any bits set
	429	MissingStartBit,
	430	/// `literals` is a treeless literals section and the decode state does not
	431	/// have a Huffman tree from a previous block
	432	TreelessLiteralsFirst,
	433	/// on the first call if one of the sequence FSE tables is set to repeat mode
	434	RepeatModeFirst,
	435	/// an FSE table has an invalid accuracy
	436	MalformedAccuracyLog,
	437	/// failed decoding an FSE table
	438	MalformedFseTable,
	439	/// input stream ends before all FSE tables are read
	440	EndOfStream,
	441	ReadFailed,
	442	InputBufferUndersize,
	443	};
	444
	445	/// Prepare the decoder to decode a compressed block. Loads the
	446	/// literals stream and Huffman tree from `literals` and reads the
	447	/// FSE tables from `in`.
	448	pub fn prepare(
	449	self: *Decode,
	450	in: *Reader,
	451	remaining: *Limit,
	452	literals: LiteralsSection,
	453	sequences_header: SequencesSection.Header,
	454	) PrepareError!void {
	455	self.literal_written_count = 0;
	456	self.literal_header = literals.header;
	457	self.literal_streams = literals.streams;
	458
	459	if (literals.huffman_tree) \|tree\| {
	460	self.huffman_tree = tree;
	461	} else if (literals.header.block_type == .treeless and self.huffman_tree == null) {
	462	return error.TreelessLiteralsFirst;
	463	}
	464
	465	switch (literals.header.block_type) {
	466	.raw, .rle => {},
	467	.compressed, .treeless => {
	468	self.literal_stream_index = 0;
	469	switch (literals.streams) {
	470	.one => \|slice\| try self.initLiteralStream(slice),
	471	.four => \|streams\| try self.initLiteralStream(streams[0]),
	472	}
	473	},
	474	}
	475
	476	if (sequences_header.sequence_count > 0) {
	477	try self.updateFseTable(in, remaining, .literal, sequences_header.literal_lengths);
	478	try self.updateFseTable(in, remaining, .offset, sequences_header.offsets);
	479	try self.updateFseTable(in, remaining, .match, sequences_header.match_lengths);
	480	self.fse_tables_undefined = false;
	481	}
	482	}
	483
	484	/// Read initial FSE states for sequence decoding.
	485	pub fn readInitialFseState(self: Decode, bit_reader: ReverseBitReader) error{EndOfStream}!void {
	486	self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
	487	self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
	488	self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log);
	489	}
	490
	491	fn updateRepeatOffset(self: *Decode, offset: u32) void {
	492	self.repeat_offsets[2] = self.repeat_offsets[1];
	493	self.repeat_offsets[1] = self.repeat_offsets[0];
	494	self.repeat_offsets[0] = offset;
	495	}
	496
	497	fn useRepeatOffset(self: *Decode, index: usize) u32 {
	498	if (index == 1)
	499	std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[1])
	500	else if (index == 2) {
	501	std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[2]);
	502	std.mem.swap(u32, &self.repeat_offsets[1], &self.repeat_offsets[2]);
	503	}
	504	return self.repeat_offsets[0];
	505	}
	506
	507	const DataType = enum { offset, match, literal };
	508
	509	/// TODO: don't use `@field`
	510	fn updateState(
	511	self: *Decode,
	512	comptime choice: DataType,
	513	bit_reader: *ReverseBitReader,
	514	) error{ MalformedFseBits, EndOfStream }!void {
	515	switch (@field(self, @tagName(choice)).table) {
	516	.rle => {},
	517	.fse => \|table\| {
	518	const data = table[@field(self, @tagName(choice)).state];
	519	const T = @TypeOf(@field(self, @tagName(choice))).State;
	520	const bits_summand = try bit_reader.readBitsNoEof(T, data.bits);
	521	const next_state = std.math.cast(
	522	@TypeOf(@field(self, @tagName(choice))).State,
	523	data.baseline + bits_summand,
	524	) orelse return error.MalformedFseBits;
	525	@field(self, @tagName(choice)).state = next_state;
	526	},
	527	}
	528	}
	529
	530	const FseTableError = error{
	531	MalformedFseTable,
	532	MalformedAccuracyLog,
	533	RepeatModeFirst,
	534	EndOfStream,
	535	};
	536
	537	/// TODO: don't use `@field`
	538	fn updateFseTable(
	539	self: *Decode,
	540	in: *Reader,
	541	remaining: *Limit,
	542	comptime choice: DataType,
	543	mode: SequencesSection.Header.Mode,
	544	) !void {
	545	const field_name = @tagName(choice);
	546	switch (mode) {
	547	.predefined => {
	548	@field(self, field_name).accuracy_log =
	549	@field(zstd.default_accuracy_log, field_name);
	550
	551	@field(self, field_name).table =
	552	@field(Table, "predefined_" ++ field_name);
	553	},
	554	.rle => {
	555	@field(self, field_name).accuracy_log = 0;
	556	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
	557	@field(self, field_name).table = .{ .rle = try in.takeByte() };
	558	},
	559	.fse => {
	560	const max_table_size = 2048;
	561	const peek_len: usize = remaining.minInt(max_table_size);
	562	if (in.buffer.len < peek_len) return error.InputBufferUndersize;
	563	const limited_buffer = try in.peek(peek_len);
	564	var bit_reader: BitReader = .{ .bytes = limited_buffer };
	565	const table_size = try Table.decode(
	566	&bit_reader,
	567	@field(zstd.table_symbol_count_max, field_name),
	568	@field(zstd.table_accuracy_log_max, field_name),
	569	@field(self, field_name ++ "_fse_buffer"),
	570	);
	571	@field(self, field_name).table = .{
	572	.fse = @field(self, field_name ++ "_fse_buffer")[0..table_size],
	573	};
	574	@field(self, field_name).accuracy_log = std.math.log2_int_ceil(usize, table_size);
	575	in.toss(bit_reader.index);
	576	remaining.* = remaining.subtract(bit_reader.index).?;
	577	},
	578	.repeat => if (self.fse_tables_undefined) return error.RepeatModeFirst,
	579	}
	580	}
	581
	582	const Sequence = struct {
	583	literal_length: u32,
	584	match_length: u32,
	585	offset: u32,
	586	};
	587
	588	fn nextSequence(
	589	self: *Decode,
	590	bit_reader: *ReverseBitReader,
	591	) error{ InvalidBitStream, EndOfStream }!Sequence {
	592	const raw_code = self.getCode(.offset);
	593	const offset_code = std.math.cast(u5, raw_code) orelse {
	594	return error.InvalidBitStream;
	595	};
	596	const offset_value = (@as(u32, 1) << offset_code) + try bit_reader.readBitsNoEof(u32, offset_code);
	597
	598	const match_code = self.getCode(.match);
	599	if (match_code >= zstd.match_length_code_table.len)
	600	return error.InvalidBitStream;
	601	const match = zstd.match_length_code_table[match_code];
	602	const match_length = match[0] + try bit_reader.readBitsNoEof(u32, match[1]);
	603
	604	const literal_code = self.getCode(.literal);
	605	if (literal_code >= zstd.literals_length_code_table.len)
	606	return error.InvalidBitStream;
	607	const literal = zstd.literals_length_code_table[literal_code];
	608	const literal_length = literal[0] + try bit_reader.readBitsNoEof(u32, literal[1]);
	609
	610	const offset = if (offset_value > 3) offset: {
	611	const offset = offset_value - 3;
	612	self.updateRepeatOffset(offset);
	613	break :offset offset;
	614	} else offset: {
	615	if (literal_length == 0) {
	616	if (offset_value == 3) {
	617	const offset = self.repeat_offsets[0] - 1;
	618	self.updateRepeatOffset(offset);
	619	break :offset offset;
	620	}
	621	break :offset self.useRepeatOffset(offset_value);
	622	}
	623	break :offset self.useRepeatOffset(offset_value - 1);
	624	};
	625
	626	if (offset == 0) return error.InvalidBitStream;
	627
	628	return .{
	629	.literal_length = literal_length,
	630	.match_length = match_length,
	631	.offset = offset,
	632	};
	633	}
	634
	635	/// Decode one sequence from `bit_reader` into `dest`. Updates FSE states
	636	/// if `last_sequence` is `false`. Assumes `prepare` called for the block
	637	/// before attempting to decode sequences.
	638	fn decodeSequence(
	639	decode: *Decode,
	640	dest: []u8,
	641	write_pos: usize,
	642	bit_reader: *ReverseBitReader,
	643	) !usize {
	644	const sequence = try decode.nextSequence(bit_reader);
	645	const literal_length: usize = sequence.literal_length;
	646	const match_length: usize = sequence.match_length;
	647	const sequence_length = literal_length + match_length;
	648
	649	const copy_start = std.math.sub(usize, write_pos + sequence.literal_length, sequence.offset) catch
	650	return error.MalformedSequence;
	651
	652	if (decode.literal_written_count + literal_length > decode.literal_header.regenerated_size)
	653	return error.MalformedLiteralsLength;
	654	var sub_bw: Writer = .fixed(dest[write_pos..]);
	655	try decodeLiterals(decode, &sub_bw, literal_length);
	656	decode.literal_written_count += literal_length;
	657	// This is not a @memmove; it intentionally repeats patterns
	658	// caused by iterating one byte at a time.
	659	for (
	660	dest[write_pos + literal_length ..][0..match_length],
	661	dest[copy_start..][0..match_length],
	662	) \|d, s\| d. = s;
	663	return sequence_length;
	664	}
	665
	666	fn nextLiteralMultiStream(self: *Decode) error{MissingStartBit}!void {
	667	self.literal_stream_index += 1;
	668	try self.initLiteralStream(self.literal_streams.four[self.literal_stream_index]);
	669	}
	670
	671	fn initLiteralStream(self: *Decode, bytes: []const u8) error{MissingStartBit}!void {
	672	self.literal_stream_reader = try ReverseBitReader.init(bytes);
	673	}
	674
	675	fn isLiteralStreamEmpty(self: *Decode) bool {
	676	switch (self.literal_streams) {
	677	.one => return self.literal_stream_reader.isEmpty(),
	678	.four => return self.literal_stream_index == 3 and self.literal_stream_reader.isEmpty(),
	679	}
	680	}
	681
	682	const LiteralBitsError = error{
	683	MissingStartBit,
	684	UnexpectedEndOfLiteralStream,
	685	};
	686	fn readLiteralsBits(
	687	self: *Decode,
	688	bit_count_to_read: u16,
	689	) LiteralBitsError!u16 {
	690	return self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch bits: {
	691	if (self.literal_streams == .four and self.literal_stream_index < 3) {
	692	try self.nextLiteralMultiStream();
	693	break :bits self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch
	694	return error.UnexpectedEndOfLiteralStream;
	695	} else {
	696	return error.UnexpectedEndOfLiteralStream;
	697	}
	698	};
	699	}
	700
	701	/// Decode `len` bytes of literals into `dest`.
	702	fn decodeLiterals(self: Decode, dest: Writer, len: usize) !void {
	703	switch (self.literal_header.block_type) {
	704	.raw => {
	705	try dest.writeAll(self.literal_streams.one[self.literal_written_count..][0..len]);
	706	},
	707	.rle => {
	708	try dest.splatByteAll(self.literal_streams.one[0], len);
	709	},
	710	.compressed, .treeless => {
	711	if (len > dest.buffer.len) return error.OutputBufferUndersize;
	712	const buf = try dest.writableSlice(len);
	713	const huffman_tree = self.huffman_tree.?;
	714	const max_bit_count = huffman_tree.max_bit_count;
	715	const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
	716	huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
	717	max_bit_count,
	718	);
	719	var bits_read: u4 = 0;
	720	var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
	721	var bit_count_to_read: u4 = starting_bit_count;
	722	for (buf) \|*out\| {
	723	var prefix: u16 = 0;
	724	while (true) {
	725	const new_bits = try self.readLiteralsBits(bit_count_to_read);
	726	prefix <<= bit_count_to_read;
	727	prefix \|= new_bits;
	728	bits_read += bit_count_to_read;
	729	const result = try huffman_tree.query(huffman_tree_index, prefix);
	730
	731	switch (result) {
	732	.symbol => \|sym\| {
	733	out.* = sym;
	734	bit_count_to_read = starting_bit_count;
	735	bits_read = 0;
	736	huffman_tree_index = huffman_tree.symbol_count_minus_one;
	737	break;
	738	},
	739	.index => \|index\| {
	740	huffman_tree_index = index;
	741	const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
	742	huffman_tree.nodes[index].weight,
	743	max_bit_count,
	744	);
	745	bit_count_to_read = bit_count - bits_read;
	746	},
	747	}
	748	}
	749	}
	750	},
	751	}
	752	}
	753
	754	/// TODO: don't use `@field`
	755	fn getCode(self: *Decode, comptime choice: DataType) u32 {
	756	return switch (@field(self, @tagName(choice)).table) {
	757	.rle => \|value\| value,
	758	.fse => \|table\| table[@field(self, @tagName(choice)).state].symbol,
	759	};
	760	}
	761	};
	762	};
	763
	764	pub const Skippable = struct {
	765	pub const magic_min: Magic = @enumFromInt(0x184D2A50);
	766	pub const magic_max: Magic = @enumFromInt(0x184D2A5F);
	767
	768	pub const Header = struct {
	769	magic_number: u32,
	770	frame_size: u32,
	771	};
	772	};
	773
	774	const InitError = error{
	775	/// Frame uses a dictionary.
	776	DictionaryIdFlagUnsupported,
	777	/// Frame does not have a valid window size.
	778	WindowSizeUnknown,
	779	/// Window size exceeds `window_size_max` or max `usize` value.
	780	WindowOversize,
	781	/// Frame header indicates a content size exceeding max `usize` value.
	782	ContentOversize,
	783	};
	784
	785	/// Validates `frame_header` and returns the associated `Frame`.
	786	pub fn init(
	787	frame_header: Frame.Zstandard.Header,
	788	window_size_max: usize,
	789	verify_checksum: bool,
	790	) InitError!Frame {
	791	if (frame_header.descriptor.dictionary_id_flag != 0)
	792	return error.DictionaryIdFlagUnsupported;
	793
	794	const window_size_raw = frame_header.windowSize() orelse return error.WindowSizeUnknown;
	795	const window_size = if (window_size_raw > window_size_max)
	796	return error.WindowOversize
	797	else
	798	std.math.cast(usize, window_size_raw) orelse return error.WindowOversize;
	799
	800	const should_compute_checksum =
	801	frame_header.descriptor.content_checksum_flag and verify_checksum;
	802
	803	const content_size = if (frame_header.content_size) \|size\|
	804	std.math.cast(usize, size) orelse return error.ContentOversize
	805	else
	806	null;
	807
	808	return .{
	809	.hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,
	810	.window_size = window_size,
	811	.has_checksum = frame_header.descriptor.content_checksum_flag,
	812	.block_size_max = @min(zstd.block_size_max, window_size),
	813	.content_size = content_size,
	814	};
	815	}
	816	};
	817
	818	pub const LiteralsSection = struct {
	819	header: Header,
	820	huffman_tree: ?HuffmanTree,
	821	streams: Streams,
	822
	823	pub const Streams = union(enum) {
	824	one: []const u8,
	825	four: [4][]const u8,
	826
	827	fn decode(size_format: u2, stream_data: []const u8) !Streams {
	828	if (size_format == 0) {
	829	return .{ .one = stream_data };
	830	}
	831
	832	if (stream_data.len < 6) return error.MalformedLiteralsSection;
	833
	834	const stream_1_length: usize = std.mem.readInt(u16, stream_data[0..2], .little);
	835	const stream_2_length: usize = std.mem.readInt(u16, stream_data[2..4], .little);
	836	const stream_3_length: usize = std.mem.readInt(u16, stream_data[4..6], .little);
	837
	838	const stream_1_start = 6;
	839	const stream_2_start = stream_1_start + stream_1_length;
	840	const stream_3_start = stream_2_start + stream_2_length;
	841	const stream_4_start = stream_3_start + stream_3_length;
	842
	843	if (stream_data.len < stream_4_start) return error.MalformedLiteralsSection;
	844
	845	return .{ .four = .{
	846	stream_data[stream_1_start .. stream_1_start + stream_1_length],
	847	stream_data[stream_2_start .. stream_2_start + stream_2_length],
	848	stream_data[stream_3_start .. stream_3_start + stream_3_length],
	849	stream_data[stream_4_start..],
	850	} };
	851	}
	852	};
	853
	854	pub const Header = struct {
	855	block_type: BlockType,
	856	size_format: u2,
	857	regenerated_size: u20,
	858	compressed_size: ?u18,
	859
	860	/// Decode a literals section header.
	861	pub fn decode(in: Reader, remaining: Limit) !Header {
	862	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
	863	const byte0 = try in.takeByte();
	864	const block_type: BlockType = @enumFromInt(byte0 & 0b11);
	865	const size_format: u2 = @intCast((byte0 & 0b1100) >> 2);
	866	var regenerated_size: u20 = undefined;
	867	var compressed_size: ?u18 = null;
	868	switch (block_type) {
	869	.raw, .rle => {
	870	switch (size_format) {
	871	0, 2 => {
	872	regenerated_size = byte0 >> 3;
	873	},
	874	1 => {
	875	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
	876	regenerated_size = (byte0 >> 4) + (@as(u20, try in.takeByte()) << 4);
	877	},
	878	3 => {
	879	remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
	880	regenerated_size = (byte0 >> 4) +
	881	(@as(u20, try in.takeByte()) << 4) +
	882	(@as(u20, try in.takeByte()) << 12);
	883	},
	884	}
	885	},
	886	.compressed, .treeless => {
	887	remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
	888	const byte1 = try in.takeByte();
	889	const byte2 = try in.takeByte();
	890	switch (size_format) {
	891	0, 1 => {
	892	regenerated_size = (byte0 >> 4) + ((@as(u20, byte1) & 0b00111111) << 4);
	893	compressed_size = ((byte1 & 0b11000000) >> 6) + (@as(u18, byte2) << 2);
	894	},
	895	2 => {
	896	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
	897	const byte3 = try in.takeByte();
	898	regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00000011) << 12);
	899	compressed_size = ((byte2 & 0b11111100) >> 2) + (@as(u18, byte3) << 6);
	900	},
	901	3 => {
	902	remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
	903	const byte3 = try in.takeByte();
	904	const byte4 = try in.takeByte();
	905	regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00111111) << 12);
	906	compressed_size = ((byte2 & 0b11000000) >> 6) + (@as(u18, byte3) << 2) + (@as(u18, byte4) << 10);
	907	},
	908	}
	909	},
	910	}
	911	return .{
	912	.block_type = block_type,
	913	.size_format = size_format,
	914	.regenerated_size = regenerated_size,
	915	.compressed_size = compressed_size,
	916	};
	917	}
	918	};
	919
	920	pub const BlockType = enum(u2) {
	921	raw,
	922	rle,
	923	compressed,
	924	treeless,
	925	};
	926
	927	pub const HuffmanTree = struct {
	928	max_bit_count: u4,
	929	symbol_count_minus_one: u8,
	930	nodes: [256]PrefixedSymbol,
	931
	932	pub const PrefixedSymbol = struct {
	933	symbol: u8,
	934	prefix: u16,
	935	weight: u4,
	936	};
	937
	938	pub const Result = union(enum) {
	939	symbol: u8,
	940	index: usize,
	941	};
	942
	943	pub fn query(self: HuffmanTree, index: usize, prefix: u16) error{HuffmanTreeIncomplete}!Result {
	944	var node = self.nodes[index];
	945	const weight = node.weight;
	946	var i: usize = index;
	947	while (node.weight == weight) {
	948	if (node.prefix == prefix) return .{ .symbol = node.symbol };
	949	if (i == 0) return error.HuffmanTreeIncomplete;
	950	i -= 1;
	951	node = self.nodes[i];
	952	}
	953	return .{ .index = i };
	954	}
	955
	956	pub fn weightToBitCount(weight: u4, max_bit_count: u4) u4 {
	957	return if (weight == 0) 0 else ((max_bit_count + 1) - weight);
	958	}
	959
	960	pub const DecodeError = Reader.Error \|\| error{
	961	MalformedHuffmanTree,
	962	MalformedFseTable,
	963	MalformedAccuracyLog,
	964	EndOfStream,
	965	MissingStartBit,
	966	};
	967
	968	pub fn decode(in: Reader, remaining: Limit) HuffmanTree.DecodeError!HuffmanTree {
	969	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
	970	const header = try in.takeByte();
	971	if (header < 128) {
	972	return decodeFse(in, remaining, header);
	973	} else {
	974	return decodeDirect(in, remaining, header - 127);
	975	}
	976	}
	977
	978	fn decodeDirect(
	979	in: *Reader,
	980	remaining: *Limit,
	981	encoded_symbol_count: usize,
	982	) HuffmanTree.DecodeError!HuffmanTree {
	983	var weights: [256]u4 = undefined;
	984	const weights_byte_count = (encoded_symbol_count + 1) / 2;
	985	remaining.* = remaining.subtract(weights_byte_count) orelse return error.EndOfStream;
	986	for (0..weights_byte_count) \|i\| {
	987	const byte = try in.takeByte();
	988	weights[2 * i] = @as(u4, @intCast(byte >> 4));
	989	weights[2 * i + 1] = @as(u4, @intCast(byte & 0xF));
	990	}
	991	const symbol_count = encoded_symbol_count + 1;
	992	return build(&weights, symbol_count);
	993	}
	994
	995	fn decodeFse(
	996	in: *Reader,
	997	remaining: *Limit,
	998	compressed_size: usize,
	999	) HuffmanTree.DecodeError!HuffmanTree {
	1000	var weights: [256]u4 = undefined;
	1001	remaining.* = remaining.subtract(compressed_size) orelse return error.EndOfStream;
	1002	const compressed_buffer = try in.take(compressed_size);
	1003	var bit_reader: BitReader = .{ .bytes = compressed_buffer };
	1004	var entries: [1 << 6]Table.Fse = undefined;
	1005	const table_size = try Table.decode(&bit_reader, 256, 6, &entries);
	1006	const accuracy_log = std.math.log2_int_ceil(usize, table_size);
	1007	const remaining_buffer = bit_reader.bytes[bit_reader.index..];
	1008	const symbol_count = try assignWeights(remaining_buffer, accuracy_log, &entries, &weights);
	1009	return build(&weights, symbol_count);
	1010	}
	1011
	1012	fn assignWeights(
	1013	huff_bits_buffer: []const u8,
	1014	accuracy_log: u16,
	1015	entries: *[1 << 6]Table.Fse,
	1016	weights: *[256]u4,
	1017	) !usize {
	1018	var huff_bits = try ReverseBitReader.init(huff_bits_buffer);
	1019
	1020	var i: usize = 0;
	1021	var even_state: u32 = try huff_bits.readBitsNoEof(u32, accuracy_log);
	1022	var odd_state: u32 = try huff_bits.readBitsNoEof(u32, accuracy_log);
	1023
	1024	while (i < 254) {
	1025	const even_data = entries[even_state];
	1026	var read_bits: u16 = 0;
	1027	const even_bits = huff_bits.readBits(u32, even_data.bits, &read_bits) catch unreachable;
	1028	weights[i] = std.math.cast(u4, even_data.symbol) orelse return error.MalformedHuffmanTree;
	1029	i += 1;
	1030	if (read_bits < even_data.bits) {
	1031	weights[i] = std.math.cast(u4, entries[odd_state].symbol) orelse return error.MalformedHuffmanTree;
	1032	i += 1;
	1033	break;
	1034	}
	1035	even_state = even_data.baseline + even_bits;
	1036
	1037	read_bits = 0;
	1038	const odd_data = entries[odd_state];
	1039	const odd_bits = huff_bits.readBits(u32, odd_data.bits, &read_bits) catch unreachable;
	1040	weights[i] = std.math.cast(u4, odd_data.symbol) orelse return error.MalformedHuffmanTree;
	1041	i += 1;
	1042	if (read_bits < odd_data.bits) {
	1043	if (i == 255) return error.MalformedHuffmanTree;
	1044	weights[i] = std.math.cast(u4, entries[even_state].symbol) orelse return error.MalformedHuffmanTree;
	1045	i += 1;
	1046	break;
	1047	}
	1048	odd_state = odd_data.baseline + odd_bits;
	1049	} else return error.MalformedHuffmanTree;
	1050
	1051	if (!huff_bits.isEmpty()) {
	1052	return error.MalformedHuffmanTree;
	1053	}
	1054
	1055	return i + 1; // stream contains all but the last symbol
	1056	}
	1057
	1058	fn assignSymbols(weight_sorted_prefixed_symbols: []PrefixedSymbol, weights: [256]u4) usize {
	1059	for (0..weight_sorted_prefixed_symbols.len) \|i\| {
	1060	weight_sorted_prefixed_symbols[i] = .{
	1061	.symbol = @as(u8, @intCast(i)),
	1062	.weight = undefined,
	1063	.prefix = undefined,
	1064	};
	1065	}
	1066
	1067	std.mem.sort(
	1068	PrefixedSymbol,
	1069	weight_sorted_prefixed_symbols,
	1070	weights,
	1071	lessThanByWeight,
	1072	);
	1073
	1074	var prefix: u16 = 0;
	1075	var prefixed_symbol_count: usize = 0;
	1076	var sorted_index: usize = 0;
	1077	const symbol_count = weight_sorted_prefixed_symbols.len;
	1078	while (sorted_index < symbol_count) {
	1079	var symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
	1080	const weight = weights[symbol];
	1081	if (weight == 0) {
	1082	sorted_index += 1;
	1083	continue;
	1084	}
	1085
	1086	while (sorted_index < symbol_count) : ({
	1087	sorted_index += 1;
	1088	prefixed_symbol_count += 1;
	1089	prefix += 1;
	1090	}) {
	1091	symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
	1092	if (weights[symbol] != weight) {
	1093	prefix = ((prefix - 1) >> (weights[symbol] - weight)) + 1;
	1094	break;
	1095	}
	1096	weight_sorted_prefixed_symbols[prefixed_symbol_count].symbol = symbol;
	1097	weight_sorted_prefixed_symbols[prefixed_symbol_count].prefix = prefix;
	1098	weight_sorted_prefixed_symbols[prefixed_symbol_count].weight = weight;
	1099	}
	1100	}
	1101	return prefixed_symbol_count;
	1102	}
	1103
	1104	fn build(weights: *[256]u4, symbol_count: usize) error{MalformedHuffmanTree}!HuffmanTree {
	1105	var weight_power_sum_big: u32 = 0;
	1106	for (weights[0 .. symbol_count - 1]) \|value\| {
	1107	weight_power_sum_big += (@as(u16, 1) << value) >> 1;
	1108	}
	1109	if (weight_power_sum_big >= 1 << 11) return error.MalformedHuffmanTree;
	1110	const weight_power_sum = @as(u16, @intCast(weight_power_sum_big));
	1111
	1112	// advance to next power of two (even if weight_power_sum is a power of 2)
	1113	// TODO: is it valid to have weight_power_sum == 0?
	1114	const max_number_of_bits = if (weight_power_sum == 0) 1 else std.math.log2_int(u16, weight_power_sum) + 1;
	1115	const next_power_of_two = @as(u16, 1) << max_number_of_bits;
	1116	weights[symbol_count - 1] = std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1;
	1117
	1118	var weight_sorted_prefixed_symbols: [256]PrefixedSymbol = undefined;
	1119	const prefixed_symbol_count = assignSymbols(weight_sorted_prefixed_symbols[0..symbol_count], weights.*);
	1120	const tree: HuffmanTree = .{
	1121	.max_bit_count = max_number_of_bits,
	1122	.symbol_count_minus_one = @as(u8, @intCast(prefixed_symbol_count - 1)),
	1123	.nodes = weight_sorted_prefixed_symbols,
	1124	};
	1125	return tree;
	1126	}
	1127
	1128	fn lessThanByWeight(
	1129	weights: [256]u4,
	1130	lhs: PrefixedSymbol,
	1131	rhs: PrefixedSymbol,
	1132	) bool {
	1133	// NOTE: this function relies on the use of a stable sorting algorithm,
	1134	// otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs;
	1135	// should be added
	1136	return weights[lhs.symbol] < weights[rhs.symbol];
	1137	}
	1138	};
	1139
	1140	pub const StreamCount = enum { one, four };
	1141	pub fn streamCount(size_format: u2, block_type: BlockType) StreamCount {
	1142	return switch (block_type) {
	1143	.raw, .rle => .one,
	1144	.compressed, .treeless => if (size_format == 0) .one else .four,
	1145	};
	1146	}
	1147
	1148	pub const DecodeError = error{
	1149	/// Invalid header.
	1150	MalformedLiteralsHeader,
	1151	/// Decoding errors.
	1152	MalformedLiteralsSection,
	1153	/// Compressed literals have invalid accuracy.
	1154	MalformedAccuracyLog,
	1155	/// Compressed literals have invalid FSE table.
	1156	MalformedFseTable,
	1157	/// Failed decoding a Huffamn tree.
	1158	MalformedHuffmanTree,
	1159	/// Not enough bytes to complete the section.
	1160	EndOfStream,
	1161	ReadFailed,
	1162	MissingStartBit,
	1163	};
	1164
	1165	pub fn decode(in: Reader, remaining: Limit, buffer: []u8) DecodeError!LiteralsSection {
	1166	const header = try Header.decode(in, remaining);
	1167	switch (header.block_type) {
	1168	.raw => {
	1169	if (buffer.len < header.regenerated_size) return error.MalformedLiteralsSection;
	1170	remaining.* = remaining.subtract(header.regenerated_size) orelse return error.EndOfStream;
	1171	try in.readSliceAll(buffer[0..header.regenerated_size]);
	1172	return .{
	1173	.header = header,
	1174	.huffman_tree = null,
	1175	.streams = .{ .one = buffer },
	1176	};
	1177	},
	1178	.rle => {
	1179	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
	1180	buffer[0] = try in.takeByte();
	1181	return .{
	1182	.header = header,
	1183	.huffman_tree = null,
	1184	.streams = .{ .one = buffer[0..1] },
	1185	};
	1186	},
	1187	.compressed, .treeless => {
	1188	const before_remaining = remaining.*;
	1189	const huffman_tree = if (header.block_type == .compressed)
	1190	try HuffmanTree.decode(in, remaining)
	1191	else
	1192	null;
	1193	const huffman_tree_size = @intFromEnum(before_remaining) - @intFromEnum(remaining.*);
	1194	const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
	1195	return error.MalformedLiteralsSection;
	1196	if (total_streams_size > buffer.len) return error.MalformedLiteralsSection;
	1197	remaining.* = remaining.subtract(total_streams_size) orelse return error.EndOfStream;
	1198	try in.readSliceAll(buffer[0..total_streams_size]);
	1199	const stream_data = buffer[0..total_streams_size];
	1200	const streams = try Streams.decode(header.size_format, stream_data);
	1201	return .{
	1202	.header = header,
	1203	.huffman_tree = huffman_tree,
	1204	.streams = streams,
	1205	};
	1206	},
	1207	}
	1208	}
	1209	};
	1210
	1211	pub const SequencesSection = struct {
	1212	header: Header,
	1213	literals_length_table: Table,
	1214	offset_table: Table,
	1215	match_length_table: Table,
	1216
	1217	pub const Header = struct {
	1218	sequence_count: u24,
	1219	match_lengths: Mode,
	1220	offsets: Mode,
	1221	literal_lengths: Mode,
	1222
	1223	pub const Mode = enum(u2) {
	1224	predefined,
	1225	rle,
	1226	fse,
	1227	repeat,
	1228	};
	1229
	1230	pub const DecodeError = error{
	1231	ReservedBitSet,
	1232	EndOfStream,
	1233	ReadFailed,
	1234	};
	1235
	1236	pub fn decode(in: Reader, remaining: Limit) DecodeError!Header {
	1237	var sequence_count: u24 = undefined;
	1238
	1239	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
	1240	const byte0 = try in.takeByte();
	1241	if (byte0 == 0) {
	1242	return .{
	1243	.sequence_count = 0,
	1244	.offsets = undefined,
	1245	.match_lengths = undefined,
	1246	.literal_lengths = undefined,
	1247	};
	1248	} else if (byte0 < 128) {
	1249	remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
	1250	sequence_count = byte0;
	1251	} else if (byte0 < 255) {
	1252	remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
	1253	sequence_count = (@as(u24, (byte0 - 128)) << 8) + try in.takeByte();
	1254	} else {
	1255	remaining.* = remaining.subtract(3) orelse return error.EndOfStream;
	1256	sequence_count = (try in.takeByte()) + (@as(u24, try in.takeByte()) << 8) + 0x7F00;
	1257	}
	1258
	1259	const compression_modes = try in.takeByte();
	1260
	1261	const matches_mode: Header.Mode = @enumFromInt((compression_modes & 0b00001100) >> 2);
	1262	const offsets_mode: Header.Mode = @enumFromInt((compression_modes & 0b00110000) >> 4);
	1263	const literal_mode: Header.Mode = @enumFromInt((compression_modes & 0b11000000) >> 6);
	1264	if (compression_modes & 0b11 != 0) return error.ReservedBitSet;
	1265
	1266	return .{
	1267	.sequence_count = sequence_count,
	1268	.offsets = offsets_mode,
	1269	.match_lengths = matches_mode,
	1270	.literal_lengths = literal_mode,
	1271	};
	1272	}
	1273	};
	1274	};
	1275
	1276	pub const Table = union(enum) {
	1277	fse: []const Fse,
	1278	rle: u8,
	1279
	1280	pub const Fse = struct {
	1281	symbol: u8,
	1282	baseline: u16,
	1283	bits: u8,
	1284	};
	1285
	1286	pub fn decode(
	1287	bit_reader: *BitReader,
	1288	expected_symbol_count: usize,
	1289	max_accuracy_log: u4,
	1290	entries: []Table.Fse,
	1291	) !usize {
	1292	const accuracy_log_biased = try bit_reader.readBitsNoEof(u4, 4);
	1293	if (accuracy_log_biased > max_accuracy_log -\| 5) return error.MalformedAccuracyLog;
	1294	const accuracy_log = accuracy_log_biased + 5;
	1295
	1296	var values: [256]u16 = undefined;
	1297	var value_count: usize = 0;
	1298
	1299	const total_probability = @as(u16, 1) << accuracy_log;
	1300	var accumulated_probability: u16 = 0;
	1301
	1302	while (accumulated_probability < total_probability) {
	1303	// WARNING: The RFC is poorly worded, and would suggest std.math.log2_int_ceil is correct here,
	1304	// but power of two (remaining probabilities + 1) need max bits set to 1 more.
	1305	const max_bits = std.math.log2_int(u16, total_probability - accumulated_probability + 1) + 1;
	1306	const small = try bit_reader.readBitsNoEof(u16, max_bits - 1);
	1307
	1308	const cutoff = (@as(u16, 1) << max_bits) - 1 - (total_probability - accumulated_probability + 1);
	1309
	1310	const value = if (small < cutoff)
	1311	small
	1312	else value: {
	1313	const value_read = small + (try bit_reader.readBitsNoEof(u16, 1) << (max_bits - 1));
	1314	break :value if (value_read < @as(u16, 1) << (max_bits - 1))
	1315	value_read
	1316	else
	1317	value_read - cutoff;
	1318	};
	1319
	1320	accumulated_probability += if (value != 0) value - 1 else 1;
	1321
	1322	values[value_count] = value;
	1323	value_count += 1;
	1324
	1325	if (value == 1) {
	1326	while (true) {
	1327	const repeat_flag = try bit_reader.readBitsNoEof(u2, 2);
	1328	if (repeat_flag + value_count > 256) return error.MalformedFseTable;
	1329	for (0..repeat_flag) \|_\| {
	1330	values[value_count] = 1;
	1331	value_count += 1;
	1332	}
	1333	if (repeat_flag < 3) break;
	1334	}
	1335	}
	1336	if (value_count == 256) break;
	1337	}
	1338	bit_reader.alignToByte();
	1339
	1340	if (value_count < 2) return error.MalformedFseTable;
	1341	if (accumulated_probability != total_probability) return error.MalformedFseTable;
	1342	if (value_count > expected_symbol_count) return error.MalformedFseTable;
	1343
	1344	const table_size = total_probability;
	1345
	1346	try build(values[0..value_count], entries[0..table_size]);
	1347	return table_size;
	1348	}
	1349
	1350	pub fn build(values: []const u16, entries: []Table.Fse) !void {
	1351	const total_probability = @as(u16, @intCast(entries.len));
	1352	const accuracy_log = std.math.log2_int(u16, total_probability);
	1353	assert(total_probability <= 1 << 9);
	1354
	1355	var less_than_one_count: usize = 0;
	1356	for (values, 0..) \|value, i\| {
	1357	if (value == 0) {
	1358	entries[entries.len - 1 - less_than_one_count] = Table.Fse{
	1359	.symbol = @as(u8, @intCast(i)),
	1360	.baseline = 0,
	1361	.bits = accuracy_log,
	1362	};
	1363	less_than_one_count += 1;
	1364	}
	1365	}
	1366
	1367	var position: usize = 0;
	1368	var temp_states: [1 << 9]u16 = undefined;
	1369	for (values, 0..) \|value, symbol\| {
	1370	if (value == 0 or value == 1) continue;
	1371	const probability = value - 1;
	1372
	1373	const state_share_dividend = std.math.ceilPowerOfTwo(u16, probability) catch
	1374	return error.MalformedFseTable;
	1375	const share_size = @divExact(total_probability, state_share_dividend);
	1376	const double_state_count = state_share_dividend - probability;
	1377	const single_state_count = probability - double_state_count;
	1378	const share_size_log = std.math.log2_int(u16, share_size);
	1379
	1380	for (0..probability) \|i\| {
	1381	temp_states[i] = @as(u16, @intCast(position));
	1382	position += (entries.len >> 1) + (entries.len >> 3) + 3;
	1383	position &= entries.len - 1;
	1384	while (position >= entries.len - less_than_one_count) {
	1385	position += (entries.len >> 1) + (entries.len >> 3) + 3;
	1386	position &= entries.len - 1;
	1387	}
	1388	}
	1389	std.mem.sort(u16, temp_states[0..probability], {}, std.sort.asc(u16));
	1390	for (0..probability) \|i\| {
	1391	entries[temp_states[i]] = if (i < double_state_count) Table.Fse{
	1392	.symbol = @as(u8, @intCast(symbol)),
	1393	.bits = share_size_log + 1,
	1394	.baseline = single_state_count * share_size + @as(u16, @intCast(i)) * 2 * share_size,
	1395	} else Table.Fse{
	1396	.symbol = @as(u8, @intCast(symbol)),
	1397	.bits = share_size_log,
	1398	.baseline = (@as(u16, @intCast(i)) - double_state_count) * share_size,
	1399	};
	1400	}
	1401	}
	1402	}
	1403
	1404	test build {
	1405	const literals_length_default_values = [36]u16{
	1406	5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2,
	1407	3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 2, 2, 2, 2, 2,
	1408	0, 0, 0, 0,
	1409	};
	1410
	1411	const match_lengths_default_values = [53]u16{
	1412	2, 5, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
	1413	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
	1414	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0,
	1415	0, 0, 0, 0, 0,
	1416	};
	1417
	1418	const offset_codes_default_values = [29]u16{
	1419	2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
	1420	2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0,
	1421	};
	1422
	1423	var entries: [64]Table.Fse = undefined;
	1424	try build(&literals_length_default_values, &entries);
	1425	try std.testing.expectEqualSlices(Table.Fse, Table.predefined_literal.fse, &entries);
	1426
	1427	try build(&match_lengths_default_values, &entries);
	1428	try std.testing.expectEqualSlices(Table.Fse, Table.predefined_match.fse, &entries);
	1429
	1430	try build(&offset_codes_default_values, entries[0..32]);
	1431	try std.testing.expectEqualSlices(Table.Fse, Table.predefined_offset.fse, entries[0..32]);
	1432	}
	1433
	1434	pub const predefined_literal: Table = .{
	1435	.fse = &[64]Table.Fse{
	1436	.{ .symbol = 0, .bits = 4, .baseline = 0 },
	1437	.{ .symbol = 0, .bits = 4, .baseline = 16 },
	1438	.{ .symbol = 1, .bits = 5, .baseline = 32 },
	1439	.{ .symbol = 3, .bits = 5, .baseline = 0 },
	1440	.{ .symbol = 4, .bits = 5, .baseline = 0 },
	1441	.{ .symbol = 6, .bits = 5, .baseline = 0 },
	1442	.{ .symbol = 7, .bits = 5, .baseline = 0 },
	1443	.{ .symbol = 9, .bits = 5, .baseline = 0 },
	1444	.{ .symbol = 10, .bits = 5, .baseline = 0 },
	1445	.{ .symbol = 12, .bits = 5, .baseline = 0 },
	1446	.{ .symbol = 14, .bits = 6, .baseline = 0 },
	1447	.{ .symbol = 16, .bits = 5, .baseline = 0 },
	1448	.{ .symbol = 18, .bits = 5, .baseline = 0 },
	1449	.{ .symbol = 19, .bits = 5, .baseline = 0 },
	1450	.{ .symbol = 21, .bits = 5, .baseline = 0 },
	1451	.{ .symbol = 22, .bits = 5, .baseline = 0 },
	1452	.{ .symbol = 24, .bits = 5, .baseline = 0 },
	1453	.{ .symbol = 25, .bits = 5, .baseline = 32 },
	1454	.{ .symbol = 26, .bits = 5, .baseline = 0 },
	1455	.{ .symbol = 27, .bits = 6, .baseline = 0 },
	1456	.{ .symbol = 29, .bits = 6, .baseline = 0 },
	1457	.{ .symbol = 31, .bits = 6, .baseline = 0 },
	1458	.{ .symbol = 0, .bits = 4, .baseline = 32 },
	1459	.{ .symbol = 1, .bits = 4, .baseline = 0 },
	1460	.{ .symbol = 2, .bits = 5, .baseline = 0 },
	1461	.{ .symbol = 4, .bits = 5, .baseline = 32 },
	1462	.{ .symbol = 5, .bits = 5, .baseline = 0 },
	1463	.{ .symbol = 7, .bits = 5, .baseline = 32 },
	1464	.{ .symbol = 8, .bits = 5, .baseline = 0 },
	1465	.{ .symbol = 10, .bits = 5, .baseline = 32 },
	1466	.{ .symbol = 11, .bits = 5, .baseline = 0 },
	1467	.{ .symbol = 13, .bits = 6, .baseline = 0 },
	1468	.{ .symbol = 16, .bits = 5, .baseline = 32 },
	1469	.{ .symbol = 17, .bits = 5, .baseline = 0 },
	1470	.{ .symbol = 19, .bits = 5, .baseline = 32 },
	1471	.{ .symbol = 20, .bits = 5, .baseline = 0 },
	1472	.{ .symbol = 22, .bits = 5, .baseline = 32 },
	1473	.{ .symbol = 23, .bits = 5, .baseline = 0 },
	1474	.{ .symbol = 25, .bits = 4, .baseline = 0 },
	1475	.{ .symbol = 25, .bits = 4, .baseline = 16 },
	1476	.{ .symbol = 26, .bits = 5, .baseline = 32 },
	1477	.{ .symbol = 28, .bits = 6, .baseline = 0 },
	1478	.{ .symbol = 30, .bits = 6, .baseline = 0 },
	1479	.{ .symbol = 0, .bits = 4, .baseline = 48 },
	1480	.{ .symbol = 1, .bits = 4, .baseline = 16 },
	1481	.{ .symbol = 2, .bits = 5, .baseline = 32 },
	1482	.{ .symbol = 3, .bits = 5, .baseline = 32 },
	1483	.{ .symbol = 5, .bits = 5, .baseline = 32 },
	1484	.{ .symbol = 6, .bits = 5, .baseline = 32 },
	1485	.{ .symbol = 8, .bits = 5, .baseline = 32 },
	1486	.{ .symbol = 9, .bits = 5, .baseline = 32 },
	1487	.{ .symbol = 11, .bits = 5, .baseline = 32 },
	1488	.{ .symbol = 12, .bits = 5, .baseline = 32 },
	1489	.{ .symbol = 15, .bits = 6, .baseline = 0 },
	1490	.{ .symbol = 17, .bits = 5, .baseline = 32 },
	1491	.{ .symbol = 18, .bits = 5, .baseline = 32 },
	1492	.{ .symbol = 20, .bits = 5, .baseline = 32 },
	1493	.{ .symbol = 21, .bits = 5, .baseline = 32 },
	1494	.{ .symbol = 23, .bits = 5, .baseline = 32 },
	1495	.{ .symbol = 24, .bits = 5, .baseline = 32 },
	1496	.{ .symbol = 35, .bits = 6, .baseline = 0 },
	1497	.{ .symbol = 34, .bits = 6, .baseline = 0 },
	1498	.{ .symbol = 33, .bits = 6, .baseline = 0 },
	1499	.{ .symbol = 32, .bits = 6, .baseline = 0 },
	1500	},
	1501	};
	1502
	1503	pub const predefined_match: Table = .{
	1504	.fse = &[64]Table.Fse{
	1505	.{ .symbol = 0, .bits = 6, .baseline = 0 },
	1506	.{ .symbol = 1, .bits = 4, .baseline = 0 },
	1507	.{ .symbol = 2, .bits = 5, .baseline = 32 },
	1508	.{ .symbol = 3, .bits = 5, .baseline = 0 },
	1509	.{ .symbol = 5, .bits = 5, .baseline = 0 },
	1510	.{ .symbol = 6, .bits = 5, .baseline = 0 },
	1511	.{ .symbol = 8, .bits = 5, .baseline = 0 },
	1512	.{ .symbol = 10, .bits = 6, .baseline = 0 },
	1513	.{ .symbol = 13, .bits = 6, .baseline = 0 },
	1514	.{ .symbol = 16, .bits = 6, .baseline = 0 },
	1515	.{ .symbol = 19, .bits = 6, .baseline = 0 },
	1516	.{ .symbol = 22, .bits = 6, .baseline = 0 },
	1517	.{ .symbol = 25, .bits = 6, .baseline = 0 },
	1518	.{ .symbol = 28, .bits = 6, .baseline = 0 },
	1519	.{ .symbol = 31, .bits = 6, .baseline = 0 },
	1520	.{ .symbol = 33, .bits = 6, .baseline = 0 },
	1521	.{ .symbol = 35, .bits = 6, .baseline = 0 },
	1522	.{ .symbol = 37, .bits = 6, .baseline = 0 },
	1523	.{ .symbol = 39, .bits = 6, .baseline = 0 },
	1524	.{ .symbol = 41, .bits = 6, .baseline = 0 },
	1525	.{ .symbol = 43, .bits = 6, .baseline = 0 },
	1526	.{ .symbol = 45, .bits = 6, .baseline = 0 },
	1527	.{ .symbol = 1, .bits = 4, .baseline = 16 },
	1528	.{ .symbol = 2, .bits = 4, .baseline = 0 },
	1529	.{ .symbol = 3, .bits = 5, .baseline = 32 },
	1530	.{ .symbol = 4, .bits = 5, .baseline = 0 },
	1531	.{ .symbol = 6, .bits = 5, .baseline = 32 },
	1532	.{ .symbol = 7, .bits = 5, .baseline = 0 },
	1533	.{ .symbol = 9, .bits = 6, .baseline = 0 },
	1534	.{ .symbol = 12, .bits = 6, .baseline = 0 },
	1535	.{ .symbol = 15, .bits = 6, .baseline = 0 },
	1536	.{ .symbol = 18, .bits = 6, .baseline = 0 },
	1537	.{ .symbol = 21, .bits = 6, .baseline = 0 },
	1538	.{ .symbol = 24, .bits = 6, .baseline = 0 },
	1539	.{ .symbol = 27, .bits = 6, .baseline = 0 },
	1540	.{ .symbol = 30, .bits = 6, .baseline = 0 },
	1541	.{ .symbol = 32, .bits = 6, .baseline = 0 },
	1542	.{ .symbol = 34, .bits = 6, .baseline = 0 },
	1543	.{ .symbol = 36, .bits = 6, .baseline = 0 },
	1544	.{ .symbol = 38, .bits = 6, .baseline = 0 },
	1545	.{ .symbol = 40, .bits = 6, .baseline = 0 },
	1546	.{ .symbol = 42, .bits = 6, .baseline = 0 },
	1547	.{ .symbol = 44, .bits = 6, .baseline = 0 },
	1548	.{ .symbol = 1, .bits = 4, .baseline = 32 },
	1549	.{ .symbol = 1, .bits = 4, .baseline = 48 },
	1550	.{ .symbol = 2, .bits = 4, .baseline = 16 },
	1551	.{ .symbol = 4, .bits = 5, .baseline = 32 },
	1552	.{ .symbol = 5, .bits = 5, .baseline = 32 },
	1553	.{ .symbol = 7, .bits = 5, .baseline = 32 },
	1554	.{ .symbol = 8, .bits = 5, .baseline = 32 },
	1555	.{ .symbol = 11, .bits = 6, .baseline = 0 },
	1556	.{ .symbol = 14, .bits = 6, .baseline = 0 },
	1557	.{ .symbol = 17, .bits = 6, .baseline = 0 },
	1558	.{ .symbol = 20, .bits = 6, .baseline = 0 },
	1559	.{ .symbol = 23, .bits = 6, .baseline = 0 },
	1560	.{ .symbol = 26, .bits = 6, .baseline = 0 },
	1561	.{ .symbol = 29, .bits = 6, .baseline = 0 },
	1562	.{ .symbol = 52, .bits = 6, .baseline = 0 },
	1563	.{ .symbol = 51, .bits = 6, .baseline = 0 },
	1564	.{ .symbol = 50, .bits = 6, .baseline = 0 },
	1565	.{ .symbol = 49, .bits = 6, .baseline = 0 },
	1566	.{ .symbol = 48, .bits = 6, .baseline = 0 },
	1567	.{ .symbol = 47, .bits = 6, .baseline = 0 },
	1568	.{ .symbol = 46, .bits = 6, .baseline = 0 },
	1569	},
	1570	};
	1571
	1572	pub const predefined_offset: Table = .{
	1573	.fse = &[32]Table.Fse{
	1574	.{ .symbol = 0, .bits = 5, .baseline = 0 },
	1575	.{ .symbol = 6, .bits = 4, .baseline = 0 },
	1576	.{ .symbol = 9, .bits = 5, .baseline = 0 },
	1577	.{ .symbol = 15, .bits = 5, .baseline = 0 },
	1578	.{ .symbol = 21, .bits = 5, .baseline = 0 },
	1579	.{ .symbol = 3, .bits = 5, .baseline = 0 },
	1580	.{ .symbol = 7, .bits = 4, .baseline = 0 },
	1581	.{ .symbol = 12, .bits = 5, .baseline = 0 },
	1582	.{ .symbol = 18, .bits = 5, .baseline = 0 },
	1583	.{ .symbol = 23, .bits = 5, .baseline = 0 },
	1584	.{ .symbol = 5, .bits = 5, .baseline = 0 },
	1585	.{ .symbol = 8, .bits = 4, .baseline = 0 },
	1586	.{ .symbol = 14, .bits = 5, .baseline = 0 },
	1587	.{ .symbol = 20, .bits = 5, .baseline = 0 },
	1588	.{ .symbol = 2, .bits = 5, .baseline = 0 },
	1589	.{ .symbol = 7, .bits = 4, .baseline = 16 },
	1590	.{ .symbol = 11, .bits = 5, .baseline = 0 },
	1591	.{ .symbol = 17, .bits = 5, .baseline = 0 },
	1592	.{ .symbol = 22, .bits = 5, .baseline = 0 },
	1593	.{ .symbol = 4, .bits = 5, .baseline = 0 },
	1594	.{ .symbol = 8, .bits = 4, .baseline = 16 },
	1595	.{ .symbol = 13, .bits = 5, .baseline = 0 },
	1596	.{ .symbol = 19, .bits = 5, .baseline = 0 },
	1597	.{ .symbol = 1, .bits = 5, .baseline = 0 },
	1598	.{ .symbol = 6, .bits = 4, .baseline = 16 },
	1599	.{ .symbol = 10, .bits = 5, .baseline = 0 },
	1600	.{ .symbol = 16, .bits = 5, .baseline = 0 },
	1601	.{ .symbol = 28, .bits = 5, .baseline = 0 },
	1602	.{ .symbol = 27, .bits = 5, .baseline = 0 },
	1603	.{ .symbol = 26, .bits = 5, .baseline = 0 },
	1604	.{ .symbol = 25, .bits = 5, .baseline = 0 },
	1605	.{ .symbol = 24, .bits = 5, .baseline = 0 },
	1606	},
	1607	};
	1608	};
	1609
	1610	const low_bit_mask = [9]u8{
	1611	0b00000000,
	1612	0b00000001,
	1613	0b00000011,
	1614	0b00000111,
	1615	0b00001111,
	1616	0b00011111,
	1617	0b00111111,
	1618	0b01111111,
	1619	0b11111111,
	1620	};
	1621
	1622	fn Bits(comptime T: type) type {
	1623	return struct { T, u16 };
	1624	}
	1625
	1626	/// For reading the reversed bit streams used to encode FSE compressed data.
	1627	const ReverseBitReader = struct {
	1628	bytes: []const u8,
	1629	remaining: usize,
	1630	bits: u8,
	1631	count: u4,
	1632
	1633	fn init(bytes: []const u8) error{MissingStartBit}!ReverseBitReader {
	1634	var result: ReverseBitReader = .{
	1635	.bytes = bytes,
	1636	.remaining = bytes.len,
	1637	.bits = 0,
	1638	.count = 0,
	1639	};
	1640	if (bytes.len == 0) return result;
	1641	for (0..8) \|_\| if (0 != (result.readBitsNoEof(u1, 1) catch unreachable)) return result;
	1642	return error.MissingStartBit;
	1643	}
	1644
	1645	fn initBits(comptime T: type, out: anytype, num: u16) Bits(T) {
	1646	const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
	1647	return .{
	1648	@bitCast(@as(UT, @intCast(out))),
	1649	num,
	1650	};
	1651	}
	1652
	1653	fn readBitsNoEof(self: *ReverseBitReader, comptime T: type, num: u16) error{EndOfStream}!T {
	1654	const b, const c = try self.readBitsTuple(T, num);
	1655	if (c < num) return error.EndOfStream;
	1656	return b;
	1657	}
	1658
	1659	fn readBits(self: ReverseBitReader, comptime T: type, num: u16, out_bits: u16) !T {
	1660	const b, const c = try self.readBitsTuple(T, num);
	1661	out_bits.* = c;
	1662	return b;
	1663	}
	1664
	1665	fn readBitsTuple(self: *ReverseBitReader, comptime T: type, num: u16) !Bits(T) {
	1666	const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
	1667	const U = if (@bitSizeOf(T) < 8) u8 else UT;
	1668
	1669	if (num <= self.count) return initBits(T, self.removeBits(@intCast(num)), num);
	1670
	1671	var out_count: u16 = self.count;
	1672	var out: U = self.removeBits(self.count);
	1673
	1674	const full_bytes_left = (num - out_count) / 8;
	1675
	1676	for (0..full_bytes_left) \|_\| {
	1677	const byte = takeByte(self) catch \|err\| switch (err) {
	1678	error.EndOfStream => return initBits(T, out, out_count),
	1679	};
	1680	if (U == u8) out = 0 else out <<= 8;
	1681	out \|= byte;
	1682	out_count += 8;
	1683	}
	1684
	1685	const bits_left = num - out_count;
	1686	const keep = 8 - bits_left;
	1687
	1688	if (bits_left == 0) return initBits(T, out, out_count);
	1689
	1690	const final_byte = takeByte(self) catch \|err\| switch (err) {
	1691	error.EndOfStream => return initBits(T, out, out_count),
	1692	};
	1693
	1694	out <<= @intCast(bits_left);
	1695	out \|= final_byte >> @intCast(keep);
	1696	self.bits = final_byte & low_bit_mask[keep];
	1697
	1698	self.count = @intCast(keep);
	1699	return initBits(T, out, num);
	1700	}
	1701
	1702	fn takeByte(rbr: *ReverseBitReader) error{EndOfStream}!u8 {
	1703	if (rbr.remaining == 0) return error.EndOfStream;
	1704	rbr.remaining -= 1;
	1705	return rbr.bytes[rbr.remaining];
	1706	}
	1707
	1708	fn isEmpty(self: *const ReverseBitReader) bool {
	1709	return self.remaining == 0 and self.count == 0;
	1710	}
	1711
	1712	fn removeBits(self: *ReverseBitReader, num: u4) u8 {
	1713	if (num == 8) {
	1714	self.count = 0;
	1715	return self.bits;
	1716	}
	1717
	1718	const keep = self.count - num;
	1719	const bits = self.bits >> @intCast(keep);
	1720	self.bits &= low_bit_mask[keep];
	1721
	1722	self.count = keep;
	1723	return bits;
	1724	}
	1725	};
	1726
	1727	const BitReader = struct {
	1728	bytes: []const u8,
	1729	index: usize = 0,
	1730	bits: u8 = 0,
	1731	count: u4 = 0,
	1732
	1733	fn initBits(comptime T: type, out: anytype, num: u16) Bits(T) {
	1734	const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
	1735	return .{
	1736	@bitCast(@as(UT, @intCast(out))),
	1737	num,
	1738	};
	1739	}
	1740
	1741	fn readBitsNoEof(self: *@This(), comptime T: type, num: u16) !T {
	1742	const b, const c = try self.readBitsTuple(T, num);
	1743	if (c < num) return error.EndOfStream;
	1744	return b;
	1745	}
	1746
	1747	fn readBits(self: @This(), comptime T: type, num: u16, out_bits: u16) !T {
	1748	const b, const c = try self.readBitsTuple(T, num);
	1749	out_bits.* = c;
	1750	return b;
	1751	}
	1752
	1753	fn readBitsTuple(self: *@This(), comptime T: type, num: u16) !Bits(T) {
	1754	const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
	1755	const U = if (@bitSizeOf(T) < 8) u8 else UT;
	1756
	1757	if (num <= self.count) return initBits(T, self.removeBits(@intCast(num)), num);
	1758
	1759	var out_count: u16 = self.count;
	1760	var out: U = self.removeBits(self.count);
	1761
	1762	const full_bytes_left = (num - out_count) / 8;
	1763
	1764	for (0..full_bytes_left) \|_\| {
	1765	const byte = takeByte(self) catch \|err\| switch (err) {
	1766	error.EndOfStream => return initBits(T, out, out_count),
	1767	};
	1768
	1769	const pos = @as(U, byte) << @intCast(out_count);
	1770	out \|= pos;
	1771	out_count += 8;
	1772	}
	1773
	1774	const bits_left = num - out_count;
	1775	const keep = 8 - bits_left;
	1776
	1777	if (bits_left == 0) return initBits(T, out, out_count);
	1778
	1779	const final_byte = takeByte(self) catch \|err\| switch (err) {
	1780	error.EndOfStream => return initBits(T, out, out_count),
	1781	};
	1782
	1783	const pos = @as(U, final_byte & low_bit_mask[bits_left]) << @intCast(out_count);
	1784	out \|= pos;
	1785	self.bits = final_byte >> @intCast(bits_left);
	1786
	1787	self.count = @intCast(keep);
	1788	return initBits(T, out, num);
	1789	}
	1790
	1791	fn takeByte(br: *BitReader) error{EndOfStream}!u8 {
	1792	if (br.bytes.len - br.index == 0) return error.EndOfStream;
	1793	const result = br.bytes[br.index];
	1794	br.index += 1;
	1795	return result;
	1796	}
	1797
	1798	fn removeBits(self: *@This(), num: u4) u8 {
	1799	if (num == 8) {
	1800	self.count = 0;
	1801	return self.bits;
	1802	}
	1803
	1804	const keep = self.count - num;
	1805	const bits = self.bits & low_bit_mask[num];
	1806	self.bits >>= @intCast(num);
	1807	self.count = keep;
	1808	return bits;
	1809	}
	1810
	1811	fn alignToByte(self: *@This()) void {
	1812	self.bits = 0;
	1813	self.count = 0;
	1814	}
	1815	};
	1816
	1817	test {
	1818	_ = Table;
	1819	}

lib/std/http/Server.zig+1-2

...	...	@@ -129,11 +129,10 @@ pub const Request = struct {
129	129	pub const Compression = union(enum) {
130	130	pub const DeflateDecompressor = std.compress.zlib.Decompressor(std.io.AnyReader);
131	131	pub const GzipDecompressor = std.compress.gzip.Decompressor(std.io.AnyReader);
132		pub const ZstdDecompressor = std.compress.zstd.Decompressor(std.io.AnyReader);
133	132
134	133	deflate: DeflateDecompressor,
135	134	gzip: GzipDecompressor,
136		zstd: ZstdDecompressor,
	135	zstd: std.compress.zstd.Decompress,
137	136	none: void,
138	137	};
139	138

src/Package/Fetch.zig+8-9

...	...	@@ -1198,7 +1198,8 @@ fn unpackResource(
1198	1198
1199	1199	switch (file_type) {
1200	1200	.tar => {
1201		var adapter = resource.reader().adaptToNewApi();
	1201	var adapter_buffer: [1024]u8 = undefined;
	1202	var adapter = resource.reader().adaptToNewApi(&adapter_buffer);
1202	1203	return unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
1203	1204	},
1204	1205	.@"tar.gz" => {
...	...	@@ -1225,16 +1226,14 @@ fn unpackResource(
1225	1226	return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
1226	1227	},
1227	1228	.@"tar.zst" => {
1228		const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len;
	1229	const window_size = std.compress.zstd.default_window_len;
1229	1230	const window_buffer = try f.arena.allocator().create([window_size]u8);
1230		const reader = resource.reader();
1231		var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1232		var dcp = std.compress.zstd.decompressor(br.reader(), .{
1233		.window_buffer = window_buffer,
	1231	var adapter_buffer: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined;
	1232	var adapter = resource.reader().adaptToNewApi(&adapter_buffer);
	1233	var decompress: std.compress.zstd.Decompress = .init(&adapter.new_interface, window_buffer, .{
	1234	.verify_checksum = false,
1234	1235	});
1235		var adapter_buffer: [1024]u8 = undefined;
1236		var adapter = dcp.reader().adaptToNewApi(&adapter_buffer);
1237		return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
	1236	return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
1238	1237	},
1239	1238	.git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch \|err\| switch (err) {
1240	1239	error.FetchFailed => return error.FetchFailed,

src/Package/Fetch/git.zig+60-5

...	...	@@ -1281,7 +1281,7 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in
1281	1281	}
1282	1282	@memset(fan_out_table[fan_out_index..], count);
1283	1283
1284		var index_hashed_writer = std.compress.hashedWriter(index_writer, Oid.Hasher.init(format));
	1284	var index_hashed_writer = hashedWriter(index_writer, Oid.Hasher.init(format));
1285	1285	const writer = index_hashed_writer.writer();
1286	1286	try writer.writeAll(IndexHeader.signature);
1287	1287	try writer.writeInt(u32, IndexHeader.supported_version, .big);
...	...	@@ -1331,7 +1331,7 @@ fn indexPackFirstPass(
1331	1331	) !Oid {
1332	1332	var pack_buffered_reader = std.io.bufferedReader(pack.deprecatedReader());
1333	1333	var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader());
1334		var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format));
	1334	var pack_hashed_reader = hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format));
1335	1335	const pack_reader = pack_hashed_reader.reader();
1336	1336
1337	1337	const pack_header = try PackHeader.read(pack_reader);
...	...	@@ -1339,13 +1339,13 @@ fn indexPackFirstPass(
1339	1339	var current_entry: u32 = 0;
1340	1340	while (current_entry < pack_header.total_objects) : (current_entry += 1) {
1341	1341	const entry_offset = pack_counting_reader.bytes_read;
1342		var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init());
	1342	var entry_crc32_reader = hashedReader(pack_reader, std.hash.Crc32.init());
1343	1343	const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader());
1344	1344	switch (entry_header) {
1345	1345	.commit, .tree, .blob, .tag => \|object\| {
1346	1346	var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());
1347	1347	var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1348		var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, Oid.Hasher.init(format));
	1348	var entry_hashed_writer = hashedWriter(std.io.null_writer, Oid.Hasher.init(format));
1349	1349	const entry_writer = entry_hashed_writer.writer();
1350	1350	// The object header is not included in the pack data but is
1351	1351	// part of the object's ID
...	...	@@ -1432,7 +1432,7 @@ fn indexPackHashDelta(
1432	1432	const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache);
1433	1433
1434	1434	var entry_hasher: Oid.Hasher = .init(format);
1435		var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, &entry_hasher);
	1435	var entry_hashed_writer = hashedWriter(std.io.null_writer, &entry_hasher);
1436	1436	try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len });
1437	1437	entry_hasher.update(base_data);
1438	1438	return entry_hasher.finalResult();
...	...	@@ -1703,3 +1703,58 @@ pub fn main() !void {
1703	1703	std.debug.print("Diagnostic: {}\n", .{err});
1704	1704	}
1705	1705	}
	1706
	1707	/// Deprecated
	1708	fn hashedReader(reader: anytype, hasher: anytype) HashedReader(@TypeOf(reader), @TypeOf(hasher)) {
	1709	return .{ .child_reader = reader, .hasher = hasher };
	1710	}
	1711
	1712	/// Deprecated
	1713	fn HashedReader(ReaderType: type, HasherType: type) type {
	1714	return struct {
	1715	child_reader: ReaderType,
	1716	hasher: HasherType,
	1717
	1718	pub const Error = ReaderType.Error;
	1719	pub const Reader = std.io.GenericReader(*@This(), Error, read);
	1720
	1721	pub fn read(self: *@This(), buf: []u8) Error!usize {
	1722	const amt = try self.child_reader.read(buf);
	1723	self.hasher.update(buf[0..amt]);
	1724	return amt;
	1725	}
	1726
	1727	pub fn reader(self: *@This()) Reader {
	1728	return .{ .context = self };
	1729	}
	1730	};
	1731	}
	1732
	1733	/// Deprecated
	1734	pub fn HashedWriter(WriterType: type, HasherType: type) type {
	1735	return struct {
	1736	child_writer: WriterType,
	1737	hasher: HasherType,
	1738
	1739	pub const Error = WriterType.Error;
	1740	pub const Writer = std.io.GenericWriter(*@This(), Error, write);
	1741
	1742	pub fn write(self: *@This(), buf: []const u8) Error!usize {
	1743	const amt = try self.child_writer.write(buf);
	1744	self.hasher.update(buf[0..amt]);
	1745	return amt;
	1746	}
	1747
	1748	pub fn writer(self: *@This()) Writer {
	1749	return .{ .context = self };
	1750	}
	1751	};
	1752	}
	1753
	1754	/// Deprecated
	1755	pub fn hashedWriter(
	1756	writer: anytype,
	1757	hasher: anytype,
	1758	) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) {
	1759	return .{ .child_writer = writer, .hasher = hasher };
	1760	}