Commit 685d55c1a49615b6601c02c174d1097489b0fe95 | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-04-30 22:40:11-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:28-07:00
log	685d55c1a49615b6601c02c174d1097489b0fe95
tree	627b77f57fdc3da485af90bc42f679cca94a19fa
parent	689a70663a1ddd6b939ec4f957869be21ceaaa34

std.compress.zstd: rewrite

WIP - passing AstGen but not yet Sema

15 files changed, 1815 insertions(+), 3015 deletions(-)

lib
- std
  - RingBuffer.zig
  - compress.zig
  - compress
    zstandard.zig
    zstandard
    decode
    block.zig
    fse.zig
    huffman.zig
    decompress.zig
    readers.zig
    types.zig
    zstd.zig
    zstd
    Decompress.zig
  - crypto
    sha2.zig
  - io
    BufferedReader.zig
    Writer
    Null.zig
  - std.zig

lib/std/RingBuffer.zig deleted-230

...	@@ -1,230 +0,0 @@
1	//! This ring buffer stores read and write indices while being able to utilise
2	//! the full backing slice by incrementing the indices modulo twice the slice's
3	//! length and reducing indices modulo the slice's length on slice access. This
4	//! means that whether the ring buffer is full or empty can be distinguished by
5	//! looking at the difference between the read and write indices without adding
6	//! an extra boolean flag or having to reserve a slot in the buffer.
7	//!
8	//! This ring buffer has not been implemented with thread safety in mind, and
9	//! therefore should not be assumed to be suitable for use cases involving
10	//! separate reader and writer threads.
11
12	const Allocator = @import("std").mem.Allocator;
13	const assert = @import("std").debug.assert;
14	const copyForwards = @import("std").mem.copyForwards;
15
16	const RingBuffer = @This();
17
18	data: []u8,
19	read_index: usize,
20	write_index: usize,
21
22	pub const Error = error{ Full, ReadLengthInvalid };
23
24	/// Allocate a new `RingBuffer`; `deinit()` should be called to free the buffer.
25	pub fn init(allocator: Allocator, capacity: usize) Allocator.Error!RingBuffer {
26	const bytes = try allocator.alloc(u8, capacity);
27	return RingBuffer{
28	.data = bytes,
29	.write_index = 0,
30	.read_index = 0,
31	};
32	}
33
34	/// Free the data backing a `RingBuffer`; must be passed the same `Allocator` as
35	/// `init()`.
36	pub fn deinit(self: *RingBuffer, allocator: Allocator) void {
37	allocator.free(self.data);
38	self.* = undefined;
39	}
40
41	/// Returns `index` modulo the length of the backing slice.
42	pub fn mask(self: RingBuffer, index: usize) usize {
43	return index % self.data.len;
44	}
45
46	/// Returns `index` modulo twice the length of the backing slice.
47	pub fn mask2(self: RingBuffer, index: usize) usize {
48	return index % (2 * self.data.len);
49	}
50
51	/// Write `byte` into the ring buffer. Returns `error.Full` if the ring
52	/// buffer is full.
53	pub fn write(self: *RingBuffer, byte: u8) Error!void {
54	if (self.isFull()) return error.Full;
55	self.writeAssumeCapacity(byte);
56	}
57
58	/// Write `byte` into the ring buffer. If the ring buffer is full, the
59	/// oldest byte is overwritten.
60	pub fn writeAssumeCapacity(self: *RingBuffer, byte: u8) void {
61	self.data[self.mask(self.write_index)] = byte;
62	self.write_index = self.mask2(self.write_index + 1);
63	}
64
65	/// Write `bytes` into the ring buffer. Returns `error.Full` if the ring
66	/// buffer does not have enough space, without writing any data.
67	/// Uses memcpy and so `bytes` must not overlap ring buffer data.
68	pub fn writeSlice(self: *RingBuffer, bytes: []const u8) Error!void {
69	if (self.len() + bytes.len > self.data.len) return error.Full;
70	self.writeSliceAssumeCapacity(bytes);
71	}
72
73	/// Write `bytes` into the ring buffer. If there is not enough space, older
74	/// bytes will be overwritten.
75	/// Uses memcpy and so `bytes` must not overlap ring buffer data.
76	pub fn writeSliceAssumeCapacity(self: *RingBuffer, bytes: []const u8) void {
77	assert(bytes.len <= self.data.len);
78	const data_start = self.mask(self.write_index);
79	const part1_data_end = @min(data_start + bytes.len, self.data.len);
80	const part1_len = part1_data_end - data_start;
81	@memcpy(self.data[data_start..part1_data_end], bytes[0..part1_len]);
82
83	const remaining = bytes.len - part1_len;
84	const to_write = @min(remaining, remaining % self.data.len + self.data.len);
85	const part2_bytes_start = bytes.len - to_write;
86	const part2_bytes_end = @min(part2_bytes_start + self.data.len, bytes.len);
87	const part2_len = part2_bytes_end - part2_bytes_start;
88	@memcpy(self.data[0..part2_len], bytes[part2_bytes_start..part2_bytes_end]);
89	if (part2_bytes_end != bytes.len) {
90	const part3_len = bytes.len - part2_bytes_end;
91	@memcpy(self.data[0..part3_len], bytes[part2_bytes_end..bytes.len]);
92	}
93	self.write_index = self.mask2(self.write_index + bytes.len);
94	}
95
96	/// Write `bytes` into the ring buffer. Returns `error.Full` if the ring
97	/// buffer does not have enough space, without writing any data.
98	/// Uses copyForwards and can write slices from this RingBuffer into itself.
99	pub fn writeSliceForwards(self: *RingBuffer, bytes: []const u8) Error!void {
100	if (self.len() + bytes.len > self.data.len) return error.Full;
101	self.writeSliceForwardsAssumeCapacity(bytes);
102	}
103
104	/// Write `bytes` into the ring buffer. If there is not enough space, older
105	/// bytes will be overwritten.
106	/// Uses copyForwards and can write slices from this RingBuffer into itself.
107	pub fn writeSliceForwardsAssumeCapacity(self: *RingBuffer, bytes: []const u8) void {
108	assert(bytes.len <= self.data.len);
109	const data_start = self.mask(self.write_index);
110	const part1_data_end = @min(data_start + bytes.len, self.data.len);
111	const part1_len = part1_data_end - data_start;
112	copyForwards(u8, self.data[data_start..], bytes[0..part1_len]);
113
114	const remaining = bytes.len - part1_len;
115	const to_write = @min(remaining, remaining % self.data.len + self.data.len);
116	const part2_bytes_start = bytes.len - to_write;
117	const part2_bytes_end = @min(part2_bytes_start + self.data.len, bytes.len);
118	copyForwards(u8, self.data[0..], bytes[part2_bytes_start..part2_bytes_end]);
119	if (part2_bytes_end != bytes.len)
120	copyForwards(u8, self.data[0..], bytes[part2_bytes_end..bytes.len]);
121	self.write_index = self.mask2(self.write_index + bytes.len);
122	}
123
124	/// Consume a byte from the ring buffer and return it. Returns `null` if the
125	/// ring buffer is empty.
126	pub fn read(self: *RingBuffer) ?u8 {
127	if (self.isEmpty()) return null;
128	return self.readAssumeLength();
129	}
130
131	/// Consume a byte from the ring buffer and return it; asserts that the buffer
132	/// is not empty.
133	pub fn readAssumeLength(self: *RingBuffer) u8 {
134	assert(!self.isEmpty());
135	const byte = self.data[self.mask(self.read_index)];
136	self.read_index = self.mask2(self.read_index + 1);
137	return byte;
138	}
139
140	/// Reads first `length` bytes written to the ring buffer into `dest`; Returns
141	/// Error.ReadLengthInvalid if length greater than ring or dest length
142	/// Uses memcpy and so `dest` must not overlap ring buffer data.
143	pub fn readFirst(self: *RingBuffer, dest: []u8, length: usize) Error!void {
144	if (length > self.len() or length > dest.len) return error.ReadLengthInvalid;
145	self.readFirstAssumeLength(dest, length);
146	}
147
148	/// Reads first `length` bytes written to the ring buffer into `dest`;
149	/// Asserts that length not greater than ring buffer or dest length
150	/// Uses memcpy and so `dest` must not overlap ring buffer data.
151	pub fn readFirstAssumeLength(self: *RingBuffer, dest: []u8, length: usize) void {
152	assert(length <= self.len() and length <= dest.len);
153	const slice = self.sliceAt(self.read_index, length);
154	slice.copyTo(dest);
155	self.read_index = self.mask2(self.read_index + length);
156	}
157
158	/// Reads last `length` bytes written to the ring buffer into `dest`; Returns
159	/// Error.ReadLengthInvalid if length greater than ring or dest length
160	/// Uses memcpy and so `dest` must not overlap ring buffer data.
161	/// Reduces write index by `length`.
162	pub fn readLast(self: *RingBuffer, dest: []u8, length: usize) Error!void {
163	if (length > self.len() or length > dest.len) return error.ReadLengthInvalid;
164	self.readLastAssumeLength(dest, length);
165	}
166
167	/// Reads last `length` bytes written to the ring buffer into `dest`;
168	/// Asserts that length not greater than ring buffer or dest length
169	/// Uses memcpy and so `dest` must not overlap ring buffer data.
170	/// Reduces write index by `length`.
171	pub fn readLastAssumeLength(self: *RingBuffer, dest: []u8, length: usize) void {
172	assert(length <= self.len() and length <= dest.len);
173	const slice = self.sliceLast(length);
174	slice.copyTo(dest);
175	self.write_index = if (self.write_index >= self.data.len)
176	self.write_index - length
177	else
178	self.mask(self.write_index + self.data.len - length);
179	}
180
181	/// Returns `true` if the ring buffer is empty and `false` otherwise.
182	pub fn isEmpty(self: RingBuffer) bool {
183	return self.write_index == self.read_index;
184	}
185
186	/// Returns `true` if the ring buffer is full and `false` otherwise.
187	pub fn isFull(self: RingBuffer) bool {
188	return self.mask2(self.write_index + self.data.len) == self.read_index;
189	}
190
191	/// Returns the length of data available for reading
192	pub fn len(self: RingBuffer) usize {
193	const wrap_offset = 2 * self.data.len * @intFromBool(self.write_index < self.read_index);
194	const adjusted_write_index = self.write_index + wrap_offset;
195	return adjusted_write_index - self.read_index;
196	}
197
198	/// A `Slice` represents a region of a ring buffer. The region is split into two
199	/// sections as the ring buffer data will not be contiguous if the desired
200	/// region wraps to the start of the backing slice.
201	pub const Slice = struct {
202	first: []u8,
203	second: []u8,
204
205	/// Copy data from `self` into `dest`
206	pub fn copyTo(self: Slice, dest: []u8) void {
207	@memcpy(dest[0..self.first.len], self.first);
208	@memcpy(dest[self.first.len..][0..self.second.len], self.second);
209	}
210	};
211
212	/// Returns a `Slice` for the region of the ring buffer starting at
213	/// `self.mask(start_unmasked)` with the specified length.
214	pub fn sliceAt(self: RingBuffer, start_unmasked: usize, length: usize) Slice {
215	assert(length <= self.data.len);
216	const slice1_start = self.mask(start_unmasked);
217	const slice1_end = @min(self.data.len, slice1_start + length);
218	const slice1 = self.data[slice1_start..slice1_end];
219	const slice2 = self.data[0 .. length - slice1.len];
220	return Slice{
221	.first = slice1,
222	.second = slice2,
223	};
224	}
225
226	/// Returns a `Slice` for the last `length` bytes written to the ring buffer.
227	/// Does not check that any bytes have been written into the region.
228	pub fn sliceLast(self: RingBuffer, length: usize) Slice {
229	return self.sliceAt(self.write_index + self.data.len - length, length);
230	}

lib/std/compress.zig+1-1

...	@@ -6,7 +6,7 @@ pub const lzma = @import("compress/lzma.zig");	...	@@ -6,7 +6,7 @@ pub const lzma = @import("compress/lzma.zig");
6	pub const lzma2 = @import("compress/lzma2.zig");	6	pub const lzma2 = @import("compress/lzma2.zig");
7	pub const xz = @import("compress/xz.zig");	7	pub const xz = @import("compress/xz.zig");
8	pub const zlib = @import("compress/zlib.zig");	8	pub const zlib = @import("compress/zlib.zig");
9	pub const zstd = @import("compress/zstandard.zig");	9	pub const zstd = @import("compress/zstd.zig");
10		10
11	test {	11	test {
12	_ = flate;	12	_ = flate;

lib/std/compress/zstandard.zig deleted-334

...	@@ -1,334 +0,0 @@
1	const std = @import("../std.zig");
2	const RingBuffer = std.RingBuffer;
3
4	const types = @import("zstandard/types.zig");
5
6	/// Recommended amount by the standard. Lower than this may result in inability
7	/// to decompress common streams.
8	pub const default_window_len = 8 * 1024 * 1024;
9
10	pub const frame = types.frame;
11	pub const compressed_block = types.compressed_block;
12
13	pub const decompress = @import("zstandard/decompress.zig");
14
15	pub const Decompressor = struct {
16	const table_size_max = types.compressed_block.table_size_max;
17
18	input: *std.io.BufferedReader,
19	bytes_read: usize,
20	state: enum { NewFrame, InFrame, LastBlock },
21	decode_state: decompress.block.DecodeState,
22	frame_context: decompress.FrameContext,
23	buffer: WindowBuffer,
24	literal_fse_buffer: [table_size_max.literal]types.compressed_block.Table.Fse,
25	match_fse_buffer: [table_size_max.match]types.compressed_block.Table.Fse,
26	offset_fse_buffer: [table_size_max.offset]types.compressed_block.Table.Fse,
27	literals_buffer: [types.block_size_max]u8,
28	sequence_buffer: [types.block_size_max]u8,
29	verify_checksum: bool,
30	checksum: ?u32,
31	current_frame_decompressed_size: usize,
32	err: ?Error = null,
33
34	pub const Options = struct {
35	verify_checksum: bool = true,
36	/// See `default_window_len`.
37	window_buffer: []u8,
38	};
39
40	const WindowBuffer = struct {
41	data: []u8 = undefined,
42	read_index: usize = 0,
43	write_index: usize = 0,
44	};
45
46	pub const Error = std.io.Reader.Error \|\| error{
47	ChecksumFailure,
48	DictionaryIdFlagUnsupported,
49	MalformedBlock,
50	MalformedFrame,
51	OutOfMemory,
52	EndOfStream,
53	};
54
55	pub fn init(input: *std.io.BufferedReader, options: Options) Decompressor {
56	return .{
57	.input = input,
58	.bytes_read = 0,
59	.state = .NewFrame,
60	.decode_state = undefined,
61	.frame_context = undefined,
62	.buffer = .{ .data = options.window_buffer },
63	.literal_fse_buffer = undefined,
64	.match_fse_buffer = undefined,
65	.offset_fse_buffer = undefined,
66	.literals_buffer = undefined,
67	.sequence_buffer = undefined,
68	.verify_checksum = options.verify_checksum,
69	.checksum = undefined,
70	.current_frame_decompressed_size = undefined,
71	};
72	}
73
74	fn frameInit(d: *Decompressor) !void {
75	const in = d.input;
76	switch (try decompress.decodeFrameHeader(in, &d.bytes_read)) {
77	.skippable => \|header\| {
78	try in.discardAll(header.frame_size);
79	d.bytes_read += header.frame_size;
80	d.state = .NewFrame;
81	},
82	.zstandard => \|header\| {
83	const frame_context = try decompress.FrameContext.init(
84	header,
85	d.buffer.data.len,
86	d.verify_checksum,
87	);
88
89	const decode_state = decompress.block.DecodeState.init(
90	&d.literal_fse_buffer,
91	&d.match_fse_buffer,
92	&d.offset_fse_buffer,
93	);
94
95	d.decode_state = decode_state;
96	d.frame_context = frame_context;
97
98	d.checksum = null;
99	d.current_frame_decompressed_size = 0;
100
101	d.state = .InFrame;
102	},
103	}
104	}
105
106	pub fn reader(self: *Decompressor) std.io.Reader {
107	return .{
108	.context = self,
109	.vtable = &.{
110	.read = read,
111	.readVec = readVec,
112	.discard = discard,
113	},
114	};
115	}
116
117	fn read(context: ?anyopaque, bw: std.io.BufferedWriter, limit: std.io.Reader.Limit) std.io.Reader.RwError!usize {
118	const buf = limit.slice(try bw.writableSliceGreedy(1));
119	const n = try readVec(context, &.{buf});
120	bw.advance(n);
121	return n;
122	}
123
124	fn discard(context: ?*anyopaque, limit: std.io.Reader.Limit) std.io.Reader.Error!usize {
125	var trash: [128]u8 = undefined;
126	const buf = limit.slice(&trash);
127	return readVec(context, &.{buf});
128	}
129
130	fn readVec(context: ?*anyopaque, data: []const []u8) std.io.Reader.Error!usize {
131	const d: *Decompressor = @ptrCast(@alignCast(context));
132	if (data.len == 0) return 0;
133	const buffer = data[0];
134	while (d.state == .NewFrame) {
135	const initial_count = d.bytes_read;
136	d.frameInit() catch \|err\| switch (err) {
137	error.DictionaryIdFlagUnsupported => {
138	d.err = error.DictionaryIdFlagUnsupported;
139	return error.ReadFailed;
140	},
141	error.EndOfStream => {
142	if (d.bytes_read == initial_count) return error.EndOfStream;
143	d.err = error.MalformedFrame;
144	return error.ReadFailed;
145	},
146	else => {
147	d.err = error.MalformedFrame;
148	return error.ReadFailed;
149	},
150	};
151	}
152	return d.readInner(buffer) catch \|err\| {
153	d.err = err;
154	return error.ReadFailed;
155	};
156	}
157
158	fn readInner(d: *Decompressor, buffer: []u8) Error!usize {
159	std.debug.assert(d.state != .NewFrame);
160
161	var ring_buffer = RingBuffer{
162	.data = d.buffer.data,
163	.read_index = d.buffer.read_index,
164	.write_index = d.buffer.write_index,
165	};
166	defer {
167	d.buffer.read_index = ring_buffer.read_index;
168	d.buffer.write_index = ring_buffer.write_index;
169	}
170
171	const in = d.input;
172	while (ring_buffer.isEmpty() and d.state != .LastBlock) {
173	const header_bytes = try in.takeArray(3);
174	d.bytes_read += header_bytes.len;
175	const block_header = decompress.block.decodeBlockHeader(header_bytes);
176
177	decompress.block.decodeBlockReader(
178	&ring_buffer,
179	in,
180	&d.bytes_read,
181	block_header,
182	&d.decode_state,
183	d.frame_context.block_size_max,
184	&d.literals_buffer,
185	&d.sequence_buffer,
186	) catch return error.MalformedBlock;
187
188	if (d.frame_context.content_size) \|size\| {
189	if (d.current_frame_decompressed_size > size) return error.MalformedFrame;
190	}
191
192	const size = ring_buffer.len();
193	d.current_frame_decompressed_size += size;
194
195	if (d.frame_context.hasher_opt) \|*hasher\| {
196	if (size > 0) {
197	const written_slice = ring_buffer.sliceLast(size);
198	hasher.update(written_slice.first);
199	hasher.update(written_slice.second);
200	}
201	}
202	if (block_header.last_block) {
203	d.state = .LastBlock;
204	if (d.frame_context.has_checksum) {
205	const checksum = in.readInt(u32, .little) catch return error.MalformedFrame;
206	d.bytes_read += 4;
207	if (d.verify_checksum) {
208	if (d.frame_context.hasher_opt) \|*hasher\| {
209	if (checksum != decompress.computeChecksum(hasher))
210	return error.ChecksumFailure;
211	}
212	}
213	}
214	if (d.frame_context.content_size) \|content_size\| {
215	if (content_size != d.current_frame_decompressed_size) {
216	return error.MalformedFrame;
217	}
218	}
219	}
220	}
221
222	const size = @min(ring_buffer.len(), buffer.len);
223	if (size > 0) {
224	ring_buffer.readFirstAssumeLength(buffer, size);
225	}
226	if (d.state == .LastBlock and ring_buffer.len() == 0) {
227	d.state = .NewFrame;
228	}
229	return size;
230	}
231	};
232
233	fn testDecompress(data: []const u8) ![]u8 {
234	const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
235	defer std.testing.allocator.free(window_buffer);
236
237	var in_stream = std.io.fixedBufferStream(data);
238	var zstd_stream: Decompressor = .init(in_stream.reader(), .{ .window_buffer = window_buffer });
239	const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
240	return result;
241	}
242
243	fn testReader(data: []const u8, comptime expected: []const u8) !void {
244	const buf = try testDecompress(data);
245	defer std.testing.allocator.free(buf);
246	try std.testing.expectEqualSlices(u8, expected, buf);
247	}
248
249	test "decompression" {
250	const uncompressed = @embedFile("testdata/rfc8478.txt");
251	const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
252	const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
253
254	const buffer = try std.testing.allocator.alloc(u8, uncompressed.len);
255	defer std.testing.allocator.free(buffer);
256
257	const res3 = try decompress.decode(buffer, compressed3, true);
258	try std.testing.expectEqual(uncompressed.len, res3);
259	try std.testing.expectEqualSlices(u8, uncompressed, buffer);
260
261	@memset(buffer, undefined);
262	const res19 = try decompress.decode(buffer, compressed19, true);
263	try std.testing.expectEqual(uncompressed.len, res19);
264	try std.testing.expectEqualSlices(u8, uncompressed, buffer);
265
266	try testReader(compressed3, uncompressed);
267	try testReader(compressed19, uncompressed);
268	}
269
270	fn expectEqualDecoded(expected: []const u8, input: []const u8) !void {
271	{
272	const result = try decompress.decodeAlloc(std.testing.allocator, input, false, 1 << 23);
273	defer std.testing.allocator.free(result);
274	try std.testing.expectEqualStrings(expected, result);
275	}
276
277	{
278	var buffer = try std.testing.allocator.alloc(u8, 2 * expected.len);
279	defer std.testing.allocator.free(buffer);
280
281	const size = try decompress.decode(buffer, input, false);
282	try std.testing.expectEqualStrings(expected, buffer[0..size]);
283	}
284	}
285
286	fn expectEqualDecodedStreaming(expected: []const u8, input: []const u8) !void {
287	const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
288	defer std.testing.allocator.free(window_buffer);
289
290	var in_stream = std.io.fixedBufferStream(input);
291	var stream: Decompressor = .init(in_stream.reader(), .{ .window_buffer = window_buffer });
292
293	const result = try stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
294	defer std.testing.allocator.free(result);
295
296	try std.testing.expectEqualStrings(expected, result);
297	}
298
299	test "zero sized block" {
300	const input_raw =
301	"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
302	"\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
303	"\x01\x00\x00"; // block header with: last_block set, block_type raw, block_size zero
304
305	const input_rle =
306	"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
307	"\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
308	"\x03\x00\x00" ++ // block header with: last_block set, block_type rle, block_size zero
309	"\xaa"; // block_content
310
311	try expectEqualDecoded("", input_raw);
312	try expectEqualDecoded("", input_rle);
313	try expectEqualDecodedStreaming("", input_raw);
314	try expectEqualDecodedStreaming("", input_rle);
315	}
316
317	test "declared raw literals size too large" {
318	const input_raw =
319	"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
320	"\x00\x00" ++ // frame header: everything unset, window descriptor zero
321	"\x95\x00\x00" ++ // block header with: last_block set, block_type compressed, block_size 18
322	"\xbc\xf3\xae" ++ // literals section header with: type raw, size_format 3, regenerated_size 716603
323	"\xa5\x9f\xe3"; // some bytes of literal content - the content is shorter than regenerated_size
324
325	// Note that the regenerated_size in the above input is larger than block maximum size, so the
326	// block can't be valid as it is a raw literals block.
327
328	var fbs = std.io.fixedBufferStream(input_raw);
329	var window: [1024]u8 = undefined;
330	var stream: Decompressor = .init(fbs.reader(), .{ .window_buffer = &window });
331
332	var buf: [1024]u8 = undefined;
333	try std.testing.expectError(error.MalformedBlock, stream.read(&buf));
334	}

lib/std/compress/zstandard/decode/block.zig deleted-1153

...	@@ -1,1153 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const RingBuffer = std.RingBuffer;
4
5	const types = @import("../types.zig");
6	const frame = types.frame;
7	const Table = types.compressed_block.Table;
8	const LiteralsSection = types.compressed_block.LiteralsSection;
9	const SequencesSection = types.compressed_block.SequencesSection;
10
11	const huffman = @import("huffman.zig");
12	const readers = @import("../readers.zig");
13
14	const decodeFseTable = @import("fse.zig").decodeFseTable;
15
16	pub const Error = error{
17	BlockSizeOverMaximum,
18	MalformedBlockSize,
19	ReservedBlock,
20	MalformedRleBlock,
21	MalformedCompressedBlock,
22	};
23
24	pub const DecodeState = struct {
25	repeat_offsets: [3]u32,
26
27	offset: StateData(8),
28	match: StateData(9),
29	literal: StateData(9),
30
31	offset_fse_buffer: []Table.Fse,
32	match_fse_buffer: []Table.Fse,
33	literal_fse_buffer: []Table.Fse,
34
35	fse_tables_undefined: bool,
36
37	literal_stream_reader: readers.ReverseBitReader,
38	literal_stream_index: usize,
39	literal_streams: LiteralsSection.Streams,
40	literal_header: LiteralsSection.Header,
41	huffman_tree: ?LiteralsSection.HuffmanTree,
42
43	literal_written_count: usize,
44	written_count: usize = 0,
45
46	fn StateData(comptime max_accuracy_log: comptime_int) type {
47	return struct {
48	state: State,
49	table: Table,
50	accuracy_log: u8,
51
52	const State = std.meta.Int(.unsigned, max_accuracy_log);
53	};
54	}
55
56	pub fn init(
57	literal_fse_buffer: []Table.Fse,
58	match_fse_buffer: []Table.Fse,
59	offset_fse_buffer: []Table.Fse,
60	) DecodeState {
61	return DecodeState{
62	.repeat_offsets = .{
63	types.compressed_block.start_repeated_offset_1,
64	types.compressed_block.start_repeated_offset_2,
65	types.compressed_block.start_repeated_offset_3,
66	},
67
68	.offset = undefined,
69	.match = undefined,
70	.literal = undefined,
71
72	.literal_fse_buffer = literal_fse_buffer,
73	.match_fse_buffer = match_fse_buffer,
74	.offset_fse_buffer = offset_fse_buffer,
75
76	.fse_tables_undefined = true,
77
78	.literal_written_count = 0,
79	.literal_header = undefined,
80	.literal_streams = undefined,
81	.literal_stream_reader = undefined,
82	.literal_stream_index = undefined,
83	.huffman_tree = null,
84
85	.written_count = 0,
86	};
87	}
88
89	/// Prepare the decoder to decode a compressed block. Loads the literals
90	/// stream and Huffman tree from `literals` and reads the FSE tables from
91	/// `source`.
92	///
93	/// Errors returned:
94	/// - `error.BitStreamHasNoStartBit` if the (reversed) literal bitstream's
95	/// first byte does not have any bits set
96	/// - `error.TreelessLiteralsFirst` `literals` is a treeless literals
97	/// section and the decode state does not have a Huffman tree from a
98	/// previous block
99	/// - `error.RepeatModeFirst` on the first call if one of the sequence FSE
100	/// tables is set to repeat mode
101	/// - `error.MalformedAccuracyLog` if an FSE table has an invalid accuracy
102	/// - `error.MalformedFseTable` if there are errors decoding an FSE table
103	/// - `error.EndOfStream` if `source` ends before all FSE tables are read
104	pub fn prepare(
105	self: *DecodeState,
106	source: anytype,
107	literals: LiteralsSection,
108	sequences_header: SequencesSection.Header,
109	) !void {
110	self.literal_written_count = 0;
111	self.literal_header = literals.header;
112	self.literal_streams = literals.streams;
113
114	if (literals.huffman_tree) \|tree\| {
115	self.huffman_tree = tree;
116	} else if (literals.header.block_type == .treeless and self.huffman_tree == null) {
117	return error.TreelessLiteralsFirst;
118	}
119
120	switch (literals.header.block_type) {
121	.raw, .rle => {},
122	.compressed, .treeless => {
123	self.literal_stream_index = 0;
124	switch (literals.streams) {
125	.one => \|slice\| try self.initLiteralStream(slice),
126	.four => \|streams\| try self.initLiteralStream(streams[0]),
127	}
128	},
129	}
130
131	if (sequences_header.sequence_count > 0) {
132	try self.updateFseTable(source, .literal, sequences_header.literal_lengths);
133	try self.updateFseTable(source, .offset, sequences_header.offsets);
134	try self.updateFseTable(source, .match, sequences_header.match_lengths);
135	self.fse_tables_undefined = false;
136	}
137	}
138
139	/// Read initial FSE states for sequence decoding.
140	///
141	/// Errors returned:
142	/// - `error.EndOfStream` if `bit_reader` does not contain enough bits.
143	pub fn readInitialFseState(self: DecodeState, bit_reader: readers.ReverseBitReader) error{EndOfStream}!void {
144	self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
145	self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
146	self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log);
147	}
148
149	fn updateRepeatOffset(self: *DecodeState, offset: u32) void {
150	self.repeat_offsets[2] = self.repeat_offsets[1];
151	self.repeat_offsets[1] = self.repeat_offsets[0];
152	self.repeat_offsets[0] = offset;
153	}
154
155	fn useRepeatOffset(self: *DecodeState, index: usize) u32 {
156	if (index == 1)
157	std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[1])
158	else if (index == 2) {
159	std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[2]);
160	std.mem.swap(u32, &self.repeat_offsets[1], &self.repeat_offsets[2]);
161	}
162	return self.repeat_offsets[0];
163	}
164
165	const DataType = enum { offset, match, literal };
166
167	fn updateState(
168	self: *DecodeState,
169	comptime choice: DataType,
170	bit_reader: *readers.ReverseBitReader,
171	) error{ MalformedFseBits, EndOfStream }!void {
172	switch (@field(self, @tagName(choice)).table) {
173	.rle => {},
174	.fse => \|table\| {
175	const data = table[@field(self, @tagName(choice)).state];
176	const T = @TypeOf(@field(self, @tagName(choice))).State;
177	const bits_summand = try bit_reader.readBitsNoEof(T, data.bits);
178	const next_state = std.math.cast(
179	@TypeOf(@field(self, @tagName(choice))).State,
180	data.baseline + bits_summand,
181	) orelse return error.MalformedFseBits;
182	@field(self, @tagName(choice)).state = next_state;
183	},
184	}
185	}
186
187	const FseTableError = error{
188	MalformedFseTable,
189	MalformedAccuracyLog,
190	RepeatModeFirst,
191	EndOfStream,
192	};
193
194	fn updateFseTable(
195	self: *DecodeState,
196	source: anytype,
197	comptime choice: DataType,
198	mode: SequencesSection.Header.Mode,
199	) !void {
200	const field_name = @tagName(choice);
201	switch (mode) {
202	.predefined => {
203	@field(self, field_name).accuracy_log =
204	@field(types.compressed_block.default_accuracy_log, field_name);
205
206	@field(self, field_name).table =
207	@field(types.compressed_block, "predefined_" ++ field_name ++ "_fse_table");
208	},
209	.rle => {
210	@field(self, field_name).accuracy_log = 0;
211	@field(self, field_name).table = .{ .rle = try source.readByte() };
212	},
213	.fse => {
214	var bit_reader = readers.bitReader(source);
215
216	const table_size = try decodeFseTable(
217	&bit_reader,
218	@field(types.compressed_block.table_symbol_count_max, field_name),
219	@field(types.compressed_block.table_accuracy_log_max, field_name),
220	@field(self, field_name ++ "_fse_buffer"),
221	);
222	@field(self, field_name).table = .{
223	.fse = @field(self, field_name ++ "_fse_buffer")[0..table_size],
224	};
225	@field(self, field_name).accuracy_log = std.math.log2_int_ceil(usize, table_size);
226	},
227	.repeat => if (self.fse_tables_undefined) return error.RepeatModeFirst,
228	}
229	}
230
231	const Sequence = struct {
232	literal_length: u32,
233	match_length: u32,
234	offset: u32,
235	};
236
237	fn nextSequence(
238	self: *DecodeState,
239	bit_reader: *readers.ReverseBitReader,
240	) error{ InvalidBitStream, EndOfStream }!Sequence {
241	const raw_code = self.getCode(.offset);
242	const offset_code = std.math.cast(u5, raw_code) orelse {
243	return error.InvalidBitStream;
244	};
245	const offset_value = (@as(u32, 1) << offset_code) + try bit_reader.readBitsNoEof(u32, offset_code);
246
247	const match_code = self.getCode(.match);
248	if (match_code >= types.compressed_block.match_length_code_table.len)
249	return error.InvalidBitStream;
250	const match = types.compressed_block.match_length_code_table[match_code];
251	const match_length = match[0] + try bit_reader.readBitsNoEof(u32, match[1]);
252
253	const literal_code = self.getCode(.literal);
254	if (literal_code >= types.compressed_block.literals_length_code_table.len)
255	return error.InvalidBitStream;
256	const literal = types.compressed_block.literals_length_code_table[literal_code];
257	const literal_length = literal[0] + try bit_reader.readBitsNoEof(u32, literal[1]);
258
259	const offset = if (offset_value > 3) offset: {
260	const offset = offset_value - 3;
261	self.updateRepeatOffset(offset);
262	break :offset offset;
263	} else offset: {
264	if (literal_length == 0) {
265	if (offset_value == 3) {
266	const offset = self.repeat_offsets[0] - 1;
267	self.updateRepeatOffset(offset);
268	break :offset offset;
269	}
270	break :offset self.useRepeatOffset(offset_value);
271	}
272	break :offset self.useRepeatOffset(offset_value - 1);
273	};
274
275	if (offset == 0) return error.InvalidBitStream;
276
277	return .{
278	.literal_length = literal_length,
279	.match_length = match_length,
280	.offset = offset,
281	};
282	}
283
284	fn executeSequenceSlice(
285	self: *DecodeState,
286	dest: []u8,
287	write_pos: usize,
288	sequence: Sequence,
289	) (error{MalformedSequence} \|\| DecodeLiteralsError)!void {
290	if (sequence.offset > write_pos + sequence.literal_length) return error.MalformedSequence;
291
292	try self.decodeLiteralsSlice(dest[write_pos..], sequence.literal_length);
293	const copy_start = write_pos + sequence.literal_length - sequence.offset;
294	for (
295	dest[write_pos + sequence.literal_length ..][0..sequence.match_length],
296	dest[copy_start..][0..sequence.match_length],
297	) \|d, s\| d. = s;
298	self.written_count += sequence.match_length;
299	}
300
301	fn executeSequenceRingBuffer(
302	self: *DecodeState,
303	dest: *RingBuffer,
304	sequence: Sequence,
305	) (error{MalformedSequence} \|\| DecodeLiteralsError)!void {
306	if (sequence.offset > @min(dest.data.len, self.written_count + sequence.literal_length))
307	return error.MalformedSequence;
308
309	try self.decodeLiteralsRingBuffer(dest, sequence.literal_length);
310	const copy_start = dest.write_index + dest.data.len - sequence.offset;
311	const copy_slice = dest.sliceAt(copy_start, sequence.match_length);
312	dest.writeSliceForwardsAssumeCapacity(copy_slice.first);
313	dest.writeSliceForwardsAssumeCapacity(copy_slice.second);
314	self.written_count += sequence.match_length;
315	}
316
317	const DecodeSequenceError = error{
318	InvalidBitStream,
319	EndOfStream,
320	MalformedSequence,
321	MalformedFseBits,
322	} \|\| DecodeLiteralsError;
323
324	/// Decode one sequence from `bit_reader` into `dest`, written starting at
325	/// `write_pos` and update FSE states if `last_sequence` is `false`.
326	/// `prepare()` must be called for the block before attempting to decode
327	/// sequences.
328	///
329	/// Errors returned:
330	/// - `error.MalformedSequence` if the decompressed sequence would be
331	/// longer than `sequence_size_limit` or the sequence's offset is too
332	/// large
333	/// - `error.UnexpectedEndOfLiteralStream` if the decoder state's literal
334	/// streams do not contain enough literals for the sequence (this may
335	/// mean the literal stream or the sequence is malformed).
336	/// - `error.InvalidBitStream` if the FSE sequence bitstream is malformed
337	/// - `error.EndOfStream` if `bit_reader` does not contain enough bits
338	/// - `error.DestTooSmall` if `dest` is not large enough to holde the
339	/// decompressed sequence
340	pub fn decodeSequenceSlice(
341	self: *DecodeState,
342	dest: []u8,
343	write_pos: usize,
344	bit_reader: *readers.ReverseBitReader,
345	sequence_size_limit: usize,
346	last_sequence: bool,
347	) (error{DestTooSmall} \|\| DecodeSequenceError)!usize {
348	const sequence = try self.nextSequence(bit_reader);
349	const sequence_length = @as(usize, sequence.literal_length) + sequence.match_length;
350	if (sequence_length > sequence_size_limit) return error.MalformedSequence;
351	if (sequence_length > dest[write_pos..].len) return error.DestTooSmall;
352
353	try self.executeSequenceSlice(dest, write_pos, sequence);
354	if (!last_sequence) {
355	try self.updateState(.literal, bit_reader);
356	try self.updateState(.match, bit_reader);
357	try self.updateState(.offset, bit_reader);
358	}
359	return sequence_length;
360	}
361
362	/// Decode one sequence from `bit_reader` into `dest`; see
363	/// `decodeSequenceSlice`.
364	pub fn decodeSequenceRingBuffer(
365	self: *DecodeState,
366	dest: *RingBuffer,
367	bit_reader: anytype,
368	sequence_size_limit: usize,
369	last_sequence: bool,
370	) DecodeSequenceError!usize {
371	const sequence = try self.nextSequence(bit_reader);
372	const sequence_length = @as(usize, sequence.literal_length) + sequence.match_length;
373	if (sequence_length > sequence_size_limit) return error.MalformedSequence;
374
375	try self.executeSequenceRingBuffer(dest, sequence);
376	if (!last_sequence) {
377	try self.updateState(.literal, bit_reader);
378	try self.updateState(.match, bit_reader);
379	try self.updateState(.offset, bit_reader);
380	}
381	return sequence_length;
382	}
383
384	fn nextLiteralMultiStream(
385	self: *DecodeState,
386	) error{BitStreamHasNoStartBit}!void {
387	self.literal_stream_index += 1;
388	try self.initLiteralStream(self.literal_streams.four[self.literal_stream_index]);
389	}
390
391	fn initLiteralStream(self: *DecodeState, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
392	try self.literal_stream_reader.init(bytes);
393	}
394
395	fn isLiteralStreamEmpty(self: *DecodeState) bool {
396	switch (self.literal_streams) {
397	.one => return self.literal_stream_reader.isEmpty(),
398	.four => return self.literal_stream_index == 3 and self.literal_stream_reader.isEmpty(),
399	}
400	}
401
402	const LiteralBitsError = error{
403	BitStreamHasNoStartBit,
404	UnexpectedEndOfLiteralStream,
405	};
406	fn readLiteralsBits(
407	self: *DecodeState,
408	bit_count_to_read: u16,
409	) LiteralBitsError!u16 {
410	return self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch bits: {
411	if (self.literal_streams == .four and self.literal_stream_index < 3) {
412	try self.nextLiteralMultiStream();
413	break :bits self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch
414	return error.UnexpectedEndOfLiteralStream;
415	} else {
416	return error.UnexpectedEndOfLiteralStream;
417	}
418	};
419	}
420
421	const DecodeLiteralsError = error{
422	MalformedLiteralsLength,
423	NotFound,
424	} \|\| LiteralBitsError;
425
426	/// Decode `len` bytes of literals into `dest`.
427	///
428	/// Errors returned:
429	/// - `error.MalformedLiteralsLength` if the number of literal bytes
430	/// decoded by `self` plus `len` is greater than the regenerated size of
431	/// `literals`
432	/// - `error.UnexpectedEndOfLiteralStream` and `error.NotFound` if there
433	/// are problems decoding Huffman compressed literals
434	pub fn decodeLiteralsSlice(
435	self: *DecodeState,
436	dest: []u8,
437	len: usize,
438	) DecodeLiteralsError!void {
439	if (self.literal_written_count + len > self.literal_header.regenerated_size)
440	return error.MalformedLiteralsLength;
441
442	switch (self.literal_header.block_type) {
443	.raw => {
444	const literal_data = self.literal_streams.one[self.literal_written_count..][0..len];
445	@memcpy(dest[0..len], literal_data);
446	self.literal_written_count += len;
447	self.written_count += len;
448	},
449	.rle => {
450	for (0..len) \|i\| {
451	dest[i] = self.literal_streams.one[0];
452	}
453	self.literal_written_count += len;
454	self.written_count += len;
455	},
456	.compressed, .treeless => {
457	// const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
458	const huffman_tree = self.huffman_tree orelse unreachable;
459	const max_bit_count = huffman_tree.max_bit_count;
460	const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
461	huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
462	max_bit_count,
463	);
464	var bits_read: u4 = 0;
465	var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
466	var bit_count_to_read: u4 = starting_bit_count;
467	for (0..len) \|i\| {
468	var prefix: u16 = 0;
469	while (true) {
470	const new_bits = self.readLiteralsBits(bit_count_to_read) catch \|err\| {
471	return err;
472	};
473	prefix <<= bit_count_to_read;
474	prefix \|= new_bits;
475	bits_read += bit_count_to_read;
476	const result = huffman_tree.query(huffman_tree_index, prefix) catch \|err\| {
477	return err;
478	};
479
480	switch (result) {
481	.symbol => \|sym\| {
482	dest[i] = sym;
483	bit_count_to_read = starting_bit_count;
484	bits_read = 0;
485	huffman_tree_index = huffman_tree.symbol_count_minus_one;
486	break;
487	},
488	.index => \|index\| {
489	huffman_tree_index = index;
490	const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
491	huffman_tree.nodes[index].weight,
492	max_bit_count,
493	);
494	bit_count_to_read = bit_count - bits_read;
495	},
496	}
497	}
498	}
499	self.literal_written_count += len;
500	self.written_count += len;
501	},
502	}
503	}
504
505	/// Decode literals into `dest`; see `decodeLiteralsSlice()`.
506	pub fn decodeLiteralsRingBuffer(
507	self: *DecodeState,
508	dest: *RingBuffer,
509	len: usize,
510	) DecodeLiteralsError!void {
511	if (self.literal_written_count + len > self.literal_header.regenerated_size)
512	return error.MalformedLiteralsLength;
513
514	switch (self.literal_header.block_type) {
515	.raw => {
516	const literals_end = self.literal_written_count + len;
517	const literal_data = self.literal_streams.one[self.literal_written_count..literals_end];
518	dest.writeSliceAssumeCapacity(literal_data);
519	self.literal_written_count += len;
520	self.written_count += len;
521	},
522	.rle => {
523	for (0..len) \|_\| {
524	dest.writeAssumeCapacity(self.literal_streams.one[0]);
525	}
526	self.literal_written_count += len;
527	self.written_count += len;
528	},
529	.compressed, .treeless => {
530	// const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
531	const huffman_tree = self.huffman_tree orelse unreachable;
532	const max_bit_count = huffman_tree.max_bit_count;
533	const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
534	huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
535	max_bit_count,
536	);
537	var bits_read: u4 = 0;
538	var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
539	var bit_count_to_read: u4 = starting_bit_count;
540	for (0..len) \|_\| {
541	var prefix: u16 = 0;
542	while (true) {
543	const new_bits = try self.readLiteralsBits(bit_count_to_read);
544	prefix <<= bit_count_to_read;
545	prefix \|= new_bits;
546	bits_read += bit_count_to_read;
547	const result = try huffman_tree.query(huffman_tree_index, prefix);
548
549	switch (result) {
550	.symbol => \|sym\| {
551	dest.writeAssumeCapacity(sym);
552	bit_count_to_read = starting_bit_count;
553	bits_read = 0;
554	huffman_tree_index = huffman_tree.symbol_count_minus_one;
555	break;
556	},
557	.index => \|index\| {
558	huffman_tree_index = index;
559	const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
560	huffman_tree.nodes[index].weight,
561	max_bit_count,
562	);
563	bit_count_to_read = bit_count - bits_read;
564	},
565	}
566	}
567	}
568	self.literal_written_count += len;
569	self.written_count += len;
570	},
571	}
572	}
573
574	fn getCode(self: *DecodeState, comptime choice: DataType) u32 {
575	return switch (@field(self, @tagName(choice)).table) {
576	.rle => \|value\| value,
577	.fse => \|table\| table[@field(self, @tagName(choice)).state].symbol,
578	};
579	}
580	};
581
582	/// Decode a single block from `src` into `dest`. The beginning of `src` must be
583	/// the start of the block content (i.e. directly after the block header).
584	/// Increments `consumed_count` by the number of bytes read from `src` to decode
585	/// the block and returns the decompressed size of the block.
586	///
587	/// Errors returned:
588	///
589	/// - `error.BlockSizeOverMaximum` if block's size is larger than 1 << 17 or
590	/// `dest[written_count..].len`
591	/// - `error.MalformedBlockSize` if `src.len` is smaller than the block size
592	/// and the block is a raw or compressed block
593	/// - `error.ReservedBlock` if the block is a reserved block
594	/// - `error.MalformedRleBlock` if the block is an RLE block and `src.len < 1`
595	/// - `error.MalformedCompressedBlock` if there are errors decoding a
596	/// compressed block
597	/// - `error.DestTooSmall` is `dest` is not large enough to hold the
598	/// decompressed block
599	pub fn decodeBlock(
600	dest: []u8,
601	src: []const u8,
602	block_header: frame.Zstandard.Block.Header,
603	decode_state: *DecodeState,
604	consumed_count: *usize,
605	block_size_max: usize,
606	written_count: usize,
607	) (error{DestTooSmall} \|\| Error)!usize {
608	const block_size = block_header.block_size;
609	if (block_size_max < block_size) return error.BlockSizeOverMaximum;
610	switch (block_header.block_type) {
611	.raw => {
612	if (src.len < block_size) return error.MalformedBlockSize;
613	if (dest[written_count..].len < block_size) return error.DestTooSmall;
614	@memcpy(dest[written_count..][0..block_size], src[0..block_size]);
615	consumed_count.* += block_size;
616	decode_state.written_count += block_size;
617	return block_size;
618	},
619	.rle => {
620	if (src.len < 1) return error.MalformedRleBlock;
621	if (dest[written_count..].len < block_size) return error.DestTooSmall;
622	for (written_count..block_size + written_count) \|write_pos\| {
623	dest[write_pos] = src[0];
624	}
625	consumed_count.* += 1;
626	decode_state.written_count += block_size;
627	return block_size;
628	},
629	.compressed => {
630	if (src.len < block_size) return error.MalformedBlockSize;
631	var bytes_read: usize = 0;
632	const literals = decodeLiteralsSectionSlice(src[0..block_size], &bytes_read) catch
633	return error.MalformedCompressedBlock;
634	var fbs: std.io.FixedBufferStream = .{ .buffer = src[bytes_read..block_size] };
635	const fbs_reader = fbs.reader();
636	const sequences_header = decodeSequencesHeader(fbs_reader) catch
637	return error.MalformedCompressedBlock;
638
639	decode_state.prepare(fbs_reader, literals, sequences_header) catch
640	return error.MalformedCompressedBlock;
641
642	bytes_read += fbs.pos;
643
644	var bytes_written: usize = 0;
645	{
646	const bit_stream_bytes = src[bytes_read..block_size];
647	var bit_stream: readers.ReverseBitReader = undefined;
648	bit_stream.init(bit_stream_bytes) catch return error.MalformedCompressedBlock;
649
650	if (sequences_header.sequence_count > 0) {
651	decode_state.readInitialFseState(&bit_stream) catch
652	return error.MalformedCompressedBlock;
653
654	var sequence_size_limit = block_size_max;
655	for (0..sequences_header.sequence_count) \|i\| {
656	const write_pos = written_count + bytes_written;
657	const decompressed_size = decode_state.decodeSequenceSlice(
658	dest,
659	write_pos,
660	&bit_stream,
661	sequence_size_limit,
662	i == sequences_header.sequence_count - 1,
663	) catch \|err\| switch (err) {
664	error.DestTooSmall => return error.DestTooSmall,
665	else => return error.MalformedCompressedBlock,
666	};
667	bytes_written += decompressed_size;
668	sequence_size_limit -= decompressed_size;
669	}
670	}
671
672	if (!bit_stream.isEmpty()) {
673	return error.MalformedCompressedBlock;
674	}
675	}
676
677	if (decode_state.literal_written_count < literals.header.regenerated_size) {
678	const len = literals.header.regenerated_size - decode_state.literal_written_count;
679	if (len > dest[written_count + bytes_written ..].len) return error.DestTooSmall;
680	decode_state.decodeLiteralsSlice(dest[written_count + bytes_written ..], len) catch
681	return error.MalformedCompressedBlock;
682	bytes_written += len;
683	}
684
685	switch (decode_state.literal_header.block_type) {
686	.treeless, .compressed => {
687	if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
688	},
689	.raw, .rle => {},
690	}
691
692	consumed_count.* += block_size;
693	return bytes_written;
694	},
695	.reserved => return error.ReservedBlock,
696	}
697	}
698
699	/// Decode a single block from `src` into `dest`; see `decodeBlock()`. Returns
700	/// the size of the decompressed block, which can be used with `dest.sliceLast()`
701	/// to get the decompressed bytes. `error.BlockSizeOverMaximum` is returned if
702	/// the block's compressed or decompressed size is larger than `block_size_max`.
703	pub fn decodeBlockRingBuffer(
704	dest: *RingBuffer,
705	src: []const u8,
706	block_header: frame.Zstandard.Block.Header,
707	decode_state: *DecodeState,
708	consumed_count: *usize,
709	block_size_max: usize,
710	) Error!usize {
711	const block_size = block_header.block_size;
712	if (block_size_max < block_size) return error.BlockSizeOverMaximum;
713	switch (block_header.block_type) {
714	.raw => {
715	if (src.len < block_size) return error.MalformedBlockSize;
716	// dest may have length zero if block_size == 0, causing division by zero in
717	// writeSliceAssumeCapacity()
718	if (block_size > 0) {
719	const data = src[0..block_size];
720	dest.writeSliceAssumeCapacity(data);
721	consumed_count.* += block_size;
722	decode_state.written_count += block_size;
723	}
724	return block_size;
725	},
726	.rle => {
727	if (src.len < 1) return error.MalformedRleBlock;
728	for (0..block_size) \|_\| {
729	dest.writeAssumeCapacity(src[0]);
730	}
731	consumed_count.* += 1;
732	decode_state.written_count += block_size;
733	return block_size;
734	},
735	.compressed => {
736	if (src.len < block_size) return error.MalformedBlockSize;
737	var bytes_read: usize = 0;
738	const literals = decodeLiteralsSectionSlice(src[0..block_size], &bytes_read) catch
739	return error.MalformedCompressedBlock;
740	var fbs: std.io.FixedBufferStream = .{ .buffer = src[bytes_read..block_size] };
741	const fbs_reader = fbs.reader();
742	const sequences_header = decodeSequencesHeader(fbs_reader) catch
743	return error.MalformedCompressedBlock;
744
745	decode_state.prepare(fbs_reader, literals, sequences_header) catch
746	return error.MalformedCompressedBlock;
747
748	bytes_read += fbs.pos;
749
750	var bytes_written: usize = 0;
751	{
752	const bit_stream_bytes = src[bytes_read..block_size];
753	var bit_stream: readers.ReverseBitReader = undefined;
754	bit_stream.init(bit_stream_bytes) catch return error.MalformedCompressedBlock;
755
756	if (sequences_header.sequence_count > 0) {
757	decode_state.readInitialFseState(&bit_stream) catch
758	return error.MalformedCompressedBlock;
759
760	var sequence_size_limit = block_size_max;
761	for (0..sequences_header.sequence_count) \|i\| {
762	const decompressed_size = decode_state.decodeSequenceRingBuffer(
763	dest,
764	&bit_stream,
765	sequence_size_limit,
766	i == sequences_header.sequence_count - 1,
767	) catch return error.MalformedCompressedBlock;
768	bytes_written += decompressed_size;
769	sequence_size_limit -= decompressed_size;
770	}
771	}
772
773	if (!bit_stream.isEmpty()) {
774	return error.MalformedCompressedBlock;
775	}
776	}
777
778	if (decode_state.literal_written_count < literals.header.regenerated_size) {
779	const len = literals.header.regenerated_size - decode_state.literal_written_count;
780	decode_state.decodeLiteralsRingBuffer(dest, len) catch
781	return error.MalformedCompressedBlock;
782	bytes_written += len;
783	}
784
785	switch (decode_state.literal_header.block_type) {
786	.treeless, .compressed => {
787	if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
788	},
789	.raw, .rle => {},
790	}
791
792	consumed_count.* += block_size;
793	if (bytes_written > block_size_max) return error.BlockSizeOverMaximum;
794	return bytes_written;
795	},
796	.reserved => return error.ReservedBlock,
797	}
798	}
799
800	/// Decode a single block from `source` into `dest`. Literal and sequence data
801	/// from the block is copied into `literals_buffer` and `sequence_buffer`, which
802	/// must be large enough or `error.LiteralsBufferTooSmall` and
803	/// `error.SequenceBufferTooSmall` are returned (the maximum block size is an
804	/// upper bound for the size of both buffers). See `decodeBlock`
805	/// and `decodeBlockRingBuffer` for function that can decode a block without
806	/// these extra copies. `error.EndOfStream` is returned if `source` does not
807	/// contain enough bytes.
808	pub fn decodeBlockReader(
809	dest: *RingBuffer,
810	in: *std.io.BufferedReader,
811	bytes_read: *usize,
812	block_header: frame.Zstandard.Block.Header,
813	decode_state: *DecodeState,
814	block_size_max: usize,
815	literals_buffer: []u8,
816	sequence_buffer: []u8,
817	) !void {
818	const block_size = block_header.block_size;
819	if (block_size_max < block_size) return error.BlockSizeOverMaximum;
820	switch (block_header.block_type) {
821	.raw => {
822	if (block_size == 0) return;
823	const slice = dest.sliceAt(dest.write_index, block_size);
824	var vecs: [2][]u8 = &.{slice.first, slice.second };
825	try in.readVecAll(&vecs);
826	assert(slice.first.len + slice.second.len == block_size);
827	bytes_read.* += block_size;
828	dest.write_index = dest.mask2(dest.write_index + block_size);
829	decode_state.written_count += block_size;
830	},
831	.rle => {
832	const byte = try in.takeByte();
833	bytes_read.* += 1;
834	for (0..block_size) \|_\| {
835	dest.writeAssumeCapacity(byte);
836	}
837	decode_state.written_count += block_size;
838	},
839	.compressed => {
840	var block_reader_limited = std.io.limitedReader(source, block_size);
841	const block_reader = block_reader_limited.reader();
842	const literals = try decodeLiteralsSection(block_reader, literals_buffer);
843	const sequences_header = try decodeSequencesHeader(block_reader);
844
845	try decode_state.prepare(block_reader, literals, sequences_header);
846
847	var bytes_written: usize = 0;
848	{
849	const size = try block_reader.readAll(sequence_buffer);
850	var bit_stream: readers.ReverseBitReader = undefined;
851	try bit_stream.init(sequence_buffer[0..size]);
852
853	if (sequences_header.sequence_count > 0) {
854	if (sequence_buffer.len < block_reader_limited.bytes_left)
855	return error.SequenceBufferTooSmall;
856
857	decode_state.readInitialFseState(&bit_stream) catch
858	return error.MalformedCompressedBlock;
859
860	var sequence_size_limit = block_size_max;
861	for (0..sequences_header.sequence_count) \|i\| {
862	const decompressed_size = decode_state.decodeSequenceRingBuffer(
863	dest,
864	&bit_stream,
865	sequence_size_limit,
866	i == sequences_header.sequence_count - 1,
867	) catch return error.MalformedCompressedBlock;
868	sequence_size_limit -= decompressed_size;
869	bytes_written += decompressed_size;
870	}
871	}
872
873	if (!bit_stream.isEmpty()) {
874	return error.MalformedCompressedBlock;
875	}
876	}
877
878	if (decode_state.literal_written_count < literals.header.regenerated_size) {
879	const len = literals.header.regenerated_size - decode_state.literal_written_count;
880	decode_state.decodeLiteralsRingBuffer(dest, len) catch
881	return error.MalformedCompressedBlock;
882	bytes_written += len;
883	}
884
885	switch (decode_state.literal_header.block_type) {
886	.treeless, .compressed => {
887	if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
888	},
889	.raw, .rle => {},
890	}
891
892	if (bytes_written > block_size_max) return error.BlockSizeOverMaximum;
893	if (block_reader_limited.bytes_left != 0) return error.MalformedCompressedBlock;
894	decode_state.literal_written_count = 0;
895	},
896	.reserved => return error.ReservedBlock,
897	}
898	}
899
900	/// Decode the header of a block.
901	pub fn decodeBlockHeader(src: *const [3]u8) frame.Zstandard.Block.Header {
902	const last_block = src[0] & 1 == 1;
903	const block_type = @as(frame.Zstandard.Block.Type, @enumFromInt((src[0] & 0b110) >> 1));
904	const block_size = ((src[0] & 0b11111000) >> 3) + (@as(u21, src[1]) << 5) + (@as(u21, src[2]) << 13);
905	return .{
906	.last_block = last_block,
907	.block_type = block_type,
908	.block_size = block_size,
909	};
910	}
911
912	/// Decode the header of a block.
913	///
914	/// Errors returned:
915	/// - `error.EndOfStream` if `src.len < 3`
916	pub fn decodeBlockHeaderSlice(src: []const u8) error{EndOfStream}!frame.Zstandard.Block.Header {
917	if (src.len < 3) return error.EndOfStream;
918	return decodeBlockHeader(src[0..3]);
919	}
920
921	/// Decode a `LiteralsSection` from `src`, incrementing `consumed_count` by the
922	/// number of bytes the section uses.
923	///
924	/// Errors returned:
925	/// - `error.MalformedLiteralsHeader` if the header is invalid
926	/// - `error.MalformedLiteralsSection` if there are decoding errors
927	/// - `error.MalformedAccuracyLog` if compressed literals have invalid
928	/// accuracy
929	/// - `error.MalformedFseTable` if compressed literals have invalid FSE table
930	/// - `error.MalformedHuffmanTree` if there are errors decoding a Huffamn tree
931	/// - `error.EndOfStream` if there are not enough bytes in `src`
932	pub fn decodeLiteralsSectionSlice(
933	src: []const u8,
934	consumed_count: *usize,
935	) (error{ MalformedLiteralsHeader, MalformedLiteralsSection, EndOfStream } \|\| huffman.Error)!LiteralsSection {
936	var bytes_read: usize = 0;
937	const header = header: {
938	var fbs: std.io.FixedBufferStream = .{ .buffer = src };
939	defer bytes_read = fbs.pos;
940	break :header decodeLiteralsHeader(fbs.reader()) catch return error.MalformedLiteralsHeader;
941	};
942	switch (header.block_type) {
943	.raw => {
944	if (src.len < bytes_read + header.regenerated_size) return error.MalformedLiteralsSection;
945	const stream = src[bytes_read..][0..header.regenerated_size];
946	consumed_count.* += header.regenerated_size + bytes_read;
947	return LiteralsSection{
948	.header = header,
949	.huffman_tree = null,
950	.streams = .{ .one = stream },
951	};
952	},
953	.rle => {
954	if (src.len < bytes_read + 1) return error.MalformedLiteralsSection;
955	const stream = src[bytes_read..][0..1];
956	consumed_count.* += 1 + bytes_read;
957	return LiteralsSection{
958	.header = header,
959	.huffman_tree = null,
960	.streams = .{ .one = stream },
961	};
962	},
963	.compressed, .treeless => {
964	const huffman_tree_start = bytes_read;
965	const huffman_tree = if (header.block_type == .compressed)
966	try huffman.decodeHuffmanTreeSlice(src[bytes_read..], &bytes_read)
967	else
968	null;
969	const huffman_tree_size = bytes_read - huffman_tree_start;
970	const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
971	return error.MalformedLiteralsSection;
972
973	if (src.len < bytes_read + total_streams_size) return error.MalformedLiteralsSection;
974	const stream_data = src[bytes_read .. bytes_read + total_streams_size];
975
976	const streams = try decodeStreams(header.size_format, stream_data);
977	consumed_count.* += bytes_read + total_streams_size;
978	return LiteralsSection{
979	.header = header,
980	.huffman_tree = huffman_tree,
981	.streams = streams,
982	};
983	},
984	}
985	}
986
987	/// Decode a `LiteralsSection` from `src`, incrementing `consumed_count` by the
988	/// number of bytes the section uses. See `decodeLiterasSectionSlice()`.
989	pub fn decodeLiteralsSection(
990	source: anytype,
991	buffer: []u8,
992	) !LiteralsSection {
993	const header = try decodeLiteralsHeader(source);
994	switch (header.block_type) {
995	.raw => {
996	if (buffer.len < header.regenerated_size) return error.LiteralsBufferTooSmall;
997	try source.readNoEof(buffer[0..header.regenerated_size]);
998	return LiteralsSection{
999	.header = header,
1000	.huffman_tree = null,
1001	.streams = .{ .one = buffer },
1002	};
1003	},
1004	.rle => {
1005	buffer[0] = try source.readByte();
1006	return LiteralsSection{
1007	.header = header,
1008	.huffman_tree = null,
1009	.streams = .{ .one = buffer[0..1] },
1010	};
1011	},
1012	.compressed, .treeless => {
1013	var counting_reader = std.io.countingReader(source);
1014	const huffman_tree = if (header.block_type == .compressed)
1015	try huffman.decodeHuffmanTree(counting_reader.reader(), buffer)
1016	else
1017	null;
1018	const huffman_tree_size = @as(usize, @intCast(counting_reader.bytes_read));
1019	const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
1020	return error.MalformedLiteralsSection;
1021
1022	if (total_streams_size > buffer.len) return error.LiteralsBufferTooSmall;
1023	try source.readNoEof(buffer[0..total_streams_size]);
1024	const stream_data = buffer[0..total_streams_size];
1025
1026	const streams = try decodeStreams(header.size_format, stream_data);
1027	return LiteralsSection{
1028	.header = header,
1029	.huffman_tree = huffman_tree,
1030	.streams = streams,
1031	};
1032	},
1033	}
1034	}
1035
1036	fn decodeStreams(size_format: u2, stream_data: []const u8) !LiteralsSection.Streams {
1037	if (size_format == 0) {
1038	return .{ .one = stream_data };
1039	}
1040
1041	if (stream_data.len < 6) return error.MalformedLiteralsSection;
1042
1043	const stream_1_length: usize = std.mem.readInt(u16, stream_data[0..2], .little);
1044	const stream_2_length: usize = std.mem.readInt(u16, stream_data[2..4], .little);
1045	const stream_3_length: usize = std.mem.readInt(u16, stream_data[4..6], .little);
1046
1047	const stream_1_start = 6;
1048	const stream_2_start = stream_1_start + stream_1_length;
1049	const stream_3_start = stream_2_start + stream_2_length;
1050	const stream_4_start = stream_3_start + stream_3_length;
1051
1052	if (stream_data.len < stream_4_start) return error.MalformedLiteralsSection;
1053
1054	return .{ .four = .{
1055	stream_data[stream_1_start .. stream_1_start + stream_1_length],
1056	stream_data[stream_2_start .. stream_2_start + stream_2_length],
1057	stream_data[stream_3_start .. stream_3_start + stream_3_length],
1058	stream_data[stream_4_start..],
1059	} };
1060	}
1061
1062	/// Decode a literals section header.
1063	///
1064	/// Errors returned:
1065	/// - `error.EndOfStream` if there are not enough bytes in `source`
1066	pub fn decodeLiteralsHeader(source: anytype) !LiteralsSection.Header {
1067	const byte0 = try source.readByte();
1068	const block_type = @as(LiteralsSection.BlockType, @enumFromInt(byte0 & 0b11));
1069	const size_format = @as(u2, @intCast((byte0 & 0b1100) >> 2));
1070	var regenerated_size: u20 = undefined;
1071	var compressed_size: ?u18 = null;
1072	switch (block_type) {
1073	.raw, .rle => {
1074	switch (size_format) {
1075	0, 2 => {
1076	regenerated_size = byte0 >> 3;
1077	},
1078	1 => regenerated_size = (byte0 >> 4) + (@as(u20, try source.readByte()) << 4),
1079	3 => regenerated_size = (byte0 >> 4) +
1080	(@as(u20, try source.readByte()) << 4) +
1081	(@as(u20, try source.readByte()) << 12),
1082	}
1083	},
1084	.compressed, .treeless => {
1085	const byte1 = try source.readByte();
1086	const byte2 = try source.readByte();
1087	switch (size_format) {
1088	0, 1 => {
1089	regenerated_size = (byte0 >> 4) + ((@as(u20, byte1) & 0b00111111) << 4);
1090	compressed_size = ((byte1 & 0b11000000) >> 6) + (@as(u18, byte2) << 2);
1091	},
1092	2 => {
1093	const byte3 = try source.readByte();
1094	regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00000011) << 12);
1095	compressed_size = ((byte2 & 0b11111100) >> 2) + (@as(u18, byte3) << 6);
1096	},
1097	3 => {
1098	const byte3 = try source.readByte();
1099	const byte4 = try source.readByte();
1100	regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00111111) << 12);
1101	compressed_size = ((byte2 & 0b11000000) >> 6) + (@as(u18, byte3) << 2) + (@as(u18, byte4) << 10);
1102	},
1103	}
1104	},
1105	}
1106	return LiteralsSection.Header{
1107	.block_type = block_type,
1108	.size_format = size_format,
1109	.regenerated_size = regenerated_size,
1110	.compressed_size = compressed_size,
1111	};
1112	}
1113
1114	/// Decode a sequences section header.
1115	///
1116	/// Errors returned:
1117	/// - `error.ReservedBitSet` if the reserved bit is set
1118	/// - `error.EndOfStream` if there are not enough bytes in `source`
1119	pub fn decodeSequencesHeader(
1120	source: anytype,
1121	) !SequencesSection.Header {
1122	var sequence_count: u24 = undefined;
1123
1124	const byte0 = try source.readByte();
1125	if (byte0 == 0) {
1126	return SequencesSection.Header{
1127	.sequence_count = 0,
1128	.offsets = undefined,
1129	.match_lengths = undefined,
1130	.literal_lengths = undefined,
1131	};
1132	} else if (byte0 < 128) {
1133	sequence_count = byte0;
1134	} else if (byte0 < 255) {
1135	sequence_count = (@as(u24, (byte0 - 128)) << 8) + try source.readByte();
1136	} else {
1137	sequence_count = (try source.readByte()) + (@as(u24, try source.readByte()) << 8) + 0x7F00;
1138	}
1139
1140	const compression_modes = try source.readByte();
1141
1142	const matches_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00001100) >> 2));
1143	const offsets_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00110000) >> 4));
1144	const literal_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b11000000) >> 6));
1145	if (compression_modes & 0b11 != 0) return error.ReservedBitSet;
1146
1147	return SequencesSection.Header{
1148	.sequence_count = sequence_count,
1149	.offsets = offsets_mode,
1150	.match_lengths = matches_mode,
1151	.literal_lengths = literal_mode,
1152	};
1153	}

lib/std/compress/zstandard/decode/fse.zig deleted-153

...	@@ -1,153 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3
4	const types = @import("../types.zig");
5	const Table = types.compressed_block.Table;
6
7	pub fn decodeFseTable(
8	bit_reader: anytype,
9	expected_symbol_count: usize,
10	max_accuracy_log: u4,
11	entries: []Table.Fse,
12	) !usize {
13	const accuracy_log_biased = try bit_reader.readBitsNoEof(u4, 4);
14	if (accuracy_log_biased > max_accuracy_log -\| 5) return error.MalformedAccuracyLog;
15	const accuracy_log = accuracy_log_biased + 5;
16
17	var values: [256]u16 = undefined;
18	var value_count: usize = 0;
19
20	const total_probability = @as(u16, 1) << accuracy_log;
21	var accumulated_probability: u16 = 0;
22
23	while (accumulated_probability < total_probability) {
24	// WARNING: The RFC is poorly worded, and would suggest std.math.log2_int_ceil is correct here,
25	// but power of two (remaining probabilities + 1) need max bits set to 1 more.
26	const max_bits = std.math.log2_int(u16, total_probability - accumulated_probability + 1) + 1;
27	const small = try bit_reader.readBitsNoEof(u16, max_bits - 1);
28
29	const cutoff = (@as(u16, 1) << max_bits) - 1 - (total_probability - accumulated_probability + 1);
30
31	const value = if (small < cutoff)
32	small
33	else value: {
34	const value_read = small + (try bit_reader.readBitsNoEof(u16, 1) << (max_bits - 1));
35	break :value if (value_read < @as(u16, 1) << (max_bits - 1))
36	value_read
37	else
38	value_read - cutoff;
39	};
40
41	accumulated_probability += if (value != 0) value - 1 else 1;
42
43	values[value_count] = value;
44	value_count += 1;
45
46	if (value == 1) {
47	while (true) {
48	const repeat_flag = try bit_reader.readBitsNoEof(u2, 2);
49	if (repeat_flag + value_count > 256) return error.MalformedFseTable;
50	for (0..repeat_flag) \|_\| {
51	values[value_count] = 1;
52	value_count += 1;
53	}
54	if (repeat_flag < 3) break;
55	}
56	}
57	if (value_count == 256) break;
58	}
59	bit_reader.alignToByte();
60
61	if (value_count < 2) return error.MalformedFseTable;
62	if (accumulated_probability != total_probability) return error.MalformedFseTable;
63	if (value_count > expected_symbol_count) return error.MalformedFseTable;
64
65	const table_size = total_probability;
66
67	try buildFseTable(values[0..value_count], entries[0..table_size]);
68	return table_size;
69	}
70
71	fn buildFseTable(values: []const u16, entries: []Table.Fse) !void {
72	const total_probability = @as(u16, @intCast(entries.len));
73	const accuracy_log = std.math.log2_int(u16, total_probability);
74	assert(total_probability <= 1 << 9);
75
76	var less_than_one_count: usize = 0;
77	for (values, 0..) \|value, i\| {
78	if (value == 0) {
79	entries[entries.len - 1 - less_than_one_count] = Table.Fse{
80	.symbol = @as(u8, @intCast(i)),
81	.baseline = 0,
82	.bits = accuracy_log,
83	};
84	less_than_one_count += 1;
85	}
86	}
87
88	var position: usize = 0;
89	var temp_states: [1 << 9]u16 = undefined;
90	for (values, 0..) \|value, symbol\| {
91	if (value == 0 or value == 1) continue;
92	const probability = value - 1;
93
94	const state_share_dividend = std.math.ceilPowerOfTwo(u16, probability) catch
95	return error.MalformedFseTable;
96	const share_size = @divExact(total_probability, state_share_dividend);
97	const double_state_count = state_share_dividend - probability;
98	const single_state_count = probability - double_state_count;
99	const share_size_log = std.math.log2_int(u16, share_size);
100
101	for (0..probability) \|i\| {
102	temp_states[i] = @as(u16, @intCast(position));
103	position += (entries.len >> 1) + (entries.len >> 3) + 3;
104	position &= entries.len - 1;
105	while (position >= entries.len - less_than_one_count) {
106	position += (entries.len >> 1) + (entries.len >> 3) + 3;
107	position &= entries.len - 1;
108	}
109	}
110	std.mem.sort(u16, temp_states[0..probability], {}, std.sort.asc(u16));
111	for (0..probability) \|i\| {
112	entries[temp_states[i]] = if (i < double_state_count) Table.Fse{
113	.symbol = @as(u8, @intCast(symbol)),
114	.bits = share_size_log + 1,
115	.baseline = single_state_count * share_size + @as(u16, @intCast(i)) * 2 * share_size,
116	} else Table.Fse{
117	.symbol = @as(u8, @intCast(symbol)),
118	.bits = share_size_log,
119	.baseline = (@as(u16, @intCast(i)) - double_state_count) * share_size,
120	};
121	}
122	}
123	}
124
125	test buildFseTable {
126	const literals_length_default_values = [36]u16{
127	5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2,
128	3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 2, 2, 2, 2, 2,
129	0, 0, 0, 0,
130	};
131
132	const match_lengths_default_values = [53]u16{
133	2, 5, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
134	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
135	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0,
136	0, 0, 0, 0, 0,
137	};
138
139	const offset_codes_default_values = [29]u16{
140	2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
141	2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0,
142	};
143
144	var entries: [64]Table.Fse = undefined;
145	try buildFseTable(&literals_length_default_values, &entries);
146	try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_literal_fse_table.fse, &entries);
147
148	try buildFseTable(&match_lengths_default_values, &entries);
149	try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_match_fse_table.fse, &entries);
150
151	try buildFseTable(&offset_codes_default_values, entries[0..32]);
152	try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_offset_fse_table.fse, entries[0..32]);
153	}

lib/std/compress/zstandard/decode/huffman.zig deleted-234

...	@@ -1,234 +0,0 @@
1	const std = @import("std");
2
3	const types = @import("../types.zig");
4	const LiteralsSection = types.compressed_block.LiteralsSection;
5	const Table = types.compressed_block.Table;
6
7	const readers = @import("../readers.zig");
8
9	const decodeFseTable = @import("fse.zig").decodeFseTable;
10
11	pub const Error = error{
12	MalformedHuffmanTree,
13	MalformedFseTable,
14	MalformedAccuracyLog,
15	EndOfStream,
16	};
17
18	fn decodeFseHuffmanTree(
19	source: anytype,
20	compressed_size: usize,
21	buffer: []u8,
22	weights: *[256]u4,
23	) !usize {
24	var stream = std.io.limitedReader(source, compressed_size);
25	var bit_reader = readers.bitReader(stream.reader());
26
27	var entries: [1 << 6]Table.Fse = undefined;
28	const table_size = decodeFseTable(&bit_reader, 256, 6, &entries) catch \|err\| switch (err) {
29	error.MalformedAccuracyLog, error.MalformedFseTable => \|e\| return e,
30	error.EndOfStream => return error.MalformedFseTable,
31	else => \|e\| return e,
32	};
33	const accuracy_log = std.math.log2_int_ceil(usize, table_size);
34
35	const amount = try stream.reader().readAll(buffer);
36	var huff_bits: readers.ReverseBitReader = undefined;
37	huff_bits.init(buffer[0..amount]) catch return error.MalformedHuffmanTree;
38
39	return assignWeights(&huff_bits, accuracy_log, &entries, weights);
40	}
41
42	fn decodeFseHuffmanTreeSlice(src: []const u8, compressed_size: usize, weights: *[256]u4) !usize {
43	if (src.len < compressed_size) return error.MalformedHuffmanTree;
44	var stream: std.io.FixedBufferStream = .{ .buffer = src[0..compressed_size] };
45	var counting_reader = std.io.countingReader(stream.reader());
46	var bit_reader = readers.bitReader(counting_reader.reader());
47
48	var entries: [1 << 6]Table.Fse = undefined;
49	const table_size = decodeFseTable(&bit_reader, 256, 6, &entries) catch \|err\| switch (err) {
50	error.MalformedAccuracyLog, error.MalformedFseTable => \|e\| return e,
51	error.EndOfStream => return error.MalformedFseTable,
52	};
53	const accuracy_log = std.math.log2_int_ceil(usize, table_size);
54
55	const start_index = std.math.cast(usize, counting_reader.bytes_read) orelse
56	return error.MalformedHuffmanTree;
57	const huff_data = src[start_index..compressed_size];
58	var huff_bits: readers.ReverseBitReader = undefined;
59	huff_bits.init(huff_data) catch return error.MalformedHuffmanTree;
60
61	return assignWeights(&huff_bits, accuracy_log, &entries, weights);
62	}
63
64	fn assignWeights(
65	huff_bits: *readers.ReverseBitReader,
66	accuracy_log: u16,
67	entries: *[1 << 6]Table.Fse,
68	weights: *[256]u4,
69	) !usize {
70	var i: usize = 0;
71	var even_state: u32 = huff_bits.readBitsNoEof(u32, accuracy_log) catch return error.MalformedHuffmanTree;
72	var odd_state: u32 = huff_bits.readBitsNoEof(u32, accuracy_log) catch return error.MalformedHuffmanTree;
73
74	while (i < 254) {
75	const even_data = entries[even_state];
76	var read_bits: u16 = 0;
77	const even_bits = huff_bits.readBits(u32, even_data.bits, &read_bits) catch unreachable;
78	weights[i] = std.math.cast(u4, even_data.symbol) orelse return error.MalformedHuffmanTree;
79	i += 1;
80	if (read_bits < even_data.bits) {
81	weights[i] = std.math.cast(u4, entries[odd_state].symbol) orelse return error.MalformedHuffmanTree;
82	i += 1;
83	break;
84	}
85	even_state = even_data.baseline + even_bits;
86
87	read_bits = 0;
88	const odd_data = entries[odd_state];
89	const odd_bits = huff_bits.readBits(u32, odd_data.bits, &read_bits) catch unreachable;
90	weights[i] = std.math.cast(u4, odd_data.symbol) orelse return error.MalformedHuffmanTree;
91	i += 1;
92	if (read_bits < odd_data.bits) {
93	if (i == 255) return error.MalformedHuffmanTree;
94	weights[i] = std.math.cast(u4, entries[even_state].symbol) orelse return error.MalformedHuffmanTree;
95	i += 1;
96	break;
97	}
98	odd_state = odd_data.baseline + odd_bits;
99	} else return error.MalformedHuffmanTree;
100
101	if (!huff_bits.isEmpty()) {
102	return error.MalformedHuffmanTree;
103	}
104
105	return i + 1; // stream contains all but the last symbol
106	}
107
108	fn decodeDirectHuffmanTree(source: anytype, encoded_symbol_count: usize, weights: *[256]u4) !usize {
109	const weights_byte_count = (encoded_symbol_count + 1) / 2;
110	for (0..weights_byte_count) \|i\| {
111	const byte = try source.readByte();
112	weights[2 * i] = @as(u4, @intCast(byte >> 4));
113	weights[2 * i + 1] = @as(u4, @intCast(byte & 0xF));
114	}
115	return encoded_symbol_count + 1;
116	}
117
118	fn assignSymbols(weight_sorted_prefixed_symbols: []LiteralsSection.HuffmanTree.PrefixedSymbol, weights: [256]u4) usize {
119	for (0..weight_sorted_prefixed_symbols.len) \|i\| {
120	weight_sorted_prefixed_symbols[i] = .{
121	.symbol = @as(u8, @intCast(i)),
122	.weight = undefined,
123	.prefix = undefined,
124	};
125	}
126
127	std.mem.sort(
128	LiteralsSection.HuffmanTree.PrefixedSymbol,
129	weight_sorted_prefixed_symbols,
130	weights,
131	lessThanByWeight,
132	);
133
134	var prefix: u16 = 0;
135	var prefixed_symbol_count: usize = 0;
136	var sorted_index: usize = 0;
137	const symbol_count = weight_sorted_prefixed_symbols.len;
138	while (sorted_index < symbol_count) {
139	var symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
140	const weight = weights[symbol];
141	if (weight == 0) {
142	sorted_index += 1;
143	continue;
144	}
145
146	while (sorted_index < symbol_count) : ({
147	sorted_index += 1;
148	prefixed_symbol_count += 1;
149	prefix += 1;
150	}) {
151	symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
152	if (weights[symbol] != weight) {
153	prefix = ((prefix - 1) >> (weights[symbol] - weight)) + 1;
154	break;
155	}
156	weight_sorted_prefixed_symbols[prefixed_symbol_count].symbol = symbol;
157	weight_sorted_prefixed_symbols[prefixed_symbol_count].prefix = prefix;
158	weight_sorted_prefixed_symbols[prefixed_symbol_count].weight = weight;
159	}
160	}
161	return prefixed_symbol_count;
162	}
163
164	fn buildHuffmanTree(weights: *[256]u4, symbol_count: usize) error{MalformedHuffmanTree}!LiteralsSection.HuffmanTree {
165	var weight_power_sum_big: u32 = 0;
166	for (weights[0 .. symbol_count - 1]) \|value\| {
167	weight_power_sum_big += (@as(u16, 1) << value) >> 1;
168	}
169	if (weight_power_sum_big >= 1 << 11) return error.MalformedHuffmanTree;
170	const weight_power_sum = @as(u16, @intCast(weight_power_sum_big));
171
172	// advance to next power of two (even if weight_power_sum is a power of 2)
173	// TODO: is it valid to have weight_power_sum == 0?
174	const max_number_of_bits = if (weight_power_sum == 0) 1 else std.math.log2_int(u16, weight_power_sum) + 1;
175	const next_power_of_two = @as(u16, 1) << max_number_of_bits;
176	weights[symbol_count - 1] = std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1;
177
178	var weight_sorted_prefixed_symbols: [256]LiteralsSection.HuffmanTree.PrefixedSymbol = undefined;
179	const prefixed_symbol_count = assignSymbols(weight_sorted_prefixed_symbols[0..symbol_count], weights.*);
180	const tree = LiteralsSection.HuffmanTree{
181	.max_bit_count = max_number_of_bits,
182	.symbol_count_minus_one = @as(u8, @intCast(prefixed_symbol_count - 1)),
183	.nodes = weight_sorted_prefixed_symbols,
184	};
185	return tree;
186	}
187
188	pub fn decodeHuffmanTree(
189	source: anytype,
190	buffer: []u8,
191	) (@TypeOf(source).Error \|\| Error)!LiteralsSection.HuffmanTree {
192	const header = try source.readByte();
193	var weights: [256]u4 = undefined;
194	const symbol_count = if (header < 128)
195	// FSE compressed weights
196	try decodeFseHuffmanTree(source, header, buffer, &weights)
197	else
198	try decodeDirectHuffmanTree(source, header - 127, &weights);
199
200	return buildHuffmanTree(&weights, symbol_count);
201	}
202
203	pub fn decodeHuffmanTreeSlice(
204	src: []const u8,
205	consumed_count: *usize,
206	) Error!LiteralsSection.HuffmanTree {
207	if (src.len == 0) return error.MalformedHuffmanTree;
208	const header = src[0];
209	var bytes_read: usize = 1;
210	var weights: [256]u4 = undefined;
211	const symbol_count = if (header < 128) count: {
212	// FSE compressed weights
213	bytes_read += header;
214	break :count try decodeFseHuffmanTreeSlice(src[1..], header, &weights);
215	} else count: {
216	var fbs: std.io.FixedBufferStream = .{ .buffer = src[1..] };
217	defer bytes_read += fbs.pos;
218	break :count try decodeDirectHuffmanTree(fbs.reader(), header - 127, &weights);
219	};
220
221	consumed_count.* += bytes_read;
222	return buildHuffmanTree(&weights, symbol_count);
223	}
224
225	fn lessThanByWeight(
226	weights: [256]u4,
227	lhs: LiteralsSection.HuffmanTree.PrefixedSymbol,
228	rhs: LiteralsSection.HuffmanTree.PrefixedSymbol,
229	) bool {
230	// NOTE: this function relies on the use of a stable sorting algorithm,
231	// otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs;
232	// should be added
233	return weights[lhs.symbol] < weights[rhs.symbol];
234	}

lib/std/compress/zstandard/decompress.zig deleted-441

...	@@ -1,441 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const RingBuffer = std.RingBuffer;
4
5	const types = @import("types.zig");
6	const frame = types.frame;
7	const LiteralsSection = types.compressed_block.LiteralsSection;
8	const SequencesSection = types.compressed_block.SequencesSection;
9	const SkippableHeader = types.frame.Skippable.Header;
10	const ZstandardHeader = types.frame.Zstandard.Header;
11	const Table = types.compressed_block.Table;
12
13	pub const block = @import("decode/block.zig");
14
15	const readers = @import("readers.zig");
16
17	/// Returns `true` is `magic` is a valid magic number for a skippable frame
18	pub fn isSkippableMagic(magic: u32) bool {
19	return frame.Skippable.magic_number_min <= magic and magic <= frame.Skippable.magic_number_max;
20	}
21
22	/// Returns the kind of frame at the beginning of `source`.
23	///
24	/// Errors returned:
25	/// - `error.BadMagic` if `source` begins with bytes not equal to the
26	/// Zstandard frame magic number, or outside the range of magic numbers for
27	/// skippable frames.
28	/// - `error.EndOfStream` if `source` contains fewer than 4 bytes
29	pub fn decodeFrameType(source: anytype) error{ BadMagic, EndOfStream }!frame.Kind {
30	const magic = try source.readInt(u32, .little);
31	return frameType(magic);
32	}
33
34	/// Returns the kind of frame associated to `magic`.
35	///
36	/// Errors returned:
37	/// - `error.BadMagic` if `magic` is not a valid magic number.
38	pub fn frameType(magic: u32) error{BadMagic}!frame.Kind {
39	return if (magic == frame.Zstandard.magic_number)
40	.zstandard
41	else if (isSkippableMagic(magic))
42	.skippable
43	else
44	error.BadMagic;
45	}
46
47	pub const FrameHeader = union(enum) {
48	zstandard: ZstandardHeader,
49	skippable: SkippableHeader,
50	};
51
52	pub const HeaderError = error{ ReadFailed, BadMagic, EndOfStream, ReservedBitSet };
53
54	/// Returns the header of the frame at the beginning of `source`.
55	///
56	/// Errors returned:
57	/// - `error.BadMagic` if `source` begins with bytes not equal to the
58	/// Zstandard frame magic number, or outside the range of magic numbers for
59	/// skippable frames.
60	/// - `error.EndOfStream` if `source` contains fewer than 4 bytes
61	/// - `error.ReservedBitSet` if the frame is a Zstandard frame and any of the
62	/// reserved bits are set
63	pub fn decodeFrameHeader(br: std.io.BufferedReader, bytes_read: usize) HeaderError!FrameHeader {
64	const magic = try br.readInt(u32, .little);
65	bytes_read.* += 4;
66	const frame_type = try frameType(magic);
67	switch (frame_type) {
68	.zstandard => return .{ .zstandard = try decodeZstandardHeader(br, bytes_read) },
69	.skippable => {
70	const result: FrameHeader = .{
71	.skippable = .{
72	.magic_number = magic,
73	.frame_size = try br.readInt(u32, .little),
74	},
75	};
76	bytes_read.* += 4;
77	return result;
78	},
79	}
80	}
81
82	pub const ReadWriteCount = struct {
83	read_count: usize,
84	write_count: usize,
85	};
86
87	/// Decodes frames from `src` into `dest`; returns the length of the result.
88	/// The stream should not have extra trailing bytes - either all bytes in `src`
89	/// will be decoded, or an error will be returned. An error will be returned if
90	/// a Zstandard frame in `src` does not declare its content size.
91	///
92	/// Errors returned:
93	/// - `error.DictionaryIdFlagUnsupported` if a `src` contains a frame that
94	/// uses a dictionary
95	/// - `error.MalformedFrame` if a frame in `src` is invalid
96	/// - `error.UnknownContentSizeUnsupported` if a frame in `src` does not
97	/// declare its content size
98	pub fn decode(dest: []u8, src: []const u8, verify_checksum: bool) error{
99	MalformedFrame,
100	UnknownContentSizeUnsupported,
101	DictionaryIdFlagUnsupported,
102	}!usize {
103	var write_count: usize = 0;
104	var read_count: usize = 0;
105	while (read_count < src.len) {
106	const counts = decodeFrame(dest, src[read_count..], verify_checksum) catch \|err\| {
107	switch (err) {
108	error.UnknownContentSizeUnsupported => return error.UnknownContentSizeUnsupported,
109	error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
110	else => return error.MalformedFrame,
111	}
112	};
113	read_count += counts.read_count;
114	write_count += counts.write_count;
115	}
116	return write_count;
117	}
118
119	/// Decodes the frame at the start of `src` into `dest`. Returns the number of
120	/// bytes read from `src` and written to `dest`. This function can only decode
121	/// frames that declare the decompressed content size.
122	///
123	/// Errors returned:
124	/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic
125	/// number for a Zstandard or skippable frame
126	/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
127	/// uncompressed content size
128	/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
129	/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
130	/// size declared by the frame header
131	/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
132	/// that is larger than `std.math.maxInt(usize)`
133	/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
134	/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
135	/// contains a checksum that does not match the checksum of the decompressed
136	/// data
137	/// - `error.ReservedBitSet` if any of the reserved bits of the frame header
138	/// are set
139	/// - `error.EndOfStream` if `src` does not contain a complete frame
140	/// - `error.BadContentSize` if the content size declared by the frame does
141	/// not equal the actual size of decompressed data
142	/// - an error in `block.Error` if there are errors decoding a block
143	/// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a
144	/// size greater than `src.len`
145	pub fn decodeFrame(
146	dest: []u8,
147	src: []const u8,
148	verify_checksum: bool,
149	) (error{
150	BadMagic,
151	UnknownContentSizeUnsupported,
152	ContentTooLarge,
153	ContentSizeTooLarge,
154	WindowSizeUnknown,
155	DictionaryIdFlagUnsupported,
156	SkippableSizeTooLarge,
157	} \|\| FrameError)!ReadWriteCount {
158	var fbs: std.io.FixedBufferStream = .{ .buffer = src };
159	switch (try decodeFrameType(fbs.reader())) {
160	.zstandard => return decodeZstandardFrame(dest, src, verify_checksum),
161	.skippable => {
162	const content_size = try fbs.reader().readInt(u32, .little);
163	if (content_size > std.math.maxInt(usize) - 8) return error.SkippableSizeTooLarge;
164	const read_count = @as(usize, content_size) + 8;
165	if (read_count > src.len) return error.SkippableSizeTooLarge;
166	return ReadWriteCount{
167	.read_count = read_count,
168	.write_count = 0,
169	};
170	},
171	}
172	}
173
174	/// Returns the frame checksum corresponding to the data fed into `hasher`
175	pub fn computeChecksum(hasher: *std.hash.XxHash64) u32 {
176	const hash = hasher.final();
177	return @as(u32, @intCast(hash & 0xFFFFFFFF));
178	}
179
180	const FrameError = error{
181	ChecksumFailure,
182	BadContentSize,
183	EndOfStream,
184	ReservedBitSet,
185	} \|\| block.Error;
186
187	/// Decode a Zstandard frame from `src` into `dest`, returning the number of
188	/// bytes read from `src` and written to `dest`. The first four bytes of `src`
189	/// must be the magic number for a Zstandard frame.
190	///
191	/// Error returned:
192	/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
193	/// uncompressed content size
194	/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
195	/// size declared by the frame header
196	/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
197	/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
198	/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
199	/// that is larger than `std.math.maxInt(usize)`
200	/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
201	/// contains a checksum that does not match the checksum of the decompressed
202	/// data
203	/// - `error.ReservedBitSet` if the reserved bit of the frame header is set
204	/// - `error.EndOfStream` if `src` does not contain a complete frame
205	/// - an error in `block.Error` if there are errors decoding a block
206	/// - `error.BadContentSize` if the content size declared by the frame does
207	/// not equal the actual size of decompressed data
208	pub fn decodeZstandardFrame(
209	dest: []u8,
210	src: []const u8,
211	verify_checksum: bool,
212	) (error{
213	UnknownContentSizeUnsupported,
214	ContentTooLarge,
215	ContentSizeTooLarge,
216	WindowSizeUnknown,
217	DictionaryIdFlagUnsupported,
218	} \|\| FrameError)!ReadWriteCount {
219	assert(std.mem.readInt(u32, src[0..4], .little) == frame.Zstandard.magic_number);
220	var consumed_count: usize = 4;
221
222	var frame_context = context: {
223	var fbs: std.io.FixedBufferStream = .{ .buffer = src[consumed_count..] };
224	const source = fbs.reader();
225	const frame_header = try decodeZstandardHeader(source);
226	consumed_count += fbs.pos;
227	break :context FrameContext.init(
228	frame_header,
229	std.math.maxInt(usize),
230	verify_checksum,
231	) catch \|err\| switch (err) {
232	error.WindowTooLarge => unreachable,
233	inline else => \|e\| return e,
234	};
235	};
236	const counts = try decodeZStandardFrameBlocks(
237	dest,
238	src[consumed_count..],
239	&frame_context,
240	);
241	return ReadWriteCount{
242	.read_count = counts.read_count + consumed_count,
243	.write_count = counts.write_count,
244	};
245	}
246
247	pub fn decodeZStandardFrameBlocks(
248	dest: []u8,
249	src: []const u8,
250	frame_context: *FrameContext,
251	) (error{ ContentTooLarge, UnknownContentSizeUnsupported } \|\| FrameError)!ReadWriteCount {
252	const content_size = frame_context.content_size orelse
253	return error.UnknownContentSizeUnsupported;
254	if (dest.len < content_size) return error.ContentTooLarge;
255
256	var consumed_count: usize = 0;
257	const written_count = decodeFrameBlocksInner(
258	dest[0..content_size],
259	src[consumed_count..],
260	&consumed_count,
261	if (frame_context.hasher_opt) \|*hasher\| hasher else null,
262	frame_context.block_size_max,
263	) catch \|err\| switch (err) {
264	error.DestTooSmall => return error.BadContentSize,
265	inline else => \|e\| return e,
266	};
267
268	if (written_count != content_size) return error.BadContentSize;
269	if (frame_context.has_checksum) {
270	if (src.len < consumed_count + 4) return error.EndOfStream;
271	const checksum = std.mem.readInt(u32, src[consumed_count..][0..4], .little);
272	consumed_count += 4;
273	if (frame_context.hasher_opt) \|*hasher\| {
274	if (checksum != computeChecksum(hasher)) return error.ChecksumFailure;
275	}
276	}
277	return ReadWriteCount{ .read_count = consumed_count, .write_count = written_count };
278	}
279
280	pub const FrameContext = struct {
281	hasher_opt: ?std.hash.XxHash64,
282	window_size: usize,
283	has_checksum: bool,
284	block_size_max: usize,
285	content_size: ?usize,
286
287	const Error = error{
288	DictionaryIdFlagUnsupported,
289	WindowSizeUnknown,
290	WindowTooLarge,
291	ContentSizeTooLarge,
292	};
293	/// Validates `frame_header` and returns the associated `FrameContext`.
294	///
295	/// Errors returned:
296	/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
297	/// - `error.WindowSizeUnknown` if the frame does not have a valid window
298	/// size
299	/// - `error.WindowTooLarge` if the window size is larger than
300	/// `window_size_max` or `std.math.intMax(usize)`
301	/// - `error.ContentSizeTooLarge` if the frame header indicates a content
302	/// size larger than `std.math.maxInt(usize)`
303	pub fn init(
304	frame_header: ZstandardHeader,
305	window_size_max: usize,
306	verify_checksum: bool,
307	) Error!FrameContext {
308	if (frame_header.descriptor.dictionary_id_flag != 0)
309	return error.DictionaryIdFlagUnsupported;
310
311	const window_size_raw = frameWindowSize(frame_header) orelse return error.WindowSizeUnknown;
312	const window_size = if (window_size_raw > window_size_max)
313	return error.WindowTooLarge
314	else
315	std.math.cast(usize, window_size_raw) orelse return error.WindowTooLarge;
316
317	const should_compute_checksum =
318	frame_header.descriptor.content_checksum_flag and verify_checksum;
319
320	const content_size = if (frame_header.content_size) \|size\|
321	std.math.cast(usize, size) orelse return error.ContentSizeTooLarge
322	else
323	null;
324
325	return .{
326	.hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,
327	.window_size = window_size,
328	.has_checksum = frame_header.descriptor.content_checksum_flag,
329	.block_size_max = @min(types.block_size_max, window_size),
330	.content_size = content_size,
331	};
332	}
333	};
334
335	fn decodeFrameBlocksInner(
336	dest: []u8,
337	src: []const u8,
338	consumed_count: *usize,
339	hash: ?*std.hash.XxHash64,
340	block_size_max: usize,
341	) (error{ EndOfStream, DestTooSmall } \|\| block.Error)!usize {
342	// These tables take 7680 bytes
343	var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined;
344	var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined;
345	var offset_fse_data: [types.compressed_block.table_size_max.offset]Table.Fse = undefined;
346
347	var block_header = try block.decodeBlockHeaderSlice(src);
348	var bytes_read: usize = 3;
349	defer consumed_count.* += bytes_read;
350	var decode_state = block.DecodeState.init(&literal_fse_data, &match_fse_data, &offset_fse_data);
351	var count: usize = 0;
352	while (true) : ({
353	block_header = try block.decodeBlockHeaderSlice(src[bytes_read..]);
354	bytes_read += 3;
355	}) {
356	const written_size = try block.decodeBlock(
357	dest,
358	src[bytes_read..],
359	block_header,
360	&decode_state,
361	&bytes_read,
362	block_size_max,
363	count,
364	);
365	if (hash) \|hash_state\| hash_state.update(dest[count .. count + written_size]);
366	count += written_size;
367	if (block_header.last_block) break;
368	}
369	return count;
370	}
371
372	/// Decode the header of a skippable frame. The first four bytes of `src` must
373	/// be a valid magic number for a skippable frame.
374	pub fn decodeSkippableHeader(src: *const [8]u8) SkippableHeader {
375	const magic = std.mem.readInt(u32, src[0..4], .little);
376	assert(isSkippableMagic(magic));
377	const frame_size = std.mem.readInt(u32, src[4..8], .little);
378	return .{
379	.magic_number = magic,
380	.frame_size = frame_size,
381	};
382	}
383
384	/// Returns the window size required to decompress a frame, or `null` if it
385	/// cannot be determined (which indicates a malformed frame header).
386	pub fn frameWindowSize(header: ZstandardHeader) ?u64 {
387	if (header.window_descriptor) \|descriptor\| {
388	const exponent = (descriptor & 0b11111000) >> 3;
389	const mantissa = descriptor & 0b00000111;
390	const window_log = 10 + exponent;
391	const window_base = @as(u64, 1) << @as(u6, @intCast(window_log));
392	const window_add = (window_base / 8) * mantissa;
393	return window_base + window_add;
394	} else return header.content_size;
395	}
396
397	/// Decode the header of a Zstandard frame.
398	///
399	/// Errors returned:
400	/// - `error.ReservedBitSet` if any of the reserved bits of the header are set
401	/// - `error.EndOfStream` if `source` does not contain a complete header
402	pub fn decodeZstandardHeader(
403	source: anytype,
404	) (@TypeOf(source).Error \|\| error{ EndOfStream, ReservedBitSet })!ZstandardHeader {
405	const descriptor = @as(ZstandardHeader.Descriptor, @bitCast(try source.readByte()));
406
407	if (descriptor.reserved) return error.ReservedBitSet;
408
409	var window_descriptor: ?u8 = null;
410	if (!descriptor.single_segment_flag) {
411	window_descriptor = try source.readByte();
412	}
413
414	var dictionary_id: ?u32 = null;
415	if (descriptor.dictionary_id_flag > 0) {
416	// if flag is 3 then field_size = 4, else field_size = flag
417	const field_size = (@as(u4, 1) << descriptor.dictionary_id_flag) >> 1;
418	dictionary_id = try source.readVarInt(u32, .little, field_size);
419	}
420
421	var content_size: ?u64 = null;
422	if (descriptor.single_segment_flag or descriptor.content_size_flag > 0) {
423	const field_size = @as(u4, 1) << descriptor.content_size_flag;
424	content_size = try source.readVarInt(u64, .little, field_size);
425	if (field_size == 2) content_size.? += 256;
426	}
427
428	const header = ZstandardHeader{
429	.descriptor = descriptor,
430	.window_descriptor = window_descriptor,
431	.dictionary_id = dictionary_id,
432	.content_size = content_size,
433	};
434	return header;
435	}
436
437	test {
438	_ = types;
439	_ = block;
440	_ = readers;
441	}

lib/std/compress/zstandard/readers.zig deleted-62

...	@@ -1,62 +0,0 @@
1	const std = @import("std");
2
3	pub const ReversedByteReader = struct {
4	remaining_bytes: usize,
5	bytes: []const u8,
6
7	const Reader = std.io.Reader(*ReversedByteReader, error{}, readFn);
8
9	pub fn init(bytes: []const u8) ReversedByteReader {
10	return .{
11	.bytes = bytes,
12	.remaining_bytes = bytes.len,
13	};
14	}
15
16	pub fn reader(self: *ReversedByteReader) Reader {
17	return .{ .context = self };
18	}
19
20	fn readFn(ctx: *ReversedByteReader, buffer: []u8) !usize {
21	if (ctx.remaining_bytes == 0) return 0;
22	const byte_index = ctx.remaining_bytes - 1;
23	buffer[0] = ctx.bytes[byte_index];
24	// buffer[0] = @bitReverse(ctx.bytes[byte_index]);
25	ctx.remaining_bytes = byte_index;
26	return 1;
27	}
28	};
29
30	/// A bit reader for reading the reversed bit streams used to encode
31	/// FSE compressed data.
32	pub const ReverseBitReader = struct {
33	byte_reader: ReversedByteReader,
34	bit_reader: std.io.BitReader(.big),
35
36	pub fn init(self: *ReverseBitReader, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
37	self.byte_reader = ReversedByteReader.init(bytes);
38	self.bit_reader = .init(self.byte_reader.reader());
39	if (bytes.len == 0) return;
40	var i: usize = 0;
41	while (i < 8 and 0 == self.readBitsNoEof(u1, 1) catch unreachable) : (i += 1) {}
42	if (i == 8) return error.BitStreamHasNoStartBit;
43	}
44
45	pub fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: u16) error{EndOfStream}!U {
46	return self.bit_reader.readBitsNoEof(U, num_bits);
47	}
48
49	pub fn readBits(self: @This(), comptime U: type, num_bits: u16, out_bits: u16) error{}!U {
50	return try self.bit_reader.readBits(U, num_bits, out_bits);
51	}
52
53	pub fn alignToByte(self: *@This()) void {
54	self.bit_reader.alignToByte();
55	}
56
57	pub fn isEmpty(self: ReverseBitReader) bool {
58	return self.byte_reader.remaining_bytes == 0 and self.bit_reader.count == 0;
59	}
60	};
61
62	pub const BitReader = std.io.BitReader(.little);

lib/std/compress/zstandard/types.zig deleted-403

...	@@ -1,403 +0,0 @@
1	pub const block_size_max = 1 << 17;
2
3	pub const frame = struct {
4	pub const Kind = enum { zstandard, skippable };
5
6	pub const Zstandard = struct {
7	pub const magic_number = 0xFD2FB528;
8
9	header: Header,
10	data_blocks: []Block,
11	checksum: ?u32,
12
13	pub const Header = struct {
14	descriptor: Descriptor,
15	window_descriptor: ?u8,
16	dictionary_id: ?u32,
17	content_size: ?u64,
18
19	pub const Descriptor = packed struct {
20	dictionary_id_flag: u2,
21	content_checksum_flag: bool,
22	reserved: bool,
23	unused: bool,
24	single_segment_flag: bool,
25	content_size_flag: u2,
26	};
27	};
28
29	pub const Block = struct {
30	pub const Header = struct {
31	last_block: bool,
32	block_type: Block.Type,
33	block_size: u21,
34	};
35
36	pub const Type = enum(u2) {
37	raw,
38	rle,
39	compressed,
40	reserved,
41	};
42	};
43	};
44
45	pub const Skippable = struct {
46	pub const magic_number_min = 0x184D2A50;
47	pub const magic_number_max = 0x184D2A5F;
48
49	pub const Header = struct {
50	magic_number: u32,
51	frame_size: u32,
52	};
53	};
54	};
55
56	pub const compressed_block = struct {
57	pub const LiteralsSection = struct {
58	header: Header,
59	huffman_tree: ?HuffmanTree,
60	streams: Streams,
61
62	pub const Streams = union(enum) {
63	one: []const u8,
64	four: [4][]const u8,
65	};
66
67	pub const Header = struct {
68	block_type: BlockType,
69	size_format: u2,
70	regenerated_size: u20,
71	compressed_size: ?u18,
72	};
73
74	pub const BlockType = enum(u2) {
75	raw,
76	rle,
77	compressed,
78	treeless,
79	};
80
81	pub const HuffmanTree = struct {
82	max_bit_count: u4,
83	symbol_count_minus_one: u8,
84	nodes: [256]PrefixedSymbol,
85
86	pub const PrefixedSymbol = struct {
87	symbol: u8,
88	prefix: u16,
89	weight: u4,
90	};
91
92	pub const Result = union(enum) {
93	symbol: u8,
94	index: usize,
95	};
96
97	pub fn query(self: HuffmanTree, index: usize, prefix: u16) error{NotFound}!Result {
98	var node = self.nodes[index];
99	const weight = node.weight;
100	var i: usize = index;
101	while (node.weight == weight) {
102	if (node.prefix == prefix) return Result{ .symbol = node.symbol };
103	if (i == 0) return error.NotFound;
104	i -= 1;
105	node = self.nodes[i];
106	}
107	return Result{ .index = i };
108	}
109
110	pub fn weightToBitCount(weight: u4, max_bit_count: u4) u4 {
111	return if (weight == 0) 0 else ((max_bit_count + 1) - weight);
112	}
113	};
114
115	pub const StreamCount = enum { one, four };
116	pub fn streamCount(size_format: u2, block_type: BlockType) StreamCount {
117	return switch (block_type) {
118	.raw, .rle => .one,
119	.compressed, .treeless => if (size_format == 0) .one else .four,
120	};
121	}
122	};
123
124	pub const SequencesSection = struct {
125	header: SequencesSection.Header,
126	literals_length_table: Table,
127	offset_table: Table,
128	match_length_table: Table,
129
130	pub const Header = struct {
131	sequence_count: u24,
132	match_lengths: Mode,
133	offsets: Mode,
134	literal_lengths: Mode,
135
136	pub const Mode = enum(u2) {
137	predefined,
138	rle,
139	fse,
140	repeat,
141	};
142	};
143	};
144
145	pub const Table = union(enum) {
146	fse: []const Fse,
147	rle: u8,
148
149	pub const Fse = struct {
150	symbol: u8,
151	baseline: u16,
152	bits: u8,
153	};
154	};
155
156	pub const literals_length_code_table = [36]struct { u32, u5 }{
157	.{ 0, 0 }, .{ 1, 0 }, .{ 2, 0 }, .{ 3, 0 },
158	.{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 },
159	.{ 8, 0 }, .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 },
160	.{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 }, .{ 15, 0 },
161	.{ 16, 1 }, .{ 18, 1 }, .{ 20, 1 }, .{ 22, 1 },
162	.{ 24, 2 }, .{ 28, 2 }, .{ 32, 3 }, .{ 40, 3 },
163	.{ 48, 4 }, .{ 64, 6 }, .{ 128, 7 }, .{ 256, 8 },
164	.{ 512, 9 }, .{ 1024, 10 }, .{ 2048, 11 }, .{ 4096, 12 },
165	.{ 8192, 13 }, .{ 16384, 14 }, .{ 32768, 15 }, .{ 65536, 16 },
166	};
167
168	pub const match_length_code_table = [53]struct { u32, u5 }{
169	.{ 3, 0 }, .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 }, .{ 8, 0 },
170	.{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 }, .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 },
171	.{ 15, 0 }, .{ 16, 0 }, .{ 17, 0 }, .{ 18, 0 }, .{ 19, 0 }, .{ 20, 0 },
172	.{ 21, 0 }, .{ 22, 0 }, .{ 23, 0 }, .{ 24, 0 }, .{ 25, 0 }, .{ 26, 0 },
173	.{ 27, 0 }, .{ 28, 0 }, .{ 29, 0 }, .{ 30, 0 }, .{ 31, 0 }, .{ 32, 0 },
174	.{ 33, 0 }, .{ 34, 0 }, .{ 35, 1 }, .{ 37, 1 }, .{ 39, 1 }, .{ 41, 1 },
175	.{ 43, 2 }, .{ 47, 2 }, .{ 51, 3 }, .{ 59, 3 }, .{ 67, 4 }, .{ 83, 4 },
176	.{ 99, 5 }, .{ 131, 7 }, .{ 259, 8 }, .{ 515, 9 }, .{ 1027, 10 }, .{ 2051, 11 },
177	.{ 4099, 12 }, .{ 8195, 13 }, .{ 16387, 14 }, .{ 32771, 15 }, .{ 65539, 16 },
178	};
179
180	pub const literals_length_default_distribution = [36]i16{
181	4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
182	2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
183	-1, -1, -1, -1,
184	};
185
186	pub const match_lengths_default_distribution = [53]i16{
187	1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
188	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
189	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
190	-1, -1, -1, -1, -1,
191	};
192
193	pub const offset_codes_default_distribution = [29]i16{
194	1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
195	1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1,
196	};
197
198	pub const predefined_literal_fse_table = Table{
199	.fse = &[64]Table.Fse{
200	.{ .symbol = 0, .bits = 4, .baseline = 0 },
201	.{ .symbol = 0, .bits = 4, .baseline = 16 },
202	.{ .symbol = 1, .bits = 5, .baseline = 32 },
203	.{ .symbol = 3, .bits = 5, .baseline = 0 },
204	.{ .symbol = 4, .bits = 5, .baseline = 0 },
205	.{ .symbol = 6, .bits = 5, .baseline = 0 },
206	.{ .symbol = 7, .bits = 5, .baseline = 0 },
207	.{ .symbol = 9, .bits = 5, .baseline = 0 },
208	.{ .symbol = 10, .bits = 5, .baseline = 0 },
209	.{ .symbol = 12, .bits = 5, .baseline = 0 },
210	.{ .symbol = 14, .bits = 6, .baseline = 0 },
211	.{ .symbol = 16, .bits = 5, .baseline = 0 },
212	.{ .symbol = 18, .bits = 5, .baseline = 0 },
213	.{ .symbol = 19, .bits = 5, .baseline = 0 },
214	.{ .symbol = 21, .bits = 5, .baseline = 0 },
215	.{ .symbol = 22, .bits = 5, .baseline = 0 },
216	.{ .symbol = 24, .bits = 5, .baseline = 0 },
217	.{ .symbol = 25, .bits = 5, .baseline = 32 },
218	.{ .symbol = 26, .bits = 5, .baseline = 0 },
219	.{ .symbol = 27, .bits = 6, .baseline = 0 },
220	.{ .symbol = 29, .bits = 6, .baseline = 0 },
221	.{ .symbol = 31, .bits = 6, .baseline = 0 },
222	.{ .symbol = 0, .bits = 4, .baseline = 32 },
223	.{ .symbol = 1, .bits = 4, .baseline = 0 },
224	.{ .symbol = 2, .bits = 5, .baseline = 0 },
225	.{ .symbol = 4, .bits = 5, .baseline = 32 },
226	.{ .symbol = 5, .bits = 5, .baseline = 0 },
227	.{ .symbol = 7, .bits = 5, .baseline = 32 },
228	.{ .symbol = 8, .bits = 5, .baseline = 0 },
229	.{ .symbol = 10, .bits = 5, .baseline = 32 },
230	.{ .symbol = 11, .bits = 5, .baseline = 0 },
231	.{ .symbol = 13, .bits = 6, .baseline = 0 },
232	.{ .symbol = 16, .bits = 5, .baseline = 32 },
233	.{ .symbol = 17, .bits = 5, .baseline = 0 },
234	.{ .symbol = 19, .bits = 5, .baseline = 32 },
235	.{ .symbol = 20, .bits = 5, .baseline = 0 },
236	.{ .symbol = 22, .bits = 5, .baseline = 32 },
237	.{ .symbol = 23, .bits = 5, .baseline = 0 },
238	.{ .symbol = 25, .bits = 4, .baseline = 0 },
239	.{ .symbol = 25, .bits = 4, .baseline = 16 },
240	.{ .symbol = 26, .bits = 5, .baseline = 32 },
241	.{ .symbol = 28, .bits = 6, .baseline = 0 },
242	.{ .symbol = 30, .bits = 6, .baseline = 0 },
243	.{ .symbol = 0, .bits = 4, .baseline = 48 },
244	.{ .symbol = 1, .bits = 4, .baseline = 16 },
245	.{ .symbol = 2, .bits = 5, .baseline = 32 },
246	.{ .symbol = 3, .bits = 5, .baseline = 32 },
247	.{ .symbol = 5, .bits = 5, .baseline = 32 },
248	.{ .symbol = 6, .bits = 5, .baseline = 32 },
249	.{ .symbol = 8, .bits = 5, .baseline = 32 },
250	.{ .symbol = 9, .bits = 5, .baseline = 32 },
251	.{ .symbol = 11, .bits = 5, .baseline = 32 },
252	.{ .symbol = 12, .bits = 5, .baseline = 32 },
253	.{ .symbol = 15, .bits = 6, .baseline = 0 },
254	.{ .symbol = 17, .bits = 5, .baseline = 32 },
255	.{ .symbol = 18, .bits = 5, .baseline = 32 },
256	.{ .symbol = 20, .bits = 5, .baseline = 32 },
257	.{ .symbol = 21, .bits = 5, .baseline = 32 },
258	.{ .symbol = 23, .bits = 5, .baseline = 32 },
259	.{ .symbol = 24, .bits = 5, .baseline = 32 },
260	.{ .symbol = 35, .bits = 6, .baseline = 0 },
261	.{ .symbol = 34, .bits = 6, .baseline = 0 },
262	.{ .symbol = 33, .bits = 6, .baseline = 0 },
263	.{ .symbol = 32, .bits = 6, .baseline = 0 },
264	},
265	};
266
267	pub const predefined_match_fse_table = Table{
268	.fse = &[64]Table.Fse{
269	.{ .symbol = 0, .bits = 6, .baseline = 0 },
270	.{ .symbol = 1, .bits = 4, .baseline = 0 },
271	.{ .symbol = 2, .bits = 5, .baseline = 32 },
272	.{ .symbol = 3, .bits = 5, .baseline = 0 },
273	.{ .symbol = 5, .bits = 5, .baseline = 0 },
274	.{ .symbol = 6, .bits = 5, .baseline = 0 },
275	.{ .symbol = 8, .bits = 5, .baseline = 0 },
276	.{ .symbol = 10, .bits = 6, .baseline = 0 },
277	.{ .symbol = 13, .bits = 6, .baseline = 0 },
278	.{ .symbol = 16, .bits = 6, .baseline = 0 },
279	.{ .symbol = 19, .bits = 6, .baseline = 0 },
280	.{ .symbol = 22, .bits = 6, .baseline = 0 },
281	.{ .symbol = 25, .bits = 6, .baseline = 0 },
282	.{ .symbol = 28, .bits = 6, .baseline = 0 },
283	.{ .symbol = 31, .bits = 6, .baseline = 0 },
284	.{ .symbol = 33, .bits = 6, .baseline = 0 },
285	.{ .symbol = 35, .bits = 6, .baseline = 0 },
286	.{ .symbol = 37, .bits = 6, .baseline = 0 },
287	.{ .symbol = 39, .bits = 6, .baseline = 0 },
288	.{ .symbol = 41, .bits = 6, .baseline = 0 },
289	.{ .symbol = 43, .bits = 6, .baseline = 0 },
290	.{ .symbol = 45, .bits = 6, .baseline = 0 },
291	.{ .symbol = 1, .bits = 4, .baseline = 16 },
292	.{ .symbol = 2, .bits = 4, .baseline = 0 },
293	.{ .symbol = 3, .bits = 5, .baseline = 32 },
294	.{ .symbol = 4, .bits = 5, .baseline = 0 },
295	.{ .symbol = 6, .bits = 5, .baseline = 32 },
296	.{ .symbol = 7, .bits = 5, .baseline = 0 },
297	.{ .symbol = 9, .bits = 6, .baseline = 0 },
298	.{ .symbol = 12, .bits = 6, .baseline = 0 },
299	.{ .symbol = 15, .bits = 6, .baseline = 0 },
300	.{ .symbol = 18, .bits = 6, .baseline = 0 },
301	.{ .symbol = 21, .bits = 6, .baseline = 0 },
302	.{ .symbol = 24, .bits = 6, .baseline = 0 },
303	.{ .symbol = 27, .bits = 6, .baseline = 0 },
304	.{ .symbol = 30, .bits = 6, .baseline = 0 },
305	.{ .symbol = 32, .bits = 6, .baseline = 0 },
306	.{ .symbol = 34, .bits = 6, .baseline = 0 },
307	.{ .symbol = 36, .bits = 6, .baseline = 0 },
308	.{ .symbol = 38, .bits = 6, .baseline = 0 },
309	.{ .symbol = 40, .bits = 6, .baseline = 0 },
310	.{ .symbol = 42, .bits = 6, .baseline = 0 },
311	.{ .symbol = 44, .bits = 6, .baseline = 0 },
312	.{ .symbol = 1, .bits = 4, .baseline = 32 },
313	.{ .symbol = 1, .bits = 4, .baseline = 48 },
314	.{ .symbol = 2, .bits = 4, .baseline = 16 },
315	.{ .symbol = 4, .bits = 5, .baseline = 32 },
316	.{ .symbol = 5, .bits = 5, .baseline = 32 },
317	.{ .symbol = 7, .bits = 5, .baseline = 32 },
318	.{ .symbol = 8, .bits = 5, .baseline = 32 },
319	.{ .symbol = 11, .bits = 6, .baseline = 0 },
320	.{ .symbol = 14, .bits = 6, .baseline = 0 },
321	.{ .symbol = 17, .bits = 6, .baseline = 0 },
322	.{ .symbol = 20, .bits = 6, .baseline = 0 },
323	.{ .symbol = 23, .bits = 6, .baseline = 0 },
324	.{ .symbol = 26, .bits = 6, .baseline = 0 },
325	.{ .symbol = 29, .bits = 6, .baseline = 0 },
326	.{ .symbol = 52, .bits = 6, .baseline = 0 },
327	.{ .symbol = 51, .bits = 6, .baseline = 0 },
328	.{ .symbol = 50, .bits = 6, .baseline = 0 },
329	.{ .symbol = 49, .bits = 6, .baseline = 0 },
330	.{ .symbol = 48, .bits = 6, .baseline = 0 },
331	.{ .symbol = 47, .bits = 6, .baseline = 0 },
332	.{ .symbol = 46, .bits = 6, .baseline = 0 },
333	},
334	};
335
336	pub const predefined_offset_fse_table = Table{
337	.fse = &[32]Table.Fse{
338	.{ .symbol = 0, .bits = 5, .baseline = 0 },
339	.{ .symbol = 6, .bits = 4, .baseline = 0 },
340	.{ .symbol = 9, .bits = 5, .baseline = 0 },
341	.{ .symbol = 15, .bits = 5, .baseline = 0 },
342	.{ .symbol = 21, .bits = 5, .baseline = 0 },
343	.{ .symbol = 3, .bits = 5, .baseline = 0 },
344	.{ .symbol = 7, .bits = 4, .baseline = 0 },
345	.{ .symbol = 12, .bits = 5, .baseline = 0 },
346	.{ .symbol = 18, .bits = 5, .baseline = 0 },
347	.{ .symbol = 23, .bits = 5, .baseline = 0 },
348	.{ .symbol = 5, .bits = 5, .baseline = 0 },
349	.{ .symbol = 8, .bits = 4, .baseline = 0 },
350	.{ .symbol = 14, .bits = 5, .baseline = 0 },
351	.{ .symbol = 20, .bits = 5, .baseline = 0 },
352	.{ .symbol = 2, .bits = 5, .baseline = 0 },
353	.{ .symbol = 7, .bits = 4, .baseline = 16 },
354	.{ .symbol = 11, .bits = 5, .baseline = 0 },
355	.{ .symbol = 17, .bits = 5, .baseline = 0 },
356	.{ .symbol = 22, .bits = 5, .baseline = 0 },
357	.{ .symbol = 4, .bits = 5, .baseline = 0 },
358	.{ .symbol = 8, .bits = 4, .baseline = 16 },
359	.{ .symbol = 13, .bits = 5, .baseline = 0 },
360	.{ .symbol = 19, .bits = 5, .baseline = 0 },
361	.{ .symbol = 1, .bits = 5, .baseline = 0 },
362	.{ .symbol = 6, .bits = 4, .baseline = 16 },
363	.{ .symbol = 10, .bits = 5, .baseline = 0 },
364	.{ .symbol = 16, .bits = 5, .baseline = 0 },
365	.{ .symbol = 28, .bits = 5, .baseline = 0 },
366	.{ .symbol = 27, .bits = 5, .baseline = 0 },
367	.{ .symbol = 26, .bits = 5, .baseline = 0 },
368	.{ .symbol = 25, .bits = 5, .baseline = 0 },
369	.{ .symbol = 24, .bits = 5, .baseline = 0 },
370	},
371	};
372	pub const start_repeated_offset_1 = 1;
373	pub const start_repeated_offset_2 = 4;
374	pub const start_repeated_offset_3 = 8;
375
376	pub const table_accuracy_log_max = struct {
377	pub const literal = 9;
378	pub const match = 9;
379	pub const offset = 8;
380	};
381
382	pub const table_symbol_count_max = struct {
383	pub const literal = 36;
384	pub const match = 53;
385	pub const offset = 32;
386	};
387
388	pub const default_accuracy_log = struct {
389	pub const literal = 6;
390	pub const match = 6;
391	pub const offset = 5;
392	};
393	pub const table_size_max = struct {
394	pub const literal = 1 << table_accuracy_log_max.literal;
395	pub const match = 1 << table_accuracy_log_max.match;
396	pub const offset = 1 << table_accuracy_log_max.offset;
397	};
398	};
399
400	test {
401	const testing = @import("std").testing;
402	testing.refAllDeclsRecursive(@This());
403	}

lib/std/compress/zstd.zig created+153

...	@@ -0,0 +1,153 @@
1	const std = @import("../std.zig");
2	const assert = std.debug.assert;
3
4	/// Recommended amount by the standard. Lower than this may result in inability
5	/// to decompress common streams.
6	pub const default_window_len = 8 * 1024 * 1024;
7
8	pub const Decompress = @import("zstd/Decompress.zig");
9
10	pub const block_size_max = 1 << 17;
11
12	pub const literals_length_default_distribution = [36]i16{
13	4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
14	2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
15	-1, -1, -1, -1,
16	};
17
18	pub const match_lengths_default_distribution = [53]i16{
19	1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
20	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
21	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
22	-1, -1, -1, -1, -1,
23	};
24
25	pub const offset_codes_default_distribution = [29]i16{
26	1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
27	1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1,
28	};
29
30	pub const start_repeated_offset_1 = 1;
31	pub const start_repeated_offset_2 = 4;
32	pub const start_repeated_offset_3 = 8;
33
34	pub const literals_length_code_table = [36]struct { u32, u5 }{
35	.{ 0, 0 }, .{ 1, 0 }, .{ 2, 0 }, .{ 3, 0 },
36	.{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 },
37	.{ 8, 0 }, .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 },
38	.{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 }, .{ 15, 0 },
39	.{ 16, 1 }, .{ 18, 1 }, .{ 20, 1 }, .{ 22, 1 },
40	.{ 24, 2 }, .{ 28, 2 }, .{ 32, 3 }, .{ 40, 3 },
41	.{ 48, 4 }, .{ 64, 6 }, .{ 128, 7 }, .{ 256, 8 },
42	.{ 512, 9 }, .{ 1024, 10 }, .{ 2048, 11 }, .{ 4096, 12 },
43	.{ 8192, 13 }, .{ 16384, 14 }, .{ 32768, 15 }, .{ 65536, 16 },
44	};
45
46	pub const match_length_code_table = [53]struct { u32, u5 }{
47	.{ 3, 0 }, .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 }, .{ 8, 0 },
48	.{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 }, .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 },
49	.{ 15, 0 }, .{ 16, 0 }, .{ 17, 0 }, .{ 18, 0 }, .{ 19, 0 }, .{ 20, 0 },
50	.{ 21, 0 }, .{ 22, 0 }, .{ 23, 0 }, .{ 24, 0 }, .{ 25, 0 }, .{ 26, 0 },
51	.{ 27, 0 }, .{ 28, 0 }, .{ 29, 0 }, .{ 30, 0 }, .{ 31, 0 }, .{ 32, 0 },
52	.{ 33, 0 }, .{ 34, 0 }, .{ 35, 1 }, .{ 37, 1 }, .{ 39, 1 }, .{ 41, 1 },
53	.{ 43, 2 }, .{ 47, 2 }, .{ 51, 3 }, .{ 59, 3 }, .{ 67, 4 }, .{ 83, 4 },
54	.{ 99, 5 }, .{ 131, 7 }, .{ 259, 8 }, .{ 515, 9 }, .{ 1027, 10 }, .{ 2051, 11 },
55	.{ 4099, 12 }, .{ 8195, 13 }, .{ 16387, 14 }, .{ 32771, 15 }, .{ 65539, 16 },
56	};
57
58	pub const table_accuracy_log_max = struct {
59	pub const literal = 9;
60	pub const match = 9;
61	pub const offset = 8;
62	};
63
64	pub const table_symbol_count_max = struct {
65	pub const literal = 36;
66	pub const match = 53;
67	pub const offset = 32;
68	};
69
70	pub const default_accuracy_log = struct {
71	pub const literal = 6;
72	pub const match = 6;
73	pub const offset = 5;
74	};
75	pub const table_size_max = struct {
76	pub const literal = 1 << table_accuracy_log_max.literal;
77	pub const match = 1 << table_accuracy_log_max.match;
78	pub const offset = 1 << table_accuracy_log_max.offset;
79	};
80
81	fn testDecompress(gpa: std.mem.Allocator, compressed: []const u8) ![]u8 {
82	var out: std.ArrayListUnmanaged(u8) = .empty;
83	defer out.deinit(gpa);
84	try out.ensureUnusedCapacity(gpa, default_window_len);
85
86	var in: std.io.BufferedReader = undefined;
87	in.initFixed(@constCast(compressed));
88	var zstd_stream: Decompress = .init(&in, .{});
89	try zstd_stream.reader().readRemainingArrayList(gpa, null, &out, .unlimited);
90
91	return out.toOwnedSlice(gpa);
92	}
93
94	fn testExpectDecompress(uncompressed: []const u8, compressed: []const u8) !void {
95	const gpa = std.testing.allocator;
96	const result = try testDecompress(gpa, compressed);
97	defer gpa.free(result);
98	try std.testing.expectEqualSlices(u8, uncompressed, result);
99	}
100
101	fn testExpectDecompressError(err: anyerror, compressed: []const u8) !void {
102	const gpa = std.testing.allocator;
103
104	var out: std.ArrayListUnmanaged(u8) = .empty;
105	defer out.deinit(gpa);
106	try out.ensureUnusedCapacity(gpa, default_window_len);
107
108	var in: std.io.BufferedReader = undefined;
109	in.initFixed(@constCast(compressed));
110	var zstd_stream: Decompress = .init(&in, .{});
111	try std.testing.expectError(error.ReadFailed, zstd_stream.reader().readRemainingArrayList(gpa, null, &out, .unlimited));
112	try std.testing.expectError(err, zstd_stream.err.?);
113
114	return error.TestFailed;
115	}
116
117	test "decompression" {
118	const uncompressed = @embedFile("testdata/rfc8478.txt");
119	const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
120	const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
121
122	try testExpectDecompress(uncompressed, compressed3);
123	try testExpectDecompress(uncompressed, compressed19);
124	}
125
126	test "zero sized block" {
127	const input_raw =
128	"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
129	"\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
130	"\x01\x00\x00"; // block header with: last_block set, block_type raw, block_size zero
131
132	const input_rle =
133	"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
134	"\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
135	"\x03\x00\x00" ++ // block header with: last_block set, block_type rle, block_size zero
136	"\xaa"; // block_content
137
138	try testExpectDecompress("", input_raw);
139	try testExpectDecompress("", input_rle);
140	}
141
142	test "declared raw literals size too large" {
143	const input_raw =
144	"\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
145	"\x00\x00" ++ // frame header: everything unset, window descriptor zero
146	"\x95\x00\x00" ++ // block header with: last_block set, block_type compressed, block_size 18
147	"\xbc\xf3\xae" ++ // literals section header with: type raw, size_format 3, regenerated_size 716603
148	"\xa5\x9f\xe3"; // some bytes of literal content - the content is shorter than regenerated_size
149
150	// Note that the regenerated_size in the above input is larger than block maximum size, so the
151	// block can't be valid as it is a raw literals block.
152	try testExpectDecompressError(error.MalformedBlock, input_raw);
153	}

lib/std/compress/zstd/Decompress.zig created+1641

...	@@ -0,0 +1,1641 @@
1	const Decompress = @This();
2	const std = @import("std");
3	const assert = std.debug.assert;
4	const Reader = std.io.Reader;
5	const BufferedWriter = std.io.BufferedWriter;
6	const BufferedReader = std.io.BufferedReader;
7	const zstd = @import("../zstd.zig");
8
9	input: *BufferedReader,
10	state: State,
11	verify_checksum: bool,
12	err: ?Error = null,
13
14	const table_size_max = zstd.compressed_block.table_size_max;
15
16	const State = union(enum) {
17	new_frame,
18	in_frame: InFrame,
19	skipping_frame: u32,
20	end,
21
22	const InFrame = struct {
23	frame: Frame,
24	checksum: ?u32,
25	decompressed_size: usize,
26	};
27	};
28
29	pub const Options = struct {
30	verify_checksum: bool = true,
31	};
32
33	pub const Error = error{
34	ChecksumFailure,
35	DictionaryIdFlagUnsupported,
36	MalformedBlock,
37	MalformedFrame,
38	EndOfStream,
39	};
40
41	pub fn init(input: *BufferedReader, options: Options) Decompress {
42	return .{
43	.input = input,
44	.state = .new_frame,
45	.verify_checksum = options.verify_checksum,
46	};
47	}
48
49	pub fn reader(self: *Decompress) Reader {
50	return .{
51	.context = self,
52	.vtable = &.{
53	.read = read,
54	.readVec = readVec,
55	.discard = discard,
56	},
57	};
58	}
59
60	fn read(context: ?anyopaque, bw: BufferedWriter, limit: Reader.Limit) Reader.RwError!usize {
61	const d: *Decompress = @ptrCast(@alignCast(context));
62	const in = d.input;
63
64	switch (d.state) {
65	.new_frame => {
66	// Allow error.EndOfStream only on the frame magic.
67	const magic = try in.takeEnumNonexhaustive(Frame.Magic, .little);
68	initFrame(d, bw.buffer.len, magic) catch \|err\| {
69	d.err = err;
70	return error.ReadFailed;
71	};
72	return readInFrame(d, bw, limit, &d.state.in_frame) catch \|err\| {
73	d.err = err;
74	return error.ReadFailed;
75	};
76	},
77	.in_frame => \|*in_frame\| {
78	return readInFrame(d, bw, limit, in_frame) catch \|err\| {
79	d.err = err;
80	return error.ReadFailed;
81	};
82	},
83	.skipping_frame => \|*remaining\| {
84	const requested = remaining.*;
85	const n = in.discard(.limited(requested)) catch \|err\| {
86	d.err = err;
87	return error.ReadFailed;
88	};
89	if (requested == n) d.state = .new_frame;
90	remaining.* = requested - n;
91	return 0;
92	},
93	.end => return error.EndOfStream,
94	}
95	}
96
97	fn initFrame(d: *Decompress, window_size_max: usize, magic: Frame.Magic) !void {
98	const in = d.input;
99	switch (magic.kind() orelse return error.BadMagic) {
100	.zstandard => {
101	const header = try Frame.Zstandard.Header.decode(in);
102	d.state = .{ .in_frame = .{
103	.frame = try Frame.init(header, window_size_max, d.verify_checksum),
104	.checksum = null,
105	.decompressed_size = 0,
106	} };
107	},
108	.skippable => {
109	const frame_size = try in.takeInt(u32, .little);
110	d.state = .{ .skipping_frame = frame_size };
111	},
112	}
113	}
114
115	fn readInFrame(d: Decompress, bw: BufferedWriter, limit: Reader.Limit, state: *State.InFrame) !usize {
116	const in = d.input;
117
118	var literal_fse_buffer: [table_size_max.literal]Table.Fse = undefined;
119	var match_fse_buffer: [table_size_max.match]Table.Fse = undefined;
120	var offset_fse_buffer: [table_size_max.offset]Table.Fse = undefined;
121	var literals_buffer: [zstd.block_size_max]u8 = undefined;
122	var sequence_buffer: [zstd.block_size_max]u8 = undefined;
123
124	var decode: Frame.Zstandard.Decode = .init(&literal_fse_buffer, &match_fse_buffer, &offset_fse_buffer);
125
126	const header_bytes = try in.takeArray(3);
127	const block_header: Frame.Zstandard.Block.Header = @bitCast(header_bytes.*);
128	const block_size = block_header.block_size;
129	if (state.frame.block_size_max < block_size) return error.BlockOversize;
130	if (@intFromEnum(limit) < block_size) return error.OutputBufferUndersize;
131	switch (block_header.block_type) {
132	.raw => {
133	try in.readAll(bw, .limited(block_size));
134	return block_size;
135	},
136	.rle => {
137	const byte = try in.takeByte();
138	try bw.splatByteAll(byte, block_size);
139	return block_size;
140	},
141	.compressed => {},
142	.reserved => return error.ReservedBlock,
143	}
144
145	var remaining: Reader.Limit = .limited(block_size);
146	const literals = try LiteralsSection.decode(in, &remaining, &literals_buffer);
147	const sequences_header = try SequencesSection.Header.decode(in, &remaining);
148
149	try decode.prepare(in, &remaining, literals, sequences_header);
150
151	var bytes_written: usize = 0;
152	{
153	if (sequence_buffer.len < @intFromEnum(remaining))
154	return error.SequenceBufferTooSmall;
155	const seq_len = try in.readSlice(remaining.slice(&sequence_buffer));
156	var bit_stream = try ReverseBitReader.init(sequence_buffer[0..seq_len]);
157
158	if (sequences_header.sequence_count > 0) {
159	try decode.readInitialFseState(&bit_stream);
160
161	var sequence_size_limit = state.frame.block_size_max;
162	for (0..sequences_header.sequence_count) \|i\| {
163	const decompressed_size = try decode.decodeSequence(
164	bw,
165	&bit_stream,
166	sequence_size_limit,
167	i == sequences_header.sequence_count - 1,
168	);
169	sequence_size_limit -= decompressed_size;
170	bytes_written += decompressed_size;
171	}
172	}
173
174	if (!bit_stream.isEmpty()) {
175	return error.MalformedCompressedBlock;
176	}
177	}
178
179	if (decode.literal_written_count < literals.header.regenerated_size) {
180	const len = literals.header.regenerated_size - decode.literal_written_count;
181	try decode.decodeLiterals(bw, len);
182	bytes_written += len;
183	}
184
185	switch (decode.literal_header.block_type) {
186	.treeless, .compressed => {
187	if (!decode.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
188	},
189	.raw, .rle => {},
190	}
191
192	if (bytes_written > state.frame.block_size_max) return error.BlockOversize;
193	if (remaining.nonzero()) return error.MalformedCompressedBlock;
194
195	state.decompressed_size += bytes_written;
196
197	if (state.frame.content_size) \|size\| {
198	if (state.decompressed_size > size) return error.MalformedFrame;
199	}
200
201	if (state.frame.hasher_opt) \|*hasher\| {
202	if (bytes_written > 0) {
203	_ = hasher;
204	@panic("TODO all those bytes written needed to go through the hasher too");
205	}
206	}
207
208	if (block_header.last_block) {
209	if (state.frame.has_checksum) {
210	const expected_checksum = try in.readInt(u32, .little);
211	if (state.frame.hasher_opt) \|*hasher\| {
212	const actual_checksum: u32 = @truncate(hasher.final());
213	if (expected_checksum != actual_checksum) return error.ChecksumFailure;
214	}
215	}
216	if (d.frame.content_size) \|content_size\| {
217	if (content_size != d.current_frame_decompressed_size) {
218	return error.MalformedFrame;
219	}
220	}
221	d.state = .new_frame;
222	}
223
224	return bytes_written;
225	}
226
227	fn discard(context: ?*anyopaque, limit: Reader.Limit) Reader.Error!usize {
228	var trash_buffer: [64]u8 = undefined;
229	var null_writer: std.io.Writer.Null = undefined;
230	var bw = null_writer.writer().buffered(&trash_buffer);
231	return @errorCast(read(context, &bw, limit));
232	}
233
234	fn readVec(context: ?*anyopaque, data: []const []u8) Reader.Error!usize {
235	var bw: BufferedWriter = undefined;
236	bw.initFixed(data[0]);
237	return @errorCast(read(context, &bw, .limited(data[0].len)));
238	}
239
240	pub const Frame = struct {
241	hasher_opt: ?std.hash.XxHash64,
242	window_size: usize,
243	has_checksum: bool,
244	block_size_max: usize,
245	content_size: ?usize,
246
247	pub const Magic = enum(u32) {
248	zstandard = 0xFD2FB528,
249	_,
250
251	pub fn kind(m: Magic) ?Kind {
252	return switch (m) {
253	.zstandard => .zstandard,
254	Skippable.magic_min...Skippable.magic_max => .skippable,
255	else => null,
256	};
257	}
258
259	pub fn isSkippable(m: Magic) bool {
260	return switch (m) {
261	Skippable.magic_min...Skippable.magic_max => true,
262	else => false,
263	};
264	}
265	};
266
267	pub const Kind = enum { zstandard, skippable };
268
269	pub const Zstandard = struct {
270	pub const magic: Magic = .zstandard;
271
272	header: Header,
273	data_blocks: []Block,
274	checksum: ?u32,
275
276	pub const Header = struct {
277	descriptor: Descriptor,
278	window_descriptor: ?u8,
279	dictionary_id: ?u32,
280	content_size: ?u64,
281
282	pub const Descriptor = packed struct {
283	dictionary_id_flag: u2,
284	content_checksum_flag: bool,
285	reserved: bool,
286	unused: bool,
287	single_segment_flag: bool,
288	content_size_flag: u2,
289	};
290
291	pub const DecodeError = Reader.Error \|\| error{ReservedBitSet};
292
293	pub fn decode(in: *BufferedReader) DecodeError!Header {
294	const descriptor: Descriptor = @bitCast(try in.takeByte());
295
296	if (descriptor.reserved) return error.ReservedBitSet;
297
298	const window_descriptor: ?u8 = if (descriptor.single_segment_flag) null else try in.takeByte();
299
300	const dictionary_id: ?u32 = if (descriptor.dictionary_id_flag > 0) d: {
301	// if flag is 3 then field_size = 4, else field_size = flag
302	const field_size = (@as(u4, 1) << descriptor.dictionary_id_flag) >> 1;
303	break :d try in.takeVarInt(u32, .little, field_size);
304	} else null;
305
306	const content_size: ?u64 = if (descriptor.single_segment_flag or descriptor.content_size_flag > 0) c: {
307	const field_size = @as(u4, 1) << descriptor.content_size_flag;
308	const content_size = try in.takeVarInt(u64, .little, field_size);
309	break :c if (field_size == 2) content_size + 256 else content_size;
310	} else null;
311
312	return .{
313	.descriptor = descriptor,
314	.window_descriptor = window_descriptor,
315	.dictionary_id = dictionary_id,
316	.content_size = content_size,
317	};
318	}
319
320	/// Returns the window size required to decompress a frame, or `null` if it
321	/// cannot be determined (which indicates a malformed frame header).
322	pub fn windowSize(header: Header) ?u64 {
323	if (header.window_descriptor) \|descriptor\| {
324	const exponent = (descriptor & 0b11111000) >> 3;
325	const mantissa = descriptor & 0b00000111;
326	const window_log = 10 + exponent;
327	const window_base = @as(u64, 1) << @as(u6, @intCast(window_log));
328	const window_add = (window_base / 8) * mantissa;
329	return window_base + window_add;
330	} else return header.content_size;
331	}
332	};
333
334	pub const Block = struct {
335	pub const Header = packed struct(u24) {
336	last: bool,
337	type: Type,
338	size: u21,
339	};
340
341	pub const Type = enum(u2) {
342	raw,
343	rle,
344	compressed,
345	reserved,
346	};
347	};
348
349	pub const Decode = struct {
350	repeat_offsets: [3]u32,
351
352	offset: StateData(8),
353	match: StateData(9),
354	literal: StateData(9),
355
356	offset_fse_buffer: []Table.Fse,
357	match_fse_buffer: []Table.Fse,
358	literal_fse_buffer: []Table.Fse,
359
360	fse_tables_undefined: bool,
361
362	literal_stream_reader: ReverseBitReader,
363	literal_stream_index: usize,
364	literal_streams: LiteralsSection.Streams,
365	literal_header: LiteralsSection.Header,
366	huffman_tree: ?LiteralsSection.HuffmanTree,
367
368	literal_written_count: usize,
369
370	fn StateData(comptime max_accuracy_log: comptime_int) type {
371	return struct {
372	state: @This().State,
373	table: Table,
374	accuracy_log: u8,
375
376	const State = std.meta.Int(.unsigned, max_accuracy_log);
377	};
378	}
379
380	pub fn init(
381	literal_fse_buffer: []Table.Fse,
382	match_fse_buffer: []Table.Fse,
383	offset_fse_buffer: []Table.Fse,
384	) Decode {
385	return .{
386	.repeat_offsets = .{
387	zstd.compressed_block.start_repeated_offset_1,
388	zstd.compressed_block.start_repeated_offset_2,
389	zstd.compressed_block.start_repeated_offset_3,
390	},
391
392	.offset = undefined,
393	.match = undefined,
394	.literal = undefined,
395
396	.literal_fse_buffer = literal_fse_buffer,
397	.match_fse_buffer = match_fse_buffer,
398	.offset_fse_buffer = offset_fse_buffer,
399
400	.fse_tables_undefined = true,
401
402	.literal_written_count = 0,
403	.literal_header = undefined,
404	.literal_streams = undefined,
405	.literal_stream_reader = undefined,
406	.literal_stream_index = undefined,
407	.huffman_tree = null,
408	};
409	}
410
411	pub const PrepareError = error{
412	/// the (reversed) literal bitstream's first byte does not have any bits set
413	BitStreamHasNoStartBit,
414	/// `literals` is a treeless literals section and the decode state does not
415	/// have a Huffman tree from a previous block
416	TreelessLiteralsFirst,
417	/// on the first call if one of the sequence FSE tables is set to repeat mode
418	RepeatModeFirst,
419	/// an FSE table has an invalid accuracy
420	MalformedAccuracyLog,
421	/// failed decoding an FSE table
422	MalformedFseTable,
423	/// input stream ends before all FSE tables are read
424	EndOfStream,
425	};
426
427	/// Prepare the decoder to decode a compressed block. Loads the literals
428	/// stream and Huffman tree from `literals` and reads the FSE tables from
429	/// `source`.
430	pub fn prepare(
431	self: *Decode,
432	in: *BufferedReader,
433	literals: LiteralsSection,
434	sequences_header: SequencesSection.Header,
435	) PrepareError!void {
436	self.literal_written_count = 0;
437	self.literal_header = literals.header;
438	self.literal_streams = literals.streams;
439
440	if (literals.huffman_tree) \|tree\| {
441	self.huffman_tree = tree;
442	} else if (literals.header.block_type == .treeless and self.huffman_tree == null) {
443	return error.TreelessLiteralsFirst;
444	}
445
446	switch (literals.header.block_type) {
447	.raw, .rle => {},
448	.compressed, .treeless => {
449	self.literal_stream_index = 0;
450	switch (literals.streams) {
451	.one => \|slice\| try self.initLiteralStream(slice),
452	.four => \|streams\| try self.initLiteralStream(streams[0]),
453	}
454	},
455	}
456
457	if (sequences_header.sequence_count > 0) {
458	try self.updateFseTable(in, .literal, sequences_header.literal_lengths);
459	try self.updateFseTable(in, .offset, sequences_header.offsets);
460	try self.updateFseTable(in, .match, sequences_header.match_lengths);
461	self.fse_tables_undefined = false;
462	}
463	}
464
465	/// Read initial FSE states for sequence decoding.
466	///
467	/// Errors returned:
468	/// - `error.EndOfStream` if `bit_reader` does not contain enough bits.
469	pub fn readInitialFseState(self: Decode, bit_reader: ReverseBitReader) error{EndOfStream}!void {
470	self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
471	self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
472	self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log);
473	}
474
475	fn updateRepeatOffset(self: *Decode, offset: u32) void {
476	self.repeat_offsets[2] = self.repeat_offsets[1];
477	self.repeat_offsets[1] = self.repeat_offsets[0];
478	self.repeat_offsets[0] = offset;
479	}
480
481	fn useRepeatOffset(self: *Decode, index: usize) u32 {
482	if (index == 1)
483	std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[1])
484	else if (index == 2) {
485	std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[2]);
486	std.mem.swap(u32, &self.repeat_offsets[1], &self.repeat_offsets[2]);
487	}
488	return self.repeat_offsets[0];
489	}
490
491	const DataType = enum { offset, match, literal };
492
493	fn updateState(
494	self: *Decode,
495	comptime choice: DataType,
496	bit_reader: *ReverseBitReader,
497	) error{ MalformedFseBits, EndOfStream }!void {
498	switch (@field(self, @tagName(choice)).table) {
499	.rle => {},
500	.fse => \|table\| {
501	const data = table[@field(self, @tagName(choice)).state];
502	const T = @TypeOf(@field(self, @tagName(choice))).State;
503	const bits_summand = try bit_reader.readBitsNoEof(T, data.bits);
504	const next_state = std.math.cast(
505	@TypeOf(@field(self, @tagName(choice))).State,
506	data.baseline + bits_summand,
507	) orelse return error.MalformedFseBits;
508	@field(self, @tagName(choice)).state = next_state;
509	},
510	}
511	}
512
513	const FseTableError = error{
514	MalformedFseTable,
515	MalformedAccuracyLog,
516	RepeatModeFirst,
517	EndOfStream,
518	};
519
520	fn updateFseTable(
521	self: *Decode,
522	source: *BufferedReader,
523	comptime choice: DataType,
524	mode: SequencesSection.Header.Mode,
525	) !void {
526	const field_name = @tagName(choice);
527	switch (mode) {
528	.predefined => {
529	@field(self, field_name).accuracy_log =
530	@field(zstd.compressed_block.default_accuracy_log, field_name);
531
532	@field(self, field_name).table =
533	@field(Table, "predefined_" ++ field_name);
534	},
535	.rle => {
536	@field(self, field_name).accuracy_log = 0;
537	@field(self, field_name).table = .{ .rle = try source.readByte() };
538	},
539	.fse => {
540	var bit_reader: std.io.BitReader(.little) = .init(source);
541
542	const table_size = try Table.decode(
543	&bit_reader,
544	@field(zstd.compressed_block.table_symbol_count_max, field_name),
545	@field(zstd.compressed_block.table_accuracy_log_max, field_name),
546	@field(self, field_name ++ "_fse_buffer"),
547	);
548	@field(self, field_name).table = .{
549	.fse = @field(self, field_name ++ "_fse_buffer")[0..table_size],
550	};
551	@field(self, field_name).accuracy_log = std.math.log2_int_ceil(usize, table_size);
552	},
553	.repeat => if (self.fse_tables_undefined) return error.RepeatModeFirst,
554	}
555	}
556
557	const Sequence = struct {
558	literal_length: u32,
559	match_length: u32,
560	offset: u32,
561	};
562
563	fn nextSequence(
564	self: *Decode,
565	bit_reader: *ReverseBitReader,
566	) error{ InvalidBitStream, EndOfStream }!Sequence {
567	const raw_code = self.getCode(.offset);
568	const offset_code = std.math.cast(u5, raw_code) orelse {
569	return error.InvalidBitStream;
570	};
571	const offset_value = (@as(u32, 1) << offset_code) + try bit_reader.readBitsNoEof(u32, offset_code);
572
573	const match_code = self.getCode(.match);
574	if (match_code >= zstd.compressed_block.match_length_code_table.len)
575	return error.InvalidBitStream;
576	const match = zstd.compressed_block.match_length_code_table[match_code];
577	const match_length = match[0] + try bit_reader.readBitsNoEof(u32, match[1]);
578
579	const literal_code = self.getCode(.literal);
580	if (literal_code >= zstd.compressed_block.literals_length_code_table.len)
581	return error.InvalidBitStream;
582	const literal = zstd.compressed_block.literals_length_code_table[literal_code];
583	const literal_length = literal[0] + try bit_reader.readBitsNoEof(u32, literal[1]);
584
585	const offset = if (offset_value > 3) offset: {
586	const offset = offset_value - 3;
587	self.updateRepeatOffset(offset);
588	break :offset offset;
589	} else offset: {
590	if (literal_length == 0) {
591	if (offset_value == 3) {
592	const offset = self.repeat_offsets[0] - 1;
593	self.updateRepeatOffset(offset);
594	break :offset offset;
595	}
596	break :offset self.useRepeatOffset(offset_value);
597	}
598	break :offset self.useRepeatOffset(offset_value - 1);
599	};
600
601	if (offset == 0) return error.InvalidBitStream;
602
603	return .{
604	.literal_length = literal_length,
605	.match_length = match_length,
606	.offset = offset,
607	};
608	}
609
610	const DecodeSequenceError = error{
611	/// The decompressed sequence would be longer than
612	/// `sequence_size_limit` or the sequence's offset is too large
613	MalformedSequence,
614	/// The decoder state's literal streams do not contain enough
615	/// literals for the sequence (this may mean the literal stream or the
616	/// sequence is malformed).
617	UnexpectedEndOfLiteralStream,
618	/// The FSE sequence bitstream is malformed
619	InvalidBitStream,
620	/// `bit_reader` does not contain enough bits
621	EndOfStream,
622	/// The `BufferedWriter` storage capacity is not large enough to
623	/// accept this stream.
624	OutputBufferUndersize,
625	};
626
627	/// Decode one sequence from `bit_reader` into `dest`. Updates FSE states
628	/// if `last_sequence` is `false`. Assumes `prepare` called for the block
629	/// before attempting to decode sequences.
630	pub fn decodeSequence(
631	self: *Decode,
632	dest: *BufferedWriter,
633	bit_reader: *ReverseBitReader,
634	sequence_size_limit: usize,
635	last_sequence: bool,
636	) DecodeSequenceError!usize {
637	const sequence = try self.nextSequence(bit_reader);
638	const literal_length: usize = sequence.literal_length;
639	const match_length: usize = sequence.match_length;
640	const sequence_length = literal_length + match_length;
641	if (sequence_length > sequence_size_limit) return error.MalformedSequence;
642	if (sequence_length > dest.buffer.len) return error.OutputBufferUndersize;
643
644	if (sequence.offset > literal_length) return error.MalformedSequence;
645	// Ensures the following call to `decodeLiterals` will not cause a
646	// flush and therefore be at the beginning of `out`.
647	const out = try dest.writableSlice(sequence_length);
648	const expected_end = dest.end + out.len;
649	try decodeLiterals(self, dest, literal_length);
650	@memmove(
651	out[literal_length..][0..match_length],
652	out[literal_length - sequence.offset ..][0..match_length],
653	);
654	dest.advance(match_length);
655	assert(dest.end == expected_end);
656
657	if (!last_sequence) {
658	try self.updateState(.literal, bit_reader);
659	try self.updateState(.match, bit_reader);
660	try self.updateState(.offset, bit_reader);
661	}
662	return sequence_length;
663	}
664
665	fn nextLiteralMultiStream(self: *Decode) error{BitStreamHasNoStartBit}!void {
666	self.literal_stream_index += 1;
667	try self.initLiteralStream(self.literal_streams.four[self.literal_stream_index]);
668	}
669
670	fn initLiteralStream(self: *Decode, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
671	try self.literal_stream_reader.init(bytes);
672	}
673
674	fn isLiteralStreamEmpty(self: *Decode) bool {
675	switch (self.literal_streams) {
676	.one => return self.literal_stream_reader.isEmpty(),
677	.four => return self.literal_stream_index == 3 and self.literal_stream_reader.isEmpty(),
678	}
679	}
680
681	const LiteralBitsError = error{
682	BitStreamHasNoStartBit,
683	UnexpectedEndOfLiteralStream,
684	};
685	fn readLiteralsBits(
686	self: *Decode,
687	bit_count_to_read: u16,
688	) LiteralBitsError!u16 {
689	return self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch bits: {
690	if (self.literal_streams == .four and self.literal_stream_index < 3) {
691	try self.nextLiteralMultiStream();
692	break :bits self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch
693	return error.UnexpectedEndOfLiteralStream;
694	} else {
695	return error.UnexpectedEndOfLiteralStream;
696	}
697	};
698	}
699
700	const DecodeLiteralsError = error{
701	/// The number of literal bytes decoded by `self` plus `len` is greater
702	/// than the regenerated size of `literals`
703	MalformedLiteralsLength,
704	/// Problems decoding Huffman compressed literals
705	UnexpectedEndOfLiteralStream,
706	OutputBufferUndersize,
707	};
708
709	/// Decode `len` bytes of literals into `dest`.
710	pub fn decodeLiterals(self: Decode, dest: BufferedWriter, len: usize) DecodeLiteralsError!void {
711	if (self.literal_written_count + len > self.literal_header.regenerated_size)
712	return error.MalformedLiteralsLength;
713
714	switch (self.literal_header.block_type) {
715	.raw => {
716	try dest.writeAll(self.literal_streams.one[self.literal_written_count..][0..len]);
717	self.literal_written_count += len;
718	},
719	.rle => {
720	try dest.splatByteAll(self.literal_streams.one[0], len);
721	self.literal_written_count += len;
722	},
723	.compressed, .treeless => {
724	if (len > dest.buffer.len) return error.OutputBufferUndersize;
725	const buf = try dest.writableSlice(len);
726	const huffman_tree = self.huffman_tree.?;
727	const max_bit_count = huffman_tree.max_bit_count;
728	const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
729	huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
730	max_bit_count,
731	);
732	var bits_read: u4 = 0;
733	var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
734	var bit_count_to_read: u4 = starting_bit_count;
735	for (buf) \|*out\| {
736	var prefix: u16 = 0;
737	while (true) {
738	const new_bits = try self.readLiteralsBits(bit_count_to_read);
739	prefix <<= bit_count_to_read;
740	prefix \|= new_bits;
741	bits_read += bit_count_to_read;
742	const result = try huffman_tree.query(huffman_tree_index, prefix);
743
744	switch (result) {
745	.symbol => \|sym\| {
746	out.* = sym;
747	bit_count_to_read = starting_bit_count;
748	bits_read = 0;
749	huffman_tree_index = huffman_tree.symbol_count_minus_one;
750	break;
751	},
752	.index => \|index\| {
753	huffman_tree_index = index;
754	const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
755	huffman_tree.nodes[index].weight,
756	max_bit_count,
757	);
758	bit_count_to_read = bit_count - bits_read;
759	},
760	}
761	}
762	}
763	self.literal_written_count += len;
764	},
765	}
766	}
767
768	fn getCode(self: *Decode, comptime choice: DataType) u32 {
769	return switch (@field(self, @tagName(choice)).table) {
770	.rle => \|value\| value,
771	.fse => \|table\| table[@field(self, @tagName(choice)).state].symbol,
772	};
773	}
774	};
775	};
776
777	pub const Skippable = struct {
778	pub const magic_min: Magic = @enumFromInt(0x184D2A50);
779	pub const magic_max: Magic = @enumFromInt(0x184D2A5F);
780
781	pub const Header = struct {
782	magic_number: u32,
783	frame_size: u32,
784	};
785	};
786
787	const InitError = error{
788	DictionaryIdFlagUnsupported,
789	WindowSizeUnknown,
790	WindowTooLarge,
791	ContentSizeTooLarge,
792	};
793	/// Validates `frame_header` and returns the associated `Frame`.
794	///
795	/// Errors returned:
796	/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
797	/// - `error.WindowSizeUnknown` if the frame does not have a valid window
798	/// size
799	/// - `error.WindowTooLarge` if the window size is larger than
800	/// `window_size_max` or `std.math.intMax(usize)`
801	/// - `error.ContentSizeTooLarge` if the frame header indicates a content
802	/// size larger than `std.math.maxInt(usize)`
803	pub fn init(
804	frame_header: Frame.Zstandard.Header,
805	window_size_max: usize,
806	verify_checksum: bool,
807	) InitError!Frame {
808	if (frame_header.descriptor.dictionary_id_flag != 0)
809	return error.DictionaryIdFlagUnsupported;
810
811	const window_size_raw = frame_header.windowSize() orelse return error.WindowSizeUnknown;
812	const window_size = if (window_size_raw > window_size_max)
813	return error.WindowTooLarge
814	else
815	std.math.cast(usize, window_size_raw) orelse return error.WindowTooLarge;
816
817	const should_compute_checksum =
818	frame_header.descriptor.content_checksum_flag and verify_checksum;
819
820	const content_size = if (frame_header.content_size) \|size\|
821	std.math.cast(usize, size) orelse return error.ContentSizeTooLarge
822	else
823	null;
824
825	return .{
826	.hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,
827	.window_size = window_size,
828	.has_checksum = frame_header.descriptor.content_checksum_flag,
829	.block_size_max = @min(zstd.block_size_max, window_size),
830	.content_size = content_size,
831	};
832	}
833	};
834
835	pub const LiteralsSection = struct {
836	header: Header,
837	huffman_tree: ?HuffmanTree,
838	streams: Streams,
839
840	pub const Streams = union(enum) {
841	one: []const u8,
842	four: [4][]const u8,
843
844	fn decode(size_format: u2, stream_data: []const u8) !Streams {
845	if (size_format == 0) {
846	return .{ .one = stream_data };
847	}
848
849	if (stream_data.len < 6) return error.MalformedLiteralsSection;
850
851	const stream_1_length: usize = std.mem.readInt(u16, stream_data[0..2], .little);
852	const stream_2_length: usize = std.mem.readInt(u16, stream_data[2..4], .little);
853	const stream_3_length: usize = std.mem.readInt(u16, stream_data[4..6], .little);
854
855	const stream_1_start = 6;
856	const stream_2_start = stream_1_start + stream_1_length;
857	const stream_3_start = stream_2_start + stream_2_length;
858	const stream_4_start = stream_3_start + stream_3_length;
859
860	if (stream_data.len < stream_4_start) return error.MalformedLiteralsSection;
861
862	return .{ .four = .{
863	stream_data[stream_1_start .. stream_1_start + stream_1_length],
864	stream_data[stream_2_start .. stream_2_start + stream_2_length],
865	stream_data[stream_3_start .. stream_3_start + stream_3_length],
866	stream_data[stream_4_start..],
867	} };
868	}
869	};
870
871	pub const Header = struct {
872	block_type: BlockType,
873	size_format: u2,
874	regenerated_size: u20,
875	compressed_size: ?u18,
876
877	/// Decode a literals section header.
878	///
879	/// Errors returned:
880	/// - `error.EndOfStream` if there are not enough bytes in `source`
881	pub fn decode(source: *BufferedReader) !Header {
882	const byte0 = try source.readByte();
883	const block_type = @as(BlockType, @enumFromInt(byte0 & 0b11));
884	const size_format = @as(u2, @intCast((byte0 & 0b1100) >> 2));
885	var regenerated_size: u20 = undefined;
886	var compressed_size: ?u18 = null;
887	switch (block_type) {
888	.raw, .rle => {
889	switch (size_format) {
890	0, 2 => {
891	regenerated_size = byte0 >> 3;
892	},
893	1 => regenerated_size = (byte0 >> 4) + (@as(u20, try source.readByte()) << 4),
894	3 => regenerated_size = (byte0 >> 4) +
895	(@as(u20, try source.readByte()) << 4) +
896	(@as(u20, try source.readByte()) << 12),
897	}
898	},
899	.compressed, .treeless => {
900	const byte1 = try source.readByte();
901	const byte2 = try source.readByte();
902	switch (size_format) {
903	0, 1 => {
904	regenerated_size = (byte0 >> 4) + ((@as(u20, byte1) & 0b00111111) << 4);
905	compressed_size = ((byte1 & 0b11000000) >> 6) + (@as(u18, byte2) << 2);
906	},
907	2 => {
908	const byte3 = try source.readByte();
909	regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00000011) << 12);
910	compressed_size = ((byte2 & 0b11111100) >> 2) + (@as(u18, byte3) << 6);
911	},
912	3 => {
913	const byte3 = try source.readByte();
914	const byte4 = try source.readByte();
915	regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00111111) << 12);
916	compressed_size = ((byte2 & 0b11000000) >> 6) + (@as(u18, byte3) << 2) + (@as(u18, byte4) << 10);
917	},
918	}
919	},
920	}
921	return .{
922	.block_type = block_type,
923	.size_format = size_format,
924	.regenerated_size = regenerated_size,
925	.compressed_size = compressed_size,
926	};
927	}
928	};
929
930	pub const BlockType = enum(u2) {
931	raw,
932	rle,
933	compressed,
934	treeless,
935	};
936
937	pub const HuffmanTree = struct {
938	max_bit_count: u4,
939	symbol_count_minus_one: u8,
940	nodes: [256]PrefixedSymbol,
941
942	pub const PrefixedSymbol = struct {
943	symbol: u8,
944	prefix: u16,
945	weight: u4,
946	};
947
948	pub const Result = union(enum) {
949	symbol: u8,
950	index: usize,
951	};
952
953	pub fn query(self: HuffmanTree, index: usize, prefix: u16) error{NotFound}!Result {
954	var node = self.nodes[index];
955	const weight = node.weight;
956	var i: usize = index;
957	while (node.weight == weight) {
958	if (node.prefix == prefix) return Result{ .symbol = node.symbol };
959	if (i == 0) return error.NotFound;
960	i -= 1;
961	node = self.nodes[i];
962	}
963	return Result{ .index = i };
964	}
965
966	pub fn weightToBitCount(weight: u4, max_bit_count: u4) u4 {
967	return if (weight == 0) 0 else ((max_bit_count + 1) - weight);
968	}
969
970	pub const DecodeError = Reader.Error \|\| error{
971	MalformedHuffmanTree,
972	MalformedFseTable,
973	MalformedAccuracyLog,
974	EndOfStream,
975	MissingStartBit,
976	};
977
978	pub fn decode(in: *BufferedReader) HuffmanTree.DecodeError!HuffmanTree {
979	const header = try in.takeByte();
980	if (header < 128) {
981	return decodeFse(in, header);
982	} else {
983	return decodeDirect(in, header - 127);
984	}
985	}
986
987	fn decodeDirect(source: *BufferedReader, encoded_symbol_count: usize) HuffmanTree.DecodeError!HuffmanTree {
988	var weights: [256]u4 = undefined;
989	const weights_byte_count = (encoded_symbol_count + 1) / 2;
990	for (0..weights_byte_count) \|i\| {
991	const byte = try source.takeByte();
992	weights[2 * i] = @as(u4, @intCast(byte >> 4));
993	weights[2 * i + 1] = @as(u4, @intCast(byte & 0xF));
994	}
995	const symbol_count = encoded_symbol_count + 1;
996	return build(&weights, symbol_count);
997	}
998
999	fn decodeFse(in: *BufferedReader, compressed_size: usize) HuffmanTree.DecodeError!HuffmanTree {
1000	var weights: [256]u4 = undefined;
1001	const compressed_buffer = try in.take(compressed_size);
1002	var limited_stream: BufferedReader = undefined;
1003	limited_stream.initFixed(compressed_buffer);
1004	var bit_reader: std.io.BitReader(.little) = .init(&limited_stream);
1005	var entries: [1 << 6]Table.Fse = undefined;
1006	const table_size = try Table.decode(&bit_reader, 256, 6, &entries);
1007	const accuracy_log = std.math.log2_int_ceil(usize, table_size);
1008	const remaining = limited_stream.bufferContents();
1009	const symbol_count = try assignWeights(remaining, accuracy_log, &entries, weights);
1010	return build(&weights, symbol_count);
1011	}
1012
1013	fn assignWeights(
1014	huff_bits_buffer: []u8,
1015	accuracy_log: u16,
1016	entries: *[1 << 6]Table.Fse,
1017	weights: *[256]u4,
1018	) !usize {
1019	var huff_bits = try ReverseBitReader.init(huff_bits_buffer);
1020
1021	var i: usize = 0;
1022	var even_state: u32 = try huff_bits.readBitsNoEof(u32, accuracy_log);
1023	var odd_state: u32 = try huff_bits.readBitsNoEof(u32, accuracy_log);
1024
1025	while (i < 254) {
1026	const even_data = entries[even_state];
1027	var read_bits: u16 = 0;
1028	const even_bits = huff_bits.readBits(u32, even_data.bits, &read_bits) catch unreachable;
1029	weights[i] = std.math.cast(u4, even_data.symbol) orelse return error.MalformedHuffmanTree;
1030	i += 1;
1031	if (read_bits < even_data.bits) {
1032	weights[i] = std.math.cast(u4, entries[odd_state].symbol) orelse return error.MalformedHuffmanTree;
1033	i += 1;
1034	break;
1035	}
1036	even_state = even_data.baseline + even_bits;
1037
1038	read_bits = 0;
1039	const odd_data = entries[odd_state];
1040	const odd_bits = huff_bits.readBits(u32, odd_data.bits, &read_bits) catch unreachable;
1041	weights[i] = std.math.cast(u4, odd_data.symbol) orelse return error.MalformedHuffmanTree;
1042	i += 1;
1043	if (read_bits < odd_data.bits) {
1044	if (i == 255) return error.MalformedHuffmanTree;
1045	weights[i] = std.math.cast(u4, entries[even_state].symbol) orelse return error.MalformedHuffmanTree;
1046	i += 1;
1047	break;
1048	}
1049	odd_state = odd_data.baseline + odd_bits;
1050	} else return error.MalformedHuffmanTree;
1051
1052	if (!huff_bits.isEmpty()) {
1053	return error.MalformedHuffmanTree;
1054	}
1055
1056	return i + 1; // stream contains all but the last symbol
1057	}
1058
1059	fn assignSymbols(weight_sorted_prefixed_symbols: []PrefixedSymbol, weights: [256]u4) usize {
1060	for (0..weight_sorted_prefixed_symbols.len) \|i\| {
1061	weight_sorted_prefixed_symbols[i] = .{
1062	.symbol = @as(u8, @intCast(i)),
1063	.weight = undefined,
1064	.prefix = undefined,
1065	};
1066	}
1067
1068	std.mem.sort(
1069	PrefixedSymbol,
1070	weight_sorted_prefixed_symbols,
1071	weights,
1072	lessThanByWeight,
1073	);
1074
1075	var prefix: u16 = 0;
1076	var prefixed_symbol_count: usize = 0;
1077	var sorted_index: usize = 0;
1078	const symbol_count = weight_sorted_prefixed_symbols.len;
1079	while (sorted_index < symbol_count) {
1080	var symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
1081	const weight = weights[symbol];
1082	if (weight == 0) {
1083	sorted_index += 1;
1084	continue;
1085	}
1086
1087	while (sorted_index < symbol_count) : ({
1088	sorted_index += 1;
1089	prefixed_symbol_count += 1;
1090	prefix += 1;
1091	}) {
1092	symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
1093	if (weights[symbol] != weight) {
1094	prefix = ((prefix - 1) >> (weights[symbol] - weight)) + 1;
1095	break;
1096	}
1097	weight_sorted_prefixed_symbols[prefixed_symbol_count].symbol = symbol;
1098	weight_sorted_prefixed_symbols[prefixed_symbol_count].prefix = prefix;
1099	weight_sorted_prefixed_symbols[prefixed_symbol_count].weight = weight;
1100	}
1101	}
1102	return prefixed_symbol_count;
1103	}
1104
1105	fn build(weights: *[256]u4, symbol_count: usize) error{MalformedHuffmanTree}!HuffmanTree {
1106	var weight_power_sum_big: u32 = 0;
1107	for (weights[0 .. symbol_count - 1]) \|value\| {
1108	weight_power_sum_big += (@as(u16, 1) << value) >> 1;
1109	}
1110	if (weight_power_sum_big >= 1 << 11) return error.MalformedHuffmanTree;
1111	const weight_power_sum = @as(u16, @intCast(weight_power_sum_big));
1112
1113	// advance to next power of two (even if weight_power_sum is a power of 2)
1114	// TODO: is it valid to have weight_power_sum == 0?
1115	const max_number_of_bits = if (weight_power_sum == 0) 1 else std.math.log2_int(u16, weight_power_sum) + 1;
1116	const next_power_of_two = @as(u16, 1) << max_number_of_bits;
1117	weights[symbol_count - 1] = std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1;
1118
1119	var weight_sorted_prefixed_symbols: [256]PrefixedSymbol = undefined;
1120	const prefixed_symbol_count = assignSymbols(weight_sorted_prefixed_symbols[0..symbol_count], weights.*);
1121	const tree: HuffmanTree = .{
1122	.max_bit_count = max_number_of_bits,
1123	.symbol_count_minus_one = @as(u8, @intCast(prefixed_symbol_count - 1)),
1124	.nodes = weight_sorted_prefixed_symbols,
1125	};
1126	return tree;
1127	}
1128
1129	fn lessThanByWeight(
1130	weights: [256]u4,
1131	lhs: PrefixedSymbol,
1132	rhs: PrefixedSymbol,
1133	) bool {
1134	// NOTE: this function relies on the use of a stable sorting algorithm,
1135	// otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs;
1136	// should be added
1137	return weights[lhs.symbol] < weights[rhs.symbol];
1138	}
1139	};
1140
1141	pub const StreamCount = enum { one, four };
1142	pub fn streamCount(size_format: u2, block_type: BlockType) StreamCount {
1143	return switch (block_type) {
1144	.raw, .rle => .one,
1145	.compressed, .treeless => if (size_format == 0) .one else .four,
1146	};
1147	}
1148
1149	pub const DecodeError = error{
1150	/// Invalid header.
1151	MalformedLiteralsHeader,
1152	/// Decoding errors.
1153	MalformedLiteralsSection,
1154	/// Compressed literals have invalid accuracy.
1155	MalformedAccuracyLog,
1156	/// Compressed literals have invalid FSE table.
1157	MalformedFseTable,
1158	/// Failed decoding a Huffamn tree.
1159	MalformedHuffmanTree,
1160	/// Not enough bytes to complete the section.
1161	EndOfStream,
1162	};
1163
1164	pub fn decode(source: *BufferedReader, buffer: []u8) DecodeError!LiteralsSection {
1165	const header = try Header.decode(source);
1166	switch (header.block_type) {
1167	.raw => {
1168	if (buffer.len < header.regenerated_size) return error.LiteralsBufferTooSmall;
1169	try source.readNoEof(buffer[0..header.regenerated_size]);
1170	return .{
1171	.header = header,
1172	.huffman_tree = null,
1173	.streams = .{ .one = buffer },
1174	};
1175	},
1176	.rle => {
1177	buffer[0] = try source.readByte();
1178	return .{
1179	.header = header,
1180	.huffman_tree = null,
1181	.streams = .{ .one = buffer[0..1] },
1182	};
1183	},
1184	.compressed, .treeless => {
1185	var counting_reader = std.io.countingReader(source);
1186	const huffman_tree = if (header.block_type == .compressed)
1187	try HuffmanTree.decode(counting_reader.reader(), buffer)
1188	else
1189	null;
1190	const huffman_tree_size = @as(usize, @intCast(counting_reader.bytes_read));
1191	const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
1192	return error.MalformedLiteralsSection;
1193
1194	if (total_streams_size > buffer.len) return error.LiteralsBufferTooSmall;
1195	try source.readNoEof(buffer[0..total_streams_size]);
1196	const stream_data = buffer[0..total_streams_size];
1197
1198	const streams = try Streams.decode(header.size_format, stream_data);
1199	return .{
1200	.header = header,
1201	.huffman_tree = huffman_tree,
1202	.streams = streams,
1203	};
1204	},
1205	}
1206	}
1207	};
1208
1209	pub const SequencesSection = struct {
1210	header: SequencesSection.Header,
1211	literals_length_table: Table,
1212	offset_table: Table,
1213	match_length_table: Table,
1214
1215	pub const Header = struct {
1216	sequence_count: u24,
1217	match_lengths: Mode,
1218	offsets: Mode,
1219	literal_lengths: Mode,
1220
1221	pub const Mode = enum(u2) {
1222	predefined,
1223	rle,
1224	fse,
1225	repeat,
1226	};
1227
1228	pub const DecodeError = error{
1229	ReservedBitSet,
1230	EndOfStream,
1231	};
1232
1233	pub fn decode(source: *BufferedReader) DecodeError!Header {
1234	var sequence_count: u24 = undefined;
1235
1236	const byte0 = try source.readByte();
1237	if (byte0 == 0) {
1238	return SequencesSection.Header{
1239	.sequence_count = 0,
1240	.offsets = undefined,
1241	.match_lengths = undefined,
1242	.literal_lengths = undefined,
1243	};
1244	} else if (byte0 < 128) {
1245	sequence_count = byte0;
1246	} else if (byte0 < 255) {
1247	sequence_count = (@as(u24, (byte0 - 128)) << 8) + try source.readByte();
1248	} else {
1249	sequence_count = (try source.readByte()) + (@as(u24, try source.readByte()) << 8) + 0x7F00;
1250	}
1251
1252	const compression_modes = try source.readByte();
1253
1254	const matches_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00001100) >> 2));
1255	const offsets_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00110000) >> 4));
1256	const literal_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b11000000) >> 6));
1257	if (compression_modes & 0b11 != 0) return error.ReservedBitSet;
1258
1259	return .{
1260	.sequence_count = sequence_count,
1261	.offsets = offsets_mode,
1262	.match_lengths = matches_mode,
1263	.literal_lengths = literal_mode,
1264	};
1265	}
1266	};
1267	};
1268
1269	pub const Table = union(enum) {
1270	fse: []const Fse,
1271	rle: u8,
1272
1273	pub const Fse = struct {
1274	symbol: u8,
1275	baseline: u16,
1276	bits: u8,
1277	};
1278
1279	pub fn decode(
1280	bit_reader: *std.io.BitReader(.little),
1281	expected_symbol_count: usize,
1282	max_accuracy_log: u4,
1283	entries: []Table.Fse,
1284	) !usize {
1285	const accuracy_log_biased = try bit_reader.readBitsNoEof(u4, 4);
1286	if (accuracy_log_biased > max_accuracy_log -\| 5) return error.MalformedAccuracyLog;
1287	const accuracy_log = accuracy_log_biased + 5;
1288
1289	var values: [256]u16 = undefined;
1290	var value_count: usize = 0;
1291
1292	const total_probability = @as(u16, 1) << accuracy_log;
1293	var accumulated_probability: u16 = 0;
1294
1295	while (accumulated_probability < total_probability) {
1296	// WARNING: The RFC is poorly worded, and would suggest std.math.log2_int_ceil is correct here,
1297	// but power of two (remaining probabilities + 1) need max bits set to 1 more.
1298	const max_bits = std.math.log2_int(u16, total_probability - accumulated_probability + 1) + 1;
1299	const small = try bit_reader.readBitsNoEof(u16, max_bits - 1);
1300
1301	const cutoff = (@as(u16, 1) << max_bits) - 1 - (total_probability - accumulated_probability + 1);
1302
1303	const value = if (small < cutoff)
1304	small
1305	else value: {
1306	const value_read = small + (try bit_reader.readBitsNoEof(u16, 1) << (max_bits - 1));
1307	break :value if (value_read < @as(u16, 1) << (max_bits - 1))
1308	value_read
1309	else
1310	value_read - cutoff;
1311	};
1312
1313	accumulated_probability += if (value != 0) value - 1 else 1;
1314
1315	values[value_count] = value;
1316	value_count += 1;
1317
1318	if (value == 1) {
1319	while (true) {
1320	const repeat_flag = try bit_reader.readBitsNoEof(u2, 2);
1321	if (repeat_flag + value_count > 256) return error.MalformedFseTable;
1322	for (0..repeat_flag) \|_\| {
1323	values[value_count] = 1;
1324	value_count += 1;
1325	}
1326	if (repeat_flag < 3) break;
1327	}
1328	}
1329	if (value_count == 256) break;
1330	}
1331	bit_reader.alignToByte();
1332
1333	if (value_count < 2) return error.MalformedFseTable;
1334	if (accumulated_probability != total_probability) return error.MalformedFseTable;
1335	if (value_count > expected_symbol_count) return error.MalformedFseTable;
1336
1337	const table_size = total_probability;
1338
1339	try build(values[0..value_count], entries[0..table_size]);
1340	return table_size;
1341	}
1342
1343	pub fn build(values: []const u16, entries: []Table.Fse) !void {
1344	const total_probability = @as(u16, @intCast(entries.len));
1345	const accuracy_log = std.math.log2_int(u16, total_probability);
1346	assert(total_probability <= 1 << 9);
1347
1348	var less_than_one_count: usize = 0;
1349	for (values, 0..) \|value, i\| {
1350	if (value == 0) {
1351	entries[entries.len - 1 - less_than_one_count] = Table.Fse{
1352	.symbol = @as(u8, @intCast(i)),
1353	.baseline = 0,
1354	.bits = accuracy_log,
1355	};
1356	less_than_one_count += 1;
1357	}
1358	}
1359
1360	var position: usize = 0;
1361	var temp_states: [1 << 9]u16 = undefined;
1362	for (values, 0..) \|value, symbol\| {
1363	if (value == 0 or value == 1) continue;
1364	const probability = value - 1;
1365
1366	const state_share_dividend = std.math.ceilPowerOfTwo(u16, probability) catch
1367	return error.MalformedFseTable;
1368	const share_size = @divExact(total_probability, state_share_dividend);
1369	const double_state_count = state_share_dividend - probability;
1370	const single_state_count = probability - double_state_count;
1371	const share_size_log = std.math.log2_int(u16, share_size);
1372
1373	for (0..probability) \|i\| {
1374	temp_states[i] = @as(u16, @intCast(position));
1375	position += (entries.len >> 1) + (entries.len >> 3) + 3;
1376	position &= entries.len - 1;
1377	while (position >= entries.len - less_than_one_count) {
1378	position += (entries.len >> 1) + (entries.len >> 3) + 3;
1379	position &= entries.len - 1;
1380	}
1381	}
1382	std.mem.sort(u16, temp_states[0..probability], {}, std.sort.asc(u16));
1383	for (0..probability) \|i\| {
1384	entries[temp_states[i]] = if (i < double_state_count) Table.Fse{
1385	.symbol = @as(u8, @intCast(symbol)),
1386	.bits = share_size_log + 1,
1387	.baseline = single_state_count * share_size + @as(u16, @intCast(i)) * 2 * share_size,
1388	} else Table.Fse{
1389	.symbol = @as(u8, @intCast(symbol)),
1390	.bits = share_size_log,
1391	.baseline = (@as(u16, @intCast(i)) - double_state_count) * share_size,
1392	};
1393	}
1394	}
1395	}
1396
1397	test build {
1398	const literals_length_default_values = [36]u16{
1399	5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2,
1400	3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 2, 2, 2, 2, 2,
1401	0, 0, 0, 0,
1402	};
1403
1404	const match_lengths_default_values = [53]u16{
1405	2, 5, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
1406	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
1407	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0,
1408	0, 0, 0, 0, 0,
1409	};
1410
1411	const offset_codes_default_values = [29]u16{
1412	2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
1413	2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0,
1414	};
1415
1416	var entries: [64]Table.Fse = undefined;
1417	try build(&literals_length_default_values, &entries);
1418	try std.testing.expectEqualSlices(Table.Fse, Table.predefined_literal.fse, &entries);
1419
1420	try build(&match_lengths_default_values, &entries);
1421	try std.testing.expectEqualSlices(Table.Fse, Table.predefined_match.fse, &entries);
1422
1423	try build(&offset_codes_default_values, entries[0..32]);
1424	try std.testing.expectEqualSlices(Table.Fse, Table.predefined_offset.fse, entries[0..32]);
1425	}
1426
1427	pub const predefined_literal: Table = .{
1428	.fse = &[64]Table.Fse{
1429	.{ .symbol = 0, .bits = 4, .baseline = 0 },
1430	.{ .symbol = 0, .bits = 4, .baseline = 16 },
1431	.{ .symbol = 1, .bits = 5, .baseline = 32 },
1432	.{ .symbol = 3, .bits = 5, .baseline = 0 },
1433	.{ .symbol = 4, .bits = 5, .baseline = 0 },
1434	.{ .symbol = 6, .bits = 5, .baseline = 0 },
1435	.{ .symbol = 7, .bits = 5, .baseline = 0 },
1436	.{ .symbol = 9, .bits = 5, .baseline = 0 },
1437	.{ .symbol = 10, .bits = 5, .baseline = 0 },
1438	.{ .symbol = 12, .bits = 5, .baseline = 0 },
1439	.{ .symbol = 14, .bits = 6, .baseline = 0 },
1440	.{ .symbol = 16, .bits = 5, .baseline = 0 },
1441	.{ .symbol = 18, .bits = 5, .baseline = 0 },
1442	.{ .symbol = 19, .bits = 5, .baseline = 0 },
1443	.{ .symbol = 21, .bits = 5, .baseline = 0 },
1444	.{ .symbol = 22, .bits = 5, .baseline = 0 },
1445	.{ .symbol = 24, .bits = 5, .baseline = 0 },
1446	.{ .symbol = 25, .bits = 5, .baseline = 32 },
1447	.{ .symbol = 26, .bits = 5, .baseline = 0 },
1448	.{ .symbol = 27, .bits = 6, .baseline = 0 },
1449	.{ .symbol = 29, .bits = 6, .baseline = 0 },
1450	.{ .symbol = 31, .bits = 6, .baseline = 0 },
1451	.{ .symbol = 0, .bits = 4, .baseline = 32 },
1452	.{ .symbol = 1, .bits = 4, .baseline = 0 },
1453	.{ .symbol = 2, .bits = 5, .baseline = 0 },
1454	.{ .symbol = 4, .bits = 5, .baseline = 32 },
1455	.{ .symbol = 5, .bits = 5, .baseline = 0 },
1456	.{ .symbol = 7, .bits = 5, .baseline = 32 },
1457	.{ .symbol = 8, .bits = 5, .baseline = 0 },
1458	.{ .symbol = 10, .bits = 5, .baseline = 32 },
1459	.{ .symbol = 11, .bits = 5, .baseline = 0 },
1460	.{ .symbol = 13, .bits = 6, .baseline = 0 },
1461	.{ .symbol = 16, .bits = 5, .baseline = 32 },
1462	.{ .symbol = 17, .bits = 5, .baseline = 0 },
1463	.{ .symbol = 19, .bits = 5, .baseline = 32 },
1464	.{ .symbol = 20, .bits = 5, .baseline = 0 },
1465	.{ .symbol = 22, .bits = 5, .baseline = 32 },
1466	.{ .symbol = 23, .bits = 5, .baseline = 0 },
1467	.{ .symbol = 25, .bits = 4, .baseline = 0 },
1468	.{ .symbol = 25, .bits = 4, .baseline = 16 },
1469	.{ .symbol = 26, .bits = 5, .baseline = 32 },
1470	.{ .symbol = 28, .bits = 6, .baseline = 0 },
1471	.{ .symbol = 30, .bits = 6, .baseline = 0 },
1472	.{ .symbol = 0, .bits = 4, .baseline = 48 },
1473	.{ .symbol = 1, .bits = 4, .baseline = 16 },
1474	.{ .symbol = 2, .bits = 5, .baseline = 32 },
1475	.{ .symbol = 3, .bits = 5, .baseline = 32 },
1476	.{ .symbol = 5, .bits = 5, .baseline = 32 },
1477	.{ .symbol = 6, .bits = 5, .baseline = 32 },
1478	.{ .symbol = 8, .bits = 5, .baseline = 32 },
1479	.{ .symbol = 9, .bits = 5, .baseline = 32 },
1480	.{ .symbol = 11, .bits = 5, .baseline = 32 },
1481	.{ .symbol = 12, .bits = 5, .baseline = 32 },
1482	.{ .symbol = 15, .bits = 6, .baseline = 0 },
1483	.{ .symbol = 17, .bits = 5, .baseline = 32 },
1484	.{ .symbol = 18, .bits = 5, .baseline = 32 },
1485	.{ .symbol = 20, .bits = 5, .baseline = 32 },
1486	.{ .symbol = 21, .bits = 5, .baseline = 32 },
1487	.{ .symbol = 23, .bits = 5, .baseline = 32 },
1488	.{ .symbol = 24, .bits = 5, .baseline = 32 },
1489	.{ .symbol = 35, .bits = 6, .baseline = 0 },
1490	.{ .symbol = 34, .bits = 6, .baseline = 0 },
1491	.{ .symbol = 33, .bits = 6, .baseline = 0 },
1492	.{ .symbol = 32, .bits = 6, .baseline = 0 },
1493	},
1494	};
1495
1496	pub const predefined_match: Table = .{
1497	.fse = &[64]Table.Fse{
1498	.{ .symbol = 0, .bits = 6, .baseline = 0 },
1499	.{ .symbol = 1, .bits = 4, .baseline = 0 },
1500	.{ .symbol = 2, .bits = 5, .baseline = 32 },
1501	.{ .symbol = 3, .bits = 5, .baseline = 0 },
1502	.{ .symbol = 5, .bits = 5, .baseline = 0 },
1503	.{ .symbol = 6, .bits = 5, .baseline = 0 },
1504	.{ .symbol = 8, .bits = 5, .baseline = 0 },
1505	.{ .symbol = 10, .bits = 6, .baseline = 0 },
1506	.{ .symbol = 13, .bits = 6, .baseline = 0 },
1507	.{ .symbol = 16, .bits = 6, .baseline = 0 },
1508	.{ .symbol = 19, .bits = 6, .baseline = 0 },
1509	.{ .symbol = 22, .bits = 6, .baseline = 0 },
1510	.{ .symbol = 25, .bits = 6, .baseline = 0 },
1511	.{ .symbol = 28, .bits = 6, .baseline = 0 },
1512	.{ .symbol = 31, .bits = 6, .baseline = 0 },
1513	.{ .symbol = 33, .bits = 6, .baseline = 0 },
1514	.{ .symbol = 35, .bits = 6, .baseline = 0 },
1515	.{ .symbol = 37, .bits = 6, .baseline = 0 },
1516	.{ .symbol = 39, .bits = 6, .baseline = 0 },
1517	.{ .symbol = 41, .bits = 6, .baseline = 0 },
1518	.{ .symbol = 43, .bits = 6, .baseline = 0 },
1519	.{ .symbol = 45, .bits = 6, .baseline = 0 },
1520	.{ .symbol = 1, .bits = 4, .baseline = 16 },
1521	.{ .symbol = 2, .bits = 4, .baseline = 0 },
1522	.{ .symbol = 3, .bits = 5, .baseline = 32 },
1523	.{ .symbol = 4, .bits = 5, .baseline = 0 },
1524	.{ .symbol = 6, .bits = 5, .baseline = 32 },
1525	.{ .symbol = 7, .bits = 5, .baseline = 0 },
1526	.{ .symbol = 9, .bits = 6, .baseline = 0 },
1527	.{ .symbol = 12, .bits = 6, .baseline = 0 },
1528	.{ .symbol = 15, .bits = 6, .baseline = 0 },
1529	.{ .symbol = 18, .bits = 6, .baseline = 0 },
1530	.{ .symbol = 21, .bits = 6, .baseline = 0 },
1531	.{ .symbol = 24, .bits = 6, .baseline = 0 },
1532	.{ .symbol = 27, .bits = 6, .baseline = 0 },
1533	.{ .symbol = 30, .bits = 6, .baseline = 0 },
1534	.{ .symbol = 32, .bits = 6, .baseline = 0 },
1535	.{ .symbol = 34, .bits = 6, .baseline = 0 },
1536	.{ .symbol = 36, .bits = 6, .baseline = 0 },
1537	.{ .symbol = 38, .bits = 6, .baseline = 0 },
1538	.{ .symbol = 40, .bits = 6, .baseline = 0 },
1539	.{ .symbol = 42, .bits = 6, .baseline = 0 },
1540	.{ .symbol = 44, .bits = 6, .baseline = 0 },
1541	.{ .symbol = 1, .bits = 4, .baseline = 32 },
1542	.{ .symbol = 1, .bits = 4, .baseline = 48 },
1543	.{ .symbol = 2, .bits = 4, .baseline = 16 },
1544	.{ .symbol = 4, .bits = 5, .baseline = 32 },
1545	.{ .symbol = 5, .bits = 5, .baseline = 32 },
1546	.{ .symbol = 7, .bits = 5, .baseline = 32 },
1547	.{ .symbol = 8, .bits = 5, .baseline = 32 },
1548	.{ .symbol = 11, .bits = 6, .baseline = 0 },
1549	.{ .symbol = 14, .bits = 6, .baseline = 0 },
1550	.{ .symbol = 17, .bits = 6, .baseline = 0 },
1551	.{ .symbol = 20, .bits = 6, .baseline = 0 },
1552	.{ .symbol = 23, .bits = 6, .baseline = 0 },
1553	.{ .symbol = 26, .bits = 6, .baseline = 0 },
1554	.{ .symbol = 29, .bits = 6, .baseline = 0 },
1555	.{ .symbol = 52, .bits = 6, .baseline = 0 },
1556	.{ .symbol = 51, .bits = 6, .baseline = 0 },
1557	.{ .symbol = 50, .bits = 6, .baseline = 0 },
1558	.{ .symbol = 49, .bits = 6, .baseline = 0 },
1559	.{ .symbol = 48, .bits = 6, .baseline = 0 },
1560	.{ .symbol = 47, .bits = 6, .baseline = 0 },
1561	.{ .symbol = 46, .bits = 6, .baseline = 0 },
1562	},
1563	};
1564
1565	pub const predefined_offset: Table = .{
1566	.fse = &[32]Table.Fse{
1567	.{ .symbol = 0, .bits = 5, .baseline = 0 },
1568	.{ .symbol = 6, .bits = 4, .baseline = 0 },
1569	.{ .symbol = 9, .bits = 5, .baseline = 0 },
1570	.{ .symbol = 15, .bits = 5, .baseline = 0 },
1571	.{ .symbol = 21, .bits = 5, .baseline = 0 },
1572	.{ .symbol = 3, .bits = 5, .baseline = 0 },
1573	.{ .symbol = 7, .bits = 4, .baseline = 0 },
1574	.{ .symbol = 12, .bits = 5, .baseline = 0 },
1575	.{ .symbol = 18, .bits = 5, .baseline = 0 },
1576	.{ .symbol = 23, .bits = 5, .baseline = 0 },
1577	.{ .symbol = 5, .bits = 5, .baseline = 0 },
1578	.{ .symbol = 8, .bits = 4, .baseline = 0 },
1579	.{ .symbol = 14, .bits = 5, .baseline = 0 },
1580	.{ .symbol = 20, .bits = 5, .baseline = 0 },
1581	.{ .symbol = 2, .bits = 5, .baseline = 0 },
1582	.{ .symbol = 7, .bits = 4, .baseline = 16 },
1583	.{ .symbol = 11, .bits = 5, .baseline = 0 },
1584	.{ .symbol = 17, .bits = 5, .baseline = 0 },
1585	.{ .symbol = 22, .bits = 5, .baseline = 0 },
1586	.{ .symbol = 4, .bits = 5, .baseline = 0 },
1587	.{ .symbol = 8, .bits = 4, .baseline = 16 },
1588	.{ .symbol = 13, .bits = 5, .baseline = 0 },
1589	.{ .symbol = 19, .bits = 5, .baseline = 0 },
1590	.{ .symbol = 1, .bits = 5, .baseline = 0 },
1591	.{ .symbol = 6, .bits = 4, .baseline = 16 },
1592	.{ .symbol = 10, .bits = 5, .baseline = 0 },
1593	.{ .symbol = 16, .bits = 5, .baseline = 0 },
1594	.{ .symbol = 28, .bits = 5, .baseline = 0 },
1595	.{ .symbol = 27, .bits = 5, .baseline = 0 },
1596	.{ .symbol = 26, .bits = 5, .baseline = 0 },
1597	.{ .symbol = 25, .bits = 5, .baseline = 0 },
1598	.{ .symbol = 24, .bits = 5, .baseline = 0 },
1599	},
1600	};
1601	};
1602
1603	/// For reading the reversed bit streams used to encode FSE compressed data.
1604	const ReverseBitReader = struct {
1605	bytes: []const u8,
1606	remaining: usize,
1607	bits: u8,
1608	count: u4,
1609
1610	fn init(bytes: []const u8) error{MissingStartBit}!ReverseBitReader {
1611	var result: ReverseBitReader = .{
1612	.bytes = bytes,
1613	.remaining = bytes.len,
1614	.bits = 0,
1615	.count = 0,
1616	};
1617	if (bytes.len == 0) return result;
1618	for (0..8) \|_\| if (0 != (result.readBitsNoEof(u1, 1) catch unreachable)) return result;
1619	return error.MissingStartBit;
1620	}
1621
1622	fn readBitsNoEof(self: *ReverseBitReader, comptime U: type, num_bits: u16) error{EndOfStream}!U {
1623	return self.bit_reader.readBitsNoEof(U, num_bits);
1624	}
1625
1626	fn readBits(self: ReverseBitReader, comptime U: type, num_bits: u16, out_bits: u16) error{}!U {
1627	return try self.bit_reader.readBits(U, num_bits, out_bits);
1628	}
1629
1630	fn alignToByte(self: *ReverseBitReader) void {
1631	self.bit_reader.alignToByte();
1632	}
1633
1634	fn isEmpty(self: *const ReverseBitReader) bool {
1635	return self.byte_reader.remaining_bytes == 0 and self.bit_reader.count == 0;
1636	}
1637	};
1638
1639	test {
1640	_ = Table;
1641	}

lib/std/crypto/sha2.zig+3-1

 @@ -293,7 +293,9 @@ fn Sha2x32(comptime iv: Iv32, digest_bits: comptime_int) type {
             var i: usize = 16;
             while (i < 64) : (i += 1) {
-                s[i] = s[i - 16] +% s[i - 7] +% (math.rotr(u32, s[i - 15], @as(u32, 7)) ^ math.rotr(u32, s[i - 15], @as(u32, 18)) ^ (s[i - 15] >> 3)) +% (math.rotr(u32, s[i - 2], @as(u32, 17)) ^ math.rotr(u32, s[i - 2], @as(u32, 19)) ^ (s[i - 2] >> 10));
+                s[i] = s[i - 16] +% s[i - 7] +%
+                    (math.rotr(u32, s[i - 15], @as(u32, 7)) ^ math.rotr(u32, s[i - 15], @as(u32, 18)) ^ (s[i - 15] >> 3)) +%
+                    (math.rotr(u32, s[i - 2], @as(u32, 17)) ^ math.rotr(u32, s[i - 2], @as(u32, 19)) ^ (s[i - 2] >> 10));
             }
             var v: [8]u32 = d.s;

lib/std/io/BufferedReader.zig+16-1

 @@ -825,17 +825,32 @@ pub inline fn peekStructEndian(br: *BufferedReader, comptime T: type, endian: st
     return res;
 }
+pub const TakeEnumError = Reader.Error || error{InvalidEnumTag};
 /// Reads an integer with the same size as the given enum's tag type. If the
 /// integer matches an enum tag, casts the integer to the enum tag and returns
 /// it. Otherwise, returns `error.InvalidEnumTag`.
 ///
 /// Asserts the buffer was initialized with a capacity at least `@sizeOf(Enum)`.
-pub fn takeEnum(br: *BufferedReader, comptime Enum: type, endian: std.builtin.Endian) (Reader.Error || std.meta.IntToEnumError)!Enum {
+pub fn takeEnum(br: *BufferedReader, comptime Enum: type, endian: std.builtin.Endian) TakeEnumError!Enum {
     const Tag = @typeInfo(Enum).@"enum".tag_type;
     const int = try br.takeInt(Tag, endian);
     return std.meta.intToEnum(Enum, int);
 }
+/// Reads an integer with the same size as the given nonexhaustive enum's tag type.
+///
+/// Asserts the buffer was initialized with a capacity at least `@sizeOf(Enum)`.
+pub fn takeEnumNonexhaustive(br: *BufferedReader, comptime Enum: type, endian: std.builtin.Endian) Reader.Error!Enum {
+    const info = @typeInfo(Enum).@"enum";
+    comptime assert(!info.is_exhaustive);
+    comptime assert(@bitSizeOf(info.tag_type) == @sizeOf(info.tag_type) * 8);
+    return takeEnum(br, Enum, endian) catch |err| switch (err) {
+        error.InvalidEnumTag => unreachable,
+        else => |e| return e,
+    };
+}
 pub const TakeLeb128Error = Reader.Error || error{Overflow};
 /// Read a single LEB128 value as type T, or `error.Overflow` if the value cannot fit.

lib/std/io/Writer/Null.zig+1-1

...	@@ -5,7 +5,7 @@ const Writer = std.io.Writer;	...	@@ -5,7 +5,7 @@ const Writer = std.io.Writer;
5		5
6	const NullWriter = @This();	6	const NullWriter = @This();
7		7
8	err: Error,	8	err: ?Error = null,
9		9
10	pub const Error = std.fs.File.StatError;	10	pub const Error = std.fs.File.StatError;
11		11

lib/std/std.zig-1

...	@@ -30,7 +30,6 @@ pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue;	...	@@ -30,7 +30,6 @@ pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue;
30	pub const PriorityDequeue = @import("priority_dequeue.zig").PriorityDequeue;	30	pub const PriorityDequeue = @import("priority_dequeue.zig").PriorityDequeue;
31	pub const Progress = @import("Progress.zig");	31	pub const Progress = @import("Progress.zig");
32	pub const Random = @import("Random.zig");	32	pub const Random = @import("Random.zig");
33	pub const RingBuffer = @import("RingBuffer.zig");
34	pub const SegmentedList = @import("segmented_list.zig").SegmentedList;	33	pub const SegmentedList = @import("segmented_list.zig").SegmentedList;
35	pub const SemanticVersion = @import("SemanticVersion.zig");	34	pub const SemanticVersion = @import("SemanticVersion.zig");
36	pub const SinglyLinkedList = @import("SinglyLinkedList.zig");	35	pub const SinglyLinkedList = @import("SinglyLinkedList.zig");