authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-04-30 22:40:11-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:28-07:00
log685d55c1a49615b6601c02c174d1097489b0fe95
tree627b77f57fdc3da485af90bc42f679cca94a19fa
parent689a70663a1ddd6b939ec4f957869be21ceaaa34

std.compress.zstd: rewrite

WIP - passing AstGen but not yet Sema

15 files changed, 1815 insertions(+), 3015 deletions(-)

lib/std/RingBuffer.zig deleted-230
......@@ -1,230 +0,0 @@
1//! This ring buffer stores read and write indices while being able to utilise
2//! the full backing slice by incrementing the indices modulo twice the slice's
3//! length and reducing indices modulo the slice's length on slice access. This
4//! means that whether the ring buffer is full or empty can be distinguished by
5//! looking at the difference between the read and write indices without adding
6//! an extra boolean flag or having to reserve a slot in the buffer.
7//!
8//! This ring buffer has not been implemented with thread safety in mind, and
9//! therefore should not be assumed to be suitable for use cases involving
10//! separate reader and writer threads.
11
12const Allocator = @import("std").mem.Allocator;
13const assert = @import("std").debug.assert;
14const copyForwards = @import("std").mem.copyForwards;
15
16const RingBuffer = @This();
17
18data: []u8,
19read_index: usize,
20write_index: usize,
21
22pub const Error = error{ Full, ReadLengthInvalid };
23
24/// Allocate a new `RingBuffer`; `deinit()` should be called to free the buffer.
25pub fn init(allocator: Allocator, capacity: usize) Allocator.Error!RingBuffer {
26 const bytes = try allocator.alloc(u8, capacity);
27 return RingBuffer{
28 .data = bytes,
29 .write_index = 0,
30 .read_index = 0,
31 };
32}
33
34/// Free the data backing a `RingBuffer`; must be passed the same `Allocator` as
35/// `init()`.
36pub fn deinit(self: *RingBuffer, allocator: Allocator) void {
37 allocator.free(self.data);
38 self.* = undefined;
39}
40
41/// Returns `index` modulo the length of the backing slice.
42pub fn mask(self: RingBuffer, index: usize) usize {
43 return index % self.data.len;
44}
45
46/// Returns `index` modulo twice the length of the backing slice.
47pub fn mask2(self: RingBuffer, index: usize) usize {
48 return index % (2 * self.data.len);
49}
50
51/// Write `byte` into the ring buffer. Returns `error.Full` if the ring
52/// buffer is full.
53pub fn write(self: *RingBuffer, byte: u8) Error!void {
54 if (self.isFull()) return error.Full;
55 self.writeAssumeCapacity(byte);
56}
57
58/// Write `byte` into the ring buffer. If the ring buffer is full, the
59/// oldest byte is overwritten.
60pub fn writeAssumeCapacity(self: *RingBuffer, byte: u8) void {
61 self.data[self.mask(self.write_index)] = byte;
62 self.write_index = self.mask2(self.write_index + 1);
63}
64
65/// Write `bytes` into the ring buffer. Returns `error.Full` if the ring
66/// buffer does not have enough space, without writing any data.
67/// Uses memcpy and so `bytes` must not overlap ring buffer data.
68pub fn writeSlice(self: *RingBuffer, bytes: []const u8) Error!void {
69 if (self.len() + bytes.len > self.data.len) return error.Full;
70 self.writeSliceAssumeCapacity(bytes);
71}
72
73/// Write `bytes` into the ring buffer. If there is not enough space, older
74/// bytes will be overwritten.
75/// Uses memcpy and so `bytes` must not overlap ring buffer data.
76pub fn writeSliceAssumeCapacity(self: *RingBuffer, bytes: []const u8) void {
77 assert(bytes.len <= self.data.len);
78 const data_start = self.mask(self.write_index);
79 const part1_data_end = @min(data_start + bytes.len, self.data.len);
80 const part1_len = part1_data_end - data_start;
81 @memcpy(self.data[data_start..part1_data_end], bytes[0..part1_len]);
82
83 const remaining = bytes.len - part1_len;
84 const to_write = @min(remaining, remaining % self.data.len + self.data.len);
85 const part2_bytes_start = bytes.len - to_write;
86 const part2_bytes_end = @min(part2_bytes_start + self.data.len, bytes.len);
87 const part2_len = part2_bytes_end - part2_bytes_start;
88 @memcpy(self.data[0..part2_len], bytes[part2_bytes_start..part2_bytes_end]);
89 if (part2_bytes_end != bytes.len) {
90 const part3_len = bytes.len - part2_bytes_end;
91 @memcpy(self.data[0..part3_len], bytes[part2_bytes_end..bytes.len]);
92 }
93 self.write_index = self.mask2(self.write_index + bytes.len);
94}
95
96/// Write `bytes` into the ring buffer. Returns `error.Full` if the ring
97/// buffer does not have enough space, without writing any data.
98/// Uses copyForwards and can write slices from this RingBuffer into itself.
99pub fn writeSliceForwards(self: *RingBuffer, bytes: []const u8) Error!void {
100 if (self.len() + bytes.len > self.data.len) return error.Full;
101 self.writeSliceForwardsAssumeCapacity(bytes);
102}
103
104/// Write `bytes` into the ring buffer. If there is not enough space, older
105/// bytes will be overwritten.
106/// Uses copyForwards and can write slices from this RingBuffer into itself.
107pub fn writeSliceForwardsAssumeCapacity(self: *RingBuffer, bytes: []const u8) void {
108 assert(bytes.len <= self.data.len);
109 const data_start = self.mask(self.write_index);
110 const part1_data_end = @min(data_start + bytes.len, self.data.len);
111 const part1_len = part1_data_end - data_start;
112 copyForwards(u8, self.data[data_start..], bytes[0..part1_len]);
113
114 const remaining = bytes.len - part1_len;
115 const to_write = @min(remaining, remaining % self.data.len + self.data.len);
116 const part2_bytes_start = bytes.len - to_write;
117 const part2_bytes_end = @min(part2_bytes_start + self.data.len, bytes.len);
118 copyForwards(u8, self.data[0..], bytes[part2_bytes_start..part2_bytes_end]);
119 if (part2_bytes_end != bytes.len)
120 copyForwards(u8, self.data[0..], bytes[part2_bytes_end..bytes.len]);
121 self.write_index = self.mask2(self.write_index + bytes.len);
122}
123
124/// Consume a byte from the ring buffer and return it. Returns `null` if the
125/// ring buffer is empty.
126pub fn read(self: *RingBuffer) ?u8 {
127 if (self.isEmpty()) return null;
128 return self.readAssumeLength();
129}
130
131/// Consume a byte from the ring buffer and return it; asserts that the buffer
132/// is not empty.
133pub fn readAssumeLength(self: *RingBuffer) u8 {
134 assert(!self.isEmpty());
135 const byte = self.data[self.mask(self.read_index)];
136 self.read_index = self.mask2(self.read_index + 1);
137 return byte;
138}
139
140/// Reads first `length` bytes written to the ring buffer into `dest`; Returns
141/// Error.ReadLengthInvalid if length greater than ring or dest length
142/// Uses memcpy and so `dest` must not overlap ring buffer data.
143pub fn readFirst(self: *RingBuffer, dest: []u8, length: usize) Error!void {
144 if (length > self.len() or length > dest.len) return error.ReadLengthInvalid;
145 self.readFirstAssumeLength(dest, length);
146}
147
148/// Reads first `length` bytes written to the ring buffer into `dest`;
149/// Asserts that length not greater than ring buffer or dest length
150/// Uses memcpy and so `dest` must not overlap ring buffer data.
151pub fn readFirstAssumeLength(self: *RingBuffer, dest: []u8, length: usize) void {
152 assert(length <= self.len() and length <= dest.len);
153 const slice = self.sliceAt(self.read_index, length);
154 slice.copyTo(dest);
155 self.read_index = self.mask2(self.read_index + length);
156}
157
158/// Reads last `length` bytes written to the ring buffer into `dest`; Returns
159/// Error.ReadLengthInvalid if length greater than ring or dest length
160/// Uses memcpy and so `dest` must not overlap ring buffer data.
161/// Reduces write index by `length`.
162pub fn readLast(self: *RingBuffer, dest: []u8, length: usize) Error!void {
163 if (length > self.len() or length > dest.len) return error.ReadLengthInvalid;
164 self.readLastAssumeLength(dest, length);
165}
166
167/// Reads last `length` bytes written to the ring buffer into `dest`;
168/// Asserts that length not greater than ring buffer or dest length
169/// Uses memcpy and so `dest` must not overlap ring buffer data.
170/// Reduces write index by `length`.
171pub fn readLastAssumeLength(self: *RingBuffer, dest: []u8, length: usize) void {
172 assert(length <= self.len() and length <= dest.len);
173 const slice = self.sliceLast(length);
174 slice.copyTo(dest);
175 self.write_index = if (self.write_index >= self.data.len)
176 self.write_index - length
177 else
178 self.mask(self.write_index + self.data.len - length);
179}
180
181/// Returns `true` if the ring buffer is empty and `false` otherwise.
182pub fn isEmpty(self: RingBuffer) bool {
183 return self.write_index == self.read_index;
184}
185
186/// Returns `true` if the ring buffer is full and `false` otherwise.
187pub fn isFull(self: RingBuffer) bool {
188 return self.mask2(self.write_index + self.data.len) == self.read_index;
189}
190
191/// Returns the length of data available for reading
192pub fn len(self: RingBuffer) usize {
193 const wrap_offset = 2 * self.data.len * @intFromBool(self.write_index < self.read_index);
194 const adjusted_write_index = self.write_index + wrap_offset;
195 return adjusted_write_index - self.read_index;
196}
197
198/// A `Slice` represents a region of a ring buffer. The region is split into two
199/// sections as the ring buffer data will not be contiguous if the desired
200/// region wraps to the start of the backing slice.
201pub const Slice = struct {
202 first: []u8,
203 second: []u8,
204
205 /// Copy data from `self` into `dest`
206 pub fn copyTo(self: Slice, dest: []u8) void {
207 @memcpy(dest[0..self.first.len], self.first);
208 @memcpy(dest[self.first.len..][0..self.second.len], self.second);
209 }
210};
211
212/// Returns a `Slice` for the region of the ring buffer starting at
213/// `self.mask(start_unmasked)` with the specified length.
214pub fn sliceAt(self: RingBuffer, start_unmasked: usize, length: usize) Slice {
215 assert(length <= self.data.len);
216 const slice1_start = self.mask(start_unmasked);
217 const slice1_end = @min(self.data.len, slice1_start + length);
218 const slice1 = self.data[slice1_start..slice1_end];
219 const slice2 = self.data[0 .. length - slice1.len];
220 return Slice{
221 .first = slice1,
222 .second = slice2,
223 };
224}
225
226/// Returns a `Slice` for the last `length` bytes written to the ring buffer.
227/// Does not check that any bytes have been written into the region.
228pub fn sliceLast(self: RingBuffer, length: usize) Slice {
229 return self.sliceAt(self.write_index + self.data.len - length, length);
230}
lib/std/compress.zig+1-1
......@@ -6,7 +6,7 @@ pub const lzma = @import("compress/lzma.zig");
66pub const lzma2 = @import("compress/lzma2.zig");
77pub const xz = @import("compress/xz.zig");
88pub const zlib = @import("compress/zlib.zig");
9pub const zstd = @import("compress/zstandard.zig");
9pub const zstd = @import("compress/zstd.zig");
1010
1111test {
1212 _ = flate;
lib/std/compress/zstandard.zig deleted-334
......@@ -1,334 +0,0 @@
1const std = @import("../std.zig");
2const RingBuffer = std.RingBuffer;
3
4const types = @import("zstandard/types.zig");
5
6/// Recommended amount by the standard. Lower than this may result in inability
7/// to decompress common streams.
8pub const default_window_len = 8 * 1024 * 1024;
9
10pub const frame = types.frame;
11pub const compressed_block = types.compressed_block;
12
13pub const decompress = @import("zstandard/decompress.zig");
14
15pub const Decompressor = struct {
16 const table_size_max = types.compressed_block.table_size_max;
17
18 input: *std.io.BufferedReader,
19 bytes_read: usize,
20 state: enum { NewFrame, InFrame, LastBlock },
21 decode_state: decompress.block.DecodeState,
22 frame_context: decompress.FrameContext,
23 buffer: WindowBuffer,
24 literal_fse_buffer: [table_size_max.literal]types.compressed_block.Table.Fse,
25 match_fse_buffer: [table_size_max.match]types.compressed_block.Table.Fse,
26 offset_fse_buffer: [table_size_max.offset]types.compressed_block.Table.Fse,
27 literals_buffer: [types.block_size_max]u8,
28 sequence_buffer: [types.block_size_max]u8,
29 verify_checksum: bool,
30 checksum: ?u32,
31 current_frame_decompressed_size: usize,
32 err: ?Error = null,
33
34 pub const Options = struct {
35 verify_checksum: bool = true,
36 /// See `default_window_len`.
37 window_buffer: []u8,
38 };
39
40 const WindowBuffer = struct {
41 data: []u8 = undefined,
42 read_index: usize = 0,
43 write_index: usize = 0,
44 };
45
46 pub const Error = std.io.Reader.Error || error{
47 ChecksumFailure,
48 DictionaryIdFlagUnsupported,
49 MalformedBlock,
50 MalformedFrame,
51 OutOfMemory,
52 EndOfStream,
53 };
54
55 pub fn init(input: *std.io.BufferedReader, options: Options) Decompressor {
56 return .{
57 .input = input,
58 .bytes_read = 0,
59 .state = .NewFrame,
60 .decode_state = undefined,
61 .frame_context = undefined,
62 .buffer = .{ .data = options.window_buffer },
63 .literal_fse_buffer = undefined,
64 .match_fse_buffer = undefined,
65 .offset_fse_buffer = undefined,
66 .literals_buffer = undefined,
67 .sequence_buffer = undefined,
68 .verify_checksum = options.verify_checksum,
69 .checksum = undefined,
70 .current_frame_decompressed_size = undefined,
71 };
72 }
73
74 fn frameInit(d: *Decompressor) !void {
75 const in = d.input;
76 switch (try decompress.decodeFrameHeader(in, &d.bytes_read)) {
77 .skippable => |header| {
78 try in.discardAll(header.frame_size);
79 d.bytes_read += header.frame_size;
80 d.state = .NewFrame;
81 },
82 .zstandard => |header| {
83 const frame_context = try decompress.FrameContext.init(
84 header,
85 d.buffer.data.len,
86 d.verify_checksum,
87 );
88
89 const decode_state = decompress.block.DecodeState.init(
90 &d.literal_fse_buffer,
91 &d.match_fse_buffer,
92 &d.offset_fse_buffer,
93 );
94
95 d.decode_state = decode_state;
96 d.frame_context = frame_context;
97
98 d.checksum = null;
99 d.current_frame_decompressed_size = 0;
100
101 d.state = .InFrame;
102 },
103 }
104 }
105
106 pub fn reader(self: *Decompressor) std.io.Reader {
107 return .{
108 .context = self,
109 .vtable = &.{
110 .read = read,
111 .readVec = readVec,
112 .discard = discard,
113 },
114 };
115 }
116
117 fn read(context: ?*anyopaque, bw: *std.io.BufferedWriter, limit: std.io.Reader.Limit) std.io.Reader.RwError!usize {
118 const buf = limit.slice(try bw.writableSliceGreedy(1));
119 const n = try readVec(context, &.{buf});
120 bw.advance(n);
121 return n;
122 }
123
124 fn discard(context: ?*anyopaque, limit: std.io.Reader.Limit) std.io.Reader.Error!usize {
125 var trash: [128]u8 = undefined;
126 const buf = limit.slice(&trash);
127 return readVec(context, &.{buf});
128 }
129
130 fn readVec(context: ?*anyopaque, data: []const []u8) std.io.Reader.Error!usize {
131 const d: *Decompressor = @ptrCast(@alignCast(context));
132 if (data.len == 0) return 0;
133 const buffer = data[0];
134 while (d.state == .NewFrame) {
135 const initial_count = d.bytes_read;
136 d.frameInit() catch |err| switch (err) {
137 error.DictionaryIdFlagUnsupported => {
138 d.err = error.DictionaryIdFlagUnsupported;
139 return error.ReadFailed;
140 },
141 error.EndOfStream => {
142 if (d.bytes_read == initial_count) return error.EndOfStream;
143 d.err = error.MalformedFrame;
144 return error.ReadFailed;
145 },
146 else => {
147 d.err = error.MalformedFrame;
148 return error.ReadFailed;
149 },
150 };
151 }
152 return d.readInner(buffer) catch |err| {
153 d.err = err;
154 return error.ReadFailed;
155 };
156 }
157
158 fn readInner(d: *Decompressor, buffer: []u8) Error!usize {
159 std.debug.assert(d.state != .NewFrame);
160
161 var ring_buffer = RingBuffer{
162 .data = d.buffer.data,
163 .read_index = d.buffer.read_index,
164 .write_index = d.buffer.write_index,
165 };
166 defer {
167 d.buffer.read_index = ring_buffer.read_index;
168 d.buffer.write_index = ring_buffer.write_index;
169 }
170
171 const in = d.input;
172 while (ring_buffer.isEmpty() and d.state != .LastBlock) {
173 const header_bytes = try in.takeArray(3);
174 d.bytes_read += header_bytes.len;
175 const block_header = decompress.block.decodeBlockHeader(header_bytes);
176
177 decompress.block.decodeBlockReader(
178 &ring_buffer,
179 in,
180 &d.bytes_read,
181 block_header,
182 &d.decode_state,
183 d.frame_context.block_size_max,
184 &d.literals_buffer,
185 &d.sequence_buffer,
186 ) catch return error.MalformedBlock;
187
188 if (d.frame_context.content_size) |size| {
189 if (d.current_frame_decompressed_size > size) return error.MalformedFrame;
190 }
191
192 const size = ring_buffer.len();
193 d.current_frame_decompressed_size += size;
194
195 if (d.frame_context.hasher_opt) |*hasher| {
196 if (size > 0) {
197 const written_slice = ring_buffer.sliceLast(size);
198 hasher.update(written_slice.first);
199 hasher.update(written_slice.second);
200 }
201 }
202 if (block_header.last_block) {
203 d.state = .LastBlock;
204 if (d.frame_context.has_checksum) {
205 const checksum = in.readInt(u32, .little) catch return error.MalformedFrame;
206 d.bytes_read += 4;
207 if (d.verify_checksum) {
208 if (d.frame_context.hasher_opt) |*hasher| {
209 if (checksum != decompress.computeChecksum(hasher))
210 return error.ChecksumFailure;
211 }
212 }
213 }
214 if (d.frame_context.content_size) |content_size| {
215 if (content_size != d.current_frame_decompressed_size) {
216 return error.MalformedFrame;
217 }
218 }
219 }
220 }
221
222 const size = @min(ring_buffer.len(), buffer.len);
223 if (size > 0) {
224 ring_buffer.readFirstAssumeLength(buffer, size);
225 }
226 if (d.state == .LastBlock and ring_buffer.len() == 0) {
227 d.state = .NewFrame;
228 }
229 return size;
230 }
231};
232
233fn testDecompress(data: []const u8) ![]u8 {
234 const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
235 defer std.testing.allocator.free(window_buffer);
236
237 var in_stream = std.io.fixedBufferStream(data);
238 var zstd_stream: Decompressor = .init(in_stream.reader(), .{ .window_buffer = window_buffer });
239 const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
240 return result;
241}
242
243fn testReader(data: []const u8, comptime expected: []const u8) !void {
244 const buf = try testDecompress(data);
245 defer std.testing.allocator.free(buf);
246 try std.testing.expectEqualSlices(u8, expected, buf);
247}
248
249test "decompression" {
250 const uncompressed = @embedFile("testdata/rfc8478.txt");
251 const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
252 const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
253
254 const buffer = try std.testing.allocator.alloc(u8, uncompressed.len);
255 defer std.testing.allocator.free(buffer);
256
257 const res3 = try decompress.decode(buffer, compressed3, true);
258 try std.testing.expectEqual(uncompressed.len, res3);
259 try std.testing.expectEqualSlices(u8, uncompressed, buffer);
260
261 @memset(buffer, undefined);
262 const res19 = try decompress.decode(buffer, compressed19, true);
263 try std.testing.expectEqual(uncompressed.len, res19);
264 try std.testing.expectEqualSlices(u8, uncompressed, buffer);
265
266 try testReader(compressed3, uncompressed);
267 try testReader(compressed19, uncompressed);
268}
269
270fn expectEqualDecoded(expected: []const u8, input: []const u8) !void {
271 {
272 const result = try decompress.decodeAlloc(std.testing.allocator, input, false, 1 << 23);
273 defer std.testing.allocator.free(result);
274 try std.testing.expectEqualStrings(expected, result);
275 }
276
277 {
278 var buffer = try std.testing.allocator.alloc(u8, 2 * expected.len);
279 defer std.testing.allocator.free(buffer);
280
281 const size = try decompress.decode(buffer, input, false);
282 try std.testing.expectEqualStrings(expected, buffer[0..size]);
283 }
284}
285
286fn expectEqualDecodedStreaming(expected: []const u8, input: []const u8) !void {
287 const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
288 defer std.testing.allocator.free(window_buffer);
289
290 var in_stream = std.io.fixedBufferStream(input);
291 var stream: Decompressor = .init(in_stream.reader(), .{ .window_buffer = window_buffer });
292
293 const result = try stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
294 defer std.testing.allocator.free(result);
295
296 try std.testing.expectEqualStrings(expected, result);
297}
298
299test "zero sized block" {
300 const input_raw =
301 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
302 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
303 "\x01\x00\x00"; // block header with: last_block set, block_type raw, block_size zero
304
305 const input_rle =
306 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
307 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
308 "\x03\x00\x00" ++ // block header with: last_block set, block_type rle, block_size zero
309 "\xaa"; // block_content
310
311 try expectEqualDecoded("", input_raw);
312 try expectEqualDecoded("", input_rle);
313 try expectEqualDecodedStreaming("", input_raw);
314 try expectEqualDecodedStreaming("", input_rle);
315}
316
317test "declared raw literals size too large" {
318 const input_raw =
319 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
320 "\x00\x00" ++ // frame header: everything unset, window descriptor zero
321 "\x95\x00\x00" ++ // block header with: last_block set, block_type compressed, block_size 18
322 "\xbc\xf3\xae" ++ // literals section header with: type raw, size_format 3, regenerated_size 716603
323 "\xa5\x9f\xe3"; // some bytes of literal content - the content is shorter than regenerated_size
324
325 // Note that the regenerated_size in the above input is larger than block maximum size, so the
326 // block can't be valid as it is a raw literals block.
327
328 var fbs = std.io.fixedBufferStream(input_raw);
329 var window: [1024]u8 = undefined;
330 var stream: Decompressor = .init(fbs.reader(), .{ .window_buffer = &window });
331
332 var buf: [1024]u8 = undefined;
333 try std.testing.expectError(error.MalformedBlock, stream.read(&buf));
334}
lib/std/compress/zstandard/decode/block.zig deleted-1153
......@@ -1,1153 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const RingBuffer = std.RingBuffer;
4
5const types = @import("../types.zig");
6const frame = types.frame;
7const Table = types.compressed_block.Table;
8const LiteralsSection = types.compressed_block.LiteralsSection;
9const SequencesSection = types.compressed_block.SequencesSection;
10
11const huffman = @import("huffman.zig");
12const readers = @import("../readers.zig");
13
14const decodeFseTable = @import("fse.zig").decodeFseTable;
15
16pub const Error = error{
17 BlockSizeOverMaximum,
18 MalformedBlockSize,
19 ReservedBlock,
20 MalformedRleBlock,
21 MalformedCompressedBlock,
22};
23
24pub const DecodeState = struct {
25 repeat_offsets: [3]u32,
26
27 offset: StateData(8),
28 match: StateData(9),
29 literal: StateData(9),
30
31 offset_fse_buffer: []Table.Fse,
32 match_fse_buffer: []Table.Fse,
33 literal_fse_buffer: []Table.Fse,
34
35 fse_tables_undefined: bool,
36
37 literal_stream_reader: readers.ReverseBitReader,
38 literal_stream_index: usize,
39 literal_streams: LiteralsSection.Streams,
40 literal_header: LiteralsSection.Header,
41 huffman_tree: ?LiteralsSection.HuffmanTree,
42
43 literal_written_count: usize,
44 written_count: usize = 0,
45
46 fn StateData(comptime max_accuracy_log: comptime_int) type {
47 return struct {
48 state: State,
49 table: Table,
50 accuracy_log: u8,
51
52 const State = std.meta.Int(.unsigned, max_accuracy_log);
53 };
54 }
55
56 pub fn init(
57 literal_fse_buffer: []Table.Fse,
58 match_fse_buffer: []Table.Fse,
59 offset_fse_buffer: []Table.Fse,
60 ) DecodeState {
61 return DecodeState{
62 .repeat_offsets = .{
63 types.compressed_block.start_repeated_offset_1,
64 types.compressed_block.start_repeated_offset_2,
65 types.compressed_block.start_repeated_offset_3,
66 },
67
68 .offset = undefined,
69 .match = undefined,
70 .literal = undefined,
71
72 .literal_fse_buffer = literal_fse_buffer,
73 .match_fse_buffer = match_fse_buffer,
74 .offset_fse_buffer = offset_fse_buffer,
75
76 .fse_tables_undefined = true,
77
78 .literal_written_count = 0,
79 .literal_header = undefined,
80 .literal_streams = undefined,
81 .literal_stream_reader = undefined,
82 .literal_stream_index = undefined,
83 .huffman_tree = null,
84
85 .written_count = 0,
86 };
87 }
88
89 /// Prepare the decoder to decode a compressed block. Loads the literals
90 /// stream and Huffman tree from `literals` and reads the FSE tables from
91 /// `source`.
92 ///
93 /// Errors returned:
94 /// - `error.BitStreamHasNoStartBit` if the (reversed) literal bitstream's
95 /// first byte does not have any bits set
96 /// - `error.TreelessLiteralsFirst` `literals` is a treeless literals
97 /// section and the decode state does not have a Huffman tree from a
98 /// previous block
99 /// - `error.RepeatModeFirst` on the first call if one of the sequence FSE
100 /// tables is set to repeat mode
101 /// - `error.MalformedAccuracyLog` if an FSE table has an invalid accuracy
102 /// - `error.MalformedFseTable` if there are errors decoding an FSE table
103 /// - `error.EndOfStream` if `source` ends before all FSE tables are read
104 pub fn prepare(
105 self: *DecodeState,
106 source: anytype,
107 literals: LiteralsSection,
108 sequences_header: SequencesSection.Header,
109 ) !void {
110 self.literal_written_count = 0;
111 self.literal_header = literals.header;
112 self.literal_streams = literals.streams;
113
114 if (literals.huffman_tree) |tree| {
115 self.huffman_tree = tree;
116 } else if (literals.header.block_type == .treeless and self.huffman_tree == null) {
117 return error.TreelessLiteralsFirst;
118 }
119
120 switch (literals.header.block_type) {
121 .raw, .rle => {},
122 .compressed, .treeless => {
123 self.literal_stream_index = 0;
124 switch (literals.streams) {
125 .one => |slice| try self.initLiteralStream(slice),
126 .four => |streams| try self.initLiteralStream(streams[0]),
127 }
128 },
129 }
130
131 if (sequences_header.sequence_count > 0) {
132 try self.updateFseTable(source, .literal, sequences_header.literal_lengths);
133 try self.updateFseTable(source, .offset, sequences_header.offsets);
134 try self.updateFseTable(source, .match, sequences_header.match_lengths);
135 self.fse_tables_undefined = false;
136 }
137 }
138
139 /// Read initial FSE states for sequence decoding.
140 ///
141 /// Errors returned:
142 /// - `error.EndOfStream` if `bit_reader` does not contain enough bits.
143 pub fn readInitialFseState(self: *DecodeState, bit_reader: *readers.ReverseBitReader) error{EndOfStream}!void {
144 self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
145 self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
146 self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log);
147 }
148
149 fn updateRepeatOffset(self: *DecodeState, offset: u32) void {
150 self.repeat_offsets[2] = self.repeat_offsets[1];
151 self.repeat_offsets[1] = self.repeat_offsets[0];
152 self.repeat_offsets[0] = offset;
153 }
154
155 fn useRepeatOffset(self: *DecodeState, index: usize) u32 {
156 if (index == 1)
157 std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[1])
158 else if (index == 2) {
159 std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[2]);
160 std.mem.swap(u32, &self.repeat_offsets[1], &self.repeat_offsets[2]);
161 }
162 return self.repeat_offsets[0];
163 }
164
165 const DataType = enum { offset, match, literal };
166
167 fn updateState(
168 self: *DecodeState,
169 comptime choice: DataType,
170 bit_reader: *readers.ReverseBitReader,
171 ) error{ MalformedFseBits, EndOfStream }!void {
172 switch (@field(self, @tagName(choice)).table) {
173 .rle => {},
174 .fse => |table| {
175 const data = table[@field(self, @tagName(choice)).state];
176 const T = @TypeOf(@field(self, @tagName(choice))).State;
177 const bits_summand = try bit_reader.readBitsNoEof(T, data.bits);
178 const next_state = std.math.cast(
179 @TypeOf(@field(self, @tagName(choice))).State,
180 data.baseline + bits_summand,
181 ) orelse return error.MalformedFseBits;
182 @field(self, @tagName(choice)).state = next_state;
183 },
184 }
185 }
186
187 const FseTableError = error{
188 MalformedFseTable,
189 MalformedAccuracyLog,
190 RepeatModeFirst,
191 EndOfStream,
192 };
193
194 fn updateFseTable(
195 self: *DecodeState,
196 source: anytype,
197 comptime choice: DataType,
198 mode: SequencesSection.Header.Mode,
199 ) !void {
200 const field_name = @tagName(choice);
201 switch (mode) {
202 .predefined => {
203 @field(self, field_name).accuracy_log =
204 @field(types.compressed_block.default_accuracy_log, field_name);
205
206 @field(self, field_name).table =
207 @field(types.compressed_block, "predefined_" ++ field_name ++ "_fse_table");
208 },
209 .rle => {
210 @field(self, field_name).accuracy_log = 0;
211 @field(self, field_name).table = .{ .rle = try source.readByte() };
212 },
213 .fse => {
214 var bit_reader = readers.bitReader(source);
215
216 const table_size = try decodeFseTable(
217 &bit_reader,
218 @field(types.compressed_block.table_symbol_count_max, field_name),
219 @field(types.compressed_block.table_accuracy_log_max, field_name),
220 @field(self, field_name ++ "_fse_buffer"),
221 );
222 @field(self, field_name).table = .{
223 .fse = @field(self, field_name ++ "_fse_buffer")[0..table_size],
224 };
225 @field(self, field_name).accuracy_log = std.math.log2_int_ceil(usize, table_size);
226 },
227 .repeat => if (self.fse_tables_undefined) return error.RepeatModeFirst,
228 }
229 }
230
231 const Sequence = struct {
232 literal_length: u32,
233 match_length: u32,
234 offset: u32,
235 };
236
237 fn nextSequence(
238 self: *DecodeState,
239 bit_reader: *readers.ReverseBitReader,
240 ) error{ InvalidBitStream, EndOfStream }!Sequence {
241 const raw_code = self.getCode(.offset);
242 const offset_code = std.math.cast(u5, raw_code) orelse {
243 return error.InvalidBitStream;
244 };
245 const offset_value = (@as(u32, 1) << offset_code) + try bit_reader.readBitsNoEof(u32, offset_code);
246
247 const match_code = self.getCode(.match);
248 if (match_code >= types.compressed_block.match_length_code_table.len)
249 return error.InvalidBitStream;
250 const match = types.compressed_block.match_length_code_table[match_code];
251 const match_length = match[0] + try bit_reader.readBitsNoEof(u32, match[1]);
252
253 const literal_code = self.getCode(.literal);
254 if (literal_code >= types.compressed_block.literals_length_code_table.len)
255 return error.InvalidBitStream;
256 const literal = types.compressed_block.literals_length_code_table[literal_code];
257 const literal_length = literal[0] + try bit_reader.readBitsNoEof(u32, literal[1]);
258
259 const offset = if (offset_value > 3) offset: {
260 const offset = offset_value - 3;
261 self.updateRepeatOffset(offset);
262 break :offset offset;
263 } else offset: {
264 if (literal_length == 0) {
265 if (offset_value == 3) {
266 const offset = self.repeat_offsets[0] - 1;
267 self.updateRepeatOffset(offset);
268 break :offset offset;
269 }
270 break :offset self.useRepeatOffset(offset_value);
271 }
272 break :offset self.useRepeatOffset(offset_value - 1);
273 };
274
275 if (offset == 0) return error.InvalidBitStream;
276
277 return .{
278 .literal_length = literal_length,
279 .match_length = match_length,
280 .offset = offset,
281 };
282 }
283
284 fn executeSequenceSlice(
285 self: *DecodeState,
286 dest: []u8,
287 write_pos: usize,
288 sequence: Sequence,
289 ) (error{MalformedSequence} || DecodeLiteralsError)!void {
290 if (sequence.offset > write_pos + sequence.literal_length) return error.MalformedSequence;
291
292 try self.decodeLiteralsSlice(dest[write_pos..], sequence.literal_length);
293 const copy_start = write_pos + sequence.literal_length - sequence.offset;
294 for (
295 dest[write_pos + sequence.literal_length ..][0..sequence.match_length],
296 dest[copy_start..][0..sequence.match_length],
297 ) |*d, s| d.* = s;
298 self.written_count += sequence.match_length;
299 }
300
301 fn executeSequenceRingBuffer(
302 self: *DecodeState,
303 dest: *RingBuffer,
304 sequence: Sequence,
305 ) (error{MalformedSequence} || DecodeLiteralsError)!void {
306 if (sequence.offset > @min(dest.data.len, self.written_count + sequence.literal_length))
307 return error.MalformedSequence;
308
309 try self.decodeLiteralsRingBuffer(dest, sequence.literal_length);
310 const copy_start = dest.write_index + dest.data.len - sequence.offset;
311 const copy_slice = dest.sliceAt(copy_start, sequence.match_length);
312 dest.writeSliceForwardsAssumeCapacity(copy_slice.first);
313 dest.writeSliceForwardsAssumeCapacity(copy_slice.second);
314 self.written_count += sequence.match_length;
315 }
316
317 const DecodeSequenceError = error{
318 InvalidBitStream,
319 EndOfStream,
320 MalformedSequence,
321 MalformedFseBits,
322 } || DecodeLiteralsError;
323
324 /// Decode one sequence from `bit_reader` into `dest`, written starting at
325 /// `write_pos` and update FSE states if `last_sequence` is `false`.
326 /// `prepare()` must be called for the block before attempting to decode
327 /// sequences.
328 ///
329 /// Errors returned:
330 /// - `error.MalformedSequence` if the decompressed sequence would be
331 /// longer than `sequence_size_limit` or the sequence's offset is too
332 /// large
333 /// - `error.UnexpectedEndOfLiteralStream` if the decoder state's literal
334 /// streams do not contain enough literals for the sequence (this may
335 /// mean the literal stream or the sequence is malformed).
336 /// - `error.InvalidBitStream` if the FSE sequence bitstream is malformed
337 /// - `error.EndOfStream` if `bit_reader` does not contain enough bits
338 /// - `error.DestTooSmall` if `dest` is not large enough to holde the
339 /// decompressed sequence
340 pub fn decodeSequenceSlice(
341 self: *DecodeState,
342 dest: []u8,
343 write_pos: usize,
344 bit_reader: *readers.ReverseBitReader,
345 sequence_size_limit: usize,
346 last_sequence: bool,
347 ) (error{DestTooSmall} || DecodeSequenceError)!usize {
348 const sequence = try self.nextSequence(bit_reader);
349 const sequence_length = @as(usize, sequence.literal_length) + sequence.match_length;
350 if (sequence_length > sequence_size_limit) return error.MalformedSequence;
351 if (sequence_length > dest[write_pos..].len) return error.DestTooSmall;
352
353 try self.executeSequenceSlice(dest, write_pos, sequence);
354 if (!last_sequence) {
355 try self.updateState(.literal, bit_reader);
356 try self.updateState(.match, bit_reader);
357 try self.updateState(.offset, bit_reader);
358 }
359 return sequence_length;
360 }
361
362 /// Decode one sequence from `bit_reader` into `dest`; see
363 /// `decodeSequenceSlice`.
364 pub fn decodeSequenceRingBuffer(
365 self: *DecodeState,
366 dest: *RingBuffer,
367 bit_reader: anytype,
368 sequence_size_limit: usize,
369 last_sequence: bool,
370 ) DecodeSequenceError!usize {
371 const sequence = try self.nextSequence(bit_reader);
372 const sequence_length = @as(usize, sequence.literal_length) + sequence.match_length;
373 if (sequence_length > sequence_size_limit) return error.MalformedSequence;
374
375 try self.executeSequenceRingBuffer(dest, sequence);
376 if (!last_sequence) {
377 try self.updateState(.literal, bit_reader);
378 try self.updateState(.match, bit_reader);
379 try self.updateState(.offset, bit_reader);
380 }
381 return sequence_length;
382 }
383
384 fn nextLiteralMultiStream(
385 self: *DecodeState,
386 ) error{BitStreamHasNoStartBit}!void {
387 self.literal_stream_index += 1;
388 try self.initLiteralStream(self.literal_streams.four[self.literal_stream_index]);
389 }
390
391 fn initLiteralStream(self: *DecodeState, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
392 try self.literal_stream_reader.init(bytes);
393 }
394
395 fn isLiteralStreamEmpty(self: *DecodeState) bool {
396 switch (self.literal_streams) {
397 .one => return self.literal_stream_reader.isEmpty(),
398 .four => return self.literal_stream_index == 3 and self.literal_stream_reader.isEmpty(),
399 }
400 }
401
402 const LiteralBitsError = error{
403 BitStreamHasNoStartBit,
404 UnexpectedEndOfLiteralStream,
405 };
406 fn readLiteralsBits(
407 self: *DecodeState,
408 bit_count_to_read: u16,
409 ) LiteralBitsError!u16 {
410 return self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch bits: {
411 if (self.literal_streams == .four and self.literal_stream_index < 3) {
412 try self.nextLiteralMultiStream();
413 break :bits self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch
414 return error.UnexpectedEndOfLiteralStream;
415 } else {
416 return error.UnexpectedEndOfLiteralStream;
417 }
418 };
419 }
420
421 const DecodeLiteralsError = error{
422 MalformedLiteralsLength,
423 NotFound,
424 } || LiteralBitsError;
425
426 /// Decode `len` bytes of literals into `dest`.
427 ///
428 /// Errors returned:
429 /// - `error.MalformedLiteralsLength` if the number of literal bytes
430 /// decoded by `self` plus `len` is greater than the regenerated size of
431 /// `literals`
432 /// - `error.UnexpectedEndOfLiteralStream` and `error.NotFound` if there
433 /// are problems decoding Huffman compressed literals
434 pub fn decodeLiteralsSlice(
435 self: *DecodeState,
436 dest: []u8,
437 len: usize,
438 ) DecodeLiteralsError!void {
439 if (self.literal_written_count + len > self.literal_header.regenerated_size)
440 return error.MalformedLiteralsLength;
441
442 switch (self.literal_header.block_type) {
443 .raw => {
444 const literal_data = self.literal_streams.one[self.literal_written_count..][0..len];
445 @memcpy(dest[0..len], literal_data);
446 self.literal_written_count += len;
447 self.written_count += len;
448 },
449 .rle => {
450 for (0..len) |i| {
451 dest[i] = self.literal_streams.one[0];
452 }
453 self.literal_written_count += len;
454 self.written_count += len;
455 },
456 .compressed, .treeless => {
457 // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
458 const huffman_tree = self.huffman_tree orelse unreachable;
459 const max_bit_count = huffman_tree.max_bit_count;
460 const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
461 huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
462 max_bit_count,
463 );
464 var bits_read: u4 = 0;
465 var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
466 var bit_count_to_read: u4 = starting_bit_count;
467 for (0..len) |i| {
468 var prefix: u16 = 0;
469 while (true) {
470 const new_bits = self.readLiteralsBits(bit_count_to_read) catch |err| {
471 return err;
472 };
473 prefix <<= bit_count_to_read;
474 prefix |= new_bits;
475 bits_read += bit_count_to_read;
476 const result = huffman_tree.query(huffman_tree_index, prefix) catch |err| {
477 return err;
478 };
479
480 switch (result) {
481 .symbol => |sym| {
482 dest[i] = sym;
483 bit_count_to_read = starting_bit_count;
484 bits_read = 0;
485 huffman_tree_index = huffman_tree.symbol_count_minus_one;
486 break;
487 },
488 .index => |index| {
489 huffman_tree_index = index;
490 const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
491 huffman_tree.nodes[index].weight,
492 max_bit_count,
493 );
494 bit_count_to_read = bit_count - bits_read;
495 },
496 }
497 }
498 }
499 self.literal_written_count += len;
500 self.written_count += len;
501 },
502 }
503 }
504
505 /// Decode literals into `dest`; see `decodeLiteralsSlice()`.
506 pub fn decodeLiteralsRingBuffer(
507 self: *DecodeState,
508 dest: *RingBuffer,
509 len: usize,
510 ) DecodeLiteralsError!void {
511 if (self.literal_written_count + len > self.literal_header.regenerated_size)
512 return error.MalformedLiteralsLength;
513
514 switch (self.literal_header.block_type) {
515 .raw => {
516 const literals_end = self.literal_written_count + len;
517 const literal_data = self.literal_streams.one[self.literal_written_count..literals_end];
518 dest.writeSliceAssumeCapacity(literal_data);
519 self.literal_written_count += len;
520 self.written_count += len;
521 },
522 .rle => {
523 for (0..len) |_| {
524 dest.writeAssumeCapacity(self.literal_streams.one[0]);
525 }
526 self.literal_written_count += len;
527 self.written_count += len;
528 },
529 .compressed, .treeless => {
530 // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
531 const huffman_tree = self.huffman_tree orelse unreachable;
532 const max_bit_count = huffman_tree.max_bit_count;
533 const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
534 huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
535 max_bit_count,
536 );
537 var bits_read: u4 = 0;
538 var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
539 var bit_count_to_read: u4 = starting_bit_count;
540 for (0..len) |_| {
541 var prefix: u16 = 0;
542 while (true) {
543 const new_bits = try self.readLiteralsBits(bit_count_to_read);
544 prefix <<= bit_count_to_read;
545 prefix |= new_bits;
546 bits_read += bit_count_to_read;
547 const result = try huffman_tree.query(huffman_tree_index, prefix);
548
549 switch (result) {
550 .symbol => |sym| {
551 dest.writeAssumeCapacity(sym);
552 bit_count_to_read = starting_bit_count;
553 bits_read = 0;
554 huffman_tree_index = huffman_tree.symbol_count_minus_one;
555 break;
556 },
557 .index => |index| {
558 huffman_tree_index = index;
559 const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
560 huffman_tree.nodes[index].weight,
561 max_bit_count,
562 );
563 bit_count_to_read = bit_count - bits_read;
564 },
565 }
566 }
567 }
568 self.literal_written_count += len;
569 self.written_count += len;
570 },
571 }
572 }
573
574 fn getCode(self: *DecodeState, comptime choice: DataType) u32 {
575 return switch (@field(self, @tagName(choice)).table) {
576 .rle => |value| value,
577 .fse => |table| table[@field(self, @tagName(choice)).state].symbol,
578 };
579 }
580};
581
582/// Decode a single block from `src` into `dest`. The beginning of `src` must be
583/// the start of the block content (i.e. directly after the block header).
584/// Increments `consumed_count` by the number of bytes read from `src` to decode
585/// the block and returns the decompressed size of the block.
586///
587/// Errors returned:
588///
589/// - `error.BlockSizeOverMaximum` if block's size is larger than 1 << 17 or
590/// `dest[written_count..].len`
591/// - `error.MalformedBlockSize` if `src.len` is smaller than the block size
592/// and the block is a raw or compressed block
593/// - `error.ReservedBlock` if the block is a reserved block
594/// - `error.MalformedRleBlock` if the block is an RLE block and `src.len < 1`
595/// - `error.MalformedCompressedBlock` if there are errors decoding a
596/// compressed block
597/// - `error.DestTooSmall` is `dest` is not large enough to hold the
598/// decompressed block
599pub fn decodeBlock(
600 dest: []u8,
601 src: []const u8,
602 block_header: frame.Zstandard.Block.Header,
603 decode_state: *DecodeState,
604 consumed_count: *usize,
605 block_size_max: usize,
606 written_count: usize,
607) (error{DestTooSmall} || Error)!usize {
608 const block_size = block_header.block_size;
609 if (block_size_max < block_size) return error.BlockSizeOverMaximum;
610 switch (block_header.block_type) {
611 .raw => {
612 if (src.len < block_size) return error.MalformedBlockSize;
613 if (dest[written_count..].len < block_size) return error.DestTooSmall;
614 @memcpy(dest[written_count..][0..block_size], src[0..block_size]);
615 consumed_count.* += block_size;
616 decode_state.written_count += block_size;
617 return block_size;
618 },
619 .rle => {
620 if (src.len < 1) return error.MalformedRleBlock;
621 if (dest[written_count..].len < block_size) return error.DestTooSmall;
622 for (written_count..block_size + written_count) |write_pos| {
623 dest[write_pos] = src[0];
624 }
625 consumed_count.* += 1;
626 decode_state.written_count += block_size;
627 return block_size;
628 },
629 .compressed => {
630 if (src.len < block_size) return error.MalformedBlockSize;
631 var bytes_read: usize = 0;
632 const literals = decodeLiteralsSectionSlice(src[0..block_size], &bytes_read) catch
633 return error.MalformedCompressedBlock;
634 var fbs: std.io.FixedBufferStream = .{ .buffer = src[bytes_read..block_size] };
635 const fbs_reader = fbs.reader();
636 const sequences_header = decodeSequencesHeader(fbs_reader) catch
637 return error.MalformedCompressedBlock;
638
639 decode_state.prepare(fbs_reader, literals, sequences_header) catch
640 return error.MalformedCompressedBlock;
641
642 bytes_read += fbs.pos;
643
644 var bytes_written: usize = 0;
645 {
646 const bit_stream_bytes = src[bytes_read..block_size];
647 var bit_stream: readers.ReverseBitReader = undefined;
648 bit_stream.init(bit_stream_bytes) catch return error.MalformedCompressedBlock;
649
650 if (sequences_header.sequence_count > 0) {
651 decode_state.readInitialFseState(&bit_stream) catch
652 return error.MalformedCompressedBlock;
653
654 var sequence_size_limit = block_size_max;
655 for (0..sequences_header.sequence_count) |i| {
656 const write_pos = written_count + bytes_written;
657 const decompressed_size = decode_state.decodeSequenceSlice(
658 dest,
659 write_pos,
660 &bit_stream,
661 sequence_size_limit,
662 i == sequences_header.sequence_count - 1,
663 ) catch |err| switch (err) {
664 error.DestTooSmall => return error.DestTooSmall,
665 else => return error.MalformedCompressedBlock,
666 };
667 bytes_written += decompressed_size;
668 sequence_size_limit -= decompressed_size;
669 }
670 }
671
672 if (!bit_stream.isEmpty()) {
673 return error.MalformedCompressedBlock;
674 }
675 }
676
677 if (decode_state.literal_written_count < literals.header.regenerated_size) {
678 const len = literals.header.regenerated_size - decode_state.literal_written_count;
679 if (len > dest[written_count + bytes_written ..].len) return error.DestTooSmall;
680 decode_state.decodeLiteralsSlice(dest[written_count + bytes_written ..], len) catch
681 return error.MalformedCompressedBlock;
682 bytes_written += len;
683 }
684
685 switch (decode_state.literal_header.block_type) {
686 .treeless, .compressed => {
687 if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
688 },
689 .raw, .rle => {},
690 }
691
692 consumed_count.* += block_size;
693 return bytes_written;
694 },
695 .reserved => return error.ReservedBlock,
696 }
697}
698
699/// Decode a single block from `src` into `dest`; see `decodeBlock()`. Returns
700/// the size of the decompressed block, which can be used with `dest.sliceLast()`
701/// to get the decompressed bytes. `error.BlockSizeOverMaximum` is returned if
702/// the block's compressed or decompressed size is larger than `block_size_max`.
703pub fn decodeBlockRingBuffer(
704 dest: *RingBuffer,
705 src: []const u8,
706 block_header: frame.Zstandard.Block.Header,
707 decode_state: *DecodeState,
708 consumed_count: *usize,
709 block_size_max: usize,
710) Error!usize {
711 const block_size = block_header.block_size;
712 if (block_size_max < block_size) return error.BlockSizeOverMaximum;
713 switch (block_header.block_type) {
714 .raw => {
715 if (src.len < block_size) return error.MalformedBlockSize;
716 // dest may have length zero if block_size == 0, causing division by zero in
717 // writeSliceAssumeCapacity()
718 if (block_size > 0) {
719 const data = src[0..block_size];
720 dest.writeSliceAssumeCapacity(data);
721 consumed_count.* += block_size;
722 decode_state.written_count += block_size;
723 }
724 return block_size;
725 },
726 .rle => {
727 if (src.len < 1) return error.MalformedRleBlock;
728 for (0..block_size) |_| {
729 dest.writeAssumeCapacity(src[0]);
730 }
731 consumed_count.* += 1;
732 decode_state.written_count += block_size;
733 return block_size;
734 },
735 .compressed => {
736 if (src.len < block_size) return error.MalformedBlockSize;
737 var bytes_read: usize = 0;
738 const literals = decodeLiteralsSectionSlice(src[0..block_size], &bytes_read) catch
739 return error.MalformedCompressedBlock;
740 var fbs: std.io.FixedBufferStream = .{ .buffer = src[bytes_read..block_size] };
741 const fbs_reader = fbs.reader();
742 const sequences_header = decodeSequencesHeader(fbs_reader) catch
743 return error.MalformedCompressedBlock;
744
745 decode_state.prepare(fbs_reader, literals, sequences_header) catch
746 return error.MalformedCompressedBlock;
747
748 bytes_read += fbs.pos;
749
750 var bytes_written: usize = 0;
751 {
752 const bit_stream_bytes = src[bytes_read..block_size];
753 var bit_stream: readers.ReverseBitReader = undefined;
754 bit_stream.init(bit_stream_bytes) catch return error.MalformedCompressedBlock;
755
756 if (sequences_header.sequence_count > 0) {
757 decode_state.readInitialFseState(&bit_stream) catch
758 return error.MalformedCompressedBlock;
759
760 var sequence_size_limit = block_size_max;
761 for (0..sequences_header.sequence_count) |i| {
762 const decompressed_size = decode_state.decodeSequenceRingBuffer(
763 dest,
764 &bit_stream,
765 sequence_size_limit,
766 i == sequences_header.sequence_count - 1,
767 ) catch return error.MalformedCompressedBlock;
768 bytes_written += decompressed_size;
769 sequence_size_limit -= decompressed_size;
770 }
771 }
772
773 if (!bit_stream.isEmpty()) {
774 return error.MalformedCompressedBlock;
775 }
776 }
777
778 if (decode_state.literal_written_count < literals.header.regenerated_size) {
779 const len = literals.header.regenerated_size - decode_state.literal_written_count;
780 decode_state.decodeLiteralsRingBuffer(dest, len) catch
781 return error.MalformedCompressedBlock;
782 bytes_written += len;
783 }
784
785 switch (decode_state.literal_header.block_type) {
786 .treeless, .compressed => {
787 if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
788 },
789 .raw, .rle => {},
790 }
791
792 consumed_count.* += block_size;
793 if (bytes_written > block_size_max) return error.BlockSizeOverMaximum;
794 return bytes_written;
795 },
796 .reserved => return error.ReservedBlock,
797 }
798}
799
800/// Decode a single block from `source` into `dest`. Literal and sequence data
801/// from the block is copied into `literals_buffer` and `sequence_buffer`, which
802/// must be large enough or `error.LiteralsBufferTooSmall` and
803/// `error.SequenceBufferTooSmall` are returned (the maximum block size is an
804/// upper bound for the size of both buffers). See `decodeBlock`
805/// and `decodeBlockRingBuffer` for function that can decode a block without
806/// these extra copies. `error.EndOfStream` is returned if `source` does not
807/// contain enough bytes.
808pub fn decodeBlockReader(
809 dest: *RingBuffer,
810 in: *std.io.BufferedReader,
811 bytes_read: *usize,
812 block_header: frame.Zstandard.Block.Header,
813 decode_state: *DecodeState,
814 block_size_max: usize,
815 literals_buffer: []u8,
816 sequence_buffer: []u8,
817) !void {
818 const block_size = block_header.block_size;
819 if (block_size_max < block_size) return error.BlockSizeOverMaximum;
820 switch (block_header.block_type) {
821 .raw => {
822 if (block_size == 0) return;
823 const slice = dest.sliceAt(dest.write_index, block_size);
824 var vecs: [2][]u8 = &.{slice.first, slice.second };
825 try in.readVecAll(&vecs);
826 assert(slice.first.len + slice.second.len == block_size);
827 bytes_read.* += block_size;
828 dest.write_index = dest.mask2(dest.write_index + block_size);
829 decode_state.written_count += block_size;
830 },
831 .rle => {
832 const byte = try in.takeByte();
833 bytes_read.* += 1;
834 for (0..block_size) |_| {
835 dest.writeAssumeCapacity(byte);
836 }
837 decode_state.written_count += block_size;
838 },
839 .compressed => {
840 var block_reader_limited = std.io.limitedReader(source, block_size);
841 const block_reader = block_reader_limited.reader();
842 const literals = try decodeLiteralsSection(block_reader, literals_buffer);
843 const sequences_header = try decodeSequencesHeader(block_reader);
844
845 try decode_state.prepare(block_reader, literals, sequences_header);
846
847 var bytes_written: usize = 0;
848 {
849 const size = try block_reader.readAll(sequence_buffer);
850 var bit_stream: readers.ReverseBitReader = undefined;
851 try bit_stream.init(sequence_buffer[0..size]);
852
853 if (sequences_header.sequence_count > 0) {
854 if (sequence_buffer.len < block_reader_limited.bytes_left)
855 return error.SequenceBufferTooSmall;
856
857 decode_state.readInitialFseState(&bit_stream) catch
858 return error.MalformedCompressedBlock;
859
860 var sequence_size_limit = block_size_max;
861 for (0..sequences_header.sequence_count) |i| {
862 const decompressed_size = decode_state.decodeSequenceRingBuffer(
863 dest,
864 &bit_stream,
865 sequence_size_limit,
866 i == sequences_header.sequence_count - 1,
867 ) catch return error.MalformedCompressedBlock;
868 sequence_size_limit -= decompressed_size;
869 bytes_written += decompressed_size;
870 }
871 }
872
873 if (!bit_stream.isEmpty()) {
874 return error.MalformedCompressedBlock;
875 }
876 }
877
878 if (decode_state.literal_written_count < literals.header.regenerated_size) {
879 const len = literals.header.regenerated_size - decode_state.literal_written_count;
880 decode_state.decodeLiteralsRingBuffer(dest, len) catch
881 return error.MalformedCompressedBlock;
882 bytes_written += len;
883 }
884
885 switch (decode_state.literal_header.block_type) {
886 .treeless, .compressed => {
887 if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
888 },
889 .raw, .rle => {},
890 }
891
892 if (bytes_written > block_size_max) return error.BlockSizeOverMaximum;
893 if (block_reader_limited.bytes_left != 0) return error.MalformedCompressedBlock;
894 decode_state.literal_written_count = 0;
895 },
896 .reserved => return error.ReservedBlock,
897 }
898}
899
900/// Decode the header of a block.
901pub fn decodeBlockHeader(src: *const [3]u8) frame.Zstandard.Block.Header {
902 const last_block = src[0] & 1 == 1;
903 const block_type = @as(frame.Zstandard.Block.Type, @enumFromInt((src[0] & 0b110) >> 1));
904 const block_size = ((src[0] & 0b11111000) >> 3) + (@as(u21, src[1]) << 5) + (@as(u21, src[2]) << 13);
905 return .{
906 .last_block = last_block,
907 .block_type = block_type,
908 .block_size = block_size,
909 };
910}
911
912/// Decode the header of a block.
913///
914/// Errors returned:
915/// - `error.EndOfStream` if `src.len < 3`
916pub fn decodeBlockHeaderSlice(src: []const u8) error{EndOfStream}!frame.Zstandard.Block.Header {
917 if (src.len < 3) return error.EndOfStream;
918 return decodeBlockHeader(src[0..3]);
919}
920
921/// Decode a `LiteralsSection` from `src`, incrementing `consumed_count` by the
922/// number of bytes the section uses.
923///
924/// Errors returned:
925/// - `error.MalformedLiteralsHeader` if the header is invalid
926/// - `error.MalformedLiteralsSection` if there are decoding errors
927/// - `error.MalformedAccuracyLog` if compressed literals have invalid
928/// accuracy
929/// - `error.MalformedFseTable` if compressed literals have invalid FSE table
930/// - `error.MalformedHuffmanTree` if there are errors decoding a Huffamn tree
931/// - `error.EndOfStream` if there are not enough bytes in `src`
932pub fn decodeLiteralsSectionSlice(
933 src: []const u8,
934 consumed_count: *usize,
935) (error{ MalformedLiteralsHeader, MalformedLiteralsSection, EndOfStream } || huffman.Error)!LiteralsSection {
936 var bytes_read: usize = 0;
937 const header = header: {
938 var fbs: std.io.FixedBufferStream = .{ .buffer = src };
939 defer bytes_read = fbs.pos;
940 break :header decodeLiteralsHeader(fbs.reader()) catch return error.MalformedLiteralsHeader;
941 };
942 switch (header.block_type) {
943 .raw => {
944 if (src.len < bytes_read + header.regenerated_size) return error.MalformedLiteralsSection;
945 const stream = src[bytes_read..][0..header.regenerated_size];
946 consumed_count.* += header.regenerated_size + bytes_read;
947 return LiteralsSection{
948 .header = header,
949 .huffman_tree = null,
950 .streams = .{ .one = stream },
951 };
952 },
953 .rle => {
954 if (src.len < bytes_read + 1) return error.MalformedLiteralsSection;
955 const stream = src[bytes_read..][0..1];
956 consumed_count.* += 1 + bytes_read;
957 return LiteralsSection{
958 .header = header,
959 .huffman_tree = null,
960 .streams = .{ .one = stream },
961 };
962 },
963 .compressed, .treeless => {
964 const huffman_tree_start = bytes_read;
965 const huffman_tree = if (header.block_type == .compressed)
966 try huffman.decodeHuffmanTreeSlice(src[bytes_read..], &bytes_read)
967 else
968 null;
969 const huffman_tree_size = bytes_read - huffman_tree_start;
970 const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
971 return error.MalformedLiteralsSection;
972
973 if (src.len < bytes_read + total_streams_size) return error.MalformedLiteralsSection;
974 const stream_data = src[bytes_read .. bytes_read + total_streams_size];
975
976 const streams = try decodeStreams(header.size_format, stream_data);
977 consumed_count.* += bytes_read + total_streams_size;
978 return LiteralsSection{
979 .header = header,
980 .huffman_tree = huffman_tree,
981 .streams = streams,
982 };
983 },
984 }
985}
986
987/// Decode a `LiteralsSection` from `src`, incrementing `consumed_count` by the
988/// number of bytes the section uses. See `decodeLiterasSectionSlice()`.
989pub fn decodeLiteralsSection(
990 source: anytype,
991 buffer: []u8,
992) !LiteralsSection {
993 const header = try decodeLiteralsHeader(source);
994 switch (header.block_type) {
995 .raw => {
996 if (buffer.len < header.regenerated_size) return error.LiteralsBufferTooSmall;
997 try source.readNoEof(buffer[0..header.regenerated_size]);
998 return LiteralsSection{
999 .header = header,
1000 .huffman_tree = null,
1001 .streams = .{ .one = buffer },
1002 };
1003 },
1004 .rle => {
1005 buffer[0] = try source.readByte();
1006 return LiteralsSection{
1007 .header = header,
1008 .huffman_tree = null,
1009 .streams = .{ .one = buffer[0..1] },
1010 };
1011 },
1012 .compressed, .treeless => {
1013 var counting_reader = std.io.countingReader(source);
1014 const huffman_tree = if (header.block_type == .compressed)
1015 try huffman.decodeHuffmanTree(counting_reader.reader(), buffer)
1016 else
1017 null;
1018 const huffman_tree_size = @as(usize, @intCast(counting_reader.bytes_read));
1019 const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
1020 return error.MalformedLiteralsSection;
1021
1022 if (total_streams_size > buffer.len) return error.LiteralsBufferTooSmall;
1023 try source.readNoEof(buffer[0..total_streams_size]);
1024 const stream_data = buffer[0..total_streams_size];
1025
1026 const streams = try decodeStreams(header.size_format, stream_data);
1027 return LiteralsSection{
1028 .header = header,
1029 .huffman_tree = huffman_tree,
1030 .streams = streams,
1031 };
1032 },
1033 }
1034}
1035
1036fn decodeStreams(size_format: u2, stream_data: []const u8) !LiteralsSection.Streams {
1037 if (size_format == 0) {
1038 return .{ .one = stream_data };
1039 }
1040
1041 if (stream_data.len < 6) return error.MalformedLiteralsSection;
1042
1043 const stream_1_length: usize = std.mem.readInt(u16, stream_data[0..2], .little);
1044 const stream_2_length: usize = std.mem.readInt(u16, stream_data[2..4], .little);
1045 const stream_3_length: usize = std.mem.readInt(u16, stream_data[4..6], .little);
1046
1047 const stream_1_start = 6;
1048 const stream_2_start = stream_1_start + stream_1_length;
1049 const stream_3_start = stream_2_start + stream_2_length;
1050 const stream_4_start = stream_3_start + stream_3_length;
1051
1052 if (stream_data.len < stream_4_start) return error.MalformedLiteralsSection;
1053
1054 return .{ .four = .{
1055 stream_data[stream_1_start .. stream_1_start + stream_1_length],
1056 stream_data[stream_2_start .. stream_2_start + stream_2_length],
1057 stream_data[stream_3_start .. stream_3_start + stream_3_length],
1058 stream_data[stream_4_start..],
1059 } };
1060}
1061
1062/// Decode a literals section header.
1063///
1064/// Errors returned:
1065/// - `error.EndOfStream` if there are not enough bytes in `source`
1066pub fn decodeLiteralsHeader(source: anytype) !LiteralsSection.Header {
1067 const byte0 = try source.readByte();
1068 const block_type = @as(LiteralsSection.BlockType, @enumFromInt(byte0 & 0b11));
1069 const size_format = @as(u2, @intCast((byte0 & 0b1100) >> 2));
1070 var regenerated_size: u20 = undefined;
1071 var compressed_size: ?u18 = null;
1072 switch (block_type) {
1073 .raw, .rle => {
1074 switch (size_format) {
1075 0, 2 => {
1076 regenerated_size = byte0 >> 3;
1077 },
1078 1 => regenerated_size = (byte0 >> 4) + (@as(u20, try source.readByte()) << 4),
1079 3 => regenerated_size = (byte0 >> 4) +
1080 (@as(u20, try source.readByte()) << 4) +
1081 (@as(u20, try source.readByte()) << 12),
1082 }
1083 },
1084 .compressed, .treeless => {
1085 const byte1 = try source.readByte();
1086 const byte2 = try source.readByte();
1087 switch (size_format) {
1088 0, 1 => {
1089 regenerated_size = (byte0 >> 4) + ((@as(u20, byte1) & 0b00111111) << 4);
1090 compressed_size = ((byte1 & 0b11000000) >> 6) + (@as(u18, byte2) << 2);
1091 },
1092 2 => {
1093 const byte3 = try source.readByte();
1094 regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00000011) << 12);
1095 compressed_size = ((byte2 & 0b11111100) >> 2) + (@as(u18, byte3) << 6);
1096 },
1097 3 => {
1098 const byte3 = try source.readByte();
1099 const byte4 = try source.readByte();
1100 regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00111111) << 12);
1101 compressed_size = ((byte2 & 0b11000000) >> 6) + (@as(u18, byte3) << 2) + (@as(u18, byte4) << 10);
1102 },
1103 }
1104 },
1105 }
1106 return LiteralsSection.Header{
1107 .block_type = block_type,
1108 .size_format = size_format,
1109 .regenerated_size = regenerated_size,
1110 .compressed_size = compressed_size,
1111 };
1112}
1113
1114/// Decode a sequences section header.
1115///
1116/// Errors returned:
1117/// - `error.ReservedBitSet` if the reserved bit is set
1118/// - `error.EndOfStream` if there are not enough bytes in `source`
1119pub fn decodeSequencesHeader(
1120 source: anytype,
1121) !SequencesSection.Header {
1122 var sequence_count: u24 = undefined;
1123
1124 const byte0 = try source.readByte();
1125 if (byte0 == 0) {
1126 return SequencesSection.Header{
1127 .sequence_count = 0,
1128 .offsets = undefined,
1129 .match_lengths = undefined,
1130 .literal_lengths = undefined,
1131 };
1132 } else if (byte0 < 128) {
1133 sequence_count = byte0;
1134 } else if (byte0 < 255) {
1135 sequence_count = (@as(u24, (byte0 - 128)) << 8) + try source.readByte();
1136 } else {
1137 sequence_count = (try source.readByte()) + (@as(u24, try source.readByte()) << 8) + 0x7F00;
1138 }
1139
1140 const compression_modes = try source.readByte();
1141
1142 const matches_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00001100) >> 2));
1143 const offsets_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00110000) >> 4));
1144 const literal_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b11000000) >> 6));
1145 if (compression_modes & 0b11 != 0) return error.ReservedBitSet;
1146
1147 return SequencesSection.Header{
1148 .sequence_count = sequence_count,
1149 .offsets = offsets_mode,
1150 .match_lengths = matches_mode,
1151 .literal_lengths = literal_mode,
1152 };
1153}
lib/std/compress/zstandard/decode/fse.zig deleted-153
......@@ -1,153 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3
4const types = @import("../types.zig");
5const Table = types.compressed_block.Table;
6
7pub fn decodeFseTable(
8 bit_reader: anytype,
9 expected_symbol_count: usize,
10 max_accuracy_log: u4,
11 entries: []Table.Fse,
12) !usize {
13 const accuracy_log_biased = try bit_reader.readBitsNoEof(u4, 4);
14 if (accuracy_log_biased > max_accuracy_log -| 5) return error.MalformedAccuracyLog;
15 const accuracy_log = accuracy_log_biased + 5;
16
17 var values: [256]u16 = undefined;
18 var value_count: usize = 0;
19
20 const total_probability = @as(u16, 1) << accuracy_log;
21 var accumulated_probability: u16 = 0;
22
23 while (accumulated_probability < total_probability) {
24 // WARNING: The RFC is poorly worded, and would suggest std.math.log2_int_ceil is correct here,
25 // but power of two (remaining probabilities + 1) need max bits set to 1 more.
26 const max_bits = std.math.log2_int(u16, total_probability - accumulated_probability + 1) + 1;
27 const small = try bit_reader.readBitsNoEof(u16, max_bits - 1);
28
29 const cutoff = (@as(u16, 1) << max_bits) - 1 - (total_probability - accumulated_probability + 1);
30
31 const value = if (small < cutoff)
32 small
33 else value: {
34 const value_read = small + (try bit_reader.readBitsNoEof(u16, 1) << (max_bits - 1));
35 break :value if (value_read < @as(u16, 1) << (max_bits - 1))
36 value_read
37 else
38 value_read - cutoff;
39 };
40
41 accumulated_probability += if (value != 0) value - 1 else 1;
42
43 values[value_count] = value;
44 value_count += 1;
45
46 if (value == 1) {
47 while (true) {
48 const repeat_flag = try bit_reader.readBitsNoEof(u2, 2);
49 if (repeat_flag + value_count > 256) return error.MalformedFseTable;
50 for (0..repeat_flag) |_| {
51 values[value_count] = 1;
52 value_count += 1;
53 }
54 if (repeat_flag < 3) break;
55 }
56 }
57 if (value_count == 256) break;
58 }
59 bit_reader.alignToByte();
60
61 if (value_count < 2) return error.MalformedFseTable;
62 if (accumulated_probability != total_probability) return error.MalformedFseTable;
63 if (value_count > expected_symbol_count) return error.MalformedFseTable;
64
65 const table_size = total_probability;
66
67 try buildFseTable(values[0..value_count], entries[0..table_size]);
68 return table_size;
69}
70
71fn buildFseTable(values: []const u16, entries: []Table.Fse) !void {
72 const total_probability = @as(u16, @intCast(entries.len));
73 const accuracy_log = std.math.log2_int(u16, total_probability);
74 assert(total_probability <= 1 << 9);
75
76 var less_than_one_count: usize = 0;
77 for (values, 0..) |value, i| {
78 if (value == 0) {
79 entries[entries.len - 1 - less_than_one_count] = Table.Fse{
80 .symbol = @as(u8, @intCast(i)),
81 .baseline = 0,
82 .bits = accuracy_log,
83 };
84 less_than_one_count += 1;
85 }
86 }
87
88 var position: usize = 0;
89 var temp_states: [1 << 9]u16 = undefined;
90 for (values, 0..) |value, symbol| {
91 if (value == 0 or value == 1) continue;
92 const probability = value - 1;
93
94 const state_share_dividend = std.math.ceilPowerOfTwo(u16, probability) catch
95 return error.MalformedFseTable;
96 const share_size = @divExact(total_probability, state_share_dividend);
97 const double_state_count = state_share_dividend - probability;
98 const single_state_count = probability - double_state_count;
99 const share_size_log = std.math.log2_int(u16, share_size);
100
101 for (0..probability) |i| {
102 temp_states[i] = @as(u16, @intCast(position));
103 position += (entries.len >> 1) + (entries.len >> 3) + 3;
104 position &= entries.len - 1;
105 while (position >= entries.len - less_than_one_count) {
106 position += (entries.len >> 1) + (entries.len >> 3) + 3;
107 position &= entries.len - 1;
108 }
109 }
110 std.mem.sort(u16, temp_states[0..probability], {}, std.sort.asc(u16));
111 for (0..probability) |i| {
112 entries[temp_states[i]] = if (i < double_state_count) Table.Fse{
113 .symbol = @as(u8, @intCast(symbol)),
114 .bits = share_size_log + 1,
115 .baseline = single_state_count * share_size + @as(u16, @intCast(i)) * 2 * share_size,
116 } else Table.Fse{
117 .symbol = @as(u8, @intCast(symbol)),
118 .bits = share_size_log,
119 .baseline = (@as(u16, @intCast(i)) - double_state_count) * share_size,
120 };
121 }
122 }
123}
124
125test buildFseTable {
126 const literals_length_default_values = [36]u16{
127 5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2,
128 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 2, 2, 2, 2, 2,
129 0, 0, 0, 0,
130 };
131
132 const match_lengths_default_values = [53]u16{
133 2, 5, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
134 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
135 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0,
136 0, 0, 0, 0, 0,
137 };
138
139 const offset_codes_default_values = [29]u16{
140 2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
141 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0,
142 };
143
144 var entries: [64]Table.Fse = undefined;
145 try buildFseTable(&literals_length_default_values, &entries);
146 try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_literal_fse_table.fse, &entries);
147
148 try buildFseTable(&match_lengths_default_values, &entries);
149 try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_match_fse_table.fse, &entries);
150
151 try buildFseTable(&offset_codes_default_values, entries[0..32]);
152 try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_offset_fse_table.fse, entries[0..32]);
153}
lib/std/compress/zstandard/decode/huffman.zig deleted-234
......@@ -1,234 +0,0 @@
1const std = @import("std");
2
3const types = @import("../types.zig");
4const LiteralsSection = types.compressed_block.LiteralsSection;
5const Table = types.compressed_block.Table;
6
7const readers = @import("../readers.zig");
8
9const decodeFseTable = @import("fse.zig").decodeFseTable;
10
11pub const Error = error{
12 MalformedHuffmanTree,
13 MalformedFseTable,
14 MalformedAccuracyLog,
15 EndOfStream,
16};
17
18fn decodeFseHuffmanTree(
19 source: anytype,
20 compressed_size: usize,
21 buffer: []u8,
22 weights: *[256]u4,
23) !usize {
24 var stream = std.io.limitedReader(source, compressed_size);
25 var bit_reader = readers.bitReader(stream.reader());
26
27 var entries: [1 << 6]Table.Fse = undefined;
28 const table_size = decodeFseTable(&bit_reader, 256, 6, &entries) catch |err| switch (err) {
29 error.MalformedAccuracyLog, error.MalformedFseTable => |e| return e,
30 error.EndOfStream => return error.MalformedFseTable,
31 else => |e| return e,
32 };
33 const accuracy_log = std.math.log2_int_ceil(usize, table_size);
34
35 const amount = try stream.reader().readAll(buffer);
36 var huff_bits: readers.ReverseBitReader = undefined;
37 huff_bits.init(buffer[0..amount]) catch return error.MalformedHuffmanTree;
38
39 return assignWeights(&huff_bits, accuracy_log, &entries, weights);
40}
41
42fn decodeFseHuffmanTreeSlice(src: []const u8, compressed_size: usize, weights: *[256]u4) !usize {
43 if (src.len < compressed_size) return error.MalformedHuffmanTree;
44 var stream: std.io.FixedBufferStream = .{ .buffer = src[0..compressed_size] };
45 var counting_reader = std.io.countingReader(stream.reader());
46 var bit_reader = readers.bitReader(counting_reader.reader());
47
48 var entries: [1 << 6]Table.Fse = undefined;
49 const table_size = decodeFseTable(&bit_reader, 256, 6, &entries) catch |err| switch (err) {
50 error.MalformedAccuracyLog, error.MalformedFseTable => |e| return e,
51 error.EndOfStream => return error.MalformedFseTable,
52 };
53 const accuracy_log = std.math.log2_int_ceil(usize, table_size);
54
55 const start_index = std.math.cast(usize, counting_reader.bytes_read) orelse
56 return error.MalformedHuffmanTree;
57 const huff_data = src[start_index..compressed_size];
58 var huff_bits: readers.ReverseBitReader = undefined;
59 huff_bits.init(huff_data) catch return error.MalformedHuffmanTree;
60
61 return assignWeights(&huff_bits, accuracy_log, &entries, weights);
62}
63
64fn assignWeights(
65 huff_bits: *readers.ReverseBitReader,
66 accuracy_log: u16,
67 entries: *[1 << 6]Table.Fse,
68 weights: *[256]u4,
69) !usize {
70 var i: usize = 0;
71 var even_state: u32 = huff_bits.readBitsNoEof(u32, accuracy_log) catch return error.MalformedHuffmanTree;
72 var odd_state: u32 = huff_bits.readBitsNoEof(u32, accuracy_log) catch return error.MalformedHuffmanTree;
73
74 while (i < 254) {
75 const even_data = entries[even_state];
76 var read_bits: u16 = 0;
77 const even_bits = huff_bits.readBits(u32, even_data.bits, &read_bits) catch unreachable;
78 weights[i] = std.math.cast(u4, even_data.symbol) orelse return error.MalformedHuffmanTree;
79 i += 1;
80 if (read_bits < even_data.bits) {
81 weights[i] = std.math.cast(u4, entries[odd_state].symbol) orelse return error.MalformedHuffmanTree;
82 i += 1;
83 break;
84 }
85 even_state = even_data.baseline + even_bits;
86
87 read_bits = 0;
88 const odd_data = entries[odd_state];
89 const odd_bits = huff_bits.readBits(u32, odd_data.bits, &read_bits) catch unreachable;
90 weights[i] = std.math.cast(u4, odd_data.symbol) orelse return error.MalformedHuffmanTree;
91 i += 1;
92 if (read_bits < odd_data.bits) {
93 if (i == 255) return error.MalformedHuffmanTree;
94 weights[i] = std.math.cast(u4, entries[even_state].symbol) orelse return error.MalformedHuffmanTree;
95 i += 1;
96 break;
97 }
98 odd_state = odd_data.baseline + odd_bits;
99 } else return error.MalformedHuffmanTree;
100
101 if (!huff_bits.isEmpty()) {
102 return error.MalformedHuffmanTree;
103 }
104
105 return i + 1; // stream contains all but the last symbol
106}
107
108fn decodeDirectHuffmanTree(source: anytype, encoded_symbol_count: usize, weights: *[256]u4) !usize {
109 const weights_byte_count = (encoded_symbol_count + 1) / 2;
110 for (0..weights_byte_count) |i| {
111 const byte = try source.readByte();
112 weights[2 * i] = @as(u4, @intCast(byte >> 4));
113 weights[2 * i + 1] = @as(u4, @intCast(byte & 0xF));
114 }
115 return encoded_symbol_count + 1;
116}
117
118fn assignSymbols(weight_sorted_prefixed_symbols: []LiteralsSection.HuffmanTree.PrefixedSymbol, weights: [256]u4) usize {
119 for (0..weight_sorted_prefixed_symbols.len) |i| {
120 weight_sorted_prefixed_symbols[i] = .{
121 .symbol = @as(u8, @intCast(i)),
122 .weight = undefined,
123 .prefix = undefined,
124 };
125 }
126
127 std.mem.sort(
128 LiteralsSection.HuffmanTree.PrefixedSymbol,
129 weight_sorted_prefixed_symbols,
130 weights,
131 lessThanByWeight,
132 );
133
134 var prefix: u16 = 0;
135 var prefixed_symbol_count: usize = 0;
136 var sorted_index: usize = 0;
137 const symbol_count = weight_sorted_prefixed_symbols.len;
138 while (sorted_index < symbol_count) {
139 var symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
140 const weight = weights[symbol];
141 if (weight == 0) {
142 sorted_index += 1;
143 continue;
144 }
145
146 while (sorted_index < symbol_count) : ({
147 sorted_index += 1;
148 prefixed_symbol_count += 1;
149 prefix += 1;
150 }) {
151 symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
152 if (weights[symbol] != weight) {
153 prefix = ((prefix - 1) >> (weights[symbol] - weight)) + 1;
154 break;
155 }
156 weight_sorted_prefixed_symbols[prefixed_symbol_count].symbol = symbol;
157 weight_sorted_prefixed_symbols[prefixed_symbol_count].prefix = prefix;
158 weight_sorted_prefixed_symbols[prefixed_symbol_count].weight = weight;
159 }
160 }
161 return prefixed_symbol_count;
162}
163
164fn buildHuffmanTree(weights: *[256]u4, symbol_count: usize) error{MalformedHuffmanTree}!LiteralsSection.HuffmanTree {
165 var weight_power_sum_big: u32 = 0;
166 for (weights[0 .. symbol_count - 1]) |value| {
167 weight_power_sum_big += (@as(u16, 1) << value) >> 1;
168 }
169 if (weight_power_sum_big >= 1 << 11) return error.MalformedHuffmanTree;
170 const weight_power_sum = @as(u16, @intCast(weight_power_sum_big));
171
172 // advance to next power of two (even if weight_power_sum is a power of 2)
173 // TODO: is it valid to have weight_power_sum == 0?
174 const max_number_of_bits = if (weight_power_sum == 0) 1 else std.math.log2_int(u16, weight_power_sum) + 1;
175 const next_power_of_two = @as(u16, 1) << max_number_of_bits;
176 weights[symbol_count - 1] = std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1;
177
178 var weight_sorted_prefixed_symbols: [256]LiteralsSection.HuffmanTree.PrefixedSymbol = undefined;
179 const prefixed_symbol_count = assignSymbols(weight_sorted_prefixed_symbols[0..symbol_count], weights.*);
180 const tree = LiteralsSection.HuffmanTree{
181 .max_bit_count = max_number_of_bits,
182 .symbol_count_minus_one = @as(u8, @intCast(prefixed_symbol_count - 1)),
183 .nodes = weight_sorted_prefixed_symbols,
184 };
185 return tree;
186}
187
188pub fn decodeHuffmanTree(
189 source: anytype,
190 buffer: []u8,
191) (@TypeOf(source).Error || Error)!LiteralsSection.HuffmanTree {
192 const header = try source.readByte();
193 var weights: [256]u4 = undefined;
194 const symbol_count = if (header < 128)
195 // FSE compressed weights
196 try decodeFseHuffmanTree(source, header, buffer, &weights)
197 else
198 try decodeDirectHuffmanTree(source, header - 127, &weights);
199
200 return buildHuffmanTree(&weights, symbol_count);
201}
202
203pub fn decodeHuffmanTreeSlice(
204 src: []const u8,
205 consumed_count: *usize,
206) Error!LiteralsSection.HuffmanTree {
207 if (src.len == 0) return error.MalformedHuffmanTree;
208 const header = src[0];
209 var bytes_read: usize = 1;
210 var weights: [256]u4 = undefined;
211 const symbol_count = if (header < 128) count: {
212 // FSE compressed weights
213 bytes_read += header;
214 break :count try decodeFseHuffmanTreeSlice(src[1..], header, &weights);
215 } else count: {
216 var fbs: std.io.FixedBufferStream = .{ .buffer = src[1..] };
217 defer bytes_read += fbs.pos;
218 break :count try decodeDirectHuffmanTree(fbs.reader(), header - 127, &weights);
219 };
220
221 consumed_count.* += bytes_read;
222 return buildHuffmanTree(&weights, symbol_count);
223}
224
225fn lessThanByWeight(
226 weights: [256]u4,
227 lhs: LiteralsSection.HuffmanTree.PrefixedSymbol,
228 rhs: LiteralsSection.HuffmanTree.PrefixedSymbol,
229) bool {
230 // NOTE: this function relies on the use of a stable sorting algorithm,
231 // otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs;
232 // should be added
233 return weights[lhs.symbol] < weights[rhs.symbol];
234}
lib/std/compress/zstandard/decompress.zig deleted-441
......@@ -1,441 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const RingBuffer = std.RingBuffer;
4
5const types = @import("types.zig");
6const frame = types.frame;
7const LiteralsSection = types.compressed_block.LiteralsSection;
8const SequencesSection = types.compressed_block.SequencesSection;
9const SkippableHeader = types.frame.Skippable.Header;
10const ZstandardHeader = types.frame.Zstandard.Header;
11const Table = types.compressed_block.Table;
12
13pub const block = @import("decode/block.zig");
14
15const readers = @import("readers.zig");
16
17/// Returns `true` is `magic` is a valid magic number for a skippable frame
18pub fn isSkippableMagic(magic: u32) bool {
19 return frame.Skippable.magic_number_min <= magic and magic <= frame.Skippable.magic_number_max;
20}
21
22/// Returns the kind of frame at the beginning of `source`.
23///
24/// Errors returned:
25/// - `error.BadMagic` if `source` begins with bytes not equal to the
26/// Zstandard frame magic number, or outside the range of magic numbers for
27/// skippable frames.
28/// - `error.EndOfStream` if `source` contains fewer than 4 bytes
29pub fn decodeFrameType(source: anytype) error{ BadMagic, EndOfStream }!frame.Kind {
30 const magic = try source.readInt(u32, .little);
31 return frameType(magic);
32}
33
34/// Returns the kind of frame associated to `magic`.
35///
36/// Errors returned:
37/// - `error.BadMagic` if `magic` is not a valid magic number.
38pub fn frameType(magic: u32) error{BadMagic}!frame.Kind {
39 return if (magic == frame.Zstandard.magic_number)
40 .zstandard
41 else if (isSkippableMagic(magic))
42 .skippable
43 else
44 error.BadMagic;
45}
46
47pub const FrameHeader = union(enum) {
48 zstandard: ZstandardHeader,
49 skippable: SkippableHeader,
50};
51
52pub const HeaderError = error{ ReadFailed, BadMagic, EndOfStream, ReservedBitSet };
53
54/// Returns the header of the frame at the beginning of `source`.
55///
56/// Errors returned:
57/// - `error.BadMagic` if `source` begins with bytes not equal to the
58/// Zstandard frame magic number, or outside the range of magic numbers for
59/// skippable frames.
60/// - `error.EndOfStream` if `source` contains fewer than 4 bytes
61/// - `error.ReservedBitSet` if the frame is a Zstandard frame and any of the
62/// reserved bits are set
63pub fn decodeFrameHeader(br: *std.io.BufferedReader, bytes_read: *usize) HeaderError!FrameHeader {
64 const magic = try br.readInt(u32, .little);
65 bytes_read.* += 4;
66 const frame_type = try frameType(magic);
67 switch (frame_type) {
68 .zstandard => return .{ .zstandard = try decodeZstandardHeader(br, bytes_read) },
69 .skippable => {
70 const result: FrameHeader = .{
71 .skippable = .{
72 .magic_number = magic,
73 .frame_size = try br.readInt(u32, .little),
74 },
75 };
76 bytes_read.* += 4;
77 return result;
78 },
79 }
80}
81
82pub const ReadWriteCount = struct {
83 read_count: usize,
84 write_count: usize,
85};
86
87/// Decodes frames from `src` into `dest`; returns the length of the result.
88/// The stream should not have extra trailing bytes - either all bytes in `src`
89/// will be decoded, or an error will be returned. An error will be returned if
90/// a Zstandard frame in `src` does not declare its content size.
91///
92/// Errors returned:
93/// - `error.DictionaryIdFlagUnsupported` if a `src` contains a frame that
94/// uses a dictionary
95/// - `error.MalformedFrame` if a frame in `src` is invalid
96/// - `error.UnknownContentSizeUnsupported` if a frame in `src` does not
97/// declare its content size
98pub fn decode(dest: []u8, src: []const u8, verify_checksum: bool) error{
99 MalformedFrame,
100 UnknownContentSizeUnsupported,
101 DictionaryIdFlagUnsupported,
102}!usize {
103 var write_count: usize = 0;
104 var read_count: usize = 0;
105 while (read_count < src.len) {
106 const counts = decodeFrame(dest, src[read_count..], verify_checksum) catch |err| {
107 switch (err) {
108 error.UnknownContentSizeUnsupported => return error.UnknownContentSizeUnsupported,
109 error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
110 else => return error.MalformedFrame,
111 }
112 };
113 read_count += counts.read_count;
114 write_count += counts.write_count;
115 }
116 return write_count;
117}
118
119/// Decodes the frame at the start of `src` into `dest`. Returns the number of
120/// bytes read from `src` and written to `dest`. This function can only decode
121/// frames that declare the decompressed content size.
122///
123/// Errors returned:
124/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic
125/// number for a Zstandard or skippable frame
126/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
127/// uncompressed content size
128/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
129/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
130/// size declared by the frame header
131/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
132/// that is larger than `std.math.maxInt(usize)`
133/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
134/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
135/// contains a checksum that does not match the checksum of the decompressed
136/// data
137/// - `error.ReservedBitSet` if any of the reserved bits of the frame header
138/// are set
139/// - `error.EndOfStream` if `src` does not contain a complete frame
140/// - `error.BadContentSize` if the content size declared by the frame does
141/// not equal the actual size of decompressed data
142/// - an error in `block.Error` if there are errors decoding a block
143/// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a
144/// size greater than `src.len`
145pub fn decodeFrame(
146 dest: []u8,
147 src: []const u8,
148 verify_checksum: bool,
149) (error{
150 BadMagic,
151 UnknownContentSizeUnsupported,
152 ContentTooLarge,
153 ContentSizeTooLarge,
154 WindowSizeUnknown,
155 DictionaryIdFlagUnsupported,
156 SkippableSizeTooLarge,
157} || FrameError)!ReadWriteCount {
158 var fbs: std.io.FixedBufferStream = .{ .buffer = src };
159 switch (try decodeFrameType(fbs.reader())) {
160 .zstandard => return decodeZstandardFrame(dest, src, verify_checksum),
161 .skippable => {
162 const content_size = try fbs.reader().readInt(u32, .little);
163 if (content_size > std.math.maxInt(usize) - 8) return error.SkippableSizeTooLarge;
164 const read_count = @as(usize, content_size) + 8;
165 if (read_count > src.len) return error.SkippableSizeTooLarge;
166 return ReadWriteCount{
167 .read_count = read_count,
168 .write_count = 0,
169 };
170 },
171 }
172}
173
174/// Returns the frame checksum corresponding to the data fed into `hasher`
175pub fn computeChecksum(hasher: *std.hash.XxHash64) u32 {
176 const hash = hasher.final();
177 return @as(u32, @intCast(hash & 0xFFFFFFFF));
178}
179
180const FrameError = error{
181 ChecksumFailure,
182 BadContentSize,
183 EndOfStream,
184 ReservedBitSet,
185} || block.Error;
186
187/// Decode a Zstandard frame from `src` into `dest`, returning the number of
188/// bytes read from `src` and written to `dest`. The first four bytes of `src`
189/// must be the magic number for a Zstandard frame.
190///
191/// Error returned:
192/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
193/// uncompressed content size
194/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
195/// size declared by the frame header
196/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
197/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
198/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
199/// that is larger than `std.math.maxInt(usize)`
200/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
201/// contains a checksum that does not match the checksum of the decompressed
202/// data
203/// - `error.ReservedBitSet` if the reserved bit of the frame header is set
204/// - `error.EndOfStream` if `src` does not contain a complete frame
205/// - an error in `block.Error` if there are errors decoding a block
206/// - `error.BadContentSize` if the content size declared by the frame does
207/// not equal the actual size of decompressed data
208pub fn decodeZstandardFrame(
209 dest: []u8,
210 src: []const u8,
211 verify_checksum: bool,
212) (error{
213 UnknownContentSizeUnsupported,
214 ContentTooLarge,
215 ContentSizeTooLarge,
216 WindowSizeUnknown,
217 DictionaryIdFlagUnsupported,
218} || FrameError)!ReadWriteCount {
219 assert(std.mem.readInt(u32, src[0..4], .little) == frame.Zstandard.magic_number);
220 var consumed_count: usize = 4;
221
222 var frame_context = context: {
223 var fbs: std.io.FixedBufferStream = .{ .buffer = src[consumed_count..] };
224 const source = fbs.reader();
225 const frame_header = try decodeZstandardHeader(source);
226 consumed_count += fbs.pos;
227 break :context FrameContext.init(
228 frame_header,
229 std.math.maxInt(usize),
230 verify_checksum,
231 ) catch |err| switch (err) {
232 error.WindowTooLarge => unreachable,
233 inline else => |e| return e,
234 };
235 };
236 const counts = try decodeZStandardFrameBlocks(
237 dest,
238 src[consumed_count..],
239 &frame_context,
240 );
241 return ReadWriteCount{
242 .read_count = counts.read_count + consumed_count,
243 .write_count = counts.write_count,
244 };
245}
246
247pub fn decodeZStandardFrameBlocks(
248 dest: []u8,
249 src: []const u8,
250 frame_context: *FrameContext,
251) (error{ ContentTooLarge, UnknownContentSizeUnsupported } || FrameError)!ReadWriteCount {
252 const content_size = frame_context.content_size orelse
253 return error.UnknownContentSizeUnsupported;
254 if (dest.len < content_size) return error.ContentTooLarge;
255
256 var consumed_count: usize = 0;
257 const written_count = decodeFrameBlocksInner(
258 dest[0..content_size],
259 src[consumed_count..],
260 &consumed_count,
261 if (frame_context.hasher_opt) |*hasher| hasher else null,
262 frame_context.block_size_max,
263 ) catch |err| switch (err) {
264 error.DestTooSmall => return error.BadContentSize,
265 inline else => |e| return e,
266 };
267
268 if (written_count != content_size) return error.BadContentSize;
269 if (frame_context.has_checksum) {
270 if (src.len < consumed_count + 4) return error.EndOfStream;
271 const checksum = std.mem.readInt(u32, src[consumed_count..][0..4], .little);
272 consumed_count += 4;
273 if (frame_context.hasher_opt) |*hasher| {
274 if (checksum != computeChecksum(hasher)) return error.ChecksumFailure;
275 }
276 }
277 return ReadWriteCount{ .read_count = consumed_count, .write_count = written_count };
278}
279
280pub const FrameContext = struct {
281 hasher_opt: ?std.hash.XxHash64,
282 window_size: usize,
283 has_checksum: bool,
284 block_size_max: usize,
285 content_size: ?usize,
286
287 const Error = error{
288 DictionaryIdFlagUnsupported,
289 WindowSizeUnknown,
290 WindowTooLarge,
291 ContentSizeTooLarge,
292 };
293 /// Validates `frame_header` and returns the associated `FrameContext`.
294 ///
295 /// Errors returned:
296 /// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
297 /// - `error.WindowSizeUnknown` if the frame does not have a valid window
298 /// size
299 /// - `error.WindowTooLarge` if the window size is larger than
300 /// `window_size_max` or `std.math.intMax(usize)`
301 /// - `error.ContentSizeTooLarge` if the frame header indicates a content
302 /// size larger than `std.math.maxInt(usize)`
303 pub fn init(
304 frame_header: ZstandardHeader,
305 window_size_max: usize,
306 verify_checksum: bool,
307 ) Error!FrameContext {
308 if (frame_header.descriptor.dictionary_id_flag != 0)
309 return error.DictionaryIdFlagUnsupported;
310
311 const window_size_raw = frameWindowSize(frame_header) orelse return error.WindowSizeUnknown;
312 const window_size = if (window_size_raw > window_size_max)
313 return error.WindowTooLarge
314 else
315 std.math.cast(usize, window_size_raw) orelse return error.WindowTooLarge;
316
317 const should_compute_checksum =
318 frame_header.descriptor.content_checksum_flag and verify_checksum;
319
320 const content_size = if (frame_header.content_size) |size|
321 std.math.cast(usize, size) orelse return error.ContentSizeTooLarge
322 else
323 null;
324
325 return .{
326 .hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,
327 .window_size = window_size,
328 .has_checksum = frame_header.descriptor.content_checksum_flag,
329 .block_size_max = @min(types.block_size_max, window_size),
330 .content_size = content_size,
331 };
332 }
333};
334
335fn decodeFrameBlocksInner(
336 dest: []u8,
337 src: []const u8,
338 consumed_count: *usize,
339 hash: ?*std.hash.XxHash64,
340 block_size_max: usize,
341) (error{ EndOfStream, DestTooSmall } || block.Error)!usize {
342 // These tables take 7680 bytes
343 var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined;
344 var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined;
345 var offset_fse_data: [types.compressed_block.table_size_max.offset]Table.Fse = undefined;
346
347 var block_header = try block.decodeBlockHeaderSlice(src);
348 var bytes_read: usize = 3;
349 defer consumed_count.* += bytes_read;
350 var decode_state = block.DecodeState.init(&literal_fse_data, &match_fse_data, &offset_fse_data);
351 var count: usize = 0;
352 while (true) : ({
353 block_header = try block.decodeBlockHeaderSlice(src[bytes_read..]);
354 bytes_read += 3;
355 }) {
356 const written_size = try block.decodeBlock(
357 dest,
358 src[bytes_read..],
359 block_header,
360 &decode_state,
361 &bytes_read,
362 block_size_max,
363 count,
364 );
365 if (hash) |hash_state| hash_state.update(dest[count .. count + written_size]);
366 count += written_size;
367 if (block_header.last_block) break;
368 }
369 return count;
370}
371
372/// Decode the header of a skippable frame. The first four bytes of `src` must
373/// be a valid magic number for a skippable frame.
374pub fn decodeSkippableHeader(src: *const [8]u8) SkippableHeader {
375 const magic = std.mem.readInt(u32, src[0..4], .little);
376 assert(isSkippableMagic(magic));
377 const frame_size = std.mem.readInt(u32, src[4..8], .little);
378 return .{
379 .magic_number = magic,
380 .frame_size = frame_size,
381 };
382}
383
384/// Returns the window size required to decompress a frame, or `null` if it
385/// cannot be determined (which indicates a malformed frame header).
386pub fn frameWindowSize(header: ZstandardHeader) ?u64 {
387 if (header.window_descriptor) |descriptor| {
388 const exponent = (descriptor & 0b11111000) >> 3;
389 const mantissa = descriptor & 0b00000111;
390 const window_log = 10 + exponent;
391 const window_base = @as(u64, 1) << @as(u6, @intCast(window_log));
392 const window_add = (window_base / 8) * mantissa;
393 return window_base + window_add;
394 } else return header.content_size;
395}
396
397/// Decode the header of a Zstandard frame.
398///
399/// Errors returned:
400/// - `error.ReservedBitSet` if any of the reserved bits of the header are set
401/// - `error.EndOfStream` if `source` does not contain a complete header
402pub fn decodeZstandardHeader(
403 source: anytype,
404) (@TypeOf(source).Error || error{ EndOfStream, ReservedBitSet })!ZstandardHeader {
405 const descriptor = @as(ZstandardHeader.Descriptor, @bitCast(try source.readByte()));
406
407 if (descriptor.reserved) return error.ReservedBitSet;
408
409 var window_descriptor: ?u8 = null;
410 if (!descriptor.single_segment_flag) {
411 window_descriptor = try source.readByte();
412 }
413
414 var dictionary_id: ?u32 = null;
415 if (descriptor.dictionary_id_flag > 0) {
416 // if flag is 3 then field_size = 4, else field_size = flag
417 const field_size = (@as(u4, 1) << descriptor.dictionary_id_flag) >> 1;
418 dictionary_id = try source.readVarInt(u32, .little, field_size);
419 }
420
421 var content_size: ?u64 = null;
422 if (descriptor.single_segment_flag or descriptor.content_size_flag > 0) {
423 const field_size = @as(u4, 1) << descriptor.content_size_flag;
424 content_size = try source.readVarInt(u64, .little, field_size);
425 if (field_size == 2) content_size.? += 256;
426 }
427
428 const header = ZstandardHeader{
429 .descriptor = descriptor,
430 .window_descriptor = window_descriptor,
431 .dictionary_id = dictionary_id,
432 .content_size = content_size,
433 };
434 return header;
435}
436
437test {
438 _ = types;
439 _ = block;
440 _ = readers;
441}
lib/std/compress/zstandard/readers.zig deleted-62
......@@ -1,62 +0,0 @@
1const std = @import("std");
2
3pub const ReversedByteReader = struct {
4 remaining_bytes: usize,
5 bytes: []const u8,
6
7 const Reader = std.io.Reader(*ReversedByteReader, error{}, readFn);
8
9 pub fn init(bytes: []const u8) ReversedByteReader {
10 return .{
11 .bytes = bytes,
12 .remaining_bytes = bytes.len,
13 };
14 }
15
16 pub fn reader(self: *ReversedByteReader) Reader {
17 return .{ .context = self };
18 }
19
20 fn readFn(ctx: *ReversedByteReader, buffer: []u8) !usize {
21 if (ctx.remaining_bytes == 0) return 0;
22 const byte_index = ctx.remaining_bytes - 1;
23 buffer[0] = ctx.bytes[byte_index];
24 // buffer[0] = @bitReverse(ctx.bytes[byte_index]);
25 ctx.remaining_bytes = byte_index;
26 return 1;
27 }
28};
29
30/// A bit reader for reading the reversed bit streams used to encode
31/// FSE compressed data.
32pub const ReverseBitReader = struct {
33 byte_reader: ReversedByteReader,
34 bit_reader: std.io.BitReader(.big),
35
36 pub fn init(self: *ReverseBitReader, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
37 self.byte_reader = ReversedByteReader.init(bytes);
38 self.bit_reader = .init(self.byte_reader.reader());
39 if (bytes.len == 0) return;
40 var i: usize = 0;
41 while (i < 8 and 0 == self.readBitsNoEof(u1, 1) catch unreachable) : (i += 1) {}
42 if (i == 8) return error.BitStreamHasNoStartBit;
43 }
44
45 pub fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: u16) error{EndOfStream}!U {
46 return self.bit_reader.readBitsNoEof(U, num_bits);
47 }
48
49 pub fn readBits(self: *@This(), comptime U: type, num_bits: u16, out_bits: *u16) error{}!U {
50 return try self.bit_reader.readBits(U, num_bits, out_bits);
51 }
52
53 pub fn alignToByte(self: *@This()) void {
54 self.bit_reader.alignToByte();
55 }
56
57 pub fn isEmpty(self: ReverseBitReader) bool {
58 return self.byte_reader.remaining_bytes == 0 and self.bit_reader.count == 0;
59 }
60};
61
62pub const BitReader = std.io.BitReader(.little);
lib/std/compress/zstandard/types.zig deleted-403
......@@ -1,403 +0,0 @@
1pub const block_size_max = 1 << 17;
2
3pub const frame = struct {
4 pub const Kind = enum { zstandard, skippable };
5
6 pub const Zstandard = struct {
7 pub const magic_number = 0xFD2FB528;
8
9 header: Header,
10 data_blocks: []Block,
11 checksum: ?u32,
12
13 pub const Header = struct {
14 descriptor: Descriptor,
15 window_descriptor: ?u8,
16 dictionary_id: ?u32,
17 content_size: ?u64,
18
19 pub const Descriptor = packed struct {
20 dictionary_id_flag: u2,
21 content_checksum_flag: bool,
22 reserved: bool,
23 unused: bool,
24 single_segment_flag: bool,
25 content_size_flag: u2,
26 };
27 };
28
29 pub const Block = struct {
30 pub const Header = struct {
31 last_block: bool,
32 block_type: Block.Type,
33 block_size: u21,
34 };
35
36 pub const Type = enum(u2) {
37 raw,
38 rle,
39 compressed,
40 reserved,
41 };
42 };
43 };
44
45 pub const Skippable = struct {
46 pub const magic_number_min = 0x184D2A50;
47 pub const magic_number_max = 0x184D2A5F;
48
49 pub const Header = struct {
50 magic_number: u32,
51 frame_size: u32,
52 };
53 };
54};
55
56pub const compressed_block = struct {
57 pub const LiteralsSection = struct {
58 header: Header,
59 huffman_tree: ?HuffmanTree,
60 streams: Streams,
61
62 pub const Streams = union(enum) {
63 one: []const u8,
64 four: [4][]const u8,
65 };
66
67 pub const Header = struct {
68 block_type: BlockType,
69 size_format: u2,
70 regenerated_size: u20,
71 compressed_size: ?u18,
72 };
73
74 pub const BlockType = enum(u2) {
75 raw,
76 rle,
77 compressed,
78 treeless,
79 };
80
81 pub const HuffmanTree = struct {
82 max_bit_count: u4,
83 symbol_count_minus_one: u8,
84 nodes: [256]PrefixedSymbol,
85
86 pub const PrefixedSymbol = struct {
87 symbol: u8,
88 prefix: u16,
89 weight: u4,
90 };
91
92 pub const Result = union(enum) {
93 symbol: u8,
94 index: usize,
95 };
96
97 pub fn query(self: HuffmanTree, index: usize, prefix: u16) error{NotFound}!Result {
98 var node = self.nodes[index];
99 const weight = node.weight;
100 var i: usize = index;
101 while (node.weight == weight) {
102 if (node.prefix == prefix) return Result{ .symbol = node.symbol };
103 if (i == 0) return error.NotFound;
104 i -= 1;
105 node = self.nodes[i];
106 }
107 return Result{ .index = i };
108 }
109
110 pub fn weightToBitCount(weight: u4, max_bit_count: u4) u4 {
111 return if (weight == 0) 0 else ((max_bit_count + 1) - weight);
112 }
113 };
114
115 pub const StreamCount = enum { one, four };
116 pub fn streamCount(size_format: u2, block_type: BlockType) StreamCount {
117 return switch (block_type) {
118 .raw, .rle => .one,
119 .compressed, .treeless => if (size_format == 0) .one else .four,
120 };
121 }
122 };
123
124 pub const SequencesSection = struct {
125 header: SequencesSection.Header,
126 literals_length_table: Table,
127 offset_table: Table,
128 match_length_table: Table,
129
130 pub const Header = struct {
131 sequence_count: u24,
132 match_lengths: Mode,
133 offsets: Mode,
134 literal_lengths: Mode,
135
136 pub const Mode = enum(u2) {
137 predefined,
138 rle,
139 fse,
140 repeat,
141 };
142 };
143 };
144
145 pub const Table = union(enum) {
146 fse: []const Fse,
147 rle: u8,
148
149 pub const Fse = struct {
150 symbol: u8,
151 baseline: u16,
152 bits: u8,
153 };
154 };
155
156 pub const literals_length_code_table = [36]struct { u32, u5 }{
157 .{ 0, 0 }, .{ 1, 0 }, .{ 2, 0 }, .{ 3, 0 },
158 .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 },
159 .{ 8, 0 }, .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 },
160 .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 }, .{ 15, 0 },
161 .{ 16, 1 }, .{ 18, 1 }, .{ 20, 1 }, .{ 22, 1 },
162 .{ 24, 2 }, .{ 28, 2 }, .{ 32, 3 }, .{ 40, 3 },
163 .{ 48, 4 }, .{ 64, 6 }, .{ 128, 7 }, .{ 256, 8 },
164 .{ 512, 9 }, .{ 1024, 10 }, .{ 2048, 11 }, .{ 4096, 12 },
165 .{ 8192, 13 }, .{ 16384, 14 }, .{ 32768, 15 }, .{ 65536, 16 },
166 };
167
168 pub const match_length_code_table = [53]struct { u32, u5 }{
169 .{ 3, 0 }, .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 }, .{ 8, 0 },
170 .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 }, .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 },
171 .{ 15, 0 }, .{ 16, 0 }, .{ 17, 0 }, .{ 18, 0 }, .{ 19, 0 }, .{ 20, 0 },
172 .{ 21, 0 }, .{ 22, 0 }, .{ 23, 0 }, .{ 24, 0 }, .{ 25, 0 }, .{ 26, 0 },
173 .{ 27, 0 }, .{ 28, 0 }, .{ 29, 0 }, .{ 30, 0 }, .{ 31, 0 }, .{ 32, 0 },
174 .{ 33, 0 }, .{ 34, 0 }, .{ 35, 1 }, .{ 37, 1 }, .{ 39, 1 }, .{ 41, 1 },
175 .{ 43, 2 }, .{ 47, 2 }, .{ 51, 3 }, .{ 59, 3 }, .{ 67, 4 }, .{ 83, 4 },
176 .{ 99, 5 }, .{ 131, 7 }, .{ 259, 8 }, .{ 515, 9 }, .{ 1027, 10 }, .{ 2051, 11 },
177 .{ 4099, 12 }, .{ 8195, 13 }, .{ 16387, 14 }, .{ 32771, 15 }, .{ 65539, 16 },
178 };
179
180 pub const literals_length_default_distribution = [36]i16{
181 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
182 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
183 -1, -1, -1, -1,
184 };
185
186 pub const match_lengths_default_distribution = [53]i16{
187 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
188 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
189 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
190 -1, -1, -1, -1, -1,
191 };
192
193 pub const offset_codes_default_distribution = [29]i16{
194 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
195 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1,
196 };
197
198 pub const predefined_literal_fse_table = Table{
199 .fse = &[64]Table.Fse{
200 .{ .symbol = 0, .bits = 4, .baseline = 0 },
201 .{ .symbol = 0, .bits = 4, .baseline = 16 },
202 .{ .symbol = 1, .bits = 5, .baseline = 32 },
203 .{ .symbol = 3, .bits = 5, .baseline = 0 },
204 .{ .symbol = 4, .bits = 5, .baseline = 0 },
205 .{ .symbol = 6, .bits = 5, .baseline = 0 },
206 .{ .symbol = 7, .bits = 5, .baseline = 0 },
207 .{ .symbol = 9, .bits = 5, .baseline = 0 },
208 .{ .symbol = 10, .bits = 5, .baseline = 0 },
209 .{ .symbol = 12, .bits = 5, .baseline = 0 },
210 .{ .symbol = 14, .bits = 6, .baseline = 0 },
211 .{ .symbol = 16, .bits = 5, .baseline = 0 },
212 .{ .symbol = 18, .bits = 5, .baseline = 0 },
213 .{ .symbol = 19, .bits = 5, .baseline = 0 },
214 .{ .symbol = 21, .bits = 5, .baseline = 0 },
215 .{ .symbol = 22, .bits = 5, .baseline = 0 },
216 .{ .symbol = 24, .bits = 5, .baseline = 0 },
217 .{ .symbol = 25, .bits = 5, .baseline = 32 },
218 .{ .symbol = 26, .bits = 5, .baseline = 0 },
219 .{ .symbol = 27, .bits = 6, .baseline = 0 },
220 .{ .symbol = 29, .bits = 6, .baseline = 0 },
221 .{ .symbol = 31, .bits = 6, .baseline = 0 },
222 .{ .symbol = 0, .bits = 4, .baseline = 32 },
223 .{ .symbol = 1, .bits = 4, .baseline = 0 },
224 .{ .symbol = 2, .bits = 5, .baseline = 0 },
225 .{ .symbol = 4, .bits = 5, .baseline = 32 },
226 .{ .symbol = 5, .bits = 5, .baseline = 0 },
227 .{ .symbol = 7, .bits = 5, .baseline = 32 },
228 .{ .symbol = 8, .bits = 5, .baseline = 0 },
229 .{ .symbol = 10, .bits = 5, .baseline = 32 },
230 .{ .symbol = 11, .bits = 5, .baseline = 0 },
231 .{ .symbol = 13, .bits = 6, .baseline = 0 },
232 .{ .symbol = 16, .bits = 5, .baseline = 32 },
233 .{ .symbol = 17, .bits = 5, .baseline = 0 },
234 .{ .symbol = 19, .bits = 5, .baseline = 32 },
235 .{ .symbol = 20, .bits = 5, .baseline = 0 },
236 .{ .symbol = 22, .bits = 5, .baseline = 32 },
237 .{ .symbol = 23, .bits = 5, .baseline = 0 },
238 .{ .symbol = 25, .bits = 4, .baseline = 0 },
239 .{ .symbol = 25, .bits = 4, .baseline = 16 },
240 .{ .symbol = 26, .bits = 5, .baseline = 32 },
241 .{ .symbol = 28, .bits = 6, .baseline = 0 },
242 .{ .symbol = 30, .bits = 6, .baseline = 0 },
243 .{ .symbol = 0, .bits = 4, .baseline = 48 },
244 .{ .symbol = 1, .bits = 4, .baseline = 16 },
245 .{ .symbol = 2, .bits = 5, .baseline = 32 },
246 .{ .symbol = 3, .bits = 5, .baseline = 32 },
247 .{ .symbol = 5, .bits = 5, .baseline = 32 },
248 .{ .symbol = 6, .bits = 5, .baseline = 32 },
249 .{ .symbol = 8, .bits = 5, .baseline = 32 },
250 .{ .symbol = 9, .bits = 5, .baseline = 32 },
251 .{ .symbol = 11, .bits = 5, .baseline = 32 },
252 .{ .symbol = 12, .bits = 5, .baseline = 32 },
253 .{ .symbol = 15, .bits = 6, .baseline = 0 },
254 .{ .symbol = 17, .bits = 5, .baseline = 32 },
255 .{ .symbol = 18, .bits = 5, .baseline = 32 },
256 .{ .symbol = 20, .bits = 5, .baseline = 32 },
257 .{ .symbol = 21, .bits = 5, .baseline = 32 },
258 .{ .symbol = 23, .bits = 5, .baseline = 32 },
259 .{ .symbol = 24, .bits = 5, .baseline = 32 },
260 .{ .symbol = 35, .bits = 6, .baseline = 0 },
261 .{ .symbol = 34, .bits = 6, .baseline = 0 },
262 .{ .symbol = 33, .bits = 6, .baseline = 0 },
263 .{ .symbol = 32, .bits = 6, .baseline = 0 },
264 },
265 };
266
267 pub const predefined_match_fse_table = Table{
268 .fse = &[64]Table.Fse{
269 .{ .symbol = 0, .bits = 6, .baseline = 0 },
270 .{ .symbol = 1, .bits = 4, .baseline = 0 },
271 .{ .symbol = 2, .bits = 5, .baseline = 32 },
272 .{ .symbol = 3, .bits = 5, .baseline = 0 },
273 .{ .symbol = 5, .bits = 5, .baseline = 0 },
274 .{ .symbol = 6, .bits = 5, .baseline = 0 },
275 .{ .symbol = 8, .bits = 5, .baseline = 0 },
276 .{ .symbol = 10, .bits = 6, .baseline = 0 },
277 .{ .symbol = 13, .bits = 6, .baseline = 0 },
278 .{ .symbol = 16, .bits = 6, .baseline = 0 },
279 .{ .symbol = 19, .bits = 6, .baseline = 0 },
280 .{ .symbol = 22, .bits = 6, .baseline = 0 },
281 .{ .symbol = 25, .bits = 6, .baseline = 0 },
282 .{ .symbol = 28, .bits = 6, .baseline = 0 },
283 .{ .symbol = 31, .bits = 6, .baseline = 0 },
284 .{ .symbol = 33, .bits = 6, .baseline = 0 },
285 .{ .symbol = 35, .bits = 6, .baseline = 0 },
286 .{ .symbol = 37, .bits = 6, .baseline = 0 },
287 .{ .symbol = 39, .bits = 6, .baseline = 0 },
288 .{ .symbol = 41, .bits = 6, .baseline = 0 },
289 .{ .symbol = 43, .bits = 6, .baseline = 0 },
290 .{ .symbol = 45, .bits = 6, .baseline = 0 },
291 .{ .symbol = 1, .bits = 4, .baseline = 16 },
292 .{ .symbol = 2, .bits = 4, .baseline = 0 },
293 .{ .symbol = 3, .bits = 5, .baseline = 32 },
294 .{ .symbol = 4, .bits = 5, .baseline = 0 },
295 .{ .symbol = 6, .bits = 5, .baseline = 32 },
296 .{ .symbol = 7, .bits = 5, .baseline = 0 },
297 .{ .symbol = 9, .bits = 6, .baseline = 0 },
298 .{ .symbol = 12, .bits = 6, .baseline = 0 },
299 .{ .symbol = 15, .bits = 6, .baseline = 0 },
300 .{ .symbol = 18, .bits = 6, .baseline = 0 },
301 .{ .symbol = 21, .bits = 6, .baseline = 0 },
302 .{ .symbol = 24, .bits = 6, .baseline = 0 },
303 .{ .symbol = 27, .bits = 6, .baseline = 0 },
304 .{ .symbol = 30, .bits = 6, .baseline = 0 },
305 .{ .symbol = 32, .bits = 6, .baseline = 0 },
306 .{ .symbol = 34, .bits = 6, .baseline = 0 },
307 .{ .symbol = 36, .bits = 6, .baseline = 0 },
308 .{ .symbol = 38, .bits = 6, .baseline = 0 },
309 .{ .symbol = 40, .bits = 6, .baseline = 0 },
310 .{ .symbol = 42, .bits = 6, .baseline = 0 },
311 .{ .symbol = 44, .bits = 6, .baseline = 0 },
312 .{ .symbol = 1, .bits = 4, .baseline = 32 },
313 .{ .symbol = 1, .bits = 4, .baseline = 48 },
314 .{ .symbol = 2, .bits = 4, .baseline = 16 },
315 .{ .symbol = 4, .bits = 5, .baseline = 32 },
316 .{ .symbol = 5, .bits = 5, .baseline = 32 },
317 .{ .symbol = 7, .bits = 5, .baseline = 32 },
318 .{ .symbol = 8, .bits = 5, .baseline = 32 },
319 .{ .symbol = 11, .bits = 6, .baseline = 0 },
320 .{ .symbol = 14, .bits = 6, .baseline = 0 },
321 .{ .symbol = 17, .bits = 6, .baseline = 0 },
322 .{ .symbol = 20, .bits = 6, .baseline = 0 },
323 .{ .symbol = 23, .bits = 6, .baseline = 0 },
324 .{ .symbol = 26, .bits = 6, .baseline = 0 },
325 .{ .symbol = 29, .bits = 6, .baseline = 0 },
326 .{ .symbol = 52, .bits = 6, .baseline = 0 },
327 .{ .symbol = 51, .bits = 6, .baseline = 0 },
328 .{ .symbol = 50, .bits = 6, .baseline = 0 },
329 .{ .symbol = 49, .bits = 6, .baseline = 0 },
330 .{ .symbol = 48, .bits = 6, .baseline = 0 },
331 .{ .symbol = 47, .bits = 6, .baseline = 0 },
332 .{ .symbol = 46, .bits = 6, .baseline = 0 },
333 },
334 };
335
336 pub const predefined_offset_fse_table = Table{
337 .fse = &[32]Table.Fse{
338 .{ .symbol = 0, .bits = 5, .baseline = 0 },
339 .{ .symbol = 6, .bits = 4, .baseline = 0 },
340 .{ .symbol = 9, .bits = 5, .baseline = 0 },
341 .{ .symbol = 15, .bits = 5, .baseline = 0 },
342 .{ .symbol = 21, .bits = 5, .baseline = 0 },
343 .{ .symbol = 3, .bits = 5, .baseline = 0 },
344 .{ .symbol = 7, .bits = 4, .baseline = 0 },
345 .{ .symbol = 12, .bits = 5, .baseline = 0 },
346 .{ .symbol = 18, .bits = 5, .baseline = 0 },
347 .{ .symbol = 23, .bits = 5, .baseline = 0 },
348 .{ .symbol = 5, .bits = 5, .baseline = 0 },
349 .{ .symbol = 8, .bits = 4, .baseline = 0 },
350 .{ .symbol = 14, .bits = 5, .baseline = 0 },
351 .{ .symbol = 20, .bits = 5, .baseline = 0 },
352 .{ .symbol = 2, .bits = 5, .baseline = 0 },
353 .{ .symbol = 7, .bits = 4, .baseline = 16 },
354 .{ .symbol = 11, .bits = 5, .baseline = 0 },
355 .{ .symbol = 17, .bits = 5, .baseline = 0 },
356 .{ .symbol = 22, .bits = 5, .baseline = 0 },
357 .{ .symbol = 4, .bits = 5, .baseline = 0 },
358 .{ .symbol = 8, .bits = 4, .baseline = 16 },
359 .{ .symbol = 13, .bits = 5, .baseline = 0 },
360 .{ .symbol = 19, .bits = 5, .baseline = 0 },
361 .{ .symbol = 1, .bits = 5, .baseline = 0 },
362 .{ .symbol = 6, .bits = 4, .baseline = 16 },
363 .{ .symbol = 10, .bits = 5, .baseline = 0 },
364 .{ .symbol = 16, .bits = 5, .baseline = 0 },
365 .{ .symbol = 28, .bits = 5, .baseline = 0 },
366 .{ .symbol = 27, .bits = 5, .baseline = 0 },
367 .{ .symbol = 26, .bits = 5, .baseline = 0 },
368 .{ .symbol = 25, .bits = 5, .baseline = 0 },
369 .{ .symbol = 24, .bits = 5, .baseline = 0 },
370 },
371 };
372 pub const start_repeated_offset_1 = 1;
373 pub const start_repeated_offset_2 = 4;
374 pub const start_repeated_offset_3 = 8;
375
376 pub const table_accuracy_log_max = struct {
377 pub const literal = 9;
378 pub const match = 9;
379 pub const offset = 8;
380 };
381
382 pub const table_symbol_count_max = struct {
383 pub const literal = 36;
384 pub const match = 53;
385 pub const offset = 32;
386 };
387
388 pub const default_accuracy_log = struct {
389 pub const literal = 6;
390 pub const match = 6;
391 pub const offset = 5;
392 };
393 pub const table_size_max = struct {
394 pub const literal = 1 << table_accuracy_log_max.literal;
395 pub const match = 1 << table_accuracy_log_max.match;
396 pub const offset = 1 << table_accuracy_log_max.offset;
397 };
398};
399
400test {
401 const testing = @import("std").testing;
402 testing.refAllDeclsRecursive(@This());
403}
lib/std/compress/zstd.zig created+153
......@@ -0,0 +1,153 @@
1const std = @import("../std.zig");
2const assert = std.debug.assert;
3
4/// Recommended amount by the standard. Lower than this may result in inability
5/// to decompress common streams.
6pub const default_window_len = 8 * 1024 * 1024;
7
8pub const Decompress = @import("zstd/Decompress.zig");
9
10pub const block_size_max = 1 << 17;
11
12pub const literals_length_default_distribution = [36]i16{
13 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
14 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
15 -1, -1, -1, -1,
16};
17
18pub const match_lengths_default_distribution = [53]i16{
19 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
20 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
21 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
22 -1, -1, -1, -1, -1,
23};
24
25pub const offset_codes_default_distribution = [29]i16{
26 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
27 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1,
28};
29
30pub const start_repeated_offset_1 = 1;
31pub const start_repeated_offset_2 = 4;
32pub const start_repeated_offset_3 = 8;
33
34pub const literals_length_code_table = [36]struct { u32, u5 }{
35 .{ 0, 0 }, .{ 1, 0 }, .{ 2, 0 }, .{ 3, 0 },
36 .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 },
37 .{ 8, 0 }, .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 },
38 .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 }, .{ 15, 0 },
39 .{ 16, 1 }, .{ 18, 1 }, .{ 20, 1 }, .{ 22, 1 },
40 .{ 24, 2 }, .{ 28, 2 }, .{ 32, 3 }, .{ 40, 3 },
41 .{ 48, 4 }, .{ 64, 6 }, .{ 128, 7 }, .{ 256, 8 },
42 .{ 512, 9 }, .{ 1024, 10 }, .{ 2048, 11 }, .{ 4096, 12 },
43 .{ 8192, 13 }, .{ 16384, 14 }, .{ 32768, 15 }, .{ 65536, 16 },
44};
45
46pub const match_length_code_table = [53]struct { u32, u5 }{
47 .{ 3, 0 }, .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 }, .{ 8, 0 },
48 .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 }, .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 },
49 .{ 15, 0 }, .{ 16, 0 }, .{ 17, 0 }, .{ 18, 0 }, .{ 19, 0 }, .{ 20, 0 },
50 .{ 21, 0 }, .{ 22, 0 }, .{ 23, 0 }, .{ 24, 0 }, .{ 25, 0 }, .{ 26, 0 },
51 .{ 27, 0 }, .{ 28, 0 }, .{ 29, 0 }, .{ 30, 0 }, .{ 31, 0 }, .{ 32, 0 },
52 .{ 33, 0 }, .{ 34, 0 }, .{ 35, 1 }, .{ 37, 1 }, .{ 39, 1 }, .{ 41, 1 },
53 .{ 43, 2 }, .{ 47, 2 }, .{ 51, 3 }, .{ 59, 3 }, .{ 67, 4 }, .{ 83, 4 },
54 .{ 99, 5 }, .{ 131, 7 }, .{ 259, 8 }, .{ 515, 9 }, .{ 1027, 10 }, .{ 2051, 11 },
55 .{ 4099, 12 }, .{ 8195, 13 }, .{ 16387, 14 }, .{ 32771, 15 }, .{ 65539, 16 },
56};
57
58pub const table_accuracy_log_max = struct {
59 pub const literal = 9;
60 pub const match = 9;
61 pub const offset = 8;
62};
63
64pub const table_symbol_count_max = struct {
65 pub const literal = 36;
66 pub const match = 53;
67 pub const offset = 32;
68};
69
70pub const default_accuracy_log = struct {
71 pub const literal = 6;
72 pub const match = 6;
73 pub const offset = 5;
74};
75pub const table_size_max = struct {
76 pub const literal = 1 << table_accuracy_log_max.literal;
77 pub const match = 1 << table_accuracy_log_max.match;
78 pub const offset = 1 << table_accuracy_log_max.offset;
79};
80
81fn testDecompress(gpa: std.mem.Allocator, compressed: []const u8) ![]u8 {
82 var out: std.ArrayListUnmanaged(u8) = .empty;
83 defer out.deinit(gpa);
84 try out.ensureUnusedCapacity(gpa, default_window_len);
85
86 var in: std.io.BufferedReader = undefined;
87 in.initFixed(@constCast(compressed));
88 var zstd_stream: Decompress = .init(&in, .{});
89 try zstd_stream.reader().readRemainingArrayList(gpa, null, &out, .unlimited);
90
91 return out.toOwnedSlice(gpa);
92}
93
94fn testExpectDecompress(uncompressed: []const u8, compressed: []const u8) !void {
95 const gpa = std.testing.allocator;
96 const result = try testDecompress(gpa, compressed);
97 defer gpa.free(result);
98 try std.testing.expectEqualSlices(u8, uncompressed, result);
99}
100
101fn testExpectDecompressError(err: anyerror, compressed: []const u8) !void {
102 const gpa = std.testing.allocator;
103
104 var out: std.ArrayListUnmanaged(u8) = .empty;
105 defer out.deinit(gpa);
106 try out.ensureUnusedCapacity(gpa, default_window_len);
107
108 var in: std.io.BufferedReader = undefined;
109 in.initFixed(@constCast(compressed));
110 var zstd_stream: Decompress = .init(&in, .{});
111 try std.testing.expectError(error.ReadFailed, zstd_stream.reader().readRemainingArrayList(gpa, null, &out, .unlimited));
112 try std.testing.expectError(err, zstd_stream.err.?);
113
114 return error.TestFailed;
115}
116
117test "decompression" {
118 const uncompressed = @embedFile("testdata/rfc8478.txt");
119 const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
120 const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
121
122 try testExpectDecompress(uncompressed, compressed3);
123 try testExpectDecompress(uncompressed, compressed19);
124}
125
126test "zero sized block" {
127 const input_raw =
128 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
129 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
130 "\x01\x00\x00"; // block header with: last_block set, block_type raw, block_size zero
131
132 const input_rle =
133 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
134 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
135 "\x03\x00\x00" ++ // block header with: last_block set, block_type rle, block_size zero
136 "\xaa"; // block_content
137
138 try testExpectDecompress("", input_raw);
139 try testExpectDecompress("", input_rle);
140}
141
142test "declared raw literals size too large" {
143 const input_raw =
144 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
145 "\x00\x00" ++ // frame header: everything unset, window descriptor zero
146 "\x95\x00\x00" ++ // block header with: last_block set, block_type compressed, block_size 18
147 "\xbc\xf3\xae" ++ // literals section header with: type raw, size_format 3, regenerated_size 716603
148 "\xa5\x9f\xe3"; // some bytes of literal content - the content is shorter than regenerated_size
149
150 // Note that the regenerated_size in the above input is larger than block maximum size, so the
151 // block can't be valid as it is a raw literals block.
152 try testExpectDecompressError(error.MalformedBlock, input_raw);
153}
lib/std/compress/zstd/Decompress.zig created+1641
......@@ -0,0 +1,1641 @@
1const Decompress = @This();
2const std = @import("std");
3const assert = std.debug.assert;
4const Reader = std.io.Reader;
5const BufferedWriter = std.io.BufferedWriter;
6const BufferedReader = std.io.BufferedReader;
7const zstd = @import("../zstd.zig");
8
9input: *BufferedReader,
10state: State,
11verify_checksum: bool,
12err: ?Error = null,
13
14const table_size_max = zstd.compressed_block.table_size_max;
15
16const State = union(enum) {
17 new_frame,
18 in_frame: InFrame,
19 skipping_frame: u32,
20 end,
21
22 const InFrame = struct {
23 frame: Frame,
24 checksum: ?u32,
25 decompressed_size: usize,
26 };
27};
28
29pub const Options = struct {
30 verify_checksum: bool = true,
31};
32
33pub const Error = error{
34 ChecksumFailure,
35 DictionaryIdFlagUnsupported,
36 MalformedBlock,
37 MalformedFrame,
38 EndOfStream,
39};
40
41pub fn init(input: *BufferedReader, options: Options) Decompress {
42 return .{
43 .input = input,
44 .state = .new_frame,
45 .verify_checksum = options.verify_checksum,
46 };
47}
48
49pub fn reader(self: *Decompress) Reader {
50 return .{
51 .context = self,
52 .vtable = &.{
53 .read = read,
54 .readVec = readVec,
55 .discard = discard,
56 },
57 };
58}
59
60fn read(context: ?*anyopaque, bw: *BufferedWriter, limit: Reader.Limit) Reader.RwError!usize {
61 const d: *Decompress = @ptrCast(@alignCast(context));
62 const in = d.input;
63
64 switch (d.state) {
65 .new_frame => {
66 // Allow error.EndOfStream only on the frame magic.
67 const magic = try in.takeEnumNonexhaustive(Frame.Magic, .little);
68 initFrame(d, bw.buffer.len, magic) catch |err| {
69 d.err = err;
70 return error.ReadFailed;
71 };
72 return readInFrame(d, bw, limit, &d.state.in_frame) catch |err| {
73 d.err = err;
74 return error.ReadFailed;
75 };
76 },
77 .in_frame => |*in_frame| {
78 return readInFrame(d, bw, limit, in_frame) catch |err| {
79 d.err = err;
80 return error.ReadFailed;
81 };
82 },
83 .skipping_frame => |*remaining| {
84 const requested = remaining.*;
85 const n = in.discard(.limited(requested)) catch |err| {
86 d.err = err;
87 return error.ReadFailed;
88 };
89 if (requested == n) d.state = .new_frame;
90 remaining.* = requested - n;
91 return 0;
92 },
93 .end => return error.EndOfStream,
94 }
95}
96
97fn initFrame(d: *Decompress, window_size_max: usize, magic: Frame.Magic) !void {
98 const in = d.input;
99 switch (magic.kind() orelse return error.BadMagic) {
100 .zstandard => {
101 const header = try Frame.Zstandard.Header.decode(in);
102 d.state = .{ .in_frame = .{
103 .frame = try Frame.init(header, window_size_max, d.verify_checksum),
104 .checksum = null,
105 .decompressed_size = 0,
106 } };
107 },
108 .skippable => {
109 const frame_size = try in.takeInt(u32, .little);
110 d.state = .{ .skipping_frame = frame_size };
111 },
112 }
113}
114
115fn readInFrame(d: *Decompress, bw: *BufferedWriter, limit: Reader.Limit, state: *State.InFrame) !usize {
116 const in = d.input;
117
118 var literal_fse_buffer: [table_size_max.literal]Table.Fse = undefined;
119 var match_fse_buffer: [table_size_max.match]Table.Fse = undefined;
120 var offset_fse_buffer: [table_size_max.offset]Table.Fse = undefined;
121 var literals_buffer: [zstd.block_size_max]u8 = undefined;
122 var sequence_buffer: [zstd.block_size_max]u8 = undefined;
123
124 var decode: Frame.Zstandard.Decode = .init(&literal_fse_buffer, &match_fse_buffer, &offset_fse_buffer);
125
126 const header_bytes = try in.takeArray(3);
127 const block_header: Frame.Zstandard.Block.Header = @bitCast(header_bytes.*);
128 const block_size = block_header.block_size;
129 if (state.frame.block_size_max < block_size) return error.BlockOversize;
130 if (@intFromEnum(limit) < block_size) return error.OutputBufferUndersize;
131 switch (block_header.block_type) {
132 .raw => {
133 try in.readAll(bw, .limited(block_size));
134 return block_size;
135 },
136 .rle => {
137 const byte = try in.takeByte();
138 try bw.splatByteAll(byte, block_size);
139 return block_size;
140 },
141 .compressed => {},
142 .reserved => return error.ReservedBlock,
143 }
144
145 var remaining: Reader.Limit = .limited(block_size);
146 const literals = try LiteralsSection.decode(in, &remaining, &literals_buffer);
147 const sequences_header = try SequencesSection.Header.decode(in, &remaining);
148
149 try decode.prepare(in, &remaining, literals, sequences_header);
150
151 var bytes_written: usize = 0;
152 {
153 if (sequence_buffer.len < @intFromEnum(remaining))
154 return error.SequenceBufferTooSmall;
155 const seq_len = try in.readSlice(remaining.slice(&sequence_buffer));
156 var bit_stream = try ReverseBitReader.init(sequence_buffer[0..seq_len]);
157
158 if (sequences_header.sequence_count > 0) {
159 try decode.readInitialFseState(&bit_stream);
160
161 var sequence_size_limit = state.frame.block_size_max;
162 for (0..sequences_header.sequence_count) |i| {
163 const decompressed_size = try decode.decodeSequence(
164 bw,
165 &bit_stream,
166 sequence_size_limit,
167 i == sequences_header.sequence_count - 1,
168 );
169 sequence_size_limit -= decompressed_size;
170 bytes_written += decompressed_size;
171 }
172 }
173
174 if (!bit_stream.isEmpty()) {
175 return error.MalformedCompressedBlock;
176 }
177 }
178
179 if (decode.literal_written_count < literals.header.regenerated_size) {
180 const len = literals.header.regenerated_size - decode.literal_written_count;
181 try decode.decodeLiterals(bw, len);
182 bytes_written += len;
183 }
184
185 switch (decode.literal_header.block_type) {
186 .treeless, .compressed => {
187 if (!decode.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
188 },
189 .raw, .rle => {},
190 }
191
192 if (bytes_written > state.frame.block_size_max) return error.BlockOversize;
193 if (remaining.nonzero()) return error.MalformedCompressedBlock;
194
195 state.decompressed_size += bytes_written;
196
197 if (state.frame.content_size) |size| {
198 if (state.decompressed_size > size) return error.MalformedFrame;
199 }
200
201 if (state.frame.hasher_opt) |*hasher| {
202 if (bytes_written > 0) {
203 _ = hasher;
204 @panic("TODO all those bytes written needed to go through the hasher too");
205 }
206 }
207
208 if (block_header.last_block) {
209 if (state.frame.has_checksum) {
210 const expected_checksum = try in.readInt(u32, .little);
211 if (state.frame.hasher_opt) |*hasher| {
212 const actual_checksum: u32 = @truncate(hasher.final());
213 if (expected_checksum != actual_checksum) return error.ChecksumFailure;
214 }
215 }
216 if (d.frame.content_size) |content_size| {
217 if (content_size != d.current_frame_decompressed_size) {
218 return error.MalformedFrame;
219 }
220 }
221 d.state = .new_frame;
222 }
223
224 return bytes_written;
225}
226
227fn discard(context: ?*anyopaque, limit: Reader.Limit) Reader.Error!usize {
228 var trash_buffer: [64]u8 = undefined;
229 var null_writer: std.io.Writer.Null = undefined;
230 var bw = null_writer.writer().buffered(&trash_buffer);
231 return @errorCast(read(context, &bw, limit));
232}
233
234fn readVec(context: ?*anyopaque, data: []const []u8) Reader.Error!usize {
235 var bw: BufferedWriter = undefined;
236 bw.initFixed(data[0]);
237 return @errorCast(read(context, &bw, .limited(data[0].len)));
238}
239
240pub const Frame = struct {
241 hasher_opt: ?std.hash.XxHash64,
242 window_size: usize,
243 has_checksum: bool,
244 block_size_max: usize,
245 content_size: ?usize,
246
247 pub const Magic = enum(u32) {
248 zstandard = 0xFD2FB528,
249 _,
250
251 pub fn kind(m: Magic) ?Kind {
252 return switch (m) {
253 .zstandard => .zstandard,
254 Skippable.magic_min...Skippable.magic_max => .skippable,
255 else => null,
256 };
257 }
258
259 pub fn isSkippable(m: Magic) bool {
260 return switch (m) {
261 Skippable.magic_min...Skippable.magic_max => true,
262 else => false,
263 };
264 }
265 };
266
267 pub const Kind = enum { zstandard, skippable };
268
269 pub const Zstandard = struct {
270 pub const magic: Magic = .zstandard;
271
272 header: Header,
273 data_blocks: []Block,
274 checksum: ?u32,
275
276 pub const Header = struct {
277 descriptor: Descriptor,
278 window_descriptor: ?u8,
279 dictionary_id: ?u32,
280 content_size: ?u64,
281
282 pub const Descriptor = packed struct {
283 dictionary_id_flag: u2,
284 content_checksum_flag: bool,
285 reserved: bool,
286 unused: bool,
287 single_segment_flag: bool,
288 content_size_flag: u2,
289 };
290
291 pub const DecodeError = Reader.Error || error{ReservedBitSet};
292
293 pub fn decode(in: *BufferedReader) DecodeError!Header {
294 const descriptor: Descriptor = @bitCast(try in.takeByte());
295
296 if (descriptor.reserved) return error.ReservedBitSet;
297
298 const window_descriptor: ?u8 = if (descriptor.single_segment_flag) null else try in.takeByte();
299
300 const dictionary_id: ?u32 = if (descriptor.dictionary_id_flag > 0) d: {
301 // if flag is 3 then field_size = 4, else field_size = flag
302 const field_size = (@as(u4, 1) << descriptor.dictionary_id_flag) >> 1;
303 break :d try in.takeVarInt(u32, .little, field_size);
304 } else null;
305
306 const content_size: ?u64 = if (descriptor.single_segment_flag or descriptor.content_size_flag > 0) c: {
307 const field_size = @as(u4, 1) << descriptor.content_size_flag;
308 const content_size = try in.takeVarInt(u64, .little, field_size);
309 break :c if (field_size == 2) content_size + 256 else content_size;
310 } else null;
311
312 return .{
313 .descriptor = descriptor,
314 .window_descriptor = window_descriptor,
315 .dictionary_id = dictionary_id,
316 .content_size = content_size,
317 };
318 }
319
320 /// Returns the window size required to decompress a frame, or `null` if it
321 /// cannot be determined (which indicates a malformed frame header).
322 pub fn windowSize(header: Header) ?u64 {
323 if (header.window_descriptor) |descriptor| {
324 const exponent = (descriptor & 0b11111000) >> 3;
325 const mantissa = descriptor & 0b00000111;
326 const window_log = 10 + exponent;
327 const window_base = @as(u64, 1) << @as(u6, @intCast(window_log));
328 const window_add = (window_base / 8) * mantissa;
329 return window_base + window_add;
330 } else return header.content_size;
331 }
332 };
333
334 pub const Block = struct {
335 pub const Header = packed struct(u24) {
336 last: bool,
337 type: Type,
338 size: u21,
339 };
340
341 pub const Type = enum(u2) {
342 raw,
343 rle,
344 compressed,
345 reserved,
346 };
347 };
348
349 pub const Decode = struct {
350 repeat_offsets: [3]u32,
351
352 offset: StateData(8),
353 match: StateData(9),
354 literal: StateData(9),
355
356 offset_fse_buffer: []Table.Fse,
357 match_fse_buffer: []Table.Fse,
358 literal_fse_buffer: []Table.Fse,
359
360 fse_tables_undefined: bool,
361
362 literal_stream_reader: ReverseBitReader,
363 literal_stream_index: usize,
364 literal_streams: LiteralsSection.Streams,
365 literal_header: LiteralsSection.Header,
366 huffman_tree: ?LiteralsSection.HuffmanTree,
367
368 literal_written_count: usize,
369
370 fn StateData(comptime max_accuracy_log: comptime_int) type {
371 return struct {
372 state: @This().State,
373 table: Table,
374 accuracy_log: u8,
375
376 const State = std.meta.Int(.unsigned, max_accuracy_log);
377 };
378 }
379
380 pub fn init(
381 literal_fse_buffer: []Table.Fse,
382 match_fse_buffer: []Table.Fse,
383 offset_fse_buffer: []Table.Fse,
384 ) Decode {
385 return .{
386 .repeat_offsets = .{
387 zstd.compressed_block.start_repeated_offset_1,
388 zstd.compressed_block.start_repeated_offset_2,
389 zstd.compressed_block.start_repeated_offset_3,
390 },
391
392 .offset = undefined,
393 .match = undefined,
394 .literal = undefined,
395
396 .literal_fse_buffer = literal_fse_buffer,
397 .match_fse_buffer = match_fse_buffer,
398 .offset_fse_buffer = offset_fse_buffer,
399
400 .fse_tables_undefined = true,
401
402 .literal_written_count = 0,
403 .literal_header = undefined,
404 .literal_streams = undefined,
405 .literal_stream_reader = undefined,
406 .literal_stream_index = undefined,
407 .huffman_tree = null,
408 };
409 }
410
411 pub const PrepareError = error{
412 /// the (reversed) literal bitstream's first byte does not have any bits set
413 BitStreamHasNoStartBit,
414 /// `literals` is a treeless literals section and the decode state does not
415 /// have a Huffman tree from a previous block
416 TreelessLiteralsFirst,
417 /// on the first call if one of the sequence FSE tables is set to repeat mode
418 RepeatModeFirst,
419 /// an FSE table has an invalid accuracy
420 MalformedAccuracyLog,
421 /// failed decoding an FSE table
422 MalformedFseTable,
423 /// input stream ends before all FSE tables are read
424 EndOfStream,
425 };
426
427 /// Prepare the decoder to decode a compressed block. Loads the literals
428 /// stream and Huffman tree from `literals` and reads the FSE tables from
429 /// `source`.
430 pub fn prepare(
431 self: *Decode,
432 in: *BufferedReader,
433 literals: LiteralsSection,
434 sequences_header: SequencesSection.Header,
435 ) PrepareError!void {
436 self.literal_written_count = 0;
437 self.literal_header = literals.header;
438 self.literal_streams = literals.streams;
439
440 if (literals.huffman_tree) |tree| {
441 self.huffman_tree = tree;
442 } else if (literals.header.block_type == .treeless and self.huffman_tree == null) {
443 return error.TreelessLiteralsFirst;
444 }
445
446 switch (literals.header.block_type) {
447 .raw, .rle => {},
448 .compressed, .treeless => {
449 self.literal_stream_index = 0;
450 switch (literals.streams) {
451 .one => |slice| try self.initLiteralStream(slice),
452 .four => |streams| try self.initLiteralStream(streams[0]),
453 }
454 },
455 }
456
457 if (sequences_header.sequence_count > 0) {
458 try self.updateFseTable(in, .literal, sequences_header.literal_lengths);
459 try self.updateFseTable(in, .offset, sequences_header.offsets);
460 try self.updateFseTable(in, .match, sequences_header.match_lengths);
461 self.fse_tables_undefined = false;
462 }
463 }
464
465 /// Read initial FSE states for sequence decoding.
466 ///
467 /// Errors returned:
468 /// - `error.EndOfStream` if `bit_reader` does not contain enough bits.
469 pub fn readInitialFseState(self: *Decode, bit_reader: *ReverseBitReader) error{EndOfStream}!void {
470 self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
471 self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
472 self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log);
473 }
474
475 fn updateRepeatOffset(self: *Decode, offset: u32) void {
476 self.repeat_offsets[2] = self.repeat_offsets[1];
477 self.repeat_offsets[1] = self.repeat_offsets[0];
478 self.repeat_offsets[0] = offset;
479 }
480
481 fn useRepeatOffset(self: *Decode, index: usize) u32 {
482 if (index == 1)
483 std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[1])
484 else if (index == 2) {
485 std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[2]);
486 std.mem.swap(u32, &self.repeat_offsets[1], &self.repeat_offsets[2]);
487 }
488 return self.repeat_offsets[0];
489 }
490
491 const DataType = enum { offset, match, literal };
492
493 fn updateState(
494 self: *Decode,
495 comptime choice: DataType,
496 bit_reader: *ReverseBitReader,
497 ) error{ MalformedFseBits, EndOfStream }!void {
498 switch (@field(self, @tagName(choice)).table) {
499 .rle => {},
500 .fse => |table| {
501 const data = table[@field(self, @tagName(choice)).state];
502 const T = @TypeOf(@field(self, @tagName(choice))).State;
503 const bits_summand = try bit_reader.readBitsNoEof(T, data.bits);
504 const next_state = std.math.cast(
505 @TypeOf(@field(self, @tagName(choice))).State,
506 data.baseline + bits_summand,
507 ) orelse return error.MalformedFseBits;
508 @field(self, @tagName(choice)).state = next_state;
509 },
510 }
511 }
512
513 const FseTableError = error{
514 MalformedFseTable,
515 MalformedAccuracyLog,
516 RepeatModeFirst,
517 EndOfStream,
518 };
519
520 fn updateFseTable(
521 self: *Decode,
522 source: *BufferedReader,
523 comptime choice: DataType,
524 mode: SequencesSection.Header.Mode,
525 ) !void {
526 const field_name = @tagName(choice);
527 switch (mode) {
528 .predefined => {
529 @field(self, field_name).accuracy_log =
530 @field(zstd.compressed_block.default_accuracy_log, field_name);
531
532 @field(self, field_name).table =
533 @field(Table, "predefined_" ++ field_name);
534 },
535 .rle => {
536 @field(self, field_name).accuracy_log = 0;
537 @field(self, field_name).table = .{ .rle = try source.readByte() };
538 },
539 .fse => {
540 var bit_reader: std.io.BitReader(.little) = .init(source);
541
542 const table_size = try Table.decode(
543 &bit_reader,
544 @field(zstd.compressed_block.table_symbol_count_max, field_name),
545 @field(zstd.compressed_block.table_accuracy_log_max, field_name),
546 @field(self, field_name ++ "_fse_buffer"),
547 );
548 @field(self, field_name).table = .{
549 .fse = @field(self, field_name ++ "_fse_buffer")[0..table_size],
550 };
551 @field(self, field_name).accuracy_log = std.math.log2_int_ceil(usize, table_size);
552 },
553 .repeat => if (self.fse_tables_undefined) return error.RepeatModeFirst,
554 }
555 }
556
557 const Sequence = struct {
558 literal_length: u32,
559 match_length: u32,
560 offset: u32,
561 };
562
563 fn nextSequence(
564 self: *Decode,
565 bit_reader: *ReverseBitReader,
566 ) error{ InvalidBitStream, EndOfStream }!Sequence {
567 const raw_code = self.getCode(.offset);
568 const offset_code = std.math.cast(u5, raw_code) orelse {
569 return error.InvalidBitStream;
570 };
571 const offset_value = (@as(u32, 1) << offset_code) + try bit_reader.readBitsNoEof(u32, offset_code);
572
573 const match_code = self.getCode(.match);
574 if (match_code >= zstd.compressed_block.match_length_code_table.len)
575 return error.InvalidBitStream;
576 const match = zstd.compressed_block.match_length_code_table[match_code];
577 const match_length = match[0] + try bit_reader.readBitsNoEof(u32, match[1]);
578
579 const literal_code = self.getCode(.literal);
580 if (literal_code >= zstd.compressed_block.literals_length_code_table.len)
581 return error.InvalidBitStream;
582 const literal = zstd.compressed_block.literals_length_code_table[literal_code];
583 const literal_length = literal[0] + try bit_reader.readBitsNoEof(u32, literal[1]);
584
585 const offset = if (offset_value > 3) offset: {
586 const offset = offset_value - 3;
587 self.updateRepeatOffset(offset);
588 break :offset offset;
589 } else offset: {
590 if (literal_length == 0) {
591 if (offset_value == 3) {
592 const offset = self.repeat_offsets[0] - 1;
593 self.updateRepeatOffset(offset);
594 break :offset offset;
595 }
596 break :offset self.useRepeatOffset(offset_value);
597 }
598 break :offset self.useRepeatOffset(offset_value - 1);
599 };
600
601 if (offset == 0) return error.InvalidBitStream;
602
603 return .{
604 .literal_length = literal_length,
605 .match_length = match_length,
606 .offset = offset,
607 };
608 }
609
610 const DecodeSequenceError = error{
611 /// The decompressed sequence would be longer than
612 /// `sequence_size_limit` or the sequence's offset is too large
613 MalformedSequence,
614 /// The decoder state's literal streams do not contain enough
615 /// literals for the sequence (this may mean the literal stream or the
616 /// sequence is malformed).
617 UnexpectedEndOfLiteralStream,
618 /// The FSE sequence bitstream is malformed
619 InvalidBitStream,
620 /// `bit_reader` does not contain enough bits
621 EndOfStream,
622 /// The `BufferedWriter` storage capacity is not large enough to
623 /// accept this stream.
624 OutputBufferUndersize,
625 };
626
627 /// Decode one sequence from `bit_reader` into `dest`. Updates FSE states
628 /// if `last_sequence` is `false`. Assumes `prepare` called for the block
629 /// before attempting to decode sequences.
630 pub fn decodeSequence(
631 self: *Decode,
632 dest: *BufferedWriter,
633 bit_reader: *ReverseBitReader,
634 sequence_size_limit: usize,
635 last_sequence: bool,
636 ) DecodeSequenceError!usize {
637 const sequence = try self.nextSequence(bit_reader);
638 const literal_length: usize = sequence.literal_length;
639 const match_length: usize = sequence.match_length;
640 const sequence_length = literal_length + match_length;
641 if (sequence_length > sequence_size_limit) return error.MalformedSequence;
642 if (sequence_length > dest.buffer.len) return error.OutputBufferUndersize;
643
644 if (sequence.offset > literal_length) return error.MalformedSequence;
645 // Ensures the following call to `decodeLiterals` will not cause a
646 // flush and therefore be at the beginning of `out`.
647 const out = try dest.writableSlice(sequence_length);
648 const expected_end = dest.end + out.len;
649 try decodeLiterals(self, dest, literal_length);
650 @memmove(
651 out[literal_length..][0..match_length],
652 out[literal_length - sequence.offset ..][0..match_length],
653 );
654 dest.advance(match_length);
655 assert(dest.end == expected_end);
656
657 if (!last_sequence) {
658 try self.updateState(.literal, bit_reader);
659 try self.updateState(.match, bit_reader);
660 try self.updateState(.offset, bit_reader);
661 }
662 return sequence_length;
663 }
664
665 fn nextLiteralMultiStream(self: *Decode) error{BitStreamHasNoStartBit}!void {
666 self.literal_stream_index += 1;
667 try self.initLiteralStream(self.literal_streams.four[self.literal_stream_index]);
668 }
669
670 fn initLiteralStream(self: *Decode, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
671 try self.literal_stream_reader.init(bytes);
672 }
673
674 fn isLiteralStreamEmpty(self: *Decode) bool {
675 switch (self.literal_streams) {
676 .one => return self.literal_stream_reader.isEmpty(),
677 .four => return self.literal_stream_index == 3 and self.literal_stream_reader.isEmpty(),
678 }
679 }
680
681 const LiteralBitsError = error{
682 BitStreamHasNoStartBit,
683 UnexpectedEndOfLiteralStream,
684 };
685 fn readLiteralsBits(
686 self: *Decode,
687 bit_count_to_read: u16,
688 ) LiteralBitsError!u16 {
689 return self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch bits: {
690 if (self.literal_streams == .four and self.literal_stream_index < 3) {
691 try self.nextLiteralMultiStream();
692 break :bits self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch
693 return error.UnexpectedEndOfLiteralStream;
694 } else {
695 return error.UnexpectedEndOfLiteralStream;
696 }
697 };
698 }
699
700 const DecodeLiteralsError = error{
701 /// The number of literal bytes decoded by `self` plus `len` is greater
702 /// than the regenerated size of `literals`
703 MalformedLiteralsLength,
704 /// Problems decoding Huffman compressed literals
705 UnexpectedEndOfLiteralStream,
706 OutputBufferUndersize,
707 };
708
709 /// Decode `len` bytes of literals into `dest`.
710 pub fn decodeLiterals(self: *Decode, dest: *BufferedWriter, len: usize) DecodeLiteralsError!void {
711 if (self.literal_written_count + len > self.literal_header.regenerated_size)
712 return error.MalformedLiteralsLength;
713
714 switch (self.literal_header.block_type) {
715 .raw => {
716 try dest.writeAll(self.literal_streams.one[self.literal_written_count..][0..len]);
717 self.literal_written_count += len;
718 },
719 .rle => {
720 try dest.splatByteAll(self.literal_streams.one[0], len);
721 self.literal_written_count += len;
722 },
723 .compressed, .treeless => {
724 if (len > dest.buffer.len) return error.OutputBufferUndersize;
725 const buf = try dest.writableSlice(len);
726 const huffman_tree = self.huffman_tree.?;
727 const max_bit_count = huffman_tree.max_bit_count;
728 const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
729 huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
730 max_bit_count,
731 );
732 var bits_read: u4 = 0;
733 var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
734 var bit_count_to_read: u4 = starting_bit_count;
735 for (buf) |*out| {
736 var prefix: u16 = 0;
737 while (true) {
738 const new_bits = try self.readLiteralsBits(bit_count_to_read);
739 prefix <<= bit_count_to_read;
740 prefix |= new_bits;
741 bits_read += bit_count_to_read;
742 const result = try huffman_tree.query(huffman_tree_index, prefix);
743
744 switch (result) {
745 .symbol => |sym| {
746 out.* = sym;
747 bit_count_to_read = starting_bit_count;
748 bits_read = 0;
749 huffman_tree_index = huffman_tree.symbol_count_minus_one;
750 break;
751 },
752 .index => |index| {
753 huffman_tree_index = index;
754 const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
755 huffman_tree.nodes[index].weight,
756 max_bit_count,
757 );
758 bit_count_to_read = bit_count - bits_read;
759 },
760 }
761 }
762 }
763 self.literal_written_count += len;
764 },
765 }
766 }
767
768 fn getCode(self: *Decode, comptime choice: DataType) u32 {
769 return switch (@field(self, @tagName(choice)).table) {
770 .rle => |value| value,
771 .fse => |table| table[@field(self, @tagName(choice)).state].symbol,
772 };
773 }
774 };
775 };
776
777 pub const Skippable = struct {
778 pub const magic_min: Magic = @enumFromInt(0x184D2A50);
779 pub const magic_max: Magic = @enumFromInt(0x184D2A5F);
780
781 pub const Header = struct {
782 magic_number: u32,
783 frame_size: u32,
784 };
785 };
786
787 const InitError = error{
788 DictionaryIdFlagUnsupported,
789 WindowSizeUnknown,
790 WindowTooLarge,
791 ContentSizeTooLarge,
792 };
793 /// Validates `frame_header` and returns the associated `Frame`.
794 ///
795 /// Errors returned:
796 /// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
797 /// - `error.WindowSizeUnknown` if the frame does not have a valid window
798 /// size
799 /// - `error.WindowTooLarge` if the window size is larger than
800 /// `window_size_max` or `std.math.intMax(usize)`
801 /// - `error.ContentSizeTooLarge` if the frame header indicates a content
802 /// size larger than `std.math.maxInt(usize)`
803 pub fn init(
804 frame_header: Frame.Zstandard.Header,
805 window_size_max: usize,
806 verify_checksum: bool,
807 ) InitError!Frame {
808 if (frame_header.descriptor.dictionary_id_flag != 0)
809 return error.DictionaryIdFlagUnsupported;
810
811 const window_size_raw = frame_header.windowSize() orelse return error.WindowSizeUnknown;
812 const window_size = if (window_size_raw > window_size_max)
813 return error.WindowTooLarge
814 else
815 std.math.cast(usize, window_size_raw) orelse return error.WindowTooLarge;
816
817 const should_compute_checksum =
818 frame_header.descriptor.content_checksum_flag and verify_checksum;
819
820 const content_size = if (frame_header.content_size) |size|
821 std.math.cast(usize, size) orelse return error.ContentSizeTooLarge
822 else
823 null;
824
825 return .{
826 .hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,
827 .window_size = window_size,
828 .has_checksum = frame_header.descriptor.content_checksum_flag,
829 .block_size_max = @min(zstd.block_size_max, window_size),
830 .content_size = content_size,
831 };
832 }
833};
834
835pub const LiteralsSection = struct {
836 header: Header,
837 huffman_tree: ?HuffmanTree,
838 streams: Streams,
839
840 pub const Streams = union(enum) {
841 one: []const u8,
842 four: [4][]const u8,
843
844 fn decode(size_format: u2, stream_data: []const u8) !Streams {
845 if (size_format == 0) {
846 return .{ .one = stream_data };
847 }
848
849 if (stream_data.len < 6) return error.MalformedLiteralsSection;
850
851 const stream_1_length: usize = std.mem.readInt(u16, stream_data[0..2], .little);
852 const stream_2_length: usize = std.mem.readInt(u16, stream_data[2..4], .little);
853 const stream_3_length: usize = std.mem.readInt(u16, stream_data[4..6], .little);
854
855 const stream_1_start = 6;
856 const stream_2_start = stream_1_start + stream_1_length;
857 const stream_3_start = stream_2_start + stream_2_length;
858 const stream_4_start = stream_3_start + stream_3_length;
859
860 if (stream_data.len < stream_4_start) return error.MalformedLiteralsSection;
861
862 return .{ .four = .{
863 stream_data[stream_1_start .. stream_1_start + stream_1_length],
864 stream_data[stream_2_start .. stream_2_start + stream_2_length],
865 stream_data[stream_3_start .. stream_3_start + stream_3_length],
866 stream_data[stream_4_start..],
867 } };
868 }
869 };
870
871 pub const Header = struct {
872 block_type: BlockType,
873 size_format: u2,
874 regenerated_size: u20,
875 compressed_size: ?u18,
876
877 /// Decode a literals section header.
878 ///
879 /// Errors returned:
880 /// - `error.EndOfStream` if there are not enough bytes in `source`
881 pub fn decode(source: *BufferedReader) !Header {
882 const byte0 = try source.readByte();
883 const block_type = @as(BlockType, @enumFromInt(byte0 & 0b11));
884 const size_format = @as(u2, @intCast((byte0 & 0b1100) >> 2));
885 var regenerated_size: u20 = undefined;
886 var compressed_size: ?u18 = null;
887 switch (block_type) {
888 .raw, .rle => {
889 switch (size_format) {
890 0, 2 => {
891 regenerated_size = byte0 >> 3;
892 },
893 1 => regenerated_size = (byte0 >> 4) + (@as(u20, try source.readByte()) << 4),
894 3 => regenerated_size = (byte0 >> 4) +
895 (@as(u20, try source.readByte()) << 4) +
896 (@as(u20, try source.readByte()) << 12),
897 }
898 },
899 .compressed, .treeless => {
900 const byte1 = try source.readByte();
901 const byte2 = try source.readByte();
902 switch (size_format) {
903 0, 1 => {
904 regenerated_size = (byte0 >> 4) + ((@as(u20, byte1) & 0b00111111) << 4);
905 compressed_size = ((byte1 & 0b11000000) >> 6) + (@as(u18, byte2) << 2);
906 },
907 2 => {
908 const byte3 = try source.readByte();
909 regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00000011) << 12);
910 compressed_size = ((byte2 & 0b11111100) >> 2) + (@as(u18, byte3) << 6);
911 },
912 3 => {
913 const byte3 = try source.readByte();
914 const byte4 = try source.readByte();
915 regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00111111) << 12);
916 compressed_size = ((byte2 & 0b11000000) >> 6) + (@as(u18, byte3) << 2) + (@as(u18, byte4) << 10);
917 },
918 }
919 },
920 }
921 return .{
922 .block_type = block_type,
923 .size_format = size_format,
924 .regenerated_size = regenerated_size,
925 .compressed_size = compressed_size,
926 };
927 }
928 };
929
930 pub const BlockType = enum(u2) {
931 raw,
932 rle,
933 compressed,
934 treeless,
935 };
936
937 pub const HuffmanTree = struct {
938 max_bit_count: u4,
939 symbol_count_minus_one: u8,
940 nodes: [256]PrefixedSymbol,
941
942 pub const PrefixedSymbol = struct {
943 symbol: u8,
944 prefix: u16,
945 weight: u4,
946 };
947
948 pub const Result = union(enum) {
949 symbol: u8,
950 index: usize,
951 };
952
953 pub fn query(self: HuffmanTree, index: usize, prefix: u16) error{NotFound}!Result {
954 var node = self.nodes[index];
955 const weight = node.weight;
956 var i: usize = index;
957 while (node.weight == weight) {
958 if (node.prefix == prefix) return Result{ .symbol = node.symbol };
959 if (i == 0) return error.NotFound;
960 i -= 1;
961 node = self.nodes[i];
962 }
963 return Result{ .index = i };
964 }
965
966 pub fn weightToBitCount(weight: u4, max_bit_count: u4) u4 {
967 return if (weight == 0) 0 else ((max_bit_count + 1) - weight);
968 }
969
970 pub const DecodeError = Reader.Error || error{
971 MalformedHuffmanTree,
972 MalformedFseTable,
973 MalformedAccuracyLog,
974 EndOfStream,
975 MissingStartBit,
976 };
977
978 pub fn decode(in: *BufferedReader) HuffmanTree.DecodeError!HuffmanTree {
979 const header = try in.takeByte();
980 if (header < 128) {
981 return decodeFse(in, header);
982 } else {
983 return decodeDirect(in, header - 127);
984 }
985 }
986
987 fn decodeDirect(source: *BufferedReader, encoded_symbol_count: usize) HuffmanTree.DecodeError!HuffmanTree {
988 var weights: [256]u4 = undefined;
989 const weights_byte_count = (encoded_symbol_count + 1) / 2;
990 for (0..weights_byte_count) |i| {
991 const byte = try source.takeByte();
992 weights[2 * i] = @as(u4, @intCast(byte >> 4));
993 weights[2 * i + 1] = @as(u4, @intCast(byte & 0xF));
994 }
995 const symbol_count = encoded_symbol_count + 1;
996 return build(&weights, symbol_count);
997 }
998
999 fn decodeFse(in: *BufferedReader, compressed_size: usize) HuffmanTree.DecodeError!HuffmanTree {
1000 var weights: [256]u4 = undefined;
1001 const compressed_buffer = try in.take(compressed_size);
1002 var limited_stream: BufferedReader = undefined;
1003 limited_stream.initFixed(compressed_buffer);
1004 var bit_reader: std.io.BitReader(.little) = .init(&limited_stream);
1005 var entries: [1 << 6]Table.Fse = undefined;
1006 const table_size = try Table.decode(&bit_reader, 256, 6, &entries);
1007 const accuracy_log = std.math.log2_int_ceil(usize, table_size);
1008 const remaining = limited_stream.bufferContents();
1009 const symbol_count = try assignWeights(remaining, accuracy_log, &entries, weights);
1010 return build(&weights, symbol_count);
1011 }
1012
1013 fn assignWeights(
1014 huff_bits_buffer: []u8,
1015 accuracy_log: u16,
1016 entries: *[1 << 6]Table.Fse,
1017 weights: *[256]u4,
1018 ) !usize {
1019 var huff_bits = try ReverseBitReader.init(huff_bits_buffer);
1020
1021 var i: usize = 0;
1022 var even_state: u32 = try huff_bits.readBitsNoEof(u32, accuracy_log);
1023 var odd_state: u32 = try huff_bits.readBitsNoEof(u32, accuracy_log);
1024
1025 while (i < 254) {
1026 const even_data = entries[even_state];
1027 var read_bits: u16 = 0;
1028 const even_bits = huff_bits.readBits(u32, even_data.bits, &read_bits) catch unreachable;
1029 weights[i] = std.math.cast(u4, even_data.symbol) orelse return error.MalformedHuffmanTree;
1030 i += 1;
1031 if (read_bits < even_data.bits) {
1032 weights[i] = std.math.cast(u4, entries[odd_state].symbol) orelse return error.MalformedHuffmanTree;
1033 i += 1;
1034 break;
1035 }
1036 even_state = even_data.baseline + even_bits;
1037
1038 read_bits = 0;
1039 const odd_data = entries[odd_state];
1040 const odd_bits = huff_bits.readBits(u32, odd_data.bits, &read_bits) catch unreachable;
1041 weights[i] = std.math.cast(u4, odd_data.symbol) orelse return error.MalformedHuffmanTree;
1042 i += 1;
1043 if (read_bits < odd_data.bits) {
1044 if (i == 255) return error.MalformedHuffmanTree;
1045 weights[i] = std.math.cast(u4, entries[even_state].symbol) orelse return error.MalformedHuffmanTree;
1046 i += 1;
1047 break;
1048 }
1049 odd_state = odd_data.baseline + odd_bits;
1050 } else return error.MalformedHuffmanTree;
1051
1052 if (!huff_bits.isEmpty()) {
1053 return error.MalformedHuffmanTree;
1054 }
1055
1056 return i + 1; // stream contains all but the last symbol
1057 }
1058
1059 fn assignSymbols(weight_sorted_prefixed_symbols: []PrefixedSymbol, weights: [256]u4) usize {
1060 for (0..weight_sorted_prefixed_symbols.len) |i| {
1061 weight_sorted_prefixed_symbols[i] = .{
1062 .symbol = @as(u8, @intCast(i)),
1063 .weight = undefined,
1064 .prefix = undefined,
1065 };
1066 }
1067
1068 std.mem.sort(
1069 PrefixedSymbol,
1070 weight_sorted_prefixed_symbols,
1071 weights,
1072 lessThanByWeight,
1073 );
1074
1075 var prefix: u16 = 0;
1076 var prefixed_symbol_count: usize = 0;
1077 var sorted_index: usize = 0;
1078 const symbol_count = weight_sorted_prefixed_symbols.len;
1079 while (sorted_index < symbol_count) {
1080 var symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
1081 const weight = weights[symbol];
1082 if (weight == 0) {
1083 sorted_index += 1;
1084 continue;
1085 }
1086
1087 while (sorted_index < symbol_count) : ({
1088 sorted_index += 1;
1089 prefixed_symbol_count += 1;
1090 prefix += 1;
1091 }) {
1092 symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
1093 if (weights[symbol] != weight) {
1094 prefix = ((prefix - 1) >> (weights[symbol] - weight)) + 1;
1095 break;
1096 }
1097 weight_sorted_prefixed_symbols[prefixed_symbol_count].symbol = symbol;
1098 weight_sorted_prefixed_symbols[prefixed_symbol_count].prefix = prefix;
1099 weight_sorted_prefixed_symbols[prefixed_symbol_count].weight = weight;
1100 }
1101 }
1102 return prefixed_symbol_count;
1103 }
1104
1105 fn build(weights: *[256]u4, symbol_count: usize) error{MalformedHuffmanTree}!HuffmanTree {
1106 var weight_power_sum_big: u32 = 0;
1107 for (weights[0 .. symbol_count - 1]) |value| {
1108 weight_power_sum_big += (@as(u16, 1) << value) >> 1;
1109 }
1110 if (weight_power_sum_big >= 1 << 11) return error.MalformedHuffmanTree;
1111 const weight_power_sum = @as(u16, @intCast(weight_power_sum_big));
1112
1113 // advance to next power of two (even if weight_power_sum is a power of 2)
1114 // TODO: is it valid to have weight_power_sum == 0?
1115 const max_number_of_bits = if (weight_power_sum == 0) 1 else std.math.log2_int(u16, weight_power_sum) + 1;
1116 const next_power_of_two = @as(u16, 1) << max_number_of_bits;
1117 weights[symbol_count - 1] = std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1;
1118
1119 var weight_sorted_prefixed_symbols: [256]PrefixedSymbol = undefined;
1120 const prefixed_symbol_count = assignSymbols(weight_sorted_prefixed_symbols[0..symbol_count], weights.*);
1121 const tree: HuffmanTree = .{
1122 .max_bit_count = max_number_of_bits,
1123 .symbol_count_minus_one = @as(u8, @intCast(prefixed_symbol_count - 1)),
1124 .nodes = weight_sorted_prefixed_symbols,
1125 };
1126 return tree;
1127 }
1128
1129 fn lessThanByWeight(
1130 weights: [256]u4,
1131 lhs: PrefixedSymbol,
1132 rhs: PrefixedSymbol,
1133 ) bool {
1134 // NOTE: this function relies on the use of a stable sorting algorithm,
1135 // otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs;
1136 // should be added
1137 return weights[lhs.symbol] < weights[rhs.symbol];
1138 }
1139 };
1140
1141 pub const StreamCount = enum { one, four };
1142 pub fn streamCount(size_format: u2, block_type: BlockType) StreamCount {
1143 return switch (block_type) {
1144 .raw, .rle => .one,
1145 .compressed, .treeless => if (size_format == 0) .one else .four,
1146 };
1147 }
1148
1149 pub const DecodeError = error{
1150 /// Invalid header.
1151 MalformedLiteralsHeader,
1152 /// Decoding errors.
1153 MalformedLiteralsSection,
1154 /// Compressed literals have invalid accuracy.
1155 MalformedAccuracyLog,
1156 /// Compressed literals have invalid FSE table.
1157 MalformedFseTable,
1158 /// Failed decoding a Huffamn tree.
1159 MalformedHuffmanTree,
1160 /// Not enough bytes to complete the section.
1161 EndOfStream,
1162 };
1163
1164 pub fn decode(source: *BufferedReader, buffer: []u8) DecodeError!LiteralsSection {
1165 const header = try Header.decode(source);
1166 switch (header.block_type) {
1167 .raw => {
1168 if (buffer.len < header.regenerated_size) return error.LiteralsBufferTooSmall;
1169 try source.readNoEof(buffer[0..header.regenerated_size]);
1170 return .{
1171 .header = header,
1172 .huffman_tree = null,
1173 .streams = .{ .one = buffer },
1174 };
1175 },
1176 .rle => {
1177 buffer[0] = try source.readByte();
1178 return .{
1179 .header = header,
1180 .huffman_tree = null,
1181 .streams = .{ .one = buffer[0..1] },
1182 };
1183 },
1184 .compressed, .treeless => {
1185 var counting_reader = std.io.countingReader(source);
1186 const huffman_tree = if (header.block_type == .compressed)
1187 try HuffmanTree.decode(counting_reader.reader(), buffer)
1188 else
1189 null;
1190 const huffman_tree_size = @as(usize, @intCast(counting_reader.bytes_read));
1191 const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
1192 return error.MalformedLiteralsSection;
1193
1194 if (total_streams_size > buffer.len) return error.LiteralsBufferTooSmall;
1195 try source.readNoEof(buffer[0..total_streams_size]);
1196 const stream_data = buffer[0..total_streams_size];
1197
1198 const streams = try Streams.decode(header.size_format, stream_data);
1199 return .{
1200 .header = header,
1201 .huffman_tree = huffman_tree,
1202 .streams = streams,
1203 };
1204 },
1205 }
1206 }
1207};
1208
1209pub const SequencesSection = struct {
1210 header: SequencesSection.Header,
1211 literals_length_table: Table,
1212 offset_table: Table,
1213 match_length_table: Table,
1214
1215 pub const Header = struct {
1216 sequence_count: u24,
1217 match_lengths: Mode,
1218 offsets: Mode,
1219 literal_lengths: Mode,
1220
1221 pub const Mode = enum(u2) {
1222 predefined,
1223 rle,
1224 fse,
1225 repeat,
1226 };
1227
1228 pub const DecodeError = error{
1229 ReservedBitSet,
1230 EndOfStream,
1231 };
1232
1233 pub fn decode(source: *BufferedReader) DecodeError!Header {
1234 var sequence_count: u24 = undefined;
1235
1236 const byte0 = try source.readByte();
1237 if (byte0 == 0) {
1238 return SequencesSection.Header{
1239 .sequence_count = 0,
1240 .offsets = undefined,
1241 .match_lengths = undefined,
1242 .literal_lengths = undefined,
1243 };
1244 } else if (byte0 < 128) {
1245 sequence_count = byte0;
1246 } else if (byte0 < 255) {
1247 sequence_count = (@as(u24, (byte0 - 128)) << 8) + try source.readByte();
1248 } else {
1249 sequence_count = (try source.readByte()) + (@as(u24, try source.readByte()) << 8) + 0x7F00;
1250 }
1251
1252 const compression_modes = try source.readByte();
1253
1254 const matches_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00001100) >> 2));
1255 const offsets_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00110000) >> 4));
1256 const literal_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b11000000) >> 6));
1257 if (compression_modes & 0b11 != 0) return error.ReservedBitSet;
1258
1259 return .{
1260 .sequence_count = sequence_count,
1261 .offsets = offsets_mode,
1262 .match_lengths = matches_mode,
1263 .literal_lengths = literal_mode,
1264 };
1265 }
1266 };
1267};
1268
1269pub const Table = union(enum) {
1270 fse: []const Fse,
1271 rle: u8,
1272
1273 pub const Fse = struct {
1274 symbol: u8,
1275 baseline: u16,
1276 bits: u8,
1277 };
1278
1279 pub fn decode(
1280 bit_reader: *std.io.BitReader(.little),
1281 expected_symbol_count: usize,
1282 max_accuracy_log: u4,
1283 entries: []Table.Fse,
1284 ) !usize {
1285 const accuracy_log_biased = try bit_reader.readBitsNoEof(u4, 4);
1286 if (accuracy_log_biased > max_accuracy_log -| 5) return error.MalformedAccuracyLog;
1287 const accuracy_log = accuracy_log_biased + 5;
1288
1289 var values: [256]u16 = undefined;
1290 var value_count: usize = 0;
1291
1292 const total_probability = @as(u16, 1) << accuracy_log;
1293 var accumulated_probability: u16 = 0;
1294
1295 while (accumulated_probability < total_probability) {
1296 // WARNING: The RFC is poorly worded, and would suggest std.math.log2_int_ceil is correct here,
1297 // but power of two (remaining probabilities + 1) need max bits set to 1 more.
1298 const max_bits = std.math.log2_int(u16, total_probability - accumulated_probability + 1) + 1;
1299 const small = try bit_reader.readBitsNoEof(u16, max_bits - 1);
1300
1301 const cutoff = (@as(u16, 1) << max_bits) - 1 - (total_probability - accumulated_probability + 1);
1302
1303 const value = if (small < cutoff)
1304 small
1305 else value: {
1306 const value_read = small + (try bit_reader.readBitsNoEof(u16, 1) << (max_bits - 1));
1307 break :value if (value_read < @as(u16, 1) << (max_bits - 1))
1308 value_read
1309 else
1310 value_read - cutoff;
1311 };
1312
1313 accumulated_probability += if (value != 0) value - 1 else 1;
1314
1315 values[value_count] = value;
1316 value_count += 1;
1317
1318 if (value == 1) {
1319 while (true) {
1320 const repeat_flag = try bit_reader.readBitsNoEof(u2, 2);
1321 if (repeat_flag + value_count > 256) return error.MalformedFseTable;
1322 for (0..repeat_flag) |_| {
1323 values[value_count] = 1;
1324 value_count += 1;
1325 }
1326 if (repeat_flag < 3) break;
1327 }
1328 }
1329 if (value_count == 256) break;
1330 }
1331 bit_reader.alignToByte();
1332
1333 if (value_count < 2) return error.MalformedFseTable;
1334 if (accumulated_probability != total_probability) return error.MalformedFseTable;
1335 if (value_count > expected_symbol_count) return error.MalformedFseTable;
1336
1337 const table_size = total_probability;
1338
1339 try build(values[0..value_count], entries[0..table_size]);
1340 return table_size;
1341 }
1342
1343 pub fn build(values: []const u16, entries: []Table.Fse) !void {
1344 const total_probability = @as(u16, @intCast(entries.len));
1345 const accuracy_log = std.math.log2_int(u16, total_probability);
1346 assert(total_probability <= 1 << 9);
1347
1348 var less_than_one_count: usize = 0;
1349 for (values, 0..) |value, i| {
1350 if (value == 0) {
1351 entries[entries.len - 1 - less_than_one_count] = Table.Fse{
1352 .symbol = @as(u8, @intCast(i)),
1353 .baseline = 0,
1354 .bits = accuracy_log,
1355 };
1356 less_than_one_count += 1;
1357 }
1358 }
1359
1360 var position: usize = 0;
1361 var temp_states: [1 << 9]u16 = undefined;
1362 for (values, 0..) |value, symbol| {
1363 if (value == 0 or value == 1) continue;
1364 const probability = value - 1;
1365
1366 const state_share_dividend = std.math.ceilPowerOfTwo(u16, probability) catch
1367 return error.MalformedFseTable;
1368 const share_size = @divExact(total_probability, state_share_dividend);
1369 const double_state_count = state_share_dividend - probability;
1370 const single_state_count = probability - double_state_count;
1371 const share_size_log = std.math.log2_int(u16, share_size);
1372
1373 for (0..probability) |i| {
1374 temp_states[i] = @as(u16, @intCast(position));
1375 position += (entries.len >> 1) + (entries.len >> 3) + 3;
1376 position &= entries.len - 1;
1377 while (position >= entries.len - less_than_one_count) {
1378 position += (entries.len >> 1) + (entries.len >> 3) + 3;
1379 position &= entries.len - 1;
1380 }
1381 }
1382 std.mem.sort(u16, temp_states[0..probability], {}, std.sort.asc(u16));
1383 for (0..probability) |i| {
1384 entries[temp_states[i]] = if (i < double_state_count) Table.Fse{
1385 .symbol = @as(u8, @intCast(symbol)),
1386 .bits = share_size_log + 1,
1387 .baseline = single_state_count * share_size + @as(u16, @intCast(i)) * 2 * share_size,
1388 } else Table.Fse{
1389 .symbol = @as(u8, @intCast(symbol)),
1390 .bits = share_size_log,
1391 .baseline = (@as(u16, @intCast(i)) - double_state_count) * share_size,
1392 };
1393 }
1394 }
1395 }
1396
1397 test build {
1398 const literals_length_default_values = [36]u16{
1399 5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2,
1400 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 2, 2, 2, 2, 2,
1401 0, 0, 0, 0,
1402 };
1403
1404 const match_lengths_default_values = [53]u16{
1405 2, 5, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
1406 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
1407 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0,
1408 0, 0, 0, 0, 0,
1409 };
1410
1411 const offset_codes_default_values = [29]u16{
1412 2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
1413 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0,
1414 };
1415
1416 var entries: [64]Table.Fse = undefined;
1417 try build(&literals_length_default_values, &entries);
1418 try std.testing.expectEqualSlices(Table.Fse, Table.predefined_literal.fse, &entries);
1419
1420 try build(&match_lengths_default_values, &entries);
1421 try std.testing.expectEqualSlices(Table.Fse, Table.predefined_match.fse, &entries);
1422
1423 try build(&offset_codes_default_values, entries[0..32]);
1424 try std.testing.expectEqualSlices(Table.Fse, Table.predefined_offset.fse, entries[0..32]);
1425 }
1426
1427 pub const predefined_literal: Table = .{
1428 .fse = &[64]Table.Fse{
1429 .{ .symbol = 0, .bits = 4, .baseline = 0 },
1430 .{ .symbol = 0, .bits = 4, .baseline = 16 },
1431 .{ .symbol = 1, .bits = 5, .baseline = 32 },
1432 .{ .symbol = 3, .bits = 5, .baseline = 0 },
1433 .{ .symbol = 4, .bits = 5, .baseline = 0 },
1434 .{ .symbol = 6, .bits = 5, .baseline = 0 },
1435 .{ .symbol = 7, .bits = 5, .baseline = 0 },
1436 .{ .symbol = 9, .bits = 5, .baseline = 0 },
1437 .{ .symbol = 10, .bits = 5, .baseline = 0 },
1438 .{ .symbol = 12, .bits = 5, .baseline = 0 },
1439 .{ .symbol = 14, .bits = 6, .baseline = 0 },
1440 .{ .symbol = 16, .bits = 5, .baseline = 0 },
1441 .{ .symbol = 18, .bits = 5, .baseline = 0 },
1442 .{ .symbol = 19, .bits = 5, .baseline = 0 },
1443 .{ .symbol = 21, .bits = 5, .baseline = 0 },
1444 .{ .symbol = 22, .bits = 5, .baseline = 0 },
1445 .{ .symbol = 24, .bits = 5, .baseline = 0 },
1446 .{ .symbol = 25, .bits = 5, .baseline = 32 },
1447 .{ .symbol = 26, .bits = 5, .baseline = 0 },
1448 .{ .symbol = 27, .bits = 6, .baseline = 0 },
1449 .{ .symbol = 29, .bits = 6, .baseline = 0 },
1450 .{ .symbol = 31, .bits = 6, .baseline = 0 },
1451 .{ .symbol = 0, .bits = 4, .baseline = 32 },
1452 .{ .symbol = 1, .bits = 4, .baseline = 0 },
1453 .{ .symbol = 2, .bits = 5, .baseline = 0 },
1454 .{ .symbol = 4, .bits = 5, .baseline = 32 },
1455 .{ .symbol = 5, .bits = 5, .baseline = 0 },
1456 .{ .symbol = 7, .bits = 5, .baseline = 32 },
1457 .{ .symbol = 8, .bits = 5, .baseline = 0 },
1458 .{ .symbol = 10, .bits = 5, .baseline = 32 },
1459 .{ .symbol = 11, .bits = 5, .baseline = 0 },
1460 .{ .symbol = 13, .bits = 6, .baseline = 0 },
1461 .{ .symbol = 16, .bits = 5, .baseline = 32 },
1462 .{ .symbol = 17, .bits = 5, .baseline = 0 },
1463 .{ .symbol = 19, .bits = 5, .baseline = 32 },
1464 .{ .symbol = 20, .bits = 5, .baseline = 0 },
1465 .{ .symbol = 22, .bits = 5, .baseline = 32 },
1466 .{ .symbol = 23, .bits = 5, .baseline = 0 },
1467 .{ .symbol = 25, .bits = 4, .baseline = 0 },
1468 .{ .symbol = 25, .bits = 4, .baseline = 16 },
1469 .{ .symbol = 26, .bits = 5, .baseline = 32 },
1470 .{ .symbol = 28, .bits = 6, .baseline = 0 },
1471 .{ .symbol = 30, .bits = 6, .baseline = 0 },
1472 .{ .symbol = 0, .bits = 4, .baseline = 48 },
1473 .{ .symbol = 1, .bits = 4, .baseline = 16 },
1474 .{ .symbol = 2, .bits = 5, .baseline = 32 },
1475 .{ .symbol = 3, .bits = 5, .baseline = 32 },
1476 .{ .symbol = 5, .bits = 5, .baseline = 32 },
1477 .{ .symbol = 6, .bits = 5, .baseline = 32 },
1478 .{ .symbol = 8, .bits = 5, .baseline = 32 },
1479 .{ .symbol = 9, .bits = 5, .baseline = 32 },
1480 .{ .symbol = 11, .bits = 5, .baseline = 32 },
1481 .{ .symbol = 12, .bits = 5, .baseline = 32 },
1482 .{ .symbol = 15, .bits = 6, .baseline = 0 },
1483 .{ .symbol = 17, .bits = 5, .baseline = 32 },
1484 .{ .symbol = 18, .bits = 5, .baseline = 32 },
1485 .{ .symbol = 20, .bits = 5, .baseline = 32 },
1486 .{ .symbol = 21, .bits = 5, .baseline = 32 },
1487 .{ .symbol = 23, .bits = 5, .baseline = 32 },
1488 .{ .symbol = 24, .bits = 5, .baseline = 32 },
1489 .{ .symbol = 35, .bits = 6, .baseline = 0 },
1490 .{ .symbol = 34, .bits = 6, .baseline = 0 },
1491 .{ .symbol = 33, .bits = 6, .baseline = 0 },
1492 .{ .symbol = 32, .bits = 6, .baseline = 0 },
1493 },
1494 };
1495
1496 pub const predefined_match: Table = .{
1497 .fse = &[64]Table.Fse{
1498 .{ .symbol = 0, .bits = 6, .baseline = 0 },
1499 .{ .symbol = 1, .bits = 4, .baseline = 0 },
1500 .{ .symbol = 2, .bits = 5, .baseline = 32 },
1501 .{ .symbol = 3, .bits = 5, .baseline = 0 },
1502 .{ .symbol = 5, .bits = 5, .baseline = 0 },
1503 .{ .symbol = 6, .bits = 5, .baseline = 0 },
1504 .{ .symbol = 8, .bits = 5, .baseline = 0 },
1505 .{ .symbol = 10, .bits = 6, .baseline = 0 },
1506 .{ .symbol = 13, .bits = 6, .baseline = 0 },
1507 .{ .symbol = 16, .bits = 6, .baseline = 0 },
1508 .{ .symbol = 19, .bits = 6, .baseline = 0 },
1509 .{ .symbol = 22, .bits = 6, .baseline = 0 },
1510 .{ .symbol = 25, .bits = 6, .baseline = 0 },
1511 .{ .symbol = 28, .bits = 6, .baseline = 0 },
1512 .{ .symbol = 31, .bits = 6, .baseline = 0 },
1513 .{ .symbol = 33, .bits = 6, .baseline = 0 },
1514 .{ .symbol = 35, .bits = 6, .baseline = 0 },
1515 .{ .symbol = 37, .bits = 6, .baseline = 0 },
1516 .{ .symbol = 39, .bits = 6, .baseline = 0 },
1517 .{ .symbol = 41, .bits = 6, .baseline = 0 },
1518 .{ .symbol = 43, .bits = 6, .baseline = 0 },
1519 .{ .symbol = 45, .bits = 6, .baseline = 0 },
1520 .{ .symbol = 1, .bits = 4, .baseline = 16 },
1521 .{ .symbol = 2, .bits = 4, .baseline = 0 },
1522 .{ .symbol = 3, .bits = 5, .baseline = 32 },
1523 .{ .symbol = 4, .bits = 5, .baseline = 0 },
1524 .{ .symbol = 6, .bits = 5, .baseline = 32 },
1525 .{ .symbol = 7, .bits = 5, .baseline = 0 },
1526 .{ .symbol = 9, .bits = 6, .baseline = 0 },
1527 .{ .symbol = 12, .bits = 6, .baseline = 0 },
1528 .{ .symbol = 15, .bits = 6, .baseline = 0 },
1529 .{ .symbol = 18, .bits = 6, .baseline = 0 },
1530 .{ .symbol = 21, .bits = 6, .baseline = 0 },
1531 .{ .symbol = 24, .bits = 6, .baseline = 0 },
1532 .{ .symbol = 27, .bits = 6, .baseline = 0 },
1533 .{ .symbol = 30, .bits = 6, .baseline = 0 },
1534 .{ .symbol = 32, .bits = 6, .baseline = 0 },
1535 .{ .symbol = 34, .bits = 6, .baseline = 0 },
1536 .{ .symbol = 36, .bits = 6, .baseline = 0 },
1537 .{ .symbol = 38, .bits = 6, .baseline = 0 },
1538 .{ .symbol = 40, .bits = 6, .baseline = 0 },
1539 .{ .symbol = 42, .bits = 6, .baseline = 0 },
1540 .{ .symbol = 44, .bits = 6, .baseline = 0 },
1541 .{ .symbol = 1, .bits = 4, .baseline = 32 },
1542 .{ .symbol = 1, .bits = 4, .baseline = 48 },
1543 .{ .symbol = 2, .bits = 4, .baseline = 16 },
1544 .{ .symbol = 4, .bits = 5, .baseline = 32 },
1545 .{ .symbol = 5, .bits = 5, .baseline = 32 },
1546 .{ .symbol = 7, .bits = 5, .baseline = 32 },
1547 .{ .symbol = 8, .bits = 5, .baseline = 32 },
1548 .{ .symbol = 11, .bits = 6, .baseline = 0 },
1549 .{ .symbol = 14, .bits = 6, .baseline = 0 },
1550 .{ .symbol = 17, .bits = 6, .baseline = 0 },
1551 .{ .symbol = 20, .bits = 6, .baseline = 0 },
1552 .{ .symbol = 23, .bits = 6, .baseline = 0 },
1553 .{ .symbol = 26, .bits = 6, .baseline = 0 },
1554 .{ .symbol = 29, .bits = 6, .baseline = 0 },
1555 .{ .symbol = 52, .bits = 6, .baseline = 0 },
1556 .{ .symbol = 51, .bits = 6, .baseline = 0 },
1557 .{ .symbol = 50, .bits = 6, .baseline = 0 },
1558 .{ .symbol = 49, .bits = 6, .baseline = 0 },
1559 .{ .symbol = 48, .bits = 6, .baseline = 0 },
1560 .{ .symbol = 47, .bits = 6, .baseline = 0 },
1561 .{ .symbol = 46, .bits = 6, .baseline = 0 },
1562 },
1563 };
1564
1565 pub const predefined_offset: Table = .{
1566 .fse = &[32]Table.Fse{
1567 .{ .symbol = 0, .bits = 5, .baseline = 0 },
1568 .{ .symbol = 6, .bits = 4, .baseline = 0 },
1569 .{ .symbol = 9, .bits = 5, .baseline = 0 },
1570 .{ .symbol = 15, .bits = 5, .baseline = 0 },
1571 .{ .symbol = 21, .bits = 5, .baseline = 0 },
1572 .{ .symbol = 3, .bits = 5, .baseline = 0 },
1573 .{ .symbol = 7, .bits = 4, .baseline = 0 },
1574 .{ .symbol = 12, .bits = 5, .baseline = 0 },
1575 .{ .symbol = 18, .bits = 5, .baseline = 0 },
1576 .{ .symbol = 23, .bits = 5, .baseline = 0 },
1577 .{ .symbol = 5, .bits = 5, .baseline = 0 },
1578 .{ .symbol = 8, .bits = 4, .baseline = 0 },
1579 .{ .symbol = 14, .bits = 5, .baseline = 0 },
1580 .{ .symbol = 20, .bits = 5, .baseline = 0 },
1581 .{ .symbol = 2, .bits = 5, .baseline = 0 },
1582 .{ .symbol = 7, .bits = 4, .baseline = 16 },
1583 .{ .symbol = 11, .bits = 5, .baseline = 0 },
1584 .{ .symbol = 17, .bits = 5, .baseline = 0 },
1585 .{ .symbol = 22, .bits = 5, .baseline = 0 },
1586 .{ .symbol = 4, .bits = 5, .baseline = 0 },
1587 .{ .symbol = 8, .bits = 4, .baseline = 16 },
1588 .{ .symbol = 13, .bits = 5, .baseline = 0 },
1589 .{ .symbol = 19, .bits = 5, .baseline = 0 },
1590 .{ .symbol = 1, .bits = 5, .baseline = 0 },
1591 .{ .symbol = 6, .bits = 4, .baseline = 16 },
1592 .{ .symbol = 10, .bits = 5, .baseline = 0 },
1593 .{ .symbol = 16, .bits = 5, .baseline = 0 },
1594 .{ .symbol = 28, .bits = 5, .baseline = 0 },
1595 .{ .symbol = 27, .bits = 5, .baseline = 0 },
1596 .{ .symbol = 26, .bits = 5, .baseline = 0 },
1597 .{ .symbol = 25, .bits = 5, .baseline = 0 },
1598 .{ .symbol = 24, .bits = 5, .baseline = 0 },
1599 },
1600 };
1601};
1602
1603/// For reading the reversed bit streams used to encode FSE compressed data.
1604const ReverseBitReader = struct {
1605 bytes: []const u8,
1606 remaining: usize,
1607 bits: u8,
1608 count: u4,
1609
1610 fn init(bytes: []const u8) error{MissingStartBit}!ReverseBitReader {
1611 var result: ReverseBitReader = .{
1612 .bytes = bytes,
1613 .remaining = bytes.len,
1614 .bits = 0,
1615 .count = 0,
1616 };
1617 if (bytes.len == 0) return result;
1618 for (0..8) |_| if (0 != (result.readBitsNoEof(u1, 1) catch unreachable)) return result;
1619 return error.MissingStartBit;
1620 }
1621
1622 fn readBitsNoEof(self: *ReverseBitReader, comptime U: type, num_bits: u16) error{EndOfStream}!U {
1623 return self.bit_reader.readBitsNoEof(U, num_bits);
1624 }
1625
1626 fn readBits(self: *ReverseBitReader, comptime U: type, num_bits: u16, out_bits: *u16) error{}!U {
1627 return try self.bit_reader.readBits(U, num_bits, out_bits);
1628 }
1629
1630 fn alignToByte(self: *ReverseBitReader) void {
1631 self.bit_reader.alignToByte();
1632 }
1633
1634 fn isEmpty(self: *const ReverseBitReader) bool {
1635 return self.byte_reader.remaining_bytes == 0 and self.bit_reader.count == 0;
1636 }
1637};
1638
1639test {
1640 _ = Table;
1641}
lib/std/crypto/sha2.zig+3-1
......@@ -293,7 +293,9 @@ fn Sha2x32(comptime iv: Iv32, digest_bits: comptime_int) type {
293293
294294 var i: usize = 16;
295295 while (i < 64) : (i += 1) {
296 s[i] = s[i - 16] +% s[i - 7] +% (math.rotr(u32, s[i - 15], @as(u32, 7)) ^ math.rotr(u32, s[i - 15], @as(u32, 18)) ^ (s[i - 15] >> 3)) +% (math.rotr(u32, s[i - 2], @as(u32, 17)) ^ math.rotr(u32, s[i - 2], @as(u32, 19)) ^ (s[i - 2] >> 10));
296 s[i] = s[i - 16] +% s[i - 7] +%
297 (math.rotr(u32, s[i - 15], @as(u32, 7)) ^ math.rotr(u32, s[i - 15], @as(u32, 18)) ^ (s[i - 15] >> 3)) +%
298 (math.rotr(u32, s[i - 2], @as(u32, 17)) ^ math.rotr(u32, s[i - 2], @as(u32, 19)) ^ (s[i - 2] >> 10));
297299 }
298300
299301 var v: [8]u32 = d.s;
lib/std/io/BufferedReader.zig+16-1
......@@ -825,17 +825,32 @@ pub inline fn peekStructEndian(br: *BufferedReader, comptime T: type, endian: st
825825 return res;
826826}
827827
828pub const TakeEnumError = Reader.Error || error{InvalidEnumTag};
829
828830/// Reads an integer with the same size as the given enum's tag type. If the
829831/// integer matches an enum tag, casts the integer to the enum tag and returns
830832/// it. Otherwise, returns `error.InvalidEnumTag`.
831833///
832834/// Asserts the buffer was initialized with a capacity at least `@sizeOf(Enum)`.
833pub fn takeEnum(br: *BufferedReader, comptime Enum: type, endian: std.builtin.Endian) (Reader.Error || std.meta.IntToEnumError)!Enum {
835pub fn takeEnum(br: *BufferedReader, comptime Enum: type, endian: std.builtin.Endian) TakeEnumError!Enum {
834836 const Tag = @typeInfo(Enum).@"enum".tag_type;
835837 const int = try br.takeInt(Tag, endian);
836838 return std.meta.intToEnum(Enum, int);
837839}
838840
841/// Reads an integer with the same size as the given nonexhaustive enum's tag type.
842///
843/// Asserts the buffer was initialized with a capacity at least `@sizeOf(Enum)`.
844pub fn takeEnumNonexhaustive(br: *BufferedReader, comptime Enum: type, endian: std.builtin.Endian) Reader.Error!Enum {
845 const info = @typeInfo(Enum).@"enum";
846 comptime assert(!info.is_exhaustive);
847 comptime assert(@bitSizeOf(info.tag_type) == @sizeOf(info.tag_type) * 8);
848 return takeEnum(br, Enum, endian) catch |err| switch (err) {
849 error.InvalidEnumTag => unreachable,
850 else => |e| return e,
851 };
852}
853
839854pub const TakeLeb128Error = Reader.Error || error{Overflow};
840855
841856/// Read a single LEB128 value as type T, or `error.Overflow` if the value cannot fit.
lib/std/io/Writer/Null.zig+1-1
......@@ -5,7 +5,7 @@ const Writer = std.io.Writer;
55
66const NullWriter = @This();
77
8err: Error,
8err: ?Error = null,
99
1010pub const Error = std.fs.File.StatError;
1111
lib/std/std.zig-1
......@@ -30,7 +30,6 @@ pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue;
3030pub const PriorityDequeue = @import("priority_dequeue.zig").PriorityDequeue;
3131pub const Progress = @import("Progress.zig");
3232pub const Random = @import("Random.zig");
33pub const RingBuffer = @import("RingBuffer.zig");
3433pub const SegmentedList = @import("segmented_list.zig").SegmentedList;
3534pub const SemanticVersion = @import("SemanticVersion.zig");
3635pub const SinglyLinkedList = @import("SinglyLinkedList.zig");