authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2024-02-18 13:21:48+11:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-23 02:37:11-07:00
log5c12783094ef6c36e2d26d58116fdb992306d044
tree07239df72ae8397ddce43485231d86e1916221c9
parent63acc856c12fec71d96c5cbce9138204f36d6568

std.compress.zstd: make DecompressStream options runtime

1 files changed, 23 insertions(+), 22 deletions(-)

lib/std/compress/zstandard.zig+23-22
......@@ -9,20 +9,14 @@ pub const decompress = @import("zstandard/decompress.zig");
99
1010pub const DecompressStreamOptions = struct {
1111 verify_checksum: bool = true,
12 window_size_max: usize = default_window_size_max,
13
14 pub const default_window_size_max = 1 << 23; // 8MiB default maximum window size
1512};
1613
1714pub fn DecompressStream(
1815 comptime ReaderType: type,
19 comptime options: DecompressStreamOptions,
2016) type {
2117 return struct {
2218 const Self = @This();
2319
24 pub const window_size_max = options.window_size_max;
25
2620 const table_size_max = types.compressed_block.table_size_max;
2721
2822 source: std.io.CountingReader(ReaderType),
......@@ -35,11 +29,12 @@ pub fn DecompressStream(
3529 offset_fse_buffer: [table_size_max.offset]types.compressed_block.Table.Fse,
3630 literals_buffer: [types.block_size_max]u8,
3731 sequence_buffer: [types.block_size_max]u8,
38 checksum: if (options.verify_checksum) ?u32 else void,
32 verify_checksum: bool,
33 checksum: ?u32,
3934 current_frame_decompressed_size: usize,
4035
4136 const WindowBuffer = struct {
42 data: *[options.window_size_max]u8 = undefined,
37 data: []u8 = undefined,
4338 read_index: usize = 0,
4439 write_index: usize = 0,
4540 };
......@@ -54,7 +49,7 @@ pub fn DecompressStream(
5449
5550 pub const Reader = std.io.Reader(*Self, Error, read);
5651
57 pub fn init(source: ReaderType, window_buffer: *[options.window_size_max]u8) Self {
52 pub fn init(source: ReaderType, window_buffer: []u8, options: DecompressStreamOptions) Self {
5853 return Self{
5954 .source = std.io.countingReader(source),
6055 .state = .NewFrame,
......@@ -66,6 +61,7 @@ pub fn DecompressStream(
6661 .offset_fse_buffer = undefined,
6762 .literals_buffer = undefined,
6863 .sequence_buffer = undefined,
64 .verify_checksum = options.verify_checksum,
6965 .checksum = undefined,
7066 .current_frame_decompressed_size = undefined,
7167 };
......@@ -81,8 +77,8 @@ pub fn DecompressStream(
8177 .zstandard => |header| {
8278 const frame_context = try decompress.FrameContext.init(
8379 header,
84 options.window_size_max,
85 options.verify_checksum,
80 self.buffer.data.len,
81 self.verify_checksum,
8682 );
8783
8884 const decode_state = decompress.block.DecodeState.init(
......@@ -94,7 +90,7 @@ pub fn DecompressStream(
9490 self.decode_state = decode_state;
9591 self.frame_context = frame_context;
9692
97 self.checksum = if (options.verify_checksum) null else {};
93 self.checksum = null;
9894 self.current_frame_decompressed_size = 0;
9995
10096 self.state = .InFrame;
......@@ -176,7 +172,7 @@ pub fn DecompressStream(
176172 if (self.frame_context.has_checksum) {
177173 const checksum = source_reader.readInt(u32, .little) catch
178174 return error.MalformedFrame;
179 if (comptime options.verify_checksum) {
175 if (self.verify_checksum) {
180176 if (self.frame_context.hasher_opt) |*hasher| {
181177 if (checksum != decompress.computeChecksum(hasher))
182178 return error.ChecksumFailure;
......@@ -213,15 +209,18 @@ pub fn decompressStreamOptions(
213209
214210pub fn decompressStream(
215211 reader: anytype,
216 window_buffer: *[DecompressStreamOptions.default_window_size_max]u8,
217) DecompressStream(@TypeOf(reader), .{}) {
218 return DecompressStream(@TypeOf(reader), .{}).init(reader, window_buffer);
212 window_buffer: []u8,
213 options: DecompressStreamOptions,
214) DecompressStream(@TypeOf(reader)) {
215 return DecompressStream(@TypeOf(reader)).init(reader, window_buffer, options);
219216}
220217
221218fn testDecompress(data: []const u8) ![]u8 {
222 var window_buffer: [DecompressStreamOptions.default_window_size_max]u8 = undefined;
219 const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
220 defer std.testing.allocator.free(window_buffer);
221
223222 var in_stream = std.io.fixedBufferStream(data);
224 var zstd_stream = decompressStream(in_stream.reader(), &window_buffer);
223 var zstd_stream = decompressStream(in_stream.reader(), window_buffer, .{});
225224 const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
226225 return result;
227226}
......@@ -251,7 +250,7 @@ test "zstandard decompression" {
251250
252251test "zstandard streaming decompression" {
253252 // default stack size for wasm32 is too low for DecompressStream - slightly
254 // over 9MiB stack space is needed via the --stack CLI flag
253 // over 1MiB stack space is needed via the --stack CLI flag
255254 if (@import("builtin").target.cpu.arch == .wasm32) return error.SkipZigTest;
256255
257256 const uncompressed = @embedFile("testdata/rfc8478.txt");
......@@ -279,9 +278,11 @@ fn expectEqualDecoded(expected: []const u8, input: []const u8) !void {
279278}
280279
281280fn expectEqualDecodedStreaming(expected: []const u8, input: []const u8) !void {
282 var window_buffer: [DecompressStreamOptions.default_window_size_max]u8 = undefined;
281 const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
282 defer std.testing.allocator.free(window_buffer);
283
283284 var in_stream = std.io.fixedBufferStream(input);
284 var stream = decompressStream(in_stream.reader(), &window_buffer);
285 var stream = decompressStream(in_stream.reader(), window_buffer, .{});
285286
286287 const result = try stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
287288 defer std.testing.allocator.free(result);
......@@ -307,7 +308,7 @@ test "zero sized block" {
307308
308309test "zero sized block streaming" {
309310 // default stack size for wasm32 is too low for DecompressStream - slightly
310 // over 9MiB stack space is needed via the --stack CLI flag
311 // over 1MiB stack space is needed via the --stack CLI flag
311312 if (@import("builtin").target.cpu.arch == .wasm32) return error.SkipZigTest;
312313
313314 const input_raw =