authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2024-02-18 13:21:48+11:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-23 02:37:11-07:00
log5c12783094ef6c36e2d26d58116fdb992306d044
tree07239df72ae8397ddce43485231d86e1916221c9
parent63acc856c12fec71d96c5cbce9138204f36d6568

std.compress.zstd: make DecompressStream options runtime

1 files changed, 23 insertions(+), 22 deletions(-)

lib/std/compress/zstandard.zig+23-22
...@@ -9,20 +9,14 @@ pub const decompress = @import("zstandard/decompress.zig");...@@ -9,20 +9,14 @@ pub const decompress = @import("zstandard/decompress.zig");
99
10pub const DecompressStreamOptions = struct {10pub const DecompressStreamOptions = struct {
11 verify_checksum: bool = true,11 verify_checksum: bool = true,
12 window_size_max: usize = default_window_size_max,
13
14 pub const default_window_size_max = 1 << 23; // 8MiB default maximum window size
15};12};
1613
17pub fn DecompressStream(14pub fn DecompressStream(
18 comptime ReaderType: type,15 comptime ReaderType: type,
19 comptime options: DecompressStreamOptions,
20) type {16) type {
21 return struct {17 return struct {
22 const Self = @This();18 const Self = @This();
2319
24 pub const window_size_max = options.window_size_max;
25
26 const table_size_max = types.compressed_block.table_size_max;20 const table_size_max = types.compressed_block.table_size_max;
2721
28 source: std.io.CountingReader(ReaderType),22 source: std.io.CountingReader(ReaderType),
...@@ -35,11 +29,12 @@ pub fn DecompressStream(...@@ -35,11 +29,12 @@ pub fn DecompressStream(
35 offset_fse_buffer: [table_size_max.offset]types.compressed_block.Table.Fse,29 offset_fse_buffer: [table_size_max.offset]types.compressed_block.Table.Fse,
36 literals_buffer: [types.block_size_max]u8,30 literals_buffer: [types.block_size_max]u8,
37 sequence_buffer: [types.block_size_max]u8,31 sequence_buffer: [types.block_size_max]u8,
38 checksum: if (options.verify_checksum) ?u32 else void,32 verify_checksum: bool,
33 checksum: ?u32,
39 current_frame_decompressed_size: usize,34 current_frame_decompressed_size: usize,
4035
41 const WindowBuffer = struct {36 const WindowBuffer = struct {
42 data: *[options.window_size_max]u8 = undefined,37 data: []u8 = undefined,
43 read_index: usize = 0,38 read_index: usize = 0,
44 write_index: usize = 0,39 write_index: usize = 0,
45 };40 };
...@@ -54,7 +49,7 @@ pub fn DecompressStream(...@@ -54,7 +49,7 @@ pub fn DecompressStream(
5449
55 pub const Reader = std.io.Reader(*Self, Error, read);50 pub const Reader = std.io.Reader(*Self, Error, read);
5651
57 pub fn init(source: ReaderType, window_buffer: *[options.window_size_max]u8) Self {52 pub fn init(source: ReaderType, window_buffer: []u8, options: DecompressStreamOptions) Self {
58 return Self{53 return Self{
59 .source = std.io.countingReader(source),54 .source = std.io.countingReader(source),
60 .state = .NewFrame,55 .state = .NewFrame,
...@@ -66,6 +61,7 @@ pub fn DecompressStream(...@@ -66,6 +61,7 @@ pub fn DecompressStream(
66 .offset_fse_buffer = undefined,61 .offset_fse_buffer = undefined,
67 .literals_buffer = undefined,62 .literals_buffer = undefined,
68 .sequence_buffer = undefined,63 .sequence_buffer = undefined,
64 .verify_checksum = options.verify_checksum,
69 .checksum = undefined,65 .checksum = undefined,
70 .current_frame_decompressed_size = undefined,66 .current_frame_decompressed_size = undefined,
71 };67 };
...@@ -81,8 +77,8 @@ pub fn DecompressStream(...@@ -81,8 +77,8 @@ pub fn DecompressStream(
81 .zstandard => |header| {77 .zstandard => |header| {
82 const frame_context = try decompress.FrameContext.init(78 const frame_context = try decompress.FrameContext.init(
83 header,79 header,
84 options.window_size_max,80 self.buffer.data.len,
85 options.verify_checksum,81 self.verify_checksum,
86 );82 );
8783
88 const decode_state = decompress.block.DecodeState.init(84 const decode_state = decompress.block.DecodeState.init(
...@@ -94,7 +90,7 @@ pub fn DecompressStream(...@@ -94,7 +90,7 @@ pub fn DecompressStream(
94 self.decode_state = decode_state;90 self.decode_state = decode_state;
95 self.frame_context = frame_context;91 self.frame_context = frame_context;
9692
97 self.checksum = if (options.verify_checksum) null else {};93 self.checksum = null;
98 self.current_frame_decompressed_size = 0;94 self.current_frame_decompressed_size = 0;
9995
100 self.state = .InFrame;96 self.state = .InFrame;
...@@ -176,7 +172,7 @@ pub fn DecompressStream(...@@ -176,7 +172,7 @@ pub fn DecompressStream(
176 if (self.frame_context.has_checksum) {172 if (self.frame_context.has_checksum) {
177 const checksum = source_reader.readInt(u32, .little) catch173 const checksum = source_reader.readInt(u32, .little) catch
178 return error.MalformedFrame;174 return error.MalformedFrame;
179 if (comptime options.verify_checksum) {175 if (self.verify_checksum) {
180 if (self.frame_context.hasher_opt) |*hasher| {176 if (self.frame_context.hasher_opt) |*hasher| {
181 if (checksum != decompress.computeChecksum(hasher))177 if (checksum != decompress.computeChecksum(hasher))
182 return error.ChecksumFailure;178 return error.ChecksumFailure;
...@@ -213,15 +209,18 @@ pub fn decompressStreamOptions(...@@ -213,15 +209,18 @@ pub fn decompressStreamOptions(
213209
214pub fn decompressStream(210pub fn decompressStream(
215 reader: anytype,211 reader: anytype,
216 window_buffer: *[DecompressStreamOptions.default_window_size_max]u8,212 window_buffer: []u8,
217) DecompressStream(@TypeOf(reader), .{}) {213 options: DecompressStreamOptions,
218 return DecompressStream(@TypeOf(reader), .{}).init(reader, window_buffer);214) DecompressStream(@TypeOf(reader)) {
215 return DecompressStream(@TypeOf(reader)).init(reader, window_buffer, options);
219}216}
220217
221fn testDecompress(data: []const u8) ![]u8 {218fn testDecompress(data: []const u8) ![]u8 {
222 var window_buffer: [DecompressStreamOptions.default_window_size_max]u8 = undefined;219 const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
220 defer std.testing.allocator.free(window_buffer);
221
223 var in_stream = std.io.fixedBufferStream(data);222 var in_stream = std.io.fixedBufferStream(data);
224 var zstd_stream = decompressStream(in_stream.reader(), &window_buffer);223 var zstd_stream = decompressStream(in_stream.reader(), window_buffer, .{});
225 const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));224 const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
226 return result;225 return result;
227}226}
...@@ -251,7 +250,7 @@ test "zstandard decompression" {...@@ -251,7 +250,7 @@ test "zstandard decompression" {
251250
252test "zstandard streaming decompression" {251test "zstandard streaming decompression" {
253 // default stack size for wasm32 is too low for DecompressStream - slightly252 // default stack size for wasm32 is too low for DecompressStream - slightly
254 // over 9MiB stack space is needed via the --stack CLI flag253 // over 1MiB stack space is needed via the --stack CLI flag
255 if (@import("builtin").target.cpu.arch == .wasm32) return error.SkipZigTest;254 if (@import("builtin").target.cpu.arch == .wasm32) return error.SkipZigTest;
256255
257 const uncompressed = @embedFile("testdata/rfc8478.txt");256 const uncompressed = @embedFile("testdata/rfc8478.txt");
...@@ -279,9 +278,11 @@ fn expectEqualDecoded(expected: []const u8, input: []const u8) !void {...@@ -279,9 +278,11 @@ fn expectEqualDecoded(expected: []const u8, input: []const u8) !void {
279}278}
280279
281fn expectEqualDecodedStreaming(expected: []const u8, input: []const u8) !void {280fn expectEqualDecodedStreaming(expected: []const u8, input: []const u8) !void {
282 var window_buffer: [DecompressStreamOptions.default_window_size_max]u8 = undefined;281 const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
282 defer std.testing.allocator.free(window_buffer);
283
283 var in_stream = std.io.fixedBufferStream(input);284 var in_stream = std.io.fixedBufferStream(input);
284 var stream = decompressStream(in_stream.reader(), &window_buffer);285 var stream = decompressStream(in_stream.reader(), window_buffer, .{});
285286
286 const result = try stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));287 const result = try stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
287 defer std.testing.allocator.free(result);288 defer std.testing.allocator.free(result);
...@@ -307,7 +308,7 @@ test "zero sized block" {...@@ -307,7 +308,7 @@ test "zero sized block" {
307308
308test "zero sized block streaming" {309test "zero sized block streaming" {
309 // default stack size for wasm32 is too low for DecompressStream - slightly310 // default stack size for wasm32 is too low for DecompressStream - slightly
310 // over 9MiB stack space is needed via the --stack CLI flag311 // over 1MiB stack space is needed via the --stack CLI flag
311 if (@import("builtin").target.cpu.arch == .wasm32) return error.SkipZigTest;312 if (@import("builtin").target.cpu.arch == .wasm32) return error.SkipZigTest;
312313
313 const input_raw =314 const input_raw =