authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-02-21 14:31:02+11:00
committergravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-02-21 14:53:02+11:00
logc6ef83efe54bd19d7cd5dfadff23678d6af10de9
treef505afd84ad546bc8858877601da95649950dedd
parentc7c35bf9e61035ead3098d109dc894b285373622

std.compress.zstandard: clean up streaming API

1 files changed, 24 insertions(+), 13 deletions(-)

lib/std/compress/zstandard.zig+24-13
...@@ -8,10 +8,14 @@ pub const compressed_block = types.compressed_block;...@@ -8,10 +8,14 @@ pub const compressed_block = types.compressed_block;
88
9pub const decompress = @import("zstandard/decompress.zig");9pub const decompress = @import("zstandard/decompress.zig");
1010
11pub const DecompressStreamOptions = struct {
12 verify_checksum: bool = true,
13 window_size_max: usize = 1 << 23, // 8MiB default maximum window size,
14};
15
11pub fn DecompressStream(16pub fn DecompressStream(
12 comptime ReaderType: type,17 comptime ReaderType: type,
13 comptime verify_checksum: bool,18 comptime options: DecompressStreamOptions,
14 comptime window_size_max: usize,
15) type {19) type {
16 return struct {20 return struct {
17 const Self = @This();21 const Self = @This();
...@@ -27,7 +31,7 @@ pub fn DecompressStream(...@@ -27,7 +31,7 @@ pub fn DecompressStream(
27 offset_fse_buffer: []types.compressed_block.Table.Fse,31 offset_fse_buffer: []types.compressed_block.Table.Fse,
28 literals_buffer: []u8,32 literals_buffer: []u8,
29 sequence_buffer: []u8,33 sequence_buffer: []u8,
30 checksum: if (verify_checksum) ?u32 else void,34 checksum: if (options.verify_checksum) ?u32 else void,
31 current_frame_decompressed_size: usize,35 current_frame_decompressed_size: usize,
3236
33 pub const Error = ReaderType.Error || error{37 pub const Error = ReaderType.Error || error{
...@@ -69,8 +73,8 @@ pub fn DecompressStream(...@@ -69,8 +73,8 @@ pub fn DecompressStream(
69 const frame_context = context: {73 const frame_context = context: {
70 break :context try decompress.FrameContext.init(74 break :context try decompress.FrameContext.init(
71 header,75 header,
72 window_size_max,76 options.window_size_max,
73 verify_checksum,77 options.verify_checksum,
74 );78 );
75 };79 };
7680
...@@ -99,10 +103,10 @@ pub fn DecompressStream(...@@ -99,10 +103,10 @@ pub fn DecompressStream(
99 );103 );
100 const buffer = try RingBuffer.init(self.allocator, frame_context.window_size);104 const buffer = try RingBuffer.init(self.allocator, frame_context.window_size);
101105
102 const literals_data = try self.allocator.alloc(u8, window_size_max);106 const literals_data = try self.allocator.alloc(u8, options.window_size_max);
103 errdefer self.allocator.free(literals_data);107 errdefer self.allocator.free(literals_data);
104108
105 const sequence_data = try self.allocator.alloc(u8, window_size_max);109 const sequence_data = try self.allocator.alloc(u8, options.window_size_max);
106 errdefer self.allocator.free(sequence_data);110 errdefer self.allocator.free(sequence_data);
107111
108 self.literal_fse_buffer = literal_fse_buffer;112 self.literal_fse_buffer = literal_fse_buffer;
...@@ -116,7 +120,7 @@ pub fn DecompressStream(...@@ -116,7 +120,7 @@ pub fn DecompressStream(
116 self.decode_state = decode_state;120 self.decode_state = decode_state;
117 self.frame_context = frame_context;121 self.frame_context = frame_context;
118122
119 self.checksum = if (verify_checksum) null else {};123 self.checksum = if (options.verify_checksum) null else {};
120 self.current_frame_decompressed_size = 0;124 self.current_frame_decompressed_size = 0;
121125
122 self.state = .InFrame;126 self.state = .InFrame;
...@@ -199,7 +203,7 @@ pub fn DecompressStream(...@@ -199,7 +203,7 @@ pub fn DecompressStream(
199 if (self.frame_context.has_checksum) {203 if (self.frame_context.has_checksum) {
200 const checksum = source_reader.readIntLittle(u32) catch204 const checksum = source_reader.readIntLittle(u32) catch
201 return error.MalformedFrame;205 return error.MalformedFrame;
202 if (comptime verify_checksum) {206 if (comptime options.verify_checksum) {
203 if (self.frame_context.hasher_opt) |*hasher| {207 if (self.frame_context.hasher_opt) |*hasher| {
204 if (checksum != decompress.computeChecksum(hasher))208 if (checksum != decompress.computeChecksum(hasher))
205 return error.ChecksumFailure;209 return error.ChecksumFailure;
...@@ -232,17 +236,24 @@ pub fn DecompressStream(...@@ -232,17 +236,24 @@ pub fn DecompressStream(
232 };236 };
233}237}
234238
239pub fn decompressStreamOptions(
240 allocator: Allocator,
241 reader: anytype,
242 comptime options: DecompressStreamOptions,
243) DecompressStream(@TypeOf(reader, options)) {
244 return DecompressStream(@TypeOf(reader), options).init(allocator, reader);
245}
246
235pub fn decompressStream(247pub fn decompressStream(
236 allocator: Allocator,248 allocator: Allocator,
237 reader: anytype,249 reader: anytype,
238 comptime window_size_max: usize,250) DecompressStream(@TypeOf(reader), .{}) {
239) DecompressStream(@TypeOf(reader), true, window_size_max) {251 return DecompressStream(@TypeOf(reader), .{}).init(allocator, reader);
240 return DecompressStream(@TypeOf(reader), true, 8 * (1 << 20)).init(allocator, reader);
241}252}
242253
243fn testDecompress(data: []const u8) ![]u8 {254fn testDecompress(data: []const u8) ![]u8 {
244 var in_stream = std.io.fixedBufferStream(data);255 var in_stream = std.io.fixedBufferStream(data);
245 var zstd_stream = decompressStream(std.testing.allocator, in_stream.reader(), 1 << 23);256 var zstd_stream = decompressStream(std.testing.allocator, in_stream.reader());
246 defer zstd_stream.deinit();257 defer zstd_stream.deinit();
247 const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));258 const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
248 return result;259 return result;