authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-18 15:05:39-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-23 02:37:11-07:00
logc44a90283699246172a77429a04288b19d38f075
treeb7bc00c264bcf9828325d1af3f22375352851da9
parent5c12783094ef6c36e2d26d58116fdb992306d044

fix zstd compilation errors from previous commit

2 files changed, 39 insertions(+), 64 deletions(-)

lib/std/compress/zstandard.zig+16-25
......@@ -7,13 +7,16 @@ pub const compressed_block = types.compressed_block;
77
88pub const decompress = @import("zstandard/decompress.zig");
99
10pub const DecompressStreamOptions = struct {
10pub const DecompressorOptions = struct {
1111 verify_checksum: bool = true,
12 window_buffer: []u8,
13
14 /// Recommended amount by the standard. Lower than this may result
15 /// in inability to decompress common streams.
16 pub const default_window_buffer_len = 8 * 1024 * 1024;
1217};
1318
14pub fn DecompressStream(
15 comptime ReaderType: type,
16) type {
19pub fn Decompressor(comptime ReaderType: type) type {
1720 return struct {
1821 const Self = @This();
1922
......@@ -49,13 +52,13 @@ pub fn DecompressStream(
4952
5053 pub const Reader = std.io.Reader(*Self, Error, read);
5154
52 pub fn init(source: ReaderType, window_buffer: []u8, options: DecompressStreamOptions) Self {
53 return Self{
55 pub fn init(source: ReaderType, options: DecompressorOptions) Self {
56 return .{
5457 .source = std.io.countingReader(source),
5558 .state = .NewFrame,
5659 .decode_state = undefined,
5760 .frame_context = undefined,
58 .buffer = .{ .data = window_buffer },
61 .buffer = .{ .data = options.window_buffer },
5962 .literal_fse_buffer = undefined,
6063 .match_fse_buffer = undefined,
6164 .offset_fse_buffer = undefined,
......@@ -199,20 +202,8 @@ pub fn DecompressStream(
199202 };
200203}
201204
202pub fn decompressStreamOptions(
203 reader: anytype,
204 comptime options: DecompressStreamOptions,
205 window_buffer: *[options.window_size_max]u8,
206) DecompressStream(@TypeOf(reader), options) {
207 return DecompressStream(@TypeOf(reader), options).init(reader, window_buffer);
208}
209
210pub fn decompressStream(
211 reader: anytype,
212 window_buffer: []u8,
213 options: DecompressStreamOptions,
214) DecompressStream(@TypeOf(reader)) {
215 return DecompressStream(@TypeOf(reader)).init(reader, window_buffer, options);
205pub fn decompressor(reader: anytype, options: DecompressorOptions) Decompressor(@TypeOf(reader)) {
206 return Decompressor(@TypeOf(reader)).init(reader, options);
216207}
217208
218209fn testDecompress(data: []const u8) ![]u8 {
......@@ -220,7 +211,7 @@ fn testDecompress(data: []const u8) ![]u8 {
220211 defer std.testing.allocator.free(window_buffer);
221212
222213 var in_stream = std.io.fixedBufferStream(data);
223 var zstd_stream = decompressStream(in_stream.reader(), window_buffer, .{});
214 var zstd_stream = decompressor(in_stream.reader(), .{ .window_buffer = window_buffer });
224215 const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
225216 return result;
226217}
......@@ -249,7 +240,7 @@ test "zstandard decompression" {
249240}
250241
251242test "zstandard streaming decompression" {
252 // default stack size for wasm32 is too low for DecompressStream - slightly
243 // default stack size for wasm32 is too low for Decompressor - slightly
253244 // over 1MiB stack space is needed via the --stack CLI flag
254245 if (@import("builtin").target.cpu.arch == .wasm32) return error.SkipZigTest;
255246
......@@ -282,7 +273,7 @@ fn expectEqualDecodedStreaming(expected: []const u8, input: []const u8) !void {
282273 defer std.testing.allocator.free(window_buffer);
283274
284275 var in_stream = std.io.fixedBufferStream(input);
285 var stream = decompressStream(in_stream.reader(), window_buffer, .{});
276 var stream = decompressor(in_stream.reader(), .{ .window_buffer = window_buffer });
286277
287278 const result = try stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
288279 defer std.testing.allocator.free(result);
......@@ -307,7 +298,7 @@ test "zero sized block" {
307298}
308299
309300test "zero sized block streaming" {
310 // default stack size for wasm32 is too low for DecompressStream - slightly
301 // default stack size for wasm32 is too low for Decompressor - slightly
311302 // over 1MiB stack space is needed via the --stack CLI flag
312303 if (@import("builtin").target.cpu.arch == .wasm32) return error.SkipZigTest;
313304
src/Package/Fetch.zig+23-39
......@@ -1109,8 +1109,29 @@ fn unpackResource(
11091109 var dcp = std.compress.gzip.decompressor(br.reader());
11101110 try unpackTarball(f, tmp_directory.handle, dcp.reader());
11111111 },
1112 .@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz),
1113 .@"tar.zst" => try unpackTarballCompressed(f, tmp_directory.handle, resource, ZstdWrapper),
1112 .@"tar.xz" => {
1113 const gpa = f.arena.child_allocator;
1114 const reader = resource.reader();
1115 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1116 var dcp = std.compress.xz.decompress(gpa, br.reader()) catch |err| {
1117 return f.fail(f.location_tok, try eb.printString(
1118 "unable to decompress tarball: {s}",
1119 .{@errorName(err)},
1120 ));
1121 };
1122 defer dcp.deinit();
1123 try unpackTarball(f, tmp_directory.handle, dcp.reader());
1124 },
1125 .@"tar.zst" => {
1126 const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len;
1127 const window_buffer = try f.arena.allocator().create([window_size]u8);
1128 const reader = resource.reader();
1129 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1130 var dcp = std.compress.zstd.decompressor(br.reader(), .{
1131 .window_buffer = window_buffer,
1132 });
1133 return unpackTarball(f, tmp_directory.handle, dcp.reader());
1134 },
11141135 .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) {
11151136 error.FetchFailed => return error.FetchFailed,
11161137 error.OutOfMemory => return error.OutOfMemory,
......@@ -1122,43 +1143,6 @@ fn unpackResource(
11221143 }
11231144}
11241145
1125// due to slight differences in the API of std.compress.(gzip|xz) and std.compress.zstd, zstd is
1126// wrapped for generic use in unpackTarballCompressed: see github.com/ziglang/zig/issues/14739
1127const ZstdWrapper = struct {
1128 fn DecompressType(comptime T: type) type {
1129 return Allocator.Error!std.compress.zstd.DecompressStream(T, .{});
1130 }
1131
1132 fn decompress(allocator: Allocator, reader: anytype) DecompressType(@TypeOf(reader)) {
1133 const window_size = std.compress.zstd.DecompressStreamOptions.default_window_size_max;
1134 const window_buffer = try allocator.create([window_size]u8);
1135 defer allocator.destroy(window_buffer);
1136 return std.compress.zstd.decompressStream(reader, window_buffer);
1137 }
1138};
1139
1140fn unpackTarballCompressed(
1141 f: *Fetch,
1142 out_dir: fs.Dir,
1143 resource: *Resource,
1144 Compression: anytype,
1145) RunError!void {
1146 const gpa = f.arena.child_allocator;
1147 const eb = &f.error_bundle;
1148 const reader = resource.reader();
1149 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1150
1151 var decompress = Compression.decompress(gpa, br.reader()) catch |err| {
1152 return f.fail(f.location_tok, try eb.printString(
1153 "unable to decompress tarball: {s}",
1154 .{@errorName(err)},
1155 ));
1156 };
1157 defer if (@hasDecl(Compression, "deinit")) decompress.deinit();
1158
1159 return unpackTarball(f, out_dir, decompress.reader());
1160}
1161
11621146fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
11631147 const eb = &f.error_bundle;
11641148 const gpa = f.arena.child_allocator;