| author |  Andrew Kelley <andrew@ziglang.org> 2023-01-24 15:09:02-07:00 |
| committer | Andrew Kelley <andrew@ziglang.org> 2023-01-24 15:24:19-07:00 |
| log | d94613c1d06fbeaf0cee88a04842cda64a10c8f9 |
| tree | 9a6350759af9a054a0282b344b36b29ff436a01f |
| parent | ea9ded87582a8b9d0ed3afd3360a1d75f0359a5c |
support xz compressed tarballs in the package manager
This includes a breaking change:
std.compress.gzip.GzipStream renamed to
std.compress.gzip.Decompress
This follows the same naming convention as std.compress.xz so that the
stream type can be passed as a comptime parameter.2 files changed, 26 insertions(+), 16 deletions(-)
lib- std
- compress
- src
lib/std/compress/gzip.zig+5-8
| ... | @@ -1,7 +1,7 @@ | ... | @@ -1,7 +1,7 @@ |
| 1 | // | 1 | // |
| 2 | // Decompressor for GZIP data streams (RFC1952) | 2 | // Decompressor for GZIP data streams (RFC1952) |
| 3 | 3 | ||
| 4 | const std = @import("std"); | 4 | const std = @import("../std.zig"); |
| 5 | const io = std.io; | 5 | const io = std.io; |
| 6 | const fs = std.fs; | 6 | const fs = std.fs; |
| 7 | const testing = std.testing; | 7 | const testing = std.testing; |
| ... | @@ -17,10 +17,7 @@ const FCOMMENT = 1 << 4; | ... | @@ -17,10 +17,7 @@ const FCOMMENT = 1 << 4; |
| 17 | 17 | ||
| 18 | const max_string_len = 1024; | 18 | const max_string_len = 1024; |
| 19 | 19 | ||
| 20 | /// TODO: the fully qualified namespace to this declaration is | 20 | pub fn Decompress(comptime ReaderType: type) type { |
| 21 | /// std.compress.gzip.GzipStream which has a redundant "gzip" in the name. | ||
| 22 | /// Instead, it should be `std.compress.gzip.Stream`. | ||
| 23 | pub fn GzipStream(comptime ReaderType: type) type { | ||
| 24 | return struct { | 21 | return struct { |
| 25 | const Self = @This(); | 22 | const Self = @This(); |
| 26 | 23 | ||
| ... | @@ -154,14 +151,14 @@ pub fn GzipStream(comptime ReaderType: type) type { | ... | @@ -154,14 +151,14 @@ pub fn GzipStream(comptime ReaderType: type) type { |
| 154 | }; | 151 | }; |
| 155 | } | 152 | } |
| 156 | 153 | ||
| 157 | pub fn gzipStream(allocator: mem.Allocator, reader: anytype) !GzipStream(@TypeOf(reader)) { | 154 | pub fn decompress(allocator: mem.Allocator, reader: anytype) !Decompress(@TypeOf(reader)) { |
| 158 | return GzipStream(@TypeOf(reader)).init(allocator, reader); | 155 | return Decompress(@TypeOf(reader)).init(allocator, reader); |
| 159 | } | 156 | } |
| 160 | 157 | ||
| 161 | fn testReader(data: []const u8, comptime expected: []const u8) !void { | 158 | fn testReader(data: []const u8, comptime expected: []const u8) !void { |
| 162 | var in_stream = io.fixedBufferStream(data); | 159 | var in_stream = io.fixedBufferStream(data); |
| 163 | 160 | ||
| 164 | var gzip_stream = try gzipStream(testing.allocator, in_stream.reader()); | 161 | var gzip_stream = try decompress(testing.allocator, in_stream.reader()); |
| 165 | defer gzip_stream.deinit(); | 162 | defer gzip_stream.deinit(); |
| 166 | 163 | ||
| 167 | // Read and decompress the whole file | 164 | // Read and decompress the whole file |
src/Package.zig+21-8
| ... | @@ -370,14 +370,11 @@ fn fetchAndUnpack( | ... | @@ -370,14 +370,11 @@ fn fetchAndUnpack( |
| 370 | if (mem.endsWith(u8, uri.path, ".tar.gz")) { | 370 | if (mem.endsWith(u8, uri.path, ".tar.gz")) { |
| 371 | // I observed the gzip stream to read 1 byte at a time, so I am using a | 371 | // I observed the gzip stream to read 1 byte at a time, so I am using a |
| 372 | // buffered reader on the front of it. | 372 | // buffered reader on the front of it. |
| 373 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader()); | 373 | try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.gzip); |
| 374 | 374 | } else if (mem.endsWith(u8, uri.path, ".tar.xz")) { | |
| 375 | var gzip_stream = try std.compress.gzip.gzipStream(gpa, br.reader()); | 375 | // I have not checked what buffer sizes the xz decompression implementation uses |
| 376 | defer gzip_stream.deinit(); | 376 | // by default, so the same logic applies for buffering the reader as for gzip. |
| 377 | 377 | try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.xz); | |
| 378 | try std.tar.pipeToFileSystem(tmp_directory.handle, gzip_stream.reader(), .{ | ||
| 379 | .strip_components = 1, | ||
| 380 | }); | ||
| 381 | } else { | 378 | } else { |
| 382 | return reportError( | 379 | return reportError( |
| 383 | ini, | 380 | ini, |
| ... | @@ -430,6 +427,22 @@ fn fetchAndUnpack( | ... | @@ -430,6 +427,22 @@ fn fetchAndUnpack( |
| 430 | return createWithDir(gpa, fqn, global_cache_directory, pkg_dir_sub_path, build_zig_basename); | 427 | return createWithDir(gpa, fqn, global_cache_directory, pkg_dir_sub_path, build_zig_basename); |
| 431 | } | 428 | } |
| 432 | 429 | ||
| 430 | fn unpackTarball( | ||
| 431 | gpa: Allocator, | ||
| 432 | req: *std.http.Client.Request, | ||
| 433 | out_dir: fs.Dir, | ||
| 434 | comptime compression: type, | ||
| 435 | ) !void { | ||
| 436 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader()); | ||
| 437 | |||
| 438 | var decompress = try compression.decompress(gpa, br.reader()); | ||
| 439 | defer decompress.deinit(); | ||
| 440 | |||
| 441 | try std.tar.pipeToFileSystem(out_dir, decompress.reader(), .{ | ||
| 442 | .strip_components = 1, | ||
| 443 | }); | ||
| 444 | } | ||
| 445 | |||
| 433 | fn reportError( | 446 | fn reportError( |
| 434 | ini: std.Ini, | 447 | ini: std.Ini, |
| 435 | comp_directory: Compilation.Directory, | 448 | comp_directory: Compilation.Directory, |