Commit d94613c1d06fbeaf0cee88a04842cda64a10c8f9 | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-01-24 15:09:02-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2023-01-24 15:24:19-07:00
log	d94613c1d06fbeaf0cee88a04842cda64a10c8f9
tree	9a6350759af9a054a0282b344b36b29ff436a01f
parent	ea9ded87582a8b9d0ed3afd3360a1d75f0359a5c

support xz compressed tarballs in the package manager

This includes a breaking change:

std.compress.gzip.GzipStream renamed to
std.compress.gzip.Decompress

This follows the same naming convention as std.compress.xz so that the
stream type can be passed as a comptime parameter.

2 files changed, 26 insertions(+), 16 deletions(-)

lib
- std
  - compress
    gzip.zig
src
- Package.zig

lib/std/compress/gzip.zig+5-8

...	...	@@ -1,7 +1,7 @@
1	1	//
2	2	// Decompressor for GZIP data streams (RFC1952)
3	3
4		const std = @import("std");
	4	const std = @import("../std.zig");
5	5	const io = std.io;
6	6	const fs = std.fs;
7	7	const testing = std.testing;
...	...	@@ -17,10 +17,7 @@ const FCOMMENT = 1 << 4;
17	17
18	18	const max_string_len = 1024;
19	19
20		/// TODO: the fully qualified namespace to this declaration is
21		/// std.compress.gzip.GzipStream which has a redundant "gzip" in the name.
22		/// Instead, it should be `std.compress.gzip.Stream`.
23		pub fn GzipStream(comptime ReaderType: type) type {
	20	pub fn Decompress(comptime ReaderType: type) type {
24	21	return struct {
25	22	const Self = @This();
26	23
...	...	@@ -154,14 +151,14 @@ pub fn GzipStream(comptime ReaderType: type) type {
154	151	};
155	152	}
156	153
157		pub fn gzipStream(allocator: mem.Allocator, reader: anytype) !GzipStream(@TypeOf(reader)) {
158		return GzipStream(@TypeOf(reader)).init(allocator, reader);
	154	pub fn decompress(allocator: mem.Allocator, reader: anytype) !Decompress(@TypeOf(reader)) {
	155	return Decompress(@TypeOf(reader)).init(allocator, reader);
159	156	}
160	157
161	158	fn testReader(data: []const u8, comptime expected: []const u8) !void {
162	159	var in_stream = io.fixedBufferStream(data);
163	160
164		var gzip_stream = try gzipStream(testing.allocator, in_stream.reader());
	161	var gzip_stream = try decompress(testing.allocator, in_stream.reader());
165	162	defer gzip_stream.deinit();
166	163
167	164	// Read and decompress the whole file

src/Package.zig+21-8

...	...	@@ -370,14 +370,11 @@ fn fetchAndUnpack(
370	370	if (mem.endsWith(u8, uri.path, ".tar.gz")) {
371	371	// I observed the gzip stream to read 1 byte at a time, so I am using a
372	372	// buffered reader on the front of it.
373		var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader());
374
375		var gzip_stream = try std.compress.gzip.gzipStream(gpa, br.reader());
376		defer gzip_stream.deinit();
377
378		try std.tar.pipeToFileSystem(tmp_directory.handle, gzip_stream.reader(), .{
379		.strip_components = 1,
380		});
	373	try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.gzip);
	374	} else if (mem.endsWith(u8, uri.path, ".tar.xz")) {
	375	// I have not checked what buffer sizes the xz decompression implementation uses
	376	// by default, so the same logic applies for buffering the reader as for gzip.
	377	try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.xz);
381	378	} else {
382	379	return reportError(
383	380	ini,
...	...	@@ -430,6 +427,22 @@ fn fetchAndUnpack(
430	427	return createWithDir(gpa, fqn, global_cache_directory, pkg_dir_sub_path, build_zig_basename);
431	428	}
432	429
	430	fn unpackTarball(
	431	gpa: Allocator,
	432	req: *std.http.Client.Request,
	433	out_dir: fs.Dir,
	434	comptime compression: type,
	435	) !void {
	436	var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader());
	437
	438	var decompress = try compression.decompress(gpa, br.reader());
	439	defer decompress.deinit();
	440
	441	try std.tar.pipeToFileSystem(out_dir, decompress.reader(), .{
	442	.strip_components = 1,
	443	});
	444	}
	445
433	446	fn reportError(
434	447	ini: std.Ini,
435	448	comp_directory: Compilation.Directory,