Commit ee21a1f988f05a5d45bcb1724095c27cc2c7259b | zig

author	Andrew Kelley <andrew@ziglang.org> 2026-02-04 21:40:06-08:00
committer	Andrew Kelley <andrew@ziglang.org> 2026-02-05 16:50:41-08:00
log	ee21a1f988f05a5d45bcb1724095c27cc2c7259b
tree	e51dea9e94a045bac9ab2368f4c52a472872ffd9
parent	df64a3a36815fce6cc8671d047e52795655b3b9b

fetch: implement recompression

After fetching a package and applying the filter by deleting files that
are not part of the hash, creates a recompressed $GLOBAL_CACHE/p/$PKG_HASH.tar.gz

Checking this cache before fetching network URLs is not yet implemented.

3 files changed, 136 insertions(+), 14 deletions(-)

lib
- std
  - Io.zig
  - compress
    flate
    Compress.zig
src
- Package
  - Fetch.zig

lib/std/Io.zig+6

...	...	@@ -1031,6 +1031,9 @@ pub const Group = struct {
1031	1031	/// Once this function is called, there are resources associated with the
1032	1032	/// group. To release those resources, `Group.await` or `Group.cancel` must
1033	1033	/// eventually be called.
	1034	///
	1035	/// If `error.Canceled` is returned from any operation this task performs,
	1036	/// it is asserted that `function` returns `error.Canceled`.
1034	1037	pub fn async(g: *Group, io: Io, function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) void {
1035	1038	const Args = @TypeOf(args);
1036	1039	const TypeErased = struct {
...	...	@@ -1050,6 +1053,9 @@ pub const Group = struct {
1050	1053	/// Once this function is called, there are resources associated with the
1051	1054	/// group. To release those resources, `Group.await` or `Group.cancel` must
1052	1055	/// eventually be called.
	1056	///
	1057	/// If `error.Canceled` is returned from any operation this task performs,
	1058	/// it is asserted that `function` returns `error.Canceled`.
1053	1059	pub fn concurrent(g: *Group, io: Io, function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) ConcurrentError!void {
1054	1060	const Args = @TypeOf(args);
1055	1061	const TypeErased = struct {

lib/std/compress/flate/Compress.zig+1-1

...	...	@@ -267,7 +267,7 @@ pub const Options = struct {
267	267	pub const best = level_9;
268	268	};
269	269
270		/// It is asserted `buffer` is least `flate.max_history_len` bytes.
	270	/// It is asserted `buffer` is least `flate.max_window_len` bytes.
271	271	/// It is asserted `output` has a capacity of at least 8 bytes.
272	272	pub fn init(
273	273	output: *Writer,

src/Package/Fetch.zig+129-13

...	...	@@ -1,28 +1,34 @@
1	1	//! Represents one independent job whose responsibility is to:
2	2	//!
3		//! 1. Check the global zig package cache to see if the hash already exists.
	3	//! 1. Check the local zig package directory to see if the hash already exists.
4	4	//! If so, load, parse, and validate the build.zig.zon file therein, and
5		//! goto step 8. Likewise if the location is a relative path, treat this
	5	//! goto step 9. Likewise if the location is a relative path, treat this
6	6	//! the same as a cache hit. Otherwise, proceed.
7		//! 2. Fetch and unpack a URL into a temporary directory.
8		//! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed
	7	//! 2. Check the global package cache for a compressed tarball matching the
	8	//! hash. If it is found, unpack the contents into a temporary directory inside
	9	//! project local zig cache. Rename this directory into the local zig package
	10	//! directory and goto step 9, skipping step 10.
	11	//! 3. Fetch and unpack a URL into a temporary directory.
	12	//! 4. Load, parse, and validate the build.zig.zon file therein. It is allowed
9	13	//! for the file to be missing, in which case this fetched package is considered
10	14	//! to be a "naked" package.
11		//! 4. Apply inclusion rules of the build.zig.zon to the temporary directory by
	15	//! 5. Apply inclusion rules of the build.zig.zon to the temporary directory by
12	16	//! deleting excluded files. If any files had errors for files that were
13	17	//! ultimately excluded, those errors should be ignored, such as failure to
14	18	//! create symlinks that weren't supposed to be included anyway.
15		//! 5. Compute the package hash based on the remaining files in the temporary
	19	//! 6. Compute the package hash based on the remaining files in the temporary
16	20	//! directory.
17		//! 6. Rename the temporary directory into the global zig package cache
18		//! directory. If the hash already exists, delete the temporary directory and
19		//! leave the zig package cache directory untouched as it may be in use by the
20		//! system. This is done even if the hash is invalid, in case the package with
21		//! the different hash is used in the future.
22		//! 7. Validate the computed hash against the expected hash. If invalid,
	21	//! 7. Rename the temporary directory into the local zig package directory. If
	22	//! the hash already exists, delete the temporary directory and leave the zig
	23	//! package directory untouched as it may be in use. This is done even if
	24	//! the hash is invalid, in case the package with the different hash is used
	25	//! in the future.
	26	//! 8. Validate the computed hash against the expected hash. If invalid,
23	27	//! this job is done.
24		//! 8. Spawn a new fetch job for each dependency in the manifest file. Use
	28	//! 9. Spawn a new fetch job for each dependency in the manifest file. Use
25	29	//! a mutex and a hash map so that redundant jobs do not get queued up.
	30	//! 10.Compress the package directory and store it into the global package
	31	//! cache.
26	32	//!
27	33	//! All of this must be done with only referring to the state inside this struct
28	34	//! because this work will be done in a dedicated thread.
...	...	@@ -110,6 +116,7 @@ pub const JobQueue = struct {
110	116	all_fetches: std.ArrayList(*Fetch) = .empty,
111	117
112	118	http_client: *std.http.Client,
	119	/// This tracks `Fetch` tasks as well as recompression tasks.
113	120	group: Io.Group = .init,
114	121	global_cache: Cache.Directory,
115	122	local_cache: Cache.Path,
...	...	@@ -293,8 +300,109 @@ pub const JobQueue = struct {
293	300	\\
294	301	);
295	302	}
	303
	304	fn recompress(jq: *JobQueue, package_hash: Package.Hash) Io.Cancelable!void {
	305	var dest_sub_path_buffer: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined;
	306	const dest_path: Cache.Path = .{
	307	.root_dir = jq.global_cache,
	308	.sub_path = std.fmt.bufPrint(&dest_sub_path_buffer, "p/{s}.tar.gz", .{
	309	package_hash.toSlice(),
	310	}) catch unreachable,
	311	};
	312
	313	const gpa = jq.http_client.allocator;
	314
	315	var arena_instance = std.heap.ArenaAllocator.init(gpa);
	316	defer arena_instance.deinit();
	317	const arena = arena_instance.allocator();
	318
	319	recompressFallible(jq, arena, dest_path, package_hash.toSlice()) catch \|err\| switch (err) {
	320	error.Canceled => \|e\| return e,
	321	error.ReadFailed => comptime unreachable,
	322	error.WriteFailed => comptime unreachable,
	323	else => \|e\| std.log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }),
	324	};
	325	}
	326
	327	fn recompressFallible(jq: *JobQueue, arena: Allocator, dest_path: Cache.Path, package_hash: []const u8) !void {
	328	const gpa = jq.http_client.allocator;
	329	const io = jq.io;
	330
	331	// We have to walk the file system up front in order to sort the file
	332	// list for determinism purposes. The hash of the recompressed file is
	333	// not critical because the true hash is based on the content alone.
	334	// However, if we want Zig users to be able to share cached package
	335	// data with each other via peer-to-peer protocols, we benefit greatly
	336	// from the data being identical on everyone's computers.
	337	var scanned_files: std.ArrayList([]const u8) = .empty;
	338	defer scanned_files.deinit(gpa);
	339
	340	var pkg_dir = try jq.root_pkg_path.openDir(io, package_hash, .{ .iterate = true });
	341	defer pkg_dir.close(io);
	342
	343	{
	344	var walker = try pkg_dir.walk(gpa);
	345	defer walker.deinit();
	346
	347	while (try walker.next(io)) \|entry\| {
	348	switch (entry.kind) {
	349	.directory => continue,
	350	.file, .sym_link => {},
	351	else => {
	352	return error.IllegalFileType;
	353	},
	354	}
	355	const entry_path = try arena.dupe(u8, entry.path);
	356	try scanned_files.append(gpa, entry_path);
	357	}
	358
	359	std.mem.sortUnstable([]const u8, scanned_files.items, {}, stringCmp);
	360	}
	361
	362	var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{
	363	.make_path = true,
	364	.replace = true,
	365	});
	366	defer atomic_file.deinit(io);
	367
	368	var file_write_buffer: [4096]u8 = undefined;
	369	var file_writer = atomic_file.file.writer(io, &file_write_buffer);
	370
	371	var compress_buffer: [std.compress.flate.max_window_len]u8 = undefined;
	372	var compress = std.compress.flate.Compress.init(&file_writer.interface, &compress_buffer, .gzip, .level_9) catch \|err\| switch (err) {
	373	error.WriteFailed => return file_writer.err.?,
	374	};
	375
	376	var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer };
	377	archiver.prefix = package_hash;
	378
	379	var file_read_buffer: [4096]u8 = undefined;
	380
	381	for (scanned_files.items) \|entry_path\| {
	382	var file = try pkg_dir.openFile(io, entry_path, .{});
	383	defer file.close(io);
	384	var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer);
	385	archiver.writeFile(entry_path, &file_reader, 0) catch \|err\| switch (err) {
	386	error.ReadFailed => return file_reader.err.?,
	387	error.WriteFailed => return file_writer.err.?,
	388	else => \|e\| return e,
	389	};
	390	}
	391
	392	// intentionally omitting the pointless trailer
	393	//try archiver.finish();
	394	compress.writer.flush() catch \|err\| switch (err) {
	395	error.WriteFailed => return file_writer.err.?,
	396	};
	397	try file_writer.flush();
	398	try atomic_file.replace(io);
	399	}
296	400	};
297	401
	402	fn stringCmp(_: void, lhs: []const u8, rhs: []const u8) bool {
	403	return std.mem.lessThan(u8, lhs, rhs);
	404	}
	405
298	406	pub const Location = union(enum) {
299	407	remote: Remote,
300	408	/// A directory found inside the parent package.
...	...	@@ -477,8 +585,11 @@ fn runResource(
477	585	remote_hash: ?Package.Hash,
478	586	) RunError!void {
479	587	const job_queue = f.job_queue;
	588	assert(!job_queue.read_only);
	589
480	590	const io = job_queue.io;
481	591	defer resource.deinit(io);
	592
482	593	const arena = f.arena.allocator();
483	594	const eb = &f.error_bundle;
484	595	const s = fs.path.sep_str;
...	...	@@ -556,6 +667,11 @@ fn runResource(
556	667	) });
557	668	return error.FetchFailed;
558	669	};
	670
	671	// Spin off a task to recompress the tarball, with filtered files deleted, into
	672	// the global cache.
	673	job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash });
	674
559	675	// Remove temporary directory root if not already renamed to global cache.
560	676	if (!package_sub_path.eql(tmp_directory_path)) {
561	677	tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch \|err\| switch (err) {