authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-04 21:40:06-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-05 16:50:41-08:00
logee21a1f988f05a5d45bcb1724095c27cc2c7259b
treee51dea9e94a045bac9ab2368f4c52a472872ffd9
parentdf64a3a36815fce6cc8671d047e52795655b3b9b

fetch: implement recompression

After fetching a package and applying the filter by deleting files that are not part of the hash, creates a recompressed $GLOBAL_CACHE/p/$PKG_HASH.tar.gz Checking this cache before fetching network URLs is not yet implemented.

3 files changed, 136 insertions(+), 14 deletions(-)

lib/std/Io.zig+6
...@@ -1031,6 +1031,9 @@ pub const Group = struct {...@@ -1031,6 +1031,9 @@ pub const Group = struct {
1031 /// Once this function is called, there are resources associated with the1031 /// Once this function is called, there are resources associated with the
1032 /// group. To release those resources, `Group.await` or `Group.cancel` must1032 /// group. To release those resources, `Group.await` or `Group.cancel` must
1033 /// eventually be called.1033 /// eventually be called.
1034 ///
1035 /// If `error.Canceled` is returned from any operation this task performs,
1036 /// it is asserted that `function` returns `error.Canceled`.
1034 pub fn async(g: *Group, io: Io, function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) void {1037 pub fn async(g: *Group, io: Io, function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) void {
1035 const Args = @TypeOf(args);1038 const Args = @TypeOf(args);
1036 const TypeErased = struct {1039 const TypeErased = struct {
...@@ -1050,6 +1053,9 @@ pub const Group = struct {...@@ -1050,6 +1053,9 @@ pub const Group = struct {
1050 /// Once this function is called, there are resources associated with the1053 /// Once this function is called, there are resources associated with the
1051 /// group. To release those resources, `Group.await` or `Group.cancel` must1054 /// group. To release those resources, `Group.await` or `Group.cancel` must
1052 /// eventually be called.1055 /// eventually be called.
1056 ///
1057 /// If `error.Canceled` is returned from any operation this task performs,
1058 /// it is asserted that `function` returns `error.Canceled`.
1053 pub fn concurrent(g: *Group, io: Io, function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) ConcurrentError!void {1059 pub fn concurrent(g: *Group, io: Io, function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) ConcurrentError!void {
1054 const Args = @TypeOf(args);1060 const Args = @TypeOf(args);
1055 const TypeErased = struct {1061 const TypeErased = struct {
lib/std/compress/flate/Compress.zig+1-1
...@@ -267,7 +267,7 @@ pub const Options = struct {...@@ -267,7 +267,7 @@ pub const Options = struct {
267 pub const best = level_9;267 pub const best = level_9;
268};268};
269269
270/// It is asserted `buffer` is least `flate.max_history_len` bytes.270/// It is asserted `buffer` is least `flate.max_window_len` bytes.
271/// It is asserted `output` has a capacity of at least 8 bytes.271/// It is asserted `output` has a capacity of at least 8 bytes.
272pub fn init(272pub fn init(
273 output: *Writer,273 output: *Writer,
src/Package/Fetch.zig+129-13
...@@ -1,28 +1,34 @@...@@ -1,28 +1,34 @@
1//! Represents one independent job whose responsibility is to:1//! Represents one independent job whose responsibility is to:
2//!2//!
3//! 1. Check the global zig package cache to see if the hash already exists.3//! 1. Check the local zig package directory to see if the hash already exists.
4//! If so, load, parse, and validate the build.zig.zon file therein, and4//! If so, load, parse, and validate the build.zig.zon file therein, and
5//! goto step 8. Likewise if the location is a relative path, treat this5//! goto step 9. Likewise if the location is a relative path, treat this
6//! the same as a cache hit. Otherwise, proceed.6//! the same as a cache hit. Otherwise, proceed.
7//! 2. Fetch and unpack a URL into a temporary directory.7//! 2. Check the global package cache for a compressed tarball matching the
8//! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed8//! hash. If it is found, unpack the contents into a temporary directory inside
9//! project local zig cache. Rename this directory into the local zig package
10//! directory and goto step 9, skipping step 10.
11//! 3. Fetch and unpack a URL into a temporary directory.
12//! 4. Load, parse, and validate the build.zig.zon file therein. It is allowed
9//! for the file to be missing, in which case this fetched package is considered13//! for the file to be missing, in which case this fetched package is considered
10//! to be a "naked" package.14//! to be a "naked" package.
11//! 4. Apply inclusion rules of the build.zig.zon to the temporary directory by15//! 5. Apply inclusion rules of the build.zig.zon to the temporary directory by
12//! deleting excluded files. If any files had errors for files that were16//! deleting excluded files. If any files had errors for files that were
13//! ultimately excluded, those errors should be ignored, such as failure to17//! ultimately excluded, those errors should be ignored, such as failure to
14//! create symlinks that weren't supposed to be included anyway.18//! create symlinks that weren't supposed to be included anyway.
15//! 5. Compute the package hash based on the remaining files in the temporary19//! 6. Compute the package hash based on the remaining files in the temporary
16//! directory.20//! directory.
17//! 6. Rename the temporary directory into the global zig package cache21//! 7. Rename the temporary directory into the local zig package directory. If
18//! directory. If the hash already exists, delete the temporary directory and22//! the hash already exists, delete the temporary directory and leave the zig
19//! leave the zig package cache directory untouched as it may be in use by the23//! package directory untouched as it may be in use. This is done even if
20//! system. This is done even if the hash is invalid, in case the package with24//! the hash is invalid, in case the package with the different hash is used
21//! the different hash is used in the future.25//! in the future.
22//! 7. Validate the computed hash against the expected hash. If invalid,26//! 8. Validate the computed hash against the expected hash. If invalid,
23//! this job is done.27//! this job is done.
24//! 8. Spawn a new fetch job for each dependency in the manifest file. Use28//! 9. Spawn a new fetch job for each dependency in the manifest file. Use
25//! a mutex and a hash map so that redundant jobs do not get queued up.29//! a mutex and a hash map so that redundant jobs do not get queued up.
30//! 10.Compress the package directory and store it into the global package
31//! cache.
26//!32//!
27//! All of this must be done with only referring to the state inside this struct33//! All of this must be done with only referring to the state inside this struct
28//! because this work will be done in a dedicated thread.34//! because this work will be done in a dedicated thread.
...@@ -110,6 +116,7 @@ pub const JobQueue = struct {...@@ -110,6 +116,7 @@ pub const JobQueue = struct {
110 all_fetches: std.ArrayList(*Fetch) = .empty,116 all_fetches: std.ArrayList(*Fetch) = .empty,
111117
112 http_client: *std.http.Client,118 http_client: *std.http.Client,
119 /// This tracks `Fetch` tasks as well as recompression tasks.
113 group: Io.Group = .init,120 group: Io.Group = .init,
114 global_cache: Cache.Directory,121 global_cache: Cache.Directory,
115 local_cache: Cache.Path,122 local_cache: Cache.Path,
...@@ -293,8 +300,109 @@ pub const JobQueue = struct {...@@ -293,8 +300,109 @@ pub const JobQueue = struct {
293 \\300 \\
294 );301 );
295 }302 }
303
304 fn recompress(jq: *JobQueue, package_hash: Package.Hash) Io.Cancelable!void {
305 var dest_sub_path_buffer: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined;
306 const dest_path: Cache.Path = .{
307 .root_dir = jq.global_cache,
308 .sub_path = std.fmt.bufPrint(&dest_sub_path_buffer, "p/{s}.tar.gz", .{
309 package_hash.toSlice(),
310 }) catch unreachable,
311 };
312
313 const gpa = jq.http_client.allocator;
314
315 var arena_instance = std.heap.ArenaAllocator.init(gpa);
316 defer arena_instance.deinit();
317 const arena = arena_instance.allocator();
318
319 recompressFallible(jq, arena, dest_path, package_hash.toSlice()) catch |err| switch (err) {
320 error.Canceled => |e| return e,
321 error.ReadFailed => comptime unreachable,
322 error.WriteFailed => comptime unreachable,
323 else => |e| std.log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }),
324 };
325 }
326
327 fn recompressFallible(jq: *JobQueue, arena: Allocator, dest_path: Cache.Path, package_hash: []const u8) !void {
328 const gpa = jq.http_client.allocator;
329 const io = jq.io;
330
331 // We have to walk the file system up front in order to sort the file
332 // list for determinism purposes. The hash of the recompressed file is
333 // not critical because the true hash is based on the content alone.
334 // However, if we want Zig users to be able to share cached package
335 // data with each other via peer-to-peer protocols, we benefit greatly
336 // from the data being identical on everyone's computers.
337 var scanned_files: std.ArrayList([]const u8) = .empty;
338 defer scanned_files.deinit(gpa);
339
340 var pkg_dir = try jq.root_pkg_path.openDir(io, package_hash, .{ .iterate = true });
341 defer pkg_dir.close(io);
342
343 {
344 var walker = try pkg_dir.walk(gpa);
345 defer walker.deinit();
346
347 while (try walker.next(io)) |entry| {
348 switch (entry.kind) {
349 .directory => continue,
350 .file, .sym_link => {},
351 else => {
352 return error.IllegalFileType;
353 },
354 }
355 const entry_path = try arena.dupe(u8, entry.path);
356 try scanned_files.append(gpa, entry_path);
357 }
358
359 std.mem.sortUnstable([]const u8, scanned_files.items, {}, stringCmp);
360 }
361
362 var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{
363 .make_path = true,
364 .replace = true,
365 });
366 defer atomic_file.deinit(io);
367
368 var file_write_buffer: [4096]u8 = undefined;
369 var file_writer = atomic_file.file.writer(io, &file_write_buffer);
370
371 var compress_buffer: [std.compress.flate.max_window_len]u8 = undefined;
372 var compress = std.compress.flate.Compress.init(&file_writer.interface, &compress_buffer, .gzip, .level_9) catch |err| switch (err) {
373 error.WriteFailed => return file_writer.err.?,
374 };
375
376 var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer };
377 archiver.prefix = package_hash;
378
379 var file_read_buffer: [4096]u8 = undefined;
380
381 for (scanned_files.items) |entry_path| {
382 var file = try pkg_dir.openFile(io, entry_path, .{});
383 defer file.close(io);
384 var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer);
385 archiver.writeFile(entry_path, &file_reader, 0) catch |err| switch (err) {
386 error.ReadFailed => return file_reader.err.?,
387 error.WriteFailed => return file_writer.err.?,
388 else => |e| return e,
389 };
390 }
391
392 // intentionally omitting the pointless trailer
393 //try archiver.finish();
394 compress.writer.flush() catch |err| switch (err) {
395 error.WriteFailed => return file_writer.err.?,
396 };
397 try file_writer.flush();
398 try atomic_file.replace(io);
399 }
296};400};
297401
402fn stringCmp(_: void, lhs: []const u8, rhs: []const u8) bool {
403 return std.mem.lessThan(u8, lhs, rhs);
404}
405
298pub const Location = union(enum) {406pub const Location = union(enum) {
299 remote: Remote,407 remote: Remote,
300 /// A directory found inside the parent package.408 /// A directory found inside the parent package.
...@@ -477,8 +585,11 @@ fn runResource(...@@ -477,8 +585,11 @@ fn runResource(
477 remote_hash: ?Package.Hash,585 remote_hash: ?Package.Hash,
478) RunError!void {586) RunError!void {
479 const job_queue = f.job_queue;587 const job_queue = f.job_queue;
588 assert(!job_queue.read_only);
589
480 const io = job_queue.io;590 const io = job_queue.io;
481 defer resource.deinit(io);591 defer resource.deinit(io);
592
482 const arena = f.arena.allocator();593 const arena = f.arena.allocator();
483 const eb = &f.error_bundle;594 const eb = &f.error_bundle;
484 const s = fs.path.sep_str;595 const s = fs.path.sep_str;
...@@ -556,6 +667,11 @@ fn runResource(...@@ -556,6 +667,11 @@ fn runResource(
556 ) });667 ) });
557 return error.FetchFailed;668 return error.FetchFailed;
558 };669 };
670
671 // Spin off a task to recompress the tarball, with filtered files deleted, into
672 // the global cache.
673 job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash });
674
559 // Remove temporary directory root if not already renamed to global cache.675 // Remove temporary directory root if not already renamed to global cache.
560 if (!package_sub_path.eql(tmp_directory_path)) {676 if (!package_sub_path.eql(tmp_directory_path)) {
561 tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch |err| switch (err) {677 tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch |err| switch (err) {