authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-04 15:55:56-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-05 16:50:41-08:00
log64dc1cdad8faf1fd8330d4b0f9149817b5205abf
tree5e8280bd5a6dd184e85462136c09681348a0b2d2
parent76d275b20f4ba420f741e019a7456ca392c040e4

fetch: download to local zig-pkg directory

rather than global cache p/ directory. closes #14283 does not recompress packages into global cache yet

2 files changed, 115 insertions(+), 85 deletions(-)

src/Package/Fetch.zig+77-71
......@@ -68,8 +68,7 @@ use_latest_commit: bool,
6868// Above this are fields provided as inputs to `run`.
6969// Below this are fields populated by `run`.
7070
71/// This will either be relative to `global_cache`, or to the build root of
72/// the root package.
71/// Relative to the build root of the root package.
7372package_root: Cache.Path,
7473error_bundle: ErrorBundle.Wip,
7574manifest: ?Manifest,
......@@ -115,6 +114,9 @@ pub const JobQueue = struct {
115114 http_client: *std.http.Client,
116115 group: Io.Group = .init,
117116 global_cache: Cache.Directory,
117 local_cache: Cache.Path,
118 /// Path to "zig-pkg" inside the package in which the user ran `zig build`.
119 root_pkg_path: Cache.Path,
118120 /// If true then, no fetching occurs, and:
119121 /// * The `global_cache` directory is assumed to be the direct parent
120122 /// directory of on-disk packages rather than having the "p/" directory
......@@ -325,11 +327,12 @@ pub const RunError = error{
325327};
326328
327329pub fn run(f: *Fetch) RunError!void {
328 const io = f.job_queue.io;
330 const job_queue = f.job_queue;
331 const io = job_queue.io;
329332 const eb = &f.error_bundle;
330333 const arena = f.arena.allocator();
331334 const gpa = f.arena.child_allocator;
332 const cache_root = f.job_queue.global_cache;
335 const local_cache_root = job_queue.local_cache;
333336
334337 try eb.init(gpa);
335338
......@@ -350,13 +353,13 @@ pub fn run(f: *Fetch) RunError!void {
350353 );
351354 // Packages fetched by URL may not use relative paths to escape outside the
352355 // fetched package directory from within the package cache.
353 if (pkg_root.root_dir.eql(cache_root)) {
356 if (pkg_root.root_dir.eql(local_cache_root.root_dir)) {
354357 // `parent_package_root.sub_path` contains a path like this:
355358 // "p/$hash", or
356359 // "p/$hash/foo", with possibly more directories after "foo".
357360 // We want to fail unless the resolved relative path has a
358361 // prefix of "p/$hash/".
359 const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len;
362 const prefix_len: usize = if (job_queue.read_only) 0 else "p/".len;
360363 const parent_sub_path = f.parent_package_root.sub_path;
361364 const end = find_end: {
362365 if (parent_sub_path.len > prefix_len) {
......@@ -379,7 +382,7 @@ pub fn run(f: *Fetch) RunError!void {
379382 f.package_root = pkg_root;
380383 try loadManifest(f, pkg_root);
381384 if (!f.has_build_zig) try checkBuildFileExistence(f);
382 if (!f.job_queue.recursive) return;
385 if (!job_queue.recursive) return;
383386 return queueJobsForDeps(f);
384387 },
385388 .remote => |remote| remote,
......@@ -411,51 +414,39 @@ pub fn run(f: *Fetch) RunError!void {
411414 };
412415
413416 if (remote.hash) |expected_hash| {
414 var prefixed_pkg_sub_path_buffer: [Package.Hash.max_len + 2]u8 = undefined;
415 prefixed_pkg_sub_path_buffer[0] = 'p';
416 prefixed_pkg_sub_path_buffer[1] = fs.path.sep;
417 const hash_slice = expected_hash.toSlice();
418 @memcpy(prefixed_pkg_sub_path_buffer[2..][0..hash_slice.len], hash_slice);
419 const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len];
420 const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;
421 const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];
422 if (cache_root.handle.access(io, pkg_sub_path, .{})) |_| {
417 const package_root = try job_queue.root_pkg_path.join(arena, expected_hash.toSlice());
418 if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| {
423419 assert(f.lazy_status != .unavailable);
424 f.package_root = .{
425 .root_dir = cache_root,
426 .sub_path = try arena.dupe(u8, pkg_sub_path),
427 };
420 f.package_root = package_root;
428421 try loadManifest(f, f.package_root);
429422 try checkBuildFileExistence(f);
430 if (!f.job_queue.recursive) return;
423 if (!job_queue.recursive) return;
431424 return queueJobsForDeps(f);
432425 } else |err| switch (err) {
433426 error.FileNotFound => {
434427 switch (f.lazy_status) {
435428 .eager => {},
436 .available => if (!f.job_queue.unlazy_set.contains(expected_hash)) {
429 .available => if (!job_queue.unlazy_set.contains(expected_hash)) {
437430 f.lazy_status = .unavailable;
438431 return;
439432 },
440433 .unavailable => unreachable,
441434 }
442 if (f.job_queue.read_only) return f.fail(
435 if (job_queue.read_only) return f.fail(
443436 f.name_tok,
444 try eb.printString("package not found at '{f}{s}'", .{
445 cache_root, pkg_sub_path,
446 }),
437 try eb.printString("package not found at '{f}'", .{package_root}),
447438 );
448439 },
449440 else => |e| {
450441 try eb.addRootErrorMessage(.{
451 .msg = try eb.printString("unable to open global package cache directory '{f}{s}': {s}", .{
452 cache_root, pkg_sub_path, @errorName(e),
442 .msg = try eb.printString("unable to open package cache directory {f}: {t}", .{
443 package_root, e,
453444 }),
454445 });
455446 return error.FetchFailed;
456447 },
457448 }
458 } else if (f.job_queue.read_only) {
449 } else if (job_queue.read_only) {
459450 try eb.addRootErrorMessage(.{
460451 .msg = try eb.addString("dependency is missing hash field"),
461452 .src_loc = try f.srcLoc(f.location_tok),
......@@ -467,7 +458,7 @@ pub fn run(f: *Fetch) RunError!void {
467458
468459 const uri = std.Uri.parse(remote.url) catch |err| return f.fail(
469460 f.location_tok,
470 try eb.printString("invalid URI: {s}", .{@errorName(err)}),
461 try eb.printString("invalid URI: {t}", .{err}),
471462 );
472463 var buffer: [init_resource_buffer_size]u8 = undefined;
473464 var resource: Resource = undefined;
......@@ -487,29 +478,30 @@ fn runResource(
487478 resource: *Resource,
488479 remote_hash: ?Package.Hash,
489480) RunError!void {
490 const io = f.job_queue.io;
481 const job_queue = f.job_queue;
482 const io = job_queue.io;
491483 defer resource.deinit(io);
492484 const arena = f.arena.allocator();
493485 const eb = &f.error_bundle;
494486 const s = fs.path.sep_str;
495 const cache_root = f.job_queue.global_cache;
487 const local_cache_root = job_queue.local_cache;
496488 const rand_int = r: {
497489 var x: u64 = undefined;
498490 io.random(@ptrCast(&x));
499491 break :r x;
500492 };
501493 const tmp_dir_sub_path = "tmp" ++ s ++ std.fmt.hex(rand_int);
494 const tmp_directory_path = try local_cache_root.join(arena, tmp_dir_sub_path);
502495
503496 const package_sub_path = blk: {
504 const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path});
505497 var tmp_directory: Cache.Directory = .{
506 .path = tmp_directory_path,
498 .path = tmp_directory_path.sub_path,
507499 .handle = handle: {
508 const dir = cache_root.handle.createDirPathOpen(io, tmp_dir_sub_path, .{
500 const dir = tmp_directory_path.root_dir.handle.createDirPathOpen(io, tmp_directory_path.sub_path, .{
509501 .open_options = .{ .iterate = true },
510502 }) catch |err| {
511503 try eb.addRootErrorMessage(.{
512 .msg = try eb.printString("unable to create temporary directory '{s}': {t}", .{
504 .msg = try eb.printString("unable to create temporary directory '{f}': {t}", .{
513505 tmp_directory_path, err,
514506 }),
515507 });
......@@ -545,36 +537,33 @@ fn runResource(
545537 // directory.
546538 f.computed_hash = try computeHash(f, pkg_path, filter);
547539
548 break :blk if (unpack_result.root_dir.len > 0)
549 try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir })
550 else
551 tmp_dir_sub_path;
540 if (unpack_result.root_dir.len > 0)
541 break :blk try tmp_directory_path.join(arena, unpack_result.root_dir);
542
543 break :blk tmp_directory_path;
552544 };
553545
554546 const computed_package_hash = computedPackageHash(f);
555547
556 // Rename the temporary directory into the global zig package cache
557 // directory. If the hash already exists, delete the temporary directory
558 // and leave the zig package cache directory untouched as it may be in use
559 // by the system. This is done even if the hash is invalid, in case the
560 // package with the different hash is used in the future.
561
562 f.package_root = .{
563 .root_dir = cache_root,
564 .sub_path = try std.fmt.allocPrint(arena, "p" ++ s ++ "{s}", .{computed_package_hash.toSlice()}),
565 };
566 renameTmpIntoCache(io, cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| {
567 const src = try cache_root.join(arena, &.{tmp_dir_sub_path});
568 const dest = try cache_root.join(arena, &.{f.package_root.sub_path});
548 // Rename the temporary directory into the local zig package directory. If
549 // the hash already exists, delete the temporary directory and leave the
550 // zig package directory untouched as it may be in use. This is done even
551 // if the hash is invalid, in case the package with the different hash is
552 // used in the future.
553 f.package_root = try job_queue.root_pkg_path.join(arena, computed_package_hash.toSlice());
554 renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| {
569555 try eb.addRootErrorMessage(.{ .msg = try eb.printString(
570 "unable to rename temporary directory '{s}' into package cache directory '{s}': {s}",
571 .{ src, dest, @errorName(err) },
556 "unable to rename temporary directory {f} into package cache directory {f}: {t}",
557 .{ package_sub_path, f.package_root, err },
572558 ) });
573559 return error.FetchFailed;
574560 };
575561 // Remove temporary directory root if not already renamed to global cache.
576 if (!std.mem.eql(u8, package_sub_path, tmp_dir_sub_path)) {
577 cache_root.handle.deleteDir(io, tmp_dir_sub_path) catch {};
562 if (!package_sub_path.eql(tmp_directory_path)) {
563 tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch |err| switch (err) {
564 error.Canceled => |e| return e,
565 else => |e| std.log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_directory_path, e }),
566 };
578567 }
579568
580569 // Validate the computed hash against the expected hash. If invalid, this
......@@ -614,7 +603,7 @@ fn runResource(
614603
615604 // Spawn a new fetch job for each dependency in the manifest file. Use
616605 // a mutex and a hash map so that redundant jobs do not get queued up.
617 if (!f.job_queue.recursive) return;
606 if (!job_queue.recursive) return;
618607 return queueJobsForDeps(f);
619608}
620609
......@@ -641,8 +630,8 @@ fn checkBuildFileExistence(f: *Fetch) RunError!void {
641630 error.FileNotFound => {},
642631 else => |e| {
643632 try eb.addRootErrorMessage(.{
644 .msg = try eb.printString("unable to access '{f}{s}': {s}", .{
645 f.package_root, Package.build_zig_basename, @errorName(e),
633 .msg = try eb.printString("unable to access '{f}{s}': {t}", .{
634 f.package_root, Package.build_zig_basename, e,
646635 }),
647636 });
648637 return error.FetchFailed;
......@@ -667,9 +656,7 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {
667656 else => |e| {
668657 const file_path = try pkg_root.join(arena, Manifest.basename);
669658 try eb.addRootErrorMessage(.{
670 .msg = try eb.printString("unable to load package manifest '{f}': {s}", .{
671 file_path, @errorName(e),
672 }),
659 .msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ file_path, e }),
673660 });
674661 return error.FetchFailed;
675662 },
......@@ -1453,14 +1440,20 @@ fn recursiveDirectoryCopy(f: *Fetch, dir: Io.Dir, tmp_dir: Io.Dir) anyerror!void
14531440 }
14541441}
14551442
1456pub fn renameTmpIntoCache(io: Io, cache_dir: Io.Dir, tmp_dir_sub_path: []const u8, dest_dir_sub_path: []const u8) !void {
1457 assert(dest_dir_sub_path[1] == fs.path.sep);
1443pub fn renameTmpIntoCache(io: Io, tmp_path: Cache.Path, dest_path: Cache.Path) !void {
14581444 var handled_missing_dir = false;
14591445 while (true) {
1460 cache_dir.rename(tmp_dir_sub_path, cache_dir, dest_dir_sub_path, io) catch |err| switch (err) {
1446 Io.Dir.rename(
1447 tmp_path.root_dir.handle,
1448 tmp_path.sub_path,
1449 dest_path.root_dir.handle,
1450 dest_path.sub_path,
1451 io,
1452 ) catch |err| switch (err) {
14611453 error.FileNotFound => {
14621454 if (handled_missing_dir) return err;
1463 cache_dir.createDir(io, dest_dir_sub_path[0..1], .default_dir) catch |mkd_err| switch (mkd_err) {
1455 const parent_sub_path = Io.Dir.path.dirname(dest_path.sub_path).?;
1456 dest_path.root_dir.handle.createDir(io, parent_sub_path, .default_dir) catch |er| switch (er) {
14641457 error.PathAlreadyExists => handled_missing_dir = true,
14651458 else => |e| return e,
14661459 };
......@@ -1468,9 +1461,11 @@ pub fn renameTmpIntoCache(io: Io, cache_dir: Io.Dir, tmp_dir_sub_path: []const u
14681461 },
14691462 error.DirNotEmpty, error.AccessDenied => {
14701463 // Package has been already downloaded and may already be in use on the system.
1471 cache_dir.deleteTree(io, tmp_dir_sub_path) catch {
1464 tmp_path.root_dir.handle.deleteTree(io, tmp_path.sub_path) catch |er| switch (er) {
1465 error.Canceled => |e| return e,
14721466 // Garbage files leftover in zig-cache/tmp/ is, as they say
14731467 // on Star Trek, "operating within normal parameters".
1468 else => |e| std.log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_path, e }),
14741469 };
14751470 },
14761471 else => |e| return e,
......@@ -2244,6 +2239,7 @@ fn saveEmbedFile(io: Io, comptime tarball_name: []const u8, dir: Io.Dir) !void {
22442239const TestFetchBuilder = struct {
22452240 http_client: std.http.Client,
22462241 global_cache_directory: Cache.Directory,
2242 local_cache_path: Cache.Path,
22472243 job_queue: Fetch.JobQueue,
22482244 fetch: Fetch,
22492245
......@@ -2254,15 +2250,25 @@ const TestFetchBuilder = struct {
22542250 cache_parent_dir: std.Io.Dir,
22552251 path_or_url: []const u8,
22562252 ) !*Fetch {
2257 const cache_dir = try cache_parent_dir.createDirPathOpen(io, "zig-global-cache", .{});
2253 const global_cache_dir = try cache_parent_dir.createDirPathOpen(io, "zig-global-cache", .{});
2254 const package_root_dir = try cache_parent_dir.createDirPathOpen(io, "local-project-root", .{});
22582255
22592256 self.http_client = .{ .allocator = allocator, .io = io };
2260 self.global_cache_directory = .{ .handle = cache_dir, .path = null };
2257 self.global_cache_directory = .{ .handle = global_cache_dir, .path = "zig-global-cache" };
2258 self.local_cache_path = .{
2259 .root_dir = .{ .handle = package_root_dir, .path = "local-project-root" },
2260 .sub_path = ".zig-cache",
2261 };
22612262
22622263 self.job_queue = .{
22632264 .io = io,
22642265 .http_client = &self.http_client,
22652266 .global_cache = self.global_cache_directory,
2267 .local_cache = self.local_cache_path,
2268 .root_pkg_path = .{
2269 .root_dir = .{ .handle = package_root_dir, .path = "local-project-root" },
2270 .sub_path = "zig-pkg",
2271 },
22662272 .recursive = false,
22672273 .read_only = false,
22682274 .debug_hash = false,
......@@ -2276,7 +2282,7 @@ const TestFetchBuilder = struct {
22762282 .hash_tok = .none,
22772283 .name_tok = 0,
22782284 .lazy_status = .eager,
2279 .parent_package_root = Cache.Path{ .root_dir = Cache.Directory{ .handle = cache_dir, .path = null } },
2285 .parent_package_root = .{ .root_dir = .{ .handle = package_root_dir, .path = null } },
22802286 .parent_manifest_ast = null,
22812287 .prog_node = std.Progress.Node.none,
22822288 .job_queue = &self.job_queue,
src/main.zig+38-14
......@@ -5239,6 +5239,8 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
52395239 .io = io,
52405240 .http_client = &http_client,
52415241 .global_cache = dirs.global_cache,
5242 .local_cache = .{ .root_dir = dirs.local_cache, .sub_path = "" },
5243 .root_pkg_path = .{ .root_dir = build_root.directory, .sub_path = "zig-pkg" },
52425244 .read_only = false,
52435245 .recursive = true,
52445246 .debug_hash = false,
......@@ -5248,12 +5250,17 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
52485250 defer job_queue.deinit();
52495251
52505252 if (system_pkg_dir_path) |p| {
5251 job_queue.global_cache = .{
5252 .path = p,
5253 .handle = Io.Dir.cwd().openDir(io, p, .{}) catch |err| {
5254 fatal("unable to open system package directory '{s}': {t}", .{ p, err });
5253 const system_pkg_path: Path = .{
5254 .root_dir = .{
5255 .path = p,
5256 .handle = Io.Dir.cwd().openDir(io, p, .{}) catch |err| {
5257 fatal("unable to open system package directory '{s}': {t}", .{ p, err });
5258 },
52555259 },
5260 .sub_path = "",
52565261 };
5262 job_queue.global_cache = system_pkg_path.root_dir;
5263 job_queue.root_pkg_path = system_pkg_path;
52575264 job_queue.read_only = true;
52585265 cleanup_build_dir = job_queue.global_cache.handle;
52595266 } else {
......@@ -6996,10 +7003,27 @@ fn cmdFetch(
69967003 };
69977004 defer global_cache_directory.handle.close(io);
69987005
7006 const cwd_path = try introspect.getResolvedCwd(io, arena);
7007
7008 var build_root = try findBuildRoot(arena, io, .{
7009 .cwd_path = cwd_path,
7010 });
7011 defer build_root.deinit(io);
7012
7013 const local_cache_path: Path = .{
7014 .root_dir = build_root.directory,
7015 .sub_path = ".zig-cache",
7016 };
7017
69997018 var job_queue: Package.Fetch.JobQueue = .{
70007019 .io = io,
70017020 .http_client = &http_client,
70027021 .global_cache = global_cache_directory,
7022 .local_cache = local_cache_path,
7023 .root_pkg_path = .{
7024 .root_dir = build_root.directory,
7025 .sub_path = "zig-pkg",
7026 },
70037027 .recursive = false,
70047028 .read_only = false,
70057029 .debug_hash = debug_hash,
......@@ -7069,13 +7093,6 @@ fn cmdFetch(
70697093 },
70707094 };
70717095
7072 const cwd_path = try introspect.getResolvedCwd(io, arena);
7073
7074 var build_root = try findBuildRoot(arena, io, .{
7075 .cwd_path = cwd_path,
7076 });
7077 defer build_root.deinit(io);
7078
70797096 // The name to use in case the manifest file needs to be created now.
70807097 const init_root_name = fs.path.basename(build_root.directory.path orelse cwd_path);
70817098 var manifest, var ast = try loadManifest(gpa, arena, io, .{
......@@ -7239,18 +7256,25 @@ fn createDependenciesModule(
72397256 defer tmp_dir.close(io);
72407257 try tmp_dir.writeFile(io, .{ .sub_path = basename, .data = source });
72417258 }
7259 const tmp_dir_path: Path = .{
7260 .root_dir = dirs.local_cache,
7261 .sub_path = tmp_dir_sub_path,
7262 };
72427263
72437264 var hh: Cache.HashHelper = .{};
72447265 hh.addBytes(build_options.version);
72457266 hh.addBytes(source);
72467267 const hex_digest = hh.final();
72477268
7248 const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest);
7249 try Package.Fetch.renameTmpIntoCache(io, dirs.local_cache.handle, tmp_dir_sub_path, o_dir_sub_path);
7269 const o_dir_path: Path = .{
7270 .root_dir = dirs.local_cache,
7271 .sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest),
7272 };
7273 try Package.Fetch.renameTmpIntoCache(io, tmp_dir_path, o_dir_path);
72507274
72517275 const deps_mod = try Package.Module.create(arena, .{
72527276 .paths = .{
7253 .root = try .fromRoot(arena, dirs, .local_cache, o_dir_sub_path),
7277 .root = try .fromRoot(arena, dirs, .local_cache, o_dir_path.sub_path),
72547278 .root_src_path = basename,
72557279 },
72567280 .fully_qualified_name = "root.@dependencies",