authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-04-20 16:35:07-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-04-20 18:45:16-07:00
log3a9b16db446379d6fd22bd614e6741e1142b456a
treec3e48977ee030d03fcd4c6574115eec03f6fed8e
parentf887bea4d34da6e4dcfdf1eb18cca6afca3dc546

zig fetch: detect global vs local mode via --save

Makes `zig fetch` only fetch globally, just like it used to. However, if `--save` (or any variant) is used, then it also fetches locally. When fetching by path, the hash is always computed, recompressed tarball is always created, always overwrites any existing global cache entry. closes #31818 closes #31866 (only requires build.zig file when --save is passed)

2 files changed, 114 insertions(+), 104 deletions(-)

src/Package/Fetch.zig+70-68
...@@ -104,6 +104,12 @@ pub const LazyStatus = enum {...@@ -104,6 +104,12 @@ pub const LazyStatus = enum {
104 unavailable,104 unavailable,
105};105};
106106
107pub const LocalStorage = struct {
108 cache_root: Cache.Path,
109 /// Path to "zig-pkg" inside the package in which the user ran `zig build`.
110 pkg_root: Cache.Path,
111};
112
107/// Contains shared state among all `Fetch` tasks.113/// Contains shared state among all `Fetch` tasks.
108pub const JobQueue = struct {114pub const JobQueue = struct {
109 io: Io,115 io: Io,
...@@ -122,9 +128,8 @@ pub const JobQueue = struct {...@@ -122,9 +128,8 @@ pub const JobQueue = struct {
122 /// This tracks `Fetch` tasks as well as recompression tasks.128 /// This tracks `Fetch` tasks as well as recompression tasks.
123 group: Io.Group = .init,129 group: Io.Group = .init,
124 global_cache: Cache.Directory,130 global_cache: Cache.Directory,
125 local_cache: Cache.Path,131 /// If `null`, indicates fetch globally only.
126 /// Path to "zig-pkg" inside the package in which the user ran `zig build`.132 local_storage: ?*const LocalStorage,
127 root_pkg_path: Cache.Path,
128 /// If true then, no fetching occurs, and:133 /// If true then, no fetching occurs, and:
129 /// * The `global_cache` directory is assumed to be the direct parent134 /// * The `global_cache` directory is assumed to be the direct parent
130 /// directory of on-disk packages rather than having the "p/" directory135 /// directory of on-disk packages rather than having the "p/" directory
...@@ -341,7 +346,7 @@ pub const JobQueue = struct {...@@ -341,7 +346,7 @@ pub const JobQueue = struct {
341 );346 );
342 }347 }
343348
344 fn recompress(jq: *JobQueue, package_hash: Package.Hash) Io.Cancelable!void {349 fn recompress(jq: *JobQueue, package_hash: Package.Hash, package_root: Cache.Path) Io.Cancelable!void {
345 const pkg_hash_slice = package_hash.toSlice();350 const pkg_hash_slice = package_hash.toSlice();
346351
347 const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice});352 const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice});
...@@ -359,7 +364,7 @@ pub const JobQueue = struct {...@@ -359,7 +364,7 @@ pub const JobQueue = struct {
359 defer arena_instance.deinit();364 defer arena_instance.deinit();
360 const arena = arena_instance.allocator();365 const arena = arena_instance.allocator();
361366
362 recompressFallible(jq, arena, dest_path, pkg_hash_slice, prog_node) catch |err| switch (err) {367 recompressFallible(jq, arena, dest_path, pkg_hash_slice, package_root, prog_node) catch |err| switch (err) {
363 error.Canceled => |e| return e,368 error.Canceled => |e| return e,
364 error.ReadFailed => comptime unreachable,369 error.ReadFailed => comptime unreachable,
365 error.WriteFailed => comptime unreachable,370 error.WriteFailed => comptime unreachable,
...@@ -372,6 +377,7 @@ pub const JobQueue = struct {...@@ -372,6 +377,7 @@ pub const JobQueue = struct {
372 arena: Allocator,377 arena: Allocator,
373 dest_path: Cache.Path,378 dest_path: Cache.Path,
374 pkg_hash_slice: []const u8,379 pkg_hash_slice: []const u8,
380 package_root: Cache.Path,
375 prog_node: std.Progress.Node,381 prog_node: std.Progress.Node,
376 ) !void {382 ) !void {
377 const gpa = jq.http_client.allocator;383 const gpa = jq.http_client.allocator;
...@@ -386,7 +392,7 @@ pub const JobQueue = struct {...@@ -386,7 +392,7 @@ pub const JobQueue = struct {
386 var scanned_files: std.ArrayList(ScannedFile) = .empty;392 var scanned_files: std.ArrayList(ScannedFile) = .empty;
387 defer scanned_files.deinit(gpa);393 defer scanned_files.deinit(gpa);
388394
389 var pkg_dir = try jq.root_pkg_path.openDir(io, pkg_hash_slice, .{ .iterate = true });395 var pkg_dir = try package_root.root_dir.handle.openDir(io, package_root.sub_path, .{ .iterate = true });
390 defer pkg_dir.close(io);396 defer pkg_dir.close(io);
391397
392 {398 {
...@@ -513,7 +519,6 @@ pub fn run(f: *Fetch) RunError!void {...@@ -513,7 +519,6 @@ pub fn run(f: *Fetch) RunError!void {
513 const eb = &f.error_bundle;519 const eb = &f.error_bundle;
514 const arena = f.arena.allocator();520 const arena = f.arena.allocator();
515 const gpa = f.arena.child_allocator;521 const gpa = f.arena.child_allocator;
516 const local_cache_root = job_queue.local_cache;
517522
518 try eb.init(gpa);523 try eb.init(gpa);
519524
...@@ -534,32 +539,16 @@ pub fn run(f: *Fetch) RunError!void {...@@ -534,32 +539,16 @@ pub fn run(f: *Fetch) RunError!void {
534 );539 );
535 // Packages fetched by URL may not use relative paths to escape outside the540 // Packages fetched by URL may not use relative paths to escape outside the
536 // fetched package directory from within the package cache.541 // fetched package directory from within the package cache.
537 if (pkg_root.root_dir.eql(local_cache_root.root_dir)) {542
538 // `parent_package_root.sub_path` contains a path like this:543 // This code path is only reachable recursively and the sub_path
539 // "p/$hash", or544 // will already have been resolved to no longer have extra ".." or
540 // "p/$hash/foo", with possibly more directories after "foo".545 // "." components.
541 // We want to fail unless the resolved relative path has a546 assert(job_queue.local_storage != null);
542 // prefix of "p/$hash/".547 assert(pkg_root.root_dir.eql(f.parent_package_root.root_dir));
543 const prefix_len: usize = if (job_queue.read_only) 0 else "p/".len;548 if (!std.mem.startsWith(u8, pkg_root.sub_path, f.parent_package_root.sub_path)) return f.fail(
544 const parent_sub_path = f.parent_package_root.sub_path;549 f.location_tok,
545 const end = find_end: {550 try eb.printString("dependency path outside project: '{f}'", .{pkg_root}),
546 if (parent_sub_path.len > prefix_len) {551 );
547 // Use `isSep` instead of `indexOfScalarPos` to account for
548 // Windows accepting both `\` and `/` as path separators.
549 for (parent_sub_path[prefix_len..], prefix_len..) |c, i| {
550 if (std.fs.path.isSep(c)) break :find_end i;
551 }
552 }
553 break :find_end parent_sub_path.len;
554 };
555 const expected_prefix = parent_sub_path[0..end];
556 if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) {
557 return f.fail(
558 f.location_tok,
559 try eb.printString("dependency path outside project: '{f}'", .{pkg_root}),
560 );
561 }
562 }
563 f.package_root = pkg_root;552 f.package_root = pkg_root;
564 try loadManifest(f, pkg_root);553 try loadManifest(f, pkg_root);
565 if (!f.has_build_zig) try checkBuildFileExistence(f);554 if (!f.has_build_zig) try checkBuildFileExistence(f);
...@@ -610,31 +599,33 @@ pub fn run(f: *Fetch) RunError!void {...@@ -610,31 +599,33 @@ pub fn run(f: *Fetch) RunError!void {
610 return queueJobsForDeps(f);599 return queueJobsForDeps(f);
611 }600 }
612601
613 const package_root = try job_queue.root_pkg_path.join(arena, expected_hash.toSlice());602 if (job_queue.local_storage) |ls| {
614 if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| {603 const package_root = try ls.pkg_root.join(arena, expected_hash.toSlice());
615 assert(f.lazy_status != .unavailable);604 if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| {
616 f.package_root = package_root;605 assert(f.lazy_status != .unavailable);
617 try loadManifest(f, f.package_root);606 f.package_root = package_root;
618 try checkBuildFileExistence(f);607 try loadManifest(f, f.package_root);
619 if (!job_queue.recursive) return;608 try checkBuildFileExistence(f);
620 return queueJobsForDeps(f);609 if (!job_queue.recursive) return;
621 } else |err| switch (err) {610 return queueJobsForDeps(f);
622 error.FileNotFound => {611 } else |err| switch (err) {
623 log.debug("FileNotFound: {f}", .{package_root});612 error.FileNotFound => {
624 if (job_queue.read_only and f.lazy_status == .eager) return f.fail(613 log.debug("FileNotFound: {f}", .{package_root});
625 f.name_tok,614 if (job_queue.read_only and f.lazy_status == .eager) return f.fail(
626 try eb.printString("package not found at '{f}'", .{package_root}),615 f.name_tok,
627 );616 try eb.printString("package not found at '{f}'", .{package_root}),
628 },617 );
629 error.Canceled => |e| return e,618 },
630 else => |e| {619 error.Canceled => |e| return e,
631 try eb.addRootErrorMessage(.{620 else => |e| {
632 .msg = try eb.printString("unable to open package cache directory {f}: {t}", .{621 try eb.addRootErrorMessage(.{
633 package_root, e,622 .msg = try eb.printString("unable to open package cache directory {f}: {t}", .{
634 }),623 package_root, e,
635 });624 }),
636 return error.FetchFailed;625 });
637 },626 return error.FetchFailed;
627 },
628 }
638 }629 }
639630
640 // Check global cache before remote fetch.631 // Check global cache before remote fetch.
...@@ -713,7 +704,14 @@ fn runResource(...@@ -713,7 +704,14 @@ fn runResource(
713 break :r x;704 break :r x;
714 };705 };
715 const tmp_dir_sub_path = ".tmp-" ++ std.fmt.hex(rand_int);706 const tmp_dir_sub_path = ".tmp-" ++ std.fmt.hex(rand_int);
716 const tmp_directory_path = try job_queue.root_pkg_path.join(arena, tmp_dir_sub_path);707 const tmp_tmp_dir_sub_path = "tmp/" ++ tmp_dir_sub_path;
708 const tmp_directory_path: Cache.Path = if (job_queue.local_storage) |ls|
709 try ls.pkg_root.join(arena, tmp_dir_sub_path)
710 else
711 .{
712 .root_dir = job_queue.global_cache,
713 .sub_path = tmp_tmp_dir_sub_path,
714 };
717715
718 const package_sub_path = blk: {716 const package_sub_path = blk: {
719 var tmp_directory: Cache.Directory = .{717 var tmp_directory: Cache.Directory = .{
...@@ -772,19 +770,23 @@ fn runResource(...@@ -772,19 +770,23 @@ fn runResource(
772 // zig package directory untouched as it may be in use. This is done even770 // zig package directory untouched as it may be in use. This is done even
773 // if the hash is invalid, in case the package with the different hash is771 // if the hash is invalid, in case the package with the different hash is
774 // used in the future.772 // used in the future.
775 f.package_root = try job_queue.root_pkg_path.join(arena, computed_package_hash.toSlice());773 if (job_queue.local_storage) |ls| {
776 renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| {774 f.package_root = try ls.pkg_root.join(arena, computed_package_hash.toSlice());
777 try eb.addRootErrorMessage(.{ .msg = try eb.printString(775 renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| {
778 "unable to rename temporary directory {f} into package cache directory {f}: {t}",776 try eb.addRootErrorMessage(.{ .msg = try eb.printString(
779 .{ package_sub_path, f.package_root, err },777 "unable to rename temporary directory {f} into package cache directory {f}: {t}",
780 ) });778 .{ package_sub_path, f.package_root, err },
781 return error.FetchFailed;779 ) });
782 };780 return error.FetchFailed;
781 };
782 } else {
783 f.package_root = tmp_directory_path;
784 }
783785
784 if (!disable_recompress) {786 if (!disable_recompress) {
785 // Spin off a task to recompress the tarball, with filtered files deleted, into787 // Spin off a task to recompress the tarball, with filtered files deleted, into
786 // the global cache.788 // the global cache.
787 job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash });789 job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash, f.package_root });
788 }790 }
789791
790 // Remove temporary directory root if not already renamed to global cache.792 // Remove temporary directory root if not already renamed to global cache.
src/main.zig+44-36
...@@ -5331,9 +5331,6 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,...@@ -5331,9 +5331,6 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
5331 .parent = root_mod,5331 .parent = root_mod,
5332 });5332 });
53335333
5334 var cleanup_build_dir: ?Io.Dir = null;
5335 defer if (cleanup_build_dir) |*dir| dir.close(io);
5336
5337 if (dev.env.supports(.fetch_command)) {5334 if (dev.env.supports(.fetch_command)) {
5338 const fetch_prog_node = root_prog_node.start("Fetch Packages", 0);5335 const fetch_prog_node = root_prog_node.start("Fetch Packages", 0);
5339 defer fetch_prog_node.end();5336 defer fetch_prog_node.end();
...@@ -5345,36 +5342,29 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,...@@ -5345,36 +5342,29 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
5345 .io = io,5342 .io = io,
5346 .http_client = &http_client,5343 .http_client = &http_client,
5347 .global_cache = dirs.global_cache,5344 .global_cache = dirs.global_cache,
5348 .local_cache = .{ .root_dir = dirs.local_cache, .sub_path = "" },5345 .local_storage = &.{
5349 .root_pkg_path = if (override_pkg_dir) |cwd_rel_path| .initCwd(cwd_rel_path) else .{5346 .cache_root = .{ .root_dir = dirs.local_cache, .sub_path = "" },
5350 .root_dir = build_root.directory,5347 .pkg_root = if (override_pkg_dir) |p|
5351 .sub_path = "zig-pkg",5348 .initCwd(p)
5349 else if (system_pkg_dir_path) |p|
5350 .initCwd(p)
5351 else
5352 .{
5353 .root_dir = build_root.directory,
5354 .sub_path = "zig-pkg",
5355 },
5352 },5356 },
5353 .read_only = false,
5354 .recursive = true,5357 .recursive = true,
5355 .debug_hash = false,5358 .debug_hash = false,
5356 .unlazy_set = unlazy_set,5359 .unlazy_set = unlazy_set,
5357 .fork_set = fork_set,5360 .fork_set = fork_set,
5358 .mode = fetch_mode,5361 .mode = fetch_mode,
5359 .prog_node = fetch_prog_node,5362 .prog_node = fetch_prog_node,
5363 .read_only = system_pkg_dir_path != null,
5360 };5364 };
5361 defer job_queue.deinit();5365 defer job_queue.deinit();
53625366
5363 if (system_pkg_dir_path) |p| {5367 if (system_pkg_dir_path == null) {
5364 const system_pkg_path: Path = .{
5365 .root_dir = .{
5366 .path = p,
5367 .handle = Io.Dir.cwd().openDir(io, p, .{}) catch |err| {
5368 fatal("unable to open system package directory '{s}': {t}", .{ p, err });
5369 },
5370 },
5371 .sub_path = "",
5372 };
5373 job_queue.global_cache = system_pkg_path.root_dir;
5374 job_queue.root_pkg_path = system_pkg_path;
5375 job_queue.read_only = true;
5376 cleanup_build_dir = job_queue.global_cache.handle;
5377 } else {
5378 try http_client.initDefaultProxies(arena, environ_map);5368 try http_client.initDefaultProxies(arena, environ_map);
5379 }5369 }
53805370
...@@ -7041,7 +7031,8 @@ const usage_fetch =...@@ -7041,7 +7031,8 @@ const usage_fetch =
7041 \\Options:7031 \\Options:
7042 \\ -h, --help Print this help and exit7032 \\ -h, --help Print this help and exit
7043 \\ --global-cache-dir [path] Override path to global Zig cache directory7033 \\ --global-cache-dir [path] Override path to global Zig cache directory
7044 \\ --pkg-dir [path] Override path to package directory7034 \\ --cache-dir [path] Override path to local cache directory
7035 \\ --pkg-dir [path] Override path to local package directory
7045 \\ --debug-hash Print verbose hash information to stdout7036 \\ --debug-hash Print verbose hash information to stdout
7046 \\ --save Add the fetched package to build.zig.zon7037 \\ --save Add the fetched package to build.zig.zon
7047 \\ --save=[name] Add the fetched package to build.zig.zon as name7038 \\ --save=[name] Add the fetched package to build.zig.zon as name
...@@ -7062,6 +7053,7 @@ fn cmdFetch(...@@ -7062,6 +7053,7 @@ fn cmdFetch(
7062 const color: Color = .auto;7053 const color: Color = .auto;
7063 var opt_path_or_url: ?[]const u8 = null;7054 var opt_path_or_url: ?[]const u8 = null;
7064 var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);7055 var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
7056 var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map);
7065 var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map);7057 var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map);
7066 var debug_hash: bool = false;7058 var debug_hash: bool = false;
7067 var save: union(enum) {7059 var save: union(enum) {
...@@ -7082,6 +7074,10 @@ fn cmdFetch(...@@ -7082,6 +7074,10 @@ fn cmdFetch(
7082 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});7074 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
7083 i += 1;7075 i += 1;
7084 override_global_cache_dir = args[i];7076 override_global_cache_dir = args[i];
7077 } else if (mem.eql(u8, arg, "--cache-dir")) {
7078 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
7079 i += 1;
7080 override_local_cache_dir = args[i];
7085 } else if (mem.eql(u8, arg, "--pkg-dir")) {7081 } else if (mem.eql(u8, arg, "--pkg-dir")) {
7086 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});7082 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
7087 i += 1;7083 i += 1;
...@@ -7128,27 +7124,39 @@ fn cmdFetch(...@@ -7128,27 +7124,39 @@ fn cmdFetch(
7128 };7124 };
7129 defer global_cache_directory.handle.close(io);7125 defer global_cache_directory.handle.close(io);
71307126
7127 var local_storage: Package.Fetch.LocalStorage = undefined;
7128 var build_root: BuildRoot = undefined;
7129 var build_root_initialized = false;
7130 defer if (build_root_initialized) build_root.deinit(io);
7131
7131 const cwd_path = try introspect.getResolvedCwd(io, arena);7132 const cwd_path = try introspect.getResolvedCwd(io, arena);
71327133
7133 var build_root = try findBuildRoot(arena, io, .{7134 const local_storage_ptr = switch (save) {
7134 .cwd_path = cwd_path,7135 .no => null,
7135 });7136 .yes, .exact => ls: {
7136 defer build_root.deinit(io);7137 build_root = try findBuildRoot(arena, io, .{ .cwd_path = cwd_path });
7138 build_root_initialized = true;
7139
7140 local_storage = .{
7141 .cache_root = if (override_local_cache_dir) |p| .initCwd(p) else .{
7142 .root_dir = build_root.directory,
7143 .sub_path = ".zig-cache",
7144 },
7145 .pkg_root = if (override_pkg_dir) |p| .initCwd(p) else .{
7146 .root_dir = build_root.directory,
7147 .sub_path = "zig-pkg",
7148 },
7149 };
71377150
7138 const local_cache_path: Path = .{7151 break :ls &local_storage;
7139 .root_dir = build_root.directory,7152 },
7140 .sub_path = ".zig-cache",
7141 };7153 };
71427154
7143 var job_queue: Package.Fetch.JobQueue = .{7155 var job_queue: Package.Fetch.JobQueue = .{
7144 .io = io,7156 .io = io,
7145 .http_client = &http_client,7157 .http_client = &http_client,
7146 .global_cache = global_cache_directory,7158 .global_cache = global_cache_directory,
7147 .local_cache = local_cache_path,7159 .local_storage = local_storage_ptr,
7148 .root_pkg_path = if (override_pkg_dir) |cwd_rel_path| .initCwd(cwd_rel_path) else .{
7149 .root_dir = build_root.directory,
7150 .sub_path = "zig-pkg",
7151 },
7152 .recursive = false,7160 .recursive = false,
7153 .read_only = false,7161 .read_only = false,
7154 .debug_hash = debug_hash,7162 .debug_hash = debug_hash,