authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-10-15 18:01:23-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-10-15 18:02:03-07:00
log215db865490b5d4d08171297603584841be97976
tree2fb0b8c2e8e6f3f33197b20814ec465a2beface7
parentd67152361ce131a7d49bbb8fbc0819578448d502

Maker: implement refetching from step-associated lazy deps

closes #37262

2 files changed, 83 insertions(+), 68 deletions(-)

lib/compiler/Maker.zig+80-65
...@@ -763,6 +763,8 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -763,6 +763,8 @@ pub fn main(init: process.Init.Minimal) !void {
763 break :s &protocol_server_allocation;763 break :s &protocol_server_allocation;
764 } else null;764 } else null;
765765
766 var discovered_unlazy_set: Fetch.JobQueue.UnlazySet = .empty; // uses global arena
767
766 configure: while (true) {768 configure: while (true) {
767 // Set of files that, if modified, imply that recompiling and rerunning769 // Set of files that, if modified, imply that recompiling and rerunning
768 // configurer is needed.770 // configurer is needed.
...@@ -792,9 +794,12 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -792,9 +794,12 @@ pub fn main(init: process.Init.Minimal) !void {
792 .system_pkg_dir_path = system_pkg_dir_path,794 .system_pkg_dir_path = system_pkg_dir_path,
793 .print_configuration = print_configuration,795 .print_configuration = print_configuration,
794 .forks = forks.items,796 .forks = forks.items,
797 .unlazy_set = &discovered_unlazy_set,
795 .src_files = &configure_source_files,798 .src_files = &configure_source_files,
796 .error_bundle = if (protocol_server != null) &configure_error_bundle else null,799 .error_bundle = if (protocol_server != null) &configure_error_bundle else null,
797 })) |scanned_config| {800 })) |scanned_config| {
801 discovered_unlazy_set.clearRetainingCapacity();
802
798 if (help_menu) {803 if (help_menu) {
799 scanned_config.printUsage(&graph, initStdoutWriter(io)) catch |err| switch (err) {804 scanned_config.printUsage(&graph, initStdoutWriter(io)) catch |err| switch (err) {
800 error.WriteFailed => return stdout_writer_allocation.err.?,805 error.WriteFailed => return stdout_writer_allocation.err.?,
...@@ -914,20 +919,15 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -914,20 +919,15 @@ pub fn main(init: process.Init.Minimal) !void {
914 try select.concurrent(.message, Server.receiveMessage, .{s});919 try select.concurrent(.message, Server.receiveMessage, .{s});
915920
916 maker.watch = body.flags.watch;921 maker.watch = body.flags.watch;
917 maker.prepare(steps, &configure_source_files) catch |err| switch (err) {922 maker.prepare(steps, &configure_source_files, &discovered_unlazy_set) catch |err| switch (err) {
918 error.DependencyLoopDetected => {923 error.OutOfMemory => |e| return e,
919 // TODO handle error.FailedButCacheIntact924 error.DependencyLoopDetected, error.InsufficientMemory => {
920 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};925 // TODO handle error.DependencyLoopDetected as error.FailedButCacheIntact
921 process.exit(1);926 // TODO handle error.InsufficientMemory as error.AlreadyReported
922 },
923 else => |e| return e,
924 };
925 maker.checkSufficientMemory() catch |err| switch (err) {
926 error.InsufficientMemory => {
927 // TODO handle as error.AlreadyReported
928 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};927 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
929 process.exit(1);928 process.exit(1);
930 },929 },
930 error.LazyDependencyNeeded => continue :configure,
931 };931 };
932932
933 try maker.makeSteps(main_progress_node, null);933 try maker.makeSteps(main_progress_node, null);
...@@ -985,20 +985,15 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -985,20 +985,15 @@ pub fn main(init: process.Init.Minimal) !void {
985 const initial_steps = try maker.resolveTopLevelSteps(step_names.items);985 const initial_steps = try maker.resolveTopLevelSteps(step_names.items);
986 defer gpa.free(initial_steps);986 defer gpa.free(initial_steps);
987987
988 maker.prepare(initial_steps, &configure_source_files) catch |err| switch (err) {988 maker.prepare(initial_steps, &configure_source_files, &discovered_unlazy_set) catch |err| switch (err) {
989 error.DependencyLoopDetected => {989 error.OutOfMemory => |e| return e,
990 // TODO handle as error.FailedButCacheIntact990 error.DependencyLoopDetected, error.InsufficientMemory => {
991 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};991 // TODO handle error.DependencyLoopDetected as error.FailedButCacheIntact
992 process.exit(1);992 // TODO handle error.InsufficientMemory as error.AlreadyReported
993 },
994 else => |e| return e,
995 };
996 maker.checkSufficientMemory() catch |err| switch (err) {
997 error.InsufficientMemory => {
998 // TODO handle as error.AlreadyReported
999 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};993 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
1000 process.exit(1);994 process.exit(1);
1001 },995 },
996 error.LazyDependencyNeeded => continue :configure,
1002 };997 };
1003998
1004 var w: Watch = w: {999 var w: Watch = w: {
...@@ -1146,6 +1141,7 @@ const ConfigureOptions = struct {...@@ -1146,6 +1141,7 @@ const ConfigureOptions = struct {
1146 system_pkg_dir_path: ?[]const u8,1141 system_pkg_dir_path: ?[]const u8,
1147 print_configuration: PrintConfiguration,1142 print_configuration: PrintConfiguration,
1148 forks: []Fork,1143 forks: []Fork,
1144 unlazy_set: *Fetch.JobQueue.UnlazySet,
1149 src_files: *Cache.Manifest.SelfContainedFiles,1145 src_files: *Cache.Manifest.SelfContainedFiles,
1150 error_bundle: ?*ErrorBundle = null,1146 error_bundle: ?*ErrorBundle = null,
1151};1147};
...@@ -1161,8 +1157,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1161,8 +1157,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
1161 var http_client: std.http.Client = .{ .allocator = gpa, .io = io };1157 var http_client: std.http.Client = .{ .allocator = gpa, .io = io };
1162 defer http_client.deinit();1158 defer http_client.deinit();
11631159
1164 var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{};1160 var fork_set: Fetch.JobQueue.ForkSet = .{};
1165 var fork_set: Package.Fetch.JobQueue.ForkSet = .{};
11661161
1167 {1162 {
1168 // Populate fork_set.1163 // Populate fork_set.
...@@ -1297,7 +1292,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1297,7 +1292,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
1297 // Reset fork match counts.1292 // Reset fork match counts.
1298 for (fork_set.keys()) |*fork| fork.uses = 0;1293 for (fork_set.keys()) |*fork| fork.uses = 0;
12991294
1300 var job_queue: Package.Fetch.JobQueue = .{1295 var job_queue: Fetch.JobQueue = .{
1301 .io = io,1296 .io = io,
1302 .http_client = &http_client,1297 .http_client = &http_client,
1303 .global_cache = graph.global_cache_root,1298 .global_cache = graph.global_cache_root,
...@@ -1307,7 +1302,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1307,7 +1302,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
1307 },1302 },
1308 .recursive = true,1303 .recursive = true,
1309 .debug_hash = false,1304 .debug_hash = false,
1310 .unlazy_set = unlazy_set,1305 .unlazy_set = options.unlazy_set.*,
1311 .fork_set = fork_set,1306 .fork_set = fork_set,
1312 .mode = options.fetch_mode,1307 .mode = options.fetch_mode,
1313 .prog_node = fetch_prog_node,1308 .prog_node = fetch_prog_node,
...@@ -1324,7 +1319,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1324,7 +1319,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
13241319
1325 const phantom_package_root: Cache.Path = .{ .root_dir = options.build_root.directory };1320 const phantom_package_root: Cache.Path = .{ .root_dir = options.build_root.directory };
13261321
1327 var fetch: Package.Fetch = .{1322 var fetch: Fetch = .{
1328 .arena = std.heap.ArenaAllocator.init(gpa),1323 .arena = std.heap.ArenaAllocator.init(gpa),
1329 .location = .{ .relative_path = phantom_package_root },1324 .location = .{ .relative_path = phantom_package_root },
1330 .location_tok = 0,1325 .location_tok = 0,
...@@ -1356,11 +1351,11 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1356,11 +1351,11 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
1356 job_queue.all_fetches.appendAssumeCapacity(&fetch);1351 job_queue.all_fetches.appendAssumeCapacity(&fetch);
13571352
1358 job_queue.table.putAssumeCapacityNoClobber(1353 job_queue.table.putAssumeCapacityNoClobber(
1359 Package.Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root),1354 Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root),
1360 &fetch,1355 &fetch,
1361 );1356 );
13621357
1363 job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" });1358 job_queue.group.async(io, Fetch.workerRun, .{ &fetch, "root" });
1364 try job_queue.group.await(io);1359 try job_queue.group.await(io);
13651360
1366 {1361 {
...@@ -1501,7 +1496,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1501,7 +1496,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
1501 const dep_names = man.dependencies.keys();1496 const dep_names = man.dependencies.keys();
1502 try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len));1497 try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len));
1503 for (dep_names, man.dependencies.values()) |name, *dep| {1498 for (dep_names, man.dependencies.values()) |name, *dep| {
1504 const dep_digest = Package.Fetch.depDigest(1499 const dep_digest = Fetch.depDigest(
1505 f.package_root,1500 f.package_root,
1506 graph.global_cache_root,1501 graph.global_cache_root,
1507 dep,1502 dep,
...@@ -1620,38 +1615,33 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1620,38 +1615,33 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
16201615
1621 if (configuration.unlazy_deps.len != 0) {1616 if (configuration.unlazy_deps.len != 0) {
1622 var any_errors = false;1617 var any_errors = false;
1618 options.unlazy_set.clearRetainingCapacity();
1623 for (configuration.unlazy_deps) |unlazy_dep| {1619 for (configuration.unlazy_deps) |unlazy_dep| {
1624 if (unlazy_dep.step.unwrap()) |step_index| {1620 if (unlazy_dep.step != .none) continue; // Handled in `prepare`.
1625 log.err("TODO implement step-associated lazy dependencies in maker", .{});1621 for (unlazy_dep.list.slice(&configuration)) |hash_string| {
1626 any_errors = true;1622 const hash = hash_string.slice(&configuration);
1627 _ = step_index;1623 assert(hash.len != 0);
1628 } else {1624 if (hash.len > Package.Hash.max_len) {
1629 for (unlazy_dep.list.slice(&configuration)) |hash_string| {1625 log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash });
1630 const hash = hash_string.slice(&configuration);1626 any_errors = true;
1631 assert(hash.len != 0);1627 continue;
1632 if (hash.len > Package.Hash.max_len) {
1633 log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash });
1634 any_errors = true;
1635 continue;
1636 }
1637 log.info("fetching lazy dependency {s}", .{hash});
1638 try unlazy_set.put(arena, .fromSlice(hash), {});
1639 }1628 }
1629 log.info("fetching lazy dependency {s}", .{hash});
1630 try options.unlazy_set.put(arena, .fromSlice(hash), {});
1640 }1631 }
1641 }1632 }
1642 if (any_errors) return error.FailedButCacheIntact;1633 if (any_errors) return error.FailedButCacheIntact;
1643 if (options.system_pkg_dir_path) |p| {1634 if (options.system_pkg_dir_path) |p| {
1644 // In this mode, the system needs to provide these packages; they1635 // In this mode, the system needs to provide these packages; they cannot be fetched by Zig.
1645 // cannot be fetched by Zig.
1646 const s = Dir.path.sep_str;1636 const s = Dir.path.sep_str;
1647 for (unlazy_set.keys()) |*hash| {1637 for (options.unlazy_set.keys()) |*hash| {
1648 log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() });1638 log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() });
1649 }1639 }
1650 log.info("remote package fetching disabled due to --system mode", .{});1640 log.info("remote package fetching disabled due to --system mode", .{});
1651 log.info("dependencies might be avoidable depending on build configuration", .{});1641 log.info("dependencies might be avoidable depending on build configuration", .{});
1652 return error.FailedButCacheIntact;1642 return error.FailedButCacheIntact;
1653 }1643 }
1654 continue :cp;1644 if (options.unlazy_set.count() != 0) continue :cp;
1655 }1645 }
16561646
1657 if (config_man) |man| for (configuration.path_deps) |path_dep| {1647 if (config_man) |man| for (configuration.path_deps) |path_dep| {
...@@ -1791,7 +1781,7 @@ const ValidFingerprintTable = std.array_hash_map.Custom(...@@ -1791,7 +1781,7 @@ const ValidFingerprintTable = std.array_hash_map.Custom(
1791fn validateFingerprints(1781fn validateFingerprints(
1792 gpa: Allocator,1782 gpa: Allocator,
1793 job_queue: *const Fetch.JobQueue,1783 job_queue: *const Fetch.JobQueue,
1794 fork_set: *const Package.Fetch.JobQueue.ForkSet,1784 fork_set: *const Fetch.JobQueue.ForkSet,
1795) !void {1785) !void {
1796 var any_errors = false;1786 var any_errors = false;
17971787
...@@ -1846,7 +1836,7 @@ fn printPkgTree(...@@ -1846,7 +1836,7 @@ fn printPkgTree(
1846 global_cache_root: Cache.Directory,1836 global_cache_root: Cache.Directory,
1847 job_queue: *const Fetch.JobQueue,1837 job_queue: *const Fetch.JobQueue,
1848 root_fetch: *const Fetch,1838 root_fetch: *const Fetch,
1849 fork_set: *const Package.Fetch.JobQueue.ForkSet,1839 fork_set: *const Fetch.JobQueue.ForkSet,
1850) !void {1840) !void {
1851 if (!root_fetch.have_manifest) return;1841 if (!root_fetch.have_manifest) return;
1852 const names = root_fetch.manifest.dependencies.keys();1842 const names = root_fetch.manifest.dependencies.keys();
...@@ -1861,7 +1851,7 @@ fn printPkgTree(...@@ -1861,7 +1851,7 @@ fn printPkgTree(
1861 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);1851 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);
1862 },1852 },
1863 .path => {1853 .path => {
1864 const child_hash = Package.Fetch.depDigest(root_fetch.package_root, global_cache_root, dep);1854 const child_hash = Fetch.depDigest(root_fetch.package_root, global_cache_root, dep);
1865 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);1855 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);
1866 },1856 },
1867 }1857 }
...@@ -1876,7 +1866,7 @@ fn printPkgTreeChild(...@@ -1876,7 +1866,7 @@ fn printPkgTreeChild(
1876 parent_print_node: *const PrintNode,1866 parent_print_node: *const PrintNode,
1877 dep_name: []const u8,1867 dep_name: []const u8,
1878 opt_pkg_hash: ?Package.Hash,1868 opt_pkg_hash: ?Package.Hash,
1879 fork_set: *const Package.Fetch.JobQueue.ForkSet,1869 fork_set: *const Fetch.JobQueue.ForkSet,
1880) !void {1870) !void {
1881 const w = terminal.writer;1871 const w = terminal.writer;
1882 try parent_print_node.printPrefixedChild(w, terminal.mode);1872 try parent_print_node.printPrefixedChild(w, terminal.mode);
...@@ -1933,7 +1923,7 @@ fn printPkgTreeChild(...@@ -1933,7 +1923,7 @@ fn printPkgTreeChild(
1933 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);1923 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);
1934 },1924 },
1935 .path => {1925 .path => {
1936 const child_hash = Package.Fetch.depDigest(child_fetch.package_root, global_cache_root, dep);1926 const child_hash = Fetch.depDigest(child_fetch.package_root, global_cache_root, dep);
1937 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);1927 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);
1938 },1928 },
1939 }1929 }
...@@ -2660,7 +2650,8 @@ fn prepare(...@@ -2660,7 +2650,8 @@ fn prepare(
2660 maker: *Maker,2650 maker: *Maker,
2661 step_indices: []const Configuration.Step.Index,2651 step_indices: []const Configuration.Step.Index,
2662 configure_source_files: *const Cache.Manifest.SelfContainedFiles,2652 configure_source_files: *const Cache.Manifest.SelfContainedFiles,
2663) !void {2653 discovered_unlazy_set: *Fetch.JobQueue.UnlazySet,
2654) error{ InsufficientMemory, OutOfMemory, DependencyLoopDetected, LazyDependencyNeeded }!void {
2664 const gpa = maker.gpa;2655 const gpa = maker.gpa;
2665 const graph = maker.graph;2656 const graph = maker.graph;
2666 const arena = graph.arena;2657 const arena = graph.arena;
...@@ -2707,6 +2698,30 @@ fn prepare(...@@ -2707,6 +2698,30 @@ fn prepare(
2707 for (starting_steps) |s| {2698 for (starting_steps) |s| {
2708 try constructGraphAndCheckForDependencyLoop(maker, s, &maker.step_stack, rand);2699 try constructGraphAndCheckForDependencyLoop(maker, s, &maker.step_stack, rand);
2709 }2700 }
2701
2702 // Notice when a step's participation in the evaluation graph implies that additional package dependencies
2703 // must be fetched.
2704 var needed_step_deps: std.array_hash_map.Auto(Configuration.Step.Index, Configuration.StringList) = .empty;
2705 defer needed_step_deps.deinit(gpa);
2706
2707 for (c.unlazy_deps) |unlazy_dep| {
2708 const step_index = unlazy_dep.step.unwrap() orelse continue; // Handled in `configure`.
2709 try needed_step_deps.put(gpa, step_index, unlazy_dep.list);
2710 }
2711
2712 assert(discovered_unlazy_set.count() == 0);
2713 for (step_stack.keys()) |step_index| {
2714 const interned_string_list = needed_step_deps.get(step_index) orelse continue;
2715 const hash_strings = interned_string_list.slice(c);
2716 for (hash_strings) |hash_string| {
2717 const hash = hash_string.slice(c);
2718 log.debug("due to step {s}, fetching lazy {s}", .{ step_index.ptr(c).name.slice(c), hash });
2719 try discovered_unlazy_set.put(arena, .fromSlice(hash), {});
2720 }
2721 }
2722 if (discovered_unlazy_set.count() != 0) return error.LazyDependencyNeeded;
2723
2724 try checkSufficientMemory(maker);
2710}2725}
27112726
2712fn checkSufficientMemory(maker: *Maker) error{InsufficientMemory}!void {2727fn checkSufficientMemory(maker: *Maker) error{InsufficientMemory}!void {
...@@ -3464,17 +3479,17 @@ fn printTreeChildLast(writer: *Writer, mode: Io.Terminal.Mode) !void {...@@ -3464,17 +3479,17 @@ fn printTreeChildLast(writer: *Writer, mode: Io.Terminal.Mode) !void {
3464 });3479 });
3465}3480}
34663481
3467/// Traverse the dependency graph depth-first and make it undirected by having3482/// Traverse the dependency graph depth-first and make it undirected by having steps know their dependants
3468/// steps know their dependants (they only know dependencies at start).3483/// (they only know dependencies at start).
3469/// Along the way, check that there is no dependency loop, and record the steps3484///
3470/// in traversal order in `step_stack`.3485/// Along the way, check that there is no dependency loop, and record the steps in traversal order in
3471/// Each step has its dependencies traversed in random order, this accomplishes3486/// `step_stack`.
3472/// two things:3487///
3473/// - `step_stack` will be in randomized-depth-first order, so the build runner3488/// Each step has its dependencies traversed in random order, this accomplishes two things:
3474/// spawns initial steps in a random order3489/// - `step_stack` will be in randomized-depth-first order, so the build runner spawns initial steps in a
3475/// - each step's `dependants` list is also filled in a random order, so that
3476/// when it finishes executing in `makeStep`, it spawns next steps to run in
3477/// random order3490/// random order
3491/// - each step's `dependants` list is also filled in a random order, so that when it finishes executing in
3492/// `makeStep`, it spawns next steps to run in random order
3478fn constructGraphAndCheckForDependencyLoop(3493fn constructGraphAndCheckForDependencyLoop(
3479 maker: *Maker,3494 maker: *Maker,
3480 step_index: Configuration.Step.Index,3495 step_index: Configuration.Step.Index,
lib/compiler/Maker/Fetch.zig+3-3
...@@ -153,11 +153,11 @@ pub const JobQueue = struct {...@@ -153,11 +153,11 @@ pub const JobQueue = struct {
153 debug_hash: bool,153 debug_hash: bool,
154 mode: Mode,154 mode: Mode,
155 /// Set of hashes that will be additionally fetched even if they are marked155 /// Set of hashes that will be additionally fetched even if they are marked
156 /// as lazy.156 /// as lazy. Read-only.
157 unlazy_set: UnlazySet = .{},157 unlazy_set: UnlazySet = .empty,
158 /// Identifies paths that override all packages in the tree with matching158 /// Identifies paths that override all packages in the tree with matching
159 /// project ids.159 /// project ids.
160 fork_set: ForkSet = .{},160 fork_set: ForkSet = .empty,
161161
162 pub const Mode = enum {162 pub const Mode = enum {
163 /// Non-lazy dependencies are always fetched.163 /// Non-lazy dependencies are always fetched.