| ... | ... | @@ -763,6 +763,8 @@ pub fn main(init: process.Init.Minimal) !void { |
| 763 | 763 | break :s &protocol_server_allocation; |
| 764 | 764 | } else null; |
| 765 | 765 | |
| 766 | var discovered_unlazy_set: Fetch.JobQueue.UnlazySet = .empty; // uses global arena |
| 767 | |
| 766 | 768 | configure: while (true) { |
| 767 | 769 | // Set of files that, if modified, imply that recompiling and rerunning |
| 768 | 770 | // configurer is needed. |
| ... | ... | @@ -792,9 +794,12 @@ pub fn main(init: process.Init.Minimal) !void { |
| 792 | 794 | .system_pkg_dir_path = system_pkg_dir_path, |
| 793 | 795 | .print_configuration = print_configuration, |
| 794 | 796 | .forks = forks.items, |
| 797 | .unlazy_set = &discovered_unlazy_set, |
| 795 | 798 | .src_files = &configure_source_files, |
| 796 | 799 | .error_bundle = if (protocol_server != null) &configure_error_bundle else null, |
| 797 | 800 | })) |scanned_config| { |
| 801 | discovered_unlazy_set.clearRetainingCapacity(); |
| 802 | |
| 798 | 803 | if (help_menu) { |
| 799 | 804 | scanned_config.printUsage(&graph, initStdoutWriter(io)) catch |err| switch (err) { |
| 800 | 805 | error.WriteFailed => return stdout_writer_allocation.err.?, |
| ... | ... | @@ -914,20 +919,15 @@ pub fn main(init: process.Init.Minimal) !void { |
| 914 | 919 | try select.concurrent(.message, Server.receiveMessage, .{s}); |
| 915 | 920 | |
| 916 | 921 | maker.watch = body.flags.watch; |
| 917 | | maker.prepare(steps, &configure_source_files) catch |err| switch (err) { |
| 918 | | error.DependencyLoopDetected => { |
| 919 | | // TODO handle error.FailedButCacheIntact |
| 920 | | _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; |
| 921 | | process.exit(1); |
| 922 | | }, |
| 923 | | else => |e| return e, |
| 924 | | }; |
| 925 | | maker.checkSufficientMemory() catch |err| switch (err) { |
| 926 | | error.InsufficientMemory => { |
| 927 | | // TODO handle as error.AlreadyReported |
| 922 | maker.prepare(steps, &configure_source_files, &discovered_unlazy_set) catch |err| switch (err) { |
| 923 | error.OutOfMemory => |e| return e, |
| 924 | error.DependencyLoopDetected, error.InsufficientMemory => { |
| 925 | // TODO handle error.DependencyLoopDetected as error.FailedButCacheIntact |
| 926 | // TODO handle error.InsufficientMemory as error.AlreadyReported |
| 928 | 927 | _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; |
| 929 | 928 | process.exit(1); |
| 930 | 929 | }, |
| 930 | error.LazyDependencyNeeded => continue :configure, |
| 931 | 931 | }; |
| 932 | 932 | |
| 933 | 933 | try maker.makeSteps(main_progress_node, null); |
| ... | ... | @@ -985,20 +985,15 @@ pub fn main(init: process.Init.Minimal) !void { |
| 985 | 985 | const initial_steps = try maker.resolveTopLevelSteps(step_names.items); |
| 986 | 986 | defer gpa.free(initial_steps); |
| 987 | 987 | |
| 988 | | maker.prepare(initial_steps, &configure_source_files) catch |err| switch (err) { |
| 989 | | error.DependencyLoopDetected => { |
| 990 | | // TODO handle as error.FailedButCacheIntact |
| 991 | | _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; |
| 992 | | process.exit(1); |
| 993 | | }, |
| 994 | | else => |e| return e, |
| 995 | | }; |
| 996 | | maker.checkSufficientMemory() catch |err| switch (err) { |
| 997 | | error.InsufficientMemory => { |
| 998 | | // TODO handle as error.AlreadyReported |
| 988 | maker.prepare(initial_steps, &configure_source_files, &discovered_unlazy_set) catch |err| switch (err) { |
| 989 | error.OutOfMemory => |e| return e, |
| 990 | error.DependencyLoopDetected, error.InsufficientMemory => { |
| 991 | // TODO handle error.DependencyLoopDetected as error.FailedButCacheIntact |
| 992 | // TODO handle error.InsufficientMemory as error.AlreadyReported |
| 999 | 993 | _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; |
| 1000 | 994 | process.exit(1); |
| 1001 | 995 | }, |
| 996 | error.LazyDependencyNeeded => continue :configure, |
| 1002 | 997 | }; |
| 1003 | 998 | |
| 1004 | 999 | var w: Watch = w: { |
| ... | ... | @@ -1146,6 +1141,7 @@ const ConfigureOptions = struct { |
| 1146 | 1141 | system_pkg_dir_path: ?[]const u8, |
| 1147 | 1142 | print_configuration: PrintConfiguration, |
| 1148 | 1143 | forks: []Fork, |
| 1144 | unlazy_set: *Fetch.JobQueue.UnlazySet, |
| 1149 | 1145 | src_files: *Cache.Manifest.SelfContainedFiles, |
| 1150 | 1146 | error_bundle: ?*ErrorBundle = null, |
| 1151 | 1147 | }; |
| ... | ... | @@ -1161,8 +1157,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { |
| 1161 | 1157 | var http_client: std.http.Client = .{ .allocator = gpa, .io = io }; |
| 1162 | 1158 | defer http_client.deinit(); |
| 1163 | 1159 | |
| 1164 | | var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; |
| 1165 | | var fork_set: Package.Fetch.JobQueue.ForkSet = .{}; |
| 1160 | var fork_set: Fetch.JobQueue.ForkSet = .{}; |
| 1166 | 1161 | |
| 1167 | 1162 | { |
| 1168 | 1163 | // Populate fork_set. |
| ... | ... | @@ -1297,7 +1292,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { |
| 1297 | 1292 | // Reset fork match counts. |
| 1298 | 1293 | for (fork_set.keys()) |*fork| fork.uses = 0; |
| 1299 | 1294 | |
| 1300 | | var job_queue: Package.Fetch.JobQueue = .{ |
| 1295 | var job_queue: Fetch.JobQueue = .{ |
| 1301 | 1296 | .io = io, |
| 1302 | 1297 | .http_client = &http_client, |
| 1303 | 1298 | .global_cache = graph.global_cache_root, |
| ... | ... | @@ -1307,7 +1302,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { |
| 1307 | 1302 | }, |
| 1308 | 1303 | .recursive = true, |
| 1309 | 1304 | .debug_hash = false, |
| 1310 | | .unlazy_set = unlazy_set, |
| 1305 | .unlazy_set = options.unlazy_set.*, |
| 1311 | 1306 | .fork_set = fork_set, |
| 1312 | 1307 | .mode = options.fetch_mode, |
| 1313 | 1308 | .prog_node = fetch_prog_node, |
| ... | ... | @@ -1324,7 +1319,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { |
| 1324 | 1319 | |
| 1325 | 1320 | const phantom_package_root: Cache.Path = .{ .root_dir = options.build_root.directory }; |
| 1326 | 1321 | |
| 1327 | | var fetch: Package.Fetch = .{ |
| 1322 | var fetch: Fetch = .{ |
| 1328 | 1323 | .arena = std.heap.ArenaAllocator.init(gpa), |
| 1329 | 1324 | .location = .{ .relative_path = phantom_package_root }, |
| 1330 | 1325 | .location_tok = 0, |
| ... | ... | @@ -1356,11 +1351,11 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { |
| 1356 | 1351 | job_queue.all_fetches.appendAssumeCapacity(&fetch); |
| 1357 | 1352 | |
| 1358 | 1353 | job_queue.table.putAssumeCapacityNoClobber( |
| 1359 | | Package.Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root), |
| 1354 | Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root), |
| 1360 | 1355 | &fetch, |
| 1361 | 1356 | ); |
| 1362 | 1357 | |
| 1363 | | job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" }); |
| 1358 | job_queue.group.async(io, Fetch.workerRun, .{ &fetch, "root" }); |
| 1364 | 1359 | try job_queue.group.await(io); |
| 1365 | 1360 | |
| 1366 | 1361 | { |
| ... | ... | @@ -1501,7 +1496,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { |
| 1501 | 1496 | const dep_names = man.dependencies.keys(); |
| 1502 | 1497 | try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); |
| 1503 | 1498 | for (dep_names, man.dependencies.values()) |name, *dep| { |
| 1504 | | const dep_digest = Package.Fetch.depDigest( |
| 1499 | const dep_digest = Fetch.depDigest( |
| 1505 | 1500 | f.package_root, |
| 1506 | 1501 | graph.global_cache_root, |
| 1507 | 1502 | dep, |
| ... | ... | @@ -1620,38 +1615,33 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { |
| 1620 | 1615 | |
| 1621 | 1616 | if (configuration.unlazy_deps.len != 0) { |
| 1622 | 1617 | var any_errors = false; |
| 1618 | options.unlazy_set.clearRetainingCapacity(); |
| 1623 | 1619 | for (configuration.unlazy_deps) |unlazy_dep| { |
| 1624 | | if (unlazy_dep.step.unwrap()) |step_index| { |
| 1625 | | log.err("TODO implement step-associated lazy dependencies in maker", .{}); |
| 1626 | | any_errors = true; |
| 1627 | | _ = step_index; |
| 1628 | | } else { |
| 1629 | | for (unlazy_dep.list.slice(&configuration)) |hash_string| { |
| 1630 | | const hash = hash_string.slice(&configuration); |
| 1631 | | assert(hash.len != 0); |
| 1632 | | if (hash.len > Package.Hash.max_len) { |
| 1633 | | log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash }); |
| 1634 | | any_errors = true; |
| 1635 | | continue; |
| 1636 | | } |
| 1637 | | log.info("fetching lazy dependency {s}", .{hash}); |
| 1638 | | try unlazy_set.put(arena, .fromSlice(hash), {}); |
| 1620 | if (unlazy_dep.step != .none) continue; // Handled in `prepare`. |
| 1621 | for (unlazy_dep.list.slice(&configuration)) |hash_string| { |
| 1622 | const hash = hash_string.slice(&configuration); |
| 1623 | assert(hash.len != 0); |
| 1624 | if (hash.len > Package.Hash.max_len) { |
| 1625 | log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash }); |
| 1626 | any_errors = true; |
| 1627 | continue; |
| 1639 | 1628 | } |
| 1629 | log.info("fetching lazy dependency {s}", .{hash}); |
| 1630 | try options.unlazy_set.put(arena, .fromSlice(hash), {}); |
| 1640 | 1631 | } |
| 1641 | 1632 | } |
| 1642 | 1633 | if (any_errors) return error.FailedButCacheIntact; |
| 1643 | 1634 | if (options.system_pkg_dir_path) |p| { |
| 1644 | | // In this mode, the system needs to provide these packages; they |
| 1645 | | // cannot be fetched by Zig. |
| 1635 | // In this mode, the system needs to provide these packages; they cannot be fetched by Zig. |
| 1646 | 1636 | const s = Dir.path.sep_str; |
| 1647 | | for (unlazy_set.keys()) |*hash| { |
| 1637 | for (options.unlazy_set.keys()) |*hash| { |
| 1648 | 1638 | log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() }); |
| 1649 | 1639 | } |
| 1650 | 1640 | log.info("remote package fetching disabled due to --system mode", .{}); |
| 1651 | 1641 | log.info("dependencies might be avoidable depending on build configuration", .{}); |
| 1652 | 1642 | return error.FailedButCacheIntact; |
| 1653 | 1643 | } |
| 1654 | | continue :cp; |
| 1644 | if (options.unlazy_set.count() != 0) continue :cp; |
| 1655 | 1645 | } |
| 1656 | 1646 | |
| 1657 | 1647 | if (config_man) |man| for (configuration.path_deps) |path_dep| { |
| ... | ... | @@ -1791,7 +1781,7 @@ const ValidFingerprintTable = std.array_hash_map.Custom( |
| 1791 | 1781 | fn validateFingerprints( |
| 1792 | 1782 | gpa: Allocator, |
| 1793 | 1783 | job_queue: *const Fetch.JobQueue, |
| 1794 | | fork_set: *const Package.Fetch.JobQueue.ForkSet, |
| 1784 | fork_set: *const Fetch.JobQueue.ForkSet, |
| 1795 | 1785 | ) !void { |
| 1796 | 1786 | var any_errors = false; |
| 1797 | 1787 | |
| ... | ... | @@ -1846,7 +1836,7 @@ fn printPkgTree( |
| 1846 | 1836 | global_cache_root: Cache.Directory, |
| 1847 | 1837 | job_queue: *const Fetch.JobQueue, |
| 1848 | 1838 | root_fetch: *const Fetch, |
| 1849 | | fork_set: *const Package.Fetch.JobQueue.ForkSet, |
| 1839 | fork_set: *const Fetch.JobQueue.ForkSet, |
| 1850 | 1840 | ) !void { |
| 1851 | 1841 | if (!root_fetch.have_manifest) return; |
| 1852 | 1842 | const names = root_fetch.manifest.dependencies.keys(); |
| ... | ... | @@ -1861,7 +1851,7 @@ fn printPkgTree( |
| 1861 | 1851 | try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set); |
| 1862 | 1852 | }, |
| 1863 | 1853 | .path => { |
| 1864 | | const child_hash = Package.Fetch.depDigest(root_fetch.package_root, global_cache_root, dep); |
| 1854 | const child_hash = Fetch.depDigest(root_fetch.package_root, global_cache_root, dep); |
| 1865 | 1855 | try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set); |
| 1866 | 1856 | }, |
| 1867 | 1857 | } |
| ... | ... | @@ -1876,7 +1866,7 @@ fn printPkgTreeChild( |
| 1876 | 1866 | parent_print_node: *const PrintNode, |
| 1877 | 1867 | dep_name: []const u8, |
| 1878 | 1868 | opt_pkg_hash: ?Package.Hash, |
| 1879 | | fork_set: *const Package.Fetch.JobQueue.ForkSet, |
| 1869 | fork_set: *const Fetch.JobQueue.ForkSet, |
| 1880 | 1870 | ) !void { |
| 1881 | 1871 | const w = terminal.writer; |
| 1882 | 1872 | try parent_print_node.printPrefixedChild(w, terminal.mode); |
| ... | ... | @@ -1933,7 +1923,7 @@ fn printPkgTreeChild( |
| 1933 | 1923 | try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set); |
| 1934 | 1924 | }, |
| 1935 | 1925 | .path => { |
| 1936 | | const child_hash = Package.Fetch.depDigest(child_fetch.package_root, global_cache_root, dep); |
| 1926 | const child_hash = Fetch.depDigest(child_fetch.package_root, global_cache_root, dep); |
| 1937 | 1927 | try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set); |
| 1938 | 1928 | }, |
| 1939 | 1929 | } |
| ... | ... | @@ -2660,7 +2650,8 @@ fn prepare( |
| 2660 | 2650 | maker: *Maker, |
| 2661 | 2651 | step_indices: []const Configuration.Step.Index, |
| 2662 | 2652 | configure_source_files: *const Cache.Manifest.SelfContainedFiles, |
| 2663 | | ) !void { |
| 2653 | discovered_unlazy_set: *Fetch.JobQueue.UnlazySet, |
| 2654 | ) error{ InsufficientMemory, OutOfMemory, DependencyLoopDetected, LazyDependencyNeeded }!void { |
| 2664 | 2655 | const gpa = maker.gpa; |
| 2665 | 2656 | const graph = maker.graph; |
| 2666 | 2657 | const arena = graph.arena; |
| ... | ... | @@ -2707,6 +2698,30 @@ fn prepare( |
| 2707 | 2698 | for (starting_steps) |s| { |
| 2708 | 2699 | try constructGraphAndCheckForDependencyLoop(maker, s, &maker.step_stack, rand); |
| 2709 | 2700 | } |
| 2701 | |
| 2702 | // Notice when a step's participation in the evaluation graph implies that additional package dependencies |
| 2703 | // must be fetched. |
| 2704 | var needed_step_deps: std.array_hash_map.Auto(Configuration.Step.Index, Configuration.StringList) = .empty; |
| 2705 | defer needed_step_deps.deinit(gpa); |
| 2706 | |
| 2707 | for (c.unlazy_deps) |unlazy_dep| { |
| 2708 | const step_index = unlazy_dep.step.unwrap() orelse continue; // Handled in `configure`. |
| 2709 | try needed_step_deps.put(gpa, step_index, unlazy_dep.list); |
| 2710 | } |
| 2711 | |
| 2712 | assert(discovered_unlazy_set.count() == 0); |
| 2713 | for (step_stack.keys()) |step_index| { |
| 2714 | const interned_string_list = needed_step_deps.get(step_index) orelse continue; |
| 2715 | const hash_strings = interned_string_list.slice(c); |
| 2716 | for (hash_strings) |hash_string| { |
| 2717 | const hash = hash_string.slice(c); |
| 2718 | log.debug("due to step {s}, fetching lazy {s}", .{ step_index.ptr(c).name.slice(c), hash }); |
| 2719 | try discovered_unlazy_set.put(arena, .fromSlice(hash), {}); |
| 2720 | } |
| 2721 | } |
| 2722 | if (discovered_unlazy_set.count() != 0) return error.LazyDependencyNeeded; |
| 2723 | |
| 2724 | try checkSufficientMemory(maker); |
| 2710 | 2725 | } |
| 2711 | 2726 | |
| 2712 | 2727 | fn checkSufficientMemory(maker: *Maker) error{InsufficientMemory}!void { |
| ... | ... | @@ -3464,17 +3479,17 @@ fn printTreeChildLast(writer: *Writer, mode: Io.Terminal.Mode) !void { |
| 3464 | 3479 | }); |
| 3465 | 3480 | } |
| 3466 | 3481 | |
| 3467 | | /// Traverse the dependency graph depth-first and make it undirected by having |
| 3468 | | /// steps know their dependants (they only know dependencies at start). |
| 3469 | | /// Along the way, check that there is no dependency loop, and record the steps |
| 3470 | | /// in traversal order in `step_stack`. |
| 3471 | | /// Each step has its dependencies traversed in random order, this accomplishes |
| 3472 | | /// two things: |
| 3473 | | /// - `step_stack` will be in randomized-depth-first order, so the build runner |
| 3474 | | /// spawns initial steps in a random order |
| 3475 | | /// - each step's `dependants` list is also filled in a random order, so that |
| 3476 | | /// when it finishes executing in `makeStep`, it spawns next steps to run in |
| 3482 | /// Traverse the dependency graph depth-first and make it undirected by having steps know their dependants |
| 3483 | /// (they only know dependencies at start). |
| 3484 | /// |
| 3485 | /// Along the way, check that there is no dependency loop, and record the steps in traversal order in |
| 3486 | /// `step_stack`. |
| 3487 | /// |
| 3488 | /// Each step has its dependencies traversed in random order, this accomplishes two things: |
| 3489 | /// - `step_stack` will be in randomized-depth-first order, so the build runner spawns initial steps in a |
| 3477 | 3490 | /// random order |
| 3491 | /// - each step's `dependants` list is also filled in a random order, so that when it finishes executing in |
| 3492 | /// `makeStep`, it spawns next steps to run in random order |
| 3478 | 3493 | fn constructGraphAndCheckForDependencyLoop( |
| 3479 | 3494 | maker: *Maker, |
| 3480 | 3495 | step_index: Configuration.Step.Index, |
Andrew Kelley <andrew@ziglang.org> 2026-10-15 18:01:23-07:00