authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-10-16 21:24:32+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-10-16 21:24:32+02:00
log4e293f3e4d6c7072962513bee0ca8643becb81aa
treeaf44b12756c31630503f4379981ec261002a6ee7
parentd29e4bd653efec872c83d66419a846eebf6c359a
parent156ab92d9de233a712cb29b6ac0ccc5780c0c18f

Merge pull request 'ability to explicitly declare dependencies, and ability to associate dependency with particular steps' (#37269) from lazy-step-dependencies into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/37269

7 files changed, 263 insertions(+), 103 deletions(-)

lib/compiler/Maker.zig+113-75
......@@ -763,6 +763,8 @@ pub fn main(init: process.Init.Minimal) !void {
763763 break :s &protocol_server_allocation;
764764 } else null;
765765
766 var discovered_unlazy_set: Fetch.JobQueue.UnlazySet = .empty; // uses global arena
767
766768 configure: while (true) {
767769 // Set of files that, if modified, imply that recompiling and rerunning
768770 // configurer is needed.
......@@ -792,9 +794,12 @@ pub fn main(init: process.Init.Minimal) !void {
792794 .system_pkg_dir_path = system_pkg_dir_path,
793795 .print_configuration = print_configuration,
794796 .forks = forks.items,
797 .unlazy_set = &discovered_unlazy_set,
795798 .src_files = &configure_source_files,
796799 .error_bundle = if (protocol_server != null) &configure_error_bundle else null,
797800 })) |scanned_config| {
801 discovered_unlazy_set.clearRetainingCapacity();
802
798803 if (help_menu) {
799804 scanned_config.printUsage(&graph, initStdoutWriter(io)) catch |err| switch (err) {
800805 error.WriteFailed => return stdout_writer_allocation.err.?,
......@@ -914,14 +919,15 @@ pub fn main(init: process.Init.Minimal) !void {
914919 try select.concurrent(.message, Server.receiveMessage, .{s});
915920
916921 maker.watch = body.flags.watch;
917 maker.prepare(steps, &configure_source_files) catch |err| switch (err) {
922 maker.prepare(steps, &configure_source_files, &discovered_unlazy_set) catch |err| switch (err) {
923 error.OutOfMemory => |e| return e,
918924 error.DependencyLoopDetected, error.InsufficientMemory => {
919 // TODO handle DependencyLoopDetected as error.FailedButCacheIntact
920 // and handle InsufficientMemory as error.AlreadyReported
925 // TODO handle error.DependencyLoopDetected as error.FailedButCacheIntact
926 // TODO handle error.InsufficientMemory as error.AlreadyReported
921927 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
922928 process.exit(1);
923929 },
924 else => |e| return e,
930 error.LazyDependencyNeeded => continue :configure,
925931 };
926932
927933 try maker.makeSteps(main_progress_node, null);
......@@ -979,14 +985,15 @@ pub fn main(init: process.Init.Minimal) !void {
979985 const initial_steps = try maker.resolveTopLevelSteps(step_names.items);
980986 defer gpa.free(initial_steps);
981987
982 maker.prepare(initial_steps, &configure_source_files) catch |err| switch (err) {
988 maker.prepare(initial_steps, &configure_source_files, &discovered_unlazy_set) catch |err| switch (err) {
989 error.OutOfMemory => |e| return e,
983990 error.DependencyLoopDetected, error.InsufficientMemory => {
984 // TODO handle DependencyLoopDetected as error.FailedButCacheIntact
985 // and handle InsufficientMemory as error.AlreadyReported
991 // TODO handle error.DependencyLoopDetected as error.FailedButCacheIntact
992 // TODO handle error.InsufficientMemory as error.AlreadyReported
986993 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
987994 process.exit(1);
988995 },
989 else => |e| return e,
996 error.LazyDependencyNeeded => continue :configure,
990997 };
991998
992999 var w: Watch = w: {
......@@ -1134,6 +1141,7 @@ const ConfigureOptions = struct {
11341141 system_pkg_dir_path: ?[]const u8,
11351142 print_configuration: PrintConfiguration,
11361143 forks: []Fork,
1144 unlazy_set: *Fetch.JobQueue.UnlazySet,
11371145 src_files: *Cache.Manifest.SelfContainedFiles,
11381146 error_bundle: ?*ErrorBundle = null,
11391147};
......@@ -1149,8 +1157,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
11491157 var http_client: std.http.Client = .{ .allocator = gpa, .io = io };
11501158 defer http_client.deinit();
11511159
1152 var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{};
1153 var fork_set: Package.Fetch.JobQueue.ForkSet = .{};
1160 var fork_set: Fetch.JobQueue.ForkSet = .{};
11541161
11551162 {
11561163 // Populate fork_set.
......@@ -1285,7 +1292,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
12851292 // Reset fork match counts.
12861293 for (fork_set.keys()) |*fork| fork.uses = 0;
12871294
1288 var job_queue: Package.Fetch.JobQueue = .{
1295 var job_queue: Fetch.JobQueue = .{
12891296 .io = io,
12901297 .http_client = &http_client,
12911298 .global_cache = graph.global_cache_root,
......@@ -1295,7 +1302,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
12951302 },
12961303 .recursive = true,
12971304 .debug_hash = false,
1298 .unlazy_set = unlazy_set,
1305 .unlazy_set = options.unlazy_set.*,
12991306 .fork_set = fork_set,
13001307 .mode = options.fetch_mode,
13011308 .prog_node = fetch_prog_node,
......@@ -1312,7 +1319,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
13121319
13131320 const phantom_package_root: Cache.Path = .{ .root_dir = options.build_root.directory };
13141321
1315 var fetch: Package.Fetch = .{
1322 var fetch: Fetch = .{
13161323 .arena = std.heap.ArenaAllocator.init(gpa),
13171324 .location = .{ .relative_path = phantom_package_root },
13181325 .location_tok = 0,
......@@ -1344,11 +1351,11 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
13441351 job_queue.all_fetches.appendAssumeCapacity(&fetch);
13451352
13461353 job_queue.table.putAssumeCapacityNoClobber(
1347 Package.Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root),
1354 Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root),
13481355 &fetch,
13491356 );
13501357
1351 job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" });
1358 job_queue.group.async(io, Fetch.workerRun, .{ &fetch, "root" });
13521359 try job_queue.group.await(io);
13531360
13541361 {
......@@ -1489,7 +1496,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
14891496 const dep_names = man.dependencies.keys();
14901497 try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len));
14911498 for (dep_names, man.dependencies.values()) |name, *dep| {
1492 const dep_digest = Package.Fetch.depDigest(
1499 const dep_digest = Fetch.depDigest(
14931500 f.package_root,
14941501 graph.global_cache_root,
14951502 dep,
......@@ -1608,30 +1615,33 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
16081615
16091616 if (configuration.unlazy_deps.len != 0) {
16101617 var any_errors = false;
1611 for (configuration.unlazy_deps) |hash_string| {
1612 const hash = hash_string.slice(&configuration);
1613 assert(hash.len != 0);
1614 if (hash.len > Package.Hash.max_len) {
1615 log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash });
1616 any_errors = true;
1617 continue;
1618 options.unlazy_set.clearRetainingCapacity();
1619 for (configuration.unlazy_deps) |unlazy_dep| {
1620 if (unlazy_dep.step != .none) continue; // Handled in `prepare`.
1621 for (unlazy_dep.list.slice(&configuration)) |hash_string| {
1622 const hash = hash_string.slice(&configuration);
1623 assert(hash.len != 0);
1624 if (hash.len > Package.Hash.max_len) {
1625 log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash });
1626 any_errors = true;
1627 continue;
1628 }
1629 log.info("fetching lazy dependency {s}", .{hash});
1630 try options.unlazy_set.put(arena, .fromSlice(hash), {});
16181631 }
1619 log.info("fetching lazy dependency {s}", .{hash});
1620 try unlazy_set.put(arena, .fromSlice(hash), {});
16211632 }
16221633 if (any_errors) return error.FailedButCacheIntact;
16231634 if (options.system_pkg_dir_path) |p| {
1624 // In this mode, the system needs to provide these packages; they
1625 // cannot be fetched by Zig.
1635 // In this mode, the system needs to provide these packages; they cannot be fetched by Zig.
16261636 const s = Dir.path.sep_str;
1627 for (unlazy_set.keys()) |*hash| {
1637 for (options.unlazy_set.keys()) |*hash| {
16281638 log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() });
16291639 }
16301640 log.info("remote package fetching disabled due to --system mode", .{});
16311641 log.info("dependencies might be avoidable depending on build configuration", .{});
16321642 return error.FailedButCacheIntact;
16331643 }
1634 continue :cp;
1644 if (options.unlazy_set.count() != 0) continue :cp;
16351645 }
16361646
16371647 if (config_man) |man| for (configuration.path_deps) |path_dep| {
......@@ -1771,7 +1781,7 @@ const ValidFingerprintTable = std.array_hash_map.Custom(
17711781fn validateFingerprints(
17721782 gpa: Allocator,
17731783 job_queue: *const Fetch.JobQueue,
1774 fork_set: *const Package.Fetch.JobQueue.ForkSet,
1784 fork_set: *const Fetch.JobQueue.ForkSet,
17751785) !void {
17761786 var any_errors = false;
17771787
......@@ -1826,7 +1836,7 @@ fn printPkgTree(
18261836 global_cache_root: Cache.Directory,
18271837 job_queue: *const Fetch.JobQueue,
18281838 root_fetch: *const Fetch,
1829 fork_set: *const Package.Fetch.JobQueue.ForkSet,
1839 fork_set: *const Fetch.JobQueue.ForkSet,
18301840) !void {
18311841 if (!root_fetch.have_manifest) return;
18321842 const names = root_fetch.manifest.dependencies.keys();
......@@ -1841,7 +1851,7 @@ fn printPkgTree(
18411851 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);
18421852 },
18431853 .path => {
1844 const child_hash = Package.Fetch.depDigest(root_fetch.package_root, global_cache_root, dep);
1854 const child_hash = Fetch.depDigest(root_fetch.package_root, global_cache_root, dep);
18451855 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);
18461856 },
18471857 }
......@@ -1856,7 +1866,7 @@ fn printPkgTreeChild(
18561866 parent_print_node: *const PrintNode,
18571867 dep_name: []const u8,
18581868 opt_pkg_hash: ?Package.Hash,
1859 fork_set: *const Package.Fetch.JobQueue.ForkSet,
1869 fork_set: *const Fetch.JobQueue.ForkSet,
18601870) !void {
18611871 const w = terminal.writer;
18621872 try parent_print_node.printPrefixedChild(w, terminal.mode);
......@@ -1913,7 +1923,7 @@ fn printPkgTreeChild(
19131923 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);
19141924 },
19151925 .path => {
1916 const child_hash = Package.Fetch.depDigest(child_fetch.package_root, global_cache_root, dep);
1926 const child_hash = Fetch.depDigest(child_fetch.package_root, global_cache_root, dep);
19171927 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);
19181928 },
19191929 }
......@@ -2640,7 +2650,8 @@ fn prepare(
26402650 maker: *Maker,
26412651 step_indices: []const Configuration.Step.Index,
26422652 configure_source_files: *const Cache.Manifest.SelfContainedFiles,
2643) !void {
2653 discovered_unlazy_set: *Fetch.JobQueue.UnlazySet,
2654) error{ InsufficientMemory, OutOfMemory, DependencyLoopDetected, LazyDependencyNeeded }!void {
26442655 const gpa = maker.gpa;
26452656 const graph = maker.graph;
26462657 const arena = graph.arena;
......@@ -2688,40 +2699,67 @@ fn prepare(
26882699 try constructGraphAndCheckForDependencyLoop(maker, s, &maker.step_stack, rand);
26892700 }
26902701
2691 {
2692 // Check that we have enough memory to complete the build.
2693 var any_problems = false;
2694 var max_needed: u64 = 0;
2695 for (step_stack.keys()) |step_index| {
2696 const make_step = maker.stepByIndex(step_index);
2697 const conf_step = step_index.ptr(c);
2698 const max_rss = conf_step.max_rss.toBytes();
2699 if (max_rss == 0) continue;
2700 max_needed = @max(max_needed, max_rss);
2701 if (max_rss > maker.available_rss) {
2702 if (maker.skip_oom_steps) {
2703 make_step.state = .skipped_oom;
2704 for (make_step.dependants.items) |dependant| {
2705 maker.stepByIndex(dependant).pending_deps -= 1;
2706 }
2707 } else {
2708 log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{
2709 conf_step.owner.depPrefixSlice(c),
2710 conf_step.name.slice(c),
2711 max_rss,
2712 maker.available_rss,
2713 });
2714 any_problems = true;
2702 // Notice when a step's participation in the evaluation graph implies that additional package dependencies
2703 // must be fetched.
2704 var needed_step_deps: std.array_hash_map.Auto(Configuration.Step.Index, Configuration.StringList) = .empty;
2705 defer needed_step_deps.deinit(gpa);
2706
2707 for (c.unlazy_deps) |unlazy_dep| {
2708 const step_index = unlazy_dep.step.unwrap() orelse continue; // Handled in `configure`.
2709 try needed_step_deps.put(gpa, step_index, unlazy_dep.list);
2710 }
2711
2712 assert(discovered_unlazy_set.count() == 0);
2713 for (step_stack.keys()) |step_index| {
2714 const interned_string_list = needed_step_deps.get(step_index) orelse continue;
2715 const hash_strings = interned_string_list.slice(c);
2716 for (hash_strings) |hash_string| {
2717 const hash = hash_string.slice(c);
2718 log.debug("due to step {s}, fetching lazy {s}", .{ step_index.ptr(c).name.slice(c), hash });
2719 try discovered_unlazy_set.put(arena, .fromSlice(hash), {});
2720 }
2721 }
2722 if (discovered_unlazy_set.count() != 0) return error.LazyDependencyNeeded;
2723
2724 try checkSufficientMemory(maker);
2725}
2726
2727fn checkSufficientMemory(maker: *Maker) error{InsufficientMemory}!void {
2728 const step_stack = &maker.step_stack;
2729 const c = &maker.scanned_config.configuration;
2730
2731 // Check that we have enough memory to complete the build.
2732 var any_problems = false;
2733 var max_needed: u64 = 0;
2734 for (step_stack.keys()) |step_index| {
2735 const make_step = maker.stepByIndex(step_index);
2736 const conf_step = step_index.ptr(c);
2737 const max_rss = conf_step.max_rss.toBytes();
2738 if (max_rss == 0) continue;
2739 max_needed = @max(max_needed, max_rss);
2740 if (max_rss > maker.available_rss) {
2741 if (maker.skip_oom_steps) {
2742 make_step.state = .skipped_oom;
2743 for (make_step.dependants.items) |dependant| {
2744 maker.stepByIndex(dependant).pending_deps -= 1;
27152745 }
2746 } else {
2747 log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{
2748 conf_step.owner.depPrefixSlice(c),
2749 conf_step.name.slice(c),
2750 max_rss,
2751 maker.available_rss,
2752 });
2753 any_problems = true;
27162754 }
27172755 }
2718 if (any_problems) {
2719 log.info("use --skip-oom-steps to proceed, skipping memory limited steps", .{});
2720 if (maker.max_rss_is_default) {
2721 log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{max_needed});
2722 }
2723 return error.InsufficientMemory;
2756 }
2757 if (any_problems) {
2758 log.info("use --skip-oom-steps to proceed, skipping memory limited steps", .{});
2759 if (maker.max_rss_is_default) {
2760 log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{max_needed});
27242761 }
2762 return error.InsufficientMemory;
27252763 }
27262764}
27272765
......@@ -3441,17 +3479,17 @@ fn printTreeChildLast(writer: *Writer, mode: Io.Terminal.Mode) !void {
34413479 });
34423480}
34433481
3444/// Traverse the dependency graph depth-first and make it undirected by having
3445/// steps know their dependants (they only know dependencies at start).
3446/// Along the way, check that there is no dependency loop, and record the steps
3447/// in traversal order in `step_stack`.
3448/// Each step has its dependencies traversed in random order, this accomplishes
3449/// two things:
3450/// - `step_stack` will be in randomized-depth-first order, so the build runner
3451/// spawns initial steps in a random order
3452/// - each step's `dependants` list is also filled in a random order, so that
3453/// when it finishes executing in `makeStep`, it spawns next steps to run in
3482/// Traverse the dependency graph depth-first and make it undirected by having steps know their dependants
3483/// (they only know dependencies at start).
3484///
3485/// Along the way, check that there is no dependency loop, and record the steps in traversal order in
3486/// `step_stack`.
3487///
3488/// Each step has its dependencies traversed in random order, this accomplishes two things:
3489/// - `step_stack` will be in randomized-depth-first order, so the build runner spawns initial steps in a
34543490/// random order
3491/// - each step's `dependants` list is also filled in a random order, so that when it finishes executing in
3492/// `makeStep`, it spawns next steps to run in random order
34553493fn constructGraphAndCheckForDependencyLoop(
34563494 maker: *Maker,
34573495 step_index: Configuration.Step.Index,
lib/compiler/Maker/Fetch.zig+4-4
......@@ -153,11 +153,11 @@ pub const JobQueue = struct {
153153 debug_hash: bool,
154154 mode: Mode,
155155 /// Set of hashes that will be additionally fetched even if they are marked
156 /// as lazy.
157 unlazy_set: UnlazySet = .{},
156 /// as lazy. Read-only.
157 unlazy_set: UnlazySet = .empty,
158158 /// Identifies paths that override all packages in the tree with matching
159159 /// project ids.
160 fork_set: ForkSet = .{},
160 fork_set: ForkSet = .empty,
161161
162162 pub const Mode = enum {
163163 /// Non-lazy dependencies are always fetched.
......@@ -297,8 +297,8 @@ pub const JobQueue = struct {
297297 \\ pub const build_zig = @import({qf});
298298 \\
299299 , .{fork.path});
300 break :b;
300301 }
301 break :b;
302302 }
303303 try w.print(
304304 \\ pub const build_zig = @import({q});
lib/compiler/Maker/ScannedConfig.zig+3-1
......@@ -55,7 +55,9 @@ pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void {
5555 {
5656 var tf = try s.beginTupleField("unlazy_deps", .{});
5757 for (c.unlazy_deps) |dep| {
58 try tf.field(dep.slice(c), .{});
58 var field = try tf.beginStructField(.{});
59 try printStruct(sc, &field, Configuration.UnlazyDep, dep);
60 try field.end();
5961 }
6062 try tf.end();
6163 }
lib/std/Build.zig+78-17
......@@ -80,7 +80,11 @@ pub const Graph = struct {
8080 system_package_mode: bool = false,
8181 zig_exe: []const u8,
8282 environ_map: process.Environ.Map,
83 needed_lazy_dependencies: std.array_hash_map.String(void) = .empty,
83 /// Managed by:
84 /// * `Build.declareDependency`
85 /// * `Build.dependency`
86 /// * `Build.dependencyLazy`
87 unlazy_deps: std.array_hash_map.Auto(Configuration.String, void) = .empty,
8488 /// Information about the native target. Computed before build() is invoked.
8589 host: ResolvedTarget,
8690 dependency_cache: PackageInstanceMap = .empty,
......@@ -1934,7 +1938,9 @@ inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, c
19341938}
19351939
19361940fn markNeededLazyDep(b: *Build, pkg_hash: []const u8) void {
1937 b.graph.needed_lazy_dependencies.put(b.graph.arena, pkg_hash, {}) catch @panic("OOM");
1941 const graph = b.graph;
1942 const interned_hash = graph.wip_configuration.addString(pkg_hash) catch @panic("OOM");
1943 graph.unlazy_deps.put(graph.arena, interned_hash, {}) catch @panic("OOM");
19381944}
19391945
19401946/// Deprecated in favor of `dependencyLazy`.
......@@ -1944,21 +1950,76 @@ pub fn lazyDependency(b: *Build, name: []const u8, args: anytype) ?*Dependency {
19441950 };
19451951}
19461952
1947/// Declares that the current configuration does in fact require a potentially
1948/// lazy dependency.
1953/// Indicates that the current configuration needs a potentially lazy dependency.
1954///
1955/// Related:
1956/// * `declareDependencyStep`
1957/// * `dependency`
1958/// * `dependencyLazy`
1959/// * `dependencyStep`
1960pub fn declareDependency(b: *Build, name: []const u8) void {
1961 const pkg_hash = findPkgHashOrFatal(b, name);
1962 markNeededLazyDep(b, pkg_hash);
1963}
1964
1965/// Indicates that the current configuration needs a potentially lazy dependency for the provided `Step`.
1966///
1967/// Related:
1968/// * `declareDependency`
1969/// * `dependency`
1970/// * `dependencyLazy`
1971/// * `dependencyStep`
1972pub fn declareDependencyStep(b: *Build, s: *Step, name: []const u8) void {
1973 const graph = b.graph;
1974 const pkg_hash = findPkgHashOrFatal(b, name);
1975 const interned_hash = graph.wip_configuration.addString(pkg_hash) catch @panic("OOM");
1976 s.unlazy_deps.put(graph.arena, interned_hash, {}) catch @panic("OOM");
1977}
1978
1979/// Indicates that the current configuration needs a potentially lazy dependency for the provided `Step`.
1980///
1981/// If the dependency is already fetched, it is returned, otherwise `null` is returned. After configuration
1982/// completes, if the dependency is needed after all, it will be fetched and configuration will re-run.
19491983///
1950/// If the dependency is already fetched, it is returned. However if the
1951/// dependency is not yet fetched, then when the build script is finished
1952/// running, the toolchain will not proceed to the make phase. Instead, the
1953/// parent process will additionally fetch all the lazy dependencies that were
1954/// actually required by running the build script, recompile the build script,
1955/// and then run it again. In other words, if this function returns
1956/// `error.LazyDependencyNeeded` it means that the only purpose of completing
1957/// the configure phase is to find out all the other lazy dependencies that are
1958/// also required. In this case, one must propagate the error all the way up
1959/// and return it from the main build function.
1984/// For non-lazy dependencies, this never returns `null`.
1985///
1986/// Related:
1987/// * `dependency`
1988/// * `dependencyLazy`
1989/// * `declareDependency`
1990/// * `declareDependencyStep`
1991pub fn dependencyStep(b: *Build, s: *Step, name: []const u8, args: anytype) ?*Dependency {
1992 const pkg_hash = findPkgHashOrFatal(b, name);
1993 const graph = b.graph;
1994 const arena = graph.arena;
1995 const entry = package_map.get(pkg_hash).?;
1996 if (!entry.available) {
1997 const interned_hash = graph.wip_configuration.addString(pkg_hash) catch @panic("OOM");
1998 s.unlazy_deps.put(arena, interned_hash, {}) catch @panic("OOM");
1999 return null;
2000 }
2001 var map: PackageOptions.Map = .empty;
2002 PackageOptions.fromArgs(arena, &map, args);
2003 return dependencyResolved(b, name, entry, &map);
2004}
2005
2006/// Declares that the current configuration does in fact require a potentially lazy dependency.
2007///
2008/// If the dependency is already fetched, it is returned. However if the dependency is not yet fetched, then
2009/// when the build script is finished running, the toolchain will not proceed to the make phase. Instead, the
2010/// parent process will additionally fetch all the lazy dependencies that were actually required by running the
2011/// build script, recompile the build script, and then run it again. In other words, if this function returns
2012/// `error.LazyDependencyNeeded` it means that the only purpose of completing the configure phase is to find
2013/// out all the other lazy dependencies that are also required. In this case, one must propagate the error all
2014/// the way up and return it from the main build function.
19602015///
19612016/// For non-lazy dependencies, this always succeeds.
2017///
2018/// Related:
2019/// * `dependency`
2020/// * `dependencyStep`
2021/// * `declareDependency`
2022/// * `declareDependencyStep`
19622023pub fn dependencyLazy(b: *Build, name: []const u8, args: anytype) error{LazyDependencyNeeded}!*Dependency {
19632024 const pkg_hash = findPkgHashOrFatal(b, name);
19642025 const entry = package_map.get(pkg_hash) orelse unreachable;
......@@ -2018,7 +2079,7 @@ pub const package_map: std.StaticStringMap(PackageEntry) = blk: {
20182079pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {
20192080 return dependencyLazy(b, name, args) catch |err| switch (err) {
20202081 error.LazyDependencyNeeded => {
2021 assert(b.graph.needed_lazy_dependencies.count() != 0);
2082 assert(b.graph.unlazy_deps.count() != 0);
20222083 serializeConfigurationExiting(b);
20232084 },
20242085 };
......@@ -2143,7 +2204,7 @@ fn dependencyResolved(
21432204pub inline fn runPackageScript(b: *Build, comptime build_zig: anytype) void {
21442205 const result: anyerror!void = build_zig.build(b);
21452206 result catch |err| switch (err) {
2146 error.LazyDependencyNeeded => assert(b.graph.needed_lazy_dependencies.count() != 0),
2207 error.LazyDependencyNeeded => assert(b.graph.unlazy_deps.count() != 0),
21472208 else => {
21482209 if (b.dep_prefix.len == 0) {
21492210 log.err("package {q} configuration failed: {t}", .{ b.dep_prefix, err });
......@@ -2151,7 +2212,7 @@ pub inline fn runPackageScript(b: *Build, comptime build_zig: anytype) void {
21512212 log.err("configuration failed: {t}", .{err});
21522213 }
21532214 if (@errorReturnTrace()) |trace| std.debug.dumpErrorReturnTrace(trace);
2154 const lazy_count = b.graph.needed_lazy_dependencies.count();
2215 const lazy_count = b.graph.unlazy_deps.count();
21552216 if (lazy_count == 0) process.exit(1);
21562217 log.info("{d} lazy dependencies detected; fetching and retrying configuration", .{lazy_count});
21572218 },
lib/std/Build/Configuration.zig+47-3
......@@ -11,7 +11,7 @@ const native_endian = builtin.target.cpu.arch.endian();
1111string_bytes: []u8,
1212steps: []Step,
1313path_deps: []PathDep,
14unlazy_deps: []String,
14unlazy_deps: []UnlazyDep,
1515system_integrations: []SystemIntegration,
1616available_options: []AvailableOption,
1717search_prefixes: []String,
......@@ -52,7 +52,7 @@ pub const Wip = struct {
5252 targets_table: TargetsTable = .empty,
5353
5454 string_bytes: std.ArrayList(u8) = .empty,
55 unlazy_deps: std.ArrayList(String) = .empty,
55 unlazy_deps: std.ArrayList(UnlazyDep) = .empty,
5656 system_integrations: std.ArrayList(SystemIntegration) = .empty,
5757 available_options: std.ArrayList(AvailableOption) = .empty,
5858 steps: std.ArrayList(Step) = .empty,
......@@ -227,6 +227,27 @@ pub const Wip = struct {
227227 return @fromBackingInt(@intCast(revert_index));
228228 }
229229
230 pub fn addStringListInterned(wip: *Wip, list: []const String) Allocator.Error!StringList {
231 // Increase size of extra to support the list. Add the string list
232 // there. Then check for duplicate, reverting list if already found.
233 const gpa = wip.gpa;
234 const revert_index: u32 = @intCast(wip.extra.items.len);
235 const added = try wip.extra.addManyAsSlice(gpa, list.len + 1);
236 added[0] = @intCast(list.len);
237 @memcpy(added[1..], @as([]const u32, @ptrCast(list)));
238 const gop = try wip.dedupe_table.getOrPutContext(gpa, .{
239 .index = revert_index,
240 .len = @intCast(added.len),
241 }, @as(ExtraSlice.Context, .{ .extra = wip.extra.items }));
242
243 if (gop.found_existing) {
244 wip.extra.items.len = revert_index;
245 return @fromBackingInt(@intCast(gop.key_ptr.index));
246 }
247
248 return @fromBackingInt(@intCast(revert_index));
249 }
250
230251 pub fn addBytes(wip: *Wip, bytes: []const u8) Allocator.Error!Bytes {
231252 try wip.string_bytes.appendSlice(wip.gpa, bytes);
232253 return .{
......@@ -474,6 +495,12 @@ pub const AvailableOption = extern struct {
474495 };
475496};
476497
498pub const UnlazyDep = extern struct {
499 /// none indicates not associated with any particular step.
500 step: Step.OptionalIndex,
501 list: StringList,
502};
503
477504pub const Step = extern struct {
478505 name: String,
479506 owner: Package.Index,
......@@ -507,6 +534,23 @@ pub const Step = extern struct {
507534 }
508535 };
509536
537 pub const OptionalIndex = enum(u32) {
538 none = max_u32,
539 _,
540
541 pub fn unwrap(this: @This()) ?Index {
542 return switch (this) {
543 .none => null,
544 else => @fromBackingInt(@intCast(@backingInt(this))),
545 };
546 }
547
548 pub fn ptr(i: @This(), c: *const Configuration) ?*const Step {
549 if (i == .none) return null;
550 return &c.steps[@backingInt(i)];
551 }
552 };
553
510554 /// Shared by all steps.
511555 pub const Flags = packed struct(u32) {
512556 tag: Tag,
......@@ -3134,7 +3178,7 @@ pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration {
31343178 .string_bytes = try arena.alloc(u8, header.string_bytes_len),
31353179 .steps = try arena.alloc(Step, header.steps_len),
31363180 .path_deps = try arena.alloc(PathDep, header.path_deps_len),
3137 .unlazy_deps = try arena.alloc(String, header.unlazy_deps_len),
3181 .unlazy_deps = try arena.alloc(UnlazyDep, header.unlazy_deps_len),
31383182 .system_integrations = try arena.alloc(SystemIntegration, header.system_integrations_len),
31393183 .available_options = try arena.alloc(AvailableOption, header.available_options_len),
31403184 .search_prefixes = try arena.alloc(String, header.search_prefixes_len),
lib/std/Build/Serialize.zig+12-3
......@@ -683,12 +683,21 @@ pub fn write(b: *std.Build, wc: *Configuration.Wip, writer: *std.Io.Writer) !voi
683683 .max_rss = .fromBytes(step.max_rss),
684684 .extended = @fromBackingInt(extended),
685685 });
686
687 if (step.unlazy_deps.count() != 0) {
688 try wc.unlazy_deps.append(gpa, .{
689 .step = @fromBackingInt(@intCast(wc.steps.items.len - 1)),
690 .list = try wc.addStringListInterned(step.unlazy_deps.keys()),
691 });
692 }
686693 }
687694 }
688695
689 try wc.unlazy_deps.ensureUnusedCapacity(gpa, graph.needed_lazy_dependencies.keys().len);
690 for (graph.needed_lazy_dependencies.keys()) |k| {
691 wc.unlazy_deps.appendAssumeCapacity(try wc.addString(k));
696 if (graph.unlazy_deps.count() != 0) {
697 try wc.unlazy_deps.append(gpa, .{
698 .step = .none,
699 .list = try wc.addStringListInterned(graph.unlazy_deps.keys()),
700 });
692701 }
693702
694703 try wc.write(writer, .{
lib/std/Build/Step.zig+6
......@@ -36,6 +36,11 @@ max_rss: u64,
3636/// to print along with debugging messages.
3737debug_stack_trace: std.debug.StackTrace,
3838
39/// Managed by:
40/// * `Build.declareDependencyStep`
41/// * `Build.dependencyStep`
42unlazy_deps: std.array_hash_map.Auto(Configuration.String, void),
43
3944pub const Tag = Configuration.Step.Tag;
4045
4146pub fn Type(comptime tag: Tag) type {
......@@ -104,6 +109,7 @@ pub fn init(options: StepOptions) Step {
104109 const first_ret_addr = options.first_ret_addr orelse @returnAddress();
105110 break :blk std.debug.captureCurrentStackTrace(.{ .first_address = first_ret_addr }, addr_buf);
106111 },
112 .unlazy_deps = .empty,
107113 };
108114}
109115