authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-10-15 16:00:57-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-10-15 16:00:57-07:00
log2c18bccf4428a6e23ed5c5572243337a560af9b5
treea6d57665dbb9066cf1b6d4a799011932f7549739
parent13ee140439e8a175dc80983b0257793fa9c2cb0e

std.Build: lazy dependencies can associate with steps

and serialize the configuration

4 files changed, 117 insertions(+), 20 deletions(-)

lib/std/Build.zig+78-17
...@@ -80,7 +80,11 @@ pub const Graph = struct {...@@ -80,7 +80,11 @@ pub const Graph = struct {
80 system_package_mode: bool = false,80 system_package_mode: bool = false,
81 zig_exe: []const u8,81 zig_exe: []const u8,
82 environ_map: process.Environ.Map,82 environ_map: process.Environ.Map,
83 needed_lazy_dependencies: std.array_hash_map.String(void) = .empty,83 /// Managed by:
84 /// * `Build.declareDependency`
85 /// * `Build.dependency`
86 /// * `Build.dependencyLazy`
87 unlazy_deps: std.array_hash_map.Auto(Configuration.String, void) = .empty,
84 /// Information about the native target. Computed before build() is invoked.88 /// Information about the native target. Computed before build() is invoked.
85 host: ResolvedTarget,89 host: ResolvedTarget,
86 dependency_cache: PackageInstanceMap = .empty,90 dependency_cache: PackageInstanceMap = .empty,
...@@ -1934,7 +1938,9 @@ inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, c...@@ -1934,7 +1938,9 @@ inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, c
1934}1938}
19351939
1936fn markNeededLazyDep(b: *Build, pkg_hash: []const u8) void {1940fn markNeededLazyDep(b: *Build, pkg_hash: []const u8) void {
1937 b.graph.needed_lazy_dependencies.put(b.graph.arena, pkg_hash, {}) catch @panic("OOM");1941 const graph = b.graph;
1942 const interned_hash = graph.wip_configuration.addString(pkg_hash) catch @panic("OOM");
1943 graph.unlazy_deps.put(graph.arena, interned_hash, {}) catch @panic("OOM");
1938}1944}
19391945
1940/// Deprecated in favor of `dependencyLazy`.1946/// Deprecated in favor of `dependencyLazy`.
...@@ -1944,21 +1950,76 @@ pub fn lazyDependency(b: *Build, name: []const u8, args: anytype) ?*Dependency {...@@ -1944,21 +1950,76 @@ pub fn lazyDependency(b: *Build, name: []const u8, args: anytype) ?*Dependency {
1944 };1950 };
1945}1951}
19461952
1947/// Declares that the current configuration does in fact require a potentially1953/// Indicates that the current configuration needs a potentially lazy dependency.
1948/// lazy dependency.1954///
1955/// Related:
1956/// * `declareDependencyStep`
1957/// * `dependency`
1958/// * `dependencyLazy`
1959/// * `dependencyStep`
1960pub fn declareDependency(b: *Build, name: []const u8) void {
1961 const pkg_hash = findPkgHashOrFatal(b, name);
1962 markNeededLazyDep(b, pkg_hash);
1963}
1964
1965/// Indicates that the current configuration needs a potentially lazy dependency for the provided `Step`.
1966///
1967/// Related:
1968/// * `declareDependency`
1969/// * `dependency`
1970/// * `dependencyLazy`
1971/// * `dependencyStep`
1972pub fn declareDependencyStep(b: *Build, s: *Step, name: []const u8) void {
1973 const graph = b.graph;
1974 const pkg_hash = findPkgHashOrFatal(b, name);
1975 const interned_hash = graph.wip_configuration.addString(pkg_hash) catch @panic("OOM");
1976 s.unlazy_deps.put(graph.arena, interned_hash, {}) catch @panic("OOM");
1977}
1978
1979/// Indicates that the current configuration needs a potentially lazy dependency for the provided `Step`.
1980///
1981/// If the dependency is already fetched, it is returned, otherwise `null` is returned. After configuration
1982/// completes, if the dependency is needed after all, it will be fetched and configuration will re-run.
1949///1983///
1950/// If the dependency is already fetched, it is returned. However if the1984/// For non-lazy dependencies, this never returns `null`.
1951/// dependency is not yet fetched, then when the build script is finished1985///
1952/// running, the toolchain will not proceed to the make phase. Instead, the1986/// Related:
1953/// parent process will additionally fetch all the lazy dependencies that were1987/// * `dependency`
1954/// actually required by running the build script, recompile the build script,1988/// * `dependencyLazy`
1955/// and then run it again. In other words, if this function returns1989/// * `declareDependency`
1956/// `error.LazyDependencyNeeded` it means that the only purpose of completing1990/// * `declareDependencyStep`
1957/// the configure phase is to find out all the other lazy dependencies that are1991pub fn dependencyStep(b: *Build, s: *Step, name: []const u8, args: anytype) ?*Dependency {
1958/// also required. In this case, one must propagate the error all the way up1992 const pkg_hash = findPkgHashOrFatal(b, name);
1959/// and return it from the main build function.1993 const graph = b.graph;
1994 const arena = graph.arena;
1995 const entry = package_map.get(pkg_hash).?;
1996 if (!entry.available) {
1997 const interned_hash = graph.wip_configuration.addString(pkg_hash) catch @panic("OOM");
1998 s.unlazy_deps.put(arena, interned_hash, {}) catch @panic("OOM");
1999 return null;
2000 }
2001 var map: PackageOptions.Map = .empty;
2002 PackageOptions.fromArgs(arena, &map, args);
2003 return dependencyResolved(b, name, entry, &map);
2004}
2005
2006/// Declares that the current configuration does in fact require a potentially lazy dependency.
2007///
2008/// If the dependency is already fetched, it is returned. However if the dependency is not yet fetched, then
2009/// when the build script is finished running, the toolchain will not proceed to the make phase. Instead, the
2010/// parent process will additionally fetch all the lazy dependencies that were actually required by running the
2011/// build script, recompile the build script, and then run it again. In other words, if this function returns
2012/// `error.LazyDependencyNeeded` it means that the only purpose of completing the configure phase is to find
2013/// out all the other lazy dependencies that are also required. In this case, one must propagate the error all
2014/// the way up and return it from the main build function.
1960///2015///
1961/// For non-lazy dependencies, this always succeeds.2016/// For non-lazy dependencies, this always succeeds.
2017///
2018/// Related:
2019/// * `dependency`
2020/// * `dependencyStep`
2021/// * `declareDependency`
2022/// * `declareDependencyStep`
1962pub fn dependencyLazy(b: *Build, name: []const u8, args: anytype) error{LazyDependencyNeeded}!*Dependency {2023pub fn dependencyLazy(b: *Build, name: []const u8, args: anytype) error{LazyDependencyNeeded}!*Dependency {
1963 const pkg_hash = findPkgHashOrFatal(b, name);2024 const pkg_hash = findPkgHashOrFatal(b, name);
1964 const entry = package_map.get(pkg_hash) orelse unreachable;2025 const entry = package_map.get(pkg_hash) orelse unreachable;
...@@ -2018,7 +2079,7 @@ pub const package_map: std.StaticStringMap(PackageEntry) = blk: {...@@ -2018,7 +2079,7 @@ pub const package_map: std.StaticStringMap(PackageEntry) = blk: {
2018pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {2079pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {
2019 return dependencyLazy(b, name, args) catch |err| switch (err) {2080 return dependencyLazy(b, name, args) catch |err| switch (err) {
2020 error.LazyDependencyNeeded => {2081 error.LazyDependencyNeeded => {
2021 assert(b.graph.needed_lazy_dependencies.count() != 0);2082 assert(b.graph.unlazy_deps.count() != 0);
2022 serializeConfigurationExiting(b);2083 serializeConfigurationExiting(b);
2023 },2084 },
2024 };2085 };
...@@ -2143,7 +2204,7 @@ fn dependencyResolved(...@@ -2143,7 +2204,7 @@ fn dependencyResolved(
2143pub inline fn runPackageScript(b: *Build, comptime build_zig: anytype) void {2204pub inline fn runPackageScript(b: *Build, comptime build_zig: anytype) void {
2144 const result: anyerror!void = build_zig.build(b);2205 const result: anyerror!void = build_zig.build(b);
2145 result catch |err| switch (err) {2206 result catch |err| switch (err) {
2146 error.LazyDependencyNeeded => assert(b.graph.needed_lazy_dependencies.count() != 0),2207 error.LazyDependencyNeeded => assert(b.graph.unlazy_deps.count() != 0),
2147 else => {2208 else => {
2148 if (b.dep_prefix.len == 0) {2209 if (b.dep_prefix.len == 0) {
2149 log.err("package {q} configuration failed: {t}", .{ b.dep_prefix, err });2210 log.err("package {q} configuration failed: {t}", .{ b.dep_prefix, err });
...@@ -2151,7 +2212,7 @@ pub inline fn runPackageScript(b: *Build, comptime build_zig: anytype) void {...@@ -2151,7 +2212,7 @@ pub inline fn runPackageScript(b: *Build, comptime build_zig: anytype) void {
2151 log.err("configuration failed: {t}", .{err});2212 log.err("configuration failed: {t}", .{err});
2152 }2213 }
2153 if (@errorReturnTrace()) |trace| std.debug.dumpErrorReturnTrace(trace);2214 if (@errorReturnTrace()) |trace| std.debug.dumpErrorReturnTrace(trace);
2154 const lazy_count = b.graph.needed_lazy_dependencies.count();2215 const lazy_count = b.graph.unlazy_deps.count();
2155 if (lazy_count == 0) process.exit(1);2216 if (lazy_count == 0) process.exit(1);
2156 log.info("{d} lazy dependencies detected; fetching and retrying configuration", .{lazy_count});2217 log.info("{d} lazy dependencies detected; fetching and retrying configuration", .{lazy_count});
2157 },2218 },
lib/std/Build/Configuration.zig+22
...@@ -227,6 +227,27 @@ pub const Wip = struct {...@@ -227,6 +227,27 @@ pub const Wip = struct {
227 return @fromBackingInt(@intCast(revert_index));227 return @fromBackingInt(@intCast(revert_index));
228 }228 }
229229
230 pub fn addStringListInterned(wip: *Wip, list: []const String) Allocator.Error!StringList {
231 // Increase size of extra to support the list. Add the string list
232 // there. Then check for duplicate, reverting list if already found.
233 const gpa = wip.gpa;
234 const revert_index: u32 = @intCast(wip.extra.items.len);
235 const added = try wip.extra.addManyAsSlice(gpa, list.len + 1);
236 added[0] = @intCast(list.len);
237 @memcpy(added[1..], @as([]const u32, @ptrCast(list)));
238 const gop = try wip.dedupe_table.getOrPutContext(gpa, .{
239 .index = revert_index,
240 .len = @intCast(added.len),
241 }, @as(ExtraSlice.Context, .{ .extra = wip.extra.items }));
242
243 if (gop.found_existing) {
244 wip.extra.items.len = revert_index;
245 return @fromBackingInt(@intCast(gop.key_ptr.index));
246 }
247
248 return @fromBackingInt(@intCast(revert_index));
249 }
250
230 pub fn addBytes(wip: *Wip, bytes: []const u8) Allocator.Error!Bytes {251 pub fn addBytes(wip: *Wip, bytes: []const u8) Allocator.Error!Bytes {
231 try wip.string_bytes.appendSlice(wip.gpa, bytes);252 try wip.string_bytes.appendSlice(wip.gpa, bytes);
232 return .{253 return .{
...@@ -479,6 +500,7 @@ pub const Step = extern struct {...@@ -479,6 +500,7 @@ pub const Step = extern struct {
479 owner: Package.Index,500 owner: Package.Index,
480 deps: Deps.Index,501 deps: Deps.Index,
481 max_rss: MaxRss,502 max_rss: MaxRss,
503 unlazy_deps: OptionalStringList,
482 extended: Storage.Extended(Flags, union(Tag) {504 extended: Storage.Extended(Flags, union(Tag) {
483 check_file: CheckFile,505 check_file: CheckFile,
484 compile: Compile,506 compile: Compile,
lib/std/Build/Serialize.zig+11-3
...@@ -681,14 +681,22 @@ pub fn write(b: *std.Build, wc: *Configuration.Wip, writer: *std.Io.Writer) !voi...@@ -681,14 +681,22 @@ pub fn write(b: *std.Build, wc: *Configuration.Wip, writer: *std.Io.Writer) !voi
681 .owner = try s.builderToPackage(step.owner),681 .owner = try s.builderToPackage(step.owner),
682 .deps = deps,682 .deps = deps,
683 .max_rss = .fromBytes(step.max_rss),683 .max_rss = .fromBytes(step.max_rss),
684 .unlazy_deps = if (step.unlazy_deps.count() != 0)
685 .init(try wc.addStringListInterned(step.unlazy_deps.keys()))
686 else
687 .none,
684 .extended = @fromBackingInt(extended),688 .extended = @fromBackingInt(extended),
685 });689 });
686 }690 }
687 }691 }
688692
689 try wc.unlazy_deps.ensureUnusedCapacity(gpa, graph.needed_lazy_dependencies.keys().len);693 {
690 for (graph.needed_lazy_dependencies.keys()) |k| {694 // TODO exchange graph.unlazy_deps.keys with the wc.unlazy_deps array list without copying
691 wc.unlazy_deps.appendAssumeCapacity(try wc.addString(k));695 // MultiArrayList should support this when there is only 1 non zero sized struct field
696 const unlazy_deps = graph.unlazy_deps.keys();
697 try wc.unlazy_deps.ensureTotalCapacityPrecise(gpa, unlazy_deps.len);
698 assert(wc.unlazy_deps.items.len == 0);
699 wc.unlazy_deps.appendSliceAssumeCapacity(unlazy_deps);
692 }700 }
693701
694 try wc.write(writer, .{702 try wc.write(writer, .{
lib/std/Build/Step.zig+6
...@@ -36,6 +36,11 @@ max_rss: u64,...@@ -36,6 +36,11 @@ max_rss: u64,
36/// to print along with debugging messages.36/// to print along with debugging messages.
37debug_stack_trace: std.debug.StackTrace,37debug_stack_trace: std.debug.StackTrace,
3838
39/// Managed by:
40/// * `Build.declareDependencyStep`
41/// * `Build.dependencyStep`
42unlazy_deps: std.array_hash_map.Auto(Configuration.String, void),
43
39pub const Tag = Configuration.Step.Tag;44pub const Tag = Configuration.Step.Tag;
4045
41pub fn Type(comptime tag: Tag) type {46pub fn Type(comptime tag: Tag) type {
...@@ -104,6 +109,7 @@ pub fn init(options: StepOptions) Step {...@@ -104,6 +109,7 @@ pub fn init(options: StepOptions) Step {
104 const first_ret_addr = options.first_ret_addr orelse @returnAddress();109 const first_ret_addr = options.first_ret_addr orelse @returnAddress();
105 break :blk std.debug.captureCurrentStackTrace(.{ .first_address = first_ret_addr }, addr_buf);110 break :blk std.debug.captureCurrentStackTrace(.{ .first_address = first_ret_addr }, addr_buf);
106 },111 },
112 .unlazy_deps = .empty,
107 };113 };
108}114}
109115