From 2c18bccf4428a6e23ed5c5572243337a560af9b5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 15 Oct 2026 16:00:57 -0700 Subject: [PATCH] std.Build: lazy dependencies can associate with steps and serialize the configuration --- lib/std/Build.zig | 97 +++++++++++++++++++++++++++------ lib/std/Build/Configuration.zig | 22 ++++++++ lib/std/Build/Serialize.zig | 14 ++++- lib/std/Build/Step.zig | 6 ++ 4 files changed, 118 insertions(+), 21 deletions(-) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index c1046962b509b152ce87271fa7403218b54819e1..ec2f57d2911a681571e9116ab48828868642e637 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -80,7 +80,11 @@ pub const Graph = struct { system_package_mode: bool = false, zig_exe: []const u8, environ_map: process.Environ.Map, - needed_lazy_dependencies: std.array_hash_map.String(void) = .empty, + /// Managed by: + /// * `Build.declareDependency` + /// * `Build.dependency` + /// * `Build.dependencyLazy` + unlazy_deps: std.array_hash_map.Auto(Configuration.String, void) = .empty, /// Information about the native target. Computed before build() is invoked. host: ResolvedTarget, dependency_cache: PackageInstanceMap = .empty, @@ -1934,7 +1938,9 @@ inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, c } fn markNeededLazyDep(b: *Build, pkg_hash: []const u8) void { - b.graph.needed_lazy_dependencies.put(b.graph.arena, pkg_hash, {}) catch @panic("OOM"); + const graph = b.graph; + const interned_hash = graph.wip_configuration.addString(pkg_hash) catch @panic("OOM"); + graph.unlazy_deps.put(graph.arena, interned_hash, {}) catch @panic("OOM"); } /// Deprecated in favor of `dependencyLazy`. @@ -1944,21 +1950,76 @@ pub fn lazyDependency(b: *Build, name: []const u8, args: anytype) ?*Dependency { }; } -/// Declares that the current configuration does in fact require a potentially -/// lazy dependency. -/// -/// If the dependency is already fetched, it is returned. However if the -/// dependency is not yet fetched, then when the build script is finished -/// running, the toolchain will not proceed to the make phase. Instead, the -/// parent process will additionally fetch all the lazy dependencies that were -/// actually required by running the build script, recompile the build script, -/// and then run it again. In other words, if this function returns -/// `error.LazyDependencyNeeded` it means that the only purpose of completing -/// the configure phase is to find out all the other lazy dependencies that are -/// also required. In this case, one must propagate the error all the way up -/// and return it from the main build function. +/// Indicates that the current configuration needs a potentially lazy dependency. +/// +/// Related: +/// * `declareDependencyStep` +/// * `dependency` +/// * `dependencyLazy` +/// * `dependencyStep` +pub fn declareDependency(b: *Build, name: []const u8) void { + const pkg_hash = findPkgHashOrFatal(b, name); + markNeededLazyDep(b, pkg_hash); +} + +/// Indicates that the current configuration needs a potentially lazy dependency for the provided `Step`. +/// +/// Related: +/// * `declareDependency` +/// * `dependency` +/// * `dependencyLazy` +/// * `dependencyStep` +pub fn declareDependencyStep(b: *Build, s: *Step, name: []const u8) void { + const graph = b.graph; + const pkg_hash = findPkgHashOrFatal(b, name); + const interned_hash = graph.wip_configuration.addString(pkg_hash) catch @panic("OOM"); + s.unlazy_deps.put(graph.arena, interned_hash, {}) catch @panic("OOM"); +} + +/// Indicates that the current configuration needs a potentially lazy dependency for the provided `Step`. +/// +/// If the dependency is already fetched, it is returned, otherwise `null` is returned. After configuration +/// completes, if the dependency is needed after all, it will be fetched and configuration will re-run. +/// +/// For non-lazy dependencies, this never returns `null`. +/// +/// Related: +/// * `dependency` +/// * `dependencyLazy` +/// * `declareDependency` +/// * `declareDependencyStep` +pub fn dependencyStep(b: *Build, s: *Step, name: []const u8, args: anytype) ?*Dependency { + const pkg_hash = findPkgHashOrFatal(b, name); + const graph = b.graph; + const arena = graph.arena; + const entry = package_map.get(pkg_hash).?; + if (!entry.available) { + const interned_hash = graph.wip_configuration.addString(pkg_hash) catch @panic("OOM"); + s.unlazy_deps.put(arena, interned_hash, {}) catch @panic("OOM"); + return null; + } + var map: PackageOptions.Map = .empty; + PackageOptions.fromArgs(arena, &map, args); + return dependencyResolved(b, name, entry, &map); +} + +/// Declares that the current configuration does in fact require a potentially lazy dependency. +/// +/// If the dependency is already fetched, it is returned. However if the dependency is not yet fetched, then +/// when the build script is finished running, the toolchain will not proceed to the make phase. Instead, the +/// parent process will additionally fetch all the lazy dependencies that were actually required by running the +/// build script, recompile the build script, and then run it again. In other words, if this function returns +/// `error.LazyDependencyNeeded` it means that the only purpose of completing the configure phase is to find +/// out all the other lazy dependencies that are also required. In this case, one must propagate the error all +/// the way up and return it from the main build function. /// /// For non-lazy dependencies, this always succeeds. +/// +/// Related: +/// * `dependency` +/// * `dependencyStep` +/// * `declareDependency` +/// * `declareDependencyStep` pub fn dependencyLazy(b: *Build, name: []const u8, args: anytype) error{LazyDependencyNeeded}!*Dependency { const pkg_hash = findPkgHashOrFatal(b, name); const entry = package_map.get(pkg_hash) orelse unreachable; @@ -2018,7 +2079,7 @@ pub const package_map: std.StaticStringMap(PackageEntry) = blk: { pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency { return dependencyLazy(b, name, args) catch |err| switch (err) { error.LazyDependencyNeeded => { - assert(b.graph.needed_lazy_dependencies.count() != 0); + assert(b.graph.unlazy_deps.count() != 0); serializeConfigurationExiting(b); }, }; @@ -2143,7 +2204,7 @@ fn dependencyResolved( pub inline fn runPackageScript(b: *Build, comptime build_zig: anytype) void { const result: anyerror!void = build_zig.build(b); result catch |err| switch (err) { - error.LazyDependencyNeeded => assert(b.graph.needed_lazy_dependencies.count() != 0), + error.LazyDependencyNeeded => assert(b.graph.unlazy_deps.count() != 0), else => { if (b.dep_prefix.len == 0) { log.err("package {q} configuration failed: {t}", .{ b.dep_prefix, err }); @@ -2151,7 +2212,7 @@ pub inline fn runPackageScript(b: *Build, comptime build_zig: anytype) void { log.err("configuration failed: {t}", .{err}); } if (@errorReturnTrace()) |trace| std.debug.dumpErrorReturnTrace(trace); - const lazy_count = b.graph.needed_lazy_dependencies.count(); + const lazy_count = b.graph.unlazy_deps.count(); if (lazy_count == 0) process.exit(1); log.info("{d} lazy dependencies detected; fetching and retrying configuration", .{lazy_count}); }, diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 35bfabdbd62e828cb87c73a5210ec6cc0fb2d578..d117212bb646595d05fab4625e4c0166e3ea420d 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -227,6 +227,27 @@ pub const Wip = struct { return @fromBackingInt(@intCast(revert_index)); } + pub fn addStringListInterned(wip: *Wip, list: []const String) Allocator.Error!StringList { + // Increase size of extra to support the list. Add the string list + // there. Then check for duplicate, reverting list if already found. + const gpa = wip.gpa; + const revert_index: u32 = @intCast(wip.extra.items.len); + const added = try wip.extra.addManyAsSlice(gpa, list.len + 1); + added[0] = @intCast(list.len); + @memcpy(added[1..], @as([]const u32, @ptrCast(list))); + const gop = try wip.dedupe_table.getOrPutContext(gpa, .{ + .index = revert_index, + .len = @intCast(added.len), + }, @as(ExtraSlice.Context, .{ .extra = wip.extra.items })); + + if (gop.found_existing) { + wip.extra.items.len = revert_index; + return @fromBackingInt(@intCast(gop.key_ptr.index)); + } + + return @fromBackingInt(@intCast(revert_index)); + } + pub fn addBytes(wip: *Wip, bytes: []const u8) Allocator.Error!Bytes { try wip.string_bytes.appendSlice(wip.gpa, bytes); return .{ @@ -479,6 +500,7 @@ pub const Step = extern struct { owner: Package.Index, deps: Deps.Index, max_rss: MaxRss, + unlazy_deps: OptionalStringList, extended: Storage.Extended(Flags, union(Tag) { check_file: CheckFile, compile: Compile, diff --git a/lib/std/Build/Serialize.zig b/lib/std/Build/Serialize.zig index 802bf4cbcff33db51ca5c406828e5325bb707895..82c2491c140a5a8248878cd49372ee3d197ae904 100644 --- a/lib/std/Build/Serialize.zig +++ b/lib/std/Build/Serialize.zig @@ -681,14 +681,22 @@ pub fn write(b: *std.Build, wc: *Configuration.Wip, writer: *std.Io.Writer) !voi .owner = try s.builderToPackage(step.owner), .deps = deps, .max_rss = .fromBytes(step.max_rss), + .unlazy_deps = if (step.unlazy_deps.count() != 0) + .init(try wc.addStringListInterned(step.unlazy_deps.keys())) + else + .none, .extended = @fromBackingInt(extended), }); } } - try wc.unlazy_deps.ensureUnusedCapacity(gpa, graph.needed_lazy_dependencies.keys().len); - for (graph.needed_lazy_dependencies.keys()) |k| { - wc.unlazy_deps.appendAssumeCapacity(try wc.addString(k)); + { + // TODO exchange graph.unlazy_deps.keys with the wc.unlazy_deps array list without copying + // MultiArrayList should support this when there is only 1 non zero sized struct field + const unlazy_deps = graph.unlazy_deps.keys(); + try wc.unlazy_deps.ensureTotalCapacityPrecise(gpa, unlazy_deps.len); + assert(wc.unlazy_deps.items.len == 0); + wc.unlazy_deps.appendSliceAssumeCapacity(unlazy_deps); } try wc.write(writer, .{ diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index a4d6987126ac87113fb73ce63dca3b75c89ffe9b..d37f227f2eb7aa58ef0c5c23ae84b00a8c1ecc56 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -36,6 +36,11 @@ max_rss: u64, /// to print along with debugging messages. debug_stack_trace: std.debug.StackTrace, +/// Managed by: +/// * `Build.declareDependencyStep` +/// * `Build.dependencyStep` +unlazy_deps: std.array_hash_map.Auto(Configuration.String, void), + pub const Tag = Configuration.Step.Tag; pub fn Type(comptime tag: Tag) type { @@ -104,6 +109,7 @@ pub fn init(options: StepOptions) Step { const first_ret_addr = options.first_ret_addr orelse @returnAddress(); break :blk std.debug.captureCurrentStackTrace(.{ .first_address = first_ret_addr }, addr_buf); }, + .unlazy_deps = .empty, }; } -- 2.54.0