From 2c18bccf4428a6e23ed5c5572243337a560af9b5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 15 Oct 2026 16:00:57 -0700 Subject: [PATCH 1/5] std.Build: lazy dependencies can associate with steps and serialize the configuration --- lib/std/Build.zig | 97 +++++++++++++++++++++++++++------ lib/std/Build/Configuration.zig | 22 ++++++++ lib/std/Build/Serialize.zig | 14 ++++- lib/std/Build/Step.zig | 6 ++ 4 files changed, 118 insertions(+), 21 deletions(-) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index c1046962b509b152ce87271fa7403218b54819e1..ec2f57d2911a681571e9116ab48828868642e637 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -80,7 +80,11 @@ pub const Graph = struct { system_package_mode: bool = false, zig_exe: []const u8, environ_map: process.Environ.Map, - needed_lazy_dependencies: std.array_hash_map.String(void) = .empty, + /// Managed by: + /// * `Build.declareDependency` + /// * `Build.dependency` + /// * `Build.dependencyLazy` + unlazy_deps: std.array_hash_map.Auto(Configuration.String, void) = .empty, /// Information about the native target. Computed before build() is invoked. host: ResolvedTarget, dependency_cache: PackageInstanceMap = .empty, @@ -1934,7 +1938,9 @@ inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, c } fn markNeededLazyDep(b: *Build, pkg_hash: []const u8) void { - b.graph.needed_lazy_dependencies.put(b.graph.arena, pkg_hash, {}) catch @panic("OOM"); + const graph = b.graph; + const interned_hash = graph.wip_configuration.addString(pkg_hash) catch @panic("OOM"); + graph.unlazy_deps.put(graph.arena, interned_hash, {}) catch @panic("OOM"); } /// Deprecated in favor of `dependencyLazy`. @@ -1944,21 +1950,76 @@ pub fn lazyDependency(b: *Build, name: []const u8, args: anytype) ?*Dependency { }; } -/// Declares that the current configuration does in fact require a potentially -/// lazy dependency. -/// -/// If the dependency is already fetched, it is returned. However if the -/// dependency is not yet fetched, then when the build script is finished -/// running, the toolchain will not proceed to the make phase. Instead, the -/// parent process will additionally fetch all the lazy dependencies that were -/// actually required by running the build script, recompile the build script, -/// and then run it again. In other words, if this function returns -/// `error.LazyDependencyNeeded` it means that the only purpose of completing -/// the configure phase is to find out all the other lazy dependencies that are -/// also required. In this case, one must propagate the error all the way up -/// and return it from the main build function. +/// Indicates that the current configuration needs a potentially lazy dependency. +/// +/// Related: +/// * `declareDependencyStep` +/// * `dependency` +/// * `dependencyLazy` +/// * `dependencyStep` +pub fn declareDependency(b: *Build, name: []const u8) void { + const pkg_hash = findPkgHashOrFatal(b, name); + markNeededLazyDep(b, pkg_hash); +} + +/// Indicates that the current configuration needs a potentially lazy dependency for the provided `Step`. +/// +/// Related: +/// * `declareDependency` +/// * `dependency` +/// * `dependencyLazy` +/// * `dependencyStep` +pub fn declareDependencyStep(b: *Build, s: *Step, name: []const u8) void { + const graph = b.graph; + const pkg_hash = findPkgHashOrFatal(b, name); + const interned_hash = graph.wip_configuration.addString(pkg_hash) catch @panic("OOM"); + s.unlazy_deps.put(graph.arena, interned_hash, {}) catch @panic("OOM"); +} + +/// Indicates that the current configuration needs a potentially lazy dependency for the provided `Step`. +/// +/// If the dependency is already fetched, it is returned, otherwise `null` is returned. After configuration +/// completes, if the dependency is needed after all, it will be fetched and configuration will re-run. +/// +/// For non-lazy dependencies, this never returns `null`. +/// +/// Related: +/// * `dependency` +/// * `dependencyLazy` +/// * `declareDependency` +/// * `declareDependencyStep` +pub fn dependencyStep(b: *Build, s: *Step, name: []const u8, args: anytype) ?*Dependency { + const pkg_hash = findPkgHashOrFatal(b, name); + const graph = b.graph; + const arena = graph.arena; + const entry = package_map.get(pkg_hash).?; + if (!entry.available) { + const interned_hash = graph.wip_configuration.addString(pkg_hash) catch @panic("OOM"); + s.unlazy_deps.put(arena, interned_hash, {}) catch @panic("OOM"); + return null; + } + var map: PackageOptions.Map = .empty; + PackageOptions.fromArgs(arena, &map, args); + return dependencyResolved(b, name, entry, &map); +} + +/// Declares that the current configuration does in fact require a potentially lazy dependency. +/// +/// If the dependency is already fetched, it is returned. However if the dependency is not yet fetched, then +/// when the build script is finished running, the toolchain will not proceed to the make phase. Instead, the +/// parent process will additionally fetch all the lazy dependencies that were actually required by running the +/// build script, recompile the build script, and then run it again. In other words, if this function returns +/// `error.LazyDependencyNeeded` it means that the only purpose of completing the configure phase is to find +/// out all the other lazy dependencies that are also required. In this case, one must propagate the error all +/// the way up and return it from the main build function. /// /// For non-lazy dependencies, this always succeeds. +/// +/// Related: +/// * `dependency` +/// * `dependencyStep` +/// * `declareDependency` +/// * `declareDependencyStep` pub fn dependencyLazy(b: *Build, name: []const u8, args: anytype) error{LazyDependencyNeeded}!*Dependency { const pkg_hash = findPkgHashOrFatal(b, name); const entry = package_map.get(pkg_hash) orelse unreachable; @@ -2018,7 +2079,7 @@ pub const package_map: std.StaticStringMap(PackageEntry) = blk: { pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency { return dependencyLazy(b, name, args) catch |err| switch (err) { error.LazyDependencyNeeded => { - assert(b.graph.needed_lazy_dependencies.count() != 0); + assert(b.graph.unlazy_deps.count() != 0); serializeConfigurationExiting(b); }, }; @@ -2143,7 +2204,7 @@ fn dependencyResolved( pub inline fn runPackageScript(b: *Build, comptime build_zig: anytype) void { const result: anyerror!void = build_zig.build(b); result catch |err| switch (err) { - error.LazyDependencyNeeded => assert(b.graph.needed_lazy_dependencies.count() != 0), + error.LazyDependencyNeeded => assert(b.graph.unlazy_deps.count() != 0), else => { if (b.dep_prefix.len == 0) { log.err("package {q} configuration failed: {t}", .{ b.dep_prefix, err }); @@ -2151,7 +2212,7 @@ pub inline fn runPackageScript(b: *Build, comptime build_zig: anytype) void { log.err("configuration failed: {t}", .{err}); } if (@errorReturnTrace()) |trace| std.debug.dumpErrorReturnTrace(trace); - const lazy_count = b.graph.needed_lazy_dependencies.count(); + const lazy_count = b.graph.unlazy_deps.count(); if (lazy_count == 0) process.exit(1); log.info("{d} lazy dependencies detected; fetching and retrying configuration", .{lazy_count}); }, diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 35bfabdbd62e828cb87c73a5210ec6cc0fb2d578..d117212bb646595d05fab4625e4c0166e3ea420d 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -227,6 +227,27 @@ pub const Wip = struct { return @fromBackingInt(@intCast(revert_index)); } + pub fn addStringListInterned(wip: *Wip, list: []const String) Allocator.Error!StringList { + // Increase size of extra to support the list. Add the string list + // there. Then check for duplicate, reverting list if already found. + const gpa = wip.gpa; + const revert_index: u32 = @intCast(wip.extra.items.len); + const added = try wip.extra.addManyAsSlice(gpa, list.len + 1); + added[0] = @intCast(list.len); + @memcpy(added[1..], @as([]const u32, @ptrCast(list))); + const gop = try wip.dedupe_table.getOrPutContext(gpa, .{ + .index = revert_index, + .len = @intCast(added.len), + }, @as(ExtraSlice.Context, .{ .extra = wip.extra.items })); + + if (gop.found_existing) { + wip.extra.items.len = revert_index; + return @fromBackingInt(@intCast(gop.key_ptr.index)); + } + + return @fromBackingInt(@intCast(revert_index)); + } + pub fn addBytes(wip: *Wip, bytes: []const u8) Allocator.Error!Bytes { try wip.string_bytes.appendSlice(wip.gpa, bytes); return .{ @@ -479,6 +500,7 @@ pub const Step = extern struct { owner: Package.Index, deps: Deps.Index, max_rss: MaxRss, + unlazy_deps: OptionalStringList, extended: Storage.Extended(Flags, union(Tag) { check_file: CheckFile, compile: Compile, diff --git a/lib/std/Build/Serialize.zig b/lib/std/Build/Serialize.zig index 802bf4cbcff33db51ca5c406828e5325bb707895..82c2491c140a5a8248878cd49372ee3d197ae904 100644 --- a/lib/std/Build/Serialize.zig +++ b/lib/std/Build/Serialize.zig @@ -681,14 +681,22 @@ pub fn write(b: *std.Build, wc: *Configuration.Wip, writer: *std.Io.Writer) !voi .owner = try s.builderToPackage(step.owner), .deps = deps, .max_rss = .fromBytes(step.max_rss), + .unlazy_deps = if (step.unlazy_deps.count() != 0) + .init(try wc.addStringListInterned(step.unlazy_deps.keys())) + else + .none, .extended = @fromBackingInt(extended), }); } } - try wc.unlazy_deps.ensureUnusedCapacity(gpa, graph.needed_lazy_dependencies.keys().len); - for (graph.needed_lazy_dependencies.keys()) |k| { - wc.unlazy_deps.appendAssumeCapacity(try wc.addString(k)); + { + // TODO exchange graph.unlazy_deps.keys with the wc.unlazy_deps array list without copying + // MultiArrayList should support this when there is only 1 non zero sized struct field + const unlazy_deps = graph.unlazy_deps.keys(); + try wc.unlazy_deps.ensureTotalCapacityPrecise(gpa, unlazy_deps.len); + assert(wc.unlazy_deps.items.len == 0); + wc.unlazy_deps.appendSliceAssumeCapacity(unlazy_deps); } try wc.write(writer, .{ diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index a4d6987126ac87113fb73ce63dca3b75c89ffe9b..d37f227f2eb7aa58ef0c5c23ae84b00a8c1ecc56 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -36,6 +36,11 @@ max_rss: u64, /// to print along with debugging messages. debug_stack_trace: std.debug.StackTrace, +/// Managed by: +/// * `Build.declareDependencyStep` +/// * `Build.dependencyStep` +unlazy_deps: std.array_hash_map.Auto(Configuration.String, void), + pub const Tag = Configuration.Step.Tag; pub fn Type(comptime tag: Tag) type { @@ -104,6 +109,7 @@ pub fn init(options: StepOptions) Step { const first_ret_addr = options.first_ret_addr orelse @returnAddress(); break :blk std.debug.captureCurrentStackTrace(.{ .first_address = first_ret_addr }, addr_buf); }, + .unlazy_deps = .empty, }; } -- 2.54.0 From c3cdf484072d84f32e19c9286071a913269c90e9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 15 Oct 2026 16:37:06 -0700 Subject: [PATCH 2/5] Configuration: store unlazy dependencies differently steps are many; unlazy dependencies are few --- lib/compiler/Maker.zig | 24 ++++++++++++++-------- lib/compiler/Maker/ScannedConfig.zig | 4 +++- lib/std/Build/Configuration.zig | 30 ++++++++++++++++++++++++---- lib/std/Build/Serialize.zig | 23 +++++++++++---------- 4 files changed, 57 insertions(+), 24 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index d9f0b132550e12a42f92acdda4be520b465439c5..04424cc4e97cebca0d5ba50d85c9935897826344 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1608,16 +1608,24 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { if (configuration.unlazy_deps.len != 0) { var any_errors = false; - for (configuration.unlazy_deps) |hash_string| { - const hash = hash_string.slice(&configuration); - assert(hash.len != 0); - if (hash.len > Package.Hash.max_len) { - log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash }); + for (configuration.unlazy_deps) |unlazy_dep| { + if (unlazy_dep.step.unwrap()) |step_index| { + log.err("TODO implement step-associated lazy dependencies in maker", .{}); any_errors = true; - continue; + _ = step_index; + } else { + for (unlazy_dep.list.slice(&configuration)) |hash_string| { + const hash = hash_string.slice(&configuration); + assert(hash.len != 0); + if (hash.len > Package.Hash.max_len) { + log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash }); + any_errors = true; + continue; + } + log.info("fetching lazy dependency {s}", .{hash}); + try unlazy_set.put(arena, .fromSlice(hash), {}); + } } - log.info("fetching lazy dependency {s}", .{hash}); - try unlazy_set.put(arena, .fromSlice(hash), {}); } if (any_errors) return error.FailedButCacheIntact; if (options.system_pkg_dir_path) |p| { diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index bae416b9bb2523a6b042bfa118d8a60ae43213ed..4e83388ee9f094c98a36e79f504e76b37fb9f148 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -55,7 +55,9 @@ pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { { var tf = try s.beginTupleField("unlazy_deps", .{}); for (c.unlazy_deps) |dep| { - try tf.field(dep.slice(c), .{}); + var field = try tf.beginStructField(.{}); + try printStruct(sc, &field, Configuration.UnlazyDep, dep); + try field.end(); } try tf.end(); } diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index d117212bb646595d05fab4625e4c0166e3ea420d..ee7b77085b177a456bacbe6115619197c0d86e49 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -11,7 +11,7 @@ const native_endian = builtin.target.cpu.arch.endian(); string_bytes: []u8, steps: []Step, path_deps: []PathDep, -unlazy_deps: []String, +unlazy_deps: []UnlazyDep, system_integrations: []SystemIntegration, available_options: []AvailableOption, search_prefixes: []String, @@ -52,7 +52,7 @@ pub const Wip = struct { targets_table: TargetsTable = .empty, string_bytes: std.ArrayList(u8) = .empty, - unlazy_deps: std.ArrayList(String) = .empty, + unlazy_deps: std.ArrayList(UnlazyDep) = .empty, system_integrations: std.ArrayList(SystemIntegration) = .empty, available_options: std.ArrayList(AvailableOption) = .empty, steps: std.ArrayList(Step) = .empty, @@ -495,12 +495,17 @@ pub const AvailableOption = extern struct { }; }; +pub const UnlazyDep = extern struct { + /// none indicates not associated with any particular step. + step: Step.OptionalIndex, + list: StringList, +}; + pub const Step = extern struct { name: String, owner: Package.Index, deps: Deps.Index, max_rss: MaxRss, - unlazy_deps: OptionalStringList, extended: Storage.Extended(Flags, union(Tag) { check_file: CheckFile, compile: Compile, @@ -529,6 +534,23 @@ pub const Step = extern struct { } }; + pub const OptionalIndex = enum(u32) { + none = max_u32, + _, + + pub fn unwrap(this: @This()) ?Index { + return switch (this) { + .none => null, + else => @fromBackingInt(@intCast(@backingInt(this))), + }; + } + + pub fn ptr(i: @This(), c: *const Configuration) ?*const Step { + if (i == .none) return null; + return &c.steps[@backingInt(i)]; + } + }; + /// Shared by all steps. pub const Flags = packed struct(u32) { tag: Tag, @@ -3156,7 +3178,7 @@ pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration { .string_bytes = try arena.alloc(u8, header.string_bytes_len), .steps = try arena.alloc(Step, header.steps_len), .path_deps = try arena.alloc(PathDep, header.path_deps_len), - .unlazy_deps = try arena.alloc(String, header.unlazy_deps_len), + .unlazy_deps = try arena.alloc(UnlazyDep, header.unlazy_deps_len), .system_integrations = try arena.alloc(SystemIntegration, header.system_integrations_len), .available_options = try arena.alloc(AvailableOption, header.available_options_len), .search_prefixes = try arena.alloc(String, header.search_prefixes_len), diff --git a/lib/std/Build/Serialize.zig b/lib/std/Build/Serialize.zig index 82c2491c140a5a8248878cd49372ee3d197ae904..3cf0ca946624a467b17c7e238be6e8aca79c12d9 100644 --- a/lib/std/Build/Serialize.zig +++ b/lib/std/Build/Serialize.zig @@ -681,22 +681,23 @@ pub fn write(b: *std.Build, wc: *Configuration.Wip, writer: *std.Io.Writer) !voi .owner = try s.builderToPackage(step.owner), .deps = deps, .max_rss = .fromBytes(step.max_rss), - .unlazy_deps = if (step.unlazy_deps.count() != 0) - .init(try wc.addStringListInterned(step.unlazy_deps.keys())) - else - .none, .extended = @fromBackingInt(extended), }); + + if (step.unlazy_deps.count() != 0) { + try wc.unlazy_deps.append(gpa, .{ + .step = @fromBackingInt(@intCast(wc.steps.items.len - 1)), + .list = try wc.addStringListInterned(step.unlazy_deps.keys()), + }); + } } } - { - // TODO exchange graph.unlazy_deps.keys with the wc.unlazy_deps array list without copying - // MultiArrayList should support this when there is only 1 non zero sized struct field - const unlazy_deps = graph.unlazy_deps.keys(); - try wc.unlazy_deps.ensureTotalCapacityPrecise(gpa, unlazy_deps.len); - assert(wc.unlazy_deps.items.len == 0); - wc.unlazy_deps.appendSliceAssumeCapacity(unlazy_deps); + if (graph.unlazy_deps.count() != 0) { + try wc.unlazy_deps.append(gpa, .{ + .step = .none, + .list = try wc.addStringListInterned(graph.unlazy_deps.keys()), + }); } try wc.write(writer, .{ -- 2.54.0 From d67152361ce131a7d49bbb8fbc0819578448d502 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 15 Oct 2026 16:51:26 -0700 Subject: [PATCH 3/5] Maker: extract checkSufficientMemory from prepare --- lib/compiler/Maker.zig | 87 +++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 36 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 04424cc4e97cebca0d5ba50d85c9935897826344..d048d3ecd3b2fd45256699ffc0940cbeedc1a91e 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -915,14 +915,20 @@ pub fn main(init: process.Init.Minimal) !void { maker.watch = body.flags.watch; maker.prepare(steps, &configure_source_files) catch |err| switch (err) { - error.DependencyLoopDetected, error.InsufficientMemory => { - // TODO handle DependencyLoopDetected as error.FailedButCacheIntact - // and handle InsufficientMemory as error.AlreadyReported + error.DependencyLoopDetected => { + // TODO handle error.FailedButCacheIntact _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; process.exit(1); }, else => |e| return e, }; + maker.checkSufficientMemory() catch |err| switch (err) { + error.InsufficientMemory => { + // TODO handle as error.AlreadyReported + _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; + process.exit(1); + }, + }; try maker.makeSteps(main_progress_node, null); @@ -980,14 +986,20 @@ pub fn main(init: process.Init.Minimal) !void { defer gpa.free(initial_steps); maker.prepare(initial_steps, &configure_source_files) catch |err| switch (err) { - error.DependencyLoopDetected, error.InsufficientMemory => { - // TODO handle DependencyLoopDetected as error.FailedButCacheIntact - // and handle InsufficientMemory as error.AlreadyReported + error.DependencyLoopDetected => { + // TODO handle as error.FailedButCacheIntact _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; process.exit(1); }, else => |e| return e, }; + maker.checkSufficientMemory() catch |err| switch (err) { + error.InsufficientMemory => { + // TODO handle as error.AlreadyReported + _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; + process.exit(1); + }, + }; var w: Watch = w: { if (!watch_flag) break :w undefined; @@ -2695,41 +2707,44 @@ fn prepare( for (starting_steps) |s| { try constructGraphAndCheckForDependencyLoop(maker, s, &maker.step_stack, rand); } +} - { - // Check that we have enough memory to complete the build. - var any_problems = false; - var max_needed: u64 = 0; - for (step_stack.keys()) |step_index| { - const make_step = maker.stepByIndex(step_index); - const conf_step = step_index.ptr(c); - const max_rss = conf_step.max_rss.toBytes(); - if (max_rss == 0) continue; - max_needed = @max(max_needed, max_rss); - if (max_rss > maker.available_rss) { - if (maker.skip_oom_steps) { - make_step.state = .skipped_oom; - for (make_step.dependants.items) |dependant| { - maker.stepByIndex(dependant).pending_deps -= 1; - } - } else { - log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{ - conf_step.owner.depPrefixSlice(c), - conf_step.name.slice(c), - max_rss, - maker.available_rss, - }); - any_problems = true; +fn checkSufficientMemory(maker: *Maker) error{InsufficientMemory}!void { + const step_stack = &maker.step_stack; + const c = &maker.scanned_config.configuration; + + // Check that we have enough memory to complete the build. + var any_problems = false; + var max_needed: u64 = 0; + for (step_stack.keys()) |step_index| { + const make_step = maker.stepByIndex(step_index); + const conf_step = step_index.ptr(c); + const max_rss = conf_step.max_rss.toBytes(); + if (max_rss == 0) continue; + max_needed = @max(max_needed, max_rss); + if (max_rss > maker.available_rss) { + if (maker.skip_oom_steps) { + make_step.state = .skipped_oom; + for (make_step.dependants.items) |dependant| { + maker.stepByIndex(dependant).pending_deps -= 1; } + } else { + log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{ + conf_step.owner.depPrefixSlice(c), + conf_step.name.slice(c), + max_rss, + maker.available_rss, + }); + any_problems = true; } } - if (any_problems) { - log.info("use --skip-oom-steps to proceed, skipping memory limited steps", .{}); - if (maker.max_rss_is_default) { - log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{max_needed}); - } - return error.InsufficientMemory; + } + if (any_problems) { + log.info("use --skip-oom-steps to proceed, skipping memory limited steps", .{}); + if (maker.max_rss_is_default) { + log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{max_needed}); } + return error.InsufficientMemory; } } -- 2.54.0 From 215db865490b5d4d08171297603584841be97976 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 15 Oct 2026 18:01:23 -0700 Subject: [PATCH 4/5] Maker: implement refetching from step-associated lazy deps closes #37262 --- lib/compiler/Maker.zig | 145 +++++++++++++++++++---------------- lib/compiler/Maker/Fetch.zig | 6 +- 2 files changed, 83 insertions(+), 68 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index d048d3ecd3b2fd45256699ffc0940cbeedc1a91e..94231ca66cc5ff978ac97ef33c1ea616afa72710 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -763,6 +763,8 @@ pub fn main(init: process.Init.Minimal) !void { break :s &protocol_server_allocation; } else null; + var discovered_unlazy_set: Fetch.JobQueue.UnlazySet = .empty; // uses global arena + configure: while (true) { // Set of files that, if modified, imply that recompiling and rerunning // configurer is needed. @@ -792,9 +794,12 @@ pub fn main(init: process.Init.Minimal) !void { .system_pkg_dir_path = system_pkg_dir_path, .print_configuration = print_configuration, .forks = forks.items, + .unlazy_set = &discovered_unlazy_set, .src_files = &configure_source_files, .error_bundle = if (protocol_server != null) &configure_error_bundle else null, })) |scanned_config| { + discovered_unlazy_set.clearRetainingCapacity(); + if (help_menu) { scanned_config.printUsage(&graph, initStdoutWriter(io)) catch |err| switch (err) { error.WriteFailed => return stdout_writer_allocation.err.?, @@ -914,20 +919,15 @@ pub fn main(init: process.Init.Minimal) !void { try select.concurrent(.message, Server.receiveMessage, .{s}); maker.watch = body.flags.watch; - maker.prepare(steps, &configure_source_files) catch |err| switch (err) { - error.DependencyLoopDetected => { - // TODO handle error.FailedButCacheIntact - _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; - process.exit(1); - }, - else => |e| return e, - }; - maker.checkSufficientMemory() catch |err| switch (err) { - error.InsufficientMemory => { - // TODO handle as error.AlreadyReported + maker.prepare(steps, &configure_source_files, &discovered_unlazy_set) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.DependencyLoopDetected, error.InsufficientMemory => { + // TODO handle error.DependencyLoopDetected as error.FailedButCacheIntact + // TODO handle error.InsufficientMemory as error.AlreadyReported _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; process.exit(1); }, + error.LazyDependencyNeeded => continue :configure, }; try maker.makeSteps(main_progress_node, null); @@ -985,20 +985,15 @@ pub fn main(init: process.Init.Minimal) !void { const initial_steps = try maker.resolveTopLevelSteps(step_names.items); defer gpa.free(initial_steps); - maker.prepare(initial_steps, &configure_source_files) catch |err| switch (err) { - error.DependencyLoopDetected => { - // TODO handle as error.FailedButCacheIntact - _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; - process.exit(1); - }, - else => |e| return e, - }; - maker.checkSufficientMemory() catch |err| switch (err) { - error.InsufficientMemory => { - // TODO handle as error.AlreadyReported + maker.prepare(initial_steps, &configure_source_files, &discovered_unlazy_set) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.DependencyLoopDetected, error.InsufficientMemory => { + // TODO handle error.DependencyLoopDetected as error.FailedButCacheIntact + // TODO handle error.InsufficientMemory as error.AlreadyReported _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; process.exit(1); }, + error.LazyDependencyNeeded => continue :configure, }; var w: Watch = w: { @@ -1146,6 +1141,7 @@ const ConfigureOptions = struct { system_pkg_dir_path: ?[]const u8, print_configuration: PrintConfiguration, forks: []Fork, + unlazy_set: *Fetch.JobQueue.UnlazySet, src_files: *Cache.Manifest.SelfContainedFiles, error_bundle: ?*ErrorBundle = null, }; @@ -1161,8 +1157,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { var http_client: std.http.Client = .{ .allocator = gpa, .io = io }; defer http_client.deinit(); - var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; - var fork_set: Package.Fetch.JobQueue.ForkSet = .{}; + var fork_set: Fetch.JobQueue.ForkSet = .{}; { // Populate fork_set. @@ -1297,7 +1292,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { // Reset fork match counts. for (fork_set.keys()) |*fork| fork.uses = 0; - var job_queue: Package.Fetch.JobQueue = .{ + var job_queue: Fetch.JobQueue = .{ .io = io, .http_client = &http_client, .global_cache = graph.global_cache_root, @@ -1307,7 +1302,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { }, .recursive = true, .debug_hash = false, - .unlazy_set = unlazy_set, + .unlazy_set = options.unlazy_set.*, .fork_set = fork_set, .mode = options.fetch_mode, .prog_node = fetch_prog_node, @@ -1324,7 +1319,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { const phantom_package_root: Cache.Path = .{ .root_dir = options.build_root.directory }; - var fetch: Package.Fetch = .{ + var fetch: Fetch = .{ .arena = std.heap.ArenaAllocator.init(gpa), .location = .{ .relative_path = phantom_package_root }, .location_tok = 0, @@ -1356,11 +1351,11 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { job_queue.all_fetches.appendAssumeCapacity(&fetch); job_queue.table.putAssumeCapacityNoClobber( - Package.Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root), + Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root), &fetch, ); - job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" }); + job_queue.group.async(io, Fetch.workerRun, .{ &fetch, "root" }); try job_queue.group.await(io); { @@ -1501,7 +1496,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { const dep_names = man.dependencies.keys(); try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); for (dep_names, man.dependencies.values()) |name, *dep| { - const dep_digest = Package.Fetch.depDigest( + const dep_digest = Fetch.depDigest( f.package_root, graph.global_cache_root, dep, @@ -1620,38 +1615,33 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { if (configuration.unlazy_deps.len != 0) { var any_errors = false; + options.unlazy_set.clearRetainingCapacity(); for (configuration.unlazy_deps) |unlazy_dep| { - if (unlazy_dep.step.unwrap()) |step_index| { - log.err("TODO implement step-associated lazy dependencies in maker", .{}); - any_errors = true; - _ = step_index; - } else { - for (unlazy_dep.list.slice(&configuration)) |hash_string| { - const hash = hash_string.slice(&configuration); - assert(hash.len != 0); - if (hash.len > Package.Hash.max_len) { - log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash }); - any_errors = true; - continue; - } - log.info("fetching lazy dependency {s}", .{hash}); - try unlazy_set.put(arena, .fromSlice(hash), {}); + if (unlazy_dep.step != .none) continue; // Handled in `prepare`. + for (unlazy_dep.list.slice(&configuration)) |hash_string| { + const hash = hash_string.slice(&configuration); + assert(hash.len != 0); + if (hash.len > Package.Hash.max_len) { + log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash }); + any_errors = true; + continue; } + log.info("fetching lazy dependency {s}", .{hash}); + try options.unlazy_set.put(arena, .fromSlice(hash), {}); } } if (any_errors) return error.FailedButCacheIntact; if (options.system_pkg_dir_path) |p| { - // In this mode, the system needs to provide these packages; they - // cannot be fetched by Zig. + // In this mode, the system needs to provide these packages; they cannot be fetched by Zig. const s = Dir.path.sep_str; - for (unlazy_set.keys()) |*hash| { + for (options.unlazy_set.keys()) |*hash| { log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() }); } log.info("remote package fetching disabled due to --system mode", .{}); log.info("dependencies might be avoidable depending on build configuration", .{}); return error.FailedButCacheIntact; } - continue :cp; + if (options.unlazy_set.count() != 0) continue :cp; } if (config_man) |man| for (configuration.path_deps) |path_dep| { @@ -1791,7 +1781,7 @@ const ValidFingerprintTable = std.array_hash_map.Custom( fn validateFingerprints( gpa: Allocator, job_queue: *const Fetch.JobQueue, - fork_set: *const Package.Fetch.JobQueue.ForkSet, + fork_set: *const Fetch.JobQueue.ForkSet, ) !void { var any_errors = false; @@ -1846,7 +1836,7 @@ fn printPkgTree( global_cache_root: Cache.Directory, job_queue: *const Fetch.JobQueue, root_fetch: *const Fetch, - fork_set: *const Package.Fetch.JobQueue.ForkSet, + fork_set: *const Fetch.JobQueue.ForkSet, ) !void { if (!root_fetch.have_manifest) return; const names = root_fetch.manifest.dependencies.keys(); @@ -1861,7 +1851,7 @@ fn printPkgTree( try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set); }, .path => { - const child_hash = Package.Fetch.depDigest(root_fetch.package_root, global_cache_root, dep); + const child_hash = Fetch.depDigest(root_fetch.package_root, global_cache_root, dep); try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set); }, } @@ -1876,7 +1866,7 @@ fn printPkgTreeChild( parent_print_node: *const PrintNode, dep_name: []const u8, opt_pkg_hash: ?Package.Hash, - fork_set: *const Package.Fetch.JobQueue.ForkSet, + fork_set: *const Fetch.JobQueue.ForkSet, ) !void { const w = terminal.writer; try parent_print_node.printPrefixedChild(w, terminal.mode); @@ -1933,7 +1923,7 @@ fn printPkgTreeChild( try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set); }, .path => { - const child_hash = Package.Fetch.depDigest(child_fetch.package_root, global_cache_root, dep); + const child_hash = Fetch.depDigest(child_fetch.package_root, global_cache_root, dep); try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set); }, } @@ -2660,7 +2650,8 @@ fn prepare( maker: *Maker, step_indices: []const Configuration.Step.Index, configure_source_files: *const Cache.Manifest.SelfContainedFiles, -) !void { + discovered_unlazy_set: *Fetch.JobQueue.UnlazySet, +) error{ InsufficientMemory, OutOfMemory, DependencyLoopDetected, LazyDependencyNeeded }!void { const gpa = maker.gpa; const graph = maker.graph; const arena = graph.arena; @@ -2707,6 +2698,30 @@ fn prepare( for (starting_steps) |s| { try constructGraphAndCheckForDependencyLoop(maker, s, &maker.step_stack, rand); } + + // Notice when a step's participation in the evaluation graph implies that additional package dependencies + // must be fetched. + var needed_step_deps: std.array_hash_map.Auto(Configuration.Step.Index, Configuration.StringList) = .empty; + defer needed_step_deps.deinit(gpa); + + for (c.unlazy_deps) |unlazy_dep| { + const step_index = unlazy_dep.step.unwrap() orelse continue; // Handled in `configure`. + try needed_step_deps.put(gpa, step_index, unlazy_dep.list); + } + + assert(discovered_unlazy_set.count() == 0); + for (step_stack.keys()) |step_index| { + const interned_string_list = needed_step_deps.get(step_index) orelse continue; + const hash_strings = interned_string_list.slice(c); + for (hash_strings) |hash_string| { + const hash = hash_string.slice(c); + log.debug("due to step {s}, fetching lazy {s}", .{ step_index.ptr(c).name.slice(c), hash }); + try discovered_unlazy_set.put(arena, .fromSlice(hash), {}); + } + } + if (discovered_unlazy_set.count() != 0) return error.LazyDependencyNeeded; + + try checkSufficientMemory(maker); } fn checkSufficientMemory(maker: *Maker) error{InsufficientMemory}!void { @@ -3464,17 +3479,17 @@ fn printTreeChildLast(writer: *Writer, mode: Io.Terminal.Mode) !void { }); } -/// Traverse the dependency graph depth-first and make it undirected by having -/// steps know their dependants (they only know dependencies at start). -/// Along the way, check that there is no dependency loop, and record the steps -/// in traversal order in `step_stack`. -/// Each step has its dependencies traversed in random order, this accomplishes -/// two things: -/// - `step_stack` will be in randomized-depth-first order, so the build runner -/// spawns initial steps in a random order -/// - each step's `dependants` list is also filled in a random order, so that -/// when it finishes executing in `makeStep`, it spawns next steps to run in +/// Traverse the dependency graph depth-first and make it undirected by having steps know their dependants +/// (they only know dependencies at start). +/// +/// Along the way, check that there is no dependency loop, and record the steps in traversal order in +/// `step_stack`. +/// +/// Each step has its dependencies traversed in random order, this accomplishes two things: +/// - `step_stack` will be in randomized-depth-first order, so the build runner spawns initial steps in a /// random order +/// - each step's `dependants` list is also filled in a random order, so that when it finishes executing in +/// `makeStep`, it spawns next steps to run in random order fn constructGraphAndCheckForDependencyLoop( maker: *Maker, step_index: Configuration.Step.Index, diff --git a/lib/compiler/Maker/Fetch.zig b/lib/compiler/Maker/Fetch.zig index 68294ffe3a856cd564bb4e365cb93b08189ab6a4..333379489592dfcdb666e127adc67a2484301dc6 100644 --- a/lib/compiler/Maker/Fetch.zig +++ b/lib/compiler/Maker/Fetch.zig @@ -153,11 +153,11 @@ pub const JobQueue = struct { debug_hash: bool, mode: Mode, /// Set of hashes that will be additionally fetched even if they are marked - /// as lazy. - unlazy_set: UnlazySet = .{}, + /// as lazy. Read-only. + unlazy_set: UnlazySet = .empty, /// Identifies paths that override all packages in the tree with matching /// project ids. - fork_set: ForkSet = .{}, + fork_set: ForkSet = .empty, pub const Mode = enum { /// Non-lazy dependencies are always fetched. -- 2.54.0 From 156ab92d9de233a712cb29b6ac0ccc5780c0c18f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 15 Oct 2026 18:04:38 -0700 Subject: [PATCH 5/5] Fetch: fix break on wrong line in createDependenciesSource closes #37261 --- lib/compiler/Maker/Fetch.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compiler/Maker/Fetch.zig b/lib/compiler/Maker/Fetch.zig index 333379489592dfcdb666e127adc67a2484301dc6..761457d6e0a4ccc404608535b7e02f13a56b1b69 100644 --- a/lib/compiler/Maker/Fetch.zig +++ b/lib/compiler/Maker/Fetch.zig @@ -297,8 +297,8 @@ pub const JobQueue = struct { \\ pub const build_zig = @import({qf}); \\ , .{fork.path}); + break :b; } - break :b; } try w.print( \\ pub const build_zig = @import({q}); -- 2.54.0