authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-10-16 21:24:32+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-10-16 21:24:32+02:00
log4e293f3e4d6c7072962513bee0ca8643becb81aa
treeaf44b12756c31630503f4379981ec261002a6ee7
parentd29e4bd653efec872c83d66419a846eebf6c359a
parent156ab92d9de233a712cb29b6ac0ccc5780c0c18f

Merge pull request 'ability to explicitly declare dependencies, and ability to associate dependency with particular steps' (#37269) from lazy-step-dependencies into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/37269

7 files changed, 263 insertions(+), 103 deletions(-)

lib/compiler/Maker.zig+113-75
...@@ -763,6 +763,8 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -763,6 +763,8 @@ pub fn main(init: process.Init.Minimal) !void {
763 break :s &protocol_server_allocation;763 break :s &protocol_server_allocation;
764 } else null;764 } else null;
765765
766 var discovered_unlazy_set: Fetch.JobQueue.UnlazySet = .empty; // uses global arena
767
766 configure: while (true) {768 configure: while (true) {
767 // Set of files that, if modified, imply that recompiling and rerunning769 // Set of files that, if modified, imply that recompiling and rerunning
768 // configurer is needed.770 // configurer is needed.
...@@ -792,9 +794,12 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -792,9 +794,12 @@ pub fn main(init: process.Init.Minimal) !void {
792 .system_pkg_dir_path = system_pkg_dir_path,794 .system_pkg_dir_path = system_pkg_dir_path,
793 .print_configuration = print_configuration,795 .print_configuration = print_configuration,
794 .forks = forks.items,796 .forks = forks.items,
797 .unlazy_set = &discovered_unlazy_set,
795 .src_files = &configure_source_files,798 .src_files = &configure_source_files,
796 .error_bundle = if (protocol_server != null) &configure_error_bundle else null,799 .error_bundle = if (protocol_server != null) &configure_error_bundle else null,
797 })) |scanned_config| {800 })) |scanned_config| {
801 discovered_unlazy_set.clearRetainingCapacity();
802
798 if (help_menu) {803 if (help_menu) {
799 scanned_config.printUsage(&graph, initStdoutWriter(io)) catch |err| switch (err) {804 scanned_config.printUsage(&graph, initStdoutWriter(io)) catch |err| switch (err) {
800 error.WriteFailed => return stdout_writer_allocation.err.?,805 error.WriteFailed => return stdout_writer_allocation.err.?,
...@@ -914,14 +919,15 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -914,14 +919,15 @@ pub fn main(init: process.Init.Minimal) !void {
914 try select.concurrent(.message, Server.receiveMessage, .{s});919 try select.concurrent(.message, Server.receiveMessage, .{s});
915920
916 maker.watch = body.flags.watch;921 maker.watch = body.flags.watch;
917 maker.prepare(steps, &configure_source_files) catch |err| switch (err) {922 maker.prepare(steps, &configure_source_files, &discovered_unlazy_set) catch |err| switch (err) {
923 error.OutOfMemory => |e| return e,
918 error.DependencyLoopDetected, error.InsufficientMemory => {924 error.DependencyLoopDetected, error.InsufficientMemory => {
919 // TODO handle DependencyLoopDetected as error.FailedButCacheIntact925 // TODO handle error.DependencyLoopDetected as error.FailedButCacheIntact
920 // and handle InsufficientMemory as error.AlreadyReported926 // TODO handle error.InsufficientMemory as error.AlreadyReported
921 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};927 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
922 process.exit(1);928 process.exit(1);
923 },929 },
924 else => |e| return e,930 error.LazyDependencyNeeded => continue :configure,
925 };931 };
926932
927 try maker.makeSteps(main_progress_node, null);933 try maker.makeSteps(main_progress_node, null);
...@@ -979,14 +985,15 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -979,14 +985,15 @@ pub fn main(init: process.Init.Minimal) !void {
979 const initial_steps = try maker.resolveTopLevelSteps(step_names.items);985 const initial_steps = try maker.resolveTopLevelSteps(step_names.items);
980 defer gpa.free(initial_steps);986 defer gpa.free(initial_steps);
981987
982 maker.prepare(initial_steps, &configure_source_files) catch |err| switch (err) {988 maker.prepare(initial_steps, &configure_source_files, &discovered_unlazy_set) catch |err| switch (err) {
989 error.OutOfMemory => |e| return e,
983 error.DependencyLoopDetected, error.InsufficientMemory => {990 error.DependencyLoopDetected, error.InsufficientMemory => {
984 // TODO handle DependencyLoopDetected as error.FailedButCacheIntact991 // TODO handle error.DependencyLoopDetected as error.FailedButCacheIntact
985 // and handle InsufficientMemory as error.AlreadyReported992 // TODO handle error.InsufficientMemory as error.AlreadyReported
986 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};993 _ = io.lockStderr(&.{}, graph.stderr_mode) catch {};
987 process.exit(1);994 process.exit(1);
988 },995 },
989 else => |e| return e,996 error.LazyDependencyNeeded => continue :configure,
990 };997 };
991998
992 var w: Watch = w: {999 var w: Watch = w: {
...@@ -1134,6 +1141,7 @@ const ConfigureOptions = struct {...@@ -1134,6 +1141,7 @@ const ConfigureOptions = struct {
1134 system_pkg_dir_path: ?[]const u8,1141 system_pkg_dir_path: ?[]const u8,
1135 print_configuration: PrintConfiguration,1142 print_configuration: PrintConfiguration,
1136 forks: []Fork,1143 forks: []Fork,
1144 unlazy_set: *Fetch.JobQueue.UnlazySet,
1137 src_files: *Cache.Manifest.SelfContainedFiles,1145 src_files: *Cache.Manifest.SelfContainedFiles,
1138 error_bundle: ?*ErrorBundle = null,1146 error_bundle: ?*ErrorBundle = null,
1139};1147};
...@@ -1149,8 +1157,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1149,8 +1157,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
1149 var http_client: std.http.Client = .{ .allocator = gpa, .io = io };1157 var http_client: std.http.Client = .{ .allocator = gpa, .io = io };
1150 defer http_client.deinit();1158 defer http_client.deinit();
11511159
1152 var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{};1160 var fork_set: Fetch.JobQueue.ForkSet = .{};
1153 var fork_set: Package.Fetch.JobQueue.ForkSet = .{};
11541161
1155 {1162 {
1156 // Populate fork_set.1163 // Populate fork_set.
...@@ -1285,7 +1292,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1285,7 +1292,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
1285 // Reset fork match counts.1292 // Reset fork match counts.
1286 for (fork_set.keys()) |*fork| fork.uses = 0;1293 for (fork_set.keys()) |*fork| fork.uses = 0;
12871294
1288 var job_queue: Package.Fetch.JobQueue = .{1295 var job_queue: Fetch.JobQueue = .{
1289 .io = io,1296 .io = io,
1290 .http_client = &http_client,1297 .http_client = &http_client,
1291 .global_cache = graph.global_cache_root,1298 .global_cache = graph.global_cache_root,
...@@ -1295,7 +1302,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1295,7 +1302,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
1295 },1302 },
1296 .recursive = true,1303 .recursive = true,
1297 .debug_hash = false,1304 .debug_hash = false,
1298 .unlazy_set = unlazy_set,1305 .unlazy_set = options.unlazy_set.*,
1299 .fork_set = fork_set,1306 .fork_set = fork_set,
1300 .mode = options.fetch_mode,1307 .mode = options.fetch_mode,
1301 .prog_node = fetch_prog_node,1308 .prog_node = fetch_prog_node,
...@@ -1312,7 +1319,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1312,7 +1319,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
13121319
1313 const phantom_package_root: Cache.Path = .{ .root_dir = options.build_root.directory };1320 const phantom_package_root: Cache.Path = .{ .root_dir = options.build_root.directory };
13141321
1315 var fetch: Package.Fetch = .{1322 var fetch: Fetch = .{
1316 .arena = std.heap.ArenaAllocator.init(gpa),1323 .arena = std.heap.ArenaAllocator.init(gpa),
1317 .location = .{ .relative_path = phantom_package_root },1324 .location = .{ .relative_path = phantom_package_root },
1318 .location_tok = 0,1325 .location_tok = 0,
...@@ -1344,11 +1351,11 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1344,11 +1351,11 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
1344 job_queue.all_fetches.appendAssumeCapacity(&fetch);1351 job_queue.all_fetches.appendAssumeCapacity(&fetch);
13451352
1346 job_queue.table.putAssumeCapacityNoClobber(1353 job_queue.table.putAssumeCapacityNoClobber(
1347 Package.Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root),1354 Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root),
1348 &fetch,1355 &fetch,
1349 );1356 );
13501357
1351 job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" });1358 job_queue.group.async(io, Fetch.workerRun, .{ &fetch, "root" });
1352 try job_queue.group.await(io);1359 try job_queue.group.await(io);
13531360
1354 {1361 {
...@@ -1489,7 +1496,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1489,7 +1496,7 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
1489 const dep_names = man.dependencies.keys();1496 const dep_names = man.dependencies.keys();
1490 try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len));1497 try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len));
1491 for (dep_names, man.dependencies.values()) |name, *dep| {1498 for (dep_names, man.dependencies.values()) |name, *dep| {
1492 const dep_digest = Package.Fetch.depDigest(1499 const dep_digest = Fetch.depDigest(
1493 f.package_root,1500 f.package_root,
1494 graph.global_cache_root,1501 graph.global_cache_root,
1495 dep,1502 dep,
...@@ -1608,30 +1615,33 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {...@@ -1608,30 +1615,33 @@ fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig {
16081615
1609 if (configuration.unlazy_deps.len != 0) {1616 if (configuration.unlazy_deps.len != 0) {
1610 var any_errors = false;1617 var any_errors = false;
1611 for (configuration.unlazy_deps) |hash_string| {1618 options.unlazy_set.clearRetainingCapacity();
1612 const hash = hash_string.slice(&configuration);1619 for (configuration.unlazy_deps) |unlazy_dep| {
1613 assert(hash.len != 0);1620 if (unlazy_dep.step != .none) continue; // Handled in `prepare`.
1614 if (hash.len > Package.Hash.max_len) {1621 for (unlazy_dep.list.slice(&configuration)) |hash_string| {
1615 log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash });1622 const hash = hash_string.slice(&configuration);
1616 any_errors = true;1623 assert(hash.len != 0);
1617 continue;1624 if (hash.len > Package.Hash.max_len) {
1625 log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash });
1626 any_errors = true;
1627 continue;
1628 }
1629 log.info("fetching lazy dependency {s}", .{hash});
1630 try options.unlazy_set.put(arena, .fromSlice(hash), {});
1618 }1631 }
1619 log.info("fetching lazy dependency {s}", .{hash});
1620 try unlazy_set.put(arena, .fromSlice(hash), {});
1621 }1632 }
1622 if (any_errors) return error.FailedButCacheIntact;1633 if (any_errors) return error.FailedButCacheIntact;
1623 if (options.system_pkg_dir_path) |p| {1634 if (options.system_pkg_dir_path) |p| {
1624 // In this mode, the system needs to provide these packages; they1635 // In this mode, the system needs to provide these packages; they cannot be fetched by Zig.
1625 // cannot be fetched by Zig.
1626 const s = Dir.path.sep_str;1636 const s = Dir.path.sep_str;
1627 for (unlazy_set.keys()) |*hash| {1637 for (options.unlazy_set.keys()) |*hash| {
1628 log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() });1638 log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() });
1629 }1639 }
1630 log.info("remote package fetching disabled due to --system mode", .{});1640 log.info("remote package fetching disabled due to --system mode", .{});
1631 log.info("dependencies might be avoidable depending on build configuration", .{});1641 log.info("dependencies might be avoidable depending on build configuration", .{});
1632 return error.FailedButCacheIntact;1642 return error.FailedButCacheIntact;
1633 }1643 }
1634 continue :cp;1644 if (options.unlazy_set.count() != 0) continue :cp;
1635 }1645 }
16361646
1637 if (config_man) |man| for (configuration.path_deps) |path_dep| {1647 if (config_man) |man| for (configuration.path_deps) |path_dep| {
...@@ -1771,7 +1781,7 @@ const ValidFingerprintTable = std.array_hash_map.Custom(...@@ -1771,7 +1781,7 @@ const ValidFingerprintTable = std.array_hash_map.Custom(
1771fn validateFingerprints(1781fn validateFingerprints(
1772 gpa: Allocator,1782 gpa: Allocator,
1773 job_queue: *const Fetch.JobQueue,1783 job_queue: *const Fetch.JobQueue,
1774 fork_set: *const Package.Fetch.JobQueue.ForkSet,1784 fork_set: *const Fetch.JobQueue.ForkSet,
1775) !void {1785) !void {
1776 var any_errors = false;1786 var any_errors = false;
17771787
...@@ -1826,7 +1836,7 @@ fn printPkgTree(...@@ -1826,7 +1836,7 @@ fn printPkgTree(
1826 global_cache_root: Cache.Directory,1836 global_cache_root: Cache.Directory,
1827 job_queue: *const Fetch.JobQueue,1837 job_queue: *const Fetch.JobQueue,
1828 root_fetch: *const Fetch,1838 root_fetch: *const Fetch,
1829 fork_set: *const Package.Fetch.JobQueue.ForkSet,1839 fork_set: *const Fetch.JobQueue.ForkSet,
1830) !void {1840) !void {
1831 if (!root_fetch.have_manifest) return;1841 if (!root_fetch.have_manifest) return;
1832 const names = root_fetch.manifest.dependencies.keys();1842 const names = root_fetch.manifest.dependencies.keys();
...@@ -1841,7 +1851,7 @@ fn printPkgTree(...@@ -1841,7 +1851,7 @@ fn printPkgTree(
1841 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);1851 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);
1842 },1852 },
1843 .path => {1853 .path => {
1844 const child_hash = Package.Fetch.depDigest(root_fetch.package_root, global_cache_root, dep);1854 const child_hash = Fetch.depDigest(root_fetch.package_root, global_cache_root, dep);
1845 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);1855 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);
1846 },1856 },
1847 }1857 }
...@@ -1856,7 +1866,7 @@ fn printPkgTreeChild(...@@ -1856,7 +1866,7 @@ fn printPkgTreeChild(
1856 parent_print_node: *const PrintNode,1866 parent_print_node: *const PrintNode,
1857 dep_name: []const u8,1867 dep_name: []const u8,
1858 opt_pkg_hash: ?Package.Hash,1868 opt_pkg_hash: ?Package.Hash,
1859 fork_set: *const Package.Fetch.JobQueue.ForkSet,1869 fork_set: *const Fetch.JobQueue.ForkSet,
1860) !void {1870) !void {
1861 const w = terminal.writer;1871 const w = terminal.writer;
1862 try parent_print_node.printPrefixedChild(w, terminal.mode);1872 try parent_print_node.printPrefixedChild(w, terminal.mode);
...@@ -1913,7 +1923,7 @@ fn printPkgTreeChild(...@@ -1913,7 +1923,7 @@ fn printPkgTreeChild(
1913 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);1923 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);
1914 },1924 },
1915 .path => {1925 .path => {
1916 const child_hash = Package.Fetch.depDigest(child_fetch.package_root, global_cache_root, dep);1926 const child_hash = Fetch.depDigest(child_fetch.package_root, global_cache_root, dep);
1917 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);1927 try printPkgTreeChild(terminal, arena, global_cache_root, job_queue, &print_node, name, child_hash, fork_set);
1918 },1928 },
1919 }1929 }
...@@ -2640,7 +2650,8 @@ fn prepare(...@@ -2640,7 +2650,8 @@ fn prepare(
2640 maker: *Maker,2650 maker: *Maker,
2641 step_indices: []const Configuration.Step.Index,2651 step_indices: []const Configuration.Step.Index,
2642 configure_source_files: *const Cache.Manifest.SelfContainedFiles,2652 configure_source_files: *const Cache.Manifest.SelfContainedFiles,
2643) !void {2653 discovered_unlazy_set: *Fetch.JobQueue.UnlazySet,
2654) error{ InsufficientMemory, OutOfMemory, DependencyLoopDetected, LazyDependencyNeeded }!void {
2644 const gpa = maker.gpa;2655 const gpa = maker.gpa;
2645 const graph = maker.graph;2656 const graph = maker.graph;
2646 const arena = graph.arena;2657 const arena = graph.arena;
...@@ -2688,40 +2699,67 @@ fn prepare(...@@ -2688,40 +2699,67 @@ fn prepare(
2688 try constructGraphAndCheckForDependencyLoop(maker, s, &maker.step_stack, rand);2699 try constructGraphAndCheckForDependencyLoop(maker, s, &maker.step_stack, rand);
2689 }2700 }
26902701
2691 {2702 // Notice when a step's participation in the evaluation graph implies that additional package dependencies
2692 // Check that we have enough memory to complete the build.2703 // must be fetched.
2693 var any_problems = false;2704 var needed_step_deps: std.array_hash_map.Auto(Configuration.Step.Index, Configuration.StringList) = .empty;
2694 var max_needed: u64 = 0;2705 defer needed_step_deps.deinit(gpa);
2695 for (step_stack.keys()) |step_index| {2706
2696 const make_step = maker.stepByIndex(step_index);2707 for (c.unlazy_deps) |unlazy_dep| {
2697 const conf_step = step_index.ptr(c);2708 const step_index = unlazy_dep.step.unwrap() orelse continue; // Handled in `configure`.
2698 const max_rss = conf_step.max_rss.toBytes();2709 try needed_step_deps.put(gpa, step_index, unlazy_dep.list);
2699 if (max_rss == 0) continue;2710 }
2700 max_needed = @max(max_needed, max_rss);2711
2701 if (max_rss > maker.available_rss) {2712 assert(discovered_unlazy_set.count() == 0);
2702 if (maker.skip_oom_steps) {2713 for (step_stack.keys()) |step_index| {
2703 make_step.state = .skipped_oom;2714 const interned_string_list = needed_step_deps.get(step_index) orelse continue;
2704 for (make_step.dependants.items) |dependant| {2715 const hash_strings = interned_string_list.slice(c);
2705 maker.stepByIndex(dependant).pending_deps -= 1;2716 for (hash_strings) |hash_string| {
2706 }2717 const hash = hash_string.slice(c);
2707 } else {2718 log.debug("due to step {s}, fetching lazy {s}", .{ step_index.ptr(c).name.slice(c), hash });
2708 log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{2719 try discovered_unlazy_set.put(arena, .fromSlice(hash), {});
2709 conf_step.owner.depPrefixSlice(c),2720 }
2710 conf_step.name.slice(c),2721 }
2711 max_rss,2722 if (discovered_unlazy_set.count() != 0) return error.LazyDependencyNeeded;
2712 maker.available_rss,2723
2713 });2724 try checkSufficientMemory(maker);
2714 any_problems = true;2725}
2726
2727fn checkSufficientMemory(maker: *Maker) error{InsufficientMemory}!void {
2728 const step_stack = &maker.step_stack;
2729 const c = &maker.scanned_config.configuration;
2730
2731 // Check that we have enough memory to complete the build.
2732 var any_problems = false;
2733 var max_needed: u64 = 0;
2734 for (step_stack.keys()) |step_index| {
2735 const make_step = maker.stepByIndex(step_index);
2736 const conf_step = step_index.ptr(c);
2737 const max_rss = conf_step.max_rss.toBytes();
2738 if (max_rss == 0) continue;
2739 max_needed = @max(max_needed, max_rss);
2740 if (max_rss > maker.available_rss) {
2741 if (maker.skip_oom_steps) {
2742 make_step.state = .skipped_oom;
2743 for (make_step.dependants.items) |dependant| {
2744 maker.stepByIndex(dependant).pending_deps -= 1;
2715 }2745 }
2746 } else {
2747 log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{
2748 conf_step.owner.depPrefixSlice(c),
2749 conf_step.name.slice(c),
2750 max_rss,
2751 maker.available_rss,
2752 });
2753 any_problems = true;
2716 }2754 }
2717 }2755 }
2718 if (any_problems) {2756 }
2719 log.info("use --skip-oom-steps to proceed, skipping memory limited steps", .{});2757 if (any_problems) {
2720 if (maker.max_rss_is_default) {2758 log.info("use --skip-oom-steps to proceed, skipping memory limited steps", .{});
2721 log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{max_needed});2759 if (maker.max_rss_is_default) {
2722 }2760 log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{max_needed});
2723 return error.InsufficientMemory;
2724 }2761 }
2762 return error.InsufficientMemory;
2725 }2763 }
2726}2764}
27272765
...@@ -3441,17 +3479,17 @@ fn printTreeChildLast(writer: *Writer, mode: Io.Terminal.Mode) !void {...@@ -3441,17 +3479,17 @@ fn printTreeChildLast(writer: *Writer, mode: Io.Terminal.Mode) !void {
3441 });3479 });
3442}3480}
34433481
3444/// Traverse the dependency graph depth-first and make it undirected by having3482/// Traverse the dependency graph depth-first and make it undirected by having steps know their dependants
3445/// steps know their dependants (they only know dependencies at start).3483/// (they only know dependencies at start).
3446/// Along the way, check that there is no dependency loop, and record the steps3484///
3447/// in traversal order in `step_stack`.3485/// Along the way, check that there is no dependency loop, and record the steps in traversal order in
3448/// Each step has its dependencies traversed in random order, this accomplishes3486/// `step_stack`.
3449/// two things:3487///
3450/// - `step_stack` will be in randomized-depth-first order, so the build runner3488/// Each step has its dependencies traversed in random order, this accomplishes two things:
3451/// spawns initial steps in a random order3489/// - `step_stack` will be in randomized-depth-first order, so the build runner spawns initial steps in a
3452/// - each step's `dependants` list is also filled in a random order, so that
3453/// when it finishes executing in `makeStep`, it spawns next steps to run in
3454/// random order3490/// random order
3491/// - each step's `dependants` list is also filled in a random order, so that when it finishes executing in
3492/// `makeStep`, it spawns next steps to run in random order
3455fn constructGraphAndCheckForDependencyLoop(3493fn constructGraphAndCheckForDependencyLoop(
3456 maker: *Maker,3494 maker: *Maker,
3457 step_index: Configuration.Step.Index,3495 step_index: Configuration.Step.Index,
lib/compiler/Maker/Fetch.zig+4-4
...@@ -153,11 +153,11 @@ pub const JobQueue = struct {...@@ -153,11 +153,11 @@ pub const JobQueue = struct {
153 debug_hash: bool,153 debug_hash: bool,
154 mode: Mode,154 mode: Mode,
155 /// Set of hashes that will be additionally fetched even if they are marked155 /// Set of hashes that will be additionally fetched even if they are marked
156 /// as lazy.156 /// as lazy. Read-only.
157 unlazy_set: UnlazySet = .{},157 unlazy_set: UnlazySet = .empty,
158 /// Identifies paths that override all packages in the tree with matching158 /// Identifies paths that override all packages in the tree with matching
159 /// project ids.159 /// project ids.
160 fork_set: ForkSet = .{},160 fork_set: ForkSet = .empty,
161161
162 pub const Mode = enum {162 pub const Mode = enum {
163 /// Non-lazy dependencies are always fetched.163 /// Non-lazy dependencies are always fetched.
...@@ -297,8 +297,8 @@ pub const JobQueue = struct {...@@ -297,8 +297,8 @@ pub const JobQueue = struct {
297 \\ pub const build_zig = @import({qf});297 \\ pub const build_zig = @import({qf});
298 \\298 \\
299 , .{fork.path});299 , .{fork.path});
300 break :b;
300 }301 }
301 break :b;
302 }302 }
303 try w.print(303 try w.print(
304 \\ pub const build_zig = @import({q});304 \\ pub const build_zig = @import({q});
lib/compiler/Maker/ScannedConfig.zig+3-1
...@@ -55,7 +55,9 @@ pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void {...@@ -55,7 +55,9 @@ pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void {
55 {55 {
56 var tf = try s.beginTupleField("unlazy_deps", .{});56 var tf = try s.beginTupleField("unlazy_deps", .{});
57 for (c.unlazy_deps) |dep| {57 for (c.unlazy_deps) |dep| {
58 try tf.field(dep.slice(c), .{});58 var field = try tf.beginStructField(.{});
59 try printStruct(sc, &field, Configuration.UnlazyDep, dep);
60 try field.end();
59 }61 }
60 try tf.end();62 try tf.end();
61 }63 }
lib/std/Build.zig+78-17
...@@ -80,7 +80,11 @@ pub const Graph = struct {...@@ -80,7 +80,11 @@ pub const Graph = struct {
80 system_package_mode: bool = false,80 system_package_mode: bool = false,
81 zig_exe: []const u8,81 zig_exe: []const u8,
82 environ_map: process.Environ.Map,82 environ_map: process.Environ.Map,
83 needed_lazy_dependencies: std.array_hash_map.String(void) = .empty,83 /// Managed by:
84 /// * `Build.declareDependency`
85 /// * `Build.dependency`
86 /// * `Build.dependencyLazy`
87 unlazy_deps: std.array_hash_map.Auto(Configuration.String, void) = .empty,
84 /// Information about the native target. Computed before build() is invoked.88 /// Information about the native target. Computed before build() is invoked.
85 host: ResolvedTarget,89 host: ResolvedTarget,
86 dependency_cache: PackageInstanceMap = .empty,90 dependency_cache: PackageInstanceMap = .empty,
...@@ -1934,7 +1938,9 @@ inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, c...@@ -1934,7 +1938,9 @@ inline fn findImportPkgHashOrFatal(b: *Build, comptime asking_build_zig: type, c
1934}1938}
19351939
1936fn markNeededLazyDep(b: *Build, pkg_hash: []const u8) void {1940fn markNeededLazyDep(b: *Build, pkg_hash: []const u8) void {
1937 b.graph.needed_lazy_dependencies.put(b.graph.arena, pkg_hash, {}) catch @panic("OOM");1941 const graph = b.graph;
1942 const interned_hash = graph.wip_configuration.addString(pkg_hash) catch @panic("OOM");
1943 graph.unlazy_deps.put(graph.arena, interned_hash, {}) catch @panic("OOM");
1938}1944}
19391945
1940/// Deprecated in favor of `dependencyLazy`.1946/// Deprecated in favor of `dependencyLazy`.
...@@ -1944,21 +1950,76 @@ pub fn lazyDependency(b: *Build, name: []const u8, args: anytype) ?*Dependency {...@@ -1944,21 +1950,76 @@ pub fn lazyDependency(b: *Build, name: []const u8, args: anytype) ?*Dependency {
1944 };1950 };
1945}1951}
19461952
1947/// Declares that the current configuration does in fact require a potentially1953/// Indicates that the current configuration needs a potentially lazy dependency.
1948/// lazy dependency.1954///
1955/// Related:
1956/// * `declareDependencyStep`
1957/// * `dependency`
1958/// * `dependencyLazy`
1959/// * `dependencyStep`
1960pub fn declareDependency(b: *Build, name: []const u8) void {
1961 const pkg_hash = findPkgHashOrFatal(b, name);
1962 markNeededLazyDep(b, pkg_hash);
1963}
1964
1965/// Indicates that the current configuration needs a potentially lazy dependency for the provided `Step`.
1966///
1967/// Related:
1968/// * `declareDependency`
1969/// * `dependency`
1970/// * `dependencyLazy`
1971/// * `dependencyStep`
1972pub fn declareDependencyStep(b: *Build, s: *Step, name: []const u8) void {
1973 const graph = b.graph;
1974 const pkg_hash = findPkgHashOrFatal(b, name);
1975 const interned_hash = graph.wip_configuration.addString(pkg_hash) catch @panic("OOM");
1976 s.unlazy_deps.put(graph.arena, interned_hash, {}) catch @panic("OOM");
1977}
1978
1979/// Indicates that the current configuration needs a potentially lazy dependency for the provided `Step`.
1980///
1981/// If the dependency is already fetched, it is returned, otherwise `null` is returned. After configuration
1982/// completes, if the dependency is needed after all, it will be fetched and configuration will re-run.
1949///1983///
1950/// If the dependency is already fetched, it is returned. However if the1984/// For non-lazy dependencies, this never returns `null`.
1951/// dependency is not yet fetched, then when the build script is finished1985///
1952/// running, the toolchain will not proceed to the make phase. Instead, the1986/// Related:
1953/// parent process will additionally fetch all the lazy dependencies that were1987/// * `dependency`
1954/// actually required by running the build script, recompile the build script,1988/// * `dependencyLazy`
1955/// and then run it again. In other words, if this function returns1989/// * `declareDependency`
1956/// `error.LazyDependencyNeeded` it means that the only purpose of completing1990/// * `declareDependencyStep`
1957/// the configure phase is to find out all the other lazy dependencies that are1991pub fn dependencyStep(b: *Build, s: *Step, name: []const u8, args: anytype) ?*Dependency {
1958/// also required. In this case, one must propagate the error all the way up1992 const pkg_hash = findPkgHashOrFatal(b, name);
1959/// and return it from the main build function.1993 const graph = b.graph;
1994 const arena = graph.arena;
1995 const entry = package_map.get(pkg_hash).?;
1996 if (!entry.available) {
1997 const interned_hash = graph.wip_configuration.addString(pkg_hash) catch @panic("OOM");
1998 s.unlazy_deps.put(arena, interned_hash, {}) catch @panic("OOM");
1999 return null;
2000 }
2001 var map: PackageOptions.Map = .empty;
2002 PackageOptions.fromArgs(arena, &map, args);
2003 return dependencyResolved(b, name, entry, &map);
2004}
2005
2006/// Declares that the current configuration does in fact require a potentially lazy dependency.
2007///
2008/// If the dependency is already fetched, it is returned. However if the dependency is not yet fetched, then
2009/// when the build script is finished running, the toolchain will not proceed to the make phase. Instead, the
2010/// parent process will additionally fetch all the lazy dependencies that were actually required by running the
2011/// build script, recompile the build script, and then run it again. In other words, if this function returns
2012/// `error.LazyDependencyNeeded` it means that the only purpose of completing the configure phase is to find
2013/// out all the other lazy dependencies that are also required. In this case, one must propagate the error all
2014/// the way up and return it from the main build function.
1960///2015///
1961/// For non-lazy dependencies, this always succeeds.2016/// For non-lazy dependencies, this always succeeds.
2017///
2018/// Related:
2019/// * `dependency`
2020/// * `dependencyStep`
2021/// * `declareDependency`
2022/// * `declareDependencyStep`
1962pub fn dependencyLazy(b: *Build, name: []const u8, args: anytype) error{LazyDependencyNeeded}!*Dependency {2023pub fn dependencyLazy(b: *Build, name: []const u8, args: anytype) error{LazyDependencyNeeded}!*Dependency {
1963 const pkg_hash = findPkgHashOrFatal(b, name);2024 const pkg_hash = findPkgHashOrFatal(b, name);
1964 const entry = package_map.get(pkg_hash) orelse unreachable;2025 const entry = package_map.get(pkg_hash) orelse unreachable;
...@@ -2018,7 +2079,7 @@ pub const package_map: std.StaticStringMap(PackageEntry) = blk: {...@@ -2018,7 +2079,7 @@ pub const package_map: std.StaticStringMap(PackageEntry) = blk: {
2018pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {2079pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {
2019 return dependencyLazy(b, name, args) catch |err| switch (err) {2080 return dependencyLazy(b, name, args) catch |err| switch (err) {
2020 error.LazyDependencyNeeded => {2081 error.LazyDependencyNeeded => {
2021 assert(b.graph.needed_lazy_dependencies.count() != 0);2082 assert(b.graph.unlazy_deps.count() != 0);
2022 serializeConfigurationExiting(b);2083 serializeConfigurationExiting(b);
2023 },2084 },
2024 };2085 };
...@@ -2143,7 +2204,7 @@ fn dependencyResolved(...@@ -2143,7 +2204,7 @@ fn dependencyResolved(
2143pub inline fn runPackageScript(b: *Build, comptime build_zig: anytype) void {2204pub inline fn runPackageScript(b: *Build, comptime build_zig: anytype) void {
2144 const result: anyerror!void = build_zig.build(b);2205 const result: anyerror!void = build_zig.build(b);
2145 result catch |err| switch (err) {2206 result catch |err| switch (err) {
2146 error.LazyDependencyNeeded => assert(b.graph.needed_lazy_dependencies.count() != 0),2207 error.LazyDependencyNeeded => assert(b.graph.unlazy_deps.count() != 0),
2147 else => {2208 else => {
2148 if (b.dep_prefix.len == 0) {2209 if (b.dep_prefix.len == 0) {
2149 log.err("package {q} configuration failed: {t}", .{ b.dep_prefix, err });2210 log.err("package {q} configuration failed: {t}", .{ b.dep_prefix, err });
...@@ -2151,7 +2212,7 @@ pub inline fn runPackageScript(b: *Build, comptime build_zig: anytype) void {...@@ -2151,7 +2212,7 @@ pub inline fn runPackageScript(b: *Build, comptime build_zig: anytype) void {
2151 log.err("configuration failed: {t}", .{err});2212 log.err("configuration failed: {t}", .{err});
2152 }2213 }
2153 if (@errorReturnTrace()) |trace| std.debug.dumpErrorReturnTrace(trace);2214 if (@errorReturnTrace()) |trace| std.debug.dumpErrorReturnTrace(trace);
2154 const lazy_count = b.graph.needed_lazy_dependencies.count();2215 const lazy_count = b.graph.unlazy_deps.count();
2155 if (lazy_count == 0) process.exit(1);2216 if (lazy_count == 0) process.exit(1);
2156 log.info("{d} lazy dependencies detected; fetching and retrying configuration", .{lazy_count});2217 log.info("{d} lazy dependencies detected; fetching and retrying configuration", .{lazy_count});
2157 },2218 },
lib/std/Build/Configuration.zig+47-3
...@@ -11,7 +11,7 @@ const native_endian = builtin.target.cpu.arch.endian();...@@ -11,7 +11,7 @@ const native_endian = builtin.target.cpu.arch.endian();
11string_bytes: []u8,11string_bytes: []u8,
12steps: []Step,12steps: []Step,
13path_deps: []PathDep,13path_deps: []PathDep,
14unlazy_deps: []String,14unlazy_deps: []UnlazyDep,
15system_integrations: []SystemIntegration,15system_integrations: []SystemIntegration,
16available_options: []AvailableOption,16available_options: []AvailableOption,
17search_prefixes: []String,17search_prefixes: []String,
...@@ -52,7 +52,7 @@ pub const Wip = struct {...@@ -52,7 +52,7 @@ pub const Wip = struct {
52 targets_table: TargetsTable = .empty,52 targets_table: TargetsTable = .empty,
5353
54 string_bytes: std.ArrayList(u8) = .empty,54 string_bytes: std.ArrayList(u8) = .empty,
55 unlazy_deps: std.ArrayList(String) = .empty,55 unlazy_deps: std.ArrayList(UnlazyDep) = .empty,
56 system_integrations: std.ArrayList(SystemIntegration) = .empty,56 system_integrations: std.ArrayList(SystemIntegration) = .empty,
57 available_options: std.ArrayList(AvailableOption) = .empty,57 available_options: std.ArrayList(AvailableOption) = .empty,
58 steps: std.ArrayList(Step) = .empty,58 steps: std.ArrayList(Step) = .empty,
...@@ -227,6 +227,27 @@ pub const Wip = struct {...@@ -227,6 +227,27 @@ pub const Wip = struct {
227 return @fromBackingInt(@intCast(revert_index));227 return @fromBackingInt(@intCast(revert_index));
228 }228 }
229229
230 pub fn addStringListInterned(wip: *Wip, list: []const String) Allocator.Error!StringList {
231 // Increase size of extra to support the list. Add the string list
232 // there. Then check for duplicate, reverting list if already found.
233 const gpa = wip.gpa;
234 const revert_index: u32 = @intCast(wip.extra.items.len);
235 const added = try wip.extra.addManyAsSlice(gpa, list.len + 1);
236 added[0] = @intCast(list.len);
237 @memcpy(added[1..], @as([]const u32, @ptrCast(list)));
238 const gop = try wip.dedupe_table.getOrPutContext(gpa, .{
239 .index = revert_index,
240 .len = @intCast(added.len),
241 }, @as(ExtraSlice.Context, .{ .extra = wip.extra.items }));
242
243 if (gop.found_existing) {
244 wip.extra.items.len = revert_index;
245 return @fromBackingInt(@intCast(gop.key_ptr.index));
246 }
247
248 return @fromBackingInt(@intCast(revert_index));
249 }
250
230 pub fn addBytes(wip: *Wip, bytes: []const u8) Allocator.Error!Bytes {251 pub fn addBytes(wip: *Wip, bytes: []const u8) Allocator.Error!Bytes {
231 try wip.string_bytes.appendSlice(wip.gpa, bytes);252 try wip.string_bytes.appendSlice(wip.gpa, bytes);
232 return .{253 return .{
...@@ -474,6 +495,12 @@ pub const AvailableOption = extern struct {...@@ -474,6 +495,12 @@ pub const AvailableOption = extern struct {
474 };495 };
475};496};
476497
498pub const UnlazyDep = extern struct {
499 /// none indicates not associated with any particular step.
500 step: Step.OptionalIndex,
501 list: StringList,
502};
503
477pub const Step = extern struct {504pub const Step = extern struct {
478 name: String,505 name: String,
479 owner: Package.Index,506 owner: Package.Index,
...@@ -507,6 +534,23 @@ pub const Step = extern struct {...@@ -507,6 +534,23 @@ pub const Step = extern struct {
507 }534 }
508 };535 };
509536
537 pub const OptionalIndex = enum(u32) {
538 none = max_u32,
539 _,
540
541 pub fn unwrap(this: @This()) ?Index {
542 return switch (this) {
543 .none => null,
544 else => @fromBackingInt(@intCast(@backingInt(this))),
545 };
546 }
547
548 pub fn ptr(i: @This(), c: *const Configuration) ?*const Step {
549 if (i == .none) return null;
550 return &c.steps[@backingInt(i)];
551 }
552 };
553
510 /// Shared by all steps.554 /// Shared by all steps.
511 pub const Flags = packed struct(u32) {555 pub const Flags = packed struct(u32) {
512 tag: Tag,556 tag: Tag,
...@@ -3134,7 +3178,7 @@ pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration {...@@ -3134,7 +3178,7 @@ pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration {
3134 .string_bytes = try arena.alloc(u8, header.string_bytes_len),3178 .string_bytes = try arena.alloc(u8, header.string_bytes_len),
3135 .steps = try arena.alloc(Step, header.steps_len),3179 .steps = try arena.alloc(Step, header.steps_len),
3136 .path_deps = try arena.alloc(PathDep, header.path_deps_len),3180 .path_deps = try arena.alloc(PathDep, header.path_deps_len),
3137 .unlazy_deps = try arena.alloc(String, header.unlazy_deps_len),3181 .unlazy_deps = try arena.alloc(UnlazyDep, header.unlazy_deps_len),
3138 .system_integrations = try arena.alloc(SystemIntegration, header.system_integrations_len),3182 .system_integrations = try arena.alloc(SystemIntegration, header.system_integrations_len),
3139 .available_options = try arena.alloc(AvailableOption, header.available_options_len),3183 .available_options = try arena.alloc(AvailableOption, header.available_options_len),
3140 .search_prefixes = try arena.alloc(String, header.search_prefixes_len),3184 .search_prefixes = try arena.alloc(String, header.search_prefixes_len),
lib/std/Build/Serialize.zig+12-3
...@@ -683,12 +683,21 @@ pub fn write(b: *std.Build, wc: *Configuration.Wip, writer: *std.Io.Writer) !voi...@@ -683,12 +683,21 @@ pub fn write(b: *std.Build, wc: *Configuration.Wip, writer: *std.Io.Writer) !voi
683 .max_rss = .fromBytes(step.max_rss),683 .max_rss = .fromBytes(step.max_rss),
684 .extended = @fromBackingInt(extended),684 .extended = @fromBackingInt(extended),
685 });685 });
686
687 if (step.unlazy_deps.count() != 0) {
688 try wc.unlazy_deps.append(gpa, .{
689 .step = @fromBackingInt(@intCast(wc.steps.items.len - 1)),
690 .list = try wc.addStringListInterned(step.unlazy_deps.keys()),
691 });
692 }
686 }693 }
687 }694 }
688695
689 try wc.unlazy_deps.ensureUnusedCapacity(gpa, graph.needed_lazy_dependencies.keys().len);696 if (graph.unlazy_deps.count() != 0) {
690 for (graph.needed_lazy_dependencies.keys()) |k| {697 try wc.unlazy_deps.append(gpa, .{
691 wc.unlazy_deps.appendAssumeCapacity(try wc.addString(k));698 .step = .none,
699 .list = try wc.addStringListInterned(graph.unlazy_deps.keys()),
700 });
692 }701 }
693702
694 try wc.write(writer, .{703 try wc.write(writer, .{
lib/std/Build/Step.zig+6
...@@ -36,6 +36,11 @@ max_rss: u64,...@@ -36,6 +36,11 @@ max_rss: u64,
36/// to print along with debugging messages.36/// to print along with debugging messages.
37debug_stack_trace: std.debug.StackTrace,37debug_stack_trace: std.debug.StackTrace,
3838
39/// Managed by:
40/// * `Build.declareDependencyStep`
41/// * `Build.dependencyStep`
42unlazy_deps: std.array_hash_map.Auto(Configuration.String, void),
43
39pub const Tag = Configuration.Step.Tag;44pub const Tag = Configuration.Step.Tag;
4045
41pub fn Type(comptime tag: Tag) type {46pub fn Type(comptime tag: Tag) type {
...@@ -104,6 +109,7 @@ pub fn init(options: StepOptions) Step {...@@ -104,6 +109,7 @@ pub fn init(options: StepOptions) Step {
104 const first_ret_addr = options.first_ret_addr orelse @returnAddress();109 const first_ret_addr = options.first_ret_addr orelse @returnAddress();
105 break :blk std.debug.captureCurrentStackTrace(.{ .first_address = first_ret_addr }, addr_buf);110 break :blk std.debug.captureCurrentStackTrace(.{ .first_address = first_ret_addr }, addr_buf);
106 },111 },
112 .unlazy_deps = .empty,
107 };113 };
108}114}
109115