| ... | @@ -52,7 +52,7 @@ max_rss_mutex: Io.Mutex, | ... | @@ -52,7 +52,7 @@ max_rss_mutex: Io.Mutex, |
| 52 | skip_oom_steps: bool, | 52 | skip_oom_steps: bool, |
| 53 | unit_test_timeout_ns: ?u64, | 53 | unit_test_timeout_ns: ?u64, |
| 54 | watch: bool, | 54 | watch: bool, |
| 55 | web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn, | 55 | web_server: ?*AvoidableWebServer, |
| 56 | /// Allocated into `gpa`. | 56 | /// Allocated into `gpa`. |
| 57 | memory_blocked_steps: std.ArrayList(Configuration.Step.Index), | 57 | memory_blocked_steps: std.ArrayList(Configuration.Step.Index), |
| 58 | /// Allocated into `gpa`. | 58 | /// Allocated into `gpa`. |
| ... | @@ -68,6 +68,8 @@ var stdio_buffer_allocation: [256]u8 = undefined; | ... | @@ -68,6 +68,8 @@ var stdio_buffer_allocation: [256]u8 = undefined; |
| 68 | var stdout_writer_allocation: Io.File.Writer = undefined; | 68 | var stdout_writer_allocation: Io.File.Writer = undefined; |
| 69 | var debug_maker_leaks: bool = false; | 69 | var debug_maker_leaks: bool = false; |
| 70 | | 70 | |
| | 71 | const AvoidableWebServer = if (builtin.single_threaded) void else WebServer; |
| | 72 | |
| 71 | const is_debug_mode = builtin.mode == .Debug; | 73 | const is_debug_mode = builtin.mode == .Debug; |
| 72 | const use_safe_allocator = switch (builtin.mode) { | 74 | const use_safe_allocator = switch (builtin.mode) { |
| 73 | .Debug, .ReleaseSafe => true, | 75 | .Debug, .ReleaseSafe => true, |
| ... | @@ -106,6 +108,7 @@ const ErrorStyle = enum { | ... | @@ -106,6 +108,7 @@ const ErrorStyle = enum { |
| 106 | }; | 108 | }; |
| 107 | const MultilineErrors = enum { indent, newline, none }; | 109 | const MultilineErrors = enum { indent, newline, none }; |
| 108 | const Summary = enum { all, new, failures, line, none }; | 110 | const Summary = enum { all, new, failures, line, none }; |
| | 111 | const PrintConfiguration = enum { none, zon, path }; |
| 109 | | 112 | |
| 110 | /// Used to build the -M flags to pass to build-exe. | 113 | /// Used to build the -M flags to pass to build-exe. |
| 111 | pub const CliModule = struct { | 114 | pub const CliModule = struct { |
| ... | @@ -195,7 +198,7 @@ pub fn main(init: process.Init.Minimal) !void { | ... | @@ -195,7 +198,7 @@ pub fn main(init: process.Init.Minimal) !void { |
| 195 | var step_names: std.ArrayList([]const u8) = .empty; | 198 | var step_names: std.ArrayList([]const u8) = .empty; |
| 196 | var help_menu = false; | 199 | var help_menu = false; |
| 197 | var steps_menu = false; | 200 | var steps_menu = false; |
| 198 | var print_configuration: enum { none, zon, path } = .none; | 201 | var print_configuration: PrintConfiguration = .none; |
| 199 | var override_install_prefix: ?[]const u8 = null; | 202 | var override_install_prefix: ?[]const u8 = null; |
| 200 | var override_lib_dir: ?[]const u8 = null; | 203 | var override_lib_dir: ?[]const u8 = null; |
| 201 | var override_bin_dir: ?[]const u8 = null; | 204 | var override_bin_dir: ?[]const u8 = null; |
| ... | @@ -546,6 +549,9 @@ pub fn main(init: process.Init.Minimal) !void { | ... | @@ -546,6 +549,9 @@ pub fn main(init: process.Init.Minimal) !void { |
| 546 | } | 549 | } |
| 547 | } | 550 | } |
| 548 | | 551 | |
| | 552 | const early_exit_mode = fetch_only or help_menu or steps_menu or print_configuration != .none; |
| | 553 | const server_mode = !early_exit_mode and (watch or webui_listen != null or fuzz != null); |
| | 554 | |
| 549 | const cwd_path = std.zig.getResolvedCwd(io, arena) catch |err| | 555 | const cwd_path = std.zig.getResolvedCwd(io, arena) catch |err| |
| 550 | fatal("resolving current directory path failed: {t}", .{err}); | 556 | fatal("resolving current directory path failed: {t}", .{err}); |
| 551 | | 557 | |
| ... | @@ -593,752 +599,809 @@ pub fn main(init: process.Init.Minimal) !void { | ... | @@ -593,752 +599,809 @@ pub fn main(init: process.Init.Minimal) !void { |
| 593 | .off => .no_color, | 599 | .off => .no_color, |
| 594 | }; | 600 | }; |
| 595 | | 601 | |
| | 602 | const pkg_root: Path = if (override_pkg_dir) |p| |
| | 603 | .initCwd(p) |
| | 604 | else if (system_pkg_dir_path) |p| |
| | 605 | .initCwd(p) |
| | 606 | else |
| | 607 | .{ |
| | 608 | .root_dir = build_root.directory, |
| | 609 | .sub_path = "zig-pkg", |
| | 610 | }; |
| | 611 | |
| 596 | const main_progress_node = std.Progress.start(io, .{ | 612 | const main_progress_node = std.Progress.start(io, .{ |
| 597 | .disable_printing = (graph.stderr_mode.? == .no_color), | 613 | .disable_printing = (graph.stderr_mode.? == .no_color), |
| 598 | }); | 614 | }); |
| 599 | defer main_progress_node.end(); | 615 | defer main_progress_node.end(); |
| 600 | | 616 | |
| 601 | const scanned_config: ScannedConfig = sc: { | 617 | const install_prefix_path: Path = if (graph.environ_map.get("DESTDIR")) |dest_dir| .{ |
| 602 | // Cache lookup for configure options. If we get a match, we can skip | 618 | .root_dir = .cwd(), |
| 603 | // execution of the configure script. If not, we get the file path to pass | 619 | .sub_path = try Dir.path.join(arena, &.{ dest_dir, override_install_prefix orelse "/usr" }), |
| 604 | // to the configure process. | 620 | } else if (override_install_prefix) |cwd_relative| .{ |
| 605 | // | 621 | .root_dir = .cwd(), |
| 606 | // In the hot path, we only check this cache, which means that also | 622 | .sub_path = cwd_relative, |
| 607 | // configure source files need to go in here. | 623 | } else .{ |
| 608 | var config_man = graph.cache.obtain(); | 624 | .root_dir = graph.build_root_directory, |
| 609 | defer config_man.deinit(); | 625 | .sub_path = "zig-out", |
| 610 | | 626 | }; |
| 611 | for (cached_passthru_configure.items) |i| | 627 | |
| 612 | config_man.hash.addBytes(configure_argv.items[i]); | 628 | const install_lib_path: Path = if (override_lib_dir) |cwd_relative| .{ |
| 613 | | 629 | .root_dir = .cwd(), |
| 614 | // Prevents a `zig build` from getting a false positive cache hit following | 630 | .sub_path = cwd_relative, |
| 615 | // a `zig build --cache-poison=ignored`. | 631 | } else try install_prefix_path.join(arena, "lib"); |
| 616 | config_man.hash.add(cache_poison == .ignored); | 632 | |
| 617 | | 633 | const install_bin_path: Path = if (override_bin_dir) |cwd_relative| .{ |
| 618 | const pkg_root: Path = if (override_pkg_dir) |p| | 634 | .root_dir = .cwd(), |
| 619 | .initCwd(p) | 635 | .sub_path = cwd_relative, |
| 620 | else if (system_pkg_dir_path) |p| | 636 | } else try install_prefix_path.join(arena, "bin"); |
| 621 | .initCwd(p) | 637 | |
| 622 | else | 638 | const install_include_path: Path = if (override_include_dir) |cwd_relative| .{ |
| 623 | .{ | 639 | .root_dir = .cwd(), |
| 624 | .root_dir = build_root.directory, | 640 | .sub_path = cwd_relative, |
| 625 | .sub_path = "zig-pkg", | 641 | } else try install_prefix_path.join(arena, "include"); |
| | 642 | |
| | 643 | const now = Io.Clock.Timestamp.now(io, .awake); |
| | 644 | |
| | 645 | var web_server_allocation: AvoidableWebServer = undefined; |
| | 646 | const web_server: ?*AvoidableWebServer = if (webui_listen) |listen_address| ws: { |
| | 647 | if (builtin.single_threaded) fatal("--webui is not yet supported on single-threaded hosts", .{}); |
| | 648 | web_server_allocation = .init(.{ |
| | 649 | .graph = &graph, |
| | 650 | .root_prog_node = main_progress_node, |
| | 651 | .listen_address = listen_address, |
| | 652 | .base_timestamp = now, |
| | 653 | }); |
| | 654 | web_server_allocation.start() catch |err| fatal("failed to start web server: {t}", .{err}); |
| | 655 | break :ws &web_server_allocation; |
| | 656 | } else null; |
| | 657 | |
| | 658 | while (true) { |
| | 659 | // If this fails, we can still start the server and wait for user |
| | 660 | // to request a rebuild. If it returns error.FailedButCacheIntact |
| | 661 | // we can even still do file system watching and automatically |
| | 662 | // rebuild on source changes. |
| | 663 | if (configure(&graph, .{ |
| | 664 | .configure_argv = configure_argv.items, |
| | 665 | .conf_argv_index_build_root = conf_argv_index_build_root, |
| | 666 | .cached_passthru_configure = cached_passthru_configure.items, |
| | 667 | |
| | 668 | .cache_poison = cache_poison, |
| | 669 | .pkg_root = pkg_root, |
| | 670 | .build_root = build_root, |
| | 671 | .cwd_path = cwd_path, |
| | 672 | .color = color, |
| | 673 | .debug_target = debug_target, |
| | 674 | .parent_progress_node = main_progress_node, |
| | 675 | .fetch_mode = fetch_mode, |
| | 676 | .system_pkg_dir_path = system_pkg_dir_path, |
| | 677 | .fetch_only = fetch_only, |
| | 678 | .print_configuration = print_configuration, |
| | 679 | .forks = forks.items, |
| | 680 | })) |scanned_config| { |
| | 681 | if (help_menu) { |
| | 682 | scanned_config.printUsage(&graph, initStdoutWriter(io)) catch |err| switch (err) { |
| | 683 | error.WriteFailed => return stdout_writer_allocation.err.?, |
| | 684 | else => |e| return e, |
| | 685 | }; |
| | 686 | try stdout_writer_allocation.flush(); |
| | 687 | return cleanExit(io, &scanned_config); |
| | 688 | } else if (steps_menu) { |
| | 689 | scanned_config.printSteps(&graph, initStdoutWriter(io)) catch |err| switch (err) { |
| | 690 | error.WriteFailed => return stdout_writer_allocation.err.?, |
| | 691 | else => |e| return e, |
| | 692 | }; |
| | 693 | try stdout_writer_allocation.flush(); |
| | 694 | return cleanExit(io, &scanned_config); |
| | 695 | } else switch (print_configuration) { |
| | 696 | .none => {}, |
| | 697 | .zon => { |
| | 698 | scanned_config.print(initStdoutWriter(io)) catch return stdout_writer_allocation.err.?; |
| | 699 | try stdout_writer_allocation.flush(); |
| | 700 | return cleanExit(io, &scanned_config); |
| | 701 | }, |
| | 702 | .path => unreachable, |
| | 703 | } |
| | 704 | |
| | 705 | var maker: Maker = .{ |
| | 706 | .gpa = gpa, |
| | 707 | .graph = &graph, |
| | 708 | .scanned_config = &scanned_config, |
| | 709 | .install_paths = .{ |
| | 710 | .prefix = install_prefix_path, |
| | 711 | .lib = install_lib_path, |
| | 712 | .bin = install_bin_path, |
| | 713 | .include = install_include_path, |
| | 714 | }, |
| | 715 | |
| | 716 | .steps = try arena.alloc(Step, scanned_config.configuration.steps.len), |
| | 717 | .generated_files = try arena.alloc(Path, scanned_config.configuration.generated_files_len), |
| | 718 | .run_args = run_args, |
| | 719 | |
| | 720 | .available_rss = max_rss, |
| | 721 | .max_rss_is_default = false, |
| | 722 | .max_rss_mutex = .init, |
| | 723 | .skip_oom_steps = skip_oom_steps, |
| | 724 | .unit_test_timeout_ns = test_timeout_ns, |
| | 725 | |
| | 726 | .watch = watch, |
| | 727 | .web_server = undefined, // set after `prepare` |
| | 728 | .memory_blocked_steps = .empty, |
| | 729 | .step_stack = .empty, |
| | 730 | .pkg_config = .{ .debug = debug_pkg_config }, |
| | 731 | |
| | 732 | .error_style = error_style, |
| | 733 | .multiline_errors = multiline_errors, |
| | 734 | .summary = summary orelse if (watch or webui_listen != null) .new else .failures, |
| | 735 | }; |
| | 736 | defer { |
| | 737 | maker.memory_blocked_steps.deinit(gpa); |
| | 738 | maker.step_stack.deinit(gpa); |
| | 739 | } |
| | 740 | |
| | 741 | if (maker.available_rss == 0) { |
| | 742 | maker.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64); |
| | 743 | maker.max_rss_is_default = true; |
| | 744 | } |
| | 745 | |
| | 746 | maker.prepare(step_names.items) catch |err| switch (err) { |
| | 747 | error.DependencyLoopDetected, error.InsufficientMemory => { |
| | 748 | _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; |
| | 749 | process.exit(1); |
| | 750 | }, |
| | 751 | else => |e| return e, |
| | 752 | }; |
| | 753 | |
| | 754 | var w: Watch = w: { |
| | 755 | if (!watch) break :w undefined; |
| | 756 | if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{native_os}); |
| | 757 | break :w try .init(&maker); |
| 626 | }; | 758 | }; |
| 627 | | 759 | |
| 628 | configure_argv.items[conf_argv_index_build_root] = build_root.directory.path orelse cwd_path; | 760 | if (web_server) |ws| try ws.updateConfiguration(&maker); |
| 629 | | 761 | |
| 630 | var http_client: std.http.Client = .{ .allocator = gpa, .io = io }; | 762 | rebuild: while (true) : (if (maker.error_style.clearOnUpdate()) { |
| 631 | defer http_client.deinit(); | 763 | const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); |
| | 764 | defer io.unlockStderr(); |
| | 765 | stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H") catch |err| switch (err) { |
| | 766 | error.WriteFailed => return stderr.file_writer.err.?, |
| | 767 | }; |
| | 768 | }) { |
| | 769 | if (web_server) |ws| ws.startBuild(); |
| 632 | | 770 | |
| 633 | var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; | 771 | try maker.makeStepNames(step_names.items, main_progress_node, fuzz); |
| 634 | var fork_set: Package.Fetch.JobQueue.ForkSet = .{}; | | |
| 635 | | 772 | |
| 636 | { | 773 | if (web_server) |ws| { |
| 637 | // Populate fork_set. | 774 | if (fuzz) |mode| if (mode != .forever) fatal( |
| 638 | var group: Io.Group = .init; | 775 | "error: limited fuzzing is not implemented yet for --webui", |
| 639 | defer group.cancel(io); | 776 | .{}, |
| 640 | | 777 | ); |
| 641 | for (forks.items) |*fork| | 778 | |
| 642 | group.async(io, Fork.load, .{ io, gpa, fork, color }); | 779 | ws.finishBuild(.{ .fuzz = fuzz != null }); |
| 643 | | 780 | } |
| 644 | try group.await(io); | 781 | |
| 645 | | 782 | if (web_server) |ws| { |
| 646 | for (forks.items) |*fork| { | 783 | const c = &scanned_config.configuration; |
| 647 | if (fork.failed) process.exit(1); | 784 | assert(!watch); // fatal error after CLI parsing |
| 648 | try fork_set.put(arena, .{ | 785 | while (true) switch (try ws.wait()) { |
| 649 | .path = fork.path, | 786 | .rebuild => { |
| 650 | .manifest_ast = fork.manifest_ast, | 787 | for (maker.step_stack.keys()) |step_index| { |
| 651 | .manifest = fork.manifest, | 788 | const step = maker.stepByIndex(step_index); |
| 652 | .uses = 0, | 789 | step.state = .precheck_done; |
| 653 | }, {}); | 790 | const deps = step_index.ptr(c).deps.slice(c); |
| | 791 | step.pending_deps = @intCast(deps.len); |
| | 792 | step.reset(&maker); |
| | 793 | } |
| | 794 | continue :rebuild; |
| | 795 | }, |
| | 796 | }; |
| | 797 | } |
| | 798 | |
| | 799 | if (!maker.watch) return; |
| | 800 | |
| | 801 | // Comptime-known guard to prevent including the logic below when `!Watch.have_impl`. |
| | 802 | if (!Watch.have_impl) unreachable; |
| | 803 | |
| | 804 | try w.update(maker.step_stack.keys()); |
| | 805 | |
| | 806 | // Wait until a file system notification arrives. Read all such events |
| | 807 | // until the buffer is empty. Then wait for a debounce interval, resetting |
| | 808 | // if any more events come in. After the debounce interval has passed, |
| | 809 | // trigger a rebuild on all steps with modified inputs, as well as their |
| | 810 | // recursive dependants. |
| | 811 | var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; |
| | 812 | const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ |
| | 813 | w.dir_count, countSubProcesses(&maker), |
| | 814 | }) catch &caption_buf; |
| | 815 | var debouncing_node = main_progress_node.start(caption, 0); |
| | 816 | var in_debounce = false; |
| | 817 | while (true) switch (try w.wait(if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) { |
| | 818 | .timeout => { |
| | 819 | assert(in_debounce); |
| | 820 | debouncing_node.end(); |
| | 821 | markFailedStepsDirty(&maker); |
| | 822 | continue :rebuild; |
| | 823 | }, |
| | 824 | .dirty => if (!in_debounce) { |
| | 825 | in_debounce = true; |
| | 826 | debouncing_node.end(); |
| | 827 | debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); |
| | 828 | }, |
| | 829 | .clean => {}, |
| | 830 | }; |
| | 831 | } |
| | 832 | } else |err| { |
| | 833 | const can_fs_watch = switch (err) { |
| | 834 | error.AlreadyReported => false, |
| | 835 | error.FailedButCacheIntact => true, |
| | 836 | else => |e| w: { |
| | 837 | log.err("configuration failed: {t}", .{e}); |
| | 838 | break :w false; |
| | 839 | }, |
| | 840 | }; |
| | 841 | if (!server_mode) process.exit(1); |
| | 842 | if (can_fs_watch) { |
| | 843 | @panic("TODO set up fs watching"); |
| | 844 | } else { |
| | 845 | @panic("TODO wait for user to request rebuild"); |
| 654 | } | 846 | } |
| 655 | } | 847 | } |
| 656 | defer Fork.deinitList(forks.items); | 848 | } |
| | 849 | } |
| 657 | | 850 | |
| 658 | var build_configurer_argv: std.ArrayList([]const u8) = .empty; | 851 | const ConfigureOptions = struct { |
| 659 | defer build_configurer_argv.deinit(gpa); | 852 | configure_argv: [][]const u8, |
| | 853 | conf_argv_index_build_root: usize, |
| | 854 | cached_passthru_configure: []const u32, |
| 660 | | 855 | |
| 661 | var dependencies_source: std.ArrayList(u8) = .empty; | 856 | cache_poison: std.Build.Graph.CachePoison, |
| 662 | defer dependencies_source.deinit(gpa); | 857 | pkg_root: Path, |
| | 858 | build_root: BuildRoot, |
| | 859 | cwd_path: []const u8, |
| | 860 | color: Color, |
| | 861 | debug_target: ?[]const u8, |
| | 862 | parent_progress_node: std.Progress.Node, |
| | 863 | fetch_mode: Fetch.JobQueue.Mode, |
| | 864 | system_pkg_dir_path: ?[]const u8, |
| | 865 | fetch_only: bool, |
| | 866 | print_configuration: PrintConfiguration, |
| | 867 | forks: []Fork, |
| | 868 | }; |
| 663 | | 869 | |
| 664 | const configurer_root_src_path: Cache.Path = .{ | 870 | fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { |
| 665 | .root_dir = graph.zig_lib_directory, | 871 | const configure_argv = options.configure_argv; |
| 666 | .sub_path = "compiler/configurer.zig", | 872 | const gpa = graph.cache.gpa; |
| 667 | }; | 873 | const io = graph.io; |
| | 874 | const arena = graph.arena; |
| 668 | | 875 | |
| 669 | const root_build_src_path: Cache.Path = .{ | 876 | // Cache lookup for configure options. If we get a match, we can skip |
| 670 | .root_dir = build_root.directory, | 877 | // execution of the configure script. If not, we get the file path to pass |
| 671 | .sub_path = build_root.build_zig_basename, | 878 | // to the configure process. |
| 672 | }; | 879 | // |
| | 880 | // In the hot path, we only check this cache, which means that also |
| | 881 | // configure source files need to go in here. |
| | 882 | var config_man = graph.cache.obtain(); |
| | 883 | defer config_man.deinit(); |
| 673 | | 884 | |
| 674 | const configurer_exe_name = "configurer"; | 885 | for (options.cached_passthru_configure) |i| |
| | 886 | config_man.hash.addBytes(configure_argv[i]); |
| 675 | | 887 | |
| 676 | try build_configurer_argv.appendSlice(gpa, &.{ | 888 | // Prevents a `zig build` from getting a false positive cache hit following |
| 677 | graph.zig_exe, "build-exe", // | 889 | // a `zig build --cache-poison=ignored`. |
| 678 | "--cache-dir", graph.local_cache_root.path orelse ".", // | 890 | config_man.hash.add(options.cache_poison == .ignored); |
| 679 | "--global-cache-dir", graph.global_cache_root.path orelse ".", // | | |
| 680 | "--zig-lib-dir", graph.zig_lib_directory.path orelse ".", // | | |
| 681 | "--name", configurer_exe_name, // | | |
| 682 | "-fsingle-threaded", // | | |
| 683 | }); | | |
| 684 | | 891 | |
| 685 | // Normally the build runner is compiled for the host target but here is | 892 | configure_argv[options.conf_argv_index_build_root] = options.build_root.directory.path orelse options.cwd_path; |
| 686 | // some code to help when debugging edits to the build runner so that you | 893 | |
| 687 | // can make sure it compiles successfully on other targets. | 894 | var http_client: std.http.Client = .{ .allocator = gpa, .io = io }; |
| 688 | const target_arch_os_abi: ?[]const u8 = if (debug_target) |triple| t: { | 895 | defer http_client.deinit(); |
| 689 | config_man.hash.addBytes(triple); | 896 | |
| 690 | try build_configurer_argv.appendSlice(gpa, &.{ "-target", triple }); | 897 | var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; |
| 691 | break :t triple; | 898 | var fork_set: Package.Fetch.JobQueue.ForkSet = .{}; |
| 692 | } else null; | 899 | |
| 693 | | 900 | { |
| 694 | if (graph.libc_file) |libc_file| { | 901 | // Populate fork_set. |
| 695 | try build_configurer_argv.appendSlice(gpa, &.{ "--libc", libc_file }); | 902 | var group: Io.Group = .init; |
| 696 | } | 903 | defer group.cancel(io); |
| 697 | if (graph.reference_trace) |n| { | 904 | |
| 698 | try build_configurer_argv.append(gpa, try allocPrint(arena, "-freference-trace={d}", .{n})); | 905 | for (options.forks) |*fork| |
| 699 | } | 906 | group.async(io, Fork.load, .{ io, gpa, fork, options.color }); |
| 700 | if (graph.debug_compile_errors) { | 907 | |
| 701 | try build_configurer_argv.append(gpa, "--debug-compile-errors"); | 908 | try group.await(io); |
| | 909 | |
| | 910 | for (options.forks) |*fork| { |
| | 911 | if (fork.failed) process.exit(1); |
| | 912 | try fork_set.put(arena, .{ |
| | 913 | .path = fork.path, |
| | 914 | .manifest_ast = fork.manifest_ast, |
| | 915 | .manifest = fork.manifest, |
| | 916 | .uses = 0, |
| | 917 | }, {}); |
| 702 | } | 918 | } |
| 703 | try build_configurer_argv.appendSlice(gpa, &.{ | 919 | } |
| 704 | "--dep", "@build", // | 920 | defer Fork.deinitList(options.forks); |
| 705 | "--dep", "@dependencies", // | | |
| 706 | try allocPrint(arena, "-Mroot={f}", .{configurer_root_src_path}), // | | |
| 707 | }); | | |
| 708 | | 921 | |
| 709 | // In the loop below, after doing the fetch operation, the argv will be | 922 | var build_configurer_argv: std.ArrayList([]const u8) = .empty; |
| 710 | // truncated at this point, dependencies added, and then the | 923 | defer build_configurer_argv.deinit(gpa); |
| 711 | // "--listen=-" arg appended at the end. | | |
| 712 | const argv_deps_index = build_configurer_argv.items.len; | | |
| 713 | | 924 | |
| 714 | const build_mod = try arena.create(CliModule); | 925 | var dependencies_source: std.ArrayList(u8) = .empty; |
| 715 | build_mod.* = .{ | 926 | defer dependencies_source.deinit(gpa); |
| 716 | .name = "@build", | | |
| 717 | .root_path = try root_build_src_path.toString(arena), | | |
| 718 | }; | | |
| 719 | | 927 | |
| 720 | const deps_mod = try arena.create(CliModule); | 928 | const configurer_root_src_path: Cache.Path = .{ |
| 721 | deps_mod.* = .{ | 929 | .root_dir = graph.zig_lib_directory, |
| 722 | .name = "@dependencies", | 930 | .sub_path = "compiler/configurer.zig", |
| 723 | .root_path = undefined, | 931 | }; |
| 724 | }; | | |
| 725 | | 932 | |
| 726 | // This loop is re-evaluated when the build script exits with an indication that it | 933 | const root_build_src_path: Cache.Path = .{ |
| 727 | // could not continue due to missing lazy dependencies. | 934 | .root_dir = options.build_root.directory, |
| 728 | const configuration_path: Path, const poisoned: bool = cp: while (true) { | 935 | .sub_path = options.build_root.build_zig_basename, |
| 729 | build_mod.deps.clearRetainingCapacity(); | 936 | }; |
| 730 | deps_mod.deps.clearRetainingCapacity(); | | |
| 731 | | 937 | |
| 732 | // We want to release all the locks before executing the child process, so we make a nice | 938 | const configurer_exe_name = "configurer"; |
| 733 | // big block here to ensure the cleanup gets run when we extract out our argv. | | |
| 734 | { | | |
| 735 | { | | |
| 736 | const fetch_prog_node = main_progress_node.start("Fetch Packages", 0); | | |
| 737 | defer fetch_prog_node.end(); | | |
| 738 | | | |
| 739 | // Reset fork match counts. | | |
| 740 | for (fork_set.keys()) |*fork| fork.uses = 0; | | |
| 741 | | | |
| 742 | var job_queue: Package.Fetch.JobQueue = .{ | | |
| 743 | .io = io, | | |
| 744 | .http_client = &http_client, | | |
| 745 | .global_cache = graph.global_cache_root, | | |
| 746 | .local_storage = &.{ | | |
| 747 | .cache_root = .{ .root_dir = graph.local_cache_root }, | | |
| 748 | .pkg_root = pkg_root, | | |
| 749 | }, | | |
| 750 | .recursive = true, | | |
| 751 | .debug_hash = false, | | |
| 752 | .unlazy_set = unlazy_set, | | |
| 753 | .fork_set = fork_set, | | |
| 754 | .mode = fetch_mode, | | |
| 755 | .prog_node = fetch_prog_node, | | |
| 756 | .read_only = system_pkg_dir_path != null, | | |
| 757 | }; | | |
| 758 | defer job_queue.deinit(); | | |
| 759 | | 939 | |
| 760 | if (system_pkg_dir_path == null) { | 940 | try build_configurer_argv.appendSlice(gpa, &.{ |
| 761 | try http_client.initDefaultProxies(arena, &graph.environ_map); | 941 | graph.zig_exe, "build-exe", // |
| 762 | } | 942 | "--cache-dir", graph.local_cache_root.path orelse ".", // |
| | 943 | "--global-cache-dir", graph.global_cache_root.path orelse ".", // |
| | 944 | "--zig-lib-dir", graph.zig_lib_directory.path orelse ".", // |
| | 945 | "--name", configurer_exe_name, // |
| | 946 | "-fsingle-threaded", // |
| | 947 | }); |
| 763 | | 948 | |
| 764 | try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); | 949 | // Normally the build runner is compiled for the host target but here is |
| 765 | try job_queue.table.ensureUnusedCapacity(gpa, 1); | 950 | // some code to help when debugging edits to the build runner so that you |
| 766 | | 951 | // can make sure it compiles successfully on other targets. |
| 767 | const phantom_package_root: Cache.Path = .{ .root_dir = build_root.directory }; | 952 | const target_arch_os_abi: ?[]const u8 = if (options.debug_target) |triple| t: { |
| 768 | | 953 | config_man.hash.addBytes(triple); |
| 769 | var fetch: Package.Fetch = .{ | 954 | try build_configurer_argv.appendSlice(gpa, &.{ "-target", triple }); |
| 770 | .arena = std.heap.ArenaAllocator.init(gpa), | 955 | break :t triple; |
| 771 | .location = .{ .relative_path = phantom_package_root }, | 956 | } else null; |
| 772 | .location_tok = 0, | | |
| 773 | .hash_tok = .none, | | |
| 774 | .name_tok = 0, | | |
| 775 | .lazy_status = .eager, | | |
| 776 | .remote_package_root = phantom_package_root, | | |
| 777 | .parent_package_root = phantom_package_root, | | |
| 778 | .parent_manifest_ast = null, | | |
| 779 | .prog_node = fetch_prog_node, | | |
| 780 | .job_queue = &job_queue, | | |
| 781 | .omit_missing_hash_error = true, | | |
| 782 | .allow_missing_paths_field = false, | | |
| 783 | .use_latest_commit = false, | | |
| 784 | | | |
| 785 | .package_root = undefined, | | |
| 786 | .error_bundle = undefined, | | |
| 787 | .manifest = undefined, | | |
| 788 | .manifest_ast = undefined, | | |
| 789 | .have_manifest = false, | | |
| 790 | .computed_hash = undefined, | | |
| 791 | .has_build_zig = true, | | |
| 792 | .oom_flag = false, | | |
| 793 | .latest_commit = null, | | |
| 794 | | | |
| 795 | .cli_module = build_mod, | | |
| 796 | }; | | |
| 797 | | 957 | |
| 798 | job_queue.all_fetches.appendAssumeCapacity(&fetch); | 958 | if (graph.libc_file) |libc_file| { |
| | 959 | try build_configurer_argv.appendSlice(gpa, &.{ "--libc", libc_file }); |
| | 960 | } |
| | 961 | if (graph.reference_trace) |n| { |
| | 962 | try build_configurer_argv.append(gpa, try allocPrint(arena, "-freference-trace={d}", .{n})); |
| | 963 | } |
| | 964 | if (graph.debug_compile_errors) { |
| | 965 | try build_configurer_argv.append(gpa, "--debug-compile-errors"); |
| | 966 | } |
| | 967 | try build_configurer_argv.appendSlice(gpa, &.{ |
| | 968 | "--dep", "@build", // |
| | 969 | "--dep", "@dependencies", // |
| | 970 | try allocPrint(arena, "-Mroot={f}", .{configurer_root_src_path}), // |
| | 971 | }); |
| 799 | | 972 | |
| 800 | job_queue.table.putAssumeCapacityNoClobber( | 973 | // In the loop below, after doing the fetch operation, the argv will be |
| 801 | Package.Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root), | 974 | // truncated at this point, dependencies added, and then the |
| 802 | &fetch, | 975 | // "--listen=-" arg appended at the end. |
| 803 | ); | 976 | const argv_deps_index = build_configurer_argv.items.len; |
| 804 | | 977 | |
| 805 | job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" }); | 978 | const build_mod = try arena.create(CliModule); |
| 806 | try job_queue.group.await(io); | 979 | build_mod.* = .{ |
| | 980 | .name = "@build", |
| | 981 | .root_path = try root_build_src_path.toString(arena), |
| | 982 | }; |
| 807 | | 983 | |
| 808 | { | 984 | const deps_mod = try arena.create(CliModule); |
| 809 | // Ensure that forks were actually used. This is done | 985 | deps_mod.* = .{ |
| 810 | // before printing manifest errors because using a fork can | 986 | .name = "@dependencies", |
| 811 | // prevent them. | 987 | .root_path = undefined, |
| 812 | var any_unused = false; | 988 | }; |
| 813 | for (fork_set.keys()) |*fork| { | | |
| 814 | if (fork.uses == 0) { | | |
| 815 | log.err("fork {f} matched no {s} packages", .{ | | |
| 816 | fork.path, fork.manifest.name, | | |
| 817 | }); | | |
| 818 | any_unused = true; | | |
| 819 | } else { | | |
| 820 | log.info("fork {f} matched {d} {s} packages", .{ | | |
| 821 | fork.path, fork.uses, fork.manifest.name, | | |
| 822 | }); | | |
| 823 | } | | |
| 824 | } | | |
| 825 | if (any_unused) process.exit(1); | | |
| 826 | } | | |
| 827 | | 989 | |
| 828 | try job_queue.consolidateErrors(); | 990 | // This loop is re-evaluated when the build script exits with an indication that it |
| | 991 | // could not continue due to missing lazy dependencies. |
| | 992 | const configuration_path: Path, const poisoned: bool = cp: while (true) { |
| | 993 | build_mod.deps.clearRetainingCapacity(); |
| | 994 | deps_mod.deps.clearRetainingCapacity(); |
| 829 | | 995 | |
| 830 | if (fetch.error_bundle.root_list.items.len > 0) { | 996 | // We want to release all the locks before executing the child process, so we make a nice |
| 831 | var errors = try fetch.error_bundle.toOwnedBundle(""); | 997 | // big block here to ensure the cleanup gets run when we extract out our argv. |
| 832 | // TODO when watching, watch and rebuild configure script rather than exit here | 998 | { |
| 833 | errors.renderToStderr(io, .{}, color) catch {}; | 999 | { |
| 834 | process.exit(1); | 1000 | const fetch_prog_node = options.parent_progress_node.start("Fetch Packages", 0); |
| 835 | } | 1001 | defer fetch_prog_node.end(); |
| | 1002 | |
| | 1003 | // Reset fork match counts. |
| | 1004 | for (fork_set.keys()) |*fork| fork.uses = 0; |
| | 1005 | |
| | 1006 | var job_queue: Package.Fetch.JobQueue = .{ |
| | 1007 | .io = io, |
| | 1008 | .http_client = &http_client, |
| | 1009 | .global_cache = graph.global_cache_root, |
| | 1010 | .local_storage = &.{ |
| | 1011 | .cache_root = .{ .root_dir = graph.local_cache_root }, |
| | 1012 | .pkg_root = options.pkg_root, |
| | 1013 | }, |
| | 1014 | .recursive = true, |
| | 1015 | .debug_hash = false, |
| | 1016 | .unlazy_set = unlazy_set, |
| | 1017 | .fork_set = fork_set, |
| | 1018 | .mode = options.fetch_mode, |
| | 1019 | .prog_node = fetch_prog_node, |
| | 1020 | .read_only = options.system_pkg_dir_path != null, |
| | 1021 | }; |
| | 1022 | defer job_queue.deinit(); |
| 836 | | 1023 | |
| 837 | if (fetch_only) return process.cleanExit(io); | 1024 | if (options.system_pkg_dir_path == null) { |
| | 1025 | try http_client.initDefaultProxies(arena, &graph.environ_map); |
| | 1026 | } |
| 838 | | 1027 | |
| 839 | // Create the dependencies.zig file for configurer to | 1028 | try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); |
| 840 | // obtain via `@import("@dependencies")`. | 1029 | try job_queue.table.ensureUnusedCapacity(gpa, 1); |
| 841 | { | 1030 | |
| 842 | { | 1031 | const phantom_package_root: Cache.Path = .{ .root_dir = options.build_root.directory }; |
| 843 | dependencies_source.clearRetainingCapacity(); | 1032 | |
| 844 | var source_writer: Io.Writer.Allocating = .fromArrayList(gpa, &dependencies_source); | 1033 | var fetch: Package.Fetch = .{ |
| 845 | defer dependencies_source = source_writer.toArrayList(); | 1034 | .arena = std.heap.ArenaAllocator.init(gpa), |
| 846 | job_queue.createDependenciesSource(&source_writer.writer) catch |err| switch (err) { | 1035 | .location = .{ .relative_path = phantom_package_root }, |
| 847 | error.WriteFailed => return error.OutOfMemory, | 1036 | .location_tok = 0, |
| 848 | }; | 1037 | .hash_tok = .none, |
| 849 | } | 1038 | .name_tok = 0, |
| 850 | // Atomically create the file in a directory named after the hash of its contents. | 1039 | .lazy_status = .eager, |
| 851 | var hh: Cache.HashHelper = .{}; | 1040 | .remote_package_root = phantom_package_root, |
| 852 | hh.addBytes(builtin.zig_version_string); | 1041 | .parent_package_root = phantom_package_root, |
| 853 | hh.addBytes(dependencies_source.items); | 1042 | .parent_manifest_ast = null, |
| 854 | const hex_digest = hh.final(); | 1043 | .prog_node = fetch_prog_node, |
| 855 | const dependencies_zig_path: Path = .{ | 1044 | .job_queue = &job_queue, |
| 856 | .root_dir = graph.local_cache_root, | 1045 | .omit_missing_hash_error = true, |
| 857 | .sub_path = try allocPrint(arena, "o/{s}/dependencies.zig", .{&hex_digest}), | 1046 | .allow_missing_paths_field = false, |
| 858 | }; | 1047 | .use_latest_commit = false, |
| 859 | var atomic_file = try dependencies_zig_path.root_dir.handle.createFileAtomic( | 1048 | |
| 860 | io, | 1049 | .package_root = undefined, |
| 861 | dependencies_zig_path.sub_path, | 1050 | .error_bundle = undefined, |
| 862 | .{ .make_path = true, .replace = true }, | 1051 | .manifest = undefined, |
| 863 | ); | 1052 | .manifest_ast = undefined, |
| 864 | defer atomic_file.deinit(io); | 1053 | .have_manifest = false, |
| 865 | atomic_file.file.writeStreamingAll(io, dependencies_source.items) catch |err| | 1054 | .computed_hash = undefined, |
| 866 | fatal("writing dependencies.zig contents: {t}", .{err}); | 1055 | .has_build_zig = true, |
| 867 | atomic_file.replace(io) catch |err| | 1056 | .oom_flag = false, |
| 868 | fatal("replacing {f}: {t}", .{ dependencies_zig_path, err }); | 1057 | .latest_commit = null, |
| 869 | | 1058 | |
| 870 | deps_mod.root_path = try dependencies_zig_path.toString(arena); | 1059 | .cli_module = build_mod, |
| 871 | } | 1060 | }; |
| 872 | | 1061 | |
| 873 | { | 1062 | job_queue.all_fetches.appendAssumeCapacity(&fetch); |
| 874 | // Add a CliModule for each package's build.zig. | | |
| 875 | const hashes = job_queue.table.keys(); | | |
| 876 | const fetches = job_queue.table.values(); | | |
| 877 | try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); | | |
| 878 | for (hashes, fetches) |*hash, f| { | | |
| 879 | if (f == &fetch) { | | |
| 880 | // The first one is a dummy package for the current project. | | |
| 881 | continue; | | |
| 882 | } | | |
| 883 | if (!f.has_build_zig) | | |
| 884 | continue; | | |
| 885 | const hash_slice = try arena.dupe(u8, hash.toSlice()); | | |
| 886 | | | |
| 887 | const m = try arena.create(CliModule); | | |
| 888 | m.* = .{ | | |
| 889 | .root_path = try f.package_root.toString(arena), | | |
| 890 | .name = hash_slice, | | |
| 891 | }; | | |
| 892 | deps_mod.deps.putAssumeCapacityNoClobber(hash_slice, m); | | |
| 893 | f.cli_module = m; | | |
| 894 | } | | |
| 895 | | 1063 | |
| 896 | // Each build.zig module needs access to each of its | 1064 | job_queue.table.putAssumeCapacityNoClobber( |
| 897 | // dependencies' build.zig modules by name. | 1065 | Package.Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root), |
| 898 | for (fetches) |f| { | 1066 | &fetch, |
| 899 | const mod = f.cli_module orelse continue; | 1067 | ); |
| 900 | if (!f.have_manifest) continue; | | |
| 901 | const man = &f.manifest; | | |
| 902 | const dep_names = man.dependencies.keys(); | | |
| 903 | try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); | | |
| 904 | for (dep_names, man.dependencies.values()) |name, dep| { | | |
| 905 | const dep_digest = Package.Fetch.depDigest( | | |
| 906 | f.package_root, | | |
| 907 | global_cache_directory, | | |
| 908 | dep, | | |
| 909 | ) orelse continue; | | |
| 910 | const dep_mod = job_queue.table.get(dep_digest).?.cli_module orelse continue; | | |
| 911 | const name_cloned = try arena.dupe(u8, name); | | |
| 912 | mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); | | |
| 913 | } | | |
| 914 | } | | |
| 915 | } | | |
| 916 | | 1068 | |
| 917 | // Lower module dependencies to CLI argv. | 1069 | job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" }); |
| 918 | build_configurer_argv.shrinkRetainingCapacity(argv_deps_index); | 1070 | try job_queue.group.await(io); |
| 919 | for (deps_mod.deps.values()) |dep| { | 1071 | |
| 920 | try build_configurer_argv.ensureUnusedCapacity(gpa, 2 * dep.deps.count() + 1); | 1072 | { |
| 921 | for (dep.deps.keys(), dep.deps.values()) |name, sub| { | 1073 | // Ensure that forks were actually used. This is done |
| 922 | build_configurer_argv.appendAssumeCapacity("--dep"); | 1074 | // before printing manifest errors because using a fork can |
| 923 | if (mem.eql(u8, name, sub.name)) { | 1075 | // prevent them. |
| 924 | build_configurer_argv.appendAssumeCapacity(sub.name); | 1076 | var any_unused = false; |
| 925 | } else { | 1077 | for (fork_set.keys()) |*fork| { |
| 926 | build_configurer_argv.appendAssumeCapacity(try allocPrint(arena, "{s}={s}", .{ | 1078 | if (fork.uses == 0) { |
| 927 | name, sub.name, | 1079 | log.err("fork {f} matched no {s} packages", .{ |
| 928 | })); | 1080 | fork.path, fork.manifest.name, |
| 929 | } | 1081 | }); |
| | 1082 | any_unused = true; |
| | 1083 | } else { |
| | 1084 | log.info("fork {f} matched {d} {s} packages", .{ |
| | 1085 | fork.path, fork.uses, fork.manifest.name, |
| | 1086 | }); |
| 930 | } | 1087 | } |
| 931 | build_configurer_argv.appendAssumeCapacity(try allocPrint(arena, "-M{s}={s}/{s}", .{ | | |
| 932 | dep.name, dep.root_path, std.zig.build_zig_basename, | | |
| 933 | })); | | |
| 934 | } | 1088 | } |
| 935 | try deps_mod.lower(arena, gpa, &build_configurer_argv); | 1089 | if (any_unused) process.exit(1); |
| 936 | try build_mod.lower(arena, gpa, &build_configurer_argv); | 1090 | } |
| | 1091 | |
| | 1092 | try job_queue.consolidateErrors(); |
| 937 | | 1093 | |
| 938 | try build_configurer_argv.append(gpa, "--listen=-"); | 1094 | if (fetch.error_bundle.root_list.items.len > 0) { |
| | 1095 | var errors = try fetch.error_bundle.toOwnedBundle(""); |
| | 1096 | errors.renderToStderr(io, .{}, options.color) catch process.exit(1); |
| | 1097 | return error.FailedButCacheIntact; |
| 939 | } | 1098 | } |
| 940 | | 1099 | |
| 941 | const compile_prog_node = main_progress_node.start("Compile Configure Script", 0); | 1100 | if (options.fetch_only) { |
| 942 | defer compile_prog_node.end(); | 1101 | _ = io.lockStderr(&.{}, .no_color) catch {}; |
| 943 | | 1102 | process.exit(0); |
| 944 | switch (cache_poison) { | | |
| 945 | .pure, .disallowed, .ignored => if (try config_man.hit()) { | | |
| 946 | const digest = config_man.final(); | | |
| 947 | break :cp .{ | | |
| 948 | .{ | | |
| 949 | .root_dir = graph.local_cache_root, | | |
| 950 | .sub_path = try allocPrint(arena, "c/{s}", .{&digest}), | | |
| 951 | }, | | |
| 952 | false, | | |
| 953 | }; | | |
| 954 | }, | | |
| 955 | .poisoned => {}, // Don't bother checking for cache hit. | | |
| 956 | } | 1103 | } |
| 957 | | 1104 | |
| 958 | const configure_exe_path: Path = if (std.zig.buildExeSubprocess(gpa, io, .{ | 1105 | // Create the dependencies.zig file for configurer to |
| 959 | .argv = build_configurer_argv.items, | 1106 | // obtain via `@import("@dependencies")`. |
| 960 | .cache_root = graph.local_cache_root, | 1107 | { |
| 961 | .root_name = configurer_exe_name, | 1108 | { |
| 962 | .environ_map = &graph.environ_map, | 1109 | dependencies_source.clearRetainingCapacity(); |
| 963 | .cache_manifest = &config_man, | 1110 | var source_writer: Io.Writer.Allocating = .fromArrayList(gpa, &dependencies_source); |
| 964 | .arch_os_abi = target_arch_os_abi, | 1111 | defer dependencies_source = source_writer.toArrayList(); |
| 965 | .progress_node = compile_prog_node, | 1112 | job_queue.createDependenciesSource(&source_writer.writer) catch |err| switch (err) { |
| 966 | })) |r| r.path else |err| switch (err) { | 1113 | error.WriteFailed => return error.OutOfMemory, |
| 967 | error.AlreadyReported => process.exit(1), | 1114 | }; |
| 968 | // If the file system inputs are populated, we can | 1115 | } |
| 969 | // still watch for changes and try again. | 1116 | // Atomically create the file in a directory named after the hash of its contents. |
| 970 | error.FailedButCacheIntact => @panic("TODO"), | 1117 | var hh: Cache.HashHelper = .{}; |
| 971 | error.Canceled, error.OutOfMemory => |e| return e, | 1118 | hh.addBytes(builtin.zig_version_string); |
| 972 | }; | 1119 | hh.addBytes(dependencies_source.items); |
| 973 | defer gpa.free(configure_exe_path.sub_path); | 1120 | const hex_digest = hh.final(); |
| | 1121 | const dependencies_zig_path: Path = .{ |
| | 1122 | .root_dir = graph.local_cache_root, |
| | 1123 | .sub_path = try allocPrint(arena, "o/{s}/dependencies.zig", .{&hex_digest}), |
| | 1124 | }; |
| | 1125 | var atomic_file = try dependencies_zig_path.root_dir.handle.createFileAtomic( |
| | 1126 | io, |
| | 1127 | dependencies_zig_path.sub_path, |
| | 1128 | .{ .make_path = true, .replace = true }, |
| | 1129 | ); |
| | 1130 | defer atomic_file.deinit(io); |
| | 1131 | atomic_file.file.writeStreamingAll(io, dependencies_source.items) catch |err| |
| | 1132 | fatal("writing dependencies.zig contents: {t}", .{err}); |
| | 1133 | atomic_file.replace(io) catch |err| |
| | 1134 | fatal("replacing {f}: {t}", .{ dependencies_zig_path, err }); |
| 974 | | 1135 | |
| 975 | configure_argv.items[0] = try configure_exe_path.toString(arena); | 1136 | deps_mod.root_path = try dependencies_zig_path.toString(arena); |
| 976 | } | 1137 | } |
| 977 | | 1138 | |
| 978 | if (!process.can_spawn) { | 1139 | { |
| 979 | fatal("cannot spawn command on {t}: {f}", .{ native_os, @as(std.zig.SubprocessCommand, .{ | 1140 | // Add a CliModule for each package's build.zig. |
| 980 | .argv = configure_argv.items, | 1141 | const hashes = job_queue.table.keys(); |
| 981 | }) }); | 1142 | const fetches = job_queue.table.values(); |
| 982 | } | 1143 | try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); |
| | 1144 | for (hashes, fetches) |*hash, f| { |
| | 1145 | if (f == &fetch) { |
| | 1146 | // The first one is a dummy package for the current project. |
| | 1147 | continue; |
| | 1148 | } |
| | 1149 | if (!f.has_build_zig) |
| | 1150 | continue; |
| | 1151 | const hash_slice = try arena.dupe(u8, hash.toSlice()); |
| | 1152 | |
| | 1153 | const m = try arena.create(CliModule); |
| | 1154 | m.* = .{ |
| | 1155 | .root_path = try f.package_root.toString(arena), |
| | 1156 | .name = hash_slice, |
| | 1157 | }; |
| | 1158 | deps_mod.deps.putAssumeCapacityNoClobber(hash_slice, m); |
| | 1159 | f.cli_module = m; |
| | 1160 | } |
| 983 | | 1161 | |
| 984 | const rand_int = randInt(io, u64); | 1162 | // Each build.zig module needs access to each of its |
| 985 | const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); | 1163 | // dependencies' build.zig modules by name. |
| 986 | const config_tmp_path: Path = .{ | 1164 | for (fetches) |f| { |
| 987 | .root_dir = graph.local_cache_root, | 1165 | const mod = f.cli_module orelse continue; |
| 988 | .sub_path = tmp_dir_sub_path, | 1166 | if (!f.have_manifest) continue; |
| 989 | }; | 1167 | const man = &f.manifest; |
| 990 | const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile( | 1168 | const dep_names = man.dependencies.keys(); |
| 991 | io, | 1169 | try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); |
| 992 | config_tmp_path.sub_path, | 1170 | for (dep_names, man.dependencies.values()) |name, dep| { |
| 993 | .{ .read = true, .exclusive = true }, | 1171 | const dep_digest = Package.Fetch.depDigest( |
| 994 | ); | 1172 | f.package_root, |
| 995 | defer config_tmp_file.close(io); | 1173 | graph.global_cache_root, |
| 996 | | 1174 | dep, |
| 997 | const term = term: { | 1175 | ) orelse continue; |
| 998 | const child_node = main_progress_node.start("Run Configure Script", 0); | 1176 | const dep_mod = job_queue.table.get(dep_digest).?.cli_module orelse continue; |
| 999 | defer child_node.end(); | 1177 | const name_cloned = try arena.dupe(u8, name); |
| 1000 | var child = process.spawn(io, .{ | 1178 | mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); |
| 1001 | .argv = configure_argv.items, | 1179 | } |
| 1002 | .stdout = .{ .file = config_tmp_file }, | | |
| 1003 | .progress_node = child_node, | | |
| 1004 | }) catch |err| fatal("failed to spawn configure script {q}: {t}", .{ configure_argv.items[0], err }); | | |
| 1005 | defer child.kill(io); | | |
| 1006 | break :term child.wait(io) catch |err| | | |
| 1007 | fatal("failed to wait configure script {q}: {t}", .{ configure_argv.items[0], err }); | | |
| 1008 | }; | | |
| 1009 | if (!term.success()) { | | |
| 1010 | // Failure to produce the configuration file. | | |
| 1011 | fatal("configure command {f}: {f}", .{ term, @as(std.zig.SubprocessCommand, .{ | | |
| 1012 | .argv = configure_argv.items, | | |
| 1013 | }) }); | | |
| 1014 | } | | |
| 1015 | // Even though the file is designed to be sent directly to make | | |
| 1016 | // runner, we must load it now because: | | |
| 1017 | // * If it contains additional file dependencies, we need to | | |
| 1018 | // add them to `config_man` before obtaining the final digest. | | |
| 1019 | // * If it contains a set of lazy packages that need to be | | |
| 1020 | // fetched, we need to fetch those now and re-run configure. | | |
| 1021 | var configuration = Configuration.loadFile(arena, io, config_tmp_file) catch |err| | | |
| 1022 | fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); | | |
| 1023 | | | |
| 1024 | if (configuration.unlazy_deps.len != 0) { | | |
| 1025 | var any_errors = false; | | |
| 1026 | for (configuration.unlazy_deps) |hash_string| { | | |
| 1027 | const hash = hash_string.slice(&configuration); | | |
| 1028 | assert(hash.len != 0); | | |
| 1029 | if (hash.len > Package.Hash.max_len) { | | |
| 1030 | log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash }); | | |
| 1031 | any_errors = true; | | |
| 1032 | continue; | | |
| 1033 | } | 1180 | } |
| 1034 | try unlazy_set.put(arena, .fromSlice(hash), {}); | | |
| 1035 | } | 1181 | } |
| 1036 | if (any_errors) process.exit(1); | 1182 | |
| 1037 | if (system_pkg_dir_path) |p| { | 1183 | // Lower module dependencies to CLI argv. |
| 1038 | // In this mode, the system needs to provide these packages; they | 1184 | build_configurer_argv.shrinkRetainingCapacity(argv_deps_index); |
| 1039 | // cannot be fetched by Zig. | 1185 | for (deps_mod.deps.values()) |dep| { |
| 1040 | const s = Dir.path.sep_str; | 1186 | try build_configurer_argv.ensureUnusedCapacity(gpa, 2 * dep.deps.count() + 1); |
| 1041 | for (unlazy_set.keys()) |*hash| { | 1187 | for (dep.deps.keys(), dep.deps.values()) |name, sub| { |
| 1042 | log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() }); | 1188 | build_configurer_argv.appendAssumeCapacity("--dep"); |
| | 1189 | if (mem.eql(u8, name, sub.name)) { |
| | 1190 | build_configurer_argv.appendAssumeCapacity(sub.name); |
| | 1191 | } else { |
| | 1192 | build_configurer_argv.appendAssumeCapacity(try allocPrint(arena, "{s}={s}", .{ |
| | 1193 | name, sub.name, |
| | 1194 | })); |
| | 1195 | } |
| 1043 | } | 1196 | } |
| 1044 | log.info("remote package fetching disabled due to --system mode", .{}); | 1197 | build_configurer_argv.appendAssumeCapacity(try allocPrint(arena, "-M{s}={s}/{s}", .{ |
| 1045 | log.info("dependencies might be avoidable depending on build configuration", .{}); | 1198 | dep.name, dep.root_path, std.zig.build_zig_basename, |
| 1046 | process.exit(1); | 1199 | })); |
| 1047 | } | 1200 | } |
| 1048 | continue :cp; | 1201 | try deps_mod.lower(arena, gpa, &build_configurer_argv); |
| 1049 | } | 1202 | try build_mod.lower(arena, gpa, &build_configurer_argv); |
| 1050 | | 1203 | |
| 1051 | for (configuration.path_deps) |path_dep| { | 1204 | try build_configurer_argv.append(gpa, "--listen=-"); |
| 1052 | try config_man.addPathPost(path_dep.toCachePath(&configuration, arena)); | | |
| 1053 | } | 1205 | } |
| 1054 | | 1206 | |
| 1055 | // If it is poisoned, there is no point in moving it to cached | 1207 | const compile_prog_node = options.parent_progress_node.start("Compile Configure Script", 0); |
| 1056 | // location. Just leave it in the tmp directory. | 1208 | defer compile_prog_node.end(); |
| 1057 | if (configuration.poisoned) { | 1209 | |
| 1058 | break :cp .{ config_tmp_path, true }; | 1210 | switch (options.cache_poison) { |
| 1059 | } else { | 1211 | .pure, .disallowed, .ignored => if (try config_man.hit()) { |
| 1060 | const digest = config_man.final(); | 1212 | const digest = config_man.final(); |
| 1061 | const final_path: Path = .{ | 1213 | break :cp .{ |
| 1062 | .root_dir = graph.local_cache_root, | 1214 | .{ |
| 1063 | .sub_path = try allocPrint(arena, "c/{s}", .{&digest}), | 1215 | .root_dir = graph.local_cache_root, |
| 1064 | }; | 1216 | .sub_path = try allocPrint(arena, "c/{s}", .{&digest}), |
| 1065 | Io.Dir.rename( | | |
| 1066 | config_tmp_path.root_dir.handle, | | |
| 1067 | config_tmp_path.sub_path, | | |
| 1068 | final_path.root_dir.handle, | | |
| 1069 | final_path.sub_path, | | |
| 1070 | io, | | |
| 1071 | ) catch |err| retry: { | | |
| 1072 | const e = switch (err) { | | |
| 1073 | error.FileNotFound => e: { | | |
| 1074 | const dir_path = final_path.dirname().?; | | |
| 1075 | dir_path.root_dir.handle.createDirPath(io, dir_path.sub_path) catch |e| | | |
| 1076 | fatal("failed to create directory {f}: {t}", .{ dir_path, e }); | | |
| 1077 | if (Io.Dir.rename( | | |
| 1078 | config_tmp_path.root_dir.handle, | | |
| 1079 | config_tmp_path.sub_path, | | |
| 1080 | final_path.root_dir.handle, | | |
| 1081 | final_path.sub_path, | | |
| 1082 | io, | | |
| 1083 | )) |_| break :retry else |e| break :e e; | | |
| 1084 | }, | 1217 | }, |
| 1085 | else => |e| e, | 1218 | false, |
| 1086 | }; | 1219 | }; |
| 1087 | fatal("failed to rename configuration file from {f} into {f}: {t}", .{ | 1220 | }, |
| 1088 | config_tmp_path, final_path, e, | 1221 | .poisoned => {}, // Don't bother checking for cache hit. |
| 1089 | }); | | |
| 1090 | }; | | |
| 1091 | config_man.writeManifest() catch |err| log.warn("failed to write cache manifest: {t}", .{err}); | | |
| 1092 | break :cp .{ final_path, false }; | | |
| 1093 | } | 1222 | } |
| 1094 | }; | | |
| 1095 | | | |
| 1096 | // Hang on to the configuration file lock until we finish loading the configuration file. | | |
| 1097 | var configuration_lock = if (!poisoned) config_man.toOwnedLock() else null; | | |
| 1098 | defer if (configuration_lock) |*l| l.release(io); | | |
| 1099 | | 1223 | |
| 1100 | switch (print_configuration) { | 1224 | const configure_exe_path: Path = if (std.zig.buildExeSubprocess(gpa, io, .{ |
| 1101 | .path => { | 1225 | .argv = build_configurer_argv.items, |
| 1102 | initStdoutWriter(io).print("{f}\n", .{configuration_path}) catch | 1226 | .cache_root = graph.local_cache_root, |
| 1103 | fatal("failed printing cache file path: {t}", .{stdout_writer_allocation.err.?}); | 1227 | .root_name = configurer_exe_name, |
| 1104 | stdout_writer_allocation.flush() catch |err| | 1228 | .environ_map = &graph.environ_map, |
| 1105 | fatal("failed printing cache file path: {t}", .{err}); | 1229 | .cache_manifest = &config_man, |
| 1106 | return process.cleanExit(io); | 1230 | .arch_os_abi = target_arch_os_abi, |
| 1107 | }, | 1231 | .progress_node = compile_prog_node, |
| 1108 | .none, .zon => {}, | 1232 | })) |r| r.path else |err| return err; |
| | 1233 | defer gpa.free(configure_exe_path.sub_path); |
| | 1234 | |
| | 1235 | configure_argv[0] = try configure_exe_path.toString(arena); |
| 1109 | } | 1236 | } |
| 1110 | | 1237 | |
| 1111 | const configuration = c: { | 1238 | if (!process.can_spawn) { |
| 1112 | var file = configuration_path.root_dir.handle.openFile(io, configuration_path.sub_path, .{}) catch |err| | 1239 | fatal("cannot spawn command on {t}: {f}", .{ native_os, @as(std.zig.SubprocessCommand, .{ |
| 1113 | fatal("failed to open configuration file {f}: {t}", .{ configuration_path, err }); | 1240 | .argv = configure_argv, |
| 1114 | defer file.close(io); | 1241 | }) }); |
| 1115 | break :c Configuration.loadFile(arena, io, file) catch |err| | | |
| 1116 | fatal("failed to load configuration file {f}: {t}", .{ configuration_path, err }); | | |
| 1117 | }; | | |
| 1118 | // Technically if the configuration is marked as poisoned, we could | | |
| 1119 | // already delete the file now, but we leave it around in case the | | |
| 1120 | // maker process fails or crashes and it's helpful to be able to repeat | | |
| 1121 | // execution of the command line or otherwise inspect the configuration file. | | |
| 1122 | const c = &configuration; | | |
| 1123 | var top_level_steps: std.array_hash_map.String(Configuration.Step.Index) = .empty; | | |
| 1124 | for (configuration.steps, 0..) |*conf_step, step_index_usize| { | | |
| 1125 | if (conf_step.owner != .root) continue; | | |
| 1126 | const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize); | | |
| 1127 | const flags = conf_step.flags(c); | | |
| 1128 | switch (flags.tag) { | | |
| 1129 | .top_level => { | | |
| 1130 | const name = step_index.ptr(c).name.slice(c); | | |
| 1131 | try top_level_steps.put(arena, name, step_index); | | |
| 1132 | }, | | |
| 1133 | else => {}, | | |
| 1134 | } | | |
| 1135 | } | | |
| 1136 | for (c.search_prefixes) |search_prefix| { | | |
| 1137 | try graph.search_prefixes.append(arena, search_prefix.slice(c)); | | |
| 1138 | } | 1242 | } |
| 1139 | break :sc .{ | | |
| 1140 | .configuration = configuration, | | |
| 1141 | .top_level_steps = top_level_steps, | | |
| 1142 | .path = configuration_path, | | |
| 1143 | }; | | |
| 1144 | }; | | |
| 1145 | | 1243 | |
| 1146 | if (help_menu) { | 1244 | const rand_int = randInt(io, u64); |
| 1147 | scanned_config.printUsage(&graph, initStdoutWriter(io)) catch |err| switch (err) { | 1245 | const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); |
| 1148 | error.WriteFailed => return stdout_writer_allocation.err.?, | 1246 | const config_tmp_path: Path = .{ |
| 1149 | else => |e| return e, | 1247 | .root_dir = graph.local_cache_root, |
| | 1248 | .sub_path = tmp_dir_sub_path, |
| 1150 | }; | 1249 | }; |
| 1151 | try stdout_writer_allocation.flush(); | 1250 | const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile( |
| 1152 | return cleanExit(io, &scanned_config); | 1251 | io, |
| 1153 | } else if (steps_menu) { | 1252 | config_tmp_path.sub_path, |
| 1154 | scanned_config.printSteps(&graph, initStdoutWriter(io)) catch |err| switch (err) { | 1253 | .{ .read = true, .exclusive = true }, |
| 1155 | error.WriteFailed => return stdout_writer_allocation.err.?, | 1254 | ); |
| 1156 | else => |e| return e, | 1255 | defer config_tmp_file.close(io); |
| | 1256 | |
| | 1257 | const term = term: { |
| | 1258 | const child_node = options.parent_progress_node.start("Run Configure Script", 0); |
| | 1259 | defer child_node.end(); |
| | 1260 | var child = process.spawn(io, .{ |
| | 1261 | .argv = configure_argv, |
| | 1262 | .stdout = .{ .file = config_tmp_file }, |
| | 1263 | .progress_node = child_node, |
| | 1264 | }) catch |err| fatal("failed to spawn configure script {q}: {t}", .{ configure_argv[0], err }); |
| | 1265 | defer child.kill(io); |
| | 1266 | break :term child.wait(io) catch |err| |
| | 1267 | fatal("failed to wait configure script {q}: {t}", .{ configure_argv[0], err }); |
| 1157 | }; | 1268 | }; |
| 1158 | try stdout_writer_allocation.flush(); | 1269 | if (!term.success()) { |
| 1159 | return cleanExit(io, &scanned_config); | 1270 | // Failure to produce the configuration file. |
| 1160 | } else switch (print_configuration) { | 1271 | fatal("configure command {f}: {f}", .{ term, @as(std.zig.SubprocessCommand, .{ |
| 1161 | .none => {}, | 1272 | .argv = configure_argv, |
| 1162 | .zon => { | 1273 | }) }); |
| 1163 | scanned_config.print(initStdoutWriter(io)) catch return stdout_writer_allocation.err.?; | 1274 | } |
| 1164 | try stdout_writer_allocation.flush(); | 1275 | // Even though the file is designed to be sent directly to make |
| 1165 | return cleanExit(io, &scanned_config); | 1276 | // runner, we must load it now because: |
| 1166 | }, | 1277 | // * If it contains additional file dependencies, we need to |
| 1167 | .path => unreachable, | 1278 | // add them to `config_man` before obtaining the final digest. |
| 1168 | } | 1279 | // * If it contains a set of lazy packages that need to be |
| | 1280 | // fetched, we need to fetch those now and re-run configure. |
| | 1281 | var configuration = Configuration.loadFile(arena, io, config_tmp_file) catch |err| |
| | 1282 | fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); |
| | 1283 | |
| | 1284 | if (configuration.unlazy_deps.len != 0) { |
| | 1285 | var any_errors = false; |
| | 1286 | for (configuration.unlazy_deps) |hash_string| { |
| | 1287 | const hash = hash_string.slice(&configuration); |
| | 1288 | assert(hash.len != 0); |
| | 1289 | if (hash.len > Package.Hash.max_len) { |
| | 1290 | log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash }); |
| | 1291 | any_errors = true; |
| | 1292 | continue; |
| | 1293 | } |
| | 1294 | try unlazy_set.put(arena, .fromSlice(hash), {}); |
| | 1295 | } |
| | 1296 | if (any_errors) process.exit(1); |
| | 1297 | if (options.system_pkg_dir_path) |p| { |
| | 1298 | // In this mode, the system needs to provide these packages; they |
| | 1299 | // cannot be fetched by Zig. |
| | 1300 | const s = Dir.path.sep_str; |
| | 1301 | for (unlazy_set.keys()) |*hash| { |
| | 1302 | log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() }); |
| | 1303 | } |
| | 1304 | log.info("remote package fetching disabled due to --system mode", .{}); |
| | 1305 | log.info("dependencies might be avoidable depending on build configuration", .{}); |
| | 1306 | process.exit(1); |
| | 1307 | } |
| | 1308 | continue :cp; |
| | 1309 | } |
| 1169 | | 1310 | |
| 1170 | if (webui_listen != null) { | 1311 | for (configuration.path_deps) |path_dep| { |
| 1171 | if (watch) fatal("using '--webui' and '--watch' together is not yet supported; consider omitting '--watch' in favour of the web UI \"Rebuild\" button", .{}); | 1312 | try config_man.addPathPost(path_dep.toCachePath(&configuration, arena)); |
| 1172 | if (builtin.single_threaded) fatal("'--webui' is not yet supported on single-threaded hosts", .{}); | 1313 | } |
| 1173 | } | | |
| 1174 | | 1314 | |
| 1175 | const install_prefix_path: Path = if (graph.environ_map.get("DESTDIR")) |dest_dir| .{ | 1315 | // If it is poisoned, there is no point in moving it to cached |
| 1176 | .root_dir = .cwd(), | 1316 | // location. Just leave it in the tmp directory. |
| 1177 | .sub_path = try Dir.path.join(arena, &.{ dest_dir, override_install_prefix orelse "/usr" }), | 1317 | if (configuration.poisoned) { |
| 1178 | } else if (override_install_prefix) |cwd_relative| .{ | 1318 | break :cp .{ config_tmp_path, true }; |
| 1179 | .root_dir = .cwd(), | 1319 | } else { |
| 1180 | .sub_path = cwd_relative, | 1320 | const digest = config_man.final(); |
| 1181 | } else .{ | 1321 | const final_path: Path = .{ |
| 1182 | .root_dir = graph.build_root_directory, | 1322 | .root_dir = graph.local_cache_root, |
| 1183 | .sub_path = "zig-out", | 1323 | .sub_path = try allocPrint(arena, "c/{s}", .{&digest}), |
| | 1324 | }; |
| | 1325 | Io.Dir.rename( |
| | 1326 | config_tmp_path.root_dir.handle, |
| | 1327 | config_tmp_path.sub_path, |
| | 1328 | final_path.root_dir.handle, |
| | 1329 | final_path.sub_path, |
| | 1330 | io, |
| | 1331 | ) catch |err| retry: { |
| | 1332 | const e = switch (err) { |
| | 1333 | error.FileNotFound => e: { |
| | 1334 | const dir_path = final_path.dirname().?; |
| | 1335 | dir_path.root_dir.handle.createDirPath(io, dir_path.sub_path) catch |e| |
| | 1336 | fatal("failed to create directory {f}: {t}", .{ dir_path, e }); |
| | 1337 | if (Io.Dir.rename( |
| | 1338 | config_tmp_path.root_dir.handle, |
| | 1339 | config_tmp_path.sub_path, |
| | 1340 | final_path.root_dir.handle, |
| | 1341 | final_path.sub_path, |
| | 1342 | io, |
| | 1343 | )) |_| break :retry else |e| break :e e; |
| | 1344 | }, |
| | 1345 | else => |e| e, |
| | 1346 | }; |
| | 1347 | fatal("failed to rename configuration file from {f} into {f}: {t}", .{ |
| | 1348 | config_tmp_path, final_path, e, |
| | 1349 | }); |
| | 1350 | }; |
| | 1351 | config_man.writeManifest() catch |err| log.warn("failed to write cache manifest: {t}", .{err}); |
| | 1352 | break :cp .{ final_path, false }; |
| | 1353 | } |
| 1184 | }; | 1354 | }; |
| 1185 | | 1355 | |
| 1186 | const install_lib_path: Path = if (override_lib_dir) |cwd_relative| .{ | 1356 | // Hang on to the configuration file lock until we finish loading the configuration file. |
| 1187 | .root_dir = .cwd(), | 1357 | var configuration_lock = if (!poisoned) config_man.toOwnedLock() else null; |
| 1188 | .sub_path = cwd_relative, | 1358 | defer if (configuration_lock) |*l| l.release(io); |
| 1189 | } else try install_prefix_path.join(arena, "lib"); | 1359 | |
| 1190 | | 1360 | switch (options.print_configuration) { |
| 1191 | const install_bin_path: Path = if (override_bin_dir) |cwd_relative| .{ | 1361 | .path => { |
| 1192 | .root_dir = .cwd(), | 1362 | initStdoutWriter(io).print("{f}\n", .{configuration_path}) catch |
| 1193 | .sub_path = cwd_relative, | 1363 | fatal("failed printing cache file path: {t}", .{stdout_writer_allocation.err.?}); |
| 1194 | } else try install_prefix_path.join(arena, "bin"); | 1364 | stdout_writer_allocation.flush() catch |err| |
| 1195 | | 1365 | fatal("failed printing cache file path: {t}", .{err}); |
| 1196 | const install_include_path: Path = if (override_include_dir) |cwd_relative| .{ | 1366 | _ = io.lockStderr(&.{}, .no_color) catch {}; |
| 1197 | .root_dir = .cwd(), | 1367 | process.exit(0); |
| 1198 | .sub_path = cwd_relative, | | |
| 1199 | } else try install_prefix_path.join(arena, "include"); | | |
| 1200 | | | |
| 1201 | var maker: Maker = .{ | | |
| 1202 | .gpa = gpa, | | |
| 1203 | .graph = &graph, | | |
| 1204 | .scanned_config = &scanned_config, | | |
| 1205 | .install_paths = .{ | | |
| 1206 | .prefix = install_prefix_path, | | |
| 1207 | .lib = install_lib_path, | | |
| 1208 | .bin = install_bin_path, | | |
| 1209 | .include = install_include_path, | | |
| 1210 | }, | 1368 | }, |
| 1211 | | 1369 | .none, .zon => {}, |
| 1212 | .steps = try arena.alloc(Step, scanned_config.configuration.steps.len), | | |
| 1213 | .generated_files = try arena.alloc(Path, scanned_config.configuration.generated_files_len), | | |
| 1214 | .run_args = run_args, | | |
| 1215 | | | |
| 1216 | .available_rss = max_rss, | | |
| 1217 | .max_rss_is_default = false, | | |
| 1218 | .max_rss_mutex = .init, | | |
| 1219 | .skip_oom_steps = skip_oom_steps, | | |
| 1220 | .unit_test_timeout_ns = test_timeout_ns, | | |
| 1221 | | | |
| 1222 | .watch = watch, | | |
| 1223 | .web_server = undefined, // set after `prepare` | | |
| 1224 | .memory_blocked_steps = .empty, | | |
| 1225 | .step_stack = .empty, | | |
| 1226 | .pkg_config = .{ .debug = debug_pkg_config }, | | |
| 1227 | | | |
| 1228 | .error_style = error_style, | | |
| 1229 | .multiline_errors = multiline_errors, | | |
| 1230 | .summary = summary orelse if (watch or webui_listen != null) .new else .failures, | | |
| 1231 | }; | | |
| 1232 | defer { | | |
| 1233 | maker.memory_blocked_steps.deinit(gpa); | | |
| 1234 | maker.step_stack.deinit(gpa); | | |
| 1235 | } | | |
| 1236 | | | |
| 1237 | if (maker.available_rss == 0) { | | |
| 1238 | maker.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64); | | |
| 1239 | maker.max_rss_is_default = true; | | |
| 1240 | } | 1370 | } |
| 1241 | | 1371 | |
| 1242 | maker.prepare(step_names.items) catch |err| switch (err) { | 1372 | const configuration = c: { |
| 1243 | error.DependencyLoopDetected, error.InsufficientMemory => { | 1373 | var file = configuration_path.root_dir.handle.openFile(io, configuration_path.sub_path, .{}) catch |err| |
| 1244 | _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; | 1374 | fatal("failed to open configuration file {f}: {t}", .{ configuration_path, err }); |
| 1245 | process.exit(1); | 1375 | defer file.close(io); |
| 1246 | }, | 1376 | break :c Configuration.loadFile(arena, io, file) catch |err| |
| 1247 | else => |e| return e, | 1377 | fatal("failed to load configuration file {f}: {t}", .{ configuration_path, err }); |
| 1248 | }; | 1378 | }; |
| 1249 | | 1379 | // Technically if the configuration is marked as poisoned, we could |
| 1250 | var w: Watch = w: { | 1380 | // already delete the file now, but we leave it around in case the |
| 1251 | if (!watch) break :w undefined; | 1381 | // maker process fails or crashes and it's helpful to be able to repeat |
| 1252 | if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{native_os}); | 1382 | // execution of the command line or otherwise inspect the configuration file. |
| 1253 | break :w try .init(&maker); | 1383 | const c = &configuration; |
| 1254 | }; | 1384 | var top_level_steps: std.array_hash_map.String(Configuration.Step.Index) = .empty; |
| 1255 | | 1385 | for (configuration.steps, 0..) |*conf_step, step_index_usize| { |
| 1256 | const now = Io.Clock.Timestamp.now(io, .awake); | 1386 | if (conf_step.owner != .root) continue; |
| 1257 | | 1387 | const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize); |
| 1258 | maker.web_server = if (webui_listen) |listen_address| ws: { | 1388 | const flags = conf_step.flags(c); |
| 1259 | if (builtin.single_threaded) unreachable; // `fatal` above | 1389 | switch (flags.tag) { |
| 1260 | break :ws .init(.{ | 1390 | .top_level => { |
| 1261 | .maker = &maker, | 1391 | const name = step_index.ptr(c).name.slice(c); |
| 1262 | .root_prog_node = main_progress_node, | 1392 | try top_level_steps.put(arena, name, step_index); |
| 1263 | .listen_address = listen_address, | | |
| 1264 | .base_timestamp = now, | | |
| 1265 | }); | | |
| 1266 | } else null; | | |
| 1267 | | | |
| 1268 | if (maker.web_server) |*ws| { | | |
| 1269 | ws.start() catch |err| fatal("failed to start web server: {t}", .{err}); | | |
| 1270 | } | | |
| 1271 | | | |
| 1272 | rebuild: while (true) : (if (maker.error_style.clearOnUpdate()) { | | |
| 1273 | const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); | | |
| 1274 | defer io.unlockStderr(); | | |
| 1275 | stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H") catch |err| switch (err) { | | |
| 1276 | error.WriteFailed => return stderr.file_writer.err.?, | | |
| 1277 | }; | | |
| 1278 | }) { | | |
| 1279 | if (maker.web_server) |*ws| ws.startBuild(); | | |
| 1280 | | | |
| 1281 | try maker.makeStepNames(step_names.items, main_progress_node, fuzz); | | |
| 1282 | | | |
| 1283 | if (maker.web_server) |*web_server| { | | |
| 1284 | if (fuzz) |mode| if (mode != .forever) fatal( | | |
| 1285 | "error: limited fuzzing is not implemented yet for --webui", | | |
| 1286 | .{}, | | |
| 1287 | ); | | |
| 1288 | | | |
| 1289 | web_server.finishBuild(.{ .fuzz = fuzz != null }); | | |
| 1290 | } | | |
| 1291 | | | |
| 1292 | if (maker.web_server) |*web_server| { | | |
| 1293 | const c = &scanned_config.configuration; | | |
| 1294 | assert(!watch); // fatal error after CLI parsing | | |
| 1295 | while (true) switch (try web_server.wait()) { | | |
| 1296 | .rebuild => { | | |
| 1297 | for (maker.step_stack.keys()) |step_index| { | | |
| 1298 | const step = maker.stepByIndex(step_index); | | |
| 1299 | step.state = .precheck_done; | | |
| 1300 | const deps = step_index.ptr(c).deps.slice(c); | | |
| 1301 | step.pending_deps = @intCast(deps.len); | | |
| 1302 | step.reset(&maker); | | |
| 1303 | } | | |
| 1304 | continue :rebuild; | | |
| 1305 | }, | | |
| 1306 | }; | | |
| 1307 | } | | |
| 1308 | | | |
| 1309 | if (!maker.watch) return; | | |
| 1310 | | | |
| 1311 | // Comptime-known guard to prevent including the logic below when `!Watch.have_impl`. | | |
| 1312 | if (!Watch.have_impl) unreachable; | | |
| 1313 | | | |
| 1314 | try w.update(maker.step_stack.keys()); | | |
| 1315 | | | |
| 1316 | // Wait until a file system notification arrives. Read all such events | | |
| 1317 | // until the buffer is empty. Then wait for a debounce interval, resetting | | |
| 1318 | // if any more events come in. After the debounce interval has passed, | | |
| 1319 | // trigger a rebuild on all steps with modified inputs, as well as their | | |
| 1320 | // recursive dependants. | | |
| 1321 | var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; | | |
| 1322 | const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ | | |
| 1323 | w.dir_count, countSubProcesses(&maker), | | |
| 1324 | }) catch &caption_buf; | | |
| 1325 | var debouncing_node = main_progress_node.start(caption, 0); | | |
| 1326 | var in_debounce = false; | | |
| 1327 | while (true) switch (try w.wait(if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) { | | |
| 1328 | .timeout => { | | |
| 1329 | assert(in_debounce); | | |
| 1330 | debouncing_node.end(); | | |
| 1331 | markFailedStepsDirty(&maker); | | |
| 1332 | continue :rebuild; | | |
| 1333 | }, | | |
| 1334 | .dirty => if (!in_debounce) { | | |
| 1335 | in_debounce = true; | | |
| 1336 | debouncing_node.end(); | | |
| 1337 | debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); | | |
| 1338 | }, | 1393 | }, |
| 1339 | .clean => {}, | 1394 | else => {}, |
| 1340 | }; | 1395 | } |
| | 1396 | } |
| | 1397 | for (c.search_prefixes) |search_prefix| { |
| | 1398 | try graph.search_prefixes.append(arena, search_prefix.slice(c)); |
| 1341 | } | 1399 | } |
| | 1400 | return .{ |
| | 1401 | .configuration = configuration, |
| | 1402 | .top_level_steps = top_level_steps, |
| | 1403 | .path = configuration_path, |
| | 1404 | }; |
| 1342 | } | 1405 | } |
| 1343 | | 1406 | |
| 1344 | fn cmdFetch(gpa: Allocator, graph: *Graph, args: []const []const u8) !void { | 1407 | fn cmdFetch(gpa: Allocator, graph: *Graph, args: []const []const u8) !void { |
| ... | @@ -1894,7 +1957,7 @@ fn prepare(maker: *Maker, step_names: []const []const u8) !void { | ... | @@ -1894,7 +1957,7 @@ fn prepare(maker: *Maker, step_names: []const []const u8) !void { |
| 1894 | fn makeStepNames( | 1957 | fn makeStepNames( |
| 1895 | maker: *Maker, | 1958 | maker: *Maker, |
| 1896 | step_names: []const []const u8, | 1959 | step_names: []const []const u8, |
| 1897 | parent_prog_node: std.Progress.Node, | 1960 | parent_progress_node: std.Progress.Node, |
| 1898 | fuzz: ?Fuzz.Mode, | 1961 | fuzz: ?Fuzz.Mode, |
| 1899 | ) !void { | 1962 | ) !void { |
| 1900 | const graph = maker.graph; | 1963 | const graph = maker.graph; |
| ... | @@ -1918,7 +1981,7 @@ fn makeStepNames( | ... | @@ -1918,7 +1981,7 @@ fn makeStepNames( |
| 1918 | } | 1981 | } |
| 1919 | } | 1982 | } |
| 1920 | | 1983 | |
| 1921 | const step_prog = parent_prog_node.start("steps", step_stack.count()); | 1984 | const step_prog = parent_progress_node.start("steps", step_stack.count()); |
| 1922 | defer step_prog.end(); | 1985 | defer step_prog.end(); |
| 1923 | | 1986 | |
| 1924 | var group: Io.Group = .init; | 1987 | var group: Io.Group = .init; |
| ... | @@ -1998,7 +2061,7 @@ fn makeStepNames( | ... | @@ -1998,7 +2061,7 @@ fn makeStepNames( |
| 1998 | } | 2061 | } |
| 1999 | | 2062 | |
| 2000 | assert(mode == .limit); | 2063 | assert(mode == .limit); |
| 2001 | var f = Fuzz.init(maker, step_stack.keys(), parent_prog_node, mode) catch |err| | 2064 | var f = Fuzz.init(maker, step_stack.keys(), parent_progress_node, mode) catch |err| |
| 2002 | fatal("failed to start fuzzer: {t}", .{err}); | 2065 | fatal("failed to start fuzzer: {t}", .{err}); |
| 2003 | defer f.deinit(); | 2066 | defer f.deinit(); |
| 2004 | | 2067 | |
| ... | @@ -2189,7 +2252,7 @@ fn makeStep( | ... | @@ -2189,7 +2252,7 @@ fn makeStep( |
| 2189 | const step_prog_node = root_prog_node.start(step_name, 0); | 2252 | const step_prog_node = root_prog_node.start(step_name, 0); |
| 2190 | defer step_prog_node.end(); | 2253 | defer step_prog_node.end(); |
| 2191 | | 2254 | |
| 2192 | if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .wip); | 2255 | if (maker.web_server) |ws| ws.updateStepStatus(step_index, .wip); |
| 2193 | | 2256 | |
| 2194 | const new_state: Step.State = for (deps) |dep_index| { | 2257 | const new_state: Step.State = for (deps) |dep_index| { |
| 2195 | const dep_make_step = maker.stepByIndex(dep_index); | 2258 | const dep_make_step = maker.stepByIndex(dep_index); |
| ... | @@ -2224,14 +2287,14 @@ fn makeStep( | ... | @@ -2224,14 +2287,14 @@ fn makeStep( |
| 2224 | .dependency_failure, | 2287 | .dependency_failure, |
| 2225 | .skipped_oom, | 2288 | .skipped_oom, |
| 2226 | => { | 2289 | => { |
| 2227 | if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .failure); | 2290 | if (maker.web_server) |ws| ws.updateStepStatus(step_index, .failure); |
| 2228 | std.Progress.setStatus(.failure_working); | 2291 | std.Progress.setStatus(.failure_working); |
| 2229 | }, | 2292 | }, |
| 2230 | | 2293 | |
| 2231 | .success, | 2294 | .success, |
| 2232 | .skipped, | 2295 | .skipped, |
| 2233 | => { | 2296 | => { |
| 2234 | if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .success); | 2297 | if (maker.web_server) |ws| ws.updateStepStatus(step_index, .success); |
| 2235 | }, | 2298 | }, |
| 2236 | } | 2299 | } |
| 2237 | } | 2300 | } |
Andrew Kelley <andrew@ziglang.org> 2026-06-24 20:01:56-07:00
lib