From abf895595189eb45df8c97f4029c58976815b450 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sun, 14 Jul 2024 19:48:08 -0700
Subject: [PATCH 1/6] make zig compiler processes live across rebuilds

Changes the `make` function signature to take an options struct, which
additionally includes `watch: bool`. I intentionally am not exposing
this information to configure phase logic.

Also adds global zig cache to the compiler cache prefixes.

Closes #20600
---
 lib/compiler/build_runner.zig            |   6 +-
 lib/std/Build.zig                        |   4 +-
 lib/std/Build/Step.zig                   | 163 ++++++++++++++++-------
 lib/std/Build/Step/CheckFile.zig         |   4 +-
 lib/std/Build/Step/CheckObject.zig       |   4 +-
 lib/std/Build/Step/Compile.zig           |  14 +-
 lib/std/Build/Step/ConfigHeader.zig      |   4 +-
 lib/std/Build/Step/Fail.zig              |   4 +-
 lib/std/Build/Step/Fmt.zig               |   4 +-
 lib/std/Build/Step/InstallArtifact.zig   |   4 +-
 lib/std/Build/Step/InstallDir.zig        |   4 +-
 lib/std/Build/Step/InstallFile.zig       |   4 +-
 lib/std/Build/Step/ObjCopy.zig           |   5 +-
 lib/std/Build/Step/Options.zig           |   6 +-
 lib/std/Build/Step/RemoveDir.zig         |   6 +-
 lib/std/Build/Step/Run.zig               |   3 +-
 lib/std/Build/Step/TranslateC.zig        |   5 +-
 lib/std/Build/Step/UpdateSourceFiles.zig |   4 +-
 lib/std/Build/Step/WriteFile.zig         |   4 +-
 lib/std/zig/Server.zig                   |   1 +
 src/Compilation.zig                      |   3 +-
 src/Zcu/PerThread.zig                    |   8 ++
 test/standalone/cmakedefine/build.zig    |   4 +-
 23 files changed, 181 insertions(+), 87 deletions(-)

diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig
index 703571eb56a6379908d8336faca66342aa675b3e..5f5a991fd624ca419152332ebe1fadbd1d9038e3 100644
--- a/lib/compiler/build_runner.zig
+++ b/lib/compiler/build_runner.zig
@@ -1031,7 +1031,11 @@ fn workerMakeOneStep(
     const sub_prog_node = prog_node.start(s.name, 0);
     defer sub_prog_node.end();
 
-    const make_result = s.make(sub_prog_node);
+    const make_result = s.make(.{
+        .progress_node = sub_prog_node,
+        .thread_pool = thread_pool,
+        .watch = run.watch,
+    });
 
     // No matter the result, we want to display error/warning messages.
     const show_compile_errors = !run.prominent_compile_errors and
diff --git a/lib/std/Build.zig b/lib/std/Build.zig
index 36f7396c8eb6e70779ba4bb6f975ea3550b3cc40..c5ff143acaa41ad2365205fb716a9382116dfb7e 100644
--- a/lib/std/Build.zig
+++ b/lib/std/Build.zig
@@ -1078,8 +1078,8 @@ pub fn getUninstallStep(b: *Build) *Step {
     return &b.uninstall_tls.step;
 }
 
-fn makeUninstall(uninstall_step: *Step, prog_node: std.Progress.Node) anyerror!void {
-    _ = prog_node;
+fn makeUninstall(uninstall_step: *Step, options: Step.MakeOptions) anyerror!void {
+    _ = options;
     const uninstall_tls: *TopLevelStep = @fieldParentPtr("step", uninstall_step);
     const b: *Build = @fieldParentPtr("uninstall_tls", uninstall_tls);
 
diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig
index 3c6cd660fff14778dcd97c30132c18ec85adbef4..397ad6c55f545c72ef95bf92c27066e5d0fd95b1 100644
--- a/lib/std/Build/Step.zig
+++ b/lib/std/Build/Step.zig
@@ -68,7 +68,13 @@ pub const TestResults = struct {
     }
 };
 
-pub const MakeFn = *const fn (step: *Step, prog_node: std.Progress.Node) anyerror!void;
+pub const MakeOptions = struct {
+    progress_node: std.Progress.Node,
+    thread_pool: *std.Thread.Pool,
+    watch: bool,
+};
+
+pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void;
 
 pub const State = enum {
     precheck_unstarted,
@@ -219,10 +225,10 @@ pub fn init(options: StepOptions) Step {
 /// If the Step's `make` function reports `error.MakeFailed`, it indicates they
 /// have already reported the error. Otherwise, we add a simple error report
 /// here.
-pub fn make(s: *Step, prog_node: std.Progress.Node) error{ MakeFailed, MakeSkipped }!void {
+pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void {
     const arena = s.owner.allocator;
 
-    s.makeFn(s, prog_node) catch |err| switch (err) {
+    s.makeFn(s, options) catch |err| switch (err) {
         error.MakeFailed => return error.MakeFailed,
         error.MakeSkipped => return error.MakeSkipped,
         else => {
@@ -260,8 +266,8 @@ pub fn getStackTrace(s: *Step) ?std.builtin.StackTrace {
     };
 }
 
-fn makeNoOp(step: *Step, prog_node: std.Progress.Node) anyerror!void {
-    _ = prog_node;
+fn makeNoOp(step: *Step, options: MakeOptions) anyerror!void {
+    _ = options;
 
     var all_cached = true;
 
@@ -352,13 +358,25 @@ pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutO
     try step.result_error_msgs.append(arena, msg);
 }
 
+pub const ZigProcess = struct {
+    child: std.process.Child,
+    poller: std.io.Poller(StreamEnum),
+
+    pub const StreamEnum = enum { stdout, stderr };
+};
+
 /// Assumes that argv contains `--listen=-` and that the process being spawned
 /// is the zig compiler - the same version that compiled the build runner.
 pub fn evalZigProcess(
     s: *Step,
     argv: []const []const u8,
     prog_node: std.Progress.Node,
+    watch: bool,
 ) !?[]const u8 {
+    if (s.getZigProcess()) |zp| {
+        assert(watch);
+        return zigProcessUpdate(s, zp, watch);
+    }
     assert(argv.len != 0);
     const b = s.owner;
     const arena = b.allocator;
@@ -378,29 +396,76 @@ pub fn evalZigProcess(
     child.spawn() catch |err| return s.fail("unable to spawn {s}: {s}", .{
         argv[0], @errorName(err),
     });
+
+    const zp = try arena.create(ZigProcess);
+    zp.* = .{
+        .child = child,
+        .poller = std.io.poll(gpa, ZigProcess.StreamEnum, .{
+            .stdout = child.stdout.?,
+            .stderr = child.stderr.?,
+        }),
+    };
+    if (watch) s.setZigProcess(zp);
+    defer if (!watch) zp.poller.deinit();
+
+    const result = try zigProcessUpdate(s, zp, watch);
+
+    if (!watch) {
+        // Send EOF to stdin.
+        zp.child.stdin.?.close();
+        zp.child.stdin = null;
+
+        const term = zp.child.wait() catch |err| {
+            return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(err) });
+        };
+        s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0;
+
+        // Special handling for Compile step that is expecting compile errors.
+        if (s.cast(Compile)) |compile| switch (term) {
+            .Exited => {
+                // Note that the exit code may be 0 in this case due to the
+                // compiler server protocol.
+                if (compile.expect_errors != null) {
+                    return error.NeedCompileErrorCheck;
+                }
+            },
+            else => {},
+        };
+
+        try handleChildProcessTerm(s, term, null, argv);
+    }
+
+    if (s.result_error_bundle.errorMessageCount() > 0) {
+        return s.fail("the following command failed with {d} compilation errors:\n{s}", .{
+            s.result_error_bundle.errorMessageCount(),
+            try allocPrintCmd(arena, null, argv),
+        });
+    }
+
+    return result;
+}
+
+fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {
+    const b = s.owner;
+    const arena = b.allocator;
+
     var timer = try std.time.Timer.start();
 
-    var poller = std.io.poll(gpa, enum { stdout, stderr }, .{
-        .stdout = child.stdout.?,
-        .stderr = child.stderr.?,
-    });
-    defer poller.deinit();
-
-    try sendMessage(child.stdin.?, .update);
-    try sendMessage(child.stdin.?, .exit);
+    try sendMessage(zp.child.stdin.?, .update);
+    if (!watch) try sendMessage(zp.child.stdin.?, .exit);
 
     const Header = std.zig.Server.Message.Header;
     var result: ?[]const u8 = null;
 
-    const stdout = poller.fifo(.stdout);
+    const stdout = zp.poller.fifo(.stdout);
 
     poll: while (true) {
         while (stdout.readableLength() < @sizeOf(Header)) {
-            if (!(try poller.poll())) break :poll;
+            if (!(try zp.poller.poll())) break :poll;
         }
         const header = stdout.reader().readStruct(Header) catch unreachable;
         while (stdout.readableLength() < header.bytes_len) {
-            if (!(try poller.poll())) break :poll;
+            if (!(try zp.poller.poll())) break :poll;
         }
         const body = stdout.readableSliceOfLen(header.bytes_len);
 
@@ -428,12 +493,22 @@ pub fn evalZigProcess(
                     .string_bytes = try arena.dupe(u8, string_bytes),
                     .extra = extra_array,
                 };
+                if (watch) {
+                    // This message indicates the end of the update.
+                    stdout.discard(body.len);
+                    break;
+                }
             },
             .emit_bin_path => {
                 const EbpHdr = std.zig.Server.Message.EmitBinPath;
                 const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
                 s.result_cached = ebp_hdr.flags.cache_hit;
                 result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
+                if (watch) {
+                    // This message indicates the end of the update.
+                    stdout.discard(body.len);
+                    break;
+                }
             },
             .file_system_inputs => {
                 s.clearWatchInputs();
@@ -470,6 +545,13 @@ pub fn evalZigProcess(
                             };
                             try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
                         },
+                        .global_cache => {
+                            const path: Build.Cache.Path = .{
+                                .root_dir = s.owner.graph.global_cache_root,
+                                .sub_path = sub_path_dirname,
+                            };
+                            try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
+                        },
                     }
                 }
             },
@@ -479,45 +561,30 @@ pub fn evalZigProcess(
         stdout.discard(body.len);
     }
 
-    const stderr = poller.fifo(.stderr);
+    s.result_duration_ns = timer.read();
+
+    const stderr = zp.poller.fifo(.stderr);
     if (stderr.readableLength() > 0) {
         try s.result_error_msgs.append(arena, try stderr.toOwnedSlice());
     }
 
-    // Send EOF to stdin.
-    child.stdin.?.close();
-    child.stdin = null;
-
-    const term = child.wait() catch |err| {
-        return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(err) });
-    };
-    s.result_duration_ns = timer.read();
-    s.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0;
-
-    // Special handling for Compile step that is expecting compile errors.
-    if (s.cast(Compile)) |compile| switch (term) {
-        .Exited => {
-            // Note that the exit code may be 0 in this case due to the
-            // compiler server protocol.
-            if (compile.expect_errors != null) {
-                return error.NeedCompileErrorCheck;
-            }
-        },
-        else => {},
-    };
-
-    try handleChildProcessTerm(s, term, null, argv);
-
-    if (s.result_error_bundle.errorMessageCount() > 0) {
-        return s.fail("the following command failed with {d} compilation errors:\n{s}", .{
-            s.result_error_bundle.errorMessageCount(),
-            try allocPrintCmd(arena, null, argv),
-        });
-    }
-
     return result;
 }
 
+fn getZigProcess(s: *Step) ?*ZigProcess {
+    return switch (s.id) {
+        .compile => s.cast(Compile).?.zig_process,
+        else => null,
+    };
+}
+
+fn setZigProcess(s: *Step, zp: *ZigProcess) void {
+    switch (s.id) {
+        .compile => s.cast(Compile).?.zig_process = zp,
+        else => unreachable,
+    }
+}
+
 fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void {
     const header: std.zig.Client.Message.Header = .{
         .tag = tag,
diff --git a/lib/std/Build/Step/CheckFile.zig b/lib/std/Build/Step/CheckFile.zig
index c7a2046c1f63effe584a96ea87fe0f2577ebe3d2..699e6d2e9d3ffe7420893c169c8473262bf6601f 100644
--- a/lib/std/Build/Step/CheckFile.zig
+++ b/lib/std/Build/Step/CheckFile.zig
@@ -46,8 +46,8 @@ pub fn setName(check_file: *CheckFile, name: []const u8) void {
     check_file.step.name = name;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const check_file: *CheckFile = @fieldParentPtr("step", step);
     try step.singleUnchangingWatchInput(check_file.source);
diff --git a/lib/std/Build/Step/CheckObject.zig b/lib/std/Build/Step/CheckObject.zig
index 93ee57e3b47bd86b9e6ff9ab2b6eb9233b639bb9..ce3300311777de18c2b9bc3c9334525d8b3c3e90 100644
--- a/lib/std/Build/Step/CheckObject.zig
+++ b/lib/std/Build/Step/CheckObject.zig
@@ -550,8 +550,8 @@ pub fn checkComputeCompare(
     check_object.checks.append(check) catch @panic("OOM");
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, make_options: Step.MakeOptions) !void {
+    _ = make_options;
     const b = step.owner;
     const gpa = b.allocator;
     const check_object: *CheckObject = @fieldParentPtr("step", step);
diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig
index 89862e75013745b4deb610908ab40b06cc1012f5..9af6ace9c0c6af82d887cbc9a10c4b1e418923b2 100644
--- a/lib/std/Build/Step/Compile.zig
+++ b/lib/std/Build/Step/Compile.zig
@@ -213,6 +213,10 @@ is_linking_libcpp: bool = false,
 
 no_builtin: bool = false,
 
+/// Populated during the make phase when there is a long-lived compiler process.
+/// Managed by the build runner, not user build script.
+zig_process: ?*Step.ZigProcess,
+
 pub const ExpectedCompileErrors = union(enum) {
     contains: []const u8,
     exact: []const []const u8,
@@ -398,6 +402,8 @@ pub fn create(owner: *std.Build, options: Options) *Compile {
 
         .use_llvm = options.use_llvm,
         .use_lld = options.use_lld,
+
+        .zig_process = null,
     };
 
     compile.root_module.init(owner, options.root_module, compile);
@@ -1735,13 +1741,17 @@ fn getZigArgs(compile: *Compile) ![][]const u8 {
     return try zig_args.toOwnedSlice();
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
+fn make(step: *Step, options: Step.MakeOptions) !void {
     const b = step.owner;
     const compile: *Compile = @fieldParentPtr("step", step);
 
     const zig_args = try getZigArgs(compile);
 
-    const maybe_output_bin_path = step.evalZigProcess(zig_args, prog_node) catch |err| switch (err) {
+    const maybe_output_bin_path = step.evalZigProcess(
+        zig_args,
+        options.progress_node,
+        options.watch,
+    ) catch |err| switch (err) {
         error.NeedCompileErrorCheck => {
             assert(compile.expect_errors != null);
             try checkCompileErrors(compile);
diff --git a/lib/std/Build/Step/ConfigHeader.zig b/lib/std/Build/Step/ConfigHeader.zig
index fd655125cf11d7d263081e1952700f384bea22b6..512460a53255d65627991e8d92221c623869f9e5 100644
--- a/lib/std/Build/Step/ConfigHeader.zig
+++ b/lib/std/Build/Step/ConfigHeader.zig
@@ -164,8 +164,8 @@ fn putValue(config_header: *ConfigHeader, field_name: []const u8, comptime T: ty
     }
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const config_header: *ConfigHeader = @fieldParentPtr("step", step);
     if (config_header.style.getPath()) |lp| try step.singleUnchangingWatchInput(lp);
diff --git a/lib/std/Build/Step/Fail.zig b/lib/std/Build/Step/Fail.zig
index 09b8515f773e64ebb8d49a1d6aafec51e8f8b075..9236c2ac7b6176ab9a7fc15f0afee83fa2107c42 100644
--- a/lib/std/Build/Step/Fail.zig
+++ b/lib/std/Build/Step/Fail.zig
@@ -24,8 +24,8 @@ pub fn create(owner: *std.Build, error_msg: []const u8) *Fail {
     return fail;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node; // No progress to report.
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options; // No progress to report.
 
     const fail: *Fail = @fieldParentPtr("step", step);
 
diff --git a/lib/std/Build/Step/Fmt.zig b/lib/std/Build/Step/Fmt.zig
index 1d3850a7ec769403550dda5e8fcb8be5705c010a..576aeb1d21db711b2029fc87462ddfe35ff116a1 100644
--- a/lib/std/Build/Step/Fmt.zig
+++ b/lib/std/Build/Step/Fmt.zig
@@ -36,7 +36,9 @@ pub fn create(owner: *std.Build, options: Options) *Fmt {
     return fmt;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    const prog_node = options.progress_node;
+
     // TODO: if check=false, this means we are modifying source files in place, which
     // is an operation that could race against other operations also modifying source files
     // in place. In this case, this step should obtain a write lock while making those
diff --git a/lib/std/Build/Step/InstallArtifact.zig b/lib/std/Build/Step/InstallArtifact.zig
index bd1d5db4a97700072a037fddcb82af4e175316cd..4e778d897cf834facb922d925f7c57581825308e 100644
--- a/lib/std/Build/Step/InstallArtifact.zig
+++ b/lib/std/Build/Step/InstallArtifact.zig
@@ -115,8 +115,8 @@ pub fn create(owner: *std.Build, artifact: *Step.Compile, options: Options) *Ins
     return install_artifact;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const install_artifact: *InstallArtifact = @fieldParentPtr("step", step);
     const b = step.owner;
     const cwd = fs.cwd();
diff --git a/lib/std/Build/Step/InstallDir.zig b/lib/std/Build/Step/InstallDir.zig
index 78281e56d9df7c70972ffccf933e63f0e29cf54c..4d4ff78cfc755eafd3dfd59358a46b141acda307 100644
--- a/lib/std/Build/Step/InstallDir.zig
+++ b/lib/std/Build/Step/InstallDir.zig
@@ -55,8 +55,8 @@ pub fn create(owner: *std.Build, options: Options) *InstallDir {
     return install_dir;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const install_dir: *InstallDir = @fieldParentPtr("step", step);
     step.clearWatchInputs();
diff --git a/lib/std/Build/Step/InstallFile.zig b/lib/std/Build/Step/InstallFile.zig
index d29ac21c1cd4206cec969c33335b0c2a018c9853..fb1c0ffc3405dcacbc2907d604929d6d1a983192 100644
--- a/lib/std/Build/Step/InstallFile.zig
+++ b/lib/std/Build/Step/InstallFile.zig
@@ -35,8 +35,8 @@ pub fn create(
     return install_file;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const install_file: *InstallFile = @fieldParentPtr("step", step);
     try step.singleUnchangingWatchInput(install_file.source);
diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig
index d314550f601d98e42005e2df789ca2bc2e535866..6ed23d3fb4c28d10d37876f7fa4851f7c6a5b12b 100644
--- a/lib/std/Build/Step/ObjCopy.zig
+++ b/lib/std/Build/Step/ObjCopy.zig
@@ -90,7 +90,8 @@ pub fn getOutputSeparatedDebug(objcopy: *const ObjCopy) ?std.Build.LazyPath {
     return if (objcopy.output_file_debug) |*file| .{ .generated = .{ .file = file } } else null;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    const prog_node = options.progress_node;
     const b = step.owner;
     const objcopy: *ObjCopy = @fieldParentPtr("step", step);
     try step.singleUnchangingWatchInput(objcopy.input_file);
@@ -158,7 +159,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
     try argv.appendSlice(&.{ full_src_path, full_dest_path });
 
     try argv.append("--listen=-");
-    _ = try step.evalZigProcess(argv.items, prog_node);
+    _ = try step.evalZigProcess(argv.items, prog_node, false);
 
     objcopy.output_file.path = full_dest_path;
     if (objcopy.output_file_debug) |*file| file.path = full_dest_path_debug;
diff --git a/lib/std/Build/Step/Options.zig b/lib/std/Build/Step/Options.zig
index b67acd40869745bc321240a26e1a305e84346e5b..7c872db170ce3ded43e9854355635135dee22dcd 100644
--- a/lib/std/Build/Step/Options.zig
+++ b/lib/std/Build/Step/Options.zig
@@ -410,9 +410,9 @@ pub fn getOutput(options: *Options) LazyPath {
     return .{ .generated = .{ .file = &options.generated_file } };
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    // This step completes so quickly that no progress is necessary.
-    _ = prog_node;
+fn make(step: *Step, make_options: Step.MakeOptions) !void {
+    // This step completes so quickly that no progress reporting is necessary.
+    _ = make_options;
 
     const b = step.owner;
     const options: *Options = @fieldParentPtr("step", step);
diff --git a/lib/std/Build/Step/RemoveDir.zig b/lib/std/Build/Step/RemoveDir.zig
index 1b7dc7feb8bedd3e827ce29eea1f74ac1e3cb283..e2d4c02abca8270603279dca01061cbe98a1f798 100644
--- a/lib/std/Build/Step/RemoveDir.zig
+++ b/lib/std/Build/Step/RemoveDir.zig
@@ -23,10 +23,8 @@ pub fn create(owner: *std.Build, doomed_path: LazyPath) *RemoveDir {
     return remove_dir;
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    // TODO update progress node while walking file system.
-    // Should the standard library support this use case??
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
 
     const b = step.owner;
     const remove_dir: *RemoveDir = @fieldParentPtr("step", step);
diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig
index bc89d1a12fe5835a92f74ef827540c7366587f4e..d79d4647b187a4597564684ec77932c19dad7090 100644
--- a/lib/std/Build/Step/Run.zig
+++ b/lib/std/Build/Step/Run.zig
@@ -595,7 +595,8 @@ const IndexedOutput = struct {
     tag: @typeInfo(Arg).Union.tag_type.?,
     output: *Output,
 };
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    const prog_node = options.progress_node;
     const b = step.owner;
     const arena = b.allocator;
     const run: *Run = @fieldParentPtr("step", step);
diff --git a/lib/std/Build/Step/TranslateC.zig b/lib/std/Build/Step/TranslateC.zig
index d7c62ed1948ead4bde9180e039be37fe39026e59..ac4729abd0cc57b2d9e3602a3fb9c43cc554d69a 100644
--- a/lib/std/Build/Step/TranslateC.zig
+++ b/lib/std/Build/Step/TranslateC.zig
@@ -116,7 +116,8 @@ pub fn defineCMacroRaw(translate_c: *TranslateC, name_and_value: []const u8) voi
     translate_c.c_macros.append(translate_c.step.owner.dupe(name_and_value)) catch @panic("OOM");
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    const prog_node = options.progress_node;
     const b = step.owner;
     const translate_c: *TranslateC = @fieldParentPtr("step", step);
 
@@ -154,7 +155,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
 
     try argv_list.append(translate_c.source.getPath2(b, step));
 
-    const output_path = try step.evalZigProcess(argv_list.items, prog_node);
+    const output_path = try step.evalZigProcess(argv_list.items, prog_node, false);
 
     translate_c.out_basename = fs.path.basename(output_path.?);
     const output_dir = fs.path.dirname(output_path.?).?;
diff --git a/lib/std/Build/Step/UpdateSourceFiles.zig b/lib/std/Build/Step/UpdateSourceFiles.zig
index 9d1c8e20fe62c88aebdcbdd33c2728dcac724565..d4a9565083fa203600c52fde32ff18525a82c993 100644
--- a/lib/std/Build/Step/UpdateSourceFiles.zig
+++ b/lib/std/Build/Step/UpdateSourceFiles.zig
@@ -67,8 +67,8 @@ pub fn addBytesToSource(usf: *UpdateSourceFiles, bytes: []const u8, sub_path: []
     }) catch @panic("OOM");
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const usf: *UpdateSourceFiles = @fieldParentPtr("step", step);
 
diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig
index c1488a23d2ed0f233d26199d2f8baedc5da7784a..29fba1c871cda22bfbf8a72c32699ce57b4c947e 100644
--- a/lib/std/Build/Step/WriteFile.zig
+++ b/lib/std/Build/Step/WriteFile.zig
@@ -171,8 +171,8 @@ fn maybeUpdateName(write_file: *WriteFile) void {
     }
 }
 
-fn make(step: *Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn make(step: *Step, options: Step.MakeOptions) !void {
+    _ = options;
     const b = step.owner;
     const arena = b.allocator;
     const gpa = arena;
diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig
index f1d3bc7b61d3df9fff2e5975d271ebd86b54e063..2d66ee628d6f5a407655a6981489743426a17c9b 100644
--- a/lib/std/zig/Server.zig
+++ b/lib/std/zig/Server.zig
@@ -36,6 +36,7 @@ pub const Message = struct {
         cwd,
         zig_lib,
         local_cache,
+        global_cache,
     };
 
     /// Trailing:
diff --git a/src/Compilation.zig b/src/Compilation.zig
index a785351df554166ab8dc0c53cd7a5313ffdd14ce..83a7501ffa1c6ca4571cb7107d200e05e1fce38b 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -1363,6 +1363,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
         cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() });
         cache.addPrefix(options.zig_lib_directory);
         cache.addPrefix(options.local_cache_directory);
+        cache.addPrefix(options.global_cache_directory);
         errdefer cache.manifest_dir.close();
 
         // This is shared hasher state common to zig source and all C source files.
@@ -2358,7 +2359,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
     }
 }
 
-fn appendFileSystemInput(
+pub fn appendFileSystemInput(
     comp: *Compilation,
     file_system_inputs: *std.ArrayListUnmanaged(u8),
     root: Cache.Path,
diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig
index 59b6b6bf0bb54f6b1033f038529523f7911b8b9a..73b4e6de5c09c302292f9dc043caf01d6dd50d3e 100644
--- a/src/Zcu/PerThread.zig
+++ b/src/Zcu/PerThread.zig
@@ -1418,6 +1418,10 @@ pub fn importPkg(pt: Zcu.PerThread, mod: *Module) !Zcu.ImportFileResult {
     const sub_file_path = try gpa.dupe(u8, mod.root_src_path);
     errdefer gpa.free(sub_file_path);
 
+    const comp = zcu.comp;
+    if (comp.file_system_inputs) |fsi|
+        try comp.appendFileSystemInput(fsi, mod.root, sub_file_path);
+
     const new_file = try gpa.create(Zcu.File);
     errdefer gpa.destroy(new_file);
 
@@ -1527,6 +1531,10 @@ pub fn importFile(
         resolved_root_path, resolved_path, sub_file_path, import_string,
     });
 
+    const comp = zcu.comp;
+    if (comp.file_system_inputs) |fsi|
+        try comp.appendFileSystemInput(fsi, mod.root, sub_file_path);
+
     const path_digest = zcu.computePathDigest(mod, sub_file_path);
     const new_file_index = try ip.createFile(gpa, pt.tid, .{
         .bin_digest = path_digest,
diff --git a/test/standalone/cmakedefine/build.zig b/test/standalone/cmakedefine/build.zig
index 3c57523373e85cc268e0b8bd34e87c237dd34d65..50df9d0b7a2d04f9e01f4addf3f29cc2dbb81470 100644
--- a/test/standalone/cmakedefine/build.zig
+++ b/test/standalone/cmakedefine/build.zig
@@ -80,8 +80,8 @@ pub fn build(b: *std.Build) void {
     test_step.dependOn(&wrapper_header.step);
 }
 
-fn compare_headers(step: *std.Build.Step, prog_node: std.Progress.Node) !void {
-    _ = prog_node;
+fn compare_headers(step: *std.Build.Step, options: std.Build.Step.MakeOptions) !void {
+    _ = options;
     const allocator = step.owner.allocator;
     const expected_fmt = "expected_{s}";
 
-- 
2.54.0


From 716b128a24ffc44a8694f3d61e3d74b5297fd564 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sun, 14 Jul 2024 21:18:09 -0700
Subject: [PATCH 2/6] frontend: add -fincremental, -fno-incremental flag

Remove --debug-incremental

This flag is also added to the build system. Importantly, this tells
Compile step whether or not to keep the compiler running between
rebuilds. It defaults off because it is currently crashing
zirUpdateRefs.
---
 lib/compiler/build_runner.zig  |  7 +++++++
 lib/std/Build.zig              |  1 +
 lib/std/Build/Step/Compile.zig |  4 +++-
 src/Compilation.zig            |  6 +++---
 src/Sema.zig                   | 12 ++++++------
 src/Zcu.zig                    |  2 +-
 src/Zcu/PerThread.zig          |  2 +-
 src/main.zig                   | 23 ++++++++++++++---------
 8 files changed, 36 insertions(+), 21 deletions(-)

diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig
index 5f5a991fd624ca419152332ebe1fadbd1d9038e3..af65d109484f18ef4cb0429eef639c13bbb82cb7 100644
--- a/lib/compiler/build_runner.zig
+++ b/lib/compiler/build_runner.zig
@@ -72,6 +72,7 @@ pub fn main() !void {
             .query = .{},
             .result = try std.zig.system.resolveTargetQuery(.{}),
         },
+        .incremental = null,
     };
 
     graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() });
@@ -235,6 +236,10 @@ pub fn main() !void {
                 prominent_compile_errors = true;
             } else if (mem.eql(u8, arg, "--watch")) {
                 watch = true;
+            } else if (mem.eql(u8, arg, "-fincremental")) {
+                graph.incremental = true;
+            } else if (mem.eql(u8, arg, "-fno-incremental")) {
+                graph.incremental = false;
             } else if (mem.eql(u8, arg, "-fwine")) {
                 builder.enable_wine = true;
             } else if (mem.eql(u8, arg, "-fno-wine")) {
@@ -1216,6 +1221,8 @@ fn usage(b: *std.Build, out_stream: anytype) !void {
         \\  --fetch                      Exit after fetching dependency tree
         \\  --watch                      Continuously rebuild when source files are modified
         \\  --debounce <ms>              Delay before rebuilding after changed file detected
+        \\     -fincremental             Enable incremental compilation
+        \\  -fno-incremental             Disable incremental compilation
         \\
         \\Project-Specific Options:
         \\
diff --git a/lib/std/Build.zig b/lib/std/Build.zig
index c5ff143acaa41ad2365205fb716a9382116dfb7e..29621f95c93e50f56a16d6134c7360cfcceccd46 100644
--- a/lib/std/Build.zig
+++ b/lib/std/Build.zig
@@ -120,6 +120,7 @@ pub const Graph = struct {
     needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{},
     /// Information about the native target. Computed before build() is invoked.
     host: ResolvedTarget,
+    incremental: ?bool,
 };
 
 const AvailableDeps = []const struct { []const u8, []const u8 };
diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig
index 9af6ace9c0c6af82d887cbc9a10c4b1e418923b2..724656ce29f12659708b28ff3ebab321d216baf1 100644
--- a/lib/std/Build/Step/Compile.zig
+++ b/lib/std/Build/Step/Compile.zig
@@ -1679,6 +1679,8 @@ fn getZigArgs(compile: *Compile) ![][]const u8 {
         b.fmt("{}", .{err_limit}),
     });
 
+    try addFlag(&zig_args, "incremental", b.graph.incremental);
+
     try zig_args.append("--listen=-");
 
     // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux
@@ -1750,7 +1752,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
     const maybe_output_bin_path = step.evalZigProcess(
         zig_args,
         options.progress_node,
-        options.watch,
+        (b.graph.incremental == true) and options.watch,
     ) catch |err| switch (err) {
         error.NeedCompileErrorCheck => {
             assert(compile.expect_errors != null);
diff --git a/src/Compilation.zig b/src/Compilation.zig
index 83a7501ffa1c6ca4571cb7107d200e05e1fce38b..76dc42fc64a9addf630c54371f4c61e9c7f85cc1 100644
--- a/src/Compilation.zig
+++ b/src/Compilation.zig
@@ -169,7 +169,7 @@ time_report: bool,
 stack_report: bool,
 debug_compiler_runtime_libs: bool,
 debug_compile_errors: bool,
-debug_incremental: bool,
+incremental: bool,
 job_queued_compiler_rt_lib: bool = false,
 job_queued_compiler_rt_obj: bool = false,
 job_queued_update_builtin_zig: bool,
@@ -1134,7 +1134,7 @@ pub const CreateOptions = struct {
     verbose_llvm_cpu_features: bool = false,
     debug_compiler_runtime_libs: bool = false,
     debug_compile_errors: bool = false,
-    debug_incremental: bool = false,
+    incremental: bool = false,
     /// Normally when you create a `Compilation`, Zig will automatically build
     /// and link in required dependencies, such as compiler-rt and libc. When
     /// building such dependencies themselves, this flag must be set to avoid
@@ -1516,7 +1516,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
             .test_name_prefix = options.test_name_prefix,
             .debug_compiler_runtime_libs = options.debug_compiler_runtime_libs,
             .debug_compile_errors = options.debug_compile_errors,
-            .debug_incremental = options.debug_incremental,
+            .incremental = options.incremental,
             .libcxx_abi_version = options.libcxx_abi_version,
             .root_name = root_name,
             .sysroot = sysroot,
diff --git a/src/Sema.zig b/src/Sema.zig
index f5ee909caf8dd7dc2cc49e780c86831acc817648..36694eef47a9ba18e7da1db5f7f8e131c3bba845 100644
--- a/src/Sema.zig
+++ b/src/Sema.zig
@@ -2726,7 +2726,7 @@ fn maybeRemoveOutdatedType(sema: *Sema, ty: InternPool.Index) !bool {
     const pt = sema.pt;
     const zcu = pt.zcu;
 
-    if (!zcu.comp.debug_incremental) return false;
+    if (!zcu.comp.incremental) return false;
 
     const decl_index = Type.fromInterned(ty).getOwnerDecl(zcu);
     const decl_as_depender = AnalUnit.wrap(.{ .decl = decl_index });
@@ -2826,7 +2826,7 @@ fn zirStructDecl(
     mod.declPtr(new_decl_index).owns_tv = true;
     errdefer pt.abortAnonDecl(new_decl_index);
 
-    if (pt.zcu.comp.debug_incremental) {
+    if (pt.zcu.comp.incremental) {
         try ip.addDependency(
             sema.gpa,
             AnalUnit.wrap(.{ .decl = new_decl_index }),
@@ -3064,7 +3064,7 @@ fn zirEnumDecl(
     new_decl.owns_tv = true;
     errdefer if (!done) pt.abortAnonDecl(new_decl_index);
 
-    if (pt.zcu.comp.debug_incremental) {
+    if (pt.zcu.comp.incremental) {
         try mod.intern_pool.addDependency(
             gpa,
             AnalUnit.wrap(.{ .decl = new_decl_index }),
@@ -3331,7 +3331,7 @@ fn zirUnionDecl(
     mod.declPtr(new_decl_index).owns_tv = true;
     errdefer pt.abortAnonDecl(new_decl_index);
 
-    if (pt.zcu.comp.debug_incremental) {
+    if (pt.zcu.comp.incremental) {
         try mod.intern_pool.addDependency(
             gpa,
             AnalUnit.wrap(.{ .decl = new_decl_index }),
@@ -3421,7 +3421,7 @@ fn zirOpaqueDecl(
     mod.declPtr(new_decl_index).owns_tv = true;
     errdefer pt.abortAnonDecl(new_decl_index);
 
-    if (pt.zcu.comp.debug_incremental) {
+    if (pt.zcu.comp.incremental) {
         try ip.addDependency(
             gpa,
             AnalUnit.wrap(.{ .decl = new_decl_index }),
@@ -38098,7 +38098,7 @@ fn isKnownZigType(sema: *Sema, ref: Air.Inst.Ref, tag: std.builtin.TypeId) bool
 
 pub fn declareDependency(sema: *Sema, dependee: InternPool.Dependee) !void {
     const zcu = sema.pt.zcu;
-    if (!zcu.comp.debug_incremental) return;
+    if (!zcu.comp.incremental) return;
 
     // Avoid creating dependencies on ourselves. This situation can arise when we analyze the fields
     // of a type and they use `@This()`. This dependency would be unnecessary, and in fact would
diff --git a/src/Zcu.zig b/src/Zcu.zig
index 15f418c6fe7f98b326650467f84a85797695a164..36b4c955718f876f8b568f640d68988169fafd73 100644
--- a/src/Zcu.zig
+++ b/src/Zcu.zig
@@ -2679,7 +2679,7 @@ fn markTransitiveDependersPotentiallyOutdated(zcu: *Zcu, maybe_outdated: AnalUni
 }
 
 pub fn findOutdatedToAnalyze(zcu: *Zcu) Allocator.Error!?AnalUnit {
-    if (!zcu.comp.debug_incremental) return null;
+    if (!zcu.comp.incremental) return null;
 
     if (zcu.outdated.count() == 0 and zcu.potentially_outdated.count() == 0) {
         log.debug("findOutdatedToAnalyze: no outdated depender", .{});
diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig
index 73b4e6de5c09c302292f9dc043caf01d6dd50d3e..c3f569cc7d06615c9bcf7b384dddea46e427a970 100644
--- a/src/Zcu/PerThread.zig
+++ b/src/Zcu/PerThread.zig
@@ -888,7 +888,7 @@ fn getFileRootStruct(
     };
     errdefer wip_ty.cancel(ip, pt.tid);
 
-    if (zcu.comp.debug_incremental) {
+    if (zcu.comp.incremental) {
         try ip.addDependency(
             gpa,
             InternPool.AnalUnit.wrap(.{ .decl = decl_index }),
diff --git a/src/main.zig b/src/main.zig
index 06c20d514c61cccd8ed2f6ce4944be89c7183a41..2b36c31865a5886026f26d86eae364503abe3417 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -404,6 +404,8 @@ const usage_build_generic =
     \\  -h, --help                Print this help and exit
     \\  --color [auto|off|on]     Enable or disable colored error messages
     \\  -j<N>                     Limit concurrent jobs (default is to use all CPU cores)
+    \\  -fincremental             Enable incremental compilation
+    \\  -fno-incremental          Disable incremental compilation
     \\  -femit-bin[=path]         (default) Output machine code
     \\  -fno-emit-bin             Do not output machine code
     \\  -femit-asm[=path]         Output .s (assembly code)
@@ -642,7 +644,6 @@ const usage_build_generic =
     \\  --debug-log [scope]          Enable printing debug/info log messages for scope
     \\  --debug-compile-errors       Crash with helpful diagnostics at the first compile error
     \\  --debug-link-snapshot        Enable dumping of the linker's state in JSON format
-    \\  --debug-incremental          Enable experimental feature: incremental compilation
     \\
 ;
 
@@ -904,7 +905,7 @@ fn buildOutputType(
     var minor_subsystem_version: ?u16 = null;
     var mingw_unicode_entry_point: bool = false;
     var enable_link_snapshots: bool = false;
-    var debug_incremental: bool = false;
+    var opt_incremental: ?bool = null;
     var install_name: ?[]const u8 = null;
     var hash_style: link.File.Elf.HashStyle = .both;
     var entitlements: ?[]const u8 = null;
@@ -1357,8 +1358,10 @@ fn buildOutputType(
                         } else {
                             enable_link_snapshots = true;
                         }
-                    } else if (mem.eql(u8, arg, "--debug-incremental")) {
-                        debug_incremental = true;
+                    } else if (mem.eql(u8, arg, "-fincremental")) {
+                        opt_incremental = true;
+                    } else if (mem.eql(u8, arg, "-fno-incremental")) {
+                        opt_incremental = false;
                     } else if (mem.eql(u8, arg, "--entitlements")) {
                         entitlements = args_iter.nextOrFatal();
                     } else if (mem.eql(u8, arg, "-fcompiler-rt")) {
@@ -3225,6 +3228,8 @@ fn buildOutputType(
         break :b .incremental;
     };
 
+    const incremental = opt_incremental orelse false;
+
     process.raiseFileDescriptorLimit();
 
     var file_system_inputs: std.ArrayListUnmanaged(u8) = .{};
@@ -3336,7 +3341,7 @@ fn buildOutputType(
         .cache_mode = cache_mode,
         .subsystem = subsystem,
         .debug_compile_errors = debug_compile_errors,
-        .debug_incremental = debug_incremental,
+        .incremental = incremental,
         .enable_link_snapshots = enable_link_snapshots,
         .install_name = install_name,
         .entitlements = entitlements,
@@ -3443,7 +3448,7 @@ fn buildOutputType(
         updateModule(comp, color, root_prog_node) catch |err| switch (err) {
             error.SemanticAnalyzeFail => {
                 assert(listen == .none);
-                saveState(comp, debug_incremental);
+                saveState(comp, incremental);
                 process.exit(1);
             },
             else => |e| return e,
@@ -3451,7 +3456,7 @@ fn buildOutputType(
     }
     if (build_options.only_c) return cleanExit();
     try comp.makeBinFileExecutable();
-    saveState(comp, debug_incremental);
+    saveState(comp, incremental);
 
     if (test_exec_args.items.len == 0 and target.ofmt == .c) default_exec_args: {
         // Default to using `zig run` to execute the produced .c code from `zig test`.
@@ -4032,8 +4037,8 @@ fn createModule(
     return mod;
 }
 
-fn saveState(comp: *Compilation, debug_incremental: bool) void {
-    if (debug_incremental) {
+fn saveState(comp: *Compilation, incremental: bool) void {
+    if (incremental) {
         comp.saveState() catch |err| {
             warn("unable to save incremental compilation state: {s}", .{@errorName(err)});
         };
-- 
2.54.0


From f33395ce6a66df517c09037c4a4f1756ef7f6f2e Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sun, 14 Jul 2024 21:38:05 -0700
Subject: [PATCH 3/6] std.Progress: add getIpcFd and have_ipc API

This makes advanced use cases possible such as a long-lived child
process whose progress node gets re-attached to a different parent.
---
 lib/std/Progress.zig | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/lib/std/Progress.zig b/lib/std/Progress.zig
index bb5adcefd1e0cc6bdc6f4dea7eab43df09ef2204..190f6678975641180499640f6a689fd3d302eb35 100644
--- a/lib/std/Progress.zig
+++ b/lib/std/Progress.zig
@@ -269,6 +269,19 @@ pub const Node = struct {
         storageByIndex(index).setIpcFd(fd);
     }
 
+    /// Posix-only. Thread-safe. Assumes the node is storing an IPC file
+    /// descriptor.
+    pub fn getIpcFd(node: Node) ?posix.fd_t {
+        const index = node.index.unwrap() orelse return null;
+        const storage = storageByIndex(index);
+        const int = @atomicLoad(u32, &storage.completed_count, .monotonic);
+        return switch (@typeInfo(posix.fd_t)) {
+            .Int => @bitCast(int),
+            .Pointer => @ptrFromInt(int),
+            else => @compileError("unsupported fd_t of " ++ @typeName(posix.fd_t)),
+        };
+    }
+
     fn storageByIndex(index: Node.Index) *Node.Storage {
         return &global_progress.node_storage[@intFromEnum(index)];
     }
@@ -329,6 +342,11 @@ var default_draw_buffer: [4096]u8 = undefined;
 
 var debug_start_trace = std.debug.Trace.init;
 
+pub const have_ipc = switch (builtin.os.tag) {
+    .wasi, .freestanding, .windows => false,
+    else => true,
+};
+
 const noop_impl = builtin.single_threaded or switch (builtin.os.tag) {
     .wasi, .freestanding => true,
     else => false,
-- 
2.54.0


From f6c1b71c220406d60ca4bdf9c949e775f7fca466 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sun, 14 Jul 2024 21:38:55 -0700
Subject: [PATCH 4/6] build system: update std.Progress.Node for long-lived
 children

---
 lib/std/Build/Step.zig | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig
index 397ad6c55f545c72ef95bf92c27066e5d0fd95b1..fc40a680c4fba8c681c0a19a3eb5eb5ea0574630 100644
--- a/lib/std/Build/Step.zig
+++ b/lib/std/Build/Step.zig
@@ -361,6 +361,7 @@ pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutO
 pub const ZigProcess = struct {
     child: std.process.Child,
     poller: std.io.Poller(StreamEnum),
+    progress_ipc_fd: if (std.Progress.have_ipc) ?std.posix.fd_t else void,
 
     pub const StreamEnum = enum { stdout, stderr };
 };
@@ -375,6 +376,7 @@ pub fn evalZigProcess(
 ) !?[]const u8 {
     if (s.getZigProcess()) |zp| {
         assert(watch);
+        if (std.Progress.have_ipc) if (zp.progress_ipc_fd) |fd| prog_node.setIpcFd(fd);
         return zigProcessUpdate(s, zp, watch);
     }
     assert(argv.len != 0);
@@ -404,6 +406,7 @@ pub fn evalZigProcess(
             .stdout = child.stdout.?,
             .stderr = child.stderr.?,
         }),
+        .progress_ipc_fd = if (std.Progress.have_ipc) child.progress_node.getIpcFd() else {},
     };
     if (watch) s.setZigProcess(zp);
     defer if (!watch) zp.poller.deinit();
@@ -435,6 +438,8 @@ pub fn evalZigProcess(
         try handleChildProcessTerm(s, term, null, argv);
     }
 
+    // This is intentionally printed for failure on the first build but not for
+    // subsequent rebuilds.
     if (s.result_error_bundle.errorMessageCount() > 0) {
         return s.fail("the following command failed with {d} compilation errors:\n{s}", .{
             s.result_error_bundle.errorMessageCount(),
-- 
2.54.0


From 987f63208e4bdd044768465f1581d03ee8964e28 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sun, 14 Jul 2024 22:17:29 -0700
Subject: [PATCH 5/6] build runner: handle compiler subprocess failures
 gracefully

Compilation errors now report a failure on rebuilds triggered by file
system watches.

Compiler crashes now report failure correctly on rebuilds triggered by
file system watches.

The compiler subprocess is restarted if a broken pipe is encountered on
a rebuild.
---
 lib/std/Build/Step.zig | 47 +++++++++++++++++++++++++++++++++++++++---
 1 file changed, 44 insertions(+), 3 deletions(-)

diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig
index fc40a680c4fba8c681c0a19a3eb5eb5ea0574630..661ee58509a4d9587a916f8051bde8461313cb8d 100644
--- a/lib/std/Build/Step.zig
+++ b/lib/std/Build/Step.zig
@@ -374,10 +374,37 @@ pub fn evalZigProcess(
     prog_node: std.Progress.Node,
     watch: bool,
 ) !?[]const u8 {
-    if (s.getZigProcess()) |zp| {
+    if (s.getZigProcess()) |zp| update: {
         assert(watch);
         if (std.Progress.have_ipc) if (zp.progress_ipc_fd) |fd| prog_node.setIpcFd(fd);
-        return zigProcessUpdate(s, zp, watch);
+        const result = zigProcessUpdate(s, zp, watch) catch |err| switch (err) {
+            error.BrokenPipe => {
+                // Process restart required.
+                const term = zp.child.wait() catch |e| {
+                    return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(e) });
+                };
+                _ = term;
+                s.clearZigProcess();
+                break :update;
+            },
+            else => |e| return e,
+        };
+
+        if (s.result_error_bundle.errorMessageCount() > 0)
+            return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()});
+
+        if (s.result_error_msgs.items.len > 0 and result == null) {
+            // Crash detected.
+            const term = zp.child.wait() catch |e| {
+                return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(e) });
+            };
+            s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0;
+            s.clearZigProcess();
+            try handleChildProcessTerm(s, term, null, argv);
+            return error.MakeFailed;
+        }
+
+        return result;
     }
     assert(argv.len != 0);
     const b = s.owner;
@@ -399,7 +426,7 @@ pub fn evalZigProcess(
         argv[0], @errorName(err),
     });
 
-    const zp = try arena.create(ZigProcess);
+    const zp = try gpa.create(ZigProcess);
     zp.* = .{
         .child = child,
         .poller = std.io.poll(gpa, ZigProcess.StreamEnum, .{
@@ -590,6 +617,20 @@ fn setZigProcess(s: *Step, zp: *ZigProcess) void {
     }
 }
 
+fn clearZigProcess(s: *Step) void {
+    const gpa = s.owner.allocator;
+    switch (s.id) {
+        .compile => {
+            const compile = s.cast(Compile).?;
+            if (compile.zig_process) |zp| {
+                gpa.destroy(zp);
+                compile.zig_process = null;
+            }
+        },
+        else => unreachable,
+    }
+}
+
 fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void {
     const header: std.zig.Client.Message.Header = .{
         .tag = tag,
-- 
2.54.0


From 445bd7a06fc34c9a59c6458774769bfaa2757a2f Mon Sep 17 00:00:00 2001
From: Andrew Kelley <andrew@ziglang.org>
Date: Sun, 14 Jul 2024 22:27:51 -0700
Subject: [PATCH 6/6] build runner: update watch caption to include
 subprocesses

---
 lib/compiler/build_runner.zig | 13 ++++++++++---
 lib/std/Build.zig             |  2 +-
 lib/std/Build/Step.zig        |  2 +-
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig
index af65d109484f18ef4cb0429eef639c13bbb82cb7..1b13881586ee7b656636573df7e883b8bf0f1dfd 100644
--- a/lib/compiler/build_runner.zig
+++ b/lib/compiler/build_runner.zig
@@ -72,7 +72,6 @@ pub fn main() !void {
             .query = .{},
             .result = try std.zig.system.resolveTargetQuery(.{}),
         },
-        .incremental = null,
     };
 
     graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() });
@@ -411,8 +410,8 @@ pub fn main() !void {
         // trigger a rebuild on all steps with modified inputs, as well as their
         // recursive dependants.
         var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined;
-        const caption = std.fmt.bufPrint(&caption_buf, "Watching {d} Directories", .{
-            w.dir_table.entries.len,
+        const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{
+            w.dir_table.entries.len, countSubProcesses(run.step_stack.keys()),
         }) catch &caption_buf;
         var debouncing_node = main_progress_node.start(caption, 0);
         var debounce_timeout: Watch.Timeout = .none;
@@ -445,6 +444,14 @@ fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void {
     };
 }
 
+fn countSubProcesses(all_steps: []const *Step) usize {
+    var count: usize = 0;
+    for (all_steps) |s| {
+        count += @intFromBool(s.getZigProcess() != null);
+    }
+    return count;
+}
+
 const Run = struct {
     max_rss: u64,
     max_rss_is_default: bool,
diff --git a/lib/std/Build.zig b/lib/std/Build.zig
index 29621f95c93e50f56a16d6134c7360cfcceccd46..06de7aa6f45e66f3b6b36ff47d524faf47c88463 100644
--- a/lib/std/Build.zig
+++ b/lib/std/Build.zig
@@ -120,7 +120,7 @@ pub const Graph = struct {
     needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{},
     /// Information about the native target. Computed before build() is invoked.
     host: ResolvedTarget,
-    incremental: ?bool,
+    incremental: ?bool = null,
 };
 
 const AvailableDeps = []const struct { []const u8, []const u8 };
diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig
index 661ee58509a4d9587a916f8051bde8461313cb8d..8f3236d867a5b8031b3700b44d1145f486101615 100644
--- a/lib/std/Build/Step.zig
+++ b/lib/std/Build/Step.zig
@@ -603,7 +603,7 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {
     return result;
 }
 
-fn getZigProcess(s: *Step) ?*ZigProcess {
+pub fn getZigProcess(s: *Step) ?*ZigProcess {
     return switch (s.id) {
         .compile => s.cast(Compile).?.zig_process,
         else => null,
-- 
2.54.0