From 4853ea1bda1131d4d57a50eaa8209d90b71c8537 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 24 Sep 2026 18:06:21 -0700 Subject: [PATCH 1/5] Maker: store generated files differently in preparation for sending them over the build server protocol (#36403), they are now stored in the same format in memory: one byte for prefix followed by the relative path. --- lib/compiler/Maker.zig | 131 ++++++++++++++++++-- lib/compiler/Maker/Step.zig | 23 ++-- lib/compiler/Maker/Step/Compile.zig | 43 ++++--- lib/compiler/Maker/Step/ConfigHeader.zig | 22 +--- lib/compiler/Maker/Step/FindProgram.zig | 2 +- lib/compiler/Maker/Step/InstallArtifact.zig | 2 +- lib/compiler/Maker/Step/ObjCopy.zig | 27 ++-- lib/compiler/Maker/Step/Options.zig | 15 +-- lib/compiler/Maker/Step/Run.zig | 69 ++++------- lib/compiler/Maker/Step/TranslateC.zig | 7 +- lib/compiler/Maker/Step/WriteFile.zig | 24 +--- 11 files changed, 209 insertions(+), 156 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index 681a91341b99cdaa0a837d783f974924a036f803..aaa6498c001a1eb1305cfa6466d94b09825cde2f 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -49,7 +49,13 @@ scanned_config: *const ScannedConfig, /// system watching does not need to make any special cases, and to avoid more /// OS-specific logic in file system watching implementation. steps: []Step, -generated_files: []Path, +/// Reserved space for all the file paths that are determined at runtime when the associated Step logic runs. +/// Each element is individually allocated so that steps can access the data without synchronization, and so +/// that steps can be arbitrarily re-run independently from each other. +/// Each element is optional because a step might populate any number of generated file paths before ultimately +/// failing. An element being non-null does not imply that the generated file was successfully generated. Only +/// the Step success state indicates that information. +generated_files: []GeneratedFile, run_args: ?[]const []const u8, available_rss: u64, @@ -121,6 +127,38 @@ const MultilineErrors = enum { indent, newline, none }; const Summary = enum { all, new, failures, line, none }; const PrintConfiguration = enum { none, zon, path }; +const GeneratedFile = struct { + data: ?[]const u8, + + const Unpacked = struct { + prefix: Server.Message.PathPrefix, + sub_path: []const u8, + }; + + fn unpack(gf: GeneratedFile) ?Unpacked { + const slice = gf.data orelse return null; + return .{ + .prefix = @fromBackingInt(slice[0]), + .sub_path = slice[1..], + }; + } + + fn toPath(gf: GeneratedFile, graph: *const Graph) ?Path { + const unpacked = unpack(gf) orelse return null; + return .{ + .root_dir = prefixToDirectory(graph, unpacked.prefix), + .sub_path = unpacked.sub_path, + }; + } + + fn clear(gf: *GeneratedFile, gpa: Allocator) void { + if (gf.data) |data| { + gpa.free(data); + gf.data = null; + } + } +}; + /// Used to build the -M flags to pass to build-exe. pub const CliModule = struct { name: []const u8, @@ -2344,7 +2382,8 @@ fn prepare( // Extra step at the end which is the autogenerated placeholder // step which indicates that we need to reconfigure. maker.steps = try arena.alloc(Step, c.steps.len + 1); - maker.generated_files = try arena.alloc(Path, c.generated_files_len); + maker.generated_files = try arena.alloc(GeneratedFile, c.generated_files_len); + @memset(maker.generated_files, .{ .data = null }); // The last element is a reserved special pseudostep which contains the // watch inputs for the configurer executable. @@ -2676,6 +2715,7 @@ fn deinit(maker: *Maker) void { const gpa = maker.gpa; const io = maker.graph.io; for (maker.steps) |*step| step.deinit(gpa, io); + for (maker.generated_files) |*gf| gf.clear(gpa); maker.memory_blocked_steps.deinit(gpa); maker.initial_steps.deinit(gpa); maker.step_stack.deinit(gpa); @@ -2787,7 +2827,8 @@ fn makeStep( if (maker.web_server) |ws| { ws.updateStepStatus(step_index, if (success) .success else .failure); } - if (maker.protocol_server != null) { + if (maker.protocol_server) |s| { + // https://codeberg.org/ziglang/zig/issues/36938 maker.protocol_server_mutex.lockUncancelable(io); defer maker.protocol_server_mutex.unlock(io); @@ -2800,11 +2841,8 @@ fn makeStep( .dependency_skipped, .skipped => .skipped, .skipped_oom => .skipped_oom, }; - serveBuildStepCompleted( - maker, - step_index, - status, - ) catch |err| std.debug.panic("TODO propagate error when failing to send protocol message: {t}", .{err}); + serveBuildStepCompleted(maker, step_index, status, s) catch |err| + std.debug.panic("TODO propagate error when failing to send protocol message: {t}", .{err}); } if (!success) std.Progress.setStatus(.failure_working); @@ -3366,7 +3404,7 @@ fn cleanTmpFiles(maker: *Maker, steps: []const Configuration.Step.Index) void { if (wf.flags.mode != .tmp) continue; const step = maker.stepByIndex(step_index); if (step.state != .success) continue; - const tmp_path = generatedPath(maker, wf.generated_directory).*; + const tmp_path = generatedPath(maker, wf.generated_directory); tmp_path.root_dir.handle.deleteTree(io, tmp_path.subPathOrDot()) catch |err| log.warn("failed to delete temporary path {f}: {t}", .{ tmp_path, err }); } @@ -3391,8 +3429,8 @@ fn serveBuildStepCompleted( maker: *Maker, step_index: Configuration.Step.Index, status: Server.Message.BuildStepCompleted.Status, + s: *Server, ) !void { - const s: *Server = maker.protocol_server.?; const step = maker.stepByIndex(step_index); const error_bundle = step.result_error_bundle; @@ -3434,7 +3472,7 @@ pub fn resolveLazyPath( .source_path => |sp| try packagePath(maker, arena, sp.owner, sp.sub_path.slice(c)), .relative => |relative| relativePath(maker, arena, relative), .generated => |gen| { - const base = generatedPath(maker, gen.index).*; + const base = generatedPath(maker, gen.index); var file_path = base; for (0..gen.flags.up) |_| { file_path.sub_path = Dir.path.dirname(file_path.sub_path) orelse { @@ -3485,10 +3523,67 @@ pub fn resolveLazyPathIndexAbs( return resolveLazyPathAbs(maker, arena, lazy_path_index.get(c), asking_step_index); } -pub fn generatedPath(maker: *const Maker, index: Configuration.GeneratedFileIndex) *Path { +pub fn generatedFile(maker: *const Maker, index: Configuration.GeneratedFileIndex) *GeneratedFile { return &maker.generated_files[@backingInt(index)]; } +pub fn generatedPath(maker: *const Maker, index: Configuration.GeneratedFileIndex) Path { + return generatedFile(maker, index).toPath(maker.graph).?; +} + +pub fn setGeneratedPath( + maker: *Maker, + index: Configuration.GeneratedFileIndex, + prefix: Server.Message.PathPrefix, + sub_paths: []const []const u8, +) Allocator.Error!Path { + const gpa = maker.gpa; + const graph = maker.graph; + assert(sub_paths.len != 0); + + var buffer: std.ArrayList(u8) = .empty; + defer buffer.deinit(gpa); + try buffer.append(gpa, @backingInt(prefix)); + for (sub_paths) |sub_path| { + try buffer.ensureUnusedCapacity(gpa, sub_path.len + 1); + buffer.appendSliceAssumeCapacity(sub_path); + while (Dir.path.isSep(buffer.items[buffer.items.len - 1])) + buffer.items.len -= 1; + buffer.appendAssumeCapacity(Dir.path.sep); + } + assert(buffer.pop().? == Dir.path.sep); + try buffer.shrinkToLen(gpa); + + const sub_path = buffer.toOwnedSliceAssert(); + const ptr = generatedFile(maker, index); + ptr.clear(gpa); + ptr.* = .{ .data = sub_path }; + return .{ + .root_dir = prefixToDirectory(graph, prefix), + .sub_path = sub_path[1..], + }; +} + +/// Prefer to use `setGeneratedPath` where possible to keep parent directory information intact. +pub fn setGeneratedPathPath( + maker: *Maker, + index: Configuration.GeneratedFileIndex, + path: Path, +) Allocator.Error!Path { + const graph = maker.graph; + if (path.root_dir.eql(graph.local_cache_root)) { + return setGeneratedPath(maker, index, .local_cache, &.{path.sub_path}); + } else if (path.root_dir.eql(graph.global_cache_root)) { + return setGeneratedPath(maker, index, .global_cache, &.{path.sub_path}); + } else if (path.root_dir.eql(graph.zig_lib_directory)) { + return setGeneratedPath(maker, index, .zig_lib, &.{path.sub_path}); + } else if (path.root_dir.eql(graph.build_root_directory)) { + return setGeneratedPath(maker, index, .build_root, &.{path.sub_path}); + } else { + return setGeneratedPath(maker, index, .cwd, &.{path.sub_path}); + } +} + pub fn packagePath( maker: *const Maker, arena: Allocator, @@ -3600,7 +3695,7 @@ pub fn installGenerated( dest_dir: Configuration.InstallDestDir, asking_step_index: Configuration.Step.Index, ) !Dir.PrevStatus { - const src_path = generatedPath(maker, source).*; + const src_path = generatedPath(maker, source); const dest_dir_path = try resolveInstallDir(maker, arena, dest_dir); const dest_path = try dest_dir_path.join(arena, src_path.basename()); return installPath(maker, arena, src_path, dest_path, asking_step_index); @@ -4170,3 +4265,13 @@ fn nextEnumArg(args: []const []const u8, i: *usize, comptime E: type) E { const next_arg = nextArg(args, i) orelse fatalEnumHint(E, arg, null); return stringToEnum(E, next_arg) orelse fatalEnumHint(E, arg, next_arg); } + +fn prefixToDirectory(graph: *const Graph, prefix: Server.Message.PathPrefix) Cache.Directory { + return switch (prefix) { + .cwd => .cwd(), + .zig_lib => graph.zig_lib_directory, + .local_cache => graph.local_cache_root, + .global_cache => graph.global_cache_root, + .build_root => graph.build_root_directory, + }; +} diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index fc1b7f5b324659d8381a7bcbe9d1be7b567cea41..f52640d0a490cb148467672db759d1051765a12a 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -460,6 +460,15 @@ pub const ZigProcess = struct { } }; +pub const OptCacheDigest = struct { + bin: ?Cache.BinDigest, + + pub fn toHex(ocd: *const OptCacheDigest) ?Cache.HexDigest { + const bin = ocd.bin orelse return null; + return Cache.binToHex(bin); + } +}; + /// Assumes that argv contains `--listen=-` and that the process being spawned /// is the zig compiler - the same version that compiled the build runner. /// @@ -470,7 +479,7 @@ pub fn evalZigProcess( argv: []const []const u8, prog_node: std.Progress.Node, watch: bool, -) (Step.ExtendedMakeError || error{NeedCompileErrorCheck})!?Path { +) (Step.ExtendedMakeError || error{NeedCompileErrorCheck})!OptCacheDigest { const s = maker.stepByIndex(step_index); const gpa = maker.gpa; const graph = maker.graph; @@ -503,7 +512,7 @@ pub fn evalZigProcess( if (s.result_error_bundle.errorMessageCount() > 0) return s.fail(maker, "{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); - if (s.result_error_msgs.items.len > 0 and result == null) { + if (s.result_error_msgs.items.len > 0 and result.bin == null) { // Crash detected. const term = zp.child.wait(io) catch |e| { return s.fail(maker, "unable to wait for {s}: {t}", .{ argv[0], e }); @@ -578,7 +587,7 @@ pub fn evalZigProcess( return result; } -fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *ZigProcess, watch: bool) !?Path { +fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *ZigProcess, watch: bool) !OptCacheDigest { const s = maker.stepByIndex(step_index); const gpa = maker.gpa; const graph = maker.graph; @@ -590,7 +599,7 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *Zi try sendMessage(io, zp.child.stdin.?, .update); if (!watch) try sendMessage(io, zp.child.stdin.?, .exit); - var result: ?Path = null; + var result: OptCacheDigest = .{ .bin = null }; var eos_err: error{EndOfStream}!void = {}; var client: std.zig.Client = .{ @@ -632,11 +641,7 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *Zi const EmitDigest = std.zig.Server.Message.EmitDigest; const emit_digest: *align(1) const EmitDigest = @ptrCast(body); s.result_cached = emit_digest.flags.cache_hit; - const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; - result = .{ - .root_dir = graph.local_cache_root, - .sub_path = try arena.dupe(u8, "o" ++ Dir.path.sep_str ++ Cache.binToHex(digest.*)), - }; + result = .{ .bin = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len].* }; }, .file_system_inputs => { clearWatchInputs(s, maker); diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 2e02f3407b6bfbbc4034eb6570732fe26b91271a..d16b1afcdd7be88b7f1fcbff095c53bf204ef94c 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -46,7 +46,7 @@ pub fn make( const incremental = conf_comp.flags4.incremental.toBool() orelse graph.incremental == true; - const maybe_output_dir = Step.evalZigProcess( + const opt_cache_digest = Step.evalZigProcess( compile_index, maker, argv.items, @@ -63,24 +63,25 @@ pub fn make( const root_module = conf_comp.root_module.get(conf); const target = root_module.resolved_target.get(conf).?.result.get(conf); - // Update generated files - if (maybe_output_dir) |output_dir| { - if (conf_comp.emit_directory.value) |gf| maker.generatedPath(gf).* = output_dir; - try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_bin.value, .bin); - try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_pdb.value, .pdb); - try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_implib.value, .implib); - try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_h.value, .h); - try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_docs.value, .docs); - try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_asm.value, .@"asm"); - try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_llvm_ir.value, .llvm_ir); - try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_llvm_bc.value, .llvm_bc); + if (opt_cache_digest.toHex()) |o_hex_digest| { + if (conf_comp.emit_directory.value) |gf| { + _ = try maker.setGeneratedPath(gf, .local_cache, &.{ "o", &o_hex_digest }); + } + try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_bin.value, .bin); + try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_pdb.value, .pdb); + try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_implib.value, .implib); + try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_h.value, .h); + try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_docs.value, .docs); + try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_asm.value, .@"asm"); + try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_llvm_ir.value, .llvm_ir); + try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_llvm_bc.value, .llvm_bc); } if (conf_comp.flags3.kind == .lib and conf_comp.flags2.linkage == .dynamic and conf_comp.version.value != null and target.flags.os_tag != .windows) { if (conf_comp.generated_bin.value) |generated_bin| { - const full_dest_path = maker.generatedPath(generated_bin).*; + const full_dest_path = maker.generatedPath(generated_bin); try maker.installSymLinks(arena, full_dest_path, compile_index, compile_index); } } @@ -97,13 +98,12 @@ fn updateGeneratedFile( maker: *Maker, arena: Allocator, conf_comp: *const Configuration.Step.Compile, - out_path: std.Build.Cache.Path, + o_hex_digest: *const std.Build.Cache.HexDigest, target: *const Configuration.TargetQuery, opt_gf: ?Configuration.GeneratedFileIndex, ea: std.zig.EmitArtifact, ) Allocator.Error!void { const gf = opt_gf orelse return; - const graph = maker.graph; const conf = &maker.scanned_config.configuration; const name = try ea.cacheName(arena, .{ .root_name = conf_comp.root_name.slice(conf), @@ -122,7 +122,7 @@ fn updateGeneratedFile( else null, }); - maker.generatedPath(gf).* = try out_path.join(graph.arena, name); + _ = try maker.setGeneratedPath(gf, .local_cache, &.{ "o", o_hex_digest, name }); } /// List of importable modules in a compilation's module graph, including @@ -945,6 +945,7 @@ pub fn rebuildInFuzzMode( progress_node: std.Progress.Node, ) !Path { const gpa = maker.gpa; + const graph = maker.graph; const step = maker.stepByIndex(compile_index); var arena_allocator: std.heap.ArenaAllocator = .init(gpa); @@ -963,8 +964,12 @@ pub fn rebuildInFuzzMode( defer argv.deinit(gpa); try lowerZigArgs(arena, compile, compile_index, maker, progress_node, &argv, true); - const maybe_output_bin_path = try Step.evalZigProcess(compile_index, maker, argv.items, progress_node, false); - return maybe_output_bin_path.?; + const opt_cache_digest = try Step.evalZigProcess(compile_index, maker, argv.items, progress_node, false); + const o_hex_digest = opt_cache_digest.toHex().?; + return .{ + .root_dir = graph.local_cache_root, + .sub_path = try Dir.path.join(arena, &.{ "o", &o_hex_digest }), + }; } fn addBool(gpa: Allocator, args: *std.ArrayList([]const u8), arg: []const u8, opt: bool) !void { @@ -1380,7 +1385,7 @@ pub fn appendIncludeDirFlags( }, .config_header_step => |ch_index| { const conf_ch = ch_index.ptr(conf).extended.get(conf.extra).config_header; - const path = maker.generatedPath(conf_ch.generated_dir).*; + const path = maker.generatedPath(conf_ch.generated_dir); zig_args.appendAssumeCapacity("-I"); zig_args.appendAssumeCapacity(try path.toString(arena)); }, diff --git a/lib/compiler/Maker/Step/ConfigHeader.zig b/lib/compiler/Maker/Step/ConfigHeader.zig index a539e2463128445541037d2feab024faa673681f..be8cdcc2f7b3dca25c9721a2f2d4bcc3069ce796 100644 --- a/lib/compiler/Maker/Step/ConfigHeader.zig +++ b/lib/compiler/Maker/Step/ConfigHeader.zig @@ -34,7 +34,6 @@ pub fn make( const conf = &maker.scanned_config.configuration; const conf_step = step_index.ptr(conf); const conf_ch = conf_step.extended.get(conf.extra).config_header; - const cache_root = graph.local_cache_root; const input_size_limit: Io.Limit = if (conf_ch.input_size_limit.value) |x| .limited64(x) else .unlimited; const include_guard_override: ?[]const u8 = if (conf_ch.include_guard.value) |s| s.slice(conf) else null; @@ -138,10 +137,7 @@ pub fn make( if (try step.cacheHit(maker, &man, progress_node)) { const digest = man.hitDigestHex(); - maker.generatedPath(conf_ch.generated_dir).* = .{ - .root_dir = cache_root, - .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), - }; + _ = try maker.setGeneratedPath(conf_ch.generated_dir, .local_cache, &.{ "o", &digest }); return; } @@ -152,22 +148,16 @@ pub fn make( // output_path is libavutil/avconfig.h // We want to open directory zig-cache/o/HASH/libavutil/ // but keep output_dir as zig-cache/o/HASH for -I include - const out_path: Path = .{ - .root_dir = cache_root, - .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, conf_ch.include_path.slice(conf) }), - }; + + const artifact_path = try maker.setGeneratedPath(conf_ch.generated_dir, .local_cache, &.{ "o", &digest }); + const out_path = try artifact_path.join(arena, conf_ch.include_path.slice(conf)); const out_path_dirname = out_path.dirname().?; out_path_dirname.root_dir.handle.createDirPath(io, out_path_dirname.sub_path) catch |err| - return step.fail(maker, "unable to make path {f}: {t}", .{ out_path_dirname, err }); + return step.fail(maker, "failed to make path {qf}: {t}", .{ out_path_dirname, err }); out_path.root_dir.handle.writeFile(io, .{ .sub_path = out_path.sub_path, .data = output }) catch |err| - return step.fail(maker, "unable to write file {f}: {t}", .{ out_path, err }); - - maker.generatedPath(conf_ch.generated_dir).* = .{ - .root_dir = cache_root, - .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), - }; + return step.fail(maker, "failed to write file {qf}: {t}", .{ out_path, err }); try step.finalizeManifest(maker, &man); } diff --git a/lib/compiler/Maker/Step/FindProgram.zig b/lib/compiler/Maker/Step/FindProgram.zig index 48eb678223883c8c863506ba083680d7cce13b1a..e0fece763899e1afa23f187099a4a1f7928032e3 100644 --- a/lib/compiler/Maker/Step/FindProgram.zig +++ b/lib/compiler/Maker/Step/FindProgram.zig @@ -77,7 +77,7 @@ fn checkCandidate( const io = graph.io; if (Io.Dir.cwd().access(io, full_path, .{ .execute = true })) |_| { - maker.generatedPath(found_path).* = .initCwd(full_path); + _ = try maker.setGeneratedPath(found_path, .cwd, &.{full_path}); return true; } else |err| switch (err) { error.Canceled => |e| return e, diff --git a/lib/compiler/Maker/Step/InstallArtifact.zig b/lib/compiler/Maker/Step/InstallArtifact.zig index 6f90157c38683adc7512a643f813e8ef1dc9d315..5caa3a92711c4d868bcac046d71b8df81b704a9e 100644 --- a/lib/compiler/Maker/Step/InstallArtifact.zig +++ b/lib/compiler/Maker/Step/InstallArtifact.zig @@ -50,7 +50,7 @@ pub fn make( }); const dest_dir = try maker.resolveInstallDir(arena, bin_dir); const dest_path = try dest_dir.join(arena, bin_sub_path); - const src_path = maker.generatedPath(generated_bin).*; + const src_path = maker.generatedPath(generated_bin); const p = try maker.installPath(arena, src_path, dest_path, step_index); all_cached = all_cached and p == .fresh; diff --git a/lib/compiler/Maker/Step/ObjCopy.zig b/lib/compiler/Maker/Step/ObjCopy.zig index 52e3f9c7d4f20e0dc9dcd6623b57e647b8a26ca2..a46c207910cdcab844ef805e1773af10a123970c 100644 --- a/lib/compiler/Maker/Step/ObjCopy.zig +++ b/lib/compiler/Maker/Step/ObjCopy.zig @@ -22,7 +22,6 @@ pub fn make( const conf = &maker.scanned_config.configuration; const conf_step = step_index.ptr(conf); const conf_oc = conf_step.extended.get(conf.extra).obj_copy; - const cache_root = graph.local_cache_root; const input_lazy_path = conf_oc.input_file.get(conf); const only_section: ?[]const u8 = if (conf_oc.only_section.value) |s| s.slice(conf) else null; const opt_basename: ?[]const u8 = if (conf_oc.basename.value) |s| s.slice(conf) else null; @@ -49,18 +48,12 @@ pub fn make( if (try step.cacheHit(maker, &man, progress_node)) { // Cache hit, skip subprocess execution. const digest = man.hitDigestHex(); - maker.generatedPath(conf_oc.output_file).* = .{ - .root_dir = cache_root, - .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), - }; + _ = try maker.setGeneratedPath(conf_oc.output_file, .local_cache, &.{ "o", &digest, basename }); if (conf_oc.debug_file.value) |debug_file| { const debug_basename = opt_debug_basename orelse try arena.print("{s}.debug", .{ Io.Dir.path.basename(input_path.sub_path), }); - maker.generatedPath(debug_file).* = .{ - .root_dir = cache_root, - .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, debug_basename }), - }; + _ = try maker.setGeneratedPath(debug_file, .local_cache, &.{ "o", &digest, debug_basename }); } return; } @@ -68,10 +61,8 @@ pub fn make( // We don't find out more input files while executing objcopy so we can // already obtain the digest and use it directly as the output path. const digest = man.missDigestHex(); - const dest_path: Path = .{ - .root_dir = cache_root, - .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), - }; + const dest_path = try maker.setGeneratedPath(conf_oc.output_file, .local_cache, &.{ "o", &digest, basename }); + const dest_dirname = dest_path.dirname().?; dest_dirname.root_dir.handle.createDirPath(io, dest_dirname.sub_path) catch |err| return step.fail(maker, "failed to create path {f}: {t}", .{ dest_dirname, err }); @@ -107,12 +98,10 @@ pub fn make( const debug_basename = opt_debug_basename orelse try arena.print("{s}.debug", .{ Io.Dir.path.basename(input_path.sub_path), }); - const debug_dest_path: Path = .{ - .root_dir = cache_root, - .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, debug_basename }), - }; + const debug_dest_path = try maker.setGeneratedPath(debug_file, .local_cache, &.{ + "o", &digest, debug_basename, + }); argv.appendAssumeCapacity(try arena.print("--extract-to={f}", .{debug_dest_path})); - maker.generatedPath(debug_file).* = debug_dest_path; } try argv.ensureUnusedCapacity(arena, conf_oc.add_section.slice.len * 2); @@ -163,8 +152,6 @@ pub fn make( else => |e| return e, }; - maker.generatedPath(conf_oc.output_file).* = dest_path; - step.finalizeManifest(maker, &man) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| try step.addError(maker, "failed writing cache manifest: {t}", .{e}), diff --git a/lib/compiler/Maker/Step/Options.zig b/lib/compiler/Maker/Step/Options.zig index 26ac62db41b7390e30e454738ce8a326abf06bf4..1b71707bef4bdd9e4cad0baaf992cb6137e6191a 100644 --- a/lib/compiler/Maker/Step/Options.zig +++ b/lib/compiler/Maker/Step/Options.zig @@ -19,7 +19,6 @@ pub fn make( const graph = maker.graph; const step = maker.stepByIndex(step_index); const io = graph.io; - const cache_root = graph.local_cache_root; const arena = graph.arena; // TODO don't leak into the process arena const conf = &maker.scanned_config.configuration; const conf_step = step_index.ptr(conf); @@ -80,19 +79,15 @@ pub fn make( if (try step.cacheHitWatched(maker, &man, progress_node)) { const digest = man.hitDigestHex(); - maker.generatedPath(conf_options.generated_file).* = .{ - .root_dir = cache_root, - .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), - }; + _ = try maker.setGeneratedPath(conf_options.generated_file, .local_cache, &.{ "o", &digest, basename }); step.result_cached = true; return; } const digest = man.missDigestHex(); - const out_path: Cache.Path = .{ - .root_dir = cache_root, - .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), - }; + const out_path = try maker.setGeneratedPath(conf_options.generated_file, .local_cache, &.{ + "o", &digest, basename, + }); var file: Io.File = out_path.root_dir.handle.createFile(io, out_path.sub_path, .{}) catch |err| switch (err) { error.Canceled => |e| return e, @@ -121,6 +116,4 @@ pub fn make( }; try step.finalizeManifestAndWatch(maker, &man); - - maker.generatedPath(conf_options.generated_file).* = out_path; } diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 7421d579ad8129e56515e90c16fdf7d1d9593e14..5b0e70a1413e0ed321c9ae41f8a6792a91811e92 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -146,7 +146,8 @@ pub fn make( const producer_make_comp_step = maker.stepByIndex(producer_index); const producer_make_comp = &producer_make_comp_step.extended.compile; - const file_path = producer_make_comp.installed_path orelse maker.generatedPath(producer.generated_bin.value.?).*; + const file_path = producer_make_comp.installed_path orelse + maker.generatedPath(producer.generated_bin.value.?); argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{ prefix, try convertPathArg(arena, run_index, maker, file_path, arg.flags.make_absolute), suffix, @@ -262,8 +263,8 @@ pub fn make( if (!has_side_effects and try step.cacheHitWatched(maker, &man, progress_node)) { // Cache hit; skip running command. const digest = man.hitDigestHex(); - try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); - try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest); + try populateGeneratedStdIo(maker, &conf_run, &digest); + try populateGeneratedPaths(maker, output_placeholders.items, &digest); step.result_cached = true; return; } @@ -272,7 +273,7 @@ pub fn make( // We already know the final output paths; use them directly. const digest = if (has_side_effects) man.hash.final() else man.missDigestHex(); const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest; - try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); + try populateGeneratedStdIo(maker, &conf_run, &digest); try populateGeneratedPathsCreateDirs(arena, run_index, maker, output_dir_path, output_placeholders.items, argv_list.items); try runCommand(arena, run, run_index, maker, progress_node, argv_list.items, has_side_effects, output_dir_path, null); if (!has_side_effects) try step.finalizeManifestAndWatch(maker, &man); @@ -291,7 +292,7 @@ pub fn make( const arg = placeholder.arg_index.get(conf); switch (arg.flags.tag) { .output_file => if (arg.flags.dep_file) { - const generated_path = maker.generatedPath(arg.generated.value.?).*; + const generated_path = maker.generatedPath(arg.generated.value.?); if (has_side_effects) { var diagnostic: Cache.DepTokenizer.Token = undefined; man.addInputDepFile(generated_path, &diagnostic) catch |err| switch (err) { @@ -362,8 +363,8 @@ pub fn make( if (!has_side_effects) try step.finalizeManifestAndWatch(maker, &man); - try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest); - try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest); + try populateGeneratedStdIo(maker, &conf_run, &digest); + try populateGeneratedPaths(maker, output_placeholders.items, &digest); // The utility functions that spawn the child process must unconditionally allocate // the failed command because at that point it is not known whether the step will @@ -1575,8 +1576,7 @@ pub fn rerunInFuzzMode( const file_path: Path = if (producer_index == conf_run.producer.value.?) run.rebuilt_executable.? else - producer_make_comp.installed_path orelse - maker.generatedPath(producer.generated_bin.value.?).*; + producer_make_comp.installed_path orelse maker.generatedPath(producer.generated_bin.value.?); argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{ prefix, try convertPathArg(arena, run_index, maker, file_path, arg.flags.make_absolute), suffix, })); @@ -1615,20 +1615,15 @@ pub fn rerunInFuzzMode( fn populateGeneratedPaths( maker: *Maker, output_placeholders: []const IndexedOutput, - cache_root: Cache.Directory, digest: *const Cache.HexDigest, ) !void { const conf = &maker.scanned_config.configuration; - const graph = maker.graph; for (output_placeholders) |placeholder| { const arg = placeholder.arg_index.get(conf); - maker.generatedPath(arg.generated.value.?).* = .{ - .root_dir = cache_root, - .sub_path = try Dir.path.join(graph.arena, &.{ - "o", digest, arg.basename.value.?.slice(conf), - }), - }; + _ = try maker.setGeneratedPath(arg.generated.value.?, .local_cache, &.{ + "o", digest, arg.basename.value.?.slice(conf), + }); } } @@ -1652,10 +1647,9 @@ fn populateGeneratedPathsCreateDirs( const suffix = if (arg.suffix.value) |p| p.slice(conf) else ""; const basename = arg.basename.value.?.slice(conf); - const generated_path: Path = .{ - .root_dir = cache_root, - .sub_path = try Dir.path.join(graph.arena, &.{ output_dir_path, basename }), - }; + const generated_path = try maker.setGeneratedPath(arg.generated.value.?, .local_cache, &.{ + output_dir_path, basename, + }); const create_path: Path = .{ .root_dir = cache_root, .sub_path = switch (arg.flags.tag) { @@ -1667,8 +1661,6 @@ fn populateGeneratedPathsCreateDirs( create_path.root_dir.handle.createDirPath(io, create_path.sub_path) catch |err| return step.fail(maker, "unable to make path {f}: {t}", .{ create_path, err }); - maker.generatedPath(arg.generated.value.?).* = generated_path; - const arg_output_path = try convertPathArg(arena, run_index, maker, generated_path, arg.flags.make_absolute); argv[placeholder.index] = try mem.concat(arena, u8, &.{ prefix, arg_output_path, suffix }); } @@ -1677,28 +1669,20 @@ fn populateGeneratedPathsCreateDirs( fn populateGeneratedStdIo( maker: *Maker, conf_run: *const Configuration.Step.Run, - cache_root: Cache.Directory, digest: *const Cache.HexDigest, ) !void { const conf = &maker.scanned_config.configuration; - const graph = maker.graph; if (conf_run.captured_stdout.value) |captured| { - maker.generatedPath(captured.generated_file).* = .{ - .root_dir = cache_root, - .sub_path = try Dir.path.join(graph.arena, &.{ - "o", digest, captured.basename.slice(conf), - }), - }; + _ = try maker.setGeneratedPath(captured.generated_file, .local_cache, &.{ + "o", digest, captured.basename.slice(conf), + }); } if (conf_run.captured_stderr.value) |captured| { - maker.generatedPath(captured.generated_file).* = .{ - .root_dir = cache_root, - .sub_path = try Dir.path.join(graph.arena, &.{ - "o", digest, captured.basename.slice(conf), - }), - }; + _ = try maker.setGeneratedPath(captured.generated_file, .local_cache, &.{ + "o", digest, captured.basename.slice(conf), + }); } } @@ -1735,7 +1719,6 @@ fn runCommand( const gpa = maker.gpa; const step = maker.stepByIndex(run_index); const io = graph.io; - const cache_root = graph.local_cache_root; const conf = &maker.scanned_config.configuration; const conf_step = run_index.ptr(conf); const conf_run = conf_step.extended.get(conf.extra).run; @@ -1989,13 +1972,9 @@ fn runCommand( }, }) |*stream| { if (stream.captured) |captured| { - const output_path: Path = .{ - .root_dir = cache_root, - .sub_path = try Dir.path.join(graph.arena, &.{ - output_dir_path, captured.basename.slice(conf), - }), - }; - maker.generatedPath(captured.generated_file).* = output_path; + const output_path = try maker.setGeneratedPath(captured.generated_file, .local_cache, &.{ + output_dir_path, captured.basename.slice(conf), + }); const sub_path_parent = output_path.dirname().?; sub_path_parent.root_dir.handle.createDirPath(io, sub_path_parent.sub_path) catch |err| diff --git a/lib/compiler/Maker/Step/TranslateC.zig b/lib/compiler/Maker/Step/TranslateC.zig index fa3f2189a11e23781d559f1175cd57f1e5cfc7aa..c645cca1ac8e23ea13d9d74d33746efe4bfe7aac 100644 --- a/lib/compiler/Maker/Step/TranslateC.zig +++ b/lib/compiler/Maker/Step/TranslateC.zig @@ -143,13 +143,14 @@ pub fn make( argv.appendAssumeCapacity(c_source_path); argv.appendAssumeCapacity("--listen=-"); - const output_dir_path = (Step.evalZigProcess(step_index, maker, argv.items, progress_node, false) catch |err| switch (err) { + const opt_cache_digest = Step.evalZigProcess(step_index, maker, argv.items, progress_node, false) catch |err| switch (err) { error.NeedCompileErrorCheck => unreachable, else => |e| return e, - }).?; + }; + const o_hex_digest = opt_cache_digest.toHex().?; const stem = Io.Dir.path.stem(Io.Dir.path.basename(c_source_path)); const out_basename = try arena.print("{s}.zig", .{stem}); - maker.generatedPath(conf_tc.output_file).* = try output_dir_path.join(arena, out_basename); + _ = try maker.setGeneratedPath(conf_tc.output_file, .local_cache, &.{ "o", &o_hex_digest, out_basename }); } diff --git a/lib/compiler/Maker/Step/WriteFile.zig b/lib/compiler/Maker/Step/WriteFile.zig index 6b142a02338315221f403397db090178b096ab1a..721e58917f2375b341f2f3b71b2e51a2c521e5c5 100644 --- a/lib/compiler/Maker/Step/WriteFile.zig +++ b/lib/compiler/Maker/Step/WriteFile.zig @@ -24,7 +24,6 @@ pub fn make( const conf = &maker.scanned_config.configuration; const conf_step = step_index.ptr(conf); const conf_wf = conf_step.extended.get(conf.extra).write_file; - const cache_root = graph.local_cache_root; const directories = conf_wf.directories.slice; const open_dir_cache = try arena.alloc(Io.Dir, directories.len); @@ -106,25 +105,17 @@ pub fn make( if (try step.cacheHit(maker, &man, progress_node)) { const digest = man.hitDigestHex(); - maker.generatedPath(conf_wf.generated_directory).* = .{ - .root_dir = cache_root, - .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), - }; + _ = try maker.setGeneratedPath(conf_wf.generated_directory, .local_cache, &.{ "o", &digest }); assert(step.result_cached); return; } const digest = man.missDigestHex(); - const out_path: Path = .{ - .root_dir = cache_root, - .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }), - }; + const out_path = try maker.setGeneratedPath(conf_wf.generated_directory, .local_cache, &.{ "o", &digest }); progress_node.setEstimatedTotalItems(total_items); try operate(maker, step_index, open_dir_cache, out_path, progress_node); try step.finalizeManifest(maker, &man); - - maker.generatedPath(conf_wf.generated_directory).* = out_path; }, .tmp => { step.result_cached = false; @@ -133,20 +124,17 @@ pub fn make( io.random(@ptrCast(&rand_int)); const hex_digest = std.fmt.hex(rand_int); - const out_path: Path = .{ - .root_dir = cache_root, - .sub_path = try Io.Dir.path.join(arena, &.{ "tmp", &hex_digest }), - }; + const out_path = try maker.setGeneratedPath(conf_wf.generated_directory, .local_cache, &.{ + "tmp", &hex_digest, + }); try operate(maker, step_index, open_dir_cache, out_path, progress_node); - - maker.generatedPath(conf_wf.generated_directory).* = out_path; }, .mutate => { step.result_cached = false; const root_path = try maker.resolveLazyPathIndex(arena, conf_wf.mutate_path.value.?, step_index); try operate(maker, step_index, open_dir_cache, root_path, progress_node); - maker.generatedPath(conf_wf.generated_directory).* = root_path; + _ = try maker.setGeneratedPathPath(conf_wf.generated_directory, root_path); }, } } -- 2.54.0 From 01eff30056ffef1e3c4f5987d2908acb35397caf Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 24 Sep 2026 20:24:28 -0700 Subject: [PATCH 2/5] Maker: fix Step.FindProgram --- lib/compiler/Maker/Step/FindProgram.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compiler/Maker/Step/FindProgram.zig b/lib/compiler/Maker/Step/FindProgram.zig index e0fece763899e1afa23f187099a4a1f7928032e3..d6cc22183b39bf6b44cfcfee8b24288152b3a5fc 100644 --- a/lib/compiler/Maker/Step/FindProgram.zig +++ b/lib/compiler/Maker/Step/FindProgram.zig @@ -96,7 +96,7 @@ fn checkCandidate( const extended_path = try std.mem.concat(arena, u8, &.{ full_path, ext }); if (Io.Dir.cwd().access(io, extended_path, .{ .execute = true })) |_| { - maker.generatedPath(found_path).* = .initCwd(extended_path); + _ = try maker.setGeneratedPath(found_path, .cwd, &.{extended_path}); return true; } else |err| switch (err) { error.Canceled => |e| return e, -- 2.54.0 From 3d2207fc7917d5a2e05cbce22ba4d94cf2cbdac4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 24 Sep 2026 23:08:49 -0700 Subject: [PATCH 3/5] build server protocol: send generated file paths closes #36403 --- lib/compiler/Maker.zig | 157 +++++++++++++++++++++++++++++++++++- lib/std/zig/ErrorBundle.zig | 18 +++++ lib/std/zig/Server.zig | 14 +++- tools/bsp.zig | 30 ++++++- 4 files changed, 216 insertions(+), 3 deletions(-) diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index aaa6498c001a1eb1305cfa6466d94b09825cde2f..0a7ea5cb1f5520d4c606f58049c7f1185f43ae09 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -3434,6 +3434,8 @@ fn serveBuildStepCompleted( const step = maker.stepByIndex(step_index); const error_bundle = step.result_error_bundle; + const gf_size = countGeneratedFiles(maker.generated_files, step_index, &maker.scanned_config.configuration); + const body: Server.Message.BuildStepCompleted = .{ .step_index = step_index, .status = status, @@ -3441,19 +3443,172 @@ fn serveBuildStepCompleted( .extra_len = @intCast(error_bundle.extra.len), .string_bytes_len = @intCast(error_bundle.string_bytes.len), }, + .generated_files_len = gf_size.count, }; const eb_bytes_len = @sizeOf(u32) * error_bundle.extra.len + error_bundle.string_bytes.len; - const bytes_len = @sizeOf(Server.Message.BuildStepCompleted) + eb_bytes_len; + const bytes_len = @sizeOf(Server.Message.BuildStepCompleted) + eb_bytes_len + + (gf_size.count * @sizeOf(Server.Message.GeneratedFile)) + gf_size.size; try s.serveMessageHeader(.{ .tag = .bsp_step_completed, .bytes_len = @intCast(bytes_len), }); try s.out.writeStruct(body, .little); + try s.out.writeSliceEndian(u32, error_bundle.extra, .little); try s.out.writeAll(error_bundle.string_bytes); + + try writeGeneratedFiles(maker.generated_files, step_index, &maker.scanned_config.configuration, s.out); + try writeGeneratedFilesData(maker.generated_files, step_index, &maker.scanned_config.configuration, s.out); + try s.out.flush(); } +fn walkGeneratedFiles( + step_index: Configuration.Step.Index, + conf: *const Configuration, + comptime Context: type, + context: Context, +) void { + const conf_step = step_index.ptr(conf); + switch (conf_step.extended.get(conf.extra)) { + .check_file, + .fail, + .fmt, + .install_artifact, + .install_dir, + .install_file, + .top_level, + .update_source_files, + => {}, + + .compile => |compile| { + walkGeneratedFilesOne(context, compile.emit_directory.value); + walkGeneratedFilesOne(context, compile.generated_docs.value); + walkGeneratedFilesOne(context, compile.generated_asm.value); + walkGeneratedFilesOne(context, compile.generated_bin.value); + walkGeneratedFilesOne(context, compile.generated_pdb.value); + walkGeneratedFilesOne(context, compile.generated_implib.value); + walkGeneratedFilesOne(context, compile.generated_llvm_bc.value); + walkGeneratedFilesOne(context, compile.generated_llvm_ir.value); + walkGeneratedFilesOne(context, compile.generated_h.value); + }, + .config_header => |config_header| { + walkGeneratedFilesOne(context, config_header.generated_dir); + }, + .find_program => |find_program| { + walkGeneratedFilesOne(context, find_program.found_path); + }, + .obj_copy => |obj_copy| { + walkGeneratedFilesOne(context, obj_copy.output_file); + walkGeneratedFilesOne(context, obj_copy.debug_file.value); + }, + .options => |options| { + walkGeneratedFilesOne(context, options.generated_file); + }, + .run => |run| { + if (run.captured_stdout.value) |v| walkGeneratedFilesOne(context, v.generated_file); + if (run.captured_stderr.value) |v| walkGeneratedFilesOne(context, v.generated_file); + for (run.args.slice) |arg| + walkGeneratedFilesOne(context, arg.get(conf).generated.value); + }, + .translate_c => |translate_c| { + walkGeneratedFilesOne(context, translate_c.output_file); + }, + .write_file => |write_file| { + walkGeneratedFilesOne(context, write_file.generated_directory); + }, + } +} + +fn walkGeneratedFilesOne( + context: anytype, + opt: ?Configuration.GeneratedFileIndex, +) void { + context.visit(opt orelse return); +} + +fn countGeneratedFiles( + generated_files: []const GeneratedFile, + step_index: Configuration.Step.Index, + conf: *const Configuration, +) struct { count: u32, size: u32 } { + const Visitor = struct { + generated_files: []const GeneratedFile, + count: u32, + size: u32, + + fn visit(this: *@This(), i: Configuration.GeneratedFileIndex) void { + const data = this.generated_files[@backingInt(i)].data orelse return; + this.count += 1; + this.size = @intCast(this.size + data.len); + } + }; + var v: Visitor = .{ + .generated_files = generated_files, + .count = 0, + .size = 0, + }; + walkGeneratedFiles(step_index, conf, *Visitor, &v); + return .{ + .count = v.count, + .size = v.size, + }; +} + +fn writeGeneratedFiles( + generated_files: []const GeneratedFile, + step_index: Configuration.Step.Index, + conf: *const Configuration, + writer: *Io.Writer, +) Io.Writer.Error!void { + const Visitor = struct { + generated_files: []const GeneratedFile, + writer: *Io.Writer, + result: Io.Writer.Error!void, + + fn visit(this: *@This(), i: Configuration.GeneratedFileIndex) void { + const unpacked = this.generated_files[@backingInt(i)].unpack() orelse return; + const payload: Server.Message.GeneratedFile = .{ + .index = i, + .path_len = @intCast(unpacked.sub_path.len), + }; + this.result = this.writer.writeAll(@ptrCast((&payload)[0..1])); + } + }; + var v: Visitor = .{ + .generated_files = generated_files, + .writer = writer, + .result = {}, + }; + walkGeneratedFiles(step_index, conf, *Visitor, &v); + return v.result; +} + +fn writeGeneratedFilesData( + generated_files: []const GeneratedFile, + step_index: Configuration.Step.Index, + conf: *const Configuration, + writer: *Io.Writer, +) Io.Writer.Error!void { + const Visitor = struct { + generated_files: []const GeneratedFile, + writer: *Io.Writer, + result: Io.Writer.Error!void, + + fn visit(this: *@This(), i: Configuration.GeneratedFileIndex) void { + const data = this.generated_files[@backingInt(i)].data orelse return; + this.result = this.writer.writeAll(data); + } + }; + var v: Visitor = .{ + .generated_files = generated_files, + .writer = writer, + .result = {}, + }; + walkGeneratedFiles(step_index, conf, *Visitor, &v); + return v.result; +} + fn initStdoutWriter(io: Io) *Writer { stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation); return &stdout_writer_allocation.interface; diff --git a/lib/std/zig/ErrorBundle.zig b/lib/std/zig/ErrorBundle.zig index 03ff0f83f22ae3184aced9a856e443f65d1273a0..d6fde276f1e9fb328d262b90a91f0112c45afb77 100644 --- a/lib/std/zig/ErrorBundle.zig +++ b/lib/std/zig/ErrorBundle.zig @@ -197,6 +197,24 @@ pub fn renderToTerminal(eb: ErrorBundle, options: RenderOptions, t: Io.Terminal) } } +pub fn readAlloc( + r: *Io.Reader, + gpa: Allocator, + extra_len: u32, + string_bytes_len: u32, +) Io.Reader.ReadAllocError!ErrorBundle { + const extra = try r.readSliceEndianAlloc(gpa, u32, extra_len, .little); + errdefer gpa.free(extra); + + const string_bytes = try r.readAlloc(gpa, string_bytes_len); + errdefer gpa.free(string_bytes); + + return .{ + .extra = extra, + .string_bytes = string_bytes, + }; +} + fn renderErrorMessage( eb: ErrorBundle, options: RenderOptions, diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index eac3555acf7074a174f333401e0f79c70c23ab7e..a96f186f4460c585bffb9fc8cd42f030675d172c 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -8,6 +8,7 @@ const OutMessage = std.zig.Server.Message; const InMessage = std.zig.Client.Message; const Reader = std.Io.Reader; const Writer = std.Io.Writer; +const Configuration = std.Build.Configuration; in: *Reader, out: *Writer, @@ -113,10 +114,15 @@ pub const Message = struct { /// Trailing: /// * error_bundle: ErrorBundle, + /// * generated_file: [generated_files_len]GeneratedFile, + /// * path_bytes: [_]u8, // for each GeneratedFile + /// - PathPrefix + /// - sub_path: [_]u8, pub const BuildStepCompleted = extern struct { - step_index: std.Build.Configuration.Step.Index, + step_index: Configuration.Step.Index, status: Status, error_bundle: ErrorBundle, + generated_files_len: u32, // TODO result_error_msgs // TODO result_stderr // TODO result_peak_rss @@ -130,6 +136,12 @@ pub const Message = struct { }; }; + pub const GeneratedFile = extern struct { + index: Configuration.GeneratedFileIndex, + /// Includes only the path bytes, not the prefix or null byte. + path_len: u32, + }; + pub const PathPrefix = enum(u8) { cwd, zig_lib, diff --git a/tools/bsp.zig b/tools/bsp.zig index b86e30c3af07b53b352602e5b480c0d785055830..e58205fd958ad569ec76fbf4703671546d0062d5 100644 --- a/tools/bsp.zig +++ b/tools/bsp.zig @@ -184,7 +184,35 @@ pub fn main(init: std.process.Init) !void { .bsp_build_started => {}, .bsp_build_completed => if (!watch) break, .bsp_step_started => {}, - .bsp_step_completed => {}, + .bsp_step_completed => { + var reader: Io.Reader = .fixed(body); + const bsc = try reader.takeStruct(Server.Message.BuildStepCompleted, .little); + log.info("step {q} {t}: ({d} generated files)", .{ + bsc.step_index.ptr(c).name.slice(c), + bsc.status, + bsc.generated_files_len, + }); + + var eb = try std.zig.ErrorBundle.readAlloc( + &reader, + gpa, + bsc.error_bundle.extra_len, + bsc.error_bundle.string_bytes_len, + ); + defer eb.deinit(gpa); + try eb.renderToStderr(io, .{}, .auto); + + const modified_files: []align(1) Server.Message.GeneratedFile = @ptrCast( + try reader.take(bsc.generated_files_len * @sizeOf(Server.Message.GeneratedFile)), + ); + for (modified_files) |modified_file| { + const prefix = try reader.takeEnum(Server.Message.PathPrefix, .little); + const sub_path = try reader.take(modified_file.path_len); + log.info("generated file: {d} {t} {q}", .{ + modified_file.index, prefix, sub_path, + }); + } + }, .bsp_configuration => @panic("TODO"), else => std.debug.panic("received unexpected message: {f}", .{fmtEnum(header.tag)}), } -- 2.54.0 From aaa314f6b2ff9481eba9d0cf72ed845c78f32c36 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 25 Sep 2026 01:56:12 -0700 Subject: [PATCH 4/5] Maker.Fuzz: fix computation of rebuilt exe path --- lib/compiler/Maker/Fuzz.zig | 9 +++++++-- lib/compiler/Maker/Step/Compile.zig | 10 ++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/compiler/Maker/Fuzz.zig b/lib/compiler/Maker/Fuzz.zig index 56f773eae9e8c4644e80419d75144419a62be49f..12e4087fa1bcd3b8c8c5d4262a8c27a6683d7dd9 100644 --- a/lib/compiler/Maker/Fuzz.zig +++ b/lib/compiler/Maker/Fuzz.zig @@ -212,7 +212,7 @@ fn rebuildTestsWorkerRunFallible( maker.printErrorMessages(comp_index, .{}, stderr.terminal(), .verbose, .indent) catch {}; } - const rebuilt_bin_path = result catch |err| switch (err) { + const rebuilt_digest = result catch |err| switch (err) { error.MakeFailed => return, else => |other| return other, }; @@ -235,7 +235,12 @@ fn rebuildTestsWorkerRunFallible( }); defer gpa.free(compile_filename); - run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile_filename); + const o_hex_digest = rebuilt_digest.toHex().?; + + run.rebuilt_executable = .{ + .root_dir = graph.local_cache_root, + .sub_path = try std.fs.path.join(gpa, &.{ "o", &o_hex_digest, compile_filename }), + }; } fn fuzzWorkerRun(fuzz: *Fuzz, run_index: Configuration.Step.Index) void { diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index d16b1afcdd7be88b7f1fcbff095c53bf204ef94c..b65019ed6e41890671ab9ea232e0df6317e13dec 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -943,9 +943,8 @@ pub fn rebuildInFuzzMode( maker: *Maker, compile_index: Configuration.Step.Index, progress_node: std.Progress.Node, -) !Path { +) !Step.OptCacheDigest { const gpa = maker.gpa; - const graph = maker.graph; const step = maker.stepByIndex(compile_index); var arena_allocator: std.heap.ArenaAllocator = .init(gpa); @@ -964,12 +963,7 @@ pub fn rebuildInFuzzMode( defer argv.deinit(gpa); try lowerZigArgs(arena, compile, compile_index, maker, progress_node, &argv, true); - const opt_cache_digest = try Step.evalZigProcess(compile_index, maker, argv.items, progress_node, false); - const o_hex_digest = opt_cache_digest.toHex().?; - return .{ - .root_dir = graph.local_cache_root, - .sub_path = try Dir.path.join(arena, &.{ "o", &o_hex_digest }), - }; + return Step.evalZigProcess(compile_index, maker, argv.items, progress_node, false); } fn addBool(gpa: Allocator, args: *std.ArrayList([]const u8), arg: []const u8, opt: bool) !void { -- 2.54.0 From 480e3f89b227c6c1d5daeb8ae8f7af46c9301637 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 25 Sep 2026 02:05:02 -0700 Subject: [PATCH 5/5] Maker.Fuzz: fix memory leaks --- lib/compiler/Maker/Fuzz.zig | 1 + lib/compiler/Maker/Step/Run.zig | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/compiler/Maker/Fuzz.zig b/lib/compiler/Maker/Fuzz.zig index 12e4087fa1bcd3b8c8c5d4262a8c27a6683d7dd9..25da25516ceb5d8ba5b68f393041276340530990 100644 --- a/lib/compiler/Maker/Fuzz.zig +++ b/lib/compiler/Maker/Fuzz.zig @@ -161,6 +161,7 @@ pub fn deinit(fuzz: *Fuzz) void { fuzz.group.cancel(io); fuzz.prog_node.end(); gpa.free(fuzz.run_steps); + fuzz.msg_queue.deinit(gpa); } fn rebuildTestsWorkerRun( diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 5b0e70a1413e0ed321c9ae41f8a6792a91811e92..7f5afff5cbb1968bb25537a716a715a9efb20506 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -18,14 +18,13 @@ const Step = @import("../Step.zig"); const Maker = @import("../../Maker.zig"); const Fuzz = @import("../../Maker/Fuzz.zig"); -/// If this is a Zig unit test binary, this tracks the names of the unit -/// tests that are also fuzz tests. Indexes cannot be used as they may -/// change between reruns. +/// If this is a Zig unit test binary, this tracks the names of the unit tests that are also fuzz tests. +/// Indexes cannot be used as they may change between reruns. Memory owned by `Maker.gpa`. fuzz_tests: std.ArrayList([]const u8) = .empty, cached_test_metadata: ?CachedTestMetadata = null, -/// Populated during the fuzz phase if this run step corresponds to a unit test -/// executable that contains fuzz tests. +/// Populated during the fuzz phase if this run step corresponds to a unit test executable that contains fuzz +/// tests. `Path.sub_path` owned by `Maker.gpa`. rebuilt_executable: ?Path = null, pub fn make( @@ -376,6 +375,7 @@ pub fn make( pub fn deinit(run: *Run, gpa: Allocator, io: Io) void { _ = io; run.fuzz_tests.deinit(gpa); + if (run.rebuilt_executable) |p| gpa.free(p.sub_path); } fn thirdPartyToggle( -- 2.54.0