authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-09-24 18:06:21-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-09-24 23:10:27-07:00
log4853ea1bda1131d4d57a50eaa8209d90b71c8537
treee6af587aa13af0dca95a7bb553ee3dd4706a0738
parent5b9147ed59b22edf7f891cf30e8b77223121da4f

Maker: store generated files differently

in preparation for sending them over the build server protocol (#36403), they are now stored in the same format in memory: one byte for prefix followed by the relative path.

11 files changed, 209 insertions(+), 156 deletions(-)

lib/compiler/Maker.zig+118-13
......@@ -49,7 +49,13 @@ scanned_config: *const ScannedConfig,
4949/// system watching does not need to make any special cases, and to avoid more
5050/// OS-specific logic in file system watching implementation.
5151steps: []Step,
52generated_files: []Path,
52/// Reserved space for all the file paths that are determined at runtime when the associated Step logic runs.
53/// Each element is individually allocated so that steps can access the data without synchronization, and so
54/// that steps can be arbitrarily re-run independently from each other.
55/// Each element is optional because a step might populate any number of generated file paths before ultimately
56/// failing. An element being non-null does not imply that the generated file was successfully generated. Only
57/// the Step success state indicates that information.
58generated_files: []GeneratedFile,
5359run_args: ?[]const []const u8,
5460
5561available_rss: u64,
......@@ -121,6 +127,38 @@ const MultilineErrors = enum { indent, newline, none };
121127const Summary = enum { all, new, failures, line, none };
122128const PrintConfiguration = enum { none, zon, path };
123129
130const GeneratedFile = struct {
131 data: ?[]const u8,
132
133 const Unpacked = struct {
134 prefix: Server.Message.PathPrefix,
135 sub_path: []const u8,
136 };
137
138 fn unpack(gf: GeneratedFile) ?Unpacked {
139 const slice = gf.data orelse return null;
140 return .{
141 .prefix = @fromBackingInt(slice[0]),
142 .sub_path = slice[1..],
143 };
144 }
145
146 fn toPath(gf: GeneratedFile, graph: *const Graph) ?Path {
147 const unpacked = unpack(gf) orelse return null;
148 return .{
149 .root_dir = prefixToDirectory(graph, unpacked.prefix),
150 .sub_path = unpacked.sub_path,
151 };
152 }
153
154 fn clear(gf: *GeneratedFile, gpa: Allocator) void {
155 if (gf.data) |data| {
156 gpa.free(data);
157 gf.data = null;
158 }
159 }
160};
161
124162/// Used to build the -M flags to pass to build-exe.
125163pub const CliModule = struct {
126164 name: []const u8,
......@@ -2344,7 +2382,8 @@ fn prepare(
23442382 // Extra step at the end which is the autogenerated placeholder
23452383 // step which indicates that we need to reconfigure.
23462384 maker.steps = try arena.alloc(Step, c.steps.len + 1);
2347 maker.generated_files = try arena.alloc(Path, c.generated_files_len);
2385 maker.generated_files = try arena.alloc(GeneratedFile, c.generated_files_len);
2386 @memset(maker.generated_files, .{ .data = null });
23482387
23492388 // The last element is a reserved special pseudostep which contains the
23502389 // watch inputs for the configurer executable.
......@@ -2676,6 +2715,7 @@ fn deinit(maker: *Maker) void {
26762715 const gpa = maker.gpa;
26772716 const io = maker.graph.io;
26782717 for (maker.steps) |*step| step.deinit(gpa, io);
2718 for (maker.generated_files) |*gf| gf.clear(gpa);
26792719 maker.memory_blocked_steps.deinit(gpa);
26802720 maker.initial_steps.deinit(gpa);
26812721 maker.step_stack.deinit(gpa);
......@@ -2787,7 +2827,8 @@ fn makeStep(
27872827 if (maker.web_server) |ws| {
27882828 ws.updateStepStatus(step_index, if (success) .success else .failure);
27892829 }
2790 if (maker.protocol_server != null) {
2830 if (maker.protocol_server) |s| {
2831 // https://codeberg.org/ziglang/zig/issues/36938
27912832 maker.protocol_server_mutex.lockUncancelable(io);
27922833 defer maker.protocol_server_mutex.unlock(io);
27932834
......@@ -2800,11 +2841,8 @@ fn makeStep(
28002841 .dependency_skipped, .skipped => .skipped,
28012842 .skipped_oom => .skipped_oom,
28022843 };
2803 serveBuildStepCompleted(
2804 maker,
2805 step_index,
2806 status,
2807 ) catch |err| std.debug.panic("TODO propagate error when failing to send protocol message: {t}", .{err});
2844 serveBuildStepCompleted(maker, step_index, status, s) catch |err|
2845 std.debug.panic("TODO propagate error when failing to send protocol message: {t}", .{err});
28082846 }
28092847
28102848 if (!success) std.Progress.setStatus(.failure_working);
......@@ -3366,7 +3404,7 @@ fn cleanTmpFiles(maker: *Maker, steps: []const Configuration.Step.Index) void {
33663404 if (wf.flags.mode != .tmp) continue;
33673405 const step = maker.stepByIndex(step_index);
33683406 if (step.state != .success) continue;
3369 const tmp_path = generatedPath(maker, wf.generated_directory).*;
3407 const tmp_path = generatedPath(maker, wf.generated_directory);
33703408 tmp_path.root_dir.handle.deleteTree(io, tmp_path.subPathOrDot()) catch |err|
33713409 log.warn("failed to delete temporary path {f}: {t}", .{ tmp_path, err });
33723410 }
......@@ -3391,8 +3429,8 @@ fn serveBuildStepCompleted(
33913429 maker: *Maker,
33923430 step_index: Configuration.Step.Index,
33933431 status: Server.Message.BuildStepCompleted.Status,
3432 s: *Server,
33943433) !void {
3395 const s: *Server = maker.protocol_server.?;
33963434 const step = maker.stepByIndex(step_index);
33973435 const error_bundle = step.result_error_bundle;
33983436
......@@ -3434,7 +3472,7 @@ pub fn resolveLazyPath(
34343472 .source_path => |sp| try packagePath(maker, arena, sp.owner, sp.sub_path.slice(c)),
34353473 .relative => |relative| relativePath(maker, arena, relative),
34363474 .generated => |gen| {
3437 const base = generatedPath(maker, gen.index).*;
3475 const base = generatedPath(maker, gen.index);
34383476 var file_path = base;
34393477 for (0..gen.flags.up) |_| {
34403478 file_path.sub_path = Dir.path.dirname(file_path.sub_path) orelse {
......@@ -3485,10 +3523,67 @@ pub fn resolveLazyPathIndexAbs(
34853523 return resolveLazyPathAbs(maker, arena, lazy_path_index.get(c), asking_step_index);
34863524}
34873525
3488pub fn generatedPath(maker: *const Maker, index: Configuration.GeneratedFileIndex) *Path {
3526pub fn generatedFile(maker: *const Maker, index: Configuration.GeneratedFileIndex) *GeneratedFile {
34893527 return &maker.generated_files[@backingInt(index)];
34903528}
34913529
3530pub fn generatedPath(maker: *const Maker, index: Configuration.GeneratedFileIndex) Path {
3531 return generatedFile(maker, index).toPath(maker.graph).?;
3532}
3533
3534pub fn setGeneratedPath(
3535 maker: *Maker,
3536 index: Configuration.GeneratedFileIndex,
3537 prefix: Server.Message.PathPrefix,
3538 sub_paths: []const []const u8,
3539) Allocator.Error!Path {
3540 const gpa = maker.gpa;
3541 const graph = maker.graph;
3542 assert(sub_paths.len != 0);
3543
3544 var buffer: std.ArrayList(u8) = .empty;
3545 defer buffer.deinit(gpa);
3546 try buffer.append(gpa, @backingInt(prefix));
3547 for (sub_paths) |sub_path| {
3548 try buffer.ensureUnusedCapacity(gpa, sub_path.len + 1);
3549 buffer.appendSliceAssumeCapacity(sub_path);
3550 while (Dir.path.isSep(buffer.items[buffer.items.len - 1]))
3551 buffer.items.len -= 1;
3552 buffer.appendAssumeCapacity(Dir.path.sep);
3553 }
3554 assert(buffer.pop().? == Dir.path.sep);
3555 try buffer.shrinkToLen(gpa);
3556
3557 const sub_path = buffer.toOwnedSliceAssert();
3558 const ptr = generatedFile(maker, index);
3559 ptr.clear(gpa);
3560 ptr.* = .{ .data = sub_path };
3561 return .{
3562 .root_dir = prefixToDirectory(graph, prefix),
3563 .sub_path = sub_path[1..],
3564 };
3565}
3566
3567/// Prefer to use `setGeneratedPath` where possible to keep parent directory information intact.
3568pub fn setGeneratedPathPath(
3569 maker: *Maker,
3570 index: Configuration.GeneratedFileIndex,
3571 path: Path,
3572) Allocator.Error!Path {
3573 const graph = maker.graph;
3574 if (path.root_dir.eql(graph.local_cache_root)) {
3575 return setGeneratedPath(maker, index, .local_cache, &.{path.sub_path});
3576 } else if (path.root_dir.eql(graph.global_cache_root)) {
3577 return setGeneratedPath(maker, index, .global_cache, &.{path.sub_path});
3578 } else if (path.root_dir.eql(graph.zig_lib_directory)) {
3579 return setGeneratedPath(maker, index, .zig_lib, &.{path.sub_path});
3580 } else if (path.root_dir.eql(graph.build_root_directory)) {
3581 return setGeneratedPath(maker, index, .build_root, &.{path.sub_path});
3582 } else {
3583 return setGeneratedPath(maker, index, .cwd, &.{path.sub_path});
3584 }
3585}
3586
34923587pub fn packagePath(
34933588 maker: *const Maker,
34943589 arena: Allocator,
......@@ -3600,7 +3695,7 @@ pub fn installGenerated(
36003695 dest_dir: Configuration.InstallDestDir,
36013696 asking_step_index: Configuration.Step.Index,
36023697) !Dir.PrevStatus {
3603 const src_path = generatedPath(maker, source).*;
3698 const src_path = generatedPath(maker, source);
36043699 const dest_dir_path = try resolveInstallDir(maker, arena, dest_dir);
36053700 const dest_path = try dest_dir_path.join(arena, src_path.basename());
36063701 return installPath(maker, arena, src_path, dest_path, asking_step_index);
......@@ -4170,3 +4265,13 @@ fn nextEnumArg(args: []const []const u8, i: *usize, comptime E: type) E {
41704265 const next_arg = nextArg(args, i) orelse fatalEnumHint(E, arg, null);
41714266 return stringToEnum(E, next_arg) orelse fatalEnumHint(E, arg, next_arg);
41724267}
4268
4269fn prefixToDirectory(graph: *const Graph, prefix: Server.Message.PathPrefix) Cache.Directory {
4270 return switch (prefix) {
4271 .cwd => .cwd(),
4272 .zig_lib => graph.zig_lib_directory,
4273 .local_cache => graph.local_cache_root,
4274 .global_cache => graph.global_cache_root,
4275 .build_root => graph.build_root_directory,
4276 };
4277}
lib/compiler/Maker/Step.zig+14-9
......@@ -460,6 +460,15 @@ pub const ZigProcess = struct {
460460 }
461461};
462462
463pub const OptCacheDigest = struct {
464 bin: ?Cache.BinDigest,
465
466 pub fn toHex(ocd: *const OptCacheDigest) ?Cache.HexDigest {
467 const bin = ocd.bin orelse return null;
468 return Cache.binToHex(bin);
469 }
470};
471
463472/// Assumes that argv contains `--listen=-` and that the process being spawned
464473/// is the zig compiler - the same version that compiled the build runner.
465474///
......@@ -470,7 +479,7 @@ pub fn evalZigProcess(
470479 argv: []const []const u8,
471480 prog_node: std.Progress.Node,
472481 watch: bool,
473) (Step.ExtendedMakeError || error{NeedCompileErrorCheck})!?Path {
482) (Step.ExtendedMakeError || error{NeedCompileErrorCheck})!OptCacheDigest {
474483 const s = maker.stepByIndex(step_index);
475484 const gpa = maker.gpa;
476485 const graph = maker.graph;
......@@ -503,7 +512,7 @@ pub fn evalZigProcess(
503512 if (s.result_error_bundle.errorMessageCount() > 0)
504513 return s.fail(maker, "{d} compilation errors", .{s.result_error_bundle.errorMessageCount()});
505514
506 if (s.result_error_msgs.items.len > 0 and result == null) {
515 if (s.result_error_msgs.items.len > 0 and result.bin == null) {
507516 // Crash detected.
508517 const term = zp.child.wait(io) catch |e| {
509518 return s.fail(maker, "unable to wait for {s}: {t}", .{ argv[0], e });
......@@ -578,7 +587,7 @@ pub fn evalZigProcess(
578587 return result;
579588}
580589
581fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *ZigProcess, watch: bool) !?Path {
590fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *ZigProcess, watch: bool) !OptCacheDigest {
582591 const s = maker.stepByIndex(step_index);
583592 const gpa = maker.gpa;
584593 const graph = maker.graph;
......@@ -590,7 +599,7 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *Zi
590599 try sendMessage(io, zp.child.stdin.?, .update);
591600 if (!watch) try sendMessage(io, zp.child.stdin.?, .exit);
592601
593 var result: ?Path = null;
602 var result: OptCacheDigest = .{ .bin = null };
594603 var eos_err: error{EndOfStream}!void = {};
595604
596605 var client: std.zig.Client = .{
......@@ -632,11 +641,7 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *Zi
632641 const EmitDigest = std.zig.Server.Message.EmitDigest;
633642 const emit_digest: *align(1) const EmitDigest = @ptrCast(body);
634643 s.result_cached = emit_digest.flags.cache_hit;
635 const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len];
636 result = .{
637 .root_dir = graph.local_cache_root,
638 .sub_path = try arena.dupe(u8, "o" ++ Dir.path.sep_str ++ Cache.binToHex(digest.*)),
639 };
644 result = .{ .bin = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len].* };
640645 },
641646 .file_system_inputs => {
642647 clearWatchInputs(s, maker);
lib/compiler/Maker/Step/Compile.zig+24-19
......@@ -46,7 +46,7 @@ pub fn make(
4646
4747 const incremental = conf_comp.flags4.incremental.toBool() orelse graph.incremental == true;
4848
49 const maybe_output_dir = Step.evalZigProcess(
49 const opt_cache_digest = Step.evalZigProcess(
5050 compile_index,
5151 maker,
5252 argv.items,
......@@ -63,24 +63,25 @@ pub fn make(
6363 const root_module = conf_comp.root_module.get(conf);
6464 const target = root_module.resolved_target.get(conf).?.result.get(conf);
6565
66 // Update generated files
67 if (maybe_output_dir) |output_dir| {
68 if (conf_comp.emit_directory.value) |gf| maker.generatedPath(gf).* = output_dir;
69 try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_bin.value, .bin);
70 try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_pdb.value, .pdb);
71 try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_implib.value, .implib);
72 try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_h.value, .h);
73 try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_docs.value, .docs);
74 try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_asm.value, .@"asm");
75 try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_llvm_ir.value, .llvm_ir);
76 try updateGeneratedFile(maker, arena, &conf_comp, output_dir, &target, conf_comp.generated_llvm_bc.value, .llvm_bc);
66 if (opt_cache_digest.toHex()) |o_hex_digest| {
67 if (conf_comp.emit_directory.value) |gf| {
68 _ = try maker.setGeneratedPath(gf, .local_cache, &.{ "o", &o_hex_digest });
69 }
70 try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_bin.value, .bin);
71 try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_pdb.value, .pdb);
72 try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_implib.value, .implib);
73 try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_h.value, .h);
74 try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_docs.value, .docs);
75 try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_asm.value, .@"asm");
76 try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_llvm_ir.value, .llvm_ir);
77 try updateGeneratedFile(maker, arena, &conf_comp, &o_hex_digest, &target, conf_comp.generated_llvm_bc.value, .llvm_bc);
7778 }
7879
7980 if (conf_comp.flags3.kind == .lib and conf_comp.flags2.linkage == .dynamic and
8081 conf_comp.version.value != null and target.flags.os_tag != .windows)
8182 {
8283 if (conf_comp.generated_bin.value) |generated_bin| {
83 const full_dest_path = maker.generatedPath(generated_bin).*;
84 const full_dest_path = maker.generatedPath(generated_bin);
8485 try maker.installSymLinks(arena, full_dest_path, compile_index, compile_index);
8586 }
8687 }
......@@ -97,13 +98,12 @@ fn updateGeneratedFile(
9798 maker: *Maker,
9899 arena: Allocator,
99100 conf_comp: *const Configuration.Step.Compile,
100 out_path: std.Build.Cache.Path,
101 o_hex_digest: *const std.Build.Cache.HexDigest,
101102 target: *const Configuration.TargetQuery,
102103 opt_gf: ?Configuration.GeneratedFileIndex,
103104 ea: std.zig.EmitArtifact,
104105) Allocator.Error!void {
105106 const gf = opt_gf orelse return;
106 const graph = maker.graph;
107107 const conf = &maker.scanned_config.configuration;
108108 const name = try ea.cacheName(arena, .{
109109 .root_name = conf_comp.root_name.slice(conf),
......@@ -122,7 +122,7 @@ fn updateGeneratedFile(
122122 else
123123 null,
124124 });
125 maker.generatedPath(gf).* = try out_path.join(graph.arena, name);
125 _ = try maker.setGeneratedPath(gf, .local_cache, &.{ "o", o_hex_digest, name });
126126}
127127
128128/// List of importable modules in a compilation's module graph, including
......@@ -945,6 +945,7 @@ pub fn rebuildInFuzzMode(
945945 progress_node: std.Progress.Node,
946946) !Path {
947947 const gpa = maker.gpa;
948 const graph = maker.graph;
948949 const step = maker.stepByIndex(compile_index);
949950
950951 var arena_allocator: std.heap.ArenaAllocator = .init(gpa);
......@@ -963,8 +964,12 @@ pub fn rebuildInFuzzMode(
963964 defer argv.deinit(gpa);
964965
965966 try lowerZigArgs(arena, compile, compile_index, maker, progress_node, &argv, true);
966 const maybe_output_bin_path = try Step.evalZigProcess(compile_index, maker, argv.items, progress_node, false);
967 return maybe_output_bin_path.?;
967 const opt_cache_digest = try Step.evalZigProcess(compile_index, maker, argv.items, progress_node, false);
968 const o_hex_digest = opt_cache_digest.toHex().?;
969 return .{
970 .root_dir = graph.local_cache_root,
971 .sub_path = try Dir.path.join(arena, &.{ "o", &o_hex_digest }),
972 };
968973}
969974
970975fn addBool(gpa: Allocator, args: *std.ArrayList([]const u8), arg: []const u8, opt: bool) !void {
......@@ -1380,7 +1385,7 @@ pub fn appendIncludeDirFlags(
13801385 },
13811386 .config_header_step => |ch_index| {
13821387 const conf_ch = ch_index.ptr(conf).extended.get(conf.extra).config_header;
1383 const path = maker.generatedPath(conf_ch.generated_dir).*;
1388 const path = maker.generatedPath(conf_ch.generated_dir);
13841389 zig_args.appendAssumeCapacity("-I");
13851390 zig_args.appendAssumeCapacity(try path.toString(arena));
13861391 },
lib/compiler/Maker/Step/ConfigHeader.zig+6-16
......@@ -34,7 +34,6 @@ pub fn make(
3434 const conf = &maker.scanned_config.configuration;
3535 const conf_step = step_index.ptr(conf);
3636 const conf_ch = conf_step.extended.get(conf.extra).config_header;
37 const cache_root = graph.local_cache_root;
3837
3938 const input_size_limit: Io.Limit = if (conf_ch.input_size_limit.value) |x| .limited64(x) else .unlimited;
4039 const include_guard_override: ?[]const u8 = if (conf_ch.include_guard.value) |s| s.slice(conf) else null;
......@@ -138,10 +137,7 @@ pub fn make(
138137
139138 if (try step.cacheHit(maker, &man, progress_node)) {
140139 const digest = man.hitDigestHex();
141 maker.generatedPath(conf_ch.generated_dir).* = .{
142 .root_dir = cache_root,
143 .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }),
144 };
140 _ = try maker.setGeneratedPath(conf_ch.generated_dir, .local_cache, &.{ "o", &digest });
145141 return;
146142 }
147143
......@@ -152,22 +148,16 @@ pub fn make(
152148 // output_path is libavutil/avconfig.h
153149 // We want to open directory zig-cache/o/HASH/libavutil/
154150 // but keep output_dir as zig-cache/o/HASH for -I include
155 const out_path: Path = .{
156 .root_dir = cache_root,
157 .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, conf_ch.include_path.slice(conf) }),
158 };
151
152 const artifact_path = try maker.setGeneratedPath(conf_ch.generated_dir, .local_cache, &.{ "o", &digest });
153 const out_path = try artifact_path.join(arena, conf_ch.include_path.slice(conf));
159154 const out_path_dirname = out_path.dirname().?;
160155
161156 out_path_dirname.root_dir.handle.createDirPath(io, out_path_dirname.sub_path) catch |err|
162 return step.fail(maker, "unable to make path {f}: {t}", .{ out_path_dirname, err });
157 return step.fail(maker, "failed to make path {qf}: {t}", .{ out_path_dirname, err });
163158
164159 out_path.root_dir.handle.writeFile(io, .{ .sub_path = out_path.sub_path, .data = output }) catch |err|
165 return step.fail(maker, "unable to write file {f}: {t}", .{ out_path, err });
166
167 maker.generatedPath(conf_ch.generated_dir).* = .{
168 .root_dir = cache_root,
169 .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }),
170 };
160 return step.fail(maker, "failed to write file {qf}: {t}", .{ out_path, err });
171161
172162 try step.finalizeManifest(maker, &man);
173163}
lib/compiler/Maker/Step/FindProgram.zig+1-1
......@@ -77,7 +77,7 @@ fn checkCandidate(
7777 const io = graph.io;
7878
7979 if (Io.Dir.cwd().access(io, full_path, .{ .execute = true })) |_| {
80 maker.generatedPath(found_path).* = .initCwd(full_path);
80 _ = try maker.setGeneratedPath(found_path, .cwd, &.{full_path});
8181 return true;
8282 } else |err| switch (err) {
8383 error.Canceled => |e| return e,
lib/compiler/Maker/Step/InstallArtifact.zig+1-1
......@@ -50,7 +50,7 @@ pub fn make(
5050 });
5151 const dest_dir = try maker.resolveInstallDir(arena, bin_dir);
5252 const dest_path = try dest_dir.join(arena, bin_sub_path);
53 const src_path = maker.generatedPath(generated_bin).*;
53 const src_path = maker.generatedPath(generated_bin);
5454 const p = try maker.installPath(arena, src_path, dest_path, step_index);
5555 all_cached = all_cached and p == .fresh;
5656
lib/compiler/Maker/Step/ObjCopy.zig+7-20
......@@ -22,7 +22,6 @@ pub fn make(
2222 const conf = &maker.scanned_config.configuration;
2323 const conf_step = step_index.ptr(conf);
2424 const conf_oc = conf_step.extended.get(conf.extra).obj_copy;
25 const cache_root = graph.local_cache_root;
2625 const input_lazy_path = conf_oc.input_file.get(conf);
2726 const only_section: ?[]const u8 = if (conf_oc.only_section.value) |s| s.slice(conf) else null;
2827 const opt_basename: ?[]const u8 = if (conf_oc.basename.value) |s| s.slice(conf) else null;
......@@ -49,18 +48,12 @@ pub fn make(
4948 if (try step.cacheHit(maker, &man, progress_node)) {
5049 // Cache hit, skip subprocess execution.
5150 const digest = man.hitDigestHex();
52 maker.generatedPath(conf_oc.output_file).* = .{
53 .root_dir = cache_root,
54 .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }),
55 };
51 _ = try maker.setGeneratedPath(conf_oc.output_file, .local_cache, &.{ "o", &digest, basename });
5652 if (conf_oc.debug_file.value) |debug_file| {
5753 const debug_basename = opt_debug_basename orelse try arena.print("{s}.debug", .{
5854 Io.Dir.path.basename(input_path.sub_path),
5955 });
60 maker.generatedPath(debug_file).* = .{
61 .root_dir = cache_root,
62 .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, debug_basename }),
63 };
56 _ = try maker.setGeneratedPath(debug_file, .local_cache, &.{ "o", &digest, debug_basename });
6457 }
6558 return;
6659 }
......@@ -68,10 +61,8 @@ pub fn make(
6861 // We don't find out more input files while executing objcopy so we can
6962 // already obtain the digest and use it directly as the output path.
7063 const digest = man.missDigestHex();
71 const dest_path: Path = .{
72 .root_dir = cache_root,
73 .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }),
74 };
64 const dest_path = try maker.setGeneratedPath(conf_oc.output_file, .local_cache, &.{ "o", &digest, basename });
65
7566 const dest_dirname = dest_path.dirname().?;
7667 dest_dirname.root_dir.handle.createDirPath(io, dest_dirname.sub_path) catch |err|
7768 return step.fail(maker, "failed to create path {f}: {t}", .{ dest_dirname, err });
......@@ -107,12 +98,10 @@ pub fn make(
10798 const debug_basename = opt_debug_basename orelse try arena.print("{s}.debug", .{
10899 Io.Dir.path.basename(input_path.sub_path),
109100 });
110 const debug_dest_path: Path = .{
111 .root_dir = cache_root,
112 .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, debug_basename }),
113 };
101 const debug_dest_path = try maker.setGeneratedPath(debug_file, .local_cache, &.{
102 "o", &digest, debug_basename,
103 });
114104 argv.appendAssumeCapacity(try arena.print("--extract-to={f}", .{debug_dest_path}));
115 maker.generatedPath(debug_file).* = debug_dest_path;
116105 }
117106
118107 try argv.ensureUnusedCapacity(arena, conf_oc.add_section.slice.len * 2);
......@@ -163,8 +152,6 @@ pub fn make(
163152 else => |e| return e,
164153 };
165154
166 maker.generatedPath(conf_oc.output_file).* = dest_path;
167
168155 step.finalizeManifest(maker, &man) catch |err| switch (err) {
169156 error.Canceled => |e| return e,
170157 else => |e| try step.addError(maker, "failed writing cache manifest: {t}", .{e}),
lib/compiler/Maker/Step/Options.zig+4-11
......@@ -19,7 +19,6 @@ pub fn make(
1919 const graph = maker.graph;
2020 const step = maker.stepByIndex(step_index);
2121 const io = graph.io;
22 const cache_root = graph.local_cache_root;
2322 const arena = graph.arena; // TODO don't leak into the process arena
2423 const conf = &maker.scanned_config.configuration;
2524 const conf_step = step_index.ptr(conf);
......@@ -80,19 +79,15 @@ pub fn make(
8079
8180 if (try step.cacheHitWatched(maker, &man, progress_node)) {
8281 const digest = man.hitDigestHex();
83 maker.generatedPath(conf_options.generated_file).* = .{
84 .root_dir = cache_root,
85 .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }),
86 };
82 _ = try maker.setGeneratedPath(conf_options.generated_file, .local_cache, &.{ "o", &digest, basename });
8783 step.result_cached = true;
8884 return;
8985 }
9086
9187 const digest = man.missDigestHex();
92 const out_path: Cache.Path = .{
93 .root_dir = cache_root,
94 .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }),
95 };
88 const out_path = try maker.setGeneratedPath(conf_options.generated_file, .local_cache, &.{
89 "o", &digest, basename,
90 });
9691
9792 var file: Io.File = out_path.root_dir.handle.createFile(io, out_path.sub_path, .{}) catch |err| switch (err) {
9893 error.Canceled => |e| return e,
......@@ -121,6 +116,4 @@ pub fn make(
121116 };
122117
123118 try step.finalizeManifestAndWatch(maker, &man);
124
125 maker.generatedPath(conf_options.generated_file).* = out_path;
126119}
lib/compiler/Maker/Step/Run.zig+24-45
......@@ -146,7 +146,8 @@ pub fn make(
146146 const producer_make_comp_step = maker.stepByIndex(producer_index);
147147 const producer_make_comp = &producer_make_comp_step.extended.compile;
148148
149 const file_path = producer_make_comp.installed_path orelse maker.generatedPath(producer.generated_bin.value.?).*;
149 const file_path = producer_make_comp.installed_path orelse
150 maker.generatedPath(producer.generated_bin.value.?);
150151
151152 argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{
152153 prefix, try convertPathArg(arena, run_index, maker, file_path, arg.flags.make_absolute), suffix,
......@@ -262,8 +263,8 @@ pub fn make(
262263 if (!has_side_effects and try step.cacheHitWatched(maker, &man, progress_node)) {
263264 // Cache hit; skip running command.
264265 const digest = man.hitDigestHex();
265 try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest);
266 try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest);
266 try populateGeneratedStdIo(maker, &conf_run, &digest);
267 try populateGeneratedPaths(maker, output_placeholders.items, &digest);
267268 step.result_cached = true;
268269 return;
269270 }
......@@ -272,7 +273,7 @@ pub fn make(
272273 // We already know the final output paths; use them directly.
273274 const digest = if (has_side_effects) man.hash.final() else man.missDigestHex();
274275 const output_dir_path = "o" ++ Dir.path.sep_str ++ &digest;
275 try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest);
276 try populateGeneratedStdIo(maker, &conf_run, &digest);
276277 try populateGeneratedPathsCreateDirs(arena, run_index, maker, output_dir_path, output_placeholders.items, argv_list.items);
277278 try runCommand(arena, run, run_index, maker, progress_node, argv_list.items, has_side_effects, output_dir_path, null);
278279 if (!has_side_effects) try step.finalizeManifestAndWatch(maker, &man);
......@@ -291,7 +292,7 @@ pub fn make(
291292 const arg = placeholder.arg_index.get(conf);
292293 switch (arg.flags.tag) {
293294 .output_file => if (arg.flags.dep_file) {
294 const generated_path = maker.generatedPath(arg.generated.value.?).*;
295 const generated_path = maker.generatedPath(arg.generated.value.?);
295296 if (has_side_effects) {
296297 var diagnostic: Cache.DepTokenizer.Token = undefined;
297298 man.addInputDepFile(generated_path, &diagnostic) catch |err| switch (err) {
......@@ -362,8 +363,8 @@ pub fn make(
362363
363364 if (!has_side_effects) try step.finalizeManifestAndWatch(maker, &man);
364365
365 try populateGeneratedStdIo(maker, &conf_run, cache_root, &digest);
366 try populateGeneratedPaths(maker, output_placeholders.items, cache_root, &digest);
366 try populateGeneratedStdIo(maker, &conf_run, &digest);
367 try populateGeneratedPaths(maker, output_placeholders.items, &digest);
367368
368369 // The utility functions that spawn the child process must unconditionally allocate
369370 // the failed command because at that point it is not known whether the step will
......@@ -1575,8 +1576,7 @@ pub fn rerunInFuzzMode(
15751576 const file_path: Path = if (producer_index == conf_run.producer.value.?)
15761577 run.rebuilt_executable.?
15771578 else
1578 producer_make_comp.installed_path orelse
1579 maker.generatedPath(producer.generated_bin.value.?).*;
1579 producer_make_comp.installed_path orelse maker.generatedPath(producer.generated_bin.value.?);
15801580 argv_list.appendAssumeCapacity(try mem.concat(arena, u8, &.{
15811581 prefix, try convertPathArg(arena, run_index, maker, file_path, arg.flags.make_absolute), suffix,
15821582 }));
......@@ -1615,20 +1615,15 @@ pub fn rerunInFuzzMode(
16151615fn populateGeneratedPaths(
16161616 maker: *Maker,
16171617 output_placeholders: []const IndexedOutput,
1618 cache_root: Cache.Directory,
16191618 digest: *const Cache.HexDigest,
16201619) !void {
16211620 const conf = &maker.scanned_config.configuration;
1622 const graph = maker.graph;
16231621
16241622 for (output_placeholders) |placeholder| {
16251623 const arg = placeholder.arg_index.get(conf);
1626 maker.generatedPath(arg.generated.value.?).* = .{
1627 .root_dir = cache_root,
1628 .sub_path = try Dir.path.join(graph.arena, &.{
1629 "o", digest, arg.basename.value.?.slice(conf),
1630 }),
1631 };
1624 _ = try maker.setGeneratedPath(arg.generated.value.?, .local_cache, &.{
1625 "o", digest, arg.basename.value.?.slice(conf),
1626 });
16321627 }
16331628}
16341629
......@@ -1652,10 +1647,9 @@ fn populateGeneratedPathsCreateDirs(
16521647 const suffix = if (arg.suffix.value) |p| p.slice(conf) else "";
16531648 const basename = arg.basename.value.?.slice(conf);
16541649
1655 const generated_path: Path = .{
1656 .root_dir = cache_root,
1657 .sub_path = try Dir.path.join(graph.arena, &.{ output_dir_path, basename }),
1658 };
1650 const generated_path = try maker.setGeneratedPath(arg.generated.value.?, .local_cache, &.{
1651 output_dir_path, basename,
1652 });
16591653 const create_path: Path = .{
16601654 .root_dir = cache_root,
16611655 .sub_path = switch (arg.flags.tag) {
......@@ -1667,8 +1661,6 @@ fn populateGeneratedPathsCreateDirs(
16671661 create_path.root_dir.handle.createDirPath(io, create_path.sub_path) catch |err|
16681662 return step.fail(maker, "unable to make path {f}: {t}", .{ create_path, err });
16691663
1670 maker.generatedPath(arg.generated.value.?).* = generated_path;
1671
16721664 const arg_output_path = try convertPathArg(arena, run_index, maker, generated_path, arg.flags.make_absolute);
16731665 argv[placeholder.index] = try mem.concat(arena, u8, &.{ prefix, arg_output_path, suffix });
16741666 }
......@@ -1677,28 +1669,20 @@ fn populateGeneratedPathsCreateDirs(
16771669fn populateGeneratedStdIo(
16781670 maker: *Maker,
16791671 conf_run: *const Configuration.Step.Run,
1680 cache_root: Cache.Directory,
16811672 digest: *const Cache.HexDigest,
16821673) !void {
16831674 const conf = &maker.scanned_config.configuration;
1684 const graph = maker.graph;
16851675
16861676 if (conf_run.captured_stdout.value) |captured| {
1687 maker.generatedPath(captured.generated_file).* = .{
1688 .root_dir = cache_root,
1689 .sub_path = try Dir.path.join(graph.arena, &.{
1690 "o", digest, captured.basename.slice(conf),
1691 }),
1692 };
1677 _ = try maker.setGeneratedPath(captured.generated_file, .local_cache, &.{
1678 "o", digest, captured.basename.slice(conf),
1679 });
16931680 }
16941681
16951682 if (conf_run.captured_stderr.value) |captured| {
1696 maker.generatedPath(captured.generated_file).* = .{
1697 .root_dir = cache_root,
1698 .sub_path = try Dir.path.join(graph.arena, &.{
1699 "o", digest, captured.basename.slice(conf),
1700 }),
1701 };
1683 _ = try maker.setGeneratedPath(captured.generated_file, .local_cache, &.{
1684 "o", digest, captured.basename.slice(conf),
1685 });
17021686 }
17031687}
17041688
......@@ -1735,7 +1719,6 @@ fn runCommand(
17351719 const gpa = maker.gpa;
17361720 const step = maker.stepByIndex(run_index);
17371721 const io = graph.io;
1738 const cache_root = graph.local_cache_root;
17391722 const conf = &maker.scanned_config.configuration;
17401723 const conf_step = run_index.ptr(conf);
17411724 const conf_run = conf_step.extended.get(conf.extra).run;
......@@ -1989,13 +1972,9 @@ fn runCommand(
19891972 },
19901973 }) |*stream| {
19911974 if (stream.captured) |captured| {
1992 const output_path: Path = .{
1993 .root_dir = cache_root,
1994 .sub_path = try Dir.path.join(graph.arena, &.{
1995 output_dir_path, captured.basename.slice(conf),
1996 }),
1997 };
1998 maker.generatedPath(captured.generated_file).* = output_path;
1975 const output_path = try maker.setGeneratedPath(captured.generated_file, .local_cache, &.{
1976 output_dir_path, captured.basename.slice(conf),
1977 });
19991978
20001979 const sub_path_parent = output_path.dirname().?;
20011980 sub_path_parent.root_dir.handle.createDirPath(io, sub_path_parent.sub_path) catch |err|
lib/compiler/Maker/Step/TranslateC.zig+4-3
......@@ -143,13 +143,14 @@ pub fn make(
143143 argv.appendAssumeCapacity(c_source_path);
144144
145145 argv.appendAssumeCapacity("--listen=-");
146 const output_dir_path = (Step.evalZigProcess(step_index, maker, argv.items, progress_node, false) catch |err| switch (err) {
146 const opt_cache_digest = Step.evalZigProcess(step_index, maker, argv.items, progress_node, false) catch |err| switch (err) {
147147 error.NeedCompileErrorCheck => unreachable,
148148 else => |e| return e,
149 }).?;
149 };
150 const o_hex_digest = opt_cache_digest.toHex().?;
150151
151152 const stem = Io.Dir.path.stem(Io.Dir.path.basename(c_source_path));
152153 const out_basename = try arena.print("{s}.zig", .{stem});
153154
154 maker.generatedPath(conf_tc.output_file).* = try output_dir_path.join(arena, out_basename);
155 _ = try maker.setGeneratedPath(conf_tc.output_file, .local_cache, &.{ "o", &o_hex_digest, out_basename });
155156}
lib/compiler/Maker/Step/WriteFile.zig+6-18
......@@ -24,7 +24,6 @@ pub fn make(
2424 const conf = &maker.scanned_config.configuration;
2525 const conf_step = step_index.ptr(conf);
2626 const conf_wf = conf_step.extended.get(conf.extra).write_file;
27 const cache_root = graph.local_cache_root;
2827 const directories = conf_wf.directories.slice;
2928
3029 const open_dir_cache = try arena.alloc(Io.Dir, directories.len);
......@@ -106,25 +105,17 @@ pub fn make(
106105
107106 if (try step.cacheHit(maker, &man, progress_node)) {
108107 const digest = man.hitDigestHex();
109 maker.generatedPath(conf_wf.generated_directory).* = .{
110 .root_dir = cache_root,
111 .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }),
112 };
108 _ = try maker.setGeneratedPath(conf_wf.generated_directory, .local_cache, &.{ "o", &digest });
113109 assert(step.result_cached);
114110 return;
115111 }
116112
117113 const digest = man.missDigestHex();
118 const out_path: Path = .{
119 .root_dir = cache_root,
120 .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest }),
121 };
114 const out_path = try maker.setGeneratedPath(conf_wf.generated_directory, .local_cache, &.{ "o", &digest });
122115
123116 progress_node.setEstimatedTotalItems(total_items);
124117 try operate(maker, step_index, open_dir_cache, out_path, progress_node);
125118 try step.finalizeManifest(maker, &man);
126
127 maker.generatedPath(conf_wf.generated_directory).* = out_path;
128119 },
129120 .tmp => {
130121 step.result_cached = false;
......@@ -133,20 +124,17 @@ pub fn make(
133124 io.random(@ptrCast(&rand_int));
134125 const hex_digest = std.fmt.hex(rand_int);
135126
136 const out_path: Path = .{
137 .root_dir = cache_root,
138 .sub_path = try Io.Dir.path.join(arena, &.{ "tmp", &hex_digest }),
139 };
127 const out_path = try maker.setGeneratedPath(conf_wf.generated_directory, .local_cache, &.{
128 "tmp", &hex_digest,
129 });
140130
141131 try operate(maker, step_index, open_dir_cache, out_path, progress_node);
142
143 maker.generatedPath(conf_wf.generated_directory).* = out_path;
144132 },
145133 .mutate => {
146134 step.result_cached = false;
147135 const root_path = try maker.resolveLazyPathIndex(arena, conf_wf.mutate_path.value.?, step_index);
148136 try operate(maker, step_index, open_dir_cache, root_path, progress_node);
149 maker.generatedPath(conf_wf.generated_directory).* = root_path;
137 _ = try maker.setGeneratedPathPath(conf_wf.generated_directory, root_path);
150138 },
151139 }
152140}