authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-01-03 13:23:40-08:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-01-03 13:23:40-08:00
logf64205b4456a81229886019e3621132ae150e053
tree7f649b262725149ae9e644ebb40d5c6b504481c1
parentce480dedbb1e8b9a13a0f138d8aa1ec04b7884c8
parent65878c16ee3e6677912343328e39c87c8f89c784
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #18262 from ziglang/fix-18259

std.Build.Step.Run: fix depfile support

2 files changed, 108 insertions(+), 40 deletions(-)

lib/std/Build/Cache.zig+15-14
...@@ -179,6 +179,7 @@ fn getPrefixSubpath(allocator: Allocator, prefix: []const u8, path: []u8) ![]u8...@@ -179,6 +179,7 @@ fn getPrefixSubpath(allocator: Allocator, prefix: []const u8, path: []u8) ![]u8
179pub const bin_digest_len = 16;179pub const bin_digest_len = 16;
180pub const hex_digest_len = bin_digest_len * 2;180pub const hex_digest_len = bin_digest_len * 2;
181pub const BinDigest = [bin_digest_len]u8;181pub const BinDigest = [bin_digest_len]u8;
182pub const HexDigest = [hex_digest_len]u8;
182183
183/// This is currently just an arbitrary non-empty string that can't match another manifest line.184/// This is currently just an arbitrary non-empty string that can't match another manifest line.
184const manifest_header = "0";185const manifest_header = "0";
...@@ -300,11 +301,11 @@ pub const HashHelper = struct {...@@ -300,11 +301,11 @@ pub const HashHelper = struct {
300 }301 }
301302
302 /// Returns a hex encoded hash of the inputs, mutating the state of the hasher.303 /// Returns a hex encoded hash of the inputs, mutating the state of the hasher.
303 pub fn final(hh: *HashHelper) [hex_digest_len]u8 {304 pub fn final(hh: *HashHelper) HexDigest {
304 var bin_digest: BinDigest = undefined;305 var bin_digest: BinDigest = undefined;
305 hh.hasher.final(&bin_digest);306 hh.hasher.final(&bin_digest);
306307
307 var out_digest: [hex_digest_len]u8 = undefined;308 var out_digest: HexDigest = undefined;
308 _ = fmt.bufPrint(309 _ = fmt.bufPrint(
309 &out_digest,310 &out_digest,
310 "{s}",311 "{s}",
...@@ -360,7 +361,7 @@ pub const Manifest = struct {...@@ -360,7 +361,7 @@ pub const Manifest = struct {
360 // will then use the same timestamp, to avoid unnecessary filesystem writes.361 // will then use the same timestamp, to avoid unnecessary filesystem writes.
361 want_refresh_timestamp: bool = true,362 want_refresh_timestamp: bool = true,
362 files: std.ArrayListUnmanaged(File) = .{},363 files: std.ArrayListUnmanaged(File) = .{},
363 hex_digest: [hex_digest_len]u8,364 hex_digest: HexDigest,
364 /// Populated when hit() returns an error because of one365 /// Populated when hit() returns an error because of one
365 /// of the files listed in the manifest.366 /// of the files listed in the manifest.
366 failed_file_index: ?usize = null,367 failed_file_index: ?usize = null,
...@@ -843,7 +844,7 @@ pub const Manifest = struct {...@@ -843,7 +844,7 @@ pub const Manifest = struct {
843 }844 }
844845
845 /// Returns a hex encoded hash of the inputs.846 /// Returns a hex encoded hash of the inputs.
846 pub fn final(self: *Manifest) [hex_digest_len]u8 {847 pub fn final(self: *Manifest) HexDigest {
847 assert(self.manifest_file != null);848 assert(self.manifest_file != null);
848849
849 // We don't close the manifest file yet, because we want to850 // We don't close the manifest file yet, because we want to
...@@ -855,7 +856,7 @@ pub const Manifest = struct {...@@ -855,7 +856,7 @@ pub const Manifest = struct {
855 var bin_digest: BinDigest = undefined;856 var bin_digest: BinDigest = undefined;
856 self.hash.hasher.final(&bin_digest);857 self.hash.hasher.final(&bin_digest);
857858
858 var out_digest: [hex_digest_len]u8 = undefined;859 var out_digest: HexDigest = undefined;
859 _ = fmt.bufPrint(860 _ = fmt.bufPrint(
860 &out_digest,861 &out_digest,
861 "{s}",862 "{s}",
...@@ -1035,8 +1036,8 @@ test "cache file and then recall it" {...@@ -1035,8 +1036,8 @@ test "cache file and then recall it" {
1035 std.time.sleep(1);1036 std.time.sleep(1);
1036 }1037 }
10371038
1038 var digest1: [hex_digest_len]u8 = undefined;1039 var digest1: HexDigest = undefined;
1039 var digest2: [hex_digest_len]u8 = undefined;1040 var digest2: HexDigest = undefined;
10401041
1041 {1042 {
1042 var cache = Cache{1043 var cache = Cache{
...@@ -1103,8 +1104,8 @@ test "check that changing a file makes cache fail" {...@@ -1103,8 +1104,8 @@ test "check that changing a file makes cache fail" {
1103 std.time.sleep(1);1104 std.time.sleep(1);
1104 }1105 }
11051106
1106 var digest1: [hex_digest_len]u8 = undefined;1107 var digest1: HexDigest = undefined;
1107 var digest2: [hex_digest_len]u8 = undefined;1108 var digest2: HexDigest = undefined;
11081109
1109 {1110 {
1110 var cache = Cache{1111 var cache = Cache{
...@@ -1166,8 +1167,8 @@ test "no file inputs" {...@@ -1166,8 +1167,8 @@ test "no file inputs" {
11661167
1167 const temp_manifest_dir = "no_file_inputs_manifest_dir";1168 const temp_manifest_dir = "no_file_inputs_manifest_dir";
11681169
1169 var digest1: [hex_digest_len]u8 = undefined;1170 var digest1: HexDigest = undefined;
1170 var digest2: [hex_digest_len]u8 = undefined;1171 var digest2: HexDigest = undefined;
11711172
1172 var cache = Cache{1173 var cache = Cache{
1173 .gpa = testing.allocator,1174 .gpa = testing.allocator,
...@@ -1225,9 +1226,9 @@ test "Manifest with files added after initial hash work" {...@@ -1225,9 +1226,9 @@ test "Manifest with files added after initial hash work" {
1225 std.time.sleep(1);1226 std.time.sleep(1);
1226 }1227 }
12271228
1228 var digest1: [hex_digest_len]u8 = undefined;1229 var digest1: HexDigest = undefined;
1229 var digest2: [hex_digest_len]u8 = undefined;1230 var digest2: HexDigest = undefined;
1230 var digest3: [hex_digest_len]u8 = undefined;1231 var digest3: HexDigest = undefined;
12311232
1232 {1233 {
1233 var cache = Cache{1234 var cache = Cache{
lib/std/Build/Step/Run.zig+93-26
...@@ -243,7 +243,7 @@ pub fn addDepFileOutputArg(self: *Run, basename: []const u8) std.Build.LazyPath...@@ -243,7 +243,7 @@ pub fn addDepFileOutputArg(self: *Run, basename: []const u8) std.Build.LazyPath
243/// Add a prefixed path argument to a dep file (.d) for the child process to243/// Add a prefixed path argument to a dep file (.d) for the child process to
244/// write its discovered additional dependencies.244/// write its discovered additional dependencies.
245/// Only one dep file argument is allowed by instance.245/// Only one dep file argument is allowed by instance.
246pub fn addPrefixedDepFileOutputArg(self: *Run, prefix: []const u8, basename: []const u8) void {246pub fn addPrefixedDepFileOutputArg(self: *Run, prefix: []const u8, basename: []const u8) std.Build.LazyPath {
247 assert(self.dep_output_file == null);247 assert(self.dep_output_file == null);
248248
249 const b = self.step.owner;249 const b = self.step.owner;
...@@ -258,6 +258,8 @@ pub fn addPrefixedDepFileOutputArg(self: *Run, prefix: []const u8, basename: []c...@@ -258,6 +258,8 @@ pub fn addPrefixedDepFileOutputArg(self: *Run, prefix: []const u8, basename: []c
258 self.dep_output_file = dep_file;258 self.dep_output_file = dep_file;
259259
260 self.argv.append(.{ .output = dep_file }) catch @panic("OOM");260 self.argv.append(.{ .output = dep_file }) catch @panic("OOM");
261
262 return .{ .generated = &dep_file.generated_file };
261}263}
262264
263pub fn addArg(self: *Run, arg: []const u8) void {265pub fn addArg(self: *Run, arg: []const u8) void {
...@@ -448,6 +450,10 @@ fn checksContainStderr(checks: []const StdIo.Check) bool {...@@ -448,6 +450,10 @@ fn checksContainStderr(checks: []const StdIo.Check) bool {
448 return false;450 return false;
449}451}
450452
453const IndexedOutput = struct {
454 index: usize,
455 output: *Output,
456};
451fn make(step: *Step, prog_node: *std.Progress.Node) !void {457fn make(step: *Step, prog_node: *std.Progress.Node) !void {
452 const b = step.owner;458 const b = step.owner;
453 const arena = b.allocator;459 const arena = b.allocator;
...@@ -455,10 +461,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {...@@ -455,10 +461,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
455 const has_side_effects = self.hasSideEffects();461 const has_side_effects = self.hasSideEffects();
456462
457 var argv_list = ArrayList([]const u8).init(arena);463 var argv_list = ArrayList([]const u8).init(arena);
458 var output_placeholders = ArrayList(struct {464 var output_placeholders = ArrayList(IndexedOutput).init(arena);
459 index: usize,
460 output: *Output,
461 }).init(arena);
462465
463 var man = b.cache.obtain();466 var man = b.cache.obtain();
464 defer man.deinit();467 defer man.deinit();
...@@ -540,32 +543,25 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {...@@ -540,32 +543,25 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
540 if (try step.cacheHit(&man)) {543 if (try step.cacheHit(&man)) {
541 // cache hit, skip running command544 // cache hit, skip running command
542 const digest = man.final();545 const digest = man.final();
543 for (output_placeholders.items) |placeholder| {
544 placeholder.output.generated_file.path = try b.cache_root.join(arena, &.{
545 "o", &digest, placeholder.output.basename,
546 });
547 }
548546
549 if (self.captured_stdout) |output| {547 try populateGeneratedPaths(
550 output.generated_file.path = try b.cache_root.join(arena, &.{548 arena,
551 "o", &digest, output.basename,549 output_placeholders.items,
552 });550 self.captured_stdout,
553 }551 self.captured_stderr,
554552 b.cache_root,
555 if (self.captured_stderr) |output| {553 &digest,
556 output.generated_file.path = try b.cache_root.join(arena, &.{554 );
557 "o", &digest, output.basename,
558 });
559 }
560555
561 step.result_cached = true;556 step.result_cached = true;
562 return;557 return;
563 }558 }
564559
565 const digest = man.final();560 const rand_int = std.crypto.random.int(u64);
561 const tmp_dir_path = "tmp" ++ fs.path.sep_str ++ std.Build.hex64(rand_int);
566562
567 for (output_placeholders.items) |placeholder| {563 for (output_placeholders.items) |placeholder| {
568 const output_components = .{ "o", &digest, placeholder.output.basename };564 const output_components = .{ tmp_dir_path, placeholder.output.basename };
569 const output_sub_path = try fs.path.join(arena, &output_components);565 const output_sub_path = try fs.path.join(arena, &output_components);
570 const output_sub_dir_path = fs.path.dirname(output_sub_path).?;566 const output_sub_dir_path = fs.path.dirname(output_sub_path).?;
571 b.cache_root.handle.makePath(output_sub_dir_path) catch |err| {567 b.cache_root.handle.makePath(output_sub_dir_path) catch |err| {
...@@ -582,12 +578,83 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {...@@ -582,12 +578,83 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
582 argv_list.items[placeholder.index] = cli_arg;578 argv_list.items[placeholder.index] = cli_arg;
583 }579 }
584580
585 try runCommand(self, argv_list.items, has_side_effects, &digest, prog_node);581 try runCommand(self, argv_list.items, has_side_effects, tmp_dir_path, prog_node);
586582
587 if (self.dep_output_file) |dep_output_file|583 if (self.dep_output_file) |dep_output_file|
588 try man.addDepFilePost(std.fs.cwd(), dep_output_file.generated_file.getPath());584 try man.addDepFilePost(std.fs.cwd(), dep_output_file.generated_file.getPath());
589585
586 const digest = man.final();
587
588 const any_output = output_placeholders.items.len > 0 or
589 self.captured_stdout != null or self.captured_stderr != null;
590
591 // Rename into place
592 if (any_output) {
593 const o_sub_path = "o" ++ fs.path.sep_str ++ &digest;
594
595 b.cache_root.handle.rename(tmp_dir_path, o_sub_path) catch |err| {
596 if (err == error.PathAlreadyExists) {
597 b.cache_root.handle.deleteTree(o_sub_path) catch |del_err| {
598 return step.fail("unable to remove dir '{}'{s}: {s}", .{
599 b.cache_root,
600 tmp_dir_path,
601 @errorName(del_err),
602 });
603 };
604 b.cache_root.handle.rename(tmp_dir_path, o_sub_path) catch |retry_err| {
605 return step.fail("unable to rename dir '{}{s}' to '{}{s}': {s}", .{
606 b.cache_root, tmp_dir_path,
607 b.cache_root, o_sub_path,
608 @errorName(retry_err),
609 });
610 };
611 } else {
612 return step.fail("unable to rename dir '{}{s}' to '{}{s}': {s}", .{
613 b.cache_root, tmp_dir_path,
614 b.cache_root, o_sub_path,
615 @errorName(err),
616 });
617 }
618 };
619 }
620
590 try step.writeManifest(&man);621 try step.writeManifest(&man);
622
623 try populateGeneratedPaths(
624 arena,
625 output_placeholders.items,
626 self.captured_stdout,
627 self.captured_stderr,
628 b.cache_root,
629 &digest,
630 );
631}
632
633fn populateGeneratedPaths(
634 arena: std.mem.Allocator,
635 output_placeholders: []const IndexedOutput,
636 captured_stdout: ?*Output,
637 captured_stderr: ?*Output,
638 cache_root: Build.Cache.Directory,
639 digest: *const Build.Cache.HexDigest,
640) !void {
641 for (output_placeholders) |placeholder| {
642 placeholder.output.generated_file.path = try cache_root.join(arena, &.{
643 "o", digest, placeholder.output.basename,
644 });
645 }
646
647 if (captured_stdout) |output| {
648 output.generated_file.path = try cache_root.join(arena, &.{
649 "o", digest, output.basename,
650 });
651 }
652
653 if (captured_stderr) |output| {
654 output.generated_file.path = try cache_root.join(arena, &.{
655 "o", digest, output.basename,
656 });
657 }
591}658}
592659
593fn formatTerm(660fn formatTerm(
...@@ -639,7 +706,7 @@ fn runCommand(...@@ -639,7 +706,7 @@ fn runCommand(
639 self: *Run,706 self: *Run,
640 argv: []const []const u8,707 argv: []const []const u8,
641 has_side_effects: bool,708 has_side_effects: bool,
642 digest: ?*const [std.Build.Cache.hex_digest_len]u8,709 tmp_dir_path: ?[]const u8,
643 prog_node: *std.Progress.Node,710 prog_node: *std.Progress.Node,
644) !void {711) !void {
645 const step = &self.step;712 const step = &self.step;
...@@ -812,7 +879,7 @@ fn runCommand(...@@ -812,7 +879,7 @@ fn runCommand(
812 },879 },
813 }) |stream| {880 }) |stream| {
814 if (stream.captured) |output| {881 if (stream.captured) |output| {
815 const output_components = .{ "o", digest.?, output.basename };882 const output_components = .{ tmp_dir_path.?, output.basename };
816 const output_path = try b.cache_root.join(arena, &output_components);883 const output_path = try b.cache_root.join(arena, &output_components);
817 output.generated_file.path = output_path;884 output.generated_file.path = output_path;
818885