authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-12-10 17:43:42-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-12-10 18:11:12-08:00
logd37ee79535188263bd6a907eb26a48364c4c12f2
tree9cff3c562afce6f3ddedfa236a7a819d92a9728a
parentc172877b81f4eff50cf214eb553c9df108fbd9eb

std.Build.Cache.hit: more discipline in error handling

Previous commits 2b0929929d67e222ca6a9523a3a594ed456c4a51 4ea2f441df36cec61e1017f4d795d4037326c98c had this text: > There are no dir components, so you would think that this was > unreachable, however we have observed on macOS two processes racing to > do openat() with O_CREAT manifest in ENOENT. This appears to have been a misunderstanding based on the issue report #12138 and corresponding PR #12139 in which the steps to reproduce removed the cache directory in a loop which also executed detached Zig compiler processes. There is no evidence for the macOS kernel bug however the ENOENT is easily explained by the removal of the cache directory. This commit reverts those commits, ultimately reporting the ENOENT as an error rather than repeating the create file operation. However this commit also adds an explicit error set to `std.Build.Cache.hit` as well as changing the `failed_file_index` to a proper diagnostic field that fully communicates what failed, leading to more informative error messages on failure to check the cache. The equivalent failure when occuring for AstGen performs a fatal process kill, reasoning being that the compiler has an invariant of the cache directory not being yanked out from underneath it while executing. This could be made a more granular error in the future but I suspect such thing is not valuable to pursue. Related to #18340 but does not solve it.

6 files changed, 140 insertions(+), 49 deletions(-)

lib/std/Build/Cache.zig+86-31
...@@ -40,7 +40,7 @@ pub fn addPrefix(cache: *Cache, directory: Directory) void {...@@ -40,7 +40,7 @@ pub fn addPrefix(cache: *Cache, directory: Directory) void {
4040
41/// Be sure to call `Manifest.deinit` after successful initialization.41/// Be sure to call `Manifest.deinit` after successful initialization.
42pub fn obtain(cache: *Cache) Manifest {42pub fn obtain(cache: *Cache) Manifest {
43 return Manifest{43 return .{
44 .cache = cache,44 .cache = cache,
45 .hash = cache.hash,45 .hash = cache.hash,
46 .manifest_file = null,46 .manifest_file = null,
...@@ -99,9 +99,9 @@ fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath {...@@ -99,9 +99,9 @@ fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath {
99}99}
100100
101fn getPrefixSubpath(allocator: Allocator, prefix: []const u8, path: []u8) ![]u8 {101fn getPrefixSubpath(allocator: Allocator, prefix: []const u8, path: []u8) ![]u8 {
102 const relative = try std.fs.path.relative(allocator, prefix, path);102 const relative = try fs.path.relative(allocator, prefix, path);
103 errdefer allocator.free(relative);103 errdefer allocator.free(relative);
104 var component_iterator = std.fs.path.NativeComponentIterator.init(relative) catch {104 var component_iterator = fs.path.NativeComponentIterator.init(relative) catch {
105 return error.NotASubPath;105 return error.NotASubPath;
106 };106 };
107 if (component_iterator.root() != null) {107 if (component_iterator.root() != null) {
...@@ -327,13 +327,27 @@ pub const Manifest = struct {...@@ -327,13 +327,27 @@ pub const Manifest = struct {
327 want_refresh_timestamp: bool = true,327 want_refresh_timestamp: bool = true,
328 files: Files = .{},328 files: Files = .{},
329 hex_digest: HexDigest,329 hex_digest: HexDigest,
330 /// Populated when hit() returns an error because of one330 diagnostic: Diagnostic = .none,
331 /// of the files listed in the manifest.
332 failed_file_index: ?usize = null,
333 /// Keeps track of the last time we performed a file system write to observe331 /// Keeps track of the last time we performed a file system write to observe
334 /// what time the file system thinks it is, according to its own granularity.332 /// what time the file system thinks it is, according to its own granularity.
335 recent_problematic_timestamp: i128 = 0,333 recent_problematic_timestamp: i128 = 0,
336334
335 pub const Diagnostic = union(enum) {
336 none,
337 manifest_create: fs.File.OpenError,
338 manifest_read: fs.File.ReadError,
339 manifest_lock: fs.File.LockError,
340 file_open: FileOp,
341 file_stat: FileOp,
342 file_read: FileOp,
343 file_hash: FileOp,
344
345 pub const FileOp = struct {
346 file_index: usize,
347 err: anyerror,
348 };
349 };
350
337 pub const Files = std.ArrayHashMapUnmanaged(File, void, FilesContext, false);351 pub const Files = std.ArrayHashMapUnmanaged(File, void, FilesContext, false);
338352
339 pub const FilesContext = struct {353 pub const FilesContext = struct {
...@@ -452,6 +466,15 @@ pub const Manifest = struct {...@@ -452,6 +466,15 @@ pub const Manifest = struct {
452 return self.addDepFileMaybePost(dir, dep_file_basename);466 return self.addDepFileMaybePost(dir, dep_file_basename);
453 }467 }
454468
469 pub const HitError = error{
470 /// Unable to check the cache for a reason that has been recorded into
471 /// the `diagnostic` field.
472 CacheCheckFailed,
473 /// A cache manifest file exists however it could not be parsed.
474 InvalidFormat,
475 OutOfMemory,
476 };
477
455 /// Check the cache to see if the input exists in it. If it exists, returns `true`.478 /// Check the cache to see if the input exists in it. If it exists, returns `true`.
456 /// A hex encoding of its hash is available by calling `final`.479 /// A hex encoding of its hash is available by calling `final`.
457 ///480 ///
...@@ -464,11 +487,11 @@ pub const Manifest = struct {...@@ -464,11 +487,11 @@ pub const Manifest = struct {
464 /// The lock on the manifest file is released when `deinit` is called. As another487 /// The lock on the manifest file is released when `deinit` is called. As another
465 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent488 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
466 /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.489 /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
467 pub fn hit(self: *Manifest) !bool {490 pub fn hit(self: *Manifest) HitError!bool {
468 const gpa = self.cache.gpa;491 const gpa = self.cache.gpa;
469 assert(self.manifest_file == null);492 assert(self.manifest_file == null);
470493
471 self.failed_file_index = null;494 self.diagnostic = .none;
472495
473 const ext = ".txt";496 const ext = ".txt";
474 var manifest_file_path: [hex_digest_len + ext.len]u8 = undefined;497 var manifest_file_path: [hex_digest_len + ext.len]u8 = undefined;
...@@ -496,17 +519,19 @@ pub const Manifest = struct {...@@ -496,17 +519,19 @@ pub const Manifest = struct {
496 break;519 break;
497 } else |err| switch (err) {520 } else |err| switch (err) {
498 error.WouldBlock => {521 error.WouldBlock => {
499 self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{522 self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{
500 .mode = .read_write,523 .mode = .read_write,
501 .lock = .shared,524 .lock = .shared,
502 });525 }) catch |e| {
526 self.diagnostic = .{ .manifest_create = e };
527 return error.CacheCheckFailed;
528 };
503 break;529 break;
504 },530 },
505 // There are no dir components, so you would think that this was531 else => |e| {
506 // unreachable, however we have observed on macOS two processes racing532 self.diagnostic = .{ .manifest_create = e };
507 // to do openat() with O_CREAT manifest in ENOENT.533 return error.CacheCheckFailed;
508 error.FileNotFound => continue,534 },
509 else => |e| return e,
510 }535 }
511 }536 }
512537
...@@ -514,7 +539,14 @@ pub const Manifest = struct {...@@ -514,7 +539,14 @@ pub const Manifest = struct {
514539
515 const input_file_count = self.files.entries.len;540 const input_file_count = self.files.entries.len;
516 while (true) : (self.unhit(bin_digest, input_file_count)) {541 while (true) : (self.unhit(bin_digest, input_file_count)) {
517 const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max);542 const file_contents = self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max) catch |err| switch (err) {
543 error.OutOfMemory => return error.OutOfMemory,
544 error.StreamTooLong => return error.OutOfMemory,
545 else => |e| {
546 self.diagnostic = .{ .manifest_read = e };
547 return error.CacheCheckFailed;
548 },
549 };
518 defer gpa.free(file_contents);550 defer gpa.free(file_contents);
519551
520 var any_file_changed = false;552 var any_file_changed = false;
...@@ -526,8 +558,11 @@ pub const Manifest = struct {...@@ -526,8 +558,11 @@ pub const Manifest = struct {
526 while (idx < input_file_count) : (idx += 1) {558 while (idx < input_file_count) : (idx += 1) {
527 const ch_file = &self.files.keys()[idx];559 const ch_file = &self.files.keys()[idx];
528 self.populateFileHash(ch_file) catch |err| {560 self.populateFileHash(ch_file) catch |err| {
529 self.failed_file_index = idx;561 self.diagnostic = .{ .file_hash = .{
530 return err;562 .file_index = idx,
563 .err = err,
564 } };
565 return error.CacheCheckFailed;
531 };566 };
532 }567 }
533 return false;568 return false;
...@@ -605,13 +640,22 @@ pub const Manifest = struct {...@@ -605,13 +640,22 @@ pub const Manifest = struct {
605 if (try self.upgradeToExclusiveLock()) continue;640 if (try self.upgradeToExclusiveLock()) continue;
606 return false;641 return false;
607 },642 },
608 else => return error.CacheUnavailable,643 else => |e| {
644 self.diagnostic = .{ .file_open = .{
645 .file_index = idx,
646 .err = e,
647 } };
648 return error.CacheCheckFailed;
649 },
609 };650 };
610 defer this_file.close();651 defer this_file.close();
611652
612 const actual_stat = this_file.stat() catch |err| {653 const actual_stat = this_file.stat() catch |err| {
613 self.failed_file_index = idx;654 self.diagnostic = .{ .file_stat = .{
614 return err;655 .file_index = idx,
656 .err = err,
657 } };
658 return error.CacheCheckFailed;
615 };659 };
616 const size_match = actual_stat.size == cache_hash_file.stat.size;660 const size_match = actual_stat.size == cache_hash_file.stat.size;
617 const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime;661 const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime;
...@@ -634,8 +678,11 @@ pub const Manifest = struct {...@@ -634,8 +678,11 @@ pub const Manifest = struct {
634678
635 var actual_digest: BinDigest = undefined;679 var actual_digest: BinDigest = undefined;
636 hashFile(this_file, &actual_digest) catch |err| {680 hashFile(this_file, &actual_digest) catch |err| {
637 self.failed_file_index = idx;681 self.diagnostic = .{ .file_read = .{
638 return err;682 .file_index = idx,
683 .err = err,
684 } };
685 return error.CacheCheckFailed;
639 };686 };
640687
641 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {688 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
...@@ -662,17 +709,22 @@ pub const Manifest = struct {...@@ -662,17 +709,22 @@ pub const Manifest = struct {
662 if (try self.upgradeToExclusiveLock()) continue;709 if (try self.upgradeToExclusiveLock()) continue;
663 self.manifest_dirty = true;710 self.manifest_dirty = true;
664 while (idx < input_file_count) : (idx += 1) {711 while (idx < input_file_count) : (idx += 1) {
665 const ch_file = &self.files.keys()[idx];712 self.populateFileHash(&self.files.keys()[idx]) catch |err| {
666 self.populateFileHash(ch_file) catch |err| {713 self.diagnostic = .{ .file_hash = .{
667 self.failed_file_index = idx;714 .file_index = idx,
668 return err;715 .err = err,
716 } };
717 return error.CacheCheckFailed;
669 };718 };
670 }719 }
671 return false;720 return false;
672 }721 }
673722
674 if (self.want_shared_lock) {723 if (self.want_shared_lock) {
675 try self.downgradeToSharedLock();724 self.downgradeToSharedLock() catch |err| {
725 self.diagnostic = .{ .manifest_lock = err };
726 return error.CacheCheckFailed;
727 };
676 }728 }
677729
678 return true;730 return true;
...@@ -1010,7 +1062,7 @@ pub const Manifest = struct {...@@ -1010,7 +1062,7 @@ pub const Manifest = struct {
1010 self.have_exclusive_lock = false;1062 self.have_exclusive_lock = false;
1011 }1063 }
10121064
1013 fn upgradeToExclusiveLock(self: *Manifest) !bool {1065 fn upgradeToExclusiveLock(self: *Manifest) error{CacheCheckFailed}!bool {
1014 if (self.have_exclusive_lock) return false;1066 if (self.have_exclusive_lock) return false;
1015 assert(self.manifest_file != null);1067 assert(self.manifest_file != null);
10161068
...@@ -1022,7 +1074,10 @@ pub const Manifest = struct {...@@ -1022,7 +1074,10 @@ pub const Manifest = struct {
1022 // Here we intentionally have a period where the lock is released, in case there are1074 // Here we intentionally have a period where the lock is released, in case there are
1023 // other processes holding a shared lock.1075 // other processes holding a shared lock.
1024 manifest_file.unlock();1076 manifest_file.unlock();
1025 try manifest_file.lock(.exclusive);1077 manifest_file.lock(.exclusive) catch |err| {
1078 self.diagnostic = .{ .manifest_lock = err };
1079 return error.CacheCheckFailed;
1080 };
1026 }1081 }
1027 self.have_exclusive_lock = true;1082 self.have_exclusive_lock = true;
1028 return true;1083 return true;
...@@ -1132,7 +1187,7 @@ pub fn writeSmallFile(dir: fs.Dir, sub_path: []const u8, data: []const u8) !void...@@ -1132,7 +1187,7 @@ pub fn writeSmallFile(dir: fs.Dir, sub_path: []const u8, data: []const u8) !void
1132 }1187 }
1133}1188}
11341189
1135fn hashFile(file: fs.File, bin_digest: *[Hasher.mac_length]u8) !void {1190fn hashFile(file: fs.File, bin_digest: *[Hasher.mac_length]u8) fs.File.PReadError!void {
1136 var buf: [1024]u8 = undefined;1191 var buf: [1024]u8 = undefined;
1137 var hasher = hasher_init;1192 var hasher = hasher_init;
1138 var off: u64 = 0;1193 var off: u64 = 0;
lib/std/Build/Step.zig+18-5
...@@ -754,11 +754,24 @@ pub fn cacheHitAndWatch(s: *Step, man: *Build.Cache.Manifest) !bool {...@@ -754,11 +754,24 @@ pub fn cacheHitAndWatch(s: *Step, man: *Build.Cache.Manifest) !bool {
754 return is_hit;754 return is_hit;
755}755}
756756
757fn failWithCacheError(s: *Step, man: *const Build.Cache.Manifest, err: anyerror) anyerror {757fn failWithCacheError(s: *Step, man: *const Build.Cache.Manifest, err: Build.Cache.Manifest.HitError) error{ OutOfMemory, MakeFailed } {
758 const i = man.failed_file_index orelse return err;758 switch (err) {
759 const pp = man.files.keys()[i].prefixed_path;759 error.CacheCheckFailed => switch (man.diagnostic) {
760 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";760 .none => unreachable,
761 return s.fail("{s}: {s}/{s}", .{ @errorName(err), prefix, pp.sub_path });761 .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail("failed to check cache: {s} {s}", .{
762 @tagName(man.diagnostic), @errorName(e),
763 }),
764 .file_open, .file_stat, .file_read, .file_hash => |op| {
765 const pp = man.files.keys()[op.file_index].prefixed_path;
766 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
767 return s.fail("failed to check cache: '{s}{s}' {s} {s}", .{
768 prefix, pp.sub_path, @tagName(man.diagnostic), @errorName(op.err),
769 });
770 },
771 },
772 error.OutOfMemory => return error.OutOfMemory,
773 error.InvalidFormat => return s.fail("failed check cache: invalid manifest file format", .{}),
774 }
762}775}
763776
764/// Prefer `writeManifestAndWatch` unless you already added watch inputs777/// Prefer `writeManifestAndWatch` unless you already added watch inputs
lib/std/posix.zig+4
...@@ -1537,6 +1537,10 @@ pub const OpenError = error{...@@ -1537,6 +1537,10 @@ pub const OpenError = error{
1537 ProcessFdQuotaExceeded,1537 ProcessFdQuotaExceeded,
1538 SystemFdQuotaExceeded,1538 SystemFdQuotaExceeded,
1539 NoDevice,1539 NoDevice,
1540 /// Either:
1541 /// * One of the path components does not exist.
1542 /// * Cwd was used, but cwd has been deleted.
1543 /// * The path associated with the open directory handle has been deleted.
1540 FileNotFound,1544 FileNotFound,
15411545
1542 /// The path exceeded `max_path_bytes` bytes.1546 /// The path exceeded `max_path_bytes` bytes.
src/Compilation.zig+23-8
...@@ -2048,15 +2048,30 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {...@@ -2048,15 +2048,30 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
2048 whole.cache_manifest = &man;2048 whole.cache_manifest = &man;
2049 try addNonIncrementalStuffToCacheManifest(comp, arena, &man);2049 try addNonIncrementalStuffToCacheManifest(comp, arena, &man);
20502050
2051 const is_hit = man.hit() catch |err| {2051 const is_hit = man.hit() catch |err| switch (err) {
2052 const i = man.failed_file_index orelse return err;2052 error.CacheCheckFailed => switch (man.diagnostic) {
2053 const pp = man.files.keys()[i].prefixed_path;2053 .none => unreachable,
2054 const prefix = man.cache.prefixes()[pp.prefix];2054 .manifest_create, .manifest_read, .manifest_lock => |e| return comp.setMiscFailure(
2055 return comp.setMiscFailure(2055 .check_whole_cache,
2056 "failed to check cache: {s} {s}",
2057 .{ @tagName(man.diagnostic), @errorName(e) },
2058 ),
2059 .file_open, .file_stat, .file_read, .file_hash => |op| {
2060 const pp = man.files.keys()[op.file_index].prefixed_path;
2061 const prefix = man.cache.prefixes()[pp.prefix];
2062 return comp.setMiscFailure(
2063 .check_whole_cache,
2064 "failed to check cache: '{}{s}' {s} {s}",
2065 .{ prefix, pp.sub_path, @tagName(man.diagnostic), @errorName(op.err) },
2066 );
2067 },
2068 },
2069 error.OutOfMemory => return error.OutOfMemory,
2070 error.InvalidFormat => return comp.setMiscFailure(
2056 .check_whole_cache,2071 .check_whole_cache,
2057 "unable to check cache: stat file '{}{s}' failed: {s}",2072 "failed check cache: invalid manifest file format",
2058 .{ prefix, pp.sub_path, @errorName(err) },2073 .{},
2059 );2074 ),
2060 };2075 };
2061 if (is_hit) {2076 if (is_hit) {
2062 // In this case the cache hit contains the full set of file system inputs. Nice!2077 // In this case the cache hit contains the full set of file system inputs. Nice!
src/Zcu/PerThread.zig+8-4
...@@ -135,10 +135,14 @@ pub fn astGenFile(...@@ -135,10 +135,14 @@ pub fn astGenFile(
135 error.PipeBusy => unreachable, // it's not a pipe135 error.PipeBusy => unreachable, // it's not a pipe
136 error.NoDevice => unreachable, // it's not a pipe136 error.NoDevice => unreachable, // it's not a pipe
137 error.WouldBlock => unreachable, // not asking for non-blocking I/O137 error.WouldBlock => unreachable, // not asking for non-blocking I/O
138 // There are no dir components, so you would think that this was138 error.FileNotFound => {
139 // unreachable, however we have observed on macOS two processes racing139 // Since there are no dir components this could only occur if
140 // to do openat() with O_CREAT manifest in ENOENT.140 // `zir_dir` is deleted after the compiler process obtains an
141 error.FileNotFound => continue,141 // open directory handle.
142 std.process.fatal("cache directory '{}' unexpectedly removed during compiler execution", .{
143 cache_directory,
144 });
145 },
142146
143 else => |e| return e, // Retryable errors are handled at callsite.147 else => |e| return e, // Retryable errors are handled at callsite.
144 };148 };
src/link.zig+1-1
...@@ -768,7 +768,7 @@ pub const File = struct {...@@ -768,7 +768,7 @@ pub const File = struct {
768 /// TODO audit this error set. most of these should be collapsed into one error,768 /// TODO audit this error set. most of these should be collapsed into one error,
769 /// and Diags.Flags should be updated to convey the meaning to the user.769 /// and Diags.Flags should be updated to convey the meaning to the user.
770 pub const FlushError = error{770 pub const FlushError = error{
771 CacheUnavailable,771 CacheCheckFailed,
772 CurrentWorkingDirectoryUnlinked,772 CurrentWorkingDirectoryUnlinked,
773 DivisionByZero,773 DivisionByZero,
774 DllImportLibraryNotFound,774 DllImportLibraryNotFound,