| ... | ... | @@ -181,6 +181,12 @@ pub const Manifest = struct { |
| 181 | 181 | hash: HashHelper, |
| 182 | 182 | manifest_file: ?fs.File, |
| 183 | 183 | manifest_dirty: bool, |
| 184 | /// Set this flag to true before calling hit() in order to indicate that |
| 185 | /// upon a cache hit, the code using the cache will not modify the files |
| 186 | /// within the cache directory. This allows multiple processes to utilize |
| 187 | /// the same cache directory at the same time. |
| 188 | want_shared_lock: bool = true, |
| 189 | have_exclusive_lock: bool = false, |
| 184 | 190 | files: std.ArrayListUnmanaged(File) = .{}, |
| 185 | 191 | hex_digest: [hex_digest_len]u8, |
| 186 | 192 | /// Populated when hit() returns an error because of one |
| ... | ... | @@ -257,7 +263,9 @@ pub const Manifest = struct { |
| 257 | 263 | /// |
| 258 | 264 | /// This function will also acquire an exclusive lock to the manifest file. This means |
| 259 | 265 | /// that a process holding a Manifest will block any other process attempting to |
| 260 | | /// acquire the lock. |
| 266 | /// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the |
| 267 | /// manifest file to be locked in shared mode, and a cache miss guarantees the manifest |
| 268 | /// file to be locked in exclusive mode. |
| 261 | 269 | /// |
| 262 | 270 | /// The lock on the manifest file is released when `deinit` is called. As another |
| 263 | 271 | /// option, one may call `toOwnedLock` to obtain a smaller object which can represent |
| ... | ... | @@ -285,31 +293,62 @@ pub const Manifest = struct { |
| 285 | 293 | mem.copy(u8, &manifest_file_path, &self.hex_digest); |
| 286 | 294 | manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*; |
| 287 | 295 | |
| 288 | | if (self.files.items.len != 0) { |
| 289 | | self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{ |
| 290 | | .read = true, |
| 291 | | .truncate = false, |
| 292 | | .lock = .Exclusive, |
| 293 | | }); |
| 294 | | } else { |
| 296 | if (self.files.items.len == 0) { |
| 295 | 297 | // If there are no file inputs, we check if the manifest file exists instead of |
| 296 | 298 | // comparing the hashes on the files used for the cached item |
| 297 | | self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{ |
| 299 | while (true) { |
| 300 | if (self.cache.manifest_dir.openFile(&manifest_file_path, .{ |
| 301 | .read = true, |
| 302 | .write = true, |
| 303 | .lock = .Exclusive, |
| 304 | .lock_nonblocking = self.want_shared_lock, |
| 305 | })) |manifest_file| { |
| 306 | self.manifest_file = manifest_file; |
| 307 | self.have_exclusive_lock = true; |
| 308 | break; |
| 309 | } else |open_err| switch (open_err) { |
| 310 | error.WouldBlock => { |
| 311 | self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{ |
| 312 | .lock = .Shared, |
| 313 | }); |
| 314 | break; |
| 315 | }, |
| 316 | error.FileNotFound => { |
| 317 | if (self.cache.manifest_dir.createFile(&manifest_file_path, .{ |
| 318 | .read = true, |
| 319 | .truncate = false, |
| 320 | .lock = .Exclusive, |
| 321 | .lock_nonblocking = self.want_shared_lock, |
| 322 | })) |manifest_file| { |
| 323 | self.manifest_file = manifest_file; |
| 324 | self.manifest_dirty = true; |
| 325 | self.have_exclusive_lock = true; |
| 326 | return false; // cache miss; exclusive lock already held |
| 327 | } else |err| switch (err) { |
| 328 | error.WouldBlock => continue, |
| 329 | else => |e| return e, |
| 330 | } |
| 331 | }, |
| 332 | else => |e| return e, |
| 333 | } |
| 334 | } |
| 335 | } else { |
| 336 | if (self.cache.manifest_dir.createFile(&manifest_file_path, .{ |
| 298 | 337 | .read = true, |
| 299 | | .write = true, |
| 338 | .truncate = false, |
| 300 | 339 | .lock = .Exclusive, |
| 301 | | }) catch |err| switch (err) { |
| 302 | | error.FileNotFound => { |
| 303 | | self.manifest_dirty = true; |
| 304 | | self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{ |
| 305 | | .read = true, |
| 306 | | .truncate = false, |
| 307 | | .lock = .Exclusive, |
| 340 | .lock_nonblocking = self.want_shared_lock, |
| 341 | })) |manifest_file| { |
| 342 | self.manifest_file = manifest_file; |
| 343 | self.have_exclusive_lock = true; |
| 344 | } else |err| switch (err) { |
| 345 | error.WouldBlock => { |
| 346 | self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{ |
| 347 | .lock = .Shared, |
| 308 | 348 | }); |
| 309 | | return false; |
| 310 | 349 | }, |
| 311 | 350 | else => |e| return e, |
| 312 | | }; |
| 351 | } |
| 313 | 352 | } |
| 314 | 353 | |
| 315 | 354 | const file_contents = try self.manifest_file.?.reader().readAllAlloc(self.cache.gpa, manifest_file_size_max); |
| ... | ... | @@ -360,7 +399,10 @@ pub const Manifest = struct { |
| 360 | 399 | } |
| 361 | 400 | |
| 362 | 401 | const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch |err| switch (err) { |
| 363 | | error.FileNotFound => return false, |
| 402 | error.FileNotFound => { |
| 403 | try self.upgradeToExclusiveLock(); |
| 404 | return false; |
| 405 | }, |
| 364 | 406 | else => return error.CacheUnavailable, |
| 365 | 407 | }; |
| 366 | 408 | defer this_file.close(); |
| ... | ... | @@ -405,6 +447,7 @@ pub const Manifest = struct { |
| 405 | 447 | // cache miss |
| 406 | 448 | // keep the manifest file open |
| 407 | 449 | self.unhit(bin_digest, input_file_count); |
| 450 | try self.upgradeToExclusiveLock(); |
| 408 | 451 | return false; |
| 409 | 452 | } |
| 410 | 453 | |
| ... | ... | @@ -417,9 +460,11 @@ pub const Manifest = struct { |
| 417 | 460 | return err; |
| 418 | 461 | }; |
| 419 | 462 | } |
| 463 | try self.upgradeToExclusiveLock(); |
| 420 | 464 | return false; |
| 421 | 465 | } |
| 422 | 466 | |
| 467 | try self.downgradeToSharedLock(); |
| 423 | 468 | return true; |
| 424 | 469 | } |
| 425 | 470 | |
| ... | ... | @@ -585,34 +630,56 @@ pub const Manifest = struct { |
| 585 | 630 | return out_digest; |
| 586 | 631 | } |
| 587 | 632 | |
| 633 | /// If `want_shared_lock` is true, this function automatically downgrades the |
| 634 | /// lock from exclusive to shared. |
| 588 | 635 | pub fn writeManifest(self: *Manifest) !void { |
| 589 | 636 | const manifest_file = self.manifest_file.?; |
| 590 | | if (!self.manifest_dirty) return; |
| 591 | | |
| 592 | | var contents = std.ArrayList(u8).init(self.cache.gpa); |
| 593 | | defer contents.deinit(); |
| 637 | if (self.manifest_dirty) { |
| 638 | self.manifest_dirty = false; |
| 639 | |
| 640 | var contents = std.ArrayList(u8).init(self.cache.gpa); |
| 641 | defer contents.deinit(); |
| 642 | |
| 643 | const writer = contents.writer(); |
| 644 | var encoded_digest: [hex_digest_len]u8 = undefined; |
| 645 | |
| 646 | for (self.files.items) |file| { |
| 647 | _ = std.fmt.bufPrint( |
| 648 | &encoded_digest, |
| 649 | "{s}", |
| 650 | .{std.fmt.fmtSliceHexLower(&file.bin_digest)}, |
| 651 | ) catch unreachable; |
| 652 | try writer.print("{d} {d} {d} {s} {s}\n", .{ |
| 653 | file.stat.size, |
| 654 | file.stat.inode, |
| 655 | file.stat.mtime, |
| 656 | &encoded_digest, |
| 657 | file.path, |
| 658 | }); |
| 659 | } |
| 594 | 660 | |
| 595 | | const writer = contents.writer(); |
| 596 | | var encoded_digest: [hex_digest_len]u8 = undefined; |
| 661 | try manifest_file.setEndPos(contents.items.len); |
| 662 | try manifest_file.pwriteAll(contents.items, 0); |
| 663 | } |
| 597 | 664 | |
| 598 | | for (self.files.items) |file| { |
| 599 | | _ = std.fmt.bufPrint( |
| 600 | | &encoded_digest, |
| 601 | | "{s}", |
| 602 | | .{std.fmt.fmtSliceHexLower(&file.bin_digest)}, |
| 603 | | ) catch unreachable; |
| 604 | | try writer.print("{d} {d} {d} {s} {s}\n", .{ |
| 605 | | file.stat.size, |
| 606 | | file.stat.inode, |
| 607 | | file.stat.mtime, |
| 608 | | &encoded_digest, |
| 609 | | file.path, |
| 610 | | }); |
| 665 | if (self.want_shared_lock) { |
| 666 | try self.downgradeToSharedLock(); |
| 611 | 667 | } |
| 668 | } |
| 669 | |
| 670 | fn downgradeToSharedLock(self: *Manifest) !void { |
| 671 | if (!self.have_exclusive_lock) return; |
| 672 | const manifest_file = self.manifest_file.?; |
| 673 | try manifest_file.setLock(.Shared, false); |
| 674 | self.have_exclusive_lock = false; |
| 675 | } |
| 612 | 676 | |
| 613 | | try manifest_file.setEndPos(contents.items.len); |
| 614 | | try manifest_file.pwriteAll(contents.items, 0); |
| 615 | | self.manifest_dirty = false; |
| 677 | fn upgradeToExclusiveLock(self: *Manifest) !void { |
| 678 | if (self.have_exclusive_lock) return; |
| 679 | const manifest_file = self.manifest_file.?; |
| 680 | try manifest_file.setLock(.None, false); |
| 681 | try manifest_file.setLock(.Exclusive, false); |
| 682 | self.have_exclusive_lock = true; |
| 616 | 683 | } |
| 617 | 684 | |
| 618 | 685 | /// Obtain only the data needed to maintain a lock on the manifest file. |
| ... | ... | @@ -881,27 +948,27 @@ test "no file inputs" { |
| 881 | 948 | defer cache.manifest_dir.close(); |
| 882 | 949 | |
| 883 | 950 | { |
| 884 | | var ch = cache.obtain(); |
| 885 | | defer ch.deinit(); |
| 951 | var man = cache.obtain(); |
| 952 | defer man.deinit(); |
| 886 | 953 | |
| 887 | | ch.hash.addBytes("1234"); |
| 954 | man.hash.addBytes("1234"); |
| 888 | 955 | |
| 889 | 956 | // There should be nothing in the cache |
| 890 | | try testing.expectEqual(false, try ch.hit()); |
| 957 | try testing.expectEqual(false, try man.hit()); |
| 891 | 958 | |
| 892 | | digest1 = ch.final(); |
| 959 | digest1 = man.final(); |
| 893 | 960 | |
| 894 | | try ch.writeManifest(); |
| 961 | try man.writeManifest(); |
| 895 | 962 | } |
| 896 | 963 | { |
| 897 | | var ch = cache.obtain(); |
| 898 | | defer ch.deinit(); |
| 964 | var man = cache.obtain(); |
| 965 | defer man.deinit(); |
| 899 | 966 | |
| 900 | | ch.hash.addBytes("1234"); |
| 967 | man.hash.addBytes("1234"); |
| 901 | 968 | |
| 902 | | try testing.expect(try ch.hit()); |
| 903 | | digest2 = ch.final(); |
| 904 | | try ch.writeManifest(); |
| 969 | try testing.expect(try man.hit()); |
| 970 | digest2 = man.final(); |
| 971 | try man.writeManifest(); |
| 905 | 972 | } |
| 906 | 973 | |
| 907 | 974 | try testing.expectEqual(digest1, digest2); |
Andrew Kelley <andrew@ziglang.org> 2021-06-27 22:33:17-07:00
src