| ... | @@ -181,6 +181,12 @@ pub const Manifest = struct { | ... | @@ -181,6 +181,12 @@ pub const Manifest = struct { |
| 181 | hash: HashHelper, | 181 | hash: HashHelper, |
| 182 | manifest_file: ?fs.File, | 182 | manifest_file: ?fs.File, |
| 183 | manifest_dirty: bool, | 183 | manifest_dirty: bool, |
| | 184 | /// Set this flag to true before calling hit() in order to indicate that |
| | 185 | /// upon a cache hit, the code using the cache will not modify the files |
| | 186 | /// within the cache directory. This allows multiple processes to utilize |
| | 187 | /// the same cache directory at the same time. |
| | 188 | want_shared_lock: bool = true, |
| | 189 | have_exclusive_lock: bool = false, |
| 184 | files: std.ArrayListUnmanaged(File) = .{}, | 190 | files: std.ArrayListUnmanaged(File) = .{}, |
| 185 | hex_digest: [hex_digest_len]u8, | 191 | hex_digest: [hex_digest_len]u8, |
| 186 | /// Populated when hit() returns an error because of one | 192 | /// Populated when hit() returns an error because of one |
| ... | @@ -257,7 +263,9 @@ pub const Manifest = struct { | ... | @@ -257,7 +263,9 @@ pub const Manifest = struct { |
| 257 | /// | 263 | /// |
| 258 | /// This function will also acquire an exclusive lock to the manifest file. This means | 264 | /// This function will also acquire an exclusive lock to the manifest file. This means |
| 259 | /// that a process holding a Manifest will block any other process attempting to | 265 | /// that a process holding a Manifest will block any other process attempting to |
| 260 | /// acquire the lock. | 266 | /// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the |
| | 267 | /// manifest file to be locked in shared mode, and a cache miss guarantees the manifest |
| | 268 | /// file to be locked in exclusive mode. |
| 261 | /// | 269 | /// |
| 262 | /// The lock on the manifest file is released when `deinit` is called. As another | 270 | /// The lock on the manifest file is released when `deinit` is called. As another |
| 263 | /// option, one may call `toOwnedLock` to obtain a smaller object which can represent | 271 | /// option, one may call `toOwnedLock` to obtain a smaller object which can represent |
| ... | @@ -285,31 +293,62 @@ pub const Manifest = struct { | ... | @@ -285,31 +293,62 @@ pub const Manifest = struct { |
| 285 | mem.copy(u8, &manifest_file_path, &self.hex_digest); | 293 | mem.copy(u8, &manifest_file_path, &self.hex_digest); |
| 286 | manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*; | 294 | manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*; |
| 287 | | 295 | |
| 288 | if (self.files.items.len != 0) { | 296 | if (self.files.items.len == 0) { |
| 289 | self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{ | | |
| 290 | .read = true, | | |
| 291 | .truncate = false, | | |
| 292 | .lock = .Exclusive, | | |
| 293 | }); | | |
| 294 | } else { | | |
| 295 | // If there are no file inputs, we check if the manifest file exists instead of | 297 | // If there are no file inputs, we check if the manifest file exists instead of |
| 296 | // comparing the hashes on the files used for the cached item | 298 | // comparing the hashes on the files used for the cached item |
| 297 | self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{ | 299 | while (true) { |
| | 300 | if (self.cache.manifest_dir.openFile(&manifest_file_path, .{ |
| | 301 | .read = true, |
| | 302 | .write = true, |
| | 303 | .lock = .Exclusive, |
| | 304 | .lock_nonblocking = self.want_shared_lock, |
| | 305 | })) |manifest_file| { |
| | 306 | self.manifest_file = manifest_file; |
| | 307 | self.have_exclusive_lock = true; |
| | 308 | break; |
| | 309 | } else |open_err| switch (open_err) { |
| | 310 | error.WouldBlock => { |
| | 311 | self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{ |
| | 312 | .lock = .Shared, |
| | 313 | }); |
| | 314 | break; |
| | 315 | }, |
| | 316 | error.FileNotFound => { |
| | 317 | if (self.cache.manifest_dir.createFile(&manifest_file_path, .{ |
| | 318 | .read = true, |
| | 319 | .truncate = false, |
| | 320 | .lock = .Exclusive, |
| | 321 | .lock_nonblocking = self.want_shared_lock, |
| | 322 | })) |manifest_file| { |
| | 323 | self.manifest_file = manifest_file; |
| | 324 | self.manifest_dirty = true; |
| | 325 | self.have_exclusive_lock = true; |
| | 326 | return false; // cache miss; exclusive lock already held |
| | 327 | } else |err| switch (err) { |
| | 328 | error.WouldBlock => continue, |
| | 329 | else => |e| return e, |
| | 330 | } |
| | 331 | }, |
| | 332 | else => |e| return e, |
| | 333 | } |
| | 334 | } |
| | 335 | } else { |
| | 336 | if (self.cache.manifest_dir.createFile(&manifest_file_path, .{ |
| 298 | .read = true, | 337 | .read = true, |
| 299 | .write = true, | 338 | .truncate = false, |
| 300 | .lock = .Exclusive, | 339 | .lock = .Exclusive, |
| 301 | }) catch |err| switch (err) { | 340 | .lock_nonblocking = self.want_shared_lock, |
| 302 | error.FileNotFound => { | 341 | })) |manifest_file| { |
| 303 | self.manifest_dirty = true; | 342 | self.manifest_file = manifest_file; |
| 304 | self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{ | 343 | self.have_exclusive_lock = true; |
| 305 | .read = true, | 344 | } else |err| switch (err) { |
| 306 | .truncate = false, | 345 | error.WouldBlock => { |
| 307 | .lock = .Exclusive, | 346 | self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{ |
| | 347 | .lock = .Shared, |
| 308 | }); | 348 | }); |
| 309 | return false; | | |
| 310 | }, | 349 | }, |
| 311 | else => |e| return e, | 350 | else => |e| return e, |
| 312 | }; | 351 | } |
| 313 | } | 352 | } |
| 314 | | 353 | |
| 315 | const file_contents = try self.manifest_file.?.reader().readAllAlloc(self.cache.gpa, manifest_file_size_max); | 354 | const file_contents = try self.manifest_file.?.reader().readAllAlloc(self.cache.gpa, manifest_file_size_max); |
| ... | @@ -360,7 +399,10 @@ pub const Manifest = struct { | ... | @@ -360,7 +399,10 @@ pub const Manifest = struct { |
| 360 | } | 399 | } |
| 361 | | 400 | |
| 362 | const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch |err| switch (err) { | 401 | const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch |err| switch (err) { |
| 363 | error.FileNotFound => return false, | 402 | error.FileNotFound => { |
| | 403 | try self.upgradeToExclusiveLock(); |
| | 404 | return false; |
| | 405 | }, |
| 364 | else => return error.CacheUnavailable, | 406 | else => return error.CacheUnavailable, |
| 365 | }; | 407 | }; |
| 366 | defer this_file.close(); | 408 | defer this_file.close(); |
| ... | @@ -405,6 +447,7 @@ pub const Manifest = struct { | ... | @@ -405,6 +447,7 @@ pub const Manifest = struct { |
| 405 | // cache miss | 447 | // cache miss |
| 406 | // keep the manifest file open | 448 | // keep the manifest file open |
| 407 | self.unhit(bin_digest, input_file_count); | 449 | self.unhit(bin_digest, input_file_count); |
| | 450 | try self.upgradeToExclusiveLock(); |
| 408 | return false; | 451 | return false; |
| 409 | } | 452 | } |
| 410 | | 453 | |
| ... | @@ -417,9 +460,11 @@ pub const Manifest = struct { | ... | @@ -417,9 +460,11 @@ pub const Manifest = struct { |
| 417 | return err; | 460 | return err; |
| 418 | }; | 461 | }; |
| 419 | } | 462 | } |
| | 463 | try self.upgradeToExclusiveLock(); |
| 420 | return false; | 464 | return false; |
| 421 | } | 465 | } |
| 422 | | 466 | |
| | 467 | try self.downgradeToSharedLock(); |
| 423 | return true; | 468 | return true; |
| 424 | } | 469 | } |
| 425 | | 470 | |
| ... | @@ -585,34 +630,56 @@ pub const Manifest = struct { | ... | @@ -585,34 +630,56 @@ pub const Manifest = struct { |
| 585 | return out_digest; | 630 | return out_digest; |
| 586 | } | 631 | } |
| 587 | | 632 | |
| | 633 | /// If `want_shared_lock` is true, this function automatically downgrades the |
| | 634 | /// lock from exclusive to shared. |
| 588 | pub fn writeManifest(self: *Manifest) !void { | 635 | pub fn writeManifest(self: *Manifest) !void { |
| 589 | const manifest_file = self.manifest_file.?; | 636 | const manifest_file = self.manifest_file.?; |
| 590 | if (!self.manifest_dirty) return; | 637 | if (self.manifest_dirty) { |
| 591 | | 638 | self.manifest_dirty = false; |
| 592 | var contents = std.ArrayList(u8).init(self.cache.gpa); | 639 | |
| 593 | defer contents.deinit(); | 640 | var contents = std.ArrayList(u8).init(self.cache.gpa); |
| | 641 | defer contents.deinit(); |
| | 642 | |
| | 643 | const writer = contents.writer(); |
| | 644 | var encoded_digest: [hex_digest_len]u8 = undefined; |
| | 645 | |
| | 646 | for (self.files.items) |file| { |
| | 647 | _ = std.fmt.bufPrint( |
| | 648 | &encoded_digest, |
| | 649 | "{s}", |
| | 650 | .{std.fmt.fmtSliceHexLower(&file.bin_digest)}, |
| | 651 | ) catch unreachable; |
| | 652 | try writer.print("{d} {d} {d} {s} {s}\n", .{ |
| | 653 | file.stat.size, |
| | 654 | file.stat.inode, |
| | 655 | file.stat.mtime, |
| | 656 | &encoded_digest, |
| | 657 | file.path, |
| | 658 | }); |
| | 659 | } |
| 594 | | 660 | |
| 595 | const writer = contents.writer(); | 661 | try manifest_file.setEndPos(contents.items.len); |
| 596 | var encoded_digest: [hex_digest_len]u8 = undefined; | 662 | try manifest_file.pwriteAll(contents.items, 0); |
| | 663 | } |
| 597 | | 664 | |
| 598 | for (self.files.items) |file| { | 665 | if (self.want_shared_lock) { |
| 599 | _ = std.fmt.bufPrint( | 666 | try self.downgradeToSharedLock(); |
| 600 | &encoded_digest, | | |
| 601 | "{s}", | | |
| 602 | .{std.fmt.fmtSliceHexLower(&file.bin_digest)}, | | |
| 603 | ) catch unreachable; | | |
| 604 | try writer.print("{d} {d} {d} {s} {s}\n", .{ | | |
| 605 | file.stat.size, | | |
| 606 | file.stat.inode, | | |
| 607 | file.stat.mtime, | | |
| 608 | &encoded_digest, | | |
| 609 | file.path, | | |
| 610 | }); | | |
| 611 | } | 667 | } |
| | 668 | } |
| | 669 | |
| | 670 | fn downgradeToSharedLock(self: *Manifest) !void { |
| | 671 | if (!self.have_exclusive_lock) return; |
| | 672 | const manifest_file = self.manifest_file.?; |
| | 673 | try manifest_file.setLock(.Shared, false); |
| | 674 | self.have_exclusive_lock = false; |
| | 675 | } |
| 612 | | 676 | |
| 613 | try manifest_file.setEndPos(contents.items.len); | 677 | fn upgradeToExclusiveLock(self: *Manifest) !void { |
| 614 | try manifest_file.pwriteAll(contents.items, 0); | 678 | if (self.have_exclusive_lock) return; |
| 615 | self.manifest_dirty = false; | 679 | const manifest_file = self.manifest_file.?; |
| | 680 | try manifest_file.setLock(.None, false); |
| | 681 | try manifest_file.setLock(.Exclusive, false); |
| | 682 | self.have_exclusive_lock = true; |
| 616 | } | 683 | } |
| 617 | | 684 | |
| 618 | /// Obtain only the data needed to maintain a lock on the manifest file. | 685 | /// Obtain only the data needed to maintain a lock on the manifest file. |
| ... | @@ -881,27 +948,27 @@ test "no file inputs" { | ... | @@ -881,27 +948,27 @@ test "no file inputs" { |
| 881 | defer cache.manifest_dir.close(); | 948 | defer cache.manifest_dir.close(); |
| 882 | | 949 | |
| 883 | { | 950 | { |
| 884 | var ch = cache.obtain(); | 951 | var man = cache.obtain(); |
| 885 | defer ch.deinit(); | 952 | defer man.deinit(); |
| 886 | | 953 | |
| 887 | ch.hash.addBytes("1234"); | 954 | man.hash.addBytes("1234"); |
| 888 | | 955 | |
| 889 | // There should be nothing in the cache | 956 | // There should be nothing in the cache |
| 890 | try testing.expectEqual(false, try ch.hit()); | 957 | try testing.expectEqual(false, try man.hit()); |
| 891 | | 958 | |
| 892 | digest1 = ch.final(); | 959 | digest1 = man.final(); |
| 893 | | 960 | |
| 894 | try ch.writeManifest(); | 961 | try man.writeManifest(); |
| 895 | } | 962 | } |
| 896 | { | 963 | { |
| 897 | var ch = cache.obtain(); | 964 | var man = cache.obtain(); |
| 898 | defer ch.deinit(); | 965 | defer man.deinit(); |
| 899 | | 966 | |
| 900 | ch.hash.addBytes("1234"); | 967 | man.hash.addBytes("1234"); |
| 901 | | 968 | |
| 902 | try testing.expect(try ch.hit()); | 969 | try testing.expect(try man.hit()); |
| 903 | digest2 = ch.final(); | 970 | digest2 = man.final(); |
| 904 | try ch.writeManifest(); | 971 | try man.writeManifest(); |
| 905 | } | 972 | } |
| 906 | | 973 | |
| 907 | try testing.expectEqual(digest1, digest2); | 974 | try testing.expectEqual(digest1, digest2); |
Andrew Kelley <andrew@ziglang.org> 2021-06-27 22:33:17-07:00
src