| ... | ... | @@ -55,7 +55,15 @@ pub fn prefixes(cache: *const Cache) []const Directory { |
| 55 | 55 | |
| 56 | 56 | const PrefixedPath = struct { |
| 57 | 57 | prefix: u8, |
| 58 | | sub_path: []u8, |
| 58 | sub_path: []const u8, |
| 59 | |
| 60 | fn eql(a: PrefixedPath, b: PrefixedPath) bool { |
| 61 | return a.prefix == b.prefix and std.mem.eql(u8, a.sub_path, b.sub_path); |
| 62 | } |
| 63 | |
| 64 | fn hash(pp: PrefixedPath) u32 { |
| 65 | return @truncate(std.hash.Wyhash.hash(pp.prefix, pp.sub_path)); |
| 66 | } |
| 59 | 67 | }; |
| 60 | 68 | |
| 61 | 69 | fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath { |
| ... | ... | @@ -132,7 +140,7 @@ pub const hasher_init: Hasher = Hasher.init(&[_]u8{ |
| 132 | 140 | }); |
| 133 | 141 | |
| 134 | 142 | pub const File = struct { |
| 135 | | prefixed_path: ?PrefixedPath, |
| 143 | prefixed_path: PrefixedPath, |
| 136 | 144 | max_file_size: ?usize, |
| 137 | 145 | stat: Stat, |
| 138 | 146 | bin_digest: BinDigest, |
| ... | ... | @@ -145,16 +153,18 @@ pub const File = struct { |
| 145 | 153 | }; |
| 146 | 154 | |
| 147 | 155 | pub fn deinit(self: *File, gpa: Allocator) void { |
| 148 | | if (self.prefixed_path) |pp| { |
| 149 | | gpa.free(pp.sub_path); |
| 150 | | self.prefixed_path = null; |
| 151 | | } |
| 156 | gpa.free(self.prefixed_path.sub_path); |
| 152 | 157 | if (self.contents) |contents| { |
| 153 | 158 | gpa.free(contents); |
| 154 | 159 | self.contents = null; |
| 155 | 160 | } |
| 156 | 161 | self.* = undefined; |
| 157 | 162 | } |
| 163 | |
| 164 | pub fn updateMaxSize(file: *File, new_max_size: ?usize) void { |
| 165 | const new = new_max_size orelse return; |
| 166 | file.max_file_size = if (file.max_file_size) |old| @max(old, new) else new; |
| 167 | } |
| 158 | 168 | }; |
| 159 | 169 | |
| 160 | 170 | pub const HashHelper = struct { |
| ... | ... | @@ -296,7 +306,7 @@ pub const Manifest = struct { |
| 296 | 306 | // order to obtain a problematic timestamp for the next call. Calls after that |
| 297 | 307 | // will then use the same timestamp, to avoid unnecessary filesystem writes. |
| 298 | 308 | want_refresh_timestamp: bool = true, |
| 299 | | files: std.ArrayListUnmanaged(File) = .{}, |
| 309 | files: Files = .{}, |
| 300 | 310 | hex_digest: HexDigest, |
| 301 | 311 | /// Populated when hit() returns an error because of one |
| 302 | 312 | /// of the files listed in the manifest. |
| ... | ... | @@ -305,6 +315,34 @@ pub const Manifest = struct { |
| 305 | 315 | /// what time the file system thinks it is, according to its own granularity. |
| 306 | 316 | recent_problematic_timestamp: i128 = 0, |
| 307 | 317 | |
| 318 | pub const Files = std.ArrayHashMapUnmanaged(File, void, FilesContext, false); |
| 319 | |
| 320 | pub const FilesContext = struct { |
| 321 | pub fn hash(fc: FilesContext, file: File) u32 { |
| 322 | _ = fc; |
| 323 | return file.prefixed_path.hash(); |
| 324 | } |
| 325 | |
| 326 | pub fn eql(fc: FilesContext, a: File, b: File, b_index: usize) bool { |
| 327 | _ = fc; |
| 328 | _ = b_index; |
| 329 | return a.prefixed_path.eql(b.prefixed_path); |
| 330 | } |
| 331 | }; |
| 332 | |
| 333 | const FilesAdapter = struct { |
| 334 | pub fn eql(context: @This(), a: PrefixedPath, b: File, b_index: usize) bool { |
| 335 | _ = context; |
| 336 | _ = b_index; |
| 337 | return a.eql(b.prefixed_path); |
| 338 | } |
| 339 | |
| 340 | pub fn hash(context: @This(), key: PrefixedPath) u32 { |
| 341 | _ = context; |
| 342 | return key.hash(); |
| 343 | } |
| 344 | }; |
| 345 | |
| 308 | 346 | /// Add a file as a dependency of process being cached. When `hit` is |
| 309 | 347 | /// called, the file's contents will be checked to ensure that it matches |
| 310 | 348 | /// the contents from previous times. |
| ... | ... | @@ -317,7 +355,7 @@ pub const Manifest = struct { |
| 317 | 355 | /// to access the contents of the file after calling `hit()` like so: |
| 318 | 356 | /// |
| 319 | 357 | /// ``` |
| 320 | | /// var file_contents = cache_hash.files.items[file_index].contents.?; |
| 358 | /// var file_contents = cache_hash.files.keys()[file_index].contents.?; |
| 321 | 359 | /// ``` |
| 322 | 360 | pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize { |
| 323 | 361 | assert(self.manifest_file == null); |
| ... | ... | @@ -327,7 +365,12 @@ pub const Manifest = struct { |
| 327 | 365 | const prefixed_path = try self.cache.findPrefix(file_path); |
| 328 | 366 | errdefer gpa.free(prefixed_path.sub_path); |
| 329 | 367 | |
| 330 | | self.files.addOneAssumeCapacity().* = .{ |
| 368 | const gop = self.files.getOrPutAssumeCapacityAdapted(prefixed_path, FilesAdapter{}); |
| 369 | if (gop.found_existing) { |
| 370 | gop.key_ptr.updateMaxSize(max_file_size); |
| 371 | return gop.index; |
| 372 | } |
| 373 | gop.key_ptr.* = .{ |
| 331 | 374 | .prefixed_path = prefixed_path, |
| 332 | 375 | .contents = null, |
| 333 | 376 | .max_file_size = max_file_size, |
| ... | ... | @@ -338,7 +381,7 @@ pub const Manifest = struct { |
| 338 | 381 | self.hash.add(prefixed_path.prefix); |
| 339 | 382 | self.hash.addBytes(prefixed_path.sub_path); |
| 340 | 383 | |
| 341 | | return self.files.items.len - 1; |
| 384 | return gop.index; |
| 342 | 385 | } |
| 343 | 386 | |
| 344 | 387 | pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void { |
| ... | ... | @@ -418,7 +461,7 @@ pub const Manifest = struct { |
| 418 | 461 | |
| 419 | 462 | self.want_refresh_timestamp = true; |
| 420 | 463 | |
| 421 | | const input_file_count = self.files.items.len; |
| 464 | const input_file_count = self.files.entries.len; |
| 422 | 465 | while (true) : (self.unhit(bin_digest, input_file_count)) { |
| 423 | 466 | const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max); |
| 424 | 467 | defer gpa.free(file_contents); |
| ... | ... | @@ -430,7 +473,7 @@ pub const Manifest = struct { |
| 430 | 473 | if (try self.upgradeToExclusiveLock()) continue; |
| 431 | 474 | self.manifest_dirty = true; |
| 432 | 475 | while (idx < input_file_count) : (idx += 1) { |
| 433 | | const ch_file = &self.files.items[idx]; |
| 476 | const ch_file = &self.files.keys()[idx]; |
| 434 | 477 | self.populateFileHash(ch_file) catch |err| { |
| 435 | 478 | self.failed_file_index = idx; |
| 436 | 479 | return err; |
| ... | ... | @@ -441,18 +484,6 @@ pub const Manifest = struct { |
| 441 | 484 | while (line_iter.next()) |line| { |
| 442 | 485 | defer idx += 1; |
| 443 | 486 | |
| 444 | | const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: { |
| 445 | | const new = try self.files.addOne(gpa); |
| 446 | | new.* = .{ |
| 447 | | .prefixed_path = null, |
| 448 | | .contents = null, |
| 449 | | .max_file_size = null, |
| 450 | | .stat = undefined, |
| 451 | | .bin_digest = undefined, |
| 452 | | }; |
| 453 | | break :blk new; |
| 454 | | }; |
| 455 | | |
| 456 | 487 | var iter = mem.tokenizeScalar(u8, line, ' '); |
| 457 | 488 | const size = iter.next() orelse return error.InvalidFormat; |
| 458 | 489 | const inode = iter.next() orelse return error.InvalidFormat; |
| ... | ... | @@ -461,30 +492,61 @@ pub const Manifest = struct { |
| 461 | 492 | const prefix_str = iter.next() orelse return error.InvalidFormat; |
| 462 | 493 | const file_path = iter.rest(); |
| 463 | 494 | |
| 464 | | cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat; |
| 465 | | cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat; |
| 466 | | cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat; |
| 467 | | _ = fmt.hexToBytes(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat; |
| 495 | const stat_size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat; |
| 496 | const stat_inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat; |
| 497 | const stat_mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat; |
| 498 | const file_bin_digest = b: { |
| 499 | if (digest_str.len != hex_digest_len) return error.InvalidFormat; |
| 500 | var bd: BinDigest = undefined; |
| 501 | _ = fmt.hexToBytes(&bd, digest_str) catch return error.InvalidFormat; |
| 502 | break :b bd; |
| 503 | }; |
| 504 | |
| 468 | 505 | const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat; |
| 469 | 506 | if (prefix >= self.cache.prefixes_len) return error.InvalidFormat; |
| 470 | 507 | |
| 471 | | if (file_path.len == 0) { |
| 472 | | return error.InvalidFormat; |
| 473 | | } |
| 474 | | if (cache_hash_file.prefixed_path) |pp| { |
| 475 | | if (pp.prefix != prefix or !mem.eql(u8, file_path, pp.sub_path)) { |
| 476 | | return error.InvalidFormat; |
| 477 | | } |
| 478 | | } |
| 508 | if (file_path.len == 0) return error.InvalidFormat; |
| 479 | 509 | |
| 480 | | if (cache_hash_file.prefixed_path == null) { |
| 481 | | cache_hash_file.prefixed_path = .{ |
| 510 | const cache_hash_file = f: { |
| 511 | const prefixed_path: PrefixedPath = .{ |
| 482 | 512 | .prefix = prefix, |
| 483 | | .sub_path = try gpa.dupe(u8, file_path), |
| 513 | .sub_path = file_path, // expires with file_contents |
| 484 | 514 | }; |
| 485 | | } |
| 515 | if (idx < input_file_count) { |
| 516 | const file = &self.files.keys()[idx]; |
| 517 | if (!file.prefixed_path.eql(prefixed_path)) |
| 518 | return error.InvalidFormat; |
| 519 | |
| 520 | file.stat = .{ |
| 521 | .size = stat_size, |
| 522 | .inode = stat_inode, |
| 523 | .mtime = stat_mtime, |
| 524 | }; |
| 525 | file.bin_digest = file_bin_digest; |
| 526 | break :f file; |
| 527 | } |
| 528 | const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{}); |
| 529 | errdefer assert(self.files.popOrNull() != null); |
| 530 | if (!gop.found_existing) { |
| 531 | gop.key_ptr.* = .{ |
| 532 | .prefixed_path = .{ |
| 533 | .prefix = prefix, |
| 534 | .sub_path = try gpa.dupe(u8, file_path), |
| 535 | }, |
| 536 | .contents = null, |
| 537 | .max_file_size = null, |
| 538 | .stat = .{ |
| 539 | .size = stat_size, |
| 540 | .inode = stat_inode, |
| 541 | .mtime = stat_mtime, |
| 542 | }, |
| 543 | .bin_digest = file_bin_digest, |
| 544 | }; |
| 545 | } |
| 546 | break :f gop.key_ptr; |
| 547 | }; |
| 486 | 548 | |
| 487 | | const pp = cache_hash_file.prefixed_path.?; |
| 549 | const pp = cache_hash_file.prefixed_path; |
| 488 | 550 | const dir = self.cache.prefixes()[pp.prefix].handle; |
| 489 | 551 | const this_file = dir.openFile(pp.sub_path, .{ .mode = .read_only }) catch |err| switch (err) { |
| 490 | 552 | error.FileNotFound => { |
| ... | ... | @@ -548,7 +610,7 @@ pub const Manifest = struct { |
| 548 | 610 | if (try self.upgradeToExclusiveLock()) continue; |
| 549 | 611 | self.manifest_dirty = true; |
| 550 | 612 | while (idx < input_file_count) : (idx += 1) { |
| 551 | | const ch_file = &self.files.items[idx]; |
| 613 | const ch_file = &self.files.keys()[idx]; |
| 552 | 614 | self.populateFileHash(ch_file) catch |err| { |
| 553 | 615 | self.failed_file_index = idx; |
| 554 | 616 | return err; |
| ... | ... | @@ -571,12 +633,12 @@ pub const Manifest = struct { |
| 571 | 633 | self.hash.hasher.update(&bin_digest); |
| 572 | 634 | |
| 573 | 635 | // Remove files not in the initial hash. |
| 574 | | for (self.files.items[input_file_count..]) |*file| { |
| 636 | for (self.files.keys()[input_file_count..]) |*file| { |
| 575 | 637 | file.deinit(self.cache.gpa); |
| 576 | 638 | } |
| 577 | 639 | self.files.shrinkRetainingCapacity(input_file_count); |
| 578 | 640 | |
| 579 | | for (self.files.items) |file| { |
| 641 | for (self.files.keys()) |file| { |
| 580 | 642 | self.hash.hasher.update(&file.bin_digest); |
| 581 | 643 | } |
| 582 | 644 | } |
| ... | ... | @@ -616,7 +678,7 @@ pub const Manifest = struct { |
| 616 | 678 | } |
| 617 | 679 | |
| 618 | 680 | fn populateFileHash(self: *Manifest, ch_file: *File) !void { |
| 619 | | const pp = ch_file.prefixed_path.?; |
| 681 | const pp = ch_file.prefixed_path; |
| 620 | 682 | const dir = self.cache.prefixes()[pp.prefix].handle; |
| 621 | 683 | const file = try dir.openFile(pp.sub_path, .{}); |
| 622 | 684 | defer file.close(); |
| ... | ... | @@ -682,7 +744,7 @@ pub const Manifest = struct { |
| 682 | 744 | .bin_digest = undefined, |
| 683 | 745 | .contents = null, |
| 684 | 746 | }; |
| 685 | | errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); |
| 747 | errdefer self.files.shrinkRetainingCapacity(self.files.entries.len - 1); |
| 686 | 748 | |
| 687 | 749 | try self.populateFileHash(new_ch_file); |
| 688 | 750 | |
| ... | ... | @@ -690,9 +752,11 @@ pub const Manifest = struct { |
| 690 | 752 | } |
| 691 | 753 | |
| 692 | 754 | /// Add a file as a dependency of process being cached, after the initial hash has been |
| 693 | | /// calculated. This is useful for processes that don't know the all the files that |
| 694 | | /// are depended on ahead of time. For example, a source file that can import other files |
| 695 | | /// will need to be recompiled if the imported file is changed. |
| 755 | /// calculated. |
| 756 | /// |
| 757 | /// This is useful for processes that don't know the all the files that are |
| 758 | /// depended on ahead of time. For example, a source file that can import |
| 759 | /// other files will need to be recompiled if the imported file is changed. |
| 696 | 760 | pub fn addFilePost(self: *Manifest, file_path: []const u8) !void { |
| 697 | 761 | assert(self.manifest_file != null); |
| 698 | 762 | |
| ... | ... | @@ -700,17 +764,26 @@ pub const Manifest = struct { |
| 700 | 764 | const prefixed_path = try self.cache.findPrefix(file_path); |
| 701 | 765 | errdefer gpa.free(prefixed_path.sub_path); |
| 702 | 766 | |
| 703 | | const new_ch_file = try self.files.addOne(gpa); |
| 704 | | new_ch_file.* = .{ |
| 767 | const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{}); |
| 768 | errdefer assert(self.files.popOrNull() != null); |
| 769 | |
| 770 | if (gop.found_existing) { |
| 771 | gpa.free(prefixed_path.sub_path); |
| 772 | return; |
| 773 | } |
| 774 | |
| 775 | gop.key_ptr.* = .{ |
| 705 | 776 | .prefixed_path = prefixed_path, |
| 706 | 777 | .max_file_size = null, |
| 707 | 778 | .stat = undefined, |
| 708 | 779 | .bin_digest = undefined, |
| 709 | 780 | .contents = null, |
| 710 | 781 | }; |
| 711 | | errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); |
| 712 | 782 | |
| 713 | | try self.populateFileHash(new_ch_file); |
| 783 | self.files.lockPointers(); |
| 784 | defer self.files.unlockPointers(); |
| 785 | |
| 786 | try self.populateFileHash(gop.key_ptr); |
| 714 | 787 | } |
| 715 | 788 | |
| 716 | 789 | /// Like `addFilePost` but when the file contents have already been loaded from disk. |
| ... | ... | @@ -724,13 +797,20 @@ pub const Manifest = struct { |
| 724 | 797 | assert(self.manifest_file != null); |
| 725 | 798 | const gpa = self.cache.gpa; |
| 726 | 799 | |
| 727 | | const ch_file = try self.files.addOne(gpa); |
| 728 | | errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); |
| 729 | | |
| 730 | 800 | const prefixed_path = try self.cache.findPrefixResolved(resolved_path); |
| 731 | 801 | errdefer gpa.free(prefixed_path.sub_path); |
| 732 | 802 | |
| 733 | | ch_file.* = .{ |
| 803 | const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{}); |
| 804 | errdefer assert(self.files.popOrNull() != null); |
| 805 | |
| 806 | if (gop.found_existing) { |
| 807 | gpa.free(prefixed_path.sub_path); |
| 808 | return; |
| 809 | } |
| 810 | |
| 811 | const new_file = gop.key_ptr; |
| 812 | |
| 813 | new_file.* = .{ |
| 734 | 814 | .prefixed_path = prefixed_path, |
| 735 | 815 | .max_file_size = null, |
| 736 | 816 | .stat = stat, |
| ... | ... | @@ -738,19 +818,19 @@ pub const Manifest = struct { |
| 738 | 818 | .contents = null, |
| 739 | 819 | }; |
| 740 | 820 | |
| 741 | | if (self.isProblematicTimestamp(ch_file.stat.mtime)) { |
| 821 | if (self.isProblematicTimestamp(new_file.stat.mtime)) { |
| 742 | 822 | // The actual file has an unreliable timestamp, force it to be hashed |
| 743 | | ch_file.stat.mtime = 0; |
| 744 | | ch_file.stat.inode = 0; |
| 823 | new_file.stat.mtime = 0; |
| 824 | new_file.stat.inode = 0; |
| 745 | 825 | } |
| 746 | 826 | |
| 747 | 827 | { |
| 748 | 828 | var hasher = hasher_init; |
| 749 | 829 | hasher.update(bytes); |
| 750 | | hasher.final(&ch_file.bin_digest); |
| 830 | hasher.final(&new_file.bin_digest); |
| 751 | 831 | } |
| 752 | 832 | |
| 753 | | self.hash.hasher.update(&ch_file.bin_digest); |
| 833 | self.hash.hasher.update(&new_file.bin_digest); |
| 754 | 834 | } |
| 755 | 835 | |
| 756 | 836 | pub fn addDepFilePost(self: *Manifest, dir: fs.Dir, dep_file_basename: []const u8) !void { |
| ... | ... | @@ -816,14 +896,14 @@ pub const Manifest = struct { |
| 816 | 896 | |
| 817 | 897 | const writer = contents.writer(); |
| 818 | 898 | try writer.writeAll(manifest_header ++ "\n"); |
| 819 | | for (self.files.items) |file| { |
| 899 | for (self.files.keys()) |file| { |
| 820 | 900 | try writer.print("{d} {d} {d} {} {d} {s}\n", .{ |
| 821 | 901 | file.stat.size, |
| 822 | 902 | file.stat.inode, |
| 823 | 903 | file.stat.mtime, |
| 824 | 904 | fmt.fmtSliceHexLower(&file.bin_digest), |
| 825 | | file.prefixed_path.?.prefix, |
| 826 | | file.prefixed_path.?.sub_path, |
| 905 | file.prefixed_path.prefix, |
| 906 | file.prefixed_path.sub_path, |
| 827 | 907 | }); |
| 828 | 908 | } |
| 829 | 909 | |
| ... | ... | @@ -892,7 +972,7 @@ pub const Manifest = struct { |
| 892 | 972 | |
| 893 | 973 | file.close(); |
| 894 | 974 | } |
| 895 | | for (self.files.items) |*file| { |
| 975 | for (self.files.keys()) |*file| { |
| 896 | 976 | file.deinit(self.cache.gpa); |
| 897 | 977 | } |
| 898 | 978 | self.files.deinit(self.cache.gpa); |
| ... | ... | @@ -1061,7 +1141,7 @@ test "check that changing a file makes cache fail" { |
| 1061 | 1141 | // There should be nothing in the cache |
| 1062 | 1142 | try testing.expectEqual(false, try ch.hit()); |
| 1063 | 1143 | |
| 1064 | | try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?)); |
| 1144 | try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.keys()[temp_file_idx].contents.?)); |
| 1065 | 1145 | |
| 1066 | 1146 | digest1 = ch.final(); |
| 1067 | 1147 | |
| ... | ... | @@ -1081,7 +1161,7 @@ test "check that changing a file makes cache fail" { |
| 1081 | 1161 | try testing.expectEqual(false, try ch.hit()); |
| 1082 | 1162 | |
| 1083 | 1163 | // The cache system does not keep the contents of re-hashed input files. |
| 1084 | | try testing.expect(ch.files.items[temp_file_idx].contents == null); |
| 1164 | try testing.expect(ch.files.keys()[temp_file_idx].contents == null); |
| 1085 | 1165 | |
| 1086 | 1166 | digest2 = ch.final(); |
| 1087 | 1167 | |
Andrew Kelley <andrew@ziglang.org> 2024-03-21 19:53:24-07:00