| ... | ... | @@ -48,7 +48,7 @@ pub fn obtain(cache: *Cache) Manifest { |
| 48 | 48 | .cache = cache, |
| 49 | 49 | .hash = cache.hash, |
| 50 | 50 | .manifest_file = null, |
| 51 | | .dirty = false, |
| 51 | .state = .input, |
| 52 | 52 | }; |
| 53 | 53 | } |
| 54 | 54 | |
| ... | ... | @@ -274,8 +274,7 @@ pub const Manifest = struct { |
| 274 | 274 | hash: HashHelper, |
| 275 | 275 | /// When this is null, `Manifest` is in "pre-check" phase. Otherwise it is in "post-check" phase. |
| 276 | 276 | manifest_file: ?Io.File, |
| 277 | | /// `finalize` has not been called yet. |
| 278 | | dirty: bool, |
| 277 | state: State, |
| 279 | 278 | /// Set this flag to true before calling `check` in order to indicate that upon a cache hit, the code |
| 280 | 279 | /// using the cache will not modify the files within the cache directory. This allows multiple processes |
| 281 | 280 | /// to utilize the same cache directory at the same time. |
| ... | ... | @@ -309,6 +308,27 @@ pub const Manifest = struct { |
| 309 | 308 | |
| 310 | 309 | pub const Files = std.array_hash_map.Custom(File.Offset, void, File.HashContext, false); |
| 311 | 310 | |
| 311 | pub const State = enum { |
| 312 | /// Call `addInputPath`, `addInputPathOptional`, and methods of `Manifest.hash`. Once all input files and state |
| 313 | /// have been added, call `check`. |
| 314 | input, |
| 315 | /// `check` has already been called and a hit occurred. Call `hitDigest` to find cache artifact directory. |
| 316 | hit, |
| 317 | /// `hitDigest` has already been called. No further operations are allowed. |
| 318 | hit_digested, |
| 319 | /// `check` has already been called and a miss occurred. `missDigest` has not been called yet. Miss diagnostic |
| 320 | /// data may still be accessed. Call `addDiscoveredPath` to add discovered files and directories. |
| 321 | miss, |
| 322 | /// A miss occurred, `missDigest` has not been called yet. Call `addDiscoveredPath` to add more discovered files |
| 323 | /// and directories. Miss diagnostic data may no longer be accessed. |
| 324 | miss_discovered, |
| 325 | /// `missDigest` has been called, but `finalize` has not been called yet. |
| 326 | miss_digested, |
| 327 | /// The cache manifest has been written to disk such that the next check for the same inputs will yield a hit. |
| 328 | /// No further operations are allowed. |
| 329 | miss_finalized, |
| 330 | }; |
| 331 | |
| 312 | 332 | /// Source files and directories whose prefix and relative path are |
| 313 | 333 | /// included when computing the cache manifest digest. It's the information |
| 314 | 334 | /// needed to lazily hash the input files only when a cache miss occurs. |
| ... | ... | @@ -656,6 +676,7 @@ pub const Manifest = struct { |
| 656 | 676 | /// See also: |
| 657 | 677 | /// * `addDiscoveredPath` |
| 658 | 678 | pub fn addInputPath(m: *Manifest, path: Path, options: AddInputPathOptions) Allocator.Error!InputPath.Index { |
| 679 | assert(m.state == .input); |
| 659 | 680 | const cache = m.cache; |
| 660 | 681 | const gpa = cache.gpa; |
| 661 | 682 | const is_directory = options.handle.isDirectory(); |
| ... | ... | @@ -733,6 +754,7 @@ pub const Manifest = struct { |
| 733 | 754 | } |
| 734 | 755 | |
| 735 | 756 | pub fn addInputPathOptional(m: *Manifest, opt_path: ?Path, options: AddInputPathOptions) Allocator.Error!void { |
| 757 | assert(m.state == .input); |
| 736 | 758 | m.hash.add(opt_path != null); |
| 737 | 759 | _ = try addInputPath(m, opt_path orelse return, options); |
| 738 | 760 | } |
| ... | ... | @@ -772,6 +794,7 @@ pub const Manifest = struct { |
| 772 | 794 | } |
| 773 | 795 | |
| 774 | 796 | pub fn checkProgressless(man: *Manifest) CheckError!CheckResult { |
| 797 | assert(man.state == .input); |
| 775 | 798 | assert(man.manifest_file == null); |
| 776 | 799 | |
| 777 | 800 | // This is *not* hashing the contents of the input files. It is the |
| ... | ... | @@ -872,6 +895,8 @@ pub const Manifest = struct { |
| 872 | 895 | // of all non-discovered files -- that is, the ones we've already been told about. The rest will be discovered |
| 873 | 896 | // through calls to `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user |
| 874 | 897 | // can use `final`, and will at some point `finalize` the file list to disk. |
| 898 | man.hash.hasher = hasher_init; |
| 899 | man.hash.hasher.update(&input_digest); |
| 875 | 900 | |
| 876 | 901 | hit: { |
| 877 | 902 | const miss_result = miss: { |
| ... | ... | @@ -891,9 +916,7 @@ pub const Manifest = struct { |
| 891 | 916 | |
| 892 | 917 | // Cache miss, but `checkLocked` guarantees that all input files have their digests computed, even on a |
| 893 | 918 | // cache miss, which is needed because they will be used in the manifest digest. |
| 894 | | man.dirty = true; |
| 895 | | man.shrinkFilesToInput(); |
| 896 | | hashFiles(man, &input_digest); |
| 919 | man.state = .miss; |
| 897 | 920 | return miss_result; |
| 898 | 921 | } |
| 899 | 922 | |
| ... | ... | @@ -904,19 +927,22 @@ pub const Manifest = struct { |
| 904 | 927 | }; |
| 905 | 928 | } |
| 906 | 929 | |
| 907 | | hashFiles(man, &input_digest); |
| 930 | man.state = .hit; |
| 908 | 931 | return .hit; |
| 909 | 932 | } |
| 910 | 933 | |
| 911 | | fn hashFiles(m: *Manifest, input_digest: *BinDigest) void { |
| 912 | | const contents = m.contents.items; |
| 913 | | |
| 914 | | m.hash.hasher = hasher_init; |
| 915 | | m.hash.hasher.update(input_digest); |
| 916 | | |
| 917 | | for (m.files.keys()) |off| { |
| 918 | | const file = off.get(contents); |
| 919 | | m.hash.hasher.update(&file.digest); |
| 934 | fn transitionToMissDiscovered(m: *Manifest) void { |
| 935 | switch (m.state) { |
| 936 | .input => unreachable, |
| 937 | .hit => unreachable, |
| 938 | .hit_digested => unreachable, |
| 939 | .miss => { |
| 940 | m.shrinkFilesToInput(); |
| 941 | m.state = .miss_discovered; |
| 942 | }, |
| 943 | .miss_discovered => {}, |
| 944 | .miss_digested => unreachable, |
| 945 | .miss_finalized => unreachable, |
| 920 | 946 | } |
| 921 | 947 | } |
| 922 | 948 | |
| ... | ... | @@ -1611,6 +1637,7 @@ pub const Manifest = struct { |
| 1611 | 1637 | /// * `addInputPath` |
| 1612 | 1638 | pub fn addDiscoveredPath(m: *Manifest, options: AddDiscoveredPathOptions) AddDiscoveredPathError!void { |
| 1613 | 1639 | assert(m.manifest_file != null); |
| 1640 | transitionToMissDiscovered(m); |
| 1614 | 1641 | const cache = m.cache; |
| 1615 | 1642 | const gpa = cache.gpa; |
| 1616 | 1643 | const io = cache.io; |
| ... | ... | @@ -1705,8 +1732,6 @@ pub const Manifest = struct { |
| 1705 | 1732 | try populateFile(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic); |
| 1706 | 1733 | }, |
| 1707 | 1734 | } |
| 1708 | | |
| 1709 | | m.hash.hasher.update(&header.digest); |
| 1710 | 1735 | } |
| 1711 | 1736 | |
| 1712 | 1737 | fn populateFile( |
| ... | ... | @@ -1840,39 +1865,63 @@ pub const Manifest = struct { |
| 1840 | 1865 | }; |
| 1841 | 1866 | } |
| 1842 | 1867 | |
| 1843 | | /// Returns a binary hash of the inputs, input file contents, and discovered file contents. |
| 1844 | | pub fn finalBin(self: *Manifest) BinDigest { |
| 1845 | | assert(self.manifest_file != null); |
| 1868 | /// See also `hitDigestHex`. |
| 1869 | pub fn hitDigest(m: *Manifest) BinDigest { |
| 1870 | assert(m.manifest_file != null); |
| 1871 | assert(m.state == .hit); |
| 1872 | m.state = .hit_digested; |
| 1873 | return finalDigest(m); |
| 1874 | } |
| 1875 | |
| 1876 | pub fn hitDigestHex(m: *Manifest) HexDigest { |
| 1877 | return binToHex(m.hitDigest()); |
| 1878 | } |
| 1879 | |
| 1880 | /// See also `missDigestHex`. |
| 1881 | pub fn missDigest(m: *Manifest) BinDigest { |
| 1882 | assert(m.manifest_file != null); |
| 1883 | transitionToMissDiscovered(m); |
| 1884 | m.state = .miss_digested; |
| 1885 | return finalDigest(m); |
| 1886 | } |
| 1846 | 1887 | |
| 1888 | pub fn missDigestHex(m: *Manifest) HexDigest { |
| 1889 | return binToHex(m.missDigest()); |
| 1890 | } |
| 1891 | |
| 1892 | fn finalDigest(m: *Manifest) BinDigest { |
| 1893 | const contents = m.contents.items; |
| 1894 | const hasher = &m.hash.hasher; |
| 1895 | |
| 1896 | for (m.files.keys()) |off| { |
| 1897 | const file = off.get(contents); |
| 1898 | hasher.update(&file.digest); |
| 1899 | } |
| 1847 | 1900 | // We don't close the manifest file yet, because we want to keep it locked until the API user is done using it. |
| 1848 | 1901 | // We also don't write out the manifest yet, because until `finalize` is called we still might be working on |
| 1849 | 1902 | // creating the artifacts to cache. |
| 1850 | | |
| 1851 | 1903 | var bin_digest: BinDigest = undefined; |
| 1852 | | self.hash.hasher.final(&bin_digest); |
| 1904 | hasher.final(&bin_digest); |
| 1853 | 1905 | return bin_digest; |
| 1854 | 1906 | } |
| 1855 | 1907 | |
| 1856 | | /// Returns a hex encoded hash of the inputs, input file contents, and discovered file contents. |
| 1857 | | pub fn final(self: *Manifest) HexDigest { |
| 1858 | | const bin_digest = self.finalBin(); |
| 1859 | | return binToHex(bin_digest); |
| 1860 | | } |
| 1861 | | |
| 1862 | 1908 | /// If `want_shared_lock` is true, this function automatically downgrades the |
| 1863 | 1909 | /// lock from exclusive to shared. |
| 1864 | 1910 | pub fn finalize(m: *Manifest) !void { |
| 1865 | | assert(m.have_exclusive_lock); |
| 1866 | 1911 | const io = m.cache.io; |
| 1867 | 1912 | const manifest_file = m.manifest_file.?; |
| 1868 | | if (m.dirty) { |
| 1913 | |
| 1914 | assert(m.state == .miss_digested); |
| 1915 | assert(m.have_exclusive_lock); |
| 1916 | |
| 1917 | { |
| 1869 | 1918 | m.contents.appendAssumeCapacity(0); |
| 1870 | 1919 | defer _ = m.contents.pop().?; |
| 1871 | 1920 | |
| 1872 | 1921 | try manifest_file.setLength(io, m.contents.items.len); |
| 1873 | 1922 | try manifest_file.writePositionalAll(io, m.contents.items, 0); |
| 1874 | 1923 | |
| 1875 | | m.dirty = false; |
| 1924 | m.state = .miss_finalized; |
| 1876 | 1925 | } |
| 1877 | 1926 | |
| 1878 | 1927 | if (m.want_shared_lock) { |
| ... | ... | @@ -2197,39 +2246,39 @@ test "cache file and then recall it" { |
| 2197 | 2246 | defer cache.manifest_dir.close(io); |
| 2198 | 2247 | |
| 2199 | 2248 | { |
| 2200 | | var ch = cache.obtain(); |
| 2201 | | defer ch.deinit(); |
| 2249 | var man = cache.obtain(); |
| 2250 | defer man.deinit(); |
| 2202 | 2251 | |
| 2203 | | ch.hash.add(true); |
| 2204 | | ch.hash.add(@as(u16, 1234)); |
| 2205 | | ch.hash.addBytes("1234"); |
| 2206 | | _ = try ch.addInputPath(.{ |
| 2252 | man.hash.add(true); |
| 2253 | man.hash.add(@as(u16, 1234)); |
| 2254 | man.hash.addBytes("1234"); |
| 2255 | _ = try man.addInputPath(.{ |
| 2207 | 2256 | .root_dir = tmp_directory, |
| 2208 | 2257 | .sub_path = temp_file, |
| 2209 | 2258 | }, .{}); |
| 2210 | 2259 | |
| 2211 | | try testing.expectEqual(.incomplete_manifest, try ch.check(.none)); |
| 2260 | try testing.expectEqual(.incomplete_manifest, try man.check(.none)); |
| 2212 | 2261 | |
| 2213 | | digest1 = ch.final(); |
| 2214 | | try ch.finalize(); |
| 2262 | digest1 = man.missDigestHex(); |
| 2263 | try man.finalize(); |
| 2215 | 2264 | } |
| 2216 | 2265 | { |
| 2217 | | var ch = cache.obtain(); |
| 2218 | | defer ch.deinit(); |
| 2266 | var man = cache.obtain(); |
| 2267 | defer man.deinit(); |
| 2219 | 2268 | |
| 2220 | | ch.hash.add(true); |
| 2221 | | ch.hash.add(@as(u16, 1234)); |
| 2222 | | ch.hash.addBytes("1234"); |
| 2223 | | _ = try ch.addInputPath(.{ |
| 2269 | man.hash.add(true); |
| 2270 | man.hash.add(@as(u16, 1234)); |
| 2271 | man.hash.addBytes("1234"); |
| 2272 | _ = try man.addInputPath(.{ |
| 2224 | 2273 | .root_dir = tmp_directory, |
| 2225 | 2274 | .sub_path = temp_file, |
| 2226 | 2275 | }, .{}); |
| 2227 | 2276 | |
| 2228 | 2277 | // Cache hit! We just "built" the same file |
| 2229 | | try testing.expectEqual(.hit, try ch.check(.none)); |
| 2230 | | digest2 = ch.final(); |
| 2278 | try testing.expectEqual(.hit, try man.check(.none)); |
| 2279 | digest2 = man.hitDigestHex(); |
| 2231 | 2280 | |
| 2232 | | try testing.expectEqual(false, ch.have_exclusive_lock); |
| 2281 | try testing.expectEqual(false, man.have_exclusive_lock); |
| 2233 | 2282 | } |
| 2234 | 2283 | |
| 2235 | 2284 | try testing.expectEqual(digest1, digest2); |
| ... | ... | @@ -2291,7 +2340,7 @@ test "check that changing a file causes cache miss" { |
| 2291 | 2340 | |
| 2292 | 2341 | try testing.expectEqualStrings(original_temp_file_contents, temp_file_idx.contents(&man)); |
| 2293 | 2342 | |
| 2294 | | digest1 = man.final(); |
| 2343 | digest1 = man.missDigestHex(); |
| 2295 | 2344 | |
| 2296 | 2345 | try man.finalize(); |
| 2297 | 2346 | } |
| ... | ... | @@ -2316,7 +2365,7 @@ test "check that changing a file causes cache miss" { |
| 2316 | 2365 | |
| 2317 | 2366 | try testing.expectEqualStrings(updated_temp_file_contents, temp_file_idx.contents(&man)); |
| 2318 | 2367 | |
| 2319 | | digest2 = man.final(); |
| 2368 | digest2 = man.missDigestHex(); |
| 2320 | 2369 | |
| 2321 | 2370 | try man.finalize(); |
| 2322 | 2371 | } |
| ... | ... | @@ -2356,7 +2405,7 @@ test "no file inputs" { |
| 2356 | 2405 | |
| 2357 | 2406 | try testing.expectEqual(.incomplete_manifest, try man.check(.none)); |
| 2358 | 2407 | |
| 2359 | | digest1 = man.final(); |
| 2408 | digest1 = man.missDigestHex(); |
| 2360 | 2409 | |
| 2361 | 2410 | try man.finalize(); |
| 2362 | 2411 | } |
| ... | ... | @@ -2367,14 +2416,14 @@ test "no file inputs" { |
| 2367 | 2416 | man.hash.addBytes("1234"); |
| 2368 | 2417 | |
| 2369 | 2418 | try testing.expectEqual(.hit, try man.check(.none)); |
| 2370 | | digest2 = man.final(); |
| 2419 | digest2 = man.hitDigestHex(); |
| 2371 | 2420 | try testing.expectEqual(false, man.have_exclusive_lock); |
| 2372 | 2421 | } |
| 2373 | 2422 | |
| 2374 | 2423 | try testing.expectEqual(digest1, digest2); |
| 2375 | 2424 | } |
| 2376 | 2425 | |
| 2377 | | test "Manifest with files added after initial hash work" { |
| 2426 | test "Manifest with files added after initial hash" { |
| 2378 | 2427 | const io = testing.io; |
| 2379 | 2428 | |
| 2380 | 2429 | var tmp = testing.tmpDir(.{}); |
| ... | ... | @@ -2433,7 +2482,7 @@ test "Manifest with files added after initial hash work" { |
| 2433 | 2482 | .sub_path = temp_file2, |
| 2434 | 2483 | } } }); |
| 2435 | 2484 | |
| 2436 | | digest1 = man.final(); |
| 2485 | digest1 = man.missDigestHex(); |
| 2437 | 2486 | try man.finalize(); |
| 2438 | 2487 | } |
| 2439 | 2488 | { |
| ... | ... | @@ -2447,7 +2496,7 @@ test "Manifest with files added after initial hash work" { |
| 2447 | 2496 | }, .{}); |
| 2448 | 2497 | |
| 2449 | 2498 | try testing.expect(.hit == try man.check(.none)); |
| 2450 | | digest2 = man.final(); |
| 2499 | digest2 = man.hitDigestHex(); |
| 2451 | 2500 | |
| 2452 | 2501 | try testing.expectEqual(false, man.have_exclusive_lock); |
| 2453 | 2502 | } |
| ... | ... | @@ -2484,7 +2533,7 @@ test "Manifest with files added after initial hash work" { |
| 2484 | 2533 | .sub_path = temp_file2, |
| 2485 | 2534 | } } }); |
| 2486 | 2535 | |
| 2487 | | digest3 = man.final(); |
| 2536 | digest3 = man.missDigestHex(); |
| 2488 | 2537 | |
| 2489 | 2538 | try man.finalize(); |
| 2490 | 2539 | } |
Andrew Kelley <andrew@ziglang.org> 2026-09-13 19:26:12-07:00