| ... | @@ -48,7 +48,7 @@ pub fn obtain(cache: *Cache) Manifest { | ... | @@ -48,7 +48,7 @@ pub fn obtain(cache: *Cache) Manifest { |
| 48 | .cache = cache, | 48 | .cache = cache, |
| 49 | .hash = cache.hash, | 49 | .hash = cache.hash, |
| 50 | .manifest_file = null, | 50 | .manifest_file = null, |
| 51 | .dirty = false, | 51 | .state = .input, |
| 52 | }; | 52 | }; |
| 53 | } | 53 | } |
| 54 | | 54 | |
| ... | @@ -274,8 +274,7 @@ pub const Manifest = struct { | ... | @@ -274,8 +274,7 @@ pub const Manifest = struct { |
| 274 | hash: HashHelper, | 274 | hash: HashHelper, |
| 275 | /// When this is null, `Manifest` is in "pre-check" phase. Otherwise it is in "post-check" phase. | 275 | /// When this is null, `Manifest` is in "pre-check" phase. Otherwise it is in "post-check" phase. |
| 276 | manifest_file: ?Io.File, | 276 | manifest_file: ?Io.File, |
| 277 | /// `finalize` has not been called yet. | 277 | state: State, |
| 278 | dirty: bool, | | |
| 279 | /// Set this flag to true before calling `check` in order to indicate that upon a cache hit, the code | 278 | /// Set this flag to true before calling `check` in order to indicate that upon a cache hit, the code |
| 280 | /// using the cache will not modify the files within the cache directory. This allows multiple processes | 279 | /// using the cache will not modify the files within the cache directory. This allows multiple processes |
| 281 | /// to utilize the same cache directory at the same time. | 280 | /// to utilize the same cache directory at the same time. |
| ... | @@ -309,6 +308,27 @@ pub const Manifest = struct { | ... | @@ -309,6 +308,27 @@ pub const Manifest = struct { |
| 309 | | 308 | |
| 310 | pub const Files = std.array_hash_map.Custom(File.Offset, void, File.HashContext, false); | 309 | pub const Files = std.array_hash_map.Custom(File.Offset, void, File.HashContext, false); |
| 311 | | 310 | |
| | 311 | pub const State = enum { |
| | 312 | /// Call `addInputPath`, `addInputPathOptional`, and methods of `Manifest.hash`. Once all input files and state |
| | 313 | /// have been added, call `check`. |
| | 314 | input, |
| | 315 | /// `check` has already been called and a hit occurred. Call `hitDigest` to find cache artifact directory. |
| | 316 | hit, |
| | 317 | /// `hitDigest` has already been called. No further operations are allowed. |
| | 318 | hit_digested, |
| | 319 | /// `check` has already been called and a miss occurred. `missDigest` has not been called yet. Miss diagnostic |
| | 320 | /// data may still be accessed. Call `addDiscoveredPath` to add discovered files and directories. |
| | 321 | miss, |
| | 322 | /// A miss occurred, `missDigest` has not been called yet. Call `addDiscoveredPath` to add more discovered files |
| | 323 | /// and directories. Miss diagnostic data may no longer be accessed. |
| | 324 | miss_discovered, |
| | 325 | /// `missDigest` has been called, but `finalize` has not been called yet. |
| | 326 | miss_digested, |
| | 327 | /// The cache manifest has been written to disk such that the next check for the same inputs will yield a hit. |
| | 328 | /// No further operations are allowed. |
| | 329 | miss_finalized, |
| | 330 | }; |
| | 331 | |
| 312 | /// Source files and directories whose prefix and relative path are | 332 | /// Source files and directories whose prefix and relative path are |
| 313 | /// included when computing the cache manifest digest. It's the information | 333 | /// included when computing the cache manifest digest. It's the information |
| 314 | /// needed to lazily hash the input files only when a cache miss occurs. | 334 | /// needed to lazily hash the input files only when a cache miss occurs. |
| ... | @@ -656,6 +676,7 @@ pub const Manifest = struct { | ... | @@ -656,6 +676,7 @@ pub const Manifest = struct { |
| 656 | /// See also: | 676 | /// See also: |
| 657 | /// * `addDiscoveredPath` | 677 | /// * `addDiscoveredPath` |
| 658 | pub fn addInputPath(m: *Manifest, path: Path, options: AddInputPathOptions) Allocator.Error!InputPath.Index { | 678 | pub fn addInputPath(m: *Manifest, path: Path, options: AddInputPathOptions) Allocator.Error!InputPath.Index { |
| | 679 | assert(m.state == .input); |
| 659 | const cache = m.cache; | 680 | const cache = m.cache; |
| 660 | const gpa = cache.gpa; | 681 | const gpa = cache.gpa; |
| 661 | const is_directory = options.handle.isDirectory(); | 682 | const is_directory = options.handle.isDirectory(); |
| ... | @@ -733,6 +754,7 @@ pub const Manifest = struct { | ... | @@ -733,6 +754,7 @@ pub const Manifest = struct { |
| 733 | } | 754 | } |
| 734 | | 755 | |
| 735 | pub fn addInputPathOptional(m: *Manifest, opt_path: ?Path, options: AddInputPathOptions) Allocator.Error!void { | 756 | pub fn addInputPathOptional(m: *Manifest, opt_path: ?Path, options: AddInputPathOptions) Allocator.Error!void { |
| | 757 | assert(m.state == .input); |
| 736 | m.hash.add(opt_path != null); | 758 | m.hash.add(opt_path != null); |
| 737 | _ = try addInputPath(m, opt_path orelse return, options); | 759 | _ = try addInputPath(m, opt_path orelse return, options); |
| 738 | } | 760 | } |
| ... | @@ -772,6 +794,7 @@ pub const Manifest = struct { | ... | @@ -772,6 +794,7 @@ pub const Manifest = struct { |
| 772 | } | 794 | } |
| 773 | | 795 | |
| 774 | pub fn checkProgressless(man: *Manifest) CheckError!CheckResult { | 796 | pub fn checkProgressless(man: *Manifest) CheckError!CheckResult { |
| | 797 | assert(man.state == .input); |
| 775 | assert(man.manifest_file == null); | 798 | assert(man.manifest_file == null); |
| 776 | | 799 | |
| 777 | // This is *not* hashing the contents of the input files. It is the | 800 | // This is *not* hashing the contents of the input files. It is the |
| ... | @@ -872,6 +895,8 @@ pub const Manifest = struct { | ... | @@ -872,6 +895,8 @@ pub const Manifest = struct { |
| 872 | // of all non-discovered files -- that is, the ones we've already been told about. The rest will be discovered | 895 | // of all non-discovered files -- that is, the ones we've already been told about. The rest will be discovered |
| 873 | // through calls to `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user | 896 | // through calls to `addDiscoveredPath` etc, which will update the hasher. After all files are added, the user |
| 874 | // can use `final`, and will at some point `finalize` the file list to disk. | 897 | // can use `final`, and will at some point `finalize` the file list to disk. |
| | 898 | man.hash.hasher = hasher_init; |
| | 899 | man.hash.hasher.update(&input_digest); |
| 875 | | 900 | |
| 876 | hit: { | 901 | hit: { |
| 877 | const miss_result = miss: { | 902 | const miss_result = miss: { |
| ... | @@ -891,9 +916,7 @@ pub const Manifest = struct { | ... | @@ -891,9 +916,7 @@ pub const Manifest = struct { |
| 891 | | 916 | |
| 892 | // Cache miss, but `checkLocked` guarantees that all input files have their digests computed, even on a | 917 | // Cache miss, but `checkLocked` guarantees that all input files have their digests computed, even on a |
| 893 | // cache miss, which is needed because they will be used in the manifest digest. | 918 | // cache miss, which is needed because they will be used in the manifest digest. |
| 894 | man.dirty = true; | 919 | man.state = .miss; |
| 895 | man.shrinkFilesToInput(); | | |
| 896 | hashFiles(man, &input_digest); | | |
| 897 | return miss_result; | 920 | return miss_result; |
| 898 | } | 921 | } |
| 899 | | 922 | |
| ... | @@ -904,19 +927,22 @@ pub const Manifest = struct { | ... | @@ -904,19 +927,22 @@ pub const Manifest = struct { |
| 904 | }; | 927 | }; |
| 905 | } | 928 | } |
| 906 | | 929 | |
| 907 | hashFiles(man, &input_digest); | 930 | man.state = .hit; |
| 908 | return .hit; | 931 | return .hit; |
| 909 | } | 932 | } |
| 910 | | 933 | |
| 911 | fn hashFiles(m: *Manifest, input_digest: *BinDigest) void { | 934 | fn transitionToMissDiscovered(m: *Manifest) void { |
| 912 | const contents = m.contents.items; | 935 | switch (m.state) { |
| 913 | | 936 | .input => unreachable, |
| 914 | m.hash.hasher = hasher_init; | 937 | .hit => unreachable, |
| 915 | m.hash.hasher.update(input_digest); | 938 | .hit_digested => unreachable, |
| 916 | | 939 | .miss => { |
| 917 | for (m.files.keys()) |off| { | 940 | m.shrinkFilesToInput(); |
| 918 | const file = off.get(contents); | 941 | m.state = .miss_discovered; |
| 919 | m.hash.hasher.update(&file.digest); | 942 | }, |
| | 943 | .miss_discovered => {}, |
| | 944 | .miss_digested => unreachable, |
| | 945 | .miss_finalized => unreachable, |
| 920 | } | 946 | } |
| 921 | } | 947 | } |
| 922 | | 948 | |
| ... | @@ -1611,6 +1637,7 @@ pub const Manifest = struct { | ... | @@ -1611,6 +1637,7 @@ pub const Manifest = struct { |
| 1611 | /// * `addInputPath` | 1637 | /// * `addInputPath` |
| 1612 | pub fn addDiscoveredPath(m: *Manifest, options: AddDiscoveredPathOptions) AddDiscoveredPathError!void { | 1638 | pub fn addDiscoveredPath(m: *Manifest, options: AddDiscoveredPathOptions) AddDiscoveredPathError!void { |
| 1613 | assert(m.manifest_file != null); | 1639 | assert(m.manifest_file != null); |
| | 1640 | transitionToMissDiscovered(m); |
| 1614 | const cache = m.cache; | 1641 | const cache = m.cache; |
| 1615 | const gpa = cache.gpa; | 1642 | const gpa = cache.gpa; |
| 1616 | const io = cache.io; | 1643 | const io = cache.io; |
| ... | @@ -1705,8 +1732,6 @@ pub const Manifest = struct { | ... | @@ -1705,8 +1732,6 @@ pub const Manifest = struct { |
| 1705 | try populateFile(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic); | 1732 | try populateFile(m, header, need_stat, handle, options.contents, metadata_only, options.diagnostic); |
| 1706 | }, | 1733 | }, |
| 1707 | } | 1734 | } |
| 1708 | | | |
| 1709 | m.hash.hasher.update(&header.digest); | | |
| 1710 | } | 1735 | } |
| 1711 | | 1736 | |
| 1712 | fn populateFile( | 1737 | fn populateFile( |
| ... | @@ -1840,39 +1865,63 @@ pub const Manifest = struct { | ... | @@ -1840,39 +1865,63 @@ pub const Manifest = struct { |
| 1840 | }; | 1865 | }; |
| 1841 | } | 1866 | } |
| 1842 | | 1867 | |
| 1843 | /// Returns a binary hash of the inputs, input file contents, and discovered file contents. | 1868 | /// See also `hitDigestHex`. |
| 1844 | pub fn finalBin(self: *Manifest) BinDigest { | 1869 | pub fn hitDigest(m: *Manifest) BinDigest { |
| 1845 | assert(self.manifest_file != null); | 1870 | assert(m.manifest_file != null); |
| | 1871 | assert(m.state == .hit); |
| | 1872 | m.state = .hit_digested; |
| | 1873 | return finalDigest(m); |
| | 1874 | } |
| | 1875 | |
| | 1876 | pub fn hitDigestHex(m: *Manifest) HexDigest { |
| | 1877 | return binToHex(m.hitDigest()); |
| | 1878 | } |
| | 1879 | |
| | 1880 | /// See also `missDigestHex`. |
| | 1881 | pub fn missDigest(m: *Manifest) BinDigest { |
| | 1882 | assert(m.manifest_file != null); |
| | 1883 | transitionToMissDiscovered(m); |
| | 1884 | m.state = .miss_digested; |
| | 1885 | return finalDigest(m); |
| | 1886 | } |
| 1846 | | 1887 | |
| | 1888 | pub fn missDigestHex(m: *Manifest) HexDigest { |
| | 1889 | return binToHex(m.missDigest()); |
| | 1890 | } |
| | 1891 | |
| | 1892 | fn finalDigest(m: *Manifest) BinDigest { |
| | 1893 | const contents = m.contents.items; |
| | 1894 | const hasher = &m.hash.hasher; |
| | 1895 | |
| | 1896 | for (m.files.keys()) |off| { |
| | 1897 | const file = off.get(contents); |
| | 1898 | hasher.update(&file.digest); |
| | 1899 | } |
| 1847 | // We don't close the manifest file yet, because we want to keep it locked until the API user is done using it. | 1900 | // We don't close the manifest file yet, because we want to keep it locked until the API user is done using it. |
| 1848 | // We also don't write out the manifest yet, because until `finalize` is called we still might be working on | 1901 | // We also don't write out the manifest yet, because until `finalize` is called we still might be working on |
| 1849 | // creating the artifacts to cache. | 1902 | // creating the artifacts to cache. |
| 1850 | | | |
| 1851 | var bin_digest: BinDigest = undefined; | 1903 | var bin_digest: BinDigest = undefined; |
| 1852 | self.hash.hasher.final(&bin_digest); | 1904 | hasher.final(&bin_digest); |
| 1853 | return bin_digest; | 1905 | return bin_digest; |
| 1854 | } | 1906 | } |
| 1855 | | 1907 | |
| 1856 | /// Returns a hex encoded hash of the inputs, input file contents, and discovered file contents. | | |
| 1857 | pub fn final(self: *Manifest) HexDigest { | | |
| 1858 | const bin_digest = self.finalBin(); | | |
| 1859 | return binToHex(bin_digest); | | |
| 1860 | } | | |
| 1861 | | | |
| 1862 | /// If `want_shared_lock` is true, this function automatically downgrades the | 1908 | /// If `want_shared_lock` is true, this function automatically downgrades the |
| 1863 | /// lock from exclusive to shared. | 1909 | /// lock from exclusive to shared. |
| 1864 | pub fn finalize(m: *Manifest) !void { | 1910 | pub fn finalize(m: *Manifest) !void { |
| 1865 | assert(m.have_exclusive_lock); | | |
| 1866 | const io = m.cache.io; | 1911 | const io = m.cache.io; |
| 1867 | const manifest_file = m.manifest_file.?; | 1912 | const manifest_file = m.manifest_file.?; |
| 1868 | if (m.dirty) { | 1913 | |
| | 1914 | assert(m.state == .miss_digested); |
| | 1915 | assert(m.have_exclusive_lock); |
| | 1916 | |
| | 1917 | { |
| 1869 | m.contents.appendAssumeCapacity(0); | 1918 | m.contents.appendAssumeCapacity(0); |
| 1870 | defer _ = m.contents.pop().?; | 1919 | defer _ = m.contents.pop().?; |
| 1871 | | 1920 | |
| 1872 | try manifest_file.setLength(io, m.contents.items.len); | 1921 | try manifest_file.setLength(io, m.contents.items.len); |
| 1873 | try manifest_file.writePositionalAll(io, m.contents.items, 0); | 1922 | try manifest_file.writePositionalAll(io, m.contents.items, 0); |
| 1874 | | 1923 | |
| 1875 | m.dirty = false; | 1924 | m.state = .miss_finalized; |
| 1876 | } | 1925 | } |
| 1877 | | 1926 | |
| 1878 | if (m.want_shared_lock) { | 1927 | if (m.want_shared_lock) { |
| ... | @@ -2197,39 +2246,39 @@ test "cache file and then recall it" { | ... | @@ -2197,39 +2246,39 @@ test "cache file and then recall it" { |
| 2197 | defer cache.manifest_dir.close(io); | 2246 | defer cache.manifest_dir.close(io); |
| 2198 | | 2247 | |
| 2199 | { | 2248 | { |
| 2200 | var ch = cache.obtain(); | 2249 | var man = cache.obtain(); |
| 2201 | defer ch.deinit(); | 2250 | defer man.deinit(); |
| 2202 | | 2251 | |
| 2203 | ch.hash.add(true); | 2252 | man.hash.add(true); |
| 2204 | ch.hash.add(@as(u16, 1234)); | 2253 | man.hash.add(@as(u16, 1234)); |
| 2205 | ch.hash.addBytes("1234"); | 2254 | man.hash.addBytes("1234"); |
| 2206 | _ = try ch.addInputPath(.{ | 2255 | _ = try man.addInputPath(.{ |
| 2207 | .root_dir = tmp_directory, | 2256 | .root_dir = tmp_directory, |
| 2208 | .sub_path = temp_file, | 2257 | .sub_path = temp_file, |
| 2209 | }, .{}); | 2258 | }, .{}); |
| 2210 | | 2259 | |
| 2211 | try testing.expectEqual(.incomplete_manifest, try ch.check(.none)); | 2260 | try testing.expectEqual(.incomplete_manifest, try man.check(.none)); |
| 2212 | | 2261 | |
| 2213 | digest1 = ch.final(); | 2262 | digest1 = man.missDigestHex(); |
| 2214 | try ch.finalize(); | 2263 | try man.finalize(); |
| 2215 | } | 2264 | } |
| 2216 | { | 2265 | { |
| 2217 | var ch = cache.obtain(); | 2266 | var man = cache.obtain(); |
| 2218 | defer ch.deinit(); | 2267 | defer man.deinit(); |
| 2219 | | 2268 | |
| 2220 | ch.hash.add(true); | 2269 | man.hash.add(true); |
| 2221 | ch.hash.add(@as(u16, 1234)); | 2270 | man.hash.add(@as(u16, 1234)); |
| 2222 | ch.hash.addBytes("1234"); | 2271 | man.hash.addBytes("1234"); |
| 2223 | _ = try ch.addInputPath(.{ | 2272 | _ = try man.addInputPath(.{ |
| 2224 | .root_dir = tmp_directory, | 2273 | .root_dir = tmp_directory, |
| 2225 | .sub_path = temp_file, | 2274 | .sub_path = temp_file, |
| 2226 | }, .{}); | 2275 | }, .{}); |
| 2227 | | 2276 | |
| 2228 | // Cache hit! We just "built" the same file | 2277 | // Cache hit! We just "built" the same file |
| 2229 | try testing.expectEqual(.hit, try ch.check(.none)); | 2278 | try testing.expectEqual(.hit, try man.check(.none)); |
| 2230 | digest2 = ch.final(); | 2279 | digest2 = man.hitDigestHex(); |
| 2231 | | 2280 | |
| 2232 | try testing.expectEqual(false, ch.have_exclusive_lock); | 2281 | try testing.expectEqual(false, man.have_exclusive_lock); |
| 2233 | } | 2282 | } |
| 2234 | | 2283 | |
| 2235 | try testing.expectEqual(digest1, digest2); | 2284 | try testing.expectEqual(digest1, digest2); |
| ... | @@ -2291,7 +2340,7 @@ test "check that changing a file causes cache miss" { | ... | @@ -2291,7 +2340,7 @@ test "check that changing a file causes cache miss" { |
| 2291 | | 2340 | |
| 2292 | try testing.expectEqualStrings(original_temp_file_contents, temp_file_idx.contents(&man)); | 2341 | try testing.expectEqualStrings(original_temp_file_contents, temp_file_idx.contents(&man)); |
| 2293 | | 2342 | |
| 2294 | digest1 = man.final(); | 2343 | digest1 = man.missDigestHex(); |
| 2295 | | 2344 | |
| 2296 | try man.finalize(); | 2345 | try man.finalize(); |
| 2297 | } | 2346 | } |
| ... | @@ -2316,7 +2365,7 @@ test "check that changing a file causes cache miss" { | ... | @@ -2316,7 +2365,7 @@ test "check that changing a file causes cache miss" { |
| 2316 | | 2365 | |
| 2317 | try testing.expectEqualStrings(updated_temp_file_contents, temp_file_idx.contents(&man)); | 2366 | try testing.expectEqualStrings(updated_temp_file_contents, temp_file_idx.contents(&man)); |
| 2318 | | 2367 | |
| 2319 | digest2 = man.final(); | 2368 | digest2 = man.missDigestHex(); |
| 2320 | | 2369 | |
| 2321 | try man.finalize(); | 2370 | try man.finalize(); |
| 2322 | } | 2371 | } |
| ... | @@ -2356,7 +2405,7 @@ test "no file inputs" { | ... | @@ -2356,7 +2405,7 @@ test "no file inputs" { |
| 2356 | | 2405 | |
| 2357 | try testing.expectEqual(.incomplete_manifest, try man.check(.none)); | 2406 | try testing.expectEqual(.incomplete_manifest, try man.check(.none)); |
| 2358 | | 2407 | |
| 2359 | digest1 = man.final(); | 2408 | digest1 = man.missDigestHex(); |
| 2360 | | 2409 | |
| 2361 | try man.finalize(); | 2410 | try man.finalize(); |
| 2362 | } | 2411 | } |
| ... | @@ -2367,14 +2416,14 @@ test "no file inputs" { | ... | @@ -2367,14 +2416,14 @@ test "no file inputs" { |
| 2367 | man.hash.addBytes("1234"); | 2416 | man.hash.addBytes("1234"); |
| 2368 | | 2417 | |
| 2369 | try testing.expectEqual(.hit, try man.check(.none)); | 2418 | try testing.expectEqual(.hit, try man.check(.none)); |
| 2370 | digest2 = man.final(); | 2419 | digest2 = man.hitDigestHex(); |
| 2371 | try testing.expectEqual(false, man.have_exclusive_lock); | 2420 | try testing.expectEqual(false, man.have_exclusive_lock); |
| 2372 | } | 2421 | } |
| 2373 | | 2422 | |
| 2374 | try testing.expectEqual(digest1, digest2); | 2423 | try testing.expectEqual(digest1, digest2); |
| 2375 | } | 2424 | } |
| 2376 | | 2425 | |
| 2377 | test "Manifest with files added after initial hash work" { | 2426 | test "Manifest with files added after initial hash" { |
| 2378 | const io = testing.io; | 2427 | const io = testing.io; |
| 2379 | | 2428 | |
| 2380 | var tmp = testing.tmpDir(.{}); | 2429 | var tmp = testing.tmpDir(.{}); |
| ... | @@ -2433,7 +2482,7 @@ test "Manifest with files added after initial hash work" { | ... | @@ -2433,7 +2482,7 @@ test "Manifest with files added after initial hash work" { |
| 2433 | .sub_path = temp_file2, | 2482 | .sub_path = temp_file2, |
| 2434 | } } }); | 2483 | } } }); |
| 2435 | | 2484 | |
| 2436 | digest1 = man.final(); | 2485 | digest1 = man.missDigestHex(); |
| 2437 | try man.finalize(); | 2486 | try man.finalize(); |
| 2438 | } | 2487 | } |
| 2439 | { | 2488 | { |
| ... | @@ -2447,7 +2496,7 @@ test "Manifest with files added after initial hash work" { | ... | @@ -2447,7 +2496,7 @@ test "Manifest with files added after initial hash work" { |
| 2447 | }, .{}); | 2496 | }, .{}); |
| 2448 | | 2497 | |
| 2449 | try testing.expect(.hit == try man.check(.none)); | 2498 | try testing.expect(.hit == try man.check(.none)); |
| 2450 | digest2 = man.final(); | 2499 | digest2 = man.hitDigestHex(); |
| 2451 | | 2500 | |
| 2452 | try testing.expectEqual(false, man.have_exclusive_lock); | 2501 | try testing.expectEqual(false, man.have_exclusive_lock); |
| 2453 | } | 2502 | } |
| ... | @@ -2484,7 +2533,7 @@ test "Manifest with files added after initial hash work" { | ... | @@ -2484,7 +2533,7 @@ test "Manifest with files added after initial hash work" { |
| 2484 | .sub_path = temp_file2, | 2533 | .sub_path = temp_file2, |
| 2485 | } } }); | 2534 | } } }); |
| 2486 | | 2535 | |
| 2487 | digest3 = man.final(); | 2536 | digest3 = man.missDigestHex(); |
| 2488 | | 2537 | |
| 2489 | try man.finalize(); | 2538 | try man.finalize(); |
| 2490 | } | 2539 | } |
Andrew Kelley <andrew@ziglang.org> 2026-09-13 19:26:12-07:00