authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2020-08-21 15:08:15+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-08-22 15:10:33-04:00
logf540dc1b7ebc1663ef5d3823da4630ff51c697b6
tree9ed45cca43ebad7f0328ff88b12e2fe5ab54d837
parent0fa3cfdb4aa04bf92c5d9344cd4d265ccb40e0dc

cache_hash: hash function change

This makes the `cache_hash` hash function easier to replace. BLAKE3 would be a natural fit for hashing large files, but: - second preimage resistance is not necessary for the cache_hash use cases - our BLAKE3 implementation is currently very slow Switch to SipHash128, which gives us an immediate speed boost.

1 files changed, 38 insertions(+), 32 deletions(-)

lib/std/cache_hash.zig+38-32
...@@ -4,7 +4,8 @@...@@ -4,7 +4,8 @@
4// The MIT license requires this copyright notice to be included in all copies4// The MIT license requires this copyright notice to be included in all copies
5// and substantial portions of the software.5// and substantial portions of the software.
6const std = @import("std.zig");6const std = @import("std.zig");
7const Blake3 = std.crypto.hash.Blake3;7const crypto = std.crypto;
8const Hasher = crypto.auth.siphash.SipHash128(1, 3); // provides enough collision resistance for the CacheHash use cases, while being one of our fastest options right now
8const fs = std.fs;9const fs = std.fs;
9const base64 = std.base64;10const base64 = std.base64;
10const ArrayList = std.ArrayList;11const ArrayList = std.ArrayList;
...@@ -16,9 +17,8 @@ const Allocator = std.mem.Allocator;...@@ -16,9 +17,8 @@ const Allocator = std.mem.Allocator;
1617
17const base64_encoder = fs.base64_encoder;18const base64_encoder = fs.base64_encoder;
18const base64_decoder = fs.base64_decoder;19const base64_decoder = fs.base64_decoder;
19/// This is 70 more bits than UUIDs. For an analysis of probability of collisions, see:20/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
20/// https://en.wikipedia.org/wiki/Universally_unique_identifier#Collisions21const BIN_DIGEST_LEN = 16;
21const BIN_DIGEST_LEN = 24;
22const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);22const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
2323
24const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;24const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;
...@@ -43,9 +43,13 @@ pub const File = struct {...@@ -43,9 +43,13 @@ pub const File = struct {
43 }43 }
44};44};
4545
46/// CacheHash manages project-local `zig-cache` directories.
47/// This is not a general-purpose cache.
48/// It was designed to be fast and simple, not to withstand attacks using specially-crafted input.
46pub const CacheHash = struct {49pub const CacheHash = struct {
47 allocator: *Allocator,50 allocator: *Allocator,
48 blake3: Blake3,51 hasher_init: Hasher, // initial state, that can be copied
52 hasher: Hasher, // current state for incremental hashing
49 manifest_dir: fs.Dir,53 manifest_dir: fs.Dir,
50 manifest_file: ?fs.File,54 manifest_file: ?fs.File,
51 manifest_dirty: bool,55 manifest_dirty: bool,
...@@ -54,9 +58,11 @@ pub const CacheHash = struct {...@@ -54,9 +58,11 @@ pub const CacheHash = struct {
5458
55 /// Be sure to call release after successful initialization.59 /// Be sure to call release after successful initialization.
56 pub fn init(allocator: *Allocator, dir: fs.Dir, manifest_dir_path: []const u8) !CacheHash {60 pub fn init(allocator: *Allocator, dir: fs.Dir, manifest_dir_path: []const u8) !CacheHash {
61 const hasher_init = Hasher.init(&[_]u8{0} ** Hasher.minimum_key_length);
57 return CacheHash{62 return CacheHash{
58 .allocator = allocator,63 .allocator = allocator,
59 .blake3 = Blake3.init(.{}),64 .hasher_init = hasher_init,
65 .hasher = hasher_init,
60 .manifest_dir = try dir.makeOpenPath(manifest_dir_path, .{}),66 .manifest_dir = try dir.makeOpenPath(manifest_dir_path, .{}),
61 .manifest_file = null,67 .manifest_file = null,
62 .manifest_dirty = false,68 .manifest_dirty = false,
...@@ -69,8 +75,8 @@ pub const CacheHash = struct {...@@ -69,8 +75,8 @@ pub const CacheHash = struct {
69 pub fn addSlice(self: *CacheHash, val: []const u8) void {75 pub fn addSlice(self: *CacheHash, val: []const u8) void {
70 assert(self.manifest_file == null);76 assert(self.manifest_file == null);
7177
72 self.blake3.update(val);78 self.hasher.update(val);
73 self.blake3.update(&[_]u8{0});79 self.hasher.update(&[_]u8{0});
74 }80 }
7581
76 /// Convert the input value into bytes and record it as a dependency of the82 /// Convert the input value into bytes and record it as a dependency of the
...@@ -133,12 +139,12 @@ pub const CacheHash = struct {...@@ -133,12 +139,12 @@ pub const CacheHash = struct {
133 assert(self.manifest_file == null);139 assert(self.manifest_file == null);
134140
135 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;141 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
136 self.blake3.final(&bin_digest);142 self.hasher.final(&bin_digest);
137143
138 base64_encoder.encode(self.b64_digest[0..], &bin_digest);144 base64_encoder.encode(self.b64_digest[0..], &bin_digest);
139145
140 self.blake3 = Blake3.init(.{});146 self.hasher = self.hasher_init;
141 self.blake3.update(&bin_digest);147 self.hasher.update(&bin_digest);
142148
143 const manifest_file_path = try fmt.allocPrint(self.allocator, "{}.txt", .{self.b64_digest});149 const manifest_file_path = try fmt.allocPrint(self.allocator, "{}.txt", .{self.b64_digest});
144 defer self.allocator.free(manifest_file_path);150 defer self.allocator.free(manifest_file_path);
...@@ -238,7 +244,7 @@ pub const CacheHash = struct {...@@ -238,7 +244,7 @@ pub const CacheHash = struct {
238 }244 }
239245
240 var actual_digest: [BIN_DIGEST_LEN]u8 = undefined;246 var actual_digest: [BIN_DIGEST_LEN]u8 = undefined;
241 try hashFile(this_file, &actual_digest);247 try hashFile(this_file, &actual_digest, self.hasher_init);
242248
243 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {249 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
244 cache_hash_file.bin_digest = actual_digest;250 cache_hash_file.bin_digest = actual_digest;
...@@ -248,7 +254,7 @@ pub const CacheHash = struct {...@@ -248,7 +254,7 @@ pub const CacheHash = struct {
248 }254 }
249255
250 if (!any_file_changed) {256 if (!any_file_changed) {
251 self.blake3.update(&cache_hash_file.bin_digest);257 self.hasher.update(&cache_hash_file.bin_digest);
252 }258 }
253 }259 }
254260
...@@ -256,8 +262,8 @@ pub const CacheHash = struct {...@@ -256,8 +262,8 @@ pub const CacheHash = struct {
256 // cache miss262 // cache miss
257 // keep the manifest file open263 // keep the manifest file open
258 // reset the hash264 // reset the hash
259 self.blake3 = Blake3.init(.{});265 self.hasher = self.hasher_init;
260 self.blake3.update(&bin_digest);266 self.hasher.update(&bin_digest);
261267
262 // Remove files not in the initial hash268 // Remove files not in the initial hash
263 for (self.files.items[input_file_count..]) |*file| {269 for (self.files.items[input_file_count..]) |*file| {
...@@ -266,7 +272,7 @@ pub const CacheHash = struct {...@@ -266,7 +272,7 @@ pub const CacheHash = struct {
266 self.files.shrink(input_file_count);272 self.files.shrink(input_file_count);
267273
268 for (self.files.items) |file| {274 for (self.files.items) |file| {
269 self.blake3.update(&file.bin_digest);275 self.hasher.update(&file.bin_digest);
270 }276 }
271 return null;277 return null;
272 }278 }
...@@ -304,23 +310,23 @@ pub const CacheHash = struct {...@@ -304,23 +310,23 @@ pub const CacheHash = struct {
304310
305 // Hash while reading from disk, to keep the contents in the cpu cache while311 // Hash while reading from disk, to keep the contents in the cpu cache while
306 // doing hashing.312 // doing hashing.
307 var blake3 = Blake3.init(.{});313 var hasher = self.hasher_init;
308 var off: usize = 0;314 var off: usize = 0;
309 while (true) {315 while (true) {
310 // give me everything you've got, captain316 // give me everything you've got, captain
311 const bytes_read = try file.read(contents[off..]);317 const bytes_read = try file.read(contents[off..]);
312 if (bytes_read == 0) break;318 if (bytes_read == 0) break;
313 blake3.update(contents[off..][0..bytes_read]);319 hasher.update(contents[off..][0..bytes_read]);
314 off += bytes_read;320 off += bytes_read;
315 }321 }
316 blake3.final(&ch_file.bin_digest);322 hasher.final(&ch_file.bin_digest);
317323
318 ch_file.contents = contents;324 ch_file.contents = contents;
319 } else {325 } else {
320 try hashFile(file, &ch_file.bin_digest);326 try hashFile(file, &ch_file.bin_digest, self.hasher_init);
321 }327 }
322328
323 self.blake3.update(&ch_file.bin_digest);329 self.hasher.update(&ch_file.bin_digest);
324 }330 }
325331
326 /// Add a file as a dependency of process being cached, after the initial hash has been332 /// Add a file as a dependency of process being cached, after the initial hash has been
...@@ -382,7 +388,7 @@ pub const CacheHash = struct {...@@ -382,7 +388,7 @@ pub const CacheHash = struct {
382 // the artifacts to cache.388 // the artifacts to cache.
383389
384 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;390 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
385 self.blake3.final(&bin_digest);391 self.hasher.final(&bin_digest);
386392
387 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;393 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;
388 base64_encoder.encode(&out_digest, &bin_digest);394 base64_encoder.encode(&out_digest, &bin_digest);
...@@ -433,17 +439,17 @@ pub const CacheHash = struct {...@@ -433,17 +439,17 @@ pub const CacheHash = struct {
433 }439 }
434};440};
435441
436fn hashFile(file: fs.File, bin_digest: []u8) !void {442fn hashFile(file: fs.File, bin_digest: []u8, hasher_init: anytype) !void {
437 var blake3 = Blake3.init(.{});
438 var buf: [1024]u8 = undefined;443 var buf: [1024]u8 = undefined;
439444
445 var hasher = hasher_init;
440 while (true) {446 while (true) {
441 const bytes_read = try file.read(&buf);447 const bytes_read = try file.read(&buf);
442 if (bytes_read == 0) break;448 if (bytes_read == 0) break;
443 blake3.update(buf[0..bytes_read]);449 hasher.update(buf[0..bytes_read]);
444 }450 }
445451
446 blake3.final(bin_digest);452 hasher.final(bin_digest);
447}453}
448454
449/// If the wall clock time, rounded to the same precision as the455/// If the wall clock time, rounded to the same precision as the
...@@ -507,7 +513,7 @@ test "cache file and then recall it" {...@@ -507,7 +513,7 @@ test "cache file and then recall it" {
507 _ = try ch.addFile(temp_file, null);513 _ = try ch.addFile(temp_file, null);
508514
509 // There should be nothing in the cache515 // There should be nothing in the cache
510 testing.expectEqual(@as(?[32]u8, null), try ch.hit());516 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());
511517
512 digest1 = ch.final();518 digest1 = ch.final();
513 }519 }
...@@ -575,7 +581,7 @@ test "check that changing a file makes cache fail" {...@@ -575,7 +581,7 @@ test "check that changing a file makes cache fail" {
575 const temp_file_idx = try ch.addFile(temp_file, 100);581 const temp_file_idx = try ch.addFile(temp_file, 100);
576582
577 // There should be nothing in the cache583 // There should be nothing in the cache
578 testing.expectEqual(@as(?[32]u8, null), try ch.hit());584 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());
579585
580 testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));586 testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
581587
...@@ -592,7 +598,7 @@ test "check that changing a file makes cache fail" {...@@ -592,7 +598,7 @@ test "check that changing a file makes cache fail" {
592 const temp_file_idx = try ch.addFile(temp_file, 100);598 const temp_file_idx = try ch.addFile(temp_file, 100);
593599
594 // A file that we depend on has been updated, so the cache should not contain an entry for it600 // A file that we depend on has been updated, so the cache should not contain an entry for it
595 testing.expectEqual(@as(?[32]u8, null), try ch.hit());601 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());
596602
597 // The cache system does not keep the contents of re-hashed input files.603 // The cache system does not keep the contents of re-hashed input files.
598 testing.expect(ch.files.items[temp_file_idx].contents == null);604 testing.expect(ch.files.items[temp_file_idx].contents == null);
...@@ -625,7 +631,7 @@ test "no file inputs" {...@@ -625,7 +631,7 @@ test "no file inputs" {
625 ch.add("1234");631 ch.add("1234");
626632
627 // There should be nothing in the cache633 // There should be nothing in the cache
628 testing.expectEqual(@as(?[32]u8, null), try ch.hit());634 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());
629635
630 digest1 = ch.final();636 digest1 = ch.final();
631 }637 }
...@@ -672,7 +678,7 @@ test "CacheHashes with files added after initial hash work" {...@@ -672,7 +678,7 @@ test "CacheHashes with files added after initial hash work" {
672 _ = try ch.addFile(temp_file1, null);678 _ = try ch.addFile(temp_file1, null);
673679
674 // There should be nothing in the cache680 // There should be nothing in the cache
675 testing.expectEqual(@as(?[32]u8, null), try ch.hit());681 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());
676682
677 _ = try ch.addFilePost(temp_file2);683 _ = try ch.addFilePost(temp_file2);
678684
...@@ -705,7 +711,7 @@ test "CacheHashes with files added after initial hash work" {...@@ -705,7 +711,7 @@ test "CacheHashes with files added after initial hash work" {
705 _ = try ch.addFile(temp_file1, null);711 _ = try ch.addFile(temp_file1, null);
706712
707 // A file that we depend on has been updated, so the cache should not contain an entry for it713 // A file that we depend on has been updated, so the cache should not contain an entry for it
708 testing.expectEqual(@as(?[32]u8, null), try ch.hit());714 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());
709715
710 _ = try ch.addFilePost(temp_file2);716 _ = try ch.addFilePost(temp_file2);
711717