authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-13 18:04:17-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-13 18:04:52-07:00
log1baa56a25f7b4cf7b3e0a78ded90c432a40c4efa
tree631b636a5ec792aa063e3092f20be06d2c7180fd
parentaf4cc20ce275aeb0e59eee3d893b2f310c1f0239

std.cache_hash: break up the API and improve implementation

into smaller exposed components and expose all of them. This makes it more flexible. `*const Cache` is now passed in with an open manifest dir handle which the caller is responsible for managing. Expose some of the base64 stuff. Extract the hash helper functions into `HashHelper` and add some more methods such as addOptional and addListOfFiles. Add `CacheHash.toOwnedLock` so that you can deinitialize everything except the open file handle which represents the file system lock on the build artifacts. Use ArrayListUnmanaged, saving space per allocated CacheHash. Avoid 1 memory allocation in hit() with a static buffer. hit() returns a bool; caller code is responsible for calling final() in either case. This is a simpler and easier to use API. writeManifest() is no longer called from deinit() with errors ignored.

1 files changed, 240 insertions(+), 160 deletions(-)

lib/std/cache_hash.zig+240-160
......@@ -7,19 +7,18 @@ const std = @import("std.zig");
77const crypto = std.crypto;
88const fs = std.fs;
99const base64 = std.base64;
10const ArrayList = std.ArrayList;
1110const assert = std.debug.assert;
1211const testing = std.testing;
1312const mem = std.mem;
1413const fmt = std.fmt;
1514const Allocator = std.mem.Allocator;
1615
17const base64_encoder = fs.base64_encoder;
18const base64_decoder = fs.base64_decoder;
16pub const base64_encoder = fs.base64_encoder;
17pub const base64_decoder = fs.base64_decoder;
1918/// 16 would be 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
2019/// We round up to 18 to avoid the `==` padding after base64 encoding.
21const BIN_DIGEST_LEN = 18;
22const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
20pub const BIN_DIGEST_LEN = 18;
21pub const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
2322
2423const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;
2524
......@@ -51,87 +50,105 @@ pub const File = struct {
5150 }
5251};
5352
54/// CacheHash manages project-local `zig-cache` directories.
55/// This is not a general-purpose cache.
56/// It is designed to be fast and simple, not to withstand attacks using specially-crafted input.
57pub const CacheHash = struct {
58 allocator: *Allocator,
59 /// Current state for incremental hashing.
60 hasher: Hasher,
53pub const Cache = struct {
54 gpa: *Allocator,
6155 manifest_dir: fs.Dir,
62 manifest_file: ?fs.File,
63 manifest_dirty: bool,
64 owns_manifest_dir: bool,
65 files: ArrayList(File),
66 b64_digest: [BASE64_DIGEST_LEN]u8,
56 hash: HashHelper = .{},
6757
68 /// Be sure to call release after successful initialization.
69 pub fn init(allocator: *Allocator, dir: fs.Dir, manifest_dir_path: []const u8) !CacheHash {
58 /// Be sure to call `CacheHash.deinit` after successful initialization.
59 pub fn obtain(cache: *const Cache) CacheHash {
7060 return CacheHash{
71 .allocator = allocator,
72 .hasher = hasher_init,
73 .manifest_dir = try dir.makeOpenPath(manifest_dir_path, .{}),
61 .cache = cache,
62 .hash = cache.hash,
7463 .manifest_file = null,
7564 .manifest_dirty = false,
76 .owns_manifest_dir = true,
77 .files = ArrayList(File).init(allocator),
7865 .b64_digest = undefined,
7966 };
8067 }
68};
8169
82 /// Allows one to fork a CacheHash instance into another one, which does not require an additional
83 /// directory handle to be opened. The new instance inherits the hash state.
84 pub fn clone(self: CacheHash) CacheHash {
85 assert(self.manifest_file == null);
86 assert(files.items.len == 0);
87 return .{
88 .allocator = self.allocator,
89 .hasher = self.hasher,
90 .manifest_dir = self.manifest_dir,
91 .manifest_file = null,
92 .manifest_dirty = false,
93 .owns_manifest_dir = false,
94 .files = ArrayList(File).init(allocator),
95 .b64_digest = undefined,
96 };
97 }
70pub const HashHelper = struct {
71 hasher: Hasher = hasher_init,
9872
9973 /// Record a slice of bytes as an dependency of the process being cached
100 pub fn addBytes(self: *CacheHash, bytes: []const u8) void {
101 assert(self.manifest_file == null);
102
103 self.hasher.update(mem.asBytes(&bytes.len));
104 self.hasher.update(bytes);
74 pub fn addBytes(hh: *HashHelper, bytes: []const u8) void {
75 hh.hasher.update(mem.asBytes(&bytes.len));
76 hh.hasher.update(bytes);
10577 }
10678
107 pub fn addListOfBytes(self: *CacheHash, list_of_bytes: []const []const u8) void {
108 assert(self.manifest_file == null);
79 pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void {
80 hh.add(optional_bytes != null);
81 hh.addBytes(optional_bytes orelse return);
82 }
10983
110 self.add(list_of_bytes.items.len);
111 for (list_of_bytes) |bytes| self.addBytes(bytes);
84 pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void {
85 hh.add(list_of_bytes.items.len);
86 for (list_of_bytes) |bytes| hh.addBytes(bytes);
11287 }
11388
11489 /// Convert the input value into bytes and record it as a dependency of the process being cached.
115 pub fn add(self: *CacheHash, x: anytype) void {
116 assert(self.manifest_file == null);
117
90 pub fn add(hh: *HashHelper, x: anytype) void {
11891 switch (@TypeOf(x)) {
11992 std.builtin.Version => {
120 self.add(x.major);
121 self.add(x.minor);
122 self.add(x.patch);
93 hh.add(x.major);
94 hh.add(x.minor);
95 hh.add(x.patch);
12396 return;
12497 },
12598 else => {},
12699 }
127100
128101 switch (@typeInfo(@TypeOf(x))) {
129 .Bool, .Int, .Enum, .Array => self.addBytes(mem.asBytes(&x)),
102 .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)),
130103 else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),
131104 }
132105 }
133106
134 /// Add a file as a dependency of process being cached. When `CacheHash.hit` is
107 pub fn addOptional(hh: *HashHelper, optional: anytype) void {
108 hh.add(optional != null);
109 hh.add(optional orelse return);
110 }
111
112 /// Returns a base64 encoded hash of the inputs, without modifying state.
113 pub fn peek(hh: HashHelper) [BASE64_DIGEST_LEN]u8 {
114 var copy = hh;
115 return copy.final();
116 }
117
118 /// Returns a base64 encoded hash of the inputs, mutating the state of the hasher.
119 pub fn final(hh: *HashHelper) [BASE64_DIGEST_LEN]u8 {
120 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
121 hh.hasher.final(&bin_digest);
122
123 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;
124 base64_encoder.encode(&out_digest, &bin_digest);
125
126 return out_digest;
127 }
128};
129
130pub const Lock = struct {
131 manifest_file: fs.File,
132
133 pub fn release(lock: *Lock) void {
134 lock.manifest_file.close();
135 lock.* = undefined;
136 }
137};
138
139/// CacheHash manages project-local `zig-cache` directories.
140/// This is not a general-purpose cache.
141/// It is designed to be fast and simple, not to withstand attacks using specially-crafted input.
142pub const CacheHash = struct {
143 cache: *const Cache,
144 /// Current state for incremental hashing.
145 hash: HashHelper,
146 manifest_file: ?fs.File,
147 manifest_dirty: bool,
148 files: std.ArrayListUnmanaged(File) = .{},
149 b64_digest: [BASE64_DIGEST_LEN]u8,
150
151 /// Add a file as a dependency of process being cached. When `hit` is
135152 /// called, the file's contents will be checked to ensure that it matches
136153 /// the contents from previous times.
137154 ///
......@@ -139,8 +156,8 @@ pub const CacheHash = struct {
139156 /// are allowed to take up in memory. If max_file_size is null, then the contents
140157 /// will not be loaded into memory.
141158 ///
142 /// Returns the index of the entry in the `CacheHash.files` ArrayList. You can use it
143 /// to access the contents of the file after calling `CacheHash.hit()` like so:
159 /// Returns the index of the entry in the `files` array list. You can use it
160 /// to access the contents of the file after calling `hit()` like so:
144161 ///
145162 /// ```
146163 /// var file_contents = cache_hash.files.items[file_index].contents.?;
......@@ -148,8 +165,8 @@ pub const CacheHash = struct {
148165 pub fn addFile(self: *CacheHash, file_path: []const u8, max_file_size: ?usize) !usize {
149166 assert(self.manifest_file == null);
150167
151 try self.files.ensureCapacity(self.files.items.len + 1);
152 const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
168 try self.files.ensureCapacity(self.cache.gpa, self.files.items.len + 1);
169 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
153170
154171 const idx = self.files.items.len;
155172 self.files.addOneAssumeCapacity().* = .{
......@@ -160,35 +177,53 @@ pub const CacheHash = struct {
160177 .bin_digest = undefined,
161178 };
162179
163 self.addBytes(resolved_path);
180 self.hash.addBytes(resolved_path);
164181
165182 return idx;
166183 }
167184
168 /// Check the cache to see if the input exists in it. If it exists, a base64 encoding
169 /// of it's hash will be returned; otherwise, null will be returned.
185 pub fn addOptionalFile(self: *CacheHash, optional_file_path: ?[]const u8) !void {
186 self.hash.add(optional_file_path != null);
187 const file_path = optional_file_path orelse return;
188 _ = try self.addFile(file_path, null);
189 }
190
191 pub fn addListOfFiles(self: *CacheHash, list_of_files: []const []const u8) !void {
192 self.hash.add(list_of_files.len);
193 for (list_of_files) |file_path| {
194 _ = try self.addFile(file_path, null);
195 }
196 }
197
198 /// Check the cache to see if the input exists in it. If it exists, returns `true`.
199 /// A base64 encoding of its hash is available by calling `final`.
170200 ///
171201 /// This function will also acquire an exclusive lock to the manifest file. This means
172202 /// that a process holding a CacheHash will block any other process attempting to
173203 /// acquire the lock.
174204 ///
175 /// The lock on the manifest file is released when `CacheHash.release` is called.
176 pub fn hit(self: *CacheHash) !?[BASE64_DIGEST_LEN]u8 {
205 /// The lock on the manifest file is released when `deinit` is called. As another
206 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
207 /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
208 pub fn hit(self: *CacheHash) !bool {
177209 assert(self.manifest_file == null);
178210
211 const ext = ".txt";
212 var manifest_file_path: [self.b64_digest.len + ext.len]u8 = undefined;
213
179214 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
180 self.hasher.final(&bin_digest);
215 self.hash.hasher.final(&bin_digest);
181216
182217 base64_encoder.encode(self.b64_digest[0..], &bin_digest);
183218
184 self.hasher = hasher_init;
185 self.hasher.update(&bin_digest);
219 self.hash.hasher = hasher_init;
220 self.hash.hasher.update(&bin_digest);
186221
187 const manifest_file_path = try fmt.allocPrint(self.allocator, "{}.txt", .{self.b64_digest});
188 defer self.allocator.free(manifest_file_path);
222 mem.copy(u8, &manifest_file_path, &self.b64_digest);
223 manifest_file_path[self.b64_digest.len..][0..ext.len].* = ext.*;
189224
190225 if (self.files.items.len != 0) {
191 self.manifest_file = try self.manifest_dir.createFile(manifest_file_path, .{
226 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
192227 .read = true,
193228 .truncate = false,
194229 .lock = .Exclusive,
......@@ -196,26 +231,26 @@ pub const CacheHash = struct {
196231 } else {
197232 // If there are no file inputs, we check if the manifest file exists instead of
198233 // comparing the hashes on the files used for the cached item
199 self.manifest_file = self.manifest_dir.openFile(manifest_file_path, .{
234 self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{
200235 .read = true,
201236 .write = true,
202237 .lock = .Exclusive,
203238 }) catch |err| switch (err) {
204239 error.FileNotFound => {
205240 self.manifest_dirty = true;
206 self.manifest_file = try self.manifest_dir.createFile(manifest_file_path, .{
241 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
207242 .read = true,
208243 .truncate = false,
209244 .lock = .Exclusive,
210245 });
211 return null;
246 return false;
212247 },
213248 else => |e| return e,
214249 };
215250 }
216251
217 const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.allocator, MANIFEST_FILE_SIZE_MAX);
218 defer self.allocator.free(file_contents);
252 const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.cache.gpa, MANIFEST_FILE_SIZE_MAX);
253 defer self.cache.gpa.free(file_contents);
219254
220255 const input_file_count = self.files.items.len;
221256 var any_file_changed = false;
......@@ -225,7 +260,7 @@ pub const CacheHash = struct {
225260 defer idx += 1;
226261
227262 const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
228 const new = try self.files.addOne();
263 const new = try self.files.addOne(self.cache.gpa);
229264 new.* = .{
230265 .path = null,
231266 .contents = null,
......@@ -258,7 +293,7 @@ pub const CacheHash = struct {
258293 }
259294
260295 if (cache_hash_file.path == null) {
261 cache_hash_file.path = try self.allocator.dupe(u8, file_path);
296 cache_hash_file.path = try self.cache.gpa.dupe(u8, file_path);
262297 }
263298
264299 const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch {
......@@ -292,7 +327,7 @@ pub const CacheHash = struct {
292327 }
293328
294329 if (!any_file_changed) {
295 self.hasher.update(&cache_hash_file.bin_digest);
330 self.hash.hasher.update(&cache_hash_file.bin_digest);
296331 }
297332 }
298333
......@@ -300,19 +335,19 @@ pub const CacheHash = struct {
300335 // cache miss
301336 // keep the manifest file open
302337 // reset the hash
303 self.hasher = hasher_init;
304 self.hasher.update(&bin_digest);
338 self.hash.hasher = hasher_init;
339 self.hash.hasher.update(&bin_digest);
305340
306341 // Remove files not in the initial hash
307342 for (self.files.items[input_file_count..]) |*file| {
308 file.deinit(self.allocator);
343 file.deinit(self.cache.gpa);
309344 }
310 self.files.shrink(input_file_count);
345 self.files.shrinkRetainingCapacity(input_file_count);
311346
312347 for (self.files.items) |file| {
313 self.hasher.update(&file.bin_digest);
348 self.hash.hasher.update(&file.bin_digest);
314349 }
315 return null;
350 return false;
316351 }
317352
318353 if (idx < input_file_count) {
......@@ -321,10 +356,10 @@ pub const CacheHash = struct {
321356 const ch_file = &self.files.items[idx];
322357 try self.populateFileHash(ch_file);
323358 }
324 return null;
359 return false;
325360 }
326361
327 return self.final();
362 return true;
328363 }
329364
330365 fn populateFileHash(self: *CacheHash, ch_file: *File) !void {
......@@ -343,8 +378,8 @@ pub const CacheHash = struct {
343378 return error.FileTooBig;
344379 }
345380
346 const contents = try self.allocator.alloc(u8, @intCast(usize, ch_file.stat.size));
347 errdefer self.allocator.free(contents);
381 const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size));
382 errdefer self.cache.gpa.free(contents);
348383
349384 // Hash while reading from disk, to keep the contents in the cpu cache while
350385 // doing hashing.
......@@ -364,7 +399,7 @@ pub const CacheHash = struct {
364399 try hashFile(file, &ch_file.bin_digest);
365400 }
366401
367 self.hasher.update(&ch_file.bin_digest);
402 self.hash.hasher.update(&ch_file.bin_digest);
368403 }
369404
370405 /// Add a file as a dependency of process being cached, after the initial hash has been
......@@ -374,10 +409,10 @@ pub const CacheHash = struct {
374409 pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 {
375410 assert(self.manifest_file != null);
376411
377 const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
378 errdefer self.allocator.free(resolved_path);
412 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
413 errdefer self.cache.gpa.free(resolved_path);
379414
380 const new_ch_file = try self.files.addOne();
415 const new_ch_file = try self.files.addOne(self.cache.gpa);
381416 new_ch_file.* = .{
382417 .path = resolved_path,
383418 .max_file_size = max_file_size,
......@@ -385,7 +420,7 @@ pub const CacheHash = struct {
385420 .bin_digest = undefined,
386421 .contents = null,
387422 };
388 errdefer self.files.shrink(self.files.items.len - 1);
423 errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
389424
390425 try self.populateFileHash(new_ch_file);
391426
......@@ -399,10 +434,10 @@ pub const CacheHash = struct {
399434 pub fn addFilePost(self: *CacheHash, file_path: []const u8) !void {
400435 assert(self.manifest_file != null);
401436
402 const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
403 errdefer self.allocator.free(resolved_path);
437 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
438 errdefer self.cache.gpa.free(resolved_path);
404439
405 const new_ch_file = try self.files.addOne();
440 const new_ch_file = try self.files.addOne(self.cache.gpa);
406441 new_ch_file.* = .{
407442 .path = resolved_path,
408443 .max_file_size = null,
......@@ -410,7 +445,7 @@ pub const CacheHash = struct {
410445 .bin_digest = undefined,
411446 .contents = null,
412447 };
413 errdefer self.files.shrink(self.files.items.len - 1);
448 errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
414449
415450 try self.populateFileHash(new_ch_file);
416451 }
......@@ -426,7 +461,7 @@ pub const CacheHash = struct {
426461 // the artifacts to cache.
427462
428463 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
429 self.hasher.final(&bin_digest);
464 self.hash.hasher.final(&bin_digest);
430465
431466 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;
432467 base64_encoder.encode(&out_digest, &bin_digest);
......@@ -436,45 +471,48 @@ pub const CacheHash = struct {
436471
437472 pub fn writeManifest(self: *CacheHash) !void {
438473 assert(self.manifest_file != null);
474 if (!self.manifest_dirty) return;
439475
440476 var encoded_digest: [BASE64_DIGEST_LEN]u8 = undefined;
441 var contents = ArrayList(u8).init(self.allocator);
442 var outStream = contents.outStream();
477 var contents = std.ArrayList(u8).init(self.cache.gpa);
478 var writer = contents.writer();
443479 defer contents.deinit();
444480
445481 for (self.files.items) |file| {
446482 base64_encoder.encode(encoded_digest[0..], &file.bin_digest);
447 try outStream.print("{} {} {} {} {}\n", .{ file.stat.size, file.stat.inode, file.stat.mtime, encoded_digest[0..], file.path });
483 try writer.print("{} {} {} {} {}\n", .{
484 file.stat.size,
485 file.stat.inode,
486 file.stat.mtime,
487 encoded_digest[0..],
488 file.path,
489 });
448490 }
449491
450492 try self.manifest_file.?.pwriteAll(contents.items, 0);
451493 self.manifest_dirty = false;
452494 }
453495
496 /// Obtain only the data needed to maintain a lock on the manifest file.
497 /// The `CacheHash` remains safe to deinit.
498 /// Don't forget to call `writeManifest` before this!
499 pub fn toOwnedLock(self: *CacheHash) Lock {
500 const manifest_file = self.manifest_file.?;
501 self.manifest_file = null;
502 return Lock{ .manifest_file = manifest_file };
503 }
504
454505 /// Releases the manifest file and frees any memory the CacheHash was using.
455506 /// `CacheHash.hit` must be called first.
456 ///
457 /// Will also attempt to write to the manifest file if the manifest is dirty.
458 /// Writing to the manifest file can fail, but this function ignores those errors.
459 /// To detect failures from writing the manifest, one may explicitly call
460 /// `writeManifest` before `release`.
461 pub fn release(self: *CacheHash) void {
507 /// Don't forget to call `writeManifest` before this!
508 pub fn deinit(self: *CacheHash) void {
462509 if (self.manifest_file) |file| {
463 if (self.manifest_dirty) {
464 // To handle these errors, API users should call
465 // writeManifest before release().
466 self.writeManifest() catch {};
467 }
468
469510 file.close();
470511 }
471
472512 for (self.files.items) |*file| {
473 file.deinit(self.allocator);
513 file.deinit(self.cache.gpa);
474514 }
475 self.files.deinit();
476 if (self.owns_manifest_dir)
477 self.manifest_dir.close();
515 self.files.deinit(self.cache.gpa);
478516 }
479517};
480518
......@@ -542,31 +580,41 @@ test "cache file and then recall it" {
542580 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
543581 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
544582
583 var cache = Cache{
584 .gpa = testing.allocator,
585 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
586 };
587 defer cache.manifest_dir.close();
588
545589 {
546 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
547 defer ch.release();
590 var ch = cache.obtain();
591 defer ch.deinit();
548592
549 ch.add(true);
550 ch.add(@as(u16, 1234));
551 ch.addBytes("1234");
593 ch.hash.add(true);
594 ch.hash.add(@as(u16, 1234));
595 ch.hash.addBytes("1234");
552596 _ = try ch.addFile(temp_file, null);
553597
554598 // There should be nothing in the cache
555 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());
599 testing.expectEqual(false, try ch.hit());
556600
557601 digest1 = ch.final();
602 try ch.writeManifest();
558603 }
559604 {
560 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
561 defer ch.release();
605 var ch = cache.obtain();
606 defer ch.deinit();
562607
563 ch.add(true);
564 ch.add(@as(u16, 1234));
565 ch.addBytes("1234");
608 ch.hash.add(true);
609 ch.hash.add(@as(u16, 1234));
610 ch.hash.addBytes("1234");
566611 _ = try ch.addFile(temp_file, null);
567612
568613 // Cache hit! We just "built" the same file
569 digest2 = (try ch.hit()).?;
614 testing.expect(try ch.hit());
615 digest2 = ch.final();
616
617 try ch.writeManifest();
570618 }
571619
572620 testing.expectEqual(digest1, digest2);
......@@ -612,37 +660,47 @@ test "check that changing a file makes cache fail" {
612660 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
613661 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
614662
663 var cache = Cache{
664 .gpa = testing.allocator,
665 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
666 };
667 defer cache.manifest_dir.close();
668
615669 {
616 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
617 defer ch.release();
670 var ch = cache.obtain();
671 defer ch.deinit();
618672
619 ch.addBytes("1234");
673 ch.hash.addBytes("1234");
620674 const temp_file_idx = try ch.addFile(temp_file, 100);
621675
622676 // There should be nothing in the cache
623 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());
677 testing.expectEqual(false, try ch.hit());
624678
625679 testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
626680
627681 digest1 = ch.final();
682
683 try ch.writeManifest();
628684 }
629685
630686 try cwd.writeFile(temp_file, updated_temp_file_contents);
631687
632688 {
633 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
634 defer ch.release();
689 var ch = cache.obtain();
690 defer ch.deinit();
635691
636 ch.addBytes("1234");
692 ch.hash.addBytes("1234");
637693 const temp_file_idx = try ch.addFile(temp_file, 100);
638694
639695 // A file that we depend on has been updated, so the cache should not contain an entry for it
640 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());
696 testing.expectEqual(false, try ch.hit());
641697
642698 // The cache system does not keep the contents of re-hashed input files.
643699 testing.expect(ch.files.items[temp_file_idx].contents == null);
644700
645701 digest2 = ch.final();
702
703 try ch.writeManifest();
646704 }
647705
648706 testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
......@@ -663,24 +721,34 @@ test "no file inputs" {
663721 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
664722 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
665723
724 var cache = Cache{
725 .gpa = testing.allocator,
726 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
727 };
728 defer cache.manifest_dir.close();
729
666730 {
667 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
668 defer ch.release();
731 var ch = cache.obtain();
732 defer ch.deinit();
669733
670 ch.addBytes("1234");
734 ch.hash.addBytes("1234");
671735
672736 // There should be nothing in the cache
673 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());
737 testing.expectEqual(false, try ch.hit());
674738
675739 digest1 = ch.final();
740
741 try ch.writeManifest();
676742 }
677743 {
678 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
679 defer ch.release();
744 var ch = cache.obtain();
745 defer ch.deinit();
680746
681 ch.addBytes("1234");
747 ch.hash.addBytes("1234");
682748
683 digest2 = (try ch.hit()).?;
749 testing.expect(try ch.hit());
750 digest2 = ch.final();
751 try ch.writeManifest();
684752 }
685753
686754 testing.expectEqual(digest1, digest2);
......@@ -709,28 +777,38 @@ test "CacheHashes with files added after initial hash work" {
709777 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
710778 var digest3: [BASE64_DIGEST_LEN]u8 = undefined;
711779
780 var cache = Cache{
781 .gpa = testing.allocator,
782 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
783 };
784 defer cache.manifest_dir.close();
785
712786 {
713 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
714 defer ch.release();
787 var ch = cache.obtain();
788 defer ch.deinit();
715789
716 ch.addBytes("1234");
790 ch.hash.addBytes("1234");
717791 _ = try ch.addFile(temp_file1, null);
718792
719793 // There should be nothing in the cache
720 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());
794 testing.expectEqual(false, try ch.hit());
721795
722796 _ = try ch.addFilePost(temp_file2);
723797
724798 digest1 = ch.final();
799 try ch.writeManifest();
725800 }
726801 {
727 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
728 defer ch.release();
802 var ch = cache.obtain();
803 defer ch.deinit();
729804
730 ch.addBytes("1234");
805 ch.hash.addBytes("1234");
731806 _ = try ch.addFile(temp_file1, null);
732807
733 digest2 = (try ch.hit()).?;
808 testing.expect(try ch.hit());
809 digest2 = ch.final();
810
811 try ch.writeManifest();
734812 }
735813 testing.expect(mem.eql(u8, &digest1, &digest2));
736814
......@@ -743,18 +821,20 @@ test "CacheHashes with files added after initial hash work" {
743821 }
744822
745823 {
746 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
747 defer ch.release();
824 var ch = cache.obtain();
825 defer ch.deinit();
748826
749 ch.addBytes("1234");
827 ch.hash.addBytes("1234");
750828 _ = try ch.addFile(temp_file1, null);
751829
752830 // A file that we depend on has been updated, so the cache should not contain an entry for it
753 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());
831 testing.expectEqual(false, try ch.hit());
754832
755833 _ = try ch.addFilePost(temp_file2);
756834
757835 digest3 = ch.final();
836
837 try ch.writeManifest();
758838 }
759839
760840 testing.expect(!mem.eql(u8, &digest1, &digest3));