authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-13 18:04:17-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-13 18:04:52-07:00
log1baa56a25f7b4cf7b3e0a78ded90c432a40c4efa
tree631b636a5ec792aa063e3092f20be06d2c7180fd
parentaf4cc20ce275aeb0e59eee3d893b2f310c1f0239

std.cache_hash: break up the API and improve implementation

into smaller exposed components and expose all of them. This makes it more flexible. `*const Cache` is now passed in with an open manifest dir handle which the caller is responsible for managing. Expose some of the base64 stuff. Extract the hash helper functions into `HashHelper` and add some more methods such as addOptional and addListOfFiles. Add `CacheHash.toOwnedLock` so that you can deinitialize everything except the open file handle which represents the file system lock on the build artifacts. Use ArrayListUnmanaged, saving space per allocated CacheHash. Avoid 1 memory allocation in hit() with a static buffer. hit() returns a bool; caller code is responsible for calling final() in either case. This is a simpler and easier to use API. writeManifest() is no longer called from deinit() with errors ignored.

1 files changed, 240 insertions(+), 160 deletions(-)

lib/std/cache_hash.zig+240-160
...@@ -7,19 +7,18 @@ const std = @import("std.zig");...@@ -7,19 +7,18 @@ const std = @import("std.zig");
7const crypto = std.crypto;7const crypto = std.crypto;
8const fs = std.fs;8const fs = std.fs;
9const base64 = std.base64;9const base64 = std.base64;
10const ArrayList = std.ArrayList;
11const assert = std.debug.assert;10const assert = std.debug.assert;
12const testing = std.testing;11const testing = std.testing;
13const mem = std.mem;12const mem = std.mem;
14const fmt = std.fmt;13const fmt = std.fmt;
15const Allocator = std.mem.Allocator;14const Allocator = std.mem.Allocator;
1615
17const base64_encoder = fs.base64_encoder;16pub const base64_encoder = fs.base64_encoder;
18const base64_decoder = fs.base64_decoder;17pub const base64_decoder = fs.base64_decoder;
19/// 16 would be 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-618/// 16 would be 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
20/// We round up to 18 to avoid the `==` padding after base64 encoding.19/// We round up to 18 to avoid the `==` padding after base64 encoding.
21const BIN_DIGEST_LEN = 18;20pub const BIN_DIGEST_LEN = 18;
22const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);21pub const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
2322
24const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;23const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;
2524
...@@ -51,87 +50,105 @@ pub const File = struct {...@@ -51,87 +50,105 @@ pub const File = struct {
51 }50 }
52};51};
5352
54/// CacheHash manages project-local `zig-cache` directories.53pub const Cache = struct {
55/// This is not a general-purpose cache.54 gpa: *Allocator,
56/// It is designed to be fast and simple, not to withstand attacks using specially-crafted input.
57pub const CacheHash = struct {
58 allocator: *Allocator,
59 /// Current state for incremental hashing.
60 hasher: Hasher,
61 manifest_dir: fs.Dir,55 manifest_dir: fs.Dir,
62 manifest_file: ?fs.File,56 hash: HashHelper = .{},
63 manifest_dirty: bool,
64 owns_manifest_dir: bool,
65 files: ArrayList(File),
66 b64_digest: [BASE64_DIGEST_LEN]u8,
6757
68 /// Be sure to call release after successful initialization.58 /// Be sure to call `CacheHash.deinit` after successful initialization.
69 pub fn init(allocator: *Allocator, dir: fs.Dir, manifest_dir_path: []const u8) !CacheHash {59 pub fn obtain(cache: *const Cache) CacheHash {
70 return CacheHash{60 return CacheHash{
71 .allocator = allocator,61 .cache = cache,
72 .hasher = hasher_init,62 .hash = cache.hash,
73 .manifest_dir = try dir.makeOpenPath(manifest_dir_path, .{}),
74 .manifest_file = null,63 .manifest_file = null,
75 .manifest_dirty = false,64 .manifest_dirty = false,
76 .owns_manifest_dir = true,
77 .files = ArrayList(File).init(allocator),
78 .b64_digest = undefined,65 .b64_digest = undefined,
79 };66 };
80 }67 }
68};
8169
82 /// Allows one to fork a CacheHash instance into another one, which does not require an additional70pub const HashHelper = struct {
83 /// directory handle to be opened. The new instance inherits the hash state.71 hasher: Hasher = hasher_init,
84 pub fn clone(self: CacheHash) CacheHash {
85 assert(self.manifest_file == null);
86 assert(files.items.len == 0);
87 return .{
88 .allocator = self.allocator,
89 .hasher = self.hasher,
90 .manifest_dir = self.manifest_dir,
91 .manifest_file = null,
92 .manifest_dirty = false,
93 .owns_manifest_dir = false,
94 .files = ArrayList(File).init(allocator),
95 .b64_digest = undefined,
96 };
97 }
9872
99 /// Record a slice of bytes as an dependency of the process being cached73 /// Record a slice of bytes as an dependency of the process being cached
100 pub fn addBytes(self: *CacheHash, bytes: []const u8) void {74 pub fn addBytes(hh: *HashHelper, bytes: []const u8) void {
101 assert(self.manifest_file == null);75 hh.hasher.update(mem.asBytes(&bytes.len));
10276 hh.hasher.update(bytes);
103 self.hasher.update(mem.asBytes(&bytes.len));
104 self.hasher.update(bytes);
105 }77 }
10678
107 pub fn addListOfBytes(self: *CacheHash, list_of_bytes: []const []const u8) void {79 pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void {
108 assert(self.manifest_file == null);80 hh.add(optional_bytes != null);
81 hh.addBytes(optional_bytes orelse return);
82 }
10983
110 self.add(list_of_bytes.items.len);84 pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void {
111 for (list_of_bytes) |bytes| self.addBytes(bytes);85 hh.add(list_of_bytes.items.len);
86 for (list_of_bytes) |bytes| hh.addBytes(bytes);
112 }87 }
11388
114 /// Convert the input value into bytes and record it as a dependency of the process being cached.89 /// Convert the input value into bytes and record it as a dependency of the process being cached.
115 pub fn add(self: *CacheHash, x: anytype) void {90 pub fn add(hh: *HashHelper, x: anytype) void {
116 assert(self.manifest_file == null);
117
118 switch (@TypeOf(x)) {91 switch (@TypeOf(x)) {
119 std.builtin.Version => {92 std.builtin.Version => {
120 self.add(x.major);93 hh.add(x.major);
121 self.add(x.minor);94 hh.add(x.minor);
122 self.add(x.patch);95 hh.add(x.patch);
123 return;96 return;
124 },97 },
125 else => {},98 else => {},
126 }99 }
127100
128 switch (@typeInfo(@TypeOf(x))) {101 switch (@typeInfo(@TypeOf(x))) {
129 .Bool, .Int, .Enum, .Array => self.addBytes(mem.asBytes(&x)),102 .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)),
130 else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),103 else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),
131 }104 }
132 }105 }
133106
134 /// Add a file as a dependency of process being cached. When `CacheHash.hit` is107 pub fn addOptional(hh: *HashHelper, optional: anytype) void {
108 hh.add(optional != null);
109 hh.add(optional orelse return);
110 }
111
112 /// Returns a base64 encoded hash of the inputs, without modifying state.
113 pub fn peek(hh: HashHelper) [BASE64_DIGEST_LEN]u8 {
114 var copy = hh;
115 return copy.final();
116 }
117
118 /// Returns a base64 encoded hash of the inputs, mutating the state of the hasher.
119 pub fn final(hh: *HashHelper) [BASE64_DIGEST_LEN]u8 {
120 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
121 hh.hasher.final(&bin_digest);
122
123 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;
124 base64_encoder.encode(&out_digest, &bin_digest);
125
126 return out_digest;
127 }
128};
129
130pub const Lock = struct {
131 manifest_file: fs.File,
132
133 pub fn release(lock: *Lock) void {
134 lock.manifest_file.close();
135 lock.* = undefined;
136 }
137};
138
139/// CacheHash manages project-local `zig-cache` directories.
140/// This is not a general-purpose cache.
141/// It is designed to be fast and simple, not to withstand attacks using specially-crafted input.
142pub const CacheHash = struct {
143 cache: *const Cache,
144 /// Current state for incremental hashing.
145 hash: HashHelper,
146 manifest_file: ?fs.File,
147 manifest_dirty: bool,
148 files: std.ArrayListUnmanaged(File) = .{},
149 b64_digest: [BASE64_DIGEST_LEN]u8,
150
151 /// Add a file as a dependency of process being cached. When `hit` is
135 /// called, the file's contents will be checked to ensure that it matches152 /// called, the file's contents will be checked to ensure that it matches
136 /// the contents from previous times.153 /// the contents from previous times.
137 ///154 ///
...@@ -139,8 +156,8 @@ pub const CacheHash = struct {...@@ -139,8 +156,8 @@ pub const CacheHash = struct {
139 /// are allowed to take up in memory. If max_file_size is null, then the contents156 /// are allowed to take up in memory. If max_file_size is null, then the contents
140 /// will not be loaded into memory.157 /// will not be loaded into memory.
141 ///158 ///
142 /// Returns the index of the entry in the `CacheHash.files` ArrayList. You can use it159 /// Returns the index of the entry in the `files` array list. You can use it
143 /// to access the contents of the file after calling `CacheHash.hit()` like so:160 /// to access the contents of the file after calling `hit()` like so:
144 ///161 ///
145 /// ```162 /// ```
146 /// var file_contents = cache_hash.files.items[file_index].contents.?;163 /// var file_contents = cache_hash.files.items[file_index].contents.?;
...@@ -148,8 +165,8 @@ pub const CacheHash = struct {...@@ -148,8 +165,8 @@ pub const CacheHash = struct {
148 pub fn addFile(self: *CacheHash, file_path: []const u8, max_file_size: ?usize) !usize {165 pub fn addFile(self: *CacheHash, file_path: []const u8, max_file_size: ?usize) !usize {
149 assert(self.manifest_file == null);166 assert(self.manifest_file == null);
150167
151 try self.files.ensureCapacity(self.files.items.len + 1);168 try self.files.ensureCapacity(self.cache.gpa, self.files.items.len + 1);
152 const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});169 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
153170
154 const idx = self.files.items.len;171 const idx = self.files.items.len;
155 self.files.addOneAssumeCapacity().* = .{172 self.files.addOneAssumeCapacity().* = .{
...@@ -160,35 +177,53 @@ pub const CacheHash = struct {...@@ -160,35 +177,53 @@ pub const CacheHash = struct {
160 .bin_digest = undefined,177 .bin_digest = undefined,
161 };178 };
162179
163 self.addBytes(resolved_path);180 self.hash.addBytes(resolved_path);
164181
165 return idx;182 return idx;
166 }183 }
167184
168 /// Check the cache to see if the input exists in it. If it exists, a base64 encoding185 pub fn addOptionalFile(self: *CacheHash, optional_file_path: ?[]const u8) !void {
169 /// of it's hash will be returned; otherwise, null will be returned.186 self.hash.add(optional_file_path != null);
187 const file_path = optional_file_path orelse return;
188 _ = try self.addFile(file_path, null);
189 }
190
191 pub fn addListOfFiles(self: *CacheHash, list_of_files: []const []const u8) !void {
192 self.hash.add(list_of_files.len);
193 for (list_of_files) |file_path| {
194 _ = try self.addFile(file_path, null);
195 }
196 }
197
198 /// Check the cache to see if the input exists in it. If it exists, returns `true`.
199 /// A base64 encoding of its hash is available by calling `final`.
170 ///200 ///
171 /// This function will also acquire an exclusive lock to the manifest file. This means201 /// This function will also acquire an exclusive lock to the manifest file. This means
172 /// that a process holding a CacheHash will block any other process attempting to202 /// that a process holding a CacheHash will block any other process attempting to
173 /// acquire the lock.203 /// acquire the lock.
174 ///204 ///
175 /// The lock on the manifest file is released when `CacheHash.release` is called.205 /// The lock on the manifest file is released when `deinit` is called. As another
176 pub fn hit(self: *CacheHash) !?[BASE64_DIGEST_LEN]u8 {206 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
207 /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
208 pub fn hit(self: *CacheHash) !bool {
177 assert(self.manifest_file == null);209 assert(self.manifest_file == null);
178210
211 const ext = ".txt";
212 var manifest_file_path: [self.b64_digest.len + ext.len]u8 = undefined;
213
179 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;214 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
180 self.hasher.final(&bin_digest);215 self.hash.hasher.final(&bin_digest);
181216
182 base64_encoder.encode(self.b64_digest[0..], &bin_digest);217 base64_encoder.encode(self.b64_digest[0..], &bin_digest);
183218
184 self.hasher = hasher_init;219 self.hash.hasher = hasher_init;
185 self.hasher.update(&bin_digest);220 self.hash.hasher.update(&bin_digest);
186221
187 const manifest_file_path = try fmt.allocPrint(self.allocator, "{}.txt", .{self.b64_digest});222 mem.copy(u8, &manifest_file_path, &self.b64_digest);
188 defer self.allocator.free(manifest_file_path);223 manifest_file_path[self.b64_digest.len..][0..ext.len].* = ext.*;
189224
190 if (self.files.items.len != 0) {225 if (self.files.items.len != 0) {
191 self.manifest_file = try self.manifest_dir.createFile(manifest_file_path, .{226 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
192 .read = true,227 .read = true,
193 .truncate = false,228 .truncate = false,
194 .lock = .Exclusive,229 .lock = .Exclusive,
...@@ -196,26 +231,26 @@ pub const CacheHash = struct {...@@ -196,26 +231,26 @@ pub const CacheHash = struct {
196 } else {231 } else {
197 // If there are no file inputs, we check if the manifest file exists instead of232 // If there are no file inputs, we check if the manifest file exists instead of
198 // comparing the hashes on the files used for the cached item233 // comparing the hashes on the files used for the cached item
199 self.manifest_file = self.manifest_dir.openFile(manifest_file_path, .{234 self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{
200 .read = true,235 .read = true,
201 .write = true,236 .write = true,
202 .lock = .Exclusive,237 .lock = .Exclusive,
203 }) catch |err| switch (err) {238 }) catch |err| switch (err) {
204 error.FileNotFound => {239 error.FileNotFound => {
205 self.manifest_dirty = true;240 self.manifest_dirty = true;
206 self.manifest_file = try self.manifest_dir.createFile(manifest_file_path, .{241 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
207 .read = true,242 .read = true,
208 .truncate = false,243 .truncate = false,
209 .lock = .Exclusive,244 .lock = .Exclusive,
210 });245 });
211 return null;246 return false;
212 },247 },
213 else => |e| return e,248 else => |e| return e,
214 };249 };
215 }250 }
216251
217 const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.allocator, MANIFEST_FILE_SIZE_MAX);252 const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.cache.gpa, MANIFEST_FILE_SIZE_MAX);
218 defer self.allocator.free(file_contents);253 defer self.cache.gpa.free(file_contents);
219254
220 const input_file_count = self.files.items.len;255 const input_file_count = self.files.items.len;
221 var any_file_changed = false;256 var any_file_changed = false;
...@@ -225,7 +260,7 @@ pub const CacheHash = struct {...@@ -225,7 +260,7 @@ pub const CacheHash = struct {
225 defer idx += 1;260 defer idx += 1;
226261
227 const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {262 const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
228 const new = try self.files.addOne();263 const new = try self.files.addOne(self.cache.gpa);
229 new.* = .{264 new.* = .{
230 .path = null,265 .path = null,
231 .contents = null,266 .contents = null,
...@@ -258,7 +293,7 @@ pub const CacheHash = struct {...@@ -258,7 +293,7 @@ pub const CacheHash = struct {
258 }293 }
259294
260 if (cache_hash_file.path == null) {295 if (cache_hash_file.path == null) {
261 cache_hash_file.path = try self.allocator.dupe(u8, file_path);296 cache_hash_file.path = try self.cache.gpa.dupe(u8, file_path);
262 }297 }
263298
264 const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch {299 const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch {
...@@ -292,7 +327,7 @@ pub const CacheHash = struct {...@@ -292,7 +327,7 @@ pub const CacheHash = struct {
292 }327 }
293328
294 if (!any_file_changed) {329 if (!any_file_changed) {
295 self.hasher.update(&cache_hash_file.bin_digest);330 self.hash.hasher.update(&cache_hash_file.bin_digest);
296 }331 }
297 }332 }
298333
...@@ -300,19 +335,19 @@ pub const CacheHash = struct {...@@ -300,19 +335,19 @@ pub const CacheHash = struct {
300 // cache miss335 // cache miss
301 // keep the manifest file open336 // keep the manifest file open
302 // reset the hash337 // reset the hash
303 self.hasher = hasher_init;338 self.hash.hasher = hasher_init;
304 self.hasher.update(&bin_digest);339 self.hash.hasher.update(&bin_digest);
305340
306 // Remove files not in the initial hash341 // Remove files not in the initial hash
307 for (self.files.items[input_file_count..]) |*file| {342 for (self.files.items[input_file_count..]) |*file| {
308 file.deinit(self.allocator);343 file.deinit(self.cache.gpa);
309 }344 }
310 self.files.shrink(input_file_count);345 self.files.shrinkRetainingCapacity(input_file_count);
311346
312 for (self.files.items) |file| {347 for (self.files.items) |file| {
313 self.hasher.update(&file.bin_digest);348 self.hash.hasher.update(&file.bin_digest);
314 }349 }
315 return null;350 return false;
316 }351 }
317352
318 if (idx < input_file_count) {353 if (idx < input_file_count) {
...@@ -321,10 +356,10 @@ pub const CacheHash = struct {...@@ -321,10 +356,10 @@ pub const CacheHash = struct {
321 const ch_file = &self.files.items[idx];356 const ch_file = &self.files.items[idx];
322 try self.populateFileHash(ch_file);357 try self.populateFileHash(ch_file);
323 }358 }
324 return null;359 return false;
325 }360 }
326361
327 return self.final();362 return true;
328 }363 }
329364
330 fn populateFileHash(self: *CacheHash, ch_file: *File) !void {365 fn populateFileHash(self: *CacheHash, ch_file: *File) !void {
...@@ -343,8 +378,8 @@ pub const CacheHash = struct {...@@ -343,8 +378,8 @@ pub const CacheHash = struct {
343 return error.FileTooBig;378 return error.FileTooBig;
344 }379 }
345380
346 const contents = try self.allocator.alloc(u8, @intCast(usize, ch_file.stat.size));381 const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size));
347 errdefer self.allocator.free(contents);382 errdefer self.cache.gpa.free(contents);
348383
349 // Hash while reading from disk, to keep the contents in the cpu cache while384 // Hash while reading from disk, to keep the contents in the cpu cache while
350 // doing hashing.385 // doing hashing.
...@@ -364,7 +399,7 @@ pub const CacheHash = struct {...@@ -364,7 +399,7 @@ pub const CacheHash = struct {
364 try hashFile(file, &ch_file.bin_digest);399 try hashFile(file, &ch_file.bin_digest);
365 }400 }
366401
367 self.hasher.update(&ch_file.bin_digest);402 self.hash.hasher.update(&ch_file.bin_digest);
368 }403 }
369404
370 /// Add a file as a dependency of process being cached, after the initial hash has been405 /// Add a file as a dependency of process being cached, after the initial hash has been
...@@ -374,10 +409,10 @@ pub const CacheHash = struct {...@@ -374,10 +409,10 @@ pub const CacheHash = struct {
374 pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 {409 pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 {
375 assert(self.manifest_file != null);410 assert(self.manifest_file != null);
376411
377 const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});412 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
378 errdefer self.allocator.free(resolved_path);413 errdefer self.cache.gpa.free(resolved_path);
379414
380 const new_ch_file = try self.files.addOne();415 const new_ch_file = try self.files.addOne(self.cache.gpa);
381 new_ch_file.* = .{416 new_ch_file.* = .{
382 .path = resolved_path,417 .path = resolved_path,
383 .max_file_size = max_file_size,418 .max_file_size = max_file_size,
...@@ -385,7 +420,7 @@ pub const CacheHash = struct {...@@ -385,7 +420,7 @@ pub const CacheHash = struct {
385 .bin_digest = undefined,420 .bin_digest = undefined,
386 .contents = null,421 .contents = null,
387 };422 };
388 errdefer self.files.shrink(self.files.items.len - 1);423 errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
389424
390 try self.populateFileHash(new_ch_file);425 try self.populateFileHash(new_ch_file);
391426
...@@ -399,10 +434,10 @@ pub const CacheHash = struct {...@@ -399,10 +434,10 @@ pub const CacheHash = struct {
399 pub fn addFilePost(self: *CacheHash, file_path: []const u8) !void {434 pub fn addFilePost(self: *CacheHash, file_path: []const u8) !void {
400 assert(self.manifest_file != null);435 assert(self.manifest_file != null);
401436
402 const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});437 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
403 errdefer self.allocator.free(resolved_path);438 errdefer self.cache.gpa.free(resolved_path);
404439
405 const new_ch_file = try self.files.addOne();440 const new_ch_file = try self.files.addOne(self.cache.gpa);
406 new_ch_file.* = .{441 new_ch_file.* = .{
407 .path = resolved_path,442 .path = resolved_path,
408 .max_file_size = null,443 .max_file_size = null,
...@@ -410,7 +445,7 @@ pub const CacheHash = struct {...@@ -410,7 +445,7 @@ pub const CacheHash = struct {
410 .bin_digest = undefined,445 .bin_digest = undefined,
411 .contents = null,446 .contents = null,
412 };447 };
413 errdefer self.files.shrink(self.files.items.len - 1);448 errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
414449
415 try self.populateFileHash(new_ch_file);450 try self.populateFileHash(new_ch_file);
416 }451 }
...@@ -426,7 +461,7 @@ pub const CacheHash = struct {...@@ -426,7 +461,7 @@ pub const CacheHash = struct {
426 // the artifacts to cache.461 // the artifacts to cache.
427462
428 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;463 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
429 self.hasher.final(&bin_digest);464 self.hash.hasher.final(&bin_digest);
430465
431 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;466 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;
432 base64_encoder.encode(&out_digest, &bin_digest);467 base64_encoder.encode(&out_digest, &bin_digest);
...@@ -436,45 +471,48 @@ pub const CacheHash = struct {...@@ -436,45 +471,48 @@ pub const CacheHash = struct {
436471
437 pub fn writeManifest(self: *CacheHash) !void {472 pub fn writeManifest(self: *CacheHash) !void {
438 assert(self.manifest_file != null);473 assert(self.manifest_file != null);
474 if (!self.manifest_dirty) return;
439475
440 var encoded_digest: [BASE64_DIGEST_LEN]u8 = undefined;476 var encoded_digest: [BASE64_DIGEST_LEN]u8 = undefined;
441 var contents = ArrayList(u8).init(self.allocator);477 var contents = std.ArrayList(u8).init(self.cache.gpa);
442 var outStream = contents.outStream();478 var writer = contents.writer();
443 defer contents.deinit();479 defer contents.deinit();
444480
445 for (self.files.items) |file| {481 for (self.files.items) |file| {
446 base64_encoder.encode(encoded_digest[0..], &file.bin_digest);482 base64_encoder.encode(encoded_digest[0..], &file.bin_digest);
447 try outStream.print("{} {} {} {} {}\n", .{ file.stat.size, file.stat.inode, file.stat.mtime, encoded_digest[0..], file.path });483 try writer.print("{} {} {} {} {}\n", .{
484 file.stat.size,
485 file.stat.inode,
486 file.stat.mtime,
487 encoded_digest[0..],
488 file.path,
489 });
448 }490 }
449491
450 try self.manifest_file.?.pwriteAll(contents.items, 0);492 try self.manifest_file.?.pwriteAll(contents.items, 0);
451 self.manifest_dirty = false;493 self.manifest_dirty = false;
452 }494 }
453495
496 /// Obtain only the data needed to maintain a lock on the manifest file.
497 /// The `CacheHash` remains safe to deinit.
498 /// Don't forget to call `writeManifest` before this!
499 pub fn toOwnedLock(self: *CacheHash) Lock {
500 const manifest_file = self.manifest_file.?;
501 self.manifest_file = null;
502 return Lock{ .manifest_file = manifest_file };
503 }
504
454 /// Releases the manifest file and frees any memory the CacheHash was using.505 /// Releases the manifest file and frees any memory the CacheHash was using.
455 /// `CacheHash.hit` must be called first.506 /// `CacheHash.hit` must be called first.
456 ///507 /// Don't forget to call `writeManifest` before this!
457 /// Will also attempt to write to the manifest file if the manifest is dirty.508 pub fn deinit(self: *CacheHash) void {
458 /// Writing to the manifest file can fail, but this function ignores those errors.
459 /// To detect failures from writing the manifest, one may explicitly call
460 /// `writeManifest` before `release`.
461 pub fn release(self: *CacheHash) void {
462 if (self.manifest_file) |file| {509 if (self.manifest_file) |file| {
463 if (self.manifest_dirty) {
464 // To handle these errors, API users should call
465 // writeManifest before release().
466 self.writeManifest() catch {};
467 }
468
469 file.close();510 file.close();
470 }511 }
471
472 for (self.files.items) |*file| {512 for (self.files.items) |*file| {
473 file.deinit(self.allocator);513 file.deinit(self.cache.gpa);
474 }514 }
475 self.files.deinit();515 self.files.deinit(self.cache.gpa);
476 if (self.owns_manifest_dir)
477 self.manifest_dir.close();
478 }516 }
479};517};
480518
...@@ -542,31 +580,41 @@ test "cache file and then recall it" {...@@ -542,31 +580,41 @@ test "cache file and then recall it" {
542 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;580 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
543 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;581 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
544582
583 var cache = Cache{
584 .gpa = testing.allocator,
585 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
586 };
587 defer cache.manifest_dir.close();
588
545 {589 {
546 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);590 var ch = cache.obtain();
547 defer ch.release();591 defer ch.deinit();
548592
549 ch.add(true);593 ch.hash.add(true);
550 ch.add(@as(u16, 1234));594 ch.hash.add(@as(u16, 1234));
551 ch.addBytes("1234");595 ch.hash.addBytes("1234");
552 _ = try ch.addFile(temp_file, null);596 _ = try ch.addFile(temp_file, null);
553597
554 // There should be nothing in the cache598 // There should be nothing in the cache
555 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());599 testing.expectEqual(false, try ch.hit());
556600
557 digest1 = ch.final();601 digest1 = ch.final();
602 try ch.writeManifest();
558 }603 }
559 {604 {
560 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);605 var ch = cache.obtain();
561 defer ch.release();606 defer ch.deinit();
562607
563 ch.add(true);608 ch.hash.add(true);
564 ch.add(@as(u16, 1234));609 ch.hash.add(@as(u16, 1234));
565 ch.addBytes("1234");610 ch.hash.addBytes("1234");
566 _ = try ch.addFile(temp_file, null);611 _ = try ch.addFile(temp_file, null);
567612
568 // Cache hit! We just "built" the same file613 // Cache hit! We just "built" the same file
569 digest2 = (try ch.hit()).?;614 testing.expect(try ch.hit());
615 digest2 = ch.final();
616
617 try ch.writeManifest();
570 }618 }
571619
572 testing.expectEqual(digest1, digest2);620 testing.expectEqual(digest1, digest2);
...@@ -612,37 +660,47 @@ test "check that changing a file makes cache fail" {...@@ -612,37 +660,47 @@ test "check that changing a file makes cache fail" {
612 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;660 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
613 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;661 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
614662
663 var cache = Cache{
664 .gpa = testing.allocator,
665 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
666 };
667 defer cache.manifest_dir.close();
668
615 {669 {
616 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);670 var ch = cache.obtain();
617 defer ch.release();671 defer ch.deinit();
618672
619 ch.addBytes("1234");673 ch.hash.addBytes("1234");
620 const temp_file_idx = try ch.addFile(temp_file, 100);674 const temp_file_idx = try ch.addFile(temp_file, 100);
621675
622 // There should be nothing in the cache676 // There should be nothing in the cache
623 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());677 testing.expectEqual(false, try ch.hit());
624678
625 testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));679 testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
626680
627 digest1 = ch.final();681 digest1 = ch.final();
682
683 try ch.writeManifest();
628 }684 }
629685
630 try cwd.writeFile(temp_file, updated_temp_file_contents);686 try cwd.writeFile(temp_file, updated_temp_file_contents);
631687
632 {688 {
633 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);689 var ch = cache.obtain();
634 defer ch.release();690 defer ch.deinit();
635691
636 ch.addBytes("1234");692 ch.hash.addBytes("1234");
637 const temp_file_idx = try ch.addFile(temp_file, 100);693 const temp_file_idx = try ch.addFile(temp_file, 100);
638694
639 // A file that we depend on has been updated, so the cache should not contain an entry for it695 // A file that we depend on has been updated, so the cache should not contain an entry for it
640 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());696 testing.expectEqual(false, try ch.hit());
641697
642 // The cache system does not keep the contents of re-hashed input files.698 // The cache system does not keep the contents of re-hashed input files.
643 testing.expect(ch.files.items[temp_file_idx].contents == null);699 testing.expect(ch.files.items[temp_file_idx].contents == null);
644700
645 digest2 = ch.final();701 digest2 = ch.final();
702
703 try ch.writeManifest();
646 }704 }
647705
648 testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));706 testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
...@@ -663,24 +721,34 @@ test "no file inputs" {...@@ -663,24 +721,34 @@ test "no file inputs" {
663 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;721 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
664 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;722 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
665723
724 var cache = Cache{
725 .gpa = testing.allocator,
726 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
727 };
728 defer cache.manifest_dir.close();
729
666 {730 {
667 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);731 var ch = cache.obtain();
668 defer ch.release();732 defer ch.deinit();
669733
670 ch.addBytes("1234");734 ch.hash.addBytes("1234");
671735
672 // There should be nothing in the cache736 // There should be nothing in the cache
673 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());737 testing.expectEqual(false, try ch.hit());
674738
675 digest1 = ch.final();739 digest1 = ch.final();
740
741 try ch.writeManifest();
676 }742 }
677 {743 {
678 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);744 var ch = cache.obtain();
679 defer ch.release();745 defer ch.deinit();
680746
681 ch.addBytes("1234");747 ch.hash.addBytes("1234");
682748
683 digest2 = (try ch.hit()).?;749 testing.expect(try ch.hit());
750 digest2 = ch.final();
751 try ch.writeManifest();
684 }752 }
685753
686 testing.expectEqual(digest1, digest2);754 testing.expectEqual(digest1, digest2);
...@@ -709,28 +777,38 @@ test "CacheHashes with files added after initial hash work" {...@@ -709,28 +777,38 @@ test "CacheHashes with files added after initial hash work" {
709 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;777 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
710 var digest3: [BASE64_DIGEST_LEN]u8 = undefined;778 var digest3: [BASE64_DIGEST_LEN]u8 = undefined;
711779
780 var cache = Cache{
781 .gpa = testing.allocator,
782 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
783 };
784 defer cache.manifest_dir.close();
785
712 {786 {
713 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);787 var ch = cache.obtain();
714 defer ch.release();788 defer ch.deinit();
715789
716 ch.addBytes("1234");790 ch.hash.addBytes("1234");
717 _ = try ch.addFile(temp_file1, null);791 _ = try ch.addFile(temp_file1, null);
718792
719 // There should be nothing in the cache793 // There should be nothing in the cache
720 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());794 testing.expectEqual(false, try ch.hit());
721795
722 _ = try ch.addFilePost(temp_file2);796 _ = try ch.addFilePost(temp_file2);
723797
724 digest1 = ch.final();798 digest1 = ch.final();
799 try ch.writeManifest();
725 }800 }
726 {801 {
727 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);802 var ch = cache.obtain();
728 defer ch.release();803 defer ch.deinit();
729804
730 ch.addBytes("1234");805 ch.hash.addBytes("1234");
731 _ = try ch.addFile(temp_file1, null);806 _ = try ch.addFile(temp_file1, null);
732807
733 digest2 = (try ch.hit()).?;808 testing.expect(try ch.hit());
809 digest2 = ch.final();
810
811 try ch.writeManifest();
734 }812 }
735 testing.expect(mem.eql(u8, &digest1, &digest2));813 testing.expect(mem.eql(u8, &digest1, &digest2));
736814
...@@ -743,18 +821,20 @@ test "CacheHashes with files added after initial hash work" {...@@ -743,18 +821,20 @@ test "CacheHashes with files added after initial hash work" {
743 }821 }
744822
745 {823 {
746 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);824 var ch = cache.obtain();
747 defer ch.release();825 defer ch.deinit();
748826
749 ch.addBytes("1234");827 ch.hash.addBytes("1234");
750 _ = try ch.addFile(temp_file1, null);828 _ = try ch.addFile(temp_file1, null);
751829
752 // A file that we depend on has been updated, so the cache should not contain an entry for it830 // A file that we depend on has been updated, so the cache should not contain an entry for it
753 testing.expectEqual(@as(?[BASE64_DIGEST_LEN]u8, null), try ch.hit());831 testing.expectEqual(false, try ch.hit());
754832
755 _ = try ch.addFilePost(temp_file2);833 _ = try ch.addFilePost(temp_file2);
756834
757 digest3 = ch.final();835 digest3 = ch.final();
836
837 try ch.writeManifest();
758 }838 }
759839
760 testing.expect(!mem.eql(u8, &digest1, &digest3));840 testing.expect(!mem.eql(u8, &digest1, &digest3));