authorgravatar for leroycepearson@geemili.xyzLeRoyce Pearson <leroycepearson@geemili.xyz> 2020-05-01 23:06:10-06:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-25 13:48:43-04:00
logc3c332c9ecf8ab982ca1b704a54b7d1b439b520d
tree2957c21b54dde838e66910ae2ca914cc79240bc5
parent5a1c6a362743882ba2923570c24f77f555a39504

Add max_file_size argument

1 files changed, 50 insertions(+), 22 deletions(-)

lib/std/cache_hash.zig+50-22
...@@ -19,6 +19,7 @@ const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;...@@ -19,6 +19,7 @@ const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;
1919
20pub const File = struct {20pub const File = struct {
21 path: ?[]const u8,21 path: ?[]const u8,
22 max_file_size: ?usize,
22 stat: fs.File.Stat,23 stat: fs.File.Stat,
23 bin_digest: [BIN_DIGEST_LEN]u8,24 bin_digest: [BIN_DIGEST_LEN]u8,
24 contents: ?[]const u8 = null,25 contents: ?[]const u8 = null,
...@@ -85,18 +86,23 @@ pub const CacheHash = struct {...@@ -85,18 +86,23 @@ pub const CacheHash = struct {
85 /// called, the file's contents will be checked to ensure that it matches86 /// called, the file's contents will be checked to ensure that it matches
86 /// the contents from previous times.87 /// the contents from previous times.
87 ///88 ///
89 /// Max file size will be used to determine the amount of space to the file contents
90 /// are allowed to take up in memory. If max_file_size is null, then the contents
91 /// will not be loaded into memory.
92 ///
88 /// Returns the index of the entry in the `CacheHash.files` ArrayList. You can use it93 /// Returns the index of the entry in the `CacheHash.files` ArrayList. You can use it
89 /// to access the contents of the file after calling `CacheHash.hit()` like so:94 /// to access the contents of the file after calling `CacheHash.hit()` like so:
90 ///95 ///
91 /// ```96 /// ```
92 /// var file_contents = cache_hash.files.items[file_index].contents.?;97 /// var file_contents = cache_hash.files.items[file_index].contents.?;
93 /// ```98 /// ```
94 pub fn addFile(self: *@This(), file_path: []const u8) !usize {99 pub fn addFile(self: *@This(), file_path: []const u8, max_file_size: ?usize) !usize {
95 debug.assert(self.manifest_file == null);100 debug.assert(self.manifest_file == null);
96101
97 const idx = self.files.items.len;102 const idx = self.files.items.len;
98 var cache_hash_file = try self.files.addOne();103 var cache_hash_file = try self.files.addOne();
99 cache_hash_file.path = try fs.path.resolve(self.alloc, &[_][]const u8{file_path});104 cache_hash_file.path = try fs.path.resolve(self.alloc, &[_][]const u8{file_path});
105 cache_hash_file.max_file_size = max_file_size;
100106
101 self.addSlice(cache_hash_file.path.?);107 self.addSlice(cache_hash_file.path.?);
102108
...@@ -168,6 +174,7 @@ pub const CacheHash = struct {...@@ -168,6 +174,7 @@ pub const CacheHash = struct {
168 } else {174 } else {
169 cache_hash_file = try self.files.addOne();175 cache_hash_file = try self.files.addOne();
170 cache_hash_file.path = null;176 cache_hash_file.path = null;
177 cache_hash_file.max_file_size = null;
171 }178 }
172179
173 var iter = mem.tokenize(line, " ");180 var iter = mem.tokenize(line, " ");
...@@ -213,7 +220,7 @@ pub const CacheHash = struct {...@@ -213,7 +220,7 @@ pub const CacheHash = struct {
213 }220 }
214221
215 var actual_digest: [BIN_DIGEST_LEN]u8 = undefined;222 var actual_digest: [BIN_DIGEST_LEN]u8 = undefined;
216 cache_hash_file.contents = try hash_file(self.alloc, &actual_digest, &this_file);223 cache_hash_file.contents = try hash_file(self.alloc, &actual_digest, &this_file, cache_hash_file.max_file_size);
217224
218 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {225 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
219 mem.copy(u8, &cache_hash_file.bin_digest, &actual_digest);226 mem.copy(u8, &cache_hash_file.bin_digest, &actual_digest);
...@@ -260,7 +267,7 @@ pub const CacheHash = struct {...@@ -260,7 +267,7 @@ pub const CacheHash = struct {
260 return self.final();267 return self.final();
261 }268 }
262269
263 fn populate_file_hash_fetch(self: *@This(), otherAlloc: *mem.Allocator, cache_hash_file: *File) ![]u8 {270 fn populate_file_hash_fetch(self: *@This(), otherAlloc: *mem.Allocator, cache_hash_file: *File) !?[]u8 {
264 debug.assert(cache_hash_file.path != null);271 debug.assert(cache_hash_file.path != null);
265272
266 const this_file = try fs.cwd().openFile(cache_hash_file.path.?, .{});273 const this_file = try fs.cwd().openFile(cache_hash_file.path.?, .{});
...@@ -273,7 +280,7 @@ pub const CacheHash = struct {...@@ -273,7 +280,7 @@ pub const CacheHash = struct {
273 cache_hash_file.stat.inode = 0;280 cache_hash_file.stat.inode = 0;
274 }281 }
275282
276 const contents = try hash_file(otherAlloc, &cache_hash_file.bin_digest, &this_file);283 const contents = try hash_file(otherAlloc, &cache_hash_file.bin_digest, &this_file, cache_hash_file.max_file_size);
277 self.blake3.update(&cache_hash_file.bin_digest);284 self.blake3.update(&cache_hash_file.bin_digest);
278285
279 return contents;286 return contents;
...@@ -289,13 +296,15 @@ pub const CacheHash = struct {...@@ -289,13 +296,15 @@ pub const CacheHash = struct {
289 /// will need to be recompiled if the imported file is changed.296 /// will need to be recompiled if the imported file is changed.
290 ///297 ///
291 /// Returns the contents of the file, allocated with the given allocator.298 /// Returns the contents of the file, allocated with the given allocator.
292 pub fn addFilePostFetch(self: *@This(), otherAlloc: *mem.Allocator, file_path: []const u8) ![]u8 {299 pub fn addFilePostFetch(self: *@This(), otherAlloc: *mem.Allocator, file_path: []const u8, max_file_size_opt: ?usize) !?[]u8 {
293 debug.assert(self.manifest_file != null);300 debug.assert(self.manifest_file != null);
294301
295 var cache_hash_file = try self.files.addOne();302 var cache_hash_file = try self.files.addOne();
296 cache_hash_file.path = try fs.path.resolve(self.alloc, &[_][]const u8{file_path});303 cache_hash_file.path = try fs.path.resolve(self.alloc, &[_][]const u8{file_path});
297304
298 return try self.populate_file_hash_fetch(otherAlloc, cache_hash_file);305 const contents = try self.populate_file_hash_fetch(otherAlloc, cache_hash_file);
306
307 return contents;
299 }308 }
300309
301 /// Add a file as a dependency of process being cached, after the initial hash has been310 /// Add a file as a dependency of process being cached, after the initial hash has been
...@@ -303,8 +312,7 @@ pub const CacheHash = struct {...@@ -303,8 +312,7 @@ pub const CacheHash = struct {
303 /// are depended on ahead of time. For example, a source file that can import other files312 /// are depended on ahead of time. For example, a source file that can import other files
304 /// will need to be recompiled if the imported file is changed.313 /// will need to be recompiled if the imported file is changed.
305 pub fn addFilePost(self: *@This(), file_path: []const u8) !void {314 pub fn addFilePost(self: *@This(), file_path: []const u8) !void {
306 const contents = try self.addFilePostFetch(self.alloc, file_path);315 _ = try self.addFilePostFetch(self.alloc, file_path, null);
307 self.alloc.free(contents);
308 }316 }
309317
310 /// Returns a base64 encoded hash of the inputs.318 /// Returns a base64 encoded hash of the inputs.
...@@ -367,16 +375,30 @@ pub const CacheHash = struct {...@@ -367,16 +375,30 @@ pub const CacheHash = struct {
367};375};
368376
369/// Hash the file, and return the contents as an array377/// Hash the file, and return the contents as an array
370fn hash_file(alloc: *Allocator, bin_digest: []u8, handle: *const fs.File) ![]u8 {378fn hash_file(alloc: *Allocator, bin_digest: []u8, handle: *const fs.File, max_file_size_opt: ?usize) !?[]u8 {
371 var blake3 = Blake3.init();379 var blake3 = Blake3.init();
380 var in_stream = handle.inStream();
372381
373 const contents = try handle.inStream().readAllAlloc(alloc, 64 * 1024);382 if (max_file_size_opt) |max_file_size| {
383 const contents = try in_stream.readAllAlloc(alloc, max_file_size);
374384
375 blake3.update(contents);385 blake3.update(contents);
376386
377 blake3.final(bin_digest);387 blake3.final(bin_digest);
378388
379 return contents;389 return contents;
390 } else {
391 var buf: [1024]u8 = undefined;
392
393 while (true) {
394 const bytes_read = try in_stream.read(buf[0..]);
395 if (bytes_read == 0) break;
396 blake3.update(buf[0..bytes_read]);
397 }
398
399 blake3.final(bin_digest);
400 return null;
401 }
380}402}
381403
382/// If the wall clock time, rounded to the same precision as the404/// If the wall clock time, rounded to the same precision as the
...@@ -407,7 +429,7 @@ test "cache file and then recall it" {...@@ -407,7 +429,7 @@ test "cache file and then recall it" {
407 ch.add(true);429 ch.add(true);
408 ch.add(@as(u16, 1234));430 ch.add(@as(u16, 1234));
409 ch.add("1234");431 ch.add("1234");
410 _ = try ch.addFile(temp_file);432 _ = try ch.addFile(temp_file, null);
411433
412 // There should be nothing in the cache434 // There should be nothing in the cache
413 testing.expectEqual(@as(?[64]u8, null), try ch.hit());435 testing.expectEqual(@as(?[64]u8, null), try ch.hit());
...@@ -421,7 +443,7 @@ test "cache file and then recall it" {...@@ -421,7 +443,7 @@ test "cache file and then recall it" {
421 ch.add(true);443 ch.add(true);
422 ch.add(@as(u16, 1234));444 ch.add(@as(u16, 1234));
423 ch.add("1234");445 ch.add("1234");
424 _ = try ch.addFile(temp_file);446 _ = try ch.addFile(temp_file, null);
425447
426 // Cache hit! We just "built" the same file448 // Cache hit! We just "built" the same file
427 digest2 = (try ch.hit()).?;449 digest2 = (try ch.hit()).?;
...@@ -448,8 +470,10 @@ test "check that changing a file makes cache fail" {...@@ -448,8 +470,10 @@ test "check that changing a file makes cache fail" {
448470
449 const temp_file = "cache_hash_change_file_test.txt";471 const temp_file = "cache_hash_change_file_test.txt";
450 const temp_manifest_dir = "cache_hash_change_file_manifest_dir";472 const temp_manifest_dir = "cache_hash_change_file_manifest_dir";
473 const original_temp_file_contents = "Hello, world!\n";
474 const updated_temp_file_contents = "Hello, world; but updated!\n";
451475
452 try cwd.writeFile(temp_file, "Hello, world!\n");476 try cwd.writeFile(temp_file, original_temp_file_contents);
453477
454 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;478 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
455 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;479 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
...@@ -459,26 +483,30 @@ test "check that changing a file makes cache fail" {...@@ -459,26 +483,30 @@ test "check that changing a file makes cache fail" {
459 defer ch.release() catch unreachable;483 defer ch.release() catch unreachable;
460484
461 ch.add("1234");485 ch.add("1234");
462 _ = try ch.addFile(temp_file);486 const temp_file_idx = try ch.addFile(temp_file, 100);
463487
464 // There should be nothing in the cache488 // There should be nothing in the cache
465 testing.expectEqual(@as(?[64]u8, null), try ch.hit());489 testing.expectEqual(@as(?[64]u8, null), try ch.hit());
466490
491 testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
492
467 digest1 = ch.final();493 digest1 = ch.final();
468 }494 }
469495
470 try cwd.writeFile(temp_file, "Hello, world; but updated!\n");496 try cwd.writeFile(temp_file, updated_temp_file_contents);
471497
472 {498 {
473 var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);499 var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);
474 defer ch.release() catch unreachable;500 defer ch.release() catch unreachable;
475501
476 ch.add("1234");502 ch.add("1234");
477 _ = try ch.addFile(temp_file);503 const temp_file_idx = try ch.addFile(temp_file, 100);
478504
479 // A file that we depend on has been updated, so the cache should not contain an entry for it505 // A file that we depend on has been updated, so the cache should not contain an entry for it
480 testing.expectEqual(@as(?[64]u8, null), try ch.hit());506 testing.expectEqual(@as(?[64]u8, null), try ch.hit());
481507
508 testing.expect(mem.eql(u8, updated_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
509
482 digest2 = ch.final();510 digest2 = ch.final();
483 }511 }
484512
...@@ -538,7 +566,7 @@ test "CacheHashes with files added after initial hash work" {...@@ -538,7 +566,7 @@ test "CacheHashes with files added after initial hash work" {
538 defer ch.release() catch unreachable;566 defer ch.release() catch unreachable;
539567
540 ch.add("1234");568 ch.add("1234");
541 _ = try ch.addFile(temp_file1);569 _ = try ch.addFile(temp_file1, null);
542570
543 // There should be nothing in the cache571 // There should be nothing in the cache
544 testing.expectEqual(@as(?[64]u8, null), try ch.hit());572 testing.expectEqual(@as(?[64]u8, null), try ch.hit());
...@@ -552,7 +580,7 @@ test "CacheHashes with files added after initial hash work" {...@@ -552,7 +580,7 @@ test "CacheHashes with files added after initial hash work" {
552 defer ch.release() catch unreachable;580 defer ch.release() catch unreachable;
553581
554 ch.add("1234");582 ch.add("1234");
555 _ = try ch.addFile(temp_file1);583 _ = try ch.addFile(temp_file1, null);
556584
557 // A file that we depend on has been updated, so the cache should not contain an entry for it585 // A file that we depend on has been updated, so the cache should not contain an entry for it
558 digest2 = (try ch.hit()).?;586 digest2 = (try ch.hit()).?;
...@@ -566,7 +594,7 @@ test "CacheHashes with files added after initial hash work" {...@@ -566,7 +594,7 @@ test "CacheHashes with files added after initial hash work" {
566 defer ch.release() catch unreachable;594 defer ch.release() catch unreachable;
567595
568 ch.add("1234");596 ch.add("1234");
569 _ = try ch.addFile(temp_file1);597 _ = try ch.addFile(temp_file1, null);
570598
571 // A file that we depend on has been updated, so the cache should not contain an entry for it599 // A file that we depend on has been updated, so the cache should not contain an entry for it
572 testing.expectEqual(@as(?[64]u8, null), try ch.hit());600 testing.expectEqual(@as(?[64]u8, null), try ch.hit());