authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-14 11:05:51-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-14 11:05:51-07:00
log778bb4bc9c9ceb62426c0ed48c079142b713b910
tree97abadae9a9c6a28d8faaaca2466ef57a4589746
parent04f6a26955bbb947983cbf3e0681d9092aaaa13f

move std.cache_hash from std to stage2

The API is pretty specific to the implementationt details of the self-hosted compiler. I don't want to have to independently support and maintain this as part of the standard library, and be obligated to not make breaking changes to it with changes to the implementation of stage2.

6 files changed, 847 insertions(+), 853 deletions(-)

lib/std/cache_hash.zig deleted-845
...@@ -1,845 +0,0 @@
1// SPDX-License-Identifier: MIT
2// Copyright (c) 2015-2020 Zig Contributors
3// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
4// The MIT license requires this copyright notice to be included in all copies
5// and substantial portions of the software.
6const std = @import("std.zig");
7const crypto = std.crypto;
8const fs = std.fs;
9const base64 = std.base64;
10const assert = std.debug.assert;
11const testing = std.testing;
12const mem = std.mem;
13const fmt = std.fmt;
14const Allocator = std.mem.Allocator;
15
16pub const base64_encoder = fs.base64_encoder;
17pub const base64_decoder = fs.base64_decoder;
18/// 16 would be 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
19/// We round up to 18 to avoid the `==` padding after base64 encoding.
20pub const BIN_DIGEST_LEN = 18;
21pub const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
22
23const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;
24
25/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it
26/// provides enough collision resistance for the CacheHash use cases, while being one of our
27/// fastest options right now.
28pub const Hasher = crypto.auth.siphash.SipHash128(1, 3);
29
30/// Initial state, that can be copied.
31pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.minimum_key_length);
32
33pub const File = struct {
34 path: ?[]const u8,
35 max_file_size: ?usize,
36 stat: fs.File.Stat,
37 bin_digest: [BIN_DIGEST_LEN]u8,
38 contents: ?[]const u8,
39
40 pub fn deinit(self: *File, allocator: *Allocator) void {
41 if (self.path) |owned_slice| {
42 allocator.free(owned_slice);
43 self.path = null;
44 }
45 if (self.contents) |contents| {
46 allocator.free(contents);
47 self.contents = null;
48 }
49 self.* = undefined;
50 }
51};
52
53pub const Cache = struct {
54 gpa: *Allocator,
55 manifest_dir: fs.Dir,
56 hash: HashHelper = .{},
57
58 /// Be sure to call `CacheHash.deinit` after successful initialization.
59 pub fn obtain(cache: *const Cache) CacheHash {
60 return CacheHash{
61 .cache = cache,
62 .hash = cache.hash,
63 .manifest_file = null,
64 .manifest_dirty = false,
65 .b64_digest = undefined,
66 };
67 }
68};
69
70pub const HashHelper = struct {
71 hasher: Hasher = hasher_init,
72
73 /// Record a slice of bytes as an dependency of the process being cached
74 pub fn addBytes(hh: *HashHelper, bytes: []const u8) void {
75 hh.hasher.update(mem.asBytes(&bytes.len));
76 hh.hasher.update(bytes);
77 }
78
79 pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void {
80 hh.add(optional_bytes != null);
81 hh.addBytes(optional_bytes orelse return);
82 }
83
84 pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void {
85 hh.add(list_of_bytes.len);
86 for (list_of_bytes) |bytes| hh.addBytes(bytes);
87 }
88
89 /// Convert the input value into bytes and record it as a dependency of the process being cached.
90 pub fn add(hh: *HashHelper, x: anytype) void {
91 switch (@TypeOf(x)) {
92 std.builtin.Version => {
93 hh.add(x.major);
94 hh.add(x.minor);
95 hh.add(x.patch);
96 return;
97 },
98 else => {},
99 }
100
101 switch (@typeInfo(@TypeOf(x))) {
102 .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)),
103 else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),
104 }
105 }
106
107 pub fn addOptional(hh: *HashHelper, optional: anytype) void {
108 hh.add(optional != null);
109 hh.add(optional orelse return);
110 }
111
112 /// Returns a base64 encoded hash of the inputs, without modifying state.
113 pub fn peek(hh: HashHelper) [BASE64_DIGEST_LEN]u8 {
114 var copy = hh;
115 return copy.final();
116 }
117
118 /// Returns a base64 encoded hash of the inputs, mutating the state of the hasher.
119 pub fn final(hh: *HashHelper) [BASE64_DIGEST_LEN]u8 {
120 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
121 hh.hasher.final(&bin_digest);
122
123 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;
124 base64_encoder.encode(&out_digest, &bin_digest);
125
126 return out_digest;
127 }
128};
129
130pub const Lock = struct {
131 manifest_file: fs.File,
132
133 pub fn release(lock: *Lock) void {
134 lock.manifest_file.close();
135 lock.* = undefined;
136 }
137};
138
139/// CacheHash manages project-local `zig-cache` directories.
140/// This is not a general-purpose cache.
141/// It is designed to be fast and simple, not to withstand attacks using specially-crafted input.
142pub const CacheHash = struct {
143 cache: *const Cache,
144 /// Current state for incremental hashing.
145 hash: HashHelper,
146 manifest_file: ?fs.File,
147 manifest_dirty: bool,
148 files: std.ArrayListUnmanaged(File) = .{},
149 b64_digest: [BASE64_DIGEST_LEN]u8,
150
151 /// Add a file as a dependency of process being cached. When `hit` is
152 /// called, the file's contents will be checked to ensure that it matches
153 /// the contents from previous times.
154 ///
155 /// Max file size will be used to determine the amount of space to the file contents
156 /// are allowed to take up in memory. If max_file_size is null, then the contents
157 /// will not be loaded into memory.
158 ///
159 /// Returns the index of the entry in the `files` array list. You can use it
160 /// to access the contents of the file after calling `hit()` like so:
161 ///
162 /// ```
163 /// var file_contents = cache_hash.files.items[file_index].contents.?;
164 /// ```
165 pub fn addFile(self: *CacheHash, file_path: []const u8, max_file_size: ?usize) !usize {
166 assert(self.manifest_file == null);
167
168 try self.files.ensureCapacity(self.cache.gpa, self.files.items.len + 1);
169 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
170
171 const idx = self.files.items.len;
172 self.files.addOneAssumeCapacity().* = .{
173 .path = resolved_path,
174 .contents = null,
175 .max_file_size = max_file_size,
176 .stat = undefined,
177 .bin_digest = undefined,
178 };
179
180 self.hash.addBytes(resolved_path);
181
182 return idx;
183 }
184
185 pub fn addOptionalFile(self: *CacheHash, optional_file_path: ?[]const u8) !void {
186 self.hash.add(optional_file_path != null);
187 const file_path = optional_file_path orelse return;
188 _ = try self.addFile(file_path, null);
189 }
190
191 pub fn addListOfFiles(self: *CacheHash, list_of_files: []const []const u8) !void {
192 self.hash.add(list_of_files.len);
193 for (list_of_files) |file_path| {
194 _ = try self.addFile(file_path, null);
195 }
196 }
197
198 /// Check the cache to see if the input exists in it. If it exists, returns `true`.
199 /// A base64 encoding of its hash is available by calling `final`.
200 ///
201 /// This function will also acquire an exclusive lock to the manifest file. This means
202 /// that a process holding a CacheHash will block any other process attempting to
203 /// acquire the lock.
204 ///
205 /// The lock on the manifest file is released when `deinit` is called. As another
206 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
207 /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
208 pub fn hit(self: *CacheHash) !bool {
209 assert(self.manifest_file == null);
210
211 const ext = ".txt";
212 var manifest_file_path: [self.b64_digest.len + ext.len]u8 = undefined;
213
214 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
215 self.hash.hasher.final(&bin_digest);
216
217 base64_encoder.encode(self.b64_digest[0..], &bin_digest);
218
219 self.hash.hasher = hasher_init;
220 self.hash.hasher.update(&bin_digest);
221
222 mem.copy(u8, &manifest_file_path, &self.b64_digest);
223 manifest_file_path[self.b64_digest.len..][0..ext.len].* = ext.*;
224
225 if (self.files.items.len != 0) {
226 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
227 .read = true,
228 .truncate = false,
229 .lock = .Exclusive,
230 });
231 } else {
232 // If there are no file inputs, we check if the manifest file exists instead of
233 // comparing the hashes on the files used for the cached item
234 self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{
235 .read = true,
236 .write = true,
237 .lock = .Exclusive,
238 }) catch |err| switch (err) {
239 error.FileNotFound => {
240 self.manifest_dirty = true;
241 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
242 .read = true,
243 .truncate = false,
244 .lock = .Exclusive,
245 });
246 return false;
247 },
248 else => |e| return e,
249 };
250 }
251
252 const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.cache.gpa, MANIFEST_FILE_SIZE_MAX);
253 defer self.cache.gpa.free(file_contents);
254
255 const input_file_count = self.files.items.len;
256 var any_file_changed = false;
257 var line_iter = mem.tokenize(file_contents, "\n");
258 var idx: usize = 0;
259 while (line_iter.next()) |line| {
260 defer idx += 1;
261
262 const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
263 const new = try self.files.addOne(self.cache.gpa);
264 new.* = .{
265 .path = null,
266 .contents = null,
267 .max_file_size = null,
268 .stat = undefined,
269 .bin_digest = undefined,
270 };
271 break :blk new;
272 };
273
274 var iter = mem.tokenize(line, " ");
275 const size = iter.next() orelse return error.InvalidFormat;
276 const inode = iter.next() orelse return error.InvalidFormat;
277 const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
278 const digest_str = iter.next() orelse return error.InvalidFormat;
279 const file_path = iter.rest();
280
281 cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;
282 cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat;
283 cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
284 base64_decoder.decode(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
285
286 if (file_path.len == 0) {
287 return error.InvalidFormat;
288 }
289 if (cache_hash_file.path) |p| {
290 if (!mem.eql(u8, file_path, p)) {
291 return error.InvalidFormat;
292 }
293 }
294
295 if (cache_hash_file.path == null) {
296 cache_hash_file.path = try self.cache.gpa.dupe(u8, file_path);
297 }
298
299 const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch {
300 return error.CacheUnavailable;
301 };
302 defer this_file.close();
303
304 const actual_stat = try this_file.stat();
305 const size_match = actual_stat.size == cache_hash_file.stat.size;
306 const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime;
307 const inode_match = actual_stat.inode == cache_hash_file.stat.inode;
308
309 if (!size_match or !mtime_match or !inode_match) {
310 self.manifest_dirty = true;
311
312 cache_hash_file.stat = actual_stat;
313
314 if (isProblematicTimestamp(cache_hash_file.stat.mtime)) {
315 cache_hash_file.stat.mtime = 0;
316 cache_hash_file.stat.inode = 0;
317 }
318
319 var actual_digest: [BIN_DIGEST_LEN]u8 = undefined;
320 try hashFile(this_file, &actual_digest);
321
322 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
323 cache_hash_file.bin_digest = actual_digest;
324 // keep going until we have the input file digests
325 any_file_changed = true;
326 }
327 }
328
329 if (!any_file_changed) {
330 self.hash.hasher.update(&cache_hash_file.bin_digest);
331 }
332 }
333
334 if (any_file_changed) {
335 // cache miss
336 // keep the manifest file open
337 // reset the hash
338 self.hash.hasher = hasher_init;
339 self.hash.hasher.update(&bin_digest);
340
341 // Remove files not in the initial hash
342 for (self.files.items[input_file_count..]) |*file| {
343 file.deinit(self.cache.gpa);
344 }
345 self.files.shrinkRetainingCapacity(input_file_count);
346
347 for (self.files.items) |file| {
348 self.hash.hasher.update(&file.bin_digest);
349 }
350 return false;
351 }
352
353 if (idx < input_file_count) {
354 self.manifest_dirty = true;
355 while (idx < input_file_count) : (idx += 1) {
356 const ch_file = &self.files.items[idx];
357 try self.populateFileHash(ch_file);
358 }
359 return false;
360 }
361
362 return true;
363 }
364
365 fn populateFileHash(self: *CacheHash, ch_file: *File) !void {
366 const file = try fs.cwd().openFile(ch_file.path.?, .{});
367 defer file.close();
368
369 ch_file.stat = try file.stat();
370
371 if (isProblematicTimestamp(ch_file.stat.mtime)) {
372 ch_file.stat.mtime = 0;
373 ch_file.stat.inode = 0;
374 }
375
376 if (ch_file.max_file_size) |max_file_size| {
377 if (ch_file.stat.size > max_file_size) {
378 return error.FileTooBig;
379 }
380
381 const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size));
382 errdefer self.cache.gpa.free(contents);
383
384 // Hash while reading from disk, to keep the contents in the cpu cache while
385 // doing hashing.
386 var hasher = hasher_init;
387 var off: usize = 0;
388 while (true) {
389 // give me everything you've got, captain
390 const bytes_read = try file.read(contents[off..]);
391 if (bytes_read == 0) break;
392 hasher.update(contents[off..][0..bytes_read]);
393 off += bytes_read;
394 }
395 hasher.final(&ch_file.bin_digest);
396
397 ch_file.contents = contents;
398 } else {
399 try hashFile(file, &ch_file.bin_digest);
400 }
401
402 self.hash.hasher.update(&ch_file.bin_digest);
403 }
404
405 /// Add a file as a dependency of process being cached, after the initial hash has been
406 /// calculated. This is useful for processes that don't know the all the files that
407 /// are depended on ahead of time. For example, a source file that can import other files
408 /// will need to be recompiled if the imported file is changed.
409 pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 {
410 assert(self.manifest_file != null);
411
412 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
413 errdefer self.cache.gpa.free(resolved_path);
414
415 const new_ch_file = try self.files.addOne(self.cache.gpa);
416 new_ch_file.* = .{
417 .path = resolved_path,
418 .max_file_size = max_file_size,
419 .stat = undefined,
420 .bin_digest = undefined,
421 .contents = null,
422 };
423 errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
424
425 try self.populateFileHash(new_ch_file);
426
427 return new_ch_file.contents.?;
428 }
429
430 /// Add a file as a dependency of process being cached, after the initial hash has been
431 /// calculated. This is useful for processes that don't know the all the files that
432 /// are depended on ahead of time. For example, a source file that can import other files
433 /// will need to be recompiled if the imported file is changed.
434 pub fn addFilePost(self: *CacheHash, file_path: []const u8) !void {
435 assert(self.manifest_file != null);
436
437 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
438 errdefer self.cache.gpa.free(resolved_path);
439
440 const new_ch_file = try self.files.addOne(self.cache.gpa);
441 new_ch_file.* = .{
442 .path = resolved_path,
443 .max_file_size = null,
444 .stat = undefined,
445 .bin_digest = undefined,
446 .contents = null,
447 };
448 errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
449
450 try self.populateFileHash(new_ch_file);
451 }
452
453 /// Returns a base64 encoded hash of the inputs.
454 pub fn final(self: *CacheHash) [BASE64_DIGEST_LEN]u8 {
455 assert(self.manifest_file != null);
456
457 // We don't close the manifest file yet, because we want to
458 // keep it locked until the API user is done using it.
459 // We also don't write out the manifest yet, because until
460 // cache_release is called we still might be working on creating
461 // the artifacts to cache.
462
463 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
464 self.hash.hasher.final(&bin_digest);
465
466 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;
467 base64_encoder.encode(&out_digest, &bin_digest);
468
469 return out_digest;
470 }
471
472 pub fn writeManifest(self: *CacheHash) !void {
473 assert(self.manifest_file != null);
474 if (!self.manifest_dirty) return;
475
476 var encoded_digest: [BASE64_DIGEST_LEN]u8 = undefined;
477 var contents = std.ArrayList(u8).init(self.cache.gpa);
478 var writer = contents.writer();
479 defer contents.deinit();
480
481 for (self.files.items) |file| {
482 base64_encoder.encode(encoded_digest[0..], &file.bin_digest);
483 try writer.print("{} {} {} {} {}\n", .{
484 file.stat.size,
485 file.stat.inode,
486 file.stat.mtime,
487 encoded_digest[0..],
488 file.path,
489 });
490 }
491
492 try self.manifest_file.?.pwriteAll(contents.items, 0);
493 self.manifest_dirty = false;
494 }
495
496 /// Obtain only the data needed to maintain a lock on the manifest file.
497 /// The `CacheHash` remains safe to deinit.
498 /// Don't forget to call `writeManifest` before this!
499 pub fn toOwnedLock(self: *CacheHash) Lock {
500 const manifest_file = self.manifest_file.?;
501 self.manifest_file = null;
502 return Lock{ .manifest_file = manifest_file };
503 }
504
505 /// Releases the manifest file and frees any memory the CacheHash was using.
506 /// `CacheHash.hit` must be called first.
507 /// Don't forget to call `writeManifest` before this!
508 pub fn deinit(self: *CacheHash) void {
509 if (self.manifest_file) |file| {
510 file.close();
511 }
512 for (self.files.items) |*file| {
513 file.deinit(self.cache.gpa);
514 }
515 self.files.deinit(self.cache.gpa);
516 }
517};
518
519fn hashFile(file: fs.File, bin_digest: []u8) !void {
520 var buf: [1024]u8 = undefined;
521
522 var hasher = hasher_init;
523 while (true) {
524 const bytes_read = try file.read(&buf);
525 if (bytes_read == 0) break;
526 hasher.update(buf[0..bytes_read]);
527 }
528
529 hasher.final(bin_digest);
530}
531
532/// If the wall clock time, rounded to the same precision as the
533/// mtime, is equal to the mtime, then we cannot rely on this mtime
534/// yet. We will instead save an mtime value that indicates the hash
535/// must be unconditionally computed.
536/// This function recognizes the precision of mtime by looking at trailing
537/// zero bits of the seconds and nanoseconds.
538fn isProblematicTimestamp(fs_clock: i128) bool {
539 const wall_clock = std.time.nanoTimestamp();
540
541 // We have to break the nanoseconds into seconds and remainder nanoseconds
542 // to detect precision of seconds, because looking at the zero bits in base
543 // 2 would not detect precision of the seconds value.
544 const fs_sec = @intCast(i64, @divFloor(fs_clock, std.time.ns_per_s));
545 const fs_nsec = @intCast(i64, @mod(fs_clock, std.time.ns_per_s));
546 var wall_sec = @intCast(i64, @divFloor(wall_clock, std.time.ns_per_s));
547 var wall_nsec = @intCast(i64, @mod(wall_clock, std.time.ns_per_s));
548
549 // First make all the least significant zero bits in the fs_clock, also zero bits in the wall clock.
550 if (fs_nsec == 0) {
551 wall_nsec = 0;
552 if (fs_sec == 0) {
553 wall_sec = 0;
554 } else {
555 wall_sec &= @as(i64, -1) << @intCast(u6, @ctz(i64, fs_sec));
556 }
557 } else {
558 wall_nsec &= @as(i64, -1) << @intCast(u6, @ctz(i64, fs_nsec));
559 }
560 return wall_nsec == fs_nsec and wall_sec == fs_sec;
561}
562
563test "cache file and then recall it" {
564 if (std.Target.current.os.tag == .wasi) {
565 // https://github.com/ziglang/zig/issues/5437
566 return error.SkipZigTest;
567 }
568 const cwd = fs.cwd();
569
570 const temp_file = "test.txt";
571 const temp_manifest_dir = "temp_manifest_dir";
572
573 const ts = std.time.nanoTimestamp();
574 try cwd.writeFile(temp_file, "Hello, world!\n");
575
576 while (isProblematicTimestamp(ts)) {
577 std.time.sleep(1);
578 }
579
580 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
581 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
582
583 var cache = Cache{
584 .gpa = testing.allocator,
585 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
586 };
587 defer cache.manifest_dir.close();
588
589 {
590 var ch = cache.obtain();
591 defer ch.deinit();
592
593 ch.hash.add(true);
594 ch.hash.add(@as(u16, 1234));
595 ch.hash.addBytes("1234");
596 _ = try ch.addFile(temp_file, null);
597
598 // There should be nothing in the cache
599 testing.expectEqual(false, try ch.hit());
600
601 digest1 = ch.final();
602 try ch.writeManifest();
603 }
604 {
605 var ch = cache.obtain();
606 defer ch.deinit();
607
608 ch.hash.add(true);
609 ch.hash.add(@as(u16, 1234));
610 ch.hash.addBytes("1234");
611 _ = try ch.addFile(temp_file, null);
612
613 // Cache hit! We just "built" the same file
614 testing.expect(try ch.hit());
615 digest2 = ch.final();
616
617 try ch.writeManifest();
618 }
619
620 testing.expectEqual(digest1, digest2);
621
622 try cwd.deleteTree(temp_manifest_dir);
623 try cwd.deleteFile(temp_file);
624}
625
626test "give problematic timestamp" {
627 var fs_clock = std.time.nanoTimestamp();
628 // to make it problematic, we make it only accurate to the second
629 fs_clock = @divTrunc(fs_clock, std.time.ns_per_s);
630 fs_clock *= std.time.ns_per_s;
631 testing.expect(isProblematicTimestamp(fs_clock));
632}
633
634test "give nonproblematic timestamp" {
635 testing.expect(!isProblematicTimestamp(std.time.nanoTimestamp() - std.time.ns_per_s));
636}
637
638test "check that changing a file makes cache fail" {
639 if (std.Target.current.os.tag == .wasi) {
640 // https://github.com/ziglang/zig/issues/5437
641 return error.SkipZigTest;
642 }
643 const cwd = fs.cwd();
644
645 const temp_file = "cache_hash_change_file_test.txt";
646 const temp_manifest_dir = "cache_hash_change_file_manifest_dir";
647 const original_temp_file_contents = "Hello, world!\n";
648 const updated_temp_file_contents = "Hello, world; but updated!\n";
649
650 try cwd.deleteTree(temp_manifest_dir);
651 try cwd.deleteTree(temp_file);
652
653 const ts = std.time.nanoTimestamp();
654 try cwd.writeFile(temp_file, original_temp_file_contents);
655
656 while (isProblematicTimestamp(ts)) {
657 std.time.sleep(1);
658 }
659
660 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
661 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
662
663 var cache = Cache{
664 .gpa = testing.allocator,
665 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
666 };
667 defer cache.manifest_dir.close();
668
669 {
670 var ch = cache.obtain();
671 defer ch.deinit();
672
673 ch.hash.addBytes("1234");
674 const temp_file_idx = try ch.addFile(temp_file, 100);
675
676 // There should be nothing in the cache
677 testing.expectEqual(false, try ch.hit());
678
679 testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
680
681 digest1 = ch.final();
682
683 try ch.writeManifest();
684 }
685
686 try cwd.writeFile(temp_file, updated_temp_file_contents);
687
688 {
689 var ch = cache.obtain();
690 defer ch.deinit();
691
692 ch.hash.addBytes("1234");
693 const temp_file_idx = try ch.addFile(temp_file, 100);
694
695 // A file that we depend on has been updated, so the cache should not contain an entry for it
696 testing.expectEqual(false, try ch.hit());
697
698 // The cache system does not keep the contents of re-hashed input files.
699 testing.expect(ch.files.items[temp_file_idx].contents == null);
700
701 digest2 = ch.final();
702
703 try ch.writeManifest();
704 }
705
706 testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
707
708 try cwd.deleteTree(temp_manifest_dir);
709 try cwd.deleteTree(temp_file);
710}
711
712test "no file inputs" {
713 if (std.Target.current.os.tag == .wasi) {
714 // https://github.com/ziglang/zig/issues/5437
715 return error.SkipZigTest;
716 }
717 const cwd = fs.cwd();
718 const temp_manifest_dir = "no_file_inputs_manifest_dir";
719 defer cwd.deleteTree(temp_manifest_dir) catch unreachable;
720
721 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
722 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
723
724 var cache = Cache{
725 .gpa = testing.allocator,
726 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
727 };
728 defer cache.manifest_dir.close();
729
730 {
731 var ch = cache.obtain();
732 defer ch.deinit();
733
734 ch.hash.addBytes("1234");
735
736 // There should be nothing in the cache
737 testing.expectEqual(false, try ch.hit());
738
739 digest1 = ch.final();
740
741 try ch.writeManifest();
742 }
743 {
744 var ch = cache.obtain();
745 defer ch.deinit();
746
747 ch.hash.addBytes("1234");
748
749 testing.expect(try ch.hit());
750 digest2 = ch.final();
751 try ch.writeManifest();
752 }
753
754 testing.expectEqual(digest1, digest2);
755}
756
757test "CacheHashes with files added after initial hash work" {
758 if (std.Target.current.os.tag == .wasi) {
759 // https://github.com/ziglang/zig/issues/5437
760 return error.SkipZigTest;
761 }
762 const cwd = fs.cwd();
763
764 const temp_file1 = "cache_hash_post_file_test1.txt";
765 const temp_file2 = "cache_hash_post_file_test2.txt";
766 const temp_manifest_dir = "cache_hash_post_file_manifest_dir";
767
768 const ts1 = std.time.nanoTimestamp();
769 try cwd.writeFile(temp_file1, "Hello, world!\n");
770 try cwd.writeFile(temp_file2, "Hello world the second!\n");
771
772 while (isProblematicTimestamp(ts1)) {
773 std.time.sleep(1);
774 }
775
776 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
777 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
778 var digest3: [BASE64_DIGEST_LEN]u8 = undefined;
779
780 var cache = Cache{
781 .gpa = testing.allocator,
782 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
783 };
784 defer cache.manifest_dir.close();
785
786 {
787 var ch = cache.obtain();
788 defer ch.deinit();
789
790 ch.hash.addBytes("1234");
791 _ = try ch.addFile(temp_file1, null);
792
793 // There should be nothing in the cache
794 testing.expectEqual(false, try ch.hit());
795
796 _ = try ch.addFilePost(temp_file2);
797
798 digest1 = ch.final();
799 try ch.writeManifest();
800 }
801 {
802 var ch = cache.obtain();
803 defer ch.deinit();
804
805 ch.hash.addBytes("1234");
806 _ = try ch.addFile(temp_file1, null);
807
808 testing.expect(try ch.hit());
809 digest2 = ch.final();
810
811 try ch.writeManifest();
812 }
813 testing.expect(mem.eql(u8, &digest1, &digest2));
814
815 // Modify the file added after initial hash
816 const ts2 = std.time.nanoTimestamp();
817 try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
818
819 while (isProblematicTimestamp(ts2)) {
820 std.time.sleep(1);
821 }
822
823 {
824 var ch = cache.obtain();
825 defer ch.deinit();
826
827 ch.hash.addBytes("1234");
828 _ = try ch.addFile(temp_file1, null);
829
830 // A file that we depend on has been updated, so the cache should not contain an entry for it
831 testing.expectEqual(false, try ch.hit());
832
833 _ = try ch.addFilePost(temp_file2);
834
835 digest3 = ch.final();
836
837 try ch.writeManifest();
838 }
839
840 testing.expect(!mem.eql(u8, &digest1, &digest3));
841
842 try cwd.deleteTree(temp_manifest_dir);
843 try cwd.deleteFile(temp_file1);
844 try cwd.deleteFile(temp_file2);
845}
lib/std/std.zig-1
...@@ -48,7 +48,6 @@ pub const base64 = @import("base64.zig");...@@ -48,7 +48,6 @@ pub const base64 = @import("base64.zig");
48pub const build = @import("build.zig");48pub const build = @import("build.zig");
49pub const builtin = @import("builtin.zig");49pub const builtin = @import("builtin.zig");
50pub const c = @import("c.zig");50pub const c = @import("c.zig");
51pub const cache_hash = @import("cache_hash.zig");
52pub const coff = @import("coff.zig");51pub const coff = @import("coff.zig");
53pub const compress = @import("compress.zig");52pub const compress = @import("compress.zig");
54pub const crypto = @import("crypto.zig");53pub const crypto = @import("crypto.zig");
src-self-hosted/Cache.zig created+839
...@@ -0,0 +1,839 @@
1gpa: *Allocator,
2manifest_dir: fs.Dir,
3hash: HashHelper = .{},
4
5const Cache = @This();
6const std = @import("std");
7const crypto = std.crypto;
8const fs = std.fs;
9const base64 = std.base64;
10const assert = std.debug.assert;
11const testing = std.testing;
12const mem = std.mem;
13const fmt = std.fmt;
14const Allocator = std.mem.Allocator;
15
16/// Be sure to call `CacheHash.deinit` after successful initialization.
17pub fn obtain(cache: *const Cache) CacheHash {
18 return CacheHash{
19 .cache = cache,
20 .hash = cache.hash,
21 .manifest_file = null,
22 .manifest_dirty = false,
23 .b64_digest = undefined,
24 };
25}
26
27pub const base64_encoder = fs.base64_encoder;
28pub const base64_decoder = fs.base64_decoder;
29/// 16 would be 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
30/// We round up to 18 to avoid the `==` padding after base64 encoding.
31pub const BIN_DIGEST_LEN = 18;
32pub const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
33
34const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;
35
36/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it
37/// provides enough collision resistance for the CacheHash use cases, while being one of our
38/// fastest options right now.
39pub const Hasher = crypto.auth.siphash.SipHash128(1, 3);
40
41/// Initial state, that can be copied.
42pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.minimum_key_length);
43
44pub const File = struct {
45 path: ?[]const u8,
46 max_file_size: ?usize,
47 stat: fs.File.Stat,
48 bin_digest: [BIN_DIGEST_LEN]u8,
49 contents: ?[]const u8,
50
51 pub fn deinit(self: *File, allocator: *Allocator) void {
52 if (self.path) |owned_slice| {
53 allocator.free(owned_slice);
54 self.path = null;
55 }
56 if (self.contents) |contents| {
57 allocator.free(contents);
58 self.contents = null;
59 }
60 self.* = undefined;
61 }
62};
63
64pub const HashHelper = struct {
65 hasher: Hasher = hasher_init,
66
67 /// Record a slice of bytes as an dependency of the process being cached
68 pub fn addBytes(hh: *HashHelper, bytes: []const u8) void {
69 hh.hasher.update(mem.asBytes(&bytes.len));
70 hh.hasher.update(bytes);
71 }
72
73 pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void {
74 hh.add(optional_bytes != null);
75 hh.addBytes(optional_bytes orelse return);
76 }
77
78 pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void {
79 hh.add(list_of_bytes.len);
80 for (list_of_bytes) |bytes| hh.addBytes(bytes);
81 }
82
83 /// Convert the input value into bytes and record it as a dependency of the process being cached.
84 pub fn add(hh: *HashHelper, x: anytype) void {
85 switch (@TypeOf(x)) {
86 std.builtin.Version => {
87 hh.add(x.major);
88 hh.add(x.minor);
89 hh.add(x.patch);
90 return;
91 },
92 else => {},
93 }
94
95 switch (@typeInfo(@TypeOf(x))) {
96 .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)),
97 else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),
98 }
99 }
100
101 pub fn addOptional(hh: *HashHelper, optional: anytype) void {
102 hh.add(optional != null);
103 hh.add(optional orelse return);
104 }
105
106 /// Returns a base64 encoded hash of the inputs, without modifying state.
107 pub fn peek(hh: HashHelper) [BASE64_DIGEST_LEN]u8 {
108 var copy = hh;
109 return copy.final();
110 }
111
112 /// Returns a base64 encoded hash of the inputs, mutating the state of the hasher.
113 pub fn final(hh: *HashHelper) [BASE64_DIGEST_LEN]u8 {
114 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
115 hh.hasher.final(&bin_digest);
116
117 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;
118 base64_encoder.encode(&out_digest, &bin_digest);
119
120 return out_digest;
121 }
122};
123
124pub const Lock = struct {
125 manifest_file: fs.File,
126
127 pub fn release(lock: *Lock) void {
128 lock.manifest_file.close();
129 lock.* = undefined;
130 }
131};
132
133/// CacheHash manages project-local `zig-cache` directories.
134/// This is not a general-purpose cache.
135/// It is designed to be fast and simple, not to withstand attacks using specially-crafted input.
136pub const CacheHash = struct {
137 cache: *const Cache,
138 /// Current state for incremental hashing.
139 hash: HashHelper,
140 manifest_file: ?fs.File,
141 manifest_dirty: bool,
142 files: std.ArrayListUnmanaged(File) = .{},
143 b64_digest: [BASE64_DIGEST_LEN]u8,
144
145 /// Add a file as a dependency of process being cached. When `hit` is
146 /// called, the file's contents will be checked to ensure that it matches
147 /// the contents from previous times.
148 ///
149 /// Max file size will be used to determine the amount of space to the file contents
150 /// are allowed to take up in memory. If max_file_size is null, then the contents
151 /// will not be loaded into memory.
152 ///
153 /// Returns the index of the entry in the `files` array list. You can use it
154 /// to access the contents of the file after calling `hit()` like so:
155 ///
156 /// ```
157 /// var file_contents = cache_hash.files.items[file_index].contents.?;
158 /// ```
159 pub fn addFile(self: *CacheHash, file_path: []const u8, max_file_size: ?usize) !usize {
160 assert(self.manifest_file == null);
161
162 try self.files.ensureCapacity(self.cache.gpa, self.files.items.len + 1);
163 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
164
165 const idx = self.files.items.len;
166 self.files.addOneAssumeCapacity().* = .{
167 .path = resolved_path,
168 .contents = null,
169 .max_file_size = max_file_size,
170 .stat = undefined,
171 .bin_digest = undefined,
172 };
173
174 self.hash.addBytes(resolved_path);
175
176 return idx;
177 }
178
179 pub fn addOptionalFile(self: *CacheHash, optional_file_path: ?[]const u8) !void {
180 self.hash.add(optional_file_path != null);
181 const file_path = optional_file_path orelse return;
182 _ = try self.addFile(file_path, null);
183 }
184
185 pub fn addListOfFiles(self: *CacheHash, list_of_files: []const []const u8) !void {
186 self.hash.add(list_of_files.len);
187 for (list_of_files) |file_path| {
188 _ = try self.addFile(file_path, null);
189 }
190 }
191
192 /// Check the cache to see if the input exists in it. If it exists, returns `true`.
193 /// A base64 encoding of its hash is available by calling `final`.
194 ///
195 /// This function will also acquire an exclusive lock to the manifest file. This means
196 /// that a process holding a CacheHash will block any other process attempting to
197 /// acquire the lock.
198 ///
199 /// The lock on the manifest file is released when `deinit` is called. As another
200 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
201 /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
202 pub fn hit(self: *CacheHash) !bool {
203 assert(self.manifest_file == null);
204
205 const ext = ".txt";
206 var manifest_file_path: [self.b64_digest.len + ext.len]u8 = undefined;
207
208 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
209 self.hash.hasher.final(&bin_digest);
210
211 base64_encoder.encode(self.b64_digest[0..], &bin_digest);
212
213 self.hash.hasher = hasher_init;
214 self.hash.hasher.update(&bin_digest);
215
216 mem.copy(u8, &manifest_file_path, &self.b64_digest);
217 manifest_file_path[self.b64_digest.len..][0..ext.len].* = ext.*;
218
219 if (self.files.items.len != 0) {
220 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
221 .read = true,
222 .truncate = false,
223 .lock = .Exclusive,
224 });
225 } else {
226 // If there are no file inputs, we check if the manifest file exists instead of
227 // comparing the hashes on the files used for the cached item
228 self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{
229 .read = true,
230 .write = true,
231 .lock = .Exclusive,
232 }) catch |err| switch (err) {
233 error.FileNotFound => {
234 self.manifest_dirty = true;
235 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
236 .read = true,
237 .truncate = false,
238 .lock = .Exclusive,
239 });
240 return false;
241 },
242 else => |e| return e,
243 };
244 }
245
246 const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.cache.gpa, MANIFEST_FILE_SIZE_MAX);
247 defer self.cache.gpa.free(file_contents);
248
249 const input_file_count = self.files.items.len;
250 var any_file_changed = false;
251 var line_iter = mem.tokenize(file_contents, "\n");
252 var idx: usize = 0;
253 while (line_iter.next()) |line| {
254 defer idx += 1;
255
256 const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
257 const new = try self.files.addOne(self.cache.gpa);
258 new.* = .{
259 .path = null,
260 .contents = null,
261 .max_file_size = null,
262 .stat = undefined,
263 .bin_digest = undefined,
264 };
265 break :blk new;
266 };
267
268 var iter = mem.tokenize(line, " ");
269 const size = iter.next() orelse return error.InvalidFormat;
270 const inode = iter.next() orelse return error.InvalidFormat;
271 const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
272 const digest_str = iter.next() orelse return error.InvalidFormat;
273 const file_path = iter.rest();
274
275 cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;
276 cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat;
277 cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
278 base64_decoder.decode(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
279
280 if (file_path.len == 0) {
281 return error.InvalidFormat;
282 }
283 if (cache_hash_file.path) |p| {
284 if (!mem.eql(u8, file_path, p)) {
285 return error.InvalidFormat;
286 }
287 }
288
289 if (cache_hash_file.path == null) {
290 cache_hash_file.path = try self.cache.gpa.dupe(u8, file_path);
291 }
292
293 const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch {
294 return error.CacheUnavailable;
295 };
296 defer this_file.close();
297
298 const actual_stat = try this_file.stat();
299 const size_match = actual_stat.size == cache_hash_file.stat.size;
300 const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime;
301 const inode_match = actual_stat.inode == cache_hash_file.stat.inode;
302
303 if (!size_match or !mtime_match or !inode_match) {
304 self.manifest_dirty = true;
305
306 cache_hash_file.stat = actual_stat;
307
308 if (isProblematicTimestamp(cache_hash_file.stat.mtime)) {
309 cache_hash_file.stat.mtime = 0;
310 cache_hash_file.stat.inode = 0;
311 }
312
313 var actual_digest: [BIN_DIGEST_LEN]u8 = undefined;
314 try hashFile(this_file, &actual_digest);
315
316 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
317 cache_hash_file.bin_digest = actual_digest;
318 // keep going until we have the input file digests
319 any_file_changed = true;
320 }
321 }
322
323 if (!any_file_changed) {
324 self.hash.hasher.update(&cache_hash_file.bin_digest);
325 }
326 }
327
328 if (any_file_changed) {
329 // cache miss
330 // keep the manifest file open
331 // reset the hash
332 self.hash.hasher = hasher_init;
333 self.hash.hasher.update(&bin_digest);
334
335 // Remove files not in the initial hash
336 for (self.files.items[input_file_count..]) |*file| {
337 file.deinit(self.cache.gpa);
338 }
339 self.files.shrinkRetainingCapacity(input_file_count);
340
341 for (self.files.items) |file| {
342 self.hash.hasher.update(&file.bin_digest);
343 }
344 return false;
345 }
346
347 if (idx < input_file_count) {
348 self.manifest_dirty = true;
349 while (idx < input_file_count) : (idx += 1) {
350 const ch_file = &self.files.items[idx];
351 try self.populateFileHash(ch_file);
352 }
353 return false;
354 }
355
356 return true;
357 }
358
359 fn populateFileHash(self: *CacheHash, ch_file: *File) !void {
360 const file = try fs.cwd().openFile(ch_file.path.?, .{});
361 defer file.close();
362
363 ch_file.stat = try file.stat();
364
365 if (isProblematicTimestamp(ch_file.stat.mtime)) {
366 ch_file.stat.mtime = 0;
367 ch_file.stat.inode = 0;
368 }
369
370 if (ch_file.max_file_size) |max_file_size| {
371 if (ch_file.stat.size > max_file_size) {
372 return error.FileTooBig;
373 }
374
375 const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size));
376 errdefer self.cache.gpa.free(contents);
377
378 // Hash while reading from disk, to keep the contents in the cpu cache while
379 // doing hashing.
380 var hasher = hasher_init;
381 var off: usize = 0;
382 while (true) {
383 // give me everything you've got, captain
384 const bytes_read = try file.read(contents[off..]);
385 if (bytes_read == 0) break;
386 hasher.update(contents[off..][0..bytes_read]);
387 off += bytes_read;
388 }
389 hasher.final(&ch_file.bin_digest);
390
391 ch_file.contents = contents;
392 } else {
393 try hashFile(file, &ch_file.bin_digest);
394 }
395
396 self.hash.hasher.update(&ch_file.bin_digest);
397 }
398
399 /// Add a file as a dependency of process being cached, after the initial hash has been
400 /// calculated. This is useful for processes that don't know the all the files that
401 /// are depended on ahead of time. For example, a source file that can import other files
402 /// will need to be recompiled if the imported file is changed.
403 pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 {
404 assert(self.manifest_file != null);
405
406 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
407 errdefer self.cache.gpa.free(resolved_path);
408
409 const new_ch_file = try self.files.addOne(self.cache.gpa);
410 new_ch_file.* = .{
411 .path = resolved_path,
412 .max_file_size = max_file_size,
413 .stat = undefined,
414 .bin_digest = undefined,
415 .contents = null,
416 };
417 errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
418
419 try self.populateFileHash(new_ch_file);
420
421 return new_ch_file.contents.?;
422 }
423
424 /// Add a file as a dependency of process being cached, after the initial hash has been
425 /// calculated. This is useful for processes that don't know the all the files that
426 /// are depended on ahead of time. For example, a source file that can import other files
427 /// will need to be recompiled if the imported file is changed.
428 pub fn addFilePost(self: *CacheHash, file_path: []const u8) !void {
429 assert(self.manifest_file != null);
430
431 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
432 errdefer self.cache.gpa.free(resolved_path);
433
434 const new_ch_file = try self.files.addOne(self.cache.gpa);
435 new_ch_file.* = .{
436 .path = resolved_path,
437 .max_file_size = null,
438 .stat = undefined,
439 .bin_digest = undefined,
440 .contents = null,
441 };
442 errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
443
444 try self.populateFileHash(new_ch_file);
445 }
446
447 /// Returns a base64 encoded hash of the inputs.
448 pub fn final(self: *CacheHash) [BASE64_DIGEST_LEN]u8 {
449 assert(self.manifest_file != null);
450
451 // We don't close the manifest file yet, because we want to
452 // keep it locked until the API user is done using it.
453 // We also don't write out the manifest yet, because until
454 // cache_release is called we still might be working on creating
455 // the artifacts to cache.
456
457 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
458 self.hash.hasher.final(&bin_digest);
459
460 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;
461 base64_encoder.encode(&out_digest, &bin_digest);
462
463 return out_digest;
464 }
465
466 pub fn writeManifest(self: *CacheHash) !void {
467 assert(self.manifest_file != null);
468 if (!self.manifest_dirty) return;
469
470 var encoded_digest: [BASE64_DIGEST_LEN]u8 = undefined;
471 var contents = std.ArrayList(u8).init(self.cache.gpa);
472 var writer = contents.writer();
473 defer contents.deinit();
474
475 for (self.files.items) |file| {
476 base64_encoder.encode(encoded_digest[0..], &file.bin_digest);
477 try writer.print("{} {} {} {} {}\n", .{
478 file.stat.size,
479 file.stat.inode,
480 file.stat.mtime,
481 encoded_digest[0..],
482 file.path,
483 });
484 }
485
486 try self.manifest_file.?.pwriteAll(contents.items, 0);
487 self.manifest_dirty = false;
488 }
489
490 /// Obtain only the data needed to maintain a lock on the manifest file.
491 /// The `CacheHash` remains safe to deinit.
492 /// Don't forget to call `writeManifest` before this!
493 pub fn toOwnedLock(self: *CacheHash) Lock {
494 const manifest_file = self.manifest_file.?;
495 self.manifest_file = null;
496 return Lock{ .manifest_file = manifest_file };
497 }
498
499 /// Releases the manifest file and frees any memory the CacheHash was using.
500 /// `CacheHash.hit` must be called first.
501 /// Don't forget to call `writeManifest` before this!
502 pub fn deinit(self: *CacheHash) void {
503 if (self.manifest_file) |file| {
504 file.close();
505 }
506 for (self.files.items) |*file| {
507 file.deinit(self.cache.gpa);
508 }
509 self.files.deinit(self.cache.gpa);
510 }
511};
512
513fn hashFile(file: fs.File, bin_digest: []u8) !void {
514 var buf: [1024]u8 = undefined;
515
516 var hasher = hasher_init;
517 while (true) {
518 const bytes_read = try file.read(&buf);
519 if (bytes_read == 0) break;
520 hasher.update(buf[0..bytes_read]);
521 }
522
523 hasher.final(bin_digest);
524}
525
526/// If the wall clock time, rounded to the same precision as the
527/// mtime, is equal to the mtime, then we cannot rely on this mtime
528/// yet. We will instead save an mtime value that indicates the hash
529/// must be unconditionally computed.
530/// This function recognizes the precision of mtime by looking at trailing
531/// zero bits of the seconds and nanoseconds.
532fn isProblematicTimestamp(fs_clock: i128) bool {
533 const wall_clock = std.time.nanoTimestamp();
534
535 // We have to break the nanoseconds into seconds and remainder nanoseconds
536 // to detect precision of seconds, because looking at the zero bits in base
537 // 2 would not detect precision of the seconds value.
538 const fs_sec = @intCast(i64, @divFloor(fs_clock, std.time.ns_per_s));
539 const fs_nsec = @intCast(i64, @mod(fs_clock, std.time.ns_per_s));
540 var wall_sec = @intCast(i64, @divFloor(wall_clock, std.time.ns_per_s));
541 var wall_nsec = @intCast(i64, @mod(wall_clock, std.time.ns_per_s));
542
543 // First make all the least significant zero bits in the fs_clock, also zero bits in the wall clock.
544 if (fs_nsec == 0) {
545 wall_nsec = 0;
546 if (fs_sec == 0) {
547 wall_sec = 0;
548 } else {
549 wall_sec &= @as(i64, -1) << @intCast(u6, @ctz(i64, fs_sec));
550 }
551 } else {
552 wall_nsec &= @as(i64, -1) << @intCast(u6, @ctz(i64, fs_nsec));
553 }
554 return wall_nsec == fs_nsec and wall_sec == fs_sec;
555}
556
557test "cache file and then recall it" {
558 if (std.Target.current.os.tag == .wasi) {
559 // https://github.com/ziglang/zig/issues/5437
560 return error.SkipZigTest;
561 }
562 const cwd = fs.cwd();
563
564 const temp_file = "test.txt";
565 const temp_manifest_dir = "temp_manifest_dir";
566
567 const ts = std.time.nanoTimestamp();
568 try cwd.writeFile(temp_file, "Hello, world!\n");
569
570 while (isProblematicTimestamp(ts)) {
571 std.time.sleep(1);
572 }
573
574 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
575 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
576
577 var cache = Cache{
578 .gpa = testing.allocator,
579 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
580 };
581 defer cache.manifest_dir.close();
582
583 {
584 var ch = cache.obtain();
585 defer ch.deinit();
586
587 ch.hash.add(true);
588 ch.hash.add(@as(u16, 1234));
589 ch.hash.addBytes("1234");
590 _ = try ch.addFile(temp_file, null);
591
592 // There should be nothing in the cache
593 testing.expectEqual(false, try ch.hit());
594
595 digest1 = ch.final();
596 try ch.writeManifest();
597 }
598 {
599 var ch = cache.obtain();
600 defer ch.deinit();
601
602 ch.hash.add(true);
603 ch.hash.add(@as(u16, 1234));
604 ch.hash.addBytes("1234");
605 _ = try ch.addFile(temp_file, null);
606
607 // Cache hit! We just "built" the same file
608 testing.expect(try ch.hit());
609 digest2 = ch.final();
610
611 try ch.writeManifest();
612 }
613
614 testing.expectEqual(digest1, digest2);
615
616 try cwd.deleteTree(temp_manifest_dir);
617 try cwd.deleteFile(temp_file);
618}
619
620test "give problematic timestamp" {
621 var fs_clock = std.time.nanoTimestamp();
622 // to make it problematic, we make it only accurate to the second
623 fs_clock = @divTrunc(fs_clock, std.time.ns_per_s);
624 fs_clock *= std.time.ns_per_s;
625 testing.expect(isProblematicTimestamp(fs_clock));
626}
627
628test "give nonproblematic timestamp" {
629 testing.expect(!isProblematicTimestamp(std.time.nanoTimestamp() - std.time.ns_per_s));
630}
631
632test "check that changing a file makes cache fail" {
633 if (std.Target.current.os.tag == .wasi) {
634 // https://github.com/ziglang/zig/issues/5437
635 return error.SkipZigTest;
636 }
637 const cwd = fs.cwd();
638
639 const temp_file = "cache_hash_change_file_test.txt";
640 const temp_manifest_dir = "cache_hash_change_file_manifest_dir";
641 const original_temp_file_contents = "Hello, world!\n";
642 const updated_temp_file_contents = "Hello, world; but updated!\n";
643
644 try cwd.deleteTree(temp_manifest_dir);
645 try cwd.deleteTree(temp_file);
646
647 const ts = std.time.nanoTimestamp();
648 try cwd.writeFile(temp_file, original_temp_file_contents);
649
650 while (isProblematicTimestamp(ts)) {
651 std.time.sleep(1);
652 }
653
654 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
655 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
656
657 var cache = Cache{
658 .gpa = testing.allocator,
659 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
660 };
661 defer cache.manifest_dir.close();
662
663 {
664 var ch = cache.obtain();
665 defer ch.deinit();
666
667 ch.hash.addBytes("1234");
668 const temp_file_idx = try ch.addFile(temp_file, 100);
669
670 // There should be nothing in the cache
671 testing.expectEqual(false, try ch.hit());
672
673 testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
674
675 digest1 = ch.final();
676
677 try ch.writeManifest();
678 }
679
680 try cwd.writeFile(temp_file, updated_temp_file_contents);
681
682 {
683 var ch = cache.obtain();
684 defer ch.deinit();
685
686 ch.hash.addBytes("1234");
687 const temp_file_idx = try ch.addFile(temp_file, 100);
688
689 // A file that we depend on has been updated, so the cache should not contain an entry for it
690 testing.expectEqual(false, try ch.hit());
691
692 // The cache system does not keep the contents of re-hashed input files.
693 testing.expect(ch.files.items[temp_file_idx].contents == null);
694
695 digest2 = ch.final();
696
697 try ch.writeManifest();
698 }
699
700 testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
701
702 try cwd.deleteTree(temp_manifest_dir);
703 try cwd.deleteTree(temp_file);
704}
705
706test "no file inputs" {
707 if (std.Target.current.os.tag == .wasi) {
708 // https://github.com/ziglang/zig/issues/5437
709 return error.SkipZigTest;
710 }
711 const cwd = fs.cwd();
712 const temp_manifest_dir = "no_file_inputs_manifest_dir";
713 defer cwd.deleteTree(temp_manifest_dir) catch unreachable;
714
715 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
716 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
717
718 var cache = Cache{
719 .gpa = testing.allocator,
720 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
721 };
722 defer cache.manifest_dir.close();
723
724 {
725 var ch = cache.obtain();
726 defer ch.deinit();
727
728 ch.hash.addBytes("1234");
729
730 // There should be nothing in the cache
731 testing.expectEqual(false, try ch.hit());
732
733 digest1 = ch.final();
734
735 try ch.writeManifest();
736 }
737 {
738 var ch = cache.obtain();
739 defer ch.deinit();
740
741 ch.hash.addBytes("1234");
742
743 testing.expect(try ch.hit());
744 digest2 = ch.final();
745 try ch.writeManifest();
746 }
747
748 testing.expectEqual(digest1, digest2);
749}
750
751test "CacheHashes with files added after initial hash work" {
752 if (std.Target.current.os.tag == .wasi) {
753 // https://github.com/ziglang/zig/issues/5437
754 return error.SkipZigTest;
755 }
756 const cwd = fs.cwd();
757
758 const temp_file1 = "cache_hash_post_file_test1.txt";
759 const temp_file2 = "cache_hash_post_file_test2.txt";
760 const temp_manifest_dir = "cache_hash_post_file_manifest_dir";
761
762 const ts1 = std.time.nanoTimestamp();
763 try cwd.writeFile(temp_file1, "Hello, world!\n");
764 try cwd.writeFile(temp_file2, "Hello world the second!\n");
765
766 while (isProblematicTimestamp(ts1)) {
767 std.time.sleep(1);
768 }
769
770 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
771 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
772 var digest3: [BASE64_DIGEST_LEN]u8 = undefined;
773
774 var cache = Cache{
775 .gpa = testing.allocator,
776 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
777 };
778 defer cache.manifest_dir.close();
779
780 {
781 var ch = cache.obtain();
782 defer ch.deinit();
783
784 ch.hash.addBytes("1234");
785 _ = try ch.addFile(temp_file1, null);
786
787 // There should be nothing in the cache
788 testing.expectEqual(false, try ch.hit());
789
790 _ = try ch.addFilePost(temp_file2);
791
792 digest1 = ch.final();
793 try ch.writeManifest();
794 }
795 {
796 var ch = cache.obtain();
797 defer ch.deinit();
798
799 ch.hash.addBytes("1234");
800 _ = try ch.addFile(temp_file1, null);
801
802 testing.expect(try ch.hit());
803 digest2 = ch.final();
804
805 try ch.writeManifest();
806 }
807 testing.expect(mem.eql(u8, &digest1, &digest2));
808
809 // Modify the file added after initial hash
810 const ts2 = std.time.nanoTimestamp();
811 try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
812
813 while (isProblematicTimestamp(ts2)) {
814 std.time.sleep(1);
815 }
816
817 {
818 var ch = cache.obtain();
819 defer ch.deinit();
820
821 ch.hash.addBytes("1234");
822 _ = try ch.addFile(temp_file1, null);
823
824 // A file that we depend on has been updated, so the cache should not contain an entry for it
825 testing.expectEqual(false, try ch.hit());
826
827 _ = try ch.addFilePost(temp_file2);
828
829 digest3 = ch.final();
830
831 try ch.writeManifest();
832 }
833
834 testing.expect(!mem.eql(u8, &digest1, &digest3));
835
836 try cwd.deleteTree(temp_manifest_dir);
837 try cwd.deleteFile(temp_file1);
838 try cwd.deleteFile(temp_file2);
839}
src-self-hosted/Compilation.zig+5-4
...@@ -17,6 +17,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation;...@@ -17,6 +17,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
17const glibc = @import("glibc.zig");17const glibc = @import("glibc.zig");
18const fatal = @import("main.zig").fatal;18const fatal = @import("main.zig").fatal;
19const Module = @import("Module.zig");19const Module = @import("Module.zig");
20const Cache = @import("Cache.zig");
2021
21/// General-purpose allocator. Used for both temporary and long-term storage.22/// General-purpose allocator. Used for both temporary and long-term storage.
22gpa: *Allocator,23gpa: *Allocator,
...@@ -46,7 +47,7 @@ disable_c_depfile: bool,...@@ -46,7 +47,7 @@ disable_c_depfile: bool,
4647
47c_source_files: []const CSourceFile,48c_source_files: []const CSourceFile,
48clang_argv: []const []const u8,49clang_argv: []const []const u8,
49cache_parent: *std.cache_hash.Cache,50cache_parent: *Cache,
50/// Path to own executable for invoking `zig clang`.51/// Path to own executable for invoking `zig clang`.
51self_exe_path: ?[]const u8,52self_exe_path: ?[]const u8,
52zig_lib_directory: Directory,53zig_lib_directory: Directory,
...@@ -78,7 +79,7 @@ owned_link_dir: ?std.fs.Dir,...@@ -78,7 +79,7 @@ owned_link_dir: ?std.fs.Dir,
78pub const InnerError = Module.InnerError;79pub const InnerError = Module.InnerError;
7980
80pub const CRTFile = struct {81pub const CRTFile = struct {
81 lock: std.cache_hash.Lock,82 lock: Cache.Lock,
82 full_object_path: []const u8,83 full_object_path: []const u8,
8384
84 fn deinit(self: *CRTFile, gpa: *Allocator) void {85 fn deinit(self: *CRTFile, gpa: *Allocator) void {
...@@ -128,7 +129,7 @@ pub const CObject = struct {...@@ -128,7 +129,7 @@ pub const CObject = struct {
128 /// This is a file system lock on the cache hash manifest representing this129 /// This is a file system lock on the cache hash manifest representing this
129 /// object. It prevents other invocations of the Zig compiler from interfering130 /// object. It prevents other invocations of the Zig compiler from interfering
130 /// with this object until released.131 /// with this object until released.
131 lock: std.cache_hash.Lock,132 lock: Cache.Lock,
132 },133 },
133 /// There will be a corresponding ErrorMsg in Compilation.failed_c_objects.134 /// There will be a corresponding ErrorMsg in Compilation.failed_c_objects.
134 failure,135 failure,
...@@ -404,7 +405,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {...@@ -404,7 +405,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation {
404 // find the same binary and incrementally update it even if there are modified source files.405 // find the same binary and incrementally update it even if there are modified source files.
405 // We do this even if outputting to the current directory because we need somewhere to store406 // We do this even if outputting to the current directory because we need somewhere to store
406 // incremental compilation metadata.407 // incremental compilation metadata.
407 const cache = try arena.create(std.cache_hash.Cache);408 const cache = try arena.create(Cache);
408 cache.* = .{409 cache.* = .{
409 .gpa = gpa,410 .gpa = gpa,
410 .manifest_dir = try options.zig_cache_directory.handle.makeOpenPath("h", .{}),411 .manifest_dir = try options.zig_cache_directory.handle.makeOpenPath("h", .{}),
src-self-hosted/introspect.zig-1
...@@ -1,7 +1,6 @@...@@ -1,7 +1,6 @@
1const std = @import("std");1const std = @import("std");
2const mem = std.mem;2const mem = std.mem;
3const fs = std.fs;3const fs = std.fs;
4const CacheHash = std.cache_hash.CacheHash;
5const Compilation = @import("Compilation.zig");4const Compilation = @import("Compilation.zig");
65
7/// Returns the sub_path that worked, or `null` if none did.6/// Returns the sub_path that worked, or `null` if none did.
src-self-hosted/link.zig+3-2
...@@ -6,6 +6,7 @@ const fs = std.fs;...@@ -6,6 +6,7 @@ const fs = std.fs;
6const trace = @import("tracy.zig").trace;6const trace = @import("tracy.zig").trace;
7const Package = @import("Package.zig");7const Package = @import("Package.zig");
8const Type = @import("type.zig").Type;8const Type = @import("type.zig").Type;
9const Cache = @import("Cache.zig");
9const build_options = @import("build_options");10const build_options = @import("build_options");
10const LibCInstallation = @import("libc_installation.zig").LibCInstallation;11const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
1112
...@@ -92,7 +93,7 @@ pub const File = struct {...@@ -92,7 +93,7 @@ pub const File = struct {
9293
93 /// Prevents other processes from clobbering files in the output directory94 /// Prevents other processes from clobbering files in the output directory
94 /// of this linking operation.95 /// of this linking operation.
95 lock: ?std.cache_hash.Lock = null,96 lock: ?Cache.Lock = null,
9697
97 pub const LinkBlock = union {98 pub const LinkBlock = union {
98 elf: Elf.TextBlock,99 elf: Elf.TextBlock,
...@@ -239,7 +240,7 @@ pub const File = struct {...@@ -239,7 +240,7 @@ pub const File = struct {
239 }240 }
240 }241 }
241242
242 pub fn toOwnedLock(self: *File) std.cache_hash.Lock {243 pub fn toOwnedLock(self: *File) Cache.Lock {
243 const lock = self.lock.?;244 const lock = self.lock.?;
244 self.lock = null;245 self.lock = null;
245 return lock;246 return lock;