authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-16 12:31:42-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-16 12:31:42-07:00
log883dcb8b18ca2f1cfa7208273f7b5dad18532a7e
treec0f39266951fdf49b268d1130706f80a0e5420c0
parenta863d899e121f11fd21c3bd14afe47a9a0ff23b5

stage2 Cache: use hex instead of base64 for file paths

2 files changed, 41 insertions(+), 47 deletions(-)

BRANCH_TODO-1
......@@ -3,7 +3,6 @@
33 instead of lld not getting the object file on the linker line for some reason.
44 - stage1 C++ code integration
55 - ok file
6 * use hex for cache hash file paths
76 * support rpaths in ELF linker code
87 * build & link against compiler-rt
98 * build & link againstn freestanding libc
src-self-hosted/Cache.zig+41-46
......@@ -6,7 +6,6 @@ const Cache = @This();
66const std = @import("std");
77const crypto = std.crypto;
88const fs = std.fs;
9const base64 = std.base64;
109const assert = std.debug.assert;
1110const testing = std.testing;
1211const mem = std.mem;
......@@ -20,18 +19,15 @@ pub fn obtain(cache: *const Cache) CacheHash {
2019 .hash = cache.hash,
2120 .manifest_file = null,
2221 .manifest_dirty = false,
23 .b64_digest = undefined,
22 .hex_digest = undefined,
2423 };
2524}
2625
27pub const base64_encoder = fs.base64_encoder;
28pub const base64_decoder = fs.base64_decoder;
2926/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
30/// Currently we use SipHash and so this value must be 16 not any higher.
31pub const BIN_DIGEST_LEN = 16;
32pub const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
27pub const bin_digest_len = 16;
28pub const hex_digest_len = bin_digest_len * 2;
3329
34const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;
30const manifest_file_size_max = 50 * 1024 * 1024;
3531
3632/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it
3733/// provides enough collision resistance for the CacheHash use cases, while being one of our
......@@ -45,7 +41,7 @@ pub const File = struct {
4541 path: ?[]const u8,
4642 max_file_size: ?usize,
4743 stat: fs.File.Stat,
48 bin_digest: [BIN_DIGEST_LEN]u8,
44 bin_digest: [bin_digest_len]u8,
4945 contents: ?[]const u8,
5046
5147 pub fn deinit(self: *File, allocator: *Allocator) void {
......@@ -118,20 +114,19 @@ pub const HashHelper = struct {
118114 hh.add(optional orelse return);
119115 }
120116
121 /// Returns a base64 encoded hash of the inputs, without modifying state.
122 pub fn peek(hh: HashHelper) [BASE64_DIGEST_LEN]u8 {
117 /// Returns a hex encoded hash of the inputs, without modifying state.
118 pub fn peek(hh: HashHelper) [hex_digest_len]u8 {
123119 var copy = hh;
124120 return copy.final();
125121 }
126122
127 /// Returns a base64 encoded hash of the inputs, mutating the state of the hasher.
128 pub fn final(hh: *HashHelper) [BASE64_DIGEST_LEN]u8 {
129 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
123 /// Returns a hex encoded hash of the inputs, mutating the state of the hasher.
124 pub fn final(hh: *HashHelper) [hex_digest_len]u8 {
125 var bin_digest: [bin_digest_len]u8 = undefined;
130126 hh.hasher.final(&bin_digest);
131127
132 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;
133 base64_encoder.encode(&out_digest, &bin_digest);
134
128 var out_digest: [hex_digest_len]u8 = undefined;
129 _ = std.fmt.bufPrint(&out_digest, "{x}", .{bin_digest}) catch unreachable;
135130 return out_digest;
136131 }
137132};
......@@ -155,7 +150,7 @@ pub const CacheHash = struct {
155150 manifest_file: ?fs.File,
156151 manifest_dirty: bool,
157152 files: std.ArrayListUnmanaged(File) = .{},
158 b64_digest: [BASE64_DIGEST_LEN]u8,
153 hex_digest: [hex_digest_len]u8,
159154
160155 /// Add a file as a dependency of process being cached. When `hit` is
161156 /// called, the file's contents will be checked to ensure that it matches
......@@ -205,7 +200,7 @@ pub const CacheHash = struct {
205200 }
206201
207202 /// Check the cache to see if the input exists in it. If it exists, returns `true`.
208 /// A base64 encoding of its hash is available by calling `final`.
203 /// A hex encoding of its hash is available by calling `final`.
209204 ///
210205 /// This function will also acquire an exclusive lock to the manifest file. This means
211206 /// that a process holding a CacheHash will block any other process attempting to
......@@ -218,18 +213,18 @@ pub const CacheHash = struct {
218213 assert(self.manifest_file == null);
219214
220215 const ext = ".txt";
221 var manifest_file_path: [self.b64_digest.len + ext.len]u8 = undefined;
216 var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined;
222217
223 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
218 var bin_digest: [bin_digest_len]u8 = undefined;
224219 self.hash.hasher.final(&bin_digest);
225220
226 base64_encoder.encode(self.b64_digest[0..], &bin_digest);
221 _ = std.fmt.bufPrint(&self.hex_digest, "{x}", .{bin_digest}) catch unreachable;
227222
228223 self.hash.hasher = hasher_init;
229224 self.hash.hasher.update(&bin_digest);
230225
231 mem.copy(u8, &manifest_file_path, &self.b64_digest);
232 manifest_file_path[self.b64_digest.len..][0..ext.len].* = ext.*;
226 mem.copy(u8, &manifest_file_path, &self.hex_digest);
227 manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*;
233228
234229 if (self.files.items.len != 0) {
235230 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
......@@ -258,7 +253,7 @@ pub const CacheHash = struct {
258253 };
259254 }
260255
261 const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.cache.gpa, MANIFEST_FILE_SIZE_MAX);
256 const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.cache.gpa, manifest_file_size_max);
262257 defer self.cache.gpa.free(file_contents);
263258
264259 const input_file_count = self.files.items.len;
......@@ -290,7 +285,7 @@ pub const CacheHash = struct {
290285 cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;
291286 cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat;
292287 cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
293 base64_decoder.decode(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
288 std.fmt.hexToBytes(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
294289
295290 if (file_path.len == 0) {
296291 return error.InvalidFormat;
......@@ -325,7 +320,7 @@ pub const CacheHash = struct {
325320 cache_hash_file.stat.inode = 0;
326321 }
327322
328 var actual_digest: [BIN_DIGEST_LEN]u8 = undefined;
323 var actual_digest: [bin_digest_len]u8 = undefined;
329324 try hashFile(this_file, &actual_digest);
330325
331326 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
......@@ -462,7 +457,7 @@ pub const CacheHash = struct {
462457 pub fn addDepFilePost(self: *CacheHash, dir: fs.Dir, dep_file_basename: []const u8) !void {
463458 assert(self.manifest_file != null);
464459
465 const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, MANIFEST_FILE_SIZE_MAX);
460 const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max);
466461 defer self.cache.gpa.free(dep_file_contents);
467462
468463 const DepTokenizer = @import("DepTokenizer.zig");
......@@ -498,8 +493,8 @@ pub const CacheHash = struct {
498493 }
499494 }
500495
501 /// Returns a base64 encoded hash of the inputs.
502 pub fn final(self: *CacheHash) [BASE64_DIGEST_LEN]u8 {
496 /// Returns a hex encoded hash of the inputs.
497 pub fn final(self: *CacheHash) [hex_digest_len]u8 {
503498 assert(self.manifest_file != null);
504499
505500 // We don't close the manifest file yet, because we want to
......@@ -508,11 +503,11 @@ pub const CacheHash = struct {
508503 // cache_release is called we still might be working on creating
509504 // the artifacts to cache.
510505
511 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
506 var bin_digest: [bin_digest_len]u8 = undefined;
512507 self.hash.hasher.final(&bin_digest);
513508
514 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;
515 base64_encoder.encode(&out_digest, &bin_digest);
509 var out_digest: [hex_digest_len]u8 = undefined;
510 _ = std.fmt.bufPrint(&out_digest, "{x}", .{bin_digest}) catch unreachable;
516511
517512 return out_digest;
518513 }
......@@ -521,18 +516,18 @@ pub const CacheHash = struct {
521516 assert(self.manifest_file != null);
522517 if (!self.manifest_dirty) return;
523518
524 var encoded_digest: [BASE64_DIGEST_LEN]u8 = undefined;
519 var encoded_digest: [hex_digest_len]u8 = undefined;
525520 var contents = std.ArrayList(u8).init(self.cache.gpa);
526521 var writer = contents.writer();
527522 defer contents.deinit();
528523
529524 for (self.files.items) |file| {
530 base64_encoder.encode(encoded_digest[0..], &file.bin_digest);
531 try writer.print("{} {} {} {} {}\n", .{
525 _ = std.fmt.bufPrint(&encoded_digest, "{x}", .{file.bin_digest}) catch unreachable;
526 try writer.print("{d} {d} {d} {s} {s}\n", .{
532527 file.stat.size,
533528 file.stat.inode,
534529 file.stat.mtime,
535 encoded_digest[0..],
530 &encoded_digest,
536531 file.path,
537532 });
538533 }
......@@ -625,8 +620,8 @@ test "cache file and then recall it" {
625620 std.time.sleep(1);
626621 }
627622
628 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
629 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
623 var digest1: [hex_digest_len]u8 = undefined;
624 var digest2: [hex_digest_len]u8 = undefined;
630625
631626 {
632627 var cache = Cache{
......@@ -707,8 +702,8 @@ test "check that changing a file makes cache fail" {
707702 std.time.sleep(1);
708703 }
709704
710 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
711 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
705 var digest1: [hex_digest_len]u8 = undefined;
706 var digest2: [hex_digest_len]u8 = undefined;
712707
713708 {
714709 var cache = Cache{
......@@ -770,8 +765,8 @@ test "no file inputs" {
770765 const temp_manifest_dir = "no_file_inputs_manifest_dir";
771766 defer cwd.deleteTree(temp_manifest_dir) catch {};
772767
773 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
774 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
768 var digest1: [hex_digest_len]u8 = undefined;
769 var digest2: [hex_digest_len]u8 = undefined;
775770
776771 var cache = Cache{
777772 .gpa = testing.allocator,
......@@ -825,9 +820,9 @@ test "CacheHashes with files added after initial hash work" {
825820 std.time.sleep(1);
826821 }
827822
828 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
829 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
830 var digest3: [BASE64_DIGEST_LEN]u8 = undefined;
823 var digest1: [hex_digest_len]u8 = undefined;
824 var digest2: [hex_digest_len]u8 = undefined;
825 var digest3: [hex_digest_len]u8 = undefined;
831826
832827 {
833828 var cache = Cache{