authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-16 12:31:42-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-16 12:31:42-07:00
log883dcb8b18ca2f1cfa7208273f7b5dad18532a7e
treec0f39266951fdf49b268d1130706f80a0e5420c0
parenta863d899e121f11fd21c3bd14afe47a9a0ff23b5

stage2 Cache: use hex instead of base64 for file paths

2 files changed, 41 insertions(+), 47 deletions(-)

BRANCH_TODO-1
...@@ -3,7 +3,6 @@...@@ -3,7 +3,6 @@
3 instead of lld not getting the object file on the linker line for some reason.3 instead of lld not getting the object file on the linker line for some reason.
4 - stage1 C++ code integration4 - stage1 C++ code integration
5 - ok file5 - ok file
6 * use hex for cache hash file paths
7 * support rpaths in ELF linker code6 * support rpaths in ELF linker code
8 * build & link against compiler-rt7 * build & link against compiler-rt
9 * build & link againstn freestanding libc8 * build & link againstn freestanding libc
src-self-hosted/Cache.zig+41-46
...@@ -6,7 +6,6 @@ const Cache = @This();...@@ -6,7 +6,6 @@ const Cache = @This();
6const std = @import("std");6const std = @import("std");
7const crypto = std.crypto;7const crypto = std.crypto;
8const fs = std.fs;8const fs = std.fs;
9const base64 = std.base64;
10const assert = std.debug.assert;9const assert = std.debug.assert;
11const testing = std.testing;10const testing = std.testing;
12const mem = std.mem;11const mem = std.mem;
...@@ -20,18 +19,15 @@ pub fn obtain(cache: *const Cache) CacheHash {...@@ -20,18 +19,15 @@ pub fn obtain(cache: *const Cache) CacheHash {
20 .hash = cache.hash,19 .hash = cache.hash,
21 .manifest_file = null,20 .manifest_file = null,
22 .manifest_dirty = false,21 .manifest_dirty = false,
23 .b64_digest = undefined,22 .hex_digest = undefined,
24 };23 };
25}24}
2625
27pub const base64_encoder = fs.base64_encoder;
28pub const base64_decoder = fs.base64_decoder;
29/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-626/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
30/// Currently we use SipHash and so this value must be 16 not any higher.27pub const bin_digest_len = 16;
31pub const BIN_DIGEST_LEN = 16;28pub const hex_digest_len = bin_digest_len * 2;
32pub const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
3329
34const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;30const manifest_file_size_max = 50 * 1024 * 1024;
3531
36/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it32/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it
37/// provides enough collision resistance for the CacheHash use cases, while being one of our33/// provides enough collision resistance for the CacheHash use cases, while being one of our
...@@ -45,7 +41,7 @@ pub const File = struct {...@@ -45,7 +41,7 @@ pub const File = struct {
45 path: ?[]const u8,41 path: ?[]const u8,
46 max_file_size: ?usize,42 max_file_size: ?usize,
47 stat: fs.File.Stat,43 stat: fs.File.Stat,
48 bin_digest: [BIN_DIGEST_LEN]u8,44 bin_digest: [bin_digest_len]u8,
49 contents: ?[]const u8,45 contents: ?[]const u8,
5046
51 pub fn deinit(self: *File, allocator: *Allocator) void {47 pub fn deinit(self: *File, allocator: *Allocator) void {
...@@ -118,20 +114,19 @@ pub const HashHelper = struct {...@@ -118,20 +114,19 @@ pub const HashHelper = struct {
118 hh.add(optional orelse return);114 hh.add(optional orelse return);
119 }115 }
120116
121 /// Returns a base64 encoded hash of the inputs, without modifying state.117 /// Returns a hex encoded hash of the inputs, without modifying state.
122 pub fn peek(hh: HashHelper) [BASE64_DIGEST_LEN]u8 {118 pub fn peek(hh: HashHelper) [hex_digest_len]u8 {
123 var copy = hh;119 var copy = hh;
124 return copy.final();120 return copy.final();
125 }121 }
126122
127 /// Returns a base64 encoded hash of the inputs, mutating the state of the hasher.123 /// Returns a hex encoded hash of the inputs, mutating the state of the hasher.
128 pub fn final(hh: *HashHelper) [BASE64_DIGEST_LEN]u8 {124 pub fn final(hh: *HashHelper) [hex_digest_len]u8 {
129 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;125 var bin_digest: [bin_digest_len]u8 = undefined;
130 hh.hasher.final(&bin_digest);126 hh.hasher.final(&bin_digest);
131127
132 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;128 var out_digest: [hex_digest_len]u8 = undefined;
133 base64_encoder.encode(&out_digest, &bin_digest);129 _ = std.fmt.bufPrint(&out_digest, "{x}", .{bin_digest}) catch unreachable;
134
135 return out_digest;130 return out_digest;
136 }131 }
137};132};
...@@ -155,7 +150,7 @@ pub const CacheHash = struct {...@@ -155,7 +150,7 @@ pub const CacheHash = struct {
155 manifest_file: ?fs.File,150 manifest_file: ?fs.File,
156 manifest_dirty: bool,151 manifest_dirty: bool,
157 files: std.ArrayListUnmanaged(File) = .{},152 files: std.ArrayListUnmanaged(File) = .{},
158 b64_digest: [BASE64_DIGEST_LEN]u8,153 hex_digest: [hex_digest_len]u8,
159154
160 /// Add a file as a dependency of process being cached. When `hit` is155 /// Add a file as a dependency of process being cached. When `hit` is
161 /// called, the file's contents will be checked to ensure that it matches156 /// called, the file's contents will be checked to ensure that it matches
...@@ -205,7 +200,7 @@ pub const CacheHash = struct {...@@ -205,7 +200,7 @@ pub const CacheHash = struct {
205 }200 }
206201
207 /// Check the cache to see if the input exists in it. If it exists, returns `true`.202 /// Check the cache to see if the input exists in it. If it exists, returns `true`.
208 /// A base64 encoding of its hash is available by calling `final`.203 /// A hex encoding of its hash is available by calling `final`.
209 ///204 ///
210 /// This function will also acquire an exclusive lock to the manifest file. This means205 /// This function will also acquire an exclusive lock to the manifest file. This means
211 /// that a process holding a CacheHash will block any other process attempting to206 /// that a process holding a CacheHash will block any other process attempting to
...@@ -218,18 +213,18 @@ pub const CacheHash = struct {...@@ -218,18 +213,18 @@ pub const CacheHash = struct {
218 assert(self.manifest_file == null);213 assert(self.manifest_file == null);
219214
220 const ext = ".txt";215 const ext = ".txt";
221 var manifest_file_path: [self.b64_digest.len + ext.len]u8 = undefined;216 var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined;
222217
223 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;218 var bin_digest: [bin_digest_len]u8 = undefined;
224 self.hash.hasher.final(&bin_digest);219 self.hash.hasher.final(&bin_digest);
225220
226 base64_encoder.encode(self.b64_digest[0..], &bin_digest);221 _ = std.fmt.bufPrint(&self.hex_digest, "{x}", .{bin_digest}) catch unreachable;
227222
228 self.hash.hasher = hasher_init;223 self.hash.hasher = hasher_init;
229 self.hash.hasher.update(&bin_digest);224 self.hash.hasher.update(&bin_digest);
230225
231 mem.copy(u8, &manifest_file_path, &self.b64_digest);226 mem.copy(u8, &manifest_file_path, &self.hex_digest);
232 manifest_file_path[self.b64_digest.len..][0..ext.len].* = ext.*;227 manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*;
233228
234 if (self.files.items.len != 0) {229 if (self.files.items.len != 0) {
235 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{230 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
...@@ -258,7 +253,7 @@ pub const CacheHash = struct {...@@ -258,7 +253,7 @@ pub const CacheHash = struct {
258 };253 };
259 }254 }
260255
261 const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.cache.gpa, MANIFEST_FILE_SIZE_MAX);256 const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.cache.gpa, manifest_file_size_max);
262 defer self.cache.gpa.free(file_contents);257 defer self.cache.gpa.free(file_contents);
263258
264 const input_file_count = self.files.items.len;259 const input_file_count = self.files.items.len;
...@@ -290,7 +285,7 @@ pub const CacheHash = struct {...@@ -290,7 +285,7 @@ pub const CacheHash = struct {
290 cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;285 cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;
291 cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat;286 cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat;
292 cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;287 cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
293 base64_decoder.decode(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;288 std.fmt.hexToBytes(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
294289
295 if (file_path.len == 0) {290 if (file_path.len == 0) {
296 return error.InvalidFormat;291 return error.InvalidFormat;
...@@ -325,7 +320,7 @@ pub const CacheHash = struct {...@@ -325,7 +320,7 @@ pub const CacheHash = struct {
325 cache_hash_file.stat.inode = 0;320 cache_hash_file.stat.inode = 0;
326 }321 }
327322
328 var actual_digest: [BIN_DIGEST_LEN]u8 = undefined;323 var actual_digest: [bin_digest_len]u8 = undefined;
329 try hashFile(this_file, &actual_digest);324 try hashFile(this_file, &actual_digest);
330325
331 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {326 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
...@@ -462,7 +457,7 @@ pub const CacheHash = struct {...@@ -462,7 +457,7 @@ pub const CacheHash = struct {
462 pub fn addDepFilePost(self: *CacheHash, dir: fs.Dir, dep_file_basename: []const u8) !void {457 pub fn addDepFilePost(self: *CacheHash, dir: fs.Dir, dep_file_basename: []const u8) !void {
463 assert(self.manifest_file != null);458 assert(self.manifest_file != null);
464459
465 const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, MANIFEST_FILE_SIZE_MAX);460 const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max);
466 defer self.cache.gpa.free(dep_file_contents);461 defer self.cache.gpa.free(dep_file_contents);
467462
468 const DepTokenizer = @import("DepTokenizer.zig");463 const DepTokenizer = @import("DepTokenizer.zig");
...@@ -498,8 +493,8 @@ pub const CacheHash = struct {...@@ -498,8 +493,8 @@ pub const CacheHash = struct {
498 }493 }
499 }494 }
500495
501 /// Returns a base64 encoded hash of the inputs.496 /// Returns a hex encoded hash of the inputs.
502 pub fn final(self: *CacheHash) [BASE64_DIGEST_LEN]u8 {497 pub fn final(self: *CacheHash) [hex_digest_len]u8 {
503 assert(self.manifest_file != null);498 assert(self.manifest_file != null);
504499
505 // We don't close the manifest file yet, because we want to500 // We don't close the manifest file yet, because we want to
...@@ -508,11 +503,11 @@ pub const CacheHash = struct {...@@ -508,11 +503,11 @@ pub const CacheHash = struct {
508 // cache_release is called we still might be working on creating503 // cache_release is called we still might be working on creating
509 // the artifacts to cache.504 // the artifacts to cache.
510505
511 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;506 var bin_digest: [bin_digest_len]u8 = undefined;
512 self.hash.hasher.final(&bin_digest);507 self.hash.hasher.final(&bin_digest);
513508
514 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;509 var out_digest: [hex_digest_len]u8 = undefined;
515 base64_encoder.encode(&out_digest, &bin_digest);510 _ = std.fmt.bufPrint(&out_digest, "{x}", .{bin_digest}) catch unreachable;
516511
517 return out_digest;512 return out_digest;
518 }513 }
...@@ -521,18 +516,18 @@ pub const CacheHash = struct {...@@ -521,18 +516,18 @@ pub const CacheHash = struct {
521 assert(self.manifest_file != null);516 assert(self.manifest_file != null);
522 if (!self.manifest_dirty) return;517 if (!self.manifest_dirty) return;
523518
524 var encoded_digest: [BASE64_DIGEST_LEN]u8 = undefined;519 var encoded_digest: [hex_digest_len]u8 = undefined;
525 var contents = std.ArrayList(u8).init(self.cache.gpa);520 var contents = std.ArrayList(u8).init(self.cache.gpa);
526 var writer = contents.writer();521 var writer = contents.writer();
527 defer contents.deinit();522 defer contents.deinit();
528523
529 for (self.files.items) |file| {524 for (self.files.items) |file| {
530 base64_encoder.encode(encoded_digest[0..], &file.bin_digest);525 _ = std.fmt.bufPrint(&encoded_digest, "{x}", .{file.bin_digest}) catch unreachable;
531 try writer.print("{} {} {} {} {}\n", .{526 try writer.print("{d} {d} {d} {s} {s}\n", .{
532 file.stat.size,527 file.stat.size,
533 file.stat.inode,528 file.stat.inode,
534 file.stat.mtime,529 file.stat.mtime,
535 encoded_digest[0..],530 &encoded_digest,
536 file.path,531 file.path,
537 });532 });
538 }533 }
...@@ -625,8 +620,8 @@ test "cache file and then recall it" {...@@ -625,8 +620,8 @@ test "cache file and then recall it" {
625 std.time.sleep(1);620 std.time.sleep(1);
626 }621 }
627622
628 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;623 var digest1: [hex_digest_len]u8 = undefined;
629 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;624 var digest2: [hex_digest_len]u8 = undefined;
630625
631 {626 {
632 var cache = Cache{627 var cache = Cache{
...@@ -707,8 +702,8 @@ test "check that changing a file makes cache fail" {...@@ -707,8 +702,8 @@ test "check that changing a file makes cache fail" {
707 std.time.sleep(1);702 std.time.sleep(1);
708 }703 }
709704
710 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;705 var digest1: [hex_digest_len]u8 = undefined;
711 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;706 var digest2: [hex_digest_len]u8 = undefined;
712707
713 {708 {
714 var cache = Cache{709 var cache = Cache{
...@@ -770,8 +765,8 @@ test "no file inputs" {...@@ -770,8 +765,8 @@ test "no file inputs" {
770 const temp_manifest_dir = "no_file_inputs_manifest_dir";765 const temp_manifest_dir = "no_file_inputs_manifest_dir";
771 defer cwd.deleteTree(temp_manifest_dir) catch {};766 defer cwd.deleteTree(temp_manifest_dir) catch {};
772767
773 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;768 var digest1: [hex_digest_len]u8 = undefined;
774 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;769 var digest2: [hex_digest_len]u8 = undefined;
775770
776 var cache = Cache{771 var cache = Cache{
777 .gpa = testing.allocator,772 .gpa = testing.allocator,
...@@ -825,9 +820,9 @@ test "CacheHashes with files added after initial hash work" {...@@ -825,9 +820,9 @@ test "CacheHashes with files added after initial hash work" {
825 std.time.sleep(1);820 std.time.sleep(1);
826 }821 }
827822
828 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;823 var digest1: [hex_digest_len]u8 = undefined;
829 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;824 var digest2: [hex_digest_len]u8 = undefined;
830 var digest3: [BASE64_DIGEST_LEN]u8 = undefined;825 var digest3: [hex_digest_len]u8 = undefined;
831826
832 {827 {
833 var cache = Cache{828 var cache = Cache{