Commit 64a590a31194a778b494937e2f91642b80e3f4d0 | zig

author	Andrew Kelley <andrew@ziglang.org> 2020-12-10 21:12:05-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2020-12-11 02:33:09-05:00
log	64a590a31194a778b494937e2f91642b80e3f4d0
tree	c5c79449e05be5d03ae436283d20ce2ff256a90f
parent	0d00938016a75ccf87b50cf55a19b725d01b4ef6

stage2: detect redundant C/C++ source files

Cache exposes BinDigest.

Compilation gains a set of a BinDigest for every C/C++ source file. We
detect when the same source/flags have already been added and emit a
compile error. This prevents a deadlock in the caching system.

Closes #7308

2 files changed, 22 insertions(+), 8 deletions(-)

src
- Cache.zig
- Compilation.zig

src/Cache.zig+9-8

...	@@ -26,6 +26,7 @@ pub fn obtain(cache: *const Cache) Manifest {	...	@@ -26,6 +26,7 @@ pub fn obtain(cache: *const Cache) Manifest {
26	/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6	26	/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
27	pub const bin_digest_len = 16;	27	pub const bin_digest_len = 16;
28	pub const hex_digest_len = bin_digest_len * 2;	28	pub const hex_digest_len = bin_digest_len * 2;
		29	pub const BinDigest = [bin_digest_len]u8;
29		30
30	const manifest_file_size_max = 50 * 1024 * 1024;	31	const manifest_file_size_max = 50 * 1024 * 1024;
31		32
...	@@ -41,7 +42,7 @@ pub const File = struct {	...	@@ -41,7 +42,7 @@ pub const File = struct {
41	path: ?[]const u8,	42	path: ?[]const u8,
42	max_file_size: ?usize,	43	max_file_size: ?usize,
43	stat: fs.File.Stat,	44	stat: fs.File.Stat,
44	bin_digest: [bin_digest_len]u8,	45	bin_digest: BinDigest,
45	contents: ?[]const u8,	46	contents: ?[]const u8,
46		47
47	pub fn deinit(self: File, allocator: Allocator) void {	48	pub fn deinit(self: File, allocator: Allocator) void {
...	@@ -139,16 +140,16 @@ pub const HashHelper = struct {	...	@@ -139,16 +140,16 @@ pub const HashHelper = struct {
139	return copy.final();	140	return copy.final();
140	}	141	}
141		142
142	pub fn peekBin(hh: HashHelper) [bin_digest_len]u8 {	143	pub fn peekBin(hh: HashHelper) BinDigest {
143	var copy = hh;	144	var copy = hh;
144	var bin_digest: [bin_digest_len]u8 = undefined;	145	var bin_digest: BinDigest = undefined;
145	copy.hasher.final(&bin_digest);	146	copy.hasher.final(&bin_digest);
146	return bin_digest;	147	return bin_digest;
147	}	148	}
148		149
149	/// Returns a hex encoded hash of the inputs, mutating the state of the hasher.	150	/// Returns a hex encoded hash of the inputs, mutating the state of the hasher.
150	pub fn final(hh: *HashHelper) [hex_digest_len]u8 {	151	pub fn final(hh: *HashHelper) [hex_digest_len]u8 {
151	var bin_digest: [bin_digest_len]u8 = undefined;	152	var bin_digest: BinDigest = undefined;
152	hh.hasher.final(&bin_digest);	153	hh.hasher.final(&bin_digest);
153		154
154	var out_digest: [hex_digest_len]u8 = undefined;	155	var out_digest: [hex_digest_len]u8 = undefined;
...	@@ -241,7 +242,7 @@ pub const Manifest = struct {	...	@@ -241,7 +242,7 @@ pub const Manifest = struct {
241	const ext = ".txt";	242	const ext = ".txt";
242	var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined;	243	var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined;
243		244
244	var bin_digest: [bin_digest_len]u8 = undefined;	245	var bin_digest: BinDigest = undefined;
245	self.hash.hasher.final(&bin_digest);	246	self.hash.hasher.final(&bin_digest);
246		247
247	_ = std.fmt.bufPrint(&self.hex_digest, "{x}", .{bin_digest}) catch unreachable;	248	_ = std.fmt.bufPrint(&self.hex_digest, "{x}", .{bin_digest}) catch unreachable;
...	@@ -347,7 +348,7 @@ pub const Manifest = struct {	...	@@ -347,7 +348,7 @@ pub const Manifest = struct {
347	cache_hash_file.stat.inode = 0;	348	cache_hash_file.stat.inode = 0;
348	}	349	}
349		350
350	var actual_digest: [bin_digest_len]u8 = undefined;	351	var actual_digest: BinDigest = undefined;
351	try hashFile(this_file, &actual_digest);	352	try hashFile(this_file, &actual_digest);
352		353
353	if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {	354	if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
...	@@ -381,7 +382,7 @@ pub const Manifest = struct {	...	@@ -381,7 +382,7 @@ pub const Manifest = struct {
381	return true;	382	return true;
382	}	383	}
383		384
384	pub fn unhit(self: *Manifest, bin_digest: [bin_digest_len]u8, input_file_count: usize) void {	385	pub fn unhit(self: *Manifest, bin_digest: BinDigest, input_file_count: usize) void {
385	// Reset the hash.	386	// Reset the hash.
386	self.hash.hasher = hasher_init;	387	self.hash.hasher = hasher_init;
387	self.hash.hasher.update(&bin_digest);	388	self.hash.hasher.update(&bin_digest);
...	@@ -530,7 +531,7 @@ pub const Manifest = struct {	...	@@ -530,7 +531,7 @@ pub const Manifest = struct {
530	// cache_release is called we still might be working on creating	531	// cache_release is called we still might be working on creating
531	// the artifacts to cache.	532	// the artifacts to cache.
532		533
533	var bin_digest: [bin_digest_len]u8 = undefined;	534	var bin_digest: BinDigest = undefined;
534	self.hash.hasher.final(&bin_digest);	535	self.hash.hasher.final(&bin_digest);
535		536
536	var out_digest: [hex_digest_len]u8 = undefined;	537	var out_digest: [hex_digest_len]u8 = undefined;

src/Compilation.zig+13

 @@ -33,6 +33,7 @@ gpa: *Allocator,
 arena_state: std.heap.ArenaAllocator.State,
 bin_file: *link.File,
 c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},
+c_object_cache_digest_set: std.AutoHashMapUnmanaged(Cache.BinDigest, void) = .{},
 stage1_lock: ?Cache.Lock = null,
 stage1_cache_manifest: *Cache.Manifest = undefined,
 @@ -1152,6 +1153,7 @@ pub fn destroy(self: *Compilation) void {
         entry.key.destroy(gpa);
     }
     self.c_object_table.deinit(gpa);
+    self.c_object_cache_digest_set.deinit(gpa);
     for (self.failed_c_objects.items()) |entry| {
         entry.value.destroy(gpa);
 @@ -1730,6 +1732,17 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
         }
     }
+    {
+        const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, man.hash.peekBin());
+        if (gop.found_existing) {
+            return comp.failCObj(
+                c_object,
+                "the same source file was already added to the same compilation with the same flags",
+                .{},
+            );
+        }
+    }
     var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
     defer arena_allocator.deinit();
     const arena = &arena_allocator.allocator;