Commit e563b166b2b70975899c84beb425c8739d05ed65 | zig

author	Stephen Gutekanst <stephen@hexops.com> 2021-12-14 01:10:43-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2021-12-14 13:56:35-08:00
log	e563b166b2b70975899c84beb425c8739d05ed65
tree	bbf22188ad607631e5e6220e1b4f6a4ef1170521
parent	5d1aab72d92d0ab512464af0d3ffc036467ac011

Fix compilation cache updating bug leading to slow builds

While investigating slow build times with [a large project](https://github.com/hexops/mach/issues/124),
I found that the compiler was reading from disk nearly every C source file in my project
when rebuilding despite no changes having been made. This accounted for several seconds of
time (approx. 20-30% of running `zig build` without any changes to the sources.)

The cause of this was that comparisons of file mtimes would _always_ fail (the mtime of the file on
disk was always newer than that stored in the cache manifest), and so the cache logic would always
fall back to byte-for-byte file content comparisons with what is on disk vs. in the cache-reading every
C source file in my project from disk during each rebuild. Because file contents were the same, a cache
hit occurred, and _despite the mtime being different the cache manifest would not be updated._

One can reproduce this by building a Zig project so the cache is populated, and then changing mtimes
of their C source files to be newer than what is in the cache (without altering file contents.)

The fix is rather simple: we should always write the updated cache manifest regardless of
whether or not a cache hit occurred (a cache hit doesn't indicate if a manifest is dirty) Luckily,
`writeManifest` already contains logic to determine if a manifest is dirty and becomes no-op if no
change to the manifest file is necessary-so we merely need to ensure it is invoked.

Signed-off-by: Stephen Gutekanst <stephen@hexops.com>

1 files changed, 16 insertions(+), 8 deletions(-)

src
- Compilation.zig

src/Compilation.zig+16-8

 @@ -2967,13 +2967,17 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
         try out_zig_file.writeAll(formatted);
-        man.writeManifest() catch |err| {
-            log.warn("failed to write cache manifest for C import: {s}", .{@errorName(err)});
-        };
         break :digest digest;
     } else man.final();
+    // Write the updated manifest. This is a no-op if the manifest is not dirty. Note that it is
+    // possible we had a hit and the manifest is dirty, for example if the file mtime changed but
+    // the contents were the same, we hit the cache but the manifest is dirty and we need to update
+    // it to prevent doing a full file content comparison the next time around.
+    man.writeManifest() catch |err| {
+        log.warn("failed to write cache manifest for C import: {s}", .{@errorName(err)});
+    };
     const out_zig_path = try comp.local_cache_directory.join(comp.gpa, &[_][]const u8{
         "o", &digest, cimport_zig_basename,
     });
 @@ -3288,13 +3292,17 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P
         defer o_dir.close();
         const tmp_basename = std.fs.path.basename(out_obj_path);
         try std.fs.rename(zig_cache_tmp_dir, tmp_basename, o_dir, o_basename);
-        man.writeManifest() catch |err| {
-            log.warn("failed to write cache manifest when compiling '{s}': {s}", .{ c_object.src.src_path, @errorName(err) });
-        };
         break :blk digest;
     };
+    // Write the updated manifest. This is a no-op if the manifest is not dirty. Note that it is
+    // possible we had a hit and the manifest is dirty, for example if the file mtime changed but
+    // the contents were the same, we hit the cache but the manifest is dirty and we need to update
+    // it to prevent doing a full file content comparison the next time around.
+    man.writeManifest() catch |err| {
+        log.warn("failed to write cache manifest when compiling '{s}': {s}", .{ c_object.src.src_path, @errorName(err) });
+    };
     const o_basename = try std.fmt.allocPrint(arena, "{s}{s}", .{ o_basename_noext, o_ext });
     c_object.status = .{