authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-16 18:15:47-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-16 18:15:47-07:00
logba656e5c9f58e0440278918e5c6cdf0ac2fe673c
tree109651051f0077d7bc6942ec6d4e3ed2c24daf7f
parentf1a9344ffeaa9b449eb111a112df33c845a1210c

zig fetch: add `--debug-hash` argument

This argument causes zig to print verbose hashing information to stdout, which can be used to diff two different fetches and find out why the hashes do not equal each other.

2 files changed, 41 insertions(+), 3 deletions(-)

src/Package/Fetch.zig+34-3
...@@ -81,6 +81,10 @@ pub const JobQueue = struct {...@@ -81,6 +81,10 @@ pub const JobQueue = struct {
81 wait_group: WaitGroup = .{},81 wait_group: WaitGroup = .{},
82 global_cache: Cache.Directory,82 global_cache: Cache.Directory,
83 recursive: bool,83 recursive: bool,
84 /// Dumps hash information to stdout which can be used to troubleshoot why
85 /// two hashes of the same package do not match.
86 /// If this is true, `recursive` must be false.
87 debug_hash: bool,
84 work_around_btrfs_bug: bool,88 work_around_btrfs_bug: bool,
8589
86 pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);90 pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);
...@@ -1315,7 +1319,7 @@ fn computeHash(...@@ -1315,7 +1319,7 @@ fn computeHash(
1315 const kind: HashedFile.Kind = switch (entry.kind) {1319 const kind: HashedFile.Kind = switch (entry.kind) {
1316 .directory => unreachable,1320 .directory => unreachable,
1317 .file => .file,1321 .file => .file,
1318 .sym_link => .sym_link,1322 .sym_link => .link,
1319 else => return f.fail(f.location_tok, try eb.printString(1323 else => return f.fail(f.location_tok, try eb.printString(
1320 "package contains '{s}' which has illegal file type '{s}'",1324 "package contains '{s}' which has illegal file type '{s}'",
1321 .{ entry.path, @tagName(entry.kind) },1325 .{ entry.path, @tagName(entry.kind) },
...@@ -1399,9 +1403,36 @@ fn computeHash(...@@ -1399,9 +1403,36 @@ fn computeHash(
1399 }1403 }
14001404
1401 if (any_failures) return error.FetchFailed;1405 if (any_failures) return error.FetchFailed;
1406
1407 if (f.job_queue.debug_hash) {
1408 assert(!f.job_queue.recursive);
1409 // Print something to stdout that can be text diffed to figure out why
1410 // the package hash is different.
1411 dumpHashInfo(all_files.items) catch |err| {
1412 std.debug.print("unable to write to stdout: {s}\n", .{@errorName(err)});
1413 std.process.exit(1);
1414 };
1415 }
1416
1402 return hasher.finalResult();1417 return hasher.finalResult();
1403}1418}
14041419
1420fn dumpHashInfo(all_files: []const *const HashedFile) !void {
1421 const stdout = std.io.getStdOut();
1422 var bw = std.io.bufferedWriter(stdout.writer());
1423 const w = bw.writer();
1424
1425 for (all_files) |hashed_file| {
1426 try w.print("{s}: {s}: {s}\n", .{
1427 @tagName(hashed_file.kind),
1428 std.fmt.fmtSliceHexLower(&hashed_file.hash),
1429 hashed_file.normalized_path,
1430 });
1431 }
1432
1433 try bw.flush();
1434}
1435
1405fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void {1436fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void {
1406 defer wg.finish();1437 defer wg.finish();
1407 hashed_file.failure = hashFileFallible(dir, hashed_file);1438 hashed_file.failure = hashFileFallible(dir, hashed_file);
...@@ -1427,7 +1458,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void...@@ -1427,7 +1458,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
1427 hasher.update(buf[0..bytes_read]);1458 hasher.update(buf[0..bytes_read]);
1428 }1459 }
1429 },1460 },
1430 .sym_link => {1461 .link => {
1431 const link_name = try dir.readLink(hashed_file.fs_path, &buf);1462 const link_name = try dir.readLink(hashed_file.fs_path, &buf);
1432 if (fs.path.sep != canonical_sep) {1463 if (fs.path.sep != canonical_sep) {
1433 // Package hashes are intended to be consistent across1464 // Package hashes are intended to be consistent across
...@@ -1480,7 +1511,7 @@ const HashedFile = struct {...@@ -1480,7 +1511,7 @@ const HashedFile = struct {
1480 fs.File.StatError ||1511 fs.File.StatError ||
1481 fs.Dir.ReadLinkError;1512 fs.Dir.ReadLinkError;
14821513
1483 const Kind = enum { file, sym_link };1514 const Kind = enum { file, link };
14841515
1485 fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool {1516 fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool {
1486 _ = context;1517 _ = context;
src/main.zig+7
...@@ -5143,6 +5143,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi...@@ -5143,6 +5143,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
5143 .thread_pool = &thread_pool,5143 .thread_pool = &thread_pool,
5144 .global_cache = global_cache_directory,5144 .global_cache = global_cache_directory,
5145 .recursive = true,5145 .recursive = true,
5146 .debug_hash = false,
5146 .work_around_btrfs_bug = work_around_btrfs_bug,5147 .work_around_btrfs_bug = work_around_btrfs_bug,
5147 };5148 };
5148 defer job_queue.deinit();5149 defer job_queue.deinit();
...@@ -6991,6 +6992,7 @@ pub const usage_fetch =...@@ -6991,6 +6992,7 @@ pub const usage_fetch =
6991 \\Options:6992 \\Options:
6992 \\ -h, --help Print this help and exit6993 \\ -h, --help Print this help and exit
6993 \\ --global-cache-dir [path] Override path to global Zig cache directory6994 \\ --global-cache-dir [path] Override path to global Zig cache directory
6995 \\ --debug-hash Print verbose hash information to stdout
6994 \\6996 \\
6995;6997;
69966998
...@@ -7004,6 +7006,7 @@ fn cmdFetch(...@@ -7004,6 +7006,7 @@ fn cmdFetch(
7004 std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND");7006 std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND");
7005 var opt_path_or_url: ?[]const u8 = null;7007 var opt_path_or_url: ?[]const u8 = null;
7006 var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");7008 var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");
7009 var debug_hash: bool = false;
70077010
7008 {7011 {
7009 var i: usize = 0;7012 var i: usize = 0;
...@@ -7019,6 +7022,9 @@ fn cmdFetch(...@@ -7019,6 +7022,9 @@ fn cmdFetch(
7019 i += 1;7022 i += 1;
7020 override_global_cache_dir = args[i];7023 override_global_cache_dir = args[i];
7021 continue;7024 continue;
7025 } else if (mem.eql(u8, arg, "--debug-hash")) {
7026 debug_hash = true;
7027 continue;
7022 } else {7028 } else {
7023 fatal("unrecognized parameter: '{s}'", .{arg});7029 fatal("unrecognized parameter: '{s}'", .{arg});
7024 }7030 }
...@@ -7057,6 +7063,7 @@ fn cmdFetch(...@@ -7057,6 +7063,7 @@ fn cmdFetch(
7057 .thread_pool = &thread_pool,7063 .thread_pool = &thread_pool,
7058 .global_cache = global_cache_directory,7064 .global_cache = global_cache_directory,
7059 .recursive = false,7065 .recursive = false,
7066 .debug_hash = debug_hash,
7060 .work_around_btrfs_bug = work_around_btrfs_bug,7067 .work_around_btrfs_bug = work_around_btrfs_bug,
7061 };7068 };
7062 defer job_queue.deinit();7069 defer job_queue.deinit();