authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-17 14:35:39-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-10-17 14:35:39-04:00
log3b21c15782a509d8f2939b0531573a9dfce415e1
tree0e1aa2e1016bf6a211b984f7a83c23ac49ea085a
parent5234b8be9c22d5735da29402620ab3a1d87aa7c6
parentba656e5c9f58e0440278918e5c6cdf0ac2fe673c
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #17562 from ziglang/fetch-symlink-normalize-sep

Package.Fetch: normalize path separators in symlinks

4 files changed, 64 insertions(+), 23 deletions(-)

lib/std/fs.zig+1-1
...@@ -1803,7 +1803,7 @@ pub const Dir = struct {...@@ -1803,7 +1803,7 @@ pub const Dir = struct {
1803 );1803 );
1804 switch (rc) {1804 switch (rc) {
1805 .SUCCESS => return result,1805 .SUCCESS => return result,
1806 .OBJECT_NAME_INVALID => unreachable,1806 .OBJECT_NAME_INVALID => return error.BadPathName,
1807 .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,1807 .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
1808 .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,1808 .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
1809 .NOT_A_DIRECTORY => return error.NotDir,1809 .NOT_A_DIRECTORY => return error.NotDir,
lib/std/mem.zig+5-6
...@@ -3810,12 +3810,11 @@ test "replace" {...@@ -3810,12 +3810,11 @@ test "replace" {
3810 try testing.expectEqualStrings(expected, output[0..expected.len]);3810 try testing.expectEqualStrings(expected, output[0..expected.len]);
3811}3811}
38123812
3813/// Replace all occurrences of `needle` with `replacement`.3813/// Replace all occurrences of `match` with `replacement`.
3814pub fn replaceScalar(comptime T: type, slice: []T, needle: T, replacement: T) void {3814pub fn replaceScalar(comptime T: type, slice: []T, match: T, replacement: T) void {
3815 for (slice, 0..) |e, i| {3815 for (slice) |*e| {
3816 if (e == needle) {3816 if (e.* == match)
3817 slice[i] = replacement;3817 e.* = replacement;
3818 }
3819 }3818 }
3820}3819}
38213820
src/Package/Fetch.zig+51-16
...@@ -81,6 +81,10 @@ pub const JobQueue = struct {...@@ -81,6 +81,10 @@ pub const JobQueue = struct {
81 wait_group: WaitGroup = .{},81 wait_group: WaitGroup = .{},
82 global_cache: Cache.Directory,82 global_cache: Cache.Directory,
83 recursive: bool,83 recursive: bool,
84 /// Dumps hash information to stdout which can be used to troubleshoot why
85 /// two hashes of the same package do not match.
86 /// If this is true, `recursive` must be false.
87 debug_hash: bool,
84 work_around_btrfs_bug: bool,88 work_around_btrfs_bug: bool,
8589
86 pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);90 pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);
...@@ -1315,7 +1319,7 @@ fn computeHash(...@@ -1315,7 +1319,7 @@ fn computeHash(
1315 const kind: HashedFile.Kind = switch (entry.kind) {1319 const kind: HashedFile.Kind = switch (entry.kind) {
1316 .directory => unreachable,1320 .directory => unreachable,
1317 .file => .file,1321 .file => .file,
1318 .sym_link => .sym_link,1322 .sym_link => .link,
1319 else => return f.fail(f.location_tok, try eb.printString(1323 else => return f.fail(f.location_tok, try eb.printString(
1320 "package contains '{s}' which has illegal file type '{s}'",1324 "package contains '{s}' which has illegal file type '{s}'",
1321 .{ entry.path, @tagName(entry.kind) },1325 .{ entry.path, @tagName(entry.kind) },
...@@ -1329,7 +1333,7 @@ fn computeHash(...@@ -1329,7 +1333,7 @@ fn computeHash(
1329 const hashed_file = try arena.create(HashedFile);1333 const hashed_file = try arena.create(HashedFile);
1330 hashed_file.* = .{1334 hashed_file.* = .{
1331 .fs_path = fs_path,1335 .fs_path = fs_path,
1332 .normalized_path = try normalizePath(arena, fs_path),1336 .normalized_path = try normalizePathAlloc(arena, fs_path),
1333 .kind = kind,1337 .kind = kind,
1334 .hash = undefined, // to be populated by the worker1338 .hash = undefined, // to be populated by the worker
1335 .failure = undefined, // to be populated by the worker1339 .failure = undefined, // to be populated by the worker
...@@ -1399,9 +1403,36 @@ fn computeHash(...@@ -1399,9 +1403,36 @@ fn computeHash(
1399 }1403 }
14001404
1401 if (any_failures) return error.FetchFailed;1405 if (any_failures) return error.FetchFailed;
1406
1407 if (f.job_queue.debug_hash) {
1408 assert(!f.job_queue.recursive);
1409 // Print something to stdout that can be text diffed to figure out why
1410 // the package hash is different.
1411 dumpHashInfo(all_files.items) catch |err| {
1412 std.debug.print("unable to write to stdout: {s}\n", .{@errorName(err)});
1413 std.process.exit(1);
1414 };
1415 }
1416
1402 return hasher.finalResult();1417 return hasher.finalResult();
1403}1418}
14041419
1420fn dumpHashInfo(all_files: []const *const HashedFile) !void {
1421 const stdout = std.io.getStdOut();
1422 var bw = std.io.bufferedWriter(stdout.writer());
1423 const w = bw.writer();
1424
1425 for (all_files) |hashed_file| {
1426 try w.print("{s}: {s}: {s}\n", .{
1427 @tagName(hashed_file.kind),
1428 std.fmt.fmtSliceHexLower(&hashed_file.hash),
1429 hashed_file.normalized_path,
1430 });
1431 }
1432
1433 try bw.flush();
1434}
1435
1405fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void {1436fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void {
1406 defer wg.finish();1437 defer wg.finish();
1407 hashed_file.failure = hashFileFallible(dir, hashed_file);1438 hashed_file.failure = hashFileFallible(dir, hashed_file);
...@@ -1427,8 +1458,14 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void...@@ -1427,8 +1458,14 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
1427 hasher.update(buf[0..bytes_read]);1458 hasher.update(buf[0..bytes_read]);
1428 }1459 }
1429 },1460 },
1430 .sym_link => {1461 .link => {
1431 const link_name = try dir.readLink(hashed_file.fs_path, &buf);1462 const link_name = try dir.readLink(hashed_file.fs_path, &buf);
1463 if (fs.path.sep != canonical_sep) {
1464 // Package hashes are intended to be consistent across
1465 // platforms which means we must normalize path separators
1466 // inside symlinks.
1467 normalizePath(link_name);
1468 }
1432 hasher.update(link_name);1469 hasher.update(link_name);
1433 },1470 },
1434 }1471 }
...@@ -1474,7 +1511,7 @@ const HashedFile = struct {...@@ -1474,7 +1511,7 @@ const HashedFile = struct {
1474 fs.File.StatError ||1511 fs.File.StatError ||
1475 fs.Dir.ReadLinkError;1512 fs.Dir.ReadLinkError;
14761513
1477 const Kind = enum { file, sym_link };1514 const Kind = enum { file, link };
14781515
1479 fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool {1516 fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool {
1480 _ = context;1517 _ = context;
...@@ -1484,22 +1521,20 @@ const HashedFile = struct {...@@ -1484,22 +1521,20 @@ const HashedFile = struct {
14841521
1485/// Make a file system path identical independently of operating system path inconsistencies.1522/// Make a file system path identical independently of operating system path inconsistencies.
1486/// This converts backslashes into forward slashes.1523/// This converts backslashes into forward slashes.
1487fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {1524fn normalizePathAlloc(arena: Allocator, fs_path: []const u8) ![]const u8 {
1488 const canonical_sep = '/';1525 if (fs.path.sep == canonical_sep) return fs_path;
1489
1490 if (fs.path.sep == canonical_sep)
1491 return fs_path;
1492
1493 const normalized = try arena.dupe(u8, fs_path);1526 const normalized = try arena.dupe(u8, fs_path);
1494 for (normalized) |*byte| {1527 normalizePath(normalized);
1495 switch (byte.*) {
1496 fs.path.sep => byte.* = canonical_sep,
1497 else => continue,
1498 }
1499 }
1500 return normalized;1528 return normalized;
1501}1529}
15021530
1531const canonical_sep = fs.path.sep_posix;
1532
1533fn normalizePath(bytes: []u8) void {
1534 assert(fs.path.sep != canonical_sep);
1535 std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep);
1536}
1537
1503const Filter = struct {1538const Filter = struct {
1504 include_paths: std.StringArrayHashMapUnmanaged(void) = .{},1539 include_paths: std.StringArrayHashMapUnmanaged(void) = .{},
15051540
src/main.zig+7
...@@ -5143,6 +5143,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi...@@ -5143,6 +5143,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
5143 .thread_pool = &thread_pool,5143 .thread_pool = &thread_pool,
5144 .global_cache = global_cache_directory,5144 .global_cache = global_cache_directory,
5145 .recursive = true,5145 .recursive = true,
5146 .debug_hash = false,
5146 .work_around_btrfs_bug = work_around_btrfs_bug,5147 .work_around_btrfs_bug = work_around_btrfs_bug,
5147 };5148 };
5148 defer job_queue.deinit();5149 defer job_queue.deinit();
...@@ -6991,6 +6992,7 @@ pub const usage_fetch =...@@ -6991,6 +6992,7 @@ pub const usage_fetch =
6991 \\Options:6992 \\Options:
6992 \\ -h, --help Print this help and exit6993 \\ -h, --help Print this help and exit
6993 \\ --global-cache-dir [path] Override path to global Zig cache directory6994 \\ --global-cache-dir [path] Override path to global Zig cache directory
6995 \\ --debug-hash Print verbose hash information to stdout
6994 \\6996 \\
6995;6997;
69966998
...@@ -7004,6 +7006,7 @@ fn cmdFetch(...@@ -7004,6 +7006,7 @@ fn cmdFetch(
7004 std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND");7006 std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND");
7005 var opt_path_or_url: ?[]const u8 = null;7007 var opt_path_or_url: ?[]const u8 = null;
7006 var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");7008 var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");
7009 var debug_hash: bool = false;
70077010
7008 {7011 {
7009 var i: usize = 0;7012 var i: usize = 0;
...@@ -7019,6 +7022,9 @@ fn cmdFetch(...@@ -7019,6 +7022,9 @@ fn cmdFetch(
7019 i += 1;7022 i += 1;
7020 override_global_cache_dir = args[i];7023 override_global_cache_dir = args[i];
7021 continue;7024 continue;
7025 } else if (mem.eql(u8, arg, "--debug-hash")) {
7026 debug_hash = true;
7027 continue;
7022 } else {7028 } else {
7023 fatal("unrecognized parameter: '{s}'", .{arg});7029 fatal("unrecognized parameter: '{s}'", .{arg});
7024 }7030 }
...@@ -7057,6 +7063,7 @@ fn cmdFetch(...@@ -7057,6 +7063,7 @@ fn cmdFetch(
7057 .thread_pool = &thread_pool,7063 .thread_pool = &thread_pool,
7058 .global_cache = global_cache_directory,7064 .global_cache = global_cache_directory,
7059 .recursive = false,7065 .recursive = false,
7066 .debug_hash = debug_hash,
7060 .work_around_btrfs_bug = work_around_btrfs_bug,7067 .work_around_btrfs_bug = work_around_btrfs_bug,
7061 };7068 };
7062 defer job_queue.deinit();7069 defer job_queue.deinit();