authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-17 14:35:39-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-10-17 14:35:39-04:00
log3b21c15782a509d8f2939b0531573a9dfce415e1
tree0e1aa2e1016bf6a211b984f7a83c23ac49ea085a
parent5234b8be9c22d5735da29402620ab3a1d87aa7c6
parentba656e5c9f58e0440278918e5c6cdf0ac2fe673c
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #17562 from ziglang/fetch-symlink-normalize-sep

Package.Fetch: normalize path separators in symlinks

4 files changed, 64 insertions(+), 23 deletions(-)

lib/std/fs.zig+1-1
......@@ -1803,7 +1803,7 @@ pub const Dir = struct {
18031803 );
18041804 switch (rc) {
18051805 .SUCCESS => return result,
1806 .OBJECT_NAME_INVALID => unreachable,
1806 .OBJECT_NAME_INVALID => return error.BadPathName,
18071807 .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
18081808 .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
18091809 .NOT_A_DIRECTORY => return error.NotDir,
lib/std/mem.zig+5-6
......@@ -3810,12 +3810,11 @@ test "replace" {
38103810 try testing.expectEqualStrings(expected, output[0..expected.len]);
38113811}
38123812
3813/// Replace all occurrences of `needle` with `replacement`.
3814pub fn replaceScalar(comptime T: type, slice: []T, needle: T, replacement: T) void {
3815 for (slice, 0..) |e, i| {
3816 if (e == needle) {
3817 slice[i] = replacement;
3818 }
3813/// Replace all occurrences of `match` with `replacement`.
3814pub fn replaceScalar(comptime T: type, slice: []T, match: T, replacement: T) void {
3815 for (slice) |*e| {
3816 if (e.* == match)
3817 e.* = replacement;
38193818 }
38203819}
38213820
src/Package/Fetch.zig+51-16
......@@ -81,6 +81,10 @@ pub const JobQueue = struct {
8181 wait_group: WaitGroup = .{},
8282 global_cache: Cache.Directory,
8383 recursive: bool,
84 /// Dumps hash information to stdout which can be used to troubleshoot why
85 /// two hashes of the same package do not match.
86 /// If this is true, `recursive` must be false.
87 debug_hash: bool,
8488 work_around_btrfs_bug: bool,
8589
8690 pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);
......@@ -1315,7 +1319,7 @@ fn computeHash(
13151319 const kind: HashedFile.Kind = switch (entry.kind) {
13161320 .directory => unreachable,
13171321 .file => .file,
1318 .sym_link => .sym_link,
1322 .sym_link => .link,
13191323 else => return f.fail(f.location_tok, try eb.printString(
13201324 "package contains '{s}' which has illegal file type '{s}'",
13211325 .{ entry.path, @tagName(entry.kind) },
......@@ -1329,7 +1333,7 @@ fn computeHash(
13291333 const hashed_file = try arena.create(HashedFile);
13301334 hashed_file.* = .{
13311335 .fs_path = fs_path,
1332 .normalized_path = try normalizePath(arena, fs_path),
1336 .normalized_path = try normalizePathAlloc(arena, fs_path),
13331337 .kind = kind,
13341338 .hash = undefined, // to be populated by the worker
13351339 .failure = undefined, // to be populated by the worker
......@@ -1399,9 +1403,36 @@ fn computeHash(
13991403 }
14001404
14011405 if (any_failures) return error.FetchFailed;
1406
1407 if (f.job_queue.debug_hash) {
1408 assert(!f.job_queue.recursive);
1409 // Print something to stdout that can be text diffed to figure out why
1410 // the package hash is different.
1411 dumpHashInfo(all_files.items) catch |err| {
1412 std.debug.print("unable to write to stdout: {s}\n", .{@errorName(err)});
1413 std.process.exit(1);
1414 };
1415 }
1416
14021417 return hasher.finalResult();
14031418}
14041419
1420fn dumpHashInfo(all_files: []const *const HashedFile) !void {
1421 const stdout = std.io.getStdOut();
1422 var bw = std.io.bufferedWriter(stdout.writer());
1423 const w = bw.writer();
1424
1425 for (all_files) |hashed_file| {
1426 try w.print("{s}: {s}: {s}\n", .{
1427 @tagName(hashed_file.kind),
1428 std.fmt.fmtSliceHexLower(&hashed_file.hash),
1429 hashed_file.normalized_path,
1430 });
1431 }
1432
1433 try bw.flush();
1434}
1435
14051436fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void {
14061437 defer wg.finish();
14071438 hashed_file.failure = hashFileFallible(dir, hashed_file);
......@@ -1427,8 +1458,14 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
14271458 hasher.update(buf[0..bytes_read]);
14281459 }
14291460 },
1430 .sym_link => {
1461 .link => {
14311462 const link_name = try dir.readLink(hashed_file.fs_path, &buf);
1463 if (fs.path.sep != canonical_sep) {
1464 // Package hashes are intended to be consistent across
1465 // platforms which means we must normalize path separators
1466 // inside symlinks.
1467 normalizePath(link_name);
1468 }
14321469 hasher.update(link_name);
14331470 },
14341471 }
......@@ -1474,7 +1511,7 @@ const HashedFile = struct {
14741511 fs.File.StatError ||
14751512 fs.Dir.ReadLinkError;
14761513
1477 const Kind = enum { file, sym_link };
1514 const Kind = enum { file, link };
14781515
14791516 fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool {
14801517 _ = context;
......@@ -1484,22 +1521,20 @@ const HashedFile = struct {
14841521
14851522/// Make a file system path identical independently of operating system path inconsistencies.
14861523/// This converts backslashes into forward slashes.
1487fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
1488 const canonical_sep = '/';
1489
1490 if (fs.path.sep == canonical_sep)
1491 return fs_path;
1492
1524fn normalizePathAlloc(arena: Allocator, fs_path: []const u8) ![]const u8 {
1525 if (fs.path.sep == canonical_sep) return fs_path;
14931526 const normalized = try arena.dupe(u8, fs_path);
1494 for (normalized) |*byte| {
1495 switch (byte.*) {
1496 fs.path.sep => byte.* = canonical_sep,
1497 else => continue,
1498 }
1499 }
1527 normalizePath(normalized);
15001528 return normalized;
15011529}
15021530
1531const canonical_sep = fs.path.sep_posix;
1532
1533fn normalizePath(bytes: []u8) void {
1534 assert(fs.path.sep != canonical_sep);
1535 std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep);
1536}
1537
15031538const Filter = struct {
15041539 include_paths: std.StringArrayHashMapUnmanaged(void) = .{},
15051540
src/main.zig+7
......@@ -5143,6 +5143,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
51435143 .thread_pool = &thread_pool,
51445144 .global_cache = global_cache_directory,
51455145 .recursive = true,
5146 .debug_hash = false,
51465147 .work_around_btrfs_bug = work_around_btrfs_bug,
51475148 };
51485149 defer job_queue.deinit();
......@@ -6991,6 +6992,7 @@ pub const usage_fetch =
69916992 \\Options:
69926993 \\ -h, --help Print this help and exit
69936994 \\ --global-cache-dir [path] Override path to global Zig cache directory
6995 \\ --debug-hash Print verbose hash information to stdout
69946996 \\
69956997;
69966998
......@@ -7004,6 +7006,7 @@ fn cmdFetch(
70047006 std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND");
70057007 var opt_path_or_url: ?[]const u8 = null;
70067008 var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");
7009 var debug_hash: bool = false;
70077010
70087011 {
70097012 var i: usize = 0;
......@@ -7019,6 +7022,9 @@ fn cmdFetch(
70197022 i += 1;
70207023 override_global_cache_dir = args[i];
70217024 continue;
7025 } else if (mem.eql(u8, arg, "--debug-hash")) {
7026 debug_hash = true;
7027 continue;
70227028 } else {
70237029 fatal("unrecognized parameter: '{s}'", .{arg});
70247030 }
......@@ -7057,6 +7063,7 @@ fn cmdFetch(
70577063 .thread_pool = &thread_pool,
70587064 .global_cache = global_cache_directory,
70597065 .recursive = false,
7066 .debug_hash = debug_hash,
70607067 .work_around_btrfs_bug = work_around_btrfs_bug,
70617068 };
70627069 defer job_queue.deinit();