authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-04-03 21:24:30-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-04-03 21:24:30-07:00
loge5d900268a1d969dcaf4f2d657e96a5a5f217e7b
tree54eb6ff054d660c6304f9eb1aa94ac64b67a25a5
parentb88ae8dbd84886d3b9b26509034720f755a0e28a
parenta60b7af2c19ca14609bd052916299ffb64063856
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #19111 from ianic/no_strip_components

package manager: handle archives without leading root folder

2 files changed, 200 insertions(+), 60 deletions(-)

lib/std/tar.zig+102-8
...@@ -29,6 +29,9 @@ pub const Diagnostics = struct {...@@ -29,6 +29,9 @@ pub const Diagnostics = struct {
29 allocator: std.mem.Allocator,29 allocator: std.mem.Allocator,
30 errors: std.ArrayListUnmanaged(Error) = .{},30 errors: std.ArrayListUnmanaged(Error) = .{},
3131
32 root_entries: usize = 0,
33 root_dir: ?[]const u8 = null,
34
32 pub const Error = union(enum) {35 pub const Error = union(enum) {
33 unable_to_create_sym_link: struct {36 unable_to_create_sym_link: struct {
34 code: anyerror,37 code: anyerror,
...@@ -45,6 +48,45 @@ pub const Diagnostics = struct {...@@ -45,6 +48,45 @@ pub const Diagnostics = struct {
45 },48 },
46 };49 };
4750
51 fn findRoot(d: *Diagnostics, path: []const u8, kind: FileKind) !void {
52 if (rootDir(path)) |root_dir| {
53 d.root_entries += 1;
54 if (kind == .directory and d.root_entries == 1) {
55 d.root_dir = try d.allocator.dupe(u8, root_dir);
56 return;
57 }
58 if (d.root_dir) |r| {
59 d.allocator.free(r);
60 d.root_dir = null;
61 }
62 }
63 }
64
65 // If path is package root returns root_dir name, otherwise null.
66 fn rootDir(path: []const u8) ?[]const u8 {
67 if (path.len == 0) return null;
68
69 const start_index: usize = if (path[0] == '/') 1 else 0;
70 const end_index: usize = if (path[path.len - 1] == '/') path.len - 1 else path.len;
71 const buf = path[start_index..end_index];
72 return if (std.mem.indexOfScalarPos(u8, buf, 0, '/') == null)
73 buf
74 else
75 null;
76 }
77
78 test rootDir {
79 const expectEqualStrings = testing.expectEqualStrings;
80 const expect = testing.expect;
81
82 try expectEqualStrings("a", rootDir("a").?);
83 try expectEqualStrings("b", rootDir("b").?);
84 try expectEqualStrings("c", rootDir("/c").?);
85 try expectEqualStrings("d", rootDir("/d/").?);
86 try expect(rootDir("a/b") == null);
87 try expect(rootDir("") == null);
88 }
89
48 pub fn deinit(d: *Diagnostics) void {90 pub fn deinit(d: *Diagnostics) void {
49 for (d.errors.items) |item| {91 for (d.errors.items) |item| {
50 switch (item) {92 switch (item) {
...@@ -61,6 +103,10 @@ pub const Diagnostics = struct {...@@ -61,6 +103,10 @@ pub const Diagnostics = struct {
61 }103 }
62 }104 }
63 d.errors.deinit(d.allocator);105 d.errors.deinit(d.allocator);
106 if (d.root_dir) |r| {
107 d.allocator.free(r);
108 d.root_dir = null;
109 }
64 d.* = undefined;110 d.* = undefined;
65 }111 }
66};112};
...@@ -580,19 +626,21 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: PipeOptions)...@@ -580,19 +626,21 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: PipeOptions)
580 .link_name_buffer = &link_name_buffer,626 .link_name_buffer = &link_name_buffer,
581 .diagnostics = options.diagnostics,627 .diagnostics = options.diagnostics,
582 });628 });
629
583 while (try iter.next()) |file| {630 while (try iter.next()) |file| {
631 const file_name = stripComponents(file.name, options.strip_components);
632 if (options.diagnostics) |d| {
633 try d.findRoot(file_name, file.kind);
634 }
635
584 switch (file.kind) {636 switch (file.kind) {
585 .directory => {637 .directory => {
586 const file_name = stripComponents(file.name, options.strip_components);
587 if (file_name.len != 0 and !options.exclude_empty_directories) {638 if (file_name.len != 0 and !options.exclude_empty_directories) {
588 try dir.makePath(file_name);639 try dir.makePath(file_name);
589 }640 }
590 },641 },
591 .file => {642 .file => {
592 if (file.size == 0 and file.name.len == 0) return;
593 const file_name = stripComponents(file.name, options.strip_components);
594 if (file_name.len == 0) return error.BadFileName;643 if (file_name.len == 0) return error.BadFileName;
595
596 if (createDirAndFile(dir, file_name, fileMode(file.mode, options))) |fs_file| {644 if (createDirAndFile(dir, file_name, fileMode(file.mode, options))) |fs_file| {
597 defer fs_file.close();645 defer fs_file.close();
598 try file.writeAll(fs_file);646 try file.writeAll(fs_file);
...@@ -605,12 +653,8 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: PipeOptions)...@@ -605,12 +653,8 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: PipeOptions)
605 }653 }
606 },654 },
607 .sym_link => {655 .sym_link => {
608 // The file system path of the symbolic link.
609 const file_name = stripComponents(file.name, options.strip_components);
610 if (file_name.len == 0) return error.BadFileName;656 if (file_name.len == 0) return error.BadFileName;
611 // The data inside the symbolic link.
612 const link_name = file.link_name;657 const link_name = file.link_name;
613
614 createDirAndSymlink(dir, link_name, file_name) catch |err| {658 createDirAndSymlink(dir, link_name, file_name) catch |err| {
615 const d = options.diagnostics orelse return error.UnableToCreateSymLink;659 const d = options.diagnostics orelse return error.UnableToCreateSymLink;
616 try d.errors.append(d.allocator, .{ .unable_to_create_sym_link = .{660 try d.errors.append(d.allocator, .{ .unable_to_create_sym_link = .{
...@@ -799,6 +843,7 @@ test PaxIterator {...@@ -799,6 +843,7 @@ test PaxIterator {
799843
800test {844test {
801 _ = @import("tar/test.zig");845 _ = @import("tar/test.zig");
846 _ = Diagnostics;
802}847}
803848
804test "header parse size" {849test "header parse size" {
...@@ -993,6 +1038,55 @@ test pipeToFileSystem {...@@ -993,6 +1038,55 @@ test pipeToFileSystem {
993 );1038 );
994}1039}
9951040
1041test "pipeToFileSystem root_dir" {
1042 const data = @embedFile("tar/testdata/example.tar");
1043 var fbs = std.io.fixedBufferStream(data);
1044 const reader = fbs.reader();
1045
1046 // with strip_components = 1
1047 {
1048 var tmp = testing.tmpDir(.{ .no_follow = true });
1049 defer tmp.cleanup();
1050 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1051 defer diagnostics.deinit();
1052
1053 pipeToFileSystem(tmp.dir, reader, .{
1054 .strip_components = 1,
1055 .diagnostics = &diagnostics,
1056 }) catch |err| {
1057 // Skip on platform which don't support symlinks
1058 if (err == error.UnableToCreateSymLink) return error.SkipZigTest;
1059 return err;
1060 };
1061
1062 // there is no root_dir
1063 try testing.expect(diagnostics.root_dir == null);
1064 try testing.expectEqual(3, diagnostics.root_entries);
1065 }
1066
1067 // with strip_components = 0
1068 {
1069 fbs.reset();
1070 var tmp = testing.tmpDir(.{ .no_follow = true });
1071 defer tmp.cleanup();
1072 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1073 defer diagnostics.deinit();
1074
1075 pipeToFileSystem(tmp.dir, reader, .{
1076 .strip_components = 0,
1077 .diagnostics = &diagnostics,
1078 }) catch |err| {
1079 // Skip on platform which don't support symlinks
1080 if (err == error.UnableToCreateSymLink) return error.SkipZigTest;
1081 return err;
1082 };
1083
1084 // root_dir found
1085 try testing.expectEqualStrings("example", diagnostics.root_dir.?);
1086 try testing.expectEqual(1, diagnostics.root_entries);
1087 }
1088}
1089
996fn normalizePath(bytes: []u8) []u8 {1090fn normalizePath(bytes: []u8) []u8 {
997 const canonical_sep = std.fs.path.sep_posix;1091 const canonical_sep = std.fs.path.sep_posix;
998 if (std.fs.path.sep == canonical_sep) return bytes;1092 if (std.fs.path.sep == canonical_sep) return bytes;
src/Package/Fetch.zig+98-52
...@@ -441,7 +441,7 @@ fn runResource(...@@ -441,7 +441,7 @@ fn runResource(
441 const rand_int = std.crypto.random.int(u64);441 const rand_int = std.crypto.random.int(u64);
442 const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);442 const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);
443443
444 {444 const package_sub_path = blk: {
445 const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path});445 const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path});
446 var tmp_directory: Cache.Directory = .{446 var tmp_directory: Cache.Directory = .{
447 .path = tmp_directory_path,447 .path = tmp_directory_path,
...@@ -461,37 +461,50 @@ fn runResource(...@@ -461,37 +461,50 @@ fn runResource(
461 };461 };
462 defer tmp_directory.handle.close();462 defer tmp_directory.handle.close();
463463
464 try unpackResource(f, resource, uri_path, tmp_directory);464 // Unpack resource into tmp_directory. A non-null return value means
465 // that the package contents are inside a `pkg_dir` sub-directory.
466 const pkg_dir = try unpackResource(f, resource, uri_path, tmp_directory);
467
468 var pkg_path: Cache.Path = .{
469 .root_dir = tmp_directory,
470 .sub_path = if (pkg_dir) |pkg_dir_name| pkg_dir_name else "",
471 };
472
473 // Apply btrfs workaround if needed. Reopen tmp_directory.
474 if (native_os == .linux and f.job_queue.work_around_btrfs_bug) {
475 // https://github.com/ziglang/zig/issues/17095
476 pkg_path.root_dir.handle.close();
477 pkg_path.root_dir.handle = cache_root.handle.makeOpenPath(tmp_dir_sub_path, .{
478 .iterate = true,
479 }) catch @panic("btrfs workaround failed");
480 }
465481
466 // Load, parse, and validate the unpacked build.zig.zon file. It is allowed482 // Load, parse, and validate the unpacked build.zig.zon file. It is allowed
467 // for the file to be missing, in which case this fetched package is483 // for the file to be missing, in which case this fetched package is
468 // considered to be a "naked" package.484 // considered to be a "naked" package.
469 try loadManifest(f, .{ .root_dir = tmp_directory });485 try loadManifest(f, pkg_path);
470
471 // Apply the manifest's inclusion rules to the temporary directory by
472 // deleting excluded files. If any error occurred for files that were
473 // ultimately excluded, those errors should be ignored, such as failure to
474 // create symlinks that weren't supposed to be included anyway.
475
476 // Empty directories have already been omitted by `unpackResource`.
477486
478 const filter: Filter = .{487 const filter: Filter = .{
479 .include_paths = if (f.manifest) |m| m.paths else .{},488 .include_paths = if (f.manifest) |m| m.paths else .{},
480 };489 };
481490
491 // TODO:
492 // If any error occurred for files that were ultimately excluded, those
493 // errors should be ignored, such as failure to create symlinks that
494 // weren't supposed to be included anyway.
495
496 // Apply the manifest's inclusion rules to the temporary directory by
497 // deleting excluded files.
498 // Empty directories have already been omitted by `unpackResource`.
482 // Compute the package hash based on the remaining files in the temporary499 // Compute the package hash based on the remaining files in the temporary
483 // directory.500 // directory.
501 f.actual_hash = try computeHash(f, pkg_path, filter);
484502
485 if (native_os == .linux and f.job_queue.work_around_btrfs_bug) {503 break :blk if (pkg_dir) |pkg_dir_name|
486 // https://github.com/ziglang/zig/issues/17095504 try fs.path.join(arena, &.{ tmp_dir_sub_path, pkg_dir_name })
487 tmp_directory.handle.close();505 else
488 tmp_directory.handle = cache_root.handle.makeOpenPath(tmp_dir_sub_path, .{506 tmp_dir_sub_path;
489 .iterate = true,507 };
490 }) catch @panic("btrfs workaround failed");
491 }
492
493 f.actual_hash = try computeHash(f, tmp_directory, filter);
494 }
495508
496 // Rename the temporary directory into the global zig package cache509 // Rename the temporary directory into the global zig package cache
497 // directory. If the hash already exists, delete the temporary directory510 // directory. If the hash already exists, delete the temporary directory
...@@ -503,7 +516,7 @@ fn runResource(...@@ -503,7 +516,7 @@ fn runResource(
503 .root_dir = cache_root,516 .root_dir = cache_root,
504 .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)),517 .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)),
505 };518 };
506 renameTmpIntoCache(cache_root.handle, tmp_dir_sub_path, f.package_root.sub_path) catch |err| {519 renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| {
507 const src = try cache_root.join(arena, &.{tmp_dir_sub_path});520 const src = try cache_root.join(arena, &.{tmp_dir_sub_path});
508 const dest = try cache_root.join(arena, &.{f.package_root.sub_path});521 const dest = try cache_root.join(arena, &.{f.package_root.sub_path});
509 try eb.addRootErrorMessage(.{ .msg = try eb.printString(522 try eb.addRootErrorMessage(.{ .msg = try eb.printString(
...@@ -512,6 +525,10 @@ fn runResource(...@@ -512,6 +525,10 @@ fn runResource(
512 ) });525 ) });
513 return error.FetchFailed;526 return error.FetchFailed;
514 };527 };
528 // Remove temporary directory root if not already renamed to global cache.
529 if (!std.mem.eql(u8, package_sub_path, tmp_dir_sub_path)) {
530 cache_root.handle.deleteDir(tmp_dir_sub_path) catch {};
531 }
515532
516 // Validate the computed hash against the expected hash. If invalid, this533 // Validate the computed hash against the expected hash. If invalid, this
517 // job is done.534 // job is done.
...@@ -867,9 +884,9 @@ const FileType = enum {...@@ -867,9 +884,9 @@ const FileType = enum {
867 try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\""));884 try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\""));
868 try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\""));885 try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\""));
869 try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz"));886 try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz"));
887 try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\""));
870888
871 try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null);889 try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null);
872 try std.testing.expect(fromContentDisposition("attachment; FileName=\"stuff.tar\"") == null);
873 try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null);890 try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null);
874 try std.testing.expect(fromContentDisposition("attachment; size=42") == null);891 try std.testing.expect(fromContentDisposition("attachment; size=42") == null);
875 try std.testing.expect(fromContentDisposition("inline; size=42") == null);892 try std.testing.expect(fromContentDisposition("inline; size=42") == null);
...@@ -1027,12 +1044,16 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re...@@ -1027,12 +1044,16 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re
1027 ));1044 ));
1028}1045}
10291046
1047/// A `null` return value indicates the `tmp_directory` is populated directly
1048/// with the package contents.
1049/// A non-null return value means that the package contents are inside a
1050/// sub-directory indicated by the named path.
1030fn unpackResource(1051fn unpackResource(
1031 f: *Fetch,1052 f: *Fetch,
1032 resource: *Resource,1053 resource: *Resource,
1033 uri_path: []const u8,1054 uri_path: []const u8,
1034 tmp_directory: Cache.Directory,1055 tmp_directory: Cache.Directory,
1035) RunError!void {1056) RunError!?[]const u8 {
1036 const eb = &f.error_bundle;1057 const eb = &f.error_bundle;
1037 const file_type = switch (resource.*) {1058 const file_type = switch (resource.*) {
1038 .file => FileType.fromPath(uri_path) orelse1059 .file => FileType.fromPath(uri_path) orelse
...@@ -1093,21 +1114,24 @@ fn unpackResource(...@@ -1093,21 +1114,24 @@ fn unpackResource(
10931114
1094 .git => .git_pack,1115 .git => .git_pack,
10951116
1096 .dir => |dir| return f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| {1117 .dir => |dir| {
1097 return f.fail(f.location_tok, try eb.printString(1118 f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| {
1098 "unable to copy directory '{s}': {s}",1119 return f.fail(f.location_tok, try eb.printString(
1099 .{ uri_path, @errorName(err) },1120 "unable to copy directory '{s}': {s}",
1100 ));1121 .{ uri_path, @errorName(err) },
1122 ));
1123 };
1124 return null;
1101 },1125 },
1102 };1126 };
11031127
1104 switch (file_type) {1128 switch (file_type) {
1105 .tar => try unpackTarball(f, tmp_directory.handle, resource.reader()),1129 .tar => return try unpackTarball(f, tmp_directory.handle, resource.reader()),
1106 .@"tar.gz" => {1130 .@"tar.gz" => {
1107 const reader = resource.reader();1131 const reader = resource.reader();
1108 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);1132 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1109 var dcp = std.compress.gzip.decompressor(br.reader());1133 var dcp = std.compress.gzip.decompressor(br.reader());
1110 try unpackTarball(f, tmp_directory.handle, dcp.reader());1134 return try unpackTarball(f, tmp_directory.handle, dcp.reader());
1111 },1135 },
1112 .@"tar.xz" => {1136 .@"tar.xz" => {
1113 const gpa = f.arena.child_allocator;1137 const gpa = f.arena.child_allocator;
...@@ -1120,7 +1144,7 @@ fn unpackResource(...@@ -1120,7 +1144,7 @@ fn unpackResource(
1120 ));1144 ));
1121 };1145 };
1122 defer dcp.deinit();1146 defer dcp.deinit();
1123 try unpackTarball(f, tmp_directory.handle, dcp.reader());1147 return try unpackTarball(f, tmp_directory.handle, dcp.reader());
1124 },1148 },
1125 .@"tar.zst" => {1149 .@"tar.zst" => {
1126 const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len;1150 const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len;
...@@ -1130,21 +1154,25 @@ fn unpackResource(...@@ -1130,21 +1154,25 @@ fn unpackResource(
1130 var dcp = std.compress.zstd.decompressor(br.reader(), .{1154 var dcp = std.compress.zstd.decompressor(br.reader(), .{
1131 .window_buffer = window_buffer,1155 .window_buffer = window_buffer,
1132 });1156 });
1133 return unpackTarball(f, tmp_directory.handle, dcp.reader());1157 return try unpackTarball(f, tmp_directory.handle, dcp.reader());
1134 },1158 },
1135 .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) {1159 .git_pack => {
1136 error.FetchFailed => return error.FetchFailed,1160 unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) {
1137 error.OutOfMemory => return error.OutOfMemory,1161 error.FetchFailed => return error.FetchFailed,
1138 else => |e| return f.fail(f.location_tok, try eb.printString(1162 error.OutOfMemory => return error.OutOfMemory,
1139 "unable to unpack git files: {s}",1163 else => |e| return f.fail(f.location_tok, try eb.printString(
1140 .{@errorName(e)},1164 "unable to unpack git files: {s}",
1141 )),1165 .{@errorName(e)},
1166 )),
1167 };
1168 return null;
1142 },1169 },
1143 }1170 }
1144}1171}
11451172
1146fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {1173fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!?[]const u8 {
1147 const eb = &f.error_bundle;1174 const eb = &f.error_bundle;
1175 const arena = f.arena.allocator();
1148 const gpa = f.arena.child_allocator;1176 const gpa = f.arena.child_allocator;
11491177
1150 var diagnostics: std.tar.Diagnostics = .{ .allocator = gpa };1178 var diagnostics: std.tar.Diagnostics = .{ .allocator = gpa };
...@@ -1152,7 +1180,7 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {...@@ -1152,7 +1180,7 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
11521180
1153 std.tar.pipeToFileSystem(out_dir, reader, .{1181 std.tar.pipeToFileSystem(out_dir, reader, .{
1154 .diagnostics = &diagnostics,1182 .diagnostics = &diagnostics,
1155 .strip_components = 1,1183 .strip_components = 0,
1156 // https://github.com/ziglang/zig/issues/174631184 // https://github.com/ziglang/zig/issues/17463
1157 .mode_mode = .ignore,1185 .mode_mode = .ignore,
1158 .exclude_empty_directories = true,1186 .exclude_empty_directories = true,
...@@ -1196,6 +1224,11 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {...@@ -1196,6 +1224,11 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
1196 }1224 }
1197 return error.FetchFailed;1225 return error.FetchFailed;
1198 }1226 }
1227
1228 return if (diagnostics.root_dir) |root_dir|
1229 return try arena.dupe(u8, root_dir)
1230 else
1231 null;
1199}1232}
12001233
1201fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!void {1234fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!void {
...@@ -1341,7 +1374,7 @@ pub fn renameTmpIntoCache(...@@ -1341,7 +1374,7 @@ pub fn renameTmpIntoCache(
1341/// function.1374/// function.
1342fn computeHash(1375fn computeHash(
1343 f: *Fetch,1376 f: *Fetch,
1344 tmp_directory: Cache.Directory,1377 pkg_path: Cache.Path,
1345 filter: Filter,1378 filter: Filter,
1346) RunError!Manifest.Digest {1379) RunError!Manifest.Digest {
1347 // All the path name strings need to be in memory for sorting.1380 // All the path name strings need to be in memory for sorting.
...@@ -1349,6 +1382,7 @@ fn computeHash(...@@ -1349,6 +1382,7 @@ fn computeHash(
1349 const gpa = f.arena.child_allocator;1382 const gpa = f.arena.child_allocator;
1350 const eb = &f.error_bundle;1383 const eb = &f.error_bundle;
1351 const thread_pool = f.job_queue.thread_pool;1384 const thread_pool = f.job_queue.thread_pool;
1385 const root_dir = pkg_path.root_dir.handle;
13521386
1353 // Collect all files, recursively, then sort.1387 // Collect all files, recursively, then sort.
1354 var all_files = std.ArrayList(*HashedFile).init(gpa);1388 var all_files = std.ArrayList(*HashedFile).init(gpa);
...@@ -1362,7 +1396,7 @@ fn computeHash(...@@ -1362,7 +1396,7 @@ fn computeHash(
1362 var sus_dirs: std.StringArrayHashMapUnmanaged(void) = .{};1396 var sus_dirs: std.StringArrayHashMapUnmanaged(void) = .{};
1363 defer sus_dirs.deinit(gpa);1397 defer sus_dirs.deinit(gpa);
13641398
1365 var walker = try tmp_directory.handle.walk(gpa);1399 var walker = try root_dir.walk(gpa);
1366 defer walker.deinit();1400 defer walker.deinit();
13671401
1368 {1402 {
...@@ -1376,13 +1410,14 @@ fn computeHash(...@@ -1376,13 +1410,14 @@ fn computeHash(
1376 while (walker.next() catch |err| {1410 while (walker.next() catch |err| {
1377 try eb.addRootErrorMessage(.{ .msg = try eb.printString(1411 try eb.addRootErrorMessage(.{ .msg = try eb.printString(
1378 "unable to walk temporary directory '{}': {s}",1412 "unable to walk temporary directory '{}': {s}",
1379 .{ tmp_directory, @errorName(err) },1413 .{ pkg_path, @errorName(err) },
1380 ) });1414 ) });
1381 return error.FetchFailed;1415 return error.FetchFailed;
1382 }) |entry| {1416 }) |entry| {
1383 if (entry.kind == .directory) continue;1417 if (entry.kind == .directory) continue;
13841418
1385 if (!filter.includePath(entry.path)) {1419 const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path);
1420 if (!filter.includePath(entry_pkg_path)) {
1386 // Delete instead of including in hash calculation.1421 // Delete instead of including in hash calculation.
1387 const fs_path = try arena.dupe(u8, entry.path);1422 const fs_path = try arena.dupe(u8, entry.path);
13881423
...@@ -1397,7 +1432,7 @@ fn computeHash(...@@ -1397,7 +1432,7 @@ fn computeHash(
1397 };1432 };
1398 wait_group.start();1433 wait_group.start();
1399 try thread_pool.spawn(workerDeleteFile, .{1434 try thread_pool.spawn(workerDeleteFile, .{
1400 tmp_directory.handle, deleted_file, &wait_group,1435 root_dir, deleted_file, &wait_group,
1401 });1436 });
1402 try deleted_files.append(deleted_file);1437 try deleted_files.append(deleted_file);
1403 continue;1438 continue;
...@@ -1420,14 +1455,14 @@ fn computeHash(...@@ -1420,14 +1455,14 @@ fn computeHash(
1420 const hashed_file = try arena.create(HashedFile);1455 const hashed_file = try arena.create(HashedFile);
1421 hashed_file.* = .{1456 hashed_file.* = .{
1422 .fs_path = fs_path,1457 .fs_path = fs_path,
1423 .normalized_path = try normalizePathAlloc(arena, fs_path),1458 .normalized_path = try normalizePathAlloc(arena, entry_pkg_path),
1424 .kind = kind,1459 .kind = kind,
1425 .hash = undefined, // to be populated by the worker1460 .hash = undefined, // to be populated by the worker
1426 .failure = undefined, // to be populated by the worker1461 .failure = undefined, // to be populated by the worker
1427 };1462 };
1428 wait_group.start();1463 wait_group.start();
1429 try thread_pool.spawn(workerHashFile, .{1464 try thread_pool.spawn(workerHashFile, .{
1430 tmp_directory.handle, hashed_file, &wait_group,1465 root_dir, hashed_file, &wait_group,
1431 });1466 });
1432 try all_files.append(hashed_file);1467 try all_files.append(hashed_file);
1433 }1468 }
...@@ -1446,7 +1481,7 @@ fn computeHash(...@@ -1446,7 +1481,7 @@ fn computeHash(
1446 var i: usize = 0;1481 var i: usize = 0;
1447 while (i < sus_dirs.count()) : (i += 1) {1482 while (i < sus_dirs.count()) : (i += 1) {
1448 const sus_dir = sus_dirs.keys()[i];1483 const sus_dir = sus_dirs.keys()[i];
1449 tmp_directory.handle.deleteDir(sus_dir) catch |err| switch (err) {1484 root_dir.deleteDir(sus_dir) catch |err| switch (err) {
1450 error.DirNotEmpty => continue,1485 error.DirNotEmpty => continue,
1451 error.FileNotFound => continue,1486 error.FileNotFound => continue,
1452 else => |e| {1487 else => |e| {
...@@ -1610,11 +1645,22 @@ const HashedFile = struct {...@@ -1610,11 +1645,22 @@ const HashedFile = struct {
1610 }1645 }
1611};1646};
16121647
1648/// Strips root directory name from file system path.
1649fn stripRoot(fs_path: []const u8, root_dir: []const u8) []const u8 {
1650 if (root_dir.len == 0 or fs_path.len <= root_dir.len) return fs_path;
1651
1652 if (std.mem.eql(u8, fs_path[0..root_dir.len], root_dir) and fs_path[root_dir.len] == fs.path.sep) {
1653 return fs_path[root_dir.len + 1 ..];
1654 }
1655
1656 return fs_path;
1657}
1658
1613/// Make a file system path identical independently of operating system path inconsistencies.1659/// Make a file system path identical independently of operating system path inconsistencies.
1614/// This converts backslashes into forward slashes.1660/// This converts backslashes into forward slashes.
1615fn normalizePathAlloc(arena: Allocator, fs_path: []const u8) ![]const u8 {1661fn normalizePathAlloc(arena: Allocator, pkg_path: []const u8) ![]const u8 {
1616 if (fs.path.sep == canonical_sep) return fs_path;1662 const normalized = try arena.dupe(u8, pkg_path);
1617 const normalized = try arena.dupe(u8, fs_path);1663 if (fs.path.sep == canonical_sep) return normalized;
1618 normalizePath(normalized);1664 normalizePath(normalized);
1619 return normalized;1665 return normalized;
1620}1666}