| ... | ... | @@ -439,10 +439,9 @@ fn runResource( |
| 439 | 439 | const s = fs.path.sep_str; |
| 440 | 440 | const cache_root = f.job_queue.global_cache; |
| 441 | 441 | const rand_int = std.crypto.random.int(u64); |
| 442 | | const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); // root of the temporary directory |
| 443 | | var tmp_package_root_sub_path: ?[]const u8 = null; // package root inside temporary directory |
| 442 | const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); |
| 444 | 443 | |
| 445 | | { |
| 444 | const package_sub_path = blk: { |
| 446 | 445 | const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path}); |
| 447 | 446 | var tmp_directory: Cache.Directory = .{ |
| 448 | 447 | .path = tmp_directory_path, |
| ... | ... | @@ -462,25 +461,21 @@ fn runResource( |
| 462 | 461 | }; |
| 463 | 462 | defer tmp_directory.handle.close(); |
| 464 | 463 | |
| 465 | | try unpackResource(f, resource, uri_path, tmp_directory); |
| 464 | const package_dir = try unpackResource(f, resource, uri_path, tmp_directory); |
| 466 | 465 | |
| 467 | | // Strip leading root directory if needed. |
| 468 | | if (findPackageRootSubPath(arena, tmp_directory) catch null) |sub_path| { |
| 469 | | // Position tmp_directory to sub_path. |
| 470 | | const handle = tmp_directory.handle.openDir(sub_path, .{ .iterate = true }) catch |err| { |
| 466 | if (package_dir) |dir_name| { |
| 467 | // Position tmp_directory to dir_name inside tmp_dir_sub_path. |
| 468 | const path = try cache_root.join(arena, &.{ tmp_dir_sub_path, dir_name }); |
| 469 | const handle = tmp_directory.handle.openDir(dir_name, .{ .iterate = true }) catch |err| { |
| 471 | 470 | try eb.addRootErrorMessage(.{ |
| 472 | | .msg = try eb.printString("fail to open temporary directory '{s}' sub path '{s}': {s}", .{ |
| 473 | | tmp_directory_path, sub_path, @errorName(err), |
| 471 | .msg = try eb.printString("unable to open temporary directory '{s}': {s}", .{ |
| 472 | path, @errorName(err), |
| 474 | 473 | }), |
| 475 | 474 | }); |
| 476 | 475 | return error.FetchFailed; |
| 477 | 476 | }; |
| 478 | | tmp_package_root_sub_path = try fs.path.join(arena, &[_][]const u8{ tmp_dir_sub_path, sub_path }); |
| 479 | 477 | tmp_directory.handle.close(); |
| 480 | | tmp_directory = .{ |
| 481 | | .path = try cache_root.join(arena, &.{tmp_package_root_sub_path.?}), |
| 482 | | .handle = handle, |
| 483 | | }; |
| 478 | tmp_directory = .{ .path = path, .handle = handle }; |
| 484 | 479 | } else { |
| 485 | 480 | // btrfs workaround; reopen tmp_directory |
| 486 | 481 | if (native_os == .linux and f.job_queue.work_around_btrfs_bug) { |
| ... | ... | @@ -511,7 +506,12 @@ fn runResource( |
| 511 | 506 | // Compute the package hash based on the remaining files in the temporary |
| 512 | 507 | // directory. |
| 513 | 508 | f.actual_hash = try computeHash(f, tmp_directory, filter); |
| 514 | | } |
| 509 | |
| 510 | break :blk if (package_dir) |dir_name| |
| 511 | try fs.path.join(arena, &.{ tmp_dir_sub_path, dir_name }) |
| 512 | else |
| 513 | tmp_dir_sub_path; |
| 514 | }; |
| 515 | 515 | |
| 516 | 516 | // Rename the temporary directory into the global zig package cache |
| 517 | 517 | // directory. If the hash already exists, delete the temporary directory |
| ... | ... | @@ -523,11 +523,7 @@ fn runResource( |
| 523 | 523 | .root_dir = cache_root, |
| 524 | 524 | .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)), |
| 525 | 525 | }; |
| 526 | | renameTmpIntoCache( |
| 527 | | cache_root.handle, |
| 528 | | if (tmp_package_root_sub_path) |p| p else tmp_dir_sub_path, |
| 529 | | f.package_root.sub_path, |
| 530 | | ) catch |err| { |
| 526 | renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| { |
| 531 | 527 | const src = try cache_root.join(arena, &.{tmp_dir_sub_path}); |
| 532 | 528 | const dest = try cache_root.join(arena, &.{f.package_root.sub_path}); |
| 533 | 529 | try eb.addRootErrorMessage(.{ .msg = try eb.printString( |
| ... | ... | @@ -536,10 +532,8 @@ fn runResource( |
| 536 | 532 | ) }); |
| 537 | 533 | return error.FetchFailed; |
| 538 | 534 | }; |
| 539 | | // Remove temporary directory root if that's not already done in rename. |
| 540 | | if (tmp_package_root_sub_path) |_| { |
| 541 | | cache_root.handle.deleteTree(tmp_dir_sub_path) catch {}; |
| 542 | | } |
| 535 | // Remove temporary directory root. |
| 536 | cache_root.handle.deleteTree(tmp_dir_sub_path) catch {}; |
| 543 | 537 | |
| 544 | 538 | // Validate the computed hash against the expected hash. If invalid, this |
| 545 | 539 | // job is done. |
| ... | ... | @@ -636,14 +630,11 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void { |
| 636 | 630 | } |
| 637 | 631 | } |
| 638 | 632 | |
| 639 | | // Finds package root subpath. |
| 640 | | // Skips single root directory, returns null in all other cases. |
| 641 | | fn findPackageRootSubPath(allocator: Allocator, parent: Cache.Directory) !?[]const u8 { |
| 642 | | var iter = parent.handle.iterate(); |
| 633 | fn archivePackageDir(arena: Allocator, out_dir: fs.Dir) !?[]const u8 { |
| 634 | var iter = out_dir.iterate(); |
| 643 | 635 | if (try iter.next()) |entry| { |
| 644 | | if (try iter.next() != null) return null; |
| 645 | | if (entry.kind == .directory) { |
| 646 | | return try allocator.dupe(u8, entry.name); |
| 636 | if (try iter.next() == null and entry.kind == .directory) { |
| 637 | return try arena.dupe(u8, entry.name); |
| 647 | 638 | } |
| 648 | 639 | } |
| 649 | 640 | return null; |
| ... | ... | @@ -1073,7 +1064,7 @@ fn unpackResource( |
| 1073 | 1064 | resource: *Resource, |
| 1074 | 1065 | uri_path: []const u8, |
| 1075 | 1066 | tmp_directory: Cache.Directory, |
| 1076 | | ) RunError!void { |
| 1067 | ) RunError!?[]const u8 { |
| 1077 | 1068 | const eb = &f.error_bundle; |
| 1078 | 1069 | const file_type = switch (resource.*) { |
| 1079 | 1070 | .file => FileType.fromPath(uri_path) orelse |
| ... | ... | @@ -1134,21 +1125,24 @@ fn unpackResource( |
| 1134 | 1125 | |
| 1135 | 1126 | .git => .git_pack, |
| 1136 | 1127 | |
| 1137 | | .dir => |dir| return f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| { |
| 1138 | | return f.fail(f.location_tok, try eb.printString( |
| 1139 | | "unable to copy directory '{s}': {s}", |
| 1140 | | .{ uri_path, @errorName(err) }, |
| 1141 | | )); |
| 1128 | .dir => |dir| { |
| 1129 | f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| { |
| 1130 | return f.fail(f.location_tok, try eb.printString( |
| 1131 | "unable to copy directory '{s}': {s}", |
| 1132 | .{ uri_path, @errorName(err) }, |
| 1133 | )); |
| 1134 | }; |
| 1135 | return null; |
| 1142 | 1136 | }, |
| 1143 | 1137 | }; |
| 1144 | 1138 | |
| 1145 | 1139 | switch (file_type) { |
| 1146 | | .tar => try unpackTarball(f, tmp_directory.handle, resource.reader()), |
| 1140 | .tar => return try unpackTarball(f, tmp_directory.handle, resource.reader()), |
| 1147 | 1141 | .@"tar.gz" => { |
| 1148 | 1142 | const reader = resource.reader(); |
| 1149 | 1143 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); |
| 1150 | 1144 | var dcp = std.compress.gzip.decompressor(br.reader()); |
| 1151 | | try unpackTarball(f, tmp_directory.handle, dcp.reader()); |
| 1145 | return try unpackTarball(f, tmp_directory.handle, dcp.reader()); |
| 1152 | 1146 | }, |
| 1153 | 1147 | .@"tar.xz" => { |
| 1154 | 1148 | const gpa = f.arena.child_allocator; |
| ... | ... | @@ -1161,7 +1155,7 @@ fn unpackResource( |
| 1161 | 1155 | )); |
| 1162 | 1156 | }; |
| 1163 | 1157 | defer dcp.deinit(); |
| 1164 | | try unpackTarball(f, tmp_directory.handle, dcp.reader()); |
| 1158 | return try unpackTarball(f, tmp_directory.handle, dcp.reader()); |
| 1165 | 1159 | }, |
| 1166 | 1160 | .@"tar.zst" => { |
| 1167 | 1161 | const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len; |
| ... | ... | @@ -1171,21 +1165,25 @@ fn unpackResource( |
| 1171 | 1165 | var dcp = std.compress.zstd.decompressor(br.reader(), .{ |
| 1172 | 1166 | .window_buffer = window_buffer, |
| 1173 | 1167 | }); |
| 1174 | | return unpackTarball(f, tmp_directory.handle, dcp.reader()); |
| 1168 | return try unpackTarball(f, tmp_directory.handle, dcp.reader()); |
| 1175 | 1169 | }, |
| 1176 | | .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) { |
| 1177 | | error.FetchFailed => return error.FetchFailed, |
| 1178 | | error.OutOfMemory => return error.OutOfMemory, |
| 1179 | | else => |e| return f.fail(f.location_tok, try eb.printString( |
| 1180 | | "unable to unpack git files: {s}", |
| 1181 | | .{@errorName(e)}, |
| 1182 | | )), |
| 1170 | .git_pack => { |
| 1171 | unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) { |
| 1172 | error.FetchFailed => return error.FetchFailed, |
| 1173 | error.OutOfMemory => return error.OutOfMemory, |
| 1174 | else => |e| return f.fail(f.location_tok, try eb.printString( |
| 1175 | "unable to unpack git files: {s}", |
| 1176 | .{@errorName(e)}, |
| 1177 | )), |
| 1178 | }; |
| 1179 | return null; |
| 1183 | 1180 | }, |
| 1184 | 1181 | } |
| 1185 | 1182 | } |
| 1186 | 1183 | |
| 1187 | | fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { |
| 1184 | fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!?[]const u8 { |
| 1188 | 1185 | const eb = &f.error_bundle; |
| 1186 | const arena = f.arena.allocator(); |
| 1189 | 1187 | const gpa = f.arena.child_allocator; |
| 1190 | 1188 | |
| 1191 | 1189 | var diagnostics: std.tar.Diagnostics = .{ .allocator = gpa }; |
| ... | ... | @@ -1193,7 +1191,7 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { |
| 1193 | 1191 | |
| 1194 | 1192 | std.tar.pipeToFileSystem(out_dir, reader, .{ |
| 1195 | 1193 | .diagnostics = &diagnostics, |
| 1196 | | .strip_components = 1, |
| 1194 | .strip_components = 0, |
| 1197 | 1195 | // https://github.com/ziglang/zig/issues/17463 |
| 1198 | 1196 | .mode_mode = .ignore, |
| 1199 | 1197 | .exclude_empty_directories = true, |
| ... | ... | @@ -1237,6 +1235,8 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { |
| 1237 | 1235 | } |
| 1238 | 1236 | return error.FetchFailed; |
| 1239 | 1237 | } |
| 1238 | |
| 1239 | return archivePackageDir(arena, out_dir) catch null; |
| 1240 | 1240 | } |
| 1241 | 1241 | |
| 1242 | 1242 | fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!void { |
| ... | ... | @@ -1742,36 +1742,33 @@ test { |
| 1742 | 1742 | _ = FileType; |
| 1743 | 1743 | } |
| 1744 | 1744 | |
| 1745 | | test "findPackageRootSubPath" { |
| 1745 | test archivePackageDir { |
| 1746 | 1746 | const testing = std.testing; |
| 1747 | 1747 | |
| 1748 | | var root = std.testing.tmpDir(.{ .iterate = true }); |
| 1749 | | defer root.cleanup(); |
| 1748 | var tmp = std.testing.tmpDir(.{ .iterate = true }); |
| 1749 | defer tmp.cleanup(); |
| 1750 | 1750 | |
| 1751 | 1751 | // folder1 |
| 1752 | 1752 | // ├── folder2 |
| 1753 | 1753 | // ├── file1 |
| 1754 | 1754 | // |
| 1755 | | try root.dir.makePath("folder1/folder2"); |
| 1756 | | (try root.dir.createFile("folder1/file1", .{})).close(); |
| 1755 | try tmp.dir.makePath("folder1/folder2"); |
| 1756 | (try tmp.dir.createFile("folder1/file1", .{})).close(); |
| 1757 | 1757 | |
| 1758 | 1758 | // start at root returns folder1 as package root |
| 1759 | | const sub_path = (try findPackageRootSubPath( |
| 1760 | | testing.allocator, |
| 1761 | | Cache.Directory{ .path = null, .handle = root.dir }, |
| 1762 | | )).?; |
| 1759 | const sub_path = (try archivePackageDir(testing.allocator, tmp.dir)).?; |
| 1763 | 1760 | try testing.expectEqualStrings("folder1", sub_path); |
| 1764 | 1761 | testing.allocator.free(sub_path); |
| 1765 | 1762 | |
| 1766 | 1763 | // start at folder1 returns null |
| 1767 | | try testing.expect(null == (try findPackageRootSubPath( |
| 1764 | try testing.expect(null == (try archivePackageDir( |
| 1768 | 1765 | testing.allocator, |
| 1769 | | Cache.Directory{ .path = null, .handle = try root.dir.openDir("folder1", .{ .iterate = true }) }, |
| 1766 | try tmp.dir.openDir("folder1", .{ .iterate = true }), |
| 1770 | 1767 | ))); |
| 1771 | 1768 | |
| 1772 | 1769 | // start at folder1/folder2 returns null |
| 1773 | | try testing.expect(null == (try findPackageRootSubPath( |
| 1770 | try testing.expect(null == (try archivePackageDir( |
| 1774 | 1771 | testing.allocator, |
| 1775 | | Cache.Directory{ .path = null, .handle = try root.dir.openDir("folder1/folder2", .{ .iterate = true }) }, |
| 1772 | try tmp.dir.openDir("folder1/folder2", .{ .iterate = true }), |
| 1776 | 1773 | ))); |
| 1777 | 1774 | } |
Igor Anić <igor.anic@gmail.com> 2024-03-26 22:27:06+01:00
src