| ... | @@ -439,10 +439,9 @@ fn runResource( | ... | @@ -439,10 +439,9 @@ fn runResource( |
| 439 | const s = fs.path.sep_str; | 439 | const s = fs.path.sep_str; |
| 440 | const cache_root = f.job_queue.global_cache; | 440 | const cache_root = f.job_queue.global_cache; |
| 441 | const rand_int = std.crypto.random.int(u64); | 441 | const rand_int = std.crypto.random.int(u64); |
| 442 | const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); // root of the temporary directory | 442 | const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); |
| 443 | var tmp_package_root_sub_path: ?[]const u8 = null; // package root inside temporary directory | | |
| 444 | | 443 | |
| 445 | { | 444 | const package_sub_path = blk: { |
| 446 | const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path}); | 445 | const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path}); |
| 447 | var tmp_directory: Cache.Directory = .{ | 446 | var tmp_directory: Cache.Directory = .{ |
| 448 | .path = tmp_directory_path, | 447 | .path = tmp_directory_path, |
| ... | @@ -462,25 +461,21 @@ fn runResource( | ... | @@ -462,25 +461,21 @@ fn runResource( |
| 462 | }; | 461 | }; |
| 463 | defer tmp_directory.handle.close(); | 462 | defer tmp_directory.handle.close(); |
| 464 | | 463 | |
| 465 | try unpackResource(f, resource, uri_path, tmp_directory); | 464 | const package_dir = try unpackResource(f, resource, uri_path, tmp_directory); |
| 466 | | 465 | |
| 467 | // Strip leading root directory if needed. | 466 | if (package_dir) |dir_name| { |
| 468 | if (findPackageRootSubPath(arena, tmp_directory) catch null) |sub_path| { | 467 | // Position tmp_directory to dir_name inside tmp_dir_sub_path. |
| 469 | // Position tmp_directory to sub_path. | 468 | const path = try cache_root.join(arena, &.{ tmp_dir_sub_path, dir_name }); |
| 470 | const handle = tmp_directory.handle.openDir(sub_path, .{ .iterate = true }) catch |err| { | 469 | const handle = tmp_directory.handle.openDir(dir_name, .{ .iterate = true }) catch |err| { |
| 471 | try eb.addRootErrorMessage(.{ | 470 | try eb.addRootErrorMessage(.{ |
| 472 | .msg = try eb.printString("fail to open temporary directory '{s}' sub path '{s}': {s}", .{ | 471 | .msg = try eb.printString("unable to open temporary directory '{s}': {s}", .{ |
| 473 | tmp_directory_path, sub_path, @errorName(err), | 472 | path, @errorName(err), |
| 474 | }), | 473 | }), |
| 475 | }); | 474 | }); |
| 476 | return error.FetchFailed; | 475 | return error.FetchFailed; |
| 477 | }; | 476 | }; |
| 478 | tmp_package_root_sub_path = try fs.path.join(arena, &[_][]const u8{ tmp_dir_sub_path, sub_path }); | | |
| 479 | tmp_directory.handle.close(); | 477 | tmp_directory.handle.close(); |
| 480 | tmp_directory = .{ | 478 | tmp_directory = .{ .path = path, .handle = handle }; |
| 481 | .path = try cache_root.join(arena, &.{tmp_package_root_sub_path.?}), | | |
| 482 | .handle = handle, | | |
| 483 | }; | | |
| 484 | } else { | 479 | } else { |
| 485 | // btrfs workaround; reopen tmp_directory | 480 | // btrfs workaround; reopen tmp_directory |
| 486 | if (native_os == .linux and f.job_queue.work_around_btrfs_bug) { | 481 | if (native_os == .linux and f.job_queue.work_around_btrfs_bug) { |
| ... | @@ -511,7 +506,12 @@ fn runResource( | ... | @@ -511,7 +506,12 @@ fn runResource( |
| 511 | // Compute the package hash based on the remaining files in the temporary | 506 | // Compute the package hash based on the remaining files in the temporary |
| 512 | // directory. | 507 | // directory. |
| 513 | f.actual_hash = try computeHash(f, tmp_directory, filter); | 508 | f.actual_hash = try computeHash(f, tmp_directory, filter); |
| 514 | } | 509 | |
| | 510 | break :blk if (package_dir) |dir_name| |
| | 511 | try fs.path.join(arena, &.{ tmp_dir_sub_path, dir_name }) |
| | 512 | else |
| | 513 | tmp_dir_sub_path; |
| | 514 | }; |
| 515 | | 515 | |
| 516 | // Rename the temporary directory into the global zig package cache | 516 | // Rename the temporary directory into the global zig package cache |
| 517 | // directory. If the hash already exists, delete the temporary directory | 517 | // directory. If the hash already exists, delete the temporary directory |
| ... | @@ -523,11 +523,7 @@ fn runResource( | ... | @@ -523,11 +523,7 @@ fn runResource( |
| 523 | .root_dir = cache_root, | 523 | .root_dir = cache_root, |
| 524 | .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)), | 524 | .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)), |
| 525 | }; | 525 | }; |
| 526 | renameTmpIntoCache( | 526 | renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| { |
| 527 | cache_root.handle, | | |
| 528 | if (tmp_package_root_sub_path) |p| p else tmp_dir_sub_path, | | |
| 529 | f.package_root.sub_path, | | |
| 530 | ) catch |err| { | | |
| 531 | const src = try cache_root.join(arena, &.{tmp_dir_sub_path}); | 527 | const src = try cache_root.join(arena, &.{tmp_dir_sub_path}); |
| 532 | const dest = try cache_root.join(arena, &.{f.package_root.sub_path}); | 528 | const dest = try cache_root.join(arena, &.{f.package_root.sub_path}); |
| 533 | try eb.addRootErrorMessage(.{ .msg = try eb.printString( | 529 | try eb.addRootErrorMessage(.{ .msg = try eb.printString( |
| ... | @@ -536,10 +532,8 @@ fn runResource( | ... | @@ -536,10 +532,8 @@ fn runResource( |
| 536 | ) }); | 532 | ) }); |
| 537 | return error.FetchFailed; | 533 | return error.FetchFailed; |
| 538 | }; | 534 | }; |
| 539 | // Remove temporary directory root if that's not already done in rename. | 535 | // Remove temporary directory root. |
| 540 | if (tmp_package_root_sub_path) |_| { | 536 | cache_root.handle.deleteTree(tmp_dir_sub_path) catch {}; |
| 541 | cache_root.handle.deleteTree(tmp_dir_sub_path) catch {}; | | |
| 542 | } | | |
| 543 | | 537 | |
| 544 | // Validate the computed hash against the expected hash. If invalid, this | 538 | // Validate the computed hash against the expected hash. If invalid, this |
| 545 | // job is done. | 539 | // job is done. |
| ... | @@ -636,14 +630,11 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void { | ... | @@ -636,14 +630,11 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void { |
| 636 | } | 630 | } |
| 637 | } | 631 | } |
| 638 | | 632 | |
| 639 | // Finds package root subpath. | 633 | fn archivePackageDir(arena: Allocator, out_dir: fs.Dir) !?[]const u8 { |
| 640 | // Skips single root directory, returns null in all other cases. | 634 | var iter = out_dir.iterate(); |
| 641 | fn findPackageRootSubPath(allocator: Allocator, parent: Cache.Directory) !?[]const u8 { | | |
| 642 | var iter = parent.handle.iterate(); | | |
| 643 | if (try iter.next()) |entry| { | 635 | if (try iter.next()) |entry| { |
| 644 | if (try iter.next() != null) return null; | 636 | if (try iter.next() == null and entry.kind == .directory) { |
| 645 | if (entry.kind == .directory) { | 637 | return try arena.dupe(u8, entry.name); |
| 646 | return try allocator.dupe(u8, entry.name); | | |
| 647 | } | 638 | } |
| 648 | } | 639 | } |
| 649 | return null; | 640 | return null; |
| ... | @@ -1073,7 +1064,7 @@ fn unpackResource( | ... | @@ -1073,7 +1064,7 @@ fn unpackResource( |
| 1073 | resource: *Resource, | 1064 | resource: *Resource, |
| 1074 | uri_path: []const u8, | 1065 | uri_path: []const u8, |
| 1075 | tmp_directory: Cache.Directory, | 1066 | tmp_directory: Cache.Directory, |
| 1076 | ) RunError!void { | 1067 | ) RunError!?[]const u8 { |
| 1077 | const eb = &f.error_bundle; | 1068 | const eb = &f.error_bundle; |
| 1078 | const file_type = switch (resource.*) { | 1069 | const file_type = switch (resource.*) { |
| 1079 | .file => FileType.fromPath(uri_path) orelse | 1070 | .file => FileType.fromPath(uri_path) orelse |
| ... | @@ -1134,21 +1125,24 @@ fn unpackResource( | ... | @@ -1134,21 +1125,24 @@ fn unpackResource( |
| 1134 | | 1125 | |
| 1135 | .git => .git_pack, | 1126 | .git => .git_pack, |
| 1136 | | 1127 | |
| 1137 | .dir => |dir| return f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| { | 1128 | .dir => |dir| { |
| 1138 | return f.fail(f.location_tok, try eb.printString( | 1129 | f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| { |
| 1139 | "unable to copy directory '{s}': {s}", | 1130 | return f.fail(f.location_tok, try eb.printString( |
| 1140 | .{ uri_path, @errorName(err) }, | 1131 | "unable to copy directory '{s}': {s}", |
| 1141 | )); | 1132 | .{ uri_path, @errorName(err) }, |
| | 1133 | )); |
| | 1134 | }; |
| | 1135 | return null; |
| 1142 | }, | 1136 | }, |
| 1143 | }; | 1137 | }; |
| 1144 | | 1138 | |
| 1145 | switch (file_type) { | 1139 | switch (file_type) { |
| 1146 | .tar => try unpackTarball(f, tmp_directory.handle, resource.reader()), | 1140 | .tar => return try unpackTarball(f, tmp_directory.handle, resource.reader()), |
| 1147 | .@"tar.gz" => { | 1141 | .@"tar.gz" => { |
| 1148 | const reader = resource.reader(); | 1142 | const reader = resource.reader(); |
| 1149 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); | 1143 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); |
| 1150 | var dcp = std.compress.gzip.decompressor(br.reader()); | 1144 | var dcp = std.compress.gzip.decompressor(br.reader()); |
| 1151 | try unpackTarball(f, tmp_directory.handle, dcp.reader()); | 1145 | return try unpackTarball(f, tmp_directory.handle, dcp.reader()); |
| 1152 | }, | 1146 | }, |
| 1153 | .@"tar.xz" => { | 1147 | .@"tar.xz" => { |
| 1154 | const gpa = f.arena.child_allocator; | 1148 | const gpa = f.arena.child_allocator; |
| ... | @@ -1161,7 +1155,7 @@ fn unpackResource( | ... | @@ -1161,7 +1155,7 @@ fn unpackResource( |
| 1161 | )); | 1155 | )); |
| 1162 | }; | 1156 | }; |
| 1163 | defer dcp.deinit(); | 1157 | defer dcp.deinit(); |
| 1164 | try unpackTarball(f, tmp_directory.handle, dcp.reader()); | 1158 | return try unpackTarball(f, tmp_directory.handle, dcp.reader()); |
| 1165 | }, | 1159 | }, |
| 1166 | .@"tar.zst" => { | 1160 | .@"tar.zst" => { |
| 1167 | const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len; | 1161 | const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len; |
| ... | @@ -1171,21 +1165,25 @@ fn unpackResource( | ... | @@ -1171,21 +1165,25 @@ fn unpackResource( |
| 1171 | var dcp = std.compress.zstd.decompressor(br.reader(), .{ | 1165 | var dcp = std.compress.zstd.decompressor(br.reader(), .{ |
| 1172 | .window_buffer = window_buffer, | 1166 | .window_buffer = window_buffer, |
| 1173 | }); | 1167 | }); |
| 1174 | return unpackTarball(f, tmp_directory.handle, dcp.reader()); | 1168 | return try unpackTarball(f, tmp_directory.handle, dcp.reader()); |
| 1175 | }, | 1169 | }, |
| 1176 | .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) { | 1170 | .git_pack => { |
| 1177 | error.FetchFailed => return error.FetchFailed, | 1171 | unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) { |
| 1178 | error.OutOfMemory => return error.OutOfMemory, | 1172 | error.FetchFailed => return error.FetchFailed, |
| 1179 | else => |e| return f.fail(f.location_tok, try eb.printString( | 1173 | error.OutOfMemory => return error.OutOfMemory, |
| 1180 | "unable to unpack git files: {s}", | 1174 | else => |e| return f.fail(f.location_tok, try eb.printString( |
| 1181 | .{@errorName(e)}, | 1175 | "unable to unpack git files: {s}", |
| 1182 | )), | 1176 | .{@errorName(e)}, |
| | 1177 | )), |
| | 1178 | }; |
| | 1179 | return null; |
| 1183 | }, | 1180 | }, |
| 1184 | } | 1181 | } |
| 1185 | } | 1182 | } |
| 1186 | | 1183 | |
| 1187 | fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { | 1184 | fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!?[]const u8 { |
| 1188 | const eb = &f.error_bundle; | 1185 | const eb = &f.error_bundle; |
| | 1186 | const arena = f.arena.allocator(); |
| 1189 | const gpa = f.arena.child_allocator; | 1187 | const gpa = f.arena.child_allocator; |
| 1190 | | 1188 | |
| 1191 | var diagnostics: std.tar.Diagnostics = .{ .allocator = gpa }; | 1189 | var diagnostics: std.tar.Diagnostics = .{ .allocator = gpa }; |
| ... | @@ -1193,7 +1191,7 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { | ... | @@ -1193,7 +1191,7 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { |
| 1193 | | 1191 | |
| 1194 | std.tar.pipeToFileSystem(out_dir, reader, .{ | 1192 | std.tar.pipeToFileSystem(out_dir, reader, .{ |
| 1195 | .diagnostics = &diagnostics, | 1193 | .diagnostics = &diagnostics, |
| 1196 | .strip_components = 1, | 1194 | .strip_components = 0, |
| 1197 | // https://github.com/ziglang/zig/issues/17463 | 1195 | // https://github.com/ziglang/zig/issues/17463 |
| 1198 | .mode_mode = .ignore, | 1196 | .mode_mode = .ignore, |
| 1199 | .exclude_empty_directories = true, | 1197 | .exclude_empty_directories = true, |
| ... | @@ -1237,6 +1235,8 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { | ... | @@ -1237,6 +1235,8 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { |
| 1237 | } | 1235 | } |
| 1238 | return error.FetchFailed; | 1236 | return error.FetchFailed; |
| 1239 | } | 1237 | } |
| | 1238 | |
| | 1239 | return archivePackageDir(arena, out_dir) catch null; |
| 1240 | } | 1240 | } |
| 1241 | | 1241 | |
| 1242 | fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!void { | 1242 | fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!void { |
| ... | @@ -1742,36 +1742,33 @@ test { | ... | @@ -1742,36 +1742,33 @@ test { |
| 1742 | _ = FileType; | 1742 | _ = FileType; |
| 1743 | } | 1743 | } |
| 1744 | | 1744 | |
| 1745 | test "findPackageRootSubPath" { | 1745 | test archivePackageDir { |
| 1746 | const testing = std.testing; | 1746 | const testing = std.testing; |
| 1747 | | 1747 | |
| 1748 | var root = std.testing.tmpDir(.{ .iterate = true }); | 1748 | var tmp = std.testing.tmpDir(.{ .iterate = true }); |
| 1749 | defer root.cleanup(); | 1749 | defer tmp.cleanup(); |
| 1750 | | 1750 | |
| 1751 | // folder1 | 1751 | // folder1 |
| 1752 | // ├── folder2 | 1752 | // ├── folder2 |
| 1753 | // ├── file1 | 1753 | // ├── file1 |
| 1754 | // | 1754 | // |
| 1755 | try root.dir.makePath("folder1/folder2"); | 1755 | try tmp.dir.makePath("folder1/folder2"); |
| 1756 | (try root.dir.createFile("folder1/file1", .{})).close(); | 1756 | (try tmp.dir.createFile("folder1/file1", .{})).close(); |
| 1757 | | 1757 | |
| 1758 | // start at root returns folder1 as package root | 1758 | // start at root returns folder1 as package root |
| 1759 | const sub_path = (try findPackageRootSubPath( | 1759 | const sub_path = (try archivePackageDir(testing.allocator, tmp.dir)).?; |
| 1760 | testing.allocator, | | |
| 1761 | Cache.Directory{ .path = null, .handle = root.dir }, | | |
| 1762 | )).?; | | |
| 1763 | try testing.expectEqualStrings("folder1", sub_path); | 1760 | try testing.expectEqualStrings("folder1", sub_path); |
| 1764 | testing.allocator.free(sub_path); | 1761 | testing.allocator.free(sub_path); |
| 1765 | | 1762 | |
| 1766 | // start at folder1 returns null | 1763 | // start at folder1 returns null |
| 1767 | try testing.expect(null == (try findPackageRootSubPath( | 1764 | try testing.expect(null == (try archivePackageDir( |
| 1768 | testing.allocator, | 1765 | testing.allocator, |
| 1769 | Cache.Directory{ .path = null, .handle = try root.dir.openDir("folder1", .{ .iterate = true }) }, | 1766 | try tmp.dir.openDir("folder1", .{ .iterate = true }), |
| 1770 | ))); | 1767 | ))); |
| 1771 | | 1768 | |
| 1772 | // start at folder1/folder2 returns null | 1769 | // start at folder1/folder2 returns null |
| 1773 | try testing.expect(null == (try findPackageRootSubPath( | 1770 | try testing.expect(null == (try archivePackageDir( |
| 1774 | testing.allocator, | 1771 | testing.allocator, |
| 1775 | Cache.Directory{ .path = null, .handle = try root.dir.openDir("folder1/folder2", .{ .iterate = true }) }, | 1772 | try tmp.dir.openDir("folder1/folder2", .{ .iterate = true }), |
| 1776 | ))); | 1773 | ))); |
| 1777 | } | 1774 | } |
Igor Anić <igor.anic@gmail.com> 2024-03-26 22:27:06+01:00
src