| ... | ... | @@ -316,36 +316,30 @@ pub fn fetchAndAddDependencies( |
| 316 | 316 | |
| 317 | 317 | // Directories do not provide a hash in build.zig.zon. |
| 318 | 318 | // Hash the path to the module rather than its contents. |
| 319 | | if (fetch_location == .directory) { |
| 320 | | if (dep.hash != null) { |
| 321 | | return report.fail(dep.hash_tok, "hash not allowed for directory package", .{}); |
| 322 | | } |
| 323 | | const hex_digest = Manifest.hexDigest(try computePathHash(gpa, directory, fetch_location.directory)); |
| 324 | | dep.hash = try gpa.dupe(u8, &hex_digest); |
| 325 | | } |
| 326 | | |
| 327 | | const sub_mod, const found_existing = try getCachedPackage( |
| 328 | | arena, |
| 329 | | fetch_location, |
| 330 | | global_cache_directory, |
| 331 | | dep.*, |
| 332 | | all_modules, |
| 333 | | root_prog_node, |
| 334 | | ) orelse .{ |
| 335 | | try fetchAndUnpack( |
| 336 | | fetch_location, |
| 337 | | thread_pool, |
| 338 | | http_client, |
| 339 | | directory, |
| 319 | const sub_mod, const found_existing = if (fetch_location == .directory) |
| 320 | try getDirectoryModule(gpa, fetch_location, directory, all_modules, dep, report) |
| 321 | else |
| 322 | try getCachedPackage( |
| 323 | gpa, |
| 340 | 324 | global_cache_directory, |
| 341 | 325 | dep.*, |
| 342 | | report, |
| 343 | 326 | all_modules, |
| 344 | 327 | root_prog_node, |
| 345 | | name, |
| 346 | | ), |
| 347 | | false, |
| 348 | | }; |
| 328 | ) orelse .{ |
| 329 | try fetchAndUnpack( |
| 330 | fetch_location, |
| 331 | thread_pool, |
| 332 | http_client, |
| 333 | directory, |
| 334 | global_cache_directory, |
| 335 | dep.*, |
| 336 | report, |
| 337 | all_modules, |
| 338 | root_prog_node, |
| 339 | name, |
| 340 | ), |
| 341 | false, |
| 342 | }; |
| 349 | 343 | |
| 350 | 344 | assert(dep.hash != null); |
| 351 | 345 | |
| ... | ... | @@ -576,14 +570,6 @@ const FetchLocation = union(enum) { |
| 576 | 570 | .resource = .{ .file = try root_dir.handle.openFile(file, .{}) }, |
| 577 | 571 | }; |
| 578 | 572 | }, |
| 579 | | .directory => |dir| { |
| 580 | | const owned_path = try gpa.dupe(u8, dir); |
| 581 | | errdefer gpa.free(owned_path); |
| 582 | | return .{ |
| 583 | | .path = owned_path, |
| 584 | | .resource = .{ .directory = try root_dir.handle.openIterableDir(dir, .{}) }, |
| 585 | | }; |
| 586 | | }, |
| 587 | 573 | .http_request => |uri| { |
| 588 | 574 | var h = std.http.Headers{ .allocator = gpa }; |
| 589 | 575 | defer h.deinit(); |
| ... | ... | @@ -606,6 +592,7 @@ const FetchLocation = union(enum) { |
| 606 | 592 | .resource = .{ .http_request = req }, |
| 607 | 593 | }; |
| 608 | 594 | }, |
| 595 | .directory => unreachable, // Directories do not require fetching |
| 609 | 596 | } |
| 610 | 597 | } |
| 611 | 598 | }; |
| ... | ... | @@ -614,7 +601,6 @@ const ReadableResource = struct { |
| 614 | 601 | path: []const u8, |
| 615 | 602 | resource: union(enum) { |
| 616 | 603 | file: fs.File, |
| 617 | | directory: fs.IterableDir, |
| 618 | 604 | http_request: std.http.Client.Request, |
| 619 | 605 | }, |
| 620 | 606 | |
| ... | ... | @@ -625,20 +611,12 @@ const ReadableResource = struct { |
| 625 | 611 | rr: *ReadableResource, |
| 626 | 612 | allocator: Allocator, |
| 627 | 613 | thread_pool: *ThreadPool, |
| 628 | | root_dir: Compilation.Directory, |
| 629 | 614 | global_cache_directory: Compilation.Directory, |
| 630 | 615 | dep: Manifest.Dependency, |
| 631 | 616 | report: Report, |
| 632 | 617 | pkg_prog_node: *std.Progress.Node, |
| 633 | 618 | ) !PackageLocation { |
| 634 | 619 | switch (rr.resource) { |
| 635 | | .directory => { |
| 636 | | return .{ |
| 637 | | .hash = try computePathHash(allocator, root_dir, rr.path), |
| 638 | | .root_src_dir_path = try allocator.dupe(u8, rr.path), |
| 639 | | .root_dir = root_dir, |
| 640 | | }; |
| 641 | | }, |
| 642 | 620 | inline .file, .http_request => |*r| { |
| 643 | 621 | const s = fs.path.sep_str; |
| 644 | 622 | const rand_int = std.crypto.random.int(u64); |
| ... | ... | @@ -710,8 +688,7 @@ const ReadableResource = struct { |
| 710 | 688 | |
| 711 | 689 | return .{ |
| 712 | 690 | .hash = actual_hash, |
| 713 | | .root_src_dir_path = relative_unpacked_path, |
| 714 | | .root_dir = global_cache_directory, |
| 691 | .relative_unpacked_path = relative_unpacked_path, |
| 715 | 692 | }; |
| 716 | 693 | }, |
| 717 | 694 | } |
| ... | ... | @@ -727,7 +704,6 @@ const ReadableResource = struct { |
| 727 | 704 | // TODO: Handle case of chunked content-length |
| 728 | 705 | .http_request => |req| return req.response.content_length, |
| 729 | 706 | .file => |f| return (try f.metadata()).size(), |
| 730 | | .directory => unreachable, |
| 731 | 707 | } |
| 732 | 708 | } |
| 733 | 709 | |
| ... | ... | @@ -737,7 +713,6 @@ const ReadableResource = struct { |
| 737 | 713 | return fileTypeFromPath(rr.path) orelse |
| 738 | 714 | return report.fail(dep.location_tok, "Unknown file type", .{}); |
| 739 | 715 | }, |
| 740 | | .directory => return error.IsDir, |
| 741 | 716 | .http_request => |req| { |
| 742 | 717 | const content_type = req.response.headers.getFirstValue("Content-Type") orelse |
| 743 | 718 | return report.fail(dep.location_tok, "Missing 'Content-Type' header", .{}); |
| ... | ... | @@ -793,7 +768,6 @@ const ReadableResource = struct { |
| 793 | 768 | gpa.free(rr.path); |
| 794 | 769 | switch (rr.resource) { |
| 795 | 770 | .file => |file| file.close(), |
| 796 | | .directory => |*dir| dir.close(), |
| 797 | 771 | .http_request => |*req| req.deinit(), |
| 798 | 772 | } |
| 799 | 773 | rr.* = undefined; |
| ... | ... | @@ -804,11 +778,10 @@ pub const PackageLocation = struct { |
| 804 | 778 | /// For packages that require unpacking, this is the hash of the package contents. |
| 805 | 779 | /// For directories, this is the hash of the absolute file path. |
| 806 | 780 | hash: [Manifest.Hash.digest_length]u8, |
| 807 | | root_src_dir_path: []const u8, |
| 808 | | root_dir: Compilation.Directory, |
| 781 | relative_unpacked_path: []const u8, |
| 809 | 782 | |
| 810 | 783 | pub fn deinit(pl: *PackageLocation, allocator: Allocator) void { |
| 811 | | allocator.free(pl.root_src_dir_path); |
| 784 | allocator.free(pl.relative_unpacked_path); |
| 812 | 785 | pl.* = undefined; |
| 813 | 786 | } |
| 814 | 787 | }; |
| ... | ... | @@ -874,19 +847,11 @@ fn ProgressReader(comptime ReaderType: type) type { |
| 874 | 847 | /// (i.e. whether or not its transitive dependencies have been fetched). |
| 875 | 848 | fn getCachedPackage( |
| 876 | 849 | gpa: Allocator, |
| 877 | | fetch_location: FetchLocation, |
| 878 | 850 | global_cache_directory: Compilation.Directory, |
| 879 | 851 | dep: Manifest.Dependency, |
| 880 | 852 | all_modules: *AllModules, |
| 881 | 853 | root_prog_node: *std.Progress.Node, |
| 882 | 854 | ) !?struct { DependencyModule, bool } { |
| 883 | | // There is no fixed location to check for directory modules. |
| 884 | | // Instead, check whether it is already listed in all_modules. |
| 885 | | if (fetch_location == .directory) { |
| 886 | | const hex_digest = dep.hash.?[0..hex_multihash_len]; |
| 887 | | return if (all_modules.get(hex_digest.*)) |mod| .{ mod.?, true } else null; |
| 888 | | } |
| 889 | | |
| 890 | 855 | const s = fs.path.sep_str; |
| 891 | 856 | // Check if the expected_hash is already present in the global package |
| 892 | 857 | // cache, and thereby avoid both fetching and unpacking. |
| ... | ... | @@ -912,34 +877,60 @@ fn getCachedPackage( |
| 912 | 877 | root_prog_node.completeOne(); |
| 913 | 878 | |
| 914 | 879 | const is_zig_mod = if (pkg_dir.access(build_zig_basename, .{})) |_| true else |_| false; |
| 880 | const basename = if (is_zig_mod) build_zig_basename else ""; |
| 881 | const pkg = try createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, basename); |
| 915 | 882 | |
| 916 | | const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}); |
| 917 | | errdefer gpa.free(build_root); |
| 918 | | |
| 919 | | const ptr = try gpa.create(Package); |
| 920 | | errdefer gpa.destroy(ptr); |
| 883 | const module: DependencyModule = if (is_zig_mod) |
| 884 | .{ .zig_pkg = pkg } |
| 885 | else |
| 886 | .{ .non_zig_pkg = pkg }; |
| 921 | 887 | |
| 922 | | const owned_src_path = if (is_zig_mod) try gpa.dupe(u8, build_zig_basename) else ""; |
| 923 | | errdefer gpa.free(owned_src_path); |
| 888 | try all_modules.put(gpa, hex_digest.*, module); |
| 889 | return .{ module, false }; |
| 890 | } |
| 924 | 891 | |
| 925 | | ptr.* = .{ |
| 926 | | .root_src_directory = .{ |
| 927 | | .path = build_root, |
| 928 | | .handle = pkg_dir, |
| 929 | | }, |
| 930 | | .root_src_directory_owned = true, |
| 931 | | .root_src_path = owned_src_path, |
| 932 | | }; |
| 892 | return null; |
| 893 | } |
| 933 | 894 | |
| 934 | | gop.value_ptr.* = if (is_zig_mod) |
| 935 | | .{ .zig_pkg = ptr } |
| 936 | | else |
| 937 | | .{ .non_zig_pkg = ptr }; |
| 895 | fn getDirectoryModule( |
| 896 | gpa: Allocator, |
| 897 | fetch_location: FetchLocation, |
| 898 | directory: Compilation.Directory, |
| 899 | all_modules: *AllModules, |
| 900 | dep: *Manifest.Dependency, |
| 901 | report: Report, |
| 902 | ) !struct { DependencyModule, bool } { |
| 903 | assert(fetch_location == .directory); |
| 938 | 904 | |
| 939 | | return .{ gop.value_ptr.*.?, false }; |
| 905 | if (dep.hash != null) { |
| 906 | return report.fail(dep.hash_tok, "hash not allowed for directory package", .{}); |
| 940 | 907 | } |
| 941 | 908 | |
| 942 | | return null; |
| 909 | const hash = try computePathHash(gpa, directory, fetch_location.directory); |
| 910 | const hex_digest = Manifest.hexDigest(hash); |
| 911 | dep.hash = try gpa.dupe(u8, &hex_digest); |
| 912 | |
| 913 | // There is no fixed location to check for directory modules. |
| 914 | // Instead, check whether it is already listed in all_modules. |
| 915 | if (all_modules.get(hex_digest)) |mod| return .{ mod.?, true }; |
| 916 | |
| 917 | var pkg_dir = directory.handle.openDir(fetch_location.directory, .{}) catch |err| switch (err) { |
| 918 | error.FileNotFound => return report.fail(dep.location_tok, "File not found: {s}", .{fetch_location.directory}), |
| 919 | else => |e| return e, |
| 920 | }; |
| 921 | defer pkg_dir.close(); |
| 922 | |
| 923 | const is_zig_mod = if (pkg_dir.access(build_zig_basename, .{})) |_| true else |_| false; |
| 924 | const basename = if (is_zig_mod) build_zig_basename else ""; |
| 925 | |
| 926 | const pkg = try createWithDir(gpa, directory, fetch_location.directory, basename); |
| 927 | const module: DependencyModule = if (is_zig_mod) |
| 928 | .{ .zig_pkg = pkg } |
| 929 | else |
| 930 | .{ .non_zig_pkg = pkg }; |
| 931 | |
| 932 | try all_modules.put(gpa, hex_digest, module); |
| 933 | return .{ module, false }; |
| 943 | 934 | } |
| 944 | 935 | |
| 945 | 936 | fn fetchAndUnpack( |
| ... | ... | @@ -956,6 +947,8 @@ fn fetchAndUnpack( |
| 956 | 947 | /// is only intended to be human-readable for progress reporting. |
| 957 | 948 | name_for_prog: []const u8, |
| 958 | 949 | ) !DependencyModule { |
| 950 | assert(fetch_location == .file or fetch_location == .http_request); |
| 951 | |
| 959 | 952 | const gpa = http_client.allocator; |
| 960 | 953 | |
| 961 | 954 | var pkg_prog_node = root_prog_node.start(name_for_prog, 0); |
| ... | ... | @@ -966,51 +959,47 @@ fn fetchAndUnpack( |
| 966 | 959 | var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep, report); |
| 967 | 960 | defer readable_resource.deinit(gpa); |
| 968 | 961 | |
| 969 | | var package_location = try readable_resource.unpack(gpa, thread_pool, directory, global_cache_directory, dep, report, &pkg_prog_node); |
| 962 | var package_location = try readable_resource.unpack(gpa, thread_pool, global_cache_directory, dep, report, &pkg_prog_node); |
| 970 | 963 | defer package_location.deinit(gpa); |
| 971 | 964 | |
| 972 | 965 | const actual_hex = Manifest.hexDigest(package_location.hash); |
| 973 | | if (readable_resource.resource != .directory) { |
| 974 | | if (dep.hash) |h| { |
| 975 | | if (!mem.eql(u8, h, &actual_hex)) { |
| 976 | | return report.fail(dep.hash_tok, "hash mismatch: expected: {s}, found: {s}", .{ |
| 977 | | h, actual_hex, |
| 978 | | }); |
| 979 | | } |
| 980 | | } else { |
| 981 | | const file_path = try report.directory.join(gpa, &.{Manifest.basename}); |
| 982 | | defer gpa.free(file_path); |
| 983 | | |
| 984 | | const eb = report.error_bundle; |
| 985 | | const notes_len = 1; |
| 986 | | try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{ |
| 987 | | .tok = dep.location_tok, |
| 988 | | .off = 0, |
| 989 | | .msg = "dependency is missing hash field", |
| 966 | if (dep.hash) |h| { |
| 967 | if (!mem.eql(u8, h, &actual_hex)) { |
| 968 | return report.fail(dep.hash_tok, "hash mismatch: expected: {s}, found: {s}", .{ |
| 969 | h, actual_hex, |
| 990 | 970 | }); |
| 991 | | const notes_start = try eb.reserveNotes(notes_len); |
| 992 | | eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ |
| 993 | | .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}), |
| 994 | | })); |
| 995 | | return error.PackageFetchFailed; |
| 996 | 971 | } |
| 972 | } else { |
| 973 | const file_path = try report.directory.join(gpa, &.{Manifest.basename}); |
| 974 | defer gpa.free(file_path); |
| 975 | |
| 976 | const eb = report.error_bundle; |
| 977 | const notes_len = 1; |
| 978 | try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{ |
| 979 | .tok = dep.location_tok, |
| 980 | .off = 0, |
| 981 | .msg = "dependency is missing hash field", |
| 982 | }); |
| 983 | const notes_start = try eb.reserveNotes(notes_len); |
| 984 | eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ |
| 985 | .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}), |
| 986 | })); |
| 987 | return error.PackageFetchFailed; |
| 997 | 988 | } |
| 998 | 989 | |
| 999 | | const build_zig_path = try std.fs.path.join(gpa, &.{ package_location.root_src_dir_path, build_zig_basename }); |
| 990 | const build_zig_path = try fs.path.join(gpa, &.{ package_location.relative_unpacked_path, build_zig_basename }); |
| 1000 | 991 | defer gpa.free(build_zig_path); |
| 1001 | 992 | |
| 1002 | | package_location.root_dir.handle.access(build_zig_path, .{}) catch |err| switch (err) { |
| 1003 | | error.FileNotFound => { |
| 1004 | | const module = try createWithDir(gpa, package_location.root_dir, package_location.root_src_dir_path, ""); |
| 1005 | | try all_modules.put(gpa, actual_hex, .{ .non_zig_pkg = module }); |
| 1006 | | return .{ .non_zig_pkg = module }; |
| 1007 | | }, |
| 1008 | | else => return err, |
| 1009 | | }; |
| 993 | const is_zig_mod = if (global_cache_directory.handle.access(build_zig_path, .{})) |_| true else |_| false; |
| 994 | const basename = if (is_zig_mod) build_zig_basename else ""; |
| 995 | const pkg = try createWithDir(gpa, global_cache_directory, package_location.relative_unpacked_path, basename); |
| 996 | const module: DependencyModule = if (is_zig_mod) |
| 997 | .{ .zig_pkg = pkg } |
| 998 | else |
| 999 | .{ .non_zig_pkg = pkg }; |
| 1010 | 1000 | |
| 1011 | | const module = try createWithDir(gpa, package_location.root_dir, package_location.root_src_dir_path, build_zig_basename); |
| 1012 | | try all_modules.put(gpa, actual_hex, .{ .zig_pkg = module }); |
| 1013 | | return .{ .zig_pkg = module }; |
| 1001 | try all_modules.put(gpa, actual_hex, module); |
| 1002 | return module; |
| 1014 | 1003 | } |
| 1015 | 1004 | |
| 1016 | 1005 | fn unpackTarball( |
Adam Goertz <adambgoertz@gmail.com> 2023-09-28 00:25:12+00:00
Andrew Kelley <andrew@ziglang.org> 2023-09-29 00:32:43-07:00
src