| ... | @@ -316,36 +316,30 @@ pub fn fetchAndAddDependencies( | ... | @@ -316,36 +316,30 @@ pub fn fetchAndAddDependencies( |
| 316 | | 316 | |
| 317 | // Directories do not provide a hash in build.zig.zon. | 317 | // Directories do not provide a hash in build.zig.zon. |
| 318 | // Hash the path to the module rather than its contents. | 318 | // Hash the path to the module rather than its contents. |
| 319 | if (fetch_location == .directory) { | 319 | const sub_mod, const found_existing = if (fetch_location == .directory) |
| 320 | if (dep.hash != null) { | 320 | try getDirectoryModule(gpa, fetch_location, directory, all_modules, dep, report) |
| 321 | return report.fail(dep.hash_tok, "hash not allowed for directory package", .{}); | 321 | else |
| 322 | } | 322 | try getCachedPackage( |
| 323 | const hex_digest = Manifest.hexDigest(try computePathHash(gpa, directory, fetch_location.directory)); | 323 | gpa, |
| 324 | dep.hash = try gpa.dupe(u8, &hex_digest); | | |
| 325 | } | | |
| 326 | | | |
| 327 | const sub_mod, const found_existing = try getCachedPackage( | | |
| 328 | arena, | | |
| 329 | fetch_location, | | |
| 330 | global_cache_directory, | | |
| 331 | dep.*, | | |
| 332 | all_modules, | | |
| 333 | root_prog_node, | | |
| 334 | ) orelse .{ | | |
| 335 | try fetchAndUnpack( | | |
| 336 | fetch_location, | | |
| 337 | thread_pool, | | |
| 338 | http_client, | | |
| 339 | directory, | | |
| 340 | global_cache_directory, | 324 | global_cache_directory, |
| 341 | dep.*, | 325 | dep.*, |
| 342 | report, | | |
| 343 | all_modules, | 326 | all_modules, |
| 344 | root_prog_node, | 327 | root_prog_node, |
| 345 | name, | 328 | ) orelse .{ |
| 346 | ), | 329 | try fetchAndUnpack( |
| 347 | false, | 330 | fetch_location, |
| 348 | }; | 331 | thread_pool, |
| | 332 | http_client, |
| | 333 | directory, |
| | 334 | global_cache_directory, |
| | 335 | dep.*, |
| | 336 | report, |
| | 337 | all_modules, |
| | 338 | root_prog_node, |
| | 339 | name, |
| | 340 | ), |
| | 341 | false, |
| | 342 | }; |
| 349 | | 343 | |
| 350 | assert(dep.hash != null); | 344 | assert(dep.hash != null); |
| 351 | | 345 | |
| ... | @@ -576,14 +570,6 @@ const FetchLocation = union(enum) { | ... | @@ -576,14 +570,6 @@ const FetchLocation = union(enum) { |
| 576 | .resource = .{ .file = try root_dir.handle.openFile(file, .{}) }, | 570 | .resource = .{ .file = try root_dir.handle.openFile(file, .{}) }, |
| 577 | }; | 571 | }; |
| 578 | }, | 572 | }, |
| 579 | .directory => |dir| { | | |
| 580 | const owned_path = try gpa.dupe(u8, dir); | | |
| 581 | errdefer gpa.free(owned_path); | | |
| 582 | return .{ | | |
| 583 | .path = owned_path, | | |
| 584 | .resource = .{ .directory = try root_dir.handle.openIterableDir(dir, .{}) }, | | |
| 585 | }; | | |
| 586 | }, | | |
| 587 | .http_request => |uri| { | 573 | .http_request => |uri| { |
| 588 | var h = std.http.Headers{ .allocator = gpa }; | 574 | var h = std.http.Headers{ .allocator = gpa }; |
| 589 | defer h.deinit(); | 575 | defer h.deinit(); |
| ... | @@ -606,6 +592,7 @@ const FetchLocation = union(enum) { | ... | @@ -606,6 +592,7 @@ const FetchLocation = union(enum) { |
| 606 | .resource = .{ .http_request = req }, | 592 | .resource = .{ .http_request = req }, |
| 607 | }; | 593 | }; |
| 608 | }, | 594 | }, |
| | 595 | .directory => unreachable, // Directories do not require fetching |
| 609 | } | 596 | } |
| 610 | } | 597 | } |
| 611 | }; | 598 | }; |
| ... | @@ -614,7 +601,6 @@ const ReadableResource = struct { | ... | @@ -614,7 +601,6 @@ const ReadableResource = struct { |
| 614 | path: []const u8, | 601 | path: []const u8, |
| 615 | resource: union(enum) { | 602 | resource: union(enum) { |
| 616 | file: fs.File, | 603 | file: fs.File, |
| 617 | directory: fs.IterableDir, | | |
| 618 | http_request: std.http.Client.Request, | 604 | http_request: std.http.Client.Request, |
| 619 | }, | 605 | }, |
| 620 | | 606 | |
| ... | @@ -625,20 +611,12 @@ const ReadableResource = struct { | ... | @@ -625,20 +611,12 @@ const ReadableResource = struct { |
| 625 | rr: *ReadableResource, | 611 | rr: *ReadableResource, |
| 626 | allocator: Allocator, | 612 | allocator: Allocator, |
| 627 | thread_pool: *ThreadPool, | 613 | thread_pool: *ThreadPool, |
| 628 | root_dir: Compilation.Directory, | | |
| 629 | global_cache_directory: Compilation.Directory, | 614 | global_cache_directory: Compilation.Directory, |
| 630 | dep: Manifest.Dependency, | 615 | dep: Manifest.Dependency, |
| 631 | report: Report, | 616 | report: Report, |
| 632 | pkg_prog_node: *std.Progress.Node, | 617 | pkg_prog_node: *std.Progress.Node, |
| 633 | ) !PackageLocation { | 618 | ) !PackageLocation { |
| 634 | switch (rr.resource) { | 619 | switch (rr.resource) { |
| 635 | .directory => { | | |
| 636 | return .{ | | |
| 637 | .hash = try computePathHash(allocator, root_dir, rr.path), | | |
| 638 | .root_src_dir_path = try allocator.dupe(u8, rr.path), | | |
| 639 | .root_dir = root_dir, | | |
| 640 | }; | | |
| 641 | }, | | |
| 642 | inline .file, .http_request => |*r| { | 620 | inline .file, .http_request => |*r| { |
| 643 | const s = fs.path.sep_str; | 621 | const s = fs.path.sep_str; |
| 644 | const rand_int = std.crypto.random.int(u64); | 622 | const rand_int = std.crypto.random.int(u64); |
| ... | @@ -710,8 +688,7 @@ const ReadableResource = struct { | ... | @@ -710,8 +688,7 @@ const ReadableResource = struct { |
| 710 | | 688 | |
| 711 | return .{ | 689 | return .{ |
| 712 | .hash = actual_hash, | 690 | .hash = actual_hash, |
| 713 | .root_src_dir_path = relative_unpacked_path, | 691 | .relative_unpacked_path = relative_unpacked_path, |
| 714 | .root_dir = global_cache_directory, | | |
| 715 | }; | 692 | }; |
| 716 | }, | 693 | }, |
| 717 | } | 694 | } |
| ... | @@ -727,7 +704,6 @@ const ReadableResource = struct { | ... | @@ -727,7 +704,6 @@ const ReadableResource = struct { |
| 727 | // TODO: Handle case of chunked content-length | 704 | // TODO: Handle case of chunked content-length |
| 728 | .http_request => |req| return req.response.content_length, | 705 | .http_request => |req| return req.response.content_length, |
| 729 | .file => |f| return (try f.metadata()).size(), | 706 | .file => |f| return (try f.metadata()).size(), |
| 730 | .directory => unreachable, | | |
| 731 | } | 707 | } |
| 732 | } | 708 | } |
| 733 | | 709 | |
| ... | @@ -737,7 +713,6 @@ const ReadableResource = struct { | ... | @@ -737,7 +713,6 @@ const ReadableResource = struct { |
| 737 | return fileTypeFromPath(rr.path) orelse | 713 | return fileTypeFromPath(rr.path) orelse |
| 738 | return report.fail(dep.location_tok, "Unknown file type", .{}); | 714 | return report.fail(dep.location_tok, "Unknown file type", .{}); |
| 739 | }, | 715 | }, |
| 740 | .directory => return error.IsDir, | | |
| 741 | .http_request => |req| { | 716 | .http_request => |req| { |
| 742 | const content_type = req.response.headers.getFirstValue("Content-Type") orelse | 717 | const content_type = req.response.headers.getFirstValue("Content-Type") orelse |
| 743 | return report.fail(dep.location_tok, "Missing 'Content-Type' header", .{}); | 718 | return report.fail(dep.location_tok, "Missing 'Content-Type' header", .{}); |
| ... | @@ -793,7 +768,6 @@ const ReadableResource = struct { | ... | @@ -793,7 +768,6 @@ const ReadableResource = struct { |
| 793 | gpa.free(rr.path); | 768 | gpa.free(rr.path); |
| 794 | switch (rr.resource) { | 769 | switch (rr.resource) { |
| 795 | .file => |file| file.close(), | 770 | .file => |file| file.close(), |
| 796 | .directory => |*dir| dir.close(), | | |
| 797 | .http_request => |*req| req.deinit(), | 771 | .http_request => |*req| req.deinit(), |
| 798 | } | 772 | } |
| 799 | rr.* = undefined; | 773 | rr.* = undefined; |
| ... | @@ -804,11 +778,10 @@ pub const PackageLocation = struct { | ... | @@ -804,11 +778,10 @@ pub const PackageLocation = struct { |
| 804 | /// For packages that require unpacking, this is the hash of the package contents. | 778 | /// For packages that require unpacking, this is the hash of the package contents. |
| 805 | /// For directories, this is the hash of the absolute file path. | 779 | /// For directories, this is the hash of the absolute file path. |
| 806 | hash: [Manifest.Hash.digest_length]u8, | 780 | hash: [Manifest.Hash.digest_length]u8, |
| 807 | root_src_dir_path: []const u8, | 781 | relative_unpacked_path: []const u8, |
| 808 | root_dir: Compilation.Directory, | | |
| 809 | | 782 | |
| 810 | pub fn deinit(pl: *PackageLocation, allocator: Allocator) void { | 783 | pub fn deinit(pl: *PackageLocation, allocator: Allocator) void { |
| 811 | allocator.free(pl.root_src_dir_path); | 784 | allocator.free(pl.relative_unpacked_path); |
| 812 | pl.* = undefined; | 785 | pl.* = undefined; |
| 813 | } | 786 | } |
| 814 | }; | 787 | }; |
| ... | @@ -874,19 +847,11 @@ fn ProgressReader(comptime ReaderType: type) type { | ... | @@ -874,19 +847,11 @@ fn ProgressReader(comptime ReaderType: type) type { |
| 874 | /// (i.e. whether or not its transitive dependencies have been fetched). | 847 | /// (i.e. whether or not its transitive dependencies have been fetched). |
| 875 | fn getCachedPackage( | 848 | fn getCachedPackage( |
| 876 | gpa: Allocator, | 849 | gpa: Allocator, |
| 877 | fetch_location: FetchLocation, | | |
| 878 | global_cache_directory: Compilation.Directory, | 850 | global_cache_directory: Compilation.Directory, |
| 879 | dep: Manifest.Dependency, | 851 | dep: Manifest.Dependency, |
| 880 | all_modules: *AllModules, | 852 | all_modules: *AllModules, |
| 881 | root_prog_node: *std.Progress.Node, | 853 | root_prog_node: *std.Progress.Node, |
| 882 | ) !?struct { DependencyModule, bool } { | 854 | ) !?struct { DependencyModule, bool } { |
| 883 | // There is no fixed location to check for directory modules. | | |
| 884 | // Instead, check whether it is already listed in all_modules. | | |
| 885 | if (fetch_location == .directory) { | | |
| 886 | const hex_digest = dep.hash.?[0..hex_multihash_len]; | | |
| 887 | return if (all_modules.get(hex_digest.*)) |mod| .{ mod.?, true } else null; | | |
| 888 | } | | |
| 889 | | | |
| 890 | const s = fs.path.sep_str; | 855 | const s = fs.path.sep_str; |
| 891 | // Check if the expected_hash is already present in the global package | 856 | // Check if the expected_hash is already present in the global package |
| 892 | // cache, and thereby avoid both fetching and unpacking. | 857 | // cache, and thereby avoid both fetching and unpacking. |
| ... | @@ -912,34 +877,60 @@ fn getCachedPackage( | ... | @@ -912,34 +877,60 @@ fn getCachedPackage( |
| 912 | root_prog_node.completeOne(); | 877 | root_prog_node.completeOne(); |
| 913 | | 878 | |
| 914 | const is_zig_mod = if (pkg_dir.access(build_zig_basename, .{})) |_| true else |_| false; | 879 | const is_zig_mod = if (pkg_dir.access(build_zig_basename, .{})) |_| true else |_| false; |
| | 880 | const basename = if (is_zig_mod) build_zig_basename else ""; |
| | 881 | const pkg = try createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, basename); |
| 915 | | 882 | |
| 916 | const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}); | 883 | const module: DependencyModule = if (is_zig_mod) |
| 917 | errdefer gpa.free(build_root); | 884 | .{ .zig_pkg = pkg } |
| 918 | | 885 | else |
| 919 | const ptr = try gpa.create(Package); | 886 | .{ .non_zig_pkg = pkg }; |
| 920 | errdefer gpa.destroy(ptr); | | |
| 921 | | 887 | |
| 922 | const owned_src_path = if (is_zig_mod) try gpa.dupe(u8, build_zig_basename) else ""; | 888 | try all_modules.put(gpa, hex_digest.*, module); |
| 923 | errdefer gpa.free(owned_src_path); | 889 | return .{ module, false }; |
| | 890 | } |
| 924 | | 891 | |
| 925 | ptr.* = .{ | 892 | return null; |
| 926 | .root_src_directory = .{ | 893 | } |
| 927 | .path = build_root, | | |
| 928 | .handle = pkg_dir, | | |
| 929 | }, | | |
| 930 | .root_src_directory_owned = true, | | |
| 931 | .root_src_path = owned_src_path, | | |
| 932 | }; | | |
| 933 | | 894 | |
| 934 | gop.value_ptr.* = if (is_zig_mod) | 895 | fn getDirectoryModule( |
| 935 | .{ .zig_pkg = ptr } | 896 | gpa: Allocator, |
| 936 | else | 897 | fetch_location: FetchLocation, |
| 937 | .{ .non_zig_pkg = ptr }; | 898 | directory: Compilation.Directory, |
| | 899 | all_modules: *AllModules, |
| | 900 | dep: *Manifest.Dependency, |
| | 901 | report: Report, |
| | 902 | ) !struct { DependencyModule, bool } { |
| | 903 | assert(fetch_location == .directory); |
| 938 | | 904 | |
| 939 | return .{ gop.value_ptr.*.?, false }; | 905 | if (dep.hash != null) { |
| | 906 | return report.fail(dep.hash_tok, "hash not allowed for directory package", .{}); |
| 940 | } | 907 | } |
| 941 | | 908 | |
| 942 | return null; | 909 | const hash = try computePathHash(gpa, directory, fetch_location.directory); |
| | 910 | const hex_digest = Manifest.hexDigest(hash); |
| | 911 | dep.hash = try gpa.dupe(u8, &hex_digest); |
| | 912 | |
| | 913 | // There is no fixed location to check for directory modules. |
| | 914 | // Instead, check whether it is already listed in all_modules. |
| | 915 | if (all_modules.get(hex_digest)) |mod| return .{ mod.?, true }; |
| | 916 | |
| | 917 | var pkg_dir = directory.handle.openDir(fetch_location.directory, .{}) catch |err| switch (err) { |
| | 918 | error.FileNotFound => return report.fail(dep.location_tok, "File not found: {s}", .{fetch_location.directory}), |
| | 919 | else => |e| return e, |
| | 920 | }; |
| | 921 | defer pkg_dir.close(); |
| | 922 | |
| | 923 | const is_zig_mod = if (pkg_dir.access(build_zig_basename, .{})) |_| true else |_| false; |
| | 924 | const basename = if (is_zig_mod) build_zig_basename else ""; |
| | 925 | |
| | 926 | const pkg = try createWithDir(gpa, directory, fetch_location.directory, basename); |
| | 927 | const module: DependencyModule = if (is_zig_mod) |
| | 928 | .{ .zig_pkg = pkg } |
| | 929 | else |
| | 930 | .{ .non_zig_pkg = pkg }; |
| | 931 | |
| | 932 | try all_modules.put(gpa, hex_digest, module); |
| | 933 | return .{ module, false }; |
| 943 | } | 934 | } |
| 944 | | 935 | |
| 945 | fn fetchAndUnpack( | 936 | fn fetchAndUnpack( |
| ... | @@ -956,6 +947,8 @@ fn fetchAndUnpack( | ... | @@ -956,6 +947,8 @@ fn fetchAndUnpack( |
| 956 | /// is only intended to be human-readable for progress reporting. | 947 | /// is only intended to be human-readable for progress reporting. |
| 957 | name_for_prog: []const u8, | 948 | name_for_prog: []const u8, |
| 958 | ) !DependencyModule { | 949 | ) !DependencyModule { |
| | 950 | assert(fetch_location == .file or fetch_location == .http_request); |
| | 951 | |
| 959 | const gpa = http_client.allocator; | 952 | const gpa = http_client.allocator; |
| 960 | | 953 | |
| 961 | var pkg_prog_node = root_prog_node.start(name_for_prog, 0); | 954 | var pkg_prog_node = root_prog_node.start(name_for_prog, 0); |
| ... | @@ -966,51 +959,47 @@ fn fetchAndUnpack( | ... | @@ -966,51 +959,47 @@ fn fetchAndUnpack( |
| 966 | var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep, report); | 959 | var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep, report); |
| 967 | defer readable_resource.deinit(gpa); | 960 | defer readable_resource.deinit(gpa); |
| 968 | | 961 | |
| 969 | var package_location = try readable_resource.unpack(gpa, thread_pool, directory, global_cache_directory, dep, report, &pkg_prog_node); | 962 | var package_location = try readable_resource.unpack(gpa, thread_pool, global_cache_directory, dep, report, &pkg_prog_node); |
| 970 | defer package_location.deinit(gpa); | 963 | defer package_location.deinit(gpa); |
| 971 | | 964 | |
| 972 | const actual_hex = Manifest.hexDigest(package_location.hash); | 965 | const actual_hex = Manifest.hexDigest(package_location.hash); |
| 973 | if (readable_resource.resource != .directory) { | 966 | if (dep.hash) |h| { |
| 974 | if (dep.hash) |h| { | 967 | if (!mem.eql(u8, h, &actual_hex)) { |
| 975 | if (!mem.eql(u8, h, &actual_hex)) { | 968 | return report.fail(dep.hash_tok, "hash mismatch: expected: {s}, found: {s}", .{ |
| 976 | return report.fail(dep.hash_tok, "hash mismatch: expected: {s}, found: {s}", .{ | 969 | h, actual_hex, |
| 977 | h, actual_hex, | | |
| 978 | }); | | |
| 979 | } | | |
| 980 | } else { | | |
| 981 | const file_path = try report.directory.join(gpa, &.{Manifest.basename}); | | |
| 982 | defer gpa.free(file_path); | | |
| 983 | | | |
| 984 | const eb = report.error_bundle; | | |
| 985 | const notes_len = 1; | | |
| 986 | try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{ | | |
| 987 | .tok = dep.location_tok, | | |
| 988 | .off = 0, | | |
| 989 | .msg = "dependency is missing hash field", | | |
| 990 | }); | 970 | }); |
| 991 | const notes_start = try eb.reserveNotes(notes_len); | | |
| 992 | eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ | | |
| 993 | .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}), | | |
| 994 | })); | | |
| 995 | return error.PackageFetchFailed; | | |
| 996 | } | 971 | } |
| | 972 | } else { |
| | 973 | const file_path = try report.directory.join(gpa, &.{Manifest.basename}); |
| | 974 | defer gpa.free(file_path); |
| | 975 | |
| | 976 | const eb = report.error_bundle; |
| | 977 | const notes_len = 1; |
| | 978 | try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{ |
| | 979 | .tok = dep.location_tok, |
| | 980 | .off = 0, |
| | 981 | .msg = "dependency is missing hash field", |
| | 982 | }); |
| | 983 | const notes_start = try eb.reserveNotes(notes_len); |
| | 984 | eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ |
| | 985 | .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}), |
| | 986 | })); |
| | 987 | return error.PackageFetchFailed; |
| 997 | } | 988 | } |
| 998 | | 989 | |
| 999 | const build_zig_path = try std.fs.path.join(gpa, &.{ package_location.root_src_dir_path, build_zig_basename }); | 990 | const build_zig_path = try fs.path.join(gpa, &.{ package_location.relative_unpacked_path, build_zig_basename }); |
| 1000 | defer gpa.free(build_zig_path); | 991 | defer gpa.free(build_zig_path); |
| 1001 | | 992 | |
| 1002 | package_location.root_dir.handle.access(build_zig_path, .{}) catch |err| switch (err) { | 993 | const is_zig_mod = if (global_cache_directory.handle.access(build_zig_path, .{})) |_| true else |_| false; |
| 1003 | error.FileNotFound => { | 994 | const basename = if (is_zig_mod) build_zig_basename else ""; |
| 1004 | const module = try createWithDir(gpa, package_location.root_dir, package_location.root_src_dir_path, ""); | 995 | const pkg = try createWithDir(gpa, global_cache_directory, package_location.relative_unpacked_path, basename); |
| 1005 | try all_modules.put(gpa, actual_hex, .{ .non_zig_pkg = module }); | 996 | const module: DependencyModule = if (is_zig_mod) |
| 1006 | return .{ .non_zig_pkg = module }; | 997 | .{ .zig_pkg = pkg } |
| 1007 | }, | 998 | else |
| 1008 | else => return err, | 999 | .{ .non_zig_pkg = pkg }; |
| 1009 | }; | | |
| 1010 | | 1000 | |
| 1011 | const module = try createWithDir(gpa, package_location.root_dir, package_location.root_src_dir_path, build_zig_basename); | 1001 | try all_modules.put(gpa, actual_hex, module); |
| 1012 | try all_modules.put(gpa, actual_hex, .{ .zig_pkg = module }); | 1002 | return module; |
| 1013 | return .{ .zig_pkg = module }; | | |
| 1014 | } | 1003 | } |
| 1015 | | 1004 | |
| 1016 | fn unpackTarball( | 1005 | fn unpackTarball( |
Adam Goertz <adambgoertz@gmail.com> 2023-09-28 00:25:12+00:00
Andrew Kelley <andrew@ziglang.org> 2023-09-29 00:32:43-07:00
src