Commit ef9966c9855dd855afda767f212abec6e5a36307 | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-10-01 23:05:01-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2023-10-02 17:02:25-07:00
log	ef9966c9855dd855afda767f212abec6e5a36307
tree	4f23d4468f3d9d44c16868becad6b0c6bb9083d8
parent	309c53295f26999065e4dc76cef4d90f8d85fb38

introduce the 'zig fetch' command + symlink support

zig fetch [options] <url>
zig fetch [options] <path>

Fetches a package which is found at <url> or <path> into the global
cache directory, printing the package hash to stdout.

Closes #16972
Related to #14280

Additionally, this commit:

* Adds uncompressed .tar support to package fetching
* Introduces symlink support to package fetching

4 files changed, 302 insertions(+), 107 deletions(-)

lib
- std
  - tar.zig
src
- Package.zig
- Package
  - hash.zig
- main.zig

lib/std/tar.zig+1-1

...	@@ -210,7 +210,7 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi	...	@@ -210,7 +210,7 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
210	while (true) {	210	while (true) {
211	const temp = try buffer.readChunk(reader, @intCast(rounded_file_size + 512 - file_off));	211	const temp = try buffer.readChunk(reader, @intCast(rounded_file_size + 512 - file_off));
212	if (temp.len == 0) return error.UnexpectedEndOfStream;	212	if (temp.len == 0) return error.UnexpectedEndOfStream;
213	const slice = temp[0..@as(usize, @intCast(@min(file_size - file_off, temp.len)))];	213	const slice = temp[0..@intCast(@min(file_size - file_off, temp.len))];
214	try file.writeAll(slice);	214	try file.writeAll(slice);
215		215
216	file_off += slice.len;	216	file_off += slice.len;

src/Package.zig+141-95

...	@@ -15,10 +15,10 @@ const Compilation = @import("Compilation.zig");	...	@@ -15,10 +15,10 @@ const Compilation = @import("Compilation.zig");
15	const Module = @import("Module.zig");	15	const Module = @import("Module.zig");
16	const Cache = std.Build.Cache;	16	const Cache = std.Build.Cache;
17	const build_options = @import("build_options");	17	const build_options = @import("build_options");
18	const Manifest = @import("Manifest.zig");
19	const git = @import("git.zig");	18	const git = @import("git.zig");
20	const computePackageHash = @import("Package/hash.zig").compute;	19	const computePackageHash = @import("Package/hash.zig").compute;
21		20
		21	pub const Manifest = @import("Manifest.zig");
22	pub const Table = std.StringHashMapUnmanaged(*Package);	22	pub const Table = std.StringHashMapUnmanaged(*Package);
23		23
24	root_src_directory: Compilation.Directory,	24	root_src_directory: Compilation.Directory,
...	@@ -454,8 +454,8 @@ pub fn createFilePkg(	...	@@ -454,8 +454,8 @@ pub fn createFilePkg(
454	return createWithDir(gpa, cache_directory, o_dir_sub_path, basename);	454	return createWithDir(gpa, cache_directory, o_dir_sub_path, basename);
455	}	455	}
456		456
457	const Report = struct {	457	pub const Report = struct {
458	ast: *const std.zig.Ast,	458	ast: ?*const std.zig.Ast,
459	directory: Compilation.Directory,	459	directory: Compilation.Directory,
460	error_bundle: *std.zig.ErrorBundle.Wip,	460	error_bundle: *std.zig.ErrorBundle.Wip,
461		461
...	@@ -465,6 +465,7 @@ const Report = struct {	...	@@ -465,6 +465,7 @@ const Report = struct {
465	comptime fmt_string: []const u8,	465	comptime fmt_string: []const u8,
466	fmt_args: anytype,	466	fmt_args: anytype,
467	) error{ PackageFetchFailed, OutOfMemory } {	467	) error{ PackageFetchFailed, OutOfMemory } {
		468	const ast = report.ast orelse main.fatal(fmt_string, fmt_args);
468	const gpa = report.error_bundle.gpa;	469	const gpa = report.error_bundle.gpa;
469		470
470	const file_path = try report.directory.join(gpa, &.{Manifest.basename});	471	const file_path = try report.directory.join(gpa, &.{Manifest.basename});
...	@@ -473,7 +474,7 @@ const Report = struct {	...	@@ -473,7 +474,7 @@ const Report = struct {
473	const msg = try std.fmt.allocPrint(gpa, fmt_string, fmt_args);	474	const msg = try std.fmt.allocPrint(gpa, fmt_string, fmt_args);
474	defer gpa.free(msg);	475	defer gpa.free(msg);
475		476
476	try addErrorMessage(report.ast.*, file_path, report.error_bundle, 0, .{	477	try addErrorMessage(ast.*, file_path, report.error_bundle, 0, .{
477	.tok = tok,	478	.tok = tok,
478	.off = 0,	479	.off = 0,
479	.msg = msg,	480	.msg = msg,
...	@@ -482,6 +483,18 @@ const Report = struct {	...	@@ -482,6 +483,18 @@ const Report = struct {
482	return error.PackageFetchFailed;	483	return error.PackageFetchFailed;
483	}	484	}
484		485
		486	fn addErrorWithNotes(
		487	report: Report,
		488	notes_len: u32,
		489	msg: Manifest.ErrorMessage,
		490	) error{OutOfMemory}!void {
		491	const ast = report.ast orelse main.fatal("{s}", .{msg.msg});
		492	const gpa = report.error_bundle.gpa;
		493	const file_path = try report.directory.join(gpa, &.{Manifest.basename});
		494	defer gpa.free(file_path);
		495	return addErrorMessage(ast.*, file_path, report.error_bundle, notes_len, msg);
		496	}
		497
485	fn addErrorMessage(	498	fn addErrorMessage(
486	ast: std.zig.Ast,	499	ast: std.zig.Ast,
487	file_path: []const u8,	500	file_path: []const u8,
...	@@ -508,7 +521,7 @@ const Report = struct {	...	@@ -508,7 +521,7 @@ const Report = struct {
508	}	521	}
509	};	522	};
510		523
511	const FetchLocation = union(enum) {	524	pub const FetchLocation = union(enum) {
512	/// The relative path to a file or directory.	525	/// The relative path to a file or directory.
513	/// This may be a file that requires unpacking (such as a .tar.gz),	526	/// This may be a file that requires unpacking (such as a .tar.gz),
514	/// or the path to the root directory of a package.	527	/// or the path to the root directory of a package.
...	@@ -517,30 +530,27 @@ const FetchLocation = union(enum) {	...	@@ -517,30 +530,27 @@ const FetchLocation = union(enum) {
517	http_request: std.Uri,	530	http_request: std.Uri,
518	git_request: std.Uri,	531	git_request: std.Uri,
519		532
520	pub fn init(gpa: Allocator, dep: Manifest.Dependency, root_dir: Compilation.Directory, report: Report) !FetchLocation {	533	pub fn init(
		534	gpa: Allocator,
		535	dep: Manifest.Dependency,
		536	root_dir: Compilation.Directory,
		537	report: Report,
		538	) !FetchLocation {
521	switch (dep.location) {	539	switch (dep.location) {
522	.url => \|url\| {	540	.url => \|url\| {
523	const uri = std.Uri.parse(url) catch \|err\| switch (err) {	541	const uri = std.Uri.parse(url) catch \|err\| switch (err) {
524	error.UnexpectedCharacter => return report.fail(dep.location_tok, "failed to parse dependency location as URI", .{}),	542	error.UnexpectedCharacter => return report.fail(dep.location_tok, "failed to parse dependency location as URI", .{}),
525	else => return err,	543	else => return err,
526	};	544	};
527	if (ascii.eqlIgnoreCase(uri.scheme, "file")) {	545	return initUri(uri, dep.location_tok, report);
528	return report.fail(dep.location_tok, "'file' scheme is not allowed for URLs. Use '.path' instead", .{});
529	} else if (ascii.eqlIgnoreCase(uri.scheme, "http") or ascii.eqlIgnoreCase(uri.scheme, "https")) {
530	return .{ .http_request = uri };
531	} else if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or ascii.eqlIgnoreCase(uri.scheme, "git+https")) {
532	return .{ .git_request = uri };
533	} else {
534	return report.fail(dep.location_tok, "Unsupported URL scheme: {s}", .{uri.scheme});
535	}
536	},	546	},
537	.path => \|path\| {	547	.path => \|path\| {
538	if (fs.path.isAbsolute(path)) {	548	if (fs.path.isAbsolute(path)) {
539	return report.fail(dep.location_tok, "Absolute paths are not allowed. Use a relative path instead", .{});	549	return report.fail(dep.location_tok, "absolute paths are not allowed. Use a relative path instead", .{});
540	}	550	}
541		551
542	const is_dir = isDirectory(root_dir, path) catch \|err\| switch (err) {	552	const is_dir = isDirectory(root_dir, path) catch \|err\| switch (err) {
543	error.FileNotFound => return report.fail(dep.location_tok, "File not found: {s}", .{path}),	553	error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{path}),
544	else => return err,	554	else => return err,
545	};	555	};
546		556
...	@@ -552,9 +562,21 @@ const FetchLocation = union(enum) {	...	@@ -552,9 +562,21 @@ const FetchLocation = union(enum) {
552	}	562	}
553	}	563	}
554		564
		565	pub fn initUri(uri: std.Uri, location_tok: std.zig.Ast.TokenIndex, report: Report) !FetchLocation {
		566	if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
		567	return report.fail(location_tok, "'file' scheme is not allowed for URLs. Use '.path' instead", .{});
		568	} else if (ascii.eqlIgnoreCase(uri.scheme, "http") or ascii.eqlIgnoreCase(uri.scheme, "https")) {
		569	return .{ .http_request = uri };
		570	} else if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or ascii.eqlIgnoreCase(uri.scheme, "git+https")) {
		571	return .{ .git_request = uri };
		572	} else {
		573	return report.fail(location_tok, "unsupported URL scheme: {s}", .{uri.scheme});
		574	}
		575	}
		576
555	pub fn deinit(f: *FetchLocation, gpa: Allocator) void {	577	pub fn deinit(f: *FetchLocation, gpa: Allocator) void {
556	switch (f.*) {	578	switch (f.*) {
557	inline .file, .directory => \|path\| gpa.free(path),	579	.file, .directory => \|path\| gpa.free(path),
558	.http_request, .git_request => {},	580	.http_request, .git_request => {},
559	}	581	}
560	f.* = undefined;	582	f.* = undefined;
...	@@ -565,7 +587,7 @@ const FetchLocation = union(enum) {	...	@@ -565,7 +587,7 @@ const FetchLocation = union(enum) {
565	gpa: Allocator,	587	gpa: Allocator,
566	root_dir: Compilation.Directory,	588	root_dir: Compilation.Directory,
567	http_client: *std.http.Client,	589	http_client: *std.http.Client,
568	dep: Manifest.Dependency,	590	dep_location_tok: std.zig.Ast.TokenIndex,
569	report: Report,	591	report: Report,
570	) !ReadableResource {	592	) !ReadableResource {
571	switch (f) {	593	switch (f) {
...	@@ -588,7 +610,7 @@ const FetchLocation = union(enum) {	...	@@ -588,7 +610,7 @@ const FetchLocation = union(enum) {
588	try req.wait();	610	try req.wait();
589		611
590	if (req.response.status != .ok) {	612	if (req.response.status != .ok) {
591	return report.fail(dep.location_tok, "Expected response status '200 OK' got '{} {s}'", .{	613	return report.fail(dep_location_tok, "expected response status '200 OK' got '{} {s}'", .{
592	@intFromEnum(req.response.status),	614	@intFromEnum(req.response.status),
593	req.response.status.phrase() orelse "",	615	req.response.status.phrase() orelse "",
594	});	616	});
...	@@ -607,7 +629,7 @@ const FetchLocation = union(enum) {	...	@@ -607,7 +629,7 @@ const FetchLocation = union(enum) {
607	session.discoverCapabilities(gpa, &redirect_uri) catch \|e\| switch (e) {	629	session.discoverCapabilities(gpa, &redirect_uri) catch \|e\| switch (e) {
608	error.Redirected => {	630	error.Redirected => {
609	defer gpa.free(redirect_uri);	631	defer gpa.free(redirect_uri);
610	return report.fail(dep.location_tok, "Repository moved to {s}", .{redirect_uri});	632	return report.fail(dep_location_tok, "repository moved to {s}", .{redirect_uri});
611	},	633	},
612	else => \|other\| return other,	634	else => \|other\| return other,
613	};	635	};
...	@@ -634,19 +656,16 @@ const FetchLocation = union(enum) {	...	@@ -634,19 +656,16 @@ const FetchLocation = union(enum) {
634	break :want_oid ref.peeled orelse ref.oid;	656	break :want_oid ref.peeled orelse ref.oid;
635	}	657	}
636	}	658	}
637	return report.fail(dep.location_tok, "Ref not found: {s}", .{want_ref});	659	return report.fail(dep_location_tok, "ref not found: {s}", .{want_ref});
638	};	660	};
639	if (uri.fragment == null) {	661	if (uri.fragment == null) {
640	const file_path = try report.directory.join(gpa, &.{Manifest.basename});
641	defer gpa.free(file_path);
642
643	const eb = report.error_bundle;
644	const notes_len = 1;	662	const notes_len = 1;
645	try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{	663	try report.addErrorWithNotes(notes_len, .{
646	.tok = dep.location_tok,	664	.tok = dep_location_tok,
647	.off = 0,	665	.off = 0,
648	.msg = "url field is missing an explicit ref",	666	.msg = "url field is missing an explicit ref",
649	});	667	});
		668	const eb = report.error_bundle;
650	const notes_start = try eb.reserveNotes(notes_len);	669	const notes_start = try eb.reserveNotes(notes_len);
651	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{	670	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
652	.msg = try eb.printString("try .url = \"{+/}#{}\",", .{ uri, std.fmt.fmtSliceHexLower(&want_oid) }),	671	.msg = try eb.printString("try .url = \"{+/}#{}\",", .{ uri, std.fmt.fmtSliceHexLower(&want_oid) }),
...	@@ -669,12 +688,13 @@ const FetchLocation = union(enum) {	...	@@ -669,12 +688,13 @@ const FetchLocation = union(enum) {
669	}	688	}
670	};	689	};
671		690
672	const ReadableResource = struct {	691	pub const ReadableResource = struct {
673	path: []const u8,	692	path: []const u8,
674	resource: union(enum) {	693	resource: union(enum) {
675	file: fs.File,	694	file: fs.File,
676	http_request: std.http.Client.Request,	695	http_request: std.http.Client.Request,
677	git_fetch_stream: git.Session.FetchStream,	696	git_fetch_stream: git.Session.FetchStream,
		697	dir: fs.IterableDir,
678	},	698	},
679		699
680	/// Unpack the package into the global cache directory.	700	/// Unpack the package into the global cache directory.
...	@@ -685,12 +705,12 @@ const ReadableResource = struct {	...	@@ -685,12 +705,12 @@ const ReadableResource = struct {
685	allocator: Allocator,	705	allocator: Allocator,
686	thread_pool: *ThreadPool,	706	thread_pool: *ThreadPool,
687	global_cache_directory: Compilation.Directory,	707	global_cache_directory: Compilation.Directory,
688	dep: Manifest.Dependency,	708	dep_location_tok: std.zig.Ast.TokenIndex,
689	report: Report,	709	report: Report,
690	pkg_prog_node: *std.Progress.Node,	710	pkg_prog_node: *std.Progress.Node,
691	) !PackageLocation {	711	) !PackageLocation {
692	switch (rr.resource) {	712	switch (rr.resource) {
693	inline .file, .http_request, .git_fetch_stream => \|*r\| {	713	inline .file, .http_request, .git_fetch_stream, .dir => \|*r, tag\| {
694	const s = fs.path.sep_str;	714	const s = fs.path.sep_str;
695	const rand_int = std.crypto.random.int(u64);	715	const rand_int = std.crypto.random.int(u64);
696	const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);	716	const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);
...	@@ -710,45 +730,58 @@ const ReadableResource = struct {	...	@@ -710,45 +730,58 @@ const ReadableResource = struct {
710	};	730	};
711	defer tmp_directory.closeAndFree(allocator);	731	defer tmp_directory.closeAndFree(allocator);
712		732
713	const opt_content_length = try rr.getSize();	733	if (tag != .dir) {
714		734	const opt_content_length = try rr.getSize();
715	var prog_reader: ProgressReader(@TypeOf(r.reader())) = .{	735
716	.child_reader = r.reader(),	736	var prog_reader: ProgressReader(@TypeOf(r.reader())) = .{
717	.prog_node = pkg_prog_node,	737	.child_reader = r.reader(),
718	.unit = if (opt_content_length) \|content_length\| unit: {	738	.prog_node = pkg_prog_node,
719	const kib = content_length / 1024;	739	.unit = if (opt_content_length) \|content_length\| unit: {
720	const mib = kib / 1024;	740	const kib = content_length / 1024;
721	if (mib > 0) {	741	const mib = kib / 1024;
722	pkg_prog_node.setEstimatedTotalItems(@intCast(mib));	742	if (mib > 0) {
723	pkg_prog_node.setUnit("MiB");	743	pkg_prog_node.setEstimatedTotalItems(@intCast(mib));
724	break :unit .mib;	744	pkg_prog_node.setUnit("MiB");
725	} else {	745	break :unit .mib;
726	pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib)));	746	} else {
727	pkg_prog_node.setUnit("KiB");	747	pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib)));
728	break :unit .kib;	748	pkg_prog_node.setUnit("KiB");
		749	break :unit .kib;
		750	}
		751	} else .any,
		752	};
		753
		754	switch (try rr.getFileType(dep_location_tok, report)) {
		755	.tar => try unpackTarball(prog_reader.reader(), tmp_directory.handle),
		756	.@"tar.gz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, std.compress.gzip),
		757	.@"tar.xz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, std.compress.xz),
		758	.git_pack => try unpackGitPack(allocator, &prog_reader, git.parseOid(rr.path) catch unreachable, tmp_directory.handle),
		759	}
		760	} else {
		761	// Recursive directory copy.
		762	var it = try r.walk(allocator);
		763	defer it.deinit();
		764	while (try it.next()) \|entry\| {
		765	switch (entry.kind) {
		766	.directory => try tmp_directory.handle.makePath(entry.path),
		767	.file => try r.dir.copyFile(
		768	entry.path,
		769	tmp_directory.handle,
		770	entry.path,
		771	.{},
		772	),
		773	.sym_link => {
		774	var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
		775	const link_name = try r.dir.readLink(entry.path, &buf);
		776	// TODO: if this would create a symlink to outside
		777	// the destination directory, fail with an error instead.
		778	try tmp_directory.handle.symLink(link_name, entry.path, .{});
		779	},
		780	else => return error.IllegalFileTypeInPackage,
729	}	781	}
730	} else .any,	782	}
731	};
732	pkg_prog_node.context.refresh();
733
734	switch (try rr.getFileType(dep, report)) {
735	.@"tar.gz" => try unpackTarball(allocator, prog_reader, tmp_directory.handle, std.compress.gzip),
736	// I have not checked what buffer sizes the xz decompression implementation uses
737	// by default, so the same logic applies for buffering the reader as for gzip.
738	.@"tar.xz" => try unpackTarball(allocator, prog_reader, tmp_directory.handle, std.compress.xz),
739	.git_pack => try unpackGitPack(allocator, &prog_reader, git.parseOid(rr.path) catch unreachable, tmp_directory.handle),
740	}	783	}
741		784
742	// Unpack completed - stop showing amount as progress
743	pkg_prog_node.setEstimatedTotalItems(0);
744	pkg_prog_node.setCompletedItems(0);
745	pkg_prog_node.context.refresh();
746
747	// TODO: delete files not included in the package prior to computing the package hash.
748	// for example, if the ini file has directives to include/not include certain files,
749	// apply those rules directly to the filesystem right here. This ensures that files
750	// not protected by the hash are not present on the file system.
751
752	break :h try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle });	785	break :h try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle });
753	};	786	};
754		787
...	@@ -769,6 +802,7 @@ const ReadableResource = struct {	...	@@ -769,6 +802,7 @@ const ReadableResource = struct {
769	}	802	}
770		803
771	const FileType = enum {	804	const FileType = enum {
		805	tar,
772	@"tar.gz",	806	@"tar.gz",
773	@"tar.xz",	807	@"tar.xz",
774	git_pack,	808	git_pack,
...	@@ -780,21 +814,28 @@ const ReadableResource = struct {	...	@@ -780,21 +814,28 @@ const ReadableResource = struct {
780	// TODO: Handle case of chunked content-length	814	// TODO: Handle case of chunked content-length
781	.http_request => \|req\| return req.response.content_length,	815	.http_request => \|req\| return req.response.content_length,
782	.git_fetch_stream => \|stream\| return stream.request.response.content_length,	816	.git_fetch_stream => \|stream\| return stream.request.response.content_length,
		817	.dir => unreachable,
783	}	818	}
784	}	819	}
785		820
786	pub fn getFileType(rr: ReadableResource, dep: Manifest.Dependency, report: Report) !FileType {	821	pub fn getFileType(
		822	rr: ReadableResource,
		823	dep_location_tok: std.zig.Ast.TokenIndex,
		824	report: Report,
		825	) !FileType {
787	switch (rr.resource) {	826	switch (rr.resource) {
788	.file => {	827	.file => {
789	return fileTypeFromPath(rr.path) orelse	828	return fileTypeFromPath(rr.path) orelse
790	return report.fail(dep.location_tok, "Unknown file type", .{});	829	return report.fail(dep_location_tok, "unknown file type", .{});
791	},	830	},
792	.http_request => \|req\| {	831	.http_request => \|req\| {
793	const content_type = req.response.headers.getFirstValue("Content-Type") orelse	832	const content_type = req.response.headers.getFirstValue("Content-Type") orelse
794	return report.fail(dep.location_tok, "Missing 'Content-Type' header", .{});	833	return report.fail(dep_location_tok, "missing 'Content-Type' header", .{});
795		834
796	// If the response has a different content type than the URI indicates, override	835	// If the response has a different content type than the URI indicates, override
797	// the previously assumed file type.	836	// the previously assumed file type.
		837	if (ascii.eqlIgnoreCase(content_type, "application/x-tar")) return .tar;
		838
798	return if (ascii.eqlIgnoreCase(content_type, "application/gzip") or	839	return if (ascii.eqlIgnoreCase(content_type, "application/gzip") or
799	ascii.eqlIgnoreCase(content_type, "application/x-gzip") or	840	ascii.eqlIgnoreCase(content_type, "application/x-gzip") or
800	ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))	841	ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))
...	@@ -805,22 +846,21 @@ const ReadableResource = struct {	...	@@ -805,22 +846,21 @@ const ReadableResource = struct {
805	// support gitlab tarball urls such as https://gitlab.com/<namespace>/<project>/-/archive/<sha>/<project>-<sha>.tar.gz	846	// support gitlab tarball urls such as https://gitlab.com/<namespace>/<project>/-/archive/<sha>/<project>-<sha>.tar.gz
806	// whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"'	847	// whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"'
807	const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse	848	const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse
808	return report.fail(dep.location_tok, "Missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});	849	return report.fail(dep_location_tok, "missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});
809	break :ty getAttachmentType(content_disposition) orelse	850	break :ty getAttachmentType(content_disposition) orelse
810	return report.fail(dep.location_tok, "Unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});	851	return report.fail(dep_location_tok, "unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});
811	} else return report.fail(dep.location_tok, "Unrecognized value for 'Content-Type' header: {s}", .{content_type});	852	} else return report.fail(dep_location_tok, "unrecognized value for 'Content-Type' header: {s}", .{content_type});
812	},	853	},
813	.git_fetch_stream => return .git_pack,	854	.git_fetch_stream => return .git_pack,
		855	.dir => unreachable,
814	}	856	}
815	}	857	}
816		858
817	fn fileTypeFromPath(file_path: []const u8) ?FileType {	859	fn fileTypeFromPath(file_path: []const u8) ?FileType {
818	return if (ascii.endsWithIgnoreCase(file_path, ".tar.gz"))	860	if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar;
819	.@"tar.gz"	861	if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz";
820	else if (ascii.endsWithIgnoreCase(file_path, ".tar.xz"))	862	if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz";
821	.@"tar.xz"	863	return null;
822	else
823	null;
824	}	864	}
825		865
826	fn getAttachmentType(content_disposition: []const u8) ?FileType {	866	fn getAttachmentType(content_disposition: []const u8) ?FileType {
...	@@ -847,6 +887,7 @@ const ReadableResource = struct {	...	@@ -847,6 +887,7 @@ const ReadableResource = struct {
847	.file => \|file\| file.close(),	887	.file => \|file\| file.close(),
848	.http_request => \|*req\| req.deinit(),	888	.http_request => \|*req\| req.deinit(),
849	.git_fetch_stream => \|*stream\| stream.deinit(),	889	.git_fetch_stream => \|*stream\| stream.deinit(),
		890	.dir => \|*dir\| dir.close(),
850	}	891	}
851	rr.* = undefined;	892	rr.* = undefined;
852	}	893	}
...	@@ -908,7 +949,7 @@ fn ProgressReader(comptime ReaderType: type) type {	...	@@ -908,7 +949,7 @@ fn ProgressReader(comptime ReaderType: type) type {
908	}	949	}
909	},	950	},
910	}	951	}
911	self.prog_node.context.maybeRefresh();	952	self.prog_node.activate();
912	return amt;	953	return amt;
913	}	954	}
914		955
...	@@ -993,7 +1034,7 @@ fn getDirectoryModule(	...	@@ -993,7 +1034,7 @@ fn getDirectoryModule(
993	if (all_modules.get(hex_digest)) \|mod\| return .{ mod.?, true };	1034	if (all_modules.get(hex_digest)) \|mod\| return .{ mod.?, true };
994		1035
995	var pkg_dir = directory.handle.openDir(fetch_location.directory, .{}) catch \|err\| switch (err) {	1036	var pkg_dir = directory.handle.openDir(fetch_location.directory, .{}) catch \|err\| switch (err) {
996	error.FileNotFound => return report.fail(dep.location_tok, "File not found: {s}", .{fetch_location.directory}),	1037	error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{fetch_location.directory}),
997	else => \|e\| return e,	1038	else => \|e\| return e,
998	};	1039	};
999	defer pkg_dir.close();	1040	defer pkg_dir.close();
...	@@ -1032,12 +1073,18 @@ fn fetchAndUnpack(	...	@@ -1032,12 +1073,18 @@ fn fetchAndUnpack(
1032	var pkg_prog_node = root_prog_node.start(name_for_prog, 0);	1073	var pkg_prog_node = root_prog_node.start(name_for_prog, 0);
1033	defer pkg_prog_node.end();	1074	defer pkg_prog_node.end();
1034	pkg_prog_node.activate();	1075	pkg_prog_node.activate();
1035	pkg_prog_node.context.refresh();
1036		1076
1037	var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep, report);	1077	var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep.location_tok, report);
1038	defer readable_resource.deinit(gpa);	1078	defer readable_resource.deinit(gpa);
1039		1079
1040	var package_location = try readable_resource.unpack(gpa, thread_pool, global_cache_directory, dep, report, &pkg_prog_node);	1080	var package_location = try readable_resource.unpack(
		1081	gpa,
		1082	thread_pool,
		1083	global_cache_directory,
		1084	dep.location_tok,
		1085	report,
		1086	&pkg_prog_node,
		1087	);
1041	defer package_location.deinit(gpa);	1088	defer package_location.deinit(gpa);
1042		1089
1043	const actual_hex = Manifest.hexDigest(package_location.hash);	1090	const actual_hex = Manifest.hexDigest(package_location.hash);
...	@@ -1048,16 +1095,13 @@ fn fetchAndUnpack(	...	@@ -1048,16 +1095,13 @@ fn fetchAndUnpack(
1048	});	1095	});
1049	}	1096	}
1050	} else {	1097	} else {
1051	const file_path = try report.directory.join(gpa, &.{Manifest.basename});
1052	defer gpa.free(file_path);
1053
1054	const eb = report.error_bundle;
1055	const notes_len = 1;	1098	const notes_len = 1;
1056	try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{	1099	try report.addErrorWithNotes(notes_len, .{
1057	.tok = dep.location_tok,	1100	.tok = dep.location_tok,
1058	.off = 0,	1101	.off = 0,
1059	.msg = "dependency is missing hash field",	1102	.msg = "dependency is missing hash field",
1060	});	1103	});
		1104	const eb = report.error_bundle;
1061	const notes_start = try eb.reserveNotes(notes_len);	1105	const notes_start = try eb.reserveNotes(notes_len);
1062	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{	1106	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
1063	.msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),	1107	.msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),
...	@@ -1080,18 +1124,22 @@ fn fetchAndUnpack(	...	@@ -1080,18 +1124,22 @@ fn fetchAndUnpack(
1080	return module;	1124	return module;
1081	}	1125	}
1082		1126
1083	fn unpackTarball(	1127	fn unpackTarballCompressed(
1084	gpa: Allocator,	1128	gpa: Allocator,
1085	reader: anytype,	1129	reader: anytype,
1086	out_dir: fs.Dir,	1130	out_dir: fs.Dir,
1087	comptime compression: type,	1131	comptime Compression: type,
1088	) !void {	1132	) !void {
1089	var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);	1133	var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1090		1134
1091	var decompress = try compression.decompress(gpa, br.reader());	1135	var decompress = try Compression.decompress(gpa, br.reader());
1092	defer decompress.deinit();	1136	defer decompress.deinit();
1093		1137
1094	try std.tar.pipeToFileSystem(out_dir, decompress.reader(), .{	1138	return unpackTarball(decompress.reader(), out_dir);
		1139	}
		1140
		1141	fn unpackTarball(reader: anytype, out_dir: fs.Dir) !void {
		1142	try std.tar.pipeToFileSystem(out_dir, reader, .{
1095	.strip_components = 1,	1143	.strip_components = 1,
1096	// TODO: we would like to set this to executable_bit_only, but two	1144	// TODO: we would like to set this to executable_bit_only, but two
1097	// things need to happen before that:	1145	// things need to happen before that:
...	@@ -1126,7 +1174,6 @@ fn unpackGitPack(	...	@@ -1126,7 +1174,6 @@ fn unpackGitPack(
1126	var index_prog_node = reader.prog_node.start("Index pack", 0);	1174	var index_prog_node = reader.prog_node.start("Index pack", 0);
1127	defer index_prog_node.end();	1175	defer index_prog_node.end();
1128	index_prog_node.activate();	1176	index_prog_node.activate();
1129	index_prog_node.context.refresh();
1130	var index_buffered_writer = std.io.bufferedWriter(index_file.writer());	1177	var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
1131	try git.indexPack(gpa, pack_file, index_buffered_writer.writer());	1178	try git.indexPack(gpa, pack_file, index_buffered_writer.writer());
1132	try index_buffered_writer.flush();	1179	try index_buffered_writer.flush();
...	@@ -1137,7 +1184,6 @@ fn unpackGitPack(	...	@@ -1137,7 +1184,6 @@ fn unpackGitPack(
1137	var checkout_prog_node = reader.prog_node.start("Checkout", 0);	1184	var checkout_prog_node = reader.prog_node.start("Checkout", 0);
1138	defer checkout_prog_node.end();	1185	defer checkout_prog_node.end();
1139	checkout_prog_node.activate();	1186	checkout_prog_node.activate();
1140	checkout_prog_node.context.refresh();
1141	var repository = try git.Repository.init(gpa, pack_file, index_file);	1187	var repository = try git.Repository.init(gpa, pack_file, index_file);
1142	defer repository.deinit();	1188	defer repository.deinit();
1143	try repository.checkout(out_dir, want_oid);	1189	try repository.checkout(out_dir, want_oid);

src/Package/hash.zig+33-11

 @@ -16,6 +16,11 @@ pub fn compute(thread_pool: *ThreadPool, pkg_dir: fs.IterableDir) ![Hash.digest_
     defer arena_instance.deinit();
     const arena = arena_instance.allocator();
+    // TODO: delete files not included in the package prior to computing the package hash.
+    // for example, if the ini file has directives to include/not include certain files,
+    // apply those rules directly to the filesystem right here. This ensures that files
+    // not protected by the hash are not present on the file system.
     // Collect all files, recursively, then sort.
     var all_files = std.ArrayList(*HashedFile).init(gpa);
     defer all_files.deinit();
 @@ -30,16 +35,18 @@ pub fn compute(thread_pool: *ThreadPool, pkg_dir: fs.IterableDir) ![Hash.digest_
         defer wait_group.wait();
         while (try walker.next()) |entry| {
-            switch (entry.kind) {
+            const kind: HashedFile.Kind = switch (entry.kind) {
                 .directory => continue,
-                .file => {},
+                .file => .file,
+                .sym_link => .sym_link,
                 else => return error.IllegalFileTypeInPackage,
-            }
+            };
             const hashed_file = try arena.create(HashedFile);
             const fs_path = try arena.dupe(u8, entry.path);
             hashed_file.* = .{
                 .fs_path = fs_path,
                 .normalized_path = try normalizePath(arena, fs_path),
+                .kind = kind,
                 .hash = undefined, // to be populated by the worker
                 .failure = undefined, // to be populated by the worker
             };
 @@ -70,8 +77,15 @@ const HashedFile = struct {
     normalized_path: []const u8,
     hash: [Hash.digest_length]u8,
     failure: Error!void,
+    kind: Kind,
-    const Error = fs.File.OpenError || fs.File.ReadError || fs.File.StatError;
+    const Error =
+        fs.File.OpenError ||
+        fs.File.ReadError ||
+        fs.File.StatError ||
+        fs.Dir.ReadLinkError;
+    const Kind = enum { file, sym_link };
     fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool {
         _ = context;
 @@ -104,15 +118,23 @@ fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void {
 fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
     var buf: [8000]u8 = undefined;
-    var file = try dir.openFile(hashed_file.fs_path, .{});
-    defer file.close();
     var hasher = Hash.init(.{});
     hasher.update(hashed_file.normalized_path);
-    hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) });
+    switch (hashed_file.kind) {
-    while (true) {
+        .file => {
-        const bytes_read = try file.read(&buf);
+            var file = try dir.openFile(hashed_file.fs_path, .{});
-        if (bytes_read == 0) break;
+            defer file.close();
-        hasher.update(buf[0..bytes_read]);
+            hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) });
+            while (true) {
+                const bytes_read = try file.read(&buf);
+                if (bytes_read == 0) break;
+                hasher.update(buf[0..bytes_read]);
+            }
+        },
+        .sym_link => {
+            const link_name = try dir.readLink(hashed_file.fs_path, &buf);
+            hasher.update(link_name);
+        },
     }
     hasher.final(&hashed_file.hash);
 }

src/main.zig+127

...	@@ -84,6 +84,7 @@ const normal_usage =	...	@@ -84,6 +84,7 @@ const normal_usage =
84	\\Commands:	84	\\Commands:
85	\\	85	\\
86	\\ build Build project from build.zig	86	\\ build Build project from build.zig
		87	\\ fetch Copy a package into global cache and print its hash
87	\\ init-exe Initialize a `zig build` application in the cwd	88	\\ init-exe Initialize a `zig build` application in the cwd
88	\\ init-lib Initialize a `zig build` library in the cwd	89	\\ init-lib Initialize a `zig build` library in the cwd
89	\\	90	\\
...	@@ -303,6 +304,8 @@ pub fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi	...	@@ -303,6 +304,8 @@ pub fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
303	return cmdFmt(gpa, arena, cmd_args);	304	return cmdFmt(gpa, arena, cmd_args);
304	} else if (mem.eql(u8, cmd, "objcopy")) {	305	} else if (mem.eql(u8, cmd, "objcopy")) {
305	return @import("objcopy.zig").cmdObjCopy(gpa, arena, cmd_args);	306	return @import("objcopy.zig").cmdObjCopy(gpa, arena, cmd_args);
		307	} else if (mem.eql(u8, cmd, "fetch")) {
		308	return cmdFetch(gpa, arena, cmd_args);
306	} else if (mem.eql(u8, cmd, "libc")) {	309	} else if (mem.eql(u8, cmd, "libc")) {
307	return cmdLibC(gpa, cmd_args);	310	return cmdLibC(gpa, cmd_args);
308	} else if (mem.eql(u8, cmd, "init-exe")) {	311	} else if (mem.eql(u8, cmd, "init-exe")) {
...	@@ -6589,3 +6592,127 @@ fn parseRcIncludes(arg: []const u8) Compilation.RcIncludes {	...	@@ -6589,3 +6592,127 @@ fn parseRcIncludes(arg: []const u8) Compilation.RcIncludes {
6589	return std.meta.stringToEnum(Compilation.RcIncludes, arg) orelse	6592	return std.meta.stringToEnum(Compilation.RcIncludes, arg) orelse
6590	fatal("unsupported rc includes type: '{s}'", .{arg});	6593	fatal("unsupported rc includes type: '{s}'", .{arg});
6591	}	6594	}
		6595
		6596	pub const usage_fetch =
		6597	\\Usage: zig fetch [options] <url>
		6598	\\Usage: zig fetch [options] <path>
		6599	\\
		6600	\\ Copy a package into the global cache and print its hash.
		6601	\\
		6602	\\Options:
		6603	\\ -h, --help Print this help and exit
		6604	\\ --global-cache-dir [path] Override path to global Zig cache directory
		6605	\\
		6606	;
		6607
		6608	fn cmdFetch(
		6609	gpa: Allocator,
		6610	arena: Allocator,
		6611	args: []const []const u8,
		6612	) !void {
		6613	var opt_url: ?[]const u8 = null;
		6614	var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");
		6615
		6616	{
		6617	var i: usize = 0;
		6618	while (i < args.len) : (i += 1) {
		6619	const arg = args[i];
		6620	if (mem.startsWith(u8, arg, "-")) {
		6621	if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
		6622	const stdout = io.getStdOut().writer();
		6623	try stdout.writeAll(usage_fetch);
		6624	return cleanExit();
		6625	} else if (mem.eql(u8, arg, "--global-cache-dir")) {
		6626	if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
		6627	i += 1;
		6628	override_global_cache_dir = args[i];
		6629	continue;
		6630	} else {
		6631	fatal("unrecognized parameter: '{s}'", .{arg});
		6632	}
		6633	} else if (opt_url != null) {
		6634	fatal("unexpected extra parameter: '{s}'", .{arg});
		6635	} else {
		6636	opt_url = arg;
		6637	}
		6638	}
		6639	}
		6640
		6641	const url = opt_url orelse fatal("missing url or path parameter", .{});
		6642
		6643	var thread_pool: ThreadPool = undefined;
		6644	try thread_pool.init(.{ .allocator = gpa });
		6645	defer thread_pool.deinit();
		6646
		6647	var http_client: std.http.Client = .{ .allocator = gpa };
		6648	defer http_client.deinit();
		6649
		6650	var progress: std.Progress = .{ .dont_print_on_dumb = true };
		6651	const root_prog_node = progress.start("Fetch", 0);
		6652	defer root_prog_node.end();
		6653
		6654	var report: Package.Report = .{
		6655	.ast = null,
		6656	.directory = undefined,
		6657	.error_bundle = undefined,
		6658	};
		6659
		6660	var global_cache_directory: Compilation.Directory = l: {
		6661	const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena);
		6662	break :l .{
		6663	.handle = try fs.cwd().makeOpenPath(p, .{}),
		6664	.path = p,
		6665	};
		6666	};
		6667	defer global_cache_directory.handle.close();
		6668
		6669	var readable_resource: Package.ReadableResource = rr: {
		6670	if (fs.cwd().openIterableDir(url, .{})) \|dir\| {
		6671	break :rr .{
		6672	.path = try gpa.dupe(u8, url),
		6673	.resource = .{ .dir = dir },
		6674	};
		6675	} else \|dir_err\| {
		6676	const file_err = if (dir_err == error.NotDir) e: {
		6677	if (fs.cwd().openFile(url, .{})) \|f\| {
		6678	break :rr .{
		6679	.path = try gpa.dupe(u8, url),
		6680	.resource = .{ .file = f },
		6681	};
		6682	} else \|err\| break :e err;
		6683	} else dir_err;
		6684
		6685	const uri = std.Uri.parse(url) catch \|uri_err\| {
		6686	fatal("'{s}' could not be recognized as a file path ({s}) or an URL ({s})", .{
		6687	url, @errorName(file_err), @errorName(uri_err),
		6688	});
		6689	};
		6690	const fetch_location = try Package.FetchLocation.initUri(uri, 0, report);
		6691	const cwd: Cache.Directory = .{
		6692	.handle = fs.cwd(),
		6693	.path = null,
		6694	};
		6695	break :rr try fetch_location.fetch(gpa, cwd, &http_client, 0, report);
		6696	}
		6697	};
		6698	defer readable_resource.deinit(gpa);
		6699
		6700	var package_location = try readable_resource.unpack(
		6701	gpa,
		6702	&thread_pool,
		6703	global_cache_directory,
		6704	0,
		6705	report,
		6706	root_prog_node,
		6707	);
		6708	defer package_location.deinit(gpa);
		6709
		6710	const hex_digest = Package.Manifest.hexDigest(package_location.hash);
		6711
		6712	progress.done = true;
		6713	progress.refresh();
		6714
		6715	try io.getStdOut().writeAll(hex_digest ++ "\n");
		6716
		6717	return cleanExit();
		6718	}