Commit df4853a6271b7963d77cafda7d6727274c6cbdaa | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-10-03 03:33:26-07:00
committer	GitHub <noreply@github.com> 2023-10-03 03:33:26-07:00
log	df4853a6271b7963d77cafda7d6727274c6cbdaa
tree	b00928173d4d73f74bfdaaae6b3021be71759eb8
parent	4930094e622c043d9be459abbfc5a7b203a22aa2
parent	573a13f8be24276a02761c30396fd75fc10ebdb0
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #17363 from ziglang/tar-symlinks

introduce the `zig fetch` subcommand and symlink support in zig packages

8 files changed, 681 insertions(+), 261 deletions(-)

CMakeLists.txt
lib
- std
  - fs.zig
  - tar.zig
  - zig
    ErrorBundle.zig
src
- Package.zig
- Package
  - hash.zig
- git.zig
- main.zig

CMakeLists.txt+1

...	@@ -527,6 +527,7 @@ set(ZIG_STAGE2_SOURCES	...	@@ -527,6 +527,7 @@ set(ZIG_STAGE2_SOURCES
527	"${CMAKE_SOURCE_DIR}/src/Liveness.zig"	527	"${CMAKE_SOURCE_DIR}/src/Liveness.zig"
528	"${CMAKE_SOURCE_DIR}/src/Module.zig"	528	"${CMAKE_SOURCE_DIR}/src/Module.zig"
529	"${CMAKE_SOURCE_DIR}/src/Package.zig"	529	"${CMAKE_SOURCE_DIR}/src/Package.zig"
		530	"${CMAKE_SOURCE_DIR}/src/Package/hash.zig"
530	"${CMAKE_SOURCE_DIR}/src/RangeSet.zig"	531	"${CMAKE_SOURCE_DIR}/src/RangeSet.zig"
531	"${CMAKE_SOURCE_DIR}/src/Sema.zig"	532	"${CMAKE_SOURCE_DIR}/src/Sema.zig"
532	"${CMAKE_SOURCE_DIR}/src/TypedValue.zig"	533	"${CMAKE_SOURCE_DIR}/src/TypedValue.zig"

lib/std/fs.zig+3-1

 @@ -2003,10 +2003,12 @@ pub const Dir = struct {
         return os.windows.CreateSymbolicLink(self.fd, sym_link_path_w, target_path_w, flags.is_directory);
     }
+    pub const ReadLinkError = os.ReadLinkError;
     /// Read value of a symbolic link.
     /// The return value is a slice of `buffer`, from index `0`.
     /// Asserts that the path parameter has no null bytes.
-    pub fn readLink(self: Dir, sub_path: []const u8, buffer: []u8) ![]u8 {
+    pub fn readLink(self: Dir, sub_path: []const u8, buffer: []u8) ReadLinkError![]u8 {
         if (builtin.os.tag == .wasi and !builtin.link_libc) {
             return self.readLinkWasi(sub_path, buffer);
         }

lib/std/tar.zig+69-7

...	@@ -3,8 +3,13 @@ pub const Options = struct {	...	@@ -3,8 +3,13 @@ pub const Options = struct {
3	strip_components: u32 = 0,	3	strip_components: u32 = 0,
4	/// How to handle the "mode" property of files from within the tar file.	4	/// How to handle the "mode" property of files from within the tar file.
5	mode_mode: ModeMode = .executable_bit_only,	5	mode_mode: ModeMode = .executable_bit_only,
		6	/// Provide this to receive detailed error messages.
		7	/// When this is provided, some errors which would otherwise be returned immediately
		8	/// will instead be added to this structure. The API user must check the errors
		9	/// in diagnostics to know whether the operation succeeded or failed.
		10	diagnostics: ?*Diagnostics = null,
6		11
7	const ModeMode = enum {	12	pub const ModeMode = enum {
8	/// The mode from the tar file is completely ignored. Files are created	13	/// The mode from the tar file is completely ignored. Files are created
9	/// with the default mode when creating files.	14	/// with the default mode when creating files.
10	ignore,	15	ignore,
...	@@ -13,12 +18,46 @@ pub const Options = struct {	...	@@ -13,12 +18,46 @@ pub const Options = struct {
13	/// Other bits of the mode are left as the default when creating files.	18	/// Other bits of the mode are left as the default when creating files.
14	executable_bit_only,	19	executable_bit_only,
15	};	20	};
		21
		22	pub const Diagnostics = struct {
		23	allocator: std.mem.Allocator,
		24	errors: std.ArrayListUnmanaged(Error) = .{},
		25
		26	pub const Error = union(enum) {
		27	unable_to_create_sym_link: struct {
		28	code: anyerror,
		29	file_name: []const u8,
		30	link_name: []const u8,
		31	},
		32	unsupported_file_type: struct {
		33	file_name: []const u8,
		34	file_type: Header.FileType,
		35	},
		36	};
		37
		38	pub fn deinit(d: *Diagnostics) void {
		39	for (d.errors.items) \|item\| {
		40	switch (item) {
		41	.unable_to_create_sym_link => \|info\| {
		42	d.allocator.free(info.file_name);
		43	d.allocator.free(info.link_name);
		44	},
		45	.unsupported_file_type => \|info\| {
		46	d.allocator.free(info.file_name);
		47	},
		48	}
		49	}
		50	d.errors.deinit(d.allocator);
		51	d.* = undefined;
		52	}
		53	};
16	};	54	};
17		55
18	pub const Header = struct {	56	pub const Header = struct {
19	bytes: *const [512]u8,	57	bytes: *const [512]u8,
20		58
21	pub const FileType = enum(u8) {	59	pub const FileType = enum(u8) {
		60	normal_alias = 0,
22	normal = '0',	61	normal = '0',
23	hard_link = '1',	62	hard_link = '1',
24	symbolic_link = '2',	63	symbolic_link = '2',
...	@@ -65,13 +104,18 @@ pub const Header = struct {	...	@@ -65,13 +104,18 @@ pub const Header = struct {
65	return str(header, 0, 0 + 100);	104	return str(header, 0, 0 + 100);
66	}	105	}
67		106
		107	pub fn linkName(header: Header) []const u8 {
		108	return str(header, 157, 157 + 100);
		109	}
		110
68	pub fn prefix(header: Header) []const u8 {	111	pub fn prefix(header: Header) []const u8 {
69	return str(header, 345, 345 + 155);	112	return str(header, 345, 345 + 155);
70	}	113	}
71		114
72	pub fn fileType(header: Header) FileType {	115	pub fn fileType(header: Header) FileType {
73	const result = @as(FileType, @enumFromInt(header.bytes[156]));	116	const result: FileType = @enumFromInt(header.bytes[156]);
74	return if (result == @as(FileType, @enumFromInt(0))) .normal else result;	117	if (result == .normal_alias) return .normal;
		118	return result;
75	}	119	}
76		120
77	fn str(header: Header, start: usize, end: usize) []const u8 {	121	fn str(header: Header, start: usize, end: usize) []const u8 {
...	@@ -148,7 +192,7 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi	...	@@ -148,7 +192,7 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
148	const header: Header = .{ .bytes = chunk[0..512] };	192	const header: Header = .{ .bytes = chunk[0..512] };
149	const file_size = try header.fileSize();	193	const file_size = try header.fileSize();
150	const rounded_file_size = std.mem.alignForward(u64, file_size, 512);	194	const rounded_file_size = std.mem.alignForward(u64, file_size, 512);
151	const pad_len = @as(usize, @intCast(rounded_file_size - file_size));	195	const pad_len: usize = @intCast(rounded_file_size - file_size);
152	const unstripped_file_name = if (file_name_override_len > 0)	196	const unstripped_file_name = if (file_name_override_len > 0)
153	file_name_buffer[0..file_name_override_len]	197	file_name_buffer[0..file_name_override_len]
154	else	198	else
...	@@ -175,7 +219,7 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi	...	@@ -175,7 +219,7 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
175	while (true) {	219	while (true) {
176	const temp = try buffer.readChunk(reader, @intCast(rounded_file_size + 512 - file_off));	220	const temp = try buffer.readChunk(reader, @intCast(rounded_file_size + 512 - file_off));
177	if (temp.len == 0) return error.UnexpectedEndOfStream;	221	if (temp.len == 0) return error.UnexpectedEndOfStream;
178	const slice = temp[0..@as(usize, @intCast(@min(file_size - file_off, temp.len)))];	222	const slice = temp[0..@intCast(@min(file_size - file_off, temp.len))];
179	try file.writeAll(slice);	223	try file.writeAll(slice);
180		224
181	file_off += slice.len;	225	file_off += slice.len;
...	@@ -228,8 +272,26 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi	...	@@ -228,8 +272,26 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
228	buffer.skip(reader, @intCast(rounded_file_size)) catch return error.TarHeadersTooBig;	272	buffer.skip(reader, @intCast(rounded_file_size)) catch return error.TarHeadersTooBig;
229	},	273	},
230	.hard_link => return error.TarUnsupportedFileType,	274	.hard_link => return error.TarUnsupportedFileType,
231	.symbolic_link => return error.TarUnsupportedFileType,	275	.symbolic_link => {
232	else => return error.TarUnsupportedFileType,	276	const file_name = try stripComponents(unstripped_file_name, options.strip_components);
		277	const link_name = header.linkName();
		278
		279	dir.symLink(link_name, file_name, .{}) catch \|err\| {
		280	const d = options.diagnostics orelse return error.UnableToCreateSymLink;
		281	try d.errors.append(d.allocator, .{ .unable_to_create_sym_link = .{
		282	.code = err,
		283	.file_name = try d.allocator.dupe(u8, file_name),
		284	.link_name = try d.allocator.dupe(u8, link_name),
		285	} });
		286	};
		287	},
		288	else => \|file_type\| {
		289	const d = options.diagnostics orelse return error.TarUnsupportedFileType;
		290	try d.errors.append(d.allocator, .{ .unsupported_file_type = .{
		291	.file_name = try d.allocator.dupe(u8, unstripped_file_name),
		292	.file_type = file_type,
		293	} });
		294	},
233	}	295	}
234	}	296	}
235	}	297	}

lib/std/zig/ErrorBundle.zig+12-12

src/Package.zig+250-238

...	@@ -10,15 +10,15 @@ const assert = std.debug.assert;	...	@@ -10,15 +10,15 @@ const assert = std.debug.assert;
10	const log = std.log.scoped(.package);	10	const log = std.log.scoped(.package);
11	const main = @import("main.zig");	11	const main = @import("main.zig");
12	const ThreadPool = std.Thread.Pool;	12	const ThreadPool = std.Thread.Pool;
13	const WaitGroup = std.Thread.WaitGroup;
14		13
15	const Compilation = @import("Compilation.zig");	14	const Compilation = @import("Compilation.zig");
16	const Module = @import("Module.zig");	15	const Module = @import("Module.zig");
17	const Cache = std.Build.Cache;	16	const Cache = std.Build.Cache;
18	const build_options = @import("build_options");	17	const build_options = @import("build_options");
19	const Manifest = @import("Manifest.zig");
20	const git = @import("git.zig");	18	const git = @import("git.zig");
		19	const computePackageHash = @import("Package/hash.zig").compute;
21		20
		21	pub const Manifest = @import("Manifest.zig");
22	pub const Table = std.StringHashMapUnmanaged(*Package);	22	pub const Table = std.StringHashMapUnmanaged(*Package);
23		23
24	root_src_directory: Compilation.Directory,	24	root_src_directory: Compilation.Directory,
...	@@ -285,7 +285,8 @@ pub fn fetchAndAddDependencies(	...	@@ -285,7 +285,8 @@ pub fn fetchAndAddDependencies(
285	if (manifest.errors.len > 0) {	285	if (manifest.errors.len > 0) {
286	const file_path = try directory.join(arena, &.{Manifest.basename});	286	const file_path = try directory.join(arena, &.{Manifest.basename});
287	for (manifest.errors) \|msg\| {	287	for (manifest.errors) \|msg\| {
288	try Report.addErrorMessage(ast, file_path, error_bundle, 0, msg);	288	const str = try error_bundle.addString(msg.msg);
		289	try Report.addErrorMessage(&ast, file_path, error_bundle, 0, str, msg.tok, msg.off);
289	}	290	}
290	return error.PackageFetchFailed;	291	return error.PackageFetchFailed;
291	}	292	}
...	@@ -454,8 +455,8 @@ pub fn createFilePkg(	...	@@ -454,8 +455,8 @@ pub fn createFilePkg(
454	return createWithDir(gpa, cache_directory, o_dir_sub_path, basename);	455	return createWithDir(gpa, cache_directory, o_dir_sub_path, basename);
455	}	456	}
456		457
457	const Report = struct {	458	pub const Report = struct {
458	ast: *const std.zig.Ast,	459	ast: ?*const std.zig.Ast,
459	directory: Compilation.Directory,	460	directory: Compilation.Directory,
460	error_bundle: *std.zig.ErrorBundle.Wip,	461	error_bundle: *std.zig.ErrorBundle.Wip,
461		462
...	@@ -464,42 +465,77 @@ const Report = struct {	...	@@ -464,42 +465,77 @@ const Report = struct {
464	tok: std.zig.Ast.TokenIndex,	465	tok: std.zig.Ast.TokenIndex,
465	comptime fmt_string: []const u8,	466	comptime fmt_string: []const u8,
466	fmt_args: anytype,	467	fmt_args: anytype,
		468	) error{ PackageFetchFailed, OutOfMemory } {
		469	const msg = try report.error_bundle.printString(fmt_string, fmt_args);
		470	return failMsg(report, tok, msg);
		471	}
		472
		473	fn failMsg(
		474	report: Report,
		475	tok: std.zig.Ast.TokenIndex,
		476	msg: u32,
467	) error{ PackageFetchFailed, OutOfMemory } {	477	) error{ PackageFetchFailed, OutOfMemory } {
468	const gpa = report.error_bundle.gpa;	478	const gpa = report.error_bundle.gpa;
469		479
470	const file_path = try report.directory.join(gpa, &.{Manifest.basename});	480	const file_path = try report.directory.join(gpa, &.{Manifest.basename});
471	defer gpa.free(file_path);	481	defer gpa.free(file_path);
472		482
473	const msg = try std.fmt.allocPrint(gpa, fmt_string, fmt_args);	483	const eb = report.error_bundle;
474	defer gpa.free(msg);
475		484
476	try addErrorMessage(report.ast.*, file_path, report.error_bundle, 0, .{	485	if (report.ast) \|ast\| {
477	.tok = tok,	486	try addErrorMessage(ast, file_path, eb, 0, msg, tok, 0);
478	.off = 0,	487	} else {
479	.msg = msg,	488	try eb.addRootErrorMessage(.{
480	});	489	.msg = msg,
		490	.src_loc = .none,
		491	.notes_len = 0,
		492	});
		493	}
481		494
482	return error.PackageFetchFailed;	495	return error.PackageFetchFailed;
483	}	496	}
484		497
		498	fn addErrorWithNotes(
		499	report: Report,
		500	notes_len: u32,
		501	msg: Manifest.ErrorMessage,
		502	) error{OutOfMemory}!void {
		503	const eb = report.error_bundle;
		504	const msg_str = try eb.addString(msg.msg);
		505	if (report.ast) \|ast\| {
		506	const gpa = eb.gpa;
		507	const file_path = try report.directory.join(gpa, &.{Manifest.basename});
		508	defer gpa.free(file_path);
		509	return addErrorMessage(ast, file_path, eb, notes_len, msg_str, msg.tok, msg.off);
		510	} else {
		511	return eb.addRootErrorMessage(.{
		512	.msg = msg_str,
		513	.src_loc = .none,
		514	.notes_len = notes_len,
		515	});
		516	}
		517	}
		518
485	fn addErrorMessage(	519	fn addErrorMessage(
486	ast: std.zig.Ast,	520	ast: *const std.zig.Ast,
487	file_path: []const u8,	521	file_path: []const u8,
488	eb: *std.zig.ErrorBundle.Wip,	522	eb: *std.zig.ErrorBundle.Wip,
489	notes_len: u32,	523	notes_len: u32,
490	msg: Manifest.ErrorMessage,	524	msg_str: u32,
		525	msg_tok: std.zig.Ast.TokenIndex,
		526	msg_off: u32,
491	) error{OutOfMemory}!void {	527	) error{OutOfMemory}!void {
492	const token_starts = ast.tokens.items(.start);	528	const token_starts = ast.tokens.items(.start);
493	const start_loc = ast.tokenLocation(0, msg.tok);	529	const start_loc = ast.tokenLocation(0, msg_tok);
494		530
495	try eb.addRootErrorMessage(.{	531	try eb.addRootErrorMessage(.{
496	.msg = try eb.addString(msg.msg),	532	.msg = msg_str,
497	.src_loc = try eb.addSourceLocation(.{	533	.src_loc = try eb.addSourceLocation(.{
498	.src_path = try eb.addString(file_path),	534	.src_path = try eb.addString(file_path),
499	.span_start = token_starts[msg.tok],	535	.span_start = token_starts[msg_tok],
500	.span_end = @as(u32, @intCast(token_starts[msg.tok] + ast.tokenSlice(msg.tok).len)),	536	.span_end = @as(u32, @intCast(token_starts[msg_tok] + ast.tokenSlice(msg_tok).len)),
501	.span_main = token_starts[msg.tok] + msg.off,	537	.span_main = token_starts[msg_tok] + msg_off,
502	.line = @as(u32, @intCast(start_loc.line)),	538	.line = @intCast(start_loc.line),
503	.column = @as(u32, @intCast(start_loc.column)),	539	.column = @as(u32, @intCast(start_loc.column)),
504	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),	540	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
505	}),	541	}),
...	@@ -508,7 +544,7 @@ const Report = struct {	...	@@ -508,7 +544,7 @@ const Report = struct {
508	}	544	}
509	};	545	};
510		546
511	const FetchLocation = union(enum) {	547	pub const FetchLocation = union(enum) {
512	/// The relative path to a file or directory.	548	/// The relative path to a file or directory.
513	/// This may be a file that requires unpacking (such as a .tar.gz),	549	/// This may be a file that requires unpacking (such as a .tar.gz),
514	/// or the path to the root directory of a package.	550	/// or the path to the root directory of a package.
...	@@ -517,30 +553,27 @@ const FetchLocation = union(enum) {	...	@@ -517,30 +553,27 @@ const FetchLocation = union(enum) {
517	http_request: std.Uri,	553	http_request: std.Uri,
518	git_request: std.Uri,	554	git_request: std.Uri,
519		555
520	pub fn init(gpa: Allocator, dep: Manifest.Dependency, root_dir: Compilation.Directory, report: Report) !FetchLocation {	556	pub fn init(
		557	gpa: Allocator,
		558	dep: Manifest.Dependency,
		559	root_dir: Compilation.Directory,
		560	report: Report,
		561	) !FetchLocation {
521	switch (dep.location) {	562	switch (dep.location) {
522	.url => \|url\| {	563	.url => \|url\| {
523	const uri = std.Uri.parse(url) catch \|err\| switch (err) {	564	const uri = std.Uri.parse(url) catch \|err\| switch (err) {
524	error.UnexpectedCharacter => return report.fail(dep.location_tok, "failed to parse dependency location as URI", .{}),	565	error.UnexpectedCharacter => return report.fail(dep.location_tok, "failed to parse dependency location as URI", .{}),
525	else => return err,	566	else => return err,
526	};	567	};
527	if (ascii.eqlIgnoreCase(uri.scheme, "file")) {	568	return initUri(uri, dep.location_tok, report);
528	return report.fail(dep.location_tok, "'file' scheme is not allowed for URLs. Use '.path' instead", .{});
529	} else if (ascii.eqlIgnoreCase(uri.scheme, "http") or ascii.eqlIgnoreCase(uri.scheme, "https")) {
530	return .{ .http_request = uri };
531	} else if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or ascii.eqlIgnoreCase(uri.scheme, "git+https")) {
532	return .{ .git_request = uri };
533	} else {
534	return report.fail(dep.location_tok, "Unsupported URL scheme: {s}", .{uri.scheme});
535	}
536	},	569	},
537	.path => \|path\| {	570	.path => \|path\| {
538	if (fs.path.isAbsolute(path)) {	571	if (fs.path.isAbsolute(path)) {
539	return report.fail(dep.location_tok, "Absolute paths are not allowed. Use a relative path instead", .{});	572	return report.fail(dep.location_tok, "absolute paths are not allowed. Use a relative path instead", .{});
540	}	573	}
541		574
542	const is_dir = isDirectory(root_dir, path) catch \|err\| switch (err) {	575	const is_dir = isDirectory(root_dir, path) catch \|err\| switch (err) {
543	error.FileNotFound => return report.fail(dep.location_tok, "File not found: {s}", .{path}),	576	error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{path}),
544	else => return err,	577	else => return err,
545	};	578	};
546		579
...	@@ -552,9 +585,21 @@ const FetchLocation = union(enum) {	...	@@ -552,9 +585,21 @@ const FetchLocation = union(enum) {
552	}	585	}
553	}	586	}
554		587
		588	pub fn initUri(uri: std.Uri, location_tok: std.zig.Ast.TokenIndex, report: Report) !FetchLocation {
		589	if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
		590	return report.fail(location_tok, "'file' scheme is not allowed for URLs. Use '.path' instead", .{});
		591	} else if (ascii.eqlIgnoreCase(uri.scheme, "http") or ascii.eqlIgnoreCase(uri.scheme, "https")) {
		592	return .{ .http_request = uri };
		593	} else if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or ascii.eqlIgnoreCase(uri.scheme, "git+https")) {
		594	return .{ .git_request = uri };
		595	} else {
		596	return report.fail(location_tok, "unsupported URL scheme: {s}", .{uri.scheme});
		597	}
		598	}
		599
555	pub fn deinit(f: *FetchLocation, gpa: Allocator) void {	600	pub fn deinit(f: *FetchLocation, gpa: Allocator) void {
556	switch (f.*) {	601	switch (f.*) {
557	inline .file, .directory => \|path\| gpa.free(path),	602	.file, .directory => \|path\| gpa.free(path),
558	.http_request, .git_request => {},	603	.http_request, .git_request => {},
559	}	604	}
560	f.* = undefined;	605	f.* = undefined;
...	@@ -565,7 +610,7 @@ const FetchLocation = union(enum) {	...	@@ -565,7 +610,7 @@ const FetchLocation = union(enum) {
565	gpa: Allocator,	610	gpa: Allocator,
566	root_dir: Compilation.Directory,	611	root_dir: Compilation.Directory,
567	http_client: *std.http.Client,	612	http_client: *std.http.Client,
568	dep: Manifest.Dependency,	613	dep_location_tok: std.zig.Ast.TokenIndex,
569	report: Report,	614	report: Report,
570	) !ReadableResource {	615	) !ReadableResource {
571	switch (f) {	616	switch (f) {
...	@@ -588,7 +633,7 @@ const FetchLocation = union(enum) {	...	@@ -588,7 +633,7 @@ const FetchLocation = union(enum) {
588	try req.wait();	633	try req.wait();
589		634
590	if (req.response.status != .ok) {	635	if (req.response.status != .ok) {
591	return report.fail(dep.location_tok, "Expected response status '200 OK' got '{} {s}'", .{	636	return report.fail(dep_location_tok, "expected response status '200 OK' got '{} {s}'", .{
592	@intFromEnum(req.response.status),	637	@intFromEnum(req.response.status),
593	req.response.status.phrase() orelse "",	638	req.response.status.phrase() orelse "",
594	});	639	});
...	@@ -607,7 +652,7 @@ const FetchLocation = union(enum) {	...	@@ -607,7 +652,7 @@ const FetchLocation = union(enum) {
607	session.discoverCapabilities(gpa, &redirect_uri) catch \|e\| switch (e) {	652	session.discoverCapabilities(gpa, &redirect_uri) catch \|e\| switch (e) {
608	error.Redirected => {	653	error.Redirected => {
609	defer gpa.free(redirect_uri);	654	defer gpa.free(redirect_uri);
610	return report.fail(dep.location_tok, "Repository moved to {s}", .{redirect_uri});	655	return report.fail(dep_location_tok, "repository moved to {s}", .{redirect_uri});
611	},	656	},
612	else => \|other\| return other,	657	else => \|other\| return other,
613	};	658	};
...	@@ -634,19 +679,16 @@ const FetchLocation = union(enum) {	...	@@ -634,19 +679,16 @@ const FetchLocation = union(enum) {
634	break :want_oid ref.peeled orelse ref.oid;	679	break :want_oid ref.peeled orelse ref.oid;
635	}	680	}
636	}	681	}
637	return report.fail(dep.location_tok, "Ref not found: {s}", .{want_ref});	682	return report.fail(dep_location_tok, "ref not found: {s}", .{want_ref});
638	};	683	};
639	if (uri.fragment == null) {	684	if (uri.fragment == null) {
640	const file_path = try report.directory.join(gpa, &.{Manifest.basename});
641	defer gpa.free(file_path);
642
643	const eb = report.error_bundle;
644	const notes_len = 1;	685	const notes_len = 1;
645	try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{	686	try report.addErrorWithNotes(notes_len, .{
646	.tok = dep.location_tok,	687	.tok = dep_location_tok,
647	.off = 0,	688	.off = 0,
648	.msg = "url field is missing an explicit ref",	689	.msg = "url field is missing an explicit ref",
649	});	690	});
		691	const eb = report.error_bundle;
650	const notes_start = try eb.reserveNotes(notes_len);	692	const notes_start = try eb.reserveNotes(notes_len);
651	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{	693	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
652	.msg = try eb.printString("try .url = \"{+/}#{}\",", .{ uri, std.fmt.fmtSliceHexLower(&want_oid) }),	694	.msg = try eb.printString("try .url = \"{+/}#{}\",", .{ uri, std.fmt.fmtSliceHexLower(&want_oid) }),
...	@@ -669,12 +711,13 @@ const FetchLocation = union(enum) {	...	@@ -669,12 +711,13 @@ const FetchLocation = union(enum) {
669	}	711	}
670	};	712	};
671		713
672	const ReadableResource = struct {	714	pub const ReadableResource = struct {
673	path: []const u8,	715	path: []const u8,
674	resource: union(enum) {	716	resource: union(enum) {
675	file: fs.File,	717	file: fs.File,
676	http_request: std.http.Client.Request,	718	http_request: std.http.Client.Request,
677	git_fetch_stream: git.Session.FetchStream,	719	git_fetch_stream: git.Session.FetchStream,
		720	dir: fs.IterableDir,
678	},	721	},
679		722
680	/// Unpack the package into the global cache directory.	723	/// Unpack the package into the global cache directory.
...	@@ -685,12 +728,12 @@ const ReadableResource = struct {	...	@@ -685,12 +728,12 @@ const ReadableResource = struct {
685	allocator: Allocator,	728	allocator: Allocator,
686	thread_pool: *ThreadPool,	729	thread_pool: *ThreadPool,
687	global_cache_directory: Compilation.Directory,	730	global_cache_directory: Compilation.Directory,
688	dep: Manifest.Dependency,	731	dep_location_tok: std.zig.Ast.TokenIndex,
689	report: Report,	732	report: Report,
690	pkg_prog_node: *std.Progress.Node,	733	pkg_prog_node: *std.Progress.Node,
691	) !PackageLocation {	734	) !PackageLocation {
692	switch (rr.resource) {	735	switch (rr.resource) {
693	inline .file, .http_request, .git_fetch_stream => \|*r\| {	736	inline .file, .http_request, .git_fetch_stream, .dir => \|*r, tag\| {
694	const s = fs.path.sep_str;	737	const s = fs.path.sep_str;
695	const rand_int = std.crypto.random.int(u64);	738	const rand_int = std.crypto.random.int(u64);
696	const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);	739	const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);
...	@@ -710,45 +753,58 @@ const ReadableResource = struct {	...	@@ -710,45 +753,58 @@ const ReadableResource = struct {
710	};	753	};
711	defer tmp_directory.closeAndFree(allocator);	754	defer tmp_directory.closeAndFree(allocator);
712		755
713	const opt_content_length = try rr.getSize();	756	if (tag != .dir) {
714		757	const opt_content_length = try rr.getSize();
715	var prog_reader: ProgressReader(@TypeOf(r.reader())) = .{	758
716	.child_reader = r.reader(),	759	var prog_reader: ProgressReader(@TypeOf(r.reader())) = .{
717	.prog_node = pkg_prog_node,	760	.child_reader = r.reader(),
718	.unit = if (opt_content_length) \|content_length\| unit: {	761	.prog_node = pkg_prog_node,
719	const kib = content_length / 1024;	762	.unit = if (opt_content_length) \|content_length\| unit: {
720	const mib = kib / 1024;	763	const kib = content_length / 1024;
721	if (mib > 0) {	764	const mib = kib / 1024;
722	pkg_prog_node.setEstimatedTotalItems(@intCast(mib));	765	if (mib > 0) {
723	pkg_prog_node.setUnit("MiB");	766	pkg_prog_node.setEstimatedTotalItems(@intCast(mib));
724	break :unit .mib;	767	pkg_prog_node.setUnit("MiB");
725	} else {	768	break :unit .mib;
726	pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib)));	769	} else {
727	pkg_prog_node.setUnit("KiB");	770	pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib)));
728	break :unit .kib;	771	pkg_prog_node.setUnit("KiB");
		772	break :unit .kib;
		773	}
		774	} else .any,
		775	};
		776
		777	switch (try rr.getFileType(dep_location_tok, report)) {
		778	.tar => try unpackTarball(allocator, prog_reader.reader(), tmp_directory.handle, dep_location_tok, report),
		779	.@"tar.gz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, dep_location_tok, report, std.compress.gzip),
		780	.@"tar.xz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, dep_location_tok, report, std.compress.xz),
		781	.git_pack => try unpackGitPack(allocator, &prog_reader, git.parseOid(rr.path) catch unreachable, tmp_directory.handle, dep_location_tok, report),
		782	}
		783	} else {
		784	// Recursive directory copy.
		785	var it = try r.walk(allocator);
		786	defer it.deinit();
		787	while (try it.next()) \|entry\| {
		788	switch (entry.kind) {
		789	.directory => try tmp_directory.handle.makePath(entry.path),
		790	.file => try r.dir.copyFile(
		791	entry.path,
		792	tmp_directory.handle,
		793	entry.path,
		794	.{},
		795	),
		796	.sym_link => {
		797	var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
		798	const link_name = try r.dir.readLink(entry.path, &buf);
		799	// TODO: if this would create a symlink to outside
		800	// the destination directory, fail with an error instead.
		801	try tmp_directory.handle.symLink(link_name, entry.path, .{});
		802	},
		803	else => return error.IllegalFileTypeInPackage,
729	}	804	}
730	} else .any,	805	}
731	};
732	pkg_prog_node.context.refresh();
733
734	switch (try rr.getFileType(dep, report)) {
735	.@"tar.gz" => try unpackTarball(allocator, prog_reader, tmp_directory.handle, std.compress.gzip),
736	// I have not checked what buffer sizes the xz decompression implementation uses
737	// by default, so the same logic applies for buffering the reader as for gzip.
738	.@"tar.xz" => try unpackTarball(allocator, prog_reader, tmp_directory.handle, std.compress.xz),
739	.git_pack => try unpackGitPack(allocator, &prog_reader, git.parseOid(rr.path) catch unreachable, tmp_directory.handle),
740	}	806	}
741		807
742	// Unpack completed - stop showing amount as progress
743	pkg_prog_node.setEstimatedTotalItems(0);
744	pkg_prog_node.setCompletedItems(0);
745	pkg_prog_node.context.refresh();
746
747	// TODO: delete files not included in the package prior to computing the package hash.
748	// for example, if the ini file has directives to include/not include certain files,
749	// apply those rules directly to the filesystem right here. This ensures that files
750	// not protected by the hash are not present on the file system.
751
752	break :h try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle });	808	break :h try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle });
753	};	809	};
754		810
...	@@ -769,6 +825,7 @@ const ReadableResource = struct {	...	@@ -769,6 +825,7 @@ const ReadableResource = struct {
769	}	825	}
770		826
771	const FileType = enum {	827	const FileType = enum {
		828	tar,
772	@"tar.gz",	829	@"tar.gz",
773	@"tar.xz",	830	@"tar.xz",
774	git_pack,	831	git_pack,
...	@@ -780,21 +837,28 @@ const ReadableResource = struct {	...	@@ -780,21 +837,28 @@ const ReadableResource = struct {
780	// TODO: Handle case of chunked content-length	837	// TODO: Handle case of chunked content-length
781	.http_request => \|req\| return req.response.content_length,	838	.http_request => \|req\| return req.response.content_length,
782	.git_fetch_stream => \|stream\| return stream.request.response.content_length,	839	.git_fetch_stream => \|stream\| return stream.request.response.content_length,
		840	.dir => unreachable,
783	}	841	}
784	}	842	}
785		843
786	pub fn getFileType(rr: ReadableResource, dep: Manifest.Dependency, report: Report) !FileType {	844	pub fn getFileType(
		845	rr: ReadableResource,
		846	dep_location_tok: std.zig.Ast.TokenIndex,
		847	report: Report,
		848	) !FileType {
787	switch (rr.resource) {	849	switch (rr.resource) {
788	.file => {	850	.file => {
789	return fileTypeFromPath(rr.path) orelse	851	return fileTypeFromPath(rr.path) orelse
790	return report.fail(dep.location_tok, "Unknown file type", .{});	852	return report.fail(dep_location_tok, "unknown file type", .{});
791	},	853	},
792	.http_request => \|req\| {	854	.http_request => \|req\| {
793	const content_type = req.response.headers.getFirstValue("Content-Type") orelse	855	const content_type = req.response.headers.getFirstValue("Content-Type") orelse
794	return report.fail(dep.location_tok, "Missing 'Content-Type' header", .{});	856	return report.fail(dep_location_tok, "missing 'Content-Type' header", .{});
795		857
796	// If the response has a different content type than the URI indicates, override	858	// If the response has a different content type than the URI indicates, override
797	// the previously assumed file type.	859	// the previously assumed file type.
		860	if (ascii.eqlIgnoreCase(content_type, "application/x-tar")) return .tar;
		861
798	return if (ascii.eqlIgnoreCase(content_type, "application/gzip") or	862	return if (ascii.eqlIgnoreCase(content_type, "application/gzip") or
799	ascii.eqlIgnoreCase(content_type, "application/x-gzip") or	863	ascii.eqlIgnoreCase(content_type, "application/x-gzip") or
800	ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))	864	ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))
...	@@ -805,22 +869,21 @@ const ReadableResource = struct {	...	@@ -805,22 +869,21 @@ const ReadableResource = struct {
805	// support gitlab tarball urls such as https://gitlab.com/<namespace>/<project>/-/archive/<sha>/<project>-<sha>.tar.gz	869	// support gitlab tarball urls such as https://gitlab.com/<namespace>/<project>/-/archive/<sha>/<project>-<sha>.tar.gz
806	// whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"'	870	// whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"'
807	const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse	871	const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse
808	return report.fail(dep.location_tok, "Missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});	872	return report.fail(dep_location_tok, "missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});
809	break :ty getAttachmentType(content_disposition) orelse	873	break :ty getAttachmentType(content_disposition) orelse
810	return report.fail(dep.location_tok, "Unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});	874	return report.fail(dep_location_tok, "unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});
811	} else return report.fail(dep.location_tok, "Unrecognized value for 'Content-Type' header: {s}", .{content_type});	875	} else return report.fail(dep_location_tok, "unrecognized value for 'Content-Type' header: {s}", .{content_type});
812	},	876	},
813	.git_fetch_stream => return .git_pack,	877	.git_fetch_stream => return .git_pack,
		878	.dir => unreachable,
814	}	879	}
815	}	880	}
816		881
817	fn fileTypeFromPath(file_path: []const u8) ?FileType {	882	fn fileTypeFromPath(file_path: []const u8) ?FileType {
818	return if (ascii.endsWithIgnoreCase(file_path, ".tar.gz"))	883	if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar;
819	.@"tar.gz"	884	if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz";
820	else if (ascii.endsWithIgnoreCase(file_path, ".tar.xz"))	885	if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz";
821	.@"tar.xz"	886	return null;
822	else
823	null;
824	}	887	}
825		888
826	fn getAttachmentType(content_disposition: []const u8) ?FileType {	889	fn getAttachmentType(content_disposition: []const u8) ?FileType {
...	@@ -847,6 +910,7 @@ const ReadableResource = struct {	...	@@ -847,6 +910,7 @@ const ReadableResource = struct {
847	.file => \|file\| file.close(),	910	.file => \|file\| file.close(),
848	.http_request => \|*req\| req.deinit(),	911	.http_request => \|*req\| req.deinit(),
849	.git_fetch_stream => \|*stream\| stream.deinit(),	912	.git_fetch_stream => \|*stream\| stream.deinit(),
		913	.dir => \|*dir\| dir.close(),
850	}	914	}
851	rr.* = undefined;	915	rr.* = undefined;
852	}	916	}
...	@@ -908,7 +972,7 @@ fn ProgressReader(comptime ReaderType: type) type {	...	@@ -908,7 +972,7 @@ fn ProgressReader(comptime ReaderType: type) type {
908	}	972	}
909	},	973	},
910	}	974	}
911	self.prog_node.context.maybeRefresh();	975	self.prog_node.activate();
912	return amt;	976	return amt;
913	}	977	}
914		978
...	@@ -993,7 +1057,7 @@ fn getDirectoryModule(	...	@@ -993,7 +1057,7 @@ fn getDirectoryModule(
993	if (all_modules.get(hex_digest)) \|mod\| return .{ mod.?, true };	1057	if (all_modules.get(hex_digest)) \|mod\| return .{ mod.?, true };
994		1058
995	var pkg_dir = directory.handle.openDir(fetch_location.directory, .{}) catch \|err\| switch (err) {	1059	var pkg_dir = directory.handle.openDir(fetch_location.directory, .{}) catch \|err\| switch (err) {
996	error.FileNotFound => return report.fail(dep.location_tok, "File not found: {s}", .{fetch_location.directory}),	1060	error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{fetch_location.directory}),
997	else => \|e\| return e,	1061	else => \|e\| return e,
998	};	1062	};
999	defer pkg_dir.close();	1063	defer pkg_dir.close();
...	@@ -1032,12 +1096,18 @@ fn fetchAndUnpack(	...	@@ -1032,12 +1096,18 @@ fn fetchAndUnpack(
1032	var pkg_prog_node = root_prog_node.start(name_for_prog, 0);	1096	var pkg_prog_node = root_prog_node.start(name_for_prog, 0);
1033	defer pkg_prog_node.end();	1097	defer pkg_prog_node.end();
1034	pkg_prog_node.activate();	1098	pkg_prog_node.activate();
1035	pkg_prog_node.context.refresh();
1036		1099
1037	var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep, report);	1100	var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep.location_tok, report);
1038	defer readable_resource.deinit(gpa);	1101	defer readable_resource.deinit(gpa);
1039		1102
1040	var package_location = try readable_resource.unpack(gpa, thread_pool, global_cache_directory, dep, report, &pkg_prog_node);	1103	var package_location = try readable_resource.unpack(
		1104	gpa,
		1105	thread_pool,
		1106	global_cache_directory,
		1107	dep.location_tok,
		1108	report,
		1109	&pkg_prog_node,
		1110	);
1041	defer package_location.deinit(gpa);	1111	defer package_location.deinit(gpa);
1042		1112
1043	const actual_hex = Manifest.hexDigest(package_location.hash);	1113	const actual_hex = Manifest.hexDigest(package_location.hash);
...	@@ -1048,16 +1118,13 @@ fn fetchAndUnpack(	...	@@ -1048,16 +1118,13 @@ fn fetchAndUnpack(
1048	});	1118	});
1049	}	1119	}
1050	} else {	1120	} else {
1051	const file_path = try report.directory.join(gpa, &.{Manifest.basename});
1052	defer gpa.free(file_path);
1053
1054	const eb = report.error_bundle;
1055	const notes_len = 1;	1121	const notes_len = 1;
1056	try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{	1122	try report.addErrorWithNotes(notes_len, .{
1057	.tok = dep.location_tok,	1123	.tok = dep.location_tok,
1058	.off = 0,	1124	.off = 0,
1059	.msg = "dependency is missing hash field",	1125	.msg = "dependency is missing hash field",
1060	});	1126	});
		1127	const eb = report.error_bundle;
1061	const notes_start = try eb.reserveNotes(notes_len);	1128	const notes_start = try eb.reserveNotes(notes_len);
1062	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{	1129	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
1063	.msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),	1130	.msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),
...	@@ -1080,18 +1147,34 @@ fn fetchAndUnpack(	...	@@ -1080,18 +1147,34 @@ fn fetchAndUnpack(
1080	return module;	1147	return module;
1081	}	1148	}
1082		1149
1083	fn unpackTarball(	1150	fn unpackTarballCompressed(
1084	gpa: Allocator,	1151	gpa: Allocator,
1085	reader: anytype,	1152	reader: anytype,
1086	out_dir: fs.Dir,	1153	out_dir: fs.Dir,
1087	comptime compression: type,	1154	dep_location_tok: std.zig.Ast.TokenIndex,
		1155	report: Report,
		1156	comptime Compression: type,
1088	) !void {	1157	) !void {
1089	var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);	1158	var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1090		1159
1091	var decompress = try compression.decompress(gpa, br.reader());	1160	var decompress = try Compression.decompress(gpa, br.reader());
1092	defer decompress.deinit();	1161	defer decompress.deinit();
1093		1162
1094	try std.tar.pipeToFileSystem(out_dir, decompress.reader(), .{	1163	return unpackTarball(gpa, decompress.reader(), out_dir, dep_location_tok, report);
		1164	}
		1165
		1166	fn unpackTarball(
		1167	gpa: Allocator,
		1168	reader: anytype,
		1169	out_dir: fs.Dir,
		1170	dep_location_tok: std.zig.Ast.TokenIndex,
		1171	report: Report,
		1172	) !void {
		1173	var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = gpa };
		1174	defer diagnostics.deinit();
		1175
		1176	try std.tar.pipeToFileSystem(out_dir, reader, .{
		1177	.diagnostics = &diagnostics,
1095	.strip_components = 1,	1178	.strip_components = 1,
1096	// TODO: we would like to set this to executable_bit_only, but two	1179	// TODO: we would like to set this to executable_bit_only, but two
1097	// things need to happen before that:	1180	// things need to happen before that:
...	@@ -1100,6 +1183,36 @@ fn unpackTarball(	...	@@ -1100,6 +1183,36 @@ fn unpackTarball(
1100	// bit on Windows from the ACLs (see the isExecutable function).	1183	// bit on Windows from the ACLs (see the isExecutable function).
1101	.mode_mode = .ignore,	1184	.mode_mode = .ignore,
1102	});	1185	});
		1186
		1187	if (diagnostics.errors.items.len > 0) {
		1188	const notes_len: u32 = @intCast(diagnostics.errors.items.len);
		1189	try report.addErrorWithNotes(notes_len, .{
		1190	.tok = dep_location_tok,
		1191	.off = 0,
		1192	.msg = "unable to unpack tarball",
		1193	});
		1194	const eb = report.error_bundle;
		1195	const notes_start = try eb.reserveNotes(notes_len);
		1196	for (diagnostics.errors.items, notes_start..) \|item, note_i\| {
		1197	switch (item) {
		1198	.unable_to_create_sym_link => \|info\| {
		1199	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
		1200	.msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{
		1201	info.file_name, info.link_name, @errorName(info.code),
		1202	}),
		1203	}));
		1204	},
		1205	.unsupported_file_type => \|info\| {
		1206	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
		1207	.msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{
		1208	info.file_name, @intFromEnum(info.file_type),
		1209	}),
		1210	}));
		1211	},
		1212	}
		1213	}
		1214	return error.InvalidTarball;
		1215	}
1103	}	1216	}
1104		1217
1105	fn unpackGitPack(	1218	fn unpackGitPack(
...	@@ -1107,6 +1220,8 @@ fn unpackGitPack(	...	@@ -1107,6 +1220,8 @@ fn unpackGitPack(
1107	reader: anytype,	1220	reader: anytype,
1108	want_oid: git.Oid,	1221	want_oid: git.Oid,
1109	out_dir: fs.Dir,	1222	out_dir: fs.Dir,
		1223	dep_location_tok: std.zig.Ast.TokenIndex,
		1224	report: Report,
1110	) !void {	1225	) !void {
1111	// The .git directory is used to store the packfile and associated index, but	1226	// The .git directory is used to store the packfile and associated index, but
1112	// we do not attempt to replicate the exact structure of a real .git	1227	// we do not attempt to replicate the exact structure of a real .git
...	@@ -1126,7 +1241,6 @@ fn unpackGitPack(	...	@@ -1126,7 +1241,6 @@ fn unpackGitPack(
1126	var index_prog_node = reader.prog_node.start("Index pack", 0);	1241	var index_prog_node = reader.prog_node.start("Index pack", 0);
1127	defer index_prog_node.end();	1242	defer index_prog_node.end();
1128	index_prog_node.activate();	1243	index_prog_node.activate();
1129	index_prog_node.context.refresh();
1130	var index_buffered_writer = std.io.bufferedWriter(index_file.writer());	1244	var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
1131	try git.indexPack(gpa, pack_file, index_buffered_writer.writer());	1245	try git.indexPack(gpa, pack_file, index_buffered_writer.writer());
1132	try index_buffered_writer.flush();	1246	try index_buffered_writer.flush();
...	@@ -1137,89 +1251,38 @@ fn unpackGitPack(	...	@@ -1137,89 +1251,38 @@ fn unpackGitPack(
1137	var checkout_prog_node = reader.prog_node.start("Checkout", 0);	1251	var checkout_prog_node = reader.prog_node.start("Checkout", 0);
1138	defer checkout_prog_node.end();	1252	defer checkout_prog_node.end();
1139	checkout_prog_node.activate();	1253	checkout_prog_node.activate();
1140	checkout_prog_node.context.refresh();
1141	var repository = try git.Repository.init(gpa, pack_file, index_file);	1254	var repository = try git.Repository.init(gpa, pack_file, index_file);
1142	defer repository.deinit();	1255	defer repository.deinit();
1143	try repository.checkout(out_dir, want_oid);	1256	var diagnostics: git.Diagnostics = .{ .allocator = gpa };
1144	}	1257	defer diagnostics.deinit();
1145	}	1258	try repository.checkout(out_dir, want_oid, &diagnostics);
1146		1259
1147	try out_dir.deleteTree(".git");	1260	if (diagnostics.errors.items.len > 0) {
1148	}	1261	const notes_len: u32 = @intCast(diagnostics.errors.items.len);
1149		1262	try report.addErrorWithNotes(notes_len, .{
1150	const HashedFile = struct {	1263	.tok = dep_location_tok,
1151	fs_path: []const u8,	1264	.off = 0,
1152	normalized_path: []const u8,	1265	.msg = "unable to unpack packfile",
1153	hash: [Manifest.Hash.digest_length]u8,	1266	});
1154	failure: Error!void,	1267	const eb = report.error_bundle;
1155		1268	const notes_start = try eb.reserveNotes(notes_len);
1156	const Error = fs.File.OpenError \|\| fs.File.ReadError \|\| fs.File.StatError;	1269	for (diagnostics.errors.items, notes_start..) \|item, note_i\| {
1157		1270	switch (item) {
1158	fn lessThan(context: void, lhs: const HashedFile, rhs: const HashedFile) bool {	1271	.unable_to_create_sym_link => \|info\| {
1159	_ = context;	1272	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
1160	return mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path);	1273	.msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{
1161	}	1274	info.file_name, info.link_name, @errorName(info.code),
1162	};	1275	}),
1163		1276	}));
1164	fn computePackageHash(	1277	},
1165	thread_pool: *ThreadPool,	1278	}
1166	pkg_dir: fs.IterableDir,	1279	}
1167	) ![Manifest.Hash.digest_length]u8 {	1280	return error.InvalidGitPack;
1168	const gpa = thread_pool.allocator;
1169
1170	// We'll use an arena allocator for the path name strings since they all
1171	// need to be in memory for sorting.
1172	var arena_instance = std.heap.ArenaAllocator.init(gpa);
1173	defer arena_instance.deinit();
1174	const arena = arena_instance.allocator();
1175
1176	// Collect all files, recursively, then sort.
1177	var all_files = std.ArrayList(*HashedFile).init(gpa);
1178	defer all_files.deinit();
1179
1180	var walker = try pkg_dir.walk(gpa);
1181	defer walker.deinit();
1182
1183	{
1184	// The final hash will be a hash of each file hashed independently. This
1185	// allows hashing in parallel.
1186	var wait_group: WaitGroup = .{};
1187	defer wait_group.wait();
1188
1189	while (try walker.next()) \|entry\| {
1190	switch (entry.kind) {
1191	.directory => continue,
1192	.file => {},
1193	else => return error.IllegalFileTypeInPackage,
1194	}	1281	}
1195	const hashed_file = try arena.create(HashedFile);
1196	const fs_path = try arena.dupe(u8, entry.path);
1197	hashed_file.* = .{
1198	.fs_path = fs_path,
1199	.normalized_path = try normalizePath(arena, fs_path),
1200	.hash = undefined, // to be populated by the worker
1201	.failure = undefined, // to be populated by the worker
1202	};
1203	wait_group.start();
1204	try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group });
1205
1206	try all_files.append(hashed_file);
1207	}	1282	}
1208	}	1283	}
1209		1284
1210	mem.sort(*HashedFile, all_files.items, {}, HashedFile.lessThan);	1285	try out_dir.deleteTree(".git");
1211
1212	var hasher = Manifest.Hash.init(.{});
1213	var any_failures = false;
1214	for (all_files.items) \|hashed_file\| {
1215	hashed_file.failure catch \|err\| {
1216	any_failures = true;
1217	std.log.err("unable to hash '{s}': {s}", .{ hashed_file.fs_path, @errorName(err) });
1218	};
1219	hasher.update(&hashed_file.hash);
1220	}
1221	if (any_failures) return error.PackageHashUnavailable;
1222	return hasher.finalResult();
1223	}	1286	}
1224		1287
1225	/// Compute the hash of a file path.	1288	/// Compute the hash of a file path.
...	@@ -1240,57 +1303,6 @@ fn isDirectory(root_dir: Compilation.Directory, path: []const u8) !bool {	...	@@ -1240,57 +1303,6 @@ fn isDirectory(root_dir: Compilation.Directory, path: []const u8) !bool {
1240	return true;	1303	return true;
1241	}	1304	}
1242		1305
1243	/// Make a file system path identical independently of operating system path inconsistencies.
1244	/// This converts backslashes into forward slashes.
1245	fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
1246	const canonical_sep = '/';
1247
1248	if (fs.path.sep == canonical_sep)
1249	return fs_path;
1250
1251	const normalized = try arena.dupe(u8, fs_path);
1252	for (normalized) \|*byte\| {
1253	switch (byte.*) {
1254	fs.path.sep => byte.* = canonical_sep,
1255	else => continue,
1256	}
1257	}
1258	return normalized;
1259	}
1260
1261	fn workerHashFile(dir: fs.Dir, hashed_file: HashedFile, wg: WaitGroup) void {
1262	defer wg.finish();
1263	hashed_file.failure = hashFileFallible(dir, hashed_file);
1264	}
1265
1266	fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
1267	var buf: [8000]u8 = undefined;
1268	var file = try dir.openFile(hashed_file.fs_path, .{});
1269	defer file.close();
1270	var hasher = Manifest.Hash.init(.{});
1271	hasher.update(hashed_file.normalized_path);
1272	hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) });
1273	while (true) {
1274	const bytes_read = try file.read(&buf);
1275	if (bytes_read == 0) break;
1276	hasher.update(buf[0..bytes_read]);
1277	}
1278	hasher.final(&hashed_file.hash);
1279	}
1280
1281	fn isExecutable(file: fs.File) !bool {
1282	if (builtin.os.tag == .windows) {
1283	// TODO check the ACL on Windows.
1284	// Until this is implemented, this could be a false negative on
1285	// Windows, which is why we do not yet set executable_bit_only above
1286	// when unpacking the tarball.
1287	return false;
1288	} else {
1289	const stat = try file.stat();
1290	return (stat.mode & std.os.S.IXUSR) != 0;
1291	}
1292	}
1293
1294	fn renameTmpIntoCache(	1306	fn renameTmpIntoCache(
1295	cache_dir: fs.Dir,	1307	cache_dir: fs.Dir,
1296	tmp_dir_sub_path: []const u8,	1308	tmp_dir_sub_path: []const u8,

src/Package/hash.zig created+153

...	@@ -0,0 +1,153 @@
1	const builtin = @import("builtin");
2	const std = @import("std");
3	const fs = std.fs;
4	const ThreadPool = std.Thread.Pool;
5	const WaitGroup = std.Thread.WaitGroup;
6	const Allocator = std.mem.Allocator;
7
8	const Hash = @import("../Manifest.zig").Hash;
9
10	pub fn compute(thread_pool: *ThreadPool, pkg_dir: fs.IterableDir) ![Hash.digest_length]u8 {
11	const gpa = thread_pool.allocator;
12
13	// We'll use an arena allocator for the path name strings since they all
14	// need to be in memory for sorting.
15	var arena_instance = std.heap.ArenaAllocator.init(gpa);
16	defer arena_instance.deinit();
17	const arena = arena_instance.allocator();
18
19	// TODO: delete files not included in the package prior to computing the package hash.
20	// for example, if the ini file has directives to include/not include certain files,
21	// apply those rules directly to the filesystem right here. This ensures that files
22	// not protected by the hash are not present on the file system.
23
24	// Collect all files, recursively, then sort.
25	var all_files = std.ArrayList(*HashedFile).init(gpa);
26	defer all_files.deinit();
27
28	var walker = try pkg_dir.walk(gpa);
29	defer walker.deinit();
30
31	{
32	// The final hash will be a hash of each file hashed independently. This
33	// allows hashing in parallel.
34	var wait_group: WaitGroup = .{};
35	defer wait_group.wait();
36
37	while (try walker.next()) \|entry\| {
38	const kind: HashedFile.Kind = switch (entry.kind) {
39	.directory => continue,
40	.file => .file,
41	.sym_link => .sym_link,
42	else => return error.IllegalFileTypeInPackage,
43	};
44	const hashed_file = try arena.create(HashedFile);
45	const fs_path = try arena.dupe(u8, entry.path);
46	hashed_file.* = .{
47	.fs_path = fs_path,
48	.normalized_path = try normalizePath(arena, fs_path),
49	.kind = kind,
50	.hash = undefined, // to be populated by the worker
51	.failure = undefined, // to be populated by the worker
52	};
53	wait_group.start();
54	try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group });
55
56	try all_files.append(hashed_file);
57	}
58	}
59
60	std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
61
62	var hasher = Hash.init(.{});
63	var any_failures = false;
64	for (all_files.items) \|hashed_file\| {
65	hashed_file.failure catch \|err\| {
66	any_failures = true;
67	std.log.err("unable to hash '{s}': {s}", .{ hashed_file.fs_path, @errorName(err) });
68	};
69	hasher.update(&hashed_file.hash);
70	}
71	if (any_failures) return error.PackageHashUnavailable;
72	return hasher.finalResult();
73	}
74
75	const HashedFile = struct {
76	fs_path: []const u8,
77	normalized_path: []const u8,
78	hash: [Hash.digest_length]u8,
79	failure: Error!void,
80	kind: Kind,
81
82	const Error =
83	fs.File.OpenError \|\|
84	fs.File.ReadError \|\|
85	fs.File.StatError \|\|
86	fs.Dir.ReadLinkError;
87
88	const Kind = enum { file, sym_link };
89
90	fn lessThan(context: void, lhs: const HashedFile, rhs: const HashedFile) bool {
91	_ = context;
92	return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path);
93	}
94	};
95
96	/// Make a file system path identical independently of operating system path inconsistencies.
97	/// This converts backslashes into forward slashes.
98	fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
99	const canonical_sep = '/';
100
101	if (fs.path.sep == canonical_sep)
102	return fs_path;
103
104	const normalized = try arena.dupe(u8, fs_path);
105	for (normalized) \|*byte\| {
106	switch (byte.*) {
107	fs.path.sep => byte.* = canonical_sep,
108	else => continue,
109	}
110	}
111	return normalized;
112	}
113
114	fn workerHashFile(dir: fs.Dir, hashed_file: HashedFile, wg: WaitGroup) void {
115	defer wg.finish();
116	hashed_file.failure = hashFileFallible(dir, hashed_file);
117	}
118
119	fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
120	var buf: [8000]u8 = undefined;
121	var hasher = Hash.init(.{});
122	hasher.update(hashed_file.normalized_path);
123	switch (hashed_file.kind) {
124	.file => {
125	var file = try dir.openFile(hashed_file.fs_path, .{});
126	defer file.close();
127	hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) });
128	while (true) {
129	const bytes_read = try file.read(&buf);
130	if (bytes_read == 0) break;
131	hasher.update(buf[0..bytes_read]);
132	}
133	},
134	.sym_link => {
135	const link_name = try dir.readLink(hashed_file.fs_path, &buf);
136	hasher.update(link_name);
137	},
138	}
139	hasher.final(&hashed_file.hash);
140	}
141
142	fn isExecutable(file: fs.File) !bool {
143	if (builtin.os.tag == .windows) {
144	// TODO check the ACL on Windows.
145	// Until this is implemented, this could be a false negative on
146	// Windows, which is why we do not yet set executable_bit_only above
147	// when unpacking the tarball.
148	return false;
149	} else {
150	const stat = try file.stat();
151	return (stat.mode & std.os.S.IXUSR) != 0;
152	}
153	}

src/git.zig+50-3

...	@@ -38,6 +38,32 @@ test parseOid {	...	@@ -38,6 +38,32 @@ test parseOid {
38	try testing.expectError(error.InvalidOid, parseOid("HEAD"));	38	try testing.expectError(error.InvalidOid, parseOid("HEAD"));
39	}	39	}
40		40
		41	pub const Diagnostics = struct {
		42	allocator: Allocator,
		43	errors: std.ArrayListUnmanaged(Error) = .{},
		44
		45	pub const Error = union(enum) {
		46	unable_to_create_sym_link: struct {
		47	code: anyerror,
		48	file_name: []const u8,
		49	link_name: []const u8,
		50	},
		51	};
		52
		53	pub fn deinit(d: *Diagnostics) void {
		54	for (d.errors.items) \|item\| {
		55	switch (item) {
		56	.unable_to_create_sym_link => \|info\| {
		57	d.allocator.free(info.file_name);
		58	d.allocator.free(info.link_name);
		59	},
		60	}
		61	}
		62	d.errors.deinit(d.allocator);
		63	d.* = undefined;
		64	}
		65	};
		66
41	pub const Repository = struct {	67	pub const Repository = struct {
42	odb: Odb,	68	odb: Odb,
43		69
...	@@ -55,6 +81,7 @@ pub const Repository = struct {	...	@@ -55,6 +81,7 @@ pub const Repository = struct {
55	repository: *Repository,	81	repository: *Repository,
56	worktree: std.fs.Dir,	82	worktree: std.fs.Dir,
57	commit_oid: Oid,	83	commit_oid: Oid,
		84	diagnostics: *Diagnostics,
58	) !void {	85	) !void {
59	try repository.odb.seekOid(commit_oid);	86	try repository.odb.seekOid(commit_oid);
60	const tree_oid = tree_oid: {	87	const tree_oid = tree_oid: {
...	@@ -62,7 +89,7 @@ pub const Repository = struct {	...	@@ -62,7 +89,7 @@ pub const Repository = struct {
62	if (commit_object.type != .commit) return error.NotACommit;	89	if (commit_object.type != .commit) return error.NotACommit;
63	break :tree_oid try getCommitTree(commit_object.data);	90	break :tree_oid try getCommitTree(commit_object.data);
64	};	91	};
65	try repository.checkoutTree(worktree, tree_oid);	92	try repository.checkoutTree(worktree, tree_oid, "", diagnostics);
66	}	93	}
67		94
68	/// Checks out the tree at `tree_oid` to `worktree`.	95	/// Checks out the tree at `tree_oid` to `worktree`.
...	@@ -70,6 +97,8 @@ pub const Repository = struct {	...	@@ -70,6 +97,8 @@ pub const Repository = struct {
70	repository: *Repository,	97	repository: *Repository,
71	dir: std.fs.Dir,	98	dir: std.fs.Dir,
72	tree_oid: Oid,	99	tree_oid: Oid,
		100	current_path: []const u8,
		101	diagnostics: *Diagnostics,
73	) !void {	102	) !void {
74	try repository.odb.seekOid(tree_oid);	103	try repository.odb.seekOid(tree_oid);
75	const tree_object = try repository.odb.readObject();	104	const tree_object = try repository.odb.readObject();
...	@@ -87,7 +116,9 @@ pub const Repository = struct {	...	@@ -87,7 +116,9 @@ pub const Repository = struct {
87	try dir.makeDir(entry.name);	116	try dir.makeDir(entry.name);
88	var subdir = try dir.openDir(entry.name, .{});	117	var subdir = try dir.openDir(entry.name, .{});
89	defer subdir.close();	118	defer subdir.close();
90	try repository.checkoutTree(subdir, entry.oid);	119	const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name });
		120	defer repository.odb.allocator.free(sub_path);
		121	try repository.checkoutTree(subdir, entry.oid, sub_path, diagnostics);
91	},	122	},
92	.file => {	123	.file => {
93	var file = try dir.createFile(entry.name, .{});	124	var file = try dir.createFile(entry.name, .{});
...	@@ -98,7 +129,23 @@ pub const Repository = struct {	...	@@ -98,7 +129,23 @@ pub const Repository = struct {
98	try file.writeAll(file_object.data);	129	try file.writeAll(file_object.data);
99	try file.sync();	130	try file.sync();
100	},	131	},
101	.symlink => return error.SymlinkNotSupported,	132	.symlink => {
		133	try repository.odb.seekOid(entry.oid);
		134	var symlink_object = try repository.odb.readObject();
		135	if (symlink_object.type != .blob) return error.InvalidFile;
		136	const link_name = symlink_object.data;
		137	dir.symLink(link_name, entry.name, .{}) catch \|e\| {
		138	const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name });
		139	errdefer diagnostics.allocator.free(file_name);
		140	const link_name_dup = try diagnostics.allocator.dupe(u8, link_name);
		141	errdefer diagnostics.allocator.free(link_name_dup);
		142	try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{
		143	.code = e,
		144	.file_name = file_name,
		145	.link_name = link_name_dup,
		146	} });
		147	};
		148	},
102	.gitlink => {	149	.gitlink => {
103	// Consistent with git archive behavior, create the directory but	150	// Consistent with git archive behavior, create the directory but
104	// do nothing else	151	// do nothing else

src/main.zig+143

...	@@ -84,6 +84,7 @@ const normal_usage =	...	@@ -84,6 +84,7 @@ const normal_usage =
84	\\Commands:	84	\\Commands:
85	\\	85	\\
86	\\ build Build project from build.zig	86	\\ build Build project from build.zig
		87	\\ fetch Copy a package into global cache and print its hash
87	\\ init-exe Initialize a `zig build` application in the cwd	88	\\ init-exe Initialize a `zig build` application in the cwd
88	\\ init-lib Initialize a `zig build` library in the cwd	89	\\ init-lib Initialize a `zig build` library in the cwd
89	\\	90	\\
...	@@ -303,6 +304,8 @@ pub fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi	...	@@ -303,6 +304,8 @@ pub fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
303	return cmdFmt(gpa, arena, cmd_args);	304	return cmdFmt(gpa, arena, cmd_args);
304	} else if (mem.eql(u8, cmd, "objcopy")) {	305	} else if (mem.eql(u8, cmd, "objcopy")) {
305	return @import("objcopy.zig").cmdObjCopy(gpa, arena, cmd_args);	306	return @import("objcopy.zig").cmdObjCopy(gpa, arena, cmd_args);
		307	} else if (mem.eql(u8, cmd, "fetch")) {
		308	return cmdFetch(gpa, arena, cmd_args);
306	} else if (mem.eql(u8, cmd, "libc")) {	309	} else if (mem.eql(u8, cmd, "libc")) {
307	return cmdLibC(gpa, cmd_args);	310	return cmdLibC(gpa, cmd_args);
308	} else if (mem.eql(u8, cmd, "init-exe")) {	311	} else if (mem.eql(u8, cmd, "init-exe")) {
...	@@ -6589,3 +6592,143 @@ fn parseRcIncludes(arg: []const u8) Compilation.RcIncludes {	...	@@ -6589,3 +6592,143 @@ fn parseRcIncludes(arg: []const u8) Compilation.RcIncludes {
6589	return std.meta.stringToEnum(Compilation.RcIncludes, arg) orelse	6592	return std.meta.stringToEnum(Compilation.RcIncludes, arg) orelse
6590	fatal("unsupported rc includes type: '{s}'", .{arg});	6593	fatal("unsupported rc includes type: '{s}'", .{arg});
6591	}	6594	}
		6595
		6596	pub const usage_fetch =
		6597	\\Usage: zig fetch [options] <url>
		6598	\\Usage: zig fetch [options] <path>
		6599	\\
		6600	\\ Copy a package into the global cache and print its hash.
		6601	\\
		6602	\\Options:
		6603	\\ -h, --help Print this help and exit
		6604	\\ --global-cache-dir [path] Override path to global Zig cache directory
		6605	\\
		6606	;
		6607
		6608	fn cmdFetch(
		6609	gpa: Allocator,
		6610	arena: Allocator,
		6611	args: []const []const u8,
		6612	) !void {
		6613	const color: Color = .auto;
		6614	var opt_url: ?[]const u8 = null;
		6615	var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");
		6616
		6617	{
		6618	var i: usize = 0;
		6619	while (i < args.len) : (i += 1) {
		6620	const arg = args[i];
		6621	if (mem.startsWith(u8, arg, "-")) {
		6622	if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
		6623	const stdout = io.getStdOut().writer();
		6624	try stdout.writeAll(usage_fetch);
		6625	return cleanExit();
		6626	} else if (mem.eql(u8, arg, "--global-cache-dir")) {
		6627	if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
		6628	i += 1;
		6629	override_global_cache_dir = args[i];
		6630	continue;
		6631	} else {
		6632	fatal("unrecognized parameter: '{s}'", .{arg});
		6633	}
		6634	} else if (opt_url != null) {
		6635	fatal("unexpected extra parameter: '{s}'", .{arg});
		6636	} else {
		6637	opt_url = arg;
		6638	}
		6639	}
		6640	}
		6641
		6642	const url = opt_url orelse fatal("missing url or path parameter", .{});
		6643
		6644	var thread_pool: ThreadPool = undefined;
		6645	try thread_pool.init(.{ .allocator = gpa });
		6646	defer thread_pool.deinit();
		6647
		6648	var http_client: std.http.Client = .{ .allocator = gpa };
		6649	defer http_client.deinit();
		6650
		6651	var progress: std.Progress = .{ .dont_print_on_dumb = true };
		6652	const root_prog_node = progress.start("Fetch", 0);
		6653	defer root_prog_node.end();
		6654
		6655	var wip_errors: std.zig.ErrorBundle.Wip = undefined;
		6656	try wip_errors.init(gpa);
		6657	defer wip_errors.deinit();
		6658
		6659	var report: Package.Report = .{
		6660	.ast = null,
		6661	.directory = .{
		6662	.handle = fs.cwd(),
		6663	.path = null,
		6664	},
		6665	.error_bundle = &wip_errors,
		6666	};
		6667
		6668	var global_cache_directory: Compilation.Directory = l: {
		6669	const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena);
		6670	break :l .{
		6671	.handle = try fs.cwd().makeOpenPath(p, .{}),
		6672	.path = p,
		6673	};
		6674	};
		6675	defer global_cache_directory.handle.close();
		6676
		6677	var readable_resource: Package.ReadableResource = rr: {
		6678	if (fs.cwd().openIterableDir(url, .{})) \|dir\| {
		6679	break :rr .{
		6680	.path = try gpa.dupe(u8, url),
		6681	.resource = .{ .dir = dir },
		6682	};
		6683	} else \|dir_err\| {
		6684	const file_err = if (dir_err == error.NotDir) e: {
		6685	if (fs.cwd().openFile(url, .{})) \|f\| {
		6686	break :rr .{
		6687	.path = try gpa.dupe(u8, url),
		6688	.resource = .{ .file = f },
		6689	};
		6690	} else \|err\| break :e err;
		6691	} else dir_err;
		6692
		6693	const uri = std.Uri.parse(url) catch \|uri_err\| {
		6694	fatal("'{s}' could not be recognized as a file path ({s}) or an URL ({s})", .{
		6695	url, @errorName(file_err), @errorName(uri_err),
		6696	});
		6697	};
		6698	const fetch_location = try Package.FetchLocation.initUri(uri, 0, report);
		6699	const cwd: Cache.Directory = .{
		6700	.handle = fs.cwd(),
		6701	.path = null,
		6702	};
		6703	break :rr try fetch_location.fetch(gpa, cwd, &http_client, 0, report);
		6704	}
		6705	};
		6706	defer readable_resource.deinit(gpa);
		6707
		6708	var package_location = readable_resource.unpack(
		6709	gpa,
		6710	&thread_pool,
		6711	global_cache_directory,
		6712	0,
		6713	report,
		6714	root_prog_node,
		6715	) catch \|err\| {
		6716	if (wip_errors.root_list.items.len > 0) {
		6717	var errors = try wip_errors.toOwnedBundle("");
		6718	defer errors.deinit(gpa);
		6719	errors.renderToStdErr(renderOptions(color));
		6720	process.exit(1);
		6721	}
		6722	fatal("unable to unpack '{s}': {s}", .{ url, @errorName(err) });
		6723	};
		6724	defer package_location.deinit(gpa);
		6725
		6726	const hex_digest = Package.Manifest.hexDigest(package_location.hash);
		6727
		6728	progress.done = true;
		6729	progress.refresh();
		6730
		6731	try io.getStdOut().writeAll(hex_digest ++ "\n");
		6732
		6733	return cleanExit();
		6734	}

...	@@ -202,7 +202,7 @@ fn renderErrorMessageToWriter(	...	@@ -202,7 +202,7 @@ fn renderErrorMessageToWriter(
202	try counting_stderr.writeAll(": ");	202	try counting_stderr.writeAll(": ");
203	// This is the length of the part before the error message:	203	// This is the length of the part before the error message:
204	// e.g. "file.zig:4:5: error: "	204	// e.g. "file.zig:4:5: error: "
205	const prefix_len = @as(usize, @intCast(counting_stderr.context.bytes_written));	205	const prefix_len: usize = @intCast(counting_stderr.context.bytes_written);
206	try ttyconf.setColor(stderr, .reset);	206	try ttyconf.setColor(stderr, .reset);
207	try ttyconf.setColor(stderr, .bold);	207	try ttyconf.setColor(stderr, .bold);
208	if (err_msg.count == 1) {	208	if (err_msg.count == 1) {
...	@@ -356,7 +356,7 @@ pub const Wip = struct {	...	@@ -356,7 +356,7 @@ pub const Wip = struct {
356	}	356	}
357		357
358	const compile_log_str_index = if (compile_log_text.len == 0) 0 else str: {	358	const compile_log_str_index = if (compile_log_text.len == 0) 0 else str: {
359	const str = @as(u32, @intCast(wip.string_bytes.items.len));	359	const str: u32 = @intCast(wip.string_bytes.items.len);
360	try wip.string_bytes.ensureUnusedCapacity(gpa, compile_log_text.len + 1);	360	try wip.string_bytes.ensureUnusedCapacity(gpa, compile_log_text.len + 1);
361	wip.string_bytes.appendSliceAssumeCapacity(compile_log_text);	361	wip.string_bytes.appendSliceAssumeCapacity(compile_log_text);
362	wip.string_bytes.appendAssumeCapacity(0);	362	wip.string_bytes.appendAssumeCapacity(0);
...	@@ -364,8 +364,8 @@ pub const Wip = struct {	...	@@ -364,8 +364,8 @@ pub const Wip = struct {
364	};	364	};
365		365
366	wip.setExtra(0, ErrorMessageList{	366	wip.setExtra(0, ErrorMessageList{
367	.len = @as(u32, @intCast(wip.root_list.items.len)),	367	.len = @intCast(wip.root_list.items.len),
368	.start = @as(u32, @intCast(wip.extra.items.len)),	368	.start = @intCast(wip.extra.items.len),
369	.compile_log_text = compile_log_str_index,	369	.compile_log_text = compile_log_str_index,
370	});	370	});
371	try wip.extra.appendSlice(gpa, @as([]const u32, @ptrCast(wip.root_list.items)));	371	try wip.extra.appendSlice(gpa, @as([]const u32, @ptrCast(wip.root_list.items)));
...	@@ -385,7 +385,7 @@ pub const Wip = struct {	...	@@ -385,7 +385,7 @@ pub const Wip = struct {
385		385
386	pub fn addString(wip: *Wip, s: []const u8) !u32 {	386	pub fn addString(wip: *Wip, s: []const u8) !u32 {
387	const gpa = wip.gpa;	387	const gpa = wip.gpa;
388	const index = @as(u32, @intCast(wip.string_bytes.items.len));	388	const index: u32 = @intCast(wip.string_bytes.items.len);
389	try wip.string_bytes.ensureUnusedCapacity(gpa, s.len + 1);	389	try wip.string_bytes.ensureUnusedCapacity(gpa, s.len + 1);
390	wip.string_bytes.appendSliceAssumeCapacity(s);	390	wip.string_bytes.appendSliceAssumeCapacity(s);
391	wip.string_bytes.appendAssumeCapacity(0);	391	wip.string_bytes.appendAssumeCapacity(0);
...	@@ -394,7 +394,7 @@ pub const Wip = struct {	...	@@ -394,7 +394,7 @@ pub const Wip = struct {
394		394
395	pub fn printString(wip: *Wip, comptime fmt: []const u8, args: anytype) !u32 {	395	pub fn printString(wip: *Wip, comptime fmt: []const u8, args: anytype) !u32 {
396	const gpa = wip.gpa;	396	const gpa = wip.gpa;
397	const index = @as(u32, @intCast(wip.string_bytes.items.len));	397	const index: u32 = @intCast(wip.string_bytes.items.len);
398	try wip.string_bytes.writer(gpa).print(fmt, args);	398	try wip.string_bytes.writer(gpa).print(fmt, args);
399	try wip.string_bytes.append(gpa, 0);	399	try wip.string_bytes.append(gpa, 0);
400	return index;	400	return index;
...	@@ -406,15 +406,15 @@ pub const Wip = struct {	...	@@ -406,15 +406,15 @@ pub const Wip = struct {
406	}	406	}
407		407
408	pub fn addErrorMessage(wip: *Wip, em: ErrorMessage) !MessageIndex {	408	pub fn addErrorMessage(wip: *Wip, em: ErrorMessage) !MessageIndex {
409	return @as(MessageIndex, @enumFromInt(try addExtra(wip, em)));	409	return @enumFromInt(try addExtra(wip, em));
410	}	410	}
411		411
412	pub fn addErrorMessageAssumeCapacity(wip: *Wip, em: ErrorMessage) MessageIndex {	412	pub fn addErrorMessageAssumeCapacity(wip: *Wip, em: ErrorMessage) MessageIndex {
413	return @as(MessageIndex, @enumFromInt(addExtraAssumeCapacity(wip, em)));	413	return @enumFromInt(addExtraAssumeCapacity(wip, em));
414	}	414	}
415		415
416	pub fn addSourceLocation(wip: *Wip, sl: SourceLocation) !SourceLocationIndex {	416	pub fn addSourceLocation(wip: *Wip, sl: SourceLocation) !SourceLocationIndex {
417	return @as(SourceLocationIndex, @enumFromInt(try addExtra(wip, sl)));	417	return @enumFromInt(try addExtra(wip, sl));
418	}	418	}
419		419
420	pub fn addReferenceTrace(wip: *Wip, rt: ReferenceTrace) !void {	420	pub fn addReferenceTrace(wip: *Wip, rt: ReferenceTrace) !void {
...	@@ -430,7 +430,7 @@ pub const Wip = struct {	...	@@ -430,7 +430,7 @@ pub const Wip = struct {
430	const other_list = other.getMessages();	430	const other_list = other.getMessages();
431		431
432	// The ensureUnusedCapacity call above guarantees this.	432	// The ensureUnusedCapacity call above guarantees this.
433	const notes_start = wip.reserveNotes(@as(u32, @intCast(other_list.len))) catch unreachable;	433	const notes_start = wip.reserveNotes(@intCast(other_list.len)) catch unreachable;
434	for (notes_start.., other_list) \|note, message\| {	434	for (notes_start.., other_list) \|note, message\| {
435	wip.extra.items[note] = @intFromEnum(wip.addOtherMessage(other, message) catch unreachable);	435	wip.extra.items[note] = @intFromEnum(wip.addOtherMessage(other, message) catch unreachable);
436	}	436	}
...	@@ -455,7 +455,7 @@ pub const Wip = struct {	...	@@ -455,7 +455,7 @@ pub const Wip = struct {
455	try wip.extra.ensureUnusedCapacity(wip.gpa, notes_len +	455	try wip.extra.ensureUnusedCapacity(wip.gpa, notes_len +
456	notes_len * @typeInfo(ErrorBundle.ErrorMessage).Struct.fields.len);	456	notes_len * @typeInfo(ErrorBundle.ErrorMessage).Struct.fields.len);
457	wip.extra.items.len += notes_len;	457	wip.extra.items.len += notes_len;
458	return @as(u32, @intCast(wip.extra.items.len - notes_len));	458	return @intCast(wip.extra.items.len - notes_len);
459	}	459	}
460		460
461	fn addOtherMessage(wip: *Wip, other: ErrorBundle, msg_index: MessageIndex) !MessageIndex {	461	fn addOtherMessage(wip: *Wip, other: ErrorBundle, msg_index: MessageIndex) !MessageIndex {
...	@@ -510,7 +510,7 @@ pub const Wip = struct {	...	@@ -510,7 +510,7 @@ pub const Wip = struct {
510		510
511	fn addExtraAssumeCapacity(wip: *Wip, extra: anytype) u32 {	511	fn addExtraAssumeCapacity(wip: *Wip, extra: anytype) u32 {
512	const fields = @typeInfo(@TypeOf(extra)).Struct.fields;	512	const fields = @typeInfo(@TypeOf(extra)).Struct.fields;
513	const result = @as(u32, @intCast(wip.extra.items.len));	513	const result: u32 = @intCast(wip.extra.items.len);
514	wip.extra.items.len += fields.len;	514	wip.extra.items.len += fields.len;
515	setExtra(wip, result, extra);	515	setExtra(wip, result, extra);
516	return result;	516	return result;