Commit df4853a6271b7963d77cafda7d6727274c6cbdaa | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-10-03 03:33:26-07:00
committer	GitHub <noreply@github.com> 2023-10-03 03:33:26-07:00
log	df4853a6271b7963d77cafda7d6727274c6cbdaa
tree	b00928173d4d73f74bfdaaae6b3021be71759eb8
parent	4930094e622c043d9be459abbfc5a7b203a22aa2
parent	573a13f8be24276a02761c30396fd75fc10ebdb0
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #17363 from ziglang/tar-symlinks

introduce the `zig fetch` subcommand and symlink support in zig packages

8 files changed, 681 insertions(+), 261 deletions(-)

CMakeLists.txt
lib
- std
  - fs.zig
  - tar.zig
  - zig
    ErrorBundle.zig
src
- Package.zig
- Package
  - hash.zig
- git.zig
- main.zig

CMakeLists.txt+1

...	...	@@ -527,6 +527,7 @@ set(ZIG_STAGE2_SOURCES
527	527	"${CMAKE_SOURCE_DIR}/src/Liveness.zig"
528	528	"${CMAKE_SOURCE_DIR}/src/Module.zig"
529	529	"${CMAKE_SOURCE_DIR}/src/Package.zig"
	530	"${CMAKE_SOURCE_DIR}/src/Package/hash.zig"
530	531	"${CMAKE_SOURCE_DIR}/src/RangeSet.zig"
531	532	"${CMAKE_SOURCE_DIR}/src/Sema.zig"
532	533	"${CMAKE_SOURCE_DIR}/src/TypedValue.zig"

lib/std/fs.zig+3-1

...	...	@@ -2003,10 +2003,12 @@ pub const Dir = struct {
2003	2003	return os.windows.CreateSymbolicLink(self.fd, sym_link_path_w, target_path_w, flags.is_directory);
2004	2004	}
2005	2005
	2006	pub const ReadLinkError = os.ReadLinkError;
	2007
2006	2008	/// Read value of a symbolic link.
2007	2009	/// The return value is a slice of `buffer`, from index `0`.
2008	2010	/// Asserts that the path parameter has no null bytes.
2009		pub fn readLink(self: Dir, sub_path: []const u8, buffer: []u8) ![]u8 {
	2011	pub fn readLink(self: Dir, sub_path: []const u8, buffer: []u8) ReadLinkError![]u8 {
2010	2012	if (builtin.os.tag == .wasi and !builtin.link_libc) {
2011	2013	return self.readLinkWasi(sub_path, buffer);
2012	2014	}

lib/std/tar.zig+69-7

...	...	@@ -3,8 +3,13 @@ pub const Options = struct {
3	3	strip_components: u32 = 0,
4	4	/// How to handle the "mode" property of files from within the tar file.
5	5	mode_mode: ModeMode = .executable_bit_only,
	6	/// Provide this to receive detailed error messages.
	7	/// When this is provided, some errors which would otherwise be returned immediately
	8	/// will instead be added to this structure. The API user must check the errors
	9	/// in diagnostics to know whether the operation succeeded or failed.
	10	diagnostics: ?*Diagnostics = null,
6	11
7		const ModeMode = enum {
	12	pub const ModeMode = enum {
8	13	/// The mode from the tar file is completely ignored. Files are created
9	14	/// with the default mode when creating files.
10	15	ignore,
...	...	@@ -13,12 +18,46 @@ pub const Options = struct {
13	18	/// Other bits of the mode are left as the default when creating files.
14	19	executable_bit_only,
15	20	};
	21
	22	pub const Diagnostics = struct {
	23	allocator: std.mem.Allocator,
	24	errors: std.ArrayListUnmanaged(Error) = .{},
	25
	26	pub const Error = union(enum) {
	27	unable_to_create_sym_link: struct {
	28	code: anyerror,
	29	file_name: []const u8,
	30	link_name: []const u8,
	31	},
	32	unsupported_file_type: struct {
	33	file_name: []const u8,
	34	file_type: Header.FileType,
	35	},
	36	};
	37
	38	pub fn deinit(d: *Diagnostics) void {
	39	for (d.errors.items) \|item\| {
	40	switch (item) {
	41	.unable_to_create_sym_link => \|info\| {
	42	d.allocator.free(info.file_name);
	43	d.allocator.free(info.link_name);
	44	},
	45	.unsupported_file_type => \|info\| {
	46	d.allocator.free(info.file_name);
	47	},
	48	}
	49	}
	50	d.errors.deinit(d.allocator);
	51	d.* = undefined;
	52	}
	53	};
16	54	};
17	55
18	56	pub const Header = struct {
19	57	bytes: *const [512]u8,
20	58
21	59	pub const FileType = enum(u8) {
	60	normal_alias = 0,
22	61	normal = '0',
23	62	hard_link = '1',
24	63	symbolic_link = '2',
...	...	@@ -65,13 +104,18 @@ pub const Header = struct {
65	104	return str(header, 0, 0 + 100);
66	105	}
67	106
	107	pub fn linkName(header: Header) []const u8 {
	108	return str(header, 157, 157 + 100);
	109	}
	110
68	111	pub fn prefix(header: Header) []const u8 {
69	112	return str(header, 345, 345 + 155);
70	113	}
71	114
72	115	pub fn fileType(header: Header) FileType {
73		const result = @as(FileType, @enumFromInt(header.bytes[156]));
74		return if (result == @as(FileType, @enumFromInt(0))) .normal else result;
	116	const result: FileType = @enumFromInt(header.bytes[156]);
	117	if (result == .normal_alias) return .normal;
	118	return result;
75	119	}
76	120
77	121	fn str(header: Header, start: usize, end: usize) []const u8 {
...	...	@@ -148,7 +192,7 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
148	192	const header: Header = .{ .bytes = chunk[0..512] };
149	193	const file_size = try header.fileSize();
150	194	const rounded_file_size = std.mem.alignForward(u64, file_size, 512);
151		const pad_len = @as(usize, @intCast(rounded_file_size - file_size));
	195	const pad_len: usize = @intCast(rounded_file_size - file_size);
152	196	const unstripped_file_name = if (file_name_override_len > 0)
153	197	file_name_buffer[0..file_name_override_len]
154	198	else
...	...	@@ -175,7 +219,7 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
175	219	while (true) {
176	220	const temp = try buffer.readChunk(reader, @intCast(rounded_file_size + 512 - file_off));
177	221	if (temp.len == 0) return error.UnexpectedEndOfStream;
178		const slice = temp[0..@as(usize, @intCast(@min(file_size - file_off, temp.len)))];
	222	const slice = temp[0..@intCast(@min(file_size - file_off, temp.len))];
179	223	try file.writeAll(slice);
180	224
181	225	file_off += slice.len;
...	...	@@ -228,8 +272,26 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
228	272	buffer.skip(reader, @intCast(rounded_file_size)) catch return error.TarHeadersTooBig;
229	273	},
230	274	.hard_link => return error.TarUnsupportedFileType,
231		.symbolic_link => return error.TarUnsupportedFileType,
232		else => return error.TarUnsupportedFileType,
	275	.symbolic_link => {
	276	const file_name = try stripComponents(unstripped_file_name, options.strip_components);
	277	const link_name = header.linkName();
	278
	279	dir.symLink(link_name, file_name, .{}) catch \|err\| {
	280	const d = options.diagnostics orelse return error.UnableToCreateSymLink;
	281	try d.errors.append(d.allocator, .{ .unable_to_create_sym_link = .{
	282	.code = err,
	283	.file_name = try d.allocator.dupe(u8, file_name),
	284	.link_name = try d.allocator.dupe(u8, link_name),
	285	} });
	286	};
	287	},
	288	else => \|file_type\| {
	289	const d = options.diagnostics orelse return error.TarUnsupportedFileType;
	290	try d.errors.append(d.allocator, .{ .unsupported_file_type = .{
	291	.file_name = try d.allocator.dupe(u8, unstripped_file_name),
	292	.file_type = file_type,
	293	} });
	294	},
233	295	}
234	296	}
235	297	}

lib/std/zig/ErrorBundle.zig+12-12

...	...	@@ -202,7 +202,7 @@ fn renderErrorMessageToWriter(
202	202	try counting_stderr.writeAll(": ");
203	203	// This is the length of the part before the error message:
204	204	// e.g. "file.zig:4:5: error: "
205		const prefix_len = @as(usize, @intCast(counting_stderr.context.bytes_written));
	205	const prefix_len: usize = @intCast(counting_stderr.context.bytes_written);
206	206	try ttyconf.setColor(stderr, .reset);
207	207	try ttyconf.setColor(stderr, .bold);
208	208	if (err_msg.count == 1) {
...	...	@@ -356,7 +356,7 @@ pub const Wip = struct {
356	356	}
357	357
358	358	const compile_log_str_index = if (compile_log_text.len == 0) 0 else str: {
359		const str = @as(u32, @intCast(wip.string_bytes.items.len));
	359	const str: u32 = @intCast(wip.string_bytes.items.len);
360	360	try wip.string_bytes.ensureUnusedCapacity(gpa, compile_log_text.len + 1);
361	361	wip.string_bytes.appendSliceAssumeCapacity(compile_log_text);
362	362	wip.string_bytes.appendAssumeCapacity(0);
...	...	@@ -364,8 +364,8 @@ pub const Wip = struct {
364	364	};
365	365
366	366	wip.setExtra(0, ErrorMessageList{
367		.len = @as(u32, @intCast(wip.root_list.items.len)),
368		.start = @as(u32, @intCast(wip.extra.items.len)),
	367	.len = @intCast(wip.root_list.items.len),
	368	.start = @intCast(wip.extra.items.len),
369	369	.compile_log_text = compile_log_str_index,
370	370	});
371	371	try wip.extra.appendSlice(gpa, @as([]const u32, @ptrCast(wip.root_list.items)));
...	...	@@ -385,7 +385,7 @@ pub const Wip = struct {
385	385
386	386	pub fn addString(wip: *Wip, s: []const u8) !u32 {
387	387	const gpa = wip.gpa;
388		const index = @as(u32, @intCast(wip.string_bytes.items.len));
	388	const index: u32 = @intCast(wip.string_bytes.items.len);
389	389	try wip.string_bytes.ensureUnusedCapacity(gpa, s.len + 1);
390	390	wip.string_bytes.appendSliceAssumeCapacity(s);
391	391	wip.string_bytes.appendAssumeCapacity(0);
...	...	@@ -394,7 +394,7 @@ pub const Wip = struct {
394	394
395	395	pub fn printString(wip: *Wip, comptime fmt: []const u8, args: anytype) !u32 {
396	396	const gpa = wip.gpa;
397		const index = @as(u32, @intCast(wip.string_bytes.items.len));
	397	const index: u32 = @intCast(wip.string_bytes.items.len);
398	398	try wip.string_bytes.writer(gpa).print(fmt, args);
399	399	try wip.string_bytes.append(gpa, 0);
400	400	return index;
...	...	@@ -406,15 +406,15 @@ pub const Wip = struct {
406	406	}
407	407
408	408	pub fn addErrorMessage(wip: *Wip, em: ErrorMessage) !MessageIndex {
409		return @as(MessageIndex, @enumFromInt(try addExtra(wip, em)));
	409	return @enumFromInt(try addExtra(wip, em));
410	410	}
411	411
412	412	pub fn addErrorMessageAssumeCapacity(wip: *Wip, em: ErrorMessage) MessageIndex {
413		return @as(MessageIndex, @enumFromInt(addExtraAssumeCapacity(wip, em)));
	413	return @enumFromInt(addExtraAssumeCapacity(wip, em));
414	414	}
415	415
416	416	pub fn addSourceLocation(wip: *Wip, sl: SourceLocation) !SourceLocationIndex {
417		return @as(SourceLocationIndex, @enumFromInt(try addExtra(wip, sl)));
	417	return @enumFromInt(try addExtra(wip, sl));
418	418	}
419	419
420	420	pub fn addReferenceTrace(wip: *Wip, rt: ReferenceTrace) !void {
...	...	@@ -430,7 +430,7 @@ pub const Wip = struct {
430	430	const other_list = other.getMessages();
431	431
432	432	// The ensureUnusedCapacity call above guarantees this.
433		const notes_start = wip.reserveNotes(@as(u32, @intCast(other_list.len))) catch unreachable;
	433	const notes_start = wip.reserveNotes(@intCast(other_list.len)) catch unreachable;
434	434	for (notes_start.., other_list) \|note, message\| {
435	435	wip.extra.items[note] = @intFromEnum(wip.addOtherMessage(other, message) catch unreachable);
436	436	}
...	...	@@ -455,7 +455,7 @@ pub const Wip = struct {
455	455	try wip.extra.ensureUnusedCapacity(wip.gpa, notes_len +
456	456	notes_len * @typeInfo(ErrorBundle.ErrorMessage).Struct.fields.len);
457	457	wip.extra.items.len += notes_len;
458		return @as(u32, @intCast(wip.extra.items.len - notes_len));
	458	return @intCast(wip.extra.items.len - notes_len);
459	459	}
460	460
461	461	fn addOtherMessage(wip: *Wip, other: ErrorBundle, msg_index: MessageIndex) !MessageIndex {
...	...	@@ -510,7 +510,7 @@ pub const Wip = struct {
510	510
511	511	fn addExtraAssumeCapacity(wip: *Wip, extra: anytype) u32 {
512	512	const fields = @typeInfo(@TypeOf(extra)).Struct.fields;
513		const result = @as(u32, @intCast(wip.extra.items.len));
	513	const result: u32 = @intCast(wip.extra.items.len);
514	514	wip.extra.items.len += fields.len;
515	515	setExtra(wip, result, extra);
516	516	return result;

src/Package.zig+250-238

...	...	@@ -10,15 +10,15 @@ const assert = std.debug.assert;
10	10	const log = std.log.scoped(.package);
11	11	const main = @import("main.zig");
12	12	const ThreadPool = std.Thread.Pool;
13		const WaitGroup = std.Thread.WaitGroup;
14	13
15	14	const Compilation = @import("Compilation.zig");
16	15	const Module = @import("Module.zig");
17	16	const Cache = std.Build.Cache;
18	17	const build_options = @import("build_options");
19		const Manifest = @import("Manifest.zig");
20	18	const git = @import("git.zig");
	19	const computePackageHash = @import("Package/hash.zig").compute;
21	20
	21	pub const Manifest = @import("Manifest.zig");
22	22	pub const Table = std.StringHashMapUnmanaged(*Package);
23	23
24	24	root_src_directory: Compilation.Directory,
...	...	@@ -285,7 +285,8 @@ pub fn fetchAndAddDependencies(
285	285	if (manifest.errors.len > 0) {
286	286	const file_path = try directory.join(arena, &.{Manifest.basename});
287	287	for (manifest.errors) \|msg\| {
288		try Report.addErrorMessage(ast, file_path, error_bundle, 0, msg);
	288	const str = try error_bundle.addString(msg.msg);
	289	try Report.addErrorMessage(&ast, file_path, error_bundle, 0, str, msg.tok, msg.off);
289	290	}
290	291	return error.PackageFetchFailed;
291	292	}
...	...	@@ -454,8 +455,8 @@ pub fn createFilePkg(
454	455	return createWithDir(gpa, cache_directory, o_dir_sub_path, basename);
455	456	}
456	457
457		const Report = struct {
458		ast: *const std.zig.Ast,
	458	pub const Report = struct {
	459	ast: ?*const std.zig.Ast,
459	460	directory: Compilation.Directory,
460	461	error_bundle: *std.zig.ErrorBundle.Wip,
461	462
...	...	@@ -464,42 +465,77 @@ const Report = struct {
464	465	tok: std.zig.Ast.TokenIndex,
465	466	comptime fmt_string: []const u8,
466	467	fmt_args: anytype,
	468	) error{ PackageFetchFailed, OutOfMemory } {
	469	const msg = try report.error_bundle.printString(fmt_string, fmt_args);
	470	return failMsg(report, tok, msg);
	471	}
	472
	473	fn failMsg(
	474	report: Report,
	475	tok: std.zig.Ast.TokenIndex,
	476	msg: u32,
467	477	) error{ PackageFetchFailed, OutOfMemory } {
468	478	const gpa = report.error_bundle.gpa;
469	479
470	480	const file_path = try report.directory.join(gpa, &.{Manifest.basename});
471	481	defer gpa.free(file_path);
472	482
473		const msg = try std.fmt.allocPrint(gpa, fmt_string, fmt_args);
474		defer gpa.free(msg);
	483	const eb = report.error_bundle;
475	484
476		try addErrorMessage(report.ast.*, file_path, report.error_bundle, 0, .{
477		.tok = tok,
478		.off = 0,
479		.msg = msg,
480		});
	485	if (report.ast) \|ast\| {
	486	try addErrorMessage(ast, file_path, eb, 0, msg, tok, 0);
	487	} else {
	488	try eb.addRootErrorMessage(.{
	489	.msg = msg,
	490	.src_loc = .none,
	491	.notes_len = 0,
	492	});
	493	}
481	494
482	495	return error.PackageFetchFailed;
483	496	}
484	497
	498	fn addErrorWithNotes(
	499	report: Report,
	500	notes_len: u32,
	501	msg: Manifest.ErrorMessage,
	502	) error{OutOfMemory}!void {
	503	const eb = report.error_bundle;
	504	const msg_str = try eb.addString(msg.msg);
	505	if (report.ast) \|ast\| {
	506	const gpa = eb.gpa;
	507	const file_path = try report.directory.join(gpa, &.{Manifest.basename});
	508	defer gpa.free(file_path);
	509	return addErrorMessage(ast, file_path, eb, notes_len, msg_str, msg.tok, msg.off);
	510	} else {
	511	return eb.addRootErrorMessage(.{
	512	.msg = msg_str,
	513	.src_loc = .none,
	514	.notes_len = notes_len,
	515	});
	516	}
	517	}
	518
485	519	fn addErrorMessage(
486		ast: std.zig.Ast,
	520	ast: *const std.zig.Ast,
487	521	file_path: []const u8,
488	522	eb: *std.zig.ErrorBundle.Wip,
489	523	notes_len: u32,
490		msg: Manifest.ErrorMessage,
	524	msg_str: u32,
	525	msg_tok: std.zig.Ast.TokenIndex,
	526	msg_off: u32,
491	527	) error{OutOfMemory}!void {
492	528	const token_starts = ast.tokens.items(.start);
493		const start_loc = ast.tokenLocation(0, msg.tok);
	529	const start_loc = ast.tokenLocation(0, msg_tok);
494	530
495	531	try eb.addRootErrorMessage(.{
496		.msg = try eb.addString(msg.msg),
	532	.msg = msg_str,
497	533	.src_loc = try eb.addSourceLocation(.{
498	534	.src_path = try eb.addString(file_path),
499		.span_start = token_starts[msg.tok],
500		.span_end = @as(u32, @intCast(token_starts[msg.tok] + ast.tokenSlice(msg.tok).len)),
501		.span_main = token_starts[msg.tok] + msg.off,
502		.line = @as(u32, @intCast(start_loc.line)),
	535	.span_start = token_starts[msg_tok],
	536	.span_end = @as(u32, @intCast(token_starts[msg_tok] + ast.tokenSlice(msg_tok).len)),
	537	.span_main = token_starts[msg_tok] + msg_off,
	538	.line = @intCast(start_loc.line),
503	539	.column = @as(u32, @intCast(start_loc.column)),
504	540	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
505	541	}),
...	...	@@ -508,7 +544,7 @@ const Report = struct {
508	544	}
509	545	};
510	546
511		const FetchLocation = union(enum) {
	547	pub const FetchLocation = union(enum) {
512	548	/// The relative path to a file or directory.
513	549	/// This may be a file that requires unpacking (such as a .tar.gz),
514	550	/// or the path to the root directory of a package.
...	...	@@ -517,30 +553,27 @@ const FetchLocation = union(enum) {
517	553	http_request: std.Uri,
518	554	git_request: std.Uri,
519	555
520		pub fn init(gpa: Allocator, dep: Manifest.Dependency, root_dir: Compilation.Directory, report: Report) !FetchLocation {
	556	pub fn init(
	557	gpa: Allocator,
	558	dep: Manifest.Dependency,
	559	root_dir: Compilation.Directory,
	560	report: Report,
	561	) !FetchLocation {
521	562	switch (dep.location) {
522	563	.url => \|url\| {
523	564	const uri = std.Uri.parse(url) catch \|err\| switch (err) {
524	565	error.UnexpectedCharacter => return report.fail(dep.location_tok, "failed to parse dependency location as URI", .{}),
525	566	else => return err,
526	567	};
527		if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
528		return report.fail(dep.location_tok, "'file' scheme is not allowed for URLs. Use '.path' instead", .{});
529		} else if (ascii.eqlIgnoreCase(uri.scheme, "http") or ascii.eqlIgnoreCase(uri.scheme, "https")) {
530		return .{ .http_request = uri };
531		} else if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or ascii.eqlIgnoreCase(uri.scheme, "git+https")) {
532		return .{ .git_request = uri };
533		} else {
534		return report.fail(dep.location_tok, "Unsupported URL scheme: {s}", .{uri.scheme});
535		}
	568	return initUri(uri, dep.location_tok, report);
536	569	},
537	570	.path => \|path\| {
538	571	if (fs.path.isAbsolute(path)) {
539		return report.fail(dep.location_tok, "Absolute paths are not allowed. Use a relative path instead", .{});
	572	return report.fail(dep.location_tok, "absolute paths are not allowed. Use a relative path instead", .{});
540	573	}
541	574
542	575	const is_dir = isDirectory(root_dir, path) catch \|err\| switch (err) {
543		error.FileNotFound => return report.fail(dep.location_tok, "File not found: {s}", .{path}),
	576	error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{path}),
544	577	else => return err,
545	578	};
546	579
...	...	@@ -552,9 +585,21 @@ const FetchLocation = union(enum) {
552	585	}
553	586	}
554	587
	588	pub fn initUri(uri: std.Uri, location_tok: std.zig.Ast.TokenIndex, report: Report) !FetchLocation {
	589	if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
	590	return report.fail(location_tok, "'file' scheme is not allowed for URLs. Use '.path' instead", .{});
	591	} else if (ascii.eqlIgnoreCase(uri.scheme, "http") or ascii.eqlIgnoreCase(uri.scheme, "https")) {
	592	return .{ .http_request = uri };
	593	} else if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or ascii.eqlIgnoreCase(uri.scheme, "git+https")) {
	594	return .{ .git_request = uri };
	595	} else {
	596	return report.fail(location_tok, "unsupported URL scheme: {s}", .{uri.scheme});
	597	}
	598	}
	599
555	600	pub fn deinit(f: *FetchLocation, gpa: Allocator) void {
556	601	switch (f.*) {
557		inline .file, .directory => \|path\| gpa.free(path),
	602	.file, .directory => \|path\| gpa.free(path),
558	603	.http_request, .git_request => {},
559	604	}
560	605	f.* = undefined;
...	...	@@ -565,7 +610,7 @@ const FetchLocation = union(enum) {
565	610	gpa: Allocator,
566	611	root_dir: Compilation.Directory,
567	612	http_client: *std.http.Client,
568		dep: Manifest.Dependency,
	613	dep_location_tok: std.zig.Ast.TokenIndex,
569	614	report: Report,
570	615	) !ReadableResource {
571	616	switch (f) {
...	...	@@ -588,7 +633,7 @@ const FetchLocation = union(enum) {
588	633	try req.wait();
589	634
590	635	if (req.response.status != .ok) {
591		return report.fail(dep.location_tok, "Expected response status '200 OK' got '{} {s}'", .{
	636	return report.fail(dep_location_tok, "expected response status '200 OK' got '{} {s}'", .{
592	637	@intFromEnum(req.response.status),
593	638	req.response.status.phrase() orelse "",
594	639	});
...	...	@@ -607,7 +652,7 @@ const FetchLocation = union(enum) {
607	652	session.discoverCapabilities(gpa, &redirect_uri) catch \|e\| switch (e) {
608	653	error.Redirected => {
609	654	defer gpa.free(redirect_uri);
610		return report.fail(dep.location_tok, "Repository moved to {s}", .{redirect_uri});
	655	return report.fail(dep_location_tok, "repository moved to {s}", .{redirect_uri});
611	656	},
612	657	else => \|other\| return other,
613	658	};
...	...	@@ -634,19 +679,16 @@ const FetchLocation = union(enum) {
634	679	break :want_oid ref.peeled orelse ref.oid;
635	680	}
636	681	}
637		return report.fail(dep.location_tok, "Ref not found: {s}", .{want_ref});
	682	return report.fail(dep_location_tok, "ref not found: {s}", .{want_ref});
638	683	};
639	684	if (uri.fragment == null) {
640		const file_path = try report.directory.join(gpa, &.{Manifest.basename});
641		defer gpa.free(file_path);
642
643		const eb = report.error_bundle;
644	685	const notes_len = 1;
645		try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{
646		.tok = dep.location_tok,
	686	try report.addErrorWithNotes(notes_len, .{
	687	.tok = dep_location_tok,
647	688	.off = 0,
648	689	.msg = "url field is missing an explicit ref",
649	690	});
	691	const eb = report.error_bundle;
650	692	const notes_start = try eb.reserveNotes(notes_len);
651	693	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
652	694	.msg = try eb.printString("try .url = \"{+/}#{}\",", .{ uri, std.fmt.fmtSliceHexLower(&want_oid) }),
...	...	@@ -669,12 +711,13 @@ const FetchLocation = union(enum) {
669	711	}
670	712	};
671	713
672		const ReadableResource = struct {
	714	pub const ReadableResource = struct {
673	715	path: []const u8,
674	716	resource: union(enum) {
675	717	file: fs.File,
676	718	http_request: std.http.Client.Request,
677	719	git_fetch_stream: git.Session.FetchStream,
	720	dir: fs.IterableDir,
678	721	},
679	722
680	723	/// Unpack the package into the global cache directory.
...	...	@@ -685,12 +728,12 @@ const ReadableResource = struct {
685	728	allocator: Allocator,
686	729	thread_pool: *ThreadPool,
687	730	global_cache_directory: Compilation.Directory,
688		dep: Manifest.Dependency,
	731	dep_location_tok: std.zig.Ast.TokenIndex,
689	732	report: Report,
690	733	pkg_prog_node: *std.Progress.Node,
691	734	) !PackageLocation {
692	735	switch (rr.resource) {
693		inline .file, .http_request, .git_fetch_stream => \|*r\| {
	736	inline .file, .http_request, .git_fetch_stream, .dir => \|*r, tag\| {
694	737	const s = fs.path.sep_str;
695	738	const rand_int = std.crypto.random.int(u64);
696	739	const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);
...	...	@@ -710,45 +753,58 @@ const ReadableResource = struct {
710	753	};
711	754	defer tmp_directory.closeAndFree(allocator);
712	755
713		const opt_content_length = try rr.getSize();
714
715		var prog_reader: ProgressReader(@TypeOf(r.reader())) = .{
716		.child_reader = r.reader(),
717		.prog_node = pkg_prog_node,
718		.unit = if (opt_content_length) \|content_length\| unit: {
719		const kib = content_length / 1024;
720		const mib = kib / 1024;
721		if (mib > 0) {
722		pkg_prog_node.setEstimatedTotalItems(@intCast(mib));
723		pkg_prog_node.setUnit("MiB");
724		break :unit .mib;
725		} else {
726		pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib)));
727		pkg_prog_node.setUnit("KiB");
728		break :unit .kib;
	756	if (tag != .dir) {
	757	const opt_content_length = try rr.getSize();
	758
	759	var prog_reader: ProgressReader(@TypeOf(r.reader())) = .{
	760	.child_reader = r.reader(),
	761	.prog_node = pkg_prog_node,
	762	.unit = if (opt_content_length) \|content_length\| unit: {
	763	const kib = content_length / 1024;
	764	const mib = kib / 1024;
	765	if (mib > 0) {
	766	pkg_prog_node.setEstimatedTotalItems(@intCast(mib));
	767	pkg_prog_node.setUnit("MiB");
	768	break :unit .mib;
	769	} else {
	770	pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib)));
	771	pkg_prog_node.setUnit("KiB");
	772	break :unit .kib;
	773	}
	774	} else .any,
	775	};
	776
	777	switch (try rr.getFileType(dep_location_tok, report)) {
	778	.tar => try unpackTarball(allocator, prog_reader.reader(), tmp_directory.handle, dep_location_tok, report),
	779	.@"tar.gz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, dep_location_tok, report, std.compress.gzip),
	780	.@"tar.xz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, dep_location_tok, report, std.compress.xz),
	781	.git_pack => try unpackGitPack(allocator, &prog_reader, git.parseOid(rr.path) catch unreachable, tmp_directory.handle, dep_location_tok, report),
	782	}
	783	} else {
	784	// Recursive directory copy.
	785	var it = try r.walk(allocator);
	786	defer it.deinit();
	787	while (try it.next()) \|entry\| {
	788	switch (entry.kind) {
	789	.directory => try tmp_directory.handle.makePath(entry.path),
	790	.file => try r.dir.copyFile(
	791	entry.path,
	792	tmp_directory.handle,
	793	entry.path,
	794	.{},
	795	),
	796	.sym_link => {
	797	var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
	798	const link_name = try r.dir.readLink(entry.path, &buf);
	799	// TODO: if this would create a symlink to outside
	800	// the destination directory, fail with an error instead.
	801	try tmp_directory.handle.symLink(link_name, entry.path, .{});
	802	},
	803	else => return error.IllegalFileTypeInPackage,
729	804	}
730		} else .any,
731		};
732		pkg_prog_node.context.refresh();
733
734		switch (try rr.getFileType(dep, report)) {
735		.@"tar.gz" => try unpackTarball(allocator, prog_reader, tmp_directory.handle, std.compress.gzip),
736		// I have not checked what buffer sizes the xz decompression implementation uses
737		// by default, so the same logic applies for buffering the reader as for gzip.
738		.@"tar.xz" => try unpackTarball(allocator, prog_reader, tmp_directory.handle, std.compress.xz),
739		.git_pack => try unpackGitPack(allocator, &prog_reader, git.parseOid(rr.path) catch unreachable, tmp_directory.handle),
	805	}
740	806	}
741	807
742		// Unpack completed - stop showing amount as progress
743		pkg_prog_node.setEstimatedTotalItems(0);
744		pkg_prog_node.setCompletedItems(0);
745		pkg_prog_node.context.refresh();
746
747		// TODO: delete files not included in the package prior to computing the package hash.
748		// for example, if the ini file has directives to include/not include certain files,
749		// apply those rules directly to the filesystem right here. This ensures that files
750		// not protected by the hash are not present on the file system.
751
752	808	break :h try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle });
753	809	};
754	810
...	...	@@ -769,6 +825,7 @@ const ReadableResource = struct {
769	825	}
770	826
771	827	const FileType = enum {
	828	tar,
772	829	@"tar.gz",
773	830	@"tar.xz",
774	831	git_pack,
...	...	@@ -780,21 +837,28 @@ const ReadableResource = struct {
780	837	// TODO: Handle case of chunked content-length
781	838	.http_request => \|req\| return req.response.content_length,
782	839	.git_fetch_stream => \|stream\| return stream.request.response.content_length,
	840	.dir => unreachable,
783	841	}
784	842	}
785	843
786		pub fn getFileType(rr: ReadableResource, dep: Manifest.Dependency, report: Report) !FileType {
	844	pub fn getFileType(
	845	rr: ReadableResource,
	846	dep_location_tok: std.zig.Ast.TokenIndex,
	847	report: Report,
	848	) !FileType {
787	849	switch (rr.resource) {
788	850	.file => {
789	851	return fileTypeFromPath(rr.path) orelse
790		return report.fail(dep.location_tok, "Unknown file type", .{});
	852	return report.fail(dep_location_tok, "unknown file type", .{});
791	853	},
792	854	.http_request => \|req\| {
793	855	const content_type = req.response.headers.getFirstValue("Content-Type") orelse
794		return report.fail(dep.location_tok, "Missing 'Content-Type' header", .{});
	856	return report.fail(dep_location_tok, "missing 'Content-Type' header", .{});
795	857
796	858	// If the response has a different content type than the URI indicates, override
797	859	// the previously assumed file type.
	860	if (ascii.eqlIgnoreCase(content_type, "application/x-tar")) return .tar;
	861
798	862	return if (ascii.eqlIgnoreCase(content_type, "application/gzip") or
799	863	ascii.eqlIgnoreCase(content_type, "application/x-gzip") or
800	864	ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))
...	...	@@ -805,22 +869,21 @@ const ReadableResource = struct {
805	869	// support gitlab tarball urls such as https://gitlab.com/<namespace>/<project>/-/archive/<sha>/<project>-<sha>.tar.gz
806	870	// whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"'
807	871	const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse
808		return report.fail(dep.location_tok, "Missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});
	872	return report.fail(dep_location_tok, "missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});
809	873	break :ty getAttachmentType(content_disposition) orelse
810		return report.fail(dep.location_tok, "Unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});
811		} else return report.fail(dep.location_tok, "Unrecognized value for 'Content-Type' header: {s}", .{content_type});
	874	return report.fail(dep_location_tok, "unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});
	875	} else return report.fail(dep_location_tok, "unrecognized value for 'Content-Type' header: {s}", .{content_type});
812	876	},
813	877	.git_fetch_stream => return .git_pack,
	878	.dir => unreachable,
814	879	}
815	880	}
816	881
817	882	fn fileTypeFromPath(file_path: []const u8) ?FileType {
818		return if (ascii.endsWithIgnoreCase(file_path, ".tar.gz"))
819		.@"tar.gz"
820		else if (ascii.endsWithIgnoreCase(file_path, ".tar.xz"))
821		.@"tar.xz"
822		else
823		null;
	883	if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar;
	884	if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz";
	885	if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz";
	886	return null;
824	887	}
825	888
826	889	fn getAttachmentType(content_disposition: []const u8) ?FileType {
...	...	@@ -847,6 +910,7 @@ const ReadableResource = struct {
847	910	.file => \|file\| file.close(),
848	911	.http_request => \|*req\| req.deinit(),
849	912	.git_fetch_stream => \|*stream\| stream.deinit(),
	913	.dir => \|*dir\| dir.close(),
850	914	}
851	915	rr.* = undefined;
852	916	}
...	...	@@ -908,7 +972,7 @@ fn ProgressReader(comptime ReaderType: type) type {
908	972	}
909	973	},
910	974	}
911		self.prog_node.context.maybeRefresh();
	975	self.prog_node.activate();
912	976	return amt;
913	977	}
914	978
...	...	@@ -993,7 +1057,7 @@ fn getDirectoryModule(
993	1057	if (all_modules.get(hex_digest)) \|mod\| return .{ mod.?, true };
994	1058
995	1059	var pkg_dir = directory.handle.openDir(fetch_location.directory, .{}) catch \|err\| switch (err) {
996		error.FileNotFound => return report.fail(dep.location_tok, "File not found: {s}", .{fetch_location.directory}),
	1060	error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{fetch_location.directory}),
997	1061	else => \|e\| return e,
998	1062	};
999	1063	defer pkg_dir.close();
...	...	@@ -1032,12 +1096,18 @@ fn fetchAndUnpack(
1032	1096	var pkg_prog_node = root_prog_node.start(name_for_prog, 0);
1033	1097	defer pkg_prog_node.end();
1034	1098	pkg_prog_node.activate();
1035		pkg_prog_node.context.refresh();
1036	1099
1037		var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep, report);
	1100	var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep.location_tok, report);
1038	1101	defer readable_resource.deinit(gpa);
1039	1102
1040		var package_location = try readable_resource.unpack(gpa, thread_pool, global_cache_directory, dep, report, &pkg_prog_node);
	1103	var package_location = try readable_resource.unpack(
	1104	gpa,
	1105	thread_pool,
	1106	global_cache_directory,
	1107	dep.location_tok,
	1108	report,
	1109	&pkg_prog_node,
	1110	);
1041	1111	defer package_location.deinit(gpa);
1042	1112
1043	1113	const actual_hex = Manifest.hexDigest(package_location.hash);
...	...	@@ -1048,16 +1118,13 @@ fn fetchAndUnpack(
1048	1118	});
1049	1119	}
1050	1120	} else {
1051		const file_path = try report.directory.join(gpa, &.{Manifest.basename});
1052		defer gpa.free(file_path);
1053
1054		const eb = report.error_bundle;
1055	1121	const notes_len = 1;
1056		try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{
	1122	try report.addErrorWithNotes(notes_len, .{
1057	1123	.tok = dep.location_tok,
1058	1124	.off = 0,
1059	1125	.msg = "dependency is missing hash field",
1060	1126	});
	1127	const eb = report.error_bundle;
1061	1128	const notes_start = try eb.reserveNotes(notes_len);
1062	1129	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
1063	1130	.msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),
...	...	@@ -1080,18 +1147,34 @@ fn fetchAndUnpack(
1080	1147	return module;
1081	1148	}
1082	1149
1083		fn unpackTarball(
	1150	fn unpackTarballCompressed(
1084	1151	gpa: Allocator,
1085	1152	reader: anytype,
1086	1153	out_dir: fs.Dir,
1087		comptime compression: type,
	1154	dep_location_tok: std.zig.Ast.TokenIndex,
	1155	report: Report,
	1156	comptime Compression: type,
1088	1157	) !void {
1089	1158	var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1090	1159
1091		var decompress = try compression.decompress(gpa, br.reader());
	1160	var decompress = try Compression.decompress(gpa, br.reader());
1092	1161	defer decompress.deinit();
1093	1162
1094		try std.tar.pipeToFileSystem(out_dir, decompress.reader(), .{
	1163	return unpackTarball(gpa, decompress.reader(), out_dir, dep_location_tok, report);
	1164	}
	1165
	1166	fn unpackTarball(
	1167	gpa: Allocator,
	1168	reader: anytype,
	1169	out_dir: fs.Dir,
	1170	dep_location_tok: std.zig.Ast.TokenIndex,
	1171	report: Report,
	1172	) !void {
	1173	var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = gpa };
	1174	defer diagnostics.deinit();
	1175
	1176	try std.tar.pipeToFileSystem(out_dir, reader, .{
	1177	.diagnostics = &diagnostics,
1095	1178	.strip_components = 1,
1096	1179	// TODO: we would like to set this to executable_bit_only, but two
1097	1180	// things need to happen before that:
...	...	@@ -1100,6 +1183,36 @@ fn unpackTarball(
1100	1183	// bit on Windows from the ACLs (see the isExecutable function).
1101	1184	.mode_mode = .ignore,
1102	1185	});
	1186
	1187	if (diagnostics.errors.items.len > 0) {
	1188	const notes_len: u32 = @intCast(diagnostics.errors.items.len);
	1189	try report.addErrorWithNotes(notes_len, .{
	1190	.tok = dep_location_tok,
	1191	.off = 0,
	1192	.msg = "unable to unpack tarball",
	1193	});
	1194	const eb = report.error_bundle;
	1195	const notes_start = try eb.reserveNotes(notes_len);
	1196	for (diagnostics.errors.items, notes_start..) \|item, note_i\| {
	1197	switch (item) {
	1198	.unable_to_create_sym_link => \|info\| {
	1199	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
	1200	.msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{
	1201	info.file_name, info.link_name, @errorName(info.code),
	1202	}),
	1203	}));
	1204	},
	1205	.unsupported_file_type => \|info\| {
	1206	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
	1207	.msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{
	1208	info.file_name, @intFromEnum(info.file_type),
	1209	}),
	1210	}));
	1211	},
	1212	}
	1213	}
	1214	return error.InvalidTarball;
	1215	}
1103	1216	}
1104	1217
1105	1218	fn unpackGitPack(
...	...	@@ -1107,6 +1220,8 @@ fn unpackGitPack(
1107	1220	reader: anytype,
1108	1221	want_oid: git.Oid,
1109	1222	out_dir: fs.Dir,
	1223	dep_location_tok: std.zig.Ast.TokenIndex,
	1224	report: Report,
1110	1225	) !void {
1111	1226	// The .git directory is used to store the packfile and associated index, but
1112	1227	// we do not attempt to replicate the exact structure of a real .git
...	...	@@ -1126,7 +1241,6 @@ fn unpackGitPack(
1126	1241	var index_prog_node = reader.prog_node.start("Index pack", 0);
1127	1242	defer index_prog_node.end();
1128	1243	index_prog_node.activate();
1129		index_prog_node.context.refresh();
1130	1244	var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
1131	1245	try git.indexPack(gpa, pack_file, index_buffered_writer.writer());
1132	1246	try index_buffered_writer.flush();
...	...	@@ -1137,89 +1251,38 @@ fn unpackGitPack(
1137	1251	var checkout_prog_node = reader.prog_node.start("Checkout", 0);
1138	1252	defer checkout_prog_node.end();
1139	1253	checkout_prog_node.activate();
1140		checkout_prog_node.context.refresh();
1141	1254	var repository = try git.Repository.init(gpa, pack_file, index_file);
1142	1255	defer repository.deinit();
1143		try repository.checkout(out_dir, want_oid);
1144		}
1145		}
1146
1147		try out_dir.deleteTree(".git");
1148		}
1149
1150		const HashedFile = struct {
1151		fs_path: []const u8,
1152		normalized_path: []const u8,
1153		hash: [Manifest.Hash.digest_length]u8,
1154		failure: Error!void,
1155
1156		const Error = fs.File.OpenError \|\| fs.File.ReadError \|\| fs.File.StatError;
1157
1158		fn lessThan(context: void, lhs: const HashedFile, rhs: const HashedFile) bool {
1159		_ = context;
1160		return mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path);
1161		}
1162		};
1163
1164		fn computePackageHash(
1165		thread_pool: *ThreadPool,
1166		pkg_dir: fs.IterableDir,
1167		) ![Manifest.Hash.digest_length]u8 {
1168		const gpa = thread_pool.allocator;
1169
1170		// We'll use an arena allocator for the path name strings since they all
1171		// need to be in memory for sorting.
1172		var arena_instance = std.heap.ArenaAllocator.init(gpa);
1173		defer arena_instance.deinit();
1174		const arena = arena_instance.allocator();
1175
1176		// Collect all files, recursively, then sort.
1177		var all_files = std.ArrayList(*HashedFile).init(gpa);
1178		defer all_files.deinit();
1179
1180		var walker = try pkg_dir.walk(gpa);
1181		defer walker.deinit();
1182
1183		{
1184		// The final hash will be a hash of each file hashed independently. This
1185		// allows hashing in parallel.
1186		var wait_group: WaitGroup = .{};
1187		defer wait_group.wait();
1188
1189		while (try walker.next()) \|entry\| {
1190		switch (entry.kind) {
1191		.directory => continue,
1192		.file => {},
1193		else => return error.IllegalFileTypeInPackage,
	1256	var diagnostics: git.Diagnostics = .{ .allocator = gpa };
	1257	defer diagnostics.deinit();
	1258	try repository.checkout(out_dir, want_oid, &diagnostics);
	1259
	1260	if (diagnostics.errors.items.len > 0) {
	1261	const notes_len: u32 = @intCast(diagnostics.errors.items.len);
	1262	try report.addErrorWithNotes(notes_len, .{
	1263	.tok = dep_location_tok,
	1264	.off = 0,
	1265	.msg = "unable to unpack packfile",
	1266	});
	1267	const eb = report.error_bundle;
	1268	const notes_start = try eb.reserveNotes(notes_len);
	1269	for (diagnostics.errors.items, notes_start..) \|item, note_i\| {
	1270	switch (item) {
	1271	.unable_to_create_sym_link => \|info\| {
	1272	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
	1273	.msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{
	1274	info.file_name, info.link_name, @errorName(info.code),
	1275	}),
	1276	}));
	1277	},
	1278	}
	1279	}
	1280	return error.InvalidGitPack;
1194	1281	}
1195		const hashed_file = try arena.create(HashedFile);
1196		const fs_path = try arena.dupe(u8, entry.path);
1197		hashed_file.* = .{
1198		.fs_path = fs_path,
1199		.normalized_path = try normalizePath(arena, fs_path),
1200		.hash = undefined, // to be populated by the worker
1201		.failure = undefined, // to be populated by the worker
1202		};
1203		wait_group.start();
1204		try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group });
1205
1206		try all_files.append(hashed_file);
1207	1282	}
1208	1283	}
1209	1284
1210		mem.sort(*HashedFile, all_files.items, {}, HashedFile.lessThan);
1211
1212		var hasher = Manifest.Hash.init(.{});
1213		var any_failures = false;
1214		for (all_files.items) \|hashed_file\| {
1215		hashed_file.failure catch \|err\| {
1216		any_failures = true;
1217		std.log.err("unable to hash '{s}': {s}", .{ hashed_file.fs_path, @errorName(err) });
1218		};
1219		hasher.update(&hashed_file.hash);
1220		}
1221		if (any_failures) return error.PackageHashUnavailable;
1222		return hasher.finalResult();
	1285	try out_dir.deleteTree(".git");
1223	1286	}
1224	1287
1225	1288	/// Compute the hash of a file path.
...	...	@@ -1240,57 +1303,6 @@ fn isDirectory(root_dir: Compilation.Directory, path: []const u8) !bool {
1240	1303	return true;
1241	1304	}
1242	1305
1243		/// Make a file system path identical independently of operating system path inconsistencies.
1244		/// This converts backslashes into forward slashes.
1245		fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
1246		const canonical_sep = '/';
1247
1248		if (fs.path.sep == canonical_sep)
1249		return fs_path;
1250
1251		const normalized = try arena.dupe(u8, fs_path);
1252		for (normalized) \|*byte\| {
1253		switch (byte.*) {
1254		fs.path.sep => byte.* = canonical_sep,
1255		else => continue,
1256		}
1257		}
1258		return normalized;
1259		}
1260
1261		fn workerHashFile(dir: fs.Dir, hashed_file: HashedFile, wg: WaitGroup) void {
1262		defer wg.finish();
1263		hashed_file.failure = hashFileFallible(dir, hashed_file);
1264		}
1265
1266		fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
1267		var buf: [8000]u8 = undefined;
1268		var file = try dir.openFile(hashed_file.fs_path, .{});
1269		defer file.close();
1270		var hasher = Manifest.Hash.init(.{});
1271		hasher.update(hashed_file.normalized_path);
1272		hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) });
1273		while (true) {
1274		const bytes_read = try file.read(&buf);
1275		if (bytes_read == 0) break;
1276		hasher.update(buf[0..bytes_read]);
1277		}
1278		hasher.final(&hashed_file.hash);
1279		}
1280
1281		fn isExecutable(file: fs.File) !bool {
1282		if (builtin.os.tag == .windows) {
1283		// TODO check the ACL on Windows.
1284		// Until this is implemented, this could be a false negative on
1285		// Windows, which is why we do not yet set executable_bit_only above
1286		// when unpacking the tarball.
1287		return false;
1288		} else {
1289		const stat = try file.stat();
1290		return (stat.mode & std.os.S.IXUSR) != 0;
1291		}
1292		}
1293
1294	1306	fn renameTmpIntoCache(
1295	1307	cache_dir: fs.Dir,
1296	1308	tmp_dir_sub_path: []const u8,

src/Package/hash.zig created+153

...	...	@@ -0,0 +1,153 @@
	1	const builtin = @import("builtin");
	2	const std = @import("std");
	3	const fs = std.fs;
	4	const ThreadPool = std.Thread.Pool;
	5	const WaitGroup = std.Thread.WaitGroup;
	6	const Allocator = std.mem.Allocator;
	7
	8	const Hash = @import("../Manifest.zig").Hash;
	9
	10	pub fn compute(thread_pool: *ThreadPool, pkg_dir: fs.IterableDir) ![Hash.digest_length]u8 {
	11	const gpa = thread_pool.allocator;
	12
	13	// We'll use an arena allocator for the path name strings since they all
	14	// need to be in memory for sorting.
	15	var arena_instance = std.heap.ArenaAllocator.init(gpa);
	16	defer arena_instance.deinit();
	17	const arena = arena_instance.allocator();
	18
	19	// TODO: delete files not included in the package prior to computing the package hash.
	20	// for example, if the ini file has directives to include/not include certain files,
	21	// apply those rules directly to the filesystem right here. This ensures that files
	22	// not protected by the hash are not present on the file system.
	23
	24	// Collect all files, recursively, then sort.
	25	var all_files = std.ArrayList(*HashedFile).init(gpa);
	26	defer all_files.deinit();
	27
	28	var walker = try pkg_dir.walk(gpa);
	29	defer walker.deinit();
	30
	31	{
	32	// The final hash will be a hash of each file hashed independently. This
	33	// allows hashing in parallel.
	34	var wait_group: WaitGroup = .{};
	35	defer wait_group.wait();
	36
	37	while (try walker.next()) \|entry\| {
	38	const kind: HashedFile.Kind = switch (entry.kind) {
	39	.directory => continue,
	40	.file => .file,
	41	.sym_link => .sym_link,
	42	else => return error.IllegalFileTypeInPackage,
	43	};
	44	const hashed_file = try arena.create(HashedFile);
	45	const fs_path = try arena.dupe(u8, entry.path);
	46	hashed_file.* = .{
	47	.fs_path = fs_path,
	48	.normalized_path = try normalizePath(arena, fs_path),
	49	.kind = kind,
	50	.hash = undefined, // to be populated by the worker
	51	.failure = undefined, // to be populated by the worker
	52	};
	53	wait_group.start();
	54	try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group });
	55
	56	try all_files.append(hashed_file);
	57	}
	58	}
	59
	60	std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
	61
	62	var hasher = Hash.init(.{});
	63	var any_failures = false;
	64	for (all_files.items) \|hashed_file\| {
	65	hashed_file.failure catch \|err\| {
	66	any_failures = true;
	67	std.log.err("unable to hash '{s}': {s}", .{ hashed_file.fs_path, @errorName(err) });
	68	};
	69	hasher.update(&hashed_file.hash);
	70	}
	71	if (any_failures) return error.PackageHashUnavailable;
	72	return hasher.finalResult();
	73	}
	74
	75	const HashedFile = struct {
	76	fs_path: []const u8,
	77	normalized_path: []const u8,
	78	hash: [Hash.digest_length]u8,
	79	failure: Error!void,
	80	kind: Kind,
	81
	82	const Error =
	83	fs.File.OpenError \|\|
	84	fs.File.ReadError \|\|
	85	fs.File.StatError \|\|
	86	fs.Dir.ReadLinkError;
	87
	88	const Kind = enum { file, sym_link };
	89
	90	fn lessThan(context: void, lhs: const HashedFile, rhs: const HashedFile) bool {
	91	_ = context;
	92	return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path);
	93	}
	94	};
	95
	96	/// Make a file system path identical independently of operating system path inconsistencies.
	97	/// This converts backslashes into forward slashes.
	98	fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
	99	const canonical_sep = '/';
	100
	101	if (fs.path.sep == canonical_sep)
	102	return fs_path;
	103
	104	const normalized = try arena.dupe(u8, fs_path);
	105	for (normalized) \|*byte\| {
	106	switch (byte.*) {
	107	fs.path.sep => byte.* = canonical_sep,
	108	else => continue,
	109	}
	110	}
	111	return normalized;
	112	}
	113
	114	fn workerHashFile(dir: fs.Dir, hashed_file: HashedFile, wg: WaitGroup) void {
	115	defer wg.finish();
	116	hashed_file.failure = hashFileFallible(dir, hashed_file);
	117	}
	118
	119	fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
	120	var buf: [8000]u8 = undefined;
	121	var hasher = Hash.init(.{});
	122	hasher.update(hashed_file.normalized_path);
	123	switch (hashed_file.kind) {
	124	.file => {
	125	var file = try dir.openFile(hashed_file.fs_path, .{});
	126	defer file.close();
	127	hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) });
	128	while (true) {
	129	const bytes_read = try file.read(&buf);
	130	if (bytes_read == 0) break;
	131	hasher.update(buf[0..bytes_read]);
	132	}
	133	},
	134	.sym_link => {
	135	const link_name = try dir.readLink(hashed_file.fs_path, &buf);
	136	hasher.update(link_name);
	137	},
	138	}
	139	hasher.final(&hashed_file.hash);
	140	}
	141
	142	fn isExecutable(file: fs.File) !bool {
	143	if (builtin.os.tag == .windows) {
	144	// TODO check the ACL on Windows.
	145	// Until this is implemented, this could be a false negative on
	146	// Windows, which is why we do not yet set executable_bit_only above
	147	// when unpacking the tarball.
	148	return false;
	149	} else {
	150	const stat = try file.stat();
	151	return (stat.mode & std.os.S.IXUSR) != 0;
	152	}
	153	}

src/git.zig+50-3

...	...	@@ -38,6 +38,32 @@ test parseOid {
38	38	try testing.expectError(error.InvalidOid, parseOid("HEAD"));
39	39	}
40	40
	41	pub const Diagnostics = struct {
	42	allocator: Allocator,
	43	errors: std.ArrayListUnmanaged(Error) = .{},
	44
	45	pub const Error = union(enum) {
	46	unable_to_create_sym_link: struct {
	47	code: anyerror,
	48	file_name: []const u8,
	49	link_name: []const u8,
	50	},
	51	};
	52
	53	pub fn deinit(d: *Diagnostics) void {
	54	for (d.errors.items) \|item\| {
	55	switch (item) {
	56	.unable_to_create_sym_link => \|info\| {
	57	d.allocator.free(info.file_name);
	58	d.allocator.free(info.link_name);
	59	},
	60	}
	61	}
	62	d.errors.deinit(d.allocator);
	63	d.* = undefined;
	64	}
	65	};
	66
41	67	pub const Repository = struct {
42	68	odb: Odb,
43	69
...	...	@@ -55,6 +81,7 @@ pub const Repository = struct {
55	81	repository: *Repository,
56	82	worktree: std.fs.Dir,
57	83	commit_oid: Oid,
	84	diagnostics: *Diagnostics,
58	85	) !void {
59	86	try repository.odb.seekOid(commit_oid);
60	87	const tree_oid = tree_oid: {
...	...	@@ -62,7 +89,7 @@ pub const Repository = struct {
62	89	if (commit_object.type != .commit) return error.NotACommit;
63	90	break :tree_oid try getCommitTree(commit_object.data);
64	91	};
65		try repository.checkoutTree(worktree, tree_oid);
	92	try repository.checkoutTree(worktree, tree_oid, "", diagnostics);
66	93	}
67	94
68	95	/// Checks out the tree at `tree_oid` to `worktree`.
...	...	@@ -70,6 +97,8 @@ pub const Repository = struct {
70	97	repository: *Repository,
71	98	dir: std.fs.Dir,
72	99	tree_oid: Oid,
	100	current_path: []const u8,
	101	diagnostics: *Diagnostics,
73	102	) !void {
74	103	try repository.odb.seekOid(tree_oid);
75	104	const tree_object = try repository.odb.readObject();
...	...	@@ -87,7 +116,9 @@ pub const Repository = struct {
87	116	try dir.makeDir(entry.name);
88	117	var subdir = try dir.openDir(entry.name, .{});
89	118	defer subdir.close();
90		try repository.checkoutTree(subdir, entry.oid);
	119	const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name });
	120	defer repository.odb.allocator.free(sub_path);
	121	try repository.checkoutTree(subdir, entry.oid, sub_path, diagnostics);
91	122	},
92	123	.file => {
93	124	var file = try dir.createFile(entry.name, .{});
...	...	@@ -98,7 +129,23 @@ pub const Repository = struct {
98	129	try file.writeAll(file_object.data);
99	130	try file.sync();
100	131	},
101		.symlink => return error.SymlinkNotSupported,
	132	.symlink => {
	133	try repository.odb.seekOid(entry.oid);
	134	var symlink_object = try repository.odb.readObject();
	135	if (symlink_object.type != .blob) return error.InvalidFile;
	136	const link_name = symlink_object.data;
	137	dir.symLink(link_name, entry.name, .{}) catch \|e\| {
	138	const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name });
	139	errdefer diagnostics.allocator.free(file_name);
	140	const link_name_dup = try diagnostics.allocator.dupe(u8, link_name);
	141	errdefer diagnostics.allocator.free(link_name_dup);
	142	try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{
	143	.code = e,
	144	.file_name = file_name,
	145	.link_name = link_name_dup,
	146	} });
	147	};
	148	},
102	149	.gitlink => {
103	150	// Consistent with git archive behavior, create the directory but
104	151	// do nothing else

src/main.zig+143

...	...	@@ -84,6 +84,7 @@ const normal_usage =
84	84	\\Commands:
85	85	\\
86	86	\\ build Build project from build.zig
	87	\\ fetch Copy a package into global cache and print its hash
87	88	\\ init-exe Initialize a `zig build` application in the cwd
88	89	\\ init-lib Initialize a `zig build` library in the cwd
89	90	\\
...	...	@@ -303,6 +304,8 @@ pub fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
303	304	return cmdFmt(gpa, arena, cmd_args);
304	305	} else if (mem.eql(u8, cmd, "objcopy")) {
305	306	return @import("objcopy.zig").cmdObjCopy(gpa, arena, cmd_args);
	307	} else if (mem.eql(u8, cmd, "fetch")) {
	308	return cmdFetch(gpa, arena, cmd_args);
306	309	} else if (mem.eql(u8, cmd, "libc")) {
307	310	return cmdLibC(gpa, cmd_args);
308	311	} else if (mem.eql(u8, cmd, "init-exe")) {
...	...	@@ -6589,3 +6592,143 @@ fn parseRcIncludes(arg: []const u8) Compilation.RcIncludes {
6589	6592	return std.meta.stringToEnum(Compilation.RcIncludes, arg) orelse
6590	6593	fatal("unsupported rc includes type: '{s}'", .{arg});
6591	6594	}
	6595
	6596	pub const usage_fetch =
	6597	\\Usage: zig fetch [options] <url>
	6598	\\Usage: zig fetch [options] <path>
	6599	\\
	6600	\\ Copy a package into the global cache and print its hash.
	6601	\\
	6602	\\Options:
	6603	\\ -h, --help Print this help and exit
	6604	\\ --global-cache-dir [path] Override path to global Zig cache directory
	6605	\\
	6606	;
	6607
	6608	fn cmdFetch(
	6609	gpa: Allocator,
	6610	arena: Allocator,
	6611	args: []const []const u8,
	6612	) !void {
	6613	const color: Color = .auto;
	6614	var opt_url: ?[]const u8 = null;
	6615	var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");
	6616
	6617	{
	6618	var i: usize = 0;
	6619	while (i < args.len) : (i += 1) {
	6620	const arg = args[i];
	6621	if (mem.startsWith(u8, arg, "-")) {
	6622	if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
	6623	const stdout = io.getStdOut().writer();
	6624	try stdout.writeAll(usage_fetch);
	6625	return cleanExit();
	6626	} else if (mem.eql(u8, arg, "--global-cache-dir")) {
	6627	if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
	6628	i += 1;
	6629	override_global_cache_dir = args[i];
	6630	continue;
	6631	} else {
	6632	fatal("unrecognized parameter: '{s}'", .{arg});
	6633	}
	6634	} else if (opt_url != null) {
	6635	fatal("unexpected extra parameter: '{s}'", .{arg});
	6636	} else {
	6637	opt_url = arg;
	6638	}
	6639	}
	6640	}
	6641
	6642	const url = opt_url orelse fatal("missing url or path parameter", .{});
	6643
	6644	var thread_pool: ThreadPool = undefined;
	6645	try thread_pool.init(.{ .allocator = gpa });
	6646	defer thread_pool.deinit();
	6647
	6648	var http_client: std.http.Client = .{ .allocator = gpa };
	6649	defer http_client.deinit();
	6650
	6651	var progress: std.Progress = .{ .dont_print_on_dumb = true };
	6652	const root_prog_node = progress.start("Fetch", 0);
	6653	defer root_prog_node.end();
	6654
	6655	var wip_errors: std.zig.ErrorBundle.Wip = undefined;
	6656	try wip_errors.init(gpa);
	6657	defer wip_errors.deinit();
	6658
	6659	var report: Package.Report = .{
	6660	.ast = null,
	6661	.directory = .{
	6662	.handle = fs.cwd(),
	6663	.path = null,
	6664	},
	6665	.error_bundle = &wip_errors,
	6666	};
	6667
	6668	var global_cache_directory: Compilation.Directory = l: {
	6669	const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena);
	6670	break :l .{
	6671	.handle = try fs.cwd().makeOpenPath(p, .{}),
	6672	.path = p,
	6673	};
	6674	};
	6675	defer global_cache_directory.handle.close();
	6676
	6677	var readable_resource: Package.ReadableResource = rr: {
	6678	if (fs.cwd().openIterableDir(url, .{})) \|dir\| {
	6679	break :rr .{
	6680	.path = try gpa.dupe(u8, url),
	6681	.resource = .{ .dir = dir },
	6682	};
	6683	} else \|dir_err\| {
	6684	const file_err = if (dir_err == error.NotDir) e: {
	6685	if (fs.cwd().openFile(url, .{})) \|f\| {
	6686	break :rr .{
	6687	.path = try gpa.dupe(u8, url),
	6688	.resource = .{ .file = f },
	6689	};
	6690	} else \|err\| break :e err;
	6691	} else dir_err;
	6692
	6693	const uri = std.Uri.parse(url) catch \|uri_err\| {
	6694	fatal("'{s}' could not be recognized as a file path ({s}) or an URL ({s})", .{
	6695	url, @errorName(file_err), @errorName(uri_err),
	6696	});
	6697	};
	6698	const fetch_location = try Package.FetchLocation.initUri(uri, 0, report);
	6699	const cwd: Cache.Directory = .{
	6700	.handle = fs.cwd(),
	6701	.path = null,
	6702	};
	6703	break :rr try fetch_location.fetch(gpa, cwd, &http_client, 0, report);
	6704	}
	6705	};
	6706	defer readable_resource.deinit(gpa);
	6707
	6708	var package_location = readable_resource.unpack(
	6709	gpa,
	6710	&thread_pool,
	6711	global_cache_directory,
	6712	0,
	6713	report,
	6714	root_prog_node,
	6715	) catch \|err\| {
	6716	if (wip_errors.root_list.items.len > 0) {
	6717	var errors = try wip_errors.toOwnedBundle("");
	6718	defer errors.deinit(gpa);
	6719	errors.renderToStdErr(renderOptions(color));
	6720	process.exit(1);
	6721	}
	6722	fatal("unable to unpack '{s}': {s}", .{ url, @errorName(err) });
	6723	};
	6724	defer package_location.deinit(gpa);
	6725
	6726	const hex_digest = Package.Manifest.hexDigest(package_location.hash);
	6727
	6728	progress.done = true;
	6729	progress.refresh();
	6730
	6731	try io.getStdOut().writeAll(hex_digest ++ "\n");
	6732
	6733	return cleanExit();
	6734	}