Commit f7bc55c0136b91805bd046a8cc8ea745d7e7567d | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-10-09 11:47:37-07:00
committer	GitHub <noreply@github.com> 2023-10-09 11:47:37-07:00
log	f7bc55c0136b91805bd046a8cc8ea745d7e7567d
tree	8379c645854d3c513ebb18e7fbabc7ab5ed1a283
parent	75b48ef503204d3ba005647ecce8fda4657a8588
parent	95907cb79578779108f3772cb93648d38354b9ec
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #17392 from ziglang/fetch

rework package manager

47 files changed, 4626 insertions(+), 4072 deletions(-)

CMakeLists.txt
build.zig
doc
- build.zig.zon.md
lib
- build_runner.zig
- std
  - Build.zig
  - Build
    Cache.zig
    Step
    Compile.zig
  - array_hash_map.zig
  - fs
    path.zig
  - multi_array_list.zig
  - process.zig
  - sort.zig
  - tar.zig
  - zig.zig
  - zig
    ErrorBundle.zig
    fmt.zig
src
- Autodoc.zig
- Compilation.zig
- Manifest.zig
- Module.zig
- Package.zig
- Package
  - Fetch.zig
  - Fetch
    git.zig
    git
    testdata
    testrepo.idx
    testrepo.pack
  - Manifest.zig
  - Module.zig
  - hash.zig
- Sema.zig
- codegen
  - llvm.zig
- crash_report.zig
- git.zig
- git
  - testdata
    testrepo.idx
    testrepo.pack
- glibc.zig
- libcxx.zig
- libtsan.zig
- libunwind.zig
- link
- main.zig
- musl.zig
test
- cases
  - compile_errors
    import_of_missing_package.zig
    import_outside_package.zig
    import_outside_package_path.zig
- compile_errors.zig

CMakeLists.txt+1-1

...	...	@@ -528,7 +528,7 @@ set(ZIG_STAGE2_SOURCES
528	528	"${CMAKE_SOURCE_DIR}/src/Liveness.zig"
529	529	"${CMAKE_SOURCE_DIR}/src/Module.zig"
530	530	"${CMAKE_SOURCE_DIR}/src/Package.zig"
531		"${CMAKE_SOURCE_DIR}/src/Package/hash.zig"
	531	"${CMAKE_SOURCE_DIR}/src/Package/Fetch.zig"
532	532	"${CMAKE_SOURCE_DIR}/src/RangeSet.zig"
533	533	"${CMAKE_SOURCE_DIR}/src/Sema.zig"
534	534	"${CMAKE_SOURCE_DIR}/src/TypedValue.zig"

build.zig+1-1

...	...	@@ -88,7 +88,7 @@ pub fn build(b: *std.Build) !void {
88	88	.name = "check-case",
89	89	.root_source_file = .{ .path = "test/src/Cases.zig" },
90	90	.optimize = optimize,
91		.main_pkg_path = .{ .path = "." },
	91	.main_mod_path = .{ .path = "." },
92	92	});
93	93	check_case_exe.stack_size = stack_size;
94	94	check_case_exe.single_threaded = single_threaded;

doc/build.zig.zon.md created+65

...	...	@@ -0,0 +1,65 @@
	1	# build.zig.zon Documentation
	2
	3	This is the manifest file for build.zig scripts. It is named build.zig.zon in
	4	order to make it clear that it is metadata specifically pertaining to
	5	build.zig.
	6
	7	- build root - the directory that contains `build.zig`
	8
	9	## Top-Level Fields
	10
	11	### `name`
	12
	13	String. Required.
	14
	15	### `version`
	16
	17	String. Required.
	18
	19	[semver](https://semver.org/)
	20
	21	### `dependencies`
	22
	23	Struct.
	24
	25	Each dependency must either provide a `url` and `hash`, or a `path`.
	26
	27	#### `url`
	28
	29	String.
	30
	31	When updating this field to a new URL, be sure to delete the corresponding
	32	`hash`, otherwise you are communicating that you expect to find the old hash at
	33	the new URL.
	34
	35	#### `hash`
	36
	37	String.
	38
	39	[multihash](https://multiformats.io/multihash/)
	40
	41	This is computed from the file contents of the directory of files that is
	42	obtained after fetching `url` and applying the inclusion rules given by
	43	`paths`.
	44
	45	This field is the source of truth; packages do not come from an `url`; they
	46	come from a `hash`. `url` is just one of many possible mirrors for how to
	47	obtain a package matching this `hash`.
	48
	49	#### `path`
	50
	51	String.
	52
	53	When this is provided, the package is found in a directory relative to the
	54	build root. In this case the package's hash is irrelevant and therefore not
	55	computed.
	56
	57	### `paths`
	58
	59	List. Required.
	60
	61	Specifies the set of files and directories that are included in this package.
	62	Paths are relative to the build root. Use the empty string (`""`) to refer to
	63	the build root itself.
	64
	65	Only files included in the package are used to compute a package's `hash`.

lib/build_runner.zig+1

...	...	@@ -997,6 +997,7 @@ fn usage(builder: *std.Build, already_ran_build: bool, out_stream: anytype) !voi
997	997	\\ -j<N> Limit concurrent jobs (default is to use all CPU cores)
998	998	\\ --maxrss <bytes> Limit memory usage (default is to use available memory)
999	999	\\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss
	1000	\\ --fetch Exit after fetching dependency tree
1000	1001	\\
1001	1002	\\Project-Specific Options:
1002	1003	\\

lib/std/Build.zig+20-5

...	...	@@ -634,6 +634,9 @@ pub const ExecutableOptions = struct {
634	634	use_llvm: ?bool = null,
635	635	use_lld: ?bool = null,
636	636	zig_lib_dir: ?LazyPath = null,
	637	main_mod_path: ?LazyPath = null,
	638
	639	/// Deprecated; use `main_mod_path`.
637	640	main_pkg_path: ?LazyPath = null,
638	641	};
639	642
...	...	@@ -652,7 +655,7 @@ pub fn addExecutable(b: Build, options: ExecutableOptions) Step.Compile {
652	655	.use_llvm = options.use_llvm,
653	656	.use_lld = options.use_lld,
654	657	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
655		.main_pkg_path = options.main_pkg_path,
	658	.main_mod_path = options.main_mod_path orelse options.main_pkg_path,
656	659	});
657	660	}
658	661
...	...	@@ -667,6 +670,9 @@ pub const ObjectOptions = struct {
667	670	use_llvm: ?bool = null,
668	671	use_lld: ?bool = null,
669	672	zig_lib_dir: ?LazyPath = null,
	673	main_mod_path: ?LazyPath = null,
	674
	675	/// Deprecated; use `main_mod_path`.
670	676	main_pkg_path: ?LazyPath = null,
671	677	};
672	678
...	...	@@ -683,7 +689,7 @@ pub fn addObject(b: Build, options: ObjectOptions) Step.Compile {
683	689	.use_llvm = options.use_llvm,
684	690	.use_lld = options.use_lld,
685	691	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
686		.main_pkg_path = options.main_pkg_path,
	692	.main_mod_path = options.main_mod_path orelse options.main_pkg_path,
687	693	});
688	694	}
689	695
...	...	@@ -699,6 +705,9 @@ pub const SharedLibraryOptions = struct {
699	705	use_llvm: ?bool = null,
700	706	use_lld: ?bool = null,
701	707	zig_lib_dir: ?LazyPath = null,
	708	main_mod_path: ?LazyPath = null,
	709
	710	/// Deprecated; use `main_mod_path`.
702	711	main_pkg_path: ?LazyPath = null,
703	712	};
704	713
...	...	@@ -717,7 +726,7 @@ pub fn addSharedLibrary(b: Build, options: SharedLibraryOptions) Step.Compile
717	726	.use_llvm = options.use_llvm,
718	727	.use_lld = options.use_lld,
719	728	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
720		.main_pkg_path = options.main_pkg_path,
	729	.main_mod_path = options.main_mod_path orelse options.main_pkg_path,
721	730	});
722	731	}
723	732
...	...	@@ -733,6 +742,9 @@ pub const StaticLibraryOptions = struct {
733	742	use_llvm: ?bool = null,
734	743	use_lld: ?bool = null,
735	744	zig_lib_dir: ?LazyPath = null,
	745	main_mod_path: ?LazyPath = null,
	746
	747	/// Deprecated; use `main_mod_path`.
736	748	main_pkg_path: ?LazyPath = null,
737	749	};
738	750
...	...	@@ -751,7 +763,7 @@ pub fn addStaticLibrary(b: Build, options: StaticLibraryOptions) Step.Compile
751	763	.use_llvm = options.use_llvm,
752	764	.use_lld = options.use_lld,
753	765	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
754		.main_pkg_path = options.main_pkg_path,
	766	.main_mod_path = options.main_mod_path orelse options.main_pkg_path,
755	767	});
756	768	}
757	769
...	...	@@ -769,6 +781,9 @@ pub const TestOptions = struct {
769	781	use_llvm: ?bool = null,
770	782	use_lld: ?bool = null,
771	783	zig_lib_dir: ?LazyPath = null,
	784	main_mod_path: ?LazyPath = null,
	785
	786	/// Deprecated; use `main_mod_path`.
772	787	main_pkg_path: ?LazyPath = null,
773	788	};
774	789
...	...	@@ -787,7 +802,7 @@ pub fn addTest(b: Build, options: TestOptions) Step.Compile {
787	802	.use_llvm = options.use_llvm,
788	803	.use_lld = options.use_lld,
789	804	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
790		.main_pkg_path = options.main_pkg_path,
	805	.main_mod_path = options.main_mod_path orelse options.main_pkg_path,
791	806	});
792	807	}
793	808

lib/std/Build/Cache.zig+19-1

...	...	@@ -9,6 +9,20 @@ pub const Directory = struct {
9	9	path: ?[]const u8,
10	10	handle: fs.Dir,
11	11
	12	pub fn clone(d: Directory, arena: Allocator) Allocator.Error!Directory {
	13	return .{
	14	.path = if (d.path) \|p\| try arena.dupe(u8, p) else null,
	15	.handle = d.handle,
	16	};
	17	}
	18
	19	pub fn cwd() Directory {
	20	return .{
	21	.path = null,
	22	.handle = fs.cwd(),
	23	};
	24	}
	25
12	26	pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
13	27	if (self.path) \|p\| {
14	28	// TODO clean way to do this with only 1 allocation
...	...	@@ -47,12 +61,16 @@ pub const Directory = struct {
47	61	writer: anytype,
48	62	) !void {
49	63	_ = options;
50		if (fmt_string.len != 0) fmt.invalidFmtError(fmt, self);
	64	if (fmt_string.len != 0) fmt.invalidFmtError(fmt_string, self);
51	65	if (self.path) \|p\| {
52	66	try writer.writeAll(p);
53	67	try writer.writeAll(fs.path.sep_str);
54	68	}
55	69	}
	70
	71	pub fn eql(self: Directory, other: Directory) bool {
	72	return self.handle.fd == other.handle.fd;
	73	}
56	74	};
57	75
58	76	gpa: Allocator,

lib/std/Build/Step/Compile.zig+9-6

...	...	@@ -68,7 +68,7 @@ c_std: std.Build.CStd,
68	68	/// Set via options; intended to be read-only after that.
69	69	zig_lib_dir: ?LazyPath,
70	70	/// Set via options; intended to be read-only after that.
71		main_pkg_path: ?LazyPath,
	71	main_mod_path: ?LazyPath,
72	72	exec_cmd_args: ?[]const ?[]const u8,
73	73	filter: ?[]const u8,
74	74	test_evented_io: bool = false,
...	...	@@ -316,6 +316,9 @@ pub const Options = struct {
316	316	use_llvm: ?bool = null,
317	317	use_lld: ?bool = null,
318	318	zig_lib_dir: ?LazyPath = null,
	319	main_mod_path: ?LazyPath = null,
	320
	321	/// deprecated; use `main_mod_path`.
319	322	main_pkg_path: ?LazyPath = null,
320	323	};
321	324
...	...	@@ -480,7 +483,7 @@ pub fn create(owner: std.Build, options: Options) Compile {
480	483	.installed_headers = ArrayList(*Step).init(owner.allocator),
481	484	.c_std = std.Build.CStd.C99,
482	485	.zig_lib_dir = null,
483		.main_pkg_path = null,
	486	.main_mod_path = null,
484	487	.exec_cmd_args = null,
485	488	.filter = options.filter,
486	489	.test_runner = options.test_runner,
...	...	@@ -515,8 +518,8 @@ pub fn create(owner: std.Build, options: Options) Compile {
515	518	lp.addStepDependencies(&self.step);
516	519	}
517	520
518		if (options.main_pkg_path) \|lp\| {
519		self.main_pkg_path = lp.dupe(self.step.owner);
	521	if (options.main_mod_path orelse options.main_pkg_path) \|lp\| {
	522	self.main_mod_path = lp.dupe(self.step.owner);
520	523	lp.addStepDependencies(&self.step);
521	524	}
522	525
...	...	@@ -1998,8 +2001,8 @@ fn make(step: Step, prog_node: std.Progress.Node) !void {
1998	2001	try zig_args.append(dir.getPath(b));
1999	2002	}
2000	2003
2001		if (self.main_pkg_path) \|dir\| {
2002		try zig_args.append("--main-pkg-path");
	2004	if (self.main_mod_path) \|dir\| {
	2005	try zig_args.append("--main-mod-path");
2003	2006	try zig_args.append(dir.getPath(b));
2004	2007	}
2005	2008

lib/std/array_hash_map.zig+30-3

...	...	@@ -1229,14 +1229,41 @@ pub fn ArrayHashMapUnmanaged(
1229	1229	/// Sorts the entries and then rebuilds the index.
1230	1230	/// `sort_ctx` must have this method:
1231	1231	/// `fn lessThan(ctx: @TypeOf(ctx), a_index: usize, b_index: usize) bool`
	1232	/// Uses a stable sorting algorithm.
1232	1233	pub inline fn sort(self: *Self, sort_ctx: anytype) void {
1233	1234	if (@sizeOf(ByIndexContext) != 0)
1234	1235	@compileError("Cannot infer context " ++ @typeName(Context) ++ ", call sortContext instead.");
1235		return self.sortContext(sort_ctx, undefined);
	1236	return sortContextInternal(self, .stable, sort_ctx, undefined);
1236	1237	}
1237	1238
1238		pub fn sortContext(self: *Self, sort_ctx: anytype, ctx: Context) void {
1239		self.entries.sort(sort_ctx);
	1239	/// Sorts the entries and then rebuilds the index.
	1240	/// `sort_ctx` must have this method:
	1241	/// `fn lessThan(ctx: @TypeOf(ctx), a_index: usize, b_index: usize) bool`
	1242	/// Uses an unstable sorting algorithm.
	1243	pub inline fn sortUnstable(self: *Self, sort_ctx: anytype) void {
	1244	if (@sizeOf(ByIndexContext) != 0)
	1245	@compileError("Cannot infer context " ++ @typeName(Context) ++ ", call sortUnstableContext instead.");
	1246	return self.sortContextInternal(.unstable, sort_ctx, undefined);
	1247	}
	1248
	1249	pub inline fn sortContext(self: *Self, sort_ctx: anytype, ctx: Context) void {
	1250	return sortContextInternal(self, .stable, sort_ctx, ctx);
	1251	}
	1252
	1253	pub inline fn sortUnstableContext(self: *Self, sort_ctx: anytype, ctx: Context) void {
	1254	return sortContextInternal(self, .unstable, sort_ctx, ctx);
	1255	}
	1256
	1257	fn sortContextInternal(
	1258	self: *Self,
	1259	comptime mode: std.sort.Mode,
	1260	sort_ctx: anytype,
	1261	ctx: Context,
	1262	) void {
	1263	switch (mode) {
	1264	.stable => self.entries.sort(sort_ctx),
	1265	.unstable => self.entries.sortUnstable(sort_ctx),
	1266	}
1240	1267	const header = self.index_header orelse return;
1241	1268	header.reset();
1242	1269	self.insertAllEntriesIntoNewHeader(if (store_hash) {} else ctx, header);

lib/std/fs/path.zig+5-3

...	...	@@ -728,15 +728,17 @@ pub fn resolvePosix(allocator: Allocator, paths: []const []const u8) Allocator.E
728	728	}
729	729	}
730	730
731		test "resolve" {
	731	test resolve {
732	732	try testResolveWindows(&[_][]const u8{ "a\\b\\c\\", "..\\..\\.." }, ".");
733	733	try testResolveWindows(&[_][]const u8{"."}, ".");
	734	try testResolveWindows(&[_][]const u8{""}, ".");
734	735
735	736	try testResolvePosix(&[_][]const u8{ "a/b/c/", "../../.." }, ".");
736	737	try testResolvePosix(&[_][]const u8{"."}, ".");
	738	try testResolvePosix(&[_][]const u8{""}, ".");
737	739	}
738	740
739		test "resolveWindows" {
	741	test resolveWindows {
740	742	try testResolveWindows(
741	743	&[_][]const u8{ "Z:\\", "/usr/local", "lib\\zig\\std\\array_list.zig" },
742	744	"Z:\\usr\\local\\lib\\zig\\std\\array_list.zig",
...	...	@@ -764,7 +766,7 @@ test "resolveWindows" {
764	766	try testResolveWindows(&[_][]const u8{"a/b"}, "a\\b");
765	767	}
766	768
767		test "resolvePosix" {
	769	test resolvePosix {
768	770	try testResolvePosix(&.{ "/a/b", "c" }, "/a/b/c");
769	771	try testResolvePosix(&.{ "/a/b", "c", "//d", "e///" }, "/d/e");
770	772	try testResolvePosix(&.{ "/a/b/c", "..", "../" }, "/a");

lib/std/multi_array_list.zig+1-1

...	...	@@ -467,7 +467,7 @@ pub fn MultiArrayList(comptime T: type) type {
467	467
468	468	/// `ctx` has the following method:
469	469	/// `fn lessThan(ctx: @TypeOf(ctx), a_index: usize, b_index: usize) bool`
470		fn sortInternal(self: Self, a: usize, b: usize, ctx: anytype, comptime mode: enum { stable, unstable }) void {
	470	fn sortInternal(self: Self, a: usize, b: usize, ctx: anytype, comptime mode: std.sort.Mode) void {
471	471	const sort_context: struct {
472	472	sub_ctx: @TypeOf(ctx),
473	473	slice: Slice,

lib/std/process.zig+1-1

...	...	@@ -46,7 +46,7 @@ pub fn getCwdAlloc(allocator: Allocator) ![]u8 {
46	46	}
47	47	}
48	48
49		test "getCwdAlloc" {
	49	test getCwdAlloc {
50	50	if (builtin.os.tag == .wasi) return error.SkipZigTest;
51	51
52	52	const cwd = try getCwdAlloc(testing.allocator);

lib/std/sort.zig+2

...	...	@@ -4,6 +4,8 @@ const testing = std.testing;
4	4	const mem = std.mem;
5	5	const math = std.math;
6	6
	7	pub const Mode = enum { stable, unstable };
	8
7	9	pub const block = @import("sort/block.zig").block;
8	10	pub const pdq = @import("sort/pdq.zig").pdq;
9	11	pub const pdqContext = @import("sort/pdq.zig").pdqContext;

lib/std/tar.zig+47-9

...	...	@@ -3,6 +3,8 @@ pub const Options = struct {
3	3	strip_components: u32 = 0,
4	4	/// How to handle the "mode" property of files from within the tar file.
5	5	mode_mode: ModeMode = .executable_bit_only,
	6	/// Prevents creation of empty directories.
	7	exclude_empty_directories: bool = false,
6	8	/// Provide this to receive detailed error messages.
7	9	/// When this is provided, some errors which would otherwise be returned immediately
8	10	/// will instead be added to this structure. The API user must check the errors
...	...	@@ -29,6 +31,10 @@ pub const Options = struct {
29	31	file_name: []const u8,
30	32	link_name: []const u8,
31	33	},
	34	unable_to_create_file: struct {
	35	code: anyerror,
	36	file_name: []const u8,
	37	},
32	38	unsupported_file_type: struct {
33	39	file_name: []const u8,
34	40	file_type: Header.FileType,
...	...	@@ -42,6 +48,9 @@ pub const Options = struct {
42	48	d.allocator.free(info.file_name);
43	49	d.allocator.free(info.link_name);
44	50	},
	51	.unable_to_create_file => \|info\| {
	52	d.allocator.free(info.file_name);
	53	},
45	54	.unsupported_file_type => \|info\| {
46	55	d.allocator.free(info.file_name);
47	56	},
...	...	@@ -201,7 +210,7 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
201	210	switch (header.fileType()) {
202	211	.directory => {
203	212	const file_name = try stripComponents(unstripped_file_name, options.strip_components);
204		if (file_name.len != 0) {
	213	if (file_name.len != 0 and !options.exclude_empty_directories) {
205	214	try dir.makePath(file_name);
206	215	}
207	216	},
...	...	@@ -209,18 +218,34 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
209	218	if (file_size == 0 and unstripped_file_name.len == 0) return;
210	219	const file_name = try stripComponents(unstripped_file_name, options.strip_components);
211	220
212		if (std.fs.path.dirname(file_name)) \|dir_name\| {
213		try dir.makePath(dir_name);
214		}
215		var file = try dir.createFile(file_name, .{});
216		defer file.close();
	221	var file = dir.createFile(file_name, .{}) catch \|err\| switch (err) {
	222	error.FileNotFound => again: {
	223	const code = code: {
	224	if (std.fs.path.dirname(file_name)) \|dir_name\| {
	225	dir.makePath(dir_name) catch \|code\| break :code code;
	226	break :again dir.createFile(file_name, .{}) catch \|code\| {
	227	break :code code;
	228	};
	229	}
	230	break :code err;
	231	};
	232	const d = options.diagnostics orelse return error.UnableToCreateFile;
	233	try d.errors.append(d.allocator, .{ .unable_to_create_file = .{
	234	.code = code,
	235	.file_name = try d.allocator.dupe(u8, file_name),
	236	} });
	237	break :again null;
	238	},
	239	else => \|e\| return e,
	240	};
	241	defer if (file) \|f\| f.close();
217	242
218	243	var file_off: usize = 0;
219	244	while (true) {
220	245	const temp = try buffer.readChunk(reader, @intCast(rounded_file_size + 512 - file_off));
221	246	if (temp.len == 0) return error.UnexpectedEndOfStream;
222	247	const slice = temp[0..@intCast(@min(file_size - file_off, temp.len))];
223		try file.writeAll(slice);
	248	if (file) \|f\| try f.writeAll(slice);
224	249
225	250	file_off += slice.len;
226	251	buffer.advance(slice.len);
...	...	@@ -273,13 +298,26 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
273	298	},
274	299	.hard_link => return error.TarUnsupportedFileType,
275	300	.symbolic_link => {
	301	// The file system path of the symbolic link.
276	302	const file_name = try stripComponents(unstripped_file_name, options.strip_components);
	303	// The data inside the symbolic link.
277	304	const link_name = header.linkName();
278	305
279		dir.symLink(link_name, file_name, .{}) catch \|err\| {
	306	dir.symLink(link_name, file_name, .{}) catch \|err\| again: {
	307	const code = code: {
	308	if (err == error.FileNotFound) {
	309	if (std.fs.path.dirname(file_name)) \|dir_name\| {
	310	dir.makePath(dir_name) catch \|code\| break :code code;
	311	break :again dir.symLink(link_name, file_name, .{}) catch \|code\| {
	312	break :code code;
	313	};
	314	}
	315	}
	316	break :code err;
	317	};
280	318	const d = options.diagnostics orelse return error.UnableToCreateSymLink;
281	319	try d.errors.append(d.allocator, .{ .unable_to_create_sym_link = .{
282		.code = err,
	320	.code = code,
283	321	.file_name = try d.allocator.dupe(u8, file_name),
284	322	.link_name = try d.allocator.dupe(u8, link_name),
285	323	} });

lib/std/zig.zig+1-1

...	...	@@ -1,6 +1,6 @@
1	1	const std = @import("std.zig");
2	2	const tokenizer = @import("zig/tokenizer.zig");
3		const fmt = @import("zig/fmt.zig");
	3	pub const fmt = @import("zig/fmt.zig");
4	4	const assert = std.debug.assert;
5	5
6	6	pub const ErrorBundle = @import("zig/ErrorBundle.zig");

lib/std/zig/ErrorBundle.zig+7-7

...	...	@@ -383,7 +383,7 @@ pub const Wip = struct {
383	383	};
384	384	}
385	385
386		pub fn addString(wip: *Wip, s: []const u8) !u32 {
	386	pub fn addString(wip: *Wip, s: []const u8) Allocator.Error!u32 {
387	387	const gpa = wip.gpa;
388	388	const index: u32 = @intCast(wip.string_bytes.items.len);
389	389	try wip.string_bytes.ensureUnusedCapacity(gpa, s.len + 1);
...	...	@@ -392,7 +392,7 @@ pub const Wip = struct {
392	392	return index;
393	393	}
394	394
395		pub fn printString(wip: *Wip, comptime fmt: []const u8, args: anytype) !u32 {
	395	pub fn printString(wip: *Wip, comptime fmt: []const u8, args: anytype) Allocator.Error!u32 {
396	396	const gpa = wip.gpa;
397	397	const index: u32 = @intCast(wip.string_bytes.items.len);
398	398	try wip.string_bytes.writer(gpa).print(fmt, args);
...	...	@@ -400,12 +400,12 @@ pub const Wip = struct {
400	400	return index;
401	401	}
402	402
403		pub fn addRootErrorMessage(wip: *Wip, em: ErrorMessage) !void {
	403	pub fn addRootErrorMessage(wip: *Wip, em: ErrorMessage) Allocator.Error!void {
404	404	try wip.root_list.ensureUnusedCapacity(wip.gpa, 1);
405	405	wip.root_list.appendAssumeCapacity(try addErrorMessage(wip, em));
406	406	}
407	407
408		pub fn addErrorMessage(wip: *Wip, em: ErrorMessage) !MessageIndex {
	408	pub fn addErrorMessage(wip: *Wip, em: ErrorMessage) Allocator.Error!MessageIndex {
409	409	return @enumFromInt(try addExtra(wip, em));
410	410	}
411	411
...	...	@@ -413,15 +413,15 @@ pub const Wip = struct {
413	413	return @enumFromInt(addExtraAssumeCapacity(wip, em));
414	414	}
415	415
416		pub fn addSourceLocation(wip: *Wip, sl: SourceLocation) !SourceLocationIndex {
	416	pub fn addSourceLocation(wip: *Wip, sl: SourceLocation) Allocator.Error!SourceLocationIndex {
417	417	return @enumFromInt(try addExtra(wip, sl));
418	418	}
419	419
420		pub fn addReferenceTrace(wip: *Wip, rt: ReferenceTrace) !void {
	420	pub fn addReferenceTrace(wip: *Wip, rt: ReferenceTrace) Allocator.Error!void {
421	421	_ = try addExtra(wip, rt);
422	422	}
423	423
424		pub fn addBundleAsNotes(wip: *Wip, other: ErrorBundle) !void {
	424	pub fn addBundleAsNotes(wip: *Wip, other: ErrorBundle) Allocator.Error!void {
425	425	const gpa = wip.gpa;
426	426
427	427	try wip.string_bytes.ensureUnusedCapacity(gpa, other.string_bytes.len);

lib/std/zig/fmt.zig+3-3

...	...	@@ -13,7 +13,7 @@ fn formatId(
13	13	return writer.writeAll(bytes);
14	14	}
15	15	try writer.writeAll("@\"");
16		try formatEscapes(bytes, "", options, writer);
	16	try stringEscape(bytes, "", options, writer);
17	17	try writer.writeByte('"');
18	18	}
19	19
...	...	@@ -47,7 +47,7 @@ test "isValidId" {
47	47	/// Print the string as escaped contents of a double quoted or single-quoted string.
48	48	/// Format `{}` treats contents as a double-quoted string.
49	49	/// Format `{'}` treats contents as a single-quoted string.
50		fn formatEscapes(
	50	pub fn stringEscape(
51	51	bytes: []const u8,
52	52	comptime fmt: []const u8,
53	53	options: std.fmt.FormatOptions,
...	...	@@ -90,7 +90,7 @@ fn formatEscapes(
90	90	/// The format specifier must be one of:
91	91	/// * `{}` treats contents as a double-quoted string.
92	92	/// * `{'}` treats contents as a single-quoted string.
93		pub fn fmtEscapes(bytes: []const u8) std.fmt.Formatter(formatEscapes) {
	93	pub fn fmtEscapes(bytes: []const u8) std.fmt.Formatter(stringEscape) {
94	94	return .{ .data = bytes };
95	95	}
96	96

src/Autodoc.zig+17-18

...	...	@@ -6,7 +6,7 @@ const Autodoc = @This();
6	6	const Compilation = @import("Compilation.zig");
7	7	const CompilationModule = @import("Module.zig");
8	8	const File = CompilationModule.File;
9		const Module = @import("Package.zig");
	9	const Module = @import("Package.zig").Module;
10	10	const Tokenizer = std.zig.Tokenizer;
11	11	const InternPool = @import("InternPool.zig");
12	12	const Zir = @import("Zir.zig");
...	...	@@ -98,9 +98,8 @@ pub fn generate(cm: *CompilationModule, output_dir: std.fs.Dir) !void {
98	98	}
99	99
100	100	fn generateZirData(self: *Autodoc, output_dir: std.fs.Dir) !void {
101		const root_src_dir = self.comp_module.main_pkg.root_src_directory;
102		const root_src_path = self.comp_module.main_pkg.root_src_path;
103		const joined_src_path = try root_src_dir.join(self.arena, &.{root_src_path});
	101	const root_src_path = self.comp_module.main_mod.root_src_path;
	102	const joined_src_path = try self.comp_module.main_mod.root.joinString(self.arena, root_src_path);
104	103	defer self.arena.free(joined_src_path);
105	104
106	105	const abs_root_src_path = try std.fs.path.resolve(self.arena, &.{ ".", joined_src_path });
...	...	@@ -295,20 +294,20 @@ fn generateZirData(self: *Autodoc, output_dir: std.fs.Dir) !void {
295	294	}
296	295
297	296	const rootName = blk: {
298		const rootName = std.fs.path.basename(self.comp_module.main_pkg.root_src_path);
	297	const rootName = std.fs.path.basename(self.comp_module.main_mod.root_src_path);
299	298	break :blk rootName[0 .. rootName.len - 4];
300	299	};
301	300
302	301	const main_type_index = self.types.items.len;
303	302	{
304		try self.modules.put(self.arena, self.comp_module.main_pkg, .{
	303	try self.modules.put(self.arena, self.comp_module.main_mod, .{
305	304	.name = rootName,
306	305	.main = main_type_index,
307	306	.table = .{},
308	307	});
309	308	try self.modules.entries.items(.value)[0].table.put(
310	309	self.arena,
311		self.comp_module.main_pkg,
	310	self.comp_module.main_mod,
312	311	.{
313	312	.name = rootName,
314	313	.value = 0,
...	...	@@ -412,7 +411,7 @@ fn generateZirData(self: *Autodoc, output_dir: std.fs.Dir) !void {
412	411
413	412	while (files_iterator.next()) \|entry\| {
414	413	const sub_file_path = entry.key_ptr.*.sub_file_path;
415		const file_module = entry.key_ptr.*.pkg;
	414	const file_module = entry.key_ptr.*.mod;
416	415	const module_name = (self.modules.get(file_module) orelse continue).name;
417	416
418	417	const file_path = std.fs.path.dirname(sub_file_path) orelse "";
...	...	@@ -986,12 +985,12 @@ fn walkInstruction(
986	985
987	986	// importFile cannot error out since all files
988	987	// are already loaded at this point
989		if (file.pkg.table.get(path)) \|other_module\| {
	988	if (file.mod.deps.get(path)) \|other_module\| {
990	989	const result = try self.modules.getOrPut(self.arena, other_module);
991	990
992	991	// Immediately add this module to the import table of our
993	992	// current module, regardless of wether it's new or not.
994		if (self.modules.getPtr(file.pkg)) \|current_module\| {
	993	if (self.modules.getPtr(file.mod)) \|current_module\| {
995	994	// TODO: apparently, in the stdlib a file gets analyzed before
996	995	// its module gets added. I guess we're importing a file
997	996	// that belongs to another module through its file path?
...	...	@@ -1025,12 +1024,12 @@ fn walkInstruction(
1025	1024	// TODO: Add this module as a dependency to the current module
1026	1025	// TODO: this seems something that could be done in bulk
1027	1026	// at the beginning or the end, or something.
1028		const root_src_dir = other_module.root_src_directory;
1029		const root_src_path = other_module.root_src_path;
1030		const joined_src_path = try root_src_dir.join(self.arena, &.{root_src_path});
1031		defer self.arena.free(joined_src_path);
1032
1033		const abs_root_src_path = try std.fs.path.resolve(self.arena, &.{ ".", joined_src_path });
	1027	const abs_root_src_path = try std.fs.path.resolve(self.arena, &.{
	1028	".",
	1029	other_module.root.root_dir.path orelse ".",
	1030	other_module.root.sub_path,
	1031	other_module.root_src_path,
	1032	});
1034	1033	defer self.arena.free(abs_root_src_path);
1035	1034
1036	1035	const new_file = self.comp_module.import_table.get(abs_root_src_path).?;
...	...	@@ -5683,7 +5682,7 @@ fn writeFileTableToJson(
5683	5682	while (it.next()) \|entry\| {
5684	5683	try jsw.beginArray();
5685	5684	try jsw.write(entry.key_ptr.*.sub_file_path);
5686		try jsw.write(mods.getIndex(entry.key_ptr.*.pkg) orelse 0);
	5685	try jsw.write(mods.getIndex(entry.key_ptr.*.mod) orelse 0);
5687	5686	try jsw.endArray();
5688	5687	}
5689	5688	try jsw.endArray();
...	...	@@ -5840,7 +5839,7 @@ fn addGuide(self: Autodoc, file: File, guide_path: []const u8, section: *Secti
5840	5839	file.sub_file_path, "..", guide_path,
5841	5840	});
5842	5841
5843		var guide_file = try file.pkg.root_src_directory.handle.openFile(resolved_path, .{});
	5842	var guide_file = try file.mod.root.openFile(resolved_path, .{});
5844	5843	defer guide_file.close();
5845	5844
5846	5845	const guide = guide_file.reader().readAllAlloc(self.arena, 1 * 1024 * 1024) catch \|err\| switch (err) {

src/Compilation.zig+135-154

...	...	@@ -41,8 +41,9 @@ const resinator = @import("resinator.zig");
41	41
42	42	/// General-purpose allocator. Used for both temporary and long-term storage.
43	43	gpa: Allocator,
44		/// Arena-allocated memory used during initialization. Should be untouched until deinit.
45		arena_state: std.heap.ArenaAllocator.State,
	44	/// Arena-allocated memory, mostly used during initialization. However, it can be used
	45	/// for other things requiring the same lifetime as the `Compilation`.
	46	arena: std.heap.ArenaAllocator,
46	47	bin_file: *link.File,
47	48	c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},
48	49	win32_resource_table: if (build_options.only_core_functionality) void else std.AutoArrayHashMapUnmanaged(*Win32Resource, void) =
...	...	@@ -124,7 +125,7 @@ cache_parent: *Cache,
124	125	/// Path to own executable for invoking `zig clang`.
125	126	self_exe_path: ?[]const u8,
126	127	/// null means -fno-emit-bin.
127		/// This is mutable memory allocated into the Compilation-lifetime arena (`arena_state`)
	128	/// This is mutable memory allocated into the Compilation-lifetime arena (`arena`)
128	129	/// of exactly the correct size for "o/[digest]/[basename]".
129	130	/// The basename is of the outputted binary file in case we don't know the directory yet.
130	131	whole_bin_sub_path: ?[]u8,
...	...	@@ -273,8 +274,8 @@ const Job = union(enum) {
273	274	/// The source file containing the Decl has been updated, and so the
274	275	/// Decl may need its line number information updated in the debug info.
275	276	update_line_number: Module.Decl.Index,
276		/// The main source file for the package needs to be analyzed.
277		analyze_pkg: *Package,
	277	/// The main source file for the module needs to be analyzed.
	278	analyze_mod: *Package.Module,
278	279
279	280	/// one of the glibc static objects
280	281	glibc_crt_file: glibc.CRTFile,
...	...	@@ -414,7 +415,7 @@ pub const MiscTask = enum {
414	415	compiler_rt,
415	416	libssp,
416	417	zig_libc,
417		analyze_pkg,
	418	analyze_mod,
418	419
419	420	@"musl crti.o",
420	421	@"musl crtn.o",
...	...	@@ -544,7 +545,7 @@ pub const InitOptions = struct {
544	545	global_cache_directory: Directory,
545	546	target: Target,
546	547	root_name: []const u8,
547		main_pkg: ?*Package,
	548	main_mod: ?*Package.Module,
548	549	output_mode: std.builtin.OutputMode,
549	550	thread_pool: *ThreadPool,
550	551	dynamic_linker: ?[]const u8 = null,
...	...	@@ -736,53 +737,55 @@ pub const InitOptions = struct {
736	737	pdb_out_path: ?[]const u8 = null,
737	738	};
738	739
739		fn addPackageTableToCacheHash(
	740	fn addModuleTableToCacheHash(
740	741	hash: *Cache.HashHelper,
741	742	arena: *std.heap.ArenaAllocator,
742		pkg_table: Package.Table,
743		seen_table: std.AutoHashMap(Package, void),
	743	mod_table: Package.Module.Deps,
	744	seen_table: std.AutoHashMap(Package.Module, void),
744	745	hash_type: union(enum) { path_bytes, files: *Cache.Manifest },
745	746	) (error{OutOfMemory} \|\| std.os.GetCwdError)!void {
746	747	const allocator = arena.allocator();
747	748
748		const packages = try allocator.alloc(Package.Table.KV, pkg_table.count());
	749	const modules = try allocator.alloc(Package.Module.Deps.KV, mod_table.count());
749	750	{
750	751	// Copy over the hashmap entries to our slice
751		var table_it = pkg_table.iterator();
	752	var table_it = mod_table.iterator();
752	753	var idx: usize = 0;
753	754	while (table_it.next()) \|entry\| : (idx += 1) {
754		packages[idx] = .{
	755	modules[idx] = .{
755	756	.key = entry.key_ptr.*,
756	757	.value = entry.value_ptr.*,
757	758	};
758	759	}
759	760	}
760	761	// Sort the slice by package name
761		mem.sort(Package.Table.KV, packages, {}, struct {
762		fn lessThan(_: void, lhs: Package.Table.KV, rhs: Package.Table.KV) bool {
	762	mem.sortUnstable(Package.Module.Deps.KV, modules, {}, struct {
	763	fn lessThan(_: void, lhs: Package.Module.Deps.KV, rhs: Package.Module.Deps.KV) bool {
763	764	return std.mem.lessThan(u8, lhs.key, rhs.key);
764	765	}
765	766	}.lessThan);
766	767
767		for (packages) \|pkg\| {
768		if ((try seen_table.getOrPut(pkg.value)).found_existing) continue;
	768	for (modules) \|mod\| {
	769	if ((try seen_table.getOrPut(mod.value)).found_existing) continue;
769	770
770	771	// Finally insert the package name and path to the cache hash.
771		hash.addBytes(pkg.key);
	772	hash.addBytes(mod.key);
772	773	switch (hash_type) {
773	774	.path_bytes => {
774		hash.addBytes(pkg.value.root_src_path);
775		hash.addOptionalBytes(pkg.value.root_src_directory.path);
	775	hash.addBytes(mod.value.root_src_path);
	776	hash.addOptionalBytes(mod.value.root.root_dir.path);
	777	hash.addBytes(mod.value.root.sub_path);
776	778	},
777	779	.files => \|man\| {
778		const pkg_zig_file = try pkg.value.root_src_directory.join(allocator, &[_][]const u8{
779		pkg.value.root_src_path,
780		});
	780	const pkg_zig_file = try mod.value.root.joinString(
	781	allocator,
	782	mod.value.root_src_path,
	783	);
781	784	_ = try man.addFile(pkg_zig_file, null);
782	785	},
783	786	}
784		// Recurse to handle the package's dependencies
785		try addPackageTableToCacheHash(hash, arena, pkg.value.table, seen_table, hash_type);
	787	// Recurse to handle the module's dependencies
	788	try addModuleTableToCacheHash(hash, arena, mod.value.deps, seen_table, hash_type);
786	789	}
787	790	}
788	791
...	...	@@ -839,7 +842,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
839	842	break :blk true;
840	843
841	844	// If we have no zig code to compile, no need for LLVM.
842		if (options.main_pkg == null)
	845	if (options.main_mod == null)
843	846	break :blk false;
844	847
845	848	// If LLVM does not support the target, then we can't use it.
...	...	@@ -869,7 +872,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
869	872	// compiler state, the second clause here can be removed so that incremental
870	873	// cache mode is used for LLVM backend too. We need some fuzz testing before
871	874	// that can be enabled.
872		const cache_mode = if ((use_llvm or options.main_pkg == null) and !options.disable_lld_caching)
	875	const cache_mode = if ((use_llvm or options.main_mod == null) and !options.disable_lld_caching)
873	876	CacheMode.whole
874	877	else
875	878	options.cache_mode;
...	...	@@ -925,7 +928,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
925	928	if (use_llvm) {
926	929	// If stage1 generates an object file, self-hosted linker is not
927	930	// yet sophisticated enough to handle that.
928		break :blk options.main_pkg != null;
	931	break :blk options.main_mod != null;
929	932	}
930	933
931	934	break :blk false;
...	...	@@ -1210,7 +1213,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1210	1213	if (options.target.os.tag == .wasi) cache.hash.add(wasi_exec_model);
1211	1214	// TODO audit this and make sure everything is in it
1212	1215
1213		const module: ?*Module = if (options.main_pkg) \|main_pkg\| blk: {
	1216	const module: ?*Module = if (options.main_mod) \|main_mod\| blk: {
1214	1217	// Options that are specific to zig source files, that cannot be
1215	1218	// modified between incremental updates.
1216	1219	var hash = cache.hash;
...	...	@@ -1223,11 +1226,12 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1223	1226	// do want to namespace different source file names because they are
1224	1227	// likely different compilations and therefore this would be likely to
1225	1228	// cause cache hits.
1226		hash.addBytes(main_pkg.root_src_path);
1227		hash.addOptionalBytes(main_pkg.root_src_directory.path);
	1229	hash.addBytes(main_mod.root_src_path);
	1230	hash.addOptionalBytes(main_mod.root.root_dir.path);
	1231	hash.addBytes(main_mod.root.sub_path);
1228	1232	{
1229		var seen_table = std.AutoHashMap(*Package, void).init(arena);
1230		try addPackageTableToCacheHash(&hash, &arena_allocator, main_pkg.table, &seen_table, .path_bytes);
	1233	var seen_table = std.AutoHashMap(*Package.Module, void).init(arena);
	1234	try addModuleTableToCacheHash(&hash, &arena_allocator, main_mod.deps, &seen_table, .path_bytes);
1231	1235	}
1232	1236	},
1233	1237	.whole => {
...	...	@@ -1283,81 +1287,83 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1283	1287	.path = try options.local_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}),
1284	1288	};
1285	1289
1286		const builtin_pkg = try Package.createWithDir(
1287		gpa,
1288		zig_cache_artifact_directory,
1289		null,
1290		"builtin.zig",
1291		);
1292		errdefer builtin_pkg.destroy(gpa);
	1290	const builtin_mod = try Package.Module.create(arena, .{
	1291	.root = .{ .root_dir = zig_cache_artifact_directory },
	1292	.root_src_path = "builtin.zig",
	1293	.fully_qualified_name = "builtin",
	1294	});
1293	1295
1294		// When you're testing std, the main module is std. In that case, we'll just set the std
1295		// module to the main one, since avoiding the errors caused by duplicating it is more
1296		// effort than it's worth.
1297		const main_pkg_is_std = m: {
	1296	// When you're testing std, the main module is std. In that case,
	1297	// we'll just set the std module to the main one, since avoiding
	1298	// the errors caused by duplicating it is more effort than it's
	1299	// worth.
	1300	const main_mod_is_std = m: {
1298	1301	const std_path = try std.fs.path.resolve(arena, &[_][]const u8{
1299	1302	options.zig_lib_directory.path orelse ".",
1300	1303	"std",
1301	1304	"std.zig",
1302	1305	});
1303		defer arena.free(std_path);
1304	1306	const main_path = try std.fs.path.resolve(arena, &[_][]const u8{
1305		main_pkg.root_src_directory.path orelse ".",
1306		main_pkg.root_src_path,
	1307	main_mod.root.root_dir.path orelse ".",
	1308	main_mod.root.sub_path,
	1309	main_mod.root_src_path,
1307	1310	});
1308		defer arena.free(main_path);
1309	1311	break :m mem.eql(u8, main_path, std_path);
1310	1312	};
1311	1313
1312		const std_pkg = if (main_pkg_is_std)
1313		main_pkg
	1314	const std_mod = if (main_mod_is_std)
	1315	main_mod
1314	1316	else
1315		try Package.createWithDir(
1316		gpa,
1317		options.zig_lib_directory,
1318		"std",
1319		"std.zig",
1320		);
1321
1322		errdefer if (!main_pkg_is_std) std_pkg.destroy(gpa);
	1317	try Package.Module.create(arena, .{
	1318	.root = .{
	1319	.root_dir = options.zig_lib_directory,
	1320	.sub_path = "std",
	1321	},
	1322	.root_src_path = "std.zig",
	1323	.fully_qualified_name = "std",
	1324	});
1323	1325
1324		const root_pkg = if (options.is_test) root_pkg: {
1325		const test_pkg = if (options.test_runner_path) \|test_runner\| test_pkg: {
1326		const test_dir = std.fs.path.dirname(test_runner);
1327		const basename = std.fs.path.basename(test_runner);
1328		const pkg = try Package.create(gpa, test_dir, basename);
	1326	const root_mod = if (options.is_test) root_mod: {
	1327	const test_mod = if (options.test_runner_path) \|test_runner\| test_mod: {
	1328	const pkg = try Package.Module.create(arena, .{
	1329	.root = .{
	1330	.root_dir = Directory.cwd(),
	1331	.sub_path = std.fs.path.dirname(test_runner) orelse "",
	1332	},
	1333	.root_src_path = std.fs.path.basename(test_runner),
	1334	.fully_qualified_name = "root",
	1335	});
1329	1336
1330		// copy package table from main_pkg to root_pkg
1331		pkg.table = try main_pkg.table.clone(gpa);
1332		break :test_pkg pkg;
1333		} else try Package.createWithDir(
1334		gpa,
1335		options.zig_lib_directory,
1336		null,
1337		"test_runner.zig",
1338		);
1339		errdefer test_pkg.destroy(gpa);
	1337	pkg.deps = try main_mod.deps.clone(arena);
	1338	break :test_mod pkg;
	1339	} else try Package.Module.create(arena, .{
	1340	.root = .{
	1341	.root_dir = options.zig_lib_directory,
	1342	},
	1343	.root_src_path = "test_runner.zig",
	1344	.fully_qualified_name = "root",
	1345	});
1340	1346
1341		break :root_pkg test_pkg;
1342		} else main_pkg;
1343		errdefer if (options.is_test) root_pkg.destroy(gpa);
	1347	break :root_mod test_mod;
	1348	} else main_mod;
1344	1349
1345		const compiler_rt_pkg = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_pkg: {
1346		break :compiler_rt_pkg try Package.createWithDir(
1347		gpa,
1348		options.zig_lib_directory,
1349		null,
1350		"compiler_rt.zig",
1351		);
	1350	const compiler_rt_mod = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_mod: {
	1351	break :compiler_rt_mod try Package.Module.create(arena, .{
	1352	.root = .{
	1353	.root_dir = options.zig_lib_directory,
	1354	},
	1355	.root_src_path = "compiler_rt.zig",
	1356	.fully_qualified_name = "compiler_rt",
	1357	});
1352	1358	} else null;
1353		errdefer if (compiler_rt_pkg) \|p\| p.destroy(gpa);
1354	1359
1355		try main_pkg.add(gpa, "builtin", builtin_pkg);
1356		try main_pkg.add(gpa, "root", root_pkg);
1357		try main_pkg.add(gpa, "std", std_pkg);
1358
1359		if (compiler_rt_pkg) \|p\| {
1360		try main_pkg.add(gpa, "compiler_rt", p);
	1360	{
	1361	try main_mod.deps.ensureUnusedCapacity(arena, 4);
	1362	main_mod.deps.putAssumeCapacity("builtin", builtin_mod);
	1363	main_mod.deps.putAssumeCapacity("root", root_mod);
	1364	main_mod.deps.putAssumeCapacity("std", std_mod);
	1365	if (compiler_rt_mod) \|m\|
	1366	main_mod.deps.putAssumeCapacity("compiler_rt", m);
1361	1367	}
1362	1368
1363	1369	// Pre-open the directory handles for cached ZIR code so that it does not need
...	...	@@ -1395,8 +1401,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1395	1401	module.* = .{
1396	1402	.gpa = gpa,
1397	1403	.comp = comp,
1398		.main_pkg = main_pkg,
1399		.root_pkg = root_pkg,
	1404	.main_mod = main_mod,
	1405	.root_mod = root_mod,
1400	1406	.zig_cache_artifact_directory = zig_cache_artifact_directory,
1401	1407	.global_zir_cache = global_zir_cache,
1402	1408	.local_zir_cache = local_zir_cache,
...	...	@@ -1664,7 +1670,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1664	1670	errdefer bin_file.destroy();
1665	1671	comp.* = .{
1666	1672	.gpa = gpa,
1667		.arena_state = arena_allocator.state,
	1673	.arena = arena_allocator,
1668	1674	.zig_lib_directory = options.zig_lib_directory,
1669	1675	.local_cache_directory = options.local_cache_directory,
1670	1676	.global_cache_directory = options.global_cache_directory,
...	...	@@ -1982,7 +1988,8 @@ pub fn destroy(self: *Compilation) void {
1982	1988	if (self.owned_link_dir) \|*dir\| dir.close();
1983	1989
1984	1990	// This destroys `self`.
1985		self.arena_state.promote(gpa).deinit();
	1991	var arena_instance = self.arena;
	1992	arena_instance.deinit();
1986	1993	}
1987	1994
1988	1995	pub fn clearMiscFailures(comp: *Compilation) void {
...	...	@@ -2005,8 +2012,8 @@ fn restorePrevZigCacheArtifactDirectory(comp: Compilation, directory: Director
2005	2012	// This is only for cleanup purposes; Module.deinit calls close
2006	2013	// on the handle of zig_cache_artifact_directory.
2007	2014	if (comp.bin_file.options.module) \|module\| {
2008		const builtin_pkg = module.main_pkg.table.get("builtin").?;
2009		module.zig_cache_artifact_directory = builtin_pkg.root_src_directory;
	2015	const builtin_mod = module.main_mod.deps.get("builtin").?;
	2016	module.zig_cache_artifact_directory = builtin_mod.root.root_dir;
2010	2017	}
2011	2018	}
2012	2019
...	...	@@ -2148,8 +2155,8 @@ pub fn update(comp: Compilation, main_progress_node: std.Progress.Node) !void
2148	2155
2149	2156	// Make sure std.zig is inside the import_table. We unconditionally need
2150	2157	// it for start.zig.
2151		const std_pkg = module.main_pkg.table.get("std").?;
2152		_ = try module.importPkg(std_pkg);
	2158	const std_mod = module.main_mod.deps.get("std").?;
	2159	_ = try module.importPkg(std_mod);
2153	2160
2154	2161	// Normally we rely on importing std to in turn import the root source file
2155	2162	// in the start code, but when using the stage1 backend that won't happen,
...	...	@@ -2158,11 +2165,11 @@ pub fn update(comp: Compilation, main_progress_node: std.Progress.Node) !void
2158	2165	// Likewise, in the case of `zig test`, the test runner is the root source file,
2159	2166	// and so there is nothing to import the main file.
2160	2167	if (comp.bin_file.options.is_test) {
2161		_ = try module.importPkg(module.main_pkg);
	2168	_ = try module.importPkg(module.main_mod);
2162	2169	}
2163	2170
2164		if (module.main_pkg.table.get("compiler_rt")) \|compiler_rt_pkg\| {
2165		_ = try module.importPkg(compiler_rt_pkg);
	2171	if (module.main_mod.deps.get("compiler_rt")) \|compiler_rt_mod\| {
	2172	_ = try module.importPkg(compiler_rt_mod);
2166	2173	}
2167	2174
2168	2175	// Put a work item in for every known source file to detect if
...	...	@@ -2185,13 +2192,13 @@ pub fn update(comp: Compilation, main_progress_node: std.Progress.Node) !void
2185	2192	}
2186	2193	}
2187	2194
2188		try comp.work_queue.writeItem(.{ .analyze_pkg = std_pkg });
	2195	try comp.work_queue.writeItem(.{ .analyze_mod = std_mod });
2189	2196	if (comp.bin_file.options.is_test) {
2190		try comp.work_queue.writeItem(.{ .analyze_pkg = module.main_pkg });
	2197	try comp.work_queue.writeItem(.{ .analyze_mod = module.main_mod });
2191	2198	}
2192	2199
2193		if (module.main_pkg.table.get("compiler_rt")) \|compiler_rt_pkg\| {
2194		try comp.work_queue.writeItem(.{ .analyze_pkg = compiler_rt_pkg });
	2200	if (module.main_mod.deps.get("compiler_rt")) \|compiler_rt_mod\| {
	2201	try comp.work_queue.writeItem(.{ .analyze_mod = compiler_rt_mod });
2195	2202	}
2196	2203	}
2197	2204
...	...	@@ -2420,19 +2427,17 @@ fn addNonIncrementalStuffToCacheManifest(comp: Compilation, man: Cache.Manifes
2420	2427	comptime assert(link_hash_implementation_version == 10);
2421	2428
2422	2429	if (comp.bin_file.options.module) \|mod\| {
2423		const main_zig_file = try mod.main_pkg.root_src_directory.join(arena, &[_][]const u8{
2424		mod.main_pkg.root_src_path,
2425		});
	2430	const main_zig_file = try mod.main_mod.root.joinString(arena, mod.main_mod.root_src_path);
2426	2431	_ = try man.addFile(main_zig_file, null);
2427	2432	{
2428		var seen_table = std.AutoHashMap(*Package, void).init(arena);
	2433	var seen_table = std.AutoHashMap(*Package.Module, void).init(arena);
2429	2434
2430	2435	// Skip builtin.zig; it is useless as an input, and we don't want to have to
2431	2436	// write it before checking for a cache hit.
2432		const builtin_pkg = mod.main_pkg.table.get("builtin").?;
2433		try seen_table.put(builtin_pkg, {});
	2437	const builtin_mod = mod.main_mod.deps.get("builtin").?;
	2438	try seen_table.put(builtin_mod, {});
2434	2439
2435		try addPackageTableToCacheHash(&man.hash, &arena_allocator, mod.main_pkg.table, &seen_table, .{ .files = man });
	2440	try addModuleTableToCacheHash(&man.hash, &arena_allocator, mod.main_mod.deps, &seen_table, .{ .files = man });
2436	2441	}
2437	2442
2438	2443	// Synchronize with other matching comments: ZigOnlyHashStuff
...	...	@@ -2616,23 +2621,19 @@ fn reportMultiModuleErrors(mod: *Module) !void {
2616	2621	errdefer for (notes[0..i]) \|*n\| n.deinit(mod.gpa);
2617	2622	note.* = switch (ref) {
2618	2623	.import => \|loc\| blk: {
2619		const name = try loc.file_scope.pkg.getName(mod.gpa, mod.*);
2620		defer mod.gpa.free(name);
2621	2624	break :blk try Module.ErrorMsg.init(
2622	2625	mod.gpa,
2623	2626	loc,
2624	2627	"imported from module {s}",
2625		.{name},
	2628	.{loc.file_scope.mod.fully_qualified_name},
2626	2629	);
2627	2630	},
2628	2631	.root => \|pkg\| blk: {
2629		const name = try pkg.getName(mod.gpa, mod.*);
2630		defer mod.gpa.free(name);
2631	2632	break :blk try Module.ErrorMsg.init(
2632	2633	mod.gpa,
2633	2634	.{ .file_scope = file, .parent_decl_node = 0, .lazy = .entire_file },
2634	2635	"root of module {s}",
2635		.{name},
	2636	.{pkg.fully_qualified_name},
2636	2637	);
2637	2638	},
2638	2639	};
...	...	@@ -3564,8 +3565,8 @@ fn processOneJob(comp: Compilation, job: Job, prog_node: std.Progress.Node) !v
3564	3565	decl.analysis = .codegen_failure_retryable;
3565	3566	};
3566	3567	},
3567		.analyze_pkg => \|pkg\| {
3568		const named_frame = tracy.namedFrame("analyze_pkg");
	3568	.analyze_mod => \|pkg\| {
	3569	const named_frame = tracy.namedFrame("analyze_mod");
3569	3570	defer named_frame.end();
3570	3571
3571	3572	const module = comp.bin_file.options.module.?;
...	...	@@ -3904,17 +3905,12 @@ pub fn obtainWin32ResourceCacheManifest(comp: *const Compilation) Cache.Manifest
3904	3905	return man;
3905	3906	}
3906	3907
3907		test "cImport" {
3908		_ = cImport;
3909		}
3910
3911	3908	pub const CImportResult = struct {
3912	3909	out_zig_path: []u8,
3913	3910	cache_hit: bool,
3914	3911	errors: std.zig.ErrorBundle,
3915	3912
3916	3913	pub fn deinit(result: *CImportResult, gpa: std.mem.Allocator) void {
3917		gpa.free(result.out_zig_path);
3918	3914	result.errors.deinit(gpa);
3919	3915	}
3920	3916	};
...	...	@@ -4059,7 +4055,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
4059	4055	};
4060	4056	}
4061	4057
4062		const out_zig_path = try comp.local_cache_directory.join(comp.gpa, &[_][]const u8{
	4058	const out_zig_path = try comp.local_cache_directory.join(comp.arena.allocator(), &.{
4063	4059	"o", &digest, cimport_zig_basename,
4064	4060	});
4065	4061	if (comp.verbose_cimport) {
...	...	@@ -4214,17 +4210,9 @@ fn reportRetryableAstGenError(
4214	4210	},
4215	4211	};
4216	4212
4217		const err_msg = if (file.pkg.root_src_directory.path) \|dir_path\|
4218		try Module.ErrorMsg.create(
4219		gpa,
4220		src_loc,
4221		"unable to load '{s}" ++ std.fs.path.sep_str ++ "{s}': {s}",
4222		.{ dir_path, file.sub_file_path, @errorName(err) },
4223		)
4224		else
4225		try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{s}': {s}", .{
4226		file.sub_file_path, @errorName(err),
4227		});
	4213	const err_msg = try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{}{s}': {s}", .{
	4214	file.mod.root, file.sub_file_path, @errorName(err),
	4215	});
4228	4216	errdefer err_msg.destroy(gpa);
4229	4217
4230	4218	{
...	...	@@ -4244,17 +4232,10 @@ fn reportRetryableEmbedFileError(
4244	4232
4245	4233	const src_loc: Module.SrcLoc = mod.declPtr(embed_file.owner_decl).srcLoc(mod);
4246	4234
4247		const err_msg = if (embed_file.pkg.root_src_directory.path) \|dir_path\|
4248		try Module.ErrorMsg.create(
4249		gpa,
4250		src_loc,
4251		"unable to load '{s}" ++ std.fs.path.sep_str ++ "{s}': {s}",
4252		.{ dir_path, embed_file.sub_file_path, @errorName(err) },
4253		)
4254		else
4255		try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{s}': {s}", .{
4256		embed_file.sub_file_path, @errorName(err),
4257		});
	4235	const err_msg = try Module.ErrorMsg.create(gpa, src_loc, "unable to load '{}{s}': {s}", .{
	4236	embed_file.mod.root, embed_file.sub_file_path, @errorName(err),
	4237	});
	4238
4258	4239	errdefer err_msg.destroy(gpa);
4259	4240
4260	4241	{
...	...	@@ -6377,13 +6358,13 @@ fn buildOutputFromZig(
6377	6358	const tracy_trace = trace(@src());
6378	6359	defer tracy_trace.end();
6379	6360
6380		std.debug.assert(output_mode != .Exe);
	6361	assert(output_mode != .Exe);
6381	6362
6382		var main_pkg: Package = .{
6383		.root_src_directory = comp.zig_lib_directory,
	6363	var main_mod: Package.Module = .{
	6364	.root = .{ .root_dir = comp.zig_lib_directory },
6384	6365	.root_src_path = src_basename,
	6366	.fully_qualified_name = "root",
6385	6367	};
6386		defer main_pkg.deinitTable(comp.gpa);
6387	6368	const root_name = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len];
6388	6369	const target = comp.getTarget();
6389	6370	const bin_basename = try std.zig.binNameAlloc(comp.gpa, .{
...	...	@@ -6404,7 +6385,7 @@ fn buildOutputFromZig(
6404	6385	.cache_mode = .whole,
6405	6386	.target = target,
6406	6387	.root_name = root_name,
6407		.main_pkg = &main_pkg,
	6388	.main_mod = &main_mod,
6408	6389	.output_mode = output_mode,
6409	6390	.thread_pool = comp.thread_pool,
6410	6391	.libc_installation = comp.bin_file.options.libc_installation,
...	...	@@ -6481,7 +6462,7 @@ pub fn build_crt_file(
6481	6462	.cache_mode = .whole,
6482	6463	.target = target,
6483	6464	.root_name = root_name,
6484		.main_pkg = null,
	6465	.main_mod = null,
6485	6466	.output_mode = output_mode,
6486	6467	.thread_pool = comp.thread_pool,
6487	6468	.libc_installation = comp.bin_file.options.libc_installation,

src/Manifest.zig deleted-519

...	...	@@ -1,519 +0,0 @@
1		pub const basename = "build.zig.zon";
2		pub const Hash = std.crypto.hash.sha2.Sha256;
3
4		pub const Dependency = struct {
5		location: union(enum) {
6		url: []const u8,
7		path: []const u8,
8		},
9		location_tok: Ast.TokenIndex,
10		hash: ?[]const u8,
11		hash_tok: Ast.TokenIndex,
12		};
13
14		pub const ErrorMessage = struct {
15		msg: []const u8,
16		tok: Ast.TokenIndex,
17		off: u32,
18		};
19
20		pub const MultihashFunction = enum(u16) {
21		identity = 0x00,
22		sha1 = 0x11,
23		@"sha2-256" = 0x12,
24		@"sha2-512" = 0x13,
25		@"sha3-512" = 0x14,
26		@"sha3-384" = 0x15,
27		@"sha3-256" = 0x16,
28		@"sha3-224" = 0x17,
29		@"sha2-384" = 0x20,
30		@"sha2-256-trunc254-padded" = 0x1012,
31		@"sha2-224" = 0x1013,
32		@"sha2-512-224" = 0x1014,
33		@"sha2-512-256" = 0x1015,
34		@"blake2b-256" = 0xb220,
35		_,
36		};
37
38		pub const multihash_function: MultihashFunction = switch (Hash) {
39		std.crypto.hash.sha2.Sha256 => .@"sha2-256",
40		else => @compileError("unreachable"),
41		};
42		comptime {
43		// We avoid unnecessary uleb128 code in hexDigest by asserting here the
44		// values are small enough to be contained in the one-byte encoding.
45		assert(@intFromEnum(multihash_function) < 127);
46		assert(Hash.digest_length < 127);
47		}
48		pub const multihash_len = 1 + 1 + Hash.digest_length;
49
50		name: []const u8,
51		version: std.SemanticVersion,
52		dependencies: std.StringArrayHashMapUnmanaged(Dependency),
53
54		errors: []ErrorMessage,
55		arena_state: std.heap.ArenaAllocator.State,
56
57		pub const Error = Allocator.Error;
58
59		pub fn parse(gpa: Allocator, ast: std.zig.Ast) Error!Manifest {
60		const node_tags = ast.nodes.items(.tag);
61		const node_datas = ast.nodes.items(.data);
62		assert(node_tags[0] == .root);
63		const main_node_index = node_datas[0].lhs;
64
65		var arena_instance = std.heap.ArenaAllocator.init(gpa);
66		errdefer arena_instance.deinit();
67
68		var p: Parse = .{
69		.gpa = gpa,
70		.ast = ast,
71		.arena = arena_instance.allocator(),
72		.errors = .{},
73
74		.name = undefined,
75		.version = undefined,
76		.dependencies = .{},
77		.buf = .{},
78		};
79		defer p.buf.deinit(gpa);
80		defer p.errors.deinit(gpa);
81		defer p.dependencies.deinit(gpa);
82
83		p.parseRoot(main_node_index) catch \|err\| switch (err) {
84		error.ParseFailure => assert(p.errors.items.len > 0),
85		else => \|e\| return e,
86		};
87
88		return .{
89		.name = p.name,
90		.version = p.version,
91		.dependencies = try p.dependencies.clone(p.arena),
92		.errors = try p.arena.dupe(ErrorMessage, p.errors.items),
93		.arena_state = arena_instance.state,
94		};
95		}
96
97		pub fn deinit(man: *Manifest, gpa: Allocator) void {
98		man.arena_state.promote(gpa).deinit();
99		man.* = undefined;
100		}
101
102		const hex_charset = "0123456789abcdef";
103
104		pub fn hex64(x: u64) [16]u8 {
105		var result: [16]u8 = undefined;
106		var i: usize = 0;
107		while (i < 8) : (i += 1) {
108		const byte = @as(u8, @truncate(x >> @as(u6, @intCast(8 * i))));
109		result[i * 2 + 0] = hex_charset[byte >> 4];
110		result[i * 2 + 1] = hex_charset[byte & 15];
111		}
112		return result;
113		}
114
115		test hex64 {
116		const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]";
117		try std.testing.expectEqualStrings("[00efcdab78563412]", s);
118		}
119
120		pub fn hexDigest(digest: [Hash.digest_length]u8) [multihash_len * 2]u8 {
121		var result: [multihash_len * 2]u8 = undefined;
122
123		result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
124		result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
125
126		result[2] = hex_charset[Hash.digest_length >> 4];
127		result[3] = hex_charset[Hash.digest_length & 15];
128
129		for (digest, 0..) \|byte, i\| {
130		result[4 + i * 2] = hex_charset[byte >> 4];
131		result[5 + i * 2] = hex_charset[byte & 15];
132		}
133		return result;
134		}
135
136		const Parse = struct {
137		gpa: Allocator,
138		ast: std.zig.Ast,
139		arena: Allocator,
140		buf: std.ArrayListUnmanaged(u8),
141		errors: std.ArrayListUnmanaged(ErrorMessage),
142
143		name: []const u8,
144		version: std.SemanticVersion,
145		dependencies: std.StringArrayHashMapUnmanaged(Dependency),
146
147		const InnerError = error{ ParseFailure, OutOfMemory };
148
149		fn parseRoot(p: *Parse, node: Ast.Node.Index) !void {
150		const ast = p.ast;
151		const main_tokens = ast.nodes.items(.main_token);
152		const main_token = main_tokens[node];
153
154		var buf: [2]Ast.Node.Index = undefined;
155		const struct_init = ast.fullStructInit(&buf, node) orelse {
156		return fail(p, main_token, "expected top level expression to be a struct", .{});
157		};
158
159		var have_name = false;
160		var have_version = false;
161
162		for (struct_init.ast.fields) \|field_init\| {
163		const name_token = ast.firstToken(field_init) - 2;
164		const field_name = try identifierTokenString(p, name_token);
165		// We could get fancy with reflection and comptime logic here but doing
166		// things manually provides an opportunity to do any additional verification
167		// that is desirable on a per-field basis.
168		if (mem.eql(u8, field_name, "dependencies")) {
169		try parseDependencies(p, field_init);
170		} else if (mem.eql(u8, field_name, "name")) {
171		p.name = try parseString(p, field_init);
172		have_name = true;
173		} else if (mem.eql(u8, field_name, "version")) {
174		const version_text = try parseString(p, field_init);
175		p.version = std.SemanticVersion.parse(version_text) catch \|err\| v: {
176		try appendError(p, main_tokens[field_init], "unable to parse semantic version: {s}", .{@errorName(err)});
177		break :v undefined;
178		};
179		have_version = true;
180		} else {
181		// Ignore unknown fields so that we can add fields in future zig
182		// versions without breaking older zig versions.
183		}
184		}
185
186		if (!have_name) {
187		try appendError(p, main_token, "missing top-level 'name' field", .{});
188		}
189
190		if (!have_version) {
191		try appendError(p, main_token, "missing top-level 'version' field", .{});
192		}
193		}
194
195		fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void {
196		const ast = p.ast;
197		const main_tokens = ast.nodes.items(.main_token);
198
199		var buf: [2]Ast.Node.Index = undefined;
200		const struct_init = ast.fullStructInit(&buf, node) orelse {
201		const tok = main_tokens[node];
202		return fail(p, tok, "expected dependencies expression to be a struct", .{});
203		};
204
205		for (struct_init.ast.fields) \|field_init\| {
206		const name_token = ast.firstToken(field_init) - 2;
207		const dep_name = try identifierTokenString(p, name_token);
208		const dep = try parseDependency(p, field_init);
209		try p.dependencies.put(p.gpa, dep_name, dep);
210		}
211		}
212
213		fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency {
214		const ast = p.ast;
215		const main_tokens = ast.nodes.items(.main_token);
216
217		var buf: [2]Ast.Node.Index = undefined;
218		const struct_init = ast.fullStructInit(&buf, node) orelse {
219		const tok = main_tokens[node];
220		return fail(p, tok, "expected dependency expression to be a struct", .{});
221		};
222
223		var dep: Dependency = .{
224		.location = undefined,
225		.location_tok = undefined,
226		.hash = null,
227		.hash_tok = undefined,
228		};
229		var has_location = false;
230
231		for (struct_init.ast.fields) \|field_init\| {
232		const name_token = ast.firstToken(field_init) - 2;
233		const field_name = try identifierTokenString(p, name_token);
234		// We could get fancy with reflection and comptime logic here but doing
235		// things manually provides an opportunity to do any additional verification
236		// that is desirable on a per-field basis.
237		if (mem.eql(u8, field_name, "url")) {
238		if (has_location) {
239		return fail(p, main_tokens[field_init], "dependency should specify only one of 'url' and 'path' fields.", .{});
240		}
241		dep.location = .{
242		.url = parseString(p, field_init) catch \|err\| switch (err) {
243		error.ParseFailure => continue,
244		else => \|e\| return e,
245		},
246		};
247		has_location = true;
248		dep.location_tok = main_tokens[field_init];
249		} else if (mem.eql(u8, field_name, "path")) {
250		if (has_location) {
251		return fail(p, main_tokens[field_init], "dependency should specify only one of 'url' and 'path' fields.", .{});
252		}
253		dep.location = .{
254		.path = parseString(p, field_init) catch \|err\| switch (err) {
255		error.ParseFailure => continue,
256		else => \|e\| return e,
257		},
258		};
259		has_location = true;
260		dep.location_tok = main_tokens[field_init];
261		} else if (mem.eql(u8, field_name, "hash")) {
262		dep.hash = parseHash(p, field_init) catch \|err\| switch (err) {
263		error.ParseFailure => continue,
264		else => \|e\| return e,
265		};
266		dep.hash_tok = main_tokens[field_init];
267		} else {
268		// Ignore unknown fields so that we can add fields in future zig
269		// versions without breaking older zig versions.
270		}
271		}
272
273		if (!has_location) {
274		try appendError(p, main_tokens[node], "dependency requires location field, one of 'url' or 'path'.", .{});
275		}
276
277		return dep;
278		}
279
280		fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 {
281		const ast = p.ast;
282		const node_tags = ast.nodes.items(.tag);
283		const main_tokens = ast.nodes.items(.main_token);
284		if (node_tags[node] != .string_literal) {
285		return fail(p, main_tokens[node], "expected string literal", .{});
286		}
287		const str_lit_token = main_tokens[node];
288		const token_bytes = ast.tokenSlice(str_lit_token);
289		p.buf.clearRetainingCapacity();
290		try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0);
291		const duped = try p.arena.dupe(u8, p.buf.items);
292		return duped;
293		}
294
295		fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 {
296		const ast = p.ast;
297		const main_tokens = ast.nodes.items(.main_token);
298		const tok = main_tokens[node];
299		const h = try parseString(p, node);
300
301		if (h.len >= 2) {
302		const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch \|err\| {
303		return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{
304		@errorName(err),
305		});
306		};
307		if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) {
308		return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{});
309		}
310		}
311
312		const hex_multihash_len = 2 * Manifest.multihash_len;
313		if (h.len != hex_multihash_len) {
314		return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{
315		hex_multihash_len, h.len,
316		});
317		}
318
319		return h;
320		}
321
322		/// TODO: try to DRY this with AstGen.identifierTokenString
323		fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 {
324		const ast = p.ast;
325		const token_tags = ast.tokens.items(.tag);
326		assert(token_tags[token] == .identifier);
327		const ident_name = ast.tokenSlice(token);
328		if (!mem.startsWith(u8, ident_name, "@")) {
329		return ident_name;
330		}
331		p.buf.clearRetainingCapacity();
332		try parseStrLit(p, token, &p.buf, ident_name, 1);
333		const duped = try p.arena.dupe(u8, p.buf.items);
334		return duped;
335		}
336
337		/// TODO: try to DRY this with AstGen.parseStrLit
338		fn parseStrLit(
339		p: *Parse,
340		token: Ast.TokenIndex,
341		buf: *std.ArrayListUnmanaged(u8),
342		bytes: []const u8,
343		offset: u32,
344		) InnerError!void {
345		const raw_string = bytes[offset..];
346		var buf_managed = buf.toManaged(p.gpa);
347		const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string);
348		buf.* = buf_managed.moveToUnmanaged();
349		switch (try result) {
350		.success => {},
351		.failure => \|err\| try p.appendStrLitError(err, token, bytes, offset),
352		}
353		}
354
355		/// TODO: try to DRY this with AstGen.failWithStrLitError
356		fn appendStrLitError(
357		p: *Parse,
358		err: std.zig.string_literal.Error,
359		token: Ast.TokenIndex,
360		bytes: []const u8,
361		offset: u32,
362		) Allocator.Error!void {
363		const raw_string = bytes[offset..];
364		switch (err) {
365		.invalid_escape_character => \|bad_index\| {
366		try p.appendErrorOff(
367		token,
368		offset + @as(u32, @intCast(bad_index)),
369		"invalid escape character: '{c}'",
370		.{raw_string[bad_index]},
371		);
372		},
373		.expected_hex_digit => \|bad_index\| {
374		try p.appendErrorOff(
375		token,
376		offset + @as(u32, @intCast(bad_index)),
377		"expected hex digit, found '{c}'",
378		.{raw_string[bad_index]},
379		);
380		},
381		.empty_unicode_escape_sequence => \|bad_index\| {
382		try p.appendErrorOff(
383		token,
384		offset + @as(u32, @intCast(bad_index)),
385		"empty unicode escape sequence",
386		.{},
387		);
388		},
389		.expected_hex_digit_or_rbrace => \|bad_index\| {
390		try p.appendErrorOff(
391		token,
392		offset + @as(u32, @intCast(bad_index)),
393		"expected hex digit or '}}', found '{c}'",
394		.{raw_string[bad_index]},
395		);
396		},
397		.invalid_unicode_codepoint => \|bad_index\| {
398		try p.appendErrorOff(
399		token,
400		offset + @as(u32, @intCast(bad_index)),
401		"unicode escape does not correspond to a valid codepoint",
402		.{},
403		);
404		},
405		.expected_lbrace => \|bad_index\| {
406		try p.appendErrorOff(
407		token,
408		offset + @as(u32, @intCast(bad_index)),
409		"expected '{{', found '{c}",
410		.{raw_string[bad_index]},
411		);
412		},
413		.expected_rbrace => \|bad_index\| {
414		try p.appendErrorOff(
415		token,
416		offset + @as(u32, @intCast(bad_index)),
417		"expected '}}', found '{c}",
418		.{raw_string[bad_index]},
419		);
420		},
421		.expected_single_quote => \|bad_index\| {
422		try p.appendErrorOff(
423		token,
424		offset + @as(u32, @intCast(bad_index)),
425		"expected single quote ('), found '{c}",
426		.{raw_string[bad_index]},
427		);
428		},
429		.invalid_character => \|bad_index\| {
430		try p.appendErrorOff(
431		token,
432		offset + @as(u32, @intCast(bad_index)),
433		"invalid byte in string or character literal: '{c}'",
434		.{raw_string[bad_index]},
435		);
436		},
437		}
438		}
439
440		fn fail(
441		p: *Parse,
442		tok: Ast.TokenIndex,
443		comptime fmt: []const u8,
444		args: anytype,
445		) InnerError {
446		try appendError(p, tok, fmt, args);
447		return error.ParseFailure;
448		}
449
450		fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void {
451		return appendErrorOff(p, tok, 0, fmt, args);
452		}
453
454		fn appendErrorOff(
455		p: *Parse,
456		tok: Ast.TokenIndex,
457		byte_offset: u32,
458		comptime fmt: []const u8,
459		args: anytype,
460		) Allocator.Error!void {
461		try p.errors.append(p.gpa, .{
462		.msg = try std.fmt.allocPrint(p.arena, fmt, args),
463		.tok = tok,
464		.off = byte_offset,
465		});
466		}
467		};
468
469		const Manifest = @This();
470		const std = @import("std");
471		const mem = std.mem;
472		const Allocator = std.mem.Allocator;
473		const assert = std.debug.assert;
474		const Ast = std.zig.Ast;
475		const testing = std.testing;
476
477		test "basic" {
478		const gpa = testing.allocator;
479
480		const example =
481		\\.{
482		\\ .name = "foo",
483		\\ .version = "3.2.1",
484		\\ .dependencies = .{
485		\\ .bar = .{
486		\\ .url = "https://example.com/baz.tar.gz",
487		\\ .hash = "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f",
488		\\ },
489		\\ },
490		\\}
491		;
492
493		var ast = try std.zig.Ast.parse(gpa, example, .zon);
494		defer ast.deinit(gpa);
495
496		try testing.expect(ast.errors.len == 0);
497
498		var manifest = try Manifest.parse(gpa, ast);
499		defer manifest.deinit(gpa);
500
501		try testing.expectEqualStrings("foo", manifest.name);
502
503		try testing.expectEqual(@as(std.SemanticVersion, .{
504		.major = 3,
505		.minor = 2,
506		.patch = 1,
507		}), manifest.version);
508
509		try testing.expect(manifest.dependencies.count() == 1);
510		try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]);
511		try testing.expectEqualStrings(
512		"https://example.com/baz.tar.gz",
513		manifest.dependencies.values()[0].url,
514		);
515		try testing.expectEqualStrings(
516		"1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f",
517		manifest.dependencies.values()[0].hash orelse return error.TestFailed,
518		);
519		}

src/Module.zig+93-114

...	...	@@ -55,10 +55,10 @@ comp: *Compilation,
55	55	/// Where build artifacts and incremental compilation metadata serialization go.
56	56	zig_cache_artifact_directory: Compilation.Directory,
57	57	/// Pointer to externally managed resource.
58		root_pkg: *Package,
59		/// Normally, `main_pkg` and `root_pkg` are the same. The exception is `zig test`, in which
60		/// `root_pkg` is the test runner, and `main_pkg` is the user's source file which has the tests.
61		main_pkg: *Package,
	58	root_mod: *Package.Module,
	59	/// Normally, `main_mod` and `root_mod` are the same. The exception is `zig test`, in which
	60	/// `root_mod` is the test runner, and `main_mod` is the user's source file which has the tests.
	61	main_mod: *Package.Module,
62	62	sema_prog_node: std.Progress.Node = undefined,
63	63
64	64	/// Used by AstGen worker to load and store ZIR cache.
...	...	@@ -973,8 +973,8 @@ pub const File = struct {
973	973	tree: Ast,
974	974	/// Whether this is populated or not depends on `zir_loaded`.
975	975	zir: Zir,
976		/// Package that this file is a part of, managed externally.
977		pkg: *Package,
	976	/// Module that this file is a part of, managed externally.
	977	mod: *Package.Module,
978	978	/// Whether this file is a part of multiple packages. This is an error condition which will be reported after AstGen.
979	979	multi_pkg: bool = false,
980	980	/// List of references to this file, used for multi-package errors.
...	...	@@ -998,8 +998,8 @@ pub const File = struct {
998	998	pub const Reference = union(enum) {
999	999	/// The file is imported directly (i.e. not as a package) with @import.
1000	1000	import: SrcLoc,
1001		/// The file is the root of a package.
1002		root: *Package,
	1001	/// The file is the root of a module.
	1002	root: *Package.Module,
1003	1003	};
1004	1004
1005	1005	pub fn unload(file: *File, gpa: Allocator) void {
...	...	@@ -1058,14 +1058,9 @@ pub const File = struct {
1058	1058	.stat = file.stat,
1059	1059	};
1060	1060
1061		const root_dir_path = file.pkg.root_src_directory.path orelse ".";
1062		log.debug("File.getSource, not cached. pkgdir={s} sub_file_path={s}", .{
1063		root_dir_path, file.sub_file_path,
1064		});
1065
1066	1061	// Keep track of inode, file size, mtime, hash so we can detect which files
1067	1062	// have been modified when an incremental update is requested.
1068		var f = try file.pkg.root_src_directory.handle.openFile(file.sub_file_path, .{});
	1063	var f = try file.mod.root.openFile(file.sub_file_path, .{});
1069	1064	defer f.close();
1070	1065
1071	1066	const stat = try f.stat();
...	...	@@ -1134,14 +1129,12 @@ pub const File = struct {
1134	1129	return ip.getOrPutTrailingString(mod.gpa, ip.string_bytes.items.len - start);
1135	1130	}
1136	1131
1137		/// Returns the full path to this file relative to its package.
1138	1132	pub fn fullPath(file: File, ally: Allocator) ![]u8 {
1139		return file.pkg.root_src_directory.join(ally, &[_][]const u8{file.sub_file_path});
	1133	return file.mod.root.joinString(ally, file.sub_file_path);
1140	1134	}
1141	1135
1142		/// Returns the full path to this file relative to its package.
1143	1136	pub fn fullPathZ(file: File, ally: Allocator) ![:0]u8 {
1144		return file.pkg.root_src_directory.joinZ(ally, &[_][]const u8{file.sub_file_path});
	1137	return file.mod.root.joinStringZ(ally, file.sub_file_path);
1145	1138	}
1146	1139
1147	1140	pub fn dumpSrc(file: *File, src: LazySrcLoc) void {
...	...	@@ -1181,10 +1174,10 @@ pub const File = struct {
1181	1174	}
1182	1175
1183	1176	const pkg = switch (ref) {
1184		.import => \|loc\| loc.file_scope.pkg,
	1177	.import => \|loc\| loc.file_scope.mod,
1185	1178	.root => \|pkg\| pkg,
1186	1179	};
1187		if (pkg != file.pkg) file.multi_pkg = true;
	1180	if (pkg != file.mod) file.multi_pkg = true;
1188	1181	}
1189	1182
1190	1183	/// Mark this file and every file referenced by it as multi_pkg and report an
...	...	@@ -1226,7 +1219,7 @@ pub const EmbedFile = struct {
1226	1219	bytes: [:0]const u8,
1227	1220	stat: Cache.File.Stat,
1228	1221	/// Package that this file is a part of, managed externally.
1229		pkg: *Package,
	1222	mod: *Package.Module,
1230	1223	/// The Decl that was created from the `@embedFile` to own this resource.
1231	1224	/// This is how zig knows what other Decl objects to invalidate if the file
1232	1225	/// changes on disk.
...	...	@@ -2542,28 +2535,6 @@ pub fn deinit(mod: *Module) void {
2542	2535	}
2543	2536
2544	2537	mod.deletion_set.deinit(gpa);
2545
2546		// The callsite of `Compilation.create` owns the `main_pkg`, however
2547		// Module owns the builtin and std packages that it adds.
2548		if (mod.main_pkg.table.fetchRemove("builtin")) \|kv\| {
2549		gpa.free(kv.key);
2550		kv.value.destroy(gpa);
2551		}
2552		if (mod.main_pkg.table.fetchRemove("std")) \|kv\| {
2553		gpa.free(kv.key);
2554		// It's possible for main_pkg to be std when running 'zig test'! In this case, we must not
2555		// destroy it, since it would lead to a double-free.
2556		if (kv.value != mod.main_pkg) {
2557		kv.value.destroy(gpa);
2558		}
2559		}
2560		if (mod.main_pkg.table.fetchRemove("root")) \|kv\| {
2561		gpa.free(kv.key);
2562		}
2563		if (mod.root_pkg != mod.main_pkg) {
2564		mod.root_pkg.destroy(gpa);
2565		}
2566
2567	2538	mod.compile_log_text.deinit(gpa);
2568	2539
2569	2540	mod.zig_cache_artifact_directory.handle.close();
...	...	@@ -2710,18 +2681,19 @@ pub fn astGenFile(mod: Module, file: File) !void {
2710	2681	const gpa = mod.gpa;
2711	2682
2712	2683	// In any case we need to examine the stat of the file to determine the course of action.
2713		var source_file = try file.pkg.root_src_directory.handle.openFile(file.sub_file_path, .{});
	2684	var source_file = try file.mod.root.openFile(file.sub_file_path, .{});
2714	2685	defer source_file.close();
2715	2686
2716	2687	const stat = try source_file.stat();
2717	2688
2718		const want_local_cache = file.pkg == mod.main_pkg;
	2689	const want_local_cache = file.mod == mod.main_mod;
2719	2690	const digest = hash: {
2720	2691	var path_hash: Cache.HashHelper = .{};
2721	2692	path_hash.addBytes(build_options.version);
2722	2693	path_hash.add(builtin.zig_backend);
2723	2694	if (!want_local_cache) {
2724		path_hash.addOptionalBytes(file.pkg.root_src_directory.path);
	2695	path_hash.addOptionalBytes(file.mod.root.root_dir.path);
	2696	path_hash.addBytes(file.mod.root.sub_path);
2725	2697	}
2726	2698	path_hash.addBytes(file.sub_file_path);
2727	2699	break :hash path_hash.final();
...	...	@@ -2946,10 +2918,8 @@ pub fn astGenFile(mod: Module, file: File) !void {
2946	2918	},
2947	2919	};
2948	2920	cache_file.writevAll(&iovecs) catch \|err\| {
2949		const pkg_path = file.pkg.root_src_directory.path orelse ".";
2950		const cache_path = cache_directory.path orelse ".";
2951		log.warn("unable to write cached ZIR code for {s}/{s} to {s}/{s}: {s}", .{
2952		pkg_path, file.sub_file_path, cache_path, &digest, @errorName(err),
	2921	log.warn("unable to write cached ZIR code for {}{s} to {}{s}: {s}", .{
	2922	file.mod.root, file.sub_file_path, cache_directory, &digest, @errorName(err),
2953	2923	});
2954	2924	};
2955	2925
...	...	@@ -3154,37 +3124,27 @@ pub fn populateBuiltinFile(mod: *Module) !void {
3154	3124	defer tracy.end();
3155	3125
3156	3126	const comp = mod.comp;
3157		const pkg_and_file = blk: {
	3127	const builtin_mod, const file = blk: {
3158	3128	comp.mutex.lock();
3159	3129	defer comp.mutex.unlock();
3160	3130
3161		const builtin_pkg = mod.main_pkg.table.get("builtin").?;
3162		const result = try mod.importPkg(builtin_pkg);
3163		break :blk .{
3164		.file = result.file,
3165		.pkg = builtin_pkg,
3166		};
	3131	const builtin_mod = mod.main_mod.deps.get("builtin").?;
	3132	const result = try mod.importPkg(builtin_mod);
	3133	break :blk .{ builtin_mod, result.file };
3167	3134	};
3168		const file = pkg_and_file.file;
3169		const builtin_pkg = pkg_and_file.pkg;
3170	3135	const gpa = mod.gpa;
3171	3136	file.source = try comp.generateBuiltinZigSource(gpa);
3172	3137	file.source_loaded = true;
3173	3138
3174		if (builtin_pkg.root_src_directory.handle.statFile(builtin_pkg.root_src_path)) \|stat\| {
	3139	if (builtin_mod.root.statFile(builtin_mod.root_src_path)) \|stat\| {
3175	3140	if (stat.size != file.source.len) {
3176		const full_path = try builtin_pkg.root_src_directory.join(gpa, &.{
3177		builtin_pkg.root_src_path,
3178		});
3179		defer gpa.free(full_path);
3180
3181	3141	log.warn(
3182		"the cached file '{s}' had the wrong size. Expected {d}, found {d}. " ++
	3142	"the cached file '{}{s}' had the wrong size. Expected {d}, found {d}. " ++
3183	3143	"Overwriting with correct file contents now",
3184		.{ full_path, file.source.len, stat.size },
	3144	.{ builtin_mod.root, builtin_mod.root_src_path, file.source.len, stat.size },
3185	3145	);
3186	3146
3187		try writeBuiltinFile(file, builtin_pkg);
	3147	try writeBuiltinFile(file, builtin_mod);
3188	3148	} else {
3189	3149	file.stat = .{
3190	3150	.size = stat.size,
...	...	@@ -3198,7 +3158,7 @@ pub fn populateBuiltinFile(mod: *Module) !void {
3198	3158	error.PipeBusy => unreachable, // it's not a pipe
3199	3159	error.WouldBlock => unreachable, // not asking for non-blocking I/O
3200	3160
3201		error.FileNotFound => try writeBuiltinFile(file, builtin_pkg),
	3161	error.FileNotFound => try writeBuiltinFile(file, builtin_mod),
3202	3162
3203	3163	else => \|e\| return e,
3204	3164	}
...	...	@@ -3212,8 +3172,8 @@ pub fn populateBuiltinFile(mod: *Module) !void {
3212	3172	file.status = .success_zir;
3213	3173	}
3214	3174
3215		fn writeBuiltinFile(file: File, builtin_pkg: Package) !void {
3216		var af = try builtin_pkg.root_src_directory.handle.atomicFile(builtin_pkg.root_src_path, .{});
	3175	fn writeBuiltinFile(file: File, builtin_mod: Package.Module) !void {
	3176	var af = try builtin_mod.root.atomicFile(builtin_mod.root_src_path, .{});
3217	3177	defer af.deinit();
3218	3178	try af.file.writeAll(file.source);
3219	3179	try af.finish();
...	...	@@ -3609,7 +3569,8 @@ pub fn updateEmbedFile(mod: Module, embed_file: EmbedFile) SemaError!void {
3609	3569	}
3610	3570	}
3611	3571
3612		pub fn semaPkg(mod: Module, pkg: Package) !void {
	3572	/// https://github.com/ziglang/zig/issues/14307
	3573	pub fn semaPkg(mod: Module, pkg: Package.Module) !void {
3613	3574	const file = (try mod.importPkg(pkg)).file;
3614	3575	return mod.semaFile(file);
3615	3576	}
...	...	@@ -3711,13 +3672,11 @@ pub fn semaFile(mod: Module, file: File) SemaError!void {
3711	3672	return error.AnalysisFail;
3712	3673	};
3713	3674
3714		const resolved_path = std.fs.path.resolve(
3715		gpa,
3716		if (file.pkg.root_src_directory.path) \|pkg_path\|
3717		&[_][]const u8{ pkg_path, file.sub_file_path }
3718		else
3719		&[_][]const u8{file.sub_file_path},
3720		) catch \|err\| {
	3675	const resolved_path = std.fs.path.resolve(gpa, &.{
	3676	file.mod.root.root_dir.path orelse ".",
	3677	file.mod.root.sub_path,
	3678	file.sub_file_path,
	3679	}) catch \|err\| {
3721	3680	try reportRetryableFileError(mod, file, "unable to resolve path: {s}", .{@errorName(err)});
3722	3681	return error.AnalysisFail;
3723	3682	};
...	...	@@ -3748,8 +3707,8 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !bool {
3748	3707
3749	3708	// TODO: figure out how this works under incremental changes to builtin.zig!
3750	3709	const builtin_type_target_index: InternPool.Index = blk: {
3751		const std_mod = mod.main_pkg.table.get("std").?;
3752		if (decl.getFileScope(mod).pkg != std_mod) break :blk .none;
	3710	const std_mod = mod.main_mod.deps.get("std").?;
	3711	if (decl.getFileScope(mod).mod != std_mod) break :blk .none;
3753	3712	// We're in the std module.
3754	3713	const std_file = (try mod.importPkg(std_mod)).file;
3755	3714	const std_decl = mod.declPtr(std_file.root_decl.unwrap().?);
...	...	@@ -4042,14 +4001,17 @@ pub const ImportFileResult = struct {
4042	4001	is_pkg: bool,
4043	4002	};
4044	4003
4045		pub fn importPkg(mod: Module, pkg: Package) !ImportFileResult {
	4004	/// https://github.com/ziglang/zig/issues/14307
	4005	pub fn importPkg(mod: Module, pkg: Package.Module) !ImportFileResult {
4046	4006	const gpa = mod.gpa;
4047	4007
4048	4008	// The resolved path is used as the key in the import table, to detect if
4049	4009	// an import refers to the same as another, despite different relative paths
4050	4010	// or differently mapped package names.
4051		const resolved_path = try std.fs.path.resolve(gpa, &[_][]const u8{
4052		pkg.root_src_directory.path orelse ".", pkg.root_src_path,
	4011	const resolved_path = try std.fs.path.resolve(gpa, &.{
	4012	pkg.root.root_dir.path orelse ".",
	4013	pkg.root.sub_path,
	4014	pkg.root_src_path,
4053	4015	});
4054	4016	var keep_resolved_path = false;
4055	4017	defer if (!keep_resolved_path) gpa.free(resolved_path);
...	...	@@ -4083,7 +4045,7 @@ pub fn importPkg(mod: Module, pkg: Package) !ImportFileResult {
4083	4045	.tree = undefined,
4084	4046	.zir = undefined,
4085	4047	.status = .never_loaded,
4086		.pkg = pkg,
	4048	.mod = pkg,
4087	4049	.root_decl = .none,
4088	4050	};
4089	4051	try new_file.addReference(mod.*, .{ .root = pkg });
...	...	@@ -4100,29 +4062,33 @@ pub fn importFile(
4100	4062	import_string: []const u8,
4101	4063	) !ImportFileResult {
4102	4064	if (std.mem.eql(u8, import_string, "std")) {
4103		return mod.importPkg(mod.main_pkg.table.get("std").?);
	4065	return mod.importPkg(mod.main_mod.deps.get("std").?);
4104	4066	}
4105	4067	if (std.mem.eql(u8, import_string, "builtin")) {
4106		return mod.importPkg(mod.main_pkg.table.get("builtin").?);
	4068	return mod.importPkg(mod.main_mod.deps.get("builtin").?);
4107	4069	}
4108	4070	if (std.mem.eql(u8, import_string, "root")) {
4109		return mod.importPkg(mod.root_pkg);
	4071	return mod.importPkg(mod.root_mod);
4110	4072	}
4111		if (cur_file.pkg.table.get(import_string)) \|pkg\| {
	4073	if (cur_file.mod.deps.get(import_string)) \|pkg\| {
4112	4074	return mod.importPkg(pkg);
4113	4075	}
4114	4076	if (!mem.endsWith(u8, import_string, ".zig")) {
4115		return error.PackageNotFound;
	4077	return error.ModuleNotFound;
4116	4078	}
4117	4079	const gpa = mod.gpa;
4118	4080
4119	4081	// The resolved path is used as the key in the import table, to detect if
4120	4082	// an import refers to the same as another, despite different relative paths
4121	4083	// or differently mapped package names.
4122		const cur_pkg_dir_path = cur_file.pkg.root_src_directory.path orelse ".";
4123		const resolved_path = try std.fs.path.resolve(gpa, &[_][]const u8{
4124		cur_pkg_dir_path, cur_file.sub_file_path, "..", import_string,
	4084	const resolved_path = try std.fs.path.resolve(gpa, &.{
	4085	cur_file.mod.root.root_dir.path orelse ".",
	4086	cur_file.mod.root.sub_path,
	4087	cur_file.sub_file_path,
	4088	"..",
	4089	import_string,
4125	4090	});
	4091
4126	4092	var keep_resolved_path = false;
4127	4093	defer if (!keep_resolved_path) gpa.free(resolved_path);
4128	4094
...	...	@@ -4137,7 +4103,10 @@ pub fn importFile(
4137	4103	const new_file = try gpa.create(File);
4138	4104	errdefer gpa.destroy(new_file);
4139	4105
4140		const resolved_root_path = try std.fs.path.resolve(gpa, &[_][]const u8{cur_pkg_dir_path});
	4106	const resolved_root_path = try std.fs.path.resolve(gpa, &.{
	4107	cur_file.mod.root.root_dir.path orelse ".",
	4108	cur_file.mod.root.sub_path,
	4109	});
4141	4110	defer gpa.free(resolved_root_path);
4142	4111
4143	4112	const sub_file_path = p: {
...	...	@@ -4151,7 +4120,7 @@ pub fn importFile(
4151	4120	{
4152	4121	break :p try gpa.dupe(u8, resolved_path);
4153	4122	}
4154		return error.ImportOutsidePkgPath;
	4123	return error.ImportOutsideModulePath;
4155	4124	};
4156	4125	errdefer gpa.free(sub_file_path);
4157	4126
...	...	@@ -4171,7 +4140,7 @@ pub fn importFile(
4171	4140	.tree = undefined,
4172	4141	.zir = undefined,
4173	4142	.status = .never_loaded,
4174		.pkg = cur_file.pkg,
	4143	.mod = cur_file.mod,
4175	4144	.root_decl = .none,
4176	4145	};
4177	4146	return ImportFileResult{
...	...	@@ -4184,9 +4153,11 @@ pub fn importFile(
4184	4153	pub fn embedFile(mod: Module, cur_file: File, import_string: []const u8) !*EmbedFile {
4185	4154	const gpa = mod.gpa;
4186	4155
4187		if (cur_file.pkg.table.get(import_string)) \|pkg\| {
4188		const resolved_path = try std.fs.path.resolve(gpa, &[_][]const u8{
4189		pkg.root_src_directory.path orelse ".", pkg.root_src_path,
	4156	if (cur_file.mod.deps.get(import_string)) \|pkg\| {
	4157	const resolved_path = try std.fs.path.resolve(gpa, &.{
	4158	pkg.root.root_dir.path orelse ".",
	4159	pkg.root.sub_path,
	4160	pkg.root_src_path,
4190	4161	});
4191	4162	var keep_resolved_path = false;
4192	4163	defer if (!keep_resolved_path) gpa.free(resolved_path);
...	...	@@ -4203,10 +4174,14 @@ pub fn embedFile(mod: Module, cur_file: File, import_string: []const u8) !*Emb
4203	4174
4204	4175	// The resolved path is used as the key in the table, to detect if a file
4205	4176	// refers to the same as another, despite different relative paths.
4206		const cur_pkg_dir_path = cur_file.pkg.root_src_directory.path orelse ".";
4207		const resolved_path = try std.fs.path.resolve(gpa, &[_][]const u8{
4208		cur_pkg_dir_path, cur_file.sub_file_path, "..", import_string,
	4177	const resolved_path = try std.fs.path.resolve(gpa, &.{
	4178	cur_file.mod.root.root_dir.path orelse ".",
	4179	cur_file.mod.root.sub_path,
	4180	cur_file.sub_file_path,
	4181	"..",
	4182	import_string,
4209	4183	});
	4184
4210	4185	var keep_resolved_path = false;
4211	4186	defer if (!keep_resolved_path) gpa.free(resolved_path);
4212	4187
...	...	@@ -4214,7 +4189,10 @@ pub fn embedFile(mod: Module, cur_file: File, import_string: []const u8) !*Emb
4214	4189	errdefer assert(mod.embed_table.remove(resolved_path));
4215	4190	if (gop.found_existing) return gop.value_ptr.*;
4216	4191
4217		const resolved_root_path = try std.fs.path.resolve(gpa, &[_][]const u8{cur_pkg_dir_path});
	4192	const resolved_root_path = try std.fs.path.resolve(gpa, &.{
	4193	cur_file.mod.root.root_dir.path orelse ".",
	4194	cur_file.mod.root.sub_path,
	4195	});
4218	4196	defer gpa.free(resolved_root_path);
4219	4197
4220	4198	const sub_file_path = p: {
...	...	@@ -4228,16 +4206,17 @@ pub fn embedFile(mod: Module, cur_file: File, import_string: []const u8) !*Emb
4228	4206	{
4229	4207	break :p try gpa.dupe(u8, resolved_path);
4230	4208	}
4231		return error.ImportOutsidePkgPath;
	4209	return error.ImportOutsideModulePath;
4232	4210	};
4233	4211	errdefer gpa.free(sub_file_path);
4234	4212
4235		return newEmbedFile(mod, cur_file.pkg, sub_file_path, resolved_path, &keep_resolved_path, gop);
	4213	return newEmbedFile(mod, cur_file.mod, sub_file_path, resolved_path, &keep_resolved_path, gop);
4236	4214	}
4237	4215
	4216	/// https://github.com/ziglang/zig/issues/14307
4238	4217	fn newEmbedFile(
4239	4218	mod: *Module,
4240		pkg: *Package,
	4219	pkg: *Package.Module,
4241	4220	sub_file_path: []const u8,
4242	4221	resolved_path: []const u8,
4243	4222	keep_resolved_path: *bool,
...	...	@@ -4248,7 +4227,7 @@ fn newEmbedFile(
4248	4227	const new_file = try gpa.create(EmbedFile);
4249	4228	errdefer gpa.destroy(new_file);
4250	4229
4251		var file = try pkg.root_src_directory.handle.openFile(sub_file_path, .{});
	4230	var file = try pkg.root.openFile(sub_file_path, .{});
4252	4231	defer file.close();
4253	4232
4254	4233	const actual_stat = try file.stat();
...	...	@@ -4275,14 +4254,14 @@ fn newEmbedFile(
4275	4254	.sub_file_path = sub_file_path,
4276	4255	.bytes = bytes,
4277	4256	.stat = stat,
4278		.pkg = pkg,
	4257	.mod = pkg,
4279	4258	.owner_decl = undefined, // Set by Sema immediately after this function returns.
4280	4259	};
4281	4260	return new_file;
4282	4261	}
4283	4262
4284	4263	pub fn detectEmbedFileUpdate(mod: Module, embed_file: EmbedFile) !void {
4285		var file = try embed_file.pkg.root_src_directory.handle.openFile(embed_file.sub_file_path, .{});
	4264	var file = try embed_file.mod.root.openFile(embed_file.sub_file_path, .{});
4286	4265	defer file.close();
4287	4266
4288	4267	const stat = try file.stat();
...	...	@@ -4455,21 +4434,21 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) Allocator.Err
4455	4434	gop.key_ptr.* = new_decl_index;
4456	4435	// Exported decls, comptime decls, usingnamespace decls, and
4457	4436	// test decls if in test mode, get analyzed.
4458		const decl_pkg = namespace.file_scope.pkg;
	4437	const decl_mod = namespace.file_scope.mod;
4459	4438	const want_analysis = is_exported or switch (decl_name_index) {
4460	4439	0 => true, // comptime or usingnamespace decl
4461	4440	1 => blk: {
4462	4441	// test decl with no name. Skip the part where we check against
4463	4442	// the test name filter.
4464	4443	if (!comp.bin_file.options.is_test) break :blk false;
4465		if (decl_pkg != mod.main_pkg) break :blk false;
	4444	if (decl_mod != mod.main_mod) break :blk false;
4466	4445	try mod.test_functions.put(gpa, new_decl_index, {});
4467	4446	break :blk true;
4468	4447	},
4469	4448	else => blk: {
4470	4449	if (!is_named_test) break :blk false;
4471	4450	if (!comp.bin_file.options.is_test) break :blk false;
4472		if (decl_pkg != mod.main_pkg) break :blk false;
	4451	if (decl_mod != mod.main_mod) break :blk false;
4473	4452	if (comp.test_filter) \|test_filter\| {
4474	4453	if (mem.indexOf(u8, ip.stringToSlice(decl_name), test_filter) == null) {
4475	4454	break :blk false;
...	...	@@ -5596,8 +5575,8 @@ pub fn populateTestFunctions(
5596	5575	) !void {
5597	5576	const gpa = mod.gpa;
5598	5577	const ip = &mod.intern_pool;
5599		const builtin_pkg = mod.main_pkg.table.get("builtin").?;
5600		const builtin_file = (mod.importPkg(builtin_pkg) catch unreachable).file;
	5578	const builtin_mod = mod.main_mod.deps.get("builtin").?;
	5579	const builtin_file = (mod.importPkg(builtin_mod) catch unreachable).file;
5601	5580	const root_decl = mod.declPtr(builtin_file.root_decl.unwrap().?);
5602	5581	const builtin_namespace = mod.namespacePtr(root_decl.src_namespace);
5603	5582	const test_functions_str = try ip.getOrPutString(gpa, "test_functions");

src/Package.zig+124-1320

...	...	@@ -1,1349 +1,153 @@
1		const Package = @This();
2
3		const builtin = @import("builtin");
4		const std = @import("std");
5		const fs = std.fs;
6		const mem = std.mem;
7		const Allocator = mem.Allocator;
8		const ascii = std.ascii;
9		const assert = std.debug.assert;
10		const log = std.log.scoped(.package);
11		const main = @import("main.zig");
12		const ThreadPool = std.Thread.Pool;
13
14		const Compilation = @import("Compilation.zig");
15		const Module = @import("Module.zig");
16		const Cache = std.Build.Cache;
17		const build_options = @import("build_options");
18		const git = @import("git.zig");
19		const computePackageHash = @import("Package/hash.zig").compute;
20
21		pub const Manifest = @import("Manifest.zig");
22		pub const Table = std.StringHashMapUnmanaged(*Package);
23
24		root_src_directory: Compilation.Directory,
25		/// Relative to `root_src_directory`. May contain path separators.
26		root_src_path: []const u8,
27		/// The dependency table of this module. Shared dependencies such as 'std', 'builtin', and 'root'
28		/// are not specified in every dependency table, but instead only in the table of `main_pkg`.
29		/// `Module.importFile` is responsible for detecting these names and using the correct package.
30		table: Table = .{},
31		/// Whether to free `root_src_directory` on `destroy`.
32		root_src_directory_owned: bool = false,
33
34		/// Allocate a Package. No references to the slices passed are kept.
35		pub fn create(
36		gpa: Allocator,
37		/// Null indicates the current working directory
38		root_src_dir_path: ?[]const u8,
39		/// Relative to root_src_dir_path
40		root_src_path: []const u8,
41		) !*Package {
42		const ptr = try gpa.create(Package);
43		errdefer gpa.destroy(ptr);
44
45		const owned_dir_path = if (root_src_dir_path) \|p\| try gpa.dupe(u8, p) else null;
46		errdefer if (owned_dir_path) \|p\| gpa.free(p);
47
48		const owned_src_path = try gpa.dupe(u8, root_src_path);
49		errdefer gpa.free(owned_src_path);
50
51		ptr.* = .{
52		.root_src_directory = .{
53		.path = owned_dir_path,
54		.handle = if (owned_dir_path) \|p\| try fs.cwd().openDir(p, .{}) else fs.cwd(),
55		},
56		.root_src_path = owned_src_path,
57		.root_src_directory_owned = true,
58		};
59
60		return ptr;
61		}
62
63		pub fn createWithDir(
64		gpa: Allocator,
65		directory: Compilation.Directory,
66		/// Relative to `directory`. If null, means `directory` is the root src dir
67		/// and is owned externally.
68		root_src_dir_path: ?[]const u8,
69		/// Relative to root_src_dir_path
70		root_src_path: []const u8,
71		) !*Package {
72		const ptr = try gpa.create(Package);
73		errdefer gpa.destroy(ptr);
74
75		const owned_src_path = try gpa.dupe(u8, root_src_path);
76		errdefer gpa.free(owned_src_path);
77
78		if (root_src_dir_path) \|p\| {
79		const owned_dir_path = try directory.join(gpa, &[1][]const u8{p});
80		errdefer gpa.free(owned_dir_path);
81
82		ptr.* = .{
83		.root_src_directory = .{
84		.path = owned_dir_path,
85		.handle = try directory.handle.openDir(p, .{}),
86		},
87		.root_src_directory_owned = true,
88		.root_src_path = owned_src_path,
89		};
90		} else {
91		ptr.* = .{
92		.root_src_directory = directory,
93		.root_src_directory_owned = false,
94		.root_src_path = owned_src_path,
	1	pub const Module = @import("Package/Module.zig");
	2	pub const Fetch = @import("Package/Fetch.zig");
	3	pub const build_zig_basename = "build.zig";
	4	pub const Manifest = @import("Package/Manifest.zig");
	5
	6	pub const Path = struct {
	7	root_dir: Cache.Directory,
	8	/// The path, relative to the root dir, that this `Path` represents.
	9	/// Empty string means the root_dir is the path.
	10	sub_path: []const u8 = "",
	11
	12	pub fn clone(p: Path, arena: Allocator) Allocator.Error!Path {
	13	return .{
	14	.root_dir = try p.root_dir.clone(arena),
	15	.sub_path = try arena.dupe(u8, p.sub_path),
95	16	};
96	17	}
97		return ptr;
98		}
99	18
100		/// Free all memory associated with this package. It does not destroy any packages
101		/// inside its table; the caller is responsible for calling destroy() on them.
102		pub fn destroy(pkg: *Package, gpa: Allocator) void {
103		gpa.free(pkg.root_src_path);
104
105		if (pkg.root_src_directory_owned) {
106		// If root_src_directory.path is null then the handle is the cwd()
107		// which shouldn't be closed.
108		if (pkg.root_src_directory.path) \|p\| {
109		gpa.free(p);
110		pkg.root_src_directory.handle.close();
111		}
	19	pub fn cwd() Path {
	20	return .{ .root_dir = Cache.Directory.cwd() };
112	21	}
113	22
114		pkg.deinitTable(gpa);
115		gpa.destroy(pkg);
116		}
117
118		/// Only frees memory associated with the table.
119		pub fn deinitTable(pkg: *Package, gpa: Allocator) void {
120		pkg.table.deinit(gpa);
121		}
122
123		pub fn add(pkg: Package, gpa: Allocator, name: []const u8, package: Package) !void {
124		try pkg.table.ensureUnusedCapacity(gpa, 1);
125		const name_dupe = try gpa.dupe(u8, name);
126		pkg.table.putAssumeCapacityNoClobber(name_dupe, package);
127		}
128
129		/// Compute a readable name for the package. The returned name should be freed from gpa. This
130		/// function is very slow, as it traverses the whole package hierarchy to find a path to this
131		/// package. It should only be used for error output.
132		pub fn getName(target: *const Package, gpa: Allocator, mod: Module) ![]const u8 {
133		// we'll do a breadth-first search from the root module to try and find a short name for this
134		// module, using a DoublyLinkedList of module/parent pairs. note that the "parent" there is
135		// just the first-found shortest path - a module may be children of arbitrarily many other
136		// modules. This path may vary between executions due to hashmap iteration order, but that
137		// doesn't matter too much.
138		var node_arena = std.heap.ArenaAllocator.init(gpa);
139		defer node_arena.deinit();
140		const Parented = struct {
141		parent: ?*const @This(),
142		mod: *const Package,
143		};
144		const Queue = std.DoublyLinkedList(Parented);
145		var to_check: Queue = .{};
146
147		{
148		const new = try node_arena.allocator().create(Queue.Node);
149		new.* = .{ .data = .{ .parent = null, .mod = mod.root_pkg } };
150		to_check.prepend(new);
151		}
152
153		if (mod.main_pkg != mod.root_pkg) {
154		const new = try node_arena.allocator().create(Queue.Node);
155		// TODO: once #12201 is resolved, we may want a way of indicating a different name for this
156		new.* = .{ .data = .{ .parent = null, .mod = mod.main_pkg } };
157		to_check.prepend(new);
158		}
159
160		// set of modules we've already checked to prevent loops
161		var checked = std.AutoHashMap(*const Package, void).init(gpa);
162		defer checked.deinit();
163
164		const linked = while (to_check.pop()) \|node\| {
165		const check = &node.data;
166
167		if (checked.contains(check.mod)) continue;
168		try checked.put(check.mod, {});
169
170		if (check.mod == target) break check;
171
172		var it = check.mod.table.iterator();
173		while (it.next()) \|kv\| {
174		var new = try node_arena.allocator().create(Queue.Node);
175		new.* = .{ .data = .{
176		.parent = check,
177		.mod = kv.value_ptr.*,
178		} };
179		to_check.prepend(new);
180		}
181		} else {
182		// this can happen for e.g. @cImport packages
183		return gpa.dupe(u8, "<unnamed>");
184		};
185
186		// we found a path to the module! unfortunately, we can only traverse up it, so we have to put
187		// all the names into a buffer so we can then print them in order.
188		var names = std.ArrayList([]const u8).init(gpa);
189		defer names.deinit();
190
191		var cur: *const Parented = linked;
192		while (cur.parent) \|parent\| : (cur = parent) {
193		// find cur's name in parent
194		var it = parent.mod.table.iterator();
195		const name = while (it.next()) \|kv\| {
196		if (kv.value_ptr.* == cur.mod) {
197		break kv.key_ptr.*;
198		}
199		} else unreachable;
200		try names.append(name);
201		}
202
203		// finally, print the names into a buffer!
204		var buf = std.ArrayList(u8).init(gpa);
205		defer buf.deinit();
206		try buf.writer().writeAll("root");
207		var i: usize = names.items.len;
208		while (i > 0) {
209		i -= 1;
210		try buf.writer().print(".{s}", .{names.items[i]});
	23	pub fn join(p: Path, arena: Allocator, sub_path: []const u8) Allocator.Error!Path {
	24	if (sub_path.len == 0) return p;
	25	const parts: []const []const u8 =
	26	if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
	27	return .{
	28	.root_dir = p.root_dir,
	29	.sub_path = try fs.path.join(arena, parts),
	30	};
211	31	}
212	32
213		return buf.toOwnedSlice();
214		}
215
216		pub const build_zig_basename = "build.zig";
217
218		/// Fetches a package and all of its dependencies recursively. Writes the
219		/// corresponding datastructures for the build runner into `dependencies_source`.
220		pub fn fetchAndAddDependencies(
221		pkg: *Package,
222		deps_pkg: *Package,
223		arena: Allocator,
224		thread_pool: *ThreadPool,
225		http_client: *std.http.Client,
226		directory: Compilation.Directory,
227		global_cache_directory: Compilation.Directory,
228		local_cache_directory: Compilation.Directory,
229		dependencies_source: *std.ArrayList(u8),
230		error_bundle: *std.zig.ErrorBundle.Wip,
231		all_modules: *AllModules,
232		root_prog_node: *std.Progress.Node,
233		/// null for the root package
234		this_hash: ?[]const u8,
235		) !void {
236		const max_bytes = 10 * 1024 * 1024;
237		const gpa = thread_pool.allocator;
238		const build_zig_zon_bytes = directory.handle.readFileAllocOptions(
239		arena,
240		Manifest.basename,
241		max_bytes,
242		null,
243		1,
244		0,
245		) catch \|err\| switch (err) {
246		error.FileNotFound => {
247		// Handle the same as no dependencies.
248		if (this_hash) \|hash\| {
249		try dependencies_source.writer().print(
250		\\ pub const {} = struct {{
251		\\ pub const build_root = "{}";
252		\\ pub const build_zig = @import("{}");
253		\\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{}};
254		\\ }};
255		\\
256		, .{
257		std.zig.fmtId(hash),
258		std.zig.fmtEscapes(pkg.root_src_directory.path.?),
259		std.zig.fmtEscapes(hash),
260		});
261		} else {
262		try dependencies_source.writer().writeAll(
263		\\pub const packages = struct {};
264		\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
265		\\
266		);
267		}
268		return;
269		},
270		else => \|e\| return e,
271		};
272
273		var ast = try std.zig.Ast.parse(gpa, build_zig_zon_bytes, .zon);
274		defer ast.deinit(gpa);
275
276		if (ast.errors.len > 0) {
277		const file_path = try directory.join(arena, &.{Manifest.basename});
278		try main.putAstErrorsIntoBundle(gpa, ast, file_path, error_bundle);
279		return error.PackageFetchFailed;
	33	pub fn resolvePosix(p: Path, arena: Allocator, sub_path: []const u8) Allocator.Error!Path {
	34	if (sub_path.len == 0) return p;
	35	return .{
	36	.root_dir = p.root_dir,
	37	.sub_path = try fs.path.resolvePosix(arena, &.{ p.sub_path, sub_path }),
	38	};
280	39	}
281	40
282		var manifest = try Manifest.parse(gpa, ast);
283		defer manifest.deinit(gpa);
284
285		if (manifest.errors.len > 0) {
286		const file_path = try directory.join(arena, &.{Manifest.basename});
287		for (manifest.errors) \|msg\| {
288		const str = try error_bundle.addString(msg.msg);
289		try Report.addErrorMessage(&ast, file_path, error_bundle, 0, str, msg.tok, msg.off);
290		}
291		return error.PackageFetchFailed;
	41	pub fn joinString(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 {
	42	const parts: []const []const u8 =
	43	if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
	44	return p.root_dir.join(allocator, parts);
292	45	}
293	46
294		const report: Report = .{
295		.ast = &ast,
296		.directory = directory,
297		.error_bundle = error_bundle,
298		};
299
300		for (manifest.dependencies.values()) \|dep\| {
301		// If the hash is invalid, let errors happen later
302		// We only want to add these for progress reporting
303		const hash = dep.hash orelse continue;
304		if (hash.len != hex_multihash_len) continue;
305		const gop = try all_modules.getOrPut(gpa, hash[0..hex_multihash_len].*);
306		if (!gop.found_existing) gop.value_ptr.* = null;
	47	pub fn joinStringZ(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![:0]u8 {
	48	const parts: []const []const u8 =
	49	if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
	50	return p.root_dir.joinZ(allocator, parts);
307	51	}
308	52
309		root_prog_node.setEstimatedTotalItems(all_modules.count());
310
311		if (this_hash == null) {
312		try dependencies_source.writer().writeAll("pub const packages = struct {\n");
	53	pub fn openFile(
	54	p: Path,
	55	sub_path: []const u8,
	56	flags: fs.File.OpenFlags,
	57	) !fs.File {
	58	var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
	59	const joined_path = if (p.sub_path.len == 0) sub_path else p: {
	60	break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
	61	p.sub_path, sub_path,
	62	}) catch return error.NameTooLong;
	63	};
	64	return p.root_dir.handle.openFile(joined_path, flags);
313	65	}
314	66
315		for (manifest.dependencies.keys(), manifest.dependencies.values()) \|name, *dep\| {
316		var fetch_location = try FetchLocation.init(gpa, dep.*, directory, report);
317		defer fetch_location.deinit(gpa);
318
319		// Directories do not provide a hash in build.zig.zon.
320		// Hash the path to the module rather than its contents.
321		const sub_mod, const found_existing = if (fetch_location == .directory)
322		try getDirectoryModule(gpa, fetch_location, directory, all_modules, dep, report)
323		else
324		try getCachedPackage(
325		gpa,
326		global_cache_directory,
327		dep.*,
328		all_modules,
329		root_prog_node,
330		) orelse .{
331		try fetchAndUnpack(
332		fetch_location,
333		thread_pool,
334		http_client,
335		directory,
336		global_cache_directory,
337		dep.*,
338		report,
339		all_modules,
340		root_prog_node,
341		name,
342		),
343		false,
344		};
345
346		assert(dep.hash != null);
347
348		switch (sub_mod) {
349		.zig_pkg => \|sub_pkg\| {
350		if (!found_existing) {
351		try sub_pkg.fetchAndAddDependencies(
352		deps_pkg,
353		arena,
354		thread_pool,
355		http_client,
356		sub_pkg.root_src_directory,
357		global_cache_directory,
358		local_cache_directory,
359		dependencies_source,
360		error_bundle,
361		all_modules,
362		root_prog_node,
363		dep.hash.?,
364		);
365		}
366
367		try pkg.add(gpa, name, sub_pkg);
368		if (deps_pkg.table.get(dep.hash.?)) \|other_sub\| {
369		// This should be the same package (and hence module) since it's the same hash
370		// TODO: dedup multiple versions of the same package
371		assert(other_sub == sub_pkg);
372		} else {
373		try deps_pkg.add(gpa, dep.hash.?, sub_pkg);
374		}
375		},
376		.non_zig_pkg => \|sub_pkg\| {
377		if (!found_existing) {
378		try dependencies_source.writer().print(
379		\\ pub const {} = struct {{
380		\\ pub const build_root = "{}";
381		\\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{}};
382		\\ }};
383		\\
384		, .{
385		std.zig.fmtId(dep.hash.?),
386		std.zig.fmtEscapes(sub_pkg.root_src_directory.path.?),
387		});
388		}
389		},
390		}
	67	pub fn makeOpenPath(p: Path, sub_path: []const u8, opts: fs.OpenDirOptions) !fs.Dir {
	68	var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
	69	const joined_path = if (p.sub_path.len == 0) sub_path else p: {
	70	break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
	71	p.sub_path, sub_path,
	72	}) catch return error.NameTooLong;
	73	};
	74	return p.root_dir.handle.makeOpenPath(joined_path, opts);
391	75	}
392	76
393		if (this_hash) \|hash\| {
394		try dependencies_source.writer().print(
395		\\ pub const {} = struct {{
396		\\ pub const build_root = "{}";
397		\\ pub const build_zig = @import("{}");
398		\\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{
399		\\
400		, .{
401		std.zig.fmtId(hash),
402		std.zig.fmtEscapes(pkg.root_src_directory.path.?),
403		std.zig.fmtEscapes(hash),
404		});
405		for (manifest.dependencies.keys(), manifest.dependencies.values()) \|name, dep\| {
406		try dependencies_source.writer().print(
407		" .{{ \"{}\", \"{}\" }},\n",
408		.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) },
409		);
410		}
411		try dependencies_source.writer().writeAll(
412		\\ };
413		\\ };
414		\\
415		);
416		} else {
417		try dependencies_source.writer().writeAll(
418		\\};
419		\\
420		\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{
421		\\
422		);
423		for (manifest.dependencies.keys(), manifest.dependencies.values()) \|name, dep\| {
424		try dependencies_source.writer().print(
425		" .{{ \"{}\", \"{}\" }},\n",
426		.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(dep.hash.?) },
427		);
428		}
429		try dependencies_source.writer().writeAll("};\n");
	77	pub fn statFile(p: Path, sub_path: []const u8) !fs.Dir.Stat {
	78	var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
	79	const joined_path = if (p.sub_path.len == 0) sub_path else p: {
	80	break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
	81	p.sub_path, sub_path,
	82	}) catch return error.NameTooLong;
	83	};
	84	return p.root_dir.handle.statFile(joined_path);
	85	}
	86
	87	pub fn atomicFile(
	88	p: Path,
	89	sub_path: []const u8,
	90	options: fs.Dir.AtomicFileOptions,
	91	) !fs.AtomicFile {
	92	var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
	93	const joined_path = if (p.sub_path.len == 0) sub_path else p: {
	94	break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
	95	p.sub_path, sub_path,
	96	}) catch return error.NameTooLong;
	97	};
	98	return p.root_dir.handle.atomicFile(joined_path, options);
430	99	}
431		}
432	100
433		pub fn createFilePkg(
434		gpa: Allocator,
435		cache_directory: Compilation.Directory,
436		basename: []const u8,
437		contents: []const u8,
438		) !*Package {
439		const rand_int = std.crypto.random.int(u64);
440		const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ Manifest.hex64(rand_int);
441		{
442		var tmp_dir = try cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{});
443		defer tmp_dir.close();
444		try tmp_dir.writeFile(basename, contents);
	101	pub fn access(p: Path, sub_path: []const u8, flags: fs.File.OpenFlags) !void {
	102	var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
	103	const joined_path = if (p.sub_path.len == 0) sub_path else p: {
	104	break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
	105	p.sub_path, sub_path,
	106	}) catch return error.NameTooLong;
	107	};
	108	return p.root_dir.handle.access(joined_path, flags);
445	109	}
446	110
447		var hh: Cache.HashHelper = .{};
448		hh.addBytes(build_options.version);
449		hh.addBytes(contents);
450		const hex_digest = hh.final();
451
452		const o_dir_sub_path = "o" ++ fs.path.sep_str ++ hex_digest;
453		try renameTmpIntoCache(cache_directory.handle, tmp_dir_sub_path, o_dir_sub_path);
454
455		return createWithDir(gpa, cache_directory, o_dir_sub_path, basename);
456		}
457
458		pub const Report = struct {
459		ast: ?*const std.zig.Ast,
460		directory: Compilation.Directory,
461		error_bundle: *std.zig.ErrorBundle.Wip,
462
463		fn fail(
464		report: Report,
465		tok: std.zig.Ast.TokenIndex,
	111	pub fn format(
	112	self: Path,
466	113	comptime fmt_string: []const u8,
467		fmt_args: anytype,
468		) error{ PackageFetchFailed, OutOfMemory } {
469		const msg = try report.error_bundle.printString(fmt_string, fmt_args);
470		return failMsg(report, tok, msg);
471		}
472
473		fn failMsg(
474		report: Report,
475		tok: std.zig.Ast.TokenIndex,
476		msg: u32,
477		) error{ PackageFetchFailed, OutOfMemory } {
478		const gpa = report.error_bundle.gpa;
479
480		const file_path = try report.directory.join(gpa, &.{Manifest.basename});
481		defer gpa.free(file_path);
482
483		const eb = report.error_bundle;
484
485		if (report.ast) \|ast\| {
486		try addErrorMessage(ast, file_path, eb, 0, msg, tok, 0);
487		} else {
488		try eb.addRootErrorMessage(.{
489		.msg = msg,
490		.src_loc = .none,
491		.notes_len = 0,
492		});
493		}
494
495		return error.PackageFetchFailed;
496		}
497
498		fn addErrorWithNotes(
499		report: Report,
500		notes_len: u32,
501		msg: Manifest.ErrorMessage,
502		) error{OutOfMemory}!void {
503		const eb = report.error_bundle;
504		const msg_str = try eb.addString(msg.msg);
505		if (report.ast) \|ast\| {
506		const gpa = eb.gpa;
507		const file_path = try report.directory.join(gpa, &.{Manifest.basename});
508		defer gpa.free(file_path);
509		return addErrorMessage(ast, file_path, eb, notes_len, msg_str, msg.tok, msg.off);
510		} else {
511		return eb.addRootErrorMessage(.{
512		.msg = msg_str,
513		.src_loc = .none,
514		.notes_len = notes_len,
515		});
516		}
517		}
518
519		fn addErrorMessage(
520		ast: *const std.zig.Ast,
521		file_path: []const u8,
522		eb: *std.zig.ErrorBundle.Wip,
523		notes_len: u32,
524		msg_str: u32,
525		msg_tok: std.zig.Ast.TokenIndex,
526		msg_off: u32,
527		) error{OutOfMemory}!void {
528		const token_starts = ast.tokens.items(.start);
529		const start_loc = ast.tokenLocation(0, msg_tok);
530
531		try eb.addRootErrorMessage(.{
532		.msg = msg_str,
533		.src_loc = try eb.addSourceLocation(.{
534		.src_path = try eb.addString(file_path),
535		.span_start = token_starts[msg_tok],
536		.span_end = @as(u32, @intCast(token_starts[msg_tok] + ast.tokenSlice(msg_tok).len)),
537		.span_main = token_starts[msg_tok] + msg_off,
538		.line = @intCast(start_loc.line),
539		.column = @as(u32, @intCast(start_loc.column)),
540		.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
541		}),
542		.notes_len = notes_len,
543		});
544		}
545		};
546
547		pub const FetchLocation = union(enum) {
548		/// The relative path to a file or directory.
549		/// This may be a file that requires unpacking (such as a .tar.gz),
550		/// or the path to the root directory of a package.
551		file: []const u8,
552		directory: []const u8,
553		http_request: std.Uri,
554		git_request: std.Uri,
555
556		pub fn init(
557		gpa: Allocator,
558		dep: Manifest.Dependency,
559		root_dir: Compilation.Directory,
560		report: Report,
561		) !FetchLocation {
562		switch (dep.location) {
563		.url => \|url\| {
564		const uri = std.Uri.parse(url) catch \|err\| switch (err) {
565		error.UnexpectedCharacter => return report.fail(dep.location_tok, "failed to parse dependency location as URI", .{}),
566		else => return err,
567		};
568		return initUri(uri, dep.location_tok, report);
569		},
570		.path => \|path\| {
571		if (fs.path.isAbsolute(path)) {
572		return report.fail(dep.location_tok, "absolute paths are not allowed. Use a relative path instead", .{});
573		}
574
575		const is_dir = isDirectory(root_dir, path) catch \|err\| switch (err) {
576		error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{path}),
577		else => return err,
578		};
579
580		return if (is_dir)
581		.{ .directory = try gpa.dupe(u8, path) }
582		else
583		.{ .file = try gpa.dupe(u8, path) };
584		},
585		}
586		}
587
588		pub fn initUri(uri: std.Uri, location_tok: std.zig.Ast.TokenIndex, report: Report) !FetchLocation {
589		if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
590		return report.fail(location_tok, "'file' scheme is not allowed for URLs. Use '.path' instead", .{});
591		} else if (ascii.eqlIgnoreCase(uri.scheme, "http") or ascii.eqlIgnoreCase(uri.scheme, "https")) {
592		return .{ .http_request = uri };
593		} else if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or ascii.eqlIgnoreCase(uri.scheme, "git+https")) {
594		return .{ .git_request = uri };
595		} else {
596		return report.fail(location_tok, "unsupported URL scheme: {s}", .{uri.scheme});
597		}
598		}
599
600		pub fn deinit(f: *FetchLocation, gpa: Allocator) void {
601		switch (f.*) {
602		.file, .directory => \|path\| gpa.free(path),
603		.http_request, .git_request => {},
604		}
605		f.* = undefined;
606		}
607
608		pub fn fetch(
609		f: FetchLocation,
610		gpa: Allocator,
611		root_dir: Compilation.Directory,
612		http_client: *std.http.Client,
613		dep_location_tok: std.zig.Ast.TokenIndex,
614		report: Report,
615		) !ReadableResource {
616		switch (f) {
617		.file => \|file\| {
618		const owned_path = try gpa.dupe(u8, file);
619		errdefer gpa.free(owned_path);
620		return .{
621		.path = owned_path,
622		.resource = .{ .file = try root_dir.handle.openFile(file, .{}) },
623		};
624		},
625		.http_request => \|uri\| {
626		var h = std.http.Headers{ .allocator = gpa };
627		defer h.deinit();
628
629		var req = try http_client.request(.GET, uri, h, .{});
630		errdefer req.deinit();
631
632		try req.start(.{});
633		try req.wait();
634
635		if (req.response.status != .ok) {
636		return report.fail(dep_location_tok, "expected response status '200 OK' got '{} {s}'", .{
637		@intFromEnum(req.response.status),
638		req.response.status.phrase() orelse "",
639		});
640		}
641
642		return .{
643		.path = try gpa.dupe(u8, uri.path),
644		.resource = .{ .http_request = req },
645		};
646		},
647		.git_request => \|uri\| {
648		var transport_uri = uri;
649		transport_uri.scheme = uri.scheme["git+".len..];
650		var redirect_uri: []u8 = undefined;
651		var session: git.Session = .{ .transport = http_client, .uri = transport_uri };
652		session.discoverCapabilities(gpa, &redirect_uri) catch \|e\| switch (e) {
653		error.Redirected => {
654		defer gpa.free(redirect_uri);
655		return report.fail(dep_location_tok, "repository moved to {s}", .{redirect_uri});
656		},
657		else => \|other\| return other,
658		};
659
660		const want_oid = want_oid: {
661		const want_ref = uri.fragment orelse "HEAD";
662		if (git.parseOid(want_ref)) \|oid\| break :want_oid oid else \|_\| {}
663
664		const want_ref_head = try std.fmt.allocPrint(gpa, "refs/heads/{s}", .{want_ref});
665		defer gpa.free(want_ref_head);
666		const want_ref_tag = try std.fmt.allocPrint(gpa, "refs/tags/{s}", .{want_ref});
667		defer gpa.free(want_ref_tag);
668
669		var ref_iterator = try session.listRefs(gpa, .{
670		.ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },
671		.include_peeled = true,
672		});
673		defer ref_iterator.deinit();
674		while (try ref_iterator.next()) \|ref\| {
675		if (mem.eql(u8, ref.name, want_ref) or
676		mem.eql(u8, ref.name, want_ref_head) or
677		mem.eql(u8, ref.name, want_ref_tag))
678		{
679		break :want_oid ref.peeled orelse ref.oid;
680		}
681		}
682		return report.fail(dep_location_tok, "ref not found: {s}", .{want_ref});
683		};
684		if (uri.fragment == null) {
685		const notes_len = 1;
686		try report.addErrorWithNotes(notes_len, .{
687		.tok = dep_location_tok,
688		.off = 0,
689		.msg = "url field is missing an explicit ref",
690		});
691		const eb = report.error_bundle;
692		const notes_start = try eb.reserveNotes(notes_len);
693		eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
694		.msg = try eb.printString("try .url = \"{+/}#{}\",", .{ uri, std.fmt.fmtSliceHexLower(&want_oid) }),
695		}));
696		return error.PackageFetchFailed;
697		}
698
699		var want_oid_buf: [git.fmt_oid_length]u8 = undefined;
700		_ = std.fmt.bufPrint(&want_oid_buf, "{}", .{std.fmt.fmtSliceHexLower(&want_oid)}) catch unreachable;
701		var fetch_stream = try session.fetch(gpa, &.{&want_oid_buf});
702		errdefer fetch_stream.deinit();
703
704		return .{
705		.path = try gpa.dupe(u8, &want_oid_buf),
706		.resource = .{ .git_fetch_stream = fetch_stream },
707		};
708		},
709		.directory => unreachable, // Directories do not require fetching
710		}
711		}
712		};
713
714		pub const ReadableResource = struct {
715		path: []const u8,
716		resource: union(enum) {
717		file: fs.File,
718		http_request: std.http.Client.Request,
719		git_fetch_stream: git.Session.FetchStream,
720		dir: fs.IterableDir,
721		},
722
723		/// Unpack the package into the global cache directory.
724		/// If `ps` does not require unpacking (for example, if it is a directory), then no caching is performed.
725		/// In either case, the hash is computed and returned along with the path to the package.
726		pub fn unpack(
727		rr: *ReadableResource,
728		allocator: Allocator,
729		thread_pool: *ThreadPool,
730		global_cache_directory: Compilation.Directory,
731		dep_location_tok: std.zig.Ast.TokenIndex,
732		report: Report,
733		pkg_prog_node: *std.Progress.Node,
734		) !PackageLocation {
735		switch (rr.resource) {
736		inline .file, .http_request, .git_fetch_stream, .dir => \|*r, tag\| {
737		const s = fs.path.sep_str;
738		const rand_int = std.crypto.random.int(u64);
739		const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);
740
741		const actual_hash = h: {
742		var tmp_directory: Compilation.Directory = d: {
743		const path = try global_cache_directory.join(allocator, &.{tmp_dir_sub_path});
744		errdefer allocator.free(path);
745
746		const iterable_dir = try global_cache_directory.handle.makeOpenPathIterable(tmp_dir_sub_path, .{});
747		errdefer iterable_dir.close();
748
749		break :d .{
750		.path = path,
751		.handle = iterable_dir.dir,
752		};
753		};
754		defer tmp_directory.closeAndFree(allocator);
755
756		if (tag != .dir) {
757		const opt_content_length = try rr.getSize();
758
759		var prog_reader: ProgressReader(@TypeOf(r.reader())) = .{
760		.child_reader = r.reader(),
761		.prog_node = pkg_prog_node,
762		.unit = if (opt_content_length) \|content_length\| unit: {
763		const kib = content_length / 1024;
764		const mib = kib / 1024;
765		if (mib > 0) {
766		pkg_prog_node.setEstimatedTotalItems(@intCast(mib));
767		pkg_prog_node.setUnit("MiB");
768		break :unit .mib;
769		} else {
770		pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib)));
771		pkg_prog_node.setUnit("KiB");
772		break :unit .kib;
773		}
774		} else .any,
775		};
776
777		switch (try rr.getFileType(dep_location_tok, report)) {
778		.tar => try unpackTarball(allocator, prog_reader.reader(), tmp_directory.handle, dep_location_tok, report),
779		.@"tar.gz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, dep_location_tok, report, std.compress.gzip),
780		.@"tar.xz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, dep_location_tok, report, std.compress.xz),
781		.git_pack => try unpackGitPack(allocator, &prog_reader, git.parseOid(rr.path) catch unreachable, tmp_directory.handle, dep_location_tok, report),
782		}
783		} else {
784		// Recursive directory copy.
785		var it = try r.walk(allocator);
786		defer it.deinit();
787		while (try it.next()) \|entry\| {
788		switch (entry.kind) {
789		.directory => try tmp_directory.handle.makePath(entry.path),
790		.file => try r.dir.copyFile(
791		entry.path,
792		tmp_directory.handle,
793		entry.path,
794		.{},
795		),
796		.sym_link => {
797		var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
798		const link_name = try r.dir.readLink(entry.path, &buf);
799		// TODO: if this would create a symlink to outside
800		// the destination directory, fail with an error instead.
801		try tmp_directory.handle.symLink(link_name, entry.path, .{});
802		},
803		else => return error.IllegalFileTypeInPackage,
804		}
805		}
806		}
807
808		break :h try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle });
809		};
810
811		const pkg_dir_sub_path = "p" ++ s ++ Manifest.hexDigest(actual_hash);
812		const unpacked_path = try global_cache_directory.join(allocator, &.{pkg_dir_sub_path});
813		defer allocator.free(unpacked_path);
814
815		const relative_unpacked_path = try fs.path.relative(allocator, global_cache_directory.path.?, unpacked_path);
816		errdefer allocator.free(relative_unpacked_path);
817		try renameTmpIntoCache(global_cache_directory.handle, tmp_dir_sub_path, relative_unpacked_path);
818
819		return .{
820		.hash = actual_hash,
821		.relative_unpacked_path = relative_unpacked_path,
822		};
823		},
824		}
825		}
826
827		const FileType = enum {
828		tar,
829		@"tar.gz",
830		@"tar.xz",
831		git_pack,
832		};
833
834		pub fn getSize(rr: ReadableResource) !?u64 {
835		switch (rr.resource) {
836		.file => \|f\| return (try f.metadata()).size(),
837		// TODO: Handle case of chunked content-length
838		.http_request => \|req\| return req.response.content_length,
839		.git_fetch_stream => \|stream\| return stream.request.response.content_length,
840		.dir => unreachable,
841		}
842		}
843
844		pub fn getFileType(
845		rr: ReadableResource,
846		dep_location_tok: std.zig.Ast.TokenIndex,
847		report: Report,
848		) !FileType {
849		switch (rr.resource) {
850		.file => {
851		return fileTypeFromPath(rr.path) orelse
852		return report.fail(dep_location_tok, "unknown file type", .{});
853		},
854		.http_request => \|req\| {
855		const content_type = req.response.headers.getFirstValue("Content-Type") orelse
856		return report.fail(dep_location_tok, "missing 'Content-Type' header", .{});
857
858		// If the response has a different content type than the URI indicates, override
859		// the previously assumed file type.
860		if (ascii.eqlIgnoreCase(content_type, "application/x-tar")) return .tar;
861
862		return if (ascii.eqlIgnoreCase(content_type, "application/gzip") or
863		ascii.eqlIgnoreCase(content_type, "application/x-gzip") or
864		ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))
865		.@"tar.gz"
866		else if (ascii.eqlIgnoreCase(content_type, "application/x-xz"))
867		.@"tar.xz"
868		else if (ascii.eqlIgnoreCase(content_type, "application/octet-stream")) ty: {
869		// support gitlab tarball urls such as https://gitlab.com/<namespace>/<project>/-/archive/<sha>/<project>-<sha>.tar.gz
870		// whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"'
871		const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse
872		return report.fail(dep_location_tok, "missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});
873		break :ty getAttachmentType(content_disposition) orelse
874		return report.fail(dep_location_tok, "unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});
875		} else return report.fail(dep_location_tok, "unrecognized value for 'Content-Type' header: {s}", .{content_type});
876		},
877		.git_fetch_stream => return .git_pack,
878		.dir => unreachable,
879		}
880		}
881
882		fn fileTypeFromPath(file_path: []const u8) ?FileType {
883		if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar;
884		if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz";
885		if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz";
886		return null;
887		}
888
889		fn getAttachmentType(content_disposition: []const u8) ?FileType {
890		const disposition_type_end = ascii.indexOfIgnoreCase(content_disposition, "attachment;") orelse return null;
891
892		var value_start = ascii.indexOfIgnoreCasePos(content_disposition, disposition_type_end + 1, "filename") orelse return null;
893		value_start += "filename".len;
894		if (content_disposition[value_start] == '*') {
895		value_start += 1;
896		}
897		if (content_disposition[value_start] != '=') return null;
898		value_start += 1;
899
900		var value_end = mem.indexOfPos(u8, content_disposition, value_start, ";") orelse content_disposition.len;
901		if (content_disposition[value_end - 1] == '\"') {
902		value_end -= 1;
903		}
904		return fileTypeFromPath(content_disposition[value_start..value_end]);
905		}
906
907		pub fn deinit(rr: *ReadableResource, gpa: Allocator) void {
908		gpa.free(rr.path);
909		switch (rr.resource) {
910		.file => \|file\| file.close(),
911		.http_request => \|*req\| req.deinit(),
912		.git_fetch_stream => \|*stream\| stream.deinit(),
913		.dir => \|*dir\| dir.close(),
914		}
915		rr.* = undefined;
916		}
917		};
918
919		pub const PackageLocation = struct {
920		/// For packages that require unpacking, this is the hash of the package contents.
921		/// For directories, this is the hash of the absolute file path.
922		hash: [Manifest.Hash.digest_length]u8,
923		relative_unpacked_path: []const u8,
924
925		pub fn deinit(pl: *PackageLocation, allocator: Allocator) void {
926		allocator.free(pl.relative_unpacked_path);
927		pl.* = undefined;
928		}
929		};
930
931		const hex_multihash_len = 2 * Manifest.multihash_len;
932		const MultiHashHexDigest = [hex_multihash_len]u8;
933
934		const DependencyModule = union(enum) {
935		zig_pkg: *Package,
936		non_zig_pkg: *Package,
937		};
938		/// This is to avoid creating multiple modules for the same build.zig file.
939		/// If the value is `null`, the package is a known dependency, but has not yet
940		/// been fetched.
941		pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, ?DependencyModule);
942
943		fn ProgressReader(comptime ReaderType: type) type {
944		return struct {
945		child_reader: ReaderType,
946		bytes_read: u64 = 0,
947		prog_node: *std.Progress.Node,
948		unit: enum {
949		kib,
950		mib,
951		any,
952		},
953
954		pub const Error = ReaderType.Error;
955		pub const Reader = std.io.Reader(*@This(), Error, read);
956
957		pub fn read(self: *@This(), buf: []u8) Error!usize {
958		const amt = try self.child_reader.read(buf);
959		self.bytes_read += amt;
960		const kib = self.bytes_read / 1024;
961		const mib = kib / 1024;
962		switch (self.unit) {
963		.kib => self.prog_node.setCompletedItems(@intCast(kib)),
964		.mib => self.prog_node.setCompletedItems(@intCast(mib)),
965		.any => {
966		if (mib > 0) {
967		self.prog_node.setUnit("MiB");
968		self.prog_node.setCompletedItems(@intCast(mib));
969		} else {
970		self.prog_node.setUnit("KiB");
971		self.prog_node.setCompletedItems(@intCast(kib));
972		}
973		},
	114	options: std.fmt.FormatOptions,
	115	writer: anytype,
	116	) !void {
	117	if (fmt_string.len == 1) {
	118	// Quote-escape the string.
	119	const stringEscape = std.zig.fmt.stringEscape;
	120	const f = switch (fmt_string[0]) {
	121	'q' => "",
	122	'\'' => '\'',
	123	else => @compileError("unsupported format string: " ++ fmt_string),
	124	};
	125	if (self.root_dir.path) \|p\| {
	126	try stringEscape(p, f, options, writer);
	127	if (self.sub_path.len > 0) try writer.writeAll(fs.path.sep_str);
974	128	}
975		self.prog_node.activate();
976		return amt;
977		}
978
979		pub fn reader(self: *@This()) Reader {
980		return .{ .context = self };
981		}
982		};
983		}
984
985		/// Get a cached package if it exists.
986		/// Returns `null` if the package has not been cached
987		/// If the package exists in the cache, returns a pointer to the package and a
988		/// boolean indicating whether this package has already been seen in the build
989		/// (i.e. whether or not its transitive dependencies have been fetched).
990		fn getCachedPackage(
991		gpa: Allocator,
992		global_cache_directory: Compilation.Directory,
993		dep: Manifest.Dependency,
994		all_modules: *AllModules,
995		root_prog_node: *std.Progress.Node,
996		) !?struct { DependencyModule, bool } {
997		const s = fs.path.sep_str;
998		// Check if the expected_hash is already present in the global package
999		// cache, and thereby avoid both fetching and unpacking.
1000		if (dep.hash) \|h\| {
1001		const hex_digest = h[0..hex_multihash_len];
1002		const pkg_dir_sub_path = "p" ++ s ++ hex_digest;
1003
1004		var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch \|err\| switch (err) {
1005		error.FileNotFound => return null,
1006		else => \|e\| return e,
1007		};
1008		errdefer pkg_dir.close();
1009
1010		// The compiler has a rule that a file must not be included in multiple modules,
1011		// so we must detect if a module has been created for this package and reuse it.
1012		const gop = try all_modules.getOrPut(gpa, hex_digest.*);
1013		if (gop.found_existing) {
1014		if (gop.value_ptr.*) \|mod\| {
1015		return .{ mod, true };
1016		}
1017		}
1018
1019		root_prog_node.completeOne();
1020
1021		const is_zig_mod = if (pkg_dir.access(build_zig_basename, .{})) \|_\| true else \|_\| false;
1022		const basename = if (is_zig_mod) build_zig_basename else "";
1023		const pkg = try createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, basename);
1024
1025		const module: DependencyModule = if (is_zig_mod)
1026		.{ .zig_pkg = pkg }
1027		else
1028		.{ .non_zig_pkg = pkg };
1029
1030		try all_modules.put(gpa, hex_digest.*, module);
1031		return .{ module, false };
1032		}
1033
1034		return null;
1035		}
1036
1037		fn getDirectoryModule(
1038		gpa: Allocator,
1039		fetch_location: FetchLocation,
1040		directory: Compilation.Directory,
1041		all_modules: *AllModules,
1042		dep: *Manifest.Dependency,
1043		report: Report,
1044		) !struct { DependencyModule, bool } {
1045		assert(fetch_location == .directory);
1046
1047		if (dep.hash != null) {
1048		return report.fail(dep.hash_tok, "hash not allowed for directory package", .{});
1049		}
1050
1051		const hash = try computePathHash(gpa, directory, fetch_location.directory);
1052		const hex_digest = Manifest.hexDigest(hash);
1053		dep.hash = try gpa.dupe(u8, &hex_digest);
1054
1055		// There is no fixed location to check for directory modules.
1056		// Instead, check whether it is already listed in all_modules.
1057		if (all_modules.get(hex_digest)) \|mod\| return .{ mod.?, true };
1058
1059		var pkg_dir = directory.handle.openDir(fetch_location.directory, .{}) catch \|err\| switch (err) {
1060		error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{fetch_location.directory}),
1061		else => \|e\| return e,
1062		};
1063		defer pkg_dir.close();
1064
1065		const is_zig_mod = if (pkg_dir.access(build_zig_basename, .{})) \|_\| true else \|_\| false;
1066		const basename = if (is_zig_mod) build_zig_basename else "";
1067
1068		const pkg = try createWithDir(gpa, directory, fetch_location.directory, basename);
1069		const module: DependencyModule = if (is_zig_mod)
1070		.{ .zig_pkg = pkg }
1071		else
1072		.{ .non_zig_pkg = pkg };
1073
1074		try all_modules.put(gpa, hex_digest, module);
1075		return .{ module, false };
1076		}
1077
1078		fn fetchAndUnpack(
1079		fetch_location: FetchLocation,
1080		thread_pool: *ThreadPool,
1081		http_client: *std.http.Client,
1082		directory: Compilation.Directory,
1083		global_cache_directory: Compilation.Directory,
1084		dep: Manifest.Dependency,
1085		report: Report,
1086		all_modules: *AllModules,
1087		root_prog_node: *std.Progress.Node,
1088		/// This does not have to be any form of canonical or fully-qualified name: it
1089		/// is only intended to be human-readable for progress reporting.
1090		name_for_prog: []const u8,
1091		) !DependencyModule {
1092		assert(fetch_location != .directory);
1093
1094		const gpa = http_client.allocator;
1095
1096		var pkg_prog_node = root_prog_node.start(name_for_prog, 0);
1097		defer pkg_prog_node.end();
1098		pkg_prog_node.activate();
1099
1100		var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep.location_tok, report);
1101		defer readable_resource.deinit(gpa);
1102
1103		var package_location = try readable_resource.unpack(
1104		gpa,
1105		thread_pool,
1106		global_cache_directory,
1107		dep.location_tok,
1108		report,
1109		&pkg_prog_node,
1110		);
1111		defer package_location.deinit(gpa);
1112
1113		const actual_hex = Manifest.hexDigest(package_location.hash);
1114		if (dep.hash) \|h\| {
1115		if (!mem.eql(u8, h, &actual_hex)) {
1116		return report.fail(dep.hash_tok, "hash mismatch: expected: {s}, found: {s}", .{
1117		h, actual_hex,
1118		});
1119		}
1120		} else {
1121		const notes_len = 1;
1122		try report.addErrorWithNotes(notes_len, .{
1123		.tok = dep.location_tok,
1124		.off = 0,
1125		.msg = "dependency is missing hash field",
1126		});
1127		const eb = report.error_bundle;
1128		const notes_start = try eb.reserveNotes(notes_len);
1129		eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
1130		.msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),
1131		}));
1132		return error.PackageFetchFailed;
1133		}
1134
1135		const build_zig_path = try fs.path.join(gpa, &.{ package_location.relative_unpacked_path, build_zig_basename });
1136		defer gpa.free(build_zig_path);
1137
1138		const is_zig_mod = if (global_cache_directory.handle.access(build_zig_path, .{})) \|_\| true else \|_\| false;
1139		const basename = if (is_zig_mod) build_zig_basename else "";
1140		const pkg = try createWithDir(gpa, global_cache_directory, package_location.relative_unpacked_path, basename);
1141		const module: DependencyModule = if (is_zig_mod)
1142		.{ .zig_pkg = pkg }
1143		else
1144		.{ .non_zig_pkg = pkg };
1145
1146		try all_modules.put(gpa, actual_hex, module);
1147		return module;
1148		}
1149
1150		fn unpackTarballCompressed(
1151		gpa: Allocator,
1152		reader: anytype,
1153		out_dir: fs.Dir,
1154		dep_location_tok: std.zig.Ast.TokenIndex,
1155		report: Report,
1156		comptime Compression: type,
1157		) !void {
1158		var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1159
1160		var decompress = try Compression.decompress(gpa, br.reader());
1161		defer decompress.deinit();
1162
1163		return unpackTarball(gpa, decompress.reader(), out_dir, dep_location_tok, report);
1164		}
1165
1166		fn unpackTarball(
1167		gpa: Allocator,
1168		reader: anytype,
1169		out_dir: fs.Dir,
1170		dep_location_tok: std.zig.Ast.TokenIndex,
1171		report: Report,
1172		) !void {
1173		var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = gpa };
1174		defer diagnostics.deinit();
1175
1176		try std.tar.pipeToFileSystem(out_dir, reader, .{
1177		.diagnostics = &diagnostics,
1178		.strip_components = 1,
1179		// TODO: we would like to set this to executable_bit_only, but two
1180		// things need to happen before that:
1181		// 1. the tar implementation needs to support it
1182		// 2. the hashing algorithm here needs to support detecting the is_executable
1183		// bit on Windows from the ACLs (see the isExecutable function).
1184		.mode_mode = .ignore,
1185		});
1186
1187		if (diagnostics.errors.items.len > 0) {
1188		const notes_len: u32 = @intCast(diagnostics.errors.items.len);
1189		try report.addErrorWithNotes(notes_len, .{
1190		.tok = dep_location_tok,
1191		.off = 0,
1192		.msg = "unable to unpack tarball",
1193		});
1194		const eb = report.error_bundle;
1195		const notes_start = try eb.reserveNotes(notes_len);
1196		for (diagnostics.errors.items, notes_start..) \|item, note_i\| {
1197		switch (item) {
1198		.unable_to_create_sym_link => \|info\| {
1199		eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
1200		.msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{
1201		info.file_name, info.link_name, @errorName(info.code),
1202		}),
1203		}));
1204		},
1205		.unsupported_file_type => \|info\| {
1206		eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
1207		.msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{
1208		info.file_name, @intFromEnum(info.file_type),
1209		}),
1210		}));
1211		},
	129	if (self.sub_path.len > 0) {
	130	try stringEscape(self.sub_path, f, options, writer);
1212	131	}
	132	return;
1213	133	}
1214		return error.InvalidTarball;
1215		}
1216		}
1217
1218		fn unpackGitPack(
1219		gpa: Allocator,
1220		reader: anytype,
1221		want_oid: git.Oid,
1222		out_dir: fs.Dir,
1223		dep_location_tok: std.zig.Ast.TokenIndex,
1224		report: Report,
1225		) !void {
1226		// The .git directory is used to store the packfile and associated index, but
1227		// we do not attempt to replicate the exact structure of a real .git
1228		// directory, since that isn't relevant for fetching a package.
1229		{
1230		var pack_dir = try out_dir.makeOpenPath(".git", .{});
1231		defer pack_dir.close();
1232		var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });
1233		defer pack_file.close();
1234		var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1235		try fifo.pump(reader.reader(), pack_file.writer());
1236		try pack_file.sync();
1237
1238		var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });
1239		defer index_file.close();
1240		{
1241		var index_prog_node = reader.prog_node.start("Index pack", 0);
1242		defer index_prog_node.end();
1243		index_prog_node.activate();
1244		var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
1245		try git.indexPack(gpa, pack_file, index_buffered_writer.writer());
1246		try index_buffered_writer.flush();
1247		try index_file.sync();
	134	if (fmt_string.len > 0)
	135	std.fmt.invalidFmtError(fmt_string, self);
	136	if (self.root_dir.path) \|p\| {
	137	try writer.writeAll(p);
	138	try writer.writeAll(fs.path.sep_str);
1248	139	}
1249
1250		{
1251		var checkout_prog_node = reader.prog_node.start("Checkout", 0);
1252		defer checkout_prog_node.end();
1253		checkout_prog_node.activate();
1254		var repository = try git.Repository.init(gpa, pack_file, index_file);
1255		defer repository.deinit();
1256		var diagnostics: git.Diagnostics = .{ .allocator = gpa };
1257		defer diagnostics.deinit();
1258		try repository.checkout(out_dir, want_oid, &diagnostics);
1259
1260		if (diagnostics.errors.items.len > 0) {
1261		const notes_len: u32 = @intCast(diagnostics.errors.items.len);
1262		try report.addErrorWithNotes(notes_len, .{
1263		.tok = dep_location_tok,
1264		.off = 0,
1265		.msg = "unable to unpack packfile",
1266		});
1267		const eb = report.error_bundle;
1268		const notes_start = try eb.reserveNotes(notes_len);
1269		for (diagnostics.errors.items, notes_start..) \|item, note_i\| {
1270		switch (item) {
1271		.unable_to_create_sym_link => \|info\| {
1272		eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
1273		.msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{
1274		info.file_name, info.link_name, @errorName(info.code),
1275		}),
1276		}));
1277		},
1278		}
1279		}
1280		return error.InvalidGitPack;
1281		}
	140	if (self.sub_path.len > 0) {
	141	try writer.writeAll(self.sub_path);
	142	try writer.writeAll(fs.path.sep_str);
1282	143	}
1283	144	}
	145	};
1284	146
1285		try out_dir.deleteTree(".git");
1286		}
1287
1288		/// Compute the hash of a file path.
1289		fn computePathHash(gpa: Allocator, dir: Compilation.Directory, path: []const u8) ![Manifest.Hash.digest_length]u8 {
1290		const resolved_path = try std.fs.path.resolve(gpa, &.{ dir.path.?, path });
1291		defer gpa.free(resolved_path);
1292		var hasher = Manifest.Hash.init(.{});
1293		hasher.update(resolved_path);
1294		return hasher.finalResult();
1295		}
1296
1297		fn isDirectory(root_dir: Compilation.Directory, path: []const u8) !bool {
1298		var dir = root_dir.handle.openDir(path, .{}) catch \|err\| switch (err) {
1299		error.NotDir => return false,
1300		else => return err,
1301		};
1302		defer dir.close();
1303		return true;
1304		}
1305
1306		fn renameTmpIntoCache(
1307		cache_dir: fs.Dir,
1308		tmp_dir_sub_path: []const u8,
1309		dest_dir_sub_path: []const u8,
1310		) !void {
1311		assert(dest_dir_sub_path[1] == fs.path.sep);
1312		var handled_missing_dir = false;
1313		while (true) {
1314		cache_dir.rename(tmp_dir_sub_path, dest_dir_sub_path) catch \|err\| switch (err) {
1315		error.FileNotFound => {
1316		if (handled_missing_dir) return err;
1317		cache_dir.makeDir(dest_dir_sub_path[0..1]) catch \|mkd_err\| switch (mkd_err) {
1318		error.PathAlreadyExists => handled_missing_dir = true,
1319		else => \|e\| return e,
1320		};
1321		continue;
1322		},
1323		error.PathAlreadyExists, error.AccessDenied => {
1324		// Package has been already downloaded and may already be in use on the system.
1325		cache_dir.deleteTree(tmp_dir_sub_path) catch \|del_err\| {
1326		std.log.warn("unable to delete temp directory: {s}", .{@errorName(del_err)});
1327		};
1328		},
1329		else => \|e\| return e,
1330		};
1331		break;
1332		}
1333		}
1334
1335		test "getAttachmentType" {
1336		try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.gz"), ReadableResource.getAttachmentType("attaChment; FILENAME=\"stuff.tar.gz\"; size=42"));
1337		try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.gz"), ReadableResource.getAttachmentType("attachment; filename*=\"stuff.tar.gz\""));
1338		try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.xz"), ReadableResource.getAttachmentType("ATTACHMENT; filename=\"stuff.tar.xz\""));
1339		try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.xz"), ReadableResource.getAttachmentType("attachment; FileName=\"stuff.tar.xz\""));
1340		try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.gz"), ReadableResource.getAttachmentType("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz"));
1341
1342		try std.testing.expect(ReadableResource.getAttachmentType("attachment FileName=\"stuff.tar.gz\"") == null);
1343		try std.testing.expect(ReadableResource.getAttachmentType("attachment; FileName=\"stuff.tar\"") == null);
1344		try std.testing.expect(ReadableResource.getAttachmentType("attachment; FileName\"stuff.gz\"") == null);
1345		try std.testing.expect(ReadableResource.getAttachmentType("attachment; size=42") == null);
1346		try std.testing.expect(ReadableResource.getAttachmentType("inline; size=42") == null);
1347		try std.testing.expect(ReadableResource.getAttachmentType("FileName=\"stuff.tar.gz\"; attachment;") == null);
1348		try std.testing.expect(ReadableResource.getAttachmentType("FileName=\"stuff.tar.gz\";") == null);
1349		}
	147	const Package = @This();
	148	const builtin = @import("builtin");
	149	const std = @import("std");
	150	const fs = std.fs;
	151	const Allocator = std.mem.Allocator;
	152	const assert = std.debug.assert;
	153	const Cache = std.Build.Cache;

src/Package/Fetch.zig created+1557

...	...	@@ -0,0 +1,1557 @@
	1	//! Represents one independent job whose responsibility is to:
	2	//!
	3	//! 1. Check the global zig package cache to see if the hash already exists.
	4	//! If so, load, parse, and validate the build.zig.zon file therein, and
	5	//! goto step 8. Likewise if the location is a relative path, treat this
	6	//! the same as a cache hit. Otherwise, proceed.
	7	//! 2. Fetch and unpack a URL into a temporary directory.
	8	//! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed
	9	//! for the file to be missing, in which case this fetched package is considered
	10	//! to be a "naked" package.
	11	//! 4. Apply inclusion rules of the build.zig.zon to the temporary directory by
	12	//! deleting excluded files. If any files had errors for files that were
	13	//! ultimately excluded, those errors should be ignored, such as failure to
	14	//! create symlinks that weren't supposed to be included anyway.
	15	//! 5. Compute the package hash based on the remaining files in the temporary
	16	//! directory.
	17	//! 6. Rename the temporary directory into the global zig package cache
	18	//! directory. If the hash already exists, delete the temporary directory and
	19	//! leave the zig package cache directory untouched as it may be in use by the
	20	//! system. This is done even if the hash is invalid, in case the package with
	21	//! the different hash is used in the future.
	22	//! 7. Validate the computed hash against the expected hash. If invalid,
	23	//! this job is done.
	24	//! 8. Spawn a new fetch job for each dependency in the manifest file. Use
	25	//! a mutex and a hash map so that redundant jobs do not get queued up.
	26	//!
	27	//! All of this must be done with only referring to the state inside this struct
	28	//! because this work will be done in a dedicated thread.
	29
	30	arena: std.heap.ArenaAllocator,
	31	location: Location,
	32	location_tok: std.zig.Ast.TokenIndex,
	33	hash_tok: std.zig.Ast.TokenIndex,
	34	parent_package_root: Package.Path,
	35	parent_manifest_ast: ?*const std.zig.Ast,
	36	prog_node: *std.Progress.Node,
	37	job_queue: *JobQueue,
	38	/// If true, don't add an error for a missing hash. This flag is not passed
	39	/// down to recursive dependencies. It's intended to be used only be the CLI.
	40	omit_missing_hash_error: bool,
	41	/// If true, don't fail when a manifest file is missing the `paths` field,
	42	/// which specifies inclusion rules. This is intended to be true for the first
	43	/// fetch task and false for the recursive dependencies.
	44	allow_missing_paths_field: bool,
	45
	46	// Above this are fields provided as inputs to `run`.
	47	// Below this are fields populated by `run`.
	48
	49	/// This will either be relative to `global_cache`, or to the build root of
	50	/// the root package.
	51	package_root: Package.Path,
	52	error_bundle: ErrorBundle.Wip,
	53	manifest: ?Manifest,
	54	manifest_ast: std.zig.Ast,
	55	actual_hash: Manifest.Digest,
	56	/// Fetch logic notices whether a package has a build.zig file and sets this flag.
	57	has_build_zig: bool,
	58	/// Indicates whether the task aborted due to an out-of-memory condition.
	59	oom_flag: bool,
	60
	61	// This field is used by the CLI only, untouched by this file.
	62
	63	/// The module for this `Fetch` tasks's package, which exposes `build.zig` as
	64	/// the root source file.
	65	module: ?*Package.Module,
	66
	67	/// Contains shared state among all `Fetch` tasks.
	68	pub const JobQueue = struct {
	69	mutex: std.Thread.Mutex = .{},
	70	/// It's an array hash map so that it can be sorted before rendering the
	71	/// dependencies.zig source file.
	72	/// Protected by `mutex`.
	73	table: Table = .{},
	74	/// `table` may be missing some tasks such as ones that failed, so this
	75	/// field contains references to all of them.
	76	/// Protected by `mutex`.
	77	all_fetches: std.ArrayListUnmanaged(*Fetch) = .{},
	78
	79	http_client: *std.http.Client,
	80	thread_pool: *ThreadPool,
	81	wait_group: WaitGroup = .{},
	82	global_cache: Cache.Directory,
	83	recursive: bool,
	84	work_around_btrfs_bug: bool,
	85
	86	pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);
	87
	88	pub fn deinit(jq: *JobQueue) void {
	89	if (jq.all_fetches.items.len == 0) return;
	90	const gpa = jq.all_fetches.items[0].arena.child_allocator;
	91	jq.table.deinit(gpa);
	92	// These must be deinitialized in reverse order because subsequent
	93	// `Fetch` instances are allocated in prior ones' arenas.
	94	// Sorry, I know it's a bit weird, but it slightly simplifies the
	95	// critical section.
	96	while (jq.all_fetches.popOrNull()) \|f\| f.deinit();
	97	jq.all_fetches.deinit(gpa);
	98	jq.* = undefined;
	99	}
	100
	101	/// Dumps all subsequent error bundles into the first one.
	102	pub fn consolidateErrors(jq: *JobQueue) !void {
	103	const root = &jq.all_fetches.items[0].error_bundle;
	104	const gpa = root.gpa;
	105	for (jq.all_fetches.items[1..]) \|fetch\| {
	106	if (fetch.error_bundle.root_list.items.len > 0) {
	107	var bundle = try fetch.error_bundle.toOwnedBundle("");
	108	defer bundle.deinit(gpa);
	109	try root.addBundleAsRoots(bundle);
	110	}
	111	}
	112	}
	113
	114	/// Creates the dependencies.zig source code for the build runner to obtain
	115	/// via `@import("@dependencies")`.
	116	pub fn createDependenciesSource(jq: JobQueue, buf: std.ArrayList(u8)) Allocator.Error!void {
	117	const keys = jq.table.keys();
	118
	119	assert(keys.len != 0); // caller should have added the first one
	120	if (keys.len == 1) {
	121	// This is the first one. It must have no dependencies.
	122	return createEmptyDependenciesSource(buf);
	123	}
	124
	125	try buf.appendSlice("pub const packages = struct {\n");
	126
	127	// Ensure the generated .zig file is deterministic.
	128	jq.table.sortUnstable(@as(struct {
	129	keys: []const Manifest.MultiHashHexDigest,
	130	pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
	131	return std.mem.lessThan(u8, &ctx.keys[a_index], &ctx.keys[b_index]);
	132	}
	133	}, .{ .keys = keys }));
	134
	135	for (keys, jq.table.values()) \|hash, fetch\| {
	136	if (fetch == jq.all_fetches.items[0]) {
	137	// The first one is a dummy package for the current project.
	138	continue;
	139	}
	140	try buf.writer().print(
	141	\\ pub const {} = struct {{
	142	\\ pub const build_root = "{q}";
	143	\\
	144	, .{ std.zig.fmtId(&hash), fetch.package_root });
	145
	146	if (fetch.has_build_zig) {
	147	try buf.writer().print(
	148	\\ pub const build_zig = @import("{}");
	149	\\
	150	, .{std.zig.fmtEscapes(&hash)});
	151	}
	152
	153	if (fetch.manifest) \|*manifest\| {
	154	try buf.appendSlice(
	155	\\ pub const deps: []const struct { []const u8, []const u8 } = &.{
	156	\\
	157	);
	158	for (manifest.dependencies.keys(), manifest.dependencies.values()) \|name, dep\| {
	159	const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue;
	160	try buf.writer().print(
	161	" .{{ \"{}\", \"{}\" }},\n",
	162	.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) },
	163	);
	164	}
	165
	166	try buf.appendSlice(
	167	\\ };
	168	\\ };
	169	\\
	170	);
	171	} else {
	172	try buf.appendSlice(
	173	\\ pub const deps: []const struct { []const u8, []const u8 } = &.{};
	174	\\ };
	175	\\
	176	);
	177	}
	178	}
	179
	180	try buf.appendSlice(
	181	\\};
	182	\\
	183	\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{
	184	\\
	185	);
	186
	187	const root_fetch = jq.all_fetches.items[0];
	188	const root_manifest = &root_fetch.manifest.?;
	189
	190	for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) \|name, dep\| {
	191	const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue;
	192	try buf.writer().print(
	193	" .{{ \"{}\", \"{}\" }},\n",
	194	.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) },
	195	);
	196	}
	197	try buf.appendSlice("};\n");
	198	}
	199
	200	pub fn createEmptyDependenciesSource(buf: *std.ArrayList(u8)) Allocator.Error!void {
	201	try buf.appendSlice(
	202	\\pub const packages = struct {};
	203	\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
	204	\\
	205	);
	206	}
	207	};
	208
	209	pub const Location = union(enum) {
	210	remote: Remote,
	211	/// A directory found inside the parent package.
	212	relative_path: Package.Path,
	213	/// Recursive Fetch tasks will never use this Location, but it may be
	214	/// passed in by the CLI. Indicates the file contents here should be copied
	215	/// into the global package cache. It may be a file relative to the cwd or
	216	/// absolute, in which case it should be treated exactly like a `file://`
	217	/// URL, or a directory, in which case it should be treated as an
	218	/// already-unpacked directory (but still needs to be copied into the
	219	/// global package cache and have inclusion rules applied).
	220	path_or_url: []const u8,
	221
	222	pub const Remote = struct {
	223	url: []const u8,
	224	/// If this is null it means the user omitted the hash field from a dependency.
	225	/// It will be an error but the logic should still fetch and print the discovered hash.
	226	hash: ?Manifest.MultiHashHexDigest,
	227	};
	228	};
	229
	230	pub const RunError = error{
	231	OutOfMemory,
	232	/// This error code is intended to be handled by inspecting the
	233	/// `error_bundle` field.
	234	FetchFailed,
	235	};
	236
	237	pub fn run(f: *Fetch) RunError!void {
	238	const eb = &f.error_bundle;
	239	const arena = f.arena.allocator();
	240	const gpa = f.arena.child_allocator;
	241	const cache_root = f.job_queue.global_cache;
	242
	243	try eb.init(gpa);
	244
	245	// Check the global zig package cache to see if the hash already exists. If
	246	// so, load, parse, and validate the build.zig.zon file therein, and skip
	247	// ahead to queuing up jobs for dependencies. Likewise if the location is a
	248	// relative path, treat this the same as a cache hit. Otherwise, proceed.
	249
	250	const remote = switch (f.location) {
	251	.relative_path => \|pkg_root\| {
	252	if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail(
	253	f.location_tok,
	254	try eb.addString("expected path relative to build root; found absolute path"),
	255	);
	256	if (f.hash_tok != 0) return f.fail(
	257	f.hash_tok,
	258	try eb.addString("path-based dependencies are not hashed"),
	259	);
	260	if (std.mem.startsWith(u8, pkg_root.sub_path, "../") or
	261	std.mem.eql(u8, pkg_root.sub_path, ".."))
	262	{
	263	return f.fail(
	264	f.location_tok,
	265	try eb.printString("dependency path outside project: '{}{s}'", .{
	266	pkg_root.root_dir, pkg_root.sub_path,
	267	}),
	268	);
	269	}
	270	f.package_root = pkg_root;
	271	try loadManifest(f, pkg_root);
	272	if (!f.has_build_zig) try checkBuildFileExistence(f);
	273	if (!f.job_queue.recursive) return;
	274	return queueJobsForDeps(f);
	275	},
	276	.remote => \|remote\| remote,
	277	.path_or_url => \|path_or_url\| {
	278	if (fs.cwd().openIterableDir(path_or_url, .{})) \|dir\| {
	279	var resource: Resource = .{ .dir = dir };
	280	return runResource(f, path_or_url, &resource, null);
	281	} else \|dir_err\| {
	282	const file_err = if (dir_err == error.NotDir) e: {
	283	if (fs.cwd().openFile(path_or_url, .{})) \|file\| {
	284	var resource: Resource = .{ .file = file };
	285	return runResource(f, path_or_url, &resource, null);
	286	} else \|err\| break :e err;
	287	} else dir_err;
	288
	289	const uri = std.Uri.parse(path_or_url) catch \|uri_err\| {
	290	return f.fail(0, try eb.printString(
	291	"'{s}' could not be recognized as a file path ({s}) or an URL ({s})",
	292	.{ path_or_url, @errorName(file_err), @errorName(uri_err) },
	293	));
	294	};
	295	var resource = try f.initResource(uri);
	296	return runResource(f, uri.path, &resource, null);
	297	}
	298	},
	299	};
	300
	301	const s = fs.path.sep_str;
	302	if (remote.hash) \|expected_hash\| {
	303	const pkg_sub_path = "p" ++ s ++ expected_hash;
	304	if (cache_root.handle.access(pkg_sub_path, .{})) \|_\| {
	305	f.package_root = .{
	306	.root_dir = cache_root,
	307	.sub_path = try arena.dupe(u8, pkg_sub_path),
	308	};
	309	try loadManifest(f, f.package_root);
	310	try checkBuildFileExistence(f);
	311	if (!f.job_queue.recursive) return;
	312	return queueJobsForDeps(f);
	313	} else \|err\| switch (err) {
	314	error.FileNotFound => {},
	315	else => \|e\| {
	316	try eb.addRootErrorMessage(.{
	317	.msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{
	318	cache_root, pkg_sub_path, @errorName(e),
	319	}),
	320	});
	321	return error.FetchFailed;
	322	},
	323	}
	324	}
	325
	326	// Fetch and unpack the remote into a temporary directory.
	327
	328	const uri = std.Uri.parse(remote.url) catch \|err\| return f.fail(
	329	f.location_tok,
	330	try eb.printString("invalid URI: {s}", .{@errorName(err)}),
	331	);
	332	var resource = try f.initResource(uri);
	333	return runResource(f, uri.path, &resource, remote.hash);
	334	}
	335
	336	pub fn deinit(f: *Fetch) void {
	337	f.error_bundle.deinit();
	338	f.arena.deinit();
	339	}
	340
	341	/// Consumes `resource`, even if an error is returned.
	342	fn runResource(
	343	f: *Fetch,
	344	uri_path: []const u8,
	345	resource: *Resource,
	346	remote_hash: ?Manifest.MultiHashHexDigest,
	347	) RunError!void {
	348	defer resource.deinit();
	349	const arena = f.arena.allocator();
	350	const eb = &f.error_bundle;
	351	const s = fs.path.sep_str;
	352	const cache_root = f.job_queue.global_cache;
	353	const rand_int = std.crypto.random.int(u64);
	354	const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);
	355
	356	const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path});
	357	var tmp_directory: Cache.Directory = .{
	358	.path = tmp_directory_path,
	359	.handle = handle: {
	360	const dir = cache_root.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch \|err\| {
	361	try eb.addRootErrorMessage(.{
	362	.msg = try eb.printString("unable to create temporary directory '{s}': {s}", .{
	363	tmp_directory_path, @errorName(err),
	364	}),
	365	});
	366	return error.FetchFailed;
	367	};
	368	break :handle dir.dir;
	369	},
	370	};
	371	defer tmp_directory.handle.close();
	372
	373	try unpackResource(f, resource, uri_path, tmp_directory);
	374
	375	// Load, parse, and validate the unpacked build.zig.zon file. It is allowed
	376	// for the file to be missing, in which case this fetched package is
	377	// considered to be a "naked" package.
	378	try loadManifest(f, .{ .root_dir = tmp_directory });
	379
	380	// Apply the manifest's inclusion rules to the temporary directory by
	381	// deleting excluded files. If any error occurred for files that were
	382	// ultimately excluded, those errors should be ignored, such as failure to
	383	// create symlinks that weren't supposed to be included anyway.
	384
	385	// Empty directories have already been omitted by `unpackResource`.
	386
	387	const filter: Filter = .{
	388	.include_paths = if (f.manifest) \|m\| m.paths else .{},
	389	};
	390
	391	// Compute the package hash based on the remaining files in the temporary
	392	// directory.
	393
	394	if (builtin.os.tag == .linux and f.job_queue.work_around_btrfs_bug) {
	395	// https://github.com/ziglang/zig/issues/17095
	396	tmp_directory.handle.close();
	397	const iterable_dir = cache_root.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch
	398	@panic("btrfs workaround failed");
	399	tmp_directory.handle = iterable_dir.dir;
	400	}
	401
	402	f.actual_hash = try computeHash(f, tmp_directory, filter);
	403
	404	// Rename the temporary directory into the global zig package cache
	405	// directory. If the hash already exists, delete the temporary directory
	406	// and leave the zig package cache directory untouched as it may be in use
	407	// by the system. This is done even if the hash is invalid, in case the
	408	// package with the different hash is used in the future.
	409
	410	f.package_root = .{
	411	.root_dir = cache_root,
	412	.sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)),
	413	};
	414	renameTmpIntoCache(cache_root.handle, tmp_dir_sub_path, f.package_root.sub_path) catch \|err\| {
	415	const src = try cache_root.join(arena, &.{tmp_dir_sub_path});
	416	const dest = try cache_root.join(arena, &.{f.package_root.sub_path});
	417	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
	418	"unable to rename temporary directory '{s}' into package cache directory '{s}': {s}",
	419	.{ src, dest, @errorName(err) },
	420	) });
	421	return error.FetchFailed;
	422	};
	423
	424	// Validate the computed hash against the expected hash. If invalid, this
	425	// job is done.
	426
	427	const actual_hex = Manifest.hexDigest(f.actual_hash);
	428	if (remote_hash) \|declared_hash\| {
	429	if (!std.mem.eql(u8, &declared_hash, &actual_hex)) {
	430	return f.fail(f.hash_tok, try eb.printString(
	431	"hash mismatch: manifest declares {s} but the fetched package has {s}",
	432	.{ declared_hash, actual_hex },
	433	));
	434	}
	435	} else if (!f.omit_missing_hash_error) {
	436	const notes_len = 1;
	437	try eb.addRootErrorMessage(.{
	438	.msg = try eb.addString("dependency is missing hash field"),
	439	.src_loc = try f.srcLoc(f.location_tok),
	440	.notes_len = notes_len,
	441	});
	442	const notes_start = try eb.reserveNotes(notes_len);
	443	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
	444	.msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),
	445	}));
	446	return error.FetchFailed;
	447	}
	448
	449	// Spawn a new fetch job for each dependency in the manifest file. Use
	450	// a mutex and a hash map so that redundant jobs do not get queued up.
	451	if (!f.job_queue.recursive) return;
	452	return queueJobsForDeps(f);
	453	}
	454
	455	/// `computeHash` gets a free check for the existence of `build.zig`, but when
	456	/// not computing a hash, we need to do a syscall to check for it.
	457	fn checkBuildFileExistence(f: *Fetch) RunError!void {
	458	const eb = &f.error_bundle;
	459	if (f.package_root.access(Package.build_zig_basename, .{})) \|_\| {
	460	f.has_build_zig = true;
	461	} else \|err\| switch (err) {
	462	error.FileNotFound => {},
	463	else => \|e\| {
	464	try eb.addRootErrorMessage(.{
	465	.msg = try eb.printString("unable to access '{}{s}': {s}", .{
	466	f.package_root, Package.build_zig_basename, @errorName(e),
	467	}),
	468	});
	469	return error.FetchFailed;
	470	},
	471	}
	472	}
	473
	474	/// This function populates `f.manifest` or leaves it `null`.
	475	fn loadManifest(f: *Fetch, pkg_root: Package.Path) RunError!void {
	476	const eb = &f.error_bundle;
	477	const arena = f.arena.allocator();
	478	const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions(
	479	arena,
	480	try fs.path.join(arena, &.{ pkg_root.sub_path, Manifest.basename }),
	481	Manifest.max_bytes,
	482	null,
	483	1,
	484	0,
	485	) catch \|err\| switch (err) {
	486	error.FileNotFound => return,
	487	else => \|e\| {
	488	const file_path = try pkg_root.join(arena, Manifest.basename);
	489	try eb.addRootErrorMessage(.{
	490	.msg = try eb.printString("unable to load package manifest '{}': {s}", .{
	491	file_path, @errorName(e),
	492	}),
	493	});
	494	return error.FetchFailed;
	495	},
	496	};
	497
	498	const ast = &f.manifest_ast;
	499	ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon);
	500
	501	if (ast.errors.len > 0) {
	502	const file_path = try std.fmt.allocPrint(arena, "{}" ++ Manifest.basename, .{pkg_root});
	503	try main.putAstErrorsIntoBundle(arena, ast.*, file_path, eb);
	504	return error.FetchFailed;
	505	}
	506
	507	f.manifest = try Manifest.parse(arena, ast.*, .{
	508	.allow_missing_paths_field = f.allow_missing_paths_field,
	509	});
	510	const manifest = &f.manifest.?;
	511
	512	if (manifest.errors.len > 0) {
	513	const src_path = try eb.printString("{}{s}", .{ pkg_root, Manifest.basename });
	514	const token_starts = ast.tokens.items(.start);
	515
	516	for (manifest.errors) \|msg\| {
	517	const start_loc = ast.tokenLocation(0, msg.tok);
	518
	519	try eb.addRootErrorMessage(.{
	520	.msg = try eb.addString(msg.msg),
	521	.src_loc = try eb.addSourceLocation(.{
	522	.src_path = src_path,
	523	.span_start = token_starts[msg.tok],
	524	.span_end = @intCast(token_starts[msg.tok] + ast.tokenSlice(msg.tok).len),
	525	.span_main = token_starts[msg.tok] + msg.off,
	526	.line = @intCast(start_loc.line),
	527	.column = @intCast(start_loc.column),
	528	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
	529	}),
	530	});
	531	}
	532	return error.FetchFailed;
	533	}
	534	}
	535
	536	fn queueJobsForDeps(f: *Fetch) RunError!void {
	537	assert(f.job_queue.recursive);
	538
	539	// If the package does not have a build.zig.zon file then there are no dependencies.
	540	const manifest = f.manifest orelse return;
	541
	542	const new_fetches, const prog_names = nf: {
	543	const parent_arena = f.arena.allocator();
	544	const gpa = f.arena.child_allocator;
	545	const cache_root = f.job_queue.global_cache;
	546	const dep_names = manifest.dependencies.keys();
	547	const deps = manifest.dependencies.values();
	548	// Grab the new tasks into a temporary buffer so we can unlock that mutex
	549	// as fast as possible.
	550	// This overallocates any fetches that get skipped by the `continue` in the
	551	// loop below.
	552	const new_fetches = try parent_arena.alloc(Fetch, deps.len);
	553	const prog_names = try parent_arena.alloc([]const u8, deps.len);
	554	var new_fetch_index: usize = 0;
	555
	556	f.job_queue.mutex.lock();
	557	defer f.job_queue.mutex.unlock();
	558
	559	try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len);
	560	try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len));
	561
	562	// There are four cases here:
	563	// * Correct hash is provided by manifest.
	564	// - Hash map already has the entry, no need to add it again.
	565	// * Incorrect hash is provided by manifest.
	566	// - Hash mismatch error emitted; `queueJobsForDeps` is not called.
	567	// * Hash is not provided by manifest.
	568	// - Hash missing error emitted; `queueJobsForDeps` is not called.
	569	// * path-based location is used without a hash.
	570	// - Hash is added to the table based on the path alone before
	571	// calling run(); no need to add it again.
	572
	573	for (dep_names, deps) \|dep_name, dep\| {
	574	const new_fetch = &new_fetches[new_fetch_index];
	575	const location: Location = switch (dep.location) {
	576	.url => \|url\| .{ .remote = .{
	577	.url = url,
	578	.hash = h: {
	579	const h = dep.hash orelse break :h null;
	580	const digest_len = @typeInfo(Manifest.MultiHashHexDigest).Array.len;
	581	const multihash_digest = h[0..digest_len].*;
	582	const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
	583	if (gop.found_existing) continue;
	584	gop.value_ptr.* = new_fetch;
	585	break :h multihash_digest;
	586	},
	587	} },
	588	.path => \|rel_path\| l: {
	589	// This might produce an invalid path, which is checked for
	590	// at the beginning of run().
	591	const new_root = try f.package_root.resolvePosix(parent_arena, rel_path);
	592	const multihash_digest = relativePathDigest(new_root, cache_root);
	593	const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
	594	if (gop.found_existing) continue;
	595	gop.value_ptr.* = new_fetch;
	596	break :l .{ .relative_path = new_root };
	597	},
	598	};
	599	prog_names[new_fetch_index] = dep_name;
	600	new_fetch_index += 1;
	601	f.job_queue.all_fetches.appendAssumeCapacity(new_fetch);
	602	new_fetch.* = .{
	603	.arena = std.heap.ArenaAllocator.init(gpa),
	604	.location = location,
	605	.location_tok = dep.location_tok,
	606	.hash_tok = dep.hash_tok,
	607	.parent_package_root = f.package_root,
	608	.parent_manifest_ast = &f.manifest_ast,
	609	.prog_node = f.prog_node,
	610	.job_queue = f.job_queue,
	611	.omit_missing_hash_error = false,
	612	.allow_missing_paths_field = true,
	613
	614	.package_root = undefined,
	615	.error_bundle = undefined,
	616	.manifest = null,
	617	.manifest_ast = undefined,
	618	.actual_hash = undefined,
	619	.has_build_zig = false,
	620	.oom_flag = false,
	621
	622	.module = null,
	623	};
	624	}
	625
	626	// job_queue mutex is locked so this is OK.
	627	f.prog_node.unprotected_estimated_total_items += new_fetch_index;
	628
	629	break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] };
	630	};
	631
	632	// Now it's time to give tasks to the thread pool.
	633	const thread_pool = f.job_queue.thread_pool;
	634
	635	for (new_fetches, prog_names) \|*new_fetch, prog_name\| {
	636	f.job_queue.wait_group.start();
	637	thread_pool.spawn(workerRun, .{ new_fetch, prog_name }) catch \|err\| switch (err) {
	638	error.OutOfMemory => {
	639	new_fetch.oom_flag = true;
	640	f.job_queue.wait_group.finish();
	641	continue;
	642	},
	643	};
	644	}
	645	}
	646
	647	pub fn relativePathDigest(
	648	pkg_root: Package.Path,
	649	cache_root: Cache.Directory,
	650	) Manifest.MultiHashHexDigest {
	651	var hasher = Manifest.Hash.init(.{});
	652	// This hash is a tuple of:
	653	// * whether it relative to the global cache directory or to the root package
	654	// * the relative file path from there to the build root of the package
	655	hasher.update(if (pkg_root.root_dir.eql(cache_root))
	656	&package_hash_prefix_cached
	657	else
	658	&package_hash_prefix_project);
	659	hasher.update(pkg_root.sub_path);
	660	return Manifest.hexDigest(hasher.finalResult());
	661	}
	662
	663	pub fn workerRun(f: *Fetch, prog_name: []const u8) void {
	664	defer f.job_queue.wait_group.finish();
	665
	666	var prog_node = f.prog_node.start(prog_name, 0);
	667	defer prog_node.end();
	668	prog_node.activate();
	669
	670	run(f) catch \|err\| switch (err) {
	671	error.OutOfMemory => f.oom_flag = true,
	672	error.FetchFailed => {
	673	// Nothing to do because the errors are already reported in `error_bundle`,
	674	// and a reference is kept to the `Fetch` task inside `all_fetches`.
	675	},
	676	};
	677	}
	678
	679	fn srcLoc(
	680	f: *Fetch,
	681	tok: std.zig.Ast.TokenIndex,
	682	) Allocator.Error!ErrorBundle.SourceLocationIndex {
	683	const ast = f.parent_manifest_ast orelse return .none;
	684	const eb = &f.error_bundle;
	685	const token_starts = ast.tokens.items(.start);
	686	const start_loc = ast.tokenLocation(0, tok);
	687	const src_path = try eb.printString("{}" ++ Manifest.basename, .{f.parent_package_root});
	688	const msg_off = 0;
	689	return eb.addSourceLocation(.{
	690	.src_path = src_path,
	691	.span_start = token_starts[tok],
	692	.span_end = @intCast(token_starts[tok] + ast.tokenSlice(tok).len),
	693	.span_main = token_starts[tok] + msg_off,
	694	.line = @intCast(start_loc.line),
	695	.column = @intCast(start_loc.column),
	696	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
	697	});
	698	}
	699
	700	fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError {
	701	const eb = &f.error_bundle;
	702	try eb.addRootErrorMessage(.{
	703	.msg = msg_str,
	704	.src_loc = try f.srcLoc(msg_tok),
	705	});
	706	return error.FetchFailed;
	707	}
	708
	709	const Resource = union(enum) {
	710	file: fs.File,
	711	http_request: std.http.Client.Request,
	712	git: Git,
	713	dir: fs.IterableDir,
	714
	715	const Git = struct {
	716	fetch_stream: git.Session.FetchStream,
	717	want_oid: [git.oid_length]u8,
	718	};
	719
	720	fn deinit(resource: *Resource) void {
	721	switch (resource.*) {
	722	.file => \|*file\| file.close(),
	723	.http_request => \|*req\| req.deinit(),
	724	.git => \|*git_resource\| git_resource.fetch_stream.deinit(),
	725	.dir => \|*dir\| dir.close(),
	726	}
	727	resource.* = undefined;
	728	}
	729
	730	fn reader(resource: *Resource) std.io.AnyReader {
	731	return .{
	732	.context = resource,
	733	.readFn = read,
	734	};
	735	}
	736
	737	fn read(context: *const anyopaque, buffer: []u8) anyerror!usize {
	738	const resource: *Resource = @constCast(@ptrCast(@alignCast(context)));
	739	switch (resource.*) {
	740	.file => \|*f\| return f.read(buffer),
	741	.http_request => \|*r\| return r.read(buffer),
	742	.git => \|*g\| return g.fetch_stream.read(buffer),
	743	.dir => unreachable,
	744	}
	745	}
	746	};
	747
	748	const FileType = enum {
	749	tar,
	750	@"tar.gz",
	751	@"tar.xz",
	752	git_pack,
	753
	754	fn fromPath(file_path: []const u8) ?FileType {
	755	if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar;
	756	if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz";
	757	if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz";
	758	return null;
	759	}
	760
	761	/// Parameter is a content-disposition header value.
	762	fn fromContentDisposition(cd_header: []const u8) ?FileType {
	763	const attach_end = ascii.indexOfIgnoreCase(cd_header, "attachment;") orelse
	764	return null;
	765
	766	var value_start = ascii.indexOfIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse
	767	return null;
	768	value_start += "filename".len;
	769	if (cd_header[value_start] == '*') {
	770	value_start += 1;
	771	}
	772	if (cd_header[value_start] != '=') return null;
	773	value_start += 1;
	774
	775	var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len;
	776	if (cd_header[value_end - 1] == '\"') {
	777	value_end -= 1;
	778	}
	779	return fromPath(cd_header[value_start..value_end]);
	780	}
	781
	782	test fromContentDisposition {
	783	try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42"));
	784	try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\""));
	785	try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\""));
	786	try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\""));
	787	try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz"));
	788
	789	try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null);
	790	try std.testing.expect(fromContentDisposition("attachment; FileName=\"stuff.tar\"") == null);
	791	try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null);
	792	try std.testing.expect(fromContentDisposition("attachment; size=42") == null);
	793	try std.testing.expect(fromContentDisposition("inline; size=42") == null);
	794	try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null);
	795	try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null);
	796	}
	797	};
	798
	799	fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
	800	const gpa = f.arena.child_allocator;
	801	const arena = f.arena.allocator();
	802	const eb = &f.error_bundle;
	803
	804	if (ascii.eqlIgnoreCase(uri.scheme, "file")) return .{
	805	.file = f.parent_package_root.openFile(uri.path, .{}) catch \|err\| {
	806	return f.fail(f.location_tok, try eb.printString("unable to open '{}{s}': {s}", .{
	807	f.parent_package_root, uri.path, @errorName(err),
	808	}));
	809	},
	810	};
	811
	812	const http_client = f.job_queue.http_client;
	813
	814	if (ascii.eqlIgnoreCase(uri.scheme, "http") or
	815	ascii.eqlIgnoreCase(uri.scheme, "https"))
	816	{
	817	var h = std.http.Headers{ .allocator = gpa };
	818	defer h.deinit();
	819
	820	var req = http_client.request(.GET, uri, h, .{}) catch \|err\| {
	821	return f.fail(f.location_tok, try eb.printString(
	822	"unable to connect to server: {s}",
	823	.{@errorName(err)},
	824	));
	825	};
	826	errdefer req.deinit(); // releases more than memory
	827
	828	req.start(.{}) catch \|err\| {
	829	return f.fail(f.location_tok, try eb.printString(
	830	"HTTP request failed: {s}",
	831	.{@errorName(err)},
	832	));
	833	};
	834	req.wait() catch \|err\| {
	835	return f.fail(f.location_tok, try eb.printString(
	836	"invalid HTTP response: {s}",
	837	.{@errorName(err)},
	838	));
	839	};
	840
	841	if (req.response.status != .ok) {
	842	return f.fail(f.location_tok, try eb.printString(
	843	"bad HTTP response code: '{d} {s}'",
	844	.{ @intFromEnum(req.response.status), req.response.status.phrase() orelse "" },
	845	));
	846	}
	847
	848	return .{ .http_request = req };
	849	}
	850
	851	if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or
	852	ascii.eqlIgnoreCase(uri.scheme, "git+https"))
	853	{
	854	var transport_uri = uri;
	855	transport_uri.scheme = uri.scheme["git+".len..];
	856	var redirect_uri: []u8 = undefined;
	857	var session: git.Session = .{ .transport = http_client, .uri = transport_uri };
	858	session.discoverCapabilities(gpa, &redirect_uri) catch \|err\| switch (err) {
	859	error.Redirected => {
	860	defer gpa.free(redirect_uri);
	861	return f.fail(f.location_tok, try eb.printString(
	862	"repository moved to {s}",
	863	.{redirect_uri},
	864	));
	865	},
	866	else => \|e\| {
	867	return f.fail(f.location_tok, try eb.printString(
	868	"unable to discover remote git server capabilities: {s}",
	869	.{@errorName(e)},
	870	));
	871	},
	872	};
	873
	874	const want_oid = want_oid: {
	875	const want_ref = uri.fragment orelse "HEAD";
	876	if (git.parseOid(want_ref)) \|oid\| break :want_oid oid else \|_\| {}
	877
	878	const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref});
	879	const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref});
	880
	881	var ref_iterator = session.listRefs(gpa, .{
	882	.ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },
	883	.include_peeled = true,
	884	}) catch \|err\| {
	885	return f.fail(f.location_tok, try eb.printString(
	886	"unable to list refs: {s}",
	887	.{@errorName(err)},
	888	));
	889	};
	890	defer ref_iterator.deinit();
	891	while (ref_iterator.next() catch \|err\| {
	892	return f.fail(f.location_tok, try eb.printString(
	893	"unable to iterate refs: {s}",
	894	.{@errorName(err)},
	895	));
	896	}) \|ref\| {
	897	if (std.mem.eql(u8, ref.name, want_ref) or
	898	std.mem.eql(u8, ref.name, want_ref_head) or
	899	std.mem.eql(u8, ref.name, want_ref_tag))
	900	{
	901	break :want_oid ref.peeled orelse ref.oid;
	902	}
	903	}
	904	return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref}));
	905	};
	906	if (uri.fragment == null) {
	907	const notes_len = 1;
	908	try eb.addRootErrorMessage(.{
	909	.msg = try eb.addString("url field is missing an explicit ref"),
	910	.src_loc = try f.srcLoc(f.location_tok),
	911	.notes_len = notes_len,
	912	});
	913	const notes_start = try eb.reserveNotes(notes_len);
	914	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
	915	.msg = try eb.printString("try .url = \"{+/}#{}\",", .{
	916	uri, std.fmt.fmtSliceHexLower(&want_oid),
	917	}),
	918	}));
	919	return error.FetchFailed;
	920	}
	921
	922	var want_oid_buf: [git.fmt_oid_length]u8 = undefined;
	923	_ = std.fmt.bufPrint(&want_oid_buf, "{}", .{
	924	std.fmt.fmtSliceHexLower(&want_oid),
	925	}) catch unreachable;
	926	var fetch_stream = session.fetch(gpa, &.{&want_oid_buf}) catch \|err\| {
	927	return f.fail(f.location_tok, try eb.printString(
	928	"unable to create fetch stream: {s}",
	929	.{@errorName(err)},
	930	));
	931	};
	932	errdefer fetch_stream.deinit();
	933
	934	return .{ .git = .{
	935	.fetch_stream = fetch_stream,
	936	.want_oid = want_oid,
	937	} };
	938	}
	939
	940	return f.fail(f.location_tok, try eb.printString(
	941	"unsupported URL scheme: {s}",
	942	.{uri.scheme},
	943	));
	944	}
	945
	946	fn unpackResource(
	947	f: *Fetch,
	948	resource: *Resource,
	949	uri_path: []const u8,
	950	tmp_directory: Cache.Directory,
	951	) RunError!void {
	952	const eb = &f.error_bundle;
	953	const file_type = switch (resource.*) {
	954	.file => FileType.fromPath(uri_path) orelse
	955	return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})),
	956
	957	.http_request => \|req\| ft: {
	958	// Content-Type takes first precedence.
	959	const content_type = req.response.headers.getFirstValue("Content-Type") orelse
	960	return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header"));
	961
	962	if (ascii.eqlIgnoreCase(content_type, "application/x-tar"))
	963	break :ft .tar;
	964
	965	if (ascii.eqlIgnoreCase(content_type, "application/gzip") or
	966	ascii.eqlIgnoreCase(content_type, "application/x-gzip") or
	967	ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))
	968	{
	969	break :ft .@"tar.gz";
	970	}
	971
	972	if (ascii.eqlIgnoreCase(content_type, "application/x-xz"))
	973	break :ft .@"tar.xz";
	974
	975	if (!ascii.eqlIgnoreCase(content_type, "application/octet-stream")) {
	976	return f.fail(f.location_tok, try eb.printString(
	977	"unrecognized 'Content-Type' header: '{s}'",
	978	.{content_type},
	979	));
	980	}
	981
	982	// Next, the filename from 'content-disposition: attachment' takes precedence.
	983	if (req.response.headers.getFirstValue("Content-Disposition")) \|cd_header\| {
	984	break :ft FileType.fromContentDisposition(cd_header) orelse {
	985	return f.fail(f.location_tok, try eb.printString(
	986	"unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",
	987	.{cd_header},
	988	));
	989	};
	990	}
	991
	992	// Finally, the path from the URI is used.
	993	break :ft FileType.fromPath(uri_path) orelse {
	994	return f.fail(f.location_tok, try eb.printString(
	995	"unknown file type: '{s}'",
	996	.{uri_path},
	997	));
	998	};
	999	},
	1000
	1001	.git => .git_pack,
	1002
	1003	.dir => \|dir\| return f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch \|err\| {
	1004	return f.fail(f.location_tok, try eb.printString(
	1005	"unable to copy directory '{s}': {s}",
	1006	.{ uri_path, @errorName(err) },
	1007	));
	1008	},
	1009	};
	1010
	1011	switch (file_type) {
	1012	.tar => try unpackTarball(f, tmp_directory.handle, resource.reader()),
	1013	.@"tar.gz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.gzip),
	1014	.@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz),
	1015	.git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch \|err\| switch (err) {
	1016	error.FetchFailed => return error.FetchFailed,
	1017	error.OutOfMemory => return error.OutOfMemory,
	1018	else => \|e\| return f.fail(f.location_tok, try eb.printString(
	1019	"unable to unpack git files: {s}",
	1020	.{@errorName(e)},
	1021	)),
	1022	},
	1023	}
	1024	}
	1025
	1026	fn unpackTarballCompressed(
	1027	f: *Fetch,
	1028	out_dir: fs.Dir,
	1029	resource: *Resource,
	1030	comptime Compression: type,
	1031	) RunError!void {
	1032	const gpa = f.arena.child_allocator;
	1033	const eb = &f.error_bundle;
	1034	const reader = resource.reader();
	1035	var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
	1036
	1037	var decompress = Compression.decompress(gpa, br.reader()) catch \|err\| {
	1038	return f.fail(f.location_tok, try eb.printString(
	1039	"unable to decompress tarball: {s}",
	1040	.{@errorName(err)},
	1041	));
	1042	};
	1043	defer decompress.deinit();
	1044
	1045	return unpackTarball(f, out_dir, decompress.reader());
	1046	}
	1047
	1048	fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
	1049	const eb = &f.error_bundle;
	1050	const gpa = f.arena.child_allocator;
	1051
	1052	var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = gpa };
	1053	defer diagnostics.deinit();
	1054
	1055	std.tar.pipeToFileSystem(out_dir, reader, .{
	1056	.diagnostics = &diagnostics,
	1057	.strip_components = 1,
	1058	// TODO: we would like to set this to executable_bit_only, but two
	1059	// things need to happen before that:
	1060	// 1. the tar implementation needs to support it
	1061	// 2. the hashing algorithm here needs to support detecting the is_executable
	1062	// bit on Windows from the ACLs (see the isExecutable function).
	1063	.mode_mode = .ignore,
	1064	.exclude_empty_directories = true,
	1065	}) catch \|err\| return f.fail(f.location_tok, try eb.printString(
	1066	"unable to unpack tarball to temporary directory: {s}",
	1067	.{@errorName(err)},
	1068	));
	1069
	1070	if (diagnostics.errors.items.len > 0) {
	1071	const notes_len: u32 = @intCast(diagnostics.errors.items.len);
	1072	try eb.addRootErrorMessage(.{
	1073	.msg = try eb.addString("unable to unpack tarball"),
	1074	.src_loc = try f.srcLoc(f.location_tok),
	1075	.notes_len = notes_len,
	1076	});
	1077	const notes_start = try eb.reserveNotes(notes_len);
	1078	for (diagnostics.errors.items, notes_start..) \|item, note_i\| {
	1079	switch (item) {
	1080	.unable_to_create_sym_link => \|info\| {
	1081	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
	1082	.msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{
	1083	info.file_name, info.link_name, @errorName(info.code),
	1084	}),
	1085	}));
	1086	},
	1087	.unable_to_create_file => \|info\| {
	1088	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
	1089	.msg = try eb.printString("unable to create file '{s}': {s}", .{
	1090	info.file_name, @errorName(info.code),
	1091	}),
	1092	}));
	1093	},
	1094	.unsupported_file_type => \|info\| {
	1095	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
	1096	.msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{
	1097	info.file_name, @intFromEnum(info.file_type),
	1098	}),
	1099	}));
	1100	},
	1101	}
	1102	}
	1103	return error.FetchFailed;
	1104	}
	1105	}
	1106
	1107	fn unpackGitPack(f: Fetch, out_dir: fs.Dir, resource: Resource) anyerror!void {
	1108	const eb = &f.error_bundle;
	1109	const gpa = f.arena.child_allocator;
	1110	const want_oid = resource.git.want_oid;
	1111	const reader = resource.git.fetch_stream.reader();
	1112	// The .git directory is used to store the packfile and associated index, but
	1113	// we do not attempt to replicate the exact structure of a real .git
	1114	// directory, since that isn't relevant for fetching a package.
	1115	{
	1116	var pack_dir = try out_dir.makeOpenPath(".git", .{});
	1117	defer pack_dir.close();
	1118	var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });
	1119	defer pack_file.close();
	1120	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
	1121	try fifo.pump(reader, pack_file.writer());
	1122	try pack_file.sync();
	1123
	1124	var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });
	1125	defer index_file.close();
	1126	{
	1127	var index_prog_node = f.prog_node.start("Index pack", 0);
	1128	defer index_prog_node.end();
	1129	index_prog_node.activate();
	1130	var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
	1131	try git.indexPack(gpa, pack_file, index_buffered_writer.writer());
	1132	try index_buffered_writer.flush();
	1133	try index_file.sync();
	1134	}
	1135
	1136	{
	1137	var checkout_prog_node = f.prog_node.start("Checkout", 0);
	1138	defer checkout_prog_node.end();
	1139	checkout_prog_node.activate();
	1140	var repository = try git.Repository.init(gpa, pack_file, index_file);
	1141	defer repository.deinit();
	1142	var diagnostics: git.Diagnostics = .{ .allocator = gpa };
	1143	defer diagnostics.deinit();
	1144	try repository.checkout(out_dir, want_oid, &diagnostics);
	1145
	1146	if (diagnostics.errors.items.len > 0) {
	1147	const notes_len: u32 = @intCast(diagnostics.errors.items.len);
	1148	try eb.addRootErrorMessage(.{
	1149	.msg = try eb.addString("unable to unpack packfile"),
	1150	.src_loc = try f.srcLoc(f.location_tok),
	1151	.notes_len = notes_len,
	1152	});
	1153	const notes_start = try eb.reserveNotes(notes_len);
	1154	for (diagnostics.errors.items, notes_start..) \|item, note_i\| {
	1155	switch (item) {
	1156	.unable_to_create_sym_link => \|info\| {
	1157	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
	1158	.msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{
	1159	info.file_name, info.link_name, @errorName(info.code),
	1160	}),
	1161	}));
	1162	},
	1163	}
	1164	}
	1165	return error.InvalidGitPack;
	1166	}
	1167	}
	1168	}
	1169
	1170	try out_dir.deleteTree(".git");
	1171	}
	1172
	1173	fn recursiveDirectoryCopy(f: *Fetch, dir: fs.IterableDir, tmp_dir: fs.Dir) anyerror!void {
	1174	const gpa = f.arena.child_allocator;
	1175	// Recursive directory copy.
	1176	var it = try dir.walk(gpa);
	1177	defer it.deinit();
	1178	while (try it.next()) \|entry\| {
	1179	switch (entry.kind) {
	1180	.directory => {}, // omit empty directories
	1181	.file => {
	1182	dir.dir.copyFile(
	1183	entry.path,
	1184	tmp_dir,
	1185	entry.path,
	1186	.{},
	1187	) catch \|err\| switch (err) {
	1188	error.FileNotFound => {
	1189	if (fs.path.dirname(entry.path)) \|dirname\| try tmp_dir.makePath(dirname);
	1190	try dir.dir.copyFile(entry.path, tmp_dir, entry.path, .{});
	1191	},
	1192	else => \|e\| return e,
	1193	};
	1194	},
	1195	.sym_link => {
	1196	var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
	1197	const link_name = try dir.dir.readLink(entry.path, &buf);
	1198	// TODO: if this would create a symlink to outside
	1199	// the destination directory, fail with an error instead.
	1200	tmp_dir.symLink(link_name, entry.path, .{}) catch \|err\| switch (err) {
	1201	error.FileNotFound => {
	1202	if (fs.path.dirname(entry.path)) \|dirname\| try tmp_dir.makePath(dirname);
	1203	try tmp_dir.symLink(link_name, entry.path, .{});
	1204	},
	1205	else => \|e\| return e,
	1206	};
	1207	},
	1208	else => return error.IllegalFileTypeInPackage,
	1209	}
	1210	}
	1211	}
	1212
	1213	pub fn renameTmpIntoCache(
	1214	cache_dir: fs.Dir,
	1215	tmp_dir_sub_path: []const u8,
	1216	dest_dir_sub_path: []const u8,
	1217	) !void {
	1218	assert(dest_dir_sub_path[1] == fs.path.sep);
	1219	var handled_missing_dir = false;
	1220	while (true) {
	1221	cache_dir.rename(tmp_dir_sub_path, dest_dir_sub_path) catch \|err\| switch (err) {
	1222	error.FileNotFound => {
	1223	if (handled_missing_dir) return err;
	1224	cache_dir.makeDir(dest_dir_sub_path[0..1]) catch \|mkd_err\| switch (mkd_err) {
	1225	error.PathAlreadyExists => handled_missing_dir = true,
	1226	else => \|e\| return e,
	1227	};
	1228	continue;
	1229	},
	1230	error.PathAlreadyExists, error.AccessDenied => {
	1231	// Package has been already downloaded and may already be in use on the system.
	1232	cache_dir.deleteTree(tmp_dir_sub_path) catch {
	1233	// Garbage files leftover in zig-cache/tmp/ is, as they say
	1234	// on Star Trek, "operating within normal parameters".
	1235	};
	1236	},
	1237	else => \|e\| return e,
	1238	};
	1239	break;
	1240	}
	1241	}
	1242
	1243	/// Assumes that files not included in the package have already been filtered
	1244	/// prior to calling this function. This ensures that files not protected by
	1245	/// the hash are not present on the file system. Empty directories are *not
	1246	/// hashed* and must not be present on the file system when calling this
	1247	/// function.
	1248	fn computeHash(
	1249	f: *Fetch,
	1250	tmp_directory: Cache.Directory,
	1251	filter: Filter,
	1252	) RunError!Manifest.Digest {
	1253	// All the path name strings need to be in memory for sorting.
	1254	const arena = f.arena.allocator();
	1255	const gpa = f.arena.child_allocator;
	1256	const eb = &f.error_bundle;
	1257	const thread_pool = f.job_queue.thread_pool;
	1258
	1259	// Collect all files, recursively, then sort.
	1260	var all_files = std.ArrayList(*HashedFile).init(gpa);
	1261	defer all_files.deinit();
	1262
	1263	var deleted_files = std.ArrayList(*DeletedFile).init(gpa);
	1264	defer deleted_files.deinit();
	1265
	1266	// Track directories which had any files deleted from them so that empty directories
	1267	// can be deleted.
	1268	var sus_dirs: std.StringArrayHashMapUnmanaged(void) = .{};
	1269	defer sus_dirs.deinit(gpa);
	1270
	1271	var walker = try @as(fs.IterableDir, .{ .dir = tmp_directory.handle }).walk(gpa);
	1272	defer walker.deinit();
	1273
	1274	{
	1275	// The final hash will be a hash of each file hashed independently. This
	1276	// allows hashing in parallel.
	1277	var wait_group: WaitGroup = .{};
	1278	// `computeHash` is called from a worker thread so there must not be
	1279	// any waiting without working or a deadlock could occur.
	1280	defer thread_pool.waitAndWork(&wait_group);
	1281
	1282	while (walker.next() catch \|err\| {
	1283	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
	1284	"unable to walk temporary directory '{}': {s}",
	1285	.{ tmp_directory, @errorName(err) },
	1286	) });
	1287	return error.FetchFailed;
	1288	}) \|entry\| {
	1289	if (entry.kind == .directory) continue;
	1290
	1291	if (!filter.includePath(entry.path)) {
	1292	// Delete instead of including in hash calculation.
	1293	const fs_path = try arena.dupe(u8, entry.path);
	1294
	1295	// Also track the parent directory in case it becomes empty.
	1296	if (fs.path.dirname(fs_path)) \|parent\|
	1297	try sus_dirs.put(gpa, parent, {});
	1298
	1299	const deleted_file = try arena.create(DeletedFile);
	1300	deleted_file.* = .{
	1301	.fs_path = fs_path,
	1302	.failure = undefined, // to be populated by the worker
	1303	};
	1304	wait_group.start();
	1305	try thread_pool.spawn(workerDeleteFile, .{
	1306	tmp_directory.handle, deleted_file, &wait_group,
	1307	});
	1308	try deleted_files.append(deleted_file);
	1309	continue;
	1310	}
	1311
	1312	const kind: HashedFile.Kind = switch (entry.kind) {
	1313	.directory => unreachable,
	1314	.file => .file,
	1315	.sym_link => .sym_link,
	1316	else => return f.fail(f.location_tok, try eb.printString(
	1317	"package contains '{s}' which has illegal file type '{s}'",
	1318	.{ entry.path, @tagName(entry.kind) },
	1319	)),
	1320	};
	1321
	1322	if (std.mem.eql(u8, entry.path, Package.build_zig_basename))
	1323	f.has_build_zig = true;
	1324
	1325	const fs_path = try arena.dupe(u8, entry.path);
	1326	const hashed_file = try arena.create(HashedFile);
	1327	hashed_file.* = .{
	1328	.fs_path = fs_path,
	1329	.normalized_path = try normalizePath(arena, fs_path),
	1330	.kind = kind,
	1331	.hash = undefined, // to be populated by the worker
	1332	.failure = undefined, // to be populated by the worker
	1333	};
	1334	wait_group.start();
	1335	try thread_pool.spawn(workerHashFile, .{
	1336	tmp_directory.handle, hashed_file, &wait_group,
	1337	});
	1338	try all_files.append(hashed_file);
	1339	}
	1340	}
	1341
	1342	{
	1343	// Sort by length, descending, so that child directories get removed first.
	1344	sus_dirs.sortUnstable(@as(struct {
	1345	keys: []const []const u8,
	1346	pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
	1347	return ctx.keys[b_index].len < ctx.keys[a_index].len;
	1348	}
	1349	}, .{ .keys = sus_dirs.keys() }));
	1350
	1351	// During this loop, more entries will be added, so we must loop by index.
	1352	var i: usize = 0;
	1353	while (i < sus_dirs.count()) : (i += 1) {
	1354	const sus_dir = sus_dirs.keys()[i];
	1355	tmp_directory.handle.deleteDir(sus_dir) catch \|err\| switch (err) {
	1356	error.DirNotEmpty => continue,
	1357	error.FileNotFound => continue,
	1358	else => \|e\| {
	1359	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
	1360	"unable to delete empty directory '{s}': {s}",
	1361	.{ sus_dir, @errorName(e) },
	1362	) });
	1363	return error.FetchFailed;
	1364	},
	1365	};
	1366	if (fs.path.dirname(sus_dir)) \|parent\| {
	1367	try sus_dirs.put(gpa, parent, {});
	1368	}
	1369	}
	1370	}
	1371
	1372	std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
	1373
	1374	var hasher = Manifest.Hash.init(.{});
	1375	var any_failures = false;
	1376	for (all_files.items) \|hashed_file\| {
	1377	hashed_file.failure catch \|err\| {
	1378	any_failures = true;
	1379	try eb.addRootErrorMessage(.{
	1380	.msg = try eb.printString("unable to hash '{s}': {s}", .{
	1381	hashed_file.fs_path, @errorName(err),
	1382	}),
	1383	});
	1384	};
	1385	hasher.update(&hashed_file.hash);
	1386	}
	1387	for (deleted_files.items) \|deleted_file\| {
	1388	deleted_file.failure catch \|err\| {
	1389	any_failures = true;
	1390	try eb.addRootErrorMessage(.{
	1391	.msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{
	1392	deleted_file.fs_path, @errorName(err),
	1393	}),
	1394	});
	1395	};
	1396	}
	1397
	1398	if (any_failures) return error.FetchFailed;
	1399	return hasher.finalResult();
	1400	}
	1401
	1402	fn workerHashFile(dir: fs.Dir, hashed_file: HashedFile, wg: WaitGroup) void {
	1403	defer wg.finish();
	1404	hashed_file.failure = hashFileFallible(dir, hashed_file);
	1405	}
	1406
	1407	fn workerDeleteFile(dir: fs.Dir, deleted_file: DeletedFile, wg: WaitGroup) void {
	1408	defer wg.finish();
	1409	deleted_file.failure = deleteFileFallible(dir, deleted_file);
	1410	}
	1411
	1412	fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
	1413	var buf: [8000]u8 = undefined;
	1414	var hasher = Manifest.Hash.init(.{});
	1415	hasher.update(hashed_file.normalized_path);
	1416	switch (hashed_file.kind) {
	1417	.file => {
	1418	var file = try dir.openFile(hashed_file.fs_path, .{});
	1419	defer file.close();
	1420	hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) });
	1421	while (true) {
	1422	const bytes_read = try file.read(&buf);
	1423	if (bytes_read == 0) break;
	1424	hasher.update(buf[0..bytes_read]);
	1425	}
	1426	},
	1427	.sym_link => {
	1428	const link_name = try dir.readLink(hashed_file.fs_path, &buf);
	1429	hasher.update(link_name);
	1430	},
	1431	}
	1432	hasher.final(&hashed_file.hash);
	1433	}
	1434
	1435	fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {
	1436	try dir.deleteFile(deleted_file.fs_path);
	1437	}
	1438
	1439	fn isExecutable(file: fs.File) !bool {
	1440	if (builtin.os.tag == .windows) {
	1441	// TODO check the ACL on Windows.
	1442	// Until this is implemented, this could be a false negative on
	1443	// Windows, which is why we do not yet set executable_bit_only above
	1444	// when unpacking the tarball.
	1445	return false;
	1446	} else {
	1447	const stat = try file.stat();
	1448	return (stat.mode & std.os.S.IXUSR) != 0;
	1449	}
	1450	}
	1451
	1452	const DeletedFile = struct {
	1453	fs_path: []const u8,
	1454	failure: Error!void,
	1455
	1456	const Error =
	1457	fs.Dir.DeleteFileError \|\|
	1458	fs.Dir.DeleteDirError;
	1459	};
	1460
	1461	const HashedFile = struct {
	1462	fs_path: []const u8,
	1463	normalized_path: []const u8,
	1464	hash: Manifest.Digest,
	1465	failure: Error!void,
	1466	kind: Kind,
	1467
	1468	const Error =
	1469	fs.File.OpenError \|\|
	1470	fs.File.ReadError \|\|
	1471	fs.File.StatError \|\|
	1472	fs.Dir.ReadLinkError;
	1473
	1474	const Kind = enum { file, sym_link };
	1475
	1476	fn lessThan(context: void, lhs: const HashedFile, rhs: const HashedFile) bool {
	1477	_ = context;
	1478	return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path);
	1479	}
	1480	};
	1481
	1482	/// Make a file system path identical independently of operating system path inconsistencies.
	1483	/// This converts backslashes into forward slashes.
	1484	fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
	1485	const canonical_sep = '/';
	1486
	1487	if (fs.path.sep == canonical_sep)
	1488	return fs_path;
	1489
	1490	const normalized = try arena.dupe(u8, fs_path);
	1491	for (normalized) \|*byte\| {
	1492	switch (byte.*) {
	1493	fs.path.sep => byte.* = canonical_sep,
	1494	else => continue,
	1495	}
	1496	}
	1497	return normalized;
	1498	}
	1499
	1500	const Filter = struct {
	1501	include_paths: std.StringArrayHashMapUnmanaged(void) = .{},
	1502
	1503	/// sub_path is relative to the package root.
	1504	pub fn includePath(self: Filter, sub_path: []const u8) bool {
	1505	if (self.include_paths.count() == 0) return true;
	1506	if (self.include_paths.contains("")) return true;
	1507	if (self.include_paths.contains(sub_path)) return true;
	1508
	1509	// Check if any included paths are parent directories of sub_path.
	1510	var dirname = sub_path;
	1511	while (std.fs.path.dirname(dirname)) \|next_dirname\| {
	1512	if (self.include_paths.contains(sub_path)) return true;
	1513	dirname = next_dirname;
	1514	}
	1515
	1516	return false;
	1517	}
	1518	};
	1519
	1520	pub fn depDigest(
	1521	pkg_root: Package.Path,
	1522	cache_root: Cache.Directory,
	1523	dep: Manifest.Dependency,
	1524	) ?Manifest.MultiHashHexDigest {
	1525	if (dep.hash) \|h\| return h[0..Manifest.multihash_hex_digest_len].*;
	1526
	1527	switch (dep.location) {
	1528	.url => return null,
	1529	.path => \|rel_path\| {
	1530	var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
	1531	var fba = std.heap.FixedBufferAllocator.init(&buf);
	1532	const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch
	1533	return null;
	1534	return relativePathDigest(new_root, cache_root);
	1535	},
	1536	}
	1537	}
	1538
	1539	// These are random bytes.
	1540	const package_hash_prefix_cached = [8]u8{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 };
	1541	const package_hash_prefix_project = [8]u8{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc };
	1542
	1543	const builtin = @import("builtin");
	1544	const std = @import("std");
	1545	const fs = std.fs;
	1546	const assert = std.debug.assert;
	1547	const ascii = std.ascii;
	1548	const Allocator = std.mem.Allocator;
	1549	const Cache = std.Build.Cache;
	1550	const ThreadPool = std.Thread.Pool;
	1551	const WaitGroup = std.Thread.WaitGroup;
	1552	const Fetch = @This();
	1553	const main = @import("../main.zig");
	1554	const git = @import("Fetch/git.zig");
	1555	const Package = @import("../Package.zig");
	1556	const Manifest = Package.Manifest;
	1557	const ErrorBundle = std.zig.ErrorBundle;

src/Package/Fetch/git.zig created+1466

...	...	@@ -0,0 +1,1466 @@
	1	//! Git support for package fetching.
	2	//!
	3	//! This is not intended to support all features of Git: it is limited to the
	4	//! basic functionality needed to clone a repository for the purpose of fetching
	5	//! a package.
	6
	7	const std = @import("std");
	8	const mem = std.mem;
	9	const testing = std.testing;
	10	const Allocator = mem.Allocator;
	11	const Sha1 = std.crypto.hash.Sha1;
	12	const assert = std.debug.assert;
	13
	14	pub const oid_length = Sha1.digest_length;
	15	pub const fmt_oid_length = 2 * oid_length;
	16	/// The ID of a Git object (an SHA-1 hash).
	17	pub const Oid = [oid_length]u8;
	18
	19	pub fn parseOid(s: []const u8) !Oid {
	20	if (s.len != fmt_oid_length) return error.InvalidOid;
	21	var oid: Oid = undefined;
	22	for (&oid, 0..) \|*b, i\| {
	23	b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid;
	24	}
	25	return oid;
	26	}
	27
	28	test parseOid {
	29	try testing.expectEqualSlices(
	30	u8,
	31	&.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 },
	32	&try parseOid("ce919ccf45951856a762ffdb8ef850301cd8c588"),
	33	);
	34	try testing.expectError(error.InvalidOid, parseOid("ce919ccf"));
	35	try testing.expectError(error.InvalidOid, parseOid("master"));
	36	try testing.expectError(error.InvalidOid, parseOid("HEAD"));
	37	}
	38
	39	pub const Diagnostics = struct {
	40	allocator: Allocator,
	41	errors: std.ArrayListUnmanaged(Error) = .{},
	42
	43	pub const Error = union(enum) {
	44	unable_to_create_sym_link: struct {
	45	code: anyerror,
	46	file_name: []const u8,
	47	link_name: []const u8,
	48	},
	49	};
	50
	51	pub fn deinit(d: *Diagnostics) void {
	52	for (d.errors.items) \|item\| {
	53	switch (item) {
	54	.unable_to_create_sym_link => \|info\| {
	55	d.allocator.free(info.file_name);
	56	d.allocator.free(info.link_name);
	57	},
	58	}
	59	}
	60	d.errors.deinit(d.allocator);
	61	d.* = undefined;
	62	}
	63	};
	64
	65	pub const Repository = struct {
	66	odb: Odb,
	67
	68	pub fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Repository {
	69	return .{ .odb = try Odb.init(allocator, pack_file, index_file) };
	70	}
	71
	72	pub fn deinit(repository: *Repository) void {
	73	repository.odb.deinit();
	74	repository.* = undefined;
	75	}
	76
	77	/// Checks out the repository at `commit_oid` to `worktree`.
	78	pub fn checkout(
	79	repository: *Repository,
	80	worktree: std.fs.Dir,
	81	commit_oid: Oid,
	82	diagnostics: *Diagnostics,
	83	) !void {
	84	try repository.odb.seekOid(commit_oid);
	85	const tree_oid = tree_oid: {
	86	var commit_object = try repository.odb.readObject();
	87	if (commit_object.type != .commit) return error.NotACommit;
	88	break :tree_oid try getCommitTree(commit_object.data);
	89	};
	90	try repository.checkoutTree(worktree, tree_oid, "", diagnostics);
	91	}
	92
	93	/// Checks out the tree at `tree_oid` to `worktree`.
	94	fn checkoutTree(
	95	repository: *Repository,
	96	dir: std.fs.Dir,
	97	tree_oid: Oid,
	98	current_path: []const u8,
	99	diagnostics: *Diagnostics,
	100	) !void {
	101	try repository.odb.seekOid(tree_oid);
	102	const tree_object = try repository.odb.readObject();
	103	if (tree_object.type != .tree) return error.NotATree;
	104	// The tree object may be evicted from the object cache while we're
	105	// iterating over it, so we can make a defensive copy here to make sure
	106	// it remains valid until we're done with it
	107	const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data);
	108	defer repository.odb.allocator.free(tree_data);
	109
	110	var tree_iter: TreeIterator = .{ .data = tree_data };
	111	while (try tree_iter.next()) \|entry\| {
	112	switch (entry.type) {
	113	.directory => {
	114	try dir.makeDir(entry.name);
	115	var subdir = try dir.openDir(entry.name, .{});
	116	defer subdir.close();
	117	const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name });
	118	defer repository.odb.allocator.free(sub_path);
	119	try repository.checkoutTree(subdir, entry.oid, sub_path, diagnostics);
	120	},
	121	.file => {
	122	var file = try dir.createFile(entry.name, .{});
	123	defer file.close();
	124	try repository.odb.seekOid(entry.oid);
	125	var file_object = try repository.odb.readObject();
	126	if (file_object.type != .blob) return error.InvalidFile;
	127	try file.writeAll(file_object.data);
	128	try file.sync();
	129	},
	130	.symlink => {
	131	try repository.odb.seekOid(entry.oid);
	132	var symlink_object = try repository.odb.readObject();
	133	if (symlink_object.type != .blob) return error.InvalidFile;
	134	const link_name = symlink_object.data;
	135	dir.symLink(link_name, entry.name, .{}) catch \|e\| {
	136	const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name });
	137	errdefer diagnostics.allocator.free(file_name);
	138	const link_name_dup = try diagnostics.allocator.dupe(u8, link_name);
	139	errdefer diagnostics.allocator.free(link_name_dup);
	140	try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{
	141	.code = e,
	142	.file_name = file_name,
	143	.link_name = link_name_dup,
	144	} });
	145	};
	146	},
	147	.gitlink => {
	148	// Consistent with git archive behavior, create the directory but
	149	// do nothing else
	150	try dir.makeDir(entry.name);
	151	},
	152	}
	153	}
	154	}
	155
	156	/// Returns the ID of the tree associated with the given commit (provided as
	157	/// raw object data).
	158	fn getCommitTree(commit_data: []const u8) !Oid {
	159	if (!mem.startsWith(u8, commit_data, "tree ") or
	160	commit_data.len < "tree ".len + fmt_oid_length + "\n".len or
	161	commit_data["tree ".len + fmt_oid_length] != '\n')
	162	{
	163	return error.InvalidCommit;
	164	}
	165	return try parseOid(commit_data["tree ".len..][0..fmt_oid_length]);
	166	}
	167
	168	const TreeIterator = struct {
	169	data: []const u8,
	170	pos: usize = 0,
	171
	172	const Entry = struct {
	173	type: Type,
	174	executable: bool,
	175	name: [:0]const u8,
	176	oid: Oid,
	177
	178	const Type = enum(u4) {
	179	directory = 0o4,
	180	file = 0o10,
	181	symlink = 0o12,
	182	gitlink = 0o16,
	183	};
	184	};
	185
	186	fn next(iterator: *TreeIterator) !?Entry {
	187	if (iterator.pos == iterator.data.len) return null;
	188
	189	const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree;
	190	const mode: packed struct {
	191	permission: u9,
	192	unused: u3,
	193	type: u4,
	194	} = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree);
	195	const @"type" = std.meta.intToEnum(Entry.Type, mode.type) catch return error.InvalidTree;
	196	const executable = switch (mode.permission) {
	197	0 => if (@"type" == .file) return error.InvalidTree else false,
	198	0o644 => if (@"type" != .file) return error.InvalidTree else false,
	199	0o755 => if (@"type" != .file) return error.InvalidTree else true,
	200	else => return error.InvalidTree,
	201	};
	202	iterator.pos = mode_end + 1;
	203
	204	const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree;
	205	const name = iterator.data[iterator.pos..name_end :0];
	206	iterator.pos = name_end + 1;
	207
	208	if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree;
	209	const oid = iterator.data[iterator.pos..][0..oid_length].*;
	210	iterator.pos += oid_length;
	211
	212	return .{ .type = @"type", .executable = executable, .name = name, .oid = oid };
	213	}
	214	};
	215	};
	216
	217	/// A Git object database backed by a packfile. A packfile index is also used
	218	/// for efficient access to objects in the packfile.
	219	///
	220	/// The format of the packfile and its associated index are documented in
	221	/// [pack-format](https://git-scm.com/docs/pack-format).
	222	const Odb = struct {
	223	pack_file: std.fs.File,
	224	index_header: IndexHeader,
	225	index_file: std.fs.File,
	226	cache: ObjectCache = .{},
	227	allocator: Allocator,
	228
	229	/// Initializes the database from open pack and index files.
	230	fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Odb {
	231	try pack_file.seekTo(0);
	232	try index_file.seekTo(0);
	233	const index_header = try IndexHeader.read(index_file.reader());
	234	return .{
	235	.pack_file = pack_file,
	236	.index_header = index_header,
	237	.index_file = index_file,
	238	.allocator = allocator,
	239	};
	240	}
	241
	242	fn deinit(odb: *Odb) void {
	243	odb.cache.deinit(odb.allocator);
	244	odb.* = undefined;
	245	}
	246
	247	/// Reads the object at the current position in the database.
	248	fn readObject(odb: *Odb) !Object {
	249	var base_offset = try odb.pack_file.getPos();
	250	var base_header: EntryHeader = undefined;
	251	var delta_offsets = std.ArrayListUnmanaged(u64){};
	252	defer delta_offsets.deinit(odb.allocator);
	253	const base_object = while (true) {
	254	if (odb.cache.get(base_offset)) \|base_object\| break base_object;
	255
	256	base_header = try EntryHeader.read(odb.pack_file.reader());
	257	switch (base_header) {
	258	.ofs_delta => \|ofs_delta\| {
	259	try delta_offsets.append(odb.allocator, base_offset);
	260	base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat;
	261	try odb.pack_file.seekTo(base_offset);
	262	},
	263	.ref_delta => \|ref_delta\| {
	264	try delta_offsets.append(odb.allocator, base_offset);
	265	try odb.seekOid(ref_delta.base_object);
	266	base_offset = try odb.pack_file.getPos();
	267	},
	268	else => {
	269	const base_data = try readObjectRaw(odb.allocator, odb.pack_file.reader(), base_header.uncompressedLength());
	270	errdefer odb.allocator.free(base_data);
	271	const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
	272	try odb.cache.put(odb.allocator, base_offset, base_object);
	273	break base_object;
	274	},
	275	}
	276	};
	277
	278	const base_data = try resolveDeltaChain(
	279	odb.allocator,
	280	odb.pack_file,
	281	base_object,
	282	delta_offsets.items,
	283	&odb.cache,
	284	);
	285
	286	return .{ .type = base_object.type, .data = base_data };
	287	}
	288
	289	/// Seeks to the beginning of the object with the given ID.
	290	fn seekOid(odb: *Odb, oid: Oid) !void {
	291	const key = oid[0];
	292	var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0;
	293	var end_index = odb.index_header.fan_out_table[key];
	294	const found_index = while (start_index < end_index) {
	295	const mid_index = start_index + (end_index - start_index) / 2;
	296	try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length);
	297	const mid_oid = try odb.index_file.reader().readBytesNoEof(oid_length);
	298	switch (mem.order(u8, &mid_oid, &oid)) {
	299	.lt => start_index = mid_index + 1,
	300	.gt => end_index = mid_index,
	301	.eq => break mid_index,
	302	}
	303	} else return error.ObjectNotFound;
	304
	305	const n_objects = odb.index_header.fan_out_table[255];
	306	const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4);
	307	try odb.index_file.seekTo(offset_values_start + found_index * 4);
	308	const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.reader().readIntBig(u32));
	309	const pack_offset = pack_offset: {
	310	if (l1_offset.big) {
	311	const l2_offset_values_start = offset_values_start + n_objects * 4;
	312	try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4);
	313	break :pack_offset try odb.index_file.reader().readIntBig(u64);
	314	} else {
	315	break :pack_offset l1_offset.value;
	316	}
	317	};
	318
	319	try odb.pack_file.seekTo(pack_offset);
	320	}
	321	};
	322
	323	const Object = struct {
	324	type: Type,
	325	data: []const u8,
	326
	327	const Type = enum {
	328	commit,
	329	tree,
	330	blob,
	331	tag,
	332	};
	333	};
	334
	335	/// A cache for object data.
	336	///
	337	/// The purpose of this cache is to speed up resolution of deltas by caching the
	338	/// results of resolving delta objects, while maintaining a maximum cache size
	339	/// to avoid excessive memory usage. If the total size of the objects in the
	340	/// cache exceeds the maximum, the cache will begin evicting the least recently
	341	/// used objects: when resolving delta chains, the most recently used objects
	342	/// will likely be more helpful as they will be further along in the chain
	343	/// (skipping earlier reconstruction steps).
	344	///
	345	/// Object data stored in the cache is managed by the cache. It should not be
	346	/// freed by the caller at any point after inserting it into the cache. Any
	347	/// objects remaining in the cache will be freed when the cache itself is freed.
	348	const ObjectCache = struct {
	349	objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .{},
	350	lru_nodes: LruList = .{},
	351	byte_size: usize = 0,
	352
	353	const max_byte_size = 128 * 1024 * 1024; // 128MiB
	354	/// A list of offsets stored in the cache, with the most recently used
	355	/// entries at the end.
	356	const LruList = std.DoublyLinkedList(u64);
	357	const CacheEntry = struct { object: Object, lru_node: *LruList.Node };
	358
	359	fn deinit(cache: *ObjectCache, allocator: Allocator) void {
	360	var object_iterator = cache.objects.iterator();
	361	while (object_iterator.next()) \|object\| {
	362	allocator.free(object.value_ptr.object.data);
	363	allocator.destroy(object.value_ptr.lru_node);
	364	}
	365	cache.objects.deinit(allocator);
	366	cache.* = undefined;
	367	}
	368
	369	/// Gets an object from the cache, moving it to the most recently used
	370	/// position if it is present.
	371	fn get(cache: *ObjectCache, offset: u64) ?Object {
	372	if (cache.objects.get(offset)) \|entry\| {
	373	cache.lru_nodes.remove(entry.lru_node);
	374	cache.lru_nodes.append(entry.lru_node);
	375	return entry.object;
	376	} else {
	377	return null;
	378	}
	379	}
	380
	381	/// Puts an object in the cache, possibly evicting older entries if the
	382	/// cache exceeds its maximum size. Note that, although old objects may
	383	/// be evicted, the object just added to the cache with this function
	384	/// will not be evicted before the next call to `put` or `deinit` even if
	385	/// it exceeds the maximum cache size.
	386	fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void {
	387	const lru_node = try allocator.create(LruList.Node);
	388	errdefer allocator.destroy(lru_node);
	389	lru_node.data = offset;
	390
	391	const gop = try cache.objects.getOrPut(allocator, offset);
	392	if (gop.found_existing) {
	393	cache.byte_size -= gop.value_ptr.object.data.len;
	394	cache.lru_nodes.remove(gop.value_ptr.lru_node);
	395	allocator.destroy(gop.value_ptr.lru_node);
	396	allocator.free(gop.value_ptr.object.data);
	397	}
	398	gop.value_ptr.* = .{ .object = object, .lru_node = lru_node };
	399	cache.byte_size += object.data.len;
	400	cache.lru_nodes.append(lru_node);
	401
	402	while (cache.byte_size > max_byte_size and cache.lru_nodes.len > 1) {
	403	// The > 1 check is to make sure that we don't evict the most
	404	// recently added node, even if it by itself happens to exceed the
	405	// maximum size of the cache.
	406	const evict_node = cache.lru_nodes.popFirst().?;
	407	const evict_offset = evict_node.data;
	408	allocator.destroy(evict_node);
	409	const evict_object = cache.objects.get(evict_offset).?.object;
	410	cache.byte_size -= evict_object.data.len;
	411	allocator.free(evict_object.data);
	412	_ = cache.objects.remove(evict_offset);
	413	}
	414	}
	415	};
	416
	417	/// A single pkt-line in the Git protocol.
	418	///
	419	/// The format of a pkt-line is documented in
	420	/// [protocol-common](https://git-scm.com/docs/protocol-common). The special
	421	/// meanings of the delimiter and response-end packets are documented in
	422	/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
	423	const Packet = union(enum) {
	424	flush,
	425	delimiter,
	426	response_end,
	427	data: []const u8,
	428
	429	const max_data_length = 65516;
	430
	431	/// Reads a packet in pkt-line format.
	432	fn read(reader: anytype, buf: *[max_data_length]u8) !Packet {
	433	const length = std.fmt.parseUnsigned(u16, &try reader.readBytesNoEof(4), 16) catch return error.InvalidPacket;
	434	switch (length) {
	435	0 => return .flush,
	436	1 => return .delimiter,
	437	2 => return .response_end,
	438	3 => return error.InvalidPacket,
	439	else => if (length - 4 > max_data_length) return error.InvalidPacket,
	440	}
	441	const data = buf[0 .. length - 4];
	442	try reader.readNoEof(data);
	443	return .{ .data = data };
	444	}
	445
	446	/// Writes a packet in pkt-line format.
	447	fn write(packet: Packet, writer: anytype) !void {
	448	switch (packet) {
	449	.flush => try writer.writeAll("0000"),
	450	.delimiter => try writer.writeAll("0001"),
	451	.response_end => try writer.writeAll("0002"),
	452	.data => \|data\| {
	453	assert(data.len <= max_data_length);
	454	try writer.print("{x:0>4}", .{data.len + 4});
	455	try writer.writeAll(data);
	456	},
	457	}
	458	}
	459	};
	460
	461	/// A client session for the Git protocol, currently limited to an HTTP(S)
	462	/// transport. Only protocol version 2 is supported, as documented in
	463	/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
	464	pub const Session = struct {
	465	transport: *std.http.Client,
	466	uri: std.Uri,
	467	supports_agent: bool = false,
	468	supports_shallow: bool = false,
	469
	470	const agent = "zig/" ++ @import("builtin").zig_version_string;
	471	const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent});
	472
	473	/// Discovers server capabilities. This should be called before using any
	474	/// other client functionality, or the client will be forced to default to
	475	/// the bare minimum server requirements, which may be considerably less
	476	/// efficient (e.g. no shallow fetches).
	477	///
	478	/// See the note on `getCapabilities` regarding `redirect_uri`.
	479	pub fn discoverCapabilities(
	480	session: *Session,
	481	allocator: Allocator,
	482	redirect_uri: *[]u8,
	483	) !void {
	484	var capability_iterator = try session.getCapabilities(allocator, redirect_uri);
	485	defer capability_iterator.deinit();
	486	while (try capability_iterator.next()) \|capability\| {
	487	if (mem.eql(u8, capability.key, "agent")) {
	488	session.supports_agent = true;
	489	} else if (mem.eql(u8, capability.key, "fetch")) {
	490	var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' ');
	491	while (feature_iterator.next()) \|feature\| {
	492	if (mem.eql(u8, feature, "shallow")) {
	493	session.supports_shallow = true;
	494	}
	495	}
	496	}
	497	}
	498	}
	499
	500	/// Returns an iterator over capabilities supported by the server.
	501	///
	502	/// If the server redirects the request, `error.Redirected` is returned and
	503	/// `redirect_uri` is populated with the URI resulting from the redirects.
	504	/// When this occurs, the value of `redirect_uri` must be freed with
	505	/// `allocator` when the caller is done with it.
	506	fn getCapabilities(
	507	session: Session,
	508	allocator: Allocator,
	509	redirect_uri: *[]u8,
	510	) !CapabilityIterator {
	511	var info_refs_uri = session.uri;
	512	info_refs_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "info/refs" });
	513	defer allocator.free(info_refs_uri.path);
	514	info_refs_uri.query = "service=git-upload-pack";
	515	info_refs_uri.fragment = null;
	516
	517	var headers = std.http.Headers.init(allocator);
	518	defer headers.deinit();
	519	try headers.append("Git-Protocol", "version=2");
	520
	521	var request = try session.transport.request(.GET, info_refs_uri, headers, .{
	522	.max_redirects = 3,
	523	});
	524	errdefer request.deinit();
	525	try request.start(.{});
	526	try request.finish();
	527
	528	try request.wait();
	529	if (request.response.status != .ok) return error.ProtocolError;
	530	if (request.redirects_left < 3) {
	531	if (!mem.endsWith(u8, request.uri.path, "/info/refs")) return error.UnparseableRedirect;
	532	var new_uri = request.uri;
	533	new_uri.path = new_uri.path[0 .. new_uri.path.len - "/info/refs".len];
	534	new_uri.query = null;
	535	redirect_uri.* = try std.fmt.allocPrint(allocator, "{+/}", .{new_uri});
	536	return error.Redirected;
	537	}
	538
	539	const reader = request.reader();
	540	var buf: [Packet.max_data_length]u8 = undefined;
	541	var state: enum { response_start, response_content } = .response_start;
	542	while (true) {
	543	// Some Git servers (at least GitHub) include an additional
	544	// '# service=git-upload-pack' informative response before sending
	545	// the expected 'version 2' packet and capability information.
	546	// This is not universal: SourceHut, for example, does not do this.
	547	// Thus, we need to skip any such useless additional responses
	548	// before we get the one we're actually looking for. The responses
	549	// will be delimited by flush packets.
	550	const packet = Packet.read(reader, &buf) catch \|e\| switch (e) {
	551	error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found
	552	else => \|other\| return other,
	553	};
	554	switch (packet) {
	555	.flush => state = .response_start,
	556	.data => \|data\| switch (state) {
	557	.response_start => if (mem.eql(u8, data, "version 2\n")) {
	558	return .{ .request = request };
	559	} else {
	560	state = .response_content;
	561	},
	562	else => {},
	563	},
	564	else => return error.UnexpectedPacket,
	565	}
	566	}
	567	}
	568
	569	const CapabilityIterator = struct {
	570	request: std.http.Client.Request,
	571	buf: [Packet.max_data_length]u8 = undefined,
	572
	573	const Capability = struct {
	574	key: []const u8,
	575	value: ?[]const u8 = null,
	576	};
	577
	578	fn deinit(iterator: *CapabilityIterator) void {
	579	iterator.request.deinit();
	580	iterator.* = undefined;
	581	}
	582
	583	fn next(iterator: *CapabilityIterator) !?Capability {
	584	switch (try Packet.read(iterator.request.reader(), &iterator.buf)) {
	585	.flush => return null,
	586	.data => \|data\| if (data.len > 0 and data[data.len - 1] == '\n') {
	587	if (mem.indexOfScalar(u8, data, '=')) \|separator_pos\| {
	588	return .{ .key = data[0..separator_pos], .value = data[separator_pos + 1 .. data.len - 1] };
	589	} else {
	590	return .{ .key = data[0 .. data.len - 1] };
	591	}
	592	} else return error.UnexpectedPacket,
	593	else => return error.UnexpectedPacket,
	594	}
	595	}
	596	};
	597
	598	const ListRefsOptions = struct {
	599	/// The ref prefixes (if any) to use to filter the refs available on the
	600	/// server. Note that the client must still check the returned refs
	601	/// against its desired filters itself: the server is not required to
	602	/// respect these prefix filters and may return other refs as well.
	603	ref_prefixes: []const []const u8 = &.{},
	604	/// Whether to include symref targets for returned symbolic refs.
	605	include_symrefs: bool = false,
	606	/// Whether to include the peeled object ID for returned tag refs.
	607	include_peeled: bool = false,
	608	};
	609
	610	/// Returns an iterator over refs known to the server.
	611	pub fn listRefs(session: Session, allocator: Allocator, options: ListRefsOptions) !RefIterator {
	612	var upload_pack_uri = session.uri;
	613	upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" });
	614	defer allocator.free(upload_pack_uri.path);
	615	upload_pack_uri.query = null;
	616	upload_pack_uri.fragment = null;
	617
	618	var headers = std.http.Headers.init(allocator);
	619	defer headers.deinit();
	620	try headers.append("Content-Type", "application/x-git-upload-pack-request");
	621	try headers.append("Git-Protocol", "version=2");
	622
	623	var body = std.ArrayListUnmanaged(u8){};
	624	defer body.deinit(allocator);
	625	const body_writer = body.writer(allocator);
	626	try Packet.write(.{ .data = "command=ls-refs\n" }, body_writer);
	627	if (session.supports_agent) {
	628	try Packet.write(.{ .data = agent_capability }, body_writer);
	629	}
	630	try Packet.write(.delimiter, body_writer);
	631	for (options.ref_prefixes) \|ref_prefix\| {
	632	const ref_prefix_packet = try std.fmt.allocPrint(allocator, "ref-prefix {s}\n", .{ref_prefix});
	633	defer allocator.free(ref_prefix_packet);
	634	try Packet.write(.{ .data = ref_prefix_packet }, body_writer);
	635	}
	636	if (options.include_symrefs) {
	637	try Packet.write(.{ .data = "symrefs\n" }, body_writer);
	638	}
	639	if (options.include_peeled) {
	640	try Packet.write(.{ .data = "peel\n" }, body_writer);
	641	}
	642	try Packet.write(.flush, body_writer);
	643
	644	var request = try session.transport.request(.POST, upload_pack_uri, headers, .{
	645	.handle_redirects = false,
	646	});
	647	errdefer request.deinit();
	648	request.transfer_encoding = .{ .content_length = body.items.len };
	649	try request.start(.{});
	650	try request.writeAll(body.items);
	651	try request.finish();
	652
	653	try request.wait();
	654	if (request.response.status != .ok) return error.ProtocolError;
	655
	656	return .{ .request = request };
	657	}
	658
	659	pub const RefIterator = struct {
	660	request: std.http.Client.Request,
	661	buf: [Packet.max_data_length]u8 = undefined,
	662
	663	pub const Ref = struct {
	664	oid: Oid,
	665	name: []const u8,
	666	symref_target: ?[]const u8,
	667	peeled: ?Oid,
	668	};
	669
	670	pub fn deinit(iterator: *RefIterator) void {
	671	iterator.request.deinit();
	672	iterator.* = undefined;
	673	}
	674
	675	pub fn next(iterator: *RefIterator) !?Ref {
	676	switch (try Packet.read(iterator.request.reader(), &iterator.buf)) {
	677	.flush => return null,
	678	.data => \|data\| {
	679	const oid_sep_pos = mem.indexOfScalar(u8, data, ' ') orelse return error.InvalidRefPacket;
	680	const oid = parseOid(data[0..oid_sep_pos]) catch return error.InvalidRefPacket;
	681
	682	const name_sep_pos = mem.indexOfAnyPos(u8, data, oid_sep_pos + 1, " \n") orelse return error.InvalidRefPacket;
	683	const name = data[oid_sep_pos + 1 .. name_sep_pos];
	684
	685	var symref_target: ?[]const u8 = null;
	686	var peeled: ?Oid = null;
	687	var last_sep_pos = name_sep_pos;
	688	while (data[last_sep_pos] == ' ') {
	689	const next_sep_pos = mem.indexOfAnyPos(u8, data, last_sep_pos + 1, " \n") orelse return error.InvalidRefPacket;
	690	const attribute = data[last_sep_pos + 1 .. next_sep_pos];
	691	if (mem.startsWith(u8, attribute, "symref-target:")) {
	692	symref_target = attribute["symref-target:".len..];
	693	} else if (mem.startsWith(u8, attribute, "peeled:")) {
	694	peeled = parseOid(attribute["peeled:".len..]) catch return error.InvalidRefPacket;
	695	}
	696	last_sep_pos = next_sep_pos;
	697	}
	698
	699	return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled };
	700	},
	701	else => return error.UnexpectedPacket,
	702	}
	703	}
	704	};
	705
	706	/// Fetches the given refs from the server. A shallow fetch (depth 1) is
	707	/// performed if the server supports it.
	708	pub fn fetch(session: Session, allocator: Allocator, wants: []const []const u8) !FetchStream {
	709	var upload_pack_uri = session.uri;
	710	upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" });
	711	defer allocator.free(upload_pack_uri.path);
	712	upload_pack_uri.query = null;
	713	upload_pack_uri.fragment = null;
	714
	715	var headers = std.http.Headers.init(allocator);
	716	defer headers.deinit();
	717	try headers.append("Content-Type", "application/x-git-upload-pack-request");
	718	try headers.append("Git-Protocol", "version=2");
	719
	720	var body = std.ArrayListUnmanaged(u8){};
	721	defer body.deinit(allocator);
	722	const body_writer = body.writer(allocator);
	723	try Packet.write(.{ .data = "command=fetch\n" }, body_writer);
	724	if (session.supports_agent) {
	725	try Packet.write(.{ .data = agent_capability }, body_writer);
	726	}
	727	try Packet.write(.delimiter, body_writer);
	728	// Our packfile parser supports the OFS_DELTA object type
	729	try Packet.write(.{ .data = "ofs-delta\n" }, body_writer);
	730	// We do not currently convey server progress information to the user
	731	try Packet.write(.{ .data = "no-progress\n" }, body_writer);
	732	if (session.supports_shallow) {
	733	try Packet.write(.{ .data = "deepen 1\n" }, body_writer);
	734	}
	735	for (wants) \|want\| {
	736	var buf: [Packet.max_data_length]u8 = undefined;
	737	const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable;
	738	try Packet.write(.{ .data = arg }, body_writer);
	739	}
	740	try Packet.write(.{ .data = "done\n" }, body_writer);
	741	try Packet.write(.flush, body_writer);
	742
	743	var request = try session.transport.request(.POST, upload_pack_uri, headers, .{
	744	.handle_redirects = false,
	745	});
	746	errdefer request.deinit();
	747	request.transfer_encoding = .{ .content_length = body.items.len };
	748	try request.start(.{});
	749	try request.writeAll(body.items);
	750	try request.finish();
	751
	752	try request.wait();
	753	if (request.response.status != .ok) return error.ProtocolError;
	754
	755	const reader = request.reader();
	756	// We are not interested in any of the sections of the returned fetch
	757	// data other than the packfile section, since we aren't doing anything
	758	// complex like ref negotiation (this is a fresh clone).
	759	var state: enum { section_start, section_content } = .section_start;
	760	while (true) {
	761	var buf: [Packet.max_data_length]u8 = undefined;
	762	const packet = try Packet.read(reader, &buf);
	763	switch (state) {
	764	.section_start => switch (packet) {
	765	.data => \|data\| if (mem.eql(u8, data, "packfile\n")) {
	766	return .{ .request = request };
	767	} else {
	768	state = .section_content;
	769	},
	770	else => return error.UnexpectedPacket,
	771	},
	772	.section_content => switch (packet) {
	773	.delimiter => state = .section_start,
	774	.data => {},
	775	else => return error.UnexpectedPacket,
	776	},
	777	}
	778	}
	779	}
	780
	781	pub const FetchStream = struct {
	782	request: std.http.Client.Request,
	783	buf: [Packet.max_data_length]u8 = undefined,
	784	pos: usize = 0,
	785	len: usize = 0,
	786
	787	pub fn deinit(stream: *FetchStream) void {
	788	stream.request.deinit();
	789	}
	790
	791	pub const ReadError = std.http.Client.Request.ReadError \|\| error{
	792	InvalidPacket,
	793	ProtocolError,
	794	UnexpectedPacket,
	795	};
	796	pub const Reader = std.io.Reader(*FetchStream, ReadError, read);
	797
	798	const StreamCode = enum(u8) {
	799	pack_data = 1,
	800	progress = 2,
	801	fatal_error = 3,
	802	_,
	803	};
	804
	805	pub fn reader(stream: *FetchStream) Reader {
	806	return .{ .context = stream };
	807	}
	808
	809	pub fn read(stream: *FetchStream, buf: []u8) !usize {
	810	if (stream.pos == stream.len) {
	811	while (true) {
	812	switch (try Packet.read(stream.request.reader(), &stream.buf)) {
	813	.flush => return 0,
	814	.data => \|data\| if (data.len > 1) switch (@as(StreamCode, @enumFromInt(data[0]))) {
	815	.pack_data => {
	816	stream.pos = 1;
	817	stream.len = data.len;
	818	break;
	819	},
	820	.fatal_error => return error.ProtocolError,
	821	else => {},
	822	},
	823	else => return error.UnexpectedPacket,
	824	}
	825	}
	826	}
	827
	828	const size = @min(buf.len, stream.len - stream.pos);
	829	@memcpy(buf[0..size], stream.buf[stream.pos .. stream.pos + size]);
	830	stream.pos += size;
	831	return size;
	832	}
	833	};
	834	};
	835
	836	const PackHeader = struct {
	837	total_objects: u32,
	838
	839	const signature = "PACK";
	840	const supported_version = 2;
	841
	842	fn read(reader: anytype) !PackHeader {
	843	const actual_signature = reader.readBytesNoEof(4) catch \|e\| switch (e) {
	844	error.EndOfStream => return error.InvalidHeader,
	845	else => \|other\| return other,
	846	};
	847	if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader;
	848	const version = reader.readIntBig(u32) catch \|e\| switch (e) {
	849	error.EndOfStream => return error.InvalidHeader,
	850	else => \|other\| return other,
	851	};
	852	if (version != supported_version) return error.UnsupportedVersion;
	853	const total_objects = reader.readIntBig(u32) catch \|e\| switch (e) {
	854	error.EndOfStream => return error.InvalidHeader,
	855	else => \|other\| return other,
	856	};
	857	return .{ .total_objects = total_objects };
	858	}
	859	};
	860
	861	const EntryHeader = union(Type) {
	862	commit: Undeltified,
	863	tree: Undeltified,
	864	blob: Undeltified,
	865	tag: Undeltified,
	866	ofs_delta: OfsDelta,
	867	ref_delta: RefDelta,
	868
	869	const Type = enum(u3) {
	870	commit = 1,
	871	tree = 2,
	872	blob = 3,
	873	tag = 4,
	874	ofs_delta = 6,
	875	ref_delta = 7,
	876	};
	877
	878	const Undeltified = struct {
	879	uncompressed_length: u64,
	880	};
	881
	882	const OfsDelta = struct {
	883	offset: u64,
	884	uncompressed_length: u64,
	885	};
	886
	887	const RefDelta = struct {
	888	base_object: Oid,
	889	uncompressed_length: u64,
	890	};
	891
	892	fn objectType(header: EntryHeader) Object.Type {
	893	return switch (header) {
	894	inline .commit, .tree, .blob, .tag => \|_, tag\| @field(Object.Type, @tagName(tag)),
	895	else => unreachable,
	896	};
	897	}
	898
	899	fn uncompressedLength(header: EntryHeader) u64 {
	900	return switch (header) {
	901	inline else => \|entry\| entry.uncompressed_length,
	902	};
	903	}
	904
	905	fn read(reader: anytype) !EntryHeader {
	906	const InitialByte = packed struct { len: u4, type: u3, has_next: bool };
	907	const initial: InitialByte = @bitCast(reader.readByte() catch \|e\| switch (e) {
	908	error.EndOfStream => return error.InvalidFormat,
	909	else => \|other\| return other,
	910	});
	911	const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0;
	912	var uncompressed_length: u64 = initial.len;
	913	uncompressed_length \|= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat;
	914	const @"type" = std.meta.intToEnum(EntryHeader.Type, initial.type) catch return error.InvalidFormat;
	915	return switch (@"type") {
	916	inline .commit, .tree, .blob, .tag => \|tag\| @unionInit(EntryHeader, @tagName(tag), .{
	917	.uncompressed_length = uncompressed_length,
	918	}),
	919	.ofs_delta => .{ .ofs_delta = .{
	920	.offset = try readOffsetVarInt(reader),
	921	.uncompressed_length = uncompressed_length,
	922	} },
	923	.ref_delta => .{ .ref_delta = .{
	924	.base_object = reader.readBytesNoEof(oid_length) catch \|e\| switch (e) {
	925	error.EndOfStream => return error.InvalidFormat,
	926	else => \|other\| return other,
	927	},
	928	.uncompressed_length = uncompressed_length,
	929	} },
	930	};
	931	}
	932	};
	933
	934	fn readSizeVarInt(r: anytype) !u64 {
	935	const Byte = packed struct { value: u7, has_next: bool };
	936	var b: Byte = @bitCast(try r.readByte());
	937	var value: u64 = b.value;
	938	var shift: u6 = 0;
	939	while (b.has_next) {
	940	b = @bitCast(try r.readByte());
	941	shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat;
	942	value \|= @as(u64, b.value) << shift;
	943	}
	944	return value;
	945	}
	946
	947	fn readOffsetVarInt(r: anytype) !u64 {
	948	const Byte = packed struct { value: u7, has_next: bool };
	949	var b: Byte = @bitCast(try r.readByte());
	950	var value: u64 = b.value;
	951	while (b.has_next) {
	952	b = @bitCast(try r.readByte());
	953	value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat;
	954	value \|= b.value;
	955	}
	956	return value;
	957	}
	958
	959	const IndexHeader = struct {
	960	fan_out_table: [256]u32,
	961
	962	const signature = "\xFFtOc";
	963	const supported_version = 2;
	964	const size = 4 + 4 + @sizeOf([256]u32);
	965
	966	fn read(reader: anytype) !IndexHeader {
	967	var header_bytes = try reader.readBytesNoEof(size);
	968	if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader;
	969	const version = mem.readIntBig(u32, header_bytes[4..8]);
	970	if (version != supported_version) return error.UnsupportedVersion;
	971
	972	var fan_out_table: [256]u32 = undefined;
	973	var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]);
	974	const fan_out_table_reader = fan_out_table_stream.reader();
	975	for (&fan_out_table) \|*entry\| {
	976	entry.* = fan_out_table_reader.readIntBig(u32) catch unreachable;
	977	}
	978	return .{ .fan_out_table = fan_out_table };
	979	}
	980	};
	981
	982	const IndexEntry = struct {
	983	offset: u64,
	984	crc32: u32,
	985	};
	986
	987	/// Writes out a version 2 index for the given packfile, as documented in
	988	/// [pack-format](https://git-scm.com/docs/pack-format).
	989	pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) !void {
	990	try pack.seekTo(0);
	991
	992	var index_entries = std.AutoHashMapUnmanaged(Oid, IndexEntry){};
	993	defer index_entries.deinit(allocator);
	994	var pending_deltas = std.ArrayListUnmanaged(IndexEntry){};
	995	defer pending_deltas.deinit(allocator);
	996
	997	const pack_checksum = try indexPackFirstPass(allocator, pack, &index_entries, &pending_deltas);
	998
	999	var cache: ObjectCache = .{};
	1000	defer cache.deinit(allocator);
	1001	var remaining_deltas = pending_deltas.items.len;
	1002	while (remaining_deltas > 0) {
	1003	var i: usize = remaining_deltas;
	1004	while (i > 0) {
	1005	i -= 1;
	1006	const delta = pending_deltas.items[i];
	1007	if (try indexPackHashDelta(allocator, pack, delta, index_entries, &cache)) \|oid\| {
	1008	try index_entries.put(allocator, oid, delta);
	1009	_ = pending_deltas.swapRemove(i);
	1010	}
	1011	}
	1012	if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack;
	1013	remaining_deltas = pending_deltas.items.len;
	1014	}
	1015
	1016	var oids = std.ArrayListUnmanaged(Oid){};
	1017	defer oids.deinit(allocator);
	1018	try oids.ensureTotalCapacityPrecise(allocator, index_entries.count());
	1019	var index_entries_iter = index_entries.iterator();
	1020	while (index_entries_iter.next()) \|entry\| {
	1021	oids.appendAssumeCapacity(entry.key_ptr.*);
	1022	}
	1023	mem.sortUnstable(Oid, oids.items, {}, struct {
	1024	fn lessThan(_: void, o1: Oid, o2: Oid) bool {
	1025	return mem.lessThan(u8, &o1, &o2);
	1026	}
	1027	}.lessThan);
	1028
	1029	var fan_out_table: [256]u32 = undefined;
	1030	var count: u32 = 0;
	1031	var fan_out_index: u8 = 0;
	1032	for (oids.items) \|oid\| {
	1033	if (oid[0] > fan_out_index) {
	1034	@memset(fan_out_table[fan_out_index..oid[0]], count);
	1035	fan_out_index = oid[0];
	1036	}
	1037	count += 1;
	1038	}
	1039	@memset(fan_out_table[fan_out_index..], count);
	1040
	1041	var index_hashed_writer = hashedWriter(index_writer, Sha1.init(.{}));
	1042	const writer = index_hashed_writer.writer();
	1043	try writer.writeAll(IndexHeader.signature);
	1044	try writer.writeIntBig(u32, IndexHeader.supported_version);
	1045	for (fan_out_table) \|fan_out_entry\| {
	1046	try writer.writeIntBig(u32, fan_out_entry);
	1047	}
	1048
	1049	for (oids.items) \|oid\| {
	1050	try writer.writeAll(&oid);
	1051	}
	1052
	1053	for (oids.items) \|oid\| {
	1054	try writer.writeIntBig(u32, index_entries.get(oid).?.crc32);
	1055	}
	1056
	1057	var big_offsets = std.ArrayListUnmanaged(u64){};
	1058	defer big_offsets.deinit(allocator);
	1059	for (oids.items) \|oid\| {
	1060	const offset = index_entries.get(oid).?.offset;
	1061	if (offset <= std.math.maxInt(u31)) {
	1062	try writer.writeIntBig(u32, @intCast(offset));
	1063	} else {
	1064	const index = big_offsets.items.len;
	1065	try big_offsets.append(allocator, offset);
	1066	try writer.writeIntBig(u32, @as(u32, @intCast(index)) \| (1 << 31));
	1067	}
	1068	}
	1069	for (big_offsets.items) \|offset\| {
	1070	try writer.writeIntBig(u64, offset);
	1071	}
	1072
	1073	try writer.writeAll(&pack_checksum);
	1074	const index_checksum = index_hashed_writer.hasher.finalResult();
	1075	try index_writer.writeAll(&index_checksum);
	1076	}
	1077
	1078	/// Performs the first pass over the packfile data for index construction.
	1079	/// This will index all non-delta objects, queue delta objects for further
	1080	/// processing, and return the pack checksum (which is part of the index
	1081	/// format).
	1082	fn indexPackFirstPass(
	1083	allocator: Allocator,
	1084	pack: std.fs.File,
	1085	index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),
	1086	pending_deltas: *std.ArrayListUnmanaged(IndexEntry),
	1087	) ![Sha1.digest_length]u8 {
	1088	var pack_buffered_reader = std.io.bufferedReader(pack.reader());
	1089	var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader());
	1090	var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Sha1.init(.{}));
	1091	const pack_reader = pack_hashed_reader.reader();
	1092
	1093	const pack_header = try PackHeader.read(pack_reader);
	1094
	1095	var current_entry: u32 = 0;
	1096	while (current_entry < pack_header.total_objects) : (current_entry += 1) {
	1097	const entry_offset = pack_counting_reader.bytes_read;
	1098	var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init());
	1099	const entry_header = try EntryHeader.read(entry_crc32_reader.reader());
	1100	switch (entry_header) {
	1101	inline .commit, .tree, .blob, .tag => \|object, tag\| {
	1102	var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader());
	1103	defer entry_decompress_stream.deinit();
	1104	var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
	1105	var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{}));
	1106	const entry_writer = entry_hashed_writer.writer();
	1107	// The object header is not included in the pack data but is
	1108	// part of the object's ID
	1109	try entry_writer.print("{s} {}\x00", .{ @tagName(tag), object.uncompressed_length });
	1110	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
	1111	try fifo.pump(entry_counting_reader.reader(), entry_writer);
	1112	if (entry_counting_reader.bytes_read != object.uncompressed_length) {
	1113	return error.InvalidObject;
	1114	}
	1115	const oid = entry_hashed_writer.hasher.finalResult();
	1116	try index_entries.put(allocator, oid, .{
	1117	.offset = entry_offset,
	1118	.crc32 = entry_crc32_reader.hasher.final(),
	1119	});
	1120	},
	1121	inline .ofs_delta, .ref_delta => \|delta\| {
	1122	var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader());
	1123	defer entry_decompress_stream.deinit();
	1124	var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
	1125	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
	1126	try fifo.pump(entry_counting_reader.reader(), std.io.null_writer);
	1127	if (entry_counting_reader.bytes_read != delta.uncompressed_length) {
	1128	return error.InvalidObject;
	1129	}
	1130	try pending_deltas.append(allocator, .{
	1131	.offset = entry_offset,
	1132	.crc32 = entry_crc32_reader.hasher.final(),
	1133	});
	1134	},
	1135	}
	1136	}
	1137
	1138	const pack_checksum = pack_hashed_reader.hasher.finalResult();
	1139	const recorded_checksum = try pack_buffered_reader.reader().readBytesNoEof(Sha1.digest_length);
	1140	if (!mem.eql(u8, &pack_checksum, &recorded_checksum)) {
	1141	return error.CorruptedPack;
	1142	}
	1143	_ = pack_buffered_reader.reader().readByte() catch \|e\| switch (e) {
	1144	error.EndOfStream => return pack_checksum,
	1145	else => \|other\| return other,
	1146	};
	1147	return error.InvalidFormat;
	1148	}
	1149
	1150	/// Attempts to determine the final object ID of the given deltified object.
	1151	/// May return null if this is not yet possible (if the delta is a ref-based
	1152	/// delta and we do not yet know the offset of the base object).
	1153	fn indexPackHashDelta(
	1154	allocator: Allocator,
	1155	pack: std.fs.File,
	1156	delta: IndexEntry,
	1157	index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),
	1158	cache: *ObjectCache,
	1159	) !?Oid {
	1160	// Figure out the chain of deltas to resolve
	1161	var base_offset = delta.offset;
	1162	var base_header: EntryHeader = undefined;
	1163	var delta_offsets = std.ArrayListUnmanaged(u64){};
	1164	defer delta_offsets.deinit(allocator);
	1165	const base_object = while (true) {
	1166	if (cache.get(base_offset)) \|base_object\| break base_object;
	1167
	1168	try pack.seekTo(base_offset);
	1169	base_header = try EntryHeader.read(pack.reader());
	1170	switch (base_header) {
	1171	.ofs_delta => \|ofs_delta\| {
	1172	try delta_offsets.append(allocator, base_offset);
	1173	base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject;
	1174	},
	1175	.ref_delta => \|ref_delta\| {
	1176	try delta_offsets.append(allocator, base_offset);
	1177	base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset;
	1178	},
	1179	else => {
	1180	const base_data = try readObjectRaw(allocator, pack.reader(), base_header.uncompressedLength());
	1181	errdefer allocator.free(base_data);
	1182	const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
	1183	try cache.put(allocator, base_offset, base_object);
	1184	break base_object;
	1185	},
	1186	}
	1187	};
	1188
	1189	const base_data = try resolveDeltaChain(allocator, pack, base_object, delta_offsets.items, cache);
	1190
	1191	var entry_hasher = Sha1.init(.{});
	1192	var entry_hashed_writer = hashedWriter(std.io.null_writer, &entry_hasher);
	1193	try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len });
	1194	entry_hasher.update(base_data);
	1195	return entry_hasher.finalResult();
	1196	}
	1197
	1198	/// Resolves a chain of deltas, returning the final base object data. `pack` is
	1199	/// assumed to be looking at the start of the object data for the base object of
	1200	/// the chain, and will then apply the deltas in `delta_offsets` in reverse order
	1201	/// to obtain the final object.
	1202	fn resolveDeltaChain(
	1203	allocator: Allocator,
	1204	pack: std.fs.File,
	1205	base_object: Object,
	1206	delta_offsets: []const u64,
	1207	cache: *ObjectCache,
	1208	) ![]const u8 {
	1209	var base_data = base_object.data;
	1210	var i: usize = delta_offsets.len;
	1211	while (i > 0) {
	1212	i -= 1;
	1213
	1214	const delta_offset = delta_offsets[i];
	1215	try pack.seekTo(delta_offset);
	1216	const delta_header = try EntryHeader.read(pack.reader());
	1217	var delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength());
	1218	defer allocator.free(delta_data);
	1219	var delta_stream = std.io.fixedBufferStream(delta_data);
	1220	const delta_reader = delta_stream.reader();
	1221	_ = try readSizeVarInt(delta_reader); // base object size
	1222	const expanded_size = try readSizeVarInt(delta_reader);
	1223
	1224	const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge;
	1225	var expanded_data = try allocator.alloc(u8, expanded_alloc_size);
	1226	errdefer allocator.free(expanded_data);
	1227	var expanded_delta_stream = std.io.fixedBufferStream(expanded_data);
	1228	var base_stream = std.io.fixedBufferStream(base_data);
	1229	try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer());
	1230	if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject;
	1231
	1232	try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data });
	1233	base_data = expanded_data;
	1234	}
	1235	return base_data;
	1236	}
	1237
	1238	/// Reads the complete contents of an object from `reader`. This function may
	1239	/// read more bytes than required from `reader`, so the reader position after
	1240	/// returning is not reliable.
	1241	fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 {
	1242	const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;
	1243	var buffered_reader = std.io.bufferedReader(reader);
	1244	var decompress_stream = try std.compress.zlib.decompressStream(allocator, buffered_reader.reader());
	1245	defer decompress_stream.deinit();
	1246	var data = try allocator.alloc(u8, alloc_size);
	1247	errdefer allocator.free(data);
	1248	try decompress_stream.reader().readNoEof(data);
	1249	_ = decompress_stream.reader().readByte() catch \|e\| switch (e) {
	1250	error.EndOfStream => return data,
	1251	else => \|other\| return other,
	1252	};
	1253	return error.InvalidFormat;
	1254	}
	1255
	1256	/// Expands delta data from `delta_reader` to `writer`. `base_object` must
	1257	/// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`).
	1258	///
	1259	/// The format of the delta data is documented in
	1260	/// [pack-format](https://git-scm.com/docs/pack-format).
	1261	fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !void {
	1262	while (true) {
	1263	const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch \|e\| switch (e) {
	1264	error.EndOfStream => return,
	1265	else => \|other\| return other,
	1266	});
	1267	if (inst.copy) {
	1268	const available: packed struct {
	1269	offset1: bool,
	1270	offset2: bool,
	1271	offset3: bool,
	1272	offset4: bool,
	1273	size1: bool,
	1274	size2: bool,
	1275	size3: bool,
	1276	} = @bitCast(inst.value);
	1277	var offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{
	1278	.offset1 = if (available.offset1) try delta_reader.readByte() else 0,
	1279	.offset2 = if (available.offset2) try delta_reader.readByte() else 0,
	1280	.offset3 = if (available.offset3) try delta_reader.readByte() else 0,
	1281	.offset4 = if (available.offset4) try delta_reader.readByte() else 0,
	1282	};
	1283	const offset: u32 = @bitCast(offset_parts);
	1284	var size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{
	1285	.size1 = if (available.size1) try delta_reader.readByte() else 0,
	1286	.size2 = if (available.size2) try delta_reader.readByte() else 0,
	1287	.size3 = if (available.size3) try delta_reader.readByte() else 0,
	1288	};
	1289	var size: u24 = @bitCast(size_parts);
	1290	if (size == 0) size = 0x10000;
	1291	try base_object.seekTo(offset);
	1292	var copy_reader = std.io.limitedReader(base_object.reader(), size);
	1293	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
	1294	try fifo.pump(copy_reader.reader(), writer);
	1295	} else if (inst.value != 0) {
	1296	var data_reader = std.io.limitedReader(delta_reader, inst.value);
	1297	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
	1298	try fifo.pump(data_reader.reader(), writer);
	1299	} else {
	1300	return error.InvalidDeltaInstruction;
	1301	}
	1302	}
	1303	}
	1304
	1305	fn HashedWriter(
	1306	comptime WriterType: anytype,
	1307	comptime HasherType: anytype,
	1308	) type {
	1309	return struct {
	1310	child_writer: WriterType,
	1311	hasher: HasherType,
	1312
	1313	const Error = WriterType.Error;
	1314	const Writer = std.io.Writer(*@This(), Error, write);
	1315
	1316	fn write(hashed_writer: *@This(), buf: []const u8) Error!usize {
	1317	const amt = try hashed_writer.child_writer.write(buf);
	1318	hashed_writer.hasher.update(buf);
	1319	return amt;
	1320	}
	1321
	1322	fn writer(hashed_writer: *@This()) Writer {
	1323	return .{ .context = hashed_writer };
	1324	}
	1325	};
	1326	}
	1327
	1328	fn hashedWriter(
	1329	writer: anytype,
	1330	hasher: anytype,
	1331	) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) {
	1332	return .{ .child_writer = writer, .hasher = hasher };
	1333	}
	1334
	1335	test "packfile indexing and checkout" {
	1336	// To verify the contents of this packfile without using the code in this
	1337	// file:
	1338	//
	1339	// 1. Create a new empty Git repository (`git init`)
	1340	// 2. `git unpack-objects <path/to/testdata.pack`
	1341	// 3. `git fsck` -> note the "dangling commit" ID (which matches the commit
	1342	// checked out below)
	1343	// 4. `git checkout dd582c0720819ab7130b103635bd7271b9fd4feb`
	1344	const testrepo_pack = @embedFile("git/testdata/testrepo.pack");
	1345
	1346	var git_dir = testing.tmpDir(.{});
	1347	defer git_dir.cleanup();
	1348	var pack_file = try git_dir.dir.createFile("testrepo.pack", .{ .read = true });
	1349	defer pack_file.close();
	1350	try pack_file.writeAll(testrepo_pack);
	1351
	1352	var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true });
	1353	defer index_file.close();
	1354	try indexPack(testing.allocator, pack_file, index_file.writer());
	1355
	1356	// Arbitrary size limit on files read while checking the repository contents
	1357	// (all files in the test repo are known to be much smaller than this)
	1358	const max_file_size = 4096;
	1359
	1360	const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size);
	1361	defer testing.allocator.free(index_file_data);
	1362	// testrepo.idx is generated by Git. The index created by this file should
	1363	// match it exactly. Running `git verify-pack -v testrepo.pack` can verify
	1364	// this.
	1365	const testrepo_idx = @embedFile("git/testdata/testrepo.idx");
	1366	try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);
	1367
	1368	var repository = try Repository.init(testing.allocator, pack_file, index_file);
	1369	defer repository.deinit();
	1370
	1371	var worktree = testing.tmpIterableDir(.{});
	1372	defer worktree.cleanup();
	1373
	1374	const commit_id = try parseOid("dd582c0720819ab7130b103635bd7271b9fd4feb");
	1375	try repository.checkout(worktree.iterable_dir.dir, commit_id);
	1376
	1377	const expected_files: []const []const u8 = &.{
	1378	"dir/file",
	1379	"dir/subdir/file",
	1380	"dir/subdir/file2",
	1381	"dir2/file",
	1382	"dir3/file",
	1383	"dir3/file2",
	1384	"file",
	1385	"file2",
	1386	"file3",
	1387	"file4",
	1388	"file5",
	1389	"file6",
	1390	"file7",
	1391	"file8",
	1392	"file9",
	1393	};
	1394	var actual_files: std.ArrayListUnmanaged([]u8) = .{};
	1395	defer actual_files.deinit(testing.allocator);
	1396	defer for (actual_files.items) \|file\| testing.allocator.free(file);
	1397	var walker = try worktree.iterable_dir.walk(testing.allocator);
	1398	defer walker.deinit();
	1399	while (try walker.next()) \|entry\| {
	1400	if (entry.kind != .file) continue;
	1401	var path = try testing.allocator.dupe(u8, entry.path);
	1402	errdefer testing.allocator.free(path);
	1403	mem.replaceScalar(u8, path, std.fs.path.sep, '/');
	1404	try actual_files.append(testing.allocator, path);
	1405	}
	1406	mem.sortUnstable([]u8, actual_files.items, {}, struct {
	1407	fn lessThan(_: void, a: []u8, b: []u8) bool {
	1408	return mem.lessThan(u8, a, b);
	1409	}
	1410	}.lessThan);
	1411	try testing.expectEqualDeep(expected_files, actual_files.items);
	1412
	1413	const expected_file_contents =
	1414	\\revision 1
	1415	\\revision 2
	1416	\\revision 4
	1417	\\revision 5
	1418	\\revision 7
	1419	\\revision 8
	1420	\\revision 9
	1421	\\revision 10
	1422	\\revision 12
	1423	\\revision 13
	1424	\\revision 14
	1425	\\revision 18
	1426	\\revision 19
	1427	\\
	1428	;
	1429	const actual_file_contents = try worktree.iterable_dir.dir.readFileAlloc(testing.allocator, "file", max_file_size);
	1430	defer testing.allocator.free(actual_file_contents);
	1431	try testing.expectEqualStrings(expected_file_contents, actual_file_contents);
	1432	}
	1433
	1434	/// Checks out a commit of a packfile. Intended for experimenting with and
	1435	/// benchmarking possible optimizations to the indexing and checkout behavior.
	1436	pub fn main() !void {
	1437	const allocator = std.heap.c_allocator;
	1438
	1439	const args = try std.process.argsAlloc(allocator);
	1440	defer std.process.argsFree(allocator, args);
	1441	if (args.len != 4) {
	1442	return error.InvalidArguments; // Arguments: packfile commit worktree
	1443	}
	1444
	1445	var pack_file = try std.fs.cwd().openFile(args[1], .{});
	1446	defer pack_file.close();
	1447	const commit = try parseOid(args[2]);
	1448	var worktree = try std.fs.cwd().makeOpenPath(args[3], .{});
	1449	defer worktree.close();
	1450
	1451	var git_dir = try worktree.makeOpenPath(".git", .{});
	1452	defer git_dir.close();
	1453
	1454	std.debug.print("Starting index...\n", .{});
	1455	var index_file = try git_dir.createFile("idx", .{ .read = true });
	1456	defer index_file.close();
	1457	var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
	1458	try indexPack(allocator, pack_file, index_buffered_writer.writer());
	1459	try index_buffered_writer.flush();
	1460	try index_file.sync();
	1461
	1462	std.debug.print("Starting checkout...\n", .{});
	1463	var repository = try Repository.init(allocator, pack_file, index_file);
	1464	defer repository.deinit();
	1465	try repository.checkout(worktree, commit);
	1466	}

src/Package/Fetch/git/testdata/testrepo.idx created

Binary files /dev/null and b/src/Package/Fetch/git/testdata/testrepo.idx differ

src/Package/Fetch/git/testdata/testrepo.pack created

Binary files /dev/null and b/src/Package/Fetch/git/testdata/testrepo.pack differ

src/Package/Manifest.zig created+566

...	...	@@ -0,0 +1,566 @@
	1	pub const max_bytes = 10 * 1024 * 1024;
	2	pub const basename = "build.zig.zon";
	3	pub const Hash = std.crypto.hash.sha2.Sha256;
	4	pub const Digest = [Hash.digest_length]u8;
	5	pub const multihash_len = 1 + 1 + Hash.digest_length;
	6	pub const multihash_hex_digest_len = 2 * multihash_len;
	7	pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
	8
	9	pub const Dependency = struct {
	10	location: Location,
	11	location_tok: Ast.TokenIndex,
	12	hash: ?[]const u8,
	13	hash_tok: Ast.TokenIndex,
	14
	15	pub const Location = union(enum) {
	16	url: []const u8,
	17	path: []const u8,
	18	};
	19	};
	20
	21	pub const ErrorMessage = struct {
	22	msg: []const u8,
	23	tok: Ast.TokenIndex,
	24	off: u32,
	25	};
	26
	27	pub const MultihashFunction = enum(u16) {
	28	identity = 0x00,
	29	sha1 = 0x11,
	30	@"sha2-256" = 0x12,
	31	@"sha2-512" = 0x13,
	32	@"sha3-512" = 0x14,
	33	@"sha3-384" = 0x15,
	34	@"sha3-256" = 0x16,
	35	@"sha3-224" = 0x17,
	36	@"sha2-384" = 0x20,
	37	@"sha2-256-trunc254-padded" = 0x1012,
	38	@"sha2-224" = 0x1013,
	39	@"sha2-512-224" = 0x1014,
	40	@"sha2-512-256" = 0x1015,
	41	@"blake2b-256" = 0xb220,
	42	_,
	43	};
	44
	45	pub const multihash_function: MultihashFunction = switch (Hash) {
	46	std.crypto.hash.sha2.Sha256 => .@"sha2-256",
	47	else => @compileError("unreachable"),
	48	};
	49	comptime {
	50	// We avoid unnecessary uleb128 code in hexDigest by asserting here the
	51	// values are small enough to be contained in the one-byte encoding.
	52	assert(@intFromEnum(multihash_function) < 127);
	53	assert(Hash.digest_length < 127);
	54	}
	55
	56	name: []const u8,
	57	version: std.SemanticVersion,
	58	dependencies: std.StringArrayHashMapUnmanaged(Dependency),
	59	paths: std.StringArrayHashMapUnmanaged(void),
	60
	61	errors: []ErrorMessage,
	62	arena_state: std.heap.ArenaAllocator.State,
	63
	64	pub const ParseOptions = struct {
	65	allow_missing_paths_field: bool = false,
	66	};
	67
	68	pub const Error = Allocator.Error;
	69
	70	pub fn parse(gpa: Allocator, ast: std.zig.Ast, options: ParseOptions) Error!Manifest {
	71	const node_tags = ast.nodes.items(.tag);
	72	const node_datas = ast.nodes.items(.data);
	73	assert(node_tags[0] == .root);
	74	const main_node_index = node_datas[0].lhs;
	75
	76	var arena_instance = std.heap.ArenaAllocator.init(gpa);
	77	errdefer arena_instance.deinit();
	78
	79	var p: Parse = .{
	80	.gpa = gpa,
	81	.ast = ast,
	82	.arena = arena_instance.allocator(),
	83	.errors = .{},
	84
	85	.name = undefined,
	86	.version = undefined,
	87	.dependencies = .{},
	88	.paths = .{},
	89	.allow_missing_paths_field = options.allow_missing_paths_field,
	90	.buf = .{},
	91	};
	92	defer p.buf.deinit(gpa);
	93	defer p.errors.deinit(gpa);
	94	defer p.dependencies.deinit(gpa);
	95	defer p.paths.deinit(gpa);
	96
	97	p.parseRoot(main_node_index) catch \|err\| switch (err) {
	98	error.ParseFailure => assert(p.errors.items.len > 0),
	99	else => \|e\| return e,
	100	};
	101
	102	return .{
	103	.name = p.name,
	104	.version = p.version,
	105	.dependencies = try p.dependencies.clone(p.arena),
	106	.paths = try p.paths.clone(p.arena),
	107	.errors = try p.arena.dupe(ErrorMessage, p.errors.items),
	108	.arena_state = arena_instance.state,
	109	};
	110	}
	111
	112	pub fn deinit(man: *Manifest, gpa: Allocator) void {
	113	man.arena_state.promote(gpa).deinit();
	114	man.* = undefined;
	115	}
	116
	117	const hex_charset = "0123456789abcdef";
	118
	119	pub fn hex64(x: u64) [16]u8 {
	120	var result: [16]u8 = undefined;
	121	var i: usize = 0;
	122	while (i < 8) : (i += 1) {
	123	const byte = @as(u8, @truncate(x >> @as(u6, @intCast(8 * i))));
	124	result[i * 2 + 0] = hex_charset[byte >> 4];
	125	result[i * 2 + 1] = hex_charset[byte & 15];
	126	}
	127	return result;
	128	}
	129
	130	test hex64 {
	131	const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]";
	132	try std.testing.expectEqualStrings("[00efcdab78563412]", s);
	133	}
	134
	135	pub fn hexDigest(digest: Digest) MultiHashHexDigest {
	136	var result: MultiHashHexDigest = undefined;
	137
	138	result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
	139	result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
	140
	141	result[2] = hex_charset[Hash.digest_length >> 4];
	142	result[3] = hex_charset[Hash.digest_length & 15];
	143
	144	for (digest, 0..) \|byte, i\| {
	145	result[4 + i * 2] = hex_charset[byte >> 4];
	146	result[5 + i * 2] = hex_charset[byte & 15];
	147	}
	148	return result;
	149	}
	150
	151	const Parse = struct {
	152	gpa: Allocator,
	153	ast: std.zig.Ast,
	154	arena: Allocator,
	155	buf: std.ArrayListUnmanaged(u8),
	156	errors: std.ArrayListUnmanaged(ErrorMessage),
	157
	158	name: []const u8,
	159	version: std.SemanticVersion,
	160	dependencies: std.StringArrayHashMapUnmanaged(Dependency),
	161	paths: std.StringArrayHashMapUnmanaged(void),
	162	allow_missing_paths_field: bool,
	163
	164	const InnerError = error{ ParseFailure, OutOfMemory };
	165
	166	fn parseRoot(p: *Parse, node: Ast.Node.Index) !void {
	167	const ast = p.ast;
	168	const main_tokens = ast.nodes.items(.main_token);
	169	const main_token = main_tokens[node];
	170
	171	var buf: [2]Ast.Node.Index = undefined;
	172	const struct_init = ast.fullStructInit(&buf, node) orelse {
	173	return fail(p, main_token, "expected top level expression to be a struct", .{});
	174	};
	175
	176	var have_name = false;
	177	var have_version = false;
	178	var have_included_paths = false;
	179
	180	for (struct_init.ast.fields) \|field_init\| {
	181	const name_token = ast.firstToken(field_init) - 2;
	182	const field_name = try identifierTokenString(p, name_token);
	183	// We could get fancy with reflection and comptime logic here but doing
	184	// things manually provides an opportunity to do any additional verification
	185	// that is desirable on a per-field basis.
	186	if (mem.eql(u8, field_name, "dependencies")) {
	187	try parseDependencies(p, field_init);
	188	} else if (mem.eql(u8, field_name, "paths")) {
	189	have_included_paths = true;
	190	try parseIncludedPaths(p, field_init);
	191	} else if (mem.eql(u8, field_name, "name")) {
	192	p.name = try parseString(p, field_init);
	193	have_name = true;
	194	} else if (mem.eql(u8, field_name, "version")) {
	195	const version_text = try parseString(p, field_init);
	196	p.version = std.SemanticVersion.parse(version_text) catch \|err\| v: {
	197	try appendError(p, main_tokens[field_init], "unable to parse semantic version: {s}", .{@errorName(err)});
	198	break :v undefined;
	199	};
	200	have_version = true;
	201	} else {
	202	// Ignore unknown fields so that we can add fields in future zig
	203	// versions without breaking older zig versions.
	204	}
	205	}
	206
	207	if (!have_name) {
	208	try appendError(p, main_token, "missing top-level 'name' field", .{});
	209	}
	210
	211	if (!have_version) {
	212	try appendError(p, main_token, "missing top-level 'version' field", .{});
	213	}
	214
	215	if (!have_included_paths) {
	216	if (p.allow_missing_paths_field) {
	217	try p.paths.put(p.gpa, "", {});
	218	} else {
	219	try appendError(p, main_token, "missing top-level 'paths' field", .{});
	220	}
	221	}
	222	}
	223
	224	fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void {
	225	const ast = p.ast;
	226	const main_tokens = ast.nodes.items(.main_token);
	227
	228	var buf: [2]Ast.Node.Index = undefined;
	229	const struct_init = ast.fullStructInit(&buf, node) orelse {
	230	const tok = main_tokens[node];
	231	return fail(p, tok, "expected dependencies expression to be a struct", .{});
	232	};
	233
	234	for (struct_init.ast.fields) \|field_init\| {
	235	const name_token = ast.firstToken(field_init) - 2;
	236	const dep_name = try identifierTokenString(p, name_token);
	237	const dep = try parseDependency(p, field_init);
	238	try p.dependencies.put(p.gpa, dep_name, dep);
	239	}
	240	}
	241
	242	fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency {
	243	const ast = p.ast;
	244	const main_tokens = ast.nodes.items(.main_token);
	245
	246	var buf: [2]Ast.Node.Index = undefined;
	247	const struct_init = ast.fullStructInit(&buf, node) orelse {
	248	const tok = main_tokens[node];
	249	return fail(p, tok, "expected dependency expression to be a struct", .{});
	250	};
	251
	252	var dep: Dependency = .{
	253	.location = undefined,
	254	.location_tok = 0,
	255	.hash = null,
	256	.hash_tok = 0,
	257	};
	258	var has_location = false;
	259
	260	for (struct_init.ast.fields) \|field_init\| {
	261	const name_token = ast.firstToken(field_init) - 2;
	262	const field_name = try identifierTokenString(p, name_token);
	263	// We could get fancy with reflection and comptime logic here but doing
	264	// things manually provides an opportunity to do any additional verification
	265	// that is desirable on a per-field basis.
	266	if (mem.eql(u8, field_name, "url")) {
	267	if (has_location) {
	268	return fail(p, main_tokens[field_init], "dependency should specify only one of 'url' and 'path' fields.", .{});
	269	}
	270	dep.location = .{
	271	.url = parseString(p, field_init) catch \|err\| switch (err) {
	272	error.ParseFailure => continue,
	273	else => \|e\| return e,
	274	},
	275	};
	276	has_location = true;
	277	dep.location_tok = main_tokens[field_init];
	278	} else if (mem.eql(u8, field_name, "path")) {
	279	if (has_location) {
	280	return fail(p, main_tokens[field_init], "dependency should specify only one of 'url' and 'path' fields.", .{});
	281	}
	282	dep.location = .{
	283	.path = parseString(p, field_init) catch \|err\| switch (err) {
	284	error.ParseFailure => continue,
	285	else => \|e\| return e,
	286	},
	287	};
	288	has_location = true;
	289	dep.location_tok = main_tokens[field_init];
	290	} else if (mem.eql(u8, field_name, "hash")) {
	291	dep.hash = parseHash(p, field_init) catch \|err\| switch (err) {
	292	error.ParseFailure => continue,
	293	else => \|e\| return e,
	294	};
	295	dep.hash_tok = main_tokens[field_init];
	296	} else {
	297	// Ignore unknown fields so that we can add fields in future zig
	298	// versions without breaking older zig versions.
	299	}
	300	}
	301
	302	if (!has_location) {
	303	try appendError(p, main_tokens[node], "dependency requires location field, one of 'url' or 'path'.", .{});
	304	}
	305
	306	return dep;
	307	}
	308
	309	fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void {
	310	const ast = p.ast;
	311	const main_tokens = ast.nodes.items(.main_token);
	312
	313	var buf: [2]Ast.Node.Index = undefined;
	314	const array_init = ast.fullArrayInit(&buf, node) orelse {
	315	const tok = main_tokens[node];
	316	return fail(p, tok, "expected paths expression to be a struct", .{});
	317	};
	318
	319	for (array_init.ast.elements) \|elem_node\| {
	320	const path_string = try parseString(p, elem_node);
	321	// This is normalized so that it can be used in string comparisons
	322	// against file system paths.
	323	const normalized = try std.fs.path.resolve(p.arena, &.{path_string});
	324	try p.paths.put(p.gpa, normalized, {});
	325	}
	326	}
	327
	328	fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 {
	329	const ast = p.ast;
	330	const node_tags = ast.nodes.items(.tag);
	331	const main_tokens = ast.nodes.items(.main_token);
	332	if (node_tags[node] != .string_literal) {
	333	return fail(p, main_tokens[node], "expected string literal", .{});
	334	}
	335	const str_lit_token = main_tokens[node];
	336	const token_bytes = ast.tokenSlice(str_lit_token);
	337	p.buf.clearRetainingCapacity();
	338	try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0);
	339	const duped = try p.arena.dupe(u8, p.buf.items);
	340	return duped;
	341	}
	342
	343	fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 {
	344	const ast = p.ast;
	345	const main_tokens = ast.nodes.items(.main_token);
	346	const tok = main_tokens[node];
	347	const h = try parseString(p, node);
	348
	349	if (h.len >= 2) {
	350	const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch \|err\| {
	351	return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{
	352	@errorName(err),
	353	});
	354	};
	355	if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) {
	356	return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{});
	357	}
	358	}
	359
	360	if (h.len != multihash_hex_digest_len) {
	361	return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{
	362	multihash_hex_digest_len, h.len,
	363	});
	364	}
	365
	366	return h;
	367	}
	368
	369	/// TODO: try to DRY this with AstGen.identifierTokenString
	370	fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 {
	371	const ast = p.ast;
	372	const token_tags = ast.tokens.items(.tag);
	373	assert(token_tags[token] == .identifier);
	374	const ident_name = ast.tokenSlice(token);
	375	if (!mem.startsWith(u8, ident_name, "@")) {
	376	return ident_name;
	377	}
	378	p.buf.clearRetainingCapacity();
	379	try parseStrLit(p, token, &p.buf, ident_name, 1);
	380	const duped = try p.arena.dupe(u8, p.buf.items);
	381	return duped;
	382	}
	383
	384	/// TODO: try to DRY this with AstGen.parseStrLit
	385	fn parseStrLit(
	386	p: *Parse,
	387	token: Ast.TokenIndex,
	388	buf: *std.ArrayListUnmanaged(u8),
	389	bytes: []const u8,
	390	offset: u32,
	391	) InnerError!void {
	392	const raw_string = bytes[offset..];
	393	var buf_managed = buf.toManaged(p.gpa);
	394	const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string);
	395	buf.* = buf_managed.moveToUnmanaged();
	396	switch (try result) {
	397	.success => {},
	398	.failure => \|err\| try p.appendStrLitError(err, token, bytes, offset),
	399	}
	400	}
	401
	402	/// TODO: try to DRY this with AstGen.failWithStrLitError
	403	fn appendStrLitError(
	404	p: *Parse,
	405	err: std.zig.string_literal.Error,
	406	token: Ast.TokenIndex,
	407	bytes: []const u8,
	408	offset: u32,
	409	) Allocator.Error!void {
	410	const raw_string = bytes[offset..];
	411	switch (err) {
	412	.invalid_escape_character => \|bad_index\| {
	413	try p.appendErrorOff(
	414	token,
	415	offset + @as(u32, @intCast(bad_index)),
	416	"invalid escape character: '{c}'",
	417	.{raw_string[bad_index]},
	418	);
	419	},
	420	.expected_hex_digit => \|bad_index\| {
	421	try p.appendErrorOff(
	422	token,
	423	offset + @as(u32, @intCast(bad_index)),
	424	"expected hex digit, found '{c}'",
	425	.{raw_string[bad_index]},
	426	);
	427	},
	428	.empty_unicode_escape_sequence => \|bad_index\| {
	429	try p.appendErrorOff(
	430	token,
	431	offset + @as(u32, @intCast(bad_index)),
	432	"empty unicode escape sequence",
	433	.{},
	434	);
	435	},
	436	.expected_hex_digit_or_rbrace => \|bad_index\| {
	437	try p.appendErrorOff(
	438	token,
	439	offset + @as(u32, @intCast(bad_index)),
	440	"expected hex digit or '}}', found '{c}'",
	441	.{raw_string[bad_index]},
	442	);
	443	},
	444	.invalid_unicode_codepoint => \|bad_index\| {
	445	try p.appendErrorOff(
	446	token,
	447	offset + @as(u32, @intCast(bad_index)),
	448	"unicode escape does not correspond to a valid codepoint",
	449	.{},
	450	);
	451	},
	452	.expected_lbrace => \|bad_index\| {
	453	try p.appendErrorOff(
	454	token,
	455	offset + @as(u32, @intCast(bad_index)),
	456	"expected '{{', found '{c}",
	457	.{raw_string[bad_index]},
	458	);
	459	},
	460	.expected_rbrace => \|bad_index\| {
	461	try p.appendErrorOff(
	462	token,
	463	offset + @as(u32, @intCast(bad_index)),
	464	"expected '}}', found '{c}",
	465	.{raw_string[bad_index]},
	466	);
	467	},
	468	.expected_single_quote => \|bad_index\| {
	469	try p.appendErrorOff(
	470	token,
	471	offset + @as(u32, @intCast(bad_index)),
	472	"expected single quote ('), found '{c}",
	473	.{raw_string[bad_index]},
	474	);
	475	},
	476	.invalid_character => \|bad_index\| {
	477	try p.appendErrorOff(
	478	token,
	479	offset + @as(u32, @intCast(bad_index)),
	480	"invalid byte in string or character literal: '{c}'",
	481	.{raw_string[bad_index]},
	482	);
	483	},
	484	}
	485	}
	486
	487	fn fail(
	488	p: *Parse,
	489	tok: Ast.TokenIndex,
	490	comptime fmt: []const u8,
	491	args: anytype,
	492	) InnerError {
	493	try appendError(p, tok, fmt, args);
	494	return error.ParseFailure;
	495	}
	496
	497	fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void {
	498	return appendErrorOff(p, tok, 0, fmt, args);
	499	}
	500
	501	fn appendErrorOff(
	502	p: *Parse,
	503	tok: Ast.TokenIndex,
	504	byte_offset: u32,
	505	comptime fmt: []const u8,
	506	args: anytype,
	507	) Allocator.Error!void {
	508	try p.errors.append(p.gpa, .{
	509	.msg = try std.fmt.allocPrint(p.arena, fmt, args),
	510	.tok = tok,
	511	.off = byte_offset,
	512	});
	513	}
	514	};
	515
	516	const Manifest = @This();
	517	const std = @import("std");
	518	const mem = std.mem;
	519	const Allocator = std.mem.Allocator;
	520	const assert = std.debug.assert;
	521	const Ast = std.zig.Ast;
	522	const testing = std.testing;
	523
	524	test "basic" {
	525	const gpa = testing.allocator;
	526
	527	const example =
	528	\\.{
	529	\\ .name = "foo",
	530	\\ .version = "3.2.1",
	531	\\ .dependencies = .{
	532	\\ .bar = .{
	533	\\ .url = "https://example.com/baz.tar.gz",
	534	\\ .hash = "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f",
	535	\\ },
	536	\\ },
	537	\\}
	538	;
	539
	540	var ast = try std.zig.Ast.parse(gpa, example, .zon);
	541	defer ast.deinit(gpa);
	542
	543	try testing.expect(ast.errors.len == 0);
	544
	545	var manifest = try Manifest.parse(gpa, ast);
	546	defer manifest.deinit(gpa);
	547
	548	try testing.expectEqualStrings("foo", manifest.name);
	549
	550	try testing.expectEqual(@as(std.SemanticVersion, .{
	551	.major = 3,
	552	.minor = 2,
	553	.patch = 1,
	554	}), manifest.version);
	555
	556	try testing.expect(manifest.dependencies.count() == 1);
	557	try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]);
	558	try testing.expectEqualStrings(
	559	"https://example.com/baz.tar.gz",
	560	manifest.dependencies.values()[0].url,
	561	);
	562	try testing.expectEqualStrings(
	563	"1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f",
	564	manifest.dependencies.values()[0].hash orelse return error.TestFailed,
	565	);
	566	}

src/Package/Module.zig created+34

...	...	@@ -0,0 +1,34 @@
	1	//! Corresponds to something that Zig source code can `@import`.
	2	//! Not to be confused with src/Module.zig which should be renamed
	3	//! to something else. https://github.com/ziglang/zig/issues/14307
	4
	5	/// Only files inside this directory can be imported.
	6	root: Package.Path,
	7	/// Relative to `root`. May contain path separators.
	8	root_src_path: []const u8,
	9	/// Name used in compile errors. Looks like "root.foo.bar".
	10	fully_qualified_name: []const u8,
	11	/// The dependency table of this module. Shared dependencies such as 'std',
	12	/// 'builtin', and 'root' are not specified in every dependency table, but
	13	/// instead only in the table of `main_mod`. `Module.importFile` is
	14	/// responsible for detecting these names and using the correct package.
	15	deps: Deps = .{},
	16
	17	pub const Deps = std.StringHashMapUnmanaged(*Module);
	18
	19	pub const Tree = struct {
	20	/// Each `Package` exposes a `Module` with build.zig as its root source file.
	21	build_module_table: std.AutoArrayHashMapUnmanaged(MultiHashHexDigest, *Module),
	22	};
	23
	24	pub fn create(allocator: Allocator, m: Module) Allocator.Error!*Module {
	25	const new = try allocator.create(Module);
	26	new.* = m;
	27	return new;
	28	}
	29
	30	const Module = @This();
	31	const Package = @import("../Package.zig");
	32	const std = @import("std");
	33	const Allocator = std.mem.Allocator;
	34	const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest;

src/Package/hash.zig deleted-153

...	...	@@ -1,153 +0,0 @@
1		const builtin = @import("builtin");
2		const std = @import("std");
3		const fs = std.fs;
4		const ThreadPool = std.Thread.Pool;
5		const WaitGroup = std.Thread.WaitGroup;
6		const Allocator = std.mem.Allocator;
7
8		const Hash = @import("../Manifest.zig").Hash;
9
10		pub fn compute(thread_pool: *ThreadPool, pkg_dir: fs.IterableDir) ![Hash.digest_length]u8 {
11		const gpa = thread_pool.allocator;
12
13		// We'll use an arena allocator for the path name strings since they all
14		// need to be in memory for sorting.
15		var arena_instance = std.heap.ArenaAllocator.init(gpa);
16		defer arena_instance.deinit();
17		const arena = arena_instance.allocator();
18
19		// TODO: delete files not included in the package prior to computing the package hash.
20		// for example, if the ini file has directives to include/not include certain files,
21		// apply those rules directly to the filesystem right here. This ensures that files
22		// not protected by the hash are not present on the file system.
23
24		// Collect all files, recursively, then sort.
25		var all_files = std.ArrayList(*HashedFile).init(gpa);
26		defer all_files.deinit();
27
28		var walker = try pkg_dir.walk(gpa);
29		defer walker.deinit();
30
31		{
32		// The final hash will be a hash of each file hashed independently. This
33		// allows hashing in parallel.
34		var wait_group: WaitGroup = .{};
35		defer wait_group.wait();
36
37		while (try walker.next()) \|entry\| {
38		const kind: HashedFile.Kind = switch (entry.kind) {
39		.directory => continue,
40		.file => .file,
41		.sym_link => .sym_link,
42		else => return error.IllegalFileTypeInPackage,
43		};
44		const hashed_file = try arena.create(HashedFile);
45		const fs_path = try arena.dupe(u8, entry.path);
46		hashed_file.* = .{
47		.fs_path = fs_path,
48		.normalized_path = try normalizePath(arena, fs_path),
49		.kind = kind,
50		.hash = undefined, // to be populated by the worker
51		.failure = undefined, // to be populated by the worker
52		};
53		wait_group.start();
54		try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group });
55
56		try all_files.append(hashed_file);
57		}
58		}
59
60		std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
61
62		var hasher = Hash.init(.{});
63		var any_failures = false;
64		for (all_files.items) \|hashed_file\| {
65		hashed_file.failure catch \|err\| {
66		any_failures = true;
67		std.log.err("unable to hash '{s}': {s}", .{ hashed_file.fs_path, @errorName(err) });
68		};
69		hasher.update(&hashed_file.hash);
70		}
71		if (any_failures) return error.PackageHashUnavailable;
72		return hasher.finalResult();
73		}
74
75		const HashedFile = struct {
76		fs_path: []const u8,
77		normalized_path: []const u8,
78		hash: [Hash.digest_length]u8,
79		failure: Error!void,
80		kind: Kind,
81
82		const Error =
83		fs.File.OpenError \|\|
84		fs.File.ReadError \|\|
85		fs.File.StatError \|\|
86		fs.Dir.ReadLinkError;
87
88		const Kind = enum { file, sym_link };
89
90		fn lessThan(context: void, lhs: const HashedFile, rhs: const HashedFile) bool {
91		_ = context;
92		return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path);
93		}
94		};
95
96		/// Make a file system path identical independently of operating system path inconsistencies.
97		/// This converts backslashes into forward slashes.
98		fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
99		const canonical_sep = '/';
100
101		if (fs.path.sep == canonical_sep)
102		return fs_path;
103
104		const normalized = try arena.dupe(u8, fs_path);
105		for (normalized) \|*byte\| {
106		switch (byte.*) {
107		fs.path.sep => byte.* = canonical_sep,
108		else => continue,
109		}
110		}
111		return normalized;
112		}
113
114		fn workerHashFile(dir: fs.Dir, hashed_file: HashedFile, wg: WaitGroup) void {
115		defer wg.finish();
116		hashed_file.failure = hashFileFallible(dir, hashed_file);
117		}
118
119		fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
120		var buf: [8000]u8 = undefined;
121		var hasher = Hash.init(.{});
122		hasher.update(hashed_file.normalized_path);
123		switch (hashed_file.kind) {
124		.file => {
125		var file = try dir.openFile(hashed_file.fs_path, .{});
126		defer file.close();
127		hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) });
128		while (true) {
129		const bytes_read = try file.read(&buf);
130		if (bytes_read == 0) break;
131		hasher.update(buf[0..bytes_read]);
132		}
133		},
134		.sym_link => {
135		const link_name = try dir.readLink(hashed_file.fs_path, &buf);
136		hasher.update(link_name);
137		},
138		}
139		hasher.final(&hashed_file.hash);
140		}
141
142		fn isExecutable(file: fs.File) !bool {
143		if (builtin.os.tag == .windows) {
144		// TODO check the ACL on Windows.
145		// Until this is implemented, this could be a false negative on
146		// Windows, which is why we do not yet set executable_bit_only above
147		// when unpacking the tarball.
148		return false;
149		} else {
150		const stat = try file.stat();
151		return (stat.mode & std.os.S.IXUSR) != 0;
152		}
153		}

src/Sema.zig+28-26

...	...	@@ -5732,6 +5732,9 @@ fn zirCImport(sema: Sema, parent_block: Block, inst: Zir.Inst.Index) CompileEr
5732	5732	const tracy = trace(@src());
5733	5733	defer tracy.end();
5734	5734
	5735	const mod = sema.mod;
	5736	const comp = mod.comp;
	5737	const gpa = sema.gpa;
5735	5738	const pl_node = sema.code.instructions.items(.data)[inst].pl_node;
5736	5739	const src = pl_node.src();
5737	5740	const extra = sema.code.extraData(Zir.Inst.Block, pl_node.payload_index);
...	...	@@ -5741,7 +5744,7 @@ fn zirCImport(sema: Sema, parent_block: Block, inst: Zir.Inst.Index) CompileEr
5741	5744	if (!@import("build_options").have_llvm)
5742	5745	return sema.fail(parent_block, src, "C import unavailable; Zig compiler built without LLVM extensions", .{});
5743	5746
5744		var c_import_buf = std.ArrayList(u8).init(sema.gpa);
	5747	var c_import_buf = std.ArrayList(u8).init(gpa);
5745	5748	defer c_import_buf.deinit();
5746	5749
5747	5750	var comptime_reason: Block.ComptimeReason = .{ .c_import = .{
...	...	@@ -5763,25 +5766,24 @@ fn zirCImport(sema: Sema, parent_block: Block, inst: Zir.Inst.Index) CompileEr
5763	5766	.runtime_loop = parent_block.runtime_loop,
5764	5767	.runtime_index = parent_block.runtime_index,
5765	5768	};
5766		defer child_block.instructions.deinit(sema.gpa);
	5769	defer child_block.instructions.deinit(gpa);
5767	5770
5768	5771	// Ignore the result, all the relevant operations have written to c_import_buf already.
5769	5772	_ = try sema.analyzeBodyBreak(&child_block, body);
5770	5773
5771		const mod = sema.mod;
5772		var c_import_res = mod.comp.cImport(c_import_buf.items) catch \|err\|
	5774	var c_import_res = comp.cImport(c_import_buf.items) catch \|err\|
5773	5775	return sema.fail(&child_block, src, "C import failed: {s}", .{@errorName(err)});
5774		defer c_import_res.deinit(mod.comp.gpa);
	5776	defer c_import_res.deinit(gpa);
5775	5777
5776	5778	if (c_import_res.errors.errorMessageCount() != 0) {
5777	5779	const msg = msg: {
5778	5780	const msg = try sema.errMsg(&child_block, src, "C import failed", .{});
5779		errdefer msg.destroy(sema.gpa);
	5781	errdefer msg.destroy(gpa);
5780	5782
5781		if (!mod.comp.bin_file.options.link_libc)
	5783	if (!comp.bin_file.options.link_libc)
5782	5784	try sema.errNote(&child_block, src, msg, "libc headers not available; compilation does not link against libc", .{});
5783	5785
5784		const gop = try mod.cimport_errors.getOrPut(sema.gpa, sema.owner_decl_index);
	5786	const gop = try mod.cimport_errors.getOrPut(gpa, sema.owner_decl_index);
5785	5787	if (!gop.found_existing) {
5786	5788	gop.value_ptr.* = c_import_res.errors;
5787	5789	c_import_res.errors = std.zig.ErrorBundle.empty;
...	...	@@ -5790,16 +5792,16 @@ fn zirCImport(sema: Sema, parent_block: Block, inst: Zir.Inst.Index) CompileEr
5790	5792	};
5791	5793	return sema.failWithOwnedErrorMsg(&child_block, msg);
5792	5794	}
5793		const c_import_pkg = Package.create(
5794		sema.gpa,
5795		null,
5796		c_import_res.out_zig_path,
5797		) catch \|err\| switch (err) {
5798		error.OutOfMemory => return error.OutOfMemory,
5799		else => unreachable, // we pass null for root_src_dir_path
5800		};
	5795	const c_import_mod = try Package.Module.create(comp.arena.allocator(), .{
	5796	.root = .{
	5797	.root_dir = Compilation.Directory.cwd(),
	5798	.sub_path = std.fs.path.dirname(c_import_res.out_zig_path) orelse "",
	5799	},
	5800	.root_src_path = std.fs.path.basename(c_import_res.out_zig_path),
	5801	.fully_qualified_name = c_import_res.out_zig_path,
	5802	});
5801	5803
5802		const result = mod.importPkg(c_import_pkg) catch \|err\|
	5804	const result = mod.importPkg(c_import_mod) catch \|err\|
5803	5805	return sema.fail(&child_block, src, "C import failed: {s}", .{@errorName(err)});
5804	5806
5805	5807	mod.astGenFile(result.file) catch \|err\|
...	...	@@ -13071,13 +13073,13 @@ fn zirImport(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.
13071	13073	const operand = inst_data.get(sema.code);
13072	13074
13073	13075	const result = mod.importFile(block.getFileScope(mod), operand) catch \|err\| switch (err) {
13074		error.ImportOutsidePkgPath => {
13075		return sema.fail(block, operand_src, "import of file outside package path: '{s}'", .{operand});
	13076	error.ImportOutsideModulePath => {
	13077	return sema.fail(block, operand_src, "import of file outside module path: '{s}'", .{operand});
13076	13078	},
13077		error.PackageNotFound => {
13078		const name = try block.getFileScope(mod).pkg.getName(sema.gpa, mod.*);
13079		defer sema.gpa.free(name);
13080		return sema.fail(block, operand_src, "no package named '{s}' available within package '{s}'", .{ operand, name });
	13079	error.ModuleNotFound => {
	13080	return sema.fail(block, operand_src, "no module named '{s}' available within module {s}", .{
	13081	operand, block.getFileScope(mod).mod.fully_qualified_name,
	13082	});
13081	13083	},
13082	13084	else => {
13083	13085	// TODO: these errors are file system errors; make sure an update() will
...	...	@@ -13106,7 +13108,7 @@ fn zirEmbedFile(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!A
13106	13108	}
13107	13109
13108	13110	const embed_file = mod.embedFile(block.getFileScope(mod), name) catch \|err\| switch (err) {
13109		error.ImportOutsidePkgPath => {
	13111	error.ImportOutsideModulePath => {
13110	13112	return sema.fail(block, operand_src, "embed of file outside package path: '{s}'", .{name});
13111	13113	},
13112	13114	else => {
...	...	@@ -36415,8 +36417,8 @@ fn getBuiltinDecl(sema: Sema, block: Block, name: []const u8) CompileError!Mod
36415	36417
36416	36418	const mod = sema.mod;
36417	36419	const ip = &mod.intern_pool;
36418		const std_pkg = mod.main_pkg.table.get("std").?;
36419		const std_file = (mod.importPkg(std_pkg) catch unreachable).file;
	36420	const std_mod = mod.main_mod.deps.get("std").?;
	36421	const std_file = (mod.importPkg(std_mod) catch unreachable).file;
36420	36422	const opt_builtin_inst = (try sema.namespaceLookupRef(
36421	36423	block,
36422	36424	src,

src/codegen/llvm.zig+21-21

...	...	@@ -892,21 +892,24 @@ pub const Object = struct {
892	892	build_options.semver.patch,
893	893	});
894	894
895		// We fully resolve all paths at this point to avoid lack of source line info in stack
896		// traces or lack of debugging information which, if relative paths were used, would
897		// be very location dependent.
	895	// We fully resolve all paths at this point to avoid lack of
	896	// source line info in stack traces or lack of debugging
	897	// information which, if relative paths were used, would be
	898	// very location dependent.
898	899	// TODO: the only concern I have with this is WASI as either host or target, should
899	900	// we leave the paths as relative then?
900	901	var buf: [std.fs.MAX_PATH_BYTES]u8 = undefined;
901		const compile_unit_dir = blk: {
902		const path = d: {
903		const mod = options.module orelse break :d ".";
904		break :d mod.root_pkg.root_src_directory.path orelse ".";
905		};
906		if (std.fs.path.isAbsolute(path)) break :blk path;
907		break :blk std.os.realpath(path, &buf) catch path; // If realpath fails, fallback to whatever path was
	902	const compile_unit_dir_z = blk: {
	903	if (options.module) \|mod\| {
	904	const d = try mod.root_mod.root.joinStringZ(builder.gpa, "");
	905	if (std.fs.path.isAbsolute(d)) break :blk d;
	906	const abs = std.fs.realpath(d, &buf) catch break :blk d;
	907	builder.gpa.free(d);
	908	break :blk try builder.gpa.dupeZ(u8, abs);
	909	}
	910	const cwd = try std.process.getCwd(&buf);
	911	break :blk try builder.gpa.dupeZ(u8, cwd);
908	912	};
909		const compile_unit_dir_z = try builder.gpa.dupeZ(u8, compile_unit_dir);
910	913	defer builder.gpa.free(compile_unit_dir_z);
911	914
912	915	builder.llvm.di_compile_unit = builder.llvm.di_builder.?.createCompileUnit(
...	...	@@ -1833,14 +1836,11 @@ pub const Object = struct {
1833	1836	}
1834	1837	const dir_path_z = d: {
1835	1838	var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
1836		const dir_path = file.pkg.root_src_directory.path orelse ".";
1837		const resolved_dir_path = if (std.fs.path.isAbsolute(dir_path))
1838		dir_path
1839		else
1840		std.os.realpath(dir_path, &buffer) catch dir_path; // If realpath fails, fallback to whatever dir_path was
1841		break :d try std.fs.path.joinZ(gpa, &.{
1842		resolved_dir_path, std.fs.path.dirname(file.sub_file_path) orelse "",
1843		});
	1839	const sub_path = std.fs.path.dirname(file.sub_file_path) orelse "";
	1840	const dir_path = try file.mod.root.joinStringZ(gpa, sub_path);
	1841	if (std.fs.path.isAbsolute(dir_path)) break :d dir_path;
	1842	const abs = std.fs.realpath(dir_path, &buffer) catch break :d dir_path;
	1843	break :d try std.fs.path.joinZ(gpa, &.{ abs, sub_path });
1844	1844	};
1845	1845	defer gpa.free(dir_path_z);
1846	1846	const sub_file_path_z = try gpa.dupeZ(u8, std.fs.path.basename(file.sub_file_path));
...	...	@@ -2828,8 +2828,8 @@ pub const Object = struct {
2828	2828	fn getStackTraceType(o: *Object) Allocator.Error!Type {
2829	2829	const mod = o.module;
2830	2830
2831		const std_pkg = mod.main_pkg.table.get("std").?;
2832		const std_file = (mod.importPkg(std_pkg) catch unreachable).file;
	2831	const std_mod = mod.main_mod.deps.get("std").?;
	2832	const std_file = (mod.importPkg(std_mod) catch unreachable).file;
2833	2833
2834	2834	const builtin_str = try mod.intern_pool.getOrPutString(mod.gpa, "builtin");
2835	2835	const std_namespace = mod.namespacePtr(mod.declPtr(std_file.root_decl.unwrap().?).src_namespace);

src/crash_report.zig+13-9

...	...	@@ -139,18 +139,22 @@ fn dumpStatusReport() !void {
139	139
140	140	var crash_heap: [16 * 4096]u8 = undefined;
141	141
142		fn writeFilePath(file: *Module.File, stream: anytype) !void {
143		if (file.pkg.root_src_directory.path) \|path\| {
144		try stream.writeAll(path);
145		try stream.writeAll(std.fs.path.sep_str);
	142	fn writeFilePath(file: *Module.File, writer: anytype) !void {
	143	if (file.mod.root.root_dir.path) \|path\| {
	144	try writer.writeAll(path);
	145	try writer.writeAll(std.fs.path.sep_str);
146	146	}
147		try stream.writeAll(file.sub_file_path);
	147	if (file.mod.root.sub_path.len > 0) {
	148	try writer.writeAll(file.mod.root.sub_path);
	149	try writer.writeAll(std.fs.path.sep_str);
	150	}
	151	try writer.writeAll(file.sub_file_path);
148	152	}
149	153
150		fn writeFullyQualifiedDeclWithFile(mod: Module, decl: Decl, stream: anytype) !void {
151		try writeFilePath(decl.getFileScope(mod), stream);
152		try stream.writeAll(": ");
153		try decl.renderFullyQualifiedDebugName(mod, stream);
	154	fn writeFullyQualifiedDeclWithFile(mod: Module, decl: Decl, writer: anytype) !void {
	155	try writeFilePath(decl.getFileScope(mod), writer);
	156	try writer.writeAll(": ");
	157	try decl.renderFullyQualifiedDebugName(mod, writer);
154	158	}
155	159
156	160	pub fn compilerPanic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, maybe_ret_addr: ?usize) noreturn {

src/git.zig deleted-1468

...	...	@@ -1,1468 +0,0 @@
1		//! Git support for package fetching.
2		//!
3		//! This is not intended to support all features of Git: it is limited to the
4		//! basic functionality needed to clone a repository for the purpose of fetching
5		//! a package.
6
7		const std = @import("std");
8		const mem = std.mem;
9		const testing = std.testing;
10		const Allocator = mem.Allocator;
11		const Sha1 = std.crypto.hash.Sha1;
12		const assert = std.debug.assert;
13
14		const ProgressReader = @import("Package.zig").ProgressReader;
15
16		pub const oid_length = Sha1.digest_length;
17		pub const fmt_oid_length = 2 * oid_length;
18		/// The ID of a Git object (an SHA-1 hash).
19		pub const Oid = [oid_length]u8;
20
21		pub fn parseOid(s: []const u8) !Oid {
22		if (s.len != fmt_oid_length) return error.InvalidOid;
23		var oid: Oid = undefined;
24		for (&oid, 0..) \|*b, i\| {
25		b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid;
26		}
27		return oid;
28		}
29
30		test parseOid {
31		try testing.expectEqualSlices(
32		u8,
33		&.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 },
34		&try parseOid("ce919ccf45951856a762ffdb8ef850301cd8c588"),
35		);
36		try testing.expectError(error.InvalidOid, parseOid("ce919ccf"));
37		try testing.expectError(error.InvalidOid, parseOid("master"));
38		try testing.expectError(error.InvalidOid, parseOid("HEAD"));
39		}
40
41		pub const Diagnostics = struct {
42		allocator: Allocator,
43		errors: std.ArrayListUnmanaged(Error) = .{},
44
45		pub const Error = union(enum) {
46		unable_to_create_sym_link: struct {
47		code: anyerror,
48		file_name: []const u8,
49		link_name: []const u8,
50		},
51		};
52
53		pub fn deinit(d: *Diagnostics) void {
54		for (d.errors.items) \|item\| {
55		switch (item) {
56		.unable_to_create_sym_link => \|info\| {
57		d.allocator.free(info.file_name);
58		d.allocator.free(info.link_name);
59		},
60		}
61		}
62		d.errors.deinit(d.allocator);
63		d.* = undefined;
64		}
65		};
66
67		pub const Repository = struct {
68		odb: Odb,
69
70		pub fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Repository {
71		return .{ .odb = try Odb.init(allocator, pack_file, index_file) };
72		}
73
74		pub fn deinit(repository: *Repository) void {
75		repository.odb.deinit();
76		repository.* = undefined;
77		}
78
79		/// Checks out the repository at `commit_oid` to `worktree`.
80		pub fn checkout(
81		repository: *Repository,
82		worktree: std.fs.Dir,
83		commit_oid: Oid,
84		diagnostics: *Diagnostics,
85		) !void {
86		try repository.odb.seekOid(commit_oid);
87		const tree_oid = tree_oid: {
88		var commit_object = try repository.odb.readObject();
89		if (commit_object.type != .commit) return error.NotACommit;
90		break :tree_oid try getCommitTree(commit_object.data);
91		};
92		try repository.checkoutTree(worktree, tree_oid, "", diagnostics);
93		}
94
95		/// Checks out the tree at `tree_oid` to `worktree`.
96		fn checkoutTree(
97		repository: *Repository,
98		dir: std.fs.Dir,
99		tree_oid: Oid,
100		current_path: []const u8,
101		diagnostics: *Diagnostics,
102		) !void {
103		try repository.odb.seekOid(tree_oid);
104		const tree_object = try repository.odb.readObject();
105		if (tree_object.type != .tree) return error.NotATree;
106		// The tree object may be evicted from the object cache while we're
107		// iterating over it, so we can make a defensive copy here to make sure
108		// it remains valid until we're done with it
109		const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data);
110		defer repository.odb.allocator.free(tree_data);
111
112		var tree_iter: TreeIterator = .{ .data = tree_data };
113		while (try tree_iter.next()) \|entry\| {
114		switch (entry.type) {
115		.directory => {
116		try dir.makeDir(entry.name);
117		var subdir = try dir.openDir(entry.name, .{});
118		defer subdir.close();
119		const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name });
120		defer repository.odb.allocator.free(sub_path);
121		try repository.checkoutTree(subdir, entry.oid, sub_path, diagnostics);
122		},
123		.file => {
124		var file = try dir.createFile(entry.name, .{});
125		defer file.close();
126		try repository.odb.seekOid(entry.oid);
127		var file_object = try repository.odb.readObject();
128		if (file_object.type != .blob) return error.InvalidFile;
129		try file.writeAll(file_object.data);
130		try file.sync();
131		},
132		.symlink => {
133		try repository.odb.seekOid(entry.oid);
134		var symlink_object = try repository.odb.readObject();
135		if (symlink_object.type != .blob) return error.InvalidFile;
136		const link_name = symlink_object.data;
137		dir.symLink(link_name, entry.name, .{}) catch \|e\| {
138		const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name });
139		errdefer diagnostics.allocator.free(file_name);
140		const link_name_dup = try diagnostics.allocator.dupe(u8, link_name);
141		errdefer diagnostics.allocator.free(link_name_dup);
142		try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{
143		.code = e,
144		.file_name = file_name,
145		.link_name = link_name_dup,
146		} });
147		};
148		},
149		.gitlink => {
150		// Consistent with git archive behavior, create the directory but
151		// do nothing else
152		try dir.makeDir(entry.name);
153		},
154		}
155		}
156		}
157
158		/// Returns the ID of the tree associated with the given commit (provided as
159		/// raw object data).
160		fn getCommitTree(commit_data: []const u8) !Oid {
161		if (!mem.startsWith(u8, commit_data, "tree ") or
162		commit_data.len < "tree ".len + fmt_oid_length + "\n".len or
163		commit_data["tree ".len + fmt_oid_length] != '\n')
164		{
165		return error.InvalidCommit;
166		}
167		return try parseOid(commit_data["tree ".len..][0..fmt_oid_length]);
168		}
169
170		const TreeIterator = struct {
171		data: []const u8,
172		pos: usize = 0,
173
174		const Entry = struct {
175		type: Type,
176		executable: bool,
177		name: [:0]const u8,
178		oid: Oid,
179
180		const Type = enum(u4) {
181		directory = 0o4,
182		file = 0o10,
183		symlink = 0o12,
184		gitlink = 0o16,
185		};
186		};
187
188		fn next(iterator: *TreeIterator) !?Entry {
189		if (iterator.pos == iterator.data.len) return null;
190
191		const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree;
192		const mode: packed struct {
193		permission: u9,
194		unused: u3,
195		type: u4,
196		} = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree);
197		const @"type" = std.meta.intToEnum(Entry.Type, mode.type) catch return error.InvalidTree;
198		const executable = switch (mode.permission) {
199		0 => if (@"type" == .file) return error.InvalidTree else false,
200		0o644 => if (@"type" != .file) return error.InvalidTree else false,
201		0o755 => if (@"type" != .file) return error.InvalidTree else true,
202		else => return error.InvalidTree,
203		};
204		iterator.pos = mode_end + 1;
205
206		const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree;
207		const name = iterator.data[iterator.pos..name_end :0];
208		iterator.pos = name_end + 1;
209
210		if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree;
211		const oid = iterator.data[iterator.pos..][0..oid_length].*;
212		iterator.pos += oid_length;
213
214		return .{ .type = @"type", .executable = executable, .name = name, .oid = oid };
215		}
216		};
217		};
218
219		/// A Git object database backed by a packfile. A packfile index is also used
220		/// for efficient access to objects in the packfile.
221		///
222		/// The format of the packfile and its associated index are documented in
223		/// [pack-format](https://git-scm.com/docs/pack-format).
224		const Odb = struct {
225		pack_file: std.fs.File,
226		index_header: IndexHeader,
227		index_file: std.fs.File,
228		cache: ObjectCache = .{},
229		allocator: Allocator,
230
231		/// Initializes the database from open pack and index files.
232		fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Odb {
233		try pack_file.seekTo(0);
234		try index_file.seekTo(0);
235		const index_header = try IndexHeader.read(index_file.reader());
236		return .{
237		.pack_file = pack_file,
238		.index_header = index_header,
239		.index_file = index_file,
240		.allocator = allocator,
241		};
242		}
243
244		fn deinit(odb: *Odb) void {
245		odb.cache.deinit(odb.allocator);
246		odb.* = undefined;
247		}
248
249		/// Reads the object at the current position in the database.
250		fn readObject(odb: *Odb) !Object {
251		var base_offset = try odb.pack_file.getPos();
252		var base_header: EntryHeader = undefined;
253		var delta_offsets = std.ArrayListUnmanaged(u64){};
254		defer delta_offsets.deinit(odb.allocator);
255		const base_object = while (true) {
256		if (odb.cache.get(base_offset)) \|base_object\| break base_object;
257
258		base_header = try EntryHeader.read(odb.pack_file.reader());
259		switch (base_header) {
260		.ofs_delta => \|ofs_delta\| {
261		try delta_offsets.append(odb.allocator, base_offset);
262		base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat;
263		try odb.pack_file.seekTo(base_offset);
264		},
265		.ref_delta => \|ref_delta\| {
266		try delta_offsets.append(odb.allocator, base_offset);
267		try odb.seekOid(ref_delta.base_object);
268		base_offset = try odb.pack_file.getPos();
269		},
270		else => {
271		const base_data = try readObjectRaw(odb.allocator, odb.pack_file.reader(), base_header.uncompressedLength());
272		errdefer odb.allocator.free(base_data);
273		const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
274		try odb.cache.put(odb.allocator, base_offset, base_object);
275		break base_object;
276		},
277		}
278		};
279
280		const base_data = try resolveDeltaChain(
281		odb.allocator,
282		odb.pack_file,
283		base_object,
284		delta_offsets.items,
285		&odb.cache,
286		);
287
288		return .{ .type = base_object.type, .data = base_data };
289		}
290
291		/// Seeks to the beginning of the object with the given ID.
292		fn seekOid(odb: *Odb, oid: Oid) !void {
293		const key = oid[0];
294		var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0;
295		var end_index = odb.index_header.fan_out_table[key];
296		const found_index = while (start_index < end_index) {
297		const mid_index = start_index + (end_index - start_index) / 2;
298		try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length);
299		const mid_oid = try odb.index_file.reader().readBytesNoEof(oid_length);
300		switch (mem.order(u8, &mid_oid, &oid)) {
301		.lt => start_index = mid_index + 1,
302		.gt => end_index = mid_index,
303		.eq => break mid_index,
304		}
305		} else return error.ObjectNotFound;
306
307		const n_objects = odb.index_header.fan_out_table[255];
308		const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4);
309		try odb.index_file.seekTo(offset_values_start + found_index * 4);
310		const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.reader().readIntBig(u32));
311		const pack_offset = pack_offset: {
312		if (l1_offset.big) {
313		const l2_offset_values_start = offset_values_start + n_objects * 4;
314		try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4);
315		break :pack_offset try odb.index_file.reader().readIntBig(u64);
316		} else {
317		break :pack_offset l1_offset.value;
318		}
319		};
320
321		try odb.pack_file.seekTo(pack_offset);
322		}
323		};
324
325		const Object = struct {
326		type: Type,
327		data: []const u8,
328
329		const Type = enum {
330		commit,
331		tree,
332		blob,
333		tag,
334		};
335		};
336
337		/// A cache for object data.
338		///
339		/// The purpose of this cache is to speed up resolution of deltas by caching the
340		/// results of resolving delta objects, while maintaining a maximum cache size
341		/// to avoid excessive memory usage. If the total size of the objects in the
342		/// cache exceeds the maximum, the cache will begin evicting the least recently
343		/// used objects: when resolving delta chains, the most recently used objects
344		/// will likely be more helpful as they will be further along in the chain
345		/// (skipping earlier reconstruction steps).
346		///
347		/// Object data stored in the cache is managed by the cache. It should not be
348		/// freed by the caller at any point after inserting it into the cache. Any
349		/// objects remaining in the cache will be freed when the cache itself is freed.
350		const ObjectCache = struct {
351		objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .{},
352		lru_nodes: LruList = .{},
353		byte_size: usize = 0,
354
355		const max_byte_size = 128 * 1024 * 1024; // 128MiB
356		/// A list of offsets stored in the cache, with the most recently used
357		/// entries at the end.
358		const LruList = std.DoublyLinkedList(u64);
359		const CacheEntry = struct { object: Object, lru_node: *LruList.Node };
360
361		fn deinit(cache: *ObjectCache, allocator: Allocator) void {
362		var object_iterator = cache.objects.iterator();
363		while (object_iterator.next()) \|object\| {
364		allocator.free(object.value_ptr.object.data);
365		allocator.destroy(object.value_ptr.lru_node);
366		}
367		cache.objects.deinit(allocator);
368		cache.* = undefined;
369		}
370
371		/// Gets an object from the cache, moving it to the most recently used
372		/// position if it is present.
373		fn get(cache: *ObjectCache, offset: u64) ?Object {
374		if (cache.objects.get(offset)) \|entry\| {
375		cache.lru_nodes.remove(entry.lru_node);
376		cache.lru_nodes.append(entry.lru_node);
377		return entry.object;
378		} else {
379		return null;
380		}
381		}
382
383		/// Puts an object in the cache, possibly evicting older entries if the
384		/// cache exceeds its maximum size. Note that, although old objects may
385		/// be evicted, the object just added to the cache with this function
386		/// will not be evicted before the next call to `put` or `deinit` even if
387		/// it exceeds the maximum cache size.
388		fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void {
389		const lru_node = try allocator.create(LruList.Node);
390		errdefer allocator.destroy(lru_node);
391		lru_node.data = offset;
392
393		const gop = try cache.objects.getOrPut(allocator, offset);
394		if (gop.found_existing) {
395		cache.byte_size -= gop.value_ptr.object.data.len;
396		cache.lru_nodes.remove(gop.value_ptr.lru_node);
397		allocator.destroy(gop.value_ptr.lru_node);
398		allocator.free(gop.value_ptr.object.data);
399		}
400		gop.value_ptr.* = .{ .object = object, .lru_node = lru_node };
401		cache.byte_size += object.data.len;
402		cache.lru_nodes.append(lru_node);
403
404		while (cache.byte_size > max_byte_size and cache.lru_nodes.len > 1) {
405		// The > 1 check is to make sure that we don't evict the most
406		// recently added node, even if it by itself happens to exceed the
407		// maximum size of the cache.
408		const evict_node = cache.lru_nodes.popFirst().?;
409		const evict_offset = evict_node.data;
410		allocator.destroy(evict_node);
411		const evict_object = cache.objects.get(evict_offset).?.object;
412		cache.byte_size -= evict_object.data.len;
413		allocator.free(evict_object.data);
414		_ = cache.objects.remove(evict_offset);
415		}
416		}
417		};
418
419		/// A single pkt-line in the Git protocol.
420		///
421		/// The format of a pkt-line is documented in
422		/// [protocol-common](https://git-scm.com/docs/protocol-common). The special
423		/// meanings of the delimiter and response-end packets are documented in
424		/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
425		const Packet = union(enum) {
426		flush,
427		delimiter,
428		response_end,
429		data: []const u8,
430
431		const max_data_length = 65516;
432
433		/// Reads a packet in pkt-line format.
434		fn read(reader: anytype, buf: *[max_data_length]u8) !Packet {
435		const length = std.fmt.parseUnsigned(u16, &try reader.readBytesNoEof(4), 16) catch return error.InvalidPacket;
436		switch (length) {
437		0 => return .flush,
438		1 => return .delimiter,
439		2 => return .response_end,
440		3 => return error.InvalidPacket,
441		else => if (length - 4 > max_data_length) return error.InvalidPacket,
442		}
443		const data = buf[0 .. length - 4];
444		try reader.readNoEof(data);
445		return .{ .data = data };
446		}
447
448		/// Writes a packet in pkt-line format.
449		fn write(packet: Packet, writer: anytype) !void {
450		switch (packet) {
451		.flush => try writer.writeAll("0000"),
452		.delimiter => try writer.writeAll("0001"),
453		.response_end => try writer.writeAll("0002"),
454		.data => \|data\| {
455		assert(data.len <= max_data_length);
456		try writer.print("{x:0>4}", .{data.len + 4});
457		try writer.writeAll(data);
458		},
459		}
460		}
461		};
462
463		/// A client session for the Git protocol, currently limited to an HTTP(S)
464		/// transport. Only protocol version 2 is supported, as documented in
465		/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
466		pub const Session = struct {
467		transport: *std.http.Client,
468		uri: std.Uri,
469		supports_agent: bool = false,
470		supports_shallow: bool = false,
471
472		const agent = "zig/" ++ @import("builtin").zig_version_string;
473		const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent});
474
475		/// Discovers server capabilities. This should be called before using any
476		/// other client functionality, or the client will be forced to default to
477		/// the bare minimum server requirements, which may be considerably less
478		/// efficient (e.g. no shallow fetches).
479		///
480		/// See the note on `getCapabilities` regarding `redirect_uri`.
481		pub fn discoverCapabilities(
482		session: *Session,
483		allocator: Allocator,
484		redirect_uri: *[]u8,
485		) !void {
486		var capability_iterator = try session.getCapabilities(allocator, redirect_uri);
487		defer capability_iterator.deinit();
488		while (try capability_iterator.next()) \|capability\| {
489		if (mem.eql(u8, capability.key, "agent")) {
490		session.supports_agent = true;
491		} else if (mem.eql(u8, capability.key, "fetch")) {
492		var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' ');
493		while (feature_iterator.next()) \|feature\| {
494		if (mem.eql(u8, feature, "shallow")) {
495		session.supports_shallow = true;
496		}
497		}
498		}
499		}
500		}
501
502		/// Returns an iterator over capabilities supported by the server.
503		///
504		/// If the server redirects the request, `error.Redirected` is returned and
505		/// `redirect_uri` is populated with the URI resulting from the redirects.
506		/// When this occurs, the value of `redirect_uri` must be freed with
507		/// `allocator` when the caller is done with it.
508		fn getCapabilities(
509		session: Session,
510		allocator: Allocator,
511		redirect_uri: *[]u8,
512		) !CapabilityIterator {
513		var info_refs_uri = session.uri;
514		info_refs_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "info/refs" });
515		defer allocator.free(info_refs_uri.path);
516		info_refs_uri.query = "service=git-upload-pack";
517		info_refs_uri.fragment = null;
518
519		var headers = std.http.Headers.init(allocator);
520		defer headers.deinit();
521		try headers.append("Git-Protocol", "version=2");
522
523		var request = try session.transport.request(.GET, info_refs_uri, headers, .{
524		.max_redirects = 3,
525		});
526		errdefer request.deinit();
527		try request.start(.{});
528		try request.finish();
529
530		try request.wait();
531		if (request.response.status != .ok) return error.ProtocolError;
532		if (request.redirects_left < 3) {
533		if (!mem.endsWith(u8, request.uri.path, "/info/refs")) return error.UnparseableRedirect;
534		var new_uri = request.uri;
535		new_uri.path = new_uri.path[0 .. new_uri.path.len - "/info/refs".len];
536		new_uri.query = null;
537		redirect_uri.* = try std.fmt.allocPrint(allocator, "{+/}", .{new_uri});
538		return error.Redirected;
539		}
540
541		const reader = request.reader();
542		var buf: [Packet.max_data_length]u8 = undefined;
543		var state: enum { response_start, response_content } = .response_start;
544		while (true) {
545		// Some Git servers (at least GitHub) include an additional
546		// '# service=git-upload-pack' informative response before sending
547		// the expected 'version 2' packet and capability information.
548		// This is not universal: SourceHut, for example, does not do this.
549		// Thus, we need to skip any such useless additional responses
550		// before we get the one we're actually looking for. The responses
551		// will be delimited by flush packets.
552		const packet = Packet.read(reader, &buf) catch \|e\| switch (e) {
553		error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found
554		else => \|other\| return other,
555		};
556		switch (packet) {
557		.flush => state = .response_start,
558		.data => \|data\| switch (state) {
559		.response_start => if (mem.eql(u8, data, "version 2\n")) {
560		return .{ .request = request };
561		} else {
562		state = .response_content;
563		},
564		else => {},
565		},
566		else => return error.UnexpectedPacket,
567		}
568		}
569		}
570
571		const CapabilityIterator = struct {
572		request: std.http.Client.Request,
573		buf: [Packet.max_data_length]u8 = undefined,
574
575		const Capability = struct {
576		key: []const u8,
577		value: ?[]const u8 = null,
578		};
579
580		fn deinit(iterator: *CapabilityIterator) void {
581		iterator.request.deinit();
582		iterator.* = undefined;
583		}
584
585		fn next(iterator: *CapabilityIterator) !?Capability {
586		switch (try Packet.read(iterator.request.reader(), &iterator.buf)) {
587		.flush => return null,
588		.data => \|data\| if (data.len > 0 and data[data.len - 1] == '\n') {
589		if (mem.indexOfScalar(u8, data, '=')) \|separator_pos\| {
590		return .{ .key = data[0..separator_pos], .value = data[separator_pos + 1 .. data.len - 1] };
591		} else {
592		return .{ .key = data[0 .. data.len - 1] };
593		}
594		} else return error.UnexpectedPacket,
595		else => return error.UnexpectedPacket,
596		}
597		}
598		};
599
600		const ListRefsOptions = struct {
601		/// The ref prefixes (if any) to use to filter the refs available on the
602		/// server. Note that the client must still check the returned refs
603		/// against its desired filters itself: the server is not required to
604		/// respect these prefix filters and may return other refs as well.
605		ref_prefixes: []const []const u8 = &.{},
606		/// Whether to include symref targets for returned symbolic refs.
607		include_symrefs: bool = false,
608		/// Whether to include the peeled object ID for returned tag refs.
609		include_peeled: bool = false,
610		};
611
612		/// Returns an iterator over refs known to the server.
613		pub fn listRefs(session: Session, allocator: Allocator, options: ListRefsOptions) !RefIterator {
614		var upload_pack_uri = session.uri;
615		upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" });
616		defer allocator.free(upload_pack_uri.path);
617		upload_pack_uri.query = null;
618		upload_pack_uri.fragment = null;
619
620		var headers = std.http.Headers.init(allocator);
621		defer headers.deinit();
622		try headers.append("Content-Type", "application/x-git-upload-pack-request");
623		try headers.append("Git-Protocol", "version=2");
624
625		var body = std.ArrayListUnmanaged(u8){};
626		defer body.deinit(allocator);
627		const body_writer = body.writer(allocator);
628		try Packet.write(.{ .data = "command=ls-refs\n" }, body_writer);
629		if (session.supports_agent) {
630		try Packet.write(.{ .data = agent_capability }, body_writer);
631		}
632		try Packet.write(.delimiter, body_writer);
633		for (options.ref_prefixes) \|ref_prefix\| {
634		const ref_prefix_packet = try std.fmt.allocPrint(allocator, "ref-prefix {s}\n", .{ref_prefix});
635		defer allocator.free(ref_prefix_packet);
636		try Packet.write(.{ .data = ref_prefix_packet }, body_writer);
637		}
638		if (options.include_symrefs) {
639		try Packet.write(.{ .data = "symrefs\n" }, body_writer);
640		}
641		if (options.include_peeled) {
642		try Packet.write(.{ .data = "peel\n" }, body_writer);
643		}
644		try Packet.write(.flush, body_writer);
645
646		var request = try session.transport.request(.POST, upload_pack_uri, headers, .{
647		.handle_redirects = false,
648		});
649		errdefer request.deinit();
650		request.transfer_encoding = .{ .content_length = body.items.len };
651		try request.start(.{});
652		try request.writeAll(body.items);
653		try request.finish();
654
655		try request.wait();
656		if (request.response.status != .ok) return error.ProtocolError;
657
658		return .{ .request = request };
659		}
660
661		pub const RefIterator = struct {
662		request: std.http.Client.Request,
663		buf: [Packet.max_data_length]u8 = undefined,
664
665		pub const Ref = struct {
666		oid: Oid,
667		name: []const u8,
668		symref_target: ?[]const u8,
669		peeled: ?Oid,
670		};
671
672		pub fn deinit(iterator: *RefIterator) void {
673		iterator.request.deinit();
674		iterator.* = undefined;
675		}
676
677		pub fn next(iterator: *RefIterator) !?Ref {
678		switch (try Packet.read(iterator.request.reader(), &iterator.buf)) {
679		.flush => return null,
680		.data => \|data\| {
681		const oid_sep_pos = mem.indexOfScalar(u8, data, ' ') orelse return error.InvalidRefPacket;
682		const oid = parseOid(data[0..oid_sep_pos]) catch return error.InvalidRefPacket;
683
684		const name_sep_pos = mem.indexOfAnyPos(u8, data, oid_sep_pos + 1, " \n") orelse return error.InvalidRefPacket;
685		const name = data[oid_sep_pos + 1 .. name_sep_pos];
686
687		var symref_target: ?[]const u8 = null;
688		var peeled: ?Oid = null;
689		var last_sep_pos = name_sep_pos;
690		while (data[last_sep_pos] == ' ') {
691		const next_sep_pos = mem.indexOfAnyPos(u8, data, last_sep_pos + 1, " \n") orelse return error.InvalidRefPacket;
692		const attribute = data[last_sep_pos + 1 .. next_sep_pos];
693		if (mem.startsWith(u8, attribute, "symref-target:")) {
694		symref_target = attribute["symref-target:".len..];
695		} else if (mem.startsWith(u8, attribute, "peeled:")) {
696		peeled = parseOid(attribute["peeled:".len..]) catch return error.InvalidRefPacket;
697		}
698		last_sep_pos = next_sep_pos;
699		}
700
701		return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled };
702		},
703		else => return error.UnexpectedPacket,
704		}
705		}
706		};
707
708		/// Fetches the given refs from the server. A shallow fetch (depth 1) is
709		/// performed if the server supports it.
710		pub fn fetch(session: Session, allocator: Allocator, wants: []const []const u8) !FetchStream {
711		var upload_pack_uri = session.uri;
712		upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" });
713		defer allocator.free(upload_pack_uri.path);
714		upload_pack_uri.query = null;
715		upload_pack_uri.fragment = null;
716
717		var headers = std.http.Headers.init(allocator);
718		defer headers.deinit();
719		try headers.append("Content-Type", "application/x-git-upload-pack-request");
720		try headers.append("Git-Protocol", "version=2");
721
722		var body = std.ArrayListUnmanaged(u8){};
723		defer body.deinit(allocator);
724		const body_writer = body.writer(allocator);
725		try Packet.write(.{ .data = "command=fetch\n" }, body_writer);
726		if (session.supports_agent) {
727		try Packet.write(.{ .data = agent_capability }, body_writer);
728		}
729		try Packet.write(.delimiter, body_writer);
730		// Our packfile parser supports the OFS_DELTA object type
731		try Packet.write(.{ .data = "ofs-delta\n" }, body_writer);
732		// We do not currently convey server progress information to the user
733		try Packet.write(.{ .data = "no-progress\n" }, body_writer);
734		if (session.supports_shallow) {
735		try Packet.write(.{ .data = "deepen 1\n" }, body_writer);
736		}
737		for (wants) \|want\| {
738		var buf: [Packet.max_data_length]u8 = undefined;
739		const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable;
740		try Packet.write(.{ .data = arg }, body_writer);
741		}
742		try Packet.write(.{ .data = "done\n" }, body_writer);
743		try Packet.write(.flush, body_writer);
744
745		var request = try session.transport.request(.POST, upload_pack_uri, headers, .{
746		.handle_redirects = false,
747		});
748		errdefer request.deinit();
749		request.transfer_encoding = .{ .content_length = body.items.len };
750		try request.start(.{});
751		try request.writeAll(body.items);
752		try request.finish();
753
754		try request.wait();
755		if (request.response.status != .ok) return error.ProtocolError;
756
757		const reader = request.reader();
758		// We are not interested in any of the sections of the returned fetch
759		// data other than the packfile section, since we aren't doing anything
760		// complex like ref negotiation (this is a fresh clone).
761		var state: enum { section_start, section_content } = .section_start;
762		while (true) {
763		var buf: [Packet.max_data_length]u8 = undefined;
764		const packet = try Packet.read(reader, &buf);
765		switch (state) {
766		.section_start => switch (packet) {
767		.data => \|data\| if (mem.eql(u8, data, "packfile\n")) {
768		return .{ .request = request };
769		} else {
770		state = .section_content;
771		},
772		else => return error.UnexpectedPacket,
773		},
774		.section_content => switch (packet) {
775		.delimiter => state = .section_start,
776		.data => {},
777		else => return error.UnexpectedPacket,
778		},
779		}
780		}
781		}
782
783		pub const FetchStream = struct {
784		request: std.http.Client.Request,
785		buf: [Packet.max_data_length]u8 = undefined,
786		pos: usize = 0,
787		len: usize = 0,
788
789		pub fn deinit(stream: *FetchStream) void {
790		stream.request.deinit();
791		}
792
793		pub const ReadError = std.http.Client.Request.ReadError \|\| error{
794		InvalidPacket,
795		ProtocolError,
796		UnexpectedPacket,
797		};
798		pub const Reader = std.io.Reader(*FetchStream, ReadError, read);
799
800		const StreamCode = enum(u8) {
801		pack_data = 1,
802		progress = 2,
803		fatal_error = 3,
804		_,
805		};
806
807		pub fn reader(stream: *FetchStream) Reader {
808		return .{ .context = stream };
809		}
810
811		pub fn read(stream: *FetchStream, buf: []u8) !usize {
812		if (stream.pos == stream.len) {
813		while (true) {
814		switch (try Packet.read(stream.request.reader(), &stream.buf)) {
815		.flush => return 0,
816		.data => \|data\| if (data.len > 1) switch (@as(StreamCode, @enumFromInt(data[0]))) {
817		.pack_data => {
818		stream.pos = 1;
819		stream.len = data.len;
820		break;
821		},
822		.fatal_error => return error.ProtocolError,
823		else => {},
824		},
825		else => return error.UnexpectedPacket,
826		}
827		}
828		}
829
830		const size = @min(buf.len, stream.len - stream.pos);
831		@memcpy(buf[0..size], stream.buf[stream.pos .. stream.pos + size]);
832		stream.pos += size;
833		return size;
834		}
835		};
836		};
837
838		const PackHeader = struct {
839		total_objects: u32,
840
841		const signature = "PACK";
842		const supported_version = 2;
843
844		fn read(reader: anytype) !PackHeader {
845		const actual_signature = reader.readBytesNoEof(4) catch \|e\| switch (e) {
846		error.EndOfStream => return error.InvalidHeader,
847		else => \|other\| return other,
848		};
849		if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader;
850		const version = reader.readIntBig(u32) catch \|e\| switch (e) {
851		error.EndOfStream => return error.InvalidHeader,
852		else => \|other\| return other,
853		};
854		if (version != supported_version) return error.UnsupportedVersion;
855		const total_objects = reader.readIntBig(u32) catch \|e\| switch (e) {
856		error.EndOfStream => return error.InvalidHeader,
857		else => \|other\| return other,
858		};
859		return .{ .total_objects = total_objects };
860		}
861		};
862
863		const EntryHeader = union(Type) {
864		commit: Undeltified,
865		tree: Undeltified,
866		blob: Undeltified,
867		tag: Undeltified,
868		ofs_delta: OfsDelta,
869		ref_delta: RefDelta,
870
871		const Type = enum(u3) {
872		commit = 1,
873		tree = 2,
874		blob = 3,
875		tag = 4,
876		ofs_delta = 6,
877		ref_delta = 7,
878		};
879
880		const Undeltified = struct {
881		uncompressed_length: u64,
882		};
883
884		const OfsDelta = struct {
885		offset: u64,
886		uncompressed_length: u64,
887		};
888
889		const RefDelta = struct {
890		base_object: Oid,
891		uncompressed_length: u64,
892		};
893
894		fn objectType(header: EntryHeader) Object.Type {
895		return switch (header) {
896		inline .commit, .tree, .blob, .tag => \|_, tag\| @field(Object.Type, @tagName(tag)),
897		else => unreachable,
898		};
899		}
900
901		fn uncompressedLength(header: EntryHeader) u64 {
902		return switch (header) {
903		inline else => \|entry\| entry.uncompressed_length,
904		};
905		}
906
907		fn read(reader: anytype) !EntryHeader {
908		const InitialByte = packed struct { len: u4, type: u3, has_next: bool };
909		const initial: InitialByte = @bitCast(reader.readByte() catch \|e\| switch (e) {
910		error.EndOfStream => return error.InvalidFormat,
911		else => \|other\| return other,
912		});
913		const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0;
914		var uncompressed_length: u64 = initial.len;
915		uncompressed_length \|= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat;
916		const @"type" = std.meta.intToEnum(EntryHeader.Type, initial.type) catch return error.InvalidFormat;
917		return switch (@"type") {
918		inline .commit, .tree, .blob, .tag => \|tag\| @unionInit(EntryHeader, @tagName(tag), .{
919		.uncompressed_length = uncompressed_length,
920		}),
921		.ofs_delta => .{ .ofs_delta = .{
922		.offset = try readOffsetVarInt(reader),
923		.uncompressed_length = uncompressed_length,
924		} },
925		.ref_delta => .{ .ref_delta = .{
926		.base_object = reader.readBytesNoEof(oid_length) catch \|e\| switch (e) {
927		error.EndOfStream => return error.InvalidFormat,
928		else => \|other\| return other,
929		},
930		.uncompressed_length = uncompressed_length,
931		} },
932		};
933		}
934		};
935
936		fn readSizeVarInt(r: anytype) !u64 {
937		const Byte = packed struct { value: u7, has_next: bool };
938		var b: Byte = @bitCast(try r.readByte());
939		var value: u64 = b.value;
940		var shift: u6 = 0;
941		while (b.has_next) {
942		b = @bitCast(try r.readByte());
943		shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat;
944		value \|= @as(u64, b.value) << shift;
945		}
946		return value;
947		}
948
949		fn readOffsetVarInt(r: anytype) !u64 {
950		const Byte = packed struct { value: u7, has_next: bool };
951		var b: Byte = @bitCast(try r.readByte());
952		var value: u64 = b.value;
953		while (b.has_next) {
954		b = @bitCast(try r.readByte());
955		value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat;
956		value \|= b.value;
957		}
958		return value;
959		}
960
961		const IndexHeader = struct {
962		fan_out_table: [256]u32,
963
964		const signature = "\xFFtOc";
965		const supported_version = 2;
966		const size = 4 + 4 + @sizeOf([256]u32);
967
968		fn read(reader: anytype) !IndexHeader {
969		var header_bytes = try reader.readBytesNoEof(size);
970		if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader;
971		const version = mem.readIntBig(u32, header_bytes[4..8]);
972		if (version != supported_version) return error.UnsupportedVersion;
973
974		var fan_out_table: [256]u32 = undefined;
975		var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]);
976		const fan_out_table_reader = fan_out_table_stream.reader();
977		for (&fan_out_table) \|*entry\| {
978		entry.* = fan_out_table_reader.readIntBig(u32) catch unreachable;
979		}
980		return .{ .fan_out_table = fan_out_table };
981		}
982		};
983
984		const IndexEntry = struct {
985		offset: u64,
986		crc32: u32,
987		};
988
989		/// Writes out a version 2 index for the given packfile, as documented in
990		/// [pack-format](https://git-scm.com/docs/pack-format).
991		pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) !void {
992		try pack.seekTo(0);
993
994		var index_entries = std.AutoHashMapUnmanaged(Oid, IndexEntry){};
995		defer index_entries.deinit(allocator);
996		var pending_deltas = std.ArrayListUnmanaged(IndexEntry){};
997		defer pending_deltas.deinit(allocator);
998
999		const pack_checksum = try indexPackFirstPass(allocator, pack, &index_entries, &pending_deltas);
1000
1001		var cache: ObjectCache = .{};
1002		defer cache.deinit(allocator);
1003		var remaining_deltas = pending_deltas.items.len;
1004		while (remaining_deltas > 0) {
1005		var i: usize = remaining_deltas;
1006		while (i > 0) {
1007		i -= 1;
1008		const delta = pending_deltas.items[i];
1009		if (try indexPackHashDelta(allocator, pack, delta, index_entries, &cache)) \|oid\| {
1010		try index_entries.put(allocator, oid, delta);
1011		_ = pending_deltas.swapRemove(i);
1012		}
1013		}
1014		if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack;
1015		remaining_deltas = pending_deltas.items.len;
1016		}
1017
1018		var oids = std.ArrayListUnmanaged(Oid){};
1019		defer oids.deinit(allocator);
1020		try oids.ensureTotalCapacityPrecise(allocator, index_entries.count());
1021		var index_entries_iter = index_entries.iterator();
1022		while (index_entries_iter.next()) \|entry\| {
1023		oids.appendAssumeCapacity(entry.key_ptr.*);
1024		}
1025		mem.sortUnstable(Oid, oids.items, {}, struct {
1026		fn lessThan(_: void, o1: Oid, o2: Oid) bool {
1027		return mem.lessThan(u8, &o1, &o2);
1028		}
1029		}.lessThan);
1030
1031		var fan_out_table: [256]u32 = undefined;
1032		var count: u32 = 0;
1033		var fan_out_index: u8 = 0;
1034		for (oids.items) \|oid\| {
1035		if (oid[0] > fan_out_index) {
1036		@memset(fan_out_table[fan_out_index..oid[0]], count);
1037		fan_out_index = oid[0];
1038		}
1039		count += 1;
1040		}
1041		@memset(fan_out_table[fan_out_index..], count);
1042
1043		var index_hashed_writer = hashedWriter(index_writer, Sha1.init(.{}));
1044		const writer = index_hashed_writer.writer();
1045		try writer.writeAll(IndexHeader.signature);
1046		try writer.writeIntBig(u32, IndexHeader.supported_version);
1047		for (fan_out_table) \|fan_out_entry\| {
1048		try writer.writeIntBig(u32, fan_out_entry);
1049		}
1050
1051		for (oids.items) \|oid\| {
1052		try writer.writeAll(&oid);
1053		}
1054
1055		for (oids.items) \|oid\| {
1056		try writer.writeIntBig(u32, index_entries.get(oid).?.crc32);
1057		}
1058
1059		var big_offsets = std.ArrayListUnmanaged(u64){};
1060		defer big_offsets.deinit(allocator);
1061		for (oids.items) \|oid\| {
1062		const offset = index_entries.get(oid).?.offset;
1063		if (offset <= std.math.maxInt(u31)) {
1064		try writer.writeIntBig(u32, @intCast(offset));
1065		} else {
1066		const index = big_offsets.items.len;
1067		try big_offsets.append(allocator, offset);
1068		try writer.writeIntBig(u32, @as(u32, @intCast(index)) \| (1 << 31));
1069		}
1070		}
1071		for (big_offsets.items) \|offset\| {
1072		try writer.writeIntBig(u64, offset);
1073		}
1074
1075		try writer.writeAll(&pack_checksum);
1076		const index_checksum = index_hashed_writer.hasher.finalResult();
1077		try index_writer.writeAll(&index_checksum);
1078		}
1079
1080		/// Performs the first pass over the packfile data for index construction.
1081		/// This will index all non-delta objects, queue delta objects for further
1082		/// processing, and return the pack checksum (which is part of the index
1083		/// format).
1084		fn indexPackFirstPass(
1085		allocator: Allocator,
1086		pack: std.fs.File,
1087		index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),
1088		pending_deltas: *std.ArrayListUnmanaged(IndexEntry),
1089		) ![Sha1.digest_length]u8 {
1090		var pack_buffered_reader = std.io.bufferedReader(pack.reader());
1091		var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader());
1092		var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Sha1.init(.{}));
1093		const pack_reader = pack_hashed_reader.reader();
1094
1095		const pack_header = try PackHeader.read(pack_reader);
1096
1097		var current_entry: u32 = 0;
1098		while (current_entry < pack_header.total_objects) : (current_entry += 1) {
1099		const entry_offset = pack_counting_reader.bytes_read;
1100		var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init());
1101		const entry_header = try EntryHeader.read(entry_crc32_reader.reader());
1102		switch (entry_header) {
1103		inline .commit, .tree, .blob, .tag => \|object, tag\| {
1104		var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader());
1105		defer entry_decompress_stream.deinit();
1106		var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1107		var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{}));
1108		const entry_writer = entry_hashed_writer.writer();
1109		// The object header is not included in the pack data but is
1110		// part of the object's ID
1111		try entry_writer.print("{s} {}\x00", .{ @tagName(tag), object.uncompressed_length });
1112		var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1113		try fifo.pump(entry_counting_reader.reader(), entry_writer);
1114		if (entry_counting_reader.bytes_read != object.uncompressed_length) {
1115		return error.InvalidObject;
1116		}
1117		const oid = entry_hashed_writer.hasher.finalResult();
1118		try index_entries.put(allocator, oid, .{
1119		.offset = entry_offset,
1120		.crc32 = entry_crc32_reader.hasher.final(),
1121		});
1122		},
1123		inline .ofs_delta, .ref_delta => \|delta\| {
1124		var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader());
1125		defer entry_decompress_stream.deinit();
1126		var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1127		var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1128		try fifo.pump(entry_counting_reader.reader(), std.io.null_writer);
1129		if (entry_counting_reader.bytes_read != delta.uncompressed_length) {
1130		return error.InvalidObject;
1131		}
1132		try pending_deltas.append(allocator, .{
1133		.offset = entry_offset,
1134		.crc32 = entry_crc32_reader.hasher.final(),
1135		});
1136		},
1137		}
1138		}
1139
1140		const pack_checksum = pack_hashed_reader.hasher.finalResult();
1141		const recorded_checksum = try pack_buffered_reader.reader().readBytesNoEof(Sha1.digest_length);
1142		if (!mem.eql(u8, &pack_checksum, &recorded_checksum)) {
1143		return error.CorruptedPack;
1144		}
1145		_ = pack_buffered_reader.reader().readByte() catch \|e\| switch (e) {
1146		error.EndOfStream => return pack_checksum,
1147		else => \|other\| return other,
1148		};
1149		return error.InvalidFormat;
1150		}
1151
1152		/// Attempts to determine the final object ID of the given deltified object.
1153		/// May return null if this is not yet possible (if the delta is a ref-based
1154		/// delta and we do not yet know the offset of the base object).
1155		fn indexPackHashDelta(
1156		allocator: Allocator,
1157		pack: std.fs.File,
1158		delta: IndexEntry,
1159		index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),
1160		cache: *ObjectCache,
1161		) !?Oid {
1162		// Figure out the chain of deltas to resolve
1163		var base_offset = delta.offset;
1164		var base_header: EntryHeader = undefined;
1165		var delta_offsets = std.ArrayListUnmanaged(u64){};
1166		defer delta_offsets.deinit(allocator);
1167		const base_object = while (true) {
1168		if (cache.get(base_offset)) \|base_object\| break base_object;
1169
1170		try pack.seekTo(base_offset);
1171		base_header = try EntryHeader.read(pack.reader());
1172		switch (base_header) {
1173		.ofs_delta => \|ofs_delta\| {
1174		try delta_offsets.append(allocator, base_offset);
1175		base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject;
1176		},
1177		.ref_delta => \|ref_delta\| {
1178		try delta_offsets.append(allocator, base_offset);
1179		base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset;
1180		},
1181		else => {
1182		const base_data = try readObjectRaw(allocator, pack.reader(), base_header.uncompressedLength());
1183		errdefer allocator.free(base_data);
1184		const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
1185		try cache.put(allocator, base_offset, base_object);
1186		break base_object;
1187		},
1188		}
1189		};
1190
1191		const base_data = try resolveDeltaChain(allocator, pack, base_object, delta_offsets.items, cache);
1192
1193		var entry_hasher = Sha1.init(.{});
1194		var entry_hashed_writer = hashedWriter(std.io.null_writer, &entry_hasher);
1195		try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len });
1196		entry_hasher.update(base_data);
1197		return entry_hasher.finalResult();
1198		}
1199
1200		/// Resolves a chain of deltas, returning the final base object data. `pack` is
1201		/// assumed to be looking at the start of the object data for the base object of
1202		/// the chain, and will then apply the deltas in `delta_offsets` in reverse order
1203		/// to obtain the final object.
1204		fn resolveDeltaChain(
1205		allocator: Allocator,
1206		pack: std.fs.File,
1207		base_object: Object,
1208		delta_offsets: []const u64,
1209		cache: *ObjectCache,
1210		) ![]const u8 {
1211		var base_data = base_object.data;
1212		var i: usize = delta_offsets.len;
1213		while (i > 0) {
1214		i -= 1;
1215
1216		const delta_offset = delta_offsets[i];
1217		try pack.seekTo(delta_offset);
1218		const delta_header = try EntryHeader.read(pack.reader());
1219		var delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength());
1220		defer allocator.free(delta_data);
1221		var delta_stream = std.io.fixedBufferStream(delta_data);
1222		const delta_reader = delta_stream.reader();
1223		_ = try readSizeVarInt(delta_reader); // base object size
1224		const expanded_size = try readSizeVarInt(delta_reader);
1225
1226		const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge;
1227		var expanded_data = try allocator.alloc(u8, expanded_alloc_size);
1228		errdefer allocator.free(expanded_data);
1229		var expanded_delta_stream = std.io.fixedBufferStream(expanded_data);
1230		var base_stream = std.io.fixedBufferStream(base_data);
1231		try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer());
1232		if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject;
1233
1234		try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data });
1235		base_data = expanded_data;
1236		}
1237		return base_data;
1238		}
1239
1240		/// Reads the complete contents of an object from `reader`. This function may
1241		/// read more bytes than required from `reader`, so the reader position after
1242		/// returning is not reliable.
1243		fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 {
1244		const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;
1245		var buffered_reader = std.io.bufferedReader(reader);
1246		var decompress_stream = try std.compress.zlib.decompressStream(allocator, buffered_reader.reader());
1247		defer decompress_stream.deinit();
1248		var data = try allocator.alloc(u8, alloc_size);
1249		errdefer allocator.free(data);
1250		try decompress_stream.reader().readNoEof(data);
1251		_ = decompress_stream.reader().readByte() catch \|e\| switch (e) {
1252		error.EndOfStream => return data,
1253		else => \|other\| return other,
1254		};
1255		return error.InvalidFormat;
1256		}
1257
1258		/// Expands delta data from `delta_reader` to `writer`. `base_object` must
1259		/// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`).
1260		///
1261		/// The format of the delta data is documented in
1262		/// [pack-format](https://git-scm.com/docs/pack-format).
1263		fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !void {
1264		while (true) {
1265		const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch \|e\| switch (e) {
1266		error.EndOfStream => return,
1267		else => \|other\| return other,
1268		});
1269		if (inst.copy) {
1270		const available: packed struct {
1271		offset1: bool,
1272		offset2: bool,
1273		offset3: bool,
1274		offset4: bool,
1275		size1: bool,
1276		size2: bool,
1277		size3: bool,
1278		} = @bitCast(inst.value);
1279		var offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{
1280		.offset1 = if (available.offset1) try delta_reader.readByte() else 0,
1281		.offset2 = if (available.offset2) try delta_reader.readByte() else 0,
1282		.offset3 = if (available.offset3) try delta_reader.readByte() else 0,
1283		.offset4 = if (available.offset4) try delta_reader.readByte() else 0,
1284		};
1285		const offset: u32 = @bitCast(offset_parts);
1286		var size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{
1287		.size1 = if (available.size1) try delta_reader.readByte() else 0,
1288		.size2 = if (available.size2) try delta_reader.readByte() else 0,
1289		.size3 = if (available.size3) try delta_reader.readByte() else 0,
1290		};
1291		var size: u24 = @bitCast(size_parts);
1292		if (size == 0) size = 0x10000;
1293		try base_object.seekTo(offset);
1294		var copy_reader = std.io.limitedReader(base_object.reader(), size);
1295		var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1296		try fifo.pump(copy_reader.reader(), writer);
1297		} else if (inst.value != 0) {
1298		var data_reader = std.io.limitedReader(delta_reader, inst.value);
1299		var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1300		try fifo.pump(data_reader.reader(), writer);
1301		} else {
1302		return error.InvalidDeltaInstruction;
1303		}
1304		}
1305		}
1306
1307		fn HashedWriter(
1308		comptime WriterType: anytype,
1309		comptime HasherType: anytype,
1310		) type {
1311		return struct {
1312		child_writer: WriterType,
1313		hasher: HasherType,
1314
1315		const Error = WriterType.Error;
1316		const Writer = std.io.Writer(*@This(), Error, write);
1317
1318		fn write(hashed_writer: *@This(), buf: []const u8) Error!usize {
1319		const amt = try hashed_writer.child_writer.write(buf);
1320		hashed_writer.hasher.update(buf);
1321		return amt;
1322		}
1323
1324		fn writer(hashed_writer: *@This()) Writer {
1325		return .{ .context = hashed_writer };
1326		}
1327		};
1328		}
1329
1330		fn hashedWriter(
1331		writer: anytype,
1332		hasher: anytype,
1333		) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) {
1334		return .{ .child_writer = writer, .hasher = hasher };
1335		}
1336
1337		test "packfile indexing and checkout" {
1338		// To verify the contents of this packfile without using the code in this
1339		// file:
1340		//
1341		// 1. Create a new empty Git repository (`git init`)
1342		// 2. `git unpack-objects <path/to/testdata.pack`
1343		// 3. `git fsck` -> note the "dangling commit" ID (which matches the commit
1344		// checked out below)
1345		// 4. `git checkout dd582c0720819ab7130b103635bd7271b9fd4feb`
1346		const testrepo_pack = @embedFile("git/testdata/testrepo.pack");
1347
1348		var git_dir = testing.tmpDir(.{});
1349		defer git_dir.cleanup();
1350		var pack_file = try git_dir.dir.createFile("testrepo.pack", .{ .read = true });
1351		defer pack_file.close();
1352		try pack_file.writeAll(testrepo_pack);
1353
1354		var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true });
1355		defer index_file.close();
1356		try indexPack(testing.allocator, pack_file, index_file.writer());
1357
1358		// Arbitrary size limit on files read while checking the repository contents
1359		// (all files in the test repo are known to be much smaller than this)
1360		const max_file_size = 4096;
1361
1362		const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size);
1363		defer testing.allocator.free(index_file_data);
1364		// testrepo.idx is generated by Git. The index created by this file should
1365		// match it exactly. Running `git verify-pack -v testrepo.pack` can verify
1366		// this.
1367		const testrepo_idx = @embedFile("git/testdata/testrepo.idx");
1368		try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);
1369
1370		var repository = try Repository.init(testing.allocator, pack_file, index_file);
1371		defer repository.deinit();
1372
1373		var worktree = testing.tmpIterableDir(.{});
1374		defer worktree.cleanup();
1375
1376		const commit_id = try parseOid("dd582c0720819ab7130b103635bd7271b9fd4feb");
1377		try repository.checkout(worktree.iterable_dir.dir, commit_id);
1378
1379		const expected_files: []const []const u8 = &.{
1380		"dir/file",
1381		"dir/subdir/file",
1382		"dir/subdir/file2",
1383		"dir2/file",
1384		"dir3/file",
1385		"dir3/file2",
1386		"file",
1387		"file2",
1388		"file3",
1389		"file4",
1390		"file5",
1391		"file6",
1392		"file7",
1393		"file8",
1394		"file9",
1395		};
1396		var actual_files: std.ArrayListUnmanaged([]u8) = .{};
1397		defer actual_files.deinit(testing.allocator);
1398		defer for (actual_files.items) \|file\| testing.allocator.free(file);
1399		var walker = try worktree.iterable_dir.walk(testing.allocator);
1400		defer walker.deinit();
1401		while (try walker.next()) \|entry\| {
1402		if (entry.kind != .file) continue;
1403		var path = try testing.allocator.dupe(u8, entry.path);
1404		errdefer testing.allocator.free(path);
1405		mem.replaceScalar(u8, path, std.fs.path.sep, '/');
1406		try actual_files.append(testing.allocator, path);
1407		}
1408		mem.sortUnstable([]u8, actual_files.items, {}, struct {
1409		fn lessThan(_: void, a: []u8, b: []u8) bool {
1410		return mem.lessThan(u8, a, b);
1411		}
1412		}.lessThan);
1413		try testing.expectEqualDeep(expected_files, actual_files.items);
1414
1415		const expected_file_contents =
1416		\\revision 1
1417		\\revision 2
1418		\\revision 4
1419		\\revision 5
1420		\\revision 7
1421		\\revision 8
1422		\\revision 9
1423		\\revision 10
1424		\\revision 12
1425		\\revision 13
1426		\\revision 14
1427		\\revision 18
1428		\\revision 19
1429		\\
1430		;
1431		const actual_file_contents = try worktree.iterable_dir.dir.readFileAlloc(testing.allocator, "file", max_file_size);
1432		defer testing.allocator.free(actual_file_contents);
1433		try testing.expectEqualStrings(expected_file_contents, actual_file_contents);
1434		}
1435
1436		/// Checks out a commit of a packfile. Intended for experimenting with and
1437		/// benchmarking possible optimizations to the indexing and checkout behavior.
1438		pub fn main() !void {
1439		const allocator = std.heap.c_allocator;
1440
1441		const args = try std.process.argsAlloc(allocator);
1442		defer std.process.argsFree(allocator, args);
1443		if (args.len != 4) {
1444		return error.InvalidArguments; // Arguments: packfile commit worktree
1445		}
1446
1447		var pack_file = try std.fs.cwd().openFile(args[1], .{});
1448		defer pack_file.close();
1449		const commit = try parseOid(args[2]);
1450		var worktree = try std.fs.cwd().makeOpenPath(args[3], .{});
1451		defer worktree.close();
1452
1453		var git_dir = try worktree.makeOpenPath(".git", .{});
1454		defer git_dir.close();
1455
1456		std.debug.print("Starting index...\n", .{});
1457		var index_file = try git_dir.createFile("idx", .{ .read = true });
1458		defer index_file.close();
1459		var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
1460		try indexPack(allocator, pack_file, index_buffered_writer.writer());
1461		try index_buffered_writer.flush();
1462		try index_file.sync();
1463
1464		std.debug.print("Starting checkout...\n", .{});
1465		var repository = try Repository.init(allocator, pack_file, index_file);
1466		defer repository.deinit();
1467		try repository.checkout(worktree, commit);
1468		}

src/git/testdata/testrepo.idx deleted

Binary files a/src/git/testdata/testrepo.idx and /dev/null differ

src/git/testdata/testrepo.pack deleted

Binary files a/src/git/testdata/testrepo.pack and /dev/null differ

src/glibc.zig+1-1

...	...	@@ -1074,7 +1074,7 @@ fn buildSharedLib(
1074	1074	.cache_mode = .whole,
1075	1075	.target = comp.getTarget(),
1076	1076	.root_name = lib.name,
1077		.main_pkg = null,
	1077	.main_mod = null,
1078	1078	.output_mode = .Lib,
1079	1079	.link_mode = .Dynamic,
1080	1080	.thread_pool = comp.thread_pool,

src/libcxx.zig+2-2

...	...	@@ -233,7 +233,7 @@ pub fn buildLibCXX(comp: Compilation, prog_node: std.Progress.Node) !void {
233	233	.cache_mode = .whole,
234	234	.target = target,
235	235	.root_name = root_name,
236		.main_pkg = null,
	236	.main_mod = null,
237	237	.output_mode = output_mode,
238	238	.thread_pool = comp.thread_pool,
239	239	.libc_installation = comp.bin_file.options.libc_installation,
...	...	@@ -396,7 +396,7 @@ pub fn buildLibCXXABI(comp: Compilation, prog_node: std.Progress.Node) !void {
396	396	.cache_mode = .whole,
397	397	.target = target,
398	398	.root_name = root_name,
399		.main_pkg = null,
	399	.main_mod = null,
400	400	.output_mode = output_mode,
401	401	.thread_pool = comp.thread_pool,
402	402	.libc_installation = comp.bin_file.options.libc_installation,

src/libtsan.zig+1-1

...	...	@@ -202,7 +202,7 @@ pub fn buildTsan(comp: Compilation, prog_node: std.Progress.Node) !void {
202	202	.cache_mode = .whole,
203	203	.target = target,
204	204	.root_name = root_name,
205		.main_pkg = null,
	205	.main_mod = null,
206	206	.output_mode = output_mode,
207	207	.thread_pool = comp.thread_pool,
208	208	.libc_installation = comp.bin_file.options.libc_installation,

src/libunwind.zig+1-1

...	...	@@ -89,7 +89,7 @@ pub fn buildStaticLib(comp: Compilation, prog_node: std.Progress.Node) !void {
89	89	.cache_mode = .whole,
90	90	.target = target,
91	91	.root_name = root_name,
92		.main_pkg = null,
	92	.main_mod = null,
93	93	.output_mode = output_mode,
94	94	.thread_pool = comp.thread_pool,
95	95	.libc_installation = comp.bin_file.options.libc_installation,

src/link/Dwarf.zig+13-5

...	...	@@ -1880,7 +1880,7 @@ pub fn writeDbgInfoHeader(self: Dwarf, module: Module, low_pc: u64, high_pc: u
1880	1880	},
1881	1881	}
1882	1882	// Write the form for the compile unit, which must match the abbrev table above.
1883		const name_strp = try self.strtab.insert(self.allocator, module.root_pkg.root_src_path);
	1883	const name_strp = try self.strtab.insert(self.allocator, module.root_mod.root_src_path);
1884	1884	var compile_unit_dir_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
1885	1885	const compile_unit_dir = resolveCompilationDir(module, &compile_unit_dir_buffer);
1886	1886	const comp_dir_strp = try self.strtab.insert(self.allocator, compile_unit_dir);
...	...	@@ -1940,9 +1940,17 @@ fn resolveCompilationDir(module: Module, buffer: [std.fs.MAX_PATH_BYTES]u8) []
1940	1940	// be very location dependent.
1941	1941	// TODO: the only concern I have with this is WASI as either host or target, should
1942	1942	// we leave the paths as relative then?
1943		const comp_dir_path = module.root_pkg.root_src_directory.path orelse ".";
1944		if (std.fs.path.isAbsolute(comp_dir_path)) return comp_dir_path;
1945		return std.os.realpath(comp_dir_path, buffer) catch comp_dir_path; // If realpath fails, fallback to whatever comp_dir_path was
	1943	const root_dir_path = module.root_mod.root.root_dir.path orelse ".";
	1944	const sub_path = module.root_mod.root.sub_path;
	1945	const realpath = if (std.fs.path.isAbsolute(root_dir_path)) r: {
	1946	@memcpy(buffer[0..root_dir_path.len], root_dir_path);
	1947	break :r root_dir_path;
	1948	} else std.fs.realpath(root_dir_path, buffer) catch return root_dir_path;
	1949	const len = realpath.len + 1 + sub_path.len;
	1950	if (buffer.len < len) return root_dir_path;
	1951	buffer[realpath.len] = '/';
	1952	@memcpy(buffer[realpath.len + 1 ..][0..sub_path.len], sub_path);
	1953	return buffer[0..len];
1946	1954	}
1947	1955
1948	1956	fn writeAddrAssumeCapacity(self: Dwarf, buf: std.ArrayList(u8), addr: u64) void {
...	...	@@ -2664,7 +2672,7 @@ fn genIncludeDirsAndFileNames(self: *Dwarf, arena: Allocator) !struct {
2664	2672	for (self.di_files.keys()) \|dif\| {
2665	2673	const dir_path = d: {
2666	2674	var buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
2667		const dir_path = dif.pkg.root_src_directory.path orelse ".";
	2675	const dir_path = try dif.mod.root.joinString(arena, dif.mod.root.sub_path);
2668	2676	const abs_dir_path = if (std.fs.path.isAbsolute(dir_path))
2669	2677	dir_path
2670	2678	else

src/link/Elf.zig+3-3

...	...	@@ -929,15 +929,15 @@ pub fn populateMissingMetadata(self: *Elf) !void {
929	929
930	930	if (self.base.options.module) \|module\| {
931	931	if (self.zig_module_index == null and !self.base.options.use_llvm) {
932		const index = @as(File.Index, @intCast(try self.files.addOne(gpa)));
	932	const index: File.Index = @intCast(try self.files.addOne(gpa));
933	933	self.files.set(index, .{ .zig_module = .{
934	934	.index = index,
935		.path = module.main_pkg.root_src_path,
	935	.path = module.main_mod.root_src_path,
936	936	} });
937	937	self.zig_module_index = index;
938	938	const zig_module = self.file(index).?.zig_module;
939	939
940		const name_off = try self.strtab.insert(gpa, std.fs.path.stem(module.main_pkg.root_src_path));
	940	const name_off = try self.strtab.insert(gpa, std.fs.path.stem(module.main_mod.root_src_path));
941	941	const symbol_index = try self.addSymbol();
942	942	try zig_module.local_symbols.append(gpa, symbol_index);
943	943	const symbol_ptr = self.symbol(symbol_index);

src/link/Plan9.zig+6-3

...	...	@@ -352,9 +352,12 @@ fn putFn(self: *Plan9, decl_index: Module.Decl.Index, out: FnDeclOutput) !void {
352	352
353	353	// getting the full file path
354	354	var buf: [std.fs.MAX_PATH_BYTES]u8 = undefined;
355		const dir = file.pkg.root_src_directory.path orelse try std.os.getcwd(&buf);
356		const sub_path = try std.fs.path.join(arena, &.{ dir, file.sub_file_path });
357		try self.addPathComponents(sub_path, &a);
	355	const full_path = try std.fs.path.join(arena, &.{
	356	file.mod.root.root_dir.path orelse try std.os.getcwd(&buf),
	357	file.mod.root.sub_path,
	358	file.sub_file_path,
	359	});
	360	try self.addPathComponents(full_path, &a);
358	361
359	362	// null terminate
360	363	try a.append(0);

src/main.zig+327-207

...	...	@@ -416,7 +416,7 @@ const usage_build_generic =
416	416	\\ dep: [[import=]name]
417	417	\\ --deps [dep],[dep],... Set dependency names for the root package
418	418	\\ dep: [[import=]name]
419		\\ --main-pkg-path Set the directory of the root package
	419	\\ --main-mod-path Set the directory of the root module
420	420	\\ -fPIC Force-enable Position Independent Code
421	421	\\ -fno-PIC Force-disable Position Independent Code
422	422	\\ -fPIE Force-enable Position Independent Executable
...	...	@@ -765,17 +765,11 @@ const Framework = struct {
765	765	};
766	766
767	767	const CliModule = struct {
768		mod: *Package,
	768	mod: *Package.Module,
769	769	/// still in CLI arg format
770	770	deps_str: []const u8,
771	771	};
772	772
773		fn cleanupModules(modules: *std.StringArrayHashMap(CliModule)) void {
774		var it = modules.iterator();
775		while (it.next()) \|kv\| kv.value_ptr.mod.destroy(modules.allocator);
776		modules.deinit();
777		}
778
779	773	fn buildOutputType(
780	774	gpa: Allocator,
781	775	arena: Allocator,
...	...	@@ -903,7 +897,7 @@ fn buildOutputType(
903	897	var override_local_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_LOCAL_CACHE_DIR");
904	898	var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");
905	899	var override_lib_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_LIB_DIR");
906		var main_pkg_path: ?[]const u8 = null;
	900	var main_mod_path: ?[]const u8 = null;
907	901	var clang_preprocessor_mode: Compilation.ClangPreprocessorMode = .no;
908	902	var subsystem: ?std.Target.SubSystem = null;
909	903	var major_subsystem_version: ?u32 = null;
...	...	@@ -950,8 +944,7 @@ fn buildOutputType(
950	944	// Contains every module specified via --mod. The dependencies are added
951	945	// after argument parsing is completed. We use a StringArrayHashMap to make
952	946	// error output consistent.
953		var modules = std.StringArrayHashMap(CliModule).init(gpa);
954		defer cleanupModules(&modules);
	947	var modules = std.StringArrayHashMap(CliModule).init(arena);
955	948
956	949	// The dependency string for the root package
957	950	var root_deps_str: ?[]const u8 = null;
...	...	@@ -1023,33 +1016,36 @@ fn buildOutputType(
1023	1016
1024	1017	for ([_][]const u8{ "std", "root", "builtin" }) \|name\| {
1025	1018	if (mem.eql(u8, mod_name, name)) {
1026		fatal("unable to add module '{s}' -> '{s}': conflicts with builtin module", .{ mod_name, root_src });
	1019	fatal("unable to add module '{s}' -> '{s}': conflicts with builtin module", .{
	1020	mod_name, root_src,
	1021	});
1027	1022	}
1028	1023	}
1029	1024
1030		var mod_it = modules.iterator();
1031		while (mod_it.next()) \|kv\| {
1032		if (std.mem.eql(u8, mod_name, kv.key_ptr.*)) {
1033		fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{ mod_name, root_src, kv.value_ptr.mod.root_src_path });
1034		}
	1025	if (modules.get(mod_name)) \|value\| {
	1026	fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{
	1027	mod_name, root_src, value.mod.root_src_path,
	1028	});
1035	1029	}
1036	1030
1037		try modules.ensureUnusedCapacity(1);
1038		modules.put(mod_name, .{
1039		.mod = try Package.create(
1040		gpa,
1041		fs.path.dirname(root_src),
1042		fs.path.basename(root_src),
1043		),
	1031	try modules.put(mod_name, .{
	1032	.mod = try Package.Module.create(arena, .{
	1033	.root = .{
	1034	.root_dir = Cache.Directory.cwd(),
	1035	.sub_path = fs.path.dirname(root_src) orelse "",
	1036	},
	1037	.root_src_path = fs.path.basename(root_src),
	1038	.fully_qualified_name = mod_name,
	1039	}),
1044	1040	.deps_str = deps_str,
1045		}) catch unreachable;
	1041	});
1046	1042	} else if (mem.eql(u8, arg, "--deps")) {
1047	1043	if (root_deps_str != null) {
1048	1044	fatal("only one --deps argument is allowed", .{});
1049	1045	}
1050	1046	root_deps_str = args_iter.nextOrFatal();
1051		} else if (mem.eql(u8, arg, "--main-pkg-path")) {
1052		main_pkg_path = args_iter.nextOrFatal();
	1047	} else if (mem.eql(u8, arg, "--main-mod-path")) {
	1048	main_mod_path = args_iter.nextOrFatal();
1053	1049	} else if (mem.eql(u8, arg, "-cflags")) {
1054	1050	extra_cflags.shrinkRetainingCapacity(0);
1055	1051	while (true) {
...	...	@@ -2461,19 +2457,26 @@ fn buildOutputType(
2461	2457	var deps_it = ModuleDepIterator.init(deps_str);
2462	2458	while (deps_it.next()) \|dep\| {
2463	2459	if (dep.expose.len == 0) {
2464		fatal("module '{s}' depends on '{s}' with a blank name", .{ kv.key_ptr.*, dep.name });
	2460	fatal("module '{s}' depends on '{s}' with a blank name", .{
	2461	kv.key_ptr.*, dep.name,
	2462	});
2465	2463	}
2466	2464
2467	2465	for ([_][]const u8{ "std", "root", "builtin" }) \|name\| {
2468	2466	if (mem.eql(u8, dep.expose, name)) {
2469		fatal("unable to add module '{s}' under name '{s}': conflicts with builtin module", .{ dep.name, dep.expose });
	2467	fatal("unable to add module '{s}' under name '{s}': conflicts with builtin module", .{
	2468	dep.name, dep.expose,
	2469	});
2470	2470	}
2471	2471	}
2472	2472
2473		const dep_mod = modules.get(dep.name) orelse
2474		fatal("module '{s}' depends on module '{s}' which does not exist", .{ kv.key_ptr.*, dep.name });
	2473	const dep_mod = modules.get(dep.name) orelse {
	2474	fatal("module '{s}' depends on module '{s}' which does not exist", .{
	2475	kv.key_ptr.*, dep.name,
	2476	});
	2477	};
2475	2478
2476		try kv.value_ptr.mod.add(gpa, dep.expose, dep_mod.mod);
	2479	try kv.value_ptr.mod.deps.put(arena, dep.expose, dep_mod.mod);
2477	2480	}
2478	2481	}
2479	2482	}
...	...	@@ -3229,31 +3232,35 @@ fn buildOutputType(
3229	3232	};
3230	3233	defer emit_implib_resolved.deinit();
3231	3234
3232		const main_pkg: ?*Package = if (root_src_file) \|unresolved_src_path\| blk: {
	3235	const main_mod: ?*Package.Module = if (root_src_file) \|unresolved_src_path\| blk: {
3233	3236	const src_path = try introspect.resolvePath(arena, unresolved_src_path);
3234		if (main_pkg_path) \|unresolved_main_pkg_path\| {
3235		const p = try introspect.resolvePath(arena, unresolved_main_pkg_path);
3236		if (p.len == 0) {
3237		break :blk try Package.create(gpa, null, src_path);
3238		} else {
3239		const rel_src_path = try fs.path.relative(arena, p, src_path);
3240		break :blk try Package.create(gpa, p, rel_src_path);
3241		}
	3237	if (main_mod_path) \|unresolved_main_mod_path\| {
	3238	const p = try introspect.resolvePath(arena, unresolved_main_mod_path);
	3239	break :blk try Package.Module.create(arena, .{
	3240	.root = .{
	3241	.root_dir = Cache.Directory.cwd(),
	3242	.sub_path = p,
	3243	},
	3244	.root_src_path = if (p.len == 0)
	3245	src_path
	3246	else
	3247	try fs.path.relative(arena, p, src_path),
	3248	.fully_qualified_name = "root",
	3249	});
3242	3250	} else {
3243		const root_src_dir_path = fs.path.dirname(src_path);
3244		break :blk Package.create(gpa, root_src_dir_path, fs.path.basename(src_path)) catch \|err\| {
3245		if (root_src_dir_path) \|p\| {
3246		fatal("unable to open '{s}': {s}", .{ p, @errorName(err) });
3247		} else {
3248		return err;
3249		}
3250		};
	3251	break :blk try Package.Module.create(arena, .{
	3252	.root = .{
	3253	.root_dir = Cache.Directory.cwd(),
	3254	.sub_path = fs.path.dirname(src_path) orelse "",
	3255	},
	3256	.root_src_path = fs.path.basename(src_path),
	3257	.fully_qualified_name = "root",
	3258	});
3251	3259	}
3252	3260	} else null;
3253		defer if (main_pkg) \|p\| p.destroy(gpa);
3254	3261
3255	3262	// Transfer packages added with --deps to the root package
3256		if (main_pkg) \|mod\| {
	3263	if (main_mod) \|mod\| {
3257	3264	var it = ModuleDepIterator.init(root_deps_str orelse "");
3258	3265	while (it.next()) \|dep\| {
3259	3266	if (dep.expose.len == 0) {
...	...	@@ -3269,7 +3276,7 @@ fn buildOutputType(
3269	3276	const dep_mod = modules.get(dep.name) orelse
3270	3277	fatal("root module depends on module '{s}' which does not exist", .{dep.name});
3271	3278
3272		try mod.add(gpa, dep.expose, dep_mod.mod);
	3279	try mod.deps.put(arena, dep.expose, dep_mod.mod);
3273	3280	}
3274	3281	}
3275	3282
...	...	@@ -3310,17 +3317,18 @@ fn buildOutputType(
3310	3317	if (arg_mode == .run) {
3311	3318	break :l global_cache_directory;
3312	3319	}
3313		if (main_pkg) \|pkg\| {
	3320	if (main_mod != null) {
3314	3321	// search upwards from cwd until we find directory with build.zig
3315	3322	const cwd_path = try process.getCwdAlloc(arena);
3316		const build_zig = "build.zig";
3317	3323	const zig_cache = "zig-cache";
3318	3324	var dirname: []const u8 = cwd_path;
3319	3325	while (true) {
3320		const joined_path = try fs.path.join(arena, &[_][]const u8{ dirname, build_zig });
	3326	const joined_path = try fs.path.join(arena, &.{
	3327	dirname, Package.build_zig_basename,
	3328	});
3321	3329	if (fs.cwd().access(joined_path, .{})) \|_\| {
3322		const cache_dir_path = try fs.path.join(arena, &[_][]const u8{ dirname, zig_cache });
3323		const dir = try pkg.root_src_directory.handle.makeOpenPath(cache_dir_path, .{});
	3330	const cache_dir_path = try fs.path.join(arena, &.{ dirname, zig_cache });
	3331	const dir = try fs.cwd().makeOpenPath(cache_dir_path, .{});
3324	3332	cleanup_local_cache_dir = dir;
3325	3333	break :l .{ .handle = dir, .path = cache_dir_path };
3326	3334	} else \|err\| switch (err) {
...	...	@@ -3389,7 +3397,7 @@ fn buildOutputType(
3389	3397	.dynamic_linker = target_info.dynamic_linker.get(),
3390	3398	.sysroot = sysroot,
3391	3399	.output_mode = output_mode,
3392		.main_pkg = main_pkg,
	3400	.main_mod = main_mod,
3393	3401	.emit_bin = emit_bin_loc,
3394	3402	.emit_h = emit_h_resolved.data,
3395	3403	.emit_asm = emit_asm_resolved.data,
...	...	@@ -4613,11 +4621,14 @@ pub const usage_build =
4613	4621	\\ --global-cache-dir [path] Override path to global Zig cache directory
4614	4622	\\ --zig-lib-dir [arg] Override path to Zig lib directory
4615	4623	\\ --build-runner [file] Override path to build runner
	4624	\\ --fetch Exit after fetching dependency tree
4616	4625	\\ -h, --help Print this help and exit
4617	4626	\\
4618	4627	;
4619	4628
4620	4629	pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
	4630	const work_around_btrfs_bug = builtin.os.tag == .linux and
	4631	std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND");
4621	4632	var color: Color = .auto;
4622	4633
4623	4634	// We want to release all the locks before executing the child process, so we make a nice
...	...	@@ -4633,6 +4644,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4633	4644	var child_argv = std.ArrayList([]const u8).init(arena);
4634	4645	var reference_trace: ?u32 = null;
4635	4646	var debug_compile_errors = false;
	4647	var fetch_only = false;
4636	4648
4637	4649	const argv_index_exe = child_argv.items.len;
4638	4650	_ = try child_argv.addOne();
...	...	@@ -4682,6 +4694,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4682	4694	} else if (mem.eql(u8, arg, "-freference-trace")) {
4683	4695	try child_argv.append(arg);
4684	4696	reference_trace = 256;
	4697	} else if (mem.eql(u8, arg, "--fetch")) {
	4698	fetch_only = true;
4685	4699	} else if (mem.startsWith(u8, arg, "-freference-trace=")) {
4686	4700	try child_argv.append(arg);
4687	4701	const num = arg["-freference-trace=".len..];
...	...	@@ -4714,8 +4728,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4714	4728	defer if (cleanup_build_dir) \|*dir\| dir.close();
4715	4729
4716	4730	const cwd_path = try process.getCwdAlloc(arena);
4717		const build_zig_basename = if (build_file) \|bf\| fs.path.basename(bf) else "build.zig";
4718		const build_directory: Compilation.Directory = blk: {
	4731	const build_zig_basename = if (build_file) \|bf\| fs.path.basename(bf) else Package.build_zig_basename;
	4732	const build_root: Compilation.Directory = blk: {
4719	4733	if (build_file) \|bf\| {
4720	4734	if (fs.path.dirname(bf)) \|dirname\| {
4721	4735	const dir = fs.cwd().openDir(dirname, .{}) catch \|err\| {
...	...	@@ -4751,7 +4765,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4751	4765	}
4752	4766	}
4753	4767	};
4754		child_argv.items[argv_index_build_file] = build_directory.path orelse cwd_path;
	4768	child_argv.items[argv_index_build_file] = build_root.path orelse cwd_path;
4755	4769
4756	4770	var global_cache_directory: Compilation.Directory = l: {
4757	4771	const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena);
...	...	@@ -4771,9 +4785,9 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4771	4785	.path = local_cache_dir_path,
4772	4786	};
4773	4787	}
4774		const cache_dir_path = try build_directory.join(arena, &[_][]const u8{"zig-cache"});
	4788	const cache_dir_path = try build_root.join(arena, &[_][]const u8{"zig-cache"});
4775	4789	break :l .{
4776		.handle = try build_directory.handle.makeOpenPath("zig-cache", .{}),
	4790	.handle = try build_root.handle.makeOpenPath("zig-cache", .{}),
4777	4791	.path = cache_dir_path,
4778	4792	};
4779	4793	};
...	...	@@ -4799,97 +4813,150 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4799	4813	try thread_pool.init(.{ .allocator = gpa });
4800	4814	defer thread_pool.deinit();
4801	4815
4802		var cleanup_build_runner_dir: ?fs.Dir = null;
4803		defer if (cleanup_build_runner_dir) \|*dir\| dir.close();
4804
4805		var main_pkg: Package = if (override_build_runner) \|build_runner_path\|
	4816	var main_mod: Package.Module = if (override_build_runner) \|build_runner_path\|
4806	4817	.{
4807		.root_src_directory = blk: {
4808		if (std.fs.path.dirname(build_runner_path)) \|dirname\| {
4809		const dir = fs.cwd().openDir(dirname, .{}) catch \|err\| {
4810		fatal("unable to open directory to build runner from argument 'build-runner', '{s}': {s}", .{ dirname, @errorName(err) });
4811		};
4812		cleanup_build_runner_dir = dir;
4813		break :blk .{ .path = dirname, .handle = dir };
4814		}
4815
4816		break :blk .{ .path = null, .handle = fs.cwd() };
	4818	.root = .{
	4819	.root_dir = Cache.Directory.cwd(),
	4820	.sub_path = fs.path.dirname(build_runner_path) orelse "",
4817	4821	},
4818		.root_src_path = std.fs.path.basename(build_runner_path),
	4822	.root_src_path = fs.path.basename(build_runner_path),
	4823	.fully_qualified_name = "root",
4819	4824	}
4820	4825	else
4821	4826	.{
4822		.root_src_directory = zig_lib_directory,
	4827	.root = .{ .root_dir = zig_lib_directory },
4823	4828	.root_src_path = "build_runner.zig",
	4829	.fully_qualified_name = "root",
4824	4830	};
4825	4831
4826		var build_pkg: Package = .{
4827		.root_src_directory = build_directory,
	4832	var build_mod: Package.Module = .{
	4833	.root = .{ .root_dir = build_root },
4828	4834	.root_src_path = build_zig_basename,
	4835	.fully_qualified_name = "root.@build",
4829	4836	};
4830	4837	if (build_options.only_core_functionality) {
4831		const deps_pkg = try Package.createFilePkg(gpa, local_cache_directory, "dependencies.zig",
4832		\\pub const packages = struct {};
4833		\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
4834		\\
4835		);
4836		try main_pkg.add(gpa, "@dependencies", deps_pkg);
	4838	try createEmptyDependenciesModule(arena, &main_mod, local_cache_directory);
4837	4839	} else {
4838	4840	var http_client: std.http.Client = .{ .allocator = gpa };
4839	4841	defer http_client.deinit();
4840	4842
4841		// Here we provide an import to the build runner that allows using reflection to find
4842		// all of the dependencies. Without this, there would be no way to use `@import` to
4843		// access dependencies by name, since `@import` requires string literals.
4844		var dependencies_source = std.ArrayList(u8).init(gpa);
4845		defer dependencies_source.deinit();
4846
4847		var all_modules: Package.AllModules = .{};
4848		defer all_modules.deinit(gpa);
4849
4850		var wip_errors: std.zig.ErrorBundle.Wip = undefined;
4851		try wip_errors.init(gpa);
4852		defer wip_errors.deinit();
4853
4854	4843	var progress: std.Progress = .{ .dont_print_on_dumb = true };
4855	4844	const root_prog_node = progress.start("Fetch Packages", 0);
4856	4845	defer root_prog_node.end();
4857	4846
4858		// Here we borrow main package's table and will replace it with a fresh
4859		// one after this process completes.
4860		const fetch_result = build_pkg.fetchAndAddDependencies(
4861		&main_pkg,
4862		arena,
4863		&thread_pool,
4864		&http_client,
4865		build_directory,
4866		global_cache_directory,
4867		local_cache_directory,
4868		&dependencies_source,
4869		&wip_errors,
4870		&all_modules,
4871		root_prog_node,
4872		null,
	4847	var job_queue: Package.Fetch.JobQueue = .{
	4848	.http_client = &http_client,
	4849	.thread_pool = &thread_pool,
	4850	.global_cache = global_cache_directory,
	4851	.recursive = true,
	4852	.work_around_btrfs_bug = work_around_btrfs_bug,
	4853	};
	4854	defer job_queue.deinit();
	4855
	4856	try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1);
	4857	try job_queue.table.ensureUnusedCapacity(gpa, 1);
	4858
	4859	var fetch: Package.Fetch = .{
	4860	.arena = std.heap.ArenaAllocator.init(gpa),
	4861	.location = .{ .relative_path = build_mod.root },
	4862	.location_tok = 0,
	4863	.hash_tok = 0,
	4864	.parent_package_root = build_mod.root,
	4865	.parent_manifest_ast = null,
	4866	.prog_node = root_prog_node,
	4867	.job_queue = &job_queue,
	4868	.omit_missing_hash_error = true,
	4869	.allow_missing_paths_field = false,
	4870
	4871	.package_root = undefined,
	4872	.error_bundle = undefined,
	4873	.manifest = null,
	4874	.manifest_ast = undefined,
	4875	.actual_hash = undefined,
	4876	.has_build_zig = true,
	4877	.oom_flag = false,
	4878
	4879	.module = &build_mod,
	4880	};
	4881	job_queue.all_fetches.appendAssumeCapacity(&fetch);
	4882
	4883	job_queue.table.putAssumeCapacityNoClobber(
	4884	Package.Fetch.relativePathDigest(build_mod.root, global_cache_directory),
	4885	&fetch,
4873	4886	);
4874		if (wip_errors.root_list.items.len > 0) {
4875		var errors = try wip_errors.toOwnedBundle("");
4876		defer errors.deinit(gpa);
	4887
	4888	job_queue.wait_group.start();
	4889	try job_queue.thread_pool.spawn(Package.Fetch.workerRun, .{ &fetch, "root" });
	4890	job_queue.wait_group.wait();
	4891
	4892	try job_queue.consolidateErrors();
	4893
	4894	if (fetch.error_bundle.root_list.items.len > 0) {
	4895	var errors = try fetch.error_bundle.toOwnedBundle("");
4877	4896	errors.renderToStdErr(renderOptions(color));
4878	4897	process.exit(1);
4879	4898	}
4880		try fetch_result;
4881	4899
4882		const deps_pkg = try Package.createFilePkg(
4883		gpa,
	4900	if (fetch_only) return cleanExit();
	4901
	4902	var source_buf = std.ArrayList(u8).init(gpa);
	4903	defer source_buf.deinit();
	4904	try job_queue.createDependenciesSource(&source_buf);
	4905	const deps_mod = try createDependenciesModule(
	4906	arena,
	4907	source_buf.items,
	4908	&main_mod,
4884	4909	local_cache_directory,
4885		"dependencies.zig",
4886		dependencies_source.items,
4887	4910	);
4888	4911
4889		mem.swap(Package.Table, &main_pkg.table, &deps_pkg.table);
4890		try main_pkg.add(gpa, "@dependencies", deps_pkg);
	4912	{
	4913	// We need a Module for each package's build.zig.
	4914	const hashes = job_queue.table.keys();
	4915	const fetches = job_queue.table.values();
	4916	try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len));
	4917	for (hashes, fetches) \|hash, f\| {
	4918	if (f == &fetch) {
	4919	// The first one is a dummy package for the current project.
	4920	continue;
	4921	}
	4922	if (!f.has_build_zig)
	4923	continue;
	4924	const m = try Package.Module.create(arena, .{
	4925	.root = try f.package_root.clone(arena),
	4926	.root_src_path = Package.build_zig_basename,
	4927	.fully_qualified_name = try std.fmt.allocPrint(
	4928	arena,
	4929	"root.@dependencies.{s}",
	4930	.{&hash},
	4931	),
	4932	});
	4933	const hash_cloned = try arena.dupe(u8, &hash);
	4934	deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m);
	4935	f.module = m;
	4936	}
	4937
	4938	// Each build.zig module needs access to each of its
	4939	// dependencies' build.zig modules by name.
	4940	for (fetches) \|f\| {
	4941	const mod = f.module orelse continue;
	4942	const man = f.manifest orelse continue;
	4943	const dep_names = man.dependencies.keys();
	4944	try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len));
	4945	for (dep_names, man.dependencies.values()) \|name, dep\| {
	4946	const dep_digest = Package.Fetch.depDigest(
	4947	f.package_root,
	4948	global_cache_directory,
	4949	dep,
	4950	) orelse continue;
	4951	const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue;
	4952	const name_cloned = try arena.dupe(u8, name);
	4953	mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod);
	4954	}
	4955	}
	4956	}
4891	4957	}
4892		try main_pkg.add(gpa, "@build", &build_pkg);
	4958
	4959	try main_mod.deps.put(arena, "@build", &build_mod);
4893	4960
4894	4961	const comp = Compilation.create(gpa, .{
4895	4962	.zig_lib_directory = zig_lib_directory,
...	...	@@ -4901,7 +4968,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4901	4968	.is_native_abi = cross_target.isNativeAbi(),
4902	4969	.dynamic_linker = target_info.dynamic_linker.get(),
4903	4970	.output_mode = .Exe,
4904		.main_pkg = &main_pkg,
	4971	.main_mod = &main_mod,
4905	4972	.emit_bin = emit_bin,
4906	4973	.emit_h = null,
4907	4974	.optimize_mode = .Debug,
...	...	@@ -5115,12 +5182,15 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void
5115	5182	.tree = tree,
5116	5183	.tree_loaded = true,
5117	5184	.zir = undefined,
5118		.pkg = undefined,
	5185	.mod = undefined,
5119	5186	.root_decl = .none,
5120	5187	};
5121	5188
5122		file.pkg = try Package.create(gpa, null, file.sub_file_path);
5123		defer file.pkg.destroy(gpa);
	5189	file.mod = try Package.Module.create(arena, .{
	5190	.root = Package.Path.cwd(),
	5191	.root_src_path = file.sub_file_path,
	5192	.fully_qualified_name = "root",
	5193	});
5124	5194
5125	5195	file.zir = try AstGen.generate(gpa, file.tree);
5126	5196	file.zir_loaded = true;
...	...	@@ -5321,12 +5391,15 @@ fn fmtPathFile(
5321	5391	.tree = tree,
5322	5392	.tree_loaded = true,
5323	5393	.zir = undefined,
5324		.pkg = undefined,
	5394	.mod = undefined,
5325	5395	.root_decl = .none,
5326	5396	};
5327	5397
5328		file.pkg = try Package.create(gpa, null, file.sub_file_path);
5329		defer file.pkg.destroy(gpa);
	5398	file.mod = try Package.Module.create(fmt.arena, .{
	5399	.root = Package.Path.cwd(),
	5400	.root_src_path = file.sub_file_path,
	5401	.fully_qualified_name = "root",
	5402	});
5330	5403
5331	5404	if (stat.size > max_src_size)
5332	5405	return error.FileTooBig;
...	...	@@ -5387,7 +5460,7 @@ pub fn putAstErrorsIntoBundle(
5387	5460	tree: Ast,
5388	5461	path: []const u8,
5389	5462	wip_errors: *std.zig.ErrorBundle.Wip,
5390		) !void {
	5463	) Allocator.Error!void {
5391	5464	var file: Module.File = .{
5392	5465	.status = .never_loaded,
5393	5466	.source_loaded = true,
...	...	@@ -5402,12 +5475,16 @@ pub fn putAstErrorsIntoBundle(
5402	5475	.tree = tree,
5403	5476	.tree_loaded = true,
5404	5477	.zir = undefined,
5405		.pkg = undefined,
	5478	.mod = undefined,
5406	5479	.root_decl = .none,
5407	5480	};
5408	5481
5409		file.pkg = try Package.create(gpa, null, path);
5410		defer file.pkg.destroy(gpa);
	5482	file.mod = try Package.Module.create(gpa, .{
	5483	.root = Package.Path.cwd(),
	5484	.root_src_path = file.sub_file_path,
	5485	.fully_qualified_name = "root",
	5486	});
	5487	defer gpa.destroy(file.mod);
5411	5488
5412	5489	file.zir = try AstGen.generate(gpa, file.tree);
5413	5490	file.zir_loaded = true;
...	...	@@ -5933,7 +6010,7 @@ pub fn cmdAstCheck(
5933	6010	.stat = undefined,
5934	6011	.tree = undefined,
5935	6012	.zir = undefined,
5936		.pkg = undefined,
	6013	.mod = undefined,
5937	6014	.root_decl = .none,
5938	6015	};
5939	6016	if (zig_source_file) \|file_name\| {
...	...	@@ -5971,8 +6048,11 @@ pub fn cmdAstCheck(
5971	6048	file.stat.size = source.len;
5972	6049	}
5973	6050
5974		file.pkg = try Package.create(gpa, null, file.sub_file_path);
5975		defer file.pkg.destroy(gpa);
	6051	file.mod = try Package.Module.create(arena, .{
	6052	.root = Package.Path.cwd(),
	6053	.root_src_path = file.sub_file_path,
	6054	.fully_qualified_name = "root",
	6055	});
5976	6056
5977	6057	file.tree = try Ast.parse(gpa, file.source, .zig);
5978	6058	file.tree_loaded = true;
...	...	@@ -6067,7 +6147,7 @@ pub fn cmdDumpZir(
6067	6147	.stat = undefined,
6068	6148	.tree = undefined,
6069	6149	.zir = try Module.loadZirCache(gpa, f),
6070		.pkg = undefined,
	6150	.mod = undefined,
6071	6151	.root_decl = .none,
6072	6152	};
6073	6153
...	...	@@ -6136,12 +6216,15 @@ pub fn cmdChangelist(
6136	6216	},
6137	6217	.tree = undefined,
6138	6218	.zir = undefined,
6139		.pkg = undefined,
	6219	.mod = undefined,
6140	6220	.root_decl = .none,
6141	6221	};
6142	6222
6143		file.pkg = try Package.create(gpa, null, file.sub_file_path);
6144		defer file.pkg.destroy(gpa);
	6223	file.mod = try Package.Module.create(arena, .{
	6224	.root = Package.Path.cwd(),
	6225	.root_src_path = file.sub_file_path,
	6226	.fully_qualified_name = "root",
	6227	});
6145	6228
6146	6229	const source = try arena.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0);
6147	6230	const amt = try f.readAll(source);
...	...	@@ -6623,7 +6706,9 @@ fn cmdFetch(
6623	6706	args: []const []const u8,
6624	6707	) !void {
6625	6708	const color: Color = .auto;
6626		var opt_url: ?[]const u8 = null;
	6709	const work_around_btrfs_bug = builtin.os.tag == .linux and
	6710	std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND");
	6711	var opt_path_or_url: ?[]const u8 = null;
6627	6712	var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");
6628	6713
6629	6714	{
...	...	@@ -6643,15 +6728,15 @@ fn cmdFetch(
6643	6728	} else {
6644	6729	fatal("unrecognized parameter: '{s}'", .{arg});
6645	6730	}
6646		} else if (opt_url != null) {
	6731	} else if (opt_path_or_url != null) {
6647	6732	fatal("unexpected extra parameter: '{s}'", .{arg});
6648	6733	} else {
6649		opt_url = arg;
	6734	opt_path_or_url = arg;
6650	6735	}
6651	6736	}
6652	6737	}
6653	6738
6654		const url = opt_url orelse fatal("missing url or path parameter", .{});
	6739	const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{});
6655	6740
6656	6741	var thread_pool: ThreadPool = undefined;
6657	6742	try thread_pool.init(.{ .allocator = gpa });
...	...	@@ -6664,19 +6749,6 @@ fn cmdFetch(
6664	6749	const root_prog_node = progress.start("Fetch", 0);
6665	6750	defer root_prog_node.end();
6666	6751
6667		var wip_errors: std.zig.ErrorBundle.Wip = undefined;
6668		try wip_errors.init(gpa);
6669		defer wip_errors.deinit();
6670
6671		var report: Package.Report = .{
6672		.ast = null,
6673		.directory = .{
6674		.handle = fs.cwd(),
6675		.path = null,
6676		},
6677		.error_bundle = &wip_errors,
6678		};
6679
6680	6752	var global_cache_directory: Compilation.Directory = l: {
6681	6753	const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena);
6682	6754	break :l .{
...	...	@@ -6686,56 +6758,51 @@ fn cmdFetch(
6686	6758	};
6687	6759	defer global_cache_directory.handle.close();
6688	6760
6689		var readable_resource: Package.ReadableResource = rr: {
6690		if (fs.cwd().openIterableDir(url, .{})) \|dir\| {
6691		break :rr .{
6692		.path = try gpa.dupe(u8, url),
6693		.resource = .{ .dir = dir },
6694		};
6695		} else \|dir_err\| {
6696		const file_err = if (dir_err == error.NotDir) e: {
6697		if (fs.cwd().openFile(url, .{})) \|f\| {
6698		break :rr .{
6699		.path = try gpa.dupe(u8, url),
6700		.resource = .{ .file = f },
6701		};
6702		} else \|err\| break :e err;
6703		} else dir_err;
6704
6705		const uri = std.Uri.parse(url) catch \|uri_err\| {
6706		fatal("'{s}' could not be recognized as a file path ({s}) or an URL ({s})", .{
6707		url, @errorName(file_err), @errorName(uri_err),
6708		});
6709		};
6710		const fetch_location = try Package.FetchLocation.initUri(uri, 0, report);
6711		const cwd: Cache.Directory = .{
6712		.handle = fs.cwd(),
6713		.path = null,
6714		};
6715		break :rr try fetch_location.fetch(gpa, cwd, &http_client, 0, report);
6716		}
	6761	var job_queue: Package.Fetch.JobQueue = .{
	6762	.http_client = &http_client,
	6763	.thread_pool = &thread_pool,
	6764	.global_cache = global_cache_directory,
	6765	.recursive = false,
	6766	.work_around_btrfs_bug = work_around_btrfs_bug,
6717	6767	};
6718		defer readable_resource.deinit(gpa);
	6768	defer job_queue.deinit();
	6769
	6770	var fetch: Package.Fetch = .{
	6771	.arena = std.heap.ArenaAllocator.init(gpa),
	6772	.location = .{ .path_or_url = path_or_url },
	6773	.location_tok = 0,
	6774	.hash_tok = 0,
	6775	.parent_package_root = undefined,
	6776	.parent_manifest_ast = null,
	6777	.prog_node = root_prog_node,
	6778	.job_queue = &job_queue,
	6779	.omit_missing_hash_error = true,
	6780	.allow_missing_paths_field = false,
	6781
	6782	.package_root = undefined,
	6783	.error_bundle = undefined,
	6784	.manifest = null,
	6785	.manifest_ast = undefined,
	6786	.actual_hash = undefined,
	6787	.has_build_zig = false,
	6788	.oom_flag = false,
	6789
	6790	.module = null,
	6791	};
	6792	defer fetch.deinit();
6719	6793
6720		var package_location = readable_resource.unpack(
6721		gpa,
6722		&thread_pool,
6723		global_cache_directory,
6724		0,
6725		report,
6726		root_prog_node,
6727		) catch \|err\| {
6728		if (wip_errors.root_list.items.len > 0) {
6729		var errors = try wip_errors.toOwnedBundle("");
6730		defer errors.deinit(gpa);
6731		errors.renderToStdErr(renderOptions(color));
6732		process.exit(1);
6733		}
6734		fatal("unable to unpack '{s}': {s}", .{ url, @errorName(err) });
	6794	fetch.run() catch \|err\| switch (err) {
	6795	error.OutOfMemory => fatal("out of memory", .{}),
	6796	error.FetchFailed => {}, // error bundle checked below
6735	6797	};
6736		defer package_location.deinit(gpa);
6737	6798
6738		const hex_digest = Package.Manifest.hexDigest(package_location.hash);
	6799	if (fetch.error_bundle.root_list.items.len > 0) {
	6800	var errors = try fetch.error_bundle.toOwnedBundle("");
	6801	errors.renderToStdErr(renderOptions(color));
	6802	process.exit(1);
	6803	}
	6804
	6805	const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
6739	6806
6740	6807	progress.done = true;
6741	6808	progress.refresh();
...	...	@@ -6744,3 +6811,56 @@ fn cmdFetch(
6744	6811
6745	6812	return cleanExit();
6746	6813	}
	6814
	6815	fn createEmptyDependenciesModule(
	6816	arena: Allocator,
	6817	main_mod: *Package.Module,
	6818	local_cache_directory: Cache.Directory,
	6819	) !void {
	6820	var source = std.ArrayList(u8).init(arena);
	6821	try Package.Fetch.JobQueue.createEmptyDependenciesSource(&source);
	6822	_ = try createDependenciesModule(arena, source.items, main_mod, local_cache_directory);
	6823	}
	6824
	6825	/// Creates the dependencies.zig file and corresponding `Package.Module` for the
	6826	/// build runner to obtain via `@import("@dependencies")`.
	6827	fn createDependenciesModule(
	6828	arena: Allocator,
	6829	source: []const u8,
	6830	main_mod: *Package.Module,
	6831	local_cache_directory: Cache.Directory,
	6832	) !*Package.Module {
	6833	// Atomically create the file in a directory named after the hash of its contents.
	6834	const basename = "dependencies.zig";
	6835	const rand_int = std.crypto.random.int(u64);
	6836	const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++
	6837	Package.Manifest.hex64(rand_int);
	6838	{
	6839	var tmp_dir = try local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{});
	6840	defer tmp_dir.close();
	6841	try tmp_dir.writeFile(basename, source);
	6842	}
	6843
	6844	var hh: Cache.HashHelper = .{};
	6845	hh.addBytes(build_options.version);
	6846	hh.addBytes(source);
	6847	const hex_digest = hh.final();
	6848
	6849	const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest);
	6850	try Package.Fetch.renameTmpIntoCache(
	6851	local_cache_directory.handle,
	6852	tmp_dir_sub_path,
	6853	o_dir_sub_path,
	6854	);
	6855
	6856	const deps_mod = try Package.Module.create(arena, .{
	6857	.root = .{
	6858	.root_dir = local_cache_directory,
	6859	.sub_path = o_dir_sub_path,
	6860	},
	6861	.root_src_path = basename,
	6862	.fully_qualified_name = "root.@dependencies",
	6863	});
	6864	try main_mod.deps.put(arena, "@dependencies", deps_mod);
	6865	return deps_mod;
	6866	}

src/musl.zig+1-1

...	...	@@ -206,7 +206,7 @@ pub fn buildCRTFile(comp: Compilation, crt_file: CRTFile, prog_node: std.Progr
206	206	.zig_lib_directory = comp.zig_lib_directory,
207	207	.target = target,
208	208	.root_name = "c",
209		.main_pkg = null,
	209	.main_mod = null,
210	210	.output_mode = .Lib,
211	211	.link_mode = .Dynamic,
212	212	.thread_pool = comp.thread_pool,

test/cases/compile_errors/import_of_missing_package.zig+1-1

...	...	@@ -7,4 +7,4 @@ comptime {
7	7	// backend=stage2
8	8	// target=native
9	9	//
10		// :1:21: error: no package named 'foo' available within package 'root'
	10	// :1:21: error: no module named 'foo' available within module root

test/cases/compile_errors/import_outside_package.zig+1-1

...	...	@@ -5,4 +5,4 @@ export fn a() usize {
5	5	// error
6	6	// target=native
7	7	//
8		// :2:20: error: import of file outside package path: '../../above.zig'
	8	// :2:20: error: import of file outside module path: '../../above.zig'

test/cases/compile_errors/import_outside_package_path.zig+1-1

...	...	@@ -6,4 +6,4 @@ comptime {
6	6	// backend=stage2
7	7	// target=native
8	8	//
9		// :2:17: error: import of file outside package path: '../a.zig'
	9	// :2:17: error: import of file outside module path: '../a.zig'

test/compile_errors.zig+1-1

...	...	@@ -129,7 +129,7 @@ pub fn addCases(ctx: *Cases) !void {
129	129	\\}
130	130	, &[_][]const u8{
131	131	":1:1: error: file exists in multiple modules",
132		":1:1: note: root of module root.foo",
	132	":1:1: note: root of module foo",
133	133	":3:17: note: imported from module root",
134	134	});
135	135	case.addSourceFile("foo.zig",