Commit 0c978ba957ad1d44f5a4952b2d6dce121a5b05e6 | zig

author	Andrew Kelley <andrew@ziglang.org> 2026-06-22 17:32:23-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2026-06-29 23:50:18-07:00
log	0c978ba957ad1d44f5a4952b2d6dce121a5b05e6
tree	b929895405196bb0a8f845bca43976643f1058e4
parent	38992fc017f11ec39769043ff309c25dba819760

WIP: migrate build and fetch commands to Maker process

32 files changed, 7249 insertions(+), 7648 deletions(-)

build.zig
lib
- compiler
  - Maker.zig
  - Maker
    Fetch.zig
    Fetch
    git.zig
    git
    testdata
    testrepo-sha1.idx
    testrepo-sha1.pack
    testrepo-sha256.idx
    testrepo-sha256.pack
    Package.zig
    Package
    Manifest.zig
  - configurer.zig
- std
  - Build
    Cache.zig
    Configuration.zig
  - zig.zig
src
- Compilation.zig
- Module.zig
- Package.zig
- Package
  - Fetch.zig
  - Fetch
    git.zig
    git
    testdata
    testrepo-sha1.idx
    testrepo-sha1.pack
    testrepo-sha256.idx
    testrepo-sha256.pack
  - Manifest.zig
  - Module.zig
- Zcu.zig
- Zcu
  - PerThread.zig
- dev.zig
- introspect.zig
- main.zig
- print_env.zig
- print_targets.zig

build.zig+3

...	...	@@ -175,6 +175,9 @@ pub fn build(b: *std.Build) !void {
175	175	".tar",
176	176	// exclude files from lib/std/zip/testdata
177	177	".zip",
	178	// exclude files from lib/compiler/Maker/Fetch/git/testdata
	179	".idx",
	180	".pack",
178	181	// others
179	182	"README.md",
180	183	},

lib/compiler/Maker.zig+1317-104

...	...	@@ -17,6 +17,10 @@ const log = std.log;
17	17	const mem = std.mem;
18	18	const process = std.process;
19	19	const Color = std.zig.Color;
	20	const EnvVar = std.zig.EnvVar;
	21	const default_local_zig_cache_basename = std.zig.default_local_zig_cache_basename;
	22	const allocPrint = std.fmt.allocPrint;
	23	const stringToEnum = std.meta.stringToEnum;
20	24
21	25	const Fuzz = @import("Maker/Fuzz.zig");
22	26	const Graph = @import("Maker/Graph.zig");
...	...	@@ -25,10 +29,11 @@ const Watch = @import("Maker/Watch.zig");
25	29	const WebServer = @import("Maker/WebServer.zig");
26	30	const ScannedConfig = @import("Maker/ScannedConfig.zig");
27	31	const PkgConfig = @import("Maker/PkgConfig.zig");
	32	const Fetch = @import("Maker/Fetch.zig");
	33	const Package = @import("Maker/Package.zig");
28	34
29	35	pub const std_options: std.Options = .{
30	36	.side_channels_mitigations = .none,
31		.http_disable_tls = true,
32	37	};
33	38
34	39	gpa: Allocator,
...	...	@@ -100,6 +105,15 @@ const ErrorStyle = enum {
100	105	const MultilineErrors = enum { indent, newline, none };
101	106	const Summary = enum { all, new, failures, line, none };
102	107
	108	/// Used to build the -M flags to pass to build-exe.
	109	const CliModule = struct {
	110	name: []const u8,
	111	root_path: []const u8,
	112	deps: Deps = .empty,
	113
	114	const Deps = std.array_hash_map.String(*CliModule);
	115	};
	116
103	117	pub fn main(init: process.Init.Minimal) !void {
104	118	// The build runner is long-lived in the following use cases:
105	119	// * `--watch` mode
...	...	@@ -124,70 +138,54 @@ pub fn main(init: process.Init.Minimal) !void {
124	138	const arena = arena_instance.allocator();
125	139
126	140	const args = try init.args.toSlice(arena);
127
128		// skip my own exe name
129		var arg_idx: usize = 1;
130
131		const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig");
132		const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir");
133		const build_root = expectArgOrFatal(args, &arg_idx, "--build-root");
134		const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache");
135		const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache");
136		const configure_path = expectArgOrFatal(args, &arg_idx, "--configuration");
	141	var arg_i: usize = 1;
	142	const cmd_name = nextArgOrFatal(args, &arg_i);
	143	const zig_lib_arg = prefixedArgOrFatal(args, &arg_i, "--zig-lib=");
	144	const zig_exe_arg = prefixedArgOrFatal(args, &arg_i, "--zig=");
	145	const global_cache_arg = prefixedArgOrFatal(args, &arg_i, "--global-cache=");
	146	const seed_arg = prefixedArgOrFatal(args, &arg_i, "--seed=");
137	147
138	148	const cwd: Dir = .cwd();
139	149
140	150	const zig_lib_directory: Cache.Directory = .{
141		.path = zig_lib_dir,
142		.handle = try cwd.openDir(io, zig_lib_dir, .{}),
143		};
144
145		const build_root_directory: Cache.Directory = .{
146		.path = build_root,
147		.handle = try cwd.openDir(io, build_root, .{}),
148		};
149
150		const local_cache_directory: Cache.Directory = .{
151		.path = local_cache_root,
152		.handle = try cwd.createDirPathOpen(io, local_cache_root, .{}),
	151	.path = zig_lib_arg,
	152	.handle = try cwd.openDir(io, zig_lib_arg, .{}),
153	153	};
154	154
155	155	const global_cache_directory: Cache.Directory = .{
156		.path = global_cache_root,
157		.handle = try cwd.createDirPathOpen(io, global_cache_root, .{}),
	156	.path = global_cache_arg,
	157	.handle = try cwd.createDirPathOpen(io, global_cache_arg, .{}),
158	158	};
159	159
160	160	var graph: Graph = .{
161	161	.io = io,
162	162	.arena = arena,
163		.cache = .{
164		.io = io,
165		.gpa = gpa,
166		.manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}),
167		.cwd = try process.currentPathAlloc(io, arena),
168		},
169		.zig_exe = zig_exe,
	163	.cache = undefined,
	164	.zig_exe = zig_exe_arg,
170	165	.environ_map = try init.environ.createMap(arena),
171	166	.global_cache_root = global_cache_directory,
172		.local_cache_root = local_cache_directory,
	167	.local_cache_root = undefined,
173	168	.zig_lib_directory = zig_lib_directory,
174		.build_root_directory = build_root_directory,
	169	.build_root_directory = undefined,
	170	.random_seed = parseRandomSeed(seed_arg),
175	171	};
176	172
177		graph.cache.addPrefix(.{ .path = null, .handle = cwd });
178		graph.cache.addPrefix(build_root_directory);
179		graph.cache.addPrefix(local_cache_directory);
180		graph.cache.addPrefix(global_cache_directory);
181		graph.cache.hash.addBytes(builtin.zig_version_string);
	173	const cmd = stringToEnum(enum { fetch, build }, cmd_name) orelse fatal("bad command name: {q}", .{ cmd_name });
	174	switch (cmd) {
	175	.fetch => return cmdFetch( gpa, &graph, args[arg_i..]),
	176	.build => {},
	177	}
182	178
183	179	var step_names: std.ArrayList([]const u8) = .empty;
184	180	var help_menu = false;
185	181	var steps_menu = false;
186		var print_configuration = false;
	182	var print_configuration: enum {none, zon, path} = .none;
187	183	var override_install_prefix: ?[]const u8 = null;
188	184	var override_lib_dir: ?[]const u8 = null;
189	185	var override_bin_dir: ?[]const u8 = null;
190	186	var override_include_dir: ?[]const u8 = null;
	187	var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(&graph.environ_map);
	188	var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(&graph.environ_map);
191	189	var error_style: ErrorStyle = .verbose;
192	190	var multiline_errors: MultilineErrors = .indent;
193	191	var summary: ?Summary = null;
...	...	@@ -201,39 +199,120 @@ pub fn main(init: process.Init.Minimal) !void {
201	199	var webui_listen: ?Io.net.IpAddress = null;
202	200	var debug_pkg_config = false;
203	201	var run_args: ?[]const []const u8 = null;
204
205		if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) \|str\| {
206		if (std.meta.stringToEnum(ErrorStyle, str)) \|style\| {
	202	var build_file: ?[]const u8 = null;
	203
	204	var configure_argv: std.ArrayList([]const u8) = .empty;
	205	var cached_passthru_configure: std.ArrayList(u32) = .empty;
	206	var forks: std.ArrayList(Fork) = .empty;
	207	var system_pkg_dir_path: ?[]const u8 = null;
	208	var fetch_only = false;
	209	var fetch_mode: Fetch.JobQueue.Mode = .needed;
	210	var debug_target: ?[]const u8 = null;
	211	var cache_poison: std.Build.Graph.CachePoison = .pure;
	212
	213	if (EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) \|str\| {
	214	if (stringToEnum(ErrorStyle, str)) \|style\| {
207	215	error_style = style;
208	216	}
209	217	}
210	218
211		if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) \|str\| {
212		if (std.meta.stringToEnum(MultilineErrors, str)) \|style\| {
	219	if (EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) \|str\| {
	220	if (stringToEnum(MultilineErrors, str)) \|style\| {
213	221	multiline_errors = style;
214	222	}
215	223	}
216	224
217		while (nextArg(args, &arg_idx)) \|arg\| {
	225	try configure_argv.ensureUnusedCapacity(arena, 16);
	226	try cached_passthru_configure.ensureUnusedCapacity(arena, 16);
	227
	228	_ = configure_argv.addOneAssumeCapacity(); // configurer executable
	229	configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", graph.zig_exe };
	230	configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined };
	231	const conf_argv_index_build_root = configure_argv.items.len - 1;
	232
	233	while (nextArg(args, &arg_i)) \|arg\| {
218	234	if (mem.startsWith(u8, arg, "-")) {
219		if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
	235	try configure_argv.ensureUnusedCapacity(arena, 2);
	236	if (mem.startsWith(u8, arg, "-D") or
	237	mem.startsWith(u8, arg, "-fsys=") or
	238	mem.startsWith(u8, arg, "-fno-sys=") or
	239	mem.startsWith(u8, arg, "--release=") or
	240	mem.eql(u8, arg, "--release"))
	241	{
	242	try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len));
	243	configure_argv.appendAssumeCapacity(arg);
	244	continue;
	245	} else if (mem.eql(u8, arg, "--system")) {
	246	system_pkg_dir_path = nextArgOrFatal(args, &arg_i);
	247
	248	try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len));
	249	configure_argv.appendAssumeCapacity(arg); // Intentionally "--system" only; not the path.
	250	continue;
	251	} else if (mem.cutPrefix(u8, arg, "--color=")) \|rest\| {
	252	color = stringToEnum(Color, rest) orelse
	253	fatal("expected --color=[auto\|on\|off]; found {q}", .{arg});
	254
	255	try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len));
	256	configure_argv.appendAssumeCapacity(arg);
	257	continue;
	258	} else if (mem.eql(u8, arg, "--cache-poison")) {
	259	cache_poison = .poisoned;
	260	configure_argv.appendAssumeCapacity("--cache-poison=poisoned");
	261	continue;
	262	} else if (mem.cutPrefix(u8, arg, "--cache-poison=")) \|rest\| {
	263	// Allow the configurer process to report parse failure.
	264	if (stringToEnum(std.Build.Graph.CachePoison, rest)) \|poison\| {
	265	cache_poison = poison;
	266	}
	267	configure_argv.appendAssumeCapacity(arg);
	268	continue;
	269	} else if (mem.eql(u8, arg, "--verbose")) {
	270	// Intentionally is added both to make and configure but
	271	// does not go into the cache hash.
	272	configure_argv.appendAssumeCapacity(arg);
	273	} else if (mem.eql(u8, arg, "--search-prefix")) {
	274	const prefix = nextArgOrFatal(args, &arg_i);
	275	// This argument is cache poisonous: it does not go into
	276	// the cache and configurer must set the poison bit when
	277	// choosing to observe it.
	278	configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ arg, prefix };
	279	continue;
	280	} else if (mem.eql(u8, arg, "--cache-dir")) {
	281	override_local_cache_dir = nextArgOrFatal(args, &arg_i);
	282	} else if (mem.eql(u8, arg, "--pkg-dir")) {
	283	override_pkg_dir = nextArgOrFatal(args, &arg_i);
	284	} else if (mem.eql(u8, arg, "--fetch")) {
	285	fetch_only = true;
	286	} else if (mem.cutPrefix(u8, arg, "--fetch=")) \|rest\| {
	287	fetch_only = true;
	288	fetch_mode = stringToEnum(Fetch.JobQueue.Mode, rest) orelse
	289	fatal("expected [needed\|all] after \"--fetch=\", found {q}", .{rest});
	290	} else if (mem.cutPrefix(u8, arg, "--fork=")) \|rest\| {
	291	try forks.append(arena, .init(rest));
	292	} else if (mem.eql(u8, arg, "--fork")) {
	293	try forks.append(arena, .init(nextArgOrFatal(args, &arg_i)));
	294	} else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
220	295	help_menu = true;
221	296	} else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) {
222	297	steps_menu = true;
223	298	} else if (mem.eql(u8, arg, "--print-configuration")) {
224		print_configuration = true;
	299	print_configuration = .zon;
	300	} else if (mem.eql(u8, arg, "--print-configuration-path")) {
	301	print_configuration = .path;
225	302	} else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) {
226		override_install_prefix = nextArgOrFatal(args, &arg_idx);
	303	override_install_prefix = nextArgOrFatal(args, &arg_i);
	304	} else if (mem.eql(u8, arg, "--build-file")) {
	305	build_file = nextArgOrFatal(args, &arg_i);
227	306	} else if (mem.eql(u8, arg, "--prefix-lib-dir")) {
228		override_lib_dir = nextArgOrFatal(args, &arg_idx);
	307	override_lib_dir = nextArgOrFatal(args, &arg_i);
229	308	} else if (mem.eql(u8, arg, "--prefix-exe-dir")) {
230		override_bin_dir = nextArgOrFatal(args, &arg_idx);
	309	override_bin_dir = nextArgOrFatal(args, &arg_i);
231	310	} else if (mem.eql(u8, arg, "--prefix-include-dir")) {
232		override_include_dir = nextArgOrFatal(args, &arg_idx);
	311	override_include_dir = nextArgOrFatal(args, &arg_i);
233	312	} else if (mem.eql(u8, arg, "--sysroot")) {
234		graph.sysroot = nextArgOrFatal(args, &arg_idx);
	313	graph.sysroot = nextArgOrFatal(args, &arg_i);
235	314	} else if (mem.eql(u8, arg, "--maxrss")) {
236		const max_rss_text = nextArgOrFatal(args, &arg_idx);
	315	const max_rss_text = nextArgOrFatal(args, &arg_i);
237	316	max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch \|err\|
238	317	fatal("invalid byte size {q}: {t}", .{ max_rss_text, err });
239	318	} else if (mem.eql(u8, arg, "--skip-oom-steps")) {
...	...	@@ -253,7 +332,7 @@ pub fn main(init: process.Init.Minimal) !void {
253	332	.{ "h", std.time.ns_per_hour },
254	333	.{ "hour", std.time.ns_per_hour },
255	334	};
256		const timeout_str = nextArgOrFatal(args, &arg_idx);
	335	const timeout_str = nextArgOrFatal(args, &arg_i);
257	336	const num_end_idx = std.mem.findLastNone(u8, timeout_str, "abcdefghijklmnopqrstuvwxyz") orelse fatal(
258	337	"invalid timeout {q}: expected unit (ns, us, ms, s, m, h)",
259	338	.{timeout_str},
...	...	@@ -274,50 +353,46 @@ pub fn main(init: process.Init.Minimal) !void {
274	353	);
275	354	test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed);
276	355	} else if (mem.eql(u8, arg, "--search-prefix")) {
277		try graph.search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx));
	356	try graph.search_prefixes.append(arena, nextArgOrFatal(args, &arg_i));
278	357	} else if (mem.eql(u8, arg, "--libc")) {
279		graph.libc_file = nextArgOrFatal(args, &arg_idx);
	358	graph.libc_file = nextArgOrFatal(args, &arg_i);
280	359	} else if (mem.eql(u8, arg, "--color")) {
281		const next_arg = nextArg(args, &arg_idx) orelse
	360	const next_arg = nextArg(args, &arg_i) orelse
282	361	fatalWithHint("expected [auto\|on\|off] after {q}", .{arg});
283		color = std.meta.stringToEnum(Color, next_arg) orelse {
	362	color = stringToEnum(Color, next_arg) orelse {
284	363	fatalWithHint("expected [auto\|on\|off] after {q}, found {q}", .{
285	364	arg, next_arg,
286	365	});
287	366	};
288	367	} else if (mem.eql(u8, arg, "--error-style")) {
289		const next_arg = nextArg(args, &arg_idx) orelse
	368	const next_arg = nextArg(args, &arg_i) orelse
290	369	fatalWithHint("expected style after {q}", .{arg});
291		error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse {
	370	error_style = stringToEnum(ErrorStyle, next_arg) orelse {
292	371	fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg });
293	372	};
294	373	} else if (mem.eql(u8, arg, "--multiline-errors")) {
295		const next_arg = nextArg(args, &arg_idx) orelse
	374	const next_arg = nextArg(args, &arg_i) orelse
296	375	fatalWithHint("expected style after {q}", .{arg});
297		multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse {
	376	multiline_errors = stringToEnum(MultilineErrors, next_arg) orelse {
298	377	fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg });
299	378	};
300	379	} else if (mem.eql(u8, arg, "--summary")) {
301		const next_arg = nextArg(args, &arg_idx) orelse
	380	const next_arg = nextArg(args, &arg_i) orelse
302	381	fatalWithHint("expected [all\|new\|failures\|line\|none] after {q}", .{arg});
303		summary = std.meta.stringToEnum(Summary, next_arg) orelse {
	382	summary = stringToEnum(Summary, next_arg) orelse {
304	383	fatalWithHint("expected [all\|new\|failures\|line\|none] after {q}, found {q}", .{
305	384	arg, next_arg,
306	385	});
307	386	};
308		} else if (mem.eql(u8, arg, "--seed")) {
309		const next_arg = nextArg(args, &arg_idx) orelse
310		fatalWithHint("expected u32 after {q}", .{arg});
311		graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch \|err\| {
312		fatal("unable to parse seed {q} as unsigned 32-bit integer: {t}", .{ next_arg, err });
313		};
	387	} else if (mem.cutPrefix(u8, arg, "--seed=")) \|rest\| {
	388	graph.random_seed = parseRandomSeed(rest);
314	389	} else if (mem.eql(u8, arg, "--build-id")) {
315	390	graph.build_id = .fast;
316	391	} else if (mem.cutPrefix(u8, arg, "--build-id=")) \|style\| {
317	392	graph.build_id = std.zig.BuildId.parse(style) catch \|err\|
318	393	fatal("unable to parse --build-id style {q}: {t}", .{ style, err });
319	394	} else if (mem.eql(u8, arg, "--debounce")) {
320		const next_arg = nextArg(args, &arg_idx) orelse
	395	const next_arg = nextArg(args, &arg_i) orelse
321	396	fatalWithHint("expected u16 after {q}", .{arg});
322	397	debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch \|err\| {
323	398	fatal("unable to parse debounce interval {q} as unsigned 16-bit integer: {t}", .{
...	...	@@ -333,8 +408,7 @@ pub fn main(init: process.Init.Minimal) !void {
333	408	fatal("invalid web UI address {q}: {t}", .{ addr_str, err });
334	409	};
335	410	} else if (mem.eql(u8, arg, "--debug-log")) {
336		const next_arg = nextArgOrFatal(args, &arg_idx);
337		try graph.debug_log_scopes.append(arena, next_arg);
	411	try graph.debug_log_scopes.append(arena, nextArgOrFatal(args, &arg_i));
338	412	} else if (mem.eql(u8, arg, "--debug-compile-errors")) {
339	413	graph.debug_compile_errors = true;
340	414	} else if (mem.eql(u8, arg, "--debug-incremental")) {
...	...	@@ -344,19 +418,21 @@ pub fn main(init: process.Init.Minimal) !void {
344	418	} else if (mem.eql(u8, arg, "--debug-rt")) {
345	419	graph.debug_compiler_runtime_libs = .Debug;
346	420	} else if (mem.cutPrefix(u8, arg, "--debug-rt=")) \|rest\| {
347		graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse
	421	graph.debug_compiler_runtime_libs = stringToEnum(std.builtin.OptimizeMode, rest) orelse
348	422	fatal("unrecognized optimization mode: {s}", .{rest});
349	423	} else if (is_debug_mode and mem.eql(u8, arg, "--debug-maker-leaks")) {
350	424	debug_maker_leaks = true;
351	425	} else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) {
352	426	// --glibc-runtimes was the old name of the flag; kept for compatibility for now.
353		graph.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx);
	427	graph.libc_runtimes_dir = nextArgOrFatal(args, &arg_i);
354	428	} else if (mem.eql(u8, arg, "--verbose")) {
355	429	graph.verbose = true;
356	430	} else if (mem.eql(u8, arg, "--verbose-air")) {
357	431	graph.verbose_air = true;
358	432	} else if (mem.eql(u8, arg, "--verbose-cc")) {
359	433	graph.verbose_cc = true;
	434	} else if (mem.eql(u8, arg, "--verbose-link")) {
	435	graph.verbose_link = true;
360	436	} else if (mem.eql(u8, arg, "--verbose-llvm-ir")) {
361	437	graph.verbose_llvm_ir = true;
362	438	} else if (mem.eql(u8, arg, "--watch")) {
...	...	@@ -439,7 +515,7 @@ pub fn main(init: process.Init.Minimal) !void {
439	515	} else if (mem.eql(u8, arg, "-fno-reference-trace")) {
440	516	graph.reference_trace = null;
441	517	} else if (mem.eql(u8, arg, "--error-limit")) {
442		const next_arg = nextArgOrFatal(args, &arg_idx);
	518	const next_arg = nextArgOrFatal(args, &arg_i);
443	519	graph.error_limit = std.fmt.parseUnsigned(u32, next_arg, 0) catch \|err\|
444	520	fatal("unable to parse error limit {q}: {t}", .{ next_arg, err });
445	521	} else if (mem.cutPrefix(u8, arg, "-j")) \|text\| {
...	...	@@ -449,7 +525,7 @@ pub fn main(init: process.Init.Minimal) !void {
449	525	threaded.setAsyncLimit(.limited(n));
450	526	graph.max_jobs = n;
451	527	} else if (mem.eql(u8, arg, "--")) {
452		run_args = argsRest(args, arg_idx);
	528	run_args = argsRest(args, arg_i);
453	529	break;
454	530	} else {
455	531	fatalWithHint("unrecognized argument: {s}", .{arg});
...	...	@@ -459,8 +535,40 @@ pub fn main(init: process.Init.Minimal) !void {
459	535	}
460	536	}
461	537
462		const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map);
463		const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map);
	538	const cwd_path = std.zig.getResolvedCwd(io, arena) catch \|err\|
	539	fatal("resolving current directory path failed: {t}", .{err});
	540
	541	const build_root = try findBuildRoot(arena, io, .{
	542	.cwd_path = cwd_path,
	543	.build_file = build_file,
	544	});
	545
	546	graph.build_root_directory = build_root.directory;
	547	graph.local_cache_root = if (override_local_cache_dir) \|unresolved_path\| std.zig.Directories.openUnresolved(
	548	arena,
	549	io,
	550	cwd_path,
	551	unresolved_path,
	552	.@"local_cache",
	553	) else .{
	554	.path = try Dir.path.join(arena, &.{build_root.directory.path orelse ".", default_local_zig_cache_basename}),
	555	.handle = try build_root.directory.handle.createDirPathOpen(io, default_local_zig_cache_basename, .{}),
	556	};
	557	graph.cache = .{
	558	.io = io,
	559	.gpa = gpa,
	560	.manifest_dir = try graph.local_cache_root.handle.createDirPathOpen(io, "h", .{}),
	561	.cwd = cwd_path,
	562	};
	563	graph.cache.addPrefix(.{ .path = null, .handle = cwd });
	564	graph.cache.addPrefix(graph.build_root_directory);
	565	graph.cache.addPrefix(zig_lib_directory);
	566	graph.cache.addPrefix(graph.local_cache_root);
	567	graph.cache.addPrefix(global_cache_directory);
	568	graph.cache.hash.addBytes(builtin.zig_version_string);
	569
	570	const NO_COLOR = EnvVar.NO_COLOR.isSet(&graph.environ_map);
	571	const CLICOLOR_FORCE = EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map);
464	572
465	573	graph.stderr_mode = switch (color) {
466	574	.auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE),
...	...	@@ -468,6 +576,555 @@ pub fn main(init: process.Init.Minimal) !void {
468	576	.off => .no_color,
469	577	};
470	578
	579	const main_progress_node = std.Progress.start(io, .{
	580	.disable_printing = (graph.stderr_mode.? == .no_color),
	581	});
	582	defer main_progress_node.end();
	583
	584	{
	585	// Cache lookup for configure options. If we get a match, we can skip
	586	// execution of the configure script. If not, we get the file path to pass
	587	// to the configure process.
	588	var config_man = graph.cache.obtain();
	589	defer config_man.deinit();
	590
	591	for (cached_passthru_configure.items) \|i\|
	592	config_man.hash.addBytes(configure_argv.items[i]);
	593
	594	// Prevents a `zig build` from getting a false positive cache hit following
	595	// a `zig build --cache-poison=ignored`.
	596	config_man.hash.add(cache_poison == .ignored);
	597
	598	// Normally the build runner is compiled for the host target but here is
	599	// some code to help when debugging edits to the build runner so that you
	600	// can make sure it compiles successfully on other targets.
	601	const resolved_target: Package.Module.ResolvedTarget = t: {
	602	if (debug_target) \|triple\| {
	603	const target_query = try std.Target.Query.parse(.{ .arch_os_abi = triple });
	604	config_man.hash.addBytes(triple);
	605	break :t .{
	606	.result = std.zig.resolveTargetQueryOrFatal(io, target_query),
	607	.is_native_os = false,
	608	.is_native_abi = false,
	609	.is_explicit_dynamic_linker = false,
	610	};
	611	}
	612	break :t .{
	613	.result = std.zig.resolveTargetQueryOrFatal(io, .{}),
	614	.is_native_os = true,
	615	.is_native_abi = true,
	616	.is_explicit_dynamic_linker = false,
	617	};
	618	};
	619
	620	const pkg_root: Path = if (override_pkg_dir) \|p\|
	621	.initCwd(p)
	622	else if (system_pkg_dir_path) \|p\|
	623	.initCwd(p)
	624	else
	625	.{
	626	.root_dir = build_root.directory,
	627	.sub_path = "zig-pkg",
	628	};
	629
	630	configure_argv.items[conf_argv_index_build_root] = build_root.directory.path orelse cwd_path;
	631
	632	var http_client: std.http.Client = .{ .allocator = gpa, .io = io };
	633	defer http_client.deinit();
	634
	635	var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{};
	636	var fork_set: Package.Fetch.JobQueue.ForkSet = .{};
	637
	638	{
	639	// Populate fork_set.
	640	var group: Io.Group = .init;
	641	defer group.cancel(io);
	642
	643	for (forks.items) \|*fork\|
	644	group.async(io, Fork.load, .{ io, gpa, fork, color });
	645
	646	try group.await(io);
	647
	648	for (forks.items) \|*fork\| {
	649	if (fork.failed) process.exit(1);
	650	try fork_set.put(arena, .{
	651	.path = fork.path,
	652	.manifest_ast = fork.manifest_ast,
	653	.manifest = fork.manifest,
	654	.uses = 0,
	655	}, {});
	656	}
	657	}
	658	defer Fork.deinitList(forks.items);
	659
	660	var file_system_inputs: std.ArrayList(u8) = .empty;
	661	defer file_system_inputs.deinit(gpa);
	662
	663	var build_configurer_argv: std.ArrayList(u8) = .empty;
	664	defer build_configurer_argv.deinit(gpa);
	665
	666	var dependencies_source: std.ArrayList(u8) = .empty;
	667	defer dependencies_source.deinit(gpa);
	668
	669	const configurer_root_src_path: Cache.Path = .{
	670	.root_dir = graph.zig_lib_directory,
	671	.sub_path = "lib/compiler/configurer.zig",
	672	};
	673
	674	const root_build_src_path: Cache.Path = .{
	675	.root_dir = build_root.directory,
	676	.sub_path = build_root.build_zig_basename,
	677	};
	678
	679	try build_configurer_argv.appendSlice(gpa, &.{
	680	graph.zig_exe, "build-exe", //
	681	"--cache-dir", graph.local_cache_root.path orelse ".", //
	682	"--global-cache-dir", graph.global_cache_root.path orelse ".", //
	683	"--zig-lib-dir", graph.zig_lib_directory.path orelse ".", //
	684	"--name", "configurer", //
	685	"-fsingle-threaded", //
	686	});
	687	if (graph.libc_file) \|libc_file\| {
	688	try build_configurer_argv.appendSlice(gpa, &.{ "--libc", libc_file});
	689	}
	690	if (graph.reference_trace) \|n\| {
	691	try build_configurer_argv.append(gpa, try allocPrint(arena, "-freference-trace={d}", .{n}));
	692	}
	693	if (graph.debug_compile_errors) {
	694	try build_configurer_argv.append(gpa, "--debug-compile-errors");
	695	}
	696	try build_configurer_argv.appendSlice(gpa, &.{
	697	"--dep", "@build", //
	698	"--dep", "@dependencies", //
	699	try allocPrint(arena, "-Mroot={f}", .{configurer_root_src_path}), //
	700	try allocPrint(arena, "-M@build={f}", .{root_build_src_path}), //
	701	});
	702
	703	// In the loop below, after doing the fetch operation, the argv will be
	704	// truncated at this point, dependencies added, and then the
	705	// "--listen=-" arg appended at the end.
	706	const argv_deps_index = build_configurer_argv.items.len - 1;
	707
	708	//const root_mod = try arena.create(CliModule);
	709	//root_mod.* = .{
	710	// .name = "root",
	711	// .root_path = try configurer_root_src_path.toString(arena),
	712	//};
	713
	714	const build_mod = try arena.create(CliModule);
	715	build_mod.* = .{
	716	.name = "@build",
	717	.root_path = try root_build_src_path.toString(arena),
	718	};
	719	defer build_mod.deps.deinit(gpa);
	720
	721	const deps_mod = try arena.create(CliModule);
	722	deps_mod.* = .{
	723	.name = "@dependencies",
	724	.root_path = undefined,
	725	};
	726	defer deps_mod.deps.deinit(gpa);
	727
	728	// This loop is re-evaluated when the build script exits with an indication that it
	729	// could not continue due to missing lazy dependencies.
	730	const configuration_path: Path, const poisoned: bool = cp: while (true) {
	731	//root_mod.deps.clearRetainingCapacity();
	732	build_mod.deps.clearRetainingCapacity();
	733	deps_mod.deps.clearRetainingCapacity();
	734
	735	// We want to release all the locks before executing the child process, so we make a nice
	736	// big block here to ensure the cleanup gets run when we extract out our argv.
	737	{
	738
	739
	740
	741	{
	742	const fetch_prog_node = main_progress_node.start("Fetch Packages", 0);
	743	defer fetch_prog_node.end();
	744
	745	// Reset fork match counts.
	746	for (fork_set.keys()) \|*fork\| fork.uses = 0;
	747
	748	var job_queue: Package.Fetch.JobQueue = .{
	749	.io = io,
	750	.http_client = &http_client,
	751	.global_cache = graph.global_cache_root,
	752	.local_storage = &.{
	753	.cache_root = .{ .root_dir = graph.local_cache_root },
	754	.pkg_root = pkg_root,
	755	},
	756	.recursive = true,
	757	.debug_hash = false,
	758	.unlazy_set = unlazy_set,
	759	.fork_set = fork_set,
	760	.mode = fetch_mode,
	761	.prog_node = fetch_prog_node,
	762	.read_only = system_pkg_dir_path != null,
	763	};
	764	defer job_queue.deinit();
	765
	766	if (system_pkg_dir_path == null) {
	767	try http_client.initDefaultProxies(arena, &graph.environ_map);
	768	}
	769
	770	try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1);
	771	try job_queue.table.ensureUnusedCapacity(gpa, 1);
	772
	773	const phantom_package_root: Cache.Path = .{ .root_dir = build_root.directory };
	774
	775	var fetch: Package.Fetch = .{
	776	.arena = std.heap.ArenaAllocator.init(gpa),
	777	.location = .{ .relative_path = phantom_package_root },
	778	.location_tok = 0,
	779	.hash_tok = .none,
	780	.name_tok = 0,
	781	.lazy_status = .eager,
	782	.remote_package_root = phantom_package_root,
	783	.parent_package_root = phantom_package_root,
	784	.parent_manifest_ast = null,
	785	.prog_node = fetch_prog_node,
	786	.job_queue = &job_queue,
	787	.omit_missing_hash_error = true,
	788	.allow_missing_paths_field = false,
	789	.use_latest_commit = false,
	790
	791	.package_root = undefined,
	792	.error_bundle = undefined,
	793	.manifest = undefined,
	794	.manifest_ast = undefined,
	795	.have_manifest = false,
	796	.computed_hash = undefined,
	797	.has_build_zig = true,
	798	.oom_flag = false,
	799	.latest_commit = null,
	800
	801	.cli_module = build_mod,
	802	};
	803
	804	job_queue.all_fetches.appendAssumeCapacity(&fetch);
	805
	806	job_queue.table.putAssumeCapacityNoClobber(
	807	Package.Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root),
	808	&fetch,
	809	);
	810
	811	job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" });
	812	try job_queue.group.await(io);
	813
	814	{
	815	// Ensure that forks were actually used. This is done
	816	// before printing manifest errors because using a fork can
	817	// prevent them.
	818	var any_unused = false;
	819	for (fork_set.keys()) \|*fork\| {
	820	if (fork.uses == 0) {
	821	std.log.err("fork {f} matched no {s} packages", .{
	822	fork.path, fork.manifest.name,
	823	});
	824	any_unused = true;
	825	} else {
	826	std.log.info("fork {f} matched {d} {s} packages", .{
	827	fork.path, fork.uses, fork.manifest.name,
	828	});
	829	}
	830	}
	831	if (any_unused) process.exit(1);
	832	}
	833
	834	try job_queue.consolidateErrors();
	835
	836	if (fetch.error_bundle.root_list.items.len > 0) {
	837	var errors = try fetch.error_bundle.toOwnedBundle("");
	838	// TODO when watching, watch and rebuild configure script rather than exit here
	839	errors.renderToStderr(io, .{}, color) catch {};
	840	process.exit(1);
	841	}
	842
	843	if (fetch_only) return cleanExit(io);
	844
	845	// Create the dependencies.zig file for configurer to
	846	// obtain via `@import("@dependencies")`.
	847	{
	848	{
	849	dependencies_source.clearRetainingCapacity();
	850	var source_writer: Io.Writer.Allocating = .fromArrayList(&dependencies_source);
	851	defer dependencies_source = source_writer.toArrayList();
	852	job_queue.createDependenciesSource(&dependencies_source) catch \|err\| switch (err) {
	853	error.WriteFailed => return error.OutOfMemory,
	854	};
	855	}
	856	// Atomically create the file in a directory named after the hash of its contents.
	857	var hh: Cache.HashHelper = .{};
	858	hh.addBytes(builtin.zig_version_string);
	859	hh.addBytes(dependencies_source.items);
	860	const hex_digest = hh.final();
	861	const dependencies_zig_path: Path = .{
	862	.root_dir = graph.local_cache_root,
	863	.sub_path = try allocPrint(arena, "o/{s}/dependencies.zig", .{ &hex_digest }),
	864	};
	865	var atomic_file = try dependencies_zig_path.root_dir.handle.createFileAtomic(
	866	io,
	867	dependencies_zig_path.sub_path, .{ .make_path = true, .replace = true },
	868	);
	869	defer atomic_file.deinit(io);
	870	atomic_file.file.writeStreamingAll(io, dependencies_source.items) catch \|err\|
	871	fatal("writing dependencies.zig contents: {t}", .{err});
	872	atomic_file.replace(io) catch \|err\|
	873	fatal("replacing {f}: {t}", .{dependencies_zig_path, err});
	874
	875	deps_mod.root_path = try dependencies_zig_path.toString(arena);
	876	}
	877
	878	{
	879	// Add a CliModule for each package's build.zig.
	880	const hashes = job_queue.table.keys();
	881	const fetches = job_queue.table.values();
	882	try deps_mod.deps.ensureUnusedCapacity(gpa, @intCast(hashes.len));
	883	for (hashes, fetches) \|*hash, f\| {
	884	if (f == &fetch) {
	885	// The first one is a dummy package for the current project.
	886	continue;
	887	}
	888	if (!f.has_build_zig)
	889	continue;
	890	const hash_slice = try arena.dupe(u8, hash.toSlice());
	891
	892	const m = try arena.create(CliModule);
	893	m.* = .{
	894	.root_path = try f.package_root.toString(arena),
	895	.name = hash_slice,
	896	};
	897	deps_mod.deps.putAssumeCapacityNoClobber(hash_slice, m);
	898	f.cli_module = m;
	899	}
	900
	901	// Each build.zig module needs access to each of its
	902	// dependencies' build.zig modules by name.
	903	for (fetches) \|f\| {
	904	const mod = f.cli_module orelse continue;
	905	if (!f.have_manifest) continue;
	906	const man = &f.manifest;
	907	const dep_names = man.dependencies.keys();
	908	try mod.deps.ensureUnusedCapacity(gpa, @intCast(dep_names.len));
	909	for (dep_names, man.dependencies.values()) \|name, dep\| {
	910	const dep_digest = Package.Fetch.depDigest(
	911	f.package_root,
	912	global_cache_directory,
	913	dep,
	914	) orelse continue;
	915	const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue;
	916	const name_cloned = try arena.dupe(u8, name);
	917	mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod);
	918	}
	919	}
	920	}
	921
	922	// Lower module dependencies to CLI argv.
	923	build_configurer_argv.shrinkRetainingCapacity(argv_deps_index);
	924	for (deps_mod.deps.values()) \|dep\| {
	925	try build_configurer_argv.ensureUnusedCapacity(gpa, 2 * dep.deps.count() + 1);
	926	for (dep.deps.values()) \|sub\| {
	927	build_configurer_argv.appendAssumeCapacity("--dep");
	928	build_configurer_argv.appendAssumeCapacity(sub.name);
	929	}
	930	build_configurer_argv.appendAssumeCapacity(try allocPrint(arena, "-M{s}={s}", .{
	931	dep.name, dep.root_path,
	932	}));
	933	}
	934	try build_configurer_argv.ensureUnusedCapacity(gpa, 2 * deps_mod.deps.count() + 1);
	935	for (deps_mod.deps.values()) \|dep\| {
	936	build_configurer_argv.appendAssumeCapacity("--dep");
	937	build_configurer_argv.appendAssumeCapacity(dep.name);
	938	}
	939	build_configurer_argv.appendAssumeCapacity(try allocPrint(arena, "-M@dependencies={s}", .{
	940	deps_mod.root_path,
	941	}));
	942	}
	943
	944	const compile_prog_node = main_progress_node.start("Compile Configure Script", 0);
	945	defer compile_prog_node.end();
	946
	947	try build_configurer_argv.append(gpa, "--listen=-");
	948
	949	file_system_inputs.clearRetainingCapacity();
	950	execute_child(build_configurer_argv, &file_system_inputs);
	951
	952	const hex_digest: []const u8 = &Cache.binToHex(comp.digest.?);
	953	const exe_path: Path = .{
	954	.root_dir = dirs.local_cache,
	955	.sub_path = try allocPrint(arena, "o/{s}/{s}", .{ hex_digest, comp.emit_bin.? }),
	956	};
	957	_ = try config_man.addFilePath(exe_path, null);
	958	configure_argv.items[0] = try exe_path.toString(arena);
	959
	960	switch (cache_poison) {
	961	.pure, .disallowed, .ignored => if (try config_man.hit()) {
	962	const digest = config_man.final();
	963	break :cp .{
	964	.{
	965	.root_dir = dirs.local_cache,
	966	.sub_path = try allocPrint(arena, "c/{s}", .{&digest}),
	967	},
	968	false,
	969	};
	970	},
	971	.poisoned => {}, // Don't bother checking for cache hit.
	972	}
	973	}
	974
	975	if (!process.can_spawn) {
	976	const cmd = try std.mem.join(arena, " ", configure_argv.items);
	977	fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd });
	978	}
	979
	980	const rand_int = randInt(io, u64);
	981	const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int);
	982	const config_tmp_path: Path = .{
	983	.root_dir = dirs.local_cache,
	984	.sub_path = tmp_dir_sub_path,
	985	};
	986	const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile(
	987	io,
	988	config_tmp_path.sub_path,
	989	.{ .read = true, .exclusive = true },
	990	);
	991	defer config_tmp_file.close(io);
	992
	993	const term = term: {
	994	const child_node = main_progress_node.start("Run Configure Script", 0);
	995	defer child_node.end();
	996	var child = std.process.spawn(io, .{
	997	.argv = configure_argv.items,
	998	.stdout = .{ .file = config_tmp_file },
	999	.progress_node = child_node,
	1000	}) catch \|err\| fatal("failed to spawn configure script {q}: {t}", .{ configure_argv.items[0], err });
	1001	defer child.kill(io);
	1002	break :term child.wait(io) catch \|err\|
	1003	fatal("failed to wait configure script {q}: {t}", .{ configure_argv.items[0], err });
	1004	};
	1005	if (!term.success()) {
	1006	// Failure to produce the configuration file.
	1007	const cmd = try std.mem.join(arena, " ", configure_argv.items);
	1008	fatal("the following configure command {f}:\n{s}", .{ term, cmd });
	1009	}
	1010	// Even though the file is designed to be sent directly to make
	1011	// runner, we must load it now because:
	1012	// * If it contains additional file dependencies, we need to
	1013	// add them to `config_man` before obtaining the final digest.
	1014	// * If it contains a set of lazy packages that need to be
	1015	// fetched, we need to fetch those now and re-run configure.
	1016	var configuration = std.Build.Configuration.loadFile(arena, io, config_tmp_file) catch \|err\|
	1017	fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err });
	1018
	1019	if (configuration.unlazy_deps.len != 0) {
	1020	if (!dev.env.supports(.fetch_command)) process.exit(1);
	1021	var any_errors = false;
	1022	for (configuration.unlazy_deps) \|hash_string\| {
	1023	const hash = hash_string.slice(&configuration);
	1024	assert(hash.len != 0);
	1025	if (hash.len > Package.Hash.max_len) {
	1026	std.log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash });
	1027	any_errors = true;
	1028	continue;
	1029	}
	1030	try unlazy_set.put(arena, .fromSlice(hash), {});
	1031	}
	1032	if (any_errors) process.exit(1);
	1033	if (system_pkg_dir_path) \|p\| {
	1034	// In this mode, the system needs to provide these packages; they
	1035	// cannot be fetched by Zig.
	1036	const s = fs.path.sep_str;
	1037	for (unlazy_set.keys()) \|*hash\| {
	1038	std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() });
	1039	}
	1040	std.log.info("remote package fetching disabled due to --system mode", .{});
	1041	std.log.info("dependencies might be avoidable depending on build configuration", .{});
	1042	process.exit(1);
	1043	}
	1044	continue :cp;
	1045	}
	1046
	1047	for (configuration.path_deps_base, configuration.path_deps_sub) \|base, sub\| {
	1048	const conf_path: std.Build.Configuration.Path = .{ .base = base, .sub = sub };
	1049	try config_man.addPathPost(conf_path.toCachePath(&configuration, arena));
	1050	}
	1051
	1052	// We need to add to the configuration cache the source files of
	1053	// configurer itself, so that the maker process can watch the file system
	1054	// for those changes and restart itself. By doing this, we make it
	1055	// possible to bypass creating a Compilation for configurer on
	1056	// Configuration cache hit.
	1057	{
	1058	var it = mem.splitScalar(u8, file_system_inputs.items, 0);
	1059	while (it.next()) \|input\| {
	1060	_ = try config_man.addPrefixedPathPost(.{
	1061	.prefix = input[0],
	1062	.sub_path = input[1..],
	1063	});
	1064	}
	1065	}
	1066
	1067	// If it is poisoned, there is no point in moving it to cached
	1068	// location. Just leave it in the tmp directory.
	1069	if (configuration.poisoned) {
	1070	break :cp .{ config_tmp_path, true };
	1071	} else {
	1072	const digest = config_man.final();
	1073	const final_path: Path = .{
	1074	.root_dir = dirs.local_cache,
	1075	.sub_path = try allocPrint(arena, "c/{s}", .{&digest}),
	1076	};
	1077	Io.Dir.rename(
	1078	config_tmp_path.root_dir.handle,
	1079	config_tmp_path.sub_path,
	1080	final_path.root_dir.handle,
	1081	final_path.sub_path,
	1082	io,
	1083	) catch \|err\| retry: {
	1084	const e = switch (err) {
	1085	error.FileNotFound => e: {
	1086	const dir_path = final_path.dirname().?;
	1087	dir_path.root_dir.handle.createDirPath(io, dir_path.sub_path) catch \|e\|
	1088	fatal("failed to create directory {f}: {t}", .{ dir_path, e });
	1089	if (Io.Dir.rename(
	1090	config_tmp_path.root_dir.handle,
	1091	config_tmp_path.sub_path,
	1092	final_path.root_dir.handle,
	1093	final_path.sub_path,
	1094	io,
	1095	)) \|_\| break :retry else \|e\| break :e e;
	1096	},
	1097	else => \|e\| e,
	1098	};
	1099	fatal("failed to rename configuration file from {f} into {f}: {t}", .{
	1100	config_tmp_path, final_path, e,
	1101	});
	1102	};
	1103	config_man.writeManifest() catch \|err\| warn("failed to write cache manifest: {t}", .{err});
	1104	break :cp .{ final_path, false };
	1105	}
	1106	};
	1107
	1108	{
	1109	// Release all file system locks just before running the maker process.
	1110	var configuration_lock = if (!poisoned) config_man.toOwnedLock() else null;
	1111	defer if (configuration_lock) \|*l\| l.release(io);
	1112
	1113	if (print_configuration_path) {
	1114	var stdout_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer);
	1115	stdout_writer.interface.print("{f}\n", .{configuration_path}) catch
	1116	fatal("failed printing cache file path: {t}", .{stdout_writer.err.?});
	1117	stdout_writer.flush() catch \|err\|
	1118	fatal("failed printing cache file path: {t}", .{err});
	1119	return cleanExit(io);
	1120	}
	1121	const make_runner = make_runner_task.await(io) catch \|err\| fatal("failed compiling maker: {t}", .{err});
	1122
	1123	make_argv.items[0] = try make_runner.exe_path.toString(arena);
	1124	make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena);
	1125	}
	1126	}
	1127
471	1128	const scanned_config: ScannedConfig = sc: {
472	1129	const configuration = c: {
473	1130	var file = cwd.openFile(io, configure_path, .{}) catch \|err\|
...	...	@@ -505,7 +1162,7 @@ pub fn main(init: process.Init.Minimal) !void {
505	1162	};
506	1163
507	1164	if (help_menu) {
508		var w = initStdoutWriter(io);
	1165	const w = initStdoutWriter(io);
509	1166	scanned_config.printUsage(&graph, w) catch \|err\| switch (err) {
510	1167	error.WriteFailed => return stdout_writer_allocation.err.?,
511	1168	else => \|e\| return e,
...	...	@@ -513,18 +1170,24 @@ pub fn main(init: process.Init.Minimal) !void {
513	1170	w.flush() catch return stdout_writer_allocation.err.?;
514	1171	return cleanExit(io, &scanned_config);
515	1172	} else if (steps_menu) {
516		var w = initStdoutWriter(io);
	1173	const w = initStdoutWriter(io);
517	1174	scanned_config.printSteps(&graph, w) catch \|err\| switch (err) {
518	1175	error.WriteFailed => return stdout_writer_allocation.err.?,
519	1176	else => \|e\| return e,
520	1177	};
521	1178	w.flush() catch return stdout_writer_allocation.err.?;
522	1179	return cleanExit(io, &scanned_config);
523		} else if (print_configuration) {
524		var w = initStdoutWriter(io);
525		scanned_config.print(w) catch return stdout_writer_allocation.err.?;
526		w.flush() catch return stdout_writer_allocation.err.?;
527		return cleanExit(io, &scanned_config);
	1180	} else switch (print_configuration) {
	1181	.none => {},
	1182	.zon => {
	1183	const w = initStdoutWriter(io);
	1184	scanned_config.print(w) catch return stdout_writer_allocation.err.?;
	1185	w.flush() catch return stdout_writer_allocation.err.?;
	1186	return cleanExit(io, &scanned_config);
	1187	},
	1188	.path => {
	1189	@panic("TODO");
	1190	},
528	1191	}
529	1192
530	1193	if (webui_listen != null) {
...	...	@@ -532,11 +1195,6 @@ pub fn main(init: process.Init.Minimal) !void {
532	1195	if (builtin.single_threaded) fatal("'--webui' is not yet supported on single-threaded hosts", .{});
533	1196	}
534	1197
535		const main_progress_node = std.Progress.start(io, .{
536		.disable_printing = (graph.stderr_mode.? == .no_color),
537		});
538		defer main_progress_node.end();
539
540	1198	const install_prefix_path: Path = if (graph.environ_map.get("DESTDIR")) \|dest_dir\| .{
541	1199	.root_dir = .cwd(),
542	1200	.sub_path = try Dir.path.join(arena, &.{ dest_dir, override_install_prefix orelse "/usr" }),
...	...	@@ -706,6 +1364,338 @@ pub fn main(init: process.Init.Minimal) !void {
706	1364	}
707	1365	}
708	1366
	1367	fn cmdFetch(
	1368	gpa: Allocator,
	1369	graph: *Graph,
	1370	args: []const []const u8
	1371	) !void {
	1372	const environ_map = &graph.environ_map;
	1373	const io = graph.io;
	1374	const arena = graph.arena;
	1375
	1376	const color: Color = Color.settingFromEnvironment(environ_map);
	1377	var opt_path_or_url: ?[]const u8 = null;
	1378	var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
	1379	var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map);
	1380	var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map);
	1381	var debug_hash: bool = false;
	1382	var save: union(enum) {
	1383	no,
	1384	yes: ?[]const u8,
	1385	exact: ?[]const u8,
	1386	} = .no;
	1387
	1388	var arg_i: usize = 0;
	1389	while (nextArg(args, &arg_i)) \|arg\| {
	1390	if (mem.startsWith(u8, arg, "-")) {
	1391	if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
	1392	try Io.File.stdout().writeStreamingAll(io, usage_fetch);
	1393	return cleanExit(io);
	1394	} else if (mem.eql(u8, arg, "--global-cache-dir")) {
	1395	override_global_cache_dir = nextArgOrFatal(args, &arg_i);
	1396	} else if (mem.eql(u8, arg, "--cache-dir")) {
	1397	override_local_cache_dir = nextArgOrFatal(args, &arg_i);
	1398	} else if (mem.eql(u8, arg, "--pkg-dir")) {
	1399	override_pkg_dir = nextArgOrFatal(args, &arg_i);
	1400	} else if (mem.eql(u8, arg, "--debug-hash")) {
	1401	debug_hash = true;
	1402	} else if (mem.eql(u8, arg, "--debug-log")) {
	1403	try graph.debug_log_scopes.append(arena, nextArgOrFatal(args, &arg_i));
	1404	} else if (mem.eql(u8, arg, "--save")) {
	1405	save = .{ .yes = null };
	1406	} else if (mem.cutPrefix(u8, arg, "--save=")) \|rest\| {
	1407	save = .{ .yes = rest };
	1408	} else if (mem.eql(u8, arg, "--save-exact")) {
	1409	save = .{ .exact = null };
	1410	} else if (mem.cutPrefix(u8, arg, "--save-exact=")) \|rest\| {
	1411	save = .{ .exact = rest };
	1412	} else {
	1413	fatal("unrecognized parameter: {q}", .{arg});
	1414	}
	1415	} else if (opt_path_or_url != null) {
	1416	fatal("unexpected extra parameter: {q}", .{arg});
	1417	} else {
	1418	opt_path_or_url = arg;
	1419	}
	1420	}
	1421
	1422	const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{});
	1423
	1424	var http_client: std.http.Client = .{ .allocator = gpa, .io = io };
	1425	defer http_client.deinit();
	1426
	1427	try http_client.initDefaultProxies(arena, environ_map);
	1428
	1429	var root_prog_node = std.Progress.start(io, .{
	1430	.root_name = "Fetch",
	1431	});
	1432	defer root_prog_node.end();
	1433
	1434	var local_storage: Fetch.LocalStorage = undefined;
	1435	var build_root: BuildRoot = undefined;
	1436	var build_root_initialized = false;
	1437	defer if (build_root_initialized) build_root.deinit(io);
	1438
	1439	const cwd_path = try std.zig.getResolvedCwd(io, arena);
	1440
	1441	const local_storage_ptr = switch (save) {
	1442	.no => null,
	1443	.yes, .exact => ls: {
	1444	build_root = try findBuildRoot(arena, io, .{ .cwd_path = cwd_path });
	1445	build_root_initialized = true;
	1446
	1447	local_storage = .{
	1448	.cache_root = if (override_local_cache_dir) \|p\| .initCwd(p) else .{
	1449	.root_dir = build_root.directory,
	1450	.sub_path = ".zig-cache",
	1451	},
	1452	.pkg_root = if (override_pkg_dir) \|p\| .initCwd(p) else .{
	1453	.root_dir = build_root.directory,
	1454	.sub_path = "zig-pkg",
	1455	},
	1456	};
	1457
	1458	break :ls &local_storage;
	1459	},
	1460	};
	1461
	1462	var job_queue: Fetch.JobQueue = .{
	1463	.io = io,
	1464	.http_client = &http_client,
	1465	.global_cache = graph.global_cache_root,
	1466	.local_storage = local_storage_ptr,
	1467	.recursive = false,
	1468	.read_only = false,
	1469	.debug_hash = debug_hash,
	1470	.mode = .all,
	1471	.prog_node = root_prog_node,
	1472	};
	1473	defer job_queue.deinit();
	1474
	1475	var fetch: Fetch = .{
	1476	.arena = std.heap.ArenaAllocator.init(gpa),
	1477	.location = .{ .path_or_url = path_or_url },
	1478	.location_tok = 0,
	1479	.hash_tok = .none,
	1480	.name_tok = 0,
	1481	.lazy_status = .eager,
	1482	.remote_package_root = undefined,
	1483	.parent_package_root = undefined,
	1484	.parent_manifest_ast = null,
	1485	.prog_node = root_prog_node,
	1486	.job_queue = &job_queue,
	1487	.omit_missing_hash_error = true,
	1488	.allow_missing_paths_field = false,
	1489	.use_latest_commit = true,
	1490
	1491	.package_root = undefined,
	1492	.error_bundle = undefined,
	1493	.manifest = undefined,
	1494	.manifest_ast = undefined,
	1495	.have_manifest = false,
	1496	.computed_hash = undefined,
	1497	.has_build_zig = false,
	1498	.oom_flag = false,
	1499	.latest_commit = null,
	1500
	1501	.module = null,
	1502	};
	1503	defer fetch.deinit();
	1504
	1505	fetch.run() catch \|err\| switch (err) {
	1506	error.OutOfMemory, error.Canceled => \|e\| return e,
	1507	error.FetchFailed => {}, // error bundle checked below
	1508	};
	1509
	1510	try job_queue.group.await(io);
	1511
	1512	if (fetch.error_bundle.root_list.items.len > 0) {
	1513	var errors = try fetch.error_bundle.toOwnedBundle("");
	1514	errors.renderToStderr(io, .{}, color) catch {};
	1515	process.exit(1);
	1516	}
	1517
	1518	const package_hash = fetch.computedPackageHash();
	1519	const package_hash_slice = package_hash.toSlice();
	1520
	1521	root_prog_node.end();
	1522	root_prog_node = .{ .index = .none };
	1523
	1524	const name = switch (save) {
	1525	.no => {
	1526	var data: [2][]const u8 = .{ package_hash_slice, "\n" };
	1527	const w = initStdoutWriter();
	1528	try w.writeVecAll(&data);
	1529	try w.flush();
	1530	return cleanExit(io);
	1531	},
	1532	.yes, .exact => \|name\| name: {
	1533	if (name) \|n\| break :name n;
	1534	if (!fetch.have_manifest)
	1535	fatal("unable to determine name; fetched package has no build.zig.zon file", .{});
	1536	break :name fetch.manifest.name;
	1537	},
	1538	};
	1539
	1540	// The name to use in case the manifest file needs to be created now.
	1541	const init_root_name = Dir.path.basename(build_root.directory.path orelse cwd_path);
	1542	var manifest, var ast = try loadManifest(gpa, arena, io, .{
	1543	.root_name = try sanitizeExampleName(arena, init_root_name),
	1544	.dir = build_root.directory.handle,
	1545	.color = color,
	1546	});
	1547	defer {
	1548	manifest.deinit(gpa);
	1549	ast.deinit(gpa);
	1550	}
	1551
	1552	var fixups: Ast.Render.Fixups = .{};
	1553	defer fixups.deinit(gpa);
	1554
	1555	var saved_path_or_url = path_or_url;
	1556
	1557	if (fetch.latest_commit) \|latest_commit\| resolved: {
	1558	const latest_commit_hex = try allocPrint(arena, "{f}", .{latest_commit});
	1559
	1560	var uri = try std.Uri.parse(path_or_url);
	1561
	1562	if (uri.fragment) \|fragment\| {
	1563	const target_ref = try fragment.toRawMaybeAlloc(arena);
	1564
	1565	// the refspec may already be fully resolved
	1566	if (std.mem.eql(u8, target_ref, latest_commit_hex)) break :resolved;
	1567
	1568	std.log.info("resolved ref {q} to commit {s}", .{ target_ref, latest_commit_hex });
	1569
	1570	// include the original refspec in a query parameter, could be used to check for updates
	1571	uri.query = .{ .percent_encoded = try allocPrint(arena, "ref={f}", .{
	1572	std.fmt.alt(fragment, .formatEscaped),
	1573	}) };
	1574	} else {
	1575	std.log.info("resolved to commit {s}", .{latest_commit_hex});
	1576	}
	1577
	1578	// replace the refspec with the resolved commit SHA
	1579	uri.fragment = .{ .raw = latest_commit_hex };
	1580
	1581	switch (save) {
	1582	.yes => saved_path_or_url = try allocPrint(arena, "{f}", .{uri}),
	1583	.no, .exact => {}, // keep the original URL
	1584	}
	1585	}
	1586
	1587	const new_node_init = try allocPrint(arena,
	1588	\\.{{
	1589	\\ .url = "{f}",
	1590	\\ .hash = "{f}",
	1591	\\ }}
	1592	, .{
	1593	std.zig.fmtString(saved_path_or_url),
	1594	std.zig.fmtString(package_hash_slice),
	1595	});
	1596
	1597	const new_node_text = try allocPrint(arena, ".{f} = {s},\n", .{
	1598	std.zig.fmtIdPU(name), new_node_init,
	1599	});
	1600
	1601	const dependencies_init = try allocPrint(arena, ".{{\n {s} }}", .{
	1602	new_node_text,
	1603	});
	1604
	1605	const dependencies_text = try allocPrint(arena, ".dependencies = {s},\n", .{
	1606	dependencies_init,
	1607	});
	1608
	1609	if (manifest.dependencies.get(name)) \|dep\| {
	1610	if (dep.hash) \|h\| {
	1611	switch (dep.location) {
	1612	.url => \|u\| {
	1613	if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) {
	1614	std.log.info("existing dependency named {q} is up-to-date", .{name});
	1615	process.exit(0);
	1616	}
	1617	},
	1618	.path => {},
	1619	}
	1620	}
	1621
	1622	const location_replace = try allocPrint(
	1623	arena,
	1624	"\"{f}\"",
	1625	.{std.zig.fmtString(saved_path_or_url)},
	1626	);
	1627	const hash_replace = try allocPrint(
	1628	arena,
	1629	"\"{f}\"",
	1630	.{std.zig.fmtString(package_hash_slice)},
	1631	);
	1632
	1633	warn("overwriting existing dependency named {q}", .{name});
	1634	try fixups.replace_nodes_with_string.put(gpa, dep.location_node, location_replace);
	1635	if (dep.hash_node.unwrap()) \|hash_node\| {
	1636	try fixups.replace_nodes_with_string.put(gpa, hash_node, hash_replace);
	1637	} else {
	1638	// https://github.com/ziglang/zig/issues/21690
	1639	}
	1640	} else if (manifest.dependencies.count() > 0) {
	1641	// Add fixup for adding another dependency.
	1642	const deps = manifest.dependencies.values();
	1643	const last_dep_node = deps[deps.len - 1].node;
	1644	try fixups.append_string_after_node.put(gpa, last_dep_node, new_node_text);
	1645	} else if (manifest.dependencies_node.unwrap()) \|dependencies_node\| {
	1646	// Add fixup for replacing the entire dependencies struct.
	1647	try fixups.replace_nodes_with_string.put(gpa, dependencies_node, dependencies_init);
	1648	} else {
	1649	// Add fixup for adding dependencies struct.
	1650	try fixups.append_string_after_node.put(gpa, manifest.version_node, dependencies_text);
	1651	}
	1652
	1653	var aw: Io.Writer.Allocating = .init(gpa);
	1654	defer aw.deinit();
	1655	try ast.render(gpa, &aw.writer, fixups);
	1656	const rendered = aw.written();
	1657
	1658	build_root.directory.handle.writeFile(io, .{ .sub_path = Package.Manifest.basename, .data = rendered }) catch \|err\| {
	1659	fatal("unable to write {s} file: {t}", .{ Package.Manifest.basename, err });
	1660	};
	1661
	1662	return cleanExit(io);
	1663	}
	1664
	1665	const usage_fetch =
	1666	\\Usage: zig fetch [options] <url>
	1667	\\Usage: zig fetch [options] <path>
	1668	\\
	1669	\\ Copy a package into the global cache and print its hash.
	1670	\\ <url> must point to one of the following:
	1671	\\ - A git+http / git+https server for the package
	1672	\\ - A tarball file (with or without compression) containing
	1673	\\ package source
	1674	\\ - A git bundle file containing package source
	1675	\\
	1676	\\Examples:
	1677	\\
	1678	\\ zig fetch --save git+https://example.com/andrewrk/fun-example-tool.git
	1679	\\ zig fetch --save https://example.com/andrewrk/fun-example-tool/archive/refs/heads/master.tar.gz
	1680	\\
	1681	\\Options:
	1682	\\ -h, --help Print this help and exit
	1683	\\ --global-cache-dir [path] Override path to global Zig cache directory
	1684	\\ --cache-dir [path] Override path to local cache directory
	1685	\\ --pkg-dir [path] Override path to local package directory
	1686	\\ --debug-hash Print verbose hash information to stdout
	1687	\\ --debug-log [scope] Enable printing debug/info log messages for scope
	1688	\\ --save Add the fetched package to build.zig.zon
	1689	\\ --save=[name] Add the fetched package to build.zig.zon as name
	1690	\\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim
	1691	\\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim
	1692	\\
	1693	;
	1694
	1695	fn cmdBuild() !void {
	1696
	1697	}
	1698
709	1699	fn markFailedStepsDirty(maker: *Maker) void {
710	1700	const all_steps = maker.step_stack.keys();
711	1701
...	...	@@ -1677,16 +2667,13 @@ fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 {
1677	2667	}
1678	2668
1679	2669	fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 {
1680		return nextArg(args, idx) orelse {
1681		fatalWithHint("expected argument after {q}", .{args[idx.* - 1]});
1682		};
	2670	return nextArg(args, idx) orelse fatalWithHint("expected argument after {q}", .{args[idx.* - 1]});
1683	2671	}
1684	2672
1685		fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 {
1686		const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first});
1687		if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg });
1688		const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first});
1689		return arg;
	2673	fn prefixedArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, prefix: []const u8) []const u8 {
	2674	const arg = args[index_ptr.*];
	2675	if (mem.cutPrefix(u8, arg, prefix)) \|rest\| return rest;
	2676	fatal("expected {q} to begin with {q}", .{arg, prefix});
1690	2677	}
1691	2678
1692	2679	fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 {
...	...	@@ -2006,11 +2993,11 @@ pub fn installSymLinks(
2006	2993	const name = conf_comp.root_name.slice(c);
2007	2994
2008	2995	const filename_major_only, const filename_name_only = if (os_tag.isDarwin()) .{
2009		try std.fmt.allocPrint(arena, "lib{s}.{d}.dylib", .{ name, version.major }),
2010		try std.fmt.allocPrint(arena, "lib{s}.dylib", .{name}),
	2996	try allocPrint(arena, "lib{s}.{d}.dylib", .{ name, version.major }),
	2997	try allocPrint(arena, "lib{s}.dylib", .{name}),
2011	2998	} else .{
2012		try std.fmt.allocPrint(arena, "lib{s}.so.{d}", .{ name, version.major }),
2013		try std.fmt.allocPrint(arena, "lib{s}.so", .{name}),
	2999	try allocPrint(arena, "lib{s}.so.{d}", .{ name, version.major }),
	3000	try allocPrint(arena, "lib{s}.so", .{name}),
2014	3001	};
2015	3002
2016	3003	return installSymLinksInner(maker, arena, output_path, asking_step_index, filename_major_only, filename_name_only);
...	...	@@ -2059,3 +3046,229 @@ inline fn debugMakerLeaks() bool {
2059	3046	if (!is_debug_mode) return false;
2060	3047	return debug_maker_leaks;
2061	3048	}
	3049
	3050	const BuildRoot = struct {
	3051	directory: Cache.Directory,
	3052	build_zig_basename: []const u8,
	3053	cleanup_build_dir: ?Io.Dir,
	3054
	3055	fn deinit(br: *BuildRoot, io: Io) void {
	3056	if (br.cleanup_build_dir) \|*dir\| dir.close(io);
	3057	br.* = undefined;
	3058	}
	3059	};
	3060
	3061	const FindBuildRootOptions = struct {
	3062	build_file: ?[]const u8 = null,
	3063	cwd_path: ?[]const u8 = null,
	3064	};
	3065
	3066	fn findBuildRoot(arena: Allocator, io: Io, options: FindBuildRootOptions) !BuildRoot {
	3067	const cwd_path = options.cwd_path orelse try std.zig.getResolvedCwd(io, arena);
	3068	const build_zig_basename = if (options.build_file) \|bf\|
	3069	Dir.path.basename(bf)
	3070	else
	3071	std.zig.build_zig_basename;
	3072
	3073	if (options.build_file) \|bf\| {
	3074	if (Dir.path.dirname(bf)) \|dirname\| {
	3075	const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch \|err\| {
	3076	fatal("failed opening directory containing {q}: {t}", .{ bf, err });
	3077	};
	3078	return .{
	3079	.build_zig_basename = build_zig_basename,
	3080	.directory = .{ .path = dirname, .handle = dir },
	3081	.cleanup_build_dir = dir,
	3082	};
	3083	}
	3084
	3085	return .{
	3086	.build_zig_basename = build_zig_basename,
	3087	.directory = .{ .path = null, .handle = Io.Dir.cwd() },
	3088	.cleanup_build_dir = null,
	3089	};
	3090	}
	3091	// Search up parent directories until we find build.zig.
	3092	var dirname: []const u8 = cwd_path;
	3093	while (true) {
	3094	const joined_path = try Dir.path.join(arena, &[_][]const u8{ dirname, build_zig_basename });
	3095	if (Io.Dir.cwd().access(io, joined_path, .{})) \|_\| {
	3096	const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch \|err\| {
	3097	fatal("unable to open directory while searching for build.zig file, {q}: {t}", .{ dirname, err });
	3098	};
	3099	return .{
	3100	.build_zig_basename = build_zig_basename,
	3101	.directory = .{
	3102	.path = dirname,
	3103	.handle = dir,
	3104	},
	3105	.cleanup_build_dir = dir,
	3106	};
	3107	} else \|err\| switch (err) {
	3108	error.FileNotFound => {
	3109	dirname = Dir.path.dirname(dirname) orelse {
	3110	std.log.info("initialize {s} template file with \"zig init\"", .{ std.zig.build_zig_basename });
	3111	std.log.info("see \"zig --help\" for more options", .{});
	3112	fatal("no build.zig file found, in the current directory or any parent directories", .{});
	3113	};
	3114	continue;
	3115	},
	3116	else => \|e\| return e,
	3117	}
	3118	}
	3119	}
	3120
	3121	const Fork = struct {
	3122	path: Path,
	3123	manifest_ast: std.zig.Ast,
	3124	manifest: Package.Manifest,
	3125	error_bundle: std.zig.ErrorBundle.Wip,
	3126	failed: bool,
	3127	arena_allocator: std.heap.ArenaAllocator,
	3128
	3129	fn init(cwd_relative_path: []const u8) Fork {
	3130	return .{
	3131	.manifest_ast = undefined,
	3132	.manifest = undefined,
	3133	.error_bundle = undefined,
	3134	.arena_allocator = undefined,
	3135	.path = .{
	3136	.root_dir = .cwd(),
	3137	.sub_path = cwd_relative_path,
	3138	},
	3139	.failed = false,
	3140	};
	3141	}
	3142
	3143	fn load(io: Io, gpa: Allocator, fork: *Fork, color: Color) Io.Cancelable!void {
	3144	loadFallible(io, gpa, fork, color) catch \|err\| switch (err) {
	3145	error.Canceled => \|e\| return e,
	3146	error.AlreadyReported => fork.failed = true,
	3147	else => \|e\| {
	3148	std.log.err("failed to load fork at {f}: {t}", .{ fork.path, e });
	3149	fork.failed = true;
	3150	},
	3151	};
	3152	}
	3153
	3154	fn loadFallible(io: Io, gpa: Allocator, fork: *Fork, color: Color) !void {
	3155	fork.arena_allocator = .init(gpa);
	3156	const arena = fork.arena_allocator.allocator();
	3157
	3158	var error_bundle: std.zig.ErrorBundle.Wip = undefined;
	3159	try error_bundle.init(gpa);
	3160	defer error_bundle.deinit();
	3161
	3162	const manifest_path = try fork.path.join(arena, Package.Manifest.basename);
	3163
	3164	Package.Manifest.load(
	3165	io,
	3166	arena,
	3167	manifest_path,
	3168	&fork.manifest_ast,
	3169	&error_bundle,
	3170	&fork.manifest,
	3171	true,
	3172	) catch \|err\| switch (err) {
	3173	error.Canceled => \|e\| return e,
	3174	error.ErrorsBundled => {
	3175	assert(error_bundle.root_list.items.len > 0);
	3176	var errors = try error_bundle.toOwnedBundle("");
	3177	errors.renderToStderr(io, .{}, color) catch {};
	3178	return error.AlreadyReported;
	3179	},
	3180	else => \|e\| {
	3181	std.log.err("failed to load package manifest {f}: {t}", .{ manifest_path, e });
	3182	return error.AlreadyReported;
	3183	},
	3184	};
	3185	}
	3186
	3187	fn deinitList(forks: []Fork) void {
	3188	for (forks) \|*fork\| fork.arena_allocator.deinit();
	3189	}
	3190	};
	3191
	3192	fn parseRandomSeed(arg: []const u8) u32 {
	3193	return std.fmt.parseUnsigned(u32, arg, 0) catch \|err\|
	3194	fatal("failed parsing random seed {q} as unsigned 32-bit integer: {t}", .{ arg, err });
	3195	}
	3196
	3197	fn randInt(io: Io, comptime T: type) T {
	3198	var x: T = undefined;
	3199	io.random(@ptrCast(&x));
	3200	return x;
	3201	}
	3202
	3203	const LoadManifestOptions = struct {
	3204	root_name: []const u8,
	3205	dir: Io.Dir,
	3206	color: Color,
	3207	};
	3208
	3209	fn loadManifest(
	3210	gpa: Allocator,
	3211	arena: Allocator,
	3212	io: Io,
	3213	options: LoadManifestOptions,
	3214	) !struct { Package.Manifest, std.zig.Ast } {
	3215	const rng: std.Random.IoSource = .{ .io = io };
	3216
	3217	const manifest_bytes = while (true) {
	3218	break options.dir.readFileAllocOptions(
	3219	io,
	3220	Package.Manifest.basename,
	3221	arena,
	3222	.limited(Package.Manifest.max_bytes),
	3223	.@"1",
	3224	0,
	3225	) catch \|err\| switch (err) {
	3226	error.FileNotFound => {
	3227	writeSimpleTemplateFile(io, Package.Manifest.basename,
	3228	\\.{{
	3229	\\ .name = .{s},
	3230	\\ .version = "{s}",
	3231	\\ .paths = .{{""}},
	3232	\\ .fingerprint = 0x{x},
	3233	\\}}
	3234	\\
	3235	, .{
	3236	options.root_name,
	3237	build_options.version,
	3238	Package.Fingerprint.generate(rng.interface(), options.root_name).int(),
	3239	}) catch \|e\| {
	3240	fatal("unable to write {s}: {t}", .{ Package.Manifest.basename, e });
	3241	};
	3242	continue;
	3243	},
	3244	else => \|e\| fatal("unable to load {s}: {t}", .{ Package.Manifest.basename, e }),
	3245	};
	3246	};
	3247	var ast = try Ast.parse(gpa, manifest_bytes, .zon);
	3248	errdefer ast.deinit(gpa);
	3249
	3250	if (ast.errors.len > 0) {
	3251	try std.zig.printAstErrorsToStderr(gpa, io, ast, Package.Manifest.basename, options.color);
	3252	process.exit(2);
	3253	}
	3254
	3255	var manifest = try Package.Manifest.parse(gpa, &ast, rng.interface(), .{});
	3256	errdefer manifest.deinit(gpa);
	3257
	3258	if (manifest.errors.len > 0) {
	3259	var wip_errors: std.zig.ErrorBundle.Wip = undefined;
	3260	try wip_errors.init(gpa);
	3261	defer wip_errors.deinit();
	3262
	3263	const src_path = try wip_errors.addString(Package.Manifest.basename);
	3264	try manifest.copyErrorsIntoBundle(ast, src_path, &wip_errors);
	3265
	3266	var error_bundle = try wip_errors.toOwnedBundle("");
	3267	defer error_bundle.deinit(gpa);
	3268	error_bundle.renderToStderr(io, .{}, options.color) catch {};
	3269
	3270	process.exit(2);
	3271	}
	3272	return .{ manifest, ast };
	3273	}
	3274

lib/compiler/Maker/Fetch.zig created+2286

...	...	@@ -0,0 +1,2286 @@
	1	//! Represents one independent job whose responsibility is to:
	2	//!
	3	//! 1. Check the local zig package directory to see if the hash already exists.
	4	//! If so, load, parse, and validate the build.zig.zon file therein, and
	5	//! goto step 9. Likewise if the location is a relative path, treat this
	6	//! the same as a cache hit. Otherwise, proceed.
	7	//! 2. Check the global package cache for a compressed tarball matching the
	8	//! hash. If it is found, unpack the contents into a temporary directory inside
	9	//! project local zig cache. Rename this directory into the local zig package
	10	//! directory and goto step 9, skipping step 10.
	11	//! 3. Fetch and unpack a URL into a temporary directory.
	12	//! 4. Load, parse, and validate the build.zig.zon file therein. It is allowed
	13	//! for the file to be missing, in which case this fetched package is considered
	14	//! to be a "naked" package.
	15	//! 5. Apply inclusion rules of the build.zig.zon to the temporary directory by
	16	//! deleting excluded files. If any files had errors for files that were
	17	//! ultimately excluded, those errors should be ignored, such as failure to
	18	//! create symlinks that weren't supposed to be included anyway.
	19	//! 6. Compute the package hash based on the remaining files in the temporary
	20	//! directory.
	21	//! 7. Rename the temporary directory into the local zig package directory. If
	22	//! the hash already exists, delete the temporary directory and leave the zig
	23	//! package directory untouched as it may be in use. This is done even if
	24	//! the hash is invalid, in case the package with the different hash is used
	25	//! in the future.
	26	//! 8. Validate the computed hash against the expected hash. If invalid,
	27	//! this job is done.
	28	//! 9. Spawn a new fetch job for each dependency in the manifest file. Use
	29	//! a mutex and a hash map so that redundant jobs do not get queued up.
	30	//! 10.Compress the package directory and store it into the global package
	31	//! cache.
	32	//!
	33	//! All of this must be done with only referring to the state inside this struct
	34	//! because this work will be done in a dedicated thread.
	35	const Fetch = @This();
	36
	37	const builtin = @import("builtin");
	38	const native_os = builtin.os.tag;
	39
	40	const std = @import("std");
	41	const Io = std.Io;
	42	const fs = std.fs;
	43	const log = std.log.scoped(.fetch);
	44	const assert = std.debug.assert;
	45	const ascii = std.ascii;
	46	const Allocator = std.mem.Allocator;
	47	const Cache = std.Build.Cache;
	48	const git = @import("Fetch/git.zig");
	49	const Package = @import("../Package.zig");
	50	const Manifest = Package.Manifest;
	51	const ErrorBundle = std.zig.ErrorBundle;
	52
	53	arena: std.heap.ArenaAllocator,
	54	location: Location,
	55	location_tok: std.zig.Ast.TokenIndex,
	56	hash_tok: std.zig.Ast.OptionalTokenIndex,
	57	name_tok: std.zig.Ast.TokenIndex,
	58	lazy_status: LazyStatus,
	59	/// Same as `parent_packge_root` except it is unchanged when recursing into
	60	/// relative file paths (as opposed to URL).
	61	remote_package_root: Cache.Path,
	62	parent_package_root: Cache.Path,
	63	parent_manifest_ast: ?*const std.zig.Ast,
	64	prog_node: std.Progress.Node,
	65	job_queue: *JobQueue,
	66	/// If true, don't add an error for a missing hash. This flag is not passed
	67	/// down to recursive dependencies. It's intended to be used only be the CLI.
	68	omit_missing_hash_error: bool,
	69	/// If true, don't fail when a manifest file is missing the `paths` field,
	70	/// which specifies inclusion rules. This is intended to be true for the first
	71	/// fetch task and false for the recursive dependencies.
	72	allow_missing_paths_field: bool,
	73	/// If true and URL points to a Git repository, will use the latest commit.
	74	use_latest_commit: bool,
	75
	76	// Above this are fields provided as inputs to `run`.
	77	// Below this are fields populated by `run`.
	78
	79	/// Relative to the build root of the root package.
	80	package_root: Cache.Path,
	81	error_bundle: ErrorBundle.Wip,
	82	manifest: Manifest,
	83	manifest_ast: std.zig.Ast,
	84	have_manifest: bool,
	85	computed_hash: ComputedHash,
	86	/// Fetch logic notices whether a package has a build.zig file and sets this flag.
	87	has_build_zig: bool,
	88	/// Indicates whether the task aborted due to an out-of-memory condition.
	89	oom_flag: bool,
	90	/// If `use_latest_commit` was true, this will be set to the commit that was used.
	91	/// If the resource pointed to by the location is not a Git-repository, this
	92	/// will be left unchanged.
	93	latest_commit: ?git.Oid,
	94
	95	// This field is used by the CLI only, untouched by this file.
	96
	97	/// The module for this `Fetch` tasks's package, which exposes `build.zig` as
	98	/// the root source file.
	99	///
	100	/// This could be an opaque "userdata" field because this code does not observe
	101	/// this data in any way but let's have some type safety because we can.
	102	cli_module: ?*@import("../Maker.zig").CliModule,
	103
	104	pub const LazyStatus = enum {
	105	/// Not lazy.
	106	eager,
	107	/// Lazy, found.
	108	available,
	109	/// Lazy, not found.
	110	unavailable,
	111	};
	112
	113	pub const LocalStorage = struct {
	114	cache_root: Cache.Path,
	115	/// Path to "zig-pkg" inside the package in which the user ran `zig build`.
	116	pkg_root: Cache.Path,
	117	};
	118
	119	/// Contains shared state among all `Fetch` tasks.
	120	pub const JobQueue = struct {
	121	io: Io,
	122	mutex: Io.Mutex = .init,
	123	/// It's an array hash map so that it can be sorted before rendering the
	124	/// dependencies.zig source file.
	125	/// Protected by `mutex`.
	126	table: Table = .{},
	127	/// `table` may be missing some tasks such as ones that failed, so this
	128	/// field contains references to all of them.
	129	/// Protected by `mutex`.
	130	all_fetches: std.ArrayList(*Fetch) = .empty,
	131	prog_node: std.Progress.Node,
	132
	133	http_client: *std.http.Client,
	134	/// This tracks `Fetch` tasks as well as recompression tasks.
	135	group: Io.Group = .init,
	136	global_cache: Cache.Directory,
	137	/// If `null`, indicates fetch globally only.
	138	local_storage: ?*const LocalStorage,
	139	/// If true then, no fetching occurs, and:
	140	/// * The `global_cache` directory is assumed to be the direct parent
	141	/// directory of on-disk packages rather than having the "p/" directory
	142	/// prefix inside of it.
	143	/// * An error occurs if any non-lazy packages are not already present in
	144	/// the package cache directory.
	145	/// * Missing hash field causes an error, and no fetching occurs so it does
	146	/// not print the correct hash like usual.
	147	read_only: bool,
	148	recursive: bool,
	149	/// Dumps hash information to stdout which can be used to troubleshoot why
	150	/// two hashes of the same package do not match.
	151	/// If this is true, `recursive` must be false.
	152	debug_hash: bool,
	153	mode: Mode,
	154	/// Set of hashes that will be additionally fetched even if they are marked
	155	/// as lazy.
	156	unlazy_set: UnlazySet = .{},
	157	/// Identifies paths that override all packages in the tree with matching
	158	/// project ids.
	159	fork_set: ForkSet = .{},
	160
	161	pub const Mode = enum {
	162	/// Non-lazy dependencies are always fetched.
	163	/// Lazy dependencies are fetched only when needed.
	164	needed,
	165	/// Both non-lazy and lazy dependencies are always fetched.
	166	all,
	167	};
	168	pub const Table = std.array_hash_map.Auto(Package.Hash, *Fetch);
	169	pub const UnlazySet = std.array_hash_map.Auto(Package.Hash, void);
	170	pub const ForkSet = std.array_hash_map.Custom(Fork, void, Fork.Context, false);
	171
	172	pub const Fork = struct {
	173	path: Cache.Path,
	174	manifest_ast: std.zig.Ast,
	175	manifest: Package.Manifest,
	176	uses: usize,
	177
	178	pub const Context = struct {
	179	pub fn hash(_: @This(), a: Fork) u32 {
	180	const project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id);
	181	return @truncate(project_id.hash());
	182	}
	183
	184	pub fn eql(_: @This(), a: Fork, b: Fork, _: usize) bool {
	185	const a_project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id);
	186	const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id);
	187	return a_project_id.eql(&b_project_id);
	188	}
	189	};
	190
	191	pub const Adapter = struct {
	192	pub fn hash(_: @This(), a: Package.ProjectId) u32 {
	193	return @truncate(a.hash());
	194	}
	195
	196	pub fn eql(_: @This(), a_project_id: Package.ProjectId, b: Fork, _: usize) bool {
	197	const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id);
	198	return a_project_id.eql(&b_project_id);
	199	}
	200	};
	201	};
	202
	203	pub fn deinit(jq: *JobQueue) void {
	204	const io = jq.io;
	205	jq.group.cancel(io);
	206	if (jq.all_fetches.items.len == 0) return;
	207	const gpa = jq.all_fetches.items[0].arena.child_allocator;
	208	jq.table.deinit(gpa);
	209	// These must be deinitialized in reverse order because subsequent
	210	// `Fetch` instances are allocated in prior ones' arenas.
	211	// Sorry, I know it's a bit weird, but it slightly simplifies the
	212	// critical section.
	213	while (jq.all_fetches.pop()) \|f\| f.deinit();
	214	jq.all_fetches.deinit(gpa);
	215	jq.* = undefined;
	216	}
	217
	218	/// Dumps all subsequent error bundles into the first one.
	219	pub fn consolidateErrors(jq: *JobQueue) !void {
	220	const root = &jq.all_fetches.items[0].error_bundle;
	221	const gpa = root.gpa;
	222	for (jq.all_fetches.items[1..]) \|fetch\| {
	223	if (fetch.error_bundle.root_list.items.len > 0) {
	224	var bundle = try fetch.error_bundle.toOwnedBundle("");
	225	defer bundle.deinit(gpa);
	226	try root.addBundleAsRoots(bundle);
	227	}
	228	}
	229	}
	230
	231	/// Creates the dependencies.zig source code for the build runner to obtain
	232	/// via `@import("@dependencies")`.
	233	pub fn createDependenciesSource(jq: JobQueue, w: Io.Writer) Io.Writer.Error!void {
	234	const keys = jq.table.keys();
	235
	236	assert(keys.len != 0); // caller should have added the first one
	237	if (keys.len == 1) {
	238	// This is the first one. It must have no dependencies.
	239	return createEmptyDependenciesSource(w);
	240	}
	241
	242	try w.writeAll("pub const packages = struct {\n");
	243
	244	// Ensure the generated .zig file is deterministic.
	245	jq.table.sortUnstable(@as(struct {
	246	keys: []const Package.Hash,
	247	pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
	248	return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes);
	249	}
	250	}, .{ .keys = keys }));
	251
	252	for (keys, jq.table.values()) \|*hash, fetch\| {
	253	if (fetch == jq.all_fetches.items[0]) {
	254	// The first one is a dummy package for the current project.
	255	continue;
	256	}
	257
	258	const hash_slice = hash.toSlice();
	259
	260	try w.print(
	261	\\ pub const {f} = struct {{
	262	\\
	263	, .{std.zig.fmtId(hash_slice)});
	264
	265	lazy: {
	266	switch (fetch.lazy_status) {
	267	.eager => break :lazy,
	268	.available => {
	269	try w.writeAll(
	270	\\ pub const available = true;
	271	\\
	272	);
	273	break :lazy;
	274	},
	275	.unavailable => {
	276	try w.writeAll(
	277	\\ pub const available = false;
	278	\\ };
	279	\\
	280	);
	281	continue;
	282	},
	283	}
	284	}
	285
	286	try w.print(
	287	\\ pub const build_root = "{f}";
	288	\\
	289	, .{std.fmt.alt(fetch.package_root, .formatEscapeString)});
	290
	291	if (fetch.has_build_zig) {
	292	try w.print(
	293	\\ pub const build_zig = @import("{f}");
	294	\\
	295	, .{std.zig.fmtString(hash_slice)});
	296	}
	297
	298	if (fetch.have_manifest) {
	299	const manifest = &fetch.manifest;
	300	try w.writeAll(
	301	\\ pub const deps: []const struct { []const u8, []const u8 } = &.{
	302	\\
	303	);
	304	for (manifest.dependencies.keys(), manifest.dependencies.values()) \|name, dep\| {
	305	const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue;
	306	try w.print(
	307	" .{{ \"{f}\", \"{f}\" }},\n",
	308	.{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) },
	309	);
	310	}
	311
	312	try w.writeAll(
	313	\\ };
	314	\\ };
	315	\\
	316	);
	317	} else {
	318	try w.writeAll(
	319	\\ pub const deps: []const struct { []const u8, []const u8 } = &.{};
	320	\\ };
	321	\\
	322	);
	323	}
	324	}
	325
	326	try w.writeAll(
	327	\\};
	328	\\
	329	\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{
	330	\\
	331	);
	332
	333	const root_fetch = jq.all_fetches.items[0];
	334	assert(root_fetch.have_manifest);
	335	const root_manifest = &root_fetch.manifest;
	336
	337	for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) \|name, dep\| {
	338	const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue;
	339	try w.print(
	340	" .{{ \"{f}\", \"{f}\" }},\n",
	341	.{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) },
	342	);
	343	}
	344	try w.appendSlice("};\n");
	345	}
	346
	347	pub fn createEmptyDependenciesSource(w: *Io.Writer) Io.Writer!void {
	348	try w.writeAll(
	349	\\pub const packages = struct {};
	350	\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
	351	\\
	352	);
	353	}
	354
	355	fn recompress(jq: *JobQueue, package_hash: Package.Hash, package_root: Cache.Path) Io.Cancelable!void {
	356	const pkg_hash_slice = package_hash.toSlice();
	357
	358	const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice});
	359	defer prog_node.end();
	360
	361	var dest_sub_path_buf: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined;
	362	const dest_path: Cache.Path = .{
	363	.root_dir = jq.global_cache,
	364	.sub_path = std.fmt.bufPrint(&dest_sub_path_buf, "p/{s}.tar.gz", .{pkg_hash_slice}) catch unreachable,
	365	};
	366
	367	const gpa = jq.http_client.allocator;
	368
	369	var arena_instance = std.heap.ArenaAllocator.init(gpa);
	370	defer arena_instance.deinit();
	371	const arena = arena_instance.allocator();
	372
	373	recompressFallible(jq, arena, dest_path, pkg_hash_slice, package_root, prog_node) catch \|err\| switch (err) {
	374	error.Canceled => \|e\| return e,
	375	error.ReadFailed => comptime unreachable,
	376	error.WriteFailed => comptime unreachable,
	377	else => \|e\| log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }),
	378	};
	379	}
	380
	381	fn recompressFallible(
	382	jq: *JobQueue,
	383	arena: Allocator,
	384	dest_path: Cache.Path,
	385	pkg_hash_slice: []const u8,
	386	package_root: Cache.Path,
	387	prog_node: std.Progress.Node,
	388	) !void {
	389	const gpa = jq.http_client.allocator;
	390	const io = jq.io;
	391
	392	// We have to walk the file system up front in order to sort the file
	393	// list for determinism purposes. The hash of the recompressed file is
	394	// not critical because the true hash is based on the content alone.
	395	// However, if we want Zig users to be able to share cached package
	396	// data with each other via peer-to-peer protocols, we benefit greatly
	397	// from the data being identical on everyone's computers.
	398	var scanned_files: std.ArrayList(ScannedFile) = .empty;
	399	defer scanned_files.deinit(gpa);
	400
	401	var pkg_dir = try package_root.root_dir.handle.openDir(io, package_root.sub_path, .{ .iterate = true });
	402	defer pkg_dir.close(io);
	403
	404	{
	405	var walker = try pkg_dir.walk(gpa);
	406	defer walker.deinit();
	407
	408	while (try walker.next(io)) \|entry\| {
	409	const symlink = switch (entry.kind) {
	410	.directory => continue,
	411	.file => false,
	412	.sym_link => true,
	413	else => return error.IllegalFileType,
	414	};
	415	const entry_path = try arena.dupe(u8, entry.path);
	416	// If necessary, normalize path separators to POSIX-style since the tar format requires that.
	417	if (comptime (std.fs.path.sep != std.fs.path.sep_posix)) {
	418	std.mem.replaceScalar(u8, entry_path, std.fs.path.sep, std.fs.path.sep_posix);
	419	}
	420	try scanned_files.append(gpa, .{
	421	.ptr = entry_path.ptr,
	422	.len = @intCast(entry_path.len),
	423	.symlink = symlink,
	424	});
	425	}
	426
	427	std.mem.sortUnstable(ScannedFile, scanned_files.items, {}, stringCmp);
	428	}
	429
	430	prog_node.setEstimatedTotalItems(scanned_files.items.len);
	431
	432	var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{
	433	.make_path = true,
	434	.replace = true,
	435	});
	436	defer atomic_file.deinit(io);
	437
	438	var file_write_buffer: [4096]u8 = undefined;
	439	var file_writer = atomic_file.file.writer(io, &file_write_buffer);
	440
	441	var compress_buffer: [std.compress.flate.max_window_len]u8 = undefined;
	442	var compress = std.compress.flate.Compress.init(&file_writer.interface, &compress_buffer, .gzip, .level_9) catch \|err\| switch (err) {
	443	error.WriteFailed => return file_writer.err.?,
	444	};
	445
	446	var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer };
	447	archiver.prefix = pkg_hash_slice;
	448
	449	var file_read_buffer: [4096]u8 = undefined;
	450	var link_buf: [fs.max_path_bytes]u8 = undefined;
	451
	452	for (scanned_files.items) \|scanned_file\| {
	453	const entry_path = scanned_file.ptr[0..scanned_file.len];
	454	if (scanned_file.symlink) {
	455	const link_name = link_buf[0..try pkg_dir.readLink(io, entry_path, &link_buf)];
	456	archiver.writeLink(entry_path, link_name, .{}) catch \|err\| switch (err) {
	457	error.WriteFailed => return file_writer.err.?,
	458	else => \|e\| return e,
	459	};
	460	} else {
	461	var file = try pkg_dir.openFile(io, entry_path, .{});
	462	defer file.close(io);
	463	var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer);
	464	archiver.writeFile(entry_path, &file_reader, 0) catch \|err\| switch (err) {
	465	error.ReadFailed => return file_reader.err.?,
	466	error.WriteFailed => return file_writer.err.?,
	467	else => \|e\| return e,
	468	};
	469	}
	470	prog_node.completeOne();
	471	}
	472
	473	// intentionally omitting the pointless trailer
	474	//try archiver.finish();
	475	compress.finish() catch \|err\| switch (err) {
	476	error.WriteFailed => return file_writer.err.?,
	477	};
	478	try file_writer.flush();
	479	try atomic_file.replace(io);
	480	}
	481	};
	482
	483	const ScannedFile = struct {
	484	ptr: [*]const u8,
	485	len: u32,
	486	symlink: bool,
	487	};
	488
	489	fn stringCmp(_: void, lhs: ScannedFile, rhs: ScannedFile) bool {
	490	return std.mem.lessThan(u8, lhs.ptr[0..lhs.len], rhs.ptr[0..rhs.len]);
	491	}
	492
	493	pub const Location = union(enum) {
	494	remote: Remote,
	495	/// A directory found inside the parent package.
	496	relative_path: Cache.Path,
	497	/// Recursive Fetch tasks will never use this Location, but it may be
	498	/// passed in by the CLI. Indicates the file contents here should be copied
	499	/// into the global package cache. It may be a file relative to the cwd or
	500	/// absolute, in which case it should be treated exactly like a `file://`
	501	/// URL, or a directory, in which case it should be treated as an
	502	/// already-unpacked directory (but still needs to be copied into the
	503	/// global package cache and have inclusion rules applied).
	504	path_or_url: []const u8,
	505
	506	pub const Remote = struct {
	507	url: []const u8,
	508	/// If this is null it means the user omitted the hash field from a dependency.
	509	/// It will be an error but the logic should still fetch and print the discovered hash.
	510	hash: ?Package.Hash,
	511	};
	512	};
	513
	514	pub const RunError = error{
	515	OutOfMemory,
	516	Canceled,
	517	/// This error code is intended to be handled by inspecting the
	518	/// `error_bundle` field.
	519	FetchFailed,
	520	};
	521
	522	pub fn run(f: *Fetch) RunError!void {
	523	const job_queue = f.job_queue;
	524	const io = job_queue.io;
	525	const eb = &f.error_bundle;
	526	const arena = f.arena.allocator();
	527	const gpa = f.arena.child_allocator;
	528
	529	try eb.init(gpa);
	530
	531	// Check the global zig package cache to see if the hash already exists. If
	532	// so, load, parse, and validate the build.zig.zon file therein, and skip
	533	// ahead to queuing up jobs for dependencies. Likewise if the location is a
	534	// relative path, treat this the same as a cache hit. Otherwise, proceed.
	535
	536	const remote = switch (f.location) {
	537	.relative_path => \|pkg_root\| {
	538	if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail(
	539	f.location_tok,
	540	try eb.addString("expected path relative to build root; found absolute path"),
	541	);
	542	if (f.hash_tok.unwrap()) \|hash_tok\| return f.fail(
	543	hash_tok,
	544	try eb.addString("path-based dependencies are not hashed"),
	545	);
	546	// Packages fetched by URL may not use relative paths to escape outside the
	547	// fetched package directory from within the package cache.
	548
	549	// This code path is only reachable recursively and the sub_path
	550	// will already have been resolved to no longer have extra ".." or
	551	// "." components.
	552	assert(job_queue.local_storage != null);
	553	log.debug("checking pkg root \"{s}\" against parent package root \"{s}\"", .{
	554	pkg_root.sub_path, f.remote_package_root.sub_path,
	555	});
	556	assert(pkg_root.root_dir.eql(f.remote_package_root.root_dir));
	557	if (!std.mem.startsWith(u8, pkg_root.sub_path, f.remote_package_root.sub_path)) return f.fail(
	558	f.location_tok,
	559	try eb.printString("dependency path outside project: '{f}'", .{pkg_root}),
	560	);
	561	f.package_root = pkg_root;
	562	try loadManifest(f, pkg_root);
	563	if (!f.has_build_zig) try checkBuildFileExistence(f);
	564	if (!job_queue.recursive) return;
	565	return queueJobsForDeps(f);
	566	},
	567	.remote => \|remote\| remote,
	568	.path_or_url => \|path_or_url\| {
	569	if (Io.Dir.cwd().openDir(io, path_or_url, .{ .iterate = true })) \|dir\| {
	570	var resource: Resource = .{ .dir = dir };
	571	return f.runResource(path_or_url, &resource, null, false);
	572	} else \|dir_err\| {
	573	var server_header_buffer: [init_resource_buffer_size]u8 = undefined;
	574
	575	const file_err = if (dir_err == error.NotDir) e: {
	576	if (Io.Dir.cwd().openFile(io, path_or_url, .{})) \|file\| {
	577	var resource: Resource = .{ .file = file.reader(io, &server_header_buffer) };
	578	return f.runResource(path_or_url, &resource, null, false);
	579	} else \|err\| break :e err;
	580	} else dir_err;
	581
	582	const uri = std.Uri.parse(path_or_url) catch \|uri_err\| {
	583	return f.fail(0, try eb.printString(
	584	"'{s}' could not be recognized as a file path ({t}) or an URL ({t})",
	585	.{ path_or_url, file_err, uri_err },
	586	));
	587	};
	588	var resource: Resource = undefined;
	589	try f.initResource(uri, &resource, &server_header_buffer);
	590	return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, false);
	591	}
	592	},
	593	};
	594
	595	var resource_buffer: [init_resource_buffer_size]u8 = undefined;
	596
	597	if (remote.hash) \|expected_hash\| {
	598	const expected_project_id: Package.ProjectId = expected_hash.projectId();
	599	if (job_queue.fork_set.getKeyPtrAdapted(expected_project_id, @as(JobQueue.Fork.Adapter, .{}))) \|fork\| {
	600	log.debug("using fork {f} for {s}", .{ fork.path, fork.manifest.name });
	601	fork.uses += 1;
	602	f.package_root = fork.path;
	603	f.remote_package_root = f.package_root;
	604	f.manifest_ast = fork.manifest_ast;
	605	f.manifest = fork.manifest;
	606	f.have_manifest = true;
	607	try checkBuildFileExistence(f);
	608	if (!job_queue.recursive) return;
	609	return queueJobsForDeps(f);
	610	}
	611
	612	if (job_queue.local_storage) \|ls\| {
	613	const package_root = try ls.pkg_root.join(arena, expected_hash.toSlice());
	614	if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) \|_\| {
	615	assert(f.lazy_status != .unavailable);
	616	f.package_root = package_root;
	617	f.remote_package_root = f.package_root;
	618	try loadManifest(f, f.package_root);
	619	try checkBuildFileExistence(f);
	620	if (!job_queue.recursive) return;
	621	return queueJobsForDeps(f);
	622	} else \|err\| switch (err) {
	623	error.FileNotFound => {
	624	log.debug("FileNotFound: {f}", .{package_root});
	625	if (job_queue.read_only and f.lazy_status == .eager) return f.fail(
	626	f.name_tok,
	627	try eb.printString("package not found at '{f}'", .{package_root}),
	628	);
	629	},
	630	error.Canceled => \|e\| return e,
	631	else => \|e\| {
	632	try eb.addRootErrorMessage(.{
	633	.msg = try eb.printString("unable to open package cache directory {f}: {t}", .{
	634	package_root, e,
	635	}),
	636	});
	637	return error.FetchFailed;
	638	},
	639	}
	640	}
	641
	642	// Check global cache before remote fetch.
	643	const cached_tarball_sub_path = try std.fmt.allocPrint(arena, "p/{s}.tar.gz", .{expected_hash.toSlice()});
	644	const cached_tarball_path: Cache.Path = .{
	645	.root_dir = job_queue.global_cache,
	646	.sub_path = cached_tarball_sub_path,
	647	};
	648	if (cached_tarball_path.root_dir.handle.openFile(io, cached_tarball_path.sub_path, .{})) \|file\| {
	649	log.debug("found global cached tarball {f}", .{cached_tarball_path});
	650	var resource: Resource = .{ .file = file.reader(io, &resource_buffer) };
	651	return f.runResource(cached_tarball_sub_path, &resource, remote.hash, true);
	652	} else \|err\| switch (err) {
	653	error.FileNotFound => log.debug("FileNotFound: {f}", .{cached_tarball_path}),
	654	error.Canceled => \|e\| return e,
	655	else => \|e\| {
	656	try eb.addRootErrorMessage(.{
	657	.msg = try eb.printString("unable to open globally cached package {f}: {t}", .{
	658	cached_tarball_path, e,
	659	}),
	660	});
	661	return error.FetchFailed;
	662	},
	663	}
	664
	665	switch (f.lazy_status) {
	666	.eager => {},
	667	.available => if (!job_queue.unlazy_set.contains(expected_hash)) {
	668	f.lazy_status = .unavailable;
	669	return;
	670	},
	671	.unavailable => unreachable,
	672	}
	673	} else if (job_queue.read_only) {
	674	try eb.addRootErrorMessage(.{
	675	.msg = try eb.addString("dependency is missing hash field"),
	676	.src_loc = try f.srcLoc(f.location_tok),
	677	});
	678	return error.FetchFailed;
	679	}
	680
	681	// Fetch and unpack the remote into a temporary directory.
	682	const uri = std.Uri.parse(remote.url) catch \|err\| return f.fail(
	683	f.location_tok,
	684	try eb.printString("invalid URI: {t}", .{err}),
	685	);
	686	var resource: Resource = undefined;
	687	try f.initResource(uri, &resource, &resource_buffer);
	688	return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, false);
	689	}
	690
	691	pub fn deinit(f: *Fetch) void {
	692	f.error_bundle.deinit();
	693	f.arena.deinit();
	694	}
	695
	696	/// Consumes `resource`, even if an error is returned.
	697	fn runResource(
	698	f: *Fetch,
	699	uri_path: []const u8,
	700	resource: *Resource,
	701	remote_hash: ?Package.Hash,
	702	disable_recompress: bool,
	703	) RunError!void {
	704	const job_queue = f.job_queue;
	705	assert(!job_queue.read_only);
	706
	707	const io = job_queue.io;
	708	defer resource.deinit(io);
	709
	710	const arena = f.arena.allocator();
	711	const eb = &f.error_bundle;
	712	const rand_int = r: {
	713	var x: u64 = undefined;
	714	io.random(@ptrCast(&x));
	715	break :r x;
	716	};
	717	const tmp_dir_sub_path = ".tmp-" ++ std.fmt.hex(rand_int);
	718	const tmp_tmp_dir_sub_path = "tmp/" ++ tmp_dir_sub_path;
	719	const tmp_directory_path: Cache.Path = if (job_queue.local_storage) \|ls\|
	720	try ls.pkg_root.join(arena, tmp_dir_sub_path)
	721	else
	722	.{
	723	.root_dir = job_queue.global_cache,
	724	.sub_path = tmp_tmp_dir_sub_path,
	725	};
	726
	727	const package_sub_path = blk: {
	728	var tmp_directory: Cache.Directory = .{
	729	.path = tmp_directory_path.sub_path,
	730	.handle = handle: {
	731	const dir = tmp_directory_path.root_dir.handle.createDirPathOpen(io, tmp_directory_path.sub_path, .{
	732	.open_options = .{ .iterate = true },
	733	}) catch \|err\| {
	734	try eb.addRootErrorMessage(.{
	735	.msg = try eb.printString("unable to create temporary directory '{f}': {t}", .{
	736	tmp_directory_path, err,
	737	}),
	738	});
	739	return error.FetchFailed;
	740	};
	741	break :handle dir;
	742	},
	743	};
	744	defer tmp_directory.handle.close(io);
	745
	746	// Fetch and unpack a resource into a temporary directory.
	747	var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory);
	748
	749	const pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir };
	750
	751	// Load, parse, and validate the unpacked build.zig.zon file. It is allowed
	752	// for the file to be missing, in which case this fetched package is
	753	// considered to be a "naked" package.
	754	try loadManifest(f, pkg_path);
	755
	756	const filter: Filter = .{
	757	.include_paths = if (f.have_manifest) f.manifest.paths else .{},
	758	};
	759
	760	// Ignore errors that were excluded by manifest, such as failure to
	761	// create symlinks that weren't supposed to be included anyway.
	762	try unpack_result.validate(f, filter);
	763
	764	// Apply the manifest's inclusion rules to the temporary directory by
	765	// deleting excluded files.
	766	// Empty directories have already been omitted by `unpackResource`.
	767	// Compute the package hash based on the remaining files in the temporary
	768	// directory.
	769	f.computed_hash = try computeHash(f, pkg_path, filter);
	770
	771	if (unpack_result.root_dir.len > 0)
	772	break :blk try tmp_directory_path.join(arena, unpack_result.root_dir);
	773
	774	break :blk tmp_directory_path;
	775	};
	776
	777	const computed_package_hash = computedPackageHash(f);
	778
	779	// Rename the temporary directory into the local zig package directory. If
	780	// the hash already exists, delete the temporary directory and leave the
	781	// zig package directory untouched as it may be in use. This is done even
	782	// if the hash is invalid, in case the package with the different hash is
	783	// used in the future.
	784	if (job_queue.local_storage) \|ls\| {
	785	f.package_root = try ls.pkg_root.join(arena, computed_package_hash.toSlice());
	786	renameTmpIntoCache(io, package_sub_path, f.package_root) catch \|err\| {
	787	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
	788	"failed renaming temporary directory {f} into package cache directory {f}: {t}",
	789	.{ package_sub_path, f.package_root, err },
	790	) });
	791	return error.FetchFailed;
	792	};
	793	} else {
	794	f.package_root = tmp_directory_path;
	795	}
	796	f.remote_package_root = f.package_root;
	797
	798	if (!disable_recompress) {
	799	// Spin off a task to recompress the tarball, with filtered files deleted, into
	800	// the global cache.
	801	job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash, f.package_root });
	802	}
	803
	804	// Remove temporary directory root if not already renamed to global cache.
	805	if (!package_sub_path.eql(tmp_directory_path)) {
	806	tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch \|err\| switch (err) {
	807	error.Canceled => \|e\| return e,
	808	else => \|e\| log.warn("failed deleting temporary directory {f}: {t}", .{ tmp_directory_path, e }),
	809	};
	810	}
	811
	812	// Validate the computed hash against the expected hash. If invalid, this
	813	// job is done.
	814
	815	if (remote_hash) \|declared_hash\| {
	816	const hash_tok = f.hash_tok.unwrap().?;
	817	if (!computed_package_hash.eql(&declared_hash)) {
	818	return f.fail(hash_tok, try eb.printString(
	819	"hash mismatch: manifest declares {s} but the fetched package has {s}",
	820	.{ declared_hash.toSlice(), computed_package_hash.toSlice() },
	821	));
	822	}
	823	} else if (!f.omit_missing_hash_error) {
	824	const notes_len = 1;
	825	try eb.addRootErrorMessage(.{
	826	.msg = try eb.addString("dependency is missing hash field"),
	827	.src_loc = try f.srcLoc(f.location_tok),
	828	.notes_len = notes_len,
	829	});
	830	const notes_start = try eb.reserveNotes(notes_len);
	831	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
	832	.msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}),
	833	}));
	834	return error.FetchFailed;
	835	}
	836
	837	// Spawn a new fetch job for each dependency in the manifest file. Use
	838	// a mutex and a hash map so that redundant jobs do not get queued up.
	839	if (!job_queue.recursive) return;
	840	return queueJobsForDeps(f);
	841	}
	842
	843	pub fn computedPackageHash(f: *const Fetch) Package.Hash {
	844	const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32);
	845	if (f.have_manifest) {
	846	const man = &f.manifest;
	847	var version_buffer: [32]u8 = undefined;
	848	const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{f}", .{man.version}) catch &version_buffer;
	849	return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size);
	850	}
	851	// In the future build.zig.zon fields will be added to allow overriding these values
	852	// for naked tarballs.
	853	return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size);
	854	}
	855
	856	/// `computeHash` gets a free check for the existence of `build.zig`, but when
	857	/// not computing a hash, we need to do a syscall to check for it.
	858	fn checkBuildFileExistence(f: *Fetch) RunError!void {
	859	const io = f.job_queue.io;
	860	const eb = &f.error_bundle;
	861	if (f.package_root.access(io, Package.build_zig_basename, .{})) \|_\| {
	862	f.has_build_zig = true;
	863	} else \|err\| switch (err) {
	864	error.FileNotFound => {},
	865	else => \|e\| {
	866	try eb.addRootErrorMessage(.{
	867	.msg = try eb.printString("unable to access '{f}{s}': {t}", .{
	868	f.package_root, Package.build_zig_basename, e,
	869	}),
	870	});
	871	return error.FetchFailed;
	872	},
	873	}
	874	}
	875
	876	/// This function populates `f.manifest` or leaves it `null`.
	877	fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {
	878	const io = f.job_queue.io;
	879	const eb = &f.error_bundle;
	880	const arena = f.arena.allocator();
	881	const manifest_path = try pkg_root.join(arena, Manifest.basename);
	882
	883	Manifest.load(
	884	io,
	885	arena,
	886	manifest_path,
	887	&f.manifest_ast,
	888	eb,
	889	&f.manifest,
	890	f.allow_missing_paths_field,
	891	) catch \|err\| switch (err) {
	892	error.FileNotFound => return,
	893	error.Canceled => \|e\| return e,
	894	error.ErrorsBundled => return error.FetchFailed,
	895	else => \|e\| {
	896	try eb.addRootErrorMessage(.{
	897	.msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ manifest_path, e }),
	898	});
	899	return error.FetchFailed;
	900	},
	901	};
	902	f.have_manifest = true;
	903	}
	904
	905	fn queueJobsForDeps(f: *Fetch) RunError!void {
	906	const io = f.job_queue.io;
	907
	908	assert(f.job_queue.recursive);
	909
	910	// If the package does not have a build.zig.zon file then there are no dependencies.
	911	if (!f.have_manifest) return;
	912	const manifest = &f.manifest;
	913
	914	const new_fetches, const prog_names = nf: {
	915	const parent_arena = f.arena.allocator();
	916	const gpa = f.arena.child_allocator;
	917	const cache_root = f.job_queue.global_cache;
	918	const dep_names = manifest.dependencies.keys();
	919	const deps = manifest.dependencies.values();
	920	// Grab the new tasks into a temporary buffer so we can unlock that mutex
	921	// as fast as possible.
	922	// This overallocates any fetches that get skipped by the `continue` in the
	923	// loop below.
	924	const new_fetches = try parent_arena.alloc(Fetch, deps.len);
	925	const prog_names = try parent_arena.alloc([]const u8, deps.len);
	926	var new_fetch_index: usize = 0;
	927
	928	try f.job_queue.mutex.lock(io);
	929	defer f.job_queue.mutex.unlock(io);
	930
	931	try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len);
	932	try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len));
	933
	934	// There are four cases here:
	935	// * Correct hash is provided by manifest.
	936	// - Hash map already has the entry, no need to add it again.
	937	// * Incorrect hash is provided by manifest.
	938	// - Hash mismatch error emitted; `queueJobsForDeps` is not called.
	939	// * Hash is not provided by manifest.
	940	// - Hash missing error emitted; `queueJobsForDeps` is not called.
	941	// * path-based location is used without a hash.
	942	// - Hash is added to the table based on the path alone before
	943	// calling run(); no need to add it again.
	944	//
	945	// If we add a dep as lazy and then later try to add the same dep as eager,
	946	// eagerness takes precedence and the existing entry is updated and re-scheduled
	947	// for fetching.
	948
	949	for (dep_names, deps) \|dep_name, dep\| {
	950	var promoted_existing_to_eager = false;
	951	const new_fetch = &new_fetches[new_fetch_index];
	952	const location: Location = switch (dep.location) {
	953	.url => \|url\| .{
	954	.remote = .{
	955	.url = url,
	956	.hash = h: {
	957	const h = dep.hash orelse break :h null;
	958	const pkg_hash: Package.Hash = .fromSlice(h);
	959	if (h.len == 0) break :h pkg_hash;
	960	const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
	961	if (gop.found_existing) {
	962	if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) {
	963	gop.value_ptr.*.lazy_status = .eager;
	964	promoted_existing_to_eager = true;
	965	} else {
	966	continue;
	967	}
	968	}
	969	gop.value_ptr.* = new_fetch;
	970	break :h pkg_hash;
	971	},
	972	},
	973	},
	974	.path => \|rel_path\| l: {
	975	// This might produce an invalid path, which is checked for
	976	// at the beginning of run().
	977	const new_root = try f.package_root.resolvePosix(parent_arena, rel_path);
	978	const pkg_hash = relativePathDigest(new_root, cache_root);
	979	const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
	980	if (gop.found_existing) {
	981	if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) {
	982	gop.value_ptr.*.lazy_status = .eager;
	983	promoted_existing_to_eager = true;
	984	} else {
	985	continue;
	986	}
	987	}
	988	gop.value_ptr.* = new_fetch;
	989	break :l .{ .relative_path = new_root };
	990	},
	991	};
	992	prog_names[new_fetch_index] = dep_name;
	993	new_fetch_index += 1;
	994	if (!promoted_existing_to_eager) {
	995	f.job_queue.all_fetches.appendAssumeCapacity(new_fetch);
	996	}
	997	new_fetch.* = .{
	998	.arena = std.heap.ArenaAllocator.init(gpa),
	999	.location = location,
	1000	.location_tok = dep.location_tok,
	1001	.hash_tok = dep.hash_tok,
	1002	.name_tok = dep.name_tok,
	1003	.lazy_status = switch (f.job_queue.mode) {
	1004	.needed => if (dep.lazy) .available else .eager,
	1005	.all => .eager,
	1006	},
	1007	.parent_package_root = f.package_root,
	1008	.remote_package_root = f.remote_package_root,
	1009	.parent_manifest_ast = &f.manifest_ast,
	1010	.prog_node = f.prog_node,
	1011	.job_queue = f.job_queue,
	1012	.omit_missing_hash_error = false,
	1013	.allow_missing_paths_field = true,
	1014	.use_latest_commit = false,
	1015
	1016	.package_root = undefined,
	1017	.error_bundle = undefined,
	1018	.manifest = undefined,
	1019	.manifest_ast = undefined,
	1020	.have_manifest = false,
	1021	.computed_hash = undefined,
	1022	.has_build_zig = false,
	1023	.oom_flag = false,
	1024	.latest_commit = null,
	1025
	1026	.cli_module = null,
	1027	};
	1028	}
	1029
	1030	f.prog_node.increaseEstimatedTotalItems(new_fetch_index);
	1031
	1032	break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] };
	1033	};
	1034
	1035	// Now it's time to dispatch tasks.
	1036	for (new_fetches, prog_names) \|*new_fetch, prog_name\| {
	1037	f.job_queue.group.async(io, workerRun, .{ new_fetch, prog_name });
	1038	}
	1039	}
	1040
	1041	pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash {
	1042	return .initPath(pkg_root.sub_path, pkg_root.root_dir.eql(cache_root));
	1043	}
	1044
	1045	pub fn workerRun(f: *Fetch, prog_name: []const u8) Io.Cancelable!void {
	1046	const prog_node = f.prog_node.start(prog_name, 0);
	1047	defer prog_node.end();
	1048
	1049	run(f) catch \|err\| switch (err) {
	1050	error.OutOfMemory => f.oom_flag = true,
	1051	error.Canceled => \|e\| return e,
	1052	error.FetchFailed => {
	1053	// Nothing to do because the errors are already reported in `error_bundle`,
	1054	// and a reference is kept to the `Fetch` task inside `all_fetches`.
	1055	},
	1056	};
	1057	}
	1058
	1059	fn srcLoc(
	1060	f: *Fetch,
	1061	tok: std.zig.Ast.TokenIndex,
	1062	) Allocator.Error!ErrorBundle.SourceLocationIndex {
	1063	const ast = f.parent_manifest_ast orelse return .none;
	1064	const eb = &f.error_bundle;
	1065	const start_loc = ast.tokenLocation(0, tok);
	1066	const src_path = try eb.printString("{f}" ++ fs.path.sep_str ++ Manifest.basename, .{f.parent_package_root});
	1067	const msg_off = 0;
	1068	return eb.addSourceLocation(.{
	1069	.src_path = src_path,
	1070	.span_start = ast.tokenStart(tok),
	1071	.span_end = @intCast(ast.tokenStart(tok) + ast.tokenSlice(tok).len),
	1072	.span_main = ast.tokenStart(tok) + msg_off,
	1073	.line = @intCast(start_loc.line),
	1074	.column = @intCast(start_loc.column),
	1075	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
	1076	});
	1077	}
	1078
	1079	fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError {
	1080	const eb = &f.error_bundle;
	1081	try eb.addRootErrorMessage(.{
	1082	.msg = msg_str,
	1083	.src_loc = try f.srcLoc(msg_tok),
	1084	});
	1085	return error.FetchFailed;
	1086	}
	1087
	1088	const Resource = union(enum) {
	1089	file: Io.File.Reader,
	1090	http_request: HttpRequest,
	1091	git: Git,
	1092	dir: Io.Dir,
	1093
	1094	const Git = struct {
	1095	session: git.Session,
	1096	fetch_stream: git.Session.FetchStream,
	1097	want_oid: git.Oid,
	1098	};
	1099
	1100	const HttpRequest = struct {
	1101	request: std.http.Client.Request,
	1102	response: std.http.Client.Response,
	1103	transfer_buffer: []u8,
	1104	decompress: std.http.Decompress,
	1105	decompress_buffer: []u8,
	1106	};
	1107
	1108	fn deinit(resource: *Resource, io: Io) void {
	1109	switch (resource.*) {
	1110	.file => \|*file_reader\| file_reader.file.close(io),
	1111	.http_request => \|*http_request\| http_request.request.deinit(),
	1112	.git => \|*git_resource\| {
	1113	git_resource.fetch_stream.deinit();
	1114	},
	1115	.dir => \|*dir\| dir.close(io),
	1116	}
	1117	resource.* = undefined;
	1118	}
	1119
	1120	fn reader(resource: Resource) Io.Reader {
	1121	return switch (resource.*) {
	1122	.file => \|*file_reader\| return &file_reader.interface,
	1123	.http_request => \|*http_request\| return http_request.response.readerDecompressing(
	1124	http_request.transfer_buffer,
	1125	&http_request.decompress,
	1126	http_request.decompress_buffer,
	1127	),
	1128	.git => \|*g\| return &g.fetch_stream.reader,
	1129	.dir => unreachable,
	1130	};
	1131	}
	1132	};
	1133
	1134	const FileType = enum {
	1135	tar,
	1136	@"tar.gz",
	1137	@"tar.xz",
	1138	@"tar.zst",
	1139	git_pack,
	1140	zip,
	1141
	1142	fn fromPath(file_path: []const u8) ?FileType {
	1143	if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar;
	1144	if (ascii.endsWithIgnoreCase(file_path, ".tgz")) return .@"tar.gz";
	1145	if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz";
	1146	if (ascii.endsWithIgnoreCase(file_path, ".txz")) return .@"tar.xz";
	1147	if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz";
	1148	if (ascii.endsWithIgnoreCase(file_path, ".tzst")) return .@"tar.zst";
	1149	if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst";
	1150	if (ascii.endsWithIgnoreCase(file_path, ".zip")) return .zip;
	1151	if (ascii.endsWithIgnoreCase(file_path, ".jar")) return .zip;
	1152	return null;
	1153	}
	1154
	1155	/// Parameter is a content-disposition header value.
	1156	fn fromContentDisposition(cd_header: []const u8) ?FileType {
	1157	const attach_end = ascii.findIgnoreCase(cd_header, "attachment;") orelse
	1158	return null;
	1159
	1160	var value_start = ascii.findIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse
	1161	return null;
	1162	value_start += "filename".len;
	1163	if (cd_header[value_start] == '*') {
	1164	value_start += 1;
	1165	}
	1166	if (cd_header[value_start] != '=') return null;
	1167	value_start += 1;
	1168
	1169	var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len;
	1170	if (cd_header[value_end - 1] == '\"') {
	1171	value_end -= 1;
	1172	}
	1173	return fromPath(cd_header[value_start..value_end]);
	1174	}
	1175
	1176	test fromContentDisposition {
	1177	try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42"));
	1178	try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\""));
	1179	try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\""));
	1180	try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\""));
	1181	try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz"));
	1182	try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\""));
	1183
	1184	try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null);
	1185	try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null);
	1186	try std.testing.expect(fromContentDisposition("attachment; size=42") == null);
	1187	try std.testing.expect(fromContentDisposition("inline; size=42") == null);
	1188	try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null);
	1189	try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null);
	1190	}
	1191	};
	1192
	1193	const init_resource_buffer_size = git.Packet.max_data_length;
	1194
	1195	fn initResource(f: Fetch, uri: std.Uri, resource: Resource, reader_buffer: []u8) RunError!void {
	1196	const io = f.job_queue.io;
	1197	const arena = f.arena.allocator();
	1198	const eb = &f.error_bundle;
	1199
	1200	if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
	1201	const path = try uri.path.toRawMaybeAlloc(arena);
	1202	const file = f.parent_package_root.openFile(io, path, .{}) catch \|err\| {
	1203	return f.fail(f.location_tok, try eb.printString("unable to open {f}/{s}: {t}", .{
	1204	f.parent_package_root, path, err,
	1205	}));
	1206	};
	1207	resource.* = .{ .file = file.reader(io, reader_buffer) };
	1208	return;
	1209	}
	1210
	1211	const http_client = f.job_queue.http_client;
	1212
	1213	if (ascii.eqlIgnoreCase(uri.scheme, "http") or
	1214	ascii.eqlIgnoreCase(uri.scheme, "https"))
	1215	{
	1216	resource.* = .{ .http_request = .{
	1217	.request = http_client.request(.GET, uri, .{}) catch \|err\|
	1218	return f.fail(f.location_tok, try eb.printString("server connection failed: {t}", .{err})),
	1219	.response = undefined,
	1220	.transfer_buffer = reader_buffer,
	1221	.decompress_buffer = &.{},
	1222	.decompress = undefined,
	1223	} };
	1224	const request = &resource.http_request.request;
	1225	errdefer request.deinit();
	1226
	1227	request.sendBodiless() catch \|err\|
	1228	return f.fail(f.location_tok, try eb.printString("HTTP request failed: {t}", .{err}));
	1229
	1230	var redirect_buffer: [8000]u8 = undefined;
	1231	const response = &resource.http_request.response;
	1232	response.* = request.receiveHead(&redirect_buffer) catch \|err\| switch (err) {
	1233	error.ReadFailed => {
	1234	return f.fail(f.location_tok, try eb.printString("HTTP response read failure: {t}", .{
	1235	request.connection.?.getReadError().?,
	1236	}));
	1237	},
	1238	else => \|e\| return f.fail(f.location_tok, try eb.printString("invalid HTTP response: {t}", .{e})),
	1239	};
	1240
	1241	if (response.head.status != .ok) return f.fail(f.location_tok, try eb.printString(
	1242	"bad HTTP response code: '{d} {s}'",
	1243	.{ response.head.status, response.head.status.phrase() orelse "" },
	1244	));
	1245
	1246	resource.http_request.decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
	1247	return;
	1248	}
	1249
	1250	if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or
	1251	ascii.eqlIgnoreCase(uri.scheme, "git+https"))
	1252	{
	1253	var transport_uri = uri;
	1254	transport_uri.scheme = uri.scheme["git+".len..];
	1255	var session = git.Session.init(arena, http_client, transport_uri, reader_buffer) catch \|err\| {
	1256	return f.fail(
	1257	f.location_tok,
	1258	try eb.printString("unable to discover remote git server capabilities: {t}", .{err}),
	1259	);
	1260	};
	1261
	1262	const want_oid = want_oid: {
	1263	const want_ref =
	1264	if (uri.fragment) \|fragment\| try fragment.toRawMaybeAlloc(arena) else "HEAD";
	1265	if (git.Oid.parseAny(want_ref)) \|oid\| break :want_oid oid else \|_\| {}
	1266
	1267	const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref});
	1268	const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref});
	1269
	1270	var ref_iterator: git.Session.RefIterator = undefined;
	1271	session.listRefs(&ref_iterator, .{
	1272	.ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },
	1273	.include_peeled = true,
	1274	.buffer = reader_buffer,
	1275	}) catch \|err\| return f.fail(f.location_tok, try eb.printString("unable to list refs: {t}", .{err}));
	1276	defer ref_iterator.deinit();
	1277	while (ref_iterator.next() catch \|err\| {
	1278	return f.fail(f.location_tok, try eb.printString(
	1279	"unable to iterate refs: {s}",
	1280	.{@errorName(err)},
	1281	));
	1282	}) \|ref\| {
	1283	if (std.mem.eql(u8, ref.name, want_ref) or
	1284	std.mem.eql(u8, ref.name, want_ref_head) or
	1285	std.mem.eql(u8, ref.name, want_ref_tag))
	1286	{
	1287	break :want_oid ref.peeled orelse ref.oid;
	1288	}
	1289	}
	1290	return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref}));
	1291	};
	1292	if (f.use_latest_commit) {
	1293	f.latest_commit = want_oid;
	1294	} else if (uri.fragment == null) {
	1295	const notes_len = 1;
	1296	try eb.addRootErrorMessage(.{
	1297	.msg = try eb.addString("url field is missing an explicit ref"),
	1298	.src_loc = try f.srcLoc(f.location_tok),
	1299	.notes_len = notes_len,
	1300	});
	1301	const notes_start = try eb.reserveNotes(notes_len);
	1302	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
	1303	.msg = try eb.printString("try .url = \"{f}#{f}\",", .{
	1304	uri.fmt(.{ .scheme = true, .authority = true, .path = true }),
	1305	want_oid,
	1306	}),
	1307	}));
	1308	return error.FetchFailed;
	1309	}
	1310
	1311	var want_oid_buf: [git.Oid.max_formatted_length]u8 = undefined;
	1312	_ = std.fmt.bufPrint(&want_oid_buf, "{f}", .{want_oid}) catch unreachable;
	1313	resource.* = .{ .git = .{
	1314	.session = session,
	1315	.fetch_stream = undefined,
	1316	.want_oid = want_oid,
	1317	} };
	1318	const fetch_stream = &resource.git.fetch_stream;
	1319	session.fetch(fetch_stream, &.{&want_oid_buf}, reader_buffer) catch \|err\| {
	1320	return f.fail(f.location_tok, try eb.printString("unable to create fetch stream: {t}", .{err}));
	1321	};
	1322	errdefer fetch_stream.deinit(fetch_stream);
	1323
	1324	return;
	1325	}
	1326
	1327	return f.fail(f.location_tok, try eb.printString("unsupported URL scheme: {s}", .{uri.scheme}));
	1328	}
	1329
	1330	fn unpackResource(
	1331	f: *Fetch,
	1332	resource: *Resource,
	1333	uri_path: []const u8,
	1334	tmp_directory: Cache.Directory,
	1335	) RunError!UnpackResult {
	1336	const eb = &f.error_bundle;
	1337	const file_type = switch (resource.*) {
	1338	.file => FileType.fromPath(uri_path) orelse
	1339	return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})),
	1340
	1341	.http_request => \|*http_request\| ft: {
	1342	const head = &http_request.response.head;
	1343
	1344	// Content-Type takes first precedence.
	1345	const content_type = head.content_type orelse
	1346	return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header"));
	1347
	1348	// Extract the MIME type, ignoring charset and boundary directives
	1349	const mime_type_end = std.mem.indexOf(u8, content_type, ";") orelse content_type.len;
	1350	const mime_type = content_type[0..mime_type_end];
	1351
	1352	if (ascii.eqlIgnoreCase(mime_type, "application/x-tar"))
	1353	break :ft .tar;
	1354
	1355	if (ascii.eqlIgnoreCase(mime_type, "application/gzip") or
	1356	ascii.eqlIgnoreCase(mime_type, "application/x-gzip") or
	1357	ascii.eqlIgnoreCase(mime_type, "application/tar+gzip") or
	1358	ascii.eqlIgnoreCase(mime_type, "application/x-tar-gz") or
	1359	ascii.eqlIgnoreCase(mime_type, "application/x-gtar-compressed"))
	1360	{
	1361	break :ft .@"tar.gz";
	1362	}
	1363
	1364	if (ascii.eqlIgnoreCase(mime_type, "application/x-xz"))
	1365	break :ft .@"tar.xz";
	1366
	1367	if (ascii.eqlIgnoreCase(mime_type, "application/zstd"))
	1368	break :ft .@"tar.zst";
	1369
	1370	if (ascii.eqlIgnoreCase(mime_type, "application/zip") or
	1371	ascii.eqlIgnoreCase(mime_type, "application/x-zip-compressed") or
	1372	ascii.eqlIgnoreCase(mime_type, "application/java-archive"))
	1373	{
	1374	break :ft .zip;
	1375	}
	1376
	1377	if (!ascii.eqlIgnoreCase(mime_type, "application/octet-stream") and
	1378	!ascii.eqlIgnoreCase(mime_type, "application/x-compressed"))
	1379	{
	1380	return f.fail(f.location_tok, try eb.printString(
	1381	"unrecognized 'Content-Type' header: '{s}'",
	1382	.{content_type},
	1383	));
	1384	}
	1385
	1386	// Next, the filename from 'content-disposition: attachment' takes precedence.
	1387	if (head.content_disposition) \|cd_header\| {
	1388	break :ft FileType.fromContentDisposition(cd_header) orelse {
	1389	return f.fail(f.location_tok, try eb.printString(
	1390	"unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",
	1391	.{cd_header},
	1392	));
	1393	};
	1394	}
	1395
	1396	// Finally, the path from the URI is used.
	1397	break :ft FileType.fromPath(uri_path) orelse {
	1398	return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path}));
	1399	};
	1400	},
	1401
	1402	.git => .git_pack,
	1403
	1404	.dir => \|dir\| {
	1405	f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch \|err\| {
	1406	return f.fail(f.location_tok, try eb.printString("unable to copy directory '{s}': {t}", .{
	1407	uri_path, err,
	1408	}));
	1409	};
	1410	return .{};
	1411	},
	1412	};
	1413
	1414	switch (file_type) {
	1415	.tar => {
	1416	return unpackTarball(f, tmp_directory.handle, resource.reader());
	1417	},
	1418	.@"tar.gz" => {
	1419	var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
	1420	var decompress: std.compress.flate.Decompress = .init(resource.reader(), .gzip, &flate_buffer);
	1421	return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
	1422	},
	1423	.@"tar.xz" => {
	1424	const gpa = f.arena.child_allocator;
	1425	var decompress = std.compress.xz.Decompress.init(resource.reader(), gpa, &.{}) catch \|err\|
	1426	return f.fail(f.location_tok, try eb.printString("unable to decompress tarball: {t}", .{err}));
	1427	defer decompress.deinit();
	1428	return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
	1429	},
	1430	.@"tar.zst" => {
	1431	const window_len = std.compress.zstd.default_window_len;
	1432	const window_buffer = try f.arena.allocator().alloc(u8, window_len + std.compress.zstd.block_size_max);
	1433	var decompress: std.compress.zstd.Decompress = .init(resource.reader(), window_buffer, .{
	1434	.verify_checksum = false,
	1435	.window_len = window_len,
	1436	});
	1437	return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
	1438	},
	1439	.git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch \|err\| switch (err) {
	1440	error.FetchFailed, error.OutOfMemory => \|e\| return e,
	1441	else => \|e\| return f.fail(f.location_tok, try eb.printString("unable to unpack git files: {t}", .{e})),
	1442	},
	1443	.zip => return unzip(f, tmp_directory.handle, resource.reader()) catch \|err\| switch (err) {
	1444	error.ReadFailed => return f.fail(f.location_tok, try eb.printString(
	1445	"failed reading resource: {t}",
	1446	.{err},
	1447	)),
	1448	else => \|e\| return e,
	1449	},
	1450	}
	1451	}
	1452
	1453	fn unpackTarball(f: Fetch, out_dir: Io.Dir, reader: Io.Reader) RunError!UnpackResult {
	1454	const eb = &f.error_bundle;
	1455	const arena = f.arena.allocator();
	1456	const io = f.job_queue.io;
	1457
	1458	var diagnostics: std.tar.Diagnostics = .{ .allocator = arena };
	1459
	1460	std.tar.pipeToFileSystem(io, out_dir, reader, .{
	1461	.diagnostics = &diagnostics,
	1462	.strip_components = 0,
	1463	.mode_mode = .ignore,
	1464	.exclude_empty_directories = true,
	1465	}) catch \|err\| return f.fail(
	1466	f.location_tok,
	1467	try eb.printString("unable to unpack tarball to temporary directory: {t}", .{err}),
	1468	);
	1469
	1470	var res: UnpackResult = .{ .root_dir = diagnostics.root_dir };
	1471	if (diagnostics.errors.items.len > 0) {
	1472	try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack tarball");
	1473	for (diagnostics.errors.items) \|item\| {
	1474	switch (item) {
	1475	.unable_to_create_file => \|i\| res.unableToCreateFile(stripRoot(i.file_name, res.root_dir), i.code),
	1476	.unable_to_create_sym_link => \|i\| res.unableToCreateSymLink(stripRoot(i.file_name, res.root_dir), i.link_name, i.code),
	1477	.unsupported_file_type => \|i\| res.unsupportedFileType(stripRoot(i.file_name, res.root_dir), @intFromEnum(i.file_type)),
	1478	.components_outside_stripped_prefix => unreachable, // unreachable with strip_components = 0
	1479	}
	1480	}
	1481	}
	1482	return res;
	1483	}
	1484
	1485	fn unzip(
	1486	f: *Fetch,
	1487	out_dir: Io.Dir,
	1488	reader: *Io.Reader,
	1489	) error{ ReadFailed, OutOfMemory, Canceled, FetchFailed }!UnpackResult {
	1490	// We write the entire contents to a file first because zip files
	1491	// must be processed back to front and they could be too large to
	1492	// load into memory.
	1493
	1494	const io = f.job_queue.io;
	1495	const cache_root = f.job_queue.global_cache;
	1496	const prefix = "tmp/";
	1497	const suffix = ".zip";
	1498	const eb = &f.error_bundle;
	1499	const random_len = @sizeOf(u64) * 2;
	1500
	1501	var zip_path: [prefix.len + random_len + suffix.len]u8 = undefined;
	1502	zip_path[0..prefix.len].* = prefix.*;
	1503	zip_path[prefix.len + random_len ..].* = suffix.*;
	1504
	1505	var zip_file = while (true) {
	1506	const random_integer = r: {
	1507	var x: u64 = undefined;
	1508	io.random(@ptrCast(&x));
	1509	break :r x;
	1510	};
	1511	zip_path[prefix.len..][0..random_len].* = std.fmt.hex(random_integer);
	1512
	1513	break cache_root.handle.createFile(io, &zip_path, .{
	1514	.exclusive = true,
	1515	.read = true,
	1516	}) catch \|err\| switch (err) {
	1517	error.PathAlreadyExists => continue,
	1518	error.FileNotFound => {
	1519	cache_root.handle.createDir(io, prefix, .default_dir) catch \|dir_err\| switch (dir_err) {
	1520	error.Canceled => \|e\| return e,
	1521	// error.PathAlreadyExists is considered a failure here because
	1522	// it implies that the prefix is not a directory.
	1523	else => \|e\| return f.fail(
	1524	f.location_tok,
	1525	try eb.printString("failed to create temporary directory: {t}", .{e}),
	1526	),
	1527	};
	1528	continue;
	1529	},
	1530	error.Canceled => \|e\| return e,
	1531	else => \|e\| return f.fail(
	1532	f.location_tok,
	1533	try eb.printString("failed to create temporary zip file: {t}", .{e}),
	1534	),
	1535	};
	1536	};
	1537	defer zip_file.close(io);
	1538	var zip_file_buffer: [4096]u8 = undefined;
	1539	var zip_file_reader = b: {
	1540	var zip_file_writer = zip_file.writer(io, &zip_file_buffer);
	1541
	1542	_ = reader.streamRemaining(&zip_file_writer.interface) catch \|err\| switch (err) {
	1543	error.ReadFailed => \|e\| return e,
	1544	error.WriteFailed => return f.fail(
	1545	f.location_tok,
	1546	try eb.printString("failed writing temporary zip file: {t}", .{err}),
	1547	),
	1548	};
	1549	zip_file_writer.interface.flush() catch \|err\| return f.fail(
	1550	f.location_tok,
	1551	try eb.printString("failed writing temporary zip file: {t}", .{err}),
	1552	);
	1553	break :b zip_file_writer.moveToReader();
	1554	};
	1555
	1556	var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() };
	1557	// no need to deinit since we are using an arena allocator
	1558
	1559	zip_file_reader.seekTo(0) catch \|err\|
	1560	return f.fail(f.location_tok, try eb.printString("failed to seek temporary zip file: {t}", .{err}));
	1561	std.zip.extract(out_dir, &zip_file_reader, .{
	1562	.allow_backslashes = true,
	1563	.diagnostics = &diagnostics,
	1564	}) catch \|err\| return f.fail(f.location_tok, try eb.printString("zip extract failed: {t}", .{err}));
	1565
	1566	cache_root.handle.deleteFile(io, &zip_path) catch \|err\|
	1567	return f.fail(f.location_tok, try eb.printString("delete temporary zip failed: {t}", .{err}));
	1568
	1569	return .{ .root_dir = diagnostics.root_dir };
	1570	}
	1571
	1572	fn unpackGitPack(f: Fetch, out_dir: Io.Dir, resource: Resource.Git) anyerror!UnpackResult {
	1573	const io = f.job_queue.io;
	1574	const arena = f.arena.allocator();
	1575	// TODO don't try to get a gpa from an arena. expose this dependency higher up
	1576	// because the backing of arena could be page allocator
	1577	const gpa = f.arena.child_allocator;
	1578	const object_format: git.Oid.Format = resource.want_oid;
	1579
	1580	var res: UnpackResult = .{};
	1581	// The .git directory is used to store the packfile and associated index, but
	1582	// we do not attempt to replicate the exact structure of a real .git
	1583	// directory, since that isn't relevant for fetching a package.
	1584	{
	1585	var pack_dir = try out_dir.createDirPathOpen(io, ".git", .{});
	1586	defer pack_dir.close(io);
	1587	var pack_file = try pack_dir.createFile(io, "pkg.pack", .{ .read = true });
	1588	defer pack_file.close(io);
	1589	var pack_file_buffer: [4096]u8 = undefined;
	1590	var pack_file_reader = b: {
	1591	var pack_file_writer = pack_file.writer(io, &pack_file_buffer);
	1592	const fetch_reader = &resource.fetch_stream.reader;
	1593	_ = try fetch_reader.streamRemaining(&pack_file_writer.interface);
	1594	try pack_file_writer.interface.flush();
	1595	break :b pack_file_writer.moveToReader();
	1596	};
	1597
	1598	var index_file = try pack_dir.createFile(io, "pkg.idx", .{ .read = true });
	1599	defer index_file.close(io);
	1600	var index_file_buffer: [2000]u8 = undefined;
	1601	var index_file_writer = index_file.writer(io, &index_file_buffer);
	1602	{
	1603	const index_prog_node = f.prog_node.start("Index pack", 0);
	1604	defer index_prog_node.end();
	1605	try git.indexPack(gpa, object_format, &pack_file_reader, &index_file_writer);
	1606	}
	1607
	1608	{
	1609	var index_file_reader = index_file.reader(io, &index_file_buffer);
	1610	const checkout_prog_node = f.prog_node.start("Checkout", 0);
	1611	defer checkout_prog_node.end();
	1612	var repository: git.Repository = undefined;
	1613	try repository.init(gpa, object_format, &pack_file_reader, &index_file_reader);
	1614	defer repository.deinit();
	1615	var diagnostics: git.Diagnostics = .{ .allocator = arena };
	1616	try repository.checkout(io, out_dir, resource.want_oid, &diagnostics);
	1617
	1618	if (diagnostics.errors.items.len > 0) {
	1619	try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile");
	1620	for (diagnostics.errors.items) \|item\| {
	1621	switch (item) {
	1622	.unable_to_create_file => \|i\| res.unableToCreateFile(i.file_name, i.code),
	1623	.unable_to_create_sym_link => \|i\| res.unableToCreateSymLink(i.file_name, i.link_name, i.code),
	1624	}
	1625	}
	1626	}
	1627	}
	1628	}
	1629
	1630	try out_dir.deleteTree(io, ".git");
	1631	return res;
	1632	}
	1633
	1634	fn recursiveDirectoryCopy(f: *Fetch, dir: Io.Dir, tmp_dir: Io.Dir) anyerror!void {
	1635	const gpa = f.arena.child_allocator;
	1636	const io = f.job_queue.io;
	1637	// Recursive directory copy.
	1638	var it = try dir.walk(gpa);
	1639	defer it.deinit();
	1640	while (try it.next(io)) \|entry\| {
	1641	switch (entry.kind) {
	1642	.directory => {}, // omit empty directories
	1643	.file => {
	1644	dir.copyFile(entry.path, tmp_dir, entry.path, io, .{}) catch \|err\| switch (err) {
	1645	error.FileNotFound => {
	1646	if (fs.path.dirname(entry.path)) \|dirname\| try tmp_dir.createDirPath(io, dirname);
	1647	try dir.copyFile(entry.path, tmp_dir, entry.path, io, .{});
	1648	},
	1649	else => \|e\| return e,
	1650	};
	1651	},
	1652	.sym_link => {
	1653	var buf: [fs.max_path_bytes]u8 = undefined;
	1654	const link_name = buf[0..try dir.readLink(io, entry.path, &buf)];
	1655	// TODO: if this would create a symlink to outside
	1656	// the destination directory, fail with an error instead.
	1657	tmp_dir.symLink(io, link_name, entry.path, .{}) catch \|err\| switch (err) {
	1658	error.FileNotFound => {
	1659	if (fs.path.dirname(entry.path)) \|dirname\| try tmp_dir.createDirPath(io, dirname);
	1660	try tmp_dir.symLink(io, link_name, entry.path, .{});
	1661	},
	1662	else => \|e\| return e,
	1663	};
	1664	},
	1665	else => return error.IllegalFileTypeInPackage,
	1666	}
	1667	}
	1668	}
	1669
	1670	pub fn renameTmpIntoCache(io: Io, tmp_path: Cache.Path, dest_path: Cache.Path) !void {
	1671	var handled_missing_dir = false;
	1672	while (true) {
	1673	Io.Dir.rename(
	1674	tmp_path.root_dir.handle,
	1675	tmp_path.sub_path,
	1676	dest_path.root_dir.handle,
	1677	dest_path.sub_path,
	1678	io,
	1679	) catch \|err\| switch (err) {
	1680	error.FileNotFound => {
	1681	if (handled_missing_dir) return err;
	1682	const parent_sub_path = Io.Dir.path.dirname(dest_path.sub_path).?;
	1683	dest_path.root_dir.handle.createDir(io, parent_sub_path, .default_dir) catch \|er\| switch (er) {
	1684	error.PathAlreadyExists => handled_missing_dir = true,
	1685	else => \|e\| return e,
	1686	};
	1687	continue;
	1688	},
	1689	error.DirNotEmpty, error.AccessDenied => {
	1690	// Package has been already downloaded and may already be in use on the system.
	1691	tmp_path.root_dir.handle.deleteTree(io, tmp_path.sub_path) catch \|er\| switch (er) {
	1692	error.Canceled => \|e\| return e,
	1693	// Garbage files leftover in zig-cache/tmp/ is, as they say
	1694	// on Star Trek, "operating within normal parameters".
	1695	else => \|e\| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_path, e }),
	1696	};
	1697	},
	1698	else => \|e\| return e,
	1699	};
	1700	break;
	1701	}
	1702	}
	1703
	1704	const ComputedHash = struct {
	1705	digest: Package.Hash.Digest,
	1706	total_size: u64,
	1707	};
	1708
	1709	/// Assumes that files not included in the package have already been filtered
	1710	/// prior to calling this function. This ensures that files not protected by
	1711	/// the hash are not present on the file system. Empty directories are *not
	1712	/// hashed* and must not be present on the file system when calling this
	1713	/// function.
	1714	fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash {
	1715	const io = f.job_queue.io;
	1716	// All the path name strings need to be in memory for sorting.
	1717	const arena = f.arena.allocator();
	1718	const gpa = f.arena.child_allocator;
	1719	const eb = &f.error_bundle;
	1720	const root_dir = pkg_path.root_dir.handle;
	1721
	1722	// Collect all files, recursively, then sort.
	1723	var all_files = std.array_list.Managed(*HashedFile).init(gpa);
	1724	defer all_files.deinit();
	1725
	1726	var deleted_files = std.array_list.Managed(*DeletedFile).init(gpa);
	1727	defer deleted_files.deinit();
	1728
	1729	// Track directories which had any files deleted from them so that empty directories
	1730	// can be deleted.
	1731	var sus_dirs: std.array_hash_map.String(void) = .empty;
	1732	defer sus_dirs.deinit(gpa);
	1733
	1734	var walker = try root_dir.walk(gpa);
	1735	defer walker.deinit();
	1736
	1737	// Total number of bytes of file contents included in the package.
	1738	var total_size: u64 = 0;
	1739
	1740	{
	1741	// The final hash will be a hash of each file hashed independently. This
	1742	// allows hashing in parallel.
	1743	var group: Io.Group = .init;
	1744	defer group.cancel(io);
	1745
	1746	while (walker.next(io) catch \|err\| {
	1747	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
	1748	"unable to walk temporary directory '{f}': {t}",
	1749	.{ pkg_path, err },
	1750	) });
	1751	return error.FetchFailed;
	1752	}) \|entry\| {
	1753	if (entry.kind == .directory) continue;
	1754
	1755	const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path);
	1756	if (!filter.includePath(entry_pkg_path)) {
	1757	// Delete instead of including in hash calculation.
	1758	const fs_path = try arena.dupe(u8, entry.path);
	1759
	1760	// Also track the parent directory in case it becomes empty.
	1761	if (fs.path.dirname(fs_path)) \|parent\|
	1762	try sus_dirs.put(gpa, parent, {});
	1763
	1764	const deleted_file = try arena.create(DeletedFile);
	1765	deleted_file.* = .{
	1766	.fs_path = fs_path,
	1767	.failure = undefined, // to be populated by the worker
	1768	};
	1769	group.async(io, workerDeleteFile, .{ io, root_dir, deleted_file });
	1770	try deleted_files.append(deleted_file);
	1771	continue;
	1772	}
	1773
	1774	const kind: HashedFile.Kind = switch (entry.kind) {
	1775	.directory => unreachable,
	1776	.file => .file,
	1777	.sym_link => .link,
	1778	else => return f.fail(f.location_tok, try eb.printString(
	1779	"package contains '{s}' which has illegal file type '{t}'",
	1780	.{ entry.path, entry.kind },
	1781	)),
	1782	};
	1783
	1784	if (std.mem.eql(u8, entry_pkg_path, Package.build_zig_basename))
	1785	f.has_build_zig = true;
	1786
	1787	const fs_path = try arena.dupe(u8, entry.path);
	1788	const hashed_file = try arena.create(HashedFile);
	1789	hashed_file.* = .{
	1790	.fs_path = fs_path,
	1791	.normalized_path = try normalizePathAlloc(arena, entry_pkg_path),
	1792	.kind = kind,
	1793	.hash = undefined, // to be populated by the worker
	1794	.failure = undefined, // to be populated by the worker
	1795	.size = undefined, // to be populated by the worker
	1796	};
	1797	group.async(io, workerHashFile, .{ io, root_dir, hashed_file });
	1798	try all_files.append(hashed_file);
	1799	}
	1800
	1801	try group.await(io);
	1802	}
	1803
	1804	{
	1805	// Sort by length, descending, so that child directories get removed first.
	1806	sus_dirs.sortUnstable(@as(struct {
	1807	keys: []const []const u8,
	1808	pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
	1809	return ctx.keys[b_index].len < ctx.keys[a_index].len;
	1810	}
	1811	}, .{ .keys = sus_dirs.keys() }));
	1812
	1813	// During this loop, more entries will be added, so we must loop by index.
	1814	var i: usize = 0;
	1815	while (i < sus_dirs.count()) : (i += 1) {
	1816	const sus_dir = sus_dirs.keys()[i];
	1817	root_dir.deleteDir(io, sus_dir) catch \|err\| switch (err) {
	1818	error.DirNotEmpty => continue,
	1819	error.FileNotFound => continue,
	1820	else => \|e\| {
	1821	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
	1822	"unable to delete empty directory '{s}': {s}",
	1823	.{ sus_dir, @errorName(e) },
	1824	) });
	1825	return error.FetchFailed;
	1826	},
	1827	};
	1828	if (fs.path.dirname(sus_dir)) \|parent\| {
	1829	try sus_dirs.put(gpa, parent, {});
	1830	}
	1831	}
	1832	}
	1833
	1834	std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
	1835
	1836	var hasher = Package.Hash.Algo.init(.{});
	1837	var any_failures = false;
	1838	for (all_files.items) \|hashed_file\| {
	1839	hashed_file.failure catch \|err\| {
	1840	any_failures = true;
	1841	try eb.addRootErrorMessage(.{
	1842	.msg = try eb.printString("unable to hash '{s}': {s}", .{
	1843	hashed_file.fs_path, @errorName(err),
	1844	}),
	1845	});
	1846	};
	1847	hasher.update(&hashed_file.hash);
	1848	total_size += hashed_file.size;
	1849	}
	1850	for (deleted_files.items) \|deleted_file\| {
	1851	deleted_file.failure catch \|err\| {
	1852	any_failures = true;
	1853	try eb.addRootErrorMessage(.{
	1854	.msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{
	1855	deleted_file.fs_path, @errorName(err),
	1856	}),
	1857	});
	1858	};
	1859	}
	1860
	1861	if (any_failures) return error.FetchFailed;
	1862
	1863	if (f.job_queue.debug_hash) {
	1864	assert(!f.job_queue.recursive);
	1865	// Print something to stdout that can be text diffed to figure out why
	1866	// the package hash is different.
	1867	dumpHashInfo(io, all_files.items) catch \|err\|
	1868	std.process.fatal("unable to write to stdout: {t}", .{err});
	1869	}
	1870
	1871	return .{
	1872	.digest = hasher.finalResult(),
	1873	.total_size = total_size,
	1874	};
	1875	}
	1876
	1877	fn dumpHashInfo(io: Io, all_files: []const *const HashedFile) !void {
	1878	var stdout_buffer: [1024]u8 = undefined;
	1879	var stdout_writer: Io.File.Writer = .initStreaming(.stdout(), io, &stdout_buffer);
	1880	dumpHashInfoWriter(&stdout_writer.interface, all_files) catch \|err\| switch (err) {
	1881	error.WriteFailed => return stdout_writer.err.?,
	1882	};
	1883	try stdout_writer.flush();
	1884	}
	1885
	1886	fn dumpHashInfoWriter(w: Io.Writer, all_files: []const const HashedFile) Io.Writer.Error!void {
	1887	for (all_files) \|hashed_file\| {
	1888	try w.print("{t}: {x}: {s}\n", .{ hashed_file.kind, &hashed_file.hash, hashed_file.normalized_path });
	1889	}
	1890	}
	1891
	1892	fn workerHashFile(io: Io, dir: Io.Dir, hashed_file: *HashedFile) void {
	1893	hashed_file.failure = hashFileFallible(io, dir, hashed_file);
	1894	}
	1895
	1896	fn workerDeleteFile(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) void {
	1897	deleted_file.failure = deleteFileFallible(io, dir, deleted_file);
	1898	}
	1899
	1900	fn hashFileFallible(io: Io, dir: Io.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
	1901	var buf: [8000]u8 = undefined;
	1902	var hasher = Package.Hash.Algo.init(.{});
	1903	hasher.update(hashed_file.normalized_path);
	1904	var file_size: u64 = 0;
	1905
	1906	switch (hashed_file.kind) {
	1907	.file => {
	1908	var file = try dir.openFile(io, hashed_file.fs_path, .{});
	1909	defer file.close(io);
	1910	// Hard-coded false executable bit: https://github.com/ziglang/zig/issues/17463
	1911	hasher.update(&.{ 0, 0 });
	1912	var file_header: FileHeader = .{};
	1913	while (true) {
	1914	const bytes_read = try file.readPositional(io, &.{&buf}, file_size);
	1915	if (bytes_read == 0) break;
	1916	file_size += bytes_read;
	1917	hasher.update(buf[0..bytes_read]);
	1918	file_header.update(buf[0..bytes_read]);
	1919	}
	1920	if (file_header.isExecutable()) {
	1921	try setExecutable(io, file);
	1922	}
	1923	},
	1924	.link => {
	1925	const link_name = buf[0..try dir.readLink(io, hashed_file.fs_path, &buf)];
	1926	if (fs.path.sep != canonical_sep) {
	1927	// Package hashes are intended to be consistent across
	1928	// platforms which means we must normalize path separators
	1929	// inside symlinks.
	1930	normalizePath(link_name);
	1931	}
	1932	hasher.update(link_name);
	1933	},
	1934	}
	1935	hasher.final(&hashed_file.hash);
	1936	hashed_file.size = file_size;
	1937	}
	1938
	1939	fn deleteFileFallible(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {
	1940	try dir.deleteFile(io, deleted_file.fs_path);
	1941	}
	1942
	1943	fn setExecutable(io: Io, file: Io.File) !void {
	1944	if (!Io.File.Permissions.has_executable_bit) return;
	1945	try file.setPermissions(io, .executable_file);
	1946	}
	1947
	1948	const DeletedFile = struct {
	1949	fs_path: []const u8,
	1950	failure: Error!void,
	1951
	1952	const Error =
	1953	Io.Dir.DeleteFileError \|\|
	1954	Io.Dir.DeleteDirError;
	1955	};
	1956
	1957	const HashedFile = struct {
	1958	fs_path: []const u8,
	1959	normalized_path: []const u8,
	1960	hash: Package.Hash.Digest,
	1961	failure: Error!void,
	1962	kind: Kind,
	1963	size: u64,
	1964
	1965	const Error =
	1966	Io.File.OpenError \|\|
	1967	Io.File.ReadPositionalError \|\|
	1968	Io.File.StatError \|\|
	1969	Io.File.SetPermissionsError \|\|
	1970	Io.Dir.ReadLinkError;
	1971
	1972	const Kind = enum { file, link };
	1973
	1974	fn lessThan(context: void, lhs: const HashedFile, rhs: const HashedFile) bool {
	1975	_ = context;
	1976	return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path);
	1977	}
	1978	};
	1979
	1980	/// Strips root directory name from file system path.
	1981	fn stripRoot(fs_path: []const u8, root_dir: []const u8) []const u8 {
	1982	if (root_dir.len == 0 or fs_path.len <= root_dir.len) return fs_path;
	1983
	1984	if (std.mem.eql(u8, fs_path[0..root_dir.len], root_dir) and fs.path.isSep(fs_path[root_dir.len])) {
	1985	return fs_path[root_dir.len + 1 ..];
	1986	}
	1987
	1988	return fs_path;
	1989	}
	1990
	1991	/// Make a file system path identical independently of operating system path inconsistencies.
	1992	/// This converts backslashes into forward slashes.
	1993	fn normalizePathAlloc(arena: Allocator, pkg_path: []const u8) ![]const u8 {
	1994	const normalized = try arena.dupe(u8, pkg_path);
	1995	if (fs.path.sep == canonical_sep) return normalized;
	1996	normalizePath(normalized);
	1997	return normalized;
	1998	}
	1999
	2000	const canonical_sep = fs.path.sep_posix;
	2001
	2002	fn normalizePath(bytes: []u8) void {
	2003	assert(fs.path.sep != canonical_sep);
	2004	std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep);
	2005	}
	2006
	2007	const Filter = struct {
	2008	include_paths: std.array_hash_map.String(void) = .empty,
	2009
	2010	/// sub_path is relative to the package root.
	2011	pub fn includePath(self: *const Filter, sub_path: []const u8) bool {
	2012	if (self.include_paths.count() == 0) return true;
	2013	if (self.include_paths.contains("")) return true;
	2014	if (self.include_paths.contains(".")) return true;
	2015	if (self.include_paths.contains(sub_path)) return true;
	2016
	2017	// Check if any included paths are parent directories of sub_path.
	2018	var dirname = sub_path;
	2019	while (std.fs.path.dirname(dirname)) \|next_dirname\| {
	2020	if (self.include_paths.contains(next_dirname)) return true;
	2021	dirname = next_dirname;
	2022	}
	2023
	2024	return false;
	2025	}
	2026
	2027	test includePath {
	2028	const gpa = std.testing.allocator;
	2029	var filter: Filter = .{};
	2030	defer filter.include_paths.deinit(gpa);
	2031
	2032	try filter.include_paths.put(gpa, "src", {});
	2033	try std.testing.expect(filter.includePath("src/core/unix/SDL_poll.c"));
	2034	try std.testing.expect(!filter.includePath(".gitignore"));
	2035	}
	2036	};
	2037
	2038	pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash {
	2039	if (dep.hash) \|h\| return .fromSlice(h);
	2040
	2041	switch (dep.location) {
	2042	.url => return null,
	2043	.path => \|rel_path\| {
	2044	var buf: [fs.max_path_bytes]u8 = undefined;
	2045	var fba = std.heap.FixedBufferAllocator.init(&buf);
	2046	const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch
	2047	return null;
	2048	return relativePathDigest(new_root, cache_root);
	2049	},
	2050	}
	2051	}
	2052
	2053	// Detects executable header: ELF or Macho-O magic header or shebang line.
	2054	const FileHeader = struct {
	2055	header: [4]u8 = undefined,
	2056	bytes_read: usize = 0,
	2057
	2058	pub fn update(self: *FileHeader, buf: []const u8) void {
	2059	if (self.bytes_read >= self.header.len) return;
	2060	const n = @min(self.header.len - self.bytes_read, buf.len);
	2061	@memcpy(self.header[self.bytes_read..][0..n], buf[0..n]);
	2062	self.bytes_read += n;
	2063	}
	2064
	2065	fn isScript(self: *FileHeader) bool {
	2066	const shebang = "#!";
	2067	return std.mem.eql(u8, self.header[0..@min(self.bytes_read, shebang.len)], shebang);
	2068	}
	2069
	2070	fn isElf(self: *FileHeader) bool {
	2071	const elf_magic = std.elf.MAGIC;
	2072	return std.mem.eql(u8, self.header[0..@min(self.bytes_read, elf_magic.len)], elf_magic);
	2073	}
	2074
	2075	fn isMachO(self: *FileHeader) bool {
	2076	if (self.bytes_read < 4) return false;
	2077	const magic_number = std.mem.readInt(u32, &self.header, builtin.cpu.arch.endian());
	2078	return magic_number == std.macho.MH_MAGIC or
	2079	magic_number == std.macho.MH_MAGIC_64 or
	2080	magic_number == std.macho.FAT_MAGIC or
	2081	magic_number == std.macho.FAT_MAGIC_64 or
	2082	magic_number == std.macho.MH_CIGAM or
	2083	magic_number == std.macho.MH_CIGAM_64 or
	2084	magic_number == std.macho.FAT_CIGAM or
	2085	magic_number == std.macho.FAT_CIGAM_64;
	2086	}
	2087
	2088	pub fn isExecutable(self: *FileHeader) bool {
	2089	return self.isScript() or self.isElf() or self.isMachO();
	2090	}
	2091	};
	2092
	2093	test FileHeader {
	2094	var h: FileHeader = .{};
	2095	try std.testing.expect(!h.isExecutable());
	2096
	2097	const elf_magic = std.elf.MAGIC;
	2098	h.update(elf_magic[0..2]);
	2099	try std.testing.expect(!h.isExecutable());
	2100	h.update(elf_magic[2..4]);
	2101	try std.testing.expect(h.isExecutable());
	2102
	2103	h.update(elf_magic[2..4]);
	2104	try std.testing.expect(h.isExecutable());
	2105
	2106	const macho64_magic_bytes = [_]u8{ 0xCF, 0xFA, 0xED, 0xFE };
	2107	h.bytes_read = 0;
	2108	h.update(&macho64_magic_bytes);
	2109	try std.testing.expect(h.isExecutable());
	2110
	2111	const macho64_cigam_bytes = [_]u8{ 0xFE, 0xED, 0xFA, 0xCF };
	2112	h.bytes_read = 0;
	2113	h.update(&macho64_cigam_bytes);
	2114	try std.testing.expect(h.isExecutable());
	2115	}
	2116
	2117	// Result of the `unpackResource` operation. Enables collecting errors from
	2118	// tar/git diagnostic, filtering that errors by manifest inclusion rules and
	2119	// emitting remaining errors to an `ErrorBundle`.
	2120	const UnpackResult = struct {
	2121	errors: []Error = undefined,
	2122	errors_count: usize = 0,
	2123	root_error_message: []const u8 = "",
	2124
	2125	// A non empty value means that the package contents are inside a
	2126	// sub-directory indicated by the named path.
	2127	root_dir: []const u8 = "",
	2128
	2129	const Error = union(enum) {
	2130	unable_to_create_sym_link: struct {
	2131	code: anyerror,
	2132	file_name: []const u8,
	2133	link_name: []const u8,
	2134	},
	2135	unable_to_create_file: struct {
	2136	code: anyerror,
	2137	file_name: []const u8,
	2138	},
	2139	unsupported_file_type: struct {
	2140	file_name: []const u8,
	2141	file_type: u8,
	2142	},
	2143
	2144	fn excluded(self: Error, filter: Filter) bool {
	2145	const file_name = switch (self) {
	2146	.unable_to_create_file => \|info\| info.file_name,
	2147	.unable_to_create_sym_link => \|info\| info.file_name,
	2148	.unsupported_file_type => \|info\| info.file_name,
	2149	};
	2150	return !filter.includePath(file_name);
	2151	}
	2152	};
	2153
	2154	fn allocErrors(self: *UnpackResult, arena: std.mem.Allocator, n: usize, root_error_message: []const u8) !void {
	2155	self.root_error_message = try arena.dupe(u8, root_error_message);
	2156	self.errors = try arena.alloc(UnpackResult.Error, n);
	2157	}
	2158
	2159	fn hasErrors(self: *UnpackResult) bool {
	2160	return self.errors_count > 0;
	2161	}
	2162
	2163	fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) void {
	2164	self.errors[self.errors_count] = .{ .unable_to_create_file = .{
	2165	.code = err,
	2166	.file_name = file_name,
	2167	} };
	2168	self.errors_count += 1;
	2169	}
	2170
	2171	fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) void {
	2172	self.errors[self.errors_count] = .{ .unable_to_create_sym_link = .{
	2173	.code = err,
	2174	.file_name = file_name,
	2175	.link_name = link_name,
	2176	} };
	2177	self.errors_count += 1;
	2178	}
	2179
	2180	fn unsupportedFileType(self: *UnpackResult, file_name: []const u8, file_type: u8) void {
	2181	self.errors[self.errors_count] = .{ .unsupported_file_type = .{
	2182	.file_name = file_name,
	2183	.file_type = file_type,
	2184	} };
	2185	self.errors_count += 1;
	2186	}
	2187
	2188	fn validate(self: UnpackResult, f: Fetch, filter: Filter) !void {
	2189	if (self.errors_count == 0) return;
	2190
	2191	var unfiltered_errors: u32 = 0;
	2192	for (self.errors) \|item\| {
	2193	if (item.excluded(filter)) continue;
	2194	unfiltered_errors += 1;
	2195	}
	2196	if (unfiltered_errors == 0) return;
	2197
	2198	// Emmit errors to an `ErrorBundle`.
	2199	const eb = &f.error_bundle;
	2200	try eb.addRootErrorMessage(.{
	2201	.msg = try eb.addString(self.root_error_message),
	2202	.src_loc = try f.srcLoc(f.location_tok),
	2203	.notes_len = unfiltered_errors,
	2204	});
	2205	var note_i: u32 = try eb.reserveNotes(unfiltered_errors);
	2206	for (self.errors) \|item\| {
	2207	if (item.excluded(filter)) continue;
	2208	switch (item) {
	2209	.unable_to_create_sym_link => \|info\| {
	2210	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
	2211	.msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{
	2212	info.file_name, info.link_name, @errorName(info.code),
	2213	}),
	2214	}));
	2215	},
	2216	.unable_to_create_file => \|info\| {
	2217	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
	2218	.msg = try eb.printString("unable to create file '{s}': {s}", .{
	2219	info.file_name, @errorName(info.code),
	2220	}),
	2221	}));
	2222	},
	2223	.unsupported_file_type => \|info\| {
	2224	eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
	2225	.msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{
	2226	info.file_name, info.file_type,
	2227	}),
	2228	}));
	2229	},
	2230	}
	2231	note_i += 1;
	2232	}
	2233
	2234	return error.FetchFailed;
	2235	}
	2236
	2237	test validate {
	2238	const gpa = std.testing.allocator;
	2239	var arena_instance = std.heap.ArenaAllocator.init(gpa);
	2240	defer arena_instance.deinit();
	2241	const arena = arena_instance.allocator();
	2242
	2243	// fill UnpackResult with errors
	2244	var res: UnpackResult = .{};
	2245	try res.allocErrors(arena, 4, "unable to unpack");
	2246	try std.testing.expectEqual(0, res.errors_count);
	2247	res.unableToCreateFile("dir1/file1", error.File1);
	2248	res.unableToCreateSymLink("dir2/file2", "filename", error.SymlinkError);
	2249	res.unableToCreateFile("dir1/file3", error.File3);
	2250	res.unsupportedFileType("dir2/file4", 'x');
	2251	try std.testing.expectEqual(4, res.errors_count);
	2252
	2253	// create filter, includes dir2, excludes dir1
	2254	var filter: Filter = .{};
	2255	try filter.include_paths.put(arena, "dir2", {});
	2256
	2257	// init Fetch
	2258	var fetch: Fetch = undefined;
	2259	fetch.parent_manifest_ast = null;
	2260	fetch.location_tok = 0;
	2261	try fetch.error_bundle.init(gpa);
	2262	defer fetch.error_bundle.deinit();
	2263
	2264	// validate errors with filter
	2265	try std.testing.expectError(error.FetchFailed, res.validate(&fetch, filter));
	2266
	2267	// output errors to string
	2268	var errors = try fetch.error_bundle.toOwnedBundle("");
	2269	defer errors.deinit(gpa);
	2270	var aw: Io.Writer.Allocating = .init(gpa);
	2271	defer aw.deinit();
	2272	try errors.renderToWriter(.{}, &aw.writer);
	2273	try std.testing.expectEqualStrings(
	2274	\\error: unable to unpack
	2275	\\ note: unable to create symlink from 'dir2/file2' to 'filename': SymlinkError
	2276	\\ note: file 'dir2/file4' has unsupported type 'x'
	2277	\\
	2278	, aw.written());
	2279	}
	2280	};
	2281
	2282	test {
	2283	_ = Filter;
	2284	_ = FileType;
	2285	_ = UnpackResult;
	2286	}

lib/compiler/Maker/Fetch/git.zig created+1750

...	...	@@ -0,0 +1,1750 @@
	1	//! Git support for package fetching.
	2	//!
	3	//! This is not intended to support all features of Git: it is limited to the
	4	//! basic functionality needed to clone a repository for the purpose of fetching
	5	//! a package.
	6
	7	const std = @import("std");
	8	const Io = std.Io;
	9	const mem = std.mem;
	10	const testing = std.testing;
	11	const Allocator = mem.Allocator;
	12	const Sha1 = std.crypto.hash.Sha1;
	13	const Sha256 = std.crypto.hash.sha2.Sha256;
	14	const assert = std.debug.assert;
	15
	16	/// The ID of a Git object.
	17	pub const Oid = union(Format) {
	18	sha1: [Sha1.digest_length]u8,
	19	sha256: [Sha256.digest_length]u8,
	20
	21	pub const max_formatted_length = len: {
	22	var max: usize = 0;
	23	for (std.enums.values(Format)) \|f\| {
	24	max = @max(max, f.formattedLength());
	25	}
	26	break :len max;
	27	};
	28
	29	pub const Format = enum {
	30	sha1,
	31	sha256,
	32
	33	pub fn byteLength(f: Format) usize {
	34	return switch (f) {
	35	.sha1 => Sha1.digest_length,
	36	.sha256 => Sha256.digest_length,
	37	};
	38	}
	39
	40	pub fn formattedLength(f: Format) usize {
	41	return 2 * f.byteLength();
	42	}
	43	};
	44
	45	const Hasher = union(Format) {
	46	sha1: Sha1,
	47	sha256: Sha256,
	48
	49	fn init(oid_format: Format) Hasher {
	50	return switch (oid_format) {
	51	.sha1 => .{ .sha1 = Sha1.init(.{}) },
	52	.sha256 => .{ .sha256 = Sha256.init(.{}) },
	53	};
	54	}
	55
	56	// Must be public for use from HashedReader and HashedWriter.
	57	pub fn update(hasher: *Hasher, b: []const u8) void {
	58	switch (hasher.*) {
	59	inline else => \|*inner\| inner.update(b),
	60	}
	61	}
	62
	63	fn finalResult(hasher: *Hasher) Oid {
	64	return switch (hasher.*) {
	65	inline else => \|*inner, tag\| @unionInit(Oid, @tagName(tag), inner.finalResult()),
	66	};
	67	}
	68	};
	69
	70	const Hashing = union(Format) {
	71	sha1: Io.Writer.Hashing(Sha1),
	72	sha256: Io.Writer.Hashing(Sha256),
	73
	74	fn init(oid_format: Format, buffer: []u8) Hashing {
	75	return switch (oid_format) {
	76	.sha1 => .{ .sha1 = .init(buffer) },
	77	.sha256 => .{ .sha256 = .init(buffer) },
	78	};
	79	}
	80
	81	fn writer(h: @This()) Io.Writer {
	82	return switch (h.*) {
	83	inline else => \|*inner\| &inner.writer,
	84	};
	85	}
	86
	87	fn final(h: *@This()) Oid {
	88	switch (h.*) {
	89	inline else => \|*inner, tag\| {
	90	inner.writer.flush() catch unreachable; // hashers cannot fail
	91	return @unionInit(Oid, @tagName(tag), inner.hasher.finalResult());
	92	},
	93	}
	94	}
	95	};
	96
	97	pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid {
	98	assert(bytes.len == oid_format.byteLength());
	99	return switch (oid_format) {
	100	inline else => \|tag\| @unionInit(Oid, @tagName(tag), bytes[0..comptime tag.byteLength()].*),
	101	};
	102	}
	103
	104	pub fn readBytes(oid_format: Format, reader: *Io.Reader) !Oid {
	105	return switch (oid_format) {
	106	inline else => \|tag\| @unionInit(Oid, @tagName(tag), (try reader.takeArray(tag.byteLength())).*),
	107	};
	108	}
	109
	110	pub fn parse(oid_format: Format, s: []const u8) error{InvalidOid}!Oid {
	111	switch (oid_format) {
	112	inline else => \|tag\| {
	113	if (s.len != tag.formattedLength()) return error.InvalidOid;
	114	var bytes: [tag.byteLength()]u8 = undefined;
	115	for (&bytes, 0..) \|*b, i\| {
	116	b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid;
	117	}
	118	return @unionInit(Oid, @tagName(tag), bytes);
	119	},
	120	}
	121	}
	122
	123	test parse {
	124	try testing.expectEqualSlices(
	125	u8,
	126	&.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 },
	127	&(try parse(.sha1, "ce919ccf45951856a762ffdb8ef850301cd8c588")).sha1,
	128	);
	129	try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf45951856a762ffdb8ef850301cd8c588"));
	130	try testing.expectError(error.InvalidOid, parse(.sha1, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a"));
	131	try testing.expectEqualSlices(
	132	u8,
	133	&.{ 0x7F, 0x44, 0x4A, 0x92, 0xBD, 0x45, 0x72, 0xEE, 0x4A, 0x28, 0xB2, 0xC6, 0x30, 0x59, 0x92, 0x4A, 0x9C, 0xA1, 0x82, 0x91, 0x38, 0x55, 0x3E, 0xF3, 0xE7, 0xC4, 0x1E, 0xE1, 0x59, 0xAF, 0xAE, 0x7A },
	134	&(try parse(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")).sha256,
	135	);
	136	try testing.expectError(error.InvalidOid, parse(.sha1, "ce919ccf"));
	137	try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf"));
	138	try testing.expectError(error.InvalidOid, parse(.sha1, "master"));
	139	try testing.expectError(error.InvalidOid, parse(.sha256, "master"));
	140	try testing.expectError(error.InvalidOid, parse(.sha1, "HEAD"));
	141	try testing.expectError(error.InvalidOid, parse(.sha256, "HEAD"));
	142	}
	143
	144	pub fn parseAny(s: []const u8) error{InvalidOid}!Oid {
	145	return for (std.enums.values(Format)) \|f\| {
	146	if (s.len == f.formattedLength()) break parse(f, s);
	147	} else error.InvalidOid;
	148	}
	149
	150	pub fn format(oid: Oid, writer: *Io.Writer) Io.Writer.Error!void {
	151	try writer.print("{x}", .{oid.slice()});
	152	}
	153
	154	pub fn slice(oid: *const Oid) []const u8 {
	155	return switch (oid.*) {
	156	inline else => \|*bytes\| bytes,
	157	};
	158	}
	159	};
	160
	161	pub const Diagnostics = struct {
	162	allocator: Allocator,
	163	errors: std.ArrayList(Error) = .empty,
	164
	165	pub const Error = union(enum) {
	166	unable_to_create_sym_link: struct {
	167	code: anyerror,
	168	file_name: []const u8,
	169	link_name: []const u8,
	170	},
	171	unable_to_create_file: struct {
	172	code: anyerror,
	173	file_name: []const u8,
	174	},
	175	};
	176
	177	pub fn deinit(d: *Diagnostics) void {
	178	for (d.errors.items) \|item\| {
	179	switch (item) {
	180	.unable_to_create_sym_link => \|info\| {
	181	d.allocator.free(info.file_name);
	182	d.allocator.free(info.link_name);
	183	},
	184	.unable_to_create_file => \|info\| {
	185	d.allocator.free(info.file_name);
	186	},
	187	}
	188	}
	189	d.errors.deinit(d.allocator);
	190	d.* = undefined;
	191	}
	192	};
	193
	194	pub const Repository = struct {
	195	odb: Odb,
	196
	197	pub fn init(
	198	repo: *Repository,
	199	allocator: Allocator,
	200	format: Oid.Format,
	201	pack_file: *Io.File.Reader,
	202	index_file: *Io.File.Reader,
	203	) !void {
	204	repo.* = .{ .odb = undefined };
	205	try repo.odb.init(allocator, format, pack_file, index_file);
	206	}
	207
	208	pub fn deinit(repository: *Repository) void {
	209	repository.odb.deinit();
	210	repository.* = undefined;
	211	}
	212
	213	/// Checks out the repository at `commit_oid` to `worktree`.
	214	pub fn checkout(
	215	repository: *Repository,
	216	io: Io,
	217	worktree: Io.Dir,
	218	commit_oid: Oid,
	219	diagnostics: *Diagnostics,
	220	) !void {
	221	try repository.odb.seekOid(commit_oid);
	222	const tree_oid = tree_oid: {
	223	const commit_object = try repository.odb.readObject();
	224	if (commit_object.type != .commit) return error.NotACommit;
	225	break :tree_oid try getCommitTree(repository.odb.format, commit_object.data);
	226	};
	227	try repository.checkoutTree(io, worktree, tree_oid, "", diagnostics);
	228	}
	229
	230	/// Checks out the tree at `tree_oid` to `worktree`.
	231	fn checkoutTree(
	232	repository: *Repository,
	233	io: Io,
	234	dir: Io.Dir,
	235	tree_oid: Oid,
	236	current_path: []const u8,
	237	diagnostics: *Diagnostics,
	238	) !void {
	239	try repository.odb.seekOid(tree_oid);
	240	const tree_object = try repository.odb.readObject();
	241	if (tree_object.type != .tree) return error.NotATree;
	242	// The tree object may be evicted from the object cache while we're
	243	// iterating over it, so we can make a defensive copy here to make sure
	244	// it remains valid until we're done with it
	245	const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data);
	246	defer repository.odb.allocator.free(tree_data);
	247
	248	var tree_iter: TreeIterator = .{
	249	.format = repository.odb.format,
	250	.data = tree_data,
	251	.pos = 0,
	252	};
	253	while (try tree_iter.next()) \|entry\| {
	254	switch (entry.type) {
	255	.directory => {
	256	try dir.createDir(io, entry.name, .default_dir);
	257	var subdir = try dir.openDir(io, entry.name, .{});
	258	defer subdir.close(io);
	259	const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name });
	260	defer repository.odb.allocator.free(sub_path);
	261	try repository.checkoutTree(io, subdir, entry.oid, sub_path, diagnostics);
	262	},
	263	.file => {
	264	try repository.odb.seekOid(entry.oid);
	265	const file_object = try repository.odb.readObject();
	266	if (file_object.type != .blob) return error.InvalidFile;
	267	var file = dir.createFile(io, entry.name, .{ .exclusive = true }) catch \|e\| {
	268	const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name });
	269	errdefer diagnostics.allocator.free(file_name);
	270	try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_file = .{
	271	.code = e,
	272	.file_name = file_name,
	273	} });
	274	continue;
	275	};
	276	defer file.close(io);
	277	try file.writePositionalAll(io, file_object.data, 0);
	278	},
	279	.symlink => {
	280	try repository.odb.seekOid(entry.oid);
	281	const symlink_object = try repository.odb.readObject();
	282	if (symlink_object.type != .blob) return error.InvalidFile;
	283	const link_name = symlink_object.data;
	284	dir.symLink(io, link_name, entry.name, .{}) catch \|e\| {
	285	const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name });
	286	errdefer diagnostics.allocator.free(file_name);
	287	const link_name_dup = try diagnostics.allocator.dupe(u8, link_name);
	288	errdefer diagnostics.allocator.free(link_name_dup);
	289	try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{
	290	.code = e,
	291	.file_name = file_name,
	292	.link_name = link_name_dup,
	293	} });
	294	};
	295	},
	296	.gitlink => {
	297	// Consistent with git archive behavior, create the directory but
	298	// do nothing else
	299	try dir.createDir(io, entry.name, .default_dir);
	300	},
	301	}
	302	}
	303	}
	304
	305	/// Returns the ID of the tree associated with the given commit (provided as
	306	/// raw object data).
	307	fn getCommitTree(format: Oid.Format, commit_data: []const u8) !Oid {
	308	if (!mem.startsWith(u8, commit_data, "tree ") or
	309	commit_data.len < "tree ".len + format.formattedLength() + "\n".len or
	310	commit_data["tree ".len + format.formattedLength()] != '\n')
	311	{
	312	return error.InvalidCommit;
	313	}
	314	return try .parse(format, commit_data["tree ".len..][0..format.formattedLength()]);
	315	}
	316
	317	const TreeIterator = struct {
	318	format: Oid.Format,
	319	data: []const u8,
	320	pos: usize,
	321
	322	const Entry = struct {
	323	type: Type,
	324	executable: bool,
	325	name: [:0]const u8,
	326	oid: Oid,
	327
	328	const Type = enum(u4) {
	329	directory = 0o4,
	330	file = 0o10,
	331	symlink = 0o12,
	332	gitlink = 0o16,
	333	};
	334	};
	335
	336	fn next(iterator: *TreeIterator) !?Entry {
	337	if (iterator.pos == iterator.data.len) return null;
	338
	339	const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree;
	340	const mode: packed struct {
	341	permission: u9,
	342	unused: u3,
	343	type: u4,
	344	} = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree);
	345	const @"type" = std.enums.fromInt(Entry.Type, mode.type) orelse return error.InvalidTree;
	346	const executable = switch (mode.permission) {
	347	0 => if (@"type" == .file) return error.InvalidTree else false,
	348	0o644 => if (@"type" != .file) return error.InvalidTree else false,
	349	0o755 => if (@"type" != .file) return error.InvalidTree else true,
	350	else => return error.InvalidTree,
	351	};
	352	iterator.pos = mode_end + 1;
	353
	354	const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree;
	355	const name = iterator.data[iterator.pos..name_end :0];
	356	iterator.pos = name_end + 1;
	357
	358	const oid_length = iterator.format.byteLength();
	359	if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree;
	360	const oid: Oid = .fromBytes(iterator.format, iterator.data[iterator.pos..][0..oid_length]);
	361	iterator.pos += oid_length;
	362
	363	return .{ .type = @"type", .executable = executable, .name = name, .oid = oid };
	364	}
	365	};
	366	};
	367
	368	/// A Git object database backed by a packfile. A packfile index is also used
	369	/// for efficient access to objects in the packfile.
	370	///
	371	/// The format of the packfile and its associated index are documented in
	372	/// [pack-format](https://git-scm.com/docs/pack-format).
	373	const Odb = struct {
	374	format: Oid.Format,
	375	pack_file: *Io.File.Reader,
	376	index_header: IndexHeader,
	377	index_file: *Io.File.Reader,
	378	cache: ObjectCache = .{},
	379	allocator: Allocator,
	380
	381	/// Initializes the database from open pack and index files.
	382	fn init(
	383	odb: *Odb,
	384	allocator: Allocator,
	385	format: Oid.Format,
	386	pack_file: *Io.File.Reader,
	387	index_file: *Io.File.Reader,
	388	) !void {
	389	try pack_file.seekTo(0);
	390	try index_file.seekTo(0);
	391	odb.* = .{
	392	.format = format,
	393	.pack_file = pack_file,
	394	.index_header = undefined,
	395	.index_file = index_file,
	396	.allocator = allocator,
	397	};
	398	try odb.index_header.read(&index_file.interface);
	399	}
	400
	401	fn deinit(odb: *Odb) void {
	402	odb.cache.deinit(odb.allocator);
	403	odb.* = undefined;
	404	}
	405
	406	/// Reads the object at the current position in the database.
	407	fn readObject(odb: *Odb) !Object {
	408	var base_offset = odb.pack_file.logicalPos();
	409	var base_header: EntryHeader = undefined;
	410	var delta_offsets: std.ArrayList(u64) = .empty;
	411	defer delta_offsets.deinit(odb.allocator);
	412	const base_object = while (true) {
	413	if (odb.cache.get(base_offset)) \|base_object\| break base_object;
	414
	415	base_header = try EntryHeader.read(odb.format, &odb.pack_file.interface);
	416	switch (base_header) {
	417	.ofs_delta => \|ofs_delta\| {
	418	try delta_offsets.append(odb.allocator, base_offset);
	419	base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat;
	420	try odb.pack_file.seekTo(base_offset);
	421	},
	422	.ref_delta => \|ref_delta\| {
	423	try delta_offsets.append(odb.allocator, base_offset);
	424	try odb.seekOid(ref_delta.base_object);
	425	base_offset = odb.pack_file.logicalPos();
	426	},
	427	else => {
	428	const base_data = try readObjectRaw(odb.allocator, &odb.pack_file.interface, base_header.uncompressedLength());
	429	errdefer odb.allocator.free(base_data);
	430	const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
	431	try odb.cache.put(odb.allocator, base_offset, base_object);
	432	break base_object;
	433	},
	434	}
	435	};
	436
	437	const base_data = try resolveDeltaChain(
	438	odb.allocator,
	439	odb.format,
	440	odb.pack_file,
	441	base_object,
	442	delta_offsets.items,
	443	&odb.cache,
	444	);
	445
	446	return .{ .type = base_object.type, .data = base_data };
	447	}
	448
	449	/// Seeks to the beginning of the object with the given ID.
	450	fn seekOid(odb: *Odb, oid: Oid) !void {
	451	const oid_length = odb.format.byteLength();
	452	const key = oid.slice()[0];
	453	var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0;
	454	var end_index = odb.index_header.fan_out_table[key];
	455	const found_index = while (start_index < end_index) {
	456	const mid_index = start_index + (end_index - start_index) / 2;
	457	try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length);
	458	const mid_oid = try Oid.readBytes(odb.format, &odb.index_file.interface);
	459	switch (mem.order(u8, mid_oid.slice(), oid.slice())) {
	460	.lt => start_index = mid_index + 1,
	461	.gt => end_index = mid_index,
	462	.eq => break mid_index,
	463	}
	464	} else return error.ObjectNotFound;
	465
	466	const n_objects = odb.index_header.fan_out_table[255];
	467	const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4);
	468	try odb.index_file.seekTo(offset_values_start + found_index * 4);
	469	const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.interface.takeInt(u32, .big));
	470	const pack_offset = pack_offset: {
	471	if (l1_offset.big) {
	472	const l2_offset_values_start = offset_values_start + n_objects * 4;
	473	try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4);
	474	break :pack_offset try odb.index_file.interface.takeInt(u64, .big);
	475	} else {
	476	break :pack_offset l1_offset.value;
	477	}
	478	};
	479
	480	try odb.pack_file.seekTo(pack_offset);
	481	}
	482	};
	483
	484	const Object = struct {
	485	type: Type,
	486	data: []const u8,
	487
	488	const Type = enum {
	489	commit,
	490	tree,
	491	blob,
	492	tag,
	493	};
	494	};
	495
	496	/// A cache for object data.
	497	///
	498	/// The purpose of this cache is to speed up resolution of deltas by caching the
	499	/// results of resolving delta objects, while maintaining a maximum cache size
	500	/// to avoid excessive memory usage. If the total size of the objects in the
	501	/// cache exceeds the maximum, the cache will begin evicting the least recently
	502	/// used objects: when resolving delta chains, the most recently used objects
	503	/// will likely be more helpful as they will be further along in the chain
	504	/// (skipping earlier reconstruction steps).
	505	///
	506	/// Object data stored in the cache is managed by the cache. It should not be
	507	/// freed by the caller at any point after inserting it into the cache. Any
	508	/// objects remaining in the cache will be freed when the cache itself is freed.
	509	const ObjectCache = struct {
	510	objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .empty,
	511	lru_nodes: std.DoublyLinkedList = .{},
	512	lru_nodes_len: usize = 0,
	513	byte_size: usize = 0,
	514
	515	const max_byte_size = 128 * 1024 * 1024; // 128MiB
	516	/// A list of offsets stored in the cache, with the most recently used
	517	/// entries at the end.
	518	const LruListNode = struct {
	519	data: u64,
	520	node: std.DoublyLinkedList.Node,
	521	};
	522	const CacheEntry = struct { object: Object, lru_node: *LruListNode };
	523
	524	fn deinit(cache: *ObjectCache, allocator: Allocator) void {
	525	var object_iterator = cache.objects.iterator();
	526	while (object_iterator.next()) \|object\| {
	527	allocator.free(object.value_ptr.object.data);
	528	allocator.destroy(object.value_ptr.lru_node);
	529	}
	530	cache.objects.deinit(allocator);
	531	cache.* = undefined;
	532	}
	533
	534	/// Gets an object from the cache, moving it to the most recently used
	535	/// position if it is present.
	536	fn get(cache: *ObjectCache, offset: u64) ?Object {
	537	if (cache.objects.get(offset)) \|entry\| {
	538	cache.lru_nodes.remove(&entry.lru_node.node);
	539	cache.lru_nodes.append(&entry.lru_node.node);
	540	return entry.object;
	541	} else {
	542	return null;
	543	}
	544	}
	545
	546	/// Puts an object in the cache, possibly evicting older entries if the
	547	/// cache exceeds its maximum size. Note that, although old objects may
	548	/// be evicted, the object just added to the cache with this function
	549	/// will not be evicted before the next call to `put` or `deinit` even if
	550	/// it exceeds the maximum cache size.
	551	fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void {
	552	const lru_node = try allocator.create(LruListNode);
	553	errdefer allocator.destroy(lru_node);
	554	lru_node.data = offset;
	555
	556	const gop = try cache.objects.getOrPut(allocator, offset);
	557	if (gop.found_existing) {
	558	cache.byte_size -= gop.value_ptr.object.data.len;
	559	cache.lru_nodes.remove(&gop.value_ptr.lru_node.node);
	560	cache.lru_nodes_len -= 1;
	561	allocator.destroy(gop.value_ptr.lru_node);
	562	allocator.free(gop.value_ptr.object.data);
	563	}
	564	gop.value_ptr.* = .{ .object = object, .lru_node = lru_node };
	565	cache.byte_size += object.data.len;
	566	cache.lru_nodes.append(&lru_node.node);
	567	cache.lru_nodes_len += 1;
	568
	569	while (cache.byte_size > max_byte_size and cache.lru_nodes_len > 1) {
	570	// The > 1 check is to make sure that we don't evict the most
	571	// recently added node, even if it by itself happens to exceed the
	572	// maximum size of the cache.
	573	const evict_node: *LruListNode = @alignCast(@fieldParentPtr("node", cache.lru_nodes.popFirst().?));
	574	cache.lru_nodes_len -= 1;
	575	const evict_offset = evict_node.data;
	576	allocator.destroy(evict_node);
	577	const evict_object = cache.objects.get(evict_offset).?.object;
	578	cache.byte_size -= evict_object.data.len;
	579	allocator.free(evict_object.data);
	580	_ = cache.objects.remove(evict_offset);
	581	}
	582	}
	583	};
	584
	585	/// A single pkt-line in the Git protocol.
	586	///
	587	/// The format of a pkt-line is documented in
	588	/// [protocol-common](https://git-scm.com/docs/protocol-common). The special
	589	/// meanings of the delimiter and response-end packets are documented in
	590	/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
	591	pub const Packet = union(enum) {
	592	flush,
	593	delimiter,
	594	response_end,
	595	data: []const u8,
	596
	597	pub const max_data_length = 65516;
	598
	599	/// Reads a packet in pkt-line format.
	600	fn read(reader: *Io.Reader) !Packet {
	601	const packet: Packet = try .peek(reader);
	602	switch (packet) {
	603	.data => \|data\| reader.toss(data.len),
	604	else => {},
	605	}
	606	return packet;
	607	}
	608
	609	/// Consumes the header of a pkt-line packet and reads any associated data
	610	/// into the reader's buffer, but does not consume the data.
	611	fn peek(reader: *Io.Reader) !Packet {
	612	const length = std.fmt.parseUnsigned(u16, try reader.take(4), 16) catch return error.InvalidPacket;
	613	switch (length) {
	614	0 => return .flush,
	615	1 => return .delimiter,
	616	2 => return .response_end,
	617	3 => return error.InvalidPacket,
	618	else => if (length - 4 > max_data_length) return error.InvalidPacket,
	619	}
	620	return .{ .data = try reader.peek(length - 4) };
	621	}
	622
	623	/// Writes a packet in pkt-line format.
	624	fn write(packet: Packet, writer: *Io.Writer) !void {
	625	switch (packet) {
	626	.flush => try writer.writeAll("0000"),
	627	.delimiter => try writer.writeAll("0001"),
	628	.response_end => try writer.writeAll("0002"),
	629	.data => \|data\| {
	630	assert(data.len <= max_data_length);
	631	try writer.print("{x:0>4}", .{data.len + 4});
	632	try writer.writeAll(data);
	633	},
	634	}
	635	}
	636
	637	/// Returns the normalized form of textual packet data, stripping any
	638	/// trailing '\n'.
	639	///
	640	/// As documented in
	641	/// [protocol-common](https://git-scm.com/docs/protocol-common#_pkt_line_format),
	642	/// non-binary (textual) pkt-line data should contain a trailing '\n', but
	643	/// is not required to do so (implementations must support both forms).
	644	fn normalizeText(data: []const u8) []const u8 {
	645	return if (mem.endsWith(u8, data, "\n"))
	646	data[0 .. data.len - 1]
	647	else
	648	data;
	649	}
	650	};
	651
	652	/// A client session for the Git protocol, currently limited to an HTTP(S)
	653	/// transport. Only protocol version 2 is supported, as documented in
	654	/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
	655	pub const Session = struct {
	656	transport: *std.http.Client,
	657	location: Location,
	658	supports_agent: bool,
	659	supports_shallow: bool,
	660	object_format: Oid.Format,
	661	arena: Allocator,
	662
	663	const agent = "zig/" ++ @import("builtin").zig_version_string;
	664	const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent});
	665
	666	/// Initializes a client session and discovers the capabilities of the
	667	/// server for optimal transport.
	668	pub fn init(
	669	arena: Allocator,
	670	transport: *std.http.Client,
	671	uri: std.Uri,
	672	/// Asserted to be at least `Packet.max_data_length`
	673	response_buffer: []u8,
	674	) !Session {
	675	assert(response_buffer.len >= Packet.max_data_length);
	676	var session: Session = .{
	677	.transport = transport,
	678	.location = try .init(arena, uri),
	679	.supports_agent = false,
	680	.supports_shallow = false,
	681	.object_format = .sha1,
	682	.arena = arena,
	683	};
	684	var capability_iterator: CapabilityIterator = undefined;
	685	try session.getCapabilities(&capability_iterator, response_buffer);
	686	defer capability_iterator.deinit();
	687	while (try capability_iterator.next()) \|capability\| {
	688	if (mem.eql(u8, capability.key, "agent")) {
	689	session.supports_agent = true;
	690	} else if (mem.eql(u8, capability.key, "fetch")) {
	691	var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' ');
	692	while (feature_iterator.next()) \|feature\| {
	693	if (mem.eql(u8, feature, "shallow")) {
	694	session.supports_shallow = true;
	695	}
	696	}
	697	} else if (mem.eql(u8, capability.key, "object-format")) {
	698	if (std.meta.stringToEnum(Oid.Format, capability.value orelse continue)) \|format\| {
	699	session.object_format = format;
	700	}
	701	}
	702	}
	703	return session;
	704	}
	705
	706	/// An owned `std.Uri` representing the location of the server (base URI).
	707	const Location = struct {
	708	uri: std.Uri,
	709
	710	fn init(arena: Allocator, uri: std.Uri) !Location {
	711	const scheme = try arena.dupe(u8, uri.scheme);
	712	const user = if (uri.user) \|user\| try std.fmt.allocPrint(arena, "{f}", .{
	713	std.fmt.alt(user, .formatUser),
	714	}) else null;
	715	const password = if (uri.password) \|password\| try std.fmt.allocPrint(arena, "{f}", .{
	716	std.fmt.alt(password, .formatPassword),
	717	}) else null;
	718	const host = if (uri.host) \|host\| try std.fmt.allocPrint(arena, "{f}", .{
	719	std.fmt.alt(host, .formatHost),
	720	}) else null;
	721	const path = try std.fmt.allocPrint(arena, "{f}", .{
	722	std.fmt.alt(uri.path, .formatPath),
	723	});
	724	// The query and fragment are not used as part of the base server URI.
	725	return .{
	726	.uri = .{
	727	.scheme = scheme,
	728	.user = if (user) \|s\| .{ .percent_encoded = s } else null,
	729	.password = if (password) \|s\| .{ .percent_encoded = s } else null,
	730	.host = if (host) \|s\| .{ .percent_encoded = s } else null,
	731	.port = uri.port,
	732	.path = .{ .percent_encoded = path },
	733	},
	734	};
	735	}
	736	};
	737
	738	/// Returns an iterator over capabilities supported by the server.
	739	///
	740	/// The `session.location` is updated if the server returns a redirect, so
	741	/// that subsequent session functions do not need to handle redirects.
	742	fn getCapabilities(session: Session, it: CapabilityIterator, response_buffer: []u8) !void {
	743	const arena = session.arena;
	744	assert(response_buffer.len >= Packet.max_data_length);
	745	var info_refs_uri = session.location.uri;
	746	{
	747	const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
	748	std.fmt.alt(session.location.uri.path, .formatPath),
	749	});
	750	info_refs_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{
	751	"/", session_uri_path, "info/refs",
	752	}) };
	753	}
	754	info_refs_uri.query = .{ .percent_encoded = "service=git-upload-pack" };
	755	info_refs_uri.fragment = null;
	756
	757	const max_redirects = 3;
	758	it.* = .{
	759	.request = try session.transport.request(.GET, info_refs_uri, .{
	760	.redirect_behavior = .init(max_redirects),
	761	.extra_headers = &.{
	762	.{ .name = "Git-Protocol", .value = "version=2" },
	763	},
	764	}),
	765	.reader = undefined,
	766	.decompress = undefined,
	767	};
	768	errdefer it.deinit();
	769	const request = &it.request;
	770	try request.sendBodiless();
	771
	772	var redirect_buffer: [1024]u8 = undefined;
	773	var response = try request.receiveHead(&redirect_buffer);
	774	if (response.head.status != .ok) return error.ProtocolError;
	775	const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects;
	776	if (any_redirects_occurred) {
	777	const request_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
	778	std.fmt.alt(request.uri.path, .formatPath),
	779	});
	780	if (!mem.endsWith(u8, request_uri_path, "/info/refs")) return error.UnparseableRedirect;
	781	var new_uri = request.uri;
	782	new_uri.path = .{ .percent_encoded = request_uri_path[0 .. request_uri_path.len - "/info/refs".len] };
	783	session.location = try .init(arena, new_uri);
	784	}
	785
	786	const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
	787	it.reader = response.readerDecompressing(response_buffer, &it.decompress, decompress_buffer);
	788	var state: enum { response_start, response_content } = .response_start;
	789	while (true) {
	790	// Some Git servers (at least GitHub) include an additional
	791	// '# service=git-upload-pack' informative response before sending
	792	// the expected 'version 2' packet and capability information.
	793	// This is not universal: SourceHut, for example, does not do this.
	794	// Thus, we need to skip any such useless additional responses
	795	// before we get the one we're actually looking for. The responses
	796	// will be delimited by flush packets.
	797	const packet = Packet.read(it.reader) catch \|err\| switch (err) {
	798	error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found
	799	else => \|e\| return e,
	800	};
	801	switch (packet) {
	802	.flush => state = .response_start,
	803	.data => \|data\| switch (state) {
	804	.response_start => if (mem.eql(u8, Packet.normalizeText(data), "version 2")) {
	805	return;
	806	} else {
	807	state = .response_content;
	808	},
	809	else => {},
	810	},
	811	else => return error.UnexpectedPacket,
	812	}
	813	}
	814	}
	815
	816	const CapabilityIterator = struct {
	817	request: std.http.Client.Request,
	818	reader: *Io.Reader,
	819	decompress: std.http.Decompress,
	820
	821	const Capability = struct {
	822	key: []const u8,
	823	value: ?[]const u8 = null,
	824
	825	fn parse(data: []const u8) Capability {
	826	return if (mem.indexOfScalar(u8, data, '=')) \|separator_pos\|
	827	.{ .key = data[0..separator_pos], .value = data[separator_pos + 1 ..] }
	828	else
	829	.{ .key = data };
	830	}
	831	};
	832
	833	fn deinit(it: *CapabilityIterator) void {
	834	it.request.deinit();
	835	it.* = undefined;
	836	}
	837
	838	fn next(it: *CapabilityIterator) !?Capability {
	839	switch (try Packet.read(it.reader)) {
	840	.flush => return null,
	841	.data => \|data\| return Capability.parse(Packet.normalizeText(data)),
	842	else => return error.UnexpectedPacket,
	843	}
	844	}
	845	};
	846
	847	const ListRefsOptions = struct {
	848	/// The ref prefixes (if any) to use to filter the refs available on the
	849	/// server. Note that the client must still check the returned refs
	850	/// against its desired filters itself: the server is not required to
	851	/// respect these prefix filters and may return other refs as well.
	852	ref_prefixes: []const []const u8 = &.{},
	853	/// Whether to include symref targets for returned symbolic refs.
	854	include_symrefs: bool = false,
	855	/// Whether to include the peeled object ID for returned tag refs.
	856	include_peeled: bool = false,
	857	/// Asserted to be at least `Packet.max_data_length`.
	858	buffer: []u8,
	859	};
	860
	861	/// Returns an iterator over refs known to the server.
	862	pub fn listRefs(session: Session, it: *RefIterator, options: ListRefsOptions) !void {
	863	const arena = session.arena;
	864	assert(options.buffer.len >= Packet.max_data_length);
	865	var upload_pack_uri = session.location.uri;
	866	{
	867	const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
	868	std.fmt.alt(session.location.uri.path, .formatPath),
	869	});
	870	upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) };
	871	}
	872	upload_pack_uri.query = null;
	873	upload_pack_uri.fragment = null;
	874
	875	var body: Io.Writer = .fixed(options.buffer);
	876	try Packet.write(.{ .data = "command=ls-refs\n" }, &body);
	877	if (session.supports_agent) {
	878	try Packet.write(.{ .data = agent_capability }, &body);
	879	}
	880	{
	881	const object_format_packet = try std.fmt.allocPrint(arena, "object-format={t}\n", .{
	882	session.object_format,
	883	});
	884	try Packet.write(.{ .data = object_format_packet }, &body);
	885	}
	886	try Packet.write(.delimiter, &body);
	887	for (options.ref_prefixes) \|ref_prefix\| {
	888	const ref_prefix_packet = try std.fmt.allocPrint(arena, "ref-prefix {s}\n", .{ref_prefix});
	889	try Packet.write(.{ .data = ref_prefix_packet }, &body);
	890	}
	891	if (options.include_symrefs) {
	892	try Packet.write(.{ .data = "symrefs\n" }, &body);
	893	}
	894	if (options.include_peeled) {
	895	try Packet.write(.{ .data = "peel\n" }, &body);
	896	}
	897	try Packet.write(.flush, &body);
	898
	899	it.* = .{
	900	.request = try session.transport.request(.POST, upload_pack_uri, .{
	901	.redirect_behavior = .unhandled,
	902	.extra_headers = &.{
	903	.{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" },
	904	.{ .name = "Git-Protocol", .value = "version=2" },
	905	},
	906	}),
	907	.reader = undefined,
	908	.format = session.object_format,
	909	.decompress = undefined,
	910	};
	911	const request = &it.request;
	912	errdefer request.deinit();
	913	try request.sendBodyComplete(body.buffered());
	914
	915	var response = try request.receiveHead(options.buffer);
	916	if (response.head.status != .ok) return error.ProtocolError;
	917	const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
	918	it.reader = response.readerDecompressing(options.buffer, &it.decompress, decompress_buffer);
	919	}
	920
	921	pub const RefIterator = struct {
	922	format: Oid.Format,
	923	request: std.http.Client.Request,
	924	reader: *Io.Reader,
	925	decompress: std.http.Decompress,
	926
	927	pub const Ref = struct {
	928	oid: Oid,
	929	name: []const u8,
	930	symref_target: ?[]const u8,
	931	peeled: ?Oid,
	932	};
	933
	934	pub fn deinit(iterator: *RefIterator) void {
	935	iterator.request.deinit();
	936	iterator.* = undefined;
	937	}
	938
	939	pub fn next(it: *RefIterator) !?Ref {
	940	switch (try Packet.read(it.reader)) {
	941	.flush => return null,
	942	.data => \|data\| {
	943	const ref_data = Packet.normalizeText(data);
	944	const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket;
	945	const oid = Oid.parse(it.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket;
	946
	947	const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len;
	948	const name = ref_data[oid_sep_pos + 1 .. name_sep_pos];
	949
	950	var symref_target: ?[]const u8 = null;
	951	var peeled: ?Oid = null;
	952	var last_sep_pos = name_sep_pos;
	953	while (last_sep_pos < ref_data.len) {
	954	const next_sep_pos = mem.indexOfScalarPos(u8, ref_data, last_sep_pos + 1, ' ') orelse ref_data.len;
	955	const attribute = ref_data[last_sep_pos + 1 .. next_sep_pos];
	956	if (mem.startsWith(u8, attribute, "symref-target:")) {
	957	symref_target = attribute["symref-target:".len..];
	958	} else if (mem.startsWith(u8, attribute, "peeled:")) {
	959	peeled = Oid.parse(it.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket;
	960	}
	961	last_sep_pos = next_sep_pos;
	962	}
	963
	964	return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled };
	965	},
	966	else => return error.UnexpectedPacket,
	967	}
	968	}
	969	};
	970
	971	/// Fetches the given refs from the server. A shallow fetch (depth 1) is
	972	/// performed if the server supports it.
	973	pub fn fetch(
	974	session: Session,
	975	fs: *FetchStream,
	976	wants: []const []const u8,
	977	/// Asserted to be at least `Packet.max_data_length`.
	978	response_buffer: []u8,
	979	) !void {
	980	const arena = session.arena;
	981	assert(response_buffer.len >= Packet.max_data_length);
	982	var upload_pack_uri = session.location.uri;
	983	{
	984	const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
	985	std.fmt.alt(session.location.uri.path, .formatPath),
	986	});
	987	upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) };
	988	}
	989	upload_pack_uri.query = null;
	990	upload_pack_uri.fragment = null;
	991
	992	var body: Io.Writer = .fixed(response_buffer);
	993	try Packet.write(.{ .data = "command=fetch\n" }, &body);
	994	if (session.supports_agent) {
	995	try Packet.write(.{ .data = agent_capability }, &body);
	996	}
	997	{
	998	const object_format_packet = try std.fmt.allocPrint(arena, "object-format={s}\n", .{@tagName(session.object_format)});
	999	try Packet.write(.{ .data = object_format_packet }, &body);
	1000	}
	1001	try Packet.write(.delimiter, &body);
	1002	// Our packfile parser supports the OFS_DELTA object type
	1003	try Packet.write(.{ .data = "ofs-delta\n" }, &body);
	1004	// We do not currently convey server progress information to the user
	1005	try Packet.write(.{ .data = "no-progress\n" }, &body);
	1006	if (session.supports_shallow) {
	1007	try Packet.write(.{ .data = "deepen 1\n" }, &body);
	1008	}
	1009	for (wants) \|want\| {
	1010	var buf: [Packet.max_data_length]u8 = undefined;
	1011	const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable;
	1012	try Packet.write(.{ .data = arg }, &body);
	1013	}
	1014	try Packet.write(.{ .data = "done\n" }, &body);
	1015	try Packet.write(.flush, &body);
	1016
	1017	fs.* = .{
	1018	.request = try session.transport.request(.POST, upload_pack_uri, .{
	1019	.redirect_behavior = .not_allowed,
	1020	.extra_headers = &.{
	1021	.{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" },
	1022	.{ .name = "Git-Protocol", .value = "version=2" },
	1023	},
	1024	}),
	1025	.input = undefined,
	1026	.reader = undefined,
	1027	.remaining_len = undefined,
	1028	.decompress = undefined,
	1029	};
	1030	const request = &fs.request;
	1031	errdefer request.deinit();
	1032
	1033	try request.sendBodyComplete(body.buffered());
	1034
	1035	var response = try request.receiveHead(&.{});
	1036	if (response.head.status != .ok) return error.ProtocolError;
	1037
	1038	const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
	1039	const reader = response.readerDecompressing(response_buffer, &fs.decompress, decompress_buffer);
	1040	// We are not interested in any of the sections of the returned fetch
	1041	// data other than the packfile section, since we aren't doing anything
	1042	// complex like ref negotiation (this is a fresh clone).
	1043	var state: enum { section_start, section_content } = .section_start;
	1044	while (true) {
	1045	const packet = try Packet.read(reader);
	1046	switch (state) {
	1047	.section_start => switch (packet) {
	1048	.data => \|data\| if (mem.eql(u8, Packet.normalizeText(data), "packfile")) {
	1049	fs.input = reader;
	1050	fs.reader = .{
	1051	.buffer = &.{},
	1052	.vtable = &.{ .stream = FetchStream.stream },
	1053	.seek = 0,
	1054	.end = 0,
	1055	};
	1056	fs.remaining_len = 0;
	1057	return;
	1058	} else {
	1059	state = .section_content;
	1060	},
	1061	else => return error.UnexpectedPacket,
	1062	},
	1063	.section_content => switch (packet) {
	1064	.delimiter => state = .section_start,
	1065	.data => {},
	1066	else => return error.UnexpectedPacket,
	1067	},
	1068	}
	1069	}
	1070	}
	1071
	1072	pub const FetchStream = struct {
	1073	request: std.http.Client.Request,
	1074	input: *Io.Reader,
	1075	reader: Io.Reader,
	1076	err: ?Error = null,
	1077	remaining_len: usize,
	1078	decompress: std.http.Decompress,
	1079
	1080	pub fn deinit(fs: *FetchStream) void {
	1081	fs.request.deinit();
	1082	}
	1083
	1084	pub const Error = error{
	1085	InvalidPacket,
	1086	ProtocolError,
	1087	UnexpectedPacket,
	1088	WriteFailed,
	1089	ReadFailed,
	1090	EndOfStream,
	1091	};
	1092
	1093	const StreamCode = enum(u8) {
	1094	pack_data = 1,
	1095	progress = 2,
	1096	fatal_error = 3,
	1097	_,
	1098	};
	1099
	1100	pub fn stream(r: Io.Reader, w: Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize {
	1101	const fs: *FetchStream = @alignCast(@fieldParentPtr("reader", r));
	1102	const input = fs.input;
	1103	if (fs.remaining_len == 0) {
	1104	while (true) {
	1105	switch (Packet.peek(input) catch \|err\| {
	1106	fs.err = err;
	1107	return error.ReadFailed;
	1108	}) {
	1109	.flush => return error.EndOfStream,
	1110	.data => \|data\| switch (@as(StreamCode, @enumFromInt(data[0]))) {
	1111	.pack_data => {
	1112	input.toss(1);
	1113	fs.remaining_len = data.len - 1;
	1114	break;
	1115	},
	1116	.fatal_error => {
	1117	fs.err = error.ProtocolError;
	1118	return error.ReadFailed;
	1119	},
	1120	else => {
	1121	input.toss(data.len);
	1122	},
	1123	},
	1124	else => {
	1125	fs.err = error.UnexpectedPacket;
	1126	return error.ReadFailed;
	1127	},
	1128	}
	1129	}
	1130	}
	1131	const buf = limit.slice(try w.writableSliceGreedy(1));
	1132	const n = @min(buf.len, fs.remaining_len);
	1133	try input.readSliceAll(buf[0..n]);
	1134	w.advance(n);
	1135	fs.remaining_len -= n;
	1136	return n;
	1137	}
	1138	};
	1139	};
	1140
	1141	const PackHeader = struct {
	1142	total_objects: u32,
	1143
	1144	const signature = "PACK";
	1145	const supported_version = 2;
	1146
	1147	fn read(reader: *Io.Reader) !PackHeader {
	1148	const actual_signature = reader.take(4) catch \|e\| switch (e) {
	1149	error.EndOfStream => return error.InvalidHeader,
	1150	else => \|other\| return other,
	1151	};
	1152	if (!mem.eql(u8, actual_signature, signature)) return error.InvalidHeader;
	1153	const version = reader.takeInt(u32, .big) catch \|e\| switch (e) {
	1154	error.EndOfStream => return error.InvalidHeader,
	1155	else => \|other\| return other,
	1156	};
	1157	if (version != supported_version) return error.UnsupportedVersion;
	1158	const total_objects = reader.takeInt(u32, .big) catch \|e\| switch (e) {
	1159	error.EndOfStream => return error.InvalidHeader,
	1160	else => \|other\| return other,
	1161	};
	1162	return .{ .total_objects = total_objects };
	1163	}
	1164	};
	1165
	1166	const EntryHeader = union(Type) {
	1167	commit: Undeltified,
	1168	tree: Undeltified,
	1169	blob: Undeltified,
	1170	tag: Undeltified,
	1171	ofs_delta: OfsDelta,
	1172	ref_delta: RefDelta,
	1173
	1174	const Type = enum(u3) {
	1175	commit = 1,
	1176	tree = 2,
	1177	blob = 3,
	1178	tag = 4,
	1179	ofs_delta = 6,
	1180	ref_delta = 7,
	1181	};
	1182
	1183	const Undeltified = struct {
	1184	uncompressed_length: u64,
	1185	};
	1186
	1187	const OfsDelta = struct {
	1188	offset: u64,
	1189	uncompressed_length: u64,
	1190	};
	1191
	1192	const RefDelta = struct {
	1193	base_object: Oid,
	1194	uncompressed_length: u64,
	1195	};
	1196
	1197	fn objectType(header: EntryHeader) Object.Type {
	1198	return switch (header) {
	1199	inline .commit, .tree, .blob, .tag => \|_, tag\| @field(Object.Type, @tagName(tag)),
	1200	else => unreachable,
	1201	};
	1202	}
	1203
	1204	fn uncompressedLength(header: EntryHeader) u64 {
	1205	return switch (header) {
	1206	inline else => \|entry\| entry.uncompressed_length,
	1207	};
	1208	}
	1209
	1210	fn read(format: Oid.Format, reader: *Io.Reader) !EntryHeader {
	1211	const InitialByte = packed struct { len: u4, type: u3, has_next: bool };
	1212	const initial: InitialByte = @bitCast(reader.takeByte() catch \|e\| switch (e) {
	1213	error.EndOfStream => return error.InvalidFormat,
	1214	else => \|other\| return other,
	1215	});
	1216	const rest_len = if (initial.has_next) try reader.takeLeb128(u64) else 0;
	1217	var uncompressed_length: u64 = initial.len;
	1218	uncompressed_length \|= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat;
	1219	const @"type" = std.enums.fromInt(EntryHeader.Type, initial.type) orelse return error.InvalidFormat;
	1220	return switch (@"type") {
	1221	inline .commit, .tree, .blob, .tag => \|tag\| @unionInit(EntryHeader, @tagName(tag), .{
	1222	.uncompressed_length = uncompressed_length,
	1223	}),
	1224	.ofs_delta => .{ .ofs_delta = .{
	1225	.offset = try readOffsetVarInt(reader),
	1226	.uncompressed_length = uncompressed_length,
	1227	} },
	1228	.ref_delta => .{ .ref_delta = .{
	1229	.base_object = Oid.readBytes(format, reader) catch \|e\| switch (e) {
	1230	error.EndOfStream => return error.InvalidFormat,
	1231	else => \|other\| return other,
	1232	},
	1233	.uncompressed_length = uncompressed_length,
	1234	} },
	1235	};
	1236	}
	1237	};
	1238
	1239	fn readOffsetVarInt(r: *Io.Reader) !u64 {
	1240	const Byte = packed struct { value: u7, has_next: bool };
	1241	var b: Byte = @bitCast(try r.takeByte());
	1242	var value: u64 = b.value;
	1243	while (b.has_next) {
	1244	b = @bitCast(try r.takeByte());
	1245	value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat;
	1246	value \|= b.value;
	1247	}
	1248	return value;
	1249	}
	1250
	1251	const IndexHeader = struct {
	1252	fan_out_table: [256]u32,
	1253
	1254	const signature = "\xFFtOc";
	1255	const supported_version = 2;
	1256	const size = 4 + 4 + @sizeOf([256]u32);
	1257
	1258	fn read(index_header: IndexHeader, reader: Io.Reader) !void {
	1259	const sig = try reader.take(4);
	1260	if (!mem.eql(u8, sig, signature)) return error.InvalidHeader;
	1261	const version = try reader.takeInt(u32, .big);
	1262	if (version != supported_version) return error.UnsupportedVersion;
	1263	try reader.readSliceEndian(u32, &index_header.fan_out_table, .big);
	1264	}
	1265	};
	1266
	1267	const IndexEntry = struct {
	1268	offset: u64,
	1269	crc32: u32,
	1270	};
	1271
	1272	/// Writes out a version 2 index for the given packfile, as documented in
	1273	/// [pack-format](https://git-scm.com/docs/pack-format).
	1274	pub fn indexPack(
	1275	allocator: Allocator,
	1276	format: Oid.Format,
	1277	pack: *Io.File.Reader,
	1278	index_writer: *Io.File.Writer,
	1279	) !void {
	1280	try pack.seekTo(0);
	1281
	1282	var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty;
	1283	defer index_entries.deinit(allocator);
	1284	var pending_deltas: std.ArrayList(IndexEntry) = .empty;
	1285	defer pending_deltas.deinit(allocator);
	1286
	1287	const pack_checksum = try indexPackFirstPass(allocator, format, pack, &index_entries, &pending_deltas);
	1288
	1289	var cache: ObjectCache = .{};
	1290	defer cache.deinit(allocator);
	1291	var remaining_deltas = pending_deltas.items.len;
	1292	while (remaining_deltas > 0) {
	1293	var i: usize = remaining_deltas;
	1294	while (i > 0) {
	1295	i -= 1;
	1296	const delta = pending_deltas.items[i];
	1297	if (try indexPackHashDelta(allocator, format, pack, delta, index_entries, &cache)) \|oid\| {
	1298	try index_entries.put(allocator, oid, delta);
	1299	_ = pending_deltas.swapRemove(i);
	1300	}
	1301	}
	1302	if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack;
	1303	remaining_deltas = pending_deltas.items.len;
	1304	}
	1305
	1306	var oids: std.ArrayList(Oid) = .empty;
	1307	defer oids.deinit(allocator);
	1308	try oids.ensureTotalCapacityPrecise(allocator, index_entries.count());
	1309	var index_entries_iter = index_entries.iterator();
	1310	while (index_entries_iter.next()) \|entry\| {
	1311	oids.appendAssumeCapacity(entry.key_ptr.*);
	1312	}
	1313	mem.sortUnstable(Oid, oids.items, {}, struct {
	1314	fn lessThan(_: void, o1: Oid, o2: Oid) bool {
	1315	return mem.lessThan(u8, o1.slice(), o2.slice());
	1316	}
	1317	}.lessThan);
	1318
	1319	var fan_out_table: [256]u32 = undefined;
	1320	var count: u32 = 0;
	1321	var fan_out_index: u8 = 0;
	1322	for (oids.items) \|oid\| {
	1323	const key = oid.slice()[0];
	1324	if (key > fan_out_index) {
	1325	@memset(fan_out_table[fan_out_index..key], count);
	1326	fan_out_index = key;
	1327	}
	1328	count += 1;
	1329	}
	1330	@memset(fan_out_table[fan_out_index..], count);
	1331
	1332	var index_hashed_writer = Io.Writer.hashed(&index_writer.interface, Oid.Hasher.init(format), &.{});
	1333	const writer = &index_hashed_writer.writer;
	1334	try writer.writeAll(IndexHeader.signature);
	1335	try writer.writeInt(u32, IndexHeader.supported_version, .big);
	1336	for (fan_out_table) \|fan_out_entry\| {
	1337	try writer.writeInt(u32, fan_out_entry, .big);
	1338	}
	1339
	1340	for (oids.items) \|oid\| {
	1341	try writer.writeAll(oid.slice());
	1342	}
	1343
	1344	for (oids.items) \|oid\| {
	1345	try writer.writeInt(u32, index_entries.get(oid).?.crc32, .big);
	1346	}
	1347
	1348	var big_offsets: std.ArrayList(u64) = .empty;
	1349	defer big_offsets.deinit(allocator);
	1350	for (oids.items) \|oid\| {
	1351	const offset = index_entries.get(oid).?.offset;
	1352	if (offset <= std.math.maxInt(u31)) {
	1353	try writer.writeInt(u32, @intCast(offset), .big);
	1354	} else {
	1355	const index = big_offsets.items.len;
	1356	try big_offsets.append(allocator, offset);
	1357	try writer.writeInt(u32, @as(u32, @intCast(index)) \| (1 << 31), .big);
	1358	}
	1359	}
	1360	for (big_offsets.items) \|offset\| {
	1361	try writer.writeInt(u64, offset, .big);
	1362	}
	1363
	1364	try writer.writeAll(pack_checksum.slice());
	1365	const index_checksum = index_hashed_writer.hasher.finalResult();
	1366	try index_writer.interface.writeAll(index_checksum.slice());
	1367	try index_writer.end();
	1368	}
	1369
	1370	/// Performs the first pass over the packfile data for index construction.
	1371	/// This will index all non-delta objects, queue delta objects for further
	1372	/// processing, and return the pack checksum (which is part of the index
	1373	/// format).
	1374	fn indexPackFirstPass(
	1375	allocator: Allocator,
	1376	format: Oid.Format,
	1377	pack: *Io.File.Reader,
	1378	index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),
	1379	pending_deltas: *std.ArrayList(IndexEntry),
	1380	) !Oid {
	1381	var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
	1382	var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system.
	1383	var pack_hashed = pack.interface.hashed(Oid.Hasher.init(format), &pack_buffer);
	1384
	1385	const pack_header = try PackHeader.read(&pack_hashed.reader);
	1386
	1387	for (0..pack_header.total_objects) \|_\| {
	1388	const entry_offset = pack.logicalPos() - pack_hashed.reader.bufferedLen();
	1389	const entry_header = try EntryHeader.read(format, &pack_hashed.reader);
	1390	switch (entry_header) {
	1391	.commit, .tree, .blob, .tag => \|object\| {
	1392	var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &.{});
	1393	var oid_hasher: Oid.Hashing = .init(format, &flate_buffer);
	1394	const oid_hasher_w = oid_hasher.writer();
	1395	// The object header is not included in the pack data but is
	1396	// part of the object's ID
	1397	try oid_hasher_w.print("{t} {d}\x00", .{ entry_header, object.uncompressed_length });
	1398	const n = try entry_decompress.reader.streamRemaining(oid_hasher_w);
	1399	if (n != object.uncompressed_length) return error.InvalidObject;
	1400	const oid = oid_hasher.final();
	1401	if (!skip_checksums) @compileError("TODO");
	1402	try index_entries.put(allocator, oid, .{
	1403	.offset = entry_offset,
	1404	.crc32 = 0,
	1405	});
	1406	},
	1407	inline .ofs_delta, .ref_delta => \|delta\| {
	1408	var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &flate_buffer);
	1409	const n = try entry_decompress.reader.discardRemaining();
	1410	if (n != delta.uncompressed_length) return error.InvalidObject;
	1411	if (!skip_checksums) @compileError("TODO");
	1412	try pending_deltas.append(allocator, .{
	1413	.offset = entry_offset,
	1414	.crc32 = 0,
	1415	});
	1416	},
	1417	}
	1418	}
	1419
	1420	if (!skip_checksums) @compileError("TODO");
	1421	return pack_hashed.hasher.finalResult();
	1422	}
	1423
	1424	/// Attempts to determine the final object ID of the given deltified object.
	1425	/// May return null if this is not yet possible (if the delta is a ref-based
	1426	/// delta and we do not yet know the offset of the base object).
	1427	fn indexPackHashDelta(
	1428	allocator: Allocator,
	1429	format: Oid.Format,
	1430	pack: *Io.File.Reader,
	1431	delta: IndexEntry,
	1432	index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),
	1433	cache: *ObjectCache,
	1434	) !?Oid {
	1435	// Figure out the chain of deltas to resolve
	1436	var base_offset = delta.offset;
	1437	var base_header: EntryHeader = undefined;
	1438	var delta_offsets: std.ArrayList(u64) = .empty;
	1439	defer delta_offsets.deinit(allocator);
	1440	const base_object = while (true) {
	1441	if (cache.get(base_offset)) \|base_object\| break base_object;
	1442
	1443	try pack.seekTo(base_offset);
	1444	base_header = try EntryHeader.read(format, &pack.interface);
	1445	switch (base_header) {
	1446	.ofs_delta => \|ofs_delta\| {
	1447	try delta_offsets.append(allocator, base_offset);
	1448	base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject;
	1449	},
	1450	.ref_delta => \|ref_delta\| {
	1451	try delta_offsets.append(allocator, base_offset);
	1452	base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset;
	1453	},
	1454	else => {
	1455	const base_data = try readObjectRaw(allocator, &pack.interface, base_header.uncompressedLength());
	1456	errdefer allocator.free(base_data);
	1457	const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
	1458	try cache.put(allocator, base_offset, base_object);
	1459	break base_object;
	1460	},
	1461	}
	1462	};
	1463
	1464	const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache);
	1465
	1466	var entry_hasher_buffer: [64]u8 = undefined;
	1467	var entry_hasher: Oid.Hashing = .init(format, &entry_hasher_buffer);
	1468	const entry_hasher_w = entry_hasher.writer();
	1469	// Writes to hashers cannot fail.
	1470	entry_hasher_w.print("{t} {d}\x00", .{ base_object.type, base_data.len }) catch unreachable;
	1471	entry_hasher_w.writeAll(base_data) catch unreachable;
	1472	return entry_hasher.final();
	1473	}
	1474
	1475	/// Resolves a chain of deltas, returning the final base object data. `pack` is
	1476	/// assumed to be looking at the start of the object data for the base object of
	1477	/// the chain, and will then apply the deltas in `delta_offsets` in reverse order
	1478	/// to obtain the final object.
	1479	fn resolveDeltaChain(
	1480	allocator: Allocator,
	1481	format: Oid.Format,
	1482	pack: *Io.File.Reader,
	1483	base_object: Object,
	1484	delta_offsets: []const u64,
	1485	cache: *ObjectCache,
	1486	) ![]const u8 {
	1487	var base_data = base_object.data;
	1488	var i: usize = delta_offsets.len;
	1489	while (i > 0) {
	1490	i -= 1;
	1491
	1492	const delta_offset = delta_offsets[i];
	1493	try pack.seekTo(delta_offset);
	1494	const delta_header = try EntryHeader.read(format, &pack.interface);
	1495	const delta_data = try readObjectRaw(allocator, &pack.interface, delta_header.uncompressedLength());
	1496	defer allocator.free(delta_data);
	1497	var delta_reader: Io.Reader = .fixed(delta_data);
	1498	_ = try delta_reader.takeLeb128(u64); // base object size
	1499	const expanded_size = try delta_reader.takeLeb128(u64);
	1500
	1501	const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge;
	1502	const expanded_data = try allocator.alloc(u8, expanded_alloc_size);
	1503	errdefer allocator.free(expanded_data);
	1504	var expanded_delta_stream: Io.Writer = .fixed(expanded_data);
	1505	try expandDelta(base_data, &delta_reader, &expanded_delta_stream);
	1506	if (expanded_delta_stream.end != expanded_size) return error.InvalidObject;
	1507
	1508	try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data });
	1509	base_data = expanded_data;
	1510	}
	1511	return base_data;
	1512	}
	1513
	1514	/// Reads the complete contents of an object from `reader`. This function may
	1515	/// read more bytes than required from `reader`, so the reader position after
	1516	/// returning is not reliable.
	1517	fn readObjectRaw(allocator: Allocator, reader: *Io.Reader, size: u64) ![]u8 {
	1518	const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;
	1519	var aw: Io.Writer.Allocating = .init(allocator);
	1520	try aw.ensureTotalCapacity(alloc_size + std.compress.flate.max_window_len);
	1521	defer aw.deinit();
	1522	var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &.{});
	1523	try decompress.reader.streamExact(&aw.writer, alloc_size);
	1524	return aw.toOwnedSlice();
	1525	}
	1526
	1527	/// Expands delta data from `delta_reader` to `writer`.
	1528	///
	1529	/// The format of the delta data is documented in
	1530	/// [pack-format](https://git-scm.com/docs/pack-format).
	1531	fn expandDelta(base_object: []const u8, delta_reader: Io.Reader, writer: Io.Writer) !void {
	1532	while (true) {
	1533	const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch \|e\| switch (e) {
	1534	error.EndOfStream => return,
	1535	else => \|other\| return other,
	1536	});
	1537	if (inst.copy) {
	1538	const available: packed struct {
	1539	offset1: bool,
	1540	offset2: bool,
	1541	offset3: bool,
	1542	offset4: bool,
	1543	size1: bool,
	1544	size2: bool,
	1545	size3: bool,
	1546	} = @bitCast(inst.value);
	1547	const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{
	1548	.offset1 = if (available.offset1) try delta_reader.takeByte() else 0,
	1549	.offset2 = if (available.offset2) try delta_reader.takeByte() else 0,
	1550	.offset3 = if (available.offset3) try delta_reader.takeByte() else 0,
	1551	.offset4 = if (available.offset4) try delta_reader.takeByte() else 0,
	1552	};
	1553	const base_offset: u32 = @bitCast(offset_parts);
	1554	const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{
	1555	.size1 = if (available.size1) try delta_reader.takeByte() else 0,
	1556	.size2 = if (available.size2) try delta_reader.takeByte() else 0,
	1557	.size3 = if (available.size3) try delta_reader.takeByte() else 0,
	1558	};
	1559	var size: u24 = @bitCast(size_parts);
	1560	if (size == 0) size = 0x10000;
	1561	try writer.writeAll(base_object[base_offset..][0..size]);
	1562	} else if (inst.value != 0) {
	1563	try delta_reader.streamExact(writer, inst.value);
	1564	} else {
	1565	return error.InvalidDeltaInstruction;
	1566	}
	1567	}
	1568	}
	1569
	1570	/// Runs the packfile indexing and checkout test.
	1571	///
	1572	/// The two testrepo repositories under testdata contain identical commit
	1573	/// histories and contents.
	1574	///
	1575	/// To verify the contents of the packfiles using Git alone, run the
	1576	/// following commands in an empty directory:
	1577	///
	1578	/// 1. `git init --object-format=(sha1\|sha256)`
	1579	/// 2. `git unpack-objects <path/to/testrepo.pack`
	1580	/// 3. `git fsck` - will print one "dangling commit":
	1581	/// - SHA-1: `dd582c0720819ab7130b103635bd7271b9fd4feb`
	1582	/// - SHA-256: `7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a`
	1583	/// 4. `git checkout $commit`
	1584	fn runRepositoryTest(io: Io, comptime format: Oid.Format, head_commit: []const u8) !void {
	1585	const testrepo_pack = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".pack");
	1586
	1587	var git_dir = testing.tmpDir(.{});
	1588	defer git_dir.cleanup();
	1589	var pack_file = try git_dir.dir.createFile(io, "testrepo.pack", .{ .read = true });
	1590	defer pack_file.close(io);
	1591	try pack_file.writeStreamingAll(io, testrepo_pack);
	1592
	1593	var pack_file_buffer: [2000]u8 = undefined;
	1594	var pack_file_reader = pack_file.reader(io, &pack_file_buffer);
	1595
	1596	var index_file = try git_dir.dir.createFile(io, "testrepo.idx", .{ .read = true });
	1597	defer index_file.close(io);
	1598	var index_file_buffer: [2000]u8 = undefined;
	1599	var index_file_writer = index_file.writer(io, &index_file_buffer);
	1600	try indexPack(testing.allocator, format, &pack_file_reader, &index_file_writer);
	1601
	1602	// Arbitrary size limit on files read while checking the repository contents
	1603	// (all files in the test repo are known to be smaller than this)
	1604	const max_file_size = 8192;
	1605
	1606	if (!skip_checksums) {
	1607	const index_file_data = try git_dir.dir.readFileAlloc(io, "testrepo.idx", testing.allocator, .limited(max_file_size));
	1608	defer testing.allocator.free(index_file_data);
	1609	// testrepo.idx is generated by Git. The index created by this file should
	1610	// match it exactly. Running `git verify-pack -v testrepo.pack` can verify
	1611	// this.
	1612	const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx");
	1613	try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);
	1614	}
	1615
	1616	var index_file_reader = index_file.reader(io, &index_file_buffer);
	1617	var repository: Repository = undefined;
	1618	try repository.init(testing.allocator, format, &pack_file_reader, &index_file_reader);
	1619	defer repository.deinit();
	1620
	1621	var worktree = testing.tmpDir(.{ .iterate = true });
	1622	defer worktree.cleanup();
	1623
	1624	const commit_id = try Oid.parse(format, head_commit);
	1625
	1626	var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
	1627	defer diagnostics.deinit();
	1628	try repository.checkout(io, worktree.dir, commit_id, &diagnostics);
	1629	try testing.expect(diagnostics.errors.items.len == 0);
	1630
	1631	const expected_files: []const []const u8 = &.{
	1632	"dir/file",
	1633	"dir/subdir/file",
	1634	"dir/subdir/file2",
	1635	"dir2/file",
	1636	"dir3/file",
	1637	"dir3/file2",
	1638	"file",
	1639	"file2",
	1640	"file3",
	1641	"file4",
	1642	"file5",
	1643	"file6",
	1644	"file7",
	1645	"file8",
	1646	"file9",
	1647	};
	1648	var actual_files: std.ArrayList([]u8) = .empty;
	1649	defer actual_files.deinit(testing.allocator);
	1650	defer for (actual_files.items) \|file\| testing.allocator.free(file);
	1651	var walker = try worktree.dir.walk(testing.allocator);
	1652	defer walker.deinit();
	1653	while (try walker.next(io)) \|entry\| {
	1654	if (entry.kind != .file) continue;
	1655	const path = try testing.allocator.dupe(u8, entry.path);
	1656	errdefer testing.allocator.free(path);
	1657	mem.replaceScalar(u8, path, std.fs.path.sep, '/');
	1658	try actual_files.append(testing.allocator, path);
	1659	}
	1660	mem.sortUnstable([]u8, actual_files.items, {}, struct {
	1661	fn lessThan(_: void, a: []u8, b: []u8) bool {
	1662	return mem.lessThan(u8, a, b);
	1663	}
	1664	}.lessThan);
	1665	try testing.expectEqualDeep(expected_files, actual_files.items);
	1666
	1667	const expected_file_contents =
	1668	\\revision 1
	1669	\\revision 2
	1670	\\revision 4
	1671	\\revision 5
	1672	\\revision 7
	1673	\\revision 8
	1674	\\revision 9
	1675	\\revision 10
	1676	\\revision 12
	1677	\\revision 13
	1678	\\revision 14
	1679	\\revision 18
	1680	\\revision 19
	1681	\\
	1682	;
	1683	const actual_file_contents = try worktree.dir.readFileAlloc(io, "file", testing.allocator, .limited(max_file_size));
	1684	defer testing.allocator.free(actual_file_contents);
	1685	try testing.expectEqualStrings(expected_file_contents, actual_file_contents);
	1686	}
	1687
	1688	/// Checksum calculation is useful for troubleshooting and debugging, but it's
	1689	/// redundant since the package manager already does content hashing at the
	1690	/// end. Let's save time by not doing that work, but, I left a cookie crumb
	1691	/// trail here if you want to restore the functionality for tinkering purposes.
	1692	const skip_checksums = true;
	1693
	1694	test "SHA-1 packfile indexing and checkout" {
	1695	try runRepositoryTest(std.testing.io, .sha1, "dd582c0720819ab7130b103635bd7271b9fd4feb");
	1696	}
	1697
	1698	test "SHA-256 packfile indexing and checkout" {
	1699	try runRepositoryTest(std.testing.io, .sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a");
	1700	}
	1701
	1702	/// Checks out a commit of a packfile. Intended for experimenting with and
	1703	/// benchmarking possible optimizations to the indexing and checkout behavior.
	1704	pub fn main() !void {
	1705	const allocator = std.heap.smp_allocator;
	1706
	1707	var threaded: Io.Threaded = .init(allocator, .{});
	1708	defer threaded.deinit();
	1709	const io = threaded.io();
	1710
	1711	const args = try std.process.argsAlloc(allocator);
	1712	defer std.process.argsFree(allocator, args);
	1713	if (args.len != 5) {
	1714	return error.InvalidArguments; // Arguments: format packfile commit worktree
	1715	}
	1716
	1717	const format = std.meta.stringToEnum(Oid.Format, args[1]) orelse return error.InvalidFormat;
	1718
	1719	var pack_file = try Io.Dir.cwd().openFile(io, args[2], .{});
	1720	defer pack_file.close(io);
	1721	var pack_file_buffer: [4096]u8 = undefined;
	1722	var pack_file_reader = pack_file.reader(io, &pack_file_buffer);
	1723
	1724	const commit = try Oid.parse(format, args[3]);
	1725	var worktree = try Io.Dir.cwd().createDirPathOpen(io, args[4], .{});
	1726	defer worktree.close(io);
	1727
	1728	var git_dir = try worktree.createDirPathOpen(io, ".git", .{});
	1729	defer git_dir.close(io);
	1730
	1731	std.debug.print("Starting index...\n", .{});
	1732	var index_file = try git_dir.createFile(io, "idx", .{ .read = true });
	1733	defer index_file.close(io);
	1734	var index_file_buffer: [4096]u8 = undefined;
	1735	var index_file_writer = index_file.writer(io, &index_file_buffer);
	1736	try indexPack(allocator, format, &pack_file_reader, &index_file_writer);
	1737
	1738	std.debug.print("Starting checkout...\n", .{});
	1739	var index_file_reader = index_file.reader(io, &index_file_buffer);
	1740	var repository: Repository = undefined;
	1741	try repository.init(allocator, format, &pack_file_reader, &index_file_reader);
	1742	defer repository.deinit();
	1743	var diagnostics: Diagnostics = .{ .allocator = allocator };
	1744	defer diagnostics.deinit();
	1745	try repository.checkout(io, worktree, commit, &diagnostics);
	1746
	1747	for (diagnostics.errors.items) \|err\| {
	1748	std.debug.print("Diagnostic: {}\n", .{err});
	1749	}
	1750	}

lib/compiler/Maker/Fetch/git/testdata/testrepo-sha1.idx created

Binary files /dev/null and b/lib/compiler/Maker/Fetch/git/testdata/testrepo-sha1.idx differ

lib/compiler/Maker/Fetch/git/testdata/testrepo-sha1.pack created

Binary files /dev/null and b/lib/compiler/Maker/Fetch/git/testdata/testrepo-sha1.pack differ

lib/compiler/Maker/Fetch/git/testdata/testrepo-sha256.idx created

Binary files /dev/null and b/lib/compiler/Maker/Fetch/git/testdata/testrepo-sha256.idx differ

lib/compiler/Maker/Fetch/git/testdata/testrepo-sha256.pack created

Binary files /dev/null and b/lib/compiler/Maker/Fetch/git/testdata/testrepo-sha256.pack differ

lib/compiler/Maker/Package.zig created+207

...	...	@@ -0,0 +1,207 @@
	1	const std = @import("std");
	2	const assert = std.debug.assert;
	3
	4	pub const Fetch = @import("Package/Fetch.zig");
	5	pub const Manifest = @import("Package/Manifest.zig");
	6
	7	pub const Fingerprint = packed struct(u64) {
	8	id: u32,
	9	checksum: u32,
	10
	11	pub fn generate(rng: std.Random, name: []const u8) Fingerprint {
	12	return .{
	13	.id = rng.intRangeLessThan(u32, 1, 0xffffffff),
	14	.checksum = std.hash.Crc32.hash(name),
	15	};
	16	}
	17
	18	pub fn validate(n: Fingerprint, name: []const u8) bool {
	19	switch (n.id) {
	20	0x00000000, 0xffffffff => return false,
	21	else => return std.hash.Crc32.hash(name) == n.checksum,
	22	}
	23	}
	24
	25	pub fn int(n: Fingerprint) u64 {
	26	return @bitCast(n);
	27	}
	28	};
	29
	30	/// A user-readable, file system safe hash that identifies an exact package
	31	/// snapshot, including file contents.
	32	///
	33	/// The hash is not only to prevent collisions but must resist attacks where
	34	/// the adversary fully controls the contents being hashed. Thus, it contains
	35	/// a full SHA-256 digest.
	36	///
	37	/// This data structure can be used to store the legacy hash format too. Legacy
	38	/// hash format is scheduled to be removed after 0.14.0 is tagged.
	39	///
	40	/// There's also a third way this structure is used. When using path rather than
	41	/// hash, a unique hash is still needed, so one is computed based on the path.
	42	pub const Hash = struct {
	43	/// Maximum size of a package hash. Unused bytes at the end are
	44	/// filled with zeroes.
	45	///
	46	/// Assumed to be already validated.
	47	bytes: [max_len]u8,
	48
	49	pub const Algo = std.crypto.hash.sha2.Sha256;
	50	pub const Digest = [Algo.digest_length]u8;
	51
	52	/// Example: "nnnn-vvvv-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh"
	53	pub const max_len = 32 + 1 + 32 + 1 + (32 + 32 + 200) / 6;
	54
	55	/// Asserts `s` is valid.
	56	pub fn fromSlice(s: []const u8) Hash {
	57	assert(validate(s) == .ok);
	58	var result: Hash = undefined;
	59	@memcpy(result.bytes[0..s.len], s);
	60	@memset(result.bytes[s.len..], 0);
	61	return result;
	62	}
	63
	64	pub const Validation = enum { ok, short, long, incomplete };
	65
	66	pub fn validate(s: []const u8) Validation {
	67	if (s.len > max_len) return .long;
	68	if (s.len < 44) return .short;
	69	const n_dashes = std.mem.countScalar(u8, s[0 .. s.len - 44], '-');
	70	if (n_dashes < 2) return .incomplete;
	71	return .ok;
	72	}
	73
	74	test validate {
	75	try std.testing.expectEqual(.short, validate(""));
	76	}
	77
	78	pub fn toSlice(ph: *const Hash) []const u8 {
	79	var end: usize = ph.bytes.len;
	80	while (true) {
	81	end -= 1;
	82	if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1];
	83	}
	84	}
	85
	86	pub fn eql(a: const Hash, b: const Hash) bool {
	87	return std.mem.eql(u8, &a.bytes, &b.bytes);
	88	}
	89
	90	/// Produces "$name-$semver-$hashplus".
	91	/// * name is the name field from build.zig.zon, asserted to be at most 32
	92	/// bytes and assumed be a valid zig identifier
	93	/// * semver is the version field from build.zig.zon, asserted to be at
	94	/// most 32 bytes
	95	/// * hashplus is the following 33-byte array, base64 encoded using -_ to make
	96	/// it filesystem safe:
	97	/// - (4 bytes) LE u32 Package ID
	98	/// - (4 bytes) LE u32 total decompressed size in bytes, overflow saturated
	99	/// - (25 bytes) truncated SHA-256 digest of hashed files of the package
	100	pub fn init(digest: Digest, name: []const u8, ver: []const u8, id: u32, size: u32) Hash {
	101	assert(name.len <= 32);
	102	assert(ver.len <= 32);
	103	var result: Hash = undefined;
	104	var buf: std.ArrayList(u8) = .initBuffer(&result.bytes);
	105	buf.appendSliceAssumeCapacity(name);
	106	buf.appendAssumeCapacity('-');
	107	buf.appendSliceAssumeCapacity(ver);
	108	buf.appendAssumeCapacity('-');
	109	var hashplus: [33]u8 = undefined;
	110	std.mem.writeInt(u32, hashplus[0..4], id, .little);
	111	std.mem.writeInt(u32, hashplus[4..8], size, .little);
	112	hashplus[8..].* = digest[0..25].*;
	113	_ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(44), &hashplus);
	114	@memset(buf.unusedCapacitySlice(), 0);
	115	return result;
	116	}
	117
	118	/// Produces a unique hash based on the path provided. The result should
	119	/// not be user-visible.
	120	pub fn initPath(sub_path: []const u8, is_global: bool) Hash {
	121	var result: Hash = .{ .bytes = @splat(0) };
	122	var i: usize = 0;
	123	if (is_global) {
	124	result.bytes[0] = '/';
	125	i += 1;
	126	}
	127	if (i + sub_path.len <= result.bytes.len) {
	128	@memcpy(result.bytes[i..][0..sub_path.len], sub_path);
	129	return result;
	130	}
	131	var bin_digest: [Algo.digest_length]u8 = undefined;
	132	Algo.hash(sub_path, &bin_digest, .{});
	133	_ = std.fmt.bufPrint(result.bytes[i..], "{x}", .{&bin_digest}) catch unreachable;
	134	return result;
	135	}
	136
	137	pub fn projectId(hash: *const Hash) ProjectId {
	138	const bytes = hash.toSlice();
	139	const name = std.mem.sliceTo(bytes, '-');
	140	const encoded_hashplus = bytes[bytes.len - 44 ..];
	141	var hashplus: [33]u8 = undefined;
	142	std.base64.url_safe_no_pad.Decoder.decode(&hashplus, encoded_hashplus) catch unreachable;
	143	const fingerprint_id = std.mem.readInt(u32, hashplus[0..4], .little);
	144	return .init(name, fingerprint_id);
	145	}
	146
	147	test projectId {
	148	const hash: Hash = .fromSlice("pulseaudio-16.1.1-9-mk_62MZkNwBaFwiZ7ZVrYRIf_3dTqqJR5PbMRCJzSuLw");
	149	const project_id = hash.projectId();
	150
	151	var expected_name: [32]u8 = @splat(0);
	152	expected_name[0.."pulseaudio".len].* = "pulseaudio".*;
	153	try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name);
	154
	155	try std.testing.expectEqual(0xd8fa4f9a, project_id.fingerprint_id);
	156	}
	157
	158	test "projectId with dashes in the base64" {
	159	const hash: Hash = .fromSlice("dvui-0.4.0-dev-AQFJmayi2gAKE7FeJoF61v5U1IV9-SupoEcFutIZYpkC");
	160	const project_id = hash.projectId();
	161
	162	var expected_name: [32]u8 = @splat(0);
	163	expected_name[0.."dvui".len].* = "dvui".*;
	164	try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name);
	165
	166	try std.testing.expectEqual(0x99490101, project_id.fingerprint_id);
	167	}
	168	};
	169
	170	/// Minimum information required to identify whether a package is an artifact
	171	/// of a given project.
	172	pub const ProjectId = struct {
	173	/// Bytes after name.len are set to zero.
	174	padded_name: [32]u8,
	175	fingerprint_id: u32,
	176
	177	pub fn init(name: []const u8, fingerprint_id: u32) ProjectId {
	178	var padded_name: [32]u8 = @splat(0);
	179	@memcpy(padded_name[0..name.len], name);
	180	return .{
	181	.padded_name = padded_name,
	182	.fingerprint_id = fingerprint_id,
	183	};
	184	}
	185
	186	pub fn eql(a: const ProjectId, b: const ProjectId) bool {
	187	return a.fingerprint_id == b.fingerprint_id and std.mem.eql(u8, &a.padded_name, &b.padded_name);
	188	}
	189
	190	pub fn hash(a: *const ProjectId) u64 {
	191	const x: u64 = @bitCast(a.padded_name[0..8].*);
	192	return std.hash.int(x \| a.fingerprint_id);
	193	}
	194	};
	195
	196	test Hash {
	197	const example_digest: Hash.Digest = .{
	198	0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87,
	199	0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f,
	200	};
	201	const result: Hash = .init(example_digest, "nasm", "2.16.1-3", 0xcafebabe, 10 * 1024 * 1024);
	202	try std.testing.expectEqualStrings("nasm-2.16.1-3-vrr-ygAAoADH9XG3tOdvPNuHen_d-XeHndOG-nNXmved", result.toSlice());
	203	}
	204
	205	test {
	206	_ = Fetch;
	207	}

lib/compiler/Maker/Package/Manifest.zig created+734

...	...	@@ -0,0 +1,734 @@
	1	const Manifest = @This();
	2
	3	const std = @import("std");
	4	const Io = std.Io;
	5	const mem = std.mem;
	6	const Allocator = std.mem.Allocator;
	7	const assert = std.debug.assert;
	8	const Ast = std.zig.Ast;
	9	const testing = std.testing;
	10
	11	const Package = @import("../Package.zig");
	12
	13	pub const max_bytes = 10 * 1024 * 1024;
	14	pub const basename = "build.zig.zon";
	15	pub const max_name_len = 32;
	16	pub const max_version_len = 32;
	17
	18	pub const Dependency = struct {
	19	location: Location,
	20	location_tok: Ast.TokenIndex,
	21	location_node: Ast.Node.Index,
	22	hash: ?[]const u8,
	23	hash_tok: Ast.OptionalTokenIndex,
	24	hash_node: Ast.Node.OptionalIndex,
	25	node: Ast.Node.Index,
	26	name_tok: Ast.TokenIndex,
	27	lazy: bool,
	28
	29	pub const Location = union(enum) {
	30	url: []const u8,
	31	path: []const u8,
	32	};
	33	};
	34
	35	pub const ErrorMessage = struct {
	36	msg: []const u8,
	37	tok: Ast.TokenIndex,
	38	off: u32,
	39	};
	40
	41	name: []const u8,
	42	id: u32,
	43	version: std.SemanticVersion,
	44	version_node: Ast.Node.Index,
	45	dependencies: std.array_hash_map.String(Dependency),
	46	dependencies_node: Ast.Node.OptionalIndex,
	47	paths: std.array_hash_map.String(void),
	48	minimum_zig_version: ?std.SemanticVersion,
	49
	50	errors: []ErrorMessage,
	51	arena_state: std.heap.ArenaAllocator.State,
	52
	53	pub const ParseOptions = struct {
	54	allow_missing_paths_field: bool = false,
	55	};
	56
	57	pub const Error = Allocator.Error;
	58
	59	pub fn parse(gpa: Allocator, ast: *const Ast, rng: std.Random, options: ParseOptions) Error!Manifest {
	60	const main_node_index = ast.nodeData(.root).node;
	61
	62	var arena_instance = std.heap.ArenaAllocator.init(gpa);
	63	errdefer arena_instance.deinit();
	64
	65	var p: Parse = .{
	66	.gpa = gpa,
	67	.ast = ast.*,
	68	.arena = arena_instance.allocator(),
	69	.errors = .empty,
	70
	71	.name = undefined,
	72	.id = 0,
	73	.version = undefined,
	74	.version_node = undefined,
	75	.dependencies = .{},
	76	.dependencies_node = .none,
	77	.paths = .empty,
	78	.allow_missing_paths_field = options.allow_missing_paths_field,
	79	.minimum_zig_version = null,
	80	.buf = .empty,
	81	};
	82	defer p.buf.deinit(gpa);
	83	defer p.errors.deinit(gpa);
	84	defer p.dependencies.deinit(gpa);
	85	defer p.paths.deinit(gpa);
	86
	87	p.parseRoot(main_node_index, rng) catch \|err\| switch (err) {
	88	error.ParseFailure => assert(p.errors.items.len > 0),
	89	else => \|e\| return e,
	90	};
	91
	92	return .{
	93	.name = p.name,
	94	.id = p.id,
	95	.version = p.version,
	96	.version_node = p.version_node,
	97	.dependencies = try p.dependencies.clone(p.arena),
	98	.dependencies_node = p.dependencies_node,
	99	.paths = try p.paths.clone(p.arena),
	100	.minimum_zig_version = p.minimum_zig_version,
	101	.errors = try p.arena.dupe(ErrorMessage, p.errors.items),
	102	.arena_state = arena_instance.state,
	103	};
	104	}
	105
	106	pub fn deinit(man: *Manifest, gpa: Allocator) void {
	107	man.arena_state.promote(gpa).deinit();
	108	man.* = undefined;
	109	}
	110
	111	pub fn copyErrorsIntoBundle(
	112	man: Manifest,
	113	ast: Ast,
	114	/// ErrorBundle null-terminated string index
	115	src_path: u32,
	116	eb: *std.zig.ErrorBundle.Wip,
	117	) Allocator.Error!void {
	118	for (man.errors) \|msg\| {
	119	const start_loc = ast.tokenLocation(0, msg.tok);
	120
	121	try eb.addRootErrorMessage(.{
	122	.msg = try eb.addString(msg.msg),
	123	.src_loc = try eb.addSourceLocation(.{
	124	.src_path = src_path,
	125	.span_start = ast.tokenStart(msg.tok),
	126	.span_end = @intCast(ast.tokenStart(msg.tok) + ast.tokenSlice(msg.tok).len),
	127	.span_main = ast.tokenStart(msg.tok) + msg.off,
	128	.line = @intCast(start_loc.line),
	129	.column = @intCast(start_loc.column),
	130	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
	131	}),
	132	});
	133	}
	134	}
	135
	136	const Parse = struct {
	137	gpa: Allocator,
	138	ast: Ast,
	139	arena: Allocator,
	140	buf: std.ArrayList(u8),
	141	errors: std.ArrayList(ErrorMessage),
	142
	143	name: []const u8,
	144	id: u32,
	145	version: std.SemanticVersion,
	146	version_node: Ast.Node.Index,
	147	dependencies: std.array_hash_map.String(Dependency),
	148	dependencies_node: Ast.Node.OptionalIndex,
	149	paths: std.array_hash_map.String(void),
	150	allow_missing_paths_field: bool,
	151	minimum_zig_version: ?std.SemanticVersion,
	152
	153	const InnerError = error{ ParseFailure, OutOfMemory };
	154
	155	fn parseRoot(p: *Parse, node: Ast.Node.Index, rng: std.Random) !void {
	156	const ast = p.ast;
	157	const main_token = ast.nodeMainToken(node);
	158
	159	var buf: [2]Ast.Node.Index = undefined;
	160	const struct_init = ast.fullStructInit(&buf, node) orelse {
	161	return fail(p, main_token, "expected top level expression to be a struct", .{});
	162	};
	163
	164	var have_name = false;
	165	var have_version = false;
	166	var have_included_paths = false;
	167	var fingerprint: ?Package.Fingerprint = null;
	168
	169	for (struct_init.ast.fields) \|field_init\| {
	170	const name_token = ast.firstToken(field_init) - 2;
	171	const field_name = try identifierTokenString(p, name_token);
	172	// We could get fancy with reflection and comptime logic here but doing
	173	// things manually provides an opportunity to do any additional verification
	174	// that is desirable on a per-field basis.
	175	if (mem.eql(u8, field_name, "dependencies")) {
	176	p.dependencies_node = field_init.toOptional();
	177	try parseDependencies(p, field_init);
	178	} else if (mem.eql(u8, field_name, "paths")) {
	179	have_included_paths = true;
	180	try parseIncludedPaths(p, field_init);
	181	} else if (mem.eql(u8, field_name, "name")) {
	182	p.name = try parseName(p, field_init);
	183	have_name = true;
	184	} else if (mem.eql(u8, field_name, "fingerprint")) {
	185	fingerprint = try parseFingerprint(p, field_init);
	186	} else if (mem.eql(u8, field_name, "version")) {
	187	p.version_node = field_init;
	188	const version_text = try parseString(p, field_init);
	189	if (version_text.len > max_version_len) {
	190	try appendError(p, ast.nodeMainToken(field_init), "version string length {d} exceeds maximum of {d}", .{ version_text.len, max_version_len });
	191	}
	192	p.version = std.SemanticVersion.parse(version_text) catch \|err\| v: {
	193	try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)});
	194	break :v undefined;
	195	};
	196	have_version = true;
	197	} else if (mem.eql(u8, field_name, "minimum_zig_version")) {
	198	const version_text = try parseString(p, field_init);
	199	p.minimum_zig_version = std.SemanticVersion.parse(version_text) catch \|err\| v: {
	200	try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)});
	201	break :v null;
	202	};
	203	} else {
	204	// Ignore unknown fields so that we can add fields in future zig
	205	// versions without breaking older zig versions.
	206	}
	207	}
	208
	209	if (!have_name) {
	210	try appendError(p, main_token, "missing top-level 'name' field", .{});
	211	} else {
	212	if (fingerprint) \|n\| {
	213	if (!n.validate(p.name)) {
	214	return fail(p, main_token, "invalid fingerprint: 0x{x}; if this is a new or forked package, use this value: 0x{x}", .{
	215	n.int(), Package.Fingerprint.generate(rng, p.name).int(),
	216	});
	217	}
	218	p.id = n.id;
	219	} else {
	220	try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{
	221	Package.Fingerprint.generate(rng, p.name).int(),
	222	});
	223	}
	224	}
	225
	226	if (!have_version) {
	227	try appendError(p, main_token, "missing top-level 'version' field", .{});
	228	}
	229
	230	if (!have_included_paths) {
	231	if (p.allow_missing_paths_field) {
	232	try p.paths.put(p.gpa, "", {});
	233	} else {
	234	try appendError(p, main_token, "missing top-level 'paths' field", .{});
	235	}
	236	}
	237	}
	238
	239	fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void {
	240	const ast = p.ast;
	241
	242	var buf: [2]Ast.Node.Index = undefined;
	243	const struct_init = ast.fullStructInit(&buf, node) orelse {
	244	const tok = ast.nodeMainToken(node);
	245	return fail(p, tok, "expected dependencies expression to be a struct", .{});
	246	};
	247
	248	for (struct_init.ast.fields) \|field_init\| {
	249	const name_token = ast.firstToken(field_init) - 2;
	250	const dep_name = try identifierTokenString(p, name_token);
	251	const dep = try parseDependency(p, field_init);
	252	try p.dependencies.put(p.gpa, dep_name, dep);
	253	}
	254	}
	255
	256	fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency {
	257	const ast = p.ast;
	258
	259	var buf: [2]Ast.Node.Index = undefined;
	260	const struct_init = ast.fullStructInit(&buf, node) orelse {
	261	const tok = ast.nodeMainToken(node);
	262	return fail(p, tok, "expected dependency expression to be a struct", .{});
	263	};
	264
	265	var dep: Dependency = .{
	266	.location = undefined,
	267	.location_tok = undefined,
	268	.location_node = undefined,
	269	.hash = null,
	270	.hash_tok = .none,
	271	.hash_node = .none,
	272	.node = node,
	273	.name_tok = undefined,
	274	.lazy = false,
	275	};
	276	var has_location = false;
	277
	278	for (struct_init.ast.fields) \|field_init\| {
	279	const name_token = ast.firstToken(field_init) - 2;
	280	dep.name_tok = name_token;
	281	const field_name = try identifierTokenString(p, name_token);
	282	// We could get fancy with reflection and comptime logic here but doing
	283	// things manually provides an opportunity to do any additional verification
	284	// that is desirable on a per-field basis.
	285	if (mem.eql(u8, field_name, "url")) {
	286	if (has_location) {
	287	return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{});
	288	}
	289	dep.location = .{
	290	.url = parseString(p, field_init) catch \|err\| switch (err) {
	291	error.ParseFailure => continue,
	292	else => \|e\| return e,
	293	},
	294	};
	295	has_location = true;
	296	dep.location_tok = ast.nodeMainToken(field_init);
	297	dep.location_node = field_init;
	298	} else if (mem.eql(u8, field_name, "path")) {
	299	if (has_location) {
	300	return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{});
	301	}
	302	dep.location = .{
	303	.path = parseString(p, field_init) catch \|err\| switch (err) {
	304	error.ParseFailure => continue,
	305	else => \|e\| return e,
	306	},
	307	};
	308	has_location = true;
	309	dep.location_tok = ast.nodeMainToken(field_init);
	310	dep.location_node = field_init;
	311	} else if (mem.eql(u8, field_name, "hash")) {
	312	dep.hash = parseHash(p, field_init) catch \|err\| switch (err) {
	313	error.ParseFailure => continue,
	314	else => \|e\| return e,
	315	};
	316	dep.hash_tok = .fromToken(ast.nodeMainToken(field_init));
	317	dep.hash_node = field_init.toOptional();
	318	} else if (mem.eql(u8, field_name, "lazy")) {
	319	dep.lazy = parseBool(p, field_init) catch \|err\| switch (err) {
	320	error.ParseFailure => continue,
	321	else => \|e\| return e,
	322	};
	323	} else {
	324	// Ignore unknown fields so that we can add fields in future zig
	325	// versions without breaking older zig versions.
	326	}
	327	}
	328
	329	if (!has_location) {
	330	try appendError(p, ast.nodeMainToken(node), "dependency requires location field, one of 'url' or 'path'.", .{});
	331	}
	332
	333	return dep;
	334	}
	335
	336	fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void {
	337	const ast = p.ast;
	338
	339	var buf: [2]Ast.Node.Index = undefined;
	340	const array_init = ast.fullArrayInit(&buf, node) orelse {
	341	const tok = ast.nodeMainToken(node);
	342	return fail(p, tok, "expected paths expression to be a list of strings", .{});
	343	};
	344
	345	for (array_init.ast.elements) \|elem_node\| {
	346	const path_string = try parseString(p, elem_node);
	347	// This is normalized so that it can be used in string comparisons
	348	// against file system paths.
	349	const normalized = try std.fs.path.resolve(p.arena, &.{path_string});
	350	try p.paths.put(p.gpa, normalized, {});
	351	}
	352	}
	353
	354	fn parseBool(p: *Parse, node: Ast.Node.Index) !bool {
	355	const ast = p.ast;
	356	if (ast.nodeTag(node) != .identifier) {
	357	return fail(p, ast.nodeMainToken(node), "expected identifier", .{});
	358	}
	359	const ident_token = ast.nodeMainToken(node);
	360	const token_bytes = ast.tokenSlice(ident_token);
	361	if (mem.eql(u8, token_bytes, "true")) {
	362	return true;
	363	} else if (mem.eql(u8, token_bytes, "false")) {
	364	return false;
	365	} else {
	366	return fail(p, ident_token, "expected boolean", .{});
	367	}
	368	}
	369
	370	fn parseFingerprint(p: *Parse, node: Ast.Node.Index) !Package.Fingerprint {
	371	const ast = p.ast;
	372	const main_token = ast.nodeMainToken(node);
	373	if (ast.nodeTag(node) != .number_literal) {
	374	return fail(p, main_token, "expected integer literal", .{});
	375	}
	376	const token_bytes = ast.tokenSlice(main_token);
	377	const parsed = std.zig.parseNumberLiteral(token_bytes);
	378	switch (parsed) {
	379	.int => \|n\| return @bitCast(n),
	380	.big_int, .float => return fail(p, main_token, "expected u64 integer literal, found {s}", .{
	381	@tagName(parsed),
	382	}),
	383	.failure => \|err\| return fail(p, main_token, "bad integer literal: {s}", .{@tagName(err)}),
	384	}
	385	}
	386
	387	fn parseName(p: *Parse, node: Ast.Node.Index) ![]const u8 {
	388	const ast = p.ast;
	389	const main_token = ast.nodeMainToken(node);
	390
	391	if (ast.nodeTag(node) != .enum_literal)
	392	return fail(p, main_token, "expected enum literal", .{});
	393
	394	const ident_name = ast.tokenSlice(main_token);
	395	if (mem.startsWith(u8, ident_name, "@"))
	396	return fail(p, main_token, "name must be a valid bare zig identifier", .{});
	397
	398	if (ident_name.len > max_name_len)
	399	return fail(p, main_token, "name '{f}' exceeds max length of {d}", .{
	400	std.zig.fmtId(ident_name), max_name_len,
	401	});
	402
	403	return ident_name;
	404	}
	405
	406	fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 {
	407	const ast = p.ast;
	408	if (ast.nodeTag(node) != .string_literal) {
	409	return fail(p, ast.nodeMainToken(node), "expected string literal", .{});
	410	}
	411	const str_lit_token = ast.nodeMainToken(node);
	412	const token_bytes = ast.tokenSlice(str_lit_token);
	413	p.buf.clearRetainingCapacity();
	414	try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0);
	415	const duped = try p.arena.dupe(u8, p.buf.items);
	416	return duped;
	417	}
	418
	419	fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 {
	420	const ast = p.ast;
	421	const tok = ast.nodeMainToken(node);
	422	const h = try parseString(p, node);
	423	switch (Package.Hash.validate(h)) {
	424	.ok => return h,
	425	else => \|t\| return fail(p, tok, "invalid hash: {t}", .{t}),
	426	}
	427	}
	428
	429	/// TODO: try to DRY this with AstGen.identifierTokenString
	430	fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 {
	431	const ast = p.ast;
	432	assert(ast.tokenTag(token) == .identifier);
	433	const ident_name = ast.tokenSlice(token);
	434	if (!mem.startsWith(u8, ident_name, "@")) {
	435	return ident_name;
	436	}
	437	p.buf.clearRetainingCapacity();
	438	try parseStrLit(p, token, &p.buf, ident_name, 1);
	439	const duped = try p.arena.dupe(u8, p.buf.items);
	440	return duped;
	441	}
	442
	443	/// TODO: try to DRY this with AstGen.parseStrLit
	444	fn parseStrLit(
	445	p: *Parse,
	446	token: Ast.TokenIndex,
	447	buf: *std.ArrayList(u8),
	448	bytes: []const u8,
	449	offset: u32,
	450	) InnerError!void {
	451	const raw_string = bytes[offset..];
	452	const result = r: {
	453	var aw: std.Io.Writer.Allocating = .fromArrayList(p.gpa, buf);
	454	defer buf.* = aw.toArrayList();
	455	break :r std.zig.string_literal.parseWrite(&aw.writer, raw_string) catch \|err\| switch (err) {
	456	error.WriteFailed => return error.OutOfMemory,
	457	};
	458	};
	459	switch (result) {
	460	.success => {},
	461	.failure => \|err\| try p.appendStrLitError(err, token, bytes, offset),
	462	}
	463	}
	464
	465	/// TODO: try to DRY this with AstGen.failWithStrLitError
	466	fn appendStrLitError(
	467	p: *Parse,
	468	err: std.zig.string_literal.Error,
	469	token: Ast.TokenIndex,
	470	bytes: []const u8,
	471	offset: u32,
	472	) Allocator.Error!void {
	473	const raw_string = bytes[offset..];
	474	switch (err) {
	475	.invalid_escape_character => \|bad_index\| {
	476	try p.appendErrorOff(
	477	token,
	478	offset + @as(u32, @intCast(bad_index)),
	479	"invalid escape character: '{c}'",
	480	.{raw_string[bad_index]},
	481	);
	482	},
	483	.expected_hex_digit => \|bad_index\| {
	484	try p.appendErrorOff(
	485	token,
	486	offset + @as(u32, @intCast(bad_index)),
	487	"expected hex digit, found '{c}'",
	488	.{raw_string[bad_index]},
	489	);
	490	},
	491	.empty_unicode_escape_sequence => \|bad_index\| {
	492	try p.appendErrorOff(
	493	token,
	494	offset + @as(u32, @intCast(bad_index)),
	495	"empty unicode escape sequence",
	496	.{},
	497	);
	498	},
	499	.expected_hex_digit_or_rbrace => \|bad_index\| {
	500	try p.appendErrorOff(
	501	token,
	502	offset + @as(u32, @intCast(bad_index)),
	503	"expected hex digit or '}}', found '{c}'",
	504	.{raw_string[bad_index]},
	505	);
	506	},
	507	.invalid_unicode_codepoint => \|bad_index\| {
	508	try p.appendErrorOff(
	509	token,
	510	offset + @as(u32, @intCast(bad_index)),
	511	"unicode escape does not correspond to a valid unicode scalar value",
	512	.{},
	513	);
	514	},
	515	.expected_lbrace => \|bad_index\| {
	516	try p.appendErrorOff(
	517	token,
	518	offset + @as(u32, @intCast(bad_index)),
	519	"expected '{{', found '{c}",
	520	.{raw_string[bad_index]},
	521	);
	522	},
	523	.expected_rbrace => \|bad_index\| {
	524	try p.appendErrorOff(
	525	token,
	526	offset + @as(u32, @intCast(bad_index)),
	527	"expected '}}', found '{c}",
	528	.{raw_string[bad_index]},
	529	);
	530	},
	531	.expected_single_quote => \|bad_index\| {
	532	try p.appendErrorOff(
	533	token,
	534	offset + @as(u32, @intCast(bad_index)),
	535	"expected single quote ('), found '{c}",
	536	.{raw_string[bad_index]},
	537	);
	538	},
	539	.invalid_character => \|bad_index\| {
	540	try p.appendErrorOff(
	541	token,
	542	offset + @as(u32, @intCast(bad_index)),
	543	"invalid byte in string or character literal: '{c}'",
	544	.{raw_string[bad_index]},
	545	);
	546	},
	547	.empty_char_literal => {
	548	try p.appendErrorOff(token, offset, "empty character literal", .{});
	549	},
	550	}
	551	}
	552
	553	fn fail(
	554	p: *Parse,
	555	tok: Ast.TokenIndex,
	556	comptime fmt: []const u8,
	557	args: anytype,
	558	) InnerError {
	559	try appendError(p, tok, fmt, args);
	560	return error.ParseFailure;
	561	}
	562
	563	fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void {
	564	return appendErrorOff(p, tok, 0, fmt, args);
	565	}
	566
	567	fn appendErrorOff(
	568	p: *Parse,
	569	tok: Ast.TokenIndex,
	570	byte_offset: u32,
	571	comptime fmt: []const u8,
	572	args: anytype,
	573	) Allocator.Error!void {
	574	try p.errors.append(p.gpa, .{
	575	.msg = try std.fmt.allocPrint(p.arena, fmt, args),
	576	.tok = tok,
	577	.off = byte_offset,
	578	});
	579	}
	580	};
	581
	582	pub fn load(
	583	io: Io,
	584	arena: Allocator,
	585	manifest_path: std.Build.Cache.Path,
	586	ast: *std.zig.Ast,
	587	error_bundle: *std.zig.ErrorBundle.Wip,
	588	manifest: *Manifest,
	589	allow_missing_paths_field: bool,
	590	) !void {
	591	const manifest_bytes = try manifest_path.root_dir.handle.readFileAllocOptions(
	592	io,
	593	manifest_path.sub_path,
	594	arena,
	595	.limited(max_bytes),
	596	.@"1",
	597	0,
	598	);
	599
	600	ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon);
	601
	602	if (ast.errors.len > 0) {
	603	const file_path = try manifest_path.joinString(arena, "");
	604	try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, error_bundle);
	605	return error.ErrorsBundled;
	606	}
	607
	608	const rng: std.Random.IoSource = .{ .io = io };
	609
	610	manifest.* = try parse(arena, ast, rng.interface(), .{
	611	.allow_missing_paths_field = allow_missing_paths_field,
	612	});
	613
	614	if (manifest.errors.len > 0) {
	615	const src_path = try error_bundle.printString("{f}", .{manifest_path});
	616	try manifest.copyErrorsIntoBundle(ast.*, src_path, error_bundle);
	617	return error.ErrorsBundled;
	618	}
	619	}
	620
	621	test "basic" {
	622	const gpa = testing.allocator;
	623
	624	const example =
	625	\\.{
	626	\\ .name = .foo,
	627	\\ .fingerprint = 0x8c736521490b23df,
	628	\\ .version = "3.2.1",
	629	\\ .paths = .{""},
	630	\\ .dependencies = .{
	631	\\ .bar = .{
	632	\\ .url = "https://example.com/baz.tar.gz",
	633	\\ .hash = "libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5",
	634	\\ },
	635	\\ },
	636	\\}
	637	;
	638
	639	var ast = try Ast.parse(gpa, example, .zon);
	640	defer ast.deinit(gpa);
	641
	642	try testing.expect(ast.errors.len == 0);
	643
	644	var rng = std.Random.DefaultPrng.init(0);
	645
	646	var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{});
	647	defer manifest.deinit(gpa);
	648
	649	try testing.expect(manifest.errors.len == 0);
	650	try testing.expectEqualStrings("foo", manifest.name);
	651
	652	try testing.expectEqual(@as(std.SemanticVersion, .{
	653	.major = 3,
	654	.minor = 2,
	655	.patch = 1,
	656	}), manifest.version);
	657
	658	try testing.expect(manifest.dependencies.count() == 1);
	659	try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]);
	660	try testing.expectEqualStrings(
	661	"https://example.com/baz.tar.gz",
	662	manifest.dependencies.values()[0].location.url,
	663	);
	664	try testing.expectEqualStrings(
	665	"libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5",
	666	manifest.dependencies.values()[0].hash orelse return error.TestFailed,
	667	);
	668
	669	try testing.expect(manifest.minimum_zig_version == null);
	670	}
	671
	672	test "minimum_zig_version" {
	673	const gpa = testing.allocator;
	674
	675	const example =
	676	\\.{
	677	\\ .name = .foo,
	678	\\ .fingerprint = 0x8c736521490b23df,
	679	\\ .version = "3.2.1",
	680	\\ .paths = .{""},
	681	\\ .minimum_zig_version = "0.11.1",
	682	\\}
	683	;
	684
	685	var ast = try Ast.parse(gpa, example, .zon);
	686	defer ast.deinit(gpa);
	687
	688	try testing.expect(ast.errors.len == 0);
	689
	690	var rng = std.Random.DefaultPrng.init(0);
	691
	692	var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{});
	693	defer manifest.deinit(gpa);
	694
	695	try testing.expect(manifest.errors.len == 0);
	696	try testing.expect(manifest.dependencies.count() == 0);
	697
	698	try testing.expect(manifest.minimum_zig_version != null);
	699
	700	try testing.expectEqual(@as(std.SemanticVersion, .{
	701	.major = 0,
	702	.minor = 11,
	703	.patch = 1,
	704	}), manifest.minimum_zig_version.?);
	705	}
	706
	707	test "minimum_zig_version - invalid version" {
	708	const gpa = testing.allocator;
	709
	710	const example =
	711	\\.{
	712	\\ .name = .foo,
	713	\\ .fingerprint = 0x8c736521490b23df,
	714	\\ .version = "3.2.1",
	715	\\ .minimum_zig_version = "X.11.1",
	716	\\ .paths = .{""},
	717	\\}
	718	;
	719
	720	var ast = try Ast.parse(gpa, example, .zon);
	721	defer ast.deinit(gpa);
	722
	723	try testing.expect(ast.errors.len == 0);
	724
	725	var rng = std.Random.DefaultPrng.init(0);
	726
	727	var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{});
	728	defer manifest.deinit(gpa);
	729
	730	try testing.expect(manifest.errors.len == 1);
	731	try testing.expect(manifest.dependencies.count() == 0);
	732
	733	try testing.expect(manifest.minimum_zig_version == null);
	734	}

lib/compiler/configurer.zig+3-2

...	...	@@ -633,7 +633,8 @@ fn serialize(b: std.Build, wc: Configuration.Wip, writer: *Io.Writer) !void {
633	633	var s: Serialize = .{ .wc = wc, .arena = arena };
634	634
635	635	try wc.path_deps.ensureTotalCapacityPrecise(gpa, graph.configure_dependencies.items.len);
636		for (
	636	// TODO remove this
	637	if (false) for (
637	638	graph.configure_dependencies.items,
638	639	wc.path_deps.addManyAsSliceAssumeCapacity(graph.configure_dependencies.items.len),
639	640	) \|src, *dest\| {
...	...	@@ -661,7 +662,7 @@ fn serialize(b: std.Build, wc: Configuration.Wip, writer: *Io.Writer) !void {
661	662	.dependency => \|d\| .init(try s.builderToPackage(d.dependency.builder)),
662	663	},
663	664	};
664		}
	665	};
665	666
666	667	// Starting from all top-level steps in `b`, traverse the entire step graph
667	668	// and add all step dependencies implied by module graphs.

lib/std/Build/Cache.zig+1-1

...	...	@@ -1035,7 +1035,7 @@ pub const Manifest = struct {
1035	1035	pub fn addPathPost(man: *Manifest, path: Path) !void {
1036	1036	_ = man;
1037	1037	_ = path;
1038		@panic("TODO");
	1038	std.log.err("TODO Build.Cache.addPathPost", .{});
1039	1039	}
1040	1040
1041	1041	/// Like `addFilePost` but when the file contents have already been loaded from disk.

lib/std/Build/Configuration.zig+1-1

...	...	@@ -1881,7 +1881,7 @@ pub const PathDep = extern struct {
1881	1881	_ = c;
1882	1882	_ = arena;
1883	1883	_ = path;
1884		@panic("TODO");
	1884	std.log.err("TODO Configuration.PathDep.toCachePath", .{});
1885	1885	}
1886	1886	};
1887	1887

lib/std/zig.zig+375-8

...	...	@@ -2,12 +2,17 @@
2	2	//! source lives here. These APIs are provided as-is and have absolutely no API
3	3	//! guarantees whatsoever.
4	4
	5	const builtin = @import("builtin");
	6
5	7	const std = @import("std.zig");
6	8	const assert = std.debug.assert;
7	9	const mem = std.mem;
8	10	const Allocator = std.mem.Allocator;
9	11	const Io = std.Io;
10	12	const Writer = std.Io.Writer;
	13	const Cache = std.Build.Cache;
	14	const fatal = std.process.fatal;
	15	const Dir = std.Io.Dir;
11	16
12	17	const tokenizer = @import("zig/tokenizer.zig");
13	18
...	...	@@ -47,6 +52,9 @@ pub const c_translation = struct {
47	52	pub const helpers = @import("zig/c_translation/helpers.zig");
48	53	};
49	54
	55	pub const default_local_zig_cache_basename = ".zig-cache";
	56	pub const build_zig_basename = "build.zig";
	57
50	58	pub const SrcHasher = std.crypto.hash.Blake3;
51	59	pub const SrcHash = [16]u8;
52	60
...	...	@@ -70,7 +78,7 @@ pub const Color = enum {
70	78	/// CLICOLOR_FORCE environment variables. Color is always disabled on WASI per
71	79	/// https://github.com/WebAssembly/WASI/issues/162
72	80	pub fn settingFromEnvironment(environ_map: *const std.process.Environ.Map) Color {
73		return if (@import("builtin").os.tag == .wasi or EnvVar.NO_COLOR.isSet(environ_map))
	81	return if (builtin.os.tag == .wasi or EnvVar.NO_COLOR.isSet(environ_map))
74	82	.off
75	83	else if (EnvVar.CLICOLOR_FORCE.isSet(environ_map))
76	84	.on
...	...	@@ -163,8 +171,8 @@ pub const BinNameOptions = struct {
163	171	os_tag: std.Target.Os.Tag,
164	172	ofmt: std.Target.ObjectFormat,
165	173	abi: std.Target.Abi,
166		output_mode: std.builtin.OutputMode,
167		link_mode: ?std.builtin.LinkMode = null,
	174	output_mode: std.lang.OutputMode,
	175	link_mode: ?std.lang.LinkMode = null,
168	176	version: ?std.SemanticVersion = null,
169	177	};
170	178
...	...	@@ -512,7 +520,7 @@ pub const FormatId = struct {
512	520	pub fn format(ctx: FormatId, writer: *Writer) Writer.Error!void {
513	521	const bytes = ctx.bytes;
514	522	if (isValidId(bytes) and
515		(ctx.flags.allow_primitive or !std.zig.isPrimitive(bytes)) and
	523	(ctx.flags.allow_primitive or !isPrimitive(bytes)) and
516	524	(ctx.flags.allow_underscore or !isUnderscore(bytes)))
517	525	{
518	526	return writer.writeAll(bytes);
...	...	@@ -592,7 +600,7 @@ pub fn isValidId(bytes: []const u8) bool {
592	600	else => return false,
593	601	}
594	602	}
595		return std.zig.Token.getKeyword(bytes) == null;
	603	return Token.getKeyword(bytes) == null;
596	604	}
597	605
598	606	test isValidId {
...	...	@@ -658,7 +666,7 @@ pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *Io.File.Reader) ![
658	666	}
659	667
660	668	pub fn printAstErrorsToStderr(gpa: Allocator, io: Io, tree: Ast, path: []const u8, color: Color) !void {
661		var wip_errors: std.zig.ErrorBundle.Wip = undefined;
	669	var wip_errors: ErrorBundle.Wip = undefined;
662	670	try wip_errors.init(gpa);
663	671	defer wip_errors.deinit();
664	672
...	...	@@ -673,7 +681,7 @@ pub fn putAstErrorsIntoBundle(
673	681	gpa: Allocator,
674	682	tree: Ast,
675	683	path: []const u8,
676		wip_errors: *std.zig.ErrorBundle.Wip,
	684	wip_errors: *ErrorBundle.Wip,
677	685	) Allocator.Error!void {
678	686	switch (tree.mode) {
679	687	.zig => {
...	...	@@ -692,7 +700,7 @@ pub fn putAstErrorsIntoBundle(
692	700	}
693	701
694	702	pub fn resolveTargetQueryOrFatal(io: Io, target_query: std.Target.Query) std.Target {
695		return std.zig.system.resolveTargetQuery(io, target_query) catch \|err\|
	703	return system.resolveTargetQuery(io, target_query) catch \|err\|
696	704	std.process.fatal("unable to resolve target: {t}", .{err});
697	705	}
698	706
...	...	@@ -1242,6 +1250,365 @@ pub fn allocPrintCmd(gpa: Allocator, argv: []const []const u8, options: AllocPri
1242	1250	return aw.toOwnedSlice();
1243	1251	}
1244	1252
	1253	/// Like `std.process.currentPathAlloc`, but also resolves the path with `Dir.path.resolve`. This
	1254	/// means the path has no repeated separators, no "." or ".." components, and no trailing separator.
	1255	/// On WASI, "" is returned instead of ".".
	1256	pub fn getResolvedCwd(io: Io, gpa: Allocator) std.process.CurrentPathAllocError![]u8 {
	1257	if (builtin.os.tag == .wasi) {
	1258	if (std.debug.runtime_safety) {
	1259	const cwd = try std.process.currentPathAlloc(io, gpa);
	1260	defer gpa.free(cwd);
	1261	assert(mem.eql(u8, cwd, "."));
	1262	}
	1263	return "";
	1264	}
	1265	const cwd = try std.process.currentPathAlloc(io, gpa);
	1266	defer gpa.free(cwd);
	1267	const resolved = try Dir.path.resolve(gpa, &.{cwd});
	1268	assert(Dir.path.isAbsolute(resolved));
	1269	return resolved;
	1270	}
	1271
	1272	pub const Directories = struct {
	1273	/// The string returned by `introspect.getResolvedCwd`. This is typically an absolute path,
	1274	/// but on WASI is the empty string "" instead, because WASI does not have absolute paths.
	1275	cwd: []const u8,
	1276	/// The Zig 'lib' directory.
	1277	/// `zig_lib.path` is resolved (`resolvePath`) or `null` for cwd.
	1278	/// Guaranteed to be a different path from `global_cache` and `local_cache`.
	1279	zig_lib: Cache.Directory,
	1280	/// The global Zig cache directory.
	1281	/// `global_cache.path` is resolved (`resolvePath`) or `null` for cwd.
	1282	global_cache: Cache.Directory,
	1283	/// The local Zig cache directory.
	1284	/// `local_cache.path` is resolved (`resolvePath`) or `null` for cwd.
	1285	/// This may be the same as `global_cache`.
	1286	local_cache: Cache.Directory,
	1287
	1288	pub fn deinit(dirs: *Directories, io: Io) void {
	1289	// The local and global caches could be the same.
	1290	const close_local = dirs.local_cache.handle.handle != dirs.global_cache.handle.handle;
	1291
	1292	dirs.global_cache.handle.close(io);
	1293	if (close_local) dirs.local_cache.handle.close(io);
	1294	dirs.zig_lib.handle.close(io);
	1295	}
	1296
	1297	/// Returns a `Directories` where `local_cache` is replaced with `global_cache`, intended for
	1298	/// use by sub-compilations (e.g. compiler_rt). Do not `deinit` the returned `Directories`; it
	1299	/// shares handles with `dirs`.
	1300	pub fn withoutLocalCache(dirs: Directories) Directories {
	1301	return .{
	1302	.cwd = dirs.cwd,
	1303	.zig_lib = dirs.zig_lib,
	1304	.global_cache = dirs.global_cache,
	1305	.local_cache = dirs.global_cache,
	1306	};
	1307	}
	1308
	1309	const LocalCacheStrategy = union(enum) {
	1310	override: []const u8,
	1311	search,
	1312	global,
	1313	};
	1314
	1315	/// Uses `std.process.fatal` on error conditions.
	1316	pub fn init(
	1317	arena: Allocator,
	1318	io: Io,
	1319	override_zig_lib: ?[]const u8,
	1320	override_global_cache: ?[]const u8,
	1321	local_cache_strat: LocalCacheStrategy,
	1322	preopens: std.process.Preopens,
	1323	self_exe_path: switch (builtin.target.os.tag) {
	1324	.wasi => void,
	1325	else => []const u8,
	1326	},
	1327	environ_map: *const std.process.Environ.Map,
	1328	cwd: []const u8,
	1329	) Directories {
	1330	const wasi = builtin.target.os.tag == .wasi;
	1331
	1332	const zig_lib: Cache.Directory = d: {
	1333	if (override_zig_lib) \|path\| break :d openUnresolved(arena, io, cwd, path, .@"zig lib");
	1334	if (wasi) break :d getPreopen(preopens, "/lib");
	1335	break :d findZigLibDirFromSelfExe(arena, io, cwd, self_exe_path) catch \|err\| {
	1336	fatal("unable to find zig installation directory {q}: {t}", .{ self_exe_path, err });
	1337	};
	1338	};
	1339
	1340	const global_cache: Cache.Directory = d: {
	1341	if (override_global_cache) \|path\| break :d openUnresolved(arena, io, cwd, path, .@"global cache");
	1342	if (wasi) break :d getPreopen(preopens, "/cache");
	1343	const path = resolveGlobalCacheDir(arena, environ_map) catch \|err\| {
	1344	fatal("unable to resolve zig cache directory: {t}", .{err});
	1345	};
	1346	break :d openUnresolved(arena, io, cwd, path, .@"global cache");
	1347	};
	1348
	1349	const local_cache = getLocalCacheDirectory(arena, io, cwd, global_cache, local_cache_strat);
	1350
	1351	if (std.mem.eql(u8, zig_lib.path orelse "", global_cache.path orelse "")) {
	1352	fatal("zig lib directory '{f}' cannot be equal to global cache directory '{f}'", .{ zig_lib, global_cache });
	1353	}
	1354	if (std.mem.eql(u8, zig_lib.path orelse "", local_cache.path orelse "")) {
	1355	fatal("zig lib directory '{f}' cannot be equal to local cache directory '{f}'", .{ zig_lib, local_cache });
	1356	}
	1357
	1358	return .{
	1359	.cwd = cwd,
	1360	.zig_lib = zig_lib,
	1361	.global_cache = global_cache,
	1362	.local_cache = local_cache,
	1363	};
	1364	}
	1365
	1366	fn getLocalCacheDirectory(
	1367	arena: Allocator,
	1368	io: Io,
	1369	cwd: []const u8,
	1370	global_cache: Cache.Directory,
	1371	local_cache_strat: LocalCacheStrategy,
	1372	) Cache.Directory {
	1373	return switch (local_cache_strat) {
	1374	.override => \|path\| openUnresolved(arena, io, cwd, path, .@"local cache"),
	1375	.search => d: {
	1376	const maybe_path = resolveSuitableLocalCacheDir(arena, io, cwd) catch \|err\|
	1377	fatal("unable to resolve zig cache directory: {t}", .{err});
	1378	const path = maybe_path orelse break :d global_cache;
	1379	break :d openUnresolved(arena, io, cwd, path, .@"local cache");
	1380	},
	1381	.global => global_cache,
	1382	};
	1383	}
	1384
	1385	fn getPreopen(preopens: std.process.Preopens, name: []const u8) Cache.Directory {
	1386	return .{
	1387	.path = if (std.mem.eql(u8, name, ".")) null else name,
	1388	.handle = switch (preopens.get(name) orelse fatal("preopen not found: {q}", .{name})) {
	1389	.file => fatal("preopen {q} is not a directory", .{name}),
	1390	.dir => \|d\| d,
	1391	},
	1392	};
	1393	}
	1394	fn openUnresolved(
	1395	arena: Allocator,
	1396	io: Io,
	1397	cwd: []const u8,
	1398	unresolved_path: []const u8,
	1399	thing: enum { @"zig lib", @"global cache", @"local cache" },
	1400	) Cache.Directory {
	1401	const path = resolvePath(arena, cwd, &.{unresolved_path}) catch \|err\| {
	1402	fatal("unable to resolve {t} directory: {t}", .{ thing, err });
	1403	};
	1404	const nonempty_path = if (path.len == 0) "." else path;
	1405	const handle_or_err = switch (thing) {
	1406	.@"zig lib" => Dir.cwd().openDir(io, nonempty_path, .{}),
	1407	.@"global cache", .@"local cache" => Dir.cwd().createDirPathOpen(io, nonempty_path, .{}),
	1408	};
	1409	return .{
	1410	.path = if (path.len == 0) null else path,
	1411	.handle = handle_or_err catch \|err\| {
	1412	const extra_str: []const u8 = e: {
	1413	if (thing == .@"global cache") switch (err) {
	1414	error.AccessDenied, error.ReadOnlyFileSystem => break :e "\n" ++
	1415	"If this location is not writable then consider specifying an alternative with " ++
	1416	"the ZIG_GLOBAL_CACHE_DIR environment variable or the --global-cache-dir option.",
	1417	else => {},
	1418	};
	1419	break :e "";
	1420	};
	1421	fatal("unable to open {t} directory {q}: {t}{s}", .{ thing, nonempty_path, err, extra_str });
	1422	},
	1423	};
	1424	}
	1425	};
	1426
	1427	/// Both the directory handle and the path are newly allocated resources which the caller now owns.
	1428	pub fn findZigLibDir(gpa: Allocator, io: Io) !Cache.Directory {
	1429	const cwd_path = try getResolvedCwd(io, gpa);
	1430	defer gpa.free(cwd_path);
	1431	const self_exe_path = try std.process.executablePathAlloc(io, gpa);
	1432	defer gpa.free(self_exe_path);
	1433
	1434	return findZigLibDirFromSelfExe(gpa, io, cwd_path, self_exe_path);
	1435	}
	1436
	1437	/// Both the directory handle and the path are newly allocated resources which the caller now owns.
	1438	pub fn findZigLibDirFromSelfExe(
	1439	allocator: Allocator,
	1440	io: Io,
	1441	/// The return value of `getResolvedCwd`.
	1442	/// Passed as an argument to avoid pointlessly repeating the call.
	1443	cwd_path: []const u8,
	1444	self_exe_path: []const u8,
	1445	) error{ OutOfMemory, FileNotFound }!Cache.Directory {
	1446	const cwd = Dir.cwd();
	1447	var cur_path: []const u8 = self_exe_path;
	1448	while (Dir.path.dirname(cur_path)) \|dirname\| : (cur_path = dirname) {
	1449	var base_dir = cwd.openDir(io, dirname, .{}) catch continue;
	1450	defer base_dir.close(io);
	1451
	1452	const sub_directory = testZigInstallPrefix(io, base_dir) orelse continue;
	1453	const p = try Dir.path.join(allocator, &.{ dirname, sub_directory.path.? });
	1454	defer allocator.free(p);
	1455
	1456	const resolved = try resolvePath(allocator, cwd_path, &.{p});
	1457	return .{
	1458	.handle = sub_directory.handle,
	1459	.path = if (resolved.len == 0) null else resolved,
	1460	};
	1461	}
	1462	return error.FileNotFound;
	1463	}
	1464
	1465	/// Returns the sub_path that worked, or `null` if none did.
	1466	/// The path of the returned Directory is relative to `base`.
	1467	/// The handle of the returned Directory is open.
	1468	fn testZigInstallPrefix(io: Io, base_dir: Dir) ?Cache.Directory {
	1469	const test_index_file = "std" ++ Dir.path.sep_str ++ "std.zig";
	1470
	1471	zig_dir: {
	1472	// Try lib/zig/std/std.zig
	1473	const lib_zig = "lib" ++ Dir.path.sep_str ++ "zig";
	1474	var test_zig_dir = base_dir.openDir(io, lib_zig, .{}) catch break :zig_dir;
	1475	const file = test_zig_dir.openFile(io, test_index_file, .{}) catch {
	1476	test_zig_dir.close(io);
	1477	break :zig_dir;
	1478	};
	1479	file.close(io);
	1480	return .{ .handle = test_zig_dir, .path = lib_zig };
	1481	}
	1482
	1483	// Try lib/std/std.zig
	1484	var test_zig_dir = base_dir.openDir(io, "lib", .{}) catch return null;
	1485	const file = test_zig_dir.openFile(io, test_index_file, .{}) catch {
	1486	test_zig_dir.close(io);
	1487	return null;
	1488	};
	1489	file.close(io);
	1490	return .{ .handle = test_zig_dir, .path = "lib" };
	1491	}
	1492
	1493	pub fn resolveGlobalCacheDir(arena: Allocator, environ_map: *const std.process.Environ.Map) ![]const u8 {
	1494	if (EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map)) \|value\| return value;
	1495
	1496	const app_name = "zig";
	1497
	1498	switch (builtin.os.tag) {
	1499	.wasi => @compileError("on WASI the global cache dir must be resolved with preopens"),
	1500	.windows => {
	1501	const local_app_data_dir = EnvVar.LOCALAPPDATA.get(environ_map) orelse
	1502	return error.AppDataDirUnavailable;
	1503	return Dir.path.join(arena, &.{ local_app_data_dir, app_name });
	1504	},
	1505	else => {
	1506	if (EnvVar.XDG_CACHE_HOME.get(environ_map)) \|cache_root\| {
	1507	if (cache_root.len > 0) {
	1508	return Dir.path.join(arena, &.{ cache_root, app_name });
	1509	}
	1510	}
	1511	if (EnvVar.HOME.get(environ_map)) \|home\| {
	1512	if (home.len > 0) {
	1513	return Dir.path.join(arena, &.{ home, ".cache", app_name });
	1514	}
	1515	}
	1516	return error.AppDataDirUnavailable;
	1517	},
	1518	}
	1519	}
	1520
	1521	/// Searches upwards from `cwd` for a directory containing a `build.zig` file.
	1522	/// If such a directory is found, returns the path to it joined to the `.zig_cache` name.
	1523	/// Otherwise, returns `null`, indicating no suitable local cache location.
	1524	pub fn resolveSuitableLocalCacheDir(arena: Allocator, io: Io, cwd: []const u8) Allocator.Error!?[]u8 {
	1525	var cur_dir = cwd;
	1526	while (true) {
	1527	const joined = try Dir.path.join(arena, &.{ cur_dir, build_zig_basename });
	1528	if (Dir.cwd().access(io, joined, .{})) \|_\| {
	1529	return try Dir.path.join(arena, &.{ cur_dir, default_local_zig_cache_basename });
	1530	} else \|err\| switch (err) {
	1531	error.FileNotFound => {
	1532	cur_dir = Dir.path.dirname(cur_dir) orelse return null;
	1533	continue;
	1534	},
	1535	else => return null,
	1536	}
	1537	}
	1538	}
	1539
	1540	/// Similar to `Dir.path.resolve`, but converts to a cwd-relative path, or, if that would
	1541	/// start with a relative up-dir (".."), an absolute path based on the cwd. Also, the cwd
	1542	/// returns the empty string ("") instead of ".".
	1543	pub fn resolvePath(
	1544	gpa: Allocator,
	1545	/// The return value of `getResolvedCwd`.
	1546	/// Passed as an argument to avoid pointlessly repeating the call.
	1547	cwd_resolved: []const u8,
	1548	paths: []const []const u8,
	1549	) Allocator.Error![]u8 {
	1550	if (builtin.target.os.tag == .wasi) {
	1551	assert(mem.eql(u8, cwd_resolved, ""));
	1552	const res = try Dir.path.resolve(gpa, paths);
	1553	if (mem.eql(u8, res, ".")) {
	1554	gpa.free(res);
	1555	return "";
	1556	}
	1557	return res;
	1558	}
	1559
	1560	// Heuristic for a fast path: if no component is absolute and ".." never appears, we just need to resolve `paths`.
	1561	for (paths) \|p\| {
	1562	if (Dir.path.isAbsolute(p)) break; // absolute path
	1563	if (mem.indexOf(u8, p, "..") != null) break; // may contain up-dir
	1564	} else {
	1565	// no absolute path, no "..".
	1566	const res = try Dir.path.resolve(gpa, paths);
	1567	if (mem.eql(u8, res, ".")) {
	1568	gpa.free(res);
	1569	return "";
	1570	}
	1571	assert(!Dir.path.isAbsolute(res));
	1572	assert(!isUpDir(res));
	1573	return res;
	1574	}
	1575
	1576	// The fast path failed; resolve the whole thing.
	1577	// Optimization: `paths` often has just one element.
	1578	const path_resolved = switch (paths.len) {
	1579	0 => unreachable,
	1580	1 => try Dir.path.resolve(gpa, &.{ cwd_resolved, paths[0] }),
	1581	else => r: {
	1582	const all_paths = try gpa.alloc([]const u8, paths.len + 1);
	1583	defer gpa.free(all_paths);
	1584	all_paths[0] = cwd_resolved;
	1585	@memcpy(all_paths[1..], paths);
	1586	break :r try Dir.path.resolve(gpa, all_paths);
	1587	},
	1588	};
	1589	errdefer gpa.free(path_resolved);
	1590
	1591	assert(Dir.path.isAbsolute(path_resolved));
	1592	assert(Dir.path.isAbsolute(cwd_resolved));
	1593
	1594	if (!std.mem.startsWith(u8, path_resolved, cwd_resolved)) return path_resolved; // not in cwd
	1595	if (path_resolved.len == cwd_resolved.len) {
	1596	// equal to cwd
	1597	gpa.free(path_resolved);
	1598	return "";
	1599	}
	1600	if (path_resolved[cwd_resolved.len] != Dir.path.sep) return path_resolved; // not in cwd (last component differs)
	1601
	1602	// in cwd; extract sub path
	1603	const sub_path = try gpa.dupe(u8, path_resolved[cwd_resolved.len + 1 ..]);
	1604	gpa.free(path_resolved);
	1605	return sub_path;
	1606	}
	1607
	1608	pub fn isUpDir(p: []const u8) bool {
	1609	return mem.startsWith(u8, p, "..") and (p.len == 2 or p[2] == Dir.path.sep);
	1610	}
	1611
1245	1612	test {
1246	1613	_ = Ast;
1247	1614	_ = AstRlAnnotate;

src/Compilation.zig+7-151

...	...	@@ -17,7 +17,6 @@ const Value = @import("Value.zig");
17	17	const Type = @import("Type.zig");
18	18	const target_util = @import("target.zig");
19	19	const Package = @import("Package.zig");
20		const introspect = @import("introspect.zig");
21	20	const link = @import("link.zig");
22	21	const tracy = @import("tracy.zig");
23	22	const trace = tracy.trace;
...	...	@@ -190,7 +189,7 @@ parent_whole_cache: ?ParentWholeCache,
190	189	/// Path to own executable for invoking `zig clang`.
191	190	self_exe_path: ?[]const u8,
192	191	/// Owned by the caller of `Compilation.create`.
193		dirs: Directories,
	192	dirs: std.zig.Directories,
194	193	libc_include_dir_list: []const []const u8,
195	194	libc_framework_dir_list: []const []const u8,
196	195	rc_includes: std.zig.RcIncludes,
...	...	@@ -431,7 +430,7 @@ pub const Path = struct {
431	430	}
432	431
433	432	/// Given a `Path`, returns the directory handle and sub path to be used to open the path.
434		pub fn openInfo(p: Path, dirs: Directories) struct { Io.Dir, []const u8 } {
	433	pub fn openInfo(p: Path, dirs: std.zig.Directories) struct { Io.Dir, []const u8 } {
435	434	const dir = switch (p.root) {
436	435	.none => {
437	436	const cwd_sub_path = absToCwdRelative(p.sub_path, dirs.cwd);
...	...	@@ -492,7 +491,7 @@ pub const Path = struct {
492	491	/// From an unresolved path (which can be made of multiple not-yet-joined strings), construct a
493	492	/// canonical `Path`.
494	493	pub fn fromUnresolved(gpa: Allocator, dirs: Compilation.Directories, unresolved_parts: []const []const u8) Allocator.Error!Path {
495		const resolved = try introspect.resolvePath(gpa, dirs.cwd, unresolved_parts);
	494	const resolved = try std.zig.resolvePath(gpa, dirs.cwd, unresolved_parts);
496	495	errdefer gpa.free(resolved);
497	496
498	497	// If, for instance, `dirs.local_cache.path` is within the lib dir, it must take priority,
...	...	@@ -626,7 +625,7 @@ pub const Path = struct {
626	625	});
627	626	}
628	627
629		pub fn toCachePath(p: Path, dirs: Directories) Cache.Path {
	628	pub fn toCachePath(p: Path, dirs: std.zig.Directories) Cache.Path {
630	629	const root_dir: Cache.Directory = switch (p.root) {
631	630	.zig_lib => dirs.zig_lib,
632	631	.global_cache => dirs.global_cache,
...	...	@@ -649,7 +648,7 @@ pub const Path = struct {
649	648	/// This should not be used for most of the compiler pipeline, but is useful when emitting
650	649	/// paths from the compilation (e.g. in debug info), because they will not depend on the cwd.
651	650	/// The returned path is owned by the caller and allocated into `gpa`.
652		pub fn toAbsolute(p: Path, dirs: Directories, gpa: Allocator) Allocator.Error![]u8 {
	651	pub fn toAbsolute(p: Path, dirs: std.zig.Directories, gpa: Allocator) Allocator.Error![]u8 {
653	652	const root_path: []const u8 = switch (p.root) {
654	653	.zig_lib => dirs.zig_lib.path orelse "",
655	654	.global_cache => dirs.global_cache.path orelse "",
...	...	@@ -680,7 +679,7 @@ pub const Path = struct {
680	679	/// Returns whether this `Path` is illegal to have as a user-imported `Zcu.File` (including
681	680	/// as the root of a module). Such paths exist in directories which the Zig compiler treats
682	681	/// specially, like 'global_cache/b/', which stores 'builtin.zig' files.
683		pub fn isIllegalZigImport(p: Path, gpa: Allocator, dirs: Directories) Allocator.Error!bool {
	682	pub fn isIllegalZigImport(p: Path, gpa: Allocator, dirs: std.zig.Directories) Allocator.Error!bool {
684	683	const zig_builtin_dir: Path = try .fromRoot(gpa, dirs, .global_cache, "b");
685	684	defer zig_builtin_dir.deinit(gpa);
686	685	return switch (p.isNested(zig_builtin_dir)) {
...	...	@@ -690,149 +689,6 @@ pub const Path = struct {
690	689	}
691	690	};
692	691
693		pub const Directories = struct {
694		/// The string returned by `introspect.getResolvedCwd`. This is typically an absolute path,
695		/// but on WASI is the empty string "" instead, because WASI does not have absolute paths.
696		cwd: []const u8,
697		/// The Zig 'lib' directory.
698		/// `zig_lib.path` is resolved (`introspect.resolvePath`) or `null` for cwd.
699		/// Guaranteed to be a different path from `global_cache` and `local_cache`.
700		zig_lib: Cache.Directory,
701		/// The global Zig cache directory.
702		/// `global_cache.path` is resolved (`introspect.resolvePath`) or `null` for cwd.
703		global_cache: Cache.Directory,
704		/// The local Zig cache directory.
705		/// `local_cache.path` is resolved (`introspect.resolvePath`) or `null` for cwd.
706		/// This may be the same as `global_cache`.
707		local_cache: Cache.Directory,
708
709		pub fn deinit(dirs: *Directories, io: Io) void {
710		// The local and global caches could be the same.
711		const close_local = dirs.local_cache.handle.handle != dirs.global_cache.handle.handle;
712
713		dirs.global_cache.handle.close(io);
714		if (close_local) dirs.local_cache.handle.close(io);
715		dirs.zig_lib.handle.close(io);
716		}
717
718		/// Returns a `Directories` where `local_cache` is replaced with `global_cache`, intended for
719		/// use by sub-compilations (e.g. compiler_rt). Do not `deinit` the returned `Directories`; it
720		/// shares handles with `dirs`.
721		pub fn withoutLocalCache(dirs: Directories) Directories {
722		return .{
723		.cwd = dirs.cwd,
724		.zig_lib = dirs.zig_lib,
725		.global_cache = dirs.global_cache,
726		.local_cache = dirs.global_cache,
727		};
728		}
729
730		/// Uses `std.process.fatal` on error conditions.
731		pub fn init(
732		arena: Allocator,
733		io: Io,
734		override_zig_lib: ?[]const u8,
735		override_global_cache: ?[]const u8,
736		local_cache_strat: union(enum) {
737		override: []const u8,
738		search,
739		global,
740		},
741		preopens: std.process.Preopens,
742		self_exe_path: switch (builtin.target.os.tag) {
743		.wasi => void,
744		else => []const u8,
745		},
746		environ_map: *const std.process.Environ.Map,
747		cwd: []const u8,
748		) Directories {
749		const wasi = builtin.target.os.tag == .wasi;
750
751		const zig_lib: Cache.Directory = d: {
752		if (override_zig_lib) \|path\| break :d openUnresolved(arena, io, cwd, path, .@"zig lib");
753		if (wasi) break :d getPreopen(preopens, "/lib");
754		break :d introspect.findZigLibDirFromSelfExe(arena, io, cwd, self_exe_path) catch \|err\| {
755		fatal("unable to find zig installation directory '{s}': {t}", .{ self_exe_path, err });
756		};
757		};
758
759		const global_cache: Cache.Directory = d: {
760		if (override_global_cache) \|path\| break :d openUnresolved(arena, io, cwd, path, .@"global cache");
761		if (wasi) break :d getPreopen(preopens, "/cache");
762		const path = introspect.resolveGlobalCacheDir(arena, environ_map) catch \|err\| {
763		fatal("unable to resolve zig cache directory: {t}", .{err});
764		};
765		break :d openUnresolved(arena, io, cwd, path, .@"global cache");
766		};
767
768		const local_cache: Cache.Directory = switch (local_cache_strat) {
769		.override => \|path\| openUnresolved(arena, io, cwd, path, .@"local cache"),
770		.search => d: {
771		const maybe_path = introspect.resolveSuitableLocalCacheDir(arena, io, cwd) catch \|err\| {
772		fatal("unable to resolve zig cache directory: {t}", .{err});
773		};
774		const path = maybe_path orelse break :d global_cache;
775		break :d openUnresolved(arena, io, cwd, path, .@"local cache");
776		},
777		.global => global_cache,
778		};
779
780		if (std.mem.eql(u8, zig_lib.path orelse "", global_cache.path orelse "")) {
781		fatal("zig lib directory '{f}' cannot be equal to global cache directory '{f}'", .{ zig_lib, global_cache });
782		}
783		if (std.mem.eql(u8, zig_lib.path orelse "", local_cache.path orelse "")) {
784		fatal("zig lib directory '{f}' cannot be equal to local cache directory '{f}'", .{ zig_lib, local_cache });
785		}
786
787		return .{
788		.cwd = cwd,
789		.zig_lib = zig_lib,
790		.global_cache = global_cache,
791		.local_cache = local_cache,
792		};
793		}
794		fn getPreopen(preopens: std.process.Preopens, name: []const u8) Cache.Directory {
795		return .{
796		.path = if (std.mem.eql(u8, name, ".")) null else name,
797		.handle = switch (preopens.get(name) orelse fatal("preopen not found: '{s}'", .{name})) {
798		.file => fatal("preopen {s} is not a directory", .{name}),
799		.dir => \|d\| d,
800		},
801		};
802		}
803		fn openUnresolved(
804		arena: Allocator,
805		io: Io,
806		cwd: []const u8,
807		unresolved_path: []const u8,
808		thing: enum { @"zig lib", @"global cache", @"local cache" },
809		) Cache.Directory {
810		const path = introspect.resolvePath(arena, cwd, &.{unresolved_path}) catch \|err\| {
811		fatal("unable to resolve {s} directory: {s}", .{ @tagName(thing), @errorName(err) });
812		};
813		const nonempty_path = if (path.len == 0) "." else path;
814		const handle_or_err = switch (thing) {
815		.@"zig lib" => Io.Dir.cwd().openDir(io, nonempty_path, .{}),
816		.@"global cache", .@"local cache" => Io.Dir.cwd().createDirPathOpen(io, nonempty_path, .{}),
817		};
818		return .{
819		.path = if (path.len == 0) null else path,
820		.handle = handle_or_err catch \|err\| {
821		const extra_str: []const u8 = e: {
822		if (thing == .@"global cache") switch (err) {
823		error.AccessDenied, error.ReadOnlyFileSystem => break :e "\n" ++
824		"If this location is not writable then consider specifying an alternative with " ++
825		"the ZIG_GLOBAL_CACHE_DIR environment variable or the --global-cache-dir option.",
826		else => {},
827		};
828		break :e "";
829		};
830		fatal("unable to open {s} directory '{s}': {s}{s}", .{ @tagName(thing), nonempty_path, @errorName(err), extra_str });
831		},
832		};
833		}
834		};
835
836	692	/// This small wrapper function just checks whether debug extensions are enabled before checking
837	693	/// `comp.debug_incremental`. It is inline so that comptime-known `false` propagates to the caller,
838	694	/// preventing debugging features from making it into release builds of the compiler.
...	...	@@ -1549,7 +1405,7 @@ const CacheUse = union(CacheMode) {
1549	1405	};
1550	1406
1551	1407	pub const CreateOptions = struct {
1552		dirs: Directories,
	1408	dirs: std.zig.Directories,
1553	1409	thread_limit: usize,
1554	1410	self_exe_path: ?[]const u8 = null,
1555	1411

src/Module.zig created+523

...	...	@@ -0,0 +1,523 @@
	1	//! Corresponds to something that Zig source code can `@import`.
	2	const Module = @This();
	3
	4	const std = @import("std");
	5	const Allocator = std.mem.Allocator;
	6	const Cache = std.Build.Cache;
	7	const assert = std.debug.assert;
	8
	9	const target_util = @import("../target.zig");
	10	const Builtin = @import("../Builtin.zig");
	11	const Compilation = @import("../Compilation.zig");
	12	const File = @import("../Zcu.zig").File;
	13
	14	/// The root directory of the module. Only files inside this directory can be imported.
	15	root: Compilation.Path,
	16	/// Path to the root source file of this module. Relative to `root`. May contain path separators.
	17	root_src_path: []const u8,
	18	/// Name used in compile errors. Looks like "root.foo.bar".
	19	fully_qualified_name: []const u8,
	20	/// The dependency table of this module. The shared dependencies 'std' and
	21	/// 'root' are not specified in every module dependency table, but are stored
	22	/// separately in `Zcu`. 'builtin' is also not stored here, although it is
	23	/// not necessarily the same between all modules. Handling of `@import` in
	24	/// the rest of the compiler must detect these special names and use the
	25	/// correct module instead of consulting `deps`.
	26	deps: Deps = .{},
	27
	28	resolved_target: ResolvedTarget,
	29	optimize_mode: std.lang.OptimizeMode,
	30	code_model: std.lang.CodeModel,
	31	single_threaded: bool,
	32	error_tracing: bool,
	33	valgrind: bool,
	34	pic: bool,
	35	strip: bool,
	36	omit_frame_pointer: bool,
	37	stack_check: bool,
	38	stack_protector: u32,
	39	red_zone: bool,
	40	sanitize_c: std.zig.SanitizeC,
	41	sanitize_thread: bool,
	42	fuzz: bool,
	43	unwind_tables: std.lang.UnwindTables,
	44	cc_argv: []const []const u8,
	45	/// (SPIR-V) whether to generate a structured control flow graph or not
	46	structured_cfg: bool,
	47	no_builtin: bool,
	48
	49	pub const Deps = std.array_hash_map.String(*Module);
	50
	51	pub const CreateOptions = struct {
	52	paths: Paths,
	53	fully_qualified_name: []const u8,
	54
	55	cc_argv: []const []const u8,
	56	inherited: Inherited,
	57	global: Compilation.Config,
	58	/// If this is null then `resolved_target` must be non-null.
	59	parent: ?*Module,
	60
	61	pub const Paths = struct {
	62	root: Compilation.Path,
	63	/// Relative to `root`. May contain path separators.
	64	root_src_path: []const u8,
	65	};
	66
	67	pub const Inherited = struct {
	68	/// If this is null then `parent` must be non-null.
	69	resolved_target: ?ResolvedTarget = null,
	70	optimize_mode: ?std.lang.OptimizeMode = null,
	71	code_model: ?std.lang.CodeModel = null,
	72	single_threaded: ?bool = null,
	73	error_tracing: ?bool = null,
	74	valgrind: ?bool = null,
	75	pic: ?bool = null,
	76	strip: ?bool = null,
	77	omit_frame_pointer: ?bool = null,
	78	stack_check: ?bool = null,
	79	/// null means default.
	80	/// 0 means no stack protector.
	81	/// other number means stack protection with that buffer size.
	82	stack_protector: ?u32 = null,
	83	red_zone: ?bool = null,
	84	unwind_tables: ?std.lang.UnwindTables = null,
	85	sanitize_c: ?std.zig.SanitizeC = null,
	86	sanitize_thread: ?bool = null,
	87	fuzz: ?bool = null,
	88	structured_cfg: ?bool = null,
	89	no_builtin: ?bool = null,
	90	};
	91	};
	92
	93	pub const ResolvedTarget = struct {
	94	result: std.Target,
	95	is_native_os: bool,
	96	is_native_abi: bool,
	97	is_explicit_dynamic_linker: bool,
	98	llvm_cpu_features: ?[*:0]const u8 = null,
	99	};
	100
	101	pub const CreateError = error{
	102	OutOfMemory,
	103	ValgrindUnsupportedOnTarget,
	104	TargetRequiresSingleThreaded,
	105	BackendRequiresSingleThreaded,
	106	TargetRequiresPic,
	107	PieRequiresPic,
	108	DynamicLinkingRequiresPic,
	109	TargetHasNoRedZone,
	110	StackCheckUnsupportedByTarget,
	111	StackProtectorUnsupportedByTarget,
	112	StackProtectorUnavailableWithoutLibC,
	113	};
	114
	115	/// At least one of `parent` and `resolved_target` must be non-null.
	116	pub fn create(arena: Allocator, options: CreateOptions) !*Module {
	117	if (options.inherited.sanitize_thread == true) assert(options.global.any_sanitize_thread);
	118	if (options.inherited.fuzz == true) assert(options.global.any_fuzz);
	119	if (options.inherited.single_threaded == false) assert(options.global.any_non_single_threaded);
	120	if (options.inherited.unwind_tables) \|uwt\| if (uwt != .none) assert(options.global.any_unwind_tables);
	121	if (options.inherited.sanitize_c) \|sc\| if (sc != .off) assert(options.global.any_sanitize_c != .off);
	122	if (options.inherited.error_tracing == true) assert(options.global.any_error_tracing);
	123
	124	const resolved_target = options.inherited.resolved_target orelse options.parent.?.resolved_target;
	125	const target = &resolved_target.result;
	126
	127	const optimize_mode = options.inherited.optimize_mode orelse
	128	if (options.parent) \|p\| p.optimize_mode else options.global.root_optimize_mode;
	129
	130	const strip = b: {
	131	if (options.inherited.strip) \|x\| break :b x;
	132	if (options.parent) \|p\| break :b p.strip;
	133	break :b options.global.root_strip;
	134	};
	135
	136	const zig_backend = target_util.zigBackend(target, options.global.use_llvm);
	137
	138	const valgrind = b: {
	139	if (!target_util.hasValgrindSupport(target, zig_backend)) {
	140	if (options.inherited.valgrind == true)
	141	return error.ValgrindUnsupportedOnTarget;
	142	break :b false;
	143	}
	144	if (options.inherited.valgrind) \|x\| break :b x;
	145	if (options.parent) \|p\| break :b p.valgrind;
	146	if (strip) break :b false;
	147	break :b optimize_mode == .Debug;
	148	};
	149
	150	const single_threaded = b: {
	151	if (target_util.alwaysSingleThreaded(target)) {
	152	if (options.inherited.single_threaded == false)
	153	return error.TargetRequiresSingleThreaded;
	154	break :b true;
	155	}
	156
	157	if (options.global.have_zcu) {
	158	if (!target_util.supportsThreads(target, zig_backend)) {
	159	if (options.inherited.single_threaded == false)
	160	return error.BackendRequiresSingleThreaded;
	161	break :b true;
	162	}
	163	}
	164
	165	if (options.inherited.single_threaded) \|x\| break :b x;
	166	if (options.parent) \|p\| break :b p.single_threaded;
	167	break :b target_util.defaultSingleThreaded(target);
	168	};
	169
	170	const error_tracing = b: {
	171	if (options.inherited.error_tracing) \|x\| break :b x;
	172	if (options.parent) \|p\| break :b p.error_tracing;
	173	break :b options.global.root_error_tracing;
	174	};
	175
	176	const pic = b: {
	177	if (target_util.requiresPic(target, options.global.link_libc)) {
	178	if (options.inherited.pic == false)
	179	return error.TargetRequiresPic;
	180	break :b true;
	181	}
	182	if (options.global.pie) {
	183	if (options.inherited.pic == false)
	184	return error.PieRequiresPic;
	185	break :b true;
	186	}
	187	if (options.global.link_mode == .dynamic and target_util.requiresPicForDynamicLink(target)) {
	188	if (options.inherited.pic == false)
	189	return error.DynamicLinkingRequiresPic;
	190	break :b true;
	191	}
	192	if (options.inherited.pic) \|x\| break :b x;
	193	if (options.parent) \|p\| break :b p.pic;
	194
	195	// Default to PIC on targets where we default to producing PIEs to make
	196	// the common case of linking objects and static libraries into an
	197	// executable work out of the box.
	198	break :b target_util.defaultPie(target);
	199	};
	200
	201	const red_zone = b: {
	202	if (!target_util.hasRedZone(target)) {
	203	if (options.inherited.red_zone == true)
	204	return error.TargetHasNoRedZone;
	205	break :b false;
	206	}
	207	if (options.inherited.red_zone) \|x\| break :b x;
	208	if (options.parent) \|p\| break :b p.red_zone;
	209	break :b true;
	210	};
	211
	212	const omit_frame_pointer = b: {
	213	if (options.inherited.omit_frame_pointer) \|x\| break :b x;
	214	if (options.parent) \|p\| break :b p.omit_frame_pointer;
	215	if (optimize_mode == .ReleaseSmall) {
	216	// On x86, in most cases, keeping the frame pointer usually results in smaller binary size.
	217	// This has to do with how instructions for memory access via the stack base pointer register (when keeping the frame pointer)
	218	// are smaller than instructions for memory access via the stack pointer register (when omitting the frame pointer).
	219	break :b !target.cpu.arch.isX86();
	220	}
	221	break :b false;
	222	};
	223
	224	const sanitize_thread = b: {
	225	if (options.inherited.sanitize_thread) \|x\| break :b x;
	226	if (options.parent) \|p\| break :b p.sanitize_thread;
	227	break :b false;
	228	};
	229
	230	const unwind_tables = b: {
	231	if (options.inherited.unwind_tables) \|x\| break :b x;
	232	if (options.parent) \|p\| break :b p.unwind_tables;
	233
	234	break :b target_util.defaultUnwindTables(
	235	target,
	236	options.global.link_libunwind,
	237	sanitize_thread or options.global.any_sanitize_thread,
	238	);
	239	};
	240
	241	const fuzz = b: {
	242	if (options.inherited.fuzz) \|x\| break :b x;
	243	if (options.parent) \|p\| break :b p.fuzz;
	244	break :b false;
	245	};
	246
	247	const code_model: std.lang.CodeModel = b: {
	248	if (options.inherited.code_model) \|x\| break :b x;
	249	if (options.parent) \|p\| break :b p.code_model;
	250	break :b .default;
	251	};
	252
	253	const is_safe_mode = switch (optimize_mode) {
	254	.Debug, .ReleaseSafe => true,
	255	.ReleaseFast, .ReleaseSmall => false,
	256	};
	257
	258	const sanitize_c: std.zig.SanitizeC = b: {
	259	if (options.inherited.sanitize_c) \|x\| break :b x;
	260	if (options.parent) \|p\| break :b p.sanitize_c;
	261	break :b switch (optimize_mode) {
	262	.Debug => .full,
	263	// It's recommended to use the minimal runtime in production
	264	// environments due to the security implications of the full runtime.
	265	// The minimal runtime doesn't provide much benefit over simply
	266	// trapping, however, so we do that instead.
	267	.ReleaseSafe => .trap,
	268	.ReleaseFast, .ReleaseSmall => .off,
	269	};
	270	};
	271
	272	const stack_check = b: {
	273	if (!target_util.supportsStackProbing(target, zig_backend)) {
	274	if (options.inherited.stack_check == true)
	275	return error.StackCheckUnsupportedByTarget;
	276	break :b false;
	277	}
	278	if (options.inherited.stack_check) \|x\| break :b x;
	279	if (options.parent) \|p\| break :b p.stack_check;
	280	break :b is_safe_mode;
	281	};
	282
	283	const stack_protector: u32 = sp: {
	284	const use_zig_backend = options.global.have_zcu or
	285	(options.global.any_c_source_files and options.global.c_frontend == .aro);
	286	if (use_zig_backend and !target_util.supportsStackProtector(target, zig_backend)) {
	287	if (options.inherited.stack_protector) \|x\| {
	288	if (x > 0) return error.StackProtectorUnsupportedByTarget;
	289	}
	290	break :sp 0;
	291	}
	292
	293	if (options.global.any_c_source_files and options.global.c_frontend == .clang and
	294	!target_util.clangSupportsStackProtector(target))
	295	{
	296	if (options.inherited.stack_protector) \|x\| {
	297	if (x > 0) return error.StackProtectorUnsupportedByTarget;
	298	}
	299	break :sp 0;
	300	}
	301
	302	// This logic is checking for linking libc because otherwise our start code
	303	// which is trying to set up TLS (i.e. the fs/gs registers) but the stack
	304	// protection code depends on fs/gs registers being already set up.
	305	// If we were able to annotate start code, or perhaps the entire std lib,
	306	// as being exempt from stack protection checks, we could change this logic
	307	// to supporting stack protection even when not linking libc.
	308	// TODO file issue about this
	309	if (!options.global.link_libc) {
	310	if (options.inherited.stack_protector) \|x\| {
	311	if (x > 0) return error.StackProtectorUnavailableWithoutLibC;
	312	}
	313	break :sp 0;
	314	}
	315
	316	if (options.inherited.stack_protector) \|x\| break :sp x;
	317	if (options.parent) \|p\| break :sp p.stack_protector;
	318	if (!is_safe_mode) break :sp 0;
	319
	320	break :sp target_util.default_stack_protector_buffer_size;
	321	};
	322
	323	const structured_cfg = b: {
	324	if (options.inherited.structured_cfg) \|x\| break :b x;
	325	if (options.parent) \|p\| break :b p.structured_cfg;
	326	// We always want a structured control flow in shaders. This option is
	327	// only relevant for OpenCL kernels.
	328	break :b switch (target.os.tag) {
	329	.opencl => false,
	330	else => true,
	331	};
	332	};
	333
	334	const no_builtin = b: {
	335	if (options.inherited.no_builtin) \|x\| break :b x;
	336	if (options.parent) \|p\| break :b p.no_builtin;
	337
	338	break :b target.cpu.arch.isBpf();
	339	};
	340
	341	const llvm_cpu_features: ?[*:0]const u8 = b: {
	342	if (resolved_target.llvm_cpu_features) \|x\| break :b x;
	343	if (!options.global.use_llvm) break :b null;
	344
	345	var buf = std.array_list.Managed(u8).init(arena);
	346	var disabled_features = std.array_list.Managed(u8).init(arena);
	347	defer disabled_features.deinit();
	348
	349	// Append disabled features after enabled ones, so that their effects aren't overwritten.
	350	for (target.cpu.arch.allFeaturesList()) \|feature\| {
	351	if (feature.llvm_name) \|llvm_name\| {
	352	// Ignore these until we figure out how to handle the concept of omitting features.
	353	// See https://github.com/ziglang/zig/issues/23539
	354	if (target_util.isDynamicAMDGCNFeature(target, feature)) continue;
	355
	356	if (target.cpu.arch.isPowerPC() and @as(std.Target.powerpc.Feature, @enumFromInt(feature.index)) == .@"64bit") continue;
	357	if (target.cpu.arch.isX86() and @as(std.Target.x86.Feature, @enumFromInt(feature.index)) == .x32) continue;
	358
	359	var is_enabled = target.cpu.features.isEnabled(feature.index);
	360	if (target.cpu.arch == .s390x and @as(std.Target.s390x.Feature, @enumFromInt(feature.index)) == .backchain) {
	361	is_enabled = !omit_frame_pointer;
	362	}
	363
	364	if (is_enabled) {
	365	try buf.ensureUnusedCapacity(2 + llvm_name.len);
	366	buf.appendAssumeCapacity('+');
	367	buf.appendSliceAssumeCapacity(llvm_name);
	368	buf.appendAssumeCapacity(',');
	369	} else {
	370	try disabled_features.ensureUnusedCapacity(2 + llvm_name.len);
	371	disabled_features.appendAssumeCapacity('-');
	372	disabled_features.appendSliceAssumeCapacity(llvm_name);
	373	disabled_features.appendAssumeCapacity(',');
	374	}
	375	}
	376	}
	377
	378	try buf.appendSlice(disabled_features.items);
	379	if (buf.items.len == 0) break :b "";
	380	assert(std.mem.endsWith(u8, buf.items, ","));
	381	buf.items[buf.items.len - 1] = 0;
	382	buf.shrinkAndFree(buf.items.len);
	383	break :b buf.items[0 .. buf.items.len - 1 :0].ptr;
	384	};
	385
	386	const mod = try arena.create(Module);
	387	mod.* = .{
	388	.root = options.paths.root,
	389	.root_src_path = options.paths.root_src_path,
	390	.fully_qualified_name = options.fully_qualified_name,
	391	.resolved_target = .{
	392	.result = target.*,
	393	.is_native_os = resolved_target.is_native_os,
	394	.is_native_abi = resolved_target.is_native_abi,
	395	.is_explicit_dynamic_linker = resolved_target.is_explicit_dynamic_linker,
	396	.llvm_cpu_features = llvm_cpu_features,
	397	},
	398	.optimize_mode = optimize_mode,
	399	.single_threaded = single_threaded,
	400	.error_tracing = error_tracing,
	401	.valgrind = valgrind,
	402	.pic = pic,
	403	.strip = strip,
	404	.omit_frame_pointer = omit_frame_pointer,
	405	.stack_check = stack_check,
	406	.stack_protector = stack_protector,
	407	.code_model = code_model,
	408	.red_zone = red_zone,
	409	.sanitize_c = sanitize_c,
	410	.sanitize_thread = sanitize_thread,
	411	.fuzz = fuzz,
	412	.unwind_tables = unwind_tables,
	413	.cc_argv = options.cc_argv,
	414	.structured_cfg = structured_cfg,
	415	.no_builtin = no_builtin,
	416	};
	417	return mod;
	418	}
	419
	420	/// All fields correspond to `CreateOptions`.
	421	pub const LimitedOptions = struct {
	422	root: Compilation.Path,
	423	root_src_path: []const u8,
	424	fully_qualified_name: []const u8,
	425	};
	426
	427	/// This one can only be used if the Module will only be used for AstGen and earlier in
	428	/// the pipeline. Illegal behavior occurs if a limited module touches Sema.
	429	pub fn createLimited(gpa: Allocator, options: LimitedOptions) Allocator.Error!*Module {
	430	const mod = try gpa.create(Module);
	431	mod.* = .{
	432	.root = options.root,
	433	.root_src_path = options.root_src_path,
	434	.fully_qualified_name = options.fully_qualified_name,
	435
	436	.resolved_target = undefined,
	437	.optimize_mode = undefined,
	438	.code_model = undefined,
	439	.single_threaded = undefined,
	440	.error_tracing = undefined,
	441	.valgrind = undefined,
	442	.pic = undefined,
	443	.strip = undefined,
	444	.omit_frame_pointer = undefined,
	445	.stack_check = undefined,
	446	.stack_protector = undefined,
	447	.red_zone = undefined,
	448	.sanitize_c = undefined,
	449	.sanitize_thread = undefined,
	450	.fuzz = undefined,
	451	.unwind_tables = undefined,
	452	.cc_argv = undefined,
	453	.structured_cfg = undefined,
	454	.no_builtin = undefined,
	455	};
	456	return mod;
	457	}
	458
	459	/// Does not ensure that the module's root directory exists on-disk; see `Builtin.updateFileOnDisk` for that task.
	460	pub fn createBuiltin(arena: Allocator, opts: Builtin, dirs: Compilation.Directories) Allocator.Error!*Module {
	461	const sub_path = "b" ++ std.fs.path.sep_str ++ Cache.binToHex(opts.hash());
	462	const new = try arena.create(Module);
	463	new.* = .{
	464	.root = try .fromRoot(arena, dirs, .global_cache, sub_path),
	465	.root_src_path = "builtin.zig",
	466	.fully_qualified_name = "builtin",
	467	.resolved_target = .{
	468	.result = opts.target,
	469	// These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code.
	470	.is_native_os = false,
	471	.is_native_abi = false,
	472	.is_explicit_dynamic_linker = false,
	473	.llvm_cpu_features = null,
	474	},
	475	.optimize_mode = opts.optimize_mode,
	476	.single_threaded = opts.single_threaded,
	477	.error_tracing = opts.error_tracing,
	478	.valgrind = opts.valgrind,
	479	.pic = opts.pic,
	480	.strip = opts.strip,
	481	.omit_frame_pointer = opts.omit_frame_pointer,
	482	.code_model = opts.code_model,
	483	.sanitize_thread = opts.sanitize_thread,
	484	.fuzz = opts.fuzz,
	485	.unwind_tables = opts.unwind_tables,
	486	.cc_argv = &.{},
	487	// These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code.
	488	.stack_check = false,
	489	.stack_protector = 0,
	490	.red_zone = false,
	491	.sanitize_c = .off,
	492	.structured_cfg = false,
	493	.no_builtin = false,
	494	};
	495	return new;
	496	}
	497
	498	/// Returns the `Builtin` which forms the contents of `@import("builtin")` for this module.
	499	pub fn getBuiltinOptions(m: Module, global: Compilation.Config) Builtin {
	500	assert(global.have_zcu);
	501	return .{
	502	.target = m.resolved_target.result,
	503	.zig_backend = target_util.zigBackend(&m.resolved_target.result, global.use_llvm),
	504	.output_mode = global.output_mode,
	505	.link_mode = global.link_mode,
	506	.unwind_tables = m.unwind_tables,
	507	.is_test = global.is_test,
	508	.single_threaded = m.single_threaded,
	509	.link_libc = global.link_libc,
	510	.link_libcpp = global.link_libcpp,
	511	.optimize_mode = m.optimize_mode,
	512	.error_tracing = m.error_tracing,
	513	.valgrind = m.valgrind,
	514	.sanitize_thread = m.sanitize_thread,
	515	.fuzz = m.fuzz,
	516	.pic = m.pic,
	517	.pie = global.pie,
	518	.strip = m.strip,
	519	.code_model = m.code_model,
	520	.omit_frame_pointer = m.omit_frame_pointer,
	521	.wasi_exec_model = global.wasi_exec_model,
	522	};
	523	}

src/Package.zig deleted-209

...	...	@@ -1,209 +0,0 @@
1		const std = @import("std");
2		const assert = std.debug.assert;
3
4		pub const Module = @import("Package/Module.zig");
5		pub const Fetch = @import("Package/Fetch.zig");
6		pub const build_zig_basename = "build.zig";
7		pub const Manifest = @import("Package/Manifest.zig");
8
9		pub const Fingerprint = packed struct(u64) {
10		id: u32,
11		checksum: u32,
12
13		pub fn generate(rng: std.Random, name: []const u8) Fingerprint {
14		return .{
15		.id = rng.intRangeLessThan(u32, 1, 0xffffffff),
16		.checksum = std.hash.Crc32.hash(name),
17		};
18		}
19
20		pub fn validate(n: Fingerprint, name: []const u8) bool {
21		switch (n.id) {
22		0x00000000, 0xffffffff => return false,
23		else => return std.hash.Crc32.hash(name) == n.checksum,
24		}
25		}
26
27		pub fn int(n: Fingerprint) u64 {
28		return @bitCast(n);
29		}
30		};
31
32		/// A user-readable, file system safe hash that identifies an exact package
33		/// snapshot, including file contents.
34		///
35		/// The hash is not only to prevent collisions but must resist attacks where
36		/// the adversary fully controls the contents being hashed. Thus, it contains
37		/// a full SHA-256 digest.
38		///
39		/// This data structure can be used to store the legacy hash format too. Legacy
40		/// hash format is scheduled to be removed after 0.14.0 is tagged.
41		///
42		/// There's also a third way this structure is used. When using path rather than
43		/// hash, a unique hash is still needed, so one is computed based on the path.
44		pub const Hash = struct {
45		/// Maximum size of a package hash. Unused bytes at the end are
46		/// filled with zeroes.
47		///
48		/// Assumed to be already validated.
49		bytes: [max_len]u8,
50
51		pub const Algo = std.crypto.hash.sha2.Sha256;
52		pub const Digest = [Algo.digest_length]u8;
53
54		/// Example: "nnnn-vvvv-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh"
55		pub const max_len = 32 + 1 + 32 + 1 + (32 + 32 + 200) / 6;
56
57		/// Asserts `s` is valid.
58		pub fn fromSlice(s: []const u8) Hash {
59		assert(validate(s) == .ok);
60		var result: Hash = undefined;
61		@memcpy(result.bytes[0..s.len], s);
62		@memset(result.bytes[s.len..], 0);
63		return result;
64		}
65
66		pub const Validation = enum { ok, short, long, incomplete };
67
68		pub fn validate(s: []const u8) Validation {
69		if (s.len > max_len) return .long;
70		if (s.len < 44) return .short;
71		const n_dashes = std.mem.countScalar(u8, s[0 .. s.len - 44], '-');
72		if (n_dashes < 2) return .incomplete;
73		return .ok;
74		}
75
76		test validate {
77		try std.testing.expectEqual(.short, validate(""));
78		}
79
80		pub fn toSlice(ph: *const Hash) []const u8 {
81		var end: usize = ph.bytes.len;
82		while (true) {
83		end -= 1;
84		if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1];
85		}
86		}
87
88		pub fn eql(a: const Hash, b: const Hash) bool {
89		return std.mem.eql(u8, &a.bytes, &b.bytes);
90		}
91
92		/// Produces "$name-$semver-$hashplus".
93		/// * name is the name field from build.zig.zon, asserted to be at most 32
94		/// bytes and assumed be a valid zig identifier
95		/// * semver is the version field from build.zig.zon, asserted to be at
96		/// most 32 bytes
97		/// * hashplus is the following 33-byte array, base64 encoded using -_ to make
98		/// it filesystem safe:
99		/// - (4 bytes) LE u32 Package ID
100		/// - (4 bytes) LE u32 total decompressed size in bytes, overflow saturated
101		/// - (25 bytes) truncated SHA-256 digest of hashed files of the package
102		pub fn init(digest: Digest, name: []const u8, ver: []const u8, id: u32, size: u32) Hash {
103		assert(name.len <= 32);
104		assert(ver.len <= 32);
105		var result: Hash = undefined;
106		var buf: std.ArrayList(u8) = .initBuffer(&result.bytes);
107		buf.appendSliceAssumeCapacity(name);
108		buf.appendAssumeCapacity('-');
109		buf.appendSliceAssumeCapacity(ver);
110		buf.appendAssumeCapacity('-');
111		var hashplus: [33]u8 = undefined;
112		std.mem.writeInt(u32, hashplus[0..4], id, .little);
113		std.mem.writeInt(u32, hashplus[4..8], size, .little);
114		hashplus[8..].* = digest[0..25].*;
115		_ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(44), &hashplus);
116		@memset(buf.unusedCapacitySlice(), 0);
117		return result;
118		}
119
120		/// Produces a unique hash based on the path provided. The result should
121		/// not be user-visible.
122		pub fn initPath(sub_path: []const u8, is_global: bool) Hash {
123		var result: Hash = .{ .bytes = @splat(0) };
124		var i: usize = 0;
125		if (is_global) {
126		result.bytes[0] = '/';
127		i += 1;
128		}
129		if (i + sub_path.len <= result.bytes.len) {
130		@memcpy(result.bytes[i..][0..sub_path.len], sub_path);
131		return result;
132		}
133		var bin_digest: [Algo.digest_length]u8 = undefined;
134		Algo.hash(sub_path, &bin_digest, .{});
135		_ = std.fmt.bufPrint(result.bytes[i..], "{x}", .{&bin_digest}) catch unreachable;
136		return result;
137		}
138
139		pub fn projectId(hash: *const Hash) ProjectId {
140		const bytes = hash.toSlice();
141		const name = std.mem.sliceTo(bytes, '-');
142		const encoded_hashplus = bytes[bytes.len - 44 ..];
143		var hashplus: [33]u8 = undefined;
144		std.base64.url_safe_no_pad.Decoder.decode(&hashplus, encoded_hashplus) catch unreachable;
145		const fingerprint_id = std.mem.readInt(u32, hashplus[0..4], .little);
146		return .init(name, fingerprint_id);
147		}
148
149		test projectId {
150		const hash: Hash = .fromSlice("pulseaudio-16.1.1-9-mk_62MZkNwBaFwiZ7ZVrYRIf_3dTqqJR5PbMRCJzSuLw");
151		const project_id = hash.projectId();
152
153		var expected_name: [32]u8 = @splat(0);
154		expected_name[0.."pulseaudio".len].* = "pulseaudio".*;
155		try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name);
156
157		try std.testing.expectEqual(0xd8fa4f9a, project_id.fingerprint_id);
158		}
159
160		test "projectId with dashes in the base64" {
161		const hash: Hash = .fromSlice("dvui-0.4.0-dev-AQFJmayi2gAKE7FeJoF61v5U1IV9-SupoEcFutIZYpkC");
162		const project_id = hash.projectId();
163
164		var expected_name: [32]u8 = @splat(0);
165		expected_name[0.."dvui".len].* = "dvui".*;
166		try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name);
167
168		try std.testing.expectEqual(0x99490101, project_id.fingerprint_id);
169		}
170		};
171
172		/// Minimum information required to identify whether a package is an artifact
173		/// of a given project.
174		pub const ProjectId = struct {
175		/// Bytes after name.len are set to zero.
176		padded_name: [32]u8,
177		fingerprint_id: u32,
178
179		pub fn init(name: []const u8, fingerprint_id: u32) ProjectId {
180		var padded_name: [32]u8 = @splat(0);
181		@memcpy(padded_name[0..name.len], name);
182		return .{
183		.padded_name = padded_name,
184		.fingerprint_id = fingerprint_id,
185		};
186		}
187
188		pub fn eql(a: const ProjectId, b: const ProjectId) bool {
189		return a.fingerprint_id == b.fingerprint_id and std.mem.eql(u8, &a.padded_name, &b.padded_name);
190		}
191
192		pub fn hash(a: *const ProjectId) u64 {
193		const x: u64 = @bitCast(a.padded_name[0..8].*);
194		return std.hash.int(x \| a.fingerprint_id);
195		}
196		};
197
198		test Hash {
199		const example_digest: Hash.Digest = .{
200		0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87,
201		0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f,
202		};
203		const result: Hash = .init(example_digest, "nasm", "2.16.1-3", 0xcafebabe, 10 * 1024 * 1024);
204		try std.testing.expectEqualStrings("nasm-2.16.1-3-vrr-ygAAoADH9XG3tOdvPNuHen_d-XeHndOG-nNXmved", result.toSlice());
205		}
206
207		test {
208		_ = Fetch;
209		}

src/Package/Fetch.zig deleted-2283

...	...	@@ -1,2283 +0,0 @@
1		//! Represents one independent job whose responsibility is to:
2		//!
3		//! 1. Check the local zig package directory to see if the hash already exists.
4		//! If so, load, parse, and validate the build.zig.zon file therein, and
5		//! goto step 9. Likewise if the location is a relative path, treat this
6		//! the same as a cache hit. Otherwise, proceed.
7		//! 2. Check the global package cache for a compressed tarball matching the
8		//! hash. If it is found, unpack the contents into a temporary directory inside
9		//! project local zig cache. Rename this directory into the local zig package
10		//! directory and goto step 9, skipping step 10.
11		//! 3. Fetch and unpack a URL into a temporary directory.
12		//! 4. Load, parse, and validate the build.zig.zon file therein. It is allowed
13		//! for the file to be missing, in which case this fetched package is considered
14		//! to be a "naked" package.
15		//! 5. Apply inclusion rules of the build.zig.zon to the temporary directory by
16		//! deleting excluded files. If any files had errors for files that were
17		//! ultimately excluded, those errors should be ignored, such as failure to
18		//! create symlinks that weren't supposed to be included anyway.
19		//! 6. Compute the package hash based on the remaining files in the temporary
20		//! directory.
21		//! 7. Rename the temporary directory into the local zig package directory. If
22		//! the hash already exists, delete the temporary directory and leave the zig
23		//! package directory untouched as it may be in use. This is done even if
24		//! the hash is invalid, in case the package with the different hash is used
25		//! in the future.
26		//! 8. Validate the computed hash against the expected hash. If invalid,
27		//! this job is done.
28		//! 9. Spawn a new fetch job for each dependency in the manifest file. Use
29		//! a mutex and a hash map so that redundant jobs do not get queued up.
30		//! 10.Compress the package directory and store it into the global package
31		//! cache.
32		//!
33		//! All of this must be done with only referring to the state inside this struct
34		//! because this work will be done in a dedicated thread.
35		const Fetch = @This();
36
37		const builtin = @import("builtin");
38		const native_os = builtin.os.tag;
39
40		const std = @import("std");
41		const Io = std.Io;
42		const fs = std.fs;
43		const log = std.log.scoped(.fetch);
44		const assert = std.debug.assert;
45		const ascii = std.ascii;
46		const Allocator = std.mem.Allocator;
47		const Cache = std.Build.Cache;
48		const git = @import("Fetch/git.zig");
49		const Package = @import("../Package.zig");
50		const Manifest = Package.Manifest;
51		const ErrorBundle = std.zig.ErrorBundle;
52
53		arena: std.heap.ArenaAllocator,
54		location: Location,
55		location_tok: std.zig.Ast.TokenIndex,
56		hash_tok: std.zig.Ast.OptionalTokenIndex,
57		name_tok: std.zig.Ast.TokenIndex,
58		lazy_status: LazyStatus,
59		/// Same as `parent_packge_root` except it is unchanged when recursing into
60		/// relative file paths (as opposed to URL).
61		remote_package_root: Cache.Path,
62		parent_package_root: Cache.Path,
63		parent_manifest_ast: ?*const std.zig.Ast,
64		prog_node: std.Progress.Node,
65		job_queue: *JobQueue,
66		/// If true, don't add an error for a missing hash. This flag is not passed
67		/// down to recursive dependencies. It's intended to be used only be the CLI.
68		omit_missing_hash_error: bool,
69		/// If true, don't fail when a manifest file is missing the `paths` field,
70		/// which specifies inclusion rules. This is intended to be true for the first
71		/// fetch task and false for the recursive dependencies.
72		allow_missing_paths_field: bool,
73		/// If true and URL points to a Git repository, will use the latest commit.
74		use_latest_commit: bool,
75
76		// Above this are fields provided as inputs to `run`.
77		// Below this are fields populated by `run`.
78
79		/// Relative to the build root of the root package.
80		package_root: Cache.Path,
81		error_bundle: ErrorBundle.Wip,
82		manifest: Manifest,
83		manifest_ast: std.zig.Ast,
84		have_manifest: bool,
85		computed_hash: ComputedHash,
86		/// Fetch logic notices whether a package has a build.zig file and sets this flag.
87		has_build_zig: bool,
88		/// Indicates whether the task aborted due to an out-of-memory condition.
89		oom_flag: bool,
90		/// If `use_latest_commit` was true, this will be set to the commit that was used.
91		/// If the resource pointed to by the location is not a Git-repository, this
92		/// will be left unchanged.
93		latest_commit: ?git.Oid,
94
95		// This field is used by the CLI only, untouched by this file.
96
97		/// The module for this `Fetch` tasks's package, which exposes `build.zig` as
98		/// the root source file.
99		module: ?*Package.Module,
100
101		pub const LazyStatus = enum {
102		/// Not lazy.
103		eager,
104		/// Lazy, found.
105		available,
106		/// Lazy, not found.
107		unavailable,
108		};
109
110		pub const LocalStorage = struct {
111		cache_root: Cache.Path,
112		/// Path to "zig-pkg" inside the package in which the user ran `zig build`.
113		pkg_root: Cache.Path,
114		};
115
116		/// Contains shared state among all `Fetch` tasks.
117		pub const JobQueue = struct {
118		io: Io,
119		mutex: Io.Mutex = .init,
120		/// It's an array hash map so that it can be sorted before rendering the
121		/// dependencies.zig source file.
122		/// Protected by `mutex`.
123		table: Table = .{},
124		/// `table` may be missing some tasks such as ones that failed, so this
125		/// field contains references to all of them.
126		/// Protected by `mutex`.
127		all_fetches: std.ArrayList(*Fetch) = .empty,
128		prog_node: std.Progress.Node,
129
130		http_client: *std.http.Client,
131		/// This tracks `Fetch` tasks as well as recompression tasks.
132		group: Io.Group = .init,
133		global_cache: Cache.Directory,
134		/// If `null`, indicates fetch globally only.
135		local_storage: ?*const LocalStorage,
136		/// If true then, no fetching occurs, and:
137		/// * The `global_cache` directory is assumed to be the direct parent
138		/// directory of on-disk packages rather than having the "p/" directory
139		/// prefix inside of it.
140		/// * An error occurs if any non-lazy packages are not already present in
141		/// the package cache directory.
142		/// * Missing hash field causes an error, and no fetching occurs so it does
143		/// not print the correct hash like usual.
144		read_only: bool,
145		recursive: bool,
146		/// Dumps hash information to stdout which can be used to troubleshoot why
147		/// two hashes of the same package do not match.
148		/// If this is true, `recursive` must be false.
149		debug_hash: bool,
150		mode: Mode,
151		/// Set of hashes that will be additionally fetched even if they are marked
152		/// as lazy.
153		unlazy_set: UnlazySet = .{},
154		/// Identifies paths that override all packages in the tree with matching
155		/// project ids.
156		fork_set: ForkSet = .{},
157
158		pub const Mode = enum {
159		/// Non-lazy dependencies are always fetched.
160		/// Lazy dependencies are fetched only when needed.
161		needed,
162		/// Both non-lazy and lazy dependencies are always fetched.
163		all,
164		};
165		pub const Table = std.array_hash_map.Auto(Package.Hash, *Fetch);
166		pub const UnlazySet = std.array_hash_map.Auto(Package.Hash, void);
167		pub const ForkSet = std.array_hash_map.Custom(Fork, void, Fork.Context, false);
168
169		pub const Fork = struct {
170		path: Cache.Path,
171		manifest_ast: std.zig.Ast,
172		manifest: Package.Manifest,
173		uses: usize,
174
175		pub const Context = struct {
176		pub fn hash(_: @This(), a: Fork) u32 {
177		const project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id);
178		return @truncate(project_id.hash());
179		}
180
181		pub fn eql(_: @This(), a: Fork, b: Fork, _: usize) bool {
182		const a_project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id);
183		const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id);
184		return a_project_id.eql(&b_project_id);
185		}
186		};
187
188		pub const Adapter = struct {
189		pub fn hash(_: @This(), a: Package.ProjectId) u32 {
190		return @truncate(a.hash());
191		}
192
193		pub fn eql(_: @This(), a_project_id: Package.ProjectId, b: Fork, _: usize) bool {
194		const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id);
195		return a_project_id.eql(&b_project_id);
196		}
197		};
198		};
199
200		pub fn deinit(jq: *JobQueue) void {
201		const io = jq.io;
202		jq.group.cancel(io);
203		if (jq.all_fetches.items.len == 0) return;
204		const gpa = jq.all_fetches.items[0].arena.child_allocator;
205		jq.table.deinit(gpa);
206		// These must be deinitialized in reverse order because subsequent
207		// `Fetch` instances are allocated in prior ones' arenas.
208		// Sorry, I know it's a bit weird, but it slightly simplifies the
209		// critical section.
210		while (jq.all_fetches.pop()) \|f\| f.deinit();
211		jq.all_fetches.deinit(gpa);
212		jq.* = undefined;
213		}
214
215		/// Dumps all subsequent error bundles into the first one.
216		pub fn consolidateErrors(jq: *JobQueue) !void {
217		const root = &jq.all_fetches.items[0].error_bundle;
218		const gpa = root.gpa;
219		for (jq.all_fetches.items[1..]) \|fetch\| {
220		if (fetch.error_bundle.root_list.items.len > 0) {
221		var bundle = try fetch.error_bundle.toOwnedBundle("");
222		defer bundle.deinit(gpa);
223		try root.addBundleAsRoots(bundle);
224		}
225		}
226		}
227
228		/// Creates the dependencies.zig source code for the build runner to obtain
229		/// via `@import("@dependencies")`.
230		pub fn createDependenciesSource(jq: JobQueue, buf: std.array_list.Managed(u8)) Allocator.Error!void {
231		const keys = jq.table.keys();
232
233		assert(keys.len != 0); // caller should have added the first one
234		if (keys.len == 1) {
235		// This is the first one. It must have no dependencies.
236		return createEmptyDependenciesSource(buf);
237		}
238
239		try buf.appendSlice("pub const packages = struct {\n");
240
241		// Ensure the generated .zig file is deterministic.
242		jq.table.sortUnstable(@as(struct {
243		keys: []const Package.Hash,
244		pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
245		return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes);
246		}
247		}, .{ .keys = keys }));
248
249		for (keys, jq.table.values()) \|*hash, fetch\| {
250		if (fetch == jq.all_fetches.items[0]) {
251		// The first one is a dummy package for the current project.
252		continue;
253		}
254
255		const hash_slice = hash.toSlice();
256
257		try buf.print(
258		\\ pub const {f} = struct {{
259		\\
260		, .{std.zig.fmtId(hash_slice)});
261
262		lazy: {
263		switch (fetch.lazy_status) {
264		.eager => break :lazy,
265		.available => {
266		try buf.appendSlice(
267		\\ pub const available = true;
268		\\
269		);
270		break :lazy;
271		},
272		.unavailable => {
273		try buf.appendSlice(
274		\\ pub const available = false;
275		\\ };
276		\\
277		);
278		continue;
279		},
280		}
281		}
282
283		try buf.print(
284		\\ pub const build_root = "{f}";
285		\\
286		, .{std.fmt.alt(fetch.package_root, .formatEscapeString)});
287
288		if (fetch.has_build_zig) {
289		try buf.print(
290		\\ pub const build_zig = @import("{f}");
291		\\
292		, .{std.zig.fmtString(hash_slice)});
293		}
294
295		if (fetch.have_manifest) {
296		const manifest = &fetch.manifest;
297		try buf.appendSlice(
298		\\ pub const deps: []const struct { []const u8, []const u8 } = &.{
299		\\
300		);
301		for (manifest.dependencies.keys(), manifest.dependencies.values()) \|name, dep\| {
302		const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue;
303		try buf.print(
304		" .{{ \"{f}\", \"{f}\" }},\n",
305		.{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) },
306		);
307		}
308
309		try buf.appendSlice(
310		\\ };
311		\\ };
312		\\
313		);
314		} else {
315		try buf.appendSlice(
316		\\ pub const deps: []const struct { []const u8, []const u8 } = &.{};
317		\\ };
318		\\
319		);
320		}
321		}
322
323		try buf.appendSlice(
324		\\};
325		\\
326		\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{
327		\\
328		);
329
330		const root_fetch = jq.all_fetches.items[0];
331		assert(root_fetch.have_manifest);
332		const root_manifest = &root_fetch.manifest;
333
334		for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) \|name, dep\| {
335		const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue;
336		try buf.print(
337		" .{{ \"{f}\", \"{f}\" }},\n",
338		.{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) },
339		);
340		}
341		try buf.appendSlice("};\n");
342		}
343
344		pub fn createEmptyDependenciesSource(buf: *std.array_list.Managed(u8)) Allocator.Error!void {
345		try buf.appendSlice(
346		\\pub const packages = struct {};
347		\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
348		\\
349		);
350		}
351
352		fn recompress(jq: *JobQueue, package_hash: Package.Hash, package_root: Cache.Path) Io.Cancelable!void {
353		const pkg_hash_slice = package_hash.toSlice();
354
355		const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice});
356		defer prog_node.end();
357
358		var dest_sub_path_buf: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined;
359		const dest_path: Cache.Path = .{
360		.root_dir = jq.global_cache,
361		.sub_path = std.fmt.bufPrint(&dest_sub_path_buf, "p/{s}.tar.gz", .{pkg_hash_slice}) catch unreachable,
362		};
363
364		const gpa = jq.http_client.allocator;
365
366		var arena_instance = std.heap.ArenaAllocator.init(gpa);
367		defer arena_instance.deinit();
368		const arena = arena_instance.allocator();
369
370		recompressFallible(jq, arena, dest_path, pkg_hash_slice, package_root, prog_node) catch \|err\| switch (err) {
371		error.Canceled => \|e\| return e,
372		error.ReadFailed => comptime unreachable,
373		error.WriteFailed => comptime unreachable,
374		else => \|e\| log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }),
375		};
376		}
377
378		fn recompressFallible(
379		jq: *JobQueue,
380		arena: Allocator,
381		dest_path: Cache.Path,
382		pkg_hash_slice: []const u8,
383		package_root: Cache.Path,
384		prog_node: std.Progress.Node,
385		) !void {
386		const gpa = jq.http_client.allocator;
387		const io = jq.io;
388
389		// We have to walk the file system up front in order to sort the file
390		// list for determinism purposes. The hash of the recompressed file is
391		// not critical because the true hash is based on the content alone.
392		// However, if we want Zig users to be able to share cached package
393		// data with each other via peer-to-peer protocols, we benefit greatly
394		// from the data being identical on everyone's computers.
395		var scanned_files: std.ArrayList(ScannedFile) = .empty;
396		defer scanned_files.deinit(gpa);
397
398		var pkg_dir = try package_root.root_dir.handle.openDir(io, package_root.sub_path, .{ .iterate = true });
399		defer pkg_dir.close(io);
400
401		{
402		var walker = try pkg_dir.walk(gpa);
403		defer walker.deinit();
404
405		while (try walker.next(io)) \|entry\| {
406		const symlink = switch (entry.kind) {
407		.directory => continue,
408		.file => false,
409		.sym_link => true,
410		else => return error.IllegalFileType,
411		};
412		const entry_path = try arena.dupe(u8, entry.path);
413		// If necessary, normalize path separators to POSIX-style since the tar format requires that.
414		if (comptime (std.fs.path.sep != std.fs.path.sep_posix)) {
415		std.mem.replaceScalar(u8, entry_path, std.fs.path.sep, std.fs.path.sep_posix);
416		}
417		try scanned_files.append(gpa, .{
418		.ptr = entry_path.ptr,
419		.len = @intCast(entry_path.len),
420		.symlink = symlink,
421		});
422		}
423
424		std.mem.sortUnstable(ScannedFile, scanned_files.items, {}, stringCmp);
425		}
426
427		prog_node.setEstimatedTotalItems(scanned_files.items.len);
428
429		var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{
430		.make_path = true,
431		.replace = true,
432		});
433		defer atomic_file.deinit(io);
434
435		var file_write_buffer: [4096]u8 = undefined;
436		var file_writer = atomic_file.file.writer(io, &file_write_buffer);
437
438		var compress_buffer: [std.compress.flate.max_window_len]u8 = undefined;
439		var compress = std.compress.flate.Compress.init(&file_writer.interface, &compress_buffer, .gzip, .level_9) catch \|err\| switch (err) {
440		error.WriteFailed => return file_writer.err.?,
441		};
442
443		var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer };
444		archiver.prefix = pkg_hash_slice;
445
446		var file_read_buffer: [4096]u8 = undefined;
447		var link_buf: [fs.max_path_bytes]u8 = undefined;
448
449		for (scanned_files.items) \|scanned_file\| {
450		const entry_path = scanned_file.ptr[0..scanned_file.len];
451		if (scanned_file.symlink) {
452		const link_name = link_buf[0..try pkg_dir.readLink(io, entry_path, &link_buf)];
453		archiver.writeLink(entry_path, link_name, .{}) catch \|err\| switch (err) {
454		error.WriteFailed => return file_writer.err.?,
455		else => \|e\| return e,
456		};
457		} else {
458		var file = try pkg_dir.openFile(io, entry_path, .{});
459		defer file.close(io);
460		var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer);
461		archiver.writeFile(entry_path, &file_reader, 0) catch \|err\| switch (err) {
462		error.ReadFailed => return file_reader.err.?,
463		error.WriteFailed => return file_writer.err.?,
464		else => \|e\| return e,
465		};
466		}
467		prog_node.completeOne();
468		}
469
470		// intentionally omitting the pointless trailer
471		//try archiver.finish();
472		compress.finish() catch \|err\| switch (err) {
473		error.WriteFailed => return file_writer.err.?,
474		};
475		try file_writer.flush();
476		try atomic_file.replace(io);
477		}
478		};
479
480		const ScannedFile = struct {
481		ptr: [*]const u8,
482		len: u32,
483		symlink: bool,
484		};
485
486		fn stringCmp(_: void, lhs: ScannedFile, rhs: ScannedFile) bool {
487		return std.mem.lessThan(u8, lhs.ptr[0..lhs.len], rhs.ptr[0..rhs.len]);
488		}
489
490		pub const Location = union(enum) {
491		remote: Remote,
492		/// A directory found inside the parent package.
493		relative_path: Cache.Path,
494		/// Recursive Fetch tasks will never use this Location, but it may be
495		/// passed in by the CLI. Indicates the file contents here should be copied
496		/// into the global package cache. It may be a file relative to the cwd or
497		/// absolute, in which case it should be treated exactly like a `file://`
498		/// URL, or a directory, in which case it should be treated as an
499		/// already-unpacked directory (but still needs to be copied into the
500		/// global package cache and have inclusion rules applied).
501		path_or_url: []const u8,
502
503		pub const Remote = struct {
504		url: []const u8,
505		/// If this is null it means the user omitted the hash field from a dependency.
506		/// It will be an error but the logic should still fetch and print the discovered hash.
507		hash: ?Package.Hash,
508		};
509		};
510
511		pub const RunError = error{
512		OutOfMemory,
513		Canceled,
514		/// This error code is intended to be handled by inspecting the
515		/// `error_bundle` field.
516		FetchFailed,
517		};
518
519		pub fn run(f: *Fetch) RunError!void {
520		const job_queue = f.job_queue;
521		const io = job_queue.io;
522		const eb = &f.error_bundle;
523		const arena = f.arena.allocator();
524		const gpa = f.arena.child_allocator;
525
526		try eb.init(gpa);
527
528		// Check the global zig package cache to see if the hash already exists. If
529		// so, load, parse, and validate the build.zig.zon file therein, and skip
530		// ahead to queuing up jobs for dependencies. Likewise if the location is a
531		// relative path, treat this the same as a cache hit. Otherwise, proceed.
532
533		const remote = switch (f.location) {
534		.relative_path => \|pkg_root\| {
535		if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail(
536		f.location_tok,
537		try eb.addString("expected path relative to build root; found absolute path"),
538		);
539		if (f.hash_tok.unwrap()) \|hash_tok\| return f.fail(
540		hash_tok,
541		try eb.addString("path-based dependencies are not hashed"),
542		);
543		// Packages fetched by URL may not use relative paths to escape outside the
544		// fetched package directory from within the package cache.
545
546		// This code path is only reachable recursively and the sub_path
547		// will already have been resolved to no longer have extra ".." or
548		// "." components.
549		assert(job_queue.local_storage != null);
550		log.debug("checking pkg root \"{s}\" against parent package root \"{s}\"", .{
551		pkg_root.sub_path, f.remote_package_root.sub_path,
552		});
553		assert(pkg_root.root_dir.eql(f.remote_package_root.root_dir));
554		if (!std.mem.startsWith(u8, pkg_root.sub_path, f.remote_package_root.sub_path)) return f.fail(
555		f.location_tok,
556		try eb.printString("dependency path outside project: '{f}'", .{pkg_root}),
557		);
558		f.package_root = pkg_root;
559		try loadManifest(f, pkg_root);
560		if (!f.has_build_zig) try checkBuildFileExistence(f);
561		if (!job_queue.recursive) return;
562		return queueJobsForDeps(f);
563		},
564		.remote => \|remote\| remote,
565		.path_or_url => \|path_or_url\| {
566		if (Io.Dir.cwd().openDir(io, path_or_url, .{ .iterate = true })) \|dir\| {
567		var resource: Resource = .{ .dir = dir };
568		return f.runResource(path_or_url, &resource, null, false);
569		} else \|dir_err\| {
570		var server_header_buffer: [init_resource_buffer_size]u8 = undefined;
571
572		const file_err = if (dir_err == error.NotDir) e: {
573		if (Io.Dir.cwd().openFile(io, path_or_url, .{})) \|file\| {
574		var resource: Resource = .{ .file = file.reader(io, &server_header_buffer) };
575		return f.runResource(path_or_url, &resource, null, false);
576		} else \|err\| break :e err;
577		} else dir_err;
578
579		const uri = std.Uri.parse(path_or_url) catch \|uri_err\| {
580		return f.fail(0, try eb.printString(
581		"'{s}' could not be recognized as a file path ({t}) or an URL ({t})",
582		.{ path_or_url, file_err, uri_err },
583		));
584		};
585		var resource: Resource = undefined;
586		try f.initResource(uri, &resource, &server_header_buffer);
587		return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, false);
588		}
589		},
590		};
591
592		var resource_buffer: [init_resource_buffer_size]u8 = undefined;
593
594		if (remote.hash) \|expected_hash\| {
595		const expected_project_id: Package.ProjectId = expected_hash.projectId();
596		if (job_queue.fork_set.getKeyPtrAdapted(expected_project_id, @as(JobQueue.Fork.Adapter, .{}))) \|fork\| {
597		log.debug("using fork {f} for {s}", .{ fork.path, fork.manifest.name });
598		fork.uses += 1;
599		f.package_root = fork.path;
600		f.remote_package_root = f.package_root;
601		f.manifest_ast = fork.manifest_ast;
602		f.manifest = fork.manifest;
603		f.have_manifest = true;
604		try checkBuildFileExistence(f);
605		if (!job_queue.recursive) return;
606		return queueJobsForDeps(f);
607		}
608
609		if (job_queue.local_storage) \|ls\| {
610		const package_root = try ls.pkg_root.join(arena, expected_hash.toSlice());
611		if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) \|_\| {
612		assert(f.lazy_status != .unavailable);
613		f.package_root = package_root;
614		f.remote_package_root = f.package_root;
615		try loadManifest(f, f.package_root);
616		try checkBuildFileExistence(f);
617		if (!job_queue.recursive) return;
618		return queueJobsForDeps(f);
619		} else \|err\| switch (err) {
620		error.FileNotFound => {
621		log.debug("FileNotFound: {f}", .{package_root});
622		if (job_queue.read_only and f.lazy_status == .eager) return f.fail(
623		f.name_tok,
624		try eb.printString("package not found at '{f}'", .{package_root}),
625		);
626		},
627		error.Canceled => \|e\| return e,
628		else => \|e\| {
629		try eb.addRootErrorMessage(.{
630		.msg = try eb.printString("unable to open package cache directory {f}: {t}", .{
631		package_root, e,
632		}),
633		});
634		return error.FetchFailed;
635		},
636		}
637		}
638
639		// Check global cache before remote fetch.
640		const cached_tarball_sub_path = try std.fmt.allocPrint(arena, "p/{s}.tar.gz", .{expected_hash.toSlice()});
641		const cached_tarball_path: Cache.Path = .{
642		.root_dir = job_queue.global_cache,
643		.sub_path = cached_tarball_sub_path,
644		};
645		if (cached_tarball_path.root_dir.handle.openFile(io, cached_tarball_path.sub_path, .{})) \|file\| {
646		log.debug("found global cached tarball {f}", .{cached_tarball_path});
647		var resource: Resource = .{ .file = file.reader(io, &resource_buffer) };
648		return f.runResource(cached_tarball_sub_path, &resource, remote.hash, true);
649		} else \|err\| switch (err) {
650		error.FileNotFound => log.debug("FileNotFound: {f}", .{cached_tarball_path}),
651		error.Canceled => \|e\| return e,
652		else => \|e\| {
653		try eb.addRootErrorMessage(.{
654		.msg = try eb.printString("unable to open globally cached package {f}: {t}", .{
655		cached_tarball_path, e,
656		}),
657		});
658		return error.FetchFailed;
659		},
660		}
661
662		switch (f.lazy_status) {
663		.eager => {},
664		.available => if (!job_queue.unlazy_set.contains(expected_hash)) {
665		f.lazy_status = .unavailable;
666		return;
667		},
668		.unavailable => unreachable,
669		}
670		} else if (job_queue.read_only) {
671		try eb.addRootErrorMessage(.{
672		.msg = try eb.addString("dependency is missing hash field"),
673		.src_loc = try f.srcLoc(f.location_tok),
674		});
675		return error.FetchFailed;
676		}
677
678		// Fetch and unpack the remote into a temporary directory.
679		const uri = std.Uri.parse(remote.url) catch \|err\| return f.fail(
680		f.location_tok,
681		try eb.printString("invalid URI: {t}", .{err}),
682		);
683		var resource: Resource = undefined;
684		try f.initResource(uri, &resource, &resource_buffer);
685		return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, false);
686		}
687
688		pub fn deinit(f: *Fetch) void {
689		f.error_bundle.deinit();
690		f.arena.deinit();
691		}
692
693		/// Consumes `resource`, even if an error is returned.
694		fn runResource(
695		f: *Fetch,
696		uri_path: []const u8,
697		resource: *Resource,
698		remote_hash: ?Package.Hash,
699		disable_recompress: bool,
700		) RunError!void {
701		const job_queue = f.job_queue;
702		assert(!job_queue.read_only);
703
704		const io = job_queue.io;
705		defer resource.deinit(io);
706
707		const arena = f.arena.allocator();
708		const eb = &f.error_bundle;
709		const rand_int = r: {
710		var x: u64 = undefined;
711		io.random(@ptrCast(&x));
712		break :r x;
713		};
714		const tmp_dir_sub_path = ".tmp-" ++ std.fmt.hex(rand_int);
715		const tmp_tmp_dir_sub_path = "tmp/" ++ tmp_dir_sub_path;
716		const tmp_directory_path: Cache.Path = if (job_queue.local_storage) \|ls\|
717		try ls.pkg_root.join(arena, tmp_dir_sub_path)
718		else
719		.{
720		.root_dir = job_queue.global_cache,
721		.sub_path = tmp_tmp_dir_sub_path,
722		};
723
724		const package_sub_path = blk: {
725		var tmp_directory: Cache.Directory = .{
726		.path = tmp_directory_path.sub_path,
727		.handle = handle: {
728		const dir = tmp_directory_path.root_dir.handle.createDirPathOpen(io, tmp_directory_path.sub_path, .{
729		.open_options = .{ .iterate = true },
730		}) catch \|err\| {
731		try eb.addRootErrorMessage(.{
732		.msg = try eb.printString("unable to create temporary directory '{f}': {t}", .{
733		tmp_directory_path, err,
734		}),
735		});
736		return error.FetchFailed;
737		};
738		break :handle dir;
739		},
740		};
741		defer tmp_directory.handle.close(io);
742
743		// Fetch and unpack a resource into a temporary directory.
744		var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory);
745
746		const pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir };
747
748		// Load, parse, and validate the unpacked build.zig.zon file. It is allowed
749		// for the file to be missing, in which case this fetched package is
750		// considered to be a "naked" package.
751		try loadManifest(f, pkg_path);
752
753		const filter: Filter = .{
754		.include_paths = if (f.have_manifest) f.manifest.paths else .{},
755		};
756
757		// Ignore errors that were excluded by manifest, such as failure to
758		// create symlinks that weren't supposed to be included anyway.
759		try unpack_result.validate(f, filter);
760
761		// Apply the manifest's inclusion rules to the temporary directory by
762		// deleting excluded files.
763		// Empty directories have already been omitted by `unpackResource`.
764		// Compute the package hash based on the remaining files in the temporary
765		// directory.
766		f.computed_hash = try computeHash(f, pkg_path, filter);
767
768		if (unpack_result.root_dir.len > 0)
769		break :blk try tmp_directory_path.join(arena, unpack_result.root_dir);
770
771		break :blk tmp_directory_path;
772		};
773
774		const computed_package_hash = computedPackageHash(f);
775
776		// Rename the temporary directory into the local zig package directory. If
777		// the hash already exists, delete the temporary directory and leave the
778		// zig package directory untouched as it may be in use. This is done even
779		// if the hash is invalid, in case the package with the different hash is
780		// used in the future.
781		if (job_queue.local_storage) \|ls\| {
782		f.package_root = try ls.pkg_root.join(arena, computed_package_hash.toSlice());
783		renameTmpIntoCache(io, package_sub_path, f.package_root) catch \|err\| {
784		try eb.addRootErrorMessage(.{ .msg = try eb.printString(
785		"failed renaming temporary directory {f} into package cache directory {f}: {t}",
786		.{ package_sub_path, f.package_root, err },
787		) });
788		return error.FetchFailed;
789		};
790		} else {
791		f.package_root = tmp_directory_path;
792		}
793		f.remote_package_root = f.package_root;
794
795		if (!disable_recompress) {
796		// Spin off a task to recompress the tarball, with filtered files deleted, into
797		// the global cache.
798		job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash, f.package_root });
799		}
800
801		// Remove temporary directory root if not already renamed to global cache.
802		if (!package_sub_path.eql(tmp_directory_path)) {
803		tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch \|err\| switch (err) {
804		error.Canceled => \|e\| return e,
805		else => \|e\| log.warn("failed deleting temporary directory {f}: {t}", .{ tmp_directory_path, e }),
806		};
807		}
808
809		// Validate the computed hash against the expected hash. If invalid, this
810		// job is done.
811
812		if (remote_hash) \|declared_hash\| {
813		const hash_tok = f.hash_tok.unwrap().?;
814		if (!computed_package_hash.eql(&declared_hash)) {
815		return f.fail(hash_tok, try eb.printString(
816		"hash mismatch: manifest declares {s} but the fetched package has {s}",
817		.{ declared_hash.toSlice(), computed_package_hash.toSlice() },
818		));
819		}
820		} else if (!f.omit_missing_hash_error) {
821		const notes_len = 1;
822		try eb.addRootErrorMessage(.{
823		.msg = try eb.addString("dependency is missing hash field"),
824		.src_loc = try f.srcLoc(f.location_tok),
825		.notes_len = notes_len,
826		});
827		const notes_start = try eb.reserveNotes(notes_len);
828		eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
829		.msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}),
830		}));
831		return error.FetchFailed;
832		}
833
834		// Spawn a new fetch job for each dependency in the manifest file. Use
835		// a mutex and a hash map so that redundant jobs do not get queued up.
836		if (!job_queue.recursive) return;
837		return queueJobsForDeps(f);
838		}
839
840		pub fn computedPackageHash(f: *const Fetch) Package.Hash {
841		const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32);
842		if (f.have_manifest) {
843		const man = &f.manifest;
844		var version_buffer: [32]u8 = undefined;
845		const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{f}", .{man.version}) catch &version_buffer;
846		return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size);
847		}
848		// In the future build.zig.zon fields will be added to allow overriding these values
849		// for naked tarballs.
850		return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size);
851		}
852
853		/// `computeHash` gets a free check for the existence of `build.zig`, but when
854		/// not computing a hash, we need to do a syscall to check for it.
855		fn checkBuildFileExistence(f: *Fetch) RunError!void {
856		const io = f.job_queue.io;
857		const eb = &f.error_bundle;
858		if (f.package_root.access(io, Package.build_zig_basename, .{})) \|_\| {
859		f.has_build_zig = true;
860		} else \|err\| switch (err) {
861		error.FileNotFound => {},
862		else => \|e\| {
863		try eb.addRootErrorMessage(.{
864		.msg = try eb.printString("unable to access '{f}{s}': {t}", .{
865		f.package_root, Package.build_zig_basename, e,
866		}),
867		});
868		return error.FetchFailed;
869		},
870		}
871		}
872
873		/// This function populates `f.manifest` or leaves it `null`.
874		fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {
875		const io = f.job_queue.io;
876		const eb = &f.error_bundle;
877		const arena = f.arena.allocator();
878		const manifest_path = try pkg_root.join(arena, Manifest.basename);
879
880		Manifest.load(
881		io,
882		arena,
883		manifest_path,
884		&f.manifest_ast,
885		eb,
886		&f.manifest,
887		f.allow_missing_paths_field,
888		) catch \|err\| switch (err) {
889		error.FileNotFound => return,
890		error.Canceled => \|e\| return e,
891		error.ErrorsBundled => return error.FetchFailed,
892		else => \|e\| {
893		try eb.addRootErrorMessage(.{
894		.msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ manifest_path, e }),
895		});
896		return error.FetchFailed;
897		},
898		};
899		f.have_manifest = true;
900		}
901
902		fn queueJobsForDeps(f: *Fetch) RunError!void {
903		const io = f.job_queue.io;
904
905		assert(f.job_queue.recursive);
906
907		// If the package does not have a build.zig.zon file then there are no dependencies.
908		if (!f.have_manifest) return;
909		const manifest = &f.manifest;
910
911		const new_fetches, const prog_names = nf: {
912		const parent_arena = f.arena.allocator();
913		const gpa = f.arena.child_allocator;
914		const cache_root = f.job_queue.global_cache;
915		const dep_names = manifest.dependencies.keys();
916		const deps = manifest.dependencies.values();
917		// Grab the new tasks into a temporary buffer so we can unlock that mutex
918		// as fast as possible.
919		// This overallocates any fetches that get skipped by the `continue` in the
920		// loop below.
921		const new_fetches = try parent_arena.alloc(Fetch, deps.len);
922		const prog_names = try parent_arena.alloc([]const u8, deps.len);
923		var new_fetch_index: usize = 0;
924
925		try f.job_queue.mutex.lock(io);
926		defer f.job_queue.mutex.unlock(io);
927
928		try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len);
929		try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len));
930
931		// There are four cases here:
932		// * Correct hash is provided by manifest.
933		// - Hash map already has the entry, no need to add it again.
934		// * Incorrect hash is provided by manifest.
935		// - Hash mismatch error emitted; `queueJobsForDeps` is not called.
936		// * Hash is not provided by manifest.
937		// - Hash missing error emitted; `queueJobsForDeps` is not called.
938		// * path-based location is used without a hash.
939		// - Hash is added to the table based on the path alone before
940		// calling run(); no need to add it again.
941		//
942		// If we add a dep as lazy and then later try to add the same dep as eager,
943		// eagerness takes precedence and the existing entry is updated and re-scheduled
944		// for fetching.
945
946		for (dep_names, deps) \|dep_name, dep\| {
947		var promoted_existing_to_eager = false;
948		const new_fetch = &new_fetches[new_fetch_index];
949		const location: Location = switch (dep.location) {
950		.url => \|url\| .{
951		.remote = .{
952		.url = url,
953		.hash = h: {
954		const h = dep.hash orelse break :h null;
955		const pkg_hash: Package.Hash = .fromSlice(h);
956		if (h.len == 0) break :h pkg_hash;
957		const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
958		if (gop.found_existing) {
959		if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) {
960		gop.value_ptr.*.lazy_status = .eager;
961		promoted_existing_to_eager = true;
962		} else {
963		continue;
964		}
965		}
966		gop.value_ptr.* = new_fetch;
967		break :h pkg_hash;
968		},
969		},
970		},
971		.path => \|rel_path\| l: {
972		// This might produce an invalid path, which is checked for
973		// at the beginning of run().
974		const new_root = try f.package_root.resolvePosix(parent_arena, rel_path);
975		const pkg_hash = relativePathDigest(new_root, cache_root);
976		const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
977		if (gop.found_existing) {
978		if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) {
979		gop.value_ptr.*.lazy_status = .eager;
980		promoted_existing_to_eager = true;
981		} else {
982		continue;
983		}
984		}
985		gop.value_ptr.* = new_fetch;
986		break :l .{ .relative_path = new_root };
987		},
988		};
989		prog_names[new_fetch_index] = dep_name;
990		new_fetch_index += 1;
991		if (!promoted_existing_to_eager) {
992		f.job_queue.all_fetches.appendAssumeCapacity(new_fetch);
993		}
994		new_fetch.* = .{
995		.arena = std.heap.ArenaAllocator.init(gpa),
996		.location = location,
997		.location_tok = dep.location_tok,
998		.hash_tok = dep.hash_tok,
999		.name_tok = dep.name_tok,
1000		.lazy_status = switch (f.job_queue.mode) {
1001		.needed => if (dep.lazy) .available else .eager,
1002		.all => .eager,
1003		},
1004		.parent_package_root = f.package_root,
1005		.remote_package_root = f.remote_package_root,
1006		.parent_manifest_ast = &f.manifest_ast,
1007		.prog_node = f.prog_node,
1008		.job_queue = f.job_queue,
1009		.omit_missing_hash_error = false,
1010		.allow_missing_paths_field = true,
1011		.use_latest_commit = false,
1012
1013		.package_root = undefined,
1014		.error_bundle = undefined,
1015		.manifest = undefined,
1016		.manifest_ast = undefined,
1017		.have_manifest = false,
1018		.computed_hash = undefined,
1019		.has_build_zig = false,
1020		.oom_flag = false,
1021		.latest_commit = null,
1022
1023		.module = null,
1024		};
1025		}
1026
1027		f.prog_node.increaseEstimatedTotalItems(new_fetch_index);
1028
1029		break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] };
1030		};
1031
1032		// Now it's time to dispatch tasks.
1033		for (new_fetches, prog_names) \|*new_fetch, prog_name\| {
1034		f.job_queue.group.async(io, workerRun, .{ new_fetch, prog_name });
1035		}
1036		}
1037
1038		pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash {
1039		return .initPath(pkg_root.sub_path, pkg_root.root_dir.eql(cache_root));
1040		}
1041
1042		pub fn workerRun(f: *Fetch, prog_name: []const u8) Io.Cancelable!void {
1043		const prog_node = f.prog_node.start(prog_name, 0);
1044		defer prog_node.end();
1045
1046		run(f) catch \|err\| switch (err) {
1047		error.OutOfMemory => f.oom_flag = true,
1048		error.Canceled => \|e\| return e,
1049		error.FetchFailed => {
1050		// Nothing to do because the errors are already reported in `error_bundle`,
1051		// and a reference is kept to the `Fetch` task inside `all_fetches`.
1052		},
1053		};
1054		}
1055
1056		fn srcLoc(
1057		f: *Fetch,
1058		tok: std.zig.Ast.TokenIndex,
1059		) Allocator.Error!ErrorBundle.SourceLocationIndex {
1060		const ast = f.parent_manifest_ast orelse return .none;
1061		const eb = &f.error_bundle;
1062		const start_loc = ast.tokenLocation(0, tok);
1063		const src_path = try eb.printString("{f}" ++ fs.path.sep_str ++ Manifest.basename, .{f.parent_package_root});
1064		const msg_off = 0;
1065		return eb.addSourceLocation(.{
1066		.src_path = src_path,
1067		.span_start = ast.tokenStart(tok),
1068		.span_end = @intCast(ast.tokenStart(tok) + ast.tokenSlice(tok).len),
1069		.span_main = ast.tokenStart(tok) + msg_off,
1070		.line = @intCast(start_loc.line),
1071		.column = @intCast(start_loc.column),
1072		.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
1073		});
1074		}
1075
1076		fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError {
1077		const eb = &f.error_bundle;
1078		try eb.addRootErrorMessage(.{
1079		.msg = msg_str,
1080		.src_loc = try f.srcLoc(msg_tok),
1081		});
1082		return error.FetchFailed;
1083		}
1084
1085		const Resource = union(enum) {
1086		file: Io.File.Reader,
1087		http_request: HttpRequest,
1088		git: Git,
1089		dir: Io.Dir,
1090
1091		const Git = struct {
1092		session: git.Session,
1093		fetch_stream: git.Session.FetchStream,
1094		want_oid: git.Oid,
1095		};
1096
1097		const HttpRequest = struct {
1098		request: std.http.Client.Request,
1099		response: std.http.Client.Response,
1100		transfer_buffer: []u8,
1101		decompress: std.http.Decompress,
1102		decompress_buffer: []u8,
1103		};
1104
1105		fn deinit(resource: *Resource, io: Io) void {
1106		switch (resource.*) {
1107		.file => \|*file_reader\| file_reader.file.close(io),
1108		.http_request => \|*http_request\| http_request.request.deinit(),
1109		.git => \|*git_resource\| {
1110		git_resource.fetch_stream.deinit();
1111		},
1112		.dir => \|*dir\| dir.close(io),
1113		}
1114		resource.* = undefined;
1115		}
1116
1117		fn reader(resource: Resource) Io.Reader {
1118		return switch (resource.*) {
1119		.file => \|*file_reader\| return &file_reader.interface,
1120		.http_request => \|*http_request\| return http_request.response.readerDecompressing(
1121		http_request.transfer_buffer,
1122		&http_request.decompress,
1123		http_request.decompress_buffer,
1124		),
1125		.git => \|*g\| return &g.fetch_stream.reader,
1126		.dir => unreachable,
1127		};
1128		}
1129		};
1130
1131		const FileType = enum {
1132		tar,
1133		@"tar.gz",
1134		@"tar.xz",
1135		@"tar.zst",
1136		git_pack,
1137		zip,
1138
1139		fn fromPath(file_path: []const u8) ?FileType {
1140		if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar;
1141		if (ascii.endsWithIgnoreCase(file_path, ".tgz")) return .@"tar.gz";
1142		if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz";
1143		if (ascii.endsWithIgnoreCase(file_path, ".txz")) return .@"tar.xz";
1144		if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz";
1145		if (ascii.endsWithIgnoreCase(file_path, ".tzst")) return .@"tar.zst";
1146		if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst";
1147		if (ascii.endsWithIgnoreCase(file_path, ".zip")) return .zip;
1148		if (ascii.endsWithIgnoreCase(file_path, ".jar")) return .zip;
1149		return null;
1150		}
1151
1152		/// Parameter is a content-disposition header value.
1153		fn fromContentDisposition(cd_header: []const u8) ?FileType {
1154		const attach_end = ascii.findIgnoreCase(cd_header, "attachment;") orelse
1155		return null;
1156
1157		var value_start = ascii.findIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse
1158		return null;
1159		value_start += "filename".len;
1160		if (cd_header[value_start] == '*') {
1161		value_start += 1;
1162		}
1163		if (cd_header[value_start] != '=') return null;
1164		value_start += 1;
1165
1166		var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len;
1167		if (cd_header[value_end - 1] == '\"') {
1168		value_end -= 1;
1169		}
1170		return fromPath(cd_header[value_start..value_end]);
1171		}
1172
1173		test fromContentDisposition {
1174		try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42"));
1175		try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\""));
1176		try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\""));
1177		try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\""));
1178		try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz"));
1179		try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\""));
1180
1181		try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null);
1182		try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null);
1183		try std.testing.expect(fromContentDisposition("attachment; size=42") == null);
1184		try std.testing.expect(fromContentDisposition("inline; size=42") == null);
1185		try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null);
1186		try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null);
1187		}
1188		};
1189
1190		const init_resource_buffer_size = git.Packet.max_data_length;
1191
1192		fn initResource(f: Fetch, uri: std.Uri, resource: Resource, reader_buffer: []u8) RunError!void {
1193		const io = f.job_queue.io;
1194		const arena = f.arena.allocator();
1195		const eb = &f.error_bundle;
1196
1197		if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
1198		const path = try uri.path.toRawMaybeAlloc(arena);
1199		const file = f.parent_package_root.openFile(io, path, .{}) catch \|err\| {
1200		return f.fail(f.location_tok, try eb.printString("unable to open {f}/{s}: {t}", .{
1201		f.parent_package_root, path, err,
1202		}));
1203		};
1204		resource.* = .{ .file = file.reader(io, reader_buffer) };
1205		return;
1206		}
1207
1208		const http_client = f.job_queue.http_client;
1209
1210		if (ascii.eqlIgnoreCase(uri.scheme, "http") or
1211		ascii.eqlIgnoreCase(uri.scheme, "https"))
1212		{
1213		resource.* = .{ .http_request = .{
1214		.request = http_client.request(.GET, uri, .{}) catch \|err\|
1215		return f.fail(f.location_tok, try eb.printString("server connection failed: {t}", .{err})),
1216		.response = undefined,
1217		.transfer_buffer = reader_buffer,
1218		.decompress_buffer = &.{},
1219		.decompress = undefined,
1220		} };
1221		const request = &resource.http_request.request;
1222		errdefer request.deinit();
1223
1224		request.sendBodiless() catch \|err\|
1225		return f.fail(f.location_tok, try eb.printString("HTTP request failed: {t}", .{err}));
1226
1227		var redirect_buffer: [8000]u8 = undefined;
1228		const response = &resource.http_request.response;
1229		response.* = request.receiveHead(&redirect_buffer) catch \|err\| switch (err) {
1230		error.ReadFailed => {
1231		return f.fail(f.location_tok, try eb.printString("HTTP response read failure: {t}", .{
1232		request.connection.?.getReadError().?,
1233		}));
1234		},
1235		else => \|e\| return f.fail(f.location_tok, try eb.printString("invalid HTTP response: {t}", .{e})),
1236		};
1237
1238		if (response.head.status != .ok) return f.fail(f.location_tok, try eb.printString(
1239		"bad HTTP response code: '{d} {s}'",
1240		.{ response.head.status, response.head.status.phrase() orelse "" },
1241		));
1242
1243		resource.http_request.decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
1244		return;
1245		}
1246
1247		if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or
1248		ascii.eqlIgnoreCase(uri.scheme, "git+https"))
1249		{
1250		var transport_uri = uri;
1251		transport_uri.scheme = uri.scheme["git+".len..];
1252		var session = git.Session.init(arena, http_client, transport_uri, reader_buffer) catch \|err\| {
1253		return f.fail(
1254		f.location_tok,
1255		try eb.printString("unable to discover remote git server capabilities: {t}", .{err}),
1256		);
1257		};
1258
1259		const want_oid = want_oid: {
1260		const want_ref =
1261		if (uri.fragment) \|fragment\| try fragment.toRawMaybeAlloc(arena) else "HEAD";
1262		if (git.Oid.parseAny(want_ref)) \|oid\| break :want_oid oid else \|_\| {}
1263
1264		const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref});
1265		const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref});
1266
1267		var ref_iterator: git.Session.RefIterator = undefined;
1268		session.listRefs(&ref_iterator, .{
1269		.ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },
1270		.include_peeled = true,
1271		.buffer = reader_buffer,
1272		}) catch \|err\| return f.fail(f.location_tok, try eb.printString("unable to list refs: {t}", .{err}));
1273		defer ref_iterator.deinit();
1274		while (ref_iterator.next() catch \|err\| {
1275		return f.fail(f.location_tok, try eb.printString(
1276		"unable to iterate refs: {s}",
1277		.{@errorName(err)},
1278		));
1279		}) \|ref\| {
1280		if (std.mem.eql(u8, ref.name, want_ref) or
1281		std.mem.eql(u8, ref.name, want_ref_head) or
1282		std.mem.eql(u8, ref.name, want_ref_tag))
1283		{
1284		break :want_oid ref.peeled orelse ref.oid;
1285		}
1286		}
1287		return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref}));
1288		};
1289		if (f.use_latest_commit) {
1290		f.latest_commit = want_oid;
1291		} else if (uri.fragment == null) {
1292		const notes_len = 1;
1293		try eb.addRootErrorMessage(.{
1294		.msg = try eb.addString("url field is missing an explicit ref"),
1295		.src_loc = try f.srcLoc(f.location_tok),
1296		.notes_len = notes_len,
1297		});
1298		const notes_start = try eb.reserveNotes(notes_len);
1299		eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
1300		.msg = try eb.printString("try .url = \"{f}#{f}\",", .{
1301		uri.fmt(.{ .scheme = true, .authority = true, .path = true }),
1302		want_oid,
1303		}),
1304		}));
1305		return error.FetchFailed;
1306		}
1307
1308		var want_oid_buf: [git.Oid.max_formatted_length]u8 = undefined;
1309		_ = std.fmt.bufPrint(&want_oid_buf, "{f}", .{want_oid}) catch unreachable;
1310		resource.* = .{ .git = .{
1311		.session = session,
1312		.fetch_stream = undefined,
1313		.want_oid = want_oid,
1314		} };
1315		const fetch_stream = &resource.git.fetch_stream;
1316		session.fetch(fetch_stream, &.{&want_oid_buf}, reader_buffer) catch \|err\| {
1317		return f.fail(f.location_tok, try eb.printString("unable to create fetch stream: {t}", .{err}));
1318		};
1319		errdefer fetch_stream.deinit(fetch_stream);
1320
1321		return;
1322		}
1323
1324		return f.fail(f.location_tok, try eb.printString("unsupported URL scheme: {s}", .{uri.scheme}));
1325		}
1326
1327		fn unpackResource(
1328		f: *Fetch,
1329		resource: *Resource,
1330		uri_path: []const u8,
1331		tmp_directory: Cache.Directory,
1332		) RunError!UnpackResult {
1333		const eb = &f.error_bundle;
1334		const file_type = switch (resource.*) {
1335		.file => FileType.fromPath(uri_path) orelse
1336		return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})),
1337
1338		.http_request => \|*http_request\| ft: {
1339		const head = &http_request.response.head;
1340
1341		// Content-Type takes first precedence.
1342		const content_type = head.content_type orelse
1343		return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header"));
1344
1345		// Extract the MIME type, ignoring charset and boundary directives
1346		const mime_type_end = std.mem.indexOf(u8, content_type, ";") orelse content_type.len;
1347		const mime_type = content_type[0..mime_type_end];
1348
1349		if (ascii.eqlIgnoreCase(mime_type, "application/x-tar"))
1350		break :ft .tar;
1351
1352		if (ascii.eqlIgnoreCase(mime_type, "application/gzip") or
1353		ascii.eqlIgnoreCase(mime_type, "application/x-gzip") or
1354		ascii.eqlIgnoreCase(mime_type, "application/tar+gzip") or
1355		ascii.eqlIgnoreCase(mime_type, "application/x-tar-gz") or
1356		ascii.eqlIgnoreCase(mime_type, "application/x-gtar-compressed"))
1357		{
1358		break :ft .@"tar.gz";
1359		}
1360
1361		if (ascii.eqlIgnoreCase(mime_type, "application/x-xz"))
1362		break :ft .@"tar.xz";
1363
1364		if (ascii.eqlIgnoreCase(mime_type, "application/zstd"))
1365		break :ft .@"tar.zst";
1366
1367		if (ascii.eqlIgnoreCase(mime_type, "application/zip") or
1368		ascii.eqlIgnoreCase(mime_type, "application/x-zip-compressed") or
1369		ascii.eqlIgnoreCase(mime_type, "application/java-archive"))
1370		{
1371		break :ft .zip;
1372		}
1373
1374		if (!ascii.eqlIgnoreCase(mime_type, "application/octet-stream") and
1375		!ascii.eqlIgnoreCase(mime_type, "application/x-compressed"))
1376		{
1377		return f.fail(f.location_tok, try eb.printString(
1378		"unrecognized 'Content-Type' header: '{s}'",
1379		.{content_type},
1380		));
1381		}
1382
1383		// Next, the filename from 'content-disposition: attachment' takes precedence.
1384		if (head.content_disposition) \|cd_header\| {
1385		break :ft FileType.fromContentDisposition(cd_header) orelse {
1386		return f.fail(f.location_tok, try eb.printString(
1387		"unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",
1388		.{cd_header},
1389		));
1390		};
1391		}
1392
1393		// Finally, the path from the URI is used.
1394		break :ft FileType.fromPath(uri_path) orelse {
1395		return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path}));
1396		};
1397		},
1398
1399		.git => .git_pack,
1400
1401		.dir => \|dir\| {
1402		f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch \|err\| {
1403		return f.fail(f.location_tok, try eb.printString("unable to copy directory '{s}': {t}", .{
1404		uri_path, err,
1405		}));
1406		};
1407		return .{};
1408		},
1409		};
1410
1411		switch (file_type) {
1412		.tar => {
1413		return unpackTarball(f, tmp_directory.handle, resource.reader());
1414		},
1415		.@"tar.gz" => {
1416		var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
1417		var decompress: std.compress.flate.Decompress = .init(resource.reader(), .gzip, &flate_buffer);
1418		return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
1419		},
1420		.@"tar.xz" => {
1421		const gpa = f.arena.child_allocator;
1422		var decompress = std.compress.xz.Decompress.init(resource.reader(), gpa, &.{}) catch \|err\|
1423		return f.fail(f.location_tok, try eb.printString("unable to decompress tarball: {t}", .{err}));
1424		defer decompress.deinit();
1425		return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
1426		},
1427		.@"tar.zst" => {
1428		const window_len = std.compress.zstd.default_window_len;
1429		const window_buffer = try f.arena.allocator().alloc(u8, window_len + std.compress.zstd.block_size_max);
1430		var decompress: std.compress.zstd.Decompress = .init(resource.reader(), window_buffer, .{
1431		.verify_checksum = false,
1432		.window_len = window_len,
1433		});
1434		return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
1435		},
1436		.git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch \|err\| switch (err) {
1437		error.FetchFailed, error.OutOfMemory => \|e\| return e,
1438		else => \|e\| return f.fail(f.location_tok, try eb.printString("unable to unpack git files: {t}", .{e})),
1439		},
1440		.zip => return unzip(f, tmp_directory.handle, resource.reader()) catch \|err\| switch (err) {
1441		error.ReadFailed => return f.fail(f.location_tok, try eb.printString(
1442		"failed reading resource: {t}",
1443		.{err},
1444		)),
1445		else => \|e\| return e,
1446		},
1447		}
1448		}
1449
1450		fn unpackTarball(f: Fetch, out_dir: Io.Dir, reader: Io.Reader) RunError!UnpackResult {
1451		const eb = &f.error_bundle;
1452		const arena = f.arena.allocator();
1453		const io = f.job_queue.io;
1454
1455		var diagnostics: std.tar.Diagnostics = .{ .allocator = arena };
1456
1457		std.tar.pipeToFileSystem(io, out_dir, reader, .{
1458		.diagnostics = &diagnostics,
1459		.strip_components = 0,
1460		.mode_mode = .ignore,
1461		.exclude_empty_directories = true,
1462		}) catch \|err\| return f.fail(
1463		f.location_tok,
1464		try eb.printString("unable to unpack tarball to temporary directory: {t}", .{err}),
1465		);
1466
1467		var res: UnpackResult = .{ .root_dir = diagnostics.root_dir };
1468		if (diagnostics.errors.items.len > 0) {
1469		try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack tarball");
1470		for (diagnostics.errors.items) \|item\| {
1471		switch (item) {
1472		.unable_to_create_file => \|i\| res.unableToCreateFile(stripRoot(i.file_name, res.root_dir), i.code),
1473		.unable_to_create_sym_link => \|i\| res.unableToCreateSymLink(stripRoot(i.file_name, res.root_dir), i.link_name, i.code),
1474		.unsupported_file_type => \|i\| res.unsupportedFileType(stripRoot(i.file_name, res.root_dir), @intFromEnum(i.file_type)),
1475		.components_outside_stripped_prefix => unreachable, // unreachable with strip_components = 0
1476		}
1477		}
1478		}
1479		return res;
1480		}
1481
1482		fn unzip(
1483		f: *Fetch,
1484		out_dir: Io.Dir,
1485		reader: *Io.Reader,
1486		) error{ ReadFailed, OutOfMemory, Canceled, FetchFailed }!UnpackResult {
1487		// We write the entire contents to a file first because zip files
1488		// must be processed back to front and they could be too large to
1489		// load into memory.
1490
1491		const io = f.job_queue.io;
1492		const cache_root = f.job_queue.global_cache;
1493		const prefix = "tmp/";
1494		const suffix = ".zip";
1495		const eb = &f.error_bundle;
1496		const random_len = @sizeOf(u64) * 2;
1497
1498		var zip_path: [prefix.len + random_len + suffix.len]u8 = undefined;
1499		zip_path[0..prefix.len].* = prefix.*;
1500		zip_path[prefix.len + random_len ..].* = suffix.*;
1501
1502		var zip_file = while (true) {
1503		const random_integer = r: {
1504		var x: u64 = undefined;
1505		io.random(@ptrCast(&x));
1506		break :r x;
1507		};
1508		zip_path[prefix.len..][0..random_len].* = std.fmt.hex(random_integer);
1509
1510		break cache_root.handle.createFile(io, &zip_path, .{
1511		.exclusive = true,
1512		.read = true,
1513		}) catch \|err\| switch (err) {
1514		error.PathAlreadyExists => continue,
1515		error.FileNotFound => {
1516		cache_root.handle.createDir(io, prefix, .default_dir) catch \|dir_err\| switch (dir_err) {
1517		error.Canceled => \|e\| return e,
1518		// error.PathAlreadyExists is considered a failure here because
1519		// it implies that the prefix is not a directory.
1520		else => \|e\| return f.fail(
1521		f.location_tok,
1522		try eb.printString("failed to create temporary directory: {t}", .{e}),
1523		),
1524		};
1525		continue;
1526		},
1527		error.Canceled => \|e\| return e,
1528		else => \|e\| return f.fail(
1529		f.location_tok,
1530		try eb.printString("failed to create temporary zip file: {t}", .{e}),
1531		),
1532		};
1533		};
1534		defer zip_file.close(io);
1535		var zip_file_buffer: [4096]u8 = undefined;
1536		var zip_file_reader = b: {
1537		var zip_file_writer = zip_file.writer(io, &zip_file_buffer);
1538
1539		_ = reader.streamRemaining(&zip_file_writer.interface) catch \|err\| switch (err) {
1540		error.ReadFailed => \|e\| return e,
1541		error.WriteFailed => return f.fail(
1542		f.location_tok,
1543		try eb.printString("failed writing temporary zip file: {t}", .{err}),
1544		),
1545		};
1546		zip_file_writer.interface.flush() catch \|err\| return f.fail(
1547		f.location_tok,
1548		try eb.printString("failed writing temporary zip file: {t}", .{err}),
1549		);
1550		break :b zip_file_writer.moveToReader();
1551		};
1552
1553		var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() };
1554		// no need to deinit since we are using an arena allocator
1555
1556		zip_file_reader.seekTo(0) catch \|err\|
1557		return f.fail(f.location_tok, try eb.printString("failed to seek temporary zip file: {t}", .{err}));
1558		std.zip.extract(out_dir, &zip_file_reader, .{
1559		.allow_backslashes = true,
1560		.diagnostics = &diagnostics,
1561		}) catch \|err\| return f.fail(f.location_tok, try eb.printString("zip extract failed: {t}", .{err}));
1562
1563		cache_root.handle.deleteFile(io, &zip_path) catch \|err\|
1564		return f.fail(f.location_tok, try eb.printString("delete temporary zip failed: {t}", .{err}));
1565
1566		return .{ .root_dir = diagnostics.root_dir };
1567		}
1568
1569		fn unpackGitPack(f: Fetch, out_dir: Io.Dir, resource: Resource.Git) anyerror!UnpackResult {
1570		const io = f.job_queue.io;
1571		const arena = f.arena.allocator();
1572		// TODO don't try to get a gpa from an arena. expose this dependency higher up
1573		// because the backing of arena could be page allocator
1574		const gpa = f.arena.child_allocator;
1575		const object_format: git.Oid.Format = resource.want_oid;
1576
1577		var res: UnpackResult = .{};
1578		// The .git directory is used to store the packfile and associated index, but
1579		// we do not attempt to replicate the exact structure of a real .git
1580		// directory, since that isn't relevant for fetching a package.
1581		{
1582		var pack_dir = try out_dir.createDirPathOpen(io, ".git", .{});
1583		defer pack_dir.close(io);
1584		var pack_file = try pack_dir.createFile(io, "pkg.pack", .{ .read = true });
1585		defer pack_file.close(io);
1586		var pack_file_buffer: [4096]u8 = undefined;
1587		var pack_file_reader = b: {
1588		var pack_file_writer = pack_file.writer(io, &pack_file_buffer);
1589		const fetch_reader = &resource.fetch_stream.reader;
1590		_ = try fetch_reader.streamRemaining(&pack_file_writer.interface);
1591		try pack_file_writer.interface.flush();
1592		break :b pack_file_writer.moveToReader();
1593		};
1594
1595		var index_file = try pack_dir.createFile(io, "pkg.idx", .{ .read = true });
1596		defer index_file.close(io);
1597		var index_file_buffer: [2000]u8 = undefined;
1598		var index_file_writer = index_file.writer(io, &index_file_buffer);
1599		{
1600		const index_prog_node = f.prog_node.start("Index pack", 0);
1601		defer index_prog_node.end();
1602		try git.indexPack(gpa, object_format, &pack_file_reader, &index_file_writer);
1603		}
1604
1605		{
1606		var index_file_reader = index_file.reader(io, &index_file_buffer);
1607		const checkout_prog_node = f.prog_node.start("Checkout", 0);
1608		defer checkout_prog_node.end();
1609		var repository: git.Repository = undefined;
1610		try repository.init(gpa, object_format, &pack_file_reader, &index_file_reader);
1611		defer repository.deinit();
1612		var diagnostics: git.Diagnostics = .{ .allocator = arena };
1613		try repository.checkout(io, out_dir, resource.want_oid, &diagnostics);
1614
1615		if (diagnostics.errors.items.len > 0) {
1616		try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile");
1617		for (diagnostics.errors.items) \|item\| {
1618		switch (item) {
1619		.unable_to_create_file => \|i\| res.unableToCreateFile(i.file_name, i.code),
1620		.unable_to_create_sym_link => \|i\| res.unableToCreateSymLink(i.file_name, i.link_name, i.code),
1621		}
1622		}
1623		}
1624		}
1625		}
1626
1627		try out_dir.deleteTree(io, ".git");
1628		return res;
1629		}
1630
1631		fn recursiveDirectoryCopy(f: *Fetch, dir: Io.Dir, tmp_dir: Io.Dir) anyerror!void {
1632		const gpa = f.arena.child_allocator;
1633		const io = f.job_queue.io;
1634		// Recursive directory copy.
1635		var it = try dir.walk(gpa);
1636		defer it.deinit();
1637		while (try it.next(io)) \|entry\| {
1638		switch (entry.kind) {
1639		.directory => {}, // omit empty directories
1640		.file => {
1641		dir.copyFile(entry.path, tmp_dir, entry.path, io, .{}) catch \|err\| switch (err) {
1642		error.FileNotFound => {
1643		if (fs.path.dirname(entry.path)) \|dirname\| try tmp_dir.createDirPath(io, dirname);
1644		try dir.copyFile(entry.path, tmp_dir, entry.path, io, .{});
1645		},
1646		else => \|e\| return e,
1647		};
1648		},
1649		.sym_link => {
1650		var buf: [fs.max_path_bytes]u8 = undefined;
1651		const link_name = buf[0..try dir.readLink(io, entry.path, &buf)];
1652		// TODO: if this would create a symlink to outside
1653		// the destination directory, fail with an error instead.
1654		tmp_dir.symLink(io, link_name, entry.path, .{}) catch \|err\| switch (err) {
1655		error.FileNotFound => {
1656		if (fs.path.dirname(entry.path)) \|dirname\| try tmp_dir.createDirPath(io, dirname);
1657		try tmp_dir.symLink(io, link_name, entry.path, .{});
1658		},
1659		else => \|e\| return e,
1660		};
1661		},
1662		else => return error.IllegalFileTypeInPackage,
1663		}
1664		}
1665		}
1666
1667		pub fn renameTmpIntoCache(io: Io, tmp_path: Cache.Path, dest_path: Cache.Path) !void {
1668		var handled_missing_dir = false;
1669		while (true) {
1670		Io.Dir.rename(
1671		tmp_path.root_dir.handle,
1672		tmp_path.sub_path,
1673		dest_path.root_dir.handle,
1674		dest_path.sub_path,
1675		io,
1676		) catch \|err\| switch (err) {
1677		error.FileNotFound => {
1678		if (handled_missing_dir) return err;
1679		const parent_sub_path = Io.Dir.path.dirname(dest_path.sub_path).?;
1680		dest_path.root_dir.handle.createDir(io, parent_sub_path, .default_dir) catch \|er\| switch (er) {
1681		error.PathAlreadyExists => handled_missing_dir = true,
1682		else => \|e\| return e,
1683		};
1684		continue;
1685		},
1686		error.DirNotEmpty, error.AccessDenied => {
1687		// Package has been already downloaded and may already be in use on the system.
1688		tmp_path.root_dir.handle.deleteTree(io, tmp_path.sub_path) catch \|er\| switch (er) {
1689		error.Canceled => \|e\| return e,
1690		// Garbage files leftover in zig-cache/tmp/ is, as they say
1691		// on Star Trek, "operating within normal parameters".
1692		else => \|e\| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_path, e }),
1693		};
1694		},
1695		else => \|e\| return e,
1696		};
1697		break;
1698		}
1699		}
1700
1701		const ComputedHash = struct {
1702		digest: Package.Hash.Digest,
1703		total_size: u64,
1704		};
1705
1706		/// Assumes that files not included in the package have already been filtered
1707		/// prior to calling this function. This ensures that files not protected by
1708		/// the hash are not present on the file system. Empty directories are *not
1709		/// hashed* and must not be present on the file system when calling this
1710		/// function.
1711		fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash {
1712		const io = f.job_queue.io;
1713		// All the path name strings need to be in memory for sorting.
1714		const arena = f.arena.allocator();
1715		const gpa = f.arena.child_allocator;
1716		const eb = &f.error_bundle;
1717		const root_dir = pkg_path.root_dir.handle;
1718
1719		// Collect all files, recursively, then sort.
1720		var all_files = std.array_list.Managed(*HashedFile).init(gpa);
1721		defer all_files.deinit();
1722
1723		var deleted_files = std.array_list.Managed(*DeletedFile).init(gpa);
1724		defer deleted_files.deinit();
1725
1726		// Track directories which had any files deleted from them so that empty directories
1727		// can be deleted.
1728		var sus_dirs: std.array_hash_map.String(void) = .empty;
1729		defer sus_dirs.deinit(gpa);
1730
1731		var walker = try root_dir.walk(gpa);
1732		defer walker.deinit();
1733
1734		// Total number of bytes of file contents included in the package.
1735		var total_size: u64 = 0;
1736
1737		{
1738		// The final hash will be a hash of each file hashed independently. This
1739		// allows hashing in parallel.
1740		var group: Io.Group = .init;
1741		defer group.cancel(io);
1742
1743		while (walker.next(io) catch \|err\| {
1744		try eb.addRootErrorMessage(.{ .msg = try eb.printString(
1745		"unable to walk temporary directory '{f}': {t}",
1746		.{ pkg_path, err },
1747		) });
1748		return error.FetchFailed;
1749		}) \|entry\| {
1750		if (entry.kind == .directory) continue;
1751
1752		const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path);
1753		if (!filter.includePath(entry_pkg_path)) {
1754		// Delete instead of including in hash calculation.
1755		const fs_path = try arena.dupe(u8, entry.path);
1756
1757		// Also track the parent directory in case it becomes empty.
1758		if (fs.path.dirname(fs_path)) \|parent\|
1759		try sus_dirs.put(gpa, parent, {});
1760
1761		const deleted_file = try arena.create(DeletedFile);
1762		deleted_file.* = .{
1763		.fs_path = fs_path,
1764		.failure = undefined, // to be populated by the worker
1765		};
1766		group.async(io, workerDeleteFile, .{ io, root_dir, deleted_file });
1767		try deleted_files.append(deleted_file);
1768		continue;
1769		}
1770
1771		const kind: HashedFile.Kind = switch (entry.kind) {
1772		.directory => unreachable,
1773		.file => .file,
1774		.sym_link => .link,
1775		else => return f.fail(f.location_tok, try eb.printString(
1776		"package contains '{s}' which has illegal file type '{t}'",
1777		.{ entry.path, entry.kind },
1778		)),
1779		};
1780
1781		if (std.mem.eql(u8, entry_pkg_path, Package.build_zig_basename))
1782		f.has_build_zig = true;
1783
1784		const fs_path = try arena.dupe(u8, entry.path);
1785		const hashed_file = try arena.create(HashedFile);
1786		hashed_file.* = .{
1787		.fs_path = fs_path,
1788		.normalized_path = try normalizePathAlloc(arena, entry_pkg_path),
1789		.kind = kind,
1790		.hash = undefined, // to be populated by the worker
1791		.failure = undefined, // to be populated by the worker
1792		.size = undefined, // to be populated by the worker
1793		};
1794		group.async(io, workerHashFile, .{ io, root_dir, hashed_file });
1795		try all_files.append(hashed_file);
1796		}
1797
1798		try group.await(io);
1799		}
1800
1801		{
1802		// Sort by length, descending, so that child directories get removed first.
1803		sus_dirs.sortUnstable(@as(struct {
1804		keys: []const []const u8,
1805		pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
1806		return ctx.keys[b_index].len < ctx.keys[a_index].len;
1807		}
1808		}, .{ .keys = sus_dirs.keys() }));
1809
1810		// During this loop, more entries will be added, so we must loop by index.
1811		var i: usize = 0;
1812		while (i < sus_dirs.count()) : (i += 1) {
1813		const sus_dir = sus_dirs.keys()[i];
1814		root_dir.deleteDir(io, sus_dir) catch \|err\| switch (err) {
1815		error.DirNotEmpty => continue,
1816		error.FileNotFound => continue,
1817		else => \|e\| {
1818		try eb.addRootErrorMessage(.{ .msg = try eb.printString(
1819		"unable to delete empty directory '{s}': {s}",
1820		.{ sus_dir, @errorName(e) },
1821		) });
1822		return error.FetchFailed;
1823		},
1824		};
1825		if (fs.path.dirname(sus_dir)) \|parent\| {
1826		try sus_dirs.put(gpa, parent, {});
1827		}
1828		}
1829		}
1830
1831		std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
1832
1833		var hasher = Package.Hash.Algo.init(.{});
1834		var any_failures = false;
1835		for (all_files.items) \|hashed_file\| {
1836		hashed_file.failure catch \|err\| {
1837		any_failures = true;
1838		try eb.addRootErrorMessage(.{
1839		.msg = try eb.printString("unable to hash '{s}': {s}", .{
1840		hashed_file.fs_path, @errorName(err),
1841		}),
1842		});
1843		};
1844		hasher.update(&hashed_file.hash);
1845		total_size += hashed_file.size;
1846		}
1847		for (deleted_files.items) \|deleted_file\| {
1848		deleted_file.failure catch \|err\| {
1849		any_failures = true;
1850		try eb.addRootErrorMessage(.{
1851		.msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{
1852		deleted_file.fs_path, @errorName(err),
1853		}),
1854		});
1855		};
1856		}
1857
1858		if (any_failures) return error.FetchFailed;
1859
1860		if (f.job_queue.debug_hash) {
1861		assert(!f.job_queue.recursive);
1862		// Print something to stdout that can be text diffed to figure out why
1863		// the package hash is different.
1864		dumpHashInfo(io, all_files.items) catch \|err\|
1865		std.process.fatal("unable to write to stdout: {t}", .{err});
1866		}
1867
1868		return .{
1869		.digest = hasher.finalResult(),
1870		.total_size = total_size,
1871		};
1872		}
1873
1874		fn dumpHashInfo(io: Io, all_files: []const *const HashedFile) !void {
1875		var stdout_buffer: [1024]u8 = undefined;
1876		var stdout_writer: Io.File.Writer = .initStreaming(.stdout(), io, &stdout_buffer);
1877		dumpHashInfoWriter(&stdout_writer.interface, all_files) catch \|err\| switch (err) {
1878		error.WriteFailed => return stdout_writer.err.?,
1879		};
1880		try stdout_writer.flush();
1881		}
1882
1883		fn dumpHashInfoWriter(w: Io.Writer, all_files: []const const HashedFile) Io.Writer.Error!void {
1884		for (all_files) \|hashed_file\| {
1885		try w.print("{t}: {x}: {s}\n", .{ hashed_file.kind, &hashed_file.hash, hashed_file.normalized_path });
1886		}
1887		}
1888
1889		fn workerHashFile(io: Io, dir: Io.Dir, hashed_file: *HashedFile) void {
1890		hashed_file.failure = hashFileFallible(io, dir, hashed_file);
1891		}
1892
1893		fn workerDeleteFile(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) void {
1894		deleted_file.failure = deleteFileFallible(io, dir, deleted_file);
1895		}
1896
1897		fn hashFileFallible(io: Io, dir: Io.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
1898		var buf: [8000]u8 = undefined;
1899		var hasher = Package.Hash.Algo.init(.{});
1900		hasher.update(hashed_file.normalized_path);
1901		var file_size: u64 = 0;
1902
1903		switch (hashed_file.kind) {
1904		.file => {
1905		var file = try dir.openFile(io, hashed_file.fs_path, .{});
1906		defer file.close(io);
1907		// Hard-coded false executable bit: https://github.com/ziglang/zig/issues/17463
1908		hasher.update(&.{ 0, 0 });
1909		var file_header: FileHeader = .{};
1910		while (true) {
1911		const bytes_read = try file.readPositional(io, &.{&buf}, file_size);
1912		if (bytes_read == 0) break;
1913		file_size += bytes_read;
1914		hasher.update(buf[0..bytes_read]);
1915		file_header.update(buf[0..bytes_read]);
1916		}
1917		if (file_header.isExecutable()) {
1918		try setExecutable(io, file);
1919		}
1920		},
1921		.link => {
1922		const link_name = buf[0..try dir.readLink(io, hashed_file.fs_path, &buf)];
1923		if (fs.path.sep != canonical_sep) {
1924		// Package hashes are intended to be consistent across
1925		// platforms which means we must normalize path separators
1926		// inside symlinks.
1927		normalizePath(link_name);
1928		}
1929		hasher.update(link_name);
1930		},
1931		}
1932		hasher.final(&hashed_file.hash);
1933		hashed_file.size = file_size;
1934		}
1935
1936		fn deleteFileFallible(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {
1937		try dir.deleteFile(io, deleted_file.fs_path);
1938		}
1939
1940		fn setExecutable(io: Io, file: Io.File) !void {
1941		if (!Io.File.Permissions.has_executable_bit) return;
1942		try file.setPermissions(io, .executable_file);
1943		}
1944
1945		const DeletedFile = struct {
1946		fs_path: []const u8,
1947		failure: Error!void,
1948
1949		const Error =
1950		Io.Dir.DeleteFileError \|\|
1951		Io.Dir.DeleteDirError;
1952		};
1953
1954		const HashedFile = struct {
1955		fs_path: []const u8,
1956		normalized_path: []const u8,
1957		hash: Package.Hash.Digest,
1958		failure: Error!void,
1959		kind: Kind,
1960		size: u64,
1961
1962		const Error =
1963		Io.File.OpenError \|\|
1964		Io.File.ReadPositionalError \|\|
1965		Io.File.StatError \|\|
1966		Io.File.SetPermissionsError \|\|
1967		Io.Dir.ReadLinkError;
1968
1969		const Kind = enum { file, link };
1970
1971		fn lessThan(context: void, lhs: const HashedFile, rhs: const HashedFile) bool {
1972		_ = context;
1973		return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path);
1974		}
1975		};
1976
1977		/// Strips root directory name from file system path.
1978		fn stripRoot(fs_path: []const u8, root_dir: []const u8) []const u8 {
1979		if (root_dir.len == 0 or fs_path.len <= root_dir.len) return fs_path;
1980
1981		if (std.mem.eql(u8, fs_path[0..root_dir.len], root_dir) and fs.path.isSep(fs_path[root_dir.len])) {
1982		return fs_path[root_dir.len + 1 ..];
1983		}
1984
1985		return fs_path;
1986		}
1987
1988		/// Make a file system path identical independently of operating system path inconsistencies.
1989		/// This converts backslashes into forward slashes.
1990		fn normalizePathAlloc(arena: Allocator, pkg_path: []const u8) ![]const u8 {
1991		const normalized = try arena.dupe(u8, pkg_path);
1992		if (fs.path.sep == canonical_sep) return normalized;
1993		normalizePath(normalized);
1994		return normalized;
1995		}
1996
1997		const canonical_sep = fs.path.sep_posix;
1998
1999		fn normalizePath(bytes: []u8) void {
2000		assert(fs.path.sep != canonical_sep);
2001		std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep);
2002		}
2003
2004		const Filter = struct {
2005		include_paths: std.array_hash_map.String(void) = .empty,
2006
2007		/// sub_path is relative to the package root.
2008		pub fn includePath(self: *const Filter, sub_path: []const u8) bool {
2009		if (self.include_paths.count() == 0) return true;
2010		if (self.include_paths.contains("")) return true;
2011		if (self.include_paths.contains(".")) return true;
2012		if (self.include_paths.contains(sub_path)) return true;
2013
2014		// Check if any included paths are parent directories of sub_path.
2015		var dirname = sub_path;
2016		while (std.fs.path.dirname(dirname)) \|next_dirname\| {
2017		if (self.include_paths.contains(next_dirname)) return true;
2018		dirname = next_dirname;
2019		}
2020
2021		return false;
2022		}
2023
2024		test includePath {
2025		const gpa = std.testing.allocator;
2026		var filter: Filter = .{};
2027		defer filter.include_paths.deinit(gpa);
2028
2029		try filter.include_paths.put(gpa, "src", {});
2030		try std.testing.expect(filter.includePath("src/core/unix/SDL_poll.c"));
2031		try std.testing.expect(!filter.includePath(".gitignore"));
2032		}
2033		};
2034
2035		pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash {
2036		if (dep.hash) \|h\| return .fromSlice(h);
2037
2038		switch (dep.location) {
2039		.url => return null,
2040		.path => \|rel_path\| {
2041		var buf: [fs.max_path_bytes]u8 = undefined;
2042		var fba = std.heap.FixedBufferAllocator.init(&buf);
2043		const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch
2044		return null;
2045		return relativePathDigest(new_root, cache_root);
2046		},
2047		}
2048		}
2049
2050		// Detects executable header: ELF or Macho-O magic header or shebang line.
2051		const FileHeader = struct {
2052		header: [4]u8 = undefined,
2053		bytes_read: usize = 0,
2054
2055		pub fn update(self: *FileHeader, buf: []const u8) void {
2056		if (self.bytes_read >= self.header.len) return;
2057		const n = @min(self.header.len - self.bytes_read, buf.len);
2058		@memcpy(self.header[self.bytes_read..][0..n], buf[0..n]);
2059		self.bytes_read += n;
2060		}
2061
2062		fn isScript(self: *FileHeader) bool {
2063		const shebang = "#!";
2064		return std.mem.eql(u8, self.header[0..@min(self.bytes_read, shebang.len)], shebang);
2065		}
2066
2067		fn isElf(self: *FileHeader) bool {
2068		const elf_magic = std.elf.MAGIC;
2069		return std.mem.eql(u8, self.header[0..@min(self.bytes_read, elf_magic.len)], elf_magic);
2070		}
2071
2072		fn isMachO(self: *FileHeader) bool {
2073		if (self.bytes_read < 4) return false;
2074		const magic_number = std.mem.readInt(u32, &self.header, builtin.cpu.arch.endian());
2075		return magic_number == std.macho.MH_MAGIC or
2076		magic_number == std.macho.MH_MAGIC_64 or
2077		magic_number == std.macho.FAT_MAGIC or
2078		magic_number == std.macho.FAT_MAGIC_64 or
2079		magic_number == std.macho.MH_CIGAM or
2080		magic_number == std.macho.MH_CIGAM_64 or
2081		magic_number == std.macho.FAT_CIGAM or
2082		magic_number == std.macho.FAT_CIGAM_64;
2083		}
2084
2085		pub fn isExecutable(self: *FileHeader) bool {
2086		return self.isScript() or self.isElf() or self.isMachO();
2087		}
2088		};
2089
2090		test FileHeader {
2091		var h: FileHeader = .{};
2092		try std.testing.expect(!h.isExecutable());
2093
2094		const elf_magic = std.elf.MAGIC;
2095		h.update(elf_magic[0..2]);
2096		try std.testing.expect(!h.isExecutable());
2097		h.update(elf_magic[2..4]);
2098		try std.testing.expect(h.isExecutable());
2099
2100		h.update(elf_magic[2..4]);
2101		try std.testing.expect(h.isExecutable());
2102
2103		const macho64_magic_bytes = [_]u8{ 0xCF, 0xFA, 0xED, 0xFE };
2104		h.bytes_read = 0;
2105		h.update(&macho64_magic_bytes);
2106		try std.testing.expect(h.isExecutable());
2107
2108		const macho64_cigam_bytes = [_]u8{ 0xFE, 0xED, 0xFA, 0xCF };
2109		h.bytes_read = 0;
2110		h.update(&macho64_cigam_bytes);
2111		try std.testing.expect(h.isExecutable());
2112		}
2113
2114		// Result of the `unpackResource` operation. Enables collecting errors from
2115		// tar/git diagnostic, filtering that errors by manifest inclusion rules and
2116		// emitting remaining errors to an `ErrorBundle`.
2117		const UnpackResult = struct {
2118		errors: []Error = undefined,
2119		errors_count: usize = 0,
2120		root_error_message: []const u8 = "",
2121
2122		// A non empty value means that the package contents are inside a
2123		// sub-directory indicated by the named path.
2124		root_dir: []const u8 = "",
2125
2126		const Error = union(enum) {
2127		unable_to_create_sym_link: struct {
2128		code: anyerror,
2129		file_name: []const u8,
2130		link_name: []const u8,
2131		},
2132		unable_to_create_file: struct {
2133		code: anyerror,
2134		file_name: []const u8,
2135		},
2136		unsupported_file_type: struct {
2137		file_name: []const u8,
2138		file_type: u8,
2139		},
2140
2141		fn excluded(self: Error, filter: Filter) bool {
2142		const file_name = switch (self) {
2143		.unable_to_create_file => \|info\| info.file_name,
2144		.unable_to_create_sym_link => \|info\| info.file_name,
2145		.unsupported_file_type => \|info\| info.file_name,
2146		};
2147		return !filter.includePath(file_name);
2148		}
2149		};
2150
2151		fn allocErrors(self: *UnpackResult, arena: std.mem.Allocator, n: usize, root_error_message: []const u8) !void {
2152		self.root_error_message = try arena.dupe(u8, root_error_message);
2153		self.errors = try arena.alloc(UnpackResult.Error, n);
2154		}
2155
2156		fn hasErrors(self: *UnpackResult) bool {
2157		return self.errors_count > 0;
2158		}
2159
2160		fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) void {
2161		self.errors[self.errors_count] = .{ .unable_to_create_file = .{
2162		.code = err,
2163		.file_name = file_name,
2164		} };
2165		self.errors_count += 1;
2166		}
2167
2168		fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) void {
2169		self.errors[self.errors_count] = .{ .unable_to_create_sym_link = .{
2170		.code = err,
2171		.file_name = file_name,
2172		.link_name = link_name,
2173		} };
2174		self.errors_count += 1;
2175		}
2176
2177		fn unsupportedFileType(self: *UnpackResult, file_name: []const u8, file_type: u8) void {
2178		self.errors[self.errors_count] = .{ .unsupported_file_type = .{
2179		.file_name = file_name,
2180		.file_type = file_type,
2181		} };
2182		self.errors_count += 1;
2183		}
2184
2185		fn validate(self: UnpackResult, f: Fetch, filter: Filter) !void {
2186		if (self.errors_count == 0) return;
2187
2188		var unfiltered_errors: u32 = 0;
2189		for (self.errors) \|item\| {
2190		if (item.excluded(filter)) continue;
2191		unfiltered_errors += 1;
2192		}
2193		if (unfiltered_errors == 0) return;
2194
2195		// Emmit errors to an `ErrorBundle`.
2196		const eb = &f.error_bundle;
2197		try eb.addRootErrorMessage(.{
2198		.msg = try eb.addString(self.root_error_message),
2199		.src_loc = try f.srcLoc(f.location_tok),
2200		.notes_len = unfiltered_errors,
2201		});
2202		var note_i: u32 = try eb.reserveNotes(unfiltered_errors);
2203		for (self.errors) \|item\| {
2204		if (item.excluded(filter)) continue;
2205		switch (item) {
2206		.unable_to_create_sym_link => \|info\| {
2207		eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
2208		.msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{
2209		info.file_name, info.link_name, @errorName(info.code),
2210		}),
2211		}));
2212		},
2213		.unable_to_create_file => \|info\| {
2214		eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
2215		.msg = try eb.printString("unable to create file '{s}': {s}", .{
2216		info.file_name, @errorName(info.code),
2217		}),
2218		}));
2219		},
2220		.unsupported_file_type => \|info\| {
2221		eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{
2222		.msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{
2223		info.file_name, info.file_type,
2224		}),
2225		}));
2226		},
2227		}
2228		note_i += 1;
2229		}
2230
2231		return error.FetchFailed;
2232		}
2233
2234		test validate {
2235		const gpa = std.testing.allocator;
2236		var arena_instance = std.heap.ArenaAllocator.init(gpa);
2237		defer arena_instance.deinit();
2238		const arena = arena_instance.allocator();
2239
2240		// fill UnpackResult with errors
2241		var res: UnpackResult = .{};
2242		try res.allocErrors(arena, 4, "unable to unpack");
2243		try std.testing.expectEqual(0, res.errors_count);
2244		res.unableToCreateFile("dir1/file1", error.File1);
2245		res.unableToCreateSymLink("dir2/file2", "filename", error.SymlinkError);
2246		res.unableToCreateFile("dir1/file3", error.File3);
2247		res.unsupportedFileType("dir2/file4", 'x');
2248		try std.testing.expectEqual(4, res.errors_count);
2249
2250		// create filter, includes dir2, excludes dir1
2251		var filter: Filter = .{};
2252		try filter.include_paths.put(arena, "dir2", {});
2253
2254		// init Fetch
2255		var fetch: Fetch = undefined;
2256		fetch.parent_manifest_ast = null;
2257		fetch.location_tok = 0;
2258		try fetch.error_bundle.init(gpa);
2259		defer fetch.error_bundle.deinit();
2260
2261		// validate errors with filter
2262		try std.testing.expectError(error.FetchFailed, res.validate(&fetch, filter));
2263
2264		// output errors to string
2265		var errors = try fetch.error_bundle.toOwnedBundle("");
2266		defer errors.deinit(gpa);
2267		var aw: Io.Writer.Allocating = .init(gpa);
2268		defer aw.deinit();
2269		try errors.renderToWriter(.{}, &aw.writer);
2270		try std.testing.expectEqualStrings(
2271		\\error: unable to unpack
2272		\\ note: unable to create symlink from 'dir2/file2' to 'filename': SymlinkError
2273		\\ note: file 'dir2/file4' has unsupported type 'x'
2274		\\
2275		, aw.written());
2276		}
2277		};
2278
2279		test {
2280		_ = Filter;
2281		_ = FileType;
2282		_ = UnpackResult;
2283		}

src/Package/Fetch/git.zig deleted-1750

...	...	@@ -1,1750 +0,0 @@
1		//! Git support for package fetching.
2		//!
3		//! This is not intended to support all features of Git: it is limited to the
4		//! basic functionality needed to clone a repository for the purpose of fetching
5		//! a package.
6
7		const std = @import("std");
8		const Io = std.Io;
9		const mem = std.mem;
10		const testing = std.testing;
11		const Allocator = mem.Allocator;
12		const Sha1 = std.crypto.hash.Sha1;
13		const Sha256 = std.crypto.hash.sha2.Sha256;
14		const assert = std.debug.assert;
15
16		/// The ID of a Git object.
17		pub const Oid = union(Format) {
18		sha1: [Sha1.digest_length]u8,
19		sha256: [Sha256.digest_length]u8,
20
21		pub const max_formatted_length = len: {
22		var max: usize = 0;
23		for (std.enums.values(Format)) \|f\| {
24		max = @max(max, f.formattedLength());
25		}
26		break :len max;
27		};
28
29		pub const Format = enum {
30		sha1,
31		sha256,
32
33		pub fn byteLength(f: Format) usize {
34		return switch (f) {
35		.sha1 => Sha1.digest_length,
36		.sha256 => Sha256.digest_length,
37		};
38		}
39
40		pub fn formattedLength(f: Format) usize {
41		return 2 * f.byteLength();
42		}
43		};
44
45		const Hasher = union(Format) {
46		sha1: Sha1,
47		sha256: Sha256,
48
49		fn init(oid_format: Format) Hasher {
50		return switch (oid_format) {
51		.sha1 => .{ .sha1 = Sha1.init(.{}) },
52		.sha256 => .{ .sha256 = Sha256.init(.{}) },
53		};
54		}
55
56		// Must be public for use from HashedReader and HashedWriter.
57		pub fn update(hasher: *Hasher, b: []const u8) void {
58		switch (hasher.*) {
59		inline else => \|*inner\| inner.update(b),
60		}
61		}
62
63		fn finalResult(hasher: *Hasher) Oid {
64		return switch (hasher.*) {
65		inline else => \|*inner, tag\| @unionInit(Oid, @tagName(tag), inner.finalResult()),
66		};
67		}
68		};
69
70		const Hashing = union(Format) {
71		sha1: Io.Writer.Hashing(Sha1),
72		sha256: Io.Writer.Hashing(Sha256),
73
74		fn init(oid_format: Format, buffer: []u8) Hashing {
75		return switch (oid_format) {
76		.sha1 => .{ .sha1 = .init(buffer) },
77		.sha256 => .{ .sha256 = .init(buffer) },
78		};
79		}
80
81		fn writer(h: @This()) Io.Writer {
82		return switch (h.*) {
83		inline else => \|*inner\| &inner.writer,
84		};
85		}
86
87		fn final(h: *@This()) Oid {
88		switch (h.*) {
89		inline else => \|*inner, tag\| {
90		inner.writer.flush() catch unreachable; // hashers cannot fail
91		return @unionInit(Oid, @tagName(tag), inner.hasher.finalResult());
92		},
93		}
94		}
95		};
96
97		pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid {
98		assert(bytes.len == oid_format.byteLength());
99		return switch (oid_format) {
100		inline else => \|tag\| @unionInit(Oid, @tagName(tag), bytes[0..comptime tag.byteLength()].*),
101		};
102		}
103
104		pub fn readBytes(oid_format: Format, reader: *Io.Reader) !Oid {
105		return switch (oid_format) {
106		inline else => \|tag\| @unionInit(Oid, @tagName(tag), (try reader.takeArray(tag.byteLength())).*),
107		};
108		}
109
110		pub fn parse(oid_format: Format, s: []const u8) error{InvalidOid}!Oid {
111		switch (oid_format) {
112		inline else => \|tag\| {
113		if (s.len != tag.formattedLength()) return error.InvalidOid;
114		var bytes: [tag.byteLength()]u8 = undefined;
115		for (&bytes, 0..) \|*b, i\| {
116		b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid;
117		}
118		return @unionInit(Oid, @tagName(tag), bytes);
119		},
120		}
121		}
122
123		test parse {
124		try testing.expectEqualSlices(
125		u8,
126		&.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 },
127		&(try parse(.sha1, "ce919ccf45951856a762ffdb8ef850301cd8c588")).sha1,
128		);
129		try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf45951856a762ffdb8ef850301cd8c588"));
130		try testing.expectError(error.InvalidOid, parse(.sha1, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a"));
131		try testing.expectEqualSlices(
132		u8,
133		&.{ 0x7F, 0x44, 0x4A, 0x92, 0xBD, 0x45, 0x72, 0xEE, 0x4A, 0x28, 0xB2, 0xC6, 0x30, 0x59, 0x92, 0x4A, 0x9C, 0xA1, 0x82, 0x91, 0x38, 0x55, 0x3E, 0xF3, 0xE7, 0xC4, 0x1E, 0xE1, 0x59, 0xAF, 0xAE, 0x7A },
134		&(try parse(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")).sha256,
135		);
136		try testing.expectError(error.InvalidOid, parse(.sha1, "ce919ccf"));
137		try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf"));
138		try testing.expectError(error.InvalidOid, parse(.sha1, "master"));
139		try testing.expectError(error.InvalidOid, parse(.sha256, "master"));
140		try testing.expectError(error.InvalidOid, parse(.sha1, "HEAD"));
141		try testing.expectError(error.InvalidOid, parse(.sha256, "HEAD"));
142		}
143
144		pub fn parseAny(s: []const u8) error{InvalidOid}!Oid {
145		return for (std.enums.values(Format)) \|f\| {
146		if (s.len == f.formattedLength()) break parse(f, s);
147		} else error.InvalidOid;
148		}
149
150		pub fn format(oid: Oid, writer: *Io.Writer) Io.Writer.Error!void {
151		try writer.print("{x}", .{oid.slice()});
152		}
153
154		pub fn slice(oid: *const Oid) []const u8 {
155		return switch (oid.*) {
156		inline else => \|*bytes\| bytes,
157		};
158		}
159		};
160
161		pub const Diagnostics = struct {
162		allocator: Allocator,
163		errors: std.ArrayList(Error) = .empty,
164
165		pub const Error = union(enum) {
166		unable_to_create_sym_link: struct {
167		code: anyerror,
168		file_name: []const u8,
169		link_name: []const u8,
170		},
171		unable_to_create_file: struct {
172		code: anyerror,
173		file_name: []const u8,
174		},
175		};
176
177		pub fn deinit(d: *Diagnostics) void {
178		for (d.errors.items) \|item\| {
179		switch (item) {
180		.unable_to_create_sym_link => \|info\| {
181		d.allocator.free(info.file_name);
182		d.allocator.free(info.link_name);
183		},
184		.unable_to_create_file => \|info\| {
185		d.allocator.free(info.file_name);
186		},
187		}
188		}
189		d.errors.deinit(d.allocator);
190		d.* = undefined;
191		}
192		};
193
194		pub const Repository = struct {
195		odb: Odb,
196
197		pub fn init(
198		repo: *Repository,
199		allocator: Allocator,
200		format: Oid.Format,
201		pack_file: *Io.File.Reader,
202		index_file: *Io.File.Reader,
203		) !void {
204		repo.* = .{ .odb = undefined };
205		try repo.odb.init(allocator, format, pack_file, index_file);
206		}
207
208		pub fn deinit(repository: *Repository) void {
209		repository.odb.deinit();
210		repository.* = undefined;
211		}
212
213		/// Checks out the repository at `commit_oid` to `worktree`.
214		pub fn checkout(
215		repository: *Repository,
216		io: Io,
217		worktree: Io.Dir,
218		commit_oid: Oid,
219		diagnostics: *Diagnostics,
220		) !void {
221		try repository.odb.seekOid(commit_oid);
222		const tree_oid = tree_oid: {
223		const commit_object = try repository.odb.readObject();
224		if (commit_object.type != .commit) return error.NotACommit;
225		break :tree_oid try getCommitTree(repository.odb.format, commit_object.data);
226		};
227		try repository.checkoutTree(io, worktree, tree_oid, "", diagnostics);
228		}
229
230		/// Checks out the tree at `tree_oid` to `worktree`.
231		fn checkoutTree(
232		repository: *Repository,
233		io: Io,
234		dir: Io.Dir,
235		tree_oid: Oid,
236		current_path: []const u8,
237		diagnostics: *Diagnostics,
238		) !void {
239		try repository.odb.seekOid(tree_oid);
240		const tree_object = try repository.odb.readObject();
241		if (tree_object.type != .tree) return error.NotATree;
242		// The tree object may be evicted from the object cache while we're
243		// iterating over it, so we can make a defensive copy here to make sure
244		// it remains valid until we're done with it
245		const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data);
246		defer repository.odb.allocator.free(tree_data);
247
248		var tree_iter: TreeIterator = .{
249		.format = repository.odb.format,
250		.data = tree_data,
251		.pos = 0,
252		};
253		while (try tree_iter.next()) \|entry\| {
254		switch (entry.type) {
255		.directory => {
256		try dir.createDir(io, entry.name, .default_dir);
257		var subdir = try dir.openDir(io, entry.name, .{});
258		defer subdir.close(io);
259		const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name });
260		defer repository.odb.allocator.free(sub_path);
261		try repository.checkoutTree(io, subdir, entry.oid, sub_path, diagnostics);
262		},
263		.file => {
264		try repository.odb.seekOid(entry.oid);
265		const file_object = try repository.odb.readObject();
266		if (file_object.type != .blob) return error.InvalidFile;
267		var file = dir.createFile(io, entry.name, .{ .exclusive = true }) catch \|e\| {
268		const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name });
269		errdefer diagnostics.allocator.free(file_name);
270		try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_file = .{
271		.code = e,
272		.file_name = file_name,
273		} });
274		continue;
275		};
276		defer file.close(io);
277		try file.writePositionalAll(io, file_object.data, 0);
278		},
279		.symlink => {
280		try repository.odb.seekOid(entry.oid);
281		const symlink_object = try repository.odb.readObject();
282		if (symlink_object.type != .blob) return error.InvalidFile;
283		const link_name = symlink_object.data;
284		dir.symLink(io, link_name, entry.name, .{}) catch \|e\| {
285		const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name });
286		errdefer diagnostics.allocator.free(file_name);
287		const link_name_dup = try diagnostics.allocator.dupe(u8, link_name);
288		errdefer diagnostics.allocator.free(link_name_dup);
289		try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{
290		.code = e,
291		.file_name = file_name,
292		.link_name = link_name_dup,
293		} });
294		};
295		},
296		.gitlink => {
297		// Consistent with git archive behavior, create the directory but
298		// do nothing else
299		try dir.createDir(io, entry.name, .default_dir);
300		},
301		}
302		}
303		}
304
305		/// Returns the ID of the tree associated with the given commit (provided as
306		/// raw object data).
307		fn getCommitTree(format: Oid.Format, commit_data: []const u8) !Oid {
308		if (!mem.startsWith(u8, commit_data, "tree ") or
309		commit_data.len < "tree ".len + format.formattedLength() + "\n".len or
310		commit_data["tree ".len + format.formattedLength()] != '\n')
311		{
312		return error.InvalidCommit;
313		}
314		return try .parse(format, commit_data["tree ".len..][0..format.formattedLength()]);
315		}
316
317		const TreeIterator = struct {
318		format: Oid.Format,
319		data: []const u8,
320		pos: usize,
321
322		const Entry = struct {
323		type: Type,
324		executable: bool,
325		name: [:0]const u8,
326		oid: Oid,
327
328		const Type = enum(u4) {
329		directory = 0o4,
330		file = 0o10,
331		symlink = 0o12,
332		gitlink = 0o16,
333		};
334		};
335
336		fn next(iterator: *TreeIterator) !?Entry {
337		if (iterator.pos == iterator.data.len) return null;
338
339		const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree;
340		const mode: packed struct {
341		permission: u9,
342		unused: u3,
343		type: u4,
344		} = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree);
345		const @"type" = std.enums.fromInt(Entry.Type, mode.type) orelse return error.InvalidTree;
346		const executable = switch (mode.permission) {
347		0 => if (@"type" == .file) return error.InvalidTree else false,
348		0o644 => if (@"type" != .file) return error.InvalidTree else false,
349		0o755 => if (@"type" != .file) return error.InvalidTree else true,
350		else => return error.InvalidTree,
351		};
352		iterator.pos = mode_end + 1;
353
354		const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree;
355		const name = iterator.data[iterator.pos..name_end :0];
356		iterator.pos = name_end + 1;
357
358		const oid_length = iterator.format.byteLength();
359		if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree;
360		const oid: Oid = .fromBytes(iterator.format, iterator.data[iterator.pos..][0..oid_length]);
361		iterator.pos += oid_length;
362
363		return .{ .type = @"type", .executable = executable, .name = name, .oid = oid };
364		}
365		};
366		};
367
368		/// A Git object database backed by a packfile. A packfile index is also used
369		/// for efficient access to objects in the packfile.
370		///
371		/// The format of the packfile and its associated index are documented in
372		/// [pack-format](https://git-scm.com/docs/pack-format).
373		const Odb = struct {
374		format: Oid.Format,
375		pack_file: *Io.File.Reader,
376		index_header: IndexHeader,
377		index_file: *Io.File.Reader,
378		cache: ObjectCache = .{},
379		allocator: Allocator,
380
381		/// Initializes the database from open pack and index files.
382		fn init(
383		odb: *Odb,
384		allocator: Allocator,
385		format: Oid.Format,
386		pack_file: *Io.File.Reader,
387		index_file: *Io.File.Reader,
388		) !void {
389		try pack_file.seekTo(0);
390		try index_file.seekTo(0);
391		odb.* = .{
392		.format = format,
393		.pack_file = pack_file,
394		.index_header = undefined,
395		.index_file = index_file,
396		.allocator = allocator,
397		};
398		try odb.index_header.read(&index_file.interface);
399		}
400
401		fn deinit(odb: *Odb) void {
402		odb.cache.deinit(odb.allocator);
403		odb.* = undefined;
404		}
405
406		/// Reads the object at the current position in the database.
407		fn readObject(odb: *Odb) !Object {
408		var base_offset = odb.pack_file.logicalPos();
409		var base_header: EntryHeader = undefined;
410		var delta_offsets: std.ArrayList(u64) = .empty;
411		defer delta_offsets.deinit(odb.allocator);
412		const base_object = while (true) {
413		if (odb.cache.get(base_offset)) \|base_object\| break base_object;
414
415		base_header = try EntryHeader.read(odb.format, &odb.pack_file.interface);
416		switch (base_header) {
417		.ofs_delta => \|ofs_delta\| {
418		try delta_offsets.append(odb.allocator, base_offset);
419		base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat;
420		try odb.pack_file.seekTo(base_offset);
421		},
422		.ref_delta => \|ref_delta\| {
423		try delta_offsets.append(odb.allocator, base_offset);
424		try odb.seekOid(ref_delta.base_object);
425		base_offset = odb.pack_file.logicalPos();
426		},
427		else => {
428		const base_data = try readObjectRaw(odb.allocator, &odb.pack_file.interface, base_header.uncompressedLength());
429		errdefer odb.allocator.free(base_data);
430		const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
431		try odb.cache.put(odb.allocator, base_offset, base_object);
432		break base_object;
433		},
434		}
435		};
436
437		const base_data = try resolveDeltaChain(
438		odb.allocator,
439		odb.format,
440		odb.pack_file,
441		base_object,
442		delta_offsets.items,
443		&odb.cache,
444		);
445
446		return .{ .type = base_object.type, .data = base_data };
447		}
448
449		/// Seeks to the beginning of the object with the given ID.
450		fn seekOid(odb: *Odb, oid: Oid) !void {
451		const oid_length = odb.format.byteLength();
452		const key = oid.slice()[0];
453		var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0;
454		var end_index = odb.index_header.fan_out_table[key];
455		const found_index = while (start_index < end_index) {
456		const mid_index = start_index + (end_index - start_index) / 2;
457		try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length);
458		const mid_oid = try Oid.readBytes(odb.format, &odb.index_file.interface);
459		switch (mem.order(u8, mid_oid.slice(), oid.slice())) {
460		.lt => start_index = mid_index + 1,
461		.gt => end_index = mid_index,
462		.eq => break mid_index,
463		}
464		} else return error.ObjectNotFound;
465
466		const n_objects = odb.index_header.fan_out_table[255];
467		const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4);
468		try odb.index_file.seekTo(offset_values_start + found_index * 4);
469		const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.interface.takeInt(u32, .big));
470		const pack_offset = pack_offset: {
471		if (l1_offset.big) {
472		const l2_offset_values_start = offset_values_start + n_objects * 4;
473		try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4);
474		break :pack_offset try odb.index_file.interface.takeInt(u64, .big);
475		} else {
476		break :pack_offset l1_offset.value;
477		}
478		};
479
480		try odb.pack_file.seekTo(pack_offset);
481		}
482		};
483
484		const Object = struct {
485		type: Type,
486		data: []const u8,
487
488		const Type = enum {
489		commit,
490		tree,
491		blob,
492		tag,
493		};
494		};
495
496		/// A cache for object data.
497		///
498		/// The purpose of this cache is to speed up resolution of deltas by caching the
499		/// results of resolving delta objects, while maintaining a maximum cache size
500		/// to avoid excessive memory usage. If the total size of the objects in the
501		/// cache exceeds the maximum, the cache will begin evicting the least recently
502		/// used objects: when resolving delta chains, the most recently used objects
503		/// will likely be more helpful as they will be further along in the chain
504		/// (skipping earlier reconstruction steps).
505		///
506		/// Object data stored in the cache is managed by the cache. It should not be
507		/// freed by the caller at any point after inserting it into the cache. Any
508		/// objects remaining in the cache will be freed when the cache itself is freed.
509		const ObjectCache = struct {
510		objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .empty,
511		lru_nodes: std.DoublyLinkedList = .{},
512		lru_nodes_len: usize = 0,
513		byte_size: usize = 0,
514
515		const max_byte_size = 128 * 1024 * 1024; // 128MiB
516		/// A list of offsets stored in the cache, with the most recently used
517		/// entries at the end.
518		const LruListNode = struct {
519		data: u64,
520		node: std.DoublyLinkedList.Node,
521		};
522		const CacheEntry = struct { object: Object, lru_node: *LruListNode };
523
524		fn deinit(cache: *ObjectCache, allocator: Allocator) void {
525		var object_iterator = cache.objects.iterator();
526		while (object_iterator.next()) \|object\| {
527		allocator.free(object.value_ptr.object.data);
528		allocator.destroy(object.value_ptr.lru_node);
529		}
530		cache.objects.deinit(allocator);
531		cache.* = undefined;
532		}
533
534		/// Gets an object from the cache, moving it to the most recently used
535		/// position if it is present.
536		fn get(cache: *ObjectCache, offset: u64) ?Object {
537		if (cache.objects.get(offset)) \|entry\| {
538		cache.lru_nodes.remove(&entry.lru_node.node);
539		cache.lru_nodes.append(&entry.lru_node.node);
540		return entry.object;
541		} else {
542		return null;
543		}
544		}
545
546		/// Puts an object in the cache, possibly evicting older entries if the
547		/// cache exceeds its maximum size. Note that, although old objects may
548		/// be evicted, the object just added to the cache with this function
549		/// will not be evicted before the next call to `put` or `deinit` even if
550		/// it exceeds the maximum cache size.
551		fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void {
552		const lru_node = try allocator.create(LruListNode);
553		errdefer allocator.destroy(lru_node);
554		lru_node.data = offset;
555
556		const gop = try cache.objects.getOrPut(allocator, offset);
557		if (gop.found_existing) {
558		cache.byte_size -= gop.value_ptr.object.data.len;
559		cache.lru_nodes.remove(&gop.value_ptr.lru_node.node);
560		cache.lru_nodes_len -= 1;
561		allocator.destroy(gop.value_ptr.lru_node);
562		allocator.free(gop.value_ptr.object.data);
563		}
564		gop.value_ptr.* = .{ .object = object, .lru_node = lru_node };
565		cache.byte_size += object.data.len;
566		cache.lru_nodes.append(&lru_node.node);
567		cache.lru_nodes_len += 1;
568
569		while (cache.byte_size > max_byte_size and cache.lru_nodes_len > 1) {
570		// The > 1 check is to make sure that we don't evict the most
571		// recently added node, even if it by itself happens to exceed the
572		// maximum size of the cache.
573		const evict_node: *LruListNode = @alignCast(@fieldParentPtr("node", cache.lru_nodes.popFirst().?));
574		cache.lru_nodes_len -= 1;
575		const evict_offset = evict_node.data;
576		allocator.destroy(evict_node);
577		const evict_object = cache.objects.get(evict_offset).?.object;
578		cache.byte_size -= evict_object.data.len;
579		allocator.free(evict_object.data);
580		_ = cache.objects.remove(evict_offset);
581		}
582		}
583		};
584
585		/// A single pkt-line in the Git protocol.
586		///
587		/// The format of a pkt-line is documented in
588		/// [protocol-common](https://git-scm.com/docs/protocol-common). The special
589		/// meanings of the delimiter and response-end packets are documented in
590		/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
591		pub const Packet = union(enum) {
592		flush,
593		delimiter,
594		response_end,
595		data: []const u8,
596
597		pub const max_data_length = 65516;
598
599		/// Reads a packet in pkt-line format.
600		fn read(reader: *Io.Reader) !Packet {
601		const packet: Packet = try .peek(reader);
602		switch (packet) {
603		.data => \|data\| reader.toss(data.len),
604		else => {},
605		}
606		return packet;
607		}
608
609		/// Consumes the header of a pkt-line packet and reads any associated data
610		/// into the reader's buffer, but does not consume the data.
611		fn peek(reader: *Io.Reader) !Packet {
612		const length = std.fmt.parseUnsigned(u16, try reader.take(4), 16) catch return error.InvalidPacket;
613		switch (length) {
614		0 => return .flush,
615		1 => return .delimiter,
616		2 => return .response_end,
617		3 => return error.InvalidPacket,
618		else => if (length - 4 > max_data_length) return error.InvalidPacket,
619		}
620		return .{ .data = try reader.peek(length - 4) };
621		}
622
623		/// Writes a packet in pkt-line format.
624		fn write(packet: Packet, writer: *Io.Writer) !void {
625		switch (packet) {
626		.flush => try writer.writeAll("0000"),
627		.delimiter => try writer.writeAll("0001"),
628		.response_end => try writer.writeAll("0002"),
629		.data => \|data\| {
630		assert(data.len <= max_data_length);
631		try writer.print("{x:0>4}", .{data.len + 4});
632		try writer.writeAll(data);
633		},
634		}
635		}
636
637		/// Returns the normalized form of textual packet data, stripping any
638		/// trailing '\n'.
639		///
640		/// As documented in
641		/// [protocol-common](https://git-scm.com/docs/protocol-common#_pkt_line_format),
642		/// non-binary (textual) pkt-line data should contain a trailing '\n', but
643		/// is not required to do so (implementations must support both forms).
644		fn normalizeText(data: []const u8) []const u8 {
645		return if (mem.endsWith(u8, data, "\n"))
646		data[0 .. data.len - 1]
647		else
648		data;
649		}
650		};
651
652		/// A client session for the Git protocol, currently limited to an HTTP(S)
653		/// transport. Only protocol version 2 is supported, as documented in
654		/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
655		pub const Session = struct {
656		transport: *std.http.Client,
657		location: Location,
658		supports_agent: bool,
659		supports_shallow: bool,
660		object_format: Oid.Format,
661		arena: Allocator,
662
663		const agent = "zig/" ++ @import("builtin").zig_version_string;
664		const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent});
665
666		/// Initializes a client session and discovers the capabilities of the
667		/// server for optimal transport.
668		pub fn init(
669		arena: Allocator,
670		transport: *std.http.Client,
671		uri: std.Uri,
672		/// Asserted to be at least `Packet.max_data_length`
673		response_buffer: []u8,
674		) !Session {
675		assert(response_buffer.len >= Packet.max_data_length);
676		var session: Session = .{
677		.transport = transport,
678		.location = try .init(arena, uri),
679		.supports_agent = false,
680		.supports_shallow = false,
681		.object_format = .sha1,
682		.arena = arena,
683		};
684		var capability_iterator: CapabilityIterator = undefined;
685		try session.getCapabilities(&capability_iterator, response_buffer);
686		defer capability_iterator.deinit();
687		while (try capability_iterator.next()) \|capability\| {
688		if (mem.eql(u8, capability.key, "agent")) {
689		session.supports_agent = true;
690		} else if (mem.eql(u8, capability.key, "fetch")) {
691		var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' ');
692		while (feature_iterator.next()) \|feature\| {
693		if (mem.eql(u8, feature, "shallow")) {
694		session.supports_shallow = true;
695		}
696		}
697		} else if (mem.eql(u8, capability.key, "object-format")) {
698		if (std.meta.stringToEnum(Oid.Format, capability.value orelse continue)) \|format\| {
699		session.object_format = format;
700		}
701		}
702		}
703		return session;
704		}
705
706		/// An owned `std.Uri` representing the location of the server (base URI).
707		const Location = struct {
708		uri: std.Uri,
709
710		fn init(arena: Allocator, uri: std.Uri) !Location {
711		const scheme = try arena.dupe(u8, uri.scheme);
712		const user = if (uri.user) \|user\| try std.fmt.allocPrint(arena, "{f}", .{
713		std.fmt.alt(user, .formatUser),
714		}) else null;
715		const password = if (uri.password) \|password\| try std.fmt.allocPrint(arena, "{f}", .{
716		std.fmt.alt(password, .formatPassword),
717		}) else null;
718		const host = if (uri.host) \|host\| try std.fmt.allocPrint(arena, "{f}", .{
719		std.fmt.alt(host, .formatHost),
720		}) else null;
721		const path = try std.fmt.allocPrint(arena, "{f}", .{
722		std.fmt.alt(uri.path, .formatPath),
723		});
724		// The query and fragment are not used as part of the base server URI.
725		return .{
726		.uri = .{
727		.scheme = scheme,
728		.user = if (user) \|s\| .{ .percent_encoded = s } else null,
729		.password = if (password) \|s\| .{ .percent_encoded = s } else null,
730		.host = if (host) \|s\| .{ .percent_encoded = s } else null,
731		.port = uri.port,
732		.path = .{ .percent_encoded = path },
733		},
734		};
735		}
736		};
737
738		/// Returns an iterator over capabilities supported by the server.
739		///
740		/// The `session.location` is updated if the server returns a redirect, so
741		/// that subsequent session functions do not need to handle redirects.
742		fn getCapabilities(session: Session, it: CapabilityIterator, response_buffer: []u8) !void {
743		const arena = session.arena;
744		assert(response_buffer.len >= Packet.max_data_length);
745		var info_refs_uri = session.location.uri;
746		{
747		const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
748		std.fmt.alt(session.location.uri.path, .formatPath),
749		});
750		info_refs_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{
751		"/", session_uri_path, "info/refs",
752		}) };
753		}
754		info_refs_uri.query = .{ .percent_encoded = "service=git-upload-pack" };
755		info_refs_uri.fragment = null;
756
757		const max_redirects = 3;
758		it.* = .{
759		.request = try session.transport.request(.GET, info_refs_uri, .{
760		.redirect_behavior = .init(max_redirects),
761		.extra_headers = &.{
762		.{ .name = "Git-Protocol", .value = "version=2" },
763		},
764		}),
765		.reader = undefined,
766		.decompress = undefined,
767		};
768		errdefer it.deinit();
769		const request = &it.request;
770		try request.sendBodiless();
771
772		var redirect_buffer: [1024]u8 = undefined;
773		var response = try request.receiveHead(&redirect_buffer);
774		if (response.head.status != .ok) return error.ProtocolError;
775		const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects;
776		if (any_redirects_occurred) {
777		const request_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
778		std.fmt.alt(request.uri.path, .formatPath),
779		});
780		if (!mem.endsWith(u8, request_uri_path, "/info/refs")) return error.UnparseableRedirect;
781		var new_uri = request.uri;
782		new_uri.path = .{ .percent_encoded = request_uri_path[0 .. request_uri_path.len - "/info/refs".len] };
783		session.location = try .init(arena, new_uri);
784		}
785
786		const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
787		it.reader = response.readerDecompressing(response_buffer, &it.decompress, decompress_buffer);
788		var state: enum { response_start, response_content } = .response_start;
789		while (true) {
790		// Some Git servers (at least GitHub) include an additional
791		// '# service=git-upload-pack' informative response before sending
792		// the expected 'version 2' packet and capability information.
793		// This is not universal: SourceHut, for example, does not do this.
794		// Thus, we need to skip any such useless additional responses
795		// before we get the one we're actually looking for. The responses
796		// will be delimited by flush packets.
797		const packet = Packet.read(it.reader) catch \|err\| switch (err) {
798		error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found
799		else => \|e\| return e,
800		};
801		switch (packet) {
802		.flush => state = .response_start,
803		.data => \|data\| switch (state) {
804		.response_start => if (mem.eql(u8, Packet.normalizeText(data), "version 2")) {
805		return;
806		} else {
807		state = .response_content;
808		},
809		else => {},
810		},
811		else => return error.UnexpectedPacket,
812		}
813		}
814		}
815
816		const CapabilityIterator = struct {
817		request: std.http.Client.Request,
818		reader: *Io.Reader,
819		decompress: std.http.Decompress,
820
821		const Capability = struct {
822		key: []const u8,
823		value: ?[]const u8 = null,
824
825		fn parse(data: []const u8) Capability {
826		return if (mem.indexOfScalar(u8, data, '=')) \|separator_pos\|
827		.{ .key = data[0..separator_pos], .value = data[separator_pos + 1 ..] }
828		else
829		.{ .key = data };
830		}
831		};
832
833		fn deinit(it: *CapabilityIterator) void {
834		it.request.deinit();
835		it.* = undefined;
836		}
837
838		fn next(it: *CapabilityIterator) !?Capability {
839		switch (try Packet.read(it.reader)) {
840		.flush => return null,
841		.data => \|data\| return Capability.parse(Packet.normalizeText(data)),
842		else => return error.UnexpectedPacket,
843		}
844		}
845		};
846
847		const ListRefsOptions = struct {
848		/// The ref prefixes (if any) to use to filter the refs available on the
849		/// server. Note that the client must still check the returned refs
850		/// against its desired filters itself: the server is not required to
851		/// respect these prefix filters and may return other refs as well.
852		ref_prefixes: []const []const u8 = &.{},
853		/// Whether to include symref targets for returned symbolic refs.
854		include_symrefs: bool = false,
855		/// Whether to include the peeled object ID for returned tag refs.
856		include_peeled: bool = false,
857		/// Asserted to be at least `Packet.max_data_length`.
858		buffer: []u8,
859		};
860
861		/// Returns an iterator over refs known to the server.
862		pub fn listRefs(session: Session, it: *RefIterator, options: ListRefsOptions) !void {
863		const arena = session.arena;
864		assert(options.buffer.len >= Packet.max_data_length);
865		var upload_pack_uri = session.location.uri;
866		{
867		const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
868		std.fmt.alt(session.location.uri.path, .formatPath),
869		});
870		upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) };
871		}
872		upload_pack_uri.query = null;
873		upload_pack_uri.fragment = null;
874
875		var body: Io.Writer = .fixed(options.buffer);
876		try Packet.write(.{ .data = "command=ls-refs\n" }, &body);
877		if (session.supports_agent) {
878		try Packet.write(.{ .data = agent_capability }, &body);
879		}
880		{
881		const object_format_packet = try std.fmt.allocPrint(arena, "object-format={t}\n", .{
882		session.object_format,
883		});
884		try Packet.write(.{ .data = object_format_packet }, &body);
885		}
886		try Packet.write(.delimiter, &body);
887		for (options.ref_prefixes) \|ref_prefix\| {
888		const ref_prefix_packet = try std.fmt.allocPrint(arena, "ref-prefix {s}\n", .{ref_prefix});
889		try Packet.write(.{ .data = ref_prefix_packet }, &body);
890		}
891		if (options.include_symrefs) {
892		try Packet.write(.{ .data = "symrefs\n" }, &body);
893		}
894		if (options.include_peeled) {
895		try Packet.write(.{ .data = "peel\n" }, &body);
896		}
897		try Packet.write(.flush, &body);
898
899		it.* = .{
900		.request = try session.transport.request(.POST, upload_pack_uri, .{
901		.redirect_behavior = .unhandled,
902		.extra_headers = &.{
903		.{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" },
904		.{ .name = "Git-Protocol", .value = "version=2" },
905		},
906		}),
907		.reader = undefined,
908		.format = session.object_format,
909		.decompress = undefined,
910		};
911		const request = &it.request;
912		errdefer request.deinit();
913		try request.sendBodyComplete(body.buffered());
914
915		var response = try request.receiveHead(options.buffer);
916		if (response.head.status != .ok) return error.ProtocolError;
917		const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
918		it.reader = response.readerDecompressing(options.buffer, &it.decompress, decompress_buffer);
919		}
920
921		pub const RefIterator = struct {
922		format: Oid.Format,
923		request: std.http.Client.Request,
924		reader: *Io.Reader,
925		decompress: std.http.Decompress,
926
927		pub const Ref = struct {
928		oid: Oid,
929		name: []const u8,
930		symref_target: ?[]const u8,
931		peeled: ?Oid,
932		};
933
934		pub fn deinit(iterator: *RefIterator) void {
935		iterator.request.deinit();
936		iterator.* = undefined;
937		}
938
939		pub fn next(it: *RefIterator) !?Ref {
940		switch (try Packet.read(it.reader)) {
941		.flush => return null,
942		.data => \|data\| {
943		const ref_data = Packet.normalizeText(data);
944		const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket;
945		const oid = Oid.parse(it.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket;
946
947		const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len;
948		const name = ref_data[oid_sep_pos + 1 .. name_sep_pos];
949
950		var symref_target: ?[]const u8 = null;
951		var peeled: ?Oid = null;
952		var last_sep_pos = name_sep_pos;
953		while (last_sep_pos < ref_data.len) {
954		const next_sep_pos = mem.indexOfScalarPos(u8, ref_data, last_sep_pos + 1, ' ') orelse ref_data.len;
955		const attribute = ref_data[last_sep_pos + 1 .. next_sep_pos];
956		if (mem.startsWith(u8, attribute, "symref-target:")) {
957		symref_target = attribute["symref-target:".len..];
958		} else if (mem.startsWith(u8, attribute, "peeled:")) {
959		peeled = Oid.parse(it.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket;
960		}
961		last_sep_pos = next_sep_pos;
962		}
963
964		return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled };
965		},
966		else => return error.UnexpectedPacket,
967		}
968		}
969		};
970
971		/// Fetches the given refs from the server. A shallow fetch (depth 1) is
972		/// performed if the server supports it.
973		pub fn fetch(
974		session: Session,
975		fs: *FetchStream,
976		wants: []const []const u8,
977		/// Asserted to be at least `Packet.max_data_length`.
978		response_buffer: []u8,
979		) !void {
980		const arena = session.arena;
981		assert(response_buffer.len >= Packet.max_data_length);
982		var upload_pack_uri = session.location.uri;
983		{
984		const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
985		std.fmt.alt(session.location.uri.path, .formatPath),
986		});
987		upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) };
988		}
989		upload_pack_uri.query = null;
990		upload_pack_uri.fragment = null;
991
992		var body: Io.Writer = .fixed(response_buffer);
993		try Packet.write(.{ .data = "command=fetch\n" }, &body);
994		if (session.supports_agent) {
995		try Packet.write(.{ .data = agent_capability }, &body);
996		}
997		{
998		const object_format_packet = try std.fmt.allocPrint(arena, "object-format={s}\n", .{@tagName(session.object_format)});
999		try Packet.write(.{ .data = object_format_packet }, &body);
1000		}
1001		try Packet.write(.delimiter, &body);
1002		// Our packfile parser supports the OFS_DELTA object type
1003		try Packet.write(.{ .data = "ofs-delta\n" }, &body);
1004		// We do not currently convey server progress information to the user
1005		try Packet.write(.{ .data = "no-progress\n" }, &body);
1006		if (session.supports_shallow) {
1007		try Packet.write(.{ .data = "deepen 1\n" }, &body);
1008		}
1009		for (wants) \|want\| {
1010		var buf: [Packet.max_data_length]u8 = undefined;
1011		const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable;
1012		try Packet.write(.{ .data = arg }, &body);
1013		}
1014		try Packet.write(.{ .data = "done\n" }, &body);
1015		try Packet.write(.flush, &body);
1016
1017		fs.* = .{
1018		.request = try session.transport.request(.POST, upload_pack_uri, .{
1019		.redirect_behavior = .not_allowed,
1020		.extra_headers = &.{
1021		.{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" },
1022		.{ .name = "Git-Protocol", .value = "version=2" },
1023		},
1024		}),
1025		.input = undefined,
1026		.reader = undefined,
1027		.remaining_len = undefined,
1028		.decompress = undefined,
1029		};
1030		const request = &fs.request;
1031		errdefer request.deinit();
1032
1033		try request.sendBodyComplete(body.buffered());
1034
1035		var response = try request.receiveHead(&.{});
1036		if (response.head.status != .ok) return error.ProtocolError;
1037
1038		const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
1039		const reader = response.readerDecompressing(response_buffer, &fs.decompress, decompress_buffer);
1040		// We are not interested in any of the sections of the returned fetch
1041		// data other than the packfile section, since we aren't doing anything
1042		// complex like ref negotiation (this is a fresh clone).
1043		var state: enum { section_start, section_content } = .section_start;
1044		while (true) {
1045		const packet = try Packet.read(reader);
1046		switch (state) {
1047		.section_start => switch (packet) {
1048		.data => \|data\| if (mem.eql(u8, Packet.normalizeText(data), "packfile")) {
1049		fs.input = reader;
1050		fs.reader = .{
1051		.buffer = &.{},
1052		.vtable = &.{ .stream = FetchStream.stream },
1053		.seek = 0,
1054		.end = 0,
1055		};
1056		fs.remaining_len = 0;
1057		return;
1058		} else {
1059		state = .section_content;
1060		},
1061		else => return error.UnexpectedPacket,
1062		},
1063		.section_content => switch (packet) {
1064		.delimiter => state = .section_start,
1065		.data => {},
1066		else => return error.UnexpectedPacket,
1067		},
1068		}
1069		}
1070		}
1071
1072		pub const FetchStream = struct {
1073		request: std.http.Client.Request,
1074		input: *Io.Reader,
1075		reader: Io.Reader,
1076		err: ?Error = null,
1077		remaining_len: usize,
1078		decompress: std.http.Decompress,
1079
1080		pub fn deinit(fs: *FetchStream) void {
1081		fs.request.deinit();
1082		}
1083
1084		pub const Error = error{
1085		InvalidPacket,
1086		ProtocolError,
1087		UnexpectedPacket,
1088		WriteFailed,
1089		ReadFailed,
1090		EndOfStream,
1091		};
1092
1093		const StreamCode = enum(u8) {
1094		pack_data = 1,
1095		progress = 2,
1096		fatal_error = 3,
1097		_,
1098		};
1099
1100		pub fn stream(r: Io.Reader, w: Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize {
1101		const fs: *FetchStream = @alignCast(@fieldParentPtr("reader", r));
1102		const input = fs.input;
1103		if (fs.remaining_len == 0) {
1104		while (true) {
1105		switch (Packet.peek(input) catch \|err\| {
1106		fs.err = err;
1107		return error.ReadFailed;
1108		}) {
1109		.flush => return error.EndOfStream,
1110		.data => \|data\| switch (@as(StreamCode, @enumFromInt(data[0]))) {
1111		.pack_data => {
1112		input.toss(1);
1113		fs.remaining_len = data.len - 1;
1114		break;
1115		},
1116		.fatal_error => {
1117		fs.err = error.ProtocolError;
1118		return error.ReadFailed;
1119		},
1120		else => {
1121		input.toss(data.len);
1122		},
1123		},
1124		else => {
1125		fs.err = error.UnexpectedPacket;
1126		return error.ReadFailed;
1127		},
1128		}
1129		}
1130		}
1131		const buf = limit.slice(try w.writableSliceGreedy(1));
1132		const n = @min(buf.len, fs.remaining_len);
1133		try input.readSliceAll(buf[0..n]);
1134		w.advance(n);
1135		fs.remaining_len -= n;
1136		return n;
1137		}
1138		};
1139		};
1140
1141		const PackHeader = struct {
1142		total_objects: u32,
1143
1144		const signature = "PACK";
1145		const supported_version = 2;
1146
1147		fn read(reader: *Io.Reader) !PackHeader {
1148		const actual_signature = reader.take(4) catch \|e\| switch (e) {
1149		error.EndOfStream => return error.InvalidHeader,
1150		else => \|other\| return other,
1151		};
1152		if (!mem.eql(u8, actual_signature, signature)) return error.InvalidHeader;
1153		const version = reader.takeInt(u32, .big) catch \|e\| switch (e) {
1154		error.EndOfStream => return error.InvalidHeader,
1155		else => \|other\| return other,
1156		};
1157		if (version != supported_version) return error.UnsupportedVersion;
1158		const total_objects = reader.takeInt(u32, .big) catch \|e\| switch (e) {
1159		error.EndOfStream => return error.InvalidHeader,
1160		else => \|other\| return other,
1161		};
1162		return .{ .total_objects = total_objects };
1163		}
1164		};
1165
1166		const EntryHeader = union(Type) {
1167		commit: Undeltified,
1168		tree: Undeltified,
1169		blob: Undeltified,
1170		tag: Undeltified,
1171		ofs_delta: OfsDelta,
1172		ref_delta: RefDelta,
1173
1174		const Type = enum(u3) {
1175		commit = 1,
1176		tree = 2,
1177		blob = 3,
1178		tag = 4,
1179		ofs_delta = 6,
1180		ref_delta = 7,
1181		};
1182
1183		const Undeltified = struct {
1184		uncompressed_length: u64,
1185		};
1186
1187		const OfsDelta = struct {
1188		offset: u64,
1189		uncompressed_length: u64,
1190		};
1191
1192		const RefDelta = struct {
1193		base_object: Oid,
1194		uncompressed_length: u64,
1195		};
1196
1197		fn objectType(header: EntryHeader) Object.Type {
1198		return switch (header) {
1199		inline .commit, .tree, .blob, .tag => \|_, tag\| @field(Object.Type, @tagName(tag)),
1200		else => unreachable,
1201		};
1202		}
1203
1204		fn uncompressedLength(header: EntryHeader) u64 {
1205		return switch (header) {
1206		inline else => \|entry\| entry.uncompressed_length,
1207		};
1208		}
1209
1210		fn read(format: Oid.Format, reader: *Io.Reader) !EntryHeader {
1211		const InitialByte = packed struct { len: u4, type: u3, has_next: bool };
1212		const initial: InitialByte = @bitCast(reader.takeByte() catch \|e\| switch (e) {
1213		error.EndOfStream => return error.InvalidFormat,
1214		else => \|other\| return other,
1215		});
1216		const rest_len = if (initial.has_next) try reader.takeLeb128(u64) else 0;
1217		var uncompressed_length: u64 = initial.len;
1218		uncompressed_length \|= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat;
1219		const @"type" = std.enums.fromInt(EntryHeader.Type, initial.type) orelse return error.InvalidFormat;
1220		return switch (@"type") {
1221		inline .commit, .tree, .blob, .tag => \|tag\| @unionInit(EntryHeader, @tagName(tag), .{
1222		.uncompressed_length = uncompressed_length,
1223		}),
1224		.ofs_delta => .{ .ofs_delta = .{
1225		.offset = try readOffsetVarInt(reader),
1226		.uncompressed_length = uncompressed_length,
1227		} },
1228		.ref_delta => .{ .ref_delta = .{
1229		.base_object = Oid.readBytes(format, reader) catch \|e\| switch (e) {
1230		error.EndOfStream => return error.InvalidFormat,
1231		else => \|other\| return other,
1232		},
1233		.uncompressed_length = uncompressed_length,
1234		} },
1235		};
1236		}
1237		};
1238
1239		fn readOffsetVarInt(r: *Io.Reader) !u64 {
1240		const Byte = packed struct { value: u7, has_next: bool };
1241		var b: Byte = @bitCast(try r.takeByte());
1242		var value: u64 = b.value;
1243		while (b.has_next) {
1244		b = @bitCast(try r.takeByte());
1245		value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat;
1246		value \|= b.value;
1247		}
1248		return value;
1249		}
1250
1251		const IndexHeader = struct {
1252		fan_out_table: [256]u32,
1253
1254		const signature = "\xFFtOc";
1255		const supported_version = 2;
1256		const size = 4 + 4 + @sizeOf([256]u32);
1257
1258		fn read(index_header: IndexHeader, reader: Io.Reader) !void {
1259		const sig = try reader.take(4);
1260		if (!mem.eql(u8, sig, signature)) return error.InvalidHeader;
1261		const version = try reader.takeInt(u32, .big);
1262		if (version != supported_version) return error.UnsupportedVersion;
1263		try reader.readSliceEndian(u32, &index_header.fan_out_table, .big);
1264		}
1265		};
1266
1267		const IndexEntry = struct {
1268		offset: u64,
1269		crc32: u32,
1270		};
1271
1272		/// Writes out a version 2 index for the given packfile, as documented in
1273		/// [pack-format](https://git-scm.com/docs/pack-format).
1274		pub fn indexPack(
1275		allocator: Allocator,
1276		format: Oid.Format,
1277		pack: *Io.File.Reader,
1278		index_writer: *Io.File.Writer,
1279		) !void {
1280		try pack.seekTo(0);
1281
1282		var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty;
1283		defer index_entries.deinit(allocator);
1284		var pending_deltas: std.ArrayList(IndexEntry) = .empty;
1285		defer pending_deltas.deinit(allocator);
1286
1287		const pack_checksum = try indexPackFirstPass(allocator, format, pack, &index_entries, &pending_deltas);
1288
1289		var cache: ObjectCache = .{};
1290		defer cache.deinit(allocator);
1291		var remaining_deltas = pending_deltas.items.len;
1292		while (remaining_deltas > 0) {
1293		var i: usize = remaining_deltas;
1294		while (i > 0) {
1295		i -= 1;
1296		const delta = pending_deltas.items[i];
1297		if (try indexPackHashDelta(allocator, format, pack, delta, index_entries, &cache)) \|oid\| {
1298		try index_entries.put(allocator, oid, delta);
1299		_ = pending_deltas.swapRemove(i);
1300		}
1301		}
1302		if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack;
1303		remaining_deltas = pending_deltas.items.len;
1304		}
1305
1306		var oids: std.ArrayList(Oid) = .empty;
1307		defer oids.deinit(allocator);
1308		try oids.ensureTotalCapacityPrecise(allocator, index_entries.count());
1309		var index_entries_iter = index_entries.iterator();
1310		while (index_entries_iter.next()) \|entry\| {
1311		oids.appendAssumeCapacity(entry.key_ptr.*);
1312		}
1313		mem.sortUnstable(Oid, oids.items, {}, struct {
1314		fn lessThan(_: void, o1: Oid, o2: Oid) bool {
1315		return mem.lessThan(u8, o1.slice(), o2.slice());
1316		}
1317		}.lessThan);
1318
1319		var fan_out_table: [256]u32 = undefined;
1320		var count: u32 = 0;
1321		var fan_out_index: u8 = 0;
1322		for (oids.items) \|oid\| {
1323		const key = oid.slice()[0];
1324		if (key > fan_out_index) {
1325		@memset(fan_out_table[fan_out_index..key], count);
1326		fan_out_index = key;
1327		}
1328		count += 1;
1329		}
1330		@memset(fan_out_table[fan_out_index..], count);
1331
1332		var index_hashed_writer = Io.Writer.hashed(&index_writer.interface, Oid.Hasher.init(format), &.{});
1333		const writer = &index_hashed_writer.writer;
1334		try writer.writeAll(IndexHeader.signature);
1335		try writer.writeInt(u32, IndexHeader.supported_version, .big);
1336		for (fan_out_table) \|fan_out_entry\| {
1337		try writer.writeInt(u32, fan_out_entry, .big);
1338		}
1339
1340		for (oids.items) \|oid\| {
1341		try writer.writeAll(oid.slice());
1342		}
1343
1344		for (oids.items) \|oid\| {
1345		try writer.writeInt(u32, index_entries.get(oid).?.crc32, .big);
1346		}
1347
1348		var big_offsets: std.ArrayList(u64) = .empty;
1349		defer big_offsets.deinit(allocator);
1350		for (oids.items) \|oid\| {
1351		const offset = index_entries.get(oid).?.offset;
1352		if (offset <= std.math.maxInt(u31)) {
1353		try writer.writeInt(u32, @intCast(offset), .big);
1354		} else {
1355		const index = big_offsets.items.len;
1356		try big_offsets.append(allocator, offset);
1357		try writer.writeInt(u32, @as(u32, @intCast(index)) \| (1 << 31), .big);
1358		}
1359		}
1360		for (big_offsets.items) \|offset\| {
1361		try writer.writeInt(u64, offset, .big);
1362		}
1363
1364		try writer.writeAll(pack_checksum.slice());
1365		const index_checksum = index_hashed_writer.hasher.finalResult();
1366		try index_writer.interface.writeAll(index_checksum.slice());
1367		try index_writer.end();
1368		}
1369
1370		/// Performs the first pass over the packfile data for index construction.
1371		/// This will index all non-delta objects, queue delta objects for further
1372		/// processing, and return the pack checksum (which is part of the index
1373		/// format).
1374		fn indexPackFirstPass(
1375		allocator: Allocator,
1376		format: Oid.Format,
1377		pack: *Io.File.Reader,
1378		index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),
1379		pending_deltas: *std.ArrayList(IndexEntry),
1380		) !Oid {
1381		var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
1382		var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system.
1383		var pack_hashed = pack.interface.hashed(Oid.Hasher.init(format), &pack_buffer);
1384
1385		const pack_header = try PackHeader.read(&pack_hashed.reader);
1386
1387		for (0..pack_header.total_objects) \|_\| {
1388		const entry_offset = pack.logicalPos() - pack_hashed.reader.bufferedLen();
1389		const entry_header = try EntryHeader.read(format, &pack_hashed.reader);
1390		switch (entry_header) {
1391		.commit, .tree, .blob, .tag => \|object\| {
1392		var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &.{});
1393		var oid_hasher: Oid.Hashing = .init(format, &flate_buffer);
1394		const oid_hasher_w = oid_hasher.writer();
1395		// The object header is not included in the pack data but is
1396		// part of the object's ID
1397		try oid_hasher_w.print("{t} {d}\x00", .{ entry_header, object.uncompressed_length });
1398		const n = try entry_decompress.reader.streamRemaining(oid_hasher_w);
1399		if (n != object.uncompressed_length) return error.InvalidObject;
1400		const oid = oid_hasher.final();
1401		if (!skip_checksums) @compileError("TODO");
1402		try index_entries.put(allocator, oid, .{
1403		.offset = entry_offset,
1404		.crc32 = 0,
1405		});
1406		},
1407		inline .ofs_delta, .ref_delta => \|delta\| {
1408		var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &flate_buffer);
1409		const n = try entry_decompress.reader.discardRemaining();
1410		if (n != delta.uncompressed_length) return error.InvalidObject;
1411		if (!skip_checksums) @compileError("TODO");
1412		try pending_deltas.append(allocator, .{
1413		.offset = entry_offset,
1414		.crc32 = 0,
1415		});
1416		},
1417		}
1418		}
1419
1420		if (!skip_checksums) @compileError("TODO");
1421		return pack_hashed.hasher.finalResult();
1422		}
1423
1424		/// Attempts to determine the final object ID of the given deltified object.
1425		/// May return null if this is not yet possible (if the delta is a ref-based
1426		/// delta and we do not yet know the offset of the base object).
1427		fn indexPackHashDelta(
1428		allocator: Allocator,
1429		format: Oid.Format,
1430		pack: *Io.File.Reader,
1431		delta: IndexEntry,
1432		index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),
1433		cache: *ObjectCache,
1434		) !?Oid {
1435		// Figure out the chain of deltas to resolve
1436		var base_offset = delta.offset;
1437		var base_header: EntryHeader = undefined;
1438		var delta_offsets: std.ArrayList(u64) = .empty;
1439		defer delta_offsets.deinit(allocator);
1440		const base_object = while (true) {
1441		if (cache.get(base_offset)) \|base_object\| break base_object;
1442
1443		try pack.seekTo(base_offset);
1444		base_header = try EntryHeader.read(format, &pack.interface);
1445		switch (base_header) {
1446		.ofs_delta => \|ofs_delta\| {
1447		try delta_offsets.append(allocator, base_offset);
1448		base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject;
1449		},
1450		.ref_delta => \|ref_delta\| {
1451		try delta_offsets.append(allocator, base_offset);
1452		base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset;
1453		},
1454		else => {
1455		const base_data = try readObjectRaw(allocator, &pack.interface, base_header.uncompressedLength());
1456		errdefer allocator.free(base_data);
1457		const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
1458		try cache.put(allocator, base_offset, base_object);
1459		break base_object;
1460		},
1461		}
1462		};
1463
1464		const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache);
1465
1466		var entry_hasher_buffer: [64]u8 = undefined;
1467		var entry_hasher: Oid.Hashing = .init(format, &entry_hasher_buffer);
1468		const entry_hasher_w = entry_hasher.writer();
1469		// Writes to hashers cannot fail.
1470		entry_hasher_w.print("{t} {d}\x00", .{ base_object.type, base_data.len }) catch unreachable;
1471		entry_hasher_w.writeAll(base_data) catch unreachable;
1472		return entry_hasher.final();
1473		}
1474
1475		/// Resolves a chain of deltas, returning the final base object data. `pack` is
1476		/// assumed to be looking at the start of the object data for the base object of
1477		/// the chain, and will then apply the deltas in `delta_offsets` in reverse order
1478		/// to obtain the final object.
1479		fn resolveDeltaChain(
1480		allocator: Allocator,
1481		format: Oid.Format,
1482		pack: *Io.File.Reader,
1483		base_object: Object,
1484		delta_offsets: []const u64,
1485		cache: *ObjectCache,
1486		) ![]const u8 {
1487		var base_data = base_object.data;
1488		var i: usize = delta_offsets.len;
1489		while (i > 0) {
1490		i -= 1;
1491
1492		const delta_offset = delta_offsets[i];
1493		try pack.seekTo(delta_offset);
1494		const delta_header = try EntryHeader.read(format, &pack.interface);
1495		const delta_data = try readObjectRaw(allocator, &pack.interface, delta_header.uncompressedLength());
1496		defer allocator.free(delta_data);
1497		var delta_reader: Io.Reader = .fixed(delta_data);
1498		_ = try delta_reader.takeLeb128(u64); // base object size
1499		const expanded_size = try delta_reader.takeLeb128(u64);
1500
1501		const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge;
1502		const expanded_data = try allocator.alloc(u8, expanded_alloc_size);
1503		errdefer allocator.free(expanded_data);
1504		var expanded_delta_stream: Io.Writer = .fixed(expanded_data);
1505		try expandDelta(base_data, &delta_reader, &expanded_delta_stream);
1506		if (expanded_delta_stream.end != expanded_size) return error.InvalidObject;
1507
1508		try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data });
1509		base_data = expanded_data;
1510		}
1511		return base_data;
1512		}
1513
1514		/// Reads the complete contents of an object from `reader`. This function may
1515		/// read more bytes than required from `reader`, so the reader position after
1516		/// returning is not reliable.
1517		fn readObjectRaw(allocator: Allocator, reader: *Io.Reader, size: u64) ![]u8 {
1518		const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;
1519		var aw: Io.Writer.Allocating = .init(allocator);
1520		try aw.ensureTotalCapacity(alloc_size + std.compress.flate.max_window_len);
1521		defer aw.deinit();
1522		var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &.{});
1523		try decompress.reader.streamExact(&aw.writer, alloc_size);
1524		return aw.toOwnedSlice();
1525		}
1526
1527		/// Expands delta data from `delta_reader` to `writer`.
1528		///
1529		/// The format of the delta data is documented in
1530		/// [pack-format](https://git-scm.com/docs/pack-format).
1531		fn expandDelta(base_object: []const u8, delta_reader: Io.Reader, writer: Io.Writer) !void {
1532		while (true) {
1533		const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch \|e\| switch (e) {
1534		error.EndOfStream => return,
1535		else => \|other\| return other,
1536		});
1537		if (inst.copy) {
1538		const available: packed struct {
1539		offset1: bool,
1540		offset2: bool,
1541		offset3: bool,
1542		offset4: bool,
1543		size1: bool,
1544		size2: bool,
1545		size3: bool,
1546		} = @bitCast(inst.value);
1547		const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{
1548		.offset1 = if (available.offset1) try delta_reader.takeByte() else 0,
1549		.offset2 = if (available.offset2) try delta_reader.takeByte() else 0,
1550		.offset3 = if (available.offset3) try delta_reader.takeByte() else 0,
1551		.offset4 = if (available.offset4) try delta_reader.takeByte() else 0,
1552		};
1553		const base_offset: u32 = @bitCast(offset_parts);
1554		const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{
1555		.size1 = if (available.size1) try delta_reader.takeByte() else 0,
1556		.size2 = if (available.size2) try delta_reader.takeByte() else 0,
1557		.size3 = if (available.size3) try delta_reader.takeByte() else 0,
1558		};
1559		var size: u24 = @bitCast(size_parts);
1560		if (size == 0) size = 0x10000;
1561		try writer.writeAll(base_object[base_offset..][0..size]);
1562		} else if (inst.value != 0) {
1563		try delta_reader.streamExact(writer, inst.value);
1564		} else {
1565		return error.InvalidDeltaInstruction;
1566		}
1567		}
1568		}
1569
1570		/// Runs the packfile indexing and checkout test.
1571		///
1572		/// The two testrepo repositories under testdata contain identical commit
1573		/// histories and contents.
1574		///
1575		/// To verify the contents of the packfiles using Git alone, run the
1576		/// following commands in an empty directory:
1577		///
1578		/// 1. `git init --object-format=(sha1\|sha256)`
1579		/// 2. `git unpack-objects <path/to/testrepo.pack`
1580		/// 3. `git fsck` - will print one "dangling commit":
1581		/// - SHA-1: `dd582c0720819ab7130b103635bd7271b9fd4feb`
1582		/// - SHA-256: `7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a`
1583		/// 4. `git checkout $commit`
1584		fn runRepositoryTest(io: Io, comptime format: Oid.Format, head_commit: []const u8) !void {
1585		const testrepo_pack = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".pack");
1586
1587		var git_dir = testing.tmpDir(.{});
1588		defer git_dir.cleanup();
1589		var pack_file = try git_dir.dir.createFile(io, "testrepo.pack", .{ .read = true });
1590		defer pack_file.close(io);
1591		try pack_file.writeStreamingAll(io, testrepo_pack);
1592
1593		var pack_file_buffer: [2000]u8 = undefined;
1594		var pack_file_reader = pack_file.reader(io, &pack_file_buffer);
1595
1596		var index_file = try git_dir.dir.createFile(io, "testrepo.idx", .{ .read = true });
1597		defer index_file.close(io);
1598		var index_file_buffer: [2000]u8 = undefined;
1599		var index_file_writer = index_file.writer(io, &index_file_buffer);
1600		try indexPack(testing.allocator, format, &pack_file_reader, &index_file_writer);
1601
1602		// Arbitrary size limit on files read while checking the repository contents
1603		// (all files in the test repo are known to be smaller than this)
1604		const max_file_size = 8192;
1605
1606		if (!skip_checksums) {
1607		const index_file_data = try git_dir.dir.readFileAlloc(io, "testrepo.idx", testing.allocator, .limited(max_file_size));
1608		defer testing.allocator.free(index_file_data);
1609		// testrepo.idx is generated by Git. The index created by this file should
1610		// match it exactly. Running `git verify-pack -v testrepo.pack` can verify
1611		// this.
1612		const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx");
1613		try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);
1614		}
1615
1616		var index_file_reader = index_file.reader(io, &index_file_buffer);
1617		var repository: Repository = undefined;
1618		try repository.init(testing.allocator, format, &pack_file_reader, &index_file_reader);
1619		defer repository.deinit();
1620
1621		var worktree = testing.tmpDir(.{ .iterate = true });
1622		defer worktree.cleanup();
1623
1624		const commit_id = try Oid.parse(format, head_commit);
1625
1626		var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1627		defer diagnostics.deinit();
1628		try repository.checkout(io, worktree.dir, commit_id, &diagnostics);
1629		try testing.expect(diagnostics.errors.items.len == 0);
1630
1631		const expected_files: []const []const u8 = &.{
1632		"dir/file",
1633		"dir/subdir/file",
1634		"dir/subdir/file2",
1635		"dir2/file",
1636		"dir3/file",
1637		"dir3/file2",
1638		"file",
1639		"file2",
1640		"file3",
1641		"file4",
1642		"file5",
1643		"file6",
1644		"file7",
1645		"file8",
1646		"file9",
1647		};
1648		var actual_files: std.ArrayList([]u8) = .empty;
1649		defer actual_files.deinit(testing.allocator);
1650		defer for (actual_files.items) \|file\| testing.allocator.free(file);
1651		var walker = try worktree.dir.walk(testing.allocator);
1652		defer walker.deinit();
1653		while (try walker.next(io)) \|entry\| {
1654		if (entry.kind != .file) continue;
1655		const path = try testing.allocator.dupe(u8, entry.path);
1656		errdefer testing.allocator.free(path);
1657		mem.replaceScalar(u8, path, std.fs.path.sep, '/');
1658		try actual_files.append(testing.allocator, path);
1659		}
1660		mem.sortUnstable([]u8, actual_files.items, {}, struct {
1661		fn lessThan(_: void, a: []u8, b: []u8) bool {
1662		return mem.lessThan(u8, a, b);
1663		}
1664		}.lessThan);
1665		try testing.expectEqualDeep(expected_files, actual_files.items);
1666
1667		const expected_file_contents =
1668		\\revision 1
1669		\\revision 2
1670		\\revision 4
1671		\\revision 5
1672		\\revision 7
1673		\\revision 8
1674		\\revision 9
1675		\\revision 10
1676		\\revision 12
1677		\\revision 13
1678		\\revision 14
1679		\\revision 18
1680		\\revision 19
1681		\\
1682		;
1683		const actual_file_contents = try worktree.dir.readFileAlloc(io, "file", testing.allocator, .limited(max_file_size));
1684		defer testing.allocator.free(actual_file_contents);
1685		try testing.expectEqualStrings(expected_file_contents, actual_file_contents);
1686		}
1687
1688		/// Checksum calculation is useful for troubleshooting and debugging, but it's
1689		/// redundant since the package manager already does content hashing at the
1690		/// end. Let's save time by not doing that work, but, I left a cookie crumb
1691		/// trail here if you want to restore the functionality for tinkering purposes.
1692		const skip_checksums = true;
1693
1694		test "SHA-1 packfile indexing and checkout" {
1695		try runRepositoryTest(std.testing.io, .sha1, "dd582c0720819ab7130b103635bd7271b9fd4feb");
1696		}
1697
1698		test "SHA-256 packfile indexing and checkout" {
1699		try runRepositoryTest(std.testing.io, .sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a");
1700		}
1701
1702		/// Checks out a commit of a packfile. Intended for experimenting with and
1703		/// benchmarking possible optimizations to the indexing and checkout behavior.
1704		pub fn main() !void {
1705		const allocator = std.heap.smp_allocator;
1706
1707		var threaded: Io.Threaded = .init(allocator, .{});
1708		defer threaded.deinit();
1709		const io = threaded.io();
1710
1711		const args = try std.process.argsAlloc(allocator);
1712		defer std.process.argsFree(allocator, args);
1713		if (args.len != 5) {
1714		return error.InvalidArguments; // Arguments: format packfile commit worktree
1715		}
1716
1717		const format = std.meta.stringToEnum(Oid.Format, args[1]) orelse return error.InvalidFormat;
1718
1719		var pack_file = try Io.Dir.cwd().openFile(io, args[2], .{});
1720		defer pack_file.close(io);
1721		var pack_file_buffer: [4096]u8 = undefined;
1722		var pack_file_reader = pack_file.reader(io, &pack_file_buffer);
1723
1724		const commit = try Oid.parse(format, args[3]);
1725		var worktree = try Io.Dir.cwd().createDirPathOpen(io, args[4], .{});
1726		defer worktree.close(io);
1727
1728		var git_dir = try worktree.createDirPathOpen(io, ".git", .{});
1729		defer git_dir.close(io);
1730
1731		std.debug.print("Starting index...\n", .{});
1732		var index_file = try git_dir.createFile(io, "idx", .{ .read = true });
1733		defer index_file.close(io);
1734		var index_file_buffer: [4096]u8 = undefined;
1735		var index_file_writer = index_file.writer(io, &index_file_buffer);
1736		try indexPack(allocator, format, &pack_file_reader, &index_file_writer);
1737
1738		std.debug.print("Starting checkout...\n", .{});
1739		var index_file_reader = index_file.reader(io, &index_file_buffer);
1740		var repository: Repository = undefined;
1741		try repository.init(allocator, format, &pack_file_reader, &index_file_reader);
1742		defer repository.deinit();
1743		var diagnostics: Diagnostics = .{ .allocator = allocator };
1744		defer diagnostics.deinit();
1745		try repository.checkout(io, worktree, commit, &diagnostics);
1746
1747		for (diagnostics.errors.items) \|err\| {
1748		std.debug.print("Diagnostic: {}\n", .{err});
1749		}
1750		}

src/Package/Fetch/git/testdata/testrepo-sha1.idx deleted

Binary files a/src/Package/Fetch/git/testdata/testrepo-sha1.idx and /dev/null differ

src/Package/Fetch/git/testdata/testrepo-sha1.pack deleted

Binary files a/src/Package/Fetch/git/testdata/testrepo-sha1.pack and /dev/null differ

src/Package/Fetch/git/testdata/testrepo-sha256.idx deleted

Binary files a/src/Package/Fetch/git/testdata/testrepo-sha256.idx and /dev/null differ

src/Package/Fetch/git/testdata/testrepo-sha256.pack deleted

Binary files a/src/Package/Fetch/git/testdata/testrepo-sha256.pack and /dev/null differ

src/Package/Manifest.zig deleted-734

...	...	@@ -1,734 +0,0 @@
1		const Manifest = @This();
2
3		const std = @import("std");
4		const Io = std.Io;
5		const mem = std.mem;
6		const Allocator = std.mem.Allocator;
7		const assert = std.debug.assert;
8		const Ast = std.zig.Ast;
9		const testing = std.testing;
10
11		const Package = @import("../Package.zig");
12
13		pub const max_bytes = 10 * 1024 * 1024;
14		pub const basename = "build.zig.zon";
15		pub const max_name_len = 32;
16		pub const max_version_len = 32;
17
18		pub const Dependency = struct {
19		location: Location,
20		location_tok: Ast.TokenIndex,
21		location_node: Ast.Node.Index,
22		hash: ?[]const u8,
23		hash_tok: Ast.OptionalTokenIndex,
24		hash_node: Ast.Node.OptionalIndex,
25		node: Ast.Node.Index,
26		name_tok: Ast.TokenIndex,
27		lazy: bool,
28
29		pub const Location = union(enum) {
30		url: []const u8,
31		path: []const u8,
32		};
33		};
34
35		pub const ErrorMessage = struct {
36		msg: []const u8,
37		tok: Ast.TokenIndex,
38		off: u32,
39		};
40
41		name: []const u8,
42		id: u32,
43		version: std.SemanticVersion,
44		version_node: Ast.Node.Index,
45		dependencies: std.array_hash_map.String(Dependency),
46		dependencies_node: Ast.Node.OptionalIndex,
47		paths: std.array_hash_map.String(void),
48		minimum_zig_version: ?std.SemanticVersion,
49
50		errors: []ErrorMessage,
51		arena_state: std.heap.ArenaAllocator.State,
52
53		pub const ParseOptions = struct {
54		allow_missing_paths_field: bool = false,
55		};
56
57		pub const Error = Allocator.Error;
58
59		pub fn parse(gpa: Allocator, ast: *const Ast, rng: std.Random, options: ParseOptions) Error!Manifest {
60		const main_node_index = ast.nodeData(.root).node;
61
62		var arena_instance = std.heap.ArenaAllocator.init(gpa);
63		errdefer arena_instance.deinit();
64
65		var p: Parse = .{
66		.gpa = gpa,
67		.ast = ast.*,
68		.arena = arena_instance.allocator(),
69		.errors = .empty,
70
71		.name = undefined,
72		.id = 0,
73		.version = undefined,
74		.version_node = undefined,
75		.dependencies = .{},
76		.dependencies_node = .none,
77		.paths = .empty,
78		.allow_missing_paths_field = options.allow_missing_paths_field,
79		.minimum_zig_version = null,
80		.buf = .empty,
81		};
82		defer p.buf.deinit(gpa);
83		defer p.errors.deinit(gpa);
84		defer p.dependencies.deinit(gpa);
85		defer p.paths.deinit(gpa);
86
87		p.parseRoot(main_node_index, rng) catch \|err\| switch (err) {
88		error.ParseFailure => assert(p.errors.items.len > 0),
89		else => \|e\| return e,
90		};
91
92		return .{
93		.name = p.name,
94		.id = p.id,
95		.version = p.version,
96		.version_node = p.version_node,
97		.dependencies = try p.dependencies.clone(p.arena),
98		.dependencies_node = p.dependencies_node,
99		.paths = try p.paths.clone(p.arena),
100		.minimum_zig_version = p.minimum_zig_version,
101		.errors = try p.arena.dupe(ErrorMessage, p.errors.items),
102		.arena_state = arena_instance.state,
103		};
104		}
105
106		pub fn deinit(man: *Manifest, gpa: Allocator) void {
107		man.arena_state.promote(gpa).deinit();
108		man.* = undefined;
109		}
110
111		pub fn copyErrorsIntoBundle(
112		man: Manifest,
113		ast: Ast,
114		/// ErrorBundle null-terminated string index
115		src_path: u32,
116		eb: *std.zig.ErrorBundle.Wip,
117		) Allocator.Error!void {
118		for (man.errors) \|msg\| {
119		const start_loc = ast.tokenLocation(0, msg.tok);
120
121		try eb.addRootErrorMessage(.{
122		.msg = try eb.addString(msg.msg),
123		.src_loc = try eb.addSourceLocation(.{
124		.src_path = src_path,
125		.span_start = ast.tokenStart(msg.tok),
126		.span_end = @intCast(ast.tokenStart(msg.tok) + ast.tokenSlice(msg.tok).len),
127		.span_main = ast.tokenStart(msg.tok) + msg.off,
128		.line = @intCast(start_loc.line),
129		.column = @intCast(start_loc.column),
130		.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
131		}),
132		});
133		}
134		}
135
136		const Parse = struct {
137		gpa: Allocator,
138		ast: Ast,
139		arena: Allocator,
140		buf: std.ArrayList(u8),
141		errors: std.ArrayList(ErrorMessage),
142
143		name: []const u8,
144		id: u32,
145		version: std.SemanticVersion,
146		version_node: Ast.Node.Index,
147		dependencies: std.array_hash_map.String(Dependency),
148		dependencies_node: Ast.Node.OptionalIndex,
149		paths: std.array_hash_map.String(void),
150		allow_missing_paths_field: bool,
151		minimum_zig_version: ?std.SemanticVersion,
152
153		const InnerError = error{ ParseFailure, OutOfMemory };
154
155		fn parseRoot(p: *Parse, node: Ast.Node.Index, rng: std.Random) !void {
156		const ast = p.ast;
157		const main_token = ast.nodeMainToken(node);
158
159		var buf: [2]Ast.Node.Index = undefined;
160		const struct_init = ast.fullStructInit(&buf, node) orelse {
161		return fail(p, main_token, "expected top level expression to be a struct", .{});
162		};
163
164		var have_name = false;
165		var have_version = false;
166		var have_included_paths = false;
167		var fingerprint: ?Package.Fingerprint = null;
168
169		for (struct_init.ast.fields) \|field_init\| {
170		const name_token = ast.firstToken(field_init) - 2;
171		const field_name = try identifierTokenString(p, name_token);
172		// We could get fancy with reflection and comptime logic here but doing
173		// things manually provides an opportunity to do any additional verification
174		// that is desirable on a per-field basis.
175		if (mem.eql(u8, field_name, "dependencies")) {
176		p.dependencies_node = field_init.toOptional();
177		try parseDependencies(p, field_init);
178		} else if (mem.eql(u8, field_name, "paths")) {
179		have_included_paths = true;
180		try parseIncludedPaths(p, field_init);
181		} else if (mem.eql(u8, field_name, "name")) {
182		p.name = try parseName(p, field_init);
183		have_name = true;
184		} else if (mem.eql(u8, field_name, "fingerprint")) {
185		fingerprint = try parseFingerprint(p, field_init);
186		} else if (mem.eql(u8, field_name, "version")) {
187		p.version_node = field_init;
188		const version_text = try parseString(p, field_init);
189		if (version_text.len > max_version_len) {
190		try appendError(p, ast.nodeMainToken(field_init), "version string length {d} exceeds maximum of {d}", .{ version_text.len, max_version_len });
191		}
192		p.version = std.SemanticVersion.parse(version_text) catch \|err\| v: {
193		try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)});
194		break :v undefined;
195		};
196		have_version = true;
197		} else if (mem.eql(u8, field_name, "minimum_zig_version")) {
198		const version_text = try parseString(p, field_init);
199		p.minimum_zig_version = std.SemanticVersion.parse(version_text) catch \|err\| v: {
200		try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)});
201		break :v null;
202		};
203		} else {
204		// Ignore unknown fields so that we can add fields in future zig
205		// versions without breaking older zig versions.
206		}
207		}
208
209		if (!have_name) {
210		try appendError(p, main_token, "missing top-level 'name' field", .{});
211		} else {
212		if (fingerprint) \|n\| {
213		if (!n.validate(p.name)) {
214		return fail(p, main_token, "invalid fingerprint: 0x{x}; if this is a new or forked package, use this value: 0x{x}", .{
215		n.int(), Package.Fingerprint.generate(rng, p.name).int(),
216		});
217		}
218		p.id = n.id;
219		} else {
220		try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{
221		Package.Fingerprint.generate(rng, p.name).int(),
222		});
223		}
224		}
225
226		if (!have_version) {
227		try appendError(p, main_token, "missing top-level 'version' field", .{});
228		}
229
230		if (!have_included_paths) {
231		if (p.allow_missing_paths_field) {
232		try p.paths.put(p.gpa, "", {});
233		} else {
234		try appendError(p, main_token, "missing top-level 'paths' field", .{});
235		}
236		}
237		}
238
239		fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void {
240		const ast = p.ast;
241
242		var buf: [2]Ast.Node.Index = undefined;
243		const struct_init = ast.fullStructInit(&buf, node) orelse {
244		const tok = ast.nodeMainToken(node);
245		return fail(p, tok, "expected dependencies expression to be a struct", .{});
246		};
247
248		for (struct_init.ast.fields) \|field_init\| {
249		const name_token = ast.firstToken(field_init) - 2;
250		const dep_name = try identifierTokenString(p, name_token);
251		const dep = try parseDependency(p, field_init);
252		try p.dependencies.put(p.gpa, dep_name, dep);
253		}
254		}
255
256		fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency {
257		const ast = p.ast;
258
259		var buf: [2]Ast.Node.Index = undefined;
260		const struct_init = ast.fullStructInit(&buf, node) orelse {
261		const tok = ast.nodeMainToken(node);
262		return fail(p, tok, "expected dependency expression to be a struct", .{});
263		};
264
265		var dep: Dependency = .{
266		.location = undefined,
267		.location_tok = undefined,
268		.location_node = undefined,
269		.hash = null,
270		.hash_tok = .none,
271		.hash_node = .none,
272		.node = node,
273		.name_tok = undefined,
274		.lazy = false,
275		};
276		var has_location = false;
277
278		for (struct_init.ast.fields) \|field_init\| {
279		const name_token = ast.firstToken(field_init) - 2;
280		dep.name_tok = name_token;
281		const field_name = try identifierTokenString(p, name_token);
282		// We could get fancy with reflection and comptime logic here but doing
283		// things manually provides an opportunity to do any additional verification
284		// that is desirable on a per-field basis.
285		if (mem.eql(u8, field_name, "url")) {
286		if (has_location) {
287		return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{});
288		}
289		dep.location = .{
290		.url = parseString(p, field_init) catch \|err\| switch (err) {
291		error.ParseFailure => continue,
292		else => \|e\| return e,
293		},
294		};
295		has_location = true;
296		dep.location_tok = ast.nodeMainToken(field_init);
297		dep.location_node = field_init;
298		} else if (mem.eql(u8, field_name, "path")) {
299		if (has_location) {
300		return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{});
301		}
302		dep.location = .{
303		.path = parseString(p, field_init) catch \|err\| switch (err) {
304		error.ParseFailure => continue,
305		else => \|e\| return e,
306		},
307		};
308		has_location = true;
309		dep.location_tok = ast.nodeMainToken(field_init);
310		dep.location_node = field_init;
311		} else if (mem.eql(u8, field_name, "hash")) {
312		dep.hash = parseHash(p, field_init) catch \|err\| switch (err) {
313		error.ParseFailure => continue,
314		else => \|e\| return e,
315		};
316		dep.hash_tok = .fromToken(ast.nodeMainToken(field_init));
317		dep.hash_node = field_init.toOptional();
318		} else if (mem.eql(u8, field_name, "lazy")) {
319		dep.lazy = parseBool(p, field_init) catch \|err\| switch (err) {
320		error.ParseFailure => continue,
321		else => \|e\| return e,
322		};
323		} else {
324		// Ignore unknown fields so that we can add fields in future zig
325		// versions without breaking older zig versions.
326		}
327		}
328
329		if (!has_location) {
330		try appendError(p, ast.nodeMainToken(node), "dependency requires location field, one of 'url' or 'path'.", .{});
331		}
332
333		return dep;
334		}
335
336		fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void {
337		const ast = p.ast;
338
339		var buf: [2]Ast.Node.Index = undefined;
340		const array_init = ast.fullArrayInit(&buf, node) orelse {
341		const tok = ast.nodeMainToken(node);
342		return fail(p, tok, "expected paths expression to be a list of strings", .{});
343		};
344
345		for (array_init.ast.elements) \|elem_node\| {
346		const path_string = try parseString(p, elem_node);
347		// This is normalized so that it can be used in string comparisons
348		// against file system paths.
349		const normalized = try std.fs.path.resolve(p.arena, &.{path_string});
350		try p.paths.put(p.gpa, normalized, {});
351		}
352		}
353
354		fn parseBool(p: *Parse, node: Ast.Node.Index) !bool {
355		const ast = p.ast;
356		if (ast.nodeTag(node) != .identifier) {
357		return fail(p, ast.nodeMainToken(node), "expected identifier", .{});
358		}
359		const ident_token = ast.nodeMainToken(node);
360		const token_bytes = ast.tokenSlice(ident_token);
361		if (mem.eql(u8, token_bytes, "true")) {
362		return true;
363		} else if (mem.eql(u8, token_bytes, "false")) {
364		return false;
365		} else {
366		return fail(p, ident_token, "expected boolean", .{});
367		}
368		}
369
370		fn parseFingerprint(p: *Parse, node: Ast.Node.Index) !Package.Fingerprint {
371		const ast = p.ast;
372		const main_token = ast.nodeMainToken(node);
373		if (ast.nodeTag(node) != .number_literal) {
374		return fail(p, main_token, "expected integer literal", .{});
375		}
376		const token_bytes = ast.tokenSlice(main_token);
377		const parsed = std.zig.parseNumberLiteral(token_bytes);
378		switch (parsed) {
379		.int => \|n\| return @bitCast(n),
380		.big_int, .float => return fail(p, main_token, "expected u64 integer literal, found {s}", .{
381		@tagName(parsed),
382		}),
383		.failure => \|err\| return fail(p, main_token, "bad integer literal: {s}", .{@tagName(err)}),
384		}
385		}
386
387		fn parseName(p: *Parse, node: Ast.Node.Index) ![]const u8 {
388		const ast = p.ast;
389		const main_token = ast.nodeMainToken(node);
390
391		if (ast.nodeTag(node) != .enum_literal)
392		return fail(p, main_token, "expected enum literal", .{});
393
394		const ident_name = ast.tokenSlice(main_token);
395		if (mem.startsWith(u8, ident_name, "@"))
396		return fail(p, main_token, "name must be a valid bare zig identifier", .{});
397
398		if (ident_name.len > max_name_len)
399		return fail(p, main_token, "name '{f}' exceeds max length of {d}", .{
400		std.zig.fmtId(ident_name), max_name_len,
401		});
402
403		return ident_name;
404		}
405
406		fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 {
407		const ast = p.ast;
408		if (ast.nodeTag(node) != .string_literal) {
409		return fail(p, ast.nodeMainToken(node), "expected string literal", .{});
410		}
411		const str_lit_token = ast.nodeMainToken(node);
412		const token_bytes = ast.tokenSlice(str_lit_token);
413		p.buf.clearRetainingCapacity();
414		try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0);
415		const duped = try p.arena.dupe(u8, p.buf.items);
416		return duped;
417		}
418
419		fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 {
420		const ast = p.ast;
421		const tok = ast.nodeMainToken(node);
422		const h = try parseString(p, node);
423		switch (Package.Hash.validate(h)) {
424		.ok => return h,
425		else => \|t\| return fail(p, tok, "invalid hash: {t}", .{t}),
426		}
427		}
428
429		/// TODO: try to DRY this with AstGen.identifierTokenString
430		fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 {
431		const ast = p.ast;
432		assert(ast.tokenTag(token) == .identifier);
433		const ident_name = ast.tokenSlice(token);
434		if (!mem.startsWith(u8, ident_name, "@")) {
435		return ident_name;
436		}
437		p.buf.clearRetainingCapacity();
438		try parseStrLit(p, token, &p.buf, ident_name, 1);
439		const duped = try p.arena.dupe(u8, p.buf.items);
440		return duped;
441		}
442
443		/// TODO: try to DRY this with AstGen.parseStrLit
444		fn parseStrLit(
445		p: *Parse,
446		token: Ast.TokenIndex,
447		buf: *std.ArrayList(u8),
448		bytes: []const u8,
449		offset: u32,
450		) InnerError!void {
451		const raw_string = bytes[offset..];
452		const result = r: {
453		var aw: std.Io.Writer.Allocating = .fromArrayList(p.gpa, buf);
454		defer buf.* = aw.toArrayList();
455		break :r std.zig.string_literal.parseWrite(&aw.writer, raw_string) catch \|err\| switch (err) {
456		error.WriteFailed => return error.OutOfMemory,
457		};
458		};
459		switch (result) {
460		.success => {},
461		.failure => \|err\| try p.appendStrLitError(err, token, bytes, offset),
462		}
463		}
464
465		/// TODO: try to DRY this with AstGen.failWithStrLitError
466		fn appendStrLitError(
467		p: *Parse,
468		err: std.zig.string_literal.Error,
469		token: Ast.TokenIndex,
470		bytes: []const u8,
471		offset: u32,
472		) Allocator.Error!void {
473		const raw_string = bytes[offset..];
474		switch (err) {
475		.invalid_escape_character => \|bad_index\| {
476		try p.appendErrorOff(
477		token,
478		offset + @as(u32, @intCast(bad_index)),
479		"invalid escape character: '{c}'",
480		.{raw_string[bad_index]},
481		);
482		},
483		.expected_hex_digit => \|bad_index\| {
484		try p.appendErrorOff(
485		token,
486		offset + @as(u32, @intCast(bad_index)),
487		"expected hex digit, found '{c}'",
488		.{raw_string[bad_index]},
489		);
490		},
491		.empty_unicode_escape_sequence => \|bad_index\| {
492		try p.appendErrorOff(
493		token,
494		offset + @as(u32, @intCast(bad_index)),
495		"empty unicode escape sequence",
496		.{},
497		);
498		},
499		.expected_hex_digit_or_rbrace => \|bad_index\| {
500		try p.appendErrorOff(
501		token,
502		offset + @as(u32, @intCast(bad_index)),
503		"expected hex digit or '}}', found '{c}'",
504		.{raw_string[bad_index]},
505		);
506		},
507		.invalid_unicode_codepoint => \|bad_index\| {
508		try p.appendErrorOff(
509		token,
510		offset + @as(u32, @intCast(bad_index)),
511		"unicode escape does not correspond to a valid unicode scalar value",
512		.{},
513		);
514		},
515		.expected_lbrace => \|bad_index\| {
516		try p.appendErrorOff(
517		token,
518		offset + @as(u32, @intCast(bad_index)),
519		"expected '{{', found '{c}",
520		.{raw_string[bad_index]},
521		);
522		},
523		.expected_rbrace => \|bad_index\| {
524		try p.appendErrorOff(
525		token,
526		offset + @as(u32, @intCast(bad_index)),
527		"expected '}}', found '{c}",
528		.{raw_string[bad_index]},
529		);
530		},
531		.expected_single_quote => \|bad_index\| {
532		try p.appendErrorOff(
533		token,
534		offset + @as(u32, @intCast(bad_index)),
535		"expected single quote ('), found '{c}",
536		.{raw_string[bad_index]},
537		);
538		},
539		.invalid_character => \|bad_index\| {
540		try p.appendErrorOff(
541		token,
542		offset + @as(u32, @intCast(bad_index)),
543		"invalid byte in string or character literal: '{c}'",
544		.{raw_string[bad_index]},
545		);
546		},
547		.empty_char_literal => {
548		try p.appendErrorOff(token, offset, "empty character literal", .{});
549		},
550		}
551		}
552
553		fn fail(
554		p: *Parse,
555		tok: Ast.TokenIndex,
556		comptime fmt: []const u8,
557		args: anytype,
558		) InnerError {
559		try appendError(p, tok, fmt, args);
560		return error.ParseFailure;
561		}
562
563		fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void {
564		return appendErrorOff(p, tok, 0, fmt, args);
565		}
566
567		fn appendErrorOff(
568		p: *Parse,
569		tok: Ast.TokenIndex,
570		byte_offset: u32,
571		comptime fmt: []const u8,
572		args: anytype,
573		) Allocator.Error!void {
574		try p.errors.append(p.gpa, .{
575		.msg = try std.fmt.allocPrint(p.arena, fmt, args),
576		.tok = tok,
577		.off = byte_offset,
578		});
579		}
580		};
581
582		pub fn load(
583		io: Io,
584		arena: Allocator,
585		manifest_path: std.Build.Cache.Path,
586		ast: *std.zig.Ast,
587		error_bundle: *std.zig.ErrorBundle.Wip,
588		manifest: *Manifest,
589		allow_missing_paths_field: bool,
590		) !void {
591		const manifest_bytes = try manifest_path.root_dir.handle.readFileAllocOptions(
592		io,
593		manifest_path.sub_path,
594		arena,
595		.limited(max_bytes),
596		.@"1",
597		0,
598		);
599
600		ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon);
601
602		if (ast.errors.len > 0) {
603		const file_path = try manifest_path.joinString(arena, "");
604		try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, error_bundle);
605		return error.ErrorsBundled;
606		}
607
608		const rng: std.Random.IoSource = .{ .io = io };
609
610		manifest.* = try parse(arena, ast, rng.interface(), .{
611		.allow_missing_paths_field = allow_missing_paths_field,
612		});
613
614		if (manifest.errors.len > 0) {
615		const src_path = try error_bundle.printString("{f}", .{manifest_path});
616		try manifest.copyErrorsIntoBundle(ast.*, src_path, error_bundle);
617		return error.ErrorsBundled;
618		}
619		}
620
621		test "basic" {
622		const gpa = testing.allocator;
623
624		const example =
625		\\.{
626		\\ .name = .foo,
627		\\ .fingerprint = 0x8c736521490b23df,
628		\\ .version = "3.2.1",
629		\\ .paths = .{""},
630		\\ .dependencies = .{
631		\\ .bar = .{
632		\\ .url = "https://example.com/baz.tar.gz",
633		\\ .hash = "libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5",
634		\\ },
635		\\ },
636		\\}
637		;
638
639		var ast = try Ast.parse(gpa, example, .zon);
640		defer ast.deinit(gpa);
641
642		try testing.expect(ast.errors.len == 0);
643
644		var rng = std.Random.DefaultPrng.init(0);
645
646		var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{});
647		defer manifest.deinit(gpa);
648
649		try testing.expect(manifest.errors.len == 0);
650		try testing.expectEqualStrings("foo", manifest.name);
651
652		try testing.expectEqual(@as(std.SemanticVersion, .{
653		.major = 3,
654		.minor = 2,
655		.patch = 1,
656		}), manifest.version);
657
658		try testing.expect(manifest.dependencies.count() == 1);
659		try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]);
660		try testing.expectEqualStrings(
661		"https://example.com/baz.tar.gz",
662		manifest.dependencies.values()[0].location.url,
663		);
664		try testing.expectEqualStrings(
665		"libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5",
666		manifest.dependencies.values()[0].hash orelse return error.TestFailed,
667		);
668
669		try testing.expect(manifest.minimum_zig_version == null);
670		}
671
672		test "minimum_zig_version" {
673		const gpa = testing.allocator;
674
675		const example =
676		\\.{
677		\\ .name = .foo,
678		\\ .fingerprint = 0x8c736521490b23df,
679		\\ .version = "3.2.1",
680		\\ .paths = .{""},
681		\\ .minimum_zig_version = "0.11.1",
682		\\}
683		;
684
685		var ast = try Ast.parse(gpa, example, .zon);
686		defer ast.deinit(gpa);
687
688		try testing.expect(ast.errors.len == 0);
689
690		var rng = std.Random.DefaultPrng.init(0);
691
692		var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{});
693		defer manifest.deinit(gpa);
694
695		try testing.expect(manifest.errors.len == 0);
696		try testing.expect(manifest.dependencies.count() == 0);
697
698		try testing.expect(manifest.minimum_zig_version != null);
699
700		try testing.expectEqual(@as(std.SemanticVersion, .{
701		.major = 0,
702		.minor = 11,
703		.patch = 1,
704		}), manifest.minimum_zig_version.?);
705		}
706
707		test "minimum_zig_version - invalid version" {
708		const gpa = testing.allocator;
709
710		const example =
711		\\.{
712		\\ .name = .foo,
713		\\ .fingerprint = 0x8c736521490b23df,
714		\\ .version = "3.2.1",
715		\\ .minimum_zig_version = "X.11.1",
716		\\ .paths = .{""},
717		\\}
718		;
719
720		var ast = try Ast.parse(gpa, example, .zon);
721		defer ast.deinit(gpa);
722
723		try testing.expect(ast.errors.len == 0);
724
725		var rng = std.Random.DefaultPrng.init(0);
726
727		var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{});
728		defer manifest.deinit(gpa);
729
730		try testing.expect(manifest.errors.len == 1);
731		try testing.expect(manifest.dependencies.count() == 0);
732
733		try testing.expect(manifest.minimum_zig_version == null);
734		}

src/Package/Module.zig deleted-529

...	...	@@ -1,529 +0,0 @@
1		//! Corresponds to something that Zig source code can `@import`.
2
3		/// The root directory of the module. Only files inside this directory can be imported.
4		root: Compilation.Path,
5		/// Path to the root source file of this module. Relative to `root`. May contain path separators.
6		root_src_path: []const u8,
7		/// Name used in compile errors. Looks like "root.foo.bar".
8		fully_qualified_name: []const u8,
9		/// The dependency table of this module. The shared dependencies 'std' and
10		/// 'root' are not specified in every module dependency table, but are stored
11		/// separately in `Zcu`. 'builtin' is also not stored here, although it is
12		/// not necessarily the same between all modules. Handling of `@import` in
13		/// the rest of the compiler must detect these special names and use the
14		/// correct module instead of consulting `deps`.
15		deps: Deps = .{},
16
17		resolved_target: ResolvedTarget,
18		optimize_mode: std.lang.OptimizeMode,
19		code_model: std.lang.CodeModel,
20		single_threaded: bool,
21		error_tracing: bool,
22		valgrind: bool,
23		pic: bool,
24		strip: bool,
25		omit_frame_pointer: bool,
26		stack_check: bool,
27		stack_protector: u32,
28		red_zone: bool,
29		sanitize_c: std.zig.SanitizeC,
30		sanitize_thread: bool,
31		fuzz: bool,
32		unwind_tables: std.lang.UnwindTables,
33		cc_argv: []const []const u8,
34		/// (SPIR-V) whether to generate a structured control flow graph or not
35		structured_cfg: bool,
36		no_builtin: bool,
37
38		pub const Deps = std.array_hash_map.String(*Module);
39
40		pub const Tree = struct {
41		/// Each `Package` exposes a `Module` with build.zig as its root source file.
42		build_module_table: std.array_hash_map.Auto(MultiHashHexDigest, *Module),
43		};
44
45		pub const CreateOptions = struct {
46		paths: Paths,
47		fully_qualified_name: []const u8,
48
49		cc_argv: []const []const u8,
50		inherited: Inherited,
51		global: Compilation.Config,
52		/// If this is null then `resolved_target` must be non-null.
53		parent: ?*Package.Module,
54
55		pub const Paths = struct {
56		root: Compilation.Path,
57		/// Relative to `root`. May contain path separators.
58		root_src_path: []const u8,
59		};
60
61		pub const Inherited = struct {
62		/// If this is null then `parent` must be non-null.
63		resolved_target: ?ResolvedTarget = null,
64		optimize_mode: ?std.lang.OptimizeMode = null,
65		code_model: ?std.lang.CodeModel = null,
66		single_threaded: ?bool = null,
67		error_tracing: ?bool = null,
68		valgrind: ?bool = null,
69		pic: ?bool = null,
70		strip: ?bool = null,
71		omit_frame_pointer: ?bool = null,
72		stack_check: ?bool = null,
73		/// null means default.
74		/// 0 means no stack protector.
75		/// other number means stack protection with that buffer size.
76		stack_protector: ?u32 = null,
77		red_zone: ?bool = null,
78		unwind_tables: ?std.lang.UnwindTables = null,
79		sanitize_c: ?std.zig.SanitizeC = null,
80		sanitize_thread: ?bool = null,
81		fuzz: ?bool = null,
82		structured_cfg: ?bool = null,
83		no_builtin: ?bool = null,
84		};
85		};
86
87		pub const ResolvedTarget = struct {
88		result: std.Target,
89		is_native_os: bool,
90		is_native_abi: bool,
91		is_explicit_dynamic_linker: bool,
92		llvm_cpu_features: ?[*:0]const u8 = null,
93		};
94
95		pub const CreateError = error{
96		OutOfMemory,
97		ValgrindUnsupportedOnTarget,
98		TargetRequiresSingleThreaded,
99		BackendRequiresSingleThreaded,
100		TargetRequiresPic,
101		PieRequiresPic,
102		DynamicLinkingRequiresPic,
103		TargetHasNoRedZone,
104		StackCheckUnsupportedByTarget,
105		StackProtectorUnsupportedByTarget,
106		StackProtectorUnavailableWithoutLibC,
107		};
108
109		/// At least one of `parent` and `resolved_target` must be non-null.
110		pub fn create(arena: Allocator, options: CreateOptions) !*Package.Module {
111		if (options.inherited.sanitize_thread == true) assert(options.global.any_sanitize_thread);
112		if (options.inherited.fuzz == true) assert(options.global.any_fuzz);
113		if (options.inherited.single_threaded == false) assert(options.global.any_non_single_threaded);
114		if (options.inherited.unwind_tables) \|uwt\| if (uwt != .none) assert(options.global.any_unwind_tables);
115		if (options.inherited.sanitize_c) \|sc\| if (sc != .off) assert(options.global.any_sanitize_c != .off);
116		if (options.inherited.error_tracing == true) assert(options.global.any_error_tracing);
117
118		const resolved_target = options.inherited.resolved_target orelse options.parent.?.resolved_target;
119		const target = &resolved_target.result;
120
121		const optimize_mode = options.inherited.optimize_mode orelse
122		if (options.parent) \|p\| p.optimize_mode else options.global.root_optimize_mode;
123
124		const strip = b: {
125		if (options.inherited.strip) \|x\| break :b x;
126		if (options.parent) \|p\| break :b p.strip;
127		break :b options.global.root_strip;
128		};
129
130		const zig_backend = target_util.zigBackend(target, options.global.use_llvm);
131
132		const valgrind = b: {
133		if (!target_util.hasValgrindSupport(target, zig_backend)) {
134		if (options.inherited.valgrind == true)
135		return error.ValgrindUnsupportedOnTarget;
136		break :b false;
137		}
138		if (options.inherited.valgrind) \|x\| break :b x;
139		if (options.parent) \|p\| break :b p.valgrind;
140		if (strip) break :b false;
141		break :b optimize_mode == .Debug;
142		};
143
144		const single_threaded = b: {
145		if (target_util.alwaysSingleThreaded(target)) {
146		if (options.inherited.single_threaded == false)
147		return error.TargetRequiresSingleThreaded;
148		break :b true;
149		}
150
151		if (options.global.have_zcu) {
152		if (!target_util.supportsThreads(target, zig_backend)) {
153		if (options.inherited.single_threaded == false)
154		return error.BackendRequiresSingleThreaded;
155		break :b true;
156		}
157		}
158
159		if (options.inherited.single_threaded) \|x\| break :b x;
160		if (options.parent) \|p\| break :b p.single_threaded;
161		break :b target_util.defaultSingleThreaded(target);
162		};
163
164		const error_tracing = b: {
165		if (options.inherited.error_tracing) \|x\| break :b x;
166		if (options.parent) \|p\| break :b p.error_tracing;
167		break :b options.global.root_error_tracing;
168		};
169
170		const pic = b: {
171		if (target_util.requiresPic(target, options.global.link_libc)) {
172		if (options.inherited.pic == false)
173		return error.TargetRequiresPic;
174		break :b true;
175		}
176		if (options.global.pie) {
177		if (options.inherited.pic == false)
178		return error.PieRequiresPic;
179		break :b true;
180		}
181		if (options.global.link_mode == .dynamic and target_util.requiresPicForDynamicLink(target)) {
182		if (options.inherited.pic == false)
183		return error.DynamicLinkingRequiresPic;
184		break :b true;
185		}
186		if (options.inherited.pic) \|x\| break :b x;
187		if (options.parent) \|p\| break :b p.pic;
188
189		// Default to PIC on targets where we default to producing PIEs to make
190		// the common case of linking objects and static libraries into an
191		// executable work out of the box.
192		break :b target_util.defaultPie(target);
193		};
194
195		const red_zone = b: {
196		if (!target_util.hasRedZone(target)) {
197		if (options.inherited.red_zone == true)
198		return error.TargetHasNoRedZone;
199		break :b false;
200		}
201		if (options.inherited.red_zone) \|x\| break :b x;
202		if (options.parent) \|p\| break :b p.red_zone;
203		break :b true;
204		};
205
206		const omit_frame_pointer = b: {
207		if (options.inherited.omit_frame_pointer) \|x\| break :b x;
208		if (options.parent) \|p\| break :b p.omit_frame_pointer;
209		if (optimize_mode == .ReleaseSmall) {
210		// On x86, in most cases, keeping the frame pointer usually results in smaller binary size.
211		// This has to do with how instructions for memory access via the stack base pointer register (when keeping the frame pointer)
212		// are smaller than instructions for memory access via the stack pointer register (when omitting the frame pointer).
213		break :b !target.cpu.arch.isX86();
214		}
215		break :b false;
216		};
217
218		const sanitize_thread = b: {
219		if (options.inherited.sanitize_thread) \|x\| break :b x;
220		if (options.parent) \|p\| break :b p.sanitize_thread;
221		break :b false;
222		};
223
224		const unwind_tables = b: {
225		if (options.inherited.unwind_tables) \|x\| break :b x;
226		if (options.parent) \|p\| break :b p.unwind_tables;
227
228		break :b target_util.defaultUnwindTables(
229		target,
230		options.global.link_libunwind,
231		sanitize_thread or options.global.any_sanitize_thread,
232		);
233		};
234
235		const fuzz = b: {
236		if (options.inherited.fuzz) \|x\| break :b x;
237		if (options.parent) \|p\| break :b p.fuzz;
238		break :b false;
239		};
240
241		const code_model: std.lang.CodeModel = b: {
242		if (options.inherited.code_model) \|x\| break :b x;
243		if (options.parent) \|p\| break :b p.code_model;
244		break :b .default;
245		};
246
247		const is_safe_mode = switch (optimize_mode) {
248		.Debug, .ReleaseSafe => true,
249		.ReleaseFast, .ReleaseSmall => false,
250		};
251
252		const sanitize_c: std.zig.SanitizeC = b: {
253		if (options.inherited.sanitize_c) \|x\| break :b x;
254		if (options.parent) \|p\| break :b p.sanitize_c;
255		break :b switch (optimize_mode) {
256		.Debug => .full,
257		// It's recommended to use the minimal runtime in production
258		// environments due to the security implications of the full runtime.
259		// The minimal runtime doesn't provide much benefit over simply
260		// trapping, however, so we do that instead.
261		.ReleaseSafe => .trap,
262		.ReleaseFast, .ReleaseSmall => .off,
263		};
264		};
265
266		const stack_check = b: {
267		if (!target_util.supportsStackProbing(target, zig_backend)) {
268		if (options.inherited.stack_check == true)
269		return error.StackCheckUnsupportedByTarget;
270		break :b false;
271		}
272		if (options.inherited.stack_check) \|x\| break :b x;
273		if (options.parent) \|p\| break :b p.stack_check;
274		break :b is_safe_mode;
275		};
276
277		const stack_protector: u32 = sp: {
278		const use_zig_backend = options.global.have_zcu or
279		(options.global.any_c_source_files and options.global.c_frontend == .aro);
280		if (use_zig_backend and !target_util.supportsStackProtector(target, zig_backend)) {
281		if (options.inherited.stack_protector) \|x\| {
282		if (x > 0) return error.StackProtectorUnsupportedByTarget;
283		}
284		break :sp 0;
285		}
286
287		if (options.global.any_c_source_files and options.global.c_frontend == .clang and
288		!target_util.clangSupportsStackProtector(target))
289		{
290		if (options.inherited.stack_protector) \|x\| {
291		if (x > 0) return error.StackProtectorUnsupportedByTarget;
292		}
293		break :sp 0;
294		}
295
296		// This logic is checking for linking libc because otherwise our start code
297		// which is trying to set up TLS (i.e. the fs/gs registers) but the stack
298		// protection code depends on fs/gs registers being already set up.
299		// If we were able to annotate start code, or perhaps the entire std lib,
300		// as being exempt from stack protection checks, we could change this logic
301		// to supporting stack protection even when not linking libc.
302		// TODO file issue about this
303		if (!options.global.link_libc) {
304		if (options.inherited.stack_protector) \|x\| {
305		if (x > 0) return error.StackProtectorUnavailableWithoutLibC;
306		}
307		break :sp 0;
308		}
309
310		if (options.inherited.stack_protector) \|x\| break :sp x;
311		if (options.parent) \|p\| break :sp p.stack_protector;
312		if (!is_safe_mode) break :sp 0;
313
314		break :sp target_util.default_stack_protector_buffer_size;
315		};
316
317		const structured_cfg = b: {
318		if (options.inherited.structured_cfg) \|x\| break :b x;
319		if (options.parent) \|p\| break :b p.structured_cfg;
320		// We always want a structured control flow in shaders. This option is
321		// only relevant for OpenCL kernels.
322		break :b switch (target.os.tag) {
323		.opencl => false,
324		else => true,
325		};
326		};
327
328		const no_builtin = b: {
329		if (options.inherited.no_builtin) \|x\| break :b x;
330		if (options.parent) \|p\| break :b p.no_builtin;
331
332		break :b target.cpu.arch.isBpf();
333		};
334
335		const llvm_cpu_features: ?[*:0]const u8 = b: {
336		if (resolved_target.llvm_cpu_features) \|x\| break :b x;
337		if (!options.global.use_llvm) break :b null;
338
339		var buf = std.array_list.Managed(u8).init(arena);
340		var disabled_features = std.array_list.Managed(u8).init(arena);
341		defer disabled_features.deinit();
342
343		// Append disabled features after enabled ones, so that their effects aren't overwritten.
344		for (target.cpu.arch.allFeaturesList()) \|feature\| {
345		if (feature.llvm_name) \|llvm_name\| {
346		// Ignore these until we figure out how to handle the concept of omitting features.
347		// See https://github.com/ziglang/zig/issues/23539
348		if (target_util.isDynamicAMDGCNFeature(target, feature)) continue;
349
350		if (target.cpu.arch.isPowerPC() and @as(std.Target.powerpc.Feature, @enumFromInt(feature.index)) == .@"64bit") continue;
351		if (target.cpu.arch.isX86() and @as(std.Target.x86.Feature, @enumFromInt(feature.index)) == .x32) continue;
352
353		var is_enabled = target.cpu.features.isEnabled(feature.index);
354		if (target.cpu.arch == .s390x and @as(std.Target.s390x.Feature, @enumFromInt(feature.index)) == .backchain) {
355		is_enabled = !omit_frame_pointer;
356		}
357
358		if (is_enabled) {
359		try buf.ensureUnusedCapacity(2 + llvm_name.len);
360		buf.appendAssumeCapacity('+');
361		buf.appendSliceAssumeCapacity(llvm_name);
362		buf.appendAssumeCapacity(',');
363		} else {
364		try disabled_features.ensureUnusedCapacity(2 + llvm_name.len);
365		disabled_features.appendAssumeCapacity('-');
366		disabled_features.appendSliceAssumeCapacity(llvm_name);
367		disabled_features.appendAssumeCapacity(',');
368		}
369		}
370		}
371
372		try buf.appendSlice(disabled_features.items);
373		if (buf.items.len == 0) break :b "";
374		assert(std.mem.endsWith(u8, buf.items, ","));
375		buf.items[buf.items.len - 1] = 0;
376		buf.shrinkAndFree(buf.items.len);
377		break :b buf.items[0 .. buf.items.len - 1 :0].ptr;
378		};
379
380		const mod = try arena.create(Module);
381		mod.* = .{
382		.root = options.paths.root,
383		.root_src_path = options.paths.root_src_path,
384		.fully_qualified_name = options.fully_qualified_name,
385		.resolved_target = .{
386		.result = target.*,
387		.is_native_os = resolved_target.is_native_os,
388		.is_native_abi = resolved_target.is_native_abi,
389		.is_explicit_dynamic_linker = resolved_target.is_explicit_dynamic_linker,
390		.llvm_cpu_features = llvm_cpu_features,
391		},
392		.optimize_mode = optimize_mode,
393		.single_threaded = single_threaded,
394		.error_tracing = error_tracing,
395		.valgrind = valgrind,
396		.pic = pic,
397		.strip = strip,
398		.omit_frame_pointer = omit_frame_pointer,
399		.stack_check = stack_check,
400		.stack_protector = stack_protector,
401		.code_model = code_model,
402		.red_zone = red_zone,
403		.sanitize_c = sanitize_c,
404		.sanitize_thread = sanitize_thread,
405		.fuzz = fuzz,
406		.unwind_tables = unwind_tables,
407		.cc_argv = options.cc_argv,
408		.structured_cfg = structured_cfg,
409		.no_builtin = no_builtin,
410		};
411		return mod;
412		}
413
414		/// All fields correspond to `CreateOptions`.
415		pub const LimitedOptions = struct {
416		root: Compilation.Path,
417		root_src_path: []const u8,
418		fully_qualified_name: []const u8,
419		};
420
421		/// This one can only be used if the Module will only be used for AstGen and earlier in
422		/// the pipeline. Illegal behavior occurs if a limited module touches Sema.
423		pub fn createLimited(gpa: Allocator, options: LimitedOptions) Allocator.Error!*Package.Module {
424		const mod = try gpa.create(Module);
425		mod.* = .{
426		.root = options.root,
427		.root_src_path = options.root_src_path,
428		.fully_qualified_name = options.fully_qualified_name,
429
430		.resolved_target = undefined,
431		.optimize_mode = undefined,
432		.code_model = undefined,
433		.single_threaded = undefined,
434		.error_tracing = undefined,
435		.valgrind = undefined,
436		.pic = undefined,
437		.strip = undefined,
438		.omit_frame_pointer = undefined,
439		.stack_check = undefined,
440		.stack_protector = undefined,
441		.red_zone = undefined,
442		.sanitize_c = undefined,
443		.sanitize_thread = undefined,
444		.fuzz = undefined,
445		.unwind_tables = undefined,
446		.cc_argv = undefined,
447		.structured_cfg = undefined,
448		.no_builtin = undefined,
449		};
450		return mod;
451		}
452
453		/// Does not ensure that the module's root directory exists on-disk; see `Builtin.updateFileOnDisk` for that task.
454		pub fn createBuiltin(arena: Allocator, opts: Builtin, dirs: Compilation.Directories) Allocator.Error!*Module {
455		const sub_path = "b" ++ std.fs.path.sep_str ++ Cache.binToHex(opts.hash());
456		const new = try arena.create(Module);
457		new.* = .{
458		.root = try .fromRoot(arena, dirs, .global_cache, sub_path),
459		.root_src_path = "builtin.zig",
460		.fully_qualified_name = "builtin",
461		.resolved_target = .{
462		.result = opts.target,
463		// These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code.
464		.is_native_os = false,
465		.is_native_abi = false,
466		.is_explicit_dynamic_linker = false,
467		.llvm_cpu_features = null,
468		},
469		.optimize_mode = opts.optimize_mode,
470		.single_threaded = opts.single_threaded,
471		.error_tracing = opts.error_tracing,
472		.valgrind = opts.valgrind,
473		.pic = opts.pic,
474		.strip = opts.strip,
475		.omit_frame_pointer = opts.omit_frame_pointer,
476		.code_model = opts.code_model,
477		.sanitize_thread = opts.sanitize_thread,
478		.fuzz = opts.fuzz,
479		.unwind_tables = opts.unwind_tables,
480		.cc_argv = &.{},
481		// These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code.
482		.stack_check = false,
483		.stack_protector = 0,
484		.red_zone = false,
485		.sanitize_c = .off,
486		.structured_cfg = false,
487		.no_builtin = false,
488		};
489		return new;
490		}
491
492		/// Returns the `Builtin` which forms the contents of `@import("builtin")` for this module.
493		pub fn getBuiltinOptions(m: Module, global: Compilation.Config) Builtin {
494		assert(global.have_zcu);
495		return .{
496		.target = m.resolved_target.result,
497		.zig_backend = target_util.zigBackend(&m.resolved_target.result, global.use_llvm),
498		.output_mode = global.output_mode,
499		.link_mode = global.link_mode,
500		.unwind_tables = m.unwind_tables,
501		.is_test = global.is_test,
502		.single_threaded = m.single_threaded,
503		.link_libc = global.link_libc,
504		.link_libcpp = global.link_libcpp,
505		.optimize_mode = m.optimize_mode,
506		.error_tracing = m.error_tracing,
507		.valgrind = m.valgrind,
508		.sanitize_thread = m.sanitize_thread,
509		.fuzz = m.fuzz,
510		.pic = m.pic,
511		.pie = global.pie,
512		.strip = m.strip,
513		.code_model = m.code_model,
514		.omit_frame_pointer = m.omit_frame_pointer,
515		.wasi_exec_model = global.wasi_exec_model,
516		};
517		}
518
519		const Module = @This();
520		const Package = @import("../Package.zig");
521		const std = @import("std");
522		const Allocator = std.mem.Allocator;
523		const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest;
524		const target_util = @import("../target.zig");
525		const Cache = std.Build.Cache;
526		const Builtin = @import("../Builtin.zig");
527		const assert = std.debug.assert;
528		const Compilation = @import("../Compilation.zig");
529		const File = @import("../Zcu.zig").File;

src/Zcu.zig-1

...	...	@@ -34,7 +34,6 @@ const AstGen = std.zig.AstGen;
34	34	const Sema = @import("Sema.zig");
35	35	const target_util = @import("target.zig");
36	36	const build_options = @import("build_options");
37		const isUpDir = @import("introspect.zig").isUpDir;
38	37	const InternPool = @import("InternPool.zig");
39	38	const Alignment = InternPool.Alignment;
40	39	const AnalUnit = InternPool.AnalUnit;

src/Zcu/PerThread.zig-1

...	...	@@ -23,7 +23,6 @@ const builtin = @import("builtin");
23	23	const dev = @import("../dev.zig");
24	24	const InternPool = @import("../InternPool.zig");
25	25	const AnalUnit = InternPool.AnalUnit;
26		const introspect = @import("../introspect.zig");
27	26	const Module = @import("../Package.zig").Module;
28	27	const Sema = @import("../Sema.zig");
29	28	const target_util = @import("../target.zig");

src/dev.zig-2

...	...	@@ -112,7 +112,6 @@ pub const Env = enum {
112	112	.translate_c_command,
113	113	.fmt_command,
114	114	.jit_command,
115		.fetch_command,
116	115	.init_command,
117	116	.targets_command,
118	117	.version_command,
...	...	@@ -252,7 +251,6 @@ pub const Feature = enum {
252	251	translate_c_command,
253	252	fmt_command,
254	253	jit_command,
255		fetch_command,
256	254	init_command,
257	255	targets_command,
258	256	version_command,

src/introspect.zig deleted-220

...	...	@@ -1,220 +0,0 @@
1		const builtin = @import("builtin");
2
3		const std = @import("std");
4		const Io = std.Io;
5		const Dir = std.Io.Dir;
6		const mem = std.mem;
7		const Allocator = std.mem.Allocator;
8		const Cache = std.Build.Cache;
9		const assert = std.debug.assert;
10
11		const build_options = @import("build_options");
12
13		const Compilation = @import("Compilation.zig");
14		const Package = @import("Package.zig");
15
16		/// Returns the sub_path that worked, or `null` if none did.
17		/// The path of the returned Directory is relative to `base`.
18		/// The handle of the returned Directory is open.
19		fn testZigInstallPrefix(io: Io, base_dir: Io.Dir) ?Cache.Directory {
20		const test_index_file = "std" ++ Dir.path.sep_str ++ "std.zig";
21
22		zig_dir: {
23		// Try lib/zig/std/std.zig
24		const lib_zig = "lib" ++ Dir.path.sep_str ++ "zig";
25		var test_zig_dir = base_dir.openDir(io, lib_zig, .{}) catch break :zig_dir;
26		const file = test_zig_dir.openFile(io, test_index_file, .{}) catch {
27		test_zig_dir.close(io);
28		break :zig_dir;
29		};
30		file.close(io);
31		return .{ .handle = test_zig_dir, .path = lib_zig };
32		}
33
34		// Try lib/std/std.zig
35		var test_zig_dir = base_dir.openDir(io, "lib", .{}) catch return null;
36		const file = test_zig_dir.openFile(io, test_index_file, .{}) catch {
37		test_zig_dir.close(io);
38		return null;
39		};
40		file.close(io);
41		return .{ .handle = test_zig_dir, .path = "lib" };
42		}
43
44		/// Both the directory handle and the path are newly allocated resources which the caller now owns.
45		pub fn findZigLibDir(gpa: Allocator, io: Io) !Cache.Directory {
46		const cwd_path = try getResolvedCwd(io, gpa);
47		defer gpa.free(cwd_path);
48		const self_exe_path = try std.process.executablePathAlloc(io, gpa);
49		defer gpa.free(self_exe_path);
50
51		return findZigLibDirFromSelfExe(gpa, io, cwd_path, self_exe_path);
52		}
53
54		/// Like `std.process.currentPathAlloc`, but also resolves the path with `Dir.path.resolve`. This
55		/// means the path has no repeated separators, no "." or ".." components, and no trailing separator.
56		/// On WASI, "" is returned instead of ".".
57		pub fn getResolvedCwd(io: Io, gpa: Allocator) std.process.CurrentPathAllocError![]u8 {
58		if (builtin.target.os.tag == .wasi) {
59		if (std.debug.runtime_safety) {
60		const cwd = try std.process.currentPathAlloc(io, gpa);
61		defer gpa.free(cwd);
62		assert(mem.eql(u8, cwd, "."));
63		}
64		return "";
65		}
66		const cwd = try std.process.currentPathAlloc(io, gpa);
67		defer gpa.free(cwd);
68		const resolved = try Dir.path.resolve(gpa, &.{cwd});
69		assert(Dir.path.isAbsolute(resolved));
70		return resolved;
71		}
72
73		/// Both the directory handle and the path are newly allocated resources which the caller now owns.
74		pub fn findZigLibDirFromSelfExe(
75		allocator: Allocator,
76		io: Io,
77		/// The return value of `getResolvedCwd`.
78		/// Passed as an argument to avoid pointlessly repeating the call.
79		cwd_path: []const u8,
80		self_exe_path: []const u8,
81		) error{ OutOfMemory, FileNotFound }!Cache.Directory {
82		const cwd = Io.Dir.cwd();
83		var cur_path: []const u8 = self_exe_path;
84		while (Dir.path.dirname(cur_path)) \|dirname\| : (cur_path = dirname) {
85		var base_dir = cwd.openDir(io, dirname, .{}) catch continue;
86		defer base_dir.close(io);
87
88		const sub_directory = testZigInstallPrefix(io, base_dir) orelse continue;
89		const p = try Dir.path.join(allocator, &.{ dirname, sub_directory.path.? });
90		defer allocator.free(p);
91
92		const resolved = try resolvePath(allocator, cwd_path, &.{p});
93		return .{
94		.handle = sub_directory.handle,
95		.path = if (resolved.len == 0) null else resolved,
96		};
97		}
98		return error.FileNotFound;
99		}
100
101		pub fn resolveGlobalCacheDir(arena: Allocator, environ_map: *const std.process.Environ.Map) ![]const u8 {
102		if (std.zig.EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map)) \|value\| return value;
103
104		const app_name = "zig";
105
106		switch (builtin.os.tag) {
107		.wasi => @compileError("on WASI the global cache dir must be resolved with preopens"),
108		.windows => {
109		const local_app_data_dir = std.zig.EnvVar.LOCALAPPDATA.get(environ_map) orelse
110		return error.AppDataDirUnavailable;
111		return Dir.path.join(arena, &.{ local_app_data_dir, app_name });
112		},
113		else => {
114		if (std.zig.EnvVar.XDG_CACHE_HOME.get(environ_map)) \|cache_root\| {
115		if (cache_root.len > 0) {
116		return Dir.path.join(arena, &.{ cache_root, app_name });
117		}
118		}
119		if (std.zig.EnvVar.HOME.get(environ_map)) \|home\| {
120		if (home.len > 0) {
121		return Dir.path.join(arena, &.{ home, ".cache", app_name });
122		}
123		}
124		return error.AppDataDirUnavailable;
125		},
126		}
127		}
128
129		/// Similar to `Dir.path.resolve`, but converts to a cwd-relative path, or, if that would
130		/// start with a relative up-dir (".."), an absolute path based on the cwd. Also, the cwd
131		/// returns the empty string ("") instead of ".".
132		pub fn resolvePath(
133		gpa: Allocator,
134		/// The return value of `getResolvedCwd`.
135		/// Passed as an argument to avoid pointlessly repeating the call.
136		cwd_resolved: []const u8,
137		paths: []const []const u8,
138		) Allocator.Error![]u8 {
139		if (builtin.target.os.tag == .wasi) {
140		assert(mem.eql(u8, cwd_resolved, ""));
141		const res = try Dir.path.resolve(gpa, paths);
142		if (mem.eql(u8, res, ".")) {
143		gpa.free(res);
144		return "";
145		}
146		return res;
147		}
148
149		// Heuristic for a fast path: if no component is absolute and ".." never appears, we just need to resolve `paths`.
150		for (paths) \|p\| {
151		if (Dir.path.isAbsolute(p)) break; // absolute path
152		if (mem.indexOf(u8, p, "..") != null) break; // may contain up-dir
153		} else {
154		// no absolute path, no "..".
155		const res = try Dir.path.resolve(gpa, paths);
156		if (mem.eql(u8, res, ".")) {
157		gpa.free(res);
158		return "";
159		}
160		assert(!Dir.path.isAbsolute(res));
161		assert(!isUpDir(res));
162		return res;
163		}
164
165		// The fast path failed; resolve the whole thing.
166		// Optimization: `paths` often has just one element.
167		const path_resolved = switch (paths.len) {
168		0 => unreachable,
169		1 => try Dir.path.resolve(gpa, &.{ cwd_resolved, paths[0] }),
170		else => r: {
171		const all_paths = try gpa.alloc([]const u8, paths.len + 1);
172		defer gpa.free(all_paths);
173		all_paths[0] = cwd_resolved;
174		@memcpy(all_paths[1..], paths);
175		break :r try Dir.path.resolve(gpa, all_paths);
176		},
177		};
178		errdefer gpa.free(path_resolved);
179
180		assert(Dir.path.isAbsolute(path_resolved));
181		assert(Dir.path.isAbsolute(cwd_resolved));
182
183		if (!std.mem.startsWith(u8, path_resolved, cwd_resolved)) return path_resolved; // not in cwd
184		if (path_resolved.len == cwd_resolved.len) {
185		// equal to cwd
186		gpa.free(path_resolved);
187		return "";
188		}
189		if (path_resolved[cwd_resolved.len] != Dir.path.sep) return path_resolved; // not in cwd (last component differs)
190
191		// in cwd; extract sub path
192		const sub_path = try gpa.dupe(u8, path_resolved[cwd_resolved.len + 1 ..]);
193		gpa.free(path_resolved);
194		return sub_path;
195		}
196
197		pub fn isUpDir(p: []const u8) bool {
198		return mem.startsWith(u8, p, "..") and (p.len == 2 or p[2] == Dir.path.sep);
199		}
200
201		pub const default_local_zig_cache_basename = ".zig-cache";
202
203		/// Searches upwards from `cwd` for a directory containing a `build.zig` file.
204		/// If such a directory is found, returns the path to it joined to the `.zig_cache` name.
205		/// Otherwise, returns `null`, indicating no suitable local cache location.
206		pub fn resolveSuitableLocalCacheDir(arena: Allocator, io: Io, cwd: []const u8) Allocator.Error!?[]u8 {
207		var cur_dir = cwd;
208		while (true) {
209		const joined = try Dir.path.join(arena, &.{ cur_dir, Package.build_zig_basename });
210		if (Io.Dir.cwd().access(io, joined, .{})) \|_\| {
211		return try Dir.path.join(arena, &.{ cur_dir, default_local_zig_cache_basename });
212		} else \|err\| switch (err) {
213		error.FileNotFound => {
214		cur_dir = Dir.path.dirname(cur_dir) orelse return null;
215		continue;
216		},
217		else => return null,
218		}
219		}
220		}

src/main.zig+40-1648

...	...	@@ -21,13 +21,13 @@ const AstGen = std.zig.AstGen;
21	21	const ZonGen = std.zig.ZonGen;
22	22	const Server = std.zig.Server;
23	23	const stringToEnum = std.meta.stringToEnum;
	24	const allocPrint = std.fmt.allocPrint;
24	25
25	26	pub const tracy = @import("tracy.zig");
26	27	const Compilation = @import("Compilation.zig");
27	28	const link = @import("link.zig");
28	29	const Package = @import("Package.zig");
29	30	const build_options = @import("build_options");
30		const introspect = @import("introspect.zig");
31	31	const wasi_libc = @import("libs/wasi_libc.zig");
32	32	const target_util = @import("target.zig");
33	33	const crash_report = @import("crash_report.zig");
...	...	@@ -353,9 +353,15 @@ fn mainArgs(
353	353	dev.check(.ar_command);
354	354	return process.exit(try llvmArMain(arena, args));
355	355	},
356		.build => {
357		dev.check(.build_command);
358		return cmdBuild(gpa, arena, io, cmd_args, environ_map);
	356	.build, .fetch => {
	357	return jitCmd(gpa, arena, io, args, environ_map, .{
	358	.cmd_name = "maker",
	359	.root_src_path = "Maker.zig",
	360	.prepend_zig_lib_dir_path = true,
	361	.prepend_global_cache_path = true,
	362	.prepend_zig_exe_path = true,
	363	.prepend_seed = true,
	364	});
359	365	},
360	366	.clang, .@"-cc1", .@"-cc1as" => {
361	367	dev.check(.clang_command);
...	...	@@ -385,7 +391,6 @@ fn mainArgs(
385	391	.depend_on_aro = true,
386	392	.prepend_zig_lib_dir_path = true,
387	393	.server = use_server,
388		.color = Color.settingFromEnvironment(environ_map),
389	394	});
390	395	},
391	396	.fmt => {
...	...	@@ -396,25 +401,19 @@ fn mainArgs(
396	401	return jitCmd(gpa, arena, io, cmd_args, environ_map, .{
397	402	.cmd_name = "objcopy",
398	403	.root_src_path = "objcopy.zig",
399		.color = Color.settingFromEnvironment(environ_map),
400	404	});
401	405	},
402	406	.objdump => {
403	407	return jitCmd(gpa, arena, io, cmd_args, environ_map, .{
404	408	.cmd_name = "objdump",
405	409	.root_src_path = "objdump.zig",
406		.color = Color.settingFromEnvironment(environ_map),
407	410	});
408	411	},
409		.fetch => {
410		return cmdFetch(gpa, arena, io, cmd_args, environ_map);
411		},
412	412	.libc => {
413	413	return jitCmd(gpa, arena, io, cmd_args, environ_map, .{
414	414	.cmd_name = "libc",
415	415	.root_src_path = "libc.zig",
416	416	.prepend_zig_lib_dir_path = true,
417		.color = Color.settingFromEnvironment(environ_map),
418	417	});
419	418	},
420	419	.std => {
...	...	@@ -424,7 +423,6 @@ fn mainArgs(
424	423	.prepend_zig_lib_dir_path = true,
425	424	.prepend_zig_exe_path = true,
426	425	.prepend_global_cache_path = true,
427		.color = Color.settingFromEnvironment(environ_map),
428	426	});
429	427	},
430	428	.init => {
...	...	@@ -461,7 +459,6 @@ fn mainArgs(
461	459	return jitCmd(gpa, arena, io, cmd_args, environ_map, .{
462	460	.cmd_name = "reduce",
463	461	.root_src_path = "reduce.zig",
464		.color = Color.settingFromEnvironment(environ_map),
465	462	});
466	463	},
467	464	.zen => {
...	...	@@ -2977,7 +2974,7 @@ fn buildOutputType(
2977	2974	while (preprocessor_args_it.next()) \|arg\| {
2978	2975	if (mem.eql(u8, arg, "-MD") or mem.eql(u8, arg, "-MMD") or mem.eql(u8, arg, "-MT")) {
2979	2976	disable_c_depfile = true;
2980		const cc_arg = try std.fmt.allocPrint(arena, "-Wp,{s},{s}", .{ arg, preprocessor_args_it.nextOrFatal() });
	2977	const cc_arg = try allocPrint(arena, "-Wp,{s},{s}", .{ arg, preprocessor_args_it.nextOrFatal() });
2981	2978	try cc_argv.append(arena, cc_arg);
2982	2979	} else {
2983	2980	fatal("unsupported preprocessor arg: {s}", .{arg});
...	...	@@ -3222,7 +3219,7 @@ fn buildOutputType(
3222	3219	else => process.executablePathAlloc(io, arena) catch \|err\| fatal("unable to find zig self exe path: {t}", .{err}),
3223	3220	};
3224	3221
3225		const cwd_path = try introspect.getResolvedCwd(io, arena);
	3222	const cwd_path = try std.zig.getResolvedCwd(io, arena);
3226	3223
3227	3224	// This `init` calls `fatal` on error.
3228	3225	var dirs: Compilation.Directories = .init(
...	...	@@ -3421,9 +3418,9 @@ fn buildOutputType(
3421	3418	.yes_default_value => if (create_module.resolved_options.output_mode == .Lib and
3422	3419	create_module.resolved_options.link_mode == .dynamic and target.ofmt == .elf)
3423	3420	if (have_version)
3424		try std.fmt.allocPrint(arena, "lib{s}.so.{d}", .{ root_name, version.major })
	3421	try allocPrint(arena, "lib{s}.so.{d}", .{ root_name, version.major })
3425	3422	else
3426		try std.fmt.allocPrint(arena, "lib{s}.so", .{root_name})
	3423	try allocPrint(arena, "lib{s}.so", .{root_name})
3427	3424	else
3428	3425	null,
3429	3426	};
...	...	@@ -3433,7 +3430,7 @@ fn buildOutputType(
3433	3430	.yes_default_path => emit: {
3434	3431	if (output_to_cache != null) break :emit .yes_cache;
3435	3432	const name = switch (clang_preprocessor_mode) {
3436		.pch => try std.fmt.allocPrint(arena, "{s}.pch", .{root_name}),
	3433	.pch => try allocPrint(arena, "{s}.pch", .{root_name}),
3437	3434	else => try std.zig.binNameAlloc(arena, .{
3438	3435	.root_name = root_name,
3439	3436	.cpu_arch = target.cpu.arch,
...	...	@@ -3469,16 +3466,16 @@ fn buildOutputType(
3469	3466	},
3470	3467	};
3471	3468
3472		const default_h_basename = try std.fmt.allocPrint(arena, "{s}.h", .{root_name});
	3469	const default_h_basename = try allocPrint(arena, "{s}.h", .{root_name});
3473	3470	const emit_h_resolved = emit_h.resolve(io, default_h_basename, output_to_cache);
3474	3471
3475		const default_asm_basename = try std.fmt.allocPrint(arena, "{s}.s", .{root_name});
	3472	const default_asm_basename = try allocPrint(arena, "{s}.s", .{root_name});
3476	3473	const emit_asm_resolved = emit_asm.resolve(io, default_asm_basename, output_to_cache);
3477	3474
3478		const default_llvm_ir_basename = try std.fmt.allocPrint(arena, "{s}.ll", .{root_name});
	3475	const default_llvm_ir_basename = try allocPrint(arena, "{s}.ll", .{root_name});
3479	3476	const emit_llvm_ir_resolved = emit_llvm_ir.resolve(io, default_llvm_ir_basename, output_to_cache);
3480	3477
3481		const default_llvm_bc_basename = try std.fmt.allocPrint(arena, "{s}.bc", .{root_name});
	3478	const default_llvm_bc_basename = try allocPrint(arena, "{s}.bc", .{root_name});
3482	3479	const emit_llvm_bc_resolved = emit_llvm_bc.resolve(io, default_llvm_bc_basename, output_to_cache);
3483	3480
3484	3481	const emit_docs_resolved = emit_docs.resolve(io, "docs", output_to_cache);
...	...	@@ -3499,7 +3496,7 @@ fn buildOutputType(
3499	3496	fatal("the argument -femit-implib is allowed only when building a Windows DLL", .{});
3500	3497	}
3501	3498	}
3502		const default_implib_basename = try std.fmt.allocPrint(arena, "{s}.lib", .{root_name});
	3499	const default_implib_basename = try allocPrint(arena, "{s}.lib", .{root_name});
3503	3500	const emit_implib_resolved: Compilation.CreateOptions.Emit = switch (emit_implib) {
3504	3501	.no => .no,
3505	3502	.yes => emit_implib.resolve(io, default_implib_basename, output_to_cache),
...	...	@@ -3528,7 +3525,7 @@ fn buildOutputType(
3528	3525
3529	3526	// "-" is stdin. Dump it to a real file.
3530	3527	const sep = fs.path.sep_str;
3531		const dump_path = try std.fmt.allocPrint(arena, "tmp" ++ sep ++ "{x}-dump-stdin{s}", .{
	3528	const dump_path = try allocPrint(arena, "tmp" ++ sep ++ "{x}-dump-stdin{s}", .{
3532	3529	randInt(io, u64), ext.canonicalName(target),
3533	3530	});
3534	3531	try dirs.local_cache.handle.createDirPath(io, "tmp");
...	...	@@ -3557,7 +3554,7 @@ fn buildOutputType(
3557	3554
3558	3555	const bin_digest: Cache.BinDigest = hasher.hasher.finalResult();
3559	3556
3560		const sub_path = try std.fmt.allocPrint(arena, "tmp" ++ sep ++ "{x}-stdin{s}", .{
	3557	const sub_path = try allocPrint(arena, "tmp" ++ sep ++ "{x}-stdin{s}", .{
3561	3558	&bin_digest, ext.canonicalName(target),
3562	3559	});
3563	3560	try dirs.local_cache.handle.rename(dump_path, dirs.local_cache.handle, sub_path, io);
...	...	@@ -4586,7 +4583,7 @@ fn runOrTest(
4586	4583	try argv.append(exe_path);
4587	4584	if (arg_mode == .zig_test) {
4588	4585	try argv.append(
4589		try std.fmt.allocPrint(arena, "--seed=0x{x}", .{randInt(io, u32)}),
	4586	try allocPrint(arena, "--seed=0x{x}", .{randInt(io, u32)}),
4590	4587	);
4591	4588	}
4592	4589	} else {
...	...	@@ -4794,7 +4791,7 @@ fn cmdTranslateC(
4794	4791	assert(comp.c_source_files.len == 1);
4795	4792	const c_source_file = comp.c_source_files[0];
4796	4793
4797		const translated_basename = try std.fmt.allocPrint(arena, "{s}.zig", .{comp.root_name});
	4794	const translated_basename = try allocPrint(arena, "{s}.zig", .{comp.root_name});
4798	4795
4799	4796	var man: Cache.Manifest = comp.obtainCObjectCacheManifest(comp.root_mod);
4800	4797	man.want_shared_lock = false;
...	...	@@ -4872,7 +4869,6 @@ pub fn translateC(
4872	4869	.root_src_path = "translate-c/main.zig",
4873	4870	.depend_on_aro = true,
4874	4871	.capture = capture,
4875		.color = Color.settingFromEnvironment(environ_map),
4876	4872	});
4877	4873	}
4878	4874
...	...	@@ -4912,7 +4908,7 @@ fn cmdInit(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8) !
4912	4908	}
4913	4909	}
4914	4910
4915		const cwd_path = try introspect.getResolvedCwd(io, arena);
	4911	const cwd_path = try std.zig.getResolvedCwd(io, arena);
4916	4912	const cwd_basename = fs.path.basename(cwd_path);
4917	4913	const sanitized_root_name = try sanitizeExampleName(arena, cwd_basename);
4918	4914
...	...	@@ -5027,1065 +5023,17 @@ test sanitizeExampleName {
5027	5023	try std.testing.expectEqualStrings("test_project", try sanitizeExampleName(arena, "test project"));
5028	5024	}
5029	5025
5030		fn cmdBuild(
5031		gpa: Allocator,
5032		arena: Allocator,
5033		io: Io,
5034		args: []const []const u8,
5035		environ_map: *process.Environ.Map,
5036		) !void {
5037		var build_file: ?[]const u8 = null;
5038		var override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map);
5039		var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
5040		var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map);
5041		var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map);
5042		var maker_optimize_mode: std.builtin.OptimizeMode = if (EnvVar.ZIG_DEBUG_CMD.isSet(environ_map))
5043		.Debug
5044		else
5045		.ReleaseSafe;
5046		var configure_argv: std.ArrayList([]const u8) = .empty;
5047		var make_argv: std.ArrayList([]const u8) = .empty;
5048		var cached_passthru_configure: std.ArrayList(u32) = .empty;
5049		var forks: std.ArrayList(Fork) = .empty;
5050		var reference_trace: ?u32 = null;
5051		var debug_compile_errors = false;
5052		var verbose_link = (native_os != .wasi or builtin.link_libc) and
5053		EnvVar.ZIG_VERBOSE_LINK.isSet(environ_map);
5054		var verbose_cc = (native_os != .wasi or builtin.link_libc) and
5055		EnvVar.ZIG_VERBOSE_CC.isSet(environ_map);
5056		var verbose_air = false;
5057		var verbose_intern_pool = false;
5058		var verbose_generic_instances = false;
5059		var verbose_llvm_ir: ?[]const u8 = null;
5060		var verbose_llvm_bc: ?[]const u8 = null;
5061		var verbose_llvm_cpu_features = false;
5062		var fetch_only = false;
5063		var fetch_mode: Package.Fetch.JobQueue.Mode = .needed;
5064		var system_pkg_dir_path: ?[]const u8 = null;
5065		var debug_target: ?[]const u8 = null;
5066		var debug_libc_paths_file: ?[]const u8 = null;
5067		var cache_poison: std.Build.Graph.CachePoison = .pure;
5068		var print_configuration_path: bool = false;
5069
5070		const self_exe_path = try process.executablePathAlloc(io, arena);
5071		const default_seed = try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)});
5072
5073		try configure_argv.ensureUnusedCapacity(arena, 16);
5074		try make_argv.ensureUnusedCapacity(arena, 16);
5075		try cached_passthru_configure.ensureUnusedCapacity(arena, 16);
5076
5077		_ = configure_argv.addOneAssumeCapacity(); // configurer executable
5078		_ = make_argv.addOneAssumeCapacity(); // maker executable
5079
5080		make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", self_exe_path };
5081		configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", self_exe_path };
5082
5083		make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig-lib-dir", undefined };
5084		const make_argv_index_zig_lib_dir = make_argv.items.len - 1;
5085
5086		make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined };
5087		const make_argv_index_build_root = make_argv.items.len - 1;
5088
5089		make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--local-cache", undefined };
5090		const make_argv_index_cache_dir = make_argv.items.len - 1;
5091
5092		make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--global-cache", undefined };
5093		const make_argv_index_global_cache_dir = make_argv.items.len - 1;
5094
5095		make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--configuration", undefined };
5096		const argv_index_configuration_file = make_argv.items.len - 1;
5097
5098		make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--seed", default_seed };
5099		const argv_index_seed = make_argv.items.len - 1;
5100
5101		configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined };
5102		const conf_argv_index_build_root = configure_argv.items.len - 1;
5103
5104		var color: Color = Color.settingFromEnvironment(environ_map);
5105		var n_jobs: ?u32 = null;
5106
5107		{
5108		var i: usize = 0;
5109		while (i < args.len) : (i += 1) {
5110		const arg = args[i];
5111		if (mem.startsWith(u8, arg, "-")) {
5112		try configure_argv.ensureUnusedCapacity(arena, 2);
5113
5114		if (mem.startsWith(u8, arg, "-D") or
5115		mem.startsWith(u8, arg, "-fsys=") or
5116		mem.startsWith(u8, arg, "-fno-sys=") or
5117		mem.startsWith(u8, arg, "--release=") or
5118		mem.eql(u8, arg, "--release"))
5119		{
5120		try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len));
5121		configure_argv.appendAssumeCapacity(arg);
5122		continue;
5123		} else if (mem.eql(u8, arg, "--system")) {
5124		if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg});
5125		i += 1;
5126		system_pkg_dir_path = args[i];
5127
5128		try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len));
5129		configure_argv.appendAssumeCapacity(arg); // Intentionally "--system" only; not the path.
5130		continue;
5131		} else if (mem.cutPrefix(u8, arg, "--color=")) \|rest\| {
5132		color = stringToEnum(Color, rest) orelse
5133		fatal("expected --color=[auto\|on\|off]; found {q}", .{arg});
5134
5135		try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len));
5136		configure_argv.appendAssumeCapacity(arg);
5137		continue;
5138		} else if (mem.eql(u8, arg, "--cache-poison")) {
5139		cache_poison = .poisoned;
5140		configure_argv.appendAssumeCapacity("--cache-poison=poisoned");
5141		continue;
5142		} else if (mem.cutPrefix(u8, arg, "--cache-poison=")) \|rest\| {
5143		// Allow the configurer process to report parse failure.
5144		if (stringToEnum(std.Build.Graph.CachePoison, rest)) \|poison\| {
5145		cache_poison = poison;
5146		}
5147		configure_argv.appendAssumeCapacity(arg);
5148		continue;
5149		} else if (mem.eql(u8, arg, "--verbose")) {
5150		// Intentionally is added both to make and configure but
5151		// does not go into the cache hash.
5152		configure_argv.appendAssumeCapacity(arg);
5153		} else if (mem.eql(u8, arg, "--search-prefix")) {
5154		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5155		i += 1;
5156		// This argument is cache poisonous: it does not go into
5157		// the cache and configurer must set the poison bit when
5158		// choosing to observe it.
5159		configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ arg, args[i] };
5160		(try make_argv.addManyAsArray(arena, 2)).* = .{ arg, args[i] };
5161		continue;
5162		} else if (mem.eql(u8, arg, "--build-file")) {
5163		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5164		i += 1;
5165		build_file = args[i];
5166		continue;
5167		} else if (mem.eql(u8, arg, "--zig-lib-dir")) {
5168		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5169		i += 1;
5170		override_lib_dir = args[i];
5171		continue;
5172		} else if (mem.eql(u8, arg, "--cache-dir")) {
5173		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5174		i += 1;
5175		override_local_cache_dir = args[i];
5176		continue;
5177		} else if (mem.eql(u8, arg, "--pkg-dir")) {
5178		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5179		i += 1;
5180		override_pkg_dir = args[i];
5181		continue;
5182		} else if (mem.eql(u8, arg, "--global-cache-dir")) {
5183		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5184		i += 1;
5185		override_global_cache_dir = args[i];
5186		continue;
5187		} else if (mem.eql(u8, arg, "--print-configuration-path")) {
5188		print_configuration_path = true;
5189		continue;
5190		} else if (mem.eql(u8, arg, "-freference-trace")) {
5191		reference_trace = 256;
5192		} else if (mem.eql(u8, arg, "--fetch")) {
5193		fetch_only = true;
5194		} else if (mem.cutPrefix(u8, arg, "--fetch=")) \|sub_arg\| {
5195		fetch_only = true;
5196		fetch_mode = stringToEnum(Package.Fetch.JobQueue.Mode, sub_arg) orelse
5197		fatal("expected [needed\|all] after \"--fetch=\", found: {s}", .{sub_arg});
5198		} else if (mem.cutPrefix(u8, arg, "--fork=")) \|sub_arg\| {
5199		try forks.append(arena, .init(sub_arg));
5200		continue;
5201		} else if (mem.eql(u8, arg, "--fork")) {
5202		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5203		i += 1;
5204		try forks.append(arena, .init(args[i]));
5205		continue;
5206		} else if (mem.cutPrefix(u8, arg, "-freference-trace=")) \|num\| {
5207		reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch \|err\| {
5208		fatal("unable to parse reference_trace count {q}: {t}", .{ num, err });
5209		};
5210		} else if (mem.eql(u8, arg, "-fno-reference-trace")) {
5211		reference_trace = null;
5212		} else if (mem.cutPrefix(u8, arg, "--maker-opt=")) \|rest\| {
5213		maker_optimize_mode = parseOptimizeMode(rest);
5214		continue;
5215		} else if (mem.eql(u8, arg, "--debug-log")) {
5216		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
5217		try make_argv.appendSlice(arena, args[i .. i + 2]);
5218		i += 1;
5219		try addDebugLog(arena, args[i]);
5220		continue;
5221		} else if (mem.eql(u8, arg, "--debug-compile-errors")) {
5222		if (build_options.enable_debug_extensions) {
5223		debug_compile_errors = true;
5224		} else {
5225		warn("Zig was compiled without debug extensions. --debug-compile-errors has no effect.", .{});
5226		}
5227		} else if (mem.eql(u8, arg, "--debug-target")) {
5228		if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg});
5229		i += 1;
5230		if (build_options.enable_debug_extensions) {
5231		debug_target = args[i];
5232		} else {
5233		warn("Zig was compiled without debug extensions. --debug-target has no effect.", .{});
5234		}
5235		continue;
5236		} else if (mem.eql(u8, arg, "--debug-libc")) {
5237		if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg});
5238		i += 1;
5239		if (build_options.enable_debug_extensions) {
5240		debug_libc_paths_file = args[i];
5241		} else {
5242		warn("Zig was compiled without debug extensions. --debug-libc has no effect.", .{});
5243		}
5244		continue;
5245		} else if (mem.eql(u8, arg, "--verbose-link")) {
5246		verbose_link = true;
5247		} else if (mem.eql(u8, arg, "--verbose-cc")) {
5248		verbose_cc = true;
5249		} else if (mem.eql(u8, arg, "--verbose-air")) {
5250		verbose_air = true;
5251		} else if (mem.eql(u8, arg, "--verbose-intern-pool")) {
5252		verbose_intern_pool = true;
5253		} else if (mem.eql(u8, arg, "--verbose-generic-instances")) {
5254		verbose_generic_instances = true;
5255		} else if (mem.eql(u8, arg, "--verbose-llvm-ir")) {
5256		verbose_llvm_ir = "-";
5257		} else if (mem.cutPrefix(u8, arg, "--verbose-llvm-ir=")) \|rest\| {
5258		verbose_llvm_ir = rest;
5259		} else if (mem.cutPrefix(u8, arg, "--verbose-llvm-bc=")) \|rest\| {
5260		verbose_llvm_bc = rest;
5261		} else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) {
5262		verbose_llvm_cpu_features = true;
5263		} else if (mem.cutPrefix(u8, arg, "-j")) \|str\| {
5264		const num = std.fmt.parseUnsigned(u32, str, 10) catch \|err\|
5265		fatal("unable to parse jobs count {s}: {t}", .{ str, err });
5266		if (num < 1) {
5267		fatal("number of jobs must be at least 1", .{});
5268		}
5269		n_jobs = num;
5270		} else if (mem.eql(u8, arg, "--seed")) {
5271		if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg});
5272		i += 1;
5273		make_argv.items[argv_index_seed] = args[i];
5274		continue;
5275		} else if (mem.eql(u8, arg, "--")) {
5276		try make_argv.appendSlice(arena, args[i..]);
5277		break;
5278		}
5279		}
5280		try make_argv.append(arena, arg);
5281		}
5282		}
5283
5284		const root_prog_node = std.Progress.start(io, .{
5285		.disable_printing = (color == .off),
5286		.root_name = "",
5287		});
5288		defer root_prog_node.end();
5289
5290		process.raiseFileDescriptorLimit();
5291
5292		const cwd_path = introspect.getResolvedCwd(io, arena) catch \|err\|
5293		fatal("failed to get current directory path: {t}", .{err});
5294
5295		const build_root = try findBuildRoot(arena, io, .{
5296		.cwd_path = cwd_path,
5297		.build_file = build_file,
5298		});
5299
5300		{
5301		// This `init` calls `fatal` on error.
5302		var dirs: Compilation.Directories = .init(
5303		arena,
5304		io,
5305		override_lib_dir,
5306		override_global_cache_dir,
5307		.{ .override = path: {
5308		if (override_local_cache_dir) \|d\| break :path d;
5309		break :path try build_root.directory.join(arena, &.{introspect.default_local_zig_cache_basename});
5310		} },
5311		.empty,
5312		self_exe_path,
5313		environ_map,
5314		cwd_path,
5315		);
5316		defer dirs.deinit(io);
5317
5318		const thread_limit = @min(
5319		@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1),
5320		std.math.maxInt(Zcu.PerThread.IdBacking),
5321		);
5322		try setThreadLimit(arena, thread_limit);
5323
5324		// Cache lookup for configure options. If we get a match, we can skip
5325		// execution of the configure script. If not, we get the file path to pass
5326		// to the configure process.
5327		var local_cache: Cache = .{
5328		.gpa = gpa,
5329		.io = io,
5330		.manifest_dir = try dirs.local_cache.handle.createDirPathOpen(io, "h", .{}),
5331		.cwd = cwd_path,
5332		};
5333		local_cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() });
5334		local_cache.addPrefix(dirs.zig_lib);
5335		local_cache.addPrefix(dirs.local_cache);
5336		local_cache.addPrefix(dirs.global_cache);
5337		defer local_cache.manifest_dir.close(io);
5338
5339		var config_man = local_cache.obtain();
5340		defer config_man.deinit();
5341		config_man.hash.addBytes(build_options.version);
5342
5343		for (cached_passthru_configure.items) \|i\|
5344		config_man.hash.addBytes(configure_argv.items[i]);
5345
5346		// Prevents a `zig build` from getting a false positive cache hit following
5347		// a `zig build --cache-poison=ignored`.
5348		config_man.hash.add(cache_poison == .ignored);
5349
5350		// Normally the build runner is compiled for the host target but here is
5351		// some code to help when debugging edits to the build runner so that you
5352		// can make sure it compiles successfully on other targets.
5353		const resolved_target: Package.Module.ResolvedTarget = t: {
5354		if (build_options.enable_debug_extensions) {
5355		if (debug_target) \|triple\| {
5356		const target_query = try std.Target.Query.parse(.{
5357		.arch_os_abi = triple,
5358		});
5359		config_man.hash.addBytes(triple);
5360		break :t .{
5361		.result = std.zig.resolveTargetQueryOrFatal(io, target_query),
5362		.is_native_os = false,
5363		.is_native_abi = false,
5364		.is_explicit_dynamic_linker = false,
5365		};
5366		}
5367		}
5368		break :t .{
5369		.result = std.zig.resolveTargetQueryOrFatal(io, .{}),
5370		.is_native_os = true,
5371		.is_native_abi = true,
5372		.is_explicit_dynamic_linker = false,
5373		};
5374		};
5375
5376		// Likewise, `--debug-libc` allows overriding the libc installation.
5377		const libc_installation: ?*const LibCInstallation = lci: {
5378		const paths_file = debug_libc_paths_file orelse break :lci null;
5379		if (!build_options.enable_debug_extensions) unreachable;
5380		const lci = try arena.create(LibCInstallation);
5381		lci.* = try .parse(arena, io, paths_file, &resolved_target.result);
5382		LibCInstallation.addToHash(lci, &config_man.hash, resolved_target.result.abi);
5383		break :lci lci;
5384		};
5385
5386		// Kick off an optimized compilation of the make runner.
5387		var make_runner_task = if (print_configuration_path) undefined else io.async(compileMakeRunner, .{ gpa, arena, io, .{
5388		.dirs = .{
5389		.cwd = dirs.cwd,
5390		.zig_lib = dirs.zig_lib,
5391		.global_cache = dirs.global_cache,
5392		.local_cache = dirs.global_cache,
5393		},
5394		.environ_map = environ_map,
5395		.parent_prog_node = root_prog_node,
5396		.resolved_target = resolved_target,
5397		.libc_installation = libc_installation,
5398		.thread_limit = thread_limit,
5399		.self_exe_path = self_exe_path,
5400		.color = color,
5401		.reference_trace = reference_trace,
5402		.optimize_mode = maker_optimize_mode,
5403		} });
5404		defer _ = if (!print_configuration_path) make_runner_task.cancel(io) catch {};
5405
5406		const pkg_root: Path = if (override_pkg_dir) \|p\|
5407		.initCwd(p)
5408		else if (system_pkg_dir_path) \|p\|
5409		.initCwd(p)
5410		else
5411		.{
5412		.root_dir = build_root.directory,
5413		.sub_path = "zig-pkg",
5414		};
5415
5416		make_argv.items[make_argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path;
5417		make_argv.items[make_argv_index_build_root] = build_root.directory.path orelse cwd_path;
5418		make_argv.items[make_argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path;
5419		make_argv.items[make_argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path;
5420
5421		configure_argv.items[conf_argv_index_build_root] = build_root.directory.path orelse cwd_path;
5422
5423		// Dummy http client that is not actually used when fetch_command is unsupported.
5424		// Prevents bootstrap from depending on a bunch of unnecessary stuff.
5425		var http_client: if (dev.env.supports(.fetch_command)) std.http.Client else struct {
5426		allocator: Allocator,
5427		io: Io,
5428		fn deinit(_: @This()) void {}
5429		} = .{ .allocator = gpa, .io = io };
5430		defer http_client.deinit();
5431
5432		var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{};
5433		var fork_set: Package.Fetch.JobQueue.ForkSet = .{};
5434
5435		{
5436		// Populate fork_set.
5437		var group: Io.Group = .init;
5438		defer group.cancel(io);
5439
5440		for (forks.items) \|*fork\|
5441		group.async(io, Fork.load, .{ io, gpa, fork, color });
5442
5443		try group.await(io);
5444
5445		for (forks.items) \|*fork\| {
5446		if (fork.failed) process.exit(1);
5447		try fork_set.put(arena, .{
5448		.path = fork.path,
5449		.manifest_ast = fork.manifest_ast,
5450		.manifest = fork.manifest,
5451		.uses = 0,
5452		}, {});
5453		}
5454		}
5455		defer Fork.deinitList(forks.items);
5456
5457		var file_system_inputs: std.ArrayList(u8) = .empty;
5458		defer file_system_inputs.deinit(gpa);
5459
5460		// This loop is re-evaluated when the build script exits with an indication that it
5461		// could not continue due to missing lazy dependencies.
5462		const configuration_path: Path, const poisoned: bool = cp: while (true) {
5463		// We want to release all the locks before executing the child process, so we make a nice
5464		// big block here to ensure the cleanup gets run when we extract out our argv.
5465		{
5466		const main_mod_paths: Package.Module.CreateOptions.Paths = .{
5467		.root = try .fromRoot(arena, dirs, .zig_lib, "compiler"),
5468		.root_src_path = "configurer.zig",
5469		};
5470
5471		const config = try Compilation.Config.resolve(.{
5472		.output_mode = .Exe,
5473		.resolved_target = resolved_target,
5474		.have_zcu = true,
5475		.emit_bin = true,
5476		.is_test = false,
5477		});
5478
5479		const root_mod = try Package.Module.create(arena, .{
5480		.paths = main_mod_paths,
5481		.fully_qualified_name = "root",
5482		.cc_argv = &.{},
5483		.inherited = .{
5484		.resolved_target = resolved_target,
5485		.single_threaded = true,
5486		},
5487		.global = config,
5488		.parent = null,
5489		});
5490
5491		const build_mod = try Package.Module.create(arena, .{
5492		.paths = .{
5493		.root = try .fromUnresolved(arena, dirs, &.{build_root.directory.path orelse "."}),
5494		.root_src_path = build_root.build_zig_basename,
5495		},
5496		.fully_qualified_name = "root.@build",
5497		.cc_argv = &.{},
5498		.inherited = .{},
5499		.global = config,
5500		.parent = root_mod,
5501		});
5502
5503		if (dev.env.supports(.fetch_command)) {
5504		const fetch_prog_node = root_prog_node.start("Fetch Packages", 0);
5505		defer fetch_prog_node.end();
5506
5507		// Reset fork match counts.
5508		for (fork_set.keys()) \|*fork\| fork.uses = 0;
5509
5510		var job_queue: Package.Fetch.JobQueue = .{
5511		.io = io,
5512		.http_client = &http_client,
5513		.global_cache = dirs.global_cache,
5514		.local_storage = &.{
5515		.cache_root = .{ .root_dir = dirs.local_cache, .sub_path = "" },
5516		.pkg_root = pkg_root,
5517		},
5518		.recursive = true,
5519		.debug_hash = false,
5520		.unlazy_set = unlazy_set,
5521		.fork_set = fork_set,
5522		.mode = fetch_mode,
5523		.prog_node = fetch_prog_node,
5524		.read_only = system_pkg_dir_path != null,
5525		};
5526		defer job_queue.deinit();
5527
5528		if (system_pkg_dir_path == null) {
5529		try http_client.initDefaultProxies(arena, environ_map);
5530		}
5531
5532		try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1);
5533		try job_queue.table.ensureUnusedCapacity(gpa, 1);
5534
5535		const phantom_package_root: Cache.Path = .{ .root_dir = build_root.directory };
5536
5537		var fetch: Package.Fetch = .{
5538		.arena = std.heap.ArenaAllocator.init(gpa),
5539		.location = .{ .relative_path = phantom_package_root },
5540		.location_tok = 0,
5541		.hash_tok = .none,
5542		.name_tok = 0,
5543		.lazy_status = .eager,
5544		.remote_package_root = phantom_package_root,
5545		.parent_package_root = phantom_package_root,
5546		.parent_manifest_ast = null,
5547		.prog_node = fetch_prog_node,
5548		.job_queue = &job_queue,
5549		.omit_missing_hash_error = true,
5550		.allow_missing_paths_field = false,
5551		.use_latest_commit = false,
5552
5553		.package_root = undefined,
5554		.error_bundle = undefined,
5555		.manifest = undefined,
5556		.manifest_ast = undefined,
5557		.have_manifest = false,
5558		.computed_hash = undefined,
5559		.has_build_zig = true,
5560		.oom_flag = false,
5561		.latest_commit = null,
5562
5563		.module = build_mod,
5564		};
5565
5566		job_queue.all_fetches.appendAssumeCapacity(&fetch);
5567
5568		job_queue.table.putAssumeCapacityNoClobber(
5569		Package.Fetch.relativePathDigest(phantom_package_root, dirs.global_cache),
5570		&fetch,
5571		);
5572
5573		job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" });
5574		try job_queue.group.await(io);
5575
5576		{
5577		// Ensure that forks were actually used. This is done
5578		// before printing manifest errors because using a fork can
5579		// prevent them.
5580		var any_unused = false;
5581		for (fork_set.keys()) \|*fork\| {
5582		if (fork.uses == 0) {
5583		std.log.err("fork {f} matched no {s} packages", .{
5584		fork.path, fork.manifest.name,
5585		});
5586		any_unused = true;
5587		} else {
5588		std.log.info("fork {f} matched {d} {s} packages", .{
5589		fork.path, fork.uses, fork.manifest.name,
5590		});
5591		}
5592		}
5593		if (any_unused) process.exit(1);
5594		}
5595
5596		try job_queue.consolidateErrors();
5597
5598		if (fetch.error_bundle.root_list.items.len > 0) {
5599		var errors = try fetch.error_bundle.toOwnedBundle("");
5600		errors.renderToStderr(io, .{}, color) catch {};
5601		process.exit(1);
5602		}
5603
5604		if (fetch_only) return cleanExit(io);
5605
5606		var source_buf = std.array_list.Managed(u8).init(gpa);
5607		defer source_buf.deinit();
5608		try job_queue.createDependenciesSource(&source_buf);
5609		const deps_mod = try createDependenciesModule(
5610		arena,
5611		io,
5612		source_buf.items,
5613		root_mod,
5614		dirs,
5615		config,
5616		);
5617
5618		{
5619		// We need a Module for each package's build.zig.
5620		const hashes = job_queue.table.keys();
5621		const fetches = job_queue.table.values();
5622		try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len));
5623		for (hashes, fetches) \|*hash, f\| {
5624		if (f == &fetch) {
5625		// The first one is a dummy package for the current project.
5626		continue;
5627		}
5628		if (!f.has_build_zig)
5629		continue;
5630		const hash_slice = hash.toSlice();
5631		const mod_root_path = try f.package_root.toString(arena);
5632		const m = try Package.Module.create(arena, .{
5633		.paths = .{
5634		.root = try .fromUnresolved(arena, dirs, &.{mod_root_path}),
5635		.root_src_path = Package.build_zig_basename,
5636		},
5637		.fully_qualified_name = try std.fmt.allocPrint(
5638		arena,
5639		"root.@dependencies.{s}",
5640		.{hash_slice},
5641		),
5642		.cc_argv = &.{},
5643		.inherited = .{},
5644		.global = config,
5645		.parent = root_mod,
5646		});
5647		const hash_cloned = try arena.dupe(u8, hash_slice);
5648		deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m);
5649		f.module = m;
5650		}
5651
5652		// Each build.zig module needs access to each of its
5653		// dependencies' build.zig modules by name.
5654		for (fetches) \|f\| {
5655		const mod = f.module orelse continue;
5656		if (!f.have_manifest) continue;
5657		const man = &f.manifest;
5658		const dep_names = man.dependencies.keys();
5659		try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len));
5660		for (dep_names, man.dependencies.values()) \|name, dep\| {
5661		const dep_digest = Package.Fetch.depDigest(
5662		f.package_root,
5663		dirs.global_cache,
5664		dep,
5665		) orelse continue;
5666		const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue;
5667		const name_cloned = try arena.dupe(u8, name);
5668		mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod);
5669		}
5670		}
5671		}
5672		} else try createEmptyDependenciesModule(
5673		arena,
5674		io,
5675		root_mod,
5676		dirs,
5677		config,
5678		);
5679
5680		const compile_prog_node = root_prog_node.start("Compile Configure Script", 0);
5681		defer compile_prog_node.end();
5682
5683		try root_mod.deps.put(arena, "@build", build_mod);
5684
5685		file_system_inputs.clearRetainingCapacity();
5686		var create_diag: Compilation.CreateDiagnostic = undefined;
5687		const comp = Compilation.create(gpa, arena, io, &create_diag, .{
5688		.libc_installation = libc_installation,
5689		.dirs = dirs,
5690		.root_name = "configure",
5691		.config = config,
5692		.root_mod = root_mod,
5693		.main_mod = build_mod,
5694		.emit_bin = .yes_cache,
5695		.self_exe_path = self_exe_path,
5696		.thread_limit = thread_limit,
5697		.verbose_cc = verbose_cc,
5698		.verbose_link = verbose_link,
5699		.verbose_air = verbose_air,
5700		.verbose_intern_pool = verbose_intern_pool,
5701		.verbose_generic_instances = verbose_generic_instances,
5702		.verbose_llvm_ir = verbose_llvm_ir,
5703		.verbose_llvm_bc = verbose_llvm_bc,
5704		.verbose_llvm_cpu_features = verbose_llvm_cpu_features,
5705		.cache_mode = .whole,
5706		.reference_trace = reference_trace,
5707		.debug_compile_errors = debug_compile_errors,
5708		.environ_map = environ_map,
5709		.file_system_inputs = &file_system_inputs,
5710		}) catch \|err\| switch (err) {
5711		error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}),
5712		else => \|e\| fatal("failed to create compilation: {t}", .{e}),
5713		};
5714		defer comp.destroy();
5715
5716		updateModule(comp, color, compile_prog_node) catch \|err\| switch (err) {
5717		error.CompileErrorsReported => process.exit(2),
5718		else => \|e\| return e,
5719		};
5720
5721		// Since incremental compilation isn't done yet, we use cache_mode = whole
5722		// above, and thus the output file is already closed.
5723		//try comp.makeBinFileExecutable();
5724		const hex_digest: []const u8 = &Cache.binToHex(comp.digest.?);
5725		const exe_path: Path = .{
5726		.root_dir = dirs.local_cache,
5727		.sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ hex_digest, comp.emit_bin.? }),
5728		};
5729		_ = try config_man.addFilePath(exe_path, null);
5730		configure_argv.items[0] = try exe_path.toString(arena);
5731
5732		switch (cache_poison) {
5733		.pure, .disallowed, .ignored => if (try config_man.hit()) {
5734		const digest = config_man.final();
5735		break :cp .{
5736		.{
5737		.root_dir = dirs.local_cache,
5738		.sub_path = try std.fmt.allocPrint(arena, "c/{s}", .{&digest}),
5739		},
5740		false,
5741		};
5742		},
5743		.poisoned => {}, // Don't bother checking for cache hit.
5744		}
5745		}
5746
5747		if (!process.can_spawn) {
5748		const cmd = try std.mem.join(arena, " ", configure_argv.items);
5749		fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd });
5750		}
5751
5752		const rand_int = randInt(io, u64);
5753		const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int);
5754		const config_tmp_path: Path = .{
5755		.root_dir = dirs.local_cache,
5756		.sub_path = tmp_dir_sub_path,
5757		};
5758		const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile(
5759		io,
5760		config_tmp_path.sub_path,
5761		.{ .read = true, .exclusive = true },
5762		);
5763		defer config_tmp_file.close(io);
5764
5765		const term = term: {
5766		const child_node = root_prog_node.start("Run Configure Script", 0);
5767		defer child_node.end();
5768		var child = std.process.spawn(io, .{
5769		.argv = configure_argv.items,
5770		.stdout = .{ .file = config_tmp_file },
5771		.progress_node = child_node,
5772		}) catch \|err\| fatal("failed to spawn configure script {q}: {t}", .{ configure_argv.items[0], err });
5773		defer child.kill(io);
5774		break :term child.wait(io) catch \|err\|
5775		fatal("failed to wait configure script {q}: {t}", .{ configure_argv.items[0], err });
5776		};
5777		if (!term.success()) {
5778		// Failure to produce the configuration file.
5779		const cmd = try std.mem.join(arena, " ", configure_argv.items);
5780		fatal("the following configure command {f}:\n{s}", .{ term, cmd });
5781		}
5782		// Even though the file is designed to be sent directly to make
5783		// runner, we must load it now because:
5784		// * If it contains additional file dependencies, we need to
5785		// add them to `config_man` before obtaining the final digest.
5786		// * If it contains a set of lazy packages that need to be
5787		// fetched, we need to fetch those now and re-run configure.
5788		var configuration = std.Build.Configuration.loadFile(arena, io, config_tmp_file) catch \|err\|
5789		fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err });
5790
5791		if (configuration.unlazy_deps.len != 0) {
5792		if (!dev.env.supports(.fetch_command)) process.exit(1);
5793		var any_errors = false;
5794		for (configuration.unlazy_deps) \|hash_string\| {
5795		const hash = hash_string.slice(&configuration);
5796		assert(hash.len != 0);
5797		if (hash.len > Package.Hash.max_len) {
5798		std.log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash });
5799		any_errors = true;
5800		continue;
5801		}
5802		try unlazy_set.put(arena, .fromSlice(hash), {});
5803		}
5804		if (any_errors) process.exit(1);
5805		if (system_pkg_dir_path) \|p\| {
5806		// In this mode, the system needs to provide these packages; they
5807		// cannot be fetched by Zig.
5808		const s = fs.path.sep_str;
5809		for (unlazy_set.keys()) \|*hash\| {
5810		std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() });
5811		}
5812		std.log.info("remote package fetching disabled due to --system mode", .{});
5813		std.log.info("dependencies might be avoidable depending on build configuration", .{});
5814		process.exit(1);
5815		}
5816		continue :cp;
5817		}
5818
5819		for (configuration.path_deps_base, configuration.path_deps_sub) \|base, sub\| {
5820		const conf_path: std.Build.Configuration.Path = .{ .base = base, .sub = sub };
5821		try config_man.addPathPost(conf_path.toCachePath(&configuration, arena));
5822		}
5823
5824		// We need to add to the configuration cache the source files of
5825		// configurer itself, so that the maker process can watch the file system
5826		// for those changes and restart itself. By doing this, we make it
5827		// possible to bypass creating a Compilation for configurer on
5828		// Configuration cache hit.
5829		{
5830		var it = mem.splitScalar(u8, file_system_inputs.items, 0);
5831		while (it.next()) \|input\| {
5832		_ = try config_man.addPrefixedPathPost(.{
5833		.prefix = input[0],
5834		.sub_path = input[1..],
5835		});
5836		}
5837		}
5838
5839		// If it is poisoned, there is no point in moving it to cached
5840		// location. Just leave it in the tmp directory.
5841		if (configuration.poisoned) {
5842		break :cp .{ config_tmp_path, true };
5843		} else {
5844		const digest = config_man.final();
5845		const final_path: Path = .{
5846		.root_dir = dirs.local_cache,
5847		.sub_path = try std.fmt.allocPrint(arena, "c/{s}", .{&digest}),
5848		};
5849		Io.Dir.rename(
5850		config_tmp_path.root_dir.handle,
5851		config_tmp_path.sub_path,
5852		final_path.root_dir.handle,
5853		final_path.sub_path,
5854		io,
5855		) catch \|err\| retry: {
5856		const e = switch (err) {
5857		error.FileNotFound => e: {
5858		const dir_path = final_path.dirname().?;
5859		dir_path.root_dir.handle.createDirPath(io, dir_path.sub_path) catch \|e\|
5860		fatal("failed to create directory {f}: {t}", .{ dir_path, e });
5861		if (Io.Dir.rename(
5862		config_tmp_path.root_dir.handle,
5863		config_tmp_path.sub_path,
5864		final_path.root_dir.handle,
5865		final_path.sub_path,
5866		io,
5867		)) \|_\| break :retry else \|e\| break :e e;
5868		},
5869		else => \|e\| e,
5870		};
5871		fatal("failed to rename configuration file from {f} into {f}: {t}", .{
5872		config_tmp_path, final_path, e,
5873		});
5874		};
5875		config_man.writeManifest() catch \|err\| warn("failed to write cache manifest: {t}", .{err});
5876		break :cp .{ final_path, false };
5877		}
5878		};
5879
5880		{
5881		// Release all file system locks just before running the maker process.
5882		var configuration_lock = if (!poisoned) config_man.toOwnedLock() else null;
5883		defer if (configuration_lock) \|*l\| l.release(io);
5884
5885		if (print_configuration_path) {
5886		var stdout_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer);
5887		stdout_writer.interface.print("{f}\n", .{configuration_path}) catch
5888		fatal("failed printing cache file path: {t}", .{stdout_writer.err.?});
5889		stdout_writer.flush() catch \|err\|
5890		fatal("failed printing cache file path: {t}", .{err});
5891		return cleanExit(io);
5892		}
5893		const make_runner = make_runner_task.await(io) catch \|err\| fatal("failed compiling maker: {t}", .{err});
5894
5895		make_argv.items[0] = try make_runner.exe_path.toString(arena);
5896		make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena);
5897		}
5898		}
5899
5900		if (!process.can_spawn) {
5901		const cmd = try std.mem.join(arena, " ", make_argv.items);
5902		fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{
5903		native_os, cmd,
5904		});
5905		}
5906
5907		const term = term: {
5908		_ = try io.lockStderr(&.{}, .no_color);
5909		defer io.unlockStderr();
5910		var child = std.process.spawn(io, .{
5911		.argv = make_argv.items,
5912		}) catch \|err\| fatal("failed spawning maker {s}: {t}", .{ make_argv.items[0], err });
5913		defer child.kill(io);
5914		break :term child.wait(io) catch \|err\|
5915		fatal("failed waiting on maker {s}: {t}", .{ make_argv.items[0], err });
5916		};
5917		if (term.success()) return cleanExit(io);
5918		const cmd = try std.mem.join(arena, " ", make_argv.items);
5919		fatal("the following maker command {f}:\n{s}", .{ term, cmd });
5920		}
5921
5922		const MakeRunner = struct {
5923		exe_path: Path,
5924
5925		const Options = struct {
5926		environ_map: *const process.Environ.Map,
5927		dirs: Compilation.Directories,
5928		parent_prog_node: std.Progress.Node,
5929		resolved_target: Package.Module.ResolvedTarget,
5930		libc_installation: ?*const LibCInstallation,
5931		self_exe_path: []const u8,
5932		thread_limit: usize,
5933		color: Color,
5934		reference_trace: ?u32,
5935		optimize_mode: std.builtin.OptimizeMode,
5936		};
5937		};
5938
5939		fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunner.Options) !MakeRunner {
5940		const compile_prog_node = options.parent_prog_node.start("Compiling Maker (first time setup)", 0);
5941		defer compile_prog_node.end();
5942
5943		const strip = options.optimize_mode != .Debug;
5944
5945		const main_mod_paths: Package.Module.CreateOptions.Paths = .{
5946		.root = try .fromRoot(arena, options.dirs, .zig_lib, "compiler"),
5947		.root_src_path = "Maker.zig",
5948		};
5949
5950		const config = try Compilation.Config.resolve(.{
5951		.output_mode = .Exe,
5952		.root_strip = strip,
5953		.root_optimize_mode = options.optimize_mode,
5954		.resolved_target = options.resolved_target,
5955		.have_zcu = true,
5956		.emit_bin = true,
5957		.is_test = false,
5958		});
5959
5960		const root_mod = try Package.Module.create(arena, .{
5961		.paths = main_mod_paths,
5962		.fully_qualified_name = "root",
5963		.cc_argv = &.{},
5964		.inherited = .{
5965		.resolved_target = options.resolved_target,
5966		.optimize_mode = options.optimize_mode,
5967		.strip = strip,
5968		},
5969		.global = config,
5970		.parent = null,
5971		});
5972
5973		var create_diag: Compilation.CreateDiagnostic = undefined;
5974		const comp = Compilation.create(gpa, arena, io, &create_diag, .{
5975		.dirs = options.dirs,
5976		.root_name = "maker",
5977		.config = config,
5978		.root_mod = root_mod,
5979		.main_mod = root_mod,
5980		.emit_bin = .yes_cache,
5981		.self_exe_path = options.self_exe_path,
5982		.thread_limit = options.thread_limit,
5983		.cache_mode = .whole,
5984		.environ_map = options.environ_map,
5985		.reference_trace = options.reference_trace,
5986		}) catch \|err\| switch (err) {
5987		error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}),
5988		error.Canceled => \|e\| return e,
5989		else => \|e\| fatal("failed to create compilation: {t}", .{e}),
5990		};
5991		defer comp.destroy();
5992
5993		try updateModule(comp, options.color, compile_prog_node);
5994
5995		const exe_path: Path = .{
5996		.root_dir = options.dirs.global_cache,
5997		.sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{
5998		&Cache.binToHex(comp.digest.?), comp.emit_bin.?,
5999		}),
6000		};
6001
6002		return .{
6003		.exe_path = exe_path,
6004		};
6005		}
6006
6007		const Fork = struct {
6008		path: Path,
6009		manifest_ast: std.zig.Ast,
6010		manifest: Package.Manifest,
6011		error_bundle: std.zig.ErrorBundle.Wip,
6012		failed: bool,
6013		arena_allocator: std.heap.ArenaAllocator,
6014
6015		fn init(cwd_relative_path: []const u8) Fork {
6016		return .{
6017		.manifest_ast = undefined,
6018		.manifest = undefined,
6019		.error_bundle = undefined,
6020		.arena_allocator = undefined,
6021		.path = .{
6022		.root_dir = .cwd(),
6023		.sub_path = cwd_relative_path,
6024		},
6025		.failed = false,
6026		};
6027		}
6028
6029		fn load(io: Io, gpa: Allocator, fork: *Fork, color: Color) Io.Cancelable!void {
6030		loadFallible(io, gpa, fork, color) catch \|err\| switch (err) {
6031		error.Canceled => \|e\| return e,
6032		error.AlreadyReported => fork.failed = true,
6033		else => \|e\| {
6034		std.log.err("failed to load fork at {f}: {t}", .{ fork.path, e });
6035		fork.failed = true;
6036		},
6037		};
6038		}
6039
6040		fn loadFallible(io: Io, gpa: Allocator, fork: *Fork, color: Color) !void {
6041		fork.arena_allocator = .init(gpa);
6042		const arena = fork.arena_allocator.allocator();
6043
6044		var error_bundle: std.zig.ErrorBundle.Wip = undefined;
6045		try error_bundle.init(gpa);
6046		defer error_bundle.deinit();
6047
6048		const manifest_path = try fork.path.join(arena, Package.Manifest.basename);
6049
6050		Package.Manifest.load(
6051		io,
6052		arena,
6053		manifest_path,
6054		&fork.manifest_ast,
6055		&error_bundle,
6056		&fork.manifest,
6057		true,
6058		) catch \|err\| switch (err) {
6059		error.Canceled => \|e\| return e,
6060		error.ErrorsBundled => {
6061		assert(error_bundle.root_list.items.len > 0);
6062		var errors = try error_bundle.toOwnedBundle("");
6063		errors.renderToStderr(io, .{}, color) catch {};
6064		return error.AlreadyReported;
6065		},
6066		else => \|e\| {
6067		std.log.err("failed to load package manifest {f}: {t}", .{ manifest_path, e });
6068		return error.AlreadyReported;
6069		},
6070		};
6071		}
6072
6073		fn deinitList(forks: []Fork) void {
6074		for (forks) \|*fork\| fork.arena_allocator.deinit();
6075		}
6076		};
6077
6078	5026	const JitCmdOptions = struct {
6079	5027	cmd_name: []const u8,
6080	5028	root_src_path: []const u8,
6081	5029	prepend_zig_lib_dir_path: bool = false,
6082	5030	prepend_global_cache_path: bool = false,
6083	5031	prepend_zig_exe_path: bool = false,
	5032	prepend_seed: bool = false,
6084	5033	depend_on_aro: bool = false,
6085	5034	capture: ?*[]u8 = null,
6086	5035	/// Send error bundles via std.zig.Server over stdout
6087	5036	server: bool = false,
6088		color: Color = .auto,
6089	5037	};
6090	5038
6091	5039	fn jitCmd(
...	...	@@ -6098,8 +5046,10 @@ fn jitCmd(
6098	5046	) !void {
6099	5047	dev.check(.jit_command);
6100	5048
	5049	const color = Color.settingFromEnvironment(environ_map);
	5050
6101	5051	const root_prog_node = std.Progress.start(io, .{
6102		.disable_printing = (options.color == .off),
	5052	.disable_printing = (color == .off),
6103	5053	});
6104	5054	defer root_prog_node.end();
6105	5055
...	...	@@ -6141,7 +5091,7 @@ fn jitCmdInner(
6141	5091	const override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map);
6142	5092	const override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
6143	5093
6144		const cwd_path = try introspect.getResolvedCwd(io, arena);
	5094	const cwd_path = try std.zig.getResolvedCwd(io, arena);
6145	5095
6146	5096	// This `init` calls `fatal` on error.
6147	5097	var dirs: Compilation.Directories = .init(
...	...	@@ -6158,7 +5108,7 @@ fn jitCmdInner(
6158	5108	defer dirs.deinit(io);
6159	5109
6160	5110	var child_argv: std.ArrayList([]const u8) = .empty;
6161		try child_argv.ensureUnusedCapacity(arena, args.len + 4);
	5111	try child_argv.ensureUnusedCapacity(arena, args.len + 5);
6162	5112
6163	5113	// We want to release all the locks before executing the child process, so we make a nice
6164	5114	// big block here to ensure the cleanup gets run when we extract out our argv.
...	...	@@ -6244,7 +5194,8 @@ fn jitCmdInner(
6244	5194	process.exit(2);
6245	5195	}
6246	5196	} else {
6247		updateModule(comp, options.color, root_prog_node) catch \|err\| switch (err) {
	5197	const color = Color.settingFromEnvironment(environ_map);
	5198	updateModule(comp, color, root_prog_node) catch \|err\| switch (err) {
6248	5199	error.CompileErrorsReported => process.exit(2),
6249	5200	else => \|e\| return e,
6250	5201	};
...	...	@@ -6259,11 +5210,13 @@ fn jitCmdInner(
6259	5210	}
6260	5211
6261	5212	if (options.prepend_zig_lib_dir_path)
6262		child_argv.appendAssumeCapacity(dirs.zig_lib.path.?);
	5213	child_argv.appendAssumeCapacity(try allocPrint(arena, "--zig-lib={s}", .{dirs.zig_lib.path.?}));
6263	5214	if (options.prepend_zig_exe_path)
6264		child_argv.appendAssumeCapacity(self_exe_path);
	5215	child_argv.appendAssumeCapacity(try allocPrint(arena, "--zig={s}", .{self_exe_path}));
6265	5216	if (options.prepend_global_cache_path)
6266		child_argv.appendAssumeCapacity(dirs.global_cache.path.?);
	5217	child_argv.appendAssumeCapacity(try allocPrint(arena, "--global-cache={s}", .{dirs.global_cache.path.?}));
	5218	if (options.prepend_seed)
	5219	child_argv.appendAssumeCapacity(try allocPrint(arena, "--seed=0x{x}", .{randInt(io, u32)}));
6267	5220
6268	5221	child_argv.appendSliceAssumeCapacity(args);
6269	5222
...	...	@@ -7199,567 +6152,6 @@ fn parseRcIncludes(arg: []const u8) std.zig.RcIncludes {
7199	6152	fatal("unsupported rc includes type: {q}", .{arg});
7200	6153	}
7201	6154
7202		const usage_fetch =
7203		\\Usage: zig fetch [options] <url>
7204		\\Usage: zig fetch [options] <path>
7205		\\
7206		\\ Copy a package into the global cache and print its hash.
7207		\\ <url> must point to one of the following:
7208		\\ - A git+http / git+https server for the package
7209		\\ - A tarball file (with or without compression) containing
7210		\\ package source
7211		\\ - A git bundle file containing package source
7212		\\
7213		\\Examples:
7214		\\
7215		\\ zig fetch --save git+https://example.com/andrewrk/fun-example-tool.git
7216		\\ zig fetch --save https://example.com/andrewrk/fun-example-tool/archive/refs/heads/master.tar.gz
7217		\\
7218		\\Options:
7219		\\ -h, --help Print this help and exit
7220		\\ --global-cache-dir [path] Override path to global Zig cache directory
7221		\\ --cache-dir [path] Override path to local cache directory
7222		\\ --pkg-dir [path] Override path to local package directory
7223		\\ --debug-hash Print verbose hash information to stdout
7224		\\ --debug-log [scope] Enable printing debug/info log messages for scope
7225		\\ --save Add the fetched package to build.zig.zon
7226		\\ --save=[name] Add the fetched package to build.zig.zon as name
7227		\\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim
7228		\\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim
7229		\\
7230		;
7231
7232		fn cmdFetch(
7233		gpa: Allocator,
7234		arena: Allocator,
7235		io: Io,
7236		args: []const []const u8,
7237		environ_map: *process.Environ.Map,
7238		) !void {
7239		dev.check(.fetch_command);
7240
7241		const color: Color = Color.settingFromEnvironment(environ_map);
7242		var opt_path_or_url: ?[]const u8 = null;
7243		var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
7244		var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map);
7245		var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map);
7246		var debug_hash: bool = false;
7247		var save: union(enum) {
7248		no,
7249		yes: ?[]const u8,
7250		exact: ?[]const u8,
7251		} = .no;
7252
7253		{
7254		var i: usize = 0;
7255		while (i < args.len) : (i += 1) {
7256		const arg = args[i];
7257		if (mem.startsWith(u8, arg, "-")) {
7258		if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
7259		try Io.File.stdout().writeStreamingAll(io, usage_fetch);
7260		return cleanExit(io);
7261		} else if (mem.eql(u8, arg, "--global-cache-dir")) {
7262		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
7263		i += 1;
7264		override_global_cache_dir = args[i];
7265		} else if (mem.eql(u8, arg, "--cache-dir")) {
7266		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
7267		i += 1;
7268		override_local_cache_dir = args[i];
7269		} else if (mem.eql(u8, arg, "--pkg-dir")) {
7270		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
7271		i += 1;
7272		override_pkg_dir = args[i];
7273		} else if (mem.eql(u8, arg, "--debug-hash")) {
7274		debug_hash = true;
7275		} else if (mem.eql(u8, arg, "--debug-log")) {
7276		if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg});
7277		i += 1;
7278		try addDebugLog(arena, args[i]);
7279		} else if (mem.eql(u8, arg, "--save")) {
7280		save = .{ .yes = null };
7281		} else if (mem.cutPrefix(u8, arg, "--save=")) \|rest\| {
7282		save = .{ .yes = rest };
7283		} else if (mem.eql(u8, arg, "--save-exact")) {
7284		save = .{ .exact = null };
7285		} else if (mem.cutPrefix(u8, arg, "--save-exact=")) \|rest\| {
7286		save = .{ .exact = rest };
7287		} else {
7288		fatal("unrecognized parameter: {q}", .{arg});
7289		}
7290		} else if (opt_path_or_url != null) {
7291		fatal("unexpected extra parameter: {q}", .{arg});
7292		} else {
7293		opt_path_or_url = arg;
7294		}
7295		}
7296		}
7297
7298		const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{});
7299
7300		var http_client: std.http.Client = .{ .allocator = gpa, .io = io };
7301		defer http_client.deinit();
7302
7303		try http_client.initDefaultProxies(arena, environ_map);
7304
7305		var root_prog_node = std.Progress.start(io, .{
7306		.root_name = "Fetch",
7307		});
7308		defer root_prog_node.end();
7309
7310		var global_cache_directory: Directory = l: {
7311		const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena, environ_map);
7312		break :l .{
7313		.handle = try Io.Dir.cwd().createDirPathOpen(io, p, .{}),
7314		.path = p,
7315		};
7316		};
7317		defer global_cache_directory.handle.close(io);
7318
7319		var local_storage: Package.Fetch.LocalStorage = undefined;
7320		var build_root: BuildRoot = undefined;
7321		var build_root_initialized = false;
7322		defer if (build_root_initialized) build_root.deinit(io);
7323
7324		const cwd_path = try introspect.getResolvedCwd(io, arena);
7325
7326		const local_storage_ptr = switch (save) {
7327		.no => null,
7328		.yes, .exact => ls: {
7329		build_root = try findBuildRoot(arena, io, .{ .cwd_path = cwd_path });
7330		build_root_initialized = true;
7331
7332		local_storage = .{
7333		.cache_root = if (override_local_cache_dir) \|p\| .initCwd(p) else .{
7334		.root_dir = build_root.directory,
7335		.sub_path = ".zig-cache",
7336		},
7337		.pkg_root = if (override_pkg_dir) \|p\| .initCwd(p) else .{
7338		.root_dir = build_root.directory,
7339		.sub_path = "zig-pkg",
7340		},
7341		};
7342
7343		break :ls &local_storage;
7344		},
7345		};
7346
7347		var job_queue: Package.Fetch.JobQueue = .{
7348		.io = io,
7349		.http_client = &http_client,
7350		.global_cache = global_cache_directory,
7351		.local_storage = local_storage_ptr,
7352		.recursive = false,
7353		.read_only = false,
7354		.debug_hash = debug_hash,
7355		.mode = .all,
7356		.prog_node = root_prog_node,
7357		};
7358		defer job_queue.deinit();
7359
7360		var fetch: Package.Fetch = .{
7361		.arena = std.heap.ArenaAllocator.init(gpa),
7362		.location = .{ .path_or_url = path_or_url },
7363		.location_tok = 0,
7364		.hash_tok = .none,
7365		.name_tok = 0,
7366		.lazy_status = .eager,
7367		.remote_package_root = undefined,
7368		.parent_package_root = undefined,
7369		.parent_manifest_ast = null,
7370		.prog_node = root_prog_node,
7371		.job_queue = &job_queue,
7372		.omit_missing_hash_error = true,
7373		.allow_missing_paths_field = false,
7374		.use_latest_commit = true,
7375
7376		.package_root = undefined,
7377		.error_bundle = undefined,
7378		.manifest = undefined,
7379		.manifest_ast = undefined,
7380		.have_manifest = false,
7381		.computed_hash = undefined,
7382		.has_build_zig = false,
7383		.oom_flag = false,
7384		.latest_commit = null,
7385
7386		.module = null,
7387		};
7388		defer fetch.deinit();
7389
7390		fetch.run() catch \|err\| switch (err) {
7391		error.OutOfMemory, error.Canceled => \|e\| return e,
7392		error.FetchFailed => {}, // error bundle checked below
7393		};
7394
7395		try job_queue.group.await(io);
7396
7397		if (fetch.error_bundle.root_list.items.len > 0) {
7398		var errors = try fetch.error_bundle.toOwnedBundle("");
7399		errors.renderToStderr(io, .{}, color) catch {};
7400		process.exit(1);
7401		}
7402
7403		const package_hash = fetch.computedPackageHash();
7404		const package_hash_slice = package_hash.toSlice();
7405
7406		root_prog_node.end();
7407		root_prog_node = .{ .index = .none };
7408
7409		const name = switch (save) {
7410		.no => {
7411		var stdout = Io.File.stdout().writerStreaming(io, &stdout_buffer);
7412		try stdout.interface.print("{s}\n", .{package_hash_slice});
7413		try stdout.interface.flush();
7414		return cleanExit(io);
7415		},
7416		.yes, .exact => \|name\| name: {
7417		if (name) \|n\| break :name n;
7418		if (!fetch.have_manifest)
7419		fatal("unable to determine name; fetched package has no build.zig.zon file", .{});
7420		break :name fetch.manifest.name;
7421		},
7422		};
7423
7424		// The name to use in case the manifest file needs to be created now.
7425		const init_root_name = fs.path.basename(build_root.directory.path orelse cwd_path);
7426		var manifest, var ast = try loadManifest(gpa, arena, io, .{
7427		.root_name = try sanitizeExampleName(arena, init_root_name),
7428		.dir = build_root.directory.handle,
7429		.color = color,
7430		});
7431		defer {
7432		manifest.deinit(gpa);
7433		ast.deinit(gpa);
7434		}
7435
7436		var fixups: Ast.Render.Fixups = .{};
7437		defer fixups.deinit(gpa);
7438
7439		var saved_path_or_url = path_or_url;
7440
7441		if (fetch.latest_commit) \|latest_commit\| resolved: {
7442		const latest_commit_hex = try std.fmt.allocPrint(arena, "{f}", .{latest_commit});
7443
7444		var uri = try std.Uri.parse(path_or_url);
7445
7446		if (uri.fragment) \|fragment\| {
7447		const target_ref = try fragment.toRawMaybeAlloc(arena);
7448
7449		// the refspec may already be fully resolved
7450		if (std.mem.eql(u8, target_ref, latest_commit_hex)) break :resolved;
7451
7452		std.log.info("resolved ref {q} to commit {s}", .{ target_ref, latest_commit_hex });
7453
7454		// include the original refspec in a query parameter, could be used to check for updates
7455		uri.query = .{ .percent_encoded = try std.fmt.allocPrint(arena, "ref={f}", .{
7456		std.fmt.alt(fragment, .formatEscaped),
7457		}) };
7458		} else {
7459		std.log.info("resolved to commit {s}", .{latest_commit_hex});
7460		}
7461
7462		// replace the refspec with the resolved commit SHA
7463		uri.fragment = .{ .raw = latest_commit_hex };
7464
7465		switch (save) {
7466		.yes => saved_path_or_url = try std.fmt.allocPrint(arena, "{f}", .{uri}),
7467		.no, .exact => {}, // keep the original URL
7468		}
7469		}
7470
7471		const new_node_init = try std.fmt.allocPrint(arena,
7472		\\.{{
7473		\\ .url = "{f}",
7474		\\ .hash = "{f}",
7475		\\ }}
7476		, .{
7477		std.zig.fmtString(saved_path_or_url),
7478		std.zig.fmtString(package_hash_slice),
7479		});
7480
7481		const new_node_text = try std.fmt.allocPrint(arena, ".{f} = {s},\n", .{
7482		std.zig.fmtIdPU(name), new_node_init,
7483		});
7484
7485		const dependencies_init = try std.fmt.allocPrint(arena, ".{{\n {s} }}", .{
7486		new_node_text,
7487		});
7488
7489		const dependencies_text = try std.fmt.allocPrint(arena, ".dependencies = {s},\n", .{
7490		dependencies_init,
7491		});
7492
7493		if (manifest.dependencies.get(name)) \|dep\| {
7494		if (dep.hash) \|h\| {
7495		switch (dep.location) {
7496		.url => \|u\| {
7497		if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) {
7498		std.log.info("existing dependency named {q} is up-to-date", .{name});
7499		process.exit(0);
7500		}
7501		},
7502		.path => {},
7503		}
7504		}
7505
7506		const location_replace = try std.fmt.allocPrint(
7507		arena,
7508		"\"{f}\"",
7509		.{std.zig.fmtString(saved_path_or_url)},
7510		);
7511		const hash_replace = try std.fmt.allocPrint(
7512		arena,
7513		"\"{f}\"",
7514		.{std.zig.fmtString(package_hash_slice)},
7515		);
7516
7517		warn("overwriting existing dependency named {q}", .{name});
7518		try fixups.replace_nodes_with_string.put(gpa, dep.location_node, location_replace);
7519		if (dep.hash_node.unwrap()) \|hash_node\| {
7520		try fixups.replace_nodes_with_string.put(gpa, hash_node, hash_replace);
7521		} else {
7522		// https://github.com/ziglang/zig/issues/21690
7523		}
7524		} else if (manifest.dependencies.count() > 0) {
7525		// Add fixup for adding another dependency.
7526		const deps = manifest.dependencies.values();
7527		const last_dep_node = deps[deps.len - 1].node;
7528		try fixups.append_string_after_node.put(gpa, last_dep_node, new_node_text);
7529		} else if (manifest.dependencies_node.unwrap()) \|dependencies_node\| {
7530		// Add fixup for replacing the entire dependencies struct.
7531		try fixups.replace_nodes_with_string.put(gpa, dependencies_node, dependencies_init);
7532		} else {
7533		// Add fixup for adding dependencies struct.
7534		try fixups.append_string_after_node.put(gpa, manifest.version_node, dependencies_text);
7535		}
7536
7537		var aw: Io.Writer.Allocating = .init(gpa);
7538		defer aw.deinit();
7539		try ast.render(gpa, &aw.writer, fixups);
7540		const rendered = aw.written();
7541
7542		build_root.directory.handle.writeFile(io, .{ .sub_path = Package.Manifest.basename, .data = rendered }) catch \|err\| {
7543		fatal("unable to write {s} file: {t}", .{ Package.Manifest.basename, err });
7544		};
7545
7546		return cleanExit(io);
7547		}
7548
7549		fn createEmptyDependenciesModule(
7550		arena: Allocator,
7551		io: Io,
7552		main_mod: *Package.Module,
7553		dirs: Compilation.Directories,
7554		global_options: Compilation.Config,
7555		) !void {
7556		var source = std.array_list.Managed(u8).init(arena);
7557		try Package.Fetch.JobQueue.createEmptyDependenciesSource(&source);
7558		_ = try createDependenciesModule(
7559		arena,
7560		io,
7561		source.items,
7562		main_mod,
7563		dirs,
7564		global_options,
7565		);
7566		}
7567
7568		/// Creates the dependencies.zig file and corresponding `Package.Module` for the
7569		/// build runner to obtain via `@import("@dependencies")`.
7570		fn createDependenciesModule(
7571		arena: Allocator,
7572		io: Io,
7573		source: []const u8,
7574		main_mod: *Package.Module,
7575		dirs: Compilation.Directories,
7576		global_options: Compilation.Config,
7577		) !*Package.Module {
7578		// Atomically create the file in a directory named after the hash of its contents.
7579		const basename = "dependencies.zig";
7580		const rand_int = randInt(io, u64);
7581		const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int);
7582		{
7583		var tmp_dir = try dirs.local_cache.handle.createDirPathOpen(io, tmp_dir_sub_path, .{});
7584		defer tmp_dir.close(io);
7585		try tmp_dir.writeFile(io, .{ .sub_path = basename, .data = source });
7586		}
7587		const tmp_dir_path: Path = .{
7588		.root_dir = dirs.local_cache,
7589		.sub_path = tmp_dir_sub_path,
7590		};
7591
7592		var hh: Cache.HashHelper = .{};
7593		hh.addBytes(build_options.version);
7594		hh.addBytes(source);
7595		const hex_digest = hh.final();
7596
7597		const o_dir_path: Path = .{
7598		.root_dir = dirs.local_cache,
7599		.sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest),
7600		};
7601		try Package.Fetch.renameTmpIntoCache(io, tmp_dir_path, o_dir_path);
7602
7603		const deps_mod = try Package.Module.create(arena, .{
7604		.paths = .{
7605		.root = try .fromRoot(arena, dirs, .local_cache, o_dir_path.sub_path),
7606		.root_src_path = basename,
7607		},
7608		.fully_qualified_name = "root.@dependencies",
7609		.parent = main_mod,
7610		.cc_argv = &.{},
7611		.inherited = .{},
7612		.global = global_options,
7613		});
7614		try main_mod.deps.put(arena, "@dependencies", deps_mod);
7615		return deps_mod;
7616		}
7617
7618		const BuildRoot = struct {
7619		directory: Cache.Directory,
7620		build_zig_basename: []const u8,
7621		cleanup_build_dir: ?Io.Dir,
7622
7623		fn deinit(br: *BuildRoot, io: Io) void {
7624		if (br.cleanup_build_dir) \|*dir\| dir.close(io);
7625		br.* = undefined;
7626		}
7627		};
7628
7629		const FindBuildRootOptions = struct {
7630		build_file: ?[]const u8 = null,
7631		cwd_path: ?[]const u8 = null,
7632		};
7633
7634		fn findBuildRoot(arena: Allocator, io: Io, options: FindBuildRootOptions) !BuildRoot {
7635		const cwd_path = options.cwd_path orelse try introspect.getResolvedCwd(io, arena);
7636		const build_zig_basename = if (options.build_file) \|bf\|
7637		fs.path.basename(bf)
7638		else
7639		Package.build_zig_basename;
7640
7641		if (options.build_file) \|bf\| {
7642		if (fs.path.dirname(bf)) \|dirname\| {
7643		const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch \|err\| {
7644		fatal("unable to open directory to build file from argument 'build-file', {q}: {t}", .{ dirname, err });
7645		};
7646		return .{
7647		.build_zig_basename = build_zig_basename,
7648		.directory = .{ .path = dirname, .handle = dir },
7649		.cleanup_build_dir = dir,
7650		};
7651		}
7652
7653		return .{
7654		.build_zig_basename = build_zig_basename,
7655		.directory = .{ .path = null, .handle = Io.Dir.cwd() },
7656		.cleanup_build_dir = null,
7657		};
7658		}
7659		// Search up parent directories until we find build.zig.
7660		var dirname: []const u8 = cwd_path;
7661		while (true) {
7662		const joined_path = try fs.path.join(arena, &[_][]const u8{ dirname, build_zig_basename });
7663		if (Io.Dir.cwd().access(io, joined_path, .{})) \|_\| {
7664		const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch \|err\| {
7665		fatal("unable to open directory while searching for build.zig file, {q}: {t}", .{ dirname, err });
7666		};
7667		return .{
7668		.build_zig_basename = build_zig_basename,
7669		.directory = .{
7670		.path = dirname,
7671		.handle = dir,
7672		},
7673		.cleanup_build_dir = dir,
7674		};
7675		} else \|err\| switch (err) {
7676		error.FileNotFound => {
7677		dirname = fs.path.dirname(dirname) orelse {
7678		std.log.info("initialize {s} template file with 'zig init'", .{
7679		Package.build_zig_basename,
7680		});
7681		std.log.info("see 'zig --help' for more options", .{});
7682		fatal("no build.zig file found, in the current directory or any parent directories", .{});
7683		};
7684		continue;
7685		},
7686		else => \|e\| return e,
7687		}
7688		}
7689		}
7690
7691		const LoadManifestOptions = struct {
7692		root_name: []const u8,
7693		dir: Io.Dir,
7694		color: Color,
7695		};
7696
7697		fn loadManifest(
7698		gpa: Allocator,
7699		arena: Allocator,
7700		io: Io,
7701		options: LoadManifestOptions,
7702		) !struct { Package.Manifest, Ast } {
7703		const rng: std.Random.IoSource = .{ .io = io };
7704
7705		const manifest_bytes = while (true) {
7706		break options.dir.readFileAllocOptions(
7707		io,
7708		Package.Manifest.basename,
7709		arena,
7710		.limited(Package.Manifest.max_bytes),
7711		.@"1",
7712		0,
7713		) catch \|err\| switch (err) {
7714		error.FileNotFound => {
7715		writeSimpleTemplateFile(io, Package.Manifest.basename,
7716		\\.{{
7717		\\ .name = .{s},
7718		\\ .version = "{s}",
7719		\\ .paths = .{{""}},
7720		\\ .fingerprint = 0x{x},
7721		\\}}
7722		\\
7723		, .{
7724		options.root_name,
7725		build_options.version,
7726		Package.Fingerprint.generate(rng.interface(), options.root_name).int(),
7727		}) catch \|e\| {
7728		fatal("unable to write {s}: {t}", .{ Package.Manifest.basename, e });
7729		};
7730		continue;
7731		},
7732		else => \|e\| fatal("unable to load {s}: {t}", .{ Package.Manifest.basename, e }),
7733		};
7734		};
7735		var ast = try Ast.parse(gpa, manifest_bytes, .zon);
7736		errdefer ast.deinit(gpa);
7737
7738		if (ast.errors.len > 0) {
7739		try std.zig.printAstErrorsToStderr(gpa, io, ast, Package.Manifest.basename, options.color);
7740		process.exit(2);
7741		}
7742
7743		var manifest = try Package.Manifest.parse(gpa, &ast, rng.interface(), .{});
7744		errdefer manifest.deinit(gpa);
7745
7746		if (manifest.errors.len > 0) {
7747		var wip_errors: std.zig.ErrorBundle.Wip = undefined;
7748		try wip_errors.init(gpa);
7749		defer wip_errors.deinit();
7750
7751		const src_path = try wip_errors.addString(Package.Manifest.basename);
7752		try manifest.copyErrorsIntoBundle(ast, src_path, &wip_errors);
7753
7754		var error_bundle = try wip_errors.toOwnedBundle("");
7755		defer error_bundle.deinit(gpa);
7756		error_bundle.renderToStderr(io, .{}, options.color) catch {};
7757
7758		process.exit(2);
7759		}
7760		return .{ manifest, ast };
7761		}
7762
7763	6155	const Templates = struct {
7764	6156	zig_lib_directory: Cache.Directory,
7765	6157	dir: Io.Dir,
...	...	@@ -7834,13 +6226,13 @@ fn writeSimpleTemplateFile(io: Io, file_name: []const u8, comptime fmt: []const
7834	6226	}
7835	6227
7836	6228	fn findTemplates(gpa: Allocator, arena: Allocator, io: Io) Templates {
7837		const cwd_path = introspect.getResolvedCwd(io, arena) catch \|err\| {
	6229	const cwd_path = std.zig.getResolvedCwd(io, arena) catch \|err\| {
7838	6230	fatal("unable to get cwd: {t}", .{err});
7839	6231	};
7840	6232	const self_exe_path = process.executablePathAlloc(io, arena) catch \|err\| {
7841	6233	fatal("unable to find self exe path: {t}", .{err});
7842	6234	};
7843		var zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, io, cwd_path, self_exe_path) catch \|err\| {
	6235	var zig_lib_directory = std.zig.findZigLibDirFromSelfExe(arena, io, cwd_path, self_exe_path) catch \|err\| {
7844	6236	fatal("unable to find zig installation directory {q}: {t}", .{ self_exe_path, err });
7845	6237	};
7846	6238

src/print_env.zig+1-2

...	...	@@ -8,7 +8,6 @@ const fatal = std.process.fatal;
8	8
9	9	const build_options = @import("build_options");
10	10	const Compilation = @import("Compilation.zig");
11		const introspect = @import("introspect.zig");
12	11
13	12	pub fn cmdEnv(
14	13	arena: Allocator,
...	...	@@ -29,7 +28,7 @@ pub fn cmdEnv(
29	28	},
30	29	};
31	30
32		const cwd_path = try introspect.getResolvedCwd(io, arena);
	31	const cwd_path = try std.zig.getResolvedCwd(io, arena);
33	32
34	33	var dirs: Compilation.Directories = .init(
35	34	arena,

src/print_targets.zig+1-2

...	...	@@ -9,7 +9,6 @@ const Target = std.Target;
9	9	const assert = std.debug.assert;
10	10
11	11	const glibc = @import("libs/glibc.zig");
12		const introspect = @import("introspect.zig");
13	12	const target = @import("target.zig");
14	13
15	14	pub fn cmdTargets(
...	...	@@ -20,7 +19,7 @@ pub fn cmdTargets(
20	19	native_target: *const Target,
21	20	) !void {
22	21	_ = args;
23		var zig_lib_directory = introspect.findZigLibDir(allocator, io) catch \|err\|
	22	var zig_lib_directory = std.zig.findZigLibDir(allocator, io) catch \|err\|
24	23	fatal("unable to find zig installation directory: {t}", .{err});
25	24	defer zig_lib_directory.handle.close(io);
26	25	defer allocator.free(zig_lib_directory.path.?);