Commit 12355cfb4cb14ba78423fb38838f4485bb563c9b | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-02-23 15:58:51-08:00
committer	Andrew Kelley <andrew@ziglang.org> 2025-02-26 11:42:03-08:00
log	12355cfb4cb14ba78423fb38838f4485bb563c9b
tree	bd3f0fbfc489ccbeb87d5913a26639a8e2033d63
parent	e0129b387ff962c0f89d62c8ab0409145a19f453

Package: new hash format

legacy format is also supported.

closes #20178

4 files changed, 269 insertions(+), 145 deletions(-)

src
- Package.zig
- Package
  - Fetch.zig
  - Manifest.zig
- main.zig

src/Package.zig+156

...	...	@@ -1,8 +1,164 @@
	1	const std = @import("std");
	2	const assert = std.debug.assert;
	3
1	4	pub const Module = @import("Package/Module.zig");
2	5	pub const Fetch = @import("Package/Fetch.zig");
3	6	pub const build_zig_basename = "build.zig";
4	7	pub const Manifest = @import("Package/Manifest.zig");
5	8
	9	pub const multihash_len = 1 + 1 + Hash.Algo.digest_length;
	10	pub const multihash_hex_digest_len = 2 * multihash_len;
	11	pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
	12
	13	/// A user-readable, file system safe hash that identifies an exact package
	14	/// snapshot, including file contents.
	15	///
	16	/// This data structure can be used to store the legacy hash format too. Legacy
	17	/// hash format is scheduled to be removed after 0.14.0 is tagged.
	18	pub const Hash = struct {
	19	/// Maximum size of a package hash. Unused bytes at the end are
	20	/// filled with zeroes.
	21	bytes: [max_len]u8,
	22
	23	pub const Algo = std.crypto.hash.sha2.Sha256;
	24	pub const Digest = [Algo.digest_length]u8;
	25
	26	pub const max_len = 32 + 1 + 32 + 1 + 12;
	27
	28	pub fn fromSlice(s: []const u8) Hash {
	29	assert(s.len <= max_len);
	30	var result: Hash = undefined;
	31	@memcpy(result.bytes[0..s.len], s);
	32	@memset(result.bytes[s.len..], 0);
	33	return result;
	34	}
	35
	36	pub fn toSlice(ph: *const Hash) []const u8 {
	37	var end: usize = ph.bytes.len;
	38	while (true) {
	39	end -= 1;
	40	if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1];
	41	}
	42	}
	43
	44	pub fn eql(a: const Hash, b: const Hash) bool {
	45	return std.mem.eql(u8, &a.bytes, &b.bytes);
	46	}
	47
	48	/// Distinguishes whether the legacy multihash format is being stored here.
	49	pub fn isOld(h: *const Hash) bool {
	50	if (h.bytes.len < 2) return false;
	51	const their_multihash_func = std.fmt.parseInt(u8, h.bytes[0..2], 16) catch return false;
	52	if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) return false;
	53	if (h.toSlice().len != multihash_hex_digest_len) return false;
	54	return std.mem.indexOfScalar(u8, &h.bytes, '-') == null;
	55	}
	56
	57	test isOld {
	58	const h: Hash = .fromSlice("1220138f4aba0c01e66b68ed9e1e1e74614c06e4743d88bc58af4f1c3dd0aae5fea7");
	59	try std.testing.expect(h.isOld());
	60	}
	61
	62	/// Produces "$name-$semver-$sizedhash".
	63	/// * name is the name field from build.zig.zon, truncated at 32 bytes and must
	64	/// be a valid zig identifier
	65	/// * semver is the version field from build.zig.zon, truncated at 32 bytes
	66	/// * sizedhash is the following 9-byte array, base64 encoded using -_ to make
	67	/// it filesystem safe:
	68	/// - (4 bytes) LE u32 total decompressed size in bytes
	69	/// - (5 bytes) truncated SHA-256 of hashed files of the package
	70	///
	71	/// example: "nasm-2.16.1-2-BWdcABvF_jM1"
	72	pub fn init(digest: Digest, name: []const u8, ver: []const u8, size: u32) Hash {
	73	var result: Hash = undefined;
	74	var buf: std.ArrayListUnmanaged(u8) = .initBuffer(&result.bytes);
	75	buf.appendSliceAssumeCapacity(name[0..@min(name.len, 32)]);
	76	buf.appendAssumeCapacity('-');
	77	buf.appendSliceAssumeCapacity(ver[0..@min(ver.len, 32)]);
	78	buf.appendAssumeCapacity('-');
	79	var sizedhash: [9]u8 = undefined;
	80	std.mem.writeInt(u32, sizedhash[0..4], size, .little);
	81	sizedhash[4..].* = digest[0..5].*;
	82	_ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(12), &sizedhash);
	83	@memset(buf.unusedCapacitySlice(), 0);
	84	return result;
	85	}
	86
	87	/// Produces "$hashiname-N-$sizedhash". For packages that lack "build.zig.zon" metadata.
	88	/// * hashiname is [5..][0..24] bytes of the SHA-256, urlsafe-base64-encoded, for a total of 32 bytes encoded
	89	/// * the semver section is replaced with a hardcoded N which stands for
	90	/// "naked". It acts as a version number so that any future updates to the
	91	/// hash format can tell this hash format apart. Note that "N" is an
	92	/// invalid semver.
	93	/// * sizedhash is the same as in `init`.
	94	///
	95	/// The hash is broken up this way so that "sizedhash" can be calculated
	96	/// exactly the same way in both cases, and so that "name" and "hashiname" can
	97	/// be used interchangeably in both cases.
	98	pub fn initNaked(digest: Digest, size: u32) Hash {
	99	var name: [32]u8 = undefined;
	100	_ = std.base64.url_safe_no_pad.Encoder.encode(&name, digest[5..][0..24]);
	101	return init(digest, &name, "N", size);
	102	}
	103	};
	104
	105	pub const MultihashFunction = enum(u16) {
	106	identity = 0x00,
	107	sha1 = 0x11,
	108	@"sha2-256" = 0x12,
	109	@"sha2-512" = 0x13,
	110	@"sha3-512" = 0x14,
	111	@"sha3-384" = 0x15,
	112	@"sha3-256" = 0x16,
	113	@"sha3-224" = 0x17,
	114	@"sha2-384" = 0x20,
	115	@"sha2-256-trunc254-padded" = 0x1012,
	116	@"sha2-224" = 0x1013,
	117	@"sha2-512-224" = 0x1014,
	118	@"sha2-512-256" = 0x1015,
	119	@"blake2b-256" = 0xb220,
	120	_,
	121	};
	122
	123	pub const multihash_function: MultihashFunction = switch (Hash.Algo) {
	124	std.crypto.hash.sha2.Sha256 => .@"sha2-256",
	125	else => @compileError("unreachable"),
	126	};
	127
	128	pub fn multiHashHexDigest(digest: Hash.Digest) MultiHashHexDigest {
	129	const hex_charset = std.fmt.hex_charset;
	130
	131	var result: MultiHashHexDigest = undefined;
	132
	133	result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
	134	result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
	135
	136	result[2] = hex_charset[Hash.Algo.digest_length >> 4];
	137	result[3] = hex_charset[Hash.Algo.digest_length & 15];
	138
	139	for (digest, 0..) \|byte, i\| {
	140	result[4 + i * 2] = hex_charset[byte >> 4];
	141	result[5 + i * 2] = hex_charset[byte & 15];
	142	}
	143	return result;
	144	}
	145
	146	comptime {
	147	// We avoid unnecessary uleb128 code in hexDigest by asserting here the
	148	// values are small enough to be contained in the one-byte encoding.
	149	assert(@intFromEnum(multihash_function) < 127);
	150	assert(Hash.Algo.digest_length < 127);
	151	}
	152
	153	test Hash {
	154	const example_digest: Hash.Digest = .{
	155	0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87,
	156	0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f,
	157	};
	158	const result: Hash = .init(example_digest, "nasm", "2.16.1-2", 10 * 1024 * 1024);
	159	try std.testing.expectEqualStrings("nasm-2.16.1-2-AACgAMf1cbe0", result.toSlice());
	160	}
	161
6	162	test {
7	163	_ = Fetch;
8	164	}

src/Package/Fetch.zig+92-62

...	...	@@ -56,7 +56,7 @@ package_root: Cache.Path,
56	56	error_bundle: ErrorBundle.Wip,
57	57	manifest: ?Manifest,
58	58	manifest_ast: std.zig.Ast,
59		actual_hash: Manifest.Digest,
	59	computed_hash: ComputedHash,
60	60	/// Fetch logic notices whether a package has a build.zig file and sets this flag.
61	61	has_build_zig: bool,
62	62	/// Indicates whether the task aborted due to an out-of-memory condition.
...	...	@@ -116,8 +116,8 @@ pub const JobQueue = struct {
116	116	/// as lazy.
117	117	unlazy_set: UnlazySet = .{},
118	118
119		pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);
120		pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, void);
	119	pub const Table = std.AutoArrayHashMapUnmanaged(Package.Hash, *Fetch);
	120	pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Package.Hash, void);
121	121
122	122	pub fn deinit(jq: *JobQueue) void {
123	123	if (jq.all_fetches.items.len == 0) return;
...	...	@@ -160,22 +160,24 @@ pub const JobQueue = struct {
160	160
161	161	// Ensure the generated .zig file is deterministic.
162	162	jq.table.sortUnstable(@as(struct {
163		keys: []const Manifest.MultiHashHexDigest,
	163	keys: []const Package.Hash,
164	164	pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
165		return std.mem.lessThan(u8, &ctx.keys[a_index], &ctx.keys[b_index]);
	165	return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes);
166	166	}
167	167	}, .{ .keys = keys }));
168	168
169		for (keys, jq.table.values()) \|hash, fetch\| {
	169	for (keys, jq.table.values()) \|*hash, fetch\| {
170	170	if (fetch == jq.all_fetches.items[0]) {
171	171	// The first one is a dummy package for the current project.
172	172	continue;
173	173	}
174	174
	175	const hash_slice = hash.toSlice();
	176
175	177	try buf.writer().print(
176	178	\\ pub const {} = struct {{
177	179	\\
178		, .{std.zig.fmtId(&hash)});
	180	, .{std.zig.fmtId(hash_slice)});
179	181
180	182	lazy: {
181	183	switch (fetch.lazy_status) {
...	...	@@ -207,7 +209,7 @@ pub const JobQueue = struct {
207	209	try buf.writer().print(
208	210	\\ pub const build_zig = @import("{}");
209	211	\\
210		, .{std.zig.fmtEscapes(&hash)});
	212	, .{std.zig.fmtEscapes(hash_slice)});
211	213	}
212	214
213	215	if (fetch.manifest) \|*manifest\| {
...	...	@@ -219,7 +221,7 @@ pub const JobQueue = struct {
219	221	const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue;
220	222	try buf.writer().print(
221	223	" .{{ \"{}\", \"{}\" }},\n",
222		.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) },
	224	.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) },
223	225	);
224	226	}
225	227
...	...	@@ -251,7 +253,7 @@ pub const JobQueue = struct {
251	253	const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue;
252	254	try buf.writer().print(
253	255	" .{{ \"{}\", \"{}\" }},\n",
254		.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) },
	256	.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) },
255	257	);
256	258	}
257	259	try buf.appendSlice("};\n");
...	...	@@ -283,7 +285,7 @@ pub const Location = union(enum) {
283	285	url: []const u8,
284	286	/// If this is null it means the user omitted the hash field from a dependency.
285	287	/// It will be an error but the logic should still fetch and print the discovered hash.
286		hash: ?Manifest.MultiHashHexDigest,
	288	hash: ?Package.Hash,
287	289	};
288	290	};
289	291
...	...	@@ -325,9 +327,11 @@ pub fn run(f: *Fetch) RunError!void {
325	327	// "p/$hash/foo", with possibly more directories after "foo".
326	328	// We want to fail unless the resolved relative path has a
327	329	// prefix of "p/$hash/".
328		const digest_len = @typeInfo(Manifest.MultiHashHexDigest).array.len;
329	330	const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len;
330		const expected_prefix = f.parent_package_root.sub_path[0 .. prefix_len + digest_len];
	331	const parent_sub_path = f.parent_package_root.sub_path;
	332	const end = std.mem.indexOfScalarPos(u8, parent_sub_path, prefix_len, fs.path.sep) orelse
	333	parent_sub_path.len;
	334	const expected_prefix = parent_sub_path[prefix_len..end];
331	335	if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) {
332	336	return f.fail(
333	337	f.location_tok,
...	...	@@ -367,9 +371,13 @@ pub fn run(f: *Fetch) RunError!void {
367	371	},
368	372	};
369	373
370		const s = fs.path.sep_str;
371	374	if (remote.hash) \|expected_hash\| {
372		const prefixed_pkg_sub_path = "p" ++ s ++ expected_hash;
	375	var prefixed_pkg_sub_path_buffer: [100]u8 = undefined;
	376	prefixed_pkg_sub_path_buffer[0] = 'p';
	377	prefixed_pkg_sub_path_buffer[1] = fs.path.sep;
	378	const hash_slice = expected_hash.toSlice();
	379	@memcpy(prefixed_pkg_sub_path_buffer[2..][0..hash_slice.len], hash_slice);
	380	const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len];
373	381	const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;
374	382	const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];
375	383	if (cache_root.handle.access(pkg_sub_path, .{})) \|_\| {
...	...	@@ -437,7 +445,7 @@ fn runResource(
437	445	f: *Fetch,
438	446	uri_path: []const u8,
439	447	resource: *Resource,
440		remote_hash: ?Manifest.MultiHashHexDigest,
	448	remote_hash: ?Package.Hash,
441	449	) RunError!void {
442	450	defer resource.deinit();
443	451	const arena = f.arena.allocator();
...	...	@@ -499,7 +507,7 @@ fn runResource(
499	507	// Empty directories have already been omitted by `unpackResource`.
500	508	// Compute the package hash based on the remaining files in the temporary
501	509	// directory.
502		f.actual_hash = try computeHash(f, pkg_path, filter);
	510	f.computed_hash = try computeHash(f, pkg_path, filter);
503	511
504	512	break :blk if (unpack_result.root_dir.len > 0)
505	513	try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir })
...	...	@@ -507,6 +515,8 @@ fn runResource(
507	515	tmp_dir_sub_path;
508	516	};
509	517
	518	const computed_package_hash = computedPackageHash(f);
	519
510	520	// Rename the temporary directory into the global zig package cache
511	521	// directory. If the hash already exists, delete the temporary directory
512	522	// and leave the zig package cache directory untouched as it may be in use
...	...	@@ -515,7 +525,7 @@ fn runResource(
515	525
516	526	f.package_root = .{
517	527	.root_dir = cache_root,
518		.sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)),
	528	.sub_path = try std.fmt.allocPrint(arena, "p" ++ s ++ "{s}", .{computed_package_hash.toSlice()}),
519	529	};
520	530	renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch \|err\| {
521	531	const src = try cache_root.join(arena, &.{tmp_dir_sub_path});
...	...	@@ -534,13 +544,22 @@ fn runResource(
534	544	// Validate the computed hash against the expected hash. If invalid, this
535	545	// job is done.
536	546
537		const actual_hex = Manifest.hexDigest(f.actual_hash);
538	547	if (remote_hash) \|declared_hash\| {
539		if (!std.mem.eql(u8, &declared_hash, &actual_hex)) {
540		return f.fail(f.hash_tok, try eb.printString(
541		"hash mismatch: manifest declares {s} but the fetched package has {s}",
542		.{ declared_hash, actual_hex },
543		));
	548	if (declared_hash.isOld()) {
	549	const actual_hex = Package.multiHashHexDigest(f.computed_hash.digest);
	550	if (!std.mem.eql(u8, declared_hash.toSlice(), &actual_hex)) {
	551	return f.fail(f.hash_tok, try eb.printString(
	552	"hash mismatch: manifest declares {s} but the fetched package has {s}",
	553	.{ declared_hash.toSlice(), actual_hex },
	554	));
	555	}
	556	} else {
	557	if (!computed_package_hash.eql(&declared_hash)) {
	558	return f.fail(f.hash_tok, try eb.printString(
	559	"hash mismatch: manifest declares {s} but the fetched package has {s}",
	560	.{ declared_hash.toSlice(), computed_package_hash.toSlice() },
	561	));
	562	}
544	563	}
545	564	} else if (!f.omit_missing_hash_error) {
546	565	const notes_len = 1;
...	...	@@ -551,7 +570,7 @@ fn runResource(
551	570	});
552	571	const notes_start = try eb.reserveNotes(notes_len);
553	572	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
554		.msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),
	573	.msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}),
555	574	}));
556	575	return error.FetchFailed;
557	576	}
...	...	@@ -562,6 +581,16 @@ fn runResource(
562	581	return queueJobsForDeps(f);
563	582	}
564	583
	584	pub fn computedPackageHash(f: *const Fetch) Package.Hash {
	585	const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32);
	586	if (f.manifest) \|man\| {
	587	var version_buffer: [32]u8 = undefined;
	588	const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{}", .{man.version}) catch &version_buffer;
	589	return .init(f.computed_hash.digest, man.name, version, saturated_size);
	590	}
	591	return .initNaked(f.computed_hash.digest, saturated_size);
	592	}
	593
565	594	/// `computeHash` gets a free check for the existence of `build.zig`, but when
566	595	/// not computing a hash, we need to do a syscall to check for it.
567	596	fn checkBuildFileExistence(f: *Fetch) RunError!void {
...	...	@@ -673,9 +702,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
673	702	.url = url,
674	703	.hash = h: {
675	704	const h = dep.hash orelse break :h null;
676		const digest_len = @typeInfo(Manifest.MultiHashHexDigest).array.len;
677		const multihash_digest = h[0..digest_len].*;
678		const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
	705	const pkg_hash: Package.Hash = .fromSlice(h);
	706	const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
679	707	if (gop.found_existing) {
680	708	if (!dep.lazy) {
681	709	gop.value_ptr.*.lazy_status = .eager;
...	...	@@ -683,15 +711,15 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
683	711	continue;
684	712	}
685	713	gop.value_ptr.* = new_fetch;
686		break :h multihash_digest;
	714	break :h pkg_hash;
687	715	},
688	716	} },
689	717	.path => \|rel_path\| l: {
690	718	// This might produce an invalid path, which is checked for
691	719	// at the beginning of run().
692	720	const new_root = try f.package_root.resolvePosix(parent_arena, rel_path);
693		const multihash_digest = relativePathDigest(new_root, cache_root);
694		const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
	721	const pkg_hash = relativePathDigest(new_root, cache_root);
	722	const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
695	723	if (gop.found_existing) {
696	724	if (!dep.lazy) {
697	725	gop.value_ptr.*.lazy_status = .eager;
...	...	@@ -724,7 +752,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
724	752	.error_bundle = undefined,
725	753	.manifest = null,
726	754	.manifest_ast = undefined,
727		.actual_hash = undefined,
	755	.computed_hash = undefined,
728	756	.has_build_zig = false,
729	757	.oom_flag = false,
730	758	.latest_commit = null,
...	...	@@ -746,11 +774,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
746	774	}
747	775	}
748	776
749		pub fn relativePathDigest(
750		pkg_root: Cache.Path,
751		cache_root: Cache.Directory,
752		) Manifest.MultiHashHexDigest {
753		var hasher = Manifest.Hash.init(.{});
	777	pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash {
	778	var hasher = Package.Hash.Algo.init(.{});
754	779	// This hash is a tuple of:
755	780	// * whether it relative to the global cache directory or to the root package
756	781	// * the relative file path from there to the build root of the package
...	...	@@ -759,7 +784,7 @@ pub fn relativePathDigest(
759	784	else
760	785	&package_hash_prefix_project);
761	786	hasher.update(pkg_root.sub_path);
762		return Manifest.hexDigest(hasher.finalResult());
	787	return .fromSlice(&hasher.finalResult());
763	788	}
764	789
765	790	pub fn workerRun(f: *Fetch, prog_name: []const u8) void {
...	...	@@ -1387,11 +1412,7 @@ fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void
1387	1412	}
1388	1413	}
1389	1414
1390		pub fn renameTmpIntoCache(
1391		cache_dir: fs.Dir,
1392		tmp_dir_sub_path: []const u8,
1393		dest_dir_sub_path: []const u8,
1394		) !void {
	1415	pub fn renameTmpIntoCache(cache_dir: fs.Dir, tmp_dir_sub_path: []const u8, dest_dir_sub_path: []const u8) !void {
1395	1416	assert(dest_dir_sub_path[1] == fs.path.sep);
1396	1417	var handled_missing_dir = false;
1397	1418	while (true) {
...	...	@@ -1417,16 +1438,17 @@ pub fn renameTmpIntoCache(
1417	1438	}
1418	1439	}
1419	1440
	1441	const ComputedHash = struct {
	1442	digest: Package.Hash.Digest,
	1443	total_size: u64,
	1444	};
	1445
1420	1446	/// Assumes that files not included in the package have already been filtered
1421	1447	/// prior to calling this function. This ensures that files not protected by
1422	1448	/// the hash are not present on the file system. Empty directories are *not
1423	1449	/// hashed* and must not be present on the file system when calling this
1424	1450	/// function.
1425		fn computeHash(
1426		f: *Fetch,
1427		pkg_path: Cache.Path,
1428		filter: Filter,
1429		) RunError!Manifest.Digest {
	1451	fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash {
1430	1452	// All the path name strings need to be in memory for sorting.
1431	1453	const arena = f.arena.allocator();
1432	1454	const gpa = f.arena.child_allocator;
...	...	@@ -1449,6 +1471,9 @@ fn computeHash(
1449	1471	var walker = try root_dir.walk(gpa);
1450	1472	defer walker.deinit();
1451	1473
	1474	// Total number of bytes of file contents included in the package.
	1475	var total_size: u64 = 0;
	1476
1452	1477	{
1453	1478	// The final hash will be a hash of each file hashed independently. This
1454	1479	// allows hashing in parallel.
...	...	@@ -1506,6 +1531,7 @@ fn computeHash(
1506	1531	.kind = kind,
1507	1532	.hash = undefined, // to be populated by the worker
1508	1533	.failure = undefined, // to be populated by the worker
	1534	.size = undefined, // to be populated by the worker
1509	1535	};
1510	1536	thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file });
1511	1537	try all_files.append(hashed_file);
...	...	@@ -1544,7 +1570,7 @@ fn computeHash(
1544	1570
1545	1571	std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
1546	1572
1547		var hasher = Manifest.Hash.init(.{});
	1573	var hasher = Package.Hash.Algo.init(.{});
1548	1574	var any_failures = false;
1549	1575	for (all_files.items) \|hashed_file\| {
1550	1576	hashed_file.failure catch \|err\| {
...	...	@@ -1556,6 +1582,7 @@ fn computeHash(
1556	1582	});
1557	1583	};
1558	1584	hasher.update(&hashed_file.hash);
	1585	total_size += hashed_file.size;
1559	1586	}
1560	1587	for (deleted_files.items) \|deleted_file\| {
1561	1588	deleted_file.failure catch \|err\| {
...	...	@@ -1580,7 +1607,10 @@ fn computeHash(
1580	1607	};
1581	1608	}
1582	1609
1583		return hasher.finalResult();
	1610	return .{
	1611	.digest = hasher.finalResult(),
	1612	.total_size = total_size,
	1613	};
1584	1614	}
1585	1615
1586	1616	fn dumpHashInfo(all_files: []const *const HashedFile) !void {
...	...	@@ -1609,8 +1639,9 @@ fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void {
1609	1639
1610	1640	fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
1611	1641	var buf: [8000]u8 = undefined;
1612		var hasher = Manifest.Hash.init(.{});
	1642	var hasher = Package.Hash.Algo.init(.{});
1613	1643	hasher.update(hashed_file.normalized_path);
	1644	var file_size: u64 = 0;
1614	1645
1615	1646	switch (hashed_file.kind) {
1616	1647	.file => {
...	...	@@ -1622,6 +1653,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
1622	1653	while (true) {
1623	1654	const bytes_read = try file.read(&buf);
1624	1655	if (bytes_read == 0) break;
	1656	file_size += bytes_read;
1625	1657	hasher.update(buf[0..bytes_read]);
1626	1658	file_header.update(buf[0..bytes_read]);
1627	1659	}
...	...	@@ -1641,6 +1673,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
1641	1673	},
1642	1674	}
1643	1675	hasher.final(&hashed_file.hash);
	1676	hashed_file.size = file_size;
1644	1677	}
1645	1678
1646	1679	fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {
...	...	@@ -1667,9 +1700,10 @@ const DeletedFile = struct {
1667	1700	const HashedFile = struct {
1668	1701	fs_path: []const u8,
1669	1702	normalized_path: []const u8,
1670		hash: Manifest.Digest,
	1703	hash: Package.Hash.Digest,
1671	1704	failure: Error!void,
1672	1705	kind: Kind,
	1706	size: u64,
1673	1707
1674	1708	const Error =
1675	1709	fs.File.OpenError \|\|
...	...	@@ -1744,12 +1778,8 @@ const Filter = struct {
1744	1778	}
1745	1779	};
1746	1780
1747		pub fn depDigest(
1748		pkg_root: Cache.Path,
1749		cache_root: Cache.Directory,
1750		dep: Manifest.Dependency,
1751		) ?Manifest.MultiHashHexDigest {
1752		if (dep.hash) \|h\| return h[0..Manifest.multihash_hex_digest_len].*;
	1781	pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash {
	1782	if (dep.hash) \|h\| return .fromSlice(h);
1753	1783
1754	1784	switch (dep.location) {
1755	1785	.url => return null,
...	...	@@ -2137,7 +2167,7 @@ test "tarball with excluded duplicate paths" {
2137	2167	defer fb.deinit();
2138	2168	try fetch.run();
2139	2169
2140		const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
	2170	const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest);
2141	2171	try std.testing.expectEqualStrings(
2142	2172	"12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da",
2143	2173	&hex_digest,
...	...	@@ -2181,7 +2211,7 @@ test "tarball without root folder" {
2181	2211	defer fb.deinit();
2182	2212	try fetch.run();
2183	2213
2184		const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
	2214	const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest);
2185	2215	try std.testing.expectEqualStrings(
2186	2216	"12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793",
2187	2217	&hex_digest,
...	...	@@ -2222,7 +2252,7 @@ test "set executable bit based on file content" {
2222	2252	try fetch.run();
2223	2253	try std.testing.expectEqualStrings(
2224	2254	"1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3",
2225		&Manifest.hexDigest(fetch.actual_hash),
	2255	&Package.multiHashHexDigest(fetch.computed_hash.digest),
2226	2256	);
2227	2257
2228	2258	var out = try fb.packageDir();
...	...	@@ -2304,7 +2334,7 @@ const TestFetchBuilder = struct {
2304	2334	.error_bundle = undefined,
2305	2335	.manifest = null,
2306	2336	.manifest_ast = undefined,
2307		.actual_hash = undefined,
	2337	.computed_hash = undefined,
2308	2338	.has_build_zig = false,
2309	2339	.oom_flag = false,
2310	2340	.latest_commit = null,

src/Package/Manifest.zig+3-66

...	...	@@ -5,15 +5,10 @@ const Allocator = std.mem.Allocator;
5	5	const assert = std.debug.assert;
6	6	const Ast = std.zig.Ast;
7	7	const testing = std.testing;
8		const hex_charset = std.fmt.hex_charset;
	8	const Package = @import("../Package.zig");
9	9
10	10	pub const max_bytes = 10 * 1024 * 1024;
11	11	pub const basename = "build.zig.zon";
12		pub const Hash = std.crypto.hash.sha2.Sha256;
13		pub const Digest = [Hash.digest_length]u8;
14		pub const multihash_len = 1 + 1 + Hash.digest_length;
15		pub const multihash_hex_digest_len = 2 * multihash_len;
16		pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
17	12
18	13	pub const Dependency = struct {
19	14	location: Location,
...	...	@@ -38,35 +33,6 @@ pub const ErrorMessage = struct {
38	33	off: u32,
39	34	};
40	35
41		pub const MultihashFunction = enum(u16) {
42		identity = 0x00,
43		sha1 = 0x11,
44		@"sha2-256" = 0x12,
45		@"sha2-512" = 0x13,
46		@"sha3-512" = 0x14,
47		@"sha3-384" = 0x15,
48		@"sha3-256" = 0x16,
49		@"sha3-224" = 0x17,
50		@"sha2-384" = 0x20,
51		@"sha2-256-trunc254-padded" = 0x1012,
52		@"sha2-224" = 0x1013,
53		@"sha2-512-224" = 0x1014,
54		@"sha2-512-256" = 0x1015,
55		@"blake2b-256" = 0xb220,
56		_,
57		};
58
59		pub const multihash_function: MultihashFunction = switch (Hash) {
60		std.crypto.hash.sha2.Sha256 => .@"sha2-256",
61		else => @compileError("unreachable"),
62		};
63		comptime {
64		// We avoid unnecessary uleb128 code in hexDigest by asserting here the
65		// values are small enough to be contained in the one-byte encoding.
66		assert(@intFromEnum(multihash_function) < 127);
67		assert(Hash.digest_length < 127);
68		}
69
70	36	name: []const u8,
71	37	version: std.SemanticVersion,
72	38	version_node: Ast.Node.Index,
...	...	@@ -164,22 +130,6 @@ pub fn copyErrorsIntoBundle(
164	130	}
165	131	}
166	132
167		pub fn hexDigest(digest: Digest) MultiHashHexDigest {
168		var result: MultiHashHexDigest = undefined;
169
170		result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
171		result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
172
173		result[2] = hex_charset[Hash.digest_length >> 4];
174		result[3] = hex_charset[Hash.digest_length & 15];
175
176		for (digest, 0..) \|byte, i\| {
177		result[4 + i * 2] = hex_charset[byte >> 4];
178		result[5 + i * 2] = hex_charset[byte & 15];
179		}
180		return result;
181		}
182
183	133	const Parse = struct {
184	134	gpa: Allocator,
185	135	ast: Ast,
...	...	@@ -421,21 +371,8 @@ const Parse = struct {
421	371	const tok = main_tokens[node];
422	372	const h = try parseString(p, node);
423	373
424		if (h.len >= 2) {
425		const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch \|err\| {
426		return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{
427		@errorName(err),
428		});
429		};
430		if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) {
431		return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{});
432		}
433		}
434
435		if (h.len != multihash_hex_digest_len) {
436		return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{
437		multihash_hex_digest_len, h.len,
438		});
	374	if (h.len > Package.Hash.max_len) {
	375	return fail(p, tok, "hash length exceeds maximum: {d}", .{h.len});
439	376	}
440	377
441	378	return h;

src/main.zig+18-17

...	...	@@ -5197,7 +5197,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5197	5197	.error_bundle = undefined,
5198	5198	.manifest = null,
5199	5199	.manifest_ast = undefined,
5200		.actual_hash = undefined,
	5200	.computed_hash = undefined,
5201	5201	.has_build_zig = true,
5202	5202	.oom_flag = false,
5203	5203	.latest_commit = null,
...	...	@@ -5244,13 +5244,14 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5244	5244	const hashes = job_queue.table.keys();
5245	5245	const fetches = job_queue.table.values();
5246	5246	try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len));
5247		for (hashes, fetches) \|hash, f\| {
	5247	for (hashes, fetches) \|*hash, f\| {
5248	5248	if (f == &fetch) {
5249	5249	// The first one is a dummy package for the current project.
5250	5250	continue;
5251	5251	}
5252	5252	if (!f.has_build_zig)
5253	5253	continue;
	5254	const hash_slice = hash.toSlice();
5254	5255	const m = try Package.Module.create(arena, .{
5255	5256	.global_cache_directory = global_cache_directory,
5256	5257	.paths = .{
...	...	@@ -5260,7 +5261,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5260	5261	.fully_qualified_name = try std.fmt.allocPrint(
5261	5262	arena,
5262	5263	"root.@dependencies.{s}",
5263		.{&hash},
	5264	.{hash_slice},
5264	5265	),
5265	5266	.cc_argv = &.{},
5266	5267	.inherited = .{},
...	...	@@ -5269,7 +5270,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5269	5270	.builtin_mod = builtin_mod,
5270	5271	.builtin_modules = null, // `builtin_mod` is specified
5271	5272	});
5272		const hash_cloned = try arena.dupe(u8, &hash);
	5273	const hash_cloned = try arena.dupe(u8, hash_slice);
5273	5274	deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m);
5274	5275	f.module = m;
5275	5276	}
...	...	@@ -5385,23 +5386,22 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5385	5386	var any_errors = false;
5386	5387	while (it.next()) \|hash\| {
5387	5388	if (hash.len == 0) continue;
5388		const digest_len = @typeInfo(Package.Manifest.MultiHashHexDigest).array.len;
5389		if (hash.len != digest_len) {
5390		std.log.err("invalid digest (length {d} instead of {d}): '{s}'", .{
5391		hash.len, digest_len, hash,
	5389	if (hash.len > Package.Hash.max_len) {
	5390	std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{
	5391	hash.len, hash,
5392	5392	});
5393	5393	any_errors = true;
5394	5394	continue;
5395	5395	}
5396		try unlazy_set.put(arena, hash[0..digest_len].*, {});
	5396	try unlazy_set.put(arena, .fromSlice(hash), {});
5397	5397	}
5398	5398	if (any_errors) process.exit(3);
5399	5399	if (system_pkg_dir_path) \|p\| {
5400	5400	// In this mode, the system needs to provide these packages; they
5401	5401	// cannot be fetched by Zig.
5402		for (unlazy_set.keys()) \|hash\| {
	5402	for (unlazy_set.keys()) \|*hash\| {
5403	5403	std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{
5404		p, hash,
	5404	p, hash.toSlice(),
5405	5405	});
5406	5406	}
5407	5407	std.log.info("remote package fetching disabled due to --system mode", .{});
...	...	@@ -7097,7 +7097,7 @@ fn cmdFetch(
7097	7097	.error_bundle = undefined,
7098	7098	.manifest = null,
7099	7099	.manifest_ast = undefined,
7100		.actual_hash = undefined,
	7100	.computed_hash = undefined,
7101	7101	.has_build_zig = false,
7102	7102	.oom_flag = false,
7103	7103	.latest_commit = null,
...	...	@@ -7117,14 +7117,15 @@ fn cmdFetch(
7117	7117	process.exit(1);
7118	7118	}
7119	7119
7120		const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
	7120	const package_hash = fetch.computedPackageHash();
	7121	const package_hash_slice = package_hash.toSlice();
7121	7122
7122	7123	root_prog_node.end();
7123	7124	root_prog_node = .{ .index = .none };
7124	7125
7125	7126	const name = switch (save) {
7126	7127	.no => {
7127		try io.getStdOut().writeAll(hex_digest ++ "\n");
	7128	try io.getStdOut().writer().print("{s}\n", .{package_hash_slice});
7128	7129	return cleanExit();
7129	7130	},
7130	7131	.yes, .exact => \|name\| name: {
...	...	@@ -7194,7 +7195,7 @@ fn cmdFetch(
7194	7195	\\ }}
7195	7196	, .{
7196	7197	std.zig.fmtEscapes(saved_path_or_url),
7197		std.zig.fmtEscapes(&hex_digest),
	7198	std.zig.fmtEscapes(package_hash_slice),
7198	7199	});
7199	7200
7200	7201	const new_node_text = try std.fmt.allocPrint(arena, ".{p_} = {s},\n", .{
...	...	@@ -7213,7 +7214,7 @@ fn cmdFetch(
7213	7214	if (dep.hash) \|h\| {
7214	7215	switch (dep.location) {
7215	7216	.url => \|u\| {
7216		if (mem.eql(u8, h, &hex_digest) and mem.eql(u8, u, saved_path_or_url)) {
	7217	if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) {
7217	7218	std.log.info("existing dependency named '{s}' is up-to-date", .{name});
7218	7219	process.exit(0);
7219	7220	}
...	...	@@ -7230,7 +7231,7 @@ fn cmdFetch(
7230	7231	const hash_replace = try std.fmt.allocPrint(
7231	7232	arena,
7232	7233	"\"{}\"",
7233		.{std.zig.fmtEscapes(&hex_digest)},
	7234	.{std.zig.fmtEscapes(package_hash_slice)},
7234	7235	);
7235	7236
7236	7237	warn("overwriting existing dependency named '{s}'", .{name});