Commit 6c3cbb0c87a33f4ae408874f6ceb40e372b65914 | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-02-28 04:40:05-05:00
committer	GitHub <noreply@github.com> 2025-02-28 04:40:05-05:00
log	6c3cbb0c87a33f4ae408874f6ceb40e372b65914
tree	2892f3585939f9f66dcf8e78fcc8606ad861ae6a
parent	6b6c1b1b0e04d70a4917f073a6a8bc87a5e8abb3
parent	de43f5eb6ae4a569efe15e3469b3de76a86d9cd1
signature	Signed by PGP key B5690EEEBB952194

Merge pull request #22994 from ziglang/newhash

implement new package hash format: `$name-$semver-$hash`

10 files changed, 526 insertions(+), 182 deletions(-)

build.zig.zon
doc
- build.zig.zon.md
lib
- init
  - build.zig
  - build.zig.zon
  - src
    main.zig
- std
  - array_list.zig
src
- Package.zig
- Package
  - Fetch.zig
  - Manifest.zig
- main.zig

build.zig.zon+2-1

...	...	@@ -1,7 +1,7 @@
1	1	// The Zig compiler is not intended to be consumed as a package.
2	2	// The sole purpose of this manifest file is to test the compiler.
3	3	.{
4		.name = "zig",
	4	.name = .zig,
5	5	.version = "0.0.0",
6	6	.dependencies = .{
7	7	.standalone_test_cases = .{
...	...	@@ -12,4 +12,5 @@
12	12	},
13	13	},
14	14	.paths = .{""},
	15	.fingerprint = 0xc1ce108124179e16,
15	16	}

doc/build.zig.zon.md+31-1

...	...	@@ -10,7 +10,7 @@ build.zig.
10	10
11	11	### `name`
12	12
13		String. Required.
	13	Enum literal. Required.
14	14
15	15	This is the default name used by packages depending on this one. For example,
16	16	when a user runs `zig fetch --save <url>`, this field is used as the key in the
...	...	@@ -20,12 +20,42 @@ will stick with this provided value.
20	20	It is redundant to include "zig" in this name because it is already within the
21	21	Zig package namespace.
22	22
	23	Must be a valid bare Zig identifier (don't `@` me), limited to 32 bytes.
	24
	25	Together with `fingerprint`, this represents a globally unique package identifier.
	26
	27	### `fingerprint`
	28
	29	Together with `name`, this represents a globally unique package identifier. This
	30	field is auto-initialized by the toolchain when the package is first created,
	31	and then never changes. This allows Zig to unambiguously detect when one
	32	package is an updated version of another.
	33
	34	When forking a Zig project, this fingerprint should be regenerated if the upstream
	35	project is still maintained. Otherwise, the fork is hostile, attempting to
	36	take control over the original project's identity. The fingerprint can be regenerated
	37	by deleting the field and running `zig build`.
	38
	39	This 64-bit integer is the combination of a 32-bit id component and a 32-bit
	40	checksum.
	41
	42	The id component within the fingerprint has these restrictions:
	43
	44	`0x00000000` is reserved for legacy packages.
	45
	46	`0xffffffff` is reserved to represent "naked" packages.
	47
	48	The checksum is computed from `name` and serves to protect Zig users from
	49	accidental id collisions.
	50
23	51	### `version`
24	52
25	53	String. Required.
26	54
27	55	[semver](https://semver.org/)
28	56
	57	Limited to 32 bytes.
	58
29	59	### `minimum_zig_version`
30	60
31	61	String. Optional.

lib/init/build.zig+3-3

...	...	@@ -42,14 +42,14 @@ pub fn build(b: *std.Build) void {
42	42	// Modules can depend on one another using the `std.Build.Module.addImport` function.
43	43	// This is what allows Zig source code to use `@import("foo")` where 'foo' is not a
44	44	// file path. In this case, we set up `exe_mod` to import `lib_mod`.
45		exe_mod.addImport("$_lib", lib_mod);
	45	exe_mod.addImport(".NAME_lib", lib_mod);
46	46
47	47	// Now, we will create a static library based on the module we created above.
48	48	// This creates a `std.Build.Step.Compile`, which is the build step responsible
49	49	// for actually invoking the compiler.
50	50	const lib = b.addLibrary(.{
51	51	.linkage = .static,
52		.name = "$",
	52	.name = ".NAME",
53	53	.root_module = lib_mod,
54	54	});
55	55
...	...	@@ -61,7 +61,7 @@ pub fn build(b: *std.Build) void {
61	61	// This creates another `std.Build.Step.Compile`, but this one builds an executable
62	62	// rather than a static library.
63	63	const exe = b.addExecutable(.{
64		.name = "$",
	64	.name = ".NAME",
65	65	.root_module = exe_mod,
66	66	});
67	67

lib/init/build.zig.zon+19-1

...	...	@@ -6,12 +6,30 @@
6	6	//
7	7	// It is redundant to include "zig" in this name because it is already
8	8	// within the Zig package namespace.
9		.name = "$",
	9	.name = .LITNAME,
10	10
11	11	// This is a [Semantic Version](https://semver.org/).
12	12	// In a future version of Zig it will be used for package deduplication.
13	13	.version = "0.0.0",
14	14
	15	// Together with name, this represents a globally unique package
	16	// identifier. This field is generated by the Zig toolchain when the
	17	// package is first created, and then never changes. This allows
	18	// unambiguous detection of one package being an updated version of
	19	// another.
	20	//
	21	// When forking a Zig project, this id should be regenerated (delete the
	22	// field and run `zig build`) if the upstream project is still maintained.
	23	// Otherwise, the fork is hostile, attempting to take control over the
	24	// original project's identity. Thus it is recommended to leave the comment
	25	// on the following line intact, so that it shows up in code reviews that
	26	// modify the field.
	27	.fingerprint = .FINGERPRINT, // Changing this has security and trust implications.
	28
	29	// Tracks the earliest Zig version that the package considers to be a
	30	// supported use case.
	31	.minimum_zig_version = ".ZIGVER",
	32
15	33	// This field is optional.
16	34	// This is currently advisory only; Zig does not yet do anything
17	35	// with this value.

lib/init/src/main.zig+1-1

...	...	@@ -43,4 +43,4 @@ test "fuzz example" {
43	43	const std = @import("std");
44	44
45	45	/// This imports the separate module containing `root.zig`. Take a look in `build.zig` for details.
46		const lib = @import("$_lib");
	46	const lib = @import(".NAME_lib");

lib/std/array_list.zig-7

...	...	@@ -2250,10 +2250,3 @@ test "return OutOfMemory when capacity would exceed maximum usize integer value"
2250	2250	try testing.expectError(error.OutOfMemory, list.ensureUnusedCapacity(2));
2251	2251	}
2252	2252	}
2253
2254		test "ArrayListAligned with non-native alignment compiles unusedCapabitySlice" {
2255		var list = ArrayListAligned(u8, 4).init(testing.allocator);
2256		defer list.deinit();
2257		try list.appendNTimes(1, 4);
2258		_ = list.unusedCapacitySlice();
2259		}

src/Package.zig+192

...	...	@@ -1,8 +1,200 @@
	1	const std = @import("std");
	2	const assert = std.debug.assert;
	3
1	4	pub const Module = @import("Package/Module.zig");
2	5	pub const Fetch = @import("Package/Fetch.zig");
3	6	pub const build_zig_basename = "build.zig";
4	7	pub const Manifest = @import("Package/Manifest.zig");
5	8
	9	pub const multihash_len = 1 + 1 + Hash.Algo.digest_length;
	10	pub const multihash_hex_digest_len = 2 * multihash_len;
	11	pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
	12
	13	pub const Fingerprint = packed struct(u64) {
	14	id: u32,
	15	checksum: u32,
	16
	17	pub fn generate(name: []const u8) Fingerprint {
	18	return .{
	19	.id = std.crypto.random.intRangeLessThan(u32, 1, 0xffffffff),
	20	.checksum = std.hash.Crc32.hash(name),
	21	};
	22	}
	23
	24	pub fn validate(n: Fingerprint, name: []const u8) bool {
	25	switch (n.id) {
	26	0x00000000, 0xffffffff => return false,
	27	else => return std.hash.Crc32.hash(name) == n.checksum,
	28	}
	29	}
	30
	31	pub fn int(n: Fingerprint) u64 {
	32	return @bitCast(n);
	33	}
	34	};
	35
	36	/// A user-readable, file system safe hash that identifies an exact package
	37	/// snapshot, including file contents.
	38	///
	39	/// The hash is not only to prevent collisions but must resist attacks where
	40	/// the adversary fully controls the contents being hashed. Thus, it contains
	41	/// a full SHA-256 digest.
	42	///
	43	/// This data structure can be used to store the legacy hash format too. Legacy
	44	/// hash format is scheduled to be removed after 0.14.0 is tagged.
	45	///
	46	/// There's also a third way this structure is used. When using path rather than
	47	/// hash, a unique hash is still needed, so one is computed based on the path.
	48	pub const Hash = struct {
	49	/// Maximum size of a package hash. Unused bytes at the end are
	50	/// filled with zeroes.
	51	bytes: [max_len]u8,
	52
	53	pub const Algo = std.crypto.hash.sha2.Sha256;
	54	pub const Digest = [Algo.digest_length]u8;
	55
	56	/// Example: "nnnn-vvvv-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh"
	57	pub const max_len = 32 + 1 + 32 + 1 + (32 + 32 + 200) / 6;
	58
	59	pub fn fromSlice(s: []const u8) Hash {
	60	assert(s.len <= max_len);
	61	var result: Hash = undefined;
	62	@memcpy(result.bytes[0..s.len], s);
	63	@memset(result.bytes[s.len..], 0);
	64	return result;
	65	}
	66
	67	pub fn toSlice(ph: *const Hash) []const u8 {
	68	var end: usize = ph.bytes.len;
	69	while (true) {
	70	end -= 1;
	71	if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1];
	72	}
	73	}
	74
	75	pub fn eql(a: const Hash, b: const Hash) bool {
	76	return std.mem.eql(u8, &a.bytes, &b.bytes);
	77	}
	78
	79	/// Distinguishes whether the legacy multihash format is being stored here.
	80	pub fn isOld(h: *const Hash) bool {
	81	if (h.bytes.len < 2) return false;
	82	const their_multihash_func = std.fmt.parseInt(u8, h.bytes[0..2], 16) catch return false;
	83	if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) return false;
	84	if (h.toSlice().len != multihash_hex_digest_len) return false;
	85	return std.mem.indexOfScalar(u8, &h.bytes, '-') == null;
	86	}
	87
	88	test isOld {
	89	const h: Hash = .fromSlice("1220138f4aba0c01e66b68ed9e1e1e74614c06e4743d88bc58af4f1c3dd0aae5fea7");
	90	try std.testing.expect(h.isOld());
	91	}
	92
	93	/// Produces "$name-$semver-$hashplus".
	94	/// * name is the name field from build.zig.zon, asserted to be at most 32
	95	/// bytes and assumed be a valid zig identifier
	96	/// * semver is the version field from build.zig.zon, asserted to be at
	97	/// most 32 bytes
	98	/// * hashplus is the following 33-byte array, base64 encoded using -_ to make
	99	/// it filesystem safe:
	100	/// - (4 bytes) LE u32 Package ID
	101	/// - (4 bytes) LE u32 total decompressed size in bytes, overflow saturated
	102	/// - (25 bytes) truncated SHA-256 digest of hashed files of the package
	103	pub fn init(digest: Digest, name: []const u8, ver: []const u8, id: u32, size: u32) Hash {
	104	assert(name.len <= 32);
	105	assert(ver.len <= 32);
	106	var result: Hash = undefined;
	107	var buf: std.ArrayListUnmanaged(u8) = .initBuffer(&result.bytes);
	108	buf.appendSliceAssumeCapacity(name);
	109	buf.appendAssumeCapacity('-');
	110	buf.appendSliceAssumeCapacity(ver);
	111	buf.appendAssumeCapacity('-');
	112	var hashplus: [33]u8 = undefined;
	113	std.mem.writeInt(u32, hashplus[0..4], id, .little);
	114	std.mem.writeInt(u32, hashplus[4..8], size, .little);
	115	hashplus[8..].* = digest[0..25].*;
	116	_ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(44), &hashplus);
	117	@memset(buf.unusedCapacitySlice(), 0);
	118	return result;
	119	}
	120
	121	/// Produces a unique hash based on the path provided. The result should
	122	/// not be user-visible.
	123	pub fn initPath(sub_path: []const u8, is_global: bool) Hash {
	124	var result: Hash = .{ .bytes = @splat(0) };
	125	var i: usize = 0;
	126	if (is_global) {
	127	result.bytes[0] = '/';
	128	i += 1;
	129	}
	130	if (i + sub_path.len <= result.bytes.len) {
	131	@memcpy(result.bytes[i..][0..sub_path.len], sub_path);
	132	return result;
	133	}
	134	var bin_digest: [Algo.digest_length]u8 = undefined;
	135	Algo.hash(sub_path, &bin_digest, .{});
	136	_ = std.fmt.bufPrint(result.bytes[i..], "{}", .{std.fmt.fmtSliceHexLower(&bin_digest)}) catch unreachable;
	137	return result;
	138	}
	139	};
	140
	141	pub const MultihashFunction = enum(u16) {
	142	identity = 0x00,
	143	sha1 = 0x11,
	144	@"sha2-256" = 0x12,
	145	@"sha2-512" = 0x13,
	146	@"sha3-512" = 0x14,
	147	@"sha3-384" = 0x15,
	148	@"sha3-256" = 0x16,
	149	@"sha3-224" = 0x17,
	150	@"sha2-384" = 0x20,
	151	@"sha2-256-trunc254-padded" = 0x1012,
	152	@"sha2-224" = 0x1013,
	153	@"sha2-512-224" = 0x1014,
	154	@"sha2-512-256" = 0x1015,
	155	@"blake2b-256" = 0xb220,
	156	_,
	157	};
	158
	159	pub const multihash_function: MultihashFunction = switch (Hash.Algo) {
	160	std.crypto.hash.sha2.Sha256 => .@"sha2-256",
	161	else => unreachable,
	162	};
	163
	164	pub fn multiHashHexDigest(digest: Hash.Digest) MultiHashHexDigest {
	165	const hex_charset = std.fmt.hex_charset;
	166
	167	var result: MultiHashHexDigest = undefined;
	168
	169	result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
	170	result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
	171
	172	result[2] = hex_charset[Hash.Algo.digest_length >> 4];
	173	result[3] = hex_charset[Hash.Algo.digest_length & 15];
	174
	175	for (digest, 0..) \|byte, i\| {
	176	result[4 + i * 2] = hex_charset[byte >> 4];
	177	result[5 + i * 2] = hex_charset[byte & 15];
	178	}
	179	return result;
	180	}
	181
	182	comptime {
	183	// We avoid unnecessary uleb128 code in hexDigest by asserting here the
	184	// values are small enough to be contained in the one-byte encoding.
	185	assert(@intFromEnum(multihash_function) < 127);
	186	assert(Hash.Algo.digest_length < 127);
	187	}
	188
	189	test Hash {
	190	const example_digest: Hash.Digest = .{
	191	0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87,
	192	0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f,
	193	};
	194	const result: Hash = .init(example_digest, "nasm", "2.16.1-3", 0xcafebabe, 10 * 1024 * 1024);
	195	try std.testing.expectEqualStrings("nasm-2.16.1-3-vrr-ygAAoADH9XG3tOdvPNuHen_d-XeHndOG-nNXmved", result.toSlice());
	196	}
	197
6	198	test {
7	199	_ = Fetch;
8	200	}

src/Package/Fetch.zig+102-75

...	...	@@ -44,6 +44,8 @@ omit_missing_hash_error: bool,
44	44	/// which specifies inclusion rules. This is intended to be true for the first
45	45	/// fetch task and false for the recursive dependencies.
46	46	allow_missing_paths_field: bool,
	47	allow_missing_fingerprint: bool,
	48	allow_name_string: bool,
47	49	/// If true and URL points to a Git repository, will use the latest commit.
48	50	use_latest_commit: bool,
49	51
...	...	@@ -56,7 +58,7 @@ package_root: Cache.Path,
56	58	error_bundle: ErrorBundle.Wip,
57	59	manifest: ?Manifest,
58	60	manifest_ast: std.zig.Ast,
59		actual_hash: Manifest.Digest,
	61	computed_hash: ComputedHash,
60	62	/// Fetch logic notices whether a package has a build.zig file and sets this flag.
61	63	has_build_zig: bool,
62	64	/// Indicates whether the task aborted due to an out-of-memory condition.
...	...	@@ -116,8 +118,8 @@ pub const JobQueue = struct {
116	118	/// as lazy.
117	119	unlazy_set: UnlazySet = .{},
118	120
119		pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);
120		pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, void);
	121	pub const Table = std.AutoArrayHashMapUnmanaged(Package.Hash, *Fetch);
	122	pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Package.Hash, void);
121	123
122	124	pub fn deinit(jq: *JobQueue) void {
123	125	if (jq.all_fetches.items.len == 0) return;
...	...	@@ -160,22 +162,24 @@ pub const JobQueue = struct {
160	162
161	163	// Ensure the generated .zig file is deterministic.
162	164	jq.table.sortUnstable(@as(struct {
163		keys: []const Manifest.MultiHashHexDigest,
	165	keys: []const Package.Hash,
164	166	pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
165		return std.mem.lessThan(u8, &ctx.keys[a_index], &ctx.keys[b_index]);
	167	return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes);
166	168	}
167	169	}, .{ .keys = keys }));
168	170
169		for (keys, jq.table.values()) \|hash, fetch\| {
	171	for (keys, jq.table.values()) \|*hash, fetch\| {
170	172	if (fetch == jq.all_fetches.items[0]) {
171	173	// The first one is a dummy package for the current project.
172	174	continue;
173	175	}
174	176
	177	const hash_slice = hash.toSlice();
	178
175	179	try buf.writer().print(
176	180	\\ pub const {} = struct {{
177	181	\\
178		, .{std.zig.fmtId(&hash)});
	182	, .{std.zig.fmtId(hash_slice)});
179	183
180	184	lazy: {
181	185	switch (fetch.lazy_status) {
...	...	@@ -207,7 +211,7 @@ pub const JobQueue = struct {
207	211	try buf.writer().print(
208	212	\\ pub const build_zig = @import("{}");
209	213	\\
210		, .{std.zig.fmtEscapes(&hash)});
	214	, .{std.zig.fmtEscapes(hash_slice)});
211	215	}
212	216
213	217	if (fetch.manifest) \|*manifest\| {
...	...	@@ -219,7 +223,7 @@ pub const JobQueue = struct {
219	223	const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue;
220	224	try buf.writer().print(
221	225	" .{{ \"{}\", \"{}\" }},\n",
222		.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) },
	226	.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) },
223	227	);
224	228	}
225	229
...	...	@@ -251,7 +255,7 @@ pub const JobQueue = struct {
251	255	const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue;
252	256	try buf.writer().print(
253	257	" .{{ \"{}\", \"{}\" }},\n",
254		.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) },
	258	.{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) },
255	259	);
256	260	}
257	261	try buf.appendSlice("};\n");
...	...	@@ -283,7 +287,7 @@ pub const Location = union(enum) {
283	287	url: []const u8,
284	288	/// If this is null it means the user omitted the hash field from a dependency.
285	289	/// It will be an error but the logic should still fetch and print the discovered hash.
286		hash: ?Manifest.MultiHashHexDigest,
	290	hash: ?Package.Hash,
287	291	};
288	292	};
289	293
...	...	@@ -325,9 +329,11 @@ pub fn run(f: *Fetch) RunError!void {
325	329	// "p/$hash/foo", with possibly more directories after "foo".
326	330	// We want to fail unless the resolved relative path has a
327	331	// prefix of "p/$hash/".
328		const digest_len = @typeInfo(Manifest.MultiHashHexDigest).array.len;
329	332	const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len;
330		const expected_prefix = f.parent_package_root.sub_path[0 .. prefix_len + digest_len];
	333	const parent_sub_path = f.parent_package_root.sub_path;
	334	const end = std.mem.indexOfScalarPos(u8, parent_sub_path, prefix_len, fs.path.sep) orelse
	335	parent_sub_path.len;
	336	const expected_prefix = parent_sub_path[prefix_len..end];
331	337	if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) {
332	338	return f.fail(
333	339	f.location_tok,
...	...	@@ -367,9 +373,13 @@ pub fn run(f: *Fetch) RunError!void {
367	373	},
368	374	};
369	375
370		const s = fs.path.sep_str;
371	376	if (remote.hash) \|expected_hash\| {
372		const prefixed_pkg_sub_path = "p" ++ s ++ expected_hash;
	377	var prefixed_pkg_sub_path_buffer: [Package.Hash.max_len + 2]u8 = undefined;
	378	prefixed_pkg_sub_path_buffer[0] = 'p';
	379	prefixed_pkg_sub_path_buffer[1] = fs.path.sep;
	380	const hash_slice = expected_hash.toSlice();
	381	@memcpy(prefixed_pkg_sub_path_buffer[2..][0..hash_slice.len], hash_slice);
	382	const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len];
373	383	const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;
374	384	const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];
375	385	if (cache_root.handle.access(pkg_sub_path, .{})) \|_\| {
...	...	@@ -437,7 +447,7 @@ fn runResource(
437	447	f: *Fetch,
438	448	uri_path: []const u8,
439	449	resource: *Resource,
440		remote_hash: ?Manifest.MultiHashHexDigest,
	450	remote_hash: ?Package.Hash,
441	451	) RunError!void {
442	452	defer resource.deinit();
443	453	const arena = f.arena.allocator();
...	...	@@ -499,7 +509,7 @@ fn runResource(
499	509	// Empty directories have already been omitted by `unpackResource`.
500	510	// Compute the package hash based on the remaining files in the temporary
501	511	// directory.
502		f.actual_hash = try computeHash(f, pkg_path, filter);
	512	f.computed_hash = try computeHash(f, pkg_path, filter);
503	513
504	514	break :blk if (unpack_result.root_dir.len > 0)
505	515	try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir })
...	...	@@ -507,6 +517,8 @@ fn runResource(
507	517	tmp_dir_sub_path;
508	518	};
509	519
	520	const computed_package_hash = computedPackageHash(f);
	521
510	522	// Rename the temporary directory into the global zig package cache
511	523	// directory. If the hash already exists, delete the temporary directory
512	524	// and leave the zig package cache directory untouched as it may be in use
...	...	@@ -515,7 +527,7 @@ fn runResource(
515	527
516	528	f.package_root = .{
517	529	.root_dir = cache_root,
518		.sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)),
	530	.sub_path = try std.fmt.allocPrint(arena, "p" ++ s ++ "{s}", .{computed_package_hash.toSlice()}),
519	531	};
520	532	renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch \|err\| {
521	533	const src = try cache_root.join(arena, &.{tmp_dir_sub_path});
...	...	@@ -534,13 +546,22 @@ fn runResource(
534	546	// Validate the computed hash against the expected hash. If invalid, this
535	547	// job is done.
536	548
537		const actual_hex = Manifest.hexDigest(f.actual_hash);
538	549	if (remote_hash) \|declared_hash\| {
539		if (!std.mem.eql(u8, &declared_hash, &actual_hex)) {
540		return f.fail(f.hash_tok, try eb.printString(
541		"hash mismatch: manifest declares {s} but the fetched package has {s}",
542		.{ declared_hash, actual_hex },
543		));
	550	if (declared_hash.isOld()) {
	551	const actual_hex = Package.multiHashHexDigest(f.computed_hash.digest);
	552	if (!std.mem.eql(u8, declared_hash.toSlice(), &actual_hex)) {
	553	return f.fail(f.hash_tok, try eb.printString(
	554	"hash mismatch: manifest declares {s} but the fetched package has {s}",
	555	.{ declared_hash.toSlice(), actual_hex },
	556	));
	557	}
	558	} else {
	559	if (!computed_package_hash.eql(&declared_hash)) {
	560	return f.fail(f.hash_tok, try eb.printString(
	561	"hash mismatch: manifest declares {s} but the fetched package has {s}",
	562	.{ declared_hash.toSlice(), computed_package_hash.toSlice() },
	563	));
	564	}
544	565	}
545	566	} else if (!f.omit_missing_hash_error) {
546	567	const notes_len = 1;
...	...	@@ -551,7 +572,7 @@ fn runResource(
551	572	});
552	573	const notes_start = try eb.reserveNotes(notes_len);
553	574	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
554		.msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),
	575	.msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}),
555	576	}));
556	577	return error.FetchFailed;
557	578	}
...	...	@@ -562,6 +583,18 @@ fn runResource(
562	583	return queueJobsForDeps(f);
563	584	}
564	585
	586	pub fn computedPackageHash(f: *const Fetch) Package.Hash {
	587	const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32);
	588	if (f.manifest) \|man\| {
	589	var version_buffer: [32]u8 = undefined;
	590	const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{}", .{man.version}) catch &version_buffer;
	591	return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size);
	592	}
	593	// In the future build.zig.zon fields will be added to allow overriding these values
	594	// for naked tarballs.
	595	return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size);
	596	}
	597
565	598	/// `computeHash` gets a free check for the existence of `build.zig`, but when
566	599	/// not computing a hash, we need to do a syscall to check for it.
567	600	fn checkBuildFileExistence(f: *Fetch) RunError!void {
...	...	@@ -616,11 +649,13 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {
616	649
617	650	f.manifest = try Manifest.parse(arena, ast.*, .{
618	651	.allow_missing_paths_field = f.allow_missing_paths_field,
	652	.allow_missing_fingerprint = f.allow_missing_fingerprint,
	653	.allow_name_string = f.allow_name_string,
619	654	});
620	655	const manifest = &f.manifest.?;
621	656
622	657	if (manifest.errors.len > 0) {
623		const src_path = try eb.printString("{}{s}", .{ pkg_root, Manifest.basename });
	658	const src_path = try eb.printString("{}" ++ fs.path.sep_str ++ "{s}", .{ pkg_root, Manifest.basename });
624	659	try manifest.copyErrorsIntoBundle(ast.*, src_path, eb);
625	660	return error.FetchFailed;
626	661	}
...	...	@@ -673,9 +708,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
673	708	.url = url,
674	709	.hash = h: {
675	710	const h = dep.hash orelse break :h null;
676		const digest_len = @typeInfo(Manifest.MultiHashHexDigest).array.len;
677		const multihash_digest = h[0..digest_len].*;
678		const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
	711	const pkg_hash: Package.Hash = .fromSlice(h);
	712	const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
679	713	if (gop.found_existing) {
680	714	if (!dep.lazy) {
681	715	gop.value_ptr.*.lazy_status = .eager;
...	...	@@ -683,15 +717,15 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
683	717	continue;
684	718	}
685	719	gop.value_ptr.* = new_fetch;
686		break :h multihash_digest;
	720	break :h pkg_hash;
687	721	},
688	722	} },
689	723	.path => \|rel_path\| l: {
690	724	// This might produce an invalid path, which is checked for
691	725	// at the beginning of run().
692	726	const new_root = try f.package_root.resolvePosix(parent_arena, rel_path);
693		const multihash_digest = relativePathDigest(new_root, cache_root);
694		const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
	727	const pkg_hash = relativePathDigest(new_root, cache_root);
	728	const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
695	729	if (gop.found_existing) {
696	730	if (!dep.lazy) {
697	731	gop.value_ptr.*.lazy_status = .eager;
...	...	@@ -718,13 +752,15 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
718	752	.job_queue = f.job_queue,
719	753	.omit_missing_hash_error = false,
720	754	.allow_missing_paths_field = true,
	755	.allow_missing_fingerprint = true,
	756	.allow_name_string = true,
721	757	.use_latest_commit = false,
722	758
723	759	.package_root = undefined,
724	760	.error_bundle = undefined,
725	761	.manifest = null,
726	762	.manifest_ast = undefined,
727		.actual_hash = undefined,
	763	.computed_hash = undefined,
728	764	.has_build_zig = false,
729	765	.oom_flag = false,
730	766	.latest_commit = null,
...	...	@@ -746,20 +782,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
746	782	}
747	783	}
748	784
749		pub fn relativePathDigest(
750		pkg_root: Cache.Path,
751		cache_root: Cache.Directory,
752		) Manifest.MultiHashHexDigest {
753		var hasher = Manifest.Hash.init(.{});
754		// This hash is a tuple of:
755		// * whether it relative to the global cache directory or to the root package
756		// * the relative file path from there to the build root of the package
757		hasher.update(if (pkg_root.root_dir.eql(cache_root))
758		&package_hash_prefix_cached
759		else
760		&package_hash_prefix_project);
761		hasher.update(pkg_root.sub_path);
762		return Manifest.hexDigest(hasher.finalResult());
	785	pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash {
	786	return .initPath(pkg_root.sub_path, pkg_root.root_dir.eql(cache_root));
763	787	}
764	788
765	789	pub fn workerRun(f: *Fetch, prog_name: []const u8) void {
...	...	@@ -1387,11 +1411,7 @@ fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void
1387	1411	}
1388	1412	}
1389	1413
1390		pub fn renameTmpIntoCache(
1391		cache_dir: fs.Dir,
1392		tmp_dir_sub_path: []const u8,
1393		dest_dir_sub_path: []const u8,
1394		) !void {
	1414	pub fn renameTmpIntoCache(cache_dir: fs.Dir, tmp_dir_sub_path: []const u8, dest_dir_sub_path: []const u8) !void {
1395	1415	assert(dest_dir_sub_path[1] == fs.path.sep);
1396	1416	var handled_missing_dir = false;
1397	1417	while (true) {
...	...	@@ -1417,16 +1437,17 @@ pub fn renameTmpIntoCache(
1417	1437	}
1418	1438	}
1419	1439
	1440	const ComputedHash = struct {
	1441	digest: Package.Hash.Digest,
	1442	total_size: u64,
	1443	};
	1444
1420	1445	/// Assumes that files not included in the package have already been filtered
1421	1446	/// prior to calling this function. This ensures that files not protected by
1422	1447	/// the hash are not present on the file system. Empty directories are *not
1423	1448	/// hashed* and must not be present on the file system when calling this
1424	1449	/// function.
1425		fn computeHash(
1426		f: *Fetch,
1427		pkg_path: Cache.Path,
1428		filter: Filter,
1429		) RunError!Manifest.Digest {
	1450	fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash {
1430	1451	// All the path name strings need to be in memory for sorting.
1431	1452	const arena = f.arena.allocator();
1432	1453	const gpa = f.arena.child_allocator;
...	...	@@ -1449,6 +1470,9 @@ fn computeHash(
1449	1470	var walker = try root_dir.walk(gpa);
1450	1471	defer walker.deinit();
1451	1472
	1473	// Total number of bytes of file contents included in the package.
	1474	var total_size: u64 = 0;
	1475
1452	1476	{
1453	1477	// The final hash will be a hash of each file hashed independently. This
1454	1478	// allows hashing in parallel.
...	...	@@ -1506,6 +1530,7 @@ fn computeHash(
1506	1530	.kind = kind,
1507	1531	.hash = undefined, // to be populated by the worker
1508	1532	.failure = undefined, // to be populated by the worker
	1533	.size = undefined, // to be populated by the worker
1509	1534	};
1510	1535	thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file });
1511	1536	try all_files.append(hashed_file);
...	...	@@ -1544,7 +1569,7 @@ fn computeHash(
1544	1569
1545	1570	std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
1546	1571
1547		var hasher = Manifest.Hash.init(.{});
	1572	var hasher = Package.Hash.Algo.init(.{});
1548	1573	var any_failures = false;
1549	1574	for (all_files.items) \|hashed_file\| {
1550	1575	hashed_file.failure catch \|err\| {
...	...	@@ -1556,6 +1581,7 @@ fn computeHash(
1556	1581	});
1557	1582	};
1558	1583	hasher.update(&hashed_file.hash);
	1584	total_size += hashed_file.size;
1559	1585	}
1560	1586	for (deleted_files.items) \|deleted_file\| {
1561	1587	deleted_file.failure catch \|err\| {
...	...	@@ -1580,7 +1606,10 @@ fn computeHash(
1580	1606	};
1581	1607	}
1582	1608
1583		return hasher.finalResult();
	1609	return .{
	1610	.digest = hasher.finalResult(),
	1611	.total_size = total_size,
	1612	};
1584	1613	}
1585	1614
1586	1615	fn dumpHashInfo(all_files: []const *const HashedFile) !void {
...	...	@@ -1609,8 +1638,9 @@ fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void {
1609	1638
1610	1639	fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
1611	1640	var buf: [8000]u8 = undefined;
1612		var hasher = Manifest.Hash.init(.{});
	1641	var hasher = Package.Hash.Algo.init(.{});
1613	1642	hasher.update(hashed_file.normalized_path);
	1643	var file_size: u64 = 0;
1614	1644
1615	1645	switch (hashed_file.kind) {
1616	1646	.file => {
...	...	@@ -1622,6 +1652,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
1622	1652	while (true) {
1623	1653	const bytes_read = try file.read(&buf);
1624	1654	if (bytes_read == 0) break;
	1655	file_size += bytes_read;
1625	1656	hasher.update(buf[0..bytes_read]);
1626	1657	file_header.update(buf[0..bytes_read]);
1627	1658	}
...	...	@@ -1641,6 +1672,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
1641	1672	},
1642	1673	}
1643	1674	hasher.final(&hashed_file.hash);
	1675	hashed_file.size = file_size;
1644	1676	}
1645	1677
1646	1678	fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {
...	...	@@ -1667,9 +1699,10 @@ const DeletedFile = struct {
1667	1699	const HashedFile = struct {
1668	1700	fs_path: []const u8,
1669	1701	normalized_path: []const u8,
1670		hash: Manifest.Digest,
	1702	hash: Package.Hash.Digest,
1671	1703	failure: Error!void,
1672	1704	kind: Kind,
	1705	size: u64,
1673	1706
1674	1707	const Error =
1675	1708	fs.File.OpenError \|\|
...	...	@@ -1744,12 +1777,8 @@ const Filter = struct {
1744	1777	}
1745	1778	};
1746	1779
1747		pub fn depDigest(
1748		pkg_root: Cache.Path,
1749		cache_root: Cache.Directory,
1750		dep: Manifest.Dependency,
1751		) ?Manifest.MultiHashHexDigest {
1752		if (dep.hash) \|h\| return h[0..Manifest.multihash_hex_digest_len].*;
	1780	pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash {
	1781	if (dep.hash) \|h\| return .fromSlice(h);
1753	1782
1754	1783	switch (dep.location) {
1755	1784	.url => return null,
...	...	@@ -1763,10 +1792,6 @@ pub fn depDigest(
1763	1792	}
1764	1793	}
1765	1794
1766		// These are random bytes.
1767		const package_hash_prefix_cached = [8]u8{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 };
1768		const package_hash_prefix_project = [8]u8{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc };
1769
1770	1795	const builtin = @import("builtin");
1771	1796	const std = @import("std");
1772	1797	const fs = std.fs;
...	...	@@ -2137,7 +2162,7 @@ test "tarball with excluded duplicate paths" {
2137	2162	defer fb.deinit();
2138	2163	try fetch.run();
2139	2164
2140		const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
	2165	const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest);
2141	2166	try std.testing.expectEqualStrings(
2142	2167	"12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da",
2143	2168	&hex_digest,
...	...	@@ -2181,7 +2206,7 @@ test "tarball without root folder" {
2181	2206	defer fb.deinit();
2182	2207	try fetch.run();
2183	2208
2184		const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
	2209	const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest);
2185	2210	try std.testing.expectEqualStrings(
2186	2211	"12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793",
2187	2212	&hex_digest,
...	...	@@ -2222,7 +2247,7 @@ test "set executable bit based on file content" {
2222	2247	try fetch.run();
2223	2248	try std.testing.expectEqualStrings(
2224	2249	"1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3",
2225		&Manifest.hexDigest(fetch.actual_hash),
	2250	&Package.multiHashHexDigest(fetch.computed_hash.digest),
2226	2251	);
2227	2252
2228	2253	var out = try fb.packageDir();
...	...	@@ -2298,13 +2323,15 @@ const TestFetchBuilder = struct {
2298	2323	.job_queue = &self.job_queue,
2299	2324	.omit_missing_hash_error = true,
2300	2325	.allow_missing_paths_field = false,
	2326	.allow_missing_fingerprint = true, // so we can keep using the old testdata .tar.gz
	2327	.allow_name_string = true, // so we can keep using the old testdata .tar.gz
2301	2328	.use_latest_commit = true,
2302	2329
2303	2330	.package_root = undefined,
2304	2331	.error_bundle = undefined,
2305	2332	.manifest = null,
2306	2333	.manifest_ast = undefined,
2307		.actual_hash = undefined,
	2334	.computed_hash = undefined,
2308	2335	.has_build_zig = false,
2309	2336	.oom_flag = false,
2310	2337	.latest_commit = null,

src/Package/Manifest.zig+92-67

...	...	@@ -5,15 +5,12 @@ const Allocator = std.mem.Allocator;
5	5	const assert = std.debug.assert;
6	6	const Ast = std.zig.Ast;
7	7	const testing = std.testing;
8		const hex_charset = std.fmt.hex_charset;
	8	const Package = @import("../Package.zig");
9	9
10	10	pub const max_bytes = 10 * 1024 * 1024;
11	11	pub const basename = "build.zig.zon";
12		pub const Hash = std.crypto.hash.sha2.Sha256;
13		pub const Digest = [Hash.digest_length]u8;
14		pub const multihash_len = 1 + 1 + Hash.digest_length;
15		pub const multihash_hex_digest_len = 2 * multihash_len;
16		pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
	12	pub const max_name_len = 32;
	13	pub const max_version_len = 32;
17	14
18	15	pub const Dependency = struct {
19	16	location: Location,
...	...	@@ -38,36 +35,8 @@ pub const ErrorMessage = struct {
38	35	off: u32,
39	36	};
40	37
41		pub const MultihashFunction = enum(u16) {
42		identity = 0x00,
43		sha1 = 0x11,
44		@"sha2-256" = 0x12,
45		@"sha2-512" = 0x13,
46		@"sha3-512" = 0x14,
47		@"sha3-384" = 0x15,
48		@"sha3-256" = 0x16,
49		@"sha3-224" = 0x17,
50		@"sha2-384" = 0x20,
51		@"sha2-256-trunc254-padded" = 0x1012,
52		@"sha2-224" = 0x1013,
53		@"sha2-512-224" = 0x1014,
54		@"sha2-512-256" = 0x1015,
55		@"blake2b-256" = 0xb220,
56		_,
57		};
58
59		pub const multihash_function: MultihashFunction = switch (Hash) {
60		std.crypto.hash.sha2.Sha256 => .@"sha2-256",
61		else => @compileError("unreachable"),
62		};
63		comptime {
64		// We avoid unnecessary uleb128 code in hexDigest by asserting here the
65		// values are small enough to be contained in the one-byte encoding.
66		assert(@intFromEnum(multihash_function) < 127);
67		assert(Hash.digest_length < 127);
68		}
69
70	38	name: []const u8,
	39	id: u32,
71	40	version: std.SemanticVersion,
72	41	version_node: Ast.Node.Index,
73	42	dependencies: std.StringArrayHashMapUnmanaged(Dependency),
...	...	@@ -80,6 +49,10 @@ arena_state: std.heap.ArenaAllocator.State,
80	49
81	50	pub const ParseOptions = struct {
82	51	allow_missing_paths_field: bool = false,
	52	/// Deprecated, to be removed after 0.14.0 is tagged.
	53	allow_name_string: bool = true,
	54	/// Deprecated, to be removed after 0.14.0 is tagged.
	55	allow_missing_fingerprint: bool = true,
83	56	};
84	57
85	58	pub const Error = Allocator.Error;
...	...	@@ -100,12 +73,15 @@ pub fn parse(gpa: Allocator, ast: Ast, options: ParseOptions) Error!Manifest {
100	73	.errors = .{},
101	74
102	75	.name = undefined,
	76	.id = 0,
103	77	.version = undefined,
104	78	.version_node = 0,
105	79	.dependencies = .{},
106	80	.dependencies_node = 0,
107	81	.paths = .{},
108	82	.allow_missing_paths_field = options.allow_missing_paths_field,
	83	.allow_name_string = options.allow_name_string,
	84	.allow_missing_fingerprint = options.allow_missing_fingerprint,
109	85	.minimum_zig_version = null,
110	86	.buf = .{},
111	87	};
...	...	@@ -121,6 +97,7 @@ pub fn parse(gpa: Allocator, ast: Ast, options: ParseOptions) Error!Manifest {
121	97
122	98	return .{
123	99	.name = p.name,
	100	.id = p.id,
124	101	.version = p.version,
125	102	.version_node = p.version_node,
126	103	.dependencies = try p.dependencies.clone(p.arena),
...	...	@@ -164,22 +141,6 @@ pub fn copyErrorsIntoBundle(
164	141	}
165	142	}
166	143
167		pub fn hexDigest(digest: Digest) MultiHashHexDigest {
168		var result: MultiHashHexDigest = undefined;
169
170		result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
171		result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
172
173		result[2] = hex_charset[Hash.digest_length >> 4];
174		result[3] = hex_charset[Hash.digest_length & 15];
175
176		for (digest, 0..) \|byte, i\| {
177		result[4 + i * 2] = hex_charset[byte >> 4];
178		result[5 + i * 2] = hex_charset[byte & 15];
179		}
180		return result;
181		}
182
183	144	const Parse = struct {
184	145	gpa: Allocator,
185	146	ast: Ast,
...	...	@@ -188,12 +149,15 @@ const Parse = struct {
188	149	errors: std.ArrayListUnmanaged(ErrorMessage),
189	150
190	151	name: []const u8,
	152	id: u32,
191	153	version: std.SemanticVersion,
192	154	version_node: Ast.Node.Index,
193	155	dependencies: std.StringArrayHashMapUnmanaged(Dependency),
194	156	dependencies_node: Ast.Node.Index,
195	157	paths: std.StringArrayHashMapUnmanaged(void),
196	158	allow_missing_paths_field: bool,
	159	allow_name_string: bool,
	160	allow_missing_fingerprint: bool,
197	161	minimum_zig_version: ?std.SemanticVersion,
198	162
199	163	const InnerError = error{ ParseFailure, OutOfMemory };
...	...	@@ -211,6 +175,7 @@ const Parse = struct {
211	175	var have_name = false;
212	176	var have_version = false;
213	177	var have_included_paths = false;
	178	var fingerprint: ?Package.Fingerprint = null;
214	179
215	180	for (struct_init.ast.fields) \|field_init\| {
216	181	const name_token = ast.firstToken(field_init) - 2;
...	...	@@ -225,11 +190,16 @@ const Parse = struct {
225	190	have_included_paths = true;
226	191	try parseIncludedPaths(p, field_init);
227	192	} else if (mem.eql(u8, field_name, "name")) {
228		p.name = try parseString(p, field_init);
	193	p.name = try parseName(p, field_init);
229	194	have_name = true;
	195	} else if (mem.eql(u8, field_name, "fingerprint")) {
	196	fingerprint = try parseFingerprint(p, field_init);
230	197	} else if (mem.eql(u8, field_name, "version")) {
231	198	p.version_node = field_init;
232	199	const version_text = try parseString(p, field_init);
	200	if (version_text.len > max_version_len) {
	201	try appendError(p, main_tokens[field_init], "version string length {d} exceeds maximum of {d}", .{ version_text.len, max_version_len });
	202	}
233	203	p.version = std.SemanticVersion.parse(version_text) catch \|err\| v: {
234	204	try appendError(p, main_tokens[field_init], "unable to parse semantic version: {s}", .{@errorName(err)});
235	205	break :v undefined;
...	...	@@ -249,6 +219,21 @@ const Parse = struct {
249	219
250	220	if (!have_name) {
251	221	try appendError(p, main_token, "missing top-level 'name' field", .{});
	222	} else {
	223	if (fingerprint) \|n\| {
	224	if (!n.validate(p.name)) {
	225	return fail(p, main_token, "invalid fingerprint: 0x{x}; if this is a new or forked package, use this value: 0x{x}", .{
	226	n.int(), Package.Fingerprint.generate(p.name).int(),
	227	});
	228	}
	229	p.id = n.id;
	230	} else if (!p.allow_missing_fingerprint) {
	231	try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{
	232	Package.Fingerprint.generate(p.name).int(),
	233	});
	234	} else {
	235	p.id = 0;
	236	}
252	237	}
253	238
254	239	if (!have_version) {
...	...	@@ -400,6 +385,59 @@ const Parse = struct {
400	385	}
401	386	}
402	387
	388	fn parseFingerprint(p: *Parse, node: Ast.Node.Index) !Package.Fingerprint {
	389	const ast = p.ast;
	390	const node_tags = ast.nodes.items(.tag);
	391	const main_tokens = ast.nodes.items(.main_token);
	392	const main_token = main_tokens[node];
	393	if (node_tags[node] != .number_literal) {
	394	return fail(p, main_token, "expected integer literal", .{});
	395	}
	396	const token_bytes = ast.tokenSlice(main_token);
	397	const parsed = std.zig.parseNumberLiteral(token_bytes);
	398	switch (parsed) {
	399	.int => \|n\| return @bitCast(n),
	400	.big_int, .float => return fail(p, main_token, "expected u64 integer literal, found {s}", .{
	401	@tagName(parsed),
	402	}),
	403	.failure => \|err\| return fail(p, main_token, "bad integer literal: {s}", .{@tagName(err)}),
	404	}
	405	}
	406
	407	fn parseName(p: *Parse, node: Ast.Node.Index) ![]const u8 {
	408	const ast = p.ast;
	409	const node_tags = ast.nodes.items(.tag);
	410	const main_tokens = ast.nodes.items(.main_token);
	411	const main_token = main_tokens[node];
	412
	413	if (p.allow_name_string and node_tags[node] == .string_literal) {
	414	const name = try parseString(p, node);
	415	if (!std.zig.isValidId(name))
	416	return fail(p, main_token, "name must be a valid bare zig identifier (hint: switch from string to enum literal)", .{});
	417
	418	if (name.len > max_name_len)
	419	return fail(p, main_token, "name '{}' exceeds max length of {d}", .{
	420	std.zig.fmtId(name), max_name_len,
	421	});
	422
	423	return name;
	424	}
	425
	426	if (node_tags[node] != .enum_literal)
	427	return fail(p, main_token, "expected enum literal", .{});
	428
	429	const ident_name = ast.tokenSlice(main_token);
	430	if (mem.startsWith(u8, ident_name, "@"))
	431	return fail(p, main_token, "name must be a valid bare zig identifier", .{});
	432
	433	if (ident_name.len > max_name_len)
	434	return fail(p, main_token, "name '{}' exceeds max length of {d}", .{
	435	std.zig.fmtId(ident_name), max_name_len,
	436	});
	437
	438	return ident_name;
	439	}
	440
403	441	fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 {
404	442	const ast = p.ast;
405	443	const node_tags = ast.nodes.items(.tag);
...	...	@@ -421,21 +459,8 @@ const Parse = struct {
421	459	const tok = main_tokens[node];
422	460	const h = try parseString(p, node);
423	461
424		if (h.len >= 2) {
425		const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch \|err\| {
426		return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{
427		@errorName(err),
428		});
429		};
430		if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) {
431		return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{});
432		}
433		}
434
435		if (h.len != multihash_hex_digest_len) {
436		return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{
437		multihash_hex_digest_len, h.len,
438		});
	462	if (h.len > Package.Hash.max_len) {
	463	return fail(p, tok, "hash length exceeds maximum: {d}", .{h.len});
439	464	}
440	465
441	466	return h;

src/main.zig+84-26

...	...	@@ -4741,6 +4741,7 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
4741	4741
4742	4742	const cwd_path = try process.getCwdAlloc(arena);
4743	4743	const cwd_basename = fs.path.basename(cwd_path);
	4744	const sanitized_root_name = try sanitizeExampleName(arena, cwd_basename);
4744	4745
4745	4746	const s = fs.path.sep_str;
4746	4747	const template_paths = [_][]const u8{
...	...	@@ -4751,8 +4752,10 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
4751	4752	};
4752	4753	var ok_count: usize = 0;
4753	4754
	4755	const fingerprint: Package.Fingerprint = .generate(sanitized_root_name);
	4756
4754	4757	for (template_paths) \|template_path\| {
4755		if (templates.write(arena, fs.cwd(), cwd_basename, template_path)) \|_\| {
	4758	if (templates.write(arena, fs.cwd(), sanitized_root_name, template_path, fingerprint)) \|_\| {
4756	4759	std.log.info("created {s}", .{template_path});
4757	4760	ok_count += 1;
4758	4761	} else \|err\| switch (err) {
...	...	@@ -4769,6 +4772,37 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
4769	4772	return cleanExit();
4770	4773	}
4771	4774
	4775	fn sanitizeExampleName(arena: Allocator, bytes: []const u8) error{OutOfMemory}![]const u8 {
	4776	var result: std.ArrayListUnmanaged(u8) = .empty;
	4777	for (bytes, 0..) \|byte, i\| switch (byte) {
	4778	'0'...'9' => {
	4779	if (i == 0) try result.append(arena, '_');
	4780	try result.append(arena, byte);
	4781	},
	4782	'_', 'a'...'z', 'A'...'Z' => try result.append(arena, byte),
	4783	'-', '.', ' ' => try result.append(arena, '_'),
	4784	else => continue,
	4785	};
	4786	if (result.items.len == 0) return "foo";
	4787	if (result.items.len > Package.Manifest.max_name_len)
	4788	result.shrinkRetainingCapacity(Package.Manifest.max_name_len);
	4789
	4790	return result.toOwnedSlice(arena);
	4791	}
	4792
	4793	test sanitizeExampleName {
	4794	var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator);
	4795	defer arena_instance.deinit();
	4796	const arena = arena_instance.allocator();
	4797
	4798	try std.testing.expectEqualStrings("foo_bar", try sanitizeExampleName(arena, "foo bar+"));
	4799	try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, ""));
	4800	try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "!"));
	4801	try std.testing.expectEqualStrings("a", try sanitizeExampleName(arena, "!a"));
	4802	try std.testing.expectEqualStrings("a_b", try sanitizeExampleName(arena, "a.b!"));
	4803	try std.testing.expectEqualStrings("_01234", try sanitizeExampleName(arena, "01234"));
	4804	}
	4805
4772	4806	fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
4773	4807	dev.check(.build_command);
4774	4808
...	...	@@ -5191,13 +5225,15 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5191	5225	.job_queue = &job_queue,
5192	5226	.omit_missing_hash_error = true,
5193	5227	.allow_missing_paths_field = false,
	5228	.allow_missing_fingerprint = false,
	5229	.allow_name_string = false,
5194	5230	.use_latest_commit = false,
5195	5231
5196	5232	.package_root = undefined,
5197	5233	.error_bundle = undefined,
5198	5234	.manifest = null,
5199	5235	.manifest_ast = undefined,
5200		.actual_hash = undefined,
	5236	.computed_hash = undefined,
5201	5237	.has_build_zig = true,
5202	5238	.oom_flag = false,
5203	5239	.latest_commit = null,
...	...	@@ -5244,13 +5280,14 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5244	5280	const hashes = job_queue.table.keys();
5245	5281	const fetches = job_queue.table.values();
5246	5282	try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len));
5247		for (hashes, fetches) \|hash, f\| {
	5283	for (hashes, fetches) \|*hash, f\| {
5248	5284	if (f == &fetch) {
5249	5285	// The first one is a dummy package for the current project.
5250	5286	continue;
5251	5287	}
5252	5288	if (!f.has_build_zig)
5253	5289	continue;
	5290	const hash_slice = hash.toSlice();
5254	5291	const m = try Package.Module.create(arena, .{
5255	5292	.global_cache_directory = global_cache_directory,
5256	5293	.paths = .{
...	...	@@ -5260,7 +5297,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5260	5297	.fully_qualified_name = try std.fmt.allocPrint(
5261	5298	arena,
5262	5299	"root.@dependencies.{s}",
5263		.{&hash},
	5300	.{hash_slice},
5264	5301	),
5265	5302	.cc_argv = &.{},
5266	5303	.inherited = .{},
...	...	@@ -5269,7 +5306,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5269	5306	.builtin_mod = builtin_mod,
5270	5307	.builtin_modules = null, // `builtin_mod` is specified
5271	5308	});
5272		const hash_cloned = try arena.dupe(u8, &hash);
	5309	const hash_cloned = try arena.dupe(u8, hash_slice);
5273	5310	deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m);
5274	5311	f.module = m;
5275	5312	}
...	...	@@ -5385,23 +5422,22 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5385	5422	var any_errors = false;
5386	5423	while (it.next()) \|hash\| {
5387	5424	if (hash.len == 0) continue;
5388		const digest_len = @typeInfo(Package.Manifest.MultiHashHexDigest).array.len;
5389		if (hash.len != digest_len) {
5390		std.log.err("invalid digest (length {d} instead of {d}): '{s}'", .{
5391		hash.len, digest_len, hash,
	5425	if (hash.len > Package.Hash.max_len) {
	5426	std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{
	5427	hash.len, hash,
5392	5428	});
5393	5429	any_errors = true;
5394	5430	continue;
5395	5431	}
5396		try unlazy_set.put(arena, hash[0..digest_len].*, {});
	5432	try unlazy_set.put(arena, .fromSlice(hash), {});
5397	5433	}
5398	5434	if (any_errors) process.exit(3);
5399	5435	if (system_pkg_dir_path) \|p\| {
5400	5436	// In this mode, the system needs to provide these packages; they
5401	5437	// cannot be fetched by Zig.
5402		for (unlazy_set.keys()) \|hash\| {
	5438	for (unlazy_set.keys()) \|*hash\| {
5403	5439	std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{
5404		p, hash,
	5440	p, hash.toSlice(),
5405	5441	});
5406	5442	}
5407	5443	std.log.info("remote package fetching disabled due to --system mode", .{});
...	...	@@ -7091,13 +7127,15 @@ fn cmdFetch(
7091	7127	.job_queue = &job_queue,
7092	7128	.omit_missing_hash_error = true,
7093	7129	.allow_missing_paths_field = false,
	7130	.allow_missing_fingerprint = true,
	7131	.allow_name_string = true,
7094	7132	.use_latest_commit = true,
7095	7133
7096	7134	.package_root = undefined,
7097	7135	.error_bundle = undefined,
7098	7136	.manifest = null,
7099	7137	.manifest_ast = undefined,
7100		.actual_hash = undefined,
	7138	.computed_hash = undefined,
7101	7139	.has_build_zig = false,
7102	7140	.oom_flag = false,
7103	7141	.latest_commit = null,
...	...	@@ -7117,14 +7155,15 @@ fn cmdFetch(
7117	7155	process.exit(1);
7118	7156	}
7119	7157
7120		const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
	7158	const package_hash = fetch.computedPackageHash();
	7159	const package_hash_slice = package_hash.toSlice();
7121	7160
7122	7161	root_prog_node.end();
7123	7162	root_prog_node = .{ .index = .none };
7124	7163
7125	7164	const name = switch (save) {
7126	7165	.no => {
7127		try io.getStdOut().writeAll(hex_digest ++ "\n");
	7166	try io.getStdOut().writer().print("{s}\n", .{package_hash_slice});
7128	7167	return cleanExit();
7129	7168	},
7130	7169	.yes, .exact => \|name\| name: {
...	...	@@ -7145,7 +7184,7 @@ fn cmdFetch(
7145	7184	// The name to use in case the manifest file needs to be created now.
7146	7185	const init_root_name = fs.path.basename(build_root.directory.path orelse cwd_path);
7147	7186	var manifest, var ast = try loadManifest(gpa, arena, .{
7148		.root_name = init_root_name,
	7187	.root_name = try sanitizeExampleName(arena, init_root_name),
7149	7188	.dir = build_root.directory.handle,
7150	7189	.color = color,
7151	7190	});
...	...	@@ -7194,7 +7233,7 @@ fn cmdFetch(
7194	7233	\\ }}
7195	7234	, .{
7196	7235	std.zig.fmtEscapes(saved_path_or_url),
7197		std.zig.fmtEscapes(&hex_digest),
	7236	std.zig.fmtEscapes(package_hash_slice),
7198	7237	});
7199	7238
7200	7239	const new_node_text = try std.fmt.allocPrint(arena, ".{p_} = {s},\n", .{
...	...	@@ -7213,7 +7252,7 @@ fn cmdFetch(
7213	7252	if (dep.hash) \|h\| {
7214	7253	switch (dep.location) {
7215	7254	.url => \|u\| {
7216		if (mem.eql(u8, h, &hex_digest) and mem.eql(u8, u, saved_path_or_url)) {
	7255	if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) {
7217	7256	std.log.info("existing dependency named '{s}' is up-to-date", .{name});
7218	7257	process.exit(0);
7219	7258	}
...	...	@@ -7230,7 +7269,7 @@ fn cmdFetch(
7230	7269	const hash_replace = try std.fmt.allocPrint(
7231	7270	arena,
7232	7271	"\"{}\"",
7233		.{std.zig.fmtEscapes(&hex_digest)},
	7272	.{std.zig.fmtEscapes(package_hash_slice)},
7234	7273	);
7235	7274
7236	7275	warn("overwriting existing dependency named '{s}'", .{name});
...	...	@@ -7429,10 +7468,10 @@ fn loadManifest(
7429	7468	0,
7430	7469	) catch \|err\| switch (err) {
7431	7470	error.FileNotFound => {
	7471	const fingerprint: Package.Fingerprint = .generate(options.root_name);
7432	7472	var templates = findTemplates(gpa, arena);
7433	7473	defer templates.deinit();
7434
7435		templates.write(arena, options.dir, options.root_name, Package.Manifest.basename) catch \|e\| {
	7474	templates.write(arena, options.dir, options.root_name, Package.Manifest.basename, fingerprint) catch \|e\| {
7436	7475	fatal("unable to write {s}: {s}", .{
7437	7476	Package.Manifest.basename, @errorName(e),
7438	7477	});
...	...	@@ -7490,6 +7529,7 @@ const Templates = struct {
7490	7529	out_dir: fs.Dir,
7491	7530	root_name: []const u8,
7492	7531	template_path: []const u8,
	7532	fingerprint: Package.Fingerprint,
7493	7533	) !void {
7494	7534	if (fs.path.dirname(template_path)) \|dirname\| {
7495	7535	out_dir.makePath(dirname) catch \|err\| {
...	...	@@ -7503,12 +7543,30 @@ const Templates = struct {
7503	7543	};
7504	7544	templates.buffer.clearRetainingCapacity();
7505	7545	try templates.buffer.ensureUnusedCapacity(contents.len);
7506		for (contents) \|c\| {
7507		if (c == '$') {
7508		try templates.buffer.appendSlice(root_name);
7509		} else {
7510		try templates.buffer.append(c);
	7546	var i: usize = 0;
	7547	while (i < contents.len) {
	7548	if (contents[i] == '.') {
	7549	if (std.mem.startsWith(u8, contents[i..], ".LITNAME")) {
	7550	try templates.buffer.append('.');
	7551	try templates.buffer.appendSlice(root_name);
	7552	i += ".LITNAME".len;
	7553	continue;
	7554	} else if (std.mem.startsWith(u8, contents[i..], ".NAME")) {
	7555	try templates.buffer.appendSlice(root_name);
	7556	i += ".NAME".len;
	7557	continue;
	7558	} else if (std.mem.startsWith(u8, contents[i..], ".FINGERPRINT")) {
	7559	try templates.buffer.writer().print("0x{x}", .{fingerprint.int()});
	7560	i += ".FINGERPRINT".len;
	7561	continue;
	7562	} else if (std.mem.startsWith(u8, contents[i..], ".ZIGVER")) {
	7563	try templates.buffer.appendSlice(build_options.version);
	7564	i += ".ZIGVER".len;
	7565	continue;
	7566	}
7511	7567	}
	7568	try templates.buffer.append(contents[i]);
	7569	i += 1;
7512	7570	}
7513	7571
7514	7572	return out_dir.writeFile(.{