authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-28 04:40:05-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-02-28 04:40:05-05:00
log6c3cbb0c87a33f4ae408874f6ceb40e372b65914
tree2892f3585939f9f66dcf8e78fcc8606ad861ae6a
parent6b6c1b1b0e04d70a4917f073a6a8bc87a5e8abb3
parentde43f5eb6ae4a569efe15e3469b3de76a86d9cd1
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #22994 from ziglang/newhash

implement new package hash format: `$name-$semver-$hash`

10 files changed, 526 insertions(+), 182 deletions(-)

build.zig.zon+2-1
...@@ -1,7 +1,7 @@...@@ -1,7 +1,7 @@
1// The Zig compiler is not intended to be consumed as a package.1// The Zig compiler is not intended to be consumed as a package.
2// The sole purpose of this manifest file is to test the compiler.2// The sole purpose of this manifest file is to test the compiler.
3.{3.{
4 .name = "zig",4 .name = .zig,
5 .version = "0.0.0",5 .version = "0.0.0",
6 .dependencies = .{6 .dependencies = .{
7 .standalone_test_cases = .{7 .standalone_test_cases = .{
...@@ -12,4 +12,5 @@...@@ -12,4 +12,5 @@
12 },12 },
13 },13 },
14 .paths = .{""},14 .paths = .{""},
15 .fingerprint = 0xc1ce108124179e16,
15}16}
doc/build.zig.zon.md+31-1
...@@ -10,7 +10,7 @@ build.zig....@@ -10,7 +10,7 @@ build.zig.
1010
11### `name`11### `name`
1212
13String. Required.13Enum literal. Required.
1414
15This is the default name used by packages depending on this one. For example,15This is the default name used by packages depending on this one. For example,
16when a user runs `zig fetch --save <url>`, this field is used as the key in the16when a user runs `zig fetch --save <url>`, this field is used as the key in the
...@@ -20,12 +20,42 @@ will stick with this provided value....@@ -20,12 +20,42 @@ will stick with this provided value.
20It is redundant to include "zig" in this name because it is already within the20It is redundant to include "zig" in this name because it is already within the
21Zig package namespace.21Zig package namespace.
2222
23Must be a valid bare Zig identifier (don't `@` me), limited to 32 bytes.
24
25Together with `fingerprint`, this represents a globally unique package identifier.
26
27### `fingerprint`
28
29Together with `name`, this represents a globally unique package identifier. This
30field is auto-initialized by the toolchain when the package is first created,
31and then *never changes*. This allows Zig to unambiguously detect when one
32package is an updated version of another.
33
34When forking a Zig project, this fingerprint should be regenerated if the upstream
35project is still maintained. Otherwise, the fork is *hostile*, attempting to
36take control over the original project's identity. The fingerprint can be regenerated
37by deleting the field and running `zig build`.
38
39This 64-bit integer is the combination of a 32-bit id component and a 32-bit
40checksum.
41
42The id component within the fingerprint has these restrictions:
43
44`0x00000000` is reserved for legacy packages.
45
46`0xffffffff` is reserved to represent "naked" packages.
47
48The checksum is computed from `name` and serves to protect Zig users from
49accidental id collisions.
50
23### `version`51### `version`
2452
25String. Required.53String. Required.
2654
27[semver](https://semver.org/)55[semver](https://semver.org/)
2856
57Limited to 32 bytes.
58
29### `minimum_zig_version`59### `minimum_zig_version`
3060
31String. Optional.61String. Optional.
lib/init/build.zig+3-3
...@@ -42,14 +42,14 @@ pub fn build(b: *std.Build) void {...@@ -42,14 +42,14 @@ pub fn build(b: *std.Build) void {
42 // Modules can depend on one another using the `std.Build.Module.addImport` function.42 // Modules can depend on one another using the `std.Build.Module.addImport` function.
43 // This is what allows Zig source code to use `@import("foo")` where 'foo' is not a43 // This is what allows Zig source code to use `@import("foo")` where 'foo' is not a
44 // file path. In this case, we set up `exe_mod` to import `lib_mod`.44 // file path. In this case, we set up `exe_mod` to import `lib_mod`.
45 exe_mod.addImport("$_lib", lib_mod);45 exe_mod.addImport(".NAME_lib", lib_mod);
4646
47 // Now, we will create a static library based on the module we created above.47 // Now, we will create a static library based on the module we created above.
48 // This creates a `std.Build.Step.Compile`, which is the build step responsible48 // This creates a `std.Build.Step.Compile`, which is the build step responsible
49 // for actually invoking the compiler.49 // for actually invoking the compiler.
50 const lib = b.addLibrary(.{50 const lib = b.addLibrary(.{
51 .linkage = .static,51 .linkage = .static,
52 .name = "$",52 .name = ".NAME",
53 .root_module = lib_mod,53 .root_module = lib_mod,
54 });54 });
5555
...@@ -61,7 +61,7 @@ pub fn build(b: *std.Build) void {...@@ -61,7 +61,7 @@ pub fn build(b: *std.Build) void {
61 // This creates another `std.Build.Step.Compile`, but this one builds an executable61 // This creates another `std.Build.Step.Compile`, but this one builds an executable
62 // rather than a static library.62 // rather than a static library.
63 const exe = b.addExecutable(.{63 const exe = b.addExecutable(.{
64 .name = "$",64 .name = ".NAME",
65 .root_module = exe_mod,65 .root_module = exe_mod,
66 });66 });
6767
lib/init/build.zig.zon+19-1
...@@ -6,12 +6,30 @@...@@ -6,12 +6,30 @@
6 //6 //
7 // It is redundant to include "zig" in this name because it is already7 // It is redundant to include "zig" in this name because it is already
8 // within the Zig package namespace.8 // within the Zig package namespace.
9 .name = "$",9 .name = .LITNAME,
1010
11 // This is a [Semantic Version](https://semver.org/).11 // This is a [Semantic Version](https://semver.org/).
12 // In a future version of Zig it will be used for package deduplication.12 // In a future version of Zig it will be used for package deduplication.
13 .version = "0.0.0",13 .version = "0.0.0",
1414
15 // Together with name, this represents a globally unique package
16 // identifier. This field is generated by the Zig toolchain when the
17 // package is first created, and then *never changes*. This allows
18 // unambiguous detection of one package being an updated version of
19 // another.
20 //
21 // When forking a Zig project, this id should be regenerated (delete the
22 // field and run `zig build`) if the upstream project is still maintained.
23 // Otherwise, the fork is *hostile*, attempting to take control over the
24 // original project's identity. Thus it is recommended to leave the comment
25 // on the following line intact, so that it shows up in code reviews that
26 // modify the field.
27 .fingerprint = .FINGERPRINT, // Changing this has security and trust implications.
28
29 // Tracks the earliest Zig version that the package considers to be a
30 // supported use case.
31 .minimum_zig_version = ".ZIGVER",
32
15 // This field is optional.33 // This field is optional.
16 // This is currently advisory only; Zig does not yet do anything34 // This is currently advisory only; Zig does not yet do anything
17 // with this value.35 // with this value.
lib/init/src/main.zig+1-1
...@@ -43,4 +43,4 @@ test "fuzz example" {...@@ -43,4 +43,4 @@ test "fuzz example" {
43const std = @import("std");43const std = @import("std");
4444
45/// This imports the separate module containing `root.zig`. Take a look in `build.zig` for details.45/// This imports the separate module containing `root.zig`. Take a look in `build.zig` for details.
46const lib = @import("$_lib");46const lib = @import(".NAME_lib");
lib/std/array_list.zig-7
...@@ -2250,10 +2250,3 @@ test "return OutOfMemory when capacity would exceed maximum usize integer value"...@@ -2250,10 +2250,3 @@ test "return OutOfMemory when capacity would exceed maximum usize integer value"
2250 try testing.expectError(error.OutOfMemory, list.ensureUnusedCapacity(2));2250 try testing.expectError(error.OutOfMemory, list.ensureUnusedCapacity(2));
2251 }2251 }
2252}2252}
2253
2254test "ArrayListAligned with non-native alignment compiles unusedCapabitySlice" {
2255 var list = ArrayListAligned(u8, 4).init(testing.allocator);
2256 defer list.deinit();
2257 try list.appendNTimes(1, 4);
2258 _ = list.unusedCapacitySlice();
2259}
src/Package.zig+192
...@@ -1,8 +1,200 @@...@@ -1,8 +1,200 @@
1const std = @import("std");
2const assert = std.debug.assert;
3
1pub const Module = @import("Package/Module.zig");4pub const Module = @import("Package/Module.zig");
2pub const Fetch = @import("Package/Fetch.zig");5pub const Fetch = @import("Package/Fetch.zig");
3pub const build_zig_basename = "build.zig";6pub const build_zig_basename = "build.zig";
4pub const Manifest = @import("Package/Manifest.zig");7pub const Manifest = @import("Package/Manifest.zig");
58
9pub const multihash_len = 1 + 1 + Hash.Algo.digest_length;
10pub const multihash_hex_digest_len = 2 * multihash_len;
11pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
12
13pub const Fingerprint = packed struct(u64) {
14 id: u32,
15 checksum: u32,
16
17 pub fn generate(name: []const u8) Fingerprint {
18 return .{
19 .id = std.crypto.random.intRangeLessThan(u32, 1, 0xffffffff),
20 .checksum = std.hash.Crc32.hash(name),
21 };
22 }
23
24 pub fn validate(n: Fingerprint, name: []const u8) bool {
25 switch (n.id) {
26 0x00000000, 0xffffffff => return false,
27 else => return std.hash.Crc32.hash(name) == n.checksum,
28 }
29 }
30
31 pub fn int(n: Fingerprint) u64 {
32 return @bitCast(n);
33 }
34};
35
36/// A user-readable, file system safe hash that identifies an exact package
37/// snapshot, including file contents.
38///
39/// The hash is not only to prevent collisions but must resist attacks where
40/// the adversary fully controls the contents being hashed. Thus, it contains
41/// a full SHA-256 digest.
42///
43/// This data structure can be used to store the legacy hash format too. Legacy
44/// hash format is scheduled to be removed after 0.14.0 is tagged.
45///
46/// There's also a third way this structure is used. When using path rather than
47/// hash, a unique hash is still needed, so one is computed based on the path.
48pub const Hash = struct {
49 /// Maximum size of a package hash. Unused bytes at the end are
50 /// filled with zeroes.
51 bytes: [max_len]u8,
52
53 pub const Algo = std.crypto.hash.sha2.Sha256;
54 pub const Digest = [Algo.digest_length]u8;
55
56 /// Example: "nnnn-vvvv-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh"
57 pub const max_len = 32 + 1 + 32 + 1 + (32 + 32 + 200) / 6;
58
59 pub fn fromSlice(s: []const u8) Hash {
60 assert(s.len <= max_len);
61 var result: Hash = undefined;
62 @memcpy(result.bytes[0..s.len], s);
63 @memset(result.bytes[s.len..], 0);
64 return result;
65 }
66
67 pub fn toSlice(ph: *const Hash) []const u8 {
68 var end: usize = ph.bytes.len;
69 while (true) {
70 end -= 1;
71 if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1];
72 }
73 }
74
75 pub fn eql(a: *const Hash, b: *const Hash) bool {
76 return std.mem.eql(u8, &a.bytes, &b.bytes);
77 }
78
79 /// Distinguishes whether the legacy multihash format is being stored here.
80 pub fn isOld(h: *const Hash) bool {
81 if (h.bytes.len < 2) return false;
82 const their_multihash_func = std.fmt.parseInt(u8, h.bytes[0..2], 16) catch return false;
83 if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) return false;
84 if (h.toSlice().len != multihash_hex_digest_len) return false;
85 return std.mem.indexOfScalar(u8, &h.bytes, '-') == null;
86 }
87
88 test isOld {
89 const h: Hash = .fromSlice("1220138f4aba0c01e66b68ed9e1e1e74614c06e4743d88bc58af4f1c3dd0aae5fea7");
90 try std.testing.expect(h.isOld());
91 }
92
93 /// Produces "$name-$semver-$hashplus".
94 /// * name is the name field from build.zig.zon, asserted to be at most 32
95 /// bytes and assumed be a valid zig identifier
96 /// * semver is the version field from build.zig.zon, asserted to be at
97 /// most 32 bytes
98 /// * hashplus is the following 33-byte array, base64 encoded using -_ to make
99 /// it filesystem safe:
100 /// - (4 bytes) LE u32 Package ID
101 /// - (4 bytes) LE u32 total decompressed size in bytes, overflow saturated
102 /// - (25 bytes) truncated SHA-256 digest of hashed files of the package
103 pub fn init(digest: Digest, name: []const u8, ver: []const u8, id: u32, size: u32) Hash {
104 assert(name.len <= 32);
105 assert(ver.len <= 32);
106 var result: Hash = undefined;
107 var buf: std.ArrayListUnmanaged(u8) = .initBuffer(&result.bytes);
108 buf.appendSliceAssumeCapacity(name);
109 buf.appendAssumeCapacity('-');
110 buf.appendSliceAssumeCapacity(ver);
111 buf.appendAssumeCapacity('-');
112 var hashplus: [33]u8 = undefined;
113 std.mem.writeInt(u32, hashplus[0..4], id, .little);
114 std.mem.writeInt(u32, hashplus[4..8], size, .little);
115 hashplus[8..].* = digest[0..25].*;
116 _ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(44), &hashplus);
117 @memset(buf.unusedCapacitySlice(), 0);
118 return result;
119 }
120
121 /// Produces a unique hash based on the path provided. The result should
122 /// not be user-visible.
123 pub fn initPath(sub_path: []const u8, is_global: bool) Hash {
124 var result: Hash = .{ .bytes = @splat(0) };
125 var i: usize = 0;
126 if (is_global) {
127 result.bytes[0] = '/';
128 i += 1;
129 }
130 if (i + sub_path.len <= result.bytes.len) {
131 @memcpy(result.bytes[i..][0..sub_path.len], sub_path);
132 return result;
133 }
134 var bin_digest: [Algo.digest_length]u8 = undefined;
135 Algo.hash(sub_path, &bin_digest, .{});
136 _ = std.fmt.bufPrint(result.bytes[i..], "{}", .{std.fmt.fmtSliceHexLower(&bin_digest)}) catch unreachable;
137 return result;
138 }
139};
140
141pub const MultihashFunction = enum(u16) {
142 identity = 0x00,
143 sha1 = 0x11,
144 @"sha2-256" = 0x12,
145 @"sha2-512" = 0x13,
146 @"sha3-512" = 0x14,
147 @"sha3-384" = 0x15,
148 @"sha3-256" = 0x16,
149 @"sha3-224" = 0x17,
150 @"sha2-384" = 0x20,
151 @"sha2-256-trunc254-padded" = 0x1012,
152 @"sha2-224" = 0x1013,
153 @"sha2-512-224" = 0x1014,
154 @"sha2-512-256" = 0x1015,
155 @"blake2b-256" = 0xb220,
156 _,
157};
158
159pub const multihash_function: MultihashFunction = switch (Hash.Algo) {
160 std.crypto.hash.sha2.Sha256 => .@"sha2-256",
161 else => unreachable,
162};
163
164pub fn multiHashHexDigest(digest: Hash.Digest) MultiHashHexDigest {
165 const hex_charset = std.fmt.hex_charset;
166
167 var result: MultiHashHexDigest = undefined;
168
169 result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
170 result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
171
172 result[2] = hex_charset[Hash.Algo.digest_length >> 4];
173 result[3] = hex_charset[Hash.Algo.digest_length & 15];
174
175 for (digest, 0..) |byte, i| {
176 result[4 + i * 2] = hex_charset[byte >> 4];
177 result[5 + i * 2] = hex_charset[byte & 15];
178 }
179 return result;
180}
181
182comptime {
183 // We avoid unnecessary uleb128 code in hexDigest by asserting here the
184 // values are small enough to be contained in the one-byte encoding.
185 assert(@intFromEnum(multihash_function) < 127);
186 assert(Hash.Algo.digest_length < 127);
187}
188
189test Hash {
190 const example_digest: Hash.Digest = .{
191 0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87,
192 0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f,
193 };
194 const result: Hash = .init(example_digest, "nasm", "2.16.1-3", 0xcafebabe, 10 * 1024 * 1024);
195 try std.testing.expectEqualStrings("nasm-2.16.1-3-vrr-ygAAoADH9XG3tOdvPNuHen_d-XeHndOG-nNXmved", result.toSlice());
196}
197
6test {198test {
7 _ = Fetch;199 _ = Fetch;
8}200}
src/Package/Fetch.zig+102-75
...@@ -44,6 +44,8 @@ omit_missing_hash_error: bool,...@@ -44,6 +44,8 @@ omit_missing_hash_error: bool,
44/// which specifies inclusion rules. This is intended to be true for the first44/// which specifies inclusion rules. This is intended to be true for the first
45/// fetch task and false for the recursive dependencies.45/// fetch task and false for the recursive dependencies.
46allow_missing_paths_field: bool,46allow_missing_paths_field: bool,
47allow_missing_fingerprint: bool,
48allow_name_string: bool,
47/// If true and URL points to a Git repository, will use the latest commit.49/// If true and URL points to a Git repository, will use the latest commit.
48use_latest_commit: bool,50use_latest_commit: bool,
4951
...@@ -56,7 +58,7 @@ package_root: Cache.Path,...@@ -56,7 +58,7 @@ package_root: Cache.Path,
56error_bundle: ErrorBundle.Wip,58error_bundle: ErrorBundle.Wip,
57manifest: ?Manifest,59manifest: ?Manifest,
58manifest_ast: std.zig.Ast,60manifest_ast: std.zig.Ast,
59actual_hash: Manifest.Digest,61computed_hash: ComputedHash,
60/// Fetch logic notices whether a package has a build.zig file and sets this flag.62/// Fetch logic notices whether a package has a build.zig file and sets this flag.
61has_build_zig: bool,63has_build_zig: bool,
62/// Indicates whether the task aborted due to an out-of-memory condition.64/// Indicates whether the task aborted due to an out-of-memory condition.
...@@ -116,8 +118,8 @@ pub const JobQueue = struct {...@@ -116,8 +118,8 @@ pub const JobQueue = struct {
116 /// as lazy.118 /// as lazy.
117 unlazy_set: UnlazySet = .{},119 unlazy_set: UnlazySet = .{},
118120
119 pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);121 pub const Table = std.AutoArrayHashMapUnmanaged(Package.Hash, *Fetch);
120 pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, void);122 pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Package.Hash, void);
121123
122 pub fn deinit(jq: *JobQueue) void {124 pub fn deinit(jq: *JobQueue) void {
123 if (jq.all_fetches.items.len == 0) return;125 if (jq.all_fetches.items.len == 0) return;
...@@ -160,22 +162,24 @@ pub const JobQueue = struct {...@@ -160,22 +162,24 @@ pub const JobQueue = struct {
160162
161 // Ensure the generated .zig file is deterministic.163 // Ensure the generated .zig file is deterministic.
162 jq.table.sortUnstable(@as(struct {164 jq.table.sortUnstable(@as(struct {
163 keys: []const Manifest.MultiHashHexDigest,165 keys: []const Package.Hash,
164 pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {166 pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
165 return std.mem.lessThan(u8, &ctx.keys[a_index], &ctx.keys[b_index]);167 return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes);
166 }168 }
167 }, .{ .keys = keys }));169 }, .{ .keys = keys }));
168170
169 for (keys, jq.table.values()) |hash, fetch| {171 for (keys, jq.table.values()) |*hash, fetch| {
170 if (fetch == jq.all_fetches.items[0]) {172 if (fetch == jq.all_fetches.items[0]) {
171 // The first one is a dummy package for the current project.173 // The first one is a dummy package for the current project.
172 continue;174 continue;
173 }175 }
174176
177 const hash_slice = hash.toSlice();
178
175 try buf.writer().print(179 try buf.writer().print(
176 \\ pub const {} = struct {{180 \\ pub const {} = struct {{
177 \\181 \\
178 , .{std.zig.fmtId(&hash)});182 , .{std.zig.fmtId(hash_slice)});
179183
180 lazy: {184 lazy: {
181 switch (fetch.lazy_status) {185 switch (fetch.lazy_status) {
...@@ -207,7 +211,7 @@ pub const JobQueue = struct {...@@ -207,7 +211,7 @@ pub const JobQueue = struct {
207 try buf.writer().print(211 try buf.writer().print(
208 \\ pub const build_zig = @import("{}");212 \\ pub const build_zig = @import("{}");
209 \\213 \\
210 , .{std.zig.fmtEscapes(&hash)});214 , .{std.zig.fmtEscapes(hash_slice)});
211 }215 }
212216
213 if (fetch.manifest) |*manifest| {217 if (fetch.manifest) |*manifest| {
...@@ -219,7 +223,7 @@ pub const JobQueue = struct {...@@ -219,7 +223,7 @@ pub const JobQueue = struct {
219 const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue;223 const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue;
220 try buf.writer().print(224 try buf.writer().print(
221 " .{{ \"{}\", \"{}\" }},\n",225 " .{{ \"{}\", \"{}\" }},\n",
222 .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) },226 .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) },
223 );227 );
224 }228 }
225229
...@@ -251,7 +255,7 @@ pub const JobQueue = struct {...@@ -251,7 +255,7 @@ pub const JobQueue = struct {
251 const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue;255 const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue;
252 try buf.writer().print(256 try buf.writer().print(
253 " .{{ \"{}\", \"{}\" }},\n",257 " .{{ \"{}\", \"{}\" }},\n",
254 .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) },258 .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) },
255 );259 );
256 }260 }
257 try buf.appendSlice("};\n");261 try buf.appendSlice("};\n");
...@@ -283,7 +287,7 @@ pub const Location = union(enum) {...@@ -283,7 +287,7 @@ pub const Location = union(enum) {
283 url: []const u8,287 url: []const u8,
284 /// If this is null it means the user omitted the hash field from a dependency.288 /// If this is null it means the user omitted the hash field from a dependency.
285 /// It will be an error but the logic should still fetch and print the discovered hash.289 /// It will be an error but the logic should still fetch and print the discovered hash.
286 hash: ?Manifest.MultiHashHexDigest,290 hash: ?Package.Hash,
287 };291 };
288};292};
289293
...@@ -325,9 +329,11 @@ pub fn run(f: *Fetch) RunError!void {...@@ -325,9 +329,11 @@ pub fn run(f: *Fetch) RunError!void {
325 // "p/$hash/foo", with possibly more directories after "foo".329 // "p/$hash/foo", with possibly more directories after "foo".
326 // We want to fail unless the resolved relative path has a330 // We want to fail unless the resolved relative path has a
327 // prefix of "p/$hash/".331 // prefix of "p/$hash/".
328 const digest_len = @typeInfo(Manifest.MultiHashHexDigest).array.len;
329 const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len;332 const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len;
330 const expected_prefix = f.parent_package_root.sub_path[0 .. prefix_len + digest_len];333 const parent_sub_path = f.parent_package_root.sub_path;
334 const end = std.mem.indexOfScalarPos(u8, parent_sub_path, prefix_len, fs.path.sep) orelse
335 parent_sub_path.len;
336 const expected_prefix = parent_sub_path[prefix_len..end];
331 if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) {337 if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) {
332 return f.fail(338 return f.fail(
333 f.location_tok,339 f.location_tok,
...@@ -367,9 +373,13 @@ pub fn run(f: *Fetch) RunError!void {...@@ -367,9 +373,13 @@ pub fn run(f: *Fetch) RunError!void {
367 },373 },
368 };374 };
369375
370 const s = fs.path.sep_str;
371 if (remote.hash) |expected_hash| {376 if (remote.hash) |expected_hash| {
372 const prefixed_pkg_sub_path = "p" ++ s ++ expected_hash;377 var prefixed_pkg_sub_path_buffer: [Package.Hash.max_len + 2]u8 = undefined;
378 prefixed_pkg_sub_path_buffer[0] = 'p';
379 prefixed_pkg_sub_path_buffer[1] = fs.path.sep;
380 const hash_slice = expected_hash.toSlice();
381 @memcpy(prefixed_pkg_sub_path_buffer[2..][0..hash_slice.len], hash_slice);
382 const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len];
373 const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;383 const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;
374 const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];384 const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];
375 if (cache_root.handle.access(pkg_sub_path, .{})) |_| {385 if (cache_root.handle.access(pkg_sub_path, .{})) |_| {
...@@ -437,7 +447,7 @@ fn runResource(...@@ -437,7 +447,7 @@ fn runResource(
437 f: *Fetch,447 f: *Fetch,
438 uri_path: []const u8,448 uri_path: []const u8,
439 resource: *Resource,449 resource: *Resource,
440 remote_hash: ?Manifest.MultiHashHexDigest,450 remote_hash: ?Package.Hash,
441) RunError!void {451) RunError!void {
442 defer resource.deinit();452 defer resource.deinit();
443 const arena = f.arena.allocator();453 const arena = f.arena.allocator();
...@@ -499,7 +509,7 @@ fn runResource(...@@ -499,7 +509,7 @@ fn runResource(
499 // Empty directories have already been omitted by `unpackResource`.509 // Empty directories have already been omitted by `unpackResource`.
500 // Compute the package hash based on the remaining files in the temporary510 // Compute the package hash based on the remaining files in the temporary
501 // directory.511 // directory.
502 f.actual_hash = try computeHash(f, pkg_path, filter);512 f.computed_hash = try computeHash(f, pkg_path, filter);
503513
504 break :blk if (unpack_result.root_dir.len > 0)514 break :blk if (unpack_result.root_dir.len > 0)
505 try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir })515 try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir })
...@@ -507,6 +517,8 @@ fn runResource(...@@ -507,6 +517,8 @@ fn runResource(
507 tmp_dir_sub_path;517 tmp_dir_sub_path;
508 };518 };
509519
520 const computed_package_hash = computedPackageHash(f);
521
510 // Rename the temporary directory into the global zig package cache522 // Rename the temporary directory into the global zig package cache
511 // directory. If the hash already exists, delete the temporary directory523 // directory. If the hash already exists, delete the temporary directory
512 // and leave the zig package cache directory untouched as it may be in use524 // and leave the zig package cache directory untouched as it may be in use
...@@ -515,7 +527,7 @@ fn runResource(...@@ -515,7 +527,7 @@ fn runResource(
515527
516 f.package_root = .{528 f.package_root = .{
517 .root_dir = cache_root,529 .root_dir = cache_root,
518 .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)),530 .sub_path = try std.fmt.allocPrint(arena, "p" ++ s ++ "{s}", .{computed_package_hash.toSlice()}),
519 };531 };
520 renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| {532 renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| {
521 const src = try cache_root.join(arena, &.{tmp_dir_sub_path});533 const src = try cache_root.join(arena, &.{tmp_dir_sub_path});
...@@ -534,13 +546,22 @@ fn runResource(...@@ -534,13 +546,22 @@ fn runResource(
534 // Validate the computed hash against the expected hash. If invalid, this546 // Validate the computed hash against the expected hash. If invalid, this
535 // job is done.547 // job is done.
536548
537 const actual_hex = Manifest.hexDigest(f.actual_hash);
538 if (remote_hash) |declared_hash| {549 if (remote_hash) |declared_hash| {
539 if (!std.mem.eql(u8, &declared_hash, &actual_hex)) {550 if (declared_hash.isOld()) {
540 return f.fail(f.hash_tok, try eb.printString(551 const actual_hex = Package.multiHashHexDigest(f.computed_hash.digest);
541 "hash mismatch: manifest declares {s} but the fetched package has {s}",552 if (!std.mem.eql(u8, declared_hash.toSlice(), &actual_hex)) {
542 .{ declared_hash, actual_hex },553 return f.fail(f.hash_tok, try eb.printString(
543 ));554 "hash mismatch: manifest declares {s} but the fetched package has {s}",
555 .{ declared_hash.toSlice(), actual_hex },
556 ));
557 }
558 } else {
559 if (!computed_package_hash.eql(&declared_hash)) {
560 return f.fail(f.hash_tok, try eb.printString(
561 "hash mismatch: manifest declares {s} but the fetched package has {s}",
562 .{ declared_hash.toSlice(), computed_package_hash.toSlice() },
563 ));
564 }
544 }565 }
545 } else if (!f.omit_missing_hash_error) {566 } else if (!f.omit_missing_hash_error) {
546 const notes_len = 1;567 const notes_len = 1;
...@@ -551,7 +572,7 @@ fn runResource(...@@ -551,7 +572,7 @@ fn runResource(
551 });572 });
552 const notes_start = try eb.reserveNotes(notes_len);573 const notes_start = try eb.reserveNotes(notes_len);
553 eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{574 eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
554 .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),575 .msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}),
555 }));576 }));
556 return error.FetchFailed;577 return error.FetchFailed;
557 }578 }
...@@ -562,6 +583,18 @@ fn runResource(...@@ -562,6 +583,18 @@ fn runResource(
562 return queueJobsForDeps(f);583 return queueJobsForDeps(f);
563}584}
564585
586pub fn computedPackageHash(f: *const Fetch) Package.Hash {
587 const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32);
588 if (f.manifest) |man| {
589 var version_buffer: [32]u8 = undefined;
590 const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{}", .{man.version}) catch &version_buffer;
591 return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size);
592 }
593 // In the future build.zig.zon fields will be added to allow overriding these values
594 // for naked tarballs.
595 return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size);
596}
597
565/// `computeHash` gets a free check for the existence of `build.zig`, but when598/// `computeHash` gets a free check for the existence of `build.zig`, but when
566/// not computing a hash, we need to do a syscall to check for it.599/// not computing a hash, we need to do a syscall to check for it.
567fn checkBuildFileExistence(f: *Fetch) RunError!void {600fn checkBuildFileExistence(f: *Fetch) RunError!void {
...@@ -616,11 +649,13 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {...@@ -616,11 +649,13 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {
616649
617 f.manifest = try Manifest.parse(arena, ast.*, .{650 f.manifest = try Manifest.parse(arena, ast.*, .{
618 .allow_missing_paths_field = f.allow_missing_paths_field,651 .allow_missing_paths_field = f.allow_missing_paths_field,
652 .allow_missing_fingerprint = f.allow_missing_fingerprint,
653 .allow_name_string = f.allow_name_string,
619 });654 });
620 const manifest = &f.manifest.?;655 const manifest = &f.manifest.?;
621656
622 if (manifest.errors.len > 0) {657 if (manifest.errors.len > 0) {
623 const src_path = try eb.printString("{}{s}", .{ pkg_root, Manifest.basename });658 const src_path = try eb.printString("{}" ++ fs.path.sep_str ++ "{s}", .{ pkg_root, Manifest.basename });
624 try manifest.copyErrorsIntoBundle(ast.*, src_path, eb);659 try manifest.copyErrorsIntoBundle(ast.*, src_path, eb);
625 return error.FetchFailed;660 return error.FetchFailed;
626 }661 }
...@@ -673,9 +708,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {...@@ -673,9 +708,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
673 .url = url,708 .url = url,
674 .hash = h: {709 .hash = h: {
675 const h = dep.hash orelse break :h null;710 const h = dep.hash orelse break :h null;
676 const digest_len = @typeInfo(Manifest.MultiHashHexDigest).array.len;711 const pkg_hash: Package.Hash = .fromSlice(h);
677 const multihash_digest = h[0..digest_len].*;712 const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
678 const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
679 if (gop.found_existing) {713 if (gop.found_existing) {
680 if (!dep.lazy) {714 if (!dep.lazy) {
681 gop.value_ptr.*.lazy_status = .eager;715 gop.value_ptr.*.lazy_status = .eager;
...@@ -683,15 +717,15 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {...@@ -683,15 +717,15 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
683 continue;717 continue;
684 }718 }
685 gop.value_ptr.* = new_fetch;719 gop.value_ptr.* = new_fetch;
686 break :h multihash_digest;720 break :h pkg_hash;
687 },721 },
688 } },722 } },
689 .path => |rel_path| l: {723 .path => |rel_path| l: {
690 // This might produce an invalid path, which is checked for724 // This might produce an invalid path, which is checked for
691 // at the beginning of run().725 // at the beginning of run().
692 const new_root = try f.package_root.resolvePosix(parent_arena, rel_path);726 const new_root = try f.package_root.resolvePosix(parent_arena, rel_path);
693 const multihash_digest = relativePathDigest(new_root, cache_root);727 const pkg_hash = relativePathDigest(new_root, cache_root);
694 const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);728 const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
695 if (gop.found_existing) {729 if (gop.found_existing) {
696 if (!dep.lazy) {730 if (!dep.lazy) {
697 gop.value_ptr.*.lazy_status = .eager;731 gop.value_ptr.*.lazy_status = .eager;
...@@ -718,13 +752,15 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {...@@ -718,13 +752,15 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
718 .job_queue = f.job_queue,752 .job_queue = f.job_queue,
719 .omit_missing_hash_error = false,753 .omit_missing_hash_error = false,
720 .allow_missing_paths_field = true,754 .allow_missing_paths_field = true,
755 .allow_missing_fingerprint = true,
756 .allow_name_string = true,
721 .use_latest_commit = false,757 .use_latest_commit = false,
722758
723 .package_root = undefined,759 .package_root = undefined,
724 .error_bundle = undefined,760 .error_bundle = undefined,
725 .manifest = null,761 .manifest = null,
726 .manifest_ast = undefined,762 .manifest_ast = undefined,
727 .actual_hash = undefined,763 .computed_hash = undefined,
728 .has_build_zig = false,764 .has_build_zig = false,
729 .oom_flag = false,765 .oom_flag = false,
730 .latest_commit = null,766 .latest_commit = null,
...@@ -746,20 +782,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {...@@ -746,20 +782,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
746 }782 }
747}783}
748784
749pub fn relativePathDigest(785pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash {
750 pkg_root: Cache.Path,786 return .initPath(pkg_root.sub_path, pkg_root.root_dir.eql(cache_root));
751 cache_root: Cache.Directory,
752) Manifest.MultiHashHexDigest {
753 var hasher = Manifest.Hash.init(.{});
754 // This hash is a tuple of:
755 // * whether it relative to the global cache directory or to the root package
756 // * the relative file path from there to the build root of the package
757 hasher.update(if (pkg_root.root_dir.eql(cache_root))
758 &package_hash_prefix_cached
759 else
760 &package_hash_prefix_project);
761 hasher.update(pkg_root.sub_path);
762 return Manifest.hexDigest(hasher.finalResult());
763}787}
764788
765pub fn workerRun(f: *Fetch, prog_name: []const u8) void {789pub fn workerRun(f: *Fetch, prog_name: []const u8) void {
...@@ -1387,11 +1411,7 @@ fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void...@@ -1387,11 +1411,7 @@ fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void
1387 }1411 }
1388}1412}
13891413
1390pub fn renameTmpIntoCache(1414pub fn renameTmpIntoCache(cache_dir: fs.Dir, tmp_dir_sub_path: []const u8, dest_dir_sub_path: []const u8) !void {
1391 cache_dir: fs.Dir,
1392 tmp_dir_sub_path: []const u8,
1393 dest_dir_sub_path: []const u8,
1394) !void {
1395 assert(dest_dir_sub_path[1] == fs.path.sep);1415 assert(dest_dir_sub_path[1] == fs.path.sep);
1396 var handled_missing_dir = false;1416 var handled_missing_dir = false;
1397 while (true) {1417 while (true) {
...@@ -1417,16 +1437,17 @@ pub fn renameTmpIntoCache(...@@ -1417,16 +1437,17 @@ pub fn renameTmpIntoCache(
1417 }1437 }
1418}1438}
14191439
1440const ComputedHash = struct {
1441 digest: Package.Hash.Digest,
1442 total_size: u64,
1443};
1444
1420/// Assumes that files not included in the package have already been filtered1445/// Assumes that files not included in the package have already been filtered
1421/// prior to calling this function. This ensures that files not protected by1446/// prior to calling this function. This ensures that files not protected by
1422/// the hash are not present on the file system. Empty directories are *not1447/// the hash are not present on the file system. Empty directories are *not
1423/// hashed* and must not be present on the file system when calling this1448/// hashed* and must not be present on the file system when calling this
1424/// function.1449/// function.
1425fn computeHash(1450fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash {
1426 f: *Fetch,
1427 pkg_path: Cache.Path,
1428 filter: Filter,
1429) RunError!Manifest.Digest {
1430 // All the path name strings need to be in memory for sorting.1451 // All the path name strings need to be in memory for sorting.
1431 const arena = f.arena.allocator();1452 const arena = f.arena.allocator();
1432 const gpa = f.arena.child_allocator;1453 const gpa = f.arena.child_allocator;
...@@ -1449,6 +1470,9 @@ fn computeHash(...@@ -1449,6 +1470,9 @@ fn computeHash(
1449 var walker = try root_dir.walk(gpa);1470 var walker = try root_dir.walk(gpa);
1450 defer walker.deinit();1471 defer walker.deinit();
14511472
1473 // Total number of bytes of file contents included in the package.
1474 var total_size: u64 = 0;
1475
1452 {1476 {
1453 // The final hash will be a hash of each file hashed independently. This1477 // The final hash will be a hash of each file hashed independently. This
1454 // allows hashing in parallel.1478 // allows hashing in parallel.
...@@ -1506,6 +1530,7 @@ fn computeHash(...@@ -1506,6 +1530,7 @@ fn computeHash(
1506 .kind = kind,1530 .kind = kind,
1507 .hash = undefined, // to be populated by the worker1531 .hash = undefined, // to be populated by the worker
1508 .failure = undefined, // to be populated by the worker1532 .failure = undefined, // to be populated by the worker
1533 .size = undefined, // to be populated by the worker
1509 };1534 };
1510 thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file });1535 thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file });
1511 try all_files.append(hashed_file);1536 try all_files.append(hashed_file);
...@@ -1544,7 +1569,7 @@ fn computeHash(...@@ -1544,7 +1569,7 @@ fn computeHash(
15441569
1545 std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);1570 std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
15461571
1547 var hasher = Manifest.Hash.init(.{});1572 var hasher = Package.Hash.Algo.init(.{});
1548 var any_failures = false;1573 var any_failures = false;
1549 for (all_files.items) |hashed_file| {1574 for (all_files.items) |hashed_file| {
1550 hashed_file.failure catch |err| {1575 hashed_file.failure catch |err| {
...@@ -1556,6 +1581,7 @@ fn computeHash(...@@ -1556,6 +1581,7 @@ fn computeHash(
1556 });1581 });
1557 };1582 };
1558 hasher.update(&hashed_file.hash);1583 hasher.update(&hashed_file.hash);
1584 total_size += hashed_file.size;
1559 }1585 }
1560 for (deleted_files.items) |deleted_file| {1586 for (deleted_files.items) |deleted_file| {
1561 deleted_file.failure catch |err| {1587 deleted_file.failure catch |err| {
...@@ -1580,7 +1606,10 @@ fn computeHash(...@@ -1580,7 +1606,10 @@ fn computeHash(
1580 };1606 };
1581 }1607 }
15821608
1583 return hasher.finalResult();1609 return .{
1610 .digest = hasher.finalResult(),
1611 .total_size = total_size,
1612 };
1584}1613}
15851614
1586fn dumpHashInfo(all_files: []const *const HashedFile) !void {1615fn dumpHashInfo(all_files: []const *const HashedFile) !void {
...@@ -1609,8 +1638,9 @@ fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void {...@@ -1609,8 +1638,9 @@ fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void {
16091638
1610fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {1639fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
1611 var buf: [8000]u8 = undefined;1640 var buf: [8000]u8 = undefined;
1612 var hasher = Manifest.Hash.init(.{});1641 var hasher = Package.Hash.Algo.init(.{});
1613 hasher.update(hashed_file.normalized_path);1642 hasher.update(hashed_file.normalized_path);
1643 var file_size: u64 = 0;
16141644
1615 switch (hashed_file.kind) {1645 switch (hashed_file.kind) {
1616 .file => {1646 .file => {
...@@ -1622,6 +1652,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void...@@ -1622,6 +1652,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
1622 while (true) {1652 while (true) {
1623 const bytes_read = try file.read(&buf);1653 const bytes_read = try file.read(&buf);
1624 if (bytes_read == 0) break;1654 if (bytes_read == 0) break;
1655 file_size += bytes_read;
1625 hasher.update(buf[0..bytes_read]);1656 hasher.update(buf[0..bytes_read]);
1626 file_header.update(buf[0..bytes_read]);1657 file_header.update(buf[0..bytes_read]);
1627 }1658 }
...@@ -1641,6 +1672,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void...@@ -1641,6 +1672,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
1641 },1672 },
1642 }1673 }
1643 hasher.final(&hashed_file.hash);1674 hasher.final(&hashed_file.hash);
1675 hashed_file.size = file_size;
1644}1676}
16451677
1646fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {1678fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {
...@@ -1667,9 +1699,10 @@ const DeletedFile = struct {...@@ -1667,9 +1699,10 @@ const DeletedFile = struct {
1667const HashedFile = struct {1699const HashedFile = struct {
1668 fs_path: []const u8,1700 fs_path: []const u8,
1669 normalized_path: []const u8,1701 normalized_path: []const u8,
1670 hash: Manifest.Digest,1702 hash: Package.Hash.Digest,
1671 failure: Error!void,1703 failure: Error!void,
1672 kind: Kind,1704 kind: Kind,
1705 size: u64,
16731706
1674 const Error =1707 const Error =
1675 fs.File.OpenError ||1708 fs.File.OpenError ||
...@@ -1744,12 +1777,8 @@ const Filter = struct {...@@ -1744,12 +1777,8 @@ const Filter = struct {
1744 }1777 }
1745};1778};
17461779
1747pub fn depDigest(1780pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash {
1748 pkg_root: Cache.Path,1781 if (dep.hash) |h| return .fromSlice(h);
1749 cache_root: Cache.Directory,
1750 dep: Manifest.Dependency,
1751) ?Manifest.MultiHashHexDigest {
1752 if (dep.hash) |h| return h[0..Manifest.multihash_hex_digest_len].*;
17531782
1754 switch (dep.location) {1783 switch (dep.location) {
1755 .url => return null,1784 .url => return null,
...@@ -1763,10 +1792,6 @@ pub fn depDigest(...@@ -1763,10 +1792,6 @@ pub fn depDigest(
1763 }1792 }
1764}1793}
17651794
1766// These are random bytes.
1767const package_hash_prefix_cached = [8]u8{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 };
1768const package_hash_prefix_project = [8]u8{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc };
1769
1770const builtin = @import("builtin");1795const builtin = @import("builtin");
1771const std = @import("std");1796const std = @import("std");
1772const fs = std.fs;1797const fs = std.fs;
...@@ -2137,7 +2162,7 @@ test "tarball with excluded duplicate paths" {...@@ -2137,7 +2162,7 @@ test "tarball with excluded duplicate paths" {
2137 defer fb.deinit();2162 defer fb.deinit();
2138 try fetch.run();2163 try fetch.run();
21392164
2140 const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);2165 const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest);
2141 try std.testing.expectEqualStrings(2166 try std.testing.expectEqualStrings(
2142 "12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da",2167 "12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da",
2143 &hex_digest,2168 &hex_digest,
...@@ -2181,7 +2206,7 @@ test "tarball without root folder" {...@@ -2181,7 +2206,7 @@ test "tarball without root folder" {
2181 defer fb.deinit();2206 defer fb.deinit();
2182 try fetch.run();2207 try fetch.run();
21832208
2184 const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);2209 const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest);
2185 try std.testing.expectEqualStrings(2210 try std.testing.expectEqualStrings(
2186 "12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793",2211 "12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793",
2187 &hex_digest,2212 &hex_digest,
...@@ -2222,7 +2247,7 @@ test "set executable bit based on file content" {...@@ -2222,7 +2247,7 @@ test "set executable bit based on file content" {
2222 try fetch.run();2247 try fetch.run();
2223 try std.testing.expectEqualStrings(2248 try std.testing.expectEqualStrings(
2224 "1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3",2249 "1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3",
2225 &Manifest.hexDigest(fetch.actual_hash),2250 &Package.multiHashHexDigest(fetch.computed_hash.digest),
2226 );2251 );
22272252
2228 var out = try fb.packageDir();2253 var out = try fb.packageDir();
...@@ -2298,13 +2323,15 @@ const TestFetchBuilder = struct {...@@ -2298,13 +2323,15 @@ const TestFetchBuilder = struct {
2298 .job_queue = &self.job_queue,2323 .job_queue = &self.job_queue,
2299 .omit_missing_hash_error = true,2324 .omit_missing_hash_error = true,
2300 .allow_missing_paths_field = false,2325 .allow_missing_paths_field = false,
2326 .allow_missing_fingerprint = true, // so we can keep using the old testdata .tar.gz
2327 .allow_name_string = true, // so we can keep using the old testdata .tar.gz
2301 .use_latest_commit = true,2328 .use_latest_commit = true,
23022329
2303 .package_root = undefined,2330 .package_root = undefined,
2304 .error_bundle = undefined,2331 .error_bundle = undefined,
2305 .manifest = null,2332 .manifest = null,
2306 .manifest_ast = undefined,2333 .manifest_ast = undefined,
2307 .actual_hash = undefined,2334 .computed_hash = undefined,
2308 .has_build_zig = false,2335 .has_build_zig = false,
2309 .oom_flag = false,2336 .oom_flag = false,
2310 .latest_commit = null,2337 .latest_commit = null,
src/Package/Manifest.zig+92-67
...@@ -5,15 +5,12 @@ const Allocator = std.mem.Allocator;...@@ -5,15 +5,12 @@ const Allocator = std.mem.Allocator;
5const assert = std.debug.assert;5const assert = std.debug.assert;
6const Ast = std.zig.Ast;6const Ast = std.zig.Ast;
7const testing = std.testing;7const testing = std.testing;
8const hex_charset = std.fmt.hex_charset;8const Package = @import("../Package.zig");
99
10pub const max_bytes = 10 * 1024 * 1024;10pub const max_bytes = 10 * 1024 * 1024;
11pub const basename = "build.zig.zon";11pub const basename = "build.zig.zon";
12pub const Hash = std.crypto.hash.sha2.Sha256;12pub const max_name_len = 32;
13pub const Digest = [Hash.digest_length]u8;13pub const max_version_len = 32;
14pub const multihash_len = 1 + 1 + Hash.digest_length;
15pub const multihash_hex_digest_len = 2 * multihash_len;
16pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
1714
18pub const Dependency = struct {15pub const Dependency = struct {
19 location: Location,16 location: Location,
...@@ -38,36 +35,8 @@ pub const ErrorMessage = struct {...@@ -38,36 +35,8 @@ pub const ErrorMessage = struct {
38 off: u32,35 off: u32,
39};36};
4037
41pub const MultihashFunction = enum(u16) {
42 identity = 0x00,
43 sha1 = 0x11,
44 @"sha2-256" = 0x12,
45 @"sha2-512" = 0x13,
46 @"sha3-512" = 0x14,
47 @"sha3-384" = 0x15,
48 @"sha3-256" = 0x16,
49 @"sha3-224" = 0x17,
50 @"sha2-384" = 0x20,
51 @"sha2-256-trunc254-padded" = 0x1012,
52 @"sha2-224" = 0x1013,
53 @"sha2-512-224" = 0x1014,
54 @"sha2-512-256" = 0x1015,
55 @"blake2b-256" = 0xb220,
56 _,
57};
58
59pub const multihash_function: MultihashFunction = switch (Hash) {
60 std.crypto.hash.sha2.Sha256 => .@"sha2-256",
61 else => @compileError("unreachable"),
62};
63comptime {
64 // We avoid unnecessary uleb128 code in hexDigest by asserting here the
65 // values are small enough to be contained in the one-byte encoding.
66 assert(@intFromEnum(multihash_function) < 127);
67 assert(Hash.digest_length < 127);
68}
69
70name: []const u8,38name: []const u8,
39id: u32,
71version: std.SemanticVersion,40version: std.SemanticVersion,
72version_node: Ast.Node.Index,41version_node: Ast.Node.Index,
73dependencies: std.StringArrayHashMapUnmanaged(Dependency),42dependencies: std.StringArrayHashMapUnmanaged(Dependency),
...@@ -80,6 +49,10 @@ arena_state: std.heap.ArenaAllocator.State,...@@ -80,6 +49,10 @@ arena_state: std.heap.ArenaAllocator.State,
8049
81pub const ParseOptions = struct {50pub const ParseOptions = struct {
82 allow_missing_paths_field: bool = false,51 allow_missing_paths_field: bool = false,
52 /// Deprecated, to be removed after 0.14.0 is tagged.
53 allow_name_string: bool = true,
54 /// Deprecated, to be removed after 0.14.0 is tagged.
55 allow_missing_fingerprint: bool = true,
83};56};
8457
85pub const Error = Allocator.Error;58pub const Error = Allocator.Error;
...@@ -100,12 +73,15 @@ pub fn parse(gpa: Allocator, ast: Ast, options: ParseOptions) Error!Manifest {...@@ -100,12 +73,15 @@ pub fn parse(gpa: Allocator, ast: Ast, options: ParseOptions) Error!Manifest {
100 .errors = .{},73 .errors = .{},
10174
102 .name = undefined,75 .name = undefined,
76 .id = 0,
103 .version = undefined,77 .version = undefined,
104 .version_node = 0,78 .version_node = 0,
105 .dependencies = .{},79 .dependencies = .{},
106 .dependencies_node = 0,80 .dependencies_node = 0,
107 .paths = .{},81 .paths = .{},
108 .allow_missing_paths_field = options.allow_missing_paths_field,82 .allow_missing_paths_field = options.allow_missing_paths_field,
83 .allow_name_string = options.allow_name_string,
84 .allow_missing_fingerprint = options.allow_missing_fingerprint,
109 .minimum_zig_version = null,85 .minimum_zig_version = null,
110 .buf = .{},86 .buf = .{},
111 };87 };
...@@ -121,6 +97,7 @@ pub fn parse(gpa: Allocator, ast: Ast, options: ParseOptions) Error!Manifest {...@@ -121,6 +97,7 @@ pub fn parse(gpa: Allocator, ast: Ast, options: ParseOptions) Error!Manifest {
12197
122 return .{98 return .{
123 .name = p.name,99 .name = p.name,
100 .id = p.id,
124 .version = p.version,101 .version = p.version,
125 .version_node = p.version_node,102 .version_node = p.version_node,
126 .dependencies = try p.dependencies.clone(p.arena),103 .dependencies = try p.dependencies.clone(p.arena),
...@@ -164,22 +141,6 @@ pub fn copyErrorsIntoBundle(...@@ -164,22 +141,6 @@ pub fn copyErrorsIntoBundle(
164 }141 }
165}142}
166143
167pub fn hexDigest(digest: Digest) MultiHashHexDigest {
168 var result: MultiHashHexDigest = undefined;
169
170 result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
171 result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
172
173 result[2] = hex_charset[Hash.digest_length >> 4];
174 result[3] = hex_charset[Hash.digest_length & 15];
175
176 for (digest, 0..) |byte, i| {
177 result[4 + i * 2] = hex_charset[byte >> 4];
178 result[5 + i * 2] = hex_charset[byte & 15];
179 }
180 return result;
181}
182
183const Parse = struct {144const Parse = struct {
184 gpa: Allocator,145 gpa: Allocator,
185 ast: Ast,146 ast: Ast,
...@@ -188,12 +149,15 @@ const Parse = struct {...@@ -188,12 +149,15 @@ const Parse = struct {
188 errors: std.ArrayListUnmanaged(ErrorMessage),149 errors: std.ArrayListUnmanaged(ErrorMessage),
189150
190 name: []const u8,151 name: []const u8,
152 id: u32,
191 version: std.SemanticVersion,153 version: std.SemanticVersion,
192 version_node: Ast.Node.Index,154 version_node: Ast.Node.Index,
193 dependencies: std.StringArrayHashMapUnmanaged(Dependency),155 dependencies: std.StringArrayHashMapUnmanaged(Dependency),
194 dependencies_node: Ast.Node.Index,156 dependencies_node: Ast.Node.Index,
195 paths: std.StringArrayHashMapUnmanaged(void),157 paths: std.StringArrayHashMapUnmanaged(void),
196 allow_missing_paths_field: bool,158 allow_missing_paths_field: bool,
159 allow_name_string: bool,
160 allow_missing_fingerprint: bool,
197 minimum_zig_version: ?std.SemanticVersion,161 minimum_zig_version: ?std.SemanticVersion,
198162
199 const InnerError = error{ ParseFailure, OutOfMemory };163 const InnerError = error{ ParseFailure, OutOfMemory };
...@@ -211,6 +175,7 @@ const Parse = struct {...@@ -211,6 +175,7 @@ const Parse = struct {
211 var have_name = false;175 var have_name = false;
212 var have_version = false;176 var have_version = false;
213 var have_included_paths = false;177 var have_included_paths = false;
178 var fingerprint: ?Package.Fingerprint = null;
214179
215 for (struct_init.ast.fields) |field_init| {180 for (struct_init.ast.fields) |field_init| {
216 const name_token = ast.firstToken(field_init) - 2;181 const name_token = ast.firstToken(field_init) - 2;
...@@ -225,11 +190,16 @@ const Parse = struct {...@@ -225,11 +190,16 @@ const Parse = struct {
225 have_included_paths = true;190 have_included_paths = true;
226 try parseIncludedPaths(p, field_init);191 try parseIncludedPaths(p, field_init);
227 } else if (mem.eql(u8, field_name, "name")) {192 } else if (mem.eql(u8, field_name, "name")) {
228 p.name = try parseString(p, field_init);193 p.name = try parseName(p, field_init);
229 have_name = true;194 have_name = true;
195 } else if (mem.eql(u8, field_name, "fingerprint")) {
196 fingerprint = try parseFingerprint(p, field_init);
230 } else if (mem.eql(u8, field_name, "version")) {197 } else if (mem.eql(u8, field_name, "version")) {
231 p.version_node = field_init;198 p.version_node = field_init;
232 const version_text = try parseString(p, field_init);199 const version_text = try parseString(p, field_init);
200 if (version_text.len > max_version_len) {
201 try appendError(p, main_tokens[field_init], "version string length {d} exceeds maximum of {d}", .{ version_text.len, max_version_len });
202 }
233 p.version = std.SemanticVersion.parse(version_text) catch |err| v: {203 p.version = std.SemanticVersion.parse(version_text) catch |err| v: {
234 try appendError(p, main_tokens[field_init], "unable to parse semantic version: {s}", .{@errorName(err)});204 try appendError(p, main_tokens[field_init], "unable to parse semantic version: {s}", .{@errorName(err)});
235 break :v undefined;205 break :v undefined;
...@@ -249,6 +219,21 @@ const Parse = struct {...@@ -249,6 +219,21 @@ const Parse = struct {
249219
250 if (!have_name) {220 if (!have_name) {
251 try appendError(p, main_token, "missing top-level 'name' field", .{});221 try appendError(p, main_token, "missing top-level 'name' field", .{});
222 } else {
223 if (fingerprint) |n| {
224 if (!n.validate(p.name)) {
225 return fail(p, main_token, "invalid fingerprint: 0x{x}; if this is a new or forked package, use this value: 0x{x}", .{
226 n.int(), Package.Fingerprint.generate(p.name).int(),
227 });
228 }
229 p.id = n.id;
230 } else if (!p.allow_missing_fingerprint) {
231 try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{
232 Package.Fingerprint.generate(p.name).int(),
233 });
234 } else {
235 p.id = 0;
236 }
252 }237 }
253238
254 if (!have_version) {239 if (!have_version) {
...@@ -400,6 +385,59 @@ const Parse = struct {...@@ -400,6 +385,59 @@ const Parse = struct {
400 }385 }
401 }386 }
402387
388 fn parseFingerprint(p: *Parse, node: Ast.Node.Index) !Package.Fingerprint {
389 const ast = p.ast;
390 const node_tags = ast.nodes.items(.tag);
391 const main_tokens = ast.nodes.items(.main_token);
392 const main_token = main_tokens[node];
393 if (node_tags[node] != .number_literal) {
394 return fail(p, main_token, "expected integer literal", .{});
395 }
396 const token_bytes = ast.tokenSlice(main_token);
397 const parsed = std.zig.parseNumberLiteral(token_bytes);
398 switch (parsed) {
399 .int => |n| return @bitCast(n),
400 .big_int, .float => return fail(p, main_token, "expected u64 integer literal, found {s}", .{
401 @tagName(parsed),
402 }),
403 .failure => |err| return fail(p, main_token, "bad integer literal: {s}", .{@tagName(err)}),
404 }
405 }
406
407 fn parseName(p: *Parse, node: Ast.Node.Index) ![]const u8 {
408 const ast = p.ast;
409 const node_tags = ast.nodes.items(.tag);
410 const main_tokens = ast.nodes.items(.main_token);
411 const main_token = main_tokens[node];
412
413 if (p.allow_name_string and node_tags[node] == .string_literal) {
414 const name = try parseString(p, node);
415 if (!std.zig.isValidId(name))
416 return fail(p, main_token, "name must be a valid bare zig identifier (hint: switch from string to enum literal)", .{});
417
418 if (name.len > max_name_len)
419 return fail(p, main_token, "name '{}' exceeds max length of {d}", .{
420 std.zig.fmtId(name), max_name_len,
421 });
422
423 return name;
424 }
425
426 if (node_tags[node] != .enum_literal)
427 return fail(p, main_token, "expected enum literal", .{});
428
429 const ident_name = ast.tokenSlice(main_token);
430 if (mem.startsWith(u8, ident_name, "@"))
431 return fail(p, main_token, "name must be a valid bare zig identifier", .{});
432
433 if (ident_name.len > max_name_len)
434 return fail(p, main_token, "name '{}' exceeds max length of {d}", .{
435 std.zig.fmtId(ident_name), max_name_len,
436 });
437
438 return ident_name;
439 }
440
403 fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 {441 fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 {
404 const ast = p.ast;442 const ast = p.ast;
405 const node_tags = ast.nodes.items(.tag);443 const node_tags = ast.nodes.items(.tag);
...@@ -421,21 +459,8 @@ const Parse = struct {...@@ -421,21 +459,8 @@ const Parse = struct {
421 const tok = main_tokens[node];459 const tok = main_tokens[node];
422 const h = try parseString(p, node);460 const h = try parseString(p, node);
423461
424 if (h.len >= 2) {462 if (h.len > Package.Hash.max_len) {
425 const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| {463 return fail(p, tok, "hash length exceeds maximum: {d}", .{h.len});
426 return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{
427 @errorName(err),
428 });
429 };
430 if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) {
431 return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{});
432 }
433 }
434
435 if (h.len != multihash_hex_digest_len) {
436 return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{
437 multihash_hex_digest_len, h.len,
438 });
439 }464 }
440465
441 return h;466 return h;
src/main.zig+84-26
...@@ -4741,6 +4741,7 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -4741,6 +4741,7 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
47414741
4742 const cwd_path = try process.getCwdAlloc(arena);4742 const cwd_path = try process.getCwdAlloc(arena);
4743 const cwd_basename = fs.path.basename(cwd_path);4743 const cwd_basename = fs.path.basename(cwd_path);
4744 const sanitized_root_name = try sanitizeExampleName(arena, cwd_basename);
47444745
4745 const s = fs.path.sep_str;4746 const s = fs.path.sep_str;
4746 const template_paths = [_][]const u8{4747 const template_paths = [_][]const u8{
...@@ -4751,8 +4752,10 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -4751,8 +4752,10 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
4751 };4752 };
4752 var ok_count: usize = 0;4753 var ok_count: usize = 0;
47534754
4755 const fingerprint: Package.Fingerprint = .generate(sanitized_root_name);
4756
4754 for (template_paths) |template_path| {4757 for (template_paths) |template_path| {
4755 if (templates.write(arena, fs.cwd(), cwd_basename, template_path)) |_| {4758 if (templates.write(arena, fs.cwd(), sanitized_root_name, template_path, fingerprint)) |_| {
4756 std.log.info("created {s}", .{template_path});4759 std.log.info("created {s}", .{template_path});
4757 ok_count += 1;4760 ok_count += 1;
4758 } else |err| switch (err) {4761 } else |err| switch (err) {
...@@ -4769,6 +4772,37 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -4769,6 +4772,37 @@ fn cmdInit(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
4769 return cleanExit();4772 return cleanExit();
4770}4773}
47714774
4775fn sanitizeExampleName(arena: Allocator, bytes: []const u8) error{OutOfMemory}![]const u8 {
4776 var result: std.ArrayListUnmanaged(u8) = .empty;
4777 for (bytes, 0..) |byte, i| switch (byte) {
4778 '0'...'9' => {
4779 if (i == 0) try result.append(arena, '_');
4780 try result.append(arena, byte);
4781 },
4782 '_', 'a'...'z', 'A'...'Z' => try result.append(arena, byte),
4783 '-', '.', ' ' => try result.append(arena, '_'),
4784 else => continue,
4785 };
4786 if (result.items.len == 0) return "foo";
4787 if (result.items.len > Package.Manifest.max_name_len)
4788 result.shrinkRetainingCapacity(Package.Manifest.max_name_len);
4789
4790 return result.toOwnedSlice(arena);
4791}
4792
4793test sanitizeExampleName {
4794 var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator);
4795 defer arena_instance.deinit();
4796 const arena = arena_instance.allocator();
4797
4798 try std.testing.expectEqualStrings("foo_bar", try sanitizeExampleName(arena, "foo bar+"));
4799 try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, ""));
4800 try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "!"));
4801 try std.testing.expectEqualStrings("a", try sanitizeExampleName(arena, "!a"));
4802 try std.testing.expectEqualStrings("a_b", try sanitizeExampleName(arena, "a.b!"));
4803 try std.testing.expectEqualStrings("_01234", try sanitizeExampleName(arena, "01234"));
4804}
4805
4772fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {4806fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
4773 dev.check(.build_command);4807 dev.check(.build_command);
47744808
...@@ -5191,13 +5225,15 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -5191,13 +5225,15 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5191 .job_queue = &job_queue,5225 .job_queue = &job_queue,
5192 .omit_missing_hash_error = true,5226 .omit_missing_hash_error = true,
5193 .allow_missing_paths_field = false,5227 .allow_missing_paths_field = false,
5228 .allow_missing_fingerprint = false,
5229 .allow_name_string = false,
5194 .use_latest_commit = false,5230 .use_latest_commit = false,
51955231
5196 .package_root = undefined,5232 .package_root = undefined,
5197 .error_bundle = undefined,5233 .error_bundle = undefined,
5198 .manifest = null,5234 .manifest = null,
5199 .manifest_ast = undefined,5235 .manifest_ast = undefined,
5200 .actual_hash = undefined,5236 .computed_hash = undefined,
5201 .has_build_zig = true,5237 .has_build_zig = true,
5202 .oom_flag = false,5238 .oom_flag = false,
5203 .latest_commit = null,5239 .latest_commit = null,
...@@ -5244,13 +5280,14 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -5244,13 +5280,14 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5244 const hashes = job_queue.table.keys();5280 const hashes = job_queue.table.keys();
5245 const fetches = job_queue.table.values();5281 const fetches = job_queue.table.values();
5246 try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len));5282 try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len));
5247 for (hashes, fetches) |hash, f| {5283 for (hashes, fetches) |*hash, f| {
5248 if (f == &fetch) {5284 if (f == &fetch) {
5249 // The first one is a dummy package for the current project.5285 // The first one is a dummy package for the current project.
5250 continue;5286 continue;
5251 }5287 }
5252 if (!f.has_build_zig)5288 if (!f.has_build_zig)
5253 continue;5289 continue;
5290 const hash_slice = hash.toSlice();
5254 const m = try Package.Module.create(arena, .{5291 const m = try Package.Module.create(arena, .{
5255 .global_cache_directory = global_cache_directory,5292 .global_cache_directory = global_cache_directory,
5256 .paths = .{5293 .paths = .{
...@@ -5260,7 +5297,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -5260,7 +5297,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5260 .fully_qualified_name = try std.fmt.allocPrint(5297 .fully_qualified_name = try std.fmt.allocPrint(
5261 arena,5298 arena,
5262 "root.@dependencies.{s}",5299 "root.@dependencies.{s}",
5263 .{&hash},5300 .{hash_slice},
5264 ),5301 ),
5265 .cc_argv = &.{},5302 .cc_argv = &.{},
5266 .inherited = .{},5303 .inherited = .{},
...@@ -5269,7 +5306,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -5269,7 +5306,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5269 .builtin_mod = builtin_mod,5306 .builtin_mod = builtin_mod,
5270 .builtin_modules = null, // `builtin_mod` is specified5307 .builtin_modules = null, // `builtin_mod` is specified
5271 });5308 });
5272 const hash_cloned = try arena.dupe(u8, &hash);5309 const hash_cloned = try arena.dupe(u8, hash_slice);
5273 deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m);5310 deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m);
5274 f.module = m;5311 f.module = m;
5275 }5312 }
...@@ -5385,23 +5422,22 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -5385,23 +5422,22 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5385 var any_errors = false;5422 var any_errors = false;
5386 while (it.next()) |hash| {5423 while (it.next()) |hash| {
5387 if (hash.len == 0) continue;5424 if (hash.len == 0) continue;
5388 const digest_len = @typeInfo(Package.Manifest.MultiHashHexDigest).array.len;5425 if (hash.len > Package.Hash.max_len) {
5389 if (hash.len != digest_len) {5426 std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{
5390 std.log.err("invalid digest (length {d} instead of {d}): '{s}'", .{5427 hash.len, hash,
5391 hash.len, digest_len, hash,
5392 });5428 });
5393 any_errors = true;5429 any_errors = true;
5394 continue;5430 continue;
5395 }5431 }
5396 try unlazy_set.put(arena, hash[0..digest_len].*, {});5432 try unlazy_set.put(arena, .fromSlice(hash), {});
5397 }5433 }
5398 if (any_errors) process.exit(3);5434 if (any_errors) process.exit(3);
5399 if (system_pkg_dir_path) |p| {5435 if (system_pkg_dir_path) |p| {
5400 // In this mode, the system needs to provide these packages; they5436 // In this mode, the system needs to provide these packages; they
5401 // cannot be fetched by Zig.5437 // cannot be fetched by Zig.
5402 for (unlazy_set.keys()) |hash| {5438 for (unlazy_set.keys()) |*hash| {
5403 std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{5439 std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{
5404 p, hash,5440 p, hash.toSlice(),
5405 });5441 });
5406 }5442 }
5407 std.log.info("remote package fetching disabled due to --system mode", .{});5443 std.log.info("remote package fetching disabled due to --system mode", .{});
...@@ -7091,13 +7127,15 @@ fn cmdFetch(...@@ -7091,13 +7127,15 @@ fn cmdFetch(
7091 .job_queue = &job_queue,7127 .job_queue = &job_queue,
7092 .omit_missing_hash_error = true,7128 .omit_missing_hash_error = true,
7093 .allow_missing_paths_field = false,7129 .allow_missing_paths_field = false,
7130 .allow_missing_fingerprint = true,
7131 .allow_name_string = true,
7094 .use_latest_commit = true,7132 .use_latest_commit = true,
70957133
7096 .package_root = undefined,7134 .package_root = undefined,
7097 .error_bundle = undefined,7135 .error_bundle = undefined,
7098 .manifest = null,7136 .manifest = null,
7099 .manifest_ast = undefined,7137 .manifest_ast = undefined,
7100 .actual_hash = undefined,7138 .computed_hash = undefined,
7101 .has_build_zig = false,7139 .has_build_zig = false,
7102 .oom_flag = false,7140 .oom_flag = false,
7103 .latest_commit = null,7141 .latest_commit = null,
...@@ -7117,14 +7155,15 @@ fn cmdFetch(...@@ -7117,14 +7155,15 @@ fn cmdFetch(
7117 process.exit(1);7155 process.exit(1);
7118 }7156 }
71197157
7120 const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);7158 const package_hash = fetch.computedPackageHash();
7159 const package_hash_slice = package_hash.toSlice();
71217160
7122 root_prog_node.end();7161 root_prog_node.end();
7123 root_prog_node = .{ .index = .none };7162 root_prog_node = .{ .index = .none };
71247163
7125 const name = switch (save) {7164 const name = switch (save) {
7126 .no => {7165 .no => {
7127 try io.getStdOut().writeAll(hex_digest ++ "\n");7166 try io.getStdOut().writer().print("{s}\n", .{package_hash_slice});
7128 return cleanExit();7167 return cleanExit();
7129 },7168 },
7130 .yes, .exact => |name| name: {7169 .yes, .exact => |name| name: {
...@@ -7145,7 +7184,7 @@ fn cmdFetch(...@@ -7145,7 +7184,7 @@ fn cmdFetch(
7145 // The name to use in case the manifest file needs to be created now.7184 // The name to use in case the manifest file needs to be created now.
7146 const init_root_name = fs.path.basename(build_root.directory.path orelse cwd_path);7185 const init_root_name = fs.path.basename(build_root.directory.path orelse cwd_path);
7147 var manifest, var ast = try loadManifest(gpa, arena, .{7186 var manifest, var ast = try loadManifest(gpa, arena, .{
7148 .root_name = init_root_name,7187 .root_name = try sanitizeExampleName(arena, init_root_name),
7149 .dir = build_root.directory.handle,7188 .dir = build_root.directory.handle,
7150 .color = color,7189 .color = color,
7151 });7190 });
...@@ -7194,7 +7233,7 @@ fn cmdFetch(...@@ -7194,7 +7233,7 @@ fn cmdFetch(
7194 \\ }}7233 \\ }}
7195 , .{7234 , .{
7196 std.zig.fmtEscapes(saved_path_or_url),7235 std.zig.fmtEscapes(saved_path_or_url),
7197 std.zig.fmtEscapes(&hex_digest),7236 std.zig.fmtEscapes(package_hash_slice),
7198 });7237 });
71997238
7200 const new_node_text = try std.fmt.allocPrint(arena, ".{p_} = {s},\n", .{7239 const new_node_text = try std.fmt.allocPrint(arena, ".{p_} = {s},\n", .{
...@@ -7213,7 +7252,7 @@ fn cmdFetch(...@@ -7213,7 +7252,7 @@ fn cmdFetch(
7213 if (dep.hash) |h| {7252 if (dep.hash) |h| {
7214 switch (dep.location) {7253 switch (dep.location) {
7215 .url => |u| {7254 .url => |u| {
7216 if (mem.eql(u8, h, &hex_digest) and mem.eql(u8, u, saved_path_or_url)) {7255 if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) {
7217 std.log.info("existing dependency named '{s}' is up-to-date", .{name});7256 std.log.info("existing dependency named '{s}' is up-to-date", .{name});
7218 process.exit(0);7257 process.exit(0);
7219 }7258 }
...@@ -7230,7 +7269,7 @@ fn cmdFetch(...@@ -7230,7 +7269,7 @@ fn cmdFetch(
7230 const hash_replace = try std.fmt.allocPrint(7269 const hash_replace = try std.fmt.allocPrint(
7231 arena,7270 arena,
7232 "\"{}\"",7271 "\"{}\"",
7233 .{std.zig.fmtEscapes(&hex_digest)},7272 .{std.zig.fmtEscapes(package_hash_slice)},
7234 );7273 );
72357274
7236 warn("overwriting existing dependency named '{s}'", .{name});7275 warn("overwriting existing dependency named '{s}'", .{name});
...@@ -7429,10 +7468,10 @@ fn loadManifest(...@@ -7429,10 +7468,10 @@ fn loadManifest(
7429 0,7468 0,
7430 ) catch |err| switch (err) {7469 ) catch |err| switch (err) {
7431 error.FileNotFound => {7470 error.FileNotFound => {
7471 const fingerprint: Package.Fingerprint = .generate(options.root_name);
7432 var templates = findTemplates(gpa, arena);7472 var templates = findTemplates(gpa, arena);
7433 defer templates.deinit();7473 defer templates.deinit();
74347474 templates.write(arena, options.dir, options.root_name, Package.Manifest.basename, fingerprint) catch |e| {
7435 templates.write(arena, options.dir, options.root_name, Package.Manifest.basename) catch |e| {
7436 fatal("unable to write {s}: {s}", .{7475 fatal("unable to write {s}: {s}", .{
7437 Package.Manifest.basename, @errorName(e),7476 Package.Manifest.basename, @errorName(e),
7438 });7477 });
...@@ -7490,6 +7529,7 @@ const Templates = struct {...@@ -7490,6 +7529,7 @@ const Templates = struct {
7490 out_dir: fs.Dir,7529 out_dir: fs.Dir,
7491 root_name: []const u8,7530 root_name: []const u8,
7492 template_path: []const u8,7531 template_path: []const u8,
7532 fingerprint: Package.Fingerprint,
7493 ) !void {7533 ) !void {
7494 if (fs.path.dirname(template_path)) |dirname| {7534 if (fs.path.dirname(template_path)) |dirname| {
7495 out_dir.makePath(dirname) catch |err| {7535 out_dir.makePath(dirname) catch |err| {
...@@ -7503,12 +7543,30 @@ const Templates = struct {...@@ -7503,12 +7543,30 @@ const Templates = struct {
7503 };7543 };
7504 templates.buffer.clearRetainingCapacity();7544 templates.buffer.clearRetainingCapacity();
7505 try templates.buffer.ensureUnusedCapacity(contents.len);7545 try templates.buffer.ensureUnusedCapacity(contents.len);
7506 for (contents) |c| {7546 var i: usize = 0;
7507 if (c == '$') {7547 while (i < contents.len) {
7508 try templates.buffer.appendSlice(root_name);7548 if (contents[i] == '.') {
7509 } else {7549 if (std.mem.startsWith(u8, contents[i..], ".LITNAME")) {
7510 try templates.buffer.append(c);7550 try templates.buffer.append('.');
7551 try templates.buffer.appendSlice(root_name);
7552 i += ".LITNAME".len;
7553 continue;
7554 } else if (std.mem.startsWith(u8, contents[i..], ".NAME")) {
7555 try templates.buffer.appendSlice(root_name);
7556 i += ".NAME".len;
7557 continue;
7558 } else if (std.mem.startsWith(u8, contents[i..], ".FINGERPRINT")) {
7559 try templates.buffer.writer().print("0x{x}", .{fingerprint.int()});
7560 i += ".FINGERPRINT".len;
7561 continue;
7562 } else if (std.mem.startsWith(u8, contents[i..], ".ZIGVER")) {
7563 try templates.buffer.appendSlice(build_options.version);
7564 i += ".ZIGVER".len;
7565 continue;
7566 }
7511 }7567 }
7568 try templates.buffer.append(contents[i]);
7569 i += 1;
7512 }7570 }
75137571
7514 return out_dir.writeFile(.{7572 return out_dir.writeFile(.{