authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-06 15:22:54-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-06 15:25:05-08:00
log355c6260015292642badf0e8b786e676fbb8c961
tree12d722c76642c4f501e027c56a0e51f5b7a1f0a3
parent6f18aca09e8ba3a3e4987cc17f6b92b6b433537d

fetch: delete legacy hash functionality

This also removes a unit test that violates project policy of having binary artifacts as test data when they can be created during the test instead.

3 files changed, 5 insertions(+), 282 deletions(-)

src/Package.zig-66
......@@ -6,10 +6,6 @@ pub const Fetch = @import("Package/Fetch.zig");
66pub const build_zig_basename = "build.zig";
77pub const Manifest = @import("Package/Manifest.zig");
88
9pub const multihash_len = 1 + 1 + Hash.Algo.digest_length;
10pub const multihash_hex_digest_len = 2 * multihash_len;
11pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
12
139pub const Fingerprint = packed struct(u64) {
1410 id: u32,
1511 checksum: u32,
......@@ -77,20 +73,6 @@ pub const Hash = struct {
7773 return std.mem.eql(u8, &a.bytes, &b.bytes);
7874 }
7975
80 /// Distinguishes whether the legacy multihash format is being stored here.
81 pub fn isOld(h: *const Hash) bool {
82 if (h.bytes.len < 2) return false;
83 const their_multihash_func = std.fmt.parseInt(u8, h.bytes[0..2], 16) catch return false;
84 if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) return false;
85 if (h.toSlice().len != multihash_hex_digest_len) return false;
86 return std.mem.indexOfScalar(u8, &h.bytes, '-') == null;
87 }
88
89 test isOld {
90 const h: Hash = .fromSlice("1220138f4aba0c01e66b68ed9e1e1e74614c06e4743d88bc58af4f1c3dd0aae5fea7");
91 try std.testing.expect(h.isOld());
92 }
93
9476 /// Produces "$name-$semver-$hashplus".
9577 /// * name is the name field from build.zig.zon, asserted to be at most 32
9678 /// bytes and assumed be a valid zig identifier
......@@ -197,54 +179,6 @@ pub const ProjectId = struct {
197179 }
198180};
199181
200pub const MultihashFunction = enum(u16) {
201 identity = 0x00,
202 sha1 = 0x11,
203 @"sha2-256" = 0x12,
204 @"sha2-512" = 0x13,
205 @"sha3-512" = 0x14,
206 @"sha3-384" = 0x15,
207 @"sha3-256" = 0x16,
208 @"sha3-224" = 0x17,
209 @"sha2-384" = 0x20,
210 @"sha2-256-trunc254-padded" = 0x1012,
211 @"sha2-224" = 0x1013,
212 @"sha2-512-224" = 0x1014,
213 @"sha2-512-256" = 0x1015,
214 @"blake2b-256" = 0xb220,
215 _,
216};
217
218pub const multihash_function: MultihashFunction = switch (Hash.Algo) {
219 std.crypto.hash.sha2.Sha256 => .@"sha2-256",
220 else => unreachable,
221};
222
223pub fn multiHashHexDigest(digest: Hash.Digest) MultiHashHexDigest {
224 const hex_charset = std.fmt.hex_charset;
225
226 var result: MultiHashHexDigest = undefined;
227
228 result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
229 result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
230
231 result[2] = hex_charset[Hash.Algo.digest_length >> 4];
232 result[3] = hex_charset[Hash.Algo.digest_length & 15];
233
234 for (digest, 0..) |byte, i| {
235 result[4 + i * 2] = hex_charset[byte >> 4];
236 result[5 + i * 2] = hex_charset[byte & 15];
237 }
238 return result;
239}
240
241comptime {
242 // We avoid unnecessary uleb128 code in hexDigest by asserting here the
243 // values are small enough to be contained in the one-byte encoding.
244 assert(@intFromEnum(multihash_function) < 127);
245 assert(Hash.Algo.digest_length < 127);
246}
247
248182test Hash {
249183 const example_digest: Hash.Digest = .{
250184 0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87,
src/Package/Fetch.zig+5-216
......@@ -779,21 +779,11 @@ fn runResource(
779779
780780 if (remote_hash) |declared_hash| {
781781 const hash_tok = f.hash_tok.unwrap().?;
782 if (declared_hash.isOld()) {
783 const actual_hex = Package.multiHashHexDigest(f.computed_hash.digest);
784 if (!std.mem.eql(u8, declared_hash.toSlice(), &actual_hex)) {
785 return f.fail(hash_tok, try eb.printString(
786 "hash mismatch: manifest declares '{s}' but the fetched package has '{s}'",
787 .{ declared_hash.toSlice(), actual_hex },
788 ));
789 }
790 } else {
791 if (!computed_package_hash.eql(&declared_hash)) {
792 return f.fail(hash_tok, try eb.printString(
793 "hash mismatch: manifest declares '{s}' but the fetched package has '{s}'",
794 .{ declared_hash.toSlice(), computed_package_hash.toSlice() },
795 ));
796 }
782 if (!computed_package_hash.eql(&declared_hash)) {
783 return f.fail(hash_tok, try eb.printString(
784 "hash mismatch: manifest declares '{s}' but the fetched package has '{s}'",
785 .{ declared_hash.toSlice(), computed_package_hash.toSlice() },
786 ));
797787 }
798788 } else if (!f.omit_missing_hash_error) {
799789 const notes_len = 1;
......@@ -2239,207 +2229,6 @@ const UnpackResult = struct {
22392229 }
22402230};
22412231
2242test "set executable bit based on file content" {
2243 if (!Io.File.Permissions.has_executable_bit) return error.SkipZigTest;
2244 const gpa = std.testing.allocator;
2245 const io = std.testing.io;
2246
2247 var tmp = std.testing.tmpDir(.{});
2248 defer tmp.cleanup();
2249
2250 const tarball_name = "executables.tar.gz";
2251 try saveEmbedFile(io, tarball_name, tmp.dir);
2252 const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name });
2253 defer gpa.free(tarball_path);
2254
2255 // $ tar -tvf executables.tar.gz
2256 // drwxrwxr-x 0 executables/
2257 // -rwxrwxr-x 170 executables/hello
2258 // lrwxrwxrwx 0 executables/hello_ln -> hello
2259 // -rw-rw-r-- 0 executables/file1
2260 // -rw-rw-r-- 17 executables/script_with_shebang_without_exec_bit
2261 // -rwxrwxr-x 7 executables/script_without_shebang
2262 // -rwxrwxr-x 17 executables/script
2263
2264 var fb: TestFetchBuilder = undefined;
2265 var fetch = try fb.build(gpa, io, tmp.dir, tarball_path);
2266 defer fb.deinit();
2267
2268 try fetch.run();
2269 try std.testing.expectEqualStrings(
2270 "1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3",
2271 &Package.multiHashHexDigest(fetch.computed_hash.digest),
2272 );
2273
2274 var out = try fb.packageDir();
2275 defer out.close(io);
2276 const S = std.posix.S;
2277 // expect executable bit not set
2278 try std.testing.expect((try out.statFile(io, "file1", .{})).permissions.toMode() & S.IXUSR == 0);
2279 try std.testing.expect((try out.statFile(io, "script_without_shebang", .{})).permissions.toMode() & S.IXUSR == 0);
2280 // expect executable bit set
2281 try std.testing.expect((try out.statFile(io, "hello", .{})).permissions.toMode() & S.IXUSR != 0);
2282 try std.testing.expect((try out.statFile(io, "script", .{})).permissions.toMode() & S.IXUSR != 0);
2283 try std.testing.expect((try out.statFile(io, "script_with_shebang_without_exec_bit", .{})).permissions.toMode() & S.IXUSR != 0);
2284 try std.testing.expect((try out.statFile(io, "hello_ln", .{})).permissions.toMode() & S.IXUSR != 0);
2285
2286 //
2287 // $ ls -al zig-cache/tmp/OCz9ovUcstDjTC_U/zig-global-cache/p/1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3
2288 // -rw-rw-r-- 1 0 Apr file1
2289 // -rwxrwxr-x 1 170 Apr hello
2290 // lrwxrwxrwx 1 5 Apr hello_ln -> hello
2291 // -rwxrwxr-x 1 17 Apr script
2292 // -rw-rw-r-- 1 7 Apr script_without_shebang
2293 // -rwxrwxr-x 1 17 Apr script_with_shebang_without_exec_bit
2294}
2295
2296fn saveEmbedFile(io: Io, comptime tarball_name: []const u8, dir: Io.Dir) !void {
2297 //const tarball_name = "duplicate_paths_excluded.tar.gz";
2298 const tarball_content = @embedFile("Fetch/testdata/" ++ tarball_name);
2299 var tmp_file = try dir.createFile(io, tarball_name, .{});
2300 defer tmp_file.close(io);
2301 try tmp_file.writeStreamingAll(io, tarball_content);
2302}
2303
2304// Builds Fetch with required dependencies, clears dependencies on deinit().
2305const TestFetchBuilder = struct {
2306 http_client: std.http.Client,
2307 global_cache_directory: Cache.Directory,
2308 local_cache_path: Cache.Path,
2309 job_queue: Fetch.JobQueue,
2310 fetch: Fetch,
2311
2312 fn build(
2313 self: *TestFetchBuilder,
2314 allocator: std.mem.Allocator,
2315 io: Io,
2316 cache_parent_dir: std.Io.Dir,
2317 path_or_url: []const u8,
2318 ) !*Fetch {
2319 const global_cache_dir = try cache_parent_dir.createDirPathOpen(io, "zig-global-cache", .{});
2320 const package_root_dir = try cache_parent_dir.createDirPathOpen(io, "local-project-root", .{});
2321
2322 self.http_client = .{ .allocator = allocator, .io = io };
2323 self.global_cache_directory = .{ .handle = global_cache_dir, .path = "zig-global-cache" };
2324 self.local_cache_path = .{
2325 .root_dir = .{ .handle = package_root_dir, .path = "local-project-root" },
2326 .sub_path = ".zig-cache",
2327 };
2328
2329 self.job_queue = .{
2330 .io = io,
2331 .http_client = &self.http_client,
2332 .global_cache = self.global_cache_directory,
2333 .local_cache = self.local_cache_path,
2334 .root_pkg_path = .{
2335 .root_dir = .{ .handle = package_root_dir, .path = "local-project-root" },
2336 .sub_path = "zig-pkg",
2337 },
2338 .recursive = false,
2339 .read_only = false,
2340 .debug_hash = false,
2341 .mode = .needed,
2342 .prog_node = std.Progress.Node.none,
2343 };
2344
2345 self.fetch = .{
2346 .arena = std.heap.ArenaAllocator.init(allocator),
2347 .location = .{ .path_or_url = path_or_url },
2348 .location_tok = 0,
2349 .hash_tok = .none,
2350 .name_tok = 0,
2351 .lazy_status = .eager,
2352 .parent_package_root = .{ .root_dir = .{ .handle = package_root_dir, .path = null } },
2353 .parent_manifest_ast = null,
2354 .prog_node = std.Progress.Node.none,
2355 .job_queue = &self.job_queue,
2356 .omit_missing_hash_error = true,
2357 .allow_missing_paths_field = false,
2358 .use_latest_commit = true,
2359
2360 .package_root = undefined,
2361 .error_bundle = undefined,
2362 .manifest = undefined,
2363 .manifest_ast = undefined,
2364 .have_manifest = false,
2365 .computed_hash = undefined,
2366 .has_build_zig = false,
2367 .oom_flag = false,
2368 .latest_commit = null,
2369
2370 .module = null,
2371 };
2372 return &self.fetch;
2373 }
2374
2375 fn deinit(self: *TestFetchBuilder) void {
2376 const io = self.job_queue.io;
2377 self.fetch.deinit();
2378 self.job_queue.deinit();
2379 self.fetch.prog_node.end();
2380 self.global_cache_directory.handle.close(io);
2381 self.http_client.deinit();
2382 }
2383
2384 fn packageDir(self: *TestFetchBuilder) !Io.Dir {
2385 const io = self.job_queue.io;
2386 const root = self.fetch.package_root;
2387 return try root.root_dir.handle.openDir(io, root.sub_path, .{ .iterate = true });
2388 }
2389
2390 // Test helper, asserts thet package dir constains expected_files.
2391 // expected_files must be sorted.
2392 fn expectPackageFiles(self: *TestFetchBuilder, expected_files: []const []const u8) !void {
2393 const io = self.job_queue.io;
2394 const gpa = std.testing.allocator;
2395
2396 var package_dir = try self.packageDir();
2397 defer package_dir.close(io);
2398
2399 var actual_files: std.ArrayList([]u8) = .empty;
2400 defer actual_files.deinit(gpa);
2401 defer for (actual_files.items) |file| gpa.free(file);
2402 var walker = try package_dir.walk(gpa);
2403 defer walker.deinit();
2404 while (try walker.next(io)) |entry| {
2405 if (entry.kind != .file) continue;
2406 const path = try gpa.dupe(u8, entry.path);
2407 errdefer gpa.free(path);
2408 std.mem.replaceScalar(u8, path, std.fs.path.sep, '/');
2409 try actual_files.append(gpa, path);
2410 }
2411 std.mem.sortUnstable([]u8, actual_files.items, {}, struct {
2412 fn lessThan(_: void, a: []u8, b: []u8) bool {
2413 return std.mem.lessThan(u8, a, b);
2414 }
2415 }.lessThan);
2416
2417 try std.testing.expectEqual(expected_files.len, actual_files.items.len);
2418 for (expected_files, 0..) |file_name, i| {
2419 try std.testing.expectEqualStrings(file_name, actual_files.items[i]);
2420 }
2421 try std.testing.expectEqualDeep(expected_files, actual_files.items);
2422 }
2423
2424 // Test helper, asserts that fetch has failed with `msg` error message.
2425 fn expectFetchErrors(self: *TestFetchBuilder, notes_len: usize, msg: []const u8) !void {
2426 const gpa = std.testing.allocator;
2427
2428 var errors = try self.fetch.error_bundle.toOwnedBundle("");
2429 defer errors.deinit(gpa);
2430
2431 const em = errors.getErrorMessage(errors.getMessages()[0]);
2432 try std.testing.expectEqual(1, em.count);
2433 if (notes_len > 0) {
2434 try std.testing.expectEqual(notes_len, em.notes_len);
2435 }
2436 var aw: Io.Writer.Allocating = .init(gpa);
2437 defer aw.deinit();
2438 try errors.renderToWriter(.{}, &aw.writer);
2439 try std.testing.expectEqualStrings(msg, aw.written());
2440 }
2441};
2442
24432232test {
24442233 _ = Filter;
24452234 _ = FileType;
src/Package/Fetch/testdata/executables.tar.gz deleted
Binary files a/src/Package/Fetch/testdata/executables.tar.gz and /dev/null differ