authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-06-03 13:44:56-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-06-03 13:44:56-07:00
log25cf0b59d86cfba7bafec7d9aeae31e843a5417e
tree0272098b8ac61279105151e8f24ee4499a189b97
parent7fc3fb955abf3d510079ad5e42027b90e897dc20

enough code to get some examples working

2 files changed, 102 insertions(+), 7 deletions(-)

src/Package/Fetch.zig+22-7
...@@ -447,7 +447,7 @@ fn runResource(...@@ -447,7 +447,7 @@ fn runResource(
447 const rand_int = std.crypto.random.int(u64);447 const rand_int = std.crypto.random.int(u64);
448 const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);448 const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);
449449
450 const package_sub_path = blk: {450 const package_sub_path, const size = blk: {
451 const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path});451 const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path});
452 var tmp_directory: Cache.Directory = .{452 var tmp_directory: Cache.Directory = .{
453 .path = tmp_directory_path,453 .path = tmp_directory_path,
...@@ -499,12 +499,14 @@ fn runResource(...@@ -499,12 +499,14 @@ fn runResource(
499 // Empty directories have already been omitted by `unpackResource`.499 // Empty directories have already been omitted by `unpackResource`.
500 // Compute the package hash based on the remaining files in the temporary500 // Compute the package hash based on the remaining files in the temporary
501 // directory.501 // directory.
502 f.actual_hash = try computeHash(f, pkg_path, filter);502 f.actual_hash, const size = try computeHash(f, pkg_path, filter);
503503
504 break :blk if (unpack_result.root_dir.len > 0)504 const pkg_sub_path = if (unpack_result.root_dir.len > 0)
505 try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir })505 try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir })
506 else506 else
507 tmp_dir_sub_path;507 tmp_dir_sub_path;
508
509 break :blk .{ pkg_sub_path, size };
508 };510 };
509511
510 // Rename the temporary directory into the global zig package cache512 // Rename the temporary directory into the global zig package cache
...@@ -513,6 +515,14 @@ fn runResource(...@@ -513,6 +515,14 @@ fn runResource(
513 // by the system. This is done even if the hash is invalid, in case the515 // by the system. This is done even if the hash is invalid, in case the
514 // package with the different hash is used in the future.516 // package with the different hash is used in the future.
515517
518 const name = f.manifest.?.name;
519 const semver = f.manifest.?.version;
520 const new_hash = Package.Manifest.NewHashDecoded.init(name, semver, @intCast(size), f.actual_hash);
521 const encoded = new_hash.encode();
522 std.debug.print("new hash: {s}\n", .{encoded.toSlice()});
523
524 if (true) std.process.exit(0);
525
516 f.package_root = .{526 f.package_root = .{
517 .root_dir = cache_root,527 .root_dir = cache_root,
518 .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)),528 .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)),
...@@ -1420,7 +1430,7 @@ fn computeHash(...@@ -1420,7 +1430,7 @@ fn computeHash(
1420 f: *Fetch,1430 f: *Fetch,
1421 pkg_path: Cache.Path,1431 pkg_path: Cache.Path,
1422 filter: Filter,1432 filter: Filter,
1423) RunError!Manifest.Digest {1433) RunError!struct { Manifest.Digest, u64 } {
1424 // All the path name strings need to be in memory for sorting.1434 // All the path name strings need to be in memory for sorting.
1425 const arena = f.arena.allocator();1435 const arena = f.arena.allocator();
1426 const gpa = f.arena.child_allocator;1436 const gpa = f.arena.child_allocator;
...@@ -1499,6 +1509,7 @@ fn computeHash(...@@ -1499,6 +1509,7 @@ fn computeHash(
1499 .normalized_path = try normalizePathAlloc(arena, entry_pkg_path),1509 .normalized_path = try normalizePathAlloc(arena, entry_pkg_path),
1500 .kind = kind,1510 .kind = kind,
1501 .hash = undefined, // to be populated by the worker1511 .hash = undefined, // to be populated by the worker
1512 .size = undefined, // to be populated by the worker
1502 .failure = undefined, // to be populated by the worker1513 .failure = undefined, // to be populated by the worker
1503 };1514 };
1504 thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file });1515 thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file });
...@@ -1540,6 +1551,7 @@ fn computeHash(...@@ -1540,6 +1551,7 @@ fn computeHash(
15401551
1541 var hasher = Manifest.Hash.init(.{});1552 var hasher = Manifest.Hash.init(.{});
1542 var any_failures = false;1553 var any_failures = false;
1554 var total_size: u64 = 0;
1543 for (all_files.items) |hashed_file| {1555 for (all_files.items) |hashed_file| {
1544 hashed_file.failure catch |err| {1556 hashed_file.failure catch |err| {
1545 any_failures = true;1557 any_failures = true;
...@@ -1550,6 +1562,7 @@ fn computeHash(...@@ -1550,6 +1562,7 @@ fn computeHash(
1550 });1562 });
1551 };1563 };
1552 hasher.update(&hashed_file.hash);1564 hasher.update(&hashed_file.hash);
1565 total_size += hashed_file.size;
1553 }1566 }
1554 for (deleted_files.items) |deleted_file| {1567 for (deleted_files.items) |deleted_file| {
1555 deleted_file.failure catch |err| {1568 deleted_file.failure catch |err| {
...@@ -1574,7 +1587,7 @@ fn computeHash(...@@ -1574,7 +1587,7 @@ fn computeHash(
1574 };1587 };
1575 }1588 }
15761589
1577 return hasher.finalResult();1590 return .{ hasher.finalResult(), total_size };
1578}1591}
15791592
1580fn dumpHashInfo(all_files: []const *const HashedFile) !void {1593fn dumpHashInfo(all_files: []const *const HashedFile) !void {
...@@ -1604,20 +1617,20 @@ fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void {...@@ -1604,20 +1617,20 @@ fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void {
1604fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {1617fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
1605 var buf: [8000]u8 = undefined;1618 var buf: [8000]u8 = undefined;
1606 var hasher = Manifest.Hash.init(.{});1619 var hasher = Manifest.Hash.init(.{});
1620 var size: u64 = 0;
1607 hasher.update(hashed_file.normalized_path);1621 hasher.update(hashed_file.normalized_path);
16081622
1609 switch (hashed_file.kind) {1623 switch (hashed_file.kind) {
1610 .file => {1624 .file => {
1611 var file = try dir.openFile(hashed_file.fs_path, .{});1625 var file = try dir.openFile(hashed_file.fs_path, .{});
1612 defer file.close();1626 defer file.close();
1613 // Hard-coded false executable bit: https://github.com/ziglang/zig/issues/17463
1614 hasher.update(&.{ 0, 0 });
1615 var file_header: FileHeader = .{};1627 var file_header: FileHeader = .{};
1616 while (true) {1628 while (true) {
1617 const bytes_read = try file.read(&buf);1629 const bytes_read = try file.read(&buf);
1618 if (bytes_read == 0) break;1630 if (bytes_read == 0) break;
1619 hasher.update(buf[0..bytes_read]);1631 hasher.update(buf[0..bytes_read]);
1620 file_header.update(buf[0..bytes_read]);1632 file_header.update(buf[0..bytes_read]);
1633 size += bytes_read;
1621 }1634 }
1622 if (file_header.isExecutable()) {1635 if (file_header.isExecutable()) {
1623 try setExecutable(file);1636 try setExecutable(file);
...@@ -1635,6 +1648,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void...@@ -1635,6 +1648,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
1635 },1648 },
1636 }1649 }
1637 hasher.final(&hashed_file.hash);1650 hasher.final(&hashed_file.hash);
1651 hashed_file.size = size;
1638}1652}
16391653
1640fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {1654fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {
...@@ -1662,6 +1676,7 @@ const HashedFile = struct {...@@ -1662,6 +1676,7 @@ const HashedFile = struct {
1662 fs_path: []const u8,1676 fs_path: []const u8,
1663 normalized_path: []const u8,1677 normalized_path: []const u8,
1664 hash: Manifest.Digest,1678 hash: Manifest.Digest,
1679 size: u64,
1665 failure: Error!void,1680 failure: Error!void,
1666 kind: Kind,1681 kind: Kind,
16671682
src/Package/Manifest.zig+80
...@@ -6,6 +6,84 @@ pub const multihash_len = 1 + 1 + Hash.digest_length;...@@ -6,6 +6,84 @@ pub const multihash_len = 1 + 1 + Hash.digest_length;
6pub const multihash_hex_digest_len = 2 * multihash_len;6pub const multihash_hex_digest_len = 2 * multihash_len;
7pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;7pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
88
9pub const NewHashDecoded = extern struct {
10 name: [16]u8,
11 semver: [16]u8,
12 size: u32 align(1),
13 hash: [5]u8,
14
15 pub fn init(name: []const u8, semver: std.SemanticVersion, size: u32, hash: Digest) NewHashDecoded {
16 var result: NewHashDecoded = undefined;
17
18 const name_len = @min(name.len, result.name.len);
19 @memcpy(result.name[0..name_len], name[0..name_len]);
20 @memset(result.name[name_len..], 0);
21
22 if (std.fmt.bufPrint(&result.semver, "{}", .{semver})) |slice| {
23 @memset(result.semver[slice.len..], 0);
24 } else |err| switch (err) {
25 error.NoSpaceLeft => {
26 result.semver[result.semver.len - 1] = '-';
27 },
28 }
29
30 result.size = size;
31
32 @memcpy(&result.hash, hash[0..result.hash.len]);
33
34 return result;
35 }
36
37 pub fn getName(h: *const NewHashDecoded) []const u8 {
38 const len = mem.indexOfScalar(u8, &h.name, 0) orelse h.name.len;
39 return h.name[0..len];
40 }
41
42 pub fn getSemVer(h: *const NewHashDecoded) []const u8 {
43 const len = mem.indexOfScalar(u8, &h.semver, 0) orelse h.semver.len;
44 return h.semver[0..len];
45 }
46
47 pub fn encode(h: NewHashDecoded) NewHashEncoded {
48 var result: NewHashEncoded = .{ .bytes = undefined };
49 var i: usize = 0;
50
51 const name = h.getName();
52 @memcpy(result.bytes[i..][0..name.len], name);
53 i += name.len;
54
55 result.bytes[i] = '-';
56 i += 1;
57
58 const semver = h.getSemVer();
59 @memcpy(result.bytes[i..][0..semver.len], semver);
60 i += semver.len;
61
62 result.bytes[i] = '-';
63 i += 1;
64
65 const hash_len = 5; // TODO `h.hash.len`
66 var decoded_bytes: [hash_len + @sizeOf(u32)]u8 = undefined;
67 std.mem.writeInt(u32, decoded_bytes[0..@sizeOf(u32)], h.size, .little);
68 @memcpy(decoded_bytes[@sizeOf(u32)..], &h.hash);
69
70 i += (std.fs.base64_encoder.encode(result.bytes[i..], &decoded_bytes)).len;
71
72 @memset(result.bytes[i..], 0);
73 return result;
74 }
75};
76
77pub const new_hash_encoded_max_len = std.fs.base64_encoder.calcSize(@sizeOf(NewHashDecoded));
78pub const NewHashEncoded = extern struct {
79 bytes: [new_hash_encoded_max_len]u8,
80
81 pub fn toSlice(h: *const NewHashEncoded) []const u8 {
82 const len = mem.indexOfScalar(u8, &h.bytes, 0) orelse h.bytes.len;
83 return h.bytes[0..len];
84 }
85};
86
9pub const Dependency = struct {87pub const Dependency = struct {
10 location: Location,88 location: Location,
11 location_tok: Ast.TokenIndex,89 location_tok: Ast.TokenIndex,
...@@ -591,12 +669,14 @@ const Parse = struct {...@@ -591,12 +669,14 @@ const Parse = struct {
591};669};
592670
593const Manifest = @This();671const Manifest = @This();
672const builtin = @import("builtin");
594const std = @import("std");673const std = @import("std");
595const mem = std.mem;674const mem = std.mem;
596const Allocator = std.mem.Allocator;675const Allocator = std.mem.Allocator;
597const assert = std.debug.assert;676const assert = std.debug.assert;
598const Ast = std.zig.Ast;677const Ast = std.zig.Ast;
599const testing = std.testing;678const testing = std.testing;
679const native_endian = builtin.cpu.arch.endian();
600680
601test "basic" {681test "basic" {
602 const gpa = testing.allocator;682 const gpa = testing.allocator;