authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-02-02 14:03:41-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-02-02 14:03:41-05:00
log7505d19e931893c01cf78760fa31d819ef10e879
tree289bfd00f0097e98eab19ad29fc571109d766786
parent6b7ad22981867cbf6ec40d540f5571f276d6801b
parent24ff8a1a5fc00c405e5506251f11d23653d6c8b5
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #14511 from ziglang/zig-build-hashes

two package hash breaking enhancements

3 files changed, 115 insertions(+), 11 deletions(-)

lib/std/fs/file.zig+2-1
......@@ -179,7 +179,7 @@ pub const File = struct {
179179 lock_nonblocking: bool = false,
180180
181181 /// For POSIX systems this is the file system mode the file will
182 /// be created with.
182 /// be created with. On other systems this is always 0.
183183 mode: Mode = default_mode,
184184
185185 /// Setting this to `.blocking` prevents `O.NONBLOCK` from being passed even
......@@ -307,6 +307,7 @@ pub const File = struct {
307307 /// is unique to each filesystem.
308308 inode: INode,
309309 size: u64,
310 /// This is available on POSIX systems and is always 0 otherwise.
310311 mode: Mode,
311312 kind: Kind,
312313
lib/std/tar.zig+23
......@@ -1,6 +1,18 @@
11pub const Options = struct {
22 /// Number of directory levels to skip when extracting files.
33 strip_components: u32 = 0,
4 /// How to handle the "mode" property of files from within the tar file.
5 mode_mode: ModeMode = .executable_bit_only,
6
7 const ModeMode = enum {
8 /// The mode from the tar file is completely ignored. Files are created
9 /// with the default mode when creating files.
10 ignore,
11 /// The mode from the tar file is inspected for the owner executable bit
12 /// only. This bit is copied to the group and other executable bits.
13 /// Other bits of the mode are left as the default when creating files.
14 executable_bit_only,
15 };
416};
517
618pub const Header = struct {
......@@ -72,6 +84,17 @@ pub const Header = struct {
7284};
7385
7486pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !void {
87 switch (options.mode_mode) {
88 .ignore => {},
89 .executable_bit_only => {
90 // This code does not look at the mode bits yet. To implement this feature,
91 // the implementation must be adjusted to look at the mode, and check the
92 // user executable bit, then call fchmod on newly created files when
93 // the executable bit is supposed to be set.
94 // It also needs to properly deal with ACLs on Windows.
95 @panic("TODO: unimplemented: tar ModeMode.executable_bit_only");
96 },
97 }
7598 var file_name_buffer: [255]u8 = undefined;
7699 var buffer: [512 * 8]u8 = undefined;
77100 var start: usize = 0;
src/Package.zig+90-10
......@@ -1,5 +1,6 @@
11const Package = @This();
22
3const builtin = @import("builtin");
34const std = @import("std");
45const fs = std.fs;
56const mem = std.mem;
......@@ -298,16 +299,37 @@ fn fetchAndUnpack(
298299 // Check if the expected_hash is already present in the global package
299300 // cache, and thereby avoid both fetching and unpacking.
300301 if (expected_hash) |h| cached: {
301 if (h.len != 2 * Hash.digest_length) {
302 const hex_multihash_len = 2 * multihash_len;
303 if (h.len >= 2) {
304 const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| {
305 return reportError(
306 ini,
307 comp_directory,
308 h.ptr,
309 "invalid multihash value: unable to parse hash function: {s}",
310 .{@errorName(err)},
311 );
312 };
313 if (@intToEnum(MultihashFunction, their_multihash_func) != multihash_function) {
314 return reportError(
315 ini,
316 comp_directory,
317 h.ptr,
318 "unsupported hash function: only sha2-256 is supported",
319 .{},
320 );
321 }
322 }
323 if (h.len != hex_multihash_len) {
302324 return reportError(
303325 ini,
304326 comp_directory,
305327 h.ptr,
306328 "wrong hash size. expected: {d}, found: {d}",
307 .{ Hash.digest_length, h.len },
329 .{ hex_multihash_len, h.len },
308330 );
309331 }
310 const hex_digest = h[0 .. 2 * Hash.digest_length];
332 const hex_digest = h[0..hex_multihash_len];
311333 const pkg_dir_sub_path = "p" ++ s ++ hex_digest;
312334 var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) {
313335 error.FileNotFound => break :cached,
......@@ -396,8 +418,8 @@ fn fetchAndUnpack(
396418 const pkg_dir_sub_path = "p" ++ s ++ hexDigest(actual_hash);
397419 try renameTmpIntoCache(global_cache_directory.handle, tmp_dir_sub_path, pkg_dir_sub_path);
398420
421 const actual_hex = hexDigest(actual_hash);
399422 if (expected_hash) |h| {
400 const actual_hex = hexDigest(actual_hash);
401423 if (!mem.eql(u8, h, &actual_hex)) {
402424 return reportError(
403425 ini,
......@@ -413,7 +435,7 @@ fn fetchAndUnpack(
413435 comp_directory,
414436 url.ptr,
415437 "url field is missing corresponding hash field: hash={s}",
416 .{std.fmt.fmtSliceHexLower(&actual_hash)},
438 .{&actual_hex},
417439 );
418440 }
419441
......@@ -440,6 +462,12 @@ fn unpackTarball(
440462
441463 try std.tar.pipeToFileSystem(out_dir, decompress.reader(), .{
442464 .strip_components = 1,
465 // TODO: we would like to set this to executable_bit_only, but two
466 // things need to happen before that:
467 // 1. the tar implementation needs to support it
468 // 2. the hashing algorithm here needs to support detecting the is_executable
469 // bit on Windows from the ACLs (see the isExecutable function).
470 .mode_mode = .ignore,
443471 });
444472}
445473
......@@ -468,7 +496,7 @@ const HashedFile = struct {
468496 hash: [Hash.digest_length]u8,
469497 failure: Error!void,
470498
471 const Error = fs.File.OpenError || fs.File.ReadError;
499 const Error = fs.File.OpenError || fs.File.ReadError || fs.File.StatError;
472500
473501 fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool {
474502 _ = context;
......@@ -544,6 +572,8 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
544572 var buf: [8000]u8 = undefined;
545573 var file = try dir.openFile(hashed_file.path, .{});
546574 var hasher = Hash.init(.{});
575 hasher.update(hashed_file.path);
576 hasher.update(&.{ 0, @boolToInt(try isExecutable(file)) });
547577 while (true) {
548578 const bytes_read = try file.read(&buf);
549579 if (bytes_read == 0) break;
......@@ -552,6 +582,19 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
552582 hasher.final(&hashed_file.hash);
553583}
554584
585fn isExecutable(file: fs.File) !bool {
586 if (builtin.os.tag == .windows) {
587 // TODO check the ACL on Windows.
588 // Until this is implemented, this could be a false negative on
589 // Windows, which is why we do not yet set executable_bit_only above
590 // when unpacking the tarball.
591 return false;
592 } else {
593 const stat = try file.stat();
594 return (stat.mode & std.os.S.IXUSR) != 0;
595 }
596}
597
555598const hex_charset = "0123456789abcdef";
556599
557600fn hex64(x: u64) [16]u8 {
......@@ -570,11 +613,30 @@ test hex64 {
570613 try std.testing.expectEqualStrings("[00efcdab78563412]", s);
571614}
572615
573fn hexDigest(digest: [Hash.digest_length]u8) [Hash.digest_length * 2]u8 {
574 var result: [Hash.digest_length * 2]u8 = undefined;
616const multihash_function: MultihashFunction = switch (Hash) {
617 std.crypto.hash.sha2.Sha256 => .@"sha2-256",
618 else => @compileError("unreachable"),
619};
620comptime {
621 // We avoid unnecessary uleb128 code in hexDigest by asserting here the
622 // values are small enough to be contained in the one-byte encoding.
623 assert(@enumToInt(multihash_function) < 127);
624 assert(Hash.digest_length < 127);
625}
626const multihash_len = 1 + 1 + Hash.digest_length;
627
628fn hexDigest(digest: [Hash.digest_length]u8) [multihash_len * 2]u8 {
629 var result: [multihash_len * 2]u8 = undefined;
630
631 result[0] = hex_charset[@enumToInt(multihash_function) >> 4];
632 result[1] = hex_charset[@enumToInt(multihash_function) & 15];
633
634 result[2] = hex_charset[Hash.digest_length >> 4];
635 result[3] = hex_charset[Hash.digest_length & 15];
636
575637 for (digest) |byte, i| {
576 result[i * 2 + 0] = hex_charset[byte >> 4];
577 result[i * 2 + 1] = hex_charset[byte & 15];
638 result[4 + i * 2] = hex_charset[byte >> 4];
639 result[5 + i * 2] = hex_charset[byte & 15];
578640 }
579641 return result;
580642}
......@@ -607,3 +669,21 @@ fn renameTmpIntoCache(
607669 break;
608670 }
609671}
672
673const MultihashFunction = enum(u16) {
674 identity = 0x00,
675 sha1 = 0x11,
676 @"sha2-256" = 0x12,
677 @"sha2-512" = 0x13,
678 @"sha3-512" = 0x14,
679 @"sha3-384" = 0x15,
680 @"sha3-256" = 0x16,
681 @"sha3-224" = 0x17,
682 @"sha2-384" = 0x20,
683 @"sha2-256-trunc254-padded" = 0x1012,
684 @"sha2-224" = 0x1013,
685 @"sha2-512-224" = 0x1014,
686 @"sha2-512-256" = 0x1015,
687 @"blake2b-256" = 0xb220,
688 _,
689};