authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-02-02 14:03:41-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-02-02 14:03:41-05:00
log7505d19e931893c01cf78760fa31d819ef10e879
tree289bfd00f0097e98eab19ad29fc571109d766786
parent6b7ad22981867cbf6ec40d540f5571f276d6801b
parent24ff8a1a5fc00c405e5506251f11d23653d6c8b5
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #14511 from ziglang/zig-build-hashes

two package hash breaking enhancements

3 files changed, 115 insertions(+), 11 deletions(-)

lib/std/fs/file.zig+2-1
...@@ -179,7 +179,7 @@ pub const File = struct {...@@ -179,7 +179,7 @@ pub const File = struct {
179 lock_nonblocking: bool = false,179 lock_nonblocking: bool = false,
180180
181 /// For POSIX systems this is the file system mode the file will181 /// For POSIX systems this is the file system mode the file will
182 /// be created with.182 /// be created with. On other systems this is always 0.
183 mode: Mode = default_mode,183 mode: Mode = default_mode,
184184
185 /// Setting this to `.blocking` prevents `O.NONBLOCK` from being passed even185 /// Setting this to `.blocking` prevents `O.NONBLOCK` from being passed even
...@@ -307,6 +307,7 @@ pub const File = struct {...@@ -307,6 +307,7 @@ pub const File = struct {
307 /// is unique to each filesystem.307 /// is unique to each filesystem.
308 inode: INode,308 inode: INode,
309 size: u64,309 size: u64,
310 /// This is available on POSIX systems and is always 0 otherwise.
310 mode: Mode,311 mode: Mode,
311 kind: Kind,312 kind: Kind,
312313
lib/std/tar.zig+23
...@@ -1,6 +1,18 @@...@@ -1,6 +1,18 @@
1pub const Options = struct {1pub const Options = struct {
2 /// Number of directory levels to skip when extracting files.2 /// Number of directory levels to skip when extracting files.
3 strip_components: u32 = 0,3 strip_components: u32 = 0,
4 /// How to handle the "mode" property of files from within the tar file.
5 mode_mode: ModeMode = .executable_bit_only,
6
7 const ModeMode = enum {
8 /// The mode from the tar file is completely ignored. Files are created
9 /// with the default mode when creating files.
10 ignore,
11 /// The mode from the tar file is inspected for the owner executable bit
12 /// only. This bit is copied to the group and other executable bits.
13 /// Other bits of the mode are left as the default when creating files.
14 executable_bit_only,
15 };
4};16};
517
6pub const Header = struct {18pub const Header = struct {
...@@ -72,6 +84,17 @@ pub const Header = struct {...@@ -72,6 +84,17 @@ pub const Header = struct {
72};84};
7385
74pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !void {86pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !void {
87 switch (options.mode_mode) {
88 .ignore => {},
89 .executable_bit_only => {
90 // This code does not look at the mode bits yet. To implement this feature,
91 // the implementation must be adjusted to look at the mode, and check the
92 // user executable bit, then call fchmod on newly created files when
93 // the executable bit is supposed to be set.
94 // It also needs to properly deal with ACLs on Windows.
95 @panic("TODO: unimplemented: tar ModeMode.executable_bit_only");
96 },
97 }
75 var file_name_buffer: [255]u8 = undefined;98 var file_name_buffer: [255]u8 = undefined;
76 var buffer: [512 * 8]u8 = undefined;99 var buffer: [512 * 8]u8 = undefined;
77 var start: usize = 0;100 var start: usize = 0;
src/Package.zig+90-10
...@@ -1,5 +1,6 @@...@@ -1,5 +1,6 @@
1const Package = @This();1const Package = @This();
22
3const builtin = @import("builtin");
3const std = @import("std");4const std = @import("std");
4const fs = std.fs;5const fs = std.fs;
5const mem = std.mem;6const mem = std.mem;
...@@ -298,16 +299,37 @@ fn fetchAndUnpack(...@@ -298,16 +299,37 @@ fn fetchAndUnpack(
298 // Check if the expected_hash is already present in the global package299 // Check if the expected_hash is already present in the global package
299 // cache, and thereby avoid both fetching and unpacking.300 // cache, and thereby avoid both fetching and unpacking.
300 if (expected_hash) |h| cached: {301 if (expected_hash) |h| cached: {
301 if (h.len != 2 * Hash.digest_length) {302 const hex_multihash_len = 2 * multihash_len;
303 if (h.len >= 2) {
304 const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| {
305 return reportError(
306 ini,
307 comp_directory,
308 h.ptr,
309 "invalid multihash value: unable to parse hash function: {s}",
310 .{@errorName(err)},
311 );
312 };
313 if (@intToEnum(MultihashFunction, their_multihash_func) != multihash_function) {
314 return reportError(
315 ini,
316 comp_directory,
317 h.ptr,
318 "unsupported hash function: only sha2-256 is supported",
319 .{},
320 );
321 }
322 }
323 if (h.len != hex_multihash_len) {
302 return reportError(324 return reportError(
303 ini,325 ini,
304 comp_directory,326 comp_directory,
305 h.ptr,327 h.ptr,
306 "wrong hash size. expected: {d}, found: {d}",328 "wrong hash size. expected: {d}, found: {d}",
307 .{ Hash.digest_length, h.len },329 .{ hex_multihash_len, h.len },
308 );330 );
309 }331 }
310 const hex_digest = h[0 .. 2 * Hash.digest_length];332 const hex_digest = h[0..hex_multihash_len];
311 const pkg_dir_sub_path = "p" ++ s ++ hex_digest;333 const pkg_dir_sub_path = "p" ++ s ++ hex_digest;
312 var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) {334 var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) {
313 error.FileNotFound => break :cached,335 error.FileNotFound => break :cached,
...@@ -396,8 +418,8 @@ fn fetchAndUnpack(...@@ -396,8 +418,8 @@ fn fetchAndUnpack(
396 const pkg_dir_sub_path = "p" ++ s ++ hexDigest(actual_hash);418 const pkg_dir_sub_path = "p" ++ s ++ hexDigest(actual_hash);
397 try renameTmpIntoCache(global_cache_directory.handle, tmp_dir_sub_path, pkg_dir_sub_path);419 try renameTmpIntoCache(global_cache_directory.handle, tmp_dir_sub_path, pkg_dir_sub_path);
398420
421 const actual_hex = hexDigest(actual_hash);
399 if (expected_hash) |h| {422 if (expected_hash) |h| {
400 const actual_hex = hexDigest(actual_hash);
401 if (!mem.eql(u8, h, &actual_hex)) {423 if (!mem.eql(u8, h, &actual_hex)) {
402 return reportError(424 return reportError(
403 ini,425 ini,
...@@ -413,7 +435,7 @@ fn fetchAndUnpack(...@@ -413,7 +435,7 @@ fn fetchAndUnpack(
413 comp_directory,435 comp_directory,
414 url.ptr,436 url.ptr,
415 "url field is missing corresponding hash field: hash={s}",437 "url field is missing corresponding hash field: hash={s}",
416 .{std.fmt.fmtSliceHexLower(&actual_hash)},438 .{&actual_hex},
417 );439 );
418 }440 }
419441
...@@ -440,6 +462,12 @@ fn unpackTarball(...@@ -440,6 +462,12 @@ fn unpackTarball(
440462
441 try std.tar.pipeToFileSystem(out_dir, decompress.reader(), .{463 try std.tar.pipeToFileSystem(out_dir, decompress.reader(), .{
442 .strip_components = 1,464 .strip_components = 1,
465 // TODO: we would like to set this to executable_bit_only, but two
466 // things need to happen before that:
467 // 1. the tar implementation needs to support it
468 // 2. the hashing algorithm here needs to support detecting the is_executable
469 // bit on Windows from the ACLs (see the isExecutable function).
470 .mode_mode = .ignore,
443 });471 });
444}472}
445473
...@@ -468,7 +496,7 @@ const HashedFile = struct {...@@ -468,7 +496,7 @@ const HashedFile = struct {
468 hash: [Hash.digest_length]u8,496 hash: [Hash.digest_length]u8,
469 failure: Error!void,497 failure: Error!void,
470498
471 const Error = fs.File.OpenError || fs.File.ReadError;499 const Error = fs.File.OpenError || fs.File.ReadError || fs.File.StatError;
472500
473 fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool {501 fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool {
474 _ = context;502 _ = context;
...@@ -544,6 +572,8 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void...@@ -544,6 +572,8 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
544 var buf: [8000]u8 = undefined;572 var buf: [8000]u8 = undefined;
545 var file = try dir.openFile(hashed_file.path, .{});573 var file = try dir.openFile(hashed_file.path, .{});
546 var hasher = Hash.init(.{});574 var hasher = Hash.init(.{});
575 hasher.update(hashed_file.path);
576 hasher.update(&.{ 0, @boolToInt(try isExecutable(file)) });
547 while (true) {577 while (true) {
548 const bytes_read = try file.read(&buf);578 const bytes_read = try file.read(&buf);
549 if (bytes_read == 0) break;579 if (bytes_read == 0) break;
...@@ -552,6 +582,19 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void...@@ -552,6 +582,19 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
552 hasher.final(&hashed_file.hash);582 hasher.final(&hashed_file.hash);
553}583}
554584
585fn isExecutable(file: fs.File) !bool {
586 if (builtin.os.tag == .windows) {
587 // TODO check the ACL on Windows.
588 // Until this is implemented, this could be a false negative on
589 // Windows, which is why we do not yet set executable_bit_only above
590 // when unpacking the tarball.
591 return false;
592 } else {
593 const stat = try file.stat();
594 return (stat.mode & std.os.S.IXUSR) != 0;
595 }
596}
597
555const hex_charset = "0123456789abcdef";598const hex_charset = "0123456789abcdef";
556599
557fn hex64(x: u64) [16]u8 {600fn hex64(x: u64) [16]u8 {
...@@ -570,11 +613,30 @@ test hex64 {...@@ -570,11 +613,30 @@ test hex64 {
570 try std.testing.expectEqualStrings("[00efcdab78563412]", s);613 try std.testing.expectEqualStrings("[00efcdab78563412]", s);
571}614}
572615
573fn hexDigest(digest: [Hash.digest_length]u8) [Hash.digest_length * 2]u8 {616const multihash_function: MultihashFunction = switch (Hash) {
574 var result: [Hash.digest_length * 2]u8 = undefined;617 std.crypto.hash.sha2.Sha256 => .@"sha2-256",
618 else => @compileError("unreachable"),
619};
620comptime {
621 // We avoid unnecessary uleb128 code in hexDigest by asserting here the
622 // values are small enough to be contained in the one-byte encoding.
623 assert(@enumToInt(multihash_function) < 127);
624 assert(Hash.digest_length < 127);
625}
626const multihash_len = 1 + 1 + Hash.digest_length;
627
628fn hexDigest(digest: [Hash.digest_length]u8) [multihash_len * 2]u8 {
629 var result: [multihash_len * 2]u8 = undefined;
630
631 result[0] = hex_charset[@enumToInt(multihash_function) >> 4];
632 result[1] = hex_charset[@enumToInt(multihash_function) & 15];
633
634 result[2] = hex_charset[Hash.digest_length >> 4];
635 result[3] = hex_charset[Hash.digest_length & 15];
636
575 for (digest) |byte, i| {637 for (digest) |byte, i| {
576 result[i * 2 + 0] = hex_charset[byte >> 4];638 result[4 + i * 2] = hex_charset[byte >> 4];
577 result[i * 2 + 1] = hex_charset[byte & 15];639 result[5 + i * 2] = hex_charset[byte & 15];
578 }640 }
579 return result;641 return result;
580}642}
...@@ -607,3 +669,21 @@ fn renameTmpIntoCache(...@@ -607,3 +669,21 @@ fn renameTmpIntoCache(
607 break;669 break;
608 }670 }
609}671}
672
673const MultihashFunction = enum(u16) {
674 identity = 0x00,
675 sha1 = 0x11,
676 @"sha2-256" = 0x12,
677 @"sha2-512" = 0x13,
678 @"sha3-512" = 0x14,
679 @"sha3-384" = 0x15,
680 @"sha3-256" = 0x16,
681 @"sha3-224" = 0x17,
682 @"sha2-384" = 0x20,
683 @"sha2-256-trunc254-padded" = 0x1012,
684 @"sha2-224" = 0x1013,
685 @"sha2-512-224" = 0x1014,
686 @"sha2-512-256" = 0x1015,
687 @"blake2b-256" = 0xb220,
688 _,
689};