| ... | @@ -134,6 +134,13 @@ pub const Header = struct { | ... | @@ -134,6 +134,13 @@ pub const Header = struct { |
| 134 | } | 134 | } |
| 135 | return header.bytes[start..i]; | 135 | return header.bytes[start..i]; |
| 136 | } | 136 | } |
| | 137 | |
| | 138 | pub fn isZeroBlock(header: Header) bool { |
| | 139 | for (header.bytes) |b| { |
| | 140 | if (b != 0) return false; |
| | 141 | } |
| | 142 | return true; |
| | 143 | } |
| 137 | }; | 144 | }; |
| 138 | | 145 | |
| 139 | fn BufferedReader(comptime ReaderType: type) type { | 146 | fn BufferedReader(comptime ReaderType: type) type { |
| ... | @@ -225,7 +232,7 @@ fn Iterator(comptime ReaderType: type) type { | ... | @@ -225,7 +232,7 @@ fn Iterator(comptime ReaderType: type) type { |
| 225 | const Self = @This(); | 232 | const Self = @This(); |
| 226 | | 233 | |
| 227 | const File = struct { | 234 | const File = struct { |
| 228 | file_name: []const u8, | 235 | name: []const u8, |
| 229 | link_name: []const u8, | 236 | link_name: []const u8, |
| 230 | size: usize, | 237 | size: usize, |
| 231 | file_type: Header.FileType, | 238 | file_type: Header.FileType, |
| ... | @@ -239,6 +246,31 @@ fn Iterator(comptime ReaderType: type) type { | ... | @@ -239,6 +246,31 @@ fn Iterator(comptime ReaderType: type) type { |
| 239 | const rounded_file_size = std.mem.alignForward(usize, self.size, 512); | 246 | const rounded_file_size = std.mem.alignForward(usize, self.size, 512); |
| 240 | try self.iter.reader.skip(rounded_file_size); | 247 | try self.iter.reader.skip(rounded_file_size); |
| 241 | } | 248 | } |
| | 249 | |
| | 250 | fn chksum(self: File) ![16]u8 { |
| | 251 | var cs = [_]u8{0} ** 16; |
| | 252 | if (self.size == 0) return cs; |
| | 253 | |
| | 254 | var buffer: [512]u8 = undefined; |
| | 255 | var h = std.crypto.hash.Md5.init(.{}); |
| | 256 | |
| | 257 | var remaining_bytes: usize = self.size; |
| | 258 | while (remaining_bytes > 0) { |
| | 259 | const copy_size = @min(buffer.len, remaining_bytes); |
| | 260 | try self.iter.reader.copy(&buffer, copy_size); |
| | 261 | h.update(buffer[0..copy_size]); |
| | 262 | remaining_bytes -= copy_size; |
| | 263 | } |
| | 264 | h.final(&cs); |
| | 265 | try self.skipPadding(); |
| | 266 | return cs; |
| | 267 | } |
| | 268 | |
| | 269 | fn skipPadding(self: File) !void { |
| | 270 | const rounded_file_size = std.mem.alignForward(usize, self.size, 512); |
| | 271 | const pad_len: usize = rounded_file_size - self.size; |
| | 272 | self.iter.reader.advance(pad_len); |
| | 273 | } |
| 242 | }; | 274 | }; |
| 243 | | 275 | |
| 244 | pub fn next(self: *Self) !?File { | 276 | pub fn next(self: *Self) !?File { |
| ... | @@ -253,6 +285,7 @@ fn Iterator(comptime ReaderType: type) type { | ... | @@ -253,6 +285,7 @@ fn Iterator(comptime ReaderType: type) type { |
| 253 | self.reader.advance(512); | 285 | self.reader.advance(512); |
| 254 | | 286 | |
| 255 | const header: Header = .{ .bytes = chunk[0..512] }; | 287 | const header: Header = .{ .bytes = chunk[0..512] }; |
| | 288 | if (header.isZeroBlock()) return null; |
| 256 | const file_size = try header.fileSize(); | 289 | const file_size = try header.fileSize(); |
| 257 | const file_type = header.fileType(); | 290 | const file_type = header.fileType(); |
| 258 | const link_name = header.linkName(); | 291 | const link_name = header.linkName(); |
| ... | @@ -266,10 +299,10 @@ fn Iterator(comptime ReaderType: type) type { | ... | @@ -266,10 +299,10 @@ fn Iterator(comptime ReaderType: type) type { |
| 266 | switch (file_type) { | 299 | switch (file_type) { |
| 267 | .directory, .normal, .symbolic_link => { | 300 | .directory, .normal, .symbolic_link => { |
| 268 | return File{ | 301 | return File{ |
| 269 | .file_name = file_name, | 302 | .name = file_name, |
| 270 | .link_name = link_name, | | |
| 271 | .size = file_size, | 303 | .size = file_size, |
| 272 | .file_type = file_type, | 304 | .file_type = file_type, |
| | 305 | .link_name = link_name, |
| 273 | .iter = self, | 306 | .iter = self, |
| 274 | }; | 307 | }; |
| 275 | }, | 308 | }, |
| ... | @@ -341,19 +374,19 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi | ... | @@ -341,19 +374,19 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi |
| 341 | | 374 | |
| 342 | var iter = iterator(reader, options.diagnostics); | 375 | var iter = iterator(reader, options.diagnostics); |
| 343 | | 376 | |
| 344 | while (try iter.next()) |iter_file| { | 377 | while (try iter.next()) |file| { |
| 345 | switch (iter_file.file_type) { | 378 | switch (file.file_type) { |
| 346 | .directory => { | 379 | .directory => { |
| 347 | const file_name = try stripComponents(iter_file.file_name, options.strip_components); | 380 | const file_name = try stripComponents(file.name, options.strip_components); |
| 348 | if (file_name.len != 0 and !options.exclude_empty_directories) { | 381 | if (file_name.len != 0 and !options.exclude_empty_directories) { |
| 349 | try dir.makePath(file_name); | 382 | try dir.makePath(file_name); |
| 350 | } | 383 | } |
| 351 | }, | 384 | }, |
| 352 | .normal => { | 385 | .normal => { |
| 353 | if (iter_file.size == 0 and iter_file.file_name.len == 0) return; | 386 | if (file.size == 0 and file.name.len == 0) return; |
| 354 | const file_name = try stripComponents(iter_file.file_name, options.strip_components); | 387 | const file_name = try stripComponents(file.name, options.strip_components); |
| 355 | | 388 | |
| 356 | const file = dir.createFile(file_name, .{}) catch |err| switch (err) { | 389 | const fs_file = dir.createFile(file_name, .{}) catch |err| switch (err) { |
| 357 | error.FileNotFound => again: { | 390 | error.FileNotFound => again: { |
| 358 | const code = code: { | 391 | const code = code: { |
| 359 | if (std.fs.path.dirname(file_name)) |dir_name| { | 392 | if (std.fs.path.dirname(file_name)) |dir_name| { |
| ... | @@ -373,19 +406,19 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi | ... | @@ -373,19 +406,19 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi |
| 373 | }, | 406 | }, |
| 374 | else => |e| return e, | 407 | else => |e| return e, |
| 375 | }; | 408 | }; |
| 376 | defer if (file) |f| f.close(); | 409 | defer if (fs_file) |f| f.close(); |
| 377 | | 410 | |
| 378 | if (file) |f| { | 411 | if (fs_file) |f| { |
| 379 | try iter_file.write(f); | 412 | try file.write(f); |
| 380 | } else { | 413 | } else { |
| 381 | try iter_file.skip(); | 414 | try file.skip(); |
| 382 | } | 415 | } |
| 383 | }, | 416 | }, |
| 384 | .symbolic_link => { | 417 | .symbolic_link => { |
| 385 | // The file system path of the symbolic link. | 418 | // The file system path of the symbolic link. |
| 386 | const file_name = try stripComponents(iter_file.file_name, options.strip_components); | 419 | const file_name = try stripComponents(file.name, options.strip_components); |
| 387 | // The data inside the symbolic link. | 420 | // The data inside the symbolic link. |
| 388 | const link_name = iter_file.link_name; | 421 | const link_name = file.link_name; |
| 389 | | 422 | |
| 390 | dir.symLink(link_name, file_name, .{}) catch |err| again: { | 423 | dir.symLink(link_name, file_name, .{}) catch |err| again: { |
| 391 | const code = code: { | 424 | const code = code: { |
| ... | @@ -473,3 +506,274 @@ test parsePaxAttribute { | ... | @@ -473,3 +506,274 @@ test parsePaxAttribute { |
| 473 | | 506 | |
| 474 | const std = @import("std.zig"); | 507 | const std = @import("std.zig"); |
| 475 | const assert = std.debug.assert; | 508 | const assert = std.debug.assert; |
| | 509 | |
| | 510 | const TestCase = struct { |
| | 511 | const File = struct { |
| | 512 | const empty_string = &[0]u8{}; |
| | 513 | |
| | 514 | name: []const u8, |
| | 515 | size: usize = 0, |
| | 516 | link_name: []const u8 = empty_string, |
| | 517 | file_type: Header.FileType = .normal, |
| | 518 | }; |
| | 519 | |
| | 520 | path: []const u8, |
| | 521 | files: []const File = &[_]TestCase.File{}, |
| | 522 | chksums: []const []const u8 = &[_][]const u8{}, |
| | 523 | err: ?anyerror = null, |
| | 524 | }; |
| | 525 | |
| | 526 | test "Go test cases" { |
| | 527 | const test_dir = try std.fs.openDirAbsolute("/usr/local/go/src/archive/tar/testdata", .{}); |
| | 528 | const cases = [_]TestCase{ |
| | 529 | .{ |
| | 530 | .path = "gnu.tar", |
| | 531 | .files = &[_]TestCase.File{ |
| | 532 | .{ |
| | 533 | .name = "small.txt", |
| | 534 | .size = 5, |
| | 535 | .file_type = .normal, |
| | 536 | }, |
| | 537 | .{ |
| | 538 | .name = "small2.txt", |
| | 539 | .size = 11, |
| | 540 | .file_type = .normal, |
| | 541 | }, |
| | 542 | }, |
| | 543 | .chksums = &[_][]const u8{ |
| | 544 | "e38b27eaccb4391bdec553a7f3ae6b2f", |
| | 545 | "c65bd2e50a56a2138bf1716f2fd56fe9", |
| | 546 | }, |
| | 547 | }, |
| | 548 | .{ |
| | 549 | .path = "sparse-formats.tar", |
| | 550 | .err = error.TarUnsupportedFileType, |
| | 551 | }, |
| | 552 | .{ |
| | 553 | .path = "star.tar", |
| | 554 | .files = &[_]TestCase.File{ |
| | 555 | .{ |
| | 556 | .name = "small.txt", |
| | 557 | .size = 5, |
| | 558 | .file_type = .normal, |
| | 559 | }, |
| | 560 | .{ |
| | 561 | .name = "small2.txt", |
| | 562 | .size = 11, |
| | 563 | .file_type = .normal, |
| | 564 | }, |
| | 565 | }, |
| | 566 | .chksums = &[_][]const u8{ |
| | 567 | "e38b27eaccb4391bdec553a7f3ae6b2f", |
| | 568 | "c65bd2e50a56a2138bf1716f2fd56fe9", |
| | 569 | }, |
| | 570 | }, |
| | 571 | .{ |
| | 572 | .path = "v7.tar", |
| | 573 | .files = &[_]TestCase.File{ |
| | 574 | .{ |
| | 575 | .name = "small.txt", |
| | 576 | .size = 5, |
| | 577 | .file_type = .normal, |
| | 578 | }, |
| | 579 | .{ |
| | 580 | .name = "small2.txt", |
| | 581 | .size = 11, |
| | 582 | .file_type = .normal, |
| | 583 | }, |
| | 584 | }, |
| | 585 | .chksums = &[_][]const u8{ |
| | 586 | "e38b27eaccb4391bdec553a7f3ae6b2f", |
| | 587 | "c65bd2e50a56a2138bf1716f2fd56fe9", |
| | 588 | }, |
| | 589 | }, |
| | 590 | .{ |
| | 591 | .path = "pax.tar", |
| | 592 | .files = &[_]TestCase.File{ |
| | 593 | .{ |
| | 594 | .name = "a/123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100", |
| | 595 | .size = 7, |
| | 596 | .file_type = .normal, |
| | 597 | }, |
| | 598 | .{ |
| | 599 | .name = "a/b", |
| | 600 | .size = 0, |
| | 601 | .file_type = .symbolic_link, |
| | 602 | .link_name = "1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545", |
| | 603 | // TODO fix reading link name from pax header |
| | 604 | // .link_name = "123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100", |
| | 605 | }, |
| | 606 | }, |
| | 607 | .chksums = &[_][]const u8{ |
| | 608 | "3c382e8f5b6631aa2db52643912ffd4a", |
| | 609 | }, |
| | 610 | }, |
| | 611 | // TODO: this should fail |
| | 612 | // .{ |
| | 613 | // .path = "pax-bad-hdr-file.tar", |
| | 614 | // .err = error.TarBadHeader, |
| | 615 | // }, |
| | 616 | // .{ |
| | 617 | // .path = "pax-bad-mtime-file.tar", |
| | 618 | // .err = error.TarBadHeader, |
| | 619 | // }, |
| | 620 | // |
| | 621 | // TODO: giving wrong result because we are not reading pax size header |
| | 622 | // .{ |
| | 623 | // .path = "pax-pos-size-file.tar", |
| | 624 | // .files = &[_]TestCase.File{ |
| | 625 | // .{ |
| | 626 | // .name = "foo", |
| | 627 | // .size = 999, |
| | 628 | // .file_type = .normal, |
| | 629 | // }, |
| | 630 | // }, |
| | 631 | // .chksums = &[_][]const u8{ |
| | 632 | // "0afb597b283fe61b5d4879669a350556", |
| | 633 | // }, |
| | 634 | // }, |
| | 635 | .{ |
| | 636 | // has pax records which we are not interested in |
| | 637 | .path = "pax-records.tar", |
| | 638 | .files = &[_]TestCase.File{ |
| | 639 | .{ |
| | 640 | .name = "file", |
| | 641 | }, |
| | 642 | }, |
| | 643 | }, |
| | 644 | .{ |
| | 645 | // has global records which we are ignoring |
| | 646 | .path = "pax-global-records.tar", |
| | 647 | .files = &[_]TestCase.File{ |
| | 648 | .{ |
| | 649 | .name = "file1", |
| | 650 | }, |
| | 651 | .{ |
| | 652 | .name = "file2", |
| | 653 | }, |
| | 654 | .{ |
| | 655 | .name = "file3", |
| | 656 | }, |
| | 657 | .{ |
| | 658 | .name = "file4", |
| | 659 | }, |
| | 660 | }, |
| | 661 | }, |
| | 662 | .{ |
| | 663 | .path = "nil-uid.tar", |
| | 664 | .files = &[_]TestCase.File{ |
| | 665 | .{ |
| | 666 | .name = "P1050238.JPG.log", |
| | 667 | .size = 14, |
| | 668 | .file_type = .normal, |
| | 669 | }, |
| | 670 | }, |
| | 671 | .chksums = &[_][]const u8{ |
| | 672 | "08d504674115e77a67244beac19668f5", |
| | 673 | }, |
| | 674 | }, |
| | 675 | .{ |
| | 676 | // has xattrs and pax records which we are ignoring |
| | 677 | .path = "xattrs.tar", |
| | 678 | .files = &[_]TestCase.File{ |
| | 679 | .{ |
| | 680 | .name = "small.txt", |
| | 681 | .size = 5, |
| | 682 | .file_type = .normal, |
| | 683 | }, |
| | 684 | .{ |
| | 685 | .name = "small2.txt", |
| | 686 | .size = 11, |
| | 687 | .file_type = .normal, |
| | 688 | }, |
| | 689 | }, |
| | 690 | .chksums = &[_][]const u8{ |
| | 691 | "e38b27eaccb4391bdec553a7f3ae6b2f", |
| | 692 | "c65bd2e50a56a2138bf1716f2fd56fe9", |
| | 693 | }, |
| | 694 | }, |
| | 695 | .{ |
| | 696 | .path = "gnu-multi-hdrs.tar", |
| | 697 | .err = error.TarUnsupportedFileType, |
| | 698 | }, |
| | 699 | .{ |
| | 700 | .path = "gnu-incremental.tar", |
| | 701 | .err = error.TarUnsupportedFileType, |
| | 702 | }, |
| | 703 | // .{ |
| | 704 | // .path = "pax-multi-hdrs.tar", |
| | 705 | // }, |
| | 706 | // .{ |
| | 707 | // .path = "gnu-long-nul.tar", |
| | 708 | // .files = &[_]TestCase.File{ |
| | 709 | // .{ |
| | 710 | // .name = "012233456789", |
| | 711 | // }, |
| | 712 | // }, |
| | 713 | // }, |
| | 714 | // .{ |
| | 715 | // .path = "gnu-utf8.tar", |
| | 716 | // .files = &[_]TestCase.File{ |
| | 717 | // .{ |
| | 718 | // .name = "012233456789", |
| | 719 | // }, |
| | 720 | // }, |
| | 721 | // }, |
| | 722 | // |
| | 723 | .{ |
| | 724 | .path = "gnu-not-utf8.tar", |
| | 725 | .files = &[_]TestCase.File{ |
| | 726 | .{ |
| | 727 | .name = "hi\x80\x81\x82\x83bye", |
| | 728 | }, |
| | 729 | }, |
| | 730 | }, |
| | 731 | // TODO some files with errors: |
| | 732 | // pax-nul-xattrs.tar, pax-nul-path.tar, neg-size.tar, issue10968.tar, issue11169.tar, issue12435.tar |
| | 733 | .{ |
| | 734 | .path = "trailing-slash.tar", |
| | 735 | .files = &[_]TestCase.File{ |
| | 736 | .{ |
| | 737 | .name = "123456789/" ** 30, |
| | 738 | .file_type = .directory, |
| | 739 | }, |
| | 740 | }, |
| | 741 | }, |
| | 742 | }; |
| | 743 | |
| | 744 | for (cases) |case| { |
| | 745 | // if (!std.mem.eql(u8, case.path, "pax.tar")) continue; |
| | 746 | |
| | 747 | var fs_file = try test_dir.openFile(case.path, .{}); |
| | 748 | defer fs_file.close(); |
| | 749 | |
| | 750 | var iter = iterator(fs_file.reader(), null); |
| | 751 | var i: usize = 0; |
| | 752 | while (iter.next() catch |err| { |
| | 753 | if (case.err) |e| { |
| | 754 | try std.testing.expectEqual(e, err); |
| | 755 | continue; |
| | 756 | } else { |
| | 757 | return err; |
| | 758 | } |
| | 759 | }) |actual| { |
| | 760 | const expected = case.files[i]; |
| | 761 | try std.testing.expectEqualStrings(expected.name, actual.name); |
| | 762 | try std.testing.expectEqual(expected.size, actual.size); |
| | 763 | try std.testing.expectEqual(expected.file_type, actual.file_type); |
| | 764 | try std.testing.expectEqualStrings(expected.link_name, actual.link_name); |
| | 765 | |
| | 766 | if (case.chksums.len > i) { |
| | 767 | var actual_chksum = try actual.chksum(); |
| | 768 | var hex_to_bytes_buffer: [16]u8 = undefined; |
| | 769 | const expected_chksum = try std.fmt.hexToBytes(&hex_to_bytes_buffer, case.chksums[i]); |
| | 770 | // std.debug.print("actual chksum: {s}\n", .{std.fmt.fmtSliceHexLower(&actual_chksum)}); |
| | 771 | try std.testing.expectEqualStrings(expected_chksum, &actual_chksum); |
| | 772 | } else { |
| | 773 | try actual.skip(); // skip file content |
| | 774 | } |
| | 775 | i += 1; |
| | 776 | } |
| | 777 | try std.testing.expectEqual(case.files.len, i); |
| | 778 | } |
| | 779 | } |
Igor Anić <igor.anic@gmail.com> 2023-11-27 21:37:30+01:00
Andrew Kelley <andrew@ziglang.org> 2024-01-13 19:37:33-07:00
lib