authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2023-12-11 22:00:49+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-01-13 19:37:33-07:00
logc07527abac7a5f56bb9111b42fcbcbf468b4917f
tree03b38b5a13125514b89e57485f9e9755ec0284c7
parentc76abe0e183ef513b9ee651b052c7f99c33c139c

tar: reorganize file, functions before tests

1 files changed, 326 insertions(+), 325 deletions(-)

lib/std/tar.zig+326-325
......@@ -15,8 +15,7 @@
1515/// GNU tar reference: https://www.gnu.org/software/tar/manual/html_node/Standard.html
1616/// pax reference: https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13
1717///
18//const std = @import("std.zig");
19const std = @import("std");
18const std = @import("std.zig");
2019const assert = std.debug.assert;
2120
2221pub const Options = struct {
......@@ -226,6 +225,276 @@ fn nullStr(str: []const u8) []const u8 {
226225 return str;
227226}
228227
228pub fn tarReader(reader: anytype, diagnostics: ?*Options.Diagnostics) TarReader(@TypeOf(reader)) {
229 return .{
230 .reader = reader,
231 .diagnostics = diagnostics,
232 };
233}
234
235fn TarReader(comptime ReaderType: type) type {
236 return struct {
237 reader: ReaderType,
238 diagnostics: ?*Options.Diagnostics,
239
240 // buffers for heeader and file attributes
241 header_buffer: [Header.SIZE]u8 = undefined,
242 file_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined,
243 link_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined,
244
245 // bytes of padding to the end of the block
246 padding: usize = 0,
247 // current tar file
248 file: File = undefined,
249
250 pub const File = struct {
251 name: []const u8, // name of file, symlink or directory
252 link_name: []const u8, // target name of symlink
253 size: usize, // size of the file in bytes
254 mode: u32,
255 kind: Header.Kind,
256
257 reader: ReaderType,
258
259 // Writes file content to writer.
260 pub fn write(self: File, writer: anytype) !void {
261 var buffer: [4096]u8 = undefined;
262
263 var n: usize = 0;
264 while (n < self.size) {
265 const buf = buffer[0..@min(buffer.len, self.size - n)];
266 try self.reader.readNoEof(buf);
267 try writer.writeAll(buf);
268 n += buf.len;
269 }
270 }
271
272 // Skips file content. Advances reader.
273 pub fn skip(self: File) !void {
274 try self.reader.skipBytes(self.size, .{});
275 }
276 };
277
278 const Self = @This();
279
280 fn readHeader(self: *Self) !?Header {
281 if (self.padding > 0) {
282 try self.reader.skipBytes(self.padding, .{});
283 }
284 const n = try self.reader.readAll(&self.header_buffer);
285 if (n == 0) return null;
286 if (n < Header.SIZE) return error.UnexpectedEndOfStream;
287 const header = Header{ .bytes = self.header_buffer[0..Header.SIZE] };
288 if (try header.checkChksum() == 0) return null;
289 return header;
290 }
291
292 inline fn readString(self: *Self, size: usize, buffer: []u8) ![]const u8 {
293 assert(buffer.len >= size);
294 const buf = buffer[0..size];
295 try self.reader.readNoEof(buf);
296 return nullStr(buf);
297 }
298
299 inline fn initFile(self: *Self) void {
300 self.file = File{
301 .name = self.file_name_buffer[0..0],
302 .link_name = self.link_name_buffer[0..0],
303 .size = 0,
304 .kind = .normal,
305 .mode = 0,
306 .reader = self.reader,
307 };
308 }
309
310 // Number of padding bytes in the last file block.
311 inline fn blockPadding(size: usize) usize {
312 const block_rounded = std.mem.alignForward(usize, size, Header.SIZE); // size rounded to te block boundary
313 return block_rounded - size;
314 }
315
316 /// Iterates through the tar archive as if it is a series of files.
317 /// Internally, the tar format often uses entries (header with optional
318 /// content) to add meta data that describes the next file. These
319 /// entries should not normally be visible to the outside. As such, this
320 /// loop iterates through one or more entries until it collects a all
321 /// file attributes.
322 pub fn next(self: *Self) !?File {
323 self.initFile();
324
325 while (try self.readHeader()) |header| {
326 const kind = header.kind();
327 const size: usize = @intCast(try header.size());
328 self.padding = blockPadding(size);
329
330 switch (kind) {
331 // File types to retrun upstream
332 .directory, .normal, .symbolic_link => {
333 self.file.kind = kind;
334 self.file.mode = try header.mode();
335
336 // set file attributes if not already set by prefix/extended headers
337 if (self.file.size == 0) {
338 self.file.size = size;
339 }
340 if (self.file.link_name.len == 0) {
341 self.file.link_name = header.linkName(self.link_name_buffer[0..Header.LINK_NAME_SIZE]);
342 }
343 if (self.file.name.len == 0) {
344 self.file.name = try header.fullName(self.file_name_buffer[0..Header.MAX_NAME_SIZE]);
345 }
346
347 self.padding = blockPadding(self.file.size);
348 return self.file;
349 },
350 // Prefix header types
351 .gnu_long_name => {
352 self.file.name = try self.readString(size, &self.file_name_buffer);
353 },
354 .gnu_long_link => {
355 self.file.link_name = try self.readString(size, &self.link_name_buffer);
356 },
357 .extended_header => {
358 // Use just attributes from last extended header.
359 self.initFile();
360
361 var rdr = paxReader(self.reader, size);
362 while (try rdr.next()) |attr| {
363 switch (attr.kind) {
364 .path => {
365 self.file.name = try attr.value(&self.file_name_buffer);
366 },
367 .linkpath => {
368 self.file.link_name = try attr.value(&self.link_name_buffer);
369 },
370 .size => {
371 var buf: [64]u8 = undefined;
372 self.file.size = try std.fmt.parseInt(usize, try attr.value(&buf), 10);
373 },
374 }
375 }
376 },
377 // Ignored header type
378 .global_extended_header => {
379 self.reader.skipBytes(size, .{}) catch return error.TarHeadersTooBig;
380 },
381 // All other are unsupported header types
382 else => {
383 const d = self.diagnostics orelse return error.TarUnsupportedHeader;
384 try d.errors.append(d.allocator, .{ .unsupported_file_type = .{
385 .file_name = try d.allocator.dupe(u8, header.name()),
386 .file_type = kind,
387 } });
388 },
389 }
390 }
391 return null;
392 }
393 };
394}
395
396// Pax attributes reader.
397// Size is length of pax extended header in reader.
398fn paxReader(reader: anytype, size: usize) PaxReader(@TypeOf(reader)) {
399 return PaxReader(@TypeOf(reader)){
400 .reader = reader,
401 .size = size,
402 };
403}
404
405const PaxAttributeKind = enum {
406 path,
407 linkpath,
408 size,
409};
410
411fn PaxReader(comptime ReaderType: type) type {
412 return struct {
413 size: usize, // cumulative size of all pax attributes
414 reader: ReaderType,
415 // scratch buffer used for reading attribute length and keyword
416 scratch: [128]u8 = undefined,
417
418 const Self = @This();
419
420 const Attribute = struct {
421 kind: PaxAttributeKind,
422 len: usize, // length of the attribute value
423 reader: ReaderType, // reader positioned at value start
424
425 // Copies pax attribute value into destination buffer.
426 // Must be called with destination buffer of size at least Attribute.len.
427 pub fn value(self: Attribute, dst: []u8) ![]const u8 {
428 assert(self.len <= dst.len);
429 const buf = dst[0..self.len];
430 const n = try self.reader.readAll(buf);
431 if (n < self.len) return error.UnexpectedEndOfStream;
432 try validateAttributeEnding(self.reader);
433 if (hasNull(buf)) return error.PaxNullInValue;
434 return buf;
435 }
436 };
437
438 // Iterates over pax attributes. Returns known only known attributes.
439 // Caller has to call value in Attribute, to advance reader across value.
440 pub fn next(self: *Self) !?Attribute {
441 // Pax extended header consists of one or more attributes, each constructed as follows:
442 // "%d %s=%s\n", <length>, <keyword>, <value>
443 while (self.size > 0) {
444 const length_buf = try self.readUntil(' ');
445 const length = try std.fmt.parseInt(usize, length_buf, 10); // record length in bytes
446
447 const keyword = try self.readUntil('=');
448 if (hasNull(keyword)) return error.PaxNullInKeyword;
449
450 // calculate value_len
451 const value_start = length_buf.len + keyword.len + 2; // 2 separators
452 if (length < value_start + 1 or self.size < length) return error.UnexpectedEndOfStream;
453 const value_len = length - value_start - 1; // \n separator at end
454 self.size -= length;
455
456 const kind: PaxAttributeKind = if (eql(keyword, "path"))
457 .path
458 else if (eql(keyword, "linkpath"))
459 .linkpath
460 else if (eql(keyword, "size"))
461 .size
462 else {
463 try self.reader.skipBytes(value_len, .{});
464 try validateAttributeEnding(self.reader);
465 continue;
466 };
467 return Attribute{
468 .kind = kind,
469 .len = value_len,
470 .reader = self.reader,
471 };
472 }
473
474 return null;
475 }
476
477 inline fn readUntil(self: *Self, delimiter: u8) ![]const u8 {
478 var fbs = std.io.fixedBufferStream(&self.scratch);
479 try self.reader.streamUntilDelimiter(fbs.writer(), delimiter, null);
480 return fbs.getWritten();
481 }
482
483 inline fn eql(a: []const u8, b: []const u8) bool {
484 return std.mem.eql(u8, a, b);
485 }
486
487 inline fn hasNull(str: []const u8) bool {
488 return (std.mem.indexOfScalar(u8, str, 0)) != null;
489 }
490
491 // Checks that each record ends with new line.
492 inline fn validateAttributeEnding(reader: ReaderType) !void {
493 if (try reader.readByte() != '\n') return error.PaxInvalidAttributeEnd;
494 }
495 };
496}
497
229498pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !void {
230499 switch (options.mode_mode) {
231500 .ignore => {},
......@@ -639,170 +908,70 @@ test "tar run Go test cases" {
639908 .{
640909 // Size in gnu extended format, and name in pax attribute.
641910 .path = "writer-big-long.tar",
642 .files = &[_]Case.File{
643 .{
644 .name = "longname/" ** 15 ++ "16gig.txt",
645 .size = 16 * 1024 * 1024 * 1024,
646 .mode = 0o644,
647 .truncated = true,
648 },
649 },
650 },
651 };
652
653 for (cases) |case| {
654 var fs_file = try test_dir.openFile(case.path, .{});
655 defer fs_file.close();
656
657 //var iter = iterator(fs_file.reader(), null);
658 var iter = tarReader(fs_file.reader(), null);
659 var i: usize = 0;
660 while (iter.next() catch |err| {
661 if (case.err) |e| {
662 try std.testing.expectEqual(e, err);
663 continue;
664 } else {
665 return err;
666 }
667 }) |actual| : (i += 1) {
668 const expected = case.files[i];
669 try std.testing.expectEqualStrings(expected.name, actual.name);
670 try std.testing.expectEqual(expected.size, actual.size);
671 try std.testing.expectEqual(expected.kind, actual.kind);
672 try std.testing.expectEqual(expected.mode, actual.mode);
673 try std.testing.expectEqualStrings(expected.link_name, actual.link_name);
674
675 if (case.chksums.len > i) {
676 var md5writer = Md5Writer{};
677 try actual.write(&md5writer);
678 const chksum = md5writer.chksum();
679 try std.testing.expectEqualStrings(case.chksums[i], &chksum);
680 } else {
681 if (!expected.truncated) try actual.skip(); // skip file content
682 }
683 }
684 try std.testing.expectEqual(case.files.len, i);
685 }
686}
687
688// used in test to calculate file chksum
689const Md5Writer = struct {
690 h: std.crypto.hash.Md5 = std.crypto.hash.Md5.init(.{}),
691
692 pub fn writeAll(self: *Md5Writer, buf: []const u8) !void {
693 self.h.update(buf);
694 }
695
696 pub fn writeByte(self: *Md5Writer, byte: u8) !void {
697 self.h.update(&[_]u8{byte});
698 }
699
700 pub fn chksum(self: *Md5Writer) [32]u8 {
701 var s = [_]u8{0} ** 16;
702 self.h.final(&s);
703 return std.fmt.bytesToHex(s, .lower);
704 }
705};
706
707fn paxReader(reader: anytype, size: usize) PaxReader(@TypeOf(reader)) {
708 return PaxReader(@TypeOf(reader)){
709 .reader = reader,
710 .size = size,
711 };
712}
713
714const PaxAttributeKind = enum {
715 path,
716 linkpath,
717 size,
718};
719
720fn PaxReader(comptime ReaderType: type) type {
721 return struct {
722 size: usize, // cumulative size of all pax attributes
723 reader: ReaderType,
724 // scratch buffer used for reading attribute length and keyword
725 scratch: [128]u8 = undefined,
726
727 const Self = @This();
728
729 const Attribute = struct {
730 kind: PaxAttributeKind,
731 len: usize, // length of the attribute value
732 reader: ReaderType, // reader positioned at value start
733
734 // Copies pax attribute value into destination buffer.
735 // Must be called with destination buffer of size at least Attribute.len.
736 pub fn value(self: Attribute, dst: []u8) ![]const u8 {
737 assert(self.len <= dst.len);
738 const buf = dst[0..self.len];
739 const n = try self.reader.readAll(buf);
740 if (n < self.len) return error.UnexpectedEndOfStream;
741 try validateAttributeEnding(self.reader);
742 if (hasNull(buf)) return error.PaxNullInValue;
743 return buf;
744 }
745 };
746
747 // Iterates over pax attributes. Returns known only known attributes.
748 // Caller has to call value in Attribute, to advance reader across value.
749 pub fn next(self: *Self) !?Attribute {
750 // Pax extended header consists of one or more attributes, each constructed as follows:
751 // "%d %s=%s\n", <length>, <keyword>, <value>
752 while (self.size > 0) {
753 const length_buf = try self.readUntil(' ');
754 const length = try std.fmt.parseInt(usize, length_buf, 10); // record length in bytes
755
756 const keyword = try self.readUntil('=');
757 if (hasNull(keyword)) return error.PaxNullInKeyword;
911 .files = &[_]Case.File{
912 .{
913 .name = "longname/" ** 15 ++ "16gig.txt",
914 .size = 16 * 1024 * 1024 * 1024,
915 .mode = 0o644,
916 .truncated = true,
917 },
918 },
919 },
920 };
758921
759 // calculate value_len
760 const value_start = length_buf.len + keyword.len + 2; // 2 separators
761 if (length < value_start + 1 or self.size < length) return error.UnexpectedEndOfStream;
762 const value_len = length - value_start - 1; // \n separator at end
763 self.size -= length;
922 for (cases) |case| {
923 var fs_file = try test_dir.openFile(case.path, .{});
924 defer fs_file.close();
764925
765 const kind: PaxAttributeKind = if (eql(keyword, "path"))
766 .path
767 else if (eql(keyword, "linkpath"))
768 .linkpath
769 else if (eql(keyword, "size"))
770 .size
771 else {
772 try self.reader.skipBytes(value_len, .{});
773 try validateAttributeEnding(self.reader);
774 continue;
775 };
776 return Attribute{
777 .kind = kind,
778 .len = value_len,
779 .reader = self.reader,
780 };
926 //var iter = iterator(fs_file.reader(), null);
927 var iter = tarReader(fs_file.reader(), null);
928 var i: usize = 0;
929 while (iter.next() catch |err| {
930 if (case.err) |e| {
931 try std.testing.expectEqual(e, err);
932 continue;
933 } else {
934 return err;
781935 }
936 }) |actual| : (i += 1) {
937 const expected = case.files[i];
938 try std.testing.expectEqualStrings(expected.name, actual.name);
939 try std.testing.expectEqual(expected.size, actual.size);
940 try std.testing.expectEqual(expected.kind, actual.kind);
941 try std.testing.expectEqual(expected.mode, actual.mode);
942 try std.testing.expectEqualStrings(expected.link_name, actual.link_name);
782943
783 return null;
944 if (case.chksums.len > i) {
945 var md5writer = Md5Writer{};
946 try actual.write(&md5writer);
947 const chksum = md5writer.chksum();
948 try std.testing.expectEqualStrings(case.chksums[i], &chksum);
949 } else {
950 if (!expected.truncated) try actual.skip(); // skip file content
951 }
784952 }
953 try std.testing.expectEqual(case.files.len, i);
954 }
955}
785956
786 inline fn readUntil(self: *Self, delimiter: u8) ![]const u8 {
787 var fbs = std.io.fixedBufferStream(&self.scratch);
788 try self.reader.streamUntilDelimiter(fbs.writer(), delimiter, null);
789 return fbs.getWritten();
790 }
957// used in test to calculate file chksum
958const Md5Writer = struct {
959 h: std.crypto.hash.Md5 = std.crypto.hash.Md5.init(.{}),
791960
792 inline fn eql(a: []const u8, b: []const u8) bool {
793 return std.mem.eql(u8, a, b);
794 }
961 pub fn writeAll(self: *Md5Writer, buf: []const u8) !void {
962 self.h.update(buf);
963 }
795964
796 inline fn hasNull(str: []const u8) bool {
797 return (std.mem.indexOfScalar(u8, str, 0)) != null;
798 }
965 pub fn writeByte(self: *Md5Writer, byte: u8) !void {
966 self.h.update(&[_]u8{byte});
967 }
799968
800 // Checks that each record ends with new line.
801 inline fn validateAttributeEnding(reader: ReaderType) !void {
802 if (try reader.readByte() != '\n') return error.PaxInvalidAttributeEnd;
803 }
804 };
805}
969 pub fn chksum(self: *Md5Writer) [32]u8 {
970 var s = [_]u8{0} ** 16;
971 self.h.final(&s);
972 return std.fmt.bytesToHex(s, .lower);
973 }
974};
806975
807976test "tar PaxReader" {
808977 const Attr = struct {
......@@ -927,171 +1096,3 @@ test "tar PaxReader" {
9271096 try std.testing.expect(case.err == null);
9281097 }
9291098}
930
931pub fn tarReader(reader: anytype, diagnostics: ?*Options.Diagnostics) TarReader(@TypeOf(reader)) {
932 return .{
933 .reader = reader,
934 .diagnostics = diagnostics,
935 };
936}
937
938fn TarReader(comptime ReaderType: type) type {
939 return struct {
940 reader: ReaderType,
941 diagnostics: ?*Options.Diagnostics,
942
943 // buffers for heeader and file attributes
944 header_buffer: [Header.SIZE]u8 = undefined,
945 file_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined,
946 link_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined,
947
948 // bytes of padding to the end of the block
949 padding: usize = 0,
950 // current tar file
951 file: File = undefined,
952
953 pub const File = struct {
954 name: []const u8, // name of file, symlink or directory
955 link_name: []const u8, // target name of symlink
956 size: usize, // size of the file in bytes
957 mode: u32,
958 kind: Header.Kind,
959
960 reader: ReaderType,
961
962 // Writes file content to writer.
963 pub fn write(self: File, writer: anytype) !void {
964 var buffer: [4096]u8 = undefined;
965
966 var n: usize = 0;
967 while (n < self.size) {
968 const buf = buffer[0..@min(buffer.len, self.size - n)];
969 try self.reader.readNoEof(buf);
970 try writer.writeAll(buf);
971 n += buf.len;
972 }
973 }
974
975 // Skips file content. Advances reader.
976 pub fn skip(self: File) !void {
977 try self.reader.skipBytes(self.size, .{});
978 }
979 };
980
981 const Self = @This();
982
983 fn readHeader(self: *Self) !?Header {
984 if (self.padding > 0) {
985 try self.reader.skipBytes(self.padding, .{});
986 }
987 const n = try self.reader.readAll(&self.header_buffer);
988 if (n == 0) return null;
989 if (n < Header.SIZE) return error.UnexpectedEndOfStream;
990 const header = Header{ .bytes = self.header_buffer[0..Header.SIZE] };
991 if (try header.checkChksum() == 0) return null;
992 return header;
993 }
994
995 inline fn readString(self: *Self, size: usize, buffer: []u8) ![]const u8 {
996 assert(buffer.len >= size);
997 const buf = buffer[0..size];
998 try self.reader.readNoEof(buf);
999 return nullStr(buf);
1000 }
1001
1002 inline fn initFile(self: *Self) void {
1003 self.file = File{
1004 .name = self.file_name_buffer[0..0],
1005 .link_name = self.link_name_buffer[0..0],
1006 .size = 0,
1007 .kind = .normal,
1008 .mode = 0,
1009 .reader = self.reader,
1010 };
1011 }
1012
1013 // Number of padding bytes in the last file block.
1014 inline fn blockPadding(size: usize) usize {
1015 const block_rounded = std.mem.alignForward(usize, size, Header.SIZE); // size rounded to te block boundary
1016 return block_rounded - size;
1017 }
1018
1019 // Externally, `next` iterates through the tar archive as if it is a
1020 // series of files. Internally, the tar format often uses fake "files"
1021 // to add meta data that describes the next file. These meta data
1022 // "files" should not normally be visible to the outside. As such, this
1023 // loop iterates through one or more "header files" until it finds a
1024 // "normal file".
1025 pub fn next(self: *Self) !?File {
1026 self.initFile();
1027
1028 while (try self.readHeader()) |header| {
1029 const kind = header.kind();
1030 const size: usize = @intCast(try header.size());
1031 self.padding = blockPadding(size);
1032
1033 switch (kind) {
1034 // File types to retrun upstream
1035 .directory, .normal, .symbolic_link => {
1036 self.file.kind = kind;
1037 self.file.mode = try header.mode();
1038
1039 // set file attributes if not already set by prefix/extended headers
1040 if (self.file.size == 0) {
1041 self.file.size = size;
1042 }
1043 if (self.file.link_name.len == 0) {
1044 self.file.link_name = header.linkName(self.link_name_buffer[0..Header.LINK_NAME_SIZE]);
1045 }
1046 if (self.file.name.len == 0) {
1047 self.file.name = try header.fullName(self.file_name_buffer[0..Header.MAX_NAME_SIZE]);
1048 }
1049
1050 self.padding = blockPadding(self.file.size);
1051 return self.file;
1052 },
1053 // Prefix header types
1054 .gnu_long_name => {
1055 self.file.name = try self.readString(size, &self.file_name_buffer);
1056 },
1057 .gnu_long_link => {
1058 self.file.link_name = try self.readString(size, &self.link_name_buffer);
1059 },
1060 .extended_header => {
1061 // Use just attributes from last extended header.
1062 self.initFile();
1063
1064 var rdr = paxReader(self.reader, size);
1065 while (try rdr.next()) |attr| {
1066 switch (attr.kind) {
1067 .path => {
1068 self.file.name = try attr.value(&self.file_name_buffer);
1069 },
1070 .linkpath => {
1071 self.file.link_name = try attr.value(&self.link_name_buffer);
1072 },
1073 .size => {
1074 var buf: [64]u8 = undefined;
1075 self.file.size = try std.fmt.parseInt(usize, try attr.value(&buf), 10);
1076 },
1077 }
1078 }
1079 },
1080 // Ignored header type
1081 .global_extended_header => {
1082 self.reader.skipBytes(size, .{}) catch return error.TarHeadersTooBig;
1083 },
1084 // All other are unsupported header types
1085 else => {
1086 const d = self.diagnostics orelse return error.TarUnsupportedHeader;
1087 try d.errors.append(d.allocator, .{ .unsupported_file_type = .{
1088 .file_name = try d.allocator.dupe(u8, header.name()),
1089 .file_type = kind,
1090 } });
1091 },
1092 }
1093 }
1094 return null;
1095 }
1096 };
1097}