authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-03-10 16:31:10+01:00
committergravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-03-11 12:25:03+01:00
logc4868b2bbc1df7ea6a3bd12206d10206db1d8965
tree8afe6f0f47b76800bcc50688d9b843b5cfe39a98
parenta9e7abda204300bd13ce08721bf13801817dd6ab

std.tar: use doctest

Make std.tar look better in docs. Remove from public interface what is not necessary. Add comment to the public methods. Add doctest as usage examples for iterator and pipeToFileSystem.

4 files changed, 164 insertions(+), 108 deletions(-)

lib/std/tar.zig+161-62
......@@ -17,10 +17,12 @@
1717
1818const std = @import("std");
1919const assert = std.debug.assert;
20const testing = std.testing;
2021
2122pub const output = @import("tar/output.zig");
2223
23pub const Options = struct {
24/// pipeToFileSystem options
25pub const PipeOptions = struct {
2426 /// Number of directory levels to skip when extracting files.
2527 strip_components: u32 = 0,
2628 /// How to handle the "mode" property of files from within the tar file.
......@@ -84,14 +86,14 @@ pub const Options = struct {
8486 };
8587};
8688
87pub const Header = struct {
89const Header = struct {
8890 const SIZE = 512;
89 pub const MAX_NAME_SIZE = 100 + 1 + 155; // name(100) + separator(1) + prefix(155)
90 pub const LINK_NAME_SIZE = 100;
91 const MAX_NAME_SIZE = 100 + 1 + 155; // name(100) + separator(1) + prefix(155)
92 const LINK_NAME_SIZE = 100;
9193
9294 bytes: *const [SIZE]u8,
9395
94 pub const Kind = enum(u8) {
96 const Kind = enum(u8) {
9597 normal_alias = 0,
9698 normal = '0',
9799 hard_link = '1',
......@@ -237,74 +239,53 @@ fn nullStr(str: []const u8) []const u8 {
237239 return str;
238240}
239241
242/// Options for iterator.
243/// Buffers should be provided by the caller.
240244pub const IteratorOptions = struct {
241245 /// Use a buffer with length `std.fs.MAX_PATH_BYTES` to match file system capabilities.
242246 file_name_buffer: []u8,
243247 /// Use a buffer with length `std.fs.MAX_PATH_BYTES` to match file system capabilities.
244248 link_name_buffer: []u8,
249 /// Provide this to receive detailed error messages.
250 /// When this is provided, some errors which would otherwise be returned immediately
251 /// will instead be added to this structure. The API user must check the errors
252 /// in diagnostics to know whether the operation succeeded or failed.
245253 diagnostics: ?*Diagnostics = null,
246254
247 pub const Diagnostics = Options.Diagnostics;
255 pub const Diagnostics = PipeOptions.Diagnostics;
248256};
249257
250258/// Iterates over files in tar archive.
251/// `next` returns each file in `reader` tar archive.
252///
253/// Init iterator with tar archive reader and provided buffers:
254///
255/// var file_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
256/// var link_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
257///
258/// var iter = std.tar.iterator(archive.reader(), .{
259/// .file_name_buffer = &file_name_buffer,
260/// .link_name_buffer = &link_name_buffer,
261/// });
262///
263/// Iterate on each tar archive file:
264///
265/// while (try iter.next()) |file| {
266/// switch (file.kind) {
267/// .directory => {
268/// // try dir.makePath(file.name);
269/// },
270/// .file => {
271/// // try file.writeAll(writer);
272/// },
273/// .sym_link => {
274/// // try dir.symLink(file.link_name, file.name, .{});
275/// },
276/// }
277/// }
278///
259/// `next` returns each file in tar archive.
279260pub fn iterator(reader: anytype, options: IteratorOptions) Iterator(@TypeOf(reader)) {
280261 return .{
281262 .reader = reader,
282263 .diagnostics = options.diagnostics,
283 .header_buffer = undefined,
284264 .file_name_buffer = options.file_name_buffer,
285265 .link_name_buffer = options.link_name_buffer,
286 .padding = 0,
287266 };
288267}
289268
269/// Type of the file returned by iterator `next` method.
290270pub const FileKind = enum {
291271 directory,
292272 sym_link,
293273 file,
294274};
295275
296fn Iterator(comptime ReaderType: type) type {
276/// Iteartor over entries in the tar file represented by reader.
277pub fn Iterator(comptime ReaderType: type) type {
297278 return struct {
298279 reader: ReaderType,
299 diagnostics: ?*Options.Diagnostics,
280 diagnostics: ?*PipeOptions.Diagnostics = null,
300281
301282 // buffers for heeader and file attributes
302 header_buffer: [Header.SIZE]u8,
283 header_buffer: [Header.SIZE]u8 = undefined,
303284 file_name_buffer: []u8,
304285 link_name_buffer: []u8,
305286
306287 // bytes of padding to the end of the block
307 padding: usize,
288 padding: usize = 0,
308289 // not consumed bytes of file from last next iteration
309290 unread_file_bytes: u64 = 0,
310291
......@@ -316,18 +297,18 @@ fn Iterator(comptime ReaderType: type) type {
316297 kind: FileKind = .file,
317298
318299 unread_bytes: *u64,
319 reader: ReaderType,
300 parent_reader: ReaderType,
320301
321 pub const Reader = std.io.Reader(*Self, ReaderType.Error, read);
302 pub const Reader = std.io.Reader(File, ReaderType.Error, File.read);
322303
323 pub fn reader(self: *Self) Reader {
304 pub fn reader(self: File) Reader {
324305 return .{ .context = self };
325306 }
326307
327 pub fn read(self: *Self, dest: []u8) ReaderType.Error!usize {
328 const buf = dest[0..@min(dest.len, self.unread_size.*)];
329 const n = try self.reader.read(buf);
330 self.unread_size.* -= n;
308 pub fn read(self: File, dest: []u8) ReaderType.Error!usize {
309 const buf = dest[0..@min(dest.len, self.unread_bytes.*)];
310 const n = try self.parent_reader.read(buf);
311 self.unread_bytes.* -= n;
331312 return n;
332313 }
333314
......@@ -337,7 +318,7 @@ fn Iterator(comptime ReaderType: type) type {
337318
338319 while (self.unread_bytes.* > 0) {
339320 const buf = buffer[0..@min(buffer.len, self.unread_bytes.*)];
340 try self.reader.readNoEof(buf);
321 try self.parent_reader.readNoEof(buf);
341322 try writer.writeAll(buf);
342323 self.unread_bytes.* -= buf.len;
343324 }
......@@ -369,7 +350,7 @@ fn Iterator(comptime ReaderType: type) type {
369350 return .{
370351 .name = self.file_name_buffer[0..0],
371352 .link_name = self.link_name_buffer[0..0],
372 .reader = self.reader,
353 .parent_reader = self.reader,
373354 .unread_bytes = &self.unread_file_bytes,
374355 };
375356 }
......@@ -594,7 +575,8 @@ fn PaxIterator(comptime ReaderType: type) type {
594575 };
595576}
596577
597pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !void {
578/// Saves tar file content to the file systems.
579pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: PipeOptions) !void {
598580 switch (options.mode_mode) {
599581 .ignore => {},
600582 .executable_bit_only => {
......@@ -699,7 +681,7 @@ fn stripComponents(path: []const u8, count: u32) []const u8 {
699681}
700682
701683test "stripComponents" {
702 const expectEqualStrings = std.testing.expectEqualStrings;
684 const expectEqualStrings = testing.expectEqualStrings;
703685 try expectEqualStrings("a/b/c", stripComponents("a/b/c", 0));
704686 try expectEqualStrings("b/c", stripComponents("a/b/c", 1));
705687 try expectEqualStrings("c", stripComponents("a/b/c", 2));
......@@ -810,24 +792,24 @@ test "PaxIterator" {
810792 var i: usize = 0;
811793 while (iter.next() catch |err| {
812794 if (case.err) |e| {
813 try std.testing.expectEqual(e, err);
795 try testing.expectEqual(e, err);
814796 continue;
815797 }
816798 return err;
817799 }) |attr| : (i += 1) {
818800 const exp = case.attrs[i];
819 try std.testing.expectEqual(exp.kind, attr.kind);
801 try testing.expectEqual(exp.kind, attr.kind);
820802 const value = attr.value(&buffer) catch |err| {
821803 if (exp.err) |e| {
822 try std.testing.expectEqual(e, err);
804 try testing.expectEqual(e, err);
823805 break :outer;
824806 }
825807 return err;
826808 };
827 try std.testing.expectEqualStrings(exp.value, value);
809 try testing.expectEqualStrings(exp.value, value);
828810 }
829 try std.testing.expectEqual(case.attrs.len, i);
830 try std.testing.expect(case.err == null);
811 try testing.expectEqual(case.attrs.len, i);
812 try testing.expect(case.err == null);
831813 }
832814}
833815
......@@ -863,9 +845,9 @@ test "header parse size" {
863845 @memcpy(bytes[124 .. 124 + case.in.len], case.in);
864846 var header = Header{ .bytes = &bytes };
865847 if (case.err) |err| {
866 try std.testing.expectError(err, header.size());
848 try testing.expectError(err, header.size());
867849 } else {
868 try std.testing.expectEqual(case.want, try header.size());
850 try testing.expectEqual(case.want, try header.size());
869851 }
870852 }
871853}
......@@ -888,15 +870,15 @@ test "header parse mode" {
888870 @memcpy(bytes[100 .. 100 + case.in.len], case.in);
889871 var header = Header{ .bytes = &bytes };
890872 if (case.err) |err| {
891 try std.testing.expectError(err, header.mode());
873 try testing.expectError(err, header.mode());
892874 } else {
893 try std.testing.expectEqual(case.want, try header.mode());
875 try testing.expectEqual(case.want, try header.mode());
894876 }
895877 }
896878}
897879
898880test "create file and symlink" {
899 var root = std.testing.tmpDir(.{});
881 var root = testing.tmpDir(.{});
900882 defer root.cleanup();
901883
902884 var file = try createDirAndFile(root.dir, "file1");
......@@ -916,3 +898,120 @@ test "create file and symlink" {
916898 file = try createDirAndFile(root.dir, "g/h/i/file4");
917899 file.close();
918900}
901
902test iterator {
903 // Example tar file is created from this tree structure:
904 // $ tree example
905 // example
906 // ├── a
907 // │   └── file
908 // ├── b
909 // │   └── symlink -> ../a/file
910 // └── empty
911 // $ cat example/a/file
912 // content
913 // $ tar -cf example.tar example
914 // $ tar -tvf example.tar
915 // example/
916 // example/b/
917 // example/b/symlink -> ../a/file
918 // example/a/
919 // example/a/file
920 // example/empty/
921
922 const data = @embedFile("tar/testdata/example.tar");
923 var fbs = std.io.fixedBufferStream(data);
924
925 // User provided buffers to the iterator
926 var file_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
927 var link_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
928 // Create iterator
929 var iter = iterator(fbs.reader(), .{
930 .file_name_buffer = &file_name_buffer,
931 .link_name_buffer = &link_name_buffer,
932 });
933 // Iterate over files in example.tar
934 var file_no: usize = 0;
935 while (try iter.next()) |file| : (file_no += 1) {
936 switch (file.kind) {
937 .directory => {
938 switch (file_no) {
939 0 => try testing.expectEqualStrings("example/", file.name),
940 1 => try testing.expectEqualStrings("example/b/", file.name),
941 3 => try testing.expectEqualStrings("example/a/", file.name),
942 5 => try testing.expectEqualStrings("example/empty/", file.name),
943 else => unreachable,
944 }
945 },
946 .file => {
947 try testing.expectEqualStrings("example/a/file", file.name);
948 // Read file content
949 var buf: [16]u8 = undefined;
950 const n = try file.reader().readAll(&buf);
951 try testing.expectEqualStrings("content\n", buf[0..n]);
952 },
953 .sym_link => {
954 try testing.expectEqualStrings("example/b/symlink", file.name);
955 try testing.expectEqualStrings("../a/file", file.link_name);
956 },
957 }
958 }
959}
960
961test pipeToFileSystem {
962 // Example tar file is created from this tree structure:
963 // $ tree example
964 // example
965 // ├── a
966 // │   └── file
967 // ├── b
968 // │   └── symlink -> ../a/file
969 // └── empty
970 // $ cat example/a/file
971 // content
972 // $ tar -cf example.tar example
973 // $ tar -tvf example.tar
974 // example/
975 // example/b/
976 // example/b/symlink -> ../a/file
977 // example/a/
978 // example/a/file
979 // example/empty/
980
981 const data = @embedFile("tar/testdata/example.tar");
982 var fbs = std.io.fixedBufferStream(data);
983 const reader = fbs.reader();
984
985 var tmp = testing.tmpDir(.{ .no_follow = true });
986 defer tmp.cleanup();
987 const dir = tmp.dir;
988
989 // Save tar from `reader` to the file system `dir`
990 pipeToFileSystem(dir, reader, .{
991 .mode_mode = .ignore,
992 .strip_components = 1,
993 .exclude_empty_directories = true,
994 }) catch |err| {
995 // Skip on platform which don't support symlinks
996 if (err == error.UnableToCreateSymLink) return error.SkipZigTest;
997 return err;
998 };
999
1000 try testing.expectError(error.FileNotFound, dir.statFile("empty"));
1001 try testing.expect((try dir.statFile("a/file")).kind == .file);
1002 try testing.expect((try dir.statFile("b/symlink")).kind == .file); // statFile follows symlink
1003
1004 var buf: [32]u8 = undefined;
1005 try testing.expectEqualSlices(
1006 u8,
1007 "../a/file",
1008 normalizePath(try dir.readLink("b/symlink", &buf)),
1009 );
1010}
1011
1012fn normalizePath(bytes: []u8) []u8 {
1013 const canonical_sep = std.fs.path.sep_posix;
1014 if (std.fs.path.sep == canonical_sep) return bytes;
1015 std.mem.replaceScalar(u8, bytes, std.fs.path.sep, canonical_sep);
1016 return bytes;
1017}
lib/std/tar/test.zig+3-46
......@@ -385,8 +385,8 @@ test "run test cases" {
385385test "pax/gnu long names with small buffer" {
386386 // should fail with insufficient buffer error
387387
388 var min_file_name_buffer: [tar.Header.MAX_NAME_SIZE]u8 = undefined;
389 var min_link_name_buffer: [tar.Header.LINK_NAME_SIZE]u8 = undefined;
388 var min_file_name_buffer: [256]u8 = undefined;
389 var min_link_name_buffer: [100]u8 = undefined;
390390 const long_name_cases = [_]Case{ cases[11], cases[25], cases[28] };
391391
392392 for (long_name_cases) |case| {
......@@ -409,7 +409,7 @@ test "pax/gnu long names with small buffer" {
409409
410410test "insufficient buffer in Header name filed" {
411411 var min_file_name_buffer: [9]u8 = undefined;
412 var min_link_name_buffer: [tar.Header.LINK_NAME_SIZE]u8 = undefined;
412 var min_link_name_buffer: [100]u8 = undefined;
413413
414414 var fsb = std.io.fixedBufferStream(cases[0].data);
415415 var iter = tar.iterator(fsb.reader(), .{
......@@ -509,46 +509,3 @@ test "case sensitivity" {
509509 try testing.expect((try root.dir.statFile("alacritty/darkermatrix.yml")).kind == .file);
510510 try testing.expect((try root.dir.statFile("alacritty/Darkermatrix.yml")).kind == .file);
511511}
512
513test "pipeToFileSystem" {
514 // $ tar tvf
515 // pipe_to_file_system_test/
516 // pipe_to_file_system_test/b/
517 // pipe_to_file_system_test/b/symlink -> ../a/file
518 // pipe_to_file_system_test/a/
519 // pipe_to_file_system_test/a/file
520 // pipe_to_file_system_test/empty/
521 const data = @embedFile("testdata/pipe_to_file_system_test.tar");
522 var fsb = std.io.fixedBufferStream(data);
523
524 var root = std.testing.tmpDir(.{ .no_follow = true });
525 defer root.cleanup();
526
527 tar.pipeToFileSystem(root.dir, fsb.reader(), .{
528 .mode_mode = .ignore,
529 .strip_components = 1,
530 .exclude_empty_directories = true,
531 }) catch |err| {
532 // Skip on platform which don't support symlinks
533 if (err == error.UnableToCreateSymLink) return error.SkipZigTest;
534 return err;
535 };
536
537 try testing.expectError(error.FileNotFound, root.dir.statFile("empty"));
538 try testing.expect((try root.dir.statFile("a/file")).kind == .file);
539 try testing.expect((try root.dir.statFile("b/symlink")).kind == .file); // statFile follows symlink
540
541 var buf: [32]u8 = undefined;
542 try testing.expectEqualSlices(
543 u8,
544 "../a/file",
545 normalizePath(try root.dir.readLink("b/symlink", &buf)),
546 );
547}
548
549fn normalizePath(bytes: []u8) []u8 {
550 const canonical_sep = std.fs.path.sep_posix;
551 if (std.fs.path.sep == canonical_sep) return bytes;
552 std.mem.replaceScalar(u8, bytes, std.fs.path.sep, canonical_sep);
553 return bytes;
554}
lib/std/tar/testdata/example.tar created
Binary files /dev/null and b/lib/std/tar/testdata/example.tar differ
lib/std/tar/testdata/pipe_to_file_system_test.tar deleted
Binary files a/lib/std/tar/testdata/pipe_to_file_system_test.tar and /dev/null differ