authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-03-10 16:31:10+01:00
committergravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-03-11 12:25:03+01:00
logc4868b2bbc1df7ea6a3bd12206d10206db1d8965
tree8afe6f0f47b76800bcc50688d9b843b5cfe39a98
parenta9e7abda204300bd13ce08721bf13801817dd6ab

std.tar: use doctest

Make std.tar look better in docs. Remove from public interface what is not necessary. Add comment to the public methods. Add doctest as usage examples for iterator and pipeToFileSystem.

4 files changed, 164 insertions(+), 108 deletions(-)

lib/std/tar.zig+161-62
...@@ -17,10 +17,12 @@...@@ -17,10 +17,12 @@
1717
18const std = @import("std");18const std = @import("std");
19const assert = std.debug.assert;19const assert = std.debug.assert;
20const testing = std.testing;
2021
21pub const output = @import("tar/output.zig");22pub const output = @import("tar/output.zig");
2223
23pub const Options = struct {24/// pipeToFileSystem options
25pub const PipeOptions = struct {
24 /// Number of directory levels to skip when extracting files.26 /// Number of directory levels to skip when extracting files.
25 strip_components: u32 = 0,27 strip_components: u32 = 0,
26 /// How to handle the "mode" property of files from within the tar file.28 /// How to handle the "mode" property of files from within the tar file.
...@@ -84,14 +86,14 @@ pub const Options = struct {...@@ -84,14 +86,14 @@ pub const Options = struct {
84 };86 };
85};87};
8688
87pub const Header = struct {89const Header = struct {
88 const SIZE = 512;90 const SIZE = 512;
89 pub const MAX_NAME_SIZE = 100 + 1 + 155; // name(100) + separator(1) + prefix(155)91 const MAX_NAME_SIZE = 100 + 1 + 155; // name(100) + separator(1) + prefix(155)
90 pub const LINK_NAME_SIZE = 100;92 const LINK_NAME_SIZE = 100;
9193
92 bytes: *const [SIZE]u8,94 bytes: *const [SIZE]u8,
9395
94 pub const Kind = enum(u8) {96 const Kind = enum(u8) {
95 normal_alias = 0,97 normal_alias = 0,
96 normal = '0',98 normal = '0',
97 hard_link = '1',99 hard_link = '1',
...@@ -237,74 +239,53 @@ fn nullStr(str: []const u8) []const u8 {...@@ -237,74 +239,53 @@ fn nullStr(str: []const u8) []const u8 {
237 return str;239 return str;
238}240}
239241
242/// Options for iterator.
243/// Buffers should be provided by the caller.
240pub const IteratorOptions = struct {244pub const IteratorOptions = struct {
241 /// Use a buffer with length `std.fs.MAX_PATH_BYTES` to match file system capabilities.245 /// Use a buffer with length `std.fs.MAX_PATH_BYTES` to match file system capabilities.
242 file_name_buffer: []u8,246 file_name_buffer: []u8,
243 /// Use a buffer with length `std.fs.MAX_PATH_BYTES` to match file system capabilities.247 /// Use a buffer with length `std.fs.MAX_PATH_BYTES` to match file system capabilities.
244 link_name_buffer: []u8,248 link_name_buffer: []u8,
249 /// Provide this to receive detailed error messages.
250 /// When this is provided, some errors which would otherwise be returned immediately
251 /// will instead be added to this structure. The API user must check the errors
252 /// in diagnostics to know whether the operation succeeded or failed.
245 diagnostics: ?*Diagnostics = null,253 diagnostics: ?*Diagnostics = null,
246254
247 pub const Diagnostics = Options.Diagnostics;255 pub const Diagnostics = PipeOptions.Diagnostics;
248};256};
249257
250/// Iterates over files in tar archive.258/// Iterates over files in tar archive.
251/// `next` returns each file in `reader` tar archive.259/// `next` returns each file in tar archive.
252///
253/// Init iterator with tar archive reader and provided buffers:
254///
255/// var file_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
256/// var link_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
257///
258/// var iter = std.tar.iterator(archive.reader(), .{
259/// .file_name_buffer = &file_name_buffer,
260/// .link_name_buffer = &link_name_buffer,
261/// });
262///
263/// Iterate on each tar archive file:
264///
265/// while (try iter.next()) |file| {
266/// switch (file.kind) {
267/// .directory => {
268/// // try dir.makePath(file.name);
269/// },
270/// .file => {
271/// // try file.writeAll(writer);
272/// },
273/// .sym_link => {
274/// // try dir.symLink(file.link_name, file.name, .{});
275/// },
276/// }
277/// }
278///
279pub fn iterator(reader: anytype, options: IteratorOptions) Iterator(@TypeOf(reader)) {260pub fn iterator(reader: anytype, options: IteratorOptions) Iterator(@TypeOf(reader)) {
280 return .{261 return .{
281 .reader = reader,262 .reader = reader,
282 .diagnostics = options.diagnostics,263 .diagnostics = options.diagnostics,
283 .header_buffer = undefined,
284 .file_name_buffer = options.file_name_buffer,264 .file_name_buffer = options.file_name_buffer,
285 .link_name_buffer = options.link_name_buffer,265 .link_name_buffer = options.link_name_buffer,
286 .padding = 0,
287 };266 };
288}267}
289268
269/// Type of the file returned by iterator `next` method.
290pub const FileKind = enum {270pub const FileKind = enum {
291 directory,271 directory,
292 sym_link,272 sym_link,
293 file,273 file,
294};274};
295275
296fn Iterator(comptime ReaderType: type) type {276/// Iteartor over entries in the tar file represented by reader.
277pub fn Iterator(comptime ReaderType: type) type {
297 return struct {278 return struct {
298 reader: ReaderType,279 reader: ReaderType,
299 diagnostics: ?*Options.Diagnostics,280 diagnostics: ?*PipeOptions.Diagnostics = null,
300281
301 // buffers for heeader and file attributes282 // buffers for heeader and file attributes
302 header_buffer: [Header.SIZE]u8,283 header_buffer: [Header.SIZE]u8 = undefined,
303 file_name_buffer: []u8,284 file_name_buffer: []u8,
304 link_name_buffer: []u8,285 link_name_buffer: []u8,
305286
306 // bytes of padding to the end of the block287 // bytes of padding to the end of the block
307 padding: usize,288 padding: usize = 0,
308 // not consumed bytes of file from last next iteration289 // not consumed bytes of file from last next iteration
309 unread_file_bytes: u64 = 0,290 unread_file_bytes: u64 = 0,
310291
...@@ -316,18 +297,18 @@ fn Iterator(comptime ReaderType: type) type {...@@ -316,18 +297,18 @@ fn Iterator(comptime ReaderType: type) type {
316 kind: FileKind = .file,297 kind: FileKind = .file,
317298
318 unread_bytes: *u64,299 unread_bytes: *u64,
319 reader: ReaderType,300 parent_reader: ReaderType,
320301
321 pub const Reader = std.io.Reader(*Self, ReaderType.Error, read);302 pub const Reader = std.io.Reader(File, ReaderType.Error, File.read);
322303
323 pub fn reader(self: *Self) Reader {304 pub fn reader(self: File) Reader {
324 return .{ .context = self };305 return .{ .context = self };
325 }306 }
326307
327 pub fn read(self: *Self, dest: []u8) ReaderType.Error!usize {308 pub fn read(self: File, dest: []u8) ReaderType.Error!usize {
328 const buf = dest[0..@min(dest.len, self.unread_size.*)];309 const buf = dest[0..@min(dest.len, self.unread_bytes.*)];
329 const n = try self.reader.read(buf);310 const n = try self.parent_reader.read(buf);
330 self.unread_size.* -= n;311 self.unread_bytes.* -= n;
331 return n;312 return n;
332 }313 }
333314
...@@ -337,7 +318,7 @@ fn Iterator(comptime ReaderType: type) type {...@@ -337,7 +318,7 @@ fn Iterator(comptime ReaderType: type) type {
337318
338 while (self.unread_bytes.* > 0) {319 while (self.unread_bytes.* > 0) {
339 const buf = buffer[0..@min(buffer.len, self.unread_bytes.*)];320 const buf = buffer[0..@min(buffer.len, self.unread_bytes.*)];
340 try self.reader.readNoEof(buf);321 try self.parent_reader.readNoEof(buf);
341 try writer.writeAll(buf);322 try writer.writeAll(buf);
342 self.unread_bytes.* -= buf.len;323 self.unread_bytes.* -= buf.len;
343 }324 }
...@@ -369,7 +350,7 @@ fn Iterator(comptime ReaderType: type) type {...@@ -369,7 +350,7 @@ fn Iterator(comptime ReaderType: type) type {
369 return .{350 return .{
370 .name = self.file_name_buffer[0..0],351 .name = self.file_name_buffer[0..0],
371 .link_name = self.link_name_buffer[0..0],352 .link_name = self.link_name_buffer[0..0],
372 .reader = self.reader,353 .parent_reader = self.reader,
373 .unread_bytes = &self.unread_file_bytes,354 .unread_bytes = &self.unread_file_bytes,
374 };355 };
375 }356 }
...@@ -594,7 +575,8 @@ fn PaxIterator(comptime ReaderType: type) type {...@@ -594,7 +575,8 @@ fn PaxIterator(comptime ReaderType: type) type {
594 };575 };
595}576}
596577
597pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !void {578/// Saves tar file content to the file systems.
579pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: PipeOptions) !void {
598 switch (options.mode_mode) {580 switch (options.mode_mode) {
599 .ignore => {},581 .ignore => {},
600 .executable_bit_only => {582 .executable_bit_only => {
...@@ -699,7 +681,7 @@ fn stripComponents(path: []const u8, count: u32) []const u8 {...@@ -699,7 +681,7 @@ fn stripComponents(path: []const u8, count: u32) []const u8 {
699}681}
700682
701test "stripComponents" {683test "stripComponents" {
702 const expectEqualStrings = std.testing.expectEqualStrings;684 const expectEqualStrings = testing.expectEqualStrings;
703 try expectEqualStrings("a/b/c", stripComponents("a/b/c", 0));685 try expectEqualStrings("a/b/c", stripComponents("a/b/c", 0));
704 try expectEqualStrings("b/c", stripComponents("a/b/c", 1));686 try expectEqualStrings("b/c", stripComponents("a/b/c", 1));
705 try expectEqualStrings("c", stripComponents("a/b/c", 2));687 try expectEqualStrings("c", stripComponents("a/b/c", 2));
...@@ -810,24 +792,24 @@ test "PaxIterator" {...@@ -810,24 +792,24 @@ test "PaxIterator" {
810 var i: usize = 0;792 var i: usize = 0;
811 while (iter.next() catch |err| {793 while (iter.next() catch |err| {
812 if (case.err) |e| {794 if (case.err) |e| {
813 try std.testing.expectEqual(e, err);795 try testing.expectEqual(e, err);
814 continue;796 continue;
815 }797 }
816 return err;798 return err;
817 }) |attr| : (i += 1) {799 }) |attr| : (i += 1) {
818 const exp = case.attrs[i];800 const exp = case.attrs[i];
819 try std.testing.expectEqual(exp.kind, attr.kind);801 try testing.expectEqual(exp.kind, attr.kind);
820 const value = attr.value(&buffer) catch |err| {802 const value = attr.value(&buffer) catch |err| {
821 if (exp.err) |e| {803 if (exp.err) |e| {
822 try std.testing.expectEqual(e, err);804 try testing.expectEqual(e, err);
823 break :outer;805 break :outer;
824 }806 }
825 return err;807 return err;
826 };808 };
827 try std.testing.expectEqualStrings(exp.value, value);809 try testing.expectEqualStrings(exp.value, value);
828 }810 }
829 try std.testing.expectEqual(case.attrs.len, i);811 try testing.expectEqual(case.attrs.len, i);
830 try std.testing.expect(case.err == null);812 try testing.expect(case.err == null);
831 }813 }
832}814}
833815
...@@ -863,9 +845,9 @@ test "header parse size" {...@@ -863,9 +845,9 @@ test "header parse size" {
863 @memcpy(bytes[124 .. 124 + case.in.len], case.in);845 @memcpy(bytes[124 .. 124 + case.in.len], case.in);
864 var header = Header{ .bytes = &bytes };846 var header = Header{ .bytes = &bytes };
865 if (case.err) |err| {847 if (case.err) |err| {
866 try std.testing.expectError(err, header.size());848 try testing.expectError(err, header.size());
867 } else {849 } else {
868 try std.testing.expectEqual(case.want, try header.size());850 try testing.expectEqual(case.want, try header.size());
869 }851 }
870 }852 }
871}853}
...@@ -888,15 +870,15 @@ test "header parse mode" {...@@ -888,15 +870,15 @@ test "header parse mode" {
888 @memcpy(bytes[100 .. 100 + case.in.len], case.in);870 @memcpy(bytes[100 .. 100 + case.in.len], case.in);
889 var header = Header{ .bytes = &bytes };871 var header = Header{ .bytes = &bytes };
890 if (case.err) |err| {872 if (case.err) |err| {
891 try std.testing.expectError(err, header.mode());873 try testing.expectError(err, header.mode());
892 } else {874 } else {
893 try std.testing.expectEqual(case.want, try header.mode());875 try testing.expectEqual(case.want, try header.mode());
894 }876 }
895 }877 }
896}878}
897879
898test "create file and symlink" {880test "create file and symlink" {
899 var root = std.testing.tmpDir(.{});881 var root = testing.tmpDir(.{});
900 defer root.cleanup();882 defer root.cleanup();
901883
902 var file = try createDirAndFile(root.dir, "file1");884 var file = try createDirAndFile(root.dir, "file1");
...@@ -916,3 +898,120 @@ test "create file and symlink" {...@@ -916,3 +898,120 @@ test "create file and symlink" {
916 file = try createDirAndFile(root.dir, "g/h/i/file4");898 file = try createDirAndFile(root.dir, "g/h/i/file4");
917 file.close();899 file.close();
918}900}
901
902test iterator {
903 // Example tar file is created from this tree structure:
904 // $ tree example
905 // example
906 // ├── a
907 // │   └── file
908 // ├── b
909 // │   └── symlink -> ../a/file
910 // └── empty
911 // $ cat example/a/file
912 // content
913 // $ tar -cf example.tar example
914 // $ tar -tvf example.tar
915 // example/
916 // example/b/
917 // example/b/symlink -> ../a/file
918 // example/a/
919 // example/a/file
920 // example/empty/
921
922 const data = @embedFile("tar/testdata/example.tar");
923 var fbs = std.io.fixedBufferStream(data);
924
925 // User provided buffers to the iterator
926 var file_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
927 var link_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
928 // Create iterator
929 var iter = iterator(fbs.reader(), .{
930 .file_name_buffer = &file_name_buffer,
931 .link_name_buffer = &link_name_buffer,
932 });
933 // Iterate over files in example.tar
934 var file_no: usize = 0;
935 while (try iter.next()) |file| : (file_no += 1) {
936 switch (file.kind) {
937 .directory => {
938 switch (file_no) {
939 0 => try testing.expectEqualStrings("example/", file.name),
940 1 => try testing.expectEqualStrings("example/b/", file.name),
941 3 => try testing.expectEqualStrings("example/a/", file.name),
942 5 => try testing.expectEqualStrings("example/empty/", file.name),
943 else => unreachable,
944 }
945 },
946 .file => {
947 try testing.expectEqualStrings("example/a/file", file.name);
948 // Read file content
949 var buf: [16]u8 = undefined;
950 const n = try file.reader().readAll(&buf);
951 try testing.expectEqualStrings("content\n", buf[0..n]);
952 },
953 .sym_link => {
954 try testing.expectEqualStrings("example/b/symlink", file.name);
955 try testing.expectEqualStrings("../a/file", file.link_name);
956 },
957 }
958 }
959}
960
961test pipeToFileSystem {
962 // Example tar file is created from this tree structure:
963 // $ tree example
964 // example
965 // ├── a
966 // │   └── file
967 // ├── b
968 // │   └── symlink -> ../a/file
969 // └── empty
970 // $ cat example/a/file
971 // content
972 // $ tar -cf example.tar example
973 // $ tar -tvf example.tar
974 // example/
975 // example/b/
976 // example/b/symlink -> ../a/file
977 // example/a/
978 // example/a/file
979 // example/empty/
980
981 const data = @embedFile("tar/testdata/example.tar");
982 var fbs = std.io.fixedBufferStream(data);
983 const reader = fbs.reader();
984
985 var tmp = testing.tmpDir(.{ .no_follow = true });
986 defer tmp.cleanup();
987 const dir = tmp.dir;
988
989 // Save tar from `reader` to the file system `dir`
990 pipeToFileSystem(dir, reader, .{
991 .mode_mode = .ignore,
992 .strip_components = 1,
993 .exclude_empty_directories = true,
994 }) catch |err| {
995 // Skip on platform which don't support symlinks
996 if (err == error.UnableToCreateSymLink) return error.SkipZigTest;
997 return err;
998 };
999
1000 try testing.expectError(error.FileNotFound, dir.statFile("empty"));
1001 try testing.expect((try dir.statFile("a/file")).kind == .file);
1002 try testing.expect((try dir.statFile("b/symlink")).kind == .file); // statFile follows symlink
1003
1004 var buf: [32]u8 = undefined;
1005 try testing.expectEqualSlices(
1006 u8,
1007 "../a/file",
1008 normalizePath(try dir.readLink("b/symlink", &buf)),
1009 );
1010}
1011
1012fn normalizePath(bytes: []u8) []u8 {
1013 const canonical_sep = std.fs.path.sep_posix;
1014 if (std.fs.path.sep == canonical_sep) return bytes;
1015 std.mem.replaceScalar(u8, bytes, std.fs.path.sep, canonical_sep);
1016 return bytes;
1017}
lib/std/tar/test.zig+3-46
...@@ -385,8 +385,8 @@ test "run test cases" {...@@ -385,8 +385,8 @@ test "run test cases" {
385test "pax/gnu long names with small buffer" {385test "pax/gnu long names with small buffer" {
386 // should fail with insufficient buffer error386 // should fail with insufficient buffer error
387387
388 var min_file_name_buffer: [tar.Header.MAX_NAME_SIZE]u8 = undefined;388 var min_file_name_buffer: [256]u8 = undefined;
389 var min_link_name_buffer: [tar.Header.LINK_NAME_SIZE]u8 = undefined;389 var min_link_name_buffer: [100]u8 = undefined;
390 const long_name_cases = [_]Case{ cases[11], cases[25], cases[28] };390 const long_name_cases = [_]Case{ cases[11], cases[25], cases[28] };
391391
392 for (long_name_cases) |case| {392 for (long_name_cases) |case| {
...@@ -409,7 +409,7 @@ test "pax/gnu long names with small buffer" {...@@ -409,7 +409,7 @@ test "pax/gnu long names with small buffer" {
409409
410test "insufficient buffer in Header name filed" {410test "insufficient buffer in Header name filed" {
411 var min_file_name_buffer: [9]u8 = undefined;411 var min_file_name_buffer: [9]u8 = undefined;
412 var min_link_name_buffer: [tar.Header.LINK_NAME_SIZE]u8 = undefined;412 var min_link_name_buffer: [100]u8 = undefined;
413413
414 var fsb = std.io.fixedBufferStream(cases[0].data);414 var fsb = std.io.fixedBufferStream(cases[0].data);
415 var iter = tar.iterator(fsb.reader(), .{415 var iter = tar.iterator(fsb.reader(), .{
...@@ -509,46 +509,3 @@ test "case sensitivity" {...@@ -509,46 +509,3 @@ test "case sensitivity" {
509 try testing.expect((try root.dir.statFile("alacritty/darkermatrix.yml")).kind == .file);509 try testing.expect((try root.dir.statFile("alacritty/darkermatrix.yml")).kind == .file);
510 try testing.expect((try root.dir.statFile("alacritty/Darkermatrix.yml")).kind == .file);510 try testing.expect((try root.dir.statFile("alacritty/Darkermatrix.yml")).kind == .file);
511}511}
512
513test "pipeToFileSystem" {
514 // $ tar tvf
515 // pipe_to_file_system_test/
516 // pipe_to_file_system_test/b/
517 // pipe_to_file_system_test/b/symlink -> ../a/file
518 // pipe_to_file_system_test/a/
519 // pipe_to_file_system_test/a/file
520 // pipe_to_file_system_test/empty/
521 const data = @embedFile("testdata/pipe_to_file_system_test.tar");
522 var fsb = std.io.fixedBufferStream(data);
523
524 var root = std.testing.tmpDir(.{ .no_follow = true });
525 defer root.cleanup();
526
527 tar.pipeToFileSystem(root.dir, fsb.reader(), .{
528 .mode_mode = .ignore,
529 .strip_components = 1,
530 .exclude_empty_directories = true,
531 }) catch |err| {
532 // Skip on platform which don't support symlinks
533 if (err == error.UnableToCreateSymLink) return error.SkipZigTest;
534 return err;
535 };
536
537 try testing.expectError(error.FileNotFound, root.dir.statFile("empty"));
538 try testing.expect((try root.dir.statFile("a/file")).kind == .file);
539 try testing.expect((try root.dir.statFile("b/symlink")).kind == .file); // statFile follows symlink
540
541 var buf: [32]u8 = undefined;
542 try testing.expectEqualSlices(
543 u8,
544 "../a/file",
545 normalizePath(try root.dir.readLink("b/symlink", &buf)),
546 );
547}
548
549fn normalizePath(bytes: []u8) []u8 {
550 const canonical_sep = std.fs.path.sep_posix;
551 if (std.fs.path.sep == canonical_sep) return bytes;
552 std.mem.replaceScalar(u8, bytes, std.fs.path.sep, canonical_sep);
553 return bytes;
554}
lib/std/tar/testdata/example.tar created
Binary files /dev/null and b/lib/std/tar/testdata/example.tar differ
lib/std/tar/testdata/pipe_to_file_system_test.tar deleted
Binary files a/lib/std/tar/testdata/pipe_to_file_system_test.tar and /dev/null differ