authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-03-02 17:52:31+01:00
committergravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-03-11 12:22:12+01:00
log614161a7cf65f46cb3e2461ffe2e09d972508a97
treee421443136c21a1b24da91c18d76ece3194f00c9
parent5ccbb196ad08fd5e58fc8874917a20f9a220d729

std.tar make iterator interface more ergonomic

for the then end users: 1. Don't require user to call file.skip() on file returned from iterator.next if file is not read. Iterator will now handle this. Previously that returned header parsing error, without knowing some tar internals it is hard to understand what is required from user. 2. Use iterator.File.kind enum which is similar to fs.File.Kind, something familiar. Internal Header.Kind has many types which are not exposed but the user needs to have else in kind switch to cover those cases. 3. Add reader interface to the iterator.File.

2 files changed, 75 insertions(+), 53 deletions(-)

lib/std/tar.zig+63-43
......@@ -258,10 +258,15 @@ pub fn iterator(reader: anytype, options: IteratorOptions) Iterator(@TypeOf(read
258258 .file_name_buffer = options.file_name_buffer,
259259 .link_name_buffer = options.link_name_buffer,
260260 .padding = 0,
261 .file = undefined,
262261 };
263262}
264263
264pub const FileKind = enum {
265 directory,
266 sym_link,
267 file,
268};
269
265270fn Iterator(comptime ReaderType: type) type {
266271 return struct {
267272 reader: ReaderType,
......@@ -274,35 +279,43 @@ fn Iterator(comptime ReaderType: type) type {
274279
275280 // bytes of padding to the end of the block
276281 padding: usize,
277 // current tar file
278 file: File,
282 // not consumed bytes of file from last next iteration
283 unread_file_bytes: usize = 0,
279284
280285 pub const File = struct {
281286 name: []const u8, // name of file, symlink or directory
282287 link_name: []const u8, // target name of symlink
283 size: u64, // size of the file in bytes
284 mode: u32,
285 kind: Header.Kind,
288 size: u64 = 0, // size of the file in bytes
289 mode: u32 = 0,
290 kind: FileKind = .file,
286291
292 unread_bytes: *usize,
287293 reader: ReaderType,
288294
295 pub const Reader = std.io.Reader(*Self, ReaderType.Error, read);
296
297 pub fn reader(self: *Self) Reader {
298 return .{ .context = self };
299 }
300
301 pub fn read(self: *Self, dest: []u8) ReaderType.Error!usize {
302 const buf = dest[0..@min(dest.len, self.unread_size.*)];
303 const n = try self.reader.read(buf);
304 self.unread_size.* -= n;
305 return n;
306 }
307
289308 // Writes file content to writer.
290 pub fn write(self: File, writer: anytype) !void {
309 pub fn writeAll(self: File, writer: anytype) !void {
291310 var buffer: [4096]u8 = undefined;
292311
293 var n: u64 = 0;
294 while (n < self.size) {
295 const buf = buffer[0..@min(buffer.len, self.size - n)];
312 while (self.unread_bytes.* > 0) {
313 const buf = buffer[0..@min(buffer.len, self.unread_bytes.*)];
296314 try self.reader.readNoEof(buf);
297315 try writer.writeAll(buf);
298 n += buf.len;
316 self.unread_bytes.* -= buf.len;
299317 }
300318 }
301
302 // Skips file content. Advances reader.
303 pub fn skip(self: File) !void {
304 try self.reader.skipBytes(self.size, .{});
305 }
306319 };
307320
308321 const Self = @This();
......@@ -326,14 +339,12 @@ fn Iterator(comptime ReaderType: type) type {
326339 return nullStr(buf);
327340 }
328341
329 fn initFile(self: *Self) void {
330 self.file = .{
342 fn newFile(self: *Self) File {
343 return .{
331344 .name = self.file_name_buffer[0..0],
332345 .link_name = self.link_name_buffer[0..0],
333 .size = 0,
334 .kind = .normal,
335 .mode = 0,
336346 .reader = self.reader,
347 .unread_bytes = &self.unread_file_bytes,
337348 };
338349 }
339350
......@@ -350,7 +361,12 @@ fn Iterator(comptime ReaderType: type) type {
350361 /// loop iterates through one or more entries until it collects a all
351362 /// file attributes.
352363 pub fn next(self: *Self) !?File {
353 self.initFile();
364 if (self.unread_file_bytes > 0) {
365 // If file content was not consumed by caller
366 try self.reader.skipBytes(self.unread_file_bytes, .{});
367 self.unread_file_bytes = 0;
368 }
369 var file: File = self.newFile();
354370
355371 while (try self.readHeader()) |header| {
356372 const kind = header.kind();
......@@ -360,46 +376,52 @@ fn Iterator(comptime ReaderType: type) type {
360376 switch (kind) {
361377 // File types to retrun upstream
362378 .directory, .normal, .symbolic_link => {
363 self.file.kind = kind;
364 self.file.mode = try header.mode();
379 file.kind = switch (kind) {
380 .directory => .directory,
381 .normal => .file,
382 .symbolic_link => .sym_link,
383 else => unreachable,
384 };
385 file.mode = try header.mode();
365386
366387 // set file attributes if not already set by prefix/extended headers
367 if (self.file.size == 0) {
368 self.file.size = size;
388 if (file.size == 0) {
389 file.size = size;
369390 }
370 if (self.file.link_name.len == 0) {
371 self.file.link_name = try header.linkName(self.link_name_buffer);
391 if (file.link_name.len == 0) {
392 file.link_name = try header.linkName(self.link_name_buffer);
372393 }
373 if (self.file.name.len == 0) {
374 self.file.name = try header.fullName(self.file_name_buffer);
394 if (file.name.len == 0) {
395 file.name = try header.fullName(self.file_name_buffer);
375396 }
376397
377 self.padding = blockPadding(self.file.size);
378 return self.file;
398 self.padding = blockPadding(file.size);
399 self.unread_file_bytes = file.size;
400 return file;
379401 },
380402 // Prefix header types
381403 .gnu_long_name => {
382 self.file.name = try self.readString(@intCast(size), self.file_name_buffer);
404 file.name = try self.readString(@intCast(size), self.file_name_buffer);
383405 },
384406 .gnu_long_link => {
385 self.file.link_name = try self.readString(@intCast(size), self.link_name_buffer);
407 file.link_name = try self.readString(@intCast(size), self.link_name_buffer);
386408 },
387409 .extended_header => {
388410 // Use just attributes from last extended header.
389 self.initFile();
411 file = self.newFile();
390412
391413 var rdr = paxIterator(self.reader, @intCast(size));
392414 while (try rdr.next()) |attr| {
393415 switch (attr.kind) {
394416 .path => {
395 self.file.name = try attr.value(self.file_name_buffer);
417 file.name = try attr.value(self.file_name_buffer);
396418 },
397419 .linkpath => {
398 self.file.link_name = try attr.value(self.link_name_buffer);
420 file.link_name = try attr.value(self.link_name_buffer);
399421 },
400422 .size => {
401423 var buf: [pax_max_size_attr_len]u8 = undefined;
402 self.file.size = try std.fmt.parseInt(u64, try attr.value(&buf), 10);
424 file.size = try std.fmt.parseInt(u64, try attr.value(&buf), 10);
403425 },
404426 }
405427 }
......@@ -574,24 +596,23 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
574596 try dir.makePath(file_name);
575597 }
576598 },
577 .normal => {
599 .file => {
578600 if (file.size == 0 and file.name.len == 0) return;
579601 const file_name = stripComponents(file.name, options.strip_components);
580602 if (file_name.len == 0) return error.BadFileName;
581603
582604 if (createDirAndFile(dir, file_name)) |fs_file| {
583605 defer fs_file.close();
584 try file.write(fs_file);
606 try file.writeAll(fs_file);
585607 } else |err| {
586608 const d = options.diagnostics orelse return err;
587609 try d.errors.append(d.allocator, .{ .unable_to_create_file = .{
588610 .code = err,
589611 .file_name = try d.allocator.dupe(u8, file_name),
590612 } });
591 try file.skip();
592613 }
593614 },
594 .symbolic_link => {
615 .sym_link => {
595616 // The file system path of the symbolic link.
596617 const file_name = stripComponents(file.name, options.strip_components);
597618 if (file_name.len == 0) return error.BadFileName;
......@@ -607,7 +628,6 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
607628 } });
608629 };
609630 },
610 else => unreachable,
611631 }
612632 }
613633}
lib/std/tar/test.zig+12-10
......@@ -8,7 +8,7 @@ const Case = struct {
88 size: u64 = 0,
99 mode: u32 = 0,
1010 link_name: []const u8 = &[0]u8{},
11 kind: tar.Header.Kind = .normal,
11 kind: tar.FileKind = .file,
1212 truncated: bool = false, // when there is no file body, just header, usefull for huge files
1313 };
1414
......@@ -91,7 +91,7 @@ const cases = [_]Case{
9191 .{
9292 .name = "a/b",
9393 .size = 0,
94 .kind = .symbolic_link,
94 .kind = .sym_link,
9595 .mode = 0o777,
9696 .link_name = "123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100",
9797 },
......@@ -112,7 +112,7 @@ const cases = [_]Case{
112112 .{
113113 .name = "foo",
114114 .size = 999,
115 .kind = .normal,
115 .kind = .file,
116116 .mode = 0o640,
117117 },
118118 },
......@@ -153,7 +153,7 @@ const cases = [_]Case{
153153 .{
154154 .name = "P1050238.JPG.log",
155155 .size = 14,
156 .kind = .normal,
156 .kind = .file,
157157 .mode = 0o664,
158158 },
159159 },
......@@ -168,13 +168,13 @@ const cases = [_]Case{
168168 .{
169169 .name = "small.txt",
170170 .size = 5,
171 .kind = .normal,
171 .kind = .file,
172172 .mode = 0o644,
173173 },
174174 .{
175175 .name = "small2.txt",
176176 .size = 11,
177 .kind = .normal,
177 .kind = .file,
178178 .mode = 0o644,
179179 },
180180 },
......@@ -189,7 +189,7 @@ const cases = [_]Case{
189189 .{
190190 .name = "GNU2/GNU2/long-path-name",
191191 .link_name = "GNU4/GNU4/long-linkpath-name",
192 .kind = .symbolic_link,
192 .kind = .sym_link,
193193 },
194194 },
195195 },
......@@ -205,7 +205,7 @@ const cases = [_]Case{
205205 .{
206206 .name = "bar",
207207 .link_name = "PAX4/PAX4/long-linkpath-name",
208 .kind = .symbolic_link,
208 .kind = .sym_link,
209209 },
210210 },
211211 },
......@@ -369,11 +369,13 @@ test "run test cases" {
369369
370370 if (case.chksums.len > i) {
371371 var md5writer = Md5Writer{};
372 try actual.write(&md5writer);
372 try actual.writeAll(&md5writer);
373373 const chksum = md5writer.chksum();
374374 try testing.expectEqualStrings(case.chksums[i], &chksum);
375375 } else {
376 if (!expected.truncated) try actual.skip(); // skip file content
376 if (expected.truncated) {
377 iter.unread_file_bytes = 0;
378 }
377379 }
378380 }
379381 try testing.expectEqual(case.files.len, i);