authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-20 20:14:28-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-22 09:41:44-07:00
log1dcea220a4daf537c1a75835de45076fac429895
treee2fe0cd20e756c7ce67e3dd209b9f2e951b0d0db
parent2ac81c76e3d176d4ac0c568d195d5b979078a938

std.tar: update to new I/O API

9 files changed, 1016 insertions(+), 1033 deletions(-)

lib/std/Build/Fuzz/WebServer.zig+8-5
...@@ -522,7 +522,9 @@ fn serveSourcesTar(ws: *WebServer, request: *std.http.Server.Request) !void {...@@ -522,7 +522,9 @@ fn serveSourcesTar(ws: *WebServer, request: *std.http.Server.Request) !void {
522522
523 var cwd_cache: ?[]const u8 = null;523 var cwd_cache: ?[]const u8 = null;
524524
525 var archiver = std.tar.writer(response.writer());525 var adapter = response.writer().adaptToNewApi();
526 var archiver: std.tar.Writer = .{ .underlying_writer = &adapter.new_interface };
527 var read_buffer: [1024]u8 = undefined;
526528
527 for (deduped_paths) |joined_path| {529 for (deduped_paths) |joined_path| {
528 var file = joined_path.root_dir.handle.openFile(joined_path.sub_path, .{}) catch |err| {530 var file = joined_path.root_dir.handle.openFile(joined_path.sub_path, .{}) catch |err| {
...@@ -530,13 +532,14 @@ fn serveSourcesTar(ws: *WebServer, request: *std.http.Server.Request) !void {...@@ -530,13 +532,14 @@ fn serveSourcesTar(ws: *WebServer, request: *std.http.Server.Request) !void {
530 continue;532 continue;
531 };533 };
532 defer file.close();534 defer file.close();
533535 const stat = try file.stat();
536 var file_reader: std.fs.File.Reader = .initSize(file, &read_buffer, stat.size);
534 archiver.prefix = joined_path.root_dir.path orelse try memoizedCwd(arena, &cwd_cache);537 archiver.prefix = joined_path.root_dir.path orelse try memoizedCwd(arena, &cwd_cache);
535 try archiver.writeFile(joined_path.sub_path, file);538 try archiver.writeFile(joined_path.sub_path, &file_reader, stat.mtime);
536 }539 }
537540
538 // intentionally omitting the pointless trailer541 // intentionally not calling `archiver.finishPedantically`
539 //try archiver.finish();542 try adapter.new_interface.flush();
540 try response.end();543 try response.end();
541}544}
542545
lib/std/Io/Reader.zig+6
...@@ -179,6 +179,12 @@ pub fn streamExact(r: *Reader, w: *Writer, n: usize) StreamError!void {...@@ -179,6 +179,12 @@ pub fn streamExact(r: *Reader, w: *Writer, n: usize) StreamError!void {
179 while (remaining != 0) remaining -= try r.stream(w, .limited(remaining));179 while (remaining != 0) remaining -= try r.stream(w, .limited(remaining));
180}180}
181181
182/// "Pump" exactly `n` bytes from the reader to the writer.
183pub fn streamExact64(r: *Reader, w: *Writer, n: u64) StreamError!void {
184 var remaining = n;
185 while (remaining != 0) remaining -= try r.stream(w, .limited64(remaining));
186}
187
182/// "Pump" data from the reader to the writer, handling `error.EndOfStream` as188/// "Pump" data from the reader to the writer, handling `error.EndOfStream` as
183/// a success case.189/// a success case.
184///190///
lib/std/crypto/md5.zig+10-2
...@@ -54,12 +54,20 @@ pub const Md5 = struct {...@@ -54,12 +54,20 @@ pub const Md5 = struct {
54 };54 };
55 }55 }
5656
57 pub fn hash(b: []const u8, out: *[digest_length]u8, options: Options) void {57 pub fn hash(data: []const u8, out: *[digest_length]u8, options: Options) void {
58 var d = Md5.init(options);58 var d = Md5.init(options);
59 d.update(b);59 d.update(data);
60 d.final(out);60 d.final(out);
61 }61 }
6262
63 pub fn hashResult(data: []const u8) [digest_length]u8 {
64 var out: [digest_length]u8 = undefined;
65 var d = Md5.init(.{});
66 d.update(data);
67 d.final(&out);
68 return out;
69 }
70
63 pub fn update(d: *Self, b: []const u8) void {71 pub fn update(d: *Self, b: []const u8) void {
64 var off: usize = 0;72 var off: usize = 0;
6573
lib/std/tar.zig+283-329
...@@ -19,7 +19,7 @@ const std = @import("std");...@@ -19,7 +19,7 @@ const std = @import("std");
19const assert = std.debug.assert;19const assert = std.debug.assert;
20const testing = std.testing;20const testing = std.testing;
2121
22pub const writer = @import("tar/writer.zig").writer;22pub const Writer = @import("tar/Writer.zig");
2323
24/// Provide this to receive detailed error messages.24/// Provide this to receive detailed error messages.
25/// When this is provided, some errors which would otherwise be returned25/// When this is provided, some errors which would otherwise be returned
...@@ -293,28 +293,6 @@ fn nullStr(str: []const u8) []const u8 {...@@ -293,28 +293,6 @@ fn nullStr(str: []const u8) []const u8 {
293 return str;293 return str;
294}294}
295295
296/// Options for iterator.
297/// Buffers should be provided by the caller.
298pub const IteratorOptions = struct {
299 /// Use a buffer with length `std.fs.max_path_bytes` to match file system capabilities.
300 file_name_buffer: []u8,
301 /// Use a buffer with length `std.fs.max_path_bytes` to match file system capabilities.
302 link_name_buffer: []u8,
303 /// Collects error messages during unpacking
304 diagnostics: ?*Diagnostics = null,
305};
306
307/// Iterates over files in tar archive.
308/// `next` returns each file in tar archive.
309pub fn iterator(reader: anytype, options: IteratorOptions) Iterator(@TypeOf(reader)) {
310 return .{
311 .reader = reader,
312 .diagnostics = options.diagnostics,
313 .file_name_buffer = options.file_name_buffer,
314 .link_name_buffer = options.link_name_buffer,
315 };
316}
317
318/// Type of the file returned by iterator `next` method.296/// Type of the file returned by iterator `next` method.
319pub const FileKind = enum {297pub const FileKind = enum {
320 directory,298 directory,
...@@ -323,206 +301,192 @@ pub const FileKind = enum {...@@ -323,206 +301,192 @@ pub const FileKind = enum {
323};301};
324302
325/// Iterator over entries in the tar file represented by reader.303/// Iterator over entries in the tar file represented by reader.
326pub fn Iterator(comptime ReaderType: type) type {304pub const Iterator = struct {
327 return struct {305 reader: *std.Io.Reader,
328 reader: ReaderType,306 diagnostics: ?*Diagnostics = null,
329 diagnostics: ?*Diagnostics = null,
330
331 // buffers for heeader and file attributes
332 header_buffer: [Header.SIZE]u8 = undefined,
333 file_name_buffer: []u8,
334 link_name_buffer: []u8,
335
336 // bytes of padding to the end of the block
337 padding: usize = 0,
338 // not consumed bytes of file from last next iteration
339 unread_file_bytes: u64 = 0,
340
341 pub const File = struct {
342 name: []const u8, // name of file, symlink or directory
343 link_name: []const u8, // target name of symlink
344 size: u64 = 0, // size of the file in bytes
345 mode: u32 = 0,
346 kind: FileKind = .file,
347
348 unread_bytes: *u64,
349 parent_reader: ReaderType,
350
351 pub const Reader = std.io.GenericReader(File, ReaderType.Error, File.read);
352307
353 pub fn reader(self: File) Reader {308 // buffers for heeader and file attributes
354 return .{ .context = self };309 header_buffer: [Header.SIZE]u8 = undefined,
355 }310 file_name_buffer: []u8,
311 link_name_buffer: []u8,
356312
357 pub fn read(self: File, dest: []u8) ReaderType.Error!usize {313 // bytes of padding to the end of the block
358 const buf = dest[0..@min(dest.len, self.unread_bytes.*)];314 padding: usize = 0,
359 const n = try self.parent_reader.read(buf);315 // not consumed bytes of file from last next iteration
360 self.unread_bytes.* -= n;316 unread_file_bytes: u64 = 0,
361 return n;
362 }
363317
364 // Writes file content to writer.318 /// Options for iterator.
365 pub fn writeAll(self: File, out_writer: anytype) !void {319 /// Buffers should be provided by the caller.
366 var buffer: [4096]u8 = undefined;320 pub const Options = struct {
321 /// Use a buffer with length `std.fs.max_path_bytes` to match file system capabilities.
322 file_name_buffer: []u8,
323 /// Use a buffer with length `std.fs.max_path_bytes` to match file system capabilities.
324 link_name_buffer: []u8,
325 /// Collects error messages during unpacking
326 diagnostics: ?*Diagnostics = null,
327 };
367328
368 while (self.unread_bytes.* > 0) {329 /// Iterates over files in tar archive.
369 const buf = buffer[0..@min(buffer.len, self.unread_bytes.*)];330 /// `next` returns each file in tar archive.
370 try self.parent_reader.readNoEof(buf);331 pub fn init(reader: *std.Io.Reader, options: Options) Iterator {
371 try out_writer.writeAll(buf);332 return .{
372 self.unread_bytes.* -= buf.len;333 .reader = reader,
373 }334 .diagnostics = options.diagnostics,
374 }335 .file_name_buffer = options.file_name_buffer,
336 .link_name_buffer = options.link_name_buffer,
375 };337 };
338 }
376339
377 const Self = @This();340 pub const File = struct {
378341 name: []const u8, // name of file, symlink or directory
379 fn readHeader(self: *Self) !?Header {342 link_name: []const u8, // target name of symlink
380 if (self.padding > 0) {343 size: u64 = 0, // size of the file in bytes
381 try self.reader.skipBytes(self.padding, .{});344 mode: u32 = 0,
382 }345 kind: FileKind = .file,
383 const n = try self.reader.readAll(&self.header_buffer);346 };
384 if (n == 0) return null;
385 if (n < Header.SIZE) return error.UnexpectedEndOfStream;
386 const header = Header{ .bytes = self.header_buffer[0..Header.SIZE] };
387 if (try header.checkChksum() == 0) return null;
388 return header;
389 }
390347
391 fn readString(self: *Self, size: usize, buffer: []u8) ![]const u8 {348 fn readHeader(self: *Iterator) !?Header {
392 if (size > buffer.len) return error.TarInsufficientBuffer;349 if (self.padding > 0) {
393 const buf = buffer[0..size];350 try self.reader.discardAll(self.padding);
394 try self.reader.readNoEof(buf);
395 return nullStr(buf);
396 }351 }
352 const n = try self.reader.readSliceShort(&self.header_buffer);
353 if (n == 0) return null;
354 if (n < Header.SIZE) return error.UnexpectedEndOfStream;
355 const header = Header{ .bytes = self.header_buffer[0..Header.SIZE] };
356 if (try header.checkChksum() == 0) return null;
357 return header;
358 }
397359
398 fn newFile(self: *Self) File {360 fn readString(self: *Iterator, size: usize, buffer: []u8) ![]const u8 {
399 return .{361 if (size > buffer.len) return error.TarInsufficientBuffer;
400 .name = self.file_name_buffer[0..0],362 const buf = buffer[0..size];
401 .link_name = self.link_name_buffer[0..0],363 try self.reader.readSliceAll(buf);
402 .parent_reader = self.reader,364 return nullStr(buf);
403 .unread_bytes = &self.unread_file_bytes,365 }
404 };
405 }
406366
407 // Number of padding bytes in the last file block.367 fn newFile(self: *Iterator) File {
408 fn blockPadding(size: u64) usize {368 return .{
409 const block_rounded = std.mem.alignForward(u64, size, Header.SIZE); // size rounded to te block boundary369 .name = self.file_name_buffer[0..0],
410 return @intCast(block_rounded - size);370 .link_name = self.link_name_buffer[0..0],
411 }371 };
372 }
412373
413 /// Iterates through the tar archive as if it is a series of files.374 // Number of padding bytes in the last file block.
414 /// Internally, the tar format often uses entries (header with optional375 fn blockPadding(size: u64) usize {
415 /// content) to add meta data that describes the next file. These376 const block_rounded = std.mem.alignForward(u64, size, Header.SIZE); // size rounded to te block boundary
416 /// entries should not normally be visible to the outside. As such, this377 return @intCast(block_rounded - size);
417 /// loop iterates through one or more entries until it collects a all378 }
418 /// file attributes.
419 pub fn next(self: *Self) !?File {
420 if (self.unread_file_bytes > 0) {
421 // If file content was not consumed by caller
422 try self.reader.skipBytes(self.unread_file_bytes, .{});
423 self.unread_file_bytes = 0;
424 }
425 var file: File = self.newFile();
426
427 while (try self.readHeader()) |header| {
428 const kind = header.kind();
429 const size: u64 = try header.size();
430 self.padding = blockPadding(size);
431
432 switch (kind) {
433 // File types to return upstream
434 .directory, .normal, .symbolic_link => {
435 file.kind = switch (kind) {
436 .directory => .directory,
437 .normal => .file,
438 .symbolic_link => .sym_link,
439 else => unreachable,
440 };
441 file.mode = try header.mode();
442
443 // set file attributes if not already set by prefix/extended headers
444 if (file.size == 0) {
445 file.size = size;
446 }
447 if (file.link_name.len == 0) {
448 file.link_name = try header.linkName(self.link_name_buffer);
449 }
450 if (file.name.len == 0) {
451 file.name = try header.fullName(self.file_name_buffer);
452 }
453379
454 self.padding = blockPadding(file.size);380 /// Iterates through the tar archive as if it is a series of files.
455 self.unread_file_bytes = file.size;381 /// Internally, the tar format often uses entries (header with optional
456 return file;382 /// content) to add meta data that describes the next file. These
457 },383 /// entries should not normally be visible to the outside. As such, this
458 // Prefix header types384 /// loop iterates through one or more entries until it collects a all
459 .gnu_long_name => {385 /// file attributes.
460 file.name = try self.readString(@intCast(size), self.file_name_buffer);386 pub fn next(self: *Iterator) !?File {
461 },387 if (self.unread_file_bytes > 0) {
462 .gnu_long_link => {388 // If file content was not consumed by caller
463 file.link_name = try self.readString(@intCast(size), self.link_name_buffer);389 try self.reader.discardAll64(self.unread_file_bytes);
464 },390 self.unread_file_bytes = 0;
465 .extended_header => {391 }
466 // Use just attributes from last extended header.392 var file: File = self.newFile();
467 file = self.newFile();393
468394 while (try self.readHeader()) |header| {
469 var rdr = paxIterator(self.reader, @intCast(size));395 const kind = header.kind();
470 while (try rdr.next()) |attr| {396 const size: u64 = try header.size();
471 switch (attr.kind) {397 self.padding = blockPadding(size);
472 .path => {398
473 file.name = try attr.value(self.file_name_buffer);399 switch (kind) {
474 },400 // File types to return upstream
475 .linkpath => {401 .directory, .normal, .symbolic_link => {
476 file.link_name = try attr.value(self.link_name_buffer);402 file.kind = switch (kind) {
477 },403 .directory => .directory,
478 .size => {404 .normal => .file,
479 var buf: [pax_max_size_attr_len]u8 = undefined;405 .symbolic_link => .sym_link,
480 file.size = try std.fmt.parseInt(u64, try attr.value(&buf), 10);406 else => unreachable,
481 },407 };
482 }408 file.mode = try header.mode();
483 }409
484 },410 // set file attributes if not already set by prefix/extended headers
485 // Ignored header type411 if (file.size == 0) {
486 .global_extended_header => {412 file.size = size;
487 self.reader.skipBytes(size, .{}) catch return error.TarHeadersTooBig;413 }
488 },414 if (file.link_name.len == 0) {
489 // All other are unsupported header types415 file.link_name = try header.linkName(self.link_name_buffer);
490 else => {416 }
491 const d = self.diagnostics orelse return error.TarUnsupportedHeader;417 if (file.name.len == 0) {
492 try d.errors.append(d.allocator, .{ .unsupported_file_type = .{418 file.name = try header.fullName(self.file_name_buffer);
493 .file_name = try d.allocator.dupe(u8, header.name()),419 }
494 .file_type = kind,420
495 } });421 self.padding = blockPadding(file.size);
496 if (kind == .gnu_sparse) {422 self.unread_file_bytes = file.size;
497 try self.skipGnuSparseExtendedHeaders(header);423 return file;
424 },
425 // Prefix header types
426 .gnu_long_name => {
427 file.name = try self.readString(@intCast(size), self.file_name_buffer);
428 },
429 .gnu_long_link => {
430 file.link_name = try self.readString(@intCast(size), self.link_name_buffer);
431 },
432 .extended_header => {
433 // Use just attributes from last extended header.
434 file = self.newFile();
435
436 var rdr: PaxIterator = .{
437 .reader = self.reader,
438 .size = @intCast(size),
439 };
440 while (try rdr.next()) |attr| {
441 switch (attr.kind) {
442 .path => {
443 file.name = try attr.value(self.file_name_buffer);
444 },
445 .linkpath => {
446 file.link_name = try attr.value(self.link_name_buffer);
447 },
448 .size => {
449 var buf: [pax_max_size_attr_len]u8 = undefined;
450 file.size = try std.fmt.parseInt(u64, try attr.value(&buf), 10);
451 },
498 }452 }
499 self.reader.skipBytes(size, .{}) catch return error.TarHeadersTooBig;453 }
500 },454 },
501 }455 // Ignored header type
456 .global_extended_header => {
457 self.reader.discardAll64(size) catch return error.TarHeadersTooBig;
458 },
459 // All other are unsupported header types
460 else => {
461 const d = self.diagnostics orelse return error.TarUnsupportedHeader;
462 try d.errors.append(d.allocator, .{ .unsupported_file_type = .{
463 .file_name = try d.allocator.dupe(u8, header.name()),
464 .file_type = kind,
465 } });
466 if (kind == .gnu_sparse) {
467 try self.skipGnuSparseExtendedHeaders(header);
468 }
469 self.reader.discardAll64(size) catch return error.TarHeadersTooBig;
470 },
502 }471 }
503 return null;
504 }472 }
473 return null;
474 }
505475
506 fn skipGnuSparseExtendedHeaders(self: *Self, header: Header) !void {476 pub fn streamRemaining(it: *Iterator, file: File, w: *std.Io.Writer) std.Io.Reader.StreamError!void {
507 var is_extended = header.bytes[482] > 0;477 try it.reader.streamExact64(w, file.size);
508 while (is_extended) {478 it.unread_file_bytes = 0;
509 var buf: [Header.SIZE]u8 = undefined;479 }
510 const n = try self.reader.readAll(&buf);
511 if (n < Header.SIZE) return error.UnexpectedEndOfStream;
512 is_extended = buf[504] > 0;
513 }
514 }
515 };
516}
517480
518/// Pax attributes iterator.481 fn skipGnuSparseExtendedHeaders(self: *Iterator, header: Header) !void {
519/// Size is length of pax extended header in reader.482 var is_extended = header.bytes[482] > 0;
520fn paxIterator(reader: anytype, size: usize) PaxIterator(@TypeOf(reader)) {483 while (is_extended) {
521 return PaxIterator(@TypeOf(reader)){484 var buf: [Header.SIZE]u8 = undefined;
522 .reader = reader,485 try self.reader.readSliceAll(&buf);
523 .size = size,486 is_extended = buf[504] > 0;
524 };487 }
525}488 }
489};
526490
527const PaxAttributeKind = enum {491const PaxAttributeKind = enum {
528 path,492 path,
...@@ -533,108 +497,99 @@ const PaxAttributeKind = enum {...@@ -533,108 +497,99 @@ const PaxAttributeKind = enum {
533// maxInt(u64) has 20 chars, base 10 in practice we got 24 chars497// maxInt(u64) has 20 chars, base 10 in practice we got 24 chars
534const pax_max_size_attr_len = 64;498const pax_max_size_attr_len = 64;
535499
536fn PaxIterator(comptime ReaderType: type) type {500pub const PaxIterator = struct {
537 return struct {501 size: usize, // cumulative size of all pax attributes
538 size: usize, // cumulative size of all pax attributes502 reader: *std.Io.Reader,
539 reader: ReaderType,
540 // scratch buffer used for reading attribute length and keyword
541 scratch: [128]u8 = undefined,
542
543 const Self = @This();
544
545 const Attribute = struct {
546 kind: PaxAttributeKind,
547 len: usize, // length of the attribute value
548 reader: ReaderType, // reader positioned at value start
549
550 // Copies pax attribute value into destination buffer.
551 // Must be called with destination buffer of size at least Attribute.len.
552 pub fn value(self: Attribute, dst: []u8) ![]const u8 {
553 if (self.len > dst.len) return error.TarInsufficientBuffer;
554 // assert(self.len <= dst.len);
555 const buf = dst[0..self.len];
556 const n = try self.reader.readAll(buf);
557 if (n < self.len) return error.UnexpectedEndOfStream;
558 try validateAttributeEnding(self.reader);
559 if (hasNull(buf)) return error.PaxNullInValue;
560 return buf;
561 }
562 };
563503
564 // Iterates over pax attributes. Returns known only known attributes.504 const Self = @This();
565 // Caller has to call value in Attribute, to advance reader across value.
566 pub fn next(self: *Self) !?Attribute {
567 // Pax extended header consists of one or more attributes, each constructed as follows:
568 // "%d %s=%s\n", <length>, <keyword>, <value>
569 while (self.size > 0) {
570 const length_buf = try self.readUntil(' ');
571 const length = try std.fmt.parseInt(usize, length_buf, 10); // record length in bytes
572
573 const keyword = try self.readUntil('=');
574 if (hasNull(keyword)) return error.PaxNullInKeyword;
575
576 // calculate value_len
577 const value_start = length_buf.len + keyword.len + 2; // 2 separators
578 if (length < value_start + 1 or self.size < length) return error.UnexpectedEndOfStream;
579 const value_len = length - value_start - 1; // \n separator at end
580 self.size -= length;
581
582 const kind: PaxAttributeKind = if (eql(keyword, "path"))
583 .path
584 else if (eql(keyword, "linkpath"))
585 .linkpath
586 else if (eql(keyword, "size"))
587 .size
588 else {
589 try self.reader.skipBytes(value_len, .{});
590 try validateAttributeEnding(self.reader);
591 continue;
592 };
593 if (kind == .size and value_len > pax_max_size_attr_len) {
594 return error.PaxSizeAttrOverflow;
595 }
596 return Attribute{
597 .kind = kind,
598 .len = value_len,
599 .reader = self.reader,
600 };
601 }
602505
603 return null;506 const Attribute = struct {
507 kind: PaxAttributeKind,
508 len: usize, // length of the attribute value
509 reader: *std.Io.Reader, // reader positioned at value start
510
511 // Copies pax attribute value into destination buffer.
512 // Must be called with destination buffer of size at least Attribute.len.
513 pub fn value(self: Attribute, dst: []u8) ![]const u8 {
514 if (self.len > dst.len) return error.TarInsufficientBuffer;
515 // assert(self.len <= dst.len);
516 const buf = dst[0..self.len];
517 const n = try self.reader.readSliceShort(buf);
518 if (n < self.len) return error.UnexpectedEndOfStream;
519 try validateAttributeEnding(self.reader);
520 if (hasNull(buf)) return error.PaxNullInValue;
521 return buf;
604 }522 }
523 };
605524
606 fn readUntil(self: *Self, delimiter: u8) ![]const u8 {525 // Iterates over pax attributes. Returns known only known attributes.
607 var fbs = std.io.fixedBufferStream(&self.scratch);526 // Caller has to call value in Attribute, to advance reader across value.
608 try self.reader.streamUntilDelimiter(fbs.writer(), delimiter, null);527 pub fn next(self: *Self) !?Attribute {
609 return fbs.getWritten();528 // Pax extended header consists of one or more attributes, each constructed as follows:
529 // "%d %s=%s\n", <length>, <keyword>, <value>
530 while (self.size > 0) {
531 const length_buf = try self.reader.takeSentinel(' ');
532 const length = try std.fmt.parseInt(usize, length_buf, 10); // record length in bytes
533
534 const keyword = try self.reader.takeSentinel('=');
535 if (hasNull(keyword)) return error.PaxNullInKeyword;
536
537 // calculate value_len
538 const value_start = length_buf.len + keyword.len + 2; // 2 separators
539 if (length < value_start + 1 or self.size < length) return error.UnexpectedEndOfStream;
540 const value_len = length - value_start - 1; // \n separator at end
541 self.size -= length;
542
543 const kind: PaxAttributeKind = if (eql(keyword, "path"))
544 .path
545 else if (eql(keyword, "linkpath"))
546 .linkpath
547 else if (eql(keyword, "size"))
548 .size
549 else {
550 try self.reader.discardAll(value_len);
551 try validateAttributeEnding(self.reader);
552 continue;
553 };
554 if (kind == .size and value_len > pax_max_size_attr_len) {
555 return error.PaxSizeAttrOverflow;
556 }
557 return .{
558 .kind = kind,
559 .len = value_len,
560 .reader = self.reader,
561 };
610 }562 }
611563
612 fn eql(a: []const u8, b: []const u8) bool {564 return null;
613 return std.mem.eql(u8, a, b);565 }
614 }
615566
616 fn hasNull(str: []const u8) bool {567 fn eql(a: []const u8, b: []const u8) bool {
617 return (std.mem.indexOfScalar(u8, str, 0)) != null;568 return std.mem.eql(u8, a, b);
618 }569 }
619570
620 // Checks that each record ends with new line.571 fn hasNull(str: []const u8) bool {
621 fn validateAttributeEnding(reader: ReaderType) !void {572 return (std.mem.indexOfScalar(u8, str, 0)) != null;
622 if (try reader.readByte() != '\n') return error.PaxInvalidAttributeEnd;573 }
623 }574
624 };575 // Checks that each record ends with new line.
625}576 fn validateAttributeEnding(reader: *std.Io.Reader) !void {
577 if (try reader.takeByte() != '\n') return error.PaxInvalidAttributeEnd;
578 }
579};
626580
627/// Saves tar file content to the file systems.581/// Saves tar file content to the file systems.
628pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: PipeOptions) !void {582pub fn pipeToFileSystem(dir: std.fs.Dir, reader: *std.Io.Reader, options: PipeOptions) !void {
629 var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;583 var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
630 var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;584 var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
631 var iter = iterator(reader, .{585 var file_contents_buffer: [1024]u8 = undefined;
586 var it: Iterator = .init(reader, .{
632 .file_name_buffer = &file_name_buffer,587 .file_name_buffer = &file_name_buffer,
633 .link_name_buffer = &link_name_buffer,588 .link_name_buffer = &link_name_buffer,
634 .diagnostics = options.diagnostics,589 .diagnostics = options.diagnostics,
635 });590 });
636591
637 while (try iter.next()) |file| {592 while (try it.next()) |file| {
638 const file_name = stripComponents(file.name, options.strip_components);593 const file_name = stripComponents(file.name, options.strip_components);
639 if (file_name.len == 0 and file.kind != .directory) {594 if (file_name.len == 0 and file.kind != .directory) {
640 const d = options.diagnostics orelse return error.TarComponentsOutsideStrippedPrefix;595 const d = options.diagnostics orelse return error.TarComponentsOutsideStrippedPrefix;
...@@ -656,7 +611,9 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: PipeOptions)...@@ -656,7 +611,9 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: PipeOptions)
656 .file => {611 .file => {
657 if (createDirAndFile(dir, file_name, fileMode(file.mode, options))) |fs_file| {612 if (createDirAndFile(dir, file_name, fileMode(file.mode, options))) |fs_file| {
658 defer fs_file.close();613 defer fs_file.close();
659 try file.writeAll(fs_file);614 var file_writer = fs_file.writer(&file_contents_buffer);
615 try it.streamRemaining(file, &file_writer.interface);
616 try file_writer.interface.flush();
660 } else |err| {617 } else |err| {
661 const d = options.diagnostics orelse return err;618 const d = options.diagnostics orelse return err;
662 try d.errors.append(d.allocator, .{ .unable_to_create_file = .{619 try d.errors.append(d.allocator, .{ .unable_to_create_file = .{
...@@ -826,11 +783,14 @@ test PaxIterator {...@@ -826,11 +783,14 @@ test PaxIterator {
826 var buffer: [1024]u8 = undefined;783 var buffer: [1024]u8 = undefined;
827784
828 outer: for (cases) |case| {785 outer: for (cases) |case| {
829 var stream = std.io.fixedBufferStream(case.data);786 var reader: std.Io.Reader = .fixed(case.data);
830 var iter = paxIterator(stream.reader(), case.data.len);787 var it: PaxIterator = .{
788 .size = case.data.len,
789 .reader = &reader,
790 };
831791
832 var i: usize = 0;792 var i: usize = 0;
833 while (iter.next() catch |err| {793 while (it.next() catch |err| {
834 if (case.err) |e| {794 if (case.err) |e| {
835 try testing.expectEqual(e, err);795 try testing.expectEqual(e, err);
836 continue;796 continue;
...@@ -853,12 +813,6 @@ test PaxIterator {...@@ -853,12 +813,6 @@ test PaxIterator {
853 }813 }
854}814}
855815
856test {
857 _ = @import("tar/test.zig");
858 _ = @import("tar/writer.zig");
859 _ = Diagnostics;
860}
861
862test "header parse size" {816test "header parse size" {
863 const cases = [_]struct {817 const cases = [_]struct {
864 in: []const u8,818 in: []const u8,
...@@ -941,7 +895,7 @@ test "create file and symlink" {...@@ -941,7 +895,7 @@ test "create file and symlink" {
941 file.close();895 file.close();
942}896}
943897
944test iterator {898test Iterator {
945 // Example tar file is created from this tree structure:899 // Example tar file is created from this tree structure:
946 // $ tree example900 // $ tree example
947 // example901 // example
...@@ -962,19 +916,19 @@ test iterator {...@@ -962,19 +916,19 @@ test iterator {
962 // example/empty/916 // example/empty/
963917
964 const data = @embedFile("tar/testdata/example.tar");918 const data = @embedFile("tar/testdata/example.tar");
965 var fbs = std.io.fixedBufferStream(data);919 var reader: std.Io.Reader = .fixed(data);
966920
967 // User provided buffers to the iterator921 // User provided buffers to the iterator
968 var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;922 var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
969 var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;923 var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
970 // Create iterator924 // Create iterator
971 var iter = iterator(fbs.reader(), .{925 var it: Iterator = .init(&reader, .{
972 .file_name_buffer = &file_name_buffer,926 .file_name_buffer = &file_name_buffer,
973 .link_name_buffer = &link_name_buffer,927 .link_name_buffer = &link_name_buffer,
974 });928 });
975 // Iterate over files in example.tar929 // Iterate over files in example.tar
976 var file_no: usize = 0;930 var file_no: usize = 0;
977 while (try iter.next()) |file| : (file_no += 1) {931 while (try it.next()) |file| : (file_no += 1) {
978 switch (file.kind) {932 switch (file.kind) {
979 .directory => {933 .directory => {
980 switch (file_no) {934 switch (file_no) {
...@@ -987,10 +941,10 @@ test iterator {...@@ -987,10 +941,10 @@ test iterator {
987 },941 },
988 .file => {942 .file => {
989 try testing.expectEqualStrings("example/a/file", file.name);943 try testing.expectEqualStrings("example/a/file", file.name);
990 // Read file content
991 var buf: [16]u8 = undefined;944 var buf: [16]u8 = undefined;
992 const n = try file.reader().readAll(&buf);945 var w: std.Io.Writer = .fixed(&buf);
993 try testing.expectEqualStrings("content\n", buf[0..n]);946 try it.streamRemaining(file, &w);
947 try testing.expectEqualStrings("content\n", w.buffered());
994 },948 },
995 .sym_link => {949 .sym_link => {
996 try testing.expectEqualStrings("example/b/symlink", file.name);950 try testing.expectEqualStrings("example/b/symlink", file.name);
...@@ -1021,15 +975,14 @@ test pipeToFileSystem {...@@ -1021,15 +975,14 @@ test pipeToFileSystem {
1021 // example/empty/975 // example/empty/
1022976
1023 const data = @embedFile("tar/testdata/example.tar");977 const data = @embedFile("tar/testdata/example.tar");
1024 var fbs = std.io.fixedBufferStream(data);978 var reader: std.Io.Reader = .fixed(data);
1025 const reader = fbs.reader();
1026979
1027 var tmp = testing.tmpDir(.{ .no_follow = true });980 var tmp = testing.tmpDir(.{ .no_follow = true });
1028 defer tmp.cleanup();981 defer tmp.cleanup();
1029 const dir = tmp.dir;982 const dir = tmp.dir;
1030983
1031 // Save tar from `reader` to the file system `dir`984 // Save tar from reader to the file system `dir`
1032 pipeToFileSystem(dir, reader, .{985 pipeToFileSystem(dir, &reader, .{
1033 .mode_mode = .ignore,986 .mode_mode = .ignore,
1034 .strip_components = 1,987 .strip_components = 1,
1035 .exclude_empty_directories = true,988 .exclude_empty_directories = true,
...@@ -1053,8 +1006,7 @@ test pipeToFileSystem {...@@ -1053,8 +1006,7 @@ test pipeToFileSystem {
10531006
1054test "pipeToFileSystem root_dir" {1007test "pipeToFileSystem root_dir" {
1055 const data = @embedFile("tar/testdata/example.tar");1008 const data = @embedFile("tar/testdata/example.tar");
1056 var fbs = std.io.fixedBufferStream(data);1009 var reader: std.Io.Reader = .fixed(data);
1057 const reader = fbs.reader();
10581010
1059 // with strip_components = 11011 // with strip_components = 1
1060 {1012 {
...@@ -1063,7 +1015,7 @@ test "pipeToFileSystem root_dir" {...@@ -1063,7 +1015,7 @@ test "pipeToFileSystem root_dir" {
1063 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };1015 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1064 defer diagnostics.deinit();1016 defer diagnostics.deinit();
10651017
1066 pipeToFileSystem(tmp.dir, reader, .{1018 pipeToFileSystem(tmp.dir, &reader, .{
1067 .strip_components = 1,1019 .strip_components = 1,
1068 .diagnostics = &diagnostics,1020 .diagnostics = &diagnostics,
1069 }) catch |err| {1021 }) catch |err| {
...@@ -1079,13 +1031,13 @@ test "pipeToFileSystem root_dir" {...@@ -1079,13 +1031,13 @@ test "pipeToFileSystem root_dir" {
10791031
1080 // with strip_components = 01032 // with strip_components = 0
1081 {1033 {
1082 fbs.reset();1034 reader = .fixed(data);
1083 var tmp = testing.tmpDir(.{ .no_follow = true });1035 var tmp = testing.tmpDir(.{ .no_follow = true });
1084 defer tmp.cleanup();1036 defer tmp.cleanup();
1085 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };1037 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1086 defer diagnostics.deinit();1038 defer diagnostics.deinit();
10871039
1088 pipeToFileSystem(tmp.dir, reader, .{1040 pipeToFileSystem(tmp.dir, &reader, .{
1089 .strip_components = 0,1041 .strip_components = 0,
1090 .diagnostics = &diagnostics,1042 .diagnostics = &diagnostics,
1091 }) catch |err| {1043 }) catch |err| {
...@@ -1102,45 +1054,42 @@ test "pipeToFileSystem root_dir" {...@@ -1102,45 +1054,42 @@ test "pipeToFileSystem root_dir" {
11021054
1103test "findRoot with single file archive" {1055test "findRoot with single file archive" {
1104 const data = @embedFile("tar/testdata/22752.tar");1056 const data = @embedFile("tar/testdata/22752.tar");
1105 var fbs = std.io.fixedBufferStream(data);1057 var reader: std.Io.Reader = .fixed(data);
1106 const reader = fbs.reader();
11071058
1108 var tmp = testing.tmpDir(.{});1059 var tmp = testing.tmpDir(.{});
1109 defer tmp.cleanup();1060 defer tmp.cleanup();
11101061
1111 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };1062 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1112 defer diagnostics.deinit();1063 defer diagnostics.deinit();
1113 try pipeToFileSystem(tmp.dir, reader, .{ .diagnostics = &diagnostics });1064 try pipeToFileSystem(tmp.dir, &reader, .{ .diagnostics = &diagnostics });
11141065
1115 try testing.expectEqualStrings("", diagnostics.root_dir);1066 try testing.expectEqualStrings("", diagnostics.root_dir);
1116}1067}
11171068
1118test "findRoot without explicit root dir" {1069test "findRoot without explicit root dir" {
1119 const data = @embedFile("tar/testdata/19820.tar");1070 const data = @embedFile("tar/testdata/19820.tar");
1120 var fbs = std.io.fixedBufferStream(data);1071 var reader: std.Io.Reader = .fixed(data);
1121 const reader = fbs.reader();
11221072
1123 var tmp = testing.tmpDir(.{});1073 var tmp = testing.tmpDir(.{});
1124 defer tmp.cleanup();1074 defer tmp.cleanup();
11251075
1126 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };1076 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1127 defer diagnostics.deinit();1077 defer diagnostics.deinit();
1128 try pipeToFileSystem(tmp.dir, reader, .{ .diagnostics = &diagnostics });1078 try pipeToFileSystem(tmp.dir, &reader, .{ .diagnostics = &diagnostics });
11291079
1130 try testing.expectEqualStrings("root", diagnostics.root_dir);1080 try testing.expectEqualStrings("root", diagnostics.root_dir);
1131}1081}
11321082
1133test "pipeToFileSystem strip_components" {1083test "pipeToFileSystem strip_components" {
1134 const data = @embedFile("tar/testdata/example.tar");1084 const data = @embedFile("tar/testdata/example.tar");
1135 var fbs = std.io.fixedBufferStream(data);1085 var reader: std.Io.Reader = .fixed(data);
1136 const reader = fbs.reader();
11371086
1138 var tmp = testing.tmpDir(.{ .no_follow = true });1087 var tmp = testing.tmpDir(.{ .no_follow = true });
1139 defer tmp.cleanup();1088 defer tmp.cleanup();
1140 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };1089 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1141 defer diagnostics.deinit();1090 defer diagnostics.deinit();
11421091
1143 pipeToFileSystem(tmp.dir, reader, .{1092 pipeToFileSystem(tmp.dir, &reader, .{
1144 .strip_components = 3,1093 .strip_components = 3,
1145 .diagnostics = &diagnostics,1094 .diagnostics = &diagnostics,
1146 }) catch |err| {1095 }) catch |err| {
...@@ -1194,13 +1143,12 @@ test "executable bit" {...@@ -1194,13 +1143,12 @@ test "executable bit" {
1194 const data = @embedFile("tar/testdata/example.tar");1143 const data = @embedFile("tar/testdata/example.tar");
11951144
1196 for ([_]PipeOptions.ModeMode{ .ignore, .executable_bit_only }) |opt| {1145 for ([_]PipeOptions.ModeMode{ .ignore, .executable_bit_only }) |opt| {
1197 var fbs = std.io.fixedBufferStream(data);1146 var reader: std.Io.Reader = .fixed(data);
1198 const reader = fbs.reader();
11991147
1200 var tmp = testing.tmpDir(.{ .no_follow = true });1148 var tmp = testing.tmpDir(.{ .no_follow = true });
1201 //defer tmp.cleanup();1149 //defer tmp.cleanup();
12021150
1203 pipeToFileSystem(tmp.dir, reader, .{1151 pipeToFileSystem(tmp.dir, &reader, .{
1204 .strip_components = 1,1152 .strip_components = 1,
1205 .exclude_empty_directories = true,1153 .exclude_empty_directories = true,
1206 .mode_mode = opt,1154 .mode_mode = opt,
...@@ -1226,3 +1174,9 @@ test "executable bit" {...@@ -1226,3 +1174,9 @@ test "executable bit" {
1226 }1174 }
1227 }1175 }
1228}1176}
1177
1178test {
1179 _ = @import("tar/test.zig");
1180 _ = Writer;
1181 _ = Diagnostics;
1182}
lib/std/tar/Writer.zig created+498
...@@ -0,0 +1,498 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const testing = std.testing;
4const Writer = @This();
5
6const block_size = @sizeOf(Header);
7
8/// Options for writing file/dir/link. If left empty 0o664 is used for
9/// file mode and current time for mtime.
10pub const Options = struct {
11 /// File system permission mode.
12 mode: u32 = 0,
13 /// File system modification time.
14 mtime: u64 = 0,
15};
16
17underlying_writer: *std.Io.Writer,
18prefix: []const u8 = "",
19mtime_now: u64 = 0,
20
21const Error = error{
22 WriteFailed,
23 OctalOverflow,
24 NameTooLong,
25};
26
27/// Sets prefix for all other write* method paths.
28pub fn setRoot(w: *Writer, root: []const u8) Error!void {
29 if (root.len > 0)
30 try w.writeDir(root, .{});
31
32 w.prefix = root;
33}
34
35pub fn writeDir(w: *Writer, sub_path: []const u8, options: Options) Error!void {
36 try w.writeHeader(.directory, sub_path, "", 0, options);
37}
38
39pub const WriteFileError = std.Io.Writer.FileError || Error || std.fs.File.GetEndPosError;
40
41pub fn writeFile(
42 w: *Writer,
43 sub_path: []const u8,
44 file_reader: *std.fs.File.Reader,
45 stat_mtime: i128,
46) WriteFileError!void {
47 const size = try file_reader.getSize();
48 const mtime: u64 = @intCast(@divFloor(stat_mtime, std.time.ns_per_s));
49
50 var header: Header = .{};
51 try w.setPath(&header, sub_path);
52 try header.setSize(size);
53 try header.setMtime(mtime);
54 try header.updateChecksum();
55
56 try w.underlying_writer.writeAll(@ptrCast((&header)[0..1]));
57 _ = try w.underlying_writer.sendFileAll(file_reader, .unlimited);
58 try w.writePadding(size);
59}
60
61pub const WriteFileStreamError = Error || std.Io.Reader.StreamError;
62
63/// Writes file reading file content from `reader`. Reads exactly `size` bytes
64/// from `reader`, or returns `error.EndOfStream`.
65pub fn writeFileStream(
66 w: *Writer,
67 sub_path: []const u8,
68 size: u64,
69 reader: *std.Io.Reader,
70 options: Options,
71) WriteFileStreamError!void {
72 try w.writeHeader(.regular, sub_path, "", size, options);
73 try reader.streamExact64(w.underlying_writer, size);
74 try w.writePadding(size);
75}
76
77/// Writes file using bytes buffer `content` for size and file content.
78pub fn writeFileBytes(w: *Writer, sub_path: []const u8, content: []const u8, options: Options) Error!void {
79 try w.writeHeader(.regular, sub_path, "", content.len, options);
80 try w.underlying_writer.writeAll(content);
81 try w.writePadding(content.len);
82}
83
84pub fn writeLink(w: *Writer, sub_path: []const u8, link_name: []const u8, options: Options) Error!void {
85 try w.writeHeader(.symbolic_link, sub_path, link_name, 0, options);
86}
87
88/// Writes fs.Dir.WalkerEntry. Uses `mtime` from file system entry and
89/// default for entry mode .
90pub fn writeEntry(w: *Writer, entry: std.fs.Dir.Walker.Entry) Error!void {
91 switch (entry.kind) {
92 .directory => {
93 try w.writeDir(entry.path, .{ .mtime = try entryMtime(entry) });
94 },
95 .file => {
96 var file = try entry.dir.openFile(entry.basename, .{});
97 defer file.close();
98 const stat = try file.stat();
99 try w.writeFile(entry.path, file, stat);
100 },
101 .sym_link => {
102 var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
103 const link_name = try entry.dir.readLink(entry.basename, &link_name_buffer);
104 try w.writeLink(entry.path, link_name, .{ .mtime = try entryMtime(entry) });
105 },
106 else => {
107 return error.UnsupportedWalkerEntryKind;
108 },
109 }
110}
111
112fn writeHeader(
113 w: *Writer,
114 typeflag: Header.FileType,
115 sub_path: []const u8,
116 link_name: []const u8,
117 size: u64,
118 options: Options,
119) Error!void {
120 var header = Header.init(typeflag);
121 try w.setPath(&header, sub_path);
122 try header.setSize(size);
123 try header.setMtime(if (options.mtime != 0) options.mtime else w.mtimeNow());
124 if (options.mode != 0)
125 try header.setMode(options.mode);
126 if (typeflag == .symbolic_link)
127 header.setLinkname(link_name) catch |err| switch (err) {
128 error.NameTooLong => try w.writeExtendedHeader(.gnu_long_link, &.{link_name}),
129 else => return err,
130 };
131 try header.write(w.underlying_writer);
132}
133
134fn mtimeNow(w: *Writer) u64 {
135 if (w.mtime_now == 0)
136 w.mtime_now = @intCast(std.time.timestamp());
137 return w.mtime_now;
138}
139
140fn entryMtime(entry: std.fs.Dir.Walker.Entry) !u64 {
141 const stat = try entry.dir.statFile(entry.basename);
142 return @intCast(@divFloor(stat.mtime, std.time.ns_per_s));
143}
144
145/// Writes path in posix header, if don't fit (in name+prefix; 100+155
146/// bytes) writes it in gnu extended header.
147fn setPath(w: *Writer, header: *Header, sub_path: []const u8) Error!void {
148 header.setPath(w.prefix, sub_path) catch |err| switch (err) {
149 error.NameTooLong => {
150 // write extended header
151 const buffers: []const []const u8 = if (w.prefix.len == 0)
152 &.{sub_path}
153 else
154 &.{ w.prefix, "/", sub_path };
155 try w.writeExtendedHeader(.gnu_long_name, buffers);
156 },
157 else => return err,
158 };
159}
160
161/// Writes gnu extended header: gnu_long_name or gnu_long_link.
162fn writeExtendedHeader(w: *Writer, typeflag: Header.FileType, buffers: []const []const u8) Error!void {
163 var len: usize = 0;
164 for (buffers) |buf| len += buf.len;
165
166 var header: Header = .init(typeflag);
167 try header.setSize(len);
168 try header.write(w.underlying_writer);
169 for (buffers) |buf|
170 try w.underlying_writer.writeAll(buf);
171 try w.writePadding(len);
172}
173
174fn writePadding(w: *Writer, bytes: usize) std.Io.Writer.Error!void {
175 const pos = bytes % block_size;
176 if (pos == 0) return;
177 try w.underlying_writer.splatByteAll(0, block_size - pos);
178}
179
180/// According to the specification, tar should finish with two zero blocks, but
181/// "reasonable system must not assume that such a block exists when reading an
182/// archive". Therefore, the Zig standard library recommends to not call this
183/// function.
184pub fn finishPedantically(w: *Writer) std.Io.Writer.Error!void {
185 try w.underlying_writer.splatByteAll(0, block_size * 2);
186}
187
188/// A struct that is exactly 512 bytes and matches tar file format. This is
189/// intended to be used for outputting tar files; for parsing there is
190/// `std.tar.Header`.
191pub const Header = extern struct {
192 // This struct was originally copied from
193 // https://github.com/mattnite/tar/blob/main/src/main.zig which is MIT
194 // licensed.
195 //
196 // The name, linkname, magic, uname, and gname are null-terminated character
197 // strings. All other fields are zero-filled octal numbers in ASCII. Each
198 // numeric field of width w contains w minus 1 digits, and a null.
199 // Reference: https://www.gnu.org/software/tar/manual/html_node/Standard.html
200 // POSIX header: byte offset
201 name: [100]u8 = [_]u8{0} ** 100, // 0
202 mode: [7:0]u8 = default_mode.file, // 100
203 uid: [7:0]u8 = [_:0]u8{0} ** 7, // unused 108
204 gid: [7:0]u8 = [_:0]u8{0} ** 7, // unused 116
205 size: [11:0]u8 = [_:0]u8{'0'} ** 11, // 124
206 mtime: [11:0]u8 = [_:0]u8{'0'} ** 11, // 136
207 checksum: [7:0]u8 = [_:0]u8{' '} ** 7, // 148
208 typeflag: FileType = .regular, // 156
209 linkname: [100]u8 = [_]u8{0} ** 100, // 157
210 magic: [6]u8 = [_]u8{ 'u', 's', 't', 'a', 'r', 0 }, // 257
211 version: [2]u8 = [_]u8{ '0', '0' }, // 263
212 uname: [32]u8 = [_]u8{0} ** 32, // unused 265
213 gname: [32]u8 = [_]u8{0} ** 32, // unused 297
214 devmajor: [7:0]u8 = [_:0]u8{0} ** 7, // unused 329
215 devminor: [7:0]u8 = [_:0]u8{0} ** 7, // unused 337
216 prefix: [155]u8 = [_]u8{0} ** 155, // 345
217 pad: [12]u8 = [_]u8{0} ** 12, // unused 500
218
219 pub const FileType = enum(u8) {
220 regular = '0',
221 symbolic_link = '2',
222 directory = '5',
223 gnu_long_name = 'L',
224 gnu_long_link = 'K',
225 };
226
227 const default_mode = struct {
228 const file = [_:0]u8{ '0', '0', '0', '0', '6', '6', '4' }; // 0o664
229 const dir = [_:0]u8{ '0', '0', '0', '0', '7', '7', '5' }; // 0o775
230 const sym_link = [_:0]u8{ '0', '0', '0', '0', '7', '7', '7' }; // 0o777
231 const other = [_:0]u8{ '0', '0', '0', '0', '0', '0', '0' }; // 0o000
232 };
233
234 pub fn init(typeflag: FileType) Header {
235 return .{
236 .typeflag = typeflag,
237 .mode = switch (typeflag) {
238 .directory => default_mode.dir,
239 .symbolic_link => default_mode.sym_link,
240 .regular => default_mode.file,
241 else => default_mode.other,
242 },
243 };
244 }
245
246 pub fn setSize(w: *Header, size: u64) error{OctalOverflow}!void {
247 try octal(&w.size, size);
248 }
249
250 fn octal(buf: []u8, value: u64) error{OctalOverflow}!void {
251 var remainder: u64 = value;
252 var pos: usize = buf.len;
253 while (remainder > 0 and pos > 0) {
254 pos -= 1;
255 const c: u8 = @as(u8, @intCast(remainder % 8)) + '0';
256 buf[pos] = c;
257 remainder /= 8;
258 if (pos == 0 and remainder > 0) return error.OctalOverflow;
259 }
260 }
261
262 pub fn setMode(w: *Header, mode: u32) error{OctalOverflow}!void {
263 try octal(&w.mode, mode);
264 }
265
266 // Integer number of seconds since January 1, 1970, 00:00 Coordinated Universal Time.
267 // mtime == 0 will use current time
268 pub fn setMtime(w: *Header, mtime: u64) error{OctalOverflow}!void {
269 try octal(&w.mtime, mtime);
270 }
271
272 pub fn updateChecksum(w: *Header) !void {
273 var checksum: usize = ' '; // other 7 w.checksum bytes are initialized to ' '
274 for (std.mem.asBytes(w)) |val|
275 checksum += val;
276 try octal(&w.checksum, checksum);
277 }
278
279 pub fn write(h: *Header, bw: *std.Io.Writer) error{ OctalOverflow, WriteFailed }!void {
280 try h.updateChecksum();
281 try bw.writeAll(std.mem.asBytes(h));
282 }
283
284 pub fn setLinkname(w: *Header, link: []const u8) !void {
285 if (link.len > w.linkname.len) return error.NameTooLong;
286 @memcpy(w.linkname[0..link.len], link);
287 }
288
289 pub fn setPath(w: *Header, prefix: []const u8, sub_path: []const u8) !void {
290 const max_prefix = w.prefix.len;
291 const max_name = w.name.len;
292 const sep = std.fs.path.sep_posix;
293
294 if (prefix.len + sub_path.len > max_name + max_prefix or prefix.len > max_prefix)
295 return error.NameTooLong;
296
297 // both fit into name
298 if (prefix.len > 0 and prefix.len + sub_path.len < max_name) {
299 @memcpy(w.name[0..prefix.len], prefix);
300 w.name[prefix.len] = sep;
301 @memcpy(w.name[prefix.len + 1 ..][0..sub_path.len], sub_path);
302 return;
303 }
304
305 // sub_path fits into name
306 // there is no prefix or prefix fits into prefix
307 if (sub_path.len <= max_name) {
308 @memcpy(w.name[0..sub_path.len], sub_path);
309 @memcpy(w.prefix[0..prefix.len], prefix);
310 return;
311 }
312
313 if (prefix.len > 0) {
314 @memcpy(w.prefix[0..prefix.len], prefix);
315 w.prefix[prefix.len] = sep;
316 }
317 const prefix_pos = if (prefix.len > 0) prefix.len + 1 else 0;
318
319 // add as much to prefix as you can, must split at /
320 const prefix_remaining = max_prefix - prefix_pos;
321 if (std.mem.lastIndexOf(u8, sub_path[0..@min(prefix_remaining, sub_path.len)], &.{'/'})) |sep_pos| {
322 @memcpy(w.prefix[prefix_pos..][0..sep_pos], sub_path[0..sep_pos]);
323 if ((sub_path.len - sep_pos - 1) > max_name) return error.NameTooLong;
324 @memcpy(w.name[0..][0 .. sub_path.len - sep_pos - 1], sub_path[sep_pos + 1 ..]);
325 return;
326 }
327
328 return error.NameTooLong;
329 }
330
331 comptime {
332 assert(@sizeOf(Header) == 512);
333 }
334
335 test "setPath" {
336 const cases = [_]struct {
337 in: []const []const u8,
338 out: []const []const u8,
339 }{
340 .{
341 .in = &.{ "", "123456789" },
342 .out = &.{ "", "123456789" },
343 },
344 // can fit into name
345 .{
346 .in = &.{ "prefix", "sub_path" },
347 .out = &.{ "", "prefix/sub_path" },
348 },
349 // no more both fits into name
350 .{
351 .in = &.{ "prefix", "0123456789/" ** 8 ++ "basename" },
352 .out = &.{ "prefix", "0123456789/" ** 8 ++ "basename" },
353 },
354 // put as much as you can into prefix the rest goes into name
355 .{
356 .in = &.{ "prefix", "0123456789/" ** 10 ++ "basename" },
357 .out = &.{ "prefix/" ++ "0123456789/" ** 9 ++ "0123456789", "basename" },
358 },
359
360 .{
361 .in = &.{ "prefix", "0123456789/" ** 15 ++ "basename" },
362 .out = &.{ "prefix/" ++ "0123456789/" ** 12 ++ "0123456789", "0123456789/0123456789/basename" },
363 },
364 .{
365 .in = &.{ "prefix", "0123456789/" ** 21 ++ "basename" },
366 .out = &.{ "prefix/" ++ "0123456789/" ** 12 ++ "0123456789", "0123456789/" ** 8 ++ "basename" },
367 },
368 .{
369 .in = &.{ "", "012345678/" ** 10 ++ "foo" },
370 .out = &.{ "012345678/" ** 9 ++ "012345678", "foo" },
371 },
372 };
373
374 for (cases) |case| {
375 var header = Header.init(.regular);
376 try header.setPath(case.in[0], case.in[1]);
377 try testing.expectEqualStrings(case.out[0], str(&header.prefix));
378 try testing.expectEqualStrings(case.out[1], str(&header.name));
379 }
380
381 const error_cases = [_]struct {
382 in: []const []const u8,
383 }{
384 // basename can't fit into name (106 characters)
385 .{ .in = &.{ "zig", "test/cases/compile_errors/regression_test_2980_base_type_u32_is_not_type_checked_properly_when_assigning_a_value_within_a_struct.zig" } },
386 // cant fit into 255 + sep
387 .{ .in = &.{ "prefix", "0123456789/" ** 22 ++ "basename" } },
388 // can fit but sub_path can't be split (there is no separator)
389 .{ .in = &.{ "prefix", "0123456789" ** 10 ++ "a" } },
390 .{ .in = &.{ "prefix", "0123456789" ** 14 ++ "basename" } },
391 };
392
393 for (error_cases) |case| {
394 var header = Header.init(.regular);
395 try testing.expectError(
396 error.NameTooLong,
397 header.setPath(case.in[0], case.in[1]),
398 );
399 }
400 }
401
402 // Breaks string on first null character.
403 fn str(s: []const u8) []const u8 {
404 for (s, 0..) |c, i| {
405 if (c == 0) return s[0..i];
406 }
407 return s;
408 }
409};
410
411test {
412 _ = Header;
413}
414
415test "write files" {
416 const files = [_]struct {
417 path: []const u8,
418 content: []const u8,
419 }{
420 .{ .path = "foo", .content = "bar" },
421 .{ .path = "a12345678/" ** 10 ++ "foo", .content = "a" ** 511 },
422 .{ .path = "b12345678/" ** 24 ++ "foo", .content = "b" ** 512 },
423 .{ .path = "c12345678/" ** 25 ++ "foo", .content = "c" ** 513 },
424 .{ .path = "d12345678/" ** 51 ++ "foo", .content = "d" ** 1025 },
425 .{ .path = "e123456789" ** 11, .content = "e" },
426 };
427
428 var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
429 var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
430
431 // with root
432 {
433 const root = "root";
434
435 var output: std.Io.Writer.Allocating = .init(testing.allocator);
436 var w: Writer = .{ .underlying_writer = &output.writer };
437 defer output.deinit();
438 try w.setRoot(root);
439 for (files) |file|
440 try w.writeFileBytes(file.path, file.content, .{});
441
442 var input: std.Io.Reader = .fixed(output.getWritten());
443 var it: std.tar.Iterator = .init(&input, .{
444 .file_name_buffer = &file_name_buffer,
445 .link_name_buffer = &link_name_buffer,
446 });
447
448 // first entry is directory with prefix
449 {
450 const actual = (try it.next()).?;
451 try testing.expectEqualStrings(root, actual.name);
452 try testing.expectEqual(std.tar.FileKind.directory, actual.kind);
453 }
454
455 var i: usize = 0;
456 while (try it.next()) |actual| {
457 defer i += 1;
458 const expected = files[i];
459 try testing.expectEqualStrings(root, actual.name[0..root.len]);
460 try testing.expectEqual('/', actual.name[root.len..][0]);
461 try testing.expectEqualStrings(expected.path, actual.name[root.len + 1 ..]);
462
463 var content: std.Io.Writer.Allocating = .init(testing.allocator);
464 defer content.deinit();
465 try it.streamRemaining(actual, &content.writer);
466 try testing.expectEqualSlices(u8, expected.content, content.getWritten());
467 }
468 }
469 // without root
470 {
471 var output: std.Io.Writer.Allocating = .init(testing.allocator);
472 var w: Writer = .{ .underlying_writer = &output.writer };
473 defer output.deinit();
474 for (files) |file| {
475 var content: std.Io.Reader = .fixed(file.content);
476 try w.writeFileStream(file.path, file.content.len, &content, .{});
477 }
478
479 var input: std.Io.Reader = .fixed(output.getWritten());
480 var it: std.tar.Iterator = .init(&input, .{
481 .file_name_buffer = &file_name_buffer,
482 .link_name_buffer = &link_name_buffer,
483 });
484
485 var i: usize = 0;
486 while (try it.next()) |actual| {
487 defer i += 1;
488 const expected = files[i];
489 try testing.expectEqualStrings(expected.path, actual.name);
490
491 var content: std.Io.Writer.Allocating = .init(testing.allocator);
492 defer content.deinit();
493 try it.streamRemaining(actual, &content.writer);
494 try testing.expectEqualSlices(u8, expected.content, content.getWritten());
495 }
496 try w.finishPedantically();
497 }
498}
lib/std/tar/test.zig+173-177
...@@ -18,31 +18,72 @@ const Case = struct {...@@ -18,31 +18,72 @@ const Case = struct {
18 err: ?anyerror = null, // parsing should fail with this error18 err: ?anyerror = null, // parsing should fail with this error
19};19};
2020
21const cases = [_]Case{21const gnu_case: Case = .{
22 .{22 .data = @embedFile("testdata/gnu.tar"),
23 .data = @embedFile("testdata/gnu.tar"),23 .files = &[_]Case.File{
24 .files = &[_]Case.File{24 .{
25 .{25 .name = "small.txt",
26 .name = "small.txt",26 .size = 5,
27 .size = 5,27 .mode = 0o640,
28 .mode = 0o640,
29 },
30 .{
31 .name = "small2.txt",
32 .size = 11,
33 .mode = 0o640,
34 },
35 },28 },
36 .chksums = &[_][]const u8{29 .{
37 "e38b27eaccb4391bdec553a7f3ae6b2f",30 .name = "small2.txt",
38 "c65bd2e50a56a2138bf1716f2fd56fe9",31 .size = 11,
32 .mode = 0o640,
33 },
34 },
35 .chksums = &[_][]const u8{
36 "e38b27eaccb4391bdec553a7f3ae6b2f",
37 "c65bd2e50a56a2138bf1716f2fd56fe9",
38 },
39};
40
41const gnu_multi_headers_case: Case = .{
42 .data = @embedFile("testdata/gnu-multi-hdrs.tar"),
43 .files = &[_]Case.File{
44 .{
45 .name = "GNU2/GNU2/long-path-name",
46 .link_name = "GNU4/GNU4/long-linkpath-name",
47 .kind = .sym_link,
39 },48 },
40 },49 },
41 .{50};
51
52const trailing_slash_case: Case = .{
53 .data = @embedFile("testdata/trailing-slash.tar"),
54 .files = &[_]Case.File{
55 .{
56 .name = "123456789/" ** 30,
57 .kind = .directory,
58 },
59 },
60};
61
62const writer_big_long_case: Case = .{
63 // Size in gnu extended format, and name in pax attribute.
64 .data = @embedFile("testdata/writer-big-long.tar"),
65 .files = &[_]Case.File{
66 .{
67 .name = "longname/" ** 15 ++ "16gig.txt",
68 .size = 16 * 1024 * 1024 * 1024,
69 .mode = 0o644,
70 .truncated = true,
71 },
72 },
73};
74
75const fuzz1_case: Case = .{
76 .data = @embedFile("testdata/fuzz1.tar"),
77 .err = error.TarInsufficientBuffer,
78};
79
80test "run test cases" {
81 try testCase(gnu_case);
82 try testCase(.{
42 .data = @embedFile("testdata/sparse-formats.tar"),83 .data = @embedFile("testdata/sparse-formats.tar"),
43 .err = error.TarUnsupportedHeader,84 .err = error.TarUnsupportedHeader,
44 },85 });
45 .{86 try testCase(.{
46 .data = @embedFile("testdata/star.tar"),87 .data = @embedFile("testdata/star.tar"),
47 .files = &[_]Case.File{88 .files = &[_]Case.File{
48 .{89 .{
...@@ -60,8 +101,8 @@ const cases = [_]Case{...@@ -60,8 +101,8 @@ const cases = [_]Case{
60 "e38b27eaccb4391bdec553a7f3ae6b2f",101 "e38b27eaccb4391bdec553a7f3ae6b2f",
61 "c65bd2e50a56a2138bf1716f2fd56fe9",102 "c65bd2e50a56a2138bf1716f2fd56fe9",
62 },103 },
63 },104 });
64 .{105 try testCase(.{
65 .data = @embedFile("testdata/v7.tar"),106 .data = @embedFile("testdata/v7.tar"),
66 .files = &[_]Case.File{107 .files = &[_]Case.File{
67 .{108 .{
...@@ -79,8 +120,8 @@ const cases = [_]Case{...@@ -79,8 +120,8 @@ const cases = [_]Case{
79 "e38b27eaccb4391bdec553a7f3ae6b2f",120 "e38b27eaccb4391bdec553a7f3ae6b2f",
80 "c65bd2e50a56a2138bf1716f2fd56fe9",121 "c65bd2e50a56a2138bf1716f2fd56fe9",
81 },122 },
82 },123 });
83 .{124 try testCase(.{
84 .data = @embedFile("testdata/pax.tar"),125 .data = @embedFile("testdata/pax.tar"),
85 .files = &[_]Case.File{126 .files = &[_]Case.File{
86 .{127 .{
...@@ -99,13 +140,13 @@ const cases = [_]Case{...@@ -99,13 +140,13 @@ const cases = [_]Case{
99 .chksums = &[_][]const u8{140 .chksums = &[_][]const u8{
100 "3c382e8f5b6631aa2db52643912ffd4a",141 "3c382e8f5b6631aa2db52643912ffd4a",
101 },142 },
102 },143 });
103 .{144 try testCase(.{
104 // pax attribute don't end with \n145 // pax attribute don't end with \n
105 .data = @embedFile("testdata/pax-bad-hdr-file.tar"),146 .data = @embedFile("testdata/pax-bad-hdr-file.tar"),
106 .err = error.PaxInvalidAttributeEnd,147 .err = error.PaxInvalidAttributeEnd,
107 },148 });
108 .{149 try testCase(.{
109 // size is in pax attribute150 // size is in pax attribute
110 .data = @embedFile("testdata/pax-pos-size-file.tar"),151 .data = @embedFile("testdata/pax-pos-size-file.tar"),
111 .files = &[_]Case.File{152 .files = &[_]Case.File{
...@@ -119,8 +160,8 @@ const cases = [_]Case{...@@ -119,8 +160,8 @@ const cases = [_]Case{
119 .chksums = &[_][]const u8{160 .chksums = &[_][]const u8{
120 "0afb597b283fe61b5d4879669a350556",161 "0afb597b283fe61b5d4879669a350556",
121 },162 },
122 },163 });
123 .{164 try testCase(.{
124 // has pax records which we are not interested in165 // has pax records which we are not interested in
125 .data = @embedFile("testdata/pax-records.tar"),166 .data = @embedFile("testdata/pax-records.tar"),
126 .files = &[_]Case.File{167 .files = &[_]Case.File{
...@@ -128,8 +169,8 @@ const cases = [_]Case{...@@ -128,8 +169,8 @@ const cases = [_]Case{
128 .name = "file",169 .name = "file",
129 },170 },
130 },171 },
131 },172 });
132 .{173 try testCase(.{
133 // has global records which we are ignoring174 // has global records which we are ignoring
134 .data = @embedFile("testdata/pax-global-records.tar"),175 .data = @embedFile("testdata/pax-global-records.tar"),
135 .files = &[_]Case.File{176 .files = &[_]Case.File{
...@@ -146,8 +187,8 @@ const cases = [_]Case{...@@ -146,8 +187,8 @@ const cases = [_]Case{
146 .name = "file4",187 .name = "file4",
147 },188 },
148 },189 },
149 },190 });
150 .{191 try testCase(.{
151 .data = @embedFile("testdata/nil-uid.tar"),192 .data = @embedFile("testdata/nil-uid.tar"),
152 .files = &[_]Case.File{193 .files = &[_]Case.File{
153 .{194 .{
...@@ -160,8 +201,8 @@ const cases = [_]Case{...@@ -160,8 +201,8 @@ const cases = [_]Case{
160 .chksums = &[_][]const u8{201 .chksums = &[_][]const u8{
161 "08d504674115e77a67244beac19668f5",202 "08d504674115e77a67244beac19668f5",
162 },203 },
163 },204 });
164 .{205 try testCase(.{
165 // has xattrs and pax records which we are ignoring206 // has xattrs and pax records which we are ignoring
166 .data = @embedFile("testdata/xattrs.tar"),207 .data = @embedFile("testdata/xattrs.tar"),
167 .files = &[_]Case.File{208 .files = &[_]Case.File{
...@@ -182,23 +223,14 @@ const cases = [_]Case{...@@ -182,23 +223,14 @@ const cases = [_]Case{
182 "e38b27eaccb4391bdec553a7f3ae6b2f",223 "e38b27eaccb4391bdec553a7f3ae6b2f",
183 "c65bd2e50a56a2138bf1716f2fd56fe9",224 "c65bd2e50a56a2138bf1716f2fd56fe9",
184 },225 },
185 },226 });
186 .{227 try testCase(gnu_multi_headers_case);
187 .data = @embedFile("testdata/gnu-multi-hdrs.tar"),228 try testCase(.{
188 .files = &[_]Case.File{
189 .{
190 .name = "GNU2/GNU2/long-path-name",
191 .link_name = "GNU4/GNU4/long-linkpath-name",
192 .kind = .sym_link,
193 },
194 },
195 },
196 .{
197 // has gnu type D (directory) and S (sparse) blocks229 // has gnu type D (directory) and S (sparse) blocks
198 .data = @embedFile("testdata/gnu-incremental.tar"),230 .data = @embedFile("testdata/gnu-incremental.tar"),
199 .err = error.TarUnsupportedHeader,231 .err = error.TarUnsupportedHeader,
200 },232 });
201 .{233 try testCase(.{
202 // should use values only from last pax header234 // should use values only from last pax header
203 .data = @embedFile("testdata/pax-multi-hdrs.tar"),235 .data = @embedFile("testdata/pax-multi-hdrs.tar"),
204 .files = &[_]Case.File{236 .files = &[_]Case.File{
...@@ -208,8 +240,8 @@ const cases = [_]Case{...@@ -208,8 +240,8 @@ const cases = [_]Case{
208 .kind = .sym_link,240 .kind = .sym_link,
209 },241 },
210 },242 },
211 },243 });
212 .{244 try testCase(.{
213 .data = @embedFile("testdata/gnu-long-nul.tar"),245 .data = @embedFile("testdata/gnu-long-nul.tar"),
214 .files = &[_]Case.File{246 .files = &[_]Case.File{
215 .{247 .{
...@@ -217,8 +249,8 @@ const cases = [_]Case{...@@ -217,8 +249,8 @@ const cases = [_]Case{
217 .mode = 0o644,249 .mode = 0o644,
218 },250 },
219 },251 },
220 },252 });
221 .{253 try testCase(.{
222 .data = @embedFile("testdata/gnu-utf8.tar"),254 .data = @embedFile("testdata/gnu-utf8.tar"),
223 .files = &[_]Case.File{255 .files = &[_]Case.File{
224 .{256 .{
...@@ -226,8 +258,8 @@ const cases = [_]Case{...@@ -226,8 +258,8 @@ const cases = [_]Case{
226 .mode = 0o644,258 .mode = 0o644,
227 },259 },
228 },260 },
229 },261 });
230 .{262 try testCase(.{
231 .data = @embedFile("testdata/gnu-not-utf8.tar"),263 .data = @embedFile("testdata/gnu-not-utf8.tar"),
232 .files = &[_]Case.File{264 .files = &[_]Case.File{
233 .{265 .{
...@@ -235,33 +267,33 @@ const cases = [_]Case{...@@ -235,33 +267,33 @@ const cases = [_]Case{
235 .mode = 0o644,267 .mode = 0o644,
236 },268 },
237 },269 },
238 },270 });
239 .{271 try testCase(.{
240 // null in pax key272 // null in pax key
241 .data = @embedFile("testdata/pax-nul-xattrs.tar"),273 .data = @embedFile("testdata/pax-nul-xattrs.tar"),
242 .err = error.PaxNullInKeyword,274 .err = error.PaxNullInKeyword,
243 },275 });
244 .{276 try testCase(.{
245 .data = @embedFile("testdata/pax-nul-path.tar"),277 .data = @embedFile("testdata/pax-nul-path.tar"),
246 .err = error.PaxNullInValue,278 .err = error.PaxNullInValue,
247 },279 });
248 .{280 try testCase(.{
249 .data = @embedFile("testdata/neg-size.tar"),281 .data = @embedFile("testdata/neg-size.tar"),
250 .err = error.TarHeader,282 .err = error.TarHeader,
251 },283 });
252 .{284 try testCase(.{
253 .data = @embedFile("testdata/issue10968.tar"),285 .data = @embedFile("testdata/issue10968.tar"),
254 .err = error.TarHeader,286 .err = error.TarHeader,
255 },287 });
256 .{288 try testCase(.{
257 .data = @embedFile("testdata/issue11169.tar"),289 .data = @embedFile("testdata/issue11169.tar"),
258 .err = error.TarHeader,290 .err = error.TarHeader,
259 },291 });
260 .{292 try testCase(.{
261 .data = @embedFile("testdata/issue12435.tar"),293 .data = @embedFile("testdata/issue12435.tar"),
262 .err = error.TarHeaderChksum,294 .err = error.TarHeaderChksum,
263 },295 });
264 .{296 try testCase(.{
265 // has magic with space at end instead of null297 // has magic with space at end instead of null
266 .data = @embedFile("testdata/invalid-go17.tar"),298 .data = @embedFile("testdata/invalid-go17.tar"),
267 .files = &[_]Case.File{299 .files = &[_]Case.File{
...@@ -269,8 +301,8 @@ const cases = [_]Case{...@@ -269,8 +301,8 @@ const cases = [_]Case{
269 .name = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/foo",301 .name = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/foo",
270 },302 },
271 },303 },
272 },304 });
273 .{305 try testCase(.{
274 .data = @embedFile("testdata/ustar-file-devs.tar"),306 .data = @embedFile("testdata/ustar-file-devs.tar"),
275 .files = &[_]Case.File{307 .files = &[_]Case.File{
276 .{308 .{
...@@ -278,17 +310,9 @@ const cases = [_]Case{...@@ -278,17 +310,9 @@ const cases = [_]Case{
278 .mode = 0o644,310 .mode = 0o644,
279 },311 },
280 },312 },
281 },313 });
282 .{314 try testCase(trailing_slash_case);
283 .data = @embedFile("testdata/trailing-slash.tar"),315 try testCase(.{
284 .files = &[_]Case.File{
285 .{
286 .name = "123456789/" ** 30,
287 .kind = .directory,
288 },
289 },
290 },
291 .{
292 // Has size in gnu extended format. To represent size bigger than 8 GB.316 // Has size in gnu extended format. To represent size bigger than 8 GB.
293 .data = @embedFile("testdata/writer-big.tar"),317 .data = @embedFile("testdata/writer-big.tar"),
294 .files = &[_]Case.File{318 .files = &[_]Case.File{
...@@ -299,120 +323,92 @@ const cases = [_]Case{...@@ -299,120 +323,92 @@ const cases = [_]Case{
299 .mode = 0o640,323 .mode = 0o640,
300 },324 },
301 },325 },
302 },326 });
303 .{327 try testCase(writer_big_long_case);
304 // Size in gnu extended format, and name in pax attribute.328 try testCase(fuzz1_case);
305 .data = @embedFile("testdata/writer-big-long.tar"),329 try testCase(.{
306 .files = &[_]Case.File{
307 .{
308 .name = "longname/" ** 15 ++ "16gig.txt",
309 .size = 16 * 1024 * 1024 * 1024,
310 .mode = 0o644,
311 .truncated = true,
312 },
313 },
314 },
315 .{
316 .data = @embedFile("testdata/fuzz1.tar"),
317 .err = error.TarInsufficientBuffer,
318 },
319 .{
320 .data = @embedFile("testdata/fuzz2.tar"),330 .data = @embedFile("testdata/fuzz2.tar"),
321 .err = error.PaxSizeAttrOverflow,331 .err = error.PaxSizeAttrOverflow,
322 },332 });
323};333}
324
325// used in test to calculate file chksum
326const Md5Writer = struct {
327 h: std.crypto.hash.Md5 = std.crypto.hash.Md5.init(.{}),
328
329 pub fn writeAll(self: *Md5Writer, buf: []const u8) !void {
330 self.h.update(buf);
331 }
332
333 pub fn writeByte(self: *Md5Writer, byte: u8) !void {
334 self.h.update(&[_]u8{byte});
335 }
336
337 pub fn chksum(self: *Md5Writer) [32]u8 {
338 var s = [_]u8{0} ** 16;
339 self.h.final(&s);
340 return std.fmt.bytesToHex(s, .lower);
341 }
342};
343334
344test "run test cases" {335fn testCase(case: Case) !void {
345 var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;336 var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
346 var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;337 var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
347338
348 for (cases) |case| {339 var br: std.io.Reader = .fixed(case.data);
349 var fsb = std.io.fixedBufferStream(case.data);340 var it: tar.Iterator = .init(&br, .{
350 var iter = tar.iterator(fsb.reader(), .{341 .file_name_buffer = &file_name_buffer,
351 .file_name_buffer = &file_name_buffer,342 .link_name_buffer = &link_name_buffer,
352 .link_name_buffer = &link_name_buffer,343 });
353 });344 var i: usize = 0;
354 var i: usize = 0;345 while (it.next() catch |err| {
355 while (iter.next() catch |err| {346 if (case.err) |e| {
356 if (case.err) |e| {347 try testing.expectEqual(e, err);
357 try testing.expectEqual(e, err);348 return;
358 continue;349 } else {
359 } else {350 return err;
360 return err;351 }
361 }352 }) |actual| : (i += 1) {
362 }) |actual| : (i += 1) {353 const expected = case.files[i];
363 const expected = case.files[i];354 try testing.expectEqualStrings(expected.name, actual.name);
364 try testing.expectEqualStrings(expected.name, actual.name);355 try testing.expectEqual(expected.size, actual.size);
365 try testing.expectEqual(expected.size, actual.size);356 try testing.expectEqual(expected.kind, actual.kind);
366 try testing.expectEqual(expected.kind, actual.kind);357 try testing.expectEqual(expected.mode, actual.mode);
367 try testing.expectEqual(expected.mode, actual.mode);358 try testing.expectEqualStrings(expected.link_name, actual.link_name);
368 try testing.expectEqualStrings(expected.link_name, actual.link_name);
369359
370 if (case.chksums.len > i) {360 if (case.chksums.len > i) {
371 var md5writer = Md5Writer{};361 var aw: std.Io.Writer.Allocating = .init(std.testing.allocator);
372 try actual.writeAll(&md5writer);362 defer aw.deinit();
373 const chksum = md5writer.chksum();363 try it.streamRemaining(actual, &aw.writer);
374 try testing.expectEqualStrings(case.chksums[i], &chksum);364 const chksum = std.fmt.bytesToHex(std.crypto.hash.Md5.hashResult(aw.getWritten()), .lower);
375 } else {365 try testing.expectEqualStrings(case.chksums[i], &chksum);
376 if (expected.truncated) {366 } else {
377 iter.unread_file_bytes = 0;367 if (expected.truncated) {
378 }368 it.unread_file_bytes = 0;
379 }369 }
380 }370 }
381 try testing.expectEqual(case.files.len, i);
382 }371 }
372 try testing.expectEqual(case.files.len, i);
383}373}
384374
385test "pax/gnu long names with small buffer" {375test "pax/gnu long names with small buffer" {
376 try testLongNameCase(gnu_multi_headers_case);
377 try testLongNameCase(trailing_slash_case);
378 try testLongNameCase(.{
379 .data = @embedFile("testdata/fuzz1.tar"),
380 .err = error.TarInsufficientBuffer,
381 });
382}
383
384fn testLongNameCase(case: Case) !void {
386 // should fail with insufficient buffer error385 // should fail with insufficient buffer error
387386
388 var min_file_name_buffer: [256]u8 = undefined;387 var min_file_name_buffer: [256]u8 = undefined;
389 var min_link_name_buffer: [100]u8 = undefined;388 var min_link_name_buffer: [100]u8 = undefined;
390 const long_name_cases = [_]Case{ cases[11], cases[25], cases[28] };
391389
392 for (long_name_cases) |case| {390 var br: std.io.Reader = .fixed(case.data);
393 var fsb = std.io.fixedBufferStream(case.data);391 var iter: tar.Iterator = .init(&br, .{
394 var iter = tar.iterator(fsb.reader(), .{392 .file_name_buffer = &min_file_name_buffer,
395 .file_name_buffer = &min_file_name_buffer,393 .link_name_buffer = &min_link_name_buffer,
396 .link_name_buffer = &min_link_name_buffer,394 });
397 });
398395
399 var iter_err: ?anyerror = null;396 var iter_err: ?anyerror = null;
400 while (iter.next() catch |err| brk: {397 while (iter.next() catch |err| brk: {
401 iter_err = err;398 iter_err = err;
402 break :brk null;399 break :brk null;
403 }) |_| {}400 }) |_| {}
404401
405 try testing.expect(iter_err != null);402 try testing.expect(iter_err != null);
406 try testing.expectEqual(error.TarInsufficientBuffer, iter_err.?);403 try testing.expectEqual(error.TarInsufficientBuffer, iter_err.?);
407 }
408}404}
409405
410test "insufficient buffer in Header name filed" {406test "insufficient buffer in Header name filed" {
411 var min_file_name_buffer: [9]u8 = undefined;407 var min_file_name_buffer: [9]u8 = undefined;
412 var min_link_name_buffer: [100]u8 = undefined;408 var min_link_name_buffer: [100]u8 = undefined;
413409
414 var fsb = std.io.fixedBufferStream(cases[0].data);410 var br: std.io.Reader = .fixed(gnu_case.data);
415 var iter = tar.iterator(fsb.reader(), .{411 var iter: tar.Iterator = .init(&br, .{
416 .file_name_buffer = &min_file_name_buffer,412 .file_name_buffer = &min_file_name_buffer,
417 .link_name_buffer = &min_link_name_buffer,413 .link_name_buffer = &min_link_name_buffer,
418 });414 });
...@@ -466,21 +462,21 @@ test "should not overwrite existing file" {...@@ -466,21 +462,21 @@ test "should not overwrite existing file" {
466 // This ensures that file is not overwritten.462 // This ensures that file is not overwritten.
467 //463 //
468 const data = @embedFile("testdata/overwrite_file.tar");464 const data = @embedFile("testdata/overwrite_file.tar");
469 var fsb = std.io.fixedBufferStream(data);465 var r: std.io.Reader = .fixed(data);
470466
471 // Unpack with strip_components = 1 should fail467 // Unpack with strip_components = 1 should fail
472 var root = std.testing.tmpDir(.{});468 var root = std.testing.tmpDir(.{});
473 defer root.cleanup();469 defer root.cleanup();
474 try testing.expectError(470 try testing.expectError(
475 error.PathAlreadyExists,471 error.PathAlreadyExists,
476 tar.pipeToFileSystem(root.dir, fsb.reader(), .{ .mode_mode = .ignore, .strip_components = 1 }),472 tar.pipeToFileSystem(root.dir, &r, .{ .mode_mode = .ignore, .strip_components = 1 }),
477 );473 );
478474
479 // Unpack with strip_components = 0 should pass475 // Unpack with strip_components = 0 should pass
480 fsb.reset();476 r = .fixed(data);
481 var root2 = std.testing.tmpDir(.{});477 var root2 = std.testing.tmpDir(.{});
482 defer root2.cleanup();478 defer root2.cleanup();
483 try tar.pipeToFileSystem(root2.dir, fsb.reader(), .{ .mode_mode = .ignore, .strip_components = 0 });479 try tar.pipeToFileSystem(root2.dir, &r, .{ .mode_mode = .ignore, .strip_components = 0 });
484}480}
485481
486test "case sensitivity" {482test "case sensitivity" {
...@@ -494,12 +490,12 @@ test "case sensitivity" {...@@ -494,12 +490,12 @@ test "case sensitivity" {
494 // 18089/alacritty/Darkermatrix.yml490 // 18089/alacritty/Darkermatrix.yml
495 //491 //
496 const data = @embedFile("testdata/18089.tar");492 const data = @embedFile("testdata/18089.tar");
497 var fsb = std.io.fixedBufferStream(data);493 var r: std.io.Reader = .fixed(data);
498494
499 var root = std.testing.tmpDir(.{});495 var root = std.testing.tmpDir(.{});
500 defer root.cleanup();496 defer root.cleanup();
501497
502 tar.pipeToFileSystem(root.dir, fsb.reader(), .{ .mode_mode = .ignore, .strip_components = 1 }) catch |err| {498 tar.pipeToFileSystem(root.dir, &r, .{ .mode_mode = .ignore, .strip_components = 1 }) catch |err| {
503 // on case insensitive fs we fail on overwrite existing file499 // on case insensitive fs we fail on overwrite existing file
504 try testing.expectEqual(error.PathAlreadyExists, err);500 try testing.expectEqual(error.PathAlreadyExists, err);
505 return;501 return;
lib/std/tar/writer.zig deleted-497
...@@ -1,497 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const testing = std.testing;
4
5/// Creates tar Writer which will write tar content to the `underlying_writer`.
6/// Use setRoot to nest all following entries under single root. If file don't
7/// fit into posix header (name+prefix: 100+155 bytes) gnu extented header will
8/// be used for long names. Options enables setting file premission mode and
9/// mtime. Default is to use current time for mtime and 0o664 for file mode.
10pub fn writer(underlying_writer: anytype) Writer(@TypeOf(underlying_writer)) {
11 return .{ .underlying_writer = underlying_writer };
12}
13
14pub fn Writer(comptime WriterType: type) type {
15 return struct {
16 const block_size = @sizeOf(Header);
17 const empty_block: [block_size]u8 = [_]u8{0} ** block_size;
18
19 /// Options for writing file/dir/link. If left empty 0o664 is used for
20 /// file mode and current time for mtime.
21 pub const Options = struct {
22 /// File system permission mode.
23 mode: u32 = 0,
24 /// File system modification time.
25 mtime: u64 = 0,
26 };
27 const Self = @This();
28
29 underlying_writer: WriterType,
30 prefix: []const u8 = "",
31 mtime_now: u64 = 0,
32
33 /// Sets prefix for all other write* method paths.
34 pub fn setRoot(self: *Self, root: []const u8) !void {
35 if (root.len > 0)
36 try self.writeDir(root, .{});
37
38 self.prefix = root;
39 }
40
41 /// Writes directory.
42 pub fn writeDir(self: *Self, sub_path: []const u8, opt: Options) !void {
43 try self.writeHeader(.directory, sub_path, "", 0, opt);
44 }
45
46 /// Writes file system file.
47 pub fn writeFile(self: *Self, sub_path: []const u8, file: std.fs.File) !void {
48 const stat = try file.stat();
49 const mtime: u64 = @intCast(@divFloor(stat.mtime, std.time.ns_per_s));
50
51 var header = Header{};
52 try self.setPath(&header, sub_path);
53 try header.setSize(stat.size);
54 try header.setMtime(mtime);
55 try header.write(self.underlying_writer);
56
57 try self.underlying_writer.writeFile(file);
58 try self.writePadding(stat.size);
59 }
60
61 /// Writes file reading file content from `reader`. Number of bytes in
62 /// reader must be equal to `size`.
63 pub fn writeFileStream(self: *Self, sub_path: []const u8, size: usize, reader: anytype, opt: Options) !void {
64 try self.writeHeader(.regular, sub_path, "", @intCast(size), opt);
65
66 var counting_reader = std.io.countingReader(reader);
67 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
68 try fifo.pump(counting_reader.reader(), self.underlying_writer);
69 if (counting_reader.bytes_read != size) return error.WrongReaderSize;
70 try self.writePadding(size);
71 }
72
73 /// Writes file using bytes buffer `content` for size and file content.
74 pub fn writeFileBytes(self: *Self, sub_path: []const u8, content: []const u8, opt: Options) !void {
75 try self.writeHeader(.regular, sub_path, "", @intCast(content.len), opt);
76 try self.underlying_writer.writeAll(content);
77 try self.writePadding(content.len);
78 }
79
80 /// Writes symlink.
81 pub fn writeLink(self: *Self, sub_path: []const u8, link_name: []const u8, opt: Options) !void {
82 try self.writeHeader(.symbolic_link, sub_path, link_name, 0, opt);
83 }
84
85 /// Writes fs.Dir.WalkerEntry. Uses `mtime` from file system entry and
86 /// default for entry mode .
87 pub fn writeEntry(self: *Self, entry: std.fs.Dir.Walker.Entry) !void {
88 switch (entry.kind) {
89 .directory => {
90 try self.writeDir(entry.path, .{ .mtime = try entryMtime(entry) });
91 },
92 .file => {
93 var file = try entry.dir.openFile(entry.basename, .{});
94 defer file.close();
95 try self.writeFile(entry.path, file);
96 },
97 .sym_link => {
98 var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
99 const link_name = try entry.dir.readLink(entry.basename, &link_name_buffer);
100 try self.writeLink(entry.path, link_name, .{ .mtime = try entryMtime(entry) });
101 },
102 else => {
103 return error.UnsupportedWalkerEntryKind;
104 },
105 }
106 }
107
108 fn writeHeader(
109 self: *Self,
110 typeflag: Header.FileType,
111 sub_path: []const u8,
112 link_name: []const u8,
113 size: u64,
114 opt: Options,
115 ) !void {
116 var header = Header.init(typeflag);
117 try self.setPath(&header, sub_path);
118 try header.setSize(size);
119 try header.setMtime(if (opt.mtime != 0) opt.mtime else self.mtimeNow());
120 if (opt.mode != 0)
121 try header.setMode(opt.mode);
122 if (typeflag == .symbolic_link)
123 header.setLinkname(link_name) catch |err| switch (err) {
124 error.NameTooLong => try self.writeExtendedHeader(.gnu_long_link, &.{link_name}),
125 else => return err,
126 };
127 try header.write(self.underlying_writer);
128 }
129
130 fn mtimeNow(self: *Self) u64 {
131 if (self.mtime_now == 0)
132 self.mtime_now = @intCast(std.time.timestamp());
133 return self.mtime_now;
134 }
135
136 fn entryMtime(entry: std.fs.Dir.Walker.Entry) !u64 {
137 const stat = try entry.dir.statFile(entry.basename);
138 return @intCast(@divFloor(stat.mtime, std.time.ns_per_s));
139 }
140
141 /// Writes path in posix header, if don't fit (in name+prefix; 100+155
142 /// bytes) writes it in gnu extended header.
143 fn setPath(self: *Self, header: *Header, sub_path: []const u8) !void {
144 header.setPath(self.prefix, sub_path) catch |err| switch (err) {
145 error.NameTooLong => {
146 // write extended header
147 const buffers: []const []const u8 = if (self.prefix.len == 0)
148 &.{sub_path}
149 else
150 &.{ self.prefix, "/", sub_path };
151 try self.writeExtendedHeader(.gnu_long_name, buffers);
152 },
153 else => return err,
154 };
155 }
156
157 /// Writes gnu extended header: gnu_long_name or gnu_long_link.
158 fn writeExtendedHeader(self: *Self, typeflag: Header.FileType, buffers: []const []const u8) !void {
159 var len: usize = 0;
160 for (buffers) |buf|
161 len += buf.len;
162
163 var header = Header.init(typeflag);
164 try header.setSize(len);
165 try header.write(self.underlying_writer);
166 for (buffers) |buf|
167 try self.underlying_writer.writeAll(buf);
168 try self.writePadding(len);
169 }
170
171 fn writePadding(self: *Self, bytes: u64) !void {
172 const pos: usize = @intCast(bytes % block_size);
173 if (pos == 0) return;
174 try self.underlying_writer.writeAll(empty_block[pos..]);
175 }
176
177 /// Tar should finish with two zero blocks, but 'reasonable system must
178 /// not assume that such a block exists when reading an archive' (from
179 /// reference). In practice it is safe to skip this finish.
180 pub fn finish(self: *Self) !void {
181 try self.underlying_writer.writeAll(&empty_block);
182 try self.underlying_writer.writeAll(&empty_block);
183 }
184 };
185}
186
187/// A struct that is exactly 512 bytes and matches tar file format. This is
188/// intended to be used for outputting tar files; for parsing there is
189/// `std.tar.Header`.
190const Header = extern struct {
191 // This struct was originally copied from
192 // https://github.com/mattnite/tar/blob/main/src/main.zig which is MIT
193 // licensed.
194 //
195 // The name, linkname, magic, uname, and gname are null-terminated character
196 // strings. All other fields are zero-filled octal numbers in ASCII. Each
197 // numeric field of width w contains w minus 1 digits, and a null.
198 // Reference: https://www.gnu.org/software/tar/manual/html_node/Standard.html
199 // POSIX header: byte offset
200 name: [100]u8 = [_]u8{0} ** 100, // 0
201 mode: [7:0]u8 = default_mode.file, // 100
202 uid: [7:0]u8 = [_:0]u8{0} ** 7, // unused 108
203 gid: [7:0]u8 = [_:0]u8{0} ** 7, // unused 116
204 size: [11:0]u8 = [_:0]u8{'0'} ** 11, // 124
205 mtime: [11:0]u8 = [_:0]u8{'0'} ** 11, // 136
206 checksum: [7:0]u8 = [_:0]u8{' '} ** 7, // 148
207 typeflag: FileType = .regular, // 156
208 linkname: [100]u8 = [_]u8{0} ** 100, // 157
209 magic: [6]u8 = [_]u8{ 'u', 's', 't', 'a', 'r', 0 }, // 257
210 version: [2]u8 = [_]u8{ '0', '0' }, // 263
211 uname: [32]u8 = [_]u8{0} ** 32, // unused 265
212 gname: [32]u8 = [_]u8{0} ** 32, // unused 297
213 devmajor: [7:0]u8 = [_:0]u8{0} ** 7, // unused 329
214 devminor: [7:0]u8 = [_:0]u8{0} ** 7, // unused 337
215 prefix: [155]u8 = [_]u8{0} ** 155, // 345
216 pad: [12]u8 = [_]u8{0} ** 12, // unused 500
217
218 pub const FileType = enum(u8) {
219 regular = '0',
220 symbolic_link = '2',
221 directory = '5',
222 gnu_long_name = 'L',
223 gnu_long_link = 'K',
224 };
225
226 const default_mode = struct {
227 const file = [_:0]u8{ '0', '0', '0', '0', '6', '6', '4' }; // 0o664
228 const dir = [_:0]u8{ '0', '0', '0', '0', '7', '7', '5' }; // 0o775
229 const sym_link = [_:0]u8{ '0', '0', '0', '0', '7', '7', '7' }; // 0o777
230 const other = [_:0]u8{ '0', '0', '0', '0', '0', '0', '0' }; // 0o000
231 };
232
233 pub fn init(typeflag: FileType) Header {
234 return .{
235 .typeflag = typeflag,
236 .mode = switch (typeflag) {
237 .directory => default_mode.dir,
238 .symbolic_link => default_mode.sym_link,
239 .regular => default_mode.file,
240 else => default_mode.other,
241 },
242 };
243 }
244
245 pub fn setSize(self: *Header, size: u64) !void {
246 try octal(&self.size, size);
247 }
248
249 fn octal(buf: []u8, value: u64) !void {
250 var remainder: u64 = value;
251 var pos: usize = buf.len;
252 while (remainder > 0 and pos > 0) {
253 pos -= 1;
254 const c: u8 = @as(u8, @intCast(remainder % 8)) + '0';
255 buf[pos] = c;
256 remainder /= 8;
257 if (pos == 0 and remainder > 0) return error.OctalOverflow;
258 }
259 }
260
261 pub fn setMode(self: *Header, mode: u32) !void {
262 try octal(&self.mode, mode);
263 }
264
265 // Integer number of seconds since January 1, 1970, 00:00 Coordinated Universal Time.
266 // mtime == 0 will use current time
267 pub fn setMtime(self: *Header, mtime: u64) !void {
268 try octal(&self.mtime, mtime);
269 }
270
271 pub fn updateChecksum(self: *Header) !void {
272 var checksum: usize = ' '; // other 7 self.checksum bytes are initialized to ' '
273 for (std.mem.asBytes(self)) |val|
274 checksum += val;
275 try octal(&self.checksum, checksum);
276 }
277
278 pub fn write(self: *Header, output_writer: anytype) !void {
279 try self.updateChecksum();
280 try output_writer.writeAll(std.mem.asBytes(self));
281 }
282
283 pub fn setLinkname(self: *Header, link: []const u8) !void {
284 if (link.len > self.linkname.len) return error.NameTooLong;
285 @memcpy(self.linkname[0..link.len], link);
286 }
287
288 pub fn setPath(self: *Header, prefix: []const u8, sub_path: []const u8) !void {
289 const max_prefix = self.prefix.len;
290 const max_name = self.name.len;
291 const sep = std.fs.path.sep_posix;
292
293 if (prefix.len + sub_path.len > max_name + max_prefix or prefix.len > max_prefix)
294 return error.NameTooLong;
295
296 // both fit into name
297 if (prefix.len > 0 and prefix.len + sub_path.len < max_name) {
298 @memcpy(self.name[0..prefix.len], prefix);
299 self.name[prefix.len] = sep;
300 @memcpy(self.name[prefix.len + 1 ..][0..sub_path.len], sub_path);
301 return;
302 }
303
304 // sub_path fits into name
305 // there is no prefix or prefix fits into prefix
306 if (sub_path.len <= max_name) {
307 @memcpy(self.name[0..sub_path.len], sub_path);
308 @memcpy(self.prefix[0..prefix.len], prefix);
309 return;
310 }
311
312 if (prefix.len > 0) {
313 @memcpy(self.prefix[0..prefix.len], prefix);
314 self.prefix[prefix.len] = sep;
315 }
316 const prefix_pos = if (prefix.len > 0) prefix.len + 1 else 0;
317
318 // add as much to prefix as you can, must split at /
319 const prefix_remaining = max_prefix - prefix_pos;
320 if (std.mem.lastIndexOf(u8, sub_path[0..@min(prefix_remaining, sub_path.len)], &.{'/'})) |sep_pos| {
321 @memcpy(self.prefix[prefix_pos..][0..sep_pos], sub_path[0..sep_pos]);
322 if ((sub_path.len - sep_pos - 1) > max_name) return error.NameTooLong;
323 @memcpy(self.name[0..][0 .. sub_path.len - sep_pos - 1], sub_path[sep_pos + 1 ..]);
324 return;
325 }
326
327 return error.NameTooLong;
328 }
329
330 comptime {
331 assert(@sizeOf(Header) == 512);
332 }
333
334 test setPath {
335 const cases = [_]struct {
336 in: []const []const u8,
337 out: []const []const u8,
338 }{
339 .{
340 .in = &.{ "", "123456789" },
341 .out = &.{ "", "123456789" },
342 },
343 // can fit into name
344 .{
345 .in = &.{ "prefix", "sub_path" },
346 .out = &.{ "", "prefix/sub_path" },
347 },
348 // no more both fits into name
349 .{
350 .in = &.{ "prefix", "0123456789/" ** 8 ++ "basename" },
351 .out = &.{ "prefix", "0123456789/" ** 8 ++ "basename" },
352 },
353 // put as much as you can into prefix the rest goes into name
354 .{
355 .in = &.{ "prefix", "0123456789/" ** 10 ++ "basename" },
356 .out = &.{ "prefix/" ++ "0123456789/" ** 9 ++ "0123456789", "basename" },
357 },
358
359 .{
360 .in = &.{ "prefix", "0123456789/" ** 15 ++ "basename" },
361 .out = &.{ "prefix/" ++ "0123456789/" ** 12 ++ "0123456789", "0123456789/0123456789/basename" },
362 },
363 .{
364 .in = &.{ "prefix", "0123456789/" ** 21 ++ "basename" },
365 .out = &.{ "prefix/" ++ "0123456789/" ** 12 ++ "0123456789", "0123456789/" ** 8 ++ "basename" },
366 },
367 .{
368 .in = &.{ "", "012345678/" ** 10 ++ "foo" },
369 .out = &.{ "012345678/" ** 9 ++ "012345678", "foo" },
370 },
371 };
372
373 for (cases) |case| {
374 var header = Header.init(.regular);
375 try header.setPath(case.in[0], case.in[1]);
376 try testing.expectEqualStrings(case.out[0], str(&header.prefix));
377 try testing.expectEqualStrings(case.out[1], str(&header.name));
378 }
379
380 const error_cases = [_]struct {
381 in: []const []const u8,
382 }{
383 // basename can't fit into name (106 characters)
384 .{ .in = &.{ "zig", "test/cases/compile_errors/regression_test_2980_base_type_u32_is_not_type_checked_properly_when_assigning_a_value_within_a_struct.zig" } },
385 // cant fit into 255 + sep
386 .{ .in = &.{ "prefix", "0123456789/" ** 22 ++ "basename" } },
387 // can fit but sub_path can't be split (there is no separator)
388 .{ .in = &.{ "prefix", "0123456789" ** 10 ++ "a" } },
389 .{ .in = &.{ "prefix", "0123456789" ** 14 ++ "basename" } },
390 };
391
392 for (error_cases) |case| {
393 var header = Header.init(.regular);
394 try testing.expectError(
395 error.NameTooLong,
396 header.setPath(case.in[0], case.in[1]),
397 );
398 }
399 }
400
401 // Breaks string on first null character.
402 fn str(s: []const u8) []const u8 {
403 for (s, 0..) |c, i| {
404 if (c == 0) return s[0..i];
405 }
406 return s;
407 }
408};
409
410test {
411 _ = Header;
412}
413
414test "write files" {
415 const files = [_]struct {
416 path: []const u8,
417 content: []const u8,
418 }{
419 .{ .path = "foo", .content = "bar" },
420 .{ .path = "a12345678/" ** 10 ++ "foo", .content = "a" ** 511 },
421 .{ .path = "b12345678/" ** 24 ++ "foo", .content = "b" ** 512 },
422 .{ .path = "c12345678/" ** 25 ++ "foo", .content = "c" ** 513 },
423 .{ .path = "d12345678/" ** 51 ++ "foo", .content = "d" ** 1025 },
424 .{ .path = "e123456789" ** 11, .content = "e" },
425 };
426
427 var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
428 var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
429
430 // with root
431 {
432 const root = "root";
433
434 var output = std.ArrayList(u8).init(testing.allocator);
435 defer output.deinit();
436 var wrt = writer(output.writer());
437 try wrt.setRoot(root);
438 for (files) |file|
439 try wrt.writeFileBytes(file.path, file.content, .{});
440
441 var input = std.io.fixedBufferStream(output.items);
442 var iter = std.tar.iterator(
443 input.reader(),
444 .{ .file_name_buffer = &file_name_buffer, .link_name_buffer = &link_name_buffer },
445 );
446
447 // first entry is directory with prefix
448 {
449 const actual = (try iter.next()).?;
450 try testing.expectEqualStrings(root, actual.name);
451 try testing.expectEqual(std.tar.FileKind.directory, actual.kind);
452 }
453
454 var i: usize = 0;
455 while (try iter.next()) |actual| {
456 defer i += 1;
457 const expected = files[i];
458 try testing.expectEqualStrings(root, actual.name[0..root.len]);
459 try testing.expectEqual('/', actual.name[root.len..][0]);
460 try testing.expectEqualStrings(expected.path, actual.name[root.len + 1 ..]);
461
462 var content = std.ArrayList(u8).init(testing.allocator);
463 defer content.deinit();
464 try actual.writeAll(content.writer());
465 try testing.expectEqualSlices(u8, expected.content, content.items);
466 }
467 }
468 // without root
469 {
470 var output = std.ArrayList(u8).init(testing.allocator);
471 defer output.deinit();
472 var wrt = writer(output.writer());
473 for (files) |file| {
474 var content = std.io.fixedBufferStream(file.content);
475 try wrt.writeFileStream(file.path, file.content.len, content.reader(), .{});
476 }
477
478 var input = std.io.fixedBufferStream(output.items);
479 var iter = std.tar.iterator(
480 input.reader(),
481 .{ .file_name_buffer = &file_name_buffer, .link_name_buffer = &link_name_buffer },
482 );
483
484 var i: usize = 0;
485 while (try iter.next()) |actual| {
486 defer i += 1;
487 const expected = files[i];
488 try testing.expectEqualStrings(expected.path, actual.name);
489
490 var content = std.ArrayList(u8).init(testing.allocator);
491 defer content.deinit();
492 try actual.writeAll(content.writer());
493 try testing.expectEqualSlices(u8, expected.content, content.items);
494 }
495 try wrt.finish();
496 }
497}
src/Compilation.zig+23-14
...@@ -4862,6 +4862,9 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {...@@ -4862,6 +4862,9 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {
4862 };4862 };
4863 defer tar_file.close();4863 defer tar_file.close();
48644864
4865 var buffer: [1024]u8 = undefined;
4866 var tar_file_writer = tar_file.writer(&buffer);
4867
4865 var seen_table: std.AutoArrayHashMapUnmanaged(*Package.Module, []const u8) = .empty;4868 var seen_table: std.AutoArrayHashMapUnmanaged(*Package.Module, []const u8) = .empty;
4866 defer seen_table.deinit(comp.gpa);4869 defer seen_table.deinit(comp.gpa);
48674870
...@@ -4871,7 +4874,7 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {...@@ -4871,7 +4874,7 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {
4871 var i: usize = 0;4874 var i: usize = 0;
4872 while (i < seen_table.count()) : (i += 1) {4875 while (i < seen_table.count()) : (i += 1) {
4873 const mod = seen_table.keys()[i];4876 const mod = seen_table.keys()[i];
4874 try comp.docsCopyModule(mod, seen_table.values()[i], tar_file);4877 try comp.docsCopyModule(mod, seen_table.values()[i], &tar_file_writer);
48754878
4876 const deps = mod.deps.values();4879 const deps = mod.deps.values();
4877 try seen_table.ensureUnusedCapacity(comp.gpa, deps.len);4880 try seen_table.ensureUnusedCapacity(comp.gpa, deps.len);
...@@ -4879,24 +4882,29 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {...@@ -4879,24 +4882,29 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {
4879 }4882 }
4880}4883}
48814884
4882fn docsCopyModule(comp: *Compilation, module: *Package.Module, name: []const u8, tar_file: fs.File) !void {4885fn docsCopyModule(
4886 comp: *Compilation,
4887 module: *Package.Module,
4888 name: []const u8,
4889 tar_file_writer: *fs.File.Writer,
4890) !void {
4883 const root = module.root;4891 const root = module.root;
4884 var mod_dir = d: {4892 var mod_dir = d: {
4885 const root_dir, const sub_path = root.openInfo(comp.dirs);4893 const root_dir, const sub_path = root.openInfo(comp.dirs);
4886 break :d root_dir.openDir(sub_path, .{ .iterate = true });4894 break :d root_dir.openDir(sub_path, .{ .iterate = true });
4887 } catch |err| {4895 } catch |err| {
4888 return comp.lockAndSetMiscFailure(.docs_copy, "unable to open directory '{f}': {s}", .{4896 return comp.lockAndSetMiscFailure(.docs_copy, "unable to open directory '{f}': {t}", .{ root.fmt(comp), err });
4889 root.fmt(comp), @errorName(err),
4890 });
4891 };4897 };
4892 defer mod_dir.close();4898 defer mod_dir.close();
48934899
4894 var walker = try mod_dir.walk(comp.gpa);4900 var walker = try mod_dir.walk(comp.gpa);
4895 defer walker.deinit();4901 defer walker.deinit();
48964902
4897 var archiver = std.tar.writer(tar_file.deprecatedWriter().any());4903 var archiver: std.tar.Writer = .{ .underlying_writer = &tar_file_writer.interface };
4898 archiver.prefix = name;4904 archiver.prefix = name;
48994905
4906 var buffer: [1024]u8 = undefined;
4907
4900 while (try walker.next()) |entry| {4908 while (try walker.next()) |entry| {
4901 switch (entry.kind) {4909 switch (entry.kind) {
4902 .file => {4910 .file => {
...@@ -4907,14 +4915,17 @@ fn docsCopyModule(comp: *Compilation, module: *Package.Module, name: []const u8,...@@ -4907,14 +4915,17 @@ fn docsCopyModule(comp: *Compilation, module: *Package.Module, name: []const u8,
4907 else => continue,4915 else => continue,
4908 }4916 }
4909 var file = mod_dir.openFile(entry.path, .{}) catch |err| {4917 var file = mod_dir.openFile(entry.path, .{}) catch |err| {
4910 return comp.lockAndSetMiscFailure(.docs_copy, "unable to open '{f}{s}': {s}", .{4918 return comp.lockAndSetMiscFailure(.docs_copy, "unable to open {f}{s}: {t}", .{
4911 root.fmt(comp), entry.path, @errorName(err),4919 root.fmt(comp), entry.path, err,
4912 });4920 });
4913 };4921 };
4914 defer file.close();4922 defer file.close();
4915 archiver.writeFile(entry.path, file) catch |err| {4923 const stat = try file.stat();
4916 return comp.lockAndSetMiscFailure(.docs_copy, "unable to archive '{f}{s}': {s}", .{4924 var file_reader: fs.File.Reader = .initSize(file, &buffer, stat.size);
4917 root.fmt(comp), entry.path, @errorName(err),4925
4926 archiver.writeFile(entry.path, &file_reader, stat.mtime) catch |err| {
4927 return comp.lockAndSetMiscFailure(.docs_copy, "unable to archive {f}{s}: {t}", .{
4928 root.fmt(comp), entry.path, err,
4918 });4929 });
4919 };4930 };
4920 }4931 }
...@@ -4926,9 +4937,7 @@ fn workerDocsWasm(comp: *Compilation, parent_prog_node: std.Progress.Node) void...@@ -4926,9 +4937,7 @@ fn workerDocsWasm(comp: *Compilation, parent_prog_node: std.Progress.Node) void
49264937
4927 workerDocsWasmFallible(comp, prog_node) catch |err| switch (err) {4938 workerDocsWasmFallible(comp, prog_node) catch |err| switch (err) {
4928 error.SubCompilationFailed => return, // error reported already4939 error.SubCompilationFailed => return, // error reported already
4929 else => comp.lockAndSetMiscFailure(.docs_wasm, "unable to build autodocs: {s}", .{4940 else => comp.lockAndSetMiscFailure(.docs_wasm, "unable to build autodocs: {t}", .{err}),
4930 @errorName(err),
4931 }),
4932 };4941 };
4933}4942}
49344943
src/Package/Fetch.zig+15-9
...@@ -1197,12 +1197,16 @@ fn unpackResource(...@@ -1197,12 +1197,16 @@ fn unpackResource(
1197 };1197 };
11981198
1199 switch (file_type) {1199 switch (file_type) {
1200 .tar => return try unpackTarball(f, tmp_directory.handle, resource.reader()),1200 .tar => {
1201 var adapter = resource.reader().adaptToNewApi();
1202 return unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
1203 },
1201 .@"tar.gz" => {1204 .@"tar.gz" => {
1202 const reader = resource.reader();1205 const reader = resource.reader();
1203 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);1206 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1204 var dcp = std.compress.gzip.decompressor(br.reader());1207 var dcp = std.compress.gzip.decompressor(br.reader());
1205 return try unpackTarball(f, tmp_directory.handle, dcp.reader());1208 var adapter = dcp.reader().adaptToNewApi();
1209 return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
1206 },1210 },
1207 .@"tar.xz" => {1211 .@"tar.xz" => {
1208 const gpa = f.arena.child_allocator;1212 const gpa = f.arena.child_allocator;
...@@ -1215,7 +1219,8 @@ fn unpackResource(...@@ -1215,7 +1219,8 @@ fn unpackResource(
1215 ));1219 ));
1216 };1220 };
1217 defer dcp.deinit();1221 defer dcp.deinit();
1218 return try unpackTarball(f, tmp_directory.handle, dcp.reader());1222 var adapter = dcp.reader().adaptToNewApi();
1223 return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
1219 },1224 },
1220 .@"tar.zst" => {1225 .@"tar.zst" => {
1221 const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len;1226 const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len;
...@@ -1225,7 +1230,8 @@ fn unpackResource(...@@ -1225,7 +1230,8 @@ fn unpackResource(
1225 var dcp = std.compress.zstd.decompressor(br.reader(), .{1230 var dcp = std.compress.zstd.decompressor(br.reader(), .{
1226 .window_buffer = window_buffer,1231 .window_buffer = window_buffer,
1227 });1232 });
1228 return try unpackTarball(f, tmp_directory.handle, dcp.reader());1233 var adapter = dcp.reader().adaptToNewApi();
1234 return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
1229 },1235 },
1230 .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) {1236 .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) {
1231 error.FetchFailed => return error.FetchFailed,1237 error.FetchFailed => return error.FetchFailed,
...@@ -1239,7 +1245,7 @@ fn unpackResource(...@@ -1239,7 +1245,7 @@ fn unpackResource(
1239 }1245 }
1240}1246}
12411247
1242fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult {1248fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: *std.Io.Reader) RunError!UnpackResult {
1243 const eb = &f.error_bundle;1249 const eb = &f.error_bundle;
1244 const arena = f.arena.allocator();1250 const arena = f.arena.allocator();
12451251
...@@ -1250,10 +1256,10 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackRes...@@ -1250,10 +1256,10 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackRes
1250 .strip_components = 0,1256 .strip_components = 0,
1251 .mode_mode = .ignore,1257 .mode_mode = .ignore,
1252 .exclude_empty_directories = true,1258 .exclude_empty_directories = true,
1253 }) catch |err| return f.fail(f.location_tok, try eb.printString(1259 }) catch |err| return f.fail(
1254 "unable to unpack tarball to temporary directory: {s}",1260 f.location_tok,
1255 .{@errorName(err)},1261 try eb.printString("unable to unpack tarball to temporary directory: {t}", .{err}),
1256 ));1262 );
12571263
1258 var res: UnpackResult = .{ .root_dir = diagnostics.root_dir };1264 var res: UnpackResult = .{ .root_dir = diagnostics.root_dir };
1259 if (diagnostics.errors.items.len > 0) {1265 if (diagnostics.errors.items.len > 0) {