Commit bc8e1a74c514e487842c03ab32d7f6e49f42c529 | zig

CMakeLists.txt-9

...	@@ -390,15 +390,6 @@ set(ZIG_STAGE2_SOURCES	...	@@ -390,15 +390,6 @@ set(ZIG_STAGE2_SOURCES
390	lib/std/Io.zig	390	lib/std/Io.zig
391	lib/std/Io/Reader.zig	391	lib/std/Io/Reader.zig
392	lib/std/Io/Writer.zig	392	lib/std/Io/Writer.zig
393	lib/std/Io/buffered_atomic_file.zig
394	lib/std/Io/buffered_writer.zig
395	lib/std/Io/change_detection_stream.zig
396	lib/std/Io/counting_reader.zig
397	lib/std/Io/counting_writer.zig
398	lib/std/Io/find_byte_writer.zig
399	lib/std/Io/fixed_buffer_stream.zig
400	lib/std/Io/limited_reader.zig
401	lib/std/Io/seekable_stream.zig
402	lib/std/Progress.zig	393	lib/std/Progress.zig
403	lib/std/Random.zig	394	lib/std/Random.zig
404	lib/std/Target.zig	395	lib/std/Target.zig

lib/compiler/std-docs.zig+15-3

 @@ -60,7 +60,9 @@ pub fn main() !void {
     const should_open_browser = force_open_browser orelse (listen_port == 0);
     const address = std.net.Address.parseIp("127.0.0.1", listen_port) catch unreachable;
-    var http_server = try address.listen(.{});
+    var http_server = try address.listen(.{
+        .reuse_address = true,
+    });
     const port = http_server.listen_address.in.getPort();
     const url_with_newline = try std.fmt.allocPrint(arena, "http://127.0.0.1:{d}/\n", .{port});
     std.fs.File.stdout().writeAll(url_with_newline) catch {};
 @@ -189,7 +191,11 @@ fn serveSourcesTar(request: *std.http.Server.Request, context: *Context) !void {
     var walker = try std_dir.walk(gpa);
     defer walker.deinit();
-    var archiver = std.tar.writer(response.writer());
+    var adapter_buffer: [500]u8 = undefined;
+    var response_writer = response.writer().adaptToNewApi();
+    response_writer.new_interface.buffer = &adapter_buffer;
+    var archiver: std.tar.Writer = .{ .underlying_writer = &response_writer.new_interface };
     archiver.prefix = "std";
     while (try walker.next()) |entry| {
 @@ -204,7 +210,13 @@ fn serveSourcesTar(request: *std.http.Server.Request, context: *Context) !void {
         }
         var file = try entry.dir.openFile(entry.basename, .{});
         defer file.close();
-        try archiver.writeFile(entry.path, file);
+        const stat = try file.stat();
+        var file_reader: std.fs.File.Reader = .{
+            .file = file,
+            .interface = std.fs.File.Reader.initInterface(&.{}),
+            .size = stat.size,
+        };
+        try archiver.writeFile(entry.path, &file_reader, stat.mtime);
     }
     {

lib/docs/wasm/main.zig+3-3

...	@@ -772,10 +772,10 @@ export fn decl_type_html(decl_index: Decl.Index) String {	...	@@ -772,10 +772,10 @@ export fn decl_type_html(decl_index: Decl.Index) String {
772	const Oom = error{OutOfMemory};	772	const Oom = error{OutOfMemory};
773		773
774	fn unpackInner(tar_bytes: []u8) !void {	774	fn unpackInner(tar_bytes: []u8) !void {
775	var fbs = std.io.fixedBufferStream(tar_bytes);	775	var reader: std.Io.Reader = .fixed(tar_bytes);
776	var file_name_buffer: [1024]u8 = undefined;	776	var file_name_buffer: [1024]u8 = undefined;
777	var link_name_buffer: [1024]u8 = undefined;	777	var link_name_buffer: [1024]u8 = undefined;
778	var it = std.tar.iterator(fbs.reader(), .{	778	var it: std.tar.Iterator = .init(&reader, .{
779	.file_name_buffer = &file_name_buffer,	779	.file_name_buffer = &file_name_buffer,
780	.link_name_buffer = &link_name_buffer,	780	.link_name_buffer = &link_name_buffer,
781	});	781	});
...	@@ -796,7 +796,7 @@ fn unpackInner(tar_bytes: []u8) !void {	...	@@ -796,7 +796,7 @@ fn unpackInner(tar_bytes: []u8) !void {
796	{	796	{
797	gop.value_ptr.* = file;	797	gop.value_ptr.* = file;
798	}	798	}
799	const file_bytes = tar_bytes[fbs.pos..][0..@intCast(tar_file.size)];	799	const file_bytes = tar_bytes[reader.seek..][0..@intCast(tar_file.size)];
800	assert(file == try Walk.add_file(file_name, file_bytes));	800	assert(file == try Walk.add_file(file_name, file_bytes));
801	}	801	}
802	} else {	802	} else {

lib/std/Build/Fuzz/WebServer.zig+8-5

...	@@ -522,7 +522,9 @@ fn serveSourcesTar(ws: WebServer, request: std.http.Server.Request) !void {	...	@@ -522,7 +522,9 @@ fn serveSourcesTar(ws: WebServer, request: std.http.Server.Request) !void {
522		522
523	var cwd_cache: ?[]const u8 = null;	523	var cwd_cache: ?[]const u8 = null;
524		524
525	var archiver = std.tar.writer(response.writer());	525	var adapter = response.writer().adaptToNewApi();
		526	var archiver: std.tar.Writer = .{ .underlying_writer = &adapter.new_interface };
		527	var read_buffer: [1024]u8 = undefined;
526		528
527	for (deduped_paths) \|joined_path\| {	529	for (deduped_paths) \|joined_path\| {
528	var file = joined_path.root_dir.handle.openFile(joined_path.sub_path, .{}) catch \|err\| {	530	var file = joined_path.root_dir.handle.openFile(joined_path.sub_path, .{}) catch \|err\| {
...	@@ -530,13 +532,14 @@ fn serveSourcesTar(ws: WebServer, request: std.http.Server.Request) !void {	...	@@ -530,13 +532,14 @@ fn serveSourcesTar(ws: WebServer, request: std.http.Server.Request) !void {
530	continue;	532	continue;
531	};	533	};
532	defer file.close();	534	defer file.close();
533		535	const stat = try file.stat();
		536	var file_reader: std.fs.File.Reader = .initSize(file, &read_buffer, stat.size);
534	archiver.prefix = joined_path.root_dir.path orelse try memoizedCwd(arena, &cwd_cache);	537	archiver.prefix = joined_path.root_dir.path orelse try memoizedCwd(arena, &cwd_cache);
535	try archiver.writeFile(joined_path.sub_path, file);	538	try archiver.writeFile(joined_path.sub_path, &file_reader, stat.mtime);
536	}	539	}
537		540
538	// intentionally omitting the pointless trailer	541	// intentionally not calling `archiver.finishPedantically`
539	//try archiver.finish();	542	try adapter.new_interface.flush();
540	try response.end();	543	try response.end();
541	}	544	}
542		545

lib/std/Io.zig+36-39

...	@@ -314,11 +314,11 @@ pub fn GenericReader(	...	@@ -314,11 +314,11 @@ pub fn GenericReader(
314	}	314	}
315		315
316	/// Helper for bridging to the new `Reader` API while upgrading.	316	/// Helper for bridging to the new `Reader` API while upgrading.
317	pub fn adaptToNewApi(self: *const Self) Adapter {	317	pub fn adaptToNewApi(self: *const Self, buffer: []u8) Adapter {
318	return .{	318	return .{
319	.derp_reader = self.*,	319	.derp_reader = self.*,
320	.new_interface = .{	320	.new_interface = .{
321	.buffer = &.{},	321	.buffer = buffer,
322	.vtable = &.{ .stream = Adapter.stream },	322	.vtable = &.{ .stream = Adapter.stream },
323	.seek = 0,	323	.seek = 0,
324	.end = 0,	324	.end = 0,
...	@@ -334,10 +334,12 @@ pub fn GenericReader(	...	@@ -334,10 +334,12 @@ pub fn GenericReader(
334	fn stream(r: Reader, w: Writer, limit: Limit) Reader.StreamError!usize {	334	fn stream(r: Reader, w: Writer, limit: Limit) Reader.StreamError!usize {
335	const a: *@This() = @alignCast(@fieldParentPtr("new_interface", r));	335	const a: *@This() = @alignCast(@fieldParentPtr("new_interface", r));
336	const buf = limit.slice(try w.writableSliceGreedy(1));	336	const buf = limit.slice(try w.writableSliceGreedy(1));
337	return a.derp_reader.read(buf) catch \|err\| {	337	const n = a.derp_reader.read(buf) catch \|err\| {
338	a.err = err;	338	a.err = err;
339	return error.ReadFailed;	339	return error.ReadFailed;
340	};	340	};
		341	w.advance(n);
		342	return n;
341	}	343	}
342	};	344	};
343	};	345	};
...	@@ -419,9 +421,14 @@ pub fn GenericWriter(	...	@@ -419,9 +421,14 @@ pub fn GenericWriter(
419	new_interface: Writer,	421	new_interface: Writer,
420	err: ?Error = null,	422	err: ?Error = null,
421		423
422	fn drain(w: *Writer, data: []const []const u8, splat: usize) Writer.Error!usize {	424	fn drain(w: *std.io.Writer, data: []const []const u8, splat: usize) std.io.Writer.Error!usize {
423	_ = splat;	425	_ = splat;
424	const a: *@This() = @alignCast(@fieldParentPtr("new_interface", w));	426	const a: *@This() = @alignCast(@fieldParentPtr("new_interface", w));
		427	const buffered = w.buffered();
		428	if (buffered.len != 0) return w.consume(a.derp_writer.write(buffered) catch \|err\| {
		429	a.err = err;
		430	return error.WriteFailed;
		431	});
425	return a.derp_writer.write(data[0]) catch \|err\| {	432	return a.derp_writer.write(data[0]) catch \|err\| {
426	a.err = err;	433	a.err = err;
427	return error.WriteFailed;	434	return error.WriteFailed;
...	@@ -435,54 +442,46 @@ pub fn GenericWriter(	...	@@ -435,54 +442,46 @@ pub fn GenericWriter(
435	pub const AnyReader = @import("Io/DeprecatedReader.zig");	442	pub const AnyReader = @import("Io/DeprecatedReader.zig");
436	/// Deprecated in favor of `Writer`.	443	/// Deprecated in favor of `Writer`.
437	pub const AnyWriter = @import("Io/DeprecatedWriter.zig");	444	pub const AnyWriter = @import("Io/DeprecatedWriter.zig");
438		445	/// Deprecated in favor of `File.Reader` and `File.Writer`.
439	pub const SeekableStream = @import("Io/seekable_stream.zig").SeekableStream;	446	pub const SeekableStream = @import("Io/seekable_stream.zig").SeekableStream;
440		447	/// Deprecated in favor of `Writer`.
441	pub const BufferedWriter = @import("Io/buffered_writer.zig").BufferedWriter;	448	pub const BufferedWriter = @import("Io/buffered_writer.zig").BufferedWriter;
		449	/// Deprecated in favor of `Writer`.
442	pub const bufferedWriter = @import("Io/buffered_writer.zig").bufferedWriter;	450	pub const bufferedWriter = @import("Io/buffered_writer.zig").bufferedWriter;
443		451	/// Deprecated in favor of `Reader`.
444	pub const BufferedReader = @import("Io/buffered_reader.zig").BufferedReader;	452	pub const BufferedReader = @import("Io/buffered_reader.zig").BufferedReader;
		453	/// Deprecated in favor of `Reader`.
445	pub const bufferedReader = @import("Io/buffered_reader.zig").bufferedReader;	454	pub const bufferedReader = @import("Io/buffered_reader.zig").bufferedReader;
		455	/// Deprecated in favor of `Reader`.
446	pub const bufferedReaderSize = @import("Io/buffered_reader.zig").bufferedReaderSize;	456	pub const bufferedReaderSize = @import("Io/buffered_reader.zig").bufferedReaderSize;
447		457	/// Deprecated in favor of `Reader`.
448	pub const FixedBufferStream = @import("Io/fixed_buffer_stream.zig").FixedBufferStream;	458	pub const FixedBufferStream = @import("Io/fixed_buffer_stream.zig").FixedBufferStream;
		459	/// Deprecated in favor of `Reader`.
449	pub const fixedBufferStream = @import("Io/fixed_buffer_stream.zig").fixedBufferStream;	460	pub const fixedBufferStream = @import("Io/fixed_buffer_stream.zig").fixedBufferStream;
450		461	/// Deprecated in favor of `Reader.Limited`.
451	pub const CWriter = @import("Io/c_writer.zig").CWriter;
452	pub const cWriter = @import("Io/c_writer.zig").cWriter;
453
454	pub const LimitedReader = @import("Io/limited_reader.zig").LimitedReader;	462	pub const LimitedReader = @import("Io/limited_reader.zig").LimitedReader;
		463	/// Deprecated in favor of `Reader.Limited`.
455	pub const limitedReader = @import("Io/limited_reader.zig").limitedReader;	464	pub const limitedReader = @import("Io/limited_reader.zig").limitedReader;
456		465	/// Deprecated with no replacement; inefficient pattern
457	pub const CountingWriter = @import("Io/counting_writer.zig").CountingWriter;	466	pub const CountingWriter = @import("Io/counting_writer.zig").CountingWriter;
		467	/// Deprecated with no replacement; inefficient pattern
458	pub const countingWriter = @import("Io/counting_writer.zig").countingWriter;	468	pub const countingWriter = @import("Io/counting_writer.zig").countingWriter;
		469	/// Deprecated with no replacement; inefficient pattern
459	pub const CountingReader = @import("Io/counting_reader.zig").CountingReader;	470	pub const CountingReader = @import("Io/counting_reader.zig").CountingReader;
		471	/// Deprecated with no replacement; inefficient pattern
460	pub const countingReader = @import("Io/counting_reader.zig").countingReader;	472	pub const countingReader = @import("Io/counting_reader.zig").countingReader;
461		473
462	pub const MultiWriter = @import("Io/multi_writer.zig").MultiWriter;
463	pub const multiWriter = @import("Io/multi_writer.zig").multiWriter;
464
465	pub const BitReader = @import("Io/bit_reader.zig").BitReader;	474	pub const BitReader = @import("Io/bit_reader.zig").BitReader;
466	pub const bitReader = @import("Io/bit_reader.zig").bitReader;	475	pub const bitReader = @import("Io/bit_reader.zig").bitReader;
467		476
468	pub const BitWriter = @import("Io/bit_writer.zig").BitWriter;	477	pub const BitWriter = @import("Io/bit_writer.zig").BitWriter;
469	pub const bitWriter = @import("Io/bit_writer.zig").bitWriter;	478	pub const bitWriter = @import("Io/bit_writer.zig").bitWriter;
470		479
471	pub const ChangeDetectionStream = @import("Io/change_detection_stream.zig").ChangeDetectionStream;
472	pub const changeDetectionStream = @import("Io/change_detection_stream.zig").changeDetectionStream;
473
474	pub const FindByteWriter = @import("Io/find_byte_writer.zig").FindByteWriter;
475	pub const findByteWriter = @import("Io/find_byte_writer.zig").findByteWriter;
476
477	pub const BufferedAtomicFile = @import("Io/buffered_atomic_file.zig").BufferedAtomicFile;
478
479	pub const StreamSource = @import("Io/stream_source.zig").StreamSource;
480
481	pub const tty = @import("Io/tty.zig");	480	pub const tty = @import("Io/tty.zig");
482		481
483	/// A Writer that doesn't write to anything.	482	/// Deprecated in favor of `Writer.Discarding`.
484	pub const null_writer: NullWriter = .{ .context = {} };	483	pub const null_writer: NullWriter = .{ .context = {} };
485		484	/// Deprecated in favor of `Writer.Discarding`.
486	pub const NullWriter = GenericWriter(void, error{}, dummyWrite);	485	pub const NullWriter = GenericWriter(void, error{}, dummyWrite);
487	fn dummyWrite(context: void, data: []const u8) error{}!usize {	486	fn dummyWrite(context: void, data: []const u8) error{}!usize {
488	_ = context;	487	_ = context;
...	@@ -898,16 +897,14 @@ test {	...	@@ -898,16 +897,14 @@ test {
898	_ = Reader;	897	_ = Reader;
899	_ = Reader.Limited;	898	_ = Reader.Limited;
900	_ = Writer;	899	_ = Writer;
901	_ = @import("Io/bit_reader.zig");	900	_ = BitReader;
902	_ = @import("Io/bit_writer.zig");	901	_ = BitWriter;
903	_ = @import("Io/buffered_atomic_file.zig");	902	_ = BufferedReader;
904	_ = @import("Io/buffered_reader.zig");	903	_ = BufferedWriter;
905	_ = @import("Io/buffered_writer.zig");	904	_ = CountingWriter;
906	_ = @import("Io/c_writer.zig");	905	_ = CountingReader;
907	_ = @import("Io/counting_writer.zig");	906	_ = FixedBufferStream;
908	_ = @import("Io/counting_reader.zig");	907	_ = SeekableStream;
909	_ = @import("Io/fixed_buffer_stream.zig");	908	_ = tty;
910	_ = @import("Io/seekable_stream.zig");
911	_ = @import("Io/stream_source.zig");
912	_ = @import("Io/test.zig");	909	_ = @import("Io/test.zig");
913	}	910	}

lib/std/Io/DeprecatedReader.zig+3-1

 @@ -393,10 +393,12 @@ pub const Adapter = struct {
     fn stream(r: *std.io.Reader, w: *std.io.Writer, limit: std.io.Limit) std.io.Reader.StreamError!usize {
         const a: *@This() = @alignCast(@fieldParentPtr("new_interface", r));
         const buf = limit.slice(try w.writableSliceGreedy(1));
-        return a.derp_reader.read(buf) catch |err| {
+        const n = a.derp_reader.read(buf) catch |err| {
             a.err = err;
             return error.ReadFailed;
         };
+        w.advance(n);
+        return n;
     }
 };

lib/std/Io/DeprecatedWriter.zig+6-1

 @@ -100,7 +100,12 @@ pub const Adapter = struct {
     fn drain(w: *std.io.Writer, data: []const []const u8, splat: usize) std.io.Writer.Error!usize {
         _ = splat;
-        const a: *@This() = @fieldParentPtr("new_interface", w);
+        const a: *@This() = @alignCast(@fieldParentPtr("new_interface", w));
+        const buffered = w.buffered();
+        if (buffered.len != 0) return w.consume(a.derp_writer.write(buffered) catch |err| {
+            a.err = err;
+            return error.WriteFailed;
+        });
         return a.derp_writer.write(data[0]) catch |err| {
             a.err = err;
             return error.WriteFailed;

lib/std/Io/Reader.zig+6

...	@@ -179,6 +179,12 @@ pub fn streamExact(r: Reader, w: Writer, n: usize) StreamError!void {	...	@@ -179,6 +179,12 @@ pub fn streamExact(r: Reader, w: Writer, n: usize) StreamError!void {
179	while (remaining != 0) remaining -= try r.stream(w, .limited(remaining));	179	while (remaining != 0) remaining -= try r.stream(w, .limited(remaining));
180	}	180	}
181		181
		182	/// "Pump" exactly `n` bytes from the reader to the writer.
		183	pub fn streamExact64(r: Reader, w: Writer, n: u64) StreamError!void {
		184	var remaining = n;
		185	while (remaining != 0) remaining -= try r.stream(w, .limited64(remaining));
		186	}
		187
182	/// "Pump" data from the reader to the writer, handling `error.EndOfStream` as	188	/// "Pump" data from the reader to the writer, handling `error.EndOfStream` as
183	/// a success case.	189	/// a success case.
184	///	190	///

lib/std/Io/buffered_atomic_file.zig deleted-55

...	@@ -1,55 +0,0 @@
1	const std = @import("../std.zig");
2	const mem = std.mem;
3	const fs = std.fs;
4	const File = std.fs.File;
5
6	pub const BufferedAtomicFile = struct {
7	atomic_file: fs.AtomicFile,
8	file_writer: File.Writer,
9	buffered_writer: BufferedWriter,
10	allocator: mem.Allocator,
11
12	pub const buffer_size = 4096;
13	pub const BufferedWriter = std.io.BufferedWriter(buffer_size, File.Writer);
14	pub const Writer = std.io.GenericWriter(*BufferedWriter, BufferedWriter.Error, BufferedWriter.write);
15
16	/// TODO when https://github.com/ziglang/zig/issues/2761 is solved
17	/// this API will not need an allocator
18	pub fn create(
19	allocator: mem.Allocator,
20	dir: fs.Dir,
21	dest_path: []const u8,
22	atomic_file_options: fs.Dir.AtomicFileOptions,
23	) !*BufferedAtomicFile {
24	var self = try allocator.create(BufferedAtomicFile);
25	self.* = BufferedAtomicFile{
26	.atomic_file = undefined,
27	.file_writer = undefined,
28	.buffered_writer = undefined,
29	.allocator = allocator,
30	};
31	errdefer allocator.destroy(self);
32
33	self.atomic_file = try dir.atomicFile(dest_path, atomic_file_options);
34	errdefer self.atomic_file.deinit();
35
36	self.file_writer = self.atomic_file.file.deprecatedWriter();
37	self.buffered_writer = .{ .unbuffered_writer = self.file_writer };
38	return self;
39	}
40
41	/// always call destroy, even after successful finish()
42	pub fn destroy(self: *BufferedAtomicFile) void {
43	self.atomic_file.deinit();
44	self.allocator.destroy(self);
45	}
46
47	pub fn finish(self: *BufferedAtomicFile) !void {
48	try self.buffered_writer.flush();
49	try self.atomic_file.finish();
50	}
51
52	pub fn writer(self: *BufferedAtomicFile) Writer {
53	return .{ .context = &self.buffered_writer };
54	}
55	};

lib/std/Io/c_writer.zig deleted-44

...	@@ -1,44 +0,0 @@
1	const std = @import("../std.zig");
2	const builtin = @import("builtin");
3	const io = std.io;
4	const testing = std.testing;
5
6	pub const CWriter = io.GenericWriter(*std.c.FILE, std.fs.File.WriteError, cWriterWrite);
7
8	pub fn cWriter(c_file: *std.c.FILE) CWriter {
9	return .{ .context = c_file };
10	}
11
12	fn cWriterWrite(c_file: *std.c.FILE, bytes: []const u8) std.fs.File.WriteError!usize {
13	const amt_written = std.c.fwrite(bytes.ptr, 1, bytes.len, c_file);
14	if (amt_written >= 0) return amt_written;
15	switch (@as(std.c.E, @enumFromInt(std.c._errno().*))) {
16	.SUCCESS => unreachable,
17	.INVAL => unreachable,
18	.FAULT => unreachable,
19	.AGAIN => unreachable, // this is a blocking API
20	.BADF => unreachable, // always a race condition
21	.DESTADDRREQ => unreachable, // connect was never called
22	.DQUOT => return error.DiskQuota,
23	.FBIG => return error.FileTooBig,
24	.IO => return error.InputOutput,
25	.NOSPC => return error.NoSpaceLeft,
26	.PERM => return error.PermissionDenied,
27	.PIPE => return error.BrokenPipe,
28	else => \|err\| return std.posix.unexpectedErrno(err),
29	}
30	}
31
32	test cWriter {
33	if (!builtin.link_libc or builtin.os.tag == .wasi) return error.SkipZigTest;
34
35	const filename = "tmp_io_test_file.txt";
36	const out_file = std.c.fopen(filename, "w") orelse return error.UnableToOpenTestFile;
37	defer {
38	_ = std.c.fclose(out_file);
39	std.fs.cwd().deleteFileZ(filename) catch {};
40	}
41
42	const writer = cWriter(out_file);
43	try writer.print("hi: {}\n", .{@as(i32, 123)});
44	}

lib/std/Io/change_detection_stream.zig deleted-55

...	@@ -1,55 +0,0 @@
1	const std = @import("../std.zig");
2	const io = std.io;
3	const mem = std.mem;
4	const assert = std.debug.assert;
5
6	/// Used to detect if the data written to a stream differs from a source buffer
7	pub fn ChangeDetectionStream(comptime WriterType: type) type {
8	return struct {
9	const Self = @This();
10	pub const Error = WriterType.Error;
11	pub const Writer = io.GenericWriter(*Self, Error, write);
12
13	anything_changed: bool,
14	underlying_writer: WriterType,
15	source_index: usize,
16	source: []const u8,
17
18	pub fn writer(self: *Self) Writer {
19	return .{ .context = self };
20	}
21
22	fn write(self: *Self, bytes: []const u8) Error!usize {
23	if (!self.anything_changed) {
24	const end = self.source_index + bytes.len;
25	if (end > self.source.len) {
26	self.anything_changed = true;
27	} else {
28	const src_slice = self.source[self.source_index..end];
29	self.source_index += bytes.len;
30	if (!mem.eql(u8, bytes, src_slice)) {
31	self.anything_changed = true;
32	}
33	}
34	}
35
36	return self.underlying_writer.write(bytes);
37	}
38
39	pub fn changeDetected(self: *Self) bool {
40	return self.anything_changed or (self.source_index != self.source.len);
41	}
42	};
43	}
44
45	pub fn changeDetectionStream(
46	source: []const u8,
47	underlying_writer: anytype,
48	) ChangeDetectionStream(@TypeOf(underlying_writer)) {
49	return ChangeDetectionStream(@TypeOf(underlying_writer)){
50	.anything_changed = false,
51	.underlying_writer = underlying_writer,
52	.source_index = 0,
53	.source = source,
54	};
55	}

lib/std/Io/find_byte_writer.zig deleted-40

...	@@ -1,40 +0,0 @@
1	const std = @import("../std.zig");
2	const io = std.io;
3	const assert = std.debug.assert;
4
5	/// A Writer that returns whether the given character has been written to it.
6	/// The contents are not written to anything.
7	pub fn FindByteWriter(comptime UnderlyingWriter: type) type {
8	return struct {
9	const Self = @This();
10	pub const Error = UnderlyingWriter.Error;
11	pub const Writer = io.GenericWriter(*Self, Error, write);
12
13	underlying_writer: UnderlyingWriter,
14	byte_found: bool,
15	byte: u8,
16
17	pub fn writer(self: *Self) Writer {
18	return .{ .context = self };
19	}
20
21	fn write(self: *Self, bytes: []const u8) Error!usize {
22	if (!self.byte_found) {
23	self.byte_found = blk: {
24	for (bytes) \|b\|
25	if (b == self.byte) break :blk true;
26	break :blk false;
27	};
28	}
29	return self.underlying_writer.write(bytes);
30	}
31	};
32	}
33
34	pub fn findByteWriter(byte: u8, underlying_writer: anytype) FindByteWriter(@TypeOf(underlying_writer)) {
35	return FindByteWriter(@TypeOf(underlying_writer)){
36	.underlying_writer = underlying_writer,
37	.byte = byte,
38	.byte_found = false,
39	};
40	}

lib/std/Io/multi_writer.zig deleted-53

...	@@ -1,53 +0,0 @@
1	const std = @import("../std.zig");
2	const io = std.io;
3
4	/// Takes a tuple of streams, and constructs a new stream that writes to all of them
5	pub fn MultiWriter(comptime Writers: type) type {
6	comptime var ErrSet = error{};
7	inline for (@typeInfo(Writers).@"struct".fields) \|field\| {
8	const StreamType = field.type;
9	ErrSet = ErrSet \|\| StreamType.Error;
10	}
11
12	return struct {
13	const Self = @This();
14
15	streams: Writers,
16
17	pub const Error = ErrSet;
18	pub const Writer = io.GenericWriter(*Self, Error, write);
19
20	pub fn writer(self: *Self) Writer {
21	return .{ .context = self };
22	}
23
24	pub fn write(self: *Self, bytes: []const u8) Error!usize {
25	inline for (self.streams) \|stream\|
26	try stream.writeAll(bytes);
27	return bytes.len;
28	}
29	};
30	}
31
32	pub fn multiWriter(streams: anytype) MultiWriter(@TypeOf(streams)) {
33	return .{ .streams = streams };
34	}
35
36	const testing = std.testing;
37
38	test "MultiWriter" {
39	var tmp = testing.tmpDir(.{});
40	defer tmp.cleanup();
41	var f = try tmp.dir.createFile("t.txt", .{});
42
43	var buf1: [255]u8 = undefined;
44	var fbs1 = io.fixedBufferStream(&buf1);
45	var buf2: [255]u8 = undefined;
46	var stream = multiWriter(.{ fbs1.writer(), f.writer() });
47
48	try stream.writer().print("HI", .{});
49	f.close();
50
51	try testing.expectEqualSlices(u8, "HI", fbs1.getWritten());
52	try testing.expectEqualSlices(u8, "HI", try tmp.dir.readFile("t.txt", &buf2));
53	}

lib/std/Io/stream_source.zig deleted-127

...	@@ -1,127 +0,0 @@
1	const std = @import("../std.zig");
2	const builtin = @import("builtin");
3	const io = std.io;
4
5	/// Provides `io.GenericReader`, `io.GenericWriter`, and `io.SeekableStream` for in-memory buffers as
6	/// well as files.
7	/// For memory sources, if the supplied byte buffer is const, then `io.GenericWriter` is not available.
8	/// The error set of the stream functions is the error set of the corresponding file functions.
9	pub const StreamSource = union(enum) {
10	// TODO: expose UEFI files to std.os in a way that allows this to be true
11	const has_file = (builtin.os.tag != .freestanding and builtin.os.tag != .uefi);
12
13	/// The stream access is redirected to this buffer.
14	buffer: io.FixedBufferStream([]u8),
15
16	/// The stream access is redirected to this buffer.
17	/// Writing to the source will always yield `error.AccessDenied`.
18	const_buffer: io.FixedBufferStream([]const u8),
19
20	/// The stream access is redirected to this file.
21	/// On freestanding, this must never be initialized!
22	file: if (has_file) std.fs.File else void,
23
24	pub const ReadError = io.FixedBufferStream([]u8).ReadError \|\| (if (has_file) std.fs.File.ReadError else error{});
25	pub const WriteError = error{AccessDenied} \|\| io.FixedBufferStream([]u8).WriteError \|\| (if (has_file) std.fs.File.WriteError else error{});
26	pub const SeekError = io.FixedBufferStream([]u8).SeekError \|\| (if (has_file) std.fs.File.SeekError else error{});
27	pub const GetSeekPosError = io.FixedBufferStream([]u8).GetSeekPosError \|\| (if (has_file) std.fs.File.GetSeekPosError else error{});
28
29	pub const Reader = io.GenericReader(*StreamSource, ReadError, read);
30	pub const Writer = io.GenericWriter(*StreamSource, WriteError, write);
31	pub const SeekableStream = io.SeekableStream(
32	*StreamSource,
33	SeekError,
34	GetSeekPosError,
35	seekTo,
36	seekBy,
37	getPos,
38	getEndPos,
39	);
40
41	pub fn read(self: *StreamSource, dest: []u8) ReadError!usize {
42	switch (self.*) {
43	.buffer => \|*x\| return x.read(dest),
44	.const_buffer => \|*x\| return x.read(dest),
45	.file => \|x\| if (!has_file) unreachable else return x.read(dest),
46	}
47	}
48
49	pub fn write(self: *StreamSource, bytes: []const u8) WriteError!usize {
50	switch (self.*) {
51	.buffer => \|*x\| return x.write(bytes),
52	.const_buffer => return error.AccessDenied,
53	.file => \|x\| if (!has_file) unreachable else return x.write(bytes),
54	}
55	}
56
57	pub fn seekTo(self: *StreamSource, pos: u64) SeekError!void {
58	switch (self.*) {
59	.buffer => \|*x\| return x.seekTo(pos),
60	.const_buffer => \|*x\| return x.seekTo(pos),
61	.file => \|x\| if (!has_file) unreachable else return x.seekTo(pos),
62	}
63	}
64
65	pub fn seekBy(self: *StreamSource, amt: i64) SeekError!void {
66	switch (self.*) {
67	.buffer => \|*x\| return x.seekBy(amt),
68	.const_buffer => \|*x\| return x.seekBy(amt),
69	.file => \|x\| if (!has_file) unreachable else return x.seekBy(amt),
70	}
71	}
72
73	pub fn getEndPos(self: *StreamSource) GetSeekPosError!u64 {
74	switch (self.*) {
75	.buffer => \|*x\| return x.getEndPos(),
76	.const_buffer => \|*x\| return x.getEndPos(),
77	.file => \|x\| if (!has_file) unreachable else return x.getEndPos(),
78	}
79	}
80
81	pub fn getPos(self: *StreamSource) GetSeekPosError!u64 {
82	switch (self.*) {
83	.buffer => \|*x\| return x.getPos(),
84	.const_buffer => \|*x\| return x.getPos(),
85	.file => \|x\| if (!has_file) unreachable else return x.getPos(),
86	}
87	}
88
89	pub fn reader(self: *StreamSource) Reader {
90	return .{ .context = self };
91	}
92
93	pub fn writer(self: *StreamSource) Writer {
94	return .{ .context = self };
95	}
96
97	pub fn seekableStream(self: *StreamSource) SeekableStream {
98	return .{ .context = self };
99	}
100	};
101
102	test "refs" {
103	std.testing.refAllDecls(StreamSource);
104	}
105
106	test "mutable buffer" {
107	var buffer: [64]u8 = undefined;
108	var source = StreamSource{ .buffer = std.io.fixedBufferStream(&buffer) };
109
110	var writer = source.writer();
111
112	try writer.writeAll("Hello, World!");
113
114	try std.testing.expectEqualStrings("Hello, World!", source.buffer.getWritten());
115	}
116
117	test "const buffer" {
118	const buffer: [64]u8 = "Hello, World!".* ++ ([1]u8{0xAA} ** 51);
119	var source = StreamSource{ .const_buffer = std.io.fixedBufferStream(&buffer) };
120
121	var reader = source.reader();
122
123	var dst_buffer: [13]u8 = undefined;
124	try reader.readNoEof(&dst_buffer);
125
126	try std.testing.expectEqualStrings("Hello, World!", &dst_buffer);
127	}

lib/std/crypto/md5.zig+10-2

 @@ -54,12 +54,20 @@ pub const Md5 = struct {
         };
     }
-    pub fn hash(b: []const u8, out: *[digest_length]u8, options: Options) void {
+    pub fn hash(data: []const u8, out: *[digest_length]u8, options: Options) void {
         var d = Md5.init(options);
-        d.update(b);
+        d.update(data);
         d.final(out);
     }
+    pub fn hashResult(data: []const u8) [digest_length]u8 {
+        var out: [digest_length]u8 = undefined;
+        var d = Md5.init(.{});
+        d.update(data);
+        d.final(&out);
+        return out;
+    }
     pub fn update(d: *Self, b: []const u8) void {
         var off: usize = 0;

lib/std/tar.zig+283-329

...	@@ -19,7 +19,7 @@ const std = @import("std");	...	@@ -19,7 +19,7 @@ const std = @import("std");
19	const assert = std.debug.assert;	19	const assert = std.debug.assert;
20	const testing = std.testing;	20	const testing = std.testing;
21		21
22	pub const writer = @import("tar/writer.zig").writer;	22	pub const Writer = @import("tar/Writer.zig");
23		23
24	/// Provide this to receive detailed error messages.	24	/// Provide this to receive detailed error messages.
25	/// When this is provided, some errors which would otherwise be returned	25	/// When this is provided, some errors which would otherwise be returned
...	@@ -293,28 +293,6 @@ fn nullStr(str: []const u8) []const u8 {	...	@@ -293,28 +293,6 @@ fn nullStr(str: []const u8) []const u8 {
293	return str;	293	return str;
294	}	294	}
295		295
296	/// Options for iterator.
297	/// Buffers should be provided by the caller.
298	pub const IteratorOptions = struct {
299	/// Use a buffer with length `std.fs.max_path_bytes` to match file system capabilities.
300	file_name_buffer: []u8,
301	/// Use a buffer with length `std.fs.max_path_bytes` to match file system capabilities.
302	link_name_buffer: []u8,
303	/// Collects error messages during unpacking
304	diagnostics: ?*Diagnostics = null,
305	};
306
307	/// Iterates over files in tar archive.
308	/// `next` returns each file in tar archive.
309	pub fn iterator(reader: anytype, options: IteratorOptions) Iterator(@TypeOf(reader)) {
310	return .{
311	.reader = reader,
312	.diagnostics = options.diagnostics,
313	.file_name_buffer = options.file_name_buffer,
314	.link_name_buffer = options.link_name_buffer,
315	};
316	}
317
318	/// Type of the file returned by iterator `next` method.	296	/// Type of the file returned by iterator `next` method.
319	pub const FileKind = enum {	297	pub const FileKind = enum {
320	directory,	298	directory,
...	@@ -323,206 +301,192 @@ pub const FileKind = enum {	...	@@ -323,206 +301,192 @@ pub const FileKind = enum {
323	};	301	};
324		302
325	/// Iterator over entries in the tar file represented by reader.	303	/// Iterator over entries in the tar file represented by reader.
326	pub fn Iterator(comptime ReaderType: type) type {	304	pub const Iterator = struct {
327	return struct {	305	reader: *std.Io.Reader,
328	reader: ReaderType,	306	diagnostics: ?*Diagnostics = null,
329	diagnostics: ?*Diagnostics = null,
330
331	// buffers for heeader and file attributes
332	header_buffer: [Header.SIZE]u8 = undefined,
333	file_name_buffer: []u8,
334	link_name_buffer: []u8,
335
336	// bytes of padding to the end of the block
337	padding: usize = 0,
338	// not consumed bytes of file from last next iteration
339	unread_file_bytes: u64 = 0,
340
341	pub const File = struct {
342	name: []const u8, // name of file, symlink or directory
343	link_name: []const u8, // target name of symlink
344	size: u64 = 0, // size of the file in bytes
345	mode: u32 = 0,
346	kind: FileKind = .file,
347
348	unread_bytes: *u64,
349	parent_reader: ReaderType,
350
351	pub const Reader = std.io.GenericReader(File, ReaderType.Error, File.read);
352		307
353	pub fn reader(self: File) Reader {	308	// buffers for heeader and file attributes
354	return .{ .context = self };	309	header_buffer: [Header.SIZE]u8 = undefined,
355	}	310	file_name_buffer: []u8,
		311	link_name_buffer: []u8,
356		312
357	pub fn read(self: File, dest: []u8) ReaderType.Error!usize {	313	// bytes of padding to the end of the block
358	const buf = dest[0..@min(dest.len, self.unread_bytes.*)];	314	padding: usize = 0,
359	const n = try self.parent_reader.read(buf);	315	// not consumed bytes of file from last next iteration
360	self.unread_bytes.* -= n;	316	unread_file_bytes: u64 = 0,
361	return n;
362	}
363		317
364	// Writes file content to writer.	318	/// Options for iterator.
365	pub fn writeAll(self: File, out_writer: anytype) !void {	319	/// Buffers should be provided by the caller.
366	var buffer: [4096]u8 = undefined;	320	pub const Options = struct {
		321	/// Use a buffer with length `std.fs.max_path_bytes` to match file system capabilities.
		322	file_name_buffer: []u8,
		323	/// Use a buffer with length `std.fs.max_path_bytes` to match file system capabilities.
		324	link_name_buffer: []u8,
		325	/// Collects error messages during unpacking
		326	diagnostics: ?*Diagnostics = null,
		327	};
367		328
368	while (self.unread_bytes.* > 0) {	329	/// Iterates over files in tar archive.
369	const buf = buffer[0..@min(buffer.len, self.unread_bytes.*)];	330	/// `next` returns each file in tar archive.
370	try self.parent_reader.readNoEof(buf);	331	pub fn init(reader: *std.Io.Reader, options: Options) Iterator {
371	try out_writer.writeAll(buf);	332	return .{
372	self.unread_bytes.* -= buf.len;	333	.reader = reader,
373	}	334	.diagnostics = options.diagnostics,
374	}	335	.file_name_buffer = options.file_name_buffer,
		336	.link_name_buffer = options.link_name_buffer,
375	};	337	};
		338	}
376		339
377	const Self = @This();	340	pub const File = struct {
378		341	name: []const u8, // name of file, symlink or directory
379	fn readHeader(self: *Self) !?Header {	342	link_name: []const u8, // target name of symlink
380	if (self.padding > 0) {	343	size: u64 = 0, // size of the file in bytes
381	try self.reader.skipBytes(self.padding, .{});	344	mode: u32 = 0,
382	}	345	kind: FileKind = .file,
383	const n = try self.reader.readAll(&self.header_buffer);	346	};
384	if (n == 0) return null;
385	if (n < Header.SIZE) return error.UnexpectedEndOfStream;
386	const header = Header{ .bytes = self.header_buffer[0..Header.SIZE] };
387	if (try header.checkChksum() == 0) return null;
388	return header;
389	}
390		347
391	fn readString(self: *Self, size: usize, buffer: []u8) ![]const u8 {	348	fn readHeader(self: *Iterator) !?Header {
392	if (size > buffer.len) return error.TarInsufficientBuffer;	349	if (self.padding > 0) {
393	const buf = buffer[0..size];	350	try self.reader.discardAll(self.padding);
394	try self.reader.readNoEof(buf);
395	return nullStr(buf);
396	}	351	}
		352	const n = try self.reader.readSliceShort(&self.header_buffer);
		353	if (n == 0) return null;
		354	if (n < Header.SIZE) return error.UnexpectedEndOfStream;
		355	const header = Header{ .bytes = self.header_buffer[0..Header.SIZE] };
		356	if (try header.checkChksum() == 0) return null;
		357	return header;
		358	}
397		359
398	fn newFile(self: *Self) File {	360	fn readString(self: *Iterator, size: usize, buffer: []u8) ![]const u8 {
399	return .{	361	if (size > buffer.len) return error.TarInsufficientBuffer;
400	.name = self.file_name_buffer[0..0],	362	const buf = buffer[0..size];
401	.link_name = self.link_name_buffer[0..0],	363	try self.reader.readSliceAll(buf);
402	.parent_reader = self.reader,	364	return nullStr(buf);
403	.unread_bytes = &self.unread_file_bytes,	365	}
404	};
405	}
406		366
407	// Number of padding bytes in the last file block.	367	fn newFile(self: *Iterator) File {
408	fn blockPadding(size: u64) usize {	368	return .{
409	const block_rounded = std.mem.alignForward(u64, size, Header.SIZE); // size rounded to te block boundary	369	.name = self.file_name_buffer[0..0],
410	return @intCast(block_rounded - size);	370	.link_name = self.link_name_buffer[0..0],
411	}	371	};
		372	}
412		373
413	/// Iterates through the tar archive as if it is a series of files.	374	// Number of padding bytes in the last file block.
414	/// Internally, the tar format often uses entries (header with optional	375	fn blockPadding(size: u64) usize {
415	/// content) to add meta data that describes the next file. These	376	const block_rounded = std.mem.alignForward(u64, size, Header.SIZE); // size rounded to te block boundary
416	/// entries should not normally be visible to the outside. As such, this	377	return @intCast(block_rounded - size);
417	/// loop iterates through one or more entries until it collects a all	378	}
418	/// file attributes.
419	pub fn next(self: *Self) !?File {
420	if (self.unread_file_bytes > 0) {
421	// If file content was not consumed by caller
422	try self.reader.skipBytes(self.unread_file_bytes, .{});
423	self.unread_file_bytes = 0;
424	}
425	var file: File = self.newFile();
426
427	while (try self.readHeader()) \|header\| {
428	const kind = header.kind();
429	const size: u64 = try header.size();
430	self.padding = blockPadding(size);
431
432	switch (kind) {
433	// File types to return upstream
434	.directory, .normal, .symbolic_link => {
435	file.kind = switch (kind) {
436	.directory => .directory,
437	.normal => .file,
438	.symbolic_link => .sym_link,
439	else => unreachable,
440	};
441	file.mode = try header.mode();
442
443	// set file attributes if not already set by prefix/extended headers
444	if (file.size == 0) {
445	file.size = size;
446	}
447	if (file.link_name.len == 0) {
448	file.link_name = try header.linkName(self.link_name_buffer);
449	}
450	if (file.name.len == 0) {
451	file.name = try header.fullName(self.file_name_buffer);
452	}
453		379
454	self.padding = blockPadding(file.size);	380	/// Iterates through the tar archive as if it is a series of files.
455	self.unread_file_bytes = file.size;	381	/// Internally, the tar format often uses entries (header with optional
456	return file;	382	/// content) to add meta data that describes the next file. These
457	},	383	/// entries should not normally be visible to the outside. As such, this
458	// Prefix header types	384	/// loop iterates through one or more entries until it collects a all
459	.gnu_long_name => {	385	/// file attributes.
460	file.name = try self.readString(@intCast(size), self.file_name_buffer);	386	pub fn next(self: *Iterator) !?File {
461	},	387	if (self.unread_file_bytes > 0) {
462	.gnu_long_link => {	388	// If file content was not consumed by caller
463	file.link_name = try self.readString(@intCast(size), self.link_name_buffer);	389	try self.reader.discardAll64(self.unread_file_bytes);
464	},	390	self.unread_file_bytes = 0;
465	.extended_header => {	391	}
466	// Use just attributes from last extended header.	392	var file: File = self.newFile();
467	file = self.newFile();	393
468		394	while (try self.readHeader()) \|header\| {
469	var rdr = paxIterator(self.reader, @intCast(size));	395	const kind = header.kind();
470	while (try rdr.next()) \|attr\| {	396	const size: u64 = try header.size();
471	switch (attr.kind) {	397	self.padding = blockPadding(size);
472	.path => {	398
473	file.name = try attr.value(self.file_name_buffer);	399	switch (kind) {
474	},	400	// File types to return upstream
475	.linkpath => {	401	.directory, .normal, .symbolic_link => {
476	file.link_name = try attr.value(self.link_name_buffer);	402	file.kind = switch (kind) {
477	},	403	.directory => .directory,
478	.size => {	404	.normal => .file,
479	var buf: [pax_max_size_attr_len]u8 = undefined;	405	.symbolic_link => .sym_link,
480	file.size = try std.fmt.parseInt(u64, try attr.value(&buf), 10);	406	else => unreachable,
481	},	407	};
482	}	408	file.mode = try header.mode();
483	}	409
484	},	410	// set file attributes if not already set by prefix/extended headers
485	// Ignored header type	411	if (file.size == 0) {
486	.global_extended_header => {	412	file.size = size;
487	self.reader.skipBytes(size, .{}) catch return error.TarHeadersTooBig;	413	}
488	},	414	if (file.link_name.len == 0) {
489	// All other are unsupported header types	415	file.link_name = try header.linkName(self.link_name_buffer);
490	else => {	416	}
491	const d = self.diagnostics orelse return error.TarUnsupportedHeader;	417	if (file.name.len == 0) {
492	try d.errors.append(d.allocator, .{ .unsupported_file_type = .{	418	file.name = try header.fullName(self.file_name_buffer);
493	.file_name = try d.allocator.dupe(u8, header.name()),	419	}
494	.file_type = kind,	420
495	} });	421	self.padding = blockPadding(file.size);
496	if (kind == .gnu_sparse) {	422	self.unread_file_bytes = file.size;
497	try self.skipGnuSparseExtendedHeaders(header);	423	return file;
		424	},
		425	// Prefix header types
		426	.gnu_long_name => {
		427	file.name = try self.readString(@intCast(size), self.file_name_buffer);
		428	},
		429	.gnu_long_link => {
		430	file.link_name = try self.readString(@intCast(size), self.link_name_buffer);
		431	},
		432	.extended_header => {
		433	// Use just attributes from last extended header.
		434	file = self.newFile();
		435
		436	var rdr: PaxIterator = .{
		437	.reader = self.reader,
		438	.size = @intCast(size),
		439	};
		440	while (try rdr.next()) \|attr\| {
		441	switch (attr.kind) {
		442	.path => {
		443	file.name = try attr.value(self.file_name_buffer);
		444	},
		445	.linkpath => {
		446	file.link_name = try attr.value(self.link_name_buffer);
		447	},
		448	.size => {
		449	var buf: [pax_max_size_attr_len]u8 = undefined;
		450	file.size = try std.fmt.parseInt(u64, try attr.value(&buf), 10);
		451	},
498	}	452	}
499	self.reader.skipBytes(size, .{}) catch return error.TarHeadersTooBig;	453	}
500	},	454	},
501	}	455	// Ignored header type
		456	.global_extended_header => {
		457	self.reader.discardAll64(size) catch return error.TarHeadersTooBig;
		458	},
		459	// All other are unsupported header types
		460	else => {
		461	const d = self.diagnostics orelse return error.TarUnsupportedHeader;
		462	try d.errors.append(d.allocator, .{ .unsupported_file_type = .{
		463	.file_name = try d.allocator.dupe(u8, header.name()),
		464	.file_type = kind,
		465	} });
		466	if (kind == .gnu_sparse) {
		467	try self.skipGnuSparseExtendedHeaders(header);
		468	}
		469	self.reader.discardAll64(size) catch return error.TarHeadersTooBig;
		470	},
502	}	471	}
503	return null;
504	}	472	}
		473	return null;
		474	}
505		475
506	fn skipGnuSparseExtendedHeaders(self: *Self, header: Header) !void {	476	pub fn streamRemaining(it: Iterator, file: File, w: std.Io.Writer) std.Io.Reader.StreamError!void {
507	var is_extended = header.bytes[482] > 0;	477	try it.reader.streamExact64(w, file.size);
508	while (is_extended) {	478	it.unread_file_bytes = 0;
509	var buf: [Header.SIZE]u8 = undefined;	479	}
510	const n = try self.reader.readAll(&buf);
511	if (n < Header.SIZE) return error.UnexpectedEndOfStream;
512	is_extended = buf[504] > 0;
513	}
514	}
515	};
516	}
517		480
518	/// Pax attributes iterator.	481	fn skipGnuSparseExtendedHeaders(self: *Iterator, header: Header) !void {
519	/// Size is length of pax extended header in reader.	482	var is_extended = header.bytes[482] > 0;
520	fn paxIterator(reader: anytype, size: usize) PaxIterator(@TypeOf(reader)) {	483	while (is_extended) {
521	return PaxIterator(@TypeOf(reader)){	484	var buf: [Header.SIZE]u8 = undefined;
522	.reader = reader,	485	try self.reader.readSliceAll(&buf);
523	.size = size,	486	is_extended = buf[504] > 0;
524	};	487	}
525	}	488	}
		489	};
526		490
527	const PaxAttributeKind = enum {	491	const PaxAttributeKind = enum {
528	path,	492	path,
...	@@ -533,108 +497,99 @@ const PaxAttributeKind = enum {	...	@@ -533,108 +497,99 @@ const PaxAttributeKind = enum {
533	// maxInt(u64) has 20 chars, base 10 in practice we got 24 chars	497	// maxInt(u64) has 20 chars, base 10 in practice we got 24 chars
534	const pax_max_size_attr_len = 64;	498	const pax_max_size_attr_len = 64;
535		499
536	fn PaxIterator(comptime ReaderType: type) type {	500	pub const PaxIterator = struct {
537	return struct {	501	size: usize, // cumulative size of all pax attributes
538	size: usize, // cumulative size of all pax attributes	502	reader: *std.Io.Reader,
539	reader: ReaderType,
540	// scratch buffer used for reading attribute length and keyword
541	scratch: [128]u8 = undefined,
542
543	const Self = @This();
544
545	const Attribute = struct {
546	kind: PaxAttributeKind,
547	len: usize, // length of the attribute value
548	reader: ReaderType, // reader positioned at value start
549
550	// Copies pax attribute value into destination buffer.
551	// Must be called with destination buffer of size at least Attribute.len.
552	pub fn value(self: Attribute, dst: []u8) ![]const u8 {
553	if (self.len > dst.len) return error.TarInsufficientBuffer;
554	// assert(self.len <= dst.len);
555	const buf = dst[0..self.len];
556	const n = try self.reader.readAll(buf);
557	if (n < self.len) return error.UnexpectedEndOfStream;
558	try validateAttributeEnding(self.reader);
559	if (hasNull(buf)) return error.PaxNullInValue;
560	return buf;
561	}
562	};
563		503
564	// Iterates over pax attributes. Returns known only known attributes.	504	const Self = @This();
565	// Caller has to call value in Attribute, to advance reader across value.
566	pub fn next(self: *Self) !?Attribute {
567	// Pax extended header consists of one or more attributes, each constructed as follows:
568	// "%d %s=%s\n", <length>, <keyword>, <value>
569	while (self.size > 0) {
570	const length_buf = try self.readUntil(' ');
571	const length = try std.fmt.parseInt(usize, length_buf, 10); // record length in bytes
572
573	const keyword = try self.readUntil('=');
574	if (hasNull(keyword)) return error.PaxNullInKeyword;
575
576	// calculate value_len
577	const value_start = length_buf.len + keyword.len + 2; // 2 separators
578	if (length < value_start + 1 or self.size < length) return error.UnexpectedEndOfStream;
579	const value_len = length - value_start - 1; // \n separator at end
580	self.size -= length;
581
582	const kind: PaxAttributeKind = if (eql(keyword, "path"))
583	.path
584	else if (eql(keyword, "linkpath"))
585	.linkpath
586	else if (eql(keyword, "size"))
587	.size
588	else {
589	try self.reader.skipBytes(value_len, .{});
590	try validateAttributeEnding(self.reader);
591	continue;
592	};
593	if (kind == .size and value_len > pax_max_size_attr_len) {
594	return error.PaxSizeAttrOverflow;
595	}
596	return Attribute{
597	.kind = kind,
598	.len = value_len,
599	.reader = self.reader,
600	};
601	}
602		505
603	return null;	506	const Attribute = struct {
		507	kind: PaxAttributeKind,
		508	len: usize, // length of the attribute value
		509	reader: *std.Io.Reader, // reader positioned at value start
		510
		511	// Copies pax attribute value into destination buffer.
		512	// Must be called with destination buffer of size at least Attribute.len.
		513	pub fn value(self: Attribute, dst: []u8) ![]const u8 {
		514	if (self.len > dst.len) return error.TarInsufficientBuffer;
		515	// assert(self.len <= dst.len);
		516	const buf = dst[0..self.len];
		517	const n = try self.reader.readSliceShort(buf);
		518	if (n < self.len) return error.UnexpectedEndOfStream;
		519	try validateAttributeEnding(self.reader);
		520	if (hasNull(buf)) return error.PaxNullInValue;
		521	return buf;
604	}	522	}
		523	};
605		524
606	fn readUntil(self: *Self, delimiter: u8) ![]const u8 {	525	// Iterates over pax attributes. Returns known only known attributes.
607	var fbs = std.io.fixedBufferStream(&self.scratch);	526	// Caller has to call value in Attribute, to advance reader across value.
608	try self.reader.streamUntilDelimiter(fbs.writer(), delimiter, null);	527	pub fn next(self: *Self) !?Attribute {
609	return fbs.getWritten();	528	// Pax extended header consists of one or more attributes, each constructed as follows:
		529	// "%d %s=%s\n", <length>, <keyword>, <value>
		530	while (self.size > 0) {
		531	const length_buf = try self.reader.takeSentinel(' ');
		532	const length = try std.fmt.parseInt(usize, length_buf, 10); // record length in bytes
		533
		534	const keyword = try self.reader.takeSentinel('=');
		535	if (hasNull(keyword)) return error.PaxNullInKeyword;
		536
		537	// calculate value_len
		538	const value_start = length_buf.len + keyword.len + 2; // 2 separators
		539	if (length < value_start + 1 or self.size < length) return error.UnexpectedEndOfStream;
		540	const value_len = length - value_start - 1; // \n separator at end
		541	self.size -= length;
		542
		543	const kind: PaxAttributeKind = if (eql(keyword, "path"))
		544	.path
		545	else if (eql(keyword, "linkpath"))
		546	.linkpath
		547	else if (eql(keyword, "size"))
		548	.size
		549	else {
		550	try self.reader.discardAll(value_len);
		551	try validateAttributeEnding(self.reader);
		552	continue;
		553	};
		554	if (kind == .size and value_len > pax_max_size_attr_len) {
		555	return error.PaxSizeAttrOverflow;
		556	}
		557	return .{
		558	.kind = kind,
		559	.len = value_len,
		560	.reader = self.reader,
		561	};
610	}	562	}
611		563
612	fn eql(a: []const u8, b: []const u8) bool {	564	return null;
613	return std.mem.eql(u8, a, b);	565	}
614	}
615		566
616	fn hasNull(str: []const u8) bool {	567	fn eql(a: []const u8, b: []const u8) bool {
617	return (std.mem.indexOfScalar(u8, str, 0)) != null;	568	return std.mem.eql(u8, a, b);
618	}	569	}
619		570
620	// Checks that each record ends with new line.	571	fn hasNull(str: []const u8) bool {
621	fn validateAttributeEnding(reader: ReaderType) !void {	572	return (std.mem.indexOfScalar(u8, str, 0)) != null;
622	if (try reader.readByte() != '\n') return error.PaxInvalidAttributeEnd;	573	}
623	}	574
624	};	575	// Checks that each record ends with new line.
625	}	576	fn validateAttributeEnding(reader: *std.Io.Reader) !void {
		577	if (try reader.takeByte() != '\n') return error.PaxInvalidAttributeEnd;
		578	}
		579	};
626		580
627	/// Saves tar file content to the file systems.	581	/// Saves tar file content to the file systems.
628	pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: PipeOptions) !void {	582	pub fn pipeToFileSystem(dir: std.fs.Dir, reader: *std.Io.Reader, options: PipeOptions) !void {
629	var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;	583	var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
630	var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;	584	var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
631	var iter = iterator(reader, .{	585	var file_contents_buffer: [1024]u8 = undefined;
		586	var it: Iterator = .init(reader, .{
632	.file_name_buffer = &file_name_buffer,	587	.file_name_buffer = &file_name_buffer,
633	.link_name_buffer = &link_name_buffer,	588	.link_name_buffer = &link_name_buffer,
634	.diagnostics = options.diagnostics,	589	.diagnostics = options.diagnostics,
635	});	590	});
636		591
637	while (try iter.next()) \|file\| {	592	while (try it.next()) \|file\| {
638	const file_name = stripComponents(file.name, options.strip_components);	593	const file_name = stripComponents(file.name, options.strip_components);
639	if (file_name.len == 0 and file.kind != .directory) {	594	if (file_name.len == 0 and file.kind != .directory) {
640	const d = options.diagnostics orelse return error.TarComponentsOutsideStrippedPrefix;	595	const d = options.diagnostics orelse return error.TarComponentsOutsideStrippedPrefix;
...	@@ -656,7 +611,9 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: PipeOptions)	...	@@ -656,7 +611,9 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: PipeOptions)
656	.file => {	611	.file => {
657	if (createDirAndFile(dir, file_name, fileMode(file.mode, options))) \|fs_file\| {	612	if (createDirAndFile(dir, file_name, fileMode(file.mode, options))) \|fs_file\| {
658	defer fs_file.close();	613	defer fs_file.close();
659	try file.writeAll(fs_file);	614	var file_writer = fs_file.writer(&file_contents_buffer);
		615	try it.streamRemaining(file, &file_writer.interface);
		616	try file_writer.interface.flush();
660	} else \|err\| {	617	} else \|err\| {
661	const d = options.diagnostics orelse return err;	618	const d = options.diagnostics orelse return err;
662	try d.errors.append(d.allocator, .{ .unable_to_create_file = .{	619	try d.errors.append(d.allocator, .{ .unable_to_create_file = .{
...	@@ -826,11 +783,14 @@ test PaxIterator {	...	@@ -826,11 +783,14 @@ test PaxIterator {
826	var buffer: [1024]u8 = undefined;	783	var buffer: [1024]u8 = undefined;
827		784
828	outer: for (cases) \|case\| {	785	outer: for (cases) \|case\| {
829	var stream = std.io.fixedBufferStream(case.data);	786	var reader: std.Io.Reader = .fixed(case.data);
830	var iter = paxIterator(stream.reader(), case.data.len);	787	var it: PaxIterator = .{
		788	.size = case.data.len,
		789	.reader = &reader,
		790	};
831		791
832	var i: usize = 0;	792	var i: usize = 0;
833	while (iter.next() catch \|err\| {	793	while (it.next() catch \|err\| {
834	if (case.err) \|e\| {	794	if (case.err) \|e\| {
835	try testing.expectEqual(e, err);	795	try testing.expectEqual(e, err);
836	continue;	796	continue;
...	@@ -853,12 +813,6 @@ test PaxIterator {	...	@@ -853,12 +813,6 @@ test PaxIterator {
853	}	813	}
854	}	814	}
855		815
856	test {
857	_ = @import("tar/test.zig");
858	_ = @import("tar/writer.zig");
859	_ = Diagnostics;
860	}
861
862	test "header parse size" {	816	test "header parse size" {
863	const cases = [_]struct {	817	const cases = [_]struct {
864	in: []const u8,	818	in: []const u8,
...	@@ -941,7 +895,7 @@ test "create file and symlink" {	...	@@ -941,7 +895,7 @@ test "create file and symlink" {
941	file.close();	895	file.close();
942	}	896	}
943		897
944	test iterator {	898	test Iterator {
945	// Example tar file is created from this tree structure:	899	// Example tar file is created from this tree structure:
946	// $ tree example	900	// $ tree example
947	// example	901	// example
...	@@ -962,19 +916,19 @@ test iterator {	...	@@ -962,19 +916,19 @@ test iterator {
962	// example/empty/	916	// example/empty/
963		917
964	const data = @embedFile("tar/testdata/example.tar");	918	const data = @embedFile("tar/testdata/example.tar");
965	var fbs = std.io.fixedBufferStream(data);	919	var reader: std.Io.Reader = .fixed(data);
966		920
967	// User provided buffers to the iterator	921	// User provided buffers to the iterator
968	var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;	922	var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
969	var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;	923	var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
970	// Create iterator	924	// Create iterator
971	var iter = iterator(fbs.reader(), .{	925	var it: Iterator = .init(&reader, .{
972	.file_name_buffer = &file_name_buffer,	926	.file_name_buffer = &file_name_buffer,
973	.link_name_buffer = &link_name_buffer,	927	.link_name_buffer = &link_name_buffer,
974	});	928	});
975	// Iterate over files in example.tar	929	// Iterate over files in example.tar
976	var file_no: usize = 0;	930	var file_no: usize = 0;
977	while (try iter.next()) \|file\| : (file_no += 1) {	931	while (try it.next()) \|file\| : (file_no += 1) {
978	switch (file.kind) {	932	switch (file.kind) {
979	.directory => {	933	.directory => {
980	switch (file_no) {	934	switch (file_no) {
...	@@ -987,10 +941,10 @@ test iterator {	...	@@ -987,10 +941,10 @@ test iterator {
987	},	941	},
988	.file => {	942	.file => {
989	try testing.expectEqualStrings("example/a/file", file.name);	943	try testing.expectEqualStrings("example/a/file", file.name);
990	// Read file content
991	var buf: [16]u8 = undefined;	944	var buf: [16]u8 = undefined;
992	const n = try file.reader().readAll(&buf);	945	var w: std.Io.Writer = .fixed(&buf);
993	try testing.expectEqualStrings("content\n", buf[0..n]);	946	try it.streamRemaining(file, &w);
		947	try testing.expectEqualStrings("content\n", w.buffered());
994	},	948	},
995	.sym_link => {	949	.sym_link => {
996	try testing.expectEqualStrings("example/b/symlink", file.name);	950	try testing.expectEqualStrings("example/b/symlink", file.name);
...	@@ -1021,15 +975,14 @@ test pipeToFileSystem {	...	@@ -1021,15 +975,14 @@ test pipeToFileSystem {
1021	// example/empty/	975	// example/empty/
1022		976
1023	const data = @embedFile("tar/testdata/example.tar");	977	const data = @embedFile("tar/testdata/example.tar");
1024	var fbs = std.io.fixedBufferStream(data);	978	var reader: std.Io.Reader = .fixed(data);
1025	const reader = fbs.reader();
1026		979
1027	var tmp = testing.tmpDir(.{ .no_follow = true });	980	var tmp = testing.tmpDir(.{ .no_follow = true });
1028	defer tmp.cleanup();	981	defer tmp.cleanup();
1029	const dir = tmp.dir;	982	const dir = tmp.dir;
1030		983
1031	// Save tar from `reader` to the file system `dir`	984	// Save tar from reader to the file system `dir`
1032	pipeToFileSystem(dir, reader, .{	985	pipeToFileSystem(dir, &reader, .{
1033	.mode_mode = .ignore,	986	.mode_mode = .ignore,
1034	.strip_components = 1,	987	.strip_components = 1,
1035	.exclude_empty_directories = true,	988	.exclude_empty_directories = true,
...	@@ -1053,8 +1006,7 @@ test pipeToFileSystem {	...	@@ -1053,8 +1006,7 @@ test pipeToFileSystem {
1053		1006
1054	test "pipeToFileSystem root_dir" {	1007	test "pipeToFileSystem root_dir" {
1055	const data = @embedFile("tar/testdata/example.tar");	1008	const data = @embedFile("tar/testdata/example.tar");
1056	var fbs = std.io.fixedBufferStream(data);	1009	var reader: std.Io.Reader = .fixed(data);
1057	const reader = fbs.reader();
1058		1010
1059	// with strip_components = 1	1011	// with strip_components = 1
1060	{	1012	{
...	@@ -1063,7 +1015,7 @@ test "pipeToFileSystem root_dir" {	...	@@ -1063,7 +1015,7 @@ test "pipeToFileSystem root_dir" {
1063	var diagnostics: Diagnostics = .{ .allocator = testing.allocator };	1015	var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1064	defer diagnostics.deinit();	1016	defer diagnostics.deinit();
1065		1017
1066	pipeToFileSystem(tmp.dir, reader, .{	1018	pipeToFileSystem(tmp.dir, &reader, .{
1067	.strip_components = 1,	1019	.strip_components = 1,
1068	.diagnostics = &diagnostics,	1020	.diagnostics = &diagnostics,
1069	}) catch \|err\| {	1021	}) catch \|err\| {
...	@@ -1079,13 +1031,13 @@ test "pipeToFileSystem root_dir" {	...	@@ -1079,13 +1031,13 @@ test "pipeToFileSystem root_dir" {
1079		1031
1080	// with strip_components = 0	1032	// with strip_components = 0
1081	{	1033	{
1082	fbs.reset();	1034	reader = .fixed(data);
1083	var tmp = testing.tmpDir(.{ .no_follow = true });	1035	var tmp = testing.tmpDir(.{ .no_follow = true });
1084	defer tmp.cleanup();	1036	defer tmp.cleanup();
1085	var diagnostics: Diagnostics = .{ .allocator = testing.allocator };	1037	var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1086	defer diagnostics.deinit();	1038	defer diagnostics.deinit();
1087		1039
1088	pipeToFileSystem(tmp.dir, reader, .{	1040	pipeToFileSystem(tmp.dir, &reader, .{
1089	.strip_components = 0,	1041	.strip_components = 0,
1090	.diagnostics = &diagnostics,	1042	.diagnostics = &diagnostics,
1091	}) catch \|err\| {	1043	}) catch \|err\| {
...	@@ -1102,45 +1054,42 @@ test "pipeToFileSystem root_dir" {	...	@@ -1102,45 +1054,42 @@ test "pipeToFileSystem root_dir" {
1102		1054
1103	test "findRoot with single file archive" {	1055	test "findRoot with single file archive" {
1104	const data = @embedFile("tar/testdata/22752.tar");	1056	const data = @embedFile("tar/testdata/22752.tar");
1105	var fbs = std.io.fixedBufferStream(data);	1057	var reader: std.Io.Reader = .fixed(data);
1106	const reader = fbs.reader();
1107		1058
1108	var tmp = testing.tmpDir(.{});	1059	var tmp = testing.tmpDir(.{});
1109	defer tmp.cleanup();	1060	defer tmp.cleanup();
1110		1061
1111	var diagnostics: Diagnostics = .{ .allocator = testing.allocator };	1062	var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1112	defer diagnostics.deinit();	1063	defer diagnostics.deinit();
1113	try pipeToFileSystem(tmp.dir, reader, .{ .diagnostics = &diagnostics });	1064	try pipeToFileSystem(tmp.dir, &reader, .{ .diagnostics = &diagnostics });
1114		1065
1115	try testing.expectEqualStrings("", diagnostics.root_dir);	1066	try testing.expectEqualStrings("", diagnostics.root_dir);
1116	}	1067	}
1117		1068
1118	test "findRoot without explicit root dir" {	1069	test "findRoot without explicit root dir" {
1119	const data = @embedFile("tar/testdata/19820.tar");	1070	const data = @embedFile("tar/testdata/19820.tar");
1120	var fbs = std.io.fixedBufferStream(data);	1071	var reader: std.Io.Reader = .fixed(data);
1121	const reader = fbs.reader();
1122		1072
1123	var tmp = testing.tmpDir(.{});	1073	var tmp = testing.tmpDir(.{});
1124	defer tmp.cleanup();	1074	defer tmp.cleanup();
1125		1075
1126	var diagnostics: Diagnostics = .{ .allocator = testing.allocator };	1076	var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1127	defer diagnostics.deinit();	1077	defer diagnostics.deinit();
1128	try pipeToFileSystem(tmp.dir, reader, .{ .diagnostics = &diagnostics });	1078	try pipeToFileSystem(tmp.dir, &reader, .{ .diagnostics = &diagnostics });
1129		1079
1130	try testing.expectEqualStrings("root", diagnostics.root_dir);	1080	try testing.expectEqualStrings("root", diagnostics.root_dir);
1131	}	1081	}
1132		1082
1133	test "pipeToFileSystem strip_components" {	1083	test "pipeToFileSystem strip_components" {
1134	const data = @embedFile("tar/testdata/example.tar");	1084	const data = @embedFile("tar/testdata/example.tar");
1135	var fbs = std.io.fixedBufferStream(data);	1085	var reader: std.Io.Reader = .fixed(data);
1136	const reader = fbs.reader();
1137		1086
1138	var tmp = testing.tmpDir(.{ .no_follow = true });	1087	var tmp = testing.tmpDir(.{ .no_follow = true });
1139	defer tmp.cleanup();	1088	defer tmp.cleanup();
1140	var diagnostics: Diagnostics = .{ .allocator = testing.allocator };	1089	var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1141	defer diagnostics.deinit();	1090	defer diagnostics.deinit();
1142		1091
1143	pipeToFileSystem(tmp.dir, reader, .{	1092	pipeToFileSystem(tmp.dir, &reader, .{
1144	.strip_components = 3,	1093	.strip_components = 3,
1145	.diagnostics = &diagnostics,	1094	.diagnostics = &diagnostics,
1146	}) catch \|err\| {	1095	}) catch \|err\| {
...	@@ -1194,13 +1143,12 @@ test "executable bit" {	...	@@ -1194,13 +1143,12 @@ test "executable bit" {
1194	const data = @embedFile("tar/testdata/example.tar");	1143	const data = @embedFile("tar/testdata/example.tar");
1195		1144
1196	for ([_]PipeOptions.ModeMode{ .ignore, .executable_bit_only }) \|opt\| {	1145	for ([_]PipeOptions.ModeMode{ .ignore, .executable_bit_only }) \|opt\| {
1197	var fbs = std.io.fixedBufferStream(data);	1146	var reader: std.Io.Reader = .fixed(data);
1198	const reader = fbs.reader();
1199		1147
1200	var tmp = testing.tmpDir(.{ .no_follow = true });	1148	var tmp = testing.tmpDir(.{ .no_follow = true });
1201	//defer tmp.cleanup();	1149	//defer tmp.cleanup();
1202		1150
1203	pipeToFileSystem(tmp.dir, reader, .{	1151	pipeToFileSystem(tmp.dir, &reader, .{
1204	.strip_components = 1,	1152	.strip_components = 1,
1205	.exclude_empty_directories = true,	1153	.exclude_empty_directories = true,
1206	.mode_mode = opt,	1154	.mode_mode = opt,
...	@@ -1226,3 +1174,9 @@ test "executable bit" {	...	@@ -1226,3 +1174,9 @@ test "executable bit" {
1226	}	1174	}
1227	}	1175	}
1228	}	1176	}
		1177
		1178	test {
		1179	_ = @import("tar/test.zig");
		1180	_ = Writer;
		1181	_ = Diagnostics;
		1182	}

lib/std/tar/Writer.zig created+462

...	@@ -0,0 +1,462 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const testing = std.testing;
4	const Writer = @This();
5
6	const block_size = @sizeOf(Header);
7
8	/// Options for writing file/dir/link. If left empty 0o664 is used for
9	/// file mode and current time for mtime.
10	pub const Options = struct {
11	/// File system permission mode.
12	mode: u32 = 0,
13	/// File system modification time.
14	mtime: u64 = 0,
15	};
16
17	underlying_writer: *std.Io.Writer,
18	prefix: []const u8 = "",
19	mtime_now: u64 = 0,
20
21	const Error = error{
22	WriteFailed,
23	OctalOverflow,
24	NameTooLong,
25	};
26
27	/// Sets prefix for all other write* method paths.
28	pub fn setRoot(w: *Writer, root: []const u8) Error!void {
29	if (root.len > 0)
30	try w.writeDir(root, .{});
31
32	w.prefix = root;
33	}
34
35	pub fn writeDir(w: *Writer, sub_path: []const u8, options: Options) Error!void {
36	try w.writeHeader(.directory, sub_path, "", 0, options);
37	}
38
39	pub const WriteFileError = std.Io.Writer.FileError \|\| Error \|\| std.fs.File.GetEndPosError;
40
41	pub fn writeFile(
42	w: *Writer,
43	sub_path: []const u8,
44	file_reader: *std.fs.File.Reader,
45	stat_mtime: i128,
46	) WriteFileError!void {
47	const size = try file_reader.getSize();
48	const mtime: u64 = @intCast(@divFloor(stat_mtime, std.time.ns_per_s));
49
50	var header: Header = .{};
51	try w.setPath(&header, sub_path);
52	try header.setSize(size);
53	try header.setMtime(mtime);
54	try header.updateChecksum();
55
56	try w.underlying_writer.writeAll(@ptrCast((&header)[0..1]));
57	_ = try w.underlying_writer.sendFileAll(file_reader, .unlimited);
58	try w.writePadding64(size);
59	}
60
61	pub const WriteFileStreamError = Error \|\| std.Io.Reader.StreamError;
62
63	/// Writes file reading file content from `reader`. Reads exactly `size` bytes
64	/// from `reader`, or returns `error.EndOfStream`.
65	pub fn writeFileStream(
66	w: *Writer,
67	sub_path: []const u8,
68	size: u64,
69	reader: *std.Io.Reader,
70	options: Options,
71	) WriteFileStreamError!void {
72	try w.writeHeader(.regular, sub_path, "", size, options);
73	try reader.streamExact64(w.underlying_writer, size);
74	try w.writePadding64(size);
75	}
76
77	/// Writes file using bytes buffer `content` for size and file content.
78	pub fn writeFileBytes(w: *Writer, sub_path: []const u8, content: []const u8, options: Options) Error!void {
79	try w.writeHeader(.regular, sub_path, "", content.len, options);
80	try w.underlying_writer.writeAll(content);
81	try w.writePadding(content.len);
82	}
83
84	pub fn writeLink(w: *Writer, sub_path: []const u8, link_name: []const u8, options: Options) Error!void {
85	try w.writeHeader(.symbolic_link, sub_path, link_name, 0, options);
86	}
87
88	fn writeHeader(
89	w: *Writer,
90	typeflag: Header.FileType,
91	sub_path: []const u8,
92	link_name: []const u8,
93	size: u64,
94	options: Options,
95	) Error!void {
96	var header = Header.init(typeflag);
97	try w.setPath(&header, sub_path);
98	try header.setSize(size);
99	try header.setMtime(options.mtime);
100	if (options.mode != 0)
101	try header.setMode(options.mode);
102	if (typeflag == .symbolic_link)
103	header.setLinkname(link_name) catch \|err\| switch (err) {
104	error.NameTooLong => try w.writeExtendedHeader(.gnu_long_link, &.{link_name}),
105	else => return err,
106	};
107	try header.write(w.underlying_writer);
108	}
109
110	/// Writes path in posix header, if don't fit (in name+prefix; 100+155
111	/// bytes) writes it in gnu extended header.
112	fn setPath(w: Writer, header: Header, sub_path: []const u8) Error!void {
113	header.setPath(w.prefix, sub_path) catch \|err\| switch (err) {
114	error.NameTooLong => {
115	// write extended header
116	const buffers: []const []const u8 = if (w.prefix.len == 0)
117	&.{sub_path}
118	else
119	&.{ w.prefix, "/", sub_path };
120	try w.writeExtendedHeader(.gnu_long_name, buffers);
121	},
122	else => return err,
123	};
124	}
125
126	/// Writes gnu extended header: gnu_long_name or gnu_long_link.
127	fn writeExtendedHeader(w: *Writer, typeflag: Header.FileType, buffers: []const []const u8) Error!void {
128	var len: usize = 0;
129	for (buffers) \|buf\| len += buf.len;
130
131	var header: Header = .init(typeflag);
132	try header.setSize(len);
133	try header.write(w.underlying_writer);
134	for (buffers) \|buf\|
135	try w.underlying_writer.writeAll(buf);
136	try w.writePadding(len);
137	}
138
139	fn writePadding(w: *Writer, bytes: usize) std.Io.Writer.Error!void {
140	return writePaddingPos(w, bytes % block_size);
141	}
142
143	fn writePadding64(w: *Writer, bytes: u64) std.Io.Writer.Error!void {
144	return writePaddingPos(w, @intCast(bytes % block_size));
145	}
146
147	fn writePaddingPos(w: *Writer, pos: usize) std.Io.Writer.Error!void {
148	if (pos == 0) return;
149	try w.underlying_writer.splatByteAll(0, block_size - pos);
150	}
151
152	/// According to the specification, tar should finish with two zero blocks, but
153	/// "reasonable system must not assume that such a block exists when reading an
154	/// archive". Therefore, the Zig standard library recommends to not call this
155	/// function.
156	pub fn finishPedantically(w: *Writer) std.Io.Writer.Error!void {
157	try w.underlying_writer.splatByteAll(0, block_size * 2);
158	}
159
160	/// A struct that is exactly 512 bytes and matches tar file format. This is
161	/// intended to be used for outputting tar files; for parsing there is
162	/// `std.tar.Header`.
163	pub const Header = extern struct {
164	// This struct was originally copied from
165	// https://github.com/mattnite/tar/blob/main/src/main.zig which is MIT
166	// licensed.
167	//
168	// The name, linkname, magic, uname, and gname are null-terminated character
169	// strings. All other fields are zero-filled octal numbers in ASCII. Each
170	// numeric field of width w contains w minus 1 digits, and a null.
171	// Reference: https://www.gnu.org/software/tar/manual/html_node/Standard.html
172	// POSIX header: byte offset
173	name: [100]u8 = [_]u8{0} ** 100, // 0
174	mode: [7:0]u8 = default_mode.file, // 100
175	uid: [7:0]u8 = [_:0]u8{0} ** 7, // unused 108
176	gid: [7:0]u8 = [_:0]u8{0} ** 7, // unused 116
177	size: [11:0]u8 = [_:0]u8{'0'} ** 11, // 124
178	mtime: [11:0]u8 = [_:0]u8{'0'} ** 11, // 136
179	checksum: [7:0]u8 = [_:0]u8{' '} ** 7, // 148
180	typeflag: FileType = .regular, // 156
181	linkname: [100]u8 = [_]u8{0} ** 100, // 157
182	magic: [6]u8 = [_]u8{ 'u', 's', 't', 'a', 'r', 0 }, // 257
183	version: [2]u8 = [_]u8{ '0', '0' }, // 263
184	uname: [32]u8 = [_]u8{0} ** 32, // unused 265
185	gname: [32]u8 = [_]u8{0} ** 32, // unused 297
186	devmajor: [7:0]u8 = [_:0]u8{0} ** 7, // unused 329
187	devminor: [7:0]u8 = [_:0]u8{0} ** 7, // unused 337
188	prefix: [155]u8 = [_]u8{0} ** 155, // 345
189	pad: [12]u8 = [_]u8{0} ** 12, // unused 500
190
191	pub const FileType = enum(u8) {
192	regular = '0',
193	symbolic_link = '2',
194	directory = '5',
195	gnu_long_name = 'L',
196	gnu_long_link = 'K',
197	};
198
199	const default_mode = struct {
200	const file = [_:0]u8{ '0', '0', '0', '0', '6', '6', '4' }; // 0o664
201	const dir = [_:0]u8{ '0', '0', '0', '0', '7', '7', '5' }; // 0o775
202	const sym_link = [_:0]u8{ '0', '0', '0', '0', '7', '7', '7' }; // 0o777
203	const other = [_:0]u8{ '0', '0', '0', '0', '0', '0', '0' }; // 0o000
204	};
205
206	pub fn init(typeflag: FileType) Header {
207	return .{
208	.typeflag = typeflag,
209	.mode = switch (typeflag) {
210	.directory => default_mode.dir,
211	.symbolic_link => default_mode.sym_link,
212	.regular => default_mode.file,
213	else => default_mode.other,
214	},
215	};
216	}
217
218	pub fn setSize(w: *Header, size: u64) error{OctalOverflow}!void {
219	try octal(&w.size, size);
220	}
221
222	fn octal(buf: []u8, value: u64) error{OctalOverflow}!void {
223	var remainder: u64 = value;
224	var pos: usize = buf.len;
225	while (remainder > 0 and pos > 0) {
226	pos -= 1;
227	const c: u8 = @as(u8, @intCast(remainder % 8)) + '0';
228	buf[pos] = c;
229	remainder /= 8;
230	if (pos == 0 and remainder > 0) return error.OctalOverflow;
231	}
232	}
233
234	pub fn setMode(w: *Header, mode: u32) error{OctalOverflow}!void {
235	try octal(&w.mode, mode);
236	}
237
238	// Integer number of seconds since January 1, 1970, 00:00 Coordinated Universal Time.
239	// mtime == 0 will use current time
240	pub fn setMtime(w: *Header, mtime: u64) error{OctalOverflow}!void {
241	try octal(&w.mtime, mtime);
242	}
243
244	pub fn updateChecksum(w: *Header) !void {
245	var checksum: usize = ' '; // other 7 w.checksum bytes are initialized to ' '
246	for (std.mem.asBytes(w)) \|val\|
247	checksum += val;
248	try octal(&w.checksum, checksum);
249	}
250
251	pub fn write(h: Header, bw: std.Io.Writer) error{ OctalOverflow, WriteFailed }!void {
252	try h.updateChecksum();
253	try bw.writeAll(std.mem.asBytes(h));
254	}
255
256	pub fn setLinkname(w: *Header, link: []const u8) !void {
257	if (link.len > w.linkname.len) return error.NameTooLong;
258	@memcpy(w.linkname[0..link.len], link);
259	}
260
261	pub fn setPath(w: *Header, prefix: []const u8, sub_path: []const u8) !void {
262	const max_prefix = w.prefix.len;
263	const max_name = w.name.len;
264	const sep = std.fs.path.sep_posix;
265
266	if (prefix.len + sub_path.len > max_name + max_prefix or prefix.len > max_prefix)
267	return error.NameTooLong;
268
269	// both fit into name
270	if (prefix.len > 0 and prefix.len + sub_path.len < max_name) {
271	@memcpy(w.name[0..prefix.len], prefix);
272	w.name[prefix.len] = sep;
273	@memcpy(w.name[prefix.len + 1 ..][0..sub_path.len], sub_path);
274	return;
275	}
276
277	// sub_path fits into name
278	// there is no prefix or prefix fits into prefix
279	if (sub_path.len <= max_name) {
280	@memcpy(w.name[0..sub_path.len], sub_path);
281	@memcpy(w.prefix[0..prefix.len], prefix);
282	return;
283	}
284
285	if (prefix.len > 0) {
286	@memcpy(w.prefix[0..prefix.len], prefix);
287	w.prefix[prefix.len] = sep;
288	}
289	const prefix_pos = if (prefix.len > 0) prefix.len + 1 else 0;
290
291	// add as much to prefix as you can, must split at /
292	const prefix_remaining = max_prefix - prefix_pos;
293	if (std.mem.lastIndexOf(u8, sub_path[0..@min(prefix_remaining, sub_path.len)], &.{'/'})) \|sep_pos\| {
294	@memcpy(w.prefix[prefix_pos..][0..sep_pos], sub_path[0..sep_pos]);
295	if ((sub_path.len - sep_pos - 1) > max_name) return error.NameTooLong;
296	@memcpy(w.name[0..][0 .. sub_path.len - sep_pos - 1], sub_path[sep_pos + 1 ..]);
297	return;
298	}
299
300	return error.NameTooLong;
301	}
302
303	comptime {
304	assert(@sizeOf(Header) == 512);
305	}
306
307	test "setPath" {
308	const cases = [_]struct {
309	in: []const []const u8,
310	out: []const []const u8,
311	}{
312	.{
313	.in = &.{ "", "123456789" },
314	.out = &.{ "", "123456789" },
315	},
316	// can fit into name
317	.{
318	.in = &.{ "prefix", "sub_path" },
319	.out = &.{ "", "prefix/sub_path" },
320	},
321	// no more both fits into name
322	.{
323	.in = &.{ "prefix", "0123456789/" ** 8 ++ "basename" },
324	.out = &.{ "prefix", "0123456789/" ** 8 ++ "basename" },
325	},
326	// put as much as you can into prefix the rest goes into name
327	.{
328	.in = &.{ "prefix", "0123456789/" ** 10 ++ "basename" },
329	.out = &.{ "prefix/" ++ "0123456789/" ** 9 ++ "0123456789", "basename" },
330	},
331
332	.{
333	.in = &.{ "prefix", "0123456789/" ** 15 ++ "basename" },
334	.out = &.{ "prefix/" ++ "0123456789/" ** 12 ++ "0123456789", "0123456789/0123456789/basename" },
335	},
336	.{
337	.in = &.{ "prefix", "0123456789/" ** 21 ++ "basename" },
338	.out = &.{ "prefix/" ++ "0123456789/" 12 ++ "0123456789", "0123456789/" 8 ++ "basename" },
339	},
340	.{
341	.in = &.{ "", "012345678/" ** 10 ++ "foo" },
342	.out = &.{ "012345678/" ** 9 ++ "012345678", "foo" },
343	},
344	};
345
346	for (cases) \|case\| {
347	var header = Header.init(.regular);
348	try header.setPath(case.in[0], case.in[1]);
349	try testing.expectEqualStrings(case.out[0], std.mem.sliceTo(&header.prefix, 0));
350	try testing.expectEqualStrings(case.out[1], std.mem.sliceTo(&header.name, 0));
351	}
352
353	const error_cases = [_]struct {
354	in: []const []const u8,
355	}{
356	// basename can't fit into name (106 characters)
357	.{ .in = &.{ "zig", "test/cases/compile_errors/regression_test_2980_base_type_u32_is_not_type_checked_properly_when_assigning_a_value_within_a_struct.zig" } },
358	// cant fit into 255 + sep
359	.{ .in = &.{ "prefix", "0123456789/" ** 22 ++ "basename" } },
360	// can fit but sub_path can't be split (there is no separator)
361	.{ .in = &.{ "prefix", "0123456789" ** 10 ++ "a" } },
362	.{ .in = &.{ "prefix", "0123456789" ** 14 ++ "basename" } },
363	};
364
365	for (error_cases) \|case\| {
366	var header = Header.init(.regular);
367	try testing.expectError(
368	error.NameTooLong,
369	header.setPath(case.in[0], case.in[1]),
370	);
371	}
372	}
373	};
374
375	test {
376	_ = Header;
377	}
378
379	test "write files" {
380	const files = [_]struct {
381	path: []const u8,
382	content: []const u8,
383	}{
384	.{ .path = "foo", .content = "bar" },
385	.{ .path = "a12345678/" 10 ++ "foo", .content = "a" 511 },
386	.{ .path = "b12345678/" 24 ++ "foo", .content = "b" 512 },
387	.{ .path = "c12345678/" 25 ++ "foo", .content = "c" 513 },
388	.{ .path = "d12345678/" 51 ++ "foo", .content = "d" 1025 },
389	.{ .path = "e123456789" ** 11, .content = "e" },
390	};
391
392	var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
393	var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
394
395	// with root
396	{
397	const root = "root";
398
399	var output: std.Io.Writer.Allocating = .init(testing.allocator);
400	var w: Writer = .{ .underlying_writer = &output.writer };
401	defer output.deinit();
402	try w.setRoot(root);
403	for (files) \|file\|
404	try w.writeFileBytes(file.path, file.content, .{});
405
406	var input: std.Io.Reader = .fixed(output.getWritten());
407	var it: std.tar.Iterator = .init(&input, .{
408	.file_name_buffer = &file_name_buffer,
409	.link_name_buffer = &link_name_buffer,
410	});
411
412	// first entry is directory with prefix
413	{
414	const actual = (try it.next()).?;
415	try testing.expectEqualStrings(root, actual.name);
416	try testing.expectEqual(std.tar.FileKind.directory, actual.kind);
417	}
418
419	var i: usize = 0;
420	while (try it.next()) \|actual\| {
421	defer i += 1;
422	const expected = files[i];
423	try testing.expectEqualStrings(root, actual.name[0..root.len]);
424	try testing.expectEqual('/', actual.name[root.len..][0]);
425	try testing.expectEqualStrings(expected.path, actual.name[root.len + 1 ..]);
426
427	var content: std.Io.Writer.Allocating = .init(testing.allocator);
428	defer content.deinit();
429	try it.streamRemaining(actual, &content.writer);
430	try testing.expectEqualSlices(u8, expected.content, content.getWritten());
431	}
432	}
433	// without root
434	{
435	var output: std.Io.Writer.Allocating = .init(testing.allocator);
436	var w: Writer = .{ .underlying_writer = &output.writer };
437	defer output.deinit();
438	for (files) \|file\| {
439	var content: std.Io.Reader = .fixed(file.content);
440	try w.writeFileStream(file.path, file.content.len, &content, .{});
441	}
442
443	var input: std.Io.Reader = .fixed(output.getWritten());
444	var it: std.tar.Iterator = .init(&input, .{
445	.file_name_buffer = &file_name_buffer,
446	.link_name_buffer = &link_name_buffer,
447	});
448
449	var i: usize = 0;
450	while (try it.next()) \|actual\| {
451	defer i += 1;
452	const expected = files[i];
453	try testing.expectEqualStrings(expected.path, actual.name);
454
455	var content: std.Io.Writer.Allocating = .init(testing.allocator);
456	defer content.deinit();
457	try it.streamRemaining(actual, &content.writer);
458	try testing.expectEqualSlices(u8, expected.content, content.getWritten());
459	}
460	try w.finishPedantically();
461	}
462	}

lib/std/tar/test.zig+173-177

 @@ -18,31 +18,72 @@ const Case = struct {
     err: ?anyerror = null, // parsing should fail with this error
 };
-const cases = [_]Case{
+const gnu_case: Case = .{
-    .{
+    .data = @embedFile("testdata/gnu.tar"),
-        .data = @embedFile("testdata/gnu.tar"),
+    .files = &[_]Case.File{
-        .files = &[_]Case.File{
+        .{
-            .{
+            .name = "small.txt",
-                .name = "small.txt",
+            .size = 5,
-                .size = 5,
+            .mode = 0o640,
-                .mode = 0o640,
-            },
-            .{
-                .name = "small2.txt",
-                .size = 11,
-                .mode = 0o640,
-            },
         },
-        .chksums = &[_][]const u8{
+        .{
-            "e38b27eaccb4391bdec553a7f3ae6b2f",
+            .name = "small2.txt",
-            "c65bd2e50a56a2138bf1716f2fd56fe9",
+            .size = 11,
+            .mode = 0o640,
+        },
+    },
+    .chksums = &[_][]const u8{
+        "e38b27eaccb4391bdec553a7f3ae6b2f",
+        "c65bd2e50a56a2138bf1716f2fd56fe9",
+    },
+};
+const gnu_multi_headers_case: Case = .{
+    .data = @embedFile("testdata/gnu-multi-hdrs.tar"),
+    .files = &[_]Case.File{
+        .{
+            .name = "GNU2/GNU2/long-path-name",
+            .link_name = "GNU4/GNU4/long-linkpath-name",
+            .kind = .sym_link,
         },
     },
-    .{
+};
+const trailing_slash_case: Case = .{
+    .data = @embedFile("testdata/trailing-slash.tar"),
+    .files = &[_]Case.File{
+        .{
+            .name = "123456789/" ** 30,
+            .kind = .directory,
+        },
+    },
+};
+const writer_big_long_case: Case = .{
+    // Size in gnu extended format, and name in pax attribute.
+    .data = @embedFile("testdata/writer-big-long.tar"),
+    .files = &[_]Case.File{
+        .{
+            .name = "longname/" ** 15 ++ "16gig.txt",
+            .size = 16 * 1024 * 1024 * 1024,
+            .mode = 0o644,
+            .truncated = true,
+        },
+    },
+};
+const fuzz1_case: Case = .{
+    .data = @embedFile("testdata/fuzz1.tar"),
+    .err = error.TarInsufficientBuffer,
+};
+test "run test cases" {
+    try testCase(gnu_case);
+    try testCase(.{
         .data = @embedFile("testdata/sparse-formats.tar"),
         .err = error.TarUnsupportedHeader,
-    },
+    });
-    .{
+    try testCase(.{
         .data = @embedFile("testdata/star.tar"),
         .files = &[_]Case.File{
             .{
 @@ -60,8 +101,8 @@ const cases = [_]Case{
             "e38b27eaccb4391bdec553a7f3ae6b2f",
             "c65bd2e50a56a2138bf1716f2fd56fe9",
         },
-    },
+    });
-    .{
+    try testCase(.{
         .data = @embedFile("testdata/v7.tar"),
         .files = &[_]Case.File{
             .{
 @@ -79,8 +120,8 @@ const cases = [_]Case{
             "e38b27eaccb4391bdec553a7f3ae6b2f",
             "c65bd2e50a56a2138bf1716f2fd56fe9",
         },
-    },
+    });
-    .{
+    try testCase(.{
         .data = @embedFile("testdata/pax.tar"),
         .files = &[_]Case.File{
             .{
 @@ -99,13 +140,13 @@ const cases = [_]Case{
         .chksums = &[_][]const u8{
             "3c382e8f5b6631aa2db52643912ffd4a",
         },
-    },
+    });
-    .{
+    try testCase(.{
         // pax attribute don't end with \n
         .data = @embedFile("testdata/pax-bad-hdr-file.tar"),
         .err = error.PaxInvalidAttributeEnd,
-    },
+    });
-    .{
+    try testCase(.{
         // size is in pax attribute
         .data = @embedFile("testdata/pax-pos-size-file.tar"),
         .files = &[_]Case.File{
 @@ -119,8 +160,8 @@ const cases = [_]Case{
         .chksums = &[_][]const u8{
             "0afb597b283fe61b5d4879669a350556",
         },
-    },
+    });
-    .{
+    try testCase(.{
         // has pax records which we are not interested in
         .data = @embedFile("testdata/pax-records.tar"),
         .files = &[_]Case.File{
 @@ -128,8 +169,8 @@ const cases = [_]Case{
                 .name = "file",
             },
         },
-    },
+    });
-    .{
+    try testCase(.{
         // has global records which we are ignoring
         .data = @embedFile("testdata/pax-global-records.tar"),
         .files = &[_]Case.File{
 @@ -146,8 +187,8 @@ const cases = [_]Case{
                 .name = "file4",
             },
         },
-    },
+    });
-    .{
+    try testCase(.{
         .data = @embedFile("testdata/nil-uid.tar"),
         .files = &[_]Case.File{
             .{
 @@ -160,8 +201,8 @@ const cases = [_]Case{
         .chksums = &[_][]const u8{
             "08d504674115e77a67244beac19668f5",
         },
-    },
+    });
-    .{
+    try testCase(.{
         // has xattrs and pax records which we are ignoring
         .data = @embedFile("testdata/xattrs.tar"),
         .files = &[_]Case.File{
 @@ -182,23 +223,14 @@ const cases = [_]Case{
             "e38b27eaccb4391bdec553a7f3ae6b2f",
             "c65bd2e50a56a2138bf1716f2fd56fe9",
         },
-    },
+    });
-    .{
+    try testCase(gnu_multi_headers_case);
-        .data = @embedFile("testdata/gnu-multi-hdrs.tar"),
+    try testCase(.{
-        .files = &[_]Case.File{
-            .{
-                .name = "GNU2/GNU2/long-path-name",
-                .link_name = "GNU4/GNU4/long-linkpath-name",
-                .kind = .sym_link,
-            },
-        },
-    },
-    .{
         // has gnu type D (directory) and S (sparse) blocks
         .data = @embedFile("testdata/gnu-incremental.tar"),
         .err = error.TarUnsupportedHeader,
-    },
+    });
-    .{
+    try testCase(.{
         // should use values only from last pax header
         .data = @embedFile("testdata/pax-multi-hdrs.tar"),
         .files = &[_]Case.File{
 @@ -208,8 +240,8 @@ const cases = [_]Case{
                 .kind = .sym_link,
             },
         },
-    },
+    });
-    .{
+    try testCase(.{
         .data = @embedFile("testdata/gnu-long-nul.tar"),
         .files = &[_]Case.File{
             .{
 @@ -217,8 +249,8 @@ const cases = [_]Case{
                 .mode = 0o644,
             },
         },
-    },
+    });
-    .{
+    try testCase(.{
         .data = @embedFile("testdata/gnu-utf8.tar"),
         .files = &[_]Case.File{
             .{
 @@ -226,8 +258,8 @@ const cases = [_]Case{
                 .mode = 0o644,
             },
         },
-    },
+    });
-    .{
+    try testCase(.{
         .data = @embedFile("testdata/gnu-not-utf8.tar"),
         .files = &[_]Case.File{
             .{
 @@ -235,33 +267,33 @@ const cases = [_]Case{
                 .mode = 0o644,
             },
         },
-    },
+    });
-    .{
+    try testCase(.{
         // null in pax key
         .data = @embedFile("testdata/pax-nul-xattrs.tar"),
         .err = error.PaxNullInKeyword,
-    },
+    });
-    .{
+    try testCase(.{
         .data = @embedFile("testdata/pax-nul-path.tar"),
         .err = error.PaxNullInValue,
-    },
+    });
-    .{
+    try testCase(.{
         .data = @embedFile("testdata/neg-size.tar"),
         .err = error.TarHeader,
-    },
+    });
-    .{
+    try testCase(.{
         .data = @embedFile("testdata/issue10968.tar"),
         .err = error.TarHeader,
-    },
+    });
-    .{
+    try testCase(.{
         .data = @embedFile("testdata/issue11169.tar"),
         .err = error.TarHeader,
-    },
+    });
-    .{
+    try testCase(.{
         .data = @embedFile("testdata/issue12435.tar"),
         .err = error.TarHeaderChksum,
-    },
+    });
-    .{
+    try testCase(.{
         // has magic with space at end instead of null
         .data = @embedFile("testdata/invalid-go17.tar"),
         .files = &[_]Case.File{
 @@ -269,8 +301,8 @@ const cases = [_]Case{
                 .name = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/foo",
             },
         },
-    },
+    });
-    .{
+    try testCase(.{
         .data = @embedFile("testdata/ustar-file-devs.tar"),
         .files = &[_]Case.File{
             .{
 @@ -278,17 +310,9 @@ const cases = [_]Case{
                 .mode = 0o644,
             },
         },
-    },
+    });
-    .{
+    try testCase(trailing_slash_case);
-        .data = @embedFile("testdata/trailing-slash.tar"),
+    try testCase(.{
-        .files = &[_]Case.File{
-            .{
-                .name = "123456789/" ** 30,
-                .kind = .directory,
-            },
-        },
-    },
-    .{
         // Has size in gnu extended format. To represent size bigger than 8 GB.
         .data = @embedFile("testdata/writer-big.tar"),
         .files = &[_]Case.File{
 @@ -299,120 +323,92 @@ const cases = [_]Case{
                 .mode = 0o640,
             },
         },
-    },
+    });
-    .{
+    try testCase(writer_big_long_case);
-        // Size in gnu extended format, and name in pax attribute.
+    try testCase(fuzz1_case);
-        .data = @embedFile("testdata/writer-big-long.tar"),
+    try testCase(.{
-        .files = &[_]Case.File{
-            .{
-                .name = "longname/" ** 15 ++ "16gig.txt",
-                .size = 16 * 1024 * 1024 * 1024,
-                .mode = 0o644,
-                .truncated = true,
-            },
-        },
-    },
-    .{
-        .data = @embedFile("testdata/fuzz1.tar"),
-        .err = error.TarInsufficientBuffer,
-    },
-    .{
         .data = @embedFile("testdata/fuzz2.tar"),
         .err = error.PaxSizeAttrOverflow,
-    },
+    });
-};
+}
-// used in test to calculate file chksum
-const Md5Writer = struct {
-    h: std.crypto.hash.Md5 = std.crypto.hash.Md5.init(.{}),
-    pub fn writeAll(self: *Md5Writer, buf: []const u8) !void {
-        self.h.update(buf);
-    }
-    pub fn writeByte(self: *Md5Writer, byte: u8) !void {
-        self.h.update(&[_]u8{byte});
-    }
-    pub fn chksum(self: *Md5Writer) [32]u8 {
-        var s = [_]u8{0} ** 16;
-        self.h.final(&s);
-        return std.fmt.bytesToHex(s, .lower);
-    }
-};
-test "run test cases" {
+fn testCase(case: Case) !void {
     var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
     var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
-    for (cases) |case| {
+    var br: std.io.Reader = .fixed(case.data);
-        var fsb = std.io.fixedBufferStream(case.data);
+    var it: tar.Iterator = .init(&br, .{
-        var iter = tar.iterator(fsb.reader(), .{
+        .file_name_buffer = &file_name_buffer,
-            .file_name_buffer = &file_name_buffer,
+        .link_name_buffer = &link_name_buffer,
-            .link_name_buffer = &link_name_buffer,
+    });
-        });
+    var i: usize = 0;
-        var i: usize = 0;
+    while (it.next() catch |err| {
-        while (iter.next() catch |err| {
+        if (case.err) |e| {
-            if (case.err) |e| {
+            try testing.expectEqual(e, err);
-                try testing.expectEqual(e, err);
+            return;
-                continue;
+        } else {
-            } else {
+            return err;
-                return err;
+        }
-            }
+    }) |actual| : (i += 1) {
-        }) |actual| : (i += 1) {
+        const expected = case.files[i];
-            const expected = case.files[i];
+        try testing.expectEqualStrings(expected.name, actual.name);
-            try testing.expectEqualStrings(expected.name, actual.name);
+        try testing.expectEqual(expected.size, actual.size);
-            try testing.expectEqual(expected.size, actual.size);
+        try testing.expectEqual(expected.kind, actual.kind);
-            try testing.expectEqual(expected.kind, actual.kind);
+        try testing.expectEqual(expected.mode, actual.mode);
-            try testing.expectEqual(expected.mode, actual.mode);
+        try testing.expectEqualStrings(expected.link_name, actual.link_name);
-            try testing.expectEqualStrings(expected.link_name, actual.link_name);
-            if (case.chksums.len > i) {
+        if (case.chksums.len > i) {
-                var md5writer = Md5Writer{};
+            var aw: std.Io.Writer.Allocating = .init(std.testing.allocator);
-                try actual.writeAll(&md5writer);
+            defer aw.deinit();
-                const chksum = md5writer.chksum();
+            try it.streamRemaining(actual, &aw.writer);
-                try testing.expectEqualStrings(case.chksums[i], &chksum);
+            const chksum = std.fmt.bytesToHex(std.crypto.hash.Md5.hashResult(aw.getWritten()), .lower);
-            } else {
+            try testing.expectEqualStrings(case.chksums[i], &chksum);
-                if (expected.truncated) {
+        } else {
-                    iter.unread_file_bytes = 0;
+            if (expected.truncated) {
-                }
+                it.unread_file_bytes = 0;
             }
         }
-        try testing.expectEqual(case.files.len, i);
     }
+    try testing.expectEqual(case.files.len, i);
 }
 test "pax/gnu long names with small buffer" {
+    try testLongNameCase(gnu_multi_headers_case);
+    try testLongNameCase(trailing_slash_case);
+    try testLongNameCase(.{
+        .data = @embedFile("testdata/fuzz1.tar"),
+        .err = error.TarInsufficientBuffer,
+    });
+}
+fn testLongNameCase(case: Case) !void {
     // should fail with insufficient buffer error
     var min_file_name_buffer: [256]u8 = undefined;
     var min_link_name_buffer: [100]u8 = undefined;
-    const long_name_cases = [_]Case{ cases[11], cases[25], cases[28] };
-    for (long_name_cases) |case| {
+    var br: std.io.Reader = .fixed(case.data);
-        var fsb = std.io.fixedBufferStream(case.data);
+    var iter: tar.Iterator = .init(&br, .{
-        var iter = tar.iterator(fsb.reader(), .{
+        .file_name_buffer = &min_file_name_buffer,
-            .file_name_buffer = &min_file_name_buffer,
+        .link_name_buffer = &min_link_name_buffer,
-            .link_name_buffer = &min_link_name_buffer,
+    });
-        });
-        var iter_err: ?anyerror = null;
+    var iter_err: ?anyerror = null;
-        while (iter.next() catch |err| brk: {
+    while (iter.next() catch |err| brk: {
-            iter_err = err;
+        iter_err = err;
-            break :brk null;
+        break :brk null;
-        }) |_| {}
+    }) |_| {}
-        try testing.expect(iter_err != null);
+    try testing.expect(iter_err != null);
-        try testing.expectEqual(error.TarInsufficientBuffer, iter_err.?);
+    try testing.expectEqual(error.TarInsufficientBuffer, iter_err.?);
-    }
 }
 test "insufficient buffer in Header name filed" {
     var min_file_name_buffer: [9]u8 = undefined;
     var min_link_name_buffer: [100]u8 = undefined;
-    var fsb = std.io.fixedBufferStream(cases[0].data);
+    var br: std.io.Reader = .fixed(gnu_case.data);
-    var iter = tar.iterator(fsb.reader(), .{
+    var iter: tar.Iterator = .init(&br, .{
         .file_name_buffer = &min_file_name_buffer,
         .link_name_buffer = &min_link_name_buffer,
     });
 @@ -466,21 +462,21 @@ test "should not overwrite existing file" {
     // This ensures that file is not overwritten.
     //
     const data = @embedFile("testdata/overwrite_file.tar");
-    var fsb = std.io.fixedBufferStream(data);
+    var r: std.io.Reader = .fixed(data);
     // Unpack with strip_components = 1 should fail
     var root = std.testing.tmpDir(.{});
     defer root.cleanup();
     try testing.expectError(
         error.PathAlreadyExists,
-        tar.pipeToFileSystem(root.dir, fsb.reader(), .{ .mode_mode = .ignore, .strip_components = 1 }),
+        tar.pipeToFileSystem(root.dir, &r, .{ .mode_mode = .ignore, .strip_components = 1 }),
     );
     // Unpack with strip_components = 0 should pass
-    fsb.reset();
+    r = .fixed(data);
     var root2 = std.testing.tmpDir(.{});
     defer root2.cleanup();
-    try tar.pipeToFileSystem(root2.dir, fsb.reader(), .{ .mode_mode = .ignore, .strip_components = 0 });
+    try tar.pipeToFileSystem(root2.dir, &r, .{ .mode_mode = .ignore, .strip_components = 0 });
 }
 test "case sensitivity" {
 @@ -494,12 +490,12 @@ test "case sensitivity" {
     //     18089/alacritty/Darkermatrix.yml
     //
     const data = @embedFile("testdata/18089.tar");
-    var fsb = std.io.fixedBufferStream(data);
+    var r: std.io.Reader = .fixed(data);
     var root = std.testing.tmpDir(.{});
     defer root.cleanup();
-    tar.pipeToFileSystem(root.dir, fsb.reader(), .{ .mode_mode = .ignore, .strip_components = 1 }) catch |err| {
+    tar.pipeToFileSystem(root.dir, &r, .{ .mode_mode = .ignore, .strip_components = 1 }) catch |err| {
         // on case insensitive fs we fail on overwrite existing file
         try testing.expectEqual(error.PathAlreadyExists, err);
         return;

lib/std/tar/writer.zig deleted-497

...	@@ -1,497 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const testing = std.testing;
4
5	/// Creates tar Writer which will write tar content to the `underlying_writer`.
6	/// Use setRoot to nest all following entries under single root. If file don't
7	/// fit into posix header (name+prefix: 100+155 bytes) gnu extented header will
8	/// be used for long names. Options enables setting file premission mode and
9	/// mtime. Default is to use current time for mtime and 0o664 for file mode.
10	pub fn writer(underlying_writer: anytype) Writer(@TypeOf(underlying_writer)) {
11	return .{ .underlying_writer = underlying_writer };
12	}
13
14	pub fn Writer(comptime WriterType: type) type {
15	return struct {
16	const block_size = @sizeOf(Header);
17	const empty_block: [block_size]u8 = [_]u8{0} ** block_size;
18
19	/// Options for writing file/dir/link. If left empty 0o664 is used for
20	/// file mode and current time for mtime.
21	pub const Options = struct {
22	/// File system permission mode.
23	mode: u32 = 0,
24	/// File system modification time.
25	mtime: u64 = 0,
26	};
27	const Self = @This();
28
29	underlying_writer: WriterType,
30	prefix: []const u8 = "",
31	mtime_now: u64 = 0,
32
33	/// Sets prefix for all other write* method paths.
34	pub fn setRoot(self: *Self, root: []const u8) !void {
35	if (root.len > 0)
36	try self.writeDir(root, .{});
37
38	self.prefix = root;
39	}
40
41	/// Writes directory.
42	pub fn writeDir(self: *Self, sub_path: []const u8, opt: Options) !void {
43	try self.writeHeader(.directory, sub_path, "", 0, opt);
44	}
45
46	/// Writes file system file.
47	pub fn writeFile(self: *Self, sub_path: []const u8, file: std.fs.File) !void {
48	const stat = try file.stat();
49	const mtime: u64 = @intCast(@divFloor(stat.mtime, std.time.ns_per_s));
50
51	var header = Header{};
52	try self.setPath(&header, sub_path);
53	try header.setSize(stat.size);
54	try header.setMtime(mtime);
55	try header.write(self.underlying_writer);
56
57	try self.underlying_writer.writeFile(file);
58	try self.writePadding(stat.size);
59	}
60
61	/// Writes file reading file content from `reader`. Number of bytes in
62	/// reader must be equal to `size`.
63	pub fn writeFileStream(self: *Self, sub_path: []const u8, size: usize, reader: anytype, opt: Options) !void {
64	try self.writeHeader(.regular, sub_path, "", @intCast(size), opt);
65
66	var counting_reader = std.io.countingReader(reader);
67	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
68	try fifo.pump(counting_reader.reader(), self.underlying_writer);
69	if (counting_reader.bytes_read != size) return error.WrongReaderSize;
70	try self.writePadding(size);
71	}
72
73	/// Writes file using bytes buffer `content` for size and file content.
74	pub fn writeFileBytes(self: *Self, sub_path: []const u8, content: []const u8, opt: Options) !void {
75	try self.writeHeader(.regular, sub_path, "", @intCast(content.len), opt);
76	try self.underlying_writer.writeAll(content);
77	try self.writePadding(content.len);
78	}
79
80	/// Writes symlink.
81	pub fn writeLink(self: *Self, sub_path: []const u8, link_name: []const u8, opt: Options) !void {
82	try self.writeHeader(.symbolic_link, sub_path, link_name, 0, opt);
83	}
84
85	/// Writes fs.Dir.WalkerEntry. Uses `mtime` from file system entry and
86	/// default for entry mode .
87	pub fn writeEntry(self: *Self, entry: std.fs.Dir.Walker.Entry) !void {
88	switch (entry.kind) {
89	.directory => {
90	try self.writeDir(entry.path, .{ .mtime = try entryMtime(entry) });
91	},
92	.file => {
93	var file = try entry.dir.openFile(entry.basename, .{});
94	defer file.close();
95	try self.writeFile(entry.path, file);
96	},
97	.sym_link => {
98	var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
99	const link_name = try entry.dir.readLink(entry.basename, &link_name_buffer);
100	try self.writeLink(entry.path, link_name, .{ .mtime = try entryMtime(entry) });
101	},
102	else => {
103	return error.UnsupportedWalkerEntryKind;
104	},
105	}
106	}
107
108	fn writeHeader(
109	self: *Self,
110	typeflag: Header.FileType,
111	sub_path: []const u8,
112	link_name: []const u8,
113	size: u64,
114	opt: Options,
115	) !void {
116	var header = Header.init(typeflag);
117	try self.setPath(&header, sub_path);
118	try header.setSize(size);
119	try header.setMtime(if (opt.mtime != 0) opt.mtime else self.mtimeNow());
120	if (opt.mode != 0)
121	try header.setMode(opt.mode);
122	if (typeflag == .symbolic_link)
123	header.setLinkname(link_name) catch \|err\| switch (err) {
124	error.NameTooLong => try self.writeExtendedHeader(.gnu_long_link, &.{link_name}),
125	else => return err,
126	};
127	try header.write(self.underlying_writer);
128	}
129
130	fn mtimeNow(self: *Self) u64 {
131	if (self.mtime_now == 0)
132	self.mtime_now = @intCast(std.time.timestamp());
133	return self.mtime_now;
134	}
135
136	fn entryMtime(entry: std.fs.Dir.Walker.Entry) !u64 {
137	const stat = try entry.dir.statFile(entry.basename);
138	return @intCast(@divFloor(stat.mtime, std.time.ns_per_s));
139	}
140
141	/// Writes path in posix header, if don't fit (in name+prefix; 100+155
142	/// bytes) writes it in gnu extended header.
143	fn setPath(self: Self, header: Header, sub_path: []const u8) !void {
144	header.setPath(self.prefix, sub_path) catch \|err\| switch (err) {
145	error.NameTooLong => {
146	// write extended header
147	const buffers: []const []const u8 = if (self.prefix.len == 0)
148	&.{sub_path}
149	else
150	&.{ self.prefix, "/", sub_path };
151	try self.writeExtendedHeader(.gnu_long_name, buffers);
152	},
153	else => return err,
154	};
155	}
156
157	/// Writes gnu extended header: gnu_long_name or gnu_long_link.
158	fn writeExtendedHeader(self: *Self, typeflag: Header.FileType, buffers: []const []const u8) !void {
159	var len: usize = 0;
160	for (buffers) \|buf\|
161	len += buf.len;
162
163	var header = Header.init(typeflag);
164	try header.setSize(len);
165	try header.write(self.underlying_writer);
166	for (buffers) \|buf\|
167	try self.underlying_writer.writeAll(buf);
168	try self.writePadding(len);
169	}
170
171	fn writePadding(self: *Self, bytes: u64) !void {
172	const pos: usize = @intCast(bytes % block_size);
173	if (pos == 0) return;
174	try self.underlying_writer.writeAll(empty_block[pos..]);
175	}
176
177	/// Tar should finish with two zero blocks, but 'reasonable system must
178	/// not assume that such a block exists when reading an archive' (from
179	/// reference). In practice it is safe to skip this finish.
180	pub fn finish(self: *Self) !void {
181	try self.underlying_writer.writeAll(&empty_block);
182	try self.underlying_writer.writeAll(&empty_block);
183	}
184	};
185	}
186
187	/// A struct that is exactly 512 bytes and matches tar file format. This is
188	/// intended to be used for outputting tar files; for parsing there is
189	/// `std.tar.Header`.
190	const Header = extern struct {
191	// This struct was originally copied from
192	// https://github.com/mattnite/tar/blob/main/src/main.zig which is MIT
193	// licensed.
194	//
195	// The name, linkname, magic, uname, and gname are null-terminated character
196	// strings. All other fields are zero-filled octal numbers in ASCII. Each
197	// numeric field of width w contains w minus 1 digits, and a null.
198	// Reference: https://www.gnu.org/software/tar/manual/html_node/Standard.html
199	// POSIX header: byte offset
200	name: [100]u8 = [_]u8{0} ** 100, // 0
201	mode: [7:0]u8 = default_mode.file, // 100
202	uid: [7:0]u8 = [_:0]u8{0} ** 7, // unused 108
203	gid: [7:0]u8 = [_:0]u8{0} ** 7, // unused 116
204	size: [11:0]u8 = [_:0]u8{'0'} ** 11, // 124
205	mtime: [11:0]u8 = [_:0]u8{'0'} ** 11, // 136
206	checksum: [7:0]u8 = [_:0]u8{' '} ** 7, // 148
207	typeflag: FileType = .regular, // 156
208	linkname: [100]u8 = [_]u8{0} ** 100, // 157
209	magic: [6]u8 = [_]u8{ 'u', 's', 't', 'a', 'r', 0 }, // 257
210	version: [2]u8 = [_]u8{ '0', '0' }, // 263
211	uname: [32]u8 = [_]u8{0} ** 32, // unused 265
212	gname: [32]u8 = [_]u8{0} ** 32, // unused 297
213	devmajor: [7:0]u8 = [_:0]u8{0} ** 7, // unused 329
214	devminor: [7:0]u8 = [_:0]u8{0} ** 7, // unused 337
215	prefix: [155]u8 = [_]u8{0} ** 155, // 345
216	pad: [12]u8 = [_]u8{0} ** 12, // unused 500
217
218	pub const FileType = enum(u8) {
219	regular = '0',
220	symbolic_link = '2',
221	directory = '5',
222	gnu_long_name = 'L',
223	gnu_long_link = 'K',
224	};
225
226	const default_mode = struct {
227	const file = [_:0]u8{ '0', '0', '0', '0', '6', '6', '4' }; // 0o664
228	const dir = [_:0]u8{ '0', '0', '0', '0', '7', '7', '5' }; // 0o775
229	const sym_link = [_:0]u8{ '0', '0', '0', '0', '7', '7', '7' }; // 0o777
230	const other = [_:0]u8{ '0', '0', '0', '0', '0', '0', '0' }; // 0o000
231	};
232
233	pub fn init(typeflag: FileType) Header {
234	return .{
235	.typeflag = typeflag,
236	.mode = switch (typeflag) {
237	.directory => default_mode.dir,
238	.symbolic_link => default_mode.sym_link,
239	.regular => default_mode.file,
240	else => default_mode.other,
241	},
242	};
243	}
244
245	pub fn setSize(self: *Header, size: u64) !void {
246	try octal(&self.size, size);
247	}
248
249	fn octal(buf: []u8, value: u64) !void {
250	var remainder: u64 = value;
251	var pos: usize = buf.len;
252	while (remainder > 0 and pos > 0) {
253	pos -= 1;
254	const c: u8 = @as(u8, @intCast(remainder % 8)) + '0';
255	buf[pos] = c;
256	remainder /= 8;
257	if (pos == 0 and remainder > 0) return error.OctalOverflow;
258	}
259	}
260
261	pub fn setMode(self: *Header, mode: u32) !void {
262	try octal(&self.mode, mode);
263	}
264
265	// Integer number of seconds since January 1, 1970, 00:00 Coordinated Universal Time.
266	// mtime == 0 will use current time
267	pub fn setMtime(self: *Header, mtime: u64) !void {
268	try octal(&self.mtime, mtime);
269	}
270
271	pub fn updateChecksum(self: *Header) !void {
272	var checksum: usize = ' '; // other 7 self.checksum bytes are initialized to ' '
273	for (std.mem.asBytes(self)) \|val\|
274	checksum += val;
275	try octal(&self.checksum, checksum);
276	}
277
278	pub fn write(self: *Header, output_writer: anytype) !void {
279	try self.updateChecksum();
280	try output_writer.writeAll(std.mem.asBytes(self));
281	}
282
283	pub fn setLinkname(self: *Header, link: []const u8) !void {
284	if (link.len > self.linkname.len) return error.NameTooLong;
285	@memcpy(self.linkname[0..link.len], link);
286	}
287
288	pub fn setPath(self: *Header, prefix: []const u8, sub_path: []const u8) !void {
289	const max_prefix = self.prefix.len;
290	const max_name = self.name.len;
291	const sep = std.fs.path.sep_posix;
292
293	if (prefix.len + sub_path.len > max_name + max_prefix or prefix.len > max_prefix)
294	return error.NameTooLong;
295
296	// both fit into name
297	if (prefix.len > 0 and prefix.len + sub_path.len < max_name) {
298	@memcpy(self.name[0..prefix.len], prefix);
299	self.name[prefix.len] = sep;
300	@memcpy(self.name[prefix.len + 1 ..][0..sub_path.len], sub_path);
301	return;
302	}
303
304	// sub_path fits into name
305	// there is no prefix or prefix fits into prefix
306	if (sub_path.len <= max_name) {
307	@memcpy(self.name[0..sub_path.len], sub_path);
308	@memcpy(self.prefix[0..prefix.len], prefix);
309	return;
310	}
311
312	if (prefix.len > 0) {
313	@memcpy(self.prefix[0..prefix.len], prefix);
314	self.prefix[prefix.len] = sep;
315	}
316	const prefix_pos = if (prefix.len > 0) prefix.len + 1 else 0;
317
318	// add as much to prefix as you can, must split at /
319	const prefix_remaining = max_prefix - prefix_pos;
320	if (std.mem.lastIndexOf(u8, sub_path[0..@min(prefix_remaining, sub_path.len)], &.{'/'})) \|sep_pos\| {
321	@memcpy(self.prefix[prefix_pos..][0..sep_pos], sub_path[0..sep_pos]);
322	if ((sub_path.len - sep_pos - 1) > max_name) return error.NameTooLong;
323	@memcpy(self.name[0..][0 .. sub_path.len - sep_pos - 1], sub_path[sep_pos + 1 ..]);
324	return;
325	}
326
327	return error.NameTooLong;
328	}
329
330	comptime {
331	assert(@sizeOf(Header) == 512);
332	}
333
334	test setPath {
335	const cases = [_]struct {
336	in: []const []const u8,
337	out: []const []const u8,
338	}{
339	.{
340	.in = &.{ "", "123456789" },
341	.out = &.{ "", "123456789" },
342	},
343	// can fit into name
344	.{
345	.in = &.{ "prefix", "sub_path" },
346	.out = &.{ "", "prefix/sub_path" },
347	},
348	// no more both fits into name
349	.{
350	.in = &.{ "prefix", "0123456789/" ** 8 ++ "basename" },
351	.out = &.{ "prefix", "0123456789/" ** 8 ++ "basename" },
352	},
353	// put as much as you can into prefix the rest goes into name
354	.{
355	.in = &.{ "prefix", "0123456789/" ** 10 ++ "basename" },
356	.out = &.{ "prefix/" ++ "0123456789/" ** 9 ++ "0123456789", "basename" },
357	},
358
359	.{
360	.in = &.{ "prefix", "0123456789/" ** 15 ++ "basename" },
361	.out = &.{ "prefix/" ++ "0123456789/" ** 12 ++ "0123456789", "0123456789/0123456789/basename" },
362	},
363	.{
364	.in = &.{ "prefix", "0123456789/" ** 21 ++ "basename" },
365	.out = &.{ "prefix/" ++ "0123456789/" 12 ++ "0123456789", "0123456789/" 8 ++ "basename" },
366	},
367	.{
368	.in = &.{ "", "012345678/" ** 10 ++ "foo" },
369	.out = &.{ "012345678/" ** 9 ++ "012345678", "foo" },
370	},
371	};
372
373	for (cases) \|case\| {
374	var header = Header.init(.regular);
375	try header.setPath(case.in[0], case.in[1]);
376	try testing.expectEqualStrings(case.out[0], str(&header.prefix));
377	try testing.expectEqualStrings(case.out[1], str(&header.name));
378	}
379
380	const error_cases = [_]struct {
381	in: []const []const u8,
382	}{
383	// basename can't fit into name (106 characters)
384	.{ .in = &.{ "zig", "test/cases/compile_errors/regression_test_2980_base_type_u32_is_not_type_checked_properly_when_assigning_a_value_within_a_struct.zig" } },
385	// cant fit into 255 + sep
386	.{ .in = &.{ "prefix", "0123456789/" ** 22 ++ "basename" } },
387	// can fit but sub_path can't be split (there is no separator)
388	.{ .in = &.{ "prefix", "0123456789" ** 10 ++ "a" } },
389	.{ .in = &.{ "prefix", "0123456789" ** 14 ++ "basename" } },
390	};
391
392	for (error_cases) \|case\| {
393	var header = Header.init(.regular);
394	try testing.expectError(
395	error.NameTooLong,
396	header.setPath(case.in[0], case.in[1]),
397	);
398	}
399	}
400
401	// Breaks string on first null character.
402	fn str(s: []const u8) []const u8 {
403	for (s, 0..) \|c, i\| {
404	if (c == 0) return s[0..i];
405	}
406	return s;
407	}
408	};
409
410	test {
411	_ = Header;
412	}
413
414	test "write files" {
415	const files = [_]struct {
416	path: []const u8,
417	content: []const u8,
418	}{
419	.{ .path = "foo", .content = "bar" },
420	.{ .path = "a12345678/" 10 ++ "foo", .content = "a" 511 },
421	.{ .path = "b12345678/" 24 ++ "foo", .content = "b" 512 },
422	.{ .path = "c12345678/" 25 ++ "foo", .content = "c" 513 },
423	.{ .path = "d12345678/" 51 ++ "foo", .content = "d" 1025 },
424	.{ .path = "e123456789" ** 11, .content = "e" },
425	};
426
427	var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
428	var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined;
429
430	// with root
431	{
432	const root = "root";
433
434	var output = std.ArrayList(u8).init(testing.allocator);
435	defer output.deinit();
436	var wrt = writer(output.writer());
437	try wrt.setRoot(root);
438	for (files) \|file\|
439	try wrt.writeFileBytes(file.path, file.content, .{});
440
441	var input = std.io.fixedBufferStream(output.items);
442	var iter = std.tar.iterator(
443	input.reader(),
444	.{ .file_name_buffer = &file_name_buffer, .link_name_buffer = &link_name_buffer },
445	);
446
447	// first entry is directory with prefix
448	{
449	const actual = (try iter.next()).?;
450	try testing.expectEqualStrings(root, actual.name);
451	try testing.expectEqual(std.tar.FileKind.directory, actual.kind);
452	}
453
454	var i: usize = 0;
455	while (try iter.next()) \|actual\| {
456	defer i += 1;
457	const expected = files[i];
458	try testing.expectEqualStrings(root, actual.name[0..root.len]);
459	try testing.expectEqual('/', actual.name[root.len..][0]);
460	try testing.expectEqualStrings(expected.path, actual.name[root.len + 1 ..]);
461
462	var content = std.ArrayList(u8).init(testing.allocator);
463	defer content.deinit();
464	try actual.writeAll(content.writer());
465	try testing.expectEqualSlices(u8, expected.content, content.items);
466	}
467	}
468	// without root
469	{
470	var output = std.ArrayList(u8).init(testing.allocator);
471	defer output.deinit();
472	var wrt = writer(output.writer());
473	for (files) \|file\| {
474	var content = std.io.fixedBufferStream(file.content);
475	try wrt.writeFileStream(file.path, file.content.len, content.reader(), .{});
476	}
477
478	var input = std.io.fixedBufferStream(output.items);
479	var iter = std.tar.iterator(
480	input.reader(),
481	.{ .file_name_buffer = &file_name_buffer, .link_name_buffer = &link_name_buffer },
482	);
483
484	var i: usize = 0;
485	while (try iter.next()) \|actual\| {
486	defer i += 1;
487	const expected = files[i];
488	try testing.expectEqualStrings(expected.path, actual.name);
489
490	var content = std.ArrayList(u8).init(testing.allocator);
491	defer content.deinit();
492	try actual.writeAll(content.writer());
493	try testing.expectEqualSlices(u8, expected.content, content.items);
494	}
495	try wrt.finish();
496	}
497	}

src/Compilation.zig+23-14

...	@@ -4862,6 +4862,9 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {	...	@@ -4862,6 +4862,9 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {
4862	};	4862	};
4863	defer tar_file.close();	4863	defer tar_file.close();
4864		4864
		4865	var buffer: [1024]u8 = undefined;
		4866	var tar_file_writer = tar_file.writer(&buffer);
		4867
4865	var seen_table: std.AutoArrayHashMapUnmanaged(*Package.Module, []const u8) = .empty;	4868	var seen_table: std.AutoArrayHashMapUnmanaged(*Package.Module, []const u8) = .empty;
4866	defer seen_table.deinit(comp.gpa);	4869	defer seen_table.deinit(comp.gpa);
4867		4870
...	@@ -4871,7 +4874,7 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {	...	@@ -4871,7 +4874,7 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {
4871	var i: usize = 0;	4874	var i: usize = 0;
4872	while (i < seen_table.count()) : (i += 1) {	4875	while (i < seen_table.count()) : (i += 1) {
4873	const mod = seen_table.keys()[i];	4876	const mod = seen_table.keys()[i];
4874	try comp.docsCopyModule(mod, seen_table.values()[i], tar_file);	4877	try comp.docsCopyModule(mod, seen_table.values()[i], &tar_file_writer);
4875		4878
4876	const deps = mod.deps.values();	4879	const deps = mod.deps.values();
4877	try seen_table.ensureUnusedCapacity(comp.gpa, deps.len);	4880	try seen_table.ensureUnusedCapacity(comp.gpa, deps.len);
...	@@ -4879,24 +4882,29 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {	...	@@ -4879,24 +4882,29 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void {
4879	}	4882	}
4880	}	4883	}
4881		4884
4882	fn docsCopyModule(comp: Compilation, module: Package.Module, name: []const u8, tar_file: fs.File) !void {	4885	fn docsCopyModule(
		4886	comp: *Compilation,
		4887	module: *Package.Module,
		4888	name: []const u8,
		4889	tar_file_writer: *fs.File.Writer,
		4890	) !void {
4883	const root = module.root;	4891	const root = module.root;
4884	var mod_dir = d: {	4892	var mod_dir = d: {
4885	const root_dir, const sub_path = root.openInfo(comp.dirs);	4893	const root_dir, const sub_path = root.openInfo(comp.dirs);
4886	break :d root_dir.openDir(sub_path, .{ .iterate = true });	4894	break :d root_dir.openDir(sub_path, .{ .iterate = true });
4887	} catch \|err\| {	4895	} catch \|err\| {
4888	return comp.lockAndSetMiscFailure(.docs_copy, "unable to open directory '{f}': {s}", .{	4896	return comp.lockAndSetMiscFailure(.docs_copy, "unable to open directory '{f}': {t}", .{ root.fmt(comp), err });
4889	root.fmt(comp), @errorName(err),
4890	});
4891	};	4897	};
4892	defer mod_dir.close();	4898	defer mod_dir.close();
4893		4899
4894	var walker = try mod_dir.walk(comp.gpa);	4900	var walker = try mod_dir.walk(comp.gpa);
4895	defer walker.deinit();	4901	defer walker.deinit();
4896		4902
4897	var archiver = std.tar.writer(tar_file.deprecatedWriter().any());	4903	var archiver: std.tar.Writer = .{ .underlying_writer = &tar_file_writer.interface };
4898	archiver.prefix = name;	4904	archiver.prefix = name;
4899		4905
		4906	var buffer: [1024]u8 = undefined;
		4907
4900	while (try walker.next()) \|entry\| {	4908	while (try walker.next()) \|entry\| {
4901	switch (entry.kind) {	4909	switch (entry.kind) {
4902	.file => {	4910	.file => {
...	@@ -4907,14 +4915,17 @@ fn docsCopyModule(comp: Compilation, module: Package.Module, name: []const u8,	...	@@ -4907,14 +4915,17 @@ fn docsCopyModule(comp: Compilation, module: Package.Module, name: []const u8,
4907	else => continue,	4915	else => continue,
4908	}	4916	}
4909	var file = mod_dir.openFile(entry.path, .{}) catch \|err\| {	4917	var file = mod_dir.openFile(entry.path, .{}) catch \|err\| {
4910	return comp.lockAndSetMiscFailure(.docs_copy, "unable to open '{f}{s}': {s}", .{	4918	return comp.lockAndSetMiscFailure(.docs_copy, "unable to open {f}{s}: {t}", .{
4911	root.fmt(comp), entry.path, @errorName(err),	4919	root.fmt(comp), entry.path, err,
4912	});	4920	});
4913	};	4921	};
4914	defer file.close();	4922	defer file.close();
4915	archiver.writeFile(entry.path, file) catch \|err\| {	4923	const stat = try file.stat();
4916	return comp.lockAndSetMiscFailure(.docs_copy, "unable to archive '{f}{s}': {s}", .{	4924	var file_reader: fs.File.Reader = .initSize(file, &buffer, stat.size);
4917	root.fmt(comp), entry.path, @errorName(err),	4925
		4926	archiver.writeFile(entry.path, &file_reader, stat.mtime) catch \|err\| {
		4927	return comp.lockAndSetMiscFailure(.docs_copy, "unable to archive {f}{s}: {t}", .{
		4928	root.fmt(comp), entry.path, err,
4918	});	4929	});
4919	};	4930	};
4920	}	4931	}
...	@@ -4926,9 +4937,7 @@ fn workerDocsWasm(comp: *Compilation, parent_prog_node: std.Progress.Node) void	...	@@ -4926,9 +4937,7 @@ fn workerDocsWasm(comp: *Compilation, parent_prog_node: std.Progress.Node) void
4926		4937
4927	workerDocsWasmFallible(comp, prog_node) catch \|err\| switch (err) {	4938	workerDocsWasmFallible(comp, prog_node) catch \|err\| switch (err) {
4928	error.SubCompilationFailed => return, // error reported already	4939	error.SubCompilationFailed => return, // error reported already
4929	else => comp.lockAndSetMiscFailure(.docs_wasm, "unable to build autodocs: {s}", .{	4940	else => comp.lockAndSetMiscFailure(.docs_wasm, "unable to build autodocs: {t}", .{err}),
4930	@errorName(err),
4931	}),
4932	};	4941	};
4933	}	4942	}
4934		4943

src/Package/Fetch.zig+18-9

 @@ -1197,12 +1197,17 @@ fn unpackResource(
     };
     switch (file_type) {
-        .tar => return try unpackTarball(f, tmp_directory.handle, resource.reader()),
+        .tar => {
+            var adapter = resource.reader().adaptToNewApi();
+            return unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
+        },
         .@"tar.gz" => {
             const reader = resource.reader();
             var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
             var dcp = std.compress.gzip.decompressor(br.reader());
-            return try unpackTarball(f, tmp_directory.handle, dcp.reader());
+            var adapter_buffer: [1024]u8 = undefined;
+            var adapter = dcp.reader().adaptToNewApi(&adapter_buffer);
+            return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
         },
         .@"tar.xz" => {
             const gpa = f.arena.child_allocator;
 @@ -1215,7 +1220,9 @@ fn unpackResource(
                 ));
             };
             defer dcp.deinit();
-            return try unpackTarball(f, tmp_directory.handle, dcp.reader());
+            var adapter_buffer: [1024]u8 = undefined;
+            var adapter = dcp.reader().adaptToNewApi(&adapter_buffer);
+            return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
         },
         .@"tar.zst" => {
             const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len;
 @@ -1225,7 +1232,9 @@ fn unpackResource(
             var dcp = std.compress.zstd.decompressor(br.reader(), .{
                 .window_buffer = window_buffer,
             });
-            return try unpackTarball(f, tmp_directory.handle, dcp.reader());
+            var adapter_buffer: [1024]u8 = undefined;
+            var adapter = dcp.reader().adaptToNewApi(&adapter_buffer);
+            return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
         },
         .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) {
             error.FetchFailed => return error.FetchFailed,
 @@ -1239,7 +1248,7 @@ fn unpackResource(
     }
 }
-fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult {
+fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: *std.Io.Reader) RunError!UnpackResult {
     const eb = &f.error_bundle;
     const arena = f.arena.allocator();
 @@ -1250,10 +1259,10 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackRes
         .strip_components = 0,
         .mode_mode = .ignore,
         .exclude_empty_directories = true,
-    }) catch |err| return f.fail(f.location_tok, try eb.printString(
+    }) catch |err| return f.fail(
-        "unable to unpack tarball to temporary directory: {s}",
+        f.location_tok,
-        .{@errorName(err)},
+        try eb.printString("unable to unpack tarball to temporary directory: {t}", .{err}),
-    ));
+    );
     var res: UnpackResult = .{ .root_dir = diagnostics.root_dir };
     if (diagnostics.errors.items.len > 0) {

author	Andrew Kelley <andrew@ziglang.org> 2025-07-23 10:02:52+02:00
committer	GitHub <noreply@github.com> 2025-07-23 10:02:52+02:00
log	bc8e1a74c514e487842c03ab32d7f6e49f42c529
tree	f985dec0dbf247c590b2ee87c38184b3b3c555a3
parent	2365392e0e2830b147d82ff055bbed3099b21171
parent	4fcb479de95775d056f378fdf8a0b9570bc315b7
signature	Signed by PGP key B5690EEEBB952194

Merge pull request #24523 from ziglang/fifno