authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-05-27 20:41:36-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:29-07:00
log7b417c6caf13e07d05bf798cae2f8beeaee4fe6c
treee2faa752f792ebfe4773374ada39b9e877aefb8f
parent74c56376ee2dfb6100fd8da6cb03425b0e48a779

std: improve the discarding writer

by making the vtable use File.Reader instead of File and Offset

11 files changed, 82 insertions(+), 226 deletions(-)

lib/std/Build/Step/Run.zig+1-1
......@@ -78,7 +78,7 @@ max_stdio_size: usize,
7878
7979/// If stderr or stdout exceeds this amount, the child process is killed and
8080/// the step fails.
81stdio_limit: std.io.Reader.Limit,
81stdio_limit: std.io.Limit,
8282
8383captured_stdout: ?*Output,
8484captured_stderr: ?*Output,
lib/std/compress/flate/Compress.zig+1-24
......@@ -73,11 +73,7 @@ pub fn readable(c: *Compress, buffer: []u8) std.io.BufferedReader {
7373 return .{
7474 .unbuffered_reader = .{
7575 .context = c,
76 .vtable = .{
77 .read = read,
78 .readVec = readVec,
79 .discard = discard,
80 },
76 .vtable = .{ .read = read },
8177 },
8278 .buffer = buffer,
8379 };
......@@ -863,25 +859,6 @@ fn read(
863859 }
864860}
865861
866fn readVec(context: ?*anyopaque, data: []const []u8) std.io.Reader.Error!usize {
867 var bw: std.io.BufferedWriter = undefined;
868 bw.initVec(data);
869 return read(context, &bw, .countVec(data)) catch |err| switch (err) {
870 error.WriteFailed => unreachable, // Prevented by the limit.
871 else => |e| return e,
872 };
873}
874
875fn discard(context: ?*anyopaque, limit: std.io.Reader.Limit) std.io.Reader.Error!usize {
876 var trash_buffer: [64]u8 = undefined;
877 var null_writer: std.io.Writer.Null = undefined;
878 var bw = null_writer.writer().buffered(&trash_buffer);
879 return read(context, &bw, limit) catch |err| switch (err) {
880 error.WriteFailed => unreachable,
881 else => |e| return e,
882 };
883}
884
885862test "generate a Huffman code from an array of frequencies" {
886863 var freqs: [19]u16 = [_]u16{
887864 8, // 0
lib/std/compress/zstd/Decompress.zig+1-24
......@@ -73,11 +73,7 @@ pub fn init(input: *BufferedReader, options: Options) Decompress {
7373pub fn reader(self: *Decompress) Reader {
7474 return .{
7575 .context = self,
76 .vtable = &.{
77 .read = read,
78 .readVec = readVec,
79 .discard = discard,
80 },
76 .vtable = &.{ .read = read },
8177 };
8278}
8379
......@@ -255,25 +251,6 @@ fn readInFrame(d: *Decompress, bw: *BufferedWriter, limit: Reader.Limit, state:
255251 return bytes_written;
256252}
257253
258fn discard(context: ?*anyopaque, limit: Reader.Limit) Reader.Error!usize {
259 var trash_buffer: [64]u8 = undefined;
260 var null_writer: std.io.Writer.Null = undefined;
261 var bw = null_writer.writer().buffered(&trash_buffer);
262 return read(context, &bw, limit) catch |err| switch (err) {
263 error.WriteFailed => unreachable,
264 else => |e| return e,
265 };
266}
267
268fn readVec(context: ?*anyopaque, data: []const []u8) Reader.Error!usize {
269 var bw: BufferedWriter = undefined;
270 bw.initVec(data);
271 return read(context, &bw, .countVec(data)) catch |err| switch (err) {
272 error.WriteFailed => unreachable,
273 else => |e| return e,
274 };
275}
276
277254pub const Frame = struct {
278255 hasher_opt: ?std.hash.XxHash64,
279256 window_size: usize,
lib/std/crypto/phc_encoding.zig+4-2
......@@ -196,9 +196,11 @@ pub fn serialize(params: anytype, str: []u8) Error![]const u8 {
196196
197197/// Compute the number of bytes required to serialize `params`
198198pub fn calcSize(params: anytype) usize {
199 var null_writer: std.io.Writer.Null = .{};
200199 var trash: [128]u8 = undefined;
201 var bw = null_writer.writable(&trash);
200 var bw: std.io.BufferedWriter = .{
201 .unbuffered_writer = .discarding,
202 .buffer = &trash,
203 };
202204 serializeTo(params, &bw) catch unreachable;
203205 return bw.count;
204206}
lib/std/crypto/scrypt.zig+4-2
......@@ -312,9 +312,11 @@ const crypt_format = struct {
312312
313313 /// Compute the number of bytes required to serialize `params`
314314 pub fn calcSize(params: anytype) usize {
315 var null_writer: std.io.Writer.Null = .{};
316315 var trash: [64]u8 = undefined;
317 var bw = null_writer.writer().buffered(&trash);
316 var bw: std.io.BufferedWriter = .{
317 .unbuffered_writer = .discarding,
318 .buffer = &trash,
319 };
318320 serializeTo(params, &bw) catch |err| switch (err) {
319321 error.WriteFailed => unreachable,
320322 };
lib/std/fmt.zig+8-4
......@@ -845,10 +845,14 @@ pub fn bufPrintZ(buf: []u8, comptime fmt: []const u8, args: anytype) BufPrintErr
845845
846846/// Count the characters needed for format.
847847pub fn count(comptime fmt: []const u8, args: anytype) usize {
848 var trash_buffer: [std.atomic.cache_line]u8 = undefined;
849 var null_writer: std.io.Writer.Null = undefined;
850 var bw = null_writer.writer().buffered(&trash_buffer);
851 bw.print(fmt, args) catch unreachable;
848 var trash_buffer: [64]u8 = undefined;
849 var bw: std.io.BufferedWriter = .{
850 .unbuffered_writer = .discarding,
851 .buffer = &trash_buffer,
852 };
853 bw.print(fmt, args) catch |err| switch (err) {
854 error.WriteFailed => unreachable,
855 };
852856 return bw.count;
853857}
854858
lib/std/fs/File.zig+1-1
......@@ -1361,7 +1361,7 @@ pub const Writer = struct {
13611361 context: ?*anyopaque,
13621362 in_file: std.fs.File,
13631363 in_offset: std.io.Writer.Offset,
1364 in_limit: std.io.Writer.Limit,
1364 in_limit: std.io.Limit,
13651365 headers_and_trailers: []const []const u8,
13661366 headers_len: usize,
13671367 ) std.io.Writer.FileError!usize {
lib/std/io/BufferedReader.zig+1-45
......@@ -203,13 +203,7 @@ fn defaultDiscard(br: *BufferedReader, limit: Limit) Reader.Error!usize {
203203 assert(br.seek == 0);
204204 assert(br.end == 0);
205205 var bw: BufferedWriter = .{
206 .unbuffered_writer = .{
207 .context = undefined,
208 .vtable = &.{
209 .writeSplat = defaultDiscardWriteSplat,
210 .writeFile = defaultDiscardWriteFile,
211 },
212 },
206 .unbuffered_writer = .discarding,
213207 .buffer = br.buffer,
214208 };
215209 const n = br.read(&bw, limit) catch |err| switch (err) {
......@@ -227,44 +221,6 @@ fn defaultDiscard(br: *BufferedReader, limit: Limit) Reader.Error!usize {
227221 return n;
228222}
229223
230fn defaultDiscardWriteSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Writer.Error!usize {
231 _ = context;
232 const headers = data[0 .. data.len - 1];
233 const pattern = data[headers.len..];
234 var written: usize = pattern.len * splat;
235 for (headers) |bytes| written += bytes.len;
236 return written;
237}
238
239fn defaultDiscardWriteFile(
240 context: ?*anyopaque,
241 file_reader: *std.fs.File.Reader,
242 limit: Limit,
243 headers_and_trailers: []const []const u8,
244 headers_len: usize,
245) Writer.FileError!usize {
246 _ = context;
247 if (file_reader.getSize()) |size| {
248 const remaining = size - file_reader.pos;
249 const seek_amt = limit.minInt(remaining);
250 // Error is observable on `file_reader` instance, and is safe to ignore
251 // depending on the caller's needs. Caller can make that decision.
252 file_reader.seekForward(seek_amt) catch {};
253 var n: usize = seek_amt;
254 for (headers_and_trailers[0..headers_len]) |bytes| n += bytes.len;
255 if (seek_amt == remaining) {
256 // Since we made it all the way through the file, the trailers are
257 // also included.
258 for (headers_and_trailers[headers_len..]) |bytes| n += bytes.len;
259 }
260 return n;
261 } else |_| {
262 // Error is observable on `file_reader` instance, and it is better to
263 // treat the file as a pipe.
264 return error.Unimplemented;
265 }
266}
267
268224/// Returns the next `len` bytes from `unbuffered_reader`, filling the buffer as
269225/// necessary.
270226///
lib/std/io/Writer.zig+57-51
......@@ -2,8 +2,7 @@ const std = @import("../std.zig");
22const assert = std.debug.assert;
33const Writer = @This();
44const Limit = std.io.Limit;
5
6pub const Null = @import("Writer/Null.zig");
5const File = std.fs.File;
76
87context: ?*anyopaque,
98vtable: *const VTable,
......@@ -37,15 +36,7 @@ pub const VTable = struct {
3736 /// efficient implementation.
3837 writeFile: *const fn (
3938 ctx: ?*anyopaque,
40 file: std.fs.File,
41 /// If this is `Offset.none`, `file` will be streamed, affecting the
42 /// seek position. Otherwise, it will be read positionally without
43 /// affecting the seek position. `error.Unseekable` is only possible
44 /// when reading positionally.
45 ///
46 /// An offset past the end of the file is treated the same as an offset
47 /// equal to the end of the file.
48 offset: Offset,
39 file_reader: *File.Reader,
4940 /// Maximum amount of bytes to read from the file. Implementations may
5041 /// assume that the file size does not exceed this amount.
5142 ///
......@@ -63,7 +54,9 @@ pub const Error = error{
6354 WriteFailed,
6455};
6556
66pub const FileError = std.fs.File.PReadError || error{
57pub const FileError = error{
58 /// Detailed diagnostics are found on the `File.Reader` struct.
59 ReadFailed,
6760 /// See the `Writer` implementation for detailed diagnostics.
6861 WriteFailed,
6962 /// Indicates the caller should do its own file reading; the callee cannot
......@@ -71,30 +64,6 @@ pub const FileError = std.fs.File.PReadError || error{
7164 Unimplemented,
7265};
7366
74pub const Offset = enum(u64) {
75 zero = 0,
76 /// Indicates to read the file as a stream.
77 none = std.math.maxInt(u64),
78 _,
79
80 pub fn init(integer: u64) Offset {
81 const result: Offset = @enumFromInt(integer);
82 assert(result != .none);
83 return result;
84 }
85
86 pub fn toInt(o: Offset) ?u64 {
87 return if (o == .none) null else @intFromEnum(o);
88 }
89
90 pub fn advance(o: Offset, amount: u64) Offset {
91 return switch (o) {
92 .none => .none,
93 else => .init(@intFromEnum(o) + amount),
94 };
95 }
96};
97
9867pub fn writeVec(w: Writer, data: []const []const u8) Error!usize {
9968 assert(data.len > 0);
10069 return w.vtable.writeSplat(w.context, data, 1);
......@@ -107,13 +76,12 @@ pub fn writeSplat(w: Writer, data: []const []const u8, splat: usize) Error!usize
10776
10877pub fn writeFile(
10978 w: Writer,
110 file: std.fs.File,
111 offset: Offset,
79 file_reader: *File.Reader,
11280 limit: Limit,
11381 headers_and_trailers: []const []const u8,
11482 headers_len: usize,
11583) FileError!usize {
116 return w.vtable.writeFile(w.context, file, offset, limit, headers_and_trailers, headers_len);
84 return w.vtable.writeFile(w.context, file_reader, limit, headers_and_trailers, headers_len);
11785}
11886
11987pub fn buffered(w: Writer, buffer: []u8) std.io.BufferedWriter {
......@@ -136,15 +104,13 @@ pub fn failingWriteSplat(context: ?*anyopaque, data: []const []const u8, splat:
136104
137105pub fn failingWriteFile(
138106 context: ?*anyopaque,
139 file: std.fs.File,
140 offset: Offset,
107 file_reader: *File.Reader,
141108 limit: Limit,
142109 headers_and_trailers: []const []const u8,
143110 headers_len: usize,
144111) FileError!usize {
145112 _ = context;
146 _ = file;
147 _ = offset;
113 _ = file_reader;
148114 _ = limit;
149115 _ = headers_and_trailers;
150116 _ = headers_len;
......@@ -159,19 +125,63 @@ pub const failing: Writer = .{
159125 },
160126};
161127
128pub fn discardingWriteSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Error!usize {
129 _ = context;
130 const headers = data[0 .. data.len - 1];
131 const pattern = data[headers.len..];
132 var written: usize = pattern.len * splat;
133 for (headers) |bytes| written += bytes.len;
134 return written;
135}
136
137pub fn discardingWriteFile(
138 context: ?*anyopaque,
139 file_reader: *std.fs.File.Reader,
140 limit: Limit,
141 headers_and_trailers: []const []const u8,
142 headers_len: usize,
143) Writer.FileError!usize {
144 _ = context;
145 if (file_reader.getSize()) |size| {
146 const remaining = size - file_reader.pos;
147 const seek_amt = limit.minInt(remaining);
148 // Error is observable on `file_reader` instance, and is safe to ignore
149 // depending on the caller's needs. Caller can make that decision.
150 file_reader.seekForward(seek_amt) catch {};
151 var n: usize = seek_amt;
152 for (headers_and_trailers[0..headers_len]) |bytes| n += bytes.len;
153 if (seek_amt == remaining) {
154 // Since we made it all the way through the file, the trailers are
155 // also included.
156 for (headers_and_trailers[headers_len..]) |bytes| n += bytes.len;
157 }
158 return n;
159 } else |_| {
160 // Error is observable on `file_reader` instance, and it is better to
161 // treat the file as a pipe.
162 return error.Unimplemented;
163 }
164}
165
166pub const discarding: Writer = .{
167 .context = undefined,
168 .vtable = &.{
169 .writeSplat = discardingWriteSplat,
170 .writeFile = discardingWriteFile,
171 },
172};
173
162174/// For use when the `Writer` implementation can cannot offer a more efficient
163175/// implementation than a basic read/write loop on the file.
164176pub fn unimplementedWriteFile(
165177 context: ?*anyopaque,
166 file: std.fs.File,
167 offset: Offset,
178 file_reader: *File.Reader,
168179 limit: Limit,
169180 headers_and_trailers: []const []const u8,
170181 headers_len: usize,
171182) FileError!usize {
172183 _ = context;
173 _ = file;
174 _ = offset;
184 _ = file_reader;
175185 _ = limit;
176186 _ = headers_and_trailers;
177187 _ = headers_len;
......@@ -255,7 +265,3 @@ pub fn Hashed(comptime Hasher: type) type {
255265 }
256266 };
257267}
258
259test {
260 _ = Null;
261}
lib/std/io/Writer/Null.zig deleted-70
......@@ -1,70 +0,0 @@
1//! A `Writer` that discards all data.
2
3const std = @import("../../std.zig");
4const Writer = std.io.Writer;
5
6const NullWriter = @This();
7
8err: ?Error = null,
9
10pub const Error = std.fs.File.StatError;
11
12pub fn writer(nw: *NullWriter) Writer {
13 return .{
14 .context = nw,
15 .vtable = &.{
16 .writeSplat = writeSplat,
17 .writeFile = writeFile,
18 },
19 };
20}
21
22pub fn writable(nw: *NullWriter, buffer: []u8) std.io.BufferedWriter {
23 return writer(nw).buffered(buffer);
24}
25
26fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Writer.Error!usize {
27 _ = context;
28 const headers = data[0 .. data.len - 1];
29 const pattern = data[headers.len..];
30 var written: usize = pattern.len * splat;
31 for (headers) |bytes| written += bytes.len;
32 return written;
33}
34
35fn writeFile(
36 context: ?*anyopaque,
37 file: std.fs.File,
38 offset: Writer.Offset,
39 limit: Writer.Limit,
40 headers_and_trailers: []const []const u8,
41 headers_len: usize,
42) Writer.FileError!usize {
43 const nw: *NullWriter = @alignCast(@ptrCast(context));
44 var n: usize = 0;
45 if (offset == .none) {
46 @panic("TODO seek the file forwards");
47 }
48 const limit_int = limit.toInt() orelse {
49 const headers = headers_and_trailers[0..headers_len];
50 for (headers) |bytes| n += bytes.len;
51 if (offset.toInt()) |off| {
52 const stat = file.stat() catch |err| {
53 nw.err = err;
54 return error.WriteFailed;
55 };
56 n += stat.size - off;
57 for (headers_and_trailers[headers_len..]) |bytes| n += bytes.len;
58 return n;
59 }
60 @panic("TODO stream from file until eof, counting");
61 };
62 for (headers_and_trailers) |bytes| n += bytes.len;
63 return limit_int + n;
64}
65
66test "writing a small string" {
67 var nw: NullWriter = undefined;
68 var bw = nw.writer().unbuffered();
69 try bw.writeAll("yay");
70}
lib/std/zon/stringify.zig+4-2
......@@ -1040,8 +1040,10 @@ pub const Serializer = struct {
10401040};
10411041
10421042test Serializer {
1043 var null_writer: std.io.Writer.Null = undefined;
1044 var bw = null_writer.writer().unbuffered();
1043 var bw: std.io.BufferedWriter = .{
1044 .unbuffered_writer = .discarding,
1045 .buffer = &.{},
1046 };
10451047 var s: Serializer = .{ .writer = &bw };
10461048 var vec2 = try s.beginStruct(.{});
10471049 try vec2.field("x", 1.5, .{});