authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-05-27 20:41:36-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:29-07:00
log7b417c6caf13e07d05bf798cae2f8beeaee4fe6c
treee2faa752f792ebfe4773374ada39b9e877aefb8f
parent74c56376ee2dfb6100fd8da6cb03425b0e48a779

std: improve the discarding writer

by making the vtable use File.Reader instead of File and Offset

11 files changed, 82 insertions(+), 226 deletions(-)

lib/std/Build/Step/Run.zig+1-1
...@@ -78,7 +78,7 @@ max_stdio_size: usize,...@@ -78,7 +78,7 @@ max_stdio_size: usize,
7878
79/// If stderr or stdout exceeds this amount, the child process is killed and79/// If stderr or stdout exceeds this amount, the child process is killed and
80/// the step fails.80/// the step fails.
81stdio_limit: std.io.Reader.Limit,81stdio_limit: std.io.Limit,
8282
83captured_stdout: ?*Output,83captured_stdout: ?*Output,
84captured_stderr: ?*Output,84captured_stderr: ?*Output,
lib/std/compress/flate/Compress.zig+1-24
...@@ -73,11 +73,7 @@ pub fn readable(c: *Compress, buffer: []u8) std.io.BufferedReader {...@@ -73,11 +73,7 @@ pub fn readable(c: *Compress, buffer: []u8) std.io.BufferedReader {
73 return .{73 return .{
74 .unbuffered_reader = .{74 .unbuffered_reader = .{
75 .context = c,75 .context = c,
76 .vtable = .{76 .vtable = .{ .read = read },
77 .read = read,
78 .readVec = readVec,
79 .discard = discard,
80 },
81 },77 },
82 .buffer = buffer,78 .buffer = buffer,
83 };79 };
...@@ -863,25 +859,6 @@ fn read(...@@ -863,25 +859,6 @@ fn read(
863 }859 }
864}860}
865861
866fn readVec(context: ?*anyopaque, data: []const []u8) std.io.Reader.Error!usize {
867 var bw: std.io.BufferedWriter = undefined;
868 bw.initVec(data);
869 return read(context, &bw, .countVec(data)) catch |err| switch (err) {
870 error.WriteFailed => unreachable, // Prevented by the limit.
871 else => |e| return e,
872 };
873}
874
875fn discard(context: ?*anyopaque, limit: std.io.Reader.Limit) std.io.Reader.Error!usize {
876 var trash_buffer: [64]u8 = undefined;
877 var null_writer: std.io.Writer.Null = undefined;
878 var bw = null_writer.writer().buffered(&trash_buffer);
879 return read(context, &bw, limit) catch |err| switch (err) {
880 error.WriteFailed => unreachable,
881 else => |e| return e,
882 };
883}
884
885test "generate a Huffman code from an array of frequencies" {862test "generate a Huffman code from an array of frequencies" {
886 var freqs: [19]u16 = [_]u16{863 var freqs: [19]u16 = [_]u16{
887 8, // 0864 8, // 0
lib/std/compress/zstd/Decompress.zig+1-24
...@@ -73,11 +73,7 @@ pub fn init(input: *BufferedReader, options: Options) Decompress {...@@ -73,11 +73,7 @@ pub fn init(input: *BufferedReader, options: Options) Decompress {
73pub fn reader(self: *Decompress) Reader {73pub fn reader(self: *Decompress) Reader {
74 return .{74 return .{
75 .context = self,75 .context = self,
76 .vtable = &.{76 .vtable = &.{ .read = read },
77 .read = read,
78 .readVec = readVec,
79 .discard = discard,
80 },
81 };77 };
82}78}
8379
...@@ -255,25 +251,6 @@ fn readInFrame(d: *Decompress, bw: *BufferedWriter, limit: Reader.Limit, state:...@@ -255,25 +251,6 @@ fn readInFrame(d: *Decompress, bw: *BufferedWriter, limit: Reader.Limit, state:
255 return bytes_written;251 return bytes_written;
256}252}
257253
258fn discard(context: ?*anyopaque, limit: Reader.Limit) Reader.Error!usize {
259 var trash_buffer: [64]u8 = undefined;
260 var null_writer: std.io.Writer.Null = undefined;
261 var bw = null_writer.writer().buffered(&trash_buffer);
262 return read(context, &bw, limit) catch |err| switch (err) {
263 error.WriteFailed => unreachable,
264 else => |e| return e,
265 };
266}
267
268fn readVec(context: ?*anyopaque, data: []const []u8) Reader.Error!usize {
269 var bw: BufferedWriter = undefined;
270 bw.initVec(data);
271 return read(context, &bw, .countVec(data)) catch |err| switch (err) {
272 error.WriteFailed => unreachable,
273 else => |e| return e,
274 };
275}
276
277pub const Frame = struct {254pub const Frame = struct {
278 hasher_opt: ?std.hash.XxHash64,255 hasher_opt: ?std.hash.XxHash64,
279 window_size: usize,256 window_size: usize,
lib/std/crypto/phc_encoding.zig+4-2
...@@ -196,9 +196,11 @@ pub fn serialize(params: anytype, str: []u8) Error![]const u8 {...@@ -196,9 +196,11 @@ pub fn serialize(params: anytype, str: []u8) Error![]const u8 {
196196
197/// Compute the number of bytes required to serialize `params`197/// Compute the number of bytes required to serialize `params`
198pub fn calcSize(params: anytype) usize {198pub fn calcSize(params: anytype) usize {
199 var null_writer: std.io.Writer.Null = .{};
200 var trash: [128]u8 = undefined;199 var trash: [128]u8 = undefined;
201 var bw = null_writer.writable(&trash);200 var bw: std.io.BufferedWriter = .{
201 .unbuffered_writer = .discarding,
202 .buffer = &trash,
203 };
202 serializeTo(params, &bw) catch unreachable;204 serializeTo(params, &bw) catch unreachable;
203 return bw.count;205 return bw.count;
204}206}
lib/std/crypto/scrypt.zig+4-2
...@@ -312,9 +312,11 @@ const crypt_format = struct {...@@ -312,9 +312,11 @@ const crypt_format = struct {
312312
313 /// Compute the number of bytes required to serialize `params`313 /// Compute the number of bytes required to serialize `params`
314 pub fn calcSize(params: anytype) usize {314 pub fn calcSize(params: anytype) usize {
315 var null_writer: std.io.Writer.Null = .{};
316 var trash: [64]u8 = undefined;315 var trash: [64]u8 = undefined;
317 var bw = null_writer.writer().buffered(&trash);316 var bw: std.io.BufferedWriter = .{
317 .unbuffered_writer = .discarding,
318 .buffer = &trash,
319 };
318 serializeTo(params, &bw) catch |err| switch (err) {320 serializeTo(params, &bw) catch |err| switch (err) {
319 error.WriteFailed => unreachable,321 error.WriteFailed => unreachable,
320 };322 };
lib/std/fmt.zig+8-4
...@@ -845,10 +845,14 @@ pub fn bufPrintZ(buf: []u8, comptime fmt: []const u8, args: anytype) BufPrintErr...@@ -845,10 +845,14 @@ pub fn bufPrintZ(buf: []u8, comptime fmt: []const u8, args: anytype) BufPrintErr
845845
846/// Count the characters needed for format.846/// Count the characters needed for format.
847pub fn count(comptime fmt: []const u8, args: anytype) usize {847pub fn count(comptime fmt: []const u8, args: anytype) usize {
848 var trash_buffer: [std.atomic.cache_line]u8 = undefined;848 var trash_buffer: [64]u8 = undefined;
849 var null_writer: std.io.Writer.Null = undefined;849 var bw: std.io.BufferedWriter = .{
850 var bw = null_writer.writer().buffered(&trash_buffer);850 .unbuffered_writer = .discarding,
851 bw.print(fmt, args) catch unreachable;851 .buffer = &trash_buffer,
852 };
853 bw.print(fmt, args) catch |err| switch (err) {
854 error.WriteFailed => unreachable,
855 };
852 return bw.count;856 return bw.count;
853}857}
854858
lib/std/fs/File.zig+1-1
...@@ -1361,7 +1361,7 @@ pub const Writer = struct {...@@ -1361,7 +1361,7 @@ pub const Writer = struct {
1361 context: ?*anyopaque,1361 context: ?*anyopaque,
1362 in_file: std.fs.File,1362 in_file: std.fs.File,
1363 in_offset: std.io.Writer.Offset,1363 in_offset: std.io.Writer.Offset,
1364 in_limit: std.io.Writer.Limit,1364 in_limit: std.io.Limit,
1365 headers_and_trailers: []const []const u8,1365 headers_and_trailers: []const []const u8,
1366 headers_len: usize,1366 headers_len: usize,
1367 ) std.io.Writer.FileError!usize {1367 ) std.io.Writer.FileError!usize {
lib/std/io/BufferedReader.zig+1-45
...@@ -203,13 +203,7 @@ fn defaultDiscard(br: *BufferedReader, limit: Limit) Reader.Error!usize {...@@ -203,13 +203,7 @@ fn defaultDiscard(br: *BufferedReader, limit: Limit) Reader.Error!usize {
203 assert(br.seek == 0);203 assert(br.seek == 0);
204 assert(br.end == 0);204 assert(br.end == 0);
205 var bw: BufferedWriter = .{205 var bw: BufferedWriter = .{
206 .unbuffered_writer = .{206 .unbuffered_writer = .discarding,
207 .context = undefined,
208 .vtable = &.{
209 .writeSplat = defaultDiscardWriteSplat,
210 .writeFile = defaultDiscardWriteFile,
211 },
212 },
213 .buffer = br.buffer,207 .buffer = br.buffer,
214 };208 };
215 const n = br.read(&bw, limit) catch |err| switch (err) {209 const n = br.read(&bw, limit) catch |err| switch (err) {
...@@ -227,44 +221,6 @@ fn defaultDiscard(br: *BufferedReader, limit: Limit) Reader.Error!usize {...@@ -227,44 +221,6 @@ fn defaultDiscard(br: *BufferedReader, limit: Limit) Reader.Error!usize {
227 return n;221 return n;
228}222}
229223
230fn defaultDiscardWriteSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Writer.Error!usize {
231 _ = context;
232 const headers = data[0 .. data.len - 1];
233 const pattern = data[headers.len..];
234 var written: usize = pattern.len * splat;
235 for (headers) |bytes| written += bytes.len;
236 return written;
237}
238
239fn defaultDiscardWriteFile(
240 context: ?*anyopaque,
241 file_reader: *std.fs.File.Reader,
242 limit: Limit,
243 headers_and_trailers: []const []const u8,
244 headers_len: usize,
245) Writer.FileError!usize {
246 _ = context;
247 if (file_reader.getSize()) |size| {
248 const remaining = size - file_reader.pos;
249 const seek_amt = limit.minInt(remaining);
250 // Error is observable on `file_reader` instance, and is safe to ignore
251 // depending on the caller's needs. Caller can make that decision.
252 file_reader.seekForward(seek_amt) catch {};
253 var n: usize = seek_amt;
254 for (headers_and_trailers[0..headers_len]) |bytes| n += bytes.len;
255 if (seek_amt == remaining) {
256 // Since we made it all the way through the file, the trailers are
257 // also included.
258 for (headers_and_trailers[headers_len..]) |bytes| n += bytes.len;
259 }
260 return n;
261 } else |_| {
262 // Error is observable on `file_reader` instance, and it is better to
263 // treat the file as a pipe.
264 return error.Unimplemented;
265 }
266}
267
268/// Returns the next `len` bytes from `unbuffered_reader`, filling the buffer as224/// Returns the next `len` bytes from `unbuffered_reader`, filling the buffer as
269/// necessary.225/// necessary.
270///226///
lib/std/io/Writer.zig+57-51
...@@ -2,8 +2,7 @@ const std = @import("../std.zig");...@@ -2,8 +2,7 @@ const std = @import("../std.zig");
2const assert = std.debug.assert;2const assert = std.debug.assert;
3const Writer = @This();3const Writer = @This();
4const Limit = std.io.Limit;4const Limit = std.io.Limit;
55const File = std.fs.File;
6pub const Null = @import("Writer/Null.zig");
76
8context: ?*anyopaque,7context: ?*anyopaque,
9vtable: *const VTable,8vtable: *const VTable,
...@@ -37,15 +36,7 @@ pub const VTable = struct {...@@ -37,15 +36,7 @@ pub const VTable = struct {
37 /// efficient implementation.36 /// efficient implementation.
38 writeFile: *const fn (37 writeFile: *const fn (
39 ctx: ?*anyopaque,38 ctx: ?*anyopaque,
40 file: std.fs.File,39 file_reader: *File.Reader,
41 /// If this is `Offset.none`, `file` will be streamed, affecting the
42 /// seek position. Otherwise, it will be read positionally without
43 /// affecting the seek position. `error.Unseekable` is only possible
44 /// when reading positionally.
45 ///
46 /// An offset past the end of the file is treated the same as an offset
47 /// equal to the end of the file.
48 offset: Offset,
49 /// Maximum amount of bytes to read from the file. Implementations may40 /// Maximum amount of bytes to read from the file. Implementations may
50 /// assume that the file size does not exceed this amount.41 /// assume that the file size does not exceed this amount.
51 ///42 ///
...@@ -63,7 +54,9 @@ pub const Error = error{...@@ -63,7 +54,9 @@ pub const Error = error{
63 WriteFailed,54 WriteFailed,
64};55};
6556
66pub const FileError = std.fs.File.PReadError || error{57pub const FileError = error{
58 /// Detailed diagnostics are found on the `File.Reader` struct.
59 ReadFailed,
67 /// See the `Writer` implementation for detailed diagnostics.60 /// See the `Writer` implementation for detailed diagnostics.
68 WriteFailed,61 WriteFailed,
69 /// Indicates the caller should do its own file reading; the callee cannot62 /// Indicates the caller should do its own file reading; the callee cannot
...@@ -71,30 +64,6 @@ pub const FileError = std.fs.File.PReadError || error{...@@ -71,30 +64,6 @@ pub const FileError = std.fs.File.PReadError || error{
71 Unimplemented,64 Unimplemented,
72};65};
7366
74pub const Offset = enum(u64) {
75 zero = 0,
76 /// Indicates to read the file as a stream.
77 none = std.math.maxInt(u64),
78 _,
79
80 pub fn init(integer: u64) Offset {
81 const result: Offset = @enumFromInt(integer);
82 assert(result != .none);
83 return result;
84 }
85
86 pub fn toInt(o: Offset) ?u64 {
87 return if (o == .none) null else @intFromEnum(o);
88 }
89
90 pub fn advance(o: Offset, amount: u64) Offset {
91 return switch (o) {
92 .none => .none,
93 else => .init(@intFromEnum(o) + amount),
94 };
95 }
96};
97
98pub fn writeVec(w: Writer, data: []const []const u8) Error!usize {67pub fn writeVec(w: Writer, data: []const []const u8) Error!usize {
99 assert(data.len > 0);68 assert(data.len > 0);
100 return w.vtable.writeSplat(w.context, data, 1);69 return w.vtable.writeSplat(w.context, data, 1);
...@@ -107,13 +76,12 @@ pub fn writeSplat(w: Writer, data: []const []const u8, splat: usize) Error!usize...@@ -107,13 +76,12 @@ pub fn writeSplat(w: Writer, data: []const []const u8, splat: usize) Error!usize
10776
108pub fn writeFile(77pub fn writeFile(
109 w: Writer,78 w: Writer,
110 file: std.fs.File,79 file_reader: *File.Reader,
111 offset: Offset,
112 limit: Limit,80 limit: Limit,
113 headers_and_trailers: []const []const u8,81 headers_and_trailers: []const []const u8,
114 headers_len: usize,82 headers_len: usize,
115) FileError!usize {83) FileError!usize {
116 return w.vtable.writeFile(w.context, file, offset, limit, headers_and_trailers, headers_len);84 return w.vtable.writeFile(w.context, file_reader, limit, headers_and_trailers, headers_len);
117}85}
11886
119pub fn buffered(w: Writer, buffer: []u8) std.io.BufferedWriter {87pub fn buffered(w: Writer, buffer: []u8) std.io.BufferedWriter {
...@@ -136,15 +104,13 @@ pub fn failingWriteSplat(context: ?*anyopaque, data: []const []const u8, splat:...@@ -136,15 +104,13 @@ pub fn failingWriteSplat(context: ?*anyopaque, data: []const []const u8, splat:
136104
137pub fn failingWriteFile(105pub fn failingWriteFile(
138 context: ?*anyopaque,106 context: ?*anyopaque,
139 file: std.fs.File,107 file_reader: *File.Reader,
140 offset: Offset,
141 limit: Limit,108 limit: Limit,
142 headers_and_trailers: []const []const u8,109 headers_and_trailers: []const []const u8,
143 headers_len: usize,110 headers_len: usize,
144) FileError!usize {111) FileError!usize {
145 _ = context;112 _ = context;
146 _ = file;113 _ = file_reader;
147 _ = offset;
148 _ = limit;114 _ = limit;
149 _ = headers_and_trailers;115 _ = headers_and_trailers;
150 _ = headers_len;116 _ = headers_len;
...@@ -159,19 +125,63 @@ pub const failing: Writer = .{...@@ -159,19 +125,63 @@ pub const failing: Writer = .{
159 },125 },
160};126};
161127
128pub fn discardingWriteSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Error!usize {
129 _ = context;
130 const headers = data[0 .. data.len - 1];
131 const pattern = data[headers.len..];
132 var written: usize = pattern.len * splat;
133 for (headers) |bytes| written += bytes.len;
134 return written;
135}
136
137pub fn discardingWriteFile(
138 context: ?*anyopaque,
139 file_reader: *std.fs.File.Reader,
140 limit: Limit,
141 headers_and_trailers: []const []const u8,
142 headers_len: usize,
143) Writer.FileError!usize {
144 _ = context;
145 if (file_reader.getSize()) |size| {
146 const remaining = size - file_reader.pos;
147 const seek_amt = limit.minInt(remaining);
148 // Error is observable on `file_reader` instance, and is safe to ignore
149 // depending on the caller's needs. Caller can make that decision.
150 file_reader.seekForward(seek_amt) catch {};
151 var n: usize = seek_amt;
152 for (headers_and_trailers[0..headers_len]) |bytes| n += bytes.len;
153 if (seek_amt == remaining) {
154 // Since we made it all the way through the file, the trailers are
155 // also included.
156 for (headers_and_trailers[headers_len..]) |bytes| n += bytes.len;
157 }
158 return n;
159 } else |_| {
160 // Error is observable on `file_reader` instance, and it is better to
161 // treat the file as a pipe.
162 return error.Unimplemented;
163 }
164}
165
166pub const discarding: Writer = .{
167 .context = undefined,
168 .vtable = &.{
169 .writeSplat = discardingWriteSplat,
170 .writeFile = discardingWriteFile,
171 },
172};
173
162/// For use when the `Writer` implementation can cannot offer a more efficient174/// For use when the `Writer` implementation can cannot offer a more efficient
163/// implementation than a basic read/write loop on the file.175/// implementation than a basic read/write loop on the file.
164pub fn unimplementedWriteFile(176pub fn unimplementedWriteFile(
165 context: ?*anyopaque,177 context: ?*anyopaque,
166 file: std.fs.File,178 file_reader: *File.Reader,
167 offset: Offset,
168 limit: Limit,179 limit: Limit,
169 headers_and_trailers: []const []const u8,180 headers_and_trailers: []const []const u8,
170 headers_len: usize,181 headers_len: usize,
171) FileError!usize {182) FileError!usize {
172 _ = context;183 _ = context;
173 _ = file;184 _ = file_reader;
174 _ = offset;
175 _ = limit;185 _ = limit;
176 _ = headers_and_trailers;186 _ = headers_and_trailers;
177 _ = headers_len;187 _ = headers_len;
...@@ -255,7 +265,3 @@ pub fn Hashed(comptime Hasher: type) type {...@@ -255,7 +265,3 @@ pub fn Hashed(comptime Hasher: type) type {
255 }265 }
256 };266 };
257}267}
258
259test {
260 _ = Null;
261}
lib/std/io/Writer/Null.zig deleted-70
...@@ -1,70 +0,0 @@
1//! A `Writer` that discards all data.
2
3const std = @import("../../std.zig");
4const Writer = std.io.Writer;
5
6const NullWriter = @This();
7
8err: ?Error = null,
9
10pub const Error = std.fs.File.StatError;
11
12pub fn writer(nw: *NullWriter) Writer {
13 return .{
14 .context = nw,
15 .vtable = &.{
16 .writeSplat = writeSplat,
17 .writeFile = writeFile,
18 },
19 };
20}
21
22pub fn writable(nw: *NullWriter, buffer: []u8) std.io.BufferedWriter {
23 return writer(nw).buffered(buffer);
24}
25
26fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Writer.Error!usize {
27 _ = context;
28 const headers = data[0 .. data.len - 1];
29 const pattern = data[headers.len..];
30 var written: usize = pattern.len * splat;
31 for (headers) |bytes| written += bytes.len;
32 return written;
33}
34
35fn writeFile(
36 context: ?*anyopaque,
37 file: std.fs.File,
38 offset: Writer.Offset,
39 limit: Writer.Limit,
40 headers_and_trailers: []const []const u8,
41 headers_len: usize,
42) Writer.FileError!usize {
43 const nw: *NullWriter = @alignCast(@ptrCast(context));
44 var n: usize = 0;
45 if (offset == .none) {
46 @panic("TODO seek the file forwards");
47 }
48 const limit_int = limit.toInt() orelse {
49 const headers = headers_and_trailers[0..headers_len];
50 for (headers) |bytes| n += bytes.len;
51 if (offset.toInt()) |off| {
52 const stat = file.stat() catch |err| {
53 nw.err = err;
54 return error.WriteFailed;
55 };
56 n += stat.size - off;
57 for (headers_and_trailers[headers_len..]) |bytes| n += bytes.len;
58 return n;
59 }
60 @panic("TODO stream from file until eof, counting");
61 };
62 for (headers_and_trailers) |bytes| n += bytes.len;
63 return limit_int + n;
64}
65
66test "writing a small string" {
67 var nw: NullWriter = undefined;
68 var bw = nw.writer().unbuffered();
69 try bw.writeAll("yay");
70}
lib/std/zon/stringify.zig+4-2
...@@ -1040,8 +1040,10 @@ pub const Serializer = struct {...@@ -1040,8 +1040,10 @@ pub const Serializer = struct {
1040};1040};
10411041
1042test Serializer {1042test Serializer {
1043 var null_writer: std.io.Writer.Null = undefined;1043 var bw: std.io.BufferedWriter = .{
1044 var bw = null_writer.writer().unbuffered();1044 .unbuffered_writer = .discarding,
1045 .buffer = &.{},
1046 };
1045 var s: Serializer = .{ .writer = &bw };1047 var s: Serializer = .{ .writer = &bw };
1046 var vec2 = try s.beginStruct(.{});1048 var vec2 = try s.beginStruct(.{});
1047 try vec2.field("x", 1.5, .{});1049 try vec2.field("x", 1.5, .{});