authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-15 18:45:21-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:25-07:00
logb26aceba7d5a4e0facb6589b3a6d659ae301d70b
tree1f3f3b776a813d252cdd451e31e021e9d0f02648
parent5356f3a30748ba3504767b68e99d215d6aac839b

thinking about splat being the only function

6 files changed, 196 insertions(+), 113 deletions(-)

lib/std/fs/File.zig+23
......@@ -1592,6 +1592,7 @@ pub fn writer(file: File) std.io.Writer {
15921592 .context = interface.handleToOpaque(file.handle),
15931593 .vtable = &.{
15941594 .writev = interface.writev,
1595 .splat = interface.splat,
15951596 .writeFile = interface.writeFile,
15961597 },
15971598 };
......@@ -1628,6 +1629,28 @@ const interface = struct {
16281629 return std.posix.writev(file, iovecs);
16291630 }
16301631
1632 fn splat(context: *anyopaque, headers: []const []const u8, pattern: []const u8, n: usize) anyerror!usize {
1633 const file = opaqueToHandle(context);
1634
1635 if (is_windows) {
1636 // TODO improve this to use WriteFileScatter
1637 if (headers.len > 0) {
1638 const first = headers[0];
1639 return windows.WriteFile(file, first, null);
1640 }
1641 if (n > 0) return windows.WriteFile(file, pattern, null);
1642 return 0;
1643 }
1644
1645 var iovecs_buffer: [max_buffers_len]std.posix.iovec_const = undefined;
1646 const iovecs = iovecs_buffer[0..@min(iovecs_buffer.len, headers.len)];
1647 for (iovecs, headers[0..iovecs.len]) |*v, d| v.* = .{
1648 .base = if (d.len == 0) "" else d.ptr, // OS sadly checks ptr addr before length.
1649 .len = d.len,
1650 };
1651 return std.posix.writev(file, iovecs);
1652 }
1653
16311654 fn writeFile(
16321655 context: *anyopaque,
16331656 in_file: std.fs.File,
lib/std/io.zig+11-3
......@@ -345,15 +345,23 @@ pub const null_writer: Writer = .{
345345 .context = undefined,
346346 .vtable = &.{
347347 .writev = null_writev,
348 .splat = null_splat,
348349 .writeFile = null_writeFile,
349350 },
350351};
351352
352353fn null_writev(context: *anyopaque, data: []const []const u8) anyerror!usize {
353354 _ = context;
354 var n: usize = 0;
355 for (data) |bytes| n += bytes.len;
356 return n;
355 var written: usize = 0;
356 for (data) |bytes| written += bytes.len;
357 return written;
358}
359
360fn null_splat(context: *anyopaque, headers: []const []const u8, pattern: []const u8, n: usize) anyerror!usize {
361 _ = context;
362 var written: usize = pattern.len * n;
363 for (headers) |bytes| written += bytes.len;
364 return written;
357365}
358366
359367fn null_writeFile(
lib/std/io/AllocatingWriter.zig+13-4
......@@ -99,20 +99,29 @@ pub fn clearRetainingCapacity(aw: *AllocatingWriter) void {
9999}
100100
101101fn writev(context: *anyopaque, data: []const []const u8) anyerror!usize {
102 return splat(context, data, &.{}, 0);
103}
104
105fn splat(context: *anyopaque, headers: []const []const u8, pattern: []const u8, n: usize) anyerror!usize {
102106 const aw: *AllocatingWriter = @alignCast(@ptrCast(context));
103107 const start_len = aw.written.len;
104108 const bw = &aw.buffered_writer;
105 assert(data[0].ptr == aw.written.ptr + start_len);
109 assert(headers[0].ptr == aw.written.ptr + start_len);
106110 var list: std.ArrayListUnmanaged(u8) = .{
107 .items = aw.written.ptr[0 .. start_len + data[0].len],
111 .items = aw.written.ptr[0 .. start_len + headers[0].len],
108112 .capacity = start_len + bw.buffer.len,
109113 };
110114 defer setArrayList(aw, list);
111 const rest = data[1..];
112 var new_capacity: usize = list.capacity;
115 const rest = headers[1..];
116 var new_capacity: usize = list.capacity + pattern.len * n;
113117 for (rest) |bytes| new_capacity += bytes.len;
114118 try list.ensureTotalCapacity(aw.allocator, new_capacity + 1);
115119 for (rest) |bytes| list.appendSliceAssumeCapacity(bytes);
120 if (pattern.len == 1) {
121 list.appendNTimesAssumeCapacity(pattern[0], n);
122 } else {
123 for (0..n) |_| list.appendSliceAssumeCapacity(pattern);
124 }
116125 aw.written = list.items;
117126 bw.buffer = list.unusedCapacitySlice();
118127 return list.items.len - start_len;
lib/std/io/BufferedWriter.zig+122-100
......@@ -6,6 +6,13 @@ const Writer = std.io.Writer;
66const testing = std.testing;
77
88/// Underlying stream to send bytes to.
9///
10/// A write will only be sent here if it could not fit into `buffer`, or if it
11/// is a `writeFile`.
12///
13/// `unbuffered_writer` may modify `buffer` if the number of bytes returned
14/// equals number of bytes provided. This property is exploited by
15/// `std.io.AllocatingWriter` for example.
916unbuffered_writer: Writer,
1017/// User-provided storage that must outlive this `BufferedWriter`.
1118///
......@@ -27,6 +34,7 @@ pub fn writer(bw: *BufferedWriter) Writer {
2734 .context = bw,
2835 .vtable = &.{
2936 .writev = passthru_writev,
37 .splat = passthru_splat,
3038 .writeFile = passthru_writeFile,
3139 },
3240 };
......@@ -34,6 +42,7 @@ pub fn writer(bw: *BufferedWriter) Writer {
3442
3543const fixed_vtable: Writer.VTable = .{
3644 .writev = fixed_writev,
45 .splat = fixed_splat,
3746 .writeFile = fixed_writeFile,
3847};
3948
......@@ -62,7 +71,8 @@ pub fn reset(bw: *BufferedWriter) void {
6271}
6372
6473pub fn flush(bw: *BufferedWriter) anyerror!void {
65 try bw.unbuffered_writer.writeAll(bw.buffer[0..bw.end]);
74 const send_buffer = bw.buffer[0..bw.end];
75 try bw.unbuffered_writer.writeAll(send_buffer);
6676 bw.end = 0;
6777}
6878
......@@ -120,6 +130,102 @@ fn passthru_writev(context: *anyopaque, data: []const []const u8) anyerror!usize
120130 return end - start_end;
121131}
122132
133fn passthru_splat(context: *anyopaque, headers: []const []const u8, pattern: []const u8, n: usize) anyerror!usize {
134 const bw: *BufferedWriter = @alignCast(@ptrCast(context));
135 const buffer = bw.buffer;
136 const start_end = bw.end;
137
138 var end = bw.end;
139 for (headers, 0..) |bytes, i| {
140 const new_end = end + bytes.len;
141 if (new_end <= buffer.len) {
142 @branchHint(.likely);
143 @memcpy(buffer[end..new_end], bytes);
144 end = new_end;
145 continue;
146 }
147 if (end == 0) return bw.unbuffered_writer.splat(headers, pattern, n);
148 var buffers: [max_buffers_len][]const u8 = undefined;
149 buffers[0] = buffer[0..end];
150 const remaining_headers = headers[i..];
151 const remaining_buffers = buffers[1..];
152 const len: usize = @min(remaining_headers.len, remaining_buffers.len);
153 @memcpy(remaining_buffers[0..len], remaining_headers[0..len]);
154 const send_buffers = buffers[0 .. len + 1];
155 if (len >= remaining_headers.len) {
156 @branchHint(.likely);
157 // Made it past the headers, so we can call `splat`.
158 const written = try bw.unbuffered_writer.splat(send_buffers, pattern, n);
159 if (written < end) {
160 @branchHint(.unlikely);
161 const remainder = buffer[written..end];
162 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
163 bw.end = remainder.len;
164 return end - start_end;
165 }
166 bw.end = 0;
167 return written - start_end;
168 }
169 const written = try bw.unbuffered_writer.writev(send_buffers);
170 if (written < end) {
171 @branchHint(.unlikely);
172 const remainder = buffer[written..end];
173 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
174 bw.end = remainder.len;
175 return end - start_end;
176 }
177 bw.end = 0;
178 return written - start_end;
179 }
180
181 switch (pattern.len) {
182 0 => {
183 bw.end = end;
184 return end - start_end;
185 },
186 1 => {
187 const new_end = end + n;
188 if (new_end <= buffer.len) {
189 @branchHint(.likely);
190 @memset(buffer[end..new_end], pattern[0]);
191 bw.end = new_end;
192 return end - start_end;
193 }
194 const written = try bw.unbuffered_writer.splat(buffer[0..end], pattern, n);
195 if (written < end) {
196 @branchHint(.unlikely);
197 const remainder = buffer[written..end];
198 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
199 bw.end = remainder.len;
200 return end - start_end;
201 }
202 bw.end = 0;
203 return written - start_end;
204 },
205 else => {
206 const new_end = end + pattern.len * n;
207 if (new_end <= buffer.len) {
208 @branchHint(.likely);
209 while (end < new_end) : (end += pattern.len) {
210 @memcpy(buffer[end..][0..pattern.len], pattern);
211 }
212 bw.end = end;
213 return end - start_end;
214 }
215 const written = try bw.unbuffered_writer.splat(buffer[0..end], pattern, n);
216 if (written < end) {
217 @branchHint(.unlikely);
218 const remainder = buffer[written..end];
219 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
220 bw.end = remainder.len;
221 return end - start_end;
222 }
223 bw.end = 0;
224 return written - start_end;
225 },
226 }
227}
228
123229fn fixed_writev(context: *anyopaque, data: []const []const u8) anyerror!usize {
124230 const bw: *BufferedWriter = @alignCast(@ptrCast(context));
125231 // When this function is called it means the buffer got full, so it's time
......@@ -131,6 +237,19 @@ fn fixed_writev(context: *anyopaque, data: []const []const u8) anyerror!usize {
131237 return error.NoSpaceLeft;
132238}
133239
240fn fixed_splat(context: *anyopaque, headers: []const []const u8, pattern: []const u8, n: usize) anyerror!usize {
241 const bw: *BufferedWriter = @alignCast(@ptrCast(context));
242 const dest = bw.buffer[bw.end..];
243 if (headers.len > 0) {
244 @memcpy(dest, headers[0][0..dest.len]);
245 } else switch (pattern.len) {
246 0 => unreachable,
247 1 => @memset(dest, pattern[0]),
248 else => for (0..n) |i| @memcpy(dest[i * pattern.len ..][0..pattern.len], pattern),
249 }
250 return error.NoSpaceLeft;
251}
252
134253pub fn write(bw: *BufferedWriter, bytes: []const u8) anyerror!usize {
135254 const buffer = bw.buffer;
136255 const end = bw.end;
......@@ -210,71 +329,7 @@ pub fn splatByteAll(bw: *BufferedWriter, byte: u8, n: usize) anyerror!void {
210329///
211330/// Does maximum of one underlying `Writer.VTable.writev`.
212331pub fn splatByte(bw: *BufferedWriter, byte: u8, n: usize) anyerror!usize {
213 const buffer = bw.buffer;
214 const end = bw.end;
215
216 const new_end = end + n;
217 if (new_end <= buffer.len) {
218 @branchHint(.likely);
219 @memset(buffer[end..][0..n], byte);
220 bw.end = new_end;
221 return n;
222 }
223
224 if (n <= buffer.len) {
225 const written = try bw.unbuffered_writer.write(buffer[0..end]);
226 if (written < end) {
227 @branchHint(.unlikely);
228 const remainder = buffer[written..end];
229 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
230 bw.end = remainder.len;
231 return 0;
232 }
233 assert(bw.buffer.ptr == buffer.ptr); // TODO this is not a valid assertion
234 @memset(buffer[0..n], byte);
235 bw.end = n;
236 return n;
237 }
238
239 // First try to use only the unused buffer region, to make an attempt for a
240 // single `writev`.
241 const free_space = buffer[end..];
242 var remaining = n - free_space.len;
243 @memset(free_space, byte);
244 var buffers: [max_buffers_len][]const u8 = undefined;
245 buffers[0] = buffer;
246 var buffer_i: usize = 1;
247 while (remaining > free_space.len and buffer_i < buffers.len) {
248 buffers[buffer_i] = free_space;
249 buffer_i += 1;
250 remaining -= free_space.len;
251 }
252 if (remaining > 0 and buffer_i < buffers.len) {
253 buffers[buffer_i] = free_space[0..remaining];
254 buffer_i += 1;
255 const written = try bw.unbuffered_writer.writev(buffers[0..buffer_i]);
256 if (written < end) {
257 @branchHint(.unlikely);
258 const remainder = buffer[written..end];
259 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
260 bw.end = remainder.len;
261 return 0;
262 }
263 bw.end = 0;
264 return written - end;
265 }
266
267 const written = try bw.unbuffered_writer.writev(buffers[0..buffer_i]);
268 if (written < end) {
269 @branchHint(.unlikely);
270 const remainder = buffer[written..end];
271 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
272 bw.end = remainder.len;
273 return 0;
274 }
275
276 bw.end = 0;
277 return written - end;
332 return passthru_splat(bw, &.{}, &.{byte}, n);
278333}
279334
280335/// Writes the same slice many times, performing the underlying write call as
......@@ -288,40 +343,7 @@ pub fn splatBytesAll(bw: *BufferedWriter, bytes: []const u8, n: usize) anyerror!
288343///
289344/// Does maximum of one underlying `Writer.VTable.writev`.
290345pub fn splatBytes(bw: *BufferedWriter, bytes: []const u8, n: usize) anyerror!usize {
291 const buffer = bw.buffer;
292 const start_end = bw.end;
293 var end = start_end;
294 var remaining = n;
295 while (remaining > 0 and end + bytes.len <= buffer.len) {
296 @memcpy(buffer[end..][0..bytes.len], bytes);
297 end += bytes.len;
298 remaining -= 1;
299 }
300
301 if (remaining == 0) {
302 bw.end = end;
303 return end - start_end;
304 }
305
306 var buffers: [max_buffers_len][]const u8 = undefined;
307 var buffer_i: usize = 1;
308 buffers[0] = buffer[0..end];
309 const remaining_buffers = buffers[1..];
310 const buffers_len: usize = @min(remaining, remaining_buffers.len);
311 @memset(remaining_buffers[0..buffers_len], bytes);
312 remaining -= buffers_len;
313 buffer_i += buffers_len;
314
315 const written = try bw.unbuffered_writer.writev(buffers[0..buffer_i]);
316 if (written < end) {
317 @branchHint(.unlikely);
318 const remainder = buffer[written..end];
319 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
320 bw.end = remainder.len;
321 return end - start_end;
322 }
323 bw.end = 0;
324 return written - start_end;
346 return passthru_splat(bw, &.{}, bytes, n);
325347}
326348
327349/// Asserts the `buffer` was initialized with a capacity of at least `@sizeOf(T)` bytes.
lib/std/io/CountingWriter.zig+14-6
......@@ -14,6 +14,7 @@ pub fn writer(cw: *CountingWriter) Writer {
1414 .context = cw,
1515 .vtable = &.{
1616 .writev = passthru_writev,
17 .splat = passthru_splat,
1718 .writeFile = passthru_writeFile,
1819 },
1920 };
......@@ -28,9 +29,16 @@ pub fn unbufferedWriter(cw: *CountingWriter) std.io.BufferedWriter {
2829
2930fn passthru_writev(context: *anyopaque, data: []const []const u8) anyerror!usize {
3031 const cw: *CountingWriter = @alignCast(@ptrCast(context));
31 const n = try cw.child_writer.writev(data);
32 cw.bytes_written += n;
33 return n;
32 const written = try cw.child_writer.writev(data);
33 cw.bytes_written += written;
34 return written;
35}
36
37fn passthru_splat(context: *anyopaque, header: []const u8, pattern: []const u8, n: usize) anyerror!usize {
38 const cw: *CountingWriter = @alignCast(@ptrCast(context));
39 const written = try cw.child_writer.splat(header, pattern, n);
40 cw.bytes_written += written;
41 return written;
3442}
3543
3644fn passthru_writeFile(
......@@ -42,9 +50,9 @@ fn passthru_writeFile(
4250 headers_len: usize,
4351) anyerror!usize {
4452 const cw: *CountingWriter = @alignCast(@ptrCast(context));
45 const n = try cw.child_writer.writeFile(file, offset, len, headers_and_trailers, headers_len);
46 cw.bytes_written += n;
47 return n;
53 const written = try cw.child_writer.writeFile(file, offset, len, headers_and_trailers, headers_len);
54 cw.bytes_written += written;
55 return written;
4856}
4957
5058test CountingWriter {
lib/std/io/Writer.zig+13
......@@ -15,6 +15,19 @@ pub const VTable = struct {
1515 /// of stream via an error.
1616 writev: *const fn (context: *anyopaque, data: []const []const u8) anyerror!usize,
1717
18 /// `headers_and_pattern` must have length of at least one. The last slice
19 /// is `pattern` which is the byte sequence to repeat `n` times. The rest
20 /// of the slices are headers to write before the pattern.
21 ///
22 /// When `n == 1`, this is equivalent to `writev`.
23 ///
24 /// Number of bytes actually written is returned.
25 ///
26 /// Number of bytes returned may be zero, which does not mean
27 /// end-of-stream. A subsequent call may return nonzero, or may signal end
28 /// of stream via an error.
29 splat: *const fn (context: *anyopaque, headers_and_pattern: []const []const u8, n: usize) anyerror!usize,
30
1831 /// Writes contents from an open file. `headers` are written first, then `len`
1932 /// bytes of `file` starting from `offset`, then `trailers`.
2033 ///