authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-10-01 10:53:39-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-10-01 10:53:39-04:00
logd1ec8377d1fcc9874c40e6603f64087f0b310677
treec96fbee62356b2ead89d8ea3f4a1d408710df7c4
parent9d4eaf1e07525a72fa54cfaa346a18bf18953af4
signature Commit is signed but in an unrecognized format.

std lib: flesh out the async I/O streaming API a bit

7 files changed, 187 insertions(+), 71 deletions(-)

src/analyze.cpp+5
......@@ -2710,6 +2710,11 @@ static Error resolve_struct_alignment(CodeGen *g, ZigType *struct_type) {
27102710 // be resolving ResolveStatusZeroBitsKnown
27112711 assert(field->type_entry != nullptr);
27122712
2713 if (type_is_invalid(field->type_entry)) {
2714 struct_type->data.structure.resolve_status = ResolveStatusInvalid;
2715 break;
2716 }
2717
27132718 if (!type_has_bits(field->type_entry))
27142719 continue;
27152720
std/atomic/int.zig+4
......@@ -29,5 +29,9 @@ pub fn Int(comptime T: type) type {
2929 pub fn xchg(self: *Self, new_value: T) T {
3030 return @atomicRmw(T, &self.unprotected_value, builtin.AtomicRmwOp.Xchg, new_value, AtomicOrder.SeqCst);
3131 }
32
33 pub fn fetchAdd(self: *Self, op: T) T {
34 return @atomicRmw(T, &self.unprotected_value, builtin.AtomicRmwOp.Add, op, AtomicOrder.SeqCst);
35 }
3236 };
3337}
std/event/fs.zig+128-46
......@@ -30,20 +30,20 @@ pub const Request = struct {
3030
3131 pub const PWriteV = struct {
3232 fd: os.FileHandle,
33 iov: []os.posix.iovec_const,
33 iov: []const os.posix.iovec_const,
3434 offset: usize,
3535 result: Error!void,
3636
37 pub const Error = os.File.WriteError;
37 pub const Error = os.PosixWriteError;
3838 };
3939
4040 pub const PReadV = struct {
4141 fd: os.FileHandle,
42 iov: []os.posix.iovec,
42 iov: []const os.posix.iovec,
4343 offset: usize,
4444 result: Error!usize,
4545
46 pub const Error = os.File.ReadError;
46 pub const Error = os.PosixReadError;
4747 };
4848
4949 pub const Open = struct {
......@@ -72,28 +72,47 @@ pub const Request = struct {
7272 };
7373};
7474
75pub const PWriteVError = error{OutOfMemory} || os.File.WriteError;
76
7577/// data - just the inner references - must live until pwritev promise completes.
76pub async fn pwritev(loop: *Loop, fd: os.FileHandle, data: []const []const u8, offset: usize) !void {
78pub async fn pwritev(loop: *Loop, fd: os.FileHandle, data: []const []const u8, offset: usize) PWriteVError!void {
79 // workaround for https://github.com/ziglang/zig/issues/1194
80 suspend {
81 resume @handle();
82 }
7783 switch (builtin.os) {
7884 builtin.Os.macosx,
7985 builtin.Os.linux,
80 => return await (async pwritevPosix(loop, fd, data, offset) catch unreachable),
81 builtin.Os.windows => return await (async pwritevWindows(loop, fd, data, offset) catch unreachable),
86 => {
87 const iovecs = try loop.allocator.alloc(os.posix.iovec_const, data.len);
88 defer loop.allocator.free(iovecs);
89
90 for (data) |buf, i| {
91 iovecs[i] = os.posix.iovec_const{
92 .iov_base = buf.ptr,
93 .iov_len = buf.len,
94 };
95 }
96
97 return await (async pwritevPosix(loop, fd, iovecs, offset) catch unreachable);
98 },
99 builtin.Os.windows => {
100 const data_copy = try std.mem.dupe(loop.allocator, []const u8, data);
101 defer loop.allocator.free(data_copy);
102 return await (async pwritevWindows(loop, fd, data, offset) catch unreachable);
103 },
82104 else => @compileError("Unsupported OS"),
83105 }
84106}
85107
86/// data - just the inner references - must live until pwritev promise completes.
87pub async fn pwritevWindows(loop: *Loop, fd: os.FileHandle, data: []const []const u8, offset: usize) !void {
108/// data must outlive the returned promise
109pub async fn pwritevWindows(loop: *Loop, fd: os.FileHandle, data: []const []const u8, offset: usize) os.WindowsWriteError!void {
88110 if (data.len == 0) return;
89111 if (data.len == 1) return await (async pwriteWindows(loop, fd, data[0], offset) catch unreachable);
90112
91 const data_copy = try std.mem.dupe(loop.allocator, []const u8, data);
92 defer loop.allocator.free(data_copy);
93
94113 // TODO do these in parallel
95114 var off = offset;
96 for (data_copy) |buf| {
115 for (data) |buf| {
97116 try await (async pwriteWindows(loop, fd, buf, off) catch unreachable);
98117 off += buf.len;
99118 }
......@@ -144,23 +163,18 @@ pub async fn pwriteWindows(loop: *Loop, fd: os.FileHandle, data: []const u8, off
144163 }
145164}
146165
147/// data - just the inner references - must live until pwritev promise completes.
148pub async fn pwritevPosix(loop: *Loop, fd: os.FileHandle, data: []const []const u8, offset: usize) !void {
166/// iovecs must live until pwritev promise completes.
167pub async fn pwritevPosix(
168 loop: *Loop,
169 fd: os.FileHandle,
170 iovecs: []const posix.iovec_const,
171 offset: usize,
172) os.PosixWriteError!void {
149173 // workaround for https://github.com/ziglang/zig/issues/1194
150174 suspend {
151175 resume @handle();
152176 }
153177
154 const iovecs = try loop.allocator.alloc(os.posix.iovec_const, data.len);
155 defer loop.allocator.free(iovecs);
156
157 for (data) |buf, i| {
158 iovecs[i] = os.posix.iovec_const{
159 .iov_base = buf.ptr,
160 .iov_len = buf.len,
161 };
162 }
163
164178 var req_node = RequestNode{
165179 .prev = null,
166180 .next = null,
......@@ -192,38 +206,59 @@ pub async fn pwritevPosix(loop: *Loop, fd: os.FileHandle, data: []const []const
192206 return req_node.data.msg.PWriteV.result;
193207}
194208
209pub const PReadVError = error{OutOfMemory} || os.File.ReadError;
210
195211/// data - just the inner references - must live until preadv promise completes.
196pub async fn preadv(loop: *Loop, fd: os.FileHandle, data: []const []u8, offset: usize) !usize {
212pub async fn preadv(loop: *Loop, fd: os.FileHandle, data: []const []u8, offset: usize) PReadVError!usize {
213 // workaround for https://github.com/ziglang/zig/issues/1194
214 suspend {
215 resume @handle();
216 }
217
197218 assert(data.len != 0);
198219 switch (builtin.os) {
199220 builtin.Os.macosx,
200221 builtin.Os.linux,
201 => return await (async preadvPosix(loop, fd, data, offset) catch unreachable),
202 builtin.Os.windows => return await (async preadvWindows(loop, fd, data, offset) catch unreachable),
222 => {
223 const iovecs = try loop.allocator.alloc(os.posix.iovec, data.len);
224 defer loop.allocator.free(iovecs);
225
226 for (data) |buf, i| {
227 iovecs[i] = os.posix.iovec{
228 .iov_base = buf.ptr,
229 .iov_len = buf.len,
230 };
231 }
232
233 return await (async preadvPosix(loop, fd, iovecs, offset) catch unreachable);
234 },
235 builtin.Os.windows => {
236 const data_copy = try std.mem.dupe(loop.allocator, []u8, data);
237 defer loop.allocator.free(data_copy);
238 return await (async preadvWindows(loop, fd, data_copy, offset) catch unreachable);
239 },
203240 else => @compileError("Unsupported OS"),
204241 }
205242}
206243
207pub async fn preadvWindows(loop: *Loop, fd: os.FileHandle, data: []const []u8, offset: u64) !usize {
244/// data must outlive the returned promise
245pub async fn preadvWindows(loop: *Loop, fd: os.FileHandle, data: []const []u8, offset: u64) os.WindowsReadError!usize {
208246 assert(data.len != 0);
209247 if (data.len == 1) return await (async preadWindows(loop, fd, data[0], offset) catch unreachable);
210248
211 const data_copy = try std.mem.dupe(loop.allocator, []u8, data);
212 defer loop.allocator.free(data_copy);
213
214249 // TODO do these in parallel?
215250 var off: usize = 0;
216251 var iov_i: usize = 0;
217252 var inner_off: usize = 0;
218253 while (true) {
219 const v = data_copy[iov_i];
254 const v = data[iov_i];
220255 const amt_read = try await (async preadWindows(loop, fd, v[inner_off .. v.len - inner_off], offset + off) catch unreachable);
221256 off += amt_read;
222257 inner_off += amt_read;
223258 if (inner_off == v.len) {
224259 iov_i += 1;
225260 inner_off = 0;
226 if (iov_i == data_copy.len) {
261 if (iov_i == data.len) {
227262 return off;
228263 }
229264 }
......@@ -275,23 +310,18 @@ pub async fn preadWindows(loop: *Loop, fd: os.FileHandle, data: []u8, offset: u6
275310 return usize(bytes_transferred);
276311}
277312
278/// data - just the inner references - must live until preadv promise completes.
279pub async fn preadvPosix(loop: *Loop, fd: os.FileHandle, data: []const []u8, offset: usize) !usize {
313/// iovecs must live until preadv promise completes
314pub async fn preadvPosix(
315 loop: *Loop,
316 fd: os.FileHandle,
317 iovecs: []const posix.iovec,
318 offset: usize,
319) os.PosixReadError!usize {
280320 // workaround for https://github.com/ziglang/zig/issues/1194
281321 suspend {
282322 resume @handle();
283323 }
284324
285 const iovecs = try loop.allocator.alloc(os.posix.iovec, data.len);
286 defer loop.allocator.free(iovecs);
287
288 for (data) |buf, i| {
289 iovecs[i] = os.posix.iovec{
290 .iov_base = buf.ptr,
291 .iov_len = buf.len,
292 };
293 }
294
295325 var req_node = RequestNode{
296326 .prev = null,
297327 .next = null,
......@@ -1339,3 +1369,55 @@ async fn testFsWatch(loop: *Loop) !void {
13391369
13401370 // TODO test deleting the file and then re-adding it. we should get events for both
13411371}
1372
1373pub const OutStream = struct {
1374 fd: os.FileHandle,
1375 stream: Stream,
1376 loop: *Loop,
1377 offset: usize,
1378
1379 pub const Error = os.File.WriteError;
1380 pub const Stream = event.io.OutStream(Error);
1381
1382 pub fn init(loop: *Loop, fd: os.FileHandle, offset: usize) OutStream {
1383 return OutStream{
1384 .fd = fd,
1385 .loop = loop,
1386 .offset = offset,
1387 .stream = Stream{ .writeFn = writeFn },
1388 };
1389 }
1390
1391 async<*mem.Allocator> fn writeFn(out_stream: *Stream, bytes: []const u8) Error!void {
1392 const self = @fieldParentPtr(OutStream, "stream", out_stream);
1393 const offset = self.offset;
1394 self.offset += bytes.len;
1395 return await (async pwritev(self.loop, self.fd, [][]const u8{bytes}, offset) catch unreachable);
1396 }
1397};
1398
1399pub const InStream = struct {
1400 fd: os.FileHandle,
1401 stream: Stream,
1402 loop: *Loop,
1403 offset: usize,
1404
1405 pub const Error = PReadVError; // TODO make this not have OutOfMemory
1406 pub const Stream = event.io.InStream(Error);
1407
1408 pub fn init(loop: *Loop, fd: os.FileHandle, offset: usize) InStream {
1409 return InStream{
1410 .fd = fd,
1411 .loop = loop,
1412 .offset = offset,
1413 .stream = Stream{ .readFn = readFn },
1414 };
1415 }
1416
1417 async<*mem.Allocator> fn readFn(in_stream: *Stream, bytes: []u8) Error!usize {
1418 const self = @fieldParentPtr(InStream, "stream", in_stream);
1419 const amt = try await (async preadv(self.loop, self.fd, [][]u8{bytes}, self.offset) catch unreachable);
1420 self.offset += amt;
1421 return amt;
1422 }
1423};
std/event/io.zig+15
......@@ -2,6 +2,7 @@ const std = @import("../index.zig");
22const builtin = @import("builtin");
33const Allocator = std.mem.Allocator;
44const assert = std.debug.assert;
5const mem = std.mem;
56
67pub fn InStream(comptime ReadError: type) type {
78 return struct {
......@@ -20,6 +21,20 @@ pub fn InStream(comptime ReadError: type) type {
2021 return await (async self.readFn(self, buffer) catch unreachable);
2122 }
2223
24 pub async fn readIntLe(self: *Self, comptime T: type) !T {
25 return await (async self.readInt(builtin.Endian.Little, T) catch unreachable);
26 }
27
28 pub async fn readIntBe(self: *Self, comptime T: type) !T {
29 return await (async self.readInt(builtin.Endian.Big, T) catch unreachable);
30 }
31
32 pub async fn readInt(self: *Self, endian: builtin.Endian, comptime T: type) !T {
33 var bytes: [@sizeOf(T)]u8 = undefined;
34 try await (async self.readFull(bytes[0..]) catch unreachable);
35 return mem.readInt(bytes, T, endian);
36 }
37
2338 /// Same as `read` but end of stream returns `error.EndOfStream`.
2439 pub async fn readFull(self: *Self, buf: []u8) !void {
2540 var index: usize = 0;
std/os/file.zig+2-9
......@@ -365,14 +365,7 @@ pub const File = struct {
365365 }
366366 }
367367
368 pub const ReadError = error{
369 FileClosed,
370 InputOutput,
371 IsDir,
372 SystemResources,
373
374 Unexpected,
375 };
368 pub const ReadError = os.WindowsReadError || os.PosixReadError;
376369
377370 pub fn read(self: File, buffer: []u8) ReadError!usize {
378371 if (is_posix) {
......@@ -386,7 +379,7 @@ pub const File = struct {
386379 posix.EINVAL => unreachable,
387380 posix.EFAULT => unreachable,
388381 posix.EAGAIN => unreachable,
389 posix.EBADF => return error.FileClosed,
382 posix.EBADF => unreachable, // always a race condition
390383 posix.EIO => return error.InputOutput,
391384 posix.EISDIR => return error.IsDir,
392385 posix.ENOBUFS => return error.SystemResources,
std/os/index.zig+27-16
......@@ -72,6 +72,7 @@ pub const windowsGetQueuedCompletionStatus = windows_util.windowsGetQueuedComple
7272pub const WindowsWaitError = windows_util.WaitError;
7373pub const WindowsOpenError = windows_util.OpenError;
7474pub const WindowsWriteError = windows_util.WriteError;
75pub const WindowsReadError = windows_util.ReadError;
7576
7677pub const FileHandle = if (is_windows) windows.HANDLE else i32;
7778
......@@ -227,6 +228,13 @@ pub fn close(handle: FileHandle) void {
227228 }
228229}
229230
231pub const PosixReadError = error{
232 InputOutput,
233 SystemResources,
234 IsDir,
235 Unexpected,
236};
237
230238/// Calls POSIX read, and keeps trying if it gets interrupted.
231239pub fn posixRead(fd: i32, buf: []u8) !void {
232240 // Linux can return EINVAL when read amount is > 0x7ffff000
......@@ -238,24 +246,27 @@ pub fn posixRead(fd: i32, buf: []u8) !void {
238246 const want_to_read = math.min(buf.len - index, usize(max_buf_len));
239247 const rc = posix.read(fd, buf.ptr + index, want_to_read);
240248 const err = posix.getErrno(rc);
241 if (err > 0) {
242 return switch (err) {
243 posix.EINTR => continue,
244 posix.EINVAL, posix.EFAULT => unreachable,
245 posix.EAGAIN => unreachable,
246 posix.EBADF => unreachable, // always a race condition
247 posix.EIO => error.InputOutput,
248 posix.EISDIR => error.IsDir,
249 posix.ENOBUFS, posix.ENOMEM => error.SystemResources,
250 else => unexpectedErrorPosix(err),
251 };
249 switch (err) {
250 0 => {
251 index += rc;
252 continue;
253 },
254 posix.EINTR => continue,
255 posix.EINVAL => unreachable,
256 posix.EFAULT => unreachable,
257 posix.EAGAIN => unreachable,
258 posix.EBADF => unreachable, // always a race condition
259 posix.EIO => return error.InputOutput,
260 posix.EISDIR => return error.IsDir,
261 posix.ENOBUFS => return error.SystemResources,
262 posix.ENOMEM => return error.SystemResources,
263 else => return unexpectedErrorPosix(err),
252264 }
253 index += rc;
254265 }
255266}
256267
257268/// Number of bytes read is returned. Upon reading end-of-file, zero is returned.
258pub fn posix_preadv(fd: i32, iov: [*]const posix.iovec, count: usize, offset: u64) !usize {
269pub fn posix_preadv(fd: i32, iov: [*]const posix.iovec, count: usize, offset: u64) PosixReadError!usize {
259270 switch (builtin.os) {
260271 builtin.Os.macosx => {
261272 // Darwin does not have preadv but it does have pread.
......@@ -284,7 +295,7 @@ pub fn posix_preadv(fd: i32, iov: [*]const posix.iovec, count: usize, offset: u6
284295 posix.EINVAL => unreachable,
285296 posix.EFAULT => unreachable,
286297 posix.ESPIPE => unreachable, // fd is not seekable
287 posix.EAGAIN => unreachable, // use posixAsyncPReadV for non blocking
298 posix.EAGAIN => unreachable, // this function is not for non blocking
288299 posix.EBADF => unreachable, // always a race condition
289300 posix.EIO => return error.InputOutput,
290301 posix.EISDIR => return error.IsDir,
......@@ -302,7 +313,7 @@ pub fn posix_preadv(fd: i32, iov: [*]const posix.iovec, count: usize, offset: u6
302313 posix.EINTR => continue,
303314 posix.EINVAL => unreachable,
304315 posix.EFAULT => unreachable,
305 posix.EAGAIN => unreachable, // use posixAsyncPReadV for non blocking
316 posix.EAGAIN => unreachable, // don't call this function for non blocking
306317 posix.EBADF => unreachable, // always a race condition
307318 posix.EIO => return error.InputOutput,
308319 posix.EISDIR => return error.IsDir,
......@@ -328,7 +339,7 @@ pub const PosixWriteError = error{
328339};
329340
330341/// Calls POSIX write, and keeps trying if it gets interrupted.
331pub fn posixWrite(fd: i32, bytes: []const u8) !void {
342pub fn posixWrite(fd: i32, bytes: []const u8) PosixWriteError!void {
332343 // Linux can return EINVAL when write amount is > 0x7ffff000
333344 // See https://github.com/ziglang/zig/pull/743#issuecomment-363165856
334345 const max_bytes_len = 0x7ffff000;
std/os/windows/util.zig+6
......@@ -42,6 +42,12 @@ pub fn windowsClose(handle: windows.HANDLE) void {
4242 assert(windows.CloseHandle(handle) != 0);
4343}
4444
45pub const ReadError = error{
46 OperationAborted,
47 BrokenPipe,
48 Unexpected,
49};
50
4551pub const WriteError = error{
4652 SystemResources,
4753 OperationAborted,