authorgravatar for joran@ronomon.comJoran Dirk Greef <joran@ronomon.com> 2020-09-19 16:18:04+02:00
committergravatar for joran@ronomon.comJoran Dirk Greef <joran@ronomon.com> 2020-09-19 16:18:04+02:00
log31533eb74300ad934d3fca11ffebd86fe67a31ba
treede4457fbdd30008be971fd194b4975d27007720e
parent873d1c80b3a34dc610946fb31de9dd88dd311d35

Move to std/os/linux

3 files changed, 802 insertions(+), 843 deletions(-)

lib/std/io_uring.zig deleted-843
...@@ -1,843 +0,0 @@
1// SPDX-License-Identifier: MIT
2// Copyright (c) 2015-2020 Zig Contributors
3// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
4// The MIT license requires this copyright notice to be included in all copies
5// and substantial portions of the software.
6const std = @import("std.zig");
7const assert = std.debug.assert;
8const builtin = std.builtin;
9const os = std.os;
10const linux = os.linux;
11const mem = std.mem;
12const net = std.net;
13const testing = std.testing;
14
15pub const io_uring_params = linux.io_uring_params;
16pub const io_uring_cqe = linux.io_uring_cqe;
17
18// TODO Update linux.zig's definition of linux.io_uring_sqe:
19// linux.io_uring_sqe uses numbered unions, i.e. `union1` etc. that are not future-proof and need to
20// be re-numbered whenever new unions are interposed by the kernel. Furthermore, Zig's unions do not
21// support assignment by any union member directly as in C, without going through the union, so the
22// kernel adding new unions would also break existing Zig code.
23// We therefore use a flat struct without unions to avoid these two issues.
24// Pending https://github.com/ziglang/zig/issues/6349.
25pub const io_uring_sqe = extern struct {
26 opcode: linux.IORING_OP,
27 flags: u8 = 0,
28 ioprio: u16 = 0,
29 fd: i32 = 0,
30 off: u64 = 0,
31 addr: u64 = 0,
32 len: u32 = 0,
33 opflags: u32 = 0,
34 user_data: u64 = 0,
35 buffer: u16 = 0,
36 personality: u16 = 0,
37 splice_fd_in: i32 = 0,
38 options: [2]u64 = [2]u64{ 0, 0 }
39};
40
41comptime {
42 assert(@sizeOf(io_uring_params) == 120);
43 assert(@sizeOf(io_uring_sqe) == 64);
44 assert(@sizeOf(io_uring_cqe) == 16);
45
46 assert(linux.IORING_OFF_SQ_RING == 0);
47 assert(linux.IORING_OFF_CQ_RING == 0x8000000);
48 assert(linux.IORING_OFF_SQES == 0x10000000);
49}
50
51pub const IO_Uring = struct {
52 fd: i32 = -1,
53 sq: SubmissionQueue,
54 cq: CompletionQueue,
55 flags: u32,
56
57 /// A friendly way to setup an io_uring, with default io_uring_params.
58 /// `entries` must be a power of two between 1 and 4096, although the kernel will make the final
59 /// call on how many entries the submission and completion queues will ultimately have,
60 /// see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L8027-L8050.
61 /// Matches the interface of io_uring_queue_init() in liburing.
62 pub fn init(entries: u32, flags: u32) !IO_Uring {
63 var params = io_uring_params {
64 .sq_entries = 0,
65 .cq_entries = 0,
66 .flags = flags,
67 .sq_thread_cpu = 0,
68 .sq_thread_idle = 1000,
69 .features = 0,
70 .wq_fd = 0,
71 .resv = [_]u32{0} ** 3,
72 .sq_off = undefined,
73 .cq_off = undefined,
74 };
75 // The kernel will zero the memory of the sq_off and cq_off structs in io_uring_create(),
76 // see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L7986-L8002.
77 return try IO_Uring.init_params(entries, &params);
78 }
79
80 /// A powerful way to setup an io_uring, if you want to tweak io_uring_params such as submission
81 /// queue thread cpu affinity or thread idle timeout (the kernel and our default is 1 second).
82 /// `params` is passed by reference because the kernel needs to modify the parameters.
83 /// You may only set the `flags`, `sq_thread_cpu` and `sq_thread_idle` parameters.
84 /// Every other parameter belongs to the kernel and must be zeroed.
85 /// Matches the interface of io_uring_queue_init_params() in liburing.
86 pub fn init_params(entries: u32, p: *io_uring_params) !IO_Uring {
87 assert(entries >= 1 and entries <= 4096 and std.math.isPowerOfTwo(entries));
88 assert(p.sq_entries == 0);
89 assert(p.cq_entries == 0);
90 assert(p.features == 0);
91 assert(p.wq_fd == 0);
92 assert(p.resv[0] == 0);
93 assert(p.resv[1] == 0);
94 assert(p.resv[2] == 0);
95 if (!supported) return error.IO_UringKernelNotSupported;
96
97 const res = linux.io_uring_setup(entries, p);
98 try check_errno(res);
99 const fd = @intCast(i32, res);
100 assert(fd >= 0);
101 errdefer os.close(fd);
102
103 // Kernel versions 5.4 and up use only one mmap() for the submission and completion queues.
104 // This is not an optional feature for us... if the kernel does it, we have to do it.
105 // The thinking on this by the kernel developers was that both the submission and the
106 // completion queue rings have sizes just over a power of two, but the submission queue ring
107 // is significantly smaller with u32 slots. By bundling both in a single mmap, the kernel
108 // gets the submission queue ring for free.
109 // See https://patchwork.kernel.org/patch/11115257 for the kernel patch.
110 // We do not support the double mmap() done before 5.4, because we want to keep the
111 // init/deinit mmap paths simple and because io_uring has had many bug fixes even since 5.4.
112 if ((p.features & linux.IORING_FEAT_SINGLE_MMAP) == 0) {
113 return error.IO_UringKernelNotSupported;
114 }
115
116 // Check that the kernel has actually set params and that "impossible is nothing".
117 assert(p.sq_entries != 0);
118 assert(p.cq_entries != 0);
119 assert(p.cq_entries >= p.sq_entries);
120
121 // From here on, we only need to read from params, so pass `p` by value as immutable.
122 // The completion queue shares the mmap with the submission queue, so pass `sq` there too.
123 var sq = try SubmissionQueue.init(fd, p.*);
124 errdefer sq.deinit();
125 var cq = try CompletionQueue.init(fd, p.*, sq);
126 errdefer cq.deinit();
127
128 // Check that our starting state is as we expect.
129 assert(sq.head.* == 0);
130 assert(sq.tail.* == 0);
131 assert(sq.mask.* == p.sq_entries - 1);
132 // Allow flags.* to be non-zero, since the kernel may set IORING_SQ_NEED_WAKEUP at any time.
133 assert(sq.dropped.* == 0);
134 assert(sq.array.len == p.sq_entries);
135 assert(sq.sqes.len == p.sq_entries);
136 assert(sq.sqe_head == 0);
137 assert(sq.sqe_tail == 0);
138
139 assert(cq.head.* == 0);
140 assert(cq.tail.* == 0);
141 assert(cq.mask.* == p.cq_entries - 1);
142 assert(cq.overflow.* == 0);
143 assert(cq.cqes.len == p.cq_entries);
144
145 return IO_Uring {
146 .fd = fd,
147 .sq = sq,
148 .cq = cq,
149 .flags = p.flags
150 };
151 }
152
153 pub fn deinit(self: *IO_Uring) void {
154 assert(self.fd >= 0);
155 // The mmaps depend on the fd, so the order of these calls is important:
156 self.cq.deinit();
157 self.sq.deinit();
158 os.close(self.fd);
159 self.fd = -1;
160 }
161
162 /// Returns a vacant SQE, or an error if the submission queue is full.
163 /// We follow the implementation (and atomics) of liburing's `io_uring_get_sqe()` exactly.
164 /// However, instead of a null we return an error to force safe handling.
165 /// Any situation where the submission queue is full tends more towards a control flow error,
166 /// and the null return in liburing is more a C idiom than anything else, for lack of a better
167 /// alternative. In Zig, we have first-class error handling... so let's use it.
168 /// Matches the implementation of io_uring_get_sqe() in liburing.
169 pub fn get_sqe(self: *IO_Uring) !*io_uring_sqe {
170 const head = @atomicLoad(u32, self.sq.head, .Acquire);
171 // Remember that these head and tail offsets wrap around every four billion operations.
172 // We must therefore use wrapping addition and subtraction to avoid a runtime crash.
173 const next = self.sq.sqe_tail +% 1;
174 if (next -% head > self.sq.sqes.len) return error.IO_UringSubmissionQueueFull;
175 var sqe = &self.sq.sqes[self.sq.sqe_tail & self.sq.mask.*];
176 self.sq.sqe_tail = next;
177 return sqe;
178 }
179
180 /// Submits the SQEs acquired via get_sqe() to the kernel. You can call this once after you have
181 /// called get_sqe() multiple times to setup multiple I/O requests.
182 /// Returns the number of SQEs submitted.
183 /// Matches the implementation of io_uring_submit() in liburing.
184 pub fn submit(self: *IO_Uring) !u32 {
185 return self.submit_and_wait(0);
186 }
187
188 /// Like submit(), but allows waiting for events as well.
189 /// Returns the number of SQEs submitted.
190 /// Matches the implementation of io_uring_submit_and_wait() in liburing.
191 pub fn submit_and_wait(self: *IO_Uring, wait_nr: u32) !u32 {
192 var submitted = self.flush_sq();
193 var flags: u32 = 0;
194 if (self.sq_ring_needs_enter(submitted, &flags) or wait_nr > 0) {
195 if (wait_nr > 0 or (self.flags & linux.IORING_SETUP_IOPOLL) != 0) {
196 flags |= linux.IORING_ENTER_GETEVENTS;
197 }
198 return try self.enter(submitted, wait_nr, flags);
199 }
200 return submitted;
201 }
202
203 // Tell the kernel we have submitted SQEs and/or want to wait for CQEs.
204 // Returns the number of SQEs submitted.
205 fn enter(self: *IO_Uring, to_submit: u32, min_complete: u32, flags: u32) !u32 {
206 assert(self.fd >= 0);
207 const res = linux.io_uring_enter(self.fd, to_submit, min_complete, flags, null);
208 try check_errno(res);
209 return @truncate(u32, res);
210 }
211
212 // Sync internal state with kernel ring state on the SQ side.
213 // Returns the number of all pending events in the SQ ring, for the shared ring.
214 // This return value includes previously flushed SQEs, as per liburing.
215 // The reasoning for this is to suggest that an io_uring_enter() call is needed rather than not.
216 // Matches the implementation of __io_uring_flush_sq() in liburing.
217 fn flush_sq(self: *IO_Uring) u32 {
218 if (self.sq.sqe_head != self.sq.sqe_tail) {
219 // Fill in SQEs that we have queued up, adding them to the kernel ring.
220 const to_submit = self.sq.sqe_tail -% self.sq.sqe_head;
221 const mask = self.sq.mask.*;
222 var tail = self.sq.tail.*;
223 var i: usize = 0;
224 while (i < to_submit) : (i += 1) {
225 self.sq.array[tail & mask] = self.sq.sqe_head & mask;
226 tail +%= 1;
227 self.sq.sqe_head +%= 1;
228 }
229 // Ensure that the kernel can actually see the SQE updates when it sees the tail update.
230 @atomicStore(u32, self.sq.tail, tail, .Release);
231 }
232 return self.sq_ready();
233 }
234
235 /// Returns true if we are not using an SQ thread (thus nobody submits but us),
236 /// or if IORING_SQ_NEED_WAKEUP is set and the SQ thread must be explicitly awakened.
237 /// For the latter case, we set the SQ thread wakeup flag.
238 /// Matches the implementation of sq_ring_needs_enter() in liburing.
239 fn sq_ring_needs_enter(self: *IO_Uring, submitted: u32, flags: *u32) bool {
240 assert(flags.* == 0);
241 if ((self.flags & linux.IORING_SETUP_SQPOLL) == 0 and submitted > 0) return true;
242 if ((@atomicLoad(u32, self.sq.flags, .Unordered) & linux.IORING_SQ_NEED_WAKEUP) != 0) {
243 flags.* |= linux.IORING_ENTER_SQ_WAKEUP;
244 return true;
245 }
246 return false;
247 }
248
249 /// Returns the number of flushed and unflushed SQEs pending in the submission queue.
250 /// In other words, this is the number of SQEs in the submission queue, i.e. its length.
251 /// These are SQEs that the kernel is yet to consume.
252 /// Matches the implementation of io_uring_sq_ready in liburing.
253 pub fn sq_ready(self: *IO_Uring) u32 {
254 // Always use the shared ring state (i.e. head and not sqe_head) to avoid going out of sync,
255 // see https://github.com/axboe/liburing/issues/92.
256 return self.sq.sqe_tail -% @atomicLoad(u32, self.sq.head, .Acquire);
257 }
258
259 /// Returns the number of CQEs in the completion queue, i.e. its length.
260 /// These are CQEs that the application is yet to consume.
261 /// Matches the implementation of io_uring_cq_ready in liburing.
262 pub fn cq_ready(self: *IO_Uring) u32 {
263 return @atomicLoad(u32, self.cq.tail, .Acquire) -% self.cq.head.*;
264 }
265
266 /// Copies as many CQEs as are ready, and that can fit into the destination `cqes` slice.
267 /// If none are available, enters into the kernel to wait for at most `wait_nr` CQEs.
268 /// Returns the number of CQEs copied, advancing the CQ ring.
269 /// Provides all the wait/peek methods found in liburing, but with batching and a single method.
270 /// The rationale for copying CQEs rather than copying pointers is that pointers are 8 bytes
271 /// whereas CQEs are not much more at only 16 bytes, and this provides a safer faster interface.
272 /// Safer, because you no longer need to call cqe_seen(), avoiding idempotency bugs.
273 /// Faster, because we can now amortize the atomic store release to `cq.head` across the batch.
274 /// See https://github.com/axboe/liburing/issues/103#issuecomment-686665007.
275 /// Matches the implementation of io_uring_peek_batch_cqe() in liburing, but supports waiting.
276 pub fn copy_cqes(self: *IO_Uring, cqes: []io_uring_cqe, wait_nr: u32) !u32 {
277 const count = self.copy_cqes_ready(cqes, wait_nr);
278 if (count > 0) return count;
279 if (self.cq_ring_needs_flush() or wait_nr > 0) {
280 _ = try self.enter(0, wait_nr, linux.IORING_ENTER_GETEVENTS);
281 return self.copy_cqes_ready(cqes, wait_nr);
282 }
283 return 0;
284 }
285
286 fn copy_cqes_ready(self: *IO_Uring, cqes: []io_uring_cqe, wait_nr: u32) u32 {
287 const ready = self.cq_ready();
288 const count = std.math.min(cqes.len, ready);
289 const mask = self.cq.mask.*;
290 var head = self.cq.head.*;
291 var tail = head +% count;
292 // TODO Optimize this by using 1 or 2 memcpy's (if the tail wraps) rather than a loop.
293 var i: usize = 0;
294 // Do not use "less-than" operator since head and tail may wrap:
295 while (head != tail) {
296 cqes[i] = self.cq.cqes[head & mask]; // Copy struct by value.
297 head +%= 1;
298 i += 1;
299 }
300 self.cq_advance(count);
301 return count;
302 }
303
304 /// Returns a copy of an I/O completion, waiting for it if necessary, and advancing the CQ ring.
305 /// A convenience method for `copy_cqes()` for when you don't need to batch or peek.
306 pub fn copy_cqe(ring: *IO_Uring) !io_uring_cqe {
307 var cqes: [1]io_uring_cqe = undefined;
308 const count = try ring.copy_cqes(&cqes, 1);
309 assert(count == 1);
310 return cqes[0];
311 }
312
313 // Matches the implementation of cq_ring_needs_flush() in liburing.
314 fn cq_ring_needs_flush(self: *IO_Uring) bool {
315 return (@atomicLoad(u32, self.sq.flags, .Unordered) & linux.IORING_SQ_CQ_OVERFLOW) != 0;
316 }
317
318 /// For advanced use cases only that implement custom completion queue methods.
319 /// If you use copy_cqes() or copy_cqe() you must not call cqe_seen() or cq_advance().
320 /// Must be called exactly once after a zero-copy CQE has been processed by your application.
321 /// Not idempotent, calling more than once will result in other CQEs being lost.
322 /// Matches the implementation of cqe_seen() in liburing.
323 pub fn cqe_seen(self: *IO_Uring, cqe: *io_uring_cqe) void {
324 self.cq_advance(1);
325 }
326
327 /// For advanced use cases only that implement custom completion queue methods.
328 /// Matches the implementation of cq_advance() in liburing.
329 pub fn cq_advance(self: *IO_Uring, count: u32) void {
330 if (count > 0) {
331 // Ensure the kernel only sees the new head value after the CQEs have been read.
332 @atomicStore(u32, self.cq.head, self.cq.head.* +% count, .Release);
333 }
334 }
335
336 /// Queues (but does not submit) an SQE to perform an `accept4(2)` on a socket.
337 /// Returns a pointer to the SQE.
338 pub fn queue_accept(
339 self: *IO_Uring,
340 user_data: u64,
341 fd: os.fd_t,
342 addr: *os.sockaddr,
343 addrlen: *os.socklen_t,
344 accept_flags: u32
345 ) !*io_uring_sqe {
346 if (!can_accept) return error.IO_UringKernelCannotAccept;
347 // "sqe->fd is the file descriptor, sqe->addr holds a pointer to struct sockaddr,
348 // sqe->addr2 holds a pointer to socklen_t, and finally sqe->accept_flags holds the flags
349 // for accept(4)." - https://lwn.net/ml/linux-block/20191025173037.13486-1-axboe@kernel.dk/
350 const sqe = try self.get_sqe();
351 sqe.* = .{
352 .opcode = .ACCEPT,
353 .fd = fd,
354 .off = @ptrToInt(addrlen), // `addr2` is a newer union member that maps to `off`.
355 .addr = @ptrToInt(addr),
356 .user_data = user_data,
357 .opflags = accept_flags
358 };
359 return sqe;
360 }
361
362 /// Queues (but does not submit) an SQE to perform an `fsync(2)`.
363 /// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
364 /// For example, for `fdatasync()` you can set `IORING_FSYNC_DATASYNC` in the SQE's `opflags`.
365 /// N.B. While SQEs are initiated in the order in which they appear in the submission queue,
366 /// operations execute in parallel and completions are unordered. Therefore, an application that
367 /// submits a write followed by an fsync in the submission queue cannot expect the fsync to
368 /// apply to the write, since the fsync may complete before the write is issued to the disk.
369 /// You should preferably use `link_with_next_sqe()` on a write's SQE to link it with an fsync,
370 /// or else insert a full write barrier using `drain_previous_sqes()` when queueing an fsync.
371 pub fn queue_fsync(self: *IO_Uring, user_data: u64, fd: os.fd_t) !*io_uring_sqe {
372 const sqe = try self.get_sqe();
373 sqe.* = .{
374 .opcode = .FSYNC,
375 .fd = fd,
376 .user_data = user_data
377 };
378 return sqe;
379 }
380
381 /// Queues (but does not submit) an SQE to perform a no-op.
382 /// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
383 /// A no-op is more useful than may appear at first glance.
384 /// For example, you could call `drain_previous_sqes()` on the returned SQE, to use the no-op to
385 /// know when the ring is idle before acting on a kill signal.
386 pub fn queue_nop(self: *IO_Uring, user_data: u64) !*io_uring_sqe {
387 const sqe = try self.get_sqe();
388 sqe.* = .{
389 .opcode = .NOP,
390 .user_data = user_data
391 };
392 return sqe;
393 }
394
395 /// Queues (but does not submit) an SQE to perform a `read(2)`.
396 /// Returns a pointer to the SQE.
397 pub fn queue_read(
398 self: *IO_Uring,
399 user_data: u64,
400 fd: os.fd_t,
401 buffer: []u8,
402 offset: u64
403 ) !*io_uring_sqe {
404 if (!can_read) return error.IO_UringKernelCannotRead;
405 const sqe = try self.get_sqe();
406 sqe.* = .{
407 .opcode = .READ,
408 .fd = fd,
409 .off = offset,
410 .addr = @ptrToInt(buffer.ptr),
411 .len = @truncate(u32, buffer.len),
412 .user_data = user_data
413 };
414 return sqe;
415 }
416
417 /// Queues (but does not submit) an SQE to perform a `write(2)`.
418 /// Returns a pointer to the SQE.
419 pub fn queue_write(
420 self: *IO_Uring,
421 user_data: u64,
422 fd: os.fd_t,
423 buffer: []const u8,
424 offset: u64
425 ) !*io_uring_sqe {
426 if (!can_write) return error.IO_UringKernelCannotWrite;
427 const sqe = try self.get_sqe();
428 sqe.* = .{
429 .opcode = .WRITE,
430 .fd = fd,
431 .off = offset,
432 .addr = @ptrToInt(buffer.ptr),
433 .len = @truncate(u32, buffer.len),
434 .user_data = user_data
435 };
436 return sqe;
437 }
438
439 /// Queues (but does not submit) an SQE to perform a `preadv()`.
440 /// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
441 /// For example, if you want to do a `preadv2()` then set `opflags` on the returned SQE.
442 /// See https://linux.die.net/man/2/preadv.
443 pub fn queue_readv(
444 self: *IO_Uring,
445 user_data: u64,
446 fd: os.fd_t,
447 iovecs: []const os.iovec,
448 offset: u64
449 ) !*io_uring_sqe {
450 const sqe = try self.get_sqe();
451 sqe.* = .{
452 .opcode = .READV,
453 .fd = fd,
454 .off = offset,
455 .addr = @ptrToInt(iovecs.ptr),
456 .len = @truncate(u32, iovecs.len),
457 .user_data = user_data
458 };
459 return sqe;
460 }
461
462 /// Queues (but does not submit) an SQE to perform a `pwritev()`.
463 /// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
464 /// For example, if you want to do a `pwritev2()` then set `opflags` on the returned SQE.
465 /// See https://linux.die.net/man/2/pwritev.
466 pub fn queue_writev(
467 self: *IO_Uring,
468 user_data: u64,
469 fd: os.fd_t,
470 iovecs: []const os.iovec_const,
471 offset: u64
472 ) !*io_uring_sqe {
473 const sqe = try self.get_sqe();
474 sqe.* = .{
475 .opcode = .WRITEV,
476 .fd = fd,
477 .off = offset,
478 .addr = @ptrToInt(iovecs.ptr),
479 .len = @truncate(u32, iovecs.len),
480 .user_data = user_data
481 };
482 return sqe;
483 }
484
485 /// The next SQE will not be started until this one completes.
486 /// This can be used to chain causally dependent SQEs, and the chain can be arbitrarily long.
487 /// The tail of the chain is denoted by the first SQE that does not have this flag set.
488 /// This flag has no effect on previous SQEs, nor does it impact SQEs outside the chain.
489 /// This means that multiple chains can be executing in parallel, along with individual SQEs.
490 /// Only members inside the chain are serialized.
491 /// A chain will be broken if any SQE in the chain ends in error, where any unexpected result is
492 /// considered an error. For example, a short read will terminate the remainder of the chain.
493 pub fn link_with_next_sqe(self: *IO_Uring, sqe: *io_uring_sqe) void {
494 sqe.flags |= linux.IOSQE_IO_LINK;
495 }
496
497 /// Like `link_with_next_sqe()` but stronger.
498 /// For when you don't want the chain to fail in the event of a completion result error.
499 /// For example, you may know that some commands will fail and may want the chain to continue.
500 /// Hard links are resilient to completion results, but are not resilient to submission errors.
501 pub fn hardlink_with_next_sqe(self: *IO_Uring, sqe: *io_uring_sqe) void {
502 sqe.flags |= linux.IOSQE_IO_HARDLINK;
503 }
504
505 /// This creates a full pipeline barrier in the submission queue.
506 /// This SQE will not be started until previous SQEs complete.
507 /// Subsequent SQEs will not be started until this SQE completes.
508 /// In other words, this stalls the entire submission queue.
509 /// You should first consider using link_with_next_sqe() for more granular SQE sequence control.
510 pub fn drain_previous_sqes(self: *IO_Uring, sqe: *io_uring_sqe) void {
511 sqe.flags |= linux.IOSQE_IO_DRAIN;
512 }
513
514 /// Registers an array of file descriptors.
515 /// Every time a file descriptor is put in an SQE and submitted to the kernel, the kernel must
516 /// retrieve a reference to the file, and once I/O has completed the file reference must be
517 /// dropped. The atomic nature of this file reference can be a slowdown for high IOPS workloads.
518 /// This slowdown can be avoided by pre-registering file descriptors.
519 /// To refer to a registered file descriptor, IOSQE_FIXED_FILE must be set in the SQE's flags,
520 /// and the SQE's fd must be set to the index of the file descriptor in the registered array.
521 /// Registering file descriptors will wait for the ring to idle.
522 /// Files are automatically unregistered by the kernel when the ring is torn down.
523 /// An application need unregister only if it wants to register a new array of file descriptors.
524 pub fn register_files(self: *IO_Uring, fds: []const i32) !void {
525 assert(self.fd >= 0);
526 const res = linux.io_uring_register(
527 self.fd,
528 .REGISTER_FILES,
529 @ptrCast(*const c_void, fds.ptr),
530 @truncate(u32, fds.len)
531 );
532 try check_errno(res);
533 }
534
535 /// Changes the semantics of the SQE's `fd` to refer to a pre-registered file descriptor.
536 pub fn use_registered_fd(self: *IO_Uring, sqe: *io_uring_sqe) void {
537 sqe.flags |= linux.IOSQE_FIXED_FILE;
538 }
539
540 /// Unregisters all registered file descriptors previously associated with the ring.
541 pub fn unregister_files(self: *IO_Uring) !void {
542 assert(self.fd >= 0);
543 const res = linux.io_uring_register(self.fd, .UNREGISTER_FILES, null, 0);
544 try check_errno(res);
545 }
546};
547
548
549pub const SubmissionQueue = struct {
550 head: *u32,
551 tail: *u32,
552 mask: *u32,
553 flags: *u32,
554 dropped: *u32,
555 array: []u32,
556 sqes: []io_uring_sqe,
557 mmap: []align(std.mem.page_size) u8,
558 mmap_sqes: []align(std.mem.page_size) u8,
559
560 // We use `sqe_head` and `sqe_tail` in the same way as liburing:
561 // We increment `sqe_tail` (but not `tail`) for each call to `get_sqe()`.
562 // We then set `tail` to `sqe_tail` once, only when these events are actually submitted.
563 // This allows us to amortize the cost of the @atomicStore to `tail` across multiple SQEs.
564 sqe_head: u32 = 0,
565 sqe_tail: u32 = 0,
566
567 pub fn init(fd: i32, p: io_uring_params) !SubmissionQueue {
568 assert(fd >= 0);
569 assert((p.features & linux.IORING_FEAT_SINGLE_MMAP) != 0);
570 const size = std.math.max(
571 p.sq_off.array + p.sq_entries * @sizeOf(u32),
572 p.cq_off.cqes + p.cq_entries * @sizeOf(io_uring_cqe)
573 );
574 const mmap = try os.mmap(
575 null,
576 size,
577 os.PROT_READ | os.PROT_WRITE,
578 os.MAP_SHARED | os.MAP_POPULATE,
579 fd,
580 linux.IORING_OFF_SQ_RING,
581 );
582 errdefer os.munmap(mmap);
583 assert(mmap.len == size);
584
585 // The motivation for the `sqes` and `array` indirection is to make it possible for the
586 // application to preallocate static io_uring_sqe entries and then replay them when needed.
587 const size_sqes = p.sq_entries * @sizeOf(io_uring_sqe);
588 const mmap_sqes = try os.mmap(
589 null,
590 size_sqes,
591 os.PROT_READ | os.PROT_WRITE,
592 os.MAP_SHARED | os.MAP_POPULATE,
593 fd,
594 linux.IORING_OFF_SQES,
595 );
596 errdefer os.munmap(mmap_sqes);
597 assert(mmap_sqes.len == size_sqes);
598
599 const array = @ptrCast([*]u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.array]));
600 const sqes = @ptrCast([*]io_uring_sqe, @alignCast(@alignOf(io_uring_sqe), &mmap_sqes[0]));
601 // We expect the kernel copies p.sq_entries to the u32 pointed to by p.sq_off.ring_entries,
602 // see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L7843-L7844.
603 assert(
604 p.sq_entries ==
605 @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.ring_entries])).*
606 );
607 return SubmissionQueue {
608 .head = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.head])),
609 .tail = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.tail])),
610 .mask = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.ring_mask])),
611 .flags = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.flags])),
612 .dropped = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.dropped])),
613 .array = array[0..p.sq_entries],
614 .sqes = sqes[0..p.sq_entries],
615 .mmap = mmap,
616 .mmap_sqes = mmap_sqes
617 };
618 }
619
620 pub fn deinit(self: *SubmissionQueue) void {
621 os.munmap(self.mmap_sqes);
622 os.munmap(self.mmap);
623 }
624};
625
626pub const CompletionQueue = struct {
627 head: *u32,
628 tail: *u32,
629 mask: *u32,
630 overflow: *u32,
631 cqes: []io_uring_cqe,
632
633 pub fn init(fd: i32, p: io_uring_params, sq: SubmissionQueue) !CompletionQueue {
634 assert(fd >= 0);
635 assert((p.features & linux.IORING_FEAT_SINGLE_MMAP) != 0);
636 const mmap = sq.mmap;
637 const cqes = @ptrCast(
638 [*]io_uring_cqe,
639 @alignCast(@alignOf(io_uring_cqe), &mmap[p.cq_off.cqes])
640 );
641 assert(
642 p.cq_entries ==
643 @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.ring_entries])).*
644 );
645 return CompletionQueue {
646 .head = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.head])),
647 .tail = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.tail])),
648 .mask = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.ring_mask])),
649 .overflow = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.overflow])),
650 .cqes = cqes[0..p.cq_entries]
651 };
652 }
653
654 pub fn deinit(self: *CompletionQueue) void {
655 // A no-op since we now share the mmap with the submission queue.
656 // Here for symmetry with the submission queue, and for any future feature support.
657 }
658};
659
660inline fn check_errno(res: usize) !void {
661 const errno = linux.getErrno(res);
662 if (errno != 0) return os.unexpectedErrno(errno);
663}
664
665const minimum = std.Target.current.os.isAtLeast;
666pub const can_single_mmap = comptime minimum(.linux, .{ .major = 5, .minor = 4 }) == true;
667pub const can_accept = comptime minimum(.linux, .{ .major = 5, .minor = 5 }) == true;
668pub const can_read = comptime minimum(.linux, .{ .major = 5, .minor = 6 }) == true;
669pub const can_write = comptime minimum(.linux, .{ .major = 5, .minor = 6 }) == true;
670pub const supported = can_single_mmap;
671
672test "queue_nop" {
673 if (!supported) return error.SkipZigTest;
674
675 var ring = try IO_Uring.init(1, 0);
676 defer {
677 ring.deinit();
678 testing.expectEqual(@as(i32, -1), ring.fd);
679 }
680
681 var sqe = try ring.queue_nop(@intCast(u64, 0xaaaaaaaa));
682 testing.expectEqual(io_uring_sqe {
683 .opcode = .NOP,
684 .flags = 0,
685 .ioprio = 0,
686 .fd = 0,
687 .off = 0,
688 .addr = 0,
689 .len = 0,
690 .opflags = 0,
691 .user_data = @intCast(u64, 0xaaaaaaaa),
692 .buffer = 0,
693 .personality = 0,
694 .splice_fd_in = 0,
695 .options = [2]u64{ 0, 0 }
696 }, sqe.*);
697
698 testing.expectEqual(@as(u32, 0), ring.sq.sqe_head);
699 testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
700 testing.expectEqual(@as(u32, 0), ring.sq.tail.*);
701 testing.expectEqual(@as(u32, 0), ring.cq.head.*);
702 testing.expectEqual(@as(u32, 1), ring.sq_ready());
703 testing.expectEqual(@as(u32, 0), ring.cq_ready());
704
705 testing.expectEqual(@as(u32, 1), try ring.submit());
706 testing.expectEqual(@as(u32, 1), ring.sq.sqe_head);
707 testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
708 testing.expectEqual(@as(u32, 1), ring.sq.tail.*);
709 testing.expectEqual(@as(u32, 0), ring.cq.head.*);
710 testing.expectEqual(@as(u32, 0), ring.sq_ready());
711
712 testing.expectEqual(io_uring_cqe {
713 .user_data = 0xaaaaaaaa,
714 .res = 0,
715 .flags = 0
716 }, try ring.copy_cqe());
717 testing.expectEqual(@as(u32, 1), ring.cq.head.*);
718 testing.expectEqual(@as(u32, 0), ring.cq_ready());
719
720 var sqe_barrier = try ring.queue_nop(@intCast(u64, 0xbbbbbbbb));
721 ring.drain_previous_sqes(sqe_barrier);
722 testing.expectEqual(@as(u8, linux.IOSQE_IO_DRAIN), sqe_barrier.flags);
723 testing.expectEqual(@as(u32, 1), try ring.submit());
724 testing.expectEqual(io_uring_cqe {
725 .user_data = 0xbbbbbbbb,
726 .res = 0,
727 .flags = 0
728 }, try ring.copy_cqe());
729 testing.expectEqual(@as(u32, 2), ring.sq.sqe_head);
730 testing.expectEqual(@as(u32, 2), ring.sq.sqe_tail);
731 testing.expectEqual(@as(u32, 2), ring.sq.tail.*);
732 testing.expectEqual(@as(u32, 2), ring.cq.head.*);
733}
734
735test "queue_readv" {
736 if (!supported) return error.SkipZigTest;
737
738 var ring = try IO_Uring.init(1, 0);
739 defer ring.deinit();
740
741 const fd = try os.openZ("/dev/zero", os.O_RDONLY | os.O_CLOEXEC, 0);
742 defer os.close(fd);
743
744 // Linux Kernel 5.4 supports IORING_REGISTER_FILES but not sparse fd sets (i.e. an fd of -1).
745 // Linux Kernel 5.5 adds support for sparse fd sets.
746 // Compare:
747 // https://github.com/torvalds/linux/blob/v5.4/fs/io_uring.c#L3119-L3124 vs
748 // https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L6687-L6691
749 // We therefore avoid stressing sparse fd sets here:
750 var registered_fds = [_]i32{0} ** 1;
751 const fd_index = 0;
752 registered_fds[fd_index] = fd;
753 try ring.register_files(registered_fds[0..]);
754
755 var buffer = [_]u8{42} ** 128;
756 var iovecs = [_]os.iovec{ os.iovec { .iov_base = &buffer, .iov_len = buffer.len } };
757 var sqe = try ring.queue_readv(0xcccccccc, fd_index, iovecs[0..], 0);
758 ring.use_registered_fd(sqe);
759 testing.expectEqual(@as(u8, linux.IOSQE_FIXED_FILE), sqe.flags);
760
761 testing.expectError(error.IO_UringSubmissionQueueFull, ring.queue_nop(0));
762 testing.expectEqual(@as(u32, 1), try ring.submit());
763 testing.expectEqual(linux.io_uring_cqe {
764 .user_data = 0xcccccccc,
765 .res = buffer.len,
766 .flags = 0,
767 }, try ring.copy_cqe());
768 testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
769
770 try ring.unregister_files();
771}
772
773test "queue_writev/queue_fsync" {
774 if (!supported) return error.SkipZigTest;
775
776 var ring = try IO_Uring.init(2, 0);
777 defer ring.deinit();
778
779 const path = "test_io_uring_queue_writev";
780 const file = try std.fs.cwd().createFile(path, .{ .truncate = true });
781 defer file.close();
782 defer std.fs.cwd().deleteFile(path) catch {};
783 const fd = file.handle;
784
785 var buffer = [_]u8{42} ** 128;
786 var iovecs = [_]os.iovec_const {
787 os.iovec_const { .iov_base = &buffer, .iov_len = buffer.len }
788 };
789 var sqe_writev = try ring.queue_writev(0xdddddddd, fd, iovecs[0..], 0);
790 ring.link_with_next_sqe(sqe_writev);
791 testing.expectEqual(@as(u8, linux.IOSQE_IO_LINK), sqe_writev.flags);
792
793 var sqe_fsync = try ring.queue_fsync(0xeeeeeeee, fd);
794 testing.expectEqual(fd, sqe_fsync.fd);
795
796 testing.expectEqual(@as(u32, 2), ring.sq_ready());
797 testing.expectEqual(@as(u32, 2), try ring.submit_and_wait(2));
798 testing.expectEqual(@as(u32, 0), ring.sq_ready());
799 testing.expectEqual(@as(u32, 2), ring.cq_ready());
800 testing.expectEqual(linux.io_uring_cqe {
801 .user_data = 0xdddddddd,
802 .res = buffer.len,
803 .flags = 0,
804 }, try ring.copy_cqe());
805 testing.expectEqual(@as(u32, 1), ring.cq_ready());
806 testing.expectEqual(linux.io_uring_cqe {
807 .user_data = 0xeeeeeeee,
808 .res = 0,
809 .flags = 0,
810 }, try ring.copy_cqe());
811 testing.expectEqual(@as(u32, 0), ring.cq_ready());
812}
813
814test "queue_write/queue_read" {
815 if (!can_read or !can_write) return error.SkipZigTest;
816
817 var ring = try IO_Uring.init(2, 0);
818 defer ring.deinit();
819
820 const path = "test_io_uring_queue_write";
821 const file = try std.fs.cwd().createFile(path, .{ .read = true, .truncate = true });
822 defer file.close();
823 defer std.fs.cwd().deleteFile(path) catch {};
824 const fd = file.handle;
825
826 var buffer_write = [_]u8{97} ** 20;
827 var buffer_read = [_]u8{98} ** 20;
828 var sqe_write = try ring.queue_write(123, fd, buffer_write[0..], 10);
829 ring.link_with_next_sqe(sqe_write);
830 var sqe_read = try ring.queue_read(456, fd, buffer_read[0..], 10);
831 testing.expectEqual(@as(u32, 2), try ring.submit());
832 testing.expectEqual(linux.io_uring_cqe {
833 .user_data = 123,
834 .res = buffer_write.len,
835 .flags = 0,
836 }, try ring.copy_cqe());
837 testing.expectEqual(linux.io_uring_cqe {
838 .user_data = 456,
839 .res = buffer_read.len,
840 .flags = 0,
841 }, try ring.copy_cqe());
842 testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
843}
lib/std/os/linux.zig+1
...@@ -31,6 +31,7 @@ pub usingnamespace switch (builtin.arch) {...@@ -31,6 +31,7 @@ pub usingnamespace switch (builtin.arch) {
31pub usingnamespace @import("bits.zig");31pub usingnamespace @import("bits.zig");
32pub const tls = @import("linux/tls.zig");32pub const tls = @import("linux/tls.zig");
33pub const BPF = @import("linux/bpf.zig");33pub const BPF = @import("linux/bpf.zig");
34pub usingnamespace @import("linux/io_uring.zig");
3435
35/// Set by startup code, used by `getauxval`.36/// Set by startup code, used by `getauxval`.
36pub var elf_aux_maybe: ?[*]std.elf.Auxv = null;37pub var elf_aux_maybe: ?[*]std.elf.Auxv = null;
lib/std/os/linux/io_uring.zig created+801
...@@ -0,0 +1,801 @@
1// SPDX-License-Identifier: MIT
2// Copyright (c) 2015-2020 Zig Contributors
3// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
4// The MIT license requires this copyright notice to be included in all copies
5// and substantial portions of the software.
6const std = @import("../../std.zig");
7const assert = std.debug.assert;
8const builtin = std.builtin;
9const os = std.os;
10const linux = os.linux;
11const mem = std.mem;
12const net = std.net;
13const testing = std.testing;
14
15const io_uring_params = linux.io_uring_params;
16const io_uring_sqe = linux.io_uring_sqe;
17const io_uring_cqe = linux.io_uring_cqe;
18
19pub const IO_Uring = struct {
20 fd: i32 = -1,
21 sq: SubmissionQueue,
22 cq: CompletionQueue,
23 flags: u32,
24
25 /// A friendly way to setup an io_uring, with default io_uring_params.
26 /// `entries` must be a power of two between 1 and 4096, although the kernel will make the final
27 /// call on how many entries the submission and completion queues will ultimately have,
28 /// see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L8027-L8050.
29 /// Matches the interface of io_uring_queue_init() in liburing.
30 pub fn init(entries: u32, flags: u32) !IO_Uring {
31 var params = io_uring_params {
32 .sq_entries = 0,
33 .cq_entries = 0,
34 .flags = flags,
35 .sq_thread_cpu = 0,
36 .sq_thread_idle = 1000,
37 .features = 0,
38 .wq_fd = 0,
39 .resv = [_]u32{0} ** 3,
40 .sq_off = undefined,
41 .cq_off = undefined,
42 };
43 // The kernel will zero the memory of the sq_off and cq_off structs in io_uring_create(),
44 // see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L7986-L8002.
45 return try IO_Uring.init_params(entries, &params);
46 }
47
48 /// A powerful way to setup an io_uring, if you want to tweak io_uring_params such as submission
49 /// queue thread cpu affinity or thread idle timeout (the kernel and our default is 1 second).
50 /// `params` is passed by reference because the kernel needs to modify the parameters.
51 /// You may only set the `flags`, `sq_thread_cpu` and `sq_thread_idle` parameters.
52 /// Every other parameter belongs to the kernel and must be zeroed.
53 /// Matches the interface of io_uring_queue_init_params() in liburing.
54 pub fn init_params(entries: u32, p: *io_uring_params) !IO_Uring {
55 assert(entries >= 1 and entries <= 4096 and std.math.isPowerOfTwo(entries));
56 assert(p.sq_entries == 0);
57 assert(p.cq_entries == 0);
58 assert(p.features == 0);
59 assert(p.wq_fd == 0);
60 assert(p.resv[0] == 0);
61 assert(p.resv[1] == 0);
62 assert(p.resv[2] == 0);
63
64 const res = linux.io_uring_setup(entries, p);
65 try check_errno(res);
66 const fd = @intCast(i32, res);
67 assert(fd >= 0);
68 errdefer os.close(fd);
69
70 // Kernel versions 5.4 and up use only one mmap() for the submission and completion queues.
71 // This is not an optional feature for us... if the kernel does it, we have to do it.
72 // The thinking on this by the kernel developers was that both the submission and the
73 // completion queue rings have sizes just over a power of two, but the submission queue ring
74 // is significantly smaller with u32 slots. By bundling both in a single mmap, the kernel
75 // gets the submission queue ring for free.
76 // See https://patchwork.kernel.org/patch/11115257 for the kernel patch.
77 // We do not support the double mmap() done before 5.4, because we want to keep the
78 // init/deinit mmap paths simple and because io_uring has had many bug fixes even since 5.4.
79 if ((p.features & linux.IORING_FEAT_SINGLE_MMAP) == 0) {
80 return error.UnsupportedKernel;
81 }
82
83 // Check that the kernel has actually set params and that "impossible is nothing".
84 assert(p.sq_entries != 0);
85 assert(p.cq_entries != 0);
86 assert(p.cq_entries >= p.sq_entries);
87
88 // From here on, we only need to read from params, so pass `p` by value as immutable.
89 // The completion queue shares the mmap with the submission queue, so pass `sq` there too.
90 var sq = try SubmissionQueue.init(fd, p.*);
91 errdefer sq.deinit();
92 var cq = try CompletionQueue.init(fd, p.*, sq);
93 errdefer cq.deinit();
94
95 // Check that our starting state is as we expect.
96 assert(sq.head.* == 0);
97 assert(sq.tail.* == 0);
98 assert(sq.mask.* == p.sq_entries - 1);
99 // Allow flags.* to be non-zero, since the kernel may set IORING_SQ_NEED_WAKEUP at any time.
100 assert(sq.dropped.* == 0);
101 assert(sq.array.len == p.sq_entries);
102 assert(sq.sqes.len == p.sq_entries);
103 assert(sq.sqe_head == 0);
104 assert(sq.sqe_tail == 0);
105
106 assert(cq.head.* == 0);
107 assert(cq.tail.* == 0);
108 assert(cq.mask.* == p.cq_entries - 1);
109 assert(cq.overflow.* == 0);
110 assert(cq.cqes.len == p.cq_entries);
111
112 return IO_Uring {
113 .fd = fd,
114 .sq = sq,
115 .cq = cq,
116 .flags = p.flags
117 };
118 }
119
120 pub fn deinit(self: *IO_Uring) void {
121 assert(self.fd >= 0);
122 // The mmaps depend on the fd, so the order of these calls is important:
123 self.cq.deinit();
124 self.sq.deinit();
125 os.close(self.fd);
126 self.fd = -1;
127 }
128
129 /// Returns a vacant SQE, or an error if the submission queue is full.
130 /// We follow the implementation (and atomics) of liburing's `io_uring_get_sqe()` exactly.
131 /// However, instead of a null we return an error to force safe handling.
132 /// Any situation where the submission queue is full tends more towards a control flow error,
133 /// and the null return in liburing is more a C idiom than anything else, for lack of a better
134 /// alternative. In Zig, we have first-class error handling... so let's use it.
135 /// Matches the implementation of io_uring_get_sqe() in liburing.
136 pub fn get_sqe(self: *IO_Uring) !*io_uring_sqe {
137 const head = @atomicLoad(u32, self.sq.head, .Acquire);
138 // Remember that these head and tail offsets wrap around every four billion operations.
139 // We must therefore use wrapping addition and subtraction to avoid a runtime crash.
140 const next = self.sq.sqe_tail +% 1;
141 if (next -% head > self.sq.sqes.len) return error.IO_UringSubmissionQueueFull;
142 var sqe = &self.sq.sqes[self.sq.sqe_tail & self.sq.mask.*];
143 self.sq.sqe_tail = next;
144 return sqe;
145 }
146
147 /// Submits the SQEs acquired via get_sqe() to the kernel. You can call this once after you have
148 /// called get_sqe() multiple times to setup multiple I/O requests.
149 /// Returns the number of SQEs submitted.
150 /// Matches the implementation of io_uring_submit() in liburing.
151 pub fn submit(self: *IO_Uring) !u32 {
152 return self.submit_and_wait(0);
153 }
154
155 /// Like submit(), but allows waiting for events as well.
156 /// Returns the number of SQEs submitted.
157 /// Matches the implementation of io_uring_submit_and_wait() in liburing.
158 pub fn submit_and_wait(self: *IO_Uring, wait_nr: u32) !u32 {
159 var submitted = self.flush_sq();
160 var flags: u32 = 0;
161 if (self.sq_ring_needs_enter(submitted, &flags) or wait_nr > 0) {
162 if (wait_nr > 0 or (self.flags & linux.IORING_SETUP_IOPOLL) != 0) {
163 flags |= linux.IORING_ENTER_GETEVENTS;
164 }
165 return try self.enter(submitted, wait_nr, flags);
166 }
167 return submitted;
168 }
169
170 // Tell the kernel we have submitted SQEs and/or want to wait for CQEs.
171 // Returns the number of SQEs submitted.
172 fn enter(self: *IO_Uring, to_submit: u32, min_complete: u32, flags: u32) !u32 {
173 assert(self.fd >= 0);
174 const res = linux.io_uring_enter(self.fd, to_submit, min_complete, flags, null);
175 try check_errno(res);
176 return @truncate(u32, res);
177 }
178
179 // Sync internal state with kernel ring state on the SQ side.
180 // Returns the number of all pending events in the SQ ring, for the shared ring.
181 // This return value includes previously flushed SQEs, as per liburing.
182 // The reasoning for this is to suggest that an io_uring_enter() call is needed rather than not.
183 // Matches the implementation of __io_uring_flush_sq() in liburing.
184 fn flush_sq(self: *IO_Uring) u32 {
185 if (self.sq.sqe_head != self.sq.sqe_tail) {
186 // Fill in SQEs that we have queued up, adding them to the kernel ring.
187 const to_submit = self.sq.sqe_tail -% self.sq.sqe_head;
188 const mask = self.sq.mask.*;
189 var tail = self.sq.tail.*;
190 var i: usize = 0;
191 while (i < to_submit) : (i += 1) {
192 self.sq.array[tail & mask] = self.sq.sqe_head & mask;
193 tail +%= 1;
194 self.sq.sqe_head +%= 1;
195 }
196 // Ensure that the kernel can actually see the SQE updates when it sees the tail update.
197 @atomicStore(u32, self.sq.tail, tail, .Release);
198 }
199 return self.sq_ready();
200 }
201
202 /// Returns true if we are not using an SQ thread (thus nobody submits but us),
203 /// or if IORING_SQ_NEED_WAKEUP is set and the SQ thread must be explicitly awakened.
204 /// For the latter case, we set the SQ thread wakeup flag.
205 /// Matches the implementation of sq_ring_needs_enter() in liburing.
206 fn sq_ring_needs_enter(self: *IO_Uring, submitted: u32, flags: *u32) bool {
207 assert(flags.* == 0);
208 if ((self.flags & linux.IORING_SETUP_SQPOLL) == 0 and submitted > 0) return true;
209 if ((@atomicLoad(u32, self.sq.flags, .Unordered) & linux.IORING_SQ_NEED_WAKEUP) != 0) {
210 flags.* |= linux.IORING_ENTER_SQ_WAKEUP;
211 return true;
212 }
213 return false;
214 }
215
216 /// Returns the number of flushed and unflushed SQEs pending in the submission queue.
217 /// In other words, this is the number of SQEs in the submission queue, i.e. its length.
218 /// These are SQEs that the kernel is yet to consume.
219 /// Matches the implementation of io_uring_sq_ready in liburing.
220 pub fn sq_ready(self: *IO_Uring) u32 {
221 // Always use the shared ring state (i.e. head and not sqe_head) to avoid going out of sync,
222 // see https://github.com/axboe/liburing/issues/92.
223 return self.sq.sqe_tail -% @atomicLoad(u32, self.sq.head, .Acquire);
224 }
225
226 /// Returns the number of CQEs in the completion queue, i.e. its length.
227 /// These are CQEs that the application is yet to consume.
228 /// Matches the implementation of io_uring_cq_ready in liburing.
229 pub fn cq_ready(self: *IO_Uring) u32 {
230 return @atomicLoad(u32, self.cq.tail, .Acquire) -% self.cq.head.*;
231 }
232
233 /// Copies as many CQEs as are ready, and that can fit into the destination `cqes` slice.
234 /// If none are available, enters into the kernel to wait for at most `wait_nr` CQEs.
235 /// Returns the number of CQEs copied, advancing the CQ ring.
236 /// Provides all the wait/peek methods found in liburing, but with batching and a single method.
237 /// The rationale for copying CQEs rather than copying pointers is that pointers are 8 bytes
238 /// whereas CQEs are not much more at only 16 bytes, and this provides a safer faster interface.
239 /// Safer, because you no longer need to call cqe_seen(), avoiding idempotency bugs.
240 /// Faster, because we can now amortize the atomic store release to `cq.head` across the batch.
241 /// See https://github.com/axboe/liburing/issues/103#issuecomment-686665007.
242 /// Matches the implementation of io_uring_peek_batch_cqe() in liburing, but supports waiting.
243 pub fn copy_cqes(self: *IO_Uring, cqes: []io_uring_cqe, wait_nr: u32) !u32 {
244 const count = self.copy_cqes_ready(cqes, wait_nr);
245 if (count > 0) return count;
246 if (self.cq_ring_needs_flush() or wait_nr > 0) {
247 _ = try self.enter(0, wait_nr, linux.IORING_ENTER_GETEVENTS);
248 return self.copy_cqes_ready(cqes, wait_nr);
249 }
250 return 0;
251 }
252
253 fn copy_cqes_ready(self: *IO_Uring, cqes: []io_uring_cqe, wait_nr: u32) u32 {
254 const ready = self.cq_ready();
255 const count = std.math.min(cqes.len, ready);
256 const mask = self.cq.mask.*;
257 var head = self.cq.head.*;
258 var tail = head +% count;
259 // TODO Optimize this by using 1 or 2 memcpy's (if the tail wraps) rather than a loop.
260 var i: usize = 0;
261 // Do not use "less-than" operator since head and tail may wrap:
262 while (head != tail) {
263 cqes[i] = self.cq.cqes[head & mask]; // Copy struct by value.
264 head +%= 1;
265 i += 1;
266 }
267 self.cq_advance(count);
268 return count;
269 }
270
271 /// Returns a copy of an I/O completion, waiting for it if necessary, and advancing the CQ ring.
272 /// A convenience method for `copy_cqes()` for when you don't need to batch or peek.
273 pub fn copy_cqe(ring: *IO_Uring) !io_uring_cqe {
274 var cqes: [1]io_uring_cqe = undefined;
275 const count = try ring.copy_cqes(&cqes, 1);
276 assert(count == 1);
277 return cqes[0];
278 }
279
280 // Matches the implementation of cq_ring_needs_flush() in liburing.
281 fn cq_ring_needs_flush(self: *IO_Uring) bool {
282 return (@atomicLoad(u32, self.sq.flags, .Unordered) & IORING_SQ_CQ_OVERFLOW) != 0;
283 }
284
285 /// For advanced use cases only that implement custom completion queue methods.
286 /// If you use copy_cqes() or copy_cqe() you must not call cqe_seen() or cq_advance().
287 /// Must be called exactly once after a zero-copy CQE has been processed by your application.
288 /// Not idempotent, calling more than once will result in other CQEs being lost.
289 /// Matches the implementation of cqe_seen() in liburing.
290 pub fn cqe_seen(self: *IO_Uring, cqe: *io_uring_cqe) void {
291 self.cq_advance(1);
292 }
293
294 /// For advanced use cases only that implement custom completion queue methods.
295 /// Matches the implementation of cq_advance() in liburing.
296 pub fn cq_advance(self: *IO_Uring, count: u32) void {
297 if (count > 0) {
298 // Ensure the kernel only sees the new head value after the CQEs have been read.
299 @atomicStore(u32, self.cq.head, self.cq.head.* +% count, .Release);
300 }
301 }
302
303 /// Queues (but does not submit) an SQE to perform an `accept4(2)` on a socket.
304 /// Returns a pointer to the SQE.
305 pub fn queue_accept(
306 self: *IO_Uring,
307 user_data: u64,
308 fd: os.fd_t,
309 addr: *os.sockaddr,
310 addrlen: *os.socklen_t,
311 accept_flags: u32
312 ) !*io_uring_sqe {
313 // "sqe->fd is the file descriptor, sqe->addr holds a pointer to struct sockaddr,
314 // sqe->addr2 holds a pointer to socklen_t, and finally sqe->accept_flags holds the flags
315 // for accept(4)." - https://lwn.net/ml/linux-block/20191025173037.13486-1-axboe@kernel.dk/
316 const sqe = try self.get_sqe();
317 sqe.* = .{
318 .opcode = .ACCEPT,
319 .fd = fd,
320 .off = @ptrToInt(addrlen), // `addr2` is a newer union member that maps to `off`.
321 .addr = @ptrToInt(addr),
322 .user_data = user_data,
323 .opflags = accept_flags
324 };
325 return sqe;
326 }
327
328 /// Queues (but does not submit) an SQE to perform an `fsync(2)`.
329 /// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
330 /// For example, for `fdatasync()` you can set `IORING_FSYNC_DATASYNC` in the SQE's `opflags`.
331 /// N.B. While SQEs are initiated in the order in which they appear in the submission queue,
332 /// operations execute in parallel and completions are unordered. Therefore, an application that
333 /// submits a write followed by an fsync in the submission queue cannot expect the fsync to
334 /// apply to the write, since the fsync may complete before the write is issued to the disk.
335 /// You should preferably use `link_with_next_sqe()` on a write's SQE to link it with an fsync,
336 /// or else insert a full write barrier using `drain_previous_sqes()` when queueing an fsync.
337 pub fn queue_fsync(self: *IO_Uring, user_data: u64, fd: os.fd_t) !*io_uring_sqe {
338 const sqe = try self.get_sqe();
339 sqe.* = .{
340 .opcode = .FSYNC,
341 .fd = fd,
342 .user_data = user_data
343 };
344 return sqe;
345 }
346
347 /// Queues (but does not submit) an SQE to perform a no-op.
348 /// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
349 /// A no-op is more useful than may appear at first glance.
350 /// For example, you could call `drain_previous_sqes()` on the returned SQE, to use the no-op to
351 /// know when the ring is idle before acting on a kill signal.
352 pub fn queue_nop(self: *IO_Uring, user_data: u64) !*io_uring_sqe {
353 const sqe = try self.get_sqe();
354 sqe.* = .{
355 .opcode = .NOP,
356 .user_data = user_data
357 };
358 return sqe;
359 }
360
361 /// Queues (but does not submit) an SQE to perform a `read(2)`.
362 /// Returns a pointer to the SQE.
363 pub fn queue_read(
364 self: *IO_Uring,
365 user_data: u64,
366 fd: os.fd_t,
367 buffer: []u8,
368 offset: u64
369 ) !*io_uring_sqe {
370 const sqe = try self.get_sqe();
371 sqe.* = .{
372 .opcode = .READ,
373 .fd = fd,
374 .off = offset,
375 .addr = @ptrToInt(buffer.ptr),
376 .len = @truncate(u32, buffer.len),
377 .user_data = user_data
378 };
379 return sqe;
380 }
381
382 /// Queues (but does not submit) an SQE to perform a `write(2)`.
383 /// Returns a pointer to the SQE.
384 pub fn queue_write(
385 self: *IO_Uring,
386 user_data: u64,
387 fd: os.fd_t,
388 buffer: []const u8,
389 offset: u64
390 ) !*io_uring_sqe {
391 const sqe = try self.get_sqe();
392 sqe.* = .{
393 .opcode = .WRITE,
394 .fd = fd,
395 .off = offset,
396 .addr = @ptrToInt(buffer.ptr),
397 .len = @truncate(u32, buffer.len),
398 .user_data = user_data
399 };
400 return sqe;
401 }
402
403 /// Queues (but does not submit) an SQE to perform a `preadv()`.
404 /// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
405 /// For example, if you want to do a `preadv2()` then set `opflags` on the returned SQE.
406 /// See https://linux.die.net/man/2/preadv.
407 pub fn queue_readv(
408 self: *IO_Uring,
409 user_data: u64,
410 fd: os.fd_t,
411 iovecs: []const os.iovec,
412 offset: u64
413 ) !*io_uring_sqe {
414 const sqe = try self.get_sqe();
415 sqe.* = .{
416 .opcode = .READV,
417 .fd = fd,
418 .off = offset,
419 .addr = @ptrToInt(iovecs.ptr),
420 .len = @truncate(u32, iovecs.len),
421 .user_data = user_data
422 };
423 return sqe;
424 }
425
426 /// Queues (but does not submit) an SQE to perform a `pwritev()`.
427 /// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
428 /// For example, if you want to do a `pwritev2()` then set `opflags` on the returned SQE.
429 /// See https://linux.die.net/man/2/pwritev.
430 pub fn queue_writev(
431 self: *IO_Uring,
432 user_data: u64,
433 fd: os.fd_t,
434 iovecs: []const os.iovec_const,
435 offset: u64
436 ) !*io_uring_sqe {
437 const sqe = try self.get_sqe();
438 sqe.* = .{
439 .opcode = .WRITEV,
440 .fd = fd,
441 .off = offset,
442 .addr = @ptrToInt(iovecs.ptr),
443 .len = @truncate(u32, iovecs.len),
444 .user_data = user_data
445 };
446 return sqe;
447 }
448
449 /// The next SQE will not be started until this one completes.
450 /// This can be used to chain causally dependent SQEs, and the chain can be arbitrarily long.
451 /// The tail of the chain is denoted by the first SQE that does not have this flag set.
452 /// This flag has no effect on previous SQEs, nor does it impact SQEs outside the chain.
453 /// This means that multiple chains can be executing in parallel, along with individual SQEs.
454 /// Only members inside the chain are serialized.
455 /// A chain will be broken if any SQE in the chain ends in error, where any unexpected result is
456 /// considered an error. For example, a short read will terminate the remainder of the chain.
457 pub fn link_with_next_sqe(self: *IO_Uring, sqe: *io_uring_sqe) void {
458 sqe.flags |= linux.IOSQE_IO_LINK;
459 }
460
461 /// Like `link_with_next_sqe()` but stronger.
462 /// For when you don't want the chain to fail in the event of a completion result error.
463 /// For example, you may know that some commands will fail and may want the chain to continue.
464 /// Hard links are resilient to completion results, but are not resilient to submission errors.
465 pub fn hardlink_with_next_sqe(self: *IO_Uring, sqe: *io_uring_sqe) void {
466 sqe.flags |= linux.IOSQE_IO_HARDLINK;
467 }
468
469 /// This creates a full pipeline barrier in the submission queue.
470 /// This SQE will not be started until previous SQEs complete.
471 /// Subsequent SQEs will not be started until this SQE completes.
472 /// In other words, this stalls the entire submission queue.
473 /// You should first consider using link_with_next_sqe() for more granular SQE sequence control.
474 pub fn drain_previous_sqes(self: *IO_Uring, sqe: *io_uring_sqe) void {
475 sqe.flags |= linux.IOSQE_IO_DRAIN;
476 }
477
478 /// Registers an array of file descriptors.
479 /// Every time a file descriptor is put in an SQE and submitted to the kernel, the kernel must
480 /// retrieve a reference to the file, and once I/O has completed the file reference must be
481 /// dropped. The atomic nature of this file reference can be a slowdown for high IOPS workloads.
482 /// This slowdown can be avoided by pre-registering file descriptors.
483 /// To refer to a registered file descriptor, IOSQE_FIXED_FILE must be set in the SQE's flags,
484 /// and the SQE's fd must be set to the index of the file descriptor in the registered array.
485 /// Registering file descriptors will wait for the ring to idle.
486 /// Files are automatically unregistered by the kernel when the ring is torn down.
487 /// An application need unregister only if it wants to register a new array of file descriptors.
488 pub fn register_files(self: *IO_Uring, fds: []const i32) !void {
489 assert(self.fd >= 0);
490 const res = linux.io_uring_register(
491 self.fd,
492 .REGISTER_FILES,
493 @ptrCast(*const c_void, fds.ptr),
494 @truncate(u32, fds.len)
495 );
496 try check_errno(res);
497 }
498
499 /// Changes the semantics of the SQE's `fd` to refer to a pre-registered file descriptor.
500 pub fn use_registered_fd(self: *IO_Uring, sqe: *io_uring_sqe) void {
501 sqe.flags |= linux.IOSQE_FIXED_FILE;
502 }
503
504 /// Unregisters all registered file descriptors previously associated with the ring.
505 pub fn unregister_files(self: *IO_Uring) !void {
506 assert(self.fd >= 0);
507 const res = linux.io_uring_register(self.fd, .UNREGISTER_FILES, null, 0);
508 try check_errno(res);
509 }
510};
511
512
513pub const SubmissionQueue = struct {
514 head: *u32,
515 tail: *u32,
516 mask: *u32,
517 flags: *u32,
518 dropped: *u32,
519 array: []u32,
520 sqes: []io_uring_sqe,
521 mmap: []align(std.mem.page_size) u8,
522 mmap_sqes: []align(std.mem.page_size) u8,
523
524 // We use `sqe_head` and `sqe_tail` in the same way as liburing:
525 // We increment `sqe_tail` (but not `tail`) for each call to `get_sqe()`.
526 // We then set `tail` to `sqe_tail` once, only when these events are actually submitted.
527 // This allows us to amortize the cost of the @atomicStore to `tail` across multiple SQEs.
528 sqe_head: u32 = 0,
529 sqe_tail: u32 = 0,
530
531 pub fn init(fd: i32, p: io_uring_params) !SubmissionQueue {
532 assert(fd >= 0);
533 assert((p.features & linux.IORING_FEAT_SINGLE_MMAP) != 0);
534 const size = std.math.max(
535 p.sq_off.array + p.sq_entries * @sizeOf(u32),
536 p.cq_off.cqes + p.cq_entries * @sizeOf(io_uring_cqe)
537 );
538 const mmap = try os.mmap(
539 null,
540 size,
541 os.PROT_READ | os.PROT_WRITE,
542 os.MAP_SHARED | os.MAP_POPULATE,
543 fd,
544 linux.IORING_OFF_SQ_RING,
545 );
546 errdefer os.munmap(mmap);
547 assert(mmap.len == size);
548
549 // The motivation for the `sqes` and `array` indirection is to make it possible for the
550 // application to preallocate static io_uring_sqe entries and then replay them when needed.
551 const size_sqes = p.sq_entries * @sizeOf(io_uring_sqe);
552 const mmap_sqes = try os.mmap(
553 null,
554 size_sqes,
555 os.PROT_READ | os.PROT_WRITE,
556 os.MAP_SHARED | os.MAP_POPULATE,
557 fd,
558 linux.IORING_OFF_SQES,
559 );
560 errdefer os.munmap(mmap_sqes);
561 assert(mmap_sqes.len == size_sqes);
562
563 const array = @ptrCast([*]u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.array]));
564 const sqes = @ptrCast([*]io_uring_sqe, @alignCast(@alignOf(io_uring_sqe), &mmap_sqes[0]));
565 // We expect the kernel copies p.sq_entries to the u32 pointed to by p.sq_off.ring_entries,
566 // see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L7843-L7844.
567 assert(
568 p.sq_entries ==
569 @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.ring_entries])).*
570 );
571 return SubmissionQueue {
572 .head = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.head])),
573 .tail = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.tail])),
574 .mask = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.ring_mask])),
575 .flags = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.flags])),
576 .dropped = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.dropped])),
577 .array = array[0..p.sq_entries],
578 .sqes = sqes[0..p.sq_entries],
579 .mmap = mmap,
580 .mmap_sqes = mmap_sqes
581 };
582 }
583
584 pub fn deinit(self: *SubmissionQueue) void {
585 os.munmap(self.mmap_sqes);
586 os.munmap(self.mmap);
587 }
588};
589
590pub const CompletionQueue = struct {
591 head: *u32,
592 tail: *u32,
593 mask: *u32,
594 overflow: *u32,
595 cqes: []io_uring_cqe,
596
597 pub fn init(fd: i32, p: io_uring_params, sq: SubmissionQueue) !CompletionQueue {
598 assert(fd >= 0);
599 assert((p.features & linux.IORING_FEAT_SINGLE_MMAP) != 0);
600 const mmap = sq.mmap;
601 const cqes = @ptrCast(
602 [*]io_uring_cqe,
603 @alignCast(@alignOf(io_uring_cqe), &mmap[p.cq_off.cqes])
604 );
605 assert(
606 p.cq_entries ==
607 @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.ring_entries])).*
608 );
609 return CompletionQueue {
610 .head = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.head])),
611 .tail = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.tail])),
612 .mask = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.ring_mask])),
613 .overflow = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.overflow])),
614 .cqes = cqes[0..p.cq_entries]
615 };
616 }
617
618 pub fn deinit(self: *CompletionQueue) void {
619 // A no-op since we now share the mmap with the submission queue.
620 // Here for symmetry with the submission queue, and for any future feature support.
621 }
622};
623
624inline fn check_errno(res: usize) !void {
625 const errno = linux.getErrno(res);
626 if (errno != 0) return os.unexpectedErrno(errno);
627}
628
629test "queue_nop" {
630 if (builtin.os.tag != .linux) return error.SkipZigTest;
631
632 var ring = try IO_Uring.init(1, 0);
633 defer {
634 ring.deinit();
635 testing.expectEqual(@as(i32, -1), ring.fd);
636 }
637
638 var sqe = try ring.queue_nop(@intCast(u64, 0xaaaaaaaa));
639 testing.expectEqual(io_uring_sqe {
640 .opcode = .NOP,
641 .flags = 0,
642 .ioprio = 0,
643 .fd = 0,
644 .off = 0,
645 .addr = 0,
646 .len = 0,
647 .opflags = 0,
648 .user_data = @intCast(u64, 0xaaaaaaaa),
649 .buf_index = 0,
650 .personality = 0,
651 .splice_fd_in = 0,
652 .__pad2 = [2]u64{ 0, 0 }
653 }, sqe.*);
654
655 testing.expectEqual(@as(u32, 0), ring.sq.sqe_head);
656 testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
657 testing.expectEqual(@as(u32, 0), ring.sq.tail.*);
658 testing.expectEqual(@as(u32, 0), ring.cq.head.*);
659 testing.expectEqual(@as(u32, 1), ring.sq_ready());
660 testing.expectEqual(@as(u32, 0), ring.cq_ready());
661
662 testing.expectEqual(@as(u32, 1), try ring.submit());
663 testing.expectEqual(@as(u32, 1), ring.sq.sqe_head);
664 testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
665 testing.expectEqual(@as(u32, 1), ring.sq.tail.*);
666 testing.expectEqual(@as(u32, 0), ring.cq.head.*);
667 testing.expectEqual(@as(u32, 0), ring.sq_ready());
668
669 testing.expectEqual(io_uring_cqe {
670 .user_data = 0xaaaaaaaa,
671 .res = 0,
672 .flags = 0
673 }, try ring.copy_cqe());
674 testing.expectEqual(@as(u32, 1), ring.cq.head.*);
675 testing.expectEqual(@as(u32, 0), ring.cq_ready());
676
677 var sqe_barrier = try ring.queue_nop(@intCast(u64, 0xbbbbbbbb));
678 ring.drain_previous_sqes(sqe_barrier);
679 testing.expectEqual(@as(u8, linux.IOSQE_IO_DRAIN), sqe_barrier.flags);
680 testing.expectEqual(@as(u32, 1), try ring.submit());
681 testing.expectEqual(io_uring_cqe {
682 .user_data = 0xbbbbbbbb,
683 .res = 0,
684 .flags = 0
685 }, try ring.copy_cqe());
686 testing.expectEqual(@as(u32, 2), ring.sq.sqe_head);
687 testing.expectEqual(@as(u32, 2), ring.sq.sqe_tail);
688 testing.expectEqual(@as(u32, 2), ring.sq.tail.*);
689 testing.expectEqual(@as(u32, 2), ring.cq.head.*);
690}
691
692test "queue_readv" {
693 if (builtin.os.tag != .linux) return error.SkipZigTest;
694
695 var ring = try IO_Uring.init(1, 0);
696 defer ring.deinit();
697
698 const fd = try os.openZ("/dev/zero", os.O_RDONLY | os.O_CLOEXEC, 0);
699 defer os.close(fd);
700
701 // Linux Kernel 5.4 supports IORING_REGISTER_FILES but not sparse fd sets (i.e. an fd of -1).
702 // Linux Kernel 5.5 adds support for sparse fd sets.
703 // Compare:
704 // https://github.com/torvalds/linux/blob/v5.4/fs/io_uring.c#L3119-L3124 vs
705 // https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L6687-L6691
706 // We therefore avoid stressing sparse fd sets here:
707 var registered_fds = [_]i32{0} ** 1;
708 const fd_index = 0;
709 registered_fds[fd_index] = fd;
710 try ring.register_files(registered_fds[0..]);
711
712 var buffer = [_]u8{42} ** 128;
713 var iovecs = [_]os.iovec{ os.iovec { .iov_base = &buffer, .iov_len = buffer.len } };
714 var sqe = try ring.queue_readv(0xcccccccc, fd_index, iovecs[0..], 0);
715 ring.use_registered_fd(sqe);
716 testing.expectEqual(@as(u8, linux.IOSQE_FIXED_FILE), sqe.flags);
717
718 testing.expectError(error.IO_UringSubmissionQueueFull, ring.queue_nop(0));
719 testing.expectEqual(@as(u32, 1), try ring.submit());
720 testing.expectEqual(linux.io_uring_cqe {
721 .user_data = 0xcccccccc,
722 .res = buffer.len,
723 .flags = 0,
724 }, try ring.copy_cqe());
725 testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
726
727 try ring.unregister_files();
728}
729
730test "queue_writev/queue_fsync" {
731 if (builtin.os.tag != .linux) return error.SkipZigTest;
732
733 var ring = try IO_Uring.init(2, 0);
734 defer ring.deinit();
735
736 const path = "test_io_uring_queue_writev";
737 const file = try std.fs.cwd().createFile(path, .{ .truncate = true });
738 defer file.close();
739 defer std.fs.cwd().deleteFile(path) catch {};
740 const fd = file.handle;
741
742 var buffer = [_]u8{42} ** 128;
743 var iovecs = [_]os.iovec_const {
744 os.iovec_const { .iov_base = &buffer, .iov_len = buffer.len }
745 };
746 var sqe_writev = try ring.queue_writev(0xdddddddd, fd, iovecs[0..], 0);
747 ring.link_with_next_sqe(sqe_writev);
748 testing.expectEqual(@as(u8, linux.IOSQE_IO_LINK), sqe_writev.flags);
749
750 var sqe_fsync = try ring.queue_fsync(0xeeeeeeee, fd);
751 testing.expectEqual(fd, sqe_fsync.fd);
752
753 testing.expectEqual(@as(u32, 2), ring.sq_ready());
754 testing.expectEqual(@as(u32, 2), try ring.submit_and_wait(2));
755 testing.expectEqual(@as(u32, 0), ring.sq_ready());
756 testing.expectEqual(@as(u32, 2), ring.cq_ready());
757 testing.expectEqual(linux.io_uring_cqe {
758 .user_data = 0xdddddddd,
759 .res = buffer.len,
760 .flags = 0,
761 }, try ring.copy_cqe());
762 testing.expectEqual(@as(u32, 1), ring.cq_ready());
763 testing.expectEqual(linux.io_uring_cqe {
764 .user_data = 0xeeeeeeee,
765 .res = 0,
766 .flags = 0,
767 }, try ring.copy_cqe());
768 testing.expectEqual(@as(u32, 0), ring.cq_ready());
769}
770
771test "queue_write/queue_read" {
772 // TODO
773 if (builtin.os.tag != .linux or true) return error.SkipZigTest;
774
775 var ring = try IO_Uring.init(2, 0);
776 defer ring.deinit();
777
778 const path = "test_io_uring_queue_write";
779 const file = try std.fs.cwd().createFile(path, .{ .read = true, .truncate = true });
780 defer file.close();
781 defer std.fs.cwd().deleteFile(path) catch {};
782 const fd = file.handle;
783
784 var buffer_write = [_]u8{97} ** 20;
785 var buffer_read = [_]u8{98} ** 20;
786 var sqe_write = try ring.queue_write(123, fd, buffer_write[0..], 10);
787 ring.link_with_next_sqe(sqe_write);
788 var sqe_read = try ring.queue_read(456, fd, buffer_read[0..], 10);
789 testing.expectEqual(@as(u32, 2), try ring.submit());
790 testing.expectEqual(linux.io_uring_cqe {
791 .user_data = 123,
792 .res = buffer_write.len,
793 .flags = 0,
794 }, try ring.copy_cqe());
795 testing.expectEqual(linux.io_uring_cqe {
796 .user_data = 456,
797 .res = buffer_read.len,
798 .flags = 0,
799 }, try ring.copy_cqe());
800 testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
801}