Meg's Mirrors

zig

1	const IoUring = @This();
2
3	const builtin = @import("builtin");
4	const is_linux = builtin.os.tag == .linux;
5
6	const std = @import("../../std.zig");
7	const Io = std.Io;
8	const Allocator = std.mem.Allocator;
9	const assert = std.debug.assert;
10	const posix = std.posix;
11	const linux = std.os.linux;
12	const testing = std.testing;
13	const page_size_min = std.heap.page_size_min;
14	const createSocketTestHarness = @import("IoUring/test.zig").createSocketTestHarness;
15
16	fd: linux.fd_t = -1,
17	sq: SubmissionQueue,
18	cq: CompletionQueue,
19	flags: u32,
20	features: u32,
21
22	/// A friendly way to setup an io_uring, with default linux.io_uring_params.
23	/// `entries` must be a power of two between 1 and 32768, although the kernel will make the final
24	/// call on how many entries the submission and completion queues will ultimately have,
25	/// see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L8027-L8050.
26	/// Matches the interface of io_uring_queue_init() in liburing.
27	pub fn init(entries: u16, flags: u32) !IoUring {
28	var params = std.mem.zeroInit(linux.io_uring_params, .{
29	.flags = flags,
30	.sq_thread_idle = 1000,
31	});
32	return try IoUring.init_params(entries, &params);
33	}
34
35	/// A powerful way to setup an io_uring, if you want to tweak linux.io_uring_params such as submission
36	/// queue thread cpu affinity or thread idle timeout (the kernel and our default is 1 second).
37	/// `params` is passed by reference because the kernel needs to modify the parameters.
38	/// Matches the interface of io_uring_queue_init_params() in liburing.
39	pub fn init_params(entries: u16, p: *linux.io_uring_params) !IoUring {
40	if (entries == 0) return error.EntriesZero;
41	if (!std.math.isPowerOfTwo(entries)) return error.EntriesNotPowerOfTwo;
42
43	assert(p.sq_entries == 0);
44	assert(p.cq_entries == 0 or p.flags & linux.IORING_SETUP_CQSIZE != 0);
45	assert(p.features == 0);
46	assert(p.wq_fd == 0 or p.flags & linux.IORING_SETUP_ATTACH_WQ != 0);
47	assert(p.resv[0] == 0);
48	assert(p.resv[1] == 0);
49	assert(p.resv[2] == 0);
50
51	const res = linux.io_uring_setup(entries, p);
52	switch (linux.errno(res)) {
53	.SUCCESS => {},
54	.FAULT => return error.ParamsOutsideAccessibleAddressSpace,
55	// The resv array contains non-zero data, p.flags contains an unsupported flag,
56	// entries out of bounds, IORING_SETUP_SQ_AFF was specified without IORING_SETUP_SQPOLL,
57	// or IORING_SETUP_CQSIZE was specified but linux.io_uring_params.cq_entries was invalid:
58	.INVAL => return error.ArgumentsInvalid,
59	.MFILE => return error.ProcessFdQuotaExceeded,
60	.NFILE => return error.SystemFdQuotaExceeded,
61	.NOMEM => return error.SystemResources,
62	// IORING_SETUP_SQPOLL was specified but effective user ID lacks sufficient privileges,
63	// or a container seccomp policy prohibits io_uring syscalls:
64	.PERM => return error.PermissionDenied,
65	.NOSYS => return error.SystemOutdated,
66	else => |errno| return posix.unexpectedErrno(errno),
67	}
68	const fd = @as(linux.fd_t, @intCast(res));
69	assert(fd >= 0);
70	errdefer _ = linux.close(fd);
71
72	// Kernel versions 5.4 and up use only one mmap() for the submission and completion queues.
73	// This is not an optional feature for us... if the kernel does it, we have to do it.
74	// The thinking on this by the kernel developers was that both the submission and the
75	// completion queue rings have sizes just over a power of two, but the submission queue ring
76	// is significantly smaller with u32 slots. By bundling both in a single mmap, the kernel
77	// gets the submission queue ring for free.
78	// See https://patchwork.kernel.org/patch/11115257 for the kernel patch.
79	// We do not support the double mmap() done before 5.4, because we want to keep the
80	// init/deinit mmap paths simple and because io_uring has had many bug fixes even since 5.4.
81	if ((p.features & linux.IORING_FEAT_SINGLE_MMAP) == 0) {
82	return error.SystemOutdated;
83	}
84
85	// Check that the kernel has actually set params and that "impossible is nothing".
86	assert(p.sq_entries != 0);
87	assert(p.cq_entries != 0);
88	assert(p.cq_entries >= p.sq_entries);
89
90	// From here on, we only need to read from params, so pass `p` by value as immutable.
91	// The completion queue shares the mmap with the submission queue, so pass `sq` there too.
92	var sq = try SubmissionQueue.init(fd, p.*);
93	errdefer sq.deinit();
94	var cq = try CompletionQueue.init(fd, p.*, sq);
95	errdefer cq.deinit();
96
97	// Check that our starting state is as we expect.
98	assert(sq.head.* == 0);
99	assert(sq.tail.* == 0);
100	assert(sq.mask == p.sq_entries - 1);
101	// Allow flags.* to be non-zero, since the kernel may set IORING_SQ_NEED_WAKEUP at any time.
102	assert(sq.dropped.* == 0);
103	assert(sq.array.len == p.sq_entries);
104	assert(sq.sqes.len == p.sq_entries);
105	assert(sq.sqe_head == 0);
106	assert(sq.sqe_tail == 0);
107
108	assert(cq.head.* == 0);
109	assert(cq.tail.* == 0);
110	assert(cq.mask == p.cq_entries - 1);
111	assert(cq.overflow.* == 0);
112	assert(cq.cqes.len == p.cq_entries);
113
114	return IoUring{
115	.fd = fd,
116	.sq = sq,
117	.cq = cq,
118	.flags = p.flags,
119	.features = p.features,
120	};
121	}
122
123	pub fn deinit(self: *IoUring) void {
124	assert(self.fd >= 0);
125	// The mmaps depend on the fd, so the order of these calls is important:
126	self.cq.deinit();
127	self.sq.deinit();
128	_ = linux.close(self.fd);
129	self.fd = -1;
130	}
131
132	/// Returns a pointer to a vacant SQE, or an error if the submission queue is full.
133	/// We follow the implementation (and atomics) of liburing's `io_uring_get_sqe()` exactly.
134	/// However, instead of a null we return an error to force safe handling.
135	/// Any situation where the submission queue is full tends more towards a control flow error,
136	/// and the null return in liburing is more a C idiom than anything else, for lack of a better
137	/// alternative. In Zig, we have first-class error handling... so let's use it.
138	/// Matches the implementation of io_uring_get_sqe() in liburing.
139	pub fn get_sqe(self: *IoUring) !*linux.io_uring_sqe {
140	const head = @atomicLoad(u32, self.sq.head, .acquire);
141	// Remember that these head and tail offsets wrap around every four billion operations.
142	// We must therefore use wrapping addition and subtraction to avoid a runtime crash.
143	const next = self.sq.sqe_tail +% 1;
144	if (next -% head > self.sq.sqes.len) return error.SubmissionQueueFull;
145	const sqe = &self.sq.sqes[self.sq.sqe_tail & self.sq.mask];
146	self.sq.sqe_tail = next;
147	return sqe;
148	}
149
150	/// Submits the SQEs acquired via get_sqe() to the kernel. You can call this once after you have
151	/// called get_sqe() multiple times to setup multiple I/O requests.
152	/// Returns the number of SQEs submitted, if not used alongside IORING_SETUP_SQPOLL.
153	/// If the io_uring instance is uses IORING_SETUP_SQPOLL, the value returned on success is not
154	/// guaranteed to match the amount of actually submitted sqes during this call. A value higher
155	/// or lower, including 0, may be returned.
156	/// Matches the implementation of io_uring_submit() in liburing.
157	pub fn submit(self: *IoUring) !u32 {
158	return self.submit_and_wait(0);
159	}
160
161	/// Like submit(), but allows waiting for events as well.
162	/// Returns the number of SQEs submitted.
163	/// Matches the implementation of io_uring_submit_and_wait() in liburing.
164	pub fn submit_and_wait(self: *IoUring, wait_nr: u32) !u32 {
165	const submitted = self.flush_sq();
166	var flags: u32 = 0;
167	if (self.sq_ring_needs_enter(&flags) or wait_nr > 0) {
168	if (wait_nr > 0 or (self.flags & linux.IORING_SETUP_IOPOLL) != 0) {
169	flags |= linux.IORING_ENTER_GETEVENTS;
170	}
171	return try self.enter(submitted, wait_nr, flags);
172	}
173	return submitted;
174	}
175
176	/// Tell the kernel we have submitted SQEs and/or want to wait for CQEs.
177	/// Returns the number of SQEs submitted.
178	pub fn enter(self: *IoUring, to_submit: u32, min_complete: u32, flags: u32) !u32 {
179	assert(self.fd >= 0);
180	const res = linux.io_uring_enter(self.fd, to_submit, min_complete, flags, null);
181	switch (linux.errno(res)) {
182	.SUCCESS => {},
183	// The kernel was unable to allocate memory or ran out of resources for the request.
184	// The application should wait for some completions and try again:
185	.AGAIN => return error.SystemResources,
186	// The SQE `fd` is invalid, or IOSQE_FIXED_FILE was set but no files were registered:
187	.BADF => return error.FileDescriptorInvalid,
188	// The file descriptor is valid, but the ring is not in the right state.
189	// See io_uring_register(2) for how to enable the ring.
190	.BADFD => return error.FileDescriptorInBadState,
191	// The application attempted to overcommit the number of requests it can have pending.
192	// The application should wait for some completions and try again:
193	.BUSY => return error.CompletionQueueOvercommitted,
194	// The SQE is invalid, or valid but the ring was setup with IORING_SETUP_IOPOLL:
195	.INVAL => return error.SubmissionQueueEntryInvalid,
196	// The buffer is outside the process' accessible address space, or IORING_OP_READ_FIXED
197	// or IORING_OP_WRITE_FIXED was specified but no buffers were registered, or the range
198	// described by `addr` and `len` is not within the buffer registered at `buf_index`:
199	.FAULT => return error.BufferInvalid,
200	.NXIO => return error.RingShuttingDown,
201	// The kernel believes our `self.fd` does not refer to an io_uring instance,
202	// or the opcode is valid but not supported by this kernel (more likely):
203	.OPNOTSUPP => return error.OpcodeNotSupported,
204	// The thread submitting the work is invalid. This may occur if IORING_ENTER_GETEVENTS
205	// and IORING_SETUP_DEFER_TASKRUN is set, but the submitting thread is not the thread
206	// that initially created or enabled the io_uring associated with fd.
207	.EXIST => return error.InvalidThread,
208	// The operation was interrupted by a delivery of a signal before it could complete.
209	// This can happen while waiting for events with IORING_ENTER_GETEVENTS:
210	.INTR => return error.SignalInterrupt,
211	else => |errno| return posix.unexpectedErrno(errno),
212	}
213	return @as(u32, @intCast(res));
214	}
215
216	/// Sync internal state with kernel ring state on the SQ side.
217	/// Returns the number of all pending events in the SQ ring, for the shared ring.
218	/// This return value includes previously flushed SQEs, as per liburing.
219	/// The rationale is to suggest that an io_uring_enter() call is needed rather than not.
220	/// Matches the implementation of __io_uring_flush_sq() in liburing.
221	pub fn flush_sq(self: *IoUring) u32 {
222	if (self.sq.sqe_head != self.sq.sqe_tail) {
223	// Fill in SQEs that we have queued up, adding them to the kernel ring.
224	const to_submit = self.sq.sqe_tail -% self.sq.sqe_head;
225	var tail = self.sq.tail.*;
226	var i: usize = 0;
227	while (i < to_submit) : (i += 1) {
228	self.sq.array[tail & self.sq.mask] = self.sq.sqe_head & self.sq.mask;
229	tail +%= 1;
230	self.sq.sqe_head +%= 1;
231	}
232	// Ensure that the kernel can actually see the SQE updates when it sees the tail update.
233	@atomicStore(u32, self.sq.tail, tail, .release);
234	}
235	return self.sq_ready();
236	}
237
238	/// Returns true if we are not using an SQ thread (thus nobody submits but us),
239	/// or if IORING_SQ_NEED_WAKEUP is set and the SQ thread must be explicitly awakened.
240	/// For the latter case, we set the SQ thread wakeup flag.
241	/// Matches the implementation of sq_ring_needs_enter() in liburing.
242	pub fn sq_ring_needs_enter(self: *IoUring, flags: *u32) bool {
243	assert(flags.* == 0);
244	if ((self.flags & linux.IORING_SETUP_SQPOLL) == 0) return true;
245	if ((@atomicLoad(u32, self.sq.flags, .unordered) & linux.IORING_SQ_NEED_WAKEUP) != 0) {
246	flags.* |= linux.IORING_ENTER_SQ_WAKEUP;
247	return true;
248	}
249	return false;
250	}
251
252	/// Returns the number of flushed and unflushed SQEs pending in the submission queue.
253	/// In other words, this is the number of SQEs in the submission queue, i.e. its length.
254	/// These are SQEs that the kernel is yet to consume.
255	/// Matches the implementation of io_uring_sq_ready in liburing.
256	pub fn sq_ready(self: *IoUring) u32 {
257	// Always use the shared ring state (i.e. head and not sqe_head) to avoid going out of sync,
258	// see https://github.com/axboe/liburing/issues/92.
259	return self.sq.sqe_tail -% @atomicLoad(u32, self.sq.head, .acquire);
260	}
261
262	/// Returns the number of CQEs in the completion queue, i.e. its length.
263	/// These are CQEs that the application is yet to consume.
264	/// Matches the implementation of io_uring_cq_ready in liburing.
265	pub fn cq_ready(self: *IoUring) u32 {
266	return @atomicLoad(u32, self.cq.tail, .acquire) -% self.cq.head.*;
267	}
268
269	/// Copies as many CQEs as are ready, and that can fit into the destination `cqes` slice.
270	/// If none are available, enters into the kernel to wait for at least `wait_nr` CQEs.
271	/// Returns the number of CQEs copied, advancing the CQ ring.
272	/// Provides all the wait/peek methods found in liburing, but with batching and a single method.
273	/// The rationale for copying CQEs rather than copying pointers is that pointers are 8 bytes
274	/// whereas CQEs are not much more at only 16 bytes, and this provides a safer faster interface.
275	/// Safer, because you no longer need to call cqe_seen(), avoiding idempotency bugs.
276	/// Faster, because we can now amortize the atomic store release to `cq.head` across the batch.
277	/// See https://github.com/axboe/liburing/issues/103#issuecomment-686665007.
278	/// Matches the implementation of io_uring_peek_batch_cqe() in liburing, but supports waiting.
279	pub fn copy_cqes(self: *IoUring, cqes: []linux.io_uring_cqe, wait_nr: u32) !u32 {
280	const count = self.copy_cqes_ready(cqes);
281	if (count > 0) return count;
282	if (self.cq_ring_needs_flush() or wait_nr > 0) {
283	_ = try self.enter(0, wait_nr, linux.IORING_ENTER_GETEVENTS);
284	return self.copy_cqes_ready(cqes);
285	}
286	return 0;
287	}
288
289	fn copy_cqes_ready(self: *IoUring, cqes: []linux.io_uring_cqe) u32 {
290	const ready = self.cq_ready();
291	const count = @min(cqes.len, ready);
292	const head = self.cq.head.* & self.cq.mask;
293
294	// before wrapping
295	const n = @min(self.cq.cqes.len - head, count);
296	@memcpy(cqes[0..n], self.cq.cqes[head..][0..n]);
297
298	if (count > n) {
299	// wrap self.cq.cqes
300	const w = count - n;
301	@memcpy(cqes[n..][0..w], self.cq.cqes[0..w]);
302	}
303
304	self.cq_advance(count);
305	return count;
306	}
307
308	/// Returns a copy of an I/O completion, waiting for it if necessary, and advancing the CQ ring.
309	/// A convenience method for `copy_cqes()` for when you don't need to batch or peek.
310	pub fn copy_cqe(ring: *IoUring) !linux.io_uring_cqe {
311	var cqes: [1]linux.io_uring_cqe = undefined;
312	while (true) {
313	const count = try ring.copy_cqes(&cqes, 1);
314	if (count > 0) return cqes[0];
315	}
316	}
317
318	/// Matches the implementation of cq_ring_needs_flush() in liburing.
319	pub fn cq_ring_needs_flush(self: *IoUring) bool {
320	return (@atomicLoad(u32, self.sq.flags, .unordered) & linux.IORING_SQ_CQ_OVERFLOW) != 0;
321	}
322
323	/// For advanced use cases only that implement custom completion queue methods.
324	/// If you use copy_cqes() or copy_cqe() you must not call cqe_seen() or cq_advance().
325	/// Must be called exactly once after a zero-copy CQE has been processed by your application.
326	/// Not idempotent, calling more than once will result in other CQEs being lost.
327	/// Matches the implementation of cqe_seen() in liburing.
328	pub fn cqe_seen(self: *IoUring, cqe: *linux.io_uring_cqe) void {
329	_ = cqe;
330	self.cq_advance(1);
331	}
332
333	/// For advanced use cases only that implement custom completion queue methods.
334	/// Matches the implementation of cq_advance() in liburing.
335	pub fn cq_advance(self: *IoUring, count: u32) void {
336	if (count > 0) {
337	// Ensure the kernel only sees the new head value after the CQEs have been read.
338	@atomicStore(u32, self.cq.head, self.cq.head.* +% count, .release);
339	}
340	}
341
342	/// Queues (but does not submit) an SQE to perform an `fsync(2)`.
343	/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
344	/// For example, for `fdatasync()` you can set `IORING_FSYNC_DATASYNC` in the SQE's `rw_flags`.
345	/// N.B. While SQEs are initiated in the order in which they appear in the submission queue,
346	/// operations execute in parallel and completions are unordered. Therefore, an application that
347	/// submits a write followed by an fsync in the submission queue cannot expect the fsync to
348	/// apply to the write, since the fsync may complete before the write is issued to the disk.
349	/// You should preferably use `link_with_next_sqe()` on a write's SQE to link it with an fsync,
350	/// or else insert a full write barrier using `drain_previous_sqes()` when queueing an fsync.
351	pub fn fsync(self: *IoUring, user_data: u64, fd: linux.fd_t, flags: u32) !*linux.io_uring_sqe {
352	const sqe = try self.get_sqe();
353	sqe.prep_fsync(fd, flags);
354	sqe.user_data = user_data;
355	return sqe;
356	}
357
358	/// Queues (but does not submit) an SQE to perform a no-op.
359	/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
360	/// A no-op is more useful than may appear at first glance.
361	/// For example, you could call `drain_previous_sqes()` on the returned SQE, to use the no-op to
362	/// know when the ring is idle before acting on a kill signal.
363	pub fn nop(self: *IoUring, user_data: u64) !*linux.io_uring_sqe {
364	const sqe = try self.get_sqe();
365	sqe.prep_nop();
366	sqe.user_data = user_data;
367	return sqe;
368	}
369
370	/// Used to select how the read should be handled.
371	pub const ReadBuffer = union(enum) {
372	/// io_uring will read directly into this buffer
373	buffer: []u8,
374
375	/// io_uring will read directly into these buffers using readv.
376	iovecs: []const posix.iovec,
377
378	/// io_uring will select a buffer that has previously been provided with `provide_buffers`.
379	/// The buffer group reference by `group_id` must contain at least one buffer for the read to work.
380	/// `len` controls the number of bytes to read into the selected buffer.
381	buffer_selection: struct {
382	group_id: u16,
383	len: usize,
384	},
385	};
386
387	/// Queues (but does not submit) an SQE to perform a `read(2)` or `preadv(2)` depending on the buffer type.
388	/// * Reading into a `ReadBuffer.buffer` uses `read(2)`
389	/// * Reading into a `ReadBuffer.iovecs` uses `preadv(2)`
390	/// If you want to do a `preadv2(2)` then set `rw_flags` on the returned SQE. See https://man7.org/linux/man-pages/man2/preadv2.2.html
391	///
392	/// Returns a pointer to the SQE.
393	pub fn read(
394	self: *IoUring,
395	user_data: u64,
396	fd: linux.fd_t,
397	buffer: ReadBuffer,
398	offset: u64,
399	) !*linux.io_uring_sqe {
400	const sqe = try self.get_sqe();
401	switch (buffer) {
402	.buffer => |slice| sqe.prep_read(fd, slice, offset),
403	.iovecs => |vecs| sqe.prep_readv(fd, vecs, offset),
404	.buffer_selection => |selection| {
405	sqe.prep_rw(.READ, fd, 0, selection.len, offset);
406	sqe.flags |= linux.IOSQE_BUFFER_SELECT;
407	sqe.buf_index = selection.group_id;
408	},
409	}
410	sqe.user_data = user_data;
411	return sqe;
412	}
413
414	/// Queues (but does not submit) an SQE to perform a `write(2)`.
415	/// Returns a pointer to the SQE.
416	pub fn write(
417	self: *IoUring,
418	user_data: u64,
419	fd: linux.fd_t,
420	buffer: []const u8,
421	offset: u64,
422	) !*linux.io_uring_sqe {
423	const sqe = try self.get_sqe();
424	sqe.prep_write(fd, buffer, offset);
425	sqe.user_data = user_data;
426	return sqe;
427	}
428
429	/// Queues (but does not submit) an SQE to perform a `splice(2)`
430	/// Either `fd_in` or `fd_out` must be a pipe.
431	/// If `fd_in` refers to a pipe, `off_in` is ignored and must be set to std.math.maxInt(u64).
432	/// If `fd_in` does not refer to a pipe and `off_in` is maxInt(u64), then `len` are read
433	/// from `fd_in` starting from the file offset, which is incremented by the number of bytes read.
434	/// If `fd_in` does not refer to a pipe and `off_in` is not maxInt(u64), then the starting offset of `fd_in` will be `off_in`.
435	/// This splice operation can be used to implement sendfile by splicing to an intermediate pipe first,
436	/// then splice to the final destination. In fact, the implementation of sendfile in kernel uses splice internally.
437	///
438	/// NOTE that even if fd_in or fd_out refers to a pipe, the splice operation can still fail with EINVAL if one of the
439	/// fd doesn't explicitly support splice peration, e.g. reading from terminal is unsupported from kernel 5.7 to 5.11.
440	/// See https://github.com/axboe/liburing/issues/291
441	///
442	/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
443	pub fn splice(self: *IoUring, user_data: u64, fd_in: linux.fd_t, off_in: u64, fd_out: linux.fd_t, off_out: u64, len: usize) !*linux.io_uring_sqe {
444	const sqe = try self.get_sqe();
445	sqe.prep_splice(fd_in, off_in, fd_out, off_out, len);
446	sqe.user_data = user_data;
447	return sqe;
448	}
449
450	/// Queues (but does not submit) an SQE to perform a IORING_OP_READ_FIXED.
451	/// The `buffer` provided must be registered with the kernel by calling `register_buffers` first.
452	/// The `buffer_index` must be the same as its index in the array provided to `register_buffers`.
453	///
454	/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
455	pub fn read_fixed(
456	self: *IoUring,
457	user_data: u64,
458	fd: linux.fd_t,
459	buffer: *posix.iovec,
460	offset: u64,
461	buffer_index: u16,
462	) !*linux.io_uring_sqe {
463	const sqe = try self.get_sqe();
464	sqe.prep_read_fixed(fd, buffer, offset, buffer_index);
465	sqe.user_data = user_data;
466	return sqe;
467	}
468
469	/// Queues (but does not submit) an SQE to perform a `pwritev()`.
470	/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
471	/// For example, if you want to do a `pwritev2()` then set `rw_flags` on the returned SQE.
472	/// See https://linux.die.net/man/2/pwritev.
473	pub fn writev(
474	self: *IoUring,
475	user_data: u64,
476	fd: linux.fd_t,
477	iovecs: []const posix.iovec_const,
478	offset: u64,
479	) !*linux.io_uring_sqe {
480	const sqe = try self.get_sqe();
481	sqe.prep_writev(fd, iovecs, offset);
482	sqe.user_data = user_data;
483	return sqe;
484	}
485
486	/// Queues (but does not submit) an SQE to perform a IORING_OP_WRITE_FIXED.
487	/// The `buffer` provided must be registered with the kernel by calling `register_buffers` first.
488	/// The `buffer_index` must be the same as its index in the array provided to `register_buffers`.
489	///
490	/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
491	pub fn write_fixed(
492	self: *IoUring,
493	user_data: u64,
494	fd: linux.fd_t,
495	buffer: *posix.iovec,
496	offset: u64,
497	buffer_index: u16,
498	) !*linux.io_uring_sqe {
499	const sqe = try self.get_sqe();
500	sqe.prep_write_fixed(fd, buffer, offset, buffer_index);
501	sqe.user_data = user_data;
502	return sqe;
503	}
504
505	/// Queues (but does not submit) an SQE to perform an `accept4(2)` on a socket.
506	/// Returns a pointer to the SQE.
507	/// Available since 5.5
508	pub fn accept(
509	self: *IoUring,
510	user_data: u64,
511	fd: linux.fd_t,
512	addr: ?*posix.sockaddr,
513	addrlen: ?*posix.socklen_t,
514	flags: u32,
515	) !*linux.io_uring_sqe {
516	const sqe = try self.get_sqe();
517	sqe.prep_accept(fd, addr, addrlen, flags);
518	sqe.user_data = user_data;
519	return sqe;
520	}
521
522	/// Queues an multishot accept on a socket.
523	///
524	/// Multishot variant allows an application to issue a single accept request,
525	/// which will repeatedly trigger a CQE when a connection request comes in.
526	/// While IORING_CQE_F_MORE flag is set in CQE flags accept will generate
527	/// further CQEs.
528	///
529	/// Available since 5.19
530	pub fn accept_multishot(
531	self: *IoUring,
532	user_data: u64,
533	fd: linux.fd_t,
534	addr: ?*posix.sockaddr,
535	addrlen: ?*posix.socklen_t,
536	flags: u32,
537	) !*linux.io_uring_sqe {
538	const sqe = try self.get_sqe();
539	sqe.prep_multishot_accept(fd, addr, addrlen, flags);
540	sqe.user_data = user_data;
541	return sqe;
542	}
543
544	/// Queues an accept using direct (registered) file descriptors.
545	///
546	/// To use an accept direct variant, the application must first have registered
547	/// a file table (with register_files). An unused table index will be
548	/// dynamically chosen and returned in the CQE res field.
549	///
550	/// After creation, they can be used by setting IOSQE_FIXED_FILE in the SQE
551	/// flags member, and setting the SQE fd field to the direct descriptor value
552	/// rather than the regular file descriptor.
553	///
554	/// Available since 5.19
555	pub fn accept_direct(
556	self: *IoUring,
557	user_data: u64,
558	fd: linux.fd_t,
559	addr: ?*posix.sockaddr,
560	addrlen: ?*posix.socklen_t,
561	flags: u32,
562	) !*linux.io_uring_sqe {
563	const sqe = try self.get_sqe();
564	sqe.prep_accept_direct(fd, addr, addrlen, flags, linux.IORING_FILE_INDEX_ALLOC);
565	sqe.user_data = user_data;
566	return sqe;
567	}
568
569	/// Queues an multishot accept using direct (registered) file descriptors.
570	/// Available since 5.19
571	pub fn accept_multishot_direct(
572	self: *IoUring,
573	user_data: u64,
574	fd: linux.fd_t,
575	addr: ?*posix.sockaddr,
576	addrlen: ?*posix.socklen_t,
577	flags: u32,
578	) !*linux.io_uring_sqe {
579	const sqe = try self.get_sqe();
580	sqe.prep_multishot_accept_direct(fd, addr, addrlen, flags);
581	sqe.user_data = user_data;
582	return sqe;
583	}
584
585	/// Queue (but does not submit) an SQE to perform a `connect(2)` on a socket.
586	/// Returns a pointer to the SQE.
587	pub fn connect(
588	self: *IoUring,
589	user_data: u64,
590	fd: linux.fd_t,
591	addr: *const posix.sockaddr,
592	addrlen: posix.socklen_t,
593	) !*linux.io_uring_sqe {
594	const sqe = try self.get_sqe();
595	sqe.prep_connect(fd, addr, addrlen);
596	sqe.user_data = user_data;
597	return sqe;
598	}
599
600	/// Queues (but does not submit) an SQE to perform a `epoll_ctl(2)`.
601	/// Returns a pointer to the SQE.
602	pub fn epoll_ctl(
603	self: *IoUring,
604	user_data: u64,
605	epfd: linux.fd_t,
606	fd: linux.fd_t,
607	op: u32,
608	ev: ?*linux.epoll_event,
609	) !*linux.io_uring_sqe {
610	const sqe = try self.get_sqe();
611	sqe.prep_epoll_ctl(epfd, fd, op, ev);
612	sqe.user_data = user_data;
613	return sqe;
614	}
615
616	/// Used to select how the recv call should be handled.
617	pub const RecvBuffer = union(enum) {
618	/// io_uring will recv directly into this buffer
619	buffer: []u8,
620
621	/// io_uring will select a buffer that has previously been provided with `provide_buffers`.
622	/// The buffer group referenced by `group_id` must contain at least one buffer for the recv call to work.
623	/// `len` controls the number of bytes to read into the selected buffer.
624	buffer_selection: struct {
625	group_id: u16,
626	len: usize,
627	},
628	};
629
630	/// Queues (but does not submit) an SQE to perform a `recv(2)`.
631	/// Returns a pointer to the SQE.
632	/// Available since 5.6
633	pub fn recv(
634	self: *IoUring,
635	user_data: u64,
636	fd: linux.fd_t,
637	buffer: RecvBuffer,
638	flags: u32,
639	) !*linux.io_uring_sqe {
640	const sqe = try self.get_sqe();
641	switch (buffer) {
642	.buffer => |slice| sqe.prep_recv(fd, slice, flags),
643	.buffer_selection => |selection| {
644	sqe.prep_rw(.RECV, fd, 0, selection.len, 0);
645	sqe.rw_flags = flags;
646	sqe.flags |= linux.IOSQE_BUFFER_SELECT;
647	sqe.buf_index = selection.group_id;
648	},
649	}
650	sqe.user_data = user_data;
651	return sqe;
652	}
653
654	/// Queues (but does not submit) an SQE to perform a `send(2)`.
655	/// Returns a pointer to the SQE.
656	/// Available since 5.6
657	pub fn send(
658	self: *IoUring,
659	user_data: u64,
660	fd: linux.fd_t,
661	buffer: []const u8,
662	flags: u32,
663	) !*linux.io_uring_sqe {
664	const sqe = try self.get_sqe();
665	sqe.prep_send(fd, buffer, flags);
666	sqe.user_data = user_data;
667	return sqe;
668	}
669
670	/// Queues (but does not submit) an SQE to perform an async zerocopy `send(2)`.
671	///
672	/// This operation will most likely produce two CQEs. The flags field of the
673	/// first cqe may likely contain IORING_CQE_F_MORE, which means that there will
674	/// be a second cqe with the user_data field set to the same value. The user
675	/// must not modify the data buffer until the notification is posted. The first
676	/// cqe follows the usual rules and so its res field will contain the number of
677	/// bytes sent or a negative error code. The notification's res field will be
678	/// set to zero and the flags field will contain IORING_CQE_F_NOTIF. The two
679	/// step model is needed because the kernel may hold on to buffers for a long
680	/// time, e.g. waiting for a TCP ACK. Notifications responsible for controlling
681	/// the lifetime of the buffers. Even errored requests may generate a
682	/// notification.
683	///
684	/// Available since 6.0
685	pub fn send_zc(
686	self: *IoUring,
687	user_data: u64,
688	fd: linux.fd_t,
689	buffer: []const u8,
690	send_flags: u32,
691	zc_flags: u16,
692	) !*linux.io_uring_sqe {
693	const sqe = try self.get_sqe();
694	sqe.prep_send_zc(fd, buffer, send_flags, zc_flags);
695	sqe.user_data = user_data;
696	return sqe;
697	}
698
699	/// Queues (but does not submit) an SQE to perform an async zerocopy `send(2)`.
700	/// Returns a pointer to the SQE.
701	/// Available since 6.0
702	pub fn send_zc_fixed(
703	self: *IoUring,
704	user_data: u64,
705	fd: linux.fd_t,
706	buffer: []const u8,
707	send_flags: u32,
708	zc_flags: u16,
709	buf_index: u16,
710	) !*linux.io_uring_sqe {
711	const sqe = try self.get_sqe();
712	sqe.prep_send_zc_fixed(fd, buffer, send_flags, zc_flags, buf_index);
713	sqe.user_data = user_data;
714	return sqe;
715	}
716
717	/// Queues (but does not submit) an SQE to perform a `recvmsg(2)`.
718	/// Returns a pointer to the SQE.
719	/// Available since 5.3
720	pub fn recvmsg(
721	self: *IoUring,
722	user_data: u64,
723	fd: linux.fd_t,
724	msg: *linux.msghdr,
725	flags: u32,
726	) !*linux.io_uring_sqe {
727	const sqe = try self.get_sqe();
728	sqe.prep_recvmsg(fd, msg, flags);
729	sqe.user_data = user_data;
730	return sqe;
731	}
732
733	/// Queues (but does not submit) an SQE to perform a `sendmsg(2)`.
734	/// Returns a pointer to the SQE.
735	/// Available since 5.3
736	pub fn sendmsg(
737	self: *IoUring,
738	user_data: u64,
739	fd: linux.fd_t,
740	msg: *const linux.msghdr_const,
741	flags: u32,
742	) !*linux.io_uring_sqe {
743	const sqe = try self.get_sqe();
744	sqe.prep_sendmsg(fd, msg, flags);
745	sqe.user_data = user_data;
746	return sqe;
747	}
748
749	/// Queues (but does not submit) an SQE to perform an async zerocopy `sendmsg(2)`.
750	/// Returns a pointer to the SQE.
751	/// Available since 6.1
752	pub fn sendmsg_zc(
753	self: *IoUring,
754	user_data: u64,
755	fd: linux.fd_t,
756	msg: *const linux.msghdr_const,
757	flags: u32,
758	) !*linux.io_uring_sqe {
759	const sqe = try self.get_sqe();
760	sqe.prep_sendmsg_zc(fd, msg, flags);
761	sqe.user_data = user_data;
762	return sqe;
763	}
764
765	/// Queues (but does not submit) an SQE to perform an `openat(2)`.
766	/// Returns a pointer to the SQE.
767	/// Available since 5.6.
768	pub fn openat(
769	self: *IoUring,
770	user_data: u64,
771	fd: linux.fd_t,
772	path: [*:0]const u8,
773	flags: linux.O,
774	mode: posix.mode_t,
775	) !*linux.io_uring_sqe {
776	const sqe = try self.get_sqe();
777	sqe.prep_openat(fd, path, flags, mode);
778	sqe.user_data = user_data;
779	return sqe;
780	}
781
782	/// Queues an openat using direct (registered) file descriptors.
783	///
784	/// To use an accept direct variant, the application must first have registered
785	/// a file table (with register_files). An unused table index will be
786	/// dynamically chosen and returned in the CQE res field.
787	///
788	/// After creation, they can be used by setting IOSQE_FIXED_FILE in the SQE
789	/// flags member, and setting the SQE fd field to the direct descriptor value
790	/// rather than the regular file descriptor.
791	///
792	/// Available since 5.15
793	pub fn openat_direct(
794	self: *IoUring,
795	user_data: u64,
796	fd: linux.fd_t,
797	path: [*:0]const u8,
798	flags: linux.O,
799	mode: posix.mode_t,
800	file_index: u32,
801	) !*linux.io_uring_sqe {
802	const sqe = try self.get_sqe();
803	sqe.prep_openat_direct(fd, path, flags, mode, file_index);
804	sqe.user_data = user_data;
805	return sqe;
806	}
807
808	/// Queues (but does not submit) an SQE to perform a `close(2)`.
809	/// Returns a pointer to the SQE.
810	/// Available since 5.6.
811	pub fn close(self: *IoUring, user_data: u64, fd: linux.fd_t) !*linux.io_uring_sqe {
812	const sqe = try self.get_sqe();
813	sqe.prep_close(fd);
814	sqe.user_data = user_data;
815	return sqe;
816	}
817
818	/// Queues close of registered file descriptor.
819	/// Available since 5.15
820	pub fn close_direct(self: *IoUring, user_data: u64, file_index: u32) !*linux.io_uring_sqe {
821	const sqe = try self.get_sqe();
822	sqe.prep_close_direct(file_index);
823	sqe.user_data = user_data;
824	return sqe;
825	}
826
827	/// Queues (but does not submit) an SQE to register a timeout operation.
828	/// Returns a pointer to the SQE.
829	///
830	/// The timeout will complete when either the timeout expires, or after the specified number of
831	/// events complete (if `count` is greater than `0`).
832	///
833	/// `flags` may be `0` for a relative timeout, or `IORING_TIMEOUT_ABS` for an absolute timeout.
834	///
835	/// The completion event result will be `-ETIME` if the timeout completed through expiration,
836	/// `0` if the timeout completed after the specified number of events, or `-ECANCELED` if the
837	/// timeout was removed before it expired.
838	///
839	/// io_uring timeouts use the `CLOCK.MONOTONIC` clock source.
840	pub fn timeout(
841	self: *IoUring,
842	user_data: u64,
843	ts: *const linux.kernel_timespec,
844	count: u32,
845	flags: u32,
846	) !*linux.io_uring_sqe {
847	const sqe = try self.get_sqe();
848	sqe.prep_timeout(ts, count, flags);
849	sqe.user_data = user_data;
850	return sqe;
851	}
852
853	/// Queues (but does not submit) an SQE to remove an existing timeout operation.
854	/// Returns a pointer to the SQE.
855	///
856	/// The timeout is identified by its `user_data`.
857	///
858	/// The completion event result will be `0` if the timeout was found and canceled successfully,
859	/// `-EBUSY` if the timeout was found but expiration was already in progress, or
860	/// `-ENOENT` if the timeout was not found.
861	pub fn timeout_remove(
862	self: *IoUring,
863	user_data: u64,
864	timeout_user_data: u64,
865	flags: u32,
866	) !*linux.io_uring_sqe {
867	const sqe = try self.get_sqe();
868	sqe.prep_timeout_remove(timeout_user_data, flags);
869	sqe.user_data = user_data;
870	return sqe;
871	}
872
873	/// Queues (but does not submit) an SQE to add a link timeout operation.
874	/// Returns a pointer to the SQE.
875	///
876	/// You need to set linux.IOSQE_IO_LINK to flags of the target operation
877	/// and then call this method right after the target operation.
878	/// See https://lwn.net/Articles/803932/ for detail.
879	///
880	/// If the dependent request finishes before the linked timeout, the timeout
881	/// is canceled. If the timeout finishes before the dependent request, the
882	/// dependent request will be canceled.
883	///
884	/// The completion event result of the link_timeout will be
885	/// `-ETIME` if the timeout finishes before the dependent request
886	/// (in this case, the completion event result of the dependent request will
887	/// be `-ECANCELED`), or
888	/// `-EALREADY` if the dependent request finishes before the linked timeout.
889	pub fn link_timeout(
890	self: *IoUring,
891	user_data: u64,
892	ts: *const linux.kernel_timespec,
893	flags: u32,
894	) !*linux.io_uring_sqe {
895	const sqe = try self.get_sqe();
896	sqe.prep_link_timeout(ts, flags);
897	sqe.user_data = user_data;
898	return sqe;
899	}
900
901	/// Queues (but does not submit) an SQE to perform a `poll(2)`.
902	/// Returns a pointer to the SQE.
903	pub fn poll_add(
904	self: *IoUring,
905	user_data: u64,
906	fd: linux.fd_t,
907	poll_mask: u32,
908	) !*linux.io_uring_sqe {
909	const sqe = try self.get_sqe();
910	sqe.prep_poll_add(fd, poll_mask);
911	sqe.user_data = user_data;
912	return sqe;
913	}
914
915	/// Queues (but does not submit) an SQE to remove an existing poll operation.
916	/// Returns a pointer to the SQE.
917	pub fn poll_remove(
918	self: *IoUring,
919	user_data: u64,
920	target_user_data: u64,
921	) !*linux.io_uring_sqe {
922	const sqe = try self.get_sqe();
923	sqe.prep_poll_remove(target_user_data);
924	sqe.user_data = user_data;
925	return sqe;
926	}
927
928	/// Queues (but does not submit) an SQE to update the user data of an existing poll
929	/// operation. Returns a pointer to the SQE.
930	pub fn poll_update(
931	self: *IoUring,
932	user_data: u64,
933	old_user_data: u64,
934	new_user_data: u64,
935	poll_mask: u32,
936	flags: u32,
937	) !*linux.io_uring_sqe {
938	const sqe = try self.get_sqe();
939	sqe.prep_poll_update(old_user_data, new_user_data, poll_mask, flags);
940	sqe.user_data = user_data;
941	return sqe;
942	}
943
944	/// Queues (but does not submit) an SQE to perform an `fallocate(2)`.
945	/// Returns a pointer to the SQE.
946	pub fn fallocate(
947	self: *IoUring,
948	user_data: u64,
949	fd: linux.fd_t,
950	mode: i32,
951	offset: u64,
952	len: u64,
953	) !*linux.io_uring_sqe {
954	const sqe = try self.get_sqe();
955	sqe.prep_fallocate(fd, mode, offset, len);
956	sqe.user_data = user_data;
957	return sqe;
958	}
959
960	/// Queues (but does not submit) an SQE to perform an `statx(2)`.
961	/// Returns a pointer to the SQE.
962	pub fn statx(
963	self: *IoUring,
964	user_data: u64,
965	fd: linux.fd_t,
966	path: [:0]const u8,
967	flags: u32,
968	mask: linux.STATX,
969	buf: *linux.Statx,
970	) !*linux.io_uring_sqe {
971	const sqe = try self.get_sqe();
972	sqe.prep_statx(fd, path, flags, mask, buf);
973	sqe.user_data = user_data;
974	return sqe;
975	}
976
977	/// Queues (but does not submit) an SQE to remove an existing operation.
978	/// Returns a pointer to the SQE.
979	///
980	/// The operation is identified by its `user_data`.
981	///
982	/// The completion event result will be `0` if the operation was found and canceled successfully,
983	/// `-EALREADY` if the operation was found but was already in progress, or
984	/// `-ENOENT` if the operation was not found.
985	pub fn cancel(
986	self: *IoUring,
987	user_data: u64,
988	cancel_user_data: u64,
989	flags: u32,
990	) !*linux.io_uring_sqe {
991	const sqe = try self.get_sqe();
992	sqe.prep_cancel(cancel_user_data, flags);
993	sqe.user_data = user_data;
994	return sqe;
995	}
996
997	/// Queues (but does not submit) an SQE to perform a `shutdown(2)`.
998	/// Returns a pointer to the SQE.
999	///
1000	/// The operation is identified by its `user_data`.
1001	pub fn shutdown(
1002	self: *IoUring,
1003	user_data: u64,
1004	sockfd: posix.socket_t,
1005	how: u32,
1006	) !*linux.io_uring_sqe {
1007	const sqe = try self.get_sqe();
1008	sqe.prep_shutdown(sockfd, how);
1009	sqe.user_data = user_data;
1010	return sqe;
1011	}
1012
1013	/// Queues (but does not submit) an SQE to perform a `renameat2(2)`.
1014	/// Returns a pointer to the SQE.
1015	pub fn renameat(
1016	self: *IoUring,
1017	user_data: u64,
1018	old_dir_fd: linux.fd_t,
1019	old_path: [*:0]const u8,
1020	new_dir_fd: linux.fd_t,
1021	new_path: [*:0]const u8,
1022	flags: u32,
1023	) !*linux.io_uring_sqe {
1024	const sqe = try self.get_sqe();
1025	sqe.prep_renameat(old_dir_fd, old_path, new_dir_fd, new_path, flags);
1026	sqe.user_data = user_data;
1027	return sqe;
1028	}
1029
1030	/// Queues (but does not submit) an SQE to perform a `unlinkat(2)`.
1031	/// Returns a pointer to the SQE.
1032	pub fn unlinkat(
1033	self: *IoUring,
1034	user_data: u64,
1035	dir_fd: linux.fd_t,
1036	path: [*:0]const u8,
1037	flags: u32,
1038	) !*linux.io_uring_sqe {
1039	const sqe = try self.get_sqe();
1040	sqe.prep_unlinkat(dir_fd, path, flags);
1041	sqe.user_data = user_data;
1042	return sqe;
1043	}
1044
1045	/// Queues (but does not submit) an SQE to perform a `mkdirat(2)`.
1046	/// Returns a pointer to the SQE.
1047	pub fn mkdirat(
1048	self: *IoUring,
1049	user_data: u64,
1050	dir_fd: linux.fd_t,
1051	path: [*:0]const u8,
1052	mode: posix.mode_t,
1053	) !*linux.io_uring_sqe {
1054	const sqe = try self.get_sqe();
1055	sqe.prep_mkdirat(dir_fd, path, mode);
1056	sqe.user_data = user_data;
1057	return sqe;
1058	}
1059
1060	/// Queues (but does not submit) an SQE to perform a `symlinkat(2)`.
1061	/// Returns a pointer to the SQE.
1062	pub fn symlinkat(
1063	self: *IoUring,
1064	user_data: u64,
1065	target: [*:0]const u8,
1066	new_dir_fd: linux.fd_t,
1067	link_path: [*:0]const u8,
1068	) !*linux.io_uring_sqe {
1069	const sqe = try self.get_sqe();
1070	sqe.prep_symlinkat(target, new_dir_fd, link_path);
1071	sqe.user_data = user_data;
1072	return sqe;
1073	}
1074
1075	/// Queues (but does not submit) an SQE to perform a `linkat(2)`.
1076	/// Returns a pointer to the SQE.
1077	pub fn linkat(
1078	self: *IoUring,
1079	user_data: u64,
1080	old_dir_fd: linux.fd_t,
1081	old_path: [*:0]const u8,
1082	new_dir_fd: linux.fd_t,
1083	new_path: [*:0]const u8,
1084	flags: u32,
1085	) !*linux.io_uring_sqe {
1086	const sqe = try self.get_sqe();
1087	sqe.prep_linkat(old_dir_fd, old_path, new_dir_fd, new_path, flags);
1088	sqe.user_data = user_data;
1089	return sqe;
1090	}
1091
1092	/// Queues (but does not submit) an SQE to provide a group of buffers used for commands that read/receive data.
1093	/// Returns a pointer to the SQE.
1094	///
1095	/// Provided buffers can be used in `read`, `recv` or `recvmsg` commands via .buffer_selection.
1096	///
1097	/// The kernel expects a contiguous block of memory of size (buffers_count * buffer_size).
1098	pub fn provide_buffers(
1099	self: *IoUring,
1100	user_data: u64,
1101	buffers: [*]u8,
1102	buffer_size: usize,
1103	buffers_count: usize,
1104	group_id: usize,
1105	buffer_id: usize,
1106	) !*linux.io_uring_sqe {
1107	const sqe = try self.get_sqe();
1108	sqe.prep_provide_buffers(buffers, buffer_size, buffers_count, group_id, buffer_id);
1109	sqe.user_data = user_data;
1110	return sqe;
1111	}
1112
1113	/// Queues (but does not submit) an SQE to remove a group of provided buffers.
1114	/// Returns a pointer to the SQE.
1115	pub fn remove_buffers(
1116	self: *IoUring,
1117	user_data: u64,
1118	buffers_count: usize,
1119	group_id: usize,
1120	) !*linux.io_uring_sqe {
1121	const sqe = try self.get_sqe();
1122	sqe.prep_remove_buffers(buffers_count, group_id);
1123	sqe.user_data = user_data;
1124	return sqe;
1125	}
1126
1127	/// Queues (but does not submit) an SQE to perform a `waitid(2)`.
1128	/// Returns a pointer to the SQE.
1129	pub fn waitid(
1130	self: *IoUring,
1131	user_data: u64,
1132	id_type: linux.P,
1133	id: i32,
1134	infop: *linux.siginfo_t,
1135	options: u32,
1136	flags: u32,
1137	) !*linux.io_uring_sqe {
1138	const sqe = try self.get_sqe();
1139	sqe.prep_waitid(id_type, id, infop, options, flags);
1140	sqe.user_data = user_data;
1141	return sqe;
1142	}
1143
1144	/// Registers an array of file descriptors.
1145	/// Every time a file descriptor is put in an SQE and submitted to the kernel, the kernel must
1146	/// retrieve a reference to the file, and once I/O has completed the file reference must be
1147	/// dropped. The atomic nature of this file reference can be a slowdown for high IOPS workloads.
1148	/// This slowdown can be avoided by pre-registering file descriptors.
1149	/// To refer to a registered file descriptor, IOSQE_FIXED_FILE must be set in the SQE's flags,
1150	/// and the SQE's fd must be set to the index of the file descriptor in the registered array.
1151	/// Registering file descriptors will wait for the ring to idle.
1152	/// Files are automatically unregistered by the kernel when the ring is torn down.
1153	/// An application need unregister only if it wants to register a new array of file descriptors.
1154	pub fn register_files(self: *IoUring, fds: []const linux.fd_t) !void {
1155	assert(self.fd >= 0);
1156	const res = linux.io_uring_register(
1157	self.fd,
1158	.REGISTER_FILES,
1159	@as(*const anyopaque, @ptrCast(fds.ptr)),
1160	@as(u32, @intCast(fds.len)),
1161	);
1162	try handle_registration_result(res);
1163	}
1164
1165	/// Updates registered file descriptors.
1166	///
1167	/// Updates are applied starting at the provided offset in the original file descriptors slice.
1168	/// There are three kind of updates:
1169	/// * turning a sparse entry (where the fd is -1) into a real one
1170	/// * removing an existing entry (set the fd to -1)
1171	/// * replacing an existing entry with a new fd
1172	/// Adding new file descriptors must be done with `register_files`.
1173	pub fn register_files_update(self: *IoUring, offset: u32, fds: []const linux.fd_t) !void {
1174	assert(self.fd >= 0);
1175
1176	const FilesUpdate = extern struct {
1177	offset: u32,
1178	resv: u32,
1179	fds: u64 align(8),
1180	};
1181	var update = FilesUpdate{
1182	.offset = offset,
1183	.resv = @as(u32, 0),
1184	.fds = @as(u64, @intFromPtr(fds.ptr)),
1185	};
1186
1187	const res = linux.io_uring_register(
1188	self.fd,
1189	.REGISTER_FILES_UPDATE,
1190	@as(*const anyopaque, @ptrCast(&update)),
1191	@as(u32, @intCast(fds.len)),
1192	);
1193	try handle_registration_result(res);
1194	}
1195
1196	/// Registers an empty (-1) file table of `nr_files` number of file descriptors.
1197	pub fn register_files_sparse(self: *IoUring, nr_files: u32) !void {
1198	assert(self.fd >= 0);
1199
1200	const reg = &linux.io_uring_rsrc_register{
1201	.nr = nr_files,
1202	.flags = linux.IORING_RSRC_REGISTER_SPARSE,
1203	.resv2 = 0,
1204	.data = 0,
1205	.tags = 0,
1206	};
1207
1208	const res = linux.io_uring_register(
1209	self.fd,
1210	.REGISTER_FILES2,
1211	@ptrCast(reg),
1212	@as(u32, @sizeOf(linux.io_uring_rsrc_register)),
1213	);
1214
1215	return handle_registration_result(res);
1216	}
1217
1218	// Registers range for fixed file allocations.
1219	// Available since 6.0
1220	pub fn register_file_alloc_range(self: *IoUring, offset: u32, len: u32) !void {
1221	assert(self.fd >= 0);
1222
1223	const range = &linux.io_uring_file_index_range{
1224	.off = offset,
1225	.len = len,
1226	.resv = 0,
1227	};
1228
1229	const res = linux.io_uring_register(
1230	self.fd,
1231	.REGISTER_FILE_ALLOC_RANGE,
1232	@ptrCast(range),
1233	0,
1234	);
1235
1236	return handle_registration_result(res);
1237	}
1238
1239	/// Registers the file descriptor for an eventfd that will be notified of completion events on
1240	/// an io_uring instance.
1241	/// Only a single a eventfd can be registered at any given point in time.
1242	pub fn register_eventfd(self: *IoUring, fd: linux.fd_t) !void {
1243	assert(self.fd >= 0);
1244	const res = linux.io_uring_register(
1245	self.fd,
1246	.REGISTER_EVENTFD,
1247	@as(*const anyopaque, @ptrCast(&fd)),
1248	1,
1249	);
1250	try handle_registration_result(res);
1251	}
1252
1253	/// Registers the file descriptor for an eventfd that will be notified of completion events on
1254	/// an io_uring instance. Notifications are only posted for events that complete in an async manner.
1255	/// This means that events that complete inline while being submitted do not trigger a notification event.
1256	/// Only a single eventfd can be registered at any given point in time.
1257	pub fn register_eventfd_async(self: *IoUring, fd: linux.fd_t) !void {
1258	assert(self.fd >= 0);
1259	const res = linux.io_uring_register(
1260	self.fd,
1261	.REGISTER_EVENTFD_ASYNC,
1262	@as(*const anyopaque, @ptrCast(&fd)),
1263	1,
1264	);
1265	try handle_registration_result(res);
1266	}
1267
1268	/// Unregister the registered eventfd file descriptor.
1269	pub fn unregister_eventfd(self: *IoUring) !void {
1270	assert(self.fd >= 0);
1271	const res = linux.io_uring_register(
1272	self.fd,
1273	.UNREGISTER_EVENTFD,
1274	null,
1275	0,
1276	);
1277	try handle_registration_result(res);
1278	}
1279
1280	pub fn register_napi(self: *IoUring, napi: *linux.io_uring_napi) !void {
1281	assert(self.fd >= 0);
1282	const res = linux.io_uring_register(self.fd, .REGISTER_NAPI, napi, 1);
1283	try handle_registration_result(res);
1284	}
1285
1286	pub fn unregister_napi(self: *IoUring, napi: *linux.io_uring_napi) !void {
1287	assert(self.fd >= 0);
1288	const res = linux.io_uring_register(self.fd, .UNREGISTER_NAPI, napi, 1);
1289	try handle_registration_result(res);
1290	}
1291
1292	/// Registers an array of buffers for use with `read_fixed` and `write_fixed`.
1293	pub fn register_buffers(self: *IoUring, buffers: []const posix.iovec) !void {
1294	assert(self.fd >= 0);
1295	const res = linux.io_uring_register(
1296	self.fd,
1297	.REGISTER_BUFFERS,
1298	buffers.ptr,
1299	@as(u32, @intCast(buffers.len)),
1300	);
1301	try handle_registration_result(res);
1302	}
1303
1304	/// Unregister the registered buffers.
1305	pub fn unregister_buffers(self: *IoUring) !void {
1306	assert(self.fd >= 0);
1307	const res = linux.io_uring_register(self.fd, .UNREGISTER_BUFFERS, null, 0);
1308	switch (linux.errno(res)) {
1309	.SUCCESS => {},
1310	.NXIO => return error.BuffersNotRegistered,
1311	else => |errno| return posix.unexpectedErrno(errno),
1312	}
1313	}
1314
1315	/// Returns a io_uring_probe which is used to probe the capabilities of the
1316	/// io_uring subsystem of the running kernel. The io_uring_probe contains the
1317	/// list of supported operations.
1318	pub fn get_probe(self: *IoUring) !linux.io_uring_probe {
1319	var probe = std.mem.zeroInit(linux.io_uring_probe, .{});
1320	const res = linux.io_uring_register(self.fd, .REGISTER_PROBE, &probe, probe.ops.len);
1321	try handle_register_buf_ring_result(res);
1322	return probe;
1323	}
1324
1325	fn handle_registration_result(res: usize) !void {
1326	switch (linux.errno(res)) {
1327	.SUCCESS => {},
1328	// One or more fds in the array are invalid, or the kernel does not support sparse sets:
1329	.BADF => return error.FileDescriptorInvalid,
1330	.BUSY => return error.FilesAlreadyRegistered,
1331	.INVAL => return error.FilesEmpty,
1332	// Adding `nr_args` file references would exceed the maximum allowed number of files the
1333	// user is allowed to have according to the per-user RLIMIT_NOFILE resource limit and
1334	// the CAP_SYS_RESOURCE capability is not set, or `nr_args` exceeds the maximum allowed
1335	// for a fixed file set (older kernels have a limit of 1024 files vs 64K files):
1336	.MFILE => return error.UserFdQuotaExceeded,
1337	// Insufficient kernel resources, or the caller had a non-zero RLIMIT_MEMLOCK soft
1338	// resource limit but tried to lock more memory than the limit permitted (not enforced
1339	// when the process is privileged with CAP_IPC_LOCK):
1340	.NOMEM => return error.SystemResources,
1341	// Attempt to register files on a ring already registering files or being torn down:
1342	.NXIO => return error.RingShuttingDownOrAlreadyRegisteringFiles,
1343	else => |errno| return posix.unexpectedErrno(errno),
1344	}
1345	}
1346
1347	/// Unregisters all registered file descriptors previously associated with the ring.
1348	pub fn unregister_files(self: *IoUring) !void {
1349	assert(self.fd >= 0);
1350	const res = linux.io_uring_register(self.fd, .UNREGISTER_FILES, null, 0);
1351	switch (linux.errno(res)) {
1352	.SUCCESS => {},
1353	.NXIO => return error.FilesNotRegistered,
1354	else => |errno| return posix.unexpectedErrno(errno),
1355	}
1356	}
1357
1358	/// Prepares a socket creation request.
1359	/// New socket fd will be returned in completion result.
1360	/// Available since 5.19
1361	pub fn socket(
1362	self: *IoUring,
1363	user_data: u64,
1364	domain: u32,
1365	socket_type: u32,
1366	protocol: u32,
1367	flags: u32,
1368	) !*linux.io_uring_sqe {
1369	const sqe = try self.get_sqe();
1370	sqe.prep_socket(domain, socket_type, protocol, flags);
1371	sqe.user_data = user_data;
1372	return sqe;
1373	}
1374
1375	/// Prepares a socket creation request for registered file at index `file_index`.
1376	/// Available since 5.19
1377	pub fn socket_direct(
1378	self: *IoUring,
1379	user_data: u64,
1380	domain: u32,
1381	socket_type: u32,
1382	protocol: u32,
1383	flags: u32,
1384	file_index: u32,
1385	) !*linux.io_uring_sqe {
1386	const sqe = try self.get_sqe();
1387	sqe.prep_socket_direct(domain, socket_type, protocol, flags, file_index);
1388	sqe.user_data = user_data;
1389	return sqe;
1390	}
1391
1392	/// Prepares a socket creation request for registered file, index chosen by kernel (file index alloc).
1393	/// File index will be returned in CQE res field.
1394	/// Available since 5.19
1395	pub fn socket_direct_alloc(
1396	self: *IoUring,
1397	user_data: u64,
1398	domain: u32,
1399	socket_type: u32,
1400	protocol: u32,
1401	flags: u32,
1402	) !*linux.io_uring_sqe {
1403	const sqe = try self.get_sqe();
1404	sqe.prep_socket_direct_alloc(domain, socket_type, protocol, flags);
1405	sqe.user_data = user_data;
1406	return sqe;
1407	}
1408
1409	/// Queues (but does not submit) an SQE to perform an `bind(2)` on a socket.
1410	/// Returns a pointer to the SQE.
1411	/// Available since 6.11
1412	pub fn bind(
1413	self: *IoUring,
1414	user_data: u64,
1415	fd: linux.fd_t,
1416	addr: *const posix.sockaddr,
1417	addrlen: posix.socklen_t,
1418	flags: u32,
1419	) !*linux.io_uring_sqe {
1420	const sqe = try self.get_sqe();
1421	sqe.prep_bind(fd, addr, addrlen, flags);
1422	sqe.user_data = user_data;
1423	return sqe;
1424	}
1425
1426	/// Queues (but does not submit) an SQE to perform an `listen(2)` on a socket.
1427	/// Returns a pointer to the SQE.
1428	/// Available since 6.11
1429	pub fn listen(
1430	self: *IoUring,
1431	user_data: u64,
1432	fd: linux.fd_t,
1433	backlog: usize,
1434	flags: u32,
1435	) !*linux.io_uring_sqe {
1436	const sqe = try self.get_sqe();
1437	sqe.prep_listen(fd, backlog, flags);
1438	sqe.user_data = user_data;
1439	return sqe;
1440	}
1441
1442	/// Prepares an cmd request for a socket.
1443	/// See: https://man7.org/linux/man-pages/man3/io_uring_prep_cmd.3.html
1444	/// Available since 6.7.
1445	pub fn cmd_sock(
1446	self: *IoUring,
1447	user_data: u64,
1448	cmd_op: linux.IO_URING_SOCKET_OP,
1449	fd: linux.fd_t,
1450	level: u32, // linux.SOL
1451	optname: u32, // linux.SO
1452	optval: u64, // pointer to the option value
1453	optlen: u32, // size of the option value
1454	) !*linux.io_uring_sqe {
1455	const sqe = try self.get_sqe();
1456	sqe.prep_cmd_sock(cmd_op, fd, level, optname, optval, optlen);
1457	sqe.user_data = user_data;
1458	return sqe;
1459	}
1460
1461	/// Prepares set socket option for the optname argument, at the protocol
1462	/// level specified by the level argument.
1463	/// Available since 6.7.n
1464	pub fn setsockopt(
1465	self: *IoUring,
1466	user_data: u64,
1467	fd: linux.fd_t,
1468	level: u32, // linux.SOL
1469	optname: u32, // linux.SO
1470	opt: []const u8,
1471	) !*linux.io_uring_sqe {
1472	return try self.cmd_sock(
1473	user_data,
1474	.SETSOCKOPT,
1475	fd,
1476	level,
1477	optname,
1478	@intFromPtr(opt.ptr),
1479	@intCast(opt.len),
1480	);
1481	}
1482
1483	/// Prepares get socket option to retrieve the value for the option specified by
1484	/// the option_name argument for the socket specified by the fd argument.
1485	/// Available since 6.7.
1486	pub fn getsockopt(
1487	self: *IoUring,
1488	user_data: u64,
1489	fd: linux.fd_t,
1490	level: u32, // linux.SOL
1491	optname: u32, // linux.SO
1492	opt: []u8,
1493	) !*linux.io_uring_sqe {
1494	return try self.cmd_sock(
1495	user_data,
1496	.GETSOCKOPT,
1497	fd,
1498	level,
1499	optname,
1500	@intFromPtr(opt.ptr),
1501	@intCast(opt.len),
1502	);
1503	}
1504
1505	pub const SubmissionQueue = struct {
1506	head: *u32,
1507	tail: *u32,
1508	mask: u32,
1509	flags: *u32,
1510	dropped: *u32,
1511	array: []u32,
1512	sqes: []linux.io_uring_sqe,
1513	mmap: []align(page_size_min) u8,
1514	mmap_sqes: []align(page_size_min) u8,
1515
1516	// We use `sqe_head` and `sqe_tail` in the same way as liburing:
1517	// We increment `sqe_tail` (but not `tail`) for each call to `get_sqe()`.
1518	// We then set `tail` to `sqe_tail` once, only when these events are actually submitted.
1519	// This allows us to amortize the cost of the @atomicStore to `tail` across multiple SQEs.
1520	sqe_head: u32 = 0,
1521	sqe_tail: u32 = 0,
1522
1523	pub fn init(fd: linux.fd_t, p: linux.io_uring_params) !SubmissionQueue {
1524	assert(fd >= 0);
1525	assert((p.features & linux.IORING_FEAT_SINGLE_MMAP) != 0);
1526	const size = @max(
1527	p.sq_off.array + p.sq_entries * @sizeOf(u32),
1528	p.cq_off.cqes + p.cq_entries * @sizeOf(linux.io_uring_cqe),
1529	);
1530	const mmap = try posix.mmap(
1531	null,
1532	size,
1533	.{ .READ = true, .WRITE = true },
1534	.{ .TYPE = .SHARED, .POPULATE = true },
1535	fd,
1536	linux.IORING_OFF_SQ_RING,
1537	);
1538	errdefer posix.munmap(mmap);
1539	assert(mmap.len == size);
1540
1541	// The motivation for the `sqes` and `array` indirection is to make it possible for the
1542	// application to preallocate static linux.io_uring_sqe entries and then replay them when needed.
1543	const size_sqes = p.sq_entries * @sizeOf(linux.io_uring_sqe);
1544	const mmap_sqes = try posix.mmap(
1545	null,
1546	size_sqes,
1547	.{ .READ = true, .WRITE = true },
1548	.{ .TYPE = .SHARED, .POPULATE = true },
1549	fd,
1550	linux.IORING_OFF_SQES,
1551	);
1552	errdefer posix.munmap(mmap_sqes);
1553	assert(mmap_sqes.len == size_sqes);
1554
1555	const array: [*]u32 = @ptrCast(@alignCast(&mmap[p.sq_off.array]));
1556	const sqes: [*]linux.io_uring_sqe = @ptrCast(@alignCast(&mmap_sqes[0]));
1557	// We expect the kernel copies p.sq_entries to the u32 pointed to by p.sq_off.ring_entries,
1558	// see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L7843-L7844.
1559	assert(p.sq_entries == @as(*u32, @ptrCast(@alignCast(&mmap[p.sq_off.ring_entries]))).*);
1560	return SubmissionQueue{
1561	.head = @ptrCast(@alignCast(&mmap[p.sq_off.head])),
1562	.tail = @ptrCast(@alignCast(&mmap[p.sq_off.tail])),
1563	.mask = @as(*u32, @ptrCast(@alignCast(&mmap[p.sq_off.ring_mask]))).*,
1564	.flags = @ptrCast(@alignCast(&mmap[p.sq_off.flags])),
1565	.dropped = @ptrCast(@alignCast(&mmap[p.sq_off.dropped])),
1566	.array = array[0..p.sq_entries],
1567	.sqes = sqes[0..p.sq_entries],
1568	.mmap = mmap,
1569	.mmap_sqes = mmap_sqes,
1570	};
1571	}
1572
1573	pub fn deinit(self: *SubmissionQueue) void {
1574	posix.munmap(self.mmap_sqes);
1575	posix.munmap(self.mmap);
1576	}
1577	};
1578
1579	pub const CompletionQueue = struct {
1580	head: *u32,
1581	tail: *u32,
1582	mask: u32,
1583	overflow: *u32,
1584	cqes: []linux.io_uring_cqe,
1585
1586	pub fn init(fd: linux.fd_t, p: linux.io_uring_params, sq: SubmissionQueue) !CompletionQueue {
1587	assert(fd >= 0);
1588	assert((p.features & linux.IORING_FEAT_SINGLE_MMAP) != 0);
1589	const mmap = sq.mmap;
1590	const cqes: [*]linux.io_uring_cqe = @ptrCast(@alignCast(&mmap[p.cq_off.cqes]));
1591	assert(p.cq_entries == @as(*u32, @ptrCast(@alignCast(&mmap[p.cq_off.ring_entries]))).*);
1592	return CompletionQueue{
1593	.head = @ptrCast(@alignCast(&mmap[p.cq_off.head])),
1594	.tail = @ptrCast(@alignCast(&mmap[p.cq_off.tail])),
1595	.mask = @as(*u32, @ptrCast(@alignCast(&mmap[p.cq_off.ring_mask]))).*,
1596	.overflow = @ptrCast(@alignCast(&mmap[p.cq_off.overflow])),
1597	.cqes = cqes[0..p.cq_entries],
1598	};
1599	}
1600
1601	pub fn deinit(self: *CompletionQueue) void {
1602	_ = self;
1603	// A no-op since we now share the mmap with the submission queue.
1604	// Here for symmetry with the submission queue, and for any future feature support.
1605	}
1606	};
1607
1608	/// Group of application provided buffers. Uses newer type, called ring mapped
1609	/// buffers, supported since kernel 5.19. Buffers are identified by a buffer
1610	/// group ID, and within that group, a buffer ID. IO_Uring can have multiple
1611	/// buffer groups, each with unique group ID.
1612	///
1613	/// In `init` application provides contiguous block of memory `buffers` for
1614	/// `buffers_count` buffers of size `buffers_size`. Application can then submit
1615	/// `recv` operation without providing buffer upfront. Once the operation is
1616	/// ready to receive data, a buffer is picked automatically and the resulting
1617	/// CQE will contain the buffer ID in `cqe.buffer_id()`. Use `get` method to get
1618	/// buffer for buffer ID identified by CQE. Once the application has processed
1619	/// the buffer, it may hand ownership back to the kernel, by calling `put`
1620	/// allowing the cycle to repeat.
1621	///
1622	/// Depending on the rate of arrival of data, it is possible that a given buffer
1623	/// group will run out of buffers before those in CQEs can be put back to the
1624	/// kernel. If this happens, a `cqe.err()` will have ENOBUFS as the error value.
1625	///
1626	pub const BufferGroup = struct {
1627	/// Parent ring for which this group is registered.
1628	ring: *IoUring,
1629	/// Pointer to the memory shared by the kernel.
1630	/// `buffers_count` of `io_uring_buf` structures are shared by the kernel.
1631	/// First `io_uring_buf` is overlaid by `io_uring_buf_ring` struct.
1632	br: *align(page_size_min) linux.io_uring_buf_ring,
1633	/// Contiguous block of memory of size (buffers_count * buffer_size).
1634	buffers: []u8,
1635	/// Size of each buffer in buffers.
1636	buffer_size: u32,
1637	/// Number of buffers in `buffers`, number of `io_uring_buf structures` in br.
1638	buffers_count: u16,
1639	/// Head of unconsumed part of each buffer, if incremental consumption is enabled
1640	heads: []u32,
1641	/// ID of this group, must be unique in ring.
1642	group_id: u16,
1643
1644	pub fn init(
1645	ring: *IoUring,
1646	allocator: Allocator,
1647	group_id: u16,
1648	buffer_size: u32,
1649	buffers_count: u16,
1650	) !BufferGroup {
1651	const buffers = try allocator.alloc(u8, buffer_size * buffers_count);
1652	errdefer allocator.free(buffers);
1653	const heads = try allocator.alloc(u32, buffers_count);
1654	errdefer allocator.free(heads);
1655
1656	const br = try setup_buf_ring(ring.fd, buffers_count, group_id, .{ .inc = true });
1657	buf_ring_init(br);
1658
1659	const mask = buf_ring_mask(buffers_count);
1660	var i: u16 = 0;
1661	while (i < buffers_count) : (i += 1) {
1662	const pos = buffer_size * i;
1663	const buf = buffers[pos .. pos + buffer_size];
1664	heads[i] = 0;
1665	buf_ring_add(br, buf, i, mask, i);
1666	}
1667	buf_ring_advance(br, buffers_count);
1668
1669	return BufferGroup{
1670	.ring = ring,
1671	.group_id = group_id,
1672	.br = br,
1673	.buffers = buffers,
1674	.heads = heads,
1675	.buffer_size = buffer_size,
1676	.buffers_count = buffers_count,
1677	};
1678	}
1679
1680	pub fn deinit(self: *BufferGroup, allocator: Allocator) void {
1681	free_buf_ring(self.ring.fd, self.br, self.buffers_count, self.group_id);
1682	allocator.free(self.buffers);
1683	allocator.free(self.heads);
1684	}
1685
1686	// Prepare recv operation which will select buffer from this group.
1687	pub fn recv(self: *BufferGroup, user_data: u64, fd: linux.fd_t, flags: u32) !*linux.io_uring_sqe {
1688	var sqe = try self.ring.get_sqe();
1689	sqe.prep_rw(.RECV, fd, 0, 0, 0);
1690	sqe.rw_flags = flags;
1691	sqe.flags |= linux.IOSQE_BUFFER_SELECT;
1692	sqe.buf_index = self.group_id;
1693	sqe.user_data = user_data;
1694	return sqe;
1695	}
1696
1697	// Prepare multishot recv operation which will select buffer from this group.
1698	pub fn recv_multishot(self: *BufferGroup, user_data: u64, fd: linux.fd_t, flags: u32) !*linux.io_uring_sqe {
1699	var sqe = try self.recv(user_data, fd, flags);
1700	sqe.ioprio |= linux.IORING_RECV_MULTISHOT;
1701	return sqe;
1702	}
1703
1704	// Get buffer by id.
1705	pub fn get_by_id(self: *BufferGroup, buffer_id: u16) []u8 {
1706	const pos = self.buffer_size * buffer_id;
1707	return self.buffers[pos .. pos + self.buffer_size][self.heads[buffer_id]..];
1708	}
1709
1710	// Get buffer by CQE.
1711	pub fn get(self: *BufferGroup, cqe: linux.io_uring_cqe) ![]u8 {
1712	const buffer_id = try cqe.buffer_id();
1713	const used_len = @as(usize, @intCast(cqe.res));
1714	return self.get_by_id(buffer_id)[0..used_len];
1715	}
1716
1717	// Release buffer from CQE to the kernel.
1718	pub fn put(self: *BufferGroup, cqe: linux.io_uring_cqe) !void {
1719	const buffer_id = try cqe.buffer_id();
1720	if (cqe.flags & linux.IORING_CQE_F_BUF_MORE == linux.IORING_CQE_F_BUF_MORE) {
1721	// Incremental consumption active, kernel will write to the this buffer again
1722	const used_len = @as(u32, @intCast(cqe.res));
1723	// Track what part of the buffer is used
1724	self.heads[buffer_id] += used_len;
1725	return;
1726	}
1727	self.heads[buffer_id] = 0;
1728
1729	// Release buffer to the kernel. const mask = buf_ring_mask(self.buffers_count);
1730	const mask = buf_ring_mask(self.buffers_count);
1731	buf_ring_add(self.br, self.get_by_id(buffer_id), buffer_id, mask, 0);
1732	buf_ring_advance(self.br, 1);
1733	}
1734	};
1735
1736	/// Registers a shared buffer ring to be used with provided buffers.
1737	/// `entries` number of `io_uring_buf` structures is mem mapped and shared by kernel.
1738	/// `fd` is IO_Uring.fd for which the provided buffer ring is being registered.
1739	/// `entries` is the number of entries requested in the buffer ring, must be power of 2.
1740	/// `group_id` is the chosen buffer group ID, unique in IO_Uring.
1741	pub fn setup_buf_ring(
1742	fd: linux.fd_t,
1743	entries: u16,
1744	group_id: u16,
1745	flags: linux.io_uring_buf_reg.Flags,
1746	) !*align(page_size_min) linux.io_uring_buf_ring {
1747	if (entries == 0 or entries > 1 << 15) return error.EntriesNotInRange;
1748	if (!std.math.isPowerOfTwo(entries)) return error.EntriesNotPowerOfTwo;
1749
1750	const mmap_size = @as(usize, entries) * @sizeOf(linux.io_uring_buf);
1751	const mmap = try posix.mmap(
1752	null,
1753	mmap_size,
1754	.{ .READ = true, .WRITE = true },
1755	.{ .TYPE = .PRIVATE, .ANONYMOUS = true },
1756	-1,
1757	0,
1758	);
1759	errdefer posix.munmap(mmap);
1760	assert(mmap.len == mmap_size);
1761
1762	const br: *align(page_size_min) linux.io_uring_buf_ring = @ptrCast(mmap.ptr);
1763	try register_buf_ring(fd, @intFromPtr(br), entries, group_id, flags);
1764	return br;
1765	}
1766
1767	fn register_buf_ring(
1768	fd: linux.fd_t,
1769	addr: u64,
1770	entries: u32,
1771	group_id: u16,
1772	flags: linux.io_uring_buf_reg.Flags,
1773	) !void {
1774	var reg = std.mem.zeroInit(linux.io_uring_buf_reg, .{
1775	.ring_addr = addr,
1776	.ring_entries = entries,
1777	.bgid = group_id,
1778	.flags = flags,
1779	});
1780	var res = linux.io_uring_register(fd, .REGISTER_PBUF_RING, @as(*const anyopaque, @ptrCast(&reg)), 1);
1781	if (linux.errno(res) == .INVAL and reg.flags.inc) {
1782	// Retry without incremental buffer consumption.
1783	// It is available since kernel 6.12. returns INVAL on older.
1784	reg.flags.inc = false;
1785	res = linux.io_uring_register(fd, .REGISTER_PBUF_RING, @as(*const anyopaque, @ptrCast(&reg)), 1);
1786	}
1787	try handle_register_buf_ring_result(res);
1788	}
1789
1790	fn unregister_buf_ring(fd: linux.fd_t, group_id: u16) !void {
1791	var reg = std.mem.zeroInit(linux.io_uring_buf_reg, .{
1792	.bgid = group_id,
1793	});
1794	const res = linux.io_uring_register(
1795	fd,
1796	.UNREGISTER_PBUF_RING,
1797	@as(*const anyopaque, @ptrCast(&reg)),
1798	1,
1799	);
1800	try handle_register_buf_ring_result(res);
1801	}
1802
1803	fn handle_register_buf_ring_result(res: usize) !void {
1804	switch (linux.errno(res)) {
1805	.SUCCESS => {},
1806	.INVAL => return error.ArgumentsInvalid,
1807	else => |errno| return posix.unexpectedErrno(errno),
1808	}
1809	}
1810
1811	// Unregisters a previously registered shared buffer ring, returned from io_uring_setup_buf_ring.
1812	pub fn free_buf_ring(fd: linux.fd_t, br: *align(page_size_min) linux.io_uring_buf_ring, entries: u32, group_id: u16) void {
1813	unregister_buf_ring(fd, group_id) catch {};
1814	var mmap: []align(page_size_min) u8 = undefined;
1815	mmap.ptr = @ptrCast(br);
1816	mmap.len = entries * @sizeOf(linux.io_uring_buf);
1817	posix.munmap(mmap);
1818	}
1819
1820	/// Initialises `br` so that it is ready to be used.
1821	pub fn buf_ring_init(br: *linux.io_uring_buf_ring) void {
1822	br.tail = 0;
1823	}
1824
1825	/// Calculates the appropriate size mask for a buffer ring.
1826	/// `entries` is the ring entries as specified in io_uring_register_buf_ring.
1827	pub fn buf_ring_mask(entries: u16) u16 {
1828	return entries - 1;
1829	}
1830
1831	/// Assigns `buffer` with the `br` buffer ring.
1832	/// `buffer_id` is identifier which will be returned in the CQE.
1833	/// `buffer_offset` is the offset to insert at from the current tail.
1834	/// If just one buffer is provided before the ring tail is committed with advance then offset should be 0.
1835	/// If buffers are provided in a loop before being committed, the offset must be incremented by one for each buffer added.
1836	pub fn buf_ring_add(
1837	br: *linux.io_uring_buf_ring,
1838	buffer: []u8,
1839	buffer_id: u16,
1840	mask: u16,
1841	buffer_offset: u16,
1842	) void {
1843	const bufs: [*]linux.io_uring_buf = @ptrCast(br);
1844	const buf: *linux.io_uring_buf = &bufs[(br.tail +% buffer_offset) & mask];
1845
1846	buf.addr = @intFromPtr(buffer.ptr);
1847	buf.len = @intCast(buffer.len);
1848	buf.bid = buffer_id;
1849	}
1850
1851	/// Make `count` new buffers visible to the kernel. Called after
1852	/// `io_uring_buf_ring_add` has been called `count` times to fill in new buffers.
1853	pub fn buf_ring_advance(br: *linux.io_uring_buf_ring, count: u16) void {
1854	const tail: u16 = br.tail +% count;
1855	@atomicStore(u16, &br.tail, tail, .release);
1856	}
1857
1858	test BufferGroup {
1859	if (builtin.target.cpu.arch.isPowerPC()) return; // https://codeberg.org/ziglang/zig/issues/31562
1860	if (!is_linux) return error.SkipZigTest;
1861
1862	const io = testing.io;
1863	_ = io;
1864
1865	// Init IoUring
1866	var ring = IoUring.init(16, 0) catch |err| switch (err) {
1867	error.SystemOutdated => return error.SkipZigTest,
1868	error.PermissionDenied => return error.SkipZigTest,
1869	else => return err,
1870	};
1871	defer ring.deinit();
1872
1873	// Init buffer group for ring
1874	const group_id: u16 = 1; // buffers group id
1875	const buffers_count: u16 = 1; // number of buffers in buffer group
1876	const buffer_size: usize = 128; // size of each buffer in group
1877	var buf_grp = BufferGroup.init(
1878	&ring,
1879	testing.allocator,
1880	group_id,
1881	buffer_size,
1882	buffers_count,
1883	) catch |err| switch (err) {
1884	// kernel older than 5.19
1885	error.ArgumentsInvalid => return error.SkipZigTest,
1886	else => return err,
1887	};
1888	defer buf_grp.deinit(testing.allocator);
1889
1890	// Create client/server fds
1891	const fds = try createSocketTestHarness(&ring);
1892	defer fds.close();
1893	const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
1894
1895	// Client sends data
1896	{
1897	_ = try ring.send(1, fds.client, data[0..], 0);
1898	const submitted = try ring.submit();
1899	try testing.expectEqual(1, submitted);
1900	const cqe_send = try ring.copy_cqe();
1901	if (cqe_send.err() == .INVAL) return error.SkipZigTest;
1902	try testing.expectEqual(linux.io_uring_cqe{ .user_data = 1, .res = data.len, .flags = 0 }, cqe_send);
1903	}
1904
1905	// Server uses buffer group receive
1906	{
1907	// Submit recv operation, buffer will be chosen from buffer group
1908	_ = try buf_grp.recv(2, fds.server, 0);
1909	const submitted = try ring.submit();
1910	try testing.expectEqual(1, submitted);
1911
1912	// ... when we have completion for recv operation
1913	const cqe = try ring.copy_cqe();
1914	try testing.expectEqual(2, cqe.user_data); // matches submitted user_data
1915	try testing.expect(cqe.res >= 0); // success
1916	try testing.expectEqual(posix.E.SUCCESS, cqe.err());
1917	try testing.expectEqual(data.len, @as(usize, @intCast(cqe.res))); // cqe.res holds received data len
1918
1919	// Get buffer from pool
1920	const buf = try buf_grp.get(cqe);
1921	try testing.expectEqualSlices(u8, &data, buf);
1922	// Release buffer to the kernel when application is done with it
1923	try buf_grp.put(cqe);
1924	}
1925	}
1926
1927	test {
1928	if (builtin.target.cpu.arch.isPowerPC()) return; // https://codeberg.org/ziglang/zig/issues/31562
1929	if (is_linux) _ = @import("IoUring/test.zig");
1930	}