Commit 265f42d472fa110c1e211e5f82fdf540ee198aa8 | zig

author	Matthew Lugg <mlugg@mlugg.co.uk> 2024-03-07 22:48:20+00:00
committer	Matthew Lugg <mlugg@mlugg.co.uk> 2024-03-08 08:02:45+00:00
log	265f42d472fa110c1e211e5f82fdf540ee198aa8
tree	5fa4bccaaf36b8e76fcd423eb0fd35f35d53b82c
parent	a7f6e738122e7f1a5d2e502ee121700883fa3818
signature	Commit is signed but in an unrecognized format.

std.os.linux: rework io_uring support

* `linux.IO_Uring` -> `linux.IoUring` to align with naming conventions.
* All functions `io_uring_prep_foo` are now methods `prep_foo` on `io_uring_sqe`, which is in a file of its own.
* `SubmissionQueue` and `CompletionQueue` are namespaced under `IoUring`.

This is a breaking change.

The new file and namespace layouts are more idiomatic, and allow us to
eliminate one more usage of `usingnamespace` from the standard library.
2 remain.

6 files changed, 4258 insertions(+), 4247 deletions(-)

CMakeLists.txt
lib
- std
  - os
    linux.zig
    linux
    IoUring.zig
    io_uring.zig
    io_uring_sqe.zig
    test.zig

CMakeLists.txt+2-1

...	...	@@ -291,7 +291,8 @@ set(ZIG_STAGE2_SOURCES
291	291	"${CMAKE_SOURCE_DIR}/lib/std/os/linux/errno/generic.zig"
292	292	"${CMAKE_SOURCE_DIR}/lib/std/os/linux/x86_64.zig"
293	293	"${CMAKE_SOURCE_DIR}/lib/std/os/linux.zig"
294		"${CMAKE_SOURCE_DIR}/lib/std/os/linux/io_uring.zig"
	294	"${CMAKE_SOURCE_DIR}/lib/std/os/linux/IoUring.zig"
	295	"${CMAKE_SOURCE_DIR}/lib/std/os/linux/io_uring_sqe.zig"
295	296	"${CMAKE_SOURCE_DIR}/lib/std/os/linux/x86_64.zig"
296	297	"${CMAKE_SOURCE_DIR}/lib/std/os/windows.zig"
297	298	"${CMAKE_SOURCE_DIR}/lib/std/os/windows/ntstatus.zig"

lib/std/os/linux.zig+3-18

...	...	@@ -383,8 +383,6 @@ pub const O = switch (native_arch) {
383	383	else => @compileError("missing std.os.linux.O constants for this architecture"),
384	384	};
385	385
386		pub usingnamespace @import("linux/io_uring.zig");
387
388	386	/// Set by startup code, used by `getauxval`.
389	387	pub var elf_aux_maybe: ?[*]std.elf.Auxv = null;
390	388
...	...	@@ -4188,22 +4186,9 @@ pub const IORING_SETUP_SINGLE_ISSUER = 1 << 12;
4188	4186	pub const IORING_SETUP_DEFER_TASKRUN = 1 << 13;
4189	4187
4190	4188	/// IO submission data structure (Submission Queue Entry)
4191		pub const io_uring_sqe = extern struct {
4192		opcode: IORING_OP,
4193		flags: u8,
4194		ioprio: u16,
4195		fd: i32,
4196		off: u64,
4197		addr: u64,
4198		len: u32,
4199		rw_flags: u32,
4200		user_data: u64,
4201		buf_index: u16,
4202		personality: u16,
4203		splice_fd_in: i32,
4204		addr3: u64,
4205		resv: u64,
4206		};
	4189	pub const io_uring_sqe = @import("linux/io_uring_sqe.zig").io_uring_sqe;
	4190
	4191	pub const IoUring = @import("linux/IoUring.zig");
4207	4192
4208	4193	/// If sqe->file_index is set to this for opcodes that instantiate a new
4209	4194	/// direct descriptor (like openat/openat2/accept), then io_uring will allocate

lib/std/os/linux/IoUring.zig created+3670

...	...	@@ -0,0 +1,3670 @@
	1	const IoUring = @This();
	2	const std = @import("../../std.zig");
	3	const builtin = @import("builtin");
	4	const assert = std.debug.assert;
	5	const mem = std.mem;
	6	const net = std.net;
	7	const os = std.os;
	8	const posix = std.posix;
	9	const linux = os.linux;
	10	const testing = std.testing;
	11
	12	fd: os.fd_t = -1,
	13	sq: SubmissionQueue,
	14	cq: CompletionQueue,
	15	flags: u32,
	16	features: u32,
	17
	18	/// A friendly way to setup an io_uring, with default linux.io_uring_params.
	19	/// `entries` must be a power of two between 1 and 32768, although the kernel will make the final
	20	/// call on how many entries the submission and completion queues will ultimately have,
	21	/// see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L8027-L8050.
	22	/// Matches the interface of io_uring_queue_init() in liburing.
	23	pub fn init(entries: u16, flags: u32) !IoUring {
	24	var params = mem.zeroInit(linux.io_uring_params, .{
	25	.flags = flags,
	26	.sq_thread_idle = 1000,
	27	});
	28	return try IoUring.init_params(entries, &params);
	29	}
	30
	31	/// A powerful way to setup an io_uring, if you want to tweak linux.io_uring_params such as submission
	32	/// queue thread cpu affinity or thread idle timeout (the kernel and our default is 1 second).
	33	/// `params` is passed by reference because the kernel needs to modify the parameters.
	34	/// Matches the interface of io_uring_queue_init_params() in liburing.
	35	pub fn init_params(entries: u16, p: *linux.io_uring_params) !IoUring {
	36	if (entries == 0) return error.EntriesZero;
	37	if (!std.math.isPowerOfTwo(entries)) return error.EntriesNotPowerOfTwo;
	38
	39	assert(p.sq_entries == 0);
	40	assert(p.cq_entries == 0 or p.flags & linux.IORING_SETUP_CQSIZE != 0);
	41	assert(p.features == 0);
	42	assert(p.wq_fd == 0 or p.flags & linux.IORING_SETUP_ATTACH_WQ != 0);
	43	assert(p.resv[0] == 0);
	44	assert(p.resv[1] == 0);
	45	assert(p.resv[2] == 0);
	46
	47	const res = linux.io_uring_setup(entries, p);
	48	switch (linux.getErrno(res)) {
	49	.SUCCESS => {},
	50	.FAULT => return error.ParamsOutsideAccessibleAddressSpace,
	51	// The resv array contains non-zero data, p.flags contains an unsupported flag,
	52	// entries out of bounds, IORING_SETUP_SQ_AFF was specified without IORING_SETUP_SQPOLL,
	53	// or IORING_SETUP_CQSIZE was specified but linux.io_uring_params.cq_entries was invalid:
	54	.INVAL => return error.ArgumentsInvalid,
	55	.MFILE => return error.ProcessFdQuotaExceeded,
	56	.NFILE => return error.SystemFdQuotaExceeded,
	57	.NOMEM => return error.SystemResources,
	58	// IORING_SETUP_SQPOLL was specified but effective user ID lacks sufficient privileges,
	59	// or a container seccomp policy prohibits io_uring syscalls:
	60	.PERM => return error.PermissionDenied,
	61	.NOSYS => return error.SystemOutdated,
	62	else => \|errno\| return os.unexpectedErrno(errno),
	63	}
	64	const fd = @as(os.fd_t, @intCast(res));
	65	assert(fd >= 0);
	66	errdefer os.close(fd);
	67
	68	// Kernel versions 5.4 and up use only one mmap() for the submission and completion queues.
	69	// This is not an optional feature for us... if the kernel does it, we have to do it.
	70	// The thinking on this by the kernel developers was that both the submission and the
	71	// completion queue rings have sizes just over a power of two, but the submission queue ring
	72	// is significantly smaller with u32 slots. By bundling both in a single mmap, the kernel
	73	// gets the submission queue ring for free.
	74	// See https://patchwork.kernel.org/patch/11115257 for the kernel patch.
	75	// We do not support the double mmap() done before 5.4, because we want to keep the
	76	// init/deinit mmap paths simple and because io_uring has had many bug fixes even since 5.4.
	77	if ((p.features & linux.IORING_FEAT_SINGLE_MMAP) == 0) {
	78	return error.SystemOutdated;
	79	}
	80
	81	// Check that the kernel has actually set params and that "impossible is nothing".
	82	assert(p.sq_entries != 0);
	83	assert(p.cq_entries != 0);
	84	assert(p.cq_entries >= p.sq_entries);
	85
	86	// From here on, we only need to read from params, so pass `p` by value as immutable.
	87	// The completion queue shares the mmap with the submission queue, so pass `sq` there too.
	88	var sq = try SubmissionQueue.init(fd, p.*);
	89	errdefer sq.deinit();
	90	var cq = try CompletionQueue.init(fd, p.*, sq);
	91	errdefer cq.deinit();
	92
	93	// Check that our starting state is as we expect.
	94	assert(sq.head.* == 0);
	95	assert(sq.tail.* == 0);
	96	assert(sq.mask == p.sq_entries - 1);
	97	// Allow flags.* to be non-zero, since the kernel may set IORING_SQ_NEED_WAKEUP at any time.
	98	assert(sq.dropped.* == 0);
	99	assert(sq.array.len == p.sq_entries);
	100	assert(sq.sqes.len == p.sq_entries);
	101	assert(sq.sqe_head == 0);
	102	assert(sq.sqe_tail == 0);
	103
	104	assert(cq.head.* == 0);
	105	assert(cq.tail.* == 0);
	106	assert(cq.mask == p.cq_entries - 1);
	107	assert(cq.overflow.* == 0);
	108	assert(cq.cqes.len == p.cq_entries);
	109
	110	return IoUring{
	111	.fd = fd,
	112	.sq = sq,
	113	.cq = cq,
	114	.flags = p.flags,
	115	.features = p.features,
	116	};
	117	}
	118
	119	pub fn deinit(self: *IoUring) void {
	120	assert(self.fd >= 0);
	121	// The mmaps depend on the fd, so the order of these calls is important:
	122	self.cq.deinit();
	123	self.sq.deinit();
	124	os.close(self.fd);
	125	self.fd = -1;
	126	}
	127
	128	/// Returns a pointer to a vacant SQE, or an error if the submission queue is full.
	129	/// We follow the implementation (and atomics) of liburing's `io_uring_get_sqe()` exactly.
	130	/// However, instead of a null we return an error to force safe handling.
	131	/// Any situation where the submission queue is full tends more towards a control flow error,
	132	/// and the null return in liburing is more a C idiom than anything else, for lack of a better
	133	/// alternative. In Zig, we have first-class error handling... so let's use it.
	134	/// Matches the implementation of io_uring_get_sqe() in liburing.
	135	pub fn get_sqe(self: IoUring) !linux.io_uring_sqe {
	136	const head = @atomicLoad(u32, self.sq.head, .Acquire);
	137	// Remember that these head and tail offsets wrap around every four billion operations.
	138	// We must therefore use wrapping addition and subtraction to avoid a runtime crash.
	139	const next = self.sq.sqe_tail +% 1;
	140	if (next -% head > self.sq.sqes.len) return error.SubmissionQueueFull;
	141	const sqe = &self.sq.sqes[self.sq.sqe_tail & self.sq.mask];
	142	self.sq.sqe_tail = next;
	143	return sqe;
	144	}
	145
	146	/// Submits the SQEs acquired via get_sqe() to the kernel. You can call this once after you have
	147	/// called get_sqe() multiple times to setup multiple I/O requests.
	148	/// Returns the number of SQEs submitted, if not used alongside IORING_SETUP_SQPOLL.
	149	/// If the io_uring instance is uses IORING_SETUP_SQPOLL, the value returned on success is not
	150	/// guaranteed to match the amount of actually submitted sqes during this call. A value higher
	151	/// or lower, including 0, may be returned.
	152	/// Matches the implementation of io_uring_submit() in liburing.
	153	pub fn submit(self: *IoUring) !u32 {
	154	return self.submit_and_wait(0);
	155	}
	156
	157	/// Like submit(), but allows waiting for events as well.
	158	/// Returns the number of SQEs submitted.
	159	/// Matches the implementation of io_uring_submit_and_wait() in liburing.
	160	pub fn submit_and_wait(self: *IoUring, wait_nr: u32) !u32 {
	161	const submitted = self.flush_sq();
	162	var flags: u32 = 0;
	163	if (self.sq_ring_needs_enter(&flags) or wait_nr > 0) {
	164	if (wait_nr > 0 or (self.flags & linux.IORING_SETUP_IOPOLL) != 0) {
	165	flags \|= linux.IORING_ENTER_GETEVENTS;
	166	}
	167	return try self.enter(submitted, wait_nr, flags);
	168	}
	169	return submitted;
	170	}
	171
	172	/// Tell the kernel we have submitted SQEs and/or want to wait for CQEs.
	173	/// Returns the number of SQEs submitted.
	174	pub fn enter(self: *IoUring, to_submit: u32, min_complete: u32, flags: u32) !u32 {
	175	assert(self.fd >= 0);
	176	const res = linux.io_uring_enter(self.fd, to_submit, min_complete, flags, null);
	177	switch (linux.getErrno(res)) {
	178	.SUCCESS => {},
	179	// The kernel was unable to allocate memory or ran out of resources for the request.
	180	// The application should wait for some completions and try again:
	181	.AGAIN => return error.SystemResources,
	182	// The SQE `fd` is invalid, or IOSQE_FIXED_FILE was set but no files were registered:
	183	.BADF => return error.FileDescriptorInvalid,
	184	// The file descriptor is valid, but the ring is not in the right state.
	185	// See io_uring_register(2) for how to enable the ring.
	186	.BADFD => return error.FileDescriptorInBadState,
	187	// The application attempted to overcommit the number of requests it can have pending.
	188	// The application should wait for some completions and try again:
	189	.BUSY => return error.CompletionQueueOvercommitted,
	190	// The SQE is invalid, or valid but the ring was setup with IORING_SETUP_IOPOLL:
	191	.INVAL => return error.SubmissionQueueEntryInvalid,
	192	// The buffer is outside the process' accessible address space, or IORING_OP_READ_FIXED
	193	// or IORING_OP_WRITE_FIXED was specified but no buffers were registered, or the range
	194	// described by `addr` and `len` is not within the buffer registered at `buf_index`:
	195	.FAULT => return error.BufferInvalid,
	196	.NXIO => return error.RingShuttingDown,
	197	// The kernel believes our `self.fd` does not refer to an io_uring instance,
	198	// or the opcode is valid but not supported by this kernel (more likely):
	199	.OPNOTSUPP => return error.OpcodeNotSupported,
	200	// The operation was interrupted by a delivery of a signal before it could complete.
	201	// This can happen while waiting for events with IORING_ENTER_GETEVENTS:
	202	.INTR => return error.SignalInterrupt,
	203	else => \|errno\| return os.unexpectedErrno(errno),
	204	}
	205	return @as(u32, @intCast(res));
	206	}
	207
	208	/// Sync internal state with kernel ring state on the SQ side.
	209	/// Returns the number of all pending events in the SQ ring, for the shared ring.
	210	/// This return value includes previously flushed SQEs, as per liburing.
	211	/// The rationale is to suggest that an io_uring_enter() call is needed rather than not.
	212	/// Matches the implementation of __io_uring_flush_sq() in liburing.
	213	pub fn flush_sq(self: *IoUring) u32 {
	214	if (self.sq.sqe_head != self.sq.sqe_tail) {
	215	// Fill in SQEs that we have queued up, adding them to the kernel ring.
	216	const to_submit = self.sq.sqe_tail -% self.sq.sqe_head;
	217	var tail = self.sq.tail.*;
	218	var i: usize = 0;
	219	while (i < to_submit) : (i += 1) {
	220	self.sq.array[tail & self.sq.mask] = self.sq.sqe_head & self.sq.mask;
	221	tail +%= 1;
	222	self.sq.sqe_head +%= 1;
	223	}
	224	// Ensure that the kernel can actually see the SQE updates when it sees the tail update.
	225	@atomicStore(u32, self.sq.tail, tail, .Release);
	226	}
	227	return self.sq_ready();
	228	}
	229
	230	/// Returns true if we are not using an SQ thread (thus nobody submits but us),
	231	/// or if IORING_SQ_NEED_WAKEUP is set and the SQ thread must be explicitly awakened.
	232	/// For the latter case, we set the SQ thread wakeup flag.
	233	/// Matches the implementation of sq_ring_needs_enter() in liburing.
	234	pub fn sq_ring_needs_enter(self: IoUring, flags: u32) bool {
	235	assert(flags.* == 0);
	236	if ((self.flags & linux.IORING_SETUP_SQPOLL) == 0) return true;
	237	if ((@atomicLoad(u32, self.sq.flags, .Unordered) & linux.IORING_SQ_NEED_WAKEUP) != 0) {
	238	flags.* \|= linux.IORING_ENTER_SQ_WAKEUP;
	239	return true;
	240	}
	241	return false;
	242	}
	243
	244	/// Returns the number of flushed and unflushed SQEs pending in the submission queue.
	245	/// In other words, this is the number of SQEs in the submission queue, i.e. its length.
	246	/// These are SQEs that the kernel is yet to consume.
	247	/// Matches the implementation of io_uring_sq_ready in liburing.
	248	pub fn sq_ready(self: *IoUring) u32 {
	249	// Always use the shared ring state (i.e. head and not sqe_head) to avoid going out of sync,
	250	// see https://github.com/axboe/liburing/issues/92.
	251	return self.sq.sqe_tail -% @atomicLoad(u32, self.sq.head, .Acquire);
	252	}
	253
	254	/// Returns the number of CQEs in the completion queue, i.e. its length.
	255	/// These are CQEs that the application is yet to consume.
	256	/// Matches the implementation of io_uring_cq_ready in liburing.
	257	pub fn cq_ready(self: *IoUring) u32 {
	258	return @atomicLoad(u32, self.cq.tail, .Acquire) -% self.cq.head.*;
	259	}
	260
	261	/// Copies as many CQEs as are ready, and that can fit into the destination `cqes` slice.
	262	/// If none are available, enters into the kernel to wait for at most `wait_nr` CQEs.
	263	/// Returns the number of CQEs copied, advancing the CQ ring.
	264	/// Provides all the wait/peek methods found in liburing, but with batching and a single method.
	265	/// The rationale for copying CQEs rather than copying pointers is that pointers are 8 bytes
	266	/// whereas CQEs are not much more at only 16 bytes, and this provides a safer faster interface.
	267	/// Safer, because you no longer need to call cqe_seen(), avoiding idempotency bugs.
	268	/// Faster, because we can now amortize the atomic store release to `cq.head` across the batch.
	269	/// See https://github.com/axboe/liburing/issues/103#issuecomment-686665007.
	270	/// Matches the implementation of io_uring_peek_batch_cqe() in liburing, but supports waiting.
	271	pub fn copy_cqes(self: *IoUring, cqes: []linux.io_uring_cqe, wait_nr: u32) !u32 {
	272	const count = self.copy_cqes_ready(cqes);
	273	if (count > 0) return count;
	274	if (self.cq_ring_needs_flush() or wait_nr > 0) {
	275	_ = try self.enter(0, wait_nr, linux.IORING_ENTER_GETEVENTS);
	276	return self.copy_cqes_ready(cqes);
	277	}
	278	return 0;
	279	}
	280
	281	fn copy_cqes_ready(self: *IoUring, cqes: []linux.io_uring_cqe) u32 {
	282	const ready = self.cq_ready();
	283	const count = @min(cqes.len, ready);
	284	const head = self.cq.head.* & self.cq.mask;
	285	const tail = (self.cq.head.* +% count) & self.cq.mask;
	286
	287	if (head <= tail) {
	288	// head behind tail -> no wrapping
	289	@memcpy(cqes[0..count], self.cq.cqes[head..tail]);
	290	} else {
	291	// head in front of tail -> buffer wraps
	292	const two_copies_required: bool = self.cq.cqes.len - head < count;
	293	const amount_to_copy_in_first = if (two_copies_required) self.cq.cqes.len - head else count;
	294	@memcpy(cqes[0..amount_to_copy_in_first], self.cq.cqes[head .. head + amount_to_copy_in_first]);
	295	if (two_copies_required) {
	296	@memcpy(cqes[amount_to_copy_in_first..count], self.cq.cqes[0..tail]);
	297	}
	298	}
	299
	300	self.cq_advance(count);
	301	return count;
	302	}
	303
	304	/// Returns a copy of an I/O completion, waiting for it if necessary, and advancing the CQ ring.
	305	/// A convenience method for `copy_cqes()` for when you don't need to batch or peek.
	306	pub fn copy_cqe(ring: *IoUring) !linux.io_uring_cqe {
	307	var cqes: [1]linux.io_uring_cqe = undefined;
	308	while (true) {
	309	const count = try ring.copy_cqes(&cqes, 1);
	310	if (count > 0) return cqes[0];
	311	}
	312	}
	313
	314	/// Matches the implementation of cq_ring_needs_flush() in liburing.
	315	pub fn cq_ring_needs_flush(self: *IoUring) bool {
	316	return (@atomicLoad(u32, self.sq.flags, .Unordered) & linux.IORING_SQ_CQ_OVERFLOW) != 0;
	317	}
	318
	319	/// For advanced use cases only that implement custom completion queue methods.
	320	/// If you use copy_cqes() or copy_cqe() you must not call cqe_seen() or cq_advance().
	321	/// Must be called exactly once after a zero-copy CQE has been processed by your application.
	322	/// Not idempotent, calling more than once will result in other CQEs being lost.
	323	/// Matches the implementation of cqe_seen() in liburing.
	324	pub fn cqe_seen(self: IoUring, cqe: linux.io_uring_cqe) void {
	325	_ = cqe;
	326	self.cq_advance(1);
	327	}
	328
	329	/// For advanced use cases only that implement custom completion queue methods.
	330	/// Matches the implementation of cq_advance() in liburing.
	331	pub fn cq_advance(self: *IoUring, count: u32) void {
	332	if (count > 0) {
	333	// Ensure the kernel only sees the new head value after the CQEs have been read.
	334	@atomicStore(u32, self.cq.head, self.cq.head.* +% count, .Release);
	335	}
	336	}
	337
	338	/// Queues (but does not submit) an SQE to perform an `fsync(2)`.
	339	/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
	340	/// For example, for `fdatasync()` you can set `IORING_FSYNC_DATASYNC` in the SQE's `rw_flags`.
	341	/// N.B. While SQEs are initiated in the order in which they appear in the submission queue,
	342	/// operations execute in parallel and completions are unordered. Therefore, an application that
	343	/// submits a write followed by an fsync in the submission queue cannot expect the fsync to
	344	/// apply to the write, since the fsync may complete before the write is issued to the disk.
	345	/// You should preferably use `link_with_next_sqe()` on a write's SQE to link it with an fsync,
	346	/// or else insert a full write barrier using `drain_previous_sqes()` when queueing an fsync.
	347	pub fn fsync(self: IoUring, user_data: u64, fd: os.fd_t, flags: u32) !linux.io_uring_sqe {
	348	const sqe = try self.get_sqe();
	349	sqe.prep_fsync(fd, flags);
	350	sqe.user_data = user_data;
	351	return sqe;
	352	}
	353
	354	/// Queues (but does not submit) an SQE to perform a no-op.
	355	/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
	356	/// A no-op is more useful than may appear at first glance.
	357	/// For example, you could call `drain_previous_sqes()` on the returned SQE, to use the no-op to
	358	/// know when the ring is idle before acting on a kill signal.
	359	pub fn nop(self: IoUring, user_data: u64) !linux.io_uring_sqe {
	360	const sqe = try self.get_sqe();
	361	sqe.prep_nop();
	362	sqe.user_data = user_data;
	363	return sqe;
	364	}
	365
	366	/// Used to select how the read should be handled.
	367	pub const ReadBuffer = union(enum) {
	368	/// io_uring will read directly into this buffer
	369	buffer: []u8,
	370
	371	/// io_uring will read directly into these buffers using readv.
	372	iovecs: []const os.iovec,
	373
	374	/// io_uring will select a buffer that has previously been provided with `provide_buffers`.
	375	/// The buffer group reference by `group_id` must contain at least one buffer for the read to work.
	376	/// `len` controls the number of bytes to read into the selected buffer.
	377	buffer_selection: struct {
	378	group_id: u16,
	379	len: usize,
	380	},
	381	};
	382
	383	/// Queues (but does not submit) an SQE to perform a `read(2)` or `preadv(2)` depending on the buffer type.
	384	/// * Reading into a `ReadBuffer.buffer` uses `read(2)`
	385	/// * Reading into a `ReadBuffer.iovecs` uses `preadv(2)`
	386	/// If you want to do a `preadv2(2)` then set `rw_flags` on the returned SQE. See https://man7.org/linux/man-pages/man2/preadv2.2.html
	387	///
	388	/// Returns a pointer to the SQE.
	389	pub fn read(
	390	self: *IoUring,
	391	user_data: u64,
	392	fd: os.fd_t,
	393	buffer: ReadBuffer,
	394	offset: u64,
	395	) !*linux.io_uring_sqe {
	396	const sqe = try self.get_sqe();
	397	switch (buffer) {
	398	.buffer => \|slice\| sqe.prep_read(fd, slice, offset),
	399	.iovecs => \|vecs\| sqe.prep_readv(fd, vecs, offset),
	400	.buffer_selection => \|selection\| {
	401	sqe.prep_rw(.READ, fd, 0, selection.len, offset);
	402	sqe.flags \|= linux.IOSQE_BUFFER_SELECT;
	403	sqe.buf_index = selection.group_id;
	404	},
	405	}
	406	sqe.user_data = user_data;
	407	return sqe;
	408	}
	409
	410	/// Queues (but does not submit) an SQE to perform a `write(2)`.
	411	/// Returns a pointer to the SQE.
	412	pub fn write(
	413	self: *IoUring,
	414	user_data: u64,
	415	fd: os.fd_t,
	416	buffer: []const u8,
	417	offset: u64,
	418	) !*linux.io_uring_sqe {
	419	const sqe = try self.get_sqe();
	420	sqe.prep_write(fd, buffer, offset);
	421	sqe.user_data = user_data;
	422	return sqe;
	423	}
	424
	425	/// Queues (but does not submit) an SQE to perform a `splice(2)`
	426	/// Either `fd_in` or `fd_out` must be a pipe.
	427	/// If `fd_in` refers to a pipe, `off_in` is ignored and must be set to std.math.maxInt(u64).
	428	/// If `fd_in` does not refer to a pipe and `off_in` is maxInt(u64), then `len` are read
	429	/// from `fd_in` starting from the file offset, which is incremented by the number of bytes read.
	430	/// If `fd_in` does not refer to a pipe and `off_in` is not maxInt(u64), then the starting offset of `fd_in` will be `off_in`.
	431	/// This splice operation can be used to implement sendfile by splicing to an intermediate pipe first,
	432	/// then splice to the final destination. In fact, the implementation of sendfile in kernel uses splice internally.
	433	///
	434	/// NOTE that even if fd_in or fd_out refers to a pipe, the splice operation can still fail with EINVAL if one of the
	435	/// fd doesn't explicitly support splice peration, e.g. reading from terminal is unsupported from kernel 5.7 to 5.11.
	436	/// See https://github.com/axboe/liburing/issues/291
	437	///
	438	/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
	439	pub fn splice(self: IoUring, user_data: u64, fd_in: os.fd_t, off_in: u64, fd_out: os.fd_t, off_out: u64, len: usize) !linux.io_uring_sqe {
	440	const sqe = try self.get_sqe();
	441	sqe.prep_splice(fd_in, off_in, fd_out, off_out, len);
	442	sqe.user_data = user_data;
	443	return sqe;
	444	}
	445
	446	/// Queues (but does not submit) an SQE to perform a IORING_OP_READ_FIXED.
	447	/// The `buffer` provided must be registered with the kernel by calling `register_buffers` first.
	448	/// The `buffer_index` must be the same as its index in the array provided to `register_buffers`.
	449	///
	450	/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
	451	pub fn read_fixed(
	452	self: *IoUring,
	453	user_data: u64,
	454	fd: os.fd_t,
	455	buffer: *os.iovec,
	456	offset: u64,
	457	buffer_index: u16,
	458	) !*linux.io_uring_sqe {
	459	const sqe = try self.get_sqe();
	460	sqe.prep_read_fixed(fd, buffer, offset, buffer_index);
	461	sqe.user_data = user_data;
	462	return sqe;
	463	}
	464
	465	/// Queues (but does not submit) an SQE to perform a `pwritev()`.
	466	/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
	467	/// For example, if you want to do a `pwritev2()` then set `rw_flags` on the returned SQE.
	468	/// See https://linux.die.net/man/2/pwritev.
	469	pub fn writev(
	470	self: *IoUring,
	471	user_data: u64,
	472	fd: os.fd_t,
	473	iovecs: []const os.iovec_const,
	474	offset: u64,
	475	) !*linux.io_uring_sqe {
	476	const sqe = try self.get_sqe();
	477	sqe.prep_writev(fd, iovecs, offset);
	478	sqe.user_data = user_data;
	479	return sqe;
	480	}
	481
	482	/// Queues (but does not submit) an SQE to perform a IORING_OP_WRITE_FIXED.
	483	/// The `buffer` provided must be registered with the kernel by calling `register_buffers` first.
	484	/// The `buffer_index` must be the same as its index in the array provided to `register_buffers`.
	485	///
	486	/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
	487	pub fn write_fixed(
	488	self: *IoUring,
	489	user_data: u64,
	490	fd: os.fd_t,
	491	buffer: *os.iovec,
	492	offset: u64,
	493	buffer_index: u16,
	494	) !*linux.io_uring_sqe {
	495	const sqe = try self.get_sqe();
	496	sqe.prep_write_fixed(fd, buffer, offset, buffer_index);
	497	sqe.user_data = user_data;
	498	return sqe;
	499	}
	500
	501	/// Queues (but does not submit) an SQE to perform an `accept4(2)` on a socket.
	502	/// Returns a pointer to the SQE.
	503	/// Available since 5.5
	504	pub fn accept(
	505	self: *IoUring,
	506	user_data: u64,
	507	fd: os.fd_t,
	508	addr: ?*os.sockaddr,
	509	addrlen: ?*os.socklen_t,
	510	flags: u32,
	511	) !*linux.io_uring_sqe {
	512	const sqe = try self.get_sqe();
	513	sqe.prep_accept(fd, addr, addrlen, flags);
	514	sqe.user_data = user_data;
	515	return sqe;
	516	}
	517
	518	/// Queues an multishot accept on a socket.
	519	///
	520	/// Multishot variant allows an application to issue a single accept request,
	521	/// which will repeatedly trigger a CQE when a connection request comes in.
	522	/// While IORING_CQE_F_MORE flag is set in CQE flags accept will generate
	523	/// further CQEs.
	524	///
	525	/// Available since 5.19
	526	pub fn accept_multishot(
	527	self: *IoUring,
	528	user_data: u64,
	529	fd: os.fd_t,
	530	addr: ?*os.sockaddr,
	531	addrlen: ?*os.socklen_t,
	532	flags: u32,
	533	) !*linux.io_uring_sqe {
	534	const sqe = try self.get_sqe();
	535	sqe.prep_multishot_accept(fd, addr, addrlen, flags);
	536	sqe.user_data = user_data;
	537	return sqe;
	538	}
	539
	540	/// Queues an accept using direct (registered) file descriptors.
	541	///
	542	/// To use an accept direct variant, the application must first have registered
	543	/// a file table (with register_files). An unused table index will be
	544	/// dynamically chosen and returned in the CQE res field.
	545	///
	546	/// After creation, they can be used by setting IOSQE_FIXED_FILE in the SQE
	547	/// flags member, and setting the SQE fd field to the direct descriptor value
	548	/// rather than the regular file descriptor.
	549	///
	550	/// Available since 5.19
	551	pub fn accept_direct(
	552	self: *IoUring,
	553	user_data: u64,
	554	fd: os.fd_t,
	555	addr: ?*os.sockaddr,
	556	addrlen: ?*os.socklen_t,
	557	flags: u32,
	558	) !*linux.io_uring_sqe {
	559	const sqe = try self.get_sqe();
	560	sqe.prep_accept_direct(fd, addr, addrlen, flags, linux.IORING_FILE_INDEX_ALLOC);
	561	sqe.user_data = user_data;
	562	return sqe;
	563	}
	564
	565	/// Queues an multishot accept using direct (registered) file descriptors.
	566	/// Available since 5.19
	567	pub fn accept_multishot_direct(
	568	self: *IoUring,
	569	user_data: u64,
	570	fd: os.fd_t,
	571	addr: ?*os.sockaddr,
	572	addrlen: ?*os.socklen_t,
	573	flags: u32,
	574	) !*linux.io_uring_sqe {
	575	const sqe = try self.get_sqe();
	576	sqe.prep_multishot_accept_direct(fd, addr, addrlen, flags);
	577	sqe.user_data = user_data;
	578	return sqe;
	579	}
	580
	581	/// Queue (but does not submit) an SQE to perform a `connect(2)` on a socket.
	582	/// Returns a pointer to the SQE.
	583	pub fn connect(
	584	self: *IoUring,
	585	user_data: u64,
	586	fd: os.fd_t,
	587	addr: *const os.sockaddr,
	588	addrlen: os.socklen_t,
	589	) !*linux.io_uring_sqe {
	590	const sqe = try self.get_sqe();
	591	sqe.prep_connect(fd, addr, addrlen);
	592	sqe.user_data = user_data;
	593	return sqe;
	594	}
	595
	596	/// Queues (but does not submit) an SQE to perform a `epoll_ctl(2)`.
	597	/// Returns a pointer to the SQE.
	598	pub fn epoll_ctl(
	599	self: *IoUring,
	600	user_data: u64,
	601	epfd: os.fd_t,
	602	fd: os.fd_t,
	603	op: u32,
	604	ev: ?*linux.epoll_event,
	605	) !*linux.io_uring_sqe {
	606	const sqe = try self.get_sqe();
	607	sqe.prep_epoll_ctl(epfd, fd, op, ev);
	608	sqe.user_data = user_data;
	609	return sqe;
	610	}
	611
	612	/// Used to select how the recv call should be handled.
	613	pub const RecvBuffer = union(enum) {
	614	/// io_uring will recv directly into this buffer
	615	buffer: []u8,
	616
	617	/// io_uring will select a buffer that has previously been provided with `provide_buffers`.
	618	/// The buffer group referenced by `group_id` must contain at least one buffer for the recv call to work.
	619	/// `len` controls the number of bytes to read into the selected buffer.
	620	buffer_selection: struct {
	621	group_id: u16,
	622	len: usize,
	623	},
	624	};
	625
	626	/// Queues (but does not submit) an SQE to perform a `recv(2)`.
	627	/// Returns a pointer to the SQE.
	628	/// Available since 5.6
	629	pub fn recv(
	630	self: *IoUring,
	631	user_data: u64,
	632	fd: os.fd_t,
	633	buffer: RecvBuffer,
	634	flags: u32,
	635	) !*linux.io_uring_sqe {
	636	const sqe = try self.get_sqe();
	637	switch (buffer) {
	638	.buffer => \|slice\| sqe.prep_recv(fd, slice, flags),
	639	.buffer_selection => \|selection\| {
	640	sqe.prep_rw(.RECV, fd, 0, selection.len, 0);
	641	sqe.rw_flags = flags;
	642	sqe.flags \|= linux.IOSQE_BUFFER_SELECT;
	643	sqe.buf_index = selection.group_id;
	644	},
	645	}
	646	sqe.user_data = user_data;
	647	return sqe;
	648	}
	649
	650	/// Queues (but does not submit) an SQE to perform a `send(2)`.
	651	/// Returns a pointer to the SQE.
	652	/// Available since 5.6
	653	pub fn send(
	654	self: *IoUring,
	655	user_data: u64,
	656	fd: os.fd_t,
	657	buffer: []const u8,
	658	flags: u32,
	659	) !*linux.io_uring_sqe {
	660	const sqe = try self.get_sqe();
	661	sqe.prep_send(fd, buffer, flags);
	662	sqe.user_data = user_data;
	663	return sqe;
	664	}
	665
	666	/// Queues (but does not submit) an SQE to perform an async zerocopy `send(2)`.
	667	///
	668	/// This operation will most likely produce two CQEs. The flags field of the
	669	/// first cqe may likely contain IORING_CQE_F_MORE, which means that there will
	670	/// be a second cqe with the user_data field set to the same value. The user
	671	/// must not modify the data buffer until the notification is posted. The first
	672	/// cqe follows the usual rules and so its res field will contain the number of
	673	/// bytes sent or a negative error code. The notification's res field will be
	674	/// set to zero and the flags field will contain IORING_CQE_F_NOTIF. The two
	675	/// step model is needed because the kernel may hold on to buffers for a long
	676	/// time, e.g. waiting for a TCP ACK. Notifications responsible for controlling
	677	/// the lifetime of the buffers. Even errored requests may generate a
	678	/// notification.
	679	///
	680	/// Available since 6.0
	681	pub fn send_zc(
	682	self: *IoUring,
	683	user_data: u64,
	684	fd: os.fd_t,
	685	buffer: []const u8,
	686	send_flags: u32,
	687	zc_flags: u16,
	688	) !*linux.io_uring_sqe {
	689	const sqe = try self.get_sqe();
	690	sqe.prep_send_zc(fd, buffer, send_flags, zc_flags);
	691	sqe.user_data = user_data;
	692	return sqe;
	693	}
	694
	695	/// Queues (but does not submit) an SQE to perform an async zerocopy `send(2)`.
	696	/// Returns a pointer to the SQE.
	697	/// Available since 6.0
	698	pub fn send_zc_fixed(
	699	self: *IoUring,
	700	user_data: u64,
	701	fd: os.fd_t,
	702	buffer: []const u8,
	703	send_flags: u32,
	704	zc_flags: u16,
	705	buf_index: u16,
	706	) !*linux.io_uring_sqe {
	707	const sqe = try self.get_sqe();
	708	sqe.prep_send_zc_fixed(fd, buffer, send_flags, zc_flags, buf_index);
	709	sqe.user_data = user_data;
	710	return sqe;
	711	}
	712
	713	/// Queues (but does not submit) an SQE to perform a `recvmsg(2)`.
	714	/// Returns a pointer to the SQE.
	715	/// Available since 5.3
	716	pub fn recvmsg(
	717	self: *IoUring,
	718	user_data: u64,
	719	fd: os.fd_t,
	720	msg: *os.msghdr,
	721	flags: u32,
	722	) !*linux.io_uring_sqe {
	723	const sqe = try self.get_sqe();
	724	sqe.prep_recvmsg(fd, msg, flags);
	725	sqe.user_data = user_data;
	726	return sqe;
	727	}
	728
	729	/// Queues (but does not submit) an SQE to perform a `sendmsg(2)`.
	730	/// Returns a pointer to the SQE.
	731	/// Available since 5.3
	732	pub fn sendmsg(
	733	self: *IoUring,
	734	user_data: u64,
	735	fd: os.fd_t,
	736	msg: *const os.msghdr_const,
	737	flags: u32,
	738	) !*linux.io_uring_sqe {
	739	const sqe = try self.get_sqe();
	740	sqe.prep_sendmsg(fd, msg, flags);
	741	sqe.user_data = user_data;
	742	return sqe;
	743	}
	744
	745	/// Queues (but does not submit) an SQE to perform an async zerocopy `sendmsg(2)`.
	746	/// Returns a pointer to the SQE.
	747	/// Available since 6.1
	748	pub fn sendmsg_zc(
	749	self: *IoUring,
	750	user_data: u64,
	751	fd: os.fd_t,
	752	msg: *const os.msghdr_const,
	753	flags: u32,
	754	) !*linux.io_uring_sqe {
	755	const sqe = try self.get_sqe();
	756	sqe.prep_sendmsg_zc(fd, msg, flags);
	757	sqe.user_data = user_data;
	758	return sqe;
	759	}
	760
	761	/// Queues (but does not submit) an SQE to perform an `openat(2)`.
	762	/// Returns a pointer to the SQE.
	763	/// Available since 5.6.
	764	pub fn openat(
	765	self: *IoUring,
	766	user_data: u64,
	767	fd: os.fd_t,
	768	path: [*:0]const u8,
	769	flags: linux.O,
	770	mode: os.mode_t,
	771	) !*linux.io_uring_sqe {
	772	const sqe = try self.get_sqe();
	773	sqe.prep_openat(fd, path, flags, mode);
	774	sqe.user_data = user_data;
	775	return sqe;
	776	}
	777
	778	/// Queues an openat using direct (registered) file descriptors.
	779	///
	780	/// To use an accept direct variant, the application must first have registered
	781	/// a file table (with register_files). An unused table index will be
	782	/// dynamically chosen and returned in the CQE res field.
	783	///
	784	/// After creation, they can be used by setting IOSQE_FIXED_FILE in the SQE
	785	/// flags member, and setting the SQE fd field to the direct descriptor value
	786	/// rather than the regular file descriptor.
	787	///
	788	/// Available since 5.15
	789	pub fn openat_direct(
	790	self: *IoUring,
	791	user_data: u64,
	792	fd: os.fd_t,
	793	path: [*:0]const u8,
	794	flags: linux.O,
	795	mode: os.mode_t,
	796	file_index: u32,
	797	) !*linux.io_uring_sqe {
	798	const sqe = try self.get_sqe();
	799	sqe.prep_openat_direct(fd, path, flags, mode, file_index);
	800	sqe.user_data = user_data;
	801	return sqe;
	802	}
	803
	804	/// Queues (but does not submit) an SQE to perform a `close(2)`.
	805	/// Returns a pointer to the SQE.
	806	/// Available since 5.6.
	807	pub fn close(self: IoUring, user_data: u64, fd: os.fd_t) !linux.io_uring_sqe {
	808	const sqe = try self.get_sqe();
	809	sqe.prep_close(fd);
	810	sqe.user_data = user_data;
	811	return sqe;
	812	}
	813
	814	/// Queues close of registered file descriptor.
	815	/// Available since 5.15
	816	pub fn close_direct(self: IoUring, user_data: u64, file_index: u32) !linux.io_uring_sqe {
	817	const sqe = try self.get_sqe();
	818	sqe.prep_close_direct(file_index);
	819	sqe.user_data = user_data;
	820	return sqe;
	821	}
	822
	823	/// Queues (but does not submit) an SQE to register a timeout operation.
	824	/// Returns a pointer to the SQE.
	825	///
	826	/// The timeout will complete when either the timeout expires, or after the specified number of
	827	/// events complete (if `count` is greater than `0`).
	828	///
	829	/// `flags` may be `0` for a relative timeout, or `IORING_TIMEOUT_ABS` for an absolute timeout.
	830	///
	831	/// The completion event result will be `-ETIME` if the timeout completed through expiration,
	832	/// `0` if the timeout completed after the specified number of events, or `-ECANCELED` if the
	833	/// timeout was removed before it expired.
	834	///
	835	/// io_uring timeouts use the `CLOCK.MONOTONIC` clock source.
	836	pub fn timeout(
	837	self: *IoUring,
	838	user_data: u64,
	839	ts: *const os.linux.kernel_timespec,
	840	count: u32,
	841	flags: u32,
	842	) !*linux.io_uring_sqe {
	843	const sqe = try self.get_sqe();
	844	sqe.prep_timeout(ts, count, flags);
	845	sqe.user_data = user_data;
	846	return sqe;
	847	}
	848
	849	/// Queues (but does not submit) an SQE to remove an existing timeout operation.
	850	/// Returns a pointer to the SQE.
	851	///
	852	/// The timeout is identified by its `user_data`.
	853	///
	854	/// The completion event result will be `0` if the timeout was found and cancelled successfully,
	855	/// `-EBUSY` if the timeout was found but expiration was already in progress, or
	856	/// `-ENOENT` if the timeout was not found.
	857	pub fn timeout_remove(
	858	self: *IoUring,
	859	user_data: u64,
	860	timeout_user_data: u64,
	861	flags: u32,
	862	) !*linux.io_uring_sqe {
	863	const sqe = try self.get_sqe();
	864	sqe.prep_timeout_remove(timeout_user_data, flags);
	865	sqe.user_data = user_data;
	866	return sqe;
	867	}
	868
	869	/// Queues (but does not submit) an SQE to add a link timeout operation.
	870	/// Returns a pointer to the SQE.
	871	///
	872	/// You need to set linux.IOSQE_IO_LINK to flags of the target operation
	873	/// and then call this method right after the target operation.
	874	/// See https://lwn.net/Articles/803932/ for detail.
	875	///
	876	/// If the dependent request finishes before the linked timeout, the timeout
	877	/// is canceled. If the timeout finishes before the dependent request, the
	878	/// dependent request will be canceled.
	879	///
	880	/// The completion event result of the link_timeout will be
	881	/// `-ETIME` if the timeout finishes before the dependent request
	882	/// (in this case, the completion event result of the dependent request will
	883	/// be `-ECANCELED`), or
	884	/// `-EALREADY` if the dependent request finishes before the linked timeout.
	885	pub fn link_timeout(
	886	self: *IoUring,
	887	user_data: u64,
	888	ts: *const os.linux.kernel_timespec,
	889	flags: u32,
	890	) !*linux.io_uring_sqe {
	891	const sqe = try self.get_sqe();
	892	sqe.prep_link_timeout(ts, flags);
	893	sqe.user_data = user_data;
	894	return sqe;
	895	}
	896
	897	/// Queues (but does not submit) an SQE to perform a `poll(2)`.
	898	/// Returns a pointer to the SQE.
	899	pub fn poll_add(
	900	self: *IoUring,
	901	user_data: u64,
	902	fd: os.fd_t,
	903	poll_mask: u32,
	904	) !*linux.io_uring_sqe {
	905	const sqe = try self.get_sqe();
	906	sqe.prep_poll_add(fd, poll_mask);
	907	sqe.user_data = user_data;
	908	return sqe;
	909	}
	910
	911	/// Queues (but does not submit) an SQE to remove an existing poll operation.
	912	/// Returns a pointer to the SQE.
	913	pub fn poll_remove(
	914	self: *IoUring,
	915	user_data: u64,
	916	target_user_data: u64,
	917	) !*linux.io_uring_sqe {
	918	const sqe = try self.get_sqe();
	919	sqe.prep_poll_remove(target_user_data);
	920	sqe.user_data = user_data;
	921	return sqe;
	922	}
	923
	924	/// Queues (but does not submit) an SQE to update the user data of an existing poll
	925	/// operation. Returns a pointer to the SQE.
	926	pub fn poll_update(
	927	self: *IoUring,
	928	user_data: u64,
	929	old_user_data: u64,
	930	new_user_data: u64,
	931	poll_mask: u32,
	932	flags: u32,
	933	) !*linux.io_uring_sqe {
	934	const sqe = try self.get_sqe();
	935	sqe.prep_poll_update(old_user_data, new_user_data, poll_mask, flags);
	936	sqe.user_data = user_data;
	937	return sqe;
	938	}
	939
	940	/// Queues (but does not submit) an SQE to perform an `fallocate(2)`.
	941	/// Returns a pointer to the SQE.
	942	pub fn fallocate(
	943	self: *IoUring,
	944	user_data: u64,
	945	fd: os.fd_t,
	946	mode: i32,
	947	offset: u64,
	948	len: u64,
	949	) !*linux.io_uring_sqe {
	950	const sqe = try self.get_sqe();
	951	sqe.prep_fallocate(fd, mode, offset, len);
	952	sqe.user_data = user_data;
	953	return sqe;
	954	}
	955
	956	/// Queues (but does not submit) an SQE to perform an `statx(2)`.
	957	/// Returns a pointer to the SQE.
	958	pub fn statx(
	959	self: *IoUring,
	960	user_data: u64,
	961	fd: os.fd_t,
	962	path: [:0]const u8,
	963	flags: u32,
	964	mask: u32,
	965	buf: *linux.Statx,
	966	) !*linux.io_uring_sqe {
	967	const sqe = try self.get_sqe();
	968	sqe.prep_statx(fd, path, flags, mask, buf);
	969	sqe.user_data = user_data;
	970	return sqe;
	971	}
	972
	973	/// Queues (but does not submit) an SQE to remove an existing operation.
	974	/// Returns a pointer to the SQE.
	975	///
	976	/// The operation is identified by its `user_data`.
	977	///
	978	/// The completion event result will be `0` if the operation was found and cancelled successfully,
	979	/// `-EALREADY` if the operation was found but was already in progress, or
	980	/// `-ENOENT` if the operation was not found.
	981	pub fn cancel(
	982	self: *IoUring,
	983	user_data: u64,
	984	cancel_user_data: u64,
	985	flags: u32,
	986	) !*linux.io_uring_sqe {
	987	const sqe = try self.get_sqe();
	988	sqe.prep_cancel(cancel_user_data, flags);
	989	sqe.user_data = user_data;
	990	return sqe;
	991	}
	992
	993	/// Queues (but does not submit) an SQE to perform a `shutdown(2)`.
	994	/// Returns a pointer to the SQE.
	995	///
	996	/// The operation is identified by its `user_data`.
	997	pub fn shutdown(
	998	self: *IoUring,
	999	user_data: u64,
	1000	sockfd: os.socket_t,
	1001	how: u32,
	1002	) !*linux.io_uring_sqe {
	1003	const sqe = try self.get_sqe();
	1004	sqe.prep_shutdown(sockfd, how);
	1005	sqe.user_data = user_data;
	1006	return sqe;
	1007	}
	1008
	1009	/// Queues (but does not submit) an SQE to perform a `renameat2(2)`.
	1010	/// Returns a pointer to the SQE.
	1011	pub fn renameat(
	1012	self: *IoUring,
	1013	user_data: u64,
	1014	old_dir_fd: os.fd_t,
	1015	old_path: [*:0]const u8,
	1016	new_dir_fd: os.fd_t,
	1017	new_path: [*:0]const u8,
	1018	flags: u32,
	1019	) !*linux.io_uring_sqe {
	1020	const sqe = try self.get_sqe();
	1021	sqe.prep_renameat(old_dir_fd, old_path, new_dir_fd, new_path, flags);
	1022	sqe.user_data = user_data;
	1023	return sqe;
	1024	}
	1025
	1026	/// Queues (but does not submit) an SQE to perform a `unlinkat(2)`.
	1027	/// Returns a pointer to the SQE.
	1028	pub fn unlinkat(
	1029	self: *IoUring,
	1030	user_data: u64,
	1031	dir_fd: os.fd_t,
	1032	path: [*:0]const u8,
	1033	flags: u32,
	1034	) !*linux.io_uring_sqe {
	1035	const sqe = try self.get_sqe();
	1036	sqe.prep_unlinkat(dir_fd, path, flags);
	1037	sqe.user_data = user_data;
	1038	return sqe;
	1039	}
	1040
	1041	/// Queues (but does not submit) an SQE to perform a `mkdirat(2)`.
	1042	/// Returns a pointer to the SQE.
	1043	pub fn mkdirat(
	1044	self: *IoUring,
	1045	user_data: u64,
	1046	dir_fd: os.fd_t,
	1047	path: [*:0]const u8,
	1048	mode: os.mode_t,
	1049	) !*linux.io_uring_sqe {
	1050	const sqe = try self.get_sqe();
	1051	sqe.prep_mkdirat(dir_fd, path, mode);
	1052	sqe.user_data = user_data;
	1053	return sqe;
	1054	}
	1055
	1056	/// Queues (but does not submit) an SQE to perform a `symlinkat(2)`.
	1057	/// Returns a pointer to the SQE.
	1058	pub fn symlinkat(
	1059	self: *IoUring,
	1060	user_data: u64,
	1061	target: [*:0]const u8,
	1062	new_dir_fd: os.fd_t,
	1063	link_path: [*:0]const u8,
	1064	) !*linux.io_uring_sqe {
	1065	const sqe = try self.get_sqe();
	1066	sqe.prep_symlinkat(target, new_dir_fd, link_path);
	1067	sqe.user_data = user_data;
	1068	return sqe;
	1069	}
	1070
	1071	/// Queues (but does not submit) an SQE to perform a `linkat(2)`.
	1072	/// Returns a pointer to the SQE.
	1073	pub fn linkat(
	1074	self: *IoUring,
	1075	user_data: u64,
	1076	old_dir_fd: os.fd_t,
	1077	old_path: [*:0]const u8,
	1078	new_dir_fd: os.fd_t,
	1079	new_path: [*:0]const u8,
	1080	flags: u32,
	1081	) !*linux.io_uring_sqe {
	1082	const sqe = try self.get_sqe();
	1083	sqe.prep_linkat(old_dir_fd, old_path, new_dir_fd, new_path, flags);
	1084	sqe.user_data = user_data;
	1085	return sqe;
	1086	}
	1087
	1088	/// Queues (but does not submit) an SQE to provide a group of buffers used for commands that read/receive data.
	1089	/// Returns a pointer to the SQE.
	1090	///
	1091	/// Provided buffers can be used in `read`, `recv` or `recvmsg` commands via .buffer_selection.
	1092	///
	1093	/// The kernel expects a contiguous block of memory of size (buffers_count * buffer_size).
	1094	pub fn provide_buffers(
	1095	self: *IoUring,
	1096	user_data: u64,
	1097	buffers: [*]u8,
	1098	buffer_size: usize,
	1099	buffers_count: usize,
	1100	group_id: usize,
	1101	buffer_id: usize,
	1102	) !*linux.io_uring_sqe {
	1103	const sqe = try self.get_sqe();
	1104	sqe.prep_provide_buffers(buffers, buffer_size, buffers_count, group_id, buffer_id);
	1105	sqe.user_data = user_data;
	1106	return sqe;
	1107	}
	1108
	1109	/// Queues (but does not submit) an SQE to remove a group of provided buffers.
	1110	/// Returns a pointer to the SQE.
	1111	pub fn remove_buffers(
	1112	self: *IoUring,
	1113	user_data: u64,
	1114	buffers_count: usize,
	1115	group_id: usize,
	1116	) !*linux.io_uring_sqe {
	1117	const sqe = try self.get_sqe();
	1118	sqe.prep_remove_buffers(buffers_count, group_id);
	1119	sqe.user_data = user_data;
	1120	return sqe;
	1121	}
	1122
	1123	/// Queues (but does not submit) an SQE to perform a `waitid(2)`.
	1124	/// Returns a pointer to the SQE.
	1125	pub fn waitid(
	1126	self: *IoUring,
	1127	user_data: u64,
	1128	id_type: linux.P,
	1129	id: i32,
	1130	infop: *linux.siginfo_t,
	1131	options: u32,
	1132	flags: u32,
	1133	) !*linux.io_uring_sqe {
	1134	const sqe = try self.get_sqe();
	1135	sqe.prep_waitid(id_type, id, infop, options, flags);
	1136	sqe.user_data = user_data;
	1137	return sqe;
	1138	}
	1139
	1140	/// Registers an array of file descriptors.
	1141	/// Every time a file descriptor is put in an SQE and submitted to the kernel, the kernel must
	1142	/// retrieve a reference to the file, and once I/O has completed the file reference must be
	1143	/// dropped. The atomic nature of this file reference can be a slowdown for high IOPS workloads.
	1144	/// This slowdown can be avoided by pre-registering file descriptors.
	1145	/// To refer to a registered file descriptor, IOSQE_FIXED_FILE must be set in the SQE's flags,
	1146	/// and the SQE's fd must be set to the index of the file descriptor in the registered array.
	1147	/// Registering file descriptors will wait for the ring to idle.
	1148	/// Files are automatically unregistered by the kernel when the ring is torn down.
	1149	/// An application need unregister only if it wants to register a new array of file descriptors.
	1150	pub fn register_files(self: *IoUring, fds: []const os.fd_t) !void {
	1151	assert(self.fd >= 0);
	1152	const res = linux.io_uring_register(
	1153	self.fd,
	1154	.REGISTER_FILES,
	1155	@as(*const anyopaque, @ptrCast(fds.ptr)),
	1156	@as(u32, @intCast(fds.len)),
	1157	);
	1158	try handle_registration_result(res);
	1159	}
	1160
	1161	/// Updates registered file descriptors.
	1162	///
	1163	/// Updates are applied starting at the provided offset in the original file descriptors slice.
	1164	/// There are three kind of updates:
	1165	/// * turning a sparse entry (where the fd is -1) into a real one
	1166	/// * removing an existing entry (set the fd to -1)
	1167	/// * replacing an existing entry with a new fd
	1168	/// Adding new file descriptors must be done with `register_files`.
	1169	pub fn register_files_update(self: *IoUring, offset: u32, fds: []const os.fd_t) !void {
	1170	assert(self.fd >= 0);
	1171
	1172	const FilesUpdate = extern struct {
	1173	offset: u32,
	1174	resv: u32,
	1175	fds: u64 align(8),
	1176	};
	1177	var update = FilesUpdate{
	1178	.offset = offset,
	1179	.resv = @as(u32, 0),
	1180	.fds = @as(u64, @intFromPtr(fds.ptr)),
	1181	};
	1182
	1183	const res = linux.io_uring_register(
	1184	self.fd,
	1185	.REGISTER_FILES_UPDATE,
	1186	@as(*const anyopaque, @ptrCast(&update)),
	1187	@as(u32, @intCast(fds.len)),
	1188	);
	1189	try handle_registration_result(res);
	1190	}
	1191
	1192	/// Registers the file descriptor for an eventfd that will be notified of completion events on
	1193	/// an io_uring instance.
	1194	/// Only a single a eventfd can be registered at any given point in time.
	1195	pub fn register_eventfd(self: *IoUring, fd: os.fd_t) !void {
	1196	assert(self.fd >= 0);
	1197	const res = linux.io_uring_register(
	1198	self.fd,
	1199	.REGISTER_EVENTFD,
	1200	@as(*const anyopaque, @ptrCast(&fd)),
	1201	1,
	1202	);
	1203	try handle_registration_result(res);
	1204	}
	1205
	1206	/// Registers the file descriptor for an eventfd that will be notified of completion events on
	1207	/// an io_uring instance. Notifications are only posted for events that complete in an async manner.
	1208	/// This means that events that complete inline while being submitted do not trigger a notification event.
	1209	/// Only a single eventfd can be registered at any given point in time.
	1210	pub fn register_eventfd_async(self: *IoUring, fd: os.fd_t) !void {
	1211	assert(self.fd >= 0);
	1212	const res = linux.io_uring_register(
	1213	self.fd,
	1214	.REGISTER_EVENTFD_ASYNC,
	1215	@as(*const anyopaque, @ptrCast(&fd)),
	1216	1,
	1217	);
	1218	try handle_registration_result(res);
	1219	}
	1220
	1221	/// Unregister the registered eventfd file descriptor.
	1222	pub fn unregister_eventfd(self: *IoUring) !void {
	1223	assert(self.fd >= 0);
	1224	const res = linux.io_uring_register(
	1225	self.fd,
	1226	.UNREGISTER_EVENTFD,
	1227	null,
	1228	0,
	1229	);
	1230	try handle_registration_result(res);
	1231	}
	1232
	1233	/// Registers an array of buffers for use with `read_fixed` and `write_fixed`.
	1234	pub fn register_buffers(self: *IoUring, buffers: []const os.iovec) !void {
	1235	assert(self.fd >= 0);
	1236	const res = linux.io_uring_register(
	1237	self.fd,
	1238	.REGISTER_BUFFERS,
	1239	buffers.ptr,
	1240	@as(u32, @intCast(buffers.len)),
	1241	);
	1242	try handle_registration_result(res);
	1243	}
	1244
	1245	/// Unregister the registered buffers.
	1246	pub fn unregister_buffers(self: *IoUring) !void {
	1247	assert(self.fd >= 0);
	1248	const res = linux.io_uring_register(self.fd, .UNREGISTER_BUFFERS, null, 0);
	1249	switch (linux.getErrno(res)) {
	1250	.SUCCESS => {},
	1251	.NXIO => return error.BuffersNotRegistered,
	1252	else => \|errno\| return os.unexpectedErrno(errno),
	1253	}
	1254	}
	1255
	1256	fn handle_registration_result(res: usize) !void {
	1257	switch (linux.getErrno(res)) {
	1258	.SUCCESS => {},
	1259	// One or more fds in the array are invalid, or the kernel does not support sparse sets:
	1260	.BADF => return error.FileDescriptorInvalid,
	1261	.BUSY => return error.FilesAlreadyRegistered,
	1262	.INVAL => return error.FilesEmpty,
	1263	// Adding `nr_args` file references would exceed the maximum allowed number of files the
	1264	// user is allowed to have according to the per-user RLIMIT_NOFILE resource limit and
	1265	// the CAP_SYS_RESOURCE capability is not set, or `nr_args` exceeds the maximum allowed
	1266	// for a fixed file set (older kernels have a limit of 1024 files vs 64K files):
	1267	.MFILE => return error.UserFdQuotaExceeded,
	1268	// Insufficient kernel resources, or the caller had a non-zero RLIMIT_MEMLOCK soft
	1269	// resource limit but tried to lock more memory than the limit permitted (not enforced
	1270	// when the process is privileged with CAP_IPC_LOCK):
	1271	.NOMEM => return error.SystemResources,
	1272	// Attempt to register files on a ring already registering files or being torn down:
	1273	.NXIO => return error.RingShuttingDownOrAlreadyRegisteringFiles,
	1274	else => \|errno\| return os.unexpectedErrno(errno),
	1275	}
	1276	}
	1277
	1278	/// Unregisters all registered file descriptors previously associated with the ring.
	1279	pub fn unregister_files(self: *IoUring) !void {
	1280	assert(self.fd >= 0);
	1281	const res = linux.io_uring_register(self.fd, .UNREGISTER_FILES, null, 0);
	1282	switch (linux.getErrno(res)) {
	1283	.SUCCESS => {},
	1284	.NXIO => return error.FilesNotRegistered,
	1285	else => \|errno\| return os.unexpectedErrno(errno),
	1286	}
	1287	}
	1288
	1289	/// Prepares a socket creation request.
	1290	/// New socket fd will be returned in completion result.
	1291	/// Available since 5.19
	1292	pub fn socket(
	1293	self: *IoUring,
	1294	user_data: u64,
	1295	domain: u32,
	1296	socket_type: u32,
	1297	protocol: u32,
	1298	flags: u32,
	1299	) !*linux.io_uring_sqe {
	1300	const sqe = try self.get_sqe();
	1301	sqe.prep_socket(domain, socket_type, protocol, flags);
	1302	sqe.user_data = user_data;
	1303	return sqe;
	1304	}
	1305
	1306	/// Prepares a socket creation request for registered file at index `file_index`.
	1307	/// Available since 5.19
	1308	pub fn socket_direct(
	1309	self: *IoUring,
	1310	user_data: u64,
	1311	domain: u32,
	1312	socket_type: u32,
	1313	protocol: u32,
	1314	flags: u32,
	1315	file_index: u32,
	1316	) !*linux.io_uring_sqe {
	1317	const sqe = try self.get_sqe();
	1318	sqe.prep_socket_direct(domain, socket_type, protocol, flags, file_index);
	1319	sqe.user_data = user_data;
	1320	return sqe;
	1321	}
	1322
	1323	/// Prepares a socket creation request for registered file, index chosen by kernel (file index alloc).
	1324	/// File index will be returned in CQE res field.
	1325	/// Available since 5.19
	1326	pub fn socket_direct_alloc(
	1327	self: *IoUring,
	1328	user_data: u64,
	1329	domain: u32,
	1330	socket_type: u32,
	1331	protocol: u32,
	1332	flags: u32,
	1333	) !*linux.io_uring_sqe {
	1334	const sqe = try self.get_sqe();
	1335	sqe.prep_socket_direct_alloc(domain, socket_type, protocol, flags);
	1336	sqe.user_data = user_data;
	1337	return sqe;
	1338	}
	1339
	1340	pub const SubmissionQueue = struct {
	1341	head: *u32,
	1342	tail: *u32,
	1343	mask: u32,
	1344	flags: *u32,
	1345	dropped: *u32,
	1346	array: []u32,
	1347	sqes: []linux.io_uring_sqe,
	1348	mmap: []align(mem.page_size) u8,
	1349	mmap_sqes: []align(mem.page_size) u8,
	1350
	1351	// We use `sqe_head` and `sqe_tail` in the same way as liburing:
	1352	// We increment `sqe_tail` (but not `tail`) for each call to `get_sqe()`.
	1353	// We then set `tail` to `sqe_tail` once, only when these events are actually submitted.
	1354	// This allows us to amortize the cost of the @atomicStore to `tail` across multiple SQEs.
	1355	sqe_head: u32 = 0,
	1356	sqe_tail: u32 = 0,
	1357
	1358	pub fn init(fd: os.fd_t, p: linux.io_uring_params) !SubmissionQueue {
	1359	assert(fd >= 0);
	1360	assert((p.features & linux.IORING_FEAT_SINGLE_MMAP) != 0);
	1361	const size = @max(
	1362	p.sq_off.array + p.sq_entries * @sizeOf(u32),
	1363	p.cq_off.cqes + p.cq_entries * @sizeOf(linux.io_uring_cqe),
	1364	);
	1365	const mmap = try os.mmap(
	1366	null,
	1367	size,
	1368	os.PROT.READ \| os.PROT.WRITE,
	1369	.{ .TYPE = .SHARED, .POPULATE = true },
	1370	fd,
	1371	linux.IORING_OFF_SQ_RING,
	1372	);
	1373	errdefer os.munmap(mmap);
	1374	assert(mmap.len == size);
	1375
	1376	// The motivation for the `sqes` and `array` indirection is to make it possible for the
	1377	// application to preallocate static linux.io_uring_sqe entries and then replay them when needed.
	1378	const size_sqes = p.sq_entries * @sizeOf(linux.io_uring_sqe);
	1379	const mmap_sqes = try os.mmap(
	1380	null,
	1381	size_sqes,
	1382	os.PROT.READ \| os.PROT.WRITE,
	1383	.{ .TYPE = .SHARED, .POPULATE = true },
	1384	fd,
	1385	linux.IORING_OFF_SQES,
	1386	);
	1387	errdefer os.munmap(mmap_sqes);
	1388	assert(mmap_sqes.len == size_sqes);
	1389
	1390	const array: [*]u32 = @ptrCast(@alignCast(&mmap[p.sq_off.array]));
	1391	const sqes: [*]linux.io_uring_sqe = @ptrCast(@alignCast(&mmap_sqes[0]));
	1392	// We expect the kernel copies p.sq_entries to the u32 pointed to by p.sq_off.ring_entries,
	1393	// see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L7843-L7844.
	1394	assert(p.sq_entries == @as(u32, @ptrCast(@alignCast(&mmap[p.sq_off.ring_entries]))).);
	1395	return SubmissionQueue{
	1396	.head = @ptrCast(@alignCast(&mmap[p.sq_off.head])),
	1397	.tail = @ptrCast(@alignCast(&mmap[p.sq_off.tail])),
	1398	.mask = @as(u32, @ptrCast(@alignCast(&mmap[p.sq_off.ring_mask]))).,
	1399	.flags = @ptrCast(@alignCast(&mmap[p.sq_off.flags])),
	1400	.dropped = @ptrCast(@alignCast(&mmap[p.sq_off.dropped])),
	1401	.array = array[0..p.sq_entries],
	1402	.sqes = sqes[0..p.sq_entries],
	1403	.mmap = mmap,
	1404	.mmap_sqes = mmap_sqes,
	1405	};
	1406	}
	1407
	1408	pub fn deinit(self: *SubmissionQueue) void {
	1409	os.munmap(self.mmap_sqes);
	1410	os.munmap(self.mmap);
	1411	}
	1412	};
	1413
	1414	pub const CompletionQueue = struct {
	1415	head: *u32,
	1416	tail: *u32,
	1417	mask: u32,
	1418	overflow: *u32,
	1419	cqes: []linux.io_uring_cqe,
	1420
	1421	pub fn init(fd: os.fd_t, p: linux.io_uring_params, sq: SubmissionQueue) !CompletionQueue {
	1422	assert(fd >= 0);
	1423	assert((p.features & linux.IORING_FEAT_SINGLE_MMAP) != 0);
	1424	const mmap = sq.mmap;
	1425	const cqes: [*]linux.io_uring_cqe = @ptrCast(@alignCast(&mmap[p.cq_off.cqes]));
	1426	assert(p.cq_entries == @as(u32, @ptrCast(@alignCast(&mmap[p.cq_off.ring_entries]))).);
	1427	return CompletionQueue{
	1428	.head = @ptrCast(@alignCast(&mmap[p.cq_off.head])),
	1429	.tail = @ptrCast(@alignCast(&mmap[p.cq_off.tail])),
	1430	.mask = @as(u32, @ptrCast(@alignCast(&mmap[p.cq_off.ring_mask]))).,
	1431	.overflow = @ptrCast(@alignCast(&mmap[p.cq_off.overflow])),
	1432	.cqes = cqes[0..p.cq_entries],
	1433	};
	1434	}
	1435
	1436	pub fn deinit(self: *CompletionQueue) void {
	1437	_ = self;
	1438	// A no-op since we now share the mmap with the submission queue.
	1439	// Here for symmetry with the submission queue, and for any future feature support.
	1440	}
	1441	};
	1442
	1443	test "structs/offsets/entries" {
	1444	if (builtin.os.tag != .linux) return error.SkipZigTest;
	1445
	1446	try testing.expectEqual(@as(usize, 120), @sizeOf(linux.io_uring_params));
	1447	try testing.expectEqual(@as(usize, 64), @sizeOf(linux.io_uring_sqe));
	1448	try testing.expectEqual(@as(usize, 16), @sizeOf(linux.io_uring_cqe));
	1449
	1450	try testing.expectEqual(0, linux.IORING_OFF_SQ_RING);
	1451	try testing.expectEqual(0x8000000, linux.IORING_OFF_CQ_RING);
	1452	try testing.expectEqual(0x10000000, linux.IORING_OFF_SQES);
	1453
	1454	try testing.expectError(error.EntriesZero, IoUring.init(0, 0));
	1455	try testing.expectError(error.EntriesNotPowerOfTwo, IoUring.init(3, 0));
	1456	}
	1457
	1458	test "nop" {
	1459	if (builtin.os.tag != .linux) return error.SkipZigTest;
	1460
	1461	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	1462	error.SystemOutdated => return error.SkipZigTest,
	1463	error.PermissionDenied => return error.SkipZigTest,
	1464	else => return err,
	1465	};
	1466	defer {
	1467	ring.deinit();
	1468	testing.expectEqual(@as(os.fd_t, -1), ring.fd) catch @panic("test failed");
	1469	}
	1470
	1471	const sqe = try ring.nop(0xaaaaaaaa);
	1472	try testing.expectEqual(linux.io_uring_sqe{
	1473	.opcode = .NOP,
	1474	.flags = 0,
	1475	.ioprio = 0,
	1476	.fd = 0,
	1477	.off = 0,
	1478	.addr = 0,
	1479	.len = 0,
	1480	.rw_flags = 0,
	1481	.user_data = 0xaaaaaaaa,
	1482	.buf_index = 0,
	1483	.personality = 0,
	1484	.splice_fd_in = 0,
	1485	.addr3 = 0,
	1486	.resv = 0,
	1487	}, sqe.*);
	1488
	1489	try testing.expectEqual(@as(u32, 0), ring.sq.sqe_head);
	1490	try testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
	1491	try testing.expectEqual(@as(u32, 0), ring.sq.tail.*);
	1492	try testing.expectEqual(@as(u32, 0), ring.cq.head.*);
	1493	try testing.expectEqual(@as(u32, 1), ring.sq_ready());
	1494	try testing.expectEqual(@as(u32, 0), ring.cq_ready());
	1495
	1496	try testing.expectEqual(@as(u32, 1), try ring.submit());
	1497	try testing.expectEqual(@as(u32, 1), ring.sq.sqe_head);
	1498	try testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
	1499	try testing.expectEqual(@as(u32, 1), ring.sq.tail.*);
	1500	try testing.expectEqual(@as(u32, 0), ring.cq.head.*);
	1501	try testing.expectEqual(@as(u32, 0), ring.sq_ready());
	1502
	1503	try testing.expectEqual(linux.io_uring_cqe{
	1504	.user_data = 0xaaaaaaaa,
	1505	.res = 0,
	1506	.flags = 0,
	1507	}, try ring.copy_cqe());
	1508	try testing.expectEqual(@as(u32, 1), ring.cq.head.*);
	1509	try testing.expectEqual(@as(u32, 0), ring.cq_ready());
	1510
	1511	const sqe_barrier = try ring.nop(0xbbbbbbbb);
	1512	sqe_barrier.flags \|= linux.IOSQE_IO_DRAIN;
	1513	try testing.expectEqual(@as(u32, 1), try ring.submit());
	1514	try testing.expectEqual(linux.io_uring_cqe{
	1515	.user_data = 0xbbbbbbbb,
	1516	.res = 0,
	1517	.flags = 0,
	1518	}, try ring.copy_cqe());
	1519	try testing.expectEqual(@as(u32, 2), ring.sq.sqe_head);
	1520	try testing.expectEqual(@as(u32, 2), ring.sq.sqe_tail);
	1521	try testing.expectEqual(@as(u32, 2), ring.sq.tail.*);
	1522	try testing.expectEqual(@as(u32, 2), ring.cq.head.*);
	1523	}
	1524
	1525	test "readv" {
	1526	if (builtin.os.tag != .linux) return error.SkipZigTest;
	1527
	1528	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	1529	error.SystemOutdated => return error.SkipZigTest,
	1530	error.PermissionDenied => return error.SkipZigTest,
	1531	else => return err,
	1532	};
	1533	defer ring.deinit();
	1534
	1535	const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
	1536	defer os.close(fd);
	1537
	1538	// Linux Kernel 5.4 supports IORING_REGISTER_FILES but not sparse fd sets (i.e. an fd of -1).
	1539	// Linux Kernel 5.5 adds support for sparse fd sets.
	1540	// Compare:
	1541	// https://github.com/torvalds/linux/blob/v5.4/fs/io_uring.c#L3119-L3124 vs
	1542	// https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L6687-L6691
	1543	// We therefore avoid stressing sparse fd sets here:
	1544	var registered_fds = [_]os.fd_t{0} ** 1;
	1545	const fd_index = 0;
	1546	registered_fds[fd_index] = fd;
	1547	try ring.register_files(registered_fds[0..]);
	1548
	1549	var buffer = [_]u8{42} ** 128;
	1550	var iovecs = [_]os.iovec{os.iovec{ .iov_base = &buffer, .iov_len = buffer.len }};
	1551	const sqe = try ring.read(0xcccccccc, fd_index, .{ .iovecs = iovecs[0..] }, 0);
	1552	try testing.expectEqual(linux.IORING_OP.READV, sqe.opcode);
	1553	sqe.flags \|= linux.IOSQE_FIXED_FILE;
	1554
	1555	try testing.expectError(error.SubmissionQueueFull, ring.nop(0));
	1556	try testing.expectEqual(@as(u32, 1), try ring.submit());
	1557	try testing.expectEqual(linux.io_uring_cqe{
	1558	.user_data = 0xcccccccc,
	1559	.res = buffer.len,
	1560	.flags = 0,
	1561	}, try ring.copy_cqe());
	1562	try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
	1563
	1564	try ring.unregister_files();
	1565	}
	1566
	1567	test "writev/fsync/readv" {
	1568	if (builtin.os.tag != .linux) return error.SkipZigTest;
	1569
	1570	var ring = IoUring.init(4, 0) catch \|err\| switch (err) {
	1571	error.SystemOutdated => return error.SkipZigTest,
	1572	error.PermissionDenied => return error.SkipZigTest,
	1573	else => return err,
	1574	};
	1575	defer ring.deinit();
	1576
	1577	var tmp = std.testing.tmpDir(.{});
	1578	defer tmp.cleanup();
	1579
	1580	const path = "test_io_uring_writev_fsync_readv";
	1581	const file = try tmp.dir.createFile(path, .{ .read = true, .truncate = true });
	1582	defer file.close();
	1583	const fd = file.handle;
	1584
	1585	const buffer_write = [_]u8{42} ** 128;
	1586	const iovecs_write = [_]os.iovec_const{
	1587	os.iovec_const{ .iov_base = &buffer_write, .iov_len = buffer_write.len },
	1588	};
	1589	var buffer_read = [_]u8{0} ** 128;
	1590	var iovecs_read = [_]os.iovec{
	1591	os.iovec{ .iov_base = &buffer_read, .iov_len = buffer_read.len },
	1592	};
	1593
	1594	const sqe_writev = try ring.writev(0xdddddddd, fd, iovecs_write[0..], 17);
	1595	try testing.expectEqual(linux.IORING_OP.WRITEV, sqe_writev.opcode);
	1596	try testing.expectEqual(@as(u64, 17), sqe_writev.off);
	1597	sqe_writev.flags \|= linux.IOSQE_IO_LINK;
	1598
	1599	const sqe_fsync = try ring.fsync(0xeeeeeeee, fd, 0);
	1600	try testing.expectEqual(linux.IORING_OP.FSYNC, sqe_fsync.opcode);
	1601	try testing.expectEqual(fd, sqe_fsync.fd);
	1602	sqe_fsync.flags \|= linux.IOSQE_IO_LINK;
	1603
	1604	const sqe_readv = try ring.read(0xffffffff, fd, .{ .iovecs = iovecs_read[0..] }, 17);
	1605	try testing.expectEqual(linux.IORING_OP.READV, sqe_readv.opcode);
	1606	try testing.expectEqual(@as(u64, 17), sqe_readv.off);
	1607
	1608	try testing.expectEqual(@as(u32, 3), ring.sq_ready());
	1609	try testing.expectEqual(@as(u32, 3), try ring.submit_and_wait(3));
	1610	try testing.expectEqual(@as(u32, 0), ring.sq_ready());
	1611	try testing.expectEqual(@as(u32, 3), ring.cq_ready());
	1612
	1613	try testing.expectEqual(linux.io_uring_cqe{
	1614	.user_data = 0xdddddddd,
	1615	.res = buffer_write.len,
	1616	.flags = 0,
	1617	}, try ring.copy_cqe());
	1618	try testing.expectEqual(@as(u32, 2), ring.cq_ready());
	1619
	1620	try testing.expectEqual(linux.io_uring_cqe{
	1621	.user_data = 0xeeeeeeee,
	1622	.res = 0,
	1623	.flags = 0,
	1624	}, try ring.copy_cqe());
	1625	try testing.expectEqual(@as(u32, 1), ring.cq_ready());
	1626
	1627	try testing.expectEqual(linux.io_uring_cqe{
	1628	.user_data = 0xffffffff,
	1629	.res = buffer_read.len,
	1630	.flags = 0,
	1631	}, try ring.copy_cqe());
	1632	try testing.expectEqual(@as(u32, 0), ring.cq_ready());
	1633
	1634	try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
	1635	}
	1636
	1637	test "write/read" {
	1638	if (builtin.os.tag != .linux) return error.SkipZigTest;
	1639
	1640	var ring = IoUring.init(2, 0) catch \|err\| switch (err) {
	1641	error.SystemOutdated => return error.SkipZigTest,
	1642	error.PermissionDenied => return error.SkipZigTest,
	1643	else => return err,
	1644	};
	1645	defer ring.deinit();
	1646
	1647	var tmp = std.testing.tmpDir(.{});
	1648	defer tmp.cleanup();
	1649	const path = "test_io_uring_write_read";
	1650	const file = try tmp.dir.createFile(path, .{ .read = true, .truncate = true });
	1651	defer file.close();
	1652	const fd = file.handle;
	1653
	1654	const buffer_write = [_]u8{97} ** 20;
	1655	var buffer_read = [_]u8{98} ** 20;
	1656	const sqe_write = try ring.write(0x11111111, fd, buffer_write[0..], 10);
	1657	try testing.expectEqual(linux.IORING_OP.WRITE, sqe_write.opcode);
	1658	try testing.expectEqual(@as(u64, 10), sqe_write.off);
	1659	sqe_write.flags \|= linux.IOSQE_IO_LINK;
	1660	const sqe_read = try ring.read(0x22222222, fd, .{ .buffer = buffer_read[0..] }, 10);
	1661	try testing.expectEqual(linux.IORING_OP.READ, sqe_read.opcode);
	1662	try testing.expectEqual(@as(u64, 10), sqe_read.off);
	1663	try testing.expectEqual(@as(u32, 2), try ring.submit());
	1664
	1665	const cqe_write = try ring.copy_cqe();
	1666	const cqe_read = try ring.copy_cqe();
	1667	// Prior to Linux Kernel 5.6 this is the only way to test for read/write support:
	1668	// https://lwn.net/Articles/809820/
	1669	if (cqe_write.err() == .INVAL) return error.SkipZigTest;
	1670	if (cqe_read.err() == .INVAL) return error.SkipZigTest;
	1671	try testing.expectEqual(linux.io_uring_cqe{
	1672	.user_data = 0x11111111,
	1673	.res = buffer_write.len,
	1674	.flags = 0,
	1675	}, cqe_write);
	1676	try testing.expectEqual(linux.io_uring_cqe{
	1677	.user_data = 0x22222222,
	1678	.res = buffer_read.len,
	1679	.flags = 0,
	1680	}, cqe_read);
	1681	try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
	1682	}
	1683
	1684	test "splice/read" {
	1685	if (builtin.os.tag != .linux) return error.SkipZigTest;
	1686
	1687	var ring = IoUring.init(4, 0) catch \|err\| switch (err) {
	1688	error.SystemOutdated => return error.SkipZigTest,
	1689	error.PermissionDenied => return error.SkipZigTest,
	1690	else => return err,
	1691	};
	1692	defer ring.deinit();
	1693
	1694	var tmp = std.testing.tmpDir(.{});
	1695	const path_src = "test_io_uring_splice_src";
	1696	const file_src = try tmp.dir.createFile(path_src, .{ .read = true, .truncate = true });
	1697	defer file_src.close();
	1698	const fd_src = file_src.handle;
	1699
	1700	const path_dst = "test_io_uring_splice_dst";
	1701	const file_dst = try tmp.dir.createFile(path_dst, .{ .read = true, .truncate = true });
	1702	defer file_dst.close();
	1703	const fd_dst = file_dst.handle;
	1704
	1705	const buffer_write = [_]u8{97} ** 20;
	1706	var buffer_read = [_]u8{98} ** 20;
	1707	_ = try file_src.write(&buffer_write);
	1708
	1709	const fds = try os.pipe();
	1710	const pipe_offset: u64 = std.math.maxInt(u64);
	1711
	1712	const sqe_splice_to_pipe = try ring.splice(0x11111111, fd_src, 0, fds[1], pipe_offset, buffer_write.len);
	1713	try testing.expectEqual(linux.IORING_OP.SPLICE, sqe_splice_to_pipe.opcode);
	1714	try testing.expectEqual(@as(u64, 0), sqe_splice_to_pipe.addr);
	1715	try testing.expectEqual(pipe_offset, sqe_splice_to_pipe.off);
	1716	sqe_splice_to_pipe.flags \|= linux.IOSQE_IO_LINK;
	1717
	1718	const sqe_splice_from_pipe = try ring.splice(0x22222222, fds[0], pipe_offset, fd_dst, 10, buffer_write.len);
	1719	try testing.expectEqual(linux.IORING_OP.SPLICE, sqe_splice_from_pipe.opcode);
	1720	try testing.expectEqual(pipe_offset, sqe_splice_from_pipe.addr);
	1721	try testing.expectEqual(@as(u64, 10), sqe_splice_from_pipe.off);
	1722	sqe_splice_from_pipe.flags \|= linux.IOSQE_IO_LINK;
	1723
	1724	const sqe_read = try ring.read(0x33333333, fd_dst, .{ .buffer = buffer_read[0..] }, 10);
	1725	try testing.expectEqual(linux.IORING_OP.READ, sqe_read.opcode);
	1726	try testing.expectEqual(@as(u64, 10), sqe_read.off);
	1727	try testing.expectEqual(@as(u32, 3), try ring.submit());
	1728
	1729	const cqe_splice_to_pipe = try ring.copy_cqe();
	1730	const cqe_splice_from_pipe = try ring.copy_cqe();
	1731	const cqe_read = try ring.copy_cqe();
	1732	// Prior to Linux Kernel 5.6 this is the only way to test for splice/read support:
	1733	// https://lwn.net/Articles/809820/
	1734	if (cqe_splice_to_pipe.err() == .INVAL) return error.SkipZigTest;
	1735	if (cqe_splice_from_pipe.err() == .INVAL) return error.SkipZigTest;
	1736	if (cqe_read.err() == .INVAL) return error.SkipZigTest;
	1737	try testing.expectEqual(linux.io_uring_cqe{
	1738	.user_data = 0x11111111,
	1739	.res = buffer_write.len,
	1740	.flags = 0,
	1741	}, cqe_splice_to_pipe);
	1742	try testing.expectEqual(linux.io_uring_cqe{
	1743	.user_data = 0x22222222,
	1744	.res = buffer_write.len,
	1745	.flags = 0,
	1746	}, cqe_splice_from_pipe);
	1747	try testing.expectEqual(linux.io_uring_cqe{
	1748	.user_data = 0x33333333,
	1749	.res = buffer_read.len,
	1750	.flags = 0,
	1751	}, cqe_read);
	1752	try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
	1753	}
	1754
	1755	test "write_fixed/read_fixed" {
	1756	if (builtin.os.tag != .linux) return error.SkipZigTest;
	1757
	1758	var ring = IoUring.init(2, 0) catch \|err\| switch (err) {
	1759	error.SystemOutdated => return error.SkipZigTest,
	1760	error.PermissionDenied => return error.SkipZigTest,
	1761	else => return err,
	1762	};
	1763	defer ring.deinit();
	1764
	1765	var tmp = std.testing.tmpDir(.{});
	1766	defer tmp.cleanup();
	1767
	1768	const path = "test_io_uring_write_read_fixed";
	1769	const file = try tmp.dir.createFile(path, .{ .read = true, .truncate = true });
	1770	defer file.close();
	1771	const fd = file.handle;
	1772
	1773	var raw_buffers: [2][11]u8 = undefined;
	1774	// First buffer will be written to the file.
	1775	@memset(&raw_buffers[0], 'z');
	1776	raw_buffers[0][0.."foobar".len].* = "foobar".*;
	1777
	1778	var buffers = [2]os.iovec{
	1779	.{ .iov_base = &raw_buffers[0], .iov_len = raw_buffers[0].len },
	1780	.{ .iov_base = &raw_buffers[1], .iov_len = raw_buffers[1].len },
	1781	};
	1782	ring.register_buffers(&buffers) catch \|err\| switch (err) {
	1783	error.SystemResources => {
	1784	// See https://github.com/ziglang/zig/issues/15362
	1785	return error.SkipZigTest;
	1786	},
	1787	else => \|e\| return e,
	1788	};
	1789
	1790	const sqe_write = try ring.write_fixed(0x45454545, fd, &buffers[0], 3, 0);
	1791	try testing.expectEqual(linux.IORING_OP.WRITE_FIXED, sqe_write.opcode);
	1792	try testing.expectEqual(@as(u64, 3), sqe_write.off);
	1793	sqe_write.flags \|= linux.IOSQE_IO_LINK;
	1794
	1795	const sqe_read = try ring.read_fixed(0x12121212, fd, &buffers[1], 0, 1);
	1796	try testing.expectEqual(linux.IORING_OP.READ_FIXED, sqe_read.opcode);
	1797	try testing.expectEqual(@as(u64, 0), sqe_read.off);
	1798
	1799	try testing.expectEqual(@as(u32, 2), try ring.submit());
	1800
	1801	const cqe_write = try ring.copy_cqe();
	1802	const cqe_read = try ring.copy_cqe();
	1803
	1804	try testing.expectEqual(linux.io_uring_cqe{
	1805	.user_data = 0x45454545,
	1806	.res = @as(i32, @intCast(buffers[0].iov_len)),
	1807	.flags = 0,
	1808	}, cqe_write);
	1809	try testing.expectEqual(linux.io_uring_cqe{
	1810	.user_data = 0x12121212,
	1811	.res = @as(i32, @intCast(buffers[1].iov_len)),
	1812	.flags = 0,
	1813	}, cqe_read);
	1814
	1815	try testing.expectEqualSlices(u8, "\x00\x00\x00", buffers[1].iov_base[0..3]);
	1816	try testing.expectEqualSlices(u8, "foobar", buffers[1].iov_base[3..9]);
	1817	try testing.expectEqualSlices(u8, "zz", buffers[1].iov_base[9..11]);
	1818	}
	1819
	1820	test "openat" {
	1821	if (builtin.os.tag != .linux) return error.SkipZigTest;
	1822
	1823	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	1824	error.SystemOutdated => return error.SkipZigTest,
	1825	error.PermissionDenied => return error.SkipZigTest,
	1826	else => return err,
	1827	};
	1828	defer ring.deinit();
	1829
	1830	var tmp = std.testing.tmpDir(.{});
	1831	defer tmp.cleanup();
	1832
	1833	const path = "test_io_uring_openat";
	1834
	1835	// Workaround for LLVM bug: https://github.com/ziglang/zig/issues/12014
	1836	const path_addr = if (builtin.zig_backend == .stage2_llvm) p: {
	1837	var workaround = path;
	1838	_ = &workaround;
	1839	break :p @intFromPtr(workaround);
	1840	} else @intFromPtr(path);
	1841
	1842	const flags: linux.O = .{ .CLOEXEC = true, .ACCMODE = .RDWR, .CREAT = true };
	1843	const mode: os.mode_t = 0o666;
	1844	const sqe_openat = try ring.openat(0x33333333, tmp.dir.fd, path, flags, mode);
	1845	try testing.expectEqual(linux.io_uring_sqe{
	1846	.opcode = .OPENAT,
	1847	.flags = 0,
	1848	.ioprio = 0,
	1849	.fd = tmp.dir.fd,
	1850	.off = 0,
	1851	.addr = path_addr,
	1852	.len = mode,
	1853	.rw_flags = @bitCast(flags),
	1854	.user_data = 0x33333333,
	1855	.buf_index = 0,
	1856	.personality = 0,
	1857	.splice_fd_in = 0,
	1858	.addr3 = 0,
	1859	.resv = 0,
	1860	}, sqe_openat.*);
	1861	try testing.expectEqual(@as(u32, 1), try ring.submit());
	1862
	1863	const cqe_openat = try ring.copy_cqe();
	1864	try testing.expectEqual(@as(u64, 0x33333333), cqe_openat.user_data);
	1865	if (cqe_openat.err() == .INVAL) return error.SkipZigTest;
	1866	if (cqe_openat.err() == .BADF) return error.SkipZigTest;
	1867	if (cqe_openat.res <= 0) std.debug.print("\ncqe_openat.res={}\n", .{cqe_openat.res});
	1868	try testing.expect(cqe_openat.res > 0);
	1869	try testing.expectEqual(@as(u32, 0), cqe_openat.flags);
	1870
	1871	os.close(cqe_openat.res);
	1872	}
	1873
	1874	test "close" {
	1875	if (builtin.os.tag != .linux) return error.SkipZigTest;
	1876
	1877	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	1878	error.SystemOutdated => return error.SkipZigTest,
	1879	error.PermissionDenied => return error.SkipZigTest,
	1880	else => return err,
	1881	};
	1882	defer ring.deinit();
	1883
	1884	var tmp = std.testing.tmpDir(.{});
	1885	defer tmp.cleanup();
	1886
	1887	const path = "test_io_uring_close";
	1888	const file = try tmp.dir.createFile(path, .{});
	1889	errdefer file.close();
	1890
	1891	const sqe_close = try ring.close(0x44444444, file.handle);
	1892	try testing.expectEqual(linux.IORING_OP.CLOSE, sqe_close.opcode);
	1893	try testing.expectEqual(file.handle, sqe_close.fd);
	1894	try testing.expectEqual(@as(u32, 1), try ring.submit());
	1895
	1896	const cqe_close = try ring.copy_cqe();
	1897	if (cqe_close.err() == .INVAL) return error.SkipZigTest;
	1898	try testing.expectEqual(linux.io_uring_cqe{
	1899	.user_data = 0x44444444,
	1900	.res = 0,
	1901	.flags = 0,
	1902	}, cqe_close);
	1903	}
	1904
	1905	test "accept/connect/send/recv" {
	1906	if (builtin.os.tag != .linux) return error.SkipZigTest;
	1907
	1908	var ring = IoUring.init(16, 0) catch \|err\| switch (err) {
	1909	error.SystemOutdated => return error.SkipZigTest,
	1910	error.PermissionDenied => return error.SkipZigTest,
	1911	else => return err,
	1912	};
	1913	defer ring.deinit();
	1914
	1915	const socket_test_harness = try createSocketTestHarness(&ring);
	1916	defer socket_test_harness.close();
	1917
	1918	const buffer_send = [_]u8{ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 };
	1919	var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
	1920
	1921	const sqe_send = try ring.send(0xeeeeeeee, socket_test_harness.client, buffer_send[0..], 0);
	1922	sqe_send.flags \|= linux.IOSQE_IO_LINK;
	1923	_ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
	1924	try testing.expectEqual(@as(u32, 2), try ring.submit());
	1925
	1926	const cqe_send = try ring.copy_cqe();
	1927	if (cqe_send.err() == .INVAL) return error.SkipZigTest;
	1928	try testing.expectEqual(linux.io_uring_cqe{
	1929	.user_data = 0xeeeeeeee,
	1930	.res = buffer_send.len,
	1931	.flags = 0,
	1932	}, cqe_send);
	1933
	1934	const cqe_recv = try ring.copy_cqe();
	1935	if (cqe_recv.err() == .INVAL) return error.SkipZigTest;
	1936	try testing.expectEqual(linux.io_uring_cqe{
	1937	.user_data = 0xffffffff,
	1938	.res = buffer_recv.len,
	1939	// ignore IORING_CQE_F_SOCK_NONEMPTY since it is only set on some systems
	1940	.flags = cqe_recv.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
	1941	}, cqe_recv);
	1942
	1943	try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
	1944	}
	1945
	1946	test "sendmsg/recvmsg" {
	1947	if (builtin.os.tag != .linux) return error.SkipZigTest;
	1948
	1949	var ring = IoUring.init(2, 0) catch \|err\| switch (err) {
	1950	error.SystemOutdated => return error.SkipZigTest,
	1951	error.PermissionDenied => return error.SkipZigTest,
	1952	else => return err,
	1953	};
	1954	defer ring.deinit();
	1955
	1956	var address_server = try net.Address.parseIp4("127.0.0.1", 0);
	1957
	1958	const server = try os.socket(address_server.any.family, os.SOCK.DGRAM, 0);
	1959	defer os.close(server);
	1960	try os.setsockopt(server, os.SOL.SOCKET, os.SO.REUSEPORT, &mem.toBytes(@as(c_int, 1)));
	1961	try os.setsockopt(server, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
	1962	try os.bind(server, &address_server.any, address_server.getOsSockLen());
	1963
	1964	// set address_server to the OS-chosen IP/port.
	1965	var slen: os.socklen_t = address_server.getOsSockLen();
	1966	try os.getsockname(server, &address_server.any, &slen);
	1967
	1968	const client = try os.socket(address_server.any.family, os.SOCK.DGRAM, 0);
	1969	defer os.close(client);
	1970
	1971	const buffer_send = [_]u8{42} ** 128;
	1972	const iovecs_send = [_]os.iovec_const{
	1973	os.iovec_const{ .iov_base = &buffer_send, .iov_len = buffer_send.len },
	1974	};
	1975	const msg_send = os.msghdr_const{
	1976	.name = &address_server.any,
	1977	.namelen = address_server.getOsSockLen(),
	1978	.iov = &iovecs_send,
	1979	.iovlen = 1,
	1980	.control = null,
	1981	.controllen = 0,
	1982	.flags = 0,
	1983	};
	1984	const sqe_sendmsg = try ring.sendmsg(0x11111111, client, &msg_send, 0);
	1985	sqe_sendmsg.flags \|= linux.IOSQE_IO_LINK;
	1986	try testing.expectEqual(linux.IORING_OP.SENDMSG, sqe_sendmsg.opcode);
	1987	try testing.expectEqual(client, sqe_sendmsg.fd);
	1988
	1989	var buffer_recv = [_]u8{0} ** 128;
	1990	var iovecs_recv = [_]os.iovec{
	1991	os.iovec{ .iov_base = &buffer_recv, .iov_len = buffer_recv.len },
	1992	};
	1993	const addr = [_]u8{0} ** 4;
	1994	var address_recv = net.Address.initIp4(addr, 0);
	1995	var msg_recv: os.msghdr = os.msghdr{
	1996	.name = &address_recv.any,
	1997	.namelen = address_recv.getOsSockLen(),
	1998	.iov = &iovecs_recv,
	1999	.iovlen = 1,
	2000	.control = null,
	2001	.controllen = 0,
	2002	.flags = 0,
	2003	};
	2004	const sqe_recvmsg = try ring.recvmsg(0x22222222, server, &msg_recv, 0);
	2005	try testing.expectEqual(linux.IORING_OP.RECVMSG, sqe_recvmsg.opcode);
	2006	try testing.expectEqual(server, sqe_recvmsg.fd);
	2007
	2008	try testing.expectEqual(@as(u32, 2), ring.sq_ready());
	2009	try testing.expectEqual(@as(u32, 2), try ring.submit_and_wait(2));
	2010	try testing.expectEqual(@as(u32, 0), ring.sq_ready());
	2011	try testing.expectEqual(@as(u32, 2), ring.cq_ready());
	2012
	2013	const cqe_sendmsg = try ring.copy_cqe();
	2014	if (cqe_sendmsg.res == -@as(i32, @intFromEnum(linux.E.INVAL))) return error.SkipZigTest;
	2015	try testing.expectEqual(linux.io_uring_cqe{
	2016	.user_data = 0x11111111,
	2017	.res = buffer_send.len,
	2018	.flags = 0,
	2019	}, cqe_sendmsg);
	2020
	2021	const cqe_recvmsg = try ring.copy_cqe();
	2022	if (cqe_recvmsg.res == -@as(i32, @intFromEnum(linux.E.INVAL))) return error.SkipZigTest;
	2023	try testing.expectEqual(linux.io_uring_cqe{
	2024	.user_data = 0x22222222,
	2025	.res = buffer_recv.len,
	2026	// ignore IORING_CQE_F_SOCK_NONEMPTY since it is set non-deterministically
	2027	.flags = cqe_recvmsg.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
	2028	}, cqe_recvmsg);
	2029
	2030	try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
	2031	}
	2032
	2033	test "timeout (after a relative time)" {
	2034	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2035
	2036	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	2037	error.SystemOutdated => return error.SkipZigTest,
	2038	error.PermissionDenied => return error.SkipZigTest,
	2039	else => return err,
	2040	};
	2041	defer ring.deinit();
	2042
	2043	const ms = 10;
	2044	const margin = 5;
	2045	const ts = os.linux.kernel_timespec{ .tv_sec = 0, .tv_nsec = ms * 1000000 };
	2046
	2047	const started = std.time.milliTimestamp();
	2048	const sqe = try ring.timeout(0x55555555, &ts, 0, 0);
	2049	try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe.opcode);
	2050	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2051	const cqe = try ring.copy_cqe();
	2052	const stopped = std.time.milliTimestamp();
	2053
	2054	try testing.expectEqual(linux.io_uring_cqe{
	2055	.user_data = 0x55555555,
	2056	.res = -@as(i32, @intFromEnum(linux.E.TIME)),
	2057	.flags = 0,
	2058	}, cqe);
	2059
	2060	// Tests should not depend on timings: skip test if outside margin.
	2061	if (!std.math.approxEqAbs(f64, ms, @as(f64, @floatFromInt(stopped - started)), margin)) return error.SkipZigTest;
	2062	}
	2063
	2064	test "timeout (after a number of completions)" {
	2065	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2066
	2067	var ring = IoUring.init(2, 0) catch \|err\| switch (err) {
	2068	error.SystemOutdated => return error.SkipZigTest,
	2069	error.PermissionDenied => return error.SkipZigTest,
	2070	else => return err,
	2071	};
	2072	defer ring.deinit();
	2073
	2074	const ts = os.linux.kernel_timespec{ .tv_sec = 3, .tv_nsec = 0 };
	2075	const count_completions: u64 = 1;
	2076	const sqe_timeout = try ring.timeout(0x66666666, &ts, count_completions, 0);
	2077	try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe_timeout.opcode);
	2078	try testing.expectEqual(count_completions, sqe_timeout.off);
	2079	_ = try ring.nop(0x77777777);
	2080	try testing.expectEqual(@as(u32, 2), try ring.submit());
	2081
	2082	const cqe_nop = try ring.copy_cqe();
	2083	try testing.expectEqual(linux.io_uring_cqe{
	2084	.user_data = 0x77777777,
	2085	.res = 0,
	2086	.flags = 0,
	2087	}, cqe_nop);
	2088
	2089	const cqe_timeout = try ring.copy_cqe();
	2090	try testing.expectEqual(linux.io_uring_cqe{
	2091	.user_data = 0x66666666,
	2092	.res = 0,
	2093	.flags = 0,
	2094	}, cqe_timeout);
	2095	}
	2096
	2097	test "timeout_remove" {
	2098	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2099
	2100	var ring = IoUring.init(2, 0) catch \|err\| switch (err) {
	2101	error.SystemOutdated => return error.SkipZigTest,
	2102	error.PermissionDenied => return error.SkipZigTest,
	2103	else => return err,
	2104	};
	2105	defer ring.deinit();
	2106
	2107	const ts = os.linux.kernel_timespec{ .tv_sec = 3, .tv_nsec = 0 };
	2108	const sqe_timeout = try ring.timeout(0x88888888, &ts, 0, 0);
	2109	try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe_timeout.opcode);
	2110	try testing.expectEqual(@as(u64, 0x88888888), sqe_timeout.user_data);
	2111
	2112	const sqe_timeout_remove = try ring.timeout_remove(0x99999999, 0x88888888, 0);
	2113	try testing.expectEqual(linux.IORING_OP.TIMEOUT_REMOVE, sqe_timeout_remove.opcode);
	2114	try testing.expectEqual(@as(u64, 0x88888888), sqe_timeout_remove.addr);
	2115	try testing.expectEqual(@as(u64, 0x99999999), sqe_timeout_remove.user_data);
	2116
	2117	try testing.expectEqual(@as(u32, 2), try ring.submit());
	2118
	2119	// The order in which the CQE arrive is not clearly documented and it changed with kernel 5.18:
	2120	// * kernel 5.10 gives user data 0x88888888 first, 0x99999999 second
	2121	// * kernel 5.18 gives user data 0x99999999 first, 0x88888888 second
	2122
	2123	var cqes: [2]os.linux.io_uring_cqe = undefined;
	2124	cqes[0] = try ring.copy_cqe();
	2125	cqes[1] = try ring.copy_cqe();
	2126
	2127	for (cqes) \|cqe\| {
	2128	// IORING_OP_TIMEOUT_REMOVE is not supported by this kernel version:
	2129	// Timeout remove operations set the fd to -1, which results in EBADF before EINVAL.
	2130	// We use IORING_FEAT_RW_CUR_POS as a safety check here to make sure we are at least pre-5.6.
	2131	// We don't want to skip this test for newer kernels.
	2132	if (cqe.user_data == 0x99999999 and
	2133	cqe.err() == .BADF and
	2134	(ring.features & linux.IORING_FEAT_RW_CUR_POS) == 0)
	2135	{
	2136	return error.SkipZigTest;
	2137	}
	2138
	2139	try testing.expect(cqe.user_data == 0x88888888 or cqe.user_data == 0x99999999);
	2140
	2141	if (cqe.user_data == 0x88888888) {
	2142	try testing.expectEqual(linux.io_uring_cqe{
	2143	.user_data = 0x88888888,
	2144	.res = -@as(i32, @intFromEnum(linux.E.CANCELED)),
	2145	.flags = 0,
	2146	}, cqe);
	2147	} else if (cqe.user_data == 0x99999999) {
	2148	try testing.expectEqual(linux.io_uring_cqe{
	2149	.user_data = 0x99999999,
	2150	.res = 0,
	2151	.flags = 0,
	2152	}, cqe);
	2153	}
	2154	}
	2155	}
	2156
	2157	test "accept/connect/recv/link_timeout" {
	2158	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2159
	2160	var ring = IoUring.init(16, 0) catch \|err\| switch (err) {
	2161	error.SystemOutdated => return error.SkipZigTest,
	2162	error.PermissionDenied => return error.SkipZigTest,
	2163	else => return err,
	2164	};
	2165	defer ring.deinit();
	2166
	2167	const socket_test_harness = try createSocketTestHarness(&ring);
	2168	defer socket_test_harness.close();
	2169
	2170	var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
	2171
	2172	const sqe_recv = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
	2173	sqe_recv.flags \|= linux.IOSQE_IO_LINK;
	2174
	2175	const ts = os.linux.kernel_timespec{ .tv_sec = 0, .tv_nsec = 1000000 };
	2176	_ = try ring.link_timeout(0x22222222, &ts, 0);
	2177
	2178	const nr_wait = try ring.submit();
	2179	try testing.expectEqual(@as(u32, 2), nr_wait);
	2180
	2181	var i: usize = 0;
	2182	while (i < nr_wait) : (i += 1) {
	2183	const cqe = try ring.copy_cqe();
	2184	switch (cqe.user_data) {
	2185	0xffffffff => {
	2186	if (cqe.res != -@as(i32, @intFromEnum(linux.E.INTR)) and
	2187	cqe.res != -@as(i32, @intFromEnum(linux.E.CANCELED)))
	2188	{
	2189	std.debug.print("Req 0x{x} got {d}\n", .{ cqe.user_data, cqe.res });
	2190	try testing.expect(false);
	2191	}
	2192	},
	2193	0x22222222 => {
	2194	if (cqe.res != -@as(i32, @intFromEnum(linux.E.ALREADY)) and
	2195	cqe.res != -@as(i32, @intFromEnum(linux.E.TIME)))
	2196	{
	2197	std.debug.print("Req 0x{x} got {d}\n", .{ cqe.user_data, cqe.res });
	2198	try testing.expect(false);
	2199	}
	2200	},
	2201	else => @panic("should not happen"),
	2202	}
	2203	}
	2204	}
	2205
	2206	test "fallocate" {
	2207	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2208
	2209	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	2210	error.SystemOutdated => return error.SkipZigTest,
	2211	error.PermissionDenied => return error.SkipZigTest,
	2212	else => return err,
	2213	};
	2214	defer ring.deinit();
	2215
	2216	var tmp = std.testing.tmpDir(.{});
	2217	defer tmp.cleanup();
	2218
	2219	const path = "test_io_uring_fallocate";
	2220	const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
	2221	defer file.close();
	2222
	2223	try testing.expectEqual(@as(u64, 0), (try file.stat()).size);
	2224
	2225	const len: u64 = 65536;
	2226	const sqe = try ring.fallocate(0xaaaaaaaa, file.handle, 0, 0, len);
	2227	try testing.expectEqual(linux.IORING_OP.FALLOCATE, sqe.opcode);
	2228	try testing.expectEqual(file.handle, sqe.fd);
	2229	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2230
	2231	const cqe = try ring.copy_cqe();
	2232	switch (cqe.err()) {
	2233	.SUCCESS => {},
	2234	// This kernel's io_uring does not yet implement fallocate():
	2235	.INVAL => return error.SkipZigTest,
	2236	// This kernel does not implement fallocate():
	2237	.NOSYS => return error.SkipZigTest,
	2238	// The filesystem containing the file referred to by fd does not support this operation;
	2239	// or the mode is not supported by the filesystem containing the file referred to by fd:
	2240	.OPNOTSUPP => return error.SkipZigTest,
	2241	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2242	}
	2243	try testing.expectEqual(linux.io_uring_cqe{
	2244	.user_data = 0xaaaaaaaa,
	2245	.res = 0,
	2246	.flags = 0,
	2247	}, cqe);
	2248
	2249	try testing.expectEqual(len, (try file.stat()).size);
	2250	}
	2251
	2252	test "statx" {
	2253	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2254
	2255	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	2256	error.SystemOutdated => return error.SkipZigTest,
	2257	error.PermissionDenied => return error.SkipZigTest,
	2258	else => return err,
	2259	};
	2260	defer ring.deinit();
	2261
	2262	var tmp = std.testing.tmpDir(.{});
	2263	defer tmp.cleanup();
	2264	const path = "test_io_uring_statx";
	2265	const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
	2266	defer file.close();
	2267
	2268	try testing.expectEqual(@as(u64, 0), (try file.stat()).size);
	2269
	2270	try file.writeAll("foobar");
	2271
	2272	var buf: linux.Statx = undefined;
	2273	const sqe = try ring.statx(
	2274	0xaaaaaaaa,
	2275	tmp.dir.fd,
	2276	path,
	2277	0,
	2278	linux.STATX_SIZE,
	2279	&buf,
	2280	);
	2281	try testing.expectEqual(linux.IORING_OP.STATX, sqe.opcode);
	2282	try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
	2283	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2284
	2285	const cqe = try ring.copy_cqe();
	2286	switch (cqe.err()) {
	2287	.SUCCESS => {},
	2288	// This kernel's io_uring does not yet implement statx():
	2289	.INVAL => return error.SkipZigTest,
	2290	// This kernel does not implement statx():
	2291	.NOSYS => return error.SkipZigTest,
	2292	// The filesystem containing the file referred to by fd does not support this operation;
	2293	// or the mode is not supported by the filesystem containing the file referred to by fd:
	2294	.OPNOTSUPP => return error.SkipZigTest,
	2295	// not supported on older kernels (5.4)
	2296	.BADF => return error.SkipZigTest,
	2297	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2298	}
	2299	try testing.expectEqual(linux.io_uring_cqe{
	2300	.user_data = 0xaaaaaaaa,
	2301	.res = 0,
	2302	.flags = 0,
	2303	}, cqe);
	2304
	2305	try testing.expect(buf.mask & os.linux.STATX_SIZE == os.linux.STATX_SIZE);
	2306	try testing.expectEqual(@as(u64, 6), buf.size);
	2307	}
	2308
	2309	test "accept/connect/recv/cancel" {
	2310	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2311
	2312	var ring = IoUring.init(16, 0) catch \|err\| switch (err) {
	2313	error.SystemOutdated => return error.SkipZigTest,
	2314	error.PermissionDenied => return error.SkipZigTest,
	2315	else => return err,
	2316	};
	2317	defer ring.deinit();
	2318
	2319	const socket_test_harness = try createSocketTestHarness(&ring);
	2320	defer socket_test_harness.close();
	2321
	2322	var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
	2323
	2324	_ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
	2325	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2326
	2327	const sqe_cancel = try ring.cancel(0x99999999, 0xffffffff, 0);
	2328	try testing.expectEqual(linux.IORING_OP.ASYNC_CANCEL, sqe_cancel.opcode);
	2329	try testing.expectEqual(@as(u64, 0xffffffff), sqe_cancel.addr);
	2330	try testing.expectEqual(@as(u64, 0x99999999), sqe_cancel.user_data);
	2331	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2332
	2333	var cqe_recv = try ring.copy_cqe();
	2334	if (cqe_recv.err() == .INVAL) return error.SkipZigTest;
	2335	var cqe_cancel = try ring.copy_cqe();
	2336	if (cqe_cancel.err() == .INVAL) return error.SkipZigTest;
	2337
	2338	// The recv/cancel CQEs may arrive in any order, the recv CQE will sometimes come first:
	2339	if (cqe_recv.user_data == 0x99999999 and cqe_cancel.user_data == 0xffffffff) {
	2340	const a = cqe_recv;
	2341	const b = cqe_cancel;
	2342	cqe_recv = b;
	2343	cqe_cancel = a;
	2344	}
	2345
	2346	try testing.expectEqual(linux.io_uring_cqe{
	2347	.user_data = 0xffffffff,
	2348	.res = -@as(i32, @intFromEnum(linux.E.CANCELED)),
	2349	.flags = 0,
	2350	}, cqe_recv);
	2351
	2352	try testing.expectEqual(linux.io_uring_cqe{
	2353	.user_data = 0x99999999,
	2354	.res = 0,
	2355	.flags = 0,
	2356	}, cqe_cancel);
	2357	}
	2358
	2359	test "register_files_update" {
	2360	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2361
	2362	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	2363	error.SystemOutdated => return error.SkipZigTest,
	2364	error.PermissionDenied => return error.SkipZigTest,
	2365	else => return err,
	2366	};
	2367	defer ring.deinit();
	2368
	2369	const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
	2370	defer os.close(fd);
	2371
	2372	var registered_fds = [_]os.fd_t{0} ** 2;
	2373	const fd_index = 0;
	2374	const fd_index2 = 1;
	2375	registered_fds[fd_index] = fd;
	2376	registered_fds[fd_index2] = -1;
	2377
	2378	ring.register_files(registered_fds[0..]) catch \|err\| switch (err) {
	2379	// Happens when the kernel doesn't support sparse entry (-1) in the file descriptors array.
	2380	error.FileDescriptorInvalid => return error.SkipZigTest,
	2381	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2382	};
	2383
	2384	// Test IORING_REGISTER_FILES_UPDATE
	2385	// Only available since Linux 5.5
	2386
	2387	const fd2 = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
	2388	defer os.close(fd2);
	2389
	2390	registered_fds[fd_index] = fd2;
	2391	registered_fds[fd_index2] = -1;
	2392	try ring.register_files_update(0, registered_fds[0..]);
	2393
	2394	var buffer = [_]u8{42} ** 128;
	2395	{
	2396	const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
	2397	try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
	2398	sqe.flags \|= linux.IOSQE_FIXED_FILE;
	2399
	2400	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2401	try testing.expectEqual(linux.io_uring_cqe{
	2402	.user_data = 0xcccccccc,
	2403	.res = buffer.len,
	2404	.flags = 0,
	2405	}, try ring.copy_cqe());
	2406	try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
	2407	}
	2408
	2409	// Test with a non-zero offset
	2410
	2411	registered_fds[fd_index] = -1;
	2412	registered_fds[fd_index2] = -1;
	2413	try ring.register_files_update(1, registered_fds[1..]);
	2414
	2415	{
	2416	// Next read should still work since fd_index in the registered file descriptors hasn't been updated yet.
	2417	const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
	2418	try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
	2419	sqe.flags \|= linux.IOSQE_FIXED_FILE;
	2420
	2421	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2422	try testing.expectEqual(linux.io_uring_cqe{
	2423	.user_data = 0xcccccccc,
	2424	.res = buffer.len,
	2425	.flags = 0,
	2426	}, try ring.copy_cqe());
	2427	try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
	2428	}
	2429
	2430	try ring.register_files_update(0, registered_fds[0..]);
	2431
	2432	{
	2433	// Now this should fail since both fds are sparse (-1)
	2434	const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
	2435	try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
	2436	sqe.flags \|= linux.IOSQE_FIXED_FILE;
	2437
	2438	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2439	const cqe = try ring.copy_cqe();
	2440	try testing.expectEqual(os.linux.E.BADF, cqe.err());
	2441	}
	2442
	2443	try ring.unregister_files();
	2444	}
	2445
	2446	test "shutdown" {
	2447	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2448
	2449	var ring = IoUring.init(16, 0) catch \|err\| switch (err) {
	2450	error.SystemOutdated => return error.SkipZigTest,
	2451	error.PermissionDenied => return error.SkipZigTest,
	2452	else => return err,
	2453	};
	2454	defer ring.deinit();
	2455
	2456	var address = try net.Address.parseIp4("127.0.0.1", 0);
	2457
	2458	// Socket bound, expect shutdown to work
	2459	{
	2460	const server = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
	2461	defer os.close(server);
	2462	try os.setsockopt(server, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
	2463	try os.bind(server, &address.any, address.getOsSockLen());
	2464	try os.listen(server, 1);
	2465
	2466	// set address to the OS-chosen IP/port.
	2467	var slen: os.socklen_t = address.getOsSockLen();
	2468	try os.getsockname(server, &address.any, &slen);
	2469
	2470	const shutdown_sqe = try ring.shutdown(0x445445445, server, os.linux.SHUT.RD);
	2471	try testing.expectEqual(linux.IORING_OP.SHUTDOWN, shutdown_sqe.opcode);
	2472	try testing.expectEqual(@as(i32, server), shutdown_sqe.fd);
	2473
	2474	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2475
	2476	const cqe = try ring.copy_cqe();
	2477	switch (cqe.err()) {
	2478	.SUCCESS => {},
	2479	// This kernel's io_uring does not yet implement shutdown (kernel version < 5.11)
	2480	.INVAL => return error.SkipZigTest,
	2481	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2482	}
	2483
	2484	try testing.expectEqual(linux.io_uring_cqe{
	2485	.user_data = 0x445445445,
	2486	.res = 0,
	2487	.flags = 0,
	2488	}, cqe);
	2489	}
	2490
	2491	// Socket not bound, expect to fail with ENOTCONN
	2492	{
	2493	const server = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
	2494	defer os.close(server);
	2495
	2496	const shutdown_sqe = ring.shutdown(0x445445445, server, os.linux.SHUT.RD) catch \|err\| switch (err) {
	2497	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2498	};
	2499	try testing.expectEqual(linux.IORING_OP.SHUTDOWN, shutdown_sqe.opcode);
	2500	try testing.expectEqual(@as(i32, server), shutdown_sqe.fd);
	2501
	2502	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2503
	2504	const cqe = try ring.copy_cqe();
	2505	try testing.expectEqual(@as(u64, 0x445445445), cqe.user_data);
	2506	try testing.expectEqual(os.linux.E.NOTCONN, cqe.err());
	2507	}
	2508	}
	2509
	2510	test "renameat" {
	2511	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2512
	2513	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	2514	error.SystemOutdated => return error.SkipZigTest,
	2515	error.PermissionDenied => return error.SkipZigTest,
	2516	else => return err,
	2517	};
	2518	defer ring.deinit();
	2519
	2520	const old_path = "test_io_uring_renameat_old";
	2521	const new_path = "test_io_uring_renameat_new";
	2522
	2523	var tmp = std.testing.tmpDir(.{});
	2524	defer tmp.cleanup();
	2525
	2526	// Write old file with data
	2527
	2528	const old_file = try tmp.dir.createFile(old_path, .{ .truncate = true, .mode = 0o666 });
	2529	defer old_file.close();
	2530	try old_file.writeAll("hello");
	2531
	2532	// Submit renameat
	2533
	2534	const sqe = try ring.renameat(
	2535	0x12121212,
	2536	tmp.dir.fd,
	2537	old_path,
	2538	tmp.dir.fd,
	2539	new_path,
	2540	0,
	2541	);
	2542	try testing.expectEqual(linux.IORING_OP.RENAMEAT, sqe.opcode);
	2543	try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
	2544	try testing.expectEqual(@as(i32, tmp.dir.fd), @as(i32, @bitCast(sqe.len)));
	2545	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2546
	2547	const cqe = try ring.copy_cqe();
	2548	switch (cqe.err()) {
	2549	.SUCCESS => {},
	2550	// This kernel's io_uring does not yet implement renameat (kernel version < 5.11)
	2551	.BADF, .INVAL => return error.SkipZigTest,
	2552	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2553	}
	2554	try testing.expectEqual(linux.io_uring_cqe{
	2555	.user_data = 0x12121212,
	2556	.res = 0,
	2557	.flags = 0,
	2558	}, cqe);
	2559
	2560	// Validate that the old file doesn't exist anymore
	2561	{
	2562	_ = tmp.dir.openFile(old_path, .{}) catch \|err\| switch (err) {
	2563	error.FileNotFound => {},
	2564	else => std.debug.panic("unexpected error: {}", .{err}),
	2565	};
	2566	}
	2567
	2568	// Validate that the new file exists with the proper content
	2569	{
	2570	const new_file = try tmp.dir.openFile(new_path, .{});
	2571	defer new_file.close();
	2572
	2573	var new_file_data: [16]u8 = undefined;
	2574	const bytes_read = try new_file.readAll(&new_file_data);
	2575	try testing.expectEqualStrings("hello", new_file_data[0..bytes_read]);
	2576	}
	2577	}
	2578
	2579	test "unlinkat" {
	2580	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2581
	2582	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	2583	error.SystemOutdated => return error.SkipZigTest,
	2584	error.PermissionDenied => return error.SkipZigTest,
	2585	else => return err,
	2586	};
	2587	defer ring.deinit();
	2588
	2589	const path = "test_io_uring_unlinkat";
	2590
	2591	var tmp = std.testing.tmpDir(.{});
	2592	defer tmp.cleanup();
	2593
	2594	// Write old file with data
	2595
	2596	const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
	2597	defer file.close();
	2598
	2599	// Submit unlinkat
	2600
	2601	const sqe = try ring.unlinkat(
	2602	0x12121212,
	2603	tmp.dir.fd,
	2604	path,
	2605	0,
	2606	);
	2607	try testing.expectEqual(linux.IORING_OP.UNLINKAT, sqe.opcode);
	2608	try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
	2609	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2610
	2611	const cqe = try ring.copy_cqe();
	2612	switch (cqe.err()) {
	2613	.SUCCESS => {},
	2614	// This kernel's io_uring does not yet implement unlinkat (kernel version < 5.11)
	2615	.BADF, .INVAL => return error.SkipZigTest,
	2616	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2617	}
	2618	try testing.expectEqual(linux.io_uring_cqe{
	2619	.user_data = 0x12121212,
	2620	.res = 0,
	2621	.flags = 0,
	2622	}, cqe);
	2623
	2624	// Validate that the file doesn't exist anymore
	2625	_ = tmp.dir.openFile(path, .{}) catch \|err\| switch (err) {
	2626	error.FileNotFound => {},
	2627	else => std.debug.panic("unexpected error: {}", .{err}),
	2628	};
	2629	}
	2630
	2631	test "mkdirat" {
	2632	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2633
	2634	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	2635	error.SystemOutdated => return error.SkipZigTest,
	2636	error.PermissionDenied => return error.SkipZigTest,
	2637	else => return err,
	2638	};
	2639	defer ring.deinit();
	2640
	2641	var tmp = std.testing.tmpDir(.{});
	2642	defer tmp.cleanup();
	2643
	2644	const path = "test_io_uring_mkdirat";
	2645
	2646	// Submit mkdirat
	2647
	2648	const sqe = try ring.mkdirat(
	2649	0x12121212,
	2650	tmp.dir.fd,
	2651	path,
	2652	0o0755,
	2653	);
	2654	try testing.expectEqual(linux.IORING_OP.MKDIRAT, sqe.opcode);
	2655	try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
	2656	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2657
	2658	const cqe = try ring.copy_cqe();
	2659	switch (cqe.err()) {
	2660	.SUCCESS => {},
	2661	// This kernel's io_uring does not yet implement mkdirat (kernel version < 5.15)
	2662	.BADF, .INVAL => return error.SkipZigTest,
	2663	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2664	}
	2665	try testing.expectEqual(linux.io_uring_cqe{
	2666	.user_data = 0x12121212,
	2667	.res = 0,
	2668	.flags = 0,
	2669	}, cqe);
	2670
	2671	// Validate that the directory exist
	2672	_ = try tmp.dir.openDir(path, .{});
	2673	}
	2674
	2675	test "symlinkat" {
	2676	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2677
	2678	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	2679	error.SystemOutdated => return error.SkipZigTest,
	2680	error.PermissionDenied => return error.SkipZigTest,
	2681	else => return err,
	2682	};
	2683	defer ring.deinit();
	2684
	2685	var tmp = std.testing.tmpDir(.{});
	2686	defer tmp.cleanup();
	2687
	2688	const path = "test_io_uring_symlinkat";
	2689	const link_path = "test_io_uring_symlinkat_link";
	2690
	2691	const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
	2692	defer file.close();
	2693
	2694	// Submit symlinkat
	2695
	2696	const sqe = try ring.symlinkat(
	2697	0x12121212,
	2698	path,
	2699	tmp.dir.fd,
	2700	link_path,
	2701	);
	2702	try testing.expectEqual(linux.IORING_OP.SYMLINKAT, sqe.opcode);
	2703	try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
	2704	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2705
	2706	const cqe = try ring.copy_cqe();
	2707	switch (cqe.err()) {
	2708	.SUCCESS => {},
	2709	// This kernel's io_uring does not yet implement symlinkat (kernel version < 5.15)
	2710	.BADF, .INVAL => return error.SkipZigTest,
	2711	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2712	}
	2713	try testing.expectEqual(linux.io_uring_cqe{
	2714	.user_data = 0x12121212,
	2715	.res = 0,
	2716	.flags = 0,
	2717	}, cqe);
	2718
	2719	// Validate that the symlink exist
	2720	_ = try tmp.dir.openFile(link_path, .{});
	2721	}
	2722
	2723	test "linkat" {
	2724	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2725
	2726	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	2727	error.SystemOutdated => return error.SkipZigTest,
	2728	error.PermissionDenied => return error.SkipZigTest,
	2729	else => return err,
	2730	};
	2731	defer ring.deinit();
	2732
	2733	var tmp = std.testing.tmpDir(.{});
	2734	defer tmp.cleanup();
	2735
	2736	const first_path = "test_io_uring_linkat_first";
	2737	const second_path = "test_io_uring_linkat_second";
	2738
	2739	// Write file with data
	2740
	2741	const first_file = try tmp.dir.createFile(first_path, .{ .truncate = true, .mode = 0o666 });
	2742	defer first_file.close();
	2743	try first_file.writeAll("hello");
	2744
	2745	// Submit linkat
	2746
	2747	const sqe = try ring.linkat(
	2748	0x12121212,
	2749	tmp.dir.fd,
	2750	first_path,
	2751	tmp.dir.fd,
	2752	second_path,
	2753	0,
	2754	);
	2755	try testing.expectEqual(linux.IORING_OP.LINKAT, sqe.opcode);
	2756	try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
	2757	try testing.expectEqual(@as(i32, tmp.dir.fd), @as(i32, @bitCast(sqe.len)));
	2758	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2759
	2760	const cqe = try ring.copy_cqe();
	2761	switch (cqe.err()) {
	2762	.SUCCESS => {},
	2763	// This kernel's io_uring does not yet implement linkat (kernel version < 5.15)
	2764	.BADF, .INVAL => return error.SkipZigTest,
	2765	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2766	}
	2767	try testing.expectEqual(linux.io_uring_cqe{
	2768	.user_data = 0x12121212,
	2769	.res = 0,
	2770	.flags = 0,
	2771	}, cqe);
	2772
	2773	// Validate the second file
	2774	const second_file = try tmp.dir.openFile(second_path, .{});
	2775	defer second_file.close();
	2776
	2777	var second_file_data: [16]u8 = undefined;
	2778	const bytes_read = try second_file.readAll(&second_file_data);
	2779	try testing.expectEqualStrings("hello", second_file_data[0..bytes_read]);
	2780	}
	2781
	2782	test "provide_buffers: read" {
	2783	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2784
	2785	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	2786	error.SystemOutdated => return error.SkipZigTest,
	2787	error.PermissionDenied => return error.SkipZigTest,
	2788	else => return err,
	2789	};
	2790	defer ring.deinit();
	2791
	2792	const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
	2793	defer os.close(fd);
	2794
	2795	const group_id = 1337;
	2796	const buffer_id = 0;
	2797
	2798	const buffer_len = 128;
	2799
	2800	var buffers: [4][buffer_len]u8 = undefined;
	2801
	2802	// Provide 4 buffers
	2803
	2804	{
	2805	const sqe = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
	2806	try testing.expectEqual(linux.IORING_OP.PROVIDE_BUFFERS, sqe.opcode);
	2807	try testing.expectEqual(@as(i32, buffers.len), sqe.fd);
	2808	try testing.expectEqual(@as(u32, buffers[0].len), sqe.len);
	2809	try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
	2810	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2811
	2812	const cqe = try ring.copy_cqe();
	2813	switch (cqe.err()) {
	2814	// Happens when the kernel is < 5.7
	2815	.INVAL => return error.SkipZigTest,
	2816	.SUCCESS => {},
	2817	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2818	}
	2819	try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
	2820	}
	2821
	2822	// Do 4 reads which should consume all buffers
	2823
	2824	var i: usize = 0;
	2825	while (i < buffers.len) : (i += 1) {
	2826	const sqe = try ring.read(0xdededede, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
	2827	try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
	2828	try testing.expectEqual(@as(i32, fd), sqe.fd);
	2829	try testing.expectEqual(@as(u64, 0), sqe.addr);
	2830	try testing.expectEqual(@as(u32, buffer_len), sqe.len);
	2831	try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
	2832	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2833
	2834	const cqe = try ring.copy_cqe();
	2835	switch (cqe.err()) {
	2836	.SUCCESS => {},
	2837	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2838	}
	2839
	2840	try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
	2841	const used_buffer_id = cqe.flags >> 16;
	2842	try testing.expect(used_buffer_id >= 0 and used_buffer_id <= 3);
	2843	try testing.expectEqual(@as(i32, buffer_len), cqe.res);
	2844
	2845	try testing.expectEqual(@as(u64, 0xdededede), cqe.user_data);
	2846	try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
	2847	}
	2848
	2849	// This read should fail
	2850
	2851	{
	2852	const sqe = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
	2853	try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
	2854	try testing.expectEqual(@as(i32, fd), sqe.fd);
	2855	try testing.expectEqual(@as(u64, 0), sqe.addr);
	2856	try testing.expectEqual(@as(u32, buffer_len), sqe.len);
	2857	try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
	2858	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2859
	2860	const cqe = try ring.copy_cqe();
	2861	switch (cqe.err()) {
	2862	// Expected
	2863	.NOBUFS => {},
	2864	.SUCCESS => std.debug.panic("unexpected success", .{}),
	2865	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2866	}
	2867	try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
	2868	}
	2869
	2870	// Provide 1 buffer again
	2871
	2872	// Deliberately put something we don't expect in the buffers
	2873	@memset(mem.sliceAsBytes(&buffers), 42);
	2874
	2875	const reprovided_buffer_id = 2;
	2876
	2877	{
	2878	_ = try ring.provide_buffers(0xabababab, @as([*]u8, @ptrCast(&buffers[reprovided_buffer_id])), buffer_len, 1, group_id, reprovided_buffer_id);
	2879	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2880
	2881	const cqe = try ring.copy_cqe();
	2882	switch (cqe.err()) {
	2883	.SUCCESS => {},
	2884	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2885	}
	2886	}
	2887
	2888	// Final read which should work
	2889
	2890	{
	2891	const sqe = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
	2892	try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
	2893	try testing.expectEqual(@as(i32, fd), sqe.fd);
	2894	try testing.expectEqual(@as(u64, 0), sqe.addr);
	2895	try testing.expectEqual(@as(u32, buffer_len), sqe.len);
	2896	try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
	2897	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2898
	2899	const cqe = try ring.copy_cqe();
	2900	switch (cqe.err()) {
	2901	.SUCCESS => {},
	2902	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2903	}
	2904
	2905	try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
	2906	const used_buffer_id = cqe.flags >> 16;
	2907	try testing.expectEqual(used_buffer_id, reprovided_buffer_id);
	2908	try testing.expectEqual(@as(i32, buffer_len), cqe.res);
	2909	try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
	2910	try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
	2911	}
	2912	}
	2913
	2914	test "remove_buffers" {
	2915	if (builtin.os.tag != .linux) return error.SkipZigTest;
	2916
	2917	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	2918	error.SystemOutdated => return error.SkipZigTest,
	2919	error.PermissionDenied => return error.SkipZigTest,
	2920	else => return err,
	2921	};
	2922	defer ring.deinit();
	2923
	2924	const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
	2925	defer os.close(fd);
	2926
	2927	const group_id = 1337;
	2928	const buffer_id = 0;
	2929
	2930	const buffer_len = 128;
	2931
	2932	var buffers: [4][buffer_len]u8 = undefined;
	2933
	2934	// Provide 4 buffers
	2935
	2936	{
	2937	_ = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
	2938	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2939
	2940	const cqe = try ring.copy_cqe();
	2941	switch (cqe.err()) {
	2942	.INVAL => return error.SkipZigTest,
	2943	.SUCCESS => {},
	2944	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2945	}
	2946	try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
	2947	}
	2948
	2949	// Remove 3 buffers
	2950
	2951	{
	2952	const sqe = try ring.remove_buffers(0xbababababa, 3, group_id);
	2953	try testing.expectEqual(linux.IORING_OP.REMOVE_BUFFERS, sqe.opcode);
	2954	try testing.expectEqual(@as(i32, 3), sqe.fd);
	2955	try testing.expectEqual(@as(u64, 0), sqe.addr);
	2956	try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
	2957	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2958
	2959	const cqe = try ring.copy_cqe();
	2960	switch (cqe.err()) {
	2961	.SUCCESS => {},
	2962	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2963	}
	2964	try testing.expectEqual(@as(u64, 0xbababababa), cqe.user_data);
	2965	}
	2966
	2967	// This read should work
	2968
	2969	{
	2970	_ = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
	2971	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2972
	2973	const cqe = try ring.copy_cqe();
	2974	switch (cqe.err()) {
	2975	.SUCCESS => {},
	2976	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2977	}
	2978
	2979	try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
	2980	const used_buffer_id = cqe.flags >> 16;
	2981	try testing.expect(used_buffer_id >= 0 and used_buffer_id < 4);
	2982	try testing.expectEqual(@as(i32, buffer_len), cqe.res);
	2983	try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
	2984	try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
	2985	}
	2986
	2987	// Final read should _not_ work
	2988
	2989	{
	2990	_ = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
	2991	try testing.expectEqual(@as(u32, 1), try ring.submit());
	2992
	2993	const cqe = try ring.copy_cqe();
	2994	switch (cqe.err()) {
	2995	// Expected
	2996	.NOBUFS => {},
	2997	.SUCCESS => std.debug.panic("unexpected success", .{}),
	2998	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	2999	}
	3000	}
	3001	}
	3002
	3003	test "provide_buffers: accept/connect/send/recv" {
	3004	if (builtin.os.tag != .linux) return error.SkipZigTest;
	3005
	3006	var ring = IoUring.init(16, 0) catch \|err\| switch (err) {
	3007	error.SystemOutdated => return error.SkipZigTest,
	3008	error.PermissionDenied => return error.SkipZigTest,
	3009	else => return err,
	3010	};
	3011	defer ring.deinit();
	3012
	3013	const group_id = 1337;
	3014	const buffer_id = 0;
	3015
	3016	const buffer_len = 128;
	3017	var buffers: [4][buffer_len]u8 = undefined;
	3018
	3019	// Provide 4 buffers
	3020
	3021	{
	3022	const sqe = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
	3023	try testing.expectEqual(linux.IORING_OP.PROVIDE_BUFFERS, sqe.opcode);
	3024	try testing.expectEqual(@as(i32, buffers.len), sqe.fd);
	3025	try testing.expectEqual(@as(u32, buffer_len), sqe.len);
	3026	try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
	3027	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3028
	3029	const cqe = try ring.copy_cqe();
	3030	switch (cqe.err()) {
	3031	// Happens when the kernel is < 5.7
	3032	.INVAL => return error.SkipZigTest,
	3033	// Happens on the kernel 5.4
	3034	.BADF => return error.SkipZigTest,
	3035	.SUCCESS => {},
	3036	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	3037	}
	3038	try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
	3039	}
	3040
	3041	const socket_test_harness = try createSocketTestHarness(&ring);
	3042	defer socket_test_harness.close();
	3043
	3044	// Do 4 send on the socket
	3045
	3046	{
	3047	var i: usize = 0;
	3048	while (i < buffers.len) : (i += 1) {
	3049	_ = try ring.send(0xdeaddead, socket_test_harness.server, &([_]u8{'z'} ** buffer_len), 0);
	3050	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3051	}
	3052
	3053	var cqes: [4]linux.io_uring_cqe = undefined;
	3054	try testing.expectEqual(@as(u32, 4), try ring.copy_cqes(&cqes, 4));
	3055	}
	3056
	3057	// Do 4 recv which should consume all buffers
	3058
	3059	// Deliberately put something we don't expect in the buffers
	3060	@memset(mem.sliceAsBytes(&buffers), 1);
	3061
	3062	var i: usize = 0;
	3063	while (i < buffers.len) : (i += 1) {
	3064	const sqe = try ring.recv(0xdededede, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
	3065	try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
	3066	try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
	3067	try testing.expectEqual(@as(u64, 0), sqe.addr);
	3068	try testing.expectEqual(@as(u32, buffer_len), sqe.len);
	3069	try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
	3070	try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
	3071	try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
	3072	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3073
	3074	const cqe = try ring.copy_cqe();
	3075	switch (cqe.err()) {
	3076	.SUCCESS => {},
	3077	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	3078	}
	3079
	3080	try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
	3081	const used_buffer_id = cqe.flags >> 16;
	3082	try testing.expect(used_buffer_id >= 0 and used_buffer_id <= 3);
	3083	try testing.expectEqual(@as(i32, buffer_len), cqe.res);
	3084
	3085	try testing.expectEqual(@as(u64, 0xdededede), cqe.user_data);
	3086	const buffer = buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))];
	3087	try testing.expectEqualSlices(u8, &([_]u8{'z'} ** buffer_len), buffer);
	3088	}
	3089
	3090	// This recv should fail
	3091
	3092	{
	3093	const sqe = try ring.recv(0xdfdfdfdf, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
	3094	try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
	3095	try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
	3096	try testing.expectEqual(@as(u64, 0), sqe.addr);
	3097	try testing.expectEqual(@as(u32, buffer_len), sqe.len);
	3098	try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
	3099	try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
	3100	try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
	3101	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3102
	3103	const cqe = try ring.copy_cqe();
	3104	switch (cqe.err()) {
	3105	// Expected
	3106	.NOBUFS => {},
	3107	.SUCCESS => std.debug.panic("unexpected success", .{}),
	3108	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	3109	}
	3110	try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
	3111	}
	3112
	3113	// Provide 1 buffer again
	3114
	3115	const reprovided_buffer_id = 2;
	3116
	3117	{
	3118	_ = try ring.provide_buffers(0xabababab, @as([*]u8, @ptrCast(&buffers[reprovided_buffer_id])), buffer_len, 1, group_id, reprovided_buffer_id);
	3119	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3120
	3121	const cqe = try ring.copy_cqe();
	3122	switch (cqe.err()) {
	3123	.SUCCESS => {},
	3124	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	3125	}
	3126	}
	3127
	3128	// Redo 1 send on the server socket
	3129
	3130	{
	3131	_ = try ring.send(0xdeaddead, socket_test_harness.server, &([_]u8{'w'} ** buffer_len), 0);
	3132	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3133
	3134	_ = try ring.copy_cqe();
	3135	}
	3136
	3137	// Final recv which should work
	3138
	3139	// Deliberately put something we don't expect in the buffers
	3140	@memset(mem.sliceAsBytes(&buffers), 1);
	3141
	3142	{
	3143	const sqe = try ring.recv(0xdfdfdfdf, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
	3144	try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
	3145	try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
	3146	try testing.expectEqual(@as(u64, 0), sqe.addr);
	3147	try testing.expectEqual(@as(u32, buffer_len), sqe.len);
	3148	try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
	3149	try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
	3150	try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
	3151	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3152
	3153	const cqe = try ring.copy_cqe();
	3154	switch (cqe.err()) {
	3155	.SUCCESS => {},
	3156	else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
	3157	}
	3158
	3159	try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
	3160	const used_buffer_id = cqe.flags >> 16;
	3161	try testing.expectEqual(used_buffer_id, reprovided_buffer_id);
	3162	try testing.expectEqual(@as(i32, buffer_len), cqe.res);
	3163	try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
	3164	const buffer = buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))];
	3165	try testing.expectEqualSlices(u8, &([_]u8{'w'} ** buffer_len), buffer);
	3166	}
	3167	}
	3168
	3169	/// Used for testing server/client interactions.
	3170	const SocketTestHarness = struct {
	3171	listener: os.socket_t,
	3172	server: os.socket_t,
	3173	client: os.socket_t,
	3174
	3175	fn close(self: SocketTestHarness) void {
	3176	posix.close(self.client);
	3177	posix.close(self.listener);
	3178	}
	3179	};
	3180
	3181	fn createSocketTestHarness(ring: *IoUring) !SocketTestHarness {
	3182	// Create a TCP server socket
	3183	var address = try net.Address.parseIp4("127.0.0.1", 0);
	3184	const listener_socket = try createListenerSocket(&address);
	3185	errdefer posix.close(listener_socket);
	3186
	3187	// Submit 1 accept
	3188	var accept_addr: os.sockaddr = undefined;
	3189	var accept_addr_len: os.socklen_t = @sizeOf(@TypeOf(accept_addr));
	3190	_ = try ring.accept(0xaaaaaaaa, listener_socket, &accept_addr, &accept_addr_len, 0);
	3191
	3192	// Create a TCP client socket
	3193	const client = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
	3194	errdefer posix.close(client);
	3195	_ = try ring.connect(0xcccccccc, client, &address.any, address.getOsSockLen());
	3196
	3197	try testing.expectEqual(@as(u32, 2), try ring.submit());
	3198
	3199	var cqe_accept = try ring.copy_cqe();
	3200	if (cqe_accept.err() == .INVAL) return error.SkipZigTest;
	3201	var cqe_connect = try ring.copy_cqe();
	3202	if (cqe_connect.err() == .INVAL) return error.SkipZigTest;
	3203
	3204	// The accept/connect CQEs may arrive in any order, the connect CQE will sometimes come first:
	3205	if (cqe_accept.user_data == 0xcccccccc and cqe_connect.user_data == 0xaaaaaaaa) {
	3206	const a = cqe_accept;
	3207	const b = cqe_connect;
	3208	cqe_accept = b;
	3209	cqe_connect = a;
	3210	}
	3211
	3212	try testing.expectEqual(@as(u64, 0xaaaaaaaa), cqe_accept.user_data);
	3213	if (cqe_accept.res <= 0) std.debug.print("\ncqe_accept.res={}\n", .{cqe_accept.res});
	3214	try testing.expect(cqe_accept.res > 0);
	3215	try testing.expectEqual(@as(u32, 0), cqe_accept.flags);
	3216	try testing.expectEqual(linux.io_uring_cqe{
	3217	.user_data = 0xcccccccc,
	3218	.res = 0,
	3219	.flags = 0,
	3220	}, cqe_connect);
	3221
	3222	// All good
	3223
	3224	return SocketTestHarness{
	3225	.listener = listener_socket,
	3226	.server = cqe_accept.res,
	3227	.client = client,
	3228	};
	3229	}
	3230
	3231	fn createListenerSocket(address: *net.Address) !os.socket_t {
	3232	const kernel_backlog = 1;
	3233	const listener_socket = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
	3234	errdefer posix.close(listener_socket);
	3235
	3236	try os.setsockopt(listener_socket, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
	3237	try os.bind(listener_socket, &address.any, address.getOsSockLen());
	3238	try os.listen(listener_socket, kernel_backlog);
	3239
	3240	// set address to the OS-chosen IP/port.
	3241	var slen: os.socklen_t = address.getOsSockLen();
	3242	try os.getsockname(listener_socket, &address.any, &slen);
	3243
	3244	return listener_socket;
	3245	}
	3246
	3247	test "accept multishot" {
	3248	if (builtin.os.tag != .linux) return error.SkipZigTest;
	3249
	3250	var ring = IoUring.init(16, 0) catch \|err\| switch (err) {
	3251	error.SystemOutdated => return error.SkipZigTest,
	3252	error.PermissionDenied => return error.SkipZigTest,
	3253	else => return err,
	3254	};
	3255	defer ring.deinit();
	3256
	3257	var address = try net.Address.parseIp4("127.0.0.1", 0);
	3258	const listener_socket = try createListenerSocket(&address);
	3259	defer posix.close(listener_socket);
	3260
	3261	// submit multishot accept operation
	3262	var addr: os.sockaddr = undefined;
	3263	var addr_len: os.socklen_t = @sizeOf(@TypeOf(addr));
	3264	const userdata: u64 = 0xaaaaaaaa;
	3265	_ = try ring.accept_multishot(userdata, listener_socket, &addr, &addr_len, 0);
	3266	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3267
	3268	var nr: usize = 4; // number of clients to connect
	3269	while (nr > 0) : (nr -= 1) {
	3270	// connect client
	3271	const client = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
	3272	errdefer posix.close(client);
	3273	try os.connect(client, &address.any, address.getOsSockLen());
	3274
	3275	// test accept completion
	3276	var cqe = try ring.copy_cqe();
	3277	if (cqe.err() == .INVAL) return error.SkipZigTest;
	3278	try testing.expect(cqe.res > 0);
	3279	try testing.expect(cqe.user_data == userdata);
	3280	try testing.expect(cqe.flags & linux.IORING_CQE_F_MORE > 0); // more flag is set
	3281
	3282	posix.close(client);
	3283	}
	3284	}
	3285
	3286	test "accept/connect/send_zc/recv" {
	3287	try skipKernelLessThan(.{ .major = 6, .minor = 0, .patch = 0 });
	3288
	3289	var ring = IoUring.init(16, 0) catch \|err\| switch (err) {
	3290	error.SystemOutdated => return error.SkipZigTest,
	3291	error.PermissionDenied => return error.SkipZigTest,
	3292	else => return err,
	3293	};
	3294	defer ring.deinit();
	3295
	3296	const socket_test_harness = try createSocketTestHarness(&ring);
	3297	defer socket_test_harness.close();
	3298
	3299	const buffer_send = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
	3300	var buffer_recv = [_]u8{0} ** 10;
	3301
	3302	// zero-copy send
	3303	const sqe_send = try ring.send_zc(0xeeeeeeee, socket_test_harness.client, buffer_send[0..], 0, 0);
	3304	sqe_send.flags \|= linux.IOSQE_IO_LINK;
	3305	_ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
	3306	try testing.expectEqual(@as(u32, 2), try ring.submit());
	3307
	3308	// First completion of zero-copy send.
	3309	// IORING_CQE_F_MORE, means that there
	3310	// will be a second completion event / notification for the
	3311	// request, with the user_data field set to the same value.
	3312	// buffer_send must be keep alive until second cqe.
	3313	var cqe_send = try ring.copy_cqe();
	3314	try testing.expectEqual(linux.io_uring_cqe{
	3315	.user_data = 0xeeeeeeee,
	3316	.res = buffer_send.len,
	3317	.flags = linux.IORING_CQE_F_MORE,
	3318	}, cqe_send);
	3319
	3320	const cqe_recv = try ring.copy_cqe();
	3321	try testing.expectEqual(linux.io_uring_cqe{
	3322	.user_data = 0xffffffff,
	3323	.res = buffer_recv.len,
	3324	.flags = cqe_recv.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
	3325	}, cqe_recv);
	3326
	3327	try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
	3328
	3329	// Second completion of zero-copy send.
	3330	// IORING_CQE_F_NOTIF in flags signals that kernel is done with send_buffer
	3331	cqe_send = try ring.copy_cqe();
	3332	try testing.expectEqual(linux.io_uring_cqe{
	3333	.user_data = 0xeeeeeeee,
	3334	.res = 0,
	3335	.flags = linux.IORING_CQE_F_NOTIF,
	3336	}, cqe_send);
	3337	}
	3338
	3339	test "accept_direct" {
	3340	try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
	3341
	3342	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	3343	error.SystemOutdated => return error.SkipZigTest,
	3344	error.PermissionDenied => return error.SkipZigTest,
	3345	else => return err,
	3346	};
	3347	defer ring.deinit();
	3348	var address = try net.Address.parseIp4("127.0.0.1", 0);
	3349
	3350	// register direct file descriptors
	3351	var registered_fds = [_]os.fd_t{-1} ** 2;
	3352	try ring.register_files(registered_fds[0..]);
	3353
	3354	const listener_socket = try createListenerSocket(&address);
	3355	defer posix.close(listener_socket);
	3356
	3357	const accept_userdata: u64 = 0xaaaaaaaa;
	3358	const read_userdata: u64 = 0xbbbbbbbb;
	3359	const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
	3360
	3361	for (0..2) \|_\| {
	3362	for (registered_fds, 0..) \|_, i\| {
	3363	var buffer_recv = [_]u8{0} ** 16;
	3364	const buffer_send: []const u8 = data[0 .. data.len - i]; // make it different at each loop
	3365
	3366	// submit accept, will chose registered fd and return index in cqe
	3367	_ = try ring.accept_direct(accept_userdata, listener_socket, null, null, 0);
	3368	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3369
	3370	// connect
	3371	const client = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
	3372	try os.connect(client, &address.any, address.getOsSockLen());
	3373	defer posix.close(client);
	3374
	3375	// accept completion
	3376	const cqe_accept = try ring.copy_cqe();
	3377	try testing.expectEqual(os.E.SUCCESS, cqe_accept.err());
	3378	const fd_index = cqe_accept.res;
	3379	try testing.expect(fd_index < registered_fds.len);
	3380	try testing.expect(cqe_accept.user_data == accept_userdata);
	3381
	3382	// send data
	3383	_ = try os.send(client, buffer_send, 0);
	3384
	3385	// Example of how to use registered fd:
	3386	// Submit receive to fixed file returned by accept (fd_index).
	3387	// Fd field is set to registered file index, returned by accept.
	3388	// Flag linux.IOSQE_FIXED_FILE must be set.
	3389	const recv_sqe = try ring.recv(read_userdata, fd_index, .{ .buffer = &buffer_recv }, 0);
	3390	recv_sqe.flags \|= linux.IOSQE_FIXED_FILE;
	3391	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3392
	3393	// accept receive
	3394	const recv_cqe = try ring.copy_cqe();
	3395	try testing.expect(recv_cqe.user_data == read_userdata);
	3396	try testing.expect(recv_cqe.res == buffer_send.len);
	3397	try testing.expectEqualSlices(u8, buffer_send, buffer_recv[0..buffer_send.len]);
	3398	}
	3399	// no more available fds, accept will get NFILE error
	3400	{
	3401	// submit accept
	3402	_ = try ring.accept_direct(accept_userdata, listener_socket, null, null, 0);
	3403	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3404	// connect
	3405	const client = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
	3406	try os.connect(client, &address.any, address.getOsSockLen());
	3407	defer posix.close(client);
	3408	// completion with error
	3409	const cqe_accept = try ring.copy_cqe();
	3410	try testing.expect(cqe_accept.user_data == accept_userdata);
	3411	try testing.expectEqual(os.E.NFILE, cqe_accept.err());
	3412	}
	3413	// return file descriptors to kernel
	3414	try ring.register_files_update(0, registered_fds[0..]);
	3415	}
	3416	try ring.unregister_files();
	3417	}
	3418
	3419	test "accept_multishot_direct" {
	3420	try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
	3421
	3422	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	3423	error.SystemOutdated => return error.SkipZigTest,
	3424	error.PermissionDenied => return error.SkipZigTest,
	3425	else => return err,
	3426	};
	3427	defer ring.deinit();
	3428
	3429	var address = try net.Address.parseIp4("127.0.0.1", 0);
	3430
	3431	var registered_fds = [_]os.fd_t{-1} ** 2;
	3432	try ring.register_files(registered_fds[0..]);
	3433
	3434	const listener_socket = try createListenerSocket(&address);
	3435	defer posix.close(listener_socket);
	3436
	3437	const accept_userdata: u64 = 0xaaaaaaaa;
	3438
	3439	for (0..2) \|_\| {
	3440	// submit multishot accept
	3441	// Will chose registered fd and return index of the selected registered file in cqe.
	3442	_ = try ring.accept_multishot_direct(accept_userdata, listener_socket, null, null, 0);
	3443	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3444
	3445	for (registered_fds) \|_\| {
	3446	// connect
	3447	const client = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
	3448	try os.connect(client, &address.any, address.getOsSockLen());
	3449	defer posix.close(client);
	3450
	3451	// accept completion
	3452	const cqe_accept = try ring.copy_cqe();
	3453	const fd_index = cqe_accept.res;
	3454	try testing.expect(fd_index < registered_fds.len);
	3455	try testing.expect(cqe_accept.user_data == accept_userdata);
	3456	try testing.expect(cqe_accept.flags & linux.IORING_CQE_F_MORE > 0); // has more is set
	3457	}
	3458	// No more available fds, accept will get NFILE error.
	3459	// Multishot is terminated (more flag is not set).
	3460	{
	3461	// connect
	3462	const client = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
	3463	try os.connect(client, &address.any, address.getOsSockLen());
	3464	defer posix.close(client);
	3465	// completion with error
	3466	const cqe_accept = try ring.copy_cqe();
	3467	try testing.expect(cqe_accept.user_data == accept_userdata);
	3468	try testing.expectEqual(os.E.NFILE, cqe_accept.err());
	3469	try testing.expect(cqe_accept.flags & linux.IORING_CQE_F_MORE == 0); // has more is not set
	3470	}
	3471	// return file descriptors to kernel
	3472	try ring.register_files_update(0, registered_fds[0..]);
	3473	}
	3474	try ring.unregister_files();
	3475	}
	3476
	3477	test "socket" {
	3478	try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
	3479
	3480	var ring = IoUring.init(1, 0) catch \|err\| switch (err) {
	3481	error.SystemOutdated => return error.SkipZigTest,
	3482	error.PermissionDenied => return error.SkipZigTest,
	3483	else => return err,
	3484	};
	3485	defer ring.deinit();
	3486
	3487	// prepare, submit socket operation
	3488	_ = try ring.socket(0, linux.AF.INET, os.SOCK.STREAM, 0, 0);
	3489	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3490
	3491	// test completion
	3492	var cqe = try ring.copy_cqe();
	3493	try testing.expectEqual(os.E.SUCCESS, cqe.err());
	3494	const fd: os.fd_t = @intCast(cqe.res);
	3495	try testing.expect(fd > 2);
	3496
	3497	os.close(fd);
	3498	}
	3499
	3500	test "socket_direct/socket_direct_alloc/close_direct" {
	3501	try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
	3502
	3503	var ring = IoUring.init(2, 0) catch \|err\| switch (err) {
	3504	error.SystemOutdated => return error.SkipZigTest,
	3505	error.PermissionDenied => return error.SkipZigTest,
	3506	else => return err,
	3507	};
	3508	defer ring.deinit();
	3509
	3510	var registered_fds = [_]os.fd_t{-1} ** 3;
	3511	try ring.register_files(registered_fds[0..]);
	3512
	3513	// create socket in registered file descriptor at index 0 (last param)
	3514	_ = try ring.socket_direct(0, linux.AF.INET, os.SOCK.STREAM, 0, 0, 0);
	3515	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3516	var cqe_socket = try ring.copy_cqe();
	3517	try testing.expectEqual(os.E.SUCCESS, cqe_socket.err());
	3518	try testing.expect(cqe_socket.res == 0);
	3519
	3520	// create socket in registered file descriptor at index 1 (last param)
	3521	_ = try ring.socket_direct(0, linux.AF.INET, os.SOCK.STREAM, 0, 0, 1);
	3522	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3523	cqe_socket = try ring.copy_cqe();
	3524	try testing.expectEqual(os.E.SUCCESS, cqe_socket.err());
	3525	try testing.expect(cqe_socket.res == 0); // res is 0 when index is specified
	3526
	3527	// create socket in kernel chosen file descriptor index (_alloc version)
	3528	// completion res has index from registered files
	3529	_ = try ring.socket_direct_alloc(0, linux.AF.INET, os.SOCK.STREAM, 0, 0);
	3530	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3531	cqe_socket = try ring.copy_cqe();
	3532	try testing.expectEqual(os.E.SUCCESS, cqe_socket.err());
	3533	try testing.expect(cqe_socket.res == 2); // returns registered file index
	3534
	3535	// use sockets from registered_fds in connect operation
	3536	var address = try net.Address.parseIp4("127.0.0.1", 0);
	3537	const listener_socket = try createListenerSocket(&address);
	3538	defer posix.close(listener_socket);
	3539	const accept_userdata: u64 = 0xaaaaaaaa;
	3540	const connect_userdata: u64 = 0xbbbbbbbb;
	3541	const close_userdata: u64 = 0xcccccccc;
	3542	for (registered_fds, 0..) \|_, fd_index\| {
	3543	// prepare accept
	3544	_ = try ring.accept(accept_userdata, listener_socket, null, null, 0);
	3545	// prepare connect with fixed socket
	3546	const connect_sqe = try ring.connect(connect_userdata, @intCast(fd_index), &address.any, address.getOsSockLen());
	3547	connect_sqe.flags \|= linux.IOSQE_FIXED_FILE; // fd is fixed file index
	3548	// submit both
	3549	try testing.expectEqual(@as(u32, 2), try ring.submit());
	3550	// get completions
	3551	var cqe_connect = try ring.copy_cqe();
	3552	var cqe_accept = try ring.copy_cqe();
	3553	// ignore order
	3554	if (cqe_connect.user_data == accept_userdata and cqe_accept.user_data == connect_userdata) {
	3555	const a = cqe_accept;
	3556	const b = cqe_connect;
	3557	cqe_accept = b;
	3558	cqe_connect = a;
	3559	}
	3560	// test connect completion
	3561	try testing.expect(cqe_connect.user_data == connect_userdata);
	3562	try testing.expectEqual(os.E.SUCCESS, cqe_connect.err());
	3563	// test accept completion
	3564	try testing.expect(cqe_accept.user_data == accept_userdata);
	3565	try testing.expectEqual(os.E.SUCCESS, cqe_accept.err());
	3566
	3567	// submit and test close_direct
	3568	_ = try ring.close_direct(close_userdata, @intCast(fd_index));
	3569	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3570	var cqe_close = try ring.copy_cqe();
	3571	try testing.expect(cqe_close.user_data == close_userdata);
	3572	try testing.expectEqual(os.E.SUCCESS, cqe_close.err());
	3573	}
	3574
	3575	try ring.unregister_files();
	3576	}
	3577
	3578	test "openat_direct/close_direct" {
	3579	try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
	3580
	3581	var ring = IoUring.init(2, 0) catch \|err\| switch (err) {
	3582	error.SystemOutdated => return error.SkipZigTest,
	3583	error.PermissionDenied => return error.SkipZigTest,
	3584	else => return err,
	3585	};
	3586	defer ring.deinit();
	3587
	3588	var registered_fds = [_]os.fd_t{-1} ** 3;
	3589	try ring.register_files(registered_fds[0..]);
	3590
	3591	var tmp = std.testing.tmpDir(.{});
	3592	defer tmp.cleanup();
	3593	const path = "test_io_uring_close_direct";
	3594	const flags: linux.O = .{ .ACCMODE = .RDWR, .CREAT = true };
	3595	const mode: os.mode_t = 0o666;
	3596	const user_data: u64 = 0;
	3597
	3598	// use registered file at index 0 (last param)
	3599	_ = try ring.openat_direct(user_data, tmp.dir.fd, path, flags, mode, 0);
	3600	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3601	var cqe = try ring.copy_cqe();
	3602	try testing.expectEqual(os.E.SUCCESS, cqe.err());
	3603	try testing.expect(cqe.res == 0);
	3604
	3605	// use registered file at index 1
	3606	_ = try ring.openat_direct(user_data, tmp.dir.fd, path, flags, mode, 1);
	3607	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3608	cqe = try ring.copy_cqe();
	3609	try testing.expectEqual(os.E.SUCCESS, cqe.err());
	3610	try testing.expect(cqe.res == 0); // res is 0 when we specify index
	3611
	3612	// let kernel choose registered file index
	3613	_ = try ring.openat_direct(user_data, tmp.dir.fd, path, flags, mode, linux.IORING_FILE_INDEX_ALLOC);
	3614	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3615	cqe = try ring.copy_cqe();
	3616	try testing.expectEqual(os.E.SUCCESS, cqe.err());
	3617	try testing.expect(cqe.res == 2); // chosen index is in res
	3618
	3619	// close all open file descriptors
	3620	for (registered_fds, 0..) \|_, fd_index\| {
	3621	_ = try ring.close_direct(user_data, @intCast(fd_index));
	3622	try testing.expectEqual(@as(u32, 1), try ring.submit());
	3623	var cqe_close = try ring.copy_cqe();
	3624	try testing.expectEqual(os.E.SUCCESS, cqe_close.err());
	3625	}
	3626	try ring.unregister_files();
	3627	}
	3628
	3629	test "waitid" {
	3630	try skipKernelLessThan(.{ .major = 6, .minor = 7, .patch = 0 });
	3631
	3632	var ring = IoUring.init(16, 0) catch \|err\| switch (err) {
	3633	error.SystemOutdated => return error.SkipZigTest,
	3634	error.PermissionDenied => return error.SkipZigTest,
	3635	else => return err,
	3636	};
	3637	defer ring.deinit();
	3638
	3639	const pid = try os.fork();
	3640	if (pid == 0) {
	3641	os.exit(7);
	3642	}
	3643
	3644	var siginfo: os.siginfo_t = undefined;
	3645	_ = try ring.waitid(0, .PID, pid, &siginfo, os.W.EXITED, 0);
	3646
	3647	try testing.expectEqual(1, try ring.submit());
	3648
	3649	const cqe_waitid = try ring.copy_cqe();
	3650	try testing.expectEqual(0, cqe_waitid.res);
	3651	try testing.expectEqual(pid, siginfo.fields.common.first.piduid.pid);
	3652	try testing.expectEqual(7, siginfo.fields.common.second.sigchld.status);
	3653	}
	3654
	3655	/// For use in tests. Returns SkipZigTest is kernel version is less than required.
	3656	inline fn skipKernelLessThan(required: std.SemanticVersion) !void {
	3657	if (builtin.os.tag != .linux) return error.SkipZigTest;
	3658
	3659	var uts: linux.utsname = undefined;
	3660	const res = linux.uname(&uts);
	3661	switch (linux.getErrno(res)) {
	3662	.SUCCESS => {},
	3663	else => \|errno\| return os.unexpectedErrno(errno),
	3664	}
	3665
	3666	const release = mem.sliceTo(&uts.release, 0);
	3667	var current = try std.SemanticVersion.parse(release);
	3668	current.pre = null; // don't check pre field
	3669	if (required.order(current) == .gt) return error.SkipZigTest;
	3670	}

lib/std/os/linux/io_uring.zig deleted-4228

...	...	@@ -1,4228 +0,0 @@
1		const std = @import("../../std.zig");
2		const builtin = @import("builtin");
3		const assert = std.debug.assert;
4		const mem = std.mem;
5		const net = std.net;
6		const os = std.os;
7		const posix = std.posix;
8		const linux = os.linux;
9		const testing = std.testing;
10
11		pub const IO_Uring = struct {
12		fd: os.fd_t = -1,
13		sq: SubmissionQueue,
14		cq: CompletionQueue,
15		flags: u32,
16		features: u32,
17
18		/// A friendly way to setup an io_uring, with default linux.io_uring_params.
19		/// `entries` must be a power of two between 1 and 32768, although the kernel will make the final
20		/// call on how many entries the submission and completion queues will ultimately have,
21		/// see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L8027-L8050.
22		/// Matches the interface of io_uring_queue_init() in liburing.
23		pub fn init(entries: u16, flags: u32) !IO_Uring {
24		var params = mem.zeroInit(linux.io_uring_params, .{
25		.flags = flags,
26		.sq_thread_idle = 1000,
27		});
28		return try IO_Uring.init_params(entries, &params);
29		}
30
31		/// A powerful way to setup an io_uring, if you want to tweak linux.io_uring_params such as submission
32		/// queue thread cpu affinity or thread idle timeout (the kernel and our default is 1 second).
33		/// `params` is passed by reference because the kernel needs to modify the parameters.
34		/// Matches the interface of io_uring_queue_init_params() in liburing.
35		pub fn init_params(entries: u16, p: *linux.io_uring_params) !IO_Uring {
36		if (entries == 0) return error.EntriesZero;
37		if (!std.math.isPowerOfTwo(entries)) return error.EntriesNotPowerOfTwo;
38
39		assert(p.sq_entries == 0);
40		assert(p.cq_entries == 0 or p.flags & linux.IORING_SETUP_CQSIZE != 0);
41		assert(p.features == 0);
42		assert(p.wq_fd == 0 or p.flags & linux.IORING_SETUP_ATTACH_WQ != 0);
43		assert(p.resv[0] == 0);
44		assert(p.resv[1] == 0);
45		assert(p.resv[2] == 0);
46
47		const res = linux.io_uring_setup(entries, p);
48		switch (linux.getErrno(res)) {
49		.SUCCESS => {},
50		.FAULT => return error.ParamsOutsideAccessibleAddressSpace,
51		// The resv array contains non-zero data, p.flags contains an unsupported flag,
52		// entries out of bounds, IORING_SETUP_SQ_AFF was specified without IORING_SETUP_SQPOLL,
53		// or IORING_SETUP_CQSIZE was specified but linux.io_uring_params.cq_entries was invalid:
54		.INVAL => return error.ArgumentsInvalid,
55		.MFILE => return error.ProcessFdQuotaExceeded,
56		.NFILE => return error.SystemFdQuotaExceeded,
57		.NOMEM => return error.SystemResources,
58		// IORING_SETUP_SQPOLL was specified but effective user ID lacks sufficient privileges,
59		// or a container seccomp policy prohibits io_uring syscalls:
60		.PERM => return error.PermissionDenied,
61		.NOSYS => return error.SystemOutdated,
62		else => \|errno\| return os.unexpectedErrno(errno),
63		}
64		const fd = @as(os.fd_t, @intCast(res));
65		assert(fd >= 0);
66		errdefer os.close(fd);
67
68		// Kernel versions 5.4 and up use only one mmap() for the submission and completion queues.
69		// This is not an optional feature for us... if the kernel does it, we have to do it.
70		// The thinking on this by the kernel developers was that both the submission and the
71		// completion queue rings have sizes just over a power of two, but the submission queue ring
72		// is significantly smaller with u32 slots. By bundling both in a single mmap, the kernel
73		// gets the submission queue ring for free.
74		// See https://patchwork.kernel.org/patch/11115257 for the kernel patch.
75		// We do not support the double mmap() done before 5.4, because we want to keep the
76		// init/deinit mmap paths simple and because io_uring has had many bug fixes even since 5.4.
77		if ((p.features & linux.IORING_FEAT_SINGLE_MMAP) == 0) {
78		return error.SystemOutdated;
79		}
80
81		// Check that the kernel has actually set params and that "impossible is nothing".
82		assert(p.sq_entries != 0);
83		assert(p.cq_entries != 0);
84		assert(p.cq_entries >= p.sq_entries);
85
86		// From here on, we only need to read from params, so pass `p` by value as immutable.
87		// The completion queue shares the mmap with the submission queue, so pass `sq` there too.
88		var sq = try SubmissionQueue.init(fd, p.*);
89		errdefer sq.deinit();
90		var cq = try CompletionQueue.init(fd, p.*, sq);
91		errdefer cq.deinit();
92
93		// Check that our starting state is as we expect.
94		assert(sq.head.* == 0);
95		assert(sq.tail.* == 0);
96		assert(sq.mask == p.sq_entries - 1);
97		// Allow flags.* to be non-zero, since the kernel may set IORING_SQ_NEED_WAKEUP at any time.
98		assert(sq.dropped.* == 0);
99		assert(sq.array.len == p.sq_entries);
100		assert(sq.sqes.len == p.sq_entries);
101		assert(sq.sqe_head == 0);
102		assert(sq.sqe_tail == 0);
103
104		assert(cq.head.* == 0);
105		assert(cq.tail.* == 0);
106		assert(cq.mask == p.cq_entries - 1);
107		assert(cq.overflow.* == 0);
108		assert(cq.cqes.len == p.cq_entries);
109
110		return IO_Uring{
111		.fd = fd,
112		.sq = sq,
113		.cq = cq,
114		.flags = p.flags,
115		.features = p.features,
116		};
117		}
118
119		pub fn deinit(self: *IO_Uring) void {
120		assert(self.fd >= 0);
121		// The mmaps depend on the fd, so the order of these calls is important:
122		self.cq.deinit();
123		self.sq.deinit();
124		os.close(self.fd);
125		self.fd = -1;
126		}
127
128		/// Returns a pointer to a vacant SQE, or an error if the submission queue is full.
129		/// We follow the implementation (and atomics) of liburing's `io_uring_get_sqe()` exactly.
130		/// However, instead of a null we return an error to force safe handling.
131		/// Any situation where the submission queue is full tends more towards a control flow error,
132		/// and the null return in liburing is more a C idiom than anything else, for lack of a better
133		/// alternative. In Zig, we have first-class error handling... so let's use it.
134		/// Matches the implementation of io_uring_get_sqe() in liburing.
135		pub fn get_sqe(self: IO_Uring) !linux.io_uring_sqe {
136		const head = @atomicLoad(u32, self.sq.head, .Acquire);
137		// Remember that these head and tail offsets wrap around every four billion operations.
138		// We must therefore use wrapping addition and subtraction to avoid a runtime crash.
139		const next = self.sq.sqe_tail +% 1;
140		if (next -% head > self.sq.sqes.len) return error.SubmissionQueueFull;
141		const sqe = &self.sq.sqes[self.sq.sqe_tail & self.sq.mask];
142		self.sq.sqe_tail = next;
143		return sqe;
144		}
145
146		/// Submits the SQEs acquired via get_sqe() to the kernel. You can call this once after you have
147		/// called get_sqe() multiple times to setup multiple I/O requests.
148		/// Returns the number of SQEs submitted, if not used alongside IORING_SETUP_SQPOLL.
149		/// If the io_uring instance is uses IORING_SETUP_SQPOLL, the value returned on success is not
150		/// guaranteed to match the amount of actually submitted sqes during this call. A value higher
151		/// or lower, including 0, may be returned.
152		/// Matches the implementation of io_uring_submit() in liburing.
153		pub fn submit(self: *IO_Uring) !u32 {
154		return self.submit_and_wait(0);
155		}
156
157		/// Like submit(), but allows waiting for events as well.
158		/// Returns the number of SQEs submitted.
159		/// Matches the implementation of io_uring_submit_and_wait() in liburing.
160		pub fn submit_and_wait(self: *IO_Uring, wait_nr: u32) !u32 {
161		const submitted = self.flush_sq();
162		var flags: u32 = 0;
163		if (self.sq_ring_needs_enter(&flags) or wait_nr > 0) {
164		if (wait_nr > 0 or (self.flags & linux.IORING_SETUP_IOPOLL) != 0) {
165		flags \|= linux.IORING_ENTER_GETEVENTS;
166		}
167		return try self.enter(submitted, wait_nr, flags);
168		}
169		return submitted;
170		}
171
172		/// Tell the kernel we have submitted SQEs and/or want to wait for CQEs.
173		/// Returns the number of SQEs submitted.
174		pub fn enter(self: *IO_Uring, to_submit: u32, min_complete: u32, flags: u32) !u32 {
175		assert(self.fd >= 0);
176		const res = linux.io_uring_enter(self.fd, to_submit, min_complete, flags, null);
177		switch (linux.getErrno(res)) {
178		.SUCCESS => {},
179		// The kernel was unable to allocate memory or ran out of resources for the request.
180		// The application should wait for some completions and try again:
181		.AGAIN => return error.SystemResources,
182		// The SQE `fd` is invalid, or IOSQE_FIXED_FILE was set but no files were registered:
183		.BADF => return error.FileDescriptorInvalid,
184		// The file descriptor is valid, but the ring is not in the right state.
185		// See io_uring_register(2) for how to enable the ring.
186		.BADFD => return error.FileDescriptorInBadState,
187		// The application attempted to overcommit the number of requests it can have pending.
188		// The application should wait for some completions and try again:
189		.BUSY => return error.CompletionQueueOvercommitted,
190		// The SQE is invalid, or valid but the ring was setup with IORING_SETUP_IOPOLL:
191		.INVAL => return error.SubmissionQueueEntryInvalid,
192		// The buffer is outside the process' accessible address space, or IORING_OP_READ_FIXED
193		// or IORING_OP_WRITE_FIXED was specified but no buffers were registered, or the range
194		// described by `addr` and `len` is not within the buffer registered at `buf_index`:
195		.FAULT => return error.BufferInvalid,
196		.NXIO => return error.RingShuttingDown,
197		// The kernel believes our `self.fd` does not refer to an io_uring instance,
198		// or the opcode is valid but not supported by this kernel (more likely):
199		.OPNOTSUPP => return error.OpcodeNotSupported,
200		// The operation was interrupted by a delivery of a signal before it could complete.
201		// This can happen while waiting for events with IORING_ENTER_GETEVENTS:
202		.INTR => return error.SignalInterrupt,
203		else => \|errno\| return os.unexpectedErrno(errno),
204		}
205		return @as(u32, @intCast(res));
206		}
207
208		/// Sync internal state with kernel ring state on the SQ side.
209		/// Returns the number of all pending events in the SQ ring, for the shared ring.
210		/// This return value includes previously flushed SQEs, as per liburing.
211		/// The rationale is to suggest that an io_uring_enter() call is needed rather than not.
212		/// Matches the implementation of __io_uring_flush_sq() in liburing.
213		pub fn flush_sq(self: *IO_Uring) u32 {
214		if (self.sq.sqe_head != self.sq.sqe_tail) {
215		// Fill in SQEs that we have queued up, adding them to the kernel ring.
216		const to_submit = self.sq.sqe_tail -% self.sq.sqe_head;
217		var tail = self.sq.tail.*;
218		var i: usize = 0;
219		while (i < to_submit) : (i += 1) {
220		self.sq.array[tail & self.sq.mask] = self.sq.sqe_head & self.sq.mask;
221		tail +%= 1;
222		self.sq.sqe_head +%= 1;
223		}
224		// Ensure that the kernel can actually see the SQE updates when it sees the tail update.
225		@atomicStore(u32, self.sq.tail, tail, .Release);
226		}
227		return self.sq_ready();
228		}
229
230		/// Returns true if we are not using an SQ thread (thus nobody submits but us),
231		/// or if IORING_SQ_NEED_WAKEUP is set and the SQ thread must be explicitly awakened.
232		/// For the latter case, we set the SQ thread wakeup flag.
233		/// Matches the implementation of sq_ring_needs_enter() in liburing.
234		pub fn sq_ring_needs_enter(self: IO_Uring, flags: u32) bool {
235		assert(flags.* == 0);
236		if ((self.flags & linux.IORING_SETUP_SQPOLL) == 0) return true;
237		if ((@atomicLoad(u32, self.sq.flags, .Unordered) & linux.IORING_SQ_NEED_WAKEUP) != 0) {
238		flags.* \|= linux.IORING_ENTER_SQ_WAKEUP;
239		return true;
240		}
241		return false;
242		}
243
244		/// Returns the number of flushed and unflushed SQEs pending in the submission queue.
245		/// In other words, this is the number of SQEs in the submission queue, i.e. its length.
246		/// These are SQEs that the kernel is yet to consume.
247		/// Matches the implementation of io_uring_sq_ready in liburing.
248		pub fn sq_ready(self: *IO_Uring) u32 {
249		// Always use the shared ring state (i.e. head and not sqe_head) to avoid going out of sync,
250		// see https://github.com/axboe/liburing/issues/92.
251		return self.sq.sqe_tail -% @atomicLoad(u32, self.sq.head, .Acquire);
252		}
253
254		/// Returns the number of CQEs in the completion queue, i.e. its length.
255		/// These are CQEs that the application is yet to consume.
256		/// Matches the implementation of io_uring_cq_ready in liburing.
257		pub fn cq_ready(self: *IO_Uring) u32 {
258		return @atomicLoad(u32, self.cq.tail, .Acquire) -% self.cq.head.*;
259		}
260
261		/// Copies as many CQEs as are ready, and that can fit into the destination `cqes` slice.
262		/// If none are available, enters into the kernel to wait for at most `wait_nr` CQEs.
263		/// Returns the number of CQEs copied, advancing the CQ ring.
264		/// Provides all the wait/peek methods found in liburing, but with batching and a single method.
265		/// The rationale for copying CQEs rather than copying pointers is that pointers are 8 bytes
266		/// whereas CQEs are not much more at only 16 bytes, and this provides a safer faster interface.
267		/// Safer, because you no longer need to call cqe_seen(), avoiding idempotency bugs.
268		/// Faster, because we can now amortize the atomic store release to `cq.head` across the batch.
269		/// See https://github.com/axboe/liburing/issues/103#issuecomment-686665007.
270		/// Matches the implementation of io_uring_peek_batch_cqe() in liburing, but supports waiting.
271		pub fn copy_cqes(self: *IO_Uring, cqes: []linux.io_uring_cqe, wait_nr: u32) !u32 {
272		const count = self.copy_cqes_ready(cqes);
273		if (count > 0) return count;
274		if (self.cq_ring_needs_flush() or wait_nr > 0) {
275		_ = try self.enter(0, wait_nr, linux.IORING_ENTER_GETEVENTS);
276		return self.copy_cqes_ready(cqes);
277		}
278		return 0;
279		}
280
281		fn copy_cqes_ready(self: *IO_Uring, cqes: []linux.io_uring_cqe) u32 {
282		const ready = self.cq_ready();
283		const count = @min(cqes.len, ready);
284		const head = self.cq.head.* & self.cq.mask;
285		const tail = (self.cq.head.* +% count) & self.cq.mask;
286
287		if (head <= tail) {
288		// head behind tail -> no wrapping
289		@memcpy(cqes[0..count], self.cq.cqes[head..tail]);
290		} else {
291		// head in front of tail -> buffer wraps
292		const two_copies_required: bool = self.cq.cqes.len - head < count;
293		const amount_to_copy_in_first = if (two_copies_required) self.cq.cqes.len - head else count;
294		@memcpy(cqes[0..amount_to_copy_in_first], self.cq.cqes[head .. head + amount_to_copy_in_first]);
295		if (two_copies_required) {
296		@memcpy(cqes[amount_to_copy_in_first..count], self.cq.cqes[0..tail]);
297		}
298		}
299
300		self.cq_advance(count);
301		return count;
302		}
303
304		/// Returns a copy of an I/O completion, waiting for it if necessary, and advancing the CQ ring.
305		/// A convenience method for `copy_cqes()` for when you don't need to batch or peek.
306		pub fn copy_cqe(ring: *IO_Uring) !linux.io_uring_cqe {
307		var cqes: [1]linux.io_uring_cqe = undefined;
308		while (true) {
309		const count = try ring.copy_cqes(&cqes, 1);
310		if (count > 0) return cqes[0];
311		}
312		}
313
314		/// Matches the implementation of cq_ring_needs_flush() in liburing.
315		pub fn cq_ring_needs_flush(self: *IO_Uring) bool {
316		return (@atomicLoad(u32, self.sq.flags, .Unordered) & linux.IORING_SQ_CQ_OVERFLOW) != 0;
317		}
318
319		/// For advanced use cases only that implement custom completion queue methods.
320		/// If you use copy_cqes() or copy_cqe() you must not call cqe_seen() or cq_advance().
321		/// Must be called exactly once after a zero-copy CQE has been processed by your application.
322		/// Not idempotent, calling more than once will result in other CQEs being lost.
323		/// Matches the implementation of cqe_seen() in liburing.
324		pub fn cqe_seen(self: IO_Uring, cqe: linux.io_uring_cqe) void {
325		_ = cqe;
326		self.cq_advance(1);
327		}
328
329		/// For advanced use cases only that implement custom completion queue methods.
330		/// Matches the implementation of cq_advance() in liburing.
331		pub fn cq_advance(self: *IO_Uring, count: u32) void {
332		if (count > 0) {
333		// Ensure the kernel only sees the new head value after the CQEs have been read.
334		@atomicStore(u32, self.cq.head, self.cq.head.* +% count, .Release);
335		}
336		}
337
338		/// Queues (but does not submit) an SQE to perform an `fsync(2)`.
339		/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
340		/// For example, for `fdatasync()` you can set `IORING_FSYNC_DATASYNC` in the SQE's `rw_flags`.
341		/// N.B. While SQEs are initiated in the order in which they appear in the submission queue,
342		/// operations execute in parallel and completions are unordered. Therefore, an application that
343		/// submits a write followed by an fsync in the submission queue cannot expect the fsync to
344		/// apply to the write, since the fsync may complete before the write is issued to the disk.
345		/// You should preferably use `link_with_next_sqe()` on a write's SQE to link it with an fsync,
346		/// or else insert a full write barrier using `drain_previous_sqes()` when queueing an fsync.
347		pub fn fsync(self: IO_Uring, user_data: u64, fd: os.fd_t, flags: u32) !linux.io_uring_sqe {
348		const sqe = try self.get_sqe();
349		io_uring_prep_fsync(sqe, fd, flags);
350		sqe.user_data = user_data;
351		return sqe;
352		}
353
354		/// Queues (but does not submit) an SQE to perform a no-op.
355		/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
356		/// A no-op is more useful than may appear at first glance.
357		/// For example, you could call `drain_previous_sqes()` on the returned SQE, to use the no-op to
358		/// know when the ring is idle before acting on a kill signal.
359		pub fn nop(self: IO_Uring, user_data: u64) !linux.io_uring_sqe {
360		const sqe = try self.get_sqe();
361		io_uring_prep_nop(sqe);
362		sqe.user_data = user_data;
363		return sqe;
364		}
365
366		/// Used to select how the read should be handled.
367		pub const ReadBuffer = union(enum) {
368		/// io_uring will read directly into this buffer
369		buffer: []u8,
370
371		/// io_uring will read directly into these buffers using readv.
372		iovecs: []const os.iovec,
373
374		/// io_uring will select a buffer that has previously been provided with `provide_buffers`.
375		/// The buffer group reference by `group_id` must contain at least one buffer for the read to work.
376		/// `len` controls the number of bytes to read into the selected buffer.
377		buffer_selection: struct {
378		group_id: u16,
379		len: usize,
380		},
381		};
382
383		/// Queues (but does not submit) an SQE to perform a `read(2)` or `preadv(2)` depending on the buffer type.
384		/// * Reading into a `ReadBuffer.buffer` uses `read(2)`
385		/// * Reading into a `ReadBuffer.iovecs` uses `preadv(2)`
386		/// If you want to do a `preadv2(2)` then set `rw_flags` on the returned SQE. See https://man7.org/linux/man-pages/man2/preadv2.2.html
387		///
388		/// Returns a pointer to the SQE.
389		pub fn read(
390		self: *IO_Uring,
391		user_data: u64,
392		fd: os.fd_t,
393		buffer: ReadBuffer,
394		offset: u64,
395		) !*linux.io_uring_sqe {
396		const sqe = try self.get_sqe();
397		switch (buffer) {
398		.buffer => \|slice\| io_uring_prep_read(sqe, fd, slice, offset),
399		.iovecs => \|vecs\| io_uring_prep_readv(sqe, fd, vecs, offset),
400		.buffer_selection => \|selection\| {
401		io_uring_prep_rw(.READ, sqe, fd, 0, selection.len, offset);
402		sqe.flags \|= linux.IOSQE_BUFFER_SELECT;
403		sqe.buf_index = selection.group_id;
404		},
405		}
406		sqe.user_data = user_data;
407		return sqe;
408		}
409
410		/// Queues (but does not submit) an SQE to perform a `write(2)`.
411		/// Returns a pointer to the SQE.
412		pub fn write(
413		self: *IO_Uring,
414		user_data: u64,
415		fd: os.fd_t,
416		buffer: []const u8,
417		offset: u64,
418		) !*linux.io_uring_sqe {
419		const sqe = try self.get_sqe();
420		io_uring_prep_write(sqe, fd, buffer, offset);
421		sqe.user_data = user_data;
422		return sqe;
423		}
424
425		/// Queues (but does not submit) an SQE to perform a `splice(2)`
426		/// Either `fd_in` or `fd_out` must be a pipe.
427		/// If `fd_in` refers to a pipe, `off_in` is ignored and must be set to std.math.maxInt(u64).
428		/// If `fd_in` does not refer to a pipe and `off_in` is maxInt(u64), then `len` are read
429		/// from `fd_in` starting from the file offset, which is incremented by the number of bytes read.
430		/// If `fd_in` does not refer to a pipe and `off_in` is not maxInt(u64), then the starting offset of `fd_in` will be `off_in`.
431		/// This splice operation can be used to implement sendfile by splicing to an intermediate pipe first,
432		/// then splice to the final destination. In fact, the implementation of sendfile in kernel uses splice internally.
433		///
434		/// NOTE that even if fd_in or fd_out refers to a pipe, the splice operation can still fail with EINVAL if one of the
435		/// fd doesn't explicitly support splice peration, e.g. reading from terminal is unsupported from kernel 5.7 to 5.11.
436		/// See https://github.com/axboe/liburing/issues/291
437		///
438		/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
439		pub fn splice(self: IO_Uring, user_data: u64, fd_in: os.fd_t, off_in: u64, fd_out: os.fd_t, off_out: u64, len: usize) !linux.io_uring_sqe {
440		const sqe = try self.get_sqe();
441		io_uring_prep_splice(sqe, fd_in, off_in, fd_out, off_out, len);
442		sqe.user_data = user_data;
443		return sqe;
444		}
445
446		/// Queues (but does not submit) an SQE to perform a IORING_OP_READ_FIXED.
447		/// The `buffer` provided must be registered with the kernel by calling `register_buffers` first.
448		/// The `buffer_index` must be the same as its index in the array provided to `register_buffers`.
449		///
450		/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
451		pub fn read_fixed(
452		self: *IO_Uring,
453		user_data: u64,
454		fd: os.fd_t,
455		buffer: *os.iovec,
456		offset: u64,
457		buffer_index: u16,
458		) !*linux.io_uring_sqe {
459		const sqe = try self.get_sqe();
460		io_uring_prep_read_fixed(sqe, fd, buffer, offset, buffer_index);
461		sqe.user_data = user_data;
462		return sqe;
463		}
464
465		/// Queues (but does not submit) an SQE to perform a `pwritev()`.
466		/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
467		/// For example, if you want to do a `pwritev2()` then set `rw_flags` on the returned SQE.
468		/// See https://linux.die.net/man/2/pwritev.
469		pub fn writev(
470		self: *IO_Uring,
471		user_data: u64,
472		fd: os.fd_t,
473		iovecs: []const os.iovec_const,
474		offset: u64,
475		) !*linux.io_uring_sqe {
476		const sqe = try self.get_sqe();
477		io_uring_prep_writev(sqe, fd, iovecs, offset);
478		sqe.user_data = user_data;
479		return sqe;
480		}
481
482		/// Queues (but does not submit) an SQE to perform a IORING_OP_WRITE_FIXED.
483		/// The `buffer` provided must be registered with the kernel by calling `register_buffers` first.
484		/// The `buffer_index` must be the same as its index in the array provided to `register_buffers`.
485		///
486		/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
487		pub fn write_fixed(
488		self: *IO_Uring,
489		user_data: u64,
490		fd: os.fd_t,
491		buffer: *os.iovec,
492		offset: u64,
493		buffer_index: u16,
494		) !*linux.io_uring_sqe {
495		const sqe = try self.get_sqe();
496		io_uring_prep_write_fixed(sqe, fd, buffer, offset, buffer_index);
497		sqe.user_data = user_data;
498		return sqe;
499		}
500
501		/// Queues (but does not submit) an SQE to perform an `accept4(2)` on a socket.
502		/// Returns a pointer to the SQE.
503		/// Available since 5.5
504		pub fn accept(
505		self: *IO_Uring,
506		user_data: u64,
507		fd: os.fd_t,
508		addr: ?*os.sockaddr,
509		addrlen: ?*os.socklen_t,
510		flags: u32,
511		) !*linux.io_uring_sqe {
512		const sqe = try self.get_sqe();
513		io_uring_prep_accept(sqe, fd, addr, addrlen, flags);
514		sqe.user_data = user_data;
515		return sqe;
516		}
517
518		/// Queues an multishot accept on a socket.
519		///
520		/// Multishot variant allows an application to issue a single accept request,
521		/// which will repeatedly trigger a CQE when a connection request comes in.
522		/// While IORING_CQE_F_MORE flag is set in CQE flags accept will generate
523		/// further CQEs.
524		///
525		/// Available since 5.19
526		pub fn accept_multishot(
527		self: *IO_Uring,
528		user_data: u64,
529		fd: os.fd_t,
530		addr: ?*os.sockaddr,
531		addrlen: ?*os.socklen_t,
532		flags: u32,
533		) !*linux.io_uring_sqe {
534		const sqe = try self.get_sqe();
535		io_uring_prep_multishot_accept(sqe, fd, addr, addrlen, flags);
536		sqe.user_data = user_data;
537		return sqe;
538		}
539
540		/// Queues an accept using direct (registered) file descriptors.
541		///
542		/// To use an accept direct variant, the application must first have registered
543		/// a file table (with register_files). An unused table index will be
544		/// dynamically chosen and returned in the CQE res field.
545		///
546		/// After creation, they can be used by setting IOSQE_FIXED_FILE in the SQE
547		/// flags member, and setting the SQE fd field to the direct descriptor value
548		/// rather than the regular file descriptor.
549		///
550		/// Available since 5.19
551		pub fn accept_direct(
552		self: *IO_Uring,
553		user_data: u64,
554		fd: os.fd_t,
555		addr: ?*os.sockaddr,
556		addrlen: ?*os.socklen_t,
557		flags: u32,
558		) !*linux.io_uring_sqe {
559		const sqe = try self.get_sqe();
560		io_uring_prep_accept_direct(sqe, fd, addr, addrlen, flags, linux.IORING_FILE_INDEX_ALLOC);
561		sqe.user_data = user_data;
562		return sqe;
563		}
564
565		/// Queues an multishot accept using direct (registered) file descriptors.
566		/// Available since 5.19
567		pub fn accept_multishot_direct(
568		self: *IO_Uring,
569		user_data: u64,
570		fd: os.fd_t,
571		addr: ?*os.sockaddr,
572		addrlen: ?*os.socklen_t,
573		flags: u32,
574		) !*linux.io_uring_sqe {
575		const sqe = try self.get_sqe();
576		io_uring_prep_multishot_accept_direct(sqe, fd, addr, addrlen, flags);
577		sqe.user_data = user_data;
578		return sqe;
579		}
580
581		/// Queue (but does not submit) an SQE to perform a `connect(2)` on a socket.
582		/// Returns a pointer to the SQE.
583		pub fn connect(
584		self: *IO_Uring,
585		user_data: u64,
586		fd: os.fd_t,
587		addr: *const os.sockaddr,
588		addrlen: os.socklen_t,
589		) !*linux.io_uring_sqe {
590		const sqe = try self.get_sqe();
591		io_uring_prep_connect(sqe, fd, addr, addrlen);
592		sqe.user_data = user_data;
593		return sqe;
594		}
595
596		/// Queues (but does not submit) an SQE to perform a `epoll_ctl(2)`.
597		/// Returns a pointer to the SQE.
598		pub fn epoll_ctl(
599		self: *IO_Uring,
600		user_data: u64,
601		epfd: os.fd_t,
602		fd: os.fd_t,
603		op: u32,
604		ev: ?*linux.epoll_event,
605		) !*linux.io_uring_sqe {
606		const sqe = try self.get_sqe();
607		io_uring_prep_epoll_ctl(sqe, epfd, fd, op, ev);
608		sqe.user_data = user_data;
609		return sqe;
610		}
611
612		/// Used to select how the recv call should be handled.
613		pub const RecvBuffer = union(enum) {
614		/// io_uring will recv directly into this buffer
615		buffer: []u8,
616
617		/// io_uring will select a buffer that has previously been provided with `provide_buffers`.
618		/// The buffer group referenced by `group_id` must contain at least one buffer for the recv call to work.
619		/// `len` controls the number of bytes to read into the selected buffer.
620		buffer_selection: struct {
621		group_id: u16,
622		len: usize,
623		},
624		};
625
626		/// Queues (but does not submit) an SQE to perform a `recv(2)`.
627		/// Returns a pointer to the SQE.
628		/// Available since 5.6
629		pub fn recv(
630		self: *IO_Uring,
631		user_data: u64,
632		fd: os.fd_t,
633		buffer: RecvBuffer,
634		flags: u32,
635		) !*linux.io_uring_sqe {
636		const sqe = try self.get_sqe();
637		switch (buffer) {
638		.buffer => \|slice\| io_uring_prep_recv(sqe, fd, slice, flags),
639		.buffer_selection => \|selection\| {
640		io_uring_prep_rw(.RECV, sqe, fd, 0, selection.len, 0);
641		sqe.rw_flags = flags;
642		sqe.flags \|= linux.IOSQE_BUFFER_SELECT;
643		sqe.buf_index = selection.group_id;
644		},
645		}
646		sqe.user_data = user_data;
647		return sqe;
648		}
649
650		/// Queues (but does not submit) an SQE to perform a `send(2)`.
651		/// Returns a pointer to the SQE.
652		/// Available since 5.6
653		pub fn send(
654		self: *IO_Uring,
655		user_data: u64,
656		fd: os.fd_t,
657		buffer: []const u8,
658		flags: u32,
659		) !*linux.io_uring_sqe {
660		const sqe = try self.get_sqe();
661		io_uring_prep_send(sqe, fd, buffer, flags);
662		sqe.user_data = user_data;
663		return sqe;
664		}
665
666		/// Queues (but does not submit) an SQE to perform an async zerocopy `send(2)`.
667		///
668		/// This operation will most likely produce two CQEs. The flags field of the
669		/// first cqe may likely contain IORING_CQE_F_MORE, which means that there will
670		/// be a second cqe with the user_data field set to the same value. The user
671		/// must not modify the data buffer until the notification is posted. The first
672		/// cqe follows the usual rules and so its res field will contain the number of
673		/// bytes sent or a negative error code. The notification's res field will be
674		/// set to zero and the flags field will contain IORING_CQE_F_NOTIF. The two
675		/// step model is needed because the kernel may hold on to buffers for a long
676		/// time, e.g. waiting for a TCP ACK. Notifications responsible for controlling
677		/// the lifetime of the buffers. Even errored requests may generate a
678		/// notification.
679		///
680		/// Available since 6.0
681		pub fn send_zc(
682		self: *IO_Uring,
683		user_data: u64,
684		fd: os.fd_t,
685		buffer: []const u8,
686		send_flags: u32,
687		zc_flags: u16,
688		) !*linux.io_uring_sqe {
689		const sqe = try self.get_sqe();
690		io_uring_prep_send_zc(sqe, fd, buffer, send_flags, zc_flags);
691		sqe.user_data = user_data;
692		return sqe;
693		}
694
695		/// Queues (but does not submit) an SQE to perform an async zerocopy `send(2)`.
696		/// Returns a pointer to the SQE.
697		/// Available since 6.0
698		pub fn send_zc_fixed(
699		self: *IO_Uring,
700		user_data: u64,
701		fd: os.fd_t,
702		buffer: []const u8,
703		send_flags: u32,
704		zc_flags: u16,
705		buf_index: u16,
706		) !*linux.io_uring_sqe {
707		const sqe = try self.get_sqe();
708		io_uring_prep_send_zc_fixed(sqe, fd, buffer, send_flags, zc_flags, buf_index);
709		sqe.user_data = user_data;
710		return sqe;
711		}
712
713		/// Queues (but does not submit) an SQE to perform a `recvmsg(2)`.
714		/// Returns a pointer to the SQE.
715		/// Available since 5.3
716		pub fn recvmsg(
717		self: *IO_Uring,
718		user_data: u64,
719		fd: os.fd_t,
720		msg: *os.msghdr,
721		flags: u32,
722		) !*linux.io_uring_sqe {
723		const sqe = try self.get_sqe();
724		io_uring_prep_recvmsg(sqe, fd, msg, flags);
725		sqe.user_data = user_data;
726		return sqe;
727		}
728
729		/// Queues (but does not submit) an SQE to perform a `sendmsg(2)`.
730		/// Returns a pointer to the SQE.
731		/// Available since 5.3
732		pub fn sendmsg(
733		self: *IO_Uring,
734		user_data: u64,
735		fd: os.fd_t,
736		msg: *const os.msghdr_const,
737		flags: u32,
738		) !*linux.io_uring_sqe {
739		const sqe = try self.get_sqe();
740		io_uring_prep_sendmsg(sqe, fd, msg, flags);
741		sqe.user_data = user_data;
742		return sqe;
743		}
744
745		/// Queues (but does not submit) an SQE to perform an async zerocopy `sendmsg(2)`.
746		/// Returns a pointer to the SQE.
747		/// Available since 6.1
748		pub fn sendmsg_zc(
749		self: *IO_Uring,
750		user_data: u64,
751		fd: os.fd_t,
752		msg: *const os.msghdr_const,
753		flags: u32,
754		) !*linux.io_uring_sqe {
755		const sqe = try self.get_sqe();
756		io_uring_prep_sendmsg_zc(sqe, fd, msg, flags);
757		sqe.user_data = user_data;
758		return sqe;
759		}
760
761		/// Queues (but does not submit) an SQE to perform an `openat(2)`.
762		/// Returns a pointer to the SQE.
763		/// Available since 5.6.
764		pub fn openat(
765		self: *IO_Uring,
766		user_data: u64,
767		fd: os.fd_t,
768		path: [*:0]const u8,
769		flags: linux.O,
770		mode: os.mode_t,
771		) !*linux.io_uring_sqe {
772		const sqe = try self.get_sqe();
773		io_uring_prep_openat(sqe, fd, path, flags, mode);
774		sqe.user_data = user_data;
775		return sqe;
776		}
777
778		/// Queues an openat using direct (registered) file descriptors.
779		///
780		/// To use an accept direct variant, the application must first have registered
781		/// a file table (with register_files). An unused table index will be
782		/// dynamically chosen and returned in the CQE res field.
783		///
784		/// After creation, they can be used by setting IOSQE_FIXED_FILE in the SQE
785		/// flags member, and setting the SQE fd field to the direct descriptor value
786		/// rather than the regular file descriptor.
787		///
788		/// Available since 5.15
789		pub fn openat_direct(
790		self: *IO_Uring,
791		user_data: u64,
792		fd: os.fd_t,
793		path: [*:0]const u8,
794		flags: linux.O,
795		mode: os.mode_t,
796		file_index: u32,
797		) !*linux.io_uring_sqe {
798		const sqe = try self.get_sqe();
799		io_uring_prep_openat_direct(sqe, fd, path, flags, mode, file_index);
800		sqe.user_data = user_data;
801		return sqe;
802		}
803
804		/// Queues (but does not submit) an SQE to perform a `close(2)`.
805		/// Returns a pointer to the SQE.
806		/// Available since 5.6.
807		pub fn close(self: IO_Uring, user_data: u64, fd: os.fd_t) !linux.io_uring_sqe {
808		const sqe = try self.get_sqe();
809		io_uring_prep_close(sqe, fd);
810		sqe.user_data = user_data;
811		return sqe;
812		}
813
814		/// Queues close of registered file descriptor.
815		/// Available since 5.15
816		pub fn close_direct(self: IO_Uring, user_data: u64, file_index: u32) !linux.io_uring_sqe {
817		const sqe = try self.get_sqe();
818		io_uring_prep_close_direct(sqe, file_index);
819		sqe.user_data = user_data;
820		return sqe;
821		}
822
823		/// Queues (but does not submit) an SQE to register a timeout operation.
824		/// Returns a pointer to the SQE.
825		///
826		/// The timeout will complete when either the timeout expires, or after the specified number of
827		/// events complete (if `count` is greater than `0`).
828		///
829		/// `flags` may be `0` for a relative timeout, or `IORING_TIMEOUT_ABS` for an absolute timeout.
830		///
831		/// The completion event result will be `-ETIME` if the timeout completed through expiration,
832		/// `0` if the timeout completed after the specified number of events, or `-ECANCELED` if the
833		/// timeout was removed before it expired.
834		///
835		/// io_uring timeouts use the `CLOCK.MONOTONIC` clock source.
836		pub fn timeout(
837		self: *IO_Uring,
838		user_data: u64,
839		ts: *const os.linux.kernel_timespec,
840		count: u32,
841		flags: u32,
842		) !*linux.io_uring_sqe {
843		const sqe = try self.get_sqe();
844		io_uring_prep_timeout(sqe, ts, count, flags);
845		sqe.user_data = user_data;
846		return sqe;
847		}
848
849		/// Queues (but does not submit) an SQE to remove an existing timeout operation.
850		/// Returns a pointer to the SQE.
851		///
852		/// The timeout is identified by its `user_data`.
853		///
854		/// The completion event result will be `0` if the timeout was found and cancelled successfully,
855		/// `-EBUSY` if the timeout was found but expiration was already in progress, or
856		/// `-ENOENT` if the timeout was not found.
857		pub fn timeout_remove(
858		self: *IO_Uring,
859		user_data: u64,
860		timeout_user_data: u64,
861		flags: u32,
862		) !*linux.io_uring_sqe {
863		const sqe = try self.get_sqe();
864		io_uring_prep_timeout_remove(sqe, timeout_user_data, flags);
865		sqe.user_data = user_data;
866		return sqe;
867		}
868
869		/// Queues (but does not submit) an SQE to add a link timeout operation.
870		/// Returns a pointer to the SQE.
871		///
872		/// You need to set linux.IOSQE_IO_LINK to flags of the target operation
873		/// and then call this method right after the target operation.
874		/// See https://lwn.net/Articles/803932/ for detail.
875		///
876		/// If the dependent request finishes before the linked timeout, the timeout
877		/// is canceled. If the timeout finishes before the dependent request, the
878		/// dependent request will be canceled.
879		///
880		/// The completion event result of the link_timeout will be
881		/// `-ETIME` if the timeout finishes before the dependent request
882		/// (in this case, the completion event result of the dependent request will
883		/// be `-ECANCELED`), or
884		/// `-EALREADY` if the dependent request finishes before the linked timeout.
885		pub fn link_timeout(
886		self: *IO_Uring,
887		user_data: u64,
888		ts: *const os.linux.kernel_timespec,
889		flags: u32,
890		) !*linux.io_uring_sqe {
891		const sqe = try self.get_sqe();
892		io_uring_prep_link_timeout(sqe, ts, flags);
893		sqe.user_data = user_data;
894		return sqe;
895		}
896
897		/// Queues (but does not submit) an SQE to perform a `poll(2)`.
898		/// Returns a pointer to the SQE.
899		pub fn poll_add(
900		self: *IO_Uring,
901		user_data: u64,
902		fd: os.fd_t,
903		poll_mask: u32,
904		) !*linux.io_uring_sqe {
905		const sqe = try self.get_sqe();
906		io_uring_prep_poll_add(sqe, fd, poll_mask);
907		sqe.user_data = user_data;
908		return sqe;
909		}
910
911		/// Queues (but does not submit) an SQE to remove an existing poll operation.
912		/// Returns a pointer to the SQE.
913		pub fn poll_remove(
914		self: *IO_Uring,
915		user_data: u64,
916		target_user_data: u64,
917		) !*linux.io_uring_sqe {
918		const sqe = try self.get_sqe();
919		io_uring_prep_poll_remove(sqe, target_user_data);
920		sqe.user_data = user_data;
921		return sqe;
922		}
923
924		/// Queues (but does not submit) an SQE to update the user data of an existing poll
925		/// operation. Returns a pointer to the SQE.
926		pub fn poll_update(
927		self: *IO_Uring,
928		user_data: u64,
929		old_user_data: u64,
930		new_user_data: u64,
931		poll_mask: u32,
932		flags: u32,
933		) !*linux.io_uring_sqe {
934		const sqe = try self.get_sqe();
935		io_uring_prep_poll_update(sqe, old_user_data, new_user_data, poll_mask, flags);
936		sqe.user_data = user_data;
937		return sqe;
938		}
939
940		/// Queues (but does not submit) an SQE to perform an `fallocate(2)`.
941		/// Returns a pointer to the SQE.
942		pub fn fallocate(
943		self: *IO_Uring,
944		user_data: u64,
945		fd: os.fd_t,
946		mode: i32,
947		offset: u64,
948		len: u64,
949		) !*linux.io_uring_sqe {
950		const sqe = try self.get_sqe();
951		io_uring_prep_fallocate(sqe, fd, mode, offset, len);
952		sqe.user_data = user_data;
953		return sqe;
954		}
955
956		/// Queues (but does not submit) an SQE to perform an `statx(2)`.
957		/// Returns a pointer to the SQE.
958		pub fn statx(
959		self: *IO_Uring,
960		user_data: u64,
961		fd: os.fd_t,
962		path: [:0]const u8,
963		flags: u32,
964		mask: u32,
965		buf: *linux.Statx,
966		) !*linux.io_uring_sqe {
967		const sqe = try self.get_sqe();
968		io_uring_prep_statx(sqe, fd, path, flags, mask, buf);
969		sqe.user_data = user_data;
970		return sqe;
971		}
972
973		/// Queues (but does not submit) an SQE to remove an existing operation.
974		/// Returns a pointer to the SQE.
975		///
976		/// The operation is identified by its `user_data`.
977		///
978		/// The completion event result will be `0` if the operation was found and cancelled successfully,
979		/// `-EALREADY` if the operation was found but was already in progress, or
980		/// `-ENOENT` if the operation was not found.
981		pub fn cancel(
982		self: *IO_Uring,
983		user_data: u64,
984		cancel_user_data: u64,
985		flags: u32,
986		) !*linux.io_uring_sqe {
987		const sqe = try self.get_sqe();
988		io_uring_prep_cancel(sqe, cancel_user_data, flags);
989		sqe.user_data = user_data;
990		return sqe;
991		}
992
993		/// Queues (but does not submit) an SQE to perform a `shutdown(2)`.
994		/// Returns a pointer to the SQE.
995		///
996		/// The operation is identified by its `user_data`.
997		pub fn shutdown(
998		self: *IO_Uring,
999		user_data: u64,
1000		sockfd: os.socket_t,
1001		how: u32,
1002		) !*linux.io_uring_sqe {
1003		const sqe = try self.get_sqe();
1004		io_uring_prep_shutdown(sqe, sockfd, how);
1005		sqe.user_data = user_data;
1006		return sqe;
1007		}
1008
1009		/// Queues (but does not submit) an SQE to perform a `renameat2(2)`.
1010		/// Returns a pointer to the SQE.
1011		pub fn renameat(
1012		self: *IO_Uring,
1013		user_data: u64,
1014		old_dir_fd: os.fd_t,
1015		old_path: [*:0]const u8,
1016		new_dir_fd: os.fd_t,
1017		new_path: [*:0]const u8,
1018		flags: u32,
1019		) !*linux.io_uring_sqe {
1020		const sqe = try self.get_sqe();
1021		io_uring_prep_renameat(sqe, old_dir_fd, old_path, new_dir_fd, new_path, flags);
1022		sqe.user_data = user_data;
1023		return sqe;
1024		}
1025
1026		/// Queues (but does not submit) an SQE to perform a `unlinkat(2)`.
1027		/// Returns a pointer to the SQE.
1028		pub fn unlinkat(
1029		self: *IO_Uring,
1030		user_data: u64,
1031		dir_fd: os.fd_t,
1032		path: [*:0]const u8,
1033		flags: u32,
1034		) !*linux.io_uring_sqe {
1035		const sqe = try self.get_sqe();
1036		io_uring_prep_unlinkat(sqe, dir_fd, path, flags);
1037		sqe.user_data = user_data;
1038		return sqe;
1039		}
1040
1041		/// Queues (but does not submit) an SQE to perform a `mkdirat(2)`.
1042		/// Returns a pointer to the SQE.
1043		pub fn mkdirat(
1044		self: *IO_Uring,
1045		user_data: u64,
1046		dir_fd: os.fd_t,
1047		path: [*:0]const u8,
1048		mode: os.mode_t,
1049		) !*linux.io_uring_sqe {
1050		const sqe = try self.get_sqe();
1051		io_uring_prep_mkdirat(sqe, dir_fd, path, mode);
1052		sqe.user_data = user_data;
1053		return sqe;
1054		}
1055
1056		/// Queues (but does not submit) an SQE to perform a `symlinkat(2)`.
1057		/// Returns a pointer to the SQE.
1058		pub fn symlinkat(
1059		self: *IO_Uring,
1060		user_data: u64,
1061		target: [*:0]const u8,
1062		new_dir_fd: os.fd_t,
1063		link_path: [*:0]const u8,
1064		) !*linux.io_uring_sqe {
1065		const sqe = try self.get_sqe();
1066		io_uring_prep_symlinkat(sqe, target, new_dir_fd, link_path);
1067		sqe.user_data = user_data;
1068		return sqe;
1069		}
1070
1071		/// Queues (but does not submit) an SQE to perform a `linkat(2)`.
1072		/// Returns a pointer to the SQE.
1073		pub fn linkat(
1074		self: *IO_Uring,
1075		user_data: u64,
1076		old_dir_fd: os.fd_t,
1077		old_path: [*:0]const u8,
1078		new_dir_fd: os.fd_t,
1079		new_path: [*:0]const u8,
1080		flags: u32,
1081		) !*linux.io_uring_sqe {
1082		const sqe = try self.get_sqe();
1083		io_uring_prep_linkat(sqe, old_dir_fd, old_path, new_dir_fd, new_path, flags);
1084		sqe.user_data = user_data;
1085		return sqe;
1086		}
1087
1088		/// Queues (but does not submit) an SQE to provide a group of buffers used for commands that read/receive data.
1089		/// Returns a pointer to the SQE.
1090		///
1091		/// Provided buffers can be used in `read`, `recv` or `recvmsg` commands via .buffer_selection.
1092		///
1093		/// The kernel expects a contiguous block of memory of size (buffers_count * buffer_size).
1094		pub fn provide_buffers(
1095		self: *IO_Uring,
1096		user_data: u64,
1097		buffers: [*]u8,
1098		buffer_size: usize,
1099		buffers_count: usize,
1100		group_id: usize,
1101		buffer_id: usize,
1102		) !*linux.io_uring_sqe {
1103		const sqe = try self.get_sqe();
1104		io_uring_prep_provide_buffers(sqe, buffers, buffer_size, buffers_count, group_id, buffer_id);
1105		sqe.user_data = user_data;
1106		return sqe;
1107		}
1108
1109		/// Queues (but does not submit) an SQE to remove a group of provided buffers.
1110		/// Returns a pointer to the SQE.
1111		pub fn remove_buffers(
1112		self: *IO_Uring,
1113		user_data: u64,
1114		buffers_count: usize,
1115		group_id: usize,
1116		) !*linux.io_uring_sqe {
1117		const sqe = try self.get_sqe();
1118		io_uring_prep_remove_buffers(sqe, buffers_count, group_id);
1119		sqe.user_data = user_data;
1120		return sqe;
1121		}
1122
1123		/// Queues (but does not submit) an SQE to perform a `waitid(2)`.
1124		/// Returns a pointer to the SQE.
1125		pub fn waitid(
1126		self: *IO_Uring,
1127		user_data: u64,
1128		id_type: linux.P,
1129		id: i32,
1130		infop: *linux.siginfo_t,
1131		options: u32,
1132		flags: u32,
1133		) !*linux.io_uring_sqe {
1134		const sqe = try self.get_sqe();
1135		io_uring_prep_waitid(sqe, id_type, id, infop, options, flags);
1136		sqe.user_data = user_data;
1137		return sqe;
1138		}
1139
1140		/// Registers an array of file descriptors.
1141		/// Every time a file descriptor is put in an SQE and submitted to the kernel, the kernel must
1142		/// retrieve a reference to the file, and once I/O has completed the file reference must be
1143		/// dropped. The atomic nature of this file reference can be a slowdown for high IOPS workloads.
1144		/// This slowdown can be avoided by pre-registering file descriptors.
1145		/// To refer to a registered file descriptor, IOSQE_FIXED_FILE must be set in the SQE's flags,
1146		/// and the SQE's fd must be set to the index of the file descriptor in the registered array.
1147		/// Registering file descriptors will wait for the ring to idle.
1148		/// Files are automatically unregistered by the kernel when the ring is torn down.
1149		/// An application need unregister only if it wants to register a new array of file descriptors.
1150		pub fn register_files(self: *IO_Uring, fds: []const os.fd_t) !void {
1151		assert(self.fd >= 0);
1152		const res = linux.io_uring_register(
1153		self.fd,
1154		.REGISTER_FILES,
1155		@as(*const anyopaque, @ptrCast(fds.ptr)),
1156		@as(u32, @intCast(fds.len)),
1157		);
1158		try handle_registration_result(res);
1159		}
1160
1161		/// Updates registered file descriptors.
1162		///
1163		/// Updates are applied starting at the provided offset in the original file descriptors slice.
1164		/// There are three kind of updates:
1165		/// * turning a sparse entry (where the fd is -1) into a real one
1166		/// * removing an existing entry (set the fd to -1)
1167		/// * replacing an existing entry with a new fd
1168		/// Adding new file descriptors must be done with `register_files`.
1169		pub fn register_files_update(self: *IO_Uring, offset: u32, fds: []const os.fd_t) !void {
1170		assert(self.fd >= 0);
1171
1172		const FilesUpdate = extern struct {
1173		offset: u32,
1174		resv: u32,
1175		fds: u64 align(8),
1176		};
1177		var update = FilesUpdate{
1178		.offset = offset,
1179		.resv = @as(u32, 0),
1180		.fds = @as(u64, @intFromPtr(fds.ptr)),
1181		};
1182
1183		const res = linux.io_uring_register(
1184		self.fd,
1185		.REGISTER_FILES_UPDATE,
1186		@as(*const anyopaque, @ptrCast(&update)),
1187		@as(u32, @intCast(fds.len)),
1188		);
1189		try handle_registration_result(res);
1190		}
1191
1192		/// Registers the file descriptor for an eventfd that will be notified of completion events on
1193		/// an io_uring instance.
1194		/// Only a single a eventfd can be registered at any given point in time.
1195		pub fn register_eventfd(self: *IO_Uring, fd: os.fd_t) !void {
1196		assert(self.fd >= 0);
1197		const res = linux.io_uring_register(
1198		self.fd,
1199		.REGISTER_EVENTFD,
1200		@as(*const anyopaque, @ptrCast(&fd)),
1201		1,
1202		);
1203		try handle_registration_result(res);
1204		}
1205
1206		/// Registers the file descriptor for an eventfd that will be notified of completion events on
1207		/// an io_uring instance. Notifications are only posted for events that complete in an async manner.
1208		/// This means that events that complete inline while being submitted do not trigger a notification event.
1209		/// Only a single eventfd can be registered at any given point in time.
1210		pub fn register_eventfd_async(self: *IO_Uring, fd: os.fd_t) !void {
1211		assert(self.fd >= 0);
1212		const res = linux.io_uring_register(
1213		self.fd,
1214		.REGISTER_EVENTFD_ASYNC,
1215		@as(*const anyopaque, @ptrCast(&fd)),
1216		1,
1217		);
1218		try handle_registration_result(res);
1219		}
1220
1221		/// Unregister the registered eventfd file descriptor.
1222		pub fn unregister_eventfd(self: *IO_Uring) !void {
1223		assert(self.fd >= 0);
1224		const res = linux.io_uring_register(
1225		self.fd,
1226		.UNREGISTER_EVENTFD,
1227		null,
1228		0,
1229		);
1230		try handle_registration_result(res);
1231		}
1232
1233		/// Registers an array of buffers for use with `read_fixed` and `write_fixed`.
1234		pub fn register_buffers(self: *IO_Uring, buffers: []const os.iovec) !void {
1235		assert(self.fd >= 0);
1236		const res = linux.io_uring_register(
1237		self.fd,
1238		.REGISTER_BUFFERS,
1239		buffers.ptr,
1240		@as(u32, @intCast(buffers.len)),
1241		);
1242		try handle_registration_result(res);
1243		}
1244
1245		/// Unregister the registered buffers.
1246		pub fn unregister_buffers(self: *IO_Uring) !void {
1247		assert(self.fd >= 0);
1248		const res = linux.io_uring_register(self.fd, .UNREGISTER_BUFFERS, null, 0);
1249		switch (linux.getErrno(res)) {
1250		.SUCCESS => {},
1251		.NXIO => return error.BuffersNotRegistered,
1252		else => \|errno\| return os.unexpectedErrno(errno),
1253		}
1254		}
1255
1256		fn handle_registration_result(res: usize) !void {
1257		switch (linux.getErrno(res)) {
1258		.SUCCESS => {},
1259		// One or more fds in the array are invalid, or the kernel does not support sparse sets:
1260		.BADF => return error.FileDescriptorInvalid,
1261		.BUSY => return error.FilesAlreadyRegistered,
1262		.INVAL => return error.FilesEmpty,
1263		// Adding `nr_args` file references would exceed the maximum allowed number of files the
1264		// user is allowed to have according to the per-user RLIMIT_NOFILE resource limit and
1265		// the CAP_SYS_RESOURCE capability is not set, or `nr_args` exceeds the maximum allowed
1266		// for a fixed file set (older kernels have a limit of 1024 files vs 64K files):
1267		.MFILE => return error.UserFdQuotaExceeded,
1268		// Insufficient kernel resources, or the caller had a non-zero RLIMIT_MEMLOCK soft
1269		// resource limit but tried to lock more memory than the limit permitted (not enforced
1270		// when the process is privileged with CAP_IPC_LOCK):
1271		.NOMEM => return error.SystemResources,
1272		// Attempt to register files on a ring already registering files or being torn down:
1273		.NXIO => return error.RingShuttingDownOrAlreadyRegisteringFiles,
1274		else => \|errno\| return os.unexpectedErrno(errno),
1275		}
1276		}
1277
1278		/// Unregisters all registered file descriptors previously associated with the ring.
1279		pub fn unregister_files(self: *IO_Uring) !void {
1280		assert(self.fd >= 0);
1281		const res = linux.io_uring_register(self.fd, .UNREGISTER_FILES, null, 0);
1282		switch (linux.getErrno(res)) {
1283		.SUCCESS => {},
1284		.NXIO => return error.FilesNotRegistered,
1285		else => \|errno\| return os.unexpectedErrno(errno),
1286		}
1287		}
1288
1289		/// Prepares a socket creation request.
1290		/// New socket fd will be returned in completion result.
1291		/// Available since 5.19
1292		pub fn socket(
1293		self: *IO_Uring,
1294		user_data: u64,
1295		domain: u32,
1296		socket_type: u32,
1297		protocol: u32,
1298		flags: u32,
1299		) !*linux.io_uring_sqe {
1300		const sqe = try self.get_sqe();
1301		io_uring_prep_socket(sqe, domain, socket_type, protocol, flags);
1302		sqe.user_data = user_data;
1303		return sqe;
1304		}
1305
1306		/// Prepares a socket creation request for registered file at index `file_index`.
1307		/// Available since 5.19
1308		pub fn socket_direct(
1309		self: *IO_Uring,
1310		user_data: u64,
1311		domain: u32,
1312		socket_type: u32,
1313		protocol: u32,
1314		flags: u32,
1315		file_index: u32,
1316		) !*linux.io_uring_sqe {
1317		const sqe = try self.get_sqe();
1318		io_uring_prep_socket_direct(sqe, domain, socket_type, protocol, flags, file_index);
1319		sqe.user_data = user_data;
1320		return sqe;
1321		}
1322
1323		/// Prepares a socket creation request for registered file, index chosen by kernel (file index alloc).
1324		/// File index will be returned in CQE res field.
1325		/// Available since 5.19
1326		pub fn socket_direct_alloc(
1327		self: *IO_Uring,
1328		user_data: u64,
1329		domain: u32,
1330		socket_type: u32,
1331		protocol: u32,
1332		flags: u32,
1333		) !*linux.io_uring_sqe {
1334		const sqe = try self.get_sqe();
1335		io_uring_prep_socket_direct_alloc(sqe, domain, socket_type, protocol, flags);
1336		sqe.user_data = user_data;
1337		return sqe;
1338		}
1339		};
1340
1341		pub const SubmissionQueue = struct {
1342		head: *u32,
1343		tail: *u32,
1344		mask: u32,
1345		flags: *u32,
1346		dropped: *u32,
1347		array: []u32,
1348		sqes: []linux.io_uring_sqe,
1349		mmap: []align(mem.page_size) u8,
1350		mmap_sqes: []align(mem.page_size) u8,
1351
1352		// We use `sqe_head` and `sqe_tail` in the same way as liburing:
1353		// We increment `sqe_tail` (but not `tail`) for each call to `get_sqe()`.
1354		// We then set `tail` to `sqe_tail` once, only when these events are actually submitted.
1355		// This allows us to amortize the cost of the @atomicStore to `tail` across multiple SQEs.
1356		sqe_head: u32 = 0,
1357		sqe_tail: u32 = 0,
1358
1359		pub fn init(fd: os.fd_t, p: linux.io_uring_params) !SubmissionQueue {
1360		assert(fd >= 0);
1361		assert((p.features & linux.IORING_FEAT_SINGLE_MMAP) != 0);
1362		const size = @max(
1363		p.sq_off.array + p.sq_entries * @sizeOf(u32),
1364		p.cq_off.cqes + p.cq_entries * @sizeOf(linux.io_uring_cqe),
1365		);
1366		const mmap = try os.mmap(
1367		null,
1368		size,
1369		os.PROT.READ \| os.PROT.WRITE,
1370		.{ .TYPE = .SHARED, .POPULATE = true },
1371		fd,
1372		linux.IORING_OFF_SQ_RING,
1373		);
1374		errdefer os.munmap(mmap);
1375		assert(mmap.len == size);
1376
1377		// The motivation for the `sqes` and `array` indirection is to make it possible for the
1378		// application to preallocate static linux.io_uring_sqe entries and then replay them when needed.
1379		const size_sqes = p.sq_entries * @sizeOf(linux.io_uring_sqe);
1380		const mmap_sqes = try os.mmap(
1381		null,
1382		size_sqes,
1383		os.PROT.READ \| os.PROT.WRITE,
1384		.{ .TYPE = .SHARED, .POPULATE = true },
1385		fd,
1386		linux.IORING_OFF_SQES,
1387		);
1388		errdefer os.munmap(mmap_sqes);
1389		assert(mmap_sqes.len == size_sqes);
1390
1391		const array: [*]u32 = @ptrCast(@alignCast(&mmap[p.sq_off.array]));
1392		const sqes: [*]linux.io_uring_sqe = @ptrCast(@alignCast(&mmap_sqes[0]));
1393		// We expect the kernel copies p.sq_entries to the u32 pointed to by p.sq_off.ring_entries,
1394		// see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L7843-L7844.
1395		assert(p.sq_entries == @as(u32, @ptrCast(@alignCast(&mmap[p.sq_off.ring_entries]))).);
1396		return SubmissionQueue{
1397		.head = @ptrCast(@alignCast(&mmap[p.sq_off.head])),
1398		.tail = @ptrCast(@alignCast(&mmap[p.sq_off.tail])),
1399		.mask = @as(u32, @ptrCast(@alignCast(&mmap[p.sq_off.ring_mask]))).,
1400		.flags = @ptrCast(@alignCast(&mmap[p.sq_off.flags])),
1401		.dropped = @ptrCast(@alignCast(&mmap[p.sq_off.dropped])),
1402		.array = array[0..p.sq_entries],
1403		.sqes = sqes[0..p.sq_entries],
1404		.mmap = mmap,
1405		.mmap_sqes = mmap_sqes,
1406		};
1407		}
1408
1409		pub fn deinit(self: *SubmissionQueue) void {
1410		os.munmap(self.mmap_sqes);
1411		os.munmap(self.mmap);
1412		}
1413		};
1414
1415		pub const CompletionQueue = struct {
1416		head: *u32,
1417		tail: *u32,
1418		mask: u32,
1419		overflow: *u32,
1420		cqes: []linux.io_uring_cqe,
1421
1422		pub fn init(fd: os.fd_t, p: linux.io_uring_params, sq: SubmissionQueue) !CompletionQueue {
1423		assert(fd >= 0);
1424		assert((p.features & linux.IORING_FEAT_SINGLE_MMAP) != 0);
1425		const mmap = sq.mmap;
1426		const cqes: [*]linux.io_uring_cqe = @ptrCast(@alignCast(&mmap[p.cq_off.cqes]));
1427		assert(p.cq_entries == @as(u32, @ptrCast(@alignCast(&mmap[p.cq_off.ring_entries]))).);
1428		return CompletionQueue{
1429		.head = @ptrCast(@alignCast(&mmap[p.cq_off.head])),
1430		.tail = @ptrCast(@alignCast(&mmap[p.cq_off.tail])),
1431		.mask = @as(u32, @ptrCast(@alignCast(&mmap[p.cq_off.ring_mask]))).,
1432		.overflow = @ptrCast(@alignCast(&mmap[p.cq_off.overflow])),
1433		.cqes = cqes[0..p.cq_entries],
1434		};
1435		}
1436
1437		pub fn deinit(self: *CompletionQueue) void {
1438		_ = self;
1439		// A no-op since we now share the mmap with the submission queue.
1440		// Here for symmetry with the submission queue, and for any future feature support.
1441		}
1442		};
1443
1444		pub fn io_uring_prep_nop(sqe: *linux.io_uring_sqe) void {
1445		sqe.* = .{
1446		.opcode = .NOP,
1447		.flags = 0,
1448		.ioprio = 0,
1449		.fd = 0,
1450		.off = 0,
1451		.addr = 0,
1452		.len = 0,
1453		.rw_flags = 0,
1454		.user_data = 0,
1455		.buf_index = 0,
1456		.personality = 0,
1457		.splice_fd_in = 0,
1458		.addr3 = 0,
1459		.resv = 0,
1460		};
1461		}
1462
1463		pub fn io_uring_prep_fsync(sqe: *linux.io_uring_sqe, fd: os.fd_t, flags: u32) void {
1464		sqe.* = .{
1465		.opcode = .FSYNC,
1466		.flags = 0,
1467		.ioprio = 0,
1468		.fd = fd,
1469		.off = 0,
1470		.addr = 0,
1471		.len = 0,
1472		.rw_flags = flags,
1473		.user_data = 0,
1474		.buf_index = 0,
1475		.personality = 0,
1476		.splice_fd_in = 0,
1477		.addr3 = 0,
1478		.resv = 0,
1479		};
1480		}
1481
1482		pub fn io_uring_prep_rw(
1483		op: linux.IORING_OP,
1484		sqe: *linux.io_uring_sqe,
1485		fd: os.fd_t,
1486		addr: u64,
1487		len: usize,
1488		offset: u64,
1489		) void {
1490		sqe.* = .{
1491		.opcode = op,
1492		.flags = 0,
1493		.ioprio = 0,
1494		.fd = fd,
1495		.off = offset,
1496		.addr = addr,
1497		.len = @as(u32, @intCast(len)),
1498		.rw_flags = 0,
1499		.user_data = 0,
1500		.buf_index = 0,
1501		.personality = 0,
1502		.splice_fd_in = 0,
1503		.addr3 = 0,
1504		.resv = 0,
1505		};
1506		}
1507
1508		pub fn io_uring_prep_read(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []u8, offset: u64) void {
1509		io_uring_prep_rw(.READ, sqe, fd, @intFromPtr(buffer.ptr), buffer.len, offset);
1510		}
1511
1512		pub fn io_uring_prep_write(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, offset: u64) void {
1513		io_uring_prep_rw(.WRITE, sqe, fd, @intFromPtr(buffer.ptr), buffer.len, offset);
1514		}
1515
1516		pub fn io_uring_prep_splice(sqe: *linux.io_uring_sqe, fd_in: os.fd_t, off_in: u64, fd_out: os.fd_t, off_out: u64, len: usize) void {
1517		io_uring_prep_rw(.SPLICE, sqe, fd_out, undefined, len, off_out);
1518		sqe.addr = off_in;
1519		sqe.splice_fd_in = fd_in;
1520		}
1521
1522		pub fn io_uring_prep_readv(
1523		sqe: *linux.io_uring_sqe,
1524		fd: os.fd_t,
1525		iovecs: []const os.iovec,
1526		offset: u64,
1527		) void {
1528		io_uring_prep_rw(.READV, sqe, fd, @intFromPtr(iovecs.ptr), iovecs.len, offset);
1529		}
1530
1531		pub fn io_uring_prep_writev(
1532		sqe: *linux.io_uring_sqe,
1533		fd: os.fd_t,
1534		iovecs: []const os.iovec_const,
1535		offset: u64,
1536		) void {
1537		io_uring_prep_rw(.WRITEV, sqe, fd, @intFromPtr(iovecs.ptr), iovecs.len, offset);
1538		}
1539
1540		pub fn io_uring_prep_read_fixed(sqe: linux.io_uring_sqe, fd: os.fd_t, buffer: os.iovec, offset: u64, buffer_index: u16) void {
1541		io_uring_prep_rw(.READ_FIXED, sqe, fd, @intFromPtr(buffer.iov_base), buffer.iov_len, offset);
1542		sqe.buf_index = buffer_index;
1543		}
1544
1545		pub fn io_uring_prep_write_fixed(sqe: linux.io_uring_sqe, fd: os.fd_t, buffer: os.iovec, offset: u64, buffer_index: u16) void {
1546		io_uring_prep_rw(.WRITE_FIXED, sqe, fd, @intFromPtr(buffer.iov_base), buffer.iov_len, offset);
1547		sqe.buf_index = buffer_index;
1548		}
1549
1550		/// Poll masks previously used to comprise of 16 bits in the flags union of
1551		/// a SQE, but were then extended to comprise of 32 bits in order to make
1552		/// room for additional option flags. To ensure that the correct bits of
1553		/// poll masks are consistently and properly read across multiple kernel
1554		/// versions, poll masks are enforced to be little-endian.
1555		/// https://www.spinics.net/lists/io-uring/msg02848.html
1556		pub inline fn __io_uring_prep_poll_mask(poll_mask: u32) u32 {
1557		return std.mem.nativeToLittle(u32, poll_mask);
1558		}
1559
1560		pub fn io_uring_prep_accept(
1561		sqe: *linux.io_uring_sqe,
1562		fd: os.fd_t,
1563		addr: ?*os.sockaddr,
1564		addrlen: ?*os.socklen_t,
1565		flags: u32,
1566		) void {
1567		// `addr` holds a pointer to `sockaddr`, and `addr2` holds a pointer to socklen_t`.
1568		// `addr2` maps to `sqe.off` (u64) instead of `sqe.len` (which is only a u32).
1569		io_uring_prep_rw(.ACCEPT, sqe, fd, @intFromPtr(addr), 0, @intFromPtr(addrlen));
1570		sqe.rw_flags = flags;
1571		}
1572
1573		pub fn io_uring_prep_accept_direct(
1574		sqe: *linux.io_uring_sqe,
1575		fd: os.fd_t,
1576		addr: ?*os.sockaddr,
1577		addrlen: ?*os.socklen_t,
1578		flags: u32,
1579		file_index: u32,
1580		) void {
1581		io_uring_prep_accept(sqe, fd, addr, addrlen, flags);
1582		__io_uring_set_target_fixed_file(sqe, file_index);
1583		}
1584
1585		pub fn io_uring_prep_multishot_accept_direct(
1586		sqe: *linux.io_uring_sqe,
1587		fd: os.fd_t,
1588		addr: ?*os.sockaddr,
1589		addrlen: ?*os.socklen_t,
1590		flags: u32,
1591		) void {
1592		io_uring_prep_multishot_accept(sqe, fd, addr, addrlen, flags);
1593		__io_uring_set_target_fixed_file(sqe, linux.IORING_FILE_INDEX_ALLOC);
1594		}
1595
1596		fn __io_uring_set_target_fixed_file(sqe: *linux.io_uring_sqe, file_index: u32) void {
1597		const sqe_file_index: u32 = if (file_index == linux.IORING_FILE_INDEX_ALLOC)
1598		linux.IORING_FILE_INDEX_ALLOC
1599		else
1600		// 0 means no fixed files, indexes should be encoded as "index + 1"
1601		file_index + 1;
1602		// This filed is overloaded in liburing:
1603		// splice_fd_in: i32
1604		// sqe_file_index: u32
1605		sqe.splice_fd_in = @bitCast(sqe_file_index);
1606		}
1607
1608		pub fn io_uring_prep_connect(
1609		sqe: *linux.io_uring_sqe,
1610		fd: os.fd_t,
1611		addr: *const os.sockaddr,
1612		addrlen: os.socklen_t,
1613		) void {
1614		// `addrlen` maps to `sqe.off` (u64) instead of `sqe.len` (which is only a u32).
1615		io_uring_prep_rw(.CONNECT, sqe, fd, @intFromPtr(addr), 0, addrlen);
1616		}
1617
1618		pub fn io_uring_prep_epoll_ctl(
1619		sqe: *linux.io_uring_sqe,
1620		epfd: os.fd_t,
1621		fd: os.fd_t,
1622		op: u32,
1623		ev: ?*linux.epoll_event,
1624		) void {
1625		io_uring_prep_rw(.EPOLL_CTL, sqe, epfd, @intFromPtr(ev), op, @as(u64, @intCast(fd)));
1626		}
1627
1628		pub fn io_uring_prep_recv(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []u8, flags: u32) void {
1629		io_uring_prep_rw(.RECV, sqe, fd, @intFromPtr(buffer.ptr), buffer.len, 0);
1630		sqe.rw_flags = flags;
1631		}
1632
1633		pub fn io_uring_prep_send(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, flags: u32) void {
1634		io_uring_prep_rw(.SEND, sqe, fd, @intFromPtr(buffer.ptr), buffer.len, 0);
1635		sqe.rw_flags = flags;
1636		}
1637
1638		pub fn io_uring_prep_send_zc(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, flags: u32, zc_flags: u16) void {
1639		io_uring_prep_rw(.SEND_ZC, sqe, fd, @intFromPtr(buffer.ptr), buffer.len, 0);
1640		sqe.rw_flags = flags;
1641		sqe.ioprio = zc_flags;
1642		}
1643
1644		pub fn io_uring_prep_send_zc_fixed(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, flags: u32, zc_flags: u16, buf_index: u16) void {
1645		io_uring_prep_send_zc(sqe, fd, buffer, flags, zc_flags);
1646		sqe.ioprio \|= linux.IORING_RECVSEND_FIXED_BUF;
1647		sqe.buf_index = buf_index;
1648		}
1649
1650		pub fn io_uring_prep_sendmsg_zc(
1651		sqe: *linux.io_uring_sqe,
1652		fd: os.fd_t,
1653		msg: *const os.msghdr_const,
1654		flags: u32,
1655		) void {
1656		io_uring_prep_sendmsg(sqe, fd, msg, flags);
1657		sqe.opcode = .SENDMSG_ZC;
1658		}
1659
1660		pub fn io_uring_prep_recvmsg(
1661		sqe: *linux.io_uring_sqe,
1662		fd: os.fd_t,
1663		msg: *os.msghdr,
1664		flags: u32,
1665		) void {
1666		linux.io_uring_prep_rw(.RECVMSG, sqe, fd, @intFromPtr(msg), 1, 0);
1667		sqe.rw_flags = flags;
1668		}
1669
1670		pub fn io_uring_prep_sendmsg(
1671		sqe: *linux.io_uring_sqe,
1672		fd: os.fd_t,
1673		msg: *const os.msghdr_const,
1674		flags: u32,
1675		) void {
1676		linux.io_uring_prep_rw(.SENDMSG, sqe, fd, @intFromPtr(msg), 1, 0);
1677		sqe.rw_flags = flags;
1678		}
1679
1680		pub fn io_uring_prep_openat(
1681		sqe: *linux.io_uring_sqe,
1682		fd: os.fd_t,
1683		path: [*:0]const u8,
1684		flags: linux.O,
1685		mode: os.mode_t,
1686		) void {
1687		io_uring_prep_rw(.OPENAT, sqe, fd, @intFromPtr(path), mode, 0);
1688		sqe.rw_flags = @bitCast(flags);
1689		}
1690
1691		pub fn io_uring_prep_openat_direct(
1692		sqe: *linux.io_uring_sqe,
1693		fd: os.fd_t,
1694		path: [*:0]const u8,
1695		flags: linux.O,
1696		mode: os.mode_t,
1697		file_index: u32,
1698		) void {
1699		io_uring_prep_openat(sqe, fd, path, flags, mode);
1700		__io_uring_set_target_fixed_file(sqe, file_index);
1701		}
1702
1703		pub fn io_uring_prep_close(sqe: *linux.io_uring_sqe, fd: os.fd_t) void {
1704		sqe.* = .{
1705		.opcode = .CLOSE,
1706		.flags = 0,
1707		.ioprio = 0,
1708		.fd = fd,
1709		.off = 0,
1710		.addr = 0,
1711		.len = 0,
1712		.rw_flags = 0,
1713		.user_data = 0,
1714		.buf_index = 0,
1715		.personality = 0,
1716		.splice_fd_in = 0,
1717		.addr3 = 0,
1718		.resv = 0,
1719		};
1720		}
1721
1722		pub fn io_uring_prep_close_direct(sqe: *linux.io_uring_sqe, file_index: u32) void {
1723		io_uring_prep_close(sqe, 0);
1724		__io_uring_set_target_fixed_file(sqe, file_index);
1725		}
1726
1727		pub fn io_uring_prep_timeout(
1728		sqe: *linux.io_uring_sqe,
1729		ts: *const os.linux.kernel_timespec,
1730		count: u32,
1731		flags: u32,
1732		) void {
1733		io_uring_prep_rw(.TIMEOUT, sqe, -1, @intFromPtr(ts), 1, count);
1734		sqe.rw_flags = flags;
1735		}
1736
1737		pub fn io_uring_prep_timeout_remove(sqe: *linux.io_uring_sqe, timeout_user_data: u64, flags: u32) void {
1738		sqe.* = .{
1739		.opcode = .TIMEOUT_REMOVE,
1740		.flags = 0,
1741		.ioprio = 0,
1742		.fd = -1,
1743		.off = 0,
1744		.addr = timeout_user_data,
1745		.len = 0,
1746		.rw_flags = flags,
1747		.user_data = 0,
1748		.buf_index = 0,
1749		.personality = 0,
1750		.splice_fd_in = 0,
1751		.addr3 = 0,
1752		.resv = 0,
1753		};
1754		}
1755
1756		pub fn io_uring_prep_link_timeout(
1757		sqe: *linux.io_uring_sqe,
1758		ts: *const os.linux.kernel_timespec,
1759		flags: u32,
1760		) void {
1761		linux.io_uring_prep_rw(.LINK_TIMEOUT, sqe, -1, @intFromPtr(ts), 1, 0);
1762		sqe.rw_flags = flags;
1763		}
1764
1765		pub fn io_uring_prep_poll_add(
1766		sqe: *linux.io_uring_sqe,
1767		fd: os.fd_t,
1768		poll_mask: u32,
1769		) void {
1770		io_uring_prep_rw(.POLL_ADD, sqe, fd, @intFromPtr(@as(?*anyopaque, null)), 0, 0);
1771		sqe.rw_flags = __io_uring_prep_poll_mask(poll_mask);
1772		}
1773
1774		pub fn io_uring_prep_poll_remove(
1775		sqe: *linux.io_uring_sqe,
1776		target_user_data: u64,
1777		) void {
1778		io_uring_prep_rw(.POLL_REMOVE, sqe, -1, target_user_data, 0, 0);
1779		}
1780
1781		pub fn io_uring_prep_poll_update(
1782		sqe: *linux.io_uring_sqe,
1783		old_user_data: u64,
1784		new_user_data: u64,
1785		poll_mask: u32,
1786		flags: u32,
1787		) void {
1788		io_uring_prep_rw(.POLL_REMOVE, sqe, -1, old_user_data, flags, new_user_data);
1789		sqe.rw_flags = __io_uring_prep_poll_mask(poll_mask);
1790		}
1791
1792		pub fn io_uring_prep_fallocate(
1793		sqe: *linux.io_uring_sqe,
1794		fd: os.fd_t,
1795		mode: i32,
1796		offset: u64,
1797		len: u64,
1798		) void {
1799		sqe.* = .{
1800		.opcode = .FALLOCATE,
1801		.flags = 0,
1802		.ioprio = 0,
1803		.fd = fd,
1804		.off = offset,
1805		.addr = len,
1806		.len = @as(u32, @intCast(mode)),
1807		.rw_flags = 0,
1808		.user_data = 0,
1809		.buf_index = 0,
1810		.personality = 0,
1811		.splice_fd_in = 0,
1812		.addr3 = 0,
1813		.resv = 0,
1814		};
1815		}
1816
1817		pub fn io_uring_prep_statx(
1818		sqe: *linux.io_uring_sqe,
1819		fd: os.fd_t,
1820		path: [*:0]const u8,
1821		flags: u32,
1822		mask: u32,
1823		buf: *linux.Statx,
1824		) void {
1825		io_uring_prep_rw(.STATX, sqe, fd, @intFromPtr(path), mask, @intFromPtr(buf));
1826		sqe.rw_flags = flags;
1827		}
1828
1829		pub fn io_uring_prep_cancel(
1830		sqe: *linux.io_uring_sqe,
1831		cancel_user_data: u64,
1832		flags: u32,
1833		) void {
1834		io_uring_prep_rw(.ASYNC_CANCEL, sqe, -1, cancel_user_data, 0, 0);
1835		sqe.rw_flags = flags;
1836		}
1837
1838		pub fn io_uring_prep_shutdown(
1839		sqe: *linux.io_uring_sqe,
1840		sockfd: os.socket_t,
1841		how: u32,
1842		) void {
1843		io_uring_prep_rw(.SHUTDOWN, sqe, sockfd, 0, how, 0);
1844		}
1845
1846		pub fn io_uring_prep_renameat(
1847		sqe: *linux.io_uring_sqe,
1848		old_dir_fd: os.fd_t,
1849		old_path: [*:0]const u8,
1850		new_dir_fd: os.fd_t,
1851		new_path: [*:0]const u8,
1852		flags: u32,
1853		) void {
1854		io_uring_prep_rw(
1855		.RENAMEAT,
1856		sqe,
1857		old_dir_fd,
1858		@intFromPtr(old_path),
1859		0,
1860		@intFromPtr(new_path),
1861		);
1862		sqe.len = @bitCast(new_dir_fd);
1863		sqe.rw_flags = flags;
1864		}
1865
1866		pub fn io_uring_prep_unlinkat(
1867		sqe: *linux.io_uring_sqe,
1868		dir_fd: os.fd_t,
1869		path: [*:0]const u8,
1870		flags: u32,
1871		) void {
1872		io_uring_prep_rw(.UNLINKAT, sqe, dir_fd, @intFromPtr(path), 0, 0);
1873		sqe.rw_flags = flags;
1874		}
1875
1876		pub fn io_uring_prep_mkdirat(
1877		sqe: *linux.io_uring_sqe,
1878		dir_fd: os.fd_t,
1879		path: [*:0]const u8,
1880		mode: os.mode_t,
1881		) void {
1882		io_uring_prep_rw(.MKDIRAT, sqe, dir_fd, @intFromPtr(path), mode, 0);
1883		}
1884
1885		pub fn io_uring_prep_symlinkat(
1886		sqe: *linux.io_uring_sqe,
1887		target: [*:0]const u8,
1888		new_dir_fd: os.fd_t,
1889		link_path: [*:0]const u8,
1890		) void {
1891		io_uring_prep_rw(
1892		.SYMLINKAT,
1893		sqe,
1894		new_dir_fd,
1895		@intFromPtr(target),
1896		0,
1897		@intFromPtr(link_path),
1898		);
1899		}
1900
1901		pub fn io_uring_prep_linkat(
1902		sqe: *linux.io_uring_sqe,
1903		old_dir_fd: os.fd_t,
1904		old_path: [*:0]const u8,
1905		new_dir_fd: os.fd_t,
1906		new_path: [*:0]const u8,
1907		flags: u32,
1908		) void {
1909		io_uring_prep_rw(
1910		.LINKAT,
1911		sqe,
1912		old_dir_fd,
1913		@intFromPtr(old_path),
1914		0,
1915		@intFromPtr(new_path),
1916		);
1917		sqe.len = @bitCast(new_dir_fd);
1918		sqe.rw_flags = flags;
1919		}
1920
1921		pub fn io_uring_prep_provide_buffers(
1922		sqe: *linux.io_uring_sqe,
1923		buffers: [*]u8,
1924		buffer_len: usize,
1925		num: usize,
1926		group_id: usize,
1927		buffer_id: usize,
1928		) void {
1929		const ptr = @intFromPtr(buffers);
1930		io_uring_prep_rw(.PROVIDE_BUFFERS, sqe, @as(i32, @intCast(num)), ptr, buffer_len, buffer_id);
1931		sqe.buf_index = @intCast(group_id);
1932		}
1933
1934		pub fn io_uring_prep_remove_buffers(
1935		sqe: *linux.io_uring_sqe,
1936		num: usize,
1937		group_id: usize,
1938		) void {
1939		io_uring_prep_rw(.REMOVE_BUFFERS, sqe, @as(i32, @intCast(num)), 0, 0, 0);
1940		sqe.buf_index = @intCast(group_id);
1941		}
1942
1943		pub fn io_uring_prep_multishot_accept(
1944		sqe: *linux.io_uring_sqe,
1945		fd: os.fd_t,
1946		addr: ?*os.sockaddr,
1947		addrlen: ?*os.socklen_t,
1948		flags: u32,
1949		) void {
1950		io_uring_prep_accept(sqe, fd, addr, addrlen, flags);
1951		sqe.ioprio \|= linux.IORING_ACCEPT_MULTISHOT;
1952		}
1953
1954		pub fn io_uring_prep_socket(
1955		sqe: *linux.io_uring_sqe,
1956		domain: u32,
1957		socket_type: u32,
1958		protocol: u32,
1959		flags: u32,
1960		) void {
1961		io_uring_prep_rw(.SOCKET, sqe, @intCast(domain), 0, protocol, socket_type);
1962		sqe.rw_flags = flags;
1963		}
1964
1965		pub fn io_uring_prep_socket_direct(
1966		sqe: *linux.io_uring_sqe,
1967		domain: u32,
1968		socket_type: u32,
1969		protocol: u32,
1970		flags: u32,
1971		file_index: u32,
1972		) void {
1973		io_uring_prep_socket(sqe, domain, socket_type, protocol, flags);
1974		__io_uring_set_target_fixed_file(sqe, file_index);
1975		}
1976
1977		pub fn io_uring_prep_socket_direct_alloc(
1978		sqe: *linux.io_uring_sqe,
1979		domain: u32,
1980		socket_type: u32,
1981		protocol: u32,
1982		flags: u32,
1983		) void {
1984		io_uring_prep_socket(sqe, domain, socket_type, protocol, flags);
1985		__io_uring_set_target_fixed_file(sqe, linux.IORING_FILE_INDEX_ALLOC);
1986		}
1987
1988		pub fn io_uring_prep_waitid(
1989		sqe: *linux.io_uring_sqe,
1990		id_type: linux.P,
1991		id: i32,
1992		infop: *linux.siginfo_t,
1993		options: u32,
1994		flags: u32,
1995		) void {
1996		io_uring_prep_rw(.WAITID, sqe, id, 0, @intFromEnum(id_type), @intFromPtr(infop));
1997		sqe.rw_flags = flags;
1998		sqe.splice_fd_in = @bitCast(options);
1999		}
2000
2001		test "structs/offsets/entries" {
2002		if (builtin.os.tag != .linux) return error.SkipZigTest;
2003
2004		try testing.expectEqual(@as(usize, 120), @sizeOf(linux.io_uring_params));
2005		try testing.expectEqual(@as(usize, 64), @sizeOf(linux.io_uring_sqe));
2006		try testing.expectEqual(@as(usize, 16), @sizeOf(linux.io_uring_cqe));
2007
2008		try testing.expectEqual(0, linux.IORING_OFF_SQ_RING);
2009		try testing.expectEqual(0x8000000, linux.IORING_OFF_CQ_RING);
2010		try testing.expectEqual(0x10000000, linux.IORING_OFF_SQES);
2011
2012		try testing.expectError(error.EntriesZero, IO_Uring.init(0, 0));
2013		try testing.expectError(error.EntriesNotPowerOfTwo, IO_Uring.init(3, 0));
2014		}
2015
2016		test "nop" {
2017		if (builtin.os.tag != .linux) return error.SkipZigTest;
2018
2019		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
2020		error.SystemOutdated => return error.SkipZigTest,
2021		error.PermissionDenied => return error.SkipZigTest,
2022		else => return err,
2023		};
2024		defer {
2025		ring.deinit();
2026		testing.expectEqual(@as(os.fd_t, -1), ring.fd) catch @panic("test failed");
2027		}
2028
2029		const sqe = try ring.nop(0xaaaaaaaa);
2030		try testing.expectEqual(linux.io_uring_sqe{
2031		.opcode = .NOP,
2032		.flags = 0,
2033		.ioprio = 0,
2034		.fd = 0,
2035		.off = 0,
2036		.addr = 0,
2037		.len = 0,
2038		.rw_flags = 0,
2039		.user_data = 0xaaaaaaaa,
2040		.buf_index = 0,
2041		.personality = 0,
2042		.splice_fd_in = 0,
2043		.addr3 = 0,
2044		.resv = 0,
2045		}, sqe.*);
2046
2047		try testing.expectEqual(@as(u32, 0), ring.sq.sqe_head);
2048		try testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
2049		try testing.expectEqual(@as(u32, 0), ring.sq.tail.*);
2050		try testing.expectEqual(@as(u32, 0), ring.cq.head.*);
2051		try testing.expectEqual(@as(u32, 1), ring.sq_ready());
2052		try testing.expectEqual(@as(u32, 0), ring.cq_ready());
2053
2054		try testing.expectEqual(@as(u32, 1), try ring.submit());
2055		try testing.expectEqual(@as(u32, 1), ring.sq.sqe_head);
2056		try testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
2057		try testing.expectEqual(@as(u32, 1), ring.sq.tail.*);
2058		try testing.expectEqual(@as(u32, 0), ring.cq.head.*);
2059		try testing.expectEqual(@as(u32, 0), ring.sq_ready());
2060
2061		try testing.expectEqual(linux.io_uring_cqe{
2062		.user_data = 0xaaaaaaaa,
2063		.res = 0,
2064		.flags = 0,
2065		}, try ring.copy_cqe());
2066		try testing.expectEqual(@as(u32, 1), ring.cq.head.*);
2067		try testing.expectEqual(@as(u32, 0), ring.cq_ready());
2068
2069		const sqe_barrier = try ring.nop(0xbbbbbbbb);
2070		sqe_barrier.flags \|= linux.IOSQE_IO_DRAIN;
2071		try testing.expectEqual(@as(u32, 1), try ring.submit());
2072		try testing.expectEqual(linux.io_uring_cqe{
2073		.user_data = 0xbbbbbbbb,
2074		.res = 0,
2075		.flags = 0,
2076		}, try ring.copy_cqe());
2077		try testing.expectEqual(@as(u32, 2), ring.sq.sqe_head);
2078		try testing.expectEqual(@as(u32, 2), ring.sq.sqe_tail);
2079		try testing.expectEqual(@as(u32, 2), ring.sq.tail.*);
2080		try testing.expectEqual(@as(u32, 2), ring.cq.head.*);
2081		}
2082
2083		test "readv" {
2084		if (builtin.os.tag != .linux) return error.SkipZigTest;
2085
2086		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
2087		error.SystemOutdated => return error.SkipZigTest,
2088		error.PermissionDenied => return error.SkipZigTest,
2089		else => return err,
2090		};
2091		defer ring.deinit();
2092
2093		const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
2094		defer os.close(fd);
2095
2096		// Linux Kernel 5.4 supports IORING_REGISTER_FILES but not sparse fd sets (i.e. an fd of -1).
2097		// Linux Kernel 5.5 adds support for sparse fd sets.
2098		// Compare:
2099		// https://github.com/torvalds/linux/blob/v5.4/fs/io_uring.c#L3119-L3124 vs
2100		// https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L6687-L6691
2101		// We therefore avoid stressing sparse fd sets here:
2102		var registered_fds = [_]os.fd_t{0} ** 1;
2103		const fd_index = 0;
2104		registered_fds[fd_index] = fd;
2105		try ring.register_files(registered_fds[0..]);
2106
2107		var buffer = [_]u8{42} ** 128;
2108		var iovecs = [_]os.iovec{os.iovec{ .iov_base = &buffer, .iov_len = buffer.len }};
2109		const sqe = try ring.read(0xcccccccc, fd_index, .{ .iovecs = iovecs[0..] }, 0);
2110		try testing.expectEqual(linux.IORING_OP.READV, sqe.opcode);
2111		sqe.flags \|= linux.IOSQE_FIXED_FILE;
2112
2113		try testing.expectError(error.SubmissionQueueFull, ring.nop(0));
2114		try testing.expectEqual(@as(u32, 1), try ring.submit());
2115		try testing.expectEqual(linux.io_uring_cqe{
2116		.user_data = 0xcccccccc,
2117		.res = buffer.len,
2118		.flags = 0,
2119		}, try ring.copy_cqe());
2120		try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
2121
2122		try ring.unregister_files();
2123		}
2124
2125		test "writev/fsync/readv" {
2126		if (builtin.os.tag != .linux) return error.SkipZigTest;
2127
2128		var ring = IO_Uring.init(4, 0) catch \|err\| switch (err) {
2129		error.SystemOutdated => return error.SkipZigTest,
2130		error.PermissionDenied => return error.SkipZigTest,
2131		else => return err,
2132		};
2133		defer ring.deinit();
2134
2135		var tmp = std.testing.tmpDir(.{});
2136		defer tmp.cleanup();
2137
2138		const path = "test_io_uring_writev_fsync_readv";
2139		const file = try tmp.dir.createFile(path, .{ .read = true, .truncate = true });
2140		defer file.close();
2141		const fd = file.handle;
2142
2143		const buffer_write = [_]u8{42} ** 128;
2144		const iovecs_write = [_]os.iovec_const{
2145		os.iovec_const{ .iov_base = &buffer_write, .iov_len = buffer_write.len },
2146		};
2147		var buffer_read = [_]u8{0} ** 128;
2148		var iovecs_read = [_]os.iovec{
2149		os.iovec{ .iov_base = &buffer_read, .iov_len = buffer_read.len },
2150		};
2151
2152		const sqe_writev = try ring.writev(0xdddddddd, fd, iovecs_write[0..], 17);
2153		try testing.expectEqual(linux.IORING_OP.WRITEV, sqe_writev.opcode);
2154		try testing.expectEqual(@as(u64, 17), sqe_writev.off);
2155		sqe_writev.flags \|= linux.IOSQE_IO_LINK;
2156
2157		const sqe_fsync = try ring.fsync(0xeeeeeeee, fd, 0);
2158		try testing.expectEqual(linux.IORING_OP.FSYNC, sqe_fsync.opcode);
2159		try testing.expectEqual(fd, sqe_fsync.fd);
2160		sqe_fsync.flags \|= linux.IOSQE_IO_LINK;
2161
2162		const sqe_readv = try ring.read(0xffffffff, fd, .{ .iovecs = iovecs_read[0..] }, 17);
2163		try testing.expectEqual(linux.IORING_OP.READV, sqe_readv.opcode);
2164		try testing.expectEqual(@as(u64, 17), sqe_readv.off);
2165
2166		try testing.expectEqual(@as(u32, 3), ring.sq_ready());
2167		try testing.expectEqual(@as(u32, 3), try ring.submit_and_wait(3));
2168		try testing.expectEqual(@as(u32, 0), ring.sq_ready());
2169		try testing.expectEqual(@as(u32, 3), ring.cq_ready());
2170
2171		try testing.expectEqual(linux.io_uring_cqe{
2172		.user_data = 0xdddddddd,
2173		.res = buffer_write.len,
2174		.flags = 0,
2175		}, try ring.copy_cqe());
2176		try testing.expectEqual(@as(u32, 2), ring.cq_ready());
2177
2178		try testing.expectEqual(linux.io_uring_cqe{
2179		.user_data = 0xeeeeeeee,
2180		.res = 0,
2181		.flags = 0,
2182		}, try ring.copy_cqe());
2183		try testing.expectEqual(@as(u32, 1), ring.cq_ready());
2184
2185		try testing.expectEqual(linux.io_uring_cqe{
2186		.user_data = 0xffffffff,
2187		.res = buffer_read.len,
2188		.flags = 0,
2189		}, try ring.copy_cqe());
2190		try testing.expectEqual(@as(u32, 0), ring.cq_ready());
2191
2192		try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
2193		}
2194
2195		test "write/read" {
2196		if (builtin.os.tag != .linux) return error.SkipZigTest;
2197
2198		var ring = IO_Uring.init(2, 0) catch \|err\| switch (err) {
2199		error.SystemOutdated => return error.SkipZigTest,
2200		error.PermissionDenied => return error.SkipZigTest,
2201		else => return err,
2202		};
2203		defer ring.deinit();
2204
2205		var tmp = std.testing.tmpDir(.{});
2206		defer tmp.cleanup();
2207		const path = "test_io_uring_write_read";
2208		const file = try tmp.dir.createFile(path, .{ .read = true, .truncate = true });
2209		defer file.close();
2210		const fd = file.handle;
2211
2212		const buffer_write = [_]u8{97} ** 20;
2213		var buffer_read = [_]u8{98} ** 20;
2214		const sqe_write = try ring.write(0x11111111, fd, buffer_write[0..], 10);
2215		try testing.expectEqual(linux.IORING_OP.WRITE, sqe_write.opcode);
2216		try testing.expectEqual(@as(u64, 10), sqe_write.off);
2217		sqe_write.flags \|= linux.IOSQE_IO_LINK;
2218		const sqe_read = try ring.read(0x22222222, fd, .{ .buffer = buffer_read[0..] }, 10);
2219		try testing.expectEqual(linux.IORING_OP.READ, sqe_read.opcode);
2220		try testing.expectEqual(@as(u64, 10), sqe_read.off);
2221		try testing.expectEqual(@as(u32, 2), try ring.submit());
2222
2223		const cqe_write = try ring.copy_cqe();
2224		const cqe_read = try ring.copy_cqe();
2225		// Prior to Linux Kernel 5.6 this is the only way to test for read/write support:
2226		// https://lwn.net/Articles/809820/
2227		if (cqe_write.err() == .INVAL) return error.SkipZigTest;
2228		if (cqe_read.err() == .INVAL) return error.SkipZigTest;
2229		try testing.expectEqual(linux.io_uring_cqe{
2230		.user_data = 0x11111111,
2231		.res = buffer_write.len,
2232		.flags = 0,
2233		}, cqe_write);
2234		try testing.expectEqual(linux.io_uring_cqe{
2235		.user_data = 0x22222222,
2236		.res = buffer_read.len,
2237		.flags = 0,
2238		}, cqe_read);
2239		try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
2240		}
2241
2242		test "splice/read" {
2243		if (builtin.os.tag != .linux) return error.SkipZigTest;
2244
2245		var ring = IO_Uring.init(4, 0) catch \|err\| switch (err) {
2246		error.SystemOutdated => return error.SkipZigTest,
2247		error.PermissionDenied => return error.SkipZigTest,
2248		else => return err,
2249		};
2250		defer ring.deinit();
2251
2252		var tmp = std.testing.tmpDir(.{});
2253		const path_src = "test_io_uring_splice_src";
2254		const file_src = try tmp.dir.createFile(path_src, .{ .read = true, .truncate = true });
2255		defer file_src.close();
2256		const fd_src = file_src.handle;
2257
2258		const path_dst = "test_io_uring_splice_dst";
2259		const file_dst = try tmp.dir.createFile(path_dst, .{ .read = true, .truncate = true });
2260		defer file_dst.close();
2261		const fd_dst = file_dst.handle;
2262
2263		const buffer_write = [_]u8{97} ** 20;
2264		var buffer_read = [_]u8{98} ** 20;
2265		_ = try file_src.write(&buffer_write);
2266
2267		const fds = try os.pipe();
2268		const pipe_offset: u64 = std.math.maxInt(u64);
2269
2270		const sqe_splice_to_pipe = try ring.splice(0x11111111, fd_src, 0, fds[1], pipe_offset, buffer_write.len);
2271		try testing.expectEqual(linux.IORING_OP.SPLICE, sqe_splice_to_pipe.opcode);
2272		try testing.expectEqual(@as(u64, 0), sqe_splice_to_pipe.addr);
2273		try testing.expectEqual(pipe_offset, sqe_splice_to_pipe.off);
2274		sqe_splice_to_pipe.flags \|= linux.IOSQE_IO_LINK;
2275
2276		const sqe_splice_from_pipe = try ring.splice(0x22222222, fds[0], pipe_offset, fd_dst, 10, buffer_write.len);
2277		try testing.expectEqual(linux.IORING_OP.SPLICE, sqe_splice_from_pipe.opcode);
2278		try testing.expectEqual(pipe_offset, sqe_splice_from_pipe.addr);
2279		try testing.expectEqual(@as(u64, 10), sqe_splice_from_pipe.off);
2280		sqe_splice_from_pipe.flags \|= linux.IOSQE_IO_LINK;
2281
2282		const sqe_read = try ring.read(0x33333333, fd_dst, .{ .buffer = buffer_read[0..] }, 10);
2283		try testing.expectEqual(linux.IORING_OP.READ, sqe_read.opcode);
2284		try testing.expectEqual(@as(u64, 10), sqe_read.off);
2285		try testing.expectEqual(@as(u32, 3), try ring.submit());
2286
2287		const cqe_splice_to_pipe = try ring.copy_cqe();
2288		const cqe_splice_from_pipe = try ring.copy_cqe();
2289		const cqe_read = try ring.copy_cqe();
2290		// Prior to Linux Kernel 5.6 this is the only way to test for splice/read support:
2291		// https://lwn.net/Articles/809820/
2292		if (cqe_splice_to_pipe.err() == .INVAL) return error.SkipZigTest;
2293		if (cqe_splice_from_pipe.err() == .INVAL) return error.SkipZigTest;
2294		if (cqe_read.err() == .INVAL) return error.SkipZigTest;
2295		try testing.expectEqual(linux.io_uring_cqe{
2296		.user_data = 0x11111111,
2297		.res = buffer_write.len,
2298		.flags = 0,
2299		}, cqe_splice_to_pipe);
2300		try testing.expectEqual(linux.io_uring_cqe{
2301		.user_data = 0x22222222,
2302		.res = buffer_write.len,
2303		.flags = 0,
2304		}, cqe_splice_from_pipe);
2305		try testing.expectEqual(linux.io_uring_cqe{
2306		.user_data = 0x33333333,
2307		.res = buffer_read.len,
2308		.flags = 0,
2309		}, cqe_read);
2310		try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
2311		}
2312
2313		test "write_fixed/read_fixed" {
2314		if (builtin.os.tag != .linux) return error.SkipZigTest;
2315
2316		var ring = IO_Uring.init(2, 0) catch \|err\| switch (err) {
2317		error.SystemOutdated => return error.SkipZigTest,
2318		error.PermissionDenied => return error.SkipZigTest,
2319		else => return err,
2320		};
2321		defer ring.deinit();
2322
2323		var tmp = std.testing.tmpDir(.{});
2324		defer tmp.cleanup();
2325
2326		const path = "test_io_uring_write_read_fixed";
2327		const file = try tmp.dir.createFile(path, .{ .read = true, .truncate = true });
2328		defer file.close();
2329		const fd = file.handle;
2330
2331		var raw_buffers: [2][11]u8 = undefined;
2332		// First buffer will be written to the file.
2333		@memset(&raw_buffers[0], 'z');
2334		raw_buffers[0][0.."foobar".len].* = "foobar".*;
2335
2336		var buffers = [2]os.iovec{
2337		.{ .iov_base = &raw_buffers[0], .iov_len = raw_buffers[0].len },
2338		.{ .iov_base = &raw_buffers[1], .iov_len = raw_buffers[1].len },
2339		};
2340		ring.register_buffers(&buffers) catch \|err\| switch (err) {
2341		error.SystemResources => {
2342		// See https://github.com/ziglang/zig/issues/15362
2343		return error.SkipZigTest;
2344		},
2345		else => \|e\| return e,
2346		};
2347
2348		const sqe_write = try ring.write_fixed(0x45454545, fd, &buffers[0], 3, 0);
2349		try testing.expectEqual(linux.IORING_OP.WRITE_FIXED, sqe_write.opcode);
2350		try testing.expectEqual(@as(u64, 3), sqe_write.off);
2351		sqe_write.flags \|= linux.IOSQE_IO_LINK;
2352
2353		const sqe_read = try ring.read_fixed(0x12121212, fd, &buffers[1], 0, 1);
2354		try testing.expectEqual(linux.IORING_OP.READ_FIXED, sqe_read.opcode);
2355		try testing.expectEqual(@as(u64, 0), sqe_read.off);
2356
2357		try testing.expectEqual(@as(u32, 2), try ring.submit());
2358
2359		const cqe_write = try ring.copy_cqe();
2360		const cqe_read = try ring.copy_cqe();
2361
2362		try testing.expectEqual(linux.io_uring_cqe{
2363		.user_data = 0x45454545,
2364		.res = @as(i32, @intCast(buffers[0].iov_len)),
2365		.flags = 0,
2366		}, cqe_write);
2367		try testing.expectEqual(linux.io_uring_cqe{
2368		.user_data = 0x12121212,
2369		.res = @as(i32, @intCast(buffers[1].iov_len)),
2370		.flags = 0,
2371		}, cqe_read);
2372
2373		try testing.expectEqualSlices(u8, "\x00\x00\x00", buffers[1].iov_base[0..3]);
2374		try testing.expectEqualSlices(u8, "foobar", buffers[1].iov_base[3..9]);
2375		try testing.expectEqualSlices(u8, "zz", buffers[1].iov_base[9..11]);
2376		}
2377
2378		test "openat" {
2379		if (builtin.os.tag != .linux) return error.SkipZigTest;
2380
2381		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
2382		error.SystemOutdated => return error.SkipZigTest,
2383		error.PermissionDenied => return error.SkipZigTest,
2384		else => return err,
2385		};
2386		defer ring.deinit();
2387
2388		var tmp = std.testing.tmpDir(.{});
2389		defer tmp.cleanup();
2390
2391		const path = "test_io_uring_openat";
2392
2393		// Workaround for LLVM bug: https://github.com/ziglang/zig/issues/12014
2394		const path_addr = if (builtin.zig_backend == .stage2_llvm) p: {
2395		var workaround = path;
2396		_ = &workaround;
2397		break :p @intFromPtr(workaround);
2398		} else @intFromPtr(path);
2399
2400		const flags: linux.O = .{ .CLOEXEC = true, .ACCMODE = .RDWR, .CREAT = true };
2401		const mode: os.mode_t = 0o666;
2402		const sqe_openat = try ring.openat(0x33333333, tmp.dir.fd, path, flags, mode);
2403		try testing.expectEqual(linux.io_uring_sqe{
2404		.opcode = .OPENAT,
2405		.flags = 0,
2406		.ioprio = 0,
2407		.fd = tmp.dir.fd,
2408		.off = 0,
2409		.addr = path_addr,
2410		.len = mode,
2411		.rw_flags = @bitCast(flags),
2412		.user_data = 0x33333333,
2413		.buf_index = 0,
2414		.personality = 0,
2415		.splice_fd_in = 0,
2416		.addr3 = 0,
2417		.resv = 0,
2418		}, sqe_openat.*);
2419		try testing.expectEqual(@as(u32, 1), try ring.submit());
2420
2421		const cqe_openat = try ring.copy_cqe();
2422		try testing.expectEqual(@as(u64, 0x33333333), cqe_openat.user_data);
2423		if (cqe_openat.err() == .INVAL) return error.SkipZigTest;
2424		if (cqe_openat.err() == .BADF) return error.SkipZigTest;
2425		if (cqe_openat.res <= 0) std.debug.print("\ncqe_openat.res={}\n", .{cqe_openat.res});
2426		try testing.expect(cqe_openat.res > 0);
2427		try testing.expectEqual(@as(u32, 0), cqe_openat.flags);
2428
2429		os.close(cqe_openat.res);
2430		}
2431
2432		test "close" {
2433		if (builtin.os.tag != .linux) return error.SkipZigTest;
2434
2435		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
2436		error.SystemOutdated => return error.SkipZigTest,
2437		error.PermissionDenied => return error.SkipZigTest,
2438		else => return err,
2439		};
2440		defer ring.deinit();
2441
2442		var tmp = std.testing.tmpDir(.{});
2443		defer tmp.cleanup();
2444
2445		const path = "test_io_uring_close";
2446		const file = try tmp.dir.createFile(path, .{});
2447		errdefer file.close();
2448
2449		const sqe_close = try ring.close(0x44444444, file.handle);
2450		try testing.expectEqual(linux.IORING_OP.CLOSE, sqe_close.opcode);
2451		try testing.expectEqual(file.handle, sqe_close.fd);
2452		try testing.expectEqual(@as(u32, 1), try ring.submit());
2453
2454		const cqe_close = try ring.copy_cqe();
2455		if (cqe_close.err() == .INVAL) return error.SkipZigTest;
2456		try testing.expectEqual(linux.io_uring_cqe{
2457		.user_data = 0x44444444,
2458		.res = 0,
2459		.flags = 0,
2460		}, cqe_close);
2461		}
2462
2463		test "accept/connect/send/recv" {
2464		if (builtin.os.tag != .linux) return error.SkipZigTest;
2465
2466		var ring = IO_Uring.init(16, 0) catch \|err\| switch (err) {
2467		error.SystemOutdated => return error.SkipZigTest,
2468		error.PermissionDenied => return error.SkipZigTest,
2469		else => return err,
2470		};
2471		defer ring.deinit();
2472
2473		const socket_test_harness = try createSocketTestHarness(&ring);
2474		defer socket_test_harness.close();
2475
2476		const buffer_send = [_]u8{ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 };
2477		var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
2478
2479		const send = try ring.send(0xeeeeeeee, socket_test_harness.client, buffer_send[0..], 0);
2480		send.flags \|= linux.IOSQE_IO_LINK;
2481		_ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
2482		try testing.expectEqual(@as(u32, 2), try ring.submit());
2483
2484		const cqe_send = try ring.copy_cqe();
2485		if (cqe_send.err() == .INVAL) return error.SkipZigTest;
2486		try testing.expectEqual(linux.io_uring_cqe{
2487		.user_data = 0xeeeeeeee,
2488		.res = buffer_send.len,
2489		.flags = 0,
2490		}, cqe_send);
2491
2492		const cqe_recv = try ring.copy_cqe();
2493		if (cqe_recv.err() == .INVAL) return error.SkipZigTest;
2494		try testing.expectEqual(linux.io_uring_cqe{
2495		.user_data = 0xffffffff,
2496		.res = buffer_recv.len,
2497		// ignore IORING_CQE_F_SOCK_NONEMPTY since it is only set on some systems
2498		.flags = cqe_recv.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
2499		}, cqe_recv);
2500
2501		try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
2502		}
2503
2504		test "sendmsg/recvmsg" {
2505		if (builtin.os.tag != .linux) return error.SkipZigTest;
2506
2507		var ring = IO_Uring.init(2, 0) catch \|err\| switch (err) {
2508		error.SystemOutdated => return error.SkipZigTest,
2509		error.PermissionDenied => return error.SkipZigTest,
2510		else => return err,
2511		};
2512		defer ring.deinit();
2513
2514		var address_server = try net.Address.parseIp4("127.0.0.1", 0);
2515
2516		const server = try os.socket(address_server.any.family, os.SOCK.DGRAM, 0);
2517		defer os.close(server);
2518		try os.setsockopt(server, os.SOL.SOCKET, os.SO.REUSEPORT, &mem.toBytes(@as(c_int, 1)));
2519		try os.setsockopt(server, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
2520		try os.bind(server, &address_server.any, address_server.getOsSockLen());
2521
2522		// set address_server to the OS-chosen IP/port.
2523		var slen: os.socklen_t = address_server.getOsSockLen();
2524		try os.getsockname(server, &address_server.any, &slen);
2525
2526		const client = try os.socket(address_server.any.family, os.SOCK.DGRAM, 0);
2527		defer os.close(client);
2528
2529		const buffer_send = [_]u8{42} ** 128;
2530		const iovecs_send = [_]os.iovec_const{
2531		os.iovec_const{ .iov_base = &buffer_send, .iov_len = buffer_send.len },
2532		};
2533		const msg_send = os.msghdr_const{
2534		.name = &address_server.any,
2535		.namelen = address_server.getOsSockLen(),
2536		.iov = &iovecs_send,
2537		.iovlen = 1,
2538		.control = null,
2539		.controllen = 0,
2540		.flags = 0,
2541		};
2542		const sqe_sendmsg = try ring.sendmsg(0x11111111, client, &msg_send, 0);
2543		sqe_sendmsg.flags \|= linux.IOSQE_IO_LINK;
2544		try testing.expectEqual(linux.IORING_OP.SENDMSG, sqe_sendmsg.opcode);
2545		try testing.expectEqual(client, sqe_sendmsg.fd);
2546
2547		var buffer_recv = [_]u8{0} ** 128;
2548		var iovecs_recv = [_]os.iovec{
2549		os.iovec{ .iov_base = &buffer_recv, .iov_len = buffer_recv.len },
2550		};
2551		const addr = [_]u8{0} ** 4;
2552		var address_recv = net.Address.initIp4(addr, 0);
2553		var msg_recv: os.msghdr = os.msghdr{
2554		.name = &address_recv.any,
2555		.namelen = address_recv.getOsSockLen(),
2556		.iov = &iovecs_recv,
2557		.iovlen = 1,
2558		.control = null,
2559		.controllen = 0,
2560		.flags = 0,
2561		};
2562		const sqe_recvmsg = try ring.recvmsg(0x22222222, server, &msg_recv, 0);
2563		try testing.expectEqual(linux.IORING_OP.RECVMSG, sqe_recvmsg.opcode);
2564		try testing.expectEqual(server, sqe_recvmsg.fd);
2565
2566		try testing.expectEqual(@as(u32, 2), ring.sq_ready());
2567		try testing.expectEqual(@as(u32, 2), try ring.submit_and_wait(2));
2568		try testing.expectEqual(@as(u32, 0), ring.sq_ready());
2569		try testing.expectEqual(@as(u32, 2), ring.cq_ready());
2570
2571		const cqe_sendmsg = try ring.copy_cqe();
2572		if (cqe_sendmsg.res == -@as(i32, @intFromEnum(linux.E.INVAL))) return error.SkipZigTest;
2573		try testing.expectEqual(linux.io_uring_cqe{
2574		.user_data = 0x11111111,
2575		.res = buffer_send.len,
2576		.flags = 0,
2577		}, cqe_sendmsg);
2578
2579		const cqe_recvmsg = try ring.copy_cqe();
2580		if (cqe_recvmsg.res == -@as(i32, @intFromEnum(linux.E.INVAL))) return error.SkipZigTest;
2581		try testing.expectEqual(linux.io_uring_cqe{
2582		.user_data = 0x22222222,
2583		.res = buffer_recv.len,
2584		// ignore IORING_CQE_F_SOCK_NONEMPTY since it is set non-deterministically
2585		.flags = cqe_recvmsg.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
2586		}, cqe_recvmsg);
2587
2588		try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
2589		}
2590
2591		test "timeout (after a relative time)" {
2592		if (builtin.os.tag != .linux) return error.SkipZigTest;
2593
2594		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
2595		error.SystemOutdated => return error.SkipZigTest,
2596		error.PermissionDenied => return error.SkipZigTest,
2597		else => return err,
2598		};
2599		defer ring.deinit();
2600
2601		const ms = 10;
2602		const margin = 5;
2603		const ts = os.linux.kernel_timespec{ .tv_sec = 0, .tv_nsec = ms * 1000000 };
2604
2605		const started = std.time.milliTimestamp();
2606		const sqe = try ring.timeout(0x55555555, &ts, 0, 0);
2607		try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe.opcode);
2608		try testing.expectEqual(@as(u32, 1), try ring.submit());
2609		const cqe = try ring.copy_cqe();
2610		const stopped = std.time.milliTimestamp();
2611
2612		try testing.expectEqual(linux.io_uring_cqe{
2613		.user_data = 0x55555555,
2614		.res = -@as(i32, @intFromEnum(linux.E.TIME)),
2615		.flags = 0,
2616		}, cqe);
2617
2618		// Tests should not depend on timings: skip test if outside margin.
2619		if (!std.math.approxEqAbs(f64, ms, @as(f64, @floatFromInt(stopped - started)), margin)) return error.SkipZigTest;
2620		}
2621
2622		test "timeout (after a number of completions)" {
2623		if (builtin.os.tag != .linux) return error.SkipZigTest;
2624
2625		var ring = IO_Uring.init(2, 0) catch \|err\| switch (err) {
2626		error.SystemOutdated => return error.SkipZigTest,
2627		error.PermissionDenied => return error.SkipZigTest,
2628		else => return err,
2629		};
2630		defer ring.deinit();
2631
2632		const ts = os.linux.kernel_timespec{ .tv_sec = 3, .tv_nsec = 0 };
2633		const count_completions: u64 = 1;
2634		const sqe_timeout = try ring.timeout(0x66666666, &ts, count_completions, 0);
2635		try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe_timeout.opcode);
2636		try testing.expectEqual(count_completions, sqe_timeout.off);
2637		_ = try ring.nop(0x77777777);
2638		try testing.expectEqual(@as(u32, 2), try ring.submit());
2639
2640		const cqe_nop = try ring.copy_cqe();
2641		try testing.expectEqual(linux.io_uring_cqe{
2642		.user_data = 0x77777777,
2643		.res = 0,
2644		.flags = 0,
2645		}, cqe_nop);
2646
2647		const cqe_timeout = try ring.copy_cqe();
2648		try testing.expectEqual(linux.io_uring_cqe{
2649		.user_data = 0x66666666,
2650		.res = 0,
2651		.flags = 0,
2652		}, cqe_timeout);
2653		}
2654
2655		test "timeout_remove" {
2656		if (builtin.os.tag != .linux) return error.SkipZigTest;
2657
2658		var ring = IO_Uring.init(2, 0) catch \|err\| switch (err) {
2659		error.SystemOutdated => return error.SkipZigTest,
2660		error.PermissionDenied => return error.SkipZigTest,
2661		else => return err,
2662		};
2663		defer ring.deinit();
2664
2665		const ts = os.linux.kernel_timespec{ .tv_sec = 3, .tv_nsec = 0 };
2666		const sqe_timeout = try ring.timeout(0x88888888, &ts, 0, 0);
2667		try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe_timeout.opcode);
2668		try testing.expectEqual(@as(u64, 0x88888888), sqe_timeout.user_data);
2669
2670		const sqe_timeout_remove = try ring.timeout_remove(0x99999999, 0x88888888, 0);
2671		try testing.expectEqual(linux.IORING_OP.TIMEOUT_REMOVE, sqe_timeout_remove.opcode);
2672		try testing.expectEqual(@as(u64, 0x88888888), sqe_timeout_remove.addr);
2673		try testing.expectEqual(@as(u64, 0x99999999), sqe_timeout_remove.user_data);
2674
2675		try testing.expectEqual(@as(u32, 2), try ring.submit());
2676
2677		// The order in which the CQE arrive is not clearly documented and it changed with kernel 5.18:
2678		// * kernel 5.10 gives user data 0x88888888 first, 0x99999999 second
2679		// * kernel 5.18 gives user data 0x99999999 first, 0x88888888 second
2680
2681		var cqes: [2]os.linux.io_uring_cqe = undefined;
2682		cqes[0] = try ring.copy_cqe();
2683		cqes[1] = try ring.copy_cqe();
2684
2685		for (cqes) \|cqe\| {
2686		// IORING_OP_TIMEOUT_REMOVE is not supported by this kernel version:
2687		// Timeout remove operations set the fd to -1, which results in EBADF before EINVAL.
2688		// We use IORING_FEAT_RW_CUR_POS as a safety check here to make sure we are at least pre-5.6.
2689		// We don't want to skip this test for newer kernels.
2690		if (cqe.user_data == 0x99999999 and
2691		cqe.err() == .BADF and
2692		(ring.features & linux.IORING_FEAT_RW_CUR_POS) == 0)
2693		{
2694		return error.SkipZigTest;
2695		}
2696
2697		try testing.expect(cqe.user_data == 0x88888888 or cqe.user_data == 0x99999999);
2698
2699		if (cqe.user_data == 0x88888888) {
2700		try testing.expectEqual(linux.io_uring_cqe{
2701		.user_data = 0x88888888,
2702		.res = -@as(i32, @intFromEnum(linux.E.CANCELED)),
2703		.flags = 0,
2704		}, cqe);
2705		} else if (cqe.user_data == 0x99999999) {
2706		try testing.expectEqual(linux.io_uring_cqe{
2707		.user_data = 0x99999999,
2708		.res = 0,
2709		.flags = 0,
2710		}, cqe);
2711		}
2712		}
2713		}
2714
2715		test "accept/connect/recv/link_timeout" {
2716		if (builtin.os.tag != .linux) return error.SkipZigTest;
2717
2718		var ring = IO_Uring.init(16, 0) catch \|err\| switch (err) {
2719		error.SystemOutdated => return error.SkipZigTest,
2720		error.PermissionDenied => return error.SkipZigTest,
2721		else => return err,
2722		};
2723		defer ring.deinit();
2724
2725		const socket_test_harness = try createSocketTestHarness(&ring);
2726		defer socket_test_harness.close();
2727
2728		var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
2729
2730		const sqe_recv = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
2731		sqe_recv.flags \|= linux.IOSQE_IO_LINK;
2732
2733		const ts = os.linux.kernel_timespec{ .tv_sec = 0, .tv_nsec = 1000000 };
2734		_ = try ring.link_timeout(0x22222222, &ts, 0);
2735
2736		const nr_wait = try ring.submit();
2737		try testing.expectEqual(@as(u32, 2), nr_wait);
2738
2739		var i: usize = 0;
2740		while (i < nr_wait) : (i += 1) {
2741		const cqe = try ring.copy_cqe();
2742		switch (cqe.user_data) {
2743		0xffffffff => {
2744		if (cqe.res != -@as(i32, @intFromEnum(linux.E.INTR)) and
2745		cqe.res != -@as(i32, @intFromEnum(linux.E.CANCELED)))
2746		{
2747		std.debug.print("Req 0x{x} got {d}\n", .{ cqe.user_data, cqe.res });
2748		try testing.expect(false);
2749		}
2750		},
2751		0x22222222 => {
2752		if (cqe.res != -@as(i32, @intFromEnum(linux.E.ALREADY)) and
2753		cqe.res != -@as(i32, @intFromEnum(linux.E.TIME)))
2754		{
2755		std.debug.print("Req 0x{x} got {d}\n", .{ cqe.user_data, cqe.res });
2756		try testing.expect(false);
2757		}
2758		},
2759		else => @panic("should not happen"),
2760		}
2761		}
2762		}
2763
2764		test "fallocate" {
2765		if (builtin.os.tag != .linux) return error.SkipZigTest;
2766
2767		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
2768		error.SystemOutdated => return error.SkipZigTest,
2769		error.PermissionDenied => return error.SkipZigTest,
2770		else => return err,
2771		};
2772		defer ring.deinit();
2773
2774		var tmp = std.testing.tmpDir(.{});
2775		defer tmp.cleanup();
2776
2777		const path = "test_io_uring_fallocate";
2778		const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
2779		defer file.close();
2780
2781		try testing.expectEqual(@as(u64, 0), (try file.stat()).size);
2782
2783		const len: u64 = 65536;
2784		const sqe = try ring.fallocate(0xaaaaaaaa, file.handle, 0, 0, len);
2785		try testing.expectEqual(linux.IORING_OP.FALLOCATE, sqe.opcode);
2786		try testing.expectEqual(file.handle, sqe.fd);
2787		try testing.expectEqual(@as(u32, 1), try ring.submit());
2788
2789		const cqe = try ring.copy_cqe();
2790		switch (cqe.err()) {
2791		.SUCCESS => {},
2792		// This kernel's io_uring does not yet implement fallocate():
2793		.INVAL => return error.SkipZigTest,
2794		// This kernel does not implement fallocate():
2795		.NOSYS => return error.SkipZigTest,
2796		// The filesystem containing the file referred to by fd does not support this operation;
2797		// or the mode is not supported by the filesystem containing the file referred to by fd:
2798		.OPNOTSUPP => return error.SkipZigTest,
2799		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
2800		}
2801		try testing.expectEqual(linux.io_uring_cqe{
2802		.user_data = 0xaaaaaaaa,
2803		.res = 0,
2804		.flags = 0,
2805		}, cqe);
2806
2807		try testing.expectEqual(len, (try file.stat()).size);
2808		}
2809
2810		test "statx" {
2811		if (builtin.os.tag != .linux) return error.SkipZigTest;
2812
2813		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
2814		error.SystemOutdated => return error.SkipZigTest,
2815		error.PermissionDenied => return error.SkipZigTest,
2816		else => return err,
2817		};
2818		defer ring.deinit();
2819
2820		var tmp = std.testing.tmpDir(.{});
2821		defer tmp.cleanup();
2822		const path = "test_io_uring_statx";
2823		const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
2824		defer file.close();
2825
2826		try testing.expectEqual(@as(u64, 0), (try file.stat()).size);
2827
2828		try file.writeAll("foobar");
2829
2830		var buf: linux.Statx = undefined;
2831		const sqe = try ring.statx(
2832		0xaaaaaaaa,
2833		tmp.dir.fd,
2834		path,
2835		0,
2836		linux.STATX_SIZE,
2837		&buf,
2838		);
2839		try testing.expectEqual(linux.IORING_OP.STATX, sqe.opcode);
2840		try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
2841		try testing.expectEqual(@as(u32, 1), try ring.submit());
2842
2843		const cqe = try ring.copy_cqe();
2844		switch (cqe.err()) {
2845		.SUCCESS => {},
2846		// This kernel's io_uring does not yet implement statx():
2847		.INVAL => return error.SkipZigTest,
2848		// This kernel does not implement statx():
2849		.NOSYS => return error.SkipZigTest,
2850		// The filesystem containing the file referred to by fd does not support this operation;
2851		// or the mode is not supported by the filesystem containing the file referred to by fd:
2852		.OPNOTSUPP => return error.SkipZigTest,
2853		// not supported on older kernels (5.4)
2854		.BADF => return error.SkipZigTest,
2855		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
2856		}
2857		try testing.expectEqual(linux.io_uring_cqe{
2858		.user_data = 0xaaaaaaaa,
2859		.res = 0,
2860		.flags = 0,
2861		}, cqe);
2862
2863		try testing.expect(buf.mask & os.linux.STATX_SIZE == os.linux.STATX_SIZE);
2864		try testing.expectEqual(@as(u64, 6), buf.size);
2865		}
2866
2867		test "accept/connect/recv/cancel" {
2868		if (builtin.os.tag != .linux) return error.SkipZigTest;
2869
2870		var ring = IO_Uring.init(16, 0) catch \|err\| switch (err) {
2871		error.SystemOutdated => return error.SkipZigTest,
2872		error.PermissionDenied => return error.SkipZigTest,
2873		else => return err,
2874		};
2875		defer ring.deinit();
2876
2877		const socket_test_harness = try createSocketTestHarness(&ring);
2878		defer socket_test_harness.close();
2879
2880		var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
2881
2882		_ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
2883		try testing.expectEqual(@as(u32, 1), try ring.submit());
2884
2885		const sqe_cancel = try ring.cancel(0x99999999, 0xffffffff, 0);
2886		try testing.expectEqual(linux.IORING_OP.ASYNC_CANCEL, sqe_cancel.opcode);
2887		try testing.expectEqual(@as(u64, 0xffffffff), sqe_cancel.addr);
2888		try testing.expectEqual(@as(u64, 0x99999999), sqe_cancel.user_data);
2889		try testing.expectEqual(@as(u32, 1), try ring.submit());
2890
2891		var cqe_recv = try ring.copy_cqe();
2892		if (cqe_recv.err() == .INVAL) return error.SkipZigTest;
2893		var cqe_cancel = try ring.copy_cqe();
2894		if (cqe_cancel.err() == .INVAL) return error.SkipZigTest;
2895
2896		// The recv/cancel CQEs may arrive in any order, the recv CQE will sometimes come first:
2897		if (cqe_recv.user_data == 0x99999999 and cqe_cancel.user_data == 0xffffffff) {
2898		const a = cqe_recv;
2899		const b = cqe_cancel;
2900		cqe_recv = b;
2901		cqe_cancel = a;
2902		}
2903
2904		try testing.expectEqual(linux.io_uring_cqe{
2905		.user_data = 0xffffffff,
2906		.res = -@as(i32, @intFromEnum(linux.E.CANCELED)),
2907		.flags = 0,
2908		}, cqe_recv);
2909
2910		try testing.expectEqual(linux.io_uring_cqe{
2911		.user_data = 0x99999999,
2912		.res = 0,
2913		.flags = 0,
2914		}, cqe_cancel);
2915		}
2916
2917		test "register_files_update" {
2918		if (builtin.os.tag != .linux) return error.SkipZigTest;
2919
2920		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
2921		error.SystemOutdated => return error.SkipZigTest,
2922		error.PermissionDenied => return error.SkipZigTest,
2923		else => return err,
2924		};
2925		defer ring.deinit();
2926
2927		const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
2928		defer os.close(fd);
2929
2930		var registered_fds = [_]os.fd_t{0} ** 2;
2931		const fd_index = 0;
2932		const fd_index2 = 1;
2933		registered_fds[fd_index] = fd;
2934		registered_fds[fd_index2] = -1;
2935
2936		ring.register_files(registered_fds[0..]) catch \|err\| switch (err) {
2937		// Happens when the kernel doesn't support sparse entry (-1) in the file descriptors array.
2938		error.FileDescriptorInvalid => return error.SkipZigTest,
2939		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
2940		};
2941
2942		// Test IORING_REGISTER_FILES_UPDATE
2943		// Only available since Linux 5.5
2944
2945		const fd2 = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
2946		defer os.close(fd2);
2947
2948		registered_fds[fd_index] = fd2;
2949		registered_fds[fd_index2] = -1;
2950		try ring.register_files_update(0, registered_fds[0..]);
2951
2952		var buffer = [_]u8{42} ** 128;
2953		{
2954		const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
2955		try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2956		sqe.flags \|= linux.IOSQE_FIXED_FILE;
2957
2958		try testing.expectEqual(@as(u32, 1), try ring.submit());
2959		try testing.expectEqual(linux.io_uring_cqe{
2960		.user_data = 0xcccccccc,
2961		.res = buffer.len,
2962		.flags = 0,
2963		}, try ring.copy_cqe());
2964		try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
2965		}
2966
2967		// Test with a non-zero offset
2968
2969		registered_fds[fd_index] = -1;
2970		registered_fds[fd_index2] = -1;
2971		try ring.register_files_update(1, registered_fds[1..]);
2972
2973		{
2974		// Next read should still work since fd_index in the registered file descriptors hasn't been updated yet.
2975		const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
2976		try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2977		sqe.flags \|= linux.IOSQE_FIXED_FILE;
2978
2979		try testing.expectEqual(@as(u32, 1), try ring.submit());
2980		try testing.expectEqual(linux.io_uring_cqe{
2981		.user_data = 0xcccccccc,
2982		.res = buffer.len,
2983		.flags = 0,
2984		}, try ring.copy_cqe());
2985		try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
2986		}
2987
2988		try ring.register_files_update(0, registered_fds[0..]);
2989
2990		{
2991		// Now this should fail since both fds are sparse (-1)
2992		const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
2993		try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2994		sqe.flags \|= linux.IOSQE_FIXED_FILE;
2995
2996		try testing.expectEqual(@as(u32, 1), try ring.submit());
2997		const cqe = try ring.copy_cqe();
2998		try testing.expectEqual(os.linux.E.BADF, cqe.err());
2999		}
3000
3001		try ring.unregister_files();
3002		}
3003
3004		test "shutdown" {
3005		if (builtin.os.tag != .linux) return error.SkipZigTest;
3006
3007		var ring = IO_Uring.init(16, 0) catch \|err\| switch (err) {
3008		error.SystemOutdated => return error.SkipZigTest,
3009		error.PermissionDenied => return error.SkipZigTest,
3010		else => return err,
3011		};
3012		defer ring.deinit();
3013
3014		var address = try net.Address.parseIp4("127.0.0.1", 0);
3015
3016		// Socket bound, expect shutdown to work
3017		{
3018		const server = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
3019		defer os.close(server);
3020		try os.setsockopt(server, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
3021		try os.bind(server, &address.any, address.getOsSockLen());
3022		try os.listen(server, 1);
3023
3024		// set address to the OS-chosen IP/port.
3025		var slen: os.socklen_t = address.getOsSockLen();
3026		try os.getsockname(server, &address.any, &slen);
3027
3028		const shutdown_sqe = try ring.shutdown(0x445445445, server, os.linux.SHUT.RD);
3029		try testing.expectEqual(linux.IORING_OP.SHUTDOWN, shutdown_sqe.opcode);
3030		try testing.expectEqual(@as(i32, server), shutdown_sqe.fd);
3031
3032		try testing.expectEqual(@as(u32, 1), try ring.submit());
3033
3034		const cqe = try ring.copy_cqe();
3035		switch (cqe.err()) {
3036		.SUCCESS => {},
3037		// This kernel's io_uring does not yet implement shutdown (kernel version < 5.11)
3038		.INVAL => return error.SkipZigTest,
3039		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3040		}
3041
3042		try testing.expectEqual(linux.io_uring_cqe{
3043		.user_data = 0x445445445,
3044		.res = 0,
3045		.flags = 0,
3046		}, cqe);
3047		}
3048
3049		// Socket not bound, expect to fail with ENOTCONN
3050		{
3051		const server = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
3052		defer os.close(server);
3053
3054		const shutdown_sqe = ring.shutdown(0x445445445, server, os.linux.SHUT.RD) catch \|err\| switch (err) {
3055		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3056		};
3057		try testing.expectEqual(linux.IORING_OP.SHUTDOWN, shutdown_sqe.opcode);
3058		try testing.expectEqual(@as(i32, server), shutdown_sqe.fd);
3059
3060		try testing.expectEqual(@as(u32, 1), try ring.submit());
3061
3062		const cqe = try ring.copy_cqe();
3063		try testing.expectEqual(@as(u64, 0x445445445), cqe.user_data);
3064		try testing.expectEqual(os.linux.E.NOTCONN, cqe.err());
3065		}
3066		}
3067
3068		test "renameat" {
3069		if (builtin.os.tag != .linux) return error.SkipZigTest;
3070
3071		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
3072		error.SystemOutdated => return error.SkipZigTest,
3073		error.PermissionDenied => return error.SkipZigTest,
3074		else => return err,
3075		};
3076		defer ring.deinit();
3077
3078		const old_path = "test_io_uring_renameat_old";
3079		const new_path = "test_io_uring_renameat_new";
3080
3081		var tmp = std.testing.tmpDir(.{});
3082		defer tmp.cleanup();
3083
3084		// Write old file with data
3085
3086		const old_file = try tmp.dir.createFile(old_path, .{ .truncate = true, .mode = 0o666 });
3087		defer old_file.close();
3088		try old_file.writeAll("hello");
3089
3090		// Submit renameat
3091
3092		const sqe = try ring.renameat(
3093		0x12121212,
3094		tmp.dir.fd,
3095		old_path,
3096		tmp.dir.fd,
3097		new_path,
3098		0,
3099		);
3100		try testing.expectEqual(linux.IORING_OP.RENAMEAT, sqe.opcode);
3101		try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
3102		try testing.expectEqual(@as(i32, tmp.dir.fd), @as(i32, @bitCast(sqe.len)));
3103		try testing.expectEqual(@as(u32, 1), try ring.submit());
3104
3105		const cqe = try ring.copy_cqe();
3106		switch (cqe.err()) {
3107		.SUCCESS => {},
3108		// This kernel's io_uring does not yet implement renameat (kernel version < 5.11)
3109		.BADF, .INVAL => return error.SkipZigTest,
3110		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3111		}
3112		try testing.expectEqual(linux.io_uring_cqe{
3113		.user_data = 0x12121212,
3114		.res = 0,
3115		.flags = 0,
3116		}, cqe);
3117
3118		// Validate that the old file doesn't exist anymore
3119		{
3120		_ = tmp.dir.openFile(old_path, .{}) catch \|err\| switch (err) {
3121		error.FileNotFound => {},
3122		else => std.debug.panic("unexpected error: {}", .{err}),
3123		};
3124		}
3125
3126		// Validate that the new file exists with the proper content
3127		{
3128		const new_file = try tmp.dir.openFile(new_path, .{});
3129		defer new_file.close();
3130
3131		var new_file_data: [16]u8 = undefined;
3132		const read = try new_file.readAll(&new_file_data);
3133		try testing.expectEqualStrings("hello", new_file_data[0..read]);
3134		}
3135		}
3136
3137		test "unlinkat" {
3138		if (builtin.os.tag != .linux) return error.SkipZigTest;
3139
3140		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
3141		error.SystemOutdated => return error.SkipZigTest,
3142		error.PermissionDenied => return error.SkipZigTest,
3143		else => return err,
3144		};
3145		defer ring.deinit();
3146
3147		const path = "test_io_uring_unlinkat";
3148
3149		var tmp = std.testing.tmpDir(.{});
3150		defer tmp.cleanup();
3151
3152		// Write old file with data
3153
3154		const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
3155		defer file.close();
3156
3157		// Submit unlinkat
3158
3159		const sqe = try ring.unlinkat(
3160		0x12121212,
3161		tmp.dir.fd,
3162		path,
3163		0,
3164		);
3165		try testing.expectEqual(linux.IORING_OP.UNLINKAT, sqe.opcode);
3166		try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
3167		try testing.expectEqual(@as(u32, 1), try ring.submit());
3168
3169		const cqe = try ring.copy_cqe();
3170		switch (cqe.err()) {
3171		.SUCCESS => {},
3172		// This kernel's io_uring does not yet implement unlinkat (kernel version < 5.11)
3173		.BADF, .INVAL => return error.SkipZigTest,
3174		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3175		}
3176		try testing.expectEqual(linux.io_uring_cqe{
3177		.user_data = 0x12121212,
3178		.res = 0,
3179		.flags = 0,
3180		}, cqe);
3181
3182		// Validate that the file doesn't exist anymore
3183		_ = tmp.dir.openFile(path, .{}) catch \|err\| switch (err) {
3184		error.FileNotFound => {},
3185		else => std.debug.panic("unexpected error: {}", .{err}),
3186		};
3187		}
3188
3189		test "mkdirat" {
3190		if (builtin.os.tag != .linux) return error.SkipZigTest;
3191
3192		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
3193		error.SystemOutdated => return error.SkipZigTest,
3194		error.PermissionDenied => return error.SkipZigTest,
3195		else => return err,
3196		};
3197		defer ring.deinit();
3198
3199		var tmp = std.testing.tmpDir(.{});
3200		defer tmp.cleanup();
3201
3202		const path = "test_io_uring_mkdirat";
3203
3204		// Submit mkdirat
3205
3206		const sqe = try ring.mkdirat(
3207		0x12121212,
3208		tmp.dir.fd,
3209		path,
3210		0o0755,
3211		);
3212		try testing.expectEqual(linux.IORING_OP.MKDIRAT, sqe.opcode);
3213		try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
3214		try testing.expectEqual(@as(u32, 1), try ring.submit());
3215
3216		const cqe = try ring.copy_cqe();
3217		switch (cqe.err()) {
3218		.SUCCESS => {},
3219		// This kernel's io_uring does not yet implement mkdirat (kernel version < 5.15)
3220		.BADF, .INVAL => return error.SkipZigTest,
3221		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3222		}
3223		try testing.expectEqual(linux.io_uring_cqe{
3224		.user_data = 0x12121212,
3225		.res = 0,
3226		.flags = 0,
3227		}, cqe);
3228
3229		// Validate that the directory exist
3230		_ = try tmp.dir.openDir(path, .{});
3231		}
3232
3233		test "symlinkat" {
3234		if (builtin.os.tag != .linux) return error.SkipZigTest;
3235
3236		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
3237		error.SystemOutdated => return error.SkipZigTest,
3238		error.PermissionDenied => return error.SkipZigTest,
3239		else => return err,
3240		};
3241		defer ring.deinit();
3242
3243		var tmp = std.testing.tmpDir(.{});
3244		defer tmp.cleanup();
3245
3246		const path = "test_io_uring_symlinkat";
3247		const link_path = "test_io_uring_symlinkat_link";
3248
3249		const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
3250		defer file.close();
3251
3252		// Submit symlinkat
3253
3254		const sqe = try ring.symlinkat(
3255		0x12121212,
3256		path,
3257		tmp.dir.fd,
3258		link_path,
3259		);
3260		try testing.expectEqual(linux.IORING_OP.SYMLINKAT, sqe.opcode);
3261		try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
3262		try testing.expectEqual(@as(u32, 1), try ring.submit());
3263
3264		const cqe = try ring.copy_cqe();
3265		switch (cqe.err()) {
3266		.SUCCESS => {},
3267		// This kernel's io_uring does not yet implement symlinkat (kernel version < 5.15)
3268		.BADF, .INVAL => return error.SkipZigTest,
3269		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3270		}
3271		try testing.expectEqual(linux.io_uring_cqe{
3272		.user_data = 0x12121212,
3273		.res = 0,
3274		.flags = 0,
3275		}, cqe);
3276
3277		// Validate that the symlink exist
3278		_ = try tmp.dir.openFile(link_path, .{});
3279		}
3280
3281		test "linkat" {
3282		if (builtin.os.tag != .linux) return error.SkipZigTest;
3283
3284		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
3285		error.SystemOutdated => return error.SkipZigTest,
3286		error.PermissionDenied => return error.SkipZigTest,
3287		else => return err,
3288		};
3289		defer ring.deinit();
3290
3291		var tmp = std.testing.tmpDir(.{});
3292		defer tmp.cleanup();
3293
3294		const first_path = "test_io_uring_linkat_first";
3295		const second_path = "test_io_uring_linkat_second";
3296
3297		// Write file with data
3298
3299		const first_file = try tmp.dir.createFile(first_path, .{ .truncate = true, .mode = 0o666 });
3300		defer first_file.close();
3301		try first_file.writeAll("hello");
3302
3303		// Submit linkat
3304
3305		const sqe = try ring.linkat(
3306		0x12121212,
3307		tmp.dir.fd,
3308		first_path,
3309		tmp.dir.fd,
3310		second_path,
3311		0,
3312		);
3313		try testing.expectEqual(linux.IORING_OP.LINKAT, sqe.opcode);
3314		try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
3315		try testing.expectEqual(@as(i32, tmp.dir.fd), @as(i32, @bitCast(sqe.len)));
3316		try testing.expectEqual(@as(u32, 1), try ring.submit());
3317
3318		const cqe = try ring.copy_cqe();
3319		switch (cqe.err()) {
3320		.SUCCESS => {},
3321		// This kernel's io_uring does not yet implement linkat (kernel version < 5.15)
3322		.BADF, .INVAL => return error.SkipZigTest,
3323		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3324		}
3325		try testing.expectEqual(linux.io_uring_cqe{
3326		.user_data = 0x12121212,
3327		.res = 0,
3328		.flags = 0,
3329		}, cqe);
3330
3331		// Validate the second file
3332		const second_file = try tmp.dir.openFile(second_path, .{});
3333		defer second_file.close();
3334
3335		var second_file_data: [16]u8 = undefined;
3336		const read = try second_file.readAll(&second_file_data);
3337		try testing.expectEqualStrings("hello", second_file_data[0..read]);
3338		}
3339
3340		test "provide_buffers: read" {
3341		if (builtin.os.tag != .linux) return error.SkipZigTest;
3342
3343		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
3344		error.SystemOutdated => return error.SkipZigTest,
3345		error.PermissionDenied => return error.SkipZigTest,
3346		else => return err,
3347		};
3348		defer ring.deinit();
3349
3350		const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
3351		defer os.close(fd);
3352
3353		const group_id = 1337;
3354		const buffer_id = 0;
3355
3356		const buffer_len = 128;
3357
3358		var buffers: [4][buffer_len]u8 = undefined;
3359
3360		// Provide 4 buffers
3361
3362		{
3363		const sqe = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
3364		try testing.expectEqual(linux.IORING_OP.PROVIDE_BUFFERS, sqe.opcode);
3365		try testing.expectEqual(@as(i32, buffers.len), sqe.fd);
3366		try testing.expectEqual(@as(u32, buffers[0].len), sqe.len);
3367		try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3368		try testing.expectEqual(@as(u32, 1), try ring.submit());
3369
3370		const cqe = try ring.copy_cqe();
3371		switch (cqe.err()) {
3372		// Happens when the kernel is < 5.7
3373		.INVAL => return error.SkipZigTest,
3374		.SUCCESS => {},
3375		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3376		}
3377		try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
3378		}
3379
3380		// Do 4 reads which should consume all buffers
3381
3382		var i: usize = 0;
3383		while (i < buffers.len) : (i += 1) {
3384		const sqe = try ring.read(0xdededede, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3385		try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
3386		try testing.expectEqual(@as(i32, fd), sqe.fd);
3387		try testing.expectEqual(@as(u64, 0), sqe.addr);
3388		try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3389		try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3390		try testing.expectEqual(@as(u32, 1), try ring.submit());
3391
3392		const cqe = try ring.copy_cqe();
3393		switch (cqe.err()) {
3394		.SUCCESS => {},
3395		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3396		}
3397
3398		try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3399		const used_buffer_id = cqe.flags >> 16;
3400		try testing.expect(used_buffer_id >= 0 and used_buffer_id <= 3);
3401		try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3402
3403		try testing.expectEqual(@as(u64, 0xdededede), cqe.user_data);
3404		try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
3405		}
3406
3407		// This read should fail
3408
3409		{
3410		const sqe = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3411		try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
3412		try testing.expectEqual(@as(i32, fd), sqe.fd);
3413		try testing.expectEqual(@as(u64, 0), sqe.addr);
3414		try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3415		try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3416		try testing.expectEqual(@as(u32, 1), try ring.submit());
3417
3418		const cqe = try ring.copy_cqe();
3419		switch (cqe.err()) {
3420		// Expected
3421		.NOBUFS => {},
3422		.SUCCESS => std.debug.panic("unexpected success", .{}),
3423		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3424		}
3425		try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3426		}
3427
3428		// Provide 1 buffer again
3429
3430		// Deliberately put something we don't expect in the buffers
3431		@memset(mem.sliceAsBytes(&buffers), 42);
3432
3433		const reprovided_buffer_id = 2;
3434
3435		{
3436		_ = try ring.provide_buffers(0xabababab, @as([*]u8, @ptrCast(&buffers[reprovided_buffer_id])), buffer_len, 1, group_id, reprovided_buffer_id);
3437		try testing.expectEqual(@as(u32, 1), try ring.submit());
3438
3439		const cqe = try ring.copy_cqe();
3440		switch (cqe.err()) {
3441		.SUCCESS => {},
3442		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3443		}
3444		}
3445
3446		// Final read which should work
3447
3448		{
3449		const sqe = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3450		try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
3451		try testing.expectEqual(@as(i32, fd), sqe.fd);
3452		try testing.expectEqual(@as(u64, 0), sqe.addr);
3453		try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3454		try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3455		try testing.expectEqual(@as(u32, 1), try ring.submit());
3456
3457		const cqe = try ring.copy_cqe();
3458		switch (cqe.err()) {
3459		.SUCCESS => {},
3460		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3461		}
3462
3463		try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3464		const used_buffer_id = cqe.flags >> 16;
3465		try testing.expectEqual(used_buffer_id, reprovided_buffer_id);
3466		try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3467		try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3468		try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
3469		}
3470		}
3471
3472		test "remove_buffers" {
3473		if (builtin.os.tag != .linux) return error.SkipZigTest;
3474
3475		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
3476		error.SystemOutdated => return error.SkipZigTest,
3477		error.PermissionDenied => return error.SkipZigTest,
3478		else => return err,
3479		};
3480		defer ring.deinit();
3481
3482		const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
3483		defer os.close(fd);
3484
3485		const group_id = 1337;
3486		const buffer_id = 0;
3487
3488		const buffer_len = 128;
3489
3490		var buffers: [4][buffer_len]u8 = undefined;
3491
3492		// Provide 4 buffers
3493
3494		{
3495		_ = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
3496		try testing.expectEqual(@as(u32, 1), try ring.submit());
3497
3498		const cqe = try ring.copy_cqe();
3499		switch (cqe.err()) {
3500		.INVAL => return error.SkipZigTest,
3501		.SUCCESS => {},
3502		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3503		}
3504		try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
3505		}
3506
3507		// Remove 3 buffers
3508
3509		{
3510		const sqe = try ring.remove_buffers(0xbababababa, 3, group_id);
3511		try testing.expectEqual(linux.IORING_OP.REMOVE_BUFFERS, sqe.opcode);
3512		try testing.expectEqual(@as(i32, 3), sqe.fd);
3513		try testing.expectEqual(@as(u64, 0), sqe.addr);
3514		try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3515		try testing.expectEqual(@as(u32, 1), try ring.submit());
3516
3517		const cqe = try ring.copy_cqe();
3518		switch (cqe.err()) {
3519		.SUCCESS => {},
3520		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3521		}
3522		try testing.expectEqual(@as(u64, 0xbababababa), cqe.user_data);
3523		}
3524
3525		// This read should work
3526
3527		{
3528		_ = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3529		try testing.expectEqual(@as(u32, 1), try ring.submit());
3530
3531		const cqe = try ring.copy_cqe();
3532		switch (cqe.err()) {
3533		.SUCCESS => {},
3534		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3535		}
3536
3537		try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3538		const used_buffer_id = cqe.flags >> 16;
3539		try testing.expect(used_buffer_id >= 0 and used_buffer_id < 4);
3540		try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3541		try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3542		try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
3543		}
3544
3545		// Final read should _not_ work
3546
3547		{
3548		_ = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3549		try testing.expectEqual(@as(u32, 1), try ring.submit());
3550
3551		const cqe = try ring.copy_cqe();
3552		switch (cqe.err()) {
3553		// Expected
3554		.NOBUFS => {},
3555		.SUCCESS => std.debug.panic("unexpected success", .{}),
3556		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3557		}
3558		}
3559		}
3560
3561		test "provide_buffers: accept/connect/send/recv" {
3562		if (builtin.os.tag != .linux) return error.SkipZigTest;
3563
3564		var ring = IO_Uring.init(16, 0) catch \|err\| switch (err) {
3565		error.SystemOutdated => return error.SkipZigTest,
3566		error.PermissionDenied => return error.SkipZigTest,
3567		else => return err,
3568		};
3569		defer ring.deinit();
3570
3571		const group_id = 1337;
3572		const buffer_id = 0;
3573
3574		const buffer_len = 128;
3575		var buffers: [4][buffer_len]u8 = undefined;
3576
3577		// Provide 4 buffers
3578
3579		{
3580		const sqe = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
3581		try testing.expectEqual(linux.IORING_OP.PROVIDE_BUFFERS, sqe.opcode);
3582		try testing.expectEqual(@as(i32, buffers.len), sqe.fd);
3583		try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3584		try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3585		try testing.expectEqual(@as(u32, 1), try ring.submit());
3586
3587		const cqe = try ring.copy_cqe();
3588		switch (cqe.err()) {
3589		// Happens when the kernel is < 5.7
3590		.INVAL => return error.SkipZigTest,
3591		// Happens on the kernel 5.4
3592		.BADF => return error.SkipZigTest,
3593		.SUCCESS => {},
3594		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3595		}
3596		try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
3597		}
3598
3599		const socket_test_harness = try createSocketTestHarness(&ring);
3600		defer socket_test_harness.close();
3601
3602		// Do 4 send on the socket
3603
3604		{
3605		var i: usize = 0;
3606		while (i < buffers.len) : (i += 1) {
3607		_ = try ring.send(0xdeaddead, socket_test_harness.server, &([_]u8{'z'} ** buffer_len), 0);
3608		try testing.expectEqual(@as(u32, 1), try ring.submit());
3609		}
3610
3611		var cqes: [4]linux.io_uring_cqe = undefined;
3612		try testing.expectEqual(@as(u32, 4), try ring.copy_cqes(&cqes, 4));
3613		}
3614
3615		// Do 4 recv which should consume all buffers
3616
3617		// Deliberately put something we don't expect in the buffers
3618		@memset(mem.sliceAsBytes(&buffers), 1);
3619
3620		var i: usize = 0;
3621		while (i < buffers.len) : (i += 1) {
3622		const sqe = try ring.recv(0xdededede, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3623		try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
3624		try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
3625		try testing.expectEqual(@as(u64, 0), sqe.addr);
3626		try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3627		try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3628		try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
3629		try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
3630		try testing.expectEqual(@as(u32, 1), try ring.submit());
3631
3632		const cqe = try ring.copy_cqe();
3633		switch (cqe.err()) {
3634		.SUCCESS => {},
3635		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3636		}
3637
3638		try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3639		const used_buffer_id = cqe.flags >> 16;
3640		try testing.expect(used_buffer_id >= 0 and used_buffer_id <= 3);
3641		try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3642
3643		try testing.expectEqual(@as(u64, 0xdededede), cqe.user_data);
3644		const buffer = buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))];
3645		try testing.expectEqualSlices(u8, &([_]u8{'z'} ** buffer_len), buffer);
3646		}
3647
3648		// This recv should fail
3649
3650		{
3651		const sqe = try ring.recv(0xdfdfdfdf, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3652		try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
3653		try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
3654		try testing.expectEqual(@as(u64, 0), sqe.addr);
3655		try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3656		try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3657		try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
3658		try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
3659		try testing.expectEqual(@as(u32, 1), try ring.submit());
3660
3661		const cqe = try ring.copy_cqe();
3662		switch (cqe.err()) {
3663		// Expected
3664		.NOBUFS => {},
3665		.SUCCESS => std.debug.panic("unexpected success", .{}),
3666		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3667		}
3668		try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3669		}
3670
3671		// Provide 1 buffer again
3672
3673		const reprovided_buffer_id = 2;
3674
3675		{
3676		_ = try ring.provide_buffers(0xabababab, @as([*]u8, @ptrCast(&buffers[reprovided_buffer_id])), buffer_len, 1, group_id, reprovided_buffer_id);
3677		try testing.expectEqual(@as(u32, 1), try ring.submit());
3678
3679		const cqe = try ring.copy_cqe();
3680		switch (cqe.err()) {
3681		.SUCCESS => {},
3682		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3683		}
3684		}
3685
3686		// Redo 1 send on the server socket
3687
3688		{
3689		_ = try ring.send(0xdeaddead, socket_test_harness.server, &([_]u8{'w'} ** buffer_len), 0);
3690		try testing.expectEqual(@as(u32, 1), try ring.submit());
3691
3692		_ = try ring.copy_cqe();
3693		}
3694
3695		// Final recv which should work
3696
3697		// Deliberately put something we don't expect in the buffers
3698		@memset(mem.sliceAsBytes(&buffers), 1);
3699
3700		{
3701		const sqe = try ring.recv(0xdfdfdfdf, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3702		try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
3703		try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
3704		try testing.expectEqual(@as(u64, 0), sqe.addr);
3705		try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3706		try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3707		try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
3708		try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
3709		try testing.expectEqual(@as(u32, 1), try ring.submit());
3710
3711		const cqe = try ring.copy_cqe();
3712		switch (cqe.err()) {
3713		.SUCCESS => {},
3714		else => \|errno\| std.debug.panic("unhandled errno: {}", .{errno}),
3715		}
3716
3717		try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3718		const used_buffer_id = cqe.flags >> 16;
3719		try testing.expectEqual(used_buffer_id, reprovided_buffer_id);
3720		try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3721		try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3722		const buffer = buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))];
3723		try testing.expectEqualSlices(u8, &([_]u8{'w'} ** buffer_len), buffer);
3724		}
3725		}
3726
3727		/// Used for testing server/client interactions.
3728		const SocketTestHarness = struct {
3729		listener: os.socket_t,
3730		server: os.socket_t,
3731		client: os.socket_t,
3732
3733		fn close(self: SocketTestHarness) void {
3734		posix.close(self.client);
3735		posix.close(self.listener);
3736		}
3737		};
3738
3739		fn createSocketTestHarness(ring: *IO_Uring) !SocketTestHarness {
3740		// Create a TCP server socket
3741		var address = try net.Address.parseIp4("127.0.0.1", 0);
3742		const listener_socket = try createListenerSocket(&address);
3743		errdefer posix.close(listener_socket);
3744
3745		// Submit 1 accept
3746		var accept_addr: os.sockaddr = undefined;
3747		var accept_addr_len: os.socklen_t = @sizeOf(@TypeOf(accept_addr));
3748		_ = try ring.accept(0xaaaaaaaa, listener_socket, &accept_addr, &accept_addr_len, 0);
3749
3750		// Create a TCP client socket
3751		const client = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
3752		errdefer posix.close(client);
3753		_ = try ring.connect(0xcccccccc, client, &address.any, address.getOsSockLen());
3754
3755		try testing.expectEqual(@as(u32, 2), try ring.submit());
3756
3757		var cqe_accept = try ring.copy_cqe();
3758		if (cqe_accept.err() == .INVAL) return error.SkipZigTest;
3759		var cqe_connect = try ring.copy_cqe();
3760		if (cqe_connect.err() == .INVAL) return error.SkipZigTest;
3761
3762		// The accept/connect CQEs may arrive in any order, the connect CQE will sometimes come first:
3763		if (cqe_accept.user_data == 0xcccccccc and cqe_connect.user_data == 0xaaaaaaaa) {
3764		const a = cqe_accept;
3765		const b = cqe_connect;
3766		cqe_accept = b;
3767		cqe_connect = a;
3768		}
3769
3770		try testing.expectEqual(@as(u64, 0xaaaaaaaa), cqe_accept.user_data);
3771		if (cqe_accept.res <= 0) std.debug.print("\ncqe_accept.res={}\n", .{cqe_accept.res});
3772		try testing.expect(cqe_accept.res > 0);
3773		try testing.expectEqual(@as(u32, 0), cqe_accept.flags);
3774		try testing.expectEqual(linux.io_uring_cqe{
3775		.user_data = 0xcccccccc,
3776		.res = 0,
3777		.flags = 0,
3778		}, cqe_connect);
3779
3780		// All good
3781
3782		return SocketTestHarness{
3783		.listener = listener_socket,
3784		.server = cqe_accept.res,
3785		.client = client,
3786		};
3787		}
3788
3789		fn createListenerSocket(address: *net.Address) !os.socket_t {
3790		const kernel_backlog = 1;
3791		const listener_socket = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
3792		errdefer posix.close(listener_socket);
3793
3794		try os.setsockopt(listener_socket, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
3795		try os.bind(listener_socket, &address.any, address.getOsSockLen());
3796		try os.listen(listener_socket, kernel_backlog);
3797
3798		// set address to the OS-chosen IP/port.
3799		var slen: os.socklen_t = address.getOsSockLen();
3800		try os.getsockname(listener_socket, &address.any, &slen);
3801
3802		return listener_socket;
3803		}
3804
3805		test "accept multishot" {
3806		if (builtin.os.tag != .linux) return error.SkipZigTest;
3807
3808		var ring = IO_Uring.init(16, 0) catch \|err\| switch (err) {
3809		error.SystemOutdated => return error.SkipZigTest,
3810		error.PermissionDenied => return error.SkipZigTest,
3811		else => return err,
3812		};
3813		defer ring.deinit();
3814
3815		var address = try net.Address.parseIp4("127.0.0.1", 0);
3816		const listener_socket = try createListenerSocket(&address);
3817		defer posix.close(listener_socket);
3818
3819		// submit multishot accept operation
3820		var addr: os.sockaddr = undefined;
3821		var addr_len: os.socklen_t = @sizeOf(@TypeOf(addr));
3822		const userdata: u64 = 0xaaaaaaaa;
3823		_ = try ring.accept_multishot(userdata, listener_socket, &addr, &addr_len, 0);
3824		try testing.expectEqual(@as(u32, 1), try ring.submit());
3825
3826		var nr: usize = 4; // number of clients to connect
3827		while (nr > 0) : (nr -= 1) {
3828		// connect client
3829		const client = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
3830		errdefer posix.close(client);
3831		try os.connect(client, &address.any, address.getOsSockLen());
3832
3833		// test accept completion
3834		var cqe = try ring.copy_cqe();
3835		if (cqe.err() == .INVAL) return error.SkipZigTest;
3836		try testing.expect(cqe.res > 0);
3837		try testing.expect(cqe.user_data == userdata);
3838		try testing.expect(cqe.flags & linux.IORING_CQE_F_MORE > 0); // more flag is set
3839
3840		posix.close(client);
3841		}
3842		}
3843
3844		test "accept/connect/send_zc/recv" {
3845		try skipKernelLessThan(.{ .major = 6, .minor = 0, .patch = 0 });
3846
3847		var ring = IO_Uring.init(16, 0) catch \|err\| switch (err) {
3848		error.SystemOutdated => return error.SkipZigTest,
3849		error.PermissionDenied => return error.SkipZigTest,
3850		else => return err,
3851		};
3852		defer ring.deinit();
3853
3854		const socket_test_harness = try createSocketTestHarness(&ring);
3855		defer socket_test_harness.close();
3856
3857		const buffer_send = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
3858		var buffer_recv = [_]u8{0} ** 10;
3859
3860		// zero-copy send
3861		const send = try ring.send_zc(0xeeeeeeee, socket_test_harness.client, buffer_send[0..], 0, 0);
3862		send.flags \|= linux.IOSQE_IO_LINK;
3863		_ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
3864		try testing.expectEqual(@as(u32, 2), try ring.submit());
3865
3866		// First completion of zero-copy send.
3867		// IORING_CQE_F_MORE, means that there
3868		// will be a second completion event / notification for the
3869		// request, with the user_data field set to the same value.
3870		// buffer_send must be keep alive until second cqe.
3871		var cqe_send = try ring.copy_cqe();
3872		try testing.expectEqual(linux.io_uring_cqe{
3873		.user_data = 0xeeeeeeee,
3874		.res = buffer_send.len,
3875		.flags = linux.IORING_CQE_F_MORE,
3876		}, cqe_send);
3877
3878		const cqe_recv = try ring.copy_cqe();
3879		try testing.expectEqual(linux.io_uring_cqe{
3880		.user_data = 0xffffffff,
3881		.res = buffer_recv.len,
3882		.flags = cqe_recv.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
3883		}, cqe_recv);
3884
3885		try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
3886
3887		// Second completion of zero-copy send.
3888		// IORING_CQE_F_NOTIF in flags signals that kernel is done with send_buffer
3889		cqe_send = try ring.copy_cqe();
3890		try testing.expectEqual(linux.io_uring_cqe{
3891		.user_data = 0xeeeeeeee,
3892		.res = 0,
3893		.flags = linux.IORING_CQE_F_NOTIF,
3894		}, cqe_send);
3895		}
3896
3897		test "accept_direct" {
3898		try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
3899
3900		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
3901		error.SystemOutdated => return error.SkipZigTest,
3902		error.PermissionDenied => return error.SkipZigTest,
3903		else => return err,
3904		};
3905		defer ring.deinit();
3906		var address = try net.Address.parseIp4("127.0.0.1", 0);
3907
3908		// register direct file descriptors
3909		var registered_fds = [_]os.fd_t{-1} ** 2;
3910		try ring.register_files(registered_fds[0..]);
3911
3912		const listener_socket = try createListenerSocket(&address);
3913		defer posix.close(listener_socket);
3914
3915		const accept_userdata: u64 = 0xaaaaaaaa;
3916		const read_userdata: u64 = 0xbbbbbbbb;
3917		const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
3918
3919		for (0..2) \|_\| {
3920		for (registered_fds, 0..) \|_, i\| {
3921		var buffer_recv = [_]u8{0} ** 16;
3922		const buffer_send: []const u8 = data[0 .. data.len - i]; // make it different at each loop
3923
3924		// submit accept, will chose registered fd and return index in cqe
3925		_ = try ring.accept_direct(accept_userdata, listener_socket, null, null, 0);
3926		try testing.expectEqual(@as(u32, 1), try ring.submit());
3927
3928		// connect
3929		const client = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
3930		try os.connect(client, &address.any, address.getOsSockLen());
3931		defer posix.close(client);
3932
3933		// accept completion
3934		const cqe_accept = try ring.copy_cqe();
3935		try testing.expectEqual(os.E.SUCCESS, cqe_accept.err());
3936		const fd_index = cqe_accept.res;
3937		try testing.expect(fd_index < registered_fds.len);
3938		try testing.expect(cqe_accept.user_data == accept_userdata);
3939
3940		// send data
3941		_ = try os.send(client, buffer_send, 0);
3942
3943		// Example of how to use registered fd:
3944		// Submit receive to fixed file returned by accept (fd_index).
3945		// Fd field is set to registered file index, returned by accept.
3946		// Flag linux.IOSQE_FIXED_FILE must be set.
3947		const recv_sqe = try ring.recv(read_userdata, fd_index, .{ .buffer = &buffer_recv }, 0);
3948		recv_sqe.flags \|= linux.IOSQE_FIXED_FILE;
3949		try testing.expectEqual(@as(u32, 1), try ring.submit());
3950
3951		// accept receive
3952		const recv_cqe = try ring.copy_cqe();
3953		try testing.expect(recv_cqe.user_data == read_userdata);
3954		try testing.expect(recv_cqe.res == buffer_send.len);
3955		try testing.expectEqualSlices(u8, buffer_send, buffer_recv[0..buffer_send.len]);
3956		}
3957		// no more available fds, accept will get NFILE error
3958		{
3959		// submit accept
3960		_ = try ring.accept_direct(accept_userdata, listener_socket, null, null, 0);
3961		try testing.expectEqual(@as(u32, 1), try ring.submit());
3962		// connect
3963		const client = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
3964		try os.connect(client, &address.any, address.getOsSockLen());
3965		defer posix.close(client);
3966		// completion with error
3967		const cqe_accept = try ring.copy_cqe();
3968		try testing.expect(cqe_accept.user_data == accept_userdata);
3969		try testing.expectEqual(os.E.NFILE, cqe_accept.err());
3970		}
3971		// return file descriptors to kernel
3972		try ring.register_files_update(0, registered_fds[0..]);
3973		}
3974		try ring.unregister_files();
3975		}
3976
3977		test "accept_multishot_direct" {
3978		try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
3979
3980		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
3981		error.SystemOutdated => return error.SkipZigTest,
3982		error.PermissionDenied => return error.SkipZigTest,
3983		else => return err,
3984		};
3985		defer ring.deinit();
3986
3987		var address = try net.Address.parseIp4("127.0.0.1", 0);
3988
3989		var registered_fds = [_]os.fd_t{-1} ** 2;
3990		try ring.register_files(registered_fds[0..]);
3991
3992		const listener_socket = try createListenerSocket(&address);
3993		defer posix.close(listener_socket);
3994
3995		const accept_userdata: u64 = 0xaaaaaaaa;
3996
3997		for (0..2) \|_\| {
3998		// submit multishot accept
3999		// Will chose registered fd and return index of the selected registered file in cqe.
4000		_ = try ring.accept_multishot_direct(accept_userdata, listener_socket, null, null, 0);
4001		try testing.expectEqual(@as(u32, 1), try ring.submit());
4002
4003		for (registered_fds) \|_\| {
4004		// connect
4005		const client = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
4006		try os.connect(client, &address.any, address.getOsSockLen());
4007		defer posix.close(client);
4008
4009		// accept completion
4010		const cqe_accept = try ring.copy_cqe();
4011		const fd_index = cqe_accept.res;
4012		try testing.expect(fd_index < registered_fds.len);
4013		try testing.expect(cqe_accept.user_data == accept_userdata);
4014		try testing.expect(cqe_accept.flags & linux.IORING_CQE_F_MORE > 0); // has more is set
4015		}
4016		// No more available fds, accept will get NFILE error.
4017		// Multishot is terminated (more flag is not set).
4018		{
4019		// connect
4020		const client = try os.socket(address.any.family, os.SOCK.STREAM \| os.SOCK.CLOEXEC, 0);
4021		try os.connect(client, &address.any, address.getOsSockLen());
4022		defer posix.close(client);
4023		// completion with error
4024		const cqe_accept = try ring.copy_cqe();
4025		try testing.expect(cqe_accept.user_data == accept_userdata);
4026		try testing.expectEqual(os.E.NFILE, cqe_accept.err());
4027		try testing.expect(cqe_accept.flags & linux.IORING_CQE_F_MORE == 0); // has more is not set
4028		}
4029		// return file descriptors to kernel
4030		try ring.register_files_update(0, registered_fds[0..]);
4031		}
4032		try ring.unregister_files();
4033		}
4034
4035		test "socket" {
4036		try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
4037
4038		var ring = IO_Uring.init(1, 0) catch \|err\| switch (err) {
4039		error.SystemOutdated => return error.SkipZigTest,
4040		error.PermissionDenied => return error.SkipZigTest,
4041		else => return err,
4042		};
4043		defer ring.deinit();
4044
4045		// prepare, submit socket operation
4046		_ = try ring.socket(0, linux.AF.INET, os.SOCK.STREAM, 0, 0);
4047		try testing.expectEqual(@as(u32, 1), try ring.submit());
4048
4049		// test completion
4050		var cqe = try ring.copy_cqe();
4051		try testing.expectEqual(os.E.SUCCESS, cqe.err());
4052		const fd: os.fd_t = @intCast(cqe.res);
4053		try testing.expect(fd > 2);
4054
4055		os.close(fd);
4056		}
4057
4058		test "socket_direct/socket_direct_alloc/close_direct" {
4059		try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
4060
4061		var ring = IO_Uring.init(2, 0) catch \|err\| switch (err) {
4062		error.SystemOutdated => return error.SkipZigTest,
4063		error.PermissionDenied => return error.SkipZigTest,
4064		else => return err,
4065		};
4066		defer ring.deinit();
4067
4068		var registered_fds = [_]os.fd_t{-1} ** 3;
4069		try ring.register_files(registered_fds[0..]);
4070
4071		// create socket in registered file descriptor at index 0 (last param)
4072		_ = try ring.socket_direct(0, linux.AF.INET, os.SOCK.STREAM, 0, 0, 0);
4073		try testing.expectEqual(@as(u32, 1), try ring.submit());
4074		var cqe_socket = try ring.copy_cqe();
4075		try testing.expectEqual(os.E.SUCCESS, cqe_socket.err());
4076		try testing.expect(cqe_socket.res == 0);
4077
4078		// create socket in registered file descriptor at index 1 (last param)
4079		_ = try ring.socket_direct(0, linux.AF.INET, os.SOCK.STREAM, 0, 0, 1);
4080		try testing.expectEqual(@as(u32, 1), try ring.submit());
4081		cqe_socket = try ring.copy_cqe();
4082		try testing.expectEqual(os.E.SUCCESS, cqe_socket.err());
4083		try testing.expect(cqe_socket.res == 0); // res is 0 when index is specified
4084
4085		// create socket in kernel chosen file descriptor index (_alloc version)
4086		// completion res has index from registered files
4087		_ = try ring.socket_direct_alloc(0, linux.AF.INET, os.SOCK.STREAM, 0, 0);
4088		try testing.expectEqual(@as(u32, 1), try ring.submit());
4089		cqe_socket = try ring.copy_cqe();
4090		try testing.expectEqual(os.E.SUCCESS, cqe_socket.err());
4091		try testing.expect(cqe_socket.res == 2); // returns registered file index
4092
4093		// use sockets from registered_fds in connect operation
4094		var address = try net.Address.parseIp4("127.0.0.1", 0);
4095		const listener_socket = try createListenerSocket(&address);
4096		defer posix.close(listener_socket);
4097		const accept_userdata: u64 = 0xaaaaaaaa;
4098		const connect_userdata: u64 = 0xbbbbbbbb;
4099		const close_userdata: u64 = 0xcccccccc;
4100		for (registered_fds, 0..) \|_, fd_index\| {
4101		// prepare accept
4102		_ = try ring.accept(accept_userdata, listener_socket, null, null, 0);
4103		// prepare connect with fixed socket
4104		const connect_sqe = try ring.connect(connect_userdata, @intCast(fd_index), &address.any, address.getOsSockLen());
4105		connect_sqe.flags \|= linux.IOSQE_FIXED_FILE; // fd is fixed file index
4106		// submit both
4107		try testing.expectEqual(@as(u32, 2), try ring.submit());
4108		// get completions
4109		var cqe_connect = try ring.copy_cqe();
4110		var cqe_accept = try ring.copy_cqe();
4111		// ignore order
4112		if (cqe_connect.user_data == accept_userdata and cqe_accept.user_data == connect_userdata) {
4113		const a = cqe_accept;
4114		const b = cqe_connect;
4115		cqe_accept = b;
4116		cqe_connect = a;
4117		}
4118		// test connect completion
4119		try testing.expect(cqe_connect.user_data == connect_userdata);
4120		try testing.expectEqual(os.E.SUCCESS, cqe_connect.err());
4121		// test accept completion
4122		try testing.expect(cqe_accept.user_data == accept_userdata);
4123		try testing.expectEqual(os.E.SUCCESS, cqe_accept.err());
4124
4125		// submit and test close_direct
4126		_ = try ring.close_direct(close_userdata, @intCast(fd_index));
4127		try testing.expectEqual(@as(u32, 1), try ring.submit());
4128		var cqe_close = try ring.copy_cqe();
4129		try testing.expect(cqe_close.user_data == close_userdata);
4130		try testing.expectEqual(os.E.SUCCESS, cqe_close.err());
4131		}
4132
4133		try ring.unregister_files();
4134		}
4135
4136		test "openat_direct/close_direct" {
4137		try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
4138
4139		var ring = IO_Uring.init(2, 0) catch \|err\| switch (err) {
4140		error.SystemOutdated => return error.SkipZigTest,
4141		error.PermissionDenied => return error.SkipZigTest,
4142		else => return err,
4143		};
4144		defer ring.deinit();
4145
4146		var registered_fds = [_]os.fd_t{-1} ** 3;
4147		try ring.register_files(registered_fds[0..]);
4148
4149		var tmp = std.testing.tmpDir(.{});
4150		defer tmp.cleanup();
4151		const path = "test_io_uring_close_direct";
4152		const flags: linux.O = .{ .ACCMODE = .RDWR, .CREAT = true };
4153		const mode: os.mode_t = 0o666;
4154		const user_data: u64 = 0;
4155
4156		// use registered file at index 0 (last param)
4157		_ = try ring.openat_direct(user_data, tmp.dir.fd, path, flags, mode, 0);
4158		try testing.expectEqual(@as(u32, 1), try ring.submit());
4159		var cqe = try ring.copy_cqe();
4160		try testing.expectEqual(os.E.SUCCESS, cqe.err());
4161		try testing.expect(cqe.res == 0);
4162
4163		// use registered file at index 1
4164		_ = try ring.openat_direct(user_data, tmp.dir.fd, path, flags, mode, 1);
4165		try testing.expectEqual(@as(u32, 1), try ring.submit());
4166		cqe = try ring.copy_cqe();
4167		try testing.expectEqual(os.E.SUCCESS, cqe.err());
4168		try testing.expect(cqe.res == 0); // res is 0 when we specify index
4169
4170		// let kernel choose registered file index
4171		_ = try ring.openat_direct(user_data, tmp.dir.fd, path, flags, mode, linux.IORING_FILE_INDEX_ALLOC);
4172		try testing.expectEqual(@as(u32, 1), try ring.submit());
4173		cqe = try ring.copy_cqe();
4174		try testing.expectEqual(os.E.SUCCESS, cqe.err());
4175		try testing.expect(cqe.res == 2); // chosen index is in res
4176
4177		// close all open file descriptors
4178		for (registered_fds, 0..) \|_, fd_index\| {
4179		_ = try ring.close_direct(user_data, @intCast(fd_index));
4180		try testing.expectEqual(@as(u32, 1), try ring.submit());
4181		var cqe_close = try ring.copy_cqe();
4182		try testing.expectEqual(os.E.SUCCESS, cqe_close.err());
4183		}
4184		try ring.unregister_files();
4185		}
4186
4187		test "waitid" {
4188		try skipKernelLessThan(.{ .major = 6, .minor = 7, .patch = 0 });
4189
4190		var ring = IO_Uring.init(16, 0) catch \|err\| switch (err) {
4191		error.SystemOutdated => return error.SkipZigTest,
4192		error.PermissionDenied => return error.SkipZigTest,
4193		else => return err,
4194		};
4195		defer ring.deinit();
4196
4197		const pid = try os.fork();
4198		if (pid == 0) {
4199		os.exit(7);
4200		}
4201
4202		var siginfo: os.siginfo_t = undefined;
4203		_ = try ring.waitid(0, .PID, pid, &siginfo, os.W.EXITED, 0);
4204
4205		try testing.expectEqual(1, try ring.submit());
4206
4207		const cqe_waitid = try ring.copy_cqe();
4208		try testing.expectEqual(0, cqe_waitid.res);
4209		try testing.expectEqual(pid, siginfo.fields.common.first.piduid.pid);
4210		try testing.expectEqual(7, siginfo.fields.common.second.sigchld.status);
4211		}
4212
4213		/// For use in tests. Returns SkipZigTest is kernel version is less than required.
4214		inline fn skipKernelLessThan(required: std.SemanticVersion) !void {
4215		if (builtin.os.tag != .linux) return error.SkipZigTest;
4216
4217		var uts: linux.utsname = undefined;
4218		const res = linux.uname(&uts);
4219		switch (linux.getErrno(res)) {
4220		.SUCCESS => {},
4221		else => \|errno\| return os.unexpectedErrno(errno),
4222		}
4223
4224		const release = mem.sliceTo(&uts.release, 0);
4225		var current = try std.SemanticVersion.parse(release);
4226		current.pre = null; // don't check pre field
4227		if (required.order(current) == .gt) return error.SkipZigTest;
4228		}

lib/std/os/linux/io_uring_sqe.zig created+579

...	...	@@ -0,0 +1,579 @@
	1	//! Contains only the definition of `io_uring_sqe`.
	2	//! Split into its own file to compartmentalize the initialization methods.
	3
	4	const std = @import("../../std.zig");
	5	const os = std.os;
	6	const linux = os.linux;
	7
	8	pub const io_uring_sqe = extern struct {
	9	opcode: linux.IORING_OP,
	10	flags: u8,
	11	ioprio: u16,
	12	fd: i32,
	13	off: u64,
	14	addr: u64,
	15	len: u32,
	16	rw_flags: u32,
	17	user_data: u64,
	18	buf_index: u16,
	19	personality: u16,
	20	splice_fd_in: i32,
	21	addr3: u64,
	22	resv: u64,
	23
	24	pub fn prep_nop(sqe: *linux.io_uring_sqe) void {
	25	sqe.* = .{
	26	.opcode = .NOP,
	27	.flags = 0,
	28	.ioprio = 0,
	29	.fd = 0,
	30	.off = 0,
	31	.addr = 0,
	32	.len = 0,
	33	.rw_flags = 0,
	34	.user_data = 0,
	35	.buf_index = 0,
	36	.personality = 0,
	37	.splice_fd_in = 0,
	38	.addr3 = 0,
	39	.resv = 0,
	40	};
	41	}
	42
	43	pub fn prep_fsync(sqe: *linux.io_uring_sqe, fd: os.fd_t, flags: u32) void {
	44	sqe.* = .{
	45	.opcode = .FSYNC,
	46	.flags = 0,
	47	.ioprio = 0,
	48	.fd = fd,
	49	.off = 0,
	50	.addr = 0,
	51	.len = 0,
	52	.rw_flags = flags,
	53	.user_data = 0,
	54	.buf_index = 0,
	55	.personality = 0,
	56	.splice_fd_in = 0,
	57	.addr3 = 0,
	58	.resv = 0,
	59	};
	60	}
	61
	62	pub fn prep_rw(
	63	sqe: *linux.io_uring_sqe,
	64	op: linux.IORING_OP,
	65	fd: os.fd_t,
	66	addr: u64,
	67	len: usize,
	68	offset: u64,
	69	) void {
	70	sqe.* = .{
	71	.opcode = op,
	72	.flags = 0,
	73	.ioprio = 0,
	74	.fd = fd,
	75	.off = offset,
	76	.addr = addr,
	77	.len = @intCast(len),
	78	.rw_flags = 0,
	79	.user_data = 0,
	80	.buf_index = 0,
	81	.personality = 0,
	82	.splice_fd_in = 0,
	83	.addr3 = 0,
	84	.resv = 0,
	85	};
	86	}
	87
	88	pub fn prep_read(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []u8, offset: u64) void {
	89	sqe.prep_rw(.READ, fd, @intFromPtr(buffer.ptr), buffer.len, offset);
	90	}
	91
	92	pub fn prep_write(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, offset: u64) void {
	93	sqe.prep_rw(.WRITE, fd, @intFromPtr(buffer.ptr), buffer.len, offset);
	94	}
	95
	96	pub fn prep_splice(sqe: *linux.io_uring_sqe, fd_in: os.fd_t, off_in: u64, fd_out: os.fd_t, off_out: u64, len: usize) void {
	97	sqe.prep_rw(.SPLICE, fd_out, undefined, len, off_out);
	98	sqe.addr = off_in;
	99	sqe.splice_fd_in = fd_in;
	100	}
	101
	102	pub fn prep_readv(
	103	sqe: *linux.io_uring_sqe,
	104	fd: os.fd_t,
	105	iovecs: []const os.iovec,
	106	offset: u64,
	107	) void {
	108	sqe.prep_rw(.READV, fd, @intFromPtr(iovecs.ptr), iovecs.len, offset);
	109	}
	110
	111	pub fn prep_writev(
	112	sqe: *linux.io_uring_sqe,
	113	fd: os.fd_t,
	114	iovecs: []const os.iovec_const,
	115	offset: u64,
	116	) void {
	117	sqe.prep_rw(.WRITEV, fd, @intFromPtr(iovecs.ptr), iovecs.len, offset);
	118	}
	119
	120	pub fn prep_read_fixed(sqe: linux.io_uring_sqe, fd: os.fd_t, buffer: os.iovec, offset: u64, buffer_index: u16) void {
	121	sqe.prep_rw(.READ_FIXED, fd, @intFromPtr(buffer.iov_base), buffer.iov_len, offset);
	122	sqe.buf_index = buffer_index;
	123	}
	124
	125	pub fn prep_write_fixed(sqe: linux.io_uring_sqe, fd: os.fd_t, buffer: os.iovec, offset: u64, buffer_index: u16) void {
	126	sqe.prep_rw(.WRITE_FIXED, fd, @intFromPtr(buffer.iov_base), buffer.iov_len, offset);
	127	sqe.buf_index = buffer_index;
	128	}
	129
	130	pub fn prep_accept(
	131	sqe: *linux.io_uring_sqe,
	132	fd: os.fd_t,
	133	addr: ?*os.sockaddr,
	134	addrlen: ?*os.socklen_t,
	135	flags: u32,
	136	) void {
	137	// `addr` holds a pointer to `sockaddr`, and `addr2` holds a pointer to socklen_t`.
	138	// `addr2` maps to `sqe.off` (u64) instead of `sqe.len` (which is only a u32).
	139	sqe.prep_rw(.ACCEPT, fd, @intFromPtr(addr), 0, @intFromPtr(addrlen));
	140	sqe.rw_flags = flags;
	141	}
	142
	143	pub fn prep_accept_direct(
	144	sqe: *linux.io_uring_sqe,
	145	fd: os.fd_t,
	146	addr: ?*os.sockaddr,
	147	addrlen: ?*os.socklen_t,
	148	flags: u32,
	149	file_index: u32,
	150	) void {
	151	prep_accept(sqe, fd, addr, addrlen, flags);
	152	__io_uring_set_target_fixed_file(sqe, file_index);
	153	}
	154
	155	pub fn prep_multishot_accept_direct(
	156	sqe: *linux.io_uring_sqe,
	157	fd: os.fd_t,
	158	addr: ?*os.sockaddr,
	159	addrlen: ?*os.socklen_t,
	160	flags: u32,
	161	) void {
	162	prep_multishot_accept(sqe, fd, addr, addrlen, flags);
	163	__io_uring_set_target_fixed_file(sqe, linux.IORING_FILE_INDEX_ALLOC);
	164	}
	165
	166	fn __io_uring_set_target_fixed_file(sqe: *linux.io_uring_sqe, file_index: u32) void {
	167	const sqe_file_index: u32 = if (file_index == linux.IORING_FILE_INDEX_ALLOC)
	168	linux.IORING_FILE_INDEX_ALLOC
	169	else
	170	// 0 means no fixed files, indexes should be encoded as "index + 1"
	171	file_index + 1;
	172	// This filed is overloaded in liburing:
	173	// splice_fd_in: i32
	174	// sqe_file_index: u32
	175	sqe.splice_fd_in = @bitCast(sqe_file_index);
	176	}
	177
	178	pub fn prep_connect(
	179	sqe: *linux.io_uring_sqe,
	180	fd: os.fd_t,
	181	addr: *const os.sockaddr,
	182	addrlen: os.socklen_t,
	183	) void {
	184	// `addrlen` maps to `sqe.off` (u64) instead of `sqe.len` (which is only a u32).
	185	sqe.prep_rw(.CONNECT, fd, @intFromPtr(addr), 0, addrlen);
	186	}
	187
	188	pub fn prep_epoll_ctl(
	189	sqe: *linux.io_uring_sqe,
	190	epfd: os.fd_t,
	191	fd: os.fd_t,
	192	op: u32,
	193	ev: ?*linux.epoll_event,
	194	) void {
	195	sqe.prep_rw(.EPOLL_CTL, epfd, @intFromPtr(ev), op, @intCast(fd));
	196	}
	197
	198	pub fn prep_recv(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []u8, flags: u32) void {
	199	sqe.prep_rw(.RECV, fd, @intFromPtr(buffer.ptr), buffer.len, 0);
	200	sqe.rw_flags = flags;
	201	}
	202
	203	pub fn prep_send(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, flags: u32) void {
	204	sqe.prep_rw(.SEND, fd, @intFromPtr(buffer.ptr), buffer.len, 0);
	205	sqe.rw_flags = flags;
	206	}
	207
	208	pub fn prep_send_zc(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, flags: u32, zc_flags: u16) void {
	209	sqe.prep_rw(.SEND_ZC, fd, @intFromPtr(buffer.ptr), buffer.len, 0);
	210	sqe.rw_flags = flags;
	211	sqe.ioprio = zc_flags;
	212	}
	213
	214	pub fn prep_send_zc_fixed(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, flags: u32, zc_flags: u16, buf_index: u16) void {
	215	prep_send_zc(sqe, fd, buffer, flags, zc_flags);
	216	sqe.ioprio \|= linux.IORING_RECVSEND_FIXED_BUF;
	217	sqe.buf_index = buf_index;
	218	}
	219
	220	pub fn prep_sendmsg_zc(
	221	sqe: *linux.io_uring_sqe,
	222	fd: os.fd_t,
	223	msg: *const os.msghdr_const,
	224	flags: u32,
	225	) void {
	226	prep_sendmsg(sqe, fd, msg, flags);
	227	sqe.opcode = .SENDMSG_ZC;
	228	}
	229
	230	pub fn prep_recvmsg(
	231	sqe: *linux.io_uring_sqe,
	232	fd: os.fd_t,
	233	msg: *os.msghdr,
	234	flags: u32,
	235	) void {
	236	sqe.prep_rw(.RECVMSG, fd, @intFromPtr(msg), 1, 0);
	237	sqe.rw_flags = flags;
	238	}
	239
	240	pub fn prep_sendmsg(
	241	sqe: *linux.io_uring_sqe,
	242	fd: os.fd_t,
	243	msg: *const os.msghdr_const,
	244	flags: u32,
	245	) void {
	246	sqe.prep_rw(.SENDMSG, fd, @intFromPtr(msg), 1, 0);
	247	sqe.rw_flags = flags;
	248	}
	249
	250	pub fn prep_openat(
	251	sqe: *linux.io_uring_sqe,
	252	fd: os.fd_t,
	253	path: [*:0]const u8,
	254	flags: linux.O,
	255	mode: os.mode_t,
	256	) void {
	257	sqe.prep_rw(.OPENAT, fd, @intFromPtr(path), mode, 0);
	258	sqe.rw_flags = @bitCast(flags);
	259	}
	260
	261	pub fn prep_openat_direct(
	262	sqe: *linux.io_uring_sqe,
	263	fd: os.fd_t,
	264	path: [*:0]const u8,
	265	flags: linux.O,
	266	mode: os.mode_t,
	267	file_index: u32,
	268	) void {
	269	prep_openat(sqe, fd, path, flags, mode);
	270	__io_uring_set_target_fixed_file(sqe, file_index);
	271	}
	272
	273	pub fn prep_close(sqe: *linux.io_uring_sqe, fd: os.fd_t) void {
	274	sqe.* = .{
	275	.opcode = .CLOSE,
	276	.flags = 0,
	277	.ioprio = 0,
	278	.fd = fd,
	279	.off = 0,
	280	.addr = 0,
	281	.len = 0,
	282	.rw_flags = 0,
	283	.user_data = 0,
	284	.buf_index = 0,
	285	.personality = 0,
	286	.splice_fd_in = 0,
	287	.addr3 = 0,
	288	.resv = 0,
	289	};
	290	}
	291
	292	pub fn prep_close_direct(sqe: *linux.io_uring_sqe, file_index: u32) void {
	293	prep_close(sqe, 0);
	294	__io_uring_set_target_fixed_file(sqe, file_index);
	295	}
	296
	297	pub fn prep_timeout(
	298	sqe: *linux.io_uring_sqe,
	299	ts: *const os.linux.kernel_timespec,
	300	count: u32,
	301	flags: u32,
	302	) void {
	303	sqe.prep_rw(.TIMEOUT, -1, @intFromPtr(ts), 1, count);
	304	sqe.rw_flags = flags;
	305	}
	306
	307	pub fn prep_timeout_remove(sqe: *linux.io_uring_sqe, timeout_user_data: u64, flags: u32) void {
	308	sqe.* = .{
	309	.opcode = .TIMEOUT_REMOVE,
	310	.flags = 0,
	311	.ioprio = 0,
	312	.fd = -1,
	313	.off = 0,
	314	.addr = timeout_user_data,
	315	.len = 0,
	316	.rw_flags = flags,
	317	.user_data = 0,
	318	.buf_index = 0,
	319	.personality = 0,
	320	.splice_fd_in = 0,
	321	.addr3 = 0,
	322	.resv = 0,
	323	};
	324	}
	325
	326	pub fn prep_link_timeout(
	327	sqe: *linux.io_uring_sqe,
	328	ts: *const os.linux.kernel_timespec,
	329	flags: u32,
	330	) void {
	331	sqe.prep_rw(.LINK_TIMEOUT, -1, @intFromPtr(ts), 1, 0);
	332	sqe.rw_flags = flags;
	333	}
	334
	335	pub fn prep_poll_add(
	336	sqe: *linux.io_uring_sqe,
	337	fd: os.fd_t,
	338	poll_mask: u32,
	339	) void {
	340	sqe.prep_rw(.POLL_ADD, fd, @intFromPtr(@as(?*anyopaque, null)), 0, 0);
	341	// Poll masks previously used to comprise of 16 bits in the flags union of
	342	// a SQE, but were then extended to comprise of 32 bits in order to make
	343	// room for additional option flags. To ensure that the correct bits of
	344	// poll masks are consistently and properly read across multiple kernel
	345	// versions, poll masks are enforced to be little-endian.
	346	// https://www.spinics.net/lists/io-uring/msg02848.html
	347	sqe.rw_flags = std.mem.nativeToLittle(u32, poll_mask);
	348	}
	349
	350	pub fn prep_poll_remove(
	351	sqe: *linux.io_uring_sqe,
	352	target_user_data: u64,
	353	) void {
	354	sqe.prep_rw(.POLL_REMOVE, -1, target_user_data, 0, 0);
	355	}
	356
	357	pub fn prep_poll_update(
	358	sqe: *linux.io_uring_sqe,
	359	old_user_data: u64,
	360	new_user_data: u64,
	361	poll_mask: u32,
	362	flags: u32,
	363	) void {
	364	sqe.prep_rw(.POLL_REMOVE, -1, old_user_data, flags, new_user_data);
	365	// Poll masks previously used to comprise of 16 bits in the flags union of
	366	// a SQE, but were then extended to comprise of 32 bits in order to make
	367	// room for additional option flags. To ensure that the correct bits of
	368	// poll masks are consistently and properly read across multiple kernel
	369	// versions, poll masks are enforced to be little-endian.
	370	// https://www.spinics.net/lists/io-uring/msg02848.html
	371	sqe.rw_flags = std.mem.nativeToLittle(u32, poll_mask);
	372	}
	373
	374	pub fn prep_fallocate(
	375	sqe: *linux.io_uring_sqe,
	376	fd: os.fd_t,
	377	mode: i32,
	378	offset: u64,
	379	len: u64,
	380	) void {
	381	sqe.* = .{
	382	.opcode = .FALLOCATE,
	383	.flags = 0,
	384	.ioprio = 0,
	385	.fd = fd,
	386	.off = offset,
	387	.addr = len,
	388	.len = @intCast(mode),
	389	.rw_flags = 0,
	390	.user_data = 0,
	391	.buf_index = 0,
	392	.personality = 0,
	393	.splice_fd_in = 0,
	394	.addr3 = 0,
	395	.resv = 0,
	396	};
	397	}
	398
	399	pub fn prep_statx(
	400	sqe: *linux.io_uring_sqe,
	401	fd: os.fd_t,
	402	path: [*:0]const u8,
	403	flags: u32,
	404	mask: u32,
	405	buf: *linux.Statx,
	406	) void {
	407	sqe.prep_rw(.STATX, fd, @intFromPtr(path), mask, @intFromPtr(buf));
	408	sqe.rw_flags = flags;
	409	}
	410
	411	pub fn prep_cancel(
	412	sqe: *linux.io_uring_sqe,
	413	cancel_user_data: u64,
	414	flags: u32,
	415	) void {
	416	sqe.prep_rw(.ASYNC_CANCEL, -1, cancel_user_data, 0, 0);
	417	sqe.rw_flags = flags;
	418	}
	419
	420	pub fn prep_shutdown(
	421	sqe: *linux.io_uring_sqe,
	422	sockfd: os.socket_t,
	423	how: u32,
	424	) void {
	425	sqe.prep_rw(.SHUTDOWN, sockfd, 0, how, 0);
	426	}
	427
	428	pub fn prep_renameat(
	429	sqe: *linux.io_uring_sqe,
	430	old_dir_fd: os.fd_t,
	431	old_path: [*:0]const u8,
	432	new_dir_fd: os.fd_t,
	433	new_path: [*:0]const u8,
	434	flags: u32,
	435	) void {
	436	sqe.prep_rw(
	437	.RENAMEAT,
	438	old_dir_fd,
	439	@intFromPtr(old_path),
	440	0,
	441	@intFromPtr(new_path),
	442	);
	443	sqe.len = @bitCast(new_dir_fd);
	444	sqe.rw_flags = flags;
	445	}
	446
	447	pub fn prep_unlinkat(
	448	sqe: *linux.io_uring_sqe,
	449	dir_fd: os.fd_t,
	450	path: [*:0]const u8,
	451	flags: u32,
	452	) void {
	453	sqe.prep_rw(.UNLINKAT, dir_fd, @intFromPtr(path), 0, 0);
	454	sqe.rw_flags = flags;
	455	}
	456
	457	pub fn prep_mkdirat(
	458	sqe: *linux.io_uring_sqe,
	459	dir_fd: os.fd_t,
	460	path: [*:0]const u8,
	461	mode: os.mode_t,
	462	) void {
	463	sqe.prep_rw(.MKDIRAT, dir_fd, @intFromPtr(path), mode, 0);
	464	}
	465
	466	pub fn prep_symlinkat(
	467	sqe: *linux.io_uring_sqe,
	468	target: [*:0]const u8,
	469	new_dir_fd: os.fd_t,
	470	link_path: [*:0]const u8,
	471	) void {
	472	sqe.prep_rw(
	473	.SYMLINKAT,
	474	new_dir_fd,
	475	@intFromPtr(target),
	476	0,
	477	@intFromPtr(link_path),
	478	);
	479	}
	480
	481	pub fn prep_linkat(
	482	sqe: *linux.io_uring_sqe,
	483	old_dir_fd: os.fd_t,
	484	old_path: [*:0]const u8,
	485	new_dir_fd: os.fd_t,
	486	new_path: [*:0]const u8,
	487	flags: u32,
	488	) void {
	489	sqe.prep_rw(
	490	.LINKAT,
	491	old_dir_fd,
	492	@intFromPtr(old_path),
	493	0,
	494	@intFromPtr(new_path),
	495	);
	496	sqe.len = @bitCast(new_dir_fd);
	497	sqe.rw_flags = flags;
	498	}
	499
	500	pub fn prep_provide_buffers(
	501	sqe: *linux.io_uring_sqe,
	502	buffers: [*]u8,
	503	buffer_len: usize,
	504	num: usize,
	505	group_id: usize,
	506	buffer_id: usize,
	507	) void {
	508	const ptr = @intFromPtr(buffers);
	509	sqe.prep_rw(.PROVIDE_BUFFERS, @intCast(num), ptr, buffer_len, buffer_id);
	510	sqe.buf_index = @intCast(group_id);
	511	}
	512
	513	pub fn prep_remove_buffers(
	514	sqe: *linux.io_uring_sqe,
	515	num: usize,
	516	group_id: usize,
	517	) void {
	518	sqe.prep_rw(.REMOVE_BUFFERS, @intCast(num), 0, 0, 0);
	519	sqe.buf_index = @intCast(group_id);
	520	}
	521
	522	pub fn prep_multishot_accept(
	523	sqe: *linux.io_uring_sqe,
	524	fd: os.fd_t,
	525	addr: ?*os.sockaddr,
	526	addrlen: ?*os.socklen_t,
	527	flags: u32,
	528	) void {
	529	prep_accept(sqe, fd, addr, addrlen, flags);
	530	sqe.ioprio \|= linux.IORING_ACCEPT_MULTISHOT;
	531	}
	532
	533	pub fn prep_socket(
	534	sqe: *linux.io_uring_sqe,
	535	domain: u32,
	536	socket_type: u32,
	537	protocol: u32,
	538	flags: u32,
	539	) void {
	540	sqe.prep_rw(.SOCKET, @intCast(domain), 0, protocol, socket_type);
	541	sqe.rw_flags = flags;
	542	}
	543
	544	pub fn prep_socket_direct(
	545	sqe: *linux.io_uring_sqe,
	546	domain: u32,
	547	socket_type: u32,
	548	protocol: u32,
	549	flags: u32,
	550	file_index: u32,
	551	) void {
	552	prep_socket(sqe, domain, socket_type, protocol, flags);
	553	__io_uring_set_target_fixed_file(sqe, file_index);
	554	}
	555
	556	pub fn prep_socket_direct_alloc(
	557	sqe: *linux.io_uring_sqe,
	558	domain: u32,
	559	socket_type: u32,
	560	protocol: u32,
	561	flags: u32,
	562	) void {
	563	prep_socket(sqe, domain, socket_type, protocol, flags);
	564	__io_uring_set_target_fixed_file(sqe, linux.IORING_FILE_INDEX_ALLOC);
	565	}
	566
	567	pub fn prep_waitid(
	568	sqe: *linux.io_uring_sqe,
	569	id_type: linux.P,
	570	id: i32,
	571	infop: *linux.siginfo_t,
	572	options: u32,
	573	flags: u32,
	574	) void {
	575	sqe.prep_rw(.WAITID, id, 0, @intFromEnum(id_type), @intFromPtr(infop));
	576	sqe.rw_flags = flags;
	577	sqe.splice_fd_in = @bitCast(options);
	578	}
	579	};

lib/std/os/linux/test.zig+4

...	...	@@ -120,3 +120,7 @@ test "fadvise" {
120	120	const ret = linux.fadvise(file.handle, 0, 0, linux.POSIX_FADV.SEQUENTIAL);
121	121	try expectEqual(@as(usize, 0), ret);
122	122	}
	123
	124	test {
	125	_ = linux.IoUring;
	126	}