authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-12-18 14:19:49-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-12-23 22:15:10-08:00
loge205b13ffbfe0e57e39bb324799785450c9a1da5
tree94c3c11fa778b803bc3f6d6a8ee33603cac4094a
parentb4bfd501aedb4abe4257b7f54a225a9654c4e08e

std.os.linux.IoUring: update for new fs API

and split out tests into a separate file

3 files changed, 2707 insertions(+), 2740 deletions(-)

lib/std/os/linux/IoUring.zig+14-2738
...@@ -3,15 +3,15 @@ const IoUring = @This();...@@ -3,15 +3,15 @@ const IoUring = @This();
3const builtin = @import("builtin");3const builtin = @import("builtin");
4const is_linux = builtin.os.tag == .linux;4const is_linux = builtin.os.tag == .linux;
55
6const std = @import("std");6const std = @import("../../std.zig");
7const Io = std.Io;7const Io = std.Io;
8const Allocator = std.mem.Allocator;
8const assert = std.debug.assert;9const assert = std.debug.assert;
9const mem = std.mem;
10const net = std.Io.net;
11const posix = std.posix;10const posix = std.posix;
12const linux = std.os.linux;11const linux = std.os.linux;
13const testing = std.testing;12const testing = std.testing;
14const page_size_min = std.heap.page_size_min;13const page_size_min = std.heap.page_size_min;
14const createSocketTestHarness = @import("IoUring/test.zig").createSocketTestHarness;
1515
16fd: linux.fd_t = -1,16fd: linux.fd_t = -1,
17sq: SubmissionQueue,17sq: SubmissionQueue,
...@@ -25,7 +25,7 @@ features: u32,...@@ -25,7 +25,7 @@ features: u32,
25/// see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L8027-L8050.25/// see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L8027-L8050.
26/// Matches the interface of io_uring_queue_init() in liburing.26/// Matches the interface of io_uring_queue_init() in liburing.
27pub fn init(entries: u16, flags: u32) !IoUring {27pub fn init(entries: u16, flags: u32) !IoUring {
28 var params = mem.zeroInit(linux.io_uring_params, .{28 var params = std.mem.zeroInit(linux.io_uring_params, .{
29 .flags = flags,29 .flags = flags,
30 .sq_thread_idle = 1000,30 .sq_thread_idle = 1000,
31 });31 });
...@@ -1312,7 +1312,7 @@ pub fn unregister_buffers(self: *IoUring) !void {...@@ -1312,7 +1312,7 @@ pub fn unregister_buffers(self: *IoUring) !void {
1312/// io_uring subsystem of the running kernel. The io_uring_probe contains the1312/// io_uring subsystem of the running kernel. The io_uring_probe contains the
1313/// list of supported operations.1313/// list of supported operations.
1314pub fn get_probe(self: *IoUring) !linux.io_uring_probe {1314pub fn get_probe(self: *IoUring) !linux.io_uring_probe {
1315 var probe = mem.zeroInit(linux.io_uring_probe, .{});1315 var probe = std.mem.zeroInit(linux.io_uring_probe, .{});
1316 const res = linux.io_uring_register(self.fd, .REGISTER_PROBE, &probe, probe.ops.len);1316 const res = linux.io_uring_register(self.fd, .REGISTER_PROBE, &probe, probe.ops.len);
1317 try handle_register_buf_ring_result(res);1317 try handle_register_buf_ring_result(res);
1318 return probe;1318 return probe;
...@@ -1639,7 +1639,7 @@ pub const BufferGroup = struct {...@@ -1639,7 +1639,7 @@ pub const BufferGroup = struct {
16391639
1640 pub fn init(1640 pub fn init(
1641 ring: *IoUring,1641 ring: *IoUring,
1642 allocator: mem.Allocator,1642 allocator: Allocator,
1643 group_id: u16,1643 group_id: u16,
1644 buffer_size: u32,1644 buffer_size: u32,
1645 buffers_count: u16,1645 buffers_count: u16,
...@@ -1673,7 +1673,7 @@ pub const BufferGroup = struct {...@@ -1673,7 +1673,7 @@ pub const BufferGroup = struct {
1673 };1673 };
1674 }1674 }
16751675
1676 pub fn deinit(self: *BufferGroup, allocator: mem.Allocator) void {1676 pub fn deinit(self: *BufferGroup, allocator: Allocator) void {
1677 free_buf_ring(self.ring.fd, self.br, self.buffers_count, self.group_id);1677 free_buf_ring(self.ring.fd, self.br, self.buffers_count, self.group_id);
1678 allocator.free(self.buffers);1678 allocator.free(self.buffers);
1679 allocator.free(self.heads);1679 allocator.free(self.heads);
...@@ -1698,7 +1698,7 @@ pub const BufferGroup = struct {...@@ -1698,7 +1698,7 @@ pub const BufferGroup = struct {
1698 }1698 }
16991699
1700 // Get buffer by id.1700 // Get buffer by id.
1701 fn get_by_id(self: *BufferGroup, buffer_id: u16) []u8 {1701 pub fn get_by_id(self: *BufferGroup, buffer_id: u16) []u8 {
1702 const pos = self.buffer_size * buffer_id;1702 const pos = self.buffer_size * buffer_id;
1703 return self.buffers[pos .. pos + self.buffer_size][self.heads[buffer_id]..];1703 return self.buffers[pos .. pos + self.buffer_size][self.heads[buffer_id]..];
1704 }1704 }
...@@ -1767,7 +1767,7 @@ fn register_buf_ring(...@@ -1767,7 +1767,7 @@ fn register_buf_ring(
1767 group_id: u16,1767 group_id: u16,
1768 flags: linux.io_uring_buf_reg.Flags,1768 flags: linux.io_uring_buf_reg.Flags,
1769) !void {1769) !void {
1770 var reg = mem.zeroInit(linux.io_uring_buf_reg, .{1770 var reg = std.mem.zeroInit(linux.io_uring_buf_reg, .{
1771 .ring_addr = addr,1771 .ring_addr = addr,
1772 .ring_entries = entries,1772 .ring_entries = entries,
1773 .bgid = group_id,1773 .bgid = group_id,
...@@ -1784,7 +1784,7 @@ fn register_buf_ring(...@@ -1784,7 +1784,7 @@ fn register_buf_ring(
1784}1784}
17851785
1786fn unregister_buf_ring(fd: linux.fd_t, group_id: u16) !void {1786fn unregister_buf_ring(fd: linux.fd_t, group_id: u16) !void {
1787 var reg = mem.zeroInit(linux.io_uring_buf_reg, .{1787 var reg = std.mem.zeroInit(linux.io_uring_buf_reg, .{
1788 .bgid = group_id,1788 .bgid = group_id,
1789 });1789 });
1790 const res = linux.io_uring_register(1790 const res = linux.io_uring_register(
...@@ -1851,2272 +1851,11 @@ pub fn buf_ring_advance(br: *linux.io_uring_buf_ring, count: u16) void {...@@ -1851,2272 +1851,11 @@ pub fn buf_ring_advance(br: *linux.io_uring_buf_ring, count: u16) void {
1851 @atomicStore(u16, &br.tail, tail, .release);1851 @atomicStore(u16, &br.tail, tail, .release);
1852}1852}
18531853
1854test "structs/offsets/entries" {
1855 if (!is_linux) return error.SkipZigTest;
1856
1857 try testing.expectEqual(@as(usize, 120), @sizeOf(linux.io_uring_params));
1858 try testing.expectEqual(@as(usize, 64), @sizeOf(linux.io_uring_sqe));
1859 try testing.expectEqual(@as(usize, 16), @sizeOf(linux.io_uring_cqe));
1860
1861 try testing.expectEqual(0, linux.IORING_OFF_SQ_RING);
1862 try testing.expectEqual(0x8000000, linux.IORING_OFF_CQ_RING);
1863 try testing.expectEqual(0x10000000, linux.IORING_OFF_SQES);
1864
1865 try testing.expectError(error.EntriesZero, IoUring.init(0, 0));
1866 try testing.expectError(error.EntriesNotPowerOfTwo, IoUring.init(3, 0));
1867}
1868
1869test "nop" {
1870 if (!is_linux) return error.SkipZigTest;
1871
1872 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1873 error.SystemOutdated => return error.SkipZigTest,
1874 error.PermissionDenied => return error.SkipZigTest,
1875 else => return err,
1876 };
1877 defer {
1878 ring.deinit();
1879 testing.expectEqual(@as(linux.fd_t, -1), ring.fd) catch @panic("test failed");
1880 }
1881
1882 const sqe = try ring.nop(0xaaaaaaaa);
1883 try testing.expectEqual(linux.io_uring_sqe{
1884 .opcode = .NOP,
1885 .flags = 0,
1886 .ioprio = 0,
1887 .fd = 0,
1888 .off = 0,
1889 .addr = 0,
1890 .len = 0,
1891 .rw_flags = 0,
1892 .user_data = 0xaaaaaaaa,
1893 .buf_index = 0,
1894 .personality = 0,
1895 .splice_fd_in = 0,
1896 .addr3 = 0,
1897 .resv = 0,
1898 }, sqe.*);
1899
1900 try testing.expectEqual(@as(u32, 0), ring.sq.sqe_head);
1901 try testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
1902 try testing.expectEqual(@as(u32, 0), ring.sq.tail.*);
1903 try testing.expectEqual(@as(u32, 0), ring.cq.head.*);
1904 try testing.expectEqual(@as(u32, 1), ring.sq_ready());
1905 try testing.expectEqual(@as(u32, 0), ring.cq_ready());
1906
1907 try testing.expectEqual(@as(u32, 1), try ring.submit());
1908 try testing.expectEqual(@as(u32, 1), ring.sq.sqe_head);
1909 try testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
1910 try testing.expectEqual(@as(u32, 1), ring.sq.tail.*);
1911 try testing.expectEqual(@as(u32, 0), ring.cq.head.*);
1912 try testing.expectEqual(@as(u32, 0), ring.sq_ready());
1913
1914 try testing.expectEqual(linux.io_uring_cqe{
1915 .user_data = 0xaaaaaaaa,
1916 .res = 0,
1917 .flags = 0,
1918 }, try ring.copy_cqe());
1919 try testing.expectEqual(@as(u32, 1), ring.cq.head.*);
1920 try testing.expectEqual(@as(u32, 0), ring.cq_ready());
1921
1922 const sqe_barrier = try ring.nop(0xbbbbbbbb);
1923 sqe_barrier.flags |= linux.IOSQE_IO_DRAIN;
1924 try testing.expectEqual(@as(u32, 1), try ring.submit());
1925 try testing.expectEqual(linux.io_uring_cqe{
1926 .user_data = 0xbbbbbbbb,
1927 .res = 0,
1928 .flags = 0,
1929 }, try ring.copy_cqe());
1930 try testing.expectEqual(@as(u32, 2), ring.sq.sqe_head);
1931 try testing.expectEqual(@as(u32, 2), ring.sq.sqe_tail);
1932 try testing.expectEqual(@as(u32, 2), ring.sq.tail.*);
1933 try testing.expectEqual(@as(u32, 2), ring.cq.head.*);
1934}
1935
1936test "readv" {
1937 if (!is_linux) return error.SkipZigTest;
1938
1939 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1940 error.SystemOutdated => return error.SkipZigTest,
1941 error.PermissionDenied => return error.SkipZigTest,
1942 else => return err,
1943 };
1944 defer ring.deinit();
1945
1946 const fd = try posix.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
1947 defer posix.close(fd);
1948
1949 // Linux Kernel 5.4 supports IORING_REGISTER_FILES but not sparse fd sets (i.e. an fd of -1).
1950 // Linux Kernel 5.5 adds support for sparse fd sets.
1951 // Compare:
1952 // https://github.com/torvalds/linux/blob/v5.4/fs/io_uring.c#L3119-L3124 vs
1953 // https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L6687-L6691
1954 // We therefore avoid stressing sparse fd sets here:
1955 var registered_fds = [_]linux.fd_t{0} ** 1;
1956 const fd_index = 0;
1957 registered_fds[fd_index] = fd;
1958 try ring.register_files(registered_fds[0..]);
1959
1960 var buffer = [_]u8{42} ** 128;
1961 var iovecs = [_]posix.iovec{posix.iovec{ .base = &buffer, .len = buffer.len }};
1962 const sqe = try ring.read(0xcccccccc, fd_index, .{ .iovecs = iovecs[0..] }, 0);
1963 try testing.expectEqual(linux.IORING_OP.READV, sqe.opcode);
1964 sqe.flags |= linux.IOSQE_FIXED_FILE;
1965
1966 try testing.expectError(error.SubmissionQueueFull, ring.nop(0));
1967 try testing.expectEqual(@as(u32, 1), try ring.submit());
1968 try testing.expectEqual(linux.io_uring_cqe{
1969 .user_data = 0xcccccccc,
1970 .res = buffer.len,
1971 .flags = 0,
1972 }, try ring.copy_cqe());
1973 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
1974
1975 try ring.unregister_files();
1976}
1977
1978test "writev/fsync/readv" {
1979 if (!is_linux) return error.SkipZigTest;
1980
1981 const io = testing.io;
1982
1983 var ring = IoUring.init(4, 0) catch |err| switch (err) {
1984 error.SystemOutdated => return error.SkipZigTest,
1985 error.PermissionDenied => return error.SkipZigTest,
1986 else => return err,
1987 };
1988 defer ring.deinit();
1989
1990 var tmp = std.testing.tmpDir(.{});
1991 defer tmp.cleanup();
1992
1993 const path = "test_io_uring_writev_fsync_readv";
1994 const file = try tmp.dir.createFile(io, path, .{ .read = true, .truncate = true });
1995 defer file.close(io);
1996 const fd = file.handle;
1997
1998 const buffer_write = [_]u8{42} ** 128;
1999 const iovecs_write = [_]posix.iovec_const{
2000 posix.iovec_const{ .base = &buffer_write, .len = buffer_write.len },
2001 };
2002 var buffer_read = [_]u8{0} ** 128;
2003 var iovecs_read = [_]posix.iovec{
2004 posix.iovec{ .base = &buffer_read, .len = buffer_read.len },
2005 };
2006
2007 const sqe_writev = try ring.writev(0xdddddddd, fd, iovecs_write[0..], 17);
2008 try testing.expectEqual(linux.IORING_OP.WRITEV, sqe_writev.opcode);
2009 try testing.expectEqual(@as(u64, 17), sqe_writev.off);
2010 sqe_writev.flags |= linux.IOSQE_IO_LINK;
2011
2012 const sqe_fsync = try ring.fsync(0xeeeeeeee, fd, 0);
2013 try testing.expectEqual(linux.IORING_OP.FSYNC, sqe_fsync.opcode);
2014 try testing.expectEqual(fd, sqe_fsync.fd);
2015 sqe_fsync.flags |= linux.IOSQE_IO_LINK;
2016
2017 const sqe_readv = try ring.read(0xffffffff, fd, .{ .iovecs = iovecs_read[0..] }, 17);
2018 try testing.expectEqual(linux.IORING_OP.READV, sqe_readv.opcode);
2019 try testing.expectEqual(@as(u64, 17), sqe_readv.off);
2020
2021 try testing.expectEqual(@as(u32, 3), ring.sq_ready());
2022 try testing.expectEqual(@as(u32, 3), try ring.submit_and_wait(3));
2023 try testing.expectEqual(@as(u32, 0), ring.sq_ready());
2024 try testing.expectEqual(@as(u32, 3), ring.cq_ready());
2025
2026 try testing.expectEqual(linux.io_uring_cqe{
2027 .user_data = 0xdddddddd,
2028 .res = buffer_write.len,
2029 .flags = 0,
2030 }, try ring.copy_cqe());
2031 try testing.expectEqual(@as(u32, 2), ring.cq_ready());
2032
2033 try testing.expectEqual(linux.io_uring_cqe{
2034 .user_data = 0xeeeeeeee,
2035 .res = 0,
2036 .flags = 0,
2037 }, try ring.copy_cqe());
2038 try testing.expectEqual(@as(u32, 1), ring.cq_ready());
2039
2040 try testing.expectEqual(linux.io_uring_cqe{
2041 .user_data = 0xffffffff,
2042 .res = buffer_read.len,
2043 .flags = 0,
2044 }, try ring.copy_cqe());
2045 try testing.expectEqual(@as(u32, 0), ring.cq_ready());
2046
2047 try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
2048}
2049
2050test "write/read" {
2051 if (!is_linux) return error.SkipZigTest;
2052
2053 const io = testing.io;
2054
2055 var ring = IoUring.init(2, 0) catch |err| switch (err) {
2056 error.SystemOutdated => return error.SkipZigTest,
2057 error.PermissionDenied => return error.SkipZigTest,
2058 else => return err,
2059 };
2060 defer ring.deinit();
2061
2062 var tmp = std.testing.tmpDir(.{});
2063 defer tmp.cleanup();
2064 const path = "test_io_uring_write_read";
2065 const file = try tmp.dir.createFile(io, path, .{ .read = true, .truncate = true });
2066 defer file.close(io);
2067 const fd = file.handle;
2068
2069 const buffer_write = [_]u8{97} ** 20;
2070 var buffer_read = [_]u8{98} ** 20;
2071 const sqe_write = try ring.write(0x11111111, fd, buffer_write[0..], 10);
2072 try testing.expectEqual(linux.IORING_OP.WRITE, sqe_write.opcode);
2073 try testing.expectEqual(@as(u64, 10), sqe_write.off);
2074 sqe_write.flags |= linux.IOSQE_IO_LINK;
2075 const sqe_read = try ring.read(0x22222222, fd, .{ .buffer = buffer_read[0..] }, 10);
2076 try testing.expectEqual(linux.IORING_OP.READ, sqe_read.opcode);
2077 try testing.expectEqual(@as(u64, 10), sqe_read.off);
2078 try testing.expectEqual(@as(u32, 2), try ring.submit());
2079
2080 const cqe_write = try ring.copy_cqe();
2081 const cqe_read = try ring.copy_cqe();
2082 // Prior to Linux Kernel 5.6 this is the only way to test for read/write support:
2083 // https://lwn.net/Articles/809820/
2084 if (cqe_write.err() == .INVAL) return error.SkipZigTest;
2085 if (cqe_read.err() == .INVAL) return error.SkipZigTest;
2086 try testing.expectEqual(linux.io_uring_cqe{
2087 .user_data = 0x11111111,
2088 .res = buffer_write.len,
2089 .flags = 0,
2090 }, cqe_write);
2091 try testing.expectEqual(linux.io_uring_cqe{
2092 .user_data = 0x22222222,
2093 .res = buffer_read.len,
2094 .flags = 0,
2095 }, cqe_read);
2096 try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
2097}
2098
2099test "splice/read" {
2100 if (!is_linux) return error.SkipZigTest;
2101
2102 const io = testing.io;
2103
2104 var ring = IoUring.init(4, 0) catch |err| switch (err) {
2105 error.SystemOutdated => return error.SkipZigTest,
2106 error.PermissionDenied => return error.SkipZigTest,
2107 else => return err,
2108 };
2109 defer ring.deinit();
2110
2111 var tmp = std.testing.tmpDir(.{});
2112 const path_src = "test_io_uring_splice_src";
2113 const file_src = try tmp.dir.createFile(io, path_src, .{ .read = true, .truncate = true });
2114 defer file_src.close(io);
2115 const fd_src = file_src.handle;
2116
2117 const path_dst = "test_io_uring_splice_dst";
2118 const file_dst = try tmp.dir.createFile(io, path_dst, .{ .read = true, .truncate = true });
2119 defer file_dst.close(io);
2120 const fd_dst = file_dst.handle;
2121
2122 const buffer_write = [_]u8{97} ** 20;
2123 var buffer_read = [_]u8{98} ** 20;
2124 _ = try file_src.write(&buffer_write);
2125
2126 const fds = try posix.pipe();
2127 const pipe_offset: u64 = std.math.maxInt(u64);
2128
2129 const sqe_splice_to_pipe = try ring.splice(0x11111111, fd_src, 0, fds[1], pipe_offset, buffer_write.len);
2130 try testing.expectEqual(linux.IORING_OP.SPLICE, sqe_splice_to_pipe.opcode);
2131 try testing.expectEqual(@as(u64, 0), sqe_splice_to_pipe.addr);
2132 try testing.expectEqual(pipe_offset, sqe_splice_to_pipe.off);
2133 sqe_splice_to_pipe.flags |= linux.IOSQE_IO_LINK;
2134
2135 const sqe_splice_from_pipe = try ring.splice(0x22222222, fds[0], pipe_offset, fd_dst, 10, buffer_write.len);
2136 try testing.expectEqual(linux.IORING_OP.SPLICE, sqe_splice_from_pipe.opcode);
2137 try testing.expectEqual(pipe_offset, sqe_splice_from_pipe.addr);
2138 try testing.expectEqual(@as(u64, 10), sqe_splice_from_pipe.off);
2139 sqe_splice_from_pipe.flags |= linux.IOSQE_IO_LINK;
2140
2141 const sqe_read = try ring.read(0x33333333, fd_dst, .{ .buffer = buffer_read[0..] }, 10);
2142 try testing.expectEqual(linux.IORING_OP.READ, sqe_read.opcode);
2143 try testing.expectEqual(@as(u64, 10), sqe_read.off);
2144 try testing.expectEqual(@as(u32, 3), try ring.submit());
2145
2146 const cqe_splice_to_pipe = try ring.copy_cqe();
2147 const cqe_splice_from_pipe = try ring.copy_cqe();
2148 const cqe_read = try ring.copy_cqe();
2149 // Prior to Linux Kernel 5.6 this is the only way to test for splice/read support:
2150 // https://lwn.net/Articles/809820/
2151 if (cqe_splice_to_pipe.err() == .INVAL) return error.SkipZigTest;
2152 if (cqe_splice_from_pipe.err() == .INVAL) return error.SkipZigTest;
2153 if (cqe_read.err() == .INVAL) return error.SkipZigTest;
2154 try testing.expectEqual(linux.io_uring_cqe{
2155 .user_data = 0x11111111,
2156 .res = buffer_write.len,
2157 .flags = 0,
2158 }, cqe_splice_to_pipe);
2159 try testing.expectEqual(linux.io_uring_cqe{
2160 .user_data = 0x22222222,
2161 .res = buffer_write.len,
2162 .flags = 0,
2163 }, cqe_splice_from_pipe);
2164 try testing.expectEqual(linux.io_uring_cqe{
2165 .user_data = 0x33333333,
2166 .res = buffer_read.len,
2167 .flags = 0,
2168 }, cqe_read);
2169 try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
2170}
2171
2172test "write_fixed/read_fixed" {
2173 if (!is_linux) return error.SkipZigTest;
2174
2175 const io = testing.io;
2176
2177 var ring = IoUring.init(2, 0) catch |err| switch (err) {
2178 error.SystemOutdated => return error.SkipZigTest,
2179 error.PermissionDenied => return error.SkipZigTest,
2180 else => return err,
2181 };
2182 defer ring.deinit();
2183
2184 var tmp = std.testing.tmpDir(.{});
2185 defer tmp.cleanup();
2186
2187 const path = "test_io_uring_write_read_fixed";
2188 const file = try tmp.dir.createFile(io, path, .{ .read = true, .truncate = true });
2189 defer file.close(io);
2190 const fd = file.handle;
2191
2192 var raw_buffers: [2][11]u8 = undefined;
2193 // First buffer will be written to the file.
2194 @memset(&raw_buffers[0], 'z');
2195 raw_buffers[0][0.."foobar".len].* = "foobar".*;
2196
2197 var buffers = [2]posix.iovec{
2198 .{ .base = &raw_buffers[0], .len = raw_buffers[0].len },
2199 .{ .base = &raw_buffers[1], .len = raw_buffers[1].len },
2200 };
2201 ring.register_buffers(&buffers) catch |err| switch (err) {
2202 error.SystemResources => {
2203 // See https://github.com/ziglang/zig/issues/15362
2204 return error.SkipZigTest;
2205 },
2206 else => |e| return e,
2207 };
2208
2209 const sqe_write = try ring.write_fixed(0x45454545, fd, &buffers[0], 3, 0);
2210 try testing.expectEqual(linux.IORING_OP.WRITE_FIXED, sqe_write.opcode);
2211 try testing.expectEqual(@as(u64, 3), sqe_write.off);
2212 sqe_write.flags |= linux.IOSQE_IO_LINK;
2213
2214 const sqe_read = try ring.read_fixed(0x12121212, fd, &buffers[1], 0, 1);
2215 try testing.expectEqual(linux.IORING_OP.READ_FIXED, sqe_read.opcode);
2216 try testing.expectEqual(@as(u64, 0), sqe_read.off);
2217
2218 try testing.expectEqual(@as(u32, 2), try ring.submit());
2219
2220 const cqe_write = try ring.copy_cqe();
2221 const cqe_read = try ring.copy_cqe();
2222
2223 try testing.expectEqual(linux.io_uring_cqe{
2224 .user_data = 0x45454545,
2225 .res = @as(i32, @intCast(buffers[0].len)),
2226 .flags = 0,
2227 }, cqe_write);
2228 try testing.expectEqual(linux.io_uring_cqe{
2229 .user_data = 0x12121212,
2230 .res = @as(i32, @intCast(buffers[1].len)),
2231 .flags = 0,
2232 }, cqe_read);
2233
2234 try testing.expectEqualSlices(u8, "\x00\x00\x00", buffers[1].base[0..3]);
2235 try testing.expectEqualSlices(u8, "foobar", buffers[1].base[3..9]);
2236 try testing.expectEqualSlices(u8, "zz", buffers[1].base[9..11]);
2237}
2238
2239test "openat" {
2240 if (!is_linux) return error.SkipZigTest;
2241
2242 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2243 error.SystemOutdated => return error.SkipZigTest,
2244 error.PermissionDenied => return error.SkipZigTest,
2245 else => return err,
2246 };
2247 defer ring.deinit();
2248
2249 var tmp = std.testing.tmpDir(.{});
2250 defer tmp.cleanup();
2251
2252 const path = "test_io_uring_openat";
2253
2254 // Workaround for LLVM bug: https://github.com/ziglang/zig/issues/12014
2255 const path_addr = if (builtin.zig_backend == .stage2_llvm) p: {
2256 var workaround = path;
2257 _ = &workaround;
2258 break :p @intFromPtr(workaround);
2259 } else @intFromPtr(path);
2260
2261 const flags: linux.O = .{ .CLOEXEC = true, .ACCMODE = .RDWR, .CREAT = true };
2262 const mode: posix.mode_t = 0o666;
2263 const sqe_openat = try ring.openat(0x33333333, tmp.dir.fd, path, flags, mode);
2264 try testing.expectEqual(linux.io_uring_sqe{
2265 .opcode = .OPENAT,
2266 .flags = 0,
2267 .ioprio = 0,
2268 .fd = tmp.dir.fd,
2269 .off = 0,
2270 .addr = path_addr,
2271 .len = mode,
2272 .rw_flags = @bitCast(flags),
2273 .user_data = 0x33333333,
2274 .buf_index = 0,
2275 .personality = 0,
2276 .splice_fd_in = 0,
2277 .addr3 = 0,
2278 .resv = 0,
2279 }, sqe_openat.*);
2280 try testing.expectEqual(@as(u32, 1), try ring.submit());
2281
2282 const cqe_openat = try ring.copy_cqe();
2283 try testing.expectEqual(@as(u64, 0x33333333), cqe_openat.user_data);
2284 if (cqe_openat.err() == .INVAL) return error.SkipZigTest;
2285 if (cqe_openat.err() == .BADF) return error.SkipZigTest;
2286 if (cqe_openat.res <= 0) std.debug.print("\ncqe_openat.res={}\n", .{cqe_openat.res});
2287 try testing.expect(cqe_openat.res > 0);
2288 try testing.expectEqual(@as(u32, 0), cqe_openat.flags);
2289
2290 posix.close(cqe_openat.res);
2291}
2292
2293test "close" {
2294 if (!is_linux) return error.SkipZigTest;
2295
2296 const io = testing.io;
2297
2298 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2299 error.SystemOutdated => return error.SkipZigTest,
2300 error.PermissionDenied => return error.SkipZigTest,
2301 else => return err,
2302 };
2303 defer ring.deinit();
2304
2305 var tmp = std.testing.tmpDir(.{});
2306 defer tmp.cleanup();
2307
2308 const path = "test_io_uring_close";
2309 const file = try tmp.dir.createFile(io, path, .{});
2310 errdefer file.close(io);
2311
2312 const sqe_close = try ring.close(0x44444444, file.handle);
2313 try testing.expectEqual(linux.IORING_OP.CLOSE, sqe_close.opcode);
2314 try testing.expectEqual(file.handle, sqe_close.fd);
2315 try testing.expectEqual(@as(u32, 1), try ring.submit());
2316
2317 const cqe_close = try ring.copy_cqe();
2318 if (cqe_close.err() == .INVAL) return error.SkipZigTest;
2319 try testing.expectEqual(linux.io_uring_cqe{
2320 .user_data = 0x44444444,
2321 .res = 0,
2322 .flags = 0,
2323 }, cqe_close);
2324}
2325
2326test "accept/connect/send/recv" {
2327 if (!is_linux) return error.SkipZigTest;
2328
2329 const io = testing.io;
2330
2331 var ring = IoUring.init(16, 0) catch |err| switch (err) {
2332 error.SystemOutdated => return error.SkipZigTest,
2333 error.PermissionDenied => return error.SkipZigTest,
2334 else => return err,
2335 };
2336 defer ring.deinit();
2337
2338 const socket_test_harness = try createSocketTestHarness(&ring);
2339 defer socket_test_harness.close(io);
2340
2341 const buffer_send = [_]u8{ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 };
2342 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
2343
2344 const sqe_send = try ring.send(0xeeeeeeee, socket_test_harness.client, buffer_send[0..], 0);
2345 sqe_send.flags |= linux.IOSQE_IO_LINK;
2346 _ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
2347 try testing.expectEqual(@as(u32, 2), try ring.submit());
2348
2349 const cqe_send = try ring.copy_cqe();
2350 if (cqe_send.err() == .INVAL) return error.SkipZigTest;
2351 try testing.expectEqual(linux.io_uring_cqe{
2352 .user_data = 0xeeeeeeee,
2353 .res = buffer_send.len,
2354 .flags = 0,
2355 }, cqe_send);
2356
2357 const cqe_recv = try ring.copy_cqe();
2358 if (cqe_recv.err() == .INVAL) return error.SkipZigTest;
2359 try testing.expectEqual(linux.io_uring_cqe{
2360 .user_data = 0xffffffff,
2361 .res = buffer_recv.len,
2362 // ignore IORING_CQE_F_SOCK_NONEMPTY since it is only set on some systems
2363 .flags = cqe_recv.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
2364 }, cqe_recv);
2365
2366 try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
2367}
2368
2369test "sendmsg/recvmsg" {
2370 if (!is_linux) return error.SkipZigTest;
2371
2372 var ring = IoUring.init(2, 0) catch |err| switch (err) {
2373 error.SystemOutdated => return error.SkipZigTest,
2374 error.PermissionDenied => return error.SkipZigTest,
2375 else => return err,
2376 };
2377 defer ring.deinit();
2378
2379 var address_server: linux.sockaddr.in = .{
2380 .port = 0,
2381 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
2382 };
2383
2384 const server = try posix.socket(address_server.family, posix.SOCK.DGRAM, 0);
2385 defer posix.close(server);
2386 try posix.setsockopt(server, posix.SOL.SOCKET, posix.SO.REUSEPORT, &mem.toBytes(@as(c_int, 1)));
2387 try posix.setsockopt(server, posix.SOL.SOCKET, posix.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
2388 try posix.bind(server, addrAny(&address_server), @sizeOf(linux.sockaddr.in));
2389
2390 // set address_server to the OS-chosen IP/port.
2391 var slen: posix.socklen_t = @sizeOf(linux.sockaddr.in);
2392 try posix.getsockname(server, addrAny(&address_server), &slen);
2393
2394 const client = try posix.socket(address_server.family, posix.SOCK.DGRAM, 0);
2395 defer posix.close(client);
2396
2397 const buffer_send = [_]u8{42} ** 128;
2398 const iovecs_send = [_]posix.iovec_const{
2399 posix.iovec_const{ .base = &buffer_send, .len = buffer_send.len },
2400 };
2401 const msg_send: linux.msghdr_const = .{
2402 .name = addrAny(&address_server),
2403 .namelen = @sizeOf(linux.sockaddr.in),
2404 .iov = &iovecs_send,
2405 .iovlen = 1,
2406 .control = null,
2407 .controllen = 0,
2408 .flags = 0,
2409 };
2410 const sqe_sendmsg = try ring.sendmsg(0x11111111, client, &msg_send, 0);
2411 sqe_sendmsg.flags |= linux.IOSQE_IO_LINK;
2412 try testing.expectEqual(linux.IORING_OP.SENDMSG, sqe_sendmsg.opcode);
2413 try testing.expectEqual(client, sqe_sendmsg.fd);
2414
2415 var buffer_recv = [_]u8{0} ** 128;
2416 var iovecs_recv = [_]posix.iovec{
2417 posix.iovec{ .base = &buffer_recv, .len = buffer_recv.len },
2418 };
2419 var address_recv: linux.sockaddr.in = .{
2420 .port = 0,
2421 .addr = 0,
2422 };
2423 var msg_recv: linux.msghdr = .{
2424 .name = addrAny(&address_recv),
2425 .namelen = @sizeOf(linux.sockaddr.in),
2426 .iov = &iovecs_recv,
2427 .iovlen = 1,
2428 .control = null,
2429 .controllen = 0,
2430 .flags = 0,
2431 };
2432 const sqe_recvmsg = try ring.recvmsg(0x22222222, server, &msg_recv, 0);
2433 try testing.expectEqual(linux.IORING_OP.RECVMSG, sqe_recvmsg.opcode);
2434 try testing.expectEqual(server, sqe_recvmsg.fd);
2435
2436 try testing.expectEqual(@as(u32, 2), ring.sq_ready());
2437 try testing.expectEqual(@as(u32, 2), try ring.submit_and_wait(2));
2438 try testing.expectEqual(@as(u32, 0), ring.sq_ready());
2439 try testing.expectEqual(@as(u32, 2), ring.cq_ready());
2440
2441 const cqe_sendmsg = try ring.copy_cqe();
2442 if (cqe_sendmsg.res == -@as(i32, @intFromEnum(linux.E.INVAL))) return error.SkipZigTest;
2443 try testing.expectEqual(linux.io_uring_cqe{
2444 .user_data = 0x11111111,
2445 .res = buffer_send.len,
2446 .flags = 0,
2447 }, cqe_sendmsg);
2448
2449 const cqe_recvmsg = try ring.copy_cqe();
2450 if (cqe_recvmsg.res == -@as(i32, @intFromEnum(linux.E.INVAL))) return error.SkipZigTest;
2451 try testing.expectEqual(linux.io_uring_cqe{
2452 .user_data = 0x22222222,
2453 .res = buffer_recv.len,
2454 // ignore IORING_CQE_F_SOCK_NONEMPTY since it is set non-deterministically
2455 .flags = cqe_recvmsg.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
2456 }, cqe_recvmsg);
2457
2458 try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
2459}
2460
2461test "timeout (after a relative time)" {
2462 if (!is_linux) return error.SkipZigTest;
2463
2464 const io = testing.io;
2465
2466 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2467 error.SystemOutdated => return error.SkipZigTest,
2468 error.PermissionDenied => return error.SkipZigTest,
2469 else => return err,
2470 };
2471 defer ring.deinit();
2472
2473 const ms = 10;
2474 const margin = 5;
2475 const ts: linux.kernel_timespec = .{ .sec = 0, .nsec = ms * 1000000 };
2476
2477 const started = try std.Io.Clock.awake.now(io);
2478 const sqe = try ring.timeout(0x55555555, &ts, 0, 0);
2479 try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe.opcode);
2480 try testing.expectEqual(@as(u32, 1), try ring.submit());
2481 const cqe = try ring.copy_cqe();
2482 const stopped = try std.Io.Clock.awake.now(io);
2483
2484 try testing.expectEqual(linux.io_uring_cqe{
2485 .user_data = 0x55555555,
2486 .res = -@as(i32, @intFromEnum(linux.E.TIME)),
2487 .flags = 0,
2488 }, cqe);
2489
2490 // Tests should not depend on timings: skip test if outside margin.
2491 const ms_elapsed = started.durationTo(stopped).toMilliseconds();
2492 if (ms_elapsed > margin) return error.SkipZigTest;
2493}
2494
2495test "timeout (after a number of completions)" {
2496 if (!is_linux) return error.SkipZigTest;
2497
2498 var ring = IoUring.init(2, 0) catch |err| switch (err) {
2499 error.SystemOutdated => return error.SkipZigTest,
2500 error.PermissionDenied => return error.SkipZigTest,
2501 else => return err,
2502 };
2503 defer ring.deinit();
2504
2505 const ts: linux.kernel_timespec = .{ .sec = 3, .nsec = 0 };
2506 const count_completions: u64 = 1;
2507 const sqe_timeout = try ring.timeout(0x66666666, &ts, count_completions, 0);
2508 try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe_timeout.opcode);
2509 try testing.expectEqual(count_completions, sqe_timeout.off);
2510 _ = try ring.nop(0x77777777);
2511 try testing.expectEqual(@as(u32, 2), try ring.submit());
2512
2513 const cqe_nop = try ring.copy_cqe();
2514 try testing.expectEqual(linux.io_uring_cqe{
2515 .user_data = 0x77777777,
2516 .res = 0,
2517 .flags = 0,
2518 }, cqe_nop);
2519
2520 const cqe_timeout = try ring.copy_cqe();
2521 try testing.expectEqual(linux.io_uring_cqe{
2522 .user_data = 0x66666666,
2523 .res = 0,
2524 .flags = 0,
2525 }, cqe_timeout);
2526}
2527
2528test "timeout_remove" {
2529 if (!is_linux) return error.SkipZigTest;
2530
2531 var ring = IoUring.init(2, 0) catch |err| switch (err) {
2532 error.SystemOutdated => return error.SkipZigTest,
2533 error.PermissionDenied => return error.SkipZigTest,
2534 else => return err,
2535 };
2536 defer ring.deinit();
2537
2538 const ts: linux.kernel_timespec = .{ .sec = 3, .nsec = 0 };
2539 const sqe_timeout = try ring.timeout(0x88888888, &ts, 0, 0);
2540 try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe_timeout.opcode);
2541 try testing.expectEqual(@as(u64, 0x88888888), sqe_timeout.user_data);
2542
2543 const sqe_timeout_remove = try ring.timeout_remove(0x99999999, 0x88888888, 0);
2544 try testing.expectEqual(linux.IORING_OP.TIMEOUT_REMOVE, sqe_timeout_remove.opcode);
2545 try testing.expectEqual(@as(u64, 0x88888888), sqe_timeout_remove.addr);
2546 try testing.expectEqual(@as(u64, 0x99999999), sqe_timeout_remove.user_data);
2547
2548 try testing.expectEqual(@as(u32, 2), try ring.submit());
2549
2550 // The order in which the CQE arrive is not clearly documented and it changed with kernel 5.18:
2551 // * kernel 5.10 gives user data 0x88888888 first, 0x99999999 second
2552 // * kernel 5.18 gives user data 0x99999999 first, 0x88888888 second
2553
2554 var cqes: [2]linux.io_uring_cqe = undefined;
2555 cqes[0] = try ring.copy_cqe();
2556 cqes[1] = try ring.copy_cqe();
2557
2558 for (cqes) |cqe| {
2559 // IORING_OP_TIMEOUT_REMOVE is not supported by this kernel version:
2560 // Timeout remove operations set the fd to -1, which results in EBADF before EINVAL.
2561 // We use IORING_FEAT_RW_CUR_POS as a safety check here to make sure we are at least pre-5.6.
2562 // We don't want to skip this test for newer kernels.
2563 if (cqe.user_data == 0x99999999 and
2564 cqe.err() == .BADF and
2565 (ring.features & linux.IORING_FEAT_RW_CUR_POS) == 0)
2566 {
2567 return error.SkipZigTest;
2568 }
2569
2570 try testing.expect(cqe.user_data == 0x88888888 or cqe.user_data == 0x99999999);
2571
2572 if (cqe.user_data == 0x88888888) {
2573 try testing.expectEqual(linux.io_uring_cqe{
2574 .user_data = 0x88888888,
2575 .res = -@as(i32, @intFromEnum(linux.E.CANCELED)),
2576 .flags = 0,
2577 }, cqe);
2578 } else if (cqe.user_data == 0x99999999) {
2579 try testing.expectEqual(linux.io_uring_cqe{
2580 .user_data = 0x99999999,
2581 .res = 0,
2582 .flags = 0,
2583 }, cqe);
2584 }
2585 }
2586}
2587
2588test "accept/connect/recv/link_timeout" {
2589 if (!is_linux) return error.SkipZigTest;
2590
2591 const io = testing.io;
2592
2593 var ring = IoUring.init(16, 0) catch |err| switch (err) {
2594 error.SystemOutdated => return error.SkipZigTest,
2595 error.PermissionDenied => return error.SkipZigTest,
2596 else => return err,
2597 };
2598 defer ring.deinit();
2599
2600 const socket_test_harness = try createSocketTestHarness(&ring);
2601 defer socket_test_harness.close(io);
2602
2603 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
2604
2605 const sqe_recv = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
2606 sqe_recv.flags |= linux.IOSQE_IO_LINK;
2607
2608 const ts = linux.kernel_timespec{ .sec = 0, .nsec = 1000000 };
2609 _ = try ring.link_timeout(0x22222222, &ts, 0);
2610
2611 const nr_wait = try ring.submit();
2612 try testing.expectEqual(@as(u32, 2), nr_wait);
2613
2614 var i: usize = 0;
2615 while (i < nr_wait) : (i += 1) {
2616 const cqe = try ring.copy_cqe();
2617 switch (cqe.user_data) {
2618 0xffffffff => {
2619 if (cqe.res != -@as(i32, @intFromEnum(linux.E.INTR)) and
2620 cqe.res != -@as(i32, @intFromEnum(linux.E.CANCELED)))
2621 {
2622 std.debug.print("Req 0x{x} got {d}\n", .{ cqe.user_data, cqe.res });
2623 try testing.expect(false);
2624 }
2625 },
2626 0x22222222 => {
2627 if (cqe.res != -@as(i32, @intFromEnum(linux.E.ALREADY)) and
2628 cqe.res != -@as(i32, @intFromEnum(linux.E.TIME)))
2629 {
2630 std.debug.print("Req 0x{x} got {d}\n", .{ cqe.user_data, cqe.res });
2631 try testing.expect(false);
2632 }
2633 },
2634 else => @panic("should not happen"),
2635 }
2636 }
2637}
2638
2639test "fallocate" {
2640 if (!is_linux) return error.SkipZigTest;
2641
2642 const io = testing.io;
2643
2644 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2645 error.SystemOutdated => return error.SkipZigTest,
2646 error.PermissionDenied => return error.SkipZigTest,
2647 else => return err,
2648 };
2649 defer ring.deinit();
2650
2651 var tmp = std.testing.tmpDir(.{});
2652 defer tmp.cleanup();
2653
2654 const path = "test_io_uring_fallocate";
2655 const file = try tmp.dir.createFile(io, path, .{ .truncate = true, .mode = 0o666 });
2656 defer file.close(io);
2657
2658 try testing.expectEqual(@as(u64, 0), (try file.stat(io)).size);
2659
2660 const len: u64 = 65536;
2661 const sqe = try ring.fallocate(0xaaaaaaaa, file.handle, 0, 0, len);
2662 try testing.expectEqual(linux.IORING_OP.FALLOCATE, sqe.opcode);
2663 try testing.expectEqual(file.handle, sqe.fd);
2664 try testing.expectEqual(@as(u32, 1), try ring.submit());
2665
2666 const cqe = try ring.copy_cqe();
2667 switch (cqe.err()) {
2668 .SUCCESS => {},
2669 // This kernel's io_uring does not yet implement fallocate():
2670 .INVAL => return error.SkipZigTest,
2671 // This kernel does not implement fallocate():
2672 .NOSYS => return error.SkipZigTest,
2673 // The filesystem containing the file referred to by fd does not support this operation;
2674 // or the mode is not supported by the filesystem containing the file referred to by fd:
2675 .OPNOTSUPP => return error.SkipZigTest,
2676 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2677 }
2678 try testing.expectEqual(linux.io_uring_cqe{
2679 .user_data = 0xaaaaaaaa,
2680 .res = 0,
2681 .flags = 0,
2682 }, cqe);
2683
2684 try testing.expectEqual(len, (try file.stat(io)).size);
2685}
2686
2687test "statx" {
2688 if (!is_linux) return error.SkipZigTest;
2689
2690 const io = testing.io;
2691
2692 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2693 error.SystemOutdated => return error.SkipZigTest,
2694 error.PermissionDenied => return error.SkipZigTest,
2695 else => return err,
2696 };
2697 defer ring.deinit();
2698
2699 var tmp = std.testing.tmpDir(.{});
2700 defer tmp.cleanup();
2701 const path = "test_io_uring_statx";
2702 const file = try tmp.dir.createFile(io, path, .{ .truncate = true, .mode = 0o666 });
2703 defer file.close(io);
2704
2705 try testing.expectEqual(@as(u64, 0), (try file.stat(io)).size);
2706
2707 try file.writeAll("foobar");
2708
2709 var buf: linux.Statx = undefined;
2710 const sqe = try ring.statx(
2711 0xaaaaaaaa,
2712 tmp.dir.fd,
2713 path,
2714 0,
2715 .{ .SIZE = true },
2716 &buf,
2717 );
2718 try testing.expectEqual(linux.IORING_OP.STATX, sqe.opcode);
2719 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
2720 try testing.expectEqual(@as(u32, 1), try ring.submit());
2721
2722 const cqe = try ring.copy_cqe();
2723 switch (cqe.err()) {
2724 .SUCCESS => {},
2725 // This kernel's io_uring does not yet implement statx():
2726 .INVAL => return error.SkipZigTest,
2727 // This kernel does not implement statx():
2728 .NOSYS => return error.SkipZigTest,
2729 // The filesystem containing the file referred to by fd does not support this operation;
2730 // or the mode is not supported by the filesystem containing the file referred to by fd:
2731 .OPNOTSUPP => return error.SkipZigTest,
2732 // not supported on older kernels (5.4)
2733 .BADF => return error.SkipZigTest,
2734 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2735 }
2736 try testing.expectEqual(linux.io_uring_cqe{
2737 .user_data = 0xaaaaaaaa,
2738 .res = 0,
2739 .flags = 0,
2740 }, cqe);
2741
2742 try testing.expect(buf.mask.SIZE);
2743 try testing.expectEqual(@as(u64, 6), buf.size);
2744}
2745
2746test "accept/connect/recv/cancel" {
2747 if (!is_linux) return error.SkipZigTest;
2748
2749 const io = testing.io;
2750
2751 var ring = IoUring.init(16, 0) catch |err| switch (err) {
2752 error.SystemOutdated => return error.SkipZigTest,
2753 error.PermissionDenied => return error.SkipZigTest,
2754 else => return err,
2755 };
2756 defer ring.deinit();
2757
2758 const socket_test_harness = try createSocketTestHarness(&ring);
2759 defer socket_test_harness.close(io);
2760
2761 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
2762
2763 _ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
2764 try testing.expectEqual(@as(u32, 1), try ring.submit());
2765
2766 const sqe_cancel = try ring.cancel(0x99999999, 0xffffffff, 0);
2767 try testing.expectEqual(linux.IORING_OP.ASYNC_CANCEL, sqe_cancel.opcode);
2768 try testing.expectEqual(@as(u64, 0xffffffff), sqe_cancel.addr);
2769 try testing.expectEqual(@as(u64, 0x99999999), sqe_cancel.user_data);
2770 try testing.expectEqual(@as(u32, 1), try ring.submit());
2771
2772 var cqe_recv = try ring.copy_cqe();
2773 if (cqe_recv.err() == .INVAL) return error.SkipZigTest;
2774 var cqe_cancel = try ring.copy_cqe();
2775 if (cqe_cancel.err() == .INVAL) return error.SkipZigTest;
2776
2777 // The recv/cancel CQEs may arrive in any order, the recv CQE will sometimes come first:
2778 if (cqe_recv.user_data == 0x99999999 and cqe_cancel.user_data == 0xffffffff) {
2779 const a = cqe_recv;
2780 const b = cqe_cancel;
2781 cqe_recv = b;
2782 cqe_cancel = a;
2783 }
2784
2785 try testing.expectEqual(linux.io_uring_cqe{
2786 .user_data = 0xffffffff,
2787 .res = -@as(i32, @intFromEnum(linux.E.CANCELED)),
2788 .flags = 0,
2789 }, cqe_recv);
2790
2791 try testing.expectEqual(linux.io_uring_cqe{
2792 .user_data = 0x99999999,
2793 .res = 0,
2794 .flags = 0,
2795 }, cqe_cancel);
2796}
2797
2798test "register_files_update" {
2799 if (!is_linux) return error.SkipZigTest;
2800
2801 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2802 error.SystemOutdated => return error.SkipZigTest,
2803 error.PermissionDenied => return error.SkipZigTest,
2804 else => return err,
2805 };
2806 defer ring.deinit();
2807
2808 const fd = try posix.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
2809 defer posix.close(fd);
2810
2811 var registered_fds = [_]linux.fd_t{0} ** 2;
2812 const fd_index = 0;
2813 const fd_index2 = 1;
2814 registered_fds[fd_index] = fd;
2815 registered_fds[fd_index2] = -1;
2816
2817 ring.register_files(registered_fds[0..]) catch |err| switch (err) {
2818 // Happens when the kernel doesn't support sparse entry (-1) in the file descriptors array.
2819 error.FileDescriptorInvalid => return error.SkipZigTest,
2820 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2821 };
2822
2823 // Test IORING_REGISTER_FILES_UPDATE
2824 // Only available since Linux 5.5
2825
2826 const fd2 = try posix.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
2827 defer posix.close(fd2);
2828
2829 registered_fds[fd_index] = fd2;
2830 registered_fds[fd_index2] = -1;
2831 try ring.register_files_update(0, registered_fds[0..]);
2832
2833 var buffer = [_]u8{42} ** 128;
2834 {
2835 const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
2836 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2837 sqe.flags |= linux.IOSQE_FIXED_FILE;
2838
2839 try testing.expectEqual(@as(u32, 1), try ring.submit());
2840 try testing.expectEqual(linux.io_uring_cqe{
2841 .user_data = 0xcccccccc,
2842 .res = buffer.len,
2843 .flags = 0,
2844 }, try ring.copy_cqe());
2845 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
2846 }
2847
2848 // Test with a non-zero offset
2849
2850 registered_fds[fd_index] = -1;
2851 registered_fds[fd_index2] = -1;
2852 try ring.register_files_update(1, registered_fds[1..]);
2853
2854 {
2855 // Next read should still work since fd_index in the registered file descriptors hasn't been updated yet.
2856 const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
2857 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2858 sqe.flags |= linux.IOSQE_FIXED_FILE;
2859
2860 try testing.expectEqual(@as(u32, 1), try ring.submit());
2861 try testing.expectEqual(linux.io_uring_cqe{
2862 .user_data = 0xcccccccc,
2863 .res = buffer.len,
2864 .flags = 0,
2865 }, try ring.copy_cqe());
2866 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
2867 }
2868
2869 try ring.register_files_update(0, registered_fds[0..]);
2870
2871 {
2872 // Now this should fail since both fds are sparse (-1)
2873 const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
2874 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2875 sqe.flags |= linux.IOSQE_FIXED_FILE;
2876
2877 try testing.expectEqual(@as(u32, 1), try ring.submit());
2878 const cqe = try ring.copy_cqe();
2879 try testing.expectEqual(linux.E.BADF, cqe.err());
2880 }
2881
2882 try ring.unregister_files();
2883}
2884
2885test "shutdown" {
2886 if (!is_linux) return error.SkipZigTest;
2887
2888 var ring = IoUring.init(16, 0) catch |err| switch (err) {
2889 error.SystemOutdated => return error.SkipZigTest,
2890 error.PermissionDenied => return error.SkipZigTest,
2891 else => return err,
2892 };
2893 defer ring.deinit();
2894
2895 var address: linux.sockaddr.in = .{
2896 .port = 0,
2897 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
2898 };
2899
2900 // Socket bound, expect shutdown to work
2901 {
2902 const server = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
2903 defer posix.close(server);
2904 try posix.setsockopt(server, posix.SOL.SOCKET, posix.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
2905 try posix.bind(server, addrAny(&address), @sizeOf(linux.sockaddr.in));
2906 try posix.listen(server, 1);
2907
2908 // set address to the OS-chosen IP/port.
2909 var slen: posix.socklen_t = @sizeOf(linux.sockaddr.in);
2910 try posix.getsockname(server, addrAny(&address), &slen);
2911
2912 const shutdown_sqe = try ring.shutdown(0x445445445, server, linux.SHUT.RD);
2913 try testing.expectEqual(linux.IORING_OP.SHUTDOWN, shutdown_sqe.opcode);
2914 try testing.expectEqual(@as(i32, server), shutdown_sqe.fd);
2915
2916 try testing.expectEqual(@as(u32, 1), try ring.submit());
2917
2918 const cqe = try ring.copy_cqe();
2919 switch (cqe.err()) {
2920 .SUCCESS => {},
2921 // This kernel's io_uring does not yet implement shutdown (kernel version < 5.11)
2922 .INVAL => return error.SkipZigTest,
2923 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2924 }
2925
2926 try testing.expectEqual(linux.io_uring_cqe{
2927 .user_data = 0x445445445,
2928 .res = 0,
2929 .flags = 0,
2930 }, cqe);
2931 }
2932
2933 // Socket not bound, expect to fail with ENOTCONN
2934 {
2935 const server = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
2936 defer posix.close(server);
2937
2938 const shutdown_sqe = ring.shutdown(0x445445445, server, linux.SHUT.RD) catch |err| switch (err) {
2939 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2940 };
2941 try testing.expectEqual(linux.IORING_OP.SHUTDOWN, shutdown_sqe.opcode);
2942 try testing.expectEqual(@as(i32, server), shutdown_sqe.fd);
2943
2944 try testing.expectEqual(@as(u32, 1), try ring.submit());
2945
2946 const cqe = try ring.copy_cqe();
2947 try testing.expectEqual(@as(u64, 0x445445445), cqe.user_data);
2948 try testing.expectEqual(linux.E.NOTCONN, cqe.err());
2949 }
2950}
2951
2952test "renameat" {
2953 if (!is_linux) return error.SkipZigTest;
2954
2955 const io = testing.io;
2956
2957 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2958 error.SystemOutdated => return error.SkipZigTest,
2959 error.PermissionDenied => return error.SkipZigTest,
2960 else => return err,
2961 };
2962 defer ring.deinit();
2963
2964 const old_path = "test_io_uring_renameat_old";
2965 const new_path = "test_io_uring_renameat_new";
2966
2967 var tmp = std.testing.tmpDir(.{});
2968 defer tmp.cleanup();
2969
2970 // Write old file with data
2971
2972 const old_file = try tmp.dir.createFile(io, old_path, .{ .truncate = true, .mode = 0o666 });
2973 defer old_file.close(io);
2974 try old_file.writeAll("hello");
2975
2976 // Submit renameat
2977
2978 const sqe = try ring.renameat(
2979 0x12121212,
2980 tmp.dir.fd,
2981 old_path,
2982 tmp.dir.fd,
2983 new_path,
2984 0,
2985 );
2986 try testing.expectEqual(linux.IORING_OP.RENAMEAT, sqe.opcode);
2987 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
2988 try testing.expectEqual(@as(i32, tmp.dir.fd), @as(i32, @bitCast(sqe.len)));
2989 try testing.expectEqual(@as(u32, 1), try ring.submit());
2990
2991 const cqe = try ring.copy_cqe();
2992 switch (cqe.err()) {
2993 .SUCCESS => {},
2994 // This kernel's io_uring does not yet implement renameat (kernel version < 5.11)
2995 .BADF, .INVAL => return error.SkipZigTest,
2996 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2997 }
2998 try testing.expectEqual(linux.io_uring_cqe{
2999 .user_data = 0x12121212,
3000 .res = 0,
3001 .flags = 0,
3002 }, cqe);
3003
3004 // Validate that the old file doesn't exist anymore
3005 try testing.expectError(error.FileNotFound, tmp.dir.openFile(io, old_path, .{}));
3006
3007 // Validate that the new file exists with the proper content
3008 var new_file_data: [16]u8 = undefined;
3009 try testing.expectEqualStrings("hello", try tmp.dir.readFile(new_path, &new_file_data));
3010}
3011
3012test "unlinkat" {
3013 if (!is_linux) return error.SkipZigTest;
3014
3015 const io = testing.io;
3016
3017 var ring = IoUring.init(1, 0) catch |err| switch (err) {
3018 error.SystemOutdated => return error.SkipZigTest,
3019 error.PermissionDenied => return error.SkipZigTest,
3020 else => return err,
3021 };
3022 defer ring.deinit();
3023
3024 const path = "test_io_uring_unlinkat";
3025
3026 var tmp = std.testing.tmpDir(.{});
3027 defer tmp.cleanup();
3028
3029 // Write old file with data
3030
3031 const file = try tmp.dir.createFile(io, path, .{ .truncate = true, .mode = 0o666 });
3032 defer file.close(io);
3033
3034 // Submit unlinkat
3035
3036 const sqe = try ring.unlinkat(
3037 0x12121212,
3038 tmp.dir.fd,
3039 path,
3040 0,
3041 );
3042 try testing.expectEqual(linux.IORING_OP.UNLINKAT, sqe.opcode);
3043 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
3044 try testing.expectEqual(@as(u32, 1), try ring.submit());
3045
3046 const cqe = try ring.copy_cqe();
3047 switch (cqe.err()) {
3048 .SUCCESS => {},
3049 // This kernel's io_uring does not yet implement unlinkat (kernel version < 5.11)
3050 .BADF, .INVAL => return error.SkipZigTest,
3051 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3052 }
3053 try testing.expectEqual(linux.io_uring_cqe{
3054 .user_data = 0x12121212,
3055 .res = 0,
3056 .flags = 0,
3057 }, cqe);
3058
3059 // Validate that the file doesn't exist anymore
3060 _ = tmp.dir.openFile(io, path, .{}) catch |err| switch (err) {
3061 error.FileNotFound => {},
3062 else => std.debug.panic("unexpected error: {}", .{err}),
3063 };
3064}
3065
3066test "mkdirat" {
3067 if (!is_linux) return error.SkipZigTest;
3068
3069 const io = testing.io;
3070
3071 var ring = IoUring.init(1, 0) catch |err| switch (err) {
3072 error.SystemOutdated => return error.SkipZigTest,
3073 error.PermissionDenied => return error.SkipZigTest,
3074 else => return err,
3075 };
3076 defer ring.deinit();
3077
3078 var tmp = std.testing.tmpDir(.{});
3079 defer tmp.cleanup();
3080
3081 const path = "test_io_uring_mkdirat";
3082
3083 // Submit mkdirat
3084
3085 const sqe = try ring.mkdirat(
3086 0x12121212,
3087 tmp.dir.fd,
3088 path,
3089 0o0755,
3090 );
3091 try testing.expectEqual(linux.IORING_OP.MKDIRAT, sqe.opcode);
3092 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
3093 try testing.expectEqual(@as(u32, 1), try ring.submit());
3094
3095 const cqe = try ring.copy_cqe();
3096 switch (cqe.err()) {
3097 .SUCCESS => {},
3098 // This kernel's io_uring does not yet implement mkdirat (kernel version < 5.15)
3099 .BADF, .INVAL => return error.SkipZigTest,
3100 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3101 }
3102 try testing.expectEqual(linux.io_uring_cqe{
3103 .user_data = 0x12121212,
3104 .res = 0,
3105 .flags = 0,
3106 }, cqe);
3107
3108 // Validate that the directory exist
3109 _ = try tmp.dir.openDir(io, path, .{});
3110}
3111
3112test "symlinkat" {
3113 if (!is_linux) return error.SkipZigTest;
3114
3115 const io = testing.io;
3116
3117 var ring = IoUring.init(1, 0) catch |err| switch (err) {
3118 error.SystemOutdated => return error.SkipZigTest,
3119 error.PermissionDenied => return error.SkipZigTest,
3120 else => return err,
3121 };
3122 defer ring.deinit();
3123
3124 var tmp = std.testing.tmpDir(.{});
3125 defer tmp.cleanup();
3126
3127 const path = "test_io_uring_symlinkat";
3128 const link_path = "test_io_uring_symlinkat_link";
3129
3130 const file = try tmp.dir.createFile(io, path, .{ .truncate = true, .mode = 0o666 });
3131 defer file.close(io);
3132
3133 // Submit symlinkat
3134
3135 const sqe = try ring.symlinkat(
3136 0x12121212,
3137 path,
3138 tmp.dir.fd,
3139 link_path,
3140 );
3141 try testing.expectEqual(linux.IORING_OP.SYMLINKAT, sqe.opcode);
3142 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
3143 try testing.expectEqual(@as(u32, 1), try ring.submit());
3144
3145 const cqe = try ring.copy_cqe();
3146 switch (cqe.err()) {
3147 .SUCCESS => {},
3148 // This kernel's io_uring does not yet implement symlinkat (kernel version < 5.15)
3149 .BADF, .INVAL => return error.SkipZigTest,
3150 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3151 }
3152 try testing.expectEqual(linux.io_uring_cqe{
3153 .user_data = 0x12121212,
3154 .res = 0,
3155 .flags = 0,
3156 }, cqe);
3157
3158 // Validate that the symlink exist
3159 _ = try tmp.dir.openFile(io, link_path, .{});
3160}
3161
3162test "linkat" {
3163 if (!is_linux) return error.SkipZigTest;
3164
3165 const io = testing.io;
3166
3167 var ring = IoUring.init(1, 0) catch |err| switch (err) {
3168 error.SystemOutdated => return error.SkipZigTest,
3169 error.PermissionDenied => return error.SkipZigTest,
3170 else => return err,
3171 };
3172 defer ring.deinit();
3173
3174 var tmp = std.testing.tmpDir(.{});
3175 defer tmp.cleanup();
3176
3177 const first_path = "test_io_uring_linkat_first";
3178 const second_path = "test_io_uring_linkat_second";
3179
3180 // Write file with data
3181
3182 const first_file = try tmp.dir.createFile(io, first_path, .{ .truncate = true, .mode = 0o666 });
3183 defer first_file.close(io);
3184 try first_file.writeAll("hello");
3185
3186 // Submit linkat
3187
3188 const sqe = try ring.linkat(
3189 0x12121212,
3190 tmp.dir.fd,
3191 first_path,
3192 tmp.dir.fd,
3193 second_path,
3194 0,
3195 );
3196 try testing.expectEqual(linux.IORING_OP.LINKAT, sqe.opcode);
3197 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
3198 try testing.expectEqual(@as(i32, tmp.dir.fd), @as(i32, @bitCast(sqe.len)));
3199 try testing.expectEqual(@as(u32, 1), try ring.submit());
3200
3201 const cqe = try ring.copy_cqe();
3202 switch (cqe.err()) {
3203 .SUCCESS => {},
3204 // This kernel's io_uring does not yet implement linkat (kernel version < 5.15)
3205 .BADF, .INVAL => return error.SkipZigTest,
3206 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3207 }
3208 try testing.expectEqual(linux.io_uring_cqe{
3209 .user_data = 0x12121212,
3210 .res = 0,
3211 .flags = 0,
3212 }, cqe);
3213
3214 // Validate the second file
3215 var second_file_data: [16]u8 = undefined;
3216 try testing.expectEqualStrings("hello", try tmp.dir.readFile(second_path, &second_file_data));
3217}
3218
3219test "provide_buffers: read" {
3220 if (!is_linux) return error.SkipZigTest;
3221
3222 var ring = IoUring.init(1, 0) catch |err| switch (err) {
3223 error.SystemOutdated => return error.SkipZigTest,
3224 error.PermissionDenied => return error.SkipZigTest,
3225 else => return err,
3226 };
3227 defer ring.deinit();
3228
3229 const fd = try posix.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
3230 defer posix.close(fd);
3231
3232 const group_id = 1337;
3233 const buffer_id = 0;
3234
3235 const buffer_len = 128;
3236
3237 var buffers: [4][buffer_len]u8 = undefined;
3238
3239 // Provide 4 buffers
3240
3241 {
3242 const sqe = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
3243 try testing.expectEqual(linux.IORING_OP.PROVIDE_BUFFERS, sqe.opcode);
3244 try testing.expectEqual(@as(i32, buffers.len), sqe.fd);
3245 try testing.expectEqual(@as(u32, buffers[0].len), sqe.len);
3246 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3247 try testing.expectEqual(@as(u32, 1), try ring.submit());
3248
3249 const cqe = try ring.copy_cqe();
3250 switch (cqe.err()) {
3251 // Happens when the kernel is < 5.7
3252 .INVAL, .BADF => return error.SkipZigTest,
3253 .SUCCESS => {},
3254 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3255 }
3256 try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
3257 }
3258
3259 // Do 4 reads which should consume all buffers
3260
3261 var i: usize = 0;
3262 while (i < buffers.len) : (i += 1) {
3263 const sqe = try ring.read(0xdededede, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3264 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
3265 try testing.expectEqual(@as(i32, fd), sqe.fd);
3266 try testing.expectEqual(@as(u64, 0), sqe.addr);
3267 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3268 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3269 try testing.expectEqual(@as(u32, 1), try ring.submit());
3270
3271 const cqe = try ring.copy_cqe();
3272 switch (cqe.err()) {
3273 .SUCCESS => {},
3274 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3275 }
3276
3277 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3278 const used_buffer_id = cqe.flags >> 16;
3279 try testing.expect(used_buffer_id >= 0 and used_buffer_id <= 3);
3280 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3281
3282 try testing.expectEqual(@as(u64, 0xdededede), cqe.user_data);
3283 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
3284 }
3285
3286 // This read should fail
3287
3288 {
3289 const sqe = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3290 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
3291 try testing.expectEqual(@as(i32, fd), sqe.fd);
3292 try testing.expectEqual(@as(u64, 0), sqe.addr);
3293 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3294 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3295 try testing.expectEqual(@as(u32, 1), try ring.submit());
3296
3297 const cqe = try ring.copy_cqe();
3298 switch (cqe.err()) {
3299 // Expected
3300 .NOBUFS => {},
3301 .SUCCESS => std.debug.panic("unexpected success", .{}),
3302 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3303 }
3304 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3305 }
3306
3307 // Provide 1 buffer again
3308
3309 // Deliberately put something we don't expect in the buffers
3310 @memset(mem.sliceAsBytes(&buffers), 42);
3311
3312 const reprovided_buffer_id = 2;
3313
3314 {
3315 _ = try ring.provide_buffers(0xabababab, @as([*]u8, @ptrCast(&buffers[reprovided_buffer_id])), buffer_len, 1, group_id, reprovided_buffer_id);
3316 try testing.expectEqual(@as(u32, 1), try ring.submit());
3317
3318 const cqe = try ring.copy_cqe();
3319 switch (cqe.err()) {
3320 .SUCCESS => {},
3321 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3322 }
3323 }
3324
3325 // Final read which should work
3326
3327 {
3328 const sqe = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3329 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
3330 try testing.expectEqual(@as(i32, fd), sqe.fd);
3331 try testing.expectEqual(@as(u64, 0), sqe.addr);
3332 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3333 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3334 try testing.expectEqual(@as(u32, 1), try ring.submit());
3335
3336 const cqe = try ring.copy_cqe();
3337 switch (cqe.err()) {
3338 .SUCCESS => {},
3339 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3340 }
3341
3342 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3343 const used_buffer_id = cqe.flags >> 16;
3344 try testing.expectEqual(used_buffer_id, reprovided_buffer_id);
3345 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3346 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3347 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
3348 }
3349}
3350
3351test "remove_buffers" {
3352 if (!is_linux) return error.SkipZigTest;
3353
3354 var ring = IoUring.init(1, 0) catch |err| switch (err) {
3355 error.SystemOutdated => return error.SkipZigTest,
3356 error.PermissionDenied => return error.SkipZigTest,
3357 else => return err,
3358 };
3359 defer ring.deinit();
3360
3361 const fd = try posix.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
3362 defer posix.close(fd);
3363
3364 const group_id = 1337;
3365 const buffer_id = 0;
3366
3367 const buffer_len = 128;
3368
3369 var buffers: [4][buffer_len]u8 = undefined;
3370
3371 // Provide 4 buffers
3372
3373 {
3374 _ = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
3375 try testing.expectEqual(@as(u32, 1), try ring.submit());
3376
3377 const cqe = try ring.copy_cqe();
3378 switch (cqe.err()) {
3379 .INVAL, .BADF => return error.SkipZigTest,
3380 .SUCCESS => {},
3381 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3382 }
3383 try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
3384 }
3385
3386 // Remove 3 buffers
3387
3388 {
3389 const sqe = try ring.remove_buffers(0xbababababa, 3, group_id);
3390 try testing.expectEqual(linux.IORING_OP.REMOVE_BUFFERS, sqe.opcode);
3391 try testing.expectEqual(@as(i32, 3), sqe.fd);
3392 try testing.expectEqual(@as(u64, 0), sqe.addr);
3393 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3394 try testing.expectEqual(@as(u32, 1), try ring.submit());
3395
3396 const cqe = try ring.copy_cqe();
3397 switch (cqe.err()) {
3398 .SUCCESS => {},
3399 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3400 }
3401 try testing.expectEqual(@as(u64, 0xbababababa), cqe.user_data);
3402 }
3403
3404 // This read should work
3405
3406 {
3407 _ = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3408 try testing.expectEqual(@as(u32, 1), try ring.submit());
3409
3410 const cqe = try ring.copy_cqe();
3411 switch (cqe.err()) {
3412 .SUCCESS => {},
3413 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3414 }
3415
3416 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3417 const used_buffer_id = cqe.flags >> 16;
3418 try testing.expect(used_buffer_id >= 0 and used_buffer_id < 4);
3419 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3420 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3421 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
3422 }
3423
3424 // Final read should _not_ work
3425
3426 {
3427 _ = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3428 try testing.expectEqual(@as(u32, 1), try ring.submit());
3429
3430 const cqe = try ring.copy_cqe();
3431 switch (cqe.err()) {
3432 // Expected
3433 .NOBUFS => {},
3434 .SUCCESS => std.debug.panic("unexpected success", .{}),
3435 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3436 }
3437 }
3438}
3439
3440test "provide_buffers: accept/connect/send/recv" {
3441 if (!is_linux) return error.SkipZigTest;
3442
3443 const io = testing.io;
3444
3445 var ring = IoUring.init(16, 0) catch |err| switch (err) {
3446 error.SystemOutdated => return error.SkipZigTest,
3447 error.PermissionDenied => return error.SkipZigTest,
3448 else => return err,
3449 };
3450 defer ring.deinit();
3451
3452 const group_id = 1337;
3453 const buffer_id = 0;
3454
3455 const buffer_len = 128;
3456 var buffers: [4][buffer_len]u8 = undefined;
3457
3458 // Provide 4 buffers
3459
3460 {
3461 const sqe = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
3462 try testing.expectEqual(linux.IORING_OP.PROVIDE_BUFFERS, sqe.opcode);
3463 try testing.expectEqual(@as(i32, buffers.len), sqe.fd);
3464 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3465 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3466 try testing.expectEqual(@as(u32, 1), try ring.submit());
3467
3468 const cqe = try ring.copy_cqe();
3469 switch (cqe.err()) {
3470 // Happens when the kernel is < 5.7
3471 .INVAL => return error.SkipZigTest,
3472 // Happens on the kernel 5.4
3473 .BADF => return error.SkipZigTest,
3474 .SUCCESS => {},
3475 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3476 }
3477 try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
3478 }
3479
3480 const socket_test_harness = try createSocketTestHarness(&ring);
3481 defer socket_test_harness.close(io);
3482
3483 // Do 4 send on the socket
3484
3485 {
3486 var i: usize = 0;
3487 while (i < buffers.len) : (i += 1) {
3488 _ = try ring.send(0xdeaddead, socket_test_harness.server, &([_]u8{'z'} ** buffer_len), 0);
3489 try testing.expectEqual(@as(u32, 1), try ring.submit());
3490 }
3491
3492 var cqes: [4]linux.io_uring_cqe = undefined;
3493 try testing.expectEqual(@as(u32, 4), try ring.copy_cqes(&cqes, 4));
3494 }
3495
3496 // Do 4 recv which should consume all buffers
3497
3498 // Deliberately put something we don't expect in the buffers
3499 @memset(mem.sliceAsBytes(&buffers), 1);
3500
3501 var i: usize = 0;
3502 while (i < buffers.len) : (i += 1) {
3503 const sqe = try ring.recv(0xdededede, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3504 try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
3505 try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
3506 try testing.expectEqual(@as(u64, 0), sqe.addr);
3507 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3508 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3509 try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
3510 try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
3511 try testing.expectEqual(@as(u32, 1), try ring.submit());
3512
3513 const cqe = try ring.copy_cqe();
3514 switch (cqe.err()) {
3515 .SUCCESS => {},
3516 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3517 }
3518
3519 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3520 const used_buffer_id = cqe.flags >> 16;
3521 try testing.expect(used_buffer_id >= 0 and used_buffer_id <= 3);
3522 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3523
3524 try testing.expectEqual(@as(u64, 0xdededede), cqe.user_data);
3525 const buffer = buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))];
3526 try testing.expectEqualSlices(u8, &([_]u8{'z'} ** buffer_len), buffer);
3527 }
3528
3529 // This recv should fail
3530
3531 {
3532 const sqe = try ring.recv(0xdfdfdfdf, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3533 try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
3534 try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
3535 try testing.expectEqual(@as(u64, 0), sqe.addr);
3536 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3537 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3538 try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
3539 try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
3540 try testing.expectEqual(@as(u32, 1), try ring.submit());
3541
3542 const cqe = try ring.copy_cqe();
3543 switch (cqe.err()) {
3544 // Expected
3545 .NOBUFS => {},
3546 .SUCCESS => std.debug.panic("unexpected success", .{}),
3547 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3548 }
3549 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3550 }
3551
3552 // Provide 1 buffer again
3553
3554 const reprovided_buffer_id = 2;
3555
3556 {
3557 _ = try ring.provide_buffers(0xabababab, @as([*]u8, @ptrCast(&buffers[reprovided_buffer_id])), buffer_len, 1, group_id, reprovided_buffer_id);
3558 try testing.expectEqual(@as(u32, 1), try ring.submit());
3559
3560 const cqe = try ring.copy_cqe();
3561 switch (cqe.err()) {
3562 .SUCCESS => {},
3563 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3564 }
3565 }
3566
3567 // Redo 1 send on the server socket
3568
3569 {
3570 _ = try ring.send(0xdeaddead, socket_test_harness.server, &([_]u8{'w'} ** buffer_len), 0);
3571 try testing.expectEqual(@as(u32, 1), try ring.submit());
3572
3573 _ = try ring.copy_cqe();
3574 }
3575
3576 // Final recv which should work
3577
3578 // Deliberately put something we don't expect in the buffers
3579 @memset(mem.sliceAsBytes(&buffers), 1);
3580
3581 {
3582 const sqe = try ring.recv(0xdfdfdfdf, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3583 try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
3584 try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
3585 try testing.expectEqual(@as(u64, 0), sqe.addr);
3586 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3587 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3588 try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
3589 try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
3590 try testing.expectEqual(@as(u32, 1), try ring.submit());
3591
3592 const cqe = try ring.copy_cqe();
3593 switch (cqe.err()) {
3594 .SUCCESS => {},
3595 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3596 }
3597
3598 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3599 const used_buffer_id = cqe.flags >> 16;
3600 try testing.expectEqual(used_buffer_id, reprovided_buffer_id);
3601 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3602 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3603 const buffer = buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))];
3604 try testing.expectEqualSlices(u8, &([_]u8{'w'} ** buffer_len), buffer);
3605 }
3606}
3607
3608/// Used for testing server/client interactions.
3609const SocketTestHarness = struct {
3610 listener: posix.socket_t,
3611 server: posix.socket_t,
3612 client: posix.socket_t,
3613
3614 fn close(self: SocketTestHarness) void {
3615 posix.close(self.client);
3616 posix.close(self.listener);
3617 }
3618};
3619
3620fn createSocketTestHarness(ring: *IoUring) !SocketTestHarness {
3621 // Create a TCP server socket
3622 var address: linux.sockaddr.in = .{
3623 .port = 0,
3624 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
3625 };
3626 const listener_socket = try createListenerSocket(&address);
3627 errdefer posix.close(listener_socket);
3628
3629 // Submit 1 accept
3630 var accept_addr: posix.sockaddr = undefined;
3631 var accept_addr_len: posix.socklen_t = @sizeOf(@TypeOf(accept_addr));
3632 _ = try ring.accept(0xaaaaaaaa, listener_socket, &accept_addr, &accept_addr_len, 0);
3633
3634 // Create a TCP client socket
3635 const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
3636 errdefer posix.close(client);
3637 _ = try ring.connect(0xcccccccc, client, addrAny(&address), @sizeOf(linux.sockaddr.in));
3638
3639 try testing.expectEqual(@as(u32, 2), try ring.submit());
3640
3641 var cqe_accept = try ring.copy_cqe();
3642 if (cqe_accept.err() == .INVAL) return error.SkipZigTest;
3643 var cqe_connect = try ring.copy_cqe();
3644 if (cqe_connect.err() == .INVAL) return error.SkipZigTest;
3645
3646 // The accept/connect CQEs may arrive in any order, the connect CQE will sometimes come first:
3647 if (cqe_accept.user_data == 0xcccccccc and cqe_connect.user_data == 0xaaaaaaaa) {
3648 const a = cqe_accept;
3649 const b = cqe_connect;
3650 cqe_accept = b;
3651 cqe_connect = a;
3652 }
3653
3654 try testing.expectEqual(@as(u64, 0xaaaaaaaa), cqe_accept.user_data);
3655 if (cqe_accept.res <= 0) std.debug.print("\ncqe_accept.res={}\n", .{cqe_accept.res});
3656 try testing.expect(cqe_accept.res > 0);
3657 try testing.expectEqual(@as(u32, 0), cqe_accept.flags);
3658 try testing.expectEqual(linux.io_uring_cqe{
3659 .user_data = 0xcccccccc,
3660 .res = 0,
3661 .flags = 0,
3662 }, cqe_connect);
3663
3664 // All good
3665
3666 return SocketTestHarness{
3667 .listener = listener_socket,
3668 .server = cqe_accept.res,
3669 .client = client,
3670 };
3671}
3672
3673fn createListenerSocket(address: *linux.sockaddr.in) !posix.socket_t {
3674 const kernel_backlog = 1;
3675 const listener_socket = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
3676 errdefer posix.close(listener_socket);
3677
3678 try posix.setsockopt(listener_socket, posix.SOL.SOCKET, posix.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
3679 try posix.bind(listener_socket, addrAny(address), @sizeOf(linux.sockaddr.in));
3680 try posix.listen(listener_socket, kernel_backlog);
3681
3682 // set address to the OS-chosen IP/port.
3683 var slen: posix.socklen_t = @sizeOf(linux.sockaddr.in);
3684 try posix.getsockname(listener_socket, addrAny(address), &slen);
3685
3686 return listener_socket;
3687}
3688
3689test "accept multishot" {
3690 if (!is_linux) return error.SkipZigTest;
3691
3692 var ring = IoUring.init(16, 0) catch |err| switch (err) {
3693 error.SystemOutdated => return error.SkipZigTest,
3694 error.PermissionDenied => return error.SkipZigTest,
3695 else => return err,
3696 };
3697 defer ring.deinit();
3698
3699 var address: linux.sockaddr.in = .{
3700 .port = 0,
3701 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
3702 };
3703 const listener_socket = try createListenerSocket(&address);
3704 defer posix.close(listener_socket);
3705
3706 // submit multishot accept operation
3707 var addr: posix.sockaddr = undefined;
3708 var addr_len: posix.socklen_t = @sizeOf(@TypeOf(addr));
3709 const userdata: u64 = 0xaaaaaaaa;
3710 _ = try ring.accept_multishot(userdata, listener_socket, &addr, &addr_len, 0);
3711 try testing.expectEqual(@as(u32, 1), try ring.submit());
3712
3713 var nr: usize = 4; // number of clients to connect
3714 while (nr > 0) : (nr -= 1) {
3715 // connect client
3716 const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
3717 errdefer posix.close(client);
3718 try posix.connect(client, addrAny(&address), @sizeOf(linux.sockaddr.in));
3719
3720 // test accept completion
3721 var cqe = try ring.copy_cqe();
3722 if (cqe.err() == .INVAL) return error.SkipZigTest;
3723 try testing.expect(cqe.res > 0);
3724 try testing.expect(cqe.user_data == userdata);
3725 try testing.expect(cqe.flags & linux.IORING_CQE_F_MORE > 0); // more flag is set
3726
3727 posix.close(client);
3728 }
3729}
3730
3731test "accept/connect/send_zc/recv" {
3732 try skipKernelLessThan(.{ .major = 6, .minor = 0, .patch = 0 });
3733
3734 const io = testing.io;
3735
3736 var ring = IoUring.init(16, 0) catch |err| switch (err) {
3737 error.SystemOutdated => return error.SkipZigTest,
3738 error.PermissionDenied => return error.SkipZigTest,
3739 else => return err,
3740 };
3741 defer ring.deinit();
3742
3743 const socket_test_harness = try createSocketTestHarness(&ring);
3744 defer socket_test_harness.close(io);
3745
3746 const buffer_send = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
3747 var buffer_recv = [_]u8{0} ** 10;
3748
3749 // zero-copy send
3750 const sqe_send = try ring.send_zc(0xeeeeeeee, socket_test_harness.client, buffer_send[0..], 0, 0);
3751 sqe_send.flags |= linux.IOSQE_IO_LINK;
3752 _ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
3753 try testing.expectEqual(@as(u32, 2), try ring.submit());
3754
3755 var cqe_send = try ring.copy_cqe();
3756 // First completion of zero-copy send.
3757 // IORING_CQE_F_MORE, means that there
3758 // will be a second completion event / notification for the
3759 // request, with the user_data field set to the same value.
3760 // buffer_send must be keep alive until second cqe.
3761 try testing.expectEqual(linux.io_uring_cqe{
3762 .user_data = 0xeeeeeeee,
3763 .res = buffer_send.len,
3764 .flags = linux.IORING_CQE_F_MORE,
3765 }, cqe_send);
3766
3767 cqe_send, const cqe_recv = brk: {
3768 const cqe1 = try ring.copy_cqe();
3769 const cqe2 = try ring.copy_cqe();
3770 break :brk if (cqe1.user_data == 0xeeeeeeee) .{ cqe1, cqe2 } else .{ cqe2, cqe1 };
3771 };
3772
3773 try testing.expectEqual(linux.io_uring_cqe{
3774 .user_data = 0xffffffff,
3775 .res = buffer_recv.len,
3776 .flags = cqe_recv.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
3777 }, cqe_recv);
3778 try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
3779
3780 // Second completion of zero-copy send.
3781 // IORING_CQE_F_NOTIF in flags signals that kernel is done with send_buffer
3782 try testing.expectEqual(linux.io_uring_cqe{
3783 .user_data = 0xeeeeeeee,
3784 .res = 0,
3785 .flags = linux.IORING_CQE_F_NOTIF,
3786 }, cqe_send);
3787}
3788
3789test "accept_direct" {
3790 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
3791
3792 var ring = IoUring.init(1, 0) catch |err| switch (err) {
3793 error.SystemOutdated => return error.SkipZigTest,
3794 error.PermissionDenied => return error.SkipZigTest,
3795 else => return err,
3796 };
3797 defer ring.deinit();
3798 var address: linux.sockaddr.in = .{
3799 .port = 0,
3800 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
3801 };
3802
3803 // register direct file descriptors
3804 var registered_fds = [_]linux.fd_t{-1} ** 2;
3805 try ring.register_files(registered_fds[0..]);
3806
3807 const listener_socket = try createListenerSocket(&address);
3808 defer posix.close(listener_socket);
3809
3810 const accept_userdata: u64 = 0xaaaaaaaa;
3811 const read_userdata: u64 = 0xbbbbbbbb;
3812 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
3813
3814 for (0..2) |_| {
3815 for (registered_fds, 0..) |_, i| {
3816 var buffer_recv = [_]u8{0} ** 16;
3817 const buffer_send: []const u8 = data[0 .. data.len - i]; // make it different at each loop
3818
3819 // submit accept, will chose registered fd and return index in cqe
3820 _ = try ring.accept_direct(accept_userdata, listener_socket, null, null, 0);
3821 try testing.expectEqual(@as(u32, 1), try ring.submit());
3822
3823 // connect
3824 const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
3825 try posix.connect(client, addrAny(&address), @sizeOf(linux.sockaddr.in));
3826 defer posix.close(client);
3827
3828 // accept completion
3829 const cqe_accept = try ring.copy_cqe();
3830 try testing.expectEqual(posix.E.SUCCESS, cqe_accept.err());
3831 const fd_index = cqe_accept.res;
3832 try testing.expect(fd_index < registered_fds.len);
3833 try testing.expect(cqe_accept.user_data == accept_userdata);
3834
3835 // send data
3836 _ = try posix.send(client, buffer_send, 0);
3837
3838 // Example of how to use registered fd:
3839 // Submit receive to fixed file returned by accept (fd_index).
3840 // Fd field is set to registered file index, returned by accept.
3841 // Flag linux.IOSQE_FIXED_FILE must be set.
3842 const recv_sqe = try ring.recv(read_userdata, fd_index, .{ .buffer = &buffer_recv }, 0);
3843 recv_sqe.flags |= linux.IOSQE_FIXED_FILE;
3844 try testing.expectEqual(@as(u32, 1), try ring.submit());
3845
3846 // accept receive
3847 const recv_cqe = try ring.copy_cqe();
3848 try testing.expect(recv_cqe.user_data == read_userdata);
3849 try testing.expect(recv_cqe.res == buffer_send.len);
3850 try testing.expectEqualSlices(u8, buffer_send, buffer_recv[0..buffer_send.len]);
3851 }
3852 // no more available fds, accept will get NFILE error
3853 {
3854 // submit accept
3855 _ = try ring.accept_direct(accept_userdata, listener_socket, null, null, 0);
3856 try testing.expectEqual(@as(u32, 1), try ring.submit());
3857 // connect
3858 const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
3859 try posix.connect(client, addrAny(&address), @sizeOf(linux.sockaddr.in));
3860 defer posix.close(client);
3861 // completion with error
3862 const cqe_accept = try ring.copy_cqe();
3863 try testing.expect(cqe_accept.user_data == accept_userdata);
3864 try testing.expectEqual(posix.E.NFILE, cqe_accept.err());
3865 }
3866 // return file descriptors to kernel
3867 try ring.register_files_update(0, registered_fds[0..]);
3868 }
3869 try ring.unregister_files();
3870}
3871
3872test "accept_multishot_direct" {
3873 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
3874
3875 if (builtin.cpu.arch == .riscv64) {
3876 // https://github.com/ziglang/zig/issues/25734
3877 return error.SkipZigTest;
3878 }
3879
3880 var ring = IoUring.init(1, 0) catch |err| switch (err) {
3881 error.SystemOutdated => return error.SkipZigTest,
3882 error.PermissionDenied => return error.SkipZigTest,
3883 else => return err,
3884 };
3885 defer ring.deinit();
3886
3887 var address: linux.sockaddr.in = .{
3888 .port = 0,
3889 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
3890 };
3891
3892 var registered_fds = [_]linux.fd_t{-1} ** 2;
3893 try ring.register_files(registered_fds[0..]);
3894
3895 const listener_socket = try createListenerSocket(&address);
3896 defer posix.close(listener_socket);
3897
3898 const accept_userdata: u64 = 0xaaaaaaaa;
3899
3900 for (0..2) |_| {
3901 // submit multishot accept
3902 // Will chose registered fd and return index of the selected registered file in cqe.
3903 _ = try ring.accept_multishot_direct(accept_userdata, listener_socket, null, null, 0);
3904 try testing.expectEqual(@as(u32, 1), try ring.submit());
3905
3906 for (registered_fds) |_| {
3907 // connect
3908 const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
3909 try posix.connect(client, addrAny(&address), @sizeOf(linux.sockaddr.in));
3910 defer posix.close(client);
3911
3912 // accept completion
3913 const cqe_accept = try ring.copy_cqe();
3914 const fd_index = cqe_accept.res;
3915 try testing.expect(fd_index < registered_fds.len);
3916 try testing.expect(cqe_accept.user_data == accept_userdata);
3917 try testing.expect(cqe_accept.flags & linux.IORING_CQE_F_MORE > 0); // has more is set
3918 }
3919 // No more available fds, accept will get NFILE error.
3920 // Multishot is terminated (more flag is not set).
3921 {
3922 // connect
3923 const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
3924 try posix.connect(client, addrAny(&address), @sizeOf(linux.sockaddr.in));
3925 defer posix.close(client);
3926 // completion with error
3927 const cqe_accept = try ring.copy_cqe();
3928 try testing.expect(cqe_accept.user_data == accept_userdata);
3929 try testing.expectEqual(posix.E.NFILE, cqe_accept.err());
3930 try testing.expect(cqe_accept.flags & linux.IORING_CQE_F_MORE == 0); // has more is not set
3931 }
3932 // return file descriptors to kernel
3933 try ring.register_files_update(0, registered_fds[0..]);
3934 }
3935 try ring.unregister_files();
3936}
3937
3938test "socket" {
3939 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
3940
3941 var ring = IoUring.init(1, 0) catch |err| switch (err) {
3942 error.SystemOutdated => return error.SkipZigTest,
3943 error.PermissionDenied => return error.SkipZigTest,
3944 else => return err,
3945 };
3946 defer ring.deinit();
3947
3948 // prepare, submit socket operation
3949 _ = try ring.socket(0, linux.AF.INET, posix.SOCK.STREAM, 0, 0);
3950 try testing.expectEqual(@as(u32, 1), try ring.submit());
3951
3952 // test completion
3953 var cqe = try ring.copy_cqe();
3954 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
3955 const fd: linux.fd_t = @intCast(cqe.res);
3956 try testing.expect(fd > 2);
3957
3958 posix.close(fd);
3959}
3960
3961test "socket_direct/socket_direct_alloc/close_direct" {
3962 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
3963
3964 var ring = IoUring.init(2, 0) catch |err| switch (err) {
3965 error.SystemOutdated => return error.SkipZigTest,
3966 error.PermissionDenied => return error.SkipZigTest,
3967 else => return err,
3968 };
3969 defer ring.deinit();
3970
3971 var registered_fds = [_]linux.fd_t{-1} ** 3;
3972 try ring.register_files(registered_fds[0..]);
3973
3974 // create socket in registered file descriptor at index 0 (last param)
3975 _ = try ring.socket_direct(0, linux.AF.INET, posix.SOCK.STREAM, 0, 0, 0);
3976 try testing.expectEqual(@as(u32, 1), try ring.submit());
3977 var cqe_socket = try ring.copy_cqe();
3978 try testing.expectEqual(posix.E.SUCCESS, cqe_socket.err());
3979 try testing.expect(cqe_socket.res == 0);
3980
3981 // create socket in registered file descriptor at index 1 (last param)
3982 _ = try ring.socket_direct(0, linux.AF.INET, posix.SOCK.STREAM, 0, 0, 1);
3983 try testing.expectEqual(@as(u32, 1), try ring.submit());
3984 cqe_socket = try ring.copy_cqe();
3985 try testing.expectEqual(posix.E.SUCCESS, cqe_socket.err());
3986 try testing.expect(cqe_socket.res == 0); // res is 0 when index is specified
3987
3988 // create socket in kernel chosen file descriptor index (_alloc version)
3989 // completion res has index from registered files
3990 _ = try ring.socket_direct_alloc(0, linux.AF.INET, posix.SOCK.STREAM, 0, 0);
3991 try testing.expectEqual(@as(u32, 1), try ring.submit());
3992 cqe_socket = try ring.copy_cqe();
3993 try testing.expectEqual(posix.E.SUCCESS, cqe_socket.err());
3994 try testing.expect(cqe_socket.res == 2); // returns registered file index
3995
3996 // use sockets from registered_fds in connect operation
3997 var address: linux.sockaddr.in = .{
3998 .port = 0,
3999 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
4000 };
4001 const listener_socket = try createListenerSocket(&address);
4002 defer posix.close(listener_socket);
4003 const accept_userdata: u64 = 0xaaaaaaaa;
4004 const connect_userdata: u64 = 0xbbbbbbbb;
4005 const close_userdata: u64 = 0xcccccccc;
4006 for (registered_fds, 0..) |_, fd_index| {
4007 // prepare accept
4008 _ = try ring.accept(accept_userdata, listener_socket, null, null, 0);
4009 // prepare connect with fixed socket
4010 const connect_sqe = try ring.connect(connect_userdata, @intCast(fd_index), addrAny(&address), @sizeOf(linux.sockaddr.in));
4011 connect_sqe.flags |= linux.IOSQE_FIXED_FILE; // fd is fixed file index
4012 // submit both
4013 try testing.expectEqual(@as(u32, 2), try ring.submit());
4014 // get completions
4015 var cqe_connect = try ring.copy_cqe();
4016 var cqe_accept = try ring.copy_cqe();
4017 // ignore order
4018 if (cqe_connect.user_data == accept_userdata and cqe_accept.user_data == connect_userdata) {
4019 const a = cqe_accept;
4020 const b = cqe_connect;
4021 cqe_accept = b;
4022 cqe_connect = a;
4023 }
4024 // test connect completion
4025 try testing.expect(cqe_connect.user_data == connect_userdata);
4026 try testing.expectEqual(posix.E.SUCCESS, cqe_connect.err());
4027 // test accept completion
4028 try testing.expect(cqe_accept.user_data == accept_userdata);
4029 try testing.expectEqual(posix.E.SUCCESS, cqe_accept.err());
4030
4031 // submit and test close_direct
4032 _ = try ring.close_direct(close_userdata, @intCast(fd_index));
4033 try testing.expectEqual(@as(u32, 1), try ring.submit());
4034 var cqe_close = try ring.copy_cqe();
4035 try testing.expect(cqe_close.user_data == close_userdata);
4036 try testing.expectEqual(posix.E.SUCCESS, cqe_close.err());
4037 }
4038
4039 try ring.unregister_files();
4040}
4041
4042test "openat_direct/close_direct" {
4043 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
4044
4045 var ring = IoUring.init(2, 0) catch |err| switch (err) {
4046 error.SystemOutdated => return error.SkipZigTest,
4047 error.PermissionDenied => return error.SkipZigTest,
4048 else => return err,
4049 };
4050 defer ring.deinit();
4051
4052 var registered_fds = [_]linux.fd_t{-1} ** 3;
4053 try ring.register_files(registered_fds[0..]);
4054
4055 var tmp = std.testing.tmpDir(.{});
4056 defer tmp.cleanup();
4057 const path = "test_io_uring_close_direct";
4058 const flags: linux.O = .{ .ACCMODE = .RDWR, .CREAT = true };
4059 const mode: posix.mode_t = 0o666;
4060 const user_data: u64 = 0;
4061
4062 // use registered file at index 0 (last param)
4063 _ = try ring.openat_direct(user_data, tmp.dir.fd, path, flags, mode, 0);
4064 try testing.expectEqual(@as(u32, 1), try ring.submit());
4065 var cqe = try ring.copy_cqe();
4066 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
4067 try testing.expect(cqe.res == 0);
4068
4069 // use registered file at index 1
4070 _ = try ring.openat_direct(user_data, tmp.dir.fd, path, flags, mode, 1);
4071 try testing.expectEqual(@as(u32, 1), try ring.submit());
4072 cqe = try ring.copy_cqe();
4073 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
4074 try testing.expect(cqe.res == 0); // res is 0 when we specify index
4075
4076 // let kernel choose registered file index
4077 _ = try ring.openat_direct(user_data, tmp.dir.fd, path, flags, mode, linux.IORING_FILE_INDEX_ALLOC);
4078 try testing.expectEqual(@as(u32, 1), try ring.submit());
4079 cqe = try ring.copy_cqe();
4080 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
4081 try testing.expect(cqe.res == 2); // chosen index is in res
4082
4083 // close all open file descriptors
4084 for (registered_fds, 0..) |_, fd_index| {
4085 _ = try ring.close_direct(user_data, @intCast(fd_index));
4086 try testing.expectEqual(@as(u32, 1), try ring.submit());
4087 var cqe_close = try ring.copy_cqe();
4088 try testing.expectEqual(posix.E.SUCCESS, cqe_close.err());
4089 }
4090 try ring.unregister_files();
4091}
4092
4093/// For use in tests. Returns SkipZigTest if kernel version is less than required.
4094inline fn skipKernelLessThan(required: std.SemanticVersion) !void {
4095 if (!is_linux) return error.SkipZigTest;
4096
4097 var uts: linux.utsname = undefined;
4098 const res = linux.uname(&uts);
4099 switch (linux.errno(res)) {
4100 .SUCCESS => {},
4101 else => |errno| return posix.unexpectedErrno(errno),
4102 }
4103
4104 const release = mem.sliceTo(&uts.release, 0);
4105 // Strips potential extra, as kernel version might not be semver compliant, example "6.8.9-300.fc40.x86_64"
4106 const extra_index = std.mem.findAny(u8, release, "-+");
4107 const stripped = release[0..(extra_index orelse release.len)];
4108 // Make sure the input don't rely on the extra we just stripped
4109 try testing.expect(required.pre == null and required.build == null);
4110
4111 var current = try std.SemanticVersion.parse(stripped);
4112 current.pre = null; // don't check pre field
4113 if (required.order(current) == .gt) return error.SkipZigTest;
4114}
4115
4116test BufferGroup {1854test BufferGroup {
4117 if (!is_linux) return error.SkipZigTest;1855 if (!is_linux) return error.SkipZigTest;
41181856
4119 const io = testing.io;1857 const io = testing.io;
1858 _ = io;
41201859
4121 // Init IoUring1860 // Init IoUring
4122 var ring = IoUring.init(16, 0) catch |err| switch (err) {1861 var ring = IoUring.init(16, 0) catch |err| switch (err) {
...@@ -4145,7 +1884,7 @@ test BufferGroup {...@@ -4145,7 +1884,7 @@ test BufferGroup {
41451884
4146 // Create client/server fds1885 // Create client/server fds
4147 const fds = try createSocketTestHarness(&ring);1886 const fds = try createSocketTestHarness(&ring);
4148 defer fds.close(io);1887 defer fds.close();
4149 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };1888 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
41501889
4151 // Client sends data1890 // Client sends data
...@@ -4180,469 +1919,6 @@ test BufferGroup {...@@ -4180,469 +1919,6 @@ test BufferGroup {
4180 }1919 }
4181}1920}
41821921
4183test "ring mapped buffers recv" {1922test {
4184 if (!is_linux) return error.SkipZigTest;1923 if (is_linux) _ = @import("IoUring/test.zig");
4185
4186 const io = testing.io;
4187
4188 var ring = IoUring.init(16, 0) catch |err| switch (err) {
4189 error.SystemOutdated => return error.SkipZigTest,
4190 error.PermissionDenied => return error.SkipZigTest,
4191 else => return err,
4192 };
4193 defer ring.deinit();
4194
4195 // init buffer group
4196 const group_id: u16 = 1; // buffers group id
4197 const buffers_count: u16 = 2; // number of buffers in buffer group
4198 const buffer_size: usize = 4; // size of each buffer in group
4199 var buf_grp = BufferGroup.init(
4200 &ring,
4201 testing.allocator,
4202 group_id,
4203 buffer_size,
4204 buffers_count,
4205 ) catch |err| switch (err) {
4206 // kernel older than 5.19
4207 error.ArgumentsInvalid => return error.SkipZigTest,
4208 else => return err,
4209 };
4210 defer buf_grp.deinit(testing.allocator);
4211
4212 // create client/server fds
4213 const fds = try createSocketTestHarness(&ring);
4214 defer fds.close(io);
4215
4216 // for random user_data in sqe/cqe
4217 var Rnd = std.Random.DefaultPrng.init(std.testing.random_seed);
4218 var rnd = Rnd.random();
4219
4220 var round: usize = 4; // repeat send/recv cycle round times
4221 while (round > 0) : (round -= 1) {
4222 // client sends data
4223 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
4224 {
4225 const user_data = rnd.int(u64);
4226 _ = try ring.send(user_data, fds.client, data[0..], 0);
4227 try testing.expectEqual(@as(u32, 1), try ring.submit());
4228 const cqe_send = try ring.copy_cqe();
4229 if (cqe_send.err() == .INVAL) return error.SkipZigTest;
4230 try testing.expectEqual(linux.io_uring_cqe{ .user_data = user_data, .res = data.len, .flags = 0 }, cqe_send);
4231 }
4232 var pos: usize = 0;
4233
4234 // read first chunk
4235 const cqe1 = try buf_grp_recv_submit_get_cqe(&ring, &buf_grp, fds.server, rnd.int(u64));
4236 var buf = try buf_grp.get(cqe1);
4237 try testing.expectEqualSlices(u8, data[pos..][0..buf.len], buf);
4238 pos += buf.len;
4239 // second chunk
4240 const cqe2 = try buf_grp_recv_submit_get_cqe(&ring, &buf_grp, fds.server, rnd.int(u64));
4241 buf = try buf_grp.get(cqe2);
4242 try testing.expectEqualSlices(u8, data[pos..][0..buf.len], buf);
4243 pos += buf.len;
4244
4245 // both buffers provided to the kernel are used so we get error
4246 // 'no more buffers', until we put buffers to the kernel
4247 {
4248 const user_data = rnd.int(u64);
4249 _ = try buf_grp.recv(user_data, fds.server, 0);
4250 try testing.expectEqual(@as(u32, 1), try ring.submit());
4251 const cqe = try ring.copy_cqe();
4252 try testing.expectEqual(user_data, cqe.user_data);
4253 try testing.expect(cqe.res < 0); // fail
4254 try testing.expectEqual(posix.E.NOBUFS, cqe.err());
4255 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == 0); // IORING_CQE_F_BUFFER flags is set on success only
4256 try testing.expectError(error.NoBufferSelected, cqe.buffer_id());
4257 }
4258
4259 // put buffers back to the kernel
4260 try buf_grp.put(cqe1);
4261 try buf_grp.put(cqe2);
4262
4263 // read remaining data
4264 while (pos < data.len) {
4265 const cqe = try buf_grp_recv_submit_get_cqe(&ring, &buf_grp, fds.server, rnd.int(u64));
4266 buf = try buf_grp.get(cqe);
4267 try testing.expectEqualSlices(u8, data[pos..][0..buf.len], buf);
4268 pos += buf.len;
4269 try buf_grp.put(cqe);
4270 }
4271 }
4272}
4273
4274test "ring mapped buffers multishot recv" {
4275 if (!is_linux) return error.SkipZigTest;
4276
4277 const io = testing.io;
4278
4279 var ring = IoUring.init(16, 0) catch |err| switch (err) {
4280 error.SystemOutdated => return error.SkipZigTest,
4281 error.PermissionDenied => return error.SkipZigTest,
4282 else => return err,
4283 };
4284 defer ring.deinit();
4285
4286 // init buffer group
4287 const group_id: u16 = 1; // buffers group id
4288 const buffers_count: u16 = 2; // number of buffers in buffer group
4289 const buffer_size: usize = 4; // size of each buffer in group
4290 var buf_grp = BufferGroup.init(
4291 &ring,
4292 testing.allocator,
4293 group_id,
4294 buffer_size,
4295 buffers_count,
4296 ) catch |err| switch (err) {
4297 // kernel older than 5.19
4298 error.ArgumentsInvalid => return error.SkipZigTest,
4299 else => return err,
4300 };
4301 defer buf_grp.deinit(testing.allocator);
4302
4303 // create client/server fds
4304 const fds = try createSocketTestHarness(&ring);
4305 defer fds.close(io);
4306
4307 // for random user_data in sqe/cqe
4308 var Rnd = std.Random.DefaultPrng.init(std.testing.random_seed);
4309 var rnd = Rnd.random();
4310
4311 var round: usize = 4; // repeat send/recv cycle round times
4312 while (round > 0) : (round -= 1) {
4313 // client sends data
4314 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf };
4315 {
4316 const user_data = rnd.int(u64);
4317 _ = try ring.send(user_data, fds.client, data[0..], 0);
4318 try testing.expectEqual(@as(u32, 1), try ring.submit());
4319 const cqe_send = try ring.copy_cqe();
4320 if (cqe_send.err() == .INVAL) return error.SkipZigTest;
4321 try testing.expectEqual(linux.io_uring_cqe{ .user_data = user_data, .res = data.len, .flags = 0 }, cqe_send);
4322 }
4323
4324 // start multishot recv
4325 var recv_user_data = rnd.int(u64);
4326 _ = try buf_grp.recv_multishot(recv_user_data, fds.server, 0);
4327 try testing.expectEqual(@as(u32, 1), try ring.submit()); // submit
4328
4329 // server reads data into provided buffers
4330 // there are 2 buffers of size 4, so each read gets only chunk of data
4331 // we read four chunks of 4, 4, 4, 4 bytes each
4332 var chunk: []const u8 = data[0..buffer_size]; // first chunk
4333 const cqe1 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);
4334 try testing.expect(cqe1.flags & linux.IORING_CQE_F_MORE > 0);
4335
4336 chunk = data[buffer_size .. buffer_size * 2]; // second chunk
4337 const cqe2 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);
4338 try testing.expect(cqe2.flags & linux.IORING_CQE_F_MORE > 0);
4339
4340 // both buffers provided to the kernel are used so we get error
4341 // 'no more buffers', until we put buffers to the kernel
4342 {
4343 const cqe = try ring.copy_cqe();
4344 try testing.expectEqual(recv_user_data, cqe.user_data);
4345 try testing.expect(cqe.res < 0); // fail
4346 try testing.expectEqual(posix.E.NOBUFS, cqe.err());
4347 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == 0); // IORING_CQE_F_BUFFER flags is set on success only
4348 // has more is not set
4349 // indicates that multishot is finished
4350 try testing.expect(cqe.flags & linux.IORING_CQE_F_MORE == 0);
4351 try testing.expectError(error.NoBufferSelected, cqe.buffer_id());
4352 }
4353
4354 // put buffers back to the kernel
4355 try buf_grp.put(cqe1);
4356 try buf_grp.put(cqe2);
4357
4358 // restart multishot
4359 recv_user_data = rnd.int(u64);
4360 _ = try buf_grp.recv_multishot(recv_user_data, fds.server, 0);
4361 try testing.expectEqual(@as(u32, 1), try ring.submit()); // submit
4362
4363 chunk = data[buffer_size * 2 .. buffer_size * 3]; // third chunk
4364 const cqe3 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);
4365 try testing.expect(cqe3.flags & linux.IORING_CQE_F_MORE > 0);
4366 try buf_grp.put(cqe3);
4367
4368 chunk = data[buffer_size * 3 ..]; // last chunk
4369 const cqe4 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);
4370 try testing.expect(cqe4.flags & linux.IORING_CQE_F_MORE > 0);
4371 try buf_grp.put(cqe4);
4372
4373 // cancel pending multishot recv operation
4374 {
4375 const cancel_user_data = rnd.int(u64);
4376 _ = try ring.cancel(cancel_user_data, recv_user_data, 0);
4377 try testing.expectEqual(@as(u32, 1), try ring.submit());
4378
4379 // expect completion of cancel operation and completion of recv operation
4380 var cqe_cancel = try ring.copy_cqe();
4381 if (cqe_cancel.err() == .INVAL) return error.SkipZigTest;
4382 var cqe_recv = try ring.copy_cqe();
4383 if (cqe_recv.err() == .INVAL) return error.SkipZigTest;
4384
4385 // don't depend on order of completions
4386 if (cqe_cancel.user_data == recv_user_data and cqe_recv.user_data == cancel_user_data) {
4387 const a = cqe_cancel;
4388 const b = cqe_recv;
4389 cqe_cancel = b;
4390 cqe_recv = a;
4391 }
4392
4393 // Note on different kernel results:
4394 // on older kernel (tested with v6.0.16, v6.1.57, v6.2.12, v6.4.16)
4395 // cqe_cancel.err() == .NOENT
4396 // cqe_recv.err() == .NOBUFS
4397 // on kernel (tested with v6.5.0, v6.5.7)
4398 // cqe_cancel.err() == .SUCCESS
4399 // cqe_recv.err() == .CANCELED
4400 // Upstream reference: https://github.com/axboe/liburing/issues/984
4401
4402 // cancel operation is success (or NOENT on older kernels)
4403 try testing.expectEqual(cancel_user_data, cqe_cancel.user_data);
4404 try testing.expect(cqe_cancel.err() == .NOENT or cqe_cancel.err() == .SUCCESS);
4405
4406 // recv operation is failed with err CANCELED (or NOBUFS on older kernels)
4407 try testing.expectEqual(recv_user_data, cqe_recv.user_data);
4408 try testing.expect(cqe_recv.res < 0);
4409 try testing.expect(cqe_recv.err() == .NOBUFS or cqe_recv.err() == .CANCELED);
4410 try testing.expect(cqe_recv.flags & linux.IORING_CQE_F_MORE == 0);
4411 }
4412 }
4413}
4414
4415// Prepare, submit recv and get cqe using buffer group.
4416fn buf_grp_recv_submit_get_cqe(
4417 ring: *IoUring,
4418 buf_grp: *BufferGroup,
4419 fd: linux.fd_t,
4420 user_data: u64,
4421) !linux.io_uring_cqe {
4422 // prepare and submit recv
4423 const sqe = try buf_grp.recv(user_data, fd, 0);
4424 try testing.expect(sqe.flags & linux.IOSQE_BUFFER_SELECT == linux.IOSQE_BUFFER_SELECT);
4425 try testing.expect(sqe.buf_index == buf_grp.group_id);
4426 try testing.expectEqual(@as(u32, 1), try ring.submit()); // submit
4427 // get cqe, expect success
4428 const cqe = try ring.copy_cqe();
4429 try testing.expectEqual(user_data, cqe.user_data);
4430 try testing.expect(cqe.res >= 0); // success
4431 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
4432 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER); // IORING_CQE_F_BUFFER flag is set
4433
4434 return cqe;
4435}
4436
4437fn expect_buf_grp_cqe(
4438 ring: *IoUring,
4439 buf_grp: *BufferGroup,
4440 user_data: u64,
4441 expected: []const u8,
4442) !linux.io_uring_cqe {
4443 // get cqe
4444 const cqe = try ring.copy_cqe();
4445 try testing.expectEqual(user_data, cqe.user_data);
4446 try testing.expect(cqe.res >= 0); // success
4447 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER); // IORING_CQE_F_BUFFER flag is set
4448 try testing.expectEqual(expected.len, @as(usize, @intCast(cqe.res)));
4449 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
4450
4451 // get buffer from pool
4452 const buffer_id = try cqe.buffer_id();
4453 const len = @as(usize, @intCast(cqe.res));
4454 const buf = buf_grp.get_by_id(buffer_id)[0..len];
4455 try testing.expectEqualSlices(u8, expected, buf);
4456
4457 return cqe;
4458}
4459
4460test "copy_cqes with wrapping sq.cqes buffer" {
4461 if (!is_linux) return error.SkipZigTest;
4462
4463 var ring = IoUring.init(2, 0) catch |err| switch (err) {
4464 error.SystemOutdated => return error.SkipZigTest,
4465 error.PermissionDenied => return error.SkipZigTest,
4466 else => return err,
4467 };
4468 defer ring.deinit();
4469
4470 try testing.expectEqual(2, ring.sq.sqes.len);
4471 try testing.expectEqual(4, ring.cq.cqes.len);
4472
4473 // submit 2 entries, receive 2 completions
4474 var cqes: [8]linux.io_uring_cqe = undefined;
4475 {
4476 for (0..2) |_| {
4477 const sqe = try ring.get_sqe();
4478 sqe.prep_timeout(&.{ .sec = 0, .nsec = 10000 }, 0, 0);
4479 try testing.expect(try ring.submit() == 1);
4480 }
4481 var cqe_count: u32 = 0;
4482 while (cqe_count < 2) {
4483 cqe_count += try ring.copy_cqes(&cqes, 2 - cqe_count);
4484 }
4485 }
4486
4487 try testing.expectEqual(2, ring.cq.head.*);
4488
4489 // sq.sqes len is 4, starting at position 2
4490 // every 4 entries submit wraps completion buffer
4491 // we are reading ring.cq.cqes at indexes 2,3,0,1
4492 for (1..1024) |i| {
4493 for (0..4) |_| {
4494 const sqe = try ring.get_sqe();
4495 sqe.prep_timeout(&.{ .sec = 0, .nsec = 10000 }, 0, 0);
4496 try testing.expect(try ring.submit() == 1);
4497 }
4498 var cqe_count: u32 = 0;
4499 while (cqe_count < 4) {
4500 cqe_count += try ring.copy_cqes(&cqes, 4 - cqe_count);
4501 }
4502 try testing.expectEqual(4, cqe_count);
4503 try testing.expectEqual(2 + 4 * i, ring.cq.head.*);
4504 }
4505}
4506
4507test "bind/listen/connect" {
4508 if (builtin.cpu.arch == .s390x) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/25956
4509
4510 var ring = IoUring.init(4, 0) catch |err| switch (err) {
4511 error.SystemOutdated => return error.SkipZigTest,
4512 error.PermissionDenied => return error.SkipZigTest,
4513 else => return err,
4514 };
4515 defer ring.deinit();
4516
4517 const probe = ring.get_probe() catch return error.SkipZigTest;
4518 // LISTEN is higher required operation
4519 if (!probe.is_supported(.LISTEN)) return error.SkipZigTest;
4520
4521 var addr: linux.sockaddr.in = .{
4522 .port = 0,
4523 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
4524 };
4525 const proto: u32 = if (addr.family == linux.AF.UNIX) 0 else linux.IPPROTO.TCP;
4526
4527 const listen_fd = brk: {
4528 // Create socket
4529 _ = try ring.socket(1, addr.family, linux.SOCK.STREAM | linux.SOCK.CLOEXEC, proto, 0);
4530 try testing.expectEqual(1, try ring.submit());
4531 var cqe = try ring.copy_cqe();
4532 try testing.expectEqual(1, cqe.user_data);
4533 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
4534 const listen_fd: linux.fd_t = @intCast(cqe.res);
4535 try testing.expect(listen_fd > 2);
4536
4537 // Prepare: set socket option * 2, bind, listen
4538 var optval: u32 = 1;
4539 (try ring.setsockopt(2, listen_fd, linux.SOL.SOCKET, linux.SO.REUSEADDR, mem.asBytes(&optval))).link_next();
4540 (try ring.setsockopt(3, listen_fd, linux.SOL.SOCKET, linux.SO.REUSEPORT, mem.asBytes(&optval))).link_next();
4541 (try ring.bind(4, listen_fd, addrAny(&addr), @sizeOf(linux.sockaddr.in), 0)).link_next();
4542 _ = try ring.listen(5, listen_fd, 1, 0);
4543 // Submit 4 operations
4544 try testing.expectEqual(4, try ring.submit());
4545 // Expect all to succeed
4546 for (2..6) |user_data| {
4547 cqe = try ring.copy_cqe();
4548 try testing.expectEqual(user_data, cqe.user_data);
4549 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
4550 }
4551
4552 // Check that socket option is set
4553 optval = 0;
4554 _ = try ring.getsockopt(5, listen_fd, linux.SOL.SOCKET, linux.SO.REUSEADDR, mem.asBytes(&optval));
4555 try testing.expectEqual(1, try ring.submit());
4556 cqe = try ring.copy_cqe();
4557 try testing.expectEqual(5, cqe.user_data);
4558 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
4559 try testing.expectEqual(1, optval);
4560
4561 // Read system assigned port into addr
4562 var addr_len: posix.socklen_t = @sizeOf(linux.sockaddr.in);
4563 try posix.getsockname(listen_fd, addrAny(&addr), &addr_len);
4564
4565 break :brk listen_fd;
4566 };
4567
4568 const connect_fd = brk: {
4569 // Create connect socket
4570 _ = try ring.socket(6, addr.family, linux.SOCK.STREAM | linux.SOCK.CLOEXEC, proto, 0);
4571 try testing.expectEqual(1, try ring.submit());
4572 const cqe = try ring.copy_cqe();
4573 try testing.expectEqual(6, cqe.user_data);
4574 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
4575 // Get connect socket fd
4576 const connect_fd: linux.fd_t = @intCast(cqe.res);
4577 try testing.expect(connect_fd > 2 and connect_fd != listen_fd);
4578 break :brk connect_fd;
4579 };
4580
4581 // Prepare accept/connect operations
4582 _ = try ring.accept(7, listen_fd, null, null, 0);
4583 _ = try ring.connect(8, connect_fd, addrAny(&addr), @sizeOf(linux.sockaddr.in));
4584 try testing.expectEqual(2, try ring.submit());
4585 // Get listener accepted socket
4586 var accept_fd: posix.socket_t = 0;
4587 for (0..2) |_| {
4588 const cqe = try ring.copy_cqe();
4589 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
4590 if (cqe.user_data == 7) {
4591 accept_fd = @intCast(cqe.res);
4592 } else {
4593 try testing.expectEqual(8, cqe.user_data);
4594 }
4595 }
4596 try testing.expect(accept_fd > 2 and accept_fd != listen_fd and accept_fd != connect_fd);
4597
4598 // Communicate
4599 try testSendRecv(&ring, connect_fd, accept_fd);
4600 try testSendRecv(&ring, accept_fd, connect_fd);
4601
4602 // Shutdown and close all sockets
4603 for ([_]posix.socket_t{ connect_fd, accept_fd, listen_fd }) |fd| {
4604 (try ring.shutdown(9, fd, posix.SHUT.RDWR)).link_next();
4605 _ = try ring.close(10, fd);
4606 try testing.expectEqual(2, try ring.submit());
4607 for (0..2) |i| {
4608 const cqe = try ring.copy_cqe();
4609 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
4610 try testing.expectEqual(9 + i, cqe.user_data);
4611 }
4612 }
4613}
4614
4615fn testSendRecv(ring: *IoUring, send_fd: posix.socket_t, recv_fd: posix.socket_t) !void {
4616 const buffer_send = "0123456789abcdf" ** 10;
4617 var buffer_recv: [buffer_send.len * 2]u8 = undefined;
4618
4619 // 2 sends
4620 _ = try ring.send(1, send_fd, buffer_send, linux.MSG.WAITALL);
4621 _ = try ring.send(2, send_fd, buffer_send, linux.MSG.WAITALL);
4622 try testing.expectEqual(2, try ring.submit());
4623 for (0..2) |i| {
4624 const cqe = try ring.copy_cqe();
4625 try testing.expectEqual(1 + i, cqe.user_data);
4626 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
4627 try testing.expectEqual(buffer_send.len, @as(usize, @intCast(cqe.res)));
4628 }
4629
4630 // receive
4631 var recv_len: usize = 0;
4632 while (recv_len < buffer_send.len * 2) {
4633 _ = try ring.recv(3, recv_fd, .{ .buffer = buffer_recv[recv_len..] }, 0);
4634 try testing.expectEqual(1, try ring.submit());
4635 const cqe = try ring.copy_cqe();
4636 try testing.expectEqual(3, cqe.user_data);
4637 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
4638 recv_len += @intCast(cqe.res);
4639 }
4640
4641 // inspect recv buffer
4642 try testing.expectEqualSlices(u8, buffer_send, buffer_recv[0..buffer_send.len]);
4643 try testing.expectEqualSlices(u8, buffer_send, buffer_recv[buffer_send.len..]);
4644}
4645
4646fn addrAny(addr: *linux.sockaddr.in) *linux.sockaddr {
4647 return @ptrCast(addr);
4648}1924}
lib/std/os/linux/IoUring/test.zig created+2691
...@@ -0,0 +1,2691 @@
1const builtin = @import("builtin");
2
3const std = @import("../../../std.zig");
4const Io = std.Io;
5const mem = std.mem;
6const assert = std.debug.assert;
7const testing = std.testing;
8const linux = std.os.linux;
9
10const IoUring = std.os.linux.IoUring;
11const BufferGroup = IoUring.BufferGroup;
12
13const posix = std.posix;
14const iovec = posix.iovec;
15const iovec_const = posix.iovec_const;
16
17comptime {
18 assert(builtin.os.tag == .linux);
19}
20
21test "structs/offsets/entries" {
22 try testing.expectEqual(@as(usize, 120), @sizeOf(linux.io_uring_params));
23 try testing.expectEqual(@as(usize, 64), @sizeOf(linux.io_uring_sqe));
24 try testing.expectEqual(@as(usize, 16), @sizeOf(linux.io_uring_cqe));
25
26 try testing.expectEqual(0, linux.IORING_OFF_SQ_RING);
27 try testing.expectEqual(0x8000000, linux.IORING_OFF_CQ_RING);
28 try testing.expectEqual(0x10000000, linux.IORING_OFF_SQES);
29
30 try testing.expectError(error.EntriesZero, IoUring.init(0, 0));
31 try testing.expectError(error.EntriesNotPowerOfTwo, IoUring.init(3, 0));
32}
33
34test "nop" {
35 var ring = IoUring.init(1, 0) catch |err| switch (err) {
36 error.SystemOutdated => return error.SkipZigTest,
37 error.PermissionDenied => return error.SkipZigTest,
38 else => return err,
39 };
40 defer {
41 ring.deinit();
42 testing.expectEqual(@as(linux.fd_t, -1), ring.fd) catch @panic("test failed");
43 }
44
45 const sqe = try ring.nop(0xaaaaaaaa);
46 try testing.expectEqual(linux.io_uring_sqe{
47 .opcode = .NOP,
48 .flags = 0,
49 .ioprio = 0,
50 .fd = 0,
51 .off = 0,
52 .addr = 0,
53 .len = 0,
54 .rw_flags = 0,
55 .user_data = 0xaaaaaaaa,
56 .buf_index = 0,
57 .personality = 0,
58 .splice_fd_in = 0,
59 .addr3 = 0,
60 .resv = 0,
61 }, sqe.*);
62
63 try testing.expectEqual(@as(u32, 0), ring.sq.sqe_head);
64 try testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
65 try testing.expectEqual(@as(u32, 0), ring.sq.tail.*);
66 try testing.expectEqual(@as(u32, 0), ring.cq.head.*);
67 try testing.expectEqual(@as(u32, 1), ring.sq_ready());
68 try testing.expectEqual(@as(u32, 0), ring.cq_ready());
69
70 try testing.expectEqual(@as(u32, 1), try ring.submit());
71 try testing.expectEqual(@as(u32, 1), ring.sq.sqe_head);
72 try testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
73 try testing.expectEqual(@as(u32, 1), ring.sq.tail.*);
74 try testing.expectEqual(@as(u32, 0), ring.cq.head.*);
75 try testing.expectEqual(@as(u32, 0), ring.sq_ready());
76
77 try testing.expectEqual(linux.io_uring_cqe{
78 .user_data = 0xaaaaaaaa,
79 .res = 0,
80 .flags = 0,
81 }, try ring.copy_cqe());
82 try testing.expectEqual(@as(u32, 1), ring.cq.head.*);
83 try testing.expectEqual(@as(u32, 0), ring.cq_ready());
84
85 const sqe_barrier = try ring.nop(0xbbbbbbbb);
86 sqe_barrier.flags |= linux.IOSQE_IO_DRAIN;
87 try testing.expectEqual(@as(u32, 1), try ring.submit());
88 try testing.expectEqual(linux.io_uring_cqe{
89 .user_data = 0xbbbbbbbb,
90 .res = 0,
91 .flags = 0,
92 }, try ring.copy_cqe());
93 try testing.expectEqual(@as(u32, 2), ring.sq.sqe_head);
94 try testing.expectEqual(@as(u32, 2), ring.sq.sqe_tail);
95 try testing.expectEqual(@as(u32, 2), ring.sq.tail.*);
96 try testing.expectEqual(@as(u32, 2), ring.cq.head.*);
97}
98
99test "readv" {
100 const io = testing.io;
101
102 var ring = IoUring.init(1, 0) catch |err| switch (err) {
103 error.SystemOutdated => return error.SkipZigTest,
104 error.PermissionDenied => return error.SkipZigTest,
105 else => return err,
106 };
107 defer ring.deinit();
108
109 const file = try Io.Dir.openFileAbsolute(io, "/dev/zero", .{});
110 defer file.close(io);
111
112 // Linux Kernel 5.4 supports IORING_REGISTER_FILES but not sparse fd sets (i.e. an fd of -1).
113 // Linux Kernel 5.5 adds support for sparse fd sets.
114 // Compare:
115 // https://github.com/torvalds/linux/blob/v5.4/fs/io_uring.c#L3119-L3124 vs
116 // https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L6687-L6691
117 // We therefore avoid stressing sparse fd sets here:
118 var registered_fds = [_]linux.fd_t{0} ** 1;
119 const fd_index = 0;
120 registered_fds[fd_index] = file.handle;
121 try ring.register_files(registered_fds[0..]);
122
123 var buffer = [_]u8{42} ** 128;
124 var iovecs = [_]iovec{iovec{ .base = &buffer, .len = buffer.len }};
125 const sqe = try ring.read(0xcccccccc, fd_index, .{ .iovecs = iovecs[0..] }, 0);
126 try testing.expectEqual(linux.IORING_OP.READV, sqe.opcode);
127 sqe.flags |= linux.IOSQE_FIXED_FILE;
128
129 try testing.expectError(error.SubmissionQueueFull, ring.nop(0));
130 try testing.expectEqual(@as(u32, 1), try ring.submit());
131 try testing.expectEqual(linux.io_uring_cqe{
132 .user_data = 0xcccccccc,
133 .res = buffer.len,
134 .flags = 0,
135 }, try ring.copy_cqe());
136 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
137
138 try ring.unregister_files();
139}
140
141test "writev/fsync/readv" {
142 const io = testing.io;
143
144 var ring = IoUring.init(4, 0) catch |err| switch (err) {
145 error.SystemOutdated => return error.SkipZigTest,
146 error.PermissionDenied => return error.SkipZigTest,
147 else => return err,
148 };
149 defer ring.deinit();
150
151 var tmp = std.testing.tmpDir(.{});
152 defer tmp.cleanup();
153
154 const path = "test_io_uring_writev_fsync_readv";
155 const file = try tmp.dir.createFile(io, path, .{ .read = true });
156 defer file.close(io);
157 const fd = file.handle;
158
159 const buffer_write = [_]u8{42} ** 128;
160 const iovecs_write = [_]iovec_const{
161 iovec_const{ .base = &buffer_write, .len = buffer_write.len },
162 };
163 var buffer_read = [_]u8{0} ** 128;
164 var iovecs_read = [_]iovec{
165 iovec{ .base = &buffer_read, .len = buffer_read.len },
166 };
167
168 const sqe_writev = try ring.writev(0xdddddddd, fd, iovecs_write[0..], 17);
169 try testing.expectEqual(linux.IORING_OP.WRITEV, sqe_writev.opcode);
170 try testing.expectEqual(@as(u64, 17), sqe_writev.off);
171 sqe_writev.flags |= linux.IOSQE_IO_LINK;
172
173 const sqe_fsync = try ring.fsync(0xeeeeeeee, fd, 0);
174 try testing.expectEqual(linux.IORING_OP.FSYNC, sqe_fsync.opcode);
175 try testing.expectEqual(fd, sqe_fsync.fd);
176 sqe_fsync.flags |= linux.IOSQE_IO_LINK;
177
178 const sqe_readv = try ring.read(0xffffffff, fd, .{ .iovecs = iovecs_read[0..] }, 17);
179 try testing.expectEqual(linux.IORING_OP.READV, sqe_readv.opcode);
180 try testing.expectEqual(@as(u64, 17), sqe_readv.off);
181
182 try testing.expectEqual(@as(u32, 3), ring.sq_ready());
183 try testing.expectEqual(@as(u32, 3), try ring.submit_and_wait(3));
184 try testing.expectEqual(@as(u32, 0), ring.sq_ready());
185 try testing.expectEqual(@as(u32, 3), ring.cq_ready());
186
187 try testing.expectEqual(linux.io_uring_cqe{
188 .user_data = 0xdddddddd,
189 .res = buffer_write.len,
190 .flags = 0,
191 }, try ring.copy_cqe());
192 try testing.expectEqual(@as(u32, 2), ring.cq_ready());
193
194 try testing.expectEqual(linux.io_uring_cqe{
195 .user_data = 0xeeeeeeee,
196 .res = 0,
197 .flags = 0,
198 }, try ring.copy_cqe());
199 try testing.expectEqual(@as(u32, 1), ring.cq_ready());
200
201 try testing.expectEqual(linux.io_uring_cqe{
202 .user_data = 0xffffffff,
203 .res = buffer_read.len,
204 .flags = 0,
205 }, try ring.copy_cqe());
206 try testing.expectEqual(@as(u32, 0), ring.cq_ready());
207
208 try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
209}
210
211test "write/read" {
212 const io = testing.io;
213
214 var ring = IoUring.init(2, 0) catch |err| switch (err) {
215 error.SystemOutdated => return error.SkipZigTest,
216 error.PermissionDenied => return error.SkipZigTest,
217 else => return err,
218 };
219 defer ring.deinit();
220
221 var tmp = std.testing.tmpDir(.{});
222 defer tmp.cleanup();
223 const path = "test_io_uring_write_read";
224 const file = try tmp.dir.createFile(io, path, .{ .read = true });
225 defer file.close(io);
226 const fd = file.handle;
227
228 const buffer_write = [_]u8{97} ** 20;
229 var buffer_read = [_]u8{98} ** 20;
230 const sqe_write = try ring.write(0x11111111, fd, buffer_write[0..], 10);
231 try testing.expectEqual(linux.IORING_OP.WRITE, sqe_write.opcode);
232 try testing.expectEqual(@as(u64, 10), sqe_write.off);
233 sqe_write.flags |= linux.IOSQE_IO_LINK;
234 const sqe_read = try ring.read(0x22222222, fd, .{ .buffer = buffer_read[0..] }, 10);
235 try testing.expectEqual(linux.IORING_OP.READ, sqe_read.opcode);
236 try testing.expectEqual(@as(u64, 10), sqe_read.off);
237 try testing.expectEqual(@as(u32, 2), try ring.submit());
238
239 const cqe_write = try ring.copy_cqe();
240 const cqe_read = try ring.copy_cqe();
241 // Prior to Linux Kernel 5.6 this is the only way to test for read/write support:
242 // https://lwn.net/Articles/809820/
243 if (cqe_write.err() == .INVAL) return error.SkipZigTest;
244 if (cqe_read.err() == .INVAL) return error.SkipZigTest;
245 try testing.expectEqual(linux.io_uring_cqe{
246 .user_data = 0x11111111,
247 .res = buffer_write.len,
248 .flags = 0,
249 }, cqe_write);
250 try testing.expectEqual(linux.io_uring_cqe{
251 .user_data = 0x22222222,
252 .res = buffer_read.len,
253 .flags = 0,
254 }, cqe_read);
255 try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
256}
257
258test "splice/read" {
259 const io = testing.io;
260
261 var ring = IoUring.init(4, 0) catch |err| switch (err) {
262 error.SystemOutdated => return error.SkipZigTest,
263 error.PermissionDenied => return error.SkipZigTest,
264 else => return err,
265 };
266 defer ring.deinit();
267
268 var tmp = std.testing.tmpDir(.{});
269 const path_src = "test_io_uring_splice_src";
270 const file_src = try tmp.dir.createFile(io, path_src, .{ .read = true });
271 defer file_src.close(io);
272 const fd_src = file_src.handle;
273
274 const path_dst = "test_io_uring_splice_dst";
275 const file_dst = try tmp.dir.createFile(io, path_dst, .{ .read = true });
276 defer file_dst.close(io);
277 const fd_dst = file_dst.handle;
278
279 const buffer_write = [_]u8{97} ** 20;
280 var buffer_read = [_]u8{98} ** 20;
281 try file_src.writeStreamingAll(io, &buffer_write);
282
283 const fds = try posix.pipe();
284 const pipe_offset: u64 = std.math.maxInt(u64);
285
286 const sqe_splice_to_pipe = try ring.splice(0x11111111, fd_src, 0, fds[1], pipe_offset, buffer_write.len);
287 try testing.expectEqual(linux.IORING_OP.SPLICE, sqe_splice_to_pipe.opcode);
288 try testing.expectEqual(@as(u64, 0), sqe_splice_to_pipe.addr);
289 try testing.expectEqual(pipe_offset, sqe_splice_to_pipe.off);
290 sqe_splice_to_pipe.flags |= linux.IOSQE_IO_LINK;
291
292 const sqe_splice_from_pipe = try ring.splice(0x22222222, fds[0], pipe_offset, fd_dst, 10, buffer_write.len);
293 try testing.expectEqual(linux.IORING_OP.SPLICE, sqe_splice_from_pipe.opcode);
294 try testing.expectEqual(pipe_offset, sqe_splice_from_pipe.addr);
295 try testing.expectEqual(@as(u64, 10), sqe_splice_from_pipe.off);
296 sqe_splice_from_pipe.flags |= linux.IOSQE_IO_LINK;
297
298 const sqe_read = try ring.read(0x33333333, fd_dst, .{ .buffer = buffer_read[0..] }, 10);
299 try testing.expectEqual(linux.IORING_OP.READ, sqe_read.opcode);
300 try testing.expectEqual(@as(u64, 10), sqe_read.off);
301 try testing.expectEqual(@as(u32, 3), try ring.submit());
302
303 const cqe_splice_to_pipe = try ring.copy_cqe();
304 const cqe_splice_from_pipe = try ring.copy_cqe();
305 const cqe_read = try ring.copy_cqe();
306 // Prior to Linux Kernel 5.6 this is the only way to test for splice/read support:
307 // https://lwn.net/Articles/809820/
308 if (cqe_splice_to_pipe.err() == .INVAL) return error.SkipZigTest;
309 if (cqe_splice_from_pipe.err() == .INVAL) return error.SkipZigTest;
310 if (cqe_read.err() == .INVAL) return error.SkipZigTest;
311 try testing.expectEqual(linux.io_uring_cqe{
312 .user_data = 0x11111111,
313 .res = buffer_write.len,
314 .flags = 0,
315 }, cqe_splice_to_pipe);
316 try testing.expectEqual(linux.io_uring_cqe{
317 .user_data = 0x22222222,
318 .res = buffer_write.len,
319 .flags = 0,
320 }, cqe_splice_from_pipe);
321 try testing.expectEqual(linux.io_uring_cqe{
322 .user_data = 0x33333333,
323 .res = buffer_read.len,
324 .flags = 0,
325 }, cqe_read);
326 try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
327}
328
329test "write_fixed/read_fixed" {
330 const io = testing.io;
331
332 var ring = IoUring.init(2, 0) catch |err| switch (err) {
333 error.SystemOutdated => return error.SkipZigTest,
334 error.PermissionDenied => return error.SkipZigTest,
335 else => return err,
336 };
337 defer ring.deinit();
338
339 var tmp = std.testing.tmpDir(.{});
340 defer tmp.cleanup();
341
342 const path = "test_io_uring_write_read_fixed";
343 const file = try tmp.dir.createFile(io, path, .{ .read = true });
344 defer file.close(io);
345 const fd = file.handle;
346
347 var raw_buffers: [2][11]u8 = undefined;
348 // First buffer will be written to the file.
349 @memset(&raw_buffers[0], 'z');
350 raw_buffers[0][0.."foobar".len].* = "foobar".*;
351
352 var buffers = [2]iovec{
353 .{ .base = &raw_buffers[0], .len = raw_buffers[0].len },
354 .{ .base = &raw_buffers[1], .len = raw_buffers[1].len },
355 };
356 ring.register_buffers(&buffers) catch |err| switch (err) {
357 error.SystemResources => {
358 // See https://github.com/ziglang/zig/issues/15362
359 return error.SkipZigTest;
360 },
361 else => |e| return e,
362 };
363
364 const sqe_write = try ring.write_fixed(0x45454545, fd, &buffers[0], 3, 0);
365 try testing.expectEqual(linux.IORING_OP.WRITE_FIXED, sqe_write.opcode);
366 try testing.expectEqual(@as(u64, 3), sqe_write.off);
367 sqe_write.flags |= linux.IOSQE_IO_LINK;
368
369 const sqe_read = try ring.read_fixed(0x12121212, fd, &buffers[1], 0, 1);
370 try testing.expectEqual(linux.IORING_OP.READ_FIXED, sqe_read.opcode);
371 try testing.expectEqual(@as(u64, 0), sqe_read.off);
372
373 try testing.expectEqual(@as(u32, 2), try ring.submit());
374
375 const cqe_write = try ring.copy_cqe();
376 const cqe_read = try ring.copy_cqe();
377
378 try testing.expectEqual(linux.io_uring_cqe{
379 .user_data = 0x45454545,
380 .res = @as(i32, @intCast(buffers[0].len)),
381 .flags = 0,
382 }, cqe_write);
383 try testing.expectEqual(linux.io_uring_cqe{
384 .user_data = 0x12121212,
385 .res = @as(i32, @intCast(buffers[1].len)),
386 .flags = 0,
387 }, cqe_read);
388
389 try testing.expectEqualSlices(u8, "\x00\x00\x00", buffers[1].base[0..3]);
390 try testing.expectEqualSlices(u8, "foobar", buffers[1].base[3..9]);
391 try testing.expectEqualSlices(u8, "zz", buffers[1].base[9..11]);
392}
393
394test "openat" {
395 var ring = IoUring.init(1, 0) catch |err| switch (err) {
396 error.SystemOutdated => return error.SkipZigTest,
397 error.PermissionDenied => return error.SkipZigTest,
398 else => return err,
399 };
400 defer ring.deinit();
401
402 var tmp = std.testing.tmpDir(.{});
403 defer tmp.cleanup();
404
405 const path = "test_io_uring_openat";
406
407 // Workaround for LLVM bug: https://github.com/ziglang/zig/issues/12014
408 const path_addr = if (builtin.zig_backend == .stage2_llvm) p: {
409 var workaround = path;
410 _ = &workaround;
411 break :p @intFromPtr(workaround);
412 } else @intFromPtr(path);
413
414 const flags: linux.O = .{ .CLOEXEC = true, .ACCMODE = .RDWR, .CREAT = true };
415 const mode: posix.mode_t = 0o666;
416 const sqe_openat = try ring.openat(0x33333333, tmp.dir.handle, path, flags, mode);
417 try testing.expectEqual(linux.io_uring_sqe{
418 .opcode = .OPENAT,
419 .flags = 0,
420 .ioprio = 0,
421 .fd = tmp.dir.handle,
422 .off = 0,
423 .addr = path_addr,
424 .len = mode,
425 .rw_flags = @bitCast(flags),
426 .user_data = 0x33333333,
427 .buf_index = 0,
428 .personality = 0,
429 .splice_fd_in = 0,
430 .addr3 = 0,
431 .resv = 0,
432 }, sqe_openat.*);
433 try testing.expectEqual(@as(u32, 1), try ring.submit());
434
435 const cqe_openat = try ring.copy_cqe();
436 try testing.expectEqual(@as(u64, 0x33333333), cqe_openat.user_data);
437 if (cqe_openat.err() == .INVAL) return error.SkipZigTest;
438 if (cqe_openat.err() == .BADF) return error.SkipZigTest;
439 if (cqe_openat.res <= 0) std.debug.print("\ncqe_openat.res={}\n", .{cqe_openat.res});
440 try testing.expect(cqe_openat.res > 0);
441 try testing.expectEqual(@as(u32, 0), cqe_openat.flags);
442
443 posix.close(cqe_openat.res);
444}
445
446test "close" {
447 const io = testing.io;
448
449 var ring = IoUring.init(1, 0) catch |err| switch (err) {
450 error.SystemOutdated => return error.SkipZigTest,
451 error.PermissionDenied => return error.SkipZigTest,
452 else => return err,
453 };
454 defer ring.deinit();
455
456 var tmp = std.testing.tmpDir(.{});
457 defer tmp.cleanup();
458
459 const path = "test_io_uring_close";
460 const file = try tmp.dir.createFile(io, path, .{});
461 errdefer file.close(io);
462
463 const sqe_close = try ring.close(0x44444444, file.handle);
464 try testing.expectEqual(linux.IORING_OP.CLOSE, sqe_close.opcode);
465 try testing.expectEqual(file.handle, sqe_close.fd);
466 try testing.expectEqual(@as(u32, 1), try ring.submit());
467
468 const cqe_close = try ring.copy_cqe();
469 if (cqe_close.err() == .INVAL) return error.SkipZigTest;
470 try testing.expectEqual(linux.io_uring_cqe{
471 .user_data = 0x44444444,
472 .res = 0,
473 .flags = 0,
474 }, cqe_close);
475}
476
477test "accept/connect/send/recv" {
478 const io = testing.io;
479 _ = io;
480
481 var ring = IoUring.init(16, 0) catch |err| switch (err) {
482 error.SystemOutdated => return error.SkipZigTest,
483 error.PermissionDenied => return error.SkipZigTest,
484 else => return err,
485 };
486 defer ring.deinit();
487
488 const socket_test_harness = try createSocketTestHarness(&ring);
489 defer socket_test_harness.close();
490
491 const buffer_send = [_]u8{ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 };
492 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
493
494 const sqe_send = try ring.send(0xeeeeeeee, socket_test_harness.client, buffer_send[0..], 0);
495 sqe_send.flags |= linux.IOSQE_IO_LINK;
496 _ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
497 try testing.expectEqual(@as(u32, 2), try ring.submit());
498
499 const cqe_send = try ring.copy_cqe();
500 if (cqe_send.err() == .INVAL) return error.SkipZigTest;
501 try testing.expectEqual(linux.io_uring_cqe{
502 .user_data = 0xeeeeeeee,
503 .res = buffer_send.len,
504 .flags = 0,
505 }, cqe_send);
506
507 const cqe_recv = try ring.copy_cqe();
508 if (cqe_recv.err() == .INVAL) return error.SkipZigTest;
509 try testing.expectEqual(linux.io_uring_cqe{
510 .user_data = 0xffffffff,
511 .res = buffer_recv.len,
512 // ignore IORING_CQE_F_SOCK_NONEMPTY since it is only set on some systems
513 .flags = cqe_recv.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
514 }, cqe_recv);
515
516 try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
517}
518
519test "sendmsg/recvmsg" {
520 var ring = IoUring.init(2, 0) catch |err| switch (err) {
521 error.SystemOutdated => return error.SkipZigTest,
522 error.PermissionDenied => return error.SkipZigTest,
523 else => return err,
524 };
525 defer ring.deinit();
526
527 var address_server: linux.sockaddr.in = .{
528 .port = 0,
529 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
530 };
531
532 const server = try posix.socket(address_server.family, posix.SOCK.DGRAM, 0);
533 defer posix.close(server);
534 try posix.setsockopt(server, posix.SOL.SOCKET, posix.SO.REUSEPORT, &mem.toBytes(@as(c_int, 1)));
535 try posix.setsockopt(server, posix.SOL.SOCKET, posix.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
536 try posix.bind(server, addrAny(&address_server), @sizeOf(linux.sockaddr.in));
537
538 // set address_server to the OS-chosen IP/port.
539 var slen: posix.socklen_t = @sizeOf(linux.sockaddr.in);
540 try posix.getsockname(server, addrAny(&address_server), &slen);
541
542 const client = try posix.socket(address_server.family, posix.SOCK.DGRAM, 0);
543 defer posix.close(client);
544
545 const buffer_send = [_]u8{42} ** 128;
546 const iovecs_send = [_]iovec_const{
547 iovec_const{ .base = &buffer_send, .len = buffer_send.len },
548 };
549 const msg_send: linux.msghdr_const = .{
550 .name = addrAny(&address_server),
551 .namelen = @sizeOf(linux.sockaddr.in),
552 .iov = &iovecs_send,
553 .iovlen = 1,
554 .control = null,
555 .controllen = 0,
556 .flags = 0,
557 };
558 const sqe_sendmsg = try ring.sendmsg(0x11111111, client, &msg_send, 0);
559 sqe_sendmsg.flags |= linux.IOSQE_IO_LINK;
560 try testing.expectEqual(linux.IORING_OP.SENDMSG, sqe_sendmsg.opcode);
561 try testing.expectEqual(client, sqe_sendmsg.fd);
562
563 var buffer_recv = [_]u8{0} ** 128;
564 var iovecs_recv = [_]iovec{
565 iovec{ .base = &buffer_recv, .len = buffer_recv.len },
566 };
567 var address_recv: linux.sockaddr.in = .{
568 .port = 0,
569 .addr = 0,
570 };
571 var msg_recv: linux.msghdr = .{
572 .name = addrAny(&address_recv),
573 .namelen = @sizeOf(linux.sockaddr.in),
574 .iov = &iovecs_recv,
575 .iovlen = 1,
576 .control = null,
577 .controllen = 0,
578 .flags = 0,
579 };
580 const sqe_recvmsg = try ring.recvmsg(0x22222222, server, &msg_recv, 0);
581 try testing.expectEqual(linux.IORING_OP.RECVMSG, sqe_recvmsg.opcode);
582 try testing.expectEqual(server, sqe_recvmsg.fd);
583
584 try testing.expectEqual(@as(u32, 2), ring.sq_ready());
585 try testing.expectEqual(@as(u32, 2), try ring.submit_and_wait(2));
586 try testing.expectEqual(@as(u32, 0), ring.sq_ready());
587 try testing.expectEqual(@as(u32, 2), ring.cq_ready());
588
589 const cqe_sendmsg = try ring.copy_cqe();
590 if (cqe_sendmsg.res == -@as(i32, @intFromEnum(linux.E.INVAL))) return error.SkipZigTest;
591 try testing.expectEqual(linux.io_uring_cqe{
592 .user_data = 0x11111111,
593 .res = buffer_send.len,
594 .flags = 0,
595 }, cqe_sendmsg);
596
597 const cqe_recvmsg = try ring.copy_cqe();
598 if (cqe_recvmsg.res == -@as(i32, @intFromEnum(linux.E.INVAL))) return error.SkipZigTest;
599 try testing.expectEqual(linux.io_uring_cqe{
600 .user_data = 0x22222222,
601 .res = buffer_recv.len,
602 // ignore IORING_CQE_F_SOCK_NONEMPTY since it is set non-deterministically
603 .flags = cqe_recvmsg.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
604 }, cqe_recvmsg);
605
606 try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
607}
608
609test "timeout (after a relative time)" {
610 const io = testing.io;
611
612 var ring = IoUring.init(1, 0) catch |err| switch (err) {
613 error.SystemOutdated => return error.SkipZigTest,
614 error.PermissionDenied => return error.SkipZigTest,
615 else => return err,
616 };
617 defer ring.deinit();
618
619 const ms = 10;
620 const margin = 5;
621 const ts: linux.kernel_timespec = .{ .sec = 0, .nsec = ms * 1000000 };
622
623 const started = try std.Io.Clock.awake.now(io);
624 const sqe = try ring.timeout(0x55555555, &ts, 0, 0);
625 try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe.opcode);
626 try testing.expectEqual(@as(u32, 1), try ring.submit());
627 const cqe = try ring.copy_cqe();
628 const stopped = try std.Io.Clock.awake.now(io);
629
630 try testing.expectEqual(linux.io_uring_cqe{
631 .user_data = 0x55555555,
632 .res = -@as(i32, @intFromEnum(linux.E.TIME)),
633 .flags = 0,
634 }, cqe);
635
636 // Tests should not depend on timings: skip test if outside margin.
637 const ms_elapsed = started.durationTo(stopped).toMilliseconds();
638 if (ms_elapsed > margin) return error.SkipZigTest;
639}
640
641test "timeout (after a number of completions)" {
642 var ring = IoUring.init(2, 0) catch |err| switch (err) {
643 error.SystemOutdated => return error.SkipZigTest,
644 error.PermissionDenied => return error.SkipZigTest,
645 else => return err,
646 };
647 defer ring.deinit();
648
649 const ts: linux.kernel_timespec = .{ .sec = 3, .nsec = 0 };
650 const count_completions: u64 = 1;
651 const sqe_timeout = try ring.timeout(0x66666666, &ts, count_completions, 0);
652 try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe_timeout.opcode);
653 try testing.expectEqual(count_completions, sqe_timeout.off);
654 _ = try ring.nop(0x77777777);
655 try testing.expectEqual(@as(u32, 2), try ring.submit());
656
657 const cqe_nop = try ring.copy_cqe();
658 try testing.expectEqual(linux.io_uring_cqe{
659 .user_data = 0x77777777,
660 .res = 0,
661 .flags = 0,
662 }, cqe_nop);
663
664 const cqe_timeout = try ring.copy_cqe();
665 try testing.expectEqual(linux.io_uring_cqe{
666 .user_data = 0x66666666,
667 .res = 0,
668 .flags = 0,
669 }, cqe_timeout);
670}
671
672test "timeout_remove" {
673 var ring = IoUring.init(2, 0) catch |err| switch (err) {
674 error.SystemOutdated => return error.SkipZigTest,
675 error.PermissionDenied => return error.SkipZigTest,
676 else => return err,
677 };
678 defer ring.deinit();
679
680 const ts: linux.kernel_timespec = .{ .sec = 3, .nsec = 0 };
681 const sqe_timeout = try ring.timeout(0x88888888, &ts, 0, 0);
682 try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe_timeout.opcode);
683 try testing.expectEqual(@as(u64, 0x88888888), sqe_timeout.user_data);
684
685 const sqe_timeout_remove = try ring.timeout_remove(0x99999999, 0x88888888, 0);
686 try testing.expectEqual(linux.IORING_OP.TIMEOUT_REMOVE, sqe_timeout_remove.opcode);
687 try testing.expectEqual(@as(u64, 0x88888888), sqe_timeout_remove.addr);
688 try testing.expectEqual(@as(u64, 0x99999999), sqe_timeout_remove.user_data);
689
690 try testing.expectEqual(@as(u32, 2), try ring.submit());
691
692 // The order in which the CQE arrive is not clearly documented and it changed with kernel 5.18:
693 // * kernel 5.10 gives user data 0x88888888 first, 0x99999999 second
694 // * kernel 5.18 gives user data 0x99999999 first, 0x88888888 second
695
696 var cqes: [2]linux.io_uring_cqe = undefined;
697 cqes[0] = try ring.copy_cqe();
698 cqes[1] = try ring.copy_cqe();
699
700 for (cqes) |cqe| {
701 // IORING_OP_TIMEOUT_REMOVE is not supported by this kernel version:
702 // Timeout remove operations set the fd to -1, which results in EBADF before EINVAL.
703 // We use IORING_FEAT_RW_CUR_POS as a safety check here to make sure we are at least pre-5.6.
704 // We don't want to skip this test for newer kernels.
705 if (cqe.user_data == 0x99999999 and
706 cqe.err() == .BADF and
707 (ring.features & linux.IORING_FEAT_RW_CUR_POS) == 0)
708 {
709 return error.SkipZigTest;
710 }
711
712 try testing.expect(cqe.user_data == 0x88888888 or cqe.user_data == 0x99999999);
713
714 if (cqe.user_data == 0x88888888) {
715 try testing.expectEqual(linux.io_uring_cqe{
716 .user_data = 0x88888888,
717 .res = -@as(i32, @intFromEnum(linux.E.CANCELED)),
718 .flags = 0,
719 }, cqe);
720 } else if (cqe.user_data == 0x99999999) {
721 try testing.expectEqual(linux.io_uring_cqe{
722 .user_data = 0x99999999,
723 .res = 0,
724 .flags = 0,
725 }, cqe);
726 }
727 }
728}
729
730test "accept/connect/recv/link_timeout" {
731 const io = testing.io;
732 _ = io;
733
734 var ring = IoUring.init(16, 0) catch |err| switch (err) {
735 error.SystemOutdated => return error.SkipZigTest,
736 error.PermissionDenied => return error.SkipZigTest,
737 else => return err,
738 };
739 defer ring.deinit();
740
741 const socket_test_harness = try createSocketTestHarness(&ring);
742 defer socket_test_harness.close();
743
744 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
745
746 const sqe_recv = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
747 sqe_recv.flags |= linux.IOSQE_IO_LINK;
748
749 const ts = linux.kernel_timespec{ .sec = 0, .nsec = 1000000 };
750 _ = try ring.link_timeout(0x22222222, &ts, 0);
751
752 const nr_wait = try ring.submit();
753 try testing.expectEqual(@as(u32, 2), nr_wait);
754
755 var i: usize = 0;
756 while (i < nr_wait) : (i += 1) {
757 const cqe = try ring.copy_cqe();
758 switch (cqe.user_data) {
759 0xffffffff => {
760 if (cqe.res != -@as(i32, @intFromEnum(linux.E.INTR)) and
761 cqe.res != -@as(i32, @intFromEnum(linux.E.CANCELED)))
762 {
763 std.debug.print("Req 0x{x} got {d}\n", .{ cqe.user_data, cqe.res });
764 try testing.expect(false);
765 }
766 },
767 0x22222222 => {
768 if (cqe.res != -@as(i32, @intFromEnum(linux.E.ALREADY)) and
769 cqe.res != -@as(i32, @intFromEnum(linux.E.TIME)))
770 {
771 std.debug.print("Req 0x{x} got {d}\n", .{ cqe.user_data, cqe.res });
772 try testing.expect(false);
773 }
774 },
775 else => @panic("should not happen"),
776 }
777 }
778}
779
780test "fallocate" {
781 const io = testing.io;
782
783 var ring = IoUring.init(1, 0) catch |err| switch (err) {
784 error.SystemOutdated => return error.SkipZigTest,
785 error.PermissionDenied => return error.SkipZigTest,
786 else => return err,
787 };
788 defer ring.deinit();
789
790 var tmp = std.testing.tmpDir(.{});
791 defer tmp.cleanup();
792
793 const path = "test_io_uring_fallocate";
794 const file = try tmp.dir.createFile(io, path, .{});
795 defer file.close(io);
796
797 try testing.expectEqual(@as(u64, 0), (try file.stat(io)).size);
798
799 const len: u64 = 65536;
800 const sqe = try ring.fallocate(0xaaaaaaaa, file.handle, 0, 0, len);
801 try testing.expectEqual(linux.IORING_OP.FALLOCATE, sqe.opcode);
802 try testing.expectEqual(file.handle, sqe.fd);
803 try testing.expectEqual(@as(u32, 1), try ring.submit());
804
805 const cqe = try ring.copy_cqe();
806 switch (cqe.err()) {
807 .SUCCESS => {},
808 // This kernel's io_uring does not yet implement fallocate():
809 .INVAL => return error.SkipZigTest,
810 // This kernel does not implement fallocate():
811 .NOSYS => return error.SkipZigTest,
812 // The filesystem containing the file referred to by fd does not support this operation;
813 // or the mode is not supported by the filesystem containing the file referred to by fd:
814 .OPNOTSUPP => return error.SkipZigTest,
815 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
816 }
817 try testing.expectEqual(linux.io_uring_cqe{
818 .user_data = 0xaaaaaaaa,
819 .res = 0,
820 .flags = 0,
821 }, cqe);
822
823 try testing.expectEqual(len, (try file.stat(io)).size);
824}
825
826test "statx" {
827 const io = testing.io;
828
829 var ring = IoUring.init(1, 0) catch |err| switch (err) {
830 error.SystemOutdated => return error.SkipZigTest,
831 error.PermissionDenied => return error.SkipZigTest,
832 else => return err,
833 };
834 defer ring.deinit();
835
836 var tmp = std.testing.tmpDir(.{});
837 defer tmp.cleanup();
838 const path = "test_io_uring_statx";
839 const file = try tmp.dir.createFile(io, path, .{});
840 defer file.close(io);
841
842 try testing.expectEqual(@as(u64, 0), (try file.stat(io)).size);
843
844 try file.writeStreamingAll(io, "foobar");
845
846 var buf: linux.Statx = undefined;
847 const sqe = try ring.statx(
848 0xaaaaaaaa,
849 tmp.dir.handle,
850 path,
851 0,
852 .{ .SIZE = true },
853 &buf,
854 );
855 try testing.expectEqual(linux.IORING_OP.STATX, sqe.opcode);
856 try testing.expectEqual(@as(i32, tmp.dir.handle), sqe.fd);
857 try testing.expectEqual(@as(u32, 1), try ring.submit());
858
859 const cqe = try ring.copy_cqe();
860 switch (cqe.err()) {
861 .SUCCESS => {},
862 // This kernel's io_uring does not yet implement statx():
863 .INVAL => return error.SkipZigTest,
864 // This kernel does not implement statx():
865 .NOSYS => return error.SkipZigTest,
866 // The filesystem containing the file referred to by fd does not support this operation;
867 // or the mode is not supported by the filesystem containing the file referred to by fd:
868 .OPNOTSUPP => return error.SkipZigTest,
869 // not supported on older kernels (5.4)
870 .BADF => return error.SkipZigTest,
871 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
872 }
873 try testing.expectEqual(linux.io_uring_cqe{
874 .user_data = 0xaaaaaaaa,
875 .res = 0,
876 .flags = 0,
877 }, cqe);
878
879 try testing.expect(buf.mask.SIZE);
880 try testing.expectEqual(@as(u64, 6), buf.size);
881}
882
883test "accept/connect/recv/cancel" {
884 const io = testing.io;
885 _ = io;
886
887 var ring = IoUring.init(16, 0) catch |err| switch (err) {
888 error.SystemOutdated => return error.SkipZigTest,
889 error.PermissionDenied => return error.SkipZigTest,
890 else => return err,
891 };
892 defer ring.deinit();
893
894 const socket_test_harness = try createSocketTestHarness(&ring);
895 defer socket_test_harness.close();
896
897 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
898
899 _ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
900 try testing.expectEqual(@as(u32, 1), try ring.submit());
901
902 const sqe_cancel = try ring.cancel(0x99999999, 0xffffffff, 0);
903 try testing.expectEqual(linux.IORING_OP.ASYNC_CANCEL, sqe_cancel.opcode);
904 try testing.expectEqual(@as(u64, 0xffffffff), sqe_cancel.addr);
905 try testing.expectEqual(@as(u64, 0x99999999), sqe_cancel.user_data);
906 try testing.expectEqual(@as(u32, 1), try ring.submit());
907
908 var cqe_recv = try ring.copy_cqe();
909 if (cqe_recv.err() == .INVAL) return error.SkipZigTest;
910 var cqe_cancel = try ring.copy_cqe();
911 if (cqe_cancel.err() == .INVAL) return error.SkipZigTest;
912
913 // The recv/cancel CQEs may arrive in any order, the recv CQE will sometimes come first:
914 if (cqe_recv.user_data == 0x99999999 and cqe_cancel.user_data == 0xffffffff) {
915 const a = cqe_recv;
916 const b = cqe_cancel;
917 cqe_recv = b;
918 cqe_cancel = a;
919 }
920
921 try testing.expectEqual(linux.io_uring_cqe{
922 .user_data = 0xffffffff,
923 .res = -@as(i32, @intFromEnum(linux.E.CANCELED)),
924 .flags = 0,
925 }, cqe_recv);
926
927 try testing.expectEqual(linux.io_uring_cqe{
928 .user_data = 0x99999999,
929 .res = 0,
930 .flags = 0,
931 }, cqe_cancel);
932}
933
934test "register_files_update" {
935 var ring = IoUring.init(1, 0) catch |err| switch (err) {
936 error.SystemOutdated => return error.SkipZigTest,
937 error.PermissionDenied => return error.SkipZigTest,
938 else => return err,
939 };
940 defer ring.deinit();
941
942 const fd = try posix.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
943 defer posix.close(fd);
944
945 var registered_fds = [_]linux.fd_t{0} ** 2;
946 const fd_index = 0;
947 const fd_index2 = 1;
948 registered_fds[fd_index] = fd;
949 registered_fds[fd_index2] = -1;
950
951 ring.register_files(registered_fds[0..]) catch |err| switch (err) {
952 // Happens when the kernel doesn't support sparse entry (-1) in the file descriptors array.
953 error.FileDescriptorInvalid => return error.SkipZigTest,
954 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
955 };
956
957 // Test IORING_REGISTER_FILES_UPDATE
958 // Only available since Linux 5.5
959
960 const fd2 = try posix.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
961 defer posix.close(fd2);
962
963 registered_fds[fd_index] = fd2;
964 registered_fds[fd_index2] = -1;
965 try ring.register_files_update(0, registered_fds[0..]);
966
967 var buffer = [_]u8{42} ** 128;
968 {
969 const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
970 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
971 sqe.flags |= linux.IOSQE_FIXED_FILE;
972
973 try testing.expectEqual(@as(u32, 1), try ring.submit());
974 try testing.expectEqual(linux.io_uring_cqe{
975 .user_data = 0xcccccccc,
976 .res = buffer.len,
977 .flags = 0,
978 }, try ring.copy_cqe());
979 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
980 }
981
982 // Test with a non-zero offset
983
984 registered_fds[fd_index] = -1;
985 registered_fds[fd_index2] = -1;
986 try ring.register_files_update(1, registered_fds[1..]);
987
988 {
989 // Next read should still work since fd_index in the registered file descriptors hasn't been updated yet.
990 const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
991 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
992 sqe.flags |= linux.IOSQE_FIXED_FILE;
993
994 try testing.expectEqual(@as(u32, 1), try ring.submit());
995 try testing.expectEqual(linux.io_uring_cqe{
996 .user_data = 0xcccccccc,
997 .res = buffer.len,
998 .flags = 0,
999 }, try ring.copy_cqe());
1000 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
1001 }
1002
1003 try ring.register_files_update(0, registered_fds[0..]);
1004
1005 {
1006 // Now this should fail since both fds are sparse (-1)
1007 const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
1008 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
1009 sqe.flags |= linux.IOSQE_FIXED_FILE;
1010
1011 try testing.expectEqual(@as(u32, 1), try ring.submit());
1012 const cqe = try ring.copy_cqe();
1013 try testing.expectEqual(linux.E.BADF, cqe.err());
1014 }
1015
1016 try ring.unregister_files();
1017}
1018
1019test "shutdown" {
1020 var ring = IoUring.init(16, 0) catch |err| switch (err) {
1021 error.SystemOutdated => return error.SkipZigTest,
1022 error.PermissionDenied => return error.SkipZigTest,
1023 else => return err,
1024 };
1025 defer ring.deinit();
1026
1027 var address: linux.sockaddr.in = .{
1028 .port = 0,
1029 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
1030 };
1031
1032 // Socket bound, expect shutdown to work
1033 {
1034 const server = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
1035 defer posix.close(server);
1036 try posix.setsockopt(server, posix.SOL.SOCKET, posix.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
1037 try posix.bind(server, addrAny(&address), @sizeOf(linux.sockaddr.in));
1038 try posix.listen(server, 1);
1039
1040 // set address to the OS-chosen IP/port.
1041 var slen: posix.socklen_t = @sizeOf(linux.sockaddr.in);
1042 try posix.getsockname(server, addrAny(&address), &slen);
1043
1044 const shutdown_sqe = try ring.shutdown(0x445445445, server, linux.SHUT.RD);
1045 try testing.expectEqual(linux.IORING_OP.SHUTDOWN, shutdown_sqe.opcode);
1046 try testing.expectEqual(@as(i32, server), shutdown_sqe.fd);
1047
1048 try testing.expectEqual(@as(u32, 1), try ring.submit());
1049
1050 const cqe = try ring.copy_cqe();
1051 switch (cqe.err()) {
1052 .SUCCESS => {},
1053 // This kernel's io_uring does not yet implement shutdown (kernel version < 5.11)
1054 .INVAL => return error.SkipZigTest,
1055 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1056 }
1057
1058 try testing.expectEqual(linux.io_uring_cqe{
1059 .user_data = 0x445445445,
1060 .res = 0,
1061 .flags = 0,
1062 }, cqe);
1063 }
1064
1065 // Socket not bound, expect to fail with ENOTCONN
1066 {
1067 const server = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
1068 defer posix.close(server);
1069
1070 const shutdown_sqe = ring.shutdown(0x445445445, server, linux.SHUT.RD) catch |err| switch (err) {
1071 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1072 };
1073 try testing.expectEqual(linux.IORING_OP.SHUTDOWN, shutdown_sqe.opcode);
1074 try testing.expectEqual(@as(i32, server), shutdown_sqe.fd);
1075
1076 try testing.expectEqual(@as(u32, 1), try ring.submit());
1077
1078 const cqe = try ring.copy_cqe();
1079 try testing.expectEqual(@as(u64, 0x445445445), cqe.user_data);
1080 try testing.expectEqual(linux.E.NOTCONN, cqe.err());
1081 }
1082}
1083
1084test "renameat" {
1085 const io = testing.io;
1086
1087 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1088 error.SystemOutdated => return error.SkipZigTest,
1089 error.PermissionDenied => return error.SkipZigTest,
1090 else => return err,
1091 };
1092 defer ring.deinit();
1093
1094 const old_path = "test_io_uring_renameat_old";
1095 const new_path = "test_io_uring_renameat_new";
1096
1097 var tmp = std.testing.tmpDir(.{});
1098 defer tmp.cleanup();
1099
1100 // Write old file with data
1101
1102 const old_file = try tmp.dir.createFile(io, old_path, .{});
1103 defer old_file.close(io);
1104 try old_file.writeStreamingAll(io, "hello");
1105
1106 // Submit renameat
1107
1108 const sqe = try ring.renameat(
1109 0x12121212,
1110 tmp.dir.handle,
1111 old_path,
1112 tmp.dir.handle,
1113 new_path,
1114 0,
1115 );
1116 try testing.expectEqual(linux.IORING_OP.RENAMEAT, sqe.opcode);
1117 try testing.expectEqual(@as(i32, tmp.dir.handle), sqe.fd);
1118 try testing.expectEqual(@as(i32, tmp.dir.handle), @as(i32, @bitCast(sqe.len)));
1119 try testing.expectEqual(@as(u32, 1), try ring.submit());
1120
1121 const cqe = try ring.copy_cqe();
1122 switch (cqe.err()) {
1123 .SUCCESS => {},
1124 // This kernel's io_uring does not yet implement renameat (kernel version < 5.11)
1125 .BADF, .INVAL => return error.SkipZigTest,
1126 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1127 }
1128 try testing.expectEqual(linux.io_uring_cqe{
1129 .user_data = 0x12121212,
1130 .res = 0,
1131 .flags = 0,
1132 }, cqe);
1133
1134 // Validate that the old file doesn't exist anymore
1135 try testing.expectError(error.FileNotFound, tmp.dir.openFile(io, old_path, .{}));
1136
1137 // Validate that the new file exists with the proper content
1138 var new_file_data: [16]u8 = undefined;
1139 try testing.expectEqualStrings("hello", try tmp.dir.readFile(io, new_path, &new_file_data));
1140}
1141
1142test "unlinkat" {
1143 const io = testing.io;
1144
1145 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1146 error.SystemOutdated => return error.SkipZigTest,
1147 error.PermissionDenied => return error.SkipZigTest,
1148 else => return err,
1149 };
1150 defer ring.deinit();
1151
1152 const path = "test_io_uring_unlinkat";
1153
1154 var tmp = std.testing.tmpDir(.{});
1155 defer tmp.cleanup();
1156
1157 // Write old file with data
1158
1159 const file = try tmp.dir.createFile(io, path, .{});
1160 defer file.close(io);
1161
1162 // Submit unlinkat
1163
1164 const sqe = try ring.unlinkat(
1165 0x12121212,
1166 tmp.dir.handle,
1167 path,
1168 0,
1169 );
1170 try testing.expectEqual(linux.IORING_OP.UNLINKAT, sqe.opcode);
1171 try testing.expectEqual(@as(i32, tmp.dir.handle), sqe.fd);
1172 try testing.expectEqual(@as(u32, 1), try ring.submit());
1173
1174 const cqe = try ring.copy_cqe();
1175 switch (cqe.err()) {
1176 .SUCCESS => {},
1177 // This kernel's io_uring does not yet implement unlinkat (kernel version < 5.11)
1178 .BADF, .INVAL => return error.SkipZigTest,
1179 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1180 }
1181 try testing.expectEqual(linux.io_uring_cqe{
1182 .user_data = 0x12121212,
1183 .res = 0,
1184 .flags = 0,
1185 }, cqe);
1186
1187 // Validate that the file doesn't exist anymore
1188 _ = tmp.dir.openFile(io, path, .{}) catch |err| switch (err) {
1189 error.FileNotFound => {},
1190 else => std.debug.panic("unexpected error: {}", .{err}),
1191 };
1192}
1193
1194test "mkdirat" {
1195 const io = testing.io;
1196
1197 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1198 error.SystemOutdated => return error.SkipZigTest,
1199 error.PermissionDenied => return error.SkipZigTest,
1200 else => return err,
1201 };
1202 defer ring.deinit();
1203
1204 var tmp = std.testing.tmpDir(.{});
1205 defer tmp.cleanup();
1206
1207 const path = "test_io_uring_mkdirat";
1208
1209 // Submit mkdirat
1210
1211 const sqe = try ring.mkdirat(
1212 0x12121212,
1213 tmp.dir.handle,
1214 path,
1215 0o0755,
1216 );
1217 try testing.expectEqual(linux.IORING_OP.MKDIRAT, sqe.opcode);
1218 try testing.expectEqual(@as(i32, tmp.dir.handle), sqe.fd);
1219 try testing.expectEqual(@as(u32, 1), try ring.submit());
1220
1221 const cqe = try ring.copy_cqe();
1222 switch (cqe.err()) {
1223 .SUCCESS => {},
1224 // This kernel's io_uring does not yet implement mkdirat (kernel version < 5.15)
1225 .BADF, .INVAL => return error.SkipZigTest,
1226 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1227 }
1228 try testing.expectEqual(linux.io_uring_cqe{
1229 .user_data = 0x12121212,
1230 .res = 0,
1231 .flags = 0,
1232 }, cqe);
1233
1234 // Validate that the directory exist
1235 _ = try tmp.dir.openDir(io, path, .{});
1236}
1237
1238test "symlinkat" {
1239 const io = testing.io;
1240
1241 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1242 error.SystemOutdated => return error.SkipZigTest,
1243 error.PermissionDenied => return error.SkipZigTest,
1244 else => return err,
1245 };
1246 defer ring.deinit();
1247
1248 var tmp = std.testing.tmpDir(.{});
1249 defer tmp.cleanup();
1250
1251 const path = "test_io_uring_symlinkat";
1252 const link_path = "test_io_uring_symlinkat_link";
1253
1254 const file = try tmp.dir.createFile(io, path, .{});
1255 defer file.close(io);
1256
1257 // Submit symlinkat
1258
1259 const sqe = try ring.symlinkat(
1260 0x12121212,
1261 path,
1262 tmp.dir.handle,
1263 link_path,
1264 );
1265 try testing.expectEqual(linux.IORING_OP.SYMLINKAT, sqe.opcode);
1266 try testing.expectEqual(@as(i32, tmp.dir.handle), sqe.fd);
1267 try testing.expectEqual(@as(u32, 1), try ring.submit());
1268
1269 const cqe = try ring.copy_cqe();
1270 switch (cqe.err()) {
1271 .SUCCESS => {},
1272 // This kernel's io_uring does not yet implement symlinkat (kernel version < 5.15)
1273 .BADF, .INVAL => return error.SkipZigTest,
1274 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1275 }
1276 try testing.expectEqual(linux.io_uring_cqe{
1277 .user_data = 0x12121212,
1278 .res = 0,
1279 .flags = 0,
1280 }, cqe);
1281
1282 // Validate that the symlink exist
1283 _ = try tmp.dir.openFile(io, link_path, .{});
1284}
1285
1286test "linkat" {
1287 const io = testing.io;
1288
1289 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1290 error.SystemOutdated => return error.SkipZigTest,
1291 error.PermissionDenied => return error.SkipZigTest,
1292 else => return err,
1293 };
1294 defer ring.deinit();
1295
1296 var tmp = std.testing.tmpDir(.{});
1297 defer tmp.cleanup();
1298
1299 const first_path = "test_io_uring_linkat_first";
1300 const second_path = "test_io_uring_linkat_second";
1301
1302 // Write file with data
1303
1304 const first_file = try tmp.dir.createFile(io, first_path, .{});
1305 defer first_file.close(io);
1306 try first_file.writeStreamingAll(io, "hello");
1307
1308 // Submit linkat
1309
1310 const sqe = try ring.linkat(
1311 0x12121212,
1312 tmp.dir.handle,
1313 first_path,
1314 tmp.dir.handle,
1315 second_path,
1316 0,
1317 );
1318 try testing.expectEqual(linux.IORING_OP.LINKAT, sqe.opcode);
1319 try testing.expectEqual(@as(i32, tmp.dir.handle), sqe.fd);
1320 try testing.expectEqual(@as(i32, tmp.dir.handle), @as(i32, @bitCast(sqe.len)));
1321 try testing.expectEqual(@as(u32, 1), try ring.submit());
1322
1323 const cqe = try ring.copy_cqe();
1324 switch (cqe.err()) {
1325 .SUCCESS => {},
1326 // This kernel's io_uring does not yet implement linkat (kernel version < 5.15)
1327 .BADF, .INVAL => return error.SkipZigTest,
1328 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1329 }
1330 try testing.expectEqual(linux.io_uring_cqe{
1331 .user_data = 0x12121212,
1332 .res = 0,
1333 .flags = 0,
1334 }, cqe);
1335
1336 // Validate the second file
1337 var second_file_data: [16]u8 = undefined;
1338 try testing.expectEqualStrings("hello", try tmp.dir.readFile(io, second_path, &second_file_data));
1339}
1340
1341test "provide_buffers: read" {
1342 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1343 error.SystemOutdated => return error.SkipZigTest,
1344 error.PermissionDenied => return error.SkipZigTest,
1345 else => return err,
1346 };
1347 defer ring.deinit();
1348
1349 const fd = try posix.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
1350 defer posix.close(fd);
1351
1352 const group_id = 1337;
1353 const buffer_id = 0;
1354
1355 const buffer_len = 128;
1356
1357 var buffers: [4][buffer_len]u8 = undefined;
1358
1359 // Provide 4 buffers
1360
1361 {
1362 const sqe = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
1363 try testing.expectEqual(linux.IORING_OP.PROVIDE_BUFFERS, sqe.opcode);
1364 try testing.expectEqual(@as(i32, buffers.len), sqe.fd);
1365 try testing.expectEqual(@as(u32, buffers[0].len), sqe.len);
1366 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
1367 try testing.expectEqual(@as(u32, 1), try ring.submit());
1368
1369 const cqe = try ring.copy_cqe();
1370 switch (cqe.err()) {
1371 // Happens when the kernel is < 5.7
1372 .INVAL, .BADF => return error.SkipZigTest,
1373 .SUCCESS => {},
1374 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1375 }
1376 try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
1377 }
1378
1379 // Do 4 reads which should consume all buffers
1380
1381 var i: usize = 0;
1382 while (i < buffers.len) : (i += 1) {
1383 const sqe = try ring.read(0xdededede, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
1384 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
1385 try testing.expectEqual(@as(i32, fd), sqe.fd);
1386 try testing.expectEqual(@as(u64, 0), sqe.addr);
1387 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
1388 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
1389 try testing.expectEqual(@as(u32, 1), try ring.submit());
1390
1391 const cqe = try ring.copy_cqe();
1392 switch (cqe.err()) {
1393 .SUCCESS => {},
1394 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1395 }
1396
1397 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
1398 const used_buffer_id = cqe.flags >> 16;
1399 try testing.expect(used_buffer_id >= 0 and used_buffer_id <= 3);
1400 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
1401
1402 try testing.expectEqual(@as(u64, 0xdededede), cqe.user_data);
1403 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
1404 }
1405
1406 // This read should fail
1407
1408 {
1409 const sqe = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
1410 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
1411 try testing.expectEqual(@as(i32, fd), sqe.fd);
1412 try testing.expectEqual(@as(u64, 0), sqe.addr);
1413 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
1414 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
1415 try testing.expectEqual(@as(u32, 1), try ring.submit());
1416
1417 const cqe = try ring.copy_cqe();
1418 switch (cqe.err()) {
1419 // Expected
1420 .NOBUFS => {},
1421 .SUCCESS => std.debug.panic("unexpected success", .{}),
1422 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1423 }
1424 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
1425 }
1426
1427 // Provide 1 buffer again
1428
1429 // Deliberately put something we don't expect in the buffers
1430 @memset(mem.sliceAsBytes(&buffers), 42);
1431
1432 const reprovided_buffer_id = 2;
1433
1434 {
1435 _ = try ring.provide_buffers(0xabababab, @as([*]u8, @ptrCast(&buffers[reprovided_buffer_id])), buffer_len, 1, group_id, reprovided_buffer_id);
1436 try testing.expectEqual(@as(u32, 1), try ring.submit());
1437
1438 const cqe = try ring.copy_cqe();
1439 switch (cqe.err()) {
1440 .SUCCESS => {},
1441 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1442 }
1443 }
1444
1445 // Final read which should work
1446
1447 {
1448 const sqe = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
1449 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
1450 try testing.expectEqual(@as(i32, fd), sqe.fd);
1451 try testing.expectEqual(@as(u64, 0), sqe.addr);
1452 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
1453 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
1454 try testing.expectEqual(@as(u32, 1), try ring.submit());
1455
1456 const cqe = try ring.copy_cqe();
1457 switch (cqe.err()) {
1458 .SUCCESS => {},
1459 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1460 }
1461
1462 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
1463 const used_buffer_id = cqe.flags >> 16;
1464 try testing.expectEqual(used_buffer_id, reprovided_buffer_id);
1465 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
1466 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
1467 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
1468 }
1469}
1470
1471test "remove_buffers" {
1472 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1473 error.SystemOutdated => return error.SkipZigTest,
1474 error.PermissionDenied => return error.SkipZigTest,
1475 else => return err,
1476 };
1477 defer ring.deinit();
1478
1479 const fd = try posix.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
1480 defer posix.close(fd);
1481
1482 const group_id = 1337;
1483 const buffer_id = 0;
1484
1485 const buffer_len = 128;
1486
1487 var buffers: [4][buffer_len]u8 = undefined;
1488
1489 // Provide 4 buffers
1490
1491 {
1492 _ = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
1493 try testing.expectEqual(@as(u32, 1), try ring.submit());
1494
1495 const cqe = try ring.copy_cqe();
1496 switch (cqe.err()) {
1497 .INVAL, .BADF => return error.SkipZigTest,
1498 .SUCCESS => {},
1499 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1500 }
1501 try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
1502 }
1503
1504 // Remove 3 buffers
1505
1506 {
1507 const sqe = try ring.remove_buffers(0xbababababa, 3, group_id);
1508 try testing.expectEqual(linux.IORING_OP.REMOVE_BUFFERS, sqe.opcode);
1509 try testing.expectEqual(@as(i32, 3), sqe.fd);
1510 try testing.expectEqual(@as(u64, 0), sqe.addr);
1511 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
1512 try testing.expectEqual(@as(u32, 1), try ring.submit());
1513
1514 const cqe = try ring.copy_cqe();
1515 switch (cqe.err()) {
1516 .SUCCESS => {},
1517 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1518 }
1519 try testing.expectEqual(@as(u64, 0xbababababa), cqe.user_data);
1520 }
1521
1522 // This read should work
1523
1524 {
1525 _ = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
1526 try testing.expectEqual(@as(u32, 1), try ring.submit());
1527
1528 const cqe = try ring.copy_cqe();
1529 switch (cqe.err()) {
1530 .SUCCESS => {},
1531 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1532 }
1533
1534 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
1535 const used_buffer_id = cqe.flags >> 16;
1536 try testing.expect(used_buffer_id >= 0 and used_buffer_id < 4);
1537 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
1538 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
1539 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
1540 }
1541
1542 // Final read should _not_ work
1543
1544 {
1545 _ = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
1546 try testing.expectEqual(@as(u32, 1), try ring.submit());
1547
1548 const cqe = try ring.copy_cqe();
1549 switch (cqe.err()) {
1550 // Expected
1551 .NOBUFS => {},
1552 .SUCCESS => std.debug.panic("unexpected success", .{}),
1553 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1554 }
1555 }
1556}
1557
1558test "provide_buffers: accept/connect/send/recv" {
1559 const io = testing.io;
1560 _ = io;
1561
1562 var ring = IoUring.init(16, 0) catch |err| switch (err) {
1563 error.SystemOutdated => return error.SkipZigTest,
1564 error.PermissionDenied => return error.SkipZigTest,
1565 else => return err,
1566 };
1567 defer ring.deinit();
1568
1569 const group_id = 1337;
1570 const buffer_id = 0;
1571
1572 const buffer_len = 128;
1573 var buffers: [4][buffer_len]u8 = undefined;
1574
1575 // Provide 4 buffers
1576
1577 {
1578 const sqe = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
1579 try testing.expectEqual(linux.IORING_OP.PROVIDE_BUFFERS, sqe.opcode);
1580 try testing.expectEqual(@as(i32, buffers.len), sqe.fd);
1581 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
1582 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
1583 try testing.expectEqual(@as(u32, 1), try ring.submit());
1584
1585 const cqe = try ring.copy_cqe();
1586 switch (cqe.err()) {
1587 // Happens when the kernel is < 5.7
1588 .INVAL => return error.SkipZigTest,
1589 // Happens on the kernel 5.4
1590 .BADF => return error.SkipZigTest,
1591 .SUCCESS => {},
1592 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1593 }
1594 try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
1595 }
1596
1597 const socket_test_harness = try createSocketTestHarness(&ring);
1598 defer socket_test_harness.close();
1599
1600 // Do 4 send on the socket
1601
1602 {
1603 var i: usize = 0;
1604 while (i < buffers.len) : (i += 1) {
1605 _ = try ring.send(0xdeaddead, socket_test_harness.server, &([_]u8{'z'} ** buffer_len), 0);
1606 try testing.expectEqual(@as(u32, 1), try ring.submit());
1607 }
1608
1609 var cqes: [4]linux.io_uring_cqe = undefined;
1610 try testing.expectEqual(@as(u32, 4), try ring.copy_cqes(&cqes, 4));
1611 }
1612
1613 // Do 4 recv which should consume all buffers
1614
1615 // Deliberately put something we don't expect in the buffers
1616 @memset(mem.sliceAsBytes(&buffers), 1);
1617
1618 var i: usize = 0;
1619 while (i < buffers.len) : (i += 1) {
1620 const sqe = try ring.recv(0xdededede, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
1621 try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
1622 try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
1623 try testing.expectEqual(@as(u64, 0), sqe.addr);
1624 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
1625 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
1626 try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
1627 try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
1628 try testing.expectEqual(@as(u32, 1), try ring.submit());
1629
1630 const cqe = try ring.copy_cqe();
1631 switch (cqe.err()) {
1632 .SUCCESS => {},
1633 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1634 }
1635
1636 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
1637 const used_buffer_id = cqe.flags >> 16;
1638 try testing.expect(used_buffer_id >= 0 and used_buffer_id <= 3);
1639 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
1640
1641 try testing.expectEqual(@as(u64, 0xdededede), cqe.user_data);
1642 const buffer = buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))];
1643 try testing.expectEqualSlices(u8, &([_]u8{'z'} ** buffer_len), buffer);
1644 }
1645
1646 // This recv should fail
1647
1648 {
1649 const sqe = try ring.recv(0xdfdfdfdf, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
1650 try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
1651 try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
1652 try testing.expectEqual(@as(u64, 0), sqe.addr);
1653 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
1654 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
1655 try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
1656 try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
1657 try testing.expectEqual(@as(u32, 1), try ring.submit());
1658
1659 const cqe = try ring.copy_cqe();
1660 switch (cqe.err()) {
1661 // Expected
1662 .NOBUFS => {},
1663 .SUCCESS => std.debug.panic("unexpected success", .{}),
1664 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1665 }
1666 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
1667 }
1668
1669 // Provide 1 buffer again
1670
1671 const reprovided_buffer_id = 2;
1672
1673 {
1674 _ = try ring.provide_buffers(0xabababab, @as([*]u8, @ptrCast(&buffers[reprovided_buffer_id])), buffer_len, 1, group_id, reprovided_buffer_id);
1675 try testing.expectEqual(@as(u32, 1), try ring.submit());
1676
1677 const cqe = try ring.copy_cqe();
1678 switch (cqe.err()) {
1679 .SUCCESS => {},
1680 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1681 }
1682 }
1683
1684 // Redo 1 send on the server socket
1685
1686 {
1687 _ = try ring.send(0xdeaddead, socket_test_harness.server, &([_]u8{'w'} ** buffer_len), 0);
1688 try testing.expectEqual(@as(u32, 1), try ring.submit());
1689
1690 _ = try ring.copy_cqe();
1691 }
1692
1693 // Final recv which should work
1694
1695 // Deliberately put something we don't expect in the buffers
1696 @memset(mem.sliceAsBytes(&buffers), 1);
1697
1698 {
1699 const sqe = try ring.recv(0xdfdfdfdf, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
1700 try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
1701 try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
1702 try testing.expectEqual(@as(u64, 0), sqe.addr);
1703 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
1704 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
1705 try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
1706 try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
1707 try testing.expectEqual(@as(u32, 1), try ring.submit());
1708
1709 const cqe = try ring.copy_cqe();
1710 switch (cqe.err()) {
1711 .SUCCESS => {},
1712 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
1713 }
1714
1715 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
1716 const used_buffer_id = cqe.flags >> 16;
1717 try testing.expectEqual(used_buffer_id, reprovided_buffer_id);
1718 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
1719 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
1720 const buffer = buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))];
1721 try testing.expectEqualSlices(u8, &([_]u8{'w'} ** buffer_len), buffer);
1722 }
1723}
1724
1725test "accept multishot" {
1726 var ring = IoUring.init(16, 0) catch |err| switch (err) {
1727 error.SystemOutdated => return error.SkipZigTest,
1728 error.PermissionDenied => return error.SkipZigTest,
1729 else => return err,
1730 };
1731 defer ring.deinit();
1732
1733 var address: linux.sockaddr.in = .{
1734 .port = 0,
1735 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
1736 };
1737 const listener_socket = try createListenerSocket(&address);
1738 defer posix.close(listener_socket);
1739
1740 // submit multishot accept operation
1741 var addr: posix.sockaddr = undefined;
1742 var addr_len: posix.socklen_t = @sizeOf(@TypeOf(addr));
1743 const userdata: u64 = 0xaaaaaaaa;
1744 _ = try ring.accept_multishot(userdata, listener_socket, &addr, &addr_len, 0);
1745 try testing.expectEqual(@as(u32, 1), try ring.submit());
1746
1747 var nr: usize = 4; // number of clients to connect
1748 while (nr > 0) : (nr -= 1) {
1749 // connect client
1750 const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
1751 errdefer posix.close(client);
1752 try posix.connect(client, addrAny(&address), @sizeOf(linux.sockaddr.in));
1753
1754 // test accept completion
1755 var cqe = try ring.copy_cqe();
1756 if (cqe.err() == .INVAL) return error.SkipZigTest;
1757 try testing.expect(cqe.res > 0);
1758 try testing.expect(cqe.user_data == userdata);
1759 try testing.expect(cqe.flags & linux.IORING_CQE_F_MORE > 0); // more flag is set
1760
1761 posix.close(client);
1762 }
1763}
1764
1765test "accept/connect/send_zc/recv" {
1766 try skipKernelLessThan(.{ .major = 6, .minor = 0, .patch = 0 });
1767
1768 const io = testing.io;
1769 _ = io;
1770
1771 var ring = IoUring.init(16, 0) catch |err| switch (err) {
1772 error.SystemOutdated => return error.SkipZigTest,
1773 error.PermissionDenied => return error.SkipZigTest,
1774 else => return err,
1775 };
1776 defer ring.deinit();
1777
1778 const socket_test_harness = try createSocketTestHarness(&ring);
1779 defer socket_test_harness.close();
1780
1781 const buffer_send = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
1782 var buffer_recv = [_]u8{0} ** 10;
1783
1784 // zero-copy send
1785 const sqe_send = try ring.send_zc(0xeeeeeeee, socket_test_harness.client, buffer_send[0..], 0, 0);
1786 sqe_send.flags |= linux.IOSQE_IO_LINK;
1787 _ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
1788 try testing.expectEqual(@as(u32, 2), try ring.submit());
1789
1790 var cqe_send = try ring.copy_cqe();
1791 // First completion of zero-copy send.
1792 // IORING_CQE_F_MORE, means that there
1793 // will be a second completion event / notification for the
1794 // request, with the user_data field set to the same value.
1795 // buffer_send must be keep alive until second cqe.
1796 try testing.expectEqual(linux.io_uring_cqe{
1797 .user_data = 0xeeeeeeee,
1798 .res = buffer_send.len,
1799 .flags = linux.IORING_CQE_F_MORE,
1800 }, cqe_send);
1801
1802 cqe_send, const cqe_recv = brk: {
1803 const cqe1 = try ring.copy_cqe();
1804 const cqe2 = try ring.copy_cqe();
1805 break :brk if (cqe1.user_data == 0xeeeeeeee) .{ cqe1, cqe2 } else .{ cqe2, cqe1 };
1806 };
1807
1808 try testing.expectEqual(linux.io_uring_cqe{
1809 .user_data = 0xffffffff,
1810 .res = buffer_recv.len,
1811 .flags = cqe_recv.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
1812 }, cqe_recv);
1813 try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
1814
1815 // Second completion of zero-copy send.
1816 // IORING_CQE_F_NOTIF in flags signals that kernel is done with send_buffer
1817 try testing.expectEqual(linux.io_uring_cqe{
1818 .user_data = 0xeeeeeeee,
1819 .res = 0,
1820 .flags = linux.IORING_CQE_F_NOTIF,
1821 }, cqe_send);
1822}
1823
1824test "accept_direct" {
1825 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
1826
1827 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1828 error.SystemOutdated => return error.SkipZigTest,
1829 error.PermissionDenied => return error.SkipZigTest,
1830 else => return err,
1831 };
1832 defer ring.deinit();
1833 var address: linux.sockaddr.in = .{
1834 .port = 0,
1835 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
1836 };
1837
1838 // register direct file descriptors
1839 var registered_fds = [_]linux.fd_t{-1} ** 2;
1840 try ring.register_files(registered_fds[0..]);
1841
1842 const listener_socket = try createListenerSocket(&address);
1843 defer posix.close(listener_socket);
1844
1845 const accept_userdata: u64 = 0xaaaaaaaa;
1846 const read_userdata: u64 = 0xbbbbbbbb;
1847 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
1848
1849 for (0..2) |_| {
1850 for (registered_fds, 0..) |_, i| {
1851 var buffer_recv = [_]u8{0} ** 16;
1852 const buffer_send: []const u8 = data[0 .. data.len - i]; // make it different at each loop
1853
1854 // submit accept, will chose registered fd and return index in cqe
1855 _ = try ring.accept_direct(accept_userdata, listener_socket, null, null, 0);
1856 try testing.expectEqual(@as(u32, 1), try ring.submit());
1857
1858 // connect
1859 const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
1860 try posix.connect(client, addrAny(&address), @sizeOf(linux.sockaddr.in));
1861 defer posix.close(client);
1862
1863 // accept completion
1864 const cqe_accept = try ring.copy_cqe();
1865 try testing.expectEqual(posix.E.SUCCESS, cqe_accept.err());
1866 const fd_index = cqe_accept.res;
1867 try testing.expect(fd_index < registered_fds.len);
1868 try testing.expect(cqe_accept.user_data == accept_userdata);
1869
1870 // send data
1871 _ = try posix.send(client, buffer_send, 0);
1872
1873 // Example of how to use registered fd:
1874 // Submit receive to fixed file returned by accept (fd_index).
1875 // Fd field is set to registered file index, returned by accept.
1876 // Flag linux.IOSQE_FIXED_FILE must be set.
1877 const recv_sqe = try ring.recv(read_userdata, fd_index, .{ .buffer = &buffer_recv }, 0);
1878 recv_sqe.flags |= linux.IOSQE_FIXED_FILE;
1879 try testing.expectEqual(@as(u32, 1), try ring.submit());
1880
1881 // accept receive
1882 const recv_cqe = try ring.copy_cqe();
1883 try testing.expect(recv_cqe.user_data == read_userdata);
1884 try testing.expect(recv_cqe.res == buffer_send.len);
1885 try testing.expectEqualSlices(u8, buffer_send, buffer_recv[0..buffer_send.len]);
1886 }
1887 // no more available fds, accept will get NFILE error
1888 {
1889 // submit accept
1890 _ = try ring.accept_direct(accept_userdata, listener_socket, null, null, 0);
1891 try testing.expectEqual(@as(u32, 1), try ring.submit());
1892 // connect
1893 const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
1894 try posix.connect(client, addrAny(&address), @sizeOf(linux.sockaddr.in));
1895 defer posix.close(client);
1896 // completion with error
1897 const cqe_accept = try ring.copy_cqe();
1898 try testing.expect(cqe_accept.user_data == accept_userdata);
1899 try testing.expectEqual(posix.E.NFILE, cqe_accept.err());
1900 }
1901 // return file descriptors to kernel
1902 try ring.register_files_update(0, registered_fds[0..]);
1903 }
1904 try ring.unregister_files();
1905}
1906
1907test "accept_multishot_direct" {
1908 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
1909
1910 if (builtin.cpu.arch == .riscv64) {
1911 // https://github.com/ziglang/zig/issues/25734
1912 return error.SkipZigTest;
1913 }
1914
1915 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1916 error.SystemOutdated => return error.SkipZigTest,
1917 error.PermissionDenied => return error.SkipZigTest,
1918 else => return err,
1919 };
1920 defer ring.deinit();
1921
1922 var address: linux.sockaddr.in = .{
1923 .port = 0,
1924 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
1925 };
1926
1927 var registered_fds = [_]linux.fd_t{-1} ** 2;
1928 try ring.register_files(registered_fds[0..]);
1929
1930 const listener_socket = try createListenerSocket(&address);
1931 defer posix.close(listener_socket);
1932
1933 const accept_userdata: u64 = 0xaaaaaaaa;
1934
1935 for (0..2) |_| {
1936 // submit multishot accept
1937 // Will chose registered fd and return index of the selected registered file in cqe.
1938 _ = try ring.accept_multishot_direct(accept_userdata, listener_socket, null, null, 0);
1939 try testing.expectEqual(@as(u32, 1), try ring.submit());
1940
1941 for (registered_fds) |_| {
1942 // connect
1943 const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
1944 try posix.connect(client, addrAny(&address), @sizeOf(linux.sockaddr.in));
1945 defer posix.close(client);
1946
1947 // accept completion
1948 const cqe_accept = try ring.copy_cqe();
1949 const fd_index = cqe_accept.res;
1950 try testing.expect(fd_index < registered_fds.len);
1951 try testing.expect(cqe_accept.user_data == accept_userdata);
1952 try testing.expect(cqe_accept.flags & linux.IORING_CQE_F_MORE > 0); // has more is set
1953 }
1954 // No more available fds, accept will get NFILE error.
1955 // Multishot is terminated (more flag is not set).
1956 {
1957 // connect
1958 const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
1959 try posix.connect(client, addrAny(&address), @sizeOf(linux.sockaddr.in));
1960 defer posix.close(client);
1961 // completion with error
1962 const cqe_accept = try ring.copy_cqe();
1963 try testing.expect(cqe_accept.user_data == accept_userdata);
1964 try testing.expectEqual(posix.E.NFILE, cqe_accept.err());
1965 try testing.expect(cqe_accept.flags & linux.IORING_CQE_F_MORE == 0); // has more is not set
1966 }
1967 // return file descriptors to kernel
1968 try ring.register_files_update(0, registered_fds[0..]);
1969 }
1970 try ring.unregister_files();
1971}
1972
1973test "socket" {
1974 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
1975
1976 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1977 error.SystemOutdated => return error.SkipZigTest,
1978 error.PermissionDenied => return error.SkipZigTest,
1979 else => return err,
1980 };
1981 defer ring.deinit();
1982
1983 // prepare, submit socket operation
1984 _ = try ring.socket(0, linux.AF.INET, posix.SOCK.STREAM, 0, 0);
1985 try testing.expectEqual(@as(u32, 1), try ring.submit());
1986
1987 // test completion
1988 var cqe = try ring.copy_cqe();
1989 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
1990 const fd: linux.fd_t = @intCast(cqe.res);
1991 try testing.expect(fd > 2);
1992
1993 posix.close(fd);
1994}
1995
1996test "socket_direct/socket_direct_alloc/close_direct" {
1997 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
1998
1999 var ring = IoUring.init(2, 0) catch |err| switch (err) {
2000 error.SystemOutdated => return error.SkipZigTest,
2001 error.PermissionDenied => return error.SkipZigTest,
2002 else => return err,
2003 };
2004 defer ring.deinit();
2005
2006 var registered_fds = [_]linux.fd_t{-1} ** 3;
2007 try ring.register_files(registered_fds[0..]);
2008
2009 // create socket in registered file descriptor at index 0 (last param)
2010 _ = try ring.socket_direct(0, linux.AF.INET, posix.SOCK.STREAM, 0, 0, 0);
2011 try testing.expectEqual(@as(u32, 1), try ring.submit());
2012 var cqe_socket = try ring.copy_cqe();
2013 try testing.expectEqual(posix.E.SUCCESS, cqe_socket.err());
2014 try testing.expect(cqe_socket.res == 0);
2015
2016 // create socket in registered file descriptor at index 1 (last param)
2017 _ = try ring.socket_direct(0, linux.AF.INET, posix.SOCK.STREAM, 0, 0, 1);
2018 try testing.expectEqual(@as(u32, 1), try ring.submit());
2019 cqe_socket = try ring.copy_cqe();
2020 try testing.expectEqual(posix.E.SUCCESS, cqe_socket.err());
2021 try testing.expect(cqe_socket.res == 0); // res is 0 when index is specified
2022
2023 // create socket in kernel chosen file descriptor index (_alloc version)
2024 // completion res has index from registered files
2025 _ = try ring.socket_direct_alloc(0, linux.AF.INET, posix.SOCK.STREAM, 0, 0);
2026 try testing.expectEqual(@as(u32, 1), try ring.submit());
2027 cqe_socket = try ring.copy_cqe();
2028 try testing.expectEqual(posix.E.SUCCESS, cqe_socket.err());
2029 try testing.expect(cqe_socket.res == 2); // returns registered file index
2030
2031 // use sockets from registered_fds in connect operation
2032 var address: linux.sockaddr.in = .{
2033 .port = 0,
2034 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
2035 };
2036 const listener_socket = try createListenerSocket(&address);
2037 defer posix.close(listener_socket);
2038 const accept_userdata: u64 = 0xaaaaaaaa;
2039 const connect_userdata: u64 = 0xbbbbbbbb;
2040 const close_userdata: u64 = 0xcccccccc;
2041 for (registered_fds, 0..) |_, fd_index| {
2042 // prepare accept
2043 _ = try ring.accept(accept_userdata, listener_socket, null, null, 0);
2044 // prepare connect with fixed socket
2045 const connect_sqe = try ring.connect(connect_userdata, @intCast(fd_index), addrAny(&address), @sizeOf(linux.sockaddr.in));
2046 connect_sqe.flags |= linux.IOSQE_FIXED_FILE; // fd is fixed file index
2047 // submit both
2048 try testing.expectEqual(@as(u32, 2), try ring.submit());
2049 // get completions
2050 var cqe_connect = try ring.copy_cqe();
2051 var cqe_accept = try ring.copy_cqe();
2052 // ignore order
2053 if (cqe_connect.user_data == accept_userdata and cqe_accept.user_data == connect_userdata) {
2054 const a = cqe_accept;
2055 const b = cqe_connect;
2056 cqe_accept = b;
2057 cqe_connect = a;
2058 }
2059 // test connect completion
2060 try testing.expect(cqe_connect.user_data == connect_userdata);
2061 try testing.expectEqual(posix.E.SUCCESS, cqe_connect.err());
2062 // test accept completion
2063 try testing.expect(cqe_accept.user_data == accept_userdata);
2064 try testing.expectEqual(posix.E.SUCCESS, cqe_accept.err());
2065
2066 // submit and test close_direct
2067 _ = try ring.close_direct(close_userdata, @intCast(fd_index));
2068 try testing.expectEqual(@as(u32, 1), try ring.submit());
2069 var cqe_close = try ring.copy_cqe();
2070 try testing.expect(cqe_close.user_data == close_userdata);
2071 try testing.expectEqual(posix.E.SUCCESS, cqe_close.err());
2072 }
2073
2074 try ring.unregister_files();
2075}
2076
2077test "openat_direct/close_direct" {
2078 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
2079
2080 var ring = IoUring.init(2, 0) catch |err| switch (err) {
2081 error.SystemOutdated => return error.SkipZigTest,
2082 error.PermissionDenied => return error.SkipZigTest,
2083 else => return err,
2084 };
2085 defer ring.deinit();
2086
2087 var registered_fds = [_]linux.fd_t{-1} ** 3;
2088 try ring.register_files(registered_fds[0..]);
2089
2090 var tmp = std.testing.tmpDir(.{});
2091 defer tmp.cleanup();
2092 const path = "test_io_uring_close_direct";
2093 const flags: linux.O = .{ .ACCMODE = .RDWR, .CREAT = true };
2094 const mode: posix.mode_t = 0o666;
2095 const user_data: u64 = 0;
2096
2097 // use registered file at index 0 (last param)
2098 _ = try ring.openat_direct(user_data, tmp.dir.handle, path, flags, mode, 0);
2099 try testing.expectEqual(@as(u32, 1), try ring.submit());
2100 var cqe = try ring.copy_cqe();
2101 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
2102 try testing.expect(cqe.res == 0);
2103
2104 // use registered file at index 1
2105 _ = try ring.openat_direct(user_data, tmp.dir.handle, path, flags, mode, 1);
2106 try testing.expectEqual(@as(u32, 1), try ring.submit());
2107 cqe = try ring.copy_cqe();
2108 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
2109 try testing.expect(cqe.res == 0); // res is 0 when we specify index
2110
2111 // let kernel choose registered file index
2112 _ = try ring.openat_direct(user_data, tmp.dir.handle, path, flags, mode, linux.IORING_FILE_INDEX_ALLOC);
2113 try testing.expectEqual(@as(u32, 1), try ring.submit());
2114 cqe = try ring.copy_cqe();
2115 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
2116 try testing.expect(cqe.res == 2); // chosen index is in res
2117
2118 // close all open file descriptors
2119 for (registered_fds, 0..) |_, fd_index| {
2120 _ = try ring.close_direct(user_data, @intCast(fd_index));
2121 try testing.expectEqual(@as(u32, 1), try ring.submit());
2122 var cqe_close = try ring.copy_cqe();
2123 try testing.expectEqual(posix.E.SUCCESS, cqe_close.err());
2124 }
2125 try ring.unregister_files();
2126}
2127
2128test "ring mapped buffers recv" {
2129 const io = testing.io;
2130 _ = io;
2131
2132 var ring = IoUring.init(16, 0) catch |err| switch (err) {
2133 error.SystemOutdated => return error.SkipZigTest,
2134 error.PermissionDenied => return error.SkipZigTest,
2135 else => return err,
2136 };
2137 defer ring.deinit();
2138
2139 // init buffer group
2140 const group_id: u16 = 1; // buffers group id
2141 const buffers_count: u16 = 2; // number of buffers in buffer group
2142 const buffer_size: usize = 4; // size of each buffer in group
2143 var buf_grp = BufferGroup.init(
2144 &ring,
2145 testing.allocator,
2146 group_id,
2147 buffer_size,
2148 buffers_count,
2149 ) catch |err| switch (err) {
2150 // kernel older than 5.19
2151 error.ArgumentsInvalid => return error.SkipZigTest,
2152 else => return err,
2153 };
2154 defer buf_grp.deinit(testing.allocator);
2155
2156 // create client/server fds
2157 const fds = try createSocketTestHarness(&ring);
2158 defer fds.close();
2159
2160 // for random user_data in sqe/cqe
2161 var Rnd = std.Random.DefaultPrng.init(std.testing.random_seed);
2162 var rnd = Rnd.random();
2163
2164 var round: usize = 4; // repeat send/recv cycle round times
2165 while (round > 0) : (round -= 1) {
2166 // client sends data
2167 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
2168 {
2169 const user_data = rnd.int(u64);
2170 _ = try ring.send(user_data, fds.client, data[0..], 0);
2171 try testing.expectEqual(@as(u32, 1), try ring.submit());
2172 const cqe_send = try ring.copy_cqe();
2173 if (cqe_send.err() == .INVAL) return error.SkipZigTest;
2174 try testing.expectEqual(linux.io_uring_cqe{ .user_data = user_data, .res = data.len, .flags = 0 }, cqe_send);
2175 }
2176 var pos: usize = 0;
2177
2178 // read first chunk
2179 const cqe1 = try buf_grp_recv_submit_get_cqe(&ring, &buf_grp, fds.server, rnd.int(u64));
2180 var buf = try buf_grp.get(cqe1);
2181 try testing.expectEqualSlices(u8, data[pos..][0..buf.len], buf);
2182 pos += buf.len;
2183 // second chunk
2184 const cqe2 = try buf_grp_recv_submit_get_cqe(&ring, &buf_grp, fds.server, rnd.int(u64));
2185 buf = try buf_grp.get(cqe2);
2186 try testing.expectEqualSlices(u8, data[pos..][0..buf.len], buf);
2187 pos += buf.len;
2188
2189 // both buffers provided to the kernel are used so we get error
2190 // 'no more buffers', until we put buffers to the kernel
2191 {
2192 const user_data = rnd.int(u64);
2193 _ = try buf_grp.recv(user_data, fds.server, 0);
2194 try testing.expectEqual(@as(u32, 1), try ring.submit());
2195 const cqe = try ring.copy_cqe();
2196 try testing.expectEqual(user_data, cqe.user_data);
2197 try testing.expect(cqe.res < 0); // fail
2198 try testing.expectEqual(posix.E.NOBUFS, cqe.err());
2199 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == 0); // IORING_CQE_F_BUFFER flags is set on success only
2200 try testing.expectError(error.NoBufferSelected, cqe.buffer_id());
2201 }
2202
2203 // put buffers back to the kernel
2204 try buf_grp.put(cqe1);
2205 try buf_grp.put(cqe2);
2206
2207 // read remaining data
2208 while (pos < data.len) {
2209 const cqe = try buf_grp_recv_submit_get_cqe(&ring, &buf_grp, fds.server, rnd.int(u64));
2210 buf = try buf_grp.get(cqe);
2211 try testing.expectEqualSlices(u8, data[pos..][0..buf.len], buf);
2212 pos += buf.len;
2213 try buf_grp.put(cqe);
2214 }
2215 }
2216}
2217
2218test "ring mapped buffers multishot recv" {
2219 const io = testing.io;
2220 _ = io;
2221
2222 var ring = IoUring.init(16, 0) catch |err| switch (err) {
2223 error.SystemOutdated => return error.SkipZigTest,
2224 error.PermissionDenied => return error.SkipZigTest,
2225 else => return err,
2226 };
2227 defer ring.deinit();
2228
2229 // init buffer group
2230 const group_id: u16 = 1; // buffers group id
2231 const buffers_count: u16 = 2; // number of buffers in buffer group
2232 const buffer_size: usize = 4; // size of each buffer in group
2233 var buf_grp = BufferGroup.init(
2234 &ring,
2235 testing.allocator,
2236 group_id,
2237 buffer_size,
2238 buffers_count,
2239 ) catch |err| switch (err) {
2240 // kernel older than 5.19
2241 error.ArgumentsInvalid => return error.SkipZigTest,
2242 else => return err,
2243 };
2244 defer buf_grp.deinit(testing.allocator);
2245
2246 // create client/server fds
2247 const fds = try createSocketTestHarness(&ring);
2248 defer fds.close();
2249
2250 // for random user_data in sqe/cqe
2251 var Rnd = std.Random.DefaultPrng.init(std.testing.random_seed);
2252 var rnd = Rnd.random();
2253
2254 var round: usize = 4; // repeat send/recv cycle round times
2255 while (round > 0) : (round -= 1) {
2256 // client sends data
2257 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf };
2258 {
2259 const user_data = rnd.int(u64);
2260 _ = try ring.send(user_data, fds.client, data[0..], 0);
2261 try testing.expectEqual(@as(u32, 1), try ring.submit());
2262 const cqe_send = try ring.copy_cqe();
2263 if (cqe_send.err() == .INVAL) return error.SkipZigTest;
2264 try testing.expectEqual(linux.io_uring_cqe{ .user_data = user_data, .res = data.len, .flags = 0 }, cqe_send);
2265 }
2266
2267 // start multishot recv
2268 var recv_user_data = rnd.int(u64);
2269 _ = try buf_grp.recv_multishot(recv_user_data, fds.server, 0);
2270 try testing.expectEqual(@as(u32, 1), try ring.submit()); // submit
2271
2272 // server reads data into provided buffers
2273 // there are 2 buffers of size 4, so each read gets only chunk of data
2274 // we read four chunks of 4, 4, 4, 4 bytes each
2275 var chunk: []const u8 = data[0..buffer_size]; // first chunk
2276 const cqe1 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);
2277 try testing.expect(cqe1.flags & linux.IORING_CQE_F_MORE > 0);
2278
2279 chunk = data[buffer_size .. buffer_size * 2]; // second chunk
2280 const cqe2 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);
2281 try testing.expect(cqe2.flags & linux.IORING_CQE_F_MORE > 0);
2282
2283 // both buffers provided to the kernel are used so we get error
2284 // 'no more buffers', until we put buffers to the kernel
2285 {
2286 const cqe = try ring.copy_cqe();
2287 try testing.expectEqual(recv_user_data, cqe.user_data);
2288 try testing.expect(cqe.res < 0); // fail
2289 try testing.expectEqual(posix.E.NOBUFS, cqe.err());
2290 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == 0); // IORING_CQE_F_BUFFER flags is set on success only
2291 // has more is not set
2292 // indicates that multishot is finished
2293 try testing.expect(cqe.flags & linux.IORING_CQE_F_MORE == 0);
2294 try testing.expectError(error.NoBufferSelected, cqe.buffer_id());
2295 }
2296
2297 // put buffers back to the kernel
2298 try buf_grp.put(cqe1);
2299 try buf_grp.put(cqe2);
2300
2301 // restart multishot
2302 recv_user_data = rnd.int(u64);
2303 _ = try buf_grp.recv_multishot(recv_user_data, fds.server, 0);
2304 try testing.expectEqual(@as(u32, 1), try ring.submit()); // submit
2305
2306 chunk = data[buffer_size * 2 .. buffer_size * 3]; // third chunk
2307 const cqe3 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);
2308 try testing.expect(cqe3.flags & linux.IORING_CQE_F_MORE > 0);
2309 try buf_grp.put(cqe3);
2310
2311 chunk = data[buffer_size * 3 ..]; // last chunk
2312 const cqe4 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);
2313 try testing.expect(cqe4.flags & linux.IORING_CQE_F_MORE > 0);
2314 try buf_grp.put(cqe4);
2315
2316 // cancel pending multishot recv operation
2317 {
2318 const cancel_user_data = rnd.int(u64);
2319 _ = try ring.cancel(cancel_user_data, recv_user_data, 0);
2320 try testing.expectEqual(@as(u32, 1), try ring.submit());
2321
2322 // expect completion of cancel operation and completion of recv operation
2323 var cqe_cancel = try ring.copy_cqe();
2324 if (cqe_cancel.err() == .INVAL) return error.SkipZigTest;
2325 var cqe_recv = try ring.copy_cqe();
2326 if (cqe_recv.err() == .INVAL) return error.SkipZigTest;
2327
2328 // don't depend on order of completions
2329 if (cqe_cancel.user_data == recv_user_data and cqe_recv.user_data == cancel_user_data) {
2330 const a = cqe_cancel;
2331 const b = cqe_recv;
2332 cqe_cancel = b;
2333 cqe_recv = a;
2334 }
2335
2336 // Note on different kernel results:
2337 // on older kernel (tested with v6.0.16, v6.1.57, v6.2.12, v6.4.16)
2338 // cqe_cancel.err() == .NOENT
2339 // cqe_recv.err() == .NOBUFS
2340 // on kernel (tested with v6.5.0, v6.5.7)
2341 // cqe_cancel.err() == .SUCCESS
2342 // cqe_recv.err() == .CANCELED
2343 // Upstream reference: https://github.com/axboe/liburing/issues/984
2344
2345 // cancel operation is success (or NOENT on older kernels)
2346 try testing.expectEqual(cancel_user_data, cqe_cancel.user_data);
2347 try testing.expect(cqe_cancel.err() == .NOENT or cqe_cancel.err() == .SUCCESS);
2348
2349 // recv operation is failed with err CANCELED (or NOBUFS on older kernels)
2350 try testing.expectEqual(recv_user_data, cqe_recv.user_data);
2351 try testing.expect(cqe_recv.res < 0);
2352 try testing.expect(cqe_recv.err() == .NOBUFS or cqe_recv.err() == .CANCELED);
2353 try testing.expect(cqe_recv.flags & linux.IORING_CQE_F_MORE == 0);
2354 }
2355 }
2356}
2357
2358test "copy_cqes with wrapping sq.cqes buffer" {
2359 var ring = IoUring.init(2, 0) catch |err| switch (err) {
2360 error.SystemOutdated => return error.SkipZigTest,
2361 error.PermissionDenied => return error.SkipZigTest,
2362 else => return err,
2363 };
2364 defer ring.deinit();
2365
2366 try testing.expectEqual(2, ring.sq.sqes.len);
2367 try testing.expectEqual(4, ring.cq.cqes.len);
2368
2369 // submit 2 entries, receive 2 completions
2370 var cqes: [8]linux.io_uring_cqe = undefined;
2371 {
2372 for (0..2) |_| {
2373 const sqe = try ring.get_sqe();
2374 sqe.prep_timeout(&.{ .sec = 0, .nsec = 10000 }, 0, 0);
2375 try testing.expect(try ring.submit() == 1);
2376 }
2377 var cqe_count: u32 = 0;
2378 while (cqe_count < 2) {
2379 cqe_count += try ring.copy_cqes(&cqes, 2 - cqe_count);
2380 }
2381 }
2382
2383 try testing.expectEqual(2, ring.cq.head.*);
2384
2385 // sq.sqes len is 4, starting at position 2
2386 // every 4 entries submit wraps completion buffer
2387 // we are reading ring.cq.cqes at indexes 2,3,0,1
2388 for (1..1024) |i| {
2389 for (0..4) |_| {
2390 const sqe = try ring.get_sqe();
2391 sqe.prep_timeout(&.{ .sec = 0, .nsec = 10000 }, 0, 0);
2392 try testing.expect(try ring.submit() == 1);
2393 }
2394 var cqe_count: u32 = 0;
2395 while (cqe_count < 4) {
2396 cqe_count += try ring.copy_cqes(&cqes, 4 - cqe_count);
2397 }
2398 try testing.expectEqual(4, cqe_count);
2399 try testing.expectEqual(2 + 4 * i, ring.cq.head.*);
2400 }
2401}
2402
2403test "bind/listen/connect" {
2404 if (builtin.cpu.arch == .s390x) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/25956
2405
2406 var ring = IoUring.init(4, 0) catch |err| switch (err) {
2407 error.SystemOutdated => return error.SkipZigTest,
2408 error.PermissionDenied => return error.SkipZigTest,
2409 else => return err,
2410 };
2411 defer ring.deinit();
2412
2413 const probe = ring.get_probe() catch return error.SkipZigTest;
2414 // LISTEN is higher required operation
2415 if (!probe.is_supported(.LISTEN)) return error.SkipZigTest;
2416
2417 var addr: linux.sockaddr.in = .{
2418 .port = 0,
2419 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
2420 };
2421 const proto: u32 = if (addr.family == linux.AF.UNIX) 0 else linux.IPPROTO.TCP;
2422
2423 const listen_fd = brk: {
2424 // Create socket
2425 _ = try ring.socket(1, addr.family, linux.SOCK.STREAM | linux.SOCK.CLOEXEC, proto, 0);
2426 try testing.expectEqual(1, try ring.submit());
2427 var cqe = try ring.copy_cqe();
2428 try testing.expectEqual(1, cqe.user_data);
2429 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
2430 const listen_fd: linux.fd_t = @intCast(cqe.res);
2431 try testing.expect(listen_fd > 2);
2432
2433 // Prepare: set socket option * 2, bind, listen
2434 var optval: u32 = 1;
2435 (try ring.setsockopt(2, listen_fd, linux.SOL.SOCKET, linux.SO.REUSEADDR, mem.asBytes(&optval))).link_next();
2436 (try ring.setsockopt(3, listen_fd, linux.SOL.SOCKET, linux.SO.REUSEPORT, mem.asBytes(&optval))).link_next();
2437 (try ring.bind(4, listen_fd, addrAny(&addr), @sizeOf(linux.sockaddr.in), 0)).link_next();
2438 _ = try ring.listen(5, listen_fd, 1, 0);
2439 // Submit 4 operations
2440 try testing.expectEqual(4, try ring.submit());
2441 // Expect all to succeed
2442 for (2..6) |user_data| {
2443 cqe = try ring.copy_cqe();
2444 try testing.expectEqual(user_data, cqe.user_data);
2445 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
2446 }
2447
2448 // Check that socket option is set
2449 optval = 0;
2450 _ = try ring.getsockopt(5, listen_fd, linux.SOL.SOCKET, linux.SO.REUSEADDR, mem.asBytes(&optval));
2451 try testing.expectEqual(1, try ring.submit());
2452 cqe = try ring.copy_cqe();
2453 try testing.expectEqual(5, cqe.user_data);
2454 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
2455 try testing.expectEqual(1, optval);
2456
2457 // Read system assigned port into addr
2458 var addr_len: posix.socklen_t = @sizeOf(linux.sockaddr.in);
2459 try posix.getsockname(listen_fd, addrAny(&addr), &addr_len);
2460
2461 break :brk listen_fd;
2462 };
2463
2464 const connect_fd = brk: {
2465 // Create connect socket
2466 _ = try ring.socket(6, addr.family, linux.SOCK.STREAM | linux.SOCK.CLOEXEC, proto, 0);
2467 try testing.expectEqual(1, try ring.submit());
2468 const cqe = try ring.copy_cqe();
2469 try testing.expectEqual(6, cqe.user_data);
2470 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
2471 // Get connect socket fd
2472 const connect_fd: linux.fd_t = @intCast(cqe.res);
2473 try testing.expect(connect_fd > 2 and connect_fd != listen_fd);
2474 break :brk connect_fd;
2475 };
2476
2477 // Prepare accept/connect operations
2478 _ = try ring.accept(7, listen_fd, null, null, 0);
2479 _ = try ring.connect(8, connect_fd, addrAny(&addr), @sizeOf(linux.sockaddr.in));
2480 try testing.expectEqual(2, try ring.submit());
2481 // Get listener accepted socket
2482 var accept_fd: posix.socket_t = 0;
2483 for (0..2) |_| {
2484 const cqe = try ring.copy_cqe();
2485 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
2486 if (cqe.user_data == 7) {
2487 accept_fd = @intCast(cqe.res);
2488 } else {
2489 try testing.expectEqual(8, cqe.user_data);
2490 }
2491 }
2492 try testing.expect(accept_fd > 2 and accept_fd != listen_fd and accept_fd != connect_fd);
2493
2494 // Communicate
2495 try testSendRecv(&ring, connect_fd, accept_fd);
2496 try testSendRecv(&ring, accept_fd, connect_fd);
2497
2498 // Shutdown and close all sockets
2499 for ([_]posix.socket_t{ connect_fd, accept_fd, listen_fd }) |fd| {
2500 (try ring.shutdown(9, fd, posix.SHUT.RDWR)).link_next();
2501 _ = try ring.close(10, fd);
2502 try testing.expectEqual(2, try ring.submit());
2503 for (0..2) |i| {
2504 const cqe = try ring.copy_cqe();
2505 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
2506 try testing.expectEqual(9 + i, cqe.user_data);
2507 }
2508 }
2509}
2510
2511// Prepare, submit recv and get cqe using buffer group.
2512fn buf_grp_recv_submit_get_cqe(
2513 ring: *IoUring,
2514 buf_grp: *BufferGroup,
2515 fd: linux.fd_t,
2516 user_data: u64,
2517) !linux.io_uring_cqe {
2518 // prepare and submit recv
2519 const sqe = try buf_grp.recv(user_data, fd, 0);
2520 try testing.expect(sqe.flags & linux.IOSQE_BUFFER_SELECT == linux.IOSQE_BUFFER_SELECT);
2521 try testing.expect(sqe.buf_index == buf_grp.group_id);
2522 try testing.expectEqual(@as(u32, 1), try ring.submit()); // submit
2523 // get cqe, expect success
2524 const cqe = try ring.copy_cqe();
2525 try testing.expectEqual(user_data, cqe.user_data);
2526 try testing.expect(cqe.res >= 0); // success
2527 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
2528 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER); // IORING_CQE_F_BUFFER flag is set
2529
2530 return cqe;
2531}
2532
2533fn expect_buf_grp_cqe(
2534 ring: *IoUring,
2535 buf_grp: *BufferGroup,
2536 user_data: u64,
2537 expected: []const u8,
2538) !linux.io_uring_cqe {
2539 // get cqe
2540 const cqe = try ring.copy_cqe();
2541 try testing.expectEqual(user_data, cqe.user_data);
2542 try testing.expect(cqe.res >= 0); // success
2543 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER); // IORING_CQE_F_BUFFER flag is set
2544 try testing.expectEqual(expected.len, @as(usize, @intCast(cqe.res)));
2545 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
2546
2547 // get buffer from pool
2548 const buffer_id = try cqe.buffer_id();
2549 const len = @as(usize, @intCast(cqe.res));
2550 const buf = buf_grp.get_by_id(buffer_id)[0..len];
2551 try testing.expectEqualSlices(u8, expected, buf);
2552
2553 return cqe;
2554}
2555
2556fn testSendRecv(ring: *IoUring, send_fd: posix.socket_t, recv_fd: posix.socket_t) !void {
2557 const buffer_send = "0123456789abcdf" ** 10;
2558 var buffer_recv: [buffer_send.len * 2]u8 = undefined;
2559
2560 // 2 sends
2561 _ = try ring.send(1, send_fd, buffer_send, linux.MSG.WAITALL);
2562 _ = try ring.send(2, send_fd, buffer_send, linux.MSG.WAITALL);
2563 try testing.expectEqual(2, try ring.submit());
2564 for (0..2) |i| {
2565 const cqe = try ring.copy_cqe();
2566 try testing.expectEqual(1 + i, cqe.user_data);
2567 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
2568 try testing.expectEqual(buffer_send.len, @as(usize, @intCast(cqe.res)));
2569 }
2570
2571 // receive
2572 var recv_len: usize = 0;
2573 while (recv_len < buffer_send.len * 2) {
2574 _ = try ring.recv(3, recv_fd, .{ .buffer = buffer_recv[recv_len..] }, 0);
2575 try testing.expectEqual(1, try ring.submit());
2576 const cqe = try ring.copy_cqe();
2577 try testing.expectEqual(3, cqe.user_data);
2578 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
2579 recv_len += @intCast(cqe.res);
2580 }
2581
2582 // inspect recv buffer
2583 try testing.expectEqualSlices(u8, buffer_send, buffer_recv[0..buffer_send.len]);
2584 try testing.expectEqualSlices(u8, buffer_send, buffer_recv[buffer_send.len..]);
2585}
2586
2587/// Used for testing server/client interactions.
2588pub const SocketTestHarness = struct {
2589 listener: posix.socket_t,
2590 server: posix.socket_t,
2591 client: posix.socket_t,
2592
2593 pub fn close(self: SocketTestHarness) void {
2594 posix.close(self.client);
2595 posix.close(self.listener);
2596 }
2597};
2598
2599pub fn createSocketTestHarness(ring: *IoUring) !SocketTestHarness {
2600 // Create a TCP server socket
2601 var address: linux.sockaddr.in = .{
2602 .port = 0,
2603 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
2604 };
2605 const listener_socket = try createListenerSocket(&address);
2606 errdefer posix.close(listener_socket);
2607
2608 // Submit 1 accept
2609 var accept_addr: posix.sockaddr = undefined;
2610 var accept_addr_len: posix.socklen_t = @sizeOf(@TypeOf(accept_addr));
2611 _ = try ring.accept(0xaaaaaaaa, listener_socket, &accept_addr, &accept_addr_len, 0);
2612
2613 // Create a TCP client socket
2614 const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
2615 errdefer posix.close(client);
2616 _ = try ring.connect(0xcccccccc, client, addrAny(&address), @sizeOf(linux.sockaddr.in));
2617
2618 try testing.expectEqual(@as(u32, 2), try ring.submit());
2619
2620 var cqe_accept = try ring.copy_cqe();
2621 if (cqe_accept.err() == .INVAL) return error.SkipZigTest;
2622 var cqe_connect = try ring.copy_cqe();
2623 if (cqe_connect.err() == .INVAL) return error.SkipZigTest;
2624
2625 // The accept/connect CQEs may arrive in any order, the connect CQE will sometimes come first:
2626 if (cqe_accept.user_data == 0xcccccccc and cqe_connect.user_data == 0xaaaaaaaa) {
2627 const a = cqe_accept;
2628 const b = cqe_connect;
2629 cqe_accept = b;
2630 cqe_connect = a;
2631 }
2632
2633 try testing.expectEqual(@as(u64, 0xaaaaaaaa), cqe_accept.user_data);
2634 if (cqe_accept.res <= 0) std.debug.print("\ncqe_accept.res={}\n", .{cqe_accept.res});
2635 try testing.expect(cqe_accept.res > 0);
2636 try testing.expectEqual(@as(u32, 0), cqe_accept.flags);
2637 try testing.expectEqual(linux.io_uring_cqe{
2638 .user_data = 0xcccccccc,
2639 .res = 0,
2640 .flags = 0,
2641 }, cqe_connect);
2642
2643 // All good
2644
2645 return SocketTestHarness{
2646 .listener = listener_socket,
2647 .server = cqe_accept.res,
2648 .client = client,
2649 };
2650}
2651
2652fn createListenerSocket(address: *linux.sockaddr.in) !posix.socket_t {
2653 const kernel_backlog = 1;
2654 const listener_socket = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0);
2655 errdefer posix.close(listener_socket);
2656
2657 try posix.setsockopt(listener_socket, posix.SOL.SOCKET, posix.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
2658 try posix.bind(listener_socket, addrAny(address), @sizeOf(linux.sockaddr.in));
2659 try posix.listen(listener_socket, kernel_backlog);
2660
2661 // set address to the OS-chosen IP/port.
2662 var slen: posix.socklen_t = @sizeOf(linux.sockaddr.in);
2663 try posix.getsockname(listener_socket, addrAny(address), &slen);
2664
2665 return listener_socket;
2666}
2667
2668/// For use in tests. Returns SkipZigTest if kernel version is less than required.
2669inline fn skipKernelLessThan(required: std.SemanticVersion) !void {
2670 var uts: linux.utsname = undefined;
2671 const res = linux.uname(&uts);
2672 switch (linux.errno(res)) {
2673 .SUCCESS => {},
2674 else => |errno| return posix.unexpectedErrno(errno),
2675 }
2676
2677 const release = mem.sliceTo(&uts.release, 0);
2678 // Strips potential extra, as kernel version might not be semver compliant, example "6.8.9-300.fc40.x86_64"
2679 const extra_index = std.mem.indexOfAny(u8, release, "-+");
2680 const stripped = release[0..(extra_index orelse release.len)];
2681 // Make sure the input don't rely on the extra we just stripped
2682 try testing.expect(required.pre == null and required.build == null);
2683
2684 var current = try std.SemanticVersion.parse(stripped);
2685 current.pre = null; // don't check pre field
2686 if (required.order(current) == .gt) return error.SkipZigTest;
2687}
2688
2689fn addrAny(addr: *linux.sockaddr.in) *linux.sockaddr {
2690 return @ptrCast(addr);
2691}
lib/std/os/linux/test.zig+2-2
...@@ -18,7 +18,7 @@ test "fallocate" {...@@ -18,7 +18,7 @@ test "fallocate" {
18 defer tmp.cleanup();18 defer tmp.cleanup();
1919
20 const path = "test_fallocate";20 const path = "test_fallocate";
21 const file = try tmp.dir.createFile(io, path, .{ .truncate = true, .mode = 0o666 });21 const file = try tmp.dir.createFile(io, path, .{ .truncate = true, .permissions = .fromMode(0o666) });
22 defer file.close(io);22 defer file.close(io);
2323
24 try expect((try file.stat(io)).size == 0);24 try expect((try file.stat(io)).size == 0);
...@@ -125,7 +125,7 @@ test "fadvise" {...@@ -125,7 +125,7 @@ test "fadvise" {
125 defer file.close(io);125 defer file.close(io);
126126
127 var buf: [2048]u8 = undefined;127 var buf: [2048]u8 = undefined;
128 try file.writeAll(&buf);128 try file.writeStreamingAll(io, &buf);
129129
130 const ret = linux.fadvise(file.handle, 0, 0, linux.POSIX_FADV.SEQUENTIAL);130 const ret = linux.fadvise(file.handle, 0, 0, linux.POSIX_FADV.SEQUENTIAL);
131 try expectEqual(@as(usize, 0), ret);131 try expectEqual(@as(usize, 0), ret);