authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-05-26 18:39:17-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-05-26 18:39:17-04:00
logba426f0a54d1c80ae9a4c73e06c5dc1265f0568b
tree3e900d8c4421e1de2b5a471c64435357807f6d51
parent326b2aa27bdf43b695798192e415db995ee9918b
parent3c58d3e281fca421879fe1d2ae7943bd31dccc0e
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #10197 from vrischmann/io_uring-provide_buffers

io_uring: implement automatic buffer selection

1 files changed, 581 insertions(+), 152 deletions(-)

lib/std/os/linux/io_uring.zig+581-152
...@@ -358,17 +358,46 @@ pub const IO_Uring = struct {...@@ -358,17 +358,46 @@ pub const IO_Uring = struct {
358 return sqe;358 return sqe;
359 }359 }
360360
361 /// Queues (but does not submit) an SQE to perform a `read(2)`.361 /// Used to select how the read should be handled.
362 pub const ReadBuffer = union(enum) {
363 /// io_uring will read directly into this buffer
364 buffer: []u8,
365
366 /// io_uring will read directly into these buffers using readv.
367 iovecs: []const os.iovec,
368
369 /// io_uring will select a buffer that has previously been provided with `provide_buffers`.
370 /// The buffer group reference by `group_id` must contain at least one buffer for the read to work.
371 /// `len` controls the number of bytes to read into the selected buffer.
372 buffer_selection: struct {
373 group_id: u16,
374 len: usize,
375 },
376 };
377
378 /// Queues (but does not submit) an SQE to perform a `read(2)` or `preadv` depending on the buffer type.
379 /// * Reading into a `ReadBuffer.buffer` uses `read(2)`
380 /// * Reading into a `ReadBuffer.iovecs` uses `preadv(2)`
381 /// If you want to do a `preadv2()` then set `rw_flags` on the returned SQE. See https://linux.die.net/man/2/preadv.
382 ///
362 /// Returns a pointer to the SQE.383 /// Returns a pointer to the SQE.
363 pub fn read(384 pub fn read(
364 self: *IO_Uring,385 self: *IO_Uring,
365 user_data: u64,386 user_data: u64,
366 fd: os.fd_t,387 fd: os.fd_t,
367 buffer: []u8,388 buffer: ReadBuffer,
368 offset: u64,389 offset: u64,
369 ) !*io_uring_sqe {390 ) !*io_uring_sqe {
370 const sqe = try self.get_sqe();391 const sqe = try self.get_sqe();
371 io_uring_prep_read(sqe, fd, buffer, offset);392 switch (buffer) {
393 .buffer => |slice| io_uring_prep_read(sqe, fd, slice, offset),
394 .iovecs => |vecs| io_uring_prep_readv(sqe, fd, vecs, offset),
395 .buffer_selection => |selection| {
396 io_uring_prep_rw(.READ, sqe, fd, 0, selection.len, offset);
397 sqe.flags |= linux.IOSQE_BUFFER_SELECT;
398 sqe.buf_index = selection.group_id;
399 },
400 }
372 sqe.user_data = user_data;401 sqe.user_data = user_data;
373 return sqe;402 return sqe;
374 }403 }
...@@ -388,23 +417,6 @@ pub const IO_Uring = struct {...@@ -388,23 +417,6 @@ pub const IO_Uring = struct {
388 return sqe;417 return sqe;
389 }418 }
390419
391 /// Queues (but does not submit) an SQE to perform a `preadv()`.
392 /// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
393 /// For example, if you want to do a `preadv2()` then set `rw_flags` on the returned SQE.
394 /// See https://linux.die.net/man/2/preadv.
395 pub fn readv(
396 self: *IO_Uring,
397 user_data: u64,
398 fd: os.fd_t,
399 iovecs: []const os.iovec,
400 offset: u64,
401 ) !*io_uring_sqe {
402 const sqe = try self.get_sqe();
403 io_uring_prep_readv(sqe, fd, iovecs, offset);
404 sqe.user_data = user_data;
405 return sqe;
406 }
407
408 /// Queues (but does not submit) an SQE to perform a IORING_OP_READ_FIXED.420 /// Queues (but does not submit) an SQE to perform a IORING_OP_READ_FIXED.
409 /// The `buffer` provided must be registered with the kernel by calling `register_buffers` first.421 /// The `buffer` provided must be registered with the kernel by calling `register_buffers` first.
410 /// The `buffer_index` must be the same as its index in the array provided to `register_buffers`.422 /// The `buffer_index` must be the same as its index in the array provided to `register_buffers`.
...@@ -507,17 +519,39 @@ pub const IO_Uring = struct {...@@ -507,17 +519,39 @@ pub const IO_Uring = struct {
507 return sqe;519 return sqe;
508 }520 }
509521
522 /// Used to select how the recv call should be handled.
523 pub const RecvBuffer = union(enum) {
524 /// io_uring will recv directly into this buffer
525 buffer: []u8,
526
527 /// io_uring will select a buffer that has previously been provided with `provide_buffers`.
528 /// The buffer group referenced by `group_id` must contain at least one buffer for the recv call to work.
529 /// `len` controls the number of bytes to read into the selected buffer.
530 buffer_selection: struct {
531 group_id: u16,
532 len: usize,
533 },
534 };
535
510 /// Queues (but does not submit) an SQE to perform a `recv(2)`.536 /// Queues (but does not submit) an SQE to perform a `recv(2)`.
511 /// Returns a pointer to the SQE.537 /// Returns a pointer to the SQE.
512 pub fn recv(538 pub fn recv(
513 self: *IO_Uring,539 self: *IO_Uring,
514 user_data: u64,540 user_data: u64,
515 fd: os.fd_t,541 fd: os.fd_t,
516 buffer: []u8,542 buffer: RecvBuffer,
517 flags: u32,543 flags: u32,
518 ) !*io_uring_sqe {544 ) !*io_uring_sqe {
519 const sqe = try self.get_sqe();545 const sqe = try self.get_sqe();
520 io_uring_prep_recv(sqe, fd, buffer, flags);546 switch (buffer) {
547 .buffer => |slice| io_uring_prep_recv(sqe, fd, slice, flags),
548 .buffer_selection => |selection| {
549 io_uring_prep_rw(.RECV, sqe, fd, 0, selection.len, 0);
550 sqe.rw_flags = flags;
551 sqe.flags |= linux.IOSQE_BUFFER_SELECT;
552 sqe.buf_index = selection.group_id;
553 },
554 }
521 sqe.user_data = user_data;555 sqe.user_data = user_data;
522 return sqe;556 return sqe;
523 }557 }
...@@ -857,6 +891,41 @@ pub const IO_Uring = struct {...@@ -857,6 +891,41 @@ pub const IO_Uring = struct {
857 return sqe;891 return sqe;
858 }892 }
859893
894 /// Queues (but does not submit) an SQE to provide a group of buffers used for commands that read/receive data.
895 /// Returns a pointer to the SQE.
896 ///
897 /// Provided buffers can be used in `read`, `recv` or `recvmsg` commands via .buffer_selection.
898 ///
899 /// The kernel expects a contiguous block of memory of size (buffers_count * buffer_size).
900 pub fn provide_buffers(
901 self: *IO_Uring,
902 user_data: u64,
903 buffers: [*]u8,
904 buffers_count: usize,
905 buffer_size: usize,
906 group_id: usize,
907 buffer_id: usize,
908 ) !*io_uring_sqe {
909 const sqe = try self.get_sqe();
910 io_uring_prep_provide_buffers(sqe, buffers, buffers_count, buffer_size, group_id, buffer_id);
911 sqe.user_data = user_data;
912 return sqe;
913 }
914
915 /// Queues (but does not submit) an SQE to remove a group of provided buffers.
916 /// Returns a pointer to the SQE.
917 pub fn remove_buffers(
918 self: *IO_Uring,
919 user_data: u64,
920 buffers_count: usize,
921 group_id: usize,
922 ) !*io_uring_sqe {
923 const sqe = try self.get_sqe();
924 io_uring_prep_remove_buffers(sqe, buffers_count, group_id);
925 sqe.user_data = user_data;
926 return sqe;
927 }
928
860 /// Registers an array of file descriptors.929 /// Registers an array of file descriptors.
861 /// Every time a file descriptor is put in an SQE and submitted to the kernel, the kernel must930 /// Every time a file descriptor is put in an SQE and submitted to the kernel, the kernel must
862 /// retrieve a reference to the file, and once I/O has completed the file reference must be931 /// retrieve a reference to the file, and once I/O has completed the file reference must be
...@@ -1508,6 +1577,28 @@ pub fn io_uring_prep_linkat(...@@ -1508,6 +1577,28 @@ pub fn io_uring_prep_linkat(
1508 sqe.rw_flags = flags;1577 sqe.rw_flags = flags;
1509}1578}
15101579
1580pub fn io_uring_prep_provide_buffers(
1581 sqe: *io_uring_sqe,
1582 buffers: [*]u8,
1583 num: usize,
1584 buffer_len: usize,
1585 group_id: usize,
1586 buffer_id: usize,
1587) void {
1588 const ptr = @ptrToInt(buffers);
1589 io_uring_prep_rw(.PROVIDE_BUFFERS, sqe, @intCast(i32, num), ptr, buffer_len, buffer_id);
1590 sqe.buf_index = @intCast(u16, group_id);
1591}
1592
1593pub fn io_uring_prep_remove_buffers(
1594 sqe: *io_uring_sqe,
1595 num: usize,
1596 group_id: usize,
1597) void {
1598 io_uring_prep_rw(.REMOVE_BUFFERS, sqe, @intCast(i32, num), 0, 0, 0);
1599 sqe.buf_index = @intCast(u16, group_id);
1600}
1601
1511test "structs/offsets/entries" {1602test "structs/offsets/entries" {
1512 if (builtin.os.tag != .linux) return error.SkipZigTest;1603 if (builtin.os.tag != .linux) return error.SkipZigTest;
15131604
...@@ -1615,7 +1706,7 @@ test "readv" {...@@ -1615,7 +1706,7 @@ test "readv" {
16151706
1616 var buffer = [_]u8{42} ** 128;1707 var buffer = [_]u8{42} ** 128;
1617 var iovecs = [_]os.iovec{os.iovec{ .iov_base = &buffer, .iov_len = buffer.len }};1708 var iovecs = [_]os.iovec{os.iovec{ .iov_base = &buffer, .iov_len = buffer.len }};
1618 const sqe = try ring.readv(0xcccccccc, fd_index, iovecs[0..], 0);1709 const sqe = try ring.read(0xcccccccc, fd_index, .{ .iovecs = iovecs[0..] }, 0);
1619 try testing.expectEqual(linux.IORING_OP.READV, sqe.opcode);1710 try testing.expectEqual(linux.IORING_OP.READV, sqe.opcode);
1620 sqe.flags |= linux.IOSQE_FIXED_FILE;1711 sqe.flags |= linux.IOSQE_FIXED_FILE;
16211712
...@@ -1666,7 +1757,7 @@ test "writev/fsync/readv" {...@@ -1666,7 +1757,7 @@ test "writev/fsync/readv" {
1666 try testing.expectEqual(fd, sqe_fsync.fd);1757 try testing.expectEqual(fd, sqe_fsync.fd);
1667 sqe_fsync.flags |= linux.IOSQE_IO_LINK;1758 sqe_fsync.flags |= linux.IOSQE_IO_LINK;
16681759
1669 const sqe_readv = try ring.readv(0xffffffff, fd, iovecs_read[0..], 17);1760 const sqe_readv = try ring.read(0xffffffff, fd, .{ .iovecs = iovecs_read[0..] }, 17);
1670 try testing.expectEqual(linux.IORING_OP.READV, sqe_readv.opcode);1761 try testing.expectEqual(linux.IORING_OP.READV, sqe_readv.opcode);
1671 try testing.expectEqual(@as(u64, 17), sqe_readv.off);1762 try testing.expectEqual(@as(u64, 17), sqe_readv.off);
16721763
...@@ -1721,7 +1812,7 @@ test "write/read" {...@@ -1721,7 +1812,7 @@ test "write/read" {
1721 try testing.expectEqual(linux.IORING_OP.WRITE, sqe_write.opcode);1812 try testing.expectEqual(linux.IORING_OP.WRITE, sqe_write.opcode);
1722 try testing.expectEqual(@as(u64, 10), sqe_write.off);1813 try testing.expectEqual(@as(u64, 10), sqe_write.off);
1723 sqe_write.flags |= linux.IOSQE_IO_LINK;1814 sqe_write.flags |= linux.IOSQE_IO_LINK;
1724 const sqe_read = try ring.read(0x22222222, fd, buffer_read[0..], 10);1815 const sqe_read = try ring.read(0x22222222, fd, .{ .buffer = buffer_read[0..] }, 10);
1725 try testing.expectEqual(linux.IORING_OP.READ, sqe_read.opcode);1816 try testing.expectEqual(linux.IORING_OP.READ, sqe_read.opcode);
1726 try testing.expectEqual(@as(u64, 10), sqe_read.off);1817 try testing.expectEqual(@as(u64, 10), sqe_read.off);
1727 try testing.expectEqual(@as(u32, 2), try ring.submit());1818 try testing.expectEqual(@as(u32, 2), try ring.submit());
...@@ -1890,53 +1981,15 @@ test "accept/connect/send/recv" {...@@ -1890,53 +1981,15 @@ test "accept/connect/send/recv" {
1890 };1981 };
1891 defer ring.deinit();1982 defer ring.deinit();
18921983
1893 const address = try net.Address.parseIp4("127.0.0.1", 3131);1984 const socket_test_harness = try createSocketTestHarness(&ring);
1894 const kernel_backlog = 1;1985 defer socket_test_harness.close();
1895 const server = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
1896 defer os.close(server);
1897 try os.setsockopt(server, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
1898 try os.bind(server, &address.any, address.getOsSockLen());
1899 try os.listen(server, kernel_backlog);
19001986
1901 const buffer_send = [_]u8{ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 };1987 const buffer_send = [_]u8{ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 };
1902 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };1988 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
19031989
1904 var accept_addr: os.sockaddr = undefined;1990 const send = try ring.send(0xeeeeeeee, socket_test_harness.client, buffer_send[0..], 0);
1905 var accept_addr_len: os.socklen_t = @sizeOf(@TypeOf(accept_addr));
1906 _ = try ring.accept(0xaaaaaaaa, server, &accept_addr, &accept_addr_len, 0);
1907 try testing.expectEqual(@as(u32, 1), try ring.submit());
1908
1909 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
1910 defer os.close(client);
1911 _ = try ring.connect(0xcccccccc, client, &address.any, address.getOsSockLen());
1912 try testing.expectEqual(@as(u32, 1), try ring.submit());
1913
1914 var cqe_accept = try ring.copy_cqe();
1915 if (cqe_accept.err() == .INVAL) return error.SkipZigTest;
1916 var cqe_connect = try ring.copy_cqe();
1917 if (cqe_connect.err() == .INVAL) return error.SkipZigTest;
1918
1919 // The accept/connect CQEs may arrive in any order, the connect CQE will sometimes come first:
1920 if (cqe_accept.user_data == 0xcccccccc and cqe_connect.user_data == 0xaaaaaaaa) {
1921 const a = cqe_accept;
1922 const b = cqe_connect;
1923 cqe_accept = b;
1924 cqe_connect = a;
1925 }
1926
1927 try testing.expectEqual(@as(u64, 0xaaaaaaaa), cqe_accept.user_data);
1928 if (cqe_accept.res <= 0) std.debug.print("\ncqe_accept.res={}\n", .{cqe_accept.res});
1929 try testing.expect(cqe_accept.res > 0);
1930 try testing.expectEqual(@as(u32, 0), cqe_accept.flags);
1931 try testing.expectEqual(linux.io_uring_cqe{
1932 .user_data = 0xcccccccc,
1933 .res = 0,
1934 .flags = 0,
1935 }, cqe_connect);
1936
1937 const send = try ring.send(0xeeeeeeee, client, buffer_send[0..], 0);
1938 send.flags |= linux.IOSQE_IO_LINK;1991 send.flags |= linux.IOSQE_IO_LINK;
1939 _ = try ring.recv(0xffffffff, cqe_accept.res, buffer_recv[0..], 0);1992 _ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
1940 try testing.expectEqual(@as(u32, 2), try ring.submit());1993 try testing.expectEqual(@as(u32, 2), try ring.submit());
19411994
1942 const cqe_send = try ring.copy_cqe();1995 const cqe_send = try ring.copy_cqe();
...@@ -2161,50 +2214,12 @@ test "accept/connect/recv/link_timeout" {...@@ -2161,50 +2214,12 @@ test "accept/connect/recv/link_timeout" {
2161 };2214 };
2162 defer ring.deinit();2215 defer ring.deinit();
21632216
2164 const address = try net.Address.parseIp4("127.0.0.1", 3131);2217 const socket_test_harness = try createSocketTestHarness(&ring);
2165 const kernel_backlog = 1;2218 defer socket_test_harness.close();
2166 const server = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
2167 defer os.close(server);
2168 try os.setsockopt(server, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
2169 try os.bind(server, &address.any, address.getOsSockLen());
2170 try os.listen(server, kernel_backlog);
21712219
2172 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };2220 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
21732221
2174 var accept_addr: os.sockaddr = undefined;2222 const sqe_recv = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
2175 var accept_addr_len: os.socklen_t = @sizeOf(@TypeOf(accept_addr));
2176 _ = try ring.accept(0xaaaaaaaa, server, &accept_addr, &accept_addr_len, 0);
2177 try testing.expectEqual(@as(u32, 1), try ring.submit());
2178
2179 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
2180 defer os.close(client);
2181 _ = try ring.connect(0xcccccccc, client, &address.any, address.getOsSockLen());
2182 try testing.expectEqual(@as(u32, 1), try ring.submit());
2183
2184 var cqe_accept = try ring.copy_cqe();
2185 if (cqe_accept.err() == .INVAL) return error.SkipZigTest;
2186 var cqe_connect = try ring.copy_cqe();
2187 if (cqe_connect.err() == .INVAL) return error.SkipZigTest;
2188
2189 // The accept/connect CQEs may arrive in any order, the connect CQE will sometimes come first:
2190 if (cqe_accept.user_data == 0xcccccccc and cqe_connect.user_data == 0xaaaaaaaa) {
2191 const a = cqe_accept;
2192 const b = cqe_connect;
2193 cqe_accept = b;
2194 cqe_connect = a;
2195 }
2196
2197 try testing.expectEqual(@as(u64, 0xaaaaaaaa), cqe_accept.user_data);
2198 if (cqe_accept.res <= 0) std.debug.print("\ncqe_accept.res={}\n", .{cqe_accept.res});
2199 try testing.expect(cqe_accept.res > 0);
2200 try testing.expectEqual(@as(u32, 0), cqe_accept.flags);
2201 try testing.expectEqual(linux.io_uring_cqe{
2202 .user_data = 0xcccccccc,
2203 .res = 0,
2204 .flags = 0,
2205 }, cqe_connect);
2206
2207 const sqe_recv = try ring.recv(0xffffffff, cqe_accept.res, buffer_recv[0..], 0);
2208 sqe_recv.flags |= linux.IOSQE_IO_LINK;2223 sqe_recv.flags |= linux.IOSQE_IO_LINK;
22092224
2210 const ts = os.linux.kernel_timespec{ .tv_sec = 0, .tv_nsec = 1000000 };2225 const ts = os.linux.kernel_timespec{ .tv_sec = 0, .tv_nsec = 1000000 };
...@@ -2348,50 +2363,12 @@ test "accept/connect/recv/cancel" {...@@ -2348,50 +2363,12 @@ test "accept/connect/recv/cancel" {
2348 };2363 };
2349 defer ring.deinit();2364 defer ring.deinit();
23502365
2351 const address = try net.Address.parseIp4("127.0.0.1", 3131);2366 const socket_test_harness = try createSocketTestHarness(&ring);
2352 const kernel_backlog = 1;2367 defer socket_test_harness.close();
2353 const server = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
2354 defer os.close(server);
2355 try os.setsockopt(server, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
2356 try os.bind(server, &address.any, address.getOsSockLen());
2357 try os.listen(server, kernel_backlog);
23582368
2359 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };2369 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
23602370
2361 var accept_addr: os.sockaddr = undefined;2371 _ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
2362 var accept_addr_len: os.socklen_t = @sizeOf(@TypeOf(accept_addr));
2363 _ = try ring.accept(0xaaaaaaaa, server, &accept_addr, &accept_addr_len, 0);
2364 try testing.expectEqual(@as(u32, 1), try ring.submit());
2365
2366 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
2367 defer os.close(client);
2368 _ = try ring.connect(0xcccccccc, client, &address.any, address.getOsSockLen());
2369 try testing.expectEqual(@as(u32, 1), try ring.submit());
2370
2371 var cqe_accept = try ring.copy_cqe();
2372 if (cqe_accept.err() == .INVAL) return error.SkipZigTest;
2373 var cqe_connect = try ring.copy_cqe();
2374 if (cqe_connect.err() == .INVAL) return error.SkipZigTest;
2375
2376 // The accept/connect CQEs may arrive in any order, the connect CQE will sometimes come first:
2377 if (cqe_accept.user_data == 0xcccccccc and cqe_connect.user_data == 0xaaaaaaaa) {
2378 const a = cqe_accept;
2379 const b = cqe_connect;
2380 cqe_accept = b;
2381 cqe_connect = a;
2382 }
2383
2384 try testing.expectEqual(@as(u64, 0xaaaaaaaa), cqe_accept.user_data);
2385 if (cqe_accept.res <= 0) std.debug.print("\ncqe_accept.res={}\n", .{cqe_accept.res});
2386 try testing.expect(cqe_accept.res > 0);
2387 try testing.expectEqual(@as(u32, 0), cqe_accept.flags);
2388 try testing.expectEqual(linux.io_uring_cqe{
2389 .user_data = 0xcccccccc,
2390 .res = 0,
2391 .flags = 0,
2392 }, cqe_connect);
2393
2394 _ = try ring.recv(0xffffffff, cqe_accept.res, buffer_recv[0..], 0);
2395 try testing.expectEqual(@as(u32, 1), try ring.submit());2372 try testing.expectEqual(@as(u32, 1), try ring.submit());
23962373
2397 const sqe_cancel = try ring.cancel(0x99999999, 0xffffffff, 0);2374 const sqe_cancel = try ring.cancel(0x99999999, 0xffffffff, 0);
...@@ -2463,7 +2440,7 @@ test "register_files_update" {...@@ -2463,7 +2440,7 @@ test "register_files_update" {
24632440
2464 var buffer = [_]u8{42} ** 128;2441 var buffer = [_]u8{42} ** 128;
2465 {2442 {
2466 const sqe = try ring.read(0xcccccccc, fd_index, &buffer, 0);2443 const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
2467 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);2444 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2468 sqe.flags |= linux.IOSQE_FIXED_FILE;2445 sqe.flags |= linux.IOSQE_FIXED_FILE;
24692446
...@@ -2484,7 +2461,7 @@ test "register_files_update" {...@@ -2484,7 +2461,7 @@ test "register_files_update" {
24842461
2485 {2462 {
2486 // Next read should still work since fd_index in the registered file descriptors hasn't been updated yet.2463 // Next read should still work since fd_index in the registered file descriptors hasn't been updated yet.
2487 const sqe = try ring.read(0xcccccccc, fd_index, &buffer, 0);2464 const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
2488 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);2465 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2489 sqe.flags |= linux.IOSQE_FIXED_FILE;2466 sqe.flags |= linux.IOSQE_FIXED_FILE;
24902467
...@@ -2501,7 +2478,7 @@ test "register_files_update" {...@@ -2501,7 +2478,7 @@ test "register_files_update" {
25012478
2502 {2479 {
2503 // Now this should fail since both fds are sparse (-1)2480 // Now this should fail since both fds are sparse (-1)
2504 const sqe = try ring.read(0xcccccccc, fd_index, &buffer, 0);2481 const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
2505 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);2482 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2506 sqe.flags |= linux.IOSQE_FIXED_FILE;2483 sqe.flags |= linux.IOSQE_FIXED_FILE;
25072484
...@@ -2843,3 +2820,455 @@ test "linkat" {...@@ -2843,3 +2820,455 @@ test "linkat" {
2843 const read = try second_file.readAll(&second_file_data);2820 const read = try second_file.readAll(&second_file_data);
2844 try testing.expectEqualStrings("hello", second_file_data[0..read]);2821 try testing.expectEqualStrings("hello", second_file_data[0..read]);
2845}2822}
2823
2824test "provide_buffers: read" {
2825 if (builtin.os.tag != .linux) return error.SkipZigTest;
2826
2827 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
2828 error.SystemOutdated => return error.SkipZigTest,
2829 error.PermissionDenied => return error.SkipZigTest,
2830 else => return err,
2831 };
2832 defer ring.deinit();
2833
2834 const fd = try os.openZ("/dev/zero", os.O.RDONLY | os.O.CLOEXEC, 0);
2835 defer os.close(fd);
2836
2837 const group_id = 1337;
2838 const buffer_id = 0;
2839
2840 const buffer_len = 128;
2841
2842 var buffers: [4][buffer_len]u8 = undefined;
2843
2844 // Provide 4 buffers
2845
2846 {
2847 const sqe = try ring.provide_buffers(0xcccccccc, @ptrCast([*]u8, &buffers), buffers.len, buffer_len, group_id, buffer_id);
2848 try testing.expectEqual(linux.IORING_OP.PROVIDE_BUFFERS, sqe.opcode);
2849 try testing.expectEqual(@as(i32, buffers.len), sqe.fd);
2850 try testing.expectEqual(@as(u32, buffers[0].len), sqe.len);
2851 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
2852 try testing.expectEqual(@as(u32, 1), try ring.submit());
2853
2854 const cqe = try ring.copy_cqe();
2855 switch (cqe.err()) {
2856 // Happens when the kernel is < 5.7
2857 .INVAL => return error.SkipZigTest,
2858 .SUCCESS => {},
2859 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2860 }
2861 try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
2862 }
2863
2864 // Do 4 reads which should consume all buffers
2865
2866 var i: usize = 0;
2867 while (i < buffers.len) : (i += 1) {
2868 var sqe = try ring.read(0xdededede, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
2869 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2870 try testing.expectEqual(@as(i32, fd), sqe.fd);
2871 try testing.expectEqual(@as(u64, 0), sqe.addr);
2872 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
2873 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
2874 try testing.expectEqual(@as(u32, 1), try ring.submit());
2875
2876 const cqe = try ring.copy_cqe();
2877 switch (cqe.err()) {
2878 .SUCCESS => {},
2879 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2880 }
2881
2882 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
2883 const used_buffer_id = cqe.flags >> 16;
2884 try testing.expect(used_buffer_id >= 0 and used_buffer_id <= 3);
2885 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
2886
2887 try testing.expectEqual(@as(u64, 0xdededede), cqe.user_data);
2888 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@intCast(usize, cqe.res)]);
2889 }
2890
2891 // This read should fail
2892
2893 {
2894 var sqe = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
2895 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2896 try testing.expectEqual(@as(i32, fd), sqe.fd);
2897 try testing.expectEqual(@as(u64, 0), sqe.addr);
2898 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
2899 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
2900 try testing.expectEqual(@as(u32, 1), try ring.submit());
2901
2902 const cqe = try ring.copy_cqe();
2903 switch (cqe.err()) {
2904 // Expected
2905 .NOBUFS => {},
2906 .SUCCESS => std.debug.panic("unexpected success", .{}),
2907 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2908 }
2909 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
2910 }
2911
2912 // Provide 1 buffer again
2913
2914 // Deliberately put something we don't expect in the buffers
2915 mem.set(u8, mem.sliceAsBytes(&buffers), 42);
2916
2917 const reprovided_buffer_id = 2;
2918
2919 {
2920 _ = try ring.provide_buffers(0xabababab, @ptrCast([*]u8, &buffers[reprovided_buffer_id]), 1, buffer_len, group_id, reprovided_buffer_id);
2921 try testing.expectEqual(@as(u32, 1), try ring.submit());
2922
2923 const cqe = try ring.copy_cqe();
2924 switch (cqe.err()) {
2925 .SUCCESS => {},
2926 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2927 }
2928 }
2929
2930 // Final read which should work
2931
2932 {
2933 var sqe = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
2934 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2935 try testing.expectEqual(@as(i32, fd), sqe.fd);
2936 try testing.expectEqual(@as(u64, 0), sqe.addr);
2937 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
2938 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
2939 try testing.expectEqual(@as(u32, 1), try ring.submit());
2940
2941 const cqe = try ring.copy_cqe();
2942 switch (cqe.err()) {
2943 .SUCCESS => {},
2944 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2945 }
2946
2947 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
2948 const used_buffer_id = cqe.flags >> 16;
2949 try testing.expectEqual(used_buffer_id, reprovided_buffer_id);
2950 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
2951 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
2952 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@intCast(usize, cqe.res)]);
2953 }
2954}
2955
2956test "remove_buffers" {
2957 if (builtin.os.tag != .linux) return error.SkipZigTest;
2958
2959 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
2960 error.SystemOutdated => return error.SkipZigTest,
2961 error.PermissionDenied => return error.SkipZigTest,
2962 else => return err,
2963 };
2964 defer ring.deinit();
2965
2966 const fd = try os.openZ("/dev/zero", os.O.RDONLY | os.O.CLOEXEC, 0);
2967 defer os.close(fd);
2968
2969 const group_id = 1337;
2970 const buffer_id = 0;
2971
2972 const buffer_len = 128;
2973
2974 var buffers: [4][buffer_len]u8 = undefined;
2975
2976 // Provide 4 buffers
2977
2978 {
2979 _ = try ring.provide_buffers(0xcccccccc, @ptrCast([*]u8, &buffers), buffers.len, buffer_len, group_id, buffer_id);
2980 try testing.expectEqual(@as(u32, 1), try ring.submit());
2981
2982 const cqe = try ring.copy_cqe();
2983 switch (cqe.err()) {
2984 .SUCCESS => {},
2985 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2986 }
2987 try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
2988 }
2989
2990 // Remove the first 3 buffers
2991
2992 {
2993 var sqe = try ring.remove_buffers(0xbababababa, 3, group_id);
2994 try testing.expectEqual(linux.IORING_OP.REMOVE_BUFFERS, sqe.opcode);
2995 try testing.expectEqual(@as(i32, 3), sqe.fd);
2996 try testing.expectEqual(@as(u64, 0), sqe.addr);
2997 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
2998 try testing.expectEqual(@as(u32, 1), try ring.submit());
2999
3000 const cqe = try ring.copy_cqe();
3001 switch (cqe.err()) {
3002 .SUCCESS => {},
3003 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3004 }
3005 try testing.expectEqual(@as(u64, 0xbababababa), cqe.user_data);
3006 }
3007
3008 // This read should work
3009
3010 {
3011 _ = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3012 try testing.expectEqual(@as(u32, 1), try ring.submit());
3013
3014 const cqe = try ring.copy_cqe();
3015 switch (cqe.err()) {
3016 .SUCCESS => {},
3017 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3018 }
3019
3020 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3021 const used_buffer_id = cqe.flags >> 16;
3022 try testing.expectEqual(used_buffer_id, 0);
3023 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3024 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3025 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@intCast(usize, cqe.res)]);
3026 }
3027
3028 // Final read should _not_ work
3029
3030 {
3031 _ = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3032 try testing.expectEqual(@as(u32, 1), try ring.submit());
3033
3034 const cqe = try ring.copy_cqe();
3035 switch (cqe.err()) {
3036 // Expected
3037 .NOBUFS => {},
3038 .SUCCESS => std.debug.panic("unexpected success", .{}),
3039 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3040 }
3041 }
3042}
3043
3044test "provide_buffers: accept/connect/send/recv" {
3045 if (builtin.os.tag != .linux) return error.SkipZigTest;
3046
3047 var ring = IO_Uring.init(16, 0) catch |err| switch (err) {
3048 error.SystemOutdated => return error.SkipZigTest,
3049 error.PermissionDenied => return error.SkipZigTest,
3050 else => return err,
3051 };
3052 defer ring.deinit();
3053
3054 const group_id = 1337;
3055 const buffer_id = 0;
3056
3057 const buffer_len = 128;
3058 var buffers: [4][buffer_len]u8 = undefined;
3059
3060 // Provide 4 buffers
3061
3062 {
3063 const sqe = try ring.provide_buffers(0xcccccccc, @ptrCast([*]u8, &buffers), buffers.len, buffer_len, group_id, buffer_id);
3064 try testing.expectEqual(linux.IORING_OP.PROVIDE_BUFFERS, sqe.opcode);
3065 try testing.expectEqual(@as(i32, buffers.len), sqe.fd);
3066 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3067 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3068 try testing.expectEqual(@as(u32, 1), try ring.submit());
3069
3070 const cqe = try ring.copy_cqe();
3071 switch (cqe.err()) {
3072 // Happens when the kernel is < 5.7
3073 .INVAL => return error.SkipZigTest,
3074 .SUCCESS => {},
3075 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3076 }
3077 try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
3078 }
3079
3080 const socket_test_harness = try createSocketTestHarness(&ring);
3081 defer socket_test_harness.close();
3082
3083 // Do 4 send on the socket
3084
3085 {
3086 var i: usize = 0;
3087 while (i < buffers.len) : (i += 1) {
3088 _ = try ring.send(0xdeaddead, socket_test_harness.server, &([_]u8{'z'} ** buffer_len), 0);
3089 try testing.expectEqual(@as(u32, 1), try ring.submit());
3090 }
3091
3092 var cqes: [4]linux.io_uring_cqe = undefined;
3093 try testing.expectEqual(@as(u32, 4), try ring.copy_cqes(&cqes, 4));
3094 }
3095
3096 // Do 4 recv which should consume all buffers
3097
3098 // Deliberately put something we don't expect in the buffers
3099 mem.set(u8, mem.sliceAsBytes(&buffers), 1);
3100
3101 var i: usize = 0;
3102 while (i < buffers.len) : (i += 1) {
3103 var sqe = try ring.recv(0xdededede, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3104 try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
3105 try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
3106 try testing.expectEqual(@as(u64, 0), sqe.addr);
3107 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3108 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3109 try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
3110 try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
3111 try testing.expectEqual(@as(u32, 1), try ring.submit());
3112
3113 const cqe = try ring.copy_cqe();
3114 switch (cqe.err()) {
3115 .SUCCESS => {},
3116 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3117 }
3118
3119 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3120 const used_buffer_id = cqe.flags >> 16;
3121 try testing.expect(used_buffer_id >= 0 and used_buffer_id <= 3);
3122 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3123
3124 try testing.expectEqual(@as(u64, 0xdededede), cqe.user_data);
3125 const buffer = buffers[used_buffer_id][0..@intCast(usize, cqe.res)];
3126 try testing.expectEqualSlices(u8, &([_]u8{'z'} ** buffer_len), buffer);
3127 }
3128
3129 // This recv should fail
3130
3131 {
3132 var sqe = try ring.recv(0xdfdfdfdf, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3133 try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
3134 try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
3135 try testing.expectEqual(@as(u64, 0), sqe.addr);
3136 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3137 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3138 try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
3139 try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
3140 try testing.expectEqual(@as(u32, 1), try ring.submit());
3141
3142 const cqe = try ring.copy_cqe();
3143 switch (cqe.err()) {
3144 // Expected
3145 .NOBUFS => {},
3146 .SUCCESS => std.debug.panic("unexpected success", .{}),
3147 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3148 }
3149 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3150 }
3151
3152 // Provide 1 buffer again
3153
3154 const reprovided_buffer_id = 2;
3155
3156 {
3157 _ = try ring.provide_buffers(0xabababab, @ptrCast([*]u8, &buffers[reprovided_buffer_id]), 1, buffer_len, group_id, reprovided_buffer_id);
3158 try testing.expectEqual(@as(u32, 1), try ring.submit());
3159
3160 const cqe = try ring.copy_cqe();
3161 switch (cqe.err()) {
3162 .SUCCESS => {},
3163 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3164 }
3165 }
3166
3167 // Redo 1 send on the server socket
3168
3169 {
3170 _ = try ring.send(0xdeaddead, socket_test_harness.server, &([_]u8{'w'} ** buffer_len), 0);
3171 try testing.expectEqual(@as(u32, 1), try ring.submit());
3172
3173 _ = try ring.copy_cqe();
3174 }
3175
3176 // Final recv which should work
3177
3178 // Deliberately put something we don't expect in the buffers
3179 mem.set(u8, mem.sliceAsBytes(&buffers), 1);
3180
3181 {
3182 var sqe = try ring.recv(0xdfdfdfdf, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3183 try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
3184 try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
3185 try testing.expectEqual(@as(u64, 0), sqe.addr);
3186 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3187 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3188 try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
3189 try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
3190 try testing.expectEqual(@as(u32, 1), try ring.submit());
3191
3192 const cqe = try ring.copy_cqe();
3193 switch (cqe.err()) {
3194 .SUCCESS => {},
3195 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3196 }
3197
3198 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3199 const used_buffer_id = cqe.flags >> 16;
3200 try testing.expectEqual(used_buffer_id, reprovided_buffer_id);
3201 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3202 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3203 const buffer = buffers[used_buffer_id][0..@intCast(usize, cqe.res)];
3204 try testing.expectEqualSlices(u8, &([_]u8{'w'} ** buffer_len), buffer);
3205 }
3206}
3207
3208/// Used for testing server/client interactions.
3209const SocketTestHarness = struct {
3210 listener: os.socket_t,
3211 server: os.socket_t,
3212 client: os.socket_t,
3213
3214 fn close(self: SocketTestHarness) void {
3215 os.closeSocket(self.client);
3216 os.closeSocket(self.listener);
3217 }
3218};
3219
3220fn createSocketTestHarness(ring: *IO_Uring) !SocketTestHarness {
3221 // Create a TCP server socket
3222
3223 const address = try net.Address.parseIp4("127.0.0.1", 3131);
3224 const kernel_backlog = 1;
3225 const listener_socket = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3226 errdefer os.closeSocket(listener_socket);
3227
3228 try os.setsockopt(listener_socket, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
3229 try os.bind(listener_socket, &address.any, address.getOsSockLen());
3230 try os.listen(listener_socket, kernel_backlog);
3231
3232 // Submit 1 accept
3233 var accept_addr: os.sockaddr = undefined;
3234 var accept_addr_len: os.socklen_t = @sizeOf(@TypeOf(accept_addr));
3235 _ = try ring.accept(0xaaaaaaaa, listener_socket, &accept_addr, &accept_addr_len, 0);
3236
3237 // Create a TCP client socket
3238 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3239 errdefer os.closeSocket(client);
3240 _ = try ring.connect(0xcccccccc, client, &address.any, address.getOsSockLen());
3241
3242 try testing.expectEqual(@as(u32, 2), try ring.submit());
3243
3244 var cqe_accept = try ring.copy_cqe();
3245 if (cqe_accept.err() == .INVAL) return error.SkipZigTest;
3246 var cqe_connect = try ring.copy_cqe();
3247 if (cqe_connect.err() == .INVAL) return error.SkipZigTest;
3248
3249 // The accept/connect CQEs may arrive in any order, the connect CQE will sometimes come first:
3250 if (cqe_accept.user_data == 0xcccccccc and cqe_connect.user_data == 0xaaaaaaaa) {
3251 const a = cqe_accept;
3252 const b = cqe_connect;
3253 cqe_accept = b;
3254 cqe_connect = a;
3255 }
3256
3257 try testing.expectEqual(@as(u64, 0xaaaaaaaa), cqe_accept.user_data);
3258 if (cqe_accept.res <= 0) std.debug.print("\ncqe_accept.res={}\n", .{cqe_accept.res});
3259 try testing.expect(cqe_accept.res > 0);
3260 try testing.expectEqual(@as(u32, 0), cqe_accept.flags);
3261 try testing.expectEqual(linux.io_uring_cqe{
3262 .user_data = 0xcccccccc,
3263 .res = 0,
3264 .flags = 0,
3265 }, cqe_connect);
3266
3267 // All good
3268
3269 return SocketTestHarness{
3270 .listener = listener_socket,
3271 .server = cqe_accept.res,
3272 .client = client,
3273 };
3274}