authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-03-16 03:34:31+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-03-15 22:34:31-04:00
log3ea1276eebfa1cd408e9c45d1704ff294e95b807
treeefec37a47393a8e7e09e5ed321a220b6294b659b
parentce4245f873c68cfcc431deb303383f25f6d50855
signaturebadge-check Signed by PGP key B5690EEEBB952194

io_uring: ring mapped buffers (#17806)

* io_uring: ring mapped buffers Ring mapped buffers are newer implementation of ring provided buffers, supported since kernel 5.19. Best described in Jens Axboe [post](https://github.com/axboe/liburing/wiki/io_uring-and-networking-in-2023#provided-buffers) This commit implements low level io_uring_*_buf_ring_* functions as mostly direct translation from liburing. It also adds BufferGroup abstraction over those low level functions. * io_uring: add multishot recv to BufferGroup Once we have ring mapped provided buffers functionality it is possible to use multishot recv operation. Multishot receive is submitted once, and completions are posted whenever data arrives on the socket. Received data are placed in a new buffer from buffer group. Reference: [io_uring and networking in 2023](https://github.com/axboe/liburing/wiki/io_uring-and-networking-in-2023#multi-shot) Getting NOENT for cancel completion result, meaning: -ENOENT The request identified by user_data could not be located. This could be because it completed before the cancelation request was issued, or if an invalid identifier is used. https://man7.org/linux/man-pages/man3/io_uring_prep_cancel.3.html https://github.com/ziglang/zig/actions/runs/6801394000/job/18492139893?pr=17806 Result in cancel/recv cqes are different depending on the kernel. on older kernel (tested with v6.0.16, v6.1.57, v6.2.12, v6.4.16) cqe_cancel.err() == .NOENT cqe_crecv.err() == .NOBUFS on kernel (tested with v6.5.0, v6.5.7) cqe_cancel.err() == .SUCCESS cqe_crecv.err() == .CANCELED

3 files changed, 610 insertions(+), 14 deletions(-)

lib/std/os/linux.zig+16-2
......@@ -4387,6 +4387,16 @@ pub const io_uring_cqe = extern struct {
43874387 }
43884388 return .SUCCESS;
43894389 }
4390
4391 // On successful completion of the provided buffers IO request, the CQE flags field
4392 // will have IORING_CQE_F_BUFFER set and the selected buffer ID will be indicated by
4393 // the upper 16-bits of the flags field.
4394 pub fn buffer_id(self: io_uring_cqe) !u16 {
4395 if (self.flags & IORING_CQE_F_BUFFER != IORING_CQE_F_BUFFER) {
4396 return error.NoBufferSelected;
4397 }
4398 return @as(u16, @intCast(self.flags >> IORING_CQE_BUFFER_SHIFT));
4399 }
43904400};
43914401
43924402// io_uring_cqe.flags
......@@ -4667,8 +4677,12 @@ pub const io_uring_buf = extern struct {
46674677 resv: u16,
46684678};
46694679
4670// io_uring_buf_ring struct omitted
4671// it's a io_uring_buf array with the resv of the first item used as a "tail" field.
4680pub const io_uring_buf_ring = extern struct {
4681 resv1: u64,
4682 resv2: u32,
4683 resv3: u16,
4684 tail: u16,
4685};
46724686
46734687/// argument for IORING_(UN)REGISTER_PBUF_RING
46744688pub const io_uring_buf_reg = extern struct {
lib/std/os/linux/IoUring.zig+564-2
......@@ -1,5 +1,5 @@
11const IoUring = @This();
2const std = @import("../../std.zig");
2const std = @import("std");
33const builtin = @import("builtin");
44const assert = std.debug.assert;
55const mem = std.mem;
......@@ -1440,6 +1440,229 @@ pub const CompletionQueue = struct {
14401440 }
14411441};
14421442
1443/// Group of application provided buffers. Uses newer type, called ring mapped
1444/// buffers, supported since kernel 5.19. Buffers are identified by a buffer
1445/// group ID, and within that group, a buffer ID. IO_Uring can have multiple
1446/// buffer groups, each with unique group ID.
1447///
1448/// In `init` application provides contiguous block of memory `buffers` for
1449/// `buffers_count` buffers of size `buffers_size`. Application can then submit
1450/// `recv` operation without providing buffer upfront. Once the operation is
1451/// ready to receive data, a buffer is picked automatically and the resulting
1452/// CQE will contain the buffer ID in `cqe.buffer_id()`. Use `get` method to get
1453/// buffer for buffer ID identified by CQE. Once the application has processed
1454/// the buffer, it may hand ownership back to the kernel, by calling `put`
1455/// allowing the cycle to repeat.
1456///
1457/// Depending on the rate of arrival of data, it is possible that a given buffer
1458/// group will run out of buffers before those in CQEs can be put back to the
1459/// kernel. If this happens, a `cqe.err()` will have ENOBUFS as the error value.
1460///
1461pub const BufferGroup = struct {
1462 /// Parent ring for which this group is registered.
1463 ring: *IoUring,
1464 /// Pointer to the memory shared by the kernel.
1465 /// `buffers_count` of `io_uring_buf` structures are shared by the kernel.
1466 /// First `io_uring_buf` is overlaid by `io_uring_buf_ring` struct.
1467 br: *align(mem.page_size) linux.io_uring_buf_ring,
1468 /// Contiguous block of memory of size (buffers_count * buffer_size).
1469 buffers: []u8,
1470 /// Size of each buffer in buffers.
1471 buffer_size: u32,
1472 // Number of buffers in `buffers`, number of `io_uring_buf structures` in br.
1473 buffers_count: u16,
1474 /// ID of this group, must be unique in ring.
1475 group_id: u16,
1476
1477 pub fn init(
1478 ring: *IoUring,
1479 group_id: u16,
1480 buffers: []u8,
1481 buffer_size: u32,
1482 buffers_count: u16,
1483 ) !BufferGroup {
1484 assert(buffers.len == buffers_count * buffer_size);
1485
1486 const br = try setup_buf_ring(ring.fd, buffers_count, group_id);
1487 buf_ring_init(br);
1488
1489 const mask = buf_ring_mask(buffers_count);
1490 var i: u16 = 0;
1491 while (i < buffers_count) : (i += 1) {
1492 const start = buffer_size * i;
1493 const buf = buffers[start .. start + buffer_size];
1494 buf_ring_add(br, buf, i, mask, i);
1495 }
1496 buf_ring_advance(br, buffers_count);
1497
1498 return BufferGroup{
1499 .ring = ring,
1500 .group_id = group_id,
1501 .br = br,
1502 .buffers = buffers,
1503 .buffer_size = buffer_size,
1504 .buffers_count = buffers_count,
1505 };
1506 }
1507
1508 // Prepare recv operation which will select buffer from this group.
1509 pub fn recv(self: *BufferGroup, user_data: u64, fd: os.fd_t, flags: u32) !*linux.io_uring_sqe {
1510 var sqe = try self.ring.get_sqe();
1511 sqe.prep_rw(.RECV, fd, 0, 0, 0);
1512 sqe.rw_flags = flags;
1513 sqe.flags |= linux.IOSQE_BUFFER_SELECT;
1514 sqe.buf_index = self.group_id;
1515 sqe.user_data = user_data;
1516 return sqe;
1517 }
1518
1519 // Prepare multishot recv operation which will select buffer from this group.
1520 pub fn recv_multishot(self: *BufferGroup, user_data: u64, fd: os.fd_t, flags: u32) !*linux.io_uring_sqe {
1521 var sqe = try self.recv(user_data, fd, flags);
1522 sqe.ioprio |= linux.IORING_RECV_MULTISHOT;
1523 return sqe;
1524 }
1525
1526 // Get buffer by id.
1527 pub fn get(self: *BufferGroup, buffer_id: u16) []u8 {
1528 const head = self.buffer_size * buffer_id;
1529 return self.buffers[head .. head + self.buffer_size];
1530 }
1531
1532 // Get buffer by CQE.
1533 pub fn get_cqe(self: *BufferGroup, cqe: linux.io_uring_cqe) ![]u8 {
1534 const buffer_id = try cqe.buffer_id();
1535 const used_len = @as(usize, @intCast(cqe.res));
1536 return self.get(buffer_id)[0..used_len];
1537 }
1538
1539 // Release buffer to the kernel.
1540 pub fn put(self: *BufferGroup, buffer_id: u16) void {
1541 const mask = buf_ring_mask(self.buffers_count);
1542 const buffer = self.get(buffer_id);
1543 buf_ring_add(self.br, buffer, buffer_id, mask, 0);
1544 buf_ring_advance(self.br, 1);
1545 }
1546
1547 // Release buffer from CQE to the kernel.
1548 pub fn put_cqe(self: *BufferGroup, cqe: linux.io_uring_cqe) !void {
1549 self.put(try cqe.buffer_id());
1550 }
1551
1552 pub fn deinit(self: *BufferGroup) void {
1553 free_buf_ring(self.ring.fd, self.br, self.buffers_count, self.group_id);
1554 }
1555};
1556
1557/// Registers a shared buffer ring to be used with provided buffers.
1558/// `entries` number of `io_uring_buf` structures is mem mapped and shared by kernel.
1559/// `fd` is IO_Uring.fd for which the provided buffer ring is being registered.
1560/// `entries` is the number of entries requested in the buffer ring, must be power of 2.
1561/// `group_id` is the chosen buffer group ID, unique in IO_Uring.
1562pub fn setup_buf_ring(fd: os.fd_t, entries: u16, group_id: u16) !*align(mem.page_size) linux.io_uring_buf_ring {
1563 if (entries == 0 or entries > 1 << 15) return error.EntriesNotInRange;
1564 if (!std.math.isPowerOfTwo(entries)) return error.EntriesNotPowerOfTwo;
1565
1566 const mmap_size = entries * @sizeOf(linux.io_uring_buf);
1567 const mmap = try os.mmap(
1568 null,
1569 mmap_size,
1570 os.PROT.READ | os.PROT.WRITE,
1571 .{ .TYPE = .PRIVATE, .ANONYMOUS = true },
1572 -1,
1573 0,
1574 );
1575 errdefer os.munmap(mmap);
1576 assert(mmap.len == mmap_size);
1577
1578 const br: *align(mem.page_size) linux.io_uring_buf_ring = @ptrCast(mmap.ptr);
1579 try register_buf_ring(fd, @intFromPtr(br), entries, group_id);
1580 return br;
1581}
1582
1583fn register_buf_ring(fd: os.fd_t, addr: u64, entries: u32, group_id: u16) !void {
1584 var reg = mem.zeroInit(linux.io_uring_buf_reg, .{
1585 .ring_addr = addr,
1586 .ring_entries = entries,
1587 .bgid = group_id,
1588 });
1589 const res = linux.io_uring_register(
1590 fd,
1591 .REGISTER_PBUF_RING,
1592 @as(*const anyopaque, @ptrCast(&reg)),
1593 1,
1594 );
1595 try handle_register_buf_ring_result(res);
1596}
1597
1598fn unregister_buf_ring(fd: os.fd_t, group_id: u16) !void {
1599 var reg = mem.zeroInit(linux.io_uring_buf_reg, .{
1600 .bgid = group_id,
1601 });
1602 const res = linux.io_uring_register(
1603 fd,
1604 .UNREGISTER_PBUF_RING,
1605 @as(*const anyopaque, @ptrCast(&reg)),
1606 1,
1607 );
1608 try handle_register_buf_ring_result(res);
1609}
1610
1611fn handle_register_buf_ring_result(res: usize) !void {
1612 switch (linux.getErrno(res)) {
1613 .SUCCESS => {},
1614 .INVAL => return error.ArgumentsInvalid,
1615 else => |errno| return os.unexpectedErrno(errno),
1616 }
1617}
1618
1619// Unregisters a previously registered shared buffer ring, returned from io_uring_setup_buf_ring.
1620pub fn free_buf_ring(fd: os.fd_t, br: *align(mem.page_size) linux.io_uring_buf_ring, entries: u32, group_id: u16) void {
1621 unregister_buf_ring(fd, group_id) catch {};
1622 var mmap: []align(mem.page_size) u8 = undefined;
1623 mmap.ptr = @ptrCast(br);
1624 mmap.len = entries * @sizeOf(linux.io_uring_buf);
1625 os.munmap(mmap);
1626}
1627
1628/// Initialises `br` so that it is ready to be used.
1629pub fn buf_ring_init(br: *linux.io_uring_buf_ring) void {
1630 br.tail = 0;
1631}
1632
1633/// Calculates the appropriate size mask for a buffer ring.
1634/// `entries` is the ring entries as specified in io_uring_register_buf_ring.
1635pub fn buf_ring_mask(entries: u16) u16 {
1636 return entries - 1;
1637}
1638
1639/// Assigns `buffer` with the `br` buffer ring.
1640/// `buffer_id` is identifier which will be returned in the CQE.
1641/// `buffer_offset` is the offset to insert at from the current tail.
1642/// If just one buffer is provided before the ring tail is committed with advance then offset should be 0.
1643/// If buffers are provided in a loop before being committed, the offset must be incremented by one for each buffer added.
1644pub fn buf_ring_add(
1645 br: *linux.io_uring_buf_ring,
1646 buffer: []u8,
1647 buffer_id: u16,
1648 mask: u16,
1649 buffer_offset: u16,
1650) void {
1651 const bufs: [*]linux.io_uring_buf = @ptrCast(br);
1652 const buf: *linux.io_uring_buf = &bufs[(br.tail +% buffer_offset) & mask];
1653
1654 buf.addr = @intFromPtr(buffer.ptr);
1655 buf.len = @intCast(buffer.len);
1656 buf.bid = buffer_id;
1657}
1658
1659/// Make `count` new buffers visible to the kernel. Called after
1660/// `io_uring_buf_ring_add` has been called `count` times to fill in new buffers.
1661pub fn buf_ring_advance(br: *linux.io_uring_buf_ring, count: u16) void {
1662 const tail: u16 = br.tail +% count;
1663 @atomicStore(u16, &br.tail, tail, .release);
1664}
1665
14431666test "structs/offsets/entries" {
14441667 if (builtin.os.tag != .linux) return error.SkipZigTest;
14451668
......@@ -3652,7 +3875,7 @@ test "waitid" {
36523875 try testing.expectEqual(7, siginfo.fields.common.second.sigchld.status);
36533876}
36543877
3655/// For use in tests. Returns SkipZigTest is kernel version is less than required.
3878/// For use in tests. Returns SkipZigTest if kernel version is less than required.
36563879inline fn skipKernelLessThan(required: std.SemanticVersion) !void {
36573880 if (builtin.os.tag != .linux) return error.SkipZigTest;
36583881
......@@ -3668,3 +3891,342 @@ inline fn skipKernelLessThan(required: std.SemanticVersion) !void {
36683891 current.pre = null; // don't check pre field
36693892 if (required.order(current) == .gt) return error.SkipZigTest;
36703893}
3894
3895test BufferGroup {
3896 if (builtin.os.tag != .linux) return error.SkipZigTest;
3897
3898 // Init IoUring
3899 var ring = IoUring.init(16, 0) catch |err| switch (err) {
3900 error.SystemOutdated => return error.SkipZigTest,
3901 error.PermissionDenied => return error.SkipZigTest,
3902 else => return err,
3903 };
3904 defer ring.deinit();
3905
3906 // Init buffer group for ring
3907 const group_id: u16 = 1; // buffers group id
3908 const buffers_count: u16 = 1; // number of buffers in buffer group
3909 const buffer_size: usize = 128; // size of each buffer in group
3910 const buffers = try testing.allocator.alloc(u8, buffers_count * buffer_size);
3911 defer testing.allocator.free(buffers);
3912 var buf_grp = BufferGroup.init(
3913 &ring,
3914 group_id,
3915 buffers,
3916 buffer_size,
3917 buffers_count,
3918 ) catch |err| switch (err) {
3919 // kernel older than 5.19
3920 error.ArgumentsInvalid => return error.SkipZigTest,
3921 else => return err,
3922 };
3923 defer buf_grp.deinit();
3924
3925 // Create client/server fds
3926 const fds = try createSocketTestHarness(&ring);
3927 defer fds.close();
3928 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
3929
3930 // Client sends data
3931 {
3932 _ = try ring.send(1, fds.client, data[0..], 0);
3933 const submitted = try ring.submit();
3934 try testing.expectEqual(1, submitted);
3935 const cqe_send = try ring.copy_cqe();
3936 if (cqe_send.err() == .INVAL) return error.SkipZigTest;
3937 try testing.expectEqual(linux.io_uring_cqe{ .user_data = 1, .res = data.len, .flags = 0 }, cqe_send);
3938 }
3939
3940 // Server uses buffer group receive
3941 {
3942 // Submit recv operation, buffer will be choosen from buffer group
3943 _ = try buf_grp.recv(2, fds.server, 0);
3944 const submitted = try ring.submit();
3945 try testing.expectEqual(1, submitted);
3946
3947 // ... when we have completion for recv operation
3948 const cqe = try ring.copy_cqe();
3949 try testing.expectEqual(2, cqe.user_data); // matches submitted user_data
3950 try testing.expect(cqe.res >= 0); // success
3951 try testing.expectEqual(os.E.SUCCESS, cqe.err());
3952 try testing.expectEqual(data.len, @as(usize, @intCast(cqe.res))); // cqe.res holds received data len
3953
3954 // Read buffer_id and used buffer len from cqe
3955 const buffer_id = try cqe.buffer_id();
3956 const len: usize = @intCast(cqe.res);
3957 // Get buffer from pool
3958 const buf = buf_grp.get(buffer_id)[0..len];
3959 try testing.expectEqualSlices(u8, &data, buf);
3960 // Releaase buffer to the kernel when application is done with it
3961 buf_grp.put(buffer_id);
3962 }
3963}
3964
3965test "ring mapped buffers recv" {
3966 if (builtin.os.tag != .linux) return error.SkipZigTest;
3967
3968 var ring = IoUring.init(16, 0) catch |err| switch (err) {
3969 error.SystemOutdated => return error.SkipZigTest,
3970 error.PermissionDenied => return error.SkipZigTest,
3971 else => return err,
3972 };
3973 defer ring.deinit();
3974
3975 // init buffer group
3976 const group_id: u16 = 1; // buffers group id
3977 const buffers_count: u16 = 2; // number of buffers in buffer group
3978 const buffer_size: usize = 4; // size of each buffer in group
3979 const buffers = try testing.allocator.alloc(u8, buffers_count * buffer_size);
3980 defer testing.allocator.free(buffers);
3981 var buf_grp = BufferGroup.init(
3982 &ring,
3983 group_id,
3984 buffers,
3985 buffer_size,
3986 buffers_count,
3987 ) catch |err| switch (err) {
3988 // kernel older than 5.19
3989 error.ArgumentsInvalid => return error.SkipZigTest,
3990 else => return err,
3991 };
3992 defer buf_grp.deinit();
3993
3994 // create client/server fds
3995 const fds = try createSocketTestHarness(&ring);
3996 defer fds.close();
3997
3998 // for random user_data in sqe/cqe
3999 var Rnd = std.rand.DefaultPrng.init(0);
4000 var rnd = Rnd.random();
4001
4002 var round: usize = 4; // repeat send/recv cycle round times
4003 while (round > 0) : (round -= 1) {
4004 // client sends data
4005 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
4006 {
4007 const user_data = rnd.int(u64);
4008 _ = try ring.send(user_data, fds.client, data[0..], 0);
4009 try testing.expectEqual(@as(u32, 1), try ring.submit());
4010 const cqe_send = try ring.copy_cqe();
4011 if (cqe_send.err() == .INVAL) return error.SkipZigTest;
4012 try testing.expectEqual(linux.io_uring_cqe{ .user_data = user_data, .res = data.len, .flags = 0 }, cqe_send);
4013 }
4014
4015 // server reads data into provided buffers
4016 // there are 2 buffers of size 4, so each read gets only chunk of data
4017 // we read four chunks of 4, 4, 4, 3 bytes each
4018 var chunk: []const u8 = data[0..buffer_size]; // first chunk
4019 const id1 = try expect_buf_grp_recv(&ring, &buf_grp, fds.server, rnd.int(u64), chunk);
4020 chunk = data[buffer_size .. buffer_size * 2]; // second chunk
4021 const id2 = try expect_buf_grp_recv(&ring, &buf_grp, fds.server, rnd.int(u64), chunk);
4022
4023 // both buffers provided to the kernel are used so we get error
4024 // 'no more buffers', until we put buffers to the kernel
4025 {
4026 const user_data = rnd.int(u64);
4027 _ = try buf_grp.recv(user_data, fds.server, 0);
4028 try testing.expectEqual(@as(u32, 1), try ring.submit());
4029 const cqe = try ring.copy_cqe();
4030 try testing.expectEqual(user_data, cqe.user_data);
4031 try testing.expect(cqe.res < 0); // fail
4032 try testing.expectEqual(os.E.NOBUFS, cqe.err());
4033 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == 0); // IORING_CQE_F_BUFFER flags is set on success only
4034 try testing.expectError(error.NoBufferSelected, cqe.buffer_id());
4035 }
4036
4037 // put buffers back to the kernel
4038 buf_grp.put(id1);
4039 buf_grp.put(id2);
4040
4041 chunk = data[buffer_size * 2 .. buffer_size * 3]; // third chunk
4042 const id3 = try expect_buf_grp_recv(&ring, &buf_grp, fds.server, rnd.int(u64), chunk);
4043 buf_grp.put(id3);
4044
4045 chunk = data[buffer_size * 3 ..]; // last chunk
4046 const id4 = try expect_buf_grp_recv(&ring, &buf_grp, fds.server, rnd.int(u64), chunk);
4047 buf_grp.put(id4);
4048 }
4049}
4050
4051test "ring mapped buffers multishot recv" {
4052 if (builtin.os.tag != .linux) return error.SkipZigTest;
4053
4054 var ring = IoUring.init(16, 0) catch |err| switch (err) {
4055 error.SystemOutdated => return error.SkipZigTest,
4056 error.PermissionDenied => return error.SkipZigTest,
4057 else => return err,
4058 };
4059 defer ring.deinit();
4060
4061 // init buffer group
4062 const group_id: u16 = 1; // buffers group id
4063 const buffers_count: u16 = 2; // number of buffers in buffer group
4064 const buffer_size: usize = 4; // size of each buffer in group
4065 const buffers = try testing.allocator.alloc(u8, buffers_count * buffer_size);
4066 defer testing.allocator.free(buffers);
4067 var buf_grp = BufferGroup.init(
4068 &ring,
4069 group_id,
4070 buffers,
4071 buffer_size,
4072 buffers_count,
4073 ) catch |err| switch (err) {
4074 // kernel older than 5.19
4075 error.ArgumentsInvalid => return error.SkipZigTest,
4076 else => return err,
4077 };
4078 defer buf_grp.deinit();
4079
4080 // create client/server fds
4081 const fds = try createSocketTestHarness(&ring);
4082 defer fds.close();
4083
4084 // for random user_data in sqe/cqe
4085 var Rnd = std.rand.DefaultPrng.init(0);
4086 var rnd = Rnd.random();
4087
4088 var round: usize = 4; // repeat send/recv cycle round times
4089 while (round > 0) : (round -= 1) {
4090 // client sends data
4091 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
4092 {
4093 const user_data = rnd.int(u64);
4094 _ = try ring.send(user_data, fds.client, data[0..], 0);
4095 try testing.expectEqual(@as(u32, 1), try ring.submit());
4096 const cqe_send = try ring.copy_cqe();
4097 if (cqe_send.err() == .INVAL) return error.SkipZigTest;
4098 try testing.expectEqual(linux.io_uring_cqe{ .user_data = user_data, .res = data.len, .flags = 0 }, cqe_send);
4099 }
4100
4101 // start multishot recv
4102 var recv_user_data = rnd.int(u64);
4103 _ = try buf_grp.recv_multishot(recv_user_data, fds.server, 0);
4104 try testing.expectEqual(@as(u32, 1), try ring.submit()); // submit
4105
4106 // server reads data into provided buffers
4107 // there are 2 buffers of size 4, so each read gets only chunk of data
4108 // we read four chunks of 4, 4, 4, 3 bytes each
4109 var chunk: []const u8 = data[0..buffer_size]; // first chunk
4110 const cqe1 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);
4111 try testing.expect(cqe1.flags & linux.IORING_CQE_F_MORE > 0);
4112
4113 chunk = data[buffer_size .. buffer_size * 2]; // second chunk
4114 const cqe2 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);
4115 try testing.expect(cqe2.flags & linux.IORING_CQE_F_MORE > 0);
4116
4117 // both buffers provided to the kernel are used so we get error
4118 // 'no more buffers', until we put buffers to the kernel
4119 {
4120 const cqe = try ring.copy_cqe();
4121 try testing.expectEqual(recv_user_data, cqe.user_data);
4122 try testing.expect(cqe.res < 0); // fail
4123 try testing.expectEqual(os.E.NOBUFS, cqe.err());
4124 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == 0); // IORING_CQE_F_BUFFER flags is set on success only
4125 // has more is not set
4126 // indicates that multishot is finished
4127 try testing.expect(cqe.flags & linux.IORING_CQE_F_MORE == 0);
4128 try testing.expectError(error.NoBufferSelected, cqe.buffer_id());
4129 }
4130
4131 // put buffers back to the kernel
4132 buf_grp.put(try cqe1.buffer_id());
4133 buf_grp.put(try cqe2.buffer_id());
4134
4135 // restart multishot
4136 recv_user_data = rnd.int(u64);
4137 _ = try buf_grp.recv_multishot(recv_user_data, fds.server, 0);
4138 try testing.expectEqual(@as(u32, 1), try ring.submit()); // submit
4139
4140 chunk = data[buffer_size * 2 .. buffer_size * 3]; // third chunk
4141 const cqe3 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);
4142 try testing.expect(cqe3.flags & linux.IORING_CQE_F_MORE > 0);
4143 buf_grp.put(try cqe3.buffer_id());
4144
4145 chunk = data[buffer_size * 3 ..]; // last chunk
4146 const cqe4 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);
4147 try testing.expect(cqe4.flags & linux.IORING_CQE_F_MORE > 0);
4148 buf_grp.put(try cqe4.buffer_id());
4149
4150 // cancel pending multishot recv operation
4151 {
4152 const cancel_user_data = rnd.int(u64);
4153 _ = try ring.cancel(cancel_user_data, recv_user_data, 0);
4154 try testing.expectEqual(@as(u32, 1), try ring.submit());
4155
4156 // expect completion of cancel operation and completion of recv operation
4157 var cqe_cancel = try ring.copy_cqe();
4158 if (cqe_cancel.err() == .INVAL) return error.SkipZigTest;
4159 var cqe_recv = try ring.copy_cqe();
4160 if (cqe_recv.err() == .INVAL) return error.SkipZigTest;
4161
4162 // don't depend on order of completions
4163 if (cqe_cancel.user_data == recv_user_data and cqe_recv.user_data == cancel_user_data) {
4164 const a = cqe_cancel;
4165 const b = cqe_recv;
4166 cqe_cancel = b;
4167 cqe_recv = a;
4168 }
4169
4170 // Note on different kernel results:
4171 // on older kernel (tested with v6.0.16, v6.1.57, v6.2.12, v6.4.16)
4172 // cqe_cancel.err() == .NOENT
4173 // cqe_recv.err() == .NOBUFS
4174 // on kernel (tested with v6.5.0, v6.5.7)
4175 // cqe_cancel.err() == .SUCCESS
4176 // cqe_recv.err() == .CANCELED
4177 // Upstream reference: https://github.com/axboe/liburing/issues/984
4178
4179 // cancel operation is success (or NOENT on older kernels)
4180 try testing.expectEqual(cancel_user_data, cqe_cancel.user_data);
4181 try testing.expect(cqe_cancel.err() == .NOENT or cqe_cancel.err() == .SUCCESS);
4182
4183 // recv operation is failed with err CANCELED (or NOBUFS on older kernels)
4184 try testing.expectEqual(recv_user_data, cqe_recv.user_data);
4185 try testing.expect(cqe_recv.res < 0);
4186 try testing.expect(cqe_recv.err() == .NOBUFS or cqe_recv.err() == .CANCELED);
4187 try testing.expect(cqe_recv.flags & linux.IORING_CQE_F_MORE == 0);
4188 }
4189 }
4190}
4191
4192// Prepare and submit recv using buffer group.
4193// Test that buffer from group, pointed by cqe, matches expected.
4194fn expect_buf_grp_recv(
4195 ring: *IoUring,
4196 buf_grp: *BufferGroup,
4197 fd: os.fd_t,
4198 user_data: u64,
4199 expected: []const u8,
4200) !u16 {
4201 // prepare and submit read
4202 const sqe = try buf_grp.recv(user_data, fd, 0);
4203 try testing.expect(sqe.flags & linux.IOSQE_BUFFER_SELECT == linux.IOSQE_BUFFER_SELECT);
4204 try testing.expect(sqe.buf_index == buf_grp.group_id);
4205 try testing.expectEqual(@as(u32, 1), try ring.submit()); // submit
4206
4207 const cqe = try expect_buf_grp_cqe(ring, buf_grp, user_data, expected);
4208 return try cqe.buffer_id();
4209}
4210
4211fn expect_buf_grp_cqe(
4212 ring: *IoUring,
4213 buf_grp: *BufferGroup,
4214 user_data: u64,
4215 expected: []const u8,
4216) !linux.io_uring_cqe {
4217 // get cqe
4218 const cqe = try ring.copy_cqe();
4219 try testing.expectEqual(user_data, cqe.user_data);
4220 try testing.expect(cqe.res >= 0); // success
4221 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER); // IORING_CQE_F_BUFFER flag is set
4222 try testing.expectEqual(expected.len, @as(usize, @intCast(cqe.res)));
4223 try testing.expectEqual(os.E.SUCCESS, cqe.err());
4224
4225 // get buffer from pool
4226 const buffer_id = try cqe.buffer_id();
4227 const len = @as(usize, @intCast(cqe.res));
4228 const buf = buf_grp.get(buffer_id)[0..len];
4229 try testing.expectEqualSlices(u8, expected, buf);
4230
4231 return cqe;
4232}
lib/std/os/linux/io_uring_sqe.zig+30-10
......@@ -200,6 +200,36 @@ pub const io_uring_sqe = extern struct {
200200 sqe.rw_flags = flags;
201201 }
202202
203 pub fn prep_recv_multishot(
204 sqe: *linux.io_uring_sqe,
205 fd: os.fd_t,
206 buffer: []u8,
207 flags: u32,
208 ) void {
209 sqe.prep_recv(fd, buffer, flags);
210 sqe.ioprio |= linux.IORING_RECV_MULTISHOT;
211 }
212
213 pub fn prep_recvmsg(
214 sqe: *linux.io_uring_sqe,
215 fd: os.fd_t,
216 msg: *os.msghdr,
217 flags: u32,
218 ) void {
219 sqe.prep_rw(.RECVMSG, fd, @intFromPtr(msg), 1, 0);
220 sqe.rw_flags = flags;
221 }
222
223 pub fn prep_recvmsg_multishot(
224 sqe: *linux.io_uring_sqe,
225 fd: os.fd_t,
226 msg: *os.msghdr,
227 flags: u32,
228 ) void {
229 sqe.prep_recvmsg(fd, msg, flags);
230 sqe.ioprio |= linux.IORING_RECV_MULTISHOT;
231 }
232
203233 pub fn prep_send(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, flags: u32) void {
204234 sqe.prep_rw(.SEND, fd, @intFromPtr(buffer.ptr), buffer.len, 0);
205235 sqe.rw_flags = flags;
......@@ -227,16 +257,6 @@ pub const io_uring_sqe = extern struct {
227257 sqe.opcode = .SENDMSG_ZC;
228258 }
229259
230 pub fn prep_recvmsg(
231 sqe: *linux.io_uring_sqe,
232 fd: os.fd_t,
233 msg: *os.msghdr,
234 flags: u32,
235 ) void {
236 sqe.prep_rw(.RECVMSG, fd, @intFromPtr(msg), 1, 0);
237 sqe.rw_flags = flags;
238 }
239
240260 pub fn prep_sendmsg(
241261 sqe: *linux.io_uring_sqe,
242262 fd: os.fd_t,