authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2025-03-03 14:37:52+01:00
committergravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2025-03-05 13:35:52+01:00
logc133171567fe3a81f817d0ea159bd9229d75291c
treed6565515345ccf2fe738744af06bf5cbf5cc6a91
parent4df039d235d5f77830fdc30a4c23121f6216364a

io_uring: incremental provided buffer consumption

[Incremental provided buffer consumption](https://github.com/axboe/liburing/wiki/What's-new-with-io_uring-in-6.11-and-6.12#incremental-provided-buffer-consumption) support is added in kernel 6.12. IoUring.BufferGroup will now use incremental consumption whenever kernel supports it. Before, provided buffers are wholly consumed when picked. Each cqe points to the different buffer. With this, cqe points to the part of the buffer. Multiple cqe's can reuse same buffer. Appropriate sizing of buffers becomes less important. There are slight changes in BufferGroup interface (it now needs to track current receive point for each buffer). Init requires allocator instead of buffers slice, it will allocate buffers slice and head pointers slice. Get and put now requires cqe becasue there we have information will the buffer be reused.

2 files changed, 106 insertions(+), 84 deletions(-)

lib/std/os/linux.zig+8-1
...@@ -5933,6 +5933,8 @@ pub const IORING_CQE_F_MORE = 1 << 1;...@@ -5933,6 +5933,8 @@ pub const IORING_CQE_F_MORE = 1 << 1;
5933pub const IORING_CQE_F_SOCK_NONEMPTY = 1 << 2;5933pub const IORING_CQE_F_SOCK_NONEMPTY = 1 << 2;
5934/// Set for notification CQEs. Can be used to distinct them from sends.5934/// Set for notification CQEs. Can be used to distinct them from sends.
5935pub const IORING_CQE_F_NOTIF = 1 << 3;5935pub const IORING_CQE_F_NOTIF = 1 << 3;
5936/// If set, the buffer ID set in the completion will get more completions.
5937pub const IORING_CQE_F_BUF_MORE = 1 << 4;
59365938
5937pub const IORING_CQE_BUFFER_SHIFT = 16;5939pub const IORING_CQE_BUFFER_SHIFT = 16;
59385940
...@@ -6222,8 +6224,13 @@ pub const io_uring_buf_reg = extern struct {...@@ -6222,8 +6224,13 @@ pub const io_uring_buf_reg = extern struct {
6222 ring_addr: u64,6224 ring_addr: u64,
6223 ring_entries: u32,6225 ring_entries: u32,
6224 bgid: u16,6226 bgid: u16,
6225 pad: u16,6227 flags: u16,
6226 resv: [3]u64,6228 resv: [3]u64,
6229
6230 pub const FLAG = struct {
6231 // Incremental buffer consummation.
6232 pub const INC: u16 = 2;
6233 };
6227};6234};
62286235
6229pub const io_uring_getevents_arg = extern struct {6236pub const io_uring_getevents_arg = extern struct {
lib/std/os/linux/IoUring.zig+98-83
...@@ -1594,28 +1594,34 @@ pub const BufferGroup = struct {...@@ -1594,28 +1594,34 @@ pub const BufferGroup = struct {
1594 buffers: []u8,1594 buffers: []u8,
1595 /// Size of each buffer in buffers.1595 /// Size of each buffer in buffers.
1596 buffer_size: u32,1596 buffer_size: u32,
1597 // Number of buffers in `buffers`, number of `io_uring_buf structures` in br.1597 /// Number of buffers in `buffers`, number of `io_uring_buf structures` in br.
1598 buffers_count: u16,1598 buffers_count: u16,
1599 /// Head of unconsumed part of each buffer, if incremental consumption is enabled
1600 heads: []u32,
1599 /// ID of this group, must be unique in ring.1601 /// ID of this group, must be unique in ring.
1600 group_id: u16,1602 group_id: u16,
16011603
1602 pub fn init(1604 pub fn init(
1603 ring: *IoUring,1605 ring: *IoUring,
1606 allocator: mem.Allocator,
1604 group_id: u16,1607 group_id: u16,
1605 buffers: []u8,
1606 buffer_size: u32,1608 buffer_size: u32,
1607 buffers_count: u16,1609 buffers_count: u16,
1608 ) !BufferGroup {1610 ) !BufferGroup {
1609 assert(buffers.len == buffers_count * buffer_size);1611 const buffers = try allocator.alloc(u8, buffer_size * buffers_count);
1612 errdefer allocator.free(buffers);
1613 const heads = try allocator.alloc(u32, buffers_count);
1614 errdefer allocator.free(heads);
16101615
1611 const br = try setup_buf_ring(ring.fd, buffers_count, group_id);1616 const br = try setup_buf_ring(ring.fd, buffers_count, group_id, linux.io_uring_buf_reg.FLAG.INC);
1612 buf_ring_init(br);1617 buf_ring_init(br);
16131618
1614 const mask = buf_ring_mask(buffers_count);1619 const mask = buf_ring_mask(buffers_count);
1615 var i: u16 = 0;1620 var i: u16 = 0;
1616 while (i < buffers_count) : (i += 1) {1621 while (i < buffers_count) : (i += 1) {
1617 const start = buffer_size * i;1622 const pos = buffer_size * i;
1618 const buf = buffers[start .. start + buffer_size];1623 const buf = buffers[pos .. pos + buffer_size];
1624 heads[i] = 0;
1619 buf_ring_add(br, buf, i, mask, i);1625 buf_ring_add(br, buf, i, mask, i);
1620 }1626 }
1621 buf_ring_advance(br, buffers_count);1627 buf_ring_advance(br, buffers_count);
...@@ -1625,11 +1631,18 @@ pub const BufferGroup = struct {...@@ -1625,11 +1631,18 @@ pub const BufferGroup = struct {
1625 .group_id = group_id,1631 .group_id = group_id,
1626 .br = br,1632 .br = br,
1627 .buffers = buffers,1633 .buffers = buffers,
1634 .heads = heads,
1628 .buffer_size = buffer_size,1635 .buffer_size = buffer_size,
1629 .buffers_count = buffers_count,1636 .buffers_count = buffers_count,
1630 };1637 };
1631 }1638 }
16321639
1640 pub fn deinit(self: *BufferGroup, allocator: mem.Allocator) void {
1641 free_buf_ring(self.ring.fd, self.br, self.buffers_count, self.group_id);
1642 allocator.free(self.buffers);
1643 allocator.free(self.heads);
1644 }
1645
1633 // Prepare recv operation which will select buffer from this group.1646 // Prepare recv operation which will select buffer from this group.
1634 pub fn recv(self: *BufferGroup, user_data: u64, fd: posix.fd_t, flags: u32) !*linux.io_uring_sqe {1647 pub fn recv(self: *BufferGroup, user_data: u64, fd: posix.fd_t, flags: u32) !*linux.io_uring_sqe {
1635 var sqe = try self.ring.get_sqe();1648 var sqe = try self.ring.get_sqe();
...@@ -1649,33 +1662,34 @@ pub const BufferGroup = struct {...@@ -1649,33 +1662,34 @@ pub const BufferGroup = struct {
1649 }1662 }
16501663
1651 // Get buffer by id.1664 // Get buffer by id.
1652 pub fn get(self: *BufferGroup, buffer_id: u16) []u8 {1665 fn get_by_id(self: *BufferGroup, buffer_id: u16) []u8 {
1653 const head = self.buffer_size * buffer_id;1666 const pos = self.buffer_size * buffer_id;
1654 return self.buffers[head .. head + self.buffer_size];1667 return self.buffers[pos .. pos + self.buffer_size][self.heads[buffer_id]..];
1655 }1668 }
16561669
1657 // Get buffer by CQE.1670 // Get buffer by CQE.
1658 pub fn get_cqe(self: *BufferGroup, cqe: linux.io_uring_cqe) ![]u8 {1671 pub fn get(self: *BufferGroup, cqe: linux.io_uring_cqe) ![]u8 {
1659 const buffer_id = try cqe.buffer_id();1672 const buffer_id = try cqe.buffer_id();
1660 const used_len = @as(usize, @intCast(cqe.res));1673 const used_len = @as(usize, @intCast(cqe.res));
1661 return self.get(buffer_id)[0..used_len];1674 return self.get_by_id(buffer_id)[0..used_len];
1662 }
1663
1664 // Release buffer to the kernel.
1665 pub fn put(self: *BufferGroup, buffer_id: u16) void {
1666 const mask = buf_ring_mask(self.buffers_count);
1667 const buffer = self.get(buffer_id);
1668 buf_ring_add(self.br, buffer, buffer_id, mask, 0);
1669 buf_ring_advance(self.br, 1);
1670 }1675 }
16711676
1672 // Release buffer from CQE to the kernel.1677 // Release buffer from CQE to the kernel.
1673 pub fn put_cqe(self: *BufferGroup, cqe: linux.io_uring_cqe) !void {1678 pub fn put(self: *BufferGroup, cqe: linux.io_uring_cqe) !void {
1674 self.put(try cqe.buffer_id());1679 const buffer_id = try cqe.buffer_id();
1675 }1680 if (cqe.flags & linux.IORING_CQE_F_BUF_MORE == linux.IORING_CQE_F_BUF_MORE) {
1681 // Incremental consumption active, kernel will write to the this buffer again
1682 const used_len = @as(u32, @intCast(cqe.res));
1683 // Track what part of the buffer is used
1684 self.heads[buffer_id] += used_len;
1685 return;
1686 }
1687 self.heads[buffer_id] = 0;
16761688
1677 pub fn deinit(self: *BufferGroup) void {1689 // Release buffer to the kernel. const mask = buf_ring_mask(self.buffers_count);
1678 free_buf_ring(self.ring.fd, self.br, self.buffers_count, self.group_id);1690 const mask = buf_ring_mask(self.buffers_count);
1691 buf_ring_add(self.br, self.get_by_id(buffer_id), buffer_id, mask, 0);
1692 buf_ring_advance(self.br, 1);
1679 }1693 }
1680};1694};
16811695
...@@ -1684,7 +1698,7 @@ pub const BufferGroup = struct {...@@ -1684,7 +1698,7 @@ pub const BufferGroup = struct {
1684/// `fd` is IO_Uring.fd for which the provided buffer ring is being registered.1698/// `fd` is IO_Uring.fd for which the provided buffer ring is being registered.
1685/// `entries` is the number of entries requested in the buffer ring, must be power of 2.1699/// `entries` is the number of entries requested in the buffer ring, must be power of 2.
1686/// `group_id` is the chosen buffer group ID, unique in IO_Uring.1700/// `group_id` is the chosen buffer group ID, unique in IO_Uring.
1687pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(page_size_min) linux.io_uring_buf_ring {1701pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16, flags: u16) !*align(page_size_min) linux.io_uring_buf_ring {
1688 if (entries == 0 or entries > 1 << 15) return error.EntriesNotInRange;1702 if (entries == 0 or entries > 1 << 15) return error.EntriesNotInRange;
1689 if (!std.math.isPowerOfTwo(entries)) return error.EntriesNotPowerOfTwo;1703 if (!std.math.isPowerOfTwo(entries)) return error.EntriesNotPowerOfTwo;
16901704
...@@ -1701,22 +1715,24 @@ pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(page_...@@ -1701,22 +1715,24 @@ pub fn setup_buf_ring(fd: posix.fd_t, entries: u16, group_id: u16) !*align(page_
1701 assert(mmap.len == mmap_size);1715 assert(mmap.len == mmap_size);
17021716
1703 const br: *align(page_size_min) linux.io_uring_buf_ring = @ptrCast(mmap.ptr);1717 const br: *align(page_size_min) linux.io_uring_buf_ring = @ptrCast(mmap.ptr);
1704 try register_buf_ring(fd, @intFromPtr(br), entries, group_id);1718 try register_buf_ring(fd, @intFromPtr(br), entries, group_id, flags);
1705 return br;1719 return br;
1706}1720}
17071721
1708fn register_buf_ring(fd: posix.fd_t, addr: u64, entries: u32, group_id: u16) !void {1722fn register_buf_ring(fd: posix.fd_t, addr: u64, entries: u32, group_id: u16, flags: u16) !void {
1709 var reg = mem.zeroInit(linux.io_uring_buf_reg, .{1723 var reg = mem.zeroInit(linux.io_uring_buf_reg, .{
1710 .ring_addr = addr,1724 .ring_addr = addr,
1711 .ring_entries = entries,1725 .ring_entries = entries,
1712 .bgid = group_id,1726 .bgid = group_id,
1727 .flags = flags,
1713 });1728 });
1714 const res = linux.io_uring_register(1729 var res = linux.io_uring_register(fd, .REGISTER_PBUF_RING, @as(*const anyopaque, @ptrCast(&reg)), 1);
1715 fd,1730 if (linux.E.init(res) == .INVAL and reg.flags & linux.io_uring_buf_reg.FLAG.INC > 0) {
1716 .REGISTER_PBUF_RING,1731 // Retry without incremental buffer consumption.
1717 @as(*const anyopaque, @ptrCast(&reg)),1732 // It is available since kernel 6.12. returns INVAL on older.
1718 1,1733 reg.flags &= ~linux.io_uring_buf_reg.FLAG.INC;
1719 );1734 res = linux.io_uring_register(fd, .REGISTER_PBUF_RING, @as(*const anyopaque, @ptrCast(&reg)), 1);
1735 }
1720 try handle_register_buf_ring_result(res);1736 try handle_register_buf_ring_result(res);
1721}1737}
17221738
...@@ -4041,12 +4057,10 @@ test BufferGroup {...@@ -4041,12 +4057,10 @@ test BufferGroup {
4041 const group_id: u16 = 1; // buffers group id4057 const group_id: u16 = 1; // buffers group id
4042 const buffers_count: u16 = 1; // number of buffers in buffer group4058 const buffers_count: u16 = 1; // number of buffers in buffer group
4043 const buffer_size: usize = 128; // size of each buffer in group4059 const buffer_size: usize = 128; // size of each buffer in group
4044 const buffers = try testing.allocator.alloc(u8, buffers_count * buffer_size);
4045 defer testing.allocator.free(buffers);
4046 var buf_grp = BufferGroup.init(4060 var buf_grp = BufferGroup.init(
4047 &ring,4061 &ring,
4062 testing.allocator,
4048 group_id,4063 group_id,
4049 buffers,
4050 buffer_size,4064 buffer_size,
4051 buffers_count,4065 buffers_count,
4052 ) catch |err| switch (err) {4066 ) catch |err| switch (err) {
...@@ -4054,7 +4068,7 @@ test BufferGroup {...@@ -4054,7 +4068,7 @@ test BufferGroup {
4054 error.ArgumentsInvalid => return error.SkipZigTest,4068 error.ArgumentsInvalid => return error.SkipZigTest,
4055 else => return err,4069 else => return err,
4056 };4070 };
4057 defer buf_grp.deinit();4071 defer buf_grp.deinit(testing.allocator);
40584072
4059 // Create client/server fds4073 // Create client/server fds
4060 const fds = try createSocketTestHarness(&ring);4074 const fds = try createSocketTestHarness(&ring);
...@@ -4085,14 +4099,11 @@ test BufferGroup {...@@ -4085,14 +4099,11 @@ test BufferGroup {
4085 try testing.expectEqual(posix.E.SUCCESS, cqe.err());4099 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
4086 try testing.expectEqual(data.len, @as(usize, @intCast(cqe.res))); // cqe.res holds received data len4100 try testing.expectEqual(data.len, @as(usize, @intCast(cqe.res))); // cqe.res holds received data len
40874101
4088 // Read buffer_id and used buffer len from cqe
4089 const buffer_id = try cqe.buffer_id();
4090 const len: usize = @intCast(cqe.res);
4091 // Get buffer from pool4102 // Get buffer from pool
4092 const buf = buf_grp.get(buffer_id)[0..len];4103 const buf = try buf_grp.get(cqe);
4093 try testing.expectEqualSlices(u8, &data, buf);4104 try testing.expectEqualSlices(u8, &data, buf);
4094 // Release buffer to the kernel when application is done with it4105 // Release buffer to the kernel when application is done with it
4095 buf_grp.put(buffer_id);4106 try buf_grp.put(cqe);
4096 }4107 }
4097}4108}
40984109
...@@ -4110,12 +4121,10 @@ test "ring mapped buffers recv" {...@@ -4110,12 +4121,10 @@ test "ring mapped buffers recv" {
4110 const group_id: u16 = 1; // buffers group id4121 const group_id: u16 = 1; // buffers group id
4111 const buffers_count: u16 = 2; // number of buffers in buffer group4122 const buffers_count: u16 = 2; // number of buffers in buffer group
4112 const buffer_size: usize = 4; // size of each buffer in group4123 const buffer_size: usize = 4; // size of each buffer in group
4113 const buffers = try testing.allocator.alloc(u8, buffers_count * buffer_size);
4114 defer testing.allocator.free(buffers);
4115 var buf_grp = BufferGroup.init(4124 var buf_grp = BufferGroup.init(
4116 &ring,4125 &ring,
4126 testing.allocator,
4117 group_id,4127 group_id,
4118 buffers,
4119 buffer_size,4128 buffer_size,
4120 buffers_count,4129 buffers_count,
4121 ) catch |err| switch (err) {4130 ) catch |err| switch (err) {
...@@ -4123,7 +4132,7 @@ test "ring mapped buffers recv" {...@@ -4123,7 +4132,7 @@ test "ring mapped buffers recv" {
4123 error.ArgumentsInvalid => return error.SkipZigTest,4132 error.ArgumentsInvalid => return error.SkipZigTest,
4124 else => return err,4133 else => return err,
4125 };4134 };
4126 defer buf_grp.deinit();4135 defer buf_grp.deinit(testing.allocator);
41274136
4128 // create client/server fds4137 // create client/server fds
4129 const fds = try createSocketTestHarness(&ring);4138 const fds = try createSocketTestHarness(&ring);
...@@ -4145,14 +4154,18 @@ test "ring mapped buffers recv" {...@@ -4145,14 +4154,18 @@ test "ring mapped buffers recv" {
4145 if (cqe_send.err() == .INVAL) return error.SkipZigTest;4154 if (cqe_send.err() == .INVAL) return error.SkipZigTest;
4146 try testing.expectEqual(linux.io_uring_cqe{ .user_data = user_data, .res = data.len, .flags = 0 }, cqe_send);4155 try testing.expectEqual(linux.io_uring_cqe{ .user_data = user_data, .res = data.len, .flags = 0 }, cqe_send);
4147 }4156 }
41484157 var pos: usize = 0;
4149 // server reads data into provided buffers4158
4150 // there are 2 buffers of size 4, so each read gets only chunk of data4159 // read first chunk
4151 // we read four chunks of 4, 4, 4, 3 bytes each4160 const cqe1 = try buf_grp_recv_submit_get_cqe(&ring, &buf_grp, fds.server, rnd.int(u64));
4152 var chunk: []const u8 = data[0..buffer_size]; // first chunk4161 var buf = try buf_grp.get(cqe1);
4153 const id1 = try expect_buf_grp_recv(&ring, &buf_grp, fds.server, rnd.int(u64), chunk);4162 try testing.expectEqualSlices(u8, data[pos..][0..buf.len], buf);
4154 chunk = data[buffer_size .. buffer_size * 2]; // second chunk4163 pos += buf.len;
4155 const id2 = try expect_buf_grp_recv(&ring, &buf_grp, fds.server, rnd.int(u64), chunk);4164 // second chunk
4165 const cqe2 = try buf_grp_recv_submit_get_cqe(&ring, &buf_grp, fds.server, rnd.int(u64));
4166 buf = try buf_grp.get(cqe2);
4167 try testing.expectEqualSlices(u8, data[pos..][0..buf.len], buf);
4168 pos += buf.len;
41564169
4157 // both buffers provided to the kernel are used so we get error4170 // both buffers provided to the kernel are used so we get error
4158 // 'no more buffers', until we put buffers to the kernel4171 // 'no more buffers', until we put buffers to the kernel
...@@ -4169,16 +4182,17 @@ test "ring mapped buffers recv" {...@@ -4169,16 +4182,17 @@ test "ring mapped buffers recv" {
4169 }4182 }
41704183
4171 // put buffers back to the kernel4184 // put buffers back to the kernel
4172 buf_grp.put(id1);4185 try buf_grp.put(cqe1);
4173 buf_grp.put(id2);4186 try buf_grp.put(cqe2);
41744187
4175 chunk = data[buffer_size * 2 .. buffer_size * 3]; // third chunk4188 // read remaining data
4176 const id3 = try expect_buf_grp_recv(&ring, &buf_grp, fds.server, rnd.int(u64), chunk);4189 while (pos < data.len) {
4177 buf_grp.put(id3);4190 const cqe = try buf_grp_recv_submit_get_cqe(&ring, &buf_grp, fds.server, rnd.int(u64));
41784191 buf = try buf_grp.get(cqe);
4179 chunk = data[buffer_size * 3 ..]; // last chunk4192 try testing.expectEqualSlices(u8, data[pos..][0..buf.len], buf);
4180 const id4 = try expect_buf_grp_recv(&ring, &buf_grp, fds.server, rnd.int(u64), chunk);4193 pos += buf.len;
4181 buf_grp.put(id4);4194 try buf_grp.put(cqe);
4195 }
4182 }4196 }
4183}4197}
41844198
...@@ -4196,12 +4210,10 @@ test "ring mapped buffers multishot recv" {...@@ -4196,12 +4210,10 @@ test "ring mapped buffers multishot recv" {
4196 const group_id: u16 = 1; // buffers group id4210 const group_id: u16 = 1; // buffers group id
4197 const buffers_count: u16 = 2; // number of buffers in buffer group4211 const buffers_count: u16 = 2; // number of buffers in buffer group
4198 const buffer_size: usize = 4; // size of each buffer in group4212 const buffer_size: usize = 4; // size of each buffer in group
4199 const buffers = try testing.allocator.alloc(u8, buffers_count * buffer_size);
4200 defer testing.allocator.free(buffers);
4201 var buf_grp = BufferGroup.init(4213 var buf_grp = BufferGroup.init(
4202 &ring,4214 &ring,
4215 testing.allocator,
4203 group_id,4216 group_id,
4204 buffers,
4205 buffer_size,4217 buffer_size,
4206 buffers_count,4218 buffers_count,
4207 ) catch |err| switch (err) {4219 ) catch |err| switch (err) {
...@@ -4209,7 +4221,7 @@ test "ring mapped buffers multishot recv" {...@@ -4209,7 +4221,7 @@ test "ring mapped buffers multishot recv" {
4209 error.ArgumentsInvalid => return error.SkipZigTest,4221 error.ArgumentsInvalid => return error.SkipZigTest,
4210 else => return err,4222 else => return err,
4211 };4223 };
4212 defer buf_grp.deinit();4224 defer buf_grp.deinit(testing.allocator);
42134225
4214 // create client/server fds4226 // create client/server fds
4215 const fds = try createSocketTestHarness(&ring);4227 const fds = try createSocketTestHarness(&ring);
...@@ -4222,7 +4234,7 @@ test "ring mapped buffers multishot recv" {...@@ -4222,7 +4234,7 @@ test "ring mapped buffers multishot recv" {
4222 var round: usize = 4; // repeat send/recv cycle round times4234 var round: usize = 4; // repeat send/recv cycle round times
4223 while (round > 0) : (round -= 1) {4235 while (round > 0) : (round -= 1) {
4224 // client sends data4236 // client sends data
4225 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };4237 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf };
4226 {4238 {
4227 const user_data = rnd.int(u64);4239 const user_data = rnd.int(u64);
4228 _ = try ring.send(user_data, fds.client, data[0..], 0);4240 _ = try ring.send(user_data, fds.client, data[0..], 0);
...@@ -4239,7 +4251,7 @@ test "ring mapped buffers multishot recv" {...@@ -4239,7 +4251,7 @@ test "ring mapped buffers multishot recv" {
42394251
4240 // server reads data into provided buffers4252 // server reads data into provided buffers
4241 // there are 2 buffers of size 4, so each read gets only chunk of data4253 // there are 2 buffers of size 4, so each read gets only chunk of data
4242 // we read four chunks of 4, 4, 4, 3 bytes each4254 // we read four chunks of 4, 4, 4, 4 bytes each
4243 var chunk: []const u8 = data[0..buffer_size]; // first chunk4255 var chunk: []const u8 = data[0..buffer_size]; // first chunk
4244 const cqe1 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);4256 const cqe1 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);
4245 try testing.expect(cqe1.flags & linux.IORING_CQE_F_MORE > 0);4257 try testing.expect(cqe1.flags & linux.IORING_CQE_F_MORE > 0);
...@@ -4263,8 +4275,8 @@ test "ring mapped buffers multishot recv" {...@@ -4263,8 +4275,8 @@ test "ring mapped buffers multishot recv" {
4263 }4275 }
42644276
4265 // put buffers back to the kernel4277 // put buffers back to the kernel
4266 buf_grp.put(try cqe1.buffer_id());4278 try buf_grp.put(cqe1);
4267 buf_grp.put(try cqe2.buffer_id());4279 try buf_grp.put(cqe2);
42684280
4269 // restart multishot4281 // restart multishot
4270 recv_user_data = rnd.int(u64);4282 recv_user_data = rnd.int(u64);
...@@ -4274,12 +4286,12 @@ test "ring mapped buffers multishot recv" {...@@ -4274,12 +4286,12 @@ test "ring mapped buffers multishot recv" {
4274 chunk = data[buffer_size * 2 .. buffer_size * 3]; // third chunk4286 chunk = data[buffer_size * 2 .. buffer_size * 3]; // third chunk
4275 const cqe3 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);4287 const cqe3 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);
4276 try testing.expect(cqe3.flags & linux.IORING_CQE_F_MORE > 0);4288 try testing.expect(cqe3.flags & linux.IORING_CQE_F_MORE > 0);
4277 buf_grp.put(try cqe3.buffer_id());4289 try buf_grp.put(cqe3);
42784290
4279 chunk = data[buffer_size * 3 ..]; // last chunk4291 chunk = data[buffer_size * 3 ..]; // last chunk
4280 const cqe4 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);4292 const cqe4 = try expect_buf_grp_cqe(&ring, &buf_grp, recv_user_data, chunk);
4281 try testing.expect(cqe4.flags & linux.IORING_CQE_F_MORE > 0);4293 try testing.expect(cqe4.flags & linux.IORING_CQE_F_MORE > 0);
4282 buf_grp.put(try cqe4.buffer_id());4294 try buf_grp.put(cqe4);
42834295
4284 // cancel pending multishot recv operation4296 // cancel pending multishot recv operation
4285 {4297 {
...@@ -4323,23 +4335,26 @@ test "ring mapped buffers multishot recv" {...@@ -4323,23 +4335,26 @@ test "ring mapped buffers multishot recv" {
4323 }4335 }
4324}4336}
43254337
4326// Prepare and submit recv using buffer group.4338// Prepare, submit recv and get cqe using buffer group.
4327// Test that buffer from group, pointed by cqe, matches expected.4339fn buf_grp_recv_submit_get_cqe(
4328fn expect_buf_grp_recv(
4329 ring: *IoUring,4340 ring: *IoUring,
4330 buf_grp: *BufferGroup,4341 buf_grp: *BufferGroup,
4331 fd: posix.fd_t,4342 fd: posix.fd_t,
4332 user_data: u64,4343 user_data: u64,
4333 expected: []const u8,4344) !linux.io_uring_cqe {
4334) !u16 {4345 // prepare and submit recv
4335 // prepare and submit read
4336 const sqe = try buf_grp.recv(user_data, fd, 0);4346 const sqe = try buf_grp.recv(user_data, fd, 0);
4337 try testing.expect(sqe.flags & linux.IOSQE_BUFFER_SELECT == linux.IOSQE_BUFFER_SELECT);4347 try testing.expect(sqe.flags & linux.IOSQE_BUFFER_SELECT == linux.IOSQE_BUFFER_SELECT);
4338 try testing.expect(sqe.buf_index == buf_grp.group_id);4348 try testing.expect(sqe.buf_index == buf_grp.group_id);
4339 try testing.expectEqual(@as(u32, 1), try ring.submit()); // submit4349 try testing.expectEqual(@as(u32, 1), try ring.submit()); // submit
4350 // get cqe, expect success
4351 const cqe = try ring.copy_cqe();
4352 try testing.expectEqual(user_data, cqe.user_data);
4353 try testing.expect(cqe.res >= 0); // success
4354 try testing.expectEqual(posix.E.SUCCESS, cqe.err());
4355 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER); // IORING_CQE_F_BUFFER flag is set
43404356
4341 const cqe = try expect_buf_grp_cqe(ring, buf_grp, user_data, expected);4357 return cqe;
4342 return try cqe.buffer_id();
4343}4358}
43444359
4345fn expect_buf_grp_cqe(4360fn expect_buf_grp_cqe(
...@@ -4359,7 +4374,7 @@ fn expect_buf_grp_cqe(...@@ -4359,7 +4374,7 @@ fn expect_buf_grp_cqe(
4359 // get buffer from pool4374 // get buffer from pool
4360 const buffer_id = try cqe.buffer_id();4375 const buffer_id = try cqe.buffer_id();
4361 const len = @as(usize, @intCast(cqe.res));4376 const len = @as(usize, @intCast(cqe.res));
4362 const buf = buf_grp.get(buffer_id)[0..len];4377 const buf = buf_grp.get_by_id(buffer_id)[0..len];
4363 try testing.expectEqualSlices(u8, expected, buf);4378 try testing.expectEqualSlices(u8, expected, buf);
43644379
4365 return cqe;4380 return cqe;