| ... | ... | @@ -85,13 +85,13 @@ pub const IO_Uring = struct { |
| 85 | 85 | /// Matches the interface of io_uring_queue_init_params() in liburing. |
| 86 | 86 | pub fn init_params(entries: u32, p: *io_uring_params) !IO_Uring { |
| 87 | 87 | assert(entries >= 1 and entries <= 4096 and std.math.isPowerOfTwo(entries)); |
| 88 | | assert(p.*.sq_entries == 0); |
| 89 | | assert(p.*.cq_entries == 0); |
| 90 | | assert(p.*.features == 0); |
| 91 | | assert(p.*.wq_fd == 0); |
| 92 | | assert(p.*.resv[0] == 0); |
| 93 | | assert(p.*.resv[1] == 0); |
| 94 | | assert(p.*.resv[2] == 0); |
| 88 | assert(p.sq_entries == 0); |
| 89 | assert(p.cq_entries == 0); |
| 90 | assert(p.features == 0); |
| 91 | assert(p.wq_fd == 0); |
| 92 | assert(p.resv[0] == 0); |
| 93 | assert(p.resv[1] == 0); |
| 94 | assert(p.resv[2] == 0); |
| 95 | 95 | if (!supported) return error.IO_UringKernelNotSupported; |
| 96 | 96 | |
| 97 | 97 | const res = linux.io_uring_setup(entries, p); |
| ... | ... | @@ -109,16 +109,16 @@ pub const IO_Uring = struct { |
| 109 | 109 | // See https://patchwork.kernel.org/patch/11115257 for the kernel patch. |
| 110 | 110 | // We do not support the double mmap() done before 5.4, because we want to keep the |
| 111 | 111 | // init/deinit mmap paths simple and because io_uring has had many bug fixes even since 5.4. |
| 112 | | if ((p.*.features & linux.IORING_FEAT_SINGLE_MMAP) == 0) { |
| 112 | if ((p.features & linux.IORING_FEAT_SINGLE_MMAP) == 0) { |
| 113 | 113 | return error.IO_UringKernelNotSupported; |
| 114 | 114 | } |
| 115 | 115 | |
| 116 | 116 | // Check that the kernel has actually set params and that "impossible is nothing". |
| 117 | | assert(p.*.sq_entries != 0); |
| 118 | | assert(p.*.cq_entries != 0); |
| 119 | | assert(p.*.cq_entries >= p.*.sq_entries); |
| 117 | assert(p.sq_entries != 0); |
| 118 | assert(p.cq_entries != 0); |
| 119 | assert(p.cq_entries >= p.sq_entries); |
| 120 | 120 | |
| 121 | | // From here on, we only need to read from params, so pass `p` by value for convenience. |
| 121 | // From here on, we only need to read from params, so pass `p` by value as immutable. |
| 122 | 122 | // The completion queue shares the mmap with the submission queue, so pass `sq` there too. |
| 123 | 123 | var sq = try SubmissionQueue.init(fd, p.*); |
| 124 | 124 | errdefer sq.deinit(); |
| ... | ... | @@ -128,25 +128,25 @@ pub const IO_Uring = struct { |
| 128 | 128 | // Check that our starting state is as we expect. |
| 129 | 129 | assert(sq.head.* == 0); |
| 130 | 130 | assert(sq.tail.* == 0); |
| 131 | | assert(sq.mask.* == p.*.sq_entries - 1); |
| 131 | assert(sq.mask.* == p.sq_entries - 1); |
| 132 | 132 | // Allow flags.* to be non-zero, since the kernel may set IORING_SQ_NEED_WAKEUP at any time. |
| 133 | 133 | assert(sq.dropped.* == 0); |
| 134 | | assert(sq.array.len == p.*.sq_entries); |
| 135 | | assert(sq.sqes.len == p.*.sq_entries); |
| 134 | assert(sq.array.len == p.sq_entries); |
| 135 | assert(sq.sqes.len == p.sq_entries); |
| 136 | 136 | assert(sq.sqe_head == 0); |
| 137 | 137 | assert(sq.sqe_tail == 0); |
| 138 | 138 | |
| 139 | 139 | assert(cq.head.* == 0); |
| 140 | 140 | assert(cq.tail.* == 0); |
| 141 | | assert(cq.mask.* == p.*.cq_entries - 1); |
| 141 | assert(cq.mask.* == p.cq_entries - 1); |
| 142 | 142 | assert(cq.overflow.* == 0); |
| 143 | | assert(cq.cqes.len == p.*.cq_entries); |
| 143 | assert(cq.cqes.len == p.cq_entries); |
| 144 | 144 | |
| 145 | 145 | return IO_Uring { |
| 146 | 146 | .fd = fd, |
| 147 | 147 | .sq = sq, |
| 148 | 148 | .cq = cq, |
| 149 | | .flags = p.*.flags |
| 149 | .flags = p.flags |
| 150 | 150 | }; |
| 151 | 151 | } |
| 152 | 152 | |
| ... | ... | @@ -491,7 +491,7 @@ pub const IO_Uring = struct { |
| 491 | 491 | /// A chain will be broken if any SQE in the chain ends in error, where any unexpected result is |
| 492 | 492 | /// considered an error. For example, a short read will terminate the remainder of the chain. |
| 493 | 493 | pub fn link_with_next_sqe(self: *IO_Uring, sqe: *io_uring_sqe) void { |
| 494 | | sqe.*.flags |= linux.IOSQE_IO_LINK; |
| 494 | sqe.flags |= linux.IOSQE_IO_LINK; |
| 495 | 495 | } |
| 496 | 496 | |
| 497 | 497 | /// Like `link_with_next_sqe()` but stronger. |
| ... | ... | @@ -499,7 +499,7 @@ pub const IO_Uring = struct { |
| 499 | 499 | /// For example, you may know that some commands will fail and may want the chain to continue. |
| 500 | 500 | /// Hard links are resilient to completion results, but are not resilient to submission errors. |
| 501 | 501 | pub fn hardlink_with_next_sqe(self: *IO_Uring, sqe: *io_uring_sqe) void { |
| 502 | | sqe.*.flags |= linux.IOSQE_IO_HARDLINK; |
| 502 | sqe.flags |= linux.IOSQE_IO_HARDLINK; |
| 503 | 503 | } |
| 504 | 504 | |
| 505 | 505 | /// This creates a full pipeline barrier in the submission queue. |
| ... | ... | @@ -508,7 +508,7 @@ pub const IO_Uring = struct { |
| 508 | 508 | /// In other words, this stalls the entire submission queue. |
| 509 | 509 | /// You should first consider using link_with_next_sqe() for more granular SQE sequence control. |
| 510 | 510 | pub fn drain_previous_sqes(self: *IO_Uring, sqe: *io_uring_sqe) void { |
| 511 | | sqe.*.flags |= linux.IOSQE_IO_DRAIN; |
| 511 | sqe.flags |= linux.IOSQE_IO_DRAIN; |
| 512 | 512 | } |
| 513 | 513 | |
| 514 | 514 | /// Registers an array of file descriptors. |
| ... | ... | @@ -534,7 +534,7 @@ pub const IO_Uring = struct { |
| 534 | 534 | |
| 535 | 535 | /// Changes the semantics of the SQE's `fd` to refer to a pre-registered file descriptor. |
| 536 | 536 | pub fn use_registered_fd(self: *IO_Uring, sqe: *io_uring_sqe) void { |
| 537 | | sqe.*.flags |= linux.IOSQE_FIXED_FILE; |
| 537 | sqe.flags |= linux.IOSQE_FIXED_FILE; |
| 538 | 538 | } |
| 539 | 539 | |
| 540 | 540 | /// Unregisters all registered file descriptors previously associated with the ring. |
| ... | ... | @@ -719,7 +719,7 @@ test "queue_nop" { |
| 719 | 719 | |
| 720 | 720 | var sqe_barrier = try ring.queue_nop(@intCast(u64, 0xbbbbbbbb)); |
| 721 | 721 | ring.drain_previous_sqes(sqe_barrier); |
| 722 | | testing.expectEqual(@as(u8, linux.IOSQE_IO_DRAIN), sqe_barrier.*.flags); |
| 722 | testing.expectEqual(@as(u8, linux.IOSQE_IO_DRAIN), sqe_barrier.flags); |
| 723 | 723 | testing.expectEqual(@as(u32, 1), try ring.submit()); |
| 724 | 724 | testing.expectEqual(io_uring_cqe { |
| 725 | 725 | .user_data = 0xbbbbbbbb, |
| ... | ... | @@ -750,7 +750,7 @@ test "queue_readv" { |
| 750 | 750 | var iovecs = [_]os.iovec{ os.iovec { .iov_base = &buffer, .iov_len = buffer.len } }; |
| 751 | 751 | var sqe = try ring.queue_readv(0xcccccccc, fd_index, iovecs[0..], 0); |
| 752 | 752 | ring.use_registered_fd(sqe); |
| 753 | | testing.expectEqual(@as(u8, linux.IOSQE_FIXED_FILE), sqe.*.flags); |
| 753 | testing.expectEqual(@as(u8, linux.IOSQE_FIXED_FILE), sqe.flags); |
| 754 | 754 | |
| 755 | 755 | testing.expectError(error.IO_UringSubmissionQueueFull, ring.queue_nop(0)); |
| 756 | 756 | testing.expectEqual(@as(u32, 1), try ring.submit()); |
| ... | ... | @@ -782,10 +782,10 @@ test "queue_writev/queue_fsync" { |
| 782 | 782 | }; |
| 783 | 783 | var sqe_writev = try ring.queue_writev(0xdddddddd, fd, iovecs[0..], 0); |
| 784 | 784 | ring.link_with_next_sqe(sqe_writev); |
| 785 | | testing.expectEqual(@as(u8, linux.IOSQE_IO_LINK), sqe_writev.*.flags); |
| 785 | testing.expectEqual(@as(u8, linux.IOSQE_IO_LINK), sqe_writev.flags); |
| 786 | 786 | |
| 787 | 787 | var sqe_fsync = try ring.queue_fsync(0xeeeeeeee, fd); |
| 788 | | testing.expectEqual(fd, sqe_fsync.*.fd); |
| 788 | testing.expectEqual(fd, sqe_fsync.fd); |
| 789 | 789 | |
| 790 | 790 | testing.expectEqual(@as(u32, 2), ring.sq_ready()); |
| 791 | 791 | testing.expectEqual(@as(u32, 2), try ring.submit_and_wait(2)); |