authorgravatar for joran@ronomon.comJoran Dirk Greef <joran@ronomon.com> 2020-09-21 11:07:00+02:00
committergravatar for joran@ronomon.comJoran Dirk Greef <joran@ronomon.com> 2020-09-21 11:07:11+02:00
log575ed941d7b53e4643ab803e5500c96e0f4698b3
tree024803d8c675b7c6f952dda2cf61aab0ae6626e2
parent843c104fc9cb544367fda2168c6ad45a625cb979

Cache mask instead of dereferencing mask pointer

1 files changed, 10 insertions(+), 12 deletions(-)

lib/std/os/linux/io_uring.zig+10-12
......@@ -108,7 +108,7 @@ pub const IO_Uring = struct {
108108 // Check that our starting state is as we expect.
109109 assert(sq.head.* == 0);
110110 assert(sq.tail.* == 0);
111 assert(sq.mask.* == p.sq_entries - 1);
111 assert(sq.mask == p.sq_entries - 1);
112112 // Allow flags.* to be non-zero, since the kernel may set IORING_SQ_NEED_WAKEUP at any time.
113113 assert(sq.dropped.* == 0);
114114 assert(sq.array.len == p.sq_entries);
......@@ -118,7 +118,7 @@ pub const IO_Uring = struct {
118118
119119 assert(cq.head.* == 0);
120120 assert(cq.tail.* == 0);
121 assert(cq.mask.* == p.cq_entries - 1);
121 assert(cq.mask == p.cq_entries - 1);
122122 assert(cq.overflow.* == 0);
123123 assert(cq.cqes.len == p.cq_entries);
124124
......@@ -152,7 +152,7 @@ pub const IO_Uring = struct {
152152 // We must therefore use wrapping addition and subtraction to avoid a runtime crash.
153153 const next = self.sq.sqe_tail +% 1;
154154 if (next -% head > self.sq.sqes.len) return error.SubmissionQueueFull;
155 var sqe = &self.sq.sqes[self.sq.sqe_tail & self.sq.mask.*];
155 var sqe = &self.sq.sqes[self.sq.sqe_tail & self.sq.mask];
156156 self.sq.sqe_tail = next;
157157 // We zero the SQE slot here in a single place, rather than in many `queue_` methods.
158158 @memset(@ptrCast([*]u8, sqe), 0, @sizeOf(io_uring_sqe));
......@@ -224,11 +224,10 @@ pub const IO_Uring = struct {
224224 if (self.sq.sqe_head != self.sq.sqe_tail) {
225225 // Fill in SQEs that we have queued up, adding them to the kernel ring.
226226 const to_submit = self.sq.sqe_tail -% self.sq.sqe_head;
227 const mask = self.sq.mask.*;
228227 var tail = self.sq.tail.*;
229228 var i: usize = 0;
230229 while (i < to_submit) : (i += 1) {
231 self.sq.array[tail & mask] = self.sq.sqe_head & mask;
230 self.sq.array[tail & self.sq.mask] = self.sq.sqe_head & self.sq.mask;
232231 tail +%= 1;
233232 self.sq.sqe_head +%= 1;
234233 }
......@@ -292,14 +291,13 @@ pub const IO_Uring = struct {
292291 fn copy_cqes_ready(self: *IO_Uring, cqes: []io_uring_cqe, wait_nr: u32) u32 {
293292 const ready = self.cq_ready();
294293 const count = std.math.min(cqes.len, ready);
295 const mask = self.cq.mask.*;
296294 var head = self.cq.head.*;
297295 var tail = head +% count;
298296 // TODO Optimize this by using 1 or 2 memcpy's (if the tail wraps) rather than a loop.
299297 var i: usize = 0;
300298 // Do not use "less-than" operator since head and tail may wrap:
301299 while (head != tail) {
302 cqes[i] = self.cq.cqes[head & mask]; // Copy struct by value.
300 cqes[i] = self.cq.cqes[head & self.cq.mask]; // Copy struct by value.
303301 head +%= 1;
304302 i += 1;
305303 }
......@@ -560,7 +558,7 @@ pub const IO_Uring = struct {
560558pub const SubmissionQueue = struct {
561559 head: *u32,
562560 tail: *u32,
563 mask: *u32,
561 mask: u32,
564562 flags: *u32,
565563 dropped: *u32,
566564 array: []u32,
......@@ -618,7 +616,7 @@ pub const SubmissionQueue = struct {
618616 return SubmissionQueue {
619617 .head = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.head])),
620618 .tail = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.tail])),
621 .mask = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.ring_mask])),
619 .mask = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.ring_mask])).*,
622620 .flags = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.flags])),
623621 .dropped = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.dropped])),
624622 .array = array[0..p.sq_entries],
......@@ -637,7 +635,7 @@ pub const SubmissionQueue = struct {
637635pub const CompletionQueue = struct {
638636 head: *u32,
639637 tail: *u32,
640 mask: *u32,
638 mask: u32,
641639 overflow: *u32,
642640 cqes: []io_uring_cqe,
643641
......@@ -656,7 +654,7 @@ pub const CompletionQueue = struct {
656654 return CompletionQueue {
657655 .head = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.head])),
658656 .tail = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.tail])),
659 .mask = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.ring_mask])),
657 .mask = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.ring_mask])).*,
660658 .overflow = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.overflow])),
661659 .cqes = cqes[0..p.cq_entries]
662660 };
......@@ -670,7 +668,7 @@ pub const CompletionQueue = struct {
670668
671669test "structs and offsets" {
672670 if (builtin.os.tag != .linux) return error.SkipZigTest;
673
671
674672 testing.expectEqual(@as(usize, 120), @sizeOf(io_uring_params));
675673 testing.expectEqual(@as(usize, 64), @sizeOf(io_uring_sqe));
676674 testing.expectEqual(@as(usize, 16), @sizeOf(io_uring_cqe));