authorgravatar for joran@ronomon.comJoran Dirk Greef <joran@ronomon.com> 2020-09-21 11:07:00+02:00
committergravatar for joran@ronomon.comJoran Dirk Greef <joran@ronomon.com> 2020-09-21 11:07:11+02:00
log575ed941d7b53e4643ab803e5500c96e0f4698b3
tree024803d8c675b7c6f952dda2cf61aab0ae6626e2
parent843c104fc9cb544367fda2168c6ad45a625cb979

Cache mask instead of dereferencing mask pointer

1 files changed, 10 insertions(+), 12 deletions(-)

lib/std/os/linux/io_uring.zig+10-12
...@@ -108,7 +108,7 @@ pub const IO_Uring = struct {...@@ -108,7 +108,7 @@ pub const IO_Uring = struct {
108 // Check that our starting state is as we expect.108 // Check that our starting state is as we expect.
109 assert(sq.head.* == 0);109 assert(sq.head.* == 0);
110 assert(sq.tail.* == 0);110 assert(sq.tail.* == 0);
111 assert(sq.mask.* == p.sq_entries - 1);111 assert(sq.mask == p.sq_entries - 1);
112 // Allow flags.* to be non-zero, since the kernel may set IORING_SQ_NEED_WAKEUP at any time.112 // Allow flags.* to be non-zero, since the kernel may set IORING_SQ_NEED_WAKEUP at any time.
113 assert(sq.dropped.* == 0);113 assert(sq.dropped.* == 0);
114 assert(sq.array.len == p.sq_entries);114 assert(sq.array.len == p.sq_entries);
...@@ -118,7 +118,7 @@ pub const IO_Uring = struct {...@@ -118,7 +118,7 @@ pub const IO_Uring = struct {
118118
119 assert(cq.head.* == 0);119 assert(cq.head.* == 0);
120 assert(cq.tail.* == 0);120 assert(cq.tail.* == 0);
121 assert(cq.mask.* == p.cq_entries - 1);121 assert(cq.mask == p.cq_entries - 1);
122 assert(cq.overflow.* == 0);122 assert(cq.overflow.* == 0);
123 assert(cq.cqes.len == p.cq_entries);123 assert(cq.cqes.len == p.cq_entries);
124124
...@@ -152,7 +152,7 @@ pub const IO_Uring = struct {...@@ -152,7 +152,7 @@ pub const IO_Uring = struct {
152 // We must therefore use wrapping addition and subtraction to avoid a runtime crash.152 // We must therefore use wrapping addition and subtraction to avoid a runtime crash.
153 const next = self.sq.sqe_tail +% 1;153 const next = self.sq.sqe_tail +% 1;
154 if (next -% head > self.sq.sqes.len) return error.SubmissionQueueFull;154 if (next -% head > self.sq.sqes.len) return error.SubmissionQueueFull;
155 var sqe = &self.sq.sqes[self.sq.sqe_tail & self.sq.mask.*];155 var sqe = &self.sq.sqes[self.sq.sqe_tail & self.sq.mask];
156 self.sq.sqe_tail = next;156 self.sq.sqe_tail = next;
157 // We zero the SQE slot here in a single place, rather than in many `queue_` methods.157 // We zero the SQE slot here in a single place, rather than in many `queue_` methods.
158 @memset(@ptrCast([*]u8, sqe), 0, @sizeOf(io_uring_sqe));158 @memset(@ptrCast([*]u8, sqe), 0, @sizeOf(io_uring_sqe));
...@@ -224,11 +224,10 @@ pub const IO_Uring = struct {...@@ -224,11 +224,10 @@ pub const IO_Uring = struct {
224 if (self.sq.sqe_head != self.sq.sqe_tail) {224 if (self.sq.sqe_head != self.sq.sqe_tail) {
225 // Fill in SQEs that we have queued up, adding them to the kernel ring.225 // Fill in SQEs that we have queued up, adding them to the kernel ring.
226 const to_submit = self.sq.sqe_tail -% self.sq.sqe_head;226 const to_submit = self.sq.sqe_tail -% self.sq.sqe_head;
227 const mask = self.sq.mask.*;
228 var tail = self.sq.tail.*;227 var tail = self.sq.tail.*;
229 var i: usize = 0;228 var i: usize = 0;
230 while (i < to_submit) : (i += 1) {229 while (i < to_submit) : (i += 1) {
231 self.sq.array[tail & mask] = self.sq.sqe_head & mask;230 self.sq.array[tail & self.sq.mask] = self.sq.sqe_head & self.sq.mask;
232 tail +%= 1;231 tail +%= 1;
233 self.sq.sqe_head +%= 1;232 self.sq.sqe_head +%= 1;
234 }233 }
...@@ -292,14 +291,13 @@ pub const IO_Uring = struct {...@@ -292,14 +291,13 @@ pub const IO_Uring = struct {
292 fn copy_cqes_ready(self: *IO_Uring, cqes: []io_uring_cqe, wait_nr: u32) u32 {291 fn copy_cqes_ready(self: *IO_Uring, cqes: []io_uring_cqe, wait_nr: u32) u32 {
293 const ready = self.cq_ready();292 const ready = self.cq_ready();
294 const count = std.math.min(cqes.len, ready);293 const count = std.math.min(cqes.len, ready);
295 const mask = self.cq.mask.*;
296 var head = self.cq.head.*;294 var head = self.cq.head.*;
297 var tail = head +% count;295 var tail = head +% count;
298 // TODO Optimize this by using 1 or 2 memcpy's (if the tail wraps) rather than a loop.296 // TODO Optimize this by using 1 or 2 memcpy's (if the tail wraps) rather than a loop.
299 var i: usize = 0;297 var i: usize = 0;
300 // Do not use "less-than" operator since head and tail may wrap:298 // Do not use "less-than" operator since head and tail may wrap:
301 while (head != tail) {299 while (head != tail) {
302 cqes[i] = self.cq.cqes[head & mask]; // Copy struct by value.300 cqes[i] = self.cq.cqes[head & self.cq.mask]; // Copy struct by value.
303 head +%= 1;301 head +%= 1;
304 i += 1;302 i += 1;
305 }303 }
...@@ -560,7 +558,7 @@ pub const IO_Uring = struct {...@@ -560,7 +558,7 @@ pub const IO_Uring = struct {
560pub const SubmissionQueue = struct {558pub const SubmissionQueue = struct {
561 head: *u32,559 head: *u32,
562 tail: *u32,560 tail: *u32,
563 mask: *u32,561 mask: u32,
564 flags: *u32,562 flags: *u32,
565 dropped: *u32,563 dropped: *u32,
566 array: []u32,564 array: []u32,
...@@ -618,7 +616,7 @@ pub const SubmissionQueue = struct {...@@ -618,7 +616,7 @@ pub const SubmissionQueue = struct {
618 return SubmissionQueue {616 return SubmissionQueue {
619 .head = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.head])),617 .head = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.head])),
620 .tail = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.tail])),618 .tail = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.tail])),
621 .mask = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.ring_mask])),619 .mask = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.ring_mask])).*,
622 .flags = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.flags])),620 .flags = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.flags])),
623 .dropped = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.dropped])),621 .dropped = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.sq_off.dropped])),
624 .array = array[0..p.sq_entries],622 .array = array[0..p.sq_entries],
...@@ -637,7 +635,7 @@ pub const SubmissionQueue = struct {...@@ -637,7 +635,7 @@ pub const SubmissionQueue = struct {
637pub const CompletionQueue = struct {635pub const CompletionQueue = struct {
638 head: *u32,636 head: *u32,
639 tail: *u32,637 tail: *u32,
640 mask: *u32,638 mask: u32,
641 overflow: *u32,639 overflow: *u32,
642 cqes: []io_uring_cqe,640 cqes: []io_uring_cqe,
643641
...@@ -656,7 +654,7 @@ pub const CompletionQueue = struct {...@@ -656,7 +654,7 @@ pub const CompletionQueue = struct {
656 return CompletionQueue {654 return CompletionQueue {
657 .head = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.head])),655 .head = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.head])),
658 .tail = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.tail])),656 .tail = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.tail])),
659 .mask = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.ring_mask])),657 .mask = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.ring_mask])).*,
660 .overflow = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.overflow])),658 .overflow = @ptrCast(*u32, @alignCast(@alignOf(u32), &mmap[p.cq_off.overflow])),
661 .cqes = cqes[0..p.cq_entries]659 .cqes = cqes[0..p.cq_entries]
662 };660 };
...@@ -670,7 +668,7 @@ pub const CompletionQueue = struct {...@@ -670,7 +668,7 @@ pub const CompletionQueue = struct {
670668
671test "structs and offsets" {669test "structs and offsets" {
672 if (builtin.os.tag != .linux) return error.SkipZigTest;670 if (builtin.os.tag != .linux) return error.SkipZigTest;
673 671
674 testing.expectEqual(@as(usize, 120), @sizeOf(io_uring_params));672 testing.expectEqual(@as(usize, 120), @sizeOf(io_uring_params));
675 testing.expectEqual(@as(usize, 64), @sizeOf(io_uring_sqe));673 testing.expectEqual(@as(usize, 64), @sizeOf(io_uring_sqe));
676 testing.expectEqual(@as(usize, 16), @sizeOf(io_uring_cqe));674 testing.expectEqual(@as(usize, 16), @sizeOf(io_uring_cqe));