authorgravatar for kbutcher6200@gmail.comkprotty <kbutcher6200@gmail.com> 2019-12-15 19:40:51-06:00
committergravatar for kbutcher6200@gmail.comkprotty <kbutcher6200@gmail.com> 2019-12-17 15:38:00-06:00
logac5ba27c2bd4058dcf0d72fb4b5ad5de443316b4
tree9020af518668f1afc0ffe0036925248a93f85779
parente67ce444e760f6ddf22cf1b8c8cd418bd511ee0b

Mutex: fix lock/spin bugs, improve perf slightly & more specialization

1 files changed, 182 insertions(+), 84 deletions(-)

lib/std/mutex.zig+182-84
...@@ -1,12 +1,13 @@...@@ -1,12 +1,13 @@
1const std = @import("std.zig");1const std = @import("std.zig");
2const builtin = @import("builtin");2const builtin = @import("builtin");
3const os = std.os;
3const testing = std.testing;4const testing = std.testing;
5const SpinLock = std.SpinLock;
4const ResetEvent = std.ResetEvent;6const ResetEvent = std.ResetEvent;
57
6/// Lock may be held only once. If the same thread8/// Lock may be held only once. If the same thread
7/// tries to acquire the same mutex twice, it deadlocks.9/// tries to acquire the same mutex twice, it deadlocks.
8/// This type supports static initialization and is based off of Webkit's WTF Lock (via rust parking_lot)10/// This type supports static initialization and is at most `@sizeOf(usize)` in size.
9/// https://github.com/Amanieu/parking_lot/blob/master/core/src/word_lock.rs
10/// When an application is built in single threaded release mode, all the functions are11/// When an application is built in single threaded release mode, all the functions are
11/// no-ops. In single threaded debug mode, there is deadlock detection.12/// no-ops. In single threaded debug mode, there is deadlock detection.
12pub const Mutex = if (builtin.single_threaded)13pub const Mutex = if (builtin.single_threaded)
...@@ -24,35 +25,114 @@ pub const Mutex = if (builtin.single_threaded)...@@ -24,35 +25,114 @@ pub const Mutex = if (builtin.single_threaded)
24 }25 }
25 }26 }
26 };27 };
28
27 pub fn init() Mutex {29 pub fn init() Mutex {
28 return Mutex{ .lock = lock_init };30 return Mutex{ .lock = lock_init };
29 }31 }
30 pub fn deinit(self: *Mutex) void {}
3132
32 pub fn acquire(self: *Mutex) Held {33 pub fn deinit(self: *Mutex) void {
33 if (std.debug.runtime_safety and self.lock) {34 self.* = undefined;
34 @panic("deadlock detected");35 }
36
37 pub fn tryAcquire(self: *Mutex) ?Held {
38 if (std.debug.runtime_safety) {
39 if (self.lock) return null;
40 self.lock = true;
35 }41 }
36 return Held{ .mutex = self };42 return Held{ .mutex = self };
37 }43 }
44
45 pub fn acquire(self: *Mutex) Held {
46 return self.tryAcquire() orelse @panic("deadlock detected");
47 }
38 }48 }
39else49else if (builtin.os == .windows)
50 // https://locklessinc.com/articles/keyed_events/
51 extern union {
52 locked: u8,
53 waiters: u32,
54
55 const WAKE = 1 << 8;
56 const WAIT = 1 << 9;
57
58 pub fn init() Mutex {
59 return Mutex{ .waiters = 0 };
60 }
61
62 pub fn deinit(self: *Mutex) void {
63 self.* = undefined;
64 }
65
66 pub fn tryAcquire(self: *Mutex) ?Held {
67 if (@atomicRmw(u8, &self.locked, .Xchg, 1, .Acquire) != 0)
68 return null;
69 return Held{ .mutex = self };
70 }
71
72 pub fn acquire(self: *Mutex) Held {
73 return self.tryAcquire() orelse self.acquireSlow();
74 }
75
76 fn acquireSlow(self: *Mutex) Held {
77 @setCold(true);
78 while (true) : (SpinLock.yield(1)) {
79 const waiters = @atomicLoad(u32, &self.waiters, .Monotonic);
80
81 // try and take lock if unlocked
82 if ((waiters & 1) == 0) {
83 if (@atomicRmw(u8, &self.locked, .Xchg, 1, .Acquire) == 0)
84 return Held{ .mutex = self };
85
86 // otherwise, try and update the waiting count.
87 // then unset the WAKE bit so that another unlocker can wake up a thread.
88 } else if (@cmpxchgWeak(u32, &self.waiters, waiters, (waiters + WAIT) | 1, .Monotonic, .Monotonic) == null) {
89 ResetEvent.OsEvent.Futex.wait(@ptrCast(*i32, &self.waiters), undefined, null) catch unreachable;
90 _ = @atomicRmw(u32, &self.waiters, .Sub, WAKE, .Monotonic);
91 }
92 }
93 }
94
95 pub const Held = struct {
96 mutex: *Mutex,
97
98 pub fn release(self: Held) void {
99 // unlock without a rmw/cmpxchg instruction
100 @atomicStore(u8, @ptrCast(*u8, &self.mutex.locked), 0, .Release);
101
102 while (true) : (SpinLock.yield(1)) {
103 const waiters = @atomicLoad(u32, &self.mutex.waiters, .Monotonic);
104
105 // no one is waiting
106 if (waiters < WAIT) return;
107 // someone grabbed the lock and will do the wake instead
108 if (waiters & 1 != 0) return;
109 // someone else is currently waking up
110 if (waiters & WAKE != 0) return;
111
112 // try to decrease the waiter count & set the WAKE bit meaning a thread is waking up
113 if (@cmpxchgWeak(u32, &self.mutex.waiters, waiters, waiters - WAIT + WAKE, .Release, .Monotonic) == null)
114 return ResetEvent.OsEvent.Futex.wake(@ptrCast(*i32, &self.mutex.waiters));
115 }
116 }
117 };
118 }
119else if (builtin.link_libc or builtin.os == .linux)
120 // stack-based version of https://github.com/Amanieu/parking_lot/blob/master/core/src/word_lock.rs
40 struct {121 struct {
41 state: usize,122 state: usize,
42123
124 /// number of times to spin trying to acquire the lock.
125 /// https://webkit.org/blog/6161/locking-in-webkit/
126 const SPIN_COUNT = 40;
127
43 const MUTEX_LOCK: usize = 1 << 0;128 const MUTEX_LOCK: usize = 1 << 0;
44 const QUEUE_LOCK: usize = 1 << 1;129 const QUEUE_LOCK: usize = 1 << 1;
45 const QUEUE_MASK: usize = ~(MUTEX_LOCK | QUEUE_LOCK);130 const QUEUE_MASK: usize = ~(MUTEX_LOCK | QUEUE_LOCK);
46 const QueueNode = std.atomic.Stack(ResetEvent).Node;
47131
48 /// number of iterations to spin yielding the cpu132 const Node = struct {
49 const SPIN_CPU = 4;133 next: ?*Node,
50134 event: ResetEvent,
51 /// number of iterations to spin in the cpu yield loop135 };
52 const SPIN_CPU_COUNT = 30;
53
54 /// number of iterations to spin yielding the thread
55 const SPIN_THREAD = 1;
56136
57 pub fn init() Mutex {137 pub fn init() Mutex {
58 return Mutex{ .state = 0 };138 return Mutex{ .state = 0 };
...@@ -62,98 +142,116 @@ else...@@ -62,98 +142,116 @@ else
62 self.* = undefined;142 self.* = undefined;
63 }143 }
64144
65 pub const Held = struct {145 fn yield() void {
66 mutex: *Mutex,146 os.sched_yield() catch SpinLock.yield(30);
147 }
67148
68 pub fn release(self: Held) void {149 pub fn tryAcquire(self: *Mutex) ?Held {
69 // since MUTEX_LOCK is the first bit, we can use (.Sub) instead of (.And, ~MUTEX_LOCK).150 if (@cmpxchgWeak(usize, &self.state, 0, MUTEX_LOCK, .Acquire, .Monotonic) != null)
70 // this is because .Sub may be implemented more efficiently than the latter151 return null;
71 // (e.g. `lock xadd` vs `cmpxchg` loop on x86)152 return Held{ .mutex = self };
72 const state = @atomicRmw(usize, &self.mutex.state, .Sub, MUTEX_LOCK, .Release);153 }
73 if ((state & QUEUE_MASK) != 0 and (state & QUEUE_LOCK) == 0) {
74 self.mutex.releaseSlow(state);
75 }
76 }
77 };
78154
79 pub fn acquire(self: *Mutex) Held {155 pub fn acquire(self: *Mutex) Held {
80 // fast path close to SpinLock fast path156 return self.tryAcquire() orelse {
81 if (@cmpxchgWeak(usize, &self.state, 0, MUTEX_LOCK, .Acquire, .Monotonic)) |current_state| {157 self.acquireSlow();
82 self.acquireSlow(current_state);158 return Held{ .mutex = self };
83 }159 };
84 return Held{ .mutex = self };
85 }160 }
86161
87 fn acquireSlow(self: *Mutex, current_state: usize) void {162 fn acquireSlow(self: *Mutex) void {
88 var spin: usize = 0;163 // inlining the fast path and hiding *Slow()
89 var state = current_state;164 // calls behind a @setCold(true) appears to
165 // improve performance in release builds.
166 @setCold(true);
90 while (true) {167 while (true) {
91168
92 // try and acquire the lock if unlocked169 // try and spin for a bit to acquire the mutex if theres currently no queue
93 if ((state & MUTEX_LOCK) == 0) {170 var spin_count: u32 = SPIN_COUNT;
94 state = @cmpxchgWeak(usize, &self.state, state, state | MUTEX_LOCK, .Acquire, .Monotonic) orelse return;171 var state = @atomicLoad(usize, &self.state, .Monotonic);
95 continue;172 while (spin_count != 0) : (spin_count -= 1) {
173 if (state & MUTEX_LOCK == 0) {
174 _ = @cmpxchgWeak(usize, &self.state, state, state | MUTEX_LOCK, .Acquire, .Monotonic) orelse return;
175 } else if (state & QUEUE_MASK == 0) {
176 break;
177 }
178 yield();
179 state = @atomicLoad(usize, &self.state, .Monotonic);
96 }180 }
97181
98 // spin only if the waiting queue isn't empty and when it hasn't spun too much already182 // create the ResetEvent node on the stack
99 if ((state & QUEUE_MASK) == 0 and spin < SPIN_CPU + SPIN_THREAD) {183 // (faster than threadlocal on platforms like OSX)
100 if (spin < SPIN_CPU) {184 var node: Node = undefined;
101 std.SpinLock.yield(SPIN_CPU_COUNT);185 node.event = ResetEvent.init();
186 defer node.event.deinit();
187
188 // we've spun too long, try and add our node to the LIFO queue.
189 // if the mutex becomes available in the process, try and grab it instead.
190 while (true) {
191 if (state & MUTEX_LOCK == 0) {
192 _ = @cmpxchgWeak(usize, &self.state, state, state | MUTEX_LOCK, .Acquire, .Monotonic) orelse return;
102 } else {193 } else {
103 std.os.sched_yield() catch std.time.sleep(0);194 node.next = @intToPtr(?*Node, state & QUEUE_MASK);
195 const new_state = @ptrToInt(&node) | (state & ~QUEUE_MASK);
196 _ = @cmpxchgWeak(usize, &self.state, state, new_state, .Release, .Monotonic) orelse {
197 node.event.wait();
198 break;
199 };
104 }200 }
201 yield();
105 state = @atomicLoad(usize, &self.state, .Monotonic);202 state = @atomicLoad(usize, &self.state, .Monotonic);
106 continue;
107 }203 }
108
109 // thread should block, try and add this event to the waiting queue
110 var node = QueueNode{
111 .next = @intToPtr(?*QueueNode, state & QUEUE_MASK),
112 .data = ResetEvent.init(),
113 };
114 defer node.data.deinit();
115 const new_state = @ptrToInt(&node) | (state & ~QUEUE_MASK);
116 state = @cmpxchgWeak(usize, &self.state, state, new_state, .Release, .Monotonic) orelse {
117 // node is in the queue, wait until a `held.release()` wakes us up.
118 _ = node.data.wait(null) catch unreachable;
119 spin = 0;
120 state = @atomicLoad(usize, &self.state, .Monotonic);
121 continue;
122 };
123 }204 }
124 }205 }
125206
126 fn releaseSlow(self: *Mutex, current_state: usize) void {207 pub const Held = struct {
127 // grab the QUEUE_LOCK in order to signal a waiting queue node's event.208 mutex: *Mutex,
128 var state = current_state;209
129 while (true) {210 pub fn release(self: Held) void {
130 if ((state & QUEUE_LOCK) != 0 or (state & QUEUE_MASK) == 0)211 // first, remove the lock bit so another possibly parallel acquire() can succeed.
212 // use .Sub since it can be usually compiled down more efficiency
213 // (`lock sub` on x86) vs .And ~MUTEX_LOCK (`lock cmpxchg` loop on x86)
214 const state = @atomicRmw(usize, &self.mutex.state, .Sub, MUTEX_LOCK, .Release);
215
216 // if the LIFO queue isnt locked and it has a node, try and wake up the node.
217 if ((state & QUEUE_LOCK) == 0 and (state & QUEUE_MASK) != 0)
218 self.mutex.releaseSlow();
219 }
220 };
221
222 fn releaseSlow(self: *Mutex) void {
223 @setCold(true);
224
225 // try and lock the LFIO queue to pop a node off,
226 // stopping altogether if its already locked or the queue is empty
227 var state = @atomicLoad(usize, &self.state, .Monotonic);
228 while (true) : (std.SpinLock.yield(1)) {
229 if (state & QUEUE_LOCK != 0 or state & QUEUE_MASK == 0)
131 return;230 return;
132 state = @cmpxchgWeak(usize, &self.state, state, state | QUEUE_LOCK, .Acquire, .Monotonic) orelse break;231 state = @cmpxchgWeak(usize, &self.state, state, state | QUEUE_LOCK, .Acquire, .Monotonic) orelse break;
133 }232 }
134233
135 while (true) {234 // acquired the QUEUE_LOCK, try and pop a node to wake it.
136 // barrier needed to observe incoming state changes235 // if the mutex is locked, then unset QUEUE_LOCK and let
137 defer @fence(.Acquire);236 // the thread who holds the mutex do the wake-up on unlock()
138237 while (true) : (std.SpinLock.yield(1)) {
139 // the mutex is currently locked. try to unset the QUEUE_LOCK and let the locker wake up the next node.
140 // avoids waking up multiple sleeping threads which try to acquire the lock again which increases contention.
141 if ((state & MUTEX_LOCK) != 0) {238 if ((state & MUTEX_LOCK) != 0) {
142 state = @cmpxchgWeak(usize, &self.state, state, state & ~QUEUE_LOCK, .Release, .Monotonic) orelse return;239 state = @cmpxchgWeak(usize, &self.state, state, state & ~QUEUE_LOCK, .Release, .Acquire) orelse return;
143 continue;240 } else {
241 const node = @intToPtr(*Node, state & QUEUE_MASK);
242 const new_state = @ptrToInt(node.next);
243 state = @cmpxchgWeak(usize, &self.state, state, new_state, .Release, .Acquire) orelse {
244 node.event.set();
245 return;
246 };
144 }247 }
145
146 // try to pop the top node on the waiting queue stack to wake it up
147 // while at the same time unsetting the QUEUE_LOCK.
148 const node = @intToPtr(*QueueNode, state & QUEUE_MASK);
149 const new_state = @ptrToInt(node.next) | (state & MUTEX_LOCK);
150 state = @cmpxchgWeak(usize, &self.state, state, new_state, .Release, .Monotonic) orelse {
151 _ = node.data.set(false);
152 return;
153 };
154 }248 }
155 }249 }
156 };250 }
251
252// for platforms without a known OS blocking
253// primitive, default to SpinLock for correctness
254else SpinLock;
157255
158const TestContext = struct {256const TestContext = struct {
159 mutex: *Mutex,257 mutex: *Mutex,