authorgravatar for kbutcher6200@gmail.comkprotty <kbutcher6200@gmail.com> 2019-12-15 19:40:51-06:00
committergravatar for kbutcher6200@gmail.comkprotty <kbutcher6200@gmail.com> 2019-12-17 15:38:00-06:00
logac5ba27c2bd4058dcf0d72fb4b5ad5de443316b4
tree9020af518668f1afc0ffe0036925248a93f85779
parente67ce444e760f6ddf22cf1b8c8cd418bd511ee0b

Mutex: fix lock/spin bugs, improve perf slightly & more specialization

1 files changed, 182 insertions(+), 84 deletions(-)

lib/std/mutex.zig+182-84
......@@ -1,12 +1,13 @@
11const std = @import("std.zig");
22const builtin = @import("builtin");
3const os = std.os;
34const testing = std.testing;
5const SpinLock = std.SpinLock;
46const ResetEvent = std.ResetEvent;
57
68/// Lock may be held only once. If the same thread
79/// tries to acquire the same mutex twice, it deadlocks.
8/// This type supports static initialization and is based off of Webkit's WTF Lock (via rust parking_lot)
9/// https://github.com/Amanieu/parking_lot/blob/master/core/src/word_lock.rs
10/// This type supports static initialization and is at most `@sizeOf(usize)` in size.
1011/// When an application is built in single threaded release mode, all the functions are
1112/// no-ops. In single threaded debug mode, there is deadlock detection.
1213pub const Mutex = if (builtin.single_threaded)
......@@ -24,35 +25,114 @@ pub const Mutex = if (builtin.single_threaded)
2425 }
2526 }
2627 };
28
2729 pub fn init() Mutex {
2830 return Mutex{ .lock = lock_init };
2931 }
30 pub fn deinit(self: *Mutex) void {}
3132
32 pub fn acquire(self: *Mutex) Held {
33 if (std.debug.runtime_safety and self.lock) {
34 @panic("deadlock detected");
33 pub fn deinit(self: *Mutex) void {
34 self.* = undefined;
35 }
36
37 pub fn tryAcquire(self: *Mutex) ?Held {
38 if (std.debug.runtime_safety) {
39 if (self.lock) return null;
40 self.lock = true;
3541 }
3642 return Held{ .mutex = self };
3743 }
44
45 pub fn acquire(self: *Mutex) Held {
46 return self.tryAcquire() orelse @panic("deadlock detected");
47 }
3848 }
39else
49else if (builtin.os == .windows)
50 // https://locklessinc.com/articles/keyed_events/
51 extern union {
52 locked: u8,
53 waiters: u32,
54
55 const WAKE = 1 << 8;
56 const WAIT = 1 << 9;
57
58 pub fn init() Mutex {
59 return Mutex{ .waiters = 0 };
60 }
61
62 pub fn deinit(self: *Mutex) void {
63 self.* = undefined;
64 }
65
66 pub fn tryAcquire(self: *Mutex) ?Held {
67 if (@atomicRmw(u8, &self.locked, .Xchg, 1, .Acquire) != 0)
68 return null;
69 return Held{ .mutex = self };
70 }
71
72 pub fn acquire(self: *Mutex) Held {
73 return self.tryAcquire() orelse self.acquireSlow();
74 }
75
76 fn acquireSlow(self: *Mutex) Held {
77 @setCold(true);
78 while (true) : (SpinLock.yield(1)) {
79 const waiters = @atomicLoad(u32, &self.waiters, .Monotonic);
80
81 // try and take lock if unlocked
82 if ((waiters & 1) == 0) {
83 if (@atomicRmw(u8, &self.locked, .Xchg, 1, .Acquire) == 0)
84 return Held{ .mutex = self };
85
86 // otherwise, try and update the waiting count.
87 // then unset the WAKE bit so that another unlocker can wake up a thread.
88 } else if (@cmpxchgWeak(u32, &self.waiters, waiters, (waiters + WAIT) | 1, .Monotonic, .Monotonic) == null) {
89 ResetEvent.OsEvent.Futex.wait(@ptrCast(*i32, &self.waiters), undefined, null) catch unreachable;
90 _ = @atomicRmw(u32, &self.waiters, .Sub, WAKE, .Monotonic);
91 }
92 }
93 }
94
95 pub const Held = struct {
96 mutex: *Mutex,
97
98 pub fn release(self: Held) void {
99 // unlock without a rmw/cmpxchg instruction
100 @atomicStore(u8, @ptrCast(*u8, &self.mutex.locked), 0, .Release);
101
102 while (true) : (SpinLock.yield(1)) {
103 const waiters = @atomicLoad(u32, &self.mutex.waiters, .Monotonic);
104
105 // no one is waiting
106 if (waiters < WAIT) return;
107 // someone grabbed the lock and will do the wake instead
108 if (waiters & 1 != 0) return;
109 // someone else is currently waking up
110 if (waiters & WAKE != 0) return;
111
112 // try to decrease the waiter count & set the WAKE bit meaning a thread is waking up
113 if (@cmpxchgWeak(u32, &self.mutex.waiters, waiters, waiters - WAIT + WAKE, .Release, .Monotonic) == null)
114 return ResetEvent.OsEvent.Futex.wake(@ptrCast(*i32, &self.mutex.waiters));
115 }
116 }
117 };
118 }
119else if (builtin.link_libc or builtin.os == .linux)
120 // stack-based version of https://github.com/Amanieu/parking_lot/blob/master/core/src/word_lock.rs
40121 struct {
41122 state: usize,
42123
124 /// number of times to spin trying to acquire the lock.
125 /// https://webkit.org/blog/6161/locking-in-webkit/
126 const SPIN_COUNT = 40;
127
43128 const MUTEX_LOCK: usize = 1 << 0;
44129 const QUEUE_LOCK: usize = 1 << 1;
45130 const QUEUE_MASK: usize = ~(MUTEX_LOCK | QUEUE_LOCK);
46 const QueueNode = std.atomic.Stack(ResetEvent).Node;
47131
48 /// number of iterations to spin yielding the cpu
49 const SPIN_CPU = 4;
50
51 /// number of iterations to spin in the cpu yield loop
52 const SPIN_CPU_COUNT = 30;
53
54 /// number of iterations to spin yielding the thread
55 const SPIN_THREAD = 1;
132 const Node = struct {
133 next: ?*Node,
134 event: ResetEvent,
135 };
56136
57137 pub fn init() Mutex {
58138 return Mutex{ .state = 0 };
......@@ -62,98 +142,116 @@ else
62142 self.* = undefined;
63143 }
64144
65 pub const Held = struct {
66 mutex: *Mutex,
145 fn yield() void {
146 os.sched_yield() catch SpinLock.yield(30);
147 }
67148
68 pub fn release(self: Held) void {
69 // since MUTEX_LOCK is the first bit, we can use (.Sub) instead of (.And, ~MUTEX_LOCK).
70 // this is because .Sub may be implemented more efficiently than the latter
71 // (e.g. `lock xadd` vs `cmpxchg` loop on x86)
72 const state = @atomicRmw(usize, &self.mutex.state, .Sub, MUTEX_LOCK, .Release);
73 if ((state & QUEUE_MASK) != 0 and (state & QUEUE_LOCK) == 0) {
74 self.mutex.releaseSlow(state);
75 }
76 }
77 };
149 pub fn tryAcquire(self: *Mutex) ?Held {
150 if (@cmpxchgWeak(usize, &self.state, 0, MUTEX_LOCK, .Acquire, .Monotonic) != null)
151 return null;
152 return Held{ .mutex = self };
153 }
78154
79155 pub fn acquire(self: *Mutex) Held {
80 // fast path close to SpinLock fast path
81 if (@cmpxchgWeak(usize, &self.state, 0, MUTEX_LOCK, .Acquire, .Monotonic)) |current_state| {
82 self.acquireSlow(current_state);
83 }
84 return Held{ .mutex = self };
156 return self.tryAcquire() orelse {
157 self.acquireSlow();
158 return Held{ .mutex = self };
159 };
85160 }
86161
87 fn acquireSlow(self: *Mutex, current_state: usize) void {
88 var spin: usize = 0;
89 var state = current_state;
162 fn acquireSlow(self: *Mutex) void {
163 // inlining the fast path and hiding *Slow()
164 // calls behind a @setCold(true) appears to
165 // improve performance in release builds.
166 @setCold(true);
90167 while (true) {
91168
92 // try and acquire the lock if unlocked
93 if ((state & MUTEX_LOCK) == 0) {
94 state = @cmpxchgWeak(usize, &self.state, state, state | MUTEX_LOCK, .Acquire, .Monotonic) orelse return;
95 continue;
169 // try and spin for a bit to acquire the mutex if theres currently no queue
170 var spin_count: u32 = SPIN_COUNT;
171 var state = @atomicLoad(usize, &self.state, .Monotonic);
172 while (spin_count != 0) : (spin_count -= 1) {
173 if (state & MUTEX_LOCK == 0) {
174 _ = @cmpxchgWeak(usize, &self.state, state, state | MUTEX_LOCK, .Acquire, .Monotonic) orelse return;
175 } else if (state & QUEUE_MASK == 0) {
176 break;
177 }
178 yield();
179 state = @atomicLoad(usize, &self.state, .Monotonic);
96180 }
97181
98 // spin only if the waiting queue isn't empty and when it hasn't spun too much already
99 if ((state & QUEUE_MASK) == 0 and spin < SPIN_CPU + SPIN_THREAD) {
100 if (spin < SPIN_CPU) {
101 std.SpinLock.yield(SPIN_CPU_COUNT);
182 // create the ResetEvent node on the stack
183 // (faster than threadlocal on platforms like OSX)
184 var node: Node = undefined;
185 node.event = ResetEvent.init();
186 defer node.event.deinit();
187
188 // we've spun too long, try and add our node to the LIFO queue.
189 // if the mutex becomes available in the process, try and grab it instead.
190 while (true) {
191 if (state & MUTEX_LOCK == 0) {
192 _ = @cmpxchgWeak(usize, &self.state, state, state | MUTEX_LOCK, .Acquire, .Monotonic) orelse return;
102193 } else {
103 std.os.sched_yield() catch std.time.sleep(0);
194 node.next = @intToPtr(?*Node, state & QUEUE_MASK);
195 const new_state = @ptrToInt(&node) | (state & ~QUEUE_MASK);
196 _ = @cmpxchgWeak(usize, &self.state, state, new_state, .Release, .Monotonic) orelse {
197 node.event.wait();
198 break;
199 };
104200 }
201 yield();
105202 state = @atomicLoad(usize, &self.state, .Monotonic);
106 continue;
107203 }
108
109 // thread should block, try and add this event to the waiting queue
110 var node = QueueNode{
111 .next = @intToPtr(?*QueueNode, state & QUEUE_MASK),
112 .data = ResetEvent.init(),
113 };
114 defer node.data.deinit();
115 const new_state = @ptrToInt(&node) | (state & ~QUEUE_MASK);
116 state = @cmpxchgWeak(usize, &self.state, state, new_state, .Release, .Monotonic) orelse {
117 // node is in the queue, wait until a `held.release()` wakes us up.
118 _ = node.data.wait(null) catch unreachable;
119 spin = 0;
120 state = @atomicLoad(usize, &self.state, .Monotonic);
121 continue;
122 };
123204 }
124205 }
125206
126 fn releaseSlow(self: *Mutex, current_state: usize) void {
127 // grab the QUEUE_LOCK in order to signal a waiting queue node's event.
128 var state = current_state;
129 while (true) {
130 if ((state & QUEUE_LOCK) != 0 or (state & QUEUE_MASK) == 0)
207 pub const Held = struct {
208 mutex: *Mutex,
209
210 pub fn release(self: Held) void {
211 // first, remove the lock bit so another possibly parallel acquire() can succeed.
212 // use .Sub since it can be usually compiled down more efficiency
213 // (`lock sub` on x86) vs .And ~MUTEX_LOCK (`lock cmpxchg` loop on x86)
214 const state = @atomicRmw(usize, &self.mutex.state, .Sub, MUTEX_LOCK, .Release);
215
216 // if the LIFO queue isnt locked and it has a node, try and wake up the node.
217 if ((state & QUEUE_LOCK) == 0 and (state & QUEUE_MASK) != 0)
218 self.mutex.releaseSlow();
219 }
220 };
221
222 fn releaseSlow(self: *Mutex) void {
223 @setCold(true);
224
225 // try and lock the LFIO queue to pop a node off,
226 // stopping altogether if its already locked or the queue is empty
227 var state = @atomicLoad(usize, &self.state, .Monotonic);
228 while (true) : (std.SpinLock.yield(1)) {
229 if (state & QUEUE_LOCK != 0 or state & QUEUE_MASK == 0)
131230 return;
132231 state = @cmpxchgWeak(usize, &self.state, state, state | QUEUE_LOCK, .Acquire, .Monotonic) orelse break;
133232 }
134233
135 while (true) {
136 // barrier needed to observe incoming state changes
137 defer @fence(.Acquire);
138
139 // the mutex is currently locked. try to unset the QUEUE_LOCK and let the locker wake up the next node.
140 // avoids waking up multiple sleeping threads which try to acquire the lock again which increases contention.
234 // acquired the QUEUE_LOCK, try and pop a node to wake it.
235 // if the mutex is locked, then unset QUEUE_LOCK and let
236 // the thread who holds the mutex do the wake-up on unlock()
237 while (true) : (std.SpinLock.yield(1)) {
141238 if ((state & MUTEX_LOCK) != 0) {
142 state = @cmpxchgWeak(usize, &self.state, state, state & ~QUEUE_LOCK, .Release, .Monotonic) orelse return;
143 continue;
239 state = @cmpxchgWeak(usize, &self.state, state, state & ~QUEUE_LOCK, .Release, .Acquire) orelse return;
240 } else {
241 const node = @intToPtr(*Node, state & QUEUE_MASK);
242 const new_state = @ptrToInt(node.next);
243 state = @cmpxchgWeak(usize, &self.state, state, new_state, .Release, .Acquire) orelse {
244 node.event.set();
245 return;
246 };
144247 }
145
146 // try to pop the top node on the waiting queue stack to wake it up
147 // while at the same time unsetting the QUEUE_LOCK.
148 const node = @intToPtr(*QueueNode, state & QUEUE_MASK);
149 const new_state = @ptrToInt(node.next) | (state & MUTEX_LOCK);
150 state = @cmpxchgWeak(usize, &self.state, state, new_state, .Release, .Monotonic) orelse {
151 _ = node.data.set(false);
152 return;
153 };
154248 }
155249 }
156 };
250 }
251
252// for platforms without a known OS blocking
253// primitive, default to SpinLock for correctness
254else SpinLock;
157255
158256const TestContext = struct {
159257 mutex: *Mutex,