authorgravatar for kbutcher6200@gmail.comkprotty <kbutcher6200@gmail.com> 2019-11-23 16:24:01-06:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-11-26 20:40:27-05:00
logca2d566ec85bee81396f64325844cc760b3cf870
tree1c0d0f009b49a881cf6811c8e69e6e56b9e79547
parenta0955990dc2c8df42879e33b308ca177ba2c771a
signaturelock-open Commit is signed but in an unrecognized format.

replace ThreadParker with ResetEvent + WordLock mutex

3 files changed, 86 insertions(+), 228 deletions(-)

lib/std/mutex.zig+85-47
......@@ -1,13 +1,12 @@
11const std = @import("std.zig");
22const builtin = @import("builtin");
33const testing = std.testing;
4const SpinLock = std.SpinLock;
5const ThreadParker = std.ThreadParker;
4const ResetEvent = std.ResetEvent;
65
76/// Lock may be held only once. If the same thread
87/// tries to acquire the same mutex twice, it deadlocks.
9/// This type supports static initialization and is based off of Golang 1.13 runtime.lock_futex:
10/// https://github.com/golang/go/blob/master/src/runtime/lock_futex.go
8/// This type supports static initialization and is based off of Webkit's WTF Lock (via rust parking_lot)
9/// https://github.com/Amanieu/parking_lot/blob/master/core/src/word_lock.rs
1110/// When an application is built in single threaded release mode, all the functions are
1211/// no-ops. In single threaded debug mode, there is deadlock detection.
1312pub const Mutex = if (builtin.single_threaded)
......@@ -39,80 +38,119 @@ pub const Mutex = if (builtin.single_threaded)
3938 }
4039else
4140 struct {
42 state: State, // TODO: make this an enum
43 parker: ThreadParker,
41 state: usize,
4442
45 const State = enum(u32) {
46 Unlocked,
47 Sleeping,
48 Locked,
49 };
43 const MUTEX_LOCK: usize = 1 << 0;
44 const QUEUE_LOCK: usize = 1 << 1;
45 const QUEUE_MASK: usize = ~(MUTEX_LOCK | QUEUE_LOCK);
46 const QueueNode = std.atomic.Stack(ResetEvent).Node;
5047
5148 /// number of iterations to spin yielding the cpu
5249 const SPIN_CPU = 4;
5350
54 /// number of iterations to perform in the cpu yield loop
51 /// number of iterations to spin in the cpu yield loop
5552 const SPIN_CPU_COUNT = 30;
5653
5754 /// number of iterations to spin yielding the thread
5855 const SPIN_THREAD = 1;
5956
6057 pub fn init() Mutex {
61 return Mutex{
62 .state = .Unlocked,
63 .parker = ThreadParker.init(),
64 };
58 return Mutex{ .state = 0 };
6559 }
6660
6761 pub fn deinit(self: *Mutex) void {
68 self.parker.deinit();
62 self.* = undefined;
6963 }
7064
7165 pub const Held = struct {
7266 mutex: *Mutex,
7367
7468 pub fn release(self: Held) void {
75 switch (@atomicRmw(State, &self.mutex.state, .Xchg, .Unlocked, .Release)) {
76 .Locked => {},
77 .Sleeping => self.mutex.parker.unpark(@ptrCast(*const u32, &self.mutex.state)),
78 .Unlocked => unreachable, // unlocking an unlocked mutex
79 else => unreachable, // should never be anything else
69 // since MUTEX_LOCK is the first bit, we can use (.Sub) instead of (.And, ~MUTEX_LOCK).
70 // this is because .Sub may be implemented more efficiently than the latter
71 // (e.g. `lock xadd` vs `cmpxchg` loop on x86)
72 const state = @atomicRmw(usize, &self.mutex.state, .Sub, MUTEX_LOCK, .Release);
73 if ((state & QUEUE_MASK) != 0 and (state & QUEUE_LOCK) == 0) {
74 self.mutex.releaseSlow(state);
8075 }
8176 }
8277 };
8378
8479 pub fn acquire(self: *Mutex) Held {
85 // Try and speculatively grab the lock.
86 // If it fails, the state is either Locked or Sleeping
87 // depending on if theres a thread stuck sleeping below.
88 var state = @atomicRmw(State, &self.state, .Xchg, .Locked, .Acquire);
89 if (state == .Unlocked)
90 return Held{ .mutex = self };
80 // fast path close to SpinLock fast path
81 if (@cmpxchgWeak(usize, &self.state, 0, MUTEX_LOCK, .Acquire, .Monotonic)) |current_state| {
82 self.acquireSlow(current_state);
83 }
84 return Held{ .mutex = self };
85 }
9186
87 fn acquireSlow(self: *Mutex, current_state: usize) void {
88 var spin: usize = 0;
89 var state = current_state;
9290 while (true) {
93 // try and acquire the lock using cpu spinning on failure
94 var spin: usize = 0;
95 while (spin < SPIN_CPU) : (spin += 1) {
96 var value = @atomicLoad(State, &self.state, .Monotonic);
97 while (value == .Unlocked)
98 value = @cmpxchgWeak(State, &self.state, .Unlocked, state, .Acquire, .Monotonic) orelse return Held{ .mutex = self };
99 SpinLock.yield(SPIN_CPU_COUNT);
91
92 // try and acquire the lock if unlocked
93 if ((state & MUTEX_LOCK) == 0) {
94 state = @cmpxchgWeak(usize, &self.state, state, state | MUTEX_LOCK, .Acquire, .Monotonic) orelse return;
95 continue;
96 }
97
98 // spin only if the waiting queue isn't empty and when it hasn't spun too much already
99 if ((state & QUEUE_MASK) == 0 and spin < SPIN_CPU + SPIN_THREAD) {
100 if (spin < SPIN_CPU) {
101 std.SpinLock.yield(SPIN_CPU_COUNT);
102 } else {
103 std.os.sched_yield() catch std.time.sleep(0);
104 }
105 state = @atomicLoad(usize, &self.state, .Monotonic);
106 continue;
100107 }
101108
102 // try and acquire the lock using thread rescheduling on failure
103 spin = 0;
104 while (spin < SPIN_THREAD) : (spin += 1) {
105 var value = @atomicLoad(State, &self.state, .Monotonic);
106 while (value == .Unlocked)
107 value = @cmpxchgWeak(State, &self.state, .Unlocked, state, .Acquire, .Monotonic) orelse return Held{ .mutex = self };
108 std.os.sched_yield() catch std.time.sleep(1);
109 // thread should block, try and add this event to the waiting queue
110 var node = QueueNode{
111 .next = @intToPtr(?*QueueNode, state & QUEUE_MASK),
112 .data = ResetEvent.init(),
113 };
114 defer node.data.deinit();
115 const new_state = @ptrToInt(&node) | (state & ~QUEUE_MASK);
116 state = @cmpxchgWeak(usize, &self.state, state, new_state, .Release, .Monotonic) orelse {
117 // node is in the queue, wait until a `held.release()` wakes us up.
118 _ = node.data.wait(null) catch unreachable;
119 spin = 0;
120 state = @atomicLoad(usize, &self.state, .Monotonic);
121 continue;
122 };
123 }
124 }
125
126 fn releaseSlow(self: *Mutex, current_state: usize) void {
127 // grab the QUEUE_LOCK in order to signal a waiting queue node's event.
128 var state = current_state;
129 while (true) {
130 if ((state & QUEUE_LOCK) != 0 or (state & QUEUE_MASK) == 0)
131 return;
132 state = @cmpxchgWeak(usize, &self.state, state, state | QUEUE_LOCK, .Acquire, .Monotonic) orelse break;
133 }
134
135 while (true) {
136 // barrier needed to observe incoming state changes
137 defer @fence(.Acquire);
138
139 // the mutex is currently locked. try to unset the QUEUE_LOCK and let the locker wake up the next node.
140 // avoids waking up multiple sleeping threads which try to acquire the lock again which increases contention.
141 if ((state & MUTEX_LOCK) != 0) {
142 state = @cmpxchgWeak(usize, &self.state, state, state & ~QUEUE_LOCK, .Release, .Monotonic) orelse return;
143 continue;
109144 }
110145
111 // failed to acquire the lock, go to sleep until woken up by `Held.release()`
112 if (@atomicRmw(State, &self.state, .Xchg, .Sleeping, .Acquire) == .Unlocked)
113 return Held{ .mutex = self };
114 state = .Sleeping;
115 self.parker.park(@ptrCast(*const u32, &self.state), @enumToInt(State.Sleeping));
146 // try to pop the top node on the waiting queue stack to wake it up
147 // while at the same time unsetting the QUEUE_LOCK.
148 const node = @intToPtr(*QueueNode, state & QUEUE_MASK);
149 const new_state = @ptrToInt(node.next) | (state & MUTEX_LOCK);
150 state = @cmpxchgWeak(usize, &self.state, state, new_state, .Release, .Monotonic) orelse {
151 _ = node.data.set(false);
152 return;
153 };
116154 }
117155 }
118156 };
lib/std/parker.zig deleted-180
......@@ -1,180 +0,0 @@
1const std = @import("std.zig");
2const builtin = @import("builtin");
3const time = std.time;
4const testing = std.testing;
5const assert = std.debug.assert;
6const SpinLock = std.SpinLock;
7const linux = std.os.linux;
8const windows = std.os.windows;
9
10pub const ThreadParker = switch (builtin.os) {
11 .linux => if (builtin.link_libc) PosixParker else LinuxParker,
12 .windows => WindowsParker,
13 else => if (builtin.link_libc) PosixParker else SpinParker,
14};
15
16const SpinParker = struct {
17 pub fn init() SpinParker {
18 return SpinParker{};
19 }
20 pub fn deinit(self: *SpinParker) void {}
21
22 pub fn unpark(self: *SpinParker, ptr: *const u32) void {}
23
24 pub fn park(self: *SpinParker, ptr: *const u32, expected: u32) void {
25 var backoff = SpinLock.Backoff.init();
26 while (@atomicLoad(u32, ptr, .Acquire) == expected)
27 backoff.yield();
28 }
29};
30
31const LinuxParker = struct {
32 pub fn init() LinuxParker {
33 return LinuxParker{};
34 }
35 pub fn deinit(self: *LinuxParker) void {}
36
37 pub fn unpark(self: *LinuxParker, ptr: *const u32) void {
38 const rc = linux.futex_wake(@ptrCast(*const i32, ptr), linux.FUTEX_WAKE | linux.FUTEX_PRIVATE_FLAG, 1);
39 assert(linux.getErrno(rc) == 0);
40 }
41
42 pub fn park(self: *LinuxParker, ptr: *const u32, expected: u32) void {
43 const value = @intCast(i32, expected);
44 while (@atomicLoad(u32, ptr, .Acquire) == expected) {
45 const rc = linux.futex_wait(@ptrCast(*const i32, ptr), linux.FUTEX_WAIT | linux.FUTEX_PRIVATE_FLAG, value, null);
46 switch (linux.getErrno(rc)) {
47 0, linux.EAGAIN => return,
48 linux.EINTR => continue,
49 linux.EINVAL => unreachable,
50 else => continue,
51 }
52 }
53 }
54};
55
56const WindowsParker = struct {
57 waiters: u32,
58
59 pub fn init() WindowsParker {
60 return WindowsParker{ .waiters = 0 };
61 }
62 pub fn deinit(self: *WindowsParker) void {}
63
64 pub fn unpark(self: *WindowsParker, ptr: *const u32) void {
65 const key = @ptrCast(*const c_void, ptr);
66 const handle = getEventHandle() orelse return;
67
68 var waiting = @atomicLoad(u32, &self.waiters, .Monotonic);
69 while (waiting != 0) {
70 waiting = @cmpxchgWeak(u32, &self.waiters, waiting, waiting - 1, .Acquire, .Monotonic) orelse {
71 const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null);
72 assert(rc == 0);
73 return;
74 };
75 }
76 }
77
78 pub fn park(self: *WindowsParker, ptr: *const u32, expected: u32) void {
79 var spin = SpinLock.Backoff.init();
80 const ev_handle = getEventHandle();
81 const key = @ptrCast(*const c_void, ptr);
82
83 while (@atomicLoad(u32, ptr, .Monotonic) == expected) {
84 if (ev_handle) |handle| {
85 _ = @atomicRmw(u32, &self.waiters, .Add, 1, .Release);
86 const rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, null);
87 assert(rc == 0);
88 } else {
89 spin.yield();
90 }
91 }
92 }
93
94 var event_handle = std.lazyInit(windows.HANDLE);
95
96 fn getEventHandle() ?windows.HANDLE {
97 if (event_handle.get()) |handle_ptr|
98 return handle_ptr.*;
99 defer event_handle.resolve();
100
101 const access_mask = windows.GENERIC_READ | windows.GENERIC_WRITE;
102 if (windows.ntdll.NtCreateKeyedEvent(&event_handle.data, access_mask, null, 0) != 0)
103 return null;
104 return event_handle.data;
105 }
106};
107
108const PosixParker = struct {
109 cond: c.pthread_cond_t,
110 mutex: c.pthread_mutex_t,
111
112 const c = std.c;
113
114 pub fn init() PosixParker {
115 return PosixParker{
116 .cond = c.PTHREAD_COND_INITIALIZER,
117 .mutex = c.PTHREAD_MUTEX_INITIALIZER,
118 };
119 }
120
121 pub fn deinit(self: *PosixParker) void {
122 // On dragonfly, the destroy functions return EINVAL if they were initialized statically.
123 const retm = c.pthread_mutex_destroy(&self.mutex);
124 assert(retm == 0 or retm == (if (builtin.os == .dragonfly) os.EINVAL else 0));
125 const retc = c.pthread_cond_destroy(&self.cond);
126 assert(retc == 0 or retc == (if (builtin.os == .dragonfly) os.EINVAL else 0));
127 }
128
129 pub fn unpark(self: *PosixParker, ptr: *const u32) void {
130 assert(c.pthread_mutex_lock(&self.mutex) == 0);
131 defer assert(c.pthread_mutex_unlock(&self.mutex) == 0);
132 assert(c.pthread_cond_signal(&self.cond) == 0);
133 }
134
135 pub fn park(self: *PosixParker, ptr: *const u32, expected: u32) void {
136 assert(c.pthread_mutex_lock(&self.mutex) == 0);
137 defer assert(c.pthread_mutex_unlock(&self.mutex) == 0);
138 while (@atomicLoad(u32, ptr, .Acquire) == expected)
139 assert(c.pthread_cond_wait(&self.cond, &self.mutex) == 0);
140 }
141};
142
143test "std.ThreadParker" {
144 if (builtin.single_threaded)
145 return error.SkipZigTest;
146
147 const Context = struct {
148 parker: ThreadParker,
149 data: u32,
150
151 fn receiver(self: *@This()) void {
152 self.parker.park(&self.data, 0); // receives 1
153 assert(@atomicRmw(u32, &self.data, .Xchg, 2, .SeqCst) == 1); // sends 2
154 self.parker.unpark(&self.data); // wakes up waiters on 2
155 self.parker.park(&self.data, 2); // receives 3
156 assert(@atomicRmw(u32, &self.data, .Xchg, 4, .SeqCst) == 3); // sends 4
157 self.parker.unpark(&self.data); // wakes up waiters on 4
158 }
159
160 fn sender(self: *@This()) void {
161 assert(@atomicRmw(u32, &self.data, .Xchg, 1, .SeqCst) == 0); // sends 1
162 self.parker.unpark(&self.data); // wakes up waiters on 1
163 self.parker.park(&self.data, 1); // receives 2
164 assert(@atomicRmw(u32, &self.data, .Xchg, 3, .SeqCst) == 2); // sends 3
165 self.parker.unpark(&self.data); // wakes up waiters on 3
166 self.parker.park(&self.data, 3); // receives 4
167 }
168 };
169
170 var context = Context{
171 .parker = ThreadParker.init(),
172 .data = 0,
173 };
174 defer context.parker.deinit();
175
176 var receiver = try std.Thread.spawn(&context, Context.receiver);
177 defer receiver.wait();
178
179 context.sender();
180}
lib/std/std.zig+1-1
......@@ -16,6 +16,7 @@ pub const PackedIntSlice = @import("packed_int_array.zig").PackedIntSlice;
1616pub const PackedIntSliceEndian = @import("packed_int_array.zig").PackedIntSliceEndian;
1717pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue;
1818pub const Progress = @import("progress.zig").Progress;
19pub const ResetEvent = @import("reset_event.zig").ResetEvent;
1920pub const SegmentedList = @import("segmented_list.zig").SegmentedList;
2021pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList;
2122pub const SpinLock = @import("spinlock.zig").SpinLock;
......@@ -23,7 +24,6 @@ pub const StringHashMap = @import("hash_map.zig").StringHashMap;
2324pub const TailQueue = @import("linked_list.zig").TailQueue;
2425pub const Target = @import("target.zig").Target;
2526pub const Thread = @import("thread.zig").Thread;
26pub const ThreadParker = @import("parker.zig").ThreadParker;
2727
2828pub const atomic = @import("atomic.zig");
2929pub const base64 = @import("base64.zig");