authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-12-22 23:39:13-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-12-22 23:39:13-05:00
log25e71216c4640a3d88c8f63912ea574ad6fa004c
tree172dbe007066e3f39f6455f5b0c54d935e1d933a
parentad92227516bf977a3be2802db525ac2d678acc7c
parent1c5a1284e3992555653a01903bc3131c4b628e86
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #3974 from kprotty/event_broadcast

ResetEvent fixes & broadcast notify

3 files changed, 87 insertions(+), 39 deletions(-)

lib/std/c.zig+1
...@@ -220,6 +220,7 @@ pub const PTHREAD_COND_INITIALIZER = pthread_cond_t{};...@@ -220,6 +220,7 @@ pub const PTHREAD_COND_INITIALIZER = pthread_cond_t{};
220pub extern "c" fn pthread_cond_wait(noalias cond: *pthread_cond_t, noalias mutex: *pthread_mutex_t) c_int;220pub extern "c" fn pthread_cond_wait(noalias cond: *pthread_cond_t, noalias mutex: *pthread_mutex_t) c_int;
221pub extern "c" fn pthread_cond_timedwait(noalias cond: *pthread_cond_t, noalias mutex: *pthread_mutex_t, noalias abstime: *const timespec) c_int;221pub extern "c" fn pthread_cond_timedwait(noalias cond: *pthread_cond_t, noalias mutex: *pthread_mutex_t, noalias abstime: *const timespec) c_int;
222pub extern "c" fn pthread_cond_signal(cond: *pthread_cond_t) c_int;222pub extern "c" fn pthread_cond_signal(cond: *pthread_cond_t) c_int;
223pub extern "c" fn pthread_cond_broadcast(cond: *pthread_cond_t) c_int;
223pub extern "c" fn pthread_cond_destroy(cond: *pthread_cond_t) c_int;224pub extern "c" fn pthread_cond_destroy(cond: *pthread_cond_t) c_int;
224225
225pub const pthread_t = *@OpaqueType();226pub const pthread_t = *@OpaqueType();
lib/std/mutex.zig+24-4
...@@ -1,6 +1,8 @@...@@ -1,6 +1,8 @@
1const std = @import("std.zig");1const std = @import("std.zig");
2const builtin = @import("builtin");2const builtin = @import("builtin");
3const os = std.os;3const os = std.os;
4const assert = std.debug.assert;
5const windows = os.windows;
4const testing = std.testing;6const testing = std.testing;
5const SpinLock = std.SpinLock;7const SpinLock = std.SpinLock;
6const ResetEvent = std.ResetEvent;8const ResetEvent = std.ResetEvent;
...@@ -73,20 +75,33 @@ else if (builtin.os == .windows)...@@ -73,20 +75,33 @@ else if (builtin.os == .windows)
73 return self.tryAcquire() orelse self.acquireSlow();75 return self.tryAcquire() orelse self.acquireSlow();
74 }76 }
7577
78 fn acquireSpinning(self: *Mutex) Held {
79 @setCold(true);
80 while (true) : (SpinLock.yield()) {
81 return self.tryAcquire() orelse continue;
82 }
83 }
84
76 fn acquireSlow(self: *Mutex) Held {85 fn acquireSlow(self: *Mutex) Held {
86 // try to use NT keyed events for blocking, falling back to spinlock if unavailable
77 @setCold(true);87 @setCold(true);
88 const handle = ResetEvent.OsEvent.Futex.getEventHandle() orelse return self.acquireSpinning();
89 const key = @ptrCast(*const c_void, &self.waiters);
90
78 while (true) : (SpinLock.loopHint(1)) {91 while (true) : (SpinLock.loopHint(1)) {
79 const waiters = @atomicLoad(u32, &self.waiters, .Monotonic);92 const waiters = @atomicLoad(u32, &self.waiters, .Monotonic);
8093
81 // try and take lock if unlocked94 // try and take lock if unlocked
82 if ((waiters & 1) == 0) {95 if ((waiters & 1) == 0) {
83 if (@atomicRmw(u8, &self.locked, .Xchg, 1, .Acquire) == 0)96 if (@atomicRmw(u8, &self.locked, .Xchg, 1, .Acquire) == 0) {
84 return Held{ .mutex = self };97 return Held{ .mutex = self };
98 }
8599
86 // otherwise, try and update the waiting count.100 // otherwise, try and update the waiting count.
87 // then unset the WAKE bit so that another unlocker can wake up a thread.101 // then unset the WAKE bit so that another unlocker can wake up a thread.
88 } else if (@cmpxchgWeak(u32, &self.waiters, waiters, (waiters + WAIT) | 1, .Monotonic, .Monotonic) == null) {102 } else if (@cmpxchgWeak(u32, &self.waiters, waiters, (waiters + WAIT) | 1, .Monotonic, .Monotonic) == null) {
89 ResetEvent.OsEvent.Futex.wait(@ptrCast(*i32, &self.waiters), undefined, null) catch unreachable;103 const rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, null);
104 assert(rc == 0);
90 _ = @atomicRmw(u32, &self.waiters, .Sub, WAKE, .Monotonic);105 _ = @atomicRmw(u32, &self.waiters, .Sub, WAKE, .Monotonic);
91 }106 }
92 }107 }
...@@ -98,6 +113,8 @@ else if (builtin.os == .windows)...@@ -98,6 +113,8 @@ else if (builtin.os == .windows)
98 pub fn release(self: Held) void {113 pub fn release(self: Held) void {
99 // unlock without a rmw/cmpxchg instruction114 // unlock without a rmw/cmpxchg instruction
100 @atomicStore(u8, @ptrCast(*u8, &self.mutex.locked), 0, .Release);115 @atomicStore(u8, @ptrCast(*u8, &self.mutex.locked), 0, .Release);
116 const handle = ResetEvent.OsEvent.Futex.getEventHandle() orelse return;
117 const key = @ptrCast(*const c_void, &self.mutex.waiters);
101118
102 while (true) : (SpinLock.loopHint(1)) {119 while (true) : (SpinLock.loopHint(1)) {
103 const waiters = @atomicLoad(u32, &self.mutex.waiters, .Monotonic);120 const waiters = @atomicLoad(u32, &self.mutex.waiters, .Monotonic);
...@@ -110,8 +127,11 @@ else if (builtin.os == .windows)...@@ -110,8 +127,11 @@ else if (builtin.os == .windows)
110 if (waiters & WAKE != 0) return;127 if (waiters & WAKE != 0) return;
111128
112 // try to decrease the waiter count & set the WAKE bit meaning a thread is waking up129 // try to decrease the waiter count & set the WAKE bit meaning a thread is waking up
113 if (@cmpxchgWeak(u32, &self.mutex.waiters, waiters, waiters - WAIT + WAKE, .Release, .Monotonic) == null)130 if (@cmpxchgWeak(u32, &self.mutex.waiters, waiters, waiters - WAIT + WAKE, .Release, .Monotonic) == null) {
114 return ResetEvent.OsEvent.Futex.wake(@ptrCast(*i32, &self.mutex.waiters));131 const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null);
132 assert(rc == 0);
133 return;
134 }
115 }135 }
116 }136 }
117 };137 };
lib/std/reset_event.zig+62-35
...@@ -36,7 +36,7 @@ pub const ResetEvent = struct {...@@ -36,7 +36,7 @@ pub const ResetEvent = struct {
36 }36 }
3737
38 /// Sets the event if not already set and38 /// Sets the event if not already set and
39 /// wakes up at least one thread waiting the event.39 /// wakes up all the threads waiting on the event.
40 pub fn set(self: *ResetEvent) void {40 pub fn set(self: *ResetEvent) void {
41 return self.os_event.set();41 return self.os_event.set();
42 }42 }
...@@ -135,7 +135,7 @@ const PosixEvent = struct {...@@ -135,7 +135,7 @@ const PosixEvent = struct {
135135
136 if (!self.is_set) {136 if (!self.is_set) {
137 self.is_set = true;137 self.is_set = true;
138 assert(c.pthread_cond_signal(&self.cond) == 0);138 assert(c.pthread_cond_broadcast(&self.cond) == 0);
139 }139 }
140 }140 }
141141
...@@ -181,40 +181,39 @@ const PosixEvent = struct {...@@ -181,40 +181,39 @@ const PosixEvent = struct {
181};181};
182182
183const AtomicEvent = struct {183const AtomicEvent = struct {
184 state: State,184 waiters: u32,
185185
186 const State = enum(i32) {186 const WAKE = 1 << 0;
187 Empty,187 const WAIT = 1 << 1;
188 Waiting,
189 Signaled,
190 };
191188
192 fn init() AtomicEvent {189 fn init() AtomicEvent {
193 return AtomicEvent{ .state = .Empty };190 return AtomicEvent{ .waiters = 0 };
194 }191 }
195192
196 fn deinit(self: *AtomicEvent) void {193 fn deinit(self: *AtomicEvent) void {
197 self.* = undefined;194 self.* = undefined;
198 }195 }
199196
200 fn isSet(self: *AtomicEvent) bool {197 fn isSet(self: *const AtomicEvent) bool {
201 return @atomicLoad(State, &self.state, .Acquire) == .Signaled;198 return @atomicLoad(u32, &self.waiters, .Acquire) == WAKE;
202 }199 }
203200
204 fn reset(self: *AtomicEvent) void {201 fn reset(self: *AtomicEvent) void {
205 @atomicStore(State, &self.state, .Empty, .Monotonic);202 @atomicStore(u32, &self.waiters, 0, .Monotonic);
206 }203 }
207204
208 fn set(self: *AtomicEvent) void {205 fn set(self: *AtomicEvent) void {
209 if (@atomicRmw(State, &self.state, .Xchg, .Signaled, .Release) == .Waiting)206 const waiters = @atomicRmw(u32, &self.waiters, .Xchg, WAKE, .Release);
210 Futex.wake(@ptrCast(*i32, &self.state));207 if (waiters >= WAIT) {
208 return Futex.wake(&self.waiters, waiters >> 1);
209 }
211 }210 }
212211
213 fn wait(self: *AtomicEvent, timeout: ?u64) !void {212 fn wait(self: *AtomicEvent, timeout: ?u64) !void {
214 var state = @atomicLoad(State, &self.state, .Monotonic);213 var waiters = @atomicLoad(u32, &self.waiters, .Acquire);
215 while (state == .Empty) {214 while (waiters != WAKE) {
216 state = @cmpxchgWeak(State, &self.state, .Empty, .Waiting, .Acquire, .Monotonic) orelse 215 waiters = @cmpxchgWeak(u32, &self.waiters, waiters, waiters + WAIT, .Acquire, .Acquire)
217 return Futex.wait(@ptrCast(*i32, &self.state), @enumToInt(State.Waiting), timeout);216 orelse return Futex.wait(&self.waiters, timeout);
218 }217 }
219 }218 }
220219
...@@ -225,15 +224,15 @@ const AtomicEvent = struct {...@@ -225,15 +224,15 @@ const AtomicEvent = struct {
225 };224 };
226225
227 const SpinFutex = struct {226 const SpinFutex = struct {
228 fn wake(ptr: *i32) void {}227 fn wake(waiters: *u32, wake_count: u32) void {}
229228
230 fn wait(ptr: *i32, expected: i32, timeout: ?u64) !void {229 fn wait(waiters: *u32, timeout: ?u64) !void {
231 // TODO: handle platforms where a monotonic timer isnt available230 // TODO: handle platforms where a monotonic timer isnt available
232 var timer: time.Timer = undefined;231 var timer: time.Timer = undefined;
233 if (timeout != null)232 if (timeout != null)
234 timer = time.Timer.start() catch unreachable;233 timer = time.Timer.start() catch unreachable;
235234
236 while (@atomicLoad(i32, ptr, .Acquire) == expected) {235 while (@atomicLoad(u32, waiters, .Acquire) != WAKE) {
237 SpinLock.yield();236 SpinLock.yield();
238 if (timeout) |timeout_ns| {237 if (timeout) |timeout_ns| {
239 if (timer.read() >= timeout_ns)238 if (timer.read() >= timeout_ns)
...@@ -244,12 +243,14 @@ const AtomicEvent = struct {...@@ -244,12 +243,14 @@ const AtomicEvent = struct {
244 };243 };
245244
246 const LinuxFutex = struct {245 const LinuxFutex = struct {
247 fn wake(ptr: *i32) void {246 fn wake(waiters: *u32, wake_count: u32) void {
248 const rc = linux.futex_wake(ptr, linux.FUTEX_WAKE | linux.FUTEX_PRIVATE_FLAG, 1);247 const waiting = std.math.maxInt(i32); // wake_count
248 const ptr = @ptrCast(*const i32, waiters);
249 const rc = linux.futex_wake(ptr, linux.FUTEX_WAKE | linux.FUTEX_PRIVATE_FLAG, waiting);
249 assert(linux.getErrno(rc) == 0);250 assert(linux.getErrno(rc) == 0);
250 }251 }
251252
252 fn wait(ptr: *i32, expected: i32, timeout: ?u64) !void {253 fn wait(waiters: *u32, timeout: ?u64) !void {
253 var ts: linux.timespec = undefined;254 var ts: linux.timespec = undefined;
254 var ts_ptr: ?*linux.timespec = null;255 var ts_ptr: ?*linux.timespec = null;
255 if (timeout) |timeout_ns| {256 if (timeout) |timeout_ns| {
...@@ -258,7 +259,12 @@ const AtomicEvent = struct {...@@ -258,7 +259,12 @@ const AtomicEvent = struct {
258 ts.tv_nsec = @intCast(isize, timeout_ns % time.ns_per_s);259 ts.tv_nsec = @intCast(isize, timeout_ns % time.ns_per_s);
259 }260 }
260261
261 while (@atomicLoad(i32, ptr, .Acquire) == expected) {262 while (true) {
263 const waiting = @atomicLoad(u32, waiters, .Acquire);
264 if (waiting == WAKE)
265 return;
266 const expected = @intCast(i32, waiting);
267 const ptr = @ptrCast(*const i32, waiters);
262 const rc = linux.futex_wait(ptr, linux.FUTEX_WAIT | linux.FUTEX_PRIVATE_FLAG, expected, ts_ptr);268 const rc = linux.futex_wait(ptr, linux.FUTEX_WAIT | linux.FUTEX_PRIVATE_FLAG, expected, ts_ptr);
263 switch (linux.getErrno(rc)) {269 switch (linux.getErrno(rc)) {
264 0 => continue,270 0 => continue,
...@@ -272,15 +278,20 @@ const AtomicEvent = struct {...@@ -272,15 +278,20 @@ const AtomicEvent = struct {
272 };278 };
273279
274 const WindowsFutex = struct {280 const WindowsFutex = struct {
275 pub fn wake(ptr: *i32) void {281 pub fn wake(waiters: *u32, wake_count: u32) void {
276 const handle = getEventHandle() orelse return SpinFutex.wake(ptr);282 const handle = getEventHandle() orelse return SpinFutex.wake(waiters, wake_count);
277 const key = @ptrCast(*const c_void, ptr);283 const key = @ptrCast(*const c_void, waiters);
278 const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null);284
279 assert(rc == 0);285 var waiting = wake_count;
286 while (waiting != 0) : (waiting -= 1) {
287 const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null);
288 assert(rc == 0);
289 }
280 }290 }
281291
282 pub fn wait(ptr: *i32, expected: i32, timeout: ?u64) !void {292 pub fn wait(waiters: *u32, timeout: ?u64) !void {
283 const handle = getEventHandle() orelse return SpinFutex.wait(ptr, expected, timeout);293 const handle = getEventHandle() orelse return SpinFutex.wait(waiters, timeout);
294 const key = @ptrCast(*const c_void, waiters);
284295
285 // NT uses timeouts in units of 100ns with negative value being relative296 // NT uses timeouts in units of 100ns with negative value being relative
286 var timeout_ptr: ?*windows.LARGE_INTEGER = null;297 var timeout_ptr: ?*windows.LARGE_INTEGER = null;
...@@ -291,10 +302,26 @@ const AtomicEvent = struct {...@@ -291,10 +302,26 @@ const AtomicEvent = struct {
291 }302 }
292303
293 // NtWaitForKeyedEvent doesnt have spurious wake-ups304 // NtWaitForKeyedEvent doesnt have spurious wake-ups
294 const key = @ptrCast(*const c_void, ptr);305 var rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, timeout_ptr);
295 const rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, timeout_ptr);
296 switch (rc) {306 switch (rc) {
297 windows.WAIT_TIMEOUT => return error.TimedOut,307 windows.WAIT_TIMEOUT => {
308 // update the wait count to signal that we're not waiting anymore.
309 // if the .set() thread already observed that we are, perform a
310 // matching NtWaitForKeyedEvent so that the .set() thread doesn't
311 // deadlock trying to run NtReleaseKeyedEvent above.
312 var waiting = @atomicLoad(u32, waiters, .Monotonic);
313 while (true) {
314 if (waiting == WAKE) {
315 rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, null);
316 assert(rc == windows.WAIT_OBJECT_0);
317 break;
318 } else {
319 waiting = @cmpxchgWeak(u32, waiters, waiting, waiting - WAIT, .Acquire, .Monotonic) orelse break;
320 continue;
321 }
322 }
323 return error.TimedOut;
324 },
298 windows.WAIT_OBJECT_0 => {},325 windows.WAIT_OBJECT_0 => {},
299 else => unreachable,326 else => unreachable,
300 }327 }