authorgravatar for kbutcher6200@gmail.comkprotty <kbutcher6200@gmail.com> 2019-11-23 16:24:01-06:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-11-26 20:40:27-05:00
logca2d566ec85bee81396f64325844cc760b3cf870
tree1c0d0f009b49a881cf6811c8e69e6e56b9e79547
parenta0955990dc2c8df42879e33b308ca177ba2c771a
signaturelock-open Commit is signed but in an unrecognized format.

replace ThreadParker with ResetEvent + WordLock mutex

3 files changed, 86 insertions(+), 228 deletions(-)

lib/std/mutex.zig+85-47
...@@ -1,13 +1,12 @@...@@ -1,13 +1,12 @@
1const std = @import("std.zig");1const std = @import("std.zig");
2const builtin = @import("builtin");2const builtin = @import("builtin");
3const testing = std.testing;3const testing = std.testing;
4const SpinLock = std.SpinLock;4const ResetEvent = std.ResetEvent;
5const ThreadParker = std.ThreadParker;
65
7/// Lock may be held only once. If the same thread6/// Lock may be held only once. If the same thread
8/// tries to acquire the same mutex twice, it deadlocks.7/// tries to acquire the same mutex twice, it deadlocks.
9/// This type supports static initialization and is based off of Golang 1.13 runtime.lock_futex:8/// This type supports static initialization and is based off of Webkit's WTF Lock (via rust parking_lot)
10/// https://github.com/golang/go/blob/master/src/runtime/lock_futex.go9/// https://github.com/Amanieu/parking_lot/blob/master/core/src/word_lock.rs
11/// When an application is built in single threaded release mode, all the functions are10/// When an application is built in single threaded release mode, all the functions are
12/// no-ops. In single threaded debug mode, there is deadlock detection.11/// no-ops. In single threaded debug mode, there is deadlock detection.
13pub const Mutex = if (builtin.single_threaded)12pub const Mutex = if (builtin.single_threaded)
...@@ -39,80 +38,119 @@ pub const Mutex = if (builtin.single_threaded)...@@ -39,80 +38,119 @@ pub const Mutex = if (builtin.single_threaded)
39 }38 }
40else39else
41 struct {40 struct {
42 state: State, // TODO: make this an enum41 state: usize,
43 parker: ThreadParker,
4442
45 const State = enum(u32) {43 const MUTEX_LOCK: usize = 1 << 0;
46 Unlocked,44 const QUEUE_LOCK: usize = 1 << 1;
47 Sleeping,45 const QUEUE_MASK: usize = ~(MUTEX_LOCK | QUEUE_LOCK);
48 Locked,46 const QueueNode = std.atomic.Stack(ResetEvent).Node;
49 };
5047
51 /// number of iterations to spin yielding the cpu48 /// number of iterations to spin yielding the cpu
52 const SPIN_CPU = 4;49 const SPIN_CPU = 4;
5350
54 /// number of iterations to perform in the cpu yield loop51 /// number of iterations to spin in the cpu yield loop
55 const SPIN_CPU_COUNT = 30;52 const SPIN_CPU_COUNT = 30;
5653
57 /// number of iterations to spin yielding the thread54 /// number of iterations to spin yielding the thread
58 const SPIN_THREAD = 1;55 const SPIN_THREAD = 1;
5956
60 pub fn init() Mutex {57 pub fn init() Mutex {
61 return Mutex{58 return Mutex{ .state = 0 };
62 .state = .Unlocked,
63 .parker = ThreadParker.init(),
64 };
65 }59 }
6660
67 pub fn deinit(self: *Mutex) void {61 pub fn deinit(self: *Mutex) void {
68 self.parker.deinit();62 self.* = undefined;
69 }63 }
7064
71 pub const Held = struct {65 pub const Held = struct {
72 mutex: *Mutex,66 mutex: *Mutex,
7367
74 pub fn release(self: Held) void {68 pub fn release(self: Held) void {
75 switch (@atomicRmw(State, &self.mutex.state, .Xchg, .Unlocked, .Release)) {69 // since MUTEX_LOCK is the first bit, we can use (.Sub) instead of (.And, ~MUTEX_LOCK).
76 .Locked => {},70 // this is because .Sub may be implemented more efficiently than the latter
77 .Sleeping => self.mutex.parker.unpark(@ptrCast(*const u32, &self.mutex.state)),71 // (e.g. `lock xadd` vs `cmpxchg` loop on x86)
78 .Unlocked => unreachable, // unlocking an unlocked mutex72 const state = @atomicRmw(usize, &self.mutex.state, .Sub, MUTEX_LOCK, .Release);
79 else => unreachable, // should never be anything else73 if ((state & QUEUE_MASK) != 0 and (state & QUEUE_LOCK) == 0) {
74 self.mutex.releaseSlow(state);
80 }75 }
81 }76 }
82 };77 };
8378
84 pub fn acquire(self: *Mutex) Held {79 pub fn acquire(self: *Mutex) Held {
85 // Try and speculatively grab the lock.80 // fast path close to SpinLock fast path
86 // If it fails, the state is either Locked or Sleeping81 if (@cmpxchgWeak(usize, &self.state, 0, MUTEX_LOCK, .Acquire, .Monotonic)) |current_state| {
87 // depending on if theres a thread stuck sleeping below.82 self.acquireSlow(current_state);
88 var state = @atomicRmw(State, &self.state, .Xchg, .Locked, .Acquire);83 }
89 if (state == .Unlocked)84 return Held{ .mutex = self };
90 return Held{ .mutex = self };85 }
9186
87 fn acquireSlow(self: *Mutex, current_state: usize) void {
88 var spin: usize = 0;
89 var state = current_state;
92 while (true) {90 while (true) {
93 // try and acquire the lock using cpu spinning on failure91
94 var spin: usize = 0;92 // try and acquire the lock if unlocked
95 while (spin < SPIN_CPU) : (spin += 1) {93 if ((state & MUTEX_LOCK) == 0) {
96 var value = @atomicLoad(State, &self.state, .Monotonic);94 state = @cmpxchgWeak(usize, &self.state, state, state | MUTEX_LOCK, .Acquire, .Monotonic) orelse return;
97 while (value == .Unlocked)95 continue;
98 value = @cmpxchgWeak(State, &self.state, .Unlocked, state, .Acquire, .Monotonic) orelse return Held{ .mutex = self };96 }
99 SpinLock.yield(SPIN_CPU_COUNT);97
98 // spin only if the waiting queue isn't empty and when it hasn't spun too much already
99 if ((state & QUEUE_MASK) == 0 and spin < SPIN_CPU + SPIN_THREAD) {
100 if (spin < SPIN_CPU) {
101 std.SpinLock.yield(SPIN_CPU_COUNT);
102 } else {
103 std.os.sched_yield() catch std.time.sleep(0);
104 }
105 state = @atomicLoad(usize, &self.state, .Monotonic);
106 continue;
100 }107 }
101108
102 // try and acquire the lock using thread rescheduling on failure109 // thread should block, try and add this event to the waiting queue
103 spin = 0;110 var node = QueueNode{
104 while (spin < SPIN_THREAD) : (spin += 1) {111 .next = @intToPtr(?*QueueNode, state & QUEUE_MASK),
105 var value = @atomicLoad(State, &self.state, .Monotonic);112 .data = ResetEvent.init(),
106 while (value == .Unlocked)113 };
107 value = @cmpxchgWeak(State, &self.state, .Unlocked, state, .Acquire, .Monotonic) orelse return Held{ .mutex = self };114 defer node.data.deinit();
108 std.os.sched_yield() catch std.time.sleep(1);115 const new_state = @ptrToInt(&node) | (state & ~QUEUE_MASK);
116 state = @cmpxchgWeak(usize, &self.state, state, new_state, .Release, .Monotonic) orelse {
117 // node is in the queue, wait until a `held.release()` wakes us up.
118 _ = node.data.wait(null) catch unreachable;
119 spin = 0;
120 state = @atomicLoad(usize, &self.state, .Monotonic);
121 continue;
122 };
123 }
124 }
125
126 fn releaseSlow(self: *Mutex, current_state: usize) void {
127 // grab the QUEUE_LOCK in order to signal a waiting queue node's event.
128 var state = current_state;
129 while (true) {
130 if ((state & QUEUE_LOCK) != 0 or (state & QUEUE_MASK) == 0)
131 return;
132 state = @cmpxchgWeak(usize, &self.state, state, state | QUEUE_LOCK, .Acquire, .Monotonic) orelse break;
133 }
134
135 while (true) {
136 // barrier needed to observe incoming state changes
137 defer @fence(.Acquire);
138
139 // the mutex is currently locked. try to unset the QUEUE_LOCK and let the locker wake up the next node.
140 // avoids waking up multiple sleeping threads which try to acquire the lock again which increases contention.
141 if ((state & MUTEX_LOCK) != 0) {
142 state = @cmpxchgWeak(usize, &self.state, state, state & ~QUEUE_LOCK, .Release, .Monotonic) orelse return;
143 continue;
109 }144 }
110145
111 // failed to acquire the lock, go to sleep until woken up by `Held.release()`146 // try to pop the top node on the waiting queue stack to wake it up
112 if (@atomicRmw(State, &self.state, .Xchg, .Sleeping, .Acquire) == .Unlocked)147 // while at the same time unsetting the QUEUE_LOCK.
113 return Held{ .mutex = self };148 const node = @intToPtr(*QueueNode, state & QUEUE_MASK);
114 state = .Sleeping;149 const new_state = @ptrToInt(node.next) | (state & MUTEX_LOCK);
115 self.parker.park(@ptrCast(*const u32, &self.state), @enumToInt(State.Sleeping));150 state = @cmpxchgWeak(usize, &self.state, state, new_state, .Release, .Monotonic) orelse {
151 _ = node.data.set(false);
152 return;
153 };
116 }154 }
117 }155 }
118 };156 };
lib/std/parker.zig deleted-180
...@@ -1,180 +0,0 @@
1const std = @import("std.zig");
2const builtin = @import("builtin");
3const time = std.time;
4const testing = std.testing;
5const assert = std.debug.assert;
6const SpinLock = std.SpinLock;
7const linux = std.os.linux;
8const windows = std.os.windows;
9
10pub const ThreadParker = switch (builtin.os) {
11 .linux => if (builtin.link_libc) PosixParker else LinuxParker,
12 .windows => WindowsParker,
13 else => if (builtin.link_libc) PosixParker else SpinParker,
14};
15
16const SpinParker = struct {
17 pub fn init() SpinParker {
18 return SpinParker{};
19 }
20 pub fn deinit(self: *SpinParker) void {}
21
22 pub fn unpark(self: *SpinParker, ptr: *const u32) void {}
23
24 pub fn park(self: *SpinParker, ptr: *const u32, expected: u32) void {
25 var backoff = SpinLock.Backoff.init();
26 while (@atomicLoad(u32, ptr, .Acquire) == expected)
27 backoff.yield();
28 }
29};
30
31const LinuxParker = struct {
32 pub fn init() LinuxParker {
33 return LinuxParker{};
34 }
35 pub fn deinit(self: *LinuxParker) void {}
36
37 pub fn unpark(self: *LinuxParker, ptr: *const u32) void {
38 const rc = linux.futex_wake(@ptrCast(*const i32, ptr), linux.FUTEX_WAKE | linux.FUTEX_PRIVATE_FLAG, 1);
39 assert(linux.getErrno(rc) == 0);
40 }
41
42 pub fn park(self: *LinuxParker, ptr: *const u32, expected: u32) void {
43 const value = @intCast(i32, expected);
44 while (@atomicLoad(u32, ptr, .Acquire) == expected) {
45 const rc = linux.futex_wait(@ptrCast(*const i32, ptr), linux.FUTEX_WAIT | linux.FUTEX_PRIVATE_FLAG, value, null);
46 switch (linux.getErrno(rc)) {
47 0, linux.EAGAIN => return,
48 linux.EINTR => continue,
49 linux.EINVAL => unreachable,
50 else => continue,
51 }
52 }
53 }
54};
55
56const WindowsParker = struct {
57 waiters: u32,
58
59 pub fn init() WindowsParker {
60 return WindowsParker{ .waiters = 0 };
61 }
62 pub fn deinit(self: *WindowsParker) void {}
63
64 pub fn unpark(self: *WindowsParker, ptr: *const u32) void {
65 const key = @ptrCast(*const c_void, ptr);
66 const handle = getEventHandle() orelse return;
67
68 var waiting = @atomicLoad(u32, &self.waiters, .Monotonic);
69 while (waiting != 0) {
70 waiting = @cmpxchgWeak(u32, &self.waiters, waiting, waiting - 1, .Acquire, .Monotonic) orelse {
71 const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null);
72 assert(rc == 0);
73 return;
74 };
75 }
76 }
77
78 pub fn park(self: *WindowsParker, ptr: *const u32, expected: u32) void {
79 var spin = SpinLock.Backoff.init();
80 const ev_handle = getEventHandle();
81 const key = @ptrCast(*const c_void, ptr);
82
83 while (@atomicLoad(u32, ptr, .Monotonic) == expected) {
84 if (ev_handle) |handle| {
85 _ = @atomicRmw(u32, &self.waiters, .Add, 1, .Release);
86 const rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, null);
87 assert(rc == 0);
88 } else {
89 spin.yield();
90 }
91 }
92 }
93
94 var event_handle = std.lazyInit(windows.HANDLE);
95
96 fn getEventHandle() ?windows.HANDLE {
97 if (event_handle.get()) |handle_ptr|
98 return handle_ptr.*;
99 defer event_handle.resolve();
100
101 const access_mask = windows.GENERIC_READ | windows.GENERIC_WRITE;
102 if (windows.ntdll.NtCreateKeyedEvent(&event_handle.data, access_mask, null, 0) != 0)
103 return null;
104 return event_handle.data;
105 }
106};
107
108const PosixParker = struct {
109 cond: c.pthread_cond_t,
110 mutex: c.pthread_mutex_t,
111
112 const c = std.c;
113
114 pub fn init() PosixParker {
115 return PosixParker{
116 .cond = c.PTHREAD_COND_INITIALIZER,
117 .mutex = c.PTHREAD_MUTEX_INITIALIZER,
118 };
119 }
120
121 pub fn deinit(self: *PosixParker) void {
122 // On dragonfly, the destroy functions return EINVAL if they were initialized statically.
123 const retm = c.pthread_mutex_destroy(&self.mutex);
124 assert(retm == 0 or retm == (if (builtin.os == .dragonfly) os.EINVAL else 0));
125 const retc = c.pthread_cond_destroy(&self.cond);
126 assert(retc == 0 or retc == (if (builtin.os == .dragonfly) os.EINVAL else 0));
127 }
128
129 pub fn unpark(self: *PosixParker, ptr: *const u32) void {
130 assert(c.pthread_mutex_lock(&self.mutex) == 0);
131 defer assert(c.pthread_mutex_unlock(&self.mutex) == 0);
132 assert(c.pthread_cond_signal(&self.cond) == 0);
133 }
134
135 pub fn park(self: *PosixParker, ptr: *const u32, expected: u32) void {
136 assert(c.pthread_mutex_lock(&self.mutex) == 0);
137 defer assert(c.pthread_mutex_unlock(&self.mutex) == 0);
138 while (@atomicLoad(u32, ptr, .Acquire) == expected)
139 assert(c.pthread_cond_wait(&self.cond, &self.mutex) == 0);
140 }
141};
142
143test "std.ThreadParker" {
144 if (builtin.single_threaded)
145 return error.SkipZigTest;
146
147 const Context = struct {
148 parker: ThreadParker,
149 data: u32,
150
151 fn receiver(self: *@This()) void {
152 self.parker.park(&self.data, 0); // receives 1
153 assert(@atomicRmw(u32, &self.data, .Xchg, 2, .SeqCst) == 1); // sends 2
154 self.parker.unpark(&self.data); // wakes up waiters on 2
155 self.parker.park(&self.data, 2); // receives 3
156 assert(@atomicRmw(u32, &self.data, .Xchg, 4, .SeqCst) == 3); // sends 4
157 self.parker.unpark(&self.data); // wakes up waiters on 4
158 }
159
160 fn sender(self: *@This()) void {
161 assert(@atomicRmw(u32, &self.data, .Xchg, 1, .SeqCst) == 0); // sends 1
162 self.parker.unpark(&self.data); // wakes up waiters on 1
163 self.parker.park(&self.data, 1); // receives 2
164 assert(@atomicRmw(u32, &self.data, .Xchg, 3, .SeqCst) == 2); // sends 3
165 self.parker.unpark(&self.data); // wakes up waiters on 3
166 self.parker.park(&self.data, 3); // receives 4
167 }
168 };
169
170 var context = Context{
171 .parker = ThreadParker.init(),
172 .data = 0,
173 };
174 defer context.parker.deinit();
175
176 var receiver = try std.Thread.spawn(&context, Context.receiver);
177 defer receiver.wait();
178
179 context.sender();
180}
lib/std/std.zig+1-1
...@@ -16,6 +16,7 @@ pub const PackedIntSlice = @import("packed_int_array.zig").PackedIntSlice;...@@ -16,6 +16,7 @@ pub const PackedIntSlice = @import("packed_int_array.zig").PackedIntSlice;
16pub const PackedIntSliceEndian = @import("packed_int_array.zig").PackedIntSliceEndian;16pub const PackedIntSliceEndian = @import("packed_int_array.zig").PackedIntSliceEndian;
17pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue;17pub const PriorityQueue = @import("priority_queue.zig").PriorityQueue;
18pub const Progress = @import("progress.zig").Progress;18pub const Progress = @import("progress.zig").Progress;
19pub const ResetEvent = @import("reset_event.zig").ResetEvent;
19pub const SegmentedList = @import("segmented_list.zig").SegmentedList;20pub const SegmentedList = @import("segmented_list.zig").SegmentedList;
20pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList;21pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList;
21pub const SpinLock = @import("spinlock.zig").SpinLock;22pub const SpinLock = @import("spinlock.zig").SpinLock;
...@@ -23,7 +24,6 @@ pub const StringHashMap = @import("hash_map.zig").StringHashMap;...@@ -23,7 +24,6 @@ pub const StringHashMap = @import("hash_map.zig").StringHashMap;
23pub const TailQueue = @import("linked_list.zig").TailQueue;24pub const TailQueue = @import("linked_list.zig").TailQueue;
24pub const Target = @import("target.zig").Target;25pub const Target = @import("target.zig").Target;
25pub const Thread = @import("thread.zig").Thread;26pub const Thread = @import("thread.zig").Thread;
26pub const ThreadParker = @import("parker.zig").ThreadParker;
2727
28pub const atomic = @import("atomic.zig");28pub const atomic = @import("atomic.zig");
29pub const base64 = @import("base64.zig");29pub const base64 = @import("base64.zig");