authorgravatar for 45520026+kprotty@users.noreply.github.comprotty <45520026+kprotty@users.noreply.github.com> 2022-04-19 19:42:15-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-04-19 19:42:15-05:00
loge3cbea934ee196b3833f4c4269bc3b2796c11928
tree4d51ad021293ddc47cfbe27624d55d767efb0754
parent2fa7f6e502724487e13a0a0b482bf499352746b2
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

std.Thread.Futex improvements (#11464)

* atomic: cache_line * Thread: Futex rewrite + more native platform support * Futex: tests compile * Futex: compiles and runs test * Futex: broadcast test * Futex: fix PosixImpl for tests * Futex: fix compile errors for bsd platforms * Futex: review changes + fix timeout=0 + more comments

5 files changed, 836 insertions(+), 358 deletions(-)

lib/std/Thread/Futex.zig+745-355
...@@ -7,9 +7,7 @@ const std = @import("../std.zig");...@@ -7,9 +7,7 @@ const std = @import("../std.zig");
7const builtin = @import("builtin");7const builtin = @import("builtin");
8const Futex = @This();8const Futex = @This();
99
10const target = builtin.target;10const os = std.os;
11const single_threaded = builtin.single_threaded;
12
13const assert = std.debug.assert;11const assert = std.debug.assert;
14const testing = std.testing;12const testing = std.testing;
1513
...@@ -21,160 +19,152 @@ const spinLoopHint = std.atomic.spinLoopHint;...@@ -21,160 +19,152 @@ const spinLoopHint = std.atomic.spinLoopHint;
21/// - The caller is unblocked by a matching `wake()`.19/// - The caller is unblocked by a matching `wake()`.
22/// - The caller is unblocked spuriously by an arbitrary internal signal.20/// - The caller is unblocked spuriously by an arbitrary internal signal.
23///21///
24/// If `timeout` is provided, and the caller is blocked for longer than `timeout` nanoseconds`, `error.TimedOut` is returned.
25///
26/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically22/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically
27/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.23/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.
28pub fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{TimedOut}!void {24pub fn wait(ptr: *const Atomic(u32), expect: u32) void {
29 if (single_threaded) {25 @setCold(true);
30 // check whether the caller should block
31 if (ptr.loadUnchecked() != expect) {
32 return;
33 }
3426
35 // There are no other threads which could notify the caller on single_threaded.27 Impl.wait(ptr, expect, null) catch |err| switch (err) {
36 // Therefor a wait() without a timeout would block indefinitely.28 error.Timeout => unreachable, // null timeout meant to wait forever
37 const timeout_ns = timeout orelse {29 };
38 @panic("deadlock");30}
39 };
4031
41 // Simulate blocking with the timeout knowing that:32/// Checks if `ptr` still contains the value `expect` and, if so, blocks the caller until either:
42 // - no other thread can change the ptr value33/// - The value at `ptr` is no longer equal to `expect`.
43 // - no other thread could unblock us if we waiting on the ptr34/// - The caller is unblocked by a matching `wake()`.
44 std.time.sleep(timeout_ns);35/// - The caller is unblocked spuriously by an arbitrary internal signal.
45 return error.TimedOut;36/// - The caller blocks for longer than the given timeout. In which case, `error.Timeout` is returned.
46 }37///
38/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically
39/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.
40pub fn timedWait(ptr: *const Atomic(u32), expect: u32, timeout_ns: u64) error{Timeout}!void {
41 @setCold(true);
4742
48 // Avoid calling into the OS for no-op waits()43 // Avoid calling into the OS for no-op timeouts.
49 if (timeout) |timeout_ns| {44 if (timeout_ns == 0) {
50 if (timeout_ns == 0) {45 if (ptr.load(.SeqCst) != expect) return;
51 if (ptr.load(.SeqCst) != expect) return;46 return error.Timeout;
52 return error.TimedOut;
53 }
54 }47 }
5548
56 return OsFutex.wait(ptr, expect, timeout);49 return Impl.wait(ptr, expect, timeout_ns);
57}50}
5851
59/// Unblocks at most `num_waiters` callers blocked in a `wait()` call on `ptr`.52/// Unblocks at most `max_waiters` callers blocked in a `wait()` call on `ptr`.
60/// `num_waiters` of 1 unblocks at most one `wait(ptr, ...)` and `maxInt(u32)` unblocks effectively all `wait(ptr, ...)`.53pub fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {
61pub fn wake(ptr: *const Atomic(u32), num_waiters: u32) void {54 @setCold(true);
62 if (single_threaded) return;55
63 if (num_waiters == 0) return;56 // Avoid calling into the OS if there's nothing to wake up.
57 if (max_waiters == 0) {
58 return;
59 }
6460
65 return OsFutex.wake(ptr, num_waiters);61 Impl.wake(ptr, max_waiters);
66}62}
6763
68const OsFutex = if (target.os.tag == .windows)64const Impl = if (builtin.single_threaded)
69 WindowsFutex65 SerialImpl
70else if (target.os.tag == .linux)66else if (builtin.os.tag == .windows)
71 LinuxFutex67 WindowsImpl
72else if (target.isDarwin())68else if (builtin.os.tag.isDarwin())
73 DarwinFutex69 DarwinImpl
74else if (builtin.link_libc)70else if (builtin.os.tag == .linux)
75 PosixFutex71 LinuxImpl
72else if (builtin.os.tag == .freebsd)
73 FreebsdImpl
74else if (builtin.os.tag == .openbsd)
75 OpenbsdImpl
76else if (builtin.os.tag == .dragonfly)
77 DragonflyImpl
78else if (std.Thread.use_pthreads)
79 PosixImpl
76else80else
77 UnsupportedFutex;81 UnsupportedImpl;
7882
79const UnsupportedFutex = struct {83/// We can't do @compileError() in the `Impl` switch statement above as its eagerly evaluated.
80 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{TimedOut}!void {84/// So instead, we @compileError() on the methods themselves for platforms which don't support futex.
85const UnsupportedImpl = struct {
86 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
81 return unsupported(.{ ptr, expect, timeout });87 return unsupported(.{ ptr, expect, timeout });
82 }88 }
8389
84 fn wake(ptr: *const Atomic(u32), num_waiters: u32) void {90 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {
85 return unsupported(.{ ptr, num_waiters });91 return unsupported(.{ ptr, max_waiters });
86 }92 }
8793
88 fn unsupported(unused: anytype) noreturn {94 fn unsupported(unused: anytype) noreturn {
89 @compileLog("Unsupported operating system", target.os.tag);
90 _ = unused;95 _ = unused;
91 unreachable;96 @compileError("Unsupported operating system " ++ @tagName(builtin.target.os.tag));
92 }97 }
93};98};
9499
95const WindowsFutex = struct {100const SerialImpl = struct {
96 const windows = std.os.windows;101 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
102 if (ptr.loadUnchecked() != expect) {
103 return;
104 }
105
106 // There are no threads to wake us up.
107 // So if we wait without a timeout we would never wake up.
108 const delay = timeout orelse {
109 unreachable; // deadlock detected
110 };
111
112 std.time.sleep(delay);
113 return error.Timeout;
114 }
115
116 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {
117 // There are no other threads to possibly wake up
118 _ = ptr;
119 _ = max_waiters;
120 }
121};
97122
98 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{TimedOut}!void {123// We use WaitOnAddress through NtDll instead of API-MS-Win-Core-Synch-l1-2-0.dll
99 var timeout_value: windows.LARGE_INTEGER = undefined;124// as it's generally already a linked target and is autoloaded into all processes anyway.
100 var timeout_ptr: ?*const windows.LARGE_INTEGER = null;125const WindowsImpl = struct {
126 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
127 var timeout_value: os.windows.LARGE_INTEGER = undefined;
128 var timeout_ptr: ?*const os.windows.LARGE_INTEGER = null;
101129
102 // NTDLL functions work with time in units of 100 nanoseconds.130 // NTDLL functions work with time in units of 100 nanoseconds.
103 // Positive values for timeouts are absolute time while negative is relative.131 // Positive values are absolute deadlines while negative values are relative durations.
104 if (timeout) |timeout_ns| {132 if (timeout) |delay| {
133 timeout_value = @intCast(os.windows.LARGE_INTEGER, delay / 100);
134 timeout_value = -timeout_value;
105 timeout_ptr = &timeout_value;135 timeout_ptr = &timeout_value;
106 timeout_value = -@intCast(windows.LARGE_INTEGER, timeout_ns / 100);
107 }136 }
108137
109 switch (windows.ntdll.RtlWaitOnAddress(138 const rc = os.windows.ntdll.RtlWaitOnAddress(
110 @ptrCast(?*const anyopaque, ptr),139 @ptrCast(?*const anyopaque, ptr),
111 @ptrCast(?*const anyopaque, &expect),140 @ptrCast(?*const anyopaque, &expect),
112 @sizeOf(@TypeOf(expect)),141 @sizeOf(@TypeOf(expect)),
113 timeout_ptr,142 timeout_ptr,
114 )) {143 );
144
145 switch (rc) {
115 .SUCCESS => {},146 .SUCCESS => {},
116 .TIMEOUT => return error.TimedOut,147 .TIMEOUT => {
148 assert(timeout != null);
149 return error.Timeout;
150 },
117 else => unreachable,151 else => unreachable,
118 }152 }
119 }153 }
120154
121 fn wake(ptr: *const Atomic(u32), num_waiters: u32) void {155 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {
122 const address = @ptrCast(?*const anyopaque, ptr);156 const address = @ptrCast(?*const anyopaque, ptr);
123 switch (num_waiters) {157 assert(max_waiters != 0);
124 1 => windows.ntdll.RtlWakeAddressSingle(address),
125 else => windows.ntdll.RtlWakeAddressAll(address),
126 }
127 }
128};
129
130const LinuxFutex = struct {
131 const linux = std.os.linux;
132
133 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{TimedOut}!void {
134 var ts: std.os.timespec = undefined;
135 var ts_ptr: ?*std.os.timespec = null;
136
137 // Futex timespec timeout is already in relative time.
138 if (timeout) |timeout_ns| {
139 ts_ptr = &ts;
140 ts.tv_sec = @intCast(@TypeOf(ts.tv_sec), timeout_ns / std.time.ns_per_s);
141 ts.tv_nsec = @intCast(@TypeOf(ts.tv_nsec), timeout_ns % std.time.ns_per_s);
142 }
143
144 switch (linux.getErrno(linux.futex_wait(
145 @ptrCast(*const i32, ptr),
146 linux.FUTEX.PRIVATE_FLAG | linux.FUTEX.WAIT,
147 @bitCast(i32, expect),
148 ts_ptr,
149 ))) {
150 .SUCCESS => {}, // notified by `wake()`
151 .INTR => {}, // spurious wakeup
152 .AGAIN => {}, // ptr.* != expect
153 .TIMEDOUT => return error.TimedOut,
154 .INVAL => {}, // possibly timeout overflow
155 .FAULT => unreachable,
156 else => unreachable,
157 }
158 }
159158
160 fn wake(ptr: *const Atomic(u32), num_waiters: u32) void {159 switch (max_waiters) {
161 switch (linux.getErrno(linux.futex_wake(160 1 => os.windows.ntdll.RtlWakeAddressSingle(address),
162 @ptrCast(*const i32, ptr),161 else => os.windows.ntdll.RtlWakeAddressAll(address),
163 linux.FUTEX.PRIVATE_FLAG | linux.FUTEX.WAKE,
164 std.math.cast(i32, num_waiters) catch std.math.maxInt(i32),
165 ))) {
166 .SUCCESS => {}, // successful wake up
167 .INVAL => {}, // invalid futex_wait() on ptr done elsewhere
168 .FAULT => {}, // pointer became invalid while doing the wake
169 else => unreachable,
170 }162 }
171 }163 }
172};164};
173165
174const DarwinFutex = struct {166const DarwinImpl = struct {
175 const darwin = std.os.darwin;167 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
176
177 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{TimedOut}!void {
178 // Darwin XNU 7195.50.7.100.1 introduced __ulock_wait2 and migrated code paths (notably pthread_cond_t) towards it:168 // Darwin XNU 7195.50.7.100.1 introduced __ulock_wait2 and migrated code paths (notably pthread_cond_t) towards it:
179 // https://github.com/apple/darwin-xnu/commit/d4061fb0260b3ed486147341b72468f836ed6c8f#diff-08f993cc40af475663274687b7c326cc6c3031e0db3ac8de7b24624610616be6169 // https://github.com/apple/darwin-xnu/commit/d4061fb0260b3ed486147341b72468f836ed6c8f#diff-08f993cc40af475663274687b7c326cc6c3031e0db3ac8de7b24624610616be6
180 //170 //
...@@ -183,58 +173,67 @@ const DarwinFutex = struct {...@@ -183,58 +173,67 @@ const DarwinFutex = struct {
183 //173 //
184 // ulock_wait() uses 32-bit micro-second timeouts where 0 = INFINITE or no-timeout174 // ulock_wait() uses 32-bit micro-second timeouts where 0 = INFINITE or no-timeout
185 // ulock_wait2() uses 64-bit nano-second timeouts (with the same convention)175 // ulock_wait2() uses 64-bit nano-second timeouts (with the same convention)
176 const supports_ulock_wait2 = builtin.target.os.version_range.semver.min.major >= 11;
177
186 var timeout_ns: u64 = 0;178 var timeout_ns: u64 = 0;
187 if (timeout) |timeout_value| {179 if (timeout) |delay| {
188 // This should be checked by the caller.180 assert(delay != 0); // handled by timedWait()
189 assert(timeout_value != 0);181 timeout_ns = delay;
190 timeout_ns = timeout_value;
191 }182 }
192 const addr = @ptrCast(*const anyopaque, ptr);183
193 const flags = darwin.UL_COMPARE_AND_WAIT | darwin.ULF_NO_ERRNO;
194 // If we're using `__ulock_wait` and `timeout` is too big to fit inside a `u32` count of184 // If we're using `__ulock_wait` and `timeout` is too big to fit inside a `u32` count of
195 // micro-seconds (around 70min), we'll request a shorter timeout. This is fine (users185 // micro-seconds (around 70min), we'll request a shorter timeout. This is fine (users
196 // should handle spurious wakeups), but we need to remember that we did so, so that186 // should handle spurious wakeups), but we need to remember that we did so, so that
197 // we don't return `TimedOut` incorrectly. If that happens, we set this variable to187 // we don't return `Timeout` incorrectly. If that happens, we set this variable to
198 // true so that we we know to ignore the ETIMEDOUT result.188 // true so that we we know to ignore the ETIMEDOUT result.
199 var timeout_overflowed = false;189 var timeout_overflowed = false;
190
191 const addr = @ptrCast(*const anyopaque, ptr);
192 const flags = os.darwin.UL_COMPARE_AND_WAIT | os.darwin.ULF_NO_ERRNO;
200 const status = blk: {193 const status = blk: {
201 if (target.os.version_range.semver.min.major >= 11) {194 if (supports_ulock_wait2) {
202 break :blk darwin.__ulock_wait2(flags, addr, expect, timeout_ns, 0);195 break :blk os.darwin.__ulock_wait2(flags, addr, expect, timeout_ns, 0);
203 } else {
204 const timeout_us = std.math.cast(u32, timeout_ns / std.time.ns_per_us) catch overflow: {
205 timeout_overflowed = true;
206 break :overflow std.math.maxInt(u32);
207 };
208 break :blk darwin.__ulock_wait(flags, addr, expect, timeout_us);
209 }196 }
197
198 const timeout_us = std.math.cast(u32, timeout_ns / std.time.ns_per_us) catch overflow: {
199 timeout_overflowed = true;
200 break :overflow std.math.maxInt(u32);
201 };
202
203 break :blk os.darwin.__ulock_wait(flags, addr, expect, timeout_us);
210 };204 };
211205
212 if (status >= 0) return;206 if (status >= 0) return;
213 switch (@intToEnum(std.os.E, -status)) {207 switch (@intToEnum(std.os.E, -status)) {
208 // Wait was interrupted by the OS or other spurious signalling.
214 .INTR => {},209 .INTR => {},
215 // Address of the futex is paged out. This is unlikely, but possible in theory, and210 // Address of the futex was paged out. This is unlikely, but possible in theory, and
216 // pthread/libdispatch on darwin bother to handle it. In this case we'll return211 // pthread/libdispatch on darwin bother to handle it. In this case we'll return
217 // without waiting, but the caller should retry anyway.212 // without waiting, but the caller should retry anyway.
218 .FAULT => {},213 .FAULT => {},
219 .TIMEDOUT => if (!timeout_overflowed) return error.TimedOut,214 // Only report Timeout if we didn't have to cap the timeout
215 .TIMEDOUT => {
216 assert(timeout != null);
217 if (!timeout_overflowed) return error.Timeout;
218 },
220 else => unreachable,219 else => unreachable,
221 }220 }
222 }221 }
223222
224 fn wake(ptr: *const Atomic(u32), num_waiters: u32) void {223 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {
225 var flags: u32 = darwin.UL_COMPARE_AND_WAIT | darwin.ULF_NO_ERRNO;224 var flags: u32 = os.darwin.UL_COMPARE_AND_WAIT | os.darwin.ULF_NO_ERRNO;
226 if (num_waiters > 1) {225 if (max_waiters > 1) {
227 flags |= darwin.ULF_WAKE_ALL;226 flags |= os.darwin.ULF_WAKE_ALL;
228 }227 }
229228
230 while (true) {229 while (true) {
231 const addr = @ptrCast(*const anyopaque, ptr);230 const addr = @ptrCast(*const anyopaque, ptr);
232 const status = darwin.__ulock_wake(flags, addr, 0);231 const status = os.darwin.__ulock_wake(flags, addr, 0);
233232
234 if (status >= 0) return;233 if (status >= 0) return;
235 switch (@intToEnum(std.os.E, -status)) {234 switch (@intToEnum(std.os.E, -status)) {
236 .INTR => continue, // spurious wake()235 .INTR => continue, // spurious wake()
237 .FAULT => continue, // address of the lock was paged out236 .FAULT => unreachable, // __ulock_wake doesn't generate EFAULT according to darwin pthread_cond_t
238 .NOENT => return, // nothing was woken up237 .NOENT => return, // nothing was woken up
239 .ALREADY => unreachable, // only for ULF_WAKE_THREAD238 .ALREADY => unreachable, // only for ULF_WAKE_THREAD
240 else => unreachable,239 else => unreachable,
...@@ -243,332 +242,723 @@ const DarwinFutex = struct {...@@ -243,332 +242,723 @@ const DarwinFutex = struct {
243 }242 }
244};243};
245244
246const PosixFutex = struct {245// https://man7.org/linux/man-pages/man2/futex.2.html
247 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{TimedOut}!void {246const LinuxImpl = struct {
248 const address = @ptrToInt(ptr);247 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
249 const bucket = Bucket.from(address);248 var ts: os.timespec = undefined;
250 var waiter: List.Node = undefined;249 if (timeout) |timeout_ns| {
250 ts.tv_sec = @intCast(@TypeOf(ts.tv_sec), timeout_ns / std.time.ns_per_s);
251 ts.tv_nsec = @intCast(@TypeOf(ts.tv_nsec), timeout_ns % std.time.ns_per_s);
252 }
251253
252 {254 const rc = os.linux.futex_wait(
253 assert(std.c.pthread_mutex_lock(&bucket.mutex) == .SUCCESS);255 @ptrCast(*const i32, &ptr.value),
254 defer assert(std.c.pthread_mutex_unlock(&bucket.mutex) == .SUCCESS);256 os.linux.FUTEX.PRIVATE_FLAG | os.linux.FUTEX.WAIT,
257 @bitCast(i32, expect),
258 if (timeout != null) &ts else null,
259 );
255260
256 if (ptr.load(.SeqCst) != expect) {261 switch (os.linux.getErrno(rc)) {
257 return;262 .SUCCESS => {}, // notified by `wake()`
258 }263 .INTR => {}, // spurious wakeup
264 .AGAIN => {}, // ptr.* != expect
265 .TIMEDOUT => {
266 assert(timeout != null);
267 return error.Timeout;
268 },
269 .INVAL => {}, // possibly timeout overflow
270 .FAULT => unreachable, // ptr was invalid
271 else => unreachable,
272 }
273 }
259274
260 waiter.data = .{ .address = address };275 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {
261 bucket.list.prepend(&waiter);276 const rc = os.linux.futex_wake(
277 @ptrCast(*const i32, &ptr.value),
278 os.linux.FUTEX.PRIVATE_FLAG | os.linux.FUTEX.WAKE,
279 std.math.cast(i32, max_waiters) catch std.math.maxInt(i32),
280 );
281
282 switch (os.linux.getErrno(rc)) {
283 .SUCCESS => {}, // successful wake up
284 .INVAL => {}, // invalid futex_wait() on ptr done elsewhere
285 .FAULT => {}, // pointer became invalid while doing the wake
286 else => unreachable,
262 }287 }
288 }
289};
263290
264 var timed_out = false;291// https://www.freebsd.org/cgi/man.cgi?query=_umtx_op&sektion=2&n=1
265 waiter.data.wait(timeout) catch {292const FreebsdImpl = struct {
266 defer if (!timed_out) {293 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
267 waiter.data.wait(null) catch unreachable;294 var tm_size: usize = 0;
268 };295 var tm: os.freebsd._umtx_time = undefined;
296 var tm_ptr: ?*const os.freebsd._umtx_time = null;
269297
270 assert(std.c.pthread_mutex_lock(&bucket.mutex) == .SUCCESS);298 if (timeout) |timeout_ns| {
271 defer assert(std.c.pthread_mutex_unlock(&bucket.mutex) == .SUCCESS);299 tm_ptr = &tm;
300 tm_size = @sizeOf(@TypeOf(tm));
272301
273 if (waiter.data.address == address) {302 tm._flags = 0; // use relative time not UMTX_ABSTIME
274 timed_out = true;303 tm._clockid = os.CLOCK.MONOTONIC;
275 bucket.list.remove(&waiter);304 tm._timeout.tv_sec = @intCast(@TypeOf(tm._timeout.tv_sec), timeout_ns / std.time.ns_per_s);
276 }305 tm._timeout.tv_nsec = @intCast(@TypeOf(tm._timeout.tv_nsec), timeout_ns % std.time.ns_per_s);
277 };306 }
278307
279 waiter.data.deinit();308 const rc = os.freebsd._umtx_op(
280 if (timed_out) {309 @ptrToInt(&ptr.value),
281 return error.TimedOut;310 @enumToInt(os.freebsd.UMTX_OP.WAIT_UINT_PRIVATE),
311 @as(c_ulong, expect),
312 tm_size,
313 @ptrToInt(tm_ptr),
314 );
315
316 switch (os.errno(rc)) {
317 .SUCCESS => {},
318 .FAULT => unreachable, // one of the args points to invalid memory
319 .INVAL => unreachable, // arguments should be correct
320 .TIMEDOUT => {
321 assert(timeout != null);
322 return error.Timeout;
323 },
324 .INTR => {}, // spurious wake
325 else => unreachable,
282 }326 }
283 }327 }
284328
285 fn wake(ptr: *const Atomic(u32), num_waiters: u32) void {329 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {
286 const address = @ptrToInt(ptr);330 const rc = os.freebsd._umtx_op(
287 const bucket = Bucket.from(address);331 @ptrToInt(&ptr.value),
288 var can_notify = num_waiters;332 @enumToInt(os.freebsd.UMTX_OP.WAKE_PRIVATE),
333 @as(c_ulong, max_waiters),
334 0, // there is no timeout struct
335 0, // there is no timeout struct pointer
336 );
289337
290 var notified = List{};338 switch (os.errno(rc)) {
291 defer while (notified.popFirst()) |waiter| {339 .SUCCESS => {},
292 waiter.data.notify();340 .FAULT => {}, // it's ok if the ptr doesn't point to valid memory
293 };341 .INVAL => unreachable, // arguments should be correct
342 else => unreachable,
343 }
344 }
345};
294346
295 assert(std.c.pthread_mutex_lock(&bucket.mutex) == .SUCCESS);347// https://man.openbsd.org/futex.2
296 defer assert(std.c.pthread_mutex_unlock(&bucket.mutex) == .SUCCESS);348const OpenbsdImpl = struct {
349 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
350 var ts: os.timespec = undefined;
351 if (timeout) |timeout_ns| {
352 ts.tv_sec = @intCast(@TypeOf(ts.tv_sec), timeout_ns / std.time.ns_per_s);
353 ts.tv_nsec = @intCast(@TypeOf(ts.tv_nsec), timeout_ns % std.time.ns_per_s);
354 }
297355
298 var waiters = bucket.list.first;356 const rc = os.openbsd.futex(
299 while (waiters) |waiter| {357 @ptrCast(*const volatile u32, &ptr.value),
300 assert(waiter.data.address != null);358 os.openbsd.FUTEX_WAIT | os.openbsd.FUTEX_PRIVATE_FLAG,
301 waiters = waiter.next;359 @bitCast(c_int, expect),
360 if (timeout != null) &ts else null,
361 null, // FUTEX_WAIT takes no requeue address
362 );
363
364 switch (os.errno(rc)) {
365 .SUCCESS => {}, // woken up by wake
366 .NOSYS => unreachable, // the futex operation shouldn't be invalid
367 .FAULT => unreachable, // ptr was invalid
368 .AGAIN => {}, // ptr != expect
369 .INVAL => unreachable, // invalid timeout
370 .TIMEDOUT => {
371 assert(timeout != null);
372 return error.Timeout;
373 },
374 .INTR => {}, // spurious wake from signal
375 .CANCELED => {}, // spurious wake from signal with SA_RESTART
376 else => unreachable,
377 }
378 }
379
380 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {
381 const rc = os.openbsd.futex(
382 @ptrCast(*const volatile u32, &ptr.value),
383 os.openbsd.FUTEX_WAKE | os.openbsd.FUTEX_PRIVATE_FLAG,
384 std.math.cast(c_int, max_waiters) catch std.math.maxInt(c_int),
385 null, // FUTEX_WAKE takes no timeout ptr
386 null, // FUTEX_WAKE takes no requeue address
387 );
388
389 // returns number of threads woken up.
390 assert(rc >= 0);
391 }
392};
302393
303 if (waiter.data.address != address) continue;394// https://man.dragonflybsd.org/?command=umtx&section=2
304 if (can_notify == 0) break;395const DragonflyImpl = struct {
305 can_notify -= 1;396 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
397 // Dragonfly uses a scheme where 0 timeout means wait until signaled or spurious wake.
398 // It's reporting of timeout's is also unrealiable so we use an external timing source (Timer) instead.
399 var timeout_us: c_int = 0;
400 var timeout_overflowed = false;
401 var sleep_timer: std.time.Timer = undefined;
306402
307 bucket.list.remove(waiter);403 if (timeout) |delay| {
308 waiter.data.address = null;404 assert(delay != 0); // handled by timedWait().
309 notified.prepend(waiter);405 timeout_us = std.math.cast(c_int, delay / std.time.ns_per_us) catch blk: {
406 timeout_overflowed = true;
407 break :blk std.math.maxInt(c_int);
408 };
409
410 // Only need to record the start time if we can provide somewhat accurate error.Timeout's
411 if (!timeout_overflowed) {
412 sleep_timer = std.time.Timer.start() catch unreachable;
413 }
414 }
415
416 const value = @bitCast(c_int, expect);
417 const addr = @ptrCast(*const volatile c_int, &ptr.value);
418 const rc = os.dragonfly.umtx_sleep(addr, value, timeout_us);
419
420 switch (os.errno(rc)) {
421 .SUCCESS => {},
422 .BUSY => {}, // ptr != expect
423 .AGAIN => { // maybe timed out, or paged out, or hit 2s kernel refresh
424 if (timeout) |timeout_ns| {
425 // Report error.Timeout only if we know the timeout duration has passed.
426 // If not, there's not much choice other than treating it as a spurious wake.
427 if (!timeout_overflowed and sleep_timer.read() >= timeout_ns) {
428 return error.Timeout;
429 }
430 }
431 },
432 .INTR => {}, // spurious wake
433 .INVAL => unreachable, // invalid timeout
434 else => unreachable,
310 }435 }
311 }436 }
312437
313 const Bucket = struct {438 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {
314 mutex: std.c.pthread_mutex_t = .{},439 // A count of zero means wake all waiters.
315 list: List = .{},440 assert(max_waiters != 0);
441 const to_wake = std.math.cast(c_int, max_waiters) catch 0;
316442
317 var buckets = [_]Bucket{.{}} ** 64;443 // https://man.dragonflybsd.org/?command=umtx&section=2
444 // > umtx_wakeup() will generally return 0 unless the address is bad.
445 // We are fine with the address being bad (e.g. for Semaphore.post() where Semaphore.wait() frees the Semaphore)
446 const addr = @ptrCast(*const volatile c_int, &ptr.value);
447 _ = os.dragonfly.umtx_wakeup(addr, to_wake);
448 }
449};
318450
319 fn from(address: usize) *Bucket {451/// Modified version of linux's futex and Go's sema to implement userspace wait queues with pthread:
320 return &buckets[address % buckets.len];452/// https://code.woboq.org/linux/linux/kernel/futex.c.html
453/// https://go.dev/src/runtime/sema.go
454const PosixImpl = struct {
455 const Event = struct {
456 cond: std.c.pthread_cond_t,
457 mutex: std.c.pthread_mutex_t,
458 state: enum { empty, waiting, notified },
459
460 fn init(self: *Event) void {
461 // Use static init instead of pthread_cond/mutex_init() since this is generally faster.
462 self.cond = .{};
463 self.mutex = .{};
464 self.state = .empty;
321 }465 }
322 };
323466
324 const List = std.TailQueue(struct {467 fn deinit(self: *Event) void {
325 address: ?usize,468 // Some platforms reportedly give EINVAL for statically initialized pthread types.
326 state: State = .empty,469 const rc = std.c.pthread_cond_destroy(&self.cond);
327 cond: std.c.pthread_cond_t = .{},470 assert(rc == .SUCCESS or rc == .INVAL);
328 mutex: std.c.pthread_mutex_t = .{},471
329472 const rm = std.c.pthread_mutex_destroy(&self.mutex);
330 const Self = @This();473 assert(rm == .SUCCESS or rm == .INVAL);
331 const State = enum {
332 empty,
333 waiting,
334 notified,
335 };
336474
337 fn deinit(self: *Self) void {475 self.* = undefined;
338 _ = std.c.pthread_cond_destroy(&self.cond);
339 _ = std.c.pthread_mutex_destroy(&self.mutex);
340 }476 }
341477
342 fn wait(self: *Self, timeout: ?u64) error{TimedOut}!void {478 fn wait(self: *Event, timeout: ?u64) error{Timeout}!void {
343 assert(std.c.pthread_mutex_lock(&self.mutex) == .SUCCESS);479 assert(std.c.pthread_mutex_lock(&self.mutex) == .SUCCESS);
344 defer assert(std.c.pthread_mutex_unlock(&self.mutex) == .SUCCESS);480 defer assert(std.c.pthread_mutex_unlock(&self.mutex) == .SUCCESS);
345481
346 switch (self.state) {482 // Early return if the event was already set.
347 .empty => self.state = .waiting,483 if (self.state == .notified) {
348 .waiting => unreachable,484 return;
349 .notified => return,
350 }485 }
351486
352 var ts: std.os.timespec = undefined;487 // Compute the absolute timeout if one was specified.
353 var ts_ptr: ?*const std.os.timespec = null;488 // POSIX requires that REALTIME is used by default for the pthread timedwait functions.
489 // This can be changed with pthread_condattr_setclock, but it's an extension and may not be available everywhere.
490 var ts: os.timespec = undefined;
354 if (timeout) |timeout_ns| {491 if (timeout) |timeout_ns| {
355 ts_ptr = &ts;492 os.clock_gettime(os.CLOCK.REALTIME, &ts) catch unreachable;
356 std.os.clock_gettime(std.os.CLOCK.REALTIME, &ts) catch unreachable;493 ts.tv_sec +|= @intCast(@TypeOf(ts.tv_sec), timeout_ns / std.time.ns_per_s);
357 ts.tv_sec += @intCast(@TypeOf(ts.tv_sec), timeout_ns / std.time.ns_per_s);
358 ts.tv_nsec += @intCast(@TypeOf(ts.tv_nsec), timeout_ns % std.time.ns_per_s);494 ts.tv_nsec += @intCast(@TypeOf(ts.tv_nsec), timeout_ns % std.time.ns_per_s);
495
359 if (ts.tv_nsec >= std.time.ns_per_s) {496 if (ts.tv_nsec >= std.time.ns_per_s) {
360 ts.tv_sec += 1;497 ts.tv_sec +|= 1;
361 ts.tv_nsec -= std.time.ns_per_s;498 ts.tv_nsec -= std.time.ns_per_s;
362 }499 }
363 }500 }
364501
365 while (true) {502 // Start waiting on the event - there can be only one thread waiting.
366 switch (self.state) {503 assert(self.state == .empty);
367 .empty => unreachable,504 self.state = .waiting;
368 .waiting => {},
369 .notified => return,
370 }
371505
372 const ts_ref = ts_ptr orelse {506 while (true) {
373 assert(std.c.pthread_cond_wait(&self.cond, &self.mutex) == .SUCCESS);507 // Block using either pthread_cond_wait or pthread_cond_timewait if there's an absolute timeout.
374 continue;508 const rc = blk: {
509 if (timeout == null) break :blk std.c.pthread_cond_wait(&self.cond, &self.mutex);
510 break :blk std.c.pthread_cond_timedwait(&self.cond, &self.mutex, &ts);
375 };511 };
376512
377 const rc = std.c.pthread_cond_timedwait(&self.cond, &self.mutex, ts_ref);513 // After waking up, check if the event was set.
514 if (self.state == .notified) {
515 return;
516 }
517
518 assert(self.state == .waiting);
378 switch (rc) {519 switch (rc) {
379 .SUCCESS => {},520 .SUCCESS => {},
380 .TIMEDOUT => {521 .TIMEDOUT => {
522 // If timed out, reset the event to avoid the set() thread doing an unnecessary signal().
381 self.state = .empty;523 self.state = .empty;
382 return error.TimedOut;524 return error.Timeout;
383 },525 },
526 .INVAL => unreachable, // cond, mutex, and potentially ts should all be valid
527 .PERM => unreachable, // mutex is locked when cond_*wait() functions are called
384 else => unreachable,528 else => unreachable,
385 }529 }
386 }530 }
387 }531 }
388532
389 fn notify(self: *Self) void {533 fn set(self: *Event) void {
390 assert(std.c.pthread_mutex_lock(&self.mutex) == .SUCCESS);534 assert(std.c.pthread_mutex_lock(&self.mutex) == .SUCCESS);
391 defer assert(std.c.pthread_mutex_unlock(&self.mutex) == .SUCCESS);535 defer assert(std.c.pthread_mutex_unlock(&self.mutex) == .SUCCESS);
392536
393 switch (self.state) {537 // Make sure that multiple calls to set() were not done on the same Event.
394 .empty => self.state = .notified,538 const old_state = self.state;
395 .waiting => {539 assert(old_state != .notified);
396 self.state = .notified;540
397 assert(std.c.pthread_cond_signal(&self.cond) == .SUCCESS);541 // Mark the event as set and wake up the waiting thread if there was one.
398 },542 // This must be done while the mutex as the wait() thread could deallocate
399 .notified => unreachable,543 // the condition variable once it observes the new state, potentially causing a UAF if done unlocked.
544 self.state = .notified;
545 if (old_state == .waiting) {
546 assert(std.c.pthread_cond_signal(&self.cond) == .SUCCESS);
400 }547 }
401 }548 }
402 });549 };
403};
404550
405test "Futex - wait/wake" {551 const Treap = std.Treap(usize, std.math.order);
406 var value = Atomic(u32).init(0);552 const Waiter = struct {
407 Futex.wait(&value, 1, null) catch unreachable;553 node: Treap.Node,
554 prev: ?*Waiter,
555 next: ?*Waiter,
556 tail: ?*Waiter,
557 is_queued: bool,
558 event: Event,
559 };
408560
409 const wait_noop_result = Futex.wait(&value, 0, 0);561 // An unordered set of Waiters
410 try testing.expectError(error.TimedOut, wait_noop_result);562 const WaitList = struct {
563 top: ?*Waiter = null,
564 len: usize = 0,
411565
412 const wait_longer_result = Futex.wait(&value, 0, std.time.ns_per_ms);566 fn push(self: *WaitList, waiter: *Waiter) void {
413 try testing.expectError(error.TimedOut, wait_longer_result);567 waiter.next = self.top;
568 self.top = waiter;
569 self.len += 1;
570 }
414571
415 Futex.wake(&value, 0);572 fn pop(self: *WaitList) ?*Waiter {
416 Futex.wake(&value, 1);573 const waiter = self.top orelse return null;
417 Futex.wake(&value, std.math.maxInt(u32));574 self.top = waiter.next;
418}575 self.len -= 1;
576 return waiter;
577 }
578 };
419579
420test "Futex - Signal" {580 const WaitQueue = struct {
421 if (single_threaded) {581 fn insert(treap: *Treap, address: usize, waiter: *Waiter) void {
422 return error.SkipZigTest;582 // prepare the waiter to be inserted.
423 }583 waiter.next = null;
584 waiter.is_queued = true;
585
586 // Find the wait queue entry associated with the address.
587 // If there isn't a wait queue on the address, this waiter creates the queue.
588 var entry = treap.getEntryFor(address);
589 const entry_node = entry.node orelse {
590 waiter.prev = null;
591 waiter.tail = waiter;
592 entry.set(&waiter.node);
593 return;
594 };
424595
425 const Paddle = struct {596 // There's a wait queue on the address; get the queue head and tail.
426 value: Atomic(u32) = Atomic(u32).init(0),597 const head = @fieldParentPtr(Waiter, "node", entry_node);
427 current: u32 = 0,598 const tail = head.tail orelse unreachable;
428599
429 fn run(self: *@This(), hit_to: *@This()) !void {600 // Push the waiter to the tail by replacing it and linking to the previous tail.
430 var iterations: usize = 4;601 head.tail = waiter;
431 while (iterations > 0) : (iterations -= 1) {602 tail.next = waiter;
432 var value: u32 = undefined;603 waiter.prev = tail;
433 while (true) {604 }
434 value = self.value.load(.Acquire);605
435 if (value != self.current) break;606 fn remove(treap: *Treap, address: usize, max_waiters: usize) WaitList {
436 Futex.wait(&self.value, self.current, null) catch unreachable;607 // Find the wait queue associated with this address and get the head/tail if any.
437 }608 var entry = treap.getEntryFor(address);
609 var queue_head = if (entry.node) |node| @fieldParentPtr(Waiter, "node", node) else null;
610 const queue_tail = if (queue_head) |head| head.tail else null;
611
612 // Once we're done updating the head, fix it's tail pointer and update the treap's queue head as well.
613 defer entry.set(blk: {
614 const new_head = queue_head orelse break :blk null;
615 new_head.tail = queue_tail;
616 break :blk &new_head.node;
617 });
618
619 var removed = WaitList{};
620 while (removed.len < max_waiters) {
621 // dequeue and collect waiters from their wait queue.
622 const waiter = queue_head orelse break;
623 queue_head = waiter.next;
624 removed.push(waiter);
625
626 // When dequeueing, we must mark is_queued as false.
627 // This ensures that a waiter which calls tryRemove() returns false.
628 assert(waiter.is_queued);
629 waiter.is_queued = false;
630 }
438631
439 try testing.expectEqual(value, self.current + 1);632 return removed;
440 self.current = value;633 }
634
635 fn tryRemove(treap: *Treap, address: usize, waiter: *Waiter) bool {
636 if (!waiter.is_queued) {
637 return false;
638 }
639
640 queue_remove: {
641 // Find the wait queue associated with the address.
642 var entry = blk: {
643 // A waiter without a previous link means it's the queue head that's in the treap so we can avoid lookup.
644 if (waiter.prev == null) {
645 assert(waiter.node.key == address);
646 break :blk treap.getEntryForExisting(&waiter.node);
647 }
648 break :blk treap.getEntryFor(address);
649 };
650
651 // The queue head and tail must exist if we're removing a queued waiter.
652 const head = @fieldParentPtr(Waiter, "node", entry.node orelse unreachable);
653 const tail = head.tail orelse unreachable;
654
655 // A waiter with a previous link is never the head of the queue.
656 if (waiter.prev) |prev| {
657 assert(waiter != head);
658 prev.next = waiter.next;
659
660 // A waiter with both a previous and next link is in the middle.
661 // We only need to update the surrounding waiter's links to remove it.
662 if (waiter.next) |next| {
663 assert(waiter != tail);
664 next.prev = waiter.prev;
665 break :queue_remove;
666 }
667
668 // A waiter with a previous but no next link means it's the tail of the queue.
669 // In that case, we need to update the head's tail reference.
670 assert(waiter == tail);
671 head.tail = waiter.prev;
672 break :queue_remove;
673 }
441674
442 _ = hit_to.value.fetchAdd(1, .Release);675 // A waiter with no previous link means it's the queue head of queue.
443 Futex.wake(&hit_to.value, 1);676 // We must replace (or remove) the head waiter reference in the treap.
677 assert(waiter == head);
678 entry.set(blk: {
679 const new_head = waiter.next orelse break :blk null;
680 new_head.tail = head.tail;
681 break :blk &new_head.node;
682 });
444 }683 }
684
685 // Mark the waiter as successfully removed.
686 waiter.is_queued = false;
687 return true;
445 }688 }
446 };689 };
447690
448 var ping = Paddle{};691 const Bucket = struct {
449 var pong = Paddle{};692 mutex: std.c.pthread_mutex_t align(std.atomic.cache_line) = .{},
693 pending: Atomic(usize) = Atomic(usize).init(0),
694 treap: Treap = .{},
450695
451 const t1 = try std.Thread.spawn(.{}, Paddle.run, .{ &ping, &pong });696 // Global array of buckets that addresses map to.
452 defer t1.join();697 // Bucket array size is pretty much arbitrary here, but it must be a power of two for fibonacci hashing.
698 var buckets = [_]Bucket{.{}} ** @bitSizeOf(usize);
453699
454 const t2 = try std.Thread.spawn(.{}, Paddle.run, .{ &pong, &ping });700 // https://github.com/Amanieu/parking_lot/blob/1cf12744d097233316afa6c8b7d37389e4211756/core/src/parking_lot.rs#L343-L353
455 defer t2.join();701 fn from(address: usize) *Bucket {
702 // The upper `@bitSizeOf(usize)` bits of the fibonacci golden ratio.
703 // Hashing this via (h * k) >> (64 - b) where k=golden-ration and b=bitsize-of-array
704 // evenly lays out h=hash values over the bit range even when the hash has poor entropy (identity-hash for pointers).
705 const max_multiplier_bits = @bitSizeOf(usize);
706 const fibonacci_multiplier = 0x9E3779B97F4A7C15 >> (64 - max_multiplier_bits);
456707
457 _ = ping.value.fetchAdd(1, .Release);708 const max_bucket_bits = @ctz(usize, buckets.len);
458 Futex.wake(&ping.value, 1);709 comptime assert(std.math.isPowerOfTwo(buckets.len));
459}
460710
461test "Futex - Broadcast" {711 const index = (address *% fibonacci_multiplier) >> (max_multiplier_bits - max_bucket_bits);
462 if (single_threaded) {712 return &buckets[index];
463 return error.SkipZigTest;713 }
464 }714 };
465715
466 const Context = struct {716 const Address = struct {
467 threads: [4]std.Thread = undefined,717 fn from(ptr: *const Atomic(u32)) usize {
468 broadcast: Atomic(u32) = Atomic(u32).init(0),718 // Get the alignment of the pointer.
469 notified: Atomic(usize) = Atomic(usize).init(0),719 const alignment = @alignOf(Atomic(u32));
720 comptime assert(std.math.isPowerOfTwo(alignment));
721
722 // Make sure the pointer is aligned,
723 // then cut off the zero bits from the alignment to get the unique address.
724 const addr = @ptrToInt(ptr);
725 assert(addr & (alignment - 1) == 0);
726 return addr >> @ctz(usize, alignment);
727 }
728 };
470729
471 const BROADCAST_EMPTY = 0;730 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
472 const BROADCAST_SENT = 1;731 const address = Address.from(ptr);
473 const BROADCAST_RECEIVED = 2;732 const bucket = Bucket.from(address);
474733
475 fn runSender(self: *@This()) !void {734 // Announce that there's a waiter in the bucket before checking the ptr/expect condition.
476 self.broadcast.store(BROADCAST_SENT, .Monotonic);735 // If the announcement is reordered after the ptr check, the waiter could deadlock:
477 Futex.wake(&self.broadcast, @intCast(u32, self.threads.len));736 //
737 // - T1: checks ptr == expect which is true
738 // - T2: updates ptr to != expect
739 // - T2: does Futex.wake(), sees no pending waiters, exits
740 // - T1: bumps pending waiters (was reordered after the ptr == expect check)
741 // - T1: goes to sleep and misses both the ptr change and T2's wake up
742 //
743 // SeqCst as Acquire barrier to ensure the announcement happens before the ptr check below.
744 // SeqCst as shared modification order to form a happens-before edge with the fence(.SeqCst)+load() in wake().
745 var pending = bucket.pending.fetchAdd(1, .SeqCst);
746 assert(pending < std.math.maxInt(usize));
747
748 // If the wait gets cancelled, remove the pending count we previously added.
749 // This is done outside the mutex lock to keep the critical section short in case of contention.
750 var cancelled = false;
751 defer if (cancelled) {
752 pending = bucket.pending.fetchSub(1, .Monotonic);
753 assert(pending > 0);
754 };
478755
479 while (true) {756 var waiter: Waiter = undefined;
480 const broadcast = self.broadcast.load(.Acquire);757 {
481 if (broadcast == BROADCAST_RECEIVED) break;758 assert(std.c.pthread_mutex_lock(&bucket.mutex) == .SUCCESS);
482 try testing.expectEqual(broadcast, BROADCAST_SENT);759 defer assert(std.c.pthread_mutex_unlock(&bucket.mutex) == .SUCCESS);
483 Futex.wait(&self.broadcast, broadcast, null) catch unreachable;760
761 cancelled = ptr.load(.Monotonic) != expect;
762 if (cancelled) {
763 return;
484 }764 }
765
766 waiter.event.init();
767 WaitQueue.insert(&bucket.treap, address, &waiter);
485 }768 }
486769
487 fn runReceiver(self: *@This()) void {770 defer {
488 while (true) {771 assert(!waiter.is_queued);
489 const broadcast = self.broadcast.load(.Acquire);772 waiter.event.deinit();
490 if (broadcast == BROADCAST_SENT) break;773 }
491 assert(broadcast == BROADCAST_EMPTY);774
492 Futex.wait(&self.broadcast, broadcast, null) catch unreachable;775 waiter.event.wait(timeout) catch {
776 // If we fail to cancel after a timeout, it means a wake() thread dequeued us and will wake us up.
777 // We must wait until the event is set as that's a signal that the wake() thread wont access the waiter memory anymore.
778 // If we return early without waiting, the waiter on the stack would be invalidated and the wake() thread risks a UAF.
779 defer if (!cancelled) waiter.event.wait(null) catch unreachable;
780
781 assert(std.c.pthread_mutex_lock(&bucket.mutex) == .SUCCESS);
782 defer assert(std.c.pthread_mutex_unlock(&bucket.mutex) == .SUCCESS);
783
784 cancelled = WaitQueue.tryRemove(&bucket.treap, address, &waiter);
785 if (cancelled) {
786 return error.Timeout;
493 }787 }
788 };
789 }
790
791 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {
792 const address = Address.from(ptr);
793 const bucket = Bucket.from(address);
494794
495 const notified = self.notified.fetchAdd(1, .Monotonic);795 // Quick check if there's even anything to wake up.
496 if (notified + 1 == self.threads.len) {796 // The change to the ptr's value must happen before we check for pending waiters.
497 self.broadcast.store(BROADCAST_RECEIVED, .Release);797 // If not, the wake() thread could miss a sleeping waiter and have it deadlock:
498 Futex.wake(&self.broadcast, 1);798 //
799 // - T2: p = has pending waiters (reordered before the ptr update)
800 // - T1: bump pending waiters
801 // - T1: if ptr == expected: sleep()
802 // - T2: update ptr != expected
803 // - T2: p is false from earlier so doesn't wake (T1 missed ptr update and T2 missed T1 sleeping)
804 //
805 // What we really want here is a Release load, but that doesn't exist under the C11 memory model.
806 // We could instead do `bucket.pending.fetchAdd(0, Release) == 0` which achieves effectively the same thing,
807 // but the RMW operation unconditionally stores which invalidates the cache-line for others causing unnecessary contention.
808 //
809 // Instead we opt to do a full-fence + load instead which avoids taking ownership of the cache-line.
810 // fence(SeqCst) effectively converts the ptr update to SeqCst and the pending load to SeqCst: creating a Store-Load barrier.
811 //
812 // The pending count increment in wait() must also now use SeqCst for the update + this pending load
813 // to be in the same modification order as our load isn't using Release/Acquire to guarantee it.
814 std.atomic.fence(.SeqCst);
815 if (bucket.pending.load(.Monotonic) == 0) {
816 return;
817 }
818
819 // Keep a list of all the waiters notified and wake then up outside the mutex critical section.
820 var notified = WaitList{};
821 defer if (notified.len > 0) {
822 const pending = bucket.pending.fetchSub(notified.len, .Monotonic);
823 assert(pending >= notified.len);
824
825 while (notified.pop()) |waiter| {
826 assert(!waiter.is_queued);
827 waiter.event.set();
499 }828 }
829 };
830
831 assert(std.c.pthread_mutex_lock(&bucket.mutex) == .SUCCESS);
832 defer assert(std.c.pthread_mutex_unlock(&bucket.mutex) == .SUCCESS);
833
834 // Another pending check again to avoid the WaitQueue lookup if not necessary.
835 if (bucket.pending.load(.Monotonic) > 0) {
836 notified = WaitQueue.remove(&bucket.treap, address, max_waiters);
500 }837 }
501 };838 }
839};
502840
503 var ctx = Context{};841test "Futex - smoke test" {
504 for (ctx.threads) |*thread|842 var value = Atomic(u32).init(0);
505 thread.* = try std.Thread.spawn(.{}, Context.runReceiver, .{&ctx});843
506 defer for (ctx.threads) |thread|844 // Try waits with invalid values.
507 thread.join();845 Futex.wait(&value, 0xdeadbeef);
846 Futex.timedWait(&value, 0xdeadbeef, 0) catch {};
508847
509 // Try to wait for the threads to start before running runSender().848 // Try timeout waits.
510 // NOTE: not actually needed for correctness.849 try testing.expectError(error.Timeout, Futex.timedWait(&value, 0, 0));
511 std.time.sleep(16 * std.time.ns_per_ms);850 try testing.expectError(error.Timeout, Futex.timedWait(&value, 0, std.time.ns_per_ms));
512 try ctx.runSender();
513851
514 const notified = ctx.notified.load(.Monotonic);852 // Try wakes
515 try testing.expectEqual(notified, ctx.threads.len);853 Futex.wake(&value, 0);
854 Futex.wake(&value, 1);
855 Futex.wake(&value, std.math.maxInt(u32));
516}856}
517857
518test "Futex - Chain" {858test "Futex - signaling" {
519 if (single_threaded) {859 // This test requires spawning threads
860 if (builtin.single_threaded) {
520 return error.SkipZigTest;861 return error.SkipZigTest;
521 }862 }
522863
523 const Signal = struct {864 const num_threads = 4;
865 const num_iterations = 4;
866
867 const Paddle = struct {
524 value: Atomic(u32) = Atomic(u32).init(0),868 value: Atomic(u32) = Atomic(u32).init(0),
869 current: u32 = 0,
525870
526 fn wait(self: *@This()) void {871 fn hit(self: *@This()) void {
527 while (true) {872 _ = self.value.fetchAdd(1, .Release);
528 const value = self.value.load(.Acquire);873 Futex.wake(&self.value, 1);
529 if (value == 1) break;
530 assert(value == 0);
531 Futex.wait(&self.value, 0, null) catch unreachable;
532 }
533 }874 }
534875
535 fn notify(self: *@This()) void {876 fn run(self: *@This(), hit_to: *@This()) !void {
536 assert(self.value.load(.Unordered) == 0);877 while (self.current < num_iterations) {
537 self.value.store(1, .Release);878 // Wait for the value to change from hit()
538 Futex.wake(&self.value, 1);879 var new_value: u32 = undefined;
880 while (true) {
881 new_value = self.value.load(.Acquire);
882 if (new_value != self.current) break;
883 Futex.wait(&self.value, self.current);
884 }
885
886 // change the internal "current" value
887 try testing.expectEqual(new_value, self.current + 1);
888 self.current = new_value;
889
890 // hit the next paddle
891 hit_to.hit();
892 }
539 }893 }
540 };894 };
541895
542 const Context = struct {896 var paddles = [_]Paddle{.{}} ** num_threads;
543 completed: Signal = .{},897 var threads = [_]std.Thread{undefined} ** num_threads;
544 threads: [4]struct {898
545 thread: std.Thread,899 // Create a circle of paddles which hit each other
546 signal: Signal,900 for (threads) |*t, i| {
547 } = undefined,901 const paddle = &paddles[i];
902 const hit_to = &paddles[(i + 1) % paddles.len];
903 t.* = try std.Thread.spawn(.{}, Paddle.run, .{ paddle, hit_to });
904 }
905
906 // Hit the first paddle and wait for them all to complete by hitting each other for num_iterations.
907 paddles[0].hit();
908 for (threads) |t| t.join();
909 for (paddles) |p| try testing.expectEqual(p.current, num_iterations);
910}
548911
549 fn run(self: *@This(), index: usize) void {912test "Futex - broadcasting" {
550 const this_signal = &self.threads[index].signal;913 // This test requires spawning threads
914 if (builtin.single_threaded) {
915 return error.SkipZigTest;
916 }
551917
552 var next_signal = &self.completed;918 const num_threads = 4;
553 if (index + 1 < self.threads.len) {919 const num_iterations = 4;
554 next_signal = &self.threads[index + 1].signal;920
921 const Barrier = struct {
922 count: Atomic(u32) = Atomic(u32).init(num_threads),
923 futex: Atomic(u32) = Atomic(u32).init(0),
924
925 fn wait(self: *@This()) !void {
926 // Decrement the counter.
927 // Release ensures stuff before this barrier.wait() happens before the last one.
928 const count = self.count.fetchSub(1, .Release);
929 try testing.expect(count <= num_threads);
930 try testing.expect(count > 0);
931
932 // First counter to reach zero wakes all other threads.
933 // Acquire for the last counter ensures stuff before previous barrier.wait()s happened before it.
934 // Release on futex update ensures stuff before all barrier.wait()'s happens before they all return.
935 if (count - 1 == 0) {
936 _ = self.count.load(.Acquire); // TODO: could be fence(Acquire) if not for TSAN
937 self.futex.store(1, .Release);
938 Futex.wake(&self.futex, num_threads - 1);
939 return;
555 }940 }
556941
557 this_signal.wait();942 // Other threads wait until last counter wakes them up.
558 next_signal.notify();943 // Acquire on futex synchronizes with last barrier count to ensure stuff before all barrier.wait()'s happen before us.
944 while (self.futex.load(.Acquire) == 0) {
945 Futex.wait(&self.futex, 0);
946 }
559 }947 }
560 };948 };
561949
562 var ctx = Context{};950 const Broadcast = struct {
563 for (ctx.threads) |*entry, index| {951 barriers: [num_iterations]Barrier = [_]Barrier{.{}} ** num_iterations,
564 entry.signal = .{};952 threads: [num_threads]std.Thread = undefined,
565 entry.thread = try std.Thread.spawn(.{}, Context.run, .{ &ctx, index });
566 }
567953
568 ctx.threads[0].signal.notify();954 fn run(self: *@This()) !void {
569 ctx.completed.wait();955 for (self.barriers) |*barrier| {
956 try barrier.wait();
957 }
958 }
959 };
570960
571 for (ctx.threads) |entry| {961 var broadcast = Broadcast{};
572 entry.thread.join();962 for (broadcast.threads) |*t| t.* = try std.Thread.spawn(.{}, Broadcast.run, .{&broadcast});
573 }963 for (broadcast.threads) |t| t.join();
574}964}
lib/std/atomic.zig+41-3
...@@ -1,5 +1,5 @@...@@ -1,5 +1,5 @@
1const std = @import("std.zig");1const std = @import("std.zig");
2const target = @import("builtin").target;2const builtin = @import("builtin");
33
4pub const Ordering = std.builtin.AtomicOrder;4pub const Ordering = std.builtin.AtomicOrder;
55
...@@ -40,7 +40,7 @@ test "fence/compilerFence" {...@@ -40,7 +40,7 @@ test "fence/compilerFence" {
4040
41/// Signals to the processor that the caller is inside a busy-wait spin-loop.41/// Signals to the processor that the caller is inside a busy-wait spin-loop.
42pub inline fn spinLoopHint() void {42pub inline fn spinLoopHint() void {
43 switch (target.cpu.arch) {43 switch (builtin.target.cpu.arch) {
44 // No-op instruction that can hint to save (or share with a hardware-thread)44 // No-op instruction that can hint to save (or share with a hardware-thread)
45 // pipelining/power resources45 // pipelining/power resources
46 // https://software.intel.com/content/www/us/en/develop/articles/benefitting-power-and-performance-sleep-loops.html46 // https://software.intel.com/content/www/us/en/develop/articles/benefitting-power-and-performance-sleep-loops.html
...@@ -59,7 +59,7 @@ pub inline fn spinLoopHint() void {...@@ -59,7 +59,7 @@ pub inline fn spinLoopHint() void {
59 // `yield` was introduced in v6k but is also available on v6m.59 // `yield` was introduced in v6k but is also available on v6m.
60 // https://www.keil.com/support/man/docs/armasm/armasm_dom1361289926796.htm60 // https://www.keil.com/support/man/docs/armasm/armasm_dom1361289926796.htm
61 .arm, .armeb, .thumb, .thumbeb => {61 .arm, .armeb, .thumb, .thumbeb => {
62 const can_yield = comptime std.Target.arm.featureSetHasAny(target.cpu.features, .{62 const can_yield = comptime std.Target.arm.featureSetHasAny(builtin.target.cpu.features, .{
63 .has_v6k, .has_v6m,63 .has_v6k, .has_v6m,
64 });64 });
65 if (can_yield) {65 if (can_yield) {
...@@ -80,3 +80,41 @@ test "spinLoopHint" {...@@ -80,3 +80,41 @@ test "spinLoopHint" {
80 spinLoopHint();80 spinLoopHint();
81 }81 }
82}82}
83
84/// The estimated size of the CPU's cache line when atomically updating memory.
85/// Add this much padding or align to this boundary to avoid atomically-updated
86/// memory from forcing cache invalidations on near, but non-atomic, memory.
87///
88// https://en.wikipedia.org/wiki/False_sharing
89// https://github.com/golang/go/search?q=CacheLinePadSize
90pub const cache_line = switch (builtin.cpu.arch) {
91 // x86_64: Starting from Intel's Sandy Bridge, the spatial prefetcher pulls in pairs of 64-byte cache lines at a time.
92 // - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf
93 // - https://github.com/facebook/folly/blob/1b5288e6eea6df074758f877c849b6e73bbb9fbb/folly/lang/Align.h#L107
94 //
95 // aarch64: Some big.LITTLE ARM archs have "big" cores with 128-byte cache lines:
96 // - https://www.mono-project.com/news/2016/09/12/arm64-icache/
97 // - https://cpufun.substack.com/p/more-m1-fun-hardware-information
98 //
99 // powerpc64: PPC has 128-byte cache lines
100 // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_ppc64x.go#L9
101 .x86_64, .aarch64, .powerpc64 => 128,
102
103 // These platforms reportedly have 32-byte cache lines
104 // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_arm.go#L7
105 // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips.go#L7
106 // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mipsle.go#L7
107 // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips64x.go#L9
108 // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_riscv64.go#L7
109 .arm, .mips, .mips64, .riscv64 => 32,
110
111 // This platform reportedly has 256-byte cache lines
112 // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_s390x.go#L7
113 .s390x => 256,
114
115 // Other x86 and WASM platforms have 64-byte cache lines.
116 // The rest of the architectures are assumed to be similar.
117 // - https://github.com/golang/go/blob/dda2991c2ea0c5914714469c4defc2562a907230/src/internal/cpu/cpu_x86.go#L9
118 // - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_wasm.go#L7
119 else => 64,
120};
lib/std/c/dragonfly.zig+3
...@@ -36,6 +36,9 @@ pub const sem_t = ?*opaque {};...@@ -36,6 +36,9 @@ pub const sem_t = ?*opaque {};
36pub extern "c" fn pthread_setname_np(thread: std.c.pthread_t, name: [*:0]const u8) E;36pub extern "c" fn pthread_setname_np(thread: std.c.pthread_t, name: [*:0]const u8) E;
37pub extern "c" fn pthread_getname_np(thread: std.c.pthread_t, name: [*:0]u8, len: usize) E;37pub extern "c" fn pthread_getname_np(thread: std.c.pthread_t, name: [*:0]u8, len: usize) E;
3838
39pub extern "c" fn umtx_sleep(ptr: *const volatile c_int, value: c_int, timeout: c_int) c_int;
40pub extern "c" fn umtx_wakeup(ptr: *const volatile c_int, count: c_int) c_int;
41
39// See:42// See:
40// - https://gitweb.dragonflybsd.org/dragonfly.git/blob/HEAD:/include/unistd.h43// - https://gitweb.dragonflybsd.org/dragonfly.git/blob/HEAD:/include/unistd.h
41// - https://gitweb.dragonflybsd.org/dragonfly.git/blob/HEAD:/sys/sys/types.h44// - https://gitweb.dragonflybsd.org/dragonfly.git/blob/HEAD:/sys/sys/types.h
lib/std/c/freebsd.zig+40
...@@ -62,6 +62,46 @@ pub const sem_t = extern struct {...@@ -62,6 +62,46 @@ pub const sem_t = extern struct {
62 _padding: u32,62 _padding: u32,
63};63};
6464
65// https://github.com/freebsd/freebsd-src/blob/main/sys/sys/umtx.h
66pub const UMTX_OP = enum(c_int) {
67 LOCK = 0,
68 UNLOCK = 1,
69 WAIT = 2,
70 WAKE = 3,
71 MUTEX_TRYLOCK = 4,
72 MUTEX_LOCK = 5,
73 MUTEX_UNLOCK = 6,
74 SET_CEILING = 7,
75 CV_WAIT = 8,
76 CV_SIGNAL = 9,
77 CV_BROADCAST = 10,
78 WAIT_UINT = 11,
79 RW_RDLOCK = 12,
80 RW_WRLOCK = 13,
81 RW_UNLOCK = 14,
82 WAIT_UINT_PRIVATE = 15,
83 WAKE_PRIVATE = 16,
84 MUTEX_WAIT = 17,
85 MUTEX_WAKE = 18, // deprecated
86 SEM_WAIT = 19, // deprecated
87 SEM_WAKE = 20, // deprecated
88 NWAKE_PRIVATE = 31,
89 MUTEX_WAKE2 = 22,
90 SEM2_WAIT = 23,
91 SEM2_WAKE = 24,
92 SHM = 25,
93 ROBUST_LISTS = 26,
94};
95
96pub const UMTX_ABSTIME = 0x01;
97pub const _umtx_time = extern struct {
98 _timeout: timespec,
99 _flags: u32,
100 _clockid: u32,
101};
102
103pub extern "c" fn _umtx_op(obj: usize, op: c_int, val: c_ulong, uaddr: usize, uaddr2: usize) c_int;
104
65pub const EAI = enum(c_int) {105pub const EAI = enum(c_int) {
66 /// address family for hostname not supported106 /// address family for hostname not supported
67 ADDRFAMILY = 1,107 ADDRFAMILY = 1,
lib/std/c/openbsd.zig+7
...@@ -45,6 +45,13 @@ pub extern "c" fn unveil(path: ?[*:0]const u8, permissions: ?[*:0]const u8) c_in...@@ -45,6 +45,13 @@ pub extern "c" fn unveil(path: ?[*:0]const u8, permissions: ?[*:0]const u8) c_in
45pub extern "c" fn pthread_set_name_np(thread: std.c.pthread_t, name: [*:0]const u8) void;45pub extern "c" fn pthread_set_name_np(thread: std.c.pthread_t, name: [*:0]const u8) void;
46pub extern "c" fn pthread_get_name_np(thread: std.c.pthread_t, name: [*:0]u8, len: usize) void;46pub extern "c" fn pthread_get_name_np(thread: std.c.pthread_t, name: [*:0]u8, len: usize) void;
4747
48// https://github.com/openbsd/src/blob/2207c4325726fdc5c4bcd0011af0fdf7d3dab137/sys/sys/futex.h
49pub const FUTEX_WAIT = 1;
50pub const FUTEX_WAKE = 2;
51pub const FUTEX_REQUEUE = 3;
52pub const FUTEX_PRIVATE_FLAG = 128;
53pub extern "c" fn futex(uaddr: ?*const volatile u32, op: c_int, val: c_int, timeout: ?*const timespec, uaddr2: ?*const volatile u32) c_int;
54
48pub const login_cap_t = extern struct {55pub const login_cap_t = extern struct {
49 lc_class: ?[*:0]const u8,56 lc_class: ?[*:0]const u8,
50 lc_cap: ?[*:0]const u8,57 lc_cap: ?[*:0]const u8,