Commit 963ac60918b39cafdda3cb99eff4cd9d20edd839 | zig

author	protty <45520026+kprotty@users.noreply.github.com> 2022-04-23 19:35:56-05:00
committer	GitHub <noreply@github.com> 2022-04-23 19:35:56-05:00
log	963ac60918b39cafdda3cb99eff4cd9d20edd839
tree	8b480e6c1c01c11758f47b1126c53e75a339d1a1
parent	daef82d06fd6b30d2cab7f5a6723cf2e3c7b48c6
signature	Signed by PGP key 4AEE18F83AFDEB23

std.Thread: Mutex and Condition improvements (#11497)

* Thread: minor cleanups

* Thread: rewrite Mutex

* Thread: introduce Futex.Deadline

* Thread: Condition rewrite + cleanup

* Mutex: optimize lock fast path

* Condition: more docs

* Thread: more mutex + condition docs

* Thread: remove broken Condition test

* Thread: zig fmt

* address review comments + fix Thread.DummyMutex in GPA

* Atomic: disable bitRmw x86 inline asm for stage2

* GPA: typo mutex_init

* Thread: remove noalias on stuff

* Thread: comment typos + clarifications

7 files changed, 810 insertions(+), 640 deletions(-)

lib
- std
  - Thread.zig
  - Thread
    Condition.zig
    Futex.zig
    Mutex.zig
  - atomic
    Atomic.zig
  - heap
    general_purpose_allocator.zig
  - os
    windows.zig

lib/std/Thread.zig+1-26

...	...	@@ -459,9 +459,8 @@ const UnsupportedImpl = struct {
459	459	}
460	460
461	461	fn unsupported(unusued: anytype) noreturn {
462		@compileLog("Unsupported operating system", target.os.tag);
463	462	_ = unusued;
464		unreachable;
	463	@compileError("Unsupported operating system " ++ @tagName(target.os.tag));
465	464	}
466	465	};
467	466
...	...	@@ -1188,27 +1187,3 @@ test "Thread.detach" {
1188	1187	event.wait();
1189	1188	try std.testing.expectEqual(value, 1);
1190	1189	}
1191
1192		fn testWaitForSignal(mutex: Mutex, cond: Condition) void {
1193		mutex.lock();
1194		defer mutex.unlock();
1195		cond.signal();
1196		cond.wait(mutex);
1197		}
1198
1199		test "Condition.signal" {
1200		if (builtin.single_threaded) return error.SkipZigTest;
1201
1202		var mutex = Mutex{};
1203		var cond = Condition{};
1204
1205		var thread: Thread = undefined;
1206		{
1207		mutex.lock();
1208		defer mutex.unlock();
1209		thread = try Thread.spawn(.{}, testWaitForSignal, .{ &mutex, &cond });
1210		cond.wait(&mutex);
1211		cond.signal();
1212		}
1213		thread.join();
1214		}

lib/std/Thread/Condition.zig+435-308

...	...	@@ -1,411 +1,538 @@
1		//! A condition provides a way for a kernel thread to block until it is signaled
2		//! to wake up. Spurious wakeups are possible.
3		//! This API supports static initialization and does not require deinitialization.
4
5		impl: Impl = .{},
	1	//! Condition variables are used with a Mutex to efficiently wait for an arbitrary condition to occur.
	2	//! It does this by atomically unlocking the mutex, blocking the thread until notified, and finally re-locking the mutex.
	3	//! Condition can be statically initialized and is at most `@sizeOf(u64)` large.
	4	//!
	5	//! Example:
	6	//! ```
	7	//! var m = Mutex{};
	8	//! var c = Condition{};
	9	//! var predicate = false;
	10	//!
	11	//! fn consumer() void {
	12	//! m.lock();
	13	//! defer m.unlock();
	14	//!
	15	//! while (!predicate) {
	16	//! c.wait(&mutex);
	17	//! }
	18	//! }
	19	//!
	20	//! fn producer() void {
	21	//! m.lock();
	22	//! defer m.unlock();
	23	//!
	24	//! predicate = true;
	25	//! c.signal();
	26	//! }
	27	//!
	28	//! const thread = try std.Thread.spawn(.{}, producer, .{});
	29	//! consumer();
	30	//! thread.join();
	31	//! ```
	32	//!
	33	//! Note that condition variables can only reliably unblock threads that are sequenced before them using the same Mutex.
	34	//! This means that the following is allowed to deadlock:
	35	//! ```
	36	//! thread-1: mutex.lock()
	37	//! thread-1: condition.wait(&mutex)
	38	//!
	39	//! thread-2: // mutex.lock() (without this, the following signal may not see the waiting thread-1)
	40	//! thread-2: // mutex.unlock() (this is optional for correctness once locked above, as signal can be called without holding the mutex)
	41	//! thread-2: condition.signal()
	42	//! ```
6	43
7	44	const std = @import("../std.zig");
8	45	const builtin = @import("builtin");
9	46	const Condition = @This();
10		const windows = std.os.windows;
11		const linux = std.os.linux;
12	47	const Mutex = std.Thread.Mutex;
	48
	49	const os = std.os;
13	50	const assert = std.debug.assert;
14	51	const testing = std.testing;
	52	const Atomic = std.atomic.Atomic;
	53	const Futex = std.Thread.Futex;
	54
	55	impl: Impl = .{},
15	56
16		pub fn wait(cond: Condition, mutex: Mutex) void {
17		cond.impl.wait(mutex);
	57	/// Atomically releases the Mutex, blocks the caller thread, then re-acquires the Mutex on return.
	58	/// "Atomically" here refers to accesses done on the Condition after acquiring the Mutex.
	59	///
	60	/// The Mutex must be locked by the caller's thread when this function is called.
	61	/// A Mutex can have multiple Conditions waiting with it concurrently, but not the opposite.
	62	/// It is undefined behavior for multiple threads to wait ith different mutexes using the same Condition concurrently.
	63	/// Once threads have finished waiting with one Mutex, the Condition can be used to wait with another Mutex.
	64	///
	65	/// A blocking call to wait() is unblocked from one of the following conditions:
	66	/// - a spurious ("at random") wake up occurs
	67	/// - a future call to `signal()` or `broadcast()` which has acquired the Mutex and is sequenced after this `wait()`.
	68	///
	69	/// Given wait() can be interrupted spuriously, the blocking condition should be checked continuously
	70	/// irrespective of any notifications from `signal()` or `broadcast()`.
	71	pub fn wait(self: Condition, mutex: Mutex) void {
	72	self.impl.wait(mutex, null) catch \|err\| switch (err) {
	73	error.Timeout => unreachable, // no timeout provided so we shouldn't have timed-out
	74	};
18	75	}
19	76
20		pub fn timedWait(cond: Condition, mutex: Mutex, timeout_ns: u64) error{TimedOut}!void {
21		try cond.impl.timedWait(mutex, timeout_ns);
	77	/// Atomically releases the Mutex, blocks the caller thread, then re-acquires the Mutex on return.
	78	/// "Atomically" here refers to accesses done on the Condition after acquiring the Mutex.
	79	///
	80	/// The Mutex must be locked by the caller's thread when this function is called.
	81	/// A Mutex can have multiple Conditions waiting with it concurrently, but not the opposite.
	82	/// It is undefined behavior for multiple threads to wait ith different mutexes using the same Condition concurrently.
	83	/// Once threads have finished waiting with one Mutex, the Condition can be used to wait with another Mutex.
	84	///
	85	/// A blocking call to `timedWait()` is unblocked from one of the following conditions:
	86	/// - a spurious ("at random") wake occurs
	87	/// - the caller was blocked for around `timeout_ns` nanoseconds, in which `error.Timeout` is returned.
	88	/// - a future call to `signal()` or `broadcast()` which has acquired the Mutex and is sequenced after this `timedWait()`.
	89	///
	90	/// Given `timedWait()` can be interrupted spuriously, the blocking condition should be checked continuously
	91	/// irrespective of any notifications from `signal()` or `broadcast()`.
	92	pub fn timedWait(self: Condition, mutex: Mutex, timeout_ns: u64) error{Timeout}!void {
	93	return self.impl.wait(mutex, timeout_ns);
22	94	}
23	95
24		pub fn signal(cond: *Condition) void {
25		cond.impl.signal();
	96	/// Unblocks at least one thread blocked in a call to `wait()` or `timedWait()` with a given Mutex.
	97	/// The blocked thread must be sequenced before this call with respect to acquiring the same Mutex in order to be observable for unblocking.
	98	/// `signal()` can be called with or without the relevant Mutex being acquired and have no "effect" if there's no observable blocked threads.
	99	pub fn signal(self: *Condition) void {
	100	self.impl.wake(.one);
26	101	}
27	102
28		pub fn broadcast(cond: *Condition) void {
29		cond.impl.broadcast();
	103	/// Unblocks all threads currently blocked in a call to `wait()` or `timedWait()` with a given Mutex.
	104	/// The blocked threads must be sequenced before this call with respect to acquiring the same Mutex in order to be observable for unblocking.
	105	/// `broadcast()` can be called with or without the relevant Mutex being acquired and have no "effect" if there's no observable blocked threads.
	106	pub fn broadcast(self: *Condition) void {
	107	self.impl.wake(.all);
30	108	}
31	109
32	110	const Impl = if (builtin.single_threaded)
33		SingleThreadedCondition
	111	SingleThreadedImpl
34	112	else if (builtin.os.tag == .windows)
35		WindowsCondition
36		else if (std.Thread.use_pthreads)
37		PthreadCondition
	113	WindowsImpl
38	114	else
39		AtomicCondition;
	115	FutexImpl;
40	116
41		pub const SingleThreadedCondition = struct {
42		pub fn wait(cond: SingleThreadedCondition, mutex: Mutex) void {
43		_ = cond;
44		_ = mutex;
45		unreachable; // deadlock detected
46		}
	117	const Notify = enum {
	118	one, // wake up only one thread
	119	all, // wake up all threads
	120	};
47	121
48		pub fn timedWait(cond: SingleThreadedCondition, mutex: Mutex, timeout_ns: u64) error{TimedOut}!void {
49		_ = cond;
	122	const SingleThreadedImpl = struct {
	123	fn wait(self: Impl, mutex: Mutex, timeout: ?u64) error{Timeout}!void {
	124	_ = self;
50	125	_ = mutex;
51		_ = timeout_ns;
52		std.time.sleep(timeout_ns);
53		return error.TimedOut;
54		}
55	126
56		pub fn signal(cond: *SingleThreadedCondition) void {
57		_ = cond;
	127	// There are no other threads to wake us up.
	128	// So if we wait without a timeout we would never wake up.
	129	const timeout_ns = timeout orelse {
	130	unreachable; // deadlock detected
	131	};
	132
	133	std.time.sleep(timeout_ns);
	134	return error.Timeout;
58	135	}
59	136
60		pub fn broadcast(cond: *SingleThreadedCondition) void {
61		_ = cond;
	137	fn wake(self: *Impl, comptime notify: Notify) void {
	138	// There are no other threads to wake up.
	139	_ = self;
	140	_ = notify;
62	141	}
63	142	};
64	143
65		pub const WindowsCondition = struct {
66		cond: windows.CONDITION_VARIABLE = windows.CONDITION_VARIABLE_INIT,
	144	const WindowsImpl = struct {
	145	condition: os.windows.CONDITION_VARIABLE = .{},
67	146
68		pub fn wait(cond: WindowsCondition, mutex: Mutex) void {
69		const rc = windows.kernel32.SleepConditionVariableSRW(
70		&cond.cond,
71		&mutex.impl.srwlock,
72		windows.INFINITE,
73		@as(windows.ULONG, 0),
74		);
75		assert(rc != windows.FALSE);
76		}
	147	fn wait(self: Impl, mutex: Mutex, timeout: ?u64) error{Timeout}!void {
	148	var timeout_overflowed = false;
	149	var timeout_ms: os.windows.DWORD = os.windows.INFINITE;
77	150
78		pub fn timedWait(cond: WindowsCondition, mutex: Mutex, timeout_ns: u64) error{TimedOut}!void {
79		var timeout_checked = std.math.cast(windows.DWORD, timeout_ns / std.time.ns_per_ms) catch overflow: {
80		break :overflow std.math.maxInt(windows.DWORD);
81		};
	151	if (timeout) \|timeout_ns\| {
	152	// Round the nanoseconds to the nearest millisecond,
	153	// then saturating cast it to windows DWORD for use in kernel32 call.
	154	const ms = (timeout_ns +\| (std.time.ns_per_ms / 2)) / std.time.ns_per_ms;
	155	timeout_ms = std.math.cast(os.windows.DWORD, ms) catch std.math.maxInt(os.windows.DWORD);
82	156
83		// Handle the case where timeout is INFINITE, otherwise SleepConditionVariableSRW's time-out never elapses
84		const timeout_overflowed = timeout_checked == windows.INFINITE;
85		timeout_checked -= @boolToInt(timeout_overflowed);
	157	// Track if the timeout overflowed into INFINITE and make sure not to wait forever.
	158	if (timeout_ms == os.windows.INFINITE) {
	159	timeout_overflowed = true;
	160	timeout_ms -= 1;
	161	}
	162	}
86	163
87		const rc = windows.kernel32.SleepConditionVariableSRW(
88		&cond.cond,
	164	const rc = os.windows.kernel32.SleepConditionVariableSRW(
	165	&self.condition,
89	166	&mutex.impl.srwlock,
90		timeout_checked,
91		@as(windows.ULONG, 0),
	167	timeout_ms,
	168	0, // the srwlock was assumed to acquired in exclusive mode not shared
92	169	);
93		if (rc == windows.FALSE and windows.kernel32.GetLastError() == windows.Win32Error.TIMEOUT) return error.TimedOut;
94		assert(rc != windows.FALSE);
95		}
96	170
97		pub fn signal(cond: *WindowsCondition) void {
98		windows.kernel32.WakeConditionVariable(&cond.cond);
	171	// Return error.Timeout if we know the timeout elapsed correctly.
	172	if (rc == os.windows.FALSE) {
	173	assert(os.windows.kernel32.GetLastError() == .TIMEOUT);
	174	if (!timeout_overflowed) return error.Timeout;
	175	}
99	176	}
100	177
101		pub fn broadcast(cond: *WindowsCondition) void {
102		windows.kernel32.WakeAllConditionVariable(&cond.cond);
	178	fn wake(self: *Impl, comptime notify: Notify) void {
	179	switch (notify) {
	180	.one => os.windows.kernel32.WakeConditionVariable(&self.condition),
	181	.all => os.windows.kernel32.WakeAllConditionVariable(&self.condition),
	182	}
103	183	}
104	184	};
105	185
106		pub const PthreadCondition = struct {
107		cond: std.c.pthread_cond_t = .{},
	186	const FutexImpl = struct {
	187	state: Atomic(u32) = Atomic(u32).init(0),
	188	epoch: Atomic(u32) = Atomic(u32).init(0),
108	189
109		pub fn wait(cond: PthreadCondition, mutex: Mutex) void {
110		const rc = std.c.pthread_cond_wait(&cond.cond, &mutex.impl.pthread_mutex);
111		assert(rc == .SUCCESS);
112		}
	190	const one_waiter = 1;
	191	const waiter_mask = 0xffff;
113	192
114		pub fn timedWait(cond: PthreadCondition, mutex: Mutex, timeout_ns: u64) error{TimedOut}!void {
115		var ts: std.os.timespec = undefined;
116		std.os.clock_gettime(std.os.CLOCK.REALTIME, &ts) catch unreachable;
117		ts.tv_sec += @intCast(@TypeOf(ts.tv_sec), timeout_ns / std.time.ns_per_s);
118		ts.tv_nsec += @intCast(@TypeOf(ts.tv_nsec), timeout_ns % std.time.ns_per_s);
119		if (ts.tv_nsec >= std.time.ns_per_s) {
120		ts.tv_sec += 1;
121		ts.tv_nsec -= std.time.ns_per_s;
122		}
	193	const one_signal = 1 << 16;
	194	const signal_mask = 0xffff << 16;
123	195
124		const rc = std.c.pthread_cond_timedwait(&cond.cond, &mutex.impl.pthread_mutex, &ts);
125		return switch (rc) {
126		.SUCCESS => {},
127		.TIMEDOUT => error.TimedOut,
128		else => unreachable,
129		};
130		}
	196	fn wait(self: Impl, mutex: Mutex, timeout: ?u64) error{Timeout}!void {
	197	// Register that we're waiting on the state by incrementing the wait count.
	198	// This assumes that there can be at most ((1<<16)-1) or 65,355 threads concurrently waiting on the same Condvar.
	199	// If this is hit in practice, then this condvar not working is the least of your concerns.
	200	var state = self.state.fetchAdd(one_waiter, .Monotonic);
	201	assert(state & waiter_mask != waiter_mask);
	202	state += one_waiter;
131	203
132		pub fn signal(cond: *PthreadCondition) void {
133		const rc = std.c.pthread_cond_signal(&cond.cond);
134		assert(rc == .SUCCESS);
135		}
	204	// Temporarily release the mutex in order to block on the condition variable.
	205	mutex.unlock();
	206	defer mutex.lock();
136	207
137		pub fn broadcast(cond: *PthreadCondition) void {
138		const rc = std.c.pthread_cond_broadcast(&cond.cond);
139		assert(rc == .SUCCESS);
140		}
141		};
	208	var futex_deadline = Futex.Deadline.init(timeout);
	209	while (true) {
	210	// Try to wake up by consuming a signal and decremented the waiter we added previously.
	211	// Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return.
	212	while (state & signal_mask != 0) {
	213	const new_state = state - one_waiter - one_signal;
	214	state = self.state.tryCompareAndSwap(state, new_state, .Acquire, .Monotonic) orelse return;
	215	}
142	216
143		pub const AtomicCondition = struct {
144		pending: bool = false,
145		queue_mutex: Mutex = .{},
146		queue_list: QueueList = .{},
147
148		pub const QueueList = std.SinglyLinkedList(QueueItem);
149
150		pub const QueueItem = struct {
151		futex: i32 = 0,
152		dequeued: bool = false,
153
154		fn wait(cond: *@This()) void {
155		while (@atomicLoad(i32, &cond.futex, .Acquire) == 0) {
156		switch (builtin.os.tag) {
157		.linux => {
158		switch (linux.getErrno(linux.futex_wait(
159		&cond.futex,
160		linux.FUTEX.PRIVATE_FLAG \| linux.FUTEX.WAIT,
161		0,
162		null,
163		))) {
164		.SUCCESS => {},
165		.INTR => {},
166		.AGAIN => {},
167		else => unreachable,
168		}
169		},
170		else => std.atomic.spinLoopHint(),
171		}
	217	// Observe the epoch, then check the state again to see if we should wake up.
	218	// The epoch must be observed before we check the state or we could potentially miss a wake() and deadlock:
	219	//
	220	// - T1: s = LOAD(&state)
	221	// - T2: UPDATE(&s, signal)
	222	// - T2: UPDATE(&epoch, 1) + FUTEX_WAKE(&epoch)
	223	// - T1: e = LOAD(&epoch) (was reordered after the state load)
	224	// - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed the state update + the epoch change)
	225	//
	226	// Acquire barrier to ensure the epoch load happens before the state load.
	227	const epoch = self.epoch.load(.Acquire);
	228	state = self.state.load(.Monotonic);
	229	if (state & signal_mask != 0) {
	230	continue;
172	231	}
173		}
174	232
175		pub fn timedWait(cond: *@This(), timeout_ns: u64) error{TimedOut}!void {
176		const start_time = std.time.nanoTimestamp();
177		while (@atomicLoad(i32, &cond.futex, .Acquire) == 0) {
178		switch (builtin.os.tag) {
179		.linux => {
180		var ts: std.os.timespec = undefined;
181		ts.tv_sec = @intCast(@TypeOf(ts.tv_sec), timeout_ns / std.time.ns_per_s);
182		ts.tv_nsec = @intCast(@TypeOf(ts.tv_nsec), timeout_ns % std.time.ns_per_s);
183		switch (linux.getErrno(linux.futex_wait(
184		&cond.futex,
185		linux.FUTEX.PRIVATE_FLAG \| linux.FUTEX.WAIT,
186		0,
187		&ts,
188		))) {
189		.SUCCESS => {},
190		.INTR => {},
191		.AGAIN => {},
192		.TIMEDOUT => return error.TimedOut,
193		.INVAL => {}, // possibly timeout overflow
194		.FAULT => unreachable,
195		else => unreachable,
196		}
197		},
198		else => {
199		if (std.time.nanoTimestamp() - start_time >= timeout_ns) {
200		return error.TimedOut;
	233	futex_deadline.wait(&self.epoch, epoch) catch \|err\| switch (err) {
	234	// On timeout, we must decrement the waiter we added above.
	235	error.Timeout => {
	236	while (true) {
	237	// If there's a signal when we're timing out, consume it and report being woken up instead.
	238	// Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return.
	239	while (state & signal_mask != 0) {
	240	const new_state = state - one_waiter - one_signal;
	241	state = self.state.tryCompareAndSwap(state, new_state, .Acquire, .Monotonic) orelse return;
201	242	}
202		std.atomic.spinLoopHint();
203		},
204		}
205		}
206		}
207	243
208		fn notify(cond: *@This()) void {
209		@atomicStore(i32, &cond.futex, 1, .Release);
210
211		switch (builtin.os.tag) {
212		.linux => {
213		switch (linux.getErrno(linux.futex_wake(
214		&cond.futex,
215		linux.FUTEX.PRIVATE_FLAG \| linux.FUTEX.WAKE,
216		1,
217		))) {
218		.SUCCESS => {},
219		.FAULT => {},
220		else => unreachable,
	244	// Remove the waiter we added and officially return timed out.
	245	const new_state = state - one_waiter;
	246	state = self.state.tryCompareAndSwap(state, new_state, .Monotonic, .Monotonic) orelse return err;
221	247	}
222	248	},
223		else => {},
	249	};
	250	}
	251	}
	252
	253	fn wake(self: *Impl, comptime notify: Notify) void {
	254	var state = self.state.load(.Monotonic);
	255	while (true) {
	256	const waiters = (state & waiter_mask) / one_waiter;
	257	const signals = (state & signal_mask) / one_signal;
	258
	259	// Reserves which waiters to wake up by incrementing the signals count.
	260	// Therefor, the signals count is always less than or equal to the waiters count.
	261	// We don't need to Futex.wake if there's nothing to wake up or if other wake() threads have reserved to wake up the current waiters.
	262	const wakeable = waiters - signals;
	263	if (wakeable == 0) {
	264	return;
224	265	}
	266
	267	const to_wake = switch (notify) {
	268	.one => 1,
	269	.all => wakeable,
	270	};
	271
	272	// Reserve the amount of waiters to wake by incrementing the signals count.
	273	// Release barrier ensures code before the wake() happens before the signal it posted and consumed by the wait() threads.
	274	const new_state = state + (one_signal * to_wake);
	275	state = self.state.tryCompareAndSwap(state, new_state, .Release, .Monotonic) orelse {
	276	// Wake up the waiting threads we reserved above by changing the epoch value.
	277	// NOTE: a waiting thread could miss a wake up if exactly ((1<<32)-1) wake()s happen between it observing the epoch and sleeping on it.
	278	// This is very unlikely due to how many precise amount of Futex.wake() calls that would be between the waiting thread's potential preemption.
	279	//
	280	// Release barrier ensures the signal being added to the state happens before the epoch is changed.
	281	// If not, the waiting thread could potentially deadlock from missing both the state and epoch change:
	282	//
	283	// - T2: UPDATE(&epoch, 1) (reordered before the state change)
	284	// - T1: e = LOAD(&epoch)
	285	// - T1: s = LOAD(&state)
	286	// - T2: UPDATE(&state, signal) + FUTEX_WAKE(&epoch)
	287	// - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed both epoch change and state change)
	288	_ = self.epoch.fetchAdd(1, .Release);
	289	Futex.wake(&self.epoch, to_wake);
	290	return;
	291	};
225	292	}
226		};
	293	}
	294	};
227	295
228		pub fn wait(cond: AtomicCondition, mutex: Mutex) void {
229		var waiter = QueueList.Node{ .data = .{} };
	296	test "Condition - smoke test" {
	297	var mutex = Mutex{};
	298	var cond = Condition{};
230	299
231		{
232		cond.queue_mutex.lock();
233		defer cond.queue_mutex.unlock();
	300	// Try to wake outside the mutex
	301	defer cond.signal();
	302	defer cond.broadcast();
234	303
235		cond.queue_list.prepend(&waiter);
236		@atomicStore(bool, &cond.pending, true, .SeqCst);
237		}
	304	mutex.lock();
	305	defer mutex.unlock();
238	306
239		mutex.unlock();
240		waiter.data.wait();
241		mutex.lock();
242		}
	307	// Try to wait with a timeout (should not deadlock)
	308	try testing.expectError(error.Timeout, cond.timedWait(&mutex, 0));
	309	try testing.expectError(error.Timeout, cond.timedWait(&mutex, std.time.ns_per_ms));
243	310
244		pub fn timedWait(cond: AtomicCondition, mutex: Mutex, timeout_ns: u64) error{TimedOut}!void {
245		var waiter = QueueList.Node{ .data = .{} };
	311	// Try to wake inside the mutex.
	312	cond.signal();
	313	cond.broadcast();
	314	}
246	315
247		{
248		cond.queue_mutex.lock();
249		defer cond.queue_mutex.unlock();
	316	// Inspired from: https://github.com/Amanieu/parking_lot/pull/129
	317	test "Condition - wait and signal" {
	318	// This test requires spawning threads
	319	if (builtin.single_threaded) {
	320	return error.SkipZigTest;
	321	}
250	322
251		cond.queue_list.prepend(&waiter);
252		@atomicStore(bool, &cond.pending, true, .SeqCst);
253		}
	323	const num_threads = 4;
254	324
255		var timed_out = false;
256		mutex.unlock();
257		defer mutex.lock();
258		waiter.data.timedWait(timeout_ns) catch \|err\| switch (err) {
259		error.TimedOut => {
260		defer if (!timed_out) {
261		waiter.data.wait();
262		};
263		cond.queue_mutex.lock();
264		defer cond.queue_mutex.unlock();
265
266		if (!waiter.data.dequeued) {
267		timed_out = true;
268		cond.queue_list.remove(&waiter);
269		}
270		},
271		else => unreachable,
272		};
	325	const MultiWait = struct {
	326	mutex: Mutex = .{},
	327	cond: Condition = .{},
	328	threads: [num_threads]std.Thread = undefined,
273	329
274		if (timed_out) {
275		return error.TimedOut;
	330	fn run(self: *@This()) void {
	331	self.mutex.lock();
	332	defer self.mutex.unlock();
	333
	334	self.cond.wait(&self.mutex);
	335	self.cond.timedWait(&self.mutex, std.time.ns_per_ms) catch {};
	336	self.cond.signal();
276	337	}
	338	};
	339
	340	var multi_wait = MultiWait{};
	341	for (multi_wait.threads) \|*t\| {
	342	t.* = try std.Thread.spawn(.{}, MultiWait.run, .{&multi_wait});
277	343	}
278	344
279		pub fn signal(cond: *AtomicCondition) void {
280		if (@atomicLoad(bool, &cond.pending, .SeqCst) == false)
281		return;
	345	std.time.sleep(100 * std.time.ns_per_ms);
282	346
283		const maybe_waiter = blk: {
284		cond.queue_mutex.lock();
285		defer cond.queue_mutex.unlock();
	347	multi_wait.cond.signal();
	348	for (multi_wait.threads) \|t\| {
	349	t.join();
	350	}
	351	}
286	352
287		const maybe_waiter = cond.queue_list.popFirst();
288		if (maybe_waiter) \|waiter\| {
289		waiter.data.dequeued = true;
	353	test "Condition - signal" {
	354	// This test requires spawning threads
	355	if (builtin.single_threaded) {
	356	return error.SkipZigTest;
	357	}
	358
	359	const num_threads = 4;
	360
	361	const SignalTest = struct {
	362	mutex: Mutex = .{},
	363	cond: Condition = .{},
	364	notified: bool = false,
	365	threads: [num_threads]std.Thread = undefined,
	366
	367	fn run(self: *@This()) void {
	368	self.mutex.lock();
	369	defer self.mutex.unlock();
	370
	371	// Use timedWait() a few times before using wait()
	372	// to test multiple threads timing out frequently.
	373	var i: usize = 0;
	374	while (!self.notified) : (i +%= 1) {
	375	if (i < 5) {
	376	self.cond.timedWait(&self.mutex, 1) catch {};
	377	} else {
	378	self.cond.wait(&self.mutex);
	379	}
290	380	}
291		@atomicStore(bool, &cond.pending, cond.queue_list.first != null, .SeqCst);
292		break :blk maybe_waiter;
293		};
294	381
295		if (maybe_waiter) \|waiter\| {
296		waiter.data.notify();
	382	// Once we received the signal, notify another thread (inside the lock).
	383	assert(self.notified);
	384	self.cond.signal();
297	385	}
298		}
299
300		pub fn broadcast(cond: *AtomicCondition) void {
301		if (@atomicLoad(bool, &cond.pending, .SeqCst) == false)
302		return;
	386	};
303	387
304		@atomicStore(bool, &cond.pending, false, .SeqCst);
	388	var signal_test = SignalTest{};
	389	for (signal_test.threads) \|*t\| {
	390	t.* = try std.Thread.spawn(.{}, SignalTest.run, .{&signal_test});
	391	}
305	392
306		var waiters = blk: {
307		cond.queue_mutex.lock();
308		defer cond.queue_mutex.unlock();
	393	{
	394	// Wait for a bit in hopes that the spawned threads start queuing up on the condvar
	395	std.time.sleep(10 * std.time.ns_per_ms);
309	396
310		const waiters = cond.queue_list;
	397	// Wake up one of them (outside the lock) after setting notified=true.
	398	defer signal_test.cond.signal();
311	399
312		var it = waiters.first;
313		while (it) \|node\| : (it = node.next) {
314		node.data.dequeued = true;
315		}
	400	signal_test.mutex.lock();
	401	defer signal_test.mutex.unlock();
316	402
317		cond.queue_list = .{};
318		break :blk waiters;
319		};
	403	try testing.expect(!signal_test.notified);
	404	signal_test.notified = true;
	405	}
320	406
321		while (waiters.popFirst()) \|waiter\| {
322		waiter.data.notify();
323		}
	407	for (signal_test.threads) \|t\| {
	408	t.join();
324	409	}
325		};
	410	}
326	411
327		test "Thread.Condition" {
	412	test "Condition - multi signal" {
	413	// This test requires spawning threads
328	414	if (builtin.single_threaded) {
329	415	return error.SkipZigTest;
330	416	}
331	417
332		const TestContext = struct {
333		cond: *Condition,
334		cond_main: *Condition,
335		mutex: *Mutex,
336		n: *i32,
337		fn worker(ctx: *@This()) void {
338		ctx.mutex.lock();
339		ctx.n.* += 1;
340		ctx.cond_main.signal();
341		ctx.cond.wait(ctx.mutex);
342		ctx.n.* -= 1;
343		ctx.cond_main.signal();
344		ctx.mutex.unlock();
	418	const num_threads = 4;
	419	const num_iterations = 4;
	420
	421	const Paddle = struct {
	422	mutex: Mutex = .{},
	423	cond: Condition = .{},
	424	value: u32 = 0,
	425
	426	fn hit(self: *@This()) void {
	427	defer self.cond.signal();
	428
	429	self.mutex.lock();
	430	defer self.mutex.unlock();
	431
	432	self.value += 1;
345	433	}
346		};
347		const num_threads = 3;
348		var threads: [num_threads]std.Thread = undefined;
349		var cond = Condition{};
350		var cond_main = Condition{};
351		var mut = Mutex{};
352		var n: i32 = 0;
353		var ctx = TestContext{ .cond = &cond, .cond_main = &cond_main, .mutex = &mut, .n = &n };
354	434
355		mut.lock();
356		for (threads) \|t\| t. = try std.Thread.spawn(.{}, TestContext.worker, .{&ctx});
357		cond_main.wait(&mut);
358		while (n < num_threads) cond_main.wait(&mut);
	435	fn run(self: @This(), hit_to: @This()) !void {
	436	self.mutex.lock();
	437	defer self.mutex.unlock();
359	438
360		cond.signal();
361		cond_main.wait(&mut);
362		try testing.expect(n == (num_threads - 1));
	439	var current: u32 = 0;
	440	while (current < num_iterations) : (current += 1) {
	441	// Wait for the value to change from hit()
	442	while (self.value == current) {
	443	self.cond.wait(&self.mutex);
	444	}
363	445
364		cond.broadcast();
365		while (n > 0) cond_main.wait(&mut);
366		try testing.expect(n == 0);
	446	// hit the next paddle
	447	try testing.expectEqual(self.value, current + 1);
	448	hit_to.hit();
	449	}
	450	}
	451	};
	452
	453	var paddles = [_]Paddle{.{}} ** num_threads;
	454	var threads = [_]std.Thread{undefined} ** num_threads;
	455
	456	// Create a circle of paddles which hit each other
	457	for (threads) \|*t, i\| {
	458	const paddle = &paddles[i];
	459	const hit_to = &paddles[(i + 1) % paddles.len];
	460	t.* = try std.Thread.spawn(.{}, Paddle.run, .{ paddle, hit_to });
	461	}
367	462
	463	// Hit the first paddle and wait for them all to complete by hitting each other for num_iterations.
	464	paddles[0].hit();
368	465	for (threads) \|t\| t.join();
	466
	467	// The first paddle will be hit one last time by the last paddle.
	468	for (paddles) \|p, i\| {
	469	const expected = @as(u32, num_iterations) + @boolToInt(i == 0);
	470	try testing.expectEqual(p.value, expected);
	471	}
369	472	}
370	473
371		test "Thread.Condition.timedWait" {
	474	test "Condition - broadcasting" {
	475	// This test requires spawning threads
372	476	if (builtin.single_threaded) {
373	477	return error.SkipZigTest;
374	478	}
375	479
376		var cond = Condition{};
377		var mut = Mutex{};
	480	const num_threads = 10;
378	481
379		// Expect a timeout, as the condition variable is never signaled
380		{
381		mut.lock();
382		defer mut.unlock();
383		try testing.expectError(error.TimedOut, cond.timedWait(&mut, 10 * std.time.ns_per_ms));
	482	const BroadcastTest = struct {
	483	mutex: Mutex = .{},
	484	cond: Condition = .{},
	485	completed: Condition = .{},
	486	count: usize = 0,
	487	threads: [num_threads]std.Thread = undefined,
	488
	489	fn run(self: *@This()) void {
	490	self.mutex.lock();
	491	defer self.mutex.unlock();
	492
	493	// The last broadcast thread to start tells the main test thread it's completed.
	494	self.count += 1;
	495	if (self.count == num_threads) {
	496	self.completed.signal();
	497	}
	498
	499	// Waits for the count to reach zero after the main test thread observes it at num_threads.
	500	// Tries to use timedWait() a bit before falling back to wait() to test multiple threads timing out.
	501	var i: usize = 0;
	502	while (self.count != 0) : (i +%= 1) {
	503	if (i < 10) {
	504	self.cond.timedWait(&self.mutex, 1) catch {};
	505	} else {
	506	self.cond.wait(&self.mutex);
	507	}
	508	}
	509	}
	510	};
	511
	512	var broadcast_test = BroadcastTest{};
	513	for (broadcast_test.threads) \|*t\| {
	514	t.* = try std.Thread.spawn(.{}, BroadcastTest.run, .{&broadcast_test});
384	515	}
385	516
386		// Expect a signal before timeout
387	517	{
388		const TestContext = struct {
389		cond: *Condition,
390		mutex: *Mutex,
391		n: *u32,
392		fn worker(ctx: *@This()) void {
393		ctx.mutex.lock();
394		defer ctx.mutex.unlock();
395		ctx.n.* = 1;
396		ctx.cond.signal();
397		}
398		};
	518	broadcast_test.mutex.lock();
	519	defer broadcast_test.mutex.unlock();
	520
	521	// Wait for all the broadcast threads to spawn.
	522	// timedWait() to detect any potential deadlocks.
	523	while (broadcast_test.count != num_threads) {
	524	try broadcast_test.completed.timedWait(
	525	&broadcast_test.mutex,
	526	1 * std.time.ns_per_s,
	527	);
	528	}
399	529
400		var n: u32 = 0;
	530	// Reset the counter and wake all the threads to exit.
	531	broadcast_test.count = 0;
	532	broadcast_test.cond.broadcast();
	533	}
401	534
402		var ctx = TestContext{ .cond = &cond, .mutex = &mut, .n = &n };
403		mut.lock();
404		var thread = try std.Thread.spawn(.{}, TestContext.worker, .{&ctx});
405		// Looped check to handle spurious wakeups
406		while (n != 1) try cond.timedWait(&mut, 500 * std.time.ns_per_ms);
407		mut.unlock();
408		try testing.expect(n == 1);
409		thread.join();
	535	for (broadcast_test.threads) \|t\| {
	536	t.join();
410	537	}
411	538	}

lib/std/Thread/Futex.zig+62-7

...	...	@@ -10,14 +10,12 @@ const Futex = @This();
10	10	const os = std.os;
11	11	const assert = std.debug.assert;
12	12	const testing = std.testing;
13
14	13	const Atomic = std.atomic.Atomic;
15		const spinLoopHint = std.atomic.spinLoopHint;
16	14
17	15	/// Checks if `ptr` still contains the value `expect` and, if so, blocks the caller until either:
18	16	/// - The value at `ptr` is no longer equal to `expect`.
19	17	/// - The caller is unblocked by a matching `wake()`.
20		/// - The caller is unblocked spuriously by an arbitrary internal signal.
	18	/// - The caller is unblocked spuriously ("at random").
21	19	///
22	20	/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically
23	21	/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.
...	...	@@ -32,7 +30,7 @@ pub fn wait(ptr: *const Atomic(u32), expect: u32) void {
32	30	/// Checks if `ptr` still contains the value `expect` and, if so, blocks the caller until either:
33	31	/// - The value at `ptr` is no longer equal to `expect`.
34	32	/// - The caller is unblocked by a matching `wake()`.
35		/// - The caller is unblocked spuriously by an arbitrary internal signal.
	33	/// - The caller is unblocked spuriously ("at random").
36	34	/// - The caller blocks for longer than the given timeout. In which case, `error.Timeout` is returned.
37	35	///
38	36	/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically
...	...	@@ -62,7 +60,7 @@ pub fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {
62	60	}
63	61
64	62	const Impl = if (builtin.single_threaded)
65		SerialImpl
	63	SingleThreadedImpl
66	64	else if (builtin.os.tag == .windows)
67	65	WindowsImpl
68	66	else if (builtin.os.tag.isDarwin())
...	...	@@ -97,7 +95,7 @@ const UnsupportedImpl = struct {
97	95	}
98	96	};
99	97
100		const SerialImpl = struct {
	98	const SingleThreadedImpl = struct {
101	99	fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
102	100	if (ptr.loadUnchecked() != expect) {
103	101	return;
...	...	@@ -804,7 +802,7 @@ const PosixImpl = struct {
804	802	//
805	803	// What we really want here is a Release load, but that doesn't exist under the C11 memory model.
806	804	// We could instead do `bucket.pending.fetchAdd(0, Release) == 0` which achieves effectively the same thing,
807		// but the RMW operation unconditionally stores which invalidates the cache-line for others causing unnecessary contention.
	805	// but the RMW operation unconditionally marks the cache-line as modified for others causing unnecessary fetching/contention.
808	806	//
809	807	// Instead we opt to do a full-fence + load instead which avoids taking ownership of the cache-line.
810	808	// fence(SeqCst) effectively converts the ptr update to SeqCst and the pending load to SeqCst: creating a Store-Load barrier.
...	...	@@ -962,3 +960,60 @@ test "Futex - broadcasting" {
962	960	for (broadcast.threads) \|t\| t. = try std.Thread.spawn(.{}, Broadcast.run, .{&broadcast});
963	961	for (broadcast.threads) \|t\| t.join();
964	962	}
	963
	964	/// Deadline is used to wait efficiently for a pointer's value to change using Futex and a fixed timeout.
	965	///
	966	/// Futex's timedWait() api uses a relative duration which suffers from over-waiting
	967	/// when used in a loop which is often required due to the possibility of spurious wakeups.
	968	///
	969	/// Deadline instead converts the relative timeout to an absolute one so that multiple calls
	970	/// to Futex timedWait() can block for and report more accurate error.Timeouts.
	971	pub const Deadline = struct {
	972	timeout: ?u64,
	973	started: std.time.Timer,
	974
	975	/// Create the deadline to expire after the given amount of time in nanoseconds passes.
	976	/// Pass in `null` to have the deadline call `Futex.wait()` and never expire.
	977	pub fn init(expires_in_ns: ?u64) Deadline {
	978	var deadline: Deadline = undefined;
	979	deadline.timeout = expires_in_ns;
	980
	981	// std.time.Timer is required to be supported for somewhat accurate reportings of error.Timeout.
	982	if (deadline.timeout != null) {
	983	deadline.started = std.time.Timer.start() catch unreachable;
	984	}
	985
	986	return deadline;
	987	}
	988
	989	/// Wait until either:
	990	/// - the `ptr`'s value changes from `expect`.
	991	/// - `Futex.wake()` is called on the `ptr`.
	992	/// - A spurious wake occurs.
	993	/// - The deadline expires; In which case `error.Timeout` is returned.
	994	pub fn wait(self: Deadline, ptr: const Atomic(u32), expect: u32) error{Timeout}!void {
	995	@setCold(true);
	996
	997	// Check if we actually have a timeout to wait until.
	998	// If not just wait "forever".
	999	const timeout_ns = self.timeout orelse {
	1000	return Futex.wait(ptr, expect);
	1001	};
	1002
	1003	// Get how much time has passed since we started waiting
	1004	// then subtract that from the init() timeout to get how much longer to wait.
	1005	// Use overflow to detect when we've been waiting longer than the init() timeout.
	1006	const elapsed_ns = self.started.read();
	1007	const until_timeout_ns = std.math.sub(u64, timeout_ns, elapsed_ns) catch 0;
	1008	return Futex.timedWait(ptr, expect, until_timeout_ns);
	1009	}
	1010	};
	1011
	1012	test "Futex - Deadline" {
	1013	var deadline = Deadline.init(100 * std.time.ns_per_ms);
	1014	var futex_word = Atomic(u32).init(0);
	1015
	1016	while (true) {
	1017	deadline.wait(&futex_word, 0) catch break;
	1018	}
	1019	}

lib/std/Thread/Mutex.zig+208-211

...	...	@@ -1,288 +1,285 @@
1		//! Lock may be held only once. If the same thread tries to acquire
2		//! the same mutex twice, it deadlocks. This type supports static
3		//! initialization and is at most `@sizeOf(usize)` in size. When an
4		//! application is built in single threaded release mode, all the
5		//! functions are no-ops. In single threaded debug mode, there is
6		//! deadlock detection.
	1	//! Mutex is a synchronization primitive which enforces atomic access to a shared region of code known as the "critical section".
	2	//! It does this by blocking ensuring only one thread is in the critical section at any given point in time by blocking the others.
	3	//! Mutex can be statically initialized and is at most `@sizeOf(u64)` large.
	4	//! Use `lock()` or `tryLock()` to enter the critical section and `unlock()` to leave it.
7	5	//!
8		//! Example usage:
	6	//! Example:
	7	//! ```
9	8	//! var m = Mutex{};
10	9	//!
11		//! m.lock();
12		//! defer m.release();
13		//! ... critical code
	10	//! {
	11	//! m.lock();
	12	//! defer m.unlock();
	13	//! // ... critical section code
	14	//! }
14	15	//!
15		//! Non-blocking:
16	16	//! if (m.tryLock()) {
17	17	//! defer m.unlock();
18		//! // ... critical section
19		//! } else {
20		//! // ... lock not acquired
	18	//! // ... critical section code
21	19	//! }
	20	//! ```
22	21
23		impl: Impl = .{},
24
25		const Mutex = @This();
26	22	const std = @import("../std.zig");
27	23	const builtin = @import("builtin");
	24	const Mutex = @This();
	25
28	26	const os = std.os;
29	27	const assert = std.debug.assert;
30		const windows = os.windows;
31		const linux = os.linux;
32	28	const testing = std.testing;
33		const StaticResetEvent = std.thread.StaticResetEvent;
	29	const Atomic = std.atomic.Atomic;
	30	const Futex = std.Thread.Futex;
	31
	32	impl: Impl = .{},
34	33
35		/// Try to acquire the mutex without blocking. Returns `false` if the mutex is
36		/// unavailable. Otherwise returns `true`. Call `unlock` on the mutex to release.
37		pub fn tryLock(m: *Mutex) bool {
38		return m.impl.tryLock();
	34	/// Tries to acquire the mutex without blocking the caller's thread.
	35	/// Returns `false` if the calling thread would have to block to acquire it.
	36	/// Otherwise, returns `true` and the caller should `unlock()` the Mutex to release it.
	37	pub fn tryLock(self: *Mutex) bool {
	38	return self.impl.tryLock();
39	39	}
40	40
41		/// Acquire the mutex. Deadlocks if the mutex is already
42		/// held by the calling thread.
43		pub fn lock(m: *Mutex) void {
44		m.impl.lock();
	41	/// Acquires the mutex, blocking the caller's thread until it can.
	42	/// It is undefined behavior if the mutex is already held by the caller's thread.
	43	/// Once acquired, call `unlock()` on the Mutex to release it.
	44	pub fn lock(self: *Mutex) void {
	45	self.impl.lock();
45	46	}
46	47
47		pub fn unlock(m: *Mutex) void {
48		m.impl.unlock();
	48	/// Releases the mutex which was previously acquired with `lock()` or `tryLock()`.
	49	/// It is undefined behavior if the mutex is unlocked from a different thread that it was locked from.
	50	pub fn unlock(self: *Mutex) void {
	51	self.impl.unlock();
49	52	}
50	53
51	54	const Impl = if (builtin.single_threaded)
52		Dummy
	55	SingleThreadedImpl
53	56	else if (builtin.os.tag == .windows)
54		WindowsMutex
55		else if (std.Thread.use_pthreads)
56		PthreadMutex
	57	WindowsImpl
	58	else if (builtin.os.tag.isDarwin())
	59	DarwinImpl
57	60	else
58		AtomicMutex;
59
60		pub const AtomicMutex = struct {
61		state: State = .unlocked,
	61	FutexImpl;
62	62
63		const State = enum(i32) {
64		unlocked,
65		locked,
66		waiting,
67		};
	63	const SingleThreadedImpl = struct {
	64	is_locked: bool = false,
68	65
69		pub fn tryLock(m: *AtomicMutex) bool {
70		return @cmpxchgStrong(
71		State,
72		&m.state,
73		.unlocked,
74		.locked,
75		.Acquire,
76		.Monotonic,
77		) == null;
	66	fn tryLock(self: *Impl) bool {
	67	if (self.is_locked) return false;
	68	self.is_locked = true;
	69	return true;
78	70	}
79	71
80		pub fn lock(m: *AtomicMutex) void {
81		switch (@atomicRmw(State, &m.state, .Xchg, .locked, .Acquire)) {
82		.unlocked => {},
83		else => \|s\| m.lockSlow(s),
	72	fn lock(self: *Impl) void {
	73	if (!self.tryLock()) {
	74	unreachable; // deadlock detected
84	75	}
85	76	}
86	77
87		pub fn unlock(m: *AtomicMutex) void {
88		switch (@atomicRmw(State, &m.state, .Xchg, .unlocked, .Release)) {
89		.unlocked => unreachable,
90		.locked => {},
91		.waiting => m.unlockSlow(),
92		}
	78	fn unlock(self: *Impl) void {
	79	assert(self.is_locked);
	80	self.is_locked = false;
93	81	}
	82	};
94	83
95		fn lockSlow(m: *AtomicMutex, current_state: State) void {
96		@setCold(true);
97		var new_state = current_state;
98
99		var spin: u8 = 0;
100		while (spin < 100) : (spin += 1) {
101		const state = @cmpxchgWeak(
102		State,
103		&m.state,
104		.unlocked,
105		new_state,
106		.Acquire,
107		.Monotonic,
108		) orelse return;
109
110		switch (state) {
111		.unlocked => {},
112		.locked => {},
113		.waiting => break,
114		}
115
116		var iter = std.math.min(32, spin + 1);
117		while (iter > 0) : (iter -= 1)
118		std.atomic.spinLoopHint();
119		}
	84	// SRWLOCK on windows is almost always faster than Futex solution.
	85	// It also implements an efficient Condition with requeue support for us.
	86	const WindowsImpl = struct {
	87	srwlock: os.windows.SRWLOCK = .{},
120	88
121		new_state = .waiting;
122		while (true) {
123		switch (@atomicRmw(State, &m.state, .Xchg, new_state, .Acquire)) {
124		.unlocked => return,
125		else => {},
126		}
127		switch (builtin.os.tag) {
128		.linux => {
129		switch (linux.getErrno(linux.futex_wait(
130		@ptrCast(*const i32, &m.state),
131		linux.FUTEX.PRIVATE_FLAG \| linux.FUTEX.WAIT,
132		@enumToInt(new_state),
133		null,
134		))) {
135		.SUCCESS => {},
136		.INTR => {},
137		.AGAIN => {},
138		else => unreachable,
139		}
140		},
141		else => std.atomic.spinLoopHint(),
142		}
143		}
	89	fn tryLock(self: *Impl) bool {
	90	return os.windows.kernel32.TryAcquireSRWLockExclusive(&self.srwlock) != os.windows.FALSE;
144	91	}
145	92
146		fn unlockSlow(m: *AtomicMutex) void {
147		@setCold(true);
	93	fn lock(self: *Impl) void {
	94	os.windows.kernel32.AcquireSRWLockExclusive(&self.srwlock);
	95	}
148	96
149		switch (builtin.os.tag) {
150		.linux => {
151		switch (linux.getErrno(linux.futex_wake(
152		@ptrCast(*const i32, &m.state),
153		linux.FUTEX.PRIVATE_FLAG \| linux.FUTEX.WAKE,
154		1,
155		))) {
156		.SUCCESS => {},
157		.FAULT => unreachable, // invalid pointer passed to futex_wake
158		else => unreachable,
159		}
160		},
161		else => {},
162		}
	97	fn unlock(self: *Impl) void {
	98	os.windows.kernel32.ReleaseSRWLockExclusive(&self.srwlock);
163	99	}
164	100	};
165	101
166		pub const PthreadMutex = struct {
167		pthread_mutex: std.c.pthread_mutex_t = .{},
	102	// os_unfair_lock on darwin supports priority inheritance and is generally faster than Futex solutions.
	103	const DarwinImpl = struct {
	104	oul: os.darwin.os_unfair_lock = .{},
168	105
169		/// Try to acquire the mutex without blocking. Returns true if
170		/// the mutex is unavailable. Otherwise returns false. Call
171		/// release when done.
172		pub fn tryLock(m: *PthreadMutex) bool {
173		return std.c.pthread_mutex_trylock(&m.pthread_mutex) == .SUCCESS;
	106	fn tryLock(self: *Impl) bool {
	107	return os.darwin.os_unfair_lock_trylock(&self.oul);
174	108	}
175	109
176		/// Acquire the mutex. Will deadlock if the mutex is already
177		/// held by the calling thread.
178		pub fn lock(m: *PthreadMutex) void {
179		switch (std.c.pthread_mutex_lock(&m.pthread_mutex)) {
180		.SUCCESS => {},
181		.INVAL => unreachable,
182		.BUSY => unreachable,
183		.AGAIN => unreachable,
184		.DEADLK => unreachable,
185		.PERM => unreachable,
186		else => unreachable,
187		}
	110	fn lock(self: *Impl) void {
	111	os.darwin.os_unfair_lock_lock(&self.oul);
188	112	}
189	113
190		pub fn unlock(m: *PthreadMutex) void {
191		switch (std.c.pthread_mutex_unlock(&m.pthread_mutex)) {
192		.SUCCESS => return,
193		.INVAL => unreachable,
194		.AGAIN => unreachable,
195		.PERM => unreachable,
196		else => unreachable,
197		}
	114	fn unlock(self: *Impl) void {
	115	os.darwin.os_unfair_lock_unlock(&self.oul);
198	116	}
199	117	};
200	118
201		/// This has the sematics as `Mutex`, however it does not actually do any
202		/// synchronization. Operations are safety-checked no-ops.
203		pub const Dummy = struct {
204		locked: @TypeOf(lock_init) = lock_init,
	119	const FutexImpl = struct {
	120	state: Atomic(u32) = Atomic(u32).init(unlocked),
205	121
206		const lock_init = if (std.debug.runtime_safety) false else {};
	122	const unlocked = 0b00;
	123	const locked = 0b01;
	124	const contended = 0b11; // must contain the `locked` bit for x86 optimization below
	125
	126	fn tryLock(self: *Impl) bool {
	127	// Lock with compareAndSwap instead of tryCompareAndSwap to avoid reporting spurious CAS failure.
	128	return self.lockFast("compareAndSwap");
	129	}
207	130
208		/// Try to acquire the mutex without blocking. Returns false if
209		/// the mutex is unavailable. Otherwise returns true.
210		pub fn tryLock(m: *Dummy) bool {
211		if (std.debug.runtime_safety) {
212		if (m.locked) return false;
213		m.locked = true;
	131	fn lock(self: *Impl) void {
	132	// Lock with tryCompareAndSwap instead of compareAndSwap due to being more inline-able on LL/SC archs like ARM.
	133	if (!self.lockFast("tryCompareAndSwap")) {
	134	self.lockSlow();
214	135	}
215		return true;
216	136	}
217	137
218		/// Acquire the mutex. Will deadlock if the mutex is already
219		/// held by the calling thread.
220		pub fn lock(m: *Dummy) void {
221		if (!m.tryLock()) {
222		@panic("deadlock detected");
	138	inline fn lockFast(self: *Impl, comptime casFn: []const u8) bool {
	139	// On x86, use `lock bts` instead of `lock cmpxchg` as:
	140	// - they both seem to mark the cache-line as modified regardless: https://stackoverflow.com/a/63350048
	141	// - `lock bts` is smaller instruction-wise which makes it better for inlining
	142	if (comptime builtin.target.cpu.arch.isX86()) {
	143	const locked_bit = @ctz(u32, @as(u32, locked));
	144	return self.state.bitSet(locked_bit, .Acquire) == 0;
223	145	}
	146
	147	// Acquire barrier ensures grabbing the lock happens before the critical section
	148	// and that the previous lock holder's critical section happens before we grab the lock.
	149	return @field(self.state, casFn)(unlocked, locked, .Acquire, .Monotonic) == null;
224	150	}
225	151
226		pub fn unlock(m: *Dummy) void {
227		if (std.debug.runtime_safety) {
228		m.locked = false;
	152	fn lockSlow(self: *Impl) void {
	153	@setCold(true);
	154
	155	// Avoid doing an atomic swap below if we already know the state is contended.
	156	// An atomic swap unconditionally stores which marks the cache-line as modified unnecessarily.
	157	if (self.state.load(.Monotonic) == contended) {
	158	Futex.wait(&self.state, contended);
	159	}
	160
	161	// Try to acquire the lock while also telling the existing lock holder that there are threads waiting.
	162	//
	163	// Once we sleep on the Futex, we must acquire the mutex using `contended` rather than `locked`.
	164	// If not, threads sleeping on the Futex wouldn't see the state change in unlock and potentially deadlock.
	165	// The downside is that the last mutex unlocker will see `contended` and do an unnecessary Futex wake
	166	// but this is better than having to wake all waiting threads on mutex unlock.
	167	//
	168	// Acquire barrier ensures grabbing the lock happens before the critical section
	169	// and that the previous lock holder's critical section happens before we grab the lock.
	170	while (self.state.swap(contended, .Acquire) != unlocked) {
	171	Futex.wait(&self.state, contended);
	172	}
	173	}
	174
	175	fn unlock(self: *Impl) void {
	176	// Unlock the mutex and wake up a waiting thread if any.
	177	//
	178	// A waiting thread will acquire with `contended` instead of `locked`
	179	// which ensures that it wakes up another thread on the next unlock().
	180	//
	181	// Release barrier ensures the critical section happens before we let go of the lock
	182	// and that our critical section happens before the next lock holder grabs the lock.
	183	const state = self.state.swap(unlocked, .Release);
	184	assert(state != unlocked);
	185
	186	if (state == contended) {
	187	Futex.wake(&self.state, 1);
229	188	}
230	189	}
231	190	};
232	191
233		pub const WindowsMutex = struct {
234		srwlock: windows.SRWLOCK = windows.SRWLOCK_INIT,
	192	test "Mutex - smoke test" {
	193	var mutex = Mutex{};
	194
	195	try testing.expect(mutex.tryLock());
	196	try testing.expect(!mutex.tryLock());
	197	mutex.unlock();
235	198
236		pub fn tryLock(m: *WindowsMutex) bool {
237		return windows.kernel32.TryAcquireSRWLockExclusive(&m.srwlock) != windows.FALSE;
238		}
	199	mutex.lock();
	200	try testing.expect(!mutex.tryLock());
	201	mutex.unlock();
	202	}
	203
	204	// A counter which is incremented without atomic instructions
	205	const NonAtomicCounter = struct {
	206	// direct u128 could maybe use xmm ops on x86 which are atomic
	207	value: [2]u64 = [_]u64{ 0, 0 },
239	208
240		pub fn lock(m: *WindowsMutex) void {
241		windows.kernel32.AcquireSRWLockExclusive(&m.srwlock);
	209	fn get(self: NonAtomicCounter) u128 {
	210	return @bitCast(u128, self.value);
242	211	}
243	212
244		pub fn unlock(m: *WindowsMutex) void {
245		windows.kernel32.ReleaseSRWLockExclusive(&m.srwlock);
	213	fn inc(self: *NonAtomicCounter) void {
	214	for (@bitCast([2]u64, self.get() + 1)) \|v, i\| {
	215	@ptrCast(volatile u64, &self.value[i]). = v;
	216	}
246	217	}
247	218	};
248	219
249		const TestContext = struct {
250		mutex: *Mutex,
251		data: i128,
	220	test "Mutex - many uncontended" {
	221	// This test requires spawning threads.
	222	if (builtin.single_threaded) {
	223	return error.SkipZigTest;
	224	}
	225
	226	const num_threads = 4;
	227	const num_increments = 1000;
252	228
253		const incr_count = 10000;
254		};
	229	const Runner = struct {
	230	mutex: Mutex = .{},
	231	thread: std.Thread = undefined,
	232	counter: NonAtomicCounter = .{},
255	233
256		test "basic usage" {
257		var mutex = Mutex{};
	234	fn run(self: *@This()) void {
	235	var i: usize = num_increments;
	236	while (i > 0) : (i -= 1) {
	237	self.mutex.lock();
	238	defer self.mutex.unlock();
258	239
259		var context = TestContext{
260		.mutex = &mutex,
261		.data = 0,
	240	self.counter.inc();
	241	}
	242	}
262	243	};
263	244
	245	var runners = [_]Runner{.{}} ** num_threads;
	246	for (runners) \|*r\| r.thread = try std.Thread.spawn(.{}, Runner.run, .{r});
	247	for (runners) \|r\| r.thread.join();
	248	for (runners) \|r\| try testing.expectEqual(r.counter.get(), num_increments);
	249	}
	250
	251	test "Mutex - many contended" {
	252	// This test requires spawning threads.
264	253	if (builtin.single_threaded) {
265		worker(&context);
266		try testing.expect(context.data == TestContext.incr_count);
267		} else {
268		const thread_count = 10;
269		var threads: [thread_count]std.Thread = undefined;
270		for (threads) \|*t\| {
271		t.* = try std.Thread.spawn(.{}, worker, .{&context});
	254	return error.SkipZigTest;
	255	}
	256
	257	const num_threads = 4;
	258	const num_increments = 1000;
	259
	260	const Runner = struct {
	261	mutex: Mutex = .{},
	262	counter: NonAtomicCounter = .{},
	263
	264	fn run(self: *@This()) void {
	265	var i: usize = num_increments;
	266	while (i > 0) : (i -= 1) {
	267	// Occasionally hint to let another thread run.
	268	defer if (i % 100 == 0) std.Thread.yield() catch {};
	269
	270	self.mutex.lock();
	271	defer self.mutex.unlock();
	272
	273	self.counter.inc();
	274	}
272	275	}
273		for (threads) \|t\|
274		t.join();
	276	};
275	277
276		try testing.expect(context.data == thread_count * TestContext.incr_count);
277		}
278		}
	278	var runner = Runner{};
279	279
280		fn worker(ctx: *TestContext) void {
281		var i: usize = 0;
282		while (i != TestContext.incr_count) : (i += 1) {
283		ctx.mutex.lock();
284		defer ctx.mutex.unlock();
	280	var threads: [num_threads]std.Thread = undefined;
	281	for (threads) \|t\| t. = try std.Thread.spawn(.{}, Runner.run, .{&runner});
	282	for (threads) \|t\| t.join();
285	283
286		ctx.data += 1;
287		}
	284	try testing.expectEqual(runner.counter.get(), num_increments * num_threads);
288	285	}

lib/std/atomic/Atomic.zig+87-81

...	...	@@ -1,7 +1,7 @@
1	1	const std = @import("../std.zig");
	2	const builtin = @import("builtin");
2	3
3	4	const testing = std.testing;
4		const target = @import("builtin").target;
5	5	const Ordering = std.atomic.Ordering;
6	6
7	7	pub fn Atomic(comptime T: type) type {
...	...	@@ -164,87 +164,13 @@ pub fn Atomic(comptime T: type) type {
164	164	return bitRmw(self, .Toggle, bit, ordering);
165	165	}
166	166
167		inline fn bitRmw(
168		self: *Self,
169		comptime op: BitRmwOp,
170		bit: Bit,
171		comptime ordering: Ordering,
172		) u1 {
	167	inline fn bitRmw(self: *Self, comptime op: BitRmwOp, bit: Bit, comptime ordering: Ordering) u1 {
173	168	// x86 supports dedicated bitwise instructions
174		if (comptime target.cpu.arch.isX86() and @sizeOf(T) >= 2 and @sizeOf(T) <= 8) {
175		const old_bit: u8 = switch (@sizeOf(T)) {
176		2 => switch (op) {
177		.Set => asm volatile ("lock btsw %[bit], %[ptr]"
178		// LLVM doesn't support u1 flag register return values
179		: [result] "={@ccc}" (-> u8),
180		: [ptr] "*p" (&self.value),
181		[bit] "X" (@as(T, bit)),
182		: "cc", "memory"
183		),
184		.Reset => asm volatile ("lock btrw %[bit], %[ptr]"
185		// LLVM doesn't support u1 flag register return values
186		: [result] "={@ccc}" (-> u8),
187		: [ptr] "*p" (&self.value),
188		[bit] "X" (@as(T, bit)),
189		: "cc", "memory"
190		),
191		.Toggle => asm volatile ("lock btcw %[bit], %[ptr]"
192		// LLVM doesn't support u1 flag register return values
193		: [result] "={@ccc}" (-> u8),
194		: [ptr] "*p" (&self.value),
195		[bit] "X" (@as(T, bit)),
196		: "cc", "memory"
197		),
198		},
199		4 => switch (op) {
200		.Set => asm volatile ("lock btsl %[bit], %[ptr]"
201		// LLVM doesn't support u1 flag register return values
202		: [result] "={@ccc}" (-> u8),
203		: [ptr] "*p" (&self.value),
204		[bit] "X" (@as(T, bit)),
205		: "cc", "memory"
206		),
207		.Reset => asm volatile ("lock btrl %[bit], %[ptr]"
208		// LLVM doesn't support u1 flag register return values
209		: [result] "={@ccc}" (-> u8),
210		: [ptr] "*p" (&self.value),
211		[bit] "X" (@as(T, bit)),
212		: "cc", "memory"
213		),
214		.Toggle => asm volatile ("lock btcl %[bit], %[ptr]"
215		// LLVM doesn't support u1 flag register return values
216		: [result] "={@ccc}" (-> u8),
217		: [ptr] "*p" (&self.value),
218		[bit] "X" (@as(T, bit)),
219		: "cc", "memory"
220		),
221		},
222		8 => switch (op) {
223		.Set => asm volatile ("lock btsq %[bit], %[ptr]"
224		// LLVM doesn't support u1 flag register return values
225		: [result] "={@ccc}" (-> u8),
226		: [ptr] "*p" (&self.value),
227		[bit] "X" (@as(T, bit)),
228		: "cc", "memory"
229		),
230		.Reset => asm volatile ("lock btrq %[bit], %[ptr]"
231		// LLVM doesn't support u1 flag register return values
232		: [result] "={@ccc}" (-> u8),
233		: [ptr] "*p" (&self.value),
234		[bit] "X" (@as(T, bit)),
235		: "cc", "memory"
236		),
237		.Toggle => asm volatile ("lock btcq %[bit], %[ptr]"
238		// LLVM doesn't support u1 flag register return values
239		: [result] "={@ccc}" (-> u8),
240		: [ptr] "*p" (&self.value),
241		[bit] "X" (@as(T, bit)),
242		: "cc", "memory"
243		),
244		},
245		else => @compileError("Invalid atomic type " ++ @typeName(T)),
246		};
247		return @intCast(u1, old_bit);
	169	if (comptime builtin.target.cpu.arch.isX86() and @sizeOf(T) >= 2 and @sizeOf(T) <= 8) {
	170	// TODO: stage2 currently doesn't like the inline asm this function emits.
	171	if (builtin.zig_backend == .stage1) {
	172	return x86BitRmw(self, op, bit, ordering);
	173	}
248	174	}
249	175
250	176	const mask = @as(T, 1) << bit;
...	...	@@ -256,6 +182,86 @@ pub fn Atomic(comptime T: type) type {
256	182
257	183	return @boolToInt(value & mask != 0);
258	184	}
	185
	186	inline fn x86BitRmw(self: *Self, comptime op: BitRmwOp, bit: Bit, comptime ordering: Ordering) u1 {
	187	const old_bit: u8 = switch (@sizeOf(T)) {
	188	2 => switch (op) {
	189	.Set => asm volatile ("lock btsw %[bit], %[ptr]"
	190	// LLVM doesn't support u1 flag register return values
	191	: [result] "={@ccc}" (-> u8),
	192	: [ptr] "*p" (&self.value),
	193	[bit] "X" (@as(T, bit)),
	194	: "cc", "memory"
	195	),
	196	.Reset => asm volatile ("lock btrw %[bit], %[ptr]"
	197	// LLVM doesn't support u1 flag register return values
	198	: [result] "={@ccc}" (-> u8),
	199	: [ptr] "*p" (&self.value),
	200	[bit] "X" (@as(T, bit)),
	201	: "cc", "memory"
	202	),
	203	.Toggle => asm volatile ("lock btcw %[bit], %[ptr]"
	204	// LLVM doesn't support u1 flag register return values
	205	: [result] "={@ccc}" (-> u8),
	206	: [ptr] "*p" (&self.value),
	207	[bit] "X" (@as(T, bit)),
	208	: "cc", "memory"
	209	),
	210	},
	211	4 => switch (op) {
	212	.Set => asm volatile ("lock btsl %[bit], %[ptr]"
	213	// LLVM doesn't support u1 flag register return values
	214	: [result] "={@ccc}" (-> u8),
	215	: [ptr] "*p" (&self.value),
	216	[bit] "X" (@as(T, bit)),
	217	: "cc", "memory"
	218	),
	219	.Reset => asm volatile ("lock btrl %[bit], %[ptr]"
	220	// LLVM doesn't support u1 flag register return values
	221	: [result] "={@ccc}" (-> u8),
	222	: [ptr] "*p" (&self.value),
	223	[bit] "X" (@as(T, bit)),
	224	: "cc", "memory"
	225	),
	226	.Toggle => asm volatile ("lock btcl %[bit], %[ptr]"
	227	// LLVM doesn't support u1 flag register return values
	228	: [result] "={@ccc}" (-> u8),
	229	: [ptr] "*p" (&self.value),
	230	[bit] "X" (@as(T, bit)),
	231	: "cc", "memory"
	232	),
	233	},
	234	8 => switch (op) {
	235	.Set => asm volatile ("lock btsq %[bit], %[ptr]"
	236	// LLVM doesn't support u1 flag register return values
	237	: [result] "={@ccc}" (-> u8),
	238	: [ptr] "*p" (&self.value),
	239	[bit] "X" (@as(T, bit)),
	240	: "cc", "memory"
	241	),
	242	.Reset => asm volatile ("lock btrq %[bit], %[ptr]"
	243	// LLVM doesn't support u1 flag register return values
	244	: [result] "={@ccc}" (-> u8),
	245	: [ptr] "*p" (&self.value),
	246	[bit] "X" (@as(T, bit)),
	247	: "cc", "memory"
	248	),
	249	.Toggle => asm volatile ("lock btcq %[bit], %[ptr]"
	250	// LLVM doesn't support u1 flag register return values
	251	: [result] "={@ccc}" (-> u8),
	252	: [ptr] "*p" (&self.value),
	253	[bit] "X" (@as(T, bit)),
	254	: "cc", "memory"
	255	),
	256	},
	257	else => @compileError("Invalid atomic type " ++ @typeName(T)),
	258	};
	259
	260	// TODO: emit appropriate tsan fence if compiling with tsan
	261	_ = ordering;
	262
	263	return @intCast(u1, old_bit);
	264	}
259	265	});
260	266	};
261	267	}

lib/std/heap/general_purpose_allocator.zig+8-3

...	...	@@ -151,12 +151,12 @@ pub const Config = struct {
151	151
152	152	/// What type of mutex you'd like to use, for thread safety.
153	153	/// when specfied, the mutex type must have the same shape as `std.Thread.Mutex` and
154		/// `std.Thread.Mutex.Dummy`, and have no required fields. Specifying this field causes
	154	/// `DummyMutex`, and have no required fields. Specifying this field causes
155	155	/// the `thread_safe` field to be ignored.
156	156	///
157	157	/// when null (default):
158	158	/// * the mutex type defaults to `std.Thread.Mutex` when thread_safe is enabled.
159		/// * the mutex type defaults to `std.Thread.Mutex.Dummy` otherwise.
	159	/// * the mutex type defaults to `DummyMutex` otherwise.
160	160	MutexType: ?type = null,
161	161
162	162	/// This is a temporary debugging trick you can use to turn segfaults into more helpful
...	...	@@ -198,7 +198,12 @@ pub fn GeneralPurposeAllocator(comptime config: Config) type {
198	198	else if (config.thread_safe)
199	199	std.Thread.Mutex{}
200	200	else
201		std.Thread.Mutex.Dummy{};
	201	DummyMutex{};
	202
	203	const DummyMutex = struct {
	204	fn lock(_: *DummyMutex) void {}
	205	fn unlock(_: *DummyMutex) void {}
	206	};
202	207
203	208	const stack_n = config.stack_trace_frames;
204	209	const one_trace_size = @sizeOf(usize) * stack_n;

lib/std/os/windows.zig+9-4

...	...	@@ -3680,10 +3680,15 @@ pub const OBJECT_NAME_INFORMATION = extern struct {
3680	3680	Name: UNICODE_STRING,
3681	3681	};
3682	3682
3683		pub const SRWLOCK = usize;
3684		pub const SRWLOCK_INIT: SRWLOCK = 0;
3685		pub const CONDITION_VARIABLE = usize;
3686		pub const CONDITION_VARIABLE_INIT: CONDITION_VARIABLE = 0;
	3683	pub const SRWLOCK_INIT = SRWLOCK{};
	3684	pub const SRWLOCK = extern struct {
	3685	Ptr: ?PVOID = null,
	3686	};
	3687
	3688	pub const CONDITION_VARIABLE_INIT = CONDITION_VARIABLE{};
	3689	pub const CONDITION_VARIABLE = extern struct {
	3690	Ptr: ?PVOID = null,
	3691	};
3687	3692
3688	3693	pub const FILE_SKIP_COMPLETION_PORT_ON_SUCCESS = 0x1;
3689	3694	pub const FILE_SKIP_SET_EVENT_ON_HANDLE = 0x2;