Commit 550da1b676d059ae39a629d60da0f9cd155a5e89 | zig

CMakeLists.txt-2

...	@@ -408,8 +408,6 @@ set(ZIG_STAGE2_SOURCES	...	@@ -408,8 +408,6 @@ set(ZIG_STAGE2_SOURCES
408	lib/std/Target/wasm.zig	408	lib/std/Target/wasm.zig
409	lib/std/Target/x86.zig	409	lib/std/Target/x86.zig
410	lib/std/Thread.zig	410	lib/std/Thread.zig
411	lib/std/Thread/Futex.zig
412	lib/std/Thread/Mutex.zig
413	lib/std/array_hash_map.zig	411	lib/std/array_hash_map.zig
414	lib/std/array_list.zig	412	lib/std/array_list.zig
415	lib/std/ascii.zig	413	lib/std/ascii.zig

lib/compiler/build_runner.zig+11-8

 @@ -30,14 +30,6 @@ pub fn main(init: process.Init.Minimal) !void {
     defer _ = debug_gpa_state.deinit();
     const gpa = debug_gpa_state.allocator();
-    // ...but we'll back our arena by `std.heap.page_allocator` for efficiency.
-    var single_threaded_arena: std.heap.ArenaAllocator = .init(std.heap.page_allocator);
-    defer single_threaded_arena.deinit();
-    var thread_safe_arena: std.heap.ThreadSafeAllocator = .{ .child_allocator = single_threaded_arena.allocator() };
-    const arena = thread_safe_arena.allocator();
-    const args = try init.args.toSlice(arena);
     var threaded: std.Io.Threaded = .init(gpa, .{
         .environ = init.environ,
         .argv0 = .init(init.args),
 @@ -45,6 +37,17 @@ pub fn main(init: process.Init.Minimal) !void {
     defer threaded.deinit();
     const io = threaded.io();
+    // ...but we'll back our arena by `std.heap.page_allocator` for efficiency.
+    var single_threaded_arena: std.heap.ArenaAllocator = .init(std.heap.page_allocator);
+    defer single_threaded_arena.deinit();
+    var thread_safe_arena: std.heap.ThreadSafeAllocator = .{
+        .child_allocator = single_threaded_arena.allocator(),
+        .io = io,
+    };
+    const arena = thread_safe_arena.allocator();
+    const args = try init.args.toSlice(arena);
     // skip my own exe name
     var arg_idx: usize = 1;

lib/compiler_rt/emutls.zig+7-2

 @@ -147,7 +147,8 @@ const ObjectArray = struct {
 // It provides thread-safety for on-demand storage of Thread Objects.
 const current_thread_storage = struct {
     var key: std.c.pthread_key_t = undefined;
-    var init_once = std.once(current_thread_storage.init);
+    var init_mutex: std.c.pthread_mutex_t = std.c.PTHREAD_MUTEX_INITIALIZER;
+    var init_done: bool = false;
     /// Return a per thread ObjectArray with at least the expected index.
     pub fn getArray(index: usize) *ObjectArray {
 @@ -183,9 +184,13 @@ const current_thread_storage = struct {
     /// Initialize pthread_key_t.
     fn init() void {
+        if (@atomicLoad(bool, &init_done, .monotonic)) return;
+        _ = std.c.pthread_mutex_lock(&init_mutex);
         if (std.c.pthread_key_create(&current_thread_storage.key, current_thread_storage.deinit) != .SUCCESS) {
             abort();
         }
+        @atomicStore(bool, &init_done, true, .release);
+        _ = std.c.pthread_mutex_unlock(&init_mutex);
     }
     /// Invoked by pthread specific destructor. the passed argument is the ObjectArray pointer.
 @@ -283,7 +288,7 @@ const emutls_control = extern struct {
     /// Get the pointer on allocated storage for emutls variable.
     pub fn getPointer(self: *emutls_control) *anyopaque {
         // ensure current_thread_storage initialization is done
-        current_thread_storage.init_once.call();
+        current_thread_storage.init();
         const index = self.getIndex();
         var array = current_thread_storage.getArray(index);

lib/fuzzer.zig+1-1

...	@@ -632,7 +632,7 @@ export fn fuzzer_main(limit_kind: abi.LimitKind, amount: u64) void {	...	@@ -632,7 +632,7 @@ export fn fuzzer_main(limit_kind: abi.LimitKind, amount: u64) void {
632		632
633	export fn fuzzer_unslide_address(addr: usize) usize {	633	export fn fuzzer_unslide_address(addr: usize) usize {
634	const si = std.debug.getSelfDebugInfo() catch @compileError("unsupported");	634	const si = std.debug.getSelfDebugInfo() catch @compileError("unsupported");
635	const slide = si.getModuleSlide(std.debug.getDebugInfoAllocator(), addr) catch \|err\| {	635	const slide = si.getModuleSlide(std.debug.getDebugInfoAllocator(), io, addr) catch \|err\| {
636	std.debug.panic("failed to find virtual address slide: {t}", .{err});	636	std.debug.panic("failed to find virtual address slide: {t}", .{err});
637	};	637	};
638	return addr - slide;	638	return addr - slide;

lib/std/Io/Threaded.zig+85-77

 @@ -1126,6 +1126,25 @@ const Thread = struct {
             return @ptrFromInt(@as(usize, @bitCast(split)));
         }
     };
+    /// Same as `Io.Mutex.lock` but avoids the VTable.
+    fn mutexLock(m: *Io.Mutex) Io.Cancelable!void {
+        const initial_state = m.state.cmpxchgWeak(
+            .unlocked,
+            .locked_once,
+            .acquire,
+            .monotonic,
+        ) orelse {
+            @branchHint(.likely);
+            return;
+        };
+        if (initial_state == .contended) {
+            try Thread.futexWait(@ptrCast(&m.state.raw), @intFromEnum(Io.Mutex.State.contended), null);
+        }
+        while (m.state.swap(.contended, .acquire) != .unlocked) {
+            try Thread.futexWait(@ptrCast(&m.state.raw), @intFromEnum(Io.Mutex.State.contended), null);
+        }
+    }
 };
 const Syscall = struct {
 @@ -1486,8 +1505,8 @@ var global_single_threaded_instance: Threaded = .init_single_threaded;
 pub const global_single_threaded: *Threaded = &global_single_threaded_instance;
 pub fn setAsyncLimit(t: *Threaded, new_limit: Io.Limit) void {
-    mutexLockUncancelable(&t.mutex);
+    mutexLockInternal(&t.mutex);
-    defer mutexUnlock(&t.mutex);
+    defer mutexUnlockInternal(&t.mutex);
     t.async_limit = new_limit;
 }
 @@ -1508,8 +1527,8 @@ pub fn deinit(t: *Threaded) void {
 fn join(t: *Threaded) void {
     if (builtin.single_threaded) return;
     {
-        mutexLockUncancelable(&t.mutex);
+        mutexLockInternal(&t.mutex);
-        defer mutexUnlock(&t.mutex);
+        defer mutexUnlockInternal(&t.mutex);
         t.join_requested = true;
     }
     condBroadcast(&t.cond);
 @@ -1574,16 +1593,16 @@ fn worker(t: *Threaded) void {
     defer t.wait_group.finish();
-    mutexLockUncancelable(&t.mutex);
+    mutexLockInternal(&t.mutex);
-    defer mutexUnlock(&t.mutex);
+    defer mutexUnlockInternal(&t.mutex);
     while (true) {
         while (t.run_queue.popFirst()) |runnable_node| {
-            mutexUnlock(&t.mutex);
+            mutexUnlockInternal(&t.mutex);
             thread.cancel_protection = .unblocked;
             const runnable: *Runnable = @fieldParentPtr("node", runnable_node);
             runnable.startFn(runnable, &thread, t);
-            mutexLockUncancelable(&t.mutex);
+            mutexLockInternal(&t.mutex);
             t.busy_count -= 1;
         }
         if (t.join_requested) break;
 @@ -2004,12 +2023,12 @@ fn async(
         },
     };
-    mutexLockUncancelable(&t.mutex);
+    mutexLockInternal(&t.mutex);
     const busy_count = t.busy_count;
     if (busy_count >= @intFromEnum(t.async_limit)) {
-        mutexUnlock(&t.mutex);
+        mutexUnlockInternal(&t.mutex);
         future.destroy(gpa);
         start(context.ptr, result.ptr);
         return null;
 @@ -2023,7 +2042,7 @@ fn async(
         const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch {
             t.wait_group.finish();
             t.busy_count = busy_count;
-            mutexUnlock(&t.mutex);
+            mutexUnlockInternal(&t.mutex);
             future.destroy(gpa);
             start(context.ptr, result.ptr);
             return null;
 @@ -2033,7 +2052,7 @@ fn async(
     t.run_queue.prepend(&future.runnable.node);
-    mutexUnlock(&t.mutex);
+    mutexUnlockInternal(&t.mutex);
     condSignal(&t.cond);
     return @ptrCast(future);
 }
 @@ -2056,8 +2075,8 @@ fn concurrent(
     };
     errdefer future.destroy(gpa);
-    mutexLockUncancelable(&t.mutex);
+    mutexLockInternal(&t.mutex);
-    defer mutexUnlock(&t.mutex);
+    defer mutexUnlockInternal(&t.mutex);
     const busy_count = t.busy_count;
 @@ -2101,12 +2120,12 @@ fn groupAsync(
         error.OutOfMemory => return groupAsyncEager(start, context.ptr),
     };
-    mutexLockUncancelable(&t.mutex);
+    mutexLockInternal(&t.mutex);
     const busy_count = t.busy_count;
     if (busy_count >= @intFromEnum(t.async_limit)) {
-        mutexUnlock(&t.mutex);
+        mutexUnlockInternal(&t.mutex);
         task.destroy(gpa);
         return groupAsyncEager(start, context.ptr);
     }
 @@ -2119,7 +2138,7 @@ fn groupAsync(
         const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch {
             t.wait_group.finish();
             t.busy_count = busy_count;
-            mutexUnlock(&t.mutex);
+            mutexUnlockInternal(&t.mutex);
             task.destroy(gpa);
             return groupAsyncEager(start, context.ptr);
         };
 @@ -2136,7 +2155,7 @@ fn groupAsync(
     }, .monotonic);
     t.run_queue.prepend(&task.runnable.node);
-    mutexUnlock(&t.mutex);
+    mutexUnlockInternal(&t.mutex);
     condSignal(&t.cond);
 }
 fn groupAsyncEager(
 @@ -2201,8 +2220,8 @@ fn groupConcurrent(
     };
     errdefer task.destroy(gpa);
-    mutexLockUncancelable(&t.mutex);
+    mutexLockInternal(&t.mutex);
-    defer mutexUnlock(&t.mutex);
+    defer mutexUnlockInternal(&t.mutex);
     const busy_count = t.busy_count;
 @@ -3838,8 +3857,8 @@ fn fileStatWindows(userdata: ?*anyopaque, file: File) File.StatError!File.Stat {
 fn systemBasicInformation(t: *Threaded) ?*const windows.SYSTEM_BASIC_INFORMATION {
     if (!t.system_basic_information.initialized.load(.acquire)) {
-        mutexLockUncancelable(&t.mutex);
+        mutexLockInternal(&t.mutex);
-        defer mutexUnlock(&t.mutex);
+        defer mutexUnlockInternal(&t.mutex);
         switch (windows.ntdll.NtQuerySystemInformation(
             .SystemBasicInformation,
 @@ -14373,8 +14392,8 @@ const WindowsEnvironStrings = struct {
 };
 fn scanEnviron(t: *Threaded) void {
-    mutexLockUncancelable(&t.mutex);
+    mutexLockInternal(&t.mutex);
-    defer mutexUnlock(&t.mutex);
+    defer mutexUnlockInternal(&t.mutex);
     if (t.environ.initialized) return;
     t.environ.initialized = true;
 @@ -14729,8 +14748,8 @@ fn spawnPosix(t: *Threaded, options: process.SpawnOptions) process.SpawnError!Sp
 fn getDevNullFd(t: *Threaded) !posix.fd_t {
     {
-        mutexLockUncancelable(&t.mutex);
+        mutexLockInternal(&t.mutex);
-        defer mutexUnlock(&t.mutex);
+        defer mutexUnlockInternal(&t.mutex);
         if (t.null_file.fd != -1) return t.null_file.fd;
     }
     const mode: u32 = 0;
 @@ -14741,8 +14760,8 @@ fn getDevNullFd(t: *Threaded) !posix.fd_t {
             .SUCCESS => {
                 syscall.finish();
                 const fresh_fd: posix.fd_t = @intCast(rc);
-                mutexLockUncancelable(&t.mutex); // Another thread might have won the race.
+                mutexLockInternal(&t.mutex); // Another thread might have won the race.
-                defer mutexUnlock(&t.mutex);
+                defer mutexUnlockInternal(&t.mutex);
                 if (t.null_file.fd != -1) {
                     posix.close(fresh_fd);
                     return t.null_file.fd;
 @@ -15402,8 +15421,8 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro
 fn getCngHandle(t: *Threaded) Io.RandomSecureError!windows.HANDLE {
     {
-        mutexLockUncancelable(&t.mutex);
+        mutexLockInternal(&t.mutex);
-        defer mutexUnlock(&t.mutex);
+        defer mutexUnlockInternal(&t.mutex);
         if (t.random_file.handle) |handle| return handle;
     }
 @@ -15437,8 +15456,8 @@ fn getCngHandle(t: *Threaded) Io.RandomSecureError!windows.HANDLE {
     )) {
         .SUCCESS => {
             syscall.finish();
-            mutexLockUncancelable(&t.mutex); // Another thread might have won the race.
+            mutexLockInternal(&t.mutex); // Another thread might have won the race.
-            defer mutexUnlock(&t.mutex);
+            defer mutexUnlockInternal(&t.mutex);
             if (t.random_file.handle) |prev_handle| {
                 windows.CloseHandle(fresh_handle);
                 return prev_handle;
 @@ -15458,8 +15477,8 @@ fn getCngHandle(t: *Threaded) Io.RandomSecureError!windows.HANDLE {
 fn getNulHandle(t: *Threaded) !windows.HANDLE {
     {
-        mutexLockUncancelable(&t.mutex);
+        mutexLockInternal(&t.mutex);
-        defer mutexUnlock(&t.mutex);
+        defer mutexUnlockInternal(&t.mutex);
         if (t.null_file.handle) |handle| return handle;
     }
 @@ -15505,8 +15524,8 @@ fn getNulHandle(t: *Threaded) !windows.HANDLE {
     )) {
         .SUCCESS => {
             syscall.finish();
-            mutexLockUncancelable(&t.mutex); // Another thread might have won the race.
+            mutexLockInternal(&t.mutex); // Another thread might have won the race.
-            defer mutexUnlock(&t.mutex);
+            defer mutexUnlockInternal(&t.mutex);
             if (t.null_file.handle) |prev_handle| {
                 windows.CloseHandle(fresh_handle);
                 return prev_handle;
 @@ -16551,15 +16570,15 @@ fn random(userdata: ?*anyopaque, buffer: []u8) void {
 }
 fn randomMainThread(t: *Threaded, buffer: []u8) void {
-    mutexLockUncancelable(&t.mutex);
+    mutexLockInternal(&t.mutex);
-    defer mutexUnlock(&t.mutex);
+    defer mutexUnlockInternal(&t.mutex);
     if (!t.csprng.isInitialized()) {
         @branchHint(.unlikely);
         var seed: [Csprng.seed_len]u8 = undefined;
         {
-            mutexUnlock(&t.mutex);
+            mutexUnlockInternal(&t.mutex);
-            defer mutexLockUncancelable(&t.mutex);
+            defer mutexLockInternal(&t.mutex);
             const prev = swapCancelProtection(t, .blocked);
             defer _ = swapCancelProtection(t, prev);
 @@ -16744,8 +16763,8 @@ fn randomSecure(userdata: ?*anyopaque, buffer: []u8) Io.RandomSecureError!void {
 fn getRandomFd(t: *Threaded) Io.RandomSecureError!posix.fd_t {
     {
-        mutexLockUncancelable(&t.mutex);
+        mutexLockInternal(&t.mutex);
-        defer mutexUnlock(&t.mutex);
+        defer mutexUnlockInternal(&t.mutex);
         if (t.random_file.fd == -2) return error.EntropyUnavailable;
         if (t.random_file.fd != -1) return t.random_file.fd;
 @@ -16785,8 +16804,8 @@ fn getRandomFd(t: *Threaded) Io.RandomSecureError!posix.fd_t {
                     .SUCCESS => {
                         syscall.finish();
                         if (!statx.mask.TYPE) return error.EntropyUnavailable;
-                        mutexLockUncancelable(&t.mutex); // Another thread might have won the race.
+                        mutexLockInternal(&t.mutex); // Another thread might have won the race.
-                        defer mutexUnlock(&t.mutex);
+                        defer mutexUnlockInternal(&t.mutex);
                         if (t.random_file.fd >= 0) {
                             posix.close(fd);
                             return t.random_file.fd;
 @@ -16813,8 +16832,8 @@ fn getRandomFd(t: *Threaded) Io.RandomSecureError!posix.fd_t {
                 switch (posix.errno(fstat_sym(fd, &stat))) {
                     .SUCCESS => {
                         syscall.finish();
-                        mutexLockUncancelable(&t.mutex); // Another thread might have won the race.
+                        mutexLockInternal(&t.mutex); // Another thread might have won the race.
-                        defer mutexUnlock(&t.mutex);
+                        defer mutexUnlockInternal(&t.mutex);
                         if (t.random_file.fd >= 0) {
                             posix.close(fd);
                             return t.random_file.fd;
 @@ -16947,8 +16966,8 @@ const parking_futex = struct {
         var status_buf: std.atomic.Value(Thread.Status) = undefined;
         {
-            mutexLockUncancelable(&bucket.mutex);
+            mutexLockInternal(&bucket.mutex);
-            defer mutexUnlock(&bucket.mutex);
+            defer mutexUnlockInternal(&bucket.mutex);
             _ = bucket.num_waiters.fetchAdd(1, .acquire);
 @@ -17017,8 +17036,8 @@ const parking_futex = struct {
                 .parked => {
                     // We saw a timeout and updated our own status from `.parked` to `.none`. It is
                     // our responsibility to remove `waiter` from `bucket`.
-                    mutexLockUncancelable(&bucket.mutex);
+                    mutexLockInternal(&bucket.mutex);
-                    defer mutexUnlock(&bucket.mutex);
+                    defer mutexUnlockInternal(&bucket.mutex);
                     bucket.waiters.remove(&waiter.node);
                     assert(bucket.num_waiters.fetchSub(1, .monotonic) > 0);
                 },
 @@ -17057,8 +17076,8 @@ const parking_futex = struct {
         // of the critical section. This forms a singly-linked list of waiters using `Waiter.node.next`.
         var waking_head: ?*std.DoublyLinkedList.Node = null;
         {
-            mutexLockUncancelable(&bucket.mutex);
+            mutexLockInternal(&bucket.mutex);
-            defer mutexUnlock(&bucket.mutex);
+            defer mutexUnlockInternal(&bucket.mutex);
             var num_removed: u32 = 0;
             var it = bucket.waiters.first;
 @@ -17113,8 +17132,8 @@ const parking_futex = struct {
     fn removeCanceledWaiter(waiter: *Waiter) void {
         const bucket = bucketForAddress(waiter.address);
-        mutexLockUncancelable(&bucket.mutex);
+        mutexLockInternal(&bucket.mutex);
-        defer mutexUnlock(&bucket.mutex);
+        defer mutexUnlockInternal(&bucket.mutex);
         bucket.waiters.remove(&waiter.node);
         assert(bucket.num_waiters.fetchSub(1, .monotonic) > 0);
         waiter.done.store(true, .release); // potentially invalidates `waiter.*`
 @@ -18163,8 +18182,8 @@ fn condWait(cond: *Condition, mutex: *Mutex) void {
         assert(prev_state.waiters < std.math.maxInt(u16)); // overflow caused by too many waiters
     }
-    mutexUnlock(mutex);
+    mutexUnlockInternal(mutex);
-    defer mutexLockUncancelable(mutex);
+    defer mutexLockInternal(mutex);
     while (true) {
         Thread.futexWaitUncancelable(&cond.epoch.raw, epoch, null);
 @@ -18189,28 +18208,13 @@ const Mutex = if (!is_windows) Io.Mutex else struct {
     const init: @This() = .{ .srwlock = .{} };
 };
-/// Same as `Io.Mutex.lockUncancelable` but avoids the VTable.
+fn mutexLockInternal(m: *Mutex) void {
-fn mutexLock(m: *Io.Mutex) Io.Cancelable!void {
+    if (is_windows) return windows.ntdll.RtlAcquireSRWLockExclusive(&m.srwlock);
-    const initial_state = m.state.cmpxchgWeak(
+    return mutexLock(m);
-        .unlocked,
-        .locked_once,
-        .acquire,
-        .monotonic,
-    ) orelse {
-        @branchHint(.likely);
-        return;
-    };
-    if (initial_state == .contended) {
-        try Thread.futexWait(@ptrCast(&m.state.raw), @intFromEnum(Io.Mutex.State.contended), null);
-    }
-    while (m.state.swap(.contended, .acquire) != .unlocked) {
-        try Thread.futexWait(@ptrCast(&m.state.raw), @intFromEnum(Io.Mutex.State.contended), null);
-    }
 }
 /// Same as `Io.Mutex.lockUncancelable` but avoids the VTable.
-fn mutexLockUncancelable(m: *Mutex) void {
+pub fn mutexLock(m: *Io.Mutex) void {
-    if (is_windows) return windows.ntdll.RtlAcquireSRWLockExclusive(&m.srwlock);
     const initial_state = m.state.cmpxchgWeak(
         .unlocked,
         .locked_once,
 @@ -18228,9 +18232,13 @@ fn mutexLockUncancelable(m: *Mutex) void {
     }
 }
-/// Same as `Io.Mutex.unlock` but avoids the VTable.
+fn mutexUnlockInternal(m: *Mutex) void {
-fn mutexUnlock(m: *Mutex) void {
     if (is_windows) return windows.ntdll.RtlReleaseSRWLockExclusive(&m.srwlock);
+    return mutexUnlock(m);
+}
+/// Same as `Io.Mutex.unlock` but avoids the VTable.
+pub fn mutexUnlock(m: *Io.Mutex) void {
     switch (m.state.swap(.unlocked, .release)) {
         .unlocked => unreachable,
         .locked_once => {},

lib/std/Thread.zig+3-11

 @@ -14,13 +14,9 @@ const posix = std.posix;
 const windows = std.os.windows;
 const testing = std.testing;
-pub const Futex = @import("Thread/Futex.zig");
+pub const Mutex = struct {
-pub const Mutex = @import("Thread/Mutex.zig");
+    pub const Recursive = @import("Thread/Mutex/Recursive.zig");
-pub const Semaphore = @import("Thread/Semaphore.zig");
+};
-pub const Condition = @import("Thread/Condition.zig");
-pub const RwLock = @import("Thread/RwLock.zig");
-pub const Pool = @compileError("deprecated; consider using 'std.Io.Group' with 'std.Io.Threaded'");
 pub const use_pthreads = native_os != .windows and native_os != .wasi and builtin.link_libc;
 @@ -1609,11 +1605,7 @@ test "setName, getName" {
 }
 test {
-    _ = Futex;
     _ = Mutex;
-    _ = Semaphore;
-    _ = Condition;
-    _ = RwLock;
 }
 fn testIncrementNotify(io: Io, value: *usize, event: *Io.Event) void {

lib/std/Thread/Condition.zig deleted-683

...	@@ -1,683 +0,0 @@
1	//! Condition variables are used with a Mutex to efficiently wait for an arbitrary condition to occur.
2	//! It does this by atomically unlocking the mutex, blocking the thread until notified, and finally re-locking the mutex.
3	//! Condition can be statically initialized and is at most `@sizeOf(u64)` large.
4	//!
5	//! Example:
6	//! ```
7	//! var m = Mutex{};
8	//! var c = Condition{};
9	//! var predicate = false;
10	//!
11	//! fn consumer() void {
12	//! m.lock();
13	//! defer m.unlock();
14	//!
15	//! while (!predicate) {
16	//! c.wait(&m);
17	//! }
18	//! }
19	//!
20	//! fn producer() void {
21	//! {
22	//! m.lock();
23	//! defer m.unlock();
24	//! predicate = true;
25	//! }
26	//! c.signal();
27	//! }
28	//!
29	//! const thread = try std.Thread.spawn(.{}, producer, .{});
30	//! consumer();
31	//! thread.join();
32	//! ```
33	//!
34	//! Note that condition variables can only reliably unblock threads that are sequenced before them using the same Mutex.
35	//! This means that the following is allowed to deadlock:
36	//! ```
37	//! thread-1: mutex.lock()
38	//! thread-1: condition.wait(&mutex)
39	//!
40	//! thread-2: // mutex.lock() (without this, the following signal may not see the waiting thread-1)
41	//! thread-2: // mutex.unlock() (this is optional for correctness once locked above, as signal can be called while holding the mutex)
42	//! thread-2: condition.signal()
43	//! ```
44
45	const std = @import("../std.zig");
46	const builtin = @import("builtin");
47	const Condition = @This();
48	const Mutex = std.Thread.Mutex;
49
50	const os = std.os;
51	const assert = std.debug.assert;
52	const testing = std.testing;
53	const Futex = std.Thread.Futex;
54
55	impl: Impl = .{},
56
57	/// Atomically releases the Mutex, blocks the caller thread, then re-acquires the Mutex on return.
58	/// "Atomically" here refers to accesses done on the Condition after acquiring the Mutex.
59	///
60	/// The Mutex must be locked by the caller's thread when this function is called.
61	/// A Mutex can have multiple Conditions waiting with it concurrently, but not the opposite.
62	/// It is undefined behavior for multiple threads to wait ith different mutexes using the same Condition concurrently.
63	/// Once threads have finished waiting with one Mutex, the Condition can be used to wait with another Mutex.
64	///
65	/// A blocking call to wait() is unblocked from one of the following conditions:
66	/// - a spurious ("at random") wake up occurs
67	/// - a future call to `signal()` or `broadcast()` which has acquired the Mutex and is sequenced after this `wait()`.
68	///
69	/// Given wait() can be interrupted spuriously, the blocking condition should be checked continuously
70	/// irrespective of any notifications from `signal()` or `broadcast()`.
71	pub fn wait(self: Condition, mutex: Mutex) void {
72	self.impl.wait(mutex, null) catch \|err\| switch (err) {
73	error.Timeout => unreachable, // no timeout provided so we shouldn't have timed-out
74	};
75	}
76
77	/// Atomically releases the Mutex, blocks the caller thread, then re-acquires the Mutex on return.
78	/// "Atomically" here refers to accesses done on the Condition after acquiring the Mutex.
79	///
80	/// The Mutex must be locked by the caller's thread when this function is called.
81	/// A Mutex can have multiple Conditions waiting with it concurrently, but not the opposite.
82	/// It is undefined behavior for multiple threads to wait ith different mutexes using the same Condition concurrently.
83	/// Once threads have finished waiting with one Mutex, the Condition can be used to wait with another Mutex.
84	///
85	/// A blocking call to `timedWait()` is unblocked from one of the following conditions:
86	/// - a spurious ("at random") wake occurs
87	/// - the caller was blocked for around `timeout_ns` nanoseconds, in which `error.Timeout` is returned.
88	/// - a future call to `signal()` or `broadcast()` which has acquired the Mutex and is sequenced after this `timedWait()`.
89	///
90	/// Given `timedWait()` can be interrupted spuriously, the blocking condition should be checked continuously
91	/// irrespective of any notifications from `signal()` or `broadcast()`.
92	pub fn timedWait(self: Condition, mutex: Mutex, timeout_ns: u64) error{Timeout}!void {
93	return self.impl.wait(mutex, timeout_ns);
94	}
95
96	/// Unblocks at least one thread blocked in a call to `wait()` or `timedWait()` with a given Mutex.
97	/// The blocked thread must be sequenced before this call with respect to acquiring the same Mutex in order to be observable for unblocking.
98	/// `signal()` can be called with or without the relevant Mutex being acquired and have no "effect" if there's no observable blocked threads.
99	pub fn signal(self: *Condition) void {
100	self.impl.wake(.one);
101	}
102
103	/// Unblocks all threads currently blocked in a call to `wait()` or `timedWait()` with a given Mutex.
104	/// The blocked threads must be sequenced before this call with respect to acquiring the same Mutex in order to be observable for unblocking.
105	/// `broadcast()` can be called with or without the relevant Mutex being acquired and have no "effect" if there's no observable blocked threads.
106	pub fn broadcast(self: *Condition) void {
107	self.impl.wake(.all);
108	}
109
110	const Impl = Impl: {
111	if (builtin.single_threaded) break :Impl SingleThreadedImpl;
112	if (builtin.os.tag == .windows) break :Impl WindowsImpl;
113
114	if (builtin.os.tag.isDarwin() or
115	builtin.target.os.tag == .linux or
116	builtin.target.os.tag == .freebsd or
117	builtin.target.os.tag == .openbsd or
118	builtin.target.os.tag == .dragonfly or
119	builtin.target.cpu.arch.isWasm())
120	{
121	// Futex is the system's synchronization primitive; use that.
122	break :Impl FutexImpl;
123	}
124
125	if (std.Thread.use_pthreads) {
126	// This system doesn't have a futex primitive, so `std.Thread.Futex` is using `PosixImpl`,
127	// which implements futex on top of pthread mutexes and conditions. Therefore, instead
128	// of going through that long inefficient path, just use pthread condition variable directly.
129	break :Impl PosixImpl;
130	}
131
132	break :Impl FutexImpl;
133	};
134
135	const Notify = enum {
136	one, // wake up only one thread
137	all, // wake up all threads
138	};
139
140	const SingleThreadedImpl = struct {
141	fn wait(self: Impl, mutex: Mutex, timeout: ?u64) error{Timeout}!void {
142	_ = self;
143	_ = mutex;
144	// There are no other threads to wake us up.
145	// So if we wait without a timeout we would never wake up.
146	assert(timeout != null); // Deadlock detected.
147	return error.Timeout;
148	}
149
150	fn wake(self: *Impl, comptime notify: Notify) void {
151	// There are no other threads to wake up.
152	_ = self;
153	_ = notify;
154	}
155	};
156
157	const WindowsImpl = struct {
158	condition: os.windows.CONDITION_VARIABLE = .{},
159
160	fn wait(self: Impl, mutex: Mutex, timeout: ?u64) error{Timeout}!void {
161	var timeout_overflowed = false;
162	var timeout_ms: os.windows.DWORD = os.windows.INFINITE;
163
164	if (timeout) \|timeout_ns\| {
165	// Round the nanoseconds to the nearest millisecond,
166	// then saturating cast it to windows DWORD for use in kernel32 call.
167	const ms = (timeout_ns +\| (std.time.ns_per_ms / 2)) / std.time.ns_per_ms;
168	timeout_ms = std.math.cast(os.windows.DWORD, ms) orelse std.math.maxInt(os.windows.DWORD);
169
170	// Track if the timeout overflowed into INFINITE and make sure not to wait forever.
171	if (timeout_ms == os.windows.INFINITE) {
172	timeout_overflowed = true;
173	timeout_ms -= 1;
174	}
175	}
176
177	if (builtin.mode == .Debug) {
178	// The internal state of the DebugMutex needs to be handled here as well.
179	mutex.impl.locking_thread.store(0, .unordered);
180	}
181	const rc = os.windows.kernel32.SleepConditionVariableSRW(
182	&self.condition,
183	if (builtin.mode == .Debug) &mutex.impl.impl.srwlock else &mutex.impl.srwlock,
184	timeout_ms,
185	0, // the srwlock was assumed to acquired in exclusive mode not shared
186	);
187	if (builtin.mode == .Debug) {
188	// The internal state of the DebugMutex needs to be handled here as well.
189	mutex.impl.locking_thread.store(std.Thread.getCurrentId(), .unordered);
190	}
191
192	// Return error.Timeout if we know the timeout elapsed correctly.
193	if (rc == os.windows.FALSE) {
194	assert(os.windows.GetLastError() == .TIMEOUT);
195	if (!timeout_overflowed) return error.Timeout;
196	}
197	}
198
199	fn wake(self: *Impl, comptime notify: Notify) void {
200	switch (notify) {
201	.one => os.windows.ntdll.RtlWakeConditionVariable(&self.condition),
202	.all => os.windows.ntdll.RtlWakeAllConditionVariable(&self.condition),
203	}
204	}
205	};
206
207	const FutexImpl = struct {
208	state: std.atomic.Value(u32) = std.atomic.Value(u32).init(0),
209	epoch: std.atomic.Value(u32) = std.atomic.Value(u32).init(0),
210
211	const one_waiter = 1;
212	const waiter_mask = 0xffff;
213
214	const one_signal = 1 << 16;
215	const signal_mask = 0xffff << 16;
216
217	fn wait(self: Impl, mutex: Mutex, timeout: ?u64) error{Timeout}!void {
218	// Observe the epoch, then check the state again to see if we should wake up.
219	// The epoch must be observed before we check the state or we could potentially miss a wake() and deadlock:
220	//
221	// - T1: s = LOAD(&state)
222	// - T2: UPDATE(&s, signal)
223	// - T2: UPDATE(&epoch, 1) + FUTEX_WAKE(&epoch)
224	// - T1: e = LOAD(&epoch) (was reordered after the state load)
225	// - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed the state update + the epoch change)
226	//
227	// Acquire barrier to ensure the epoch load happens before the state load.
228	var epoch = self.epoch.load(.acquire);
229	var state = self.state.fetchAdd(one_waiter, .monotonic);
230	assert(state & waiter_mask != waiter_mask);
231	state += one_waiter;
232
233	mutex.unlock();
234	defer mutex.lock();
235
236	var futex_deadline = Futex.Deadline.init(timeout);
237
238	while (true) {
239	futex_deadline.wait(&self.epoch, epoch) catch \|err\| switch (err) {
240	// On timeout, we must decrement the waiter we added above.
241	error.Timeout => {
242	while (true) {
243	// If there's a signal when we're timing out, consume it and report being woken up instead.
244	// Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return.
245	while (state & signal_mask != 0) {
246	const new_state = state - one_waiter - one_signal;
247	state = self.state.cmpxchgWeak(state, new_state, .acquire, .monotonic) orelse return;
248	}
249
250	// Remove the waiter we added and officially return timed out.
251	const new_state = state - one_waiter;
252	state = self.state.cmpxchgWeak(state, new_state, .monotonic, .monotonic) orelse return err;
253	}
254	},
255	};
256
257	epoch = self.epoch.load(.acquire);
258	state = self.state.load(.monotonic);
259
260	// Try to wake up by consuming a signal and decremented the waiter we added previously.
261	// Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return.
262	while (state & signal_mask != 0) {
263	const new_state = state - one_waiter - one_signal;
264	state = self.state.cmpxchgWeak(state, new_state, .acquire, .monotonic) orelse return;
265	}
266	}
267	}
268
269	fn wake(self: *Impl, comptime notify: Notify) void {
270	var state = self.state.load(.monotonic);
271	while (true) {
272	const waiters = (state & waiter_mask) / one_waiter;
273	const signals = (state & signal_mask) / one_signal;
274
275	// Reserves which waiters to wake up by incrementing the signals count.
276	// Therefore, the signals count is always less than or equal to the waiters count.
277	// We don't need to Futex.wake if there's nothing to wake up or if other wake() threads have reserved to wake up the current waiters.
278	const wakeable = waiters - signals;
279	if (wakeable == 0) {
280	return;
281	}
282
283	const to_wake = switch (notify) {
284	.one => 1,
285	.all => wakeable,
286	};
287
288	// Reserve the amount of waiters to wake by incrementing the signals count.
289	// Release barrier ensures code before the wake() happens before the signal it posted and consumed by the wait() threads.
290	const new_state = state + (one_signal * to_wake);
291	state = self.state.cmpxchgWeak(state, new_state, .release, .monotonic) orelse {
292	// Wake up the waiting threads we reserved above by changing the epoch value.
293	// NOTE: a waiting thread could miss a wake up if exactly ((1<<32)-1) wake()s happen between it observing the epoch and sleeping on it.
294	// This is very unlikely due to how many precise amount of Futex.wake() calls that would be between the waiting thread's potential preemption.
295	//
296	// Release barrier ensures the signal being added to the state happens before the epoch is changed.
297	// If not, the waiting thread could potentially deadlock from missing both the state and epoch change:
298	//
299	// - T2: UPDATE(&epoch, 1) (reordered before the state change)
300	// - T1: e = LOAD(&epoch)
301	// - T1: s = LOAD(&state)
302	// - T2: UPDATE(&state, signal) + FUTEX_WAKE(&epoch)
303	// - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed both epoch change and state change)
304	_ = self.epoch.fetchAdd(1, .release);
305	Futex.wake(&self.epoch, to_wake);
306	return;
307	};
308	}
309	}
310	};
311
312	const PosixImpl = struct {
313	cond: std.c.pthread_cond_t = .{},
314
315	fn wait(self: Impl, mutex: Mutex, timeout: ?u64) error{Timeout}!void {
316	if (builtin.mode == .Debug) {
317	mutex.impl.locking_thread.store(0, .unordered);
318	}
319	defer if (builtin.mode == .Debug) {
320	mutex.impl.locking_thread.store(std.Thread.getCurrentId(), .unordered);
321	};
322
323	const mtx = if (builtin.mode == .Debug) &mutex.impl.impl.mutex else &mutex.impl.mutex;
324
325	if (timeout) \|t\| {
326	switch (std.c.pthread_cond_timedwait(&self.cond, mtx, &.{
327	.sec = @intCast(@divFloor(t, std.time.ns_per_s)),
328	.nsec = @intCast(@mod(t, std.time.ns_per_s)),
329	})) {
330	.SUCCESS => return,
331	.TIMEDOUT => return error.Timeout,
332	else => unreachable,
333	}
334	}
335
336	assert(std.c.pthread_cond_wait(&self.cond, mtx) == .SUCCESS);
337	}
338
339	fn wake(self: *Impl, comptime notify: Notify) void {
340	assert(switch (notify) {
341	.one => std.c.pthread_cond_signal(&self.cond),
342	.all => std.c.pthread_cond_broadcast(&self.cond),
343	} == .SUCCESS);
344	}
345	};
346
347	test "smoke test" {
348	var mutex = Mutex{};
349	var cond = Condition{};
350
351	// Try to wake outside the mutex
352	defer cond.signal();
353	defer cond.broadcast();
354
355	mutex.lock();
356	defer mutex.unlock();
357
358	// Try to wait with a timeout (should not deadlock)
359	try testing.expectError(error.Timeout, cond.timedWait(&mutex, 0));
360	try testing.expectError(error.Timeout, cond.timedWait(&mutex, std.time.ns_per_ms));
361
362	// Try to wake inside the mutex.
363	cond.signal();
364	cond.broadcast();
365	}
366
367	// Inspired from: https://github.com/Amanieu/parking_lot/pull/129
368	test "wait and signal" {
369	// This test requires spawning threads
370	if (builtin.single_threaded) {
371	return error.SkipZigTest;
372	}
373
374	const io = testing.io;
375
376	const num_threads = 4;
377
378	const MultiWait = struct {
379	mutex: Mutex = .{},
380	cond: Condition = .{},
381	threads: [num_threads]std.Thread = undefined,
382	spawn_count: std.math.IntFittingRange(0, num_threads) = 0,
383
384	fn run(self: *@This()) void {
385	self.mutex.lock();
386	defer self.mutex.unlock();
387	self.spawn_count += 1;
388
389	self.cond.wait(&self.mutex);
390	self.cond.timedWait(&self.mutex, std.time.ns_per_ms) catch {};
391	self.cond.signal();
392	}
393	};
394
395	var multi_wait = MultiWait{};
396	for (&multi_wait.threads) \|*t\| {
397	t.* = try std.Thread.spawn(.{}, MultiWait.run, .{&multi_wait});
398	}
399
400	while (true) {
401	try std.Io.Clock.Duration.sleep(.{ .clock = .awake, .raw = .fromMilliseconds(100) }, io);
402
403	multi_wait.mutex.lock();
404	defer multi_wait.mutex.unlock();
405	// Make sure all of the threads have finished spawning to avoid a deadlock.
406	if (multi_wait.spawn_count == num_threads) break;
407	}
408
409	multi_wait.cond.signal();
410	for (multi_wait.threads) \|t\| {
411	t.join();
412	}
413	}
414
415	test signal {
416	// This test requires spawning threads
417	if (builtin.single_threaded) {
418	return error.SkipZigTest;
419	}
420
421	const io = testing.io;
422
423	const num_threads = 4;
424
425	const SignalTest = struct {
426	mutex: Mutex = .{},
427	cond: Condition = .{},
428	notified: bool = false,
429	threads: [num_threads]std.Thread = undefined,
430	spawn_count: std.math.IntFittingRange(0, num_threads) = 0,
431
432	fn run(self: *@This()) void {
433	self.mutex.lock();
434	defer self.mutex.unlock();
435	self.spawn_count += 1;
436
437	// Use timedWait() a few times before using wait()
438	// to test multiple threads timing out frequently.
439	var i: usize = 0;
440	while (!self.notified) : (i +%= 1) {
441	if (i < 5) {
442	self.cond.timedWait(&self.mutex, 1) catch {};
443	} else {
444	self.cond.wait(&self.mutex);
445	}
446	}
447
448	// Once we received the signal, notify another thread (inside the lock).
449	assert(self.notified);
450	self.cond.signal();
451	}
452	};
453
454	var signal_test = SignalTest{};
455	for (&signal_test.threads) \|*t\| {
456	t.* = try std.Thread.spawn(.{}, SignalTest.run, .{&signal_test});
457	}
458
459	while (true) {
460	try std.Io.Clock.Duration.sleep(.{ .clock = .awake, .raw = .fromMilliseconds(10) }, io);
461
462	signal_test.mutex.lock();
463	defer signal_test.mutex.unlock();
464	// Make sure at least one thread has finished spawning to avoid testing nothing.
465	if (signal_test.spawn_count > 0) break;
466	}
467
468	{
469	// Wake up one of them (outside the lock) after setting notified=true.
470	defer signal_test.cond.signal();
471
472	signal_test.mutex.lock();
473	defer signal_test.mutex.unlock();
474
475	try testing.expect(!signal_test.notified);
476	signal_test.notified = true;
477	}
478
479	for (signal_test.threads) \|t\| {
480	t.join();
481	}
482	}
483
484	test "multi signal" {
485	// This test requires spawning threads
486	if (builtin.single_threaded) {
487	return error.SkipZigTest;
488	}
489
490	const num_threads = 4;
491	const num_iterations = 4;
492
493	const Paddle = struct {
494	mutex: Mutex = .{},
495	cond: Condition = .{},
496	value: u32 = 0,
497
498	fn hit(self: *@This()) void {
499	defer self.cond.signal();
500
501	self.mutex.lock();
502	defer self.mutex.unlock();
503
504	self.value += 1;
505	}
506
507	fn run(self: @This(), hit_to: @This()) !void {
508	self.mutex.lock();
509	defer self.mutex.unlock();
510
511	var current: u32 = 0;
512	while (current < num_iterations) : (current += 1) {
513	// Wait for the value to change from hit()
514	while (self.value == current) {
515	self.cond.wait(&self.mutex);
516	}
517
518	// hit the next paddle
519	try testing.expectEqual(self.value, current + 1);
520	hit_to.hit();
521	}
522	}
523	};
524
525	var paddles = [_]Paddle{.{}} ** num_threads;
526	var threads = [_]std.Thread{undefined} ** num_threads;
527
528	// Create a circle of paddles which hit each other
529	for (&threads, 0..) \|*t, i\| {
530	const paddle = &paddles[i];
531	const hit_to = &paddles[(i + 1) % paddles.len];
532	t.* = try std.Thread.spawn(.{}, Paddle.run, .{ paddle, hit_to });
533	}
534
535	// Hit the first paddle and wait for them all to complete by hitting each other for num_iterations.
536	paddles[0].hit();
537	for (threads) \|t\| t.join();
538
539	// The first paddle will be hit one last time by the last paddle.
540	for (paddles, 0..) \|p, i\| {
541	const expected = @as(u32, num_iterations) + @intFromBool(i == 0);
542	try testing.expectEqual(p.value, expected);
543	}
544	}
545
546	test broadcast {
547	// This test requires spawning threads
548	if (builtin.single_threaded) {
549	return error.SkipZigTest;
550	}
551
552	const num_threads = 10;
553
554	const BroadcastTest = struct {
555	mutex: Mutex = .{},
556	cond: Condition = .{},
557	completed: Condition = .{},
558	count: usize = 0,
559	threads: [num_threads]std.Thread = undefined,
560
561	fn run(self: *@This()) void {
562	self.mutex.lock();
563	defer self.mutex.unlock();
564
565	// The last broadcast thread to start tells the main test thread it's completed.
566	self.count += 1;
567	if (self.count == num_threads) {
568	self.completed.signal();
569	}
570
571	// Waits for the count to reach zero after the main test thread observes it at num_threads.
572	// Tries to use timedWait() a bit before falling back to wait() to test multiple threads timing out.
573	var i: usize = 0;
574	while (self.count != 0) : (i +%= 1) {
575	if (i < 10) {
576	self.cond.timedWait(&self.mutex, 1) catch {};
577	} else {
578	self.cond.wait(&self.mutex);
579	}
580	}
581	}
582	};
583
584	var broadcast_test = BroadcastTest{};
585	for (&broadcast_test.threads) \|*t\| {
586	t.* = try std.Thread.spawn(.{}, BroadcastTest.run, .{&broadcast_test});
587	}
588
589	{
590	broadcast_test.mutex.lock();
591	defer broadcast_test.mutex.unlock();
592
593	// Wait for all the broadcast threads to spawn.
594	// timedWait() to detect any potential deadlocks.
595	while (broadcast_test.count != num_threads) {
596	broadcast_test.completed.timedWait(
597	&broadcast_test.mutex,
598	1 * std.time.ns_per_s,
599	) catch {};
600	}
601
602	// Reset the counter and wake all the threads to exit.
603	broadcast_test.count = 0;
604	broadcast_test.cond.broadcast();
605	}
606
607	for (broadcast_test.threads) \|t\| {
608	t.join();
609	}
610	}
611
612	test "broadcasting - wake all threads" {
613	// Tests issue #12877
614	// This test requires spawning threads
615	if (builtin.single_threaded) {
616	return error.SkipZigTest;
617	}
618
619	var num_runs: usize = 1;
620	const num_threads = 10;
621
622	while (num_runs > 0) : (num_runs -= 1) {
623	const BroadcastTest = struct {
624	mutex: Mutex = .{},
625	cond: Condition = .{},
626	completed: Condition = .{},
627	count: usize = 0,
628	thread_id_to_wake: usize = 0,
629	threads: [num_threads]std.Thread = undefined,
630	wakeups: usize = 0,
631
632	fn run(self: *@This(), thread_id: usize) void {
633	self.mutex.lock();
634	defer self.mutex.unlock();
635
636	// The last broadcast thread to start tells the main test thread it's completed.
637	self.count += 1;
638	if (self.count == num_threads) {
639	self.completed.signal();
640	}
641
642	while (self.thread_id_to_wake != thread_id) {
643	self.cond.timedWait(&self.mutex, 1 * std.time.ns_per_s) catch {};
644	self.wakeups += 1;
645	}
646	if (self.thread_id_to_wake <= num_threads) {
647	// Signal next thread to wake up.
648	self.thread_id_to_wake += 1;
649	self.cond.broadcast();
650	}
651	}
652	};
653
654	var broadcast_test = BroadcastTest{};
655	var thread_id: usize = 1;
656	for (&broadcast_test.threads) \|*t\| {
657	t.* = try std.Thread.spawn(.{}, BroadcastTest.run, .{ &broadcast_test, thread_id });
658	thread_id += 1;
659	}
660
661	{
662	broadcast_test.mutex.lock();
663	defer broadcast_test.mutex.unlock();
664
665	// Wait for all the broadcast threads to spawn.
666	// timedWait() to detect any potential deadlocks.
667	while (broadcast_test.count != num_threads) {
668	broadcast_test.completed.timedWait(
669	&broadcast_test.mutex,
670	1 * std.time.ns_per_s,
671	) catch {};
672	}
673
674	// Signal thread 1 to wake up
675	broadcast_test.thread_id_to_wake = 1;
676	broadcast_test.cond.broadcast();
677	}
678
679	for (broadcast_test.threads) \|t\| {
680	t.join();
681	}
682	}
683	}

lib/std/Thread/Futex.zig deleted-1063

...	@@ -1,1063 +0,0 @@
1	//! A mechanism used to block (`wait`) and unblock (`wake`) threads using a
2	//! 32bit memory address as hints.
3	//!
4	//! Blocking a thread is acknowledged only if the 32bit memory address is equal
5	//! to a given value. This check helps avoid block/unblock deadlocks which
6	//! occur if a `wake()` happens before a `wait()`.
7	//!
8	//! Using Futex, other Thread synchronization primitives can be built which
9	//! efficiently wait for cross-thread events or signals.
10
11	const std = @import("../std.zig");
12	const builtin = @import("builtin");
13	const Futex = @This();
14	const windows = std.os.windows;
15	const linux = std.os.linux;
16	const c = std.c;
17
18	const assert = std.debug.assert;
19	const testing = std.testing;
20	const atomic = std.atomic;
21
22	/// Checks if `ptr` still contains the value `expect` and, if so, blocks the caller until either:
23	/// - The value at `ptr` is no longer equal to `expect` and `wake()` is called on the same address.
24	/// - The caller is unblocked spuriously ("at random").
25	///
26	/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically
27	/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.
28	pub fn wait(ptr: *const atomic.Value(u32), expect: u32) void {
29	@branchHint(.cold);
30
31	Impl.wait(ptr, expect, null) catch \|err\| switch (err) {
32	error.Timeout => unreachable, // null timeout meant to wait forever
33	};
34	}
35
36	/// Checks if `ptr` still contains the value `expect` and, if so, blocks the caller until either:
37	/// - The value at `ptr` is no longer equal to `expect`.
38	/// - The caller is unblocked by a matching `wake()`.
39	/// - The caller is unblocked spuriously ("at random").
40	/// - The caller blocks for longer than the given timeout. In which case, `error.Timeout` is returned.
41	///
42	/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically
43	/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.
44	pub fn timedWait(ptr: *const atomic.Value(u32), expect: u32, timeout_ns: u64) error{Timeout}!void {
45	@branchHint(.cold);
46
47	// Avoid calling into the OS for no-op timeouts.
48	if (timeout_ns == 0) {
49	if (ptr.load(.seq_cst) != expect) return;
50	return error.Timeout;
51	}
52
53	return Impl.wait(ptr, expect, timeout_ns);
54	}
55
56	/// Unblocks at most `max_waiters` callers blocked in a `wait()` call on `ptr`.
57	pub fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
58	@branchHint(.cold);
59
60	// Avoid calling into the OS if there's nothing to wake up.
61	if (max_waiters == 0) {
62	return;
63	}
64
65	Impl.wake(ptr, max_waiters);
66	}
67
68	const Impl = if (builtin.single_threaded)
69	SingleThreadedImpl
70	else if (builtin.os.tag == .windows)
71	WindowsImpl
72	else if (builtin.os.tag.isDarwin())
73	DarwinImpl
74	else if (builtin.os.tag == .linux)
75	LinuxImpl
76	else if (builtin.os.tag == .freebsd)
77	FreebsdImpl
78	else if (builtin.os.tag == .openbsd)
79	OpenbsdImpl
80	else if (builtin.os.tag == .dragonfly)
81	DragonflyImpl
82	else if (builtin.target.cpu.arch.isWasm())
83	WasmImpl
84	else if (std.Thread.use_pthreads)
85	PosixImpl
86	else
87	UnsupportedImpl;
88
89	/// We can't do @compileError() in the `Impl` switch statement above as its eagerly evaluated.
90	/// So instead, we @compileError() on the methods themselves for platforms which don't support futex.
91	const UnsupportedImpl = struct {
92	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
93	return unsupported(.{ ptr, expect, timeout });
94	}
95
96	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
97	return unsupported(.{ ptr, max_waiters });
98	}
99
100	fn unsupported(unused: anytype) noreturn {
101	_ = unused;
102	@compileError("Unsupported operating system " ++ @tagName(builtin.target.os.tag));
103	}
104	};
105
106	const SingleThreadedImpl = struct {
107	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
108	if (ptr.raw != expect) {
109	return;
110	}
111
112	// There are no threads to wake us up.
113	// So if we wait without a timeout we would never wake up.
114	const delay = timeout orelse {
115	unreachable; // deadlock detected
116	};
117
118	_ = delay;
119	return error.Timeout;
120	}
121
122	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
123	// There are no other threads to possibly wake up
124	_ = ptr;
125	_ = max_waiters;
126	}
127	};
128
129	// We use WaitOnAddress through NtDll instead of API-MS-Win-Core-Synch-l1-2-0.dll
130	// as it's generally already a linked target and is autoloaded into all processes anyway.
131	const WindowsImpl = struct {
132	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
133	var timeout_value: windows.LARGE_INTEGER = undefined;
134	var timeout_ptr: ?*const windows.LARGE_INTEGER = null;
135
136	// NTDLL functions work with time in units of 100 nanoseconds.
137	// Positive values are absolute deadlines while negative values are relative durations.
138	if (timeout) \|delay\| {
139	timeout_value = @as(windows.LARGE_INTEGER, @intCast(delay / 100));
140	timeout_value = -timeout_value;
141	timeout_ptr = &timeout_value;
142	}
143
144	const rc = windows.ntdll.RtlWaitOnAddress(
145	ptr,
146	&expect,
147	@sizeOf(@TypeOf(expect)),
148	timeout_ptr,
149	);
150
151	switch (rc) {
152	.SUCCESS => {},
153	.TIMEOUT => {
154	assert(timeout != null);
155	return error.Timeout;
156	},
157	else => unreachable,
158	}
159	}
160
161	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
162	const address: ?*const anyopaque = ptr;
163	assert(max_waiters != 0);
164
165	switch (max_waiters) {
166	1 => windows.ntdll.RtlWakeAddressSingle(address),
167	else => windows.ntdll.RtlWakeAddressAll(address),
168	}
169	}
170	};
171
172	const DarwinImpl = struct {
173	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
174	// Darwin XNU 7195.50.7.100.1 introduced __ulock_wait2 and migrated code paths (notably pthread_cond_t) towards it:
175	// https://github.com/apple/darwin-xnu/commit/d4061fb0260b3ed486147341b72468f836ed6c8f#diff-08f993cc40af475663274687b7c326cc6c3031e0db3ac8de7b24624610616be6
176	//
177	// This XNU version appears to correspond to 11.0.1:
178	// https://kernelshaman.blogspot.com/2021/01/building-xnu-for-macos-big-sur-1101.html
179	//
180	// ulock_wait() uses 32-bit micro-second timeouts where 0 = INFINITE or no-timeout
181	// ulock_wait2() uses 64-bit nano-second timeouts (with the same convention)
182	const supports_ulock_wait2 = builtin.target.os.version_range.semver.min.major >= 11;
183
184	var timeout_ns: u64 = 0;
185	if (timeout) \|delay\| {
186	assert(delay != 0); // handled by timedWait()
187	timeout_ns = delay;
188	}
189
190	// If we're using `__ulock_wait` and `timeout` is too big to fit inside a `u32` count of
191	// micro-seconds (around 70min), we'll request a shorter timeout. This is fine (users
192	// should handle spurious wakeups), but we need to remember that we did so, so that
193	// we don't return `Timeout` incorrectly. If that happens, we set this variable to
194	// true so that we we know to ignore the ETIMEDOUT result.
195	var timeout_overflowed = false;
196
197	const addr: *const anyopaque = ptr;
198	const flags: c.UL = .{
199	.op = .COMPARE_AND_WAIT,
200	.NO_ERRNO = true,
201	};
202	const status = blk: {
203	if (supports_ulock_wait2) {
204	break :blk c.__ulock_wait2(flags, addr, expect, timeout_ns, 0);
205	}
206
207	const timeout_us = std.math.cast(u32, timeout_ns / std.time.ns_per_us) orelse overflow: {
208	timeout_overflowed = true;
209	break :overflow std.math.maxInt(u32);
210	};
211
212	break :blk c.__ulock_wait(flags, addr, expect, timeout_us);
213	};
214
215	if (status >= 0) return;
216	switch (@as(c.E, @enumFromInt(-status))) {
217	// Wait was interrupted by the OS or other spurious signalling.
218	.INTR => {},
219	// Address of the futex was paged out. This is unlikely, but possible in theory, and
220	// pthread/libdispatch on darwin bother to handle it. In this case we'll return
221	// without waiting, but the caller should retry anyway.
222	.FAULT => {},
223	// Only report Timeout if we didn't have to cap the timeout
224	.TIMEDOUT => {
225	assert(timeout != null);
226	if (!timeout_overflowed) return error.Timeout;
227	},
228	else => unreachable,
229	}
230	}
231
232	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
233	const flags: c.UL = .{
234	.op = .COMPARE_AND_WAIT,
235	.NO_ERRNO = true,
236	.WAKE_ALL = max_waiters > 1,
237	};
238
239	while (true) {
240	const addr: *const anyopaque = ptr;
241	const status = c.__ulock_wake(flags, addr, 0);
242
243	if (status >= 0) return;
244	switch (@as(c.E, @enumFromInt(-status))) {
245	.INTR => continue, // spurious wake()
246	.FAULT => unreachable, // __ulock_wake doesn't generate EFAULT according to darwin pthread_cond_t
247	.NOENT => return, // nothing was woken up
248	.ALREADY => unreachable, // only for UL.Op.WAKE_THREAD
249	else => unreachable,
250	}
251	}
252	}
253	};
254
255	// https://man7.org/linux/man-pages/man2/futex.2.html
256	const LinuxImpl = struct {
257	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
258	var ts: linux.timespec = undefined;
259	if (timeout) \|timeout_ns\| {
260	ts.sec = @as(@TypeOf(ts.sec), @intCast(timeout_ns / std.time.ns_per_s));
261	ts.nsec = @as(@TypeOf(ts.nsec), @intCast(timeout_ns % std.time.ns_per_s));
262	}
263
264	const rc = linux.futex_4arg(
265	&ptr.raw,
266	.{ .cmd = .WAIT, .private = true },
267	expect,
268	if (timeout != null) &ts else null,
269	);
270
271	switch (linux.errno(rc)) {
272	.SUCCESS => {}, // notified by `wake()`
273	.INTR => {}, // spurious wakeup
274	.AGAIN => {}, // ptr.* != expect
275	.TIMEDOUT => {
276	assert(timeout != null);
277	return error.Timeout;
278	},
279	.INVAL => {}, // possibly timeout overflow
280	.FAULT => unreachable, // ptr was invalid
281	else => unreachable,
282	}
283	}
284
285	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
286	const rc = linux.futex_3arg(
287	&ptr.raw,
288	.{ .cmd = .WAKE, .private = true },
289	@min(max_waiters, std.math.maxInt(i32)),
290	);
291
292	switch (linux.errno(rc)) {
293	.SUCCESS => {}, // successful wake up
294	.INVAL => {}, // invalid futex_wait() on ptr done elsewhere
295	.FAULT => {}, // pointer became invalid while doing the wake
296	else => unreachable,
297	}
298	}
299	};
300
301	// https://www.freebsd.org/cgi/man.cgi?query=_umtx_op&sektion=2&n=1
302	const FreebsdImpl = struct {
303	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
304	var tm_size: usize = 0;
305	var tm: c._umtx_time = undefined;
306	var tm_ptr: ?*const c._umtx_time = null;
307
308	if (timeout) \|timeout_ns\| {
309	tm_ptr = &tm;
310	tm_size = @sizeOf(@TypeOf(tm));
311
312	tm.flags = 0; // use relative time not UMTX_ABSTIME
313	tm.clockid = .MONOTONIC;
314	tm.timeout.sec = @as(@TypeOf(tm.timeout.sec), @intCast(timeout_ns / std.time.ns_per_s));
315	tm.timeout.nsec = @as(@TypeOf(tm.timeout.nsec), @intCast(timeout_ns % std.time.ns_per_s));
316	}
317
318	const rc = c._umtx_op(
319	@intFromPtr(&ptr.raw),
320	@intFromEnum(c.UMTX_OP.WAIT_UINT_PRIVATE),
321	@as(c_ulong, expect),
322	tm_size,
323	@intFromPtr(tm_ptr),
324	);
325
326	switch (std.posix.errno(rc)) {
327	.SUCCESS => {},
328	.FAULT => unreachable, // one of the args points to invalid memory
329	.INVAL => unreachable, // arguments should be correct
330	.TIMEDOUT => {
331	assert(timeout != null);
332	return error.Timeout;
333	},
334	.INTR => {}, // spurious wake
335	else => unreachable,
336	}
337	}
338
339	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
340	const rc = c._umtx_op(
341	@intFromPtr(&ptr.raw),
342	@intFromEnum(c.UMTX_OP.WAKE_PRIVATE),
343	@as(c_ulong, max_waiters),
344	0, // there is no timeout struct
345	0, // there is no timeout struct pointer
346	);
347
348	switch (std.posix.errno(rc)) {
349	.SUCCESS => {},
350	.FAULT => {}, // it's ok if the ptr doesn't point to valid memory
351	.INVAL => unreachable, // arguments should be correct
352	else => unreachable,
353	}
354	}
355	};
356
357	// https://man.openbsd.org/futex.2
358	const OpenbsdImpl = struct {
359	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
360	var ts: c.timespec = undefined;
361	if (timeout) \|timeout_ns\| {
362	ts.sec = @as(@TypeOf(ts.sec), @intCast(timeout_ns / std.time.ns_per_s));
363	ts.nsec = @as(@TypeOf(ts.nsec), @intCast(timeout_ns % std.time.ns_per_s));
364	}
365
366	const rc = c.futex(
367	@as(*const volatile u32, @ptrCast(&ptr.raw)),
368	c.FUTEX.WAIT \| c.FUTEX.PRIVATE_FLAG,
369	@as(c_int, @bitCast(expect)),
370	if (timeout != null) &ts else null,
371	null, // FUTEX.WAIT takes no requeue address
372	);
373
374	switch (std.posix.errno(rc)) {
375	.SUCCESS => {}, // woken up by wake
376	.NOSYS => unreachable, // the futex operation shouldn't be invalid
377	.FAULT => unreachable, // ptr was invalid
378	.AGAIN => {}, // ptr != expect
379	.INVAL => unreachable, // invalid timeout
380	.TIMEDOUT => {
381	assert(timeout != null);
382	return error.Timeout;
383	},
384	.INTR => {}, // spurious wake from signal
385	.CANCELED => {}, // spurious wake from signal with SA_RESTART
386	else => unreachable,
387	}
388	}
389
390	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
391	const rc = c.futex(
392	@as(*const volatile u32, @ptrCast(&ptr.raw)),
393	c.FUTEX.WAKE \| c.FUTEX.PRIVATE_FLAG,
394	std.math.cast(c_int, max_waiters) orelse std.math.maxInt(c_int),
395	null, // FUTEX.WAKE takes no timeout ptr
396	null, // FUTEX.WAKE takes no requeue address
397	);
398
399	// returns number of threads woken up.
400	assert(rc >= 0);
401	}
402	};
403
404	// https://man.dragonflybsd.org/?command=umtx&section=2
405	const DragonflyImpl = struct {
406	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
407	// Dragonfly uses a scheme where 0 timeout means wait until signaled or spurious wake.
408	// It's reporting of timeout's is also unrealiable so we use an external timing source (Timer) instead.
409	var timeout_us: c_int = 0;
410	var timeout_overflowed = false;
411	var sleep_timer: std.time.Timer = undefined;
412
413	if (timeout) \|delay\| {
414	assert(delay != 0); // handled by timedWait().
415	timeout_us = std.math.cast(c_int, delay / std.time.ns_per_us) orelse blk: {
416	timeout_overflowed = true;
417	break :blk std.math.maxInt(c_int);
418	};
419
420	// Only need to record the start time if we can provide somewhat accurate error.Timeout's
421	if (!timeout_overflowed) {
422	sleep_timer = std.time.Timer.start() catch unreachable;
423	}
424	}
425
426	const value = @as(c_int, @bitCast(expect));
427	const addr = @as(*const volatile c_int, @ptrCast(&ptr.raw));
428	const rc = c.umtx_sleep(addr, value, timeout_us);
429
430	switch (std.posix.errno(rc)) {
431	.SUCCESS => {},
432	.BUSY => {}, // ptr != expect
433	.AGAIN => { // maybe timed out, or paged out, or hit 2s kernel refresh
434	if (timeout) \|timeout_ns\| {
435	// Report error.Timeout only if we know the timeout duration has passed.
436	// If not, there's not much choice other than treating it as a spurious wake.
437	if (!timeout_overflowed and sleep_timer.read() >= timeout_ns) {
438	return error.Timeout;
439	}
440	}
441	},
442	.INTR => {}, // spurious wake
443	.INVAL => unreachable, // invalid timeout
444	else => unreachable,
445	}
446	}
447
448	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
449	// A count of zero means wake all waiters.
450	assert(max_waiters != 0);
451	const to_wake = std.math.cast(c_int, max_waiters) orelse 0;
452
453	// https://man.dragonflybsd.org/?command=umtx&section=2
454	// > umtx_wakeup() will generally return 0 unless the address is bad.
455	// We are fine with the address being bad (e.g. for Semaphore.post() where Semaphore.wait() frees the Semaphore)
456	const addr = @as(*const volatile c_int, @ptrCast(&ptr.raw));
457	_ = c.umtx_wakeup(addr, to_wake);
458	}
459	};
460
461	const WasmImpl = struct {
462	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
463	if (!comptime builtin.cpu.has(.wasm, .atomics)) @compileError("WASI target missing cpu feature 'atomics'");
464
465	const to: i64 = if (timeout) \|to\| @intCast(to) else -1;
466	const result = asm volatile (
467	\\local.get %[ptr]
468	\\local.get %[expected]
469	\\local.get %[timeout]
470	\\memory.atomic.wait32 0
471	\\local.set %[ret]
472	: [ret] "=r" (-> u32),
473	: [ptr] "r" (&ptr.raw),
474	[expected] "r" (@as(i32, @bitCast(expect))),
475	[timeout] "r" (to),
476	);
477	switch (result) {
478	0 => {}, // ok
479	1 => {}, // expected =! loaded
480	2 => return error.Timeout,
481	else => unreachable,
482	}
483	}
484
485	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
486	if (!comptime builtin.cpu.has(.wasm, .atomics)) @compileError("WASI target missing cpu feature 'atomics'");
487
488	assert(max_waiters != 0);
489	const woken_count = asm volatile (
490	\\local.get %[ptr]
491	\\local.get %[waiters]
492	\\memory.atomic.notify 0
493	\\local.set %[ret]
494	: [ret] "=r" (-> u32),
495	: [ptr] "r" (&ptr.raw),
496	[waiters] "r" (max_waiters),
497	);
498	_ = woken_count; // can be 0 when linker flag 'shared-memory' is not enabled
499	}
500	};
501
502	/// Modified version of linux's futex and Go's sema to implement userspace wait queues with pthread:
503	/// https://code.woboq.org/linux/linux/kernel/futex.c.html
504	/// https://go.dev/src/runtime/sema.go
505	const PosixImpl = struct {
506	const Event = struct {
507	cond: c.pthread_cond_t,
508	mutex: c.pthread_mutex_t,
509	state: enum { empty, waiting, notified },
510
511	fn init(self: *Event) void {
512	// Use static init instead of pthread_cond/mutex_init() since this is generally faster.
513	self.cond = .{};
514	self.mutex = .{};
515	self.state = .empty;
516	}
517
518	fn deinit(self: *Event) void {
519	// Some platforms reportedly give EINVAL for statically initialized pthread types.
520	const rc = c.pthread_cond_destroy(&self.cond);
521	assert(rc == .SUCCESS or rc == .INVAL);
522
523	const rm = c.pthread_mutex_destroy(&self.mutex);
524	assert(rm == .SUCCESS or rm == .INVAL);
525
526	self.* = undefined;
527	}
528
529	fn wait(self: *Event, timeout: ?u64) error{Timeout}!void {
530	assert(c.pthread_mutex_lock(&self.mutex) == .SUCCESS);
531	defer assert(c.pthread_mutex_unlock(&self.mutex) == .SUCCESS);
532
533	// Early return if the event was already set.
534	if (self.state == .notified) {
535	return;
536	}
537
538	// Compute the absolute timeout if one was specified.
539	// POSIX requires that REALTIME is used by default for the pthread timedwait functions.
540	// This can be changed with pthread_condattr_setclock, but it's an extension and may not be available everywhere.
541	var ts: c.timespec = undefined;
542	if (timeout) \|timeout_ns\| {
543	ts = std.posix.clock_gettime(c.CLOCK.REALTIME) catch unreachable;
544	ts.sec +\|= @as(@TypeOf(ts.sec), @intCast(timeout_ns / std.time.ns_per_s));
545	ts.nsec += @as(@TypeOf(ts.nsec), @intCast(timeout_ns % std.time.ns_per_s));
546
547	if (ts.nsec >= std.time.ns_per_s) {
548	ts.sec +\|= 1;
549	ts.nsec -= std.time.ns_per_s;
550	}
551	}
552
553	// Start waiting on the event - there can be only one thread waiting.
554	assert(self.state == .empty);
555	self.state = .waiting;
556
557	while (true) {
558	// Block using either pthread_cond_wait or pthread_cond_timewait if there's an absolute timeout.
559	const rc = blk: {
560	if (timeout == null) break :blk c.pthread_cond_wait(&self.cond, &self.mutex);
561	break :blk c.pthread_cond_timedwait(&self.cond, &self.mutex, &ts);
562	};
563
564	// After waking up, check if the event was set.
565	if (self.state == .notified) {
566	return;
567	}
568
569	assert(self.state == .waiting);
570	switch (rc) {
571	.SUCCESS => {},
572	.TIMEDOUT => {
573	// If timed out, reset the event to avoid the set() thread doing an unnecessary signal().
574	self.state = .empty;
575	return error.Timeout;
576	},
577	.INVAL => unreachable, // cond, mutex, and potentially ts should all be valid
578	.PERM => unreachable, // mutex is locked when cond_*wait() functions are called
579	else => unreachable,
580	}
581	}
582	}
583
584	fn set(self: *Event) void {
585	assert(c.pthread_mutex_lock(&self.mutex) == .SUCCESS);
586	defer assert(c.pthread_mutex_unlock(&self.mutex) == .SUCCESS);
587
588	// Make sure that multiple calls to set() were not done on the same Event.
589	const old_state = self.state;
590	assert(old_state != .notified);
591
592	// Mark the event as set and wake up the waiting thread if there was one.
593	// This must be done while the mutex as the wait() thread could deallocate
594	// the condition variable once it observes the new state, potentially causing a UAF if done unlocked.
595	self.state = .notified;
596	if (old_state == .waiting) {
597	assert(c.pthread_cond_signal(&self.cond) == .SUCCESS);
598	}
599	}
600	};
601
602	const Treap = std.Treap(usize, std.math.order);
603	const Waiter = struct {
604	node: Treap.Node,
605	prev: ?*Waiter,
606	next: ?*Waiter,
607	tail: ?*Waiter,
608	is_queued: bool,
609	event: Event,
610	};
611
612	// An unordered set of Waiters
613	const WaitList = struct {
614	top: ?*Waiter = null,
615	len: usize = 0,
616
617	fn push(self: WaitList, waiter: Waiter) void {
618	waiter.next = self.top;
619	self.top = waiter;
620	self.len += 1;
621	}
622
623	fn pop(self: WaitList) ?Waiter {
624	const waiter = self.top orelse return null;
625	self.top = waiter.next;
626	self.len -= 1;
627	return waiter;
628	}
629	};
630
631	const WaitQueue = struct {
632	fn insert(treap: Treap, address: usize, waiter: Waiter) void {
633	// prepare the waiter to be inserted.
634	waiter.next = null;
635	waiter.is_queued = true;
636
637	// Find the wait queue entry associated with the address.
638	// If there isn't a wait queue on the address, this waiter creates the queue.
639	var entry = treap.getEntryFor(address);
640	const entry_node = entry.node orelse {
641	waiter.prev = null;
642	waiter.tail = waiter;
643	entry.set(&waiter.node);
644	return;
645	};
646
647	// There's a wait queue on the address; get the queue head and tail.
648	const head: *Waiter = @fieldParentPtr("node", entry_node);
649	const tail = head.tail orelse unreachable;
650
651	// Push the waiter to the tail by replacing it and linking to the previous tail.
652	head.tail = waiter;
653	tail.next = waiter;
654	waiter.prev = tail;
655	}
656
657	fn remove(treap: *Treap, address: usize, max_waiters: usize) WaitList {
658	// Find the wait queue associated with this address and get the head/tail if any.
659	var entry = treap.getEntryFor(address);
660	var queue_head: ?*Waiter = if (entry.node) \|node\| @fieldParentPtr("node", node) else null;
661	const queue_tail = if (queue_head) \|head\| head.tail else null;
662
663	// Once we're done updating the head, fix it's tail pointer and update the treap's queue head as well.
664	defer entry.set(blk: {
665	const new_head = queue_head orelse break :blk null;
666	new_head.tail = queue_tail;
667	break :blk &new_head.node;
668	});
669
670	var removed = WaitList{};
671	while (removed.len < max_waiters) {
672	// dequeue and collect waiters from their wait queue.
673	const waiter = queue_head orelse break;
674	queue_head = waiter.next;
675	removed.push(waiter);
676
677	// When dequeueing, we must mark is_queued as false.
678	// This ensures that a waiter which calls tryRemove() returns false.
679	assert(waiter.is_queued);
680	waiter.is_queued = false;
681	}
682
683	return removed;
684	}
685
686	fn tryRemove(treap: Treap, address: usize, waiter: Waiter) bool {
687	if (!waiter.is_queued) {
688	return false;
689	}
690
691	queue_remove: {
692	// Find the wait queue associated with the address.
693	var entry = blk: {
694	// A waiter without a previous link means it's the queue head that's in the treap so we can avoid lookup.
695	if (waiter.prev == null) {
696	assert(waiter.node.key == address);
697	break :blk treap.getEntryForExisting(&waiter.node);
698	}
699	break :blk treap.getEntryFor(address);
700	};
701
702	// The queue head and tail must exist if we're removing a queued waiter.
703	const head: *Waiter = @fieldParentPtr("node", entry.node orelse unreachable);
704	const tail = head.tail orelse unreachable;
705
706	// A waiter with a previous link is never the head of the queue.
707	if (waiter.prev) \|prev\| {
708	assert(waiter != head);
709	prev.next = waiter.next;
710
711	// A waiter with both a previous and next link is in the middle.
712	// We only need to update the surrounding waiter's links to remove it.
713	if (waiter.next) \|next\| {
714	assert(waiter != tail);
715	next.prev = waiter.prev;
716	break :queue_remove;
717	}
718
719	// A waiter with a previous but no next link means it's the tail of the queue.
720	// In that case, we need to update the head's tail reference.
721	assert(waiter == tail);
722	head.tail = waiter.prev;
723	break :queue_remove;
724	}
725
726	// A waiter with no previous link means it's the queue head of queue.
727	// We must replace (or remove) the head waiter reference in the treap.
728	assert(waiter == head);
729	entry.set(blk: {
730	const new_head = waiter.next orelse break :blk null;
731	new_head.tail = head.tail;
732	break :blk &new_head.node;
733	});
734	}
735
736	// Mark the waiter as successfully removed.
737	waiter.is_queued = false;
738	return true;
739	}
740	};
741
742	const Bucket = struct {
743	mutex: c.pthread_mutex_t align(atomic.cache_line) = .{},
744	pending: atomic.Value(usize) = atomic.Value(usize).init(0),
745	treap: Treap = .{},
746
747	// Global array of buckets that addresses map to.
748	// Bucket array size is pretty much arbitrary here, but it must be a power of two for fibonacci hashing.
749	var buckets = [_]Bucket{.{}} ** @bitSizeOf(usize);
750
751	// https://github.com/Amanieu/parking_lot/blob/1cf12744d097233316afa6c8b7d37389e4211756/core/src/parking_lot.rs#L343-L353
752	fn from(address: usize) *Bucket {
753	// The upper `@bitSizeOf(usize)` bits of the fibonacci golden ratio.
754	// Hashing this via (h * k) >> (64 - b) where k=golden-ration and b=bitsize-of-array
755	// evenly lays out h=hash values over the bit range even when the hash has poor entropy (identity-hash for pointers).
756	const max_multiplier_bits = @bitSizeOf(usize);
757	const fibonacci_multiplier = 0x9E3779B97F4A7C15 >> (64 - max_multiplier_bits);
758
759	const max_bucket_bits = @ctz(buckets.len);
760	comptime assert(std.math.isPowerOfTwo(buckets.len));
761
762	const index = (address *% fibonacci_multiplier) >> (max_multiplier_bits - max_bucket_bits);
763	return &buckets[index];
764	}
765	};
766
767	const Address = struct {
768	fn from(ptr: *const atomic.Value(u32)) usize {
769	// Get the alignment of the pointer.
770	const alignment = @alignOf(atomic.Value(u32));
771	comptime assert(std.math.isPowerOfTwo(alignment));
772
773	// Make sure the pointer is aligned,
774	// then cut off the zero bits from the alignment to get the unique address.
775	const addr = @intFromPtr(ptr);
776	assert(addr & (alignment - 1) == 0);
777	return addr >> @ctz(@as(usize, alignment));
778	}
779	};
780
781	fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
782	const address = Address.from(ptr);
783	const bucket = Bucket.from(address);
784
785	// Announce that there's a waiter in the bucket before checking the ptr/expect condition.
786	// If the announcement is reordered after the ptr check, the waiter could deadlock:
787	//
788	// - T1: checks ptr == expect which is true
789	// - T2: updates ptr to != expect
790	// - T2: does Futex.wake(), sees no pending waiters, exits
791	// - T1: bumps pending waiters (was reordered after the ptr == expect check)
792	// - T1: goes to sleep and misses both the ptr change and T2's wake up
793	//
794	// acquire barrier to ensure the announcement happens before the ptr check below.
795	var pending = bucket.pending.fetchAdd(1, .acquire);
796	assert(pending < std.math.maxInt(usize));
797
798	// If the wait gets canceled, remove the pending count we previously added.
799	// This is done outside the mutex lock to keep the critical section short in case of contention.
800	var canceled = false;
801	defer if (canceled) {
802	pending = bucket.pending.fetchSub(1, .monotonic);
803	assert(pending > 0);
804	};
805
806	var waiter: Waiter = undefined;
807	{
808	assert(c.pthread_mutex_lock(&bucket.mutex) == .SUCCESS);
809	defer assert(c.pthread_mutex_unlock(&bucket.mutex) == .SUCCESS);
810
811	canceled = ptr.load(.monotonic) != expect;
812	if (canceled) {
813	return;
814	}
815
816	waiter.event.init();
817	WaitQueue.insert(&bucket.treap, address, &waiter);
818	}
819
820	defer {
821	assert(!waiter.is_queued);
822	waiter.event.deinit();
823	}
824
825	waiter.event.wait(timeout) catch {
826	// If we fail to cancel after a timeout, it means a wake() thread dequeued us and will wake us up.
827	// We must wait until the event is set as that's a signal that the wake() thread won't access the waiter memory anymore.
828	// If we return early without waiting, the waiter on the stack would be invalidated and the wake() thread risks a UAF.
829	defer if (!canceled) waiter.event.wait(null) catch unreachable;
830
831	assert(c.pthread_mutex_lock(&bucket.mutex) == .SUCCESS);
832	defer assert(c.pthread_mutex_unlock(&bucket.mutex) == .SUCCESS);
833
834	canceled = WaitQueue.tryRemove(&bucket.treap, address, &waiter);
835	if (canceled) {
836	return error.Timeout;
837	}
838	};
839	}
840
841	fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
842	const address = Address.from(ptr);
843	const bucket = Bucket.from(address);
844
845	// Quick check if there's even anything to wake up.
846	// The change to the ptr's value must happen before we check for pending waiters.
847	// If not, the wake() thread could miss a sleeping waiter and have it deadlock:
848	//
849	// - T2: p = has pending waiters (reordered before the ptr update)
850	// - T1: bump pending waiters
851	// - T1: if ptr == expected: sleep()
852	// - T2: update ptr != expected
853	// - T2: p is false from earlier so doesn't wake (T1 missed ptr update and T2 missed T1 sleeping)
854	//
855	// What we really want here is a Release load, but that doesn't exist under the C11 memory model.
856	// We could instead do `bucket.pending.fetchAdd(0, Release) == 0` which achieves effectively the same thing,
857	// LLVM lowers the fetchAdd(0, .release) into an mfence+load which avoids gaining ownership of the cache-line.
858	if (bucket.pending.fetchAdd(0, .release) == 0) {
859	return;
860	}
861
862	// Keep a list of all the waiters notified and wake then up outside the mutex critical section.
863	var notified = WaitList{};
864	defer if (notified.len > 0) {
865	const pending = bucket.pending.fetchSub(notified.len, .monotonic);
866	assert(pending >= notified.len);
867
868	while (notified.pop()) \|waiter\| {
869	assert(!waiter.is_queued);
870	waiter.event.set();
871	}
872	};
873
874	assert(c.pthread_mutex_lock(&bucket.mutex) == .SUCCESS);
875	defer assert(c.pthread_mutex_unlock(&bucket.mutex) == .SUCCESS);
876
877	// Another pending check again to avoid the WaitQueue lookup if not necessary.
878	if (bucket.pending.load(.monotonic) > 0) {
879	notified = WaitQueue.remove(&bucket.treap, address, max_waiters);
880	}
881	}
882	};
883
884	test "smoke test" {
885	var value = atomic.Value(u32).init(0);
886
887	// Try waits with invalid values.
888	Futex.wait(&value, 0xdeadbeef);
889	Futex.timedWait(&value, 0xdeadbeef, 0) catch {};
890
891	// Try timeout waits.
892	try testing.expectError(error.Timeout, Futex.timedWait(&value, 0, 0));
893	try testing.expectError(error.Timeout, Futex.timedWait(&value, 0, std.time.ns_per_ms));
894
895	// Try wakes
896	Futex.wake(&value, 0);
897	Futex.wake(&value, 1);
898	Futex.wake(&value, std.math.maxInt(u32));
899	}
900
901	test "signaling" {
902	// This test requires spawning threads
903	if (builtin.single_threaded) {
904	return error.SkipZigTest;
905	}
906
907	const num_threads = 4;
908	const num_iterations = 4;
909
910	const Paddle = struct {
911	value: atomic.Value(u32) = atomic.Value(u32).init(0),
912	current: u32 = 0,
913
914	fn hit(self: *@This()) void {
915	_ = self.value.fetchAdd(1, .release);
916	Futex.wake(&self.value, 1);
917	}
918
919	fn run(self: @This(), hit_to: @This()) !void {
920	while (self.current < num_iterations) {
921	// Wait for the value to change from hit()
922	var new_value: u32 = undefined;
923	while (true) {
924	new_value = self.value.load(.acquire);
925	if (new_value != self.current) break;
926	Futex.wait(&self.value, self.current);
927	}
928
929	// change the internal "current" value
930	try testing.expectEqual(new_value, self.current + 1);
931	self.current = new_value;
932
933	// hit the next paddle
934	hit_to.hit();
935	}
936	}
937	};
938
939	var paddles = [_]Paddle{.{}} ** num_threads;
940	var threads = [_]std.Thread{undefined} ** num_threads;
941
942	// Create a circle of paddles which hit each other
943	for (&threads, 0..) \|*t, i\| {
944	const paddle = &paddles[i];
945	const hit_to = &paddles[(i + 1) % paddles.len];
946	t.* = try std.Thread.spawn(.{}, Paddle.run, .{ paddle, hit_to });
947	}
948
949	// Hit the first paddle and wait for them all to complete by hitting each other for num_iterations.
950	paddles[0].hit();
951	for (threads) \|t\| t.join();
952	for (paddles) \|p\| try testing.expectEqual(p.current, num_iterations);
953	}
954
955	test "broadcasting" {
956	// This test requires spawning threads
957	if (builtin.single_threaded) {
958	return error.SkipZigTest;
959	}
960
961	const num_threads = 4;
962	const num_iterations = 4;
963
964	const Barrier = struct {
965	count: atomic.Value(u32) = atomic.Value(u32).init(num_threads),
966	futex: atomic.Value(u32) = atomic.Value(u32).init(0),
967
968	fn wait(self: *@This()) !void {
969	// Decrement the counter.
970	// Release ensures stuff before this barrier.wait() happens before the last one.
971	// Acquire for the last counter ensures stuff before previous barrier.wait()s happened before it.
972	const count = self.count.fetchSub(1, .acq_rel);
973	try testing.expect(count <= num_threads);
974	try testing.expect(count > 0);
975
976	// First counter to reach zero wakes all other threads.
977	// Release on futex update ensures stuff before all barrier.wait()'s happens before they all return.
978	if (count - 1 == 0) {
979	self.futex.store(1, .release);
980	Futex.wake(&self.futex, num_threads - 1);
981	return;
982	}
983
984	// Other threads wait until last counter wakes them up.
985	// Acquire on futex synchronizes with last barrier count to ensure stuff before all barrier.wait()'s happen before us.
986	while (self.futex.load(.acquire) == 0) {
987	Futex.wait(&self.futex, 0);
988	}
989	}
990	};
991
992	const Broadcast = struct {
993	barriers: [num_iterations]Barrier = [_]Barrier{.{}} ** num_iterations,
994	threads: [num_threads]std.Thread = undefined,
995
996	fn run(self: *@This()) !void {
997	for (&self.barriers) \|*barrier\| {
998	try barrier.wait();
999	}
1000	}
1001	};
1002
1003	var broadcast = Broadcast{};
1004	for (&broadcast.threads) \|t\| t. = try std.Thread.spawn(.{}, Broadcast.run, .{&broadcast});
1005	for (broadcast.threads) \|t\| t.join();
1006	}
1007
1008	/// Deadline is used to wait efficiently for a pointer's value to change using Futex and a fixed timeout.
1009	///
1010	/// Futex's timedWait() api uses a relative duration which suffers from over-waiting
1011	/// when used in a loop which is often required due to the possibility of spurious wakeups.
1012	///
1013	/// Deadline instead converts the relative timeout to an absolute one so that multiple calls
1014	/// to Futex timedWait() can block for and report more accurate error.Timeouts.
1015	pub const Deadline = struct {
1016	timeout: ?u64,
1017	started: std.time.Timer,
1018
1019	/// Create the deadline to expire after the given amount of time in nanoseconds passes.
1020	/// Pass in `null` to have the deadline call `Futex.wait()` and never expire.
1021	pub fn init(expires_in_ns: ?u64) Deadline {
1022	var deadline: Deadline = undefined;
1023	deadline.timeout = expires_in_ns;
1024
1025	// std.time.Timer is required to be supported for somewhat accurate reportings of error.Timeout.
1026	if (deadline.timeout != null) {
1027	deadline.started = std.time.Timer.start() catch unreachable;
1028	}
1029
1030	return deadline;
1031	}
1032
1033	/// Wait until either:
1034	/// - the `ptr`'s value changes from `expect`.
1035	/// - `Futex.wake()` is called on the `ptr`.
1036	/// - A spurious wake occurs.
1037	/// - The deadline expires; In which case `error.Timeout` is returned.
1038	pub fn wait(self: Deadline, ptr: const atomic.Value(u32), expect: u32) error{Timeout}!void {
1039	@branchHint(.cold);
1040
1041	// Check if we actually have a timeout to wait until.
1042	// If not just wait "forever".
1043	const timeout_ns = self.timeout orelse {
1044	return Futex.wait(ptr, expect);
1045	};
1046
1047	// Get how much time has passed since we started waiting
1048	// then subtract that from the init() timeout to get how much longer to wait.
1049	// Use overflow to detect when we've been waiting longer than the init() timeout.
1050	const elapsed_ns = self.started.read();
1051	const until_timeout_ns = std.math.sub(u64, timeout_ns, elapsed_ns) catch 0;
1052	return Futex.timedWait(ptr, expect, until_timeout_ns);
1053	}
1054	};
1055
1056	test "Deadline" {
1057	var deadline = Deadline.init(100 * std.time.ns_per_ms);
1058	var futex_word = atomic.Value(u32).init(0);
1059
1060	while (true) {
1061	deadline.wait(&futex_word, 0) catch break;
1062	}
1063	}

lib/std/Thread/Mutex.zig deleted-367

...	@@ -1,367 +0,0 @@
1	//! Mutex is a synchronization primitive which enforces atomic access to a
2	//! shared region of code known as the "critical section".
3	//!
4	//! It does this by blocking ensuring only one thread is in the critical
5	//! section at any given point in time by blocking the others.
6	//!
7	//! Mutex can be statically initialized and is at most `@sizeOf(u64)` large.
8	//! Use `lock()` or `tryLock()` to enter the critical section and `unlock()` to leave it.
9
10	const std = @import("../std.zig");
11	const builtin = @import("builtin");
12	const Mutex = @This();
13
14	const assert = std.debug.assert;
15	const testing = std.testing;
16	const Thread = std.Thread;
17	const Futex = Thread.Futex;
18
19	impl: Impl = .{},
20
21	pub const Recursive = @import("Mutex/Recursive.zig");
22
23	/// Tries to acquire the mutex without blocking the caller's thread.
24	/// Returns `false` if the calling thread would have to block to acquire it.
25	/// Otherwise, returns `true` and the caller should `unlock()` the Mutex to release it.
26	pub fn tryLock(self: *Mutex) bool {
27	return self.impl.tryLock();
28	}
29
30	/// Acquires the mutex, blocking the caller's thread until it can.
31	/// It is undefined behavior if the mutex is already held by the caller's thread.
32	/// Once acquired, call `unlock()` on the Mutex to release it.
33	pub fn lock(self: *Mutex) void {
34	self.impl.lock();
35	}
36
37	/// Releases the mutex which was previously acquired with `lock()` or `tryLock()`.
38	/// It is undefined behavior if the mutex is unlocked from a different thread that it was locked from.
39	pub fn unlock(self: *Mutex) void {
40	self.impl.unlock();
41	}
42
43	const Impl = if (builtin.mode == .Debug and !builtin.single_threaded)
44	DebugImpl
45	else
46	ReleaseImpl;
47
48	const ReleaseImpl = Impl: {
49	if (builtin.single_threaded) break :Impl SingleThreadedImpl;
50	if (builtin.os.tag == .windows) break :Impl WindowsImpl;
51	if (builtin.os.tag.isDarwin()) break :Impl DarwinImpl;
52
53	if (builtin.target.os.tag == .linux or
54	builtin.target.os.tag == .freebsd or
55	builtin.target.os.tag == .openbsd or
56	builtin.target.os.tag == .dragonfly or
57	builtin.target.cpu.arch.isWasm())
58	{
59	// Futex is the system's synchronization primitive; use that.
60	break :Impl FutexImpl;
61	}
62
63	if (std.Thread.use_pthreads) {
64	// This system doesn't have a futex primitive, so `std.Thread.Futex` is using `PosixImpl`,
65	// which implements futex on top of pthread mutexes and conditions. Therefore, instead
66	// of going through that long inefficient path, just use pthread mutex directly.
67	break :Impl PosixImpl;
68	}
69
70	break :Impl FutexImpl;
71	};
72
73	const DebugImpl = struct {
74	locking_thread: std.atomic.Value(Thread.Id) = std.atomic.Value(Thread.Id).init(0), // 0 means it's not locked.
75	impl: ReleaseImpl = .{},
76
77	inline fn tryLock(self: *@This()) bool {
78	const locking = self.impl.tryLock();
79	if (locking) {
80	self.locking_thread.store(Thread.getCurrentId(), .unordered);
81	}
82	return locking;
83	}
84
85	inline fn lock(self: *@This()) void {
86	const current_id = Thread.getCurrentId();
87	if (self.locking_thread.load(.unordered) == current_id and current_id != 0) {
88	@panic("Deadlock detected");
89	}
90	self.impl.lock();
91	self.locking_thread.store(current_id, .unordered);
92	}
93
94	inline fn unlock(self: *@This()) void {
95	assert(self.locking_thread.load(.unordered) == Thread.getCurrentId());
96	self.locking_thread.store(0, .unordered);
97	self.impl.unlock();
98	}
99	};
100
101	const SingleThreadedImpl = struct {
102	is_locked: bool = false,
103
104	fn tryLock(self: *@This()) bool {
105	if (self.is_locked) return false;
106	self.is_locked = true;
107	return true;
108	}
109
110	fn lock(self: *@This()) void {
111	if (!self.tryLock()) {
112	unreachable; // deadlock detected
113	}
114	}
115
116	fn unlock(self: *@This()) void {
117	assert(self.is_locked);
118	self.is_locked = false;
119	}
120	};
121
122	/// SRWLOCK on windows is almost always faster than Futex solution.
123	/// It also implements an efficient Condition with requeue support for us.
124	const WindowsImpl = struct {
125	srwlock: windows.SRWLOCK = .{},
126
127	fn tryLock(self: *@This()) bool {
128	return windows.ntdll.RtlTryAcquireSRWLockExclusive(&self.srwlock) != windows.FALSE;
129	}
130
131	fn lock(self: *@This()) void {
132	windows.ntdll.RtlAcquireSRWLockExclusive(&self.srwlock);
133	}
134
135	fn unlock(self: *@This()) void {
136	windows.ntdll.RtlReleaseSRWLockExclusive(&self.srwlock);
137	}
138
139	const windows = std.os.windows;
140	};
141
142	/// os_unfair_lock on darwin supports priority inheritance and is generally faster than Futex solutions.
143	const DarwinImpl = struct {
144	oul: c.os_unfair_lock = .{},
145
146	fn tryLock(self: *@This()) bool {
147	return c.os_unfair_lock_trylock(&self.oul);
148	}
149
150	fn lock(self: *@This()) void {
151	c.os_unfair_lock_lock(&self.oul);
152	}
153
154	fn unlock(self: *@This()) void {
155	c.os_unfair_lock_unlock(&self.oul);
156	}
157
158	const c = std.c;
159	};
160
161	const FutexImpl = struct {
162	state: std.atomic.Value(u32) = std.atomic.Value(u32).init(unlocked),
163
164	const unlocked: u32 = 0b00;
165	const locked: u32 = 0b01;
166	const contended: u32 = 0b11; // must contain the `locked` bit for x86 optimization below
167
168	fn lock(self: *@This()) void {
169	if (!self.tryLock())
170	self.lockSlow();
171	}
172
173	fn tryLock(self: *@This()) bool {
174	// On x86, use `lock bts` instead of `lock cmpxchg` as:
175	// - they both seem to mark the cache-line as modified regardless: https://stackoverflow.com/a/63350048
176	// - `lock bts` is smaller instruction-wise which makes it better for inlining
177	if (builtin.target.cpu.arch.isX86()) {
178	const locked_bit = @ctz(locked);
179	return self.state.bitSet(locked_bit, .acquire) == 0;
180	}
181
182	// Acquire barrier ensures grabbing the lock happens before the critical section
183	// and that the previous lock holder's critical section happens before we grab the lock.
184	return self.state.cmpxchgWeak(unlocked, locked, .acquire, .monotonic) == null;
185	}
186
187	fn lockSlow(self: *@This()) void {
188	@branchHint(.cold);
189
190	// Avoid doing an atomic swap below if we already know the state is contended.
191	// An atomic swap unconditionally stores which marks the cache-line as modified unnecessarily.
192	if (self.state.load(.monotonic) == contended) {
193	Futex.wait(&self.state, contended);
194	}
195
196	// Try to acquire the lock while also telling the existing lock holder that there are threads waiting.
197	//
198	// Once we sleep on the Futex, we must acquire the mutex using `contended` rather than `locked`.
199	// If not, threads sleeping on the Futex wouldn't see the state change in unlock and potentially deadlock.
200	// The downside is that the last mutex unlocker will see `contended` and do an unnecessary Futex wake
201	// but this is better than having to wake all waiting threads on mutex unlock.
202	//
203	// Acquire barrier ensures grabbing the lock happens before the critical section
204	// and that the previous lock holder's critical section happens before we grab the lock.
205	while (self.state.swap(contended, .acquire) != unlocked) {
206	Futex.wait(&self.state, contended);
207	}
208	}
209
210	fn unlock(self: *@This()) void {
211	// Unlock the mutex and wake up a waiting thread if any.
212	//
213	// A waiting thread will acquire with `contended` instead of `locked`
214	// which ensures that it wakes up another thread on the next unlock().
215	//
216	// Release barrier ensures the critical section happens before we let go of the lock
217	// and that our critical section happens before the next lock holder grabs the lock.
218	const state = self.state.swap(unlocked, .release);
219	assert(state != unlocked);
220
221	if (state == contended) {
222	Futex.wake(&self.state, 1);
223	}
224	}
225	};
226
227	const PosixImpl = struct {
228	mutex: std.c.pthread_mutex_t = .{},
229
230	fn tryLock(impl: *PosixImpl) bool {
231	switch (std.c.pthread_mutex_trylock(&impl.mutex)) {
232	.SUCCESS => return true,
233	.BUSY => return false,
234	.INVAL => unreachable, // mutex is initialized correctly
235	else => unreachable,
236	}
237	}
238
239	fn lock(impl: *PosixImpl) void {
240	switch (std.c.pthread_mutex_lock(&impl.mutex)) {
241	.SUCCESS => return,
242	.INVAL => unreachable, // mutex is initialized correctly
243	.DEADLK => unreachable, // not an error checking mutex
244	else => unreachable,
245	}
246	}
247
248	fn unlock(impl: *PosixImpl) void {
249	switch (std.c.pthread_mutex_unlock(&impl.mutex)) {
250	.SUCCESS => return,
251	.INVAL => unreachable, // mutex is initialized correctly
252	.PERM => unreachable, // not an error checking mutex
253	else => unreachable,
254	}
255	}
256	};
257
258	test "smoke test" {
259	var mutex = Mutex{};
260
261	try testing.expect(mutex.tryLock());
262	try testing.expect(!mutex.tryLock());
263	mutex.unlock();
264
265	mutex.lock();
266	try testing.expect(!mutex.tryLock());
267	mutex.unlock();
268	}
269
270	// A counter which is incremented without atomic instructions
271	const NonAtomicCounter = struct {
272	// direct u128 could maybe use xmm ops on x86 which are atomic
273	value: [2]u64 = [_]u64{ 0, 0 },
274
275	fn get(self: NonAtomicCounter) u128 {
276	return @as(u128, @bitCast(self.value));
277	}
278
279	fn inc(self: *NonAtomicCounter) void {
280	for (@as([2]u64, @bitCast(self.get() + 1)), 0..) \|v, i\| {
281	@as(volatile u64, @ptrCast(&self.value[i])). = v;
282	}
283	}
284	};
285
286	test "many uncontended" {
287	// This test requires spawning threads.
288	if (builtin.single_threaded) {
289	return error.SkipZigTest;
290	}
291
292	const num_threads = 4;
293	const num_increments = 1000;
294
295	const Runner = struct {
296	mutex: Mutex = .{},
297	thread: Thread = undefined,
298	counter: NonAtomicCounter = .{},
299
300	fn run(self: *@This()) void {
301	var i: usize = num_increments;
302	while (i > 0) : (i -= 1) {
303	self.mutex.lock();
304	defer self.mutex.unlock();
305
306	self.counter.inc();
307	}
308	}
309	};
310
311	var runners = [_]Runner{.{}} ** num_threads;
312	for (&runners) \|*r\| r.thread = try Thread.spawn(.{}, Runner.run, .{r});
313	for (runners) \|r\| r.thread.join();
314	for (runners) \|r\| try testing.expectEqual(r.counter.get(), num_increments);
315	}
316
317	test "many contended" {
318	// This test requires spawning threads.
319	if (builtin.single_threaded) {
320	return error.SkipZigTest;
321	}
322
323	const num_threads = 4;
324	const num_increments = 1000;
325
326	const Runner = struct {
327	mutex: Mutex = .{},
328	counter: NonAtomicCounter = .{},
329
330	fn run(self: *@This()) void {
331	var i: usize = num_increments;
332	while (i > 0) : (i -= 1) {
333	// Occasionally hint to let another thread run.
334	defer if (i % 100 == 0) Thread.yield() catch {};
335
336	self.mutex.lock();
337	defer self.mutex.unlock();
338
339	self.counter.inc();
340	}
341	}
342	};
343
344	var runner = Runner{};
345
346	var threads: [num_threads]Thread = undefined;
347	for (&threads) \|t\| t. = try Thread.spawn(.{}, Runner.run, .{&runner});
348	for (threads) \|t\| t.join();
349
350	try testing.expectEqual(runner.counter.get(), num_increments * num_threads);
351	}
352
353	// https://github.com/ziglang/zig/issues/19295
354	//test @This() {
355	// var m: Mutex = .{};
356	//
357	// {
358	// m.lock();
359	// defer m.unlock();
360	// // ... critical section code
361	// }
362	//
363	// if (m.tryLock()) {
364	// defer m.unlock();
365	// // ... critical section code
366	// }
367	//}

lib/std/Thread/Mutex/Recursive.zig+6-6

 @@ -7,18 +7,18 @@
 //! A recursive mutex is an abstraction layer on top of a regular mutex;
 //! therefore it is recommended to use instead `std.Mutex` unless there is a
 //! specific reason a recursive mutex is warranted.
+const Recursive = @This();
 const std = @import("../../std.zig");
-const Recursive = @This();
+const Io = std.Io;
-const Mutex = std.Thread.Mutex;
 const assert = std.debug.assert;
-mutex: Mutex,
+mutex: Io.Mutex,
 thread_id: std.Thread.Id,
 lock_count: usize,
 pub const init: Recursive = .{
-    .mutex = .{},
+    .mutex = .init,
     .thread_id = invalid_thread_id,
     .lock_count = 0,
 };
 @@ -49,7 +49,7 @@ pub fn tryLock(r: *Recursive) bool {
 pub fn lock(r: *Recursive) void {
     const current_thread_id = std.Thread.getCurrentId();
     if (@atomicLoad(std.Thread.Id, &r.thread_id, .unordered) != current_thread_id) {
-        r.mutex.lock();
+        Io.Threaded.mutexLock(&r.mutex);
         assert(r.lock_count == 0);
         @atomicStore(std.Thread.Id, &r.thread_id, current_thread_id, .unordered);
     }
 @@ -64,7 +64,7 @@ pub fn unlock(r: *Recursive) void {
     r.lock_count -= 1;
     if (r.lock_count == 0) {
         @atomicStore(std.Thread.Id, &r.thread_id, invalid_thread_id, .unordered);
-        r.mutex.unlock();
+        Io.Threaded.mutexUnlock(&r.mutex);
     }
 }

lib/std/Thread/RwLock.zig deleted-386

...	@@ -1,386 +0,0 @@
1	//! A lock that supports one writer or many readers.
2	//! This API is for kernel threads, not evented I/O.
3	//! This API requires being initialized at runtime, and initialization
4	//! can fail. Once initialized, the core operations cannot fail.
5
6	impl: Impl = .{},
7
8	const RwLock = @This();
9	const std = @import("../std.zig");
10	const builtin = @import("builtin");
11	const assert = std.debug.assert;
12	const testing = std.testing;
13
14	pub const Impl = if (builtin.single_threaded)
15	SingleThreadedRwLock
16	else if (std.Thread.use_pthreads)
17	PthreadRwLock
18	else
19	DefaultRwLock;
20
21	/// Attempts to obtain exclusive lock ownership.
22	/// Returns `true` if the lock is obtained, `false` otherwise.
23	pub fn tryLock(rwl: *RwLock) bool {
24	return rwl.impl.tryLock();
25	}
26
27	/// Blocks until exclusive lock ownership is acquired.
28	pub fn lock(rwl: *RwLock) void {
29	return rwl.impl.lock();
30	}
31
32	/// Releases a held exclusive lock.
33	/// Asserts the lock is held exclusively.
34	pub fn unlock(rwl: *RwLock) void {
35	return rwl.impl.unlock();
36	}
37
38	/// Attempts to obtain shared lock ownership.
39	/// Returns `true` if the lock is obtained, `false` otherwise.
40	pub fn tryLockShared(rwl: *RwLock) bool {
41	return rwl.impl.tryLockShared();
42	}
43
44	/// Obtains shared lock ownership.
45	/// Blocks if another thread has exclusive ownership.
46	/// May block if another thread is attempting to get exclusive ownership.
47	pub fn lockShared(rwl: *RwLock) void {
48	return rwl.impl.lockShared();
49	}
50
51	/// Releases a held shared lock.
52	pub fn unlockShared(rwl: *RwLock) void {
53	return rwl.impl.unlockShared();
54	}
55
56	/// Single-threaded applications use this for deadlock checks in
57	/// debug mode, and no-ops in release modes.
58	pub const SingleThreadedRwLock = struct {
59	state: enum { unlocked, locked_exclusive, locked_shared } = .unlocked,
60	shared_count: usize = 0,
61
62	/// Attempts to obtain exclusive lock ownership.
63	/// Returns `true` if the lock is obtained, `false` otherwise.
64	pub fn tryLock(rwl: *SingleThreadedRwLock) bool {
65	switch (rwl.state) {
66	.unlocked => {
67	assert(rwl.shared_count == 0);
68	rwl.state = .locked_exclusive;
69	return true;
70	},
71	.locked_exclusive, .locked_shared => return false,
72	}
73	}
74
75	/// Blocks until exclusive lock ownership is acquired.
76	pub fn lock(rwl: *SingleThreadedRwLock) void {
77	assert(rwl.state == .unlocked); // deadlock detected
78	assert(rwl.shared_count == 0); // corrupted state detected
79	rwl.state = .locked_exclusive;
80	}
81
82	/// Releases a held exclusive lock.
83	/// Asserts the lock is held exclusively.
84	pub fn unlock(rwl: *SingleThreadedRwLock) void {
85	assert(rwl.state == .locked_exclusive);
86	assert(rwl.shared_count == 0); // corrupted state detected
87	rwl.state = .unlocked;
88	}
89
90	/// Attempts to obtain shared lock ownership.
91	/// Returns `true` if the lock is obtained, `false` otherwise.
92	pub fn tryLockShared(rwl: *SingleThreadedRwLock) bool {
93	switch (rwl.state) {
94	.unlocked => {
95	rwl.state = .locked_shared;
96	assert(rwl.shared_count == 0);
97	rwl.shared_count = 1;
98	return true;
99	},
100	.locked_shared => {
101	rwl.shared_count += 1;
102	return true;
103	},
104	.locked_exclusive => return false,
105	}
106	}
107
108	/// Blocks until shared lock ownership is acquired.
109	pub fn lockShared(rwl: *SingleThreadedRwLock) void {
110	switch (rwl.state) {
111	.unlocked => {
112	rwl.state = .locked_shared;
113	assert(rwl.shared_count == 0);
114	rwl.shared_count = 1;
115	},
116	.locked_shared => {
117	rwl.shared_count += 1;
118	},
119	.locked_exclusive => unreachable, // deadlock detected
120	}
121	}
122
123	/// Releases a held shared lock.
124	pub fn unlockShared(rwl: *SingleThreadedRwLock) void {
125	switch (rwl.state) {
126	.unlocked => unreachable, // too many calls to `unlockShared`
127	.locked_exclusive => unreachable, // exclusively held lock
128	.locked_shared => {
129	rwl.shared_count -= 1;
130	if (rwl.shared_count == 0) {
131	rwl.state = .unlocked;
132	}
133	},
134	}
135	}
136	};
137
138	pub const PthreadRwLock = struct {
139	rwlock: std.c.pthread_rwlock_t = .{},
140
141	pub fn tryLock(rwl: *PthreadRwLock) bool {
142	return std.c.pthread_rwlock_trywrlock(&rwl.rwlock) == .SUCCESS;
143	}
144
145	pub fn lock(rwl: *PthreadRwLock) void {
146	const rc = std.c.pthread_rwlock_wrlock(&rwl.rwlock);
147	assert(rc == .SUCCESS);
148	}
149
150	pub fn unlock(rwl: *PthreadRwLock) void {
151	const rc = std.c.pthread_rwlock_unlock(&rwl.rwlock);
152	assert(rc == .SUCCESS);
153	}
154
155	pub fn tryLockShared(rwl: *PthreadRwLock) bool {
156	return std.c.pthread_rwlock_tryrdlock(&rwl.rwlock) == .SUCCESS;
157	}
158
159	pub fn lockShared(rwl: *PthreadRwLock) void {
160	const rc = std.c.pthread_rwlock_rdlock(&rwl.rwlock);
161	assert(rc == .SUCCESS);
162	}
163
164	pub fn unlockShared(rwl: *PthreadRwLock) void {
165	const rc = std.c.pthread_rwlock_unlock(&rwl.rwlock);
166	assert(rc == .SUCCESS);
167	}
168	};
169
170	pub const DefaultRwLock = struct {
171	state: usize = 0,
172	mutex: std.Thread.Mutex = .{},
173	semaphore: std.Thread.Semaphore = .{},
174
175	const IS_WRITING: usize = 1;
176	const WRITER: usize = 1 << 1;
177	const READER: usize = 1 << (1 + @bitSizeOf(Count));
178	const WRITER_MASK: usize = std.math.maxInt(Count) << @ctz(WRITER);
179	const READER_MASK: usize = std.math.maxInt(Count) << @ctz(READER);
180	const Count = std.meta.Int(.unsigned, @divFloor(@bitSizeOf(usize) - 1, 2));
181
182	pub fn tryLock(rwl: *DefaultRwLock) bool {
183	if (rwl.mutex.tryLock()) {
184	const state = @atomicLoad(usize, &rwl.state, .seq_cst);
185	if (state & READER_MASK == 0) {
186	_ = @atomicRmw(usize, &rwl.state, .Or, IS_WRITING, .seq_cst);
187	return true;
188	}
189
190	rwl.mutex.unlock();
191	}
192
193	return false;
194	}
195
196	pub fn lock(rwl: *DefaultRwLock) void {
197	_ = @atomicRmw(usize, &rwl.state, .Add, WRITER, .seq_cst);
198	rwl.mutex.lock();
199
200	const state = @atomicRmw(usize, &rwl.state, .Add, IS_WRITING -% WRITER, .seq_cst);
201	if (state & READER_MASK != 0)
202	rwl.semaphore.wait();
203	}
204
205	pub fn unlock(rwl: *DefaultRwLock) void {
206	_ = @atomicRmw(usize, &rwl.state, .And, ~IS_WRITING, .seq_cst);
207	rwl.mutex.unlock();
208	}
209
210	pub fn tryLockShared(rwl: *DefaultRwLock) bool {
211	const state = @atomicLoad(usize, &rwl.state, .seq_cst);
212	if (state & (IS_WRITING \| WRITER_MASK) == 0) {
213	_ = @cmpxchgStrong(
214	usize,
215	&rwl.state,
216	state,
217	state + READER,
218	.seq_cst,
219	.seq_cst,
220	) orelse return true;
221	}
222
223	if (rwl.mutex.tryLock()) {
224	_ = @atomicRmw(usize, &rwl.state, .Add, READER, .seq_cst);
225	rwl.mutex.unlock();
226	return true;
227	}
228
229	return false;
230	}
231
232	pub fn lockShared(rwl: *DefaultRwLock) void {
233	var state = @atomicLoad(usize, &rwl.state, .seq_cst);
234	while (state & (IS_WRITING \| WRITER_MASK) == 0) {
235	state = @cmpxchgWeak(
236	usize,
237	&rwl.state,
238	state,
239	state + READER,
240	.seq_cst,
241	.seq_cst,
242	) orelse return;
243	}
244
245	rwl.mutex.lock();
246	_ = @atomicRmw(usize, &rwl.state, .Add, READER, .seq_cst);
247	rwl.mutex.unlock();
248	}
249
250	pub fn unlockShared(rwl: *DefaultRwLock) void {
251	const state = @atomicRmw(usize, &rwl.state, .Sub, READER, .seq_cst);
252
253	if ((state & READER_MASK == READER) and (state & IS_WRITING != 0))
254	rwl.semaphore.post();
255	}
256	};
257
258	test "DefaultRwLock - internal state" {
259	var rwl = DefaultRwLock{};
260
261	// The following failed prior to the fix for Issue #13163,
262	// where the WRITER flag was subtracted by the lock method.
263
264	rwl.lock();
265	rwl.unlock();
266	try testing.expectEqual(rwl, DefaultRwLock{});
267	}
268
269	test "smoke test" {
270	var rwl = RwLock{};
271
272	rwl.lock();
273	try testing.expect(!rwl.tryLock());
274	try testing.expect(!rwl.tryLockShared());
275	rwl.unlock();
276
277	try testing.expect(rwl.tryLock());
278	try testing.expect(!rwl.tryLock());
279	try testing.expect(!rwl.tryLockShared());
280	rwl.unlock();
281
282	rwl.lockShared();
283	try testing.expect(!rwl.tryLock());
284	try testing.expect(rwl.tryLockShared());
285	rwl.unlockShared();
286	rwl.unlockShared();
287
288	try testing.expect(rwl.tryLockShared());
289	try testing.expect(!rwl.tryLock());
290	try testing.expect(rwl.tryLockShared());
291	rwl.unlockShared();
292	rwl.unlockShared();
293
294	rwl.lock();
295	rwl.unlock();
296	}
297
298	test "concurrent access" {
299	if (builtin.single_threaded)
300	return;
301
302	const num_writers: usize = 2;
303	const num_readers: usize = 4;
304	const num_writes: usize = 1000;
305	const num_reads: usize = 2000;
306
307	const Runner = struct {
308	const Runner = @This();
309
310	rwl: RwLock,
311	writes: usize,
312	reads: std.atomic.Value(usize),
313
314	val_a: usize,
315	val_b: usize,
316
317	fn reader(run: *Runner, thread_idx: usize) !void {
318	var prng = std.Random.DefaultPrng.init(thread_idx);
319	const rnd = prng.random();
320	while (true) {
321	run.rwl.lockShared();
322	defer run.rwl.unlockShared();
323
324	try testing.expect(run.writes <= num_writes);
325	if (run.reads.fetchAdd(1, .monotonic) >= num_reads) break;
326
327	// We use `volatile` accesses so that we can make sure the memory is accessed either
328	// side of a yield, maximising chances of a race.
329	const a_ptr: *const volatile usize = &run.val_a;
330	const b_ptr: *const volatile usize = &run.val_b;
331
332	const old_a = a_ptr.*;
333	if (rnd.boolean()) try std.Thread.yield();
334	const old_b = b_ptr.*;
335	try testing.expect(old_a == old_b);
336	}
337	}
338
339	fn writer(run: *Runner, thread_idx: usize) !void {
340	var prng = std.Random.DefaultPrng.init(thread_idx);
341	const rnd = prng.random();
342	while (true) {
343	run.rwl.lock();
344	defer run.rwl.unlock();
345
346	try testing.expect(run.writes <= num_writes);
347	if (run.writes == num_writes) break;
348
349	// We use `volatile` accesses so that we can make sure the memory is accessed either
350	// side of a yield, maximising chances of a race.
351	const a_ptr: *volatile usize = &run.val_a;
352	const b_ptr: *volatile usize = &run.val_b;
353
354	const new_val = rnd.int(usize);
355
356	const old_a = a_ptr.*;
357	a_ptr.* = new_val;
358	if (rnd.boolean()) try std.Thread.yield();
359	const old_b = b_ptr.*;
360	b_ptr.* = new_val;
361	try testing.expect(old_a == old_b);
362
363	run.writes += 1;
364	}
365	}
366	};
367
368	var run: Runner = .{
369	.rwl = .{},
370	.writes = 0,
371	.reads = .init(0),
372	.val_a = 0,
373	.val_b = 0,
374	};
375	var write_threads: [num_writers]std.Thread = undefined;
376	var read_threads: [num_readers]std.Thread = undefined;
377
378	for (&write_threads, 0..) \|t, i\| t. = try .spawn(.{}, Runner.writer, .{ &run, i });
379	for (&read_threads, num_writers..) \|t, i\| t. = try .spawn(.{}, Runner.reader, .{ &run, i });
380
381	for (write_threads) \|t\| t.join();
382	for (read_threads) \|t\| t.join();
383
384	try testing.expect(run.writes == num_writes);
385	try testing.expect(run.reads.raw >= num_reads);
386	}

lib/std/Thread/Semaphore.zig deleted-111

...	@@ -1,111 +0,0 @@
1	//! A semaphore is an unsigned integer that blocks the kernel thread if
2	//! the number would become negative.
3	//! This API supports static initialization and does not require deinitialization.
4	//!
5	//! Example:
6	//! ```
7	//! var s = Semaphore{};
8	//!
9	//! fn consumer() void {
10	//! s.wait();
11	//! }
12	//!
13	//! fn producer() void {
14	//! s.post();
15	//! }
16	//!
17	//! const thread = try std.Thread.spawn(.{}, producer, .{});
18	//! consumer();
19	//! thread.join();
20	//! ```
21
22	mutex: Mutex = .{},
23	cond: Condition = .{},
24	/// It is OK to initialize this field to any value.
25	permits: usize = 0,
26
27	const Semaphore = @This();
28	const std = @import("../std.zig");
29	const Mutex = std.Thread.Mutex;
30	const Condition = std.Thread.Condition;
31	const builtin = @import("builtin");
32	const testing = std.testing;
33
34	pub fn wait(sem: *Semaphore) void {
35	sem.mutex.lock();
36	defer sem.mutex.unlock();
37
38	while (sem.permits == 0)
39	sem.cond.wait(&sem.mutex);
40
41	sem.permits -= 1;
42	if (sem.permits > 0)
43	sem.cond.signal();
44	}
45
46	pub fn timedWait(sem: *Semaphore, timeout_ns: u64) error{Timeout}!void {
47	var timeout_timer = std.time.Timer.start() catch unreachable;
48
49	sem.mutex.lock();
50	defer sem.mutex.unlock();
51
52	while (sem.permits == 0) {
53	const elapsed = timeout_timer.read();
54	if (elapsed > timeout_ns)
55	return error.Timeout;
56
57	const local_timeout_ns = timeout_ns - elapsed;
58	try sem.cond.timedWait(&sem.mutex, local_timeout_ns);
59	}
60
61	sem.permits -= 1;
62	if (sem.permits > 0)
63	sem.cond.signal();
64	}
65
66	pub fn post(sem: *Semaphore) void {
67	sem.mutex.lock();
68	defer sem.mutex.unlock();
69
70	sem.permits += 1;
71	sem.cond.signal();
72	}
73
74	test Semaphore {
75	if (builtin.single_threaded) {
76	return error.SkipZigTest;
77	}
78
79	const TestContext = struct {
80	sem: *Semaphore,
81	n: *i32,
82	fn worker(ctx: *@This()) void {
83	ctx.sem.wait();
84	ctx.n.* += 1;
85	ctx.sem.post();
86	}
87	};
88	const num_threads = 3;
89	var sem = Semaphore{ .permits = 1 };
90	var threads: [num_threads]std.Thread = undefined;
91	var n: i32 = 0;
92	var ctx = TestContext{ .sem = &sem, .n = &n };
93
94	for (&threads) \|t\| t. = try std.Thread.spawn(.{}, TestContext.worker, .{&ctx});
95	for (threads) \|t\| t.join();
96	sem.wait();
97	try testing.expect(n == num_threads);
98	}
99
100	test timedWait {
101	var sem = Semaphore{};
102	try testing.expectEqual(0, sem.permits);
103
104	try testing.expectError(error.Timeout, sem.timedWait(1));
105
106	sem.post();
107	try testing.expectEqual(1, sem.permits);
108
109	try sem.timedWait(1);
110	try testing.expectEqual(0, sem.permits);
111	}

lib/std/debug.zig+6-3

lib/std/debug/Coverage.zig+11-9

 @@ -1,11 +1,12 @@
+const Coverage = @This();
 const std = @import("../std.zig");
+const Io = std.Io;
 const Allocator = std.mem.Allocator;
 const Hash = std.hash.Wyhash;
 const Dwarf = std.debug.Dwarf;
 const assert = std.debug.assert;
-const Coverage = @This();
 /// Provides a globally-scoped integer index for directories.
 ///
 /// As opposed to, for example, a directory index that is compilation-unit
 @@ -23,12 +24,12 @@ directories: std.ArrayHashMapUnmanaged(String, void, String.MapContext, false),
 files: std.ArrayHashMapUnmanaged(File, void, File.MapContext, false),
 string_bytes: std.ArrayList(u8),
 /// Protects the other fields.
-mutex: std.Thread.Mutex,
+mutex: Io.Mutex,
 pub const init: Coverage = .{
     .directories = .{},
     .files = .{},
-    .mutex = .{},
+    .mutex = .init,
     .string_bytes = .{},
 };
 @@ -140,11 +141,12 @@ pub fn stringAt(cov: *Coverage, index: String) [:0]const u8 {
     return span(cov.string_bytes.items[@intFromEnum(index)..]);
 }
-pub const ResolveAddressesDwarfError = Dwarf.ScanError;
+pub const ResolveAddressesDwarfError = Dwarf.ScanError || Io.Cancelable;
 pub fn resolveAddressesDwarf(
     cov: *Coverage,
     gpa: Allocator,
+    io: Io,
     endian: std.builtin.Endian,
     /// Asserts the addresses are in ascending order.
     sorted_pc_addrs: []const u64,
 @@ -161,8 +163,8 @@ pub fn resolveAddressesDwarf(
     var prev_pc: u64 = 0;
     var prev_cu: ?*std.debug.Dwarf.CompileUnit = null;
     // Protects directories and files tables from other threads.
-    cov.mutex.lock();
+    try cov.mutex.lock(io);
-    defer cov.mutex.unlock();
+    defer cov.mutex.unlock(io);
     next_pc: for (sorted_pc_addrs, output) |pc, *out| {
         assert(pc >= prev_pc);
         prev_pc = pc;
 @@ -183,8 +185,8 @@ pub fn resolveAddressesDwarf(
         if (cu != prev_cu) {
             prev_cu = cu;
             if (cu.src_loc_cache == null) {
-                cov.mutex.unlock();
+                cov.mutex.unlock(io);
-                defer cov.mutex.lock();
+                defer cov.mutex.lockUncancelable(io);
                 d.populateSrcLocCache(gpa, endian, cu) catch |err| switch (err) {
                     error.MissingDebugInfo, error.InvalidDebugInfo => {
                         out.* = SourceLocation.invalid;

lib/std/debug/Info.zig+2-2

...	@@ -93,7 +93,7 @@ pub fn resolveAddresses(	...	@@ -93,7 +93,7 @@ pub fn resolveAddresses(
93	) ResolveAddressesError!void {	93	) ResolveAddressesError!void {
94	assert(sorted_pc_addrs.len == output.len);	94	assert(sorted_pc_addrs.len == output.len);
95	switch (info.impl) {	95	switch (info.impl) {
96	.elf => \|*ef\| return info.coverage.resolveAddressesDwarf(gpa, ef.endian, sorted_pc_addrs, output, &ef.dwarf.?),	96	.elf => \|*ef\| return info.coverage.resolveAddressesDwarf(gpa, io, ef.endian, sorted_pc_addrs, output, &ef.dwarf.?),
97	.macho => \|*mf\| {	97	.macho => \|*mf\| {
98	// Resolving all of the addresses at once unfortunately isn't so easy in Mach-O binaries	98	// Resolving all of the addresses at once unfortunately isn't so easy in Mach-O binaries
99	// due to split debug information. For now, we'll just resolve the addreses one by one.	99	// due to split debug information. For now, we'll just resolve the addreses one by one.
...	@@ -112,7 +112,7 @@ pub fn resolveAddresses(	...	@@ -112,7 +112,7 @@ pub fn resolveAddresses(
112	else => \|e\| return e,	112	else => \|e\| return e,
113	};	113	};
114	}	114	}
115	try info.coverage.resolveAddressesDwarf(gpa, .little, &.{dwarf_pc_addr}, src_loc[0..1], dwarf);	115	try info.coverage.resolveAddressesDwarf(gpa, io, .little, &.{dwarf_pc_addr}, src_loc[0..1], dwarf);
116	}	116	}
117	},	117	},
118	}	118	}

lib/std/debug/SelfInfo/Elf.zig+22-28

 @@ -1,4 +1,4 @@
-rwlock: std.Thread.RwLock,
+mutex: Io.Mutex,
 modules: std.ArrayList(Module),
 ranges: std.ArrayList(Module.Range),
 @@ -6,7 +6,7 @@ ranges: std.ArrayList(Module.Range),
 unwind_cache: if (can_unwind) ?[]Dwarf.SelfUnwinder.CacheEntry else ?noreturn,
 pub const init: SelfInfo = .{
-    .rwlock = .{},
+    .mutex = .init,
     .modules = .empty,
     .ranges = .empty,
     .unwind_cache = null,
 @@ -29,8 +29,8 @@ pub fn deinit(si: *SelfInfo, gpa: Allocator) void {
 }
 pub fn getSymbol(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!std.debug.Symbol {
-    const module = try si.findModule(gpa, address, .exclusive);
+    const module = try si.findModule(gpa, io, address, .exclusive);
-    defer si.rwlock.unlock();
+    defer si.mutex.unlock(io);
     const vaddr = address - module.load_offset;
 @@ -73,15 +73,15 @@ pub fn getSymbol(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!st
         error.OutOfMemory => |e| return e,
     };
 }
-pub fn getModuleName(si: *SelfInfo, gpa: Allocator, address: usize) Error![]const u8 {
+pub fn getModuleName(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error![]const u8 {
-    const module = try si.findModule(gpa, address, .shared);
+    const module = try si.findModule(gpa, io, address, .shared);
-    defer si.rwlock.unlockShared();
+    defer si.mutex.unlock(io);
     if (module.name.len == 0) return error.MissingDebugInfo;
     return module.name;
 }
-pub fn getModuleSlide(si: *SelfInfo, gpa: Allocator, address: usize) Error!usize {
+pub fn getModuleSlide(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!usize {
-    const module = try si.findModule(gpa, address, .shared);
+    const module = try si.findModule(gpa, io, address, .shared);
-    defer si.rwlock.unlockShared();
+    defer si.mutex.unlock(io);
     return module.load_offset;
 }
 @@ -183,8 +183,8 @@ pub fn unwindFrame(si: *SelfInfo, gpa: Allocator, io: Io, context: *UnwindContex
     comptime assert(can_unwind);
     {
-        si.rwlock.lockShared();
+        try si.mutex.lock(io);
-        defer si.rwlock.unlockShared();
+        defer si.mutex.unlock(io);
         if (si.unwind_cache) |cache| {
             if (Dwarf.SelfUnwinder.CacheEntry.find(cache, context.pc)) |entry| {
                 return context.next(gpa, entry);
 @@ -192,8 +192,8 @@ pub fn unwindFrame(si: *SelfInfo, gpa: Allocator, io: Io, context: *UnwindContex
         }
     }
-    const module = try si.findModule(gpa, context.pc, .exclusive);
+    const module = try si.findModule(gpa, io, context.pc, .exclusive);
-    defer si.rwlock.unlock();
+    defer si.mutex.unlock(io);
     if (si.unwind_cache == null) {
         si.unwind_cache = try gpa.alloc(Dwarf.SelfUnwinder.CacheEntry, 2048);
 @@ -375,11 +375,11 @@ const Module = struct {
     }
 };
-fn findModule(si: *SelfInfo, gpa: Allocator, address: usize, lock: enum { shared, exclusive }) Error!*Module {
+fn findModule(si: *SelfInfo, gpa: Allocator, io: Io, address: usize, lock: enum { shared, exclusive }) Error!*Module {
     // With the requested lock, scan the module ranges looking for `address`.
     switch (lock) {
-        .shared => si.rwlock.lockShared(),
+        .shared => try si.mutex.lock(io),
-        .exclusive => si.rwlock.lock(),
+        .exclusive => try si.mutex.lock(io),
     }
     for (si.ranges.items) |*range| {
         if (address >= range.start and address < range.start + range.len) {
 @@ -389,15 +389,12 @@ fn findModule(si: *SelfInfo, gpa: Allocator, address: usize, lock: enum { shared
     // The address wasn't in a known range. We will rebuild the module/range lists, since it's possible
     // a new module was loaded. Upgrade to an exclusive lock if necessary.
     switch (lock) {
-        .shared => {
+        .shared => {},
-            si.rwlock.unlockShared();
-            si.rwlock.lock();
-        },
         .exclusive => {},
     }
     // Rebuild module list with the exclusive lock.
     {
-        errdefer si.rwlock.unlock();
+        errdefer si.mutex.unlock(io);
         for (si.modules.items) |*mod| {
             unwind: {
                 const u = &(mod.unwind orelse break :unwind catch break :unwind);
 @@ -415,10 +412,7 @@ fn findModule(si: *SelfInfo, gpa: Allocator, address: usize, lock: enum { shared
     }
     // Downgrade the lock back to shared if necessary.
     switch (lock) {
-        .shared => {
+        .shared => {},
-            si.rwlock.unlock();
-            si.rwlock.lockShared();
-        },
         .exclusive => {},
     }
     // Scan the newly rebuilt module ranges.
 @@ -429,8 +423,8 @@ fn findModule(si: *SelfInfo, gpa: Allocator, address: usize, lock: enum { shared
     }
     // Still nothing; unlock and error.
     switch (lock) {
-        .shared => si.rwlock.unlockShared(),
+        .shared => si.mutex.unlock(io),
-        .exclusive => si.rwlock.unlock(),
+        .exclusive => si.mutex.unlock(io),
     }
     return error.MissingDebugInfo;
 }

lib/std/debug/SelfInfo/MachO.zig+16-16

 @@ -1,9 +1,9 @@
-mutex: std.Thread.Mutex,
+mutex: Io.Mutex,
 /// Accessed through `Module.Adapter`.
 modules: std.ArrayHashMapUnmanaged(Module, void, Module.Context, false),
 pub const init: SelfInfo = .{
-    .mutex = .{},
+    .mutex = .init,
     .modules = .empty,
 };
 pub fn deinit(si: *SelfInfo, gpa: Allocator) void {
 @@ -21,8 +21,8 @@ pub fn deinit(si: *SelfInfo, gpa: Allocator) void {
 }
 pub fn getSymbol(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!std.debug.Symbol {
-    const module = try si.findModule(gpa, address);
+    const module = try si.findModule(gpa, io, address);
-    defer si.mutex.unlock();
+    defer si.mutex.unlock(io);
     const file = try module.getFile(gpa, io);
 @@ -76,9 +76,10 @@ pub fn getSymbol(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!st
         ) catch null,
     };
 }
-pub fn getModuleName(si: *SelfInfo, gpa: Allocator, address: usize) Error![]const u8 {
+pub fn getModuleName(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error![]const u8 {
     _ = si;
     _ = gpa;
+    _ = io;
     // This function is marked as deprecated; however, it is significantly more
     // performant than `dladdr` (since the latter also does a very slow symbol
     // lookup), so let's use it since it's still available.
 @@ -86,9 +87,9 @@ pub fn getModuleName(si: *SelfInfo, gpa: Allocator, address: usize) Error![]cons
         @ptrFromInt(address),
     ) orelse return error.MissingDebugInfo);
 }
-pub fn getModuleSlide(si: *SelfInfo, gpa: Allocator, address: usize) Error!usize {
+pub fn getModuleSlide(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!usize {
-    const module = try si.findModule(gpa, address);
+    const module = try si.findModule(gpa, io, address);
-    defer si.mutex.unlock();
+    defer si.mutex.unlock(io);
     const header: *std.macho.mach_header_64 = @ptrFromInt(module.text_base);
     const raw_macho: [*]u8 = @ptrCast(header);
     var it = macho.LoadCommandIterator.init(header, raw_macho[@sizeOf(macho.mach_header_64)..][0..header.sizeofcmds]) catch unreachable;
 @@ -107,8 +108,7 @@ pub const UnwindContext = std.debug.Dwarf.SelfUnwinder;
 /// If the compact encoding can't encode a way to unwind a frame, it will
 /// defer unwinding to DWARF, in which case `__eh_frame` will be used if available.
 pub fn unwindFrame(si: *SelfInfo, gpa: Allocator, io: Io, context: *UnwindContext) Error!usize {
-    _ = io;
+    return unwindFrameInner(si, gpa, io, context) catch |err| switch (err) {
-    return unwindFrameInner(si, gpa, context) catch |err| switch (err) {
         error.InvalidDebugInfo,
         error.MissingDebugInfo,
         error.UnsupportedDebugInfo,
 @@ -134,9 +134,9 @@ pub fn unwindFrame(si: *SelfInfo, gpa: Allocator, io: Io, context: *UnwindContex
         => return error.InvalidDebugInfo,
     };
 }
-fn unwindFrameInner(si: *SelfInfo, gpa: Allocator, context: *UnwindContext) !usize {
+fn unwindFrameInner(si: *SelfInfo, gpa: Allocator, io: Io, context: *UnwindContext) !usize {
-    const module = try si.findModule(gpa, context.pc);
+    const module = try si.findModule(gpa, io, context.pc);
-    defer si.mutex.unlock();
+    defer si.mutex.unlock(io);
     const unwind: *Module.Unwind = try module.getUnwindInfo(gpa);
 @@ -430,15 +430,15 @@ fn unwindFrameInner(si: *SelfInfo, gpa: Allocator, context: *UnwindContext) !usi
 }
 /// Acquires the mutex on success.
-fn findModule(si: *SelfInfo, gpa: Allocator, address: usize) Error!*Module {
+fn findModule(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!*Module {
     // This function is marked as deprecated; however, it is significantly more
     // performant than `dladdr` (since the latter also does a very slow symbol
     // lookup), so let's use it since it's still available.
     const text_base = std.c._dyld_get_image_header_containing_address(
         @ptrFromInt(address),
     ) orelse return error.MissingDebugInfo;
-    si.mutex.lock();
+    try si.mutex.lock(io);
-    errdefer si.mutex.unlock();
+    errdefer si.mutex.unlock(io);
     const gop = try si.modules.getOrPutAdapted(gpa, @intFromPtr(text_base), Module.Adapter{});
     errdefer comptime unreachable;
     if (!gop.found_existing) gop.key_ptr.* = .{

lib/std/debug/SelfInfo/Windows.zig+10-10

...	@@ -1,9 +1,9 @@	...	@@ -1,9 +1,9 @@
1	mutex: std.Thread.Mutex,	1	mutex: Io.Mutex,
2	modules: std.ArrayList(Module),	2	modules: std.ArrayList(Module),
3	module_name_arena: std.heap.ArenaAllocator.State,	3	module_name_arena: std.heap.ArenaAllocator.State,
4		4
5	pub const init: SelfInfo = .{	5	pub const init: SelfInfo = .{
6	.mutex = .{},	6	.mutex = .init,
7	.modules = .empty,	7	.modules = .empty,
8	.module_name_arena = .{},	8	.module_name_arena = .{},
9	};	9	};
...	@@ -21,21 +21,21 @@ pub fn deinit(si: *SelfInfo, gpa: Allocator) void {	...	@@ -21,21 +21,21 @@ pub fn deinit(si: *SelfInfo, gpa: Allocator) void {
21	}	21	}
22		22
23	pub fn getSymbol(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!std.debug.Symbol {	23	pub fn getSymbol(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!std.debug.Symbol {
24	si.mutex.lock();	24	try si.mutex.lock(io);
25	defer si.mutex.unlock();	25	defer si.mutex.unlock(io);
26	const module = try si.findModule(gpa, address);	26	const module = try si.findModule(gpa, address);
27	const di = try module.getDebugInfo(gpa, io);	27	const di = try module.getDebugInfo(gpa, io);
28	return di.getSymbol(gpa, address - module.base_address);	28	return di.getSymbol(gpa, address - module.base_address);
29	}	29	}
30	pub fn getModuleName(si: *SelfInfo, gpa: Allocator, address: usize) Error![]const u8 {	30	pub fn getModuleName(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error![]const u8 {
31	si.mutex.lock();	31	try si.mutex.lock(io);
32	defer si.mutex.unlock();	32	defer si.mutex.unlock(io);
33	const module = try si.findModule(gpa, address);	33	const module = try si.findModule(gpa, address);
34	return module.name;	34	return module.name;
35	}	35	}
36	pub fn getModuleSlide(si: *SelfInfo, gpa: Allocator, address: usize) Error!usize {	36	pub fn getModuleSlide(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!usize {
37	si.mutex.lock();	37	try si.mutex.lock(io);
38	defer si.mutex.unlock();	38	defer si.mutex.unlock(io);
39	const module = try si.findModule(gpa, address);	39	const module = try si.findModule(gpa, address);
40	return module.base_address;	40	return module.base_address;
41	}	41	}

lib/std/heap/ThreadSafeAllocator.zig+21-14

 @@ -1,7 +1,14 @@
-//! Wraps a non-thread-safe allocator and makes it thread-safe.
+//! Deprecated. Thread safety should be built into each Allocator instance
+//! directly rather than trying to do this "composable allocators" thing.
+const ThreadSafeAllocator = @This();
+const std = @import("../std.zig");
+const Io = std.Io;
+const Allocator = std.mem.Allocator;
 child_allocator: Allocator,
-mutex: std.Thread.Mutex = .{},
+io: Io,
+mutex: Io.Mutex = .init,
 pub fn allocator(self: *ThreadSafeAllocator) Allocator {
     return .{
 @@ -17,39 +24,39 @@ pub fn allocator(self: *ThreadSafeAllocator) Allocator {
 fn alloc(ctx: *anyopaque, n: usize, alignment: std.mem.Alignment, ra: usize) ?[*]u8 {
     const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx));
-    self.mutex.lock();
+    const io = self.io;
-    defer self.mutex.unlock();
+    self.mutex.lockUncancelable(io);
+    defer self.mutex.unlock(io);
     return self.child_allocator.rawAlloc(n, alignment, ra);
 }
 fn resize(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool {
     const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx));
+    const io = self.io;
-    self.mutex.lock();
+    self.mutex.lockUncancelable(io);
-    defer self.mutex.unlock();
+    defer self.mutex.unlock(io);
     return self.child_allocator.rawResize(buf, alignment, new_len, ret_addr);
 }
 fn remap(context: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, return_address: usize) ?[*]u8 {
     const self: *ThreadSafeAllocator = @ptrCast(@alignCast(context));
+    const io = self.io;
-    self.mutex.lock();
+    self.mutex.lockUncancelable(io);
-    defer self.mutex.unlock();
+    defer self.mutex.unlock(io);
     return self.child_allocator.rawRemap(memory, alignment, new_len, return_address);
 }
 fn free(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, ret_addr: usize) void {
     const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx));
+    const io = self.io;
-    self.mutex.lock();
+    self.mutex.lockUncancelable(io);
-    defer self.mutex.unlock();
+    defer self.mutex.unlock(io);
     return self.child_allocator.rawFree(buf, alignment, ret_addr);
 }
-const std = @import("../std.zig");
-const ThreadSafeAllocator = @This();
-const Allocator = std.mem.Allocator;

lib/std/heap/debug_allocator.zig+10-41

 @@ -126,16 +126,6 @@ pub const Config = struct {
     /// Whether the allocator may be used simultaneously from multiple threads.
     thread_safe: bool = !builtin.single_threaded,
-    /// What type of mutex you'd like to use, for thread safety.
-    /// when specified, the mutex type must have the same shape as `std.Thread.Mutex` and
-    /// `DummyMutex`, and have no required fields. Specifying this field causes
-    /// the `thread_safe` field to be ignored.
-    ///
-    /// when null (default):
-    /// * the mutex type defaults to `std.Thread.Mutex` when thread_safe is enabled.
-    /// * the mutex type defaults to `DummyMutex` otherwise.
-    MutexType: ?type = null,
     /// This is a temporary debugging trick you can use to turn segfaults into more helpful
     /// logged error messages with stack trace details. The downside is that every allocation
     /// will be leaked, unless used with retain_metadata!
 @@ -204,17 +194,8 @@ pub fn DebugAllocator(comptime config: Config) type {
         const total_requested_bytes_init = if (config.enable_memory_limit) @as(usize, 0) else {};
         const requested_memory_limit_init = if (config.enable_memory_limit) @as(usize, math.maxInt(usize)) else {};
-        const mutex_init = if (config.MutexType) |T|
+        const have_mutex = config.thread_safe;
-            T{}
+        const mutex_init = if (have_mutex) std.Io.Mutex.init else {};
-        else if (config.thread_safe)
-            std.Thread.Mutex{}
-        else
-            DummyMutex{};
-        const DummyMutex = struct {
-            inline fn lock(_: DummyMutex) void {}
-            inline fn unlock(_: DummyMutex) void {}
-        };
         const stack_n = config.stack_trace_frames;
         const one_trace_size = @sizeOf(usize) * stack_n;
 @@ -737,8 +718,8 @@ pub fn DebugAllocator(comptime config: Config) type {
         fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ret_addr: usize) ?[*]u8 {
             const self: *Self = @ptrCast(@alignCast(context));
-            self.mutex.lock();
+            if (have_mutex) std.Io.Threaded.mutexLock(&self.mutex);
-            defer self.mutex.unlock();
+            defer if (have_mutex) std.Io.Threaded.mutexUnlock(&self.mutex);
             if (config.enable_memory_limit) {
                 const new_req_bytes = self.total_requested_bytes + len;
 @@ -850,8 +831,8 @@ pub fn DebugAllocator(comptime config: Config) type {
             return_address: usize,
         ) bool {
             const self: *Self = @ptrCast(@alignCast(context));
-            self.mutex.lock();
+            if (have_mutex) std.Io.Threaded.mutexLock(&self.mutex);
-            defer self.mutex.unlock();
+            defer if (have_mutex) std.Io.Threaded.mutexUnlock(&self.mutex);
             const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment));
             if (size_class_index >= self.buckets.len) {
 @@ -869,8 +850,8 @@ pub fn DebugAllocator(comptime config: Config) type {
             return_address: usize,
         ) ?[*]u8 {
             const self: *Self = @ptrCast(@alignCast(context));
-            self.mutex.lock();
+            if (have_mutex) std.Io.Threaded.mutexLock(&self.mutex);
-            defer self.mutex.unlock();
+            defer if (have_mutex) std.Io.Threaded.mutexUnlock(&self.mutex);
             const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment));
             if (size_class_index >= self.buckets.len) {
 @@ -887,8 +868,8 @@ pub fn DebugAllocator(comptime config: Config) type {
             return_address: usize,
         ) void {
             const self: *Self = @ptrCast(@alignCast(context));
-            self.mutex.lock();
+            if (have_mutex) std.Io.Threaded.mutexLock(&self.mutex);
-            defer self.mutex.unlock();
+            defer if (have_mutex) std.Io.Threaded.mutexUnlock(&self.mutex);
             const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(old_memory.len - 1), @intFromEnum(alignment));
             if (size_class_index >= self.buckets.len) {
 @@ -1331,18 +1312,6 @@ test "realloc large object to small object" {
     try std.testing.expect(slice[16] == 0x34);
 }
-test "overridable mutexes" {
-    var gpa = DebugAllocator(.{ .MutexType = std.Thread.Mutex }){
-        .backing_allocator = std.testing.allocator,
-        .mutex = std.Thread.Mutex{},
-    };
-    defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
-    const allocator = gpa.allocator();
-    const ptr = try allocator.create(i32);
-    defer allocator.destroy(ptr);
-}
 test "non-page-allocator backing allocator" {
     var gpa: DebugAllocator(.{
         .backing_allocator_zeroes = false,

lib/std/heap/sbrk_allocator.zig+10-8

 @@ -1,5 +1,7 @@
-const std = @import("../std.zig");
 const builtin = @import("builtin");
+const std = @import("../std.zig");
+const Io = std.Io;
 const math = std.math;
 const Allocator = std.mem.Allocator;
 const mem = std.mem;
 @@ -39,12 +41,12 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
         var big_frees = [1]usize{0} ** big_size_class_count;
         // TODO don't do the naive locking strategy
-        var lock: std.Thread.Mutex = .{};
+        var mutex: Io.Mutex = .{};
         fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, return_address: usize) ?[*]u8 {
             _ = ctx;
             _ = return_address;
-            lock.lock();
+            Io.Threaded.mutexLock(&mutex);
-            defer lock.unlock();
+            defer Io.Threaded.mutexUnlock(&mutex);
             // Make room for the freelist next pointer.
             const actual_len = @max(len +| @sizeOf(usize), alignment.toByteUnits());
             const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null;
 @@ -88,8 +90,8 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
         ) bool {
             _ = ctx;
             _ = return_address;
-            lock.lock();
+            Io.Threaded.mutexLock(&mutex);
-            defer lock.unlock();
+            defer Io.Threaded.mutexUnlock(&mutex);
             // We don't want to move anything from one size class to another, but we
             // can recover bytes in between powers of two.
             const buf_align = alignment.toByteUnits();
 @@ -127,8 +129,8 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
         ) void {
             _ = ctx;
             _ = return_address;
-            lock.lock();
+            Io.Threaded.mutexLock(&mutex);
-            defer lock.unlock();
+            defer Io.Threaded.mutexUnlock(&mutex);
             const buf_align = alignment.toByteUnits();
             const actual_len = @max(buf.len + @sizeOf(usize), buf_align);
             const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len);

lib/std/http/Client.zig+36-31

 @@ -3,22 +3,25 @@
 //! Connections are opened in a thread-safe manner, but individual Requests are not.
 //!
 //! TLS support may be disabled via `std.options.http_disable_tls`.
+//!
+//! TODO all the lockUncancelable in this file should be changed to regular lock and
+//! `error.Canceled` added to more error sets.
+const Client = @This();
-const std = @import("../std.zig");
 const builtin = @import("builtin");
+const std = @import("../std.zig");
+const Io = std.Io;
 const testing = std.testing;
 const http = std.http;
 const mem = std.mem;
 const Uri = std.Uri;
-const Allocator = mem.Allocator;
+const Allocator = std.mem.Allocator;
 const assert = std.debug.assert;
-const Io = std.Io;
 const Writer = std.Io.Writer;
 const Reader = std.Io.Reader;
 const HostName = std.Io.net.HostName;
-const Client = @This();
 pub const disable_tls = std.options.http_disable_tls;
 /// Used for all client allocations. Must be thread-safe.
 @@ -27,7 +30,7 @@ allocator: Allocator,
 io: Io,
 ca_bundle: if (disable_tls) void else std.crypto.Certificate.Bundle = if (disable_tls) {} else .{},
-ca_bundle_mutex: std.Thread.Mutex = .{},
+ca_bundle_mutex: Io.Mutex = .init,
 /// Used both for the reader and writer buffers.
 tls_buffer_size: if (disable_tls) u0 else usize = if (disable_tls) 0 else std.crypto.tls.Client.min_buffer_len,
 /// If non-null, ssl secrets are logged to a stream. Creating such a stream
 @@ -62,7 +65,7 @@ https_proxy: ?*Proxy = null,
 /// A Least-Recently-Used cache of open connections to be reused.
 pub const ConnectionPool = struct {
-    mutex: std.Thread.Mutex = .{},
+    mutex: Io.Mutex = .init,
     /// Open connections that are currently in use.
     used: std.DoublyLinkedList = .{},
     /// Open connections that are not currently in use.
 @@ -81,9 +84,9 @@ pub const ConnectionPool = struct {
     /// If no connection is found, null is returned.
     ///
     /// Threadsafe.
-    pub fn findConnection(pool: *ConnectionPool, criteria: Criteria) ?*Connection {
+    pub fn findConnection(pool: *ConnectionPool, io: Io, criteria: Criteria) ?*Connection {
-        pool.mutex.lock();
+        pool.mutex.lockUncancelable(io);
-        defer pool.mutex.unlock();
+        defer pool.mutex.unlock(io);
         var next = pool.free.last;
         while (next) |node| : (next = node.prev) {
 @@ -110,9 +113,9 @@ pub const ConnectionPool = struct {
     }
     /// Acquires an existing connection from the connection pool. This function is threadsafe.
-    pub fn acquire(pool: *ConnectionPool, connection: *Connection) void {
+    pub fn acquire(pool: *ConnectionPool, io: Io, connection: *Connection) void {
-        pool.mutex.lock();
+        pool.mutex.lockUncancelable(io);
-        defer pool.mutex.unlock();
+        defer pool.mutex.unlock(io);
         return pool.acquireUnsafe(connection);
     }
 @@ -122,8 +125,8 @@ pub const ConnectionPool = struct {
     ///
     /// Threadsafe.
     pub fn release(pool: *ConnectionPool, connection: *Connection, io: Io) void {
-        pool.mutex.lock();
+        pool.mutex.lockUncancelable(io);
-        defer pool.mutex.unlock();
+        defer pool.mutex.unlock(io);
         pool.used.remove(&connection.pool_node);
 @@ -147,9 +150,9 @@ pub const ConnectionPool = struct {
     }
     /// Adds a newly created node to the pool of used connections. This function is threadsafe.
-    pub fn addUsed(pool: *ConnectionPool, connection: *Connection) void {
+    pub fn addUsed(pool: *ConnectionPool, io: Io, connection: *Connection) void {
-        pool.mutex.lock();
+        pool.mutex.lockUncancelable(io);
-        defer pool.mutex.unlock();
+        defer pool.mutex.unlock(io);
         pool.used.append(&connection.pool_node);
     }
 @@ -159,9 +162,9 @@ pub const ConnectionPool = struct {
     /// If the new size is smaller than the current size, then idle connections will be closed until the pool is the new size.
     ///
     /// Threadsafe.
-    pub fn resize(pool: *ConnectionPool, allocator: Allocator, new_size: usize) void {
+    pub fn resize(pool: *ConnectionPool, io: Io, allocator: Allocator, new_size: usize) void {
-        pool.mutex.lock();
+        pool.mutex.lockUncancelable(io);
-        defer pool.mutex.unlock();
+        defer pool.mutex.unlock(io);
         const next = pool.free.first;
         _ = next;
 @@ -182,7 +185,7 @@ pub const ConnectionPool = struct {
     ///
     /// Threadsafe.
     pub fn deinit(pool: *ConnectionPool, io: Io) void {
-        pool.mutex.lock();
+        pool.mutex.lockUncancelable(io);
         var next = pool.free.first;
         while (next) |node| {
 @@ -1308,9 +1311,11 @@ pub fn deinit(client: *Client) void {
 /// Uses `arena` for a few small allocations that must outlive the client, or
 /// at least until those fields are set to different values.
 pub fn initDefaultProxies(client: *Client, arena: Allocator, environ_map: *std.process.Environ.Map) !void {
+    const io = client.io;
     // Prevent any new connections from being created.
-    client.connection_pool.mutex.lock();
+    client.connection_pool.mutex.lockUncancelable(io);
-    defer client.connection_pool.mutex.unlock();
+    defer client.connection_pool.mutex.unlock(io);
     assert(client.connection_pool.used.first == null); // There are active requests.
 @@ -1437,7 +1442,7 @@ pub fn connectTcpOptions(client: *Client, options: ConnectTcpOptions) ConnectTcp
     const proxied_host = options.proxied_host orelse host;
     const proxied_port = options.proxied_port orelse port;
-    if (client.connection_pool.findConnection(.{
+    if (client.connection_pool.findConnection(io, .{
         .host = proxied_host,
         .port = proxied_port,
         .protocol = protocol,
 @@ -1455,12 +1460,12 @@ pub fn connectTcpOptions(client: *Client, options: ConnectTcpOptions) ConnectTcp
                 error.Canceled => |e| return e,
                 else => return error.TlsInitializationFailed,
             };
-            client.connection_pool.addUsed(&tc.connection);
+            client.connection_pool.addUsed(io, &tc.connection);
             return &tc.connection;
         },
         .plain => {
             const pc = try Connection.Plain.create(client, proxied_host, proxied_port, stream);
-            client.connection_pool.addUsed(&pc.connection);
+            client.connection_pool.addUsed(io, &pc.connection);
             return &pc.connection;
         },
     }
 @@ -1474,7 +1479,7 @@ pub const ConnectUnixError = Allocator.Error || std.posix.SocketError || error{N
 pub fn connectUnix(client: *Client, path: []const u8) ConnectUnixError!*Connection {
     const io = client.io;
-    if (client.connection_pool.findConnection(.{
+    if (client.connection_pool.findConnection(io, .{
         .host = path,
         .port = 0,
         .protocol = .plain,
 @@ -1516,7 +1521,7 @@ pub fn connectProxied(
     const io = client.io;
     if (!proxy.supports_connect) return error.TunnelNotSupported;
-    if (client.connection_pool.findConnection(.{
+    if (client.connection_pool.findConnection(io, .{
         .host = proxied_host,
         .port = proxied_port,
         .protocol = proxy.protocol,
 @@ -1691,8 +1696,8 @@ pub fn request(
     if (protocol == .tls) {
         if (disable_tls) unreachable;
         {
-            client.ca_bundle_mutex.lock();
+            client.ca_bundle_mutex.lockUncancelable(io);
-            defer client.ca_bundle_mutex.unlock();
+            defer client.ca_bundle_mutex.unlock(io);
             if (client.now == null) {
                 const now = try Io.Clock.real.now(io);

lib/std/once.zig deleted-71

...	@@ -1,71 +0,0 @@
1	const std = @import("std.zig");
2	const builtin = @import("builtin");
3	const testing = std.testing;
4
5	pub fn once(comptime f: fn () void) Once(f) {
6	return Once(f){};
7	}
8
9	/// An object that executes the function `f` just once.
10	/// It is undefined behavior if `f` re-enters the same Once instance.
11	pub fn Once(comptime f: fn () void) type {
12	return struct {
13	done: bool = false,
14	mutex: std.Thread.Mutex = std.Thread.Mutex{},
15
16	/// Call the function `f`.
17	/// If `call` is invoked multiple times `f` will be executed only the
18	/// first time.
19	/// The invocations are thread-safe.
20	pub fn call(self: *@This()) void {
21	if (@atomicLoad(bool, &self.done, .acquire))
22	return;
23
24	return self.callSlow();
25	}
26
27	fn callSlow(self: *@This()) void {
28	@branchHint(.cold);
29
30	self.mutex.lock();
31	defer self.mutex.unlock();
32
33	// The first thread to acquire the mutex gets to run the initializer
34	if (!self.done) {
35	f();
36	@atomicStore(bool, &self.done, true, .release);
37	}
38	}
39	};
40	}
41
42	var global_number: i32 = 0;
43	var global_once = once(incr);
44
45	fn incr() void {
46	global_number += 1;
47	}
48
49	test "Once executes its function just once" {
50	if (builtin.single_threaded) {
51	global_once.call();
52	global_once.call();
53	} else {
54	var threads: [10]std.Thread = undefined;
55	var thread_count: usize = 0;
56	defer for (threads[0..thread_count]) \|handle\| handle.join();
57
58	for (&threads) \|*handle\| {
59	handle.* = try std.Thread.spawn(.{}, struct {
60	fn thread_fn(x: u8) void {
61	_ = x;
62	global_once.call();
63	if (global_number != 1) @panic("memory ordering bug");
64	}
65	}.thread_fn, .{0});
66	thread_count += 1;
67	}
68	}
69
70	try testing.expectEqual(@as(i32, 1), global_number);
71	}

lib/std/std.zig-1

...	@@ -86,7 +86,6 @@ pub const math = @import("math.zig");	...	@@ -86,7 +86,6 @@ pub const math = @import("math.zig");
86	pub const mem = @import("mem.zig");	86	pub const mem = @import("mem.zig");
87	pub const meta = @import("meta.zig");	87	pub const meta = @import("meta.zig");
88	pub const os = @import("os.zig");	88	pub const os = @import("os.zig");
89	pub const once = @import("once.zig").once;
90	pub const pdb = @import("pdb.zig");	89	pub const pdb = @import("pdb.zig");
91	pub const pie = @import("pie.zig");	90	pub const pie = @import("pie.zig");
92	pub const posix = @import("posix.zig");	91	pub const posix = @import("posix.zig");

author	Andrew Kelley <andrew@ziglang.org> 2026-02-01 17:33:49-08:00
committer	Andrew Kelley <andrew@ziglang.org> 2026-02-02 18:57:17-08:00
log	550da1b676d059ae39a629d60da0f9cd155a5e89
tree	187210e7355cb0ad754cd513cc0ccb69a9aaad7c
parent	255aeb57b24bc24b604744460a61ebf7c44e42ea

...	@@ -696,7 +696,7 @@ pub noinline fn writeCurrentStackTrace(options: StackUnwindOptions, t: Io.Termin	...	@@ -696,7 +696,7 @@ pub noinline fn writeCurrentStackTrace(options: StackUnwindOptions, t: Io.Termin
696	.useless, .unsafe => {},	696	.useless, .unsafe => {},
697	.safe, .ideal => continue, // no need to even warn	697	.safe, .ideal => continue, // no need to even warn
698	}	698	}
699	const module_name = di.getModuleName(di_gpa, unwind_error.address) catch "???";	699	const module_name = di.getModuleName(di_gpa, io, unwind_error.address) catch "???";
700	const caption: []const u8 = switch (unwind_error.err) {	700	const caption: []const u8 = switch (unwind_error.err) {
701	error.MissingDebugInfo => "unwind info unavailable",	701	error.MissingDebugInfo => "unwind info unavailable",
702	error.InvalidDebugInfo => "unwind info invalid",	702	error.InvalidDebugInfo => "unwind info invalid",
...	@@ -1141,7 +1141,7 @@ fn printSourceAtAddress(	...	@@ -1141,7 +1141,7 @@ fn printSourceAtAddress(
1141	symbol.source_location,	1141	symbol.source_location,
1142	address,	1142	address,
1143	symbol.name orelse "???",	1143	symbol.name orelse "???",
1144	symbol.compile_unit_name orelse debug_info.getModuleName(gpa, address) catch "???",	1144	symbol.compile_unit_name orelse debug_info.getModuleName(gpa, io, address) catch "???",
1145	);	1145	);
1146	}	1146	}
1147	fn printLineInfo(	1147	fn printLineInfo(
...	@@ -1356,7 +1356,10 @@ pub fn getDebugInfoAllocator() Allocator {	...	@@ -1356,7 +1356,10 @@ pub fn getDebugInfoAllocator() Allocator {
1356	// Otherwise, use a global arena backed by the page allocator	1356	// Otherwise, use a global arena backed by the page allocator
1357	const S = struct {	1357	const S = struct {
1358	var arena: std.heap.ArenaAllocator = .init(std.heap.page_allocator);	1358	var arena: std.heap.ArenaAllocator = .init(std.heap.page_allocator);
1359	var ts_arena: std.heap.ThreadSafeAllocator = .{ .child_allocator = arena.allocator() };	1359	var ts_arena: std.heap.ThreadSafeAllocator = .{
		1360	.child_allocator = arena.allocator(),
		1361	.io = std.Options.debug_io,
		1362	};
1360	};	1363	};
1361	return S.ts_arena.allocator();	1364	return S.ts_arena.allocator();
1362	}	1365	}

std: migrate remaining sync primitives to Io