authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-02 17:20:03-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-02 18:55:31-08:00
log2bc89a5198eeb8b598c76ae18b1813616a330b5b
treea45032afa515203315d18cdcec37b05fa1142b4a
parente9eadee00654f5f762abe3cdc596359b79893eab

std.Io.Threaded: make parking_futex lock-free

And therefore no longer depend on a mutex API. The idea here is to rely on cache line operations being very fast. Buckets span exactly one cache line each, storing only waiter pointers. Additions and removals do linear weak cmpxchg scan over the cache line, repeating until sucess. If there are more parked threads with bucket hash collisions than fits into a cache line then wait() degrades into a spin lock.

1 files changed, 36 insertions(+), 58 deletions(-)

lib/std/Io/Threaded.zig+36-58
......@@ -16873,29 +16873,37 @@ const parking_futex = struct {
1687316873 }
1687416874
1687516875 const Bucket = struct {
16876 /// Used as a fast check for `wake` to avoid having to acquire `mutex` to discover there are no
16877 /// waiters. It is important for `wait` to increment this *before* checking the futex value to
16878 /// avoid a race.
16879 num_waiters: std.atomic.Value(u32),
16880 /// Protects `waiters`.
16881 mutex: Mutex,
16882 waiters: std.DoublyLinkedList,
16883
16884 /// Prevent false sharing between buckets.
16885 _: void align(std.atomic.cache_line) = {},
16886
16887 const init: Bucket = .{ .num_waiters = .init(0), .mutex = .init, .waiters = .{} };
16876 /// The alignment prevents false sharing between buckets.
16877 waiters: [capacity]?*Waiter align(std.atomic.cache_line) = @splat(null),
16878 const capacity = std.atomic.cache_line / @sizeOf(?*Waiter);
16879
16880 /// Store the waiter into the bucket, atomic, lock-free.
16881 fn add(b: *Bucket, w: *Waiter) void {
16882 while (true) for (&b.waiters) |*slot| {
16883 if (@cmpxchgWeak(?*Waiter, slot, null, w, .acq_rel, .monotonic) == null) {
16884 return;
16885 }
16886 };
16887 }
16888
16889 /// Delete the waiter from the bucket, atomic, lock-free.
16890 fn remove(b: *Bucket, w: *Waiter) void {
16891 while (true) for (&b.waiters) |*slot| {
16892 if (@cmpxchgWeak(?*Waiter, slot, w, null, .acq_rel, .monotonic) == null) {
16893 return;
16894 }
16895 };
16896 }
1688816897 };
1688916898
1689016899 const Waiter = struct {
16891 node: std.DoublyLinkedList.Node,
16900 node: std.SinglyLinkedList.Node,
1689216901 address: usize,
1689316902 tid: std.Thread.Id,
1689416903 /// `thread_status.cancelation` is `.parked` while the thread is waiting. The single thread
1689516904 /// which atomically updates it (to `.none` or `.canceling`) is responsible for:
1689616905 ///
1689716906 /// * Removing the `Waiter` from `Bucket.waiters`
16898 /// * Decrementing `Bucket.num_waiters`
1689916907 /// * Atomically setting `done` (after this, the `Waiter` may go out of scope at any time,
1690016908 /// so must not be referenced again)
1690116909 /// * Unparking the thread (last, so that the unparked thread definitely sees `done`)
......@@ -16911,7 +16919,7 @@ const parking_futex = struct {
1691116919 /// between different futexes. This length seems like it'll provide a reasonable balance
1691216920 /// between contention and memory usage: assuming a 128-byte `Bucket` (due to cache line
1691316921 /// alignment), this uses 32 KiB of memory.
16914 var buckets: [256]Bucket = @splat(.init);
16922 var buckets: [256]Bucket = @splat(.{});
1691516923 };
1691616924
1691716925 // Here we use Fibonacci hashing: the golden ratio can be used to evenly redistribute input
......@@ -16947,16 +16955,7 @@ const parking_futex = struct {
1694716955 var status_buf: std.atomic.Value(Thread.Status) = undefined;
1694816956
1694916957 {
16950 mutexLockUncancelable(&bucket.mutex);
16951 defer mutexUnlock(&bucket.mutex);
16952
16953 _ = bucket.num_waiters.fetchAdd(1, .acquire);
16954
16955 if (@atomicLoad(u32, ptr, .monotonic) != expect) {
16956 assert(bucket.num_waiters.fetchSub(1, .monotonic) > 0);
16957 return;
16958 }
16959
16958 if (@atomicLoad(u32, ptr, .monotonic) != expect) return;
1696016959 // This is in the critical section to avoid marking the thread as parked until we're
1696116960 // certain that we're actually going to park.
1696216961 waiter.thread_status = status: {
......@@ -16974,11 +16973,7 @@ const parking_futex = struct {
1697416973 );
1697516974 switch (old_status.cancelation) {
1697616975 .none => {}, // status is now `.parked`
16977 .canceling => {
16978 // status is now `.canceled`
16979 assert(bucket.num_waiters.fetchSub(1, .monotonic) > 0);
16980 return error.Canceled;
16981 },
16976 .canceling => return error.Canceled, // status is now `.canceled`
1698216977 .canceled => break :cancelable, // status is still `.canceled`
1698316978 .parked => unreachable,
1698416979 .blocked => unreachable,
......@@ -16996,7 +16991,7 @@ const parking_futex = struct {
1699616991 break :status &status_buf;
1699716992 };
1699816993
16999 bucket.waiters.append(&waiter.node);
16994 bucket.add(&waiter);
1700016995 }
1700116996
1700216997 const deadline: ?Io.Clock.Timestamp = switch (timeout) {
......@@ -17017,10 +17012,7 @@ const parking_futex = struct {
1701717012 .parked => {
1701817013 // We saw a timeout and updated our own status from `.parked` to `.none`. It is
1701917014 // our responsibility to remove `waiter` from `bucket`.
17020 mutexLockUncancelable(&bucket.mutex);
17021 defer mutexUnlock(&bucket.mutex);
17022 bucket.waiters.remove(&waiter.node);
17023 assert(bucket.num_waiters.fetchSub(1, .monotonic) > 0);
17015 bucket.remove(&waiter);
1702417016 },
1702517017 .none, .canceling => {
1702617018 // Race condition: the timeout was reached, then `wake` or a cancelation tried
......@@ -17046,25 +17038,16 @@ const parking_futex = struct {
1704617038
1704717039 const bucket = bucketForAddress(@intFromPtr(ptr));
1704817040
17049 // To ensure the store to `ptr` is ordered before this check, we effectively want a `.release`
17050 // load, but that doesn't exist in the C11 memory model, so emulate it with a non-mutating rmw.
17051 if (bucket.num_waiters.fetchAdd(0, .release) == 0) {
17052 @branchHint(.likely);
17053 return; // no waiters
17054 }
17055
1705617041 // Waiters removed from the linked list under the mutex so we can unpark their threads outside
1705717042 // of the critical section. This forms a singly-linked list of waiters using `Waiter.node.next`.
17058 var waking_head: ?*std.DoublyLinkedList.Node = null;
17043 var waking_head: ?*std.SinglyLinkedList.Node = null;
1705917044 {
17060 mutexLockUncancelable(&bucket.mutex);
17061 defer mutexUnlock(&bucket.mutex);
17062
1706317045 var num_removed: u32 = 0;
17064 var it = bucket.waiters.first;
17065 while (num_removed < max_waiters) {
17066 const waiter: *Waiter = @fieldParentPtr("node", it orelse break);
17067 it = waiter.node.next;
17046 var i: usize = 0;
17047 while (num_removed < max_waiters) : (i += 1) {
17048 const waiter: *Waiter = while (bucket.waiters.len - i != 0) : (i += 1) {
17049 break @atomicLoad(?*Waiter, &bucket.waiters[i], .monotonic) orelse continue;
17050 } else break;
1706817051 if (waiter.address != @intFromPtr(ptr)) continue;
1706917052 const old_status = waiter.thread_status.fetchAnd(
1707017053 .{ .cancelation = @enumFromInt(0b110), .awaitable = .all_ones },
......@@ -17072,7 +17055,7 @@ const parking_futex = struct {
1707217055 );
1707317056 switch (old_status.cancelation) {
1707417057 .parked => {}, // state updated to `.none`
17075 .none => continue, // race with timeout; they are about to lock `bucket.mutex` and remove themselves from the bucket
17058 .none => continue, // race with timeout; they are about to remove themselves from the bucket
1707617059 .canceling => continue, // race with a canceler who hasn't called `removeCanceledWaiter` yet
1707717060 .canceled => unreachable,
1707817061 .blocked => unreachable,
......@@ -17081,13 +17064,11 @@ const parking_futex = struct {
1708117064 .blocked_canceling => unreachable,
1708217065 }
1708317066 // We're waking this waiter. Remove them from the bucket and add them to our local list.
17084 bucket.waiters.remove(&waiter.node);
17067 @atomicStore(?*Waiter, &bucket.waiters[i], null, .release);
1708517068 waiter.node.next = waking_head;
1708617069 waking_head = &waiter.node;
1708717070 num_removed += 1;
1708817071 }
17089
17090 _ = bucket.num_waiters.fetchSub(num_removed, .monotonic);
1709117072 }
1709217073
1709317074 var unpark_buf: [128]UnparkTid = undefined;
......@@ -17113,10 +17094,7 @@ const parking_futex = struct {
1711317094
1711417095 fn removeCanceledWaiter(waiter: *Waiter) void {
1711517096 const bucket = bucketForAddress(waiter.address);
17116 mutexLockUncancelable(&bucket.mutex);
17117 defer mutexUnlock(&bucket.mutex);
17118 bucket.waiters.remove(&waiter.node);
17119 assert(bucket.num_waiters.fetchSub(1, .monotonic) > 0);
17097 bucket.remove(waiter);
1712017098 waiter.done.store(true, .release); // potentially invalidates `waiter.*`
1712117099 }
1712217100};