authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-02 17:20:03-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-02 18:55:31-08:00
log2bc89a5198eeb8b598c76ae18b1813616a330b5b
treea45032afa515203315d18cdcec37b05fa1142b4a
parente9eadee00654f5f762abe3cdc596359b79893eab

std.Io.Threaded: make parking_futex lock-free

And therefore no longer depend on a mutex API. The idea here is to rely on cache line operations being very fast. Buckets span exactly one cache line each, storing only waiter pointers. Additions and removals do linear weak cmpxchg scan over the cache line, repeating until sucess. If there are more parked threads with bucket hash collisions than fits into a cache line then wait() degrades into a spin lock.

1 files changed, 36 insertions(+), 58 deletions(-)

lib/std/Io/Threaded.zig+36-58
...@@ -16873,29 +16873,37 @@ const parking_futex = struct {...@@ -16873,29 +16873,37 @@ const parking_futex = struct {
16873 }16873 }
1687416874
16875 const Bucket = struct {16875 const Bucket = struct {
16876 /// Used as a fast check for `wake` to avoid having to acquire `mutex` to discover there are no16876 /// The alignment prevents false sharing between buckets.
16877 /// waiters. It is important for `wait` to increment this *before* checking the futex value to16877 waiters: [capacity]?*Waiter align(std.atomic.cache_line) = @splat(null),
16878 /// avoid a race.16878 const capacity = std.atomic.cache_line / @sizeOf(?*Waiter);
16879 num_waiters: std.atomic.Value(u32),16879
16880 /// Protects `waiters`.16880 /// Store the waiter into the bucket, atomic, lock-free.
16881 mutex: Mutex,16881 fn add(b: *Bucket, w: *Waiter) void {
16882 waiters: std.DoublyLinkedList,16882 while (true) for (&b.waiters) |*slot| {
1688316883 if (@cmpxchgWeak(?*Waiter, slot, null, w, .acq_rel, .monotonic) == null) {
16884 /// Prevent false sharing between buckets.16884 return;
16885 _: void align(std.atomic.cache_line) = {},16885 }
1688616886 };
16887 const init: Bucket = .{ .num_waiters = .init(0), .mutex = .init, .waiters = .{} };16887 }
16888
16889 /// Delete the waiter from the bucket, atomic, lock-free.
16890 fn remove(b: *Bucket, w: *Waiter) void {
16891 while (true) for (&b.waiters) |*slot| {
16892 if (@cmpxchgWeak(?*Waiter, slot, w, null, .acq_rel, .monotonic) == null) {
16893 return;
16894 }
16895 };
16896 }
16888 };16897 };
1688916898
16890 const Waiter = struct {16899 const Waiter = struct {
16891 node: std.DoublyLinkedList.Node,16900 node: std.SinglyLinkedList.Node,
16892 address: usize,16901 address: usize,
16893 tid: std.Thread.Id,16902 tid: std.Thread.Id,
16894 /// `thread_status.cancelation` is `.parked` while the thread is waiting. The single thread16903 /// `thread_status.cancelation` is `.parked` while the thread is waiting. The single thread
16895 /// which atomically updates it (to `.none` or `.canceling`) is responsible for:16904 /// which atomically updates it (to `.none` or `.canceling`) is responsible for:
16896 ///16905 ///
16897 /// * Removing the `Waiter` from `Bucket.waiters`16906 /// * Removing the `Waiter` from `Bucket.waiters`
16898 /// * Decrementing `Bucket.num_waiters`
16899 /// * Atomically setting `done` (after this, the `Waiter` may go out of scope at any time,16907 /// * Atomically setting `done` (after this, the `Waiter` may go out of scope at any time,
16900 /// so must not be referenced again)16908 /// so must not be referenced again)
16901 /// * Unparking the thread (last, so that the unparked thread definitely sees `done`)16909 /// * Unparking the thread (last, so that the unparked thread definitely sees `done`)
...@@ -16911,7 +16919,7 @@ const parking_futex = struct {...@@ -16911,7 +16919,7 @@ const parking_futex = struct {
16911 /// between different futexes. This length seems like it'll provide a reasonable balance16919 /// between different futexes. This length seems like it'll provide a reasonable balance
16912 /// between contention and memory usage: assuming a 128-byte `Bucket` (due to cache line16920 /// between contention and memory usage: assuming a 128-byte `Bucket` (due to cache line
16913 /// alignment), this uses 32 KiB of memory.16921 /// alignment), this uses 32 KiB of memory.
16914 var buckets: [256]Bucket = @splat(.init);16922 var buckets: [256]Bucket = @splat(.{});
16915 };16923 };
1691616924
16917 // Here we use Fibonacci hashing: the golden ratio can be used to evenly redistribute input16925 // Here we use Fibonacci hashing: the golden ratio can be used to evenly redistribute input
...@@ -16947,16 +16955,7 @@ const parking_futex = struct {...@@ -16947,16 +16955,7 @@ const parking_futex = struct {
16947 var status_buf: std.atomic.Value(Thread.Status) = undefined;16955 var status_buf: std.atomic.Value(Thread.Status) = undefined;
1694816956
16949 {16957 {
16950 mutexLockUncancelable(&bucket.mutex);16958 if (@atomicLoad(u32, ptr, .monotonic) != expect) return;
16951 defer mutexUnlock(&bucket.mutex);
16952
16953 _ = bucket.num_waiters.fetchAdd(1, .acquire);
16954
16955 if (@atomicLoad(u32, ptr, .monotonic) != expect) {
16956 assert(bucket.num_waiters.fetchSub(1, .monotonic) > 0);
16957 return;
16958 }
16959
16960 // This is in the critical section to avoid marking the thread as parked until we're16959 // This is in the critical section to avoid marking the thread as parked until we're
16961 // certain that we're actually going to park.16960 // certain that we're actually going to park.
16962 waiter.thread_status = status: {16961 waiter.thread_status = status: {
...@@ -16974,11 +16973,7 @@ const parking_futex = struct {...@@ -16974,11 +16973,7 @@ const parking_futex = struct {
16974 );16973 );
16975 switch (old_status.cancelation) {16974 switch (old_status.cancelation) {
16976 .none => {}, // status is now `.parked`16975 .none => {}, // status is now `.parked`
16977 .canceling => {16976 .canceling => return error.Canceled, // status is now `.canceled`
16978 // status is now `.canceled`
16979 assert(bucket.num_waiters.fetchSub(1, .monotonic) > 0);
16980 return error.Canceled;
16981 },
16982 .canceled => break :cancelable, // status is still `.canceled`16977 .canceled => break :cancelable, // status is still `.canceled`
16983 .parked => unreachable,16978 .parked => unreachable,
16984 .blocked => unreachable,16979 .blocked => unreachable,
...@@ -16996,7 +16991,7 @@ const parking_futex = struct {...@@ -16996,7 +16991,7 @@ const parking_futex = struct {
16996 break :status &status_buf;16991 break :status &status_buf;
16997 };16992 };
1699816993
16999 bucket.waiters.append(&waiter.node);16994 bucket.add(&waiter);
17000 }16995 }
1700116996
17002 const deadline: ?Io.Clock.Timestamp = switch (timeout) {16997 const deadline: ?Io.Clock.Timestamp = switch (timeout) {
...@@ -17017,10 +17012,7 @@ const parking_futex = struct {...@@ -17017,10 +17012,7 @@ const parking_futex = struct {
17017 .parked => {17012 .parked => {
17018 // We saw a timeout and updated our own status from `.parked` to `.none`. It is17013 // We saw a timeout and updated our own status from `.parked` to `.none`. It is
17019 // our responsibility to remove `waiter` from `bucket`.17014 // our responsibility to remove `waiter` from `bucket`.
17020 mutexLockUncancelable(&bucket.mutex);17015 bucket.remove(&waiter);
17021 defer mutexUnlock(&bucket.mutex);
17022 bucket.waiters.remove(&waiter.node);
17023 assert(bucket.num_waiters.fetchSub(1, .monotonic) > 0);
17024 },17016 },
17025 .none, .canceling => {17017 .none, .canceling => {
17026 // Race condition: the timeout was reached, then `wake` or a cancelation tried17018 // Race condition: the timeout was reached, then `wake` or a cancelation tried
...@@ -17046,25 +17038,16 @@ const parking_futex = struct {...@@ -17046,25 +17038,16 @@ const parking_futex = struct {
1704617038
17047 const bucket = bucketForAddress(@intFromPtr(ptr));17039 const bucket = bucketForAddress(@intFromPtr(ptr));
1704817040
17049 // To ensure the store to `ptr` is ordered before this check, we effectively want a `.release`
17050 // load, but that doesn't exist in the C11 memory model, so emulate it with a non-mutating rmw.
17051 if (bucket.num_waiters.fetchAdd(0, .release) == 0) {
17052 @branchHint(.likely);
17053 return; // no waiters
17054 }
17055
17056 // Waiters removed from the linked list under the mutex so we can unpark their threads outside17041 // Waiters removed from the linked list under the mutex so we can unpark their threads outside
17057 // of the critical section. This forms a singly-linked list of waiters using `Waiter.node.next`.17042 // of the critical section. This forms a singly-linked list of waiters using `Waiter.node.next`.
17058 var waking_head: ?*std.DoublyLinkedList.Node = null;17043 var waking_head: ?*std.SinglyLinkedList.Node = null;
17059 {17044 {
17060 mutexLockUncancelable(&bucket.mutex);
17061 defer mutexUnlock(&bucket.mutex);
17062
17063 var num_removed: u32 = 0;17045 var num_removed: u32 = 0;
17064 var it = bucket.waiters.first;17046 var i: usize = 0;
17065 while (num_removed < max_waiters) {17047 while (num_removed < max_waiters) : (i += 1) {
17066 const waiter: *Waiter = @fieldParentPtr("node", it orelse break);17048 const waiter: *Waiter = while (bucket.waiters.len - i != 0) : (i += 1) {
17067 it = waiter.node.next;17049 break @atomicLoad(?*Waiter, &bucket.waiters[i], .monotonic) orelse continue;
17050 } else break;
17068 if (waiter.address != @intFromPtr(ptr)) continue;17051 if (waiter.address != @intFromPtr(ptr)) continue;
17069 const old_status = waiter.thread_status.fetchAnd(17052 const old_status = waiter.thread_status.fetchAnd(
17070 .{ .cancelation = @enumFromInt(0b110), .awaitable = .all_ones },17053 .{ .cancelation = @enumFromInt(0b110), .awaitable = .all_ones },
...@@ -17072,7 +17055,7 @@ const parking_futex = struct {...@@ -17072,7 +17055,7 @@ const parking_futex = struct {
17072 );17055 );
17073 switch (old_status.cancelation) {17056 switch (old_status.cancelation) {
17074 .parked => {}, // state updated to `.none`17057 .parked => {}, // state updated to `.none`
17075 .none => continue, // race with timeout; they are about to lock `bucket.mutex` and remove themselves from the bucket17058 .none => continue, // race with timeout; they are about to remove themselves from the bucket
17076 .canceling => continue, // race with a canceler who hasn't called `removeCanceledWaiter` yet17059 .canceling => continue, // race with a canceler who hasn't called `removeCanceledWaiter` yet
17077 .canceled => unreachable,17060 .canceled => unreachable,
17078 .blocked => unreachable,17061 .blocked => unreachable,
...@@ -17081,13 +17064,11 @@ const parking_futex = struct {...@@ -17081,13 +17064,11 @@ const parking_futex = struct {
17081 .blocked_canceling => unreachable,17064 .blocked_canceling => unreachable,
17082 }17065 }
17083 // We're waking this waiter. Remove them from the bucket and add them to our local list.17066 // We're waking this waiter. Remove them from the bucket and add them to our local list.
17084 bucket.waiters.remove(&waiter.node);17067 @atomicStore(?*Waiter, &bucket.waiters[i], null, .release);
17085 waiter.node.next = waking_head;17068 waiter.node.next = waking_head;
17086 waking_head = &waiter.node;17069 waking_head = &waiter.node;
17087 num_removed += 1;17070 num_removed += 1;
17088 }17071 }
17089
17090 _ = bucket.num_waiters.fetchSub(num_removed, .monotonic);
17091 }17072 }
1709217073
17093 var unpark_buf: [128]UnparkTid = undefined;17074 var unpark_buf: [128]UnparkTid = undefined;
...@@ -17113,10 +17094,7 @@ const parking_futex = struct {...@@ -17113,10 +17094,7 @@ const parking_futex = struct {
1711317094
17114 fn removeCanceledWaiter(waiter: *Waiter) void {17095 fn removeCanceledWaiter(waiter: *Waiter) void {
17115 const bucket = bucketForAddress(waiter.address);17096 const bucket = bucketForAddress(waiter.address);
17116 mutexLockUncancelable(&bucket.mutex);17097 bucket.remove(waiter);
17117 defer mutexUnlock(&bucket.mutex);
17118 bucket.waiters.remove(&waiter.node);
17119 assert(bucket.num_waiters.fetchSub(1, .monotonic) > 0);
17120 waiter.done.store(true, .release); // potentially invalidates `waiter.*`17098 waiter.done.store(true, .release); // potentially invalidates `waiter.*`
17121 }17099 }
17122};17100};