authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-02-03 22:14:42+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-02-03 22:50:35+00:00
log6d6532dd9eb862dfd6e59ceed5c762342d2cc0d5
treea6fe1596bb142355c8b808ba56e20c37c8598258
parent7c08f77efa6052750118881b506f7b59918993e0
signaturelock-open Commit is signed but in an unrecognized format.

Io.Threaded: add ParkingMutex, and deal with spurious unparks on NetBSD

We can't use Io.Mutex in parking_futex; instead, we need a simple parking-based mutex implementation. That's fairly simple to do. Also deal with spurious unparks on NetBSD, where they *can* happen (as opposed to Windows, where they cannot).

1 files changed, 219 insertions(+), 65 deletions(-)

lib/std/Io/Threaded.zig+219-65
......@@ -2662,7 +2662,7 @@ fn batchAwaitConcurrent(userdata: ?*anyopaque, b: *Io.Batch, timeout: Io.Timeout
26622662 while (b.pending.head != .none and b.completions.head == .none) {
26632663 var delay_interval: windows.LARGE_INTEGER = interval: {
26642664 const d = deadline orelse break :interval std.math.minInt(windows.LARGE_INTEGER);
2665 break :interval t.deadlineToWindowsInterval(d);
2665 break :interval timeoutToWindowsInterval(.{ .deadline = d }).?;
26662666 };
26672667 const alertable_syscall = try AlertableSyscall.start();
26682668 const delay_rc = windows.ntdll.NtDelayExecution(windows.TRUE, &delay_interval);
......@@ -4339,7 +4339,10 @@ fn dirCreateFileWindows(
43394339 // kernel bug with retry attempts.
43404340 syscall.finish();
43414341 if (max_attempts - attempt == 0) return error.FileBusy;
4342 try parking_sleep.windowsRetrySleep((@as(u32, 1) << attempt) >> 1);
4342 try parking_sleep.sleep(.{ .duration = .{
4343 .raw = .fromMilliseconds((@as(u32, 1) << attempt) >> 1),
4344 .clock = .awake,
4345 } });
43434346 attempt += 1;
43444347 syscall = try .start();
43454348 continue;
......@@ -4352,7 +4355,10 @@ fn dirCreateFileWindows(
43524355 // fixed by sleeping and retrying until the error goes away.
43534356 syscall.finish();
43544357 if (max_attempts - attempt == 0) return error.FileBusy;
4355 try parking_sleep.windowsRetrySleep((@as(u32, 1) << attempt) >> 1);
4358 try parking_sleep.sleep(.{ .duration = .{
4359 .raw = .fromMilliseconds((@as(u32, 1) << attempt) >> 1),
4360 .clock = .awake,
4361 } });
43564362 attempt += 1;
43574363 syscall = try .start();
43584364 continue;
......@@ -7382,7 +7388,10 @@ fn dirReadLinkWindows(dir: Dir, sub_path: []const u8, buffer: []u8) Dir.ReadLink
73827388 // kernel bug with retry attempts.
73837389 syscall.finish();
73847390 if (max_attempts - attempt == 0) return error.FileBusy;
7385 try parking_sleep.windowsRetrySleep((@as(u32, 1) << attempt) >> 1);
7391 try parking_sleep.sleep(.{ .duration = .{
7392 .raw = .fromMilliseconds((@as(u32, 1) << attempt) >> 1),
7393 .clock = .awake,
7394 } });
73867395 attempt += 1;
73877396 syscall = try .start();
73887397 continue;
......@@ -7395,7 +7404,10 @@ fn dirReadLinkWindows(dir: Dir, sub_path: []const u8, buffer: []u8) Dir.ReadLink
73957404 // fixed by sleeping and retrying until the error goes away.
73967405 syscall.finish();
73977406 if (max_attempts - attempt == 0) return error.FileBusy;
7398 try parking_sleep.windowsRetrySleep((@as(u32, 1) << attempt) >> 1);
7407 try parking_sleep.sleep(.{ .duration = .{
7408 .raw = .fromMilliseconds((@as(u32, 1) << attempt) >> 1),
7409 .clock = .awake,
7410 } });
73997411 attempt += 1;
74007412 syscall = try .start();
74017413 continue;
......@@ -10956,7 +10968,7 @@ fn nowWasi(clock: Io.Clock) Io.Timestamp {
1095610968fn sleep(userdata: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void {
1095710969 const t: *Threaded = @ptrCast(@alignCast(userdata));
1095810970 if (timeout == .none) return;
10959 if (use_parking_sleep) return parking_sleep.sleep(timeout.toTimestamp(ioBasic(t)));
10971 if (use_parking_sleep) return parking_sleep.sleep(timeout);
1096010972 if (native_os == .wasi) return sleepWasi(t, timeout);
1096110973 if (@TypeOf(posix.system.clock_nanosleep) != void) return sleepPosix(timeout);
1096210974 return sleepNanosleep(t, timeout);
......@@ -14363,7 +14375,7 @@ const Wsa = struct {
1436314375
1436414376fn initializeWsa(t: *Threaded) error{ NetworkDown, Canceled }!void {
1436514377 const wsa = &t.wsa;
14366 try mutexLock(&wsa.mutex);
14378 mutexLock(&wsa.mutex);
1436714379 defer mutexUnlock(&wsa.mutex);
1436814380 switch (wsa.status) {
1436914381 .uninitialized => {
......@@ -16943,7 +16955,7 @@ const parking_futex = struct {
1694316955 /// avoid a race.
1694416956 num_waiters: std.atomic.Value(u32),
1694516957 /// Protects `waiters`.
16946 mutex: Io.Mutex,
16958 mutex: ParkingMutex,
1694716959 waiters: std.DoublyLinkedList,
1694816960
1694916961 /// Prevent false sharing between buckets.
......@@ -17007,8 +17019,8 @@ const parking_futex = struct {
1700717019 var status_buf: std.atomic.Value(Thread.Status) = undefined;
1700817020
1700917021 {
17010 mutexLock(&bucket.mutex);
17011 defer mutexUnlock(&bucket.mutex);
17022 bucket.mutex.lock();
17023 defer bucket.mutex.unlock();
1701217024
1701317025 _ = bucket.num_waiters.fetchAdd(1, .acquire);
1701417026
......@@ -17059,7 +17071,7 @@ const parking_futex = struct {
1705917071 bucket.waiters.append(&waiter.node);
1706017072 }
1706117073
17062 if (park(timeout, ptr)) {
17074 if (park(timeout, ptr, waiter.thread_status)) {
1706317075 // We were unparked by either `wake` or cancelation, so our current status is either
1706417076 // `.none` or `.canceling`. In either case, they've already removed `waiter` from
1706517077 // `bucket`, so we have nothing more to do!
......@@ -17084,7 +17096,7 @@ const parking_futex = struct {
1708417096 // to unpark us. Whoever did that will remove us from `bucket`. Wait for
1708517097 // that (and drop the unpark request in doing so).
1708617098 // New status is `.none` or `.canceling` respectively.
17087 park(.none, ptr) catch |e| switch (e) {
17099 park(.none, ptr, waiter.thread_status) catch |e| switch (e) {
1708817100 error.Timeout => unreachable,
1708917101 };
1709017102 },
......@@ -17114,8 +17126,8 @@ const parking_futex = struct {
1711417126 // of the critical section. This forms a singly-linked list of waiters using `Waiter.node.next`.
1711517127 var waking_head: ?*std.DoublyLinkedList.Node = null;
1711617128 {
17117 mutexLock(&bucket.mutex);
17118 defer mutexUnlock(&bucket.mutex);
17129 bucket.mutex.lock();
17130 defer bucket.mutex.unlock();
1711917131
1712017132 var num_removed: u32 = 0;
1712117133 var it = bucket.waiters.first;
......@@ -17171,8 +17183,8 @@ const parking_futex = struct {
1717117183
1717217184 fn removeCanceledWaiter(waiter: *Waiter) void {
1717317185 const bucket = bucketForAddress(waiter.address);
17174 mutexLock(&bucket.mutex);
17175 defer mutexUnlock(&bucket.mutex);
17186 bucket.mutex.lock();
17187 defer bucket.mutex.unlock();
1717617188 bucket.waiters.remove(&waiter.node);
1717717189 assert(bucket.num_waiters.fetchSub(1, .monotonic) > 0);
1717817190 }
......@@ -17206,7 +17218,7 @@ const parking_sleep = struct {
1720617218 .blocked_canceling => unreachable,
1720717219 }
1720817220 }
17209 if (park(timeout, null)) {
17221 if (park(timeout, null, &thread.status)) {
1721017222 // The only reason this could possibly happen is cancelation.
1721117223 const old_status = thread.status.load(.monotonic);
1721217224 assert(old_status.cancelation == .canceling);
......@@ -17229,7 +17241,7 @@ const parking_sleep = struct {
1722917241 // us for a cancelation. Whoever did that will have called `unpark`, so
1723017242 // drop that unpark request by waiting for it.
1723117243 // Status is still `.canceling`.
17232 park(.none, null) catch |e| switch (e) {
17244 park(.none, null, &thread.status) catch |e| switch (e) {
1723317245 error.Timeout => unreachable,
1723417246 };
1723517247 return;
......@@ -17245,32 +17257,183 @@ const parking_sleep = struct {
1724517257 }
1724617258 }
1724717259 // Uncancelable sleep; we expect not to be manually unparked.
17248 if (park(timeout, null)) {
17260 var dummy_status: std.atomic.Value(Thread.Status) = .init(.{ .cancelation = .parked, .awaitable = .null });
17261 if (park(timeout, null, &dummy_status)) {
1724917262 unreachable; // unexpected unpark
1725017263 } else |err| switch (err) {
1725117264 error.Timeout => return,
1725217265 }
1725317266 }
1725417267};
17268const ParkingMutex = struct {
17269 state: std.atomic.Value(State),
1725517270
17256/// `addr_hint` has no semantic effect, but may allow the OS to optimize this operation.
17257fn park(timeout: Io.Timeout, addr_hint: ?*const anyopaque) error{Timeout}!void {
17271 const init: ParkingMutex = .{ .state = .init(.unlocked) };
17272
17273 comptime {
17274 assert(use_parking_futex);
17275 }
17276
17277 const State = enum(usize) {
17278 unlocked = 1,
17279 /// This value is intentionally 0 so that `waiter` returns `null`.
17280 locked_once = 0,
17281 /// Contended; value is a `*Waiter`.
17282 _,
17283 /// Returns the head of the waiter list. Illegal to call if `s == .unlocked`.
17284 fn waiter(s: State) ?*Waiter {
17285 return @ptrFromInt(@intFromEnum(s));
17286 }
17287 /// Returns a locked state where `w` is contending the lock.
17288 /// If `w` is `null`, returns `.locked_once`.
17289 fn fromWaiter(w: ?*Waiter) State {
17290 return @enumFromInt(@intFromPtr(w));
17291 }
17292 };
17293 const Waiter = struct {
17294 status: std.atomic.Value(Thread.Status),
17295 /// Never modified once the `Waiter` is in the linked list.
17296 next: ?*Waiter,
17297 /// Never modified once the `Waiter` is in the linked list.
17298 tid: std.Thread.Id,
17299 };
17300 fn lock(m: *ParkingMutex) void {
17301 state: switch (State.unlocked) { // assume 'unlocked' to optimize for uncontended case
17302 .unlocked => continue :state m.state.cmpxchgWeak(
17303 .unlocked,
17304 .locked_once,
17305 .acquire, // acquire lock
17306 .monotonic,
17307 ) orelse {
17308 @branchHint(.likely);
17309 return;
17310 },
17311
17312 .locked_once, _ => |last_state| {
17313 const old_waiter = last_state.waiter();
17314 const self_tid = if (Thread.current) |t| t.id else std.Thread.getCurrentId();
17315 var waiter: Waiter = .{
17316 .next = old_waiter,
17317 .status = .init(.{ .cancelation = .parked, .awaitable = .null }),
17318 .tid = self_tid,
17319 };
17320 if (m.state.cmpxchgWeak(
17321 .fromWaiter(old_waiter),
17322 .fromWaiter(&waiter),
17323 .release, // release `waiter`
17324 .monotonic,
17325 )) |new_state| {
17326 continue :state new_state;
17327 }
17328 // We're now in the list of waiters---park until we're given the lock.
17329 park(.none, m, &waiter.status) catch |err| switch (err) {
17330 error.Timeout => unreachable,
17331 };
17332 // We now hold the lock.
17333 assert(waiter.status.load(.monotonic).cancelation == .none);
17334 return;
17335 },
17336 }
17337 }
17338 fn unlock(m: *ParkingMutex) void {
17339 state: switch (State.locked_once) { // assume 'locked_once' to optimize for uncontended case
17340 .unlocked => unreachable, // we hold the lock
17341
17342 .locked_once => continue :state m.state.cmpxchgWeak(
17343 .locked_once,
17344 .unlocked,
17345 .release, // release lock
17346 .acquire, // acquire any `Waiter` memory
17347 ) orelse {
17348 @branchHint(.likely);
17349 return;
17350 },
17351
17352 _ => |last_state| {
17353 // The logic here does not have ABA problems, and does some accesses non-atomically,
17354 // because `Waiter.next` is owned by the lock holder (that's us!) once the waiter is
17355 // in the linked list, up until we set `Waiter.status` to `.none`.
17356
17357 // Run through the waiter list to the end to ensure fairness. This is obviously not
17358 // ideal, but it shouldn't be a big deal in practice provided the critical section
17359 // is fairly small (so we won't get too many threads contending the mutex at once).
17360 // There's a *chance* we could get away with a LIFO queue for our use case, but I
17361 // don't wanna risk that.
17362 var parent: ?*Waiter = null;
17363 var waiter: *Waiter = last_state.waiter().?;
17364 while (waiter.next) |next| {
17365 parent = waiter;
17366 waiter = next;
17367 }
17368 // `waiter` is next in line for the lock. Remove them from the list.
17369 if (parent) |p| {
17370 assert(p.next == waiter);
17371 p.next = null;
17372 } else {
17373 // We're waking the last waiter, so clear the list head.
17374 if (m.state.cmpxchgWeak(
17375 .fromWaiter(last_state.waiter().?),
17376 .locked_once,
17377 .acquire,
17378 .acquire, // acquire any new `Waiter` memory
17379 )) |new_state| {
17380 continue :state new_state;
17381 }
17382 }
17383 // Now we're ready to actually hand the lock over to them.
17384 const tid = waiter.tid; // load this before the store below potentially invalidates `waiter`
17385 waiter.status.store(.{ .cancelation = .none, .awaitable = .null }, .release); // release lock
17386 unpark(&.{tid}, m);
17387 return;
17388 },
17389 }
17390 }
17391};
17392
17393fn timeoutToWindowsInterval(timeout: Io.Timeout) ?windows.LARGE_INTEGER {
17394 // ntdll only supports two combinations:
17395 // * real-time (`.real`) sleeps with absolute deadlines
17396 // * monotonic (`.awake`/`.boot`) sleeps with relative durations
17397 const clock = switch (timeout) {
17398 .none => return null,
17399 .duration => |d| d.clock,
17400 .deadline => |d| d.clock,
17401 };
17402 switch (clock) {
17403 .cpu_process, .cpu_thread => unreachable, // cannot sleep for CPU time
17404 .real => {
17405 const deadline = switch (timeout) {
17406 .none => unreachable,
17407 .duration => |d| nowWindows(clock).addDuration(d.raw),
17408 .deadline => |d| d.raw,
17409 };
17410 return @intCast(@max(@divTrunc(deadline.nanoseconds, 100), 0));
17411 },
17412 .awake, .boot => {
17413 const duration = switch (timeout) {
17414 .none => unreachable,
17415 .duration => |d| d.raw,
17416 .deadline => |d| nowWindows(clock).durationTo(d.raw),
17417 };
17418 return @intCast(@min(@divTrunc(-duration.nanoseconds, 100), -1));
17419 },
17420 }
17421}
17422
17423fn park(
17424 timeout: Io.Timeout,
17425 /// This value has no semantic effect, but may allow the OS to optimize the operation.
17426 addr_hint: ?*const anyopaque,
17427 /// The API on NetBSD and Illumos sucks and can unpark spuriously (well, it *can't*, but signals
17428 /// cause an indistinguishable unblock, and libpthread really likes to leave unparks pending).
17429 /// As such, on these targets only, this `status` is checked to determine if an unpark is real.
17430 /// no way to differentiate
17431 status: *std.atomic.Value(Thread.Status),
17432) error{Timeout}!void {
1725817433 comptime assert(use_parking_futex or use_parking_sleep);
1725917434 switch (native_os) {
1726017435 .windows => {
17261 var timeout_buf: windows.LARGE_INTEGER = undefined;
17262 const raw_timeout: ?*windows.LARGE_INTEGER = timeout: switch (timeout) {
17263 .none => null,
17264 .deadline => |timestamp| continue :timeout .{ .duration = .{
17265 .clock = timestamp.clock,
17266 .raw = (nowWindows(timestamp.clock) catch unreachable).durationTo(timestamp.raw),
17267 } },
17268 .duration => |duration| {
17269 _ = duration.clock; // Windows only supports monotonic
17270 timeout_buf = @intCast(@divTrunc(-duration.raw.nanoseconds, 100));
17271 break :timeout &timeout_buf;
17272 },
17273 };
17436 const raw_timeout = timeoutToWindowsInterval(timeout);
1727417437 // `RtlWaitOnAddress` passes the futex address in as the first argument to this call,
1727517438 // but it's unclear what that actually does, especially since `NtAlertThreadByThreadId`
1727617439 // does *not* accept the address so the kernel can't really be using it as a hint. An
......@@ -17284,7 +17447,10 @@ fn park(timeout: Io.Timeout, addr_hint: ?*const anyopaque) error{Timeout}!void {
1728417447 // this parameter). However, to err on the side of caution, let's match the behavior of
1728517448 // `RtlWaitOnAddress` and pass the pointer, in case the kernel ever does something
1728617449 // stupid such as trying to dereference it.
17287 switch (windows.ntdll.NtWaitForAlertByThreadId(addr_hint, raw_timeout)) {
17450 switch (windows.ntdll.NtWaitForAlertByThreadId(
17451 addr_hint,
17452 if (raw_timeout) |*t| t else null,
17453 )) {
1728817454 .ALERTED => return,
1728917455 .TIMEOUT => return error.Timeout,
1729017456 else => unreachable,
......@@ -17303,19 +17469,23 @@ fn park(timeout: Io.Timeout, addr_hint: ?*const anyopaque) error{Timeout}!void {
1730317469 break :timeout .{ &ts_buf, false, duration.clock == .real };
1730417470 },
1730517471 };
17306 switch (posix.errno(std.c._lwp_park(
17307 if (clock_real) .REALTIME else .MONOTONIC,
17308 .{ .ABSTIME = abstime },
17309 ts,
17310 0,
17311 addr_hint,
17312 null,
17313 ))) {
17314 .SUCCESS, .ALREADY, .INTR => return,
17315 .TIMEDOUT => return error.Timeout,
17316 .INVAL => unreachable,
17317 .SRCH => unreachable,
17318 else => unreachable,
17472 // It's okay to pass the same timeout in a loop. If it's a duration, the OS actually
17473 // writes the remaining time into the buffer when the syscall returns.
17474 while (status.load(.monotonic).cancelation == .parked) {
17475 switch (posix.errno(std.c._lwp_park(
17476 if (clock_real) .REALTIME else .MONOTONIC,
17477 .{ .ABSTIME = abstime },
17478 ts,
17479 0,
17480 addr_hint,
17481 null,
17482 ))) {
17483 .SUCCESS, .ALREADY, .INTR => {},
17484 .TIMEDOUT => return error.Timeout,
17485 .INVAL => unreachable,
17486 .SRCH => unreachable,
17487 else => unreachable,
17488 }
1731917489 }
1732017490 },
1732117491 .illumos => @panic("TODO: illumos lwp_park"),
......@@ -17323,24 +17493,8 @@ fn park(timeout: Io.Timeout, addr_hint: ?*const anyopaque) error{Timeout}!void {
1732317493 }
1732417494}
1732517495
17326fn deadlineToWindowsInterval(t: *Io.Threaded, deadline: Io.Clock.Timestamp) windows.LARGE_INTEGER {
17327 // ntdll only supports two combinations:
17328 // * real-time (`.real`) sleeps with absolute deadlines
17329 // * monotonic (`.awake`/`.boot`) sleeps with relative durations
17330 switch (deadline.clock) {
17331 .cpu_process, .cpu_thread => return 0,
17332 .real => {
17333 return @intCast(@max(@divTrunc(deadline.raw.nanoseconds, 100), 0));
17334 },
17335 .awake, .boot => {
17336 const duration = deadline.durationFromNow(ioBasic(t));
17337 return @intCast(@min(@divTrunc(-duration.raw.nanoseconds, 100), -1));
17338 },
17339 }
17340}
17341
1734217496const UnparkTid = switch (native_os) {
17343 // `NtAlertMultipleThreadByThreadId` is weird and wants 64-bit thread handles?
17497 // `NtAlertMultipleThreadByThreadId` is weird and wants 64-bit thread IDs?
1734417498 .windows => usize,
1734517499 else => std.Thread.Id,
1734617500};