authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-10-27 14:34:42-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-10-29 06:20:51-07:00
log4e95c2eb1b996d7f4dfeaad8112b62733e077273
tree4e26b1d9a1674c6e405647434d815b7b22ac474c
parenta87fd37bf59a975ba057bec408d2908d6168a084

std.Io.Threaded: implement futexes for freebsd

1 files changed, 192 insertions(+), 144 deletions(-)

lib/std/Io/Threaded.zig+192-144
...@@ -5047,7 +5047,7 @@ fn statFromLinux(stx: *const std.os.linux.Statx) Io.File.Stat {...@@ -5047,7 +5047,7 @@ fn statFromLinux(stx: *const std.os.linux.Statx) Io.File.Stat {
5047 };5047 };
5048}5048}
50495049
5050fn statFromPosix(st: *const std.posix.Stat) Io.File.Stat {5050fn statFromPosix(st: *const posix.Stat) Io.File.Stat {
5051 const atime = st.atime();5051 const atime = st.atime();
5052 const mtime = st.mtime();5052 const mtime = st.mtime();
5053 const ctime = st.ctime();5053 const ctime = st.ctime();
...@@ -5056,20 +5056,20 @@ fn statFromPosix(st: *const std.posix.Stat) Io.File.Stat {...@@ -5056,20 +5056,20 @@ fn statFromPosix(st: *const std.posix.Stat) Io.File.Stat {
5056 .size = @bitCast(st.size),5056 .size = @bitCast(st.size),
5057 .mode = st.mode,5057 .mode = st.mode,
5058 .kind = k: {5058 .kind = k: {
5059 const m = st.mode & std.posix.S.IFMT;5059 const m = st.mode & posix.S.IFMT;
5060 switch (m) {5060 switch (m) {
5061 std.posix.S.IFBLK => break :k .block_device,5061 posix.S.IFBLK => break :k .block_device,
5062 std.posix.S.IFCHR => break :k .character_device,5062 posix.S.IFCHR => break :k .character_device,
5063 std.posix.S.IFDIR => break :k .directory,5063 posix.S.IFDIR => break :k .directory,
5064 std.posix.S.IFIFO => break :k .named_pipe,5064 posix.S.IFIFO => break :k .named_pipe,
5065 std.posix.S.IFLNK => break :k .sym_link,5065 posix.S.IFLNK => break :k .sym_link,
5066 std.posix.S.IFREG => break :k .file,5066 posix.S.IFREG => break :k .file,
5067 std.posix.S.IFSOCK => break :k .unix_domain_socket,5067 posix.S.IFSOCK => break :k .unix_domain_socket,
5068 else => {},5068 else => {},
5069 }5069 }
5070 if (native_os == .illumos) switch (m) {5070 if (native_os == .illumos) switch (m) {
5071 std.posix.S.IFDOOR => break :k .door,5071 posix.S.IFDOOR => break :k .door,
5072 std.posix.S.IFPORT => break :k .event_port,5072 posix.S.IFPORT => break :k .event_port,
5073 else => {},5073 else => {},
5074 };5074 };
50755075
...@@ -5101,11 +5101,11 @@ fn statFromWasi(st: *const std.os.wasi.filestat_t) Io.File.Stat {...@@ -5101,11 +5101,11 @@ fn statFromWasi(st: *const std.os.wasi.filestat_t) Io.File.Stat {
5101 };5101 };
5102}5102}
51035103
5104fn timestampFromPosix(timespec: *const std.posix.timespec) Io.Timestamp {5104fn timestampFromPosix(timespec: *const posix.timespec) Io.Timestamp {
5105 return .{ .nanoseconds = @intCast(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec) };5105 return .{ .nanoseconds = @intCast(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec) };
5106}5106}
51075107
5108fn timestampToPosix(nanoseconds: i96) std.posix.timespec {5108fn timestampToPosix(nanoseconds: i96) posix.timespec {
5109 return .{5109 return .{
5110 .sec = @intCast(@divFloor(nanoseconds, std.time.ns_per_s)),5110 .sec = @intCast(@divFloor(nanoseconds, std.time.ns_per_s)),
5111 .nsec = @intCast(@mod(nanoseconds, std.time.ns_per_s)),5111 .nsec = @intCast(@mod(nanoseconds, std.time.ns_per_s)),
...@@ -5503,44 +5503,7 @@ const darwin_supports_ulock_wait2 = builtin.os.version_range.semver.min.major >=...@@ -5503,44 +5503,7 @@ const darwin_supports_ulock_wait2 = builtin.os.version_range.semver.min.major >=
5503fn futexWait(t: *Threaded, ptr: *const std.atomic.Value(u32), expect: u32) Io.Cancelable!void {5503fn futexWait(t: *Threaded, ptr: *const std.atomic.Value(u32), expect: u32) Io.Cancelable!void {
5504 @branchHint(.cold);5504 @branchHint(.cold);
55055505
5506 if (native_os == .linux) {5506 if (builtin.cpu.arch.isWasm()) {
5507 const linux = std.os.linux;
5508 try t.checkCancel();
5509 const rc = linux.futex_4arg(ptr, .{ .cmd = .WAIT, .private = true }, expect, null);
5510 if (is_debug) switch (linux.E.init(rc)) {
5511 .SUCCESS => {}, // notified by `wake()`
5512 .INTR => {}, // gives caller a chance to check cancellation
5513 .AGAIN => {}, // ptr.* != expect
5514 .INVAL => {}, // possibly timeout overflow
5515 .TIMEDOUT => unreachable,
5516 .FAULT => unreachable, // ptr was invalid
5517 else => unreachable,
5518 };
5519 } else if (native_os.isDarwin()) {
5520 const c = std.c;
5521 const flags: c.UL = .{
5522 .op = .COMPARE_AND_WAIT,
5523 .NO_ERRNO = true,
5524 };
5525 try t.checkCancel();
5526 const status = if (darwin_supports_ulock_wait2)
5527 c.__ulock_wait2(flags, ptr, expect, 0, 0)
5528 else
5529 c.__ulock_wait(flags, ptr, expect, 0);
5530
5531 if (status >= 0) return;
5532
5533 if (is_debug) switch (@as(c.E, @enumFromInt(-status))) {
5534 // Wait was interrupted by the OS or other spurious signalling.
5535 .INTR => {},
5536 // Address of the futex was paged out. This is unlikely, but possible in theory, and
5537 // pthread/libdispatch on darwin bother to handle it. In this case we'll return
5538 // without waiting, but the caller should retry anyway.
5539 .FAULT => {},
5540 .TIMEDOUT => unreachable,
5541 else => unreachable,
5542 };
5543 } else if (builtin.cpu.arch.isWasm()) {
5544 comptime assert(builtin.cpu.has(.wasm, .atomics));5507 comptime assert(builtin.cpu.has(.wasm, .atomics));
5545 try t.checkCancel();5508 try t.checkCancel();
5546 const timeout: i64 = -1;5509 const timeout: i64 = -1;
...@@ -5562,57 +5525,74 @@ fn futexWait(t: *Threaded, ptr: *const std.atomic.Value(u32), expect: u32) Io.Ca...@@ -5562,57 +5525,74 @@ fn futexWait(t: *Threaded, ptr: *const std.atomic.Value(u32), expect: u32) Io.Ca
5562 2 => assert(!is_debug), // timeout5525 2 => assert(!is_debug), // timeout
5563 else => assert(!is_debug),5526 else => assert(!is_debug),
5564 }5527 }
5565 } else if (is_windows) {5528 } else switch (native_os) {
5566 try t.checkCancel();5529 .linux => {
5567 switch (windows.ntdll.RtlWaitOnAddress(ptr, &expect, @sizeOf(@TypeOf(expect)), null)) {5530 const linux = std.os.linux;
5568 .SUCCESS => {},5531 try t.checkCancel();
5569 .CANCELLED => return error.Canceled,5532 const rc = linux.futex_4arg(ptr, .{ .cmd = .WAIT, .private = true }, expect, null);
5570 else => recoverableOsBugDetected(),5533 if (is_debug) switch (linux.E.init(rc)) {
5571 }5534 .SUCCESS => {}, // notified by `wake()`
5572 } else {5535 .INTR => {}, // gives caller a chance to check cancellation
5573 @compileError("TODO");5536 .AGAIN => {}, // ptr.* != expect
5537 .INVAL => {}, // possibly timeout overflow
5538 .TIMEDOUT => unreachable,
5539 .FAULT => unreachable, // ptr was invalid
5540 else => unreachable,
5541 };
5542 },
5543 .driverkit, .ios, .macos, .tvos, .visionos, .watchos => {
5544 const c = std.c;
5545 const flags: c.UL = .{
5546 .op = .COMPARE_AND_WAIT,
5547 .NO_ERRNO = true,
5548 };
5549 try t.checkCancel();
5550 const status = if (darwin_supports_ulock_wait2)
5551 c.__ulock_wait2(flags, ptr, expect, 0, 0)
5552 else
5553 c.__ulock_wait(flags, ptr, expect, 0);
5554
5555 if (status >= 0) return;
5556
5557 if (is_debug) switch (@as(c.E, @enumFromInt(-status))) {
5558 .INTR => {}, // spurious wake
5559 // Address of the futex was paged out. This is unlikely, but possible in theory, and
5560 // pthread/libdispatch on darwin bother to handle it. In this case we'll return
5561 // without waiting, but the caller should retry anyway.
5562 .FAULT => {},
5563 .TIMEDOUT => unreachable,
5564 else => unreachable,
5565 };
5566 },
5567 .windows => {
5568 try t.checkCancel();
5569 switch (windows.ntdll.RtlWaitOnAddress(ptr, &expect, @sizeOf(@TypeOf(expect)), null)) {
5570 .SUCCESS => {},
5571 .CANCELLED => return error.Canceled,
5572 else => recoverableOsBugDetected(),
5573 }
5574 },
5575 .freebsd => {
5576 const flags = @intFromEnum(std.c.UMTX_OP.WAIT_UINT_PRIVATE);
5577 try t.checkCancel();
5578 const rc = std.c._umtx_op(@intFromPtr(&ptr.raw), flags, @as(c_ulong, expect), 0, 0);
5579 if (is_debug) switch (posix.errno(rc)) {
5580 .SUCCESS => {},
5581 .FAULT => unreachable, // one of the args points to invalid memory
5582 .INVAL => unreachable, // arguments should be correct
5583 .TIMEDOUT => unreachable, // no timeout provided
5584 .INTR => {}, // spurious wake
5585 else => unreachable,
5586 };
5587 },
5588 else => @compileError("unimplemented: futexWait"),
5574 }5589 }
5575}5590}
55765591
5577pub fn futexWaitUncancelable(ptr: *const std.atomic.Value(u32), expect: u32) void {5592pub fn futexWaitUncancelable(ptr: *const std.atomic.Value(u32), expect: u32) void {
5578 @branchHint(.cold);5593 @branchHint(.cold);
55795594
5580 if (native_os == .linux) {5595 if (builtin.cpu.arch.isWasm()) {
5581 const linux = std.os.linux;
5582 const rc = linux.futex_4arg(ptr, .{ .cmd = .WAIT, .private = true }, expect, null);
5583 if (is_debug) switch (linux.E.init(rc)) {
5584 .SUCCESS => {}, // notified by `wake()`
5585 .INTR => {}, // gives caller a chance to check cancellation
5586 .AGAIN => {}, // ptr.* != expect
5587 .INVAL => {}, // possibly timeout overflow
5588 .TIMEDOUT => unreachable,
5589 .FAULT => unreachable, // ptr was invalid
5590 else => unreachable,
5591 };
5592 } else if (native_os.isDarwin()) {
5593 const c = std.c;
5594 const flags: c.UL = .{
5595 .op = .COMPARE_AND_WAIT,
5596 .NO_ERRNO = true,
5597 };
5598 const status = if (darwin_supports_ulock_wait2)
5599 c.__ulock_wait2(flags, ptr, expect, 0, 0)
5600 else
5601 c.__ulock_wait(flags, ptr, expect, 0);
5602
5603 if (status >= 0) return;
5604
5605 if (is_debug) switch (@as(c.E, @enumFromInt(-status))) {
5606 // Wait was interrupted by the OS or other spurious signalling.
5607 .INTR => {},
5608 // Address of the futex was paged out. This is unlikely, but possible in theory, and
5609 // pthread/libdispatch on darwin bother to handle it. In this case we'll return
5610 // without waiting, but the caller should retry anyway.
5611 .FAULT => {},
5612 .TIMEDOUT => unreachable,
5613 else => unreachable,
5614 };
5615 } else if (builtin.cpu.arch.isWasm()) {
5616 comptime assert(builtin.cpu.has(.wasm, .atomics));5596 comptime assert(builtin.cpu.has(.wasm, .atomics));
5617 const timeout: i64 = -1;5597 const timeout: i64 = -1;
5618 const signed_expect: i32 = @bitCast(expect);5598 const signed_expect: i32 = @bitCast(expect);
...@@ -5630,16 +5610,66 @@ pub fn futexWaitUncancelable(ptr: *const std.atomic.Value(u32), expect: u32) voi...@@ -5630,16 +5610,66 @@ pub fn futexWaitUncancelable(ptr: *const std.atomic.Value(u32), expect: u32) voi
5630 switch (result) {5610 switch (result) {
5631 0 => {}, // ok5611 0 => {}, // ok
5632 1 => {}, // expected != loaded5612 1 => {}, // expected != loaded
5633 2 => assert(!is_debug), // timeout5613 2 => recoverableOsBugDetected(), // timeout
5634 else => assert(!is_debug),
5635 }
5636 } else if (is_windows) {
5637 switch (windows.ntdll.RtlWaitOnAddress(ptr, &expect, @sizeOf(@TypeOf(expect)), null)) {
5638 .SUCCESS, .CANCELLED => {},
5639 else => recoverableOsBugDetected(),5614 else => recoverableOsBugDetected(),
5640 }5615 }
5641 } else {5616 } else switch (native_os) {
5642 @compileError("TODO");5617 .linux => {
5618 const linux = std.os.linux;
5619 const rc = linux.futex_4arg(ptr, .{ .cmd = .WAIT, .private = true }, expect, null);
5620 switch (linux.E.init(rc)) {
5621 .SUCCESS => {}, // notified by `wake()`
5622 .INTR => {}, // gives caller a chance to check cancellation
5623 .AGAIN => {}, // ptr.* != expect
5624 .INVAL => {}, // possibly timeout overflow
5625 .TIMEDOUT => recoverableOsBugDetected(),
5626 .FAULT => recoverableOsBugDetected(), // ptr was invalid
5627 else => recoverableOsBugDetected(),
5628 }
5629 },
5630 .driverkit, .ios, .macos, .tvos, .visionos, .watchos => {
5631 const c = std.c;
5632 const flags: c.UL = .{
5633 .op = .COMPARE_AND_WAIT,
5634 .NO_ERRNO = true,
5635 };
5636 const status = if (darwin_supports_ulock_wait2)
5637 c.__ulock_wait2(flags, ptr, expect, 0, 0)
5638 else
5639 c.__ulock_wait(flags, ptr, expect, 0);
5640
5641 if (status >= 0) return;
5642
5643 switch (@as(c.E, @enumFromInt(-status))) {
5644 // Wait was interrupted by the OS or other spurious signalling.
5645 .INTR => {},
5646 // Address of the futex was paged out. This is unlikely, but possible in theory, and
5647 // pthread/libdispatch on darwin bother to handle it. In this case we'll return
5648 // without waiting, but the caller should retry anyway.
5649 .FAULT => {},
5650 .TIMEDOUT => recoverableOsBugDetected(),
5651 else => recoverableOsBugDetected(),
5652 }
5653 },
5654 .windows => {
5655 switch (windows.ntdll.RtlWaitOnAddress(ptr, &expect, @sizeOf(@TypeOf(expect)), null)) {
5656 .SUCCESS, .CANCELLED => {},
5657 else => recoverableOsBugDetected(),
5658 }
5659 },
5660 .freebsd => {
5661 const flags = @intFromEnum(std.c.UMTX_OP.WAIT_UINT_PRIVATE);
5662 const rc = std.c._umtx_op(@intFromPtr(&ptr.raw), flags, @as(c_ulong, expect), 0, 0);
5663 switch (posix.errno(rc)) {
5664 .SUCCESS => {},
5665 .INTR => {}, // spurious wake
5666 .FAULT => recoverableOsBugDetected(), // one of the args points to invalid memory
5667 .INVAL => recoverableOsBugDetected(), // arguments should be correct
5668 .TIMEDOUT => recoverableOsBugDetected(), // no timeout provided
5669 else => recoverableOsBugDetected(),
5670 }
5671 },
5672 else => @compileError("unimplemented: futexWaitUncancelable"),
5643 }5673 }
5644}5674}
56455675
...@@ -5668,38 +5698,7 @@ pub fn futexWaitDurationUncancelable(ptr: *const std.atomic.Value(u32), expect:...@@ -5668,38 +5698,7 @@ pub fn futexWaitDurationUncancelable(ptr: *const std.atomic.Value(u32), expect:
5668pub fn futexWake(ptr: *const std.atomic.Value(u32), max_waiters: u32) void {5698pub fn futexWake(ptr: *const std.atomic.Value(u32), max_waiters: u32) void {
5669 @branchHint(.cold);5699 @branchHint(.cold);
56705700
5671 if (native_os == .linux) {5701 if (builtin.cpu.arch.isWasm()) {
5672 const linux = std.os.linux;
5673 const rc = linux.futex_3arg(
5674 &ptr.raw,
5675 .{ .cmd = .WAKE, .private = true },
5676 @min(max_waiters, std.math.maxInt(i32)),
5677 );
5678 if (is_debug) switch (linux.E.init(rc)) {
5679 .SUCCESS => {}, // successful wake up
5680 .INVAL => {}, // invalid futex_wait() on ptr done elsewhere
5681 .FAULT => {}, // pointer became invalid while doing the wake
5682 else => unreachable,
5683 };
5684 } else if (native_os.isDarwin()) {
5685 const c = std.c;
5686 const flags: c.UL = .{
5687 .op = .COMPARE_AND_WAIT,
5688 .NO_ERRNO = true,
5689 .WAKE_ALL = max_waiters > 1,
5690 };
5691 while (true) {
5692 const status = c.__ulock_wake(flags, ptr, 0);
5693 if (status >= 0) return;
5694 switch (@as(c.E, @enumFromInt(-status))) {
5695 .INTR, .CANCELED => continue, // spurious wake()
5696 .FAULT => assert(!is_debug), // __ulock_wake doesn't generate EFAULT according to darwin pthread_cond_t
5697 .NOENT => return, // nothing was woken up
5698 .ALREADY => assert(!is_debug), // only for UL.Op.WAKE_THREAD
5699 else => assert(!is_debug),
5700 }
5701 }
5702 } else if (builtin.cpu.arch.isWasm()) {
5703 comptime assert(builtin.cpu.has(.wasm, .atomics));5702 comptime assert(builtin.cpu.has(.wasm, .atomics));
5704 assert(max_waiters != 0);5703 assert(max_waiters != 0);
5705 const woken_count = asm volatile (5704 const woken_count = asm volatile (
...@@ -5712,14 +5711,63 @@ pub fn futexWake(ptr: *const std.atomic.Value(u32), max_waiters: u32) void {...@@ -5712,14 +5711,63 @@ pub fn futexWake(ptr: *const std.atomic.Value(u32), max_waiters: u32) void {
5712 [waiters] "r" (max_waiters),5711 [waiters] "r" (max_waiters),
5713 );5712 );
5714 _ = woken_count; // can be 0 when linker flag 'shared-memory' is not enabled5713 _ = woken_count; // can be 0 when linker flag 'shared-memory' is not enabled
5715 } else if (is_windows) {5714 } else switch (native_os) {
5716 assert(max_waiters != 0);5715 .linux => {
5717 switch (max_waiters) {5716 const linux = std.os.linux;
5718 1 => windows.ntdll.RtlWakeAddressSingle(ptr),5717 const rc = linux.futex_3arg(
5719 else => windows.ntdll.RtlWakeAddressAll(ptr),5718 &ptr.raw,
5720 }5719 .{ .cmd = .WAKE, .private = true },
5721 } else {5720 @min(max_waiters, std.math.maxInt(i32)),
5722 @compileError("TODO");5721 );
5722 if (is_debug) switch (linux.E.init(rc)) {
5723 .SUCCESS => {}, // successful wake up
5724 .INVAL => {}, // invalid futex_wait() on ptr done elsewhere
5725 .FAULT => {}, // pointer became invalid while doing the wake
5726 else => unreachable, // deadlock due to operating system bug
5727 };
5728 },
5729 .driverkit, .ios, .macos, .tvos, .visionos, .watchos => {
5730 const c = std.c;
5731 const flags: c.UL = .{
5732 .op = .COMPARE_AND_WAIT,
5733 .NO_ERRNO = true,
5734 .WAKE_ALL = max_waiters > 1,
5735 };
5736 while (true) {
5737 const status = c.__ulock_wake(flags, ptr, 0);
5738 if (status >= 0) return;
5739 switch (@as(c.E, @enumFromInt(-status))) {
5740 .INTR, .CANCELED => continue, // spurious wake()
5741 .FAULT => unreachable, // __ulock_wake doesn't generate EFAULT according to darwin pthread_cond_t
5742 .NOENT => return, // nothing was woken up
5743 .ALREADY => unreachable, // only for UL.Op.WAKE_THREAD
5744 else => unreachable, // deadlock due to operating system bug
5745 }
5746 }
5747 },
5748 .windows => {
5749 assert(max_waiters != 0);
5750 switch (max_waiters) {
5751 1 => windows.ntdll.RtlWakeAddressSingle(ptr),
5752 else => windows.ntdll.RtlWakeAddressAll(ptr),
5753 }
5754 },
5755 .freebsd => {
5756 const rc = std.c._umtx_op(
5757 @intFromPtr(&ptr.raw),
5758 @intFromEnum(std.c.UMTX_OP.WAKE_PRIVATE),
5759 @as(c_ulong, max_waiters),
5760 0, // there is no timeout struct
5761 0, // there is no timeout struct pointer
5762 );
5763 switch (posix.errno(rc)) {
5764 .SUCCESS => {},
5765 .FAULT => {}, // it's ok if the ptr doesn't point to valid memory
5766 .INVAL => unreachable, // arguments should be correct
5767 else => unreachable, // deadlock due to operating system bug
5768 }
5769 },
5770 else => @compileError("unimplemented: futexWake"),
5723 }5771 }
5724}5772}
57255773