authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2026-02-14 04:49:03-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2026-02-14 05:52:59-05:00
logb7f93695f90e92b4437346bd993bbdb4d86f7b07
tree5e0a50dfadb2410c3a243c63d390765a90ebdce0
parentf996d28666e7d562b6a18e9f80fe31a3463456d0

Io.Dispatch.Mutex: fix deadlock conditions

2 files changed, 150 insertions(+), 131 deletions(-)

lib/std/Io/Dispatch.zig+147-128
...@@ -280,7 +280,7 @@ const Fiber = struct {...@@ -280,7 +280,7 @@ const Fiber = struct {
280 .select => if (@atomicRmw(i32, &fiber.await_count, .Add, 1, .monotonic) == -1) {280 .select => if (@atomicRmw(i32, &fiber.await_count, .Add, 1, .monotonic) == -1) {
281 ev.queue.async(fiber, &Fiber.@"resume");281 ev.queue.async(fiber, &Fiber.@"resume");
282 },282 },
283 _ => |awaiting| awaiting.toCancelable().canceled(),283 _ => |awaiting| awaiting.toCancelable().async(),
284 }284 }
285 }285 }
286286
...@@ -484,7 +484,7 @@ pub fn io(ev: *Evented) Io {...@@ -484,7 +484,7 @@ pub fn io(ev: *Evented) Io {
484484
485pub const InitOptions = struct {485pub const InitOptions = struct {
486 backing_allocator_needs_mutex: bool = true,486 backing_allocator_needs_mutex: bool = true,
487 queue: ?c.dispatch.queue_t = null,487 target_queue: ?c.dispatch.queue_t = .TARGET_DEFAULT,
488 /// Upper limit on the allowable delay in processing timeouts in order to improve power488 /// Upper limit on the allowable delay in processing timeouts in order to improve power
489 /// consumption and system performance.489 /// consumption and system performance.
490 leeway: Io.Duration = .fromMilliseconds(10),490 leeway: Io.Duration = .fromMilliseconds(10),
...@@ -499,11 +499,11 @@ pub const InitOptions = struct {...@@ -499,11 +499,11 @@ pub const InitOptions = struct {
499};499};
500500
501pub fn init(ev: *Evented, backing_allocator: Allocator, options: InitOptions) !void {501pub fn init(ev: *Evented, backing_allocator: Allocator, options: InitOptions) !void {
502 const queue = if (options.queue) |queue| queue: {502 const queue = c.dispatch.queue_create_with_target(
503 queue.as_object().retain();503 "org.ziglang.std.Io.Dispatch",
504 break :queue queue;504 .CONCURRENT(),
505 } else c.dispatch.queue_create("org.ziglang.std.Io.Dispatch", .CONCURRENT()) orelse505 options.target_queue,
506 return error.SystemResources;506 ) orelse return error.SystemResources;
507 errdefer queue.as_object().release();507 errdefer queue.as_object().release();
508 const main_loop_stack = try backing_allocator.alignedAlloc(508 const main_loop_stack = try backing_allocator.alignedAlloc(
509 u8,509 u8,
...@@ -727,9 +727,9 @@ const Cancelable = struct {...@@ -727,9 +727,9 @@ const Cancelable = struct {
727727
728 const blocked: Cancelable = .{ .queue = undefined, .cancel = is_blocked };728 const blocked: Cancelable = .{ .queue = undefined, .cancel = is_blocked };
729729
730 const AwaitError = error{CancelRequested};730 const RequestedError = error{CancelRequested};
731731
732 fn await(cancelable: *Cancelable, fiber: *Fiber) AwaitError!void {732 fn enter(cancelable: *Cancelable, fiber: *Fiber) RequestedError!void {
733 const function = cancelable.cancel;733 const function = cancelable.cancel;
734 assert(function != is_requested);734 assert(function != is_requested);
735 if (function == is_blocked) {735 if (function == is_blocked) {
...@@ -750,13 +750,42 @@ const Cancelable = struct {...@@ -750,13 +750,42 @@ const Cancelable = struct {
750 }750 }
751 }751 }
752752
753 fn canceled(cancelable: *Cancelable) void {753 fn leave(cancelable: *Cancelable, fiber: *Fiber) RequestedError!void {
754 assert(cancelable.cancel != is_blocked);754 const function = cancelable.cancel;
755 assert(cancelable.cancel != is_requested);755 assert(function != is_requested);
756 cancelable.queue.async(cancelable, cancelable.cancel);756 if (function == is_blocked) {
757 @branchHint(.unlikely);
758 return;
759 }
760 const cancel_status = @atomicRmw(Fiber.CancelStatus, &fiber.cancel_status, .And, .{
761 .requested = true,
762 .awaiting = .nothing,
763 }, .monotonic);
764 assert(cancel_status.awaiting.toCancelable() == cancelable);
765 if (cancel_status.requested) return error.CancelRequested;
766 }
767
768 fn async(cancelable: *Cancelable) void {
769 const function = cancelable.cancel;
770 assert(function != is_blocked and function != is_requested);
771 cancelable.queue.async(cancelable, function);
772 }
773
774 fn requested(cancelable: *Cancelable, fiber: *Fiber) void {
775 const function = cancelable.cancel;
776 assert(function != is_blocked and function != is_requested);
777 assert(@atomicLoad(Fiber.CancelStatus, &fiber.cancel_status, .monotonic) == Fiber.CancelStatus{
778 .requested = true,
779 .awaiting = .fromCancelable(cancelable),
780 });
781 cancelable.cancel = is_requested;
782 @atomicStore(Fiber.CancelStatus, &fiber.cancel_status, .{
783 .requested = true,
784 .awaiting = .nothing,
785 }, .monotonic);
757 }786 }
758787
759 fn check(cancelable: *Cancelable, fiber: *Fiber) Io.Cancelable!void {788 fn acknowledge(cancelable: *Cancelable, fiber: *Fiber) Io.Cancelable!void {
760 if (cancelable.cancel == is_requested) {789 if (cancelable.cancel == is_requested) {
761 @branchHint(.unlikely);790 @branchHint(.unlikely);
762 fiber.cancel_protection.acknowledge();791 fiber.cancel_protection.acknowledge();
...@@ -784,81 +813,97 @@ const Sleeper = struct {...@@ -784,81 +813,97 @@ const Sleeper = struct {
784};813};
785814
786const Mutex = struct {815const Mutex = struct {
787 /// including the locker816 state: State,
788 num_waiters: usize,
789 queue: c.dispatch.queue_t,817 queue: c.dispatch.queue_t,
790 waiters: std.DoublyLinkedList,818 waiters: std.DoublyLinkedList,
791819
820 const State = packed struct(usize) {
821 locked: bool,
822 num_waiters: NumWaiters,
823
824 const NumWaiters = @Int(.unsigned, @bitSizeOf(usize) - 1);
825 };
826
792 const Waiter = struct {827 const Waiter = struct {
793 sleeper: Sleeper = undefined,828 sleeper: Sleeper = undefined,
794 cancelable: Cancelable,829 cancelable: Cancelable,
795 mutex: *Mutex,830 mutex: *Mutex,
796 node: std.DoublyLinkedList.Node = .{},831 node: std.DoublyLinkedList.Node = undefined,
797832
798 fn add(context: ?*anyopaque) callconv(.c) void {833 fn add(context: ?*anyopaque) callconv(.c) void {
799 const waiter: *Waiter = @ptrCast(@alignCast(context));834 const waiter: *Waiter = @ptrCast(@alignCast(context));
800 waiter.tryAdd() catch |err| switch (err) {835 waiter.cancelable.enter(waiter.sleeper.fiber) catch |err| switch (err) {
836 error.CancelRequested => return waiter.wake(),
837 };
838 var state = @atomicRmw(State, &waiter.mutex.state, .Add, .{
839 .locked = false,
840 .num_waiters = 1,
841 }, .monotonic);
842 state.num_waiters += 1;
843 while (!state.locked) {
844 @branchHint(.unlikely);
845 state = @cmpxchgWeak(State, &waiter.mutex.state, state, .{
846 .locked = true,
847 .num_waiters = state.num_waiters - 1,
848 }, .acquire, .monotonic) orelse break;
849 } else return waiter.mutex.waiters.append(&waiter.node);
850 waiter.cancelable.leave(waiter.sleeper.fiber) catch |err| switch (err) {
801 error.CancelRequested => {851 error.CancelRequested => {
802 waiter.wake();852 waiter.node.next = &waiter.node;
803 assert(@atomicRmw(usize, &waiter.mutex.num_waiters, .Sub, 1, .monotonic) >= 1);853 return;
804 },854 },
805 };855 };
806 }856 waiter.wake();
807
808 fn tryAdd(waiter: *Waiter) Cancelable.AwaitError!void {
809 switch (@atomicLoad(usize, &waiter.mutex.num_waiters, .acquire)) {
810 0 => unreachable,
811 1 => return waiter.wake(), // already locked exclusively
812 else => try waiter.cancelable.await(waiter.sleeper.fiber),
813 }
814 waiter.mutex.waiters.append(&waiter.node);
815 }857 }
816858
817 fn canceled(context: ?*anyopaque) callconv(.c) void {859 fn canceled(context: ?*anyopaque) callconv(.c) void {
818 const cancelable: *Cancelable = @ptrCast(@alignCast(context));860 const cancelable: *Cancelable = @ptrCast(@alignCast(context));
819 cancelable.cancel = Cancelable.is_requested;
820 const waiter: *Waiter = @fieldParentPtr("cancelable", cancelable);861 const waiter: *Waiter = @fieldParentPtr("cancelable", cancelable);
821 assert(@atomicRmw(862 cancelable.requested(waiter.sleeper.fiber);
822 Fiber.CancelStatus,
823 &waiter.sleeper.fiber.cancel_status,
824 .Xchg,
825 .{ .requested = true, .awaiting = .nothing },
826 .monotonic,
827 ) == Fiber.CancelStatus{ .requested = true, .awaiting = .fromCancelable(cancelable) });
828 const mutex = waiter.mutex;863 const mutex = waiter.mutex;
829 mutex.waiters.remove(&waiter.node);864 if (waiter.node.next != &waiter.node) {
865 @branchHint(.likely);
866 mutex.waiters.remove(&waiter.node);
867 assert(@atomicRmw(State, &mutex.state, .Sub, .{
868 .locked = false,
869 .num_waiters = 1,
870 }, .monotonic).num_waiters >= 1);
871 }
872 waiter.node = undefined;
830 waiter.wake();873 waiter.wake();
831 assert(@atomicRmw(usize, &mutex.num_waiters, .Sub, 1, .monotonic) >= 1);
832 }874 }
833875
834 fn remove(context: ?*anyopaque) callconv(.c) void {876 fn remove(context: ?*anyopaque) callconv(.c) void {
835 const mutex: *Mutex = @ptrCast(@alignCast(context));877 const mutex: *Mutex = @ptrCast(@alignCast(context));
836 var stop_node: ?*std.DoublyLinkedList.Node = null;878 var state = @atomicLoad(State, &mutex.state, .monotonic);
837 while (mutex.waiters.first != stop_node) {879 while (!state.locked and state.num_waiters > 0) {
880 @branchHint(.likely);
881 state = @cmpxchgWeak(State, &mutex.state, state, .{
882 .locked = true,
883 .num_waiters = state.num_waiters - 1,
884 }, .acquire, .monotonic) orelse break;
885 } else return;
886 var num_removed: State.NumWaiters = 0;
887 while (mutex.waiters.popFirst()) |node| {
838 @branchHint(.likely);888 @branchHint(.likely);
839 const waiter: *Waiter = @fieldParentPtr("node", mutex.waiters.popFirst().?);889 const waiter: *Waiter = @fieldParentPtr("node", node);
840 if (waiter.cancelable.cancel != Cancelable.is_blocked) {890 node.* = undefined;
841 @branchHint(.likely);891 waiter.cancelable.leave(waiter.sleeper.fiber) catch |err| switch (err) {
842 const cancel_status = @atomicRmw(892 error.CancelRequested => {
843 Fiber.CancelStatus,893 num_removed += 1;
844 &waiter.sleeper.fiber.cancel_status,894 node.next = node;
845 .And,
846 .{ .requested = true, .awaiting = .nothing },
847 .monotonic,
848 );
849 assert(cancel_status.awaiting.toCancelable() == &waiter.cancelable);
850 if (cancel_status.requested) {
851 @branchHint(.unlikely);
852 // carefully place the hot potato out of the way
853 mutex.waiters.append(&waiter.node);
854 if (stop_node == null) stop_node = &waiter.node;
855 continue;895 continue;
856 }896 },
857 }897 };
858 waiter.wake();898 break;
859 return;899 }
900 if (num_removed > 0) {
901 @branchHint(.unlikely);
902 assert(@atomicRmw(State, &mutex.state, .Sub, .{
903 .locked = false,
904 .num_waiters = num_removed,
905 }, .monotonic).num_waiters >= num_removed);
860 }906 }
861 // everyone is about to die, nobody will wake up ;-(
862 }907 }
863908
864 fn wake(waiter: *Waiter) void {909 fn wake(waiter: *Waiter) void {
...@@ -868,7 +913,7 @@ const Mutex = struct {...@@ -868,7 +913,7 @@ const Mutex = struct {
868913
869 fn init(mutex: *Mutex, queue: c.dispatch.queue_t) error{SystemResources}!void {914 fn init(mutex: *Mutex, queue: c.dispatch.queue_t) error{SystemResources}!void {
870 mutex.* = .{915 mutex.* = .{
871 .num_waiters = 0,916 .state = .{ .locked = false, .num_waiters = 0 },
872 .queue = c.dispatch.queue_create_with_target(917 .queue = c.dispatch.queue_create_with_target(
873 "org.ziglang.std.Io.Dispatch.Mutex",918 "org.ziglang.std.Io.Dispatch.Mutex",
874 .SERIAL(),919 .SERIAL(),
...@@ -879,56 +924,48 @@ const Mutex = struct {...@@ -879,56 +924,48 @@ const Mutex = struct {
879 }924 }
880925
881 fn deinit(mutex: *Mutex) void {926 fn deinit(mutex: *Mutex) void {
882 assert(mutex.num_waiters == 0 and mutex.waiters.first == null and mutex.waiters.last == null);927 assert(mutex.state == State{ .locked = false, .num_waiters = 0 });
928 assert(mutex.waiters.first == null and mutex.waiters.last == null);
883 mutex.queue.as_object().release();929 mutex.queue.as_object().release();
884 mutex.* = undefined;930 mutex.* = undefined;
885 }931 }
886932
887 fn tryLock(mutex: *Mutex) bool {933 fn tryLock(mutex: *Mutex) bool {
888 if (@cmpxchgWeak(usize, &mutex.num_waiters, 0, 1, .acquire, .monotonic) == null) {934 const state =
889 @branchHint(.likely);935 @atomicRmw(State, &mutex.state, .Or, .{ .locked = true, .num_waiters = 0 }, .acquire);
890 return true;936 if (state.locked) {
937 @branchHint(.unlikely);
891 }938 }
892 return false;939 return !state.locked;
893 }940 }
894941
895 fn lock(mutex: *Mutex, ev: *Evented) Io.Cancelable!void {942 fn lock(mutex: *Mutex, ev: *Evented) Io.Cancelable!void {
896 switch (@atomicRmw(usize, &mutex.num_waiters, .Add, 1, .acquire)) {943 if (mutex.tryLock()) return;
897 0 => {},944 var waiter: Waiter = .{
898 else => {945 .cancelable = .{ .queue = mutex.queue, .cancel = &Mutex.Waiter.canceled },
899 @branchHint(.unlikely);946 .mutex = mutex,
900 var waiter: Waiter = .{947 };
901 .cancelable = .{ .queue = mutex.queue, .cancel = &Mutex.Waiter.canceled },948 ev.yield(.{ .mutex_wait = &waiter });
902 .mutex = mutex,949 try waiter.cancelable.acknowledge(waiter.sleeper.fiber);
903 };
904 ev.yield(.{ .mutex_wait = &waiter });
905 try waiter.cancelable.check(waiter.sleeper.fiber);
906 },
907 }
908 }950 }
909951
910 fn lockUncancelable(mutex: *Mutex, ev: *Evented) void {952 fn lockUncancelable(mutex: *Mutex, ev: *Evented) void {
911 switch (@atomicRmw(usize, &mutex.num_waiters, .Add, 1, .acquire)) {953 if (mutex.tryLock()) return;
912 0 => {},954 var waiter: Waiter = .{ .cancelable = .blocked, .mutex = mutex };
913 else => {955 ev.yield(.{ .mutex_wait = &waiter });
914 @branchHint(.unlikely);956 waiter.cancelable.acknowledge(waiter.sleeper.fiber) catch |err| switch (err) {
915 var waiter: Waiter = .{ .cancelable = .blocked, .mutex = mutex };957 error.Canceled => unreachable, // blocked
916 ev.yield(.{ .mutex_wait = &waiter });958 };
917 waiter.cancelable.check(waiter.sleeper.fiber) catch |err| switch (err) {
918 error.Canceled => unreachable, // blocked
919 };
920 },
921 }
922 }959 }
923960
924 fn unlock(mutex: *Mutex) void {961 fn unlock(mutex: *Mutex) void {
925 switch (@atomicRmw(usize, &mutex.num_waiters, .Sub, 1, .release)) {962 const state = @atomicRmw(State, &mutex.state, .And, .{
926 0 => unreachable,963 .locked = false,
927 1 => {},964 .num_waiters = std.math.maxInt(State.NumWaiters),
928 else => {965 }, .release);
929 @branchHint(.unlikely);966 if (state.num_waiters > 0) {
930 mutex.queue.async(mutex, &Waiter.remove);967 @branchHint(.unlikely);
931 },968 mutex.queue.async(mutex, &Waiter.remove);
932 }969 }
933 }970 }
934};971};
...@@ -1517,10 +1554,10 @@ const Futex = struct {...@@ -1517,10 +1554,10 @@ const Futex = struct {
1517 };1554 };
1518 }1555 }
15191556
1520 fn tryAdd(waiter: *Waiter) Cancelable.AwaitError!void {1557 fn tryAdd(waiter: *Waiter) Cancelable.RequestedError!void {
1521 if (@atomicLoad(u32, waiter.ptr, .monotonic) != waiter.expected)1558 if (@atomicLoad(u32, waiter.ptr, .monotonic) != waiter.expected)
1522 return error.CancelRequested;1559 return error.CancelRequested;
1523 try waiter.cancelable.await(waiter.sleeper.fiber);1560 try waiter.cancelable.enter(waiter.sleeper.fiber);
1524 const futex = waiter.futex;1561 const futex = waiter.futex;
1525 switch (waiter.timeout) {1562 switch (waiter.timeout) {
1526 .FOREVER => {},1563 .FOREVER => {},
...@@ -1542,46 +1579,28 @@ const Futex = struct {...@@ -1542,46 +1579,28 @@ const Futex = struct {
15421579
1543 fn canceled(context: ?*anyopaque) callconv(.c) void {1580 fn canceled(context: ?*anyopaque) callconv(.c) void {
1544 const cancelable: *Cancelable = @ptrCast(@alignCast(context));1581 const cancelable: *Cancelable = @ptrCast(@alignCast(context));
1545 cancelable.cancel = Cancelable.is_requested;
1546 const waiter: *Waiter = @fieldParentPtr("cancelable", cancelable);1582 const waiter: *Waiter = @fieldParentPtr("cancelable", cancelable);
1547 assert(@atomicRmw(1583 cancelable.requested(waiter.sleeper.fiber);
1548 Fiber.CancelStatus,
1549 &waiter.sleeper.fiber.cancel_status,
1550 .Xchg,
1551 .{ .requested = true, .awaiting = .nothing },
1552 .monotonic,
1553 ) == Fiber.CancelStatus{ .requested = true, .awaiting = .fromCancelable(cancelable) });
1554 const futex = waiter.futex;1584 const futex = waiter.futex;
1555 waiter.removeUncancelable();1585 waiter.remove();
1556 assert(@atomicRmw(usize, &futex.num_waiters, .Sub, 1, .monotonic) >= 1);1586 assert(@atomicRmw(usize, &futex.num_waiters, .Sub, 1, .monotonic) >= 1);
1557 }1587 }
15581588
1559 fn timedOut(context: ?*anyopaque) callconv(.c) void {1589 fn timedOut(context: ?*anyopaque) callconv(.c) void {
1560 const waiter: *Waiter = @ptrCast(@alignCast(context));1590 const waiter: *Waiter = @ptrCast(@alignCast(context));
1561 const futex = waiter.futex;1591 const futex = waiter.futex;
1562 waiter.remove() catch |err| switch (err) {1592 waiter.tryRemove() catch |err| switch (err) {
1563 error.CancelRequested => return,1593 error.CancelRequested => return,
1564 };1594 };
1565 assert(@atomicRmw(usize, &futex.num_waiters, .Sub, 1, .monotonic) >= 1);1595 assert(@atomicRmw(usize, &futex.num_waiters, .Sub, 1, .monotonic) >= 1);
1566 }1596 }
15671597
1568 fn remove(waiter: *Waiter) Cancelable.AwaitError!void {1598 fn tryRemove(waiter: *Waiter) Cancelable.RequestedError!void {
1569 if (waiter.cancelable.cancel != Cancelable.is_blocked) {1599 try waiter.cancelable.leave(waiter.sleeper.fiber);
1570 @branchHint(.likely);1600 waiter.remove();
1571 const cancel_status = @atomicRmw(
1572 Fiber.CancelStatus,
1573 &waiter.sleeper.fiber.cancel_status,
1574 .And,
1575 .{ .requested = true, .awaiting = .nothing },
1576 .monotonic,
1577 );
1578 assert(cancel_status.awaiting.toCancelable() == &waiter.cancelable);
1579 if (cancel_status.requested) return error.CancelRequested;
1580 }
1581 waiter.removeUncancelable();
1582 }1601 }
15831602
1584 fn removeUncancelable(waiter: *Waiter) void {1603 fn remove(waiter: *Waiter) void {
1585 waiter.futex.waiters.remove(&waiter.node);1604 waiter.futex.waiters.remove(&waiter.node);
1586 if (waiter.timer) |timer| timer.cancel() else wake(waiter);1605 if (waiter.timer) |timer| timer.cancel() else wake(waiter);
1587 }1606 }
...@@ -1614,7 +1633,7 @@ const Futex = struct {...@@ -1614,7 +1633,7 @@ const Futex = struct {
1614 @branchHint(.unlikely);1633 @branchHint(.unlikely);
1615 continue;1634 continue;
1616 }1635 }
1617 waiter.remove() catch |err| switch (err) {1636 waiter.tryRemove() catch |err| switch (err) {
1618 error.CancelRequested => continue,1637 error.CancelRequested => continue,
1619 };1638 };
1620 num_removed += 1;1639 num_removed += 1;
...@@ -1720,7 +1739,7 @@ fn futexWait(...@@ -1720,7 +1739,7 @@ fn futexWait(
1720 .leeway = ev.leeway,1739 .leeway = ev.leeway,
1721 };1740 };
1722 ev.yield(.{ .futex_wait = &waiter });1741 ev.yield(.{ .futex_wait = &waiter });
1723 try waiter.cancelable.check(waiter.sleeper.fiber);1742 try waiter.cancelable.acknowledge(waiter.sleeper.fiber);
1724}1743}
17251744
1726fn futexWaitUncancelable(userdata: ?*anyopaque, ptr: *const u32, expected: u32) void {1745fn futexWaitUncancelable(userdata: ?*anyopaque, ptr: *const u32, expected: u32) void {
...@@ -1734,7 +1753,7 @@ fn futexWaitUncancelable(userdata: ?*anyopaque, ptr: *const u32, expected: u32)...@@ -1734,7 +1753,7 @@ fn futexWaitUncancelable(userdata: ?*anyopaque, ptr: *const u32, expected: u32)
1734 .leeway = ev.leeway,1753 .leeway = ev.leeway,
1735 };1754 };
1736 ev.yield(.{ .futex_wait = &waiter });1755 ev.yield(.{ .futex_wait = &waiter });
1737 waiter.cancelable.check(waiter.sleeper.fiber) catch |err| switch (err) {1756 waiter.cancelable.acknowledge(waiter.sleeper.fiber) catch |err| switch (err) {
1738 error.Canceled => unreachable, // blocked1757 error.Canceled => unreachable, // blocked
1739 };1758 };
1740}1759}
lib/std/c/darwin/dispatch.zig+3-3
...@@ -39,14 +39,13 @@ pub const once_t = enum(isize) {...@@ -39,14 +39,13 @@ pub const once_t = enum(isize) {
39 _,39 _,
4040
41 pub inline fn once(predicate: *once_t, context: ?*anyopaque, function: function_t) void {41 pub inline fn once(predicate: *once_t, context: ?*anyopaque, function: function_t) void {
42 if (@atomicLoad(once_t, predicate, .unordered) != .done) {42 if (predicate.* != .done) {
43 @branchHint(.unlikely);43 @branchHint(.unlikely);
44 once_f(predicate, context, function);44 once_f(predicate, context, function);
45 } else asm volatile ("" ::: .{ .memory = true });45 } else asm volatile ("" ::: .{ .memory = true });
46 switch (builtin.mode) {46 switch (builtin.mode) {
47 .Debug, .ReleaseSafe => {},47 .Debug, .ReleaseSafe => {},
48 .ReleaseFast, .ReleaseSmall => if (@atomicLoad(once_t, predicate, .unordered) != .done)48 .ReleaseFast, .ReleaseSmall => if (predicate.* != .done) unreachable,
49 unreachable,
50 }49 }
51 }50 }
52};51};
...@@ -110,6 +109,7 @@ const queue_s = opaque {...@@ -110,6 +109,7 @@ const queue_s = opaque {
110 pub const get_current = get_current_queue;109 pub const get_current = get_current_queue;
111 pub const get_main = get_main_queue;110 pub const get_main = get_main_queue;
112 pub const get_global = get_global_queue;111 pub const get_global = get_global_queue;
112 pub const TARGET_DEFAULT = TARGET_QUEUE_DEFAULT;
113 pub const create_with_target = queue_create_with_target;113 pub const create_with_target = queue_create_with_target;
114 pub const create = queue_create;114 pub const create = queue_create;
115 pub const get_label = queue_get_label;115 pub const get_label = queue_get_label;