authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-03 07:52:02+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-03 07:52:02+01:00
loge7e700334d1432efec7d19887c6656d956e260e7
treed16099222c409fbe01cc1b0a3544119a04951532
parente9eadee00654f5f762abe3cdc596359b79893eab
parent4c4e9d054e37afe82a856f5845c548f177996bd4

Merge pull request 'std.Thread sync primitives roundup' (#31084) from sync-cleanup into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/31084

29 files changed, 602 insertions(+), 2957 deletions(-)

CMakeLists.txt-2
......@@ -408,8 +408,6 @@ set(ZIG_STAGE2_SOURCES
408408 lib/std/Target/wasm.zig
409409 lib/std/Target/x86.zig
410410 lib/std/Thread.zig
411 lib/std/Thread/Futex.zig
412 lib/std/Thread/Mutex.zig
413411 lib/std/array_hash_map.zig
414412 lib/std/array_list.zig
415413 lib/std/ascii.zig
lib/compiler/build_runner.zig+11-8
......@@ -30,14 +30,6 @@ pub fn main(init: process.Init.Minimal) !void {
3030 defer _ = debug_gpa_state.deinit();
3131 const gpa = debug_gpa_state.allocator();
3232
33 // ...but we'll back our arena by `std.heap.page_allocator` for efficiency.
34 var single_threaded_arena: std.heap.ArenaAllocator = .init(std.heap.page_allocator);
35 defer single_threaded_arena.deinit();
36 var thread_safe_arena: std.heap.ThreadSafeAllocator = .{ .child_allocator = single_threaded_arena.allocator() };
37 const arena = thread_safe_arena.allocator();
38
39 const args = try init.args.toSlice(arena);
40
4133 var threaded: std.Io.Threaded = .init(gpa, .{
4234 .environ = init.environ,
4335 .argv0 = .init(init.args),
......@@ -45,6 +37,17 @@ pub fn main(init: process.Init.Minimal) !void {
4537 defer threaded.deinit();
4638 const io = threaded.io();
4739
40 // ...but we'll back our arena by `std.heap.page_allocator` for efficiency.
41 var single_threaded_arena: std.heap.ArenaAllocator = .init(std.heap.page_allocator);
42 defer single_threaded_arena.deinit();
43 var thread_safe_arena: std.heap.ThreadSafeAllocator = .{
44 .child_allocator = single_threaded_arena.allocator(),
45 .io = io,
46 };
47 const arena = thread_safe_arena.allocator();
48
49 const args = try init.args.toSlice(arena);
50
4851 // skip my own exe name
4952 var arg_idx: usize = 1;
5053
lib/compiler_rt/emutls.zig+7-2
......@@ -147,7 +147,8 @@ const ObjectArray = struct {
147147// It provides thread-safety for on-demand storage of Thread Objects.
148148const current_thread_storage = struct {
149149 var key: std.c.pthread_key_t = undefined;
150 var init_once = std.once(current_thread_storage.init);
150 var init_mutex: std.c.pthread_mutex_t = std.c.PTHREAD_MUTEX_INITIALIZER;
151 var init_done: bool = false;
151152
152153 /// Return a per thread ObjectArray with at least the expected index.
153154 pub fn getArray(index: usize) *ObjectArray {
......@@ -183,9 +184,13 @@ const current_thread_storage = struct {
183184
184185 /// Initialize pthread_key_t.
185186 fn init() void {
187 if (@atomicLoad(bool, &init_done, .monotonic)) return;
188 _ = std.c.pthread_mutex_lock(&init_mutex);
186189 if (std.c.pthread_key_create(&current_thread_storage.key, current_thread_storage.deinit) != .SUCCESS) {
187190 abort();
188191 }
192 @atomicStore(bool, &init_done, true, .release);
193 _ = std.c.pthread_mutex_unlock(&init_mutex);
189194 }
190195
191196 /// Invoked by pthread specific destructor. the passed argument is the ObjectArray pointer.
......@@ -283,7 +288,7 @@ const emutls_control = extern struct {
283288 /// Get the pointer on allocated storage for emutls variable.
284289 pub fn getPointer(self: *emutls_control) *anyopaque {
285290 // ensure current_thread_storage initialization is done
286 current_thread_storage.init_once.call();
291 current_thread_storage.init();
287292
288293 const index = self.getIndex();
289294 var array = current_thread_storage.getArray(index);
lib/fuzzer.zig+1-1
......@@ -632,7 +632,7 @@ export fn fuzzer_main(limit_kind: abi.LimitKind, amount: u64) void {
632632
633633export fn fuzzer_unslide_address(addr: usize) usize {
634634 const si = std.debug.getSelfDebugInfo() catch @compileError("unsupported");
635 const slide = si.getModuleSlide(std.debug.getDebugInfoAllocator(), addr) catch |err| {
635 const slide = si.getModuleSlide(std.debug.getDebugInfoAllocator(), io, addr) catch |err| {
636636 std.debug.panic("failed to find virtual address slide: {t}", .{err});
637637 };
638638 return addr - slide;
lib/std/Io.zig+14-1
......@@ -47,6 +47,9 @@ pub const Dir = @import("Io/Dir.zig");
4747pub const File = @import("Io/File.zig");
4848pub const Terminal = @import("Io/Terminal.zig");
4949
50pub const RwLock = @import("Io/RwLock.zig");
51pub const Semaphore = @import("Io/Semaphore.zig");
52
5053pub const VTable = struct {
5154 /// If it returns `null` it means `result` has been already populated and
5255 /// `await` will be a no-op.
......@@ -882,7 +885,7 @@ pub const Timeout = union(enum) {
882885
883886 pub const Error = error{ Timeout, UnsupportedClock };
884887
885 pub fn toDeadline(t: Timeout, io: Io) Clock.Error!?Clock.Timestamp {
888 pub fn toTimestamp(t: Timeout, io: Io) Clock.Error!?Clock.Timestamp {
886889 return switch (t) {
887890 .none => null,
888891 .duration => |d| try .fromNow(io, d),
......@@ -890,6 +893,14 @@ pub const Timeout = union(enum) {
890893 };
891894 }
892895
896 pub fn toDeadline(t: Timeout, io: Io) Timeout {
897 return switch (t) {
898 .none => .none,
899 .duration => |d| .{ .deadline = Clock.Timestamp.fromNow(io, d) catch @panic("TODO") },
900 .deadline => |d| .{ .deadline = d },
901 };
902 }
903
893904 pub fn toDurationFromNow(t: Timeout, io: Io) Clock.Error!?Clock.Duration {
894905 return switch (t) {
895906 .none => null,
......@@ -2153,5 +2164,7 @@ test {
21532164 _ = Writer;
21542165 _ = Evented;
21552166 _ = Threaded;
2167 _ = RwLock;
2168 _ = Semaphore;
21562169 _ = @import("Io/test.zig");
21572170}
lib/std/Io/RwLock.zig created+238
......@@ -0,0 +1,238 @@
1//! A lock that supports one writer or many readers.
2const RwLock = @This();
3
4const builtin = @import("builtin");
5
6const std = @import("../std.zig");
7const Io = std.Io;
8const assert = std.debug.assert;
9const testing = std.testing;
10
11state: usize,
12mutex: Io.Mutex,
13semaphore: Io.Semaphore,
14
15pub const init: RwLock = .{
16 .state = 0,
17 .mutex = .init,
18 .semaphore = .{},
19};
20
21const is_writing: usize = 1;
22const writer: usize = 1 << 1;
23const reader: usize = 1 << (1 + @bitSizeOf(Count));
24const writer_mask: usize = std.math.maxInt(Count) << @ctz(writer);
25const reader_mask: usize = std.math.maxInt(Count) << @ctz(reader);
26const Count = @Int(.unsigned, @divFloor(@bitSizeOf(usize) - 1, 2));
27
28pub fn tryLock(rl: *RwLock, io: Io) bool {
29 if (rl.mutex.tryLock()) {
30 const state = @atomicLoad(usize, &rl.state, .seq_cst);
31 if (state & reader_mask == 0) {
32 _ = @atomicRmw(usize, &rl.state, .Or, is_writing, .seq_cst);
33 return true;
34 }
35
36 rl.mutex.unlock(io);
37 }
38
39 return false;
40}
41
42pub fn lockUncancelable(rl: *RwLock, io: Io) void {
43 _ = @atomicRmw(usize, &rl.state, .Add, writer, .seq_cst);
44 rl.mutex.lockUncancelable(io);
45
46 const state = @atomicRmw(usize, &rl.state, .Add, is_writing -% writer, .seq_cst);
47 if (state & reader_mask != 0)
48 rl.semaphore.waitUncancelable(io);
49}
50
51pub fn unlock(rl: *RwLock, io: Io) void {
52 _ = @atomicRmw(usize, &rl.state, .And, ~is_writing, .seq_cst);
53 rl.mutex.unlock(io);
54}
55
56pub fn tryLockShared(rl: *RwLock, io: Io) bool {
57 const state = @atomicLoad(usize, &rl.state, .seq_cst);
58 if (state & (is_writing | writer_mask) == 0) {
59 _ = @cmpxchgStrong(
60 usize,
61 &rl.state,
62 state,
63 state + reader,
64 .seq_cst,
65 .seq_cst,
66 ) orelse return true;
67 }
68
69 if (rl.mutex.tryLock()) {
70 _ = @atomicRmw(usize, &rl.state, .Add, reader, .seq_cst);
71 rl.mutex.unlock(io);
72 return true;
73 }
74
75 return false;
76}
77
78pub fn lockSharedUncancelable(rl: *RwLock, io: Io) void {
79 var state = @atomicLoad(usize, &rl.state, .seq_cst);
80 while (state & (is_writing | writer_mask) == 0) {
81 state = @cmpxchgWeak(
82 usize,
83 &rl.state,
84 state,
85 state + reader,
86 .seq_cst,
87 .seq_cst,
88 ) orelse return;
89 }
90
91 rl.mutex.lockUncancelable(io);
92 _ = @atomicRmw(usize, &rl.state, .Add, reader, .seq_cst);
93 rl.mutex.unlock(io);
94}
95
96pub fn unlockShared(rl: *RwLock, io: Io) void {
97 const state = @atomicRmw(usize, &rl.state, .Sub, reader, .seq_cst);
98
99 if ((state & reader_mask == reader) and (state & is_writing != 0))
100 rl.semaphore.post(io);
101}
102
103test "internal state" {
104 const io = testing.io;
105
106 var rl: Io.RwLock = .init;
107
108 // The following failed prior to the fix for Issue #13163,
109 // where the WRITER flag was subtracted by the lock method.
110
111 rl.lockUncancelable(io);
112 rl.unlock(io);
113 try testing.expectEqual(rl, Io.RwLock.init);
114}
115
116test "smoke test" {
117 const io = testing.io;
118
119 var rl: Io.RwLock = .init;
120
121 rl.lockUncancelable(io);
122 try testing.expect(!rl.tryLock(io));
123 try testing.expect(!rl.tryLockShared(io));
124 rl.unlock(io);
125
126 try testing.expect(rl.tryLock(io));
127 try testing.expect(!rl.tryLock(io));
128 try testing.expect(!rl.tryLockShared(io));
129 rl.unlock(io);
130
131 rl.lockSharedUncancelable(io);
132 try testing.expect(!rl.tryLock(io));
133 try testing.expect(rl.tryLockShared(io));
134 rl.unlockShared(io);
135 rl.unlockShared(io);
136
137 try testing.expect(rl.tryLockShared(io));
138 try testing.expect(!rl.tryLock(io));
139 try testing.expect(rl.tryLockShared(io));
140 rl.unlockShared(io);
141 rl.unlockShared(io);
142
143 rl.lockUncancelable(io);
144 rl.unlock(io);
145}
146
147test "concurrent access" {
148 if (builtin.single_threaded) return;
149
150 const io = testing.io;
151 const num_writers: usize = 2;
152 const num_readers: usize = 4;
153 const num_writes: usize = 1000;
154 const num_reads: usize = 2000;
155
156 const Runner = struct {
157 const Runner = @This();
158
159 io: Io,
160
161 rl: Io.RwLock,
162 writes: usize,
163 reads: std.atomic.Value(usize),
164
165 val_a: usize,
166 val_b: usize,
167
168 fn reader(run: *Runner, thread_idx: usize) !void {
169 var prng = std.Random.DefaultPrng.init(thread_idx);
170 const rnd = prng.random();
171 while (true) {
172 run.rl.lockSharedUncancelable(run.io);
173 defer run.rl.unlockShared(run.io);
174
175 try testing.expect(run.writes <= num_writes);
176 if (run.reads.fetchAdd(1, .monotonic) >= num_reads) break;
177
178 // We use `volatile` accesses so that we can make sure the memory is accessed either
179 // side of a yield, maximising chances of a race.
180 const a_ptr: *const volatile usize = &run.val_a;
181 const b_ptr: *const volatile usize = &run.val_b;
182
183 const old_a = a_ptr.*;
184 if (rnd.boolean()) try std.Thread.yield();
185 const old_b = b_ptr.*;
186 try testing.expect(old_a == old_b);
187 }
188 }
189
190 fn writer(run: *Runner, thread_idx: usize) !void {
191 var prng = std.Random.DefaultPrng.init(thread_idx);
192 const rnd = prng.random();
193 while (true) {
194 run.rl.lockUncancelable(run.io);
195 defer run.rl.unlock(run.io);
196
197 try testing.expect(run.writes <= num_writes);
198 if (run.writes == num_writes) break;
199
200 // We use `volatile` accesses so that we can make sure the memory is accessed either
201 // side of a yield, maximising chances of a race.
202 const a_ptr: *volatile usize = &run.val_a;
203 const b_ptr: *volatile usize = &run.val_b;
204
205 const new_val = rnd.int(usize);
206
207 const old_a = a_ptr.*;
208 a_ptr.* = new_val;
209 if (rnd.boolean()) try std.Thread.yield();
210 const old_b = b_ptr.*;
211 b_ptr.* = new_val;
212 try testing.expect(old_a == old_b);
213
214 run.writes += 1;
215 }
216 }
217 };
218
219 var run: Runner = .{
220 .io = io,
221 .rl = .init,
222 .writes = 0,
223 .reads = .init(0),
224 .val_a = 0,
225 .val_b = 0,
226 };
227 var write_threads: [num_writers]std.Thread = undefined;
228 var read_threads: [num_readers]std.Thread = undefined;
229
230 for (&write_threads, 0..) |*t, i| t.* = try .spawn(.{}, Runner.writer, .{ &run, i });
231 for (&read_threads, num_writers..) |*t, i| t.* = try .spawn(.{}, Runner.reader, .{ &run, i });
232
233 for (write_threads) |t| t.join();
234 for (read_threads) |t| t.join();
235
236 try testing.expect(run.writes == num_writes);
237 try testing.expect(run.reads.raw >= num_reads);
238}
lib/std/Io/Semaphore.zig created+65
......@@ -0,0 +1,65 @@
1//! An unsigned integer that blocks the kernel thread if the number would
2//! become negative.
3//!
4//! This API supports static initialization and does not require deinitialization.
5const Semaphore = @This();
6
7const builtin = @import("builtin");
8
9const std = @import("../std.zig");
10const Io = std.Io;
11const testing = std.testing;
12
13mutex: Io.Mutex = .init,
14cond: Io.Condition = .init,
15/// It is OK to initialize this field to any value.
16permits: usize = 0,
17
18pub fn wait(s: *Semaphore, io: Io) Io.Cancelable!void {
19 try s.mutex.lock(io);
20 defer s.mutex.unlock(io);
21 while (s.permits == 0) try s.cond.wait(io, &s.mutex);
22 s.permits -= 1;
23 if (s.permits > 0) s.cond.signal(io);
24}
25
26pub fn waitUncancelable(s: *Semaphore, io: Io) void {
27 s.mutex.lockUncancelable(io);
28 defer s.mutex.unlock(io);
29 while (s.permits == 0) s.cond.waitUncancelable(io, &s.mutex);
30 s.permits -= 1;
31 if (s.permits > 0) s.cond.signal(io);
32}
33
34pub fn post(s: *Semaphore, io: Io) void {
35 s.mutex.lockUncancelable(io);
36 defer s.mutex.unlock(io);
37
38 s.permits += 1;
39 s.cond.signal(io);
40}
41
42test Semaphore {
43 if (builtin.single_threaded) return error.SkipZigTest;
44 const io = testing.io;
45
46 const TestContext = struct {
47 sem: *Semaphore,
48 n: *i32,
49 fn worker(ctx: *@This()) !void {
50 try ctx.sem.wait(io);
51 ctx.n.* += 1;
52 ctx.sem.post(io);
53 }
54 };
55 const num_threads = 3;
56 var sem: Semaphore = .{ .permits = 1 };
57 var threads: [num_threads]std.Thread = undefined;
58 var n: i32 = 0;
59 var ctx = TestContext{ .sem = &sem, .n = &n };
60
61 for (&threads) |*t| t.* = try std.Thread.spawn(.{}, TestContext.worker, .{&ctx});
62 for (threads) |t| t.join();
63 try sem.wait(io);
64 try testing.expect(n == num_threads);
65}
lib/std/Io/Threaded.zig+89-81
......@@ -1126,6 +1126,25 @@ const Thread = struct {
11261126 return @ptrFromInt(@as(usize, @bitCast(split)));
11271127 }
11281128 };
1129
1130 /// Same as `Io.Mutex.lock` but avoids the VTable.
1131 fn mutexLock(m: *Io.Mutex) Io.Cancelable!void {
1132 const initial_state = m.state.cmpxchgWeak(
1133 .unlocked,
1134 .locked_once,
1135 .acquire,
1136 .monotonic,
1137 ) orelse {
1138 @branchHint(.likely);
1139 return;
1140 };
1141 if (initial_state == .contended) {
1142 try Thread.futexWait(@ptrCast(&m.state.raw), @intFromEnum(Io.Mutex.State.contended), null);
1143 }
1144 while (m.state.swap(.contended, .acquire) != .unlocked) {
1145 try Thread.futexWait(@ptrCast(&m.state.raw), @intFromEnum(Io.Mutex.State.contended), null);
1146 }
1147 }
11291148};
11301149
11311150const Syscall = struct {
......@@ -1486,8 +1505,8 @@ var global_single_threaded_instance: Threaded = .init_single_threaded;
14861505pub const global_single_threaded: *Threaded = &global_single_threaded_instance;
14871506
14881507pub fn setAsyncLimit(t: *Threaded, new_limit: Io.Limit) void {
1489 mutexLockUncancelable(&t.mutex);
1490 defer mutexUnlock(&t.mutex);
1508 mutexLockInternal(&t.mutex);
1509 defer mutexUnlockInternal(&t.mutex);
14911510 t.async_limit = new_limit;
14921511}
14931512
......@@ -1508,8 +1527,8 @@ pub fn deinit(t: *Threaded) void {
15081527fn join(t: *Threaded) void {
15091528 if (builtin.single_threaded) return;
15101529 {
1511 mutexLockUncancelable(&t.mutex);
1512 defer mutexUnlock(&t.mutex);
1530 mutexLockInternal(&t.mutex);
1531 defer mutexUnlockInternal(&t.mutex);
15131532 t.join_requested = true;
15141533 }
15151534 condBroadcast(&t.cond);
......@@ -1574,16 +1593,16 @@ fn worker(t: *Threaded) void {
15741593
15751594 defer t.wait_group.finish();
15761595
1577 mutexLockUncancelable(&t.mutex);
1578 defer mutexUnlock(&t.mutex);
1596 mutexLockInternal(&t.mutex);
1597 defer mutexUnlockInternal(&t.mutex);
15791598
15801599 while (true) {
15811600 while (t.run_queue.popFirst()) |runnable_node| {
1582 mutexUnlock(&t.mutex);
1601 mutexUnlockInternal(&t.mutex);
15831602 thread.cancel_protection = .unblocked;
15841603 const runnable: *Runnable = @fieldParentPtr("node", runnable_node);
15851604 runnable.startFn(runnable, &thread, t);
1586 mutexLockUncancelable(&t.mutex);
1605 mutexLockInternal(&t.mutex);
15871606 t.busy_count -= 1;
15881607 }
15891608 if (t.join_requested) break;
......@@ -2004,12 +2023,12 @@ fn async(
20042023 },
20052024 };
20062025
2007 mutexLockUncancelable(&t.mutex);
2026 mutexLockInternal(&t.mutex);
20082027
20092028 const busy_count = t.busy_count;
20102029
20112030 if (busy_count >= @intFromEnum(t.async_limit)) {
2012 mutexUnlock(&t.mutex);
2031 mutexUnlockInternal(&t.mutex);
20132032 future.destroy(gpa);
20142033 start(context.ptr, result.ptr);
20152034 return null;
......@@ -2023,7 +2042,7 @@ fn async(
20232042 const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch {
20242043 t.wait_group.finish();
20252044 t.busy_count = busy_count;
2026 mutexUnlock(&t.mutex);
2045 mutexUnlockInternal(&t.mutex);
20272046 future.destroy(gpa);
20282047 start(context.ptr, result.ptr);
20292048 return null;
......@@ -2033,7 +2052,7 @@ fn async(
20332052
20342053 t.run_queue.prepend(&future.runnable.node);
20352054
2036 mutexUnlock(&t.mutex);
2055 mutexUnlockInternal(&t.mutex);
20372056 condSignal(&t.cond);
20382057 return @ptrCast(future);
20392058}
......@@ -2056,8 +2075,8 @@ fn concurrent(
20562075 };
20572076 errdefer future.destroy(gpa);
20582077
2059 mutexLockUncancelable(&t.mutex);
2060 defer mutexUnlock(&t.mutex);
2078 mutexLockInternal(&t.mutex);
2079 defer mutexUnlockInternal(&t.mutex);
20612080
20622081 const busy_count = t.busy_count;
20632082
......@@ -2101,12 +2120,12 @@ fn groupAsync(
21012120 error.OutOfMemory => return groupAsyncEager(start, context.ptr),
21022121 };
21032122
2104 mutexLockUncancelable(&t.mutex);
2123 mutexLockInternal(&t.mutex);
21052124
21062125 const busy_count = t.busy_count;
21072126
21082127 if (busy_count >= @intFromEnum(t.async_limit)) {
2109 mutexUnlock(&t.mutex);
2128 mutexUnlockInternal(&t.mutex);
21102129 task.destroy(gpa);
21112130 return groupAsyncEager(start, context.ptr);
21122131 }
......@@ -2119,7 +2138,7 @@ fn groupAsync(
21192138 const thread = std.Thread.spawn(.{ .stack_size = t.stack_size }, worker, .{t}) catch {
21202139 t.wait_group.finish();
21212140 t.busy_count = busy_count;
2122 mutexUnlock(&t.mutex);
2141 mutexUnlockInternal(&t.mutex);
21232142 task.destroy(gpa);
21242143 return groupAsyncEager(start, context.ptr);
21252144 };
......@@ -2136,7 +2155,7 @@ fn groupAsync(
21362155 }, .monotonic);
21372156 t.run_queue.prepend(&task.runnable.node);
21382157
2139 mutexUnlock(&t.mutex);
2158 mutexUnlockInternal(&t.mutex);
21402159 condSignal(&t.cond);
21412160}
21422161fn groupAsyncEager(
......@@ -2201,8 +2220,8 @@ fn groupConcurrent(
22012220 };
22022221 errdefer task.destroy(gpa);
22032222
2204 mutexLockUncancelable(&t.mutex);
2205 defer mutexUnlock(&t.mutex);
2223 mutexLockInternal(&t.mutex);
2224 defer mutexUnlockInternal(&t.mutex);
22062225
22072226 const busy_count = t.busy_count;
22082227
......@@ -2636,7 +2655,7 @@ fn batchAwaitAsync(userdata: ?*anyopaque, b: *Io.Batch) Io.Cancelable!void {
26362655fn batchAwaitConcurrent(userdata: ?*anyopaque, b: *Io.Batch, timeout: Io.Timeout) Io.Batch.AwaitConcurrentError!void {
26372656 const t: *Threaded = @ptrCast(@alignCast(userdata));
26382657 if (is_windows) {
2639 const deadline: ?Io.Clock.Timestamp = timeout.toDeadline(ioBasic(t)) catch |err| switch (err) {
2658 const deadline: ?Io.Clock.Timestamp = timeout.toTimestamp(ioBasic(t)) catch |err| switch (err) {
26402659 error.Unexpected => deadline: {
26412660 recoverableOsBugDetected();
26422661 break :deadline .{ .raw = .{ .nanoseconds = 0 }, .clock = .awake };
......@@ -2735,7 +2754,7 @@ fn batchAwaitConcurrent(userdata: ?*anyopaque, b: *Io.Batch, timeout: Io.Timeout
27352754 else => {},
27362755 }
27372756 const t_io = ioBasic(t);
2738 const deadline = timeout.toDeadline(t_io) catch return error.UnsupportedClock;
2757 const deadline = timeout.toTimestamp(t_io) catch return error.UnsupportedClock;
27392758 while (true) {
27402759 const timeout_ms: i32 = t: {
27412760 if (b.completions.head != .none) {
......@@ -3838,8 +3857,8 @@ fn fileStatWindows(userdata: ?*anyopaque, file: File) File.StatError!File.Stat {
38383857
38393858fn systemBasicInformation(t: *Threaded) ?*const windows.SYSTEM_BASIC_INFORMATION {
38403859 if (!t.system_basic_information.initialized.load(.acquire)) {
3841 mutexLockUncancelable(&t.mutex);
3842 defer mutexUnlock(&t.mutex);
3860 mutexLockInternal(&t.mutex);
3861 defer mutexUnlockInternal(&t.mutex);
38433862
38443863 switch (windows.ntdll.NtQuerySystemInformation(
38453864 .SystemBasicInformation,
......@@ -10899,7 +10918,7 @@ fn nowWasi(clock: Io.Clock) Io.Clock.Error!Io.Timestamp {
1089910918fn sleep(userdata: ?*anyopaque, timeout: Io.Timeout) Io.SleepError!void {
1090010919 const t: *Threaded = @ptrCast(@alignCast(userdata));
1090110920 if (timeout == .none) return;
10902 if (use_parking_sleep) return parking_sleep.sleep(try timeout.toDeadline(ioBasic(t)));
10921 if (use_parking_sleep) return parking_sleep.sleep(try timeout.toTimestamp(ioBasic(t)));
1090310922 if (native_os == .wasi) return sleepWasi(t, timeout);
1090410923 if (@TypeOf(posix.system.clock_nanosleep) != void) return sleepPosix(timeout);
1090510924 return sleepNanosleep(t, timeout);
......@@ -12611,7 +12630,7 @@ fn netReceivePosix(
1261112630 var message_i: usize = 0;
1261212631 var data_i: usize = 0;
1261312632
12614 const deadline = timeout.toDeadline(t_io) catch |err| return .{ err, message_i };
12633 const deadline = timeout.toTimestamp(t_io) catch |err| return .{ err, message_i };
1261512634
1261612635 recv: while (true) {
1261712636 if (message_buffer.len - message_i == 0) return .{ null, message_i };
......@@ -14373,8 +14392,8 @@ const WindowsEnvironStrings = struct {
1437314392};
1437414393
1437514394fn scanEnviron(t: *Threaded) void {
14376 mutexLockUncancelable(&t.mutex);
14377 defer mutexUnlock(&t.mutex);
14395 mutexLockInternal(&t.mutex);
14396 defer mutexUnlockInternal(&t.mutex);
1437814397
1437914398 if (t.environ.initialized) return;
1438014399 t.environ.initialized = true;
......@@ -14729,8 +14748,8 @@ fn spawnPosix(t: *Threaded, options: process.SpawnOptions) process.SpawnError!Sp
1472914748
1473014749fn getDevNullFd(t: *Threaded) !posix.fd_t {
1473114750 {
14732 mutexLockUncancelable(&t.mutex);
14733 defer mutexUnlock(&t.mutex);
14751 mutexLockInternal(&t.mutex);
14752 defer mutexUnlockInternal(&t.mutex);
1473414753 if (t.null_file.fd != -1) return t.null_file.fd;
1473514754 }
1473614755 const mode: u32 = 0;
......@@ -14741,8 +14760,8 @@ fn getDevNullFd(t: *Threaded) !posix.fd_t {
1474114760 .SUCCESS => {
1474214761 syscall.finish();
1474314762 const fresh_fd: posix.fd_t = @intCast(rc);
14744 mutexLockUncancelable(&t.mutex); // Another thread might have won the race.
14745 defer mutexUnlock(&t.mutex);
14763 mutexLockInternal(&t.mutex); // Another thread might have won the race.
14764 defer mutexUnlockInternal(&t.mutex);
1474614765 if (t.null_file.fd != -1) {
1474714766 posix.close(fresh_fd);
1474814767 return t.null_file.fd;
......@@ -15402,8 +15421,8 @@ fn processSpawnWindows(userdata: ?*anyopaque, options: process.SpawnOptions) pro
1540215421
1540315422fn getCngHandle(t: *Threaded) Io.RandomSecureError!windows.HANDLE {
1540415423 {
15405 mutexLockUncancelable(&t.mutex);
15406 defer mutexUnlock(&t.mutex);
15424 mutexLockInternal(&t.mutex);
15425 defer mutexUnlockInternal(&t.mutex);
1540715426 if (t.random_file.handle) |handle| return handle;
1540815427 }
1540915428
......@@ -15437,8 +15456,8 @@ fn getCngHandle(t: *Threaded) Io.RandomSecureError!windows.HANDLE {
1543715456 )) {
1543815457 .SUCCESS => {
1543915458 syscall.finish();
15440 mutexLockUncancelable(&t.mutex); // Another thread might have won the race.
15441 defer mutexUnlock(&t.mutex);
15459 mutexLockInternal(&t.mutex); // Another thread might have won the race.
15460 defer mutexUnlockInternal(&t.mutex);
1544215461 if (t.random_file.handle) |prev_handle| {
1544315462 windows.CloseHandle(fresh_handle);
1544415463 return prev_handle;
......@@ -15458,8 +15477,8 @@ fn getCngHandle(t: *Threaded) Io.RandomSecureError!windows.HANDLE {
1545815477
1545915478fn getNulHandle(t: *Threaded) !windows.HANDLE {
1546015479 {
15461 mutexLockUncancelable(&t.mutex);
15462 defer mutexUnlock(&t.mutex);
15480 mutexLockInternal(&t.mutex);
15481 defer mutexUnlockInternal(&t.mutex);
1546315482 if (t.null_file.handle) |handle| return handle;
1546415483 }
1546515484
......@@ -15505,8 +15524,8 @@ fn getNulHandle(t: *Threaded) !windows.HANDLE {
1550515524 )) {
1550615525 .SUCCESS => {
1550715526 syscall.finish();
15508 mutexLockUncancelable(&t.mutex); // Another thread might have won the race.
15509 defer mutexUnlock(&t.mutex);
15527 mutexLockInternal(&t.mutex); // Another thread might have won the race.
15528 defer mutexUnlockInternal(&t.mutex);
1551015529 if (t.null_file.handle) |prev_handle| {
1551115530 windows.CloseHandle(fresh_handle);
1551215531 return prev_handle;
......@@ -16551,15 +16570,15 @@ fn random(userdata: ?*anyopaque, buffer: []u8) void {
1655116570}
1655216571
1655316572fn randomMainThread(t: *Threaded, buffer: []u8) void {
16554 mutexLockUncancelable(&t.mutex);
16555 defer mutexUnlock(&t.mutex);
16573 mutexLockInternal(&t.mutex);
16574 defer mutexUnlockInternal(&t.mutex);
1655616575
1655716576 if (!t.csprng.isInitialized()) {
1655816577 @branchHint(.unlikely);
1655916578 var seed: [Csprng.seed_len]u8 = undefined;
1656016579 {
16561 mutexUnlock(&t.mutex);
16562 defer mutexLockUncancelable(&t.mutex);
16580 mutexUnlockInternal(&t.mutex);
16581 defer mutexLockInternal(&t.mutex);
1656316582
1656416583 const prev = swapCancelProtection(t, .blocked);
1656516584 defer _ = swapCancelProtection(t, prev);
......@@ -16744,8 +16763,8 @@ fn randomSecure(userdata: ?*anyopaque, buffer: []u8) Io.RandomSecureError!void {
1674416763
1674516764fn getRandomFd(t: *Threaded) Io.RandomSecureError!posix.fd_t {
1674616765 {
16747 mutexLockUncancelable(&t.mutex);
16748 defer mutexUnlock(&t.mutex);
16766 mutexLockInternal(&t.mutex);
16767 defer mutexUnlockInternal(&t.mutex);
1674916768
1675016769 if (t.random_file.fd == -2) return error.EntropyUnavailable;
1675116770 if (t.random_file.fd != -1) return t.random_file.fd;
......@@ -16785,8 +16804,8 @@ fn getRandomFd(t: *Threaded) Io.RandomSecureError!posix.fd_t {
1678516804 .SUCCESS => {
1678616805 syscall.finish();
1678716806 if (!statx.mask.TYPE) return error.EntropyUnavailable;
16788 mutexLockUncancelable(&t.mutex); // Another thread might have won the race.
16789 defer mutexUnlock(&t.mutex);
16807 mutexLockInternal(&t.mutex); // Another thread might have won the race.
16808 defer mutexUnlockInternal(&t.mutex);
1679016809 if (t.random_file.fd >= 0) {
1679116810 posix.close(fd);
1679216811 return t.random_file.fd;
......@@ -16813,8 +16832,8 @@ fn getRandomFd(t: *Threaded) Io.RandomSecureError!posix.fd_t {
1681316832 switch (posix.errno(fstat_sym(fd, &stat))) {
1681416833 .SUCCESS => {
1681516834 syscall.finish();
16816 mutexLockUncancelable(&t.mutex); // Another thread might have won the race.
16817 defer mutexUnlock(&t.mutex);
16835 mutexLockInternal(&t.mutex); // Another thread might have won the race.
16836 defer mutexUnlockInternal(&t.mutex);
1681816837 if (t.random_file.fd >= 0) {
1681916838 posix.close(fd);
1682016839 return t.random_file.fd;
......@@ -16947,8 +16966,8 @@ const parking_futex = struct {
1694716966 var status_buf: std.atomic.Value(Thread.Status) = undefined;
1694816967
1694916968 {
16950 mutexLockUncancelable(&bucket.mutex);
16951 defer mutexUnlock(&bucket.mutex);
16969 mutexLockInternal(&bucket.mutex);
16970 defer mutexUnlockInternal(&bucket.mutex);
1695216971
1695316972 _ = bucket.num_waiters.fetchAdd(1, .acquire);
1695416973
......@@ -17017,8 +17036,8 @@ const parking_futex = struct {
1701717036 .parked => {
1701817037 // We saw a timeout and updated our own status from `.parked` to `.none`. It is
1701917038 // our responsibility to remove `waiter` from `bucket`.
17020 mutexLockUncancelable(&bucket.mutex);
17021 defer mutexUnlock(&bucket.mutex);
17039 mutexLockInternal(&bucket.mutex);
17040 defer mutexUnlockInternal(&bucket.mutex);
1702217041 bucket.waiters.remove(&waiter.node);
1702317042 assert(bucket.num_waiters.fetchSub(1, .monotonic) > 0);
1702417043 },
......@@ -17057,8 +17076,8 @@ const parking_futex = struct {
1705717076 // of the critical section. This forms a singly-linked list of waiters using `Waiter.node.next`.
1705817077 var waking_head: ?*std.DoublyLinkedList.Node = null;
1705917078 {
17060 mutexLockUncancelable(&bucket.mutex);
17061 defer mutexUnlock(&bucket.mutex);
17079 mutexLockInternal(&bucket.mutex);
17080 defer mutexUnlockInternal(&bucket.mutex);
1706217081
1706317082 var num_removed: u32 = 0;
1706417083 var it = bucket.waiters.first;
......@@ -17113,8 +17132,8 @@ const parking_futex = struct {
1711317132
1711417133 fn removeCanceledWaiter(waiter: *Waiter) void {
1711517134 const bucket = bucketForAddress(waiter.address);
17116 mutexLockUncancelable(&bucket.mutex);
17117 defer mutexUnlock(&bucket.mutex);
17135 mutexLockInternal(&bucket.mutex);
17136 defer mutexUnlockInternal(&bucket.mutex);
1711817137 bucket.waiters.remove(&waiter.node);
1711917138 assert(bucket.num_waiters.fetchSub(1, .monotonic) > 0);
1712017139 waiter.done.store(true, .release); // potentially invalidates `waiter.*`
......@@ -18163,8 +18182,8 @@ fn condWait(cond: *Condition, mutex: *Mutex) void {
1816318182 assert(prev_state.waiters < std.math.maxInt(u16)); // overflow caused by too many waiters
1816418183 }
1816518184
18166 mutexUnlock(mutex);
18167 defer mutexLockUncancelable(mutex);
18185 mutexUnlockInternal(mutex);
18186 defer mutexLockInternal(mutex);
1816818187
1816918188 while (true) {
1817018189 Thread.futexWaitUncancelable(&cond.epoch.raw, epoch, null);
......@@ -18189,28 +18208,13 @@ const Mutex = if (!is_windows) Io.Mutex else struct {
1818918208 const init: @This() = .{ .srwlock = .{} };
1819018209};
1819118210
18192/// Same as `Io.Mutex.lockUncancelable` but avoids the VTable.
18193fn mutexLock(m: *Io.Mutex) Io.Cancelable!void {
18194 const initial_state = m.state.cmpxchgWeak(
18195 .unlocked,
18196 .locked_once,
18197 .acquire,
18198 .monotonic,
18199 ) orelse {
18200 @branchHint(.likely);
18201 return;
18202 };
18203 if (initial_state == .contended) {
18204 try Thread.futexWait(@ptrCast(&m.state.raw), @intFromEnum(Io.Mutex.State.contended), null);
18205 }
18206 while (m.state.swap(.contended, .acquire) != .unlocked) {
18207 try Thread.futexWait(@ptrCast(&m.state.raw), @intFromEnum(Io.Mutex.State.contended), null);
18208 }
18211fn mutexLockInternal(m: *Mutex) void {
18212 if (is_windows) return windows.ntdll.RtlAcquireSRWLockExclusive(&m.srwlock);
18213 return mutexLock(m);
1820918214}
1821018215
1821118216/// Same as `Io.Mutex.lockUncancelable` but avoids the VTable.
18212fn mutexLockUncancelable(m: *Mutex) void {
18213 if (is_windows) return windows.ntdll.RtlAcquireSRWLockExclusive(&m.srwlock);
18217pub fn mutexLock(m: *Io.Mutex) void {
1821418218 const initial_state = m.state.cmpxchgWeak(
1821518219 .unlocked,
1821618220 .locked_once,
......@@ -18228,9 +18232,13 @@ fn mutexLockUncancelable(m: *Mutex) void {
1822818232 }
1822918233}
1823018234
18231/// Same as `Io.Mutex.unlock` but avoids the VTable.
18232fn mutexUnlock(m: *Mutex) void {
18235fn mutexUnlockInternal(m: *Mutex) void {
1823318236 if (is_windows) return windows.ntdll.RtlReleaseSRWLockExclusive(&m.srwlock);
18237 return mutexUnlock(m);
18238}
18239
18240/// Same as `Io.Mutex.unlock` but avoids the VTable.
18241pub fn mutexUnlock(m: *Io.Mutex) void {
1823418242 switch (m.state.swap(.unlocked, .release)) {
1823518243 .unlocked => unreachable,
1823618244 .locked_once => {},
lib/std/Thread.zig+3-11
......@@ -14,13 +14,9 @@ const posix = std.posix;
1414const windows = std.os.windows;
1515const testing = std.testing;
1616
17pub const Futex = @import("Thread/Futex.zig");
18pub const Mutex = @import("Thread/Mutex.zig");
19pub const Semaphore = @import("Thread/Semaphore.zig");
20pub const Condition = @import("Thread/Condition.zig");
21pub const RwLock = @import("Thread/RwLock.zig");
22
23pub const Pool = @compileError("deprecated; consider using 'std.Io.Group' with 'std.Io.Threaded'");
17pub const Mutex = struct {
18 pub const Recursive = @import("Thread/Mutex/Recursive.zig");
19};
2420
2521pub const use_pthreads = native_os != .windows and native_os != .wasi and builtin.link_libc;
2622
......@@ -1609,11 +1605,7 @@ test "setName, getName" {
16091605}
16101606
16111607test {
1612 _ = Futex;
16131608 _ = Mutex;
1614 _ = Semaphore;
1615 _ = Condition;
1616 _ = RwLock;
16171609}
16181610
16191611fn testIncrementNotify(io: Io, value: *usize, event: *Io.Event) void {
lib/std/Thread/Condition.zig deleted-683
......@@ -1,683 +0,0 @@
1//! Condition variables are used with a Mutex to efficiently wait for an arbitrary condition to occur.
2//! It does this by atomically unlocking the mutex, blocking the thread until notified, and finally re-locking the mutex.
3//! Condition can be statically initialized and is at most `@sizeOf(u64)` large.
4//!
5//! Example:
6//! ```
7//! var m = Mutex{};
8//! var c = Condition{};
9//! var predicate = false;
10//!
11//! fn consumer() void {
12//! m.lock();
13//! defer m.unlock();
14//!
15//! while (!predicate) {
16//! c.wait(&m);
17//! }
18//! }
19//!
20//! fn producer() void {
21//! {
22//! m.lock();
23//! defer m.unlock();
24//! predicate = true;
25//! }
26//! c.signal();
27//! }
28//!
29//! const thread = try std.Thread.spawn(.{}, producer, .{});
30//! consumer();
31//! thread.join();
32//! ```
33//!
34//! Note that condition variables can only reliably unblock threads that are sequenced before them using the same Mutex.
35//! This means that the following is allowed to deadlock:
36//! ```
37//! thread-1: mutex.lock()
38//! thread-1: condition.wait(&mutex)
39//!
40//! thread-2: // mutex.lock() (without this, the following signal may not see the waiting thread-1)
41//! thread-2: // mutex.unlock() (this is optional for correctness once locked above, as signal can be called while holding the mutex)
42//! thread-2: condition.signal()
43//! ```
44
45const std = @import("../std.zig");
46const builtin = @import("builtin");
47const Condition = @This();
48const Mutex = std.Thread.Mutex;
49
50const os = std.os;
51const assert = std.debug.assert;
52const testing = std.testing;
53const Futex = std.Thread.Futex;
54
55impl: Impl = .{},
56
57/// Atomically releases the Mutex, blocks the caller thread, then re-acquires the Mutex on return.
58/// "Atomically" here refers to accesses done on the Condition after acquiring the Mutex.
59///
60/// The Mutex must be locked by the caller's thread when this function is called.
61/// A Mutex can have multiple Conditions waiting with it concurrently, but not the opposite.
62/// It is undefined behavior for multiple threads to wait ith different mutexes using the same Condition concurrently.
63/// Once threads have finished waiting with one Mutex, the Condition can be used to wait with another Mutex.
64///
65/// A blocking call to wait() is unblocked from one of the following conditions:
66/// - a spurious ("at random") wake up occurs
67/// - a future call to `signal()` or `broadcast()` which has acquired the Mutex and is sequenced after this `wait()`.
68///
69/// Given wait() can be interrupted spuriously, the blocking condition should be checked continuously
70/// irrespective of any notifications from `signal()` or `broadcast()`.
71pub fn wait(self: *Condition, mutex: *Mutex) void {
72 self.impl.wait(mutex, null) catch |err| switch (err) {
73 error.Timeout => unreachable, // no timeout provided so we shouldn't have timed-out
74 };
75}
76
77/// Atomically releases the Mutex, blocks the caller thread, then re-acquires the Mutex on return.
78/// "Atomically" here refers to accesses done on the Condition after acquiring the Mutex.
79///
80/// The Mutex must be locked by the caller's thread when this function is called.
81/// A Mutex can have multiple Conditions waiting with it concurrently, but not the opposite.
82/// It is undefined behavior for multiple threads to wait ith different mutexes using the same Condition concurrently.
83/// Once threads have finished waiting with one Mutex, the Condition can be used to wait with another Mutex.
84///
85/// A blocking call to `timedWait()` is unblocked from one of the following conditions:
86/// - a spurious ("at random") wake occurs
87/// - the caller was blocked for around `timeout_ns` nanoseconds, in which `error.Timeout` is returned.
88/// - a future call to `signal()` or `broadcast()` which has acquired the Mutex and is sequenced after this `timedWait()`.
89///
90/// Given `timedWait()` can be interrupted spuriously, the blocking condition should be checked continuously
91/// irrespective of any notifications from `signal()` or `broadcast()`.
92pub fn timedWait(self: *Condition, mutex: *Mutex, timeout_ns: u64) error{Timeout}!void {
93 return self.impl.wait(mutex, timeout_ns);
94}
95
96/// Unblocks at least one thread blocked in a call to `wait()` or `timedWait()` with a given Mutex.
97/// The blocked thread must be sequenced before this call with respect to acquiring the same Mutex in order to be observable for unblocking.
98/// `signal()` can be called with or without the relevant Mutex being acquired and have no "effect" if there's no observable blocked threads.
99pub fn signal(self: *Condition) void {
100 self.impl.wake(.one);
101}
102
103/// Unblocks all threads currently blocked in a call to `wait()` or `timedWait()` with a given Mutex.
104/// The blocked threads must be sequenced before this call with respect to acquiring the same Mutex in order to be observable for unblocking.
105/// `broadcast()` can be called with or without the relevant Mutex being acquired and have no "effect" if there's no observable blocked threads.
106pub fn broadcast(self: *Condition) void {
107 self.impl.wake(.all);
108}
109
110const Impl = Impl: {
111 if (builtin.single_threaded) break :Impl SingleThreadedImpl;
112 if (builtin.os.tag == .windows) break :Impl WindowsImpl;
113
114 if (builtin.os.tag.isDarwin() or
115 builtin.target.os.tag == .linux or
116 builtin.target.os.tag == .freebsd or
117 builtin.target.os.tag == .openbsd or
118 builtin.target.os.tag == .dragonfly or
119 builtin.target.cpu.arch.isWasm())
120 {
121 // Futex is the system's synchronization primitive; use that.
122 break :Impl FutexImpl;
123 }
124
125 if (std.Thread.use_pthreads) {
126 // This system doesn't have a futex primitive, so `std.Thread.Futex` is using `PosixImpl`,
127 // which implements futex *on top of* pthread mutexes and conditions. Therefore, instead
128 // of going through that long inefficient path, just use pthread condition variable directly.
129 break :Impl PosixImpl;
130 }
131
132 break :Impl FutexImpl;
133};
134
135const Notify = enum {
136 one, // wake up only one thread
137 all, // wake up all threads
138};
139
140const SingleThreadedImpl = struct {
141 fn wait(self: *Impl, mutex: *Mutex, timeout: ?u64) error{Timeout}!void {
142 _ = self;
143 _ = mutex;
144 // There are no other threads to wake us up.
145 // So if we wait without a timeout we would never wake up.
146 assert(timeout != null); // Deadlock detected.
147 return error.Timeout;
148 }
149
150 fn wake(self: *Impl, comptime notify: Notify) void {
151 // There are no other threads to wake up.
152 _ = self;
153 _ = notify;
154 }
155};
156
157const WindowsImpl = struct {
158 condition: os.windows.CONDITION_VARIABLE = .{},
159
160 fn wait(self: *Impl, mutex: *Mutex, timeout: ?u64) error{Timeout}!void {
161 var timeout_overflowed = false;
162 var timeout_ms: os.windows.DWORD = os.windows.INFINITE;
163
164 if (timeout) |timeout_ns| {
165 // Round the nanoseconds to the nearest millisecond,
166 // then saturating cast it to windows DWORD for use in kernel32 call.
167 const ms = (timeout_ns +| (std.time.ns_per_ms / 2)) / std.time.ns_per_ms;
168 timeout_ms = std.math.cast(os.windows.DWORD, ms) orelse std.math.maxInt(os.windows.DWORD);
169
170 // Track if the timeout overflowed into INFINITE and make sure not to wait forever.
171 if (timeout_ms == os.windows.INFINITE) {
172 timeout_overflowed = true;
173 timeout_ms -= 1;
174 }
175 }
176
177 if (builtin.mode == .Debug) {
178 // The internal state of the DebugMutex needs to be handled here as well.
179 mutex.impl.locking_thread.store(0, .unordered);
180 }
181 const rc = os.windows.kernel32.SleepConditionVariableSRW(
182 &self.condition,
183 if (builtin.mode == .Debug) &mutex.impl.impl.srwlock else &mutex.impl.srwlock,
184 timeout_ms,
185 0, // the srwlock was assumed to acquired in exclusive mode not shared
186 );
187 if (builtin.mode == .Debug) {
188 // The internal state of the DebugMutex needs to be handled here as well.
189 mutex.impl.locking_thread.store(std.Thread.getCurrentId(), .unordered);
190 }
191
192 // Return error.Timeout if we know the timeout elapsed correctly.
193 if (rc == os.windows.FALSE) {
194 assert(os.windows.GetLastError() == .TIMEOUT);
195 if (!timeout_overflowed) return error.Timeout;
196 }
197 }
198
199 fn wake(self: *Impl, comptime notify: Notify) void {
200 switch (notify) {
201 .one => os.windows.ntdll.RtlWakeConditionVariable(&self.condition),
202 .all => os.windows.ntdll.RtlWakeAllConditionVariable(&self.condition),
203 }
204 }
205};
206
207const FutexImpl = struct {
208 state: std.atomic.Value(u32) = std.atomic.Value(u32).init(0),
209 epoch: std.atomic.Value(u32) = std.atomic.Value(u32).init(0),
210
211 const one_waiter = 1;
212 const waiter_mask = 0xffff;
213
214 const one_signal = 1 << 16;
215 const signal_mask = 0xffff << 16;
216
217 fn wait(self: *Impl, mutex: *Mutex, timeout: ?u64) error{Timeout}!void {
218 // Observe the epoch, then check the state again to see if we should wake up.
219 // The epoch must be observed before we check the state or we could potentially miss a wake() and deadlock:
220 //
221 // - T1: s = LOAD(&state)
222 // - T2: UPDATE(&s, signal)
223 // - T2: UPDATE(&epoch, 1) + FUTEX_WAKE(&epoch)
224 // - T1: e = LOAD(&epoch) (was reordered after the state load)
225 // - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed the state update + the epoch change)
226 //
227 // Acquire barrier to ensure the epoch load happens before the state load.
228 var epoch = self.epoch.load(.acquire);
229 var state = self.state.fetchAdd(one_waiter, .monotonic);
230 assert(state & waiter_mask != waiter_mask);
231 state += one_waiter;
232
233 mutex.unlock();
234 defer mutex.lock();
235
236 var futex_deadline = Futex.Deadline.init(timeout);
237
238 while (true) {
239 futex_deadline.wait(&self.epoch, epoch) catch |err| switch (err) {
240 // On timeout, we must decrement the waiter we added above.
241 error.Timeout => {
242 while (true) {
243 // If there's a signal when we're timing out, consume it and report being woken up instead.
244 // Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return.
245 while (state & signal_mask != 0) {
246 const new_state = state - one_waiter - one_signal;
247 state = self.state.cmpxchgWeak(state, new_state, .acquire, .monotonic) orelse return;
248 }
249
250 // Remove the waiter we added and officially return timed out.
251 const new_state = state - one_waiter;
252 state = self.state.cmpxchgWeak(state, new_state, .monotonic, .monotonic) orelse return err;
253 }
254 },
255 };
256
257 epoch = self.epoch.load(.acquire);
258 state = self.state.load(.monotonic);
259
260 // Try to wake up by consuming a signal and decremented the waiter we added previously.
261 // Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return.
262 while (state & signal_mask != 0) {
263 const new_state = state - one_waiter - one_signal;
264 state = self.state.cmpxchgWeak(state, new_state, .acquire, .monotonic) orelse return;
265 }
266 }
267 }
268
269 fn wake(self: *Impl, comptime notify: Notify) void {
270 var state = self.state.load(.monotonic);
271 while (true) {
272 const waiters = (state & waiter_mask) / one_waiter;
273 const signals = (state & signal_mask) / one_signal;
274
275 // Reserves which waiters to wake up by incrementing the signals count.
276 // Therefore, the signals count is always less than or equal to the waiters count.
277 // We don't need to Futex.wake if there's nothing to wake up or if other wake() threads have reserved to wake up the current waiters.
278 const wakeable = waiters - signals;
279 if (wakeable == 0) {
280 return;
281 }
282
283 const to_wake = switch (notify) {
284 .one => 1,
285 .all => wakeable,
286 };
287
288 // Reserve the amount of waiters to wake by incrementing the signals count.
289 // Release barrier ensures code before the wake() happens before the signal it posted and consumed by the wait() threads.
290 const new_state = state + (one_signal * to_wake);
291 state = self.state.cmpxchgWeak(state, new_state, .release, .monotonic) orelse {
292 // Wake up the waiting threads we reserved above by changing the epoch value.
293 // NOTE: a waiting thread could miss a wake up if *exactly* ((1<<32)-1) wake()s happen between it observing the epoch and sleeping on it.
294 // This is very unlikely due to how many precise amount of Futex.wake() calls that would be between the waiting thread's potential preemption.
295 //
296 // Release barrier ensures the signal being added to the state happens before the epoch is changed.
297 // If not, the waiting thread could potentially deadlock from missing both the state and epoch change:
298 //
299 // - T2: UPDATE(&epoch, 1) (reordered before the state change)
300 // - T1: e = LOAD(&epoch)
301 // - T1: s = LOAD(&state)
302 // - T2: UPDATE(&state, signal) + FUTEX_WAKE(&epoch)
303 // - T1: s & signals == 0 -> FUTEX_WAIT(&epoch, e) (missed both epoch change and state change)
304 _ = self.epoch.fetchAdd(1, .release);
305 Futex.wake(&self.epoch, to_wake);
306 return;
307 };
308 }
309 }
310};
311
312const PosixImpl = struct {
313 cond: std.c.pthread_cond_t = .{},
314
315 fn wait(self: *Impl, mutex: *Mutex, timeout: ?u64) error{Timeout}!void {
316 if (builtin.mode == .Debug) {
317 mutex.impl.locking_thread.store(0, .unordered);
318 }
319 defer if (builtin.mode == .Debug) {
320 mutex.impl.locking_thread.store(std.Thread.getCurrentId(), .unordered);
321 };
322
323 const mtx = if (builtin.mode == .Debug) &mutex.impl.impl.mutex else &mutex.impl.mutex;
324
325 if (timeout) |t| {
326 switch (std.c.pthread_cond_timedwait(&self.cond, mtx, &.{
327 .sec = @intCast(@divFloor(t, std.time.ns_per_s)),
328 .nsec = @intCast(@mod(t, std.time.ns_per_s)),
329 })) {
330 .SUCCESS => return,
331 .TIMEDOUT => return error.Timeout,
332 else => unreachable,
333 }
334 }
335
336 assert(std.c.pthread_cond_wait(&self.cond, mtx) == .SUCCESS);
337 }
338
339 fn wake(self: *Impl, comptime notify: Notify) void {
340 assert(switch (notify) {
341 .one => std.c.pthread_cond_signal(&self.cond),
342 .all => std.c.pthread_cond_broadcast(&self.cond),
343 } == .SUCCESS);
344 }
345};
346
347test "smoke test" {
348 var mutex = Mutex{};
349 var cond = Condition{};
350
351 // Try to wake outside the mutex
352 defer cond.signal();
353 defer cond.broadcast();
354
355 mutex.lock();
356 defer mutex.unlock();
357
358 // Try to wait with a timeout (should not deadlock)
359 try testing.expectError(error.Timeout, cond.timedWait(&mutex, 0));
360 try testing.expectError(error.Timeout, cond.timedWait(&mutex, std.time.ns_per_ms));
361
362 // Try to wake inside the mutex.
363 cond.signal();
364 cond.broadcast();
365}
366
367// Inspired from: https://github.com/Amanieu/parking_lot/pull/129
368test "wait and signal" {
369 // This test requires spawning threads
370 if (builtin.single_threaded) {
371 return error.SkipZigTest;
372 }
373
374 const io = testing.io;
375
376 const num_threads = 4;
377
378 const MultiWait = struct {
379 mutex: Mutex = .{},
380 cond: Condition = .{},
381 threads: [num_threads]std.Thread = undefined,
382 spawn_count: std.math.IntFittingRange(0, num_threads) = 0,
383
384 fn run(self: *@This()) void {
385 self.mutex.lock();
386 defer self.mutex.unlock();
387 self.spawn_count += 1;
388
389 self.cond.wait(&self.mutex);
390 self.cond.timedWait(&self.mutex, std.time.ns_per_ms) catch {};
391 self.cond.signal();
392 }
393 };
394
395 var multi_wait = MultiWait{};
396 for (&multi_wait.threads) |*t| {
397 t.* = try std.Thread.spawn(.{}, MultiWait.run, .{&multi_wait});
398 }
399
400 while (true) {
401 try std.Io.Clock.Duration.sleep(.{ .clock = .awake, .raw = .fromMilliseconds(100) }, io);
402
403 multi_wait.mutex.lock();
404 defer multi_wait.mutex.unlock();
405 // Make sure all of the threads have finished spawning to avoid a deadlock.
406 if (multi_wait.spawn_count == num_threads) break;
407 }
408
409 multi_wait.cond.signal();
410 for (multi_wait.threads) |t| {
411 t.join();
412 }
413}
414
415test signal {
416 // This test requires spawning threads
417 if (builtin.single_threaded) {
418 return error.SkipZigTest;
419 }
420
421 const io = testing.io;
422
423 const num_threads = 4;
424
425 const SignalTest = struct {
426 mutex: Mutex = .{},
427 cond: Condition = .{},
428 notified: bool = false,
429 threads: [num_threads]std.Thread = undefined,
430 spawn_count: std.math.IntFittingRange(0, num_threads) = 0,
431
432 fn run(self: *@This()) void {
433 self.mutex.lock();
434 defer self.mutex.unlock();
435 self.spawn_count += 1;
436
437 // Use timedWait() a few times before using wait()
438 // to test multiple threads timing out frequently.
439 var i: usize = 0;
440 while (!self.notified) : (i +%= 1) {
441 if (i < 5) {
442 self.cond.timedWait(&self.mutex, 1) catch {};
443 } else {
444 self.cond.wait(&self.mutex);
445 }
446 }
447
448 // Once we received the signal, notify another thread (inside the lock).
449 assert(self.notified);
450 self.cond.signal();
451 }
452 };
453
454 var signal_test = SignalTest{};
455 for (&signal_test.threads) |*t| {
456 t.* = try std.Thread.spawn(.{}, SignalTest.run, .{&signal_test});
457 }
458
459 while (true) {
460 try std.Io.Clock.Duration.sleep(.{ .clock = .awake, .raw = .fromMilliseconds(10) }, io);
461
462 signal_test.mutex.lock();
463 defer signal_test.mutex.unlock();
464 // Make sure at least one thread has finished spawning to avoid testing nothing.
465 if (signal_test.spawn_count > 0) break;
466 }
467
468 {
469 // Wake up one of them (outside the lock) after setting notified=true.
470 defer signal_test.cond.signal();
471
472 signal_test.mutex.lock();
473 defer signal_test.mutex.unlock();
474
475 try testing.expect(!signal_test.notified);
476 signal_test.notified = true;
477 }
478
479 for (signal_test.threads) |t| {
480 t.join();
481 }
482}
483
484test "multi signal" {
485 // This test requires spawning threads
486 if (builtin.single_threaded) {
487 return error.SkipZigTest;
488 }
489
490 const num_threads = 4;
491 const num_iterations = 4;
492
493 const Paddle = struct {
494 mutex: Mutex = .{},
495 cond: Condition = .{},
496 value: u32 = 0,
497
498 fn hit(self: *@This()) void {
499 defer self.cond.signal();
500
501 self.mutex.lock();
502 defer self.mutex.unlock();
503
504 self.value += 1;
505 }
506
507 fn run(self: *@This(), hit_to: *@This()) !void {
508 self.mutex.lock();
509 defer self.mutex.unlock();
510
511 var current: u32 = 0;
512 while (current < num_iterations) : (current += 1) {
513 // Wait for the value to change from hit()
514 while (self.value == current) {
515 self.cond.wait(&self.mutex);
516 }
517
518 // hit the next paddle
519 try testing.expectEqual(self.value, current + 1);
520 hit_to.hit();
521 }
522 }
523 };
524
525 var paddles = [_]Paddle{.{}} ** num_threads;
526 var threads = [_]std.Thread{undefined} ** num_threads;
527
528 // Create a circle of paddles which hit each other
529 for (&threads, 0..) |*t, i| {
530 const paddle = &paddles[i];
531 const hit_to = &paddles[(i + 1) % paddles.len];
532 t.* = try std.Thread.spawn(.{}, Paddle.run, .{ paddle, hit_to });
533 }
534
535 // Hit the first paddle and wait for them all to complete by hitting each other for num_iterations.
536 paddles[0].hit();
537 for (threads) |t| t.join();
538
539 // The first paddle will be hit one last time by the last paddle.
540 for (paddles, 0..) |p, i| {
541 const expected = @as(u32, num_iterations) + @intFromBool(i == 0);
542 try testing.expectEqual(p.value, expected);
543 }
544}
545
546test broadcast {
547 // This test requires spawning threads
548 if (builtin.single_threaded) {
549 return error.SkipZigTest;
550 }
551
552 const num_threads = 10;
553
554 const BroadcastTest = struct {
555 mutex: Mutex = .{},
556 cond: Condition = .{},
557 completed: Condition = .{},
558 count: usize = 0,
559 threads: [num_threads]std.Thread = undefined,
560
561 fn run(self: *@This()) void {
562 self.mutex.lock();
563 defer self.mutex.unlock();
564
565 // The last broadcast thread to start tells the main test thread it's completed.
566 self.count += 1;
567 if (self.count == num_threads) {
568 self.completed.signal();
569 }
570
571 // Waits for the count to reach zero after the main test thread observes it at num_threads.
572 // Tries to use timedWait() a bit before falling back to wait() to test multiple threads timing out.
573 var i: usize = 0;
574 while (self.count != 0) : (i +%= 1) {
575 if (i < 10) {
576 self.cond.timedWait(&self.mutex, 1) catch {};
577 } else {
578 self.cond.wait(&self.mutex);
579 }
580 }
581 }
582 };
583
584 var broadcast_test = BroadcastTest{};
585 for (&broadcast_test.threads) |*t| {
586 t.* = try std.Thread.spawn(.{}, BroadcastTest.run, .{&broadcast_test});
587 }
588
589 {
590 broadcast_test.mutex.lock();
591 defer broadcast_test.mutex.unlock();
592
593 // Wait for all the broadcast threads to spawn.
594 // timedWait() to detect any potential deadlocks.
595 while (broadcast_test.count != num_threads) {
596 broadcast_test.completed.timedWait(
597 &broadcast_test.mutex,
598 1 * std.time.ns_per_s,
599 ) catch {};
600 }
601
602 // Reset the counter and wake all the threads to exit.
603 broadcast_test.count = 0;
604 broadcast_test.cond.broadcast();
605 }
606
607 for (broadcast_test.threads) |t| {
608 t.join();
609 }
610}
611
612test "broadcasting - wake all threads" {
613 // Tests issue #12877
614 // This test requires spawning threads
615 if (builtin.single_threaded) {
616 return error.SkipZigTest;
617 }
618
619 var num_runs: usize = 1;
620 const num_threads = 10;
621
622 while (num_runs > 0) : (num_runs -= 1) {
623 const BroadcastTest = struct {
624 mutex: Mutex = .{},
625 cond: Condition = .{},
626 completed: Condition = .{},
627 count: usize = 0,
628 thread_id_to_wake: usize = 0,
629 threads: [num_threads]std.Thread = undefined,
630 wakeups: usize = 0,
631
632 fn run(self: *@This(), thread_id: usize) void {
633 self.mutex.lock();
634 defer self.mutex.unlock();
635
636 // The last broadcast thread to start tells the main test thread it's completed.
637 self.count += 1;
638 if (self.count == num_threads) {
639 self.completed.signal();
640 }
641
642 while (self.thread_id_to_wake != thread_id) {
643 self.cond.timedWait(&self.mutex, 1 * std.time.ns_per_s) catch {};
644 self.wakeups += 1;
645 }
646 if (self.thread_id_to_wake <= num_threads) {
647 // Signal next thread to wake up.
648 self.thread_id_to_wake += 1;
649 self.cond.broadcast();
650 }
651 }
652 };
653
654 var broadcast_test = BroadcastTest{};
655 var thread_id: usize = 1;
656 for (&broadcast_test.threads) |*t| {
657 t.* = try std.Thread.spawn(.{}, BroadcastTest.run, .{ &broadcast_test, thread_id });
658 thread_id += 1;
659 }
660
661 {
662 broadcast_test.mutex.lock();
663 defer broadcast_test.mutex.unlock();
664
665 // Wait for all the broadcast threads to spawn.
666 // timedWait() to detect any potential deadlocks.
667 while (broadcast_test.count != num_threads) {
668 broadcast_test.completed.timedWait(
669 &broadcast_test.mutex,
670 1 * std.time.ns_per_s,
671 ) catch {};
672 }
673
674 // Signal thread 1 to wake up
675 broadcast_test.thread_id_to_wake = 1;
676 broadcast_test.cond.broadcast();
677 }
678
679 for (broadcast_test.threads) |t| {
680 t.join();
681 }
682 }
683}
lib/std/Thread/Futex.zig deleted-1063
......@@ -1,1063 +0,0 @@
1//! A mechanism used to block (`wait`) and unblock (`wake`) threads using a
2//! 32bit memory address as hints.
3//!
4//! Blocking a thread is acknowledged only if the 32bit memory address is equal
5//! to a given value. This check helps avoid block/unblock deadlocks which
6//! occur if a `wake()` happens before a `wait()`.
7//!
8//! Using Futex, other Thread synchronization primitives can be built which
9//! efficiently wait for cross-thread events or signals.
10
11const std = @import("../std.zig");
12const builtin = @import("builtin");
13const Futex = @This();
14const windows = std.os.windows;
15const linux = std.os.linux;
16const c = std.c;
17
18const assert = std.debug.assert;
19const testing = std.testing;
20const atomic = std.atomic;
21
22/// Checks if `ptr` still contains the value `expect` and, if so, blocks the caller until either:
23/// - The value at `ptr` is no longer equal to `expect` and `wake()` is called on the same address.
24/// - The caller is unblocked spuriously ("at random").
25///
26/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically
27/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.
28pub fn wait(ptr: *const atomic.Value(u32), expect: u32) void {
29 @branchHint(.cold);
30
31 Impl.wait(ptr, expect, null) catch |err| switch (err) {
32 error.Timeout => unreachable, // null timeout meant to wait forever
33 };
34}
35
36/// Checks if `ptr` still contains the value `expect` and, if so, blocks the caller until either:
37/// - The value at `ptr` is no longer equal to `expect`.
38/// - The caller is unblocked by a matching `wake()`.
39/// - The caller is unblocked spuriously ("at random").
40/// - The caller blocks for longer than the given timeout. In which case, `error.Timeout` is returned.
41///
42/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically
43/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.
44pub fn timedWait(ptr: *const atomic.Value(u32), expect: u32, timeout_ns: u64) error{Timeout}!void {
45 @branchHint(.cold);
46
47 // Avoid calling into the OS for no-op timeouts.
48 if (timeout_ns == 0) {
49 if (ptr.load(.seq_cst) != expect) return;
50 return error.Timeout;
51 }
52
53 return Impl.wait(ptr, expect, timeout_ns);
54}
55
56/// Unblocks at most `max_waiters` callers blocked in a `wait()` call on `ptr`.
57pub fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
58 @branchHint(.cold);
59
60 // Avoid calling into the OS if there's nothing to wake up.
61 if (max_waiters == 0) {
62 return;
63 }
64
65 Impl.wake(ptr, max_waiters);
66}
67
68const Impl = if (builtin.single_threaded)
69 SingleThreadedImpl
70else if (builtin.os.tag == .windows)
71 WindowsImpl
72else if (builtin.os.tag.isDarwin())
73 DarwinImpl
74else if (builtin.os.tag == .linux)
75 LinuxImpl
76else if (builtin.os.tag == .freebsd)
77 FreebsdImpl
78else if (builtin.os.tag == .openbsd)
79 OpenbsdImpl
80else if (builtin.os.tag == .dragonfly)
81 DragonflyImpl
82else if (builtin.target.cpu.arch.isWasm())
83 WasmImpl
84else if (std.Thread.use_pthreads)
85 PosixImpl
86else
87 UnsupportedImpl;
88
89/// We can't do @compileError() in the `Impl` switch statement above as its eagerly evaluated.
90/// So instead, we @compileError() on the methods themselves for platforms which don't support futex.
91const UnsupportedImpl = struct {
92 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
93 return unsupported(.{ ptr, expect, timeout });
94 }
95
96 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
97 return unsupported(.{ ptr, max_waiters });
98 }
99
100 fn unsupported(unused: anytype) noreturn {
101 _ = unused;
102 @compileError("Unsupported operating system " ++ @tagName(builtin.target.os.tag));
103 }
104};
105
106const SingleThreadedImpl = struct {
107 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
108 if (ptr.raw != expect) {
109 return;
110 }
111
112 // There are no threads to wake us up.
113 // So if we wait without a timeout we would never wake up.
114 const delay = timeout orelse {
115 unreachable; // deadlock detected
116 };
117
118 _ = delay;
119 return error.Timeout;
120 }
121
122 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
123 // There are no other threads to possibly wake up
124 _ = ptr;
125 _ = max_waiters;
126 }
127};
128
129// We use WaitOnAddress through NtDll instead of API-MS-Win-Core-Synch-l1-2-0.dll
130// as it's generally already a linked target and is autoloaded into all processes anyway.
131const WindowsImpl = struct {
132 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
133 var timeout_value: windows.LARGE_INTEGER = undefined;
134 var timeout_ptr: ?*const windows.LARGE_INTEGER = null;
135
136 // NTDLL functions work with time in units of 100 nanoseconds.
137 // Positive values are absolute deadlines while negative values are relative durations.
138 if (timeout) |delay| {
139 timeout_value = @as(windows.LARGE_INTEGER, @intCast(delay / 100));
140 timeout_value = -timeout_value;
141 timeout_ptr = &timeout_value;
142 }
143
144 const rc = windows.ntdll.RtlWaitOnAddress(
145 ptr,
146 &expect,
147 @sizeOf(@TypeOf(expect)),
148 timeout_ptr,
149 );
150
151 switch (rc) {
152 .SUCCESS => {},
153 .TIMEOUT => {
154 assert(timeout != null);
155 return error.Timeout;
156 },
157 else => unreachable,
158 }
159 }
160
161 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
162 const address: ?*const anyopaque = ptr;
163 assert(max_waiters != 0);
164
165 switch (max_waiters) {
166 1 => windows.ntdll.RtlWakeAddressSingle(address),
167 else => windows.ntdll.RtlWakeAddressAll(address),
168 }
169 }
170};
171
172const DarwinImpl = struct {
173 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
174 // Darwin XNU 7195.50.7.100.1 introduced __ulock_wait2 and migrated code paths (notably pthread_cond_t) towards it:
175 // https://github.com/apple/darwin-xnu/commit/d4061fb0260b3ed486147341b72468f836ed6c8f#diff-08f993cc40af475663274687b7c326cc6c3031e0db3ac8de7b24624610616be6
176 //
177 // This XNU version appears to correspond to 11.0.1:
178 // https://kernelshaman.blogspot.com/2021/01/building-xnu-for-macos-big-sur-1101.html
179 //
180 // ulock_wait() uses 32-bit micro-second timeouts where 0 = INFINITE or no-timeout
181 // ulock_wait2() uses 64-bit nano-second timeouts (with the same convention)
182 const supports_ulock_wait2 = builtin.target.os.version_range.semver.min.major >= 11;
183
184 var timeout_ns: u64 = 0;
185 if (timeout) |delay| {
186 assert(delay != 0); // handled by timedWait()
187 timeout_ns = delay;
188 }
189
190 // If we're using `__ulock_wait` and `timeout` is too big to fit inside a `u32` count of
191 // micro-seconds (around 70min), we'll request a shorter timeout. This is fine (users
192 // should handle spurious wakeups), but we need to remember that we did so, so that
193 // we don't return `Timeout` incorrectly. If that happens, we set this variable to
194 // true so that we we know to ignore the ETIMEDOUT result.
195 var timeout_overflowed = false;
196
197 const addr: *const anyopaque = ptr;
198 const flags: c.UL = .{
199 .op = .COMPARE_AND_WAIT,
200 .NO_ERRNO = true,
201 };
202 const status = blk: {
203 if (supports_ulock_wait2) {
204 break :blk c.__ulock_wait2(flags, addr, expect, timeout_ns, 0);
205 }
206
207 const timeout_us = std.math.cast(u32, timeout_ns / std.time.ns_per_us) orelse overflow: {
208 timeout_overflowed = true;
209 break :overflow std.math.maxInt(u32);
210 };
211
212 break :blk c.__ulock_wait(flags, addr, expect, timeout_us);
213 };
214
215 if (status >= 0) return;
216 switch (@as(c.E, @enumFromInt(-status))) {
217 // Wait was interrupted by the OS or other spurious signalling.
218 .INTR => {},
219 // Address of the futex was paged out. This is unlikely, but possible in theory, and
220 // pthread/libdispatch on darwin bother to handle it. In this case we'll return
221 // without waiting, but the caller should retry anyway.
222 .FAULT => {},
223 // Only report Timeout if we didn't have to cap the timeout
224 .TIMEDOUT => {
225 assert(timeout != null);
226 if (!timeout_overflowed) return error.Timeout;
227 },
228 else => unreachable,
229 }
230 }
231
232 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
233 const flags: c.UL = .{
234 .op = .COMPARE_AND_WAIT,
235 .NO_ERRNO = true,
236 .WAKE_ALL = max_waiters > 1,
237 };
238
239 while (true) {
240 const addr: *const anyopaque = ptr;
241 const status = c.__ulock_wake(flags, addr, 0);
242
243 if (status >= 0) return;
244 switch (@as(c.E, @enumFromInt(-status))) {
245 .INTR => continue, // spurious wake()
246 .FAULT => unreachable, // __ulock_wake doesn't generate EFAULT according to darwin pthread_cond_t
247 .NOENT => return, // nothing was woken up
248 .ALREADY => unreachable, // only for UL.Op.WAKE_THREAD
249 else => unreachable,
250 }
251 }
252 }
253};
254
255// https://man7.org/linux/man-pages/man2/futex.2.html
256const LinuxImpl = struct {
257 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
258 var ts: linux.timespec = undefined;
259 if (timeout) |timeout_ns| {
260 ts.sec = @as(@TypeOf(ts.sec), @intCast(timeout_ns / std.time.ns_per_s));
261 ts.nsec = @as(@TypeOf(ts.nsec), @intCast(timeout_ns % std.time.ns_per_s));
262 }
263
264 const rc = linux.futex_4arg(
265 &ptr.raw,
266 .{ .cmd = .WAIT, .private = true },
267 expect,
268 if (timeout != null) &ts else null,
269 );
270
271 switch (linux.errno(rc)) {
272 .SUCCESS => {}, // notified by `wake()`
273 .INTR => {}, // spurious wakeup
274 .AGAIN => {}, // ptr.* != expect
275 .TIMEDOUT => {
276 assert(timeout != null);
277 return error.Timeout;
278 },
279 .INVAL => {}, // possibly timeout overflow
280 .FAULT => unreachable, // ptr was invalid
281 else => unreachable,
282 }
283 }
284
285 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
286 const rc = linux.futex_3arg(
287 &ptr.raw,
288 .{ .cmd = .WAKE, .private = true },
289 @min(max_waiters, std.math.maxInt(i32)),
290 );
291
292 switch (linux.errno(rc)) {
293 .SUCCESS => {}, // successful wake up
294 .INVAL => {}, // invalid futex_wait() on ptr done elsewhere
295 .FAULT => {}, // pointer became invalid while doing the wake
296 else => unreachable,
297 }
298 }
299};
300
301// https://www.freebsd.org/cgi/man.cgi?query=_umtx_op&sektion=2&n=1
302const FreebsdImpl = struct {
303 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
304 var tm_size: usize = 0;
305 var tm: c._umtx_time = undefined;
306 var tm_ptr: ?*const c._umtx_time = null;
307
308 if (timeout) |timeout_ns| {
309 tm_ptr = &tm;
310 tm_size = @sizeOf(@TypeOf(tm));
311
312 tm.flags = 0; // use relative time not UMTX_ABSTIME
313 tm.clockid = .MONOTONIC;
314 tm.timeout.sec = @as(@TypeOf(tm.timeout.sec), @intCast(timeout_ns / std.time.ns_per_s));
315 tm.timeout.nsec = @as(@TypeOf(tm.timeout.nsec), @intCast(timeout_ns % std.time.ns_per_s));
316 }
317
318 const rc = c._umtx_op(
319 @intFromPtr(&ptr.raw),
320 @intFromEnum(c.UMTX_OP.WAIT_UINT_PRIVATE),
321 @as(c_ulong, expect),
322 tm_size,
323 @intFromPtr(tm_ptr),
324 );
325
326 switch (std.posix.errno(rc)) {
327 .SUCCESS => {},
328 .FAULT => unreachable, // one of the args points to invalid memory
329 .INVAL => unreachable, // arguments should be correct
330 .TIMEDOUT => {
331 assert(timeout != null);
332 return error.Timeout;
333 },
334 .INTR => {}, // spurious wake
335 else => unreachable,
336 }
337 }
338
339 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
340 const rc = c._umtx_op(
341 @intFromPtr(&ptr.raw),
342 @intFromEnum(c.UMTX_OP.WAKE_PRIVATE),
343 @as(c_ulong, max_waiters),
344 0, // there is no timeout struct
345 0, // there is no timeout struct pointer
346 );
347
348 switch (std.posix.errno(rc)) {
349 .SUCCESS => {},
350 .FAULT => {}, // it's ok if the ptr doesn't point to valid memory
351 .INVAL => unreachable, // arguments should be correct
352 else => unreachable,
353 }
354 }
355};
356
357// https://man.openbsd.org/futex.2
358const OpenbsdImpl = struct {
359 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
360 var ts: c.timespec = undefined;
361 if (timeout) |timeout_ns| {
362 ts.sec = @as(@TypeOf(ts.sec), @intCast(timeout_ns / std.time.ns_per_s));
363 ts.nsec = @as(@TypeOf(ts.nsec), @intCast(timeout_ns % std.time.ns_per_s));
364 }
365
366 const rc = c.futex(
367 @as(*const volatile u32, @ptrCast(&ptr.raw)),
368 c.FUTEX.WAIT | c.FUTEX.PRIVATE_FLAG,
369 @as(c_int, @bitCast(expect)),
370 if (timeout != null) &ts else null,
371 null, // FUTEX.WAIT takes no requeue address
372 );
373
374 switch (std.posix.errno(rc)) {
375 .SUCCESS => {}, // woken up by wake
376 .NOSYS => unreachable, // the futex operation shouldn't be invalid
377 .FAULT => unreachable, // ptr was invalid
378 .AGAIN => {}, // ptr != expect
379 .INVAL => unreachable, // invalid timeout
380 .TIMEDOUT => {
381 assert(timeout != null);
382 return error.Timeout;
383 },
384 .INTR => {}, // spurious wake from signal
385 .CANCELED => {}, // spurious wake from signal with SA_RESTART
386 else => unreachable,
387 }
388 }
389
390 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
391 const rc = c.futex(
392 @as(*const volatile u32, @ptrCast(&ptr.raw)),
393 c.FUTEX.WAKE | c.FUTEX.PRIVATE_FLAG,
394 std.math.cast(c_int, max_waiters) orelse std.math.maxInt(c_int),
395 null, // FUTEX.WAKE takes no timeout ptr
396 null, // FUTEX.WAKE takes no requeue address
397 );
398
399 // returns number of threads woken up.
400 assert(rc >= 0);
401 }
402};
403
404// https://man.dragonflybsd.org/?command=umtx&section=2
405const DragonflyImpl = struct {
406 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
407 // Dragonfly uses a scheme where 0 timeout means wait until signaled or spurious wake.
408 // It's reporting of timeout's is also unrealiable so we use an external timing source (Timer) instead.
409 var timeout_us: c_int = 0;
410 var timeout_overflowed = false;
411 var sleep_timer: std.time.Timer = undefined;
412
413 if (timeout) |delay| {
414 assert(delay != 0); // handled by timedWait().
415 timeout_us = std.math.cast(c_int, delay / std.time.ns_per_us) orelse blk: {
416 timeout_overflowed = true;
417 break :blk std.math.maxInt(c_int);
418 };
419
420 // Only need to record the start time if we can provide somewhat accurate error.Timeout's
421 if (!timeout_overflowed) {
422 sleep_timer = std.time.Timer.start() catch unreachable;
423 }
424 }
425
426 const value = @as(c_int, @bitCast(expect));
427 const addr = @as(*const volatile c_int, @ptrCast(&ptr.raw));
428 const rc = c.umtx_sleep(addr, value, timeout_us);
429
430 switch (std.posix.errno(rc)) {
431 .SUCCESS => {},
432 .BUSY => {}, // ptr != expect
433 .AGAIN => { // maybe timed out, or paged out, or hit 2s kernel refresh
434 if (timeout) |timeout_ns| {
435 // Report error.Timeout only if we know the timeout duration has passed.
436 // If not, there's not much choice other than treating it as a spurious wake.
437 if (!timeout_overflowed and sleep_timer.read() >= timeout_ns) {
438 return error.Timeout;
439 }
440 }
441 },
442 .INTR => {}, // spurious wake
443 .INVAL => unreachable, // invalid timeout
444 else => unreachable,
445 }
446 }
447
448 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
449 // A count of zero means wake all waiters.
450 assert(max_waiters != 0);
451 const to_wake = std.math.cast(c_int, max_waiters) orelse 0;
452
453 // https://man.dragonflybsd.org/?command=umtx&section=2
454 // > umtx_wakeup() will generally return 0 unless the address is bad.
455 // We are fine with the address being bad (e.g. for Semaphore.post() where Semaphore.wait() frees the Semaphore)
456 const addr = @as(*const volatile c_int, @ptrCast(&ptr.raw));
457 _ = c.umtx_wakeup(addr, to_wake);
458 }
459};
460
461const WasmImpl = struct {
462 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
463 if (!comptime builtin.cpu.has(.wasm, .atomics)) @compileError("WASI target missing cpu feature 'atomics'");
464
465 const to: i64 = if (timeout) |to| @intCast(to) else -1;
466 const result = asm volatile (
467 \\local.get %[ptr]
468 \\local.get %[expected]
469 \\local.get %[timeout]
470 \\memory.atomic.wait32 0
471 \\local.set %[ret]
472 : [ret] "=r" (-> u32),
473 : [ptr] "r" (&ptr.raw),
474 [expected] "r" (@as(i32, @bitCast(expect))),
475 [timeout] "r" (to),
476 );
477 switch (result) {
478 0 => {}, // ok
479 1 => {}, // expected =! loaded
480 2 => return error.Timeout,
481 else => unreachable,
482 }
483 }
484
485 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
486 if (!comptime builtin.cpu.has(.wasm, .atomics)) @compileError("WASI target missing cpu feature 'atomics'");
487
488 assert(max_waiters != 0);
489 const woken_count = asm volatile (
490 \\local.get %[ptr]
491 \\local.get %[waiters]
492 \\memory.atomic.notify 0
493 \\local.set %[ret]
494 : [ret] "=r" (-> u32),
495 : [ptr] "r" (&ptr.raw),
496 [waiters] "r" (max_waiters),
497 );
498 _ = woken_count; // can be 0 when linker flag 'shared-memory' is not enabled
499 }
500};
501
502/// Modified version of linux's futex and Go's sema to implement userspace wait queues with pthread:
503/// https://code.woboq.org/linux/linux/kernel/futex.c.html
504/// https://go.dev/src/runtime/sema.go
505const PosixImpl = struct {
506 const Event = struct {
507 cond: c.pthread_cond_t,
508 mutex: c.pthread_mutex_t,
509 state: enum { empty, waiting, notified },
510
511 fn init(self: *Event) void {
512 // Use static init instead of pthread_cond/mutex_init() since this is generally faster.
513 self.cond = .{};
514 self.mutex = .{};
515 self.state = .empty;
516 }
517
518 fn deinit(self: *Event) void {
519 // Some platforms reportedly give EINVAL for statically initialized pthread types.
520 const rc = c.pthread_cond_destroy(&self.cond);
521 assert(rc == .SUCCESS or rc == .INVAL);
522
523 const rm = c.pthread_mutex_destroy(&self.mutex);
524 assert(rm == .SUCCESS or rm == .INVAL);
525
526 self.* = undefined;
527 }
528
529 fn wait(self: *Event, timeout: ?u64) error{Timeout}!void {
530 assert(c.pthread_mutex_lock(&self.mutex) == .SUCCESS);
531 defer assert(c.pthread_mutex_unlock(&self.mutex) == .SUCCESS);
532
533 // Early return if the event was already set.
534 if (self.state == .notified) {
535 return;
536 }
537
538 // Compute the absolute timeout if one was specified.
539 // POSIX requires that REALTIME is used by default for the pthread timedwait functions.
540 // This can be changed with pthread_condattr_setclock, but it's an extension and may not be available everywhere.
541 var ts: c.timespec = undefined;
542 if (timeout) |timeout_ns| {
543 ts = std.posix.clock_gettime(c.CLOCK.REALTIME) catch unreachable;
544 ts.sec +|= @as(@TypeOf(ts.sec), @intCast(timeout_ns / std.time.ns_per_s));
545 ts.nsec += @as(@TypeOf(ts.nsec), @intCast(timeout_ns % std.time.ns_per_s));
546
547 if (ts.nsec >= std.time.ns_per_s) {
548 ts.sec +|= 1;
549 ts.nsec -= std.time.ns_per_s;
550 }
551 }
552
553 // Start waiting on the event - there can be only one thread waiting.
554 assert(self.state == .empty);
555 self.state = .waiting;
556
557 while (true) {
558 // Block using either pthread_cond_wait or pthread_cond_timewait if there's an absolute timeout.
559 const rc = blk: {
560 if (timeout == null) break :blk c.pthread_cond_wait(&self.cond, &self.mutex);
561 break :blk c.pthread_cond_timedwait(&self.cond, &self.mutex, &ts);
562 };
563
564 // After waking up, check if the event was set.
565 if (self.state == .notified) {
566 return;
567 }
568
569 assert(self.state == .waiting);
570 switch (rc) {
571 .SUCCESS => {},
572 .TIMEDOUT => {
573 // If timed out, reset the event to avoid the set() thread doing an unnecessary signal().
574 self.state = .empty;
575 return error.Timeout;
576 },
577 .INVAL => unreachable, // cond, mutex, and potentially ts should all be valid
578 .PERM => unreachable, // mutex is locked when cond_*wait() functions are called
579 else => unreachable,
580 }
581 }
582 }
583
584 fn set(self: *Event) void {
585 assert(c.pthread_mutex_lock(&self.mutex) == .SUCCESS);
586 defer assert(c.pthread_mutex_unlock(&self.mutex) == .SUCCESS);
587
588 // Make sure that multiple calls to set() were not done on the same Event.
589 const old_state = self.state;
590 assert(old_state != .notified);
591
592 // Mark the event as set and wake up the waiting thread if there was one.
593 // This must be done while the mutex as the wait() thread could deallocate
594 // the condition variable once it observes the new state, potentially causing a UAF if done unlocked.
595 self.state = .notified;
596 if (old_state == .waiting) {
597 assert(c.pthread_cond_signal(&self.cond) == .SUCCESS);
598 }
599 }
600 };
601
602 const Treap = std.Treap(usize, std.math.order);
603 const Waiter = struct {
604 node: Treap.Node,
605 prev: ?*Waiter,
606 next: ?*Waiter,
607 tail: ?*Waiter,
608 is_queued: bool,
609 event: Event,
610 };
611
612 // An unordered set of Waiters
613 const WaitList = struct {
614 top: ?*Waiter = null,
615 len: usize = 0,
616
617 fn push(self: *WaitList, waiter: *Waiter) void {
618 waiter.next = self.top;
619 self.top = waiter;
620 self.len += 1;
621 }
622
623 fn pop(self: *WaitList) ?*Waiter {
624 const waiter = self.top orelse return null;
625 self.top = waiter.next;
626 self.len -= 1;
627 return waiter;
628 }
629 };
630
631 const WaitQueue = struct {
632 fn insert(treap: *Treap, address: usize, waiter: *Waiter) void {
633 // prepare the waiter to be inserted.
634 waiter.next = null;
635 waiter.is_queued = true;
636
637 // Find the wait queue entry associated with the address.
638 // If there isn't a wait queue on the address, this waiter creates the queue.
639 var entry = treap.getEntryFor(address);
640 const entry_node = entry.node orelse {
641 waiter.prev = null;
642 waiter.tail = waiter;
643 entry.set(&waiter.node);
644 return;
645 };
646
647 // There's a wait queue on the address; get the queue head and tail.
648 const head: *Waiter = @fieldParentPtr("node", entry_node);
649 const tail = head.tail orelse unreachable;
650
651 // Push the waiter to the tail by replacing it and linking to the previous tail.
652 head.tail = waiter;
653 tail.next = waiter;
654 waiter.prev = tail;
655 }
656
657 fn remove(treap: *Treap, address: usize, max_waiters: usize) WaitList {
658 // Find the wait queue associated with this address and get the head/tail if any.
659 var entry = treap.getEntryFor(address);
660 var queue_head: ?*Waiter = if (entry.node) |node| @fieldParentPtr("node", node) else null;
661 const queue_tail = if (queue_head) |head| head.tail else null;
662
663 // Once we're done updating the head, fix it's tail pointer and update the treap's queue head as well.
664 defer entry.set(blk: {
665 const new_head = queue_head orelse break :blk null;
666 new_head.tail = queue_tail;
667 break :blk &new_head.node;
668 });
669
670 var removed = WaitList{};
671 while (removed.len < max_waiters) {
672 // dequeue and collect waiters from their wait queue.
673 const waiter = queue_head orelse break;
674 queue_head = waiter.next;
675 removed.push(waiter);
676
677 // When dequeueing, we must mark is_queued as false.
678 // This ensures that a waiter which calls tryRemove() returns false.
679 assert(waiter.is_queued);
680 waiter.is_queued = false;
681 }
682
683 return removed;
684 }
685
686 fn tryRemove(treap: *Treap, address: usize, waiter: *Waiter) bool {
687 if (!waiter.is_queued) {
688 return false;
689 }
690
691 queue_remove: {
692 // Find the wait queue associated with the address.
693 var entry = blk: {
694 // A waiter without a previous link means it's the queue head that's in the treap so we can avoid lookup.
695 if (waiter.prev == null) {
696 assert(waiter.node.key == address);
697 break :blk treap.getEntryForExisting(&waiter.node);
698 }
699 break :blk treap.getEntryFor(address);
700 };
701
702 // The queue head and tail must exist if we're removing a queued waiter.
703 const head: *Waiter = @fieldParentPtr("node", entry.node orelse unreachable);
704 const tail = head.tail orelse unreachable;
705
706 // A waiter with a previous link is never the head of the queue.
707 if (waiter.prev) |prev| {
708 assert(waiter != head);
709 prev.next = waiter.next;
710
711 // A waiter with both a previous and next link is in the middle.
712 // We only need to update the surrounding waiter's links to remove it.
713 if (waiter.next) |next| {
714 assert(waiter != tail);
715 next.prev = waiter.prev;
716 break :queue_remove;
717 }
718
719 // A waiter with a previous but no next link means it's the tail of the queue.
720 // In that case, we need to update the head's tail reference.
721 assert(waiter == tail);
722 head.tail = waiter.prev;
723 break :queue_remove;
724 }
725
726 // A waiter with no previous link means it's the queue head of queue.
727 // We must replace (or remove) the head waiter reference in the treap.
728 assert(waiter == head);
729 entry.set(blk: {
730 const new_head = waiter.next orelse break :blk null;
731 new_head.tail = head.tail;
732 break :blk &new_head.node;
733 });
734 }
735
736 // Mark the waiter as successfully removed.
737 waiter.is_queued = false;
738 return true;
739 }
740 };
741
742 const Bucket = struct {
743 mutex: c.pthread_mutex_t align(atomic.cache_line) = .{},
744 pending: atomic.Value(usize) = atomic.Value(usize).init(0),
745 treap: Treap = .{},
746
747 // Global array of buckets that addresses map to.
748 // Bucket array size is pretty much arbitrary here, but it must be a power of two for fibonacci hashing.
749 var buckets = [_]Bucket{.{}} ** @bitSizeOf(usize);
750
751 // https://github.com/Amanieu/parking_lot/blob/1cf12744d097233316afa6c8b7d37389e4211756/core/src/parking_lot.rs#L343-L353
752 fn from(address: usize) *Bucket {
753 // The upper `@bitSizeOf(usize)` bits of the fibonacci golden ratio.
754 // Hashing this via (h * k) >> (64 - b) where k=golden-ration and b=bitsize-of-array
755 // evenly lays out h=hash values over the bit range even when the hash has poor entropy (identity-hash for pointers).
756 const max_multiplier_bits = @bitSizeOf(usize);
757 const fibonacci_multiplier = 0x9E3779B97F4A7C15 >> (64 - max_multiplier_bits);
758
759 const max_bucket_bits = @ctz(buckets.len);
760 comptime assert(std.math.isPowerOfTwo(buckets.len));
761
762 const index = (address *% fibonacci_multiplier) >> (max_multiplier_bits - max_bucket_bits);
763 return &buckets[index];
764 }
765 };
766
767 const Address = struct {
768 fn from(ptr: *const atomic.Value(u32)) usize {
769 // Get the alignment of the pointer.
770 const alignment = @alignOf(atomic.Value(u32));
771 comptime assert(std.math.isPowerOfTwo(alignment));
772
773 // Make sure the pointer is aligned,
774 // then cut off the zero bits from the alignment to get the unique address.
775 const addr = @intFromPtr(ptr);
776 assert(addr & (alignment - 1) == 0);
777 return addr >> @ctz(@as(usize, alignment));
778 }
779 };
780
781 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
782 const address = Address.from(ptr);
783 const bucket = Bucket.from(address);
784
785 // Announce that there's a waiter in the bucket before checking the ptr/expect condition.
786 // If the announcement is reordered after the ptr check, the waiter could deadlock:
787 //
788 // - T1: checks ptr == expect which is true
789 // - T2: updates ptr to != expect
790 // - T2: does Futex.wake(), sees no pending waiters, exits
791 // - T1: bumps pending waiters (was reordered after the ptr == expect check)
792 // - T1: goes to sleep and misses both the ptr change and T2's wake up
793 //
794 // acquire barrier to ensure the announcement happens before the ptr check below.
795 var pending = bucket.pending.fetchAdd(1, .acquire);
796 assert(pending < std.math.maxInt(usize));
797
798 // If the wait gets canceled, remove the pending count we previously added.
799 // This is done outside the mutex lock to keep the critical section short in case of contention.
800 var canceled = false;
801 defer if (canceled) {
802 pending = bucket.pending.fetchSub(1, .monotonic);
803 assert(pending > 0);
804 };
805
806 var waiter: Waiter = undefined;
807 {
808 assert(c.pthread_mutex_lock(&bucket.mutex) == .SUCCESS);
809 defer assert(c.pthread_mutex_unlock(&bucket.mutex) == .SUCCESS);
810
811 canceled = ptr.load(.monotonic) != expect;
812 if (canceled) {
813 return;
814 }
815
816 waiter.event.init();
817 WaitQueue.insert(&bucket.treap, address, &waiter);
818 }
819
820 defer {
821 assert(!waiter.is_queued);
822 waiter.event.deinit();
823 }
824
825 waiter.event.wait(timeout) catch {
826 // If we fail to cancel after a timeout, it means a wake() thread dequeued us and will wake us up.
827 // We must wait until the event is set as that's a signal that the wake() thread won't access the waiter memory anymore.
828 // If we return early without waiting, the waiter on the stack would be invalidated and the wake() thread risks a UAF.
829 defer if (!canceled) waiter.event.wait(null) catch unreachable;
830
831 assert(c.pthread_mutex_lock(&bucket.mutex) == .SUCCESS);
832 defer assert(c.pthread_mutex_unlock(&bucket.mutex) == .SUCCESS);
833
834 canceled = WaitQueue.tryRemove(&bucket.treap, address, &waiter);
835 if (canceled) {
836 return error.Timeout;
837 }
838 };
839 }
840
841 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
842 const address = Address.from(ptr);
843 const bucket = Bucket.from(address);
844
845 // Quick check if there's even anything to wake up.
846 // The change to the ptr's value must happen before we check for pending waiters.
847 // If not, the wake() thread could miss a sleeping waiter and have it deadlock:
848 //
849 // - T2: p = has pending waiters (reordered before the ptr update)
850 // - T1: bump pending waiters
851 // - T1: if ptr == expected: sleep()
852 // - T2: update ptr != expected
853 // - T2: p is false from earlier so doesn't wake (T1 missed ptr update and T2 missed T1 sleeping)
854 //
855 // What we really want here is a Release load, but that doesn't exist under the C11 memory model.
856 // We could instead do `bucket.pending.fetchAdd(0, Release) == 0` which achieves effectively the same thing,
857 // LLVM lowers the fetchAdd(0, .release) into an mfence+load which avoids gaining ownership of the cache-line.
858 if (bucket.pending.fetchAdd(0, .release) == 0) {
859 return;
860 }
861
862 // Keep a list of all the waiters notified and wake then up outside the mutex critical section.
863 var notified = WaitList{};
864 defer if (notified.len > 0) {
865 const pending = bucket.pending.fetchSub(notified.len, .monotonic);
866 assert(pending >= notified.len);
867
868 while (notified.pop()) |waiter| {
869 assert(!waiter.is_queued);
870 waiter.event.set();
871 }
872 };
873
874 assert(c.pthread_mutex_lock(&bucket.mutex) == .SUCCESS);
875 defer assert(c.pthread_mutex_unlock(&bucket.mutex) == .SUCCESS);
876
877 // Another pending check again to avoid the WaitQueue lookup if not necessary.
878 if (bucket.pending.load(.monotonic) > 0) {
879 notified = WaitQueue.remove(&bucket.treap, address, max_waiters);
880 }
881 }
882};
883
884test "smoke test" {
885 var value = atomic.Value(u32).init(0);
886
887 // Try waits with invalid values.
888 Futex.wait(&value, 0xdeadbeef);
889 Futex.timedWait(&value, 0xdeadbeef, 0) catch {};
890
891 // Try timeout waits.
892 try testing.expectError(error.Timeout, Futex.timedWait(&value, 0, 0));
893 try testing.expectError(error.Timeout, Futex.timedWait(&value, 0, std.time.ns_per_ms));
894
895 // Try wakes
896 Futex.wake(&value, 0);
897 Futex.wake(&value, 1);
898 Futex.wake(&value, std.math.maxInt(u32));
899}
900
901test "signaling" {
902 // This test requires spawning threads
903 if (builtin.single_threaded) {
904 return error.SkipZigTest;
905 }
906
907 const num_threads = 4;
908 const num_iterations = 4;
909
910 const Paddle = struct {
911 value: atomic.Value(u32) = atomic.Value(u32).init(0),
912 current: u32 = 0,
913
914 fn hit(self: *@This()) void {
915 _ = self.value.fetchAdd(1, .release);
916 Futex.wake(&self.value, 1);
917 }
918
919 fn run(self: *@This(), hit_to: *@This()) !void {
920 while (self.current < num_iterations) {
921 // Wait for the value to change from hit()
922 var new_value: u32 = undefined;
923 while (true) {
924 new_value = self.value.load(.acquire);
925 if (new_value != self.current) break;
926 Futex.wait(&self.value, self.current);
927 }
928
929 // change the internal "current" value
930 try testing.expectEqual(new_value, self.current + 1);
931 self.current = new_value;
932
933 // hit the next paddle
934 hit_to.hit();
935 }
936 }
937 };
938
939 var paddles = [_]Paddle{.{}} ** num_threads;
940 var threads = [_]std.Thread{undefined} ** num_threads;
941
942 // Create a circle of paddles which hit each other
943 for (&threads, 0..) |*t, i| {
944 const paddle = &paddles[i];
945 const hit_to = &paddles[(i + 1) % paddles.len];
946 t.* = try std.Thread.spawn(.{}, Paddle.run, .{ paddle, hit_to });
947 }
948
949 // Hit the first paddle and wait for them all to complete by hitting each other for num_iterations.
950 paddles[0].hit();
951 for (threads) |t| t.join();
952 for (paddles) |p| try testing.expectEqual(p.current, num_iterations);
953}
954
955test "broadcasting" {
956 // This test requires spawning threads
957 if (builtin.single_threaded) {
958 return error.SkipZigTest;
959 }
960
961 const num_threads = 4;
962 const num_iterations = 4;
963
964 const Barrier = struct {
965 count: atomic.Value(u32) = atomic.Value(u32).init(num_threads),
966 futex: atomic.Value(u32) = atomic.Value(u32).init(0),
967
968 fn wait(self: *@This()) !void {
969 // Decrement the counter.
970 // Release ensures stuff before this barrier.wait() happens before the last one.
971 // Acquire for the last counter ensures stuff before previous barrier.wait()s happened before it.
972 const count = self.count.fetchSub(1, .acq_rel);
973 try testing.expect(count <= num_threads);
974 try testing.expect(count > 0);
975
976 // First counter to reach zero wakes all other threads.
977 // Release on futex update ensures stuff before all barrier.wait()'s happens before they all return.
978 if (count - 1 == 0) {
979 self.futex.store(1, .release);
980 Futex.wake(&self.futex, num_threads - 1);
981 return;
982 }
983
984 // Other threads wait until last counter wakes them up.
985 // Acquire on futex synchronizes with last barrier count to ensure stuff before all barrier.wait()'s happen before us.
986 while (self.futex.load(.acquire) == 0) {
987 Futex.wait(&self.futex, 0);
988 }
989 }
990 };
991
992 const Broadcast = struct {
993 barriers: [num_iterations]Barrier = [_]Barrier{.{}} ** num_iterations,
994 threads: [num_threads]std.Thread = undefined,
995
996 fn run(self: *@This()) !void {
997 for (&self.barriers) |*barrier| {
998 try barrier.wait();
999 }
1000 }
1001 };
1002
1003 var broadcast = Broadcast{};
1004 for (&broadcast.threads) |*t| t.* = try std.Thread.spawn(.{}, Broadcast.run, .{&broadcast});
1005 for (broadcast.threads) |t| t.join();
1006}
1007
1008/// Deadline is used to wait efficiently for a pointer's value to change using Futex and a fixed timeout.
1009///
1010/// Futex's timedWait() api uses a relative duration which suffers from over-waiting
1011/// when used in a loop which is often required due to the possibility of spurious wakeups.
1012///
1013/// Deadline instead converts the relative timeout to an absolute one so that multiple calls
1014/// to Futex timedWait() can block for and report more accurate error.Timeouts.
1015pub const Deadline = struct {
1016 timeout: ?u64,
1017 started: std.time.Timer,
1018
1019 /// Create the deadline to expire after the given amount of time in nanoseconds passes.
1020 /// Pass in `null` to have the deadline call `Futex.wait()` and never expire.
1021 pub fn init(expires_in_ns: ?u64) Deadline {
1022 var deadline: Deadline = undefined;
1023 deadline.timeout = expires_in_ns;
1024
1025 // std.time.Timer is required to be supported for somewhat accurate reportings of error.Timeout.
1026 if (deadline.timeout != null) {
1027 deadline.started = std.time.Timer.start() catch unreachable;
1028 }
1029
1030 return deadline;
1031 }
1032
1033 /// Wait until either:
1034 /// - the `ptr`'s value changes from `expect`.
1035 /// - `Futex.wake()` is called on the `ptr`.
1036 /// - A spurious wake occurs.
1037 /// - The deadline expires; In which case `error.Timeout` is returned.
1038 pub fn wait(self: *Deadline, ptr: *const atomic.Value(u32), expect: u32) error{Timeout}!void {
1039 @branchHint(.cold);
1040
1041 // Check if we actually have a timeout to wait until.
1042 // If not just wait "forever".
1043 const timeout_ns = self.timeout orelse {
1044 return Futex.wait(ptr, expect);
1045 };
1046
1047 // Get how much time has passed since we started waiting
1048 // then subtract that from the init() timeout to get how much longer to wait.
1049 // Use overflow to detect when we've been waiting longer than the init() timeout.
1050 const elapsed_ns = self.started.read();
1051 const until_timeout_ns = std.math.sub(u64, timeout_ns, elapsed_ns) catch 0;
1052 return Futex.timedWait(ptr, expect, until_timeout_ns);
1053 }
1054};
1055
1056test "Deadline" {
1057 var deadline = Deadline.init(100 * std.time.ns_per_ms);
1058 var futex_word = atomic.Value(u32).init(0);
1059
1060 while (true) {
1061 deadline.wait(&futex_word, 0) catch break;
1062 }
1063}
lib/std/Thread/Mutex.zig deleted-367
......@@ -1,367 +0,0 @@
1//! Mutex is a synchronization primitive which enforces atomic access to a
2//! shared region of code known as the "critical section".
3//!
4//! It does this by blocking ensuring only one thread is in the critical
5//! section at any given point in time by blocking the others.
6//!
7//! Mutex can be statically initialized and is at most `@sizeOf(u64)` large.
8//! Use `lock()` or `tryLock()` to enter the critical section and `unlock()` to leave it.
9
10const std = @import("../std.zig");
11const builtin = @import("builtin");
12const Mutex = @This();
13
14const assert = std.debug.assert;
15const testing = std.testing;
16const Thread = std.Thread;
17const Futex = Thread.Futex;
18
19impl: Impl = .{},
20
21pub const Recursive = @import("Mutex/Recursive.zig");
22
23/// Tries to acquire the mutex without blocking the caller's thread.
24/// Returns `false` if the calling thread would have to block to acquire it.
25/// Otherwise, returns `true` and the caller should `unlock()` the Mutex to release it.
26pub fn tryLock(self: *Mutex) bool {
27 return self.impl.tryLock();
28}
29
30/// Acquires the mutex, blocking the caller's thread until it can.
31/// It is undefined behavior if the mutex is already held by the caller's thread.
32/// Once acquired, call `unlock()` on the Mutex to release it.
33pub fn lock(self: *Mutex) void {
34 self.impl.lock();
35}
36
37/// Releases the mutex which was previously acquired with `lock()` or `tryLock()`.
38/// It is undefined behavior if the mutex is unlocked from a different thread that it was locked from.
39pub fn unlock(self: *Mutex) void {
40 self.impl.unlock();
41}
42
43const Impl = if (builtin.mode == .Debug and !builtin.single_threaded)
44 DebugImpl
45else
46 ReleaseImpl;
47
48const ReleaseImpl = Impl: {
49 if (builtin.single_threaded) break :Impl SingleThreadedImpl;
50 if (builtin.os.tag == .windows) break :Impl WindowsImpl;
51 if (builtin.os.tag.isDarwin()) break :Impl DarwinImpl;
52
53 if (builtin.target.os.tag == .linux or
54 builtin.target.os.tag == .freebsd or
55 builtin.target.os.tag == .openbsd or
56 builtin.target.os.tag == .dragonfly or
57 builtin.target.cpu.arch.isWasm())
58 {
59 // Futex is the system's synchronization primitive; use that.
60 break :Impl FutexImpl;
61 }
62
63 if (std.Thread.use_pthreads) {
64 // This system doesn't have a futex primitive, so `std.Thread.Futex` is using `PosixImpl`,
65 // which implements futex *on top of* pthread mutexes and conditions. Therefore, instead
66 // of going through that long inefficient path, just use pthread mutex directly.
67 break :Impl PosixImpl;
68 }
69
70 break :Impl FutexImpl;
71};
72
73const DebugImpl = struct {
74 locking_thread: std.atomic.Value(Thread.Id) = std.atomic.Value(Thread.Id).init(0), // 0 means it's not locked.
75 impl: ReleaseImpl = .{},
76
77 inline fn tryLock(self: *@This()) bool {
78 const locking = self.impl.tryLock();
79 if (locking) {
80 self.locking_thread.store(Thread.getCurrentId(), .unordered);
81 }
82 return locking;
83 }
84
85 inline fn lock(self: *@This()) void {
86 const current_id = Thread.getCurrentId();
87 if (self.locking_thread.load(.unordered) == current_id and current_id != 0) {
88 @panic("Deadlock detected");
89 }
90 self.impl.lock();
91 self.locking_thread.store(current_id, .unordered);
92 }
93
94 inline fn unlock(self: *@This()) void {
95 assert(self.locking_thread.load(.unordered) == Thread.getCurrentId());
96 self.locking_thread.store(0, .unordered);
97 self.impl.unlock();
98 }
99};
100
101const SingleThreadedImpl = struct {
102 is_locked: bool = false,
103
104 fn tryLock(self: *@This()) bool {
105 if (self.is_locked) return false;
106 self.is_locked = true;
107 return true;
108 }
109
110 fn lock(self: *@This()) void {
111 if (!self.tryLock()) {
112 unreachable; // deadlock detected
113 }
114 }
115
116 fn unlock(self: *@This()) void {
117 assert(self.is_locked);
118 self.is_locked = false;
119 }
120};
121
122/// SRWLOCK on windows is almost always faster than Futex solution.
123/// It also implements an efficient Condition with requeue support for us.
124const WindowsImpl = struct {
125 srwlock: windows.SRWLOCK = .{},
126
127 fn tryLock(self: *@This()) bool {
128 return windows.ntdll.RtlTryAcquireSRWLockExclusive(&self.srwlock) != windows.FALSE;
129 }
130
131 fn lock(self: *@This()) void {
132 windows.ntdll.RtlAcquireSRWLockExclusive(&self.srwlock);
133 }
134
135 fn unlock(self: *@This()) void {
136 windows.ntdll.RtlReleaseSRWLockExclusive(&self.srwlock);
137 }
138
139 const windows = std.os.windows;
140};
141
142/// os_unfair_lock on darwin supports priority inheritance and is generally faster than Futex solutions.
143const DarwinImpl = struct {
144 oul: c.os_unfair_lock = .{},
145
146 fn tryLock(self: *@This()) bool {
147 return c.os_unfair_lock_trylock(&self.oul);
148 }
149
150 fn lock(self: *@This()) void {
151 c.os_unfair_lock_lock(&self.oul);
152 }
153
154 fn unlock(self: *@This()) void {
155 c.os_unfair_lock_unlock(&self.oul);
156 }
157
158 const c = std.c;
159};
160
161const FutexImpl = struct {
162 state: std.atomic.Value(u32) = std.atomic.Value(u32).init(unlocked),
163
164 const unlocked: u32 = 0b00;
165 const locked: u32 = 0b01;
166 const contended: u32 = 0b11; // must contain the `locked` bit for x86 optimization below
167
168 fn lock(self: *@This()) void {
169 if (!self.tryLock())
170 self.lockSlow();
171 }
172
173 fn tryLock(self: *@This()) bool {
174 // On x86, use `lock bts` instead of `lock cmpxchg` as:
175 // - they both seem to mark the cache-line as modified regardless: https://stackoverflow.com/a/63350048
176 // - `lock bts` is smaller instruction-wise which makes it better for inlining
177 if (builtin.target.cpu.arch.isX86()) {
178 const locked_bit = @ctz(locked);
179 return self.state.bitSet(locked_bit, .acquire) == 0;
180 }
181
182 // Acquire barrier ensures grabbing the lock happens before the critical section
183 // and that the previous lock holder's critical section happens before we grab the lock.
184 return self.state.cmpxchgWeak(unlocked, locked, .acquire, .monotonic) == null;
185 }
186
187 fn lockSlow(self: *@This()) void {
188 @branchHint(.cold);
189
190 // Avoid doing an atomic swap below if we already know the state is contended.
191 // An atomic swap unconditionally stores which marks the cache-line as modified unnecessarily.
192 if (self.state.load(.monotonic) == contended) {
193 Futex.wait(&self.state, contended);
194 }
195
196 // Try to acquire the lock while also telling the existing lock holder that there are threads waiting.
197 //
198 // Once we sleep on the Futex, we must acquire the mutex using `contended` rather than `locked`.
199 // If not, threads sleeping on the Futex wouldn't see the state change in unlock and potentially deadlock.
200 // The downside is that the last mutex unlocker will see `contended` and do an unnecessary Futex wake
201 // but this is better than having to wake all waiting threads on mutex unlock.
202 //
203 // Acquire barrier ensures grabbing the lock happens before the critical section
204 // and that the previous lock holder's critical section happens before we grab the lock.
205 while (self.state.swap(contended, .acquire) != unlocked) {
206 Futex.wait(&self.state, contended);
207 }
208 }
209
210 fn unlock(self: *@This()) void {
211 // Unlock the mutex and wake up a waiting thread if any.
212 //
213 // A waiting thread will acquire with `contended` instead of `locked`
214 // which ensures that it wakes up another thread on the next unlock().
215 //
216 // Release barrier ensures the critical section happens before we let go of the lock
217 // and that our critical section happens before the next lock holder grabs the lock.
218 const state = self.state.swap(unlocked, .release);
219 assert(state != unlocked);
220
221 if (state == contended) {
222 Futex.wake(&self.state, 1);
223 }
224 }
225};
226
227const PosixImpl = struct {
228 mutex: std.c.pthread_mutex_t = .{},
229
230 fn tryLock(impl: *PosixImpl) bool {
231 switch (std.c.pthread_mutex_trylock(&impl.mutex)) {
232 .SUCCESS => return true,
233 .BUSY => return false,
234 .INVAL => unreachable, // mutex is initialized correctly
235 else => unreachable,
236 }
237 }
238
239 fn lock(impl: *PosixImpl) void {
240 switch (std.c.pthread_mutex_lock(&impl.mutex)) {
241 .SUCCESS => return,
242 .INVAL => unreachable, // mutex is initialized correctly
243 .DEADLK => unreachable, // not an error checking mutex
244 else => unreachable,
245 }
246 }
247
248 fn unlock(impl: *PosixImpl) void {
249 switch (std.c.pthread_mutex_unlock(&impl.mutex)) {
250 .SUCCESS => return,
251 .INVAL => unreachable, // mutex is initialized correctly
252 .PERM => unreachable, // not an error checking mutex
253 else => unreachable,
254 }
255 }
256};
257
258test "smoke test" {
259 var mutex = Mutex{};
260
261 try testing.expect(mutex.tryLock());
262 try testing.expect(!mutex.tryLock());
263 mutex.unlock();
264
265 mutex.lock();
266 try testing.expect(!mutex.tryLock());
267 mutex.unlock();
268}
269
270// A counter which is incremented without atomic instructions
271const NonAtomicCounter = struct {
272 // direct u128 could maybe use xmm ops on x86 which are atomic
273 value: [2]u64 = [_]u64{ 0, 0 },
274
275 fn get(self: NonAtomicCounter) u128 {
276 return @as(u128, @bitCast(self.value));
277 }
278
279 fn inc(self: *NonAtomicCounter) void {
280 for (@as([2]u64, @bitCast(self.get() + 1)), 0..) |v, i| {
281 @as(*volatile u64, @ptrCast(&self.value[i])).* = v;
282 }
283 }
284};
285
286test "many uncontended" {
287 // This test requires spawning threads.
288 if (builtin.single_threaded) {
289 return error.SkipZigTest;
290 }
291
292 const num_threads = 4;
293 const num_increments = 1000;
294
295 const Runner = struct {
296 mutex: Mutex = .{},
297 thread: Thread = undefined,
298 counter: NonAtomicCounter = .{},
299
300 fn run(self: *@This()) void {
301 var i: usize = num_increments;
302 while (i > 0) : (i -= 1) {
303 self.mutex.lock();
304 defer self.mutex.unlock();
305
306 self.counter.inc();
307 }
308 }
309 };
310
311 var runners = [_]Runner{.{}} ** num_threads;
312 for (&runners) |*r| r.thread = try Thread.spawn(.{}, Runner.run, .{r});
313 for (runners) |r| r.thread.join();
314 for (runners) |r| try testing.expectEqual(r.counter.get(), num_increments);
315}
316
317test "many contended" {
318 // This test requires spawning threads.
319 if (builtin.single_threaded) {
320 return error.SkipZigTest;
321 }
322
323 const num_threads = 4;
324 const num_increments = 1000;
325
326 const Runner = struct {
327 mutex: Mutex = .{},
328 counter: NonAtomicCounter = .{},
329
330 fn run(self: *@This()) void {
331 var i: usize = num_increments;
332 while (i > 0) : (i -= 1) {
333 // Occasionally hint to let another thread run.
334 defer if (i % 100 == 0) Thread.yield() catch {};
335
336 self.mutex.lock();
337 defer self.mutex.unlock();
338
339 self.counter.inc();
340 }
341 }
342 };
343
344 var runner = Runner{};
345
346 var threads: [num_threads]Thread = undefined;
347 for (&threads) |*t| t.* = try Thread.spawn(.{}, Runner.run, .{&runner});
348 for (threads) |t| t.join();
349
350 try testing.expectEqual(runner.counter.get(), num_increments * num_threads);
351}
352
353// https://github.com/ziglang/zig/issues/19295
354//test @This() {
355// var m: Mutex = .{};
356//
357// {
358// m.lock();
359// defer m.unlock();
360// // ... critical section code
361// }
362//
363// if (m.tryLock()) {
364// defer m.unlock();
365// // ... critical section code
366// }
367//}
lib/std/Thread/Mutex/Recursive.zig+6-6
......@@ -7,18 +7,18 @@
77//! A recursive mutex is an abstraction layer on top of a regular mutex;
88//! therefore it is recommended to use instead `std.Mutex` unless there is a
99//! specific reason a recursive mutex is warranted.
10const Recursive = @This();
1011
1112const std = @import("../../std.zig");
12const Recursive = @This();
13const Mutex = std.Thread.Mutex;
13const Io = std.Io;
1414const assert = std.debug.assert;
1515
16mutex: Mutex,
16mutex: Io.Mutex,
1717thread_id: std.Thread.Id,
1818lock_count: usize,
1919
2020pub const init: Recursive = .{
21 .mutex = .{},
21 .mutex = .init,
2222 .thread_id = invalid_thread_id,
2323 .lock_count = 0,
2424};
......@@ -49,7 +49,7 @@ pub fn tryLock(r: *Recursive) bool {
4949pub fn lock(r: *Recursive) void {
5050 const current_thread_id = std.Thread.getCurrentId();
5151 if (@atomicLoad(std.Thread.Id, &r.thread_id, .unordered) != current_thread_id) {
52 r.mutex.lock();
52 Io.Threaded.mutexLock(&r.mutex);
5353 assert(r.lock_count == 0);
5454 @atomicStore(std.Thread.Id, &r.thread_id, current_thread_id, .unordered);
5555 }
......@@ -64,7 +64,7 @@ pub fn unlock(r: *Recursive) void {
6464 r.lock_count -= 1;
6565 if (r.lock_count == 0) {
6666 @atomicStore(std.Thread.Id, &r.thread_id, invalid_thread_id, .unordered);
67 r.mutex.unlock();
67 Io.Threaded.mutexUnlock(&r.mutex);
6868 }
6969}
7070
lib/std/Thread/RwLock.zig deleted-386
......@@ -1,386 +0,0 @@
1//! A lock that supports one writer or many readers.
2//! This API is for kernel threads, not evented I/O.
3//! This API requires being initialized at runtime, and initialization
4//! can fail. Once initialized, the core operations cannot fail.
5
6impl: Impl = .{},
7
8const RwLock = @This();
9const std = @import("../std.zig");
10const builtin = @import("builtin");
11const assert = std.debug.assert;
12const testing = std.testing;
13
14pub const Impl = if (builtin.single_threaded)
15 SingleThreadedRwLock
16else if (std.Thread.use_pthreads)
17 PthreadRwLock
18else
19 DefaultRwLock;
20
21/// Attempts to obtain exclusive lock ownership.
22/// Returns `true` if the lock is obtained, `false` otherwise.
23pub fn tryLock(rwl: *RwLock) bool {
24 return rwl.impl.tryLock();
25}
26
27/// Blocks until exclusive lock ownership is acquired.
28pub fn lock(rwl: *RwLock) void {
29 return rwl.impl.lock();
30}
31
32/// Releases a held exclusive lock.
33/// Asserts the lock is held exclusively.
34pub fn unlock(rwl: *RwLock) void {
35 return rwl.impl.unlock();
36}
37
38/// Attempts to obtain shared lock ownership.
39/// Returns `true` if the lock is obtained, `false` otherwise.
40pub fn tryLockShared(rwl: *RwLock) bool {
41 return rwl.impl.tryLockShared();
42}
43
44/// Obtains shared lock ownership.
45/// Blocks if another thread has exclusive ownership.
46/// May block if another thread is attempting to get exclusive ownership.
47pub fn lockShared(rwl: *RwLock) void {
48 return rwl.impl.lockShared();
49}
50
51/// Releases a held shared lock.
52pub fn unlockShared(rwl: *RwLock) void {
53 return rwl.impl.unlockShared();
54}
55
56/// Single-threaded applications use this for deadlock checks in
57/// debug mode, and no-ops in release modes.
58pub const SingleThreadedRwLock = struct {
59 state: enum { unlocked, locked_exclusive, locked_shared } = .unlocked,
60 shared_count: usize = 0,
61
62 /// Attempts to obtain exclusive lock ownership.
63 /// Returns `true` if the lock is obtained, `false` otherwise.
64 pub fn tryLock(rwl: *SingleThreadedRwLock) bool {
65 switch (rwl.state) {
66 .unlocked => {
67 assert(rwl.shared_count == 0);
68 rwl.state = .locked_exclusive;
69 return true;
70 },
71 .locked_exclusive, .locked_shared => return false,
72 }
73 }
74
75 /// Blocks until exclusive lock ownership is acquired.
76 pub fn lock(rwl: *SingleThreadedRwLock) void {
77 assert(rwl.state == .unlocked); // deadlock detected
78 assert(rwl.shared_count == 0); // corrupted state detected
79 rwl.state = .locked_exclusive;
80 }
81
82 /// Releases a held exclusive lock.
83 /// Asserts the lock is held exclusively.
84 pub fn unlock(rwl: *SingleThreadedRwLock) void {
85 assert(rwl.state == .locked_exclusive);
86 assert(rwl.shared_count == 0); // corrupted state detected
87 rwl.state = .unlocked;
88 }
89
90 /// Attempts to obtain shared lock ownership.
91 /// Returns `true` if the lock is obtained, `false` otherwise.
92 pub fn tryLockShared(rwl: *SingleThreadedRwLock) bool {
93 switch (rwl.state) {
94 .unlocked => {
95 rwl.state = .locked_shared;
96 assert(rwl.shared_count == 0);
97 rwl.shared_count = 1;
98 return true;
99 },
100 .locked_shared => {
101 rwl.shared_count += 1;
102 return true;
103 },
104 .locked_exclusive => return false,
105 }
106 }
107
108 /// Blocks until shared lock ownership is acquired.
109 pub fn lockShared(rwl: *SingleThreadedRwLock) void {
110 switch (rwl.state) {
111 .unlocked => {
112 rwl.state = .locked_shared;
113 assert(rwl.shared_count == 0);
114 rwl.shared_count = 1;
115 },
116 .locked_shared => {
117 rwl.shared_count += 1;
118 },
119 .locked_exclusive => unreachable, // deadlock detected
120 }
121 }
122
123 /// Releases a held shared lock.
124 pub fn unlockShared(rwl: *SingleThreadedRwLock) void {
125 switch (rwl.state) {
126 .unlocked => unreachable, // too many calls to `unlockShared`
127 .locked_exclusive => unreachable, // exclusively held lock
128 .locked_shared => {
129 rwl.shared_count -= 1;
130 if (rwl.shared_count == 0) {
131 rwl.state = .unlocked;
132 }
133 },
134 }
135 }
136};
137
138pub const PthreadRwLock = struct {
139 rwlock: std.c.pthread_rwlock_t = .{},
140
141 pub fn tryLock(rwl: *PthreadRwLock) bool {
142 return std.c.pthread_rwlock_trywrlock(&rwl.rwlock) == .SUCCESS;
143 }
144
145 pub fn lock(rwl: *PthreadRwLock) void {
146 const rc = std.c.pthread_rwlock_wrlock(&rwl.rwlock);
147 assert(rc == .SUCCESS);
148 }
149
150 pub fn unlock(rwl: *PthreadRwLock) void {
151 const rc = std.c.pthread_rwlock_unlock(&rwl.rwlock);
152 assert(rc == .SUCCESS);
153 }
154
155 pub fn tryLockShared(rwl: *PthreadRwLock) bool {
156 return std.c.pthread_rwlock_tryrdlock(&rwl.rwlock) == .SUCCESS;
157 }
158
159 pub fn lockShared(rwl: *PthreadRwLock) void {
160 const rc = std.c.pthread_rwlock_rdlock(&rwl.rwlock);
161 assert(rc == .SUCCESS);
162 }
163
164 pub fn unlockShared(rwl: *PthreadRwLock) void {
165 const rc = std.c.pthread_rwlock_unlock(&rwl.rwlock);
166 assert(rc == .SUCCESS);
167 }
168};
169
170pub const DefaultRwLock = struct {
171 state: usize = 0,
172 mutex: std.Thread.Mutex = .{},
173 semaphore: std.Thread.Semaphore = .{},
174
175 const IS_WRITING: usize = 1;
176 const WRITER: usize = 1 << 1;
177 const READER: usize = 1 << (1 + @bitSizeOf(Count));
178 const WRITER_MASK: usize = std.math.maxInt(Count) << @ctz(WRITER);
179 const READER_MASK: usize = std.math.maxInt(Count) << @ctz(READER);
180 const Count = std.meta.Int(.unsigned, @divFloor(@bitSizeOf(usize) - 1, 2));
181
182 pub fn tryLock(rwl: *DefaultRwLock) bool {
183 if (rwl.mutex.tryLock()) {
184 const state = @atomicLoad(usize, &rwl.state, .seq_cst);
185 if (state & READER_MASK == 0) {
186 _ = @atomicRmw(usize, &rwl.state, .Or, IS_WRITING, .seq_cst);
187 return true;
188 }
189
190 rwl.mutex.unlock();
191 }
192
193 return false;
194 }
195
196 pub fn lock(rwl: *DefaultRwLock) void {
197 _ = @atomicRmw(usize, &rwl.state, .Add, WRITER, .seq_cst);
198 rwl.mutex.lock();
199
200 const state = @atomicRmw(usize, &rwl.state, .Add, IS_WRITING -% WRITER, .seq_cst);
201 if (state & READER_MASK != 0)
202 rwl.semaphore.wait();
203 }
204
205 pub fn unlock(rwl: *DefaultRwLock) void {
206 _ = @atomicRmw(usize, &rwl.state, .And, ~IS_WRITING, .seq_cst);
207 rwl.mutex.unlock();
208 }
209
210 pub fn tryLockShared(rwl: *DefaultRwLock) bool {
211 const state = @atomicLoad(usize, &rwl.state, .seq_cst);
212 if (state & (IS_WRITING | WRITER_MASK) == 0) {
213 _ = @cmpxchgStrong(
214 usize,
215 &rwl.state,
216 state,
217 state + READER,
218 .seq_cst,
219 .seq_cst,
220 ) orelse return true;
221 }
222
223 if (rwl.mutex.tryLock()) {
224 _ = @atomicRmw(usize, &rwl.state, .Add, READER, .seq_cst);
225 rwl.mutex.unlock();
226 return true;
227 }
228
229 return false;
230 }
231
232 pub fn lockShared(rwl: *DefaultRwLock) void {
233 var state = @atomicLoad(usize, &rwl.state, .seq_cst);
234 while (state & (IS_WRITING | WRITER_MASK) == 0) {
235 state = @cmpxchgWeak(
236 usize,
237 &rwl.state,
238 state,
239 state + READER,
240 .seq_cst,
241 .seq_cst,
242 ) orelse return;
243 }
244
245 rwl.mutex.lock();
246 _ = @atomicRmw(usize, &rwl.state, .Add, READER, .seq_cst);
247 rwl.mutex.unlock();
248 }
249
250 pub fn unlockShared(rwl: *DefaultRwLock) void {
251 const state = @atomicRmw(usize, &rwl.state, .Sub, READER, .seq_cst);
252
253 if ((state & READER_MASK == READER) and (state & IS_WRITING != 0))
254 rwl.semaphore.post();
255 }
256};
257
258test "DefaultRwLock - internal state" {
259 var rwl = DefaultRwLock{};
260
261 // The following failed prior to the fix for Issue #13163,
262 // where the WRITER flag was subtracted by the lock method.
263
264 rwl.lock();
265 rwl.unlock();
266 try testing.expectEqual(rwl, DefaultRwLock{});
267}
268
269test "smoke test" {
270 var rwl = RwLock{};
271
272 rwl.lock();
273 try testing.expect(!rwl.tryLock());
274 try testing.expect(!rwl.tryLockShared());
275 rwl.unlock();
276
277 try testing.expect(rwl.tryLock());
278 try testing.expect(!rwl.tryLock());
279 try testing.expect(!rwl.tryLockShared());
280 rwl.unlock();
281
282 rwl.lockShared();
283 try testing.expect(!rwl.tryLock());
284 try testing.expect(rwl.tryLockShared());
285 rwl.unlockShared();
286 rwl.unlockShared();
287
288 try testing.expect(rwl.tryLockShared());
289 try testing.expect(!rwl.tryLock());
290 try testing.expect(rwl.tryLockShared());
291 rwl.unlockShared();
292 rwl.unlockShared();
293
294 rwl.lock();
295 rwl.unlock();
296}
297
298test "concurrent access" {
299 if (builtin.single_threaded)
300 return;
301
302 const num_writers: usize = 2;
303 const num_readers: usize = 4;
304 const num_writes: usize = 1000;
305 const num_reads: usize = 2000;
306
307 const Runner = struct {
308 const Runner = @This();
309
310 rwl: RwLock,
311 writes: usize,
312 reads: std.atomic.Value(usize),
313
314 val_a: usize,
315 val_b: usize,
316
317 fn reader(run: *Runner, thread_idx: usize) !void {
318 var prng = std.Random.DefaultPrng.init(thread_idx);
319 const rnd = prng.random();
320 while (true) {
321 run.rwl.lockShared();
322 defer run.rwl.unlockShared();
323
324 try testing.expect(run.writes <= num_writes);
325 if (run.reads.fetchAdd(1, .monotonic) >= num_reads) break;
326
327 // We use `volatile` accesses so that we can make sure the memory is accessed either
328 // side of a yield, maximising chances of a race.
329 const a_ptr: *const volatile usize = &run.val_a;
330 const b_ptr: *const volatile usize = &run.val_b;
331
332 const old_a = a_ptr.*;
333 if (rnd.boolean()) try std.Thread.yield();
334 const old_b = b_ptr.*;
335 try testing.expect(old_a == old_b);
336 }
337 }
338
339 fn writer(run: *Runner, thread_idx: usize) !void {
340 var prng = std.Random.DefaultPrng.init(thread_idx);
341 const rnd = prng.random();
342 while (true) {
343 run.rwl.lock();
344 defer run.rwl.unlock();
345
346 try testing.expect(run.writes <= num_writes);
347 if (run.writes == num_writes) break;
348
349 // We use `volatile` accesses so that we can make sure the memory is accessed either
350 // side of a yield, maximising chances of a race.
351 const a_ptr: *volatile usize = &run.val_a;
352 const b_ptr: *volatile usize = &run.val_b;
353
354 const new_val = rnd.int(usize);
355
356 const old_a = a_ptr.*;
357 a_ptr.* = new_val;
358 if (rnd.boolean()) try std.Thread.yield();
359 const old_b = b_ptr.*;
360 b_ptr.* = new_val;
361 try testing.expect(old_a == old_b);
362
363 run.writes += 1;
364 }
365 }
366 };
367
368 var run: Runner = .{
369 .rwl = .{},
370 .writes = 0,
371 .reads = .init(0),
372 .val_a = 0,
373 .val_b = 0,
374 };
375 var write_threads: [num_writers]std.Thread = undefined;
376 var read_threads: [num_readers]std.Thread = undefined;
377
378 for (&write_threads, 0..) |*t, i| t.* = try .spawn(.{}, Runner.writer, .{ &run, i });
379 for (&read_threads, num_writers..) |*t, i| t.* = try .spawn(.{}, Runner.reader, .{ &run, i });
380
381 for (write_threads) |t| t.join();
382 for (read_threads) |t| t.join();
383
384 try testing.expect(run.writes == num_writes);
385 try testing.expect(run.reads.raw >= num_reads);
386}
lib/std/Thread/Semaphore.zig deleted-111
......@@ -1,111 +0,0 @@
1//! A semaphore is an unsigned integer that blocks the kernel thread if
2//! the number would become negative.
3//! This API supports static initialization and does not require deinitialization.
4//!
5//! Example:
6//! ```
7//! var s = Semaphore{};
8//!
9//! fn consumer() void {
10//! s.wait();
11//! }
12//!
13//! fn producer() void {
14//! s.post();
15//! }
16//!
17//! const thread = try std.Thread.spawn(.{}, producer, .{});
18//! consumer();
19//! thread.join();
20//! ```
21
22mutex: Mutex = .{},
23cond: Condition = .{},
24/// It is OK to initialize this field to any value.
25permits: usize = 0,
26
27const Semaphore = @This();
28const std = @import("../std.zig");
29const Mutex = std.Thread.Mutex;
30const Condition = std.Thread.Condition;
31const builtin = @import("builtin");
32const testing = std.testing;
33
34pub fn wait(sem: *Semaphore) void {
35 sem.mutex.lock();
36 defer sem.mutex.unlock();
37
38 while (sem.permits == 0)
39 sem.cond.wait(&sem.mutex);
40
41 sem.permits -= 1;
42 if (sem.permits > 0)
43 sem.cond.signal();
44}
45
46pub fn timedWait(sem: *Semaphore, timeout_ns: u64) error{Timeout}!void {
47 var timeout_timer = std.time.Timer.start() catch unreachable;
48
49 sem.mutex.lock();
50 defer sem.mutex.unlock();
51
52 while (sem.permits == 0) {
53 const elapsed = timeout_timer.read();
54 if (elapsed > timeout_ns)
55 return error.Timeout;
56
57 const local_timeout_ns = timeout_ns - elapsed;
58 try sem.cond.timedWait(&sem.mutex, local_timeout_ns);
59 }
60
61 sem.permits -= 1;
62 if (sem.permits > 0)
63 sem.cond.signal();
64}
65
66pub fn post(sem: *Semaphore) void {
67 sem.mutex.lock();
68 defer sem.mutex.unlock();
69
70 sem.permits += 1;
71 sem.cond.signal();
72}
73
74test Semaphore {
75 if (builtin.single_threaded) {
76 return error.SkipZigTest;
77 }
78
79 const TestContext = struct {
80 sem: *Semaphore,
81 n: *i32,
82 fn worker(ctx: *@This()) void {
83 ctx.sem.wait();
84 ctx.n.* += 1;
85 ctx.sem.post();
86 }
87 };
88 const num_threads = 3;
89 var sem = Semaphore{ .permits = 1 };
90 var threads: [num_threads]std.Thread = undefined;
91 var n: i32 = 0;
92 var ctx = TestContext{ .sem = &sem, .n = &n };
93
94 for (&threads) |*t| t.* = try std.Thread.spawn(.{}, TestContext.worker, .{&ctx});
95 for (threads) |t| t.join();
96 sem.wait();
97 try testing.expect(n == num_threads);
98}
99
100test timedWait {
101 var sem = Semaphore{};
102 try testing.expectEqual(0, sem.permits);
103
104 try testing.expectError(error.Timeout, sem.timedWait(1));
105
106 sem.post();
107 try testing.expectEqual(1, sem.permits);
108
109 try sem.timedWait(1);
110 try testing.expectEqual(0, sem.permits);
111}
lib/std/atomic.zig+21-4
......@@ -1,3 +1,10 @@
1const builtin = @import("builtin");
2
3const std = @import("std.zig");
4const AtomicOrder = std.builtin.AtomicOrder;
5const testing = std.testing;
6const assert = std.debug.assert;
7
18/// This is a thin wrapper around a primitive value to prevent accidental data races.
29pub fn Value(comptime T: type) type {
310 return extern struct {
......@@ -496,7 +503,17 @@ test "current CPU has a cache line size" {
496503 _ = cache_line;
497504}
498505
499const std = @import("std.zig");
500const builtin = @import("builtin");
501const AtomicOrder = std.builtin.AtomicOrder;
502const testing = std.testing;
506/// A lock-free single-owner resource.
507pub const Mutex = enum(u8) {
508 unlocked,
509 locked,
510
511 pub fn tryLock(m: *Mutex) bool {
512 return @cmpxchgWeak(Mutex, m, .unlocked, .locked, .acquire, .monotonic) == null;
513 }
514
515 pub fn unlock(m: *Mutex) void {
516 assert(m.* == .locked);
517 @atomicStore(Mutex, m, .unlocked, .release);
518 }
519};
lib/std/debug.zig+6-3
......@@ -696,7 +696,7 @@ pub noinline fn writeCurrentStackTrace(options: StackUnwindOptions, t: Io.Termin
696696 .useless, .unsafe => {},
697697 .safe, .ideal => continue, // no need to even warn
698698 }
699 const module_name = di.getModuleName(di_gpa, unwind_error.address) catch "???";
699 const module_name = di.getModuleName(di_gpa, io, unwind_error.address) catch "???";
700700 const caption: []const u8 = switch (unwind_error.err) {
701701 error.MissingDebugInfo => "unwind info unavailable",
702702 error.InvalidDebugInfo => "unwind info invalid",
......@@ -1141,7 +1141,7 @@ fn printSourceAtAddress(
11411141 symbol.source_location,
11421142 address,
11431143 symbol.name orelse "???",
1144 symbol.compile_unit_name orelse debug_info.getModuleName(gpa, address) catch "???",
1144 symbol.compile_unit_name orelse debug_info.getModuleName(gpa, io, address) catch "???",
11451145 );
11461146}
11471147fn printLineInfo(
......@@ -1356,7 +1356,10 @@ pub fn getDebugInfoAllocator() Allocator {
13561356 // Otherwise, use a global arena backed by the page allocator
13571357 const S = struct {
13581358 var arena: std.heap.ArenaAllocator = .init(std.heap.page_allocator);
1359 var ts_arena: std.heap.ThreadSafeAllocator = .{ .child_allocator = arena.allocator() };
1359 var ts_arena: std.heap.ThreadSafeAllocator = .{
1360 .child_allocator = arena.allocator(),
1361 .io = std.Options.debug_io,
1362 };
13601363 };
13611364 return S.ts_arena.allocator();
13621365}
lib/std/debug/Coverage.zig+11-9
......@@ -1,11 +1,12 @@
1const Coverage = @This();
2
13const std = @import("../std.zig");
4const Io = std.Io;
25const Allocator = std.mem.Allocator;
36const Hash = std.hash.Wyhash;
47const Dwarf = std.debug.Dwarf;
58const assert = std.debug.assert;
69
7const Coverage = @This();
8
910/// Provides a globally-scoped integer index for directories.
1011///
1112/// As opposed to, for example, a directory index that is compilation-unit
......@@ -23,12 +24,12 @@ directories: std.ArrayHashMapUnmanaged(String, void, String.MapContext, false),
2324files: std.ArrayHashMapUnmanaged(File, void, File.MapContext, false),
2425string_bytes: std.ArrayList(u8),
2526/// Protects the other fields.
26mutex: std.Thread.Mutex,
27mutex: Io.Mutex,
2728
2829pub const init: Coverage = .{
2930 .directories = .{},
3031 .files = .{},
31 .mutex = .{},
32 .mutex = .init,
3233 .string_bytes = .{},
3334};
3435
......@@ -140,11 +141,12 @@ pub fn stringAt(cov: *Coverage, index: String) [:0]const u8 {
140141 return span(cov.string_bytes.items[@intFromEnum(index)..]);
141142}
142143
143pub const ResolveAddressesDwarfError = Dwarf.ScanError;
144pub const ResolveAddressesDwarfError = Dwarf.ScanError || Io.Cancelable;
144145
145146pub fn resolveAddressesDwarf(
146147 cov: *Coverage,
147148 gpa: Allocator,
149 io: Io,
148150 endian: std.builtin.Endian,
149151 /// Asserts the addresses are in ascending order.
150152 sorted_pc_addrs: []const u64,
......@@ -161,8 +163,8 @@ pub fn resolveAddressesDwarf(
161163 var prev_pc: u64 = 0;
162164 var prev_cu: ?*std.debug.Dwarf.CompileUnit = null;
163165 // Protects directories and files tables from other threads.
164 cov.mutex.lock();
165 defer cov.mutex.unlock();
166 try cov.mutex.lock(io);
167 defer cov.mutex.unlock(io);
166168 next_pc: for (sorted_pc_addrs, output) |pc, *out| {
167169 assert(pc >= prev_pc);
168170 prev_pc = pc;
......@@ -183,8 +185,8 @@ pub fn resolveAddressesDwarf(
183185 if (cu != prev_cu) {
184186 prev_cu = cu;
185187 if (cu.src_loc_cache == null) {
186 cov.mutex.unlock();
187 defer cov.mutex.lock();
188 cov.mutex.unlock(io);
189 defer cov.mutex.lockUncancelable(io);
188190 d.populateSrcLocCache(gpa, endian, cu) catch |err| switch (err) {
189191 error.MissingDebugInfo, error.InvalidDebugInfo => {
190192 out.* = SourceLocation.invalid;
lib/std/debug/Info.zig+2-2
......@@ -93,7 +93,7 @@ pub fn resolveAddresses(
9393) ResolveAddressesError!void {
9494 assert(sorted_pc_addrs.len == output.len);
9595 switch (info.impl) {
96 .elf => |*ef| return info.coverage.resolveAddressesDwarf(gpa, ef.endian, sorted_pc_addrs, output, &ef.dwarf.?),
96 .elf => |*ef| return info.coverage.resolveAddressesDwarf(gpa, io, ef.endian, sorted_pc_addrs, output, &ef.dwarf.?),
9797 .macho => |*mf| {
9898 // Resolving all of the addresses at once unfortunately isn't so easy in Mach-O binaries
9999 // due to split debug information. For now, we'll just resolve the addreses one by one.
......@@ -112,7 +112,7 @@ pub fn resolveAddresses(
112112 else => |e| return e,
113113 };
114114 }
115 try info.coverage.resolveAddressesDwarf(gpa, .little, &.{dwarf_pc_addr}, src_loc[0..1], dwarf);
115 try info.coverage.resolveAddressesDwarf(gpa, io, .little, &.{dwarf_pc_addr}, src_loc[0..1], dwarf);
116116 }
117117 },
118118 }
lib/std/debug/SelfInfo/Elf.zig+24-24
......@@ -1,4 +1,4 @@
1rwlock: std.Thread.RwLock,
1rwlock: Io.RwLock,
22
33modules: std.ArrayList(Module),
44ranges: std.ArrayList(Module.Range),
......@@ -6,7 +6,7 @@ ranges: std.ArrayList(Module.Range),
66unwind_cache: if (can_unwind) ?[]Dwarf.SelfUnwinder.CacheEntry else ?noreturn,
77
88pub const init: SelfInfo = .{
9 .rwlock = .{},
9 .rwlock = .init,
1010 .modules = .empty,
1111 .ranges = .empty,
1212 .unwind_cache = null,
......@@ -29,8 +29,8 @@ pub fn deinit(si: *SelfInfo, gpa: Allocator) void {
2929}
3030
3131pub fn getSymbol(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!std.debug.Symbol {
32 const module = try si.findModule(gpa, address, .exclusive);
33 defer si.rwlock.unlock();
32 const module = try si.findModule(gpa, io, address, .exclusive);
33 defer si.rwlock.unlock(io);
3434
3535 const vaddr = address - module.load_offset;
3636
......@@ -73,15 +73,15 @@ pub fn getSymbol(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!st
7373 error.OutOfMemory => |e| return e,
7474 };
7575}
76pub fn getModuleName(si: *SelfInfo, gpa: Allocator, address: usize) Error![]const u8 {
77 const module = try si.findModule(gpa, address, .shared);
78 defer si.rwlock.unlockShared();
76pub fn getModuleName(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error![]const u8 {
77 const module = try si.findModule(gpa, io, address, .shared);
78 defer si.rwlock.unlockShared(io);
7979 if (module.name.len == 0) return error.MissingDebugInfo;
8080 return module.name;
8181}
82pub fn getModuleSlide(si: *SelfInfo, gpa: Allocator, address: usize) Error!usize {
83 const module = try si.findModule(gpa, address, .shared);
84 defer si.rwlock.unlockShared();
82pub fn getModuleSlide(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!usize {
83 const module = try si.findModule(gpa, io, address, .shared);
84 defer si.rwlock.unlockShared(io);
8585 return module.load_offset;
8686}
8787
......@@ -183,8 +183,8 @@ pub fn unwindFrame(si: *SelfInfo, gpa: Allocator, io: Io, context: *UnwindContex
183183 comptime assert(can_unwind);
184184
185185 {
186 si.rwlock.lockShared();
187 defer si.rwlock.unlockShared();
186 si.rwlock.lockSharedUncancelable(io);
187 defer si.rwlock.unlockShared(io);
188188 if (si.unwind_cache) |cache| {
189189 if (Dwarf.SelfUnwinder.CacheEntry.find(cache, context.pc)) |entry| {
190190 return context.next(gpa, entry);
......@@ -192,8 +192,8 @@ pub fn unwindFrame(si: *SelfInfo, gpa: Allocator, io: Io, context: *UnwindContex
192192 }
193193 }
194194
195 const module = try si.findModule(gpa, context.pc, .exclusive);
196 defer si.rwlock.unlock();
195 const module = try si.findModule(gpa, io, context.pc, .exclusive);
196 defer si.rwlock.unlock(io);
197197
198198 if (si.unwind_cache == null) {
199199 si.unwind_cache = try gpa.alloc(Dwarf.SelfUnwinder.CacheEntry, 2048);
......@@ -375,11 +375,11 @@ const Module = struct {
375375 }
376376};
377377
378fn findModule(si: *SelfInfo, gpa: Allocator, address: usize, lock: enum { shared, exclusive }) Error!*Module {
378fn findModule(si: *SelfInfo, gpa: Allocator, io: Io, address: usize, lock: enum { shared, exclusive }) Error!*Module {
379379 // With the requested lock, scan the module ranges looking for `address`.
380380 switch (lock) {
381 .shared => si.rwlock.lockShared(),
382 .exclusive => si.rwlock.lock(),
381 .shared => si.rwlock.lockSharedUncancelable(io),
382 .exclusive => si.rwlock.lockUncancelable(io),
383383 }
384384 for (si.ranges.items) |*range| {
385385 if (address >= range.start and address < range.start + range.len) {
......@@ -390,14 +390,14 @@ fn findModule(si: *SelfInfo, gpa: Allocator, address: usize, lock: enum { shared
390390 // a new module was loaded. Upgrade to an exclusive lock if necessary.
391391 switch (lock) {
392392 .shared => {
393 si.rwlock.unlockShared();
394 si.rwlock.lock();
393 si.rwlock.unlockShared(io);
394 si.rwlock.lockUncancelable(io);
395395 },
396396 .exclusive => {},
397397 }
398398 // Rebuild module list with the exclusive lock.
399399 {
400 errdefer si.rwlock.unlock();
400 errdefer si.rwlock.unlock(io);
401401 for (si.modules.items) |*mod| {
402402 unwind: {
403403 const u = &(mod.unwind orelse break :unwind catch break :unwind);
......@@ -416,8 +416,8 @@ fn findModule(si: *SelfInfo, gpa: Allocator, address: usize, lock: enum { shared
416416 // Downgrade the lock back to shared if necessary.
417417 switch (lock) {
418418 .shared => {
419 si.rwlock.unlock();
420 si.rwlock.lockShared();
419 si.rwlock.unlock(io);
420 si.rwlock.lockSharedUncancelable(io);
421421 },
422422 .exclusive => {},
423423 }
......@@ -429,8 +429,8 @@ fn findModule(si: *SelfInfo, gpa: Allocator, address: usize, lock: enum { shared
429429 }
430430 // Still nothing; unlock and error.
431431 switch (lock) {
432 .shared => si.rwlock.unlockShared(),
433 .exclusive => si.rwlock.unlock(),
432 .shared => si.rwlock.unlockShared(io),
433 .exclusive => si.rwlock.unlock(io),
434434 }
435435 return error.MissingDebugInfo;
436436}
lib/std/debug/SelfInfo/MachO.zig+16-16
......@@ -1,9 +1,9 @@
1mutex: std.Thread.Mutex,
1mutex: Io.Mutex,
22/// Accessed through `Module.Adapter`.
33modules: std.ArrayHashMapUnmanaged(Module, void, Module.Context, false),
44
55pub const init: SelfInfo = .{
6 .mutex = .{},
6 .mutex = .init,
77 .modules = .empty,
88};
99pub fn deinit(si: *SelfInfo, gpa: Allocator) void {
......@@ -21,8 +21,8 @@ pub fn deinit(si: *SelfInfo, gpa: Allocator) void {
2121}
2222
2323pub fn getSymbol(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!std.debug.Symbol {
24 const module = try si.findModule(gpa, address);
25 defer si.mutex.unlock();
24 const module = try si.findModule(gpa, io, address);
25 defer si.mutex.unlock(io);
2626
2727 const file = try module.getFile(gpa, io);
2828
......@@ -76,9 +76,10 @@ pub fn getSymbol(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!st
7676 ) catch null,
7777 };
7878}
79pub fn getModuleName(si: *SelfInfo, gpa: Allocator, address: usize) Error![]const u8 {
79pub fn getModuleName(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error![]const u8 {
8080 _ = si;
8181 _ = gpa;
82 _ = io;
8283 // This function is marked as deprecated; however, it is significantly more
8384 // performant than `dladdr` (since the latter also does a very slow symbol
8485 // lookup), so let's use it since it's still available.
......@@ -86,9 +87,9 @@ pub fn getModuleName(si: *SelfInfo, gpa: Allocator, address: usize) Error![]cons
8687 @ptrFromInt(address),
8788 ) orelse return error.MissingDebugInfo);
8889}
89pub fn getModuleSlide(si: *SelfInfo, gpa: Allocator, address: usize) Error!usize {
90 const module = try si.findModule(gpa, address);
91 defer si.mutex.unlock();
90pub fn getModuleSlide(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!usize {
91 const module = try si.findModule(gpa, io, address);
92 defer si.mutex.unlock(io);
9293 const header: *std.macho.mach_header_64 = @ptrFromInt(module.text_base);
9394 const raw_macho: [*]u8 = @ptrCast(header);
9495 var it = macho.LoadCommandIterator.init(header, raw_macho[@sizeOf(macho.mach_header_64)..][0..header.sizeofcmds]) catch unreachable;
......@@ -107,8 +108,7 @@ pub const UnwindContext = std.debug.Dwarf.SelfUnwinder;
107108/// If the compact encoding can't encode a way to unwind a frame, it will
108109/// defer unwinding to DWARF, in which case `__eh_frame` will be used if available.
109110pub fn unwindFrame(si: *SelfInfo, gpa: Allocator, io: Io, context: *UnwindContext) Error!usize {
110 _ = io;
111 return unwindFrameInner(si, gpa, context) catch |err| switch (err) {
111 return unwindFrameInner(si, gpa, io, context) catch |err| switch (err) {
112112 error.InvalidDebugInfo,
113113 error.MissingDebugInfo,
114114 error.UnsupportedDebugInfo,
......@@ -134,9 +134,9 @@ pub fn unwindFrame(si: *SelfInfo, gpa: Allocator, io: Io, context: *UnwindContex
134134 => return error.InvalidDebugInfo,
135135 };
136136}
137fn unwindFrameInner(si: *SelfInfo, gpa: Allocator, context: *UnwindContext) !usize {
138 const module = try si.findModule(gpa, context.pc);
139 defer si.mutex.unlock();
137fn unwindFrameInner(si: *SelfInfo, gpa: Allocator, io: Io, context: *UnwindContext) !usize {
138 const module = try si.findModule(gpa, io, context.pc);
139 defer si.mutex.unlock(io);
140140
141141 const unwind: *Module.Unwind = try module.getUnwindInfo(gpa);
142142
......@@ -430,15 +430,15 @@ fn unwindFrameInner(si: *SelfInfo, gpa: Allocator, context: *UnwindContext) !usi
430430}
431431
432432/// Acquires the mutex on success.
433fn findModule(si: *SelfInfo, gpa: Allocator, address: usize) Error!*Module {
433fn findModule(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!*Module {
434434 // This function is marked as deprecated; however, it is significantly more
435435 // performant than `dladdr` (since the latter also does a very slow symbol
436436 // lookup), so let's use it since it's still available.
437437 const text_base = std.c._dyld_get_image_header_containing_address(
438438 @ptrFromInt(address),
439439 ) orelse return error.MissingDebugInfo;
440 si.mutex.lock();
441 errdefer si.mutex.unlock();
440 try si.mutex.lock(io);
441 errdefer si.mutex.unlock(io);
442442 const gop = try si.modules.getOrPutAdapted(gpa, @intFromPtr(text_base), Module.Adapter{});
443443 errdefer comptime unreachable;
444444 if (!gop.found_existing) gop.key_ptr.* = .{
lib/std/debug/SelfInfo/Windows.zig+10-10
......@@ -1,9 +1,9 @@
1mutex: std.Thread.Mutex,
1mutex: Io.Mutex,
22modules: std.ArrayList(Module),
33module_name_arena: std.heap.ArenaAllocator.State,
44
55pub const init: SelfInfo = .{
6 .mutex = .{},
6 .mutex = .init,
77 .modules = .empty,
88 .module_name_arena = .{},
99};
......@@ -21,21 +21,21 @@ pub fn deinit(si: *SelfInfo, gpa: Allocator) void {
2121}
2222
2323pub fn getSymbol(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!std.debug.Symbol {
24 si.mutex.lock();
25 defer si.mutex.unlock();
24 try si.mutex.lock(io);
25 defer si.mutex.unlock(io);
2626 const module = try si.findModule(gpa, address);
2727 const di = try module.getDebugInfo(gpa, io);
2828 return di.getSymbol(gpa, address - module.base_address);
2929}
30pub fn getModuleName(si: *SelfInfo, gpa: Allocator, address: usize) Error![]const u8 {
31 si.mutex.lock();
32 defer si.mutex.unlock();
30pub fn getModuleName(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error![]const u8 {
31 try si.mutex.lock(io);
32 defer si.mutex.unlock(io);
3333 const module = try si.findModule(gpa, address);
3434 return module.name;
3535}
36pub fn getModuleSlide(si: *SelfInfo, gpa: Allocator, address: usize) Error!usize {
37 si.mutex.lock();
38 defer si.mutex.unlock();
36pub fn getModuleSlide(si: *SelfInfo, gpa: Allocator, io: Io, address: usize) Error!usize {
37 try si.mutex.lock(io);
38 defer si.mutex.unlock(io);
3939 const module = try si.findModule(gpa, address);
4040 return module.base_address;
4141}
lib/std/heap/SmpAllocator.zig+1-1
......@@ -62,7 +62,7 @@ const Thread = struct {
6262 ///
6363 /// Threads lock this before accessing their own state in order
6464 /// to support freelist reclamation.
65 mutex: std.Thread.Mutex = .{},
65 mutex: std.atomic.Mutex = .unlocked,
6666
6767 /// For each size class, tracks the next address to be returned from
6868 /// `alloc` when the freelist is empty.
lib/std/heap/ThreadSafeAllocator.zig+21-14
......@@ -1,7 +1,14 @@
1//! Wraps a non-thread-safe allocator and makes it thread-safe.
1//! Deprecated. Thread safety should be built into each Allocator instance
2//! directly rather than trying to do this "composable allocators" thing.
3const ThreadSafeAllocator = @This();
4
5const std = @import("../std.zig");
6const Io = std.Io;
7const Allocator = std.mem.Allocator;
28
39child_allocator: Allocator,
4mutex: std.Thread.Mutex = .{},
10io: Io,
11mutex: Io.Mutex = .init,
512
613pub fn allocator(self: *ThreadSafeAllocator) Allocator {
714 return .{
......@@ -17,39 +24,39 @@ pub fn allocator(self: *ThreadSafeAllocator) Allocator {
1724
1825fn alloc(ctx: *anyopaque, n: usize, alignment: std.mem.Alignment, ra: usize) ?[*]u8 {
1926 const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx));
20 self.mutex.lock();
21 defer self.mutex.unlock();
27 const io = self.io;
28 self.mutex.lockUncancelable(io);
29 defer self.mutex.unlock(io);
2230
2331 return self.child_allocator.rawAlloc(n, alignment, ra);
2432}
2533
2634fn resize(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool {
2735 const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx));
36 const io = self.io;
2837
29 self.mutex.lock();
30 defer self.mutex.unlock();
38 self.mutex.lockUncancelable(io);
39 defer self.mutex.unlock(io);
3140
3241 return self.child_allocator.rawResize(buf, alignment, new_len, ret_addr);
3342}
3443
3544fn remap(context: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, return_address: usize) ?[*]u8 {
3645 const self: *ThreadSafeAllocator = @ptrCast(@alignCast(context));
46 const io = self.io;
3747
38 self.mutex.lock();
39 defer self.mutex.unlock();
48 self.mutex.lockUncancelable(io);
49 defer self.mutex.unlock(io);
4050
4151 return self.child_allocator.rawRemap(memory, alignment, new_len, return_address);
4252}
4353
4454fn free(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, ret_addr: usize) void {
4555 const self: *ThreadSafeAllocator = @ptrCast(@alignCast(ctx));
56 const io = self.io;
4657
47 self.mutex.lock();
48 defer self.mutex.unlock();
58 self.mutex.lockUncancelable(io);
59 defer self.mutex.unlock(io);
4960
5061 return self.child_allocator.rawFree(buf, alignment, ret_addr);
5162}
52
53const std = @import("../std.zig");
54const ThreadSafeAllocator = @This();
55const Allocator = std.mem.Allocator;
lib/std/heap/debug_allocator.zig+10-41
......@@ -126,16 +126,6 @@ pub const Config = struct {
126126 /// Whether the allocator may be used simultaneously from multiple threads.
127127 thread_safe: bool = !builtin.single_threaded,
128128
129 /// What type of mutex you'd like to use, for thread safety.
130 /// when specified, the mutex type must have the same shape as `std.Thread.Mutex` and
131 /// `DummyMutex`, and have no required fields. Specifying this field causes
132 /// the `thread_safe` field to be ignored.
133 ///
134 /// when null (default):
135 /// * the mutex type defaults to `std.Thread.Mutex` when thread_safe is enabled.
136 /// * the mutex type defaults to `DummyMutex` otherwise.
137 MutexType: ?type = null,
138
139129 /// This is a temporary debugging trick you can use to turn segfaults into more helpful
140130 /// logged error messages with stack trace details. The downside is that every allocation
141131 /// will be leaked, unless used with retain_metadata!
......@@ -204,17 +194,8 @@ pub fn DebugAllocator(comptime config: Config) type {
204194 const total_requested_bytes_init = if (config.enable_memory_limit) @as(usize, 0) else {};
205195 const requested_memory_limit_init = if (config.enable_memory_limit) @as(usize, math.maxInt(usize)) else {};
206196
207 const mutex_init = if (config.MutexType) |T|
208 T{}
209 else if (config.thread_safe)
210 std.Thread.Mutex{}
211 else
212 DummyMutex{};
213
214 const DummyMutex = struct {
215 inline fn lock(_: DummyMutex) void {}
216 inline fn unlock(_: DummyMutex) void {}
217 };
197 const have_mutex = config.thread_safe;
198 const mutex_init = if (have_mutex) std.Io.Mutex.init else {};
218199
219200 const stack_n = config.stack_trace_frames;
220201 const one_trace_size = @sizeOf(usize) * stack_n;
......@@ -737,8 +718,8 @@ pub fn DebugAllocator(comptime config: Config) type {
737718
738719 fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ret_addr: usize) ?[*]u8 {
739720 const self: *Self = @ptrCast(@alignCast(context));
740 self.mutex.lock();
741 defer self.mutex.unlock();
721 if (have_mutex) std.Io.Threaded.mutexLock(&self.mutex);
722 defer if (have_mutex) std.Io.Threaded.mutexUnlock(&self.mutex);
742723
743724 if (config.enable_memory_limit) {
744725 const new_req_bytes = self.total_requested_bytes + len;
......@@ -850,8 +831,8 @@ pub fn DebugAllocator(comptime config: Config) type {
850831 return_address: usize,
851832 ) bool {
852833 const self: *Self = @ptrCast(@alignCast(context));
853 self.mutex.lock();
854 defer self.mutex.unlock();
834 if (have_mutex) std.Io.Threaded.mutexLock(&self.mutex);
835 defer if (have_mutex) std.Io.Threaded.mutexUnlock(&self.mutex);
855836
856837 const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment));
857838 if (size_class_index >= self.buckets.len) {
......@@ -869,8 +850,8 @@ pub fn DebugAllocator(comptime config: Config) type {
869850 return_address: usize,
870851 ) ?[*]u8 {
871852 const self: *Self = @ptrCast(@alignCast(context));
872 self.mutex.lock();
873 defer self.mutex.unlock();
853 if (have_mutex) std.Io.Threaded.mutexLock(&self.mutex);
854 defer if (have_mutex) std.Io.Threaded.mutexUnlock(&self.mutex);
874855
875856 const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(memory.len - 1), @intFromEnum(alignment));
876857 if (size_class_index >= self.buckets.len) {
......@@ -887,8 +868,8 @@ pub fn DebugAllocator(comptime config: Config) type {
887868 return_address: usize,
888869 ) void {
889870 const self: *Self = @ptrCast(@alignCast(context));
890 self.mutex.lock();
891 defer self.mutex.unlock();
871 if (have_mutex) std.Io.Threaded.mutexLock(&self.mutex);
872 defer if (have_mutex) std.Io.Threaded.mutexUnlock(&self.mutex);
892873
893874 const size_class_index: usize = @max(@bitSizeOf(usize) - @clz(old_memory.len - 1), @intFromEnum(alignment));
894875 if (size_class_index >= self.buckets.len) {
......@@ -1331,18 +1312,6 @@ test "realloc large object to small object" {
13311312 try std.testing.expect(slice[16] == 0x34);
13321313}
13331314
1334test "overridable mutexes" {
1335 var gpa = DebugAllocator(.{ .MutexType = std.Thread.Mutex }){
1336 .backing_allocator = std.testing.allocator,
1337 .mutex = std.Thread.Mutex{},
1338 };
1339 defer std.testing.expect(gpa.deinit() == .ok) catch @panic("leak");
1340 const allocator = gpa.allocator();
1341
1342 const ptr = try allocator.create(i32);
1343 defer allocator.destroy(ptr);
1344}
1345
13461315test "non-page-allocator backing allocator" {
13471316 var gpa: DebugAllocator(.{
13481317 .backing_allocator_zeroes = false,
lib/std/heap/sbrk_allocator.zig+10-8
......@@ -1,5 +1,7 @@
1const std = @import("../std.zig");
21const builtin = @import("builtin");
2
3const std = @import("../std.zig");
4const Io = std.Io;
35const math = std.math;
46const Allocator = std.mem.Allocator;
57const mem = std.mem;
......@@ -39,12 +41,12 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
3941 var big_frees = [1]usize{0} ** big_size_class_count;
4042
4143 // TODO don't do the naive locking strategy
42 var lock: std.Thread.Mutex = .{};
44 var mutex: Io.Mutex = .{};
4345 fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, return_address: usize) ?[*]u8 {
4446 _ = ctx;
4547 _ = return_address;
46 lock.lock();
47 defer lock.unlock();
48 Io.Threaded.mutexLock(&mutex);
49 defer Io.Threaded.mutexUnlock(&mutex);
4850 // Make room for the freelist next pointer.
4951 const actual_len = @max(len +| @sizeOf(usize), alignment.toByteUnits());
5052 const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null;
......@@ -88,8 +90,8 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
8890 ) bool {
8991 _ = ctx;
9092 _ = return_address;
91 lock.lock();
92 defer lock.unlock();
93 Io.Threaded.mutexLock(&mutex);
94 defer Io.Threaded.mutexUnlock(&mutex);
9395 // We don't want to move anything from one size class to another, but we
9496 // can recover bytes in between powers of two.
9597 const buf_align = alignment.toByteUnits();
......@@ -127,8 +129,8 @@ pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
127129 ) void {
128130 _ = ctx;
129131 _ = return_address;
130 lock.lock();
131 defer lock.unlock();
132 Io.Threaded.mutexLock(&mutex);
133 defer Io.Threaded.mutexUnlock(&mutex);
132134 const buf_align = alignment.toByteUnits();
133135 const actual_len = @max(buf.len + @sizeOf(usize), buf_align);
134136 const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len);
lib/std/http/Client.zig+36-31
......@@ -3,22 +3,25 @@
33//! Connections are opened in a thread-safe manner, but individual Requests are not.
44//!
55//! TLS support may be disabled via `std.options.http_disable_tls`.
6//!
7//! TODO all the lockUncancelable in this file should be changed to regular lock and
8//! `error.Canceled` added to more error sets.
9const Client = @This();
610
7const std = @import("../std.zig");
811const builtin = @import("builtin");
12
13const std = @import("../std.zig");
14const Io = std.Io;
915const testing = std.testing;
1016const http = std.http;
1117const mem = std.mem;
1218const Uri = std.Uri;
13const Allocator = mem.Allocator;
19const Allocator = std.mem.Allocator;
1420const assert = std.debug.assert;
15const Io = std.Io;
1621const Writer = std.Io.Writer;
1722const Reader = std.Io.Reader;
1823const HostName = std.Io.net.HostName;
1924
20const Client = @This();
21
2225pub const disable_tls = std.options.http_disable_tls;
2326
2427/// Used for all client allocations. Must be thread-safe.
......@@ -27,7 +30,7 @@ allocator: Allocator,
2730io: Io,
2831
2932ca_bundle: if (disable_tls) void else std.crypto.Certificate.Bundle = if (disable_tls) {} else .{},
30ca_bundle_mutex: std.Thread.Mutex = .{},
33ca_bundle_mutex: Io.Mutex = .init,
3134/// Used both for the reader and writer buffers.
3235tls_buffer_size: if (disable_tls) u0 else usize = if (disable_tls) 0 else std.crypto.tls.Client.min_buffer_len,
3336/// If non-null, ssl secrets are logged to a stream. Creating such a stream
......@@ -62,7 +65,7 @@ https_proxy: ?*Proxy = null,
6265
6366/// A Least-Recently-Used cache of open connections to be reused.
6467pub const ConnectionPool = struct {
65 mutex: std.Thread.Mutex = .{},
68 mutex: Io.Mutex = .init,
6669 /// Open connections that are currently in use.
6770 used: std.DoublyLinkedList = .{},
6871 /// Open connections that are not currently in use.
......@@ -81,9 +84,9 @@ pub const ConnectionPool = struct {
8184 /// If no connection is found, null is returned.
8285 ///
8386 /// Threadsafe.
84 pub fn findConnection(pool: *ConnectionPool, criteria: Criteria) ?*Connection {
85 pool.mutex.lock();
86 defer pool.mutex.unlock();
87 pub fn findConnection(pool: *ConnectionPool, io: Io, criteria: Criteria) ?*Connection {
88 pool.mutex.lockUncancelable(io);
89 defer pool.mutex.unlock(io);
8790
8891 var next = pool.free.last;
8992 while (next) |node| : (next = node.prev) {
......@@ -110,9 +113,9 @@ pub const ConnectionPool = struct {
110113 }
111114
112115 /// Acquires an existing connection from the connection pool. This function is threadsafe.
113 pub fn acquire(pool: *ConnectionPool, connection: *Connection) void {
114 pool.mutex.lock();
115 defer pool.mutex.unlock();
116 pub fn acquire(pool: *ConnectionPool, io: Io, connection: *Connection) void {
117 pool.mutex.lockUncancelable(io);
118 defer pool.mutex.unlock(io);
116119
117120 return pool.acquireUnsafe(connection);
118121 }
......@@ -122,8 +125,8 @@ pub const ConnectionPool = struct {
122125 ///
123126 /// Threadsafe.
124127 pub fn release(pool: *ConnectionPool, connection: *Connection, io: Io) void {
125 pool.mutex.lock();
126 defer pool.mutex.unlock();
128 pool.mutex.lockUncancelable(io);
129 defer pool.mutex.unlock(io);
127130
128131 pool.used.remove(&connection.pool_node);
129132
......@@ -147,9 +150,9 @@ pub const ConnectionPool = struct {
147150 }
148151
149152 /// Adds a newly created node to the pool of used connections. This function is threadsafe.
150 pub fn addUsed(pool: *ConnectionPool, connection: *Connection) void {
151 pool.mutex.lock();
152 defer pool.mutex.unlock();
153 pub fn addUsed(pool: *ConnectionPool, io: Io, connection: *Connection) void {
154 pool.mutex.lockUncancelable(io);
155 defer pool.mutex.unlock(io);
153156
154157 pool.used.append(&connection.pool_node);
155158 }
......@@ -159,9 +162,9 @@ pub const ConnectionPool = struct {
159162 /// If the new size is smaller than the current size, then idle connections will be closed until the pool is the new size.
160163 ///
161164 /// Threadsafe.
162 pub fn resize(pool: *ConnectionPool, allocator: Allocator, new_size: usize) void {
163 pool.mutex.lock();
164 defer pool.mutex.unlock();
165 pub fn resize(pool: *ConnectionPool, io: Io, allocator: Allocator, new_size: usize) void {
166 pool.mutex.lockUncancelable(io);
167 defer pool.mutex.unlock(io);
165168
166169 const next = pool.free.first;
167170 _ = next;
......@@ -182,7 +185,7 @@ pub const ConnectionPool = struct {
182185 ///
183186 /// Threadsafe.
184187 pub fn deinit(pool: *ConnectionPool, io: Io) void {
185 pool.mutex.lock();
188 pool.mutex.lockUncancelable(io);
186189
187190 var next = pool.free.first;
188191 while (next) |node| {
......@@ -1308,9 +1311,11 @@ pub fn deinit(client: *Client) void {
13081311/// Uses `arena` for a few small allocations that must outlive the client, or
13091312/// at least until those fields are set to different values.
13101313pub fn initDefaultProxies(client: *Client, arena: Allocator, environ_map: *std.process.Environ.Map) !void {
1314 const io = client.io;
1315
13111316 // Prevent any new connections from being created.
1312 client.connection_pool.mutex.lock();
1313 defer client.connection_pool.mutex.unlock();
1317 client.connection_pool.mutex.lockUncancelable(io);
1318 defer client.connection_pool.mutex.unlock(io);
13141319
13151320 assert(client.connection_pool.used.first == null); // There are active requests.
13161321
......@@ -1437,7 +1442,7 @@ pub fn connectTcpOptions(client: *Client, options: ConnectTcpOptions) ConnectTcp
14371442 const proxied_host = options.proxied_host orelse host;
14381443 const proxied_port = options.proxied_port orelse port;
14391444
1440 if (client.connection_pool.findConnection(.{
1445 if (client.connection_pool.findConnection(io, .{
14411446 .host = proxied_host,
14421447 .port = proxied_port,
14431448 .protocol = protocol,
......@@ -1455,12 +1460,12 @@ pub fn connectTcpOptions(client: *Client, options: ConnectTcpOptions) ConnectTcp
14551460 error.Canceled => |e| return e,
14561461 else => return error.TlsInitializationFailed,
14571462 };
1458 client.connection_pool.addUsed(&tc.connection);
1463 client.connection_pool.addUsed(io, &tc.connection);
14591464 return &tc.connection;
14601465 },
14611466 .plain => {
14621467 const pc = try Connection.Plain.create(client, proxied_host, proxied_port, stream);
1463 client.connection_pool.addUsed(&pc.connection);
1468 client.connection_pool.addUsed(io, &pc.connection);
14641469 return &pc.connection;
14651470 },
14661471 }
......@@ -1474,7 +1479,7 @@ pub const ConnectUnixError = Allocator.Error || std.posix.SocketError || error{N
14741479pub fn connectUnix(client: *Client, path: []const u8) ConnectUnixError!*Connection {
14751480 const io = client.io;
14761481
1477 if (client.connection_pool.findConnection(.{
1482 if (client.connection_pool.findConnection(io, .{
14781483 .host = path,
14791484 .port = 0,
14801485 .protocol = .plain,
......@@ -1516,7 +1521,7 @@ pub fn connectProxied(
15161521 const io = client.io;
15171522 if (!proxy.supports_connect) return error.TunnelNotSupported;
15181523
1519 if (client.connection_pool.findConnection(.{
1524 if (client.connection_pool.findConnection(io, .{
15201525 .host = proxied_host,
15211526 .port = proxied_port,
15221527 .protocol = proxy.protocol,
......@@ -1691,8 +1696,8 @@ pub fn request(
16911696 if (protocol == .tls) {
16921697 if (disable_tls) unreachable;
16931698 {
1694 client.ca_bundle_mutex.lock();
1695 defer client.ca_bundle_mutex.unlock();
1699 client.ca_bundle_mutex.lockUncancelable(io);
1700 defer client.ca_bundle_mutex.unlock(io);
16961701
16971702 if (client.now == null) {
16981703 const now = try Io.Clock.real.now(io);
lib/std/once.zig deleted-71
......@@ -1,71 +0,0 @@
1const std = @import("std.zig");
2const builtin = @import("builtin");
3const testing = std.testing;
4
5pub fn once(comptime f: fn () void) Once(f) {
6 return Once(f){};
7}
8
9/// An object that executes the function `f` just once.
10/// It is undefined behavior if `f` re-enters the same Once instance.
11pub fn Once(comptime f: fn () void) type {
12 return struct {
13 done: bool = false,
14 mutex: std.Thread.Mutex = std.Thread.Mutex{},
15
16 /// Call the function `f`.
17 /// If `call` is invoked multiple times `f` will be executed only the
18 /// first time.
19 /// The invocations are thread-safe.
20 pub fn call(self: *@This()) void {
21 if (@atomicLoad(bool, &self.done, .acquire))
22 return;
23
24 return self.callSlow();
25 }
26
27 fn callSlow(self: *@This()) void {
28 @branchHint(.cold);
29
30 self.mutex.lock();
31 defer self.mutex.unlock();
32
33 // The first thread to acquire the mutex gets to run the initializer
34 if (!self.done) {
35 f();
36 @atomicStore(bool, &self.done, true, .release);
37 }
38 }
39 };
40}
41
42var global_number: i32 = 0;
43var global_once = once(incr);
44
45fn incr() void {
46 global_number += 1;
47}
48
49test "Once executes its function just once" {
50 if (builtin.single_threaded) {
51 global_once.call();
52 global_once.call();
53 } else {
54 var threads: [10]std.Thread = undefined;
55 var thread_count: usize = 0;
56 defer for (threads[0..thread_count]) |handle| handle.join();
57
58 for (&threads) |*handle| {
59 handle.* = try std.Thread.spawn(.{}, struct {
60 fn thread_fn(x: u8) void {
61 _ = x;
62 global_once.call();
63 if (global_number != 1) @panic("memory ordering bug");
64 }
65 }.thread_fn, .{0});
66 thread_count += 1;
67 }
68 }
69
70 try testing.expectEqual(@as(i32, 1), global_number);
71}
lib/std/std.zig-1
......@@ -86,7 +86,6 @@ pub const math = @import("math.zig");
8686pub const mem = @import("mem.zig");
8787pub const meta = @import("meta.zig");
8888pub const os = @import("os.zig");
89pub const once = @import("once.zig").once;
9089pub const pdb = @import("pdb.zig");
9190pub const pie = @import("pie.zig");
9291pub const posix = @import("posix.zig");