| ... | ... | @@ -27,7 +27,8 @@ const os = std.os; |
| 27 | 27 | const assert = std.debug.assert; |
| 28 | 28 | const testing = std.testing; |
| 29 | 29 | const Atomic = std.atomic.Atomic; |
| 30 | | const Futex = std.Thread.Futex; |
| 30 | const Thread = std.Thread; |
| 31 | const Futex = Thread.Futex; |
| 31 | 32 | |
| 32 | 33 | impl: Impl = .{}, |
| 33 | 34 | |
| ... | ... | @@ -51,7 +52,12 @@ pub fn unlock(self: *Mutex) void { |
| 51 | 52 | self.impl.unlock(); |
| 52 | 53 | } |
| 53 | 54 | |
| 54 | | const Impl = if (builtin.single_threaded) |
| 55 | const Impl = if (builtin.mode == .Debug and !builtin.single_threaded) |
| 56 | DebugImpl |
| 57 | else |
| 58 | ReleaseImpl; |
| 59 | |
| 60 | const ReleaseImpl = if (builtin.single_threaded) |
| 55 | 61 | SingleThreadedImpl |
| 56 | 62 | else if (builtin.os.tag == .windows) |
| 57 | 63 | WindowsImpl |
| ... | ... | @@ -60,22 +66,50 @@ else if (builtin.os.tag.isDarwin()) |
| 60 | 66 | else |
| 61 | 67 | FutexImpl; |
| 62 | 68 | |
| 69 | const DebugImpl = struct { |
| 70 | locking_thread: Atomic(Thread.Id) = Atomic(Thread.Id).init(0), // 0 means it's not locked. |
| 71 | impl: ReleaseImpl = .{}, |
| 72 | |
| 73 | inline fn tryLock(self: *@This()) bool { |
| 74 | const locking = self.impl.tryLock(); |
| 75 | if (locking) { |
| 76 | self.locking_thread.store(Thread.getCurrentId(), .Unordered); |
| 77 | } |
| 78 | return locking; |
| 79 | } |
| 80 | |
| 81 | inline fn lock(self: *@This()) void { |
| 82 | const current_id = Thread.getCurrentId(); |
| 83 | if (self.locking_thread.load(.Unordered) == current_id and current_id != 0) { |
| 84 | @panic("Deadlock detected"); |
| 85 | } |
| 86 | self.impl.lock(); |
| 87 | self.locking_thread.store(current_id, .Unordered); |
| 88 | } |
| 89 | |
| 90 | inline fn unlock(self: *@This()) void { |
| 91 | assert(self.locking_thread.load(.Unordered) == Thread.getCurrentId()); |
| 92 | self.locking_thread.store(0, .Unordered); |
| 93 | self.impl.unlock(); |
| 94 | } |
| 95 | }; |
| 96 | |
| 63 | 97 | const SingleThreadedImpl = struct { |
| 64 | 98 | is_locked: bool = false, |
| 65 | 99 | |
| 66 | | fn tryLock(self: *Impl) bool { |
| 100 | fn tryLock(self: *@This()) bool { |
| 67 | 101 | if (self.is_locked) return false; |
| 68 | 102 | self.is_locked = true; |
| 69 | 103 | return true; |
| 70 | 104 | } |
| 71 | 105 | |
| 72 | | fn lock(self: *Impl) void { |
| 106 | fn lock(self: *@This()) void { |
| 73 | 107 | if (!self.tryLock()) { |
| 74 | 108 | unreachable; // deadlock detected |
| 75 | 109 | } |
| 76 | 110 | } |
| 77 | 111 | |
| 78 | | fn unlock(self: *Impl) void { |
| 112 | fn unlock(self: *@This()) void { |
| 79 | 113 | assert(self.is_locked); |
| 80 | 114 | self.is_locked = false; |
| 81 | 115 | } |
| ... | ... | @@ -86,15 +120,15 @@ const SingleThreadedImpl = struct { |
| 86 | 120 | const WindowsImpl = struct { |
| 87 | 121 | srwlock: os.windows.SRWLOCK = .{}, |
| 88 | 122 | |
| 89 | | fn tryLock(self: *Impl) bool { |
| 123 | fn tryLock(self: *@This()) bool { |
| 90 | 124 | return os.windows.kernel32.TryAcquireSRWLockExclusive(&self.srwlock) != os.windows.FALSE; |
| 91 | 125 | } |
| 92 | 126 | |
| 93 | | fn lock(self: *Impl) void { |
| 127 | fn lock(self: *@This()) void { |
| 94 | 128 | os.windows.kernel32.AcquireSRWLockExclusive(&self.srwlock); |
| 95 | 129 | } |
| 96 | 130 | |
| 97 | | fn unlock(self: *Impl) void { |
| 131 | fn unlock(self: *@This()) void { |
| 98 | 132 | os.windows.kernel32.ReleaseSRWLockExclusive(&self.srwlock); |
| 99 | 133 | } |
| 100 | 134 | }; |
| ... | ... | @@ -103,15 +137,15 @@ const WindowsImpl = struct { |
| 103 | 137 | const DarwinImpl = struct { |
| 104 | 138 | oul: os.darwin.os_unfair_lock = .{}, |
| 105 | 139 | |
| 106 | | fn tryLock(self: *Impl) bool { |
| 140 | fn tryLock(self: *@This()) bool { |
| 107 | 141 | return os.darwin.os_unfair_lock_trylock(&self.oul); |
| 108 | 142 | } |
| 109 | 143 | |
| 110 | | fn lock(self: *Impl) void { |
| 144 | fn lock(self: *@This()) void { |
| 111 | 145 | os.darwin.os_unfair_lock_lock(&self.oul); |
| 112 | 146 | } |
| 113 | 147 | |
| 114 | | fn unlock(self: *Impl) void { |
| 148 | fn unlock(self: *@This()) void { |
| 115 | 149 | os.darwin.os_unfair_lock_unlock(&self.oul); |
| 116 | 150 | } |
| 117 | 151 | }; |
| ... | ... | @@ -123,19 +157,19 @@ const FutexImpl = struct { |
| 123 | 157 | const locked = 0b01; |
| 124 | 158 | const contended = 0b11; // must contain the `locked` bit for x86 optimization below |
| 125 | 159 | |
| 126 | | fn tryLock(self: *Impl) bool { |
| 160 | fn tryLock(self: *@This()) bool { |
| 127 | 161 | // Lock with compareAndSwap instead of tryCompareAndSwap to avoid reporting spurious CAS failure. |
| 128 | 162 | return self.lockFast("compareAndSwap"); |
| 129 | 163 | } |
| 130 | 164 | |
| 131 | | fn lock(self: *Impl) void { |
| 165 | fn lock(self: *@This()) void { |
| 132 | 166 | // Lock with tryCompareAndSwap instead of compareAndSwap due to being more inline-able on LL/SC archs like ARM. |
| 133 | 167 | if (!self.lockFast("tryCompareAndSwap")) { |
| 134 | 168 | self.lockSlow(); |
| 135 | 169 | } |
| 136 | 170 | } |
| 137 | 171 | |
| 138 | | inline fn lockFast(self: *Impl, comptime casFn: []const u8) bool { |
| 172 | inline fn lockFast(self: *@This(), comptime casFn: []const u8) bool { |
| 139 | 173 | // On x86, use `lock bts` instead of `lock cmpxchg` as: |
| 140 | 174 | // - they both seem to mark the cache-line as modified regardless: https://stackoverflow.com/a/63350048 |
| 141 | 175 | // - `lock bts` is smaller instruction-wise which makes it better for inlining |
| ... | ... | @@ -149,7 +183,7 @@ const FutexImpl = struct { |
| 149 | 183 | return @field(self.state, casFn)(unlocked, locked, .Acquire, .Monotonic) == null; |
| 150 | 184 | } |
| 151 | 185 | |
| 152 | | fn lockSlow(self: *Impl) void { |
| 186 | fn lockSlow(self: *@This()) void { |
| 153 | 187 | @setCold(true); |
| 154 | 188 | |
| 155 | 189 | // Avoid doing an atomic swap below if we already know the state is contended. |
| ... | ... | @@ -172,7 +206,7 @@ const FutexImpl = struct { |
| 172 | 206 | } |
| 173 | 207 | } |
| 174 | 208 | |
| 175 | | fn unlock(self: *Impl) void { |
| 209 | fn unlock(self: *@This()) void { |
| 176 | 210 | // Unlock the mutex and wake up a waiting thread if any. |
| 177 | 211 | // |
| 178 | 212 | // A waiting thread will acquire with `contended` instead of `locked` |
| ... | ... | @@ -228,7 +262,7 @@ test "Mutex - many uncontended" { |
| 228 | 262 | |
| 229 | 263 | const Runner = struct { |
| 230 | 264 | mutex: Mutex = .{}, |
| 231 | | thread: std.Thread = undefined, |
| 265 | thread: Thread = undefined, |
| 232 | 266 | counter: NonAtomicCounter = .{}, |
| 233 | 267 | |
| 234 | 268 | fn run(self: *@This()) void { |
| ... | ... | @@ -243,7 +277,7 @@ test "Mutex - many uncontended" { |
| 243 | 277 | }; |
| 244 | 278 | |
| 245 | 279 | var runners = [_]Runner{.{}} ** num_threads; |
| 246 | | for (runners) |*r| r.thread = try std.Thread.spawn(.{}, Runner.run, .{r}); |
| 280 | for (runners) |*r| r.thread = try Thread.spawn(.{}, Runner.run, .{r}); |
| 247 | 281 | for (runners) |r| r.thread.join(); |
| 248 | 282 | for (runners) |r| try testing.expectEqual(r.counter.get(), num_increments); |
| 249 | 283 | } |
| ... | ... | @@ -265,7 +299,7 @@ test "Mutex - many contended" { |
| 265 | 299 | var i: usize = num_increments; |
| 266 | 300 | while (i > 0) : (i -= 1) { |
| 267 | 301 | // Occasionally hint to let another thread run. |
| 268 | | defer if (i % 100 == 0) std.Thread.yield() catch {}; |
| 302 | defer if (i % 100 == 0) Thread.yield() catch {}; |
| 269 | 303 | |
| 270 | 304 | self.mutex.lock(); |
| 271 | 305 | defer self.mutex.unlock(); |
| ... | ... | @@ -277,8 +311,8 @@ test "Mutex - many contended" { |
| 277 | 311 | |
| 278 | 312 | var runner = Runner{}; |
| 279 | 313 | |
| 280 | | var threads: [num_threads]std.Thread = undefined; |
| 281 | | for (threads) |*t| t.* = try std.Thread.spawn(.{}, Runner.run, .{&runner}); |
| 314 | var threads: [num_threads]Thread = undefined; |
| 315 | for (threads) |*t| t.* = try Thread.spawn(.{}, Runner.run, .{&runner}); |
| 282 | 316 | for (threads) |t| t.join(); |
| 283 | 317 | |
| 284 | 318 | try testing.expectEqual(runner.counter.get(), num_increments * num_threads); |