| ... | @@ -360,7 +360,11 @@ const Thread = struct { | ... | @@ -360,7 +360,11 @@ const Thread = struct { |
| 360 | else => unreachable, | 360 | else => unreachable, |
| 361 | }; | 361 | }; |
| 362 | }, | 362 | }, |
| 363 | else => @compileError("unimplemented: futexWait"), | 363 | else => if (std.Thread.use_pthreads) { |
| | 364 | return pthreads_futex.wait(ptr, expect, timeout_ns); |
| | 365 | } else { |
| | 366 | @compileError("unimplemented: futexWait"); |
| | 367 | }, |
| 364 | } | 368 | } |
| 365 | } | 369 | } |
| 366 | | 370 | |
| ... | @@ -436,7 +440,11 @@ const Thread = struct { | ... | @@ -436,7 +440,11 @@ const Thread = struct { |
| 436 | else => unreachable, // deadlock due to operating system bug | 440 | else => unreachable, // deadlock due to operating system bug |
| 437 | } | 441 | } |
| 438 | }, | 442 | }, |
| 439 | else => @compileError("unimplemented: futexWake"), | 443 | else => if (std.Thread.use_pthreads) { |
| | 444 | return pthreads_futex.wake(ptr, max_waiters); |
| | 445 | } else { |
| | 446 | @compileError("unimplemented: futexWake"); |
| | 447 | }, |
| 440 | } | 448 | } |
| 441 | } | 449 | } |
| 442 | }; | 450 | }; |
| ... | @@ -4025,9 +4033,7 @@ fn fileRealPathPosix(userdata: ?*anyopaque, file: File, out_buffer: []u8) File.R | ... | @@ -4025,9 +4033,7 @@ fn fileRealPathPosix(userdata: ?*anyopaque, file: File, out_buffer: []u8) File.R |
| 4025 | | 4033 | |
| 4026 | fn realPathPosix(current_thread: *Thread, fd: posix.fd_t, out_buffer: []u8) File.RealPathError!usize { | 4034 | fn realPathPosix(current_thread: *Thread, fd: posix.fd_t, out_buffer: []u8) File.RealPathError!usize { |
| 4027 | switch (native_os) { | 4035 | switch (native_os) { |
| 4028 | .driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => { | 4036 | .netbsd, .dragonfly, .driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => { |
| 4029 | // On macOS, we can use F.GETPATH fcntl command to query the OS for | | |
| 4030 | // the path to the file descriptor. | | |
| 4031 | var sufficient_buffer: [posix.PATH_MAX]u8 = undefined; | 4037 | var sufficient_buffer: [posix.PATH_MAX]u8 = undefined; |
| 4032 | @memset(&sufficient_buffer, 0); | 4038 | @memset(&sufficient_buffer, 0); |
| 4033 | try current_thread.beginSyscall(); | 4039 | try current_thread.beginSyscall(); |
| ... | @@ -4045,9 +4051,12 @@ fn realPathPosix(current_thread: *Thread, fd: posix.fd_t, out_buffer: []u8) File | ... | @@ -4045,9 +4051,12 @@ fn realPathPosix(current_thread: *Thread, fd: posix.fd_t, out_buffer: []u8) File |
| 4045 | else => |e| { | 4051 | else => |e| { |
| 4046 | current_thread.endSyscall(); | 4052 | current_thread.endSyscall(); |
| 4047 | switch (e) { | 4053 | switch (e) { |
| | 4054 | .ACCES => return error.AccessDenied, |
| 4048 | .BADF => return error.FileNotFound, | 4055 | .BADF => return error.FileNotFound, |
| 4049 | .NOSPC => return error.NameTooLong, | | |
| 4050 | .NOENT => return error.FileNotFound, | 4056 | .NOENT => return error.FileNotFound, |
| | 4057 | .NOMEM => return error.SystemResources, |
| | 4058 | .NOSPC => return error.NameTooLong, |
| | 4059 | .RANGE => return error.NameTooLong, |
| 4051 | else => |err| return posix.unexpectedErrno(err), | 4060 | else => |err| return posix.unexpectedErrno(err), |
| 4052 | } | 4061 | } |
| 4053 | }, | 4062 | }, |
| ... | @@ -4095,11 +4104,11 @@ fn realPathPosix(current_thread: *Thread, fd: posix.fd_t, out_buffer: []u8) File | ... | @@ -4095,11 +4104,11 @@ fn realPathPosix(current_thread: *Thread, fd: posix.fd_t, out_buffer: []u8) File |
| 4095 | } | 4104 | } |
| 4096 | }, | 4105 | }, |
| 4097 | .freebsd => { | 4106 | .freebsd => { |
| 4098 | var kfile: std.c.kinfo_file = undefined; | 4107 | var k_file: std.c.kinfo_file = undefined; |
| 4099 | kfile.structsize = std.c.KINFO_FILE_SIZE; | 4108 | k_file.structsize = std.c.KINFO_FILE_SIZE; |
| 4100 | try current_thread.beginSyscall(); | 4109 | try current_thread.beginSyscall(); |
| 4101 | while (true) { | 4110 | while (true) { |
| 4102 | switch (posix.errno(std.c.fcntl(fd, std.c.F.KINFO, @intFromPtr(&kfile)))) { | 4111 | switch (posix.errno(std.c.fcntl(fd, std.c.F.KINFO, @intFromPtr(&k_file)))) { |
| 4103 | .SUCCESS => { | 4112 | .SUCCESS => { |
| 4104 | current_thread.endSyscall(); | 4113 | current_thread.endSyscall(); |
| 4105 | break; | 4114 | break; |
| ... | @@ -4118,39 +4127,10 @@ fn realPathPosix(current_thread: *Thread, fd: posix.fd_t, out_buffer: []u8) File | ... | @@ -4118,39 +4127,10 @@ fn realPathPosix(current_thread: *Thread, fd: posix.fd_t, out_buffer: []u8) File |
| 4118 | }, | 4127 | }, |
| 4119 | } | 4128 | } |
| 4120 | } | 4129 | } |
| 4121 | const len = std.mem.indexOfScalar(u8, &kfile.path, 0) orelse kfile.path.len; | 4130 | const len = std.mem.indexOfScalar(u8, &k_file.path, 0) orelse k_file.path.len; |
| 4122 | if (len == 0) return error.NameTooLong; | 4131 | if (len == 0) return error.NameTooLong; |
| 4123 | return len; | 4132 | return len; |
| 4124 | }, | 4133 | }, |
| 4125 | .netbsd, .dragonfly => { | | |
| 4126 | @memset(out_buffer[0..Dir.max_path_bytes], 0); | | |
| 4127 | try current_thread.beginSyscall(); | | |
| 4128 | while (true) { | | |
| 4129 | switch (posix.errno(std.c.fcntl(fd, posix.F.GETPATH, out_buffer))) { | | |
| 4130 | .SUCCESS => { | | |
| 4131 | current_thread.endSyscall(); | | |
| 4132 | break; | | |
| 4133 | }, | | |
| 4134 | .INTR => { | | |
| 4135 | try current_thread.checkCancel(); | | |
| 4136 | continue; | | |
| 4137 | }, | | |
| 4138 | .CANCELED => return current_thread.endSyscallCanceled(), | | |
| 4139 | else => |e| { | | |
| 4140 | current_thread.endSyscall(); | | |
| 4141 | switch (e) { | | |
| 4142 | .ACCES => return error.AccessDenied, | | |
| 4143 | .BADF => return error.FileNotFound, | | |
| 4144 | .NOENT => return error.FileNotFound, | | |
| 4145 | .NOMEM => return error.SystemResources, | | |
| 4146 | .RANGE => return error.NameTooLong, | | |
| 4147 | else => |err| return posix.unexpectedErrno(err), | | |
| 4148 | } | | |
| 4149 | }, | | |
| 4150 | } | | |
| 4151 | } | | |
| 4152 | return std.mem.indexOfScalar(u8, &out_buffer, 0) orelse out_buffer.len; | | |
| 4153 | }, | | |
| 4154 | else => return error.OperationUnsupported, | 4134 | else => return error.OperationUnsupported, |
| 4155 | } | 4135 | } |
| 4156 | comptime unreachable; | 4136 | comptime unreachable; |
| ... | @@ -7049,11 +7029,32 @@ fn processExecutablePath(userdata: ?*anyopaque, out_buffer: []u8) std.process.Ex | ... | @@ -7049,11 +7029,32 @@ fn processExecutablePath(userdata: ?*anyopaque, out_buffer: []u8) std.process.Ex |
| 7049 | }, | 7029 | }, |
| 7050 | .netbsd => { | 7030 | .netbsd => { |
| 7051 | const current_thread = Thread.getCurrent(t); | 7031 | const current_thread = Thread.getCurrent(t); |
| 7052 | try current_thread.checkCancel(); | | |
| 7053 | var mib = [4]c_int{ posix.CTL.KERN, posix.KERN.PROC_ARGS, -1, posix.KERN.PROC_PATHNAME }; | 7032 | var mib = [4]c_int{ posix.CTL.KERN, posix.KERN.PROC_ARGS, -1, posix.KERN.PROC_PATHNAME }; |
| 7054 | var out_len: usize = out_buffer.len; | 7033 | var out_len: usize = out_buffer.len; |
| 7055 | try posix.sysctl(&mib, out_buffer.ptr, &out_len, null, 0); | 7034 | try current_thread.beginSyscall(); |
| 7056 | return out_len; | 7035 | while (true) { |
| | 7036 | switch (posix.errno(posix.system.sysctl(&mib, mib.len, out_buffer.ptr, &out_len, null, 0))) { |
| | 7037 | .SUCCESS => { |
| | 7038 | current_thread.endSyscall(); |
| | 7039 | return out_len; |
| | 7040 | }, |
| | 7041 | .INTR => { |
| | 7042 | try current_thread.checkCancel(); |
| | 7043 | continue; |
| | 7044 | }, |
| | 7045 | .CANCELED => return current_thread.endSyscallCanceled(), |
| | 7046 | else => |e| { |
| | 7047 | current_thread.endSyscall(); |
| | 7048 | switch (e) { |
| | 7049 | .FAULT => |err| return errnoBug(err), |
| | 7050 | .PERM => return error.PermissionDenied, |
| | 7051 | .NOMEM => return error.SystemResources, |
| | 7052 | .NOENT => |err| return errnoBug(err), |
| | 7053 | else => |err| return posix.unexpectedErrno(err), |
| | 7054 | } |
| | 7055 | }, |
| | 7056 | } |
| | 7057 | } |
| 7057 | }, | 7058 | }, |
| 7058 | .openbsd, .haiku => { | 7059 | .openbsd, .haiku => { |
| 7059 | // OpenBSD doesn't support getting the path of a running process, so try to guess it | 7060 | // OpenBSD doesn't support getting the path of a running process, so try to guess it |
| ... | @@ -11320,6 +11321,388 @@ fn initializeWsa(t: *Threaded) error{ NetworkDown, Canceled }!void { | ... | @@ -11320,6 +11321,388 @@ fn initializeWsa(t: *Threaded) error{ NetworkDown, Canceled }!void { |
| 11320 | | 11321 | |
| 11321 | fn doNothingSignalHandler(_: posix.SIG) callconv(.c) void {} | 11322 | fn doNothingSignalHandler(_: posix.SIG) callconv(.c) void {} |
| 11322 | | 11323 | |
| | 11324 | const pthreads_futex = struct { |
| | 11325 | const c = std.c; |
| | 11326 | const atomic = std.atomic; |
| | 11327 | |
| | 11328 | const Event = struct { |
| | 11329 | cond: c.pthread_cond_t, |
| | 11330 | mutex: c.pthread_mutex_t, |
| | 11331 | state: enum { empty, waiting, notified }, |
| | 11332 | |
| | 11333 | fn init(self: *Event) void { |
| | 11334 | // Use static init instead of pthread_cond/mutex_init() since this is generally faster. |
| | 11335 | self.cond = .{}; |
| | 11336 | self.mutex = .{}; |
| | 11337 | self.state = .empty; |
| | 11338 | } |
| | 11339 | |
| | 11340 | fn deinit(self: *Event) void { |
| | 11341 | // Some platforms reportedly give EINVAL for statically initialized pthread types. |
| | 11342 | const rc = c.pthread_cond_destroy(&self.cond); |
| | 11343 | assert(rc == .SUCCESS or rc == .INVAL); |
| | 11344 | |
| | 11345 | const rm = c.pthread_mutex_destroy(&self.mutex); |
| | 11346 | assert(rm == .SUCCESS or rm == .INVAL); |
| | 11347 | |
| | 11348 | self.* = undefined; |
| | 11349 | } |
| | 11350 | |
| | 11351 | fn wait(self: *Event, timeout: ?u64) error{Timeout}!void { |
| | 11352 | assert(c.pthread_mutex_lock(&self.mutex) == .SUCCESS); |
| | 11353 | defer assert(c.pthread_mutex_unlock(&self.mutex) == .SUCCESS); |
| | 11354 | |
| | 11355 | // Early return if the event was already set. |
| | 11356 | if (self.state == .notified) { |
| | 11357 | return; |
| | 11358 | } |
| | 11359 | |
| | 11360 | // Compute the absolute timeout if one was specified. |
| | 11361 | // POSIX requires that REALTIME is used by default for the pthread timedwait functions. |
| | 11362 | // This can be changed with pthread_condattr_setclock, but it's an extension and may not be available everywhere. |
| | 11363 | var ts: c.timespec = undefined; |
| | 11364 | if (timeout) |timeout_ns| { |
| | 11365 | ts = std.posix.clock_gettime(c.CLOCK.REALTIME) catch unreachable; |
| | 11366 | ts.sec +|= @as(@TypeOf(ts.sec), @intCast(timeout_ns / std.time.ns_per_s)); |
| | 11367 | ts.nsec += @as(@TypeOf(ts.nsec), @intCast(timeout_ns % std.time.ns_per_s)); |
| | 11368 | |
| | 11369 | if (ts.nsec >= std.time.ns_per_s) { |
| | 11370 | ts.sec +|= 1; |
| | 11371 | ts.nsec -= std.time.ns_per_s; |
| | 11372 | } |
| | 11373 | } |
| | 11374 | |
| | 11375 | // Start waiting on the event - there can be only one thread waiting. |
| | 11376 | assert(self.state == .empty); |
| | 11377 | self.state = .waiting; |
| | 11378 | |
| | 11379 | while (true) { |
| | 11380 | // Block using either pthread_cond_wait or pthread_cond_timewait if there's an absolute timeout. |
| | 11381 | const rc = blk: { |
| | 11382 | if (timeout == null) break :blk c.pthread_cond_wait(&self.cond, &self.mutex); |
| | 11383 | break :blk c.pthread_cond_timedwait(&self.cond, &self.mutex, &ts); |
| | 11384 | }; |
| | 11385 | |
| | 11386 | // After waking up, check if the event was set. |
| | 11387 | if (self.state == .notified) { |
| | 11388 | return; |
| | 11389 | } |
| | 11390 | |
| | 11391 | assert(self.state == .waiting); |
| | 11392 | switch (rc) { |
| | 11393 | .SUCCESS => {}, |
| | 11394 | .TIMEDOUT => { |
| | 11395 | // If timed out, reset the event to avoid the set() thread doing an unnecessary signal(). |
| | 11396 | self.state = .empty; |
| | 11397 | return error.Timeout; |
| | 11398 | }, |
| | 11399 | .INVAL => unreachable, // cond, mutex, and potentially ts should all be valid |
| | 11400 | .PERM => unreachable, // mutex is locked when cond_*wait() functions are called |
| | 11401 | else => unreachable, |
| | 11402 | } |
| | 11403 | } |
| | 11404 | } |
| | 11405 | |
| | 11406 | fn set(self: *Event) void { |
| | 11407 | assert(c.pthread_mutex_lock(&self.mutex) == .SUCCESS); |
| | 11408 | defer assert(c.pthread_mutex_unlock(&self.mutex) == .SUCCESS); |
| | 11409 | |
| | 11410 | // Make sure that multiple calls to set() were not done on the same Event. |
| | 11411 | const old_state = self.state; |
| | 11412 | assert(old_state != .notified); |
| | 11413 | |
| | 11414 | // Mark the event as set and wake up the waiting thread if there was one. |
| | 11415 | // This must be done while the mutex as the wait() thread could deallocate |
| | 11416 | // the condition variable once it observes the new state, potentially causing a UAF if done unlocked. |
| | 11417 | self.state = .notified; |
| | 11418 | if (old_state == .waiting) { |
| | 11419 | assert(c.pthread_cond_signal(&self.cond) == .SUCCESS); |
| | 11420 | } |
| | 11421 | } |
| | 11422 | }; |
| | 11423 | |
| | 11424 | const Treap = std.Treap(usize, std.math.order); |
| | 11425 | const Waiter = struct { |
| | 11426 | node: Treap.Node, |
| | 11427 | prev: ?*Waiter, |
| | 11428 | next: ?*Waiter, |
| | 11429 | tail: ?*Waiter, |
| | 11430 | is_queued: bool, |
| | 11431 | event: Event, |
| | 11432 | }; |
| | 11433 | |
| | 11434 | // An unordered set of Waiters |
| | 11435 | const WaitList = struct { |
| | 11436 | top: ?*Waiter = null, |
| | 11437 | len: usize = 0, |
| | 11438 | |
| | 11439 | fn push(self: *WaitList, waiter: *Waiter) void { |
| | 11440 | waiter.next = self.top; |
| | 11441 | self.top = waiter; |
| | 11442 | self.len += 1; |
| | 11443 | } |
| | 11444 | |
| | 11445 | fn pop(self: *WaitList) ?*Waiter { |
| | 11446 | const waiter = self.top orelse return null; |
| | 11447 | self.top = waiter.next; |
| | 11448 | self.len -= 1; |
| | 11449 | return waiter; |
| | 11450 | } |
| | 11451 | }; |
| | 11452 | |
| | 11453 | const WaitQueue = struct { |
| | 11454 | fn insert(treap: *Treap, address: usize, waiter: *Waiter) void { |
| | 11455 | // prepare the waiter to be inserted. |
| | 11456 | waiter.next = null; |
| | 11457 | waiter.is_queued = true; |
| | 11458 | |
| | 11459 | // Find the wait queue entry associated with the address. |
| | 11460 | // If there isn't a wait queue on the address, this waiter creates the queue. |
| | 11461 | var entry = treap.getEntryFor(address); |
| | 11462 | const entry_node = entry.node orelse { |
| | 11463 | waiter.prev = null; |
| | 11464 | waiter.tail = waiter; |
| | 11465 | entry.set(&waiter.node); |
| | 11466 | return; |
| | 11467 | }; |
| | 11468 | |
| | 11469 | // There's a wait queue on the address; get the queue head and tail. |
| | 11470 | const head: *Waiter = @fieldParentPtr("node", entry_node); |
| | 11471 | const tail = head.tail orelse unreachable; |
| | 11472 | |
| | 11473 | // Push the waiter to the tail by replacing it and linking to the previous tail. |
| | 11474 | head.tail = waiter; |
| | 11475 | tail.next = waiter; |
| | 11476 | waiter.prev = tail; |
| | 11477 | } |
| | 11478 | |
| | 11479 | fn remove(treap: *Treap, address: usize, max_waiters: usize) WaitList { |
| | 11480 | // Find the wait queue associated with this address and get the head/tail if any. |
| | 11481 | var entry = treap.getEntryFor(address); |
| | 11482 | var queue_head: ?*Waiter = if (entry.node) |node| @fieldParentPtr("node", node) else null; |
| | 11483 | const queue_tail = if (queue_head) |head| head.tail else null; |
| | 11484 | |
| | 11485 | // Once we're done updating the head, fix it's tail pointer and update the treap's queue head as well. |
| | 11486 | defer entry.set(blk: { |
| | 11487 | const new_head = queue_head orelse break :blk null; |
| | 11488 | new_head.tail = queue_tail; |
| | 11489 | break :blk &new_head.node; |
| | 11490 | }); |
| | 11491 | |
| | 11492 | var removed = WaitList{}; |
| | 11493 | while (removed.len < max_waiters) { |
| | 11494 | // dequeue and collect waiters from their wait queue. |
| | 11495 | const waiter = queue_head orelse break; |
| | 11496 | queue_head = waiter.next; |
| | 11497 | removed.push(waiter); |
| | 11498 | |
| | 11499 | // When dequeueing, we must mark is_queued as false. |
| | 11500 | // This ensures that a waiter which calls tryRemove() returns false. |
| | 11501 | assert(waiter.is_queued); |
| | 11502 | waiter.is_queued = false; |
| | 11503 | } |
| | 11504 | |
| | 11505 | return removed; |
| | 11506 | } |
| | 11507 | |
| | 11508 | fn tryRemove(treap: *Treap, address: usize, waiter: *Waiter) bool { |
| | 11509 | if (!waiter.is_queued) { |
| | 11510 | return false; |
| | 11511 | } |
| | 11512 | |
| | 11513 | queue_remove: { |
| | 11514 | // Find the wait queue associated with the address. |
| | 11515 | var entry = blk: { |
| | 11516 | // A waiter without a previous link means it's the queue head that's in the treap so we can avoid lookup. |
| | 11517 | if (waiter.prev == null) { |
| | 11518 | assert(waiter.node.key == address); |
| | 11519 | break :blk treap.getEntryForExisting(&waiter.node); |
| | 11520 | } |
| | 11521 | break :blk treap.getEntryFor(address); |
| | 11522 | }; |
| | 11523 | |
| | 11524 | // The queue head and tail must exist if we're removing a queued waiter. |
| | 11525 | const head: *Waiter = @fieldParentPtr("node", entry.node orelse unreachable); |
| | 11526 | const tail = head.tail orelse unreachable; |
| | 11527 | |
| | 11528 | // A waiter with a previous link is never the head of the queue. |
| | 11529 | if (waiter.prev) |prev| { |
| | 11530 | assert(waiter != head); |
| | 11531 | prev.next = waiter.next; |
| | 11532 | |
| | 11533 | // A waiter with both a previous and next link is in the middle. |
| | 11534 | // We only need to update the surrounding waiter's links to remove it. |
| | 11535 | if (waiter.next) |next| { |
| | 11536 | assert(waiter != tail); |
| | 11537 | next.prev = waiter.prev; |
| | 11538 | break :queue_remove; |
| | 11539 | } |
| | 11540 | |
| | 11541 | // A waiter with a previous but no next link means it's the tail of the queue. |
| | 11542 | // In that case, we need to update the head's tail reference. |
| | 11543 | assert(waiter == tail); |
| | 11544 | head.tail = waiter.prev; |
| | 11545 | break :queue_remove; |
| | 11546 | } |
| | 11547 | |
| | 11548 | // A waiter with no previous link means it's the queue head of queue. |
| | 11549 | // We must replace (or remove) the head waiter reference in the treap. |
| | 11550 | assert(waiter == head); |
| | 11551 | entry.set(blk: { |
| | 11552 | const new_head = waiter.next orelse break :blk null; |
| | 11553 | new_head.tail = head.tail; |
| | 11554 | break :blk &new_head.node; |
| | 11555 | }); |
| | 11556 | } |
| | 11557 | |
| | 11558 | // Mark the waiter as successfully removed. |
| | 11559 | waiter.is_queued = false; |
| | 11560 | return true; |
| | 11561 | } |
| | 11562 | }; |
| | 11563 | |
| | 11564 | const Bucket = struct { |
| | 11565 | mutex: c.pthread_mutex_t align(atomic.cache_line) = .{}, |
| | 11566 | pending: atomic.Value(usize) = atomic.Value(usize).init(0), |
| | 11567 | treap: Treap = .{}, |
| | 11568 | |
| | 11569 | // Global array of buckets that addresses map to. |
| | 11570 | // Bucket array size is pretty much arbitrary here, but it must be a power of two for fibonacci hashing. |
| | 11571 | var buckets = [_]Bucket{.{}} ** @bitSizeOf(usize); |
| | 11572 | |
| | 11573 | // https://github.com/Amanieu/parking_lot/blob/1cf12744d097233316afa6c8b7d37389e4211756/core/src/parking_lot.rs#L343-L353 |
| | 11574 | fn from(address: usize) *Bucket { |
| | 11575 | // The upper `@bitSizeOf(usize)` bits of the fibonacci golden ratio. |
| | 11576 | // Hashing this via (h * k) >> (64 - b) where k=golden-ration and b=bitsize-of-array |
| | 11577 | // evenly lays out h=hash values over the bit range even when the hash has poor entropy (identity-hash for pointers). |
| | 11578 | const max_multiplier_bits = @bitSizeOf(usize); |
| | 11579 | const fibonacci_multiplier = 0x9E3779B97F4A7C15 >> (64 - max_multiplier_bits); |
| | 11580 | |
| | 11581 | const max_bucket_bits = @ctz(buckets.len); |
| | 11582 | comptime assert(std.math.isPowerOfTwo(buckets.len)); |
| | 11583 | |
| | 11584 | const index = (address *% fibonacci_multiplier) >> (max_multiplier_bits - max_bucket_bits); |
| | 11585 | return &buckets[index]; |
| | 11586 | } |
| | 11587 | }; |
| | 11588 | |
| | 11589 | const Address = struct { |
| | 11590 | fn from(ptr: *const u32) usize { |
| | 11591 | // Get the alignment of the pointer. |
| | 11592 | const alignment = @alignOf(atomic.Value(u32)); |
| | 11593 | comptime assert(std.math.isPowerOfTwo(alignment)); |
| | 11594 | |
| | 11595 | // Make sure the pointer is aligned, |
| | 11596 | // then cut off the zero bits from the alignment to get the unique address. |
| | 11597 | const addr = @intFromPtr(ptr); |
| | 11598 | assert(addr & (alignment - 1) == 0); |
| | 11599 | return addr >> @ctz(@as(usize, alignment)); |
| | 11600 | } |
| | 11601 | }; |
| | 11602 | |
| | 11603 | fn wait(ptr: *const u32, expect: u32, timeout: ?u64) error{Timeout}!void { |
| | 11604 | const address = Address.from(ptr); |
| | 11605 | const bucket = Bucket.from(address); |
| | 11606 | |
| | 11607 | // Announce that there's a waiter in the bucket before checking the ptr/expect condition. |
| | 11608 | // If the announcement is reordered after the ptr check, the waiter could deadlock: |
| | 11609 | // |
| | 11610 | // - T1: checks ptr == expect which is true |
| | 11611 | // - T2: updates ptr to != expect |
| | 11612 | // - T2: does Futex.wake(), sees no pending waiters, exits |
| | 11613 | // - T1: bumps pending waiters (was reordered after the ptr == expect check) |
| | 11614 | // - T1: goes to sleep and misses both the ptr change and T2's wake up |
| | 11615 | // |
| | 11616 | // acquire barrier to ensure the announcement happens before the ptr check below. |
| | 11617 | var pending = bucket.pending.fetchAdd(1, .acquire); |
| | 11618 | assert(pending < std.math.maxInt(usize)); |
| | 11619 | |
| | 11620 | // If the wait gets canceled, remove the pending count we previously added. |
| | 11621 | // This is done outside the mutex lock to keep the critical section short in case of contention. |
| | 11622 | var canceled = false; |
| | 11623 | defer if (canceled) { |
| | 11624 | pending = bucket.pending.fetchSub(1, .monotonic); |
| | 11625 | assert(pending > 0); |
| | 11626 | }; |
| | 11627 | |
| | 11628 | var waiter: Waiter = undefined; |
| | 11629 | { |
| | 11630 | assert(c.pthread_mutex_lock(&bucket.mutex) == .SUCCESS); |
| | 11631 | defer assert(c.pthread_mutex_unlock(&bucket.mutex) == .SUCCESS); |
| | 11632 | |
| | 11633 | canceled = @atomicLoad(u32, ptr, .monotonic) != expect; |
| | 11634 | if (canceled) { |
| | 11635 | return; |
| | 11636 | } |
| | 11637 | |
| | 11638 | waiter.event.init(); |
| | 11639 | WaitQueue.insert(&bucket.treap, address, &waiter); |
| | 11640 | } |
| | 11641 | |
| | 11642 | defer { |
| | 11643 | assert(!waiter.is_queued); |
| | 11644 | waiter.event.deinit(); |
| | 11645 | } |
| | 11646 | |
| | 11647 | waiter.event.wait(timeout) catch { |
| | 11648 | // If we fail to cancel after a timeout, it means a wake() thread dequeued us and will wake us up. |
| | 11649 | // We must wait until the event is set as that's a signal that the wake() thread won't access the waiter memory anymore. |
| | 11650 | // If we return early without waiting, the waiter on the stack would be invalidated and the wake() thread risks a UAF. |
| | 11651 | defer if (!canceled) waiter.event.wait(null) catch unreachable; |
| | 11652 | |
| | 11653 | assert(c.pthread_mutex_lock(&bucket.mutex) == .SUCCESS); |
| | 11654 | defer assert(c.pthread_mutex_unlock(&bucket.mutex) == .SUCCESS); |
| | 11655 | |
| | 11656 | canceled = WaitQueue.tryRemove(&bucket.treap, address, &waiter); |
| | 11657 | if (canceled) { |
| | 11658 | return error.Timeout; |
| | 11659 | } |
| | 11660 | }; |
| | 11661 | } |
| | 11662 | |
| | 11663 | fn wake(ptr: *const u32, max_waiters: u32) void { |
| | 11664 | const address = Address.from(ptr); |
| | 11665 | const bucket = Bucket.from(address); |
| | 11666 | |
| | 11667 | // Quick check if there's even anything to wake up. |
| | 11668 | // The change to the ptr's value must happen before we check for pending waiters. |
| | 11669 | // If not, the wake() thread could miss a sleeping waiter and have it deadlock: |
| | 11670 | // |
| | 11671 | // - T2: p = has pending waiters (reordered before the ptr update) |
| | 11672 | // - T1: bump pending waiters |
| | 11673 | // - T1: if ptr == expected: sleep() |
| | 11674 | // - T2: update ptr != expected |
| | 11675 | // - T2: p is false from earlier so doesn't wake (T1 missed ptr update and T2 missed T1 sleeping) |
| | 11676 | // |
| | 11677 | // What we really want here is a Release load, but that doesn't exist under the C11 memory model. |
| | 11678 | // We could instead do `bucket.pending.fetchAdd(0, Release) == 0` which achieves effectively the same thing, |
| | 11679 | // LLVM lowers the fetchAdd(0, .release) into an mfence+load which avoids gaining ownership of the cache-line. |
| | 11680 | if (bucket.pending.fetchAdd(0, .release) == 0) { |
| | 11681 | return; |
| | 11682 | } |
| | 11683 | |
| | 11684 | // Keep a list of all the waiters notified and wake then up outside the mutex critical section. |
| | 11685 | var notified = WaitList{}; |
| | 11686 | defer if (notified.len > 0) { |
| | 11687 | const pending = bucket.pending.fetchSub(notified.len, .monotonic); |
| | 11688 | assert(pending >= notified.len); |
| | 11689 | |
| | 11690 | while (notified.pop()) |waiter| { |
| | 11691 | assert(!waiter.is_queued); |
| | 11692 | waiter.event.set(); |
| | 11693 | } |
| | 11694 | }; |
| | 11695 | |
| | 11696 | assert(c.pthread_mutex_lock(&bucket.mutex) == .SUCCESS); |
| | 11697 | defer assert(c.pthread_mutex_unlock(&bucket.mutex) == .SUCCESS); |
| | 11698 | |
| | 11699 | // Another pending check again to avoid the WaitQueue lookup if not necessary. |
| | 11700 | if (bucket.pending.load(.monotonic) > 0) { |
| | 11701 | notified = WaitQueue.remove(&bucket.treap, address, max_waiters); |
| | 11702 | } |
| | 11703 | } |
| | 11704 | }; |
| | 11705 | |
| 11323 | test { | 11706 | test { |
| 11324 | _ = @import("Threaded/test.zig"); | 11707 | _ = @import("Threaded/test.zig"); |
| 11325 | } | 11708 | } |
Andrew Kelley <andrew@ziglang.org> 2025-12-19 15:57:53-08:00
lib