| ... | ... | @@ -0,0 +1,378 @@ |
| 1 | /* |
| 2 | * Copyright (c) 2016 Apple Inc. All rights reserved. |
| 3 | * |
| 4 | * @APPLE_APACHE_LICENSE_HEADER_START@ |
| 5 | * |
| 6 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 7 | * you may not use this file except in compliance with the License. |
| 8 | * You may obtain a copy of the License at |
| 9 | * |
| 10 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | * |
| 12 | * Unless required by applicable law or agreed to in writing, software |
| 13 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | * See the License for the specific language governing permissions and |
| 16 | * limitations under the License. |
| 17 | * |
| 18 | * @APPLE_APACHE_LICENSE_HEADER_END@ |
| 19 | */ |
| 20 | |
| 21 | #ifndef __OS_SYNC_WAIT_ON_ADDRESS__ |
| 22 | #define __OS_SYNC_WAIT_ON_ADDRESS__ |
| 23 | |
| 24 | #include <Availability.h> |
| 25 | #include <stddef.h> |
| 26 | #include <stdint.h> |
| 27 | #include <os/base.h> |
| 28 | #include <os/clock.h> |
| 29 | |
| 30 | OS_ASSUME_NONNULL_BEGIN |
| 31 | |
| 32 | /*! @header |
| 33 | * Low-level Darwin's futex style APIs. |
| 34 | */ |
| 35 | |
| 36 | __BEGIN_DECLS |
| 37 | |
| 38 | #define OS_WAIT_ON_ADDR_AVAILABILITY \ |
| 39 | __API_AVAILABLE(macos(14.4), ios(17.4), tvos(17.4), watchos(10.4)) |
| 40 | |
| 41 | /*! |
| 42 | * @typedef os_sync_wait_on_address_flags_t |
| 43 | * |
| 44 | * @const OS_SYNC_WAIT_ON_ADDRESS_NONE |
| 45 | * This flag should be used as a default flag when no other flags listed below |
| 46 | * are required. |
| 47 | * |
| 48 | * @const OS_SYNC_WAIT_ON_ADDRESS_SHARED |
| 49 | * This flag should be used when synchronizing among multiple processes by |
| 50 | * placing the @addr passed to os_sync_wait_on_address and its variants |
| 51 | * in a shared memory region. |
| 52 | * |
| 53 | * When using this flag, it is important to pass OS_SYNC_WAKE_BY_ADDRESS_SHARED |
| 54 | * flag along with the exact same @addr to os_sync_wake_by_address_any and |
| 55 | * its variants to correctly find and wake up blocked waiters on the @addr. |
| 56 | * |
| 57 | * This flag should not be used when synchronizing among multiple threads of |
| 58 | * a single process. It allows the kernel to perform performance optimizations |
| 59 | * as the @addr is local to the calling process. |
| 60 | */ |
| 61 | OS_OPTIONS(os_sync_wait_on_address_flags, uint32_t, |
| 62 | OS_SYNC_WAIT_ON_ADDRESS_NONE |
| 63 | OS_WAIT_ON_ADDR_AVAILABILITY = 0x00000000, |
| 64 | OS_SYNC_WAIT_ON_ADDRESS_SHARED |
| 65 | OS_WAIT_ON_ADDR_AVAILABILITY = 0x00000001, |
| 66 | ); |
| 67 | |
| 68 | /*! |
| 69 | * @function os_sync_wait_on_address |
| 70 | * |
| 71 | * This function provides an atomic compare-and-wait functionality that |
| 72 | * can be used to implement other higher level synchronization primitives. |
| 73 | * |
| 74 | * It reads a value from @addr, compares it to expected @value and blocks |
| 75 | * the calling thread if they are equal. This sequence of operations is |
| 76 | * done atomically with respect to other concurrent operations that can |
| 77 | * be performed on this @addr by other threads using this same function |
| 78 | * or os_sync_wake_by_addr variants. At this point, the blocked calling |
| 79 | * thread is considered to be a waiter on this @addr, waiting to be woken |
| 80 | * up by a call to os_sync_wake_by_addr variants. If the value at @addr |
| 81 | * turns out to be different than expected, the calling thread returns |
| 82 | * immediately without blocking. |
| 83 | * |
| 84 | * This function is expected to be used for implementing synchronization |
| 85 | * primitives that do not have a sense of ownership (e.g. condition |
| 86 | * variables, semaphores) as it does not provide priority inversion avoidance. |
| 87 | * For locking primitives, it is recommended that you use existing OS |
| 88 | * primitives such as os_unfair_lock API family / pthread mutex or |
| 89 | * std::mutex. |
| 90 | * |
| 91 | * @param addr |
| 92 | * The userspace address to be used for atomic compare-and-wait. |
| 93 | * This address must be aligned to @size. |
| 94 | * |
| 95 | * @param value |
| 96 | * The value expected at @addr. |
| 97 | * |
| 98 | * @param size |
| 99 | * The size of @value, in bytes. This can be either 4 or 8 today. |
| 100 | * For @value of @size 4 bytes, the upper 4 bytes of @value are ignored. |
| 101 | * |
| 102 | * @param flags |
| 103 | * Flags to alter behavior of os_sync_wait_on_address. |
| 104 | * See os_sync_wait_on_address_flags_t. |
| 105 | * |
| 106 | * @return |
| 107 | * If the calling thread is woken up by a call to os_sync_wake_by_addr |
| 108 | * variants or the value at @addr is different than expected, this function |
| 109 | * returns successfully and the return value indicates the number |
| 110 | * of outstanding waiters blocked on this address. |
| 111 | * In the event of an error, returns -1 with errno set to indicate the error. |
| 112 | * |
| 113 | * EINVAL : Invalid flags or size. |
| 114 | * EINVAL : The @addr passed is NULL or misaligned. |
| 115 | * EINVAL : The operation associated with existing kernel state |
| 116 | * at this @addr is inconsistent with what the caller |
| 117 | * has requested. |
| 118 | * It is important to make sure consistent values are |
| 119 | * passed across wait and wake APIs for @addr, @size |
| 120 | * and the shared memory specification |
| 121 | * (See os_sync_wait_on_address_flags_t). |
| 122 | * |
| 123 | * It is possible for the os_sync_wait_on_address and its variants to perform |
| 124 | * an early return in the event of following errors where user may want to |
| 125 | * re-try the wait operation. E.g. low memory conditions could cause such early |
| 126 | * return. |
| 127 | * It is important to read the current value at the @addr before re-trying |
| 128 | * to ensure that the new value still requires waiting on @addr. |
| 129 | * |
| 130 | * ENOMEM : Unable to allocate memory for kernel internal data |
| 131 | * structures. |
| 132 | * EINTR : The syscall was interrupted / spurious wake up. |
| 133 | * EFAULT : Unable to read value from the @addr. Kernel copyin failed. |
| 134 | * It is possible to receive EFAULT error in following cases: |
| 135 | * 1. The @addr is an invalid address. This is a programmer error. |
| 136 | * 2. The @addr is valid; but, this is a transient error such as |
| 137 | * due to low memory conditions. User may want to re-try the wait |
| 138 | * operation. |
| 139 | * Following code snippet illustrates a possible re-try loop. |
| 140 | * <code> |
| 141 | * retry: |
| 142 | * current = atomic_load_explicit(addr, memory_order_relaxed); |
| 143 | * if (current != expected) { |
| 144 | * int ret = os_sync_wait_on_address(addr, current, size, flags); |
| 145 | * if ((ret < 0) && ((errno == EINTR) || (errno == EFAULT))) { |
| 146 | * goto retry; |
| 147 | * } |
| 148 | * } |
| 149 | * </code> |
| 150 | */ |
| 151 | OS_WAIT_ON_ADDR_AVAILABILITY |
| 152 | OS_EXPORT OS_NOTHROW OS_NONNULL_ALL |
| 153 | int |
| 154 | os_sync_wait_on_address(void *addr, |
| 155 | uint64_t value, |
| 156 | size_t size, |
| 157 | os_sync_wait_on_address_flags_t flags); |
| 158 | |
| 159 | /*! |
| 160 | * @function os_sync_wait_on_address_with_deadline |
| 161 | * |
| 162 | * This function is a variant of os_sync_wait_on_address that |
| 163 | * allows the calling thread to specify a deadline |
| 164 | * until which it is willing to block. |
| 165 | * |
| 166 | * @param addr |
| 167 | * The userspace address to be used for atomic compare-and-wait. |
| 168 | * This address must be aligned to @size. |
| 169 | * |
| 170 | * @param value |
| 171 | * The value expected at @addr. |
| 172 | * |
| 173 | * @param size |
| 174 | * The size of @value, in bytes. This can be either 4 or 8 today. |
| 175 | * For @value of @size 4 bytes, the upper 4 bytes of @value are ignored. |
| 176 | * |
| 177 | * @param flags |
| 178 | * Flags to alter behavior of os_sync_wait_on_address_with_deadline. |
| 179 | * See os_sync_wait_on_address_flags_t. |
| 180 | * |
| 181 | * @param clockid |
| 182 | * This value anchors @deadline argument to a specific clock id. |
| 183 | * See os_clockid_t. |
| 184 | * |
| 185 | * @param deadline |
| 186 | * This value is used to specify a deadline until which the calling |
| 187 | * thread is willing to block. |
| 188 | * Passing zero for the @deadline results in an error being returned. |
| 189 | * It is recommended to use os_sync_wait_on_address API to block |
| 190 | * indefinitely until woken up by a call to os_sync_wake_by_address_any |
| 191 | * or os_sync_wake_by_address_all APIs. |
| 192 | * |
| 193 | * @return |
| 194 | * If the calling thread is woken up by a call to os_sync_wake_by_addr |
| 195 | * variants or the value at @addr is different than expected, this function |
| 196 | * returns successfully and the return value indicates the number |
| 197 | * of outstanding waiters blocked on this address. |
| 198 | * In the event of an error, returns -1 with errno set to indicate the error. |
| 199 | * |
| 200 | * In addition to errors returned by os_sync_wait_on_address, this function |
| 201 | * can return the following additional error codes. |
| 202 | * |
| 203 | * EINVAL : Invalid clock id. |
| 204 | * EINVAL : The @deadline passed is 0. |
| 205 | * ETIMEDOUT : Deadline expired. |
| 206 | */ |
| 207 | OS_WAIT_ON_ADDR_AVAILABILITY |
| 208 | OS_EXPORT OS_NOTHROW OS_NONNULL_ALL |
| 209 | int |
| 210 | os_sync_wait_on_address_with_deadline(void *addr, |
| 211 | uint64_t value, |
| 212 | size_t size, |
| 213 | os_sync_wait_on_address_flags_t flags, |
| 214 | os_clockid_t clockid, |
| 215 | uint64_t deadline); |
| 216 | |
| 217 | /*! |
| 218 | * @function os_sync_wait_on_address_with_timeout |
| 219 | * |
| 220 | * This function is a variant of os_sync_wait_on_address that |
| 221 | * allows the calling thread to specify a timeout |
| 222 | * until which it is willing to block. |
| 223 | * |
| 224 | * @param addr |
| 225 | * The userspace address to be used for atomic compare-and-wait. |
| 226 | * This address must be aligned to @size. |
| 227 | * |
| 228 | * @param value |
| 229 | * The value expected at @addr. |
| 230 | * |
| 231 | * @param size |
| 232 | * The size of @value, in bytes. This can be either 4 or 8 today. |
| 233 | * For @value of @size 4 bytes, the upper 4 bytes of @value are ignored. |
| 234 | * |
| 235 | * @param flags |
| 236 | * Flags to alter behavior of os_sync_wait_on_address_with_timeout. |
| 237 | * See os_sync_wait_on_address_flags_t. |
| 238 | * |
| 239 | * @param clockid |
| 240 | * This value anchors @timeout_ns argument to a specific clock id. |
| 241 | * See os_clockid_t. |
| 242 | * |
| 243 | * @param timeout_ns |
| 244 | * This value is used to specify a timeout in nanoseconds until which |
| 245 | * the calling thread is willing to block. |
| 246 | * Passing zero for the @timeout_ns results in an error being returned. |
| 247 | * It is recommended to use os_sync_wait_on_address API to block |
| 248 | * indefinitely until woken up by a call to os_sync_wake_by_address_any |
| 249 | * or os_sync_wake_by_address_all APIs. |
| 250 | * |
| 251 | * @return |
| 252 | * If the calling thread is woken up by a call to os_sync_wake_by_address |
| 253 | * variants or the value at @addr is different than expected, this function |
| 254 | * returns successfully and the return value indicates the number |
| 255 | * of outstanding waiters blocked on this address. |
| 256 | * In the event of an error, returns -1 with errno set to indicate the error. |
| 257 | * |
| 258 | * In addition to errors returned by os_sync_wait_on_address, this function |
| 259 | * can return the following additional error codes. |
| 260 | * |
| 261 | * EINVAL : Invalid clock id. |
| 262 | * EINVAL : The @timeout_ns passed is 0. |
| 263 | * ETIMEDOUT : Timeout expired. |
| 264 | */ |
| 265 | OS_WAIT_ON_ADDR_AVAILABILITY |
| 266 | OS_EXPORT OS_NOTHROW OS_NONNULL_ALL |
| 267 | int |
| 268 | os_sync_wait_on_address_with_timeout(void *addr, |
| 269 | uint64_t value, |
| 270 | size_t size, |
| 271 | os_sync_wait_on_address_flags_t flags, |
| 272 | os_clockid_t clockid, |
| 273 | uint64_t timeout_ns); |
| 274 | |
| 275 | /*! |
| 276 | * @typedef os_sync_wake_by_address_flags_t |
| 277 | * |
| 278 | * @const OS_SYNC_WAKE_BY_ADDRESS_NONE |
| 279 | * This flag should be used as a default flag when no other flags listed below |
| 280 | * are required. |
| 281 | * |
| 282 | * @const OS_SYNC_WAKE_BY_ADDRESS_SHARED |
| 283 | * This flag should be used when synchronizing among multiple processes by |
| 284 | * placing the @addr passed to os_sync_wake_by_address_any and its variants |
| 285 | * in a shared memory region. |
| 286 | * |
| 287 | * When using this flag, it is important to pass OS_SYNC_WAIT_ON_ADDRESS_SHARED |
| 288 | * flag along with the exact same @addr to os_sync_wait_on_address and |
| 289 | * its variants to correctly find and wake up blocked waiters on the @addr. |
| 290 | * |
| 291 | * This flag should not be used when synchronizing among multiple threads of |
| 292 | * a single process. It allows the kernel to perform performance optimizations |
| 293 | * as the @addr is local the calling process. |
| 294 | */ |
| 295 | OS_OPTIONS(os_sync_wake_by_address_flags, uint32_t, |
| 296 | OS_SYNC_WAKE_BY_ADDRESS_NONE |
| 297 | OS_WAIT_ON_ADDR_AVAILABILITY = 0x00000000, |
| 298 | OS_SYNC_WAKE_BY_ADDRESS_SHARED |
| 299 | OS_WAIT_ON_ADDR_AVAILABILITY = 0x00000001, |
| 300 | ); |
| 301 | |
| 302 | /*! |
| 303 | * @function os_sync_wake_by_address_any |
| 304 | * |
| 305 | * This function wakes up one waiter out of all those blocked in os_sync_wait_on_address |
| 306 | * or its variants on the @addr. No guarantee is provided about which |
| 307 | * specific waiter is woken up. |
| 308 | * |
| 309 | * @param addr |
| 310 | * The userspace address to be used for waking up the blocked waiter. |
| 311 | * It should be same as what is passed to os_sync_wait_on_address or its variants. |
| 312 | * |
| 313 | * @param size |
| 314 | * The size of lock value, in bytes. This can be either 4 or 8 today. |
| 315 | * It should be same as what is passed to os_sync_wait_on_address or its variants. |
| 316 | * |
| 317 | * @param flags |
| 318 | * Flags to alter behavior of os_sync_wake_by_address_any. |
| 319 | * See os_sync_wake_by_address_flags_t. |
| 320 | * |
| 321 | * @return |
| 322 | * Returns 0 on success. |
| 323 | * In the event of an error, returns -1 with errno set to indicate the error. |
| 324 | * |
| 325 | * EINVAL : Invalid flags or size. |
| 326 | * EINVAL : The @addr passed is NULL. |
| 327 | * EINVAL : The operation associated with existing kernel state |
| 328 | * at this @addr is inconsistent with what caller |
| 329 | * has requested. |
| 330 | * It is important to make sure consistent values are |
| 331 | * passed across wait and wake APIs for @addr, @size |
| 332 | * and the shared memory specification |
| 333 | * (See os_sync_wake_by_address_flags_t). |
| 334 | * ENOENT : No waiter(s) found waiting on the @addr. |
| 335 | */ |
| 336 | OS_WAIT_ON_ADDR_AVAILABILITY |
| 337 | OS_EXPORT OS_NOTHROW OS_NONNULL_ALL |
| 338 | int |
| 339 | os_sync_wake_by_address_any(void *addr, |
| 340 | size_t size, |
| 341 | os_sync_wake_by_address_flags_t flags); |
| 342 | |
| 343 | /*! |
| 344 | * @function os_sync_wake_by_address_all |
| 345 | * |
| 346 | * This function is a variant of os_sync_wake_by_address_any that wakes up all waiters |
| 347 | * blocked in os_sync_wait_on_address or its variants. |
| 348 | * |
| 349 | * @param addr |
| 350 | * The userspace address to be used for waking up the blocked waiters. |
| 351 | * It should be same as what is passed to os_sync_wait_on_address or its variants. |
| 352 | * |
| 353 | * @param size |
| 354 | * The size of lock value, in bytes. This can be either 4 or 8 today. |
| 355 | * It should be same as what is passed to os_sync_wait_on_address or its variants. |
| 356 | * |
| 357 | * @param flags |
| 358 | * Flags to alter behavior of os_sync_wake_by_address_all. |
| 359 | * See os_sync_wake_by_address_flags_t. |
| 360 | * |
| 361 | * @return |
| 362 | * Returns 0 on success. |
| 363 | * In the event of an error, returns -1 with errno set to indicate the error. |
| 364 | * |
| 365 | * This function returns same error codes as returned by os_sync_wait_on_address. |
| 366 | */ |
| 367 | OS_WAIT_ON_ADDR_AVAILABILITY |
| 368 | OS_EXPORT OS_NOTHROW OS_NONNULL_ALL |
| 369 | int |
| 370 | os_sync_wake_by_address_all(void *addr, |
| 371 | size_t size, |
| 372 | os_sync_wake_by_address_flags_t flags); |
| 373 | |
| 374 | __END_DECLS |
| 375 | |
| 376 | OS_ASSUME_NONNULL_END |
| 377 | |
| 378 | #endif // __OS_SYNC_WAIT_ON_ADDRESS__ |
Alex Rønne Petersen <alex@alexrp.com> 2026-10-02 02:38:06+02:00