1//===-- tsan_interceptors_mac.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of ThreadSanitizer (TSan), a race detector.
10//
11// Mac-specific interceptors.
12//===----------------------------------------------------------------------===//
13
14#include "sanitizer_common/sanitizer_platform.h"
15#if SANITIZER_APPLE
16
17# include <errno.h>
18# include <libkern/OSAtomic.h>
19# include <objc/objc-sync.h>
20# include <os/lock.h>
21# include <sys/ucontext.h>
22
23# include "interception/interception.h"
24# include "sanitizer_common/sanitizer_addrhashmap.h"
25# include "tsan_interceptors.h"
26# include "tsan_interface.h"
27# include "tsan_interface_ann.h"
28
29# if defined(__has_include) && __has_include(<xpc/xpc.h>)
30# include <xpc/xpc.h>
31# endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
32
33typedef long long_t;
34
35extern "C" {
36int getcontext(ucontext_t *ucp) __attribute__((returns_twice));
37int setcontext(const ucontext_t *ucp);
38}
39
40namespace __tsan {
41
42// The non-barrier versions of OSAtomic* functions are semantically mo_relaxed,
43// but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are
44// actually aliases of each other, and we cannot have different interceptors for
45// them, because they're actually the same function. Thus, we have to stay
46// conservative and treat the non-barrier versions as mo_acq_rel.
47static constexpr morder kMacOrderBarrier = mo_acq_rel;
48static constexpr morder kMacOrderNonBarrier = mo_acq_rel;
49static constexpr morder kMacFailureOrder = mo_relaxed;
50
51# define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
52 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
53 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
54 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \
55 }
56
57# define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, \
58 mo) \
59 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
60 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
61 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \
62 }
63
64# define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
65 mo) \
66 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
67 SCOPED_TSAN_INTERCEPTOR(f, ptr); \
68 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \
69 }
70
71# define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
72 mo) \
73 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
74 SCOPED_TSAN_INTERCEPTOR(f, ptr); \
75 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \
76 }
77
78# define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \
79 m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
80 kMacOrderNonBarrier) \
81 m(int32_t, int32_t, a32, f##32##Barrier, \
82 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) \
83 m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \
84 kMacOrderNonBarrier) \
85 m(int64_t, int64_t, a64, f##64##Barrier, \
86 __tsan_atomic64_##tsan_atomic_f, kMacOrderBarrier)
87
88# define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \
89 m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
90 kMacOrderNonBarrier) \
91 m(int32_t, uint32_t, a32, f##32##Barrier, \
92 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) \
93 m_orig(int32_t, uint32_t, a32, f##32##Orig, \
94 __tsan_atomic32_##tsan_atomic_f, kMacOrderNonBarrier) \
95 m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \
96 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
97
98# pragma clang diagnostic push // OSAtomic* deprecation
99# pragma clang diagnostic ignored "-Wdeprecated-declarations"
100OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,
101 OSATOMIC_INTERCEPTOR_PLUS_X)
102OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,
103 OSATOMIC_INTERCEPTOR_PLUS_1)
104OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub,
105 OSATOMIC_INTERCEPTOR_MINUS_1)
106OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X,
107 OSATOMIC_INTERCEPTOR)
108OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,
109 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
110OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
111 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
112# pragma clang diagnostic pop // OSAtomic* deprecation
113
114# define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \
115 TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \
116 SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
117 return tsan_atomic_f##_compare_exchange_strong( \
118 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
119 kMacOrderNonBarrier, kMacFailureOrder); \
120 } \
121 \
122 TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \
123 t volatile *ptr) { \
124 SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
125 return tsan_atomic_f##_compare_exchange_strong( \
126 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
127 kMacOrderBarrier, kMacFailureOrder); \
128 }
129
130# pragma clang diagnostic push // OSAtomicCompareAndSwap* deprecation
131# pragma clang diagnostic ignored "-Wdeprecated-declarations"
132OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
133OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,
134 long_t)
135OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64,
136 void *)
137OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
138 int32_t)
139OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
140 int64_t)
141# pragma clang diagnostic pop // OSAtomicCompareAndSwap* deprecation
142
143# define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \
144 TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
145 SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \
146 volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \
147 char bit = 0x80u >> (n & 7); \
148 char mask = clear ? ~bit : bit; \
149 char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \
150 return orig_byte & bit; \
151 }
152
153# define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \
154 OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
155 OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
156
157# pragma clang diagnostic push // OSAtomicTestAnd* deprecation
158# pragma clang diagnostic ignored "-Wdeprecated-declarations"
159OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)
160OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,
161 true)
162# pragma clang diagnostic pop // OSAtomicTestAnd* deprecation
163
164TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item,
165 size_t offset) {
166 SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset);
167 __tsan_release(item);
168 REAL(OSAtomicEnqueue)(list, item, offset);
169}
170
171TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) {
172 SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);
173 void *item = REAL(OSAtomicDequeue)(list, offset);
174 if (item)
175 __tsan_acquire(item);
176 return item;
177}
178
179// OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.
180# if !SANITIZER_IOS
181
182TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item,
183 size_t offset) {
184 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset);
185 __tsan_release(item);
186 REAL(OSAtomicFifoEnqueue)(list, item, offset);
187}
188
189TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list,
190 size_t offset) {
191 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);
192 void *item = REAL(OSAtomicFifoDequeue)(list, offset);
193 if (item)
194 __tsan_acquire(item);
195 return item;
196}
197
198# endif
199
200// If `OSSPINLOCK_USE_INLINED=1` is set, then SDK headers don't declare these
201// as functions, but macros that call non-deprecated APIs. Undefine these
202// macros so they don't interfere with the interceptor machinery.
203# undef OSSpinLockLock
204# undef OSSpinLockTry
205# undef OSSpinLockUnlock
206
207# pragma clang diagnostic push // OSSpinLock* deprecation
208# pragma clang diagnostic ignored "-Wdeprecated-declarations"
209
210TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
211 CHECK(!cur_thread()->is_dead);
212 if (!cur_thread()->is_inited) {
213 return REAL(OSSpinLockLock)(lock);
214 }
215 SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock);
216 REAL(OSSpinLockLock)(lock);
217 Acquire(thr, pc, (uptr)lock);
218}
219
220TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) {
221 CHECK(!cur_thread()->is_dead);
222 if (!cur_thread()->is_inited) {
223 return REAL(OSSpinLockTry)(lock);
224 }
225 SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock);
226 bool result = REAL(OSSpinLockTry)(lock);
227 if (result)
228 Acquire(thr, pc, (uptr)lock);
229 return result;
230}
231
232TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {
233 CHECK(!cur_thread()->is_dead);
234 if (!cur_thread()->is_inited) {
235 return REAL(OSSpinLockUnlock)(lock);
236 }
237 SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock);
238 Release(thr, pc, (uptr)lock);
239 REAL(OSSpinLockUnlock)(lock);
240}
241# pragma clang diagnostic pop // OSSpinLock* deprecation
242
243TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {
244 CHECK(!cur_thread()->is_dead);
245 if (!cur_thread()->is_inited) {
246 return REAL(os_lock_lock)(lock);
247 }
248 SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock);
249 REAL(os_lock_lock)(lock);
250 Acquire(thr, pc, (uptr)lock);
251}
252
253TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) {
254 CHECK(!cur_thread()->is_dead);
255 if (!cur_thread()->is_inited) {
256 return REAL(os_lock_trylock)(lock);
257 }
258 SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock);
259 bool result = REAL(os_lock_trylock)(lock);
260 if (result)
261 Acquire(thr, pc, (uptr)lock);
262 return result;
263}
264
265TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
266 CHECK(!cur_thread()->is_dead);
267 if (!cur_thread()->is_inited) {
268 return REAL(os_lock_unlock)(lock);
269 }
270 SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock);
271 Release(thr, pc, (uptr)lock);
272 REAL(os_lock_unlock)(lock);
273}
274
275TSAN_INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) {
276 if (!cur_thread()->is_inited || cur_thread()->is_dead) {
277 return REAL(os_unfair_lock_lock)(lock);
278 }
279 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock, lock);
280 REAL(os_unfair_lock_lock)(lock);
281 Acquire(thr, pc, (uptr)lock);
282}
283
284// os_unfair_lock_lock_with_flags was introduced in macOS 15
285# if defined(__MAC_15_0) || defined(__IPHONE_18_0) || defined(__TVOS_18_0) || \
286 defined(__VISIONOS_2_0) || defined(__WATCHOS_11_0)
287# pragma clang diagnostic push
288# pragma clang diagnostic ignored "-Wunguarded-availability-new"
289// We're just intercepting this - if it doesn't exist on the platform, then the
290// process shouldn't have called it in the first place.
291TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_flags, os_unfair_lock_t lock,
292 os_unfair_lock_flags_t flags) {
293 if (!cur_thread()->is_inited || cur_thread()->is_dead) {
294 return REAL(os_unfair_lock_lock_with_flags)(lock, flags);
295 }
296 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_flags, lock, flags);
297 REAL(os_unfair_lock_lock_with_flags)(lock, flags);
298 Acquire(thr, pc, (uptr)lock);
299}
300# pragma clang diagnostic pop
301# endif
302
303TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_options, os_unfair_lock_t lock,
304 u32 options) {
305 if (!cur_thread()->is_inited || cur_thread()->is_dead) {
306 return REAL(os_unfair_lock_lock_with_options)(lock, options);
307 }
308 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_options, lock, options);
309 REAL(os_unfair_lock_lock_with_options)(lock, options);
310 Acquire(thr, pc, (uptr)lock);
311}
312
313TSAN_INTERCEPTOR(bool, os_unfair_lock_trylock, os_unfair_lock_t lock) {
314 if (!cur_thread()->is_inited || cur_thread()->is_dead) {
315 return REAL(os_unfair_lock_trylock)(lock);
316 }
317 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_trylock, lock);
318 bool result = REAL(os_unfair_lock_trylock)(lock);
319 if (result)
320 Acquire(thr, pc, (uptr)lock);
321 return result;
322}
323
324TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) {
325 if (!cur_thread()->is_inited || cur_thread()->is_dead) {
326 return REAL(os_unfair_lock_unlock)(lock);
327 }
328 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_unlock, lock);
329 Release(thr, pc, (uptr)lock);
330 REAL(os_unfair_lock_unlock)(lock);
331}
332
333# if defined(__has_include) && __has_include(<xpc/xpc.h>)
334
335TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
336 xpc_connection_t connection, xpc_handler_t handler) {
337 SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
338 handler);
339 Release(thr, pc, (uptr)connection);
340 xpc_handler_t new_handler = ^(xpc_object_t object) {
341 {
342 SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler);
343 Acquire(thr, pc, (uptr)connection);
344 }
345 handler(object);
346 };
347 REAL(xpc_connection_set_event_handler)(connection, new_handler);
348}
349
350TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection,
351 dispatch_block_t barrier) {
352 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier);
353 Release(thr, pc, (uptr)connection);
354 dispatch_block_t new_barrier = ^() {
355 {
356 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier);
357 Acquire(thr, pc, (uptr)connection);
358 }
359 barrier();
360 };
361 REAL(xpc_connection_send_barrier)(connection, new_barrier);
362}
363
364TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply,
365 xpc_connection_t connection, xpc_object_t message,
366 dispatch_queue_t replyq, xpc_handler_t handler) {
367 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection,
368 message, replyq, handler);
369 Release(thr, pc, (uptr)connection);
370 xpc_handler_t new_handler = ^(xpc_object_t object) {
371 {
372 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply);
373 Acquire(thr, pc, (uptr)connection);
374 }
375 handler(object);
376 };
377 REAL(xpc_connection_send_message_with_reply)
378 (connection, message, replyq, new_handler);
379}
380
381TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
382 SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection);
383 Release(thr, pc, (uptr)connection);
384 REAL(xpc_connection_cancel)(connection);
385}
386
387# endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
388
389// Determines whether the Obj-C object pointer is a tagged pointer. Tagged
390// pointers encode the object data directly in their pointer bits and do not
391// have an associated memory allocation. The Obj-C runtime uses tagged pointers
392// to transparently optimize small objects.
393static bool IsTaggedObjCPointer(id obj) {
394 const uptr kPossibleTaggedBits = 0x8000000000000001ull;
395 return ((uptr)obj & kPossibleTaggedBits) != 0;
396}
397
398// Returns an address which can be used to inform TSan about synchronization
399// points (MutexLock/Unlock). The TSan infrastructure expects this to be a valid
400// address in the process space. We do a small allocation here to obtain a
401// stable address (the array backing the hash map can change). The memory is
402// never free'd (leaked) and allocation and locking are slow, but this code only
403// runs for @synchronized with tagged pointers, which is very rare.
404static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) {
405 typedef AddrHashMap<uptr, 5> Map;
406 static Map Addresses;
407 Map::Handle h(&Addresses, addr);
408 if (h.created()) {
409 ThreadIgnoreBegin(thr, pc);
410 *h = (uptr)user_alloc(thr, pc, /*size=*/1);
411 ThreadIgnoreEnd(thr);
412 }
413 return *h;
414}
415
416// Returns an address on which we can synchronize given an Obj-C object pointer.
417// For normal object pointers, this is just the address of the object in memory.
418// Tagged pointers are not backed by an actual memory allocation, so we need to
419// synthesize a valid address.
420static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) {
421 if (IsTaggedObjCPointer(obj))
422 return GetOrCreateSyncAddress((uptr)obj, thr, pc);
423 return (uptr)obj;
424}
425
426TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) {
427 SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj);
428 if (!obj)
429 return REAL(objc_sync_enter)(obj);
430 uptr addr = SyncAddressForObjCObject(obj, thr, pc);
431 MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant);
432 int result = REAL(objc_sync_enter)(obj);
433 CHECK_EQ(result, OBJC_SYNC_SUCCESS);
434 MutexPostLock(thr, pc, addr, MutexFlagWriteReentrant);
435 return result;
436}
437
438TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) {
439 SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj);
440 if (!obj)
441 return REAL(objc_sync_exit)(obj);
442 uptr addr = SyncAddressForObjCObject(obj, thr, pc);
443 MutexUnlock(thr, pc, addr);
444 int result = REAL(objc_sync_exit)(obj);
445 if (result != OBJC_SYNC_SUCCESS)
446 MutexInvalidAccess(thr, pc, addr);
447 return result;
448}
449
450TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) {
451 {
452 SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp);
453 }
454 // Because of swapcontext() semantics we have no option but to copy its
455 // implementation here
456 if (!oucp || !ucp) {
457 errno = EINVAL;
458 return -1;
459 }
460 ThreadState *thr = cur_thread();
461 const int UCF_SWAPPED = 0x80000000;
462 oucp->uc_onstack &= ~UCF_SWAPPED;
463 thr->ignore_interceptors++;
464 int ret = getcontext(oucp);
465 if (!(oucp->uc_onstack & UCF_SWAPPED)) {
466 thr->ignore_interceptors--;
467 if (!ret) {
468 oucp->uc_onstack |= UCF_SWAPPED;
469 ret = setcontext(ucp);
470 }
471 }
472 return ret;
473}
474
475// On macOS, libc++ is always linked dynamically, so intercepting works the
476// usual way.
477# define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
478
479namespace {
480struct fake_shared_weak_count {
481 volatile a64 shared_owners;
482 volatile a64 shared_weak_owners;
483 virtual void _unused_0x0() = 0;
484 virtual void _unused_0x8() = 0;
485 virtual void on_zero_shared() = 0;
486 virtual void _unused_0x18() = 0;
487 virtual void on_zero_shared_weak() = 0;
488 virtual ~fake_shared_weak_count() = 0; // suppress -Wnon-virtual-dtor
489};
490} // namespace
491
492// The following code adds libc++ interceptors for:
493// void __shared_weak_count::__release_shared() _NOEXCEPT;
494// bool __shared_count::__release_shared() _NOEXCEPT;
495// Shared and weak pointers in C++ maintain reference counts via atomics in
496// libc++.dylib, which are TSan-invisible, and this leads to false positives in
497// destructor code. These interceptors re-implements the whole functions so that
498// the mo_acq_rel semantics of the atomic decrement are visible.
499//
500// Unfortunately, the interceptors cannot simply Acquire/Release some sync
501// object and call the original function, because it would have a race between
502// the sync and the destruction of the object. Calling both under a lock will
503// not work because the destructor can invoke this interceptor again (and even
504// in a different thread, so recursive locks don't help).
505
506STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv,
507 fake_shared_weak_count *o) {
508 if (!flags()->shared_ptr_interceptor)
509 return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o);
510
511 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv,
512 o);
513 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
514 Acquire(thr, pc, (uptr)&o->shared_owners);
515 o->on_zero_shared();
516 if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) ==
517 0) {
518 Acquire(thr, pc, (uptr)&o->shared_weak_owners);
519 o->on_zero_shared_weak();
520 }
521 }
522}
523
524STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv,
525 fake_shared_weak_count *o) {
526 if (!flags()->shared_ptr_interceptor)
527 return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o);
528
529 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o);
530 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
531 Acquire(thr, pc, (uptr)&o->shared_owners);
532 o->on_zero_shared();
533 return true;
534 }
535 return false;
536}
537
538namespace {
539struct call_once_callback_args {
540 void (*orig_func)(void *arg);
541 void *orig_arg;
542 void *flag;
543};
544
545void call_once_callback_wrapper(void *arg) {
546 call_once_callback_args *new_args = (call_once_callback_args *)arg;
547 new_args->orig_func(new_args->orig_arg);
548 __tsan_release(new_args->flag);
549}
550} // namespace
551
552// This adds a libc++ interceptor for:
553// void __call_once(volatile unsigned long&, void*, void(*)(void*));
554// C++11 call_once is implemented via an internal function __call_once which is
555// inside libc++.dylib, and the atomic release store inside it is thus
556// TSan-invisible. To avoid false positives, this interceptor wraps the callback
557// function and performs an explicit Release after the user code has run.
558STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag,
559 void *arg, void (*func)(void *arg)) {
560 call_once_callback_args new_args = {func, arg, flag};
561 REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args,
562 call_once_callback_wrapper);
563}
564
565} // namespace __tsan
566
567#endif // SANITIZER_APPLE