From 6498f963bc7ecf275173b82ec4da67046116db1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20R=C3=B8nne=20Petersen?= Date: Mon, 3 Aug 2026 13:06:07 +0200 Subject: [PATCH] libtsan: update to LLVM 23 --- lib/libtsan/interception/interception.h | 10 + .../interception/interception_linux.cpp | 63 ++- lib/libtsan/interception/interception_win.cpp | 25 + .../sanitizer_common/sanitizer_allocator.cpp | 5 +- .../sanitizer_common/sanitizer_allocator.h | 6 - .../sanitizer_allocator_dlsym.h | 10 +- .../sanitizer_allocator_internal.h | 5 +- lib/libtsan/sanitizer_common/sanitizer_asm.h | 6 +- .../sanitizer_common/sanitizer_common.h | 23 +- .../sanitizer_common_interceptors.inc | 67 ++- .../sanitizer_common_interface.inc | 1 - .../sanitizer_common/sanitizer_dense_map.h | 152 +++--- .../sanitizer_dense_map_info.h | 25 - .../sanitizer_common/sanitizer_errno.h | 2 + .../sanitizer_common/sanitizer_flag_parser.h | 4 +- .../sanitizer_common/sanitizer_fuchsia.cpp | 4 +- .../sanitizer_common/sanitizer_haiku.cpp | 4 + .../sanitizer_interface_internal.h | 5 - .../sanitizer_internal_defs.h | 28 +- .../sanitizer_common/sanitizer_linux.cpp | 93 +++- .../sanitizer_common/sanitizer_linux.h | 3 +- .../sanitizer_linux_libcdep.cpp | 8 +- lib/libtsan/sanitizer_common/sanitizer_lzw.h | 4 +- .../sanitizer_common/sanitizer_mac.cpp | 32 +- .../sanitizer_common/sanitizer_netbsd.cpp | 4 + .../sanitizer_common/sanitizer_platform.h | 35 +- .../sanitizer_platform_interceptors.h | 7 +- .../sanitizer_platform_limits_linux.cpp | 2 +- .../sanitizer_platform_limits_posix.cpp | 82 +++- .../sanitizer_platform_limits_posix.h | 82 +++- .../sanitizer_platform_limits_solaris.cpp | 1 + .../sanitizer_common/sanitizer_posix.cpp | 16 +- .../sanitizer_common/sanitizer_posix.h | 4 +- .../sanitizer_posix_libcdep.cpp | 21 +- .../sanitizer_redefine_builtins.h | 22 +- .../sanitizer_common/sanitizer_solaris.cpp | 4 + .../sanitizer_stoptheworld_linux_libcdep.cpp | 14 +- .../sanitizer_symbolizer_posix_libcdep.cpp | 14 +- .../sanitizer_symbolizer_report.cpp | 13 +- .../sanitizer_common/sanitizer_unwind_win.cpp | 49 +- .../sanitizer_common/sanitizer_win.cpp | 7 +- lib/libtsan/tsan_adaptive_delay.cpp | 433 ++++++++++++++++++ lib/libtsan/tsan_adaptive_delay.h | 173 +++++++ lib/libtsan/tsan_flags.cpp | 2 +- lib/libtsan/tsan_flags.inc | 27 ++ lib/libtsan/tsan_interceptors_posix.cpp | 26 +- lib/libtsan/tsan_interface_ann.cpp | 14 +- lib/libtsan/tsan_interface_atomic.cpp | 12 + lib/libtsan/tsan_platform.h | 17 +- lib/libtsan/tsan_platform_linux.cpp | 12 + lib/libtsan/tsan_report.cpp | 5 +- lib/libtsan/tsan_rtl.cpp | 5 + lib/libtsan/tsan_rtl.h | 3 + lib/libtsan/ubsan/ubsan_flags.h | 4 + src/libs/libtsan.zig | 1 + 55 files changed, 1399 insertions(+), 297 deletions(-) create mode 100644 lib/libtsan/tsan_adaptive_delay.cpp create mode 100644 lib/libtsan/tsan_adaptive_delay.h diff --git a/lib/libtsan/interception/interception.h b/lib/libtsan/interception/interception.h index 9fe7d3db308bffd5f2d9a744e217583255664c95..71c59ed36b4ad1060215bc3f7bea4de5b9c0d705 100644 --- a/lib/libtsan/interception/interception.h +++ b/lib/libtsan/interception/interception.h @@ -362,6 +362,16 @@ const interpose_substitution substitution_##func_name[] \ // so we use casts via uintptr_t (the local __sanitizer::uptr equivalent). namespace __interception { +// Dynamic library loading helpers (dlopen/LoadLibrary, dlsym/GetProcAddress). +// Implemented in interception_linux.cpp on non-Windows targets and +// interception_win.cpp on Windows. +bool DynamicLoaderAvailable(); +void* OpenLibrary(const char* name); +void* LookupSymbol(void* handle, const char* symbol); +void* LookupSymbolDefault(const char* symbol); +void* LookupSymbolNext(const char* symbol); +void* LookupSymbolNextVersioned(const char* symbol, const char* version); + #if defined(__ELF__) && !SANITIZER_FUCHSIA // The use of interceptors makes many sanitizers unusable for static linking. // Define a function, if called, will cause a linker error (undefined _DYNAMIC). diff --git a/lib/libtsan/interception/interception_linux.cpp b/lib/libtsan/interception/interception_linux.cpp index f900ae6a593bda366b8255270e5b0302871449ec..5919f8c813f285cce30880af1fd916f7ca5ca604 100644 --- a/lib/libtsan/interception/interception_linux.cpp +++ b/lib/libtsan/interception/interception_linux.cpp @@ -9,15 +9,68 @@ // This file is a part of AddressSanitizer, an address sanity checker. // // Linux-specific interception methods. +// +// POSIX dynamic-library helpers (dlopen / dlsym) live here for every +// non-Windows interception target (Linux, *BSD, Darwin, AIX, Fuchsia, ...). +// macOS/AIX/Fuchsia compile this TU for RTInterception but do not use the +// Linux-specific InterceptFunction helpers below. //===----------------------------------------------------------------------===// #include "interception.h" +#if !SANITIZER_WINDOWS + +# include + +# pragma weak dlopen +# pragma weak dlsym +#if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD +# pragma weak dlvsym +#endif + +namespace __interception { + +bool DynamicLoaderAvailable() { return dlopen != nullptr && dlsym != nullptr; } + +void* OpenLibrary(const char* name) { + if (!DynamicLoaderAvailable()) + return nullptr; + return dlopen(name, RTLD_LAZY | RTLD_LOCAL); +} + +void* LookupSymbol(void* handle, const char* symbol) { + if (!DynamicLoaderAvailable()) + return nullptr; + return dlsym(handle, symbol); +} + +void* LookupSymbolDefault(const char* symbol) { + if (!DynamicLoaderAvailable()) + return nullptr; + return dlsym(RTLD_DEFAULT, symbol); +} + +void* LookupSymbolNext(const char* symbol) { + if (!DynamicLoaderAvailable()) + return nullptr; + return dlsym(RTLD_NEXT, symbol); +} + +#if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD +void* LookupSymbolNextVersioned(const char* symbol, const char* version) { + if (!DynamicLoaderAvailable() || dlvsym == nullptr) + return nullptr; + return dlvsym(RTLD_NEXT, symbol, version); +} +#endif // SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD + +} // namespace __interception + +#endif // !SANITIZER_WINDOWS + #if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \ SANITIZER_SOLARIS || SANITIZER_HAIKU -#include // for dlsym() and dlvsym() - namespace __interception { #if SANITIZER_NETBSD @@ -39,14 +92,14 @@ static void *GetFuncAddr(const char *name, uptr trampoline) { if (StrCmp(name, "sigaction")) name = "__sigaction14"; #endif - void *addr = dlsym(RTLD_NEXT, name); + void* addr = LookupSymbolNext(name); if (!addr) { // If the lookup using RTLD_NEXT failed, the sanitizer runtime library is // later in the library search order than the DSO that we are trying to // intercept, which means that we cannot intercept this function. We still // want the address of the real definition, though, so look it up using // RTLD_DEFAULT. - addr = dlsym(RTLD_DEFAULT, name); + addr = LookupSymbolDefault(name); // In case `name' is not loaded, dlsym ends up finding the actual wrapper. // We don't want to intercept the wrapper and have it point to itself. @@ -66,7 +119,7 @@ bool InterceptFunction(const char *name, uptr *ptr_to_real, uptr func, // dlvsym is a GNU extension supported by some other platforms. #if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD static void *GetFuncAddr(const char *name, const char *ver) { - return dlvsym(RTLD_NEXT, name, ver); + return LookupSymbolNextVersioned(name, ver); } bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real, diff --git a/lib/libtsan/interception/interception_win.cpp b/lib/libtsan/interception/interception_win.cpp index 856872425117abfda4dee3a433fce48e27cad8f0..19bccb3566d7287ee6bac5538612b2a0d2b9c568 100644 --- a/lib/libtsan/interception/interception_win.cpp +++ b/lib/libtsan/interception/interception_win.cpp @@ -134,6 +134,30 @@ namespace __interception { +bool DynamicLoaderAvailable() { return true; } + +void* OpenLibrary(const char* name) { + if (!name) + return reinterpret_cast(GetModuleHandleA(nullptr)); + return reinterpret_cast(LoadLibraryA(name)); +} + +void* LookupSymbol(void* handle, const char* symbol) { + if (!handle) + return nullptr; + return reinterpret_cast(reinterpret_cast<__sanitizer::uptr>( + GetProcAddress(reinterpret_cast(handle), symbol))); +} + +void* LookupSymbolDefault(const char* symbol) { + return LookupSymbol(reinterpret_cast(GetModuleHandleA(nullptr)), + symbol); +} + +void* LookupSymbolNext(const char*) { return nullptr; } + +void* LookupSymbolNextVersioned(const char*, const char*) { return nullptr; } + static const int kAddressLength = FIRST_32_SECOND_64(4, 8); static const int kJumpInstructionLength = 5; static const int kShortJumpInstructionLength = 2; @@ -655,6 +679,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { return 2; case 0x3980: // 80 39 XX : cmp BYTE PTR [rcx], XX + case 0x3a80: // 80 3A XX : cmp BYTE PTR [rdx], XX case 0x4D8B: // 8B 4D XX : mov XX(%ebp), ecx case 0x558B: // 8B 55 XX : mov XX(%ebp), edx case 0x758B: // 8B 75 XX : mov XX(%ebp), esp diff --git a/lib/libtsan/sanitizer_common/sanitizer_allocator.cpp b/lib/libtsan/sanitizer_common/sanitizer_allocator.cpp index 9d899371c2dd90feeed3038c7ffbd7e19cb472bd..743f2b764f660efcf36f56fcbf4926add4ae16a4 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_allocator.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_allocator.cpp @@ -109,14 +109,15 @@ void *InternalReallocArray(void *addr, uptr count, uptr size, return InternalRealloc(addr, count * size, cache); } -void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) { +void* InternalCalloc(uptr count, uptr size, InternalAllocatorCache* cache, + uptr alignment) { if (UNLIKELY(CheckForCallocOverflow(count, size))) { Report("FATAL: %s: calloc parameters overflow: count * size (%zd * %zd) " "cannot be represented in type size_t\n", SanitizerToolName, count, size); Die(); } - void *p = InternalAlloc(count * size, cache); + void* p = InternalAlloc(count * size, cache, alignment); if (LIKELY(p)) internal_memset(p, 0, count * size); return p; diff --git a/lib/libtsan/sanitizer_common/sanitizer_allocator.h b/lib/libtsan/sanitizer_common/sanitizer_allocator.h index 0b28f86d140842df908ee69ba77362390ddc1d94..6154f7810334b5362ee160f0102857afd418092a 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_allocator.h +++ b/lib/libtsan/sanitizer_common/sanitizer_allocator.h @@ -47,12 +47,6 @@ void PrintHintAllocatorCannotReturnNull(); // Callback type for iterating over chunks. typedef void (*ForEachChunkCallback)(uptr chunk, void *arg); -inline u32 Rand(u32 *state) { // ANSI C linear congruential PRNG. - return (*state = *state * 1103515245 + 12345) >> 16; -} - -inline u32 RandN(u32 *state, u32 n) { return Rand(state) % n; } // [0, n) - template inline void RandomShuffle(T *a, u32 n, u32 *rand_state) { if (n <= 1) return; diff --git a/lib/libtsan/sanitizer_common/sanitizer_allocator_dlsym.h b/lib/libtsan/sanitizer_common/sanitizer_allocator_dlsym.h index 6e6cdbd9eeaed066366f5d0bdd2838ad5410c8fc..2e8e2b12156aa971c452c8eda3dc5dbb1c29c4c8 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_allocator_dlsym.h +++ b/lib/libtsan/sanitizer_common/sanitizer_allocator_dlsym.h @@ -40,8 +40,8 @@ struct DlSymAllocator { return ptr; } - static void *Callocate(usize nmemb, usize size) { - void *ptr = InternalCalloc(nmemb, size); + static void* Callocate(usize nmemb, usize size, uptr align = kWordSize) { + void* ptr = InternalCalloc(nmemb, size, nullptr, align); CHECK(internal_allocator()->FromPrimary(ptr)); Details::OnAllocate(ptr, GetSize(ptr)); return ptr; @@ -53,9 +53,9 @@ struct DlSymAllocator { InternalFree(ptr); } - static void *Realloc(void *ptr, uptr new_size) { + static void* Realloc(void* ptr, uptr new_size, uptr align = kWordSize) { if (!ptr) - return Allocate(new_size); + return Allocate(new_size, align); CHECK(internal_allocator()->FromPrimary(ptr)); if (!new_size) { Free(ptr); @@ -63,7 +63,7 @@ struct DlSymAllocator { } uptr size = GetSize(ptr); uptr memcpy_size = Min(new_size, size); - void *new_ptr = Allocate(new_size); + void* new_ptr = Allocate(new_size, align); if (new_ptr) internal_memcpy(new_ptr, ptr, memcpy_size); Free(ptr); diff --git a/lib/libtsan/sanitizer_common/sanitizer_allocator_internal.h b/lib/libtsan/sanitizer_common/sanitizer_allocator_internal.h index 6c2020e6373c04601595b07a2cc4fd3c14766420..1e0daf88352133f1ed1aaf7a3ae87579fd71ea72 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_allocator_internal.h +++ b/lib/libtsan/sanitizer_common/sanitizer_allocator_internal.h @@ -45,8 +45,9 @@ void *InternalRealloc(void *p, uptr size, InternalAllocatorCache *cache = nullptr); void *InternalReallocArray(void *p, uptr count, uptr size, InternalAllocatorCache *cache = nullptr); -void *InternalCalloc(uptr count, uptr size, - InternalAllocatorCache *cache = nullptr); +void* InternalCalloc(uptr count, uptr size, + InternalAllocatorCache* cache = nullptr, + uptr alignment = 0); void InternalFree(void *p, InternalAllocatorCache *cache = nullptr); void InternalAllocatorLock(); void InternalAllocatorUnlock(); diff --git a/lib/libtsan/sanitizer_common/sanitizer_asm.h b/lib/libtsan/sanitizer_common/sanitizer_asm.h index 30e9d15184e5d55690576306ec8eadde93424d80..d12beafccd021f329fb8151b4a9ab855086d0f45 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_asm.h +++ b/lib/libtsan/sanitizer_common/sanitizer_asm.h @@ -61,6 +61,8 @@ # define ASM_TAIL_CALL jg #elif defined(__riscv) # define ASM_TAIL_CALL tail +#elif defined(__hexagon__) +# define ASM_TAIL_CALL jump #endif // Currently, almost all of the shared libraries rely on the value of @@ -103,8 +105,8 @@ # define ASM_SIZE(symbol) .size symbol, .-symbol # define ASM_SYMBOL(symbol) symbol # define ASM_SYMBOL_INTERCEPTOR(symbol) symbol -# if defined(__i386__) || defined(__powerpc__) || defined(__s390__) || \ - defined(__sparc__) +# if defined(__i386__) || defined(__powerpc__) || defined(__s390__) || \ + defined(__sparc__) || defined(__alpha__) // For details, see interception.h # define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol # define ASM_TRAMPOLINE_ALIAS(symbol, name) \ diff --git a/lib/libtsan/sanitizer_common/sanitizer_common.h b/lib/libtsan/sanitizer_common/sanitizer_common.h index 515a7c9cdf60fbebf16f454c9cb879499538ad84..6f76d10a28cf6e62f4a76a67d65fc7a13d40c9f1 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_common.h +++ b/lib/libtsan/sanitizer_common/sanitizer_common.h @@ -387,8 +387,8 @@ void ReportDeadlySignal(const SignalContext &sig, u32 tid, const void *unwind_context); // Alternative signal stack (POSIX-only). -void SetAlternateSignalStack(); -void UnsetAlternateSignalStack(); +void* SetAlternateSignalStack(); +void UnsetAlternateSignalStack(void* altstack_base); bool IsSignalHandlerFromSanitizer(int signum); bool SetSignalHandlerFromSanitizer(int signum, bool new_state); @@ -906,7 +906,14 @@ class LoadedModule { class ListOfModules { public: ListOfModules() : initialized(false) {} - ~ListOfModules() { clear(); } + ~ListOfModules() { + clear(); + if (initialized) + modules_.Destroy(); + } + ListOfModules(const ListOfModules&) = delete; + ListOfModules& operator=(const ListOfModules&) = delete; + void init(); void fallbackInit(); // Uses fallback init if available, otherwise clears const LoadedModule *begin() const { return modules_.begin(); } @@ -1085,7 +1092,9 @@ struct StackDepotStats { // indicate that sanitizer allocator should not attempt to release memory to OS. const s32 kReleaseToOSIntervalNever = -1; -void CheckNoDeepBind(const char *filename, int flag); +// Platform hook invoked before dlopen. Performs platform-specific dlopen flag +// checks (e.g. RTLD_DEEPBIND on Linux). +void OnDlOpen(const char* filename, int flag); // Returns the requested amount of random data (up to 256 bytes) that can then // be used to seed a PRNG. Defaults to blocking like the underlying syscall. @@ -1100,6 +1109,12 @@ inline u32 GetNumberOfCPUsCached() { return NumberOfCPUsCached; } +inline u32 Rand(u32* state) { // ANSI C linear congruential PRNG. + return (*state = *state * 1103515245 + 12345) >> 16; +} + +inline u32 RandN(u32* state, u32 n) { return Rand(state) % n; } // [0, n) + } // namespace __sanitizer inline void *operator new(__sanitizer::usize size, diff --git a/lib/libtsan/sanitizer_common/sanitizer_common_interceptors.inc b/lib/libtsan/sanitizer_common/sanitizer_common_interceptors.inc index b10ce7fa44afc3d86629637e42251b4b752c7a15..caf5a039263f0cc071f620bdf73b37405763df09 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_common_interceptors.inc +++ b/lib/libtsan/sanitizer_common/sanitizer_common_interceptors.inc @@ -277,8 +277,11 @@ extern const short *_tolower_tab_; common_flags()->strict_string_checks ? (internal_strlen(s)) + 1 : (n) ) #ifndef COMMON_INTERCEPTOR_DLOPEN -#define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \ - ({ CheckNoDeepBind(filename, flag); REAL(dlopen)(filename, flag); }) +# define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \ + ({ \ + OnDlOpen(filename, flag); \ + REAL(dlopen)(filename, flag); \ + }) #endif #ifndef COMMON_INTERCEPTOR_GET_TLS_RANGE @@ -1023,6 +1026,25 @@ INTERCEPTOR(SSIZE_T, read, int fd, void *ptr, SIZE_T count) { #define INIT_READ #endif +#if SANITIZER_INTERCEPT___READ_CHK +INTERCEPTOR(SSIZE_T, __read_chk, int fd, void* ptr, SIZE_T count, + SIZE_T buflen) { + void* ctx; + COMMON_INTERCEPTOR_ENTER(ctx, __read_chk, fd, ptr, count, buflen); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + SSIZE_T res = + COMMON_INTERCEPTOR_BLOCK_REAL(__read_chk)(fd, ptr, count, buflen); + if (res > 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res); + if (res >= 0 && fd >= 0) + COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); + return res; +} +# define INIT___READ_CHK COMMON_INTERCEPT_FUNCTION(__read_chk) +#else +# define INIT___READ_CHK +#endif + #if SANITIZER_INTERCEPT_FREAD INTERCEPTOR(SIZE_T, fread, void *ptr, SIZE_T size, SIZE_T nmemb, void *file) { // libc file streams can call user-supplied functions, see fopencookie. @@ -1058,6 +1080,25 @@ INTERCEPTOR(SSIZE_T, pread, int fd, void *ptr, SIZE_T count, OFF_T offset) { #define INIT_PREAD #endif +#if SANITIZER_INTERCEPT___PREAD_CHK +INTERCEPTOR(SSIZE_T, __pread_chk, int fd, void* ptr, SIZE_T count, OFF_T offset, + SIZE_T buflen) { + void* ctx; + COMMON_INTERCEPTOR_ENTER(ctx, __pread_chk, fd, ptr, count, offset, buflen); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(__pread_chk)(fd, ptr, count, + offset, buflen); + if (res > 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res); + if (res >= 0 && fd >= 0) + COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); + return res; +} +# define INIT___PREAD_CHK COMMON_INTERCEPT_FUNCTION(__pread_chk) +#else +# define INIT___PREAD_CHK +#endif + #if SANITIZER_INTERCEPT_PREAD64 INTERCEPTOR(SSIZE_T, pread64, int fd, void *ptr, SIZE_T count, OFF64_T offset) { void *ctx; @@ -1076,6 +1117,25 @@ INTERCEPTOR(SSIZE_T, pread64, int fd, void *ptr, SIZE_T count, OFF64_T offset) { #define INIT_PREAD64 #endif +#if SANITIZER_INTERCEPT___PREAD64_CHK +INTERCEPTOR(SSIZE_T, __pread64_chk, int fd, void* ptr, SIZE_T count, + OFF64_T offset, SIZE_T buflen) { + void* ctx; + COMMON_INTERCEPTOR_ENTER(ctx, __pread64_chk, fd, ptr, count, offset, buflen); + COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); + SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(__pread64_chk)(fd, ptr, count, + offset, buflen); + if (res > 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res); + if (res >= 0 && fd >= 0) + COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); + return res; +} +# define INIT___PREAD64_CHK COMMON_INTERCEPT_FUNCTION(__pread64_chk) +#else +# define INIT___PREAD64_CHK +#endif + #if SANITIZER_INTERCEPT_READV INTERCEPTOR_WITH_SUFFIX(SSIZE_T, readv, int fd, __sanitizer_iovec *iov, int iovcnt) { @@ -10428,9 +10488,12 @@ static void InitializeCommonInterceptors() { INIT_MEMRCHR; INIT_MEMMEM; INIT_READ; + INIT___READ_CHK; INIT_FREAD; INIT_PREAD; + INIT___PREAD_CHK; INIT_PREAD64; + INIT___PREAD64_CHK; INIT_READV; INIT_PREADV; INIT_PREADV64; diff --git a/lib/libtsan/sanitizer_common/sanitizer_common_interface.inc b/lib/libtsan/sanitizer_common/sanitizer_common_interface.inc index 4ea75cdd67cb93820af0a3b621bd54bf828eefac..66744aa021e62b09b24bf8b4de569db1664a151d 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_common_interface.inc +++ b/lib/libtsan/sanitizer_common/sanitizer_common_interface.inc @@ -10,7 +10,6 @@ INTERFACE_FUNCTION(__sanitizer_acquire_crash_state) INTERFACE_FUNCTION(__sanitizer_annotate_contiguous_container) INTERFACE_FUNCTION(__sanitizer_annotate_double_ended_contiguous_container) -INTERFACE_FUNCTION(__sanitizer_copy_contiguous_container_annotations) INTERFACE_FUNCTION(__sanitizer_contiguous_container_find_bad_address) INTERFACE_FUNCTION( __sanitizer_double_ended_contiguous_container_find_bad_address) diff --git a/lib/libtsan/sanitizer_common/sanitizer_dense_map.h b/lib/libtsan/sanitizer_common/sanitizer_dense_map.h index c63788653de756ec6f134c35ad8a07c2db8fc69f..9dc196de851f39cb944f86b497607d54ca6c1d10 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_dense_map.h +++ b/lib/libtsan/sanitizer_common/sanitizer_dense_map.h @@ -44,10 +44,10 @@ class DenseMapBase { } void clear() { - if (getNumEntries() == 0 && getNumTombstones() == 0) + if (getNumEntries() == 0) return; - const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey(); + const KeyT EmptyKey = getEmptyKey(); if (__sanitizer::is_trivially_destructible::value) { // Use a simpler loop when values don't need destruction. for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) @@ -56,17 +56,14 @@ class DenseMapBase { unsigned NumEntries = getNumEntries(); for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) { if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey)) { - if (!KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) { - P->getSecond().~ValueT(); - --NumEntries; - } + P->getSecond().~ValueT(); + --NumEntries; P->getFirst() = EmptyKey; } } CHECK_EQ(NumEntries, 0); } setNumEntries(0); - setNumTombstones(0); } /// Return true if the specified key is in the map, false otherwise. @@ -171,20 +168,13 @@ class DenseMapBase { if (!TheBucket) return false; // not in map. - TheBucket->getSecond().~ValueT(); - TheBucket->getFirst() = getTombstoneKey(); - decrementNumEntries(); - incrementNumTombstones(); + eraseFromFilledBucket(TheBucket); return true; } void erase(value_type *I) { CHECK_NE(I, nullptr); - BucketT *TheBucket = &*I; - TheBucket->getSecond().~ValueT(); - TheBucket->getFirst() = getTombstoneKey(); - decrementNumEntries(); - incrementNumTombstones(); + eraseFromFilledBucket(I); } value_type &FindAndConstruct(const KeyT &Key) { @@ -214,11 +204,10 @@ class DenseMapBase { /// Function can return fast to stop the process. template void forEach(Fn fn) { - const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey(); + const KeyT EmptyKey = getEmptyKey(); for (auto *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) { const KeyT K = P->getFirst(); - if (!KeyInfoT::isEqual(K, EmptyKey) && - !KeyInfoT::isEqual(K, TombstoneKey)) { + if (!KeyInfoT::isEqual(K, EmptyKey)) { if (!fn(*P)) return; } @@ -238,10 +227,9 @@ class DenseMapBase { if (getNumBuckets() == 0) // Nothing to do. return; - const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey(); + const KeyT EmptyKey = getEmptyKey(); for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) { - if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey) && - !KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) + if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey)) P->getSecond().~ValueT(); P->getFirst().~KeyT(); } @@ -249,7 +237,6 @@ class DenseMapBase { void initEmpty() { setNumEntries(0); - setNumTombstones(0); CHECK_EQ((getNumBuckets() & (getNumBuckets() - 1)), 0); const KeyT EmptyKey = getEmptyKey(); @@ -273,10 +260,8 @@ class DenseMapBase { // Insert all the old elements. const KeyT EmptyKey = getEmptyKey(); - const KeyT TombstoneKey = getTombstoneKey(); for (BucketT *B = OldBucketsBegin, *E = OldBucketsEnd; B != E; ++B) { - if (!KeyInfoT::isEqual(B->getFirst(), EmptyKey) && - !KeyInfoT::isEqual(B->getFirst(), TombstoneKey)) { + if (!KeyInfoT::isEqual(B->getFirst(), EmptyKey)) { // Insert the key/value into the new table. BucketT *DestBucket; bool FoundVal = LookupBucketFor(B->getFirst(), DestBucket); @@ -301,7 +286,6 @@ class DenseMapBase { CHECK_EQ(getNumBuckets(), other.getNumBuckets()); setNumEntries(other.getNumEntries()); - setNumTombstones(other.getNumTombstones()); if (__sanitizer::is_trivially_copyable::value && __sanitizer::is_trivially_copyable::value) @@ -311,8 +295,7 @@ class DenseMapBase { for (uptr i = 0; i < getNumBuckets(); ++i) { ::new (&getBuckets()[i].getFirst()) KeyT(other.getBuckets()[i].getFirst()); - if (!KeyInfoT::isEqual(getBuckets()[i].getFirst(), getEmptyKey()) && - !KeyInfoT::isEqual(getBuckets()[i].getFirst(), getTombstoneKey())) + if (!KeyInfoT::isEqual(getBuckets()[i].getFirst(), getEmptyKey())) ::new (&getBuckets()[i].getSecond()) ValueT(other.getBuckets()[i].getSecond()); } @@ -329,9 +312,40 @@ class DenseMapBase { static const KeyT getEmptyKey() { return KeyInfoT::getEmptyKey(); } - static const KeyT getTombstoneKey() { return KeyInfoT::getTombstoneKey(); } - private: + /// Erase the entry at \p TheBucket and close the resulting hole via Knuth + /// TAOCP 6.4 Algorithm R: walk forward over the cluster, shifting back any + /// entry whose linear-probe chain from its home bucket passes through the + /// hole, until an empty bucket terminates the cluster. + void eraseFromFilledBucket(BucketT* TheBucket) { + TheBucket->getSecond().~ValueT(); + decrementNumEntries(); + + BucketT* BucketsPtr = getBuckets(); + const unsigned NumBuckets = getNumBuckets(); + const unsigned Mask = NumBuckets - 1; + const KeyT EmptyKey = getEmptyKey(); + unsigned I = static_cast(TheBucket - BucketsPtr); + unsigned J = I; + while (true) { + J = (J + 1) & Mask; + BucketT& BJ = BucketsPtr[J]; + if (KeyInfoT::isEqual(BJ.getFirst(), EmptyKey)) + break; + unsigned Ideal = getHashValue(BJ.getFirst()) & Mask; + // If the hole (I) lies on the linear-probe chain from the home bucket + // (Ideal) to J, shift J into the hole and make J the new hole. + if (((I - Ideal) & Mask) < ((J - Ideal) & Mask)) { + BucketT& BI = BucketsPtr[I]; + BI.getFirst() = __sanitizer::move(BJ.getFirst()); + ::new (&BI.getSecond()) ValueT(__sanitizer::move(BJ.getSecond())); + BJ.getSecond().~ValueT(); + I = J; + } + } + BucketsPtr[I].getFirst() = EmptyKey; + } + unsigned getNumEntries() const { return static_cast(this)->getNumEntries(); } @@ -344,18 +358,6 @@ class DenseMapBase { void decrementNumEntries() { setNumEntries(getNumEntries() - 1); } - unsigned getNumTombstones() const { - return static_cast(this)->getNumTombstones(); - } - - void setNumTombstones(unsigned Num) { - static_cast(this)->setNumTombstones(Num); - } - - void incrementNumTombstones() { setNumTombstones(getNumTombstones() + 1); } - - void decrementNumTombstones() { setNumTombstones(getNumTombstones() - 1); } - const BucketT *getBuckets() const { return static_cast(this)->getBuckets(); } @@ -398,25 +400,16 @@ class DenseMapBase { template BucketT *InsertIntoBucketImpl(const KeyT &Key, const LookupKeyT &Lookup, BucketT *TheBucket) { - // If the load of the hash table is more than 3/4, or if fewer than 1/8 of - // the buckets are empty (meaning that many are filled with tombstones), - // grow the table. - // - // The later case is tricky. For example, if we had one empty bucket with - // tons of tombstones, failing lookups (e.g. for insertion) would have to - // probe almost the entire table until it found the empty bucket. If the - // table completely filled with tombstones, no lookup would ever succeed, - // causing infinite loops in lookup. + // Grow the table if the load factor would exceed 3/4 after insertion. + // Linear probing with gap-closing deletion (Knuth Algorithm R) keeps every + // chain compact and bounded by the table's empty-bucket count, so no + // tombstone-driven resize is needed. unsigned NewNumEntries = getNumEntries() + 1; unsigned NumBuckets = getNumBuckets(); if (UNLIKELY(NewNumEntries * 4 >= NumBuckets * 3)) { this->grow(NumBuckets * 2); LookupBucketFor(Lookup, TheBucket); NumBuckets = getNumBuckets(); - } else if (UNLIKELY(NumBuckets - (NewNumEntries + getNumTombstones()) <= - NumBuckets / 8)) { - this->grow(NumBuckets); - LookupBucketFor(Lookup, TheBucket); } CHECK(TheBucket); @@ -424,11 +417,6 @@ class DenseMapBase { // so that when growing buckets we have self-consistent entry count. incrementNumEntries(); - // If we are writing over a tombstone, remember this. - const KeyT EmptyKey = getEmptyKey(); - if (!KeyInfoT::isEqual(TheBucket->getFirst(), EmptyKey)) - decrementNumTombstones(); - return TheBucket; } @@ -441,7 +429,6 @@ class DenseMapBase { const KeyT EmptyKey = getEmptyKey(); unsigned BucketNo = getHashValue(Val) & (NumBuckets - 1); - unsigned ProbeAmt = 1; while (true) { BucketT *Bucket = BucketsPtr + BucketNo; if (LIKELY(KeyInfoT::isEqual(Val, Bucket->getFirst()))) @@ -449,10 +436,8 @@ class DenseMapBase { if (LIKELY(KeyInfoT::isEqual(Bucket->getFirst(), EmptyKey))) return nullptr; - // Otherwise, it's a hash collision or a tombstone, continue quadratic - // probing. - BucketNo += ProbeAmt++; - BucketNo &= NumBuckets - 1; + // Hash collision: continue linear probing. + BucketNo = (BucketNo + 1) & (NumBuckets - 1); } } @@ -463,8 +448,8 @@ class DenseMapBase { /// LookupBucketFor - Lookup the appropriate bucket for Val, returning it in /// FoundBucket. If the bucket contains the key and a value, this returns - /// true, otherwise it returns a bucket with an empty marker or tombstone and - /// returns false. + /// true, otherwise it returns a bucket with an empty marker and returns + /// false. template bool LookupBucketFor(const LookupKeyT &Val, const BucketT *&FoundBucket) const { @@ -476,15 +461,10 @@ class DenseMapBase { return false; } - // FoundTombstone - Keep track of whether we find a tombstone while probing. - const BucketT *FoundTombstone = nullptr; const KeyT EmptyKey = getEmptyKey(); - const KeyT TombstoneKey = getTombstoneKey(); CHECK(!KeyInfoT::isEqual(Val, EmptyKey)); - CHECK(!KeyInfoT::isEqual(Val, TombstoneKey)); unsigned BucketNo = getHashValue(Val) & (NumBuckets - 1); - unsigned ProbeAmt = 1; while (true) { const BucketT *ThisBucket = BucketsPtr + BucketNo; // Found Val's bucket? If so, return it. @@ -494,24 +474,14 @@ class DenseMapBase { } // If we found an empty bucket, the key doesn't exist in the set. - // Insert it and return the default value. + // Return it as the insertion point. if (LIKELY(KeyInfoT::isEqual(ThisBucket->getFirst(), EmptyKey))) { - // If we've already seen a tombstone while probing, fill it in instead - // of the empty bucket we eventually probed to. - FoundBucket = FoundTombstone ? FoundTombstone : ThisBucket; + FoundBucket = ThisBucket; return false; } - // If this is a tombstone, remember it. If Val ends up not in the map, we - // prefer to return it than something that would require more probing. - if (KeyInfoT::isEqual(ThisBucket->getFirst(), TombstoneKey) && - !FoundTombstone) - FoundTombstone = ThisBucket; // Remember the first tombstone found. - - // Otherwise, it's a hash collision or a tombstone, continue quadratic - // probing. - BucketNo += ProbeAmt++; - BucketNo &= (NumBuckets - 1); + // Hash collision: continue linear probing. + BucketNo = (BucketNo + 1) & (NumBuckets - 1); } } @@ -587,7 +557,6 @@ class DenseMap : public DenseMapBase, BucketT *Buckets = nullptr; unsigned NumEntries = 0; - unsigned NumTombstones = 0; unsigned NumBuckets = 0; public: @@ -614,7 +583,6 @@ class DenseMap : public DenseMapBase, void swap(DenseMap &RHS) { Swap(Buckets, RHS.Buckets); Swap(NumEntries, RHS.NumEntries); - Swap(NumTombstones, RHS.NumTombstones); Swap(NumBuckets, RHS.NumBuckets); } @@ -639,7 +607,6 @@ class DenseMap : public DenseMapBase, this->BaseT::copyFrom(other); } else { NumEntries = 0; - NumTombstones = 0; } } @@ -649,7 +616,6 @@ class DenseMap : public DenseMapBase, this->BaseT::initEmpty(); } else { NumEntries = 0; - NumTombstones = 0; } } @@ -675,10 +641,6 @@ class DenseMap : public DenseMapBase, void setNumEntries(unsigned Num) { NumEntries = Num; } - unsigned getNumTombstones() const { return NumTombstones; } - - void setNumTombstones(unsigned Num) { NumTombstones = Num; } - BucketT *getBuckets() const { return Buckets; } unsigned getNumBuckets() const { return NumBuckets; } diff --git a/lib/libtsan/sanitizer_common/sanitizer_dense_map_info.h b/lib/libtsan/sanitizer_common/sanitizer_dense_map_info.h index f4640369ae588a7f79593eb558c1a7779a42e888..a3c7b6d9571ed00f2ce4bf289c229996d2117382 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_dense_map_info.h +++ b/lib/libtsan/sanitizer_common/sanitizer_dense_map_info.h @@ -62,7 +62,6 @@ struct DenseMapPair { template struct DenseMapInfo { // static T getEmptyKey(); - // static T getTombstoneKey(); // static unsigned getHashValue(const T &Val); // static bool isEqual(const T &LHS, const T &RHS); }; @@ -86,12 +85,6 @@ struct DenseMapInfo { return reinterpret_cast(Val); } - static constexpr T *getTombstoneKey() { - uptr Val = static_cast(-2); - Val <<= Log2MaxAlign; - return reinterpret_cast(Val); - } - static constexpr unsigned getHashValue(const T *PtrVal) { return (unsigned((uptr)PtrVal) >> 4) ^ (unsigned((uptr)PtrVal) >> 9); } @@ -105,7 +98,6 @@ struct DenseMapInfo { template <> struct DenseMapInfo { static constexpr char getEmptyKey() { return ~0; } - static constexpr char getTombstoneKey() { return ~0 - 1; } static constexpr unsigned getHashValue(const char &Val) { return Val * 37U; } static constexpr bool isEqual(const char &LHS, const char &RHS) { @@ -117,7 +109,6 @@ struct DenseMapInfo { template <> struct DenseMapInfo { static constexpr unsigned char getEmptyKey() { return ~0; } - static constexpr unsigned char getTombstoneKey() { return ~0 - 1; } static constexpr unsigned getHashValue(const unsigned char &Val) { return Val * 37U; } @@ -132,7 +123,6 @@ struct DenseMapInfo { template <> struct DenseMapInfo { static constexpr unsigned short getEmptyKey() { return 0xFFFF; } - static constexpr unsigned short getTombstoneKey() { return 0xFFFF - 1; } static constexpr unsigned getHashValue(const unsigned short &Val) { return Val * 37U; } @@ -147,7 +137,6 @@ struct DenseMapInfo { template <> struct DenseMapInfo { static constexpr unsigned getEmptyKey() { return ~0U; } - static constexpr unsigned getTombstoneKey() { return ~0U - 1; } static constexpr unsigned getHashValue(const unsigned &Val) { return Val * 37U; } @@ -161,7 +150,6 @@ struct DenseMapInfo { template <> struct DenseMapInfo { static constexpr unsigned long getEmptyKey() { return ~0UL; } - static constexpr unsigned long getTombstoneKey() { return ~0UL - 1L; } static constexpr unsigned getHashValue(const unsigned long &Val) { return (unsigned)(Val * 37UL); @@ -177,7 +165,6 @@ struct DenseMapInfo { template <> struct DenseMapInfo { static constexpr unsigned long long getEmptyKey() { return ~0ULL; } - static constexpr unsigned long long getTombstoneKey() { return ~0ULL - 1ULL; } static constexpr unsigned getHashValue(const unsigned long long &Val) { return (unsigned)(Val * 37ULL); @@ -193,7 +180,6 @@ struct DenseMapInfo { template <> struct DenseMapInfo { static constexpr short getEmptyKey() { return 0x7FFF; } - static constexpr short getTombstoneKey() { return -0x7FFF - 1; } static constexpr unsigned getHashValue(const short &Val) { return Val * 37U; } static constexpr bool isEqual(const short &LHS, const short &RHS) { return LHS == RHS; @@ -204,7 +190,6 @@ struct DenseMapInfo { template <> struct DenseMapInfo { static constexpr int getEmptyKey() { return 0x7fffffff; } - static constexpr int getTombstoneKey() { return -0x7fffffff - 1; } static constexpr unsigned getHashValue(const int &Val) { return (unsigned)(Val * 37U); } @@ -221,8 +206,6 @@ struct DenseMapInfo { return (1UL << (sizeof(long) * 8 - 1)) - 1UL; } - static constexpr long getTombstoneKey() { return getEmptyKey() - 1L; } - static constexpr unsigned getHashValue(const long &Val) { return (unsigned)(Val * 37UL); } @@ -236,9 +219,6 @@ struct DenseMapInfo { template <> struct DenseMapInfo { static constexpr long long getEmptyKey() { return 0x7fffffffffffffffLL; } - static constexpr long long getTombstoneKey() { - return -0x7fffffffffffffffLL - 1; - } static constexpr unsigned getHashValue(const long long &Val) { return (unsigned)(Val * 37ULL); @@ -261,11 +241,6 @@ struct DenseMapInfo> { SecondInfo::getEmptyKey()); } - static constexpr Pair getTombstoneKey() { - return detail::DenseMapPair(FirstInfo::getTombstoneKey(), - SecondInfo::getTombstoneKey()); - } - static constexpr unsigned getHashValue(const Pair &PairVal) { return detail::combineHashValue(FirstInfo::getHashValue(PairVal.first), SecondInfo::getHashValue(PairVal.second)); diff --git a/lib/libtsan/sanitizer_common/sanitizer_errno.h b/lib/libtsan/sanitizer_common/sanitizer_errno.h index 76919da57d942ac543c841aa9c046812d4a76b5a..0d2a9307cde6fbc8ae4348b468283ba67cdafeec 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_errno.h +++ b/lib/libtsan/sanitizer_common/sanitizer_errno.h @@ -31,6 +31,8 @@ # define __errno_location _errno #elif SANITIZER_HAIKU # define __errno_location _errnop +#elif SANITIZER_AIX +# define __errno_location _Errno #endif extern "C" int *__errno_location(); diff --git a/lib/libtsan/sanitizer_common/sanitizer_flag_parser.h b/lib/libtsan/sanitizer_common/sanitizer_flag_parser.h index dccdee4da2bd02bcf8c681f822567851321ff2f4..3dd2c5109311650865a90966cb58e26edc38f146 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_flag_parser.h +++ b/lib/libtsan/sanitizer_common/sanitizer_flag_parser.h @@ -189,8 +189,8 @@ class FlagParser { }; template -static void RegisterFlag(FlagParser *parser, const char *name, const char *desc, - T *var) { +void RegisterFlag(FlagParser* parser, const char* name, const char* desc, + T* var) { FlagHandler *fh = new (GetGlobalLowLevelAllocator()) FlagHandler(var); parser->RegisterHandler(name, fh, desc); } diff --git a/lib/libtsan/sanitizer_common/sanitizer_fuchsia.cpp b/lib/libtsan/sanitizer_common/sanitizer_fuchsia.cpp index 3c61b60802996092824d2a8a8c0cb03b5f2aa203..4dc31f4e51efad9d3b30b247565fbbdb1a2a9f19 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_fuchsia.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_fuchsia.cpp @@ -93,8 +93,8 @@ void CheckMPROTECT() {} void PlatformPrepareForSandboxing(void *args) {} void DisableCoreDumperIfNecessary() {} void InstallDeadlySignalHandlers(SignalHandlerType handler) {} -void SetAlternateSignalStack() {} -void UnsetAlternateSignalStack() {} +void* SetAlternateSignalStack() { return nullptr; } +void UnsetAlternateSignalStack(void* altstack_base) {} bool SignalContext::IsStackOverflow() const { return false; } void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); } diff --git a/lib/libtsan/sanitizer_common/sanitizer_haiku.cpp b/lib/libtsan/sanitizer_common/sanitizer_haiku.cpp index 7c114417560e559585216dbbd004ac23d04ae36a..b48ad5e14aa03231982cb6299d5d430504ef6e91 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_haiku.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_haiku.cpp @@ -128,6 +128,10 @@ uptr internal_close(fd_t fd) { RETURN_AND_SET_ERRNO(_kern_close(fd)); } +uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags) { + return -1; // Not supported. +} + uptr internal_open(const char *filename, int flags) { CHECK(&_kern_open); RETURN_AND_SET_ERRNO(_kern_open(-1, filename, flags, 0)); diff --git a/lib/libtsan/sanitizer_common/sanitizer_interface_internal.h b/lib/libtsan/sanitizer_common/sanitizer_interface_internal.h index 387a4d87d97bf9468e01655a72f6e2dbeb933d12..c424ab1cecf94f376a13391298037261096bcfe6 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_interface_internal.h +++ b/lib/libtsan/sanitizer_common/sanitizer_interface_internal.h @@ -76,11 +76,6 @@ void __sanitizer_annotate_double_ended_contiguous_container( const void *old_container_beg, const void *old_container_end, const void *new_container_beg, const void *new_container_end); SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_copy_contiguous_container_annotations(const void *src_begin, - const void *src_end, - const void *dst_begin, - const void *dst_end); -SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_verify_contiguous_container(const void *beg, const void *mid, const void *end); SANITIZER_INTERFACE_ATTRIBUTE diff --git a/lib/libtsan/sanitizer_common/sanitizer_internal_defs.h b/lib/libtsan/sanitizer_common/sanitizer_internal_defs.h index c719e2a8ef6002927ed640f30bc8d0f6e4a66a71..c694897b6556b9b76b4dfb1ef51d6e9a4fb00529 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_internal_defs.h +++ b/lib/libtsan/sanitizer_common/sanitizer_internal_defs.h @@ -29,20 +29,24 @@ // Only use SANITIZER_*ATTRIBUTE* before the function return type! #if SANITIZER_WINDOWS -#if SANITIZER_IMPORT_INTERFACE -# define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllimport) -#else -# define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllexport) -#endif -# define SANITIZER_WEAK_ATTRIBUTE -# define SANITIZER_WEAK_IMPORT -#elif SANITIZER_GO -# define SANITIZER_INTERFACE_ATTRIBUTE -# define SANITIZER_WEAK_ATTRIBUTE +# if SANITIZER_IMPORT_INTERFACE +# define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllimport) +# else +# define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllexport) +# endif +# define SANITIZER_WEAK_ATTRIBUTE # define SANITIZER_WEAK_IMPORT #else -# define SANITIZER_INTERFACE_ATTRIBUTE __attribute__((visibility("default"))) -# define SANITIZER_WEAK_ATTRIBUTE __attribute__((weak)) +# if SANITIZER_GO +# define SANITIZER_INTERFACE_ATTRIBUTE +# define SANITIZER_WEAK_ATTRIBUTE +# elif SANITIZER_AMDGPU || SANITIZER_NVPTX +# define SANITIZER_INTERFACE_ATTRIBUTE __attribute__((visibility("hidden"))) +# define SANITIZER_WEAK_ATTRIBUTE __attribute__((weak)) +# else +# define SANITIZER_INTERFACE_ATTRIBUTE __attribute__((visibility("default"))) +# define SANITIZER_WEAK_ATTRIBUTE __attribute__((weak)) +# endif // SANITIZER_GO # if SANITIZER_APPLE # define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak_import)) # else diff --git a/lib/libtsan/sanitizer_common/sanitizer_linux.cpp b/lib/libtsan/sanitizer_common/sanitizer_linux.cpp index 58608ef72b2bcef2394832c2ed87438082229085..b2d1e9abc2e91618388a3992ab59dfd8b00daad3 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_linux.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_linux.cpp @@ -90,10 +90,18 @@ extern "C" SANITIZER_WEAK_ATTRIBUTE const char *strerrorname_np(int); # endif -# if SANITIZER_LINUX && defined(__loongarch__) +# if SANITIZER_LINUX && \ + (defined(__loongarch__) || defined(__hexagon__) || defined(__alpha__)) # include # endif +// Hexagon uses statx() instead of stat64(). glibc provides struct statx +// through , but musl does not — pull it from . +// On this musl/hexagon combination the two headers coexist without conflict. +# if SANITIZER_LINUX && defined(__hexagon__) +# include +# endif + # if SANITIZER_LINUX && defined(__powerpc64__) # include # endif @@ -254,6 +262,8 @@ ScopedBlockSignals::~ScopedBlockSignals() { SetSigProcMask(&saved_, nullptr); } # include "sanitizer_syscall_linux_hexagon.inc" # elif SANITIZER_LINUX && SANITIZER_LOONGARCH64 # include "sanitizer_syscall_linux_loongarch64.inc" +# elif SANITIZER_LINUX && SANITIZER_ALPHA +# include "sanitizer_syscall_linux_alpha.inc" # else # include "sanitizer_syscall_generic.inc" # endif @@ -296,11 +306,13 @@ int internal_madvise(uptr addr, uptr length, int advice) { return internal_syscall(SYSCALL(madvise), addr, length, advice); } -# if SANITIZER_FREEBSD uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags) { +# if SANITIZER_FREEBSD || (SANITIZER_LINUX && defined(__NR_close_range)) return internal_syscall(SYSCALL(close_range), lowfd, highfd, flags); -} # endif + return -1; // Not supported. +} + uptr internal_close(fd_t fd) { return internal_syscall(SYSCALL(close), fd); } uptr internal_open(const char *filename, int flags) { @@ -341,7 +353,8 @@ uptr internal_ftruncate(fd_t fd, uptr size) { return res; } -# if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && SANITIZER_LINUX +# if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && SANITIZER_LINUX && \ + !defined(__hexagon__) static void stat64_to_stat(struct stat64 *in, struct stat *out) { internal_memset(out, 0, sizeof(*out)); out->st_dev = in->st_dev; @@ -360,7 +373,8 @@ static void stat64_to_stat(struct stat64 *in, struct stat *out) { } # endif -# if SANITIZER_LINUX && defined(__loongarch__) +# if SANITIZER_LINUX && \ + (defined(__loongarch__) || defined(__hexagon__) || defined(__alpha__)) static void statx_to_stat(struct statx *in, struct stat *out) { internal_memset(out, 0, sizeof(*out)); out->st_dev = makedev(in->stx_dev_major, in->stx_dev_minor); @@ -440,7 +454,7 @@ uptr internal_stat(const char *path, void *buf) { # if SANITIZER_FREEBSD return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0); # elif SANITIZER_LINUX -# if defined(__loongarch__) +# if defined(__loongarch__) || defined(__hexagon__) || defined(__alpha__) struct statx bufx; int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path, AT_NO_AUTOMOUNT, STATX_BASIC_STATS, (uptr)&bufx); @@ -478,7 +492,7 @@ uptr internal_lstat(const char *path, void *buf) { return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, AT_SYMLINK_NOFOLLOW); # elif SANITIZER_LINUX -# if defined(__loongarch__) +# if defined(__loongarch__) || defined(__hexagon__) || defined(__alpha__) struct statx bufx; int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path, AT_SYMLINK_NOFOLLOW | AT_NO_AUTOMOUNT, @@ -526,7 +540,7 @@ uptr internal_fstat(fd_t fd, void *buf) { int res = internal_syscall(SYSCALL(fstat64), fd, &kbuf); kernel_stat_to_stat(&kbuf, (struct stat *)buf); return res; -# elif SANITIZER_LINUX && defined(__loongarch__) +# elif SANITIZER_LINUX && (defined(__loongarch__) || defined(__alpha__)) struct statx bufx; int res = internal_syscall(SYSCALL(statx), fd, "", AT_EMPTY_PATH, STATX_BASIC_STATS, (uptr)&bufx); @@ -535,6 +549,13 @@ uptr internal_fstat(fd_t fd, void *buf) { # else return internal_syscall(SYSCALL(fstat), fd, (uptr)buf); # endif +# elif SANITIZER_LINUX && defined(__hexagon__) + // Hexagon musl lacks struct stat64; use statx() instead. + struct statx bufx; + int res = internal_syscall(SYSCALL(statx), fd, "", AT_EMPTY_PATH, + STATX_BASIC_STATS, (uptr)&bufx); + statx_to_stat(&bufx, (struct stat*)buf); + return res; # else struct stat64 buf64; int res = internal_syscall(SYSCALL(fstat64), fd, &buf64); @@ -1003,7 +1024,7 @@ int internal_sigaction_norestorer(int signum, const void *act, void *oldact) { // rt_sigaction, so we need to do the same (we'll need to reimplement the // restorers; for x86_64 the restorer address can be obtained from // oldact->sa_restorer upon a call to sigaction(xxx, NULL, oldact). -# if !SANITIZER_ANDROID || !SANITIZER_MIPS32 +# if (!SANITIZER_ANDROID || !SANITIZER_MIPS32) && !defined(__alpha__) k_act.sa_restorer = u_act->sa_restorer; # endif } @@ -1019,7 +1040,7 @@ int internal_sigaction_norestorer(int signum, const void *act, void *oldact) { internal_memcpy(&u_oldact->sa_mask, &k_oldact.sa_mask, sizeof(__sanitizer_kernel_sigset_t)); u_oldact->sa_flags = k_oldact.sa_flags; -# if !SANITIZER_ANDROID || !SANITIZER_MIPS32 +# if (!SANITIZER_ANDROID || !SANITIZER_MIPS32) && !defined(__alpha__) u_oldact->sa_restorer = k_oldact.sa_restorer; # endif } @@ -1228,6 +1249,16 @@ uptr GetMaxVirtualAddress() { // loongarch64 also has multiple address space layouts: default is 47-bit. // RISC-V 64 also has multiple address space layouts: 39, 48 and 57-bit. return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1; +# elif SANITIZER_ALPHA + // Linux/Alpha uses a 42-bit user VAS (TASK_SIZE = 0x40000000000). With + // fixed shadow offset 0x10000000000 (1 TiB) the layout is: + // LowMem: [0x000000000000, 0x00ffffffffff] (1 TiB) + // LowShadow: [0x010000000000, 0x011fffffffff] (128 GiB) + // ShadowGap: [0x012000000000, 0x012fffffffff] + // HighShadow:[0x013000000000, 0x017fffffffff] (256 GiB) + // HighMem: [0x018000000000, 0x03ffffffffff] (2.5 TiB, stack near top) + // Capping at TASK_SIZE - 1 avoids treating kernel addresses as HighMem. + return (1ULL << 42) - 1; // TASK_SIZE - 1 # elif SANITIZER_MIPS64 return (1ULL << 40) - 1; // 0x000000ffffffffffUL; # elif defined(__s390x__) @@ -1894,6 +1925,39 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, : "memory"); return res; } +# elif defined(__hexagon__) +uptr internal_clone(int (*fn)(void*), void* child_stack, int flags, void* arg, + int* parent_tidptr, void* newtls, int* child_tidptr) { + if (!fn || !child_stack) + return -EINVAL; + child_stack = (char*)child_stack - 2 * sizeof(unsigned int); + ((unsigned int*)child_stack)[0] = (uptr)fn; + ((unsigned int*)child_stack)[1] = (uptr)arg; + + // Hexagon clone syscall uses the generic argument order (no + // CONFIG_CLONE_BACKWARDS): flags, stack, ptid, ctid, tls. + register int r0 __asm__("r0") = flags; + register void* r1 __asm__("r1") = child_stack; + register int* r2 __asm__("r2") = parent_tidptr; + register int* r3 __asm__("r3") = child_tidptr; + register void* r4 __asm__("r4") = newtls; + register int r6 __asm__("r6") = __NR_clone; + + __asm__ __volatile__( + "trap0(#1)\n" /* syscall */ + "{ p0 = cmp.eq(r0, #0)\n" /* child? */ + " if (!p0.new) jump:nt 1f }\n" + "r1 = memw(r29 + #0)\n" /* r1 = fn */ + "r0 = memw(r29 + #4)\n" /* r0 = arg */ + "callr r1\n" /* fn(arg) */ + "r6 = #%7\n" /* __NR_exit */ + "trap0(#1)\n" + "1:\n" + : "=r"(r0) + : "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r6), "i"(__NR_exit) + : "memory", "p0", "r1", "lr"); + return (uptr)r0; +} # endif # endif // SANITIZER_LINUX @@ -2427,7 +2491,7 @@ static void DumpSingleReg(ucontext_t *ctx, int RegNum) { # if SANITIZER_LINUX ctx->uc_mcontext.gregs[RegNum] # elif SANITIZER_NETBSD - ctx->uc_mcontext.__gregs[RegNum] + (unsigned long long)ctx->uc_mcontext.__gregs[RegNum] # endif ); # elif defined(__i386__) @@ -2729,6 +2793,11 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { *pc = ucontext->uc_mcontext.__pc; *bp = ucontext->uc_mcontext.__gregs[22]; *sp = ucontext->uc_mcontext.__gregs[3]; +# elif defined(__alpha__) + ucontext_t* ucontext = (ucontext_t*)context; + *pc = ucontext->uc_mcontext.sc_pc; + *bp = ucontext->uc_mcontext.sc_regs[15]; // $fp / $s6 + *sp = ucontext->uc_mcontext.sc_regs[30]; // $sp # else # error "Unsupported arch" # endif @@ -2819,7 +2888,7 @@ void CheckMPROTECT() { # endif } -void CheckNoDeepBind(const char *filename, int flag) { +void OnDlOpen(const char* filename, int flag) { # ifdef RTLD_DEEPBIND if (flag & RTLD_DEEPBIND) { Report( diff --git a/lib/libtsan/sanitizer_common/sanitizer_linux.h b/lib/libtsan/sanitizer_common/sanitizer_linux.h index e621799c4bdf939638ba0cd42937cf3de069d39c..2f73dba3bc1cfddfaffd69d125af1c7d6f05a8be 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_linux.h +++ b/lib/libtsan/sanitizer_common/sanitizer_linux.h @@ -86,7 +86,8 @@ int internal_sigaction_norestorer(int signum, const void *act, void *oldact); void internal_sigdelset(__sanitizer_sigset_t *set, int signum); # if defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) || \ defined(__powerpc64__) || defined(__s390__) || defined(__i386__) || \ - defined(__arm__) || SANITIZER_RISCV64 || SANITIZER_LOONGARCH64 + defined(__arm__) || defined(__hexagon__) || SANITIZER_RISCV64 || \ + SANITIZER_LOONGARCH64 uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, int *parent_tidptr, void *newtls, int *child_tidptr); # endif diff --git a/lib/libtsan/sanitizer_common/sanitizer_linux_libcdep.cpp b/lib/libtsan/sanitizer_common/sanitizer_linux_libcdep.cpp index fb99bc0886ad8cf479b05bef7b64eb9209250078..ba4333cdf08e6908f74bafa3b8dd1dff39e3ca47 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_linux_libcdep.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_linux_libcdep.cpp @@ -284,6 +284,10 @@ static uptr ThreadDescriptorSizeFallback() { # if defined(__powerpc64__) return 1776; // from glibc.ppc64le 2.20-8.fc21 # endif + +# if defined(__alpha__) + return 1824; // from glibc 2.43 +# endif } # endif // SANITIZER_GLIBC && !SANITIZER_GO @@ -494,10 +498,10 @@ __attribute__((unused)) static void GetStaticTlsBoundary(uptr *addr, uptr *size, // loader places static TLS blocks this way not to waste space. uptr l = one; *align = ranges[l].align; - while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l].align) + while (l != 0 && ranges[l].begin <= ranges[l - 1].end + ranges[l].align) *align = Max(*align, ranges[--l].align); uptr r = one + 1; - while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r].align) + while (r != len && ranges[r].begin <= ranges[r - 1].end + ranges[r].align) *align = Max(*align, ranges[r++].align); *addr = ranges[l].begin; *size = ranges[r - 1].end - ranges[l].begin; diff --git a/lib/libtsan/sanitizer_common/sanitizer_lzw.h b/lib/libtsan/sanitizer_common/sanitizer_lzw.h index 42acfbdcea09239fb2684836116362d2a9491bdf..d8e7d1f21762763bbe7c9fdc0501075e5949dfe7 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_lzw.h +++ b/lib/libtsan/sanitizer_common/sanitizer_lzw.h @@ -26,9 +26,7 @@ ItOut LzwEncode(ItIn begin, ItIn end, ItOut out) { // Sentinel value for substrings of len 1. static constexpr LzwCodeType kNoPrefix = - Min(DenseMapInfo::getEmptyKey().first, - DenseMapInfo::getTombstoneKey().first) - - 1; + DenseMapInfo::getEmptyKey().first - 1; DenseMap prefix_to_code; { // Add all substring of len 1 as initial dictionary. diff --git a/lib/libtsan/sanitizer_common/sanitizer_mac.cpp b/lib/libtsan/sanitizer_common/sanitizer_mac.cpp index 940175791f3765862de92974c72899b4b9c05d11..64c8bfebd47fcdf0178f71cb09b1aafb3d58c3ff 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_mac.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_mac.cpp @@ -170,6 +170,10 @@ uptr internal_close(fd_t fd) { return close(fd); } +uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags) { + return -1; // Not supported. +} + uptr internal_open(const char *filename, int flags) { return open(filename, flags); } @@ -609,7 +613,13 @@ static uptr ApproximateOSVersionViaKernelVersion(VersStr vers) { u16 os_major = kernel_major - offset; const char *format = "%d.0"; - if (TARGET_OS_OSX) { + if (kernel_major >= 27) { + // with kernel major 27 <=> OSes 27.0, OS versions are aligned with kernel + os_major = kernel_major; + } else if (kernel_major >= 25) { + // with kernel_major 25 <=> OSes 26.0, OS versions are aligned + os_major = kernel_major + 1; + } else if (TARGET_OS_OSX) { if (os_major >= 16) { // macOS 11+ os_major -= 5; } else { // macOS 10.15 and below @@ -666,6 +676,24 @@ static void MapToMacos(u16 *major, u16 *minor) { if (TARGET_OS_OSX) return; + // All supported platforms (including DriverKit) have + // aligned version numbers in macOS 27+ + if (*major >= 27) + return; + +# if TARGET_OS_DRIVERKIT + // Driverkit 25.0+ aligns with macOS 26+ + if (*major >= 25) { + *major += 1; + return; + } +# else + // macOS 26 and later have aligned version strings. + if (*major >= 26) + return; +# endif + + // Below are mappings for pre-macOS-25-aligned releases if (TARGET_OS_IOS || TARGET_OS_TV) *major += 2; else if (TARGET_OS_WATCH) @@ -1529,7 +1557,7 @@ void DumpProcessMap() { Printf("End of module map.\n"); } -void CheckNoDeepBind(const char *filename, int flag) { +void OnDlOpen(const char* filename, int flag) { // Do nothing. } diff --git a/lib/libtsan/sanitizer_common/sanitizer_netbsd.cpp b/lib/libtsan/sanitizer_common/sanitizer_netbsd.cpp index 737e336dfbe872e0e19ca0780ea0032c81d54692..d83adc34c9ac6e06ae2117c7b380f9469ef08953 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_netbsd.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_netbsd.cpp @@ -126,6 +126,10 @@ uptr internal_close(fd_t fd) { return _sys_close(fd); } +uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags) { + return -1; // Not supported. +} + uptr internal_open(const char *filename, int flags) { CHECK(&_sys_open); return _sys_open(filename, flags); diff --git a/lib/libtsan/sanitizer_common/sanitizer_platform.h b/lib/libtsan/sanitizer_common/sanitizer_platform.h index acd2da26117c3862133ecae61a8abffaa7e630e9..3d85959ac94a67f0fceedb87629506036f954aea 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_platform.h +++ b/lib/libtsan/sanitizer_common/sanitizer_platform.h @@ -15,7 +15,8 @@ #if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__NetBSD__) && \ !defined(__APPLE__) && !defined(_WIN32) && !defined(__Fuchsia__) && \ !(defined(__sun__) && defined(__svr4__)) && !defined(__HAIKU__) && \ - !defined(__wasi__) + !defined(__wasi__) && !defined(__NVPTX__) && !defined(__AMDGPU__) && \ + !defined(__SPIRV__) && !defined(_AIX) # error "This operating system is not supported" #endif @@ -32,6 +33,12 @@ # define SANITIZER_LINUX 0 #endif +#if defined(_AIX) +# define SANITIZER_AIX 1 +#else +# define SANITIZER_AIX 0 +#endif + #if defined(__GLIBC__) # define SANITIZER_GLIBC 1 #else @@ -151,7 +158,7 @@ #define SANITIZER_POSIX \ (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_APPLE || \ - SANITIZER_NETBSD || SANITIZER_SOLARIS || SANITIZER_HAIKU) + SANITIZER_NETBSD || SANITIZER_SOLARIS || SANITIZER_HAIKU || SANITIZER_AIX) #if __LP64__ || defined(_WIN64) # define SANITIZER_WORDSIZE 64 @@ -302,6 +309,30 @@ # define SANITIZER_LOONGARCH64 0 #endif +#if defined(__alpha__) +# define SANITIZER_ALPHA 1 +#else +# define SANITIZER_ALPHA 0 +#endif + +#if defined(__AMDGPU__) +# define SANITIZER_AMDGPU 1 +#else +# define SANITIZER_AMDGPU 0 +#endif + +#if defined(__NVPTX__) +# define SANITIZER_NVPTX 1 +#else +# define SANITIZER_NVPTX 0 +#endif + +#if defined(__SPIRV__) +# define SANITIZER_SPIRV 1 +#else +# define SANITIZER_SPIRV 0 +#endif + // By default we allow to use SizeClassAllocator64 on 64-bit platform. // But in some cases SizeClassAllocator64 does not work well and we need to // fallback to SizeClassAllocator32. diff --git a/lib/libtsan/sanitizer_common/sanitizer_platform_interceptors.h b/lib/libtsan/sanitizer_common/sanitizer_platform_interceptors.h index 1b300bc7533b6f90b1f85d1ff0aa7d54671bfe7c..702c960434a0fa7d00be7b154005e2ec3d756764 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_platform_interceptors.h +++ b/lib/libtsan/sanitizer_common/sanitizer_platform_interceptors.h @@ -201,6 +201,9 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment, #define SANITIZER_INTERCEPT_READ SI_POSIX #define SANITIZER_INTERCEPT_PREAD SI_POSIX +#define SANITIZER_INTERCEPT___READ_CHK SI_GLIBC +#define SANITIZER_INTERCEPT___PREAD_CHK SI_GLIBC +#define SANITIZER_INTERCEPT___PREAD64_CHK SI_GLIBC #define SANITIZER_INTERCEPT_WRITE SI_POSIX #define SANITIZER_INTERCEPT_PWRITE SI_POSIX @@ -393,6 +396,8 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment, #define SANITIZER_INTERCEPT_SHMCTL \ (((SI_FREEBSD || SI_LINUX_NOT_ANDROID) && SANITIZER_WORDSIZE == 64) || \ SI_NETBSD || SI_SOLARIS) +// shmat calls REAL(shmctl), so it requires shmctl interception. +#define SANITIZER_INTERCEPT_SHMAT SANITIZER_INTERCEPT_SHMCTL #define SANITIZER_INTERCEPT_RANDOM_R SI_GLIBC #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_POSIX #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \ @@ -545,7 +550,7 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment, #define SANITIZER_INTERCEPT___LIBC_MEMALIGN SI_GLIBC #define SANITIZER_INTERCEPT_PVALLOC (SI_GLIBC || SI_ANDROID) #define SANITIZER_INTERCEPT_CFREE (SI_GLIBC && !SANITIZER_RISCV64) -#define SANITIZER_INTERCEPT_REALLOCARRAY SI_POSIX +#define SANITIZER_INTERCEPT_REALLOCARRAY (SI_POSIX || SI_FUCHSIA) #define SANITIZER_INTERCEPT_ALIGNED_ALLOC \ (!SI_MAC || SI_MAC_SDK_10_15_AVAILABLE) #define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD) diff --git a/lib/libtsan/sanitizer_common/sanitizer_platform_limits_linux.cpp b/lib/libtsan/sanitizer_common/sanitizer_platform_limits_linux.cpp index bf0f355847cb13332fac82fb756fe2afa068991a..a0cafd0b6cbde7e7c49c7f432a443c8af20bd8e5 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_platform_limits_linux.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_platform_limits_linux.cpp @@ -60,7 +60,7 @@ using namespace __sanitizer; # if !defined(__powerpc64__) && !defined(__x86_64__) && \ !defined(__aarch64__) && !defined(__mips__) && !defined(__s390__) && \ !defined(__sparc__) && !defined(__riscv) && !defined(__hexagon__) && \ - !defined(__loongarch__) + !defined(__loongarch__) && !defined(__alpha__) COMPILER_CHECK(struct___old_kernel_stat_sz == sizeof(struct __old_kernel_stat)); #endif diff --git a/lib/libtsan/sanitizer_common/sanitizer_platform_limits_posix.cpp b/lib/libtsan/sanitizer_common/sanitizer_platform_limits_posix.cpp index 47eb1dc32677210403860a12b00456cb914d9a14..f88ffc52dc692f86936a5387178806e7290ee74d 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_platform_limits_posix.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_platform_limits_posix.cpp @@ -24,7 +24,7 @@ // Must go after undef _FILE_OFFSET_BITS. #include "sanitizer_platform.h" -#if SANITIZER_LINUX || SANITIZER_APPLE || SANITIZER_HAIKU +#if SANITIZER_LINUX || SANITIZER_APPLE || SANITIZER_HAIKU || SANITIZER_AIX // Must go after undef _FILE_OFFSET_BITS. #include "sanitizer_glibc_version.h" @@ -61,11 +61,11 @@ #endif #if !SANITIZER_ANDROID -#if !SANITIZER_HAIKU -#include -#endif -#include -#include +# if !SANITIZER_HAIKU && !SANITIZER_AIX +# include +# endif +# include +# include #endif #if SANITIZER_LINUX @@ -113,11 +113,15 @@ typedef struct user_fpregs elf_fpregset_t; #endif #if !SANITIZER_ANDROID -#include -#if !SANITIZER_HAIKU -#include -#include -#endif +# if !SANITIZER_AIX +# include +# else +# include +# endif +# if !SANITIZER_HAIKU +# include +# include +# endif #endif #if SANITIZER_LINUX @@ -182,6 +186,17 @@ typedef struct user_fpregs elf_fpregset_t; #include #endif +# if SANITIZER_AIX +# include +# include +# include +# include +# include +# if HAVE_RPC_XDR_H +# include +# endif +# endif + // Include these after system headers to avoid name clashes and ambiguities. # include "sanitizer_common.h" # include "sanitizer_internal_defs.h" @@ -293,7 +308,7 @@ namespace __sanitizer { #define SIZEOF_STRUCT_USTAT 32 # elif defined(__arm__) || defined(__i386__) || defined(__mips__) || \ defined(__powerpc__) || defined(__s390__) || defined(__sparc__) || \ - defined(__hexagon__) + defined(__hexagon__) || defined(__alpha__) # define SIZEOF_STRUCT_USTAT 20 # elif defined(__loongarch__) // Not used. The minimum Glibc version available for LoongArch is 2.36 @@ -305,9 +320,12 @@ namespace __sanitizer { unsigned struct_ustat_sz = SIZEOF_STRUCT_USTAT; unsigned struct_rlimit64_sz = sizeof(struct rlimit64); unsigned struct_statvfs64_sz = sizeof(struct statvfs64); -#endif // SANITIZER_GLIBC +# elif SANITIZER_MUSL + // On musl, rlimit64 is an alias for rlimit. + unsigned struct_rlimit64_sz = sizeof(struct rlimit); +# endif // SANITIZER_GLIBC -#if SANITIZER_LINUX && !SANITIZER_ANDROID +# if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned struct_timex_sz = sizeof(struct timex); unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds); unsigned struct_mq_attr_sz = sizeof(struct mq_attr); @@ -556,13 +574,13 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); const unsigned IOCTL_NOT_PRESENT = 0; unsigned IOCTL_FIONBIO = FIONBIO; -#if !SANITIZER_HAIKU +# if !SANITIZER_HAIKU unsigned IOCTL_FIOASYNC = FIOASYNC; unsigned IOCTL_FIOCLEX = FIOCLEX; unsigned IOCTL_FIOGETOWN = FIOGETOWN; unsigned IOCTL_FIONCLEX = FIONCLEX; unsigned IOCTL_FIOSETOWN = FIOSETOWN; -#endif +# endif unsigned IOCTL_SIOCADDMULTI = SIOCADDMULTI; unsigned IOCTL_SIOCATMARK = SIOCATMARK; unsigned IOCTL_SIOCDELMULTI = SIOCDELMULTI; @@ -584,14 +602,14 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned IOCTL_SIOCSIFNETMASK = SIOCSIFNETMASK; unsigned IOCTL_SIOCSPGRP = SIOCSPGRP; -#if !SANITIZER_HAIKU +# if !SANITIZER_HAIKU unsigned IOCTL_TIOCCONS = TIOCCONS; unsigned IOCTL_TIOCGETD = TIOCGETD; unsigned IOCTL_TIOCNOTTY = TIOCNOTTY; unsigned IOCTL_TIOCPKT = TIOCPKT; unsigned IOCTL_TIOCSETD = TIOCSETD; unsigned IOCTL_TIOCSTI = TIOCSTI; -#endif +# endif unsigned IOCTL_TIOCEXCL = TIOCEXCL; unsigned IOCTL_TIOCGPGRP = TIOCGPGRP; @@ -602,10 +620,12 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned IOCTL_TIOCMSET = TIOCMSET; unsigned IOCTL_TIOCNXCL = TIOCNXCL; unsigned IOCTL_TIOCOUTQ = TIOCOUTQ; +# if !SANITIZER_AIX unsigned IOCTL_TIOCSCTTY = TIOCSCTTY; +# endif unsigned IOCTL_TIOCSPGRP = TIOCSPGRP; unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ; -#if SANITIZER_LINUX && !SANITIZER_ANDROID +# if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned IOCTL_SIOCGETSGCNT = SIOCGETSGCNT; unsigned IOCTL_SIOCGETVIFCNT = SIOCGETVIFCNT; #endif @@ -1067,6 +1087,9 @@ CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol); CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen); CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname); CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr); +# if SANITIZER_AIX +CHECK_SIZE_AND_OFFSET(addrinfo, ai_eflags); +# endif CHECK_TYPE_SIZE(hostent); CHECK_SIZE_AND_OFFSET(hostent, h_name); @@ -1113,11 +1136,13 @@ COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent)); CHECK_SIZE_AND_OFFSET(dirent, d_ino); #if SANITIZER_APPLE CHECK_SIZE_AND_OFFSET(dirent, d_seekoff); -#elif SANITIZER_FREEBSD || SANITIZER_HAIKU +# elif SANITIZER_AIX +CHECK_SIZE_AND_OFFSET(dirent, d_offset); +# elif SANITIZER_FREEBSD || SANITIZER_HAIKU // There is no 'd_off' field on FreeBSD. -#else +# else CHECK_SIZE_AND_OFFSET(dirent, d_off); -#endif +# endif CHECK_SIZE_AND_OFFSET(dirent, d_reclen); #if SANITIZER_GLIBC @@ -1151,7 +1176,8 @@ CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask); // didn't exist. CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_flags); #endif -#if SANITIZER_LINUX && (!SANITIZER_ANDROID || !SANITIZER_MIPS32) +# if SANITIZER_LINUX && (!SANITIZER_ANDROID || !SANITIZER_MIPS32) && \ + !defined(__alpha__) CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_restorer); #endif @@ -1192,6 +1218,10 @@ CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc); CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv); CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs); #endif +# if SANITIZER_AIX +CHECK_SIZE_AND_OFFSET(wordexp_t, we_sflags); +CHECK_SIZE_AND_OFFSET(wordexp_t, we_soffs); +# endif CHECK_TYPE_SIZE(tm); CHECK_SIZE_AND_OFFSET(tm, tm_sec); @@ -1203,10 +1233,12 @@ CHECK_SIZE_AND_OFFSET(tm, tm_year); CHECK_SIZE_AND_OFFSET(tm, tm_wday); CHECK_SIZE_AND_OFFSET(tm, tm_yday); CHECK_SIZE_AND_OFFSET(tm, tm_isdst); +# if !SANITIZER_AIX CHECK_SIZE_AND_OFFSET(tm, tm_gmtoff); CHECK_SIZE_AND_OFFSET(tm, tm_zone); +# endif -#if SANITIZER_LINUX +# if SANITIZER_LINUX CHECK_TYPE_SIZE(mntent); CHECK_SIZE_AND_OFFSET(mntent, mnt_fsname); CHECK_SIZE_AND_OFFSET(mntent, mnt_dir); @@ -1256,7 +1288,7 @@ CHECK_TYPE_SIZE(clock_t); CHECK_TYPE_SIZE(clockid_t); #endif -#if !SANITIZER_ANDROID && !SANITIZER_HAIKU +# if !SANITIZER_ANDROID && !SANITIZER_HAIKU && !SANITIZER_AIX CHECK_TYPE_SIZE(ifaddrs); CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next); CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name); diff --git a/lib/libtsan/sanitizer_common/sanitizer_platform_limits_posix.h b/lib/libtsan/sanitizer_common/sanitizer_platform_limits_posix.h index 05ebee49f2ab4949add32997daa2cf7a0b007af2..73eb958c827210a5023793e03414852a901754c6 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/lib/libtsan/sanitizer_common/sanitizer_platform_limits_posix.h @@ -14,7 +14,7 @@ #ifndef SANITIZER_PLATFORM_LIMITS_POSIX_H #define SANITIZER_PLATFORM_LIMITS_POSIX_H -#if SANITIZER_LINUX || SANITIZER_APPLE || SANITIZER_HAIKU +#if SANITIZER_LINUX || SANITIZER_APPLE || SANITIZER_HAIKU || SANITIZER_AIX # include "sanitizer_internal_defs.h" # include "sanitizer_mallinfo.h" @@ -29,7 +29,7 @@ # define SANITIZER_HAS_STAT64 0 # define SANITIZER_HAS_STATFS64 0 # endif -# elif SANITIZER_GLIBC || SANITIZER_ANDROID +# elif SANITIZER_GLIBC || SANITIZER_ANDROID || SANITIZER_AIX # define SANITIZER_HAS_STAT64 1 # define SANITIZER_HAS_STATFS64 1 # elif SANITIZER_HAIKU @@ -103,9 +103,15 @@ const unsigned struct_kernel_stat64_sz = 104; const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID ? FIRST_32_SECOND_64(104, 128) # if defined(_ABIN32) && _MIPS_SIM == _ABIN32 +# if defined(_TIME_BITS) && _TIME_BITS == 64 + : FIRST_32_SECOND_64(112, 216); +# else : FIRST_32_SECOND_64(176, 216); +# endif # elif SANITIZER_MUSL : FIRST_32_SECOND_64(160, 208); +# elif defined(_TIME_BITS) && _TIME_BITS == 64 + : FIRST_32_SECOND_64(112, 216); # else : FIRST_32_SECOND_64(160, 216); # endif @@ -133,6 +139,9 @@ const unsigned struct_kernel_stat64_sz = 0; # elif defined(__loongarch__) const unsigned struct_kernel_stat_sz = 128; const unsigned struct_kernel_stat64_sz = 0; +# elif defined(__alpha__) +const unsigned struct_kernel_stat_sz = 80; +const unsigned struct_kernel_stat64_sz = 136; # endif struct __sanitizer_perf_event_attr { unsigned type; @@ -323,7 +332,7 @@ struct __sanitizer_iovec { usize iov_len; }; -# if !SANITIZER_ANDROID +# if !SANITIZER_ANDROID && !SANITIZER_AIX struct __sanitizer_ifaddrs { struct __sanitizer_ifaddrs *ifa_next; char *ifa_name; @@ -337,7 +346,7 @@ struct __sanitizer_ifaddrs { void *ifa_dstaddr; // (struct sockaddr *) void *ifa_data; }; -# endif // !SANITIZER_ANDROID +# endif // !SANITIZER_ANDROID && !SANITIZER_AIX # if SANITIZER_APPLE typedef unsigned long __sanitizer_pthread_key_t; @@ -345,7 +354,7 @@ typedef unsigned long __sanitizer_pthread_key_t; typedef unsigned __sanitizer_pthread_key_t; # endif -# if SANITIZER_LINUX && !SANITIZER_ANDROID +# if (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_AIX struct __sanitizer_XDR { int x_op; @@ -440,12 +449,14 @@ struct __sanitizer_tm { int tm_wday; int tm_yday; int tm_isdst; -# if SANITIZER_HAIKU +# if !SANITIZER_AIX +# if SANITIZER_HAIKU int tm_gmtoff; # else long int tm_gmtoff; # endif const char *tm_zone; +# endif }; # if SANITIZER_LINUX @@ -513,11 +524,19 @@ struct __sanitizer_msghdr { struct __sanitizer_iovec *msg_iov; uptr msg_iovlen; void *msg_control; +# if !SANITIZER_AIX uptr msg_controllen; +# else + unsigned msg_controllen; +# endif int msg_flags; }; struct __sanitizer_cmsghdr { +# if !SANITIZER_AIX uptr cmsg_len; +# else + unsigned cmsg_len; +# endif int cmsg_level; int cmsg_type; }; @@ -554,10 +573,23 @@ struct __sanitizer_dirent { unsigned short d_reclen; // more fields that we don't care about }; +# elif defined(__alpha__) +struct __sanitizer_dirent { + unsigned int d_ino; // ino_t is 32-bit on Alpha + int __pad; // explicit padding before d_off + unsigned long d_off; + unsigned short d_reclen; + // more fields that we don't care about +}; # else struct __sanitizer_dirent { +# if SANITIZER_AIX + uptr d_offset; + uptr d_ino; +# else uptr d_ino; uptr d_off; +# endif unsigned short d_reclen; // more fields that we don't care about }; @@ -573,7 +605,7 @@ struct __sanitizer_dirent64 { extern unsigned struct_sock_fprog_sz; # endif -# if SANITIZER_HAIKU +# if SANITIZER_HAIKU || SANITIZER_AIX typedef int __sanitizer_clock_t; # elif defined(__x86_64__) && !defined(_LP64) typedef long long __sanitizer_clock_t; @@ -581,8 +613,10 @@ typedef long long __sanitizer_clock_t; typedef long __sanitizer_clock_t; # endif -# if SANITIZER_LINUX || SANITIZER_HAIKU +# if SANITIZER_LINUX || SANITIZER_HAIKU || SANITIZER_AIX typedef int __sanitizer_clockid_t; +# endif +# if SANITIZER_LINUX || SANITIZER_HAIKU typedef unsigned long long __sanitizer_eventfd_t; # endif @@ -637,6 +671,14 @@ struct __sanitizer_sigset_t { // The size is determined by looking at sizeof of real sigset_t on linux. uptr val[128 / sizeof(uptr)]; }; +# elif SANITIZER_AIX +struct __sanitizer_sigset_t { +# if SANITIZER_WORDSIZE == 64 + uptr val[4]; +# else + uptr val[2]; +# endif +}; # endif struct __sanitizer_siginfo_pad { @@ -741,7 +783,7 @@ struct __sanitizer_sigaction { # endif # endif # endif -# if SANITIZER_LINUX || SANITIZER_HAIKU +# if (SANITIZER_LINUX || SANITIZER_HAIKU) && !defined(__alpha__) void (*sa_restorer)(); # endif # if defined(__mips__) && (SANITIZER_WORDSIZE == 32) && !SANITIZER_MUSL @@ -828,8 +870,12 @@ struct __sanitizer_addrinfo { int ai_family; int ai_socktype; int ai_protocol; -# if SANITIZER_ANDROID || SANITIZER_APPLE || SANITIZER_HAIKU +# if SANITIZER_ANDROID || SANITIZER_APPLE || SANITIZER_HAIKU || SANITIZER_AIX +# if SANITIZER_AIX // AIX ai_addrlen type is size_t + uptr ai_addrlen; +# else unsigned ai_addrlen; +# endif char *ai_canonname; void *ai_addr; # else // LINUX @@ -838,6 +884,9 @@ struct __sanitizer_addrinfo { char *ai_canonname; # endif struct __sanitizer_addrinfo *ai_next; +# if SANITIZER_AIX + int ai_eflags; +# endif }; struct __sanitizer_hostent { @@ -854,7 +903,7 @@ struct __sanitizer_pollfd { short revents; }; -# if SANITIZER_ANDROID || SANITIZER_APPLE +# if SANITIZER_ANDROID || SANITIZER_APPLE || SANITIZER_AIX typedef unsigned __sanitizer_nfds_t; # else typedef unsigned long __sanitizer_nfds_t; @@ -892,6 +941,10 @@ struct __sanitizer_wordexp_t { uptr we_wordc; char **we_wordv; uptr we_offs; +# if SANITIZER_AIX + int we_sflags; + uptr we_soffs; +# endif }; # if SANITIZER_LINUX && !SANITIZER_ANDROID @@ -1023,7 +1076,7 @@ struct __sanitizer_cookie_io_functions_t { # define IOC_NRBITS 8 # define IOC_TYPEBITS 8 # if defined(__powerpc__) || defined(__powerpc64__) || defined(__mips__) || \ - defined(__sparc__) + defined(__sparc__) || defined(__alpha__) # define IOC_SIZEBITS 13 # define IOC_DIRBITS 3 # define IOC_NONE 1U @@ -1193,7 +1246,9 @@ extern unsigned IOCTL_TIOCMGET; extern unsigned IOCTL_TIOCMSET; extern unsigned IOCTL_TIOCNXCL; extern unsigned IOCTL_TIOCOUTQ; +# if !SANITIZER_AIX extern unsigned IOCTL_TIOCSCTTY; +# endif extern unsigned IOCTL_TIOCSPGRP; extern unsigned IOCTL_TIOCSWINSZ; # if SANITIZER_LINUX && !SANITIZER_ANDROID @@ -1593,6 +1648,7 @@ extern const int si_SEGV_ACCERR; typedef void *__sanitizer_timer_t; # endif -#endif // SANITIZER_LINUX || SANITIZER_APPLE || SANITIZER_HAIKU +#endif // SANITIZER_LINUX || SANITIZER_APPLE || SANITIZER_HAIKU || + // SANITIZER_AIX #endif diff --git a/lib/libtsan/sanitizer_common/sanitizer_platform_limits_solaris.cpp b/lib/libtsan/sanitizer_common/sanitizer_platform_limits_solaris.cpp index 7ea6134b702bf7be66526f7d7faf89c06620e178..1cc64a38aaed82b002700648bbdea13c66e21713 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_platform_limits_solaris.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_platform_limits_solaris.cpp @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include diff --git a/lib/libtsan/sanitizer_common/sanitizer_posix.cpp b/lib/libtsan/sanitizer_common/sanitizer_posix.cpp index 5b2c4e668ca8f9dd431f62fc1e1bd829def5a3a1..87e2878c6ce22dd3d72f84e285c5ef264d797ab6 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_posix.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_posix.cpp @@ -27,12 +27,13 @@ #include #include -#if SANITIZER_FREEBSD +# if SANITIZER_FREEBSD || SANITIZER_AIX // The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before // that, it was never implemented. So just define it to zero. -#undef MAP_NORESERVE -#define MAP_NORESERVE 0 -#endif +// Similarly, AIX does not define MAP_NORESERVE. +# undef MAP_NORESERVE +# define MAP_NORESERVE 0 +# endif namespace __sanitizer { @@ -357,9 +358,10 @@ int GetNamedMappingFd(const char *name, uptr size, int *flags) { if (!common_flags()->decorate_proc_maps || !name) return -1; char shmname[200]; - CHECK(internal_strlen(name) < sizeof(shmname) - 10); - internal_snprintf(shmname, sizeof(shmname), "/dev/shm/%zu [%s]", - internal_getpid(), name); + int len = + internal_snprintf(shmname, sizeof(shmname), "/dev/shm/%zu.%llu [%s]", + internal_getpid(), GetTid(), name); + CHECK_LT(len, sizeof(shmname)); int o_cloexec = 0; #if defined(O_CLOEXEC) o_cloexec = O_CLOEXEC; diff --git a/lib/libtsan/sanitizer_common/sanitizer_posix.h b/lib/libtsan/sanitizer_common/sanitizer_posix.h index dc9c3b8822d307bf8a65c755f22742468560b0ce..2c3cd40f5019d76ab1913bd0a331c2cf39a0cfdf 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_posix.h +++ b/lib/libtsan/sanitizer_common/sanitizer_posix.h @@ -28,9 +28,9 @@ namespace __sanitizer { // Don't use directly, use __sanitizer::OpenFile() instead. uptr internal_open(const char *filename, int flags); uptr internal_open(const char *filename, int flags, u32 mode); -# if SANITIZER_FREEBSD +// Closes all file descriptors from lowfd to highfd (inclusive). +// Returns 0 on success or non-zero if not supported on this platform. uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags); -# endif uptr internal_close(fd_t fd); uptr internal_read(fd_t fd, void *buf, uptr count); diff --git a/lib/libtsan/sanitizer_common/sanitizer_posix_libcdep.cpp b/lib/libtsan/sanitizer_common/sanitizer_posix_libcdep.cpp index 8e5e87938c37238441bcb6a28b7e927cab0461c6..056eb677f0441859d950296ff1b9000fc6a239c5 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_posix_libcdep.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_posix_libcdep.cpp @@ -188,12 +188,13 @@ static uptr GetAltStackSize() { return SIGSTKSZ * 4; } -void SetAlternateSignalStack() { +void* SetAlternateSignalStack() { stack_t altstack, oldstack; CHECK_EQ(0, sigaltstack(nullptr, &oldstack)); // If the alternate stack is already in place, do nothing. // Android always sets an alternate stack, but it's too small for us. - if (!SANITIZER_ANDROID && !(oldstack.ss_flags & SS_DISABLE)) return; + if (!SANITIZER_ANDROID && !(oldstack.ss_flags & SS_DISABLE)) + return nullptr; // TODO(glider): the mapped stack should have the MAP_STACK flag in the // future. It is not required by man 2 sigaltstack now (they're using // malloc()). @@ -201,15 +202,18 @@ void SetAlternateSignalStack() { altstack.ss_sp = (char *)MmapOrDie(altstack.ss_size, __func__); altstack.ss_flags = 0; CHECK_EQ(0, sigaltstack(&altstack, nullptr)); + return altstack.ss_sp; } -void UnsetAlternateSignalStack() { +void UnsetAlternateSignalStack(void* altstack_base) { stack_t altstack, oldstack; altstack.ss_sp = nullptr; altstack.ss_flags = SS_DISABLE; altstack.ss_size = GetAltStackSize(); // Some sane value required on Darwin. CHECK_EQ(0, sigaltstack(&altstack, &oldstack)); - UnmapOrDie(oldstack.ss_sp, oldstack.ss_size); + if (altstack_base && altstack_base == oldstack.ss_sp) { + UnmapOrDie(oldstack.ss_sp, oldstack.ss_size); + } } bool IsSignalHandlerFromSanitizer(int signum) { @@ -562,11 +566,10 @@ pid_t StartSubprocess(const char *program, const char *const argv[], internal_close(stderr_fd); } -# if SANITIZER_FREEBSD - internal_close_range(3, ~static_cast(0), 0); -# else - for (int fd = sysconf(_SC_OPEN_MAX); fd > 2; fd--) internal_close(fd); -# endif + // Close all fds except stdin/stdout/stderr before exec. + // Fallback to the loop if close_range is not supported. + if (internal_close_range(3, ~static_cast(0), 0) != 0) + for (int fd = sysconf(_SC_OPEN_MAX); fd > 2; fd--) internal_close(fd); internal_execve(program, const_cast(&argv[0]), const_cast(envp)); diff --git a/lib/libtsan/sanitizer_common/sanitizer_redefine_builtins.h b/lib/libtsan/sanitizer_common/sanitizer_redefine_builtins.h index 7d889111765775aed6cd64afd229b8543f0e81ed..6a7d1dc79a52c28736b5bd56976e889b373e3e21 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_redefine_builtins.h +++ b/lib/libtsan/sanitizer_common/sanitizer_redefine_builtins.h @@ -17,11 +17,23 @@ // The asm hack only works with GCC and Clang. # if !defined(_WIN32) && !defined(_AIX) && !defined(__APPLE__) -asm(R"( - .set memcpy, __sanitizer_internal_memcpy - .set memmove, __sanitizer_internal_memmove - .set memset, __sanitizer_internal_memset - )"); +# if defined(__hexagon__) + +# define SANITIZER_REDEFINE_BUILTIN_ASM(name) \ + asm(".set " #name ", __sanitizer_internal_" #name) + +# else + +# define SANITIZER_REDEFINE_BUILTIN_ASM(name) \ + asm(#name " = __sanitizer_internal_" #name) + +# endif + +SANITIZER_REDEFINE_BUILTIN_ASM(memcpy); +SANITIZER_REDEFINE_BUILTIN_ASM(memmove); +SANITIZER_REDEFINE_BUILTIN_ASM(memset); + +# undef SANITIZER_REDEFINE_BUILTIN_ASM # if defined(__cplusplus) && \ !defined(SANITIZER_COMMON_REDEFINE_BUILTINS_IN_STD) diff --git a/lib/libtsan/sanitizer_common/sanitizer_solaris.cpp b/lib/libtsan/sanitizer_common/sanitizer_solaris.cpp index 62c40affc9ac5b4d04578c0e5e2e4f6fabbb794a..fe9ca92831e22c45b6e8b6552449cdab69b30c9b 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_solaris.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_solaris.cpp @@ -102,6 +102,10 @@ uptr internal_open(const char *filename, int flags, u32 mode) { return _REAL64(open)(filename, flags, mode); } +uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags) { + return -1; // Not supported. +} + DECLARE__REAL_AND_INTERNAL(uptr, read, fd_t fd, void *buf, uptr count) { return _REAL(read)(fd, buf, count); } diff --git a/lib/libtsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp b/lib/libtsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp index 2bf547f4a7216f4ee7cb8ea081f5cb358cffb83a..96bba38bceb4bea894ce60e050546cf1884d9721 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp @@ -16,7 +16,8 @@ #if SANITIZER_LINUX && \ (defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) || \ defined(__powerpc64__) || defined(__s390__) || defined(__i386__) || \ - defined(__arm__) || SANITIZER_RISCV64 || SANITIZER_LOONGARCH64) + defined(__arm__) || defined(__hexagon__) || SANITIZER_RISCV64 || \ + SANITIZER_LOONGARCH64) #include "sanitizer_stoptheworld.h" @@ -32,8 +33,8 @@ #include // for iovec #include // for NT_PRSTATUS #if (defined(__aarch64__) || defined(__powerpc64__) || \ - SANITIZER_RISCV64 || SANITIZER_LOONGARCH64) && \ - !SANITIZER_ANDROID + defined(__hexagon__) || SANITIZER_RISCV64 || \ + SANITIZER_LOONGARCH64) && !SANITIZER_ANDROID // GLIBC 2.20+ sys/user does not include asm/ptrace.h # include #endif @@ -613,6 +614,13 @@ typedef _user_regs_struct regs_struct; static constexpr uptr kExtraRegs[] = {0}; #define ARCH_IOVEC_FOR_GETREGSET +#elif defined(__hexagon__) +#include +typedef struct user_regs_struct regs_struct; +#define REG_SP r29 +static constexpr uptr kExtraRegs[] = {0}; +#define ARCH_IOVEC_FOR_GETREGSET + #else #error "Unsupported architecture" #endif // SANITIZER_ANDROID && defined(__arm__) diff --git a/lib/libtsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp b/lib/libtsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp index ab6aee7c9fba779810353b9737194160668cd0b2..eb04288382a1c5427f1a7170bea813e9acb42d4d 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp @@ -475,6 +475,13 @@ static SymbolizerTool *ChooseExternalSymbolizer(LowLevelAllocator *allocator) { return new (*allocator) Addr2LinePool(found_path, allocator); } } + +# if SANITIZER_APPLE + Report( + "WARN: No external symbolizers found. Symbols may be missing or " + "unreliable.\n"); + Report("HINT: Is PATH set? Does sandbox allow file-read of /usr/bin/atos?\n"); +# endif return nullptr; # endif // SANITIZER_DISABLE_SYMBOLIZER_PATH_SEARCH } @@ -509,13 +516,6 @@ static void ChooseSymbolizerTools(IntrusiveList *list, } # if SANITIZER_APPLE - if (list->empty()) { - Report( - "WARN: No external symbolizers found. Symbols may be missing or " - "unreliable.\n"); - Report( - "HINT: Is PATH set? Does sandbox allow file-read of /usr/bin/atos?\n"); - } VReport(2, "Using dladdr symbolizer.\n"); list->push_back(new (*allocator) DlAddrSymbolizer()); # endif // SANITIZER_APPLE diff --git a/lib/libtsan/sanitizer_common/sanitizer_symbolizer_report.cpp b/lib/libtsan/sanitizer_common/sanitizer_symbolizer_report.cpp index 351e00db6fb2dc9920c8b5b215d1741ed285a075..0f553a05b2326fb7819dacf9f448fb750ad3445a 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_symbolizer_report.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_symbolizer_report.cpp @@ -184,7 +184,7 @@ static void MaybeReportNonExecRegion(uptr pc) { MemoryMappedSegment segment; while (proc_maps.Next(&segment)) { if (pc >= segment.start && pc < segment.end && !segment.IsExecutable()) - Report("Hint: PC is at a non-executable region. Maybe a wild jump?\n"); + Report("HINT: PC is at a non-executable region. Maybe a wild jump?\n"); } #endif } @@ -254,7 +254,7 @@ static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid, (void *)sig.bp, (void *)sig.sp, tid); Printf("%s", d.Default()); if (sig.pc < GetPageSizeCached()) - Report("Hint: pc points to the zero page.\n"); + Report("HINT: pc points to the zero page.\n"); if (sig.is_memory_access) { const char *access_type = sig.write_flag == SignalContext::Write @@ -262,11 +262,12 @@ static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid, : (sig.write_flag == SignalContext::Read ? "READ" : "UNKNOWN"); Report("The signal is caused by a %s memory access.\n", access_type); if (!sig.is_true_faulting_addr) - Report("Hint: this fault was caused by a dereference of a high value " - "address (see register values below). Disassemble the provided " - "pc to learn which register was used.\n"); + Report( + "HINT: this fault was caused by a dereference of a high value " + "address (see register values below). Disassemble the provided " + "pc to learn which register was used.\n"); else if (sig.addr < GetPageSizeCached()) - Report("Hint: address points to the zero page.\n"); + Report("HINT: address points to the zero page.\n"); } MaybeReportNonExecRegion(sig.pc); InternalMmapVector stack_buffer(1); diff --git a/lib/libtsan/sanitizer_common/sanitizer_unwind_win.cpp b/lib/libtsan/sanitizer_common/sanitizer_unwind_win.cpp index 30ba812afc4b071c8f10c1b9335da73c096a1902..f1b2a157c3538fd94d09a7b919aa019f79665023 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_unwind_win.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_unwind_win.cpp @@ -43,10 +43,47 @@ void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) { trace_buffer[0] = pc; } -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wframe-larger-than=" -#endif +PVOID CALLBACK FallbackFunctionTableAccess(HANDLE hProcess, + DWORD64 dwAddrBase) { + // First try DbgHelp's function. + if (PVOID pResult = + __sanitizer::SymFunctionTableAccess64(hProcess, dwAddrBase)) { + return pResult; + } + + // Fall back to RtlLookupFunctionEntry for dynamic code. + // Function registered with RtlAddFunctionTable is not necessarily registered + // with DbgHelp, so this is required to cover some edge cases (e.g. JIT + // compilers can use Rtl* functions). +# if SANITIZER_WINDOWS64 + DWORD64 dw64ImageBase = 0; + return RtlLookupFunctionEntry(dwAddrBase, &dw64ImageBase, nullptr); +# else + return nullptr; +# endif +} + +DWORD64 CALLBACK FallbackGetModuleBase(HANDLE hProcess, DWORD64 dwAddr) { + if (DWORD64 dwResult = __sanitizer::SymGetModuleBase64(hProcess, dwAddr)) { + return dwResult; + } + + // Both GetModuleBase and FunctionTableAccess must provide this fallback, + // otherwise dynamic functions won't be properly unwound. +# if SANITIZER_WINDOWS64 + DWORD64 dw64ImageBase = 0; + if (RtlLookupFunctionEntry(dwAddr, &dw64ImageBase, nullptr)) { + return dw64ImageBase; + } +# endif + + return 0; +} + +# ifdef __clang__ +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wframe-larger-than=" +# endif void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) { CHECK(context); CHECK_GE(max_depth, 2); @@ -91,8 +128,8 @@ void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) { stack_frame.AddrFrame.Mode = AddrModeFlat; stack_frame.AddrStack.Mode = AddrModeFlat; while (StackWalk64(machine_type, GetCurrentProcess(), GetCurrentThread(), - &stack_frame, &ctx, NULL, SymFunctionTableAccess64, - SymGetModuleBase64, NULL) && + &stack_frame, &ctx, NULL, FallbackFunctionTableAccess, + FallbackGetModuleBase, NULL) && size < Min(max_depth, kStackTraceMax)) { trace_buffer[size++] = (uptr)stack_frame.AddrPC.Offset; } diff --git a/lib/libtsan/sanitizer_common/sanitizer_win.cpp b/lib/libtsan/sanitizer_common/sanitizer_win.cpp index ed4f60deeffc88c8f30b09104011e5860c5fb77f..3a1d1257a3481598b552b15a03d5bbeb184fa8c4 100644 --- a/lib/libtsan/sanitizer_common/sanitizer_win.cpp +++ b/lib/libtsan/sanitizer_common/sanitizer_win.cpp @@ -913,11 +913,12 @@ void ReportFile::Write(const char *buffer, uptr length) { } } -void SetAlternateSignalStack() { +void* SetAlternateSignalStack() { // FIXME: Decide what to do on Windows. + return nullptr; } -void UnsetAlternateSignalStack() { +void UnsetAlternateSignalStack(void* altstack_base) { // FIXME: Decide what to do on Windows. } @@ -1222,7 +1223,7 @@ int WaitForProcess(pid_t pid) { return -1; } // FIXME implement on this platform. void GetMemoryProfile(fill_profile_f cb, uptr *stats) {} -void CheckNoDeepBind(const char *filename, int flag) { +void OnDlOpen(const char* filename, int flag) { // Do nothing. } diff --git a/lib/libtsan/tsan_adaptive_delay.cpp b/lib/libtsan/tsan_adaptive_delay.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b75d0bb8db98a9938cd7ad2633d4b1dcebedc73f --- /dev/null +++ b/lib/libtsan/tsan_adaptive_delay.cpp @@ -0,0 +1,433 @@ +//===-- tsan_adaptive_delay.h -----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#include "tsan_adaptive_delay.h" + +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_allocator_internal.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_errno_codes.h" +#include "tsan_interface.h" +#include "tsan_rtl.h" + +namespace __tsan { + +namespace { + +// ============================================================================= +// DelaySpec: Represents a delay configuration parsed from flag strings +// ============================================================================= +// +// Delay can be specified as: +// - "spin=N" : Spin for up to N cycles (very short delays) +// - "yield" : Call sched_yield() once +// - "sleep_us=N" : Sleep for up to N microseconds + +enum class DelayType { Spin, Yield, SleepUs }; + +struct DelaySpec { + DelayType type; + int value; // spin cycles or sleep_us value; ignored for yield + + // Both estimates below are used internally as a very rough estimate for + // delay overhead calculation, to cap the overall delay to the + // adaptive_delay_aggressiveness option. They're not intended to be 100% + // accurate on any or all architectures/operating systems, or for use in any + // other contexts. + // + // Estimated nanoseconds per spin cycle (volatile loop iteration). + static constexpr u64 kNsPerSpinCycle = 1; + // Estimated nanoseconds for a yield (context switch overhead) + static constexpr u64 kNsPerYield = 500; + + static DelaySpec Parse(const char* str) { + DelaySpec spec; + if (internal_strncmp(str, "spin=", 5) == 0) { + spec.type = DelayType::Spin; + spec.value = internal_atoll(str + 5); + if (spec.value <= 0 || spec.value > 10000) { + Printf( + "FATAL: Invalid TSAN_OPTIONS spin value '%s'; value must be " + "between 1 and 10000\n", + str); + Die(); + } + } else if (internal_strcmp(str, "yield") == 0) { + spec.type = DelayType::Yield; + spec.value = 0; + } else if (internal_strncmp(str, "sleep_us=", 9) == 0) { + spec.type = DelayType::SleepUs; + spec.value = internal_atoll(str + 9); + if (spec.value <= 0) { + Printf( + "FATAL: Invalid TSAN_OPTIONS sleep_us value '%s'; value must be a " + "positive integer\n", + str); + Die(); + } + } else { + Printf("FATAL: Unrecognized delay spec '%s', check TSAN_OPTIONS\n", str); + Die(); + } + return spec; + } + + const char* TypeName() const { + switch (type) { + case DelayType::Spin: + return "spin"; + case DelayType::Yield: + return "yield"; + case DelayType::SleepUs: + return "sleep_us"; + } + return "unknown"; + } +}; + +} // namespace + +// ============================================================================= +// AdaptiveDelayImpl: Time-budget aware delay injection for race exposure +// ============================================================================= +// +// This implementation injects delays to expose data races while maintaining a +// configurable overhead target. It uses several strategies: +// +// 1. Time-Budget Controller: Tracks cumulative delays vs wall-clock time +// and adjusts delay probability to maintain target overhead. +// +// 2. Tiered Delays: Different delay strategies for different op types: +// - Relaxed atomics: Very rare sampling, tiny spin delays +// - Sync atomics (acq/rel/seq_cst): Moderate sampling, small usleep +// - Mutex/CV ops: Higher sampling, larger delays +// - Thread create/join: Always delay (rare but high value) +// +// 3. Address-based Sampling: Exponential backoff per address to avoid +// repeatedly delaying hot atomics. + +struct AdaptiveDelayImpl { + ALWAYS_INLINE static AdaptiveDelayState* TLS() { + return &cur_thread()->adaptive_delay_state; + } + ALWAYS_INLINE static unsigned int* GetRandomSeed() { + return &TLS()->tls_random_seed_; + } + ALWAYS_INLINE static void SetRandomSeed(unsigned int seed) { + TLS()->tls_random_seed_ = seed; + } + + // The public facing option is adaptive_delay_aggressiveness, which is an + // opaque value for the user to tune the amount of delay injected into the + // program. Internally, the implementation maps the aggressiveness to a target + // percent delay for the overall program runtime. It's not easy to implement + // a true wall clock delay target (e.g., 25% program wall time slowdown) + // because 1) spin loops and yield are hard to calculate actual wall time + // slowness and 2) usleep(N) is often slower than advertised. Thus, we keep + // the user facing parameter opaque to not under deliver on a promise of + // percent wall time slowdown. + struct TimeBudget { + int target_overhead_pct_; + Percent target_low_; + Percent target_high_; + + void Init(int target_pct) { + target_overhead_pct_ = target_pct; + target_low_ = Percent::FromPct( + target_overhead_pct_ >= 5 ? target_overhead_pct_ - 5 : 0); + target_high_ = Percent::FromPct(target_overhead_pct_ + 5); + } + + static constexpr u64 BucketDurationNs = 30'000'000'000ULL; + + void RecordDelay(u64 delay_ns) { + u64 now = NanoTime(); + u64 elapsed_ns = now - TLS()->bucket_start_ns_; + + if (elapsed_ns >= BucketDurationNs) { + // Shift: old bucket is discarded, new becomes old, start fresh new + TLS()->delay_buckets_ns_[0] = TLS()->delay_buckets_ns_[1]; + TLS()->delay_buckets_ns_[1] = 0; + TLS()->bucket_start_ns_ = now; + TLS()->bucket0_window_ns = BucketDurationNs; + } + + TLS()->delay_buckets_ns_[1] += delay_ns; + } + + Percent GetOverheadPercent() { + u64 now = NanoTime(); + u64 elapsed_ns = now - TLS()->bucket_start_ns_; + + // Need at least 1ms to calculate + if (elapsed_ns < 1'000'000ULL) + return Percent::FromPct(0); + + if (elapsed_ns > BucketDurationNs * 2) { + // Both buckets are stale + return Percent::FromPct(0); + } else if (elapsed_ns > BucketDurationNs) { + // bucket[0] is stale, use only bucket[1] (current bucket) + u64 total_delay_ns = TLS()->delay_buckets_ns_[1]; + return Percent::FromRatio(total_delay_ns, elapsed_ns); + } else { + u64 total_delay_ns = + TLS()->delay_buckets_ns_[0] + TLS()->delay_buckets_ns_[1]; + u64 window_ns = TLS()->bucket0_window_ns + elapsed_ns; + return Percent::FromRatio(total_delay_ns, window_ns); + } + } + + bool ShouldDelay() { + Percent ratio = GetOverheadPercent(); + + if (ratio < target_low_) + return true; + if (ratio > target_high_) + return false; + + // Linear interpolation: at target_low -> 100%, at target_high -> 0% + Percent prob = (target_high_ - ratio) / (target_high_ - target_low_); + return prob.RandomCheck(GetRandomSeed()); + } + }; + + // Address Sampler with Exponential Backoff + struct AddressSampler { + static constexpr u64 TABLE_SIZE = 2048; + struct Entry { + atomic_uintptr_t addr_; + atomic_uint32_t count_; + }; + Entry table_[TABLE_SIZE]; + static constexpr u32 ExponentialBackoffCap = 64; + + void Init() { + for (u64 i = 0; i < TABLE_SIZE; ++i) { + atomic_store(&table_[i].addr_, 0, memory_order_relaxed); + atomic_store(&table_[i].count_, 0, memory_order_relaxed); + } + } + + static ALWAYS_INLINE u64 splitmix64(u64 x) { + x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9ULL; + x = (x ^ (x >> 27)) * 0x94D049BB133111EBULL; + x = x ^ (x >> 31); + return x; + } + + // Uses exponential backoff: delay on 1st, 2nd, 4th, 8th, 16th, ... + bool ShouldDelayAddr(uptr addr) { + u64 idx = splitmix64(addr >> 3) & (TABLE_SIZE - 1); + Entry& e = table_[idx]; + + // This function is not thread safe. + // If two threads access the same hashed entry in parallel, + // worst case, we may end up returning true too often. This is + // acceptable...instead of full locking. + + uptr stored_addr = atomic_load(&e.addr_, memory_order_relaxed); + if (stored_addr != addr) { + // Hash Collision - reset + atomic_store(&e.addr_, addr, memory_order_relaxed); + atomic_store(&e.count_, 1, memory_order_relaxed); + return true; + } + + u32 count = atomic_fetch_add(&e.count_, 1, memory_order_relaxed) + 1; + + if ((count & (count - 1)) == 0 && count <= ExponentialBackoffCap) + return true; + return false; + } + }; + + TimeBudget budget_; + AddressSampler sampler_; + + int relaxed_sample_rate_; + int sync_atomic_sample_rate_; + int mutex_sample_rate_; + DelaySpec atomic_delay_; + DelaySpec sync_delay_; + + void Init() { InitTls(); } + + void InitTls() { + TLS()->bucket_start_ns_ = NanoTime(); + TLS()->delay_buckets_ns_[0] = 0; + TLS()->delay_buckets_ns_[1] = 0; + TLS()->bucket0_window_ns = 0; + + SetRandomSeed(NanoTime()); + TLS()->tls_initialized_ = true; + } + + bool IsTlsInitialized() const { return TLS()->tls_initialized_; } + + AdaptiveDelayImpl() { + relaxed_sample_rate_ = flags()->adaptive_delay_relaxed_sample_rate; + sync_atomic_sample_rate_ = flags()->adaptive_delay_sync_atomic_sample_rate; + mutex_sample_rate_ = flags()->adaptive_delay_mutex_sample_rate; + atomic_delay_ = DelaySpec::Parse(flags()->adaptive_delay_max_atomic); + sync_delay_ = DelaySpec::Parse(flags()->adaptive_delay_max_sync); + + int delay_aggressiveness = flags()->adaptive_delay_aggressiveness; + if (delay_aggressiveness < 1) + delay_aggressiveness = 1; + + budget_.Init(delay_aggressiveness); + sampler_.Init(); + + VPrintf(1, "INFO: ThreadSanitizer AdaptiveDelay initialized\n"); + VPrintf(1, " Delay aggressiveness: %d\n", delay_aggressiveness); + VPrintf(1, " Relaxed atomic sample rate: 1/%d\n", relaxed_sample_rate_); + VPrintf(1, " Sync atomic sample rate: 1/%d\n", sync_atomic_sample_rate_); + VPrintf(1, " Mutex sample rate: 1/%d\n", mutex_sample_rate_); + VPrintf(1, " Atomic delay: %s=%d\n", atomic_delay_.TypeName(), + atomic_delay_.value); + VPrintf(1, " Sync delay: %s=%d\n", sync_delay_.TypeName(), + sync_delay_.value); + } + + void DoSpinDelay(int iters) { + volatile int v = 0; + for (int i = 0; i < iters; ++i) v = i; + (void)v; + budget_.RecordDelay(iters * DelaySpec::kNsPerSpinCycle); + } + + void DoYieldDelay() { + internal_sched_yield(); + budget_.RecordDelay(DelaySpec::kNsPerYield); + } + + void DoSleepUsDelay(int max_us) { + // Use two Rand() calls to get full 32-bit range for larger sleep values + u32 rnd = ((u32)Rand(GetRandomSeed()) << 16) | Rand(GetRandomSeed()); + int delay_us = 1 + (rnd % max_us); + internal_usleep(delay_us); + budget_.RecordDelay(delay_us * 1000ULL); + } + + void ExecuteDelay(const DelaySpec& spec) { + switch (spec.type) { + case DelayType::Spin: { + int iters = 1 + (Rand(GetRandomSeed()) % spec.value); + DoSpinDelay(iters); + break; + } + case DelayType::Yield: + DoYieldDelay(); + break; + case DelayType::SleepUs: + DoSleepUsDelay(spec.value); + break; + } + } + + void AtomicRelaxedOpDelay() { + if ((Rand(GetRandomSeed()) % relaxed_sample_rate_) != 0) + return; + if (!budget_.ShouldDelay()) + return; + + int iters = 10 + (Rand(GetRandomSeed()) % 10); + DoSpinDelay(iters); + } + + void AtomicSyncOpDelay(uptr* addr) { + if ((Rand(GetRandomSeed()) % sync_atomic_sample_rate_) != 0) + return; + if (!budget_.ShouldDelay()) + return; + + if (addr && !sampler_.ShouldDelayAddr(*addr)) + return; + + ExecuteDelay(atomic_delay_); + } + + void AtomicOpFence(int mo) { + CHECK(IsTlsInitialized()); + + if (mo < mo_acquire) + AtomicRelaxedOpDelay(); + else + AtomicSyncOpDelay(nullptr); + } + + void AtomicOpAddr(uptr addr, int mo) { + CHECK(IsTlsInitialized()); + + if (mo < mo_acquire) + AtomicRelaxedOpDelay(); + else + AtomicSyncOpDelay(&addr); + } + + void UnsampledDelay() { + CHECK(IsTlsInitialized()); + + if (!budget_.ShouldDelay()) + return; + + ExecuteDelay(sync_delay_); + } + + void SyncOp() { + CHECK(IsTlsInitialized()); + + if ((Rand(GetRandomSeed()) % mutex_sample_rate_) != 0) + return; + if (!budget_.ShouldDelay()) + return; + + ExecuteDelay(sync_delay_); + } + + void BeforeChildThreadRuns() { + InitTls(); + UnsampledDelay(); + } + + void AfterThreadCreation() { UnsampledDelay(); } +}; + +AdaptiveDelayImpl& GetImpl() { + static AdaptiveDelayImpl impl; + return impl; +} + +bool AdaptiveDelay::is_adaptive_delay_enabled; + +void AdaptiveDelay::InitImpl() { + AdaptiveDelay::is_adaptive_delay_enabled = flags()->enable_adaptive_delay; + if (!AdaptiveDelay::is_adaptive_delay_enabled) + return; + + GetImpl().Init(); +} + +void AdaptiveDelay::SyncOpImpl() { GetImpl().SyncOp(); } +void AdaptiveDelay::AtomicOpFenceImpl(int mo) { GetImpl().AtomicOpFence(mo); } +void AdaptiveDelay::AtomicOpAddrImpl(__sanitizer::uptr addr, int mo) { + GetImpl().AtomicOpAddr(addr, mo); +} +void AdaptiveDelay::AfterThreadCreationImpl() { + GetImpl().AfterThreadCreation(); +} +void AdaptiveDelay::BeforeChildThreadRunsImpl() { + GetImpl().BeforeChildThreadRuns(); +} + +} // namespace __tsan diff --git a/lib/libtsan/tsan_adaptive_delay.h b/lib/libtsan/tsan_adaptive_delay.h new file mode 100644 index 0000000000000000000000000000000000000000..aacb9de258d104fa52b30eb697f51d43bcd5a6a1 --- /dev/null +++ b/lib/libtsan/tsan_adaptive_delay.h @@ -0,0 +1,173 @@ +//===-- tsan_adaptive_delay.h -----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#ifndef TSAN_ADAPTIVE_DELAY_H +#define TSAN_ADAPTIVE_DELAY_H + +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_internal_defs.h" + +namespace __tsan { + +// AdaptiveDelay injects delays at synchronization points, atomic operations, +// and thread lifecycle events to increase the likelihood of exposing data +// races. The delay injection is controlled by an approximate time budget to +// maintain a configurable overhead target. +// +// SyncOp() delays non-atomic synchronization points (those with clear +// happens-before relationships): +// - Acquire operations like locking a mutex delays before the mutex is locked. +// - Release operations like unlocking a mutex delays after the mutex is +// unlocked +// These are more likely to expose interesting (rare) thread interleavings. +// For example, delaying a thread that unlocks a mutex from running to allow +// newly woken thread to execute before the unlocking thread would normally +// execute. +// +// TODO: +// - Move the adaptive delay implementation into sanitizer_common so that +// ASAN can also leverage it in pthread_* interceptors +// - Integrate into other interceptors like libdispatch. +struct AdaptiveDelay { + ALWAYS_INLINE static void Init() { InitImpl(); } + + ALWAYS_INLINE static void SyncOp() { + if (!is_adaptive_delay_enabled) + return; + SyncOpImpl(); + } + + ALWAYS_INLINE static void AtomicOpFence(int mo) { + if (!is_adaptive_delay_enabled) + return; + AtomicOpFenceImpl(mo); + } + + ALWAYS_INLINE static void AtomicOpAddr(__sanitizer::uptr addr, int mo) { + if (!is_adaptive_delay_enabled) + return; + AtomicOpAddrImpl(addr, mo); + } + + ALWAYS_INLINE static void AfterThreadCreation() { + if (!is_adaptive_delay_enabled) + return; + AfterThreadCreationImpl(); + } + + ALWAYS_INLINE static void BeforeChildThreadRuns() { + if (!is_adaptive_delay_enabled) + return; + BeforeChildThreadRunsImpl(); + } + + private: + static void InitImpl(); + + static void SyncOpImpl(); + + static void AtomicOpFenceImpl(int mo); + static void AtomicOpAddrImpl(__sanitizer::uptr addr, int mo); + + static void AfterThreadCreationImpl(); + static void BeforeChildThreadRunsImpl(); + + static bool is_adaptive_delay_enabled; +}; + +// The runtime defines cur_thread() to retrieve TLS thread state, and it +// takes care of platform specific implementation details. The AdaptiveDelay +// implementation stores per-thread data in this struct, which is embedded +// in cur_thread(). +struct AdaptiveDelayState { + // For the adaptive delay implementation + // Sliding window delay tracking: 2 buckets of 30 seconds each + u64 delay_buckets_ns_[2]; // [0] = older 30s, [1] = newer 30s + u64 bucket_start_ns_; // When current bucket (index 1) started + u64 bucket0_window_ns; // 0ns before the first bucket has rolled, and set to + // the bucket window time after This handles the case + // where, before the program has ran one bucket window + // duration, we should not include the previous bucket + // duration in the overhead percent calculation. + unsigned int tls_random_seed_; + bool tls_initialized_; +}; + +// Fixed-point arithmetic type that mimics floating point operations +class Percent { + using u32 = __sanitizer::u32; + using u64 = __sanitizer::u64; + + u32 bp_{}; // basis points (0-10000 represents 0.0-1.0) + bool is_valid_{}; + + static constexpr u32 kBasisPointsPerUnit = 10000; + + Percent(u32 bp, bool is_valid) : bp_(bp), is_valid_(is_valid) {} + + public: + Percent() = default; + Percent(const Percent&) = default; + Percent& operator=(const Percent&) = default; + Percent(Percent&&) = default; + Percent& operator=(Percent&&) = default; + + static Percent FromPct(u32 pct) { return Percent{pct * 100, true}; } + static Percent FromRatio(u64 numerator, u64 denominator) { + if (denominator == 0) + return Percent{0, false}; + // Avoid overflow: scale down if needed + if (numerator > UINT64_MAX / kBasisPointsPerUnit) { + return Percent{(u32)((numerator / denominator) * kBasisPointsPerUnit), + true}; + } + return Percent{(u32)((numerator * kBasisPointsPerUnit) / denominator), + true}; + } + + bool IsValid() const { return is_valid_; } + + // Returns true with probability equal to the percentage. + bool RandomCheck(u32* seed) const { + return (Rand(seed) % kBasisPointsPerUnit) < bp_; + } + + int GetPct() const { return bp_ / 100; } + int GetBasisPoints() const { return bp_; } + + bool operator==(const Percent& other) const { return bp_ == other.bp_; } + bool operator!=(const Percent& other) const { return bp_ != other.bp_; } + bool operator<(const Percent& other) const { return bp_ < other.bp_; } + bool operator>(const Percent& other) const { return bp_ > other.bp_; } + bool operator<=(const Percent& other) const { return bp_ <= other.bp_; } + bool operator>=(const Percent& other) const { return bp_ >= other.bp_; } + + Percent operator-(const Percent& other) const { + if (!is_valid_ || !other.is_valid_) + return Percent{0, false}; + if (bp_ < other.bp_) + return Percent{0, false}; + return Percent{bp_ - other.bp_, true}; + } + + Percent operator/(const Percent& other) const { + if (!is_valid_ || !other.is_valid_) + return Percent{0, false}; + if (other.bp_ == 0) + return Percent{0, false}; + return Percent{(bp_ * kBasisPointsPerUnit) / other.bp_, true}; + } +}; + +} // namespace __tsan + +#endif // TSAN_ADAPTIVE_DELAY_H diff --git a/lib/libtsan/tsan_flags.cpp b/lib/libtsan/tsan_flags.cpp index efaaef8b7ae98e69d3f190dc603652fc08c77f22..fba976594b29d997b53c56eefc69879782bfa7d3 100644 --- a/lib/libtsan/tsan_flags.cpp +++ b/lib/libtsan/tsan_flags.cpp @@ -37,7 +37,7 @@ inline bool FlagHandler::Parse(const char *value) { *t_ = kNoLockDuringWritesAllProcesses; return true; } - Printf("ERROR: Invalid value for signal handler option: '%s'\n", value); + Printf("ERROR: Invalid value for lock_during_write option: '%s'\n", value); return false; } diff --git a/lib/libtsan/tsan_flags.inc b/lib/libtsan/tsan_flags.inc index 77ab910f08fbc062ccabfac8b9c6d00b7ab697e8..68d4ba660debb708292cff19bbd8f777332a1595 100644 --- a/lib/libtsan/tsan_flags.inc +++ b/lib/libtsan/tsan_flags.inc @@ -92,3 +92,30 @@ TSAN_FLAG(LockDuringWriteSetting, lock_during_write, kLockDuringAllWrites, "\"disable_for_all_processes\" - don't lock during all writes in " "the current process and it's children processes.") #endif + +TSAN_FLAG(bool, enable_adaptive_delay, false, + "Enable adaptive delay injection to expose data races. When " + "enabled, delays are strategically injected at synchronization " + "points, atomic operations, and thread lifecycle events to increase " + "the likelihood of exposing races while maintaining a configurable " + "overhead budget.") + +TSAN_FLAG( + int, adaptive_delay_aggressiveness, 25, + "Controls delay injection intensity for race detection. Higher values " + "inject more delays to expose races. Suggested values: 10 (minimal delay), " + "50 (moderate delay), 200 (aggressive). " + "This is a tuning parameter; actual overhead varies by workload and " + "platform.") +TSAN_FLAG(int, adaptive_delay_relaxed_sample_rate, 10000, + "Sample 1 in N relaxed atomic operations for delay") +TSAN_FLAG(int, adaptive_delay_sync_atomic_sample_rate, 100, + "Sample 1 in N acquire/release/seq_cst atomic operations for delay") +TSAN_FLAG(int, adaptive_delay_mutex_sample_rate, 10, + "Sample 1 in N mutex/cv operations for delay") +TSAN_FLAG(const char*, adaptive_delay_max_atomic, "sleep_us=50", + "Delay for atomic operations: 'spin=N' (max N spins), 'yield', or " + "'sleep_us=N' (max N>0 us sleep)") +TSAN_FLAG(const char*, adaptive_delay_max_sync, "sleep_us=500", + "Delay for sync operations: 'spin=N' (max N spins), 'yield', or " + "'sleep_us=N' (max N>0 us sleep)") diff --git a/lib/libtsan/tsan_interceptors_posix.cpp b/lib/libtsan/tsan_interceptors_posix.cpp index 714220a0109a80429c5b04c390cf5423be5c481e..e00853bd1b4262949b68f56671b5db16778187e8 100644 --- a/lib/libtsan/tsan_interceptors_posix.cpp +++ b/lib/libtsan/tsan_interceptors_posix.cpp @@ -34,6 +34,7 @@ #if SANITIZER_APPLE && !SANITIZER_GO # include "tsan_flags.h" #endif +#include "tsan_adaptive_delay.h" #include "tsan_interceptors.h" #include "tsan_interface.h" #include "tsan_mman.h" @@ -1065,6 +1066,9 @@ extern "C" void *__tsan_thread_start_func(void *arg) { ThreadStart(thr, p->tid, GetTid(), ThreadType::Regular); p->started.Post(); } + + AdaptiveDelay::BeforeChildThreadRuns(); + void *res = callback(param); // Prevent the callback from being tail called, // it mixes up stack traces. @@ -1128,6 +1132,7 @@ TSAN_INTERCEPTOR(int, pthread_create, } if (attr == &myattr) pthread_attr_destroy(&myattr); + AdaptiveDelay::AfterThreadCreation(); return res; } @@ -1423,6 +1428,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock, m); MutexPreLock(thr, pc, (uptr)m); + AdaptiveDelay::SyncOp(); int res = BLOCK_REAL(pthread_mutex_lock)(m); if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); @@ -1435,6 +1441,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) { TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m); + AdaptiveDelay::SyncOp(); int res = REAL(pthread_mutex_trylock)(m); if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); @@ -1446,6 +1453,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime); + AdaptiveDelay::SyncOp(); int res = REAL(pthread_mutex_timedlock)(m, abstime); if (res == 0) { MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); @@ -1458,6 +1466,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_unlock, m); MutexUnlock(thr, pc, (uptr)m); int res = REAL(pthread_mutex_unlock)(m); + AdaptiveDelay::SyncOp(); if (res == errno_EINVAL) MutexInvalidAccess(thr, pc, (uptr)m); return res; @@ -1468,6 +1477,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_clocklock, void *m, __sanitizer_clockid_t clock, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_clocklock, m, clock, abstime); MutexPreLock(thr, pc, (uptr)m); + AdaptiveDelay::SyncOp(); int res = BLOCK_REAL(pthread_mutex_clocklock)(m, clock, abstime); if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); @@ -1486,6 +1496,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_clocklock, void *m, TSAN_INTERCEPTOR(int, __pthread_mutex_lock, void *m) { SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_lock, m); MutexPreLock(thr, pc, (uptr)m); + AdaptiveDelay::SyncOp(); int res = BLOCK_REAL(__pthread_mutex_lock)(m); if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); @@ -1500,6 +1511,7 @@ TSAN_INTERCEPTOR(int, __pthread_mutex_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_unlock, m); MutexUnlock(thr, pc, (uptr)m); int res = REAL(__pthread_mutex_unlock)(m); + AdaptiveDelay::SyncOp(); if (res == errno_EINVAL) MutexInvalidAccess(thr, pc, (uptr)m); return res; @@ -1529,6 +1541,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) { TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m); MutexPreLock(thr, pc, (uptr)m); + AdaptiveDelay::SyncOp(); int res = BLOCK_REAL(pthread_spin_lock)(m); if (res == 0) { MutexPostLock(thr, pc, (uptr)m); @@ -1538,6 +1551,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) { TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m); + AdaptiveDelay::SyncOp(); int res = REAL(pthread_spin_trylock)(m); if (res == 0) { MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); @@ -1549,6 +1563,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m); MutexUnlock(thr, pc, (uptr)m); int res = REAL(pthread_spin_unlock)(m); + AdaptiveDelay::SyncOp(); return res; } #endif @@ -1574,6 +1589,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) { TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m); MutexPreReadLock(thr, pc, (uptr)m); + AdaptiveDelay::SyncOp(); int res = REAL(pthread_rwlock_rdlock)(m); if (res == 0) { MutexPostReadLock(thr, pc, (uptr)m); @@ -1583,6 +1599,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) { TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m); + AdaptiveDelay::SyncOp(); int res = REAL(pthread_rwlock_tryrdlock)(m); if (res == 0) { MutexPostReadLock(thr, pc, (uptr)m, MutexFlagTryLock); @@ -1593,6 +1610,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime); + AdaptiveDelay::SyncOp(); int res = REAL(pthread_rwlock_timedrdlock)(m, abstime); if (res == 0) { MutexPostReadLock(thr, pc, (uptr)m); @@ -1604,6 +1622,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m); MutexPreLock(thr, pc, (uptr)m); + AdaptiveDelay::SyncOp(); int res = BLOCK_REAL(pthread_rwlock_wrlock)(m); if (res == 0) { MutexPostLock(thr, pc, (uptr)m); @@ -1613,6 +1632,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) { TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m); + AdaptiveDelay::SyncOp(); int res = REAL(pthread_rwlock_trywrlock)(m); if (res == 0) { MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); @@ -1623,6 +1643,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime); + AdaptiveDelay::SyncOp(); int res = REAL(pthread_rwlock_timedwrlock)(m, abstime); if (res == 0) { MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); @@ -1635,6 +1656,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m); MutexReadOrWriteUnlock(thr, pc, (uptr)m); int res = REAL(pthread_rwlock_unlock)(m); + AdaptiveDelay::SyncOp(); return res; } @@ -2574,9 +2596,9 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, #define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \ ({ \ - CheckNoDeepBind(filename, flag); \ + OnDlOpen(filename, flag); \ ThreadIgnoreBegin(thr, 0); \ - void *res = REAL(dlopen)(filename, flag); \ + void* res = REAL(dlopen)(filename, flag); \ ThreadIgnoreEnd(thr); \ res; \ }) diff --git a/lib/libtsan/tsan_interface_ann.cpp b/lib/libtsan/tsan_interface_ann.cpp index 02ca82369aae127af8d01289868a10c08b131c80..490f6cb3bce01b45753f06f34aa23dbf1c0265bd 100644 --- a/lib/libtsan/tsan_interface_ann.cpp +++ b/lib/libtsan/tsan_interface_ann.cpp @@ -9,17 +9,19 @@ // This file is a part of ThreadSanitizer (TSan), a race detector. // //===----------------------------------------------------------------------===// -#include "sanitizer_common/sanitizer_libc.h" +#include "tsan_interface_ann.h" + #include "sanitizer_common/sanitizer_internal_defs.h" +#include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_stacktrace.h" #include "sanitizer_common/sanitizer_vector.h" -#include "tsan_interface_ann.h" -#include "tsan_report.h" -#include "tsan_rtl.h" -#include "tsan_mman.h" +#include "tsan_adaptive_delay.h" #include "tsan_flags.h" +#include "tsan_mman.h" #include "tsan_platform.h" +#include "tsan_report.h" +#include "tsan_rtl.h" #define CALLERPC ((uptr)__builtin_return_address(0)) @@ -370,6 +372,7 @@ void __tsan_mutex_pre_lock(void *m, unsigned flagz) { } ThreadIgnoreBegin(thr, 0); ThreadIgnoreSyncBegin(thr, 0); + AdaptiveDelay::SyncOp(); } INTERFACE_ATTRIBUTE @@ -402,6 +405,7 @@ int __tsan_mutex_pre_unlock(void *m, unsigned flagz) { INTERFACE_ATTRIBUTE void __tsan_mutex_post_unlock(void *m, unsigned flagz) { + AdaptiveDelay::SyncOp(); SCOPED_ANNOTATION(__tsan_mutex_post_unlock); ThreadIgnoreSyncEnd(thr); ThreadIgnoreEnd(thr); diff --git a/lib/libtsan/tsan_interface_atomic.cpp b/lib/libtsan/tsan_interface_atomic.cpp index 527e5a9b4a8d8aec1d20a5428d8994575d5b3557..5c2461634d2d4afb0423c45add3700b49abc03ee 100644 --- a/lib/libtsan/tsan_interface_atomic.cpp +++ b/lib/libtsan/tsan_interface_atomic.cpp @@ -21,6 +21,7 @@ #include "sanitizer_common/sanitizer_mutex.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_stacktrace.h" +#include "tsan_adaptive_delay.h" #include "tsan_flags.h" #include "tsan_interface.h" #include "tsan_rtl.h" @@ -520,8 +521,19 @@ static morder to_morder(int mo) { return res; } +template +ALWAYS_INLINE auto AtomicDelayImpl(morder mo, Types... args) { + AdaptiveDelay::AtomicOpFence(mo); +} + +template +ALWAYS_INLINE auto AtomicDelayImpl(morder mo, AddrType addr, Types... args) { + AdaptiveDelay::AtomicOpAddr((uptr)addr, (int)mo); +} + template ALWAYS_INLINE auto AtomicImpl(morder mo, Types... args) { + AtomicDelayImpl(mo, args...); ThreadState *const thr = cur_thread(); ProcessPendingSignals(thr); if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) diff --git a/lib/libtsan/tsan_platform.h b/lib/libtsan/tsan_platform.h index 7089be4d5d7f7c69e6bb488d7c9a06491751f5da..5edec59d5af52f99f79a051b91599fca8398883b 100644 --- a/lib/libtsan/tsan_platform.h +++ b/lib/libtsan/tsan_platform.h @@ -404,7 +404,7 @@ struct MappingRiscv64_39 { static const uptr kHeapMemBeg = 0x2c00000000ull; static const uptr kHeapMemEnd = 0x2c00000000ull; static const uptr kHiAppMemBeg = 0x3c00000000ull; - static const uptr kHiAppMemEnd = 0x3fffffffffull; + static const uptr kHiAppMemEnd = 0x4000000000ull; static const uptr kShadowMsk = 0x3800000000ull; static const uptr kShadowXor = 0x0800000000ull; static const uptr kShadowAdd = 0x0000000000ull; @@ -434,7 +434,7 @@ struct MappingRiscv64_48 { static const uptr kHeapMemBeg = 0x5a0000000000ull; static const uptr kHeapMemEnd = 0x5a0000000000ull; static const uptr kHiAppMemBeg = 0x7a0000000000ull; - static const uptr kHiAppMemEnd = 0x7fffffffffffull; + static const uptr kHiAppMemEnd = 0x800000000000ull; static const uptr kShadowMsk = 0x700000000000ull; static const uptr kShadowXor = 0x100000000000ull; static const uptr kShadowAdd = 0x000000000000ull; @@ -738,13 +738,16 @@ struct MappingGoRiscv64_48 { Go on linux/s390x 0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB 1000 0000 0000 - 4000 0000 0000: - -4000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app) -6000 0000 0000 - 9000 0000 0000: - -9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app) +4000 0000 0000 - 6000 0000 0000: shadow - 32 TiB (2 * app) +6000 0000 0000 - 7000 0000 0000: - +7000 0000 0000 - 7800 0000 0000: metainfo - 8 TiB (0.5 * app) +7800 0000 0000 - 8000 0000 0000: - */ struct MappingGoS390x { - static const uptr kMetaShadowBeg = 0x900000000000ull; - static const uptr kMetaShadowEnd = 0x980000000000ull; + // Keep the mapping below 2^47 for QEMU linux-user on x86-64 hosts with + // four-level page tables. + static const uptr kMetaShadowBeg = 0x700000000000ull; + static const uptr kMetaShadowEnd = 0x780000000000ull; static const uptr kShadowBeg = 0x400000000000ull; static const uptr kShadowEnd = 0x600000000000ull; static const uptr kLoAppMemBeg = 0x000000001000ull; diff --git a/lib/libtsan/tsan_platform_linux.cpp b/lib/libtsan/tsan_platform_linux.cpp index c974f549acbcbf81d3b188b9022e77d1134b18d3..b0296a0756f3b6583561706568dc329c8908ba6d 100644 --- a/lib/libtsan/tsan_platform_linux.cpp +++ b/lib/libtsan/tsan_platform_linux.cpp @@ -27,6 +27,18 @@ #include "tsan_platform.h" #include "tsan_rtl.h" +#if SANITIZER_NETBSD +# // for __lwp_gettcb_fast() / __lwp_getprivate_fast() +# define _RTLD_SOURCE +# include +# include +# undef _RTLD_SOURCE +# include +# if __NetBSD_Version__ >= 1099001200 +# include +# endif +#endif + #include #include #include diff --git a/lib/libtsan/tsan_report.cpp b/lib/libtsan/tsan_report.cpp index 22ba428cc58b77b716d5a6cd54026b2510d854d0..17fd3064fb85a14b2cdc85c8cdb2c4ae94232f1c 100644 --- a/lib/libtsan/tsan_report.cpp +++ b/lib/libtsan/tsan_report.cpp @@ -317,8 +317,9 @@ void PrintReport(const ReportDesc *rep) { } else { PrintStack(rep->stacks[i]); if (i == 0) - Printf(" Hint: use TSAN_OPTIONS=second_deadlock_stack=1 " - "to get more informative warning message\n\n"); + Printf( + " HINT: use TSAN_OPTIONS=second_deadlock_stack=1 " + "to get more informative warning message\n\n"); } } } else { diff --git a/lib/libtsan/tsan_rtl.cpp b/lib/libtsan/tsan_rtl.cpp index feee566f448298eab49d057b2964904a21cd1e0b..a9147f09e34cba30adef0408b34c5dc291d4cef7 100644 --- a/lib/libtsan/tsan_rtl.cpp +++ b/lib/libtsan/tsan_rtl.cpp @@ -21,6 +21,7 @@ #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_stackdepot.h" #include "sanitizer_common/sanitizer_symbolizer.h" +#include "tsan_adaptive_delay.h" #include "tsan_defs.h" #include "tsan_interface.h" #include "tsan_mman.h" @@ -775,6 +776,10 @@ void Initialize(ThreadState *thr) { while (__tsan_resumed == 0) {} } +#if !SANITIZER_GO + AdaptiveDelay::Init(); +#endif + OnInitialize(); } diff --git a/lib/libtsan/tsan_rtl.h b/lib/libtsan/tsan_rtl.h index 635654616b7811b317c444fc7f06267d2b50abc3..3d1018accafc402a28dc8e650355bab1ecf8ef7a 100644 --- a/lib/libtsan/tsan_rtl.h +++ b/lib/libtsan/tsan_rtl.h @@ -34,6 +34,7 @@ #include "sanitizer_common/sanitizer_suppressions.h" #include "sanitizer_common/sanitizer_thread_registry.h" #include "sanitizer_common/sanitizer_vector.h" +#include "tsan_adaptive_delay.h" #include "tsan_defs.h" #include "tsan_flags.h" #include "tsan_ignoreset.h" @@ -240,6 +241,8 @@ struct alignas(SANITIZER_CACHE_LINE_SIZE) ThreadState { bool in_internal_write_call; #endif + AdaptiveDelayState adaptive_delay_state; + explicit ThreadState(Tid tid); }; diff --git a/lib/libtsan/ubsan/ubsan_flags.h b/lib/libtsan/ubsan/ubsan_flags.h index c47009bafe5399c3ac5f78a832c9681aa776bf90..1e4a6dc7bd65a88ddef6a9101fd03da755120482 100644 --- a/lib/libtsan/ubsan/ubsan_flags.h +++ b/lib/libtsan/ubsan/ubsan_flags.h @@ -41,6 +41,10 @@ extern "C" { // override the default flag values. SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char *__ubsan_default_options(); +// Users may provide their own implementation of __ubsan_default_suppressions to +// override the default suppression values. +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char * +__ubsan_default_suppressions(); } // extern "C" #endif // UBSAN_FLAGS_H diff --git a/src/libs/libtsan.zig b/src/libs/libtsan.zig index 2621391c769f63c678ab419c0bc8db7c95f2326e..7212a17e38e5dba06208c23feb698ad46527a20e 100644 --- a/src/libs/libtsan.zig +++ b/src/libs/libtsan.zig @@ -350,6 +350,7 @@ fn addCcArgs(target: *const std.Target, args: *std.array_list.Managed([]const u8 } const tsan_sources = [_][]const u8{ + "tsan_adaptive_delay.cpp", "tsan_debugging.cpp", "tsan_external.cpp", "tsan_fd.cpp", -- 2.54.0