Commit 6498f963bc7ecf275173b82ec4da67046116db1a | zig

author	Alex Rønne Petersen <alex@alexrp.com> 2026-08-03 13:06:07+02:00
committer	Alex Rønne Petersen <alex@alexrp.com> 2026-09-05 14:54:30+02:00
log	6498f963bc7ecf275173b82ec4da67046116db1a
tree	b8aa64edb788fd62b3a7078d5056d72d4bd55396
parent	5c17ee74127366ad8cc4d4ce82f11aa51899e28a
signature	Signed by SSH key SHA256:7B/LJ7bpR1eX8aCXSr4mtd5M45VMPKcx9zY8e95b5QM

libtsan: update to LLVM 23

55 files changed, 1399 insertions(+), 297 deletions(-)

lib
- libtsan
src
- libs
  - libtsan.zig

lib/libtsan/interception/interception.h+10

...	...	@@ -362,6 +362,16 @@ const interpose_substitution substitution_##func_name[] \
362	362	// so we use casts via uintptr_t (the local __sanitizer::uptr equivalent).
363	363	namespace __interception {
364	364
	365	// Dynamic library loading helpers (dlopen/LoadLibrary, dlsym/GetProcAddress).
	366	// Implemented in interception_linux.cpp on non-Windows targets and
	367	// interception_win.cpp on Windows.
	368	bool DynamicLoaderAvailable();
	369	void* OpenLibrary(const char* name);
	370	void* LookupSymbol(void* handle, const char* symbol);
	371	void* LookupSymbolDefault(const char* symbol);
	372	void* LookupSymbolNext(const char* symbol);
	373	void* LookupSymbolNextVersioned(const char* symbol, const char* version);
	374
365	375	#if defined(__ELF__) && !SANITIZER_FUCHSIA
366	376	// The use of interceptors makes many sanitizers unusable for static linking.
367	377	// Define a function, if called, will cause a linker error (undefined _DYNAMIC).

lib/libtsan/interception/interception_linux.cpp+58-5

...	...	@@ -9,15 +9,68 @@
9	9	// This file is a part of AddressSanitizer, an address sanity checker.
10	10	//
11	11	// Linux-specific interception methods.
	12	//
	13	// POSIX dynamic-library helpers (dlopen / dlsym) live here for every
	14	// non-Windows interception target (Linux, *BSD, Darwin, AIX, Fuchsia, ...).
	15	// macOS/AIX/Fuchsia compile this TU for RTInterception but do not use the
	16	// Linux-specific InterceptFunction helpers below.
12	17	//===----------------------------------------------------------------------===//
13	18
14	19	#include "interception.h"
15	20
	21	#if !SANITIZER_WINDOWS
	22
	23	# include <dlfcn.h>
	24
	25	# pragma weak dlopen
	26	# pragma weak dlsym
	27	#if SANITIZER_GLIBC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
	28	# pragma weak dlvsym
	29	#endif
	30
	31	namespace __interception {
	32
	33	bool DynamicLoaderAvailable() { return dlopen != nullptr && dlsym != nullptr; }
	34
	35	void* OpenLibrary(const char* name) {
	36	if (!DynamicLoaderAvailable())
	37	return nullptr;
	38	return dlopen(name, RTLD_LAZY \| RTLD_LOCAL);
	39	}
	40
	41	void* LookupSymbol(void* handle, const char* symbol) {
	42	if (!DynamicLoaderAvailable())
	43	return nullptr;
	44	return dlsym(handle, symbol);
	45	}
	46
	47	void* LookupSymbolDefault(const char* symbol) {
	48	if (!DynamicLoaderAvailable())
	49	return nullptr;
	50	return dlsym(RTLD_DEFAULT, symbol);
	51	}
	52
	53	void* LookupSymbolNext(const char* symbol) {
	54	if (!DynamicLoaderAvailable())
	55	return nullptr;
	56	return dlsym(RTLD_NEXT, symbol);
	57	}
	58
	59	#if SANITIZER_GLIBC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
	60	void* LookupSymbolNextVersioned(const char* symbol, const char* version) {
	61	if (!DynamicLoaderAvailable() \|\| dlvsym == nullptr)
	62	return nullptr;
	63	return dlvsym(RTLD_NEXT, symbol, version);
	64	}
	65	#endif // SANITIZER_GLIBC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
	66
	67	} // namespace __interception
	68
	69	#endif // !SANITIZER_WINDOWS
	70
16	71	#if SANITIZER_LINUX \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| \
17	72	SANITIZER_SOLARIS \|\| SANITIZER_HAIKU
18	73
19		#include <dlfcn.h> // for dlsym() and dlvsym()
20
21	74	namespace __interception {
22	75
23	76	#if SANITIZER_NETBSD
...	...	@@ -39,14 +92,14 @@ static void GetFuncAddr(const char name, uptr trampoline) {
39	92	if (StrCmp(name, "sigaction"))
40	93	name = "__sigaction14";
41	94	#endif
42		void *addr = dlsym(RTLD_NEXT, name);
	95	void* addr = LookupSymbolNext(name);
43	96	if (!addr) {
44	97	// If the lookup using RTLD_NEXT failed, the sanitizer runtime library is
45	98	// later in the library search order than the DSO that we are trying to
46	99	// intercept, which means that we cannot intercept this function. We still
47	100	// want the address of the real definition, though, so look it up using
48	101	// RTLD_DEFAULT.
49		addr = dlsym(RTLD_DEFAULT, name);
	102	addr = LookupSymbolDefault(name);
50	103
51	104	// In case `name' is not loaded, dlsym ends up finding the actual wrapper.
52	105	// We don't want to intercept the wrapper and have it point to itself.
...	...	@@ -66,7 +119,7 @@ bool InterceptFunction(const char name, uptr ptr_to_real, uptr func,
66	119	// dlvsym is a GNU extension supported by some other platforms.
67	120	#if SANITIZER_GLIBC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
68	121	static void GetFuncAddr(const char name, const char *ver) {
69		return dlvsym(RTLD_NEXT, name, ver);
	122	return LookupSymbolNextVersioned(name, ver);
70	123	}
71	124
72	125	bool InterceptFunction(const char name, const char ver, uptr *ptr_to_real,

lib/libtsan/interception/interception_win.cpp+25

...	...	@@ -134,6 +134,30 @@
134	134
135	135	namespace __interception {
136	136
	137	bool DynamicLoaderAvailable() { return true; }
	138
	139	void* OpenLibrary(const char* name) {
	140	if (!name)
	141	return reinterpret_cast<void*>(GetModuleHandleA(nullptr));
	142	return reinterpret_cast<void*>(LoadLibraryA(name));
	143	}
	144
	145	void* LookupSymbol(void* handle, const char* symbol) {
	146	if (!handle)
	147	return nullptr;
	148	return reinterpret_cast<void*>(reinterpret_cast<__sanitizer::uptr>(
	149	GetProcAddress(reinterpret_cast<HMODULE>(handle), symbol)));
	150	}
	151
	152	void* LookupSymbolDefault(const char* symbol) {
	153	return LookupSymbol(reinterpret_cast<void*>(GetModuleHandleA(nullptr)),
	154	symbol);
	155	}
	156
	157	void* LookupSymbolNext(const char*) { return nullptr; }
	158
	159	void* LookupSymbolNextVersioned(const char, const char) { return nullptr; }
	160
137	161	static const int kAddressLength = FIRST_32_SECOND_64(4, 8);
138	162	static const int kJumpInstructionLength = 5;
139	163	static const int kShortJumpInstructionLength = 2;
...	...	@@ -655,6 +679,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
655	679	return 2;
656	680
657	681	case 0x3980: // 80 39 XX : cmp BYTE PTR [rcx], XX
	682	case 0x3a80: // 80 3A XX : cmp BYTE PTR [rdx], XX
658	683	case 0x4D8B: // 8B 4D XX : mov XX(%ebp), ecx
659	684	case 0x558B: // 8B 55 XX : mov XX(%ebp), edx
660	685	case 0x758B: // 8B 75 XX : mov XX(%ebp), esp

lib/libtsan/sanitizer_common/sanitizer_allocator.cpp+3-2

...	...	@@ -109,14 +109,15 @@ void InternalReallocArray(void addr, uptr count, uptr size,
109	109	return InternalRealloc(addr, count * size, cache);
110	110	}
111	111
112		void InternalCalloc(uptr count, uptr size, InternalAllocatorCache cache) {
	112	void* InternalCalloc(uptr count, uptr size, InternalAllocatorCache* cache,
	113	uptr alignment) {
113	114	if (UNLIKELY(CheckForCallocOverflow(count, size))) {
114	115	Report("FATAL: %s: calloc parameters overflow: count * size (%zd * %zd) "
115	116	"cannot be represented in type size_t\n", SanitizerToolName, count,
116	117	size);
117	118	Die();
118	119	}
119		void p = InternalAlloc(count size, cache);
	120	void* p = InternalAlloc(count * size, cache, alignment);
120	121	if (LIKELY(p))
121	122	internal_memset(p, 0, count * size);
122	123	return p;

lib/libtsan/sanitizer_common/sanitizer_allocator.h-6

...	...	@@ -47,12 +47,6 @@ void PrintHintAllocatorCannotReturnNull();
47	47	// Callback type for iterating over chunks.
48	48	typedef void (ForEachChunkCallback)(uptr chunk, void arg);
49	49
50		inline u32 Rand(u32 *state) { // ANSI C linear congruential PRNG.
51		return (state = state * 1103515245 + 12345) >> 16;
52		}
53
54		inline u32 RandN(u32 *state, u32 n) { return Rand(state) % n; } // [0, n)
55
56	50	template<typename T>
57	51	inline void RandomShuffle(T a, u32 n, u32 rand_state) {
58	52	if (n <= 1) return;

lib/libtsan/sanitizer_common/sanitizer_allocator_dlsym.h+5-5

...	...	@@ -40,8 +40,8 @@ struct DlSymAllocator {
40	40	return ptr;
41	41	}
42	42
43		static void *Callocate(usize nmemb, usize size) {
44		void *ptr = InternalCalloc(nmemb, size);
	43	static void* Callocate(usize nmemb, usize size, uptr align = kWordSize) {
	44	void* ptr = InternalCalloc(nmemb, size, nullptr, align);
45	45	CHECK(internal_allocator()->FromPrimary(ptr));
46	46	Details::OnAllocate(ptr, GetSize(ptr));
47	47	return ptr;
...	...	@@ -53,9 +53,9 @@ struct DlSymAllocator {
53	53	InternalFree(ptr);
54	54	}
55	55
56		static void Realloc(void ptr, uptr new_size) {
	56	static void* Realloc(void* ptr, uptr new_size, uptr align = kWordSize) {
57	57	if (!ptr)
58		return Allocate(new_size);
	58	return Allocate(new_size, align);
59	59	CHECK(internal_allocator()->FromPrimary(ptr));
60	60	if (!new_size) {
61	61	Free(ptr);
...	...	@@ -63,7 +63,7 @@ struct DlSymAllocator {
63	63	}
64	64	uptr size = GetSize(ptr);
65	65	uptr memcpy_size = Min(new_size, size);
66		void *new_ptr = Allocate(new_size);
	66	void* new_ptr = Allocate(new_size, align);
67	67	if (new_ptr)
68	68	internal_memcpy(new_ptr, ptr, memcpy_size);
69	69	Free(ptr);

lib/libtsan/sanitizer_common/sanitizer_allocator_internal.h+3-2

...	...	@@ -45,8 +45,9 @@ void InternalRealloc(void p, uptr size,
45	45	InternalAllocatorCache *cache = nullptr);
46	46	void InternalReallocArray(void p, uptr count, uptr size,
47	47	InternalAllocatorCache *cache = nullptr);
48		void *InternalCalloc(uptr count, uptr size,
49		InternalAllocatorCache *cache = nullptr);
	48	void* InternalCalloc(uptr count, uptr size,
	49	InternalAllocatorCache* cache = nullptr,
	50	uptr alignment = 0);
50	51	void InternalFree(void p, InternalAllocatorCache cache = nullptr);
51	52	void InternalAllocatorLock();
52	53	void InternalAllocatorUnlock();

lib/libtsan/sanitizer_common/sanitizer_asm.h+4-2

...	...	@@ -61,6 +61,8 @@
61	61	# define ASM_TAIL_CALL jg
62	62	#elif defined(__riscv)
63	63	# define ASM_TAIL_CALL tail
	64	#elif defined(__hexagon__)
	65	# define ASM_TAIL_CALL jump
64	66	#endif
65	67
66	68	// Currently, almost all of the shared libraries rely on the value of
...	...	@@ -103,8 +105,8 @@
103	105	# define ASM_SIZE(symbol) .size symbol, .-symbol
104	106	# define ASM_SYMBOL(symbol) symbol
105	107	# define ASM_SYMBOL_INTERCEPTOR(symbol) symbol
106		# if defined(__i386__) \|\| defined(__powerpc__) \|\| defined(__s390__) \|\| \
107		defined(__sparc__)
	108	# if defined(__i386__) \|\| defined(__powerpc__) \|\| defined(__s390__) \|\| \
	109	defined(__sparc__) \|\| defined(__alpha__)
108	110	// For details, see interception.h
109	111	# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol
110	112	# define ASM_TRAMPOLINE_ALIAS(symbol, name) \

lib/libtsan/sanitizer_common/sanitizer_common.h+19-4

...	...	@@ -387,8 +387,8 @@ void ReportDeadlySignal(const SignalContext &sig, u32 tid,
387	387	const void *unwind_context);
388	388
389	389	// Alternative signal stack (POSIX-only).
390		void SetAlternateSignalStack();
391		void UnsetAlternateSignalStack();
	390	void* SetAlternateSignalStack();
	391	void UnsetAlternateSignalStack(void* altstack_base);
392	392
393	393	bool IsSignalHandlerFromSanitizer(int signum);
394	394	bool SetSignalHandlerFromSanitizer(int signum, bool new_state);
...	...	@@ -906,7 +906,14 @@ class LoadedModule {
906	906	class ListOfModules {
907	907	public:
908	908	ListOfModules() : initialized(false) {}
909		~ListOfModules() { clear(); }
	909	~ListOfModules() {
	910	clear();
	911	if (initialized)
	912	modules_.Destroy();
	913	}
	914	ListOfModules(const ListOfModules&) = delete;
	915	ListOfModules& operator=(const ListOfModules&) = delete;
	916
910	917	void init();
911	918	void fallbackInit(); // Uses fallback init if available, otherwise clears
912	919	const LoadedModule *begin() const { return modules_.begin(); }
...	...	@@ -1085,7 +1092,9 @@ struct StackDepotStats {
1085	1092	// indicate that sanitizer allocator should not attempt to release memory to OS.
1086	1093	const s32 kReleaseToOSIntervalNever = -1;
1087	1094
1088		void CheckNoDeepBind(const char *filename, int flag);
	1095	// Platform hook invoked before dlopen. Performs platform-specific dlopen flag
	1096	// checks (e.g. RTLD_DEEPBIND on Linux).
	1097	void OnDlOpen(const char* filename, int flag);
1089	1098
1090	1099	// Returns the requested amount of random data (up to 256 bytes) that can then
1091	1100	// be used to seed a PRNG. Defaults to blocking like the underlying syscall.
...	...	@@ -1100,6 +1109,12 @@ inline u32 GetNumberOfCPUsCached() {
1100	1109	return NumberOfCPUsCached;
1101	1110	}
1102	1111
	1112	inline u32 Rand(u32* state) { // ANSI C linear congruential PRNG.
	1113	return (state = state * 1103515245 + 12345) >> 16;
	1114	}
	1115
	1116	inline u32 RandN(u32* state, u32 n) { return Rand(state) % n; } // [0, n)
	1117
1103	1118	} // namespace __sanitizer
1104	1119
1105	1120	inline void *operator new(__sanitizer::usize size,

lib/libtsan/sanitizer_common/sanitizer_common_interceptors.inc+65-2

...	...	@@ -277,8 +277,11 @@ extern const short *_tolower_tab_;
277	277	common_flags()->strict_string_checks ? (internal_strlen(s)) + 1 : (n) )
278	278
279	279	#ifndef COMMON_INTERCEPTOR_DLOPEN
280		#define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \
281		({ CheckNoDeepBind(filename, flag); REAL(dlopen)(filename, flag); })
	280	# define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \
	281	({ \
	282	OnDlOpen(filename, flag); \
	283	REAL(dlopen)(filename, flag); \
	284	})
282	285	#endif
283	286
284	287	#ifndef COMMON_INTERCEPTOR_GET_TLS_RANGE
...	...	@@ -1023,6 +1026,25 @@ INTERCEPTOR(SSIZE_T, read, int fd, void *ptr, SIZE_T count) {
1023	1026	#define INIT_READ
1024	1027	#endif
1025	1028
	1029	#if SANITIZER_INTERCEPT___READ_CHK
	1030	INTERCEPTOR(SSIZE_T, __read_chk, int fd, void* ptr, SIZE_T count,
	1031	SIZE_T buflen) {
	1032	void* ctx;
	1033	COMMON_INTERCEPTOR_ENTER(ctx, __read_chk, fd, ptr, count, buflen);
	1034	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
	1035	SSIZE_T res =
	1036	COMMON_INTERCEPTOR_BLOCK_REAL(__read_chk)(fd, ptr, count, buflen);
	1037	if (res > 0)
	1038	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res);
	1039	if (res >= 0 && fd >= 0)
	1040	COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
	1041	return res;
	1042	}
	1043	# define INIT___READ_CHK COMMON_INTERCEPT_FUNCTION(__read_chk)
	1044	#else
	1045	# define INIT___READ_CHK
	1046	#endif
	1047
1026	1048	#if SANITIZER_INTERCEPT_FREAD
1027	1049	INTERCEPTOR(SIZE_T, fread, void ptr, SIZE_T size, SIZE_T nmemb, void file) {
1028	1050	// libc file streams can call user-supplied functions, see fopencookie.
...	...	@@ -1058,6 +1080,25 @@ INTERCEPTOR(SSIZE_T, pread, int fd, void *ptr, SIZE_T count, OFF_T offset) {
1058	1080	#define INIT_PREAD
1059	1081	#endif
1060	1082
	1083	#if SANITIZER_INTERCEPT___PREAD_CHK
	1084	INTERCEPTOR(SSIZE_T, __pread_chk, int fd, void* ptr, SIZE_T count, OFF_T offset,
	1085	SIZE_T buflen) {
	1086	void* ctx;
	1087	COMMON_INTERCEPTOR_ENTER(ctx, __pread_chk, fd, ptr, count, offset, buflen);
	1088	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
	1089	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(__pread_chk)(fd, ptr, count,
	1090	offset, buflen);
	1091	if (res > 0)
	1092	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res);
	1093	if (res >= 0 && fd >= 0)
	1094	COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
	1095	return res;
	1096	}
	1097	# define INIT___PREAD_CHK COMMON_INTERCEPT_FUNCTION(__pread_chk)
	1098	#else
	1099	# define INIT___PREAD_CHK
	1100	#endif
	1101
1061	1102	#if SANITIZER_INTERCEPT_PREAD64
1062	1103	INTERCEPTOR(SSIZE_T, pread64, int fd, void *ptr, SIZE_T count, OFF64_T offset) {
1063	1104	void *ctx;
...	...	@@ -1076,6 +1117,25 @@ INTERCEPTOR(SSIZE_T, pread64, int fd, void *ptr, SIZE_T count, OFF64_T offset) {
1076	1117	#define INIT_PREAD64
1077	1118	#endif
1078	1119
	1120	#if SANITIZER_INTERCEPT___PREAD64_CHK
	1121	INTERCEPTOR(SSIZE_T, __pread64_chk, int fd, void* ptr, SIZE_T count,
	1122	OFF64_T offset, SIZE_T buflen) {
	1123	void* ctx;
	1124	COMMON_INTERCEPTOR_ENTER(ctx, __pread64_chk, fd, ptr, count, offset, buflen);
	1125	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
	1126	SSIZE_T res = COMMON_INTERCEPTOR_BLOCK_REAL(__pread64_chk)(fd, ptr, count,
	1127	offset, buflen);
	1128	if (res > 0)
	1129	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res);
	1130	if (res >= 0 && fd >= 0)
	1131	COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
	1132	return res;
	1133	}
	1134	# define INIT___PREAD64_CHK COMMON_INTERCEPT_FUNCTION(__pread64_chk)
	1135	#else
	1136	# define INIT___PREAD64_CHK
	1137	#endif
	1138
1079	1139	#if SANITIZER_INTERCEPT_READV
1080	1140	INTERCEPTOR_WITH_SUFFIX(SSIZE_T, readv, int fd, __sanitizer_iovec *iov,
1081	1141	int iovcnt) {
...	...	@@ -10428,9 +10488,12 @@ static void InitializeCommonInterceptors() {
10428	10488	INIT_MEMRCHR;
10429	10489	INIT_MEMMEM;
10430	10490	INIT_READ;
	10491	INIT___READ_CHK;
10431	10492	INIT_FREAD;
10432	10493	INIT_PREAD;
	10494	INIT___PREAD_CHK;
10433	10495	INIT_PREAD64;
	10496	INIT___PREAD64_CHK;
10434	10497	INIT_READV;
10435	10498	INIT_PREADV;
10436	10499	INIT_PREADV64;

lib/libtsan/sanitizer_common/sanitizer_common_interface.inc-1

...	...	@@ -10,7 +10,6 @@
10	10	INTERFACE_FUNCTION(__sanitizer_acquire_crash_state)
11	11	INTERFACE_FUNCTION(__sanitizer_annotate_contiguous_container)
12	12	INTERFACE_FUNCTION(__sanitizer_annotate_double_ended_contiguous_container)
13		INTERFACE_FUNCTION(__sanitizer_copy_contiguous_container_annotations)
14	13	INTERFACE_FUNCTION(__sanitizer_contiguous_container_find_bad_address)
15	14	INTERFACE_FUNCTION(
16	15	__sanitizer_double_ended_contiguous_container_find_bad_address)

lib/libtsan/sanitizer_common/sanitizer_dense_map.h+57-95

...	...	@@ -44,10 +44,10 @@ class DenseMapBase {
44	44	}
45	45
46	46	void clear() {
47		if (getNumEntries() == 0 && getNumTombstones() == 0)
	47	if (getNumEntries() == 0)
48	48	return;
49	49
50		const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey();
	50	const KeyT EmptyKey = getEmptyKey();
51	51	if (__sanitizer::is_trivially_destructible<ValueT>::value) {
52	52	// Use a simpler loop when values don't need destruction.
53	53	for (BucketT P = getBuckets(), E = getBucketsEnd(); P != E; ++P)
...	...	@@ -56,17 +56,14 @@ class DenseMapBase {
56	56	unsigned NumEntries = getNumEntries();
57	57	for (BucketT P = getBuckets(), E = getBucketsEnd(); P != E; ++P) {
58	58	if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey)) {
59		if (!KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) {
60		P->getSecond().~ValueT();
61		--NumEntries;
62		}
	59	P->getSecond().~ValueT();
	60	--NumEntries;
63	61	P->getFirst() = EmptyKey;
64	62	}
65	63	}
66	64	CHECK_EQ(NumEntries, 0);
67	65	}
68	66	setNumEntries(0);
69		setNumTombstones(0);
70	67	}
71	68
72	69	/// Return true if the specified key is in the map, false otherwise.
...	...	@@ -171,20 +168,13 @@ class DenseMapBase {
171	168	if (!TheBucket)
172	169	return false; // not in map.
173	170
174		TheBucket->getSecond().~ValueT();
175		TheBucket->getFirst() = getTombstoneKey();
176		decrementNumEntries();
177		incrementNumTombstones();
	171	eraseFromFilledBucket(TheBucket);
178	172	return true;
179	173	}
180	174
181	175	void erase(value_type *I) {
182	176	CHECK_NE(I, nullptr);
183		BucketT TheBucket = &I;
184		TheBucket->getSecond().~ValueT();
185		TheBucket->getFirst() = getTombstoneKey();
186		decrementNumEntries();
187		incrementNumTombstones();
	177	eraseFromFilledBucket(I);
188	178	}
189	179
190	180	value_type &FindAndConstruct(const KeyT &Key) {
...	...	@@ -214,11 +204,10 @@ class DenseMapBase {
214	204	/// Function can return fast to stop the process.
215	205	template <class Fn>
216	206	void forEach(Fn fn) {
217		const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey();
	207	const KeyT EmptyKey = getEmptyKey();
218	208	for (auto P = getBuckets(), E = getBucketsEnd(); P != E; ++P) {
219	209	const KeyT K = P->getFirst();
220		if (!KeyInfoT::isEqual(K, EmptyKey) &&
221		!KeyInfoT::isEqual(K, TombstoneKey)) {
	210	if (!KeyInfoT::isEqual(K, EmptyKey)) {
222	211	if (!fn(*P))
223	212	return;
224	213	}
...	...	@@ -238,10 +227,9 @@ class DenseMapBase {
238	227	if (getNumBuckets() == 0) // Nothing to do.
239	228	return;
240	229
241		const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey();
	230	const KeyT EmptyKey = getEmptyKey();
242	231	for (BucketT P = getBuckets(), E = getBucketsEnd(); P != E; ++P) {
243		if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey) &&
244		!KeyInfoT::isEqual(P->getFirst(), TombstoneKey))
	232	if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey))
245	233	P->getSecond().~ValueT();
246	234	P->getFirst().~KeyT();
247	235	}
...	...	@@ -249,7 +237,6 @@ class DenseMapBase {
249	237
250	238	void initEmpty() {
251	239	setNumEntries(0);
252		setNumTombstones(0);
253	240
254	241	CHECK_EQ((getNumBuckets() & (getNumBuckets() - 1)), 0);
255	242	const KeyT EmptyKey = getEmptyKey();
...	...	@@ -273,10 +260,8 @@ class DenseMapBase {
273	260
274	261	// Insert all the old elements.
275	262	const KeyT EmptyKey = getEmptyKey();
276		const KeyT TombstoneKey = getTombstoneKey();
277	263	for (BucketT B = OldBucketsBegin, E = OldBucketsEnd; B != E; ++B) {
278		if (!KeyInfoT::isEqual(B->getFirst(), EmptyKey) &&
279		!KeyInfoT::isEqual(B->getFirst(), TombstoneKey)) {
	264	if (!KeyInfoT::isEqual(B->getFirst(), EmptyKey)) {
280	265	// Insert the key/value into the new table.
281	266	BucketT *DestBucket;
282	267	bool FoundVal = LookupBucketFor(B->getFirst(), DestBucket);
...	...	@@ -301,7 +286,6 @@ class DenseMapBase {
301	286	CHECK_EQ(getNumBuckets(), other.getNumBuckets());
302	287
303	288	setNumEntries(other.getNumEntries());
304		setNumTombstones(other.getNumTombstones());
305	289
306	290	if (__sanitizer::is_trivially_copyable<KeyT>::value &&
307	291	__sanitizer::is_trivially_copyable<ValueT>::value)
...	...	@@ -311,8 +295,7 @@ class DenseMapBase {
311	295	for (uptr i = 0; i < getNumBuckets(); ++i) {
312	296	::new (&getBuckets()[i].getFirst())
313	297	KeyT(other.getBuckets()[i].getFirst());
314		if (!KeyInfoT::isEqual(getBuckets()[i].getFirst(), getEmptyKey()) &&
315		!KeyInfoT::isEqual(getBuckets()[i].getFirst(), getTombstoneKey()))
	298	if (!KeyInfoT::isEqual(getBuckets()[i].getFirst(), getEmptyKey()))
316	299	::new (&getBuckets()[i].getSecond())
317	300	ValueT(other.getBuckets()[i].getSecond());
318	301	}
...	...	@@ -329,9 +312,40 @@ class DenseMapBase {
329	312
330	313	static const KeyT getEmptyKey() { return KeyInfoT::getEmptyKey(); }
331	314
332		static const KeyT getTombstoneKey() { return KeyInfoT::getTombstoneKey(); }
333
334	315	private:
	316	/// Erase the entry at \p TheBucket and close the resulting hole via Knuth
	317	/// TAOCP 6.4 Algorithm R: walk forward over the cluster, shifting back any
	318	/// entry whose linear-probe chain from its home bucket passes through the
	319	/// hole, until an empty bucket terminates the cluster.
	320	void eraseFromFilledBucket(BucketT* TheBucket) {
	321	TheBucket->getSecond().~ValueT();
	322	decrementNumEntries();
	323
	324	BucketT* BucketsPtr = getBuckets();
	325	const unsigned NumBuckets = getNumBuckets();
	326	const unsigned Mask = NumBuckets - 1;
	327	const KeyT EmptyKey = getEmptyKey();
	328	unsigned I = static_cast<unsigned>(TheBucket - BucketsPtr);
	329	unsigned J = I;
	330	while (true) {
	331	J = (J + 1) & Mask;
	332	BucketT& BJ = BucketsPtr[J];
	333	if (KeyInfoT::isEqual(BJ.getFirst(), EmptyKey))
	334	break;
	335	unsigned Ideal = getHashValue(BJ.getFirst()) & Mask;
	336	// If the hole (I) lies on the linear-probe chain from the home bucket
	337	// (Ideal) to J, shift J into the hole and make J the new hole.
	338	if (((I - Ideal) & Mask) < ((J - Ideal) & Mask)) {
	339	BucketT& BI = BucketsPtr[I];
	340	BI.getFirst() = __sanitizer::move(BJ.getFirst());
	341	::new (&BI.getSecond()) ValueT(__sanitizer::move(BJ.getSecond()));
	342	BJ.getSecond().~ValueT();
	343	I = J;
	344	}
	345	}
	346	BucketsPtr[I].getFirst() = EmptyKey;
	347	}
	348
335	349	unsigned getNumEntries() const {
336	350	return static_cast<const DerivedT *>(this)->getNumEntries();
337	351	}
...	...	@@ -344,18 +358,6 @@ class DenseMapBase {
344	358
345	359	void decrementNumEntries() { setNumEntries(getNumEntries() - 1); }
346	360
347		unsigned getNumTombstones() const {
348		return static_cast<const DerivedT *>(this)->getNumTombstones();
349		}
350
351		void setNumTombstones(unsigned Num) {
352		static_cast<DerivedT *>(this)->setNumTombstones(Num);
353		}
354
355		void incrementNumTombstones() { setNumTombstones(getNumTombstones() + 1); }
356
357		void decrementNumTombstones() { setNumTombstones(getNumTombstones() - 1); }
358
359	361	const BucketT *getBuckets() const {
360	362	return static_cast<const DerivedT *>(this)->getBuckets();
361	363	}
...	...	@@ -398,25 +400,16 @@ class DenseMapBase {
398	400	template <typename LookupKeyT>
399	401	BucketT *InsertIntoBucketImpl(const KeyT &Key, const LookupKeyT &Lookup,
400	402	BucketT *TheBucket) {
401		// If the load of the hash table is more than 3/4, or if fewer than 1/8 of
402		// the buckets are empty (meaning that many are filled with tombstones),
403		// grow the table.
404		//
405		// The later case is tricky. For example, if we had one empty bucket with
406		// tons of tombstones, failing lookups (e.g. for insertion) would have to
407		// probe almost the entire table until it found the empty bucket. If the
408		// table completely filled with tombstones, no lookup would ever succeed,
409		// causing infinite loops in lookup.
	403	// Grow the table if the load factor would exceed 3/4 after insertion.
	404	// Linear probing with gap-closing deletion (Knuth Algorithm R) keeps every
	405	// chain compact and bounded by the table's empty-bucket count, so no
	406	// tombstone-driven resize is needed.
410	407	unsigned NewNumEntries = getNumEntries() + 1;
411	408	unsigned NumBuckets = getNumBuckets();
412	409	if (UNLIKELY(NewNumEntries * 4 >= NumBuckets * 3)) {
413	410	this->grow(NumBuckets * 2);
414	411	LookupBucketFor(Lookup, TheBucket);
415	412	NumBuckets = getNumBuckets();
416		} else if (UNLIKELY(NumBuckets - (NewNumEntries + getNumTombstones()) <=
417		NumBuckets / 8)) {
418		this->grow(NumBuckets);
419		LookupBucketFor(Lookup, TheBucket);
420	413	}
421	414	CHECK(TheBucket);
422	415
...	...	@@ -424,11 +417,6 @@ class DenseMapBase {
424	417	// so that when growing buckets we have self-consistent entry count.
425	418	incrementNumEntries();
426	419
427		// If we are writing over a tombstone, remember this.
428		const KeyT EmptyKey = getEmptyKey();
429		if (!KeyInfoT::isEqual(TheBucket->getFirst(), EmptyKey))
430		decrementNumTombstones();
431
432	420	return TheBucket;
433	421	}
434	422
...	...	@@ -441,7 +429,6 @@ class DenseMapBase {
441	429
442	430	const KeyT EmptyKey = getEmptyKey();
443	431	unsigned BucketNo = getHashValue(Val) & (NumBuckets - 1);
444		unsigned ProbeAmt = 1;
445	432	while (true) {
446	433	BucketT *Bucket = BucketsPtr + BucketNo;
447	434	if (LIKELY(KeyInfoT::isEqual(Val, Bucket->getFirst())))
...	...	@@ -449,10 +436,8 @@ class DenseMapBase {
449	436	if (LIKELY(KeyInfoT::isEqual(Bucket->getFirst(), EmptyKey)))
450	437	return nullptr;
451	438
452		// Otherwise, it's a hash collision or a tombstone, continue quadratic
453		// probing.
454		BucketNo += ProbeAmt++;
455		BucketNo &= NumBuckets - 1;
	439	// Hash collision: continue linear probing.
	440	BucketNo = (BucketNo + 1) & (NumBuckets - 1);
456	441	}
457	442	}
458	443
...	...	@@ -463,8 +448,8 @@ class DenseMapBase {
463	448
464	449	/// LookupBucketFor - Lookup the appropriate bucket for Val, returning it in
465	450	/// FoundBucket. If the bucket contains the key and a value, this returns
466		/// true, otherwise it returns a bucket with an empty marker or tombstone and
467		/// returns false.
	451	/// true, otherwise it returns a bucket with an empty marker and returns
	452	/// false.
468	453	template <typename LookupKeyT>
469	454	bool LookupBucketFor(const LookupKeyT &Val,
470	455	const BucketT *&FoundBucket) const {
...	...	@@ -476,15 +461,10 @@ class DenseMapBase {
476	461	return false;
477	462	}
478	463
479		// FoundTombstone - Keep track of whether we find a tombstone while probing.
480		const BucketT *FoundTombstone = nullptr;
481	464	const KeyT EmptyKey = getEmptyKey();
482		const KeyT TombstoneKey = getTombstoneKey();
483	465	CHECK(!KeyInfoT::isEqual(Val, EmptyKey));
484		CHECK(!KeyInfoT::isEqual(Val, TombstoneKey));
485	466
486	467	unsigned BucketNo = getHashValue(Val) & (NumBuckets - 1);
487		unsigned ProbeAmt = 1;
488	468	while (true) {
489	469	const BucketT *ThisBucket = BucketsPtr + BucketNo;
490	470	// Found Val's bucket? If so, return it.
...	...	@@ -494,24 +474,14 @@ class DenseMapBase {
494	474	}
495	475
496	476	// If we found an empty bucket, the key doesn't exist in the set.
497		// Insert it and return the default value.
	477	// Return it as the insertion point.
498	478	if (LIKELY(KeyInfoT::isEqual(ThisBucket->getFirst(), EmptyKey))) {
499		// If we've already seen a tombstone while probing, fill it in instead
500		// of the empty bucket we eventually probed to.
501		FoundBucket = FoundTombstone ? FoundTombstone : ThisBucket;
	479	FoundBucket = ThisBucket;
502	480	return false;
503	481	}
504	482
505		// If this is a tombstone, remember it. If Val ends up not in the map, we
506		// prefer to return it than something that would require more probing.
507		if (KeyInfoT::isEqual(ThisBucket->getFirst(), TombstoneKey) &&
508		!FoundTombstone)
509		FoundTombstone = ThisBucket; // Remember the first tombstone found.
510
511		// Otherwise, it's a hash collision or a tombstone, continue quadratic
512		// probing.
513		BucketNo += ProbeAmt++;
514		BucketNo &= (NumBuckets - 1);
	483	// Hash collision: continue linear probing.
	484	BucketNo = (BucketNo + 1) & (NumBuckets - 1);
515	485	}
516	486	}
517	487
...	...	@@ -587,7 +557,6 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
587	557
588	558	BucketT *Buckets = nullptr;
589	559	unsigned NumEntries = 0;
590		unsigned NumTombstones = 0;
591	560	unsigned NumBuckets = 0;
592	561
593	562	public:
...	...	@@ -614,7 +583,6 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
614	583	void swap(DenseMap &RHS) {
615	584	Swap(Buckets, RHS.Buckets);
616	585	Swap(NumEntries, RHS.NumEntries);
617		Swap(NumTombstones, RHS.NumTombstones);
618	586	Swap(NumBuckets, RHS.NumBuckets);
619	587	}
620	588
...	...	@@ -639,7 +607,6 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
639	607	this->BaseT::copyFrom(other);
640	608	} else {
641	609	NumEntries = 0;
642		NumTombstones = 0;
643	610	}
644	611	}
645	612
...	...	@@ -649,7 +616,6 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
649	616	this->BaseT::initEmpty();
650	617	} else {
651	618	NumEntries = 0;
652		NumTombstones = 0;
653	619	}
654	620	}
655	621
...	...	@@ -675,10 +641,6 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
675	641
676	642	void setNumEntries(unsigned Num) { NumEntries = Num; }
677	643
678		unsigned getNumTombstones() const { return NumTombstones; }
679
680		void setNumTombstones(unsigned Num) { NumTombstones = Num; }
681
682	644	BucketT *getBuckets() const { return Buckets; }
683	645
684	646	unsigned getNumBuckets() const { return NumBuckets; }

lib/libtsan/sanitizer_common/sanitizer_dense_map_info.h-25

...	...	@@ -62,7 +62,6 @@ struct DenseMapPair {
62	62	template <typename T>
63	63	struct DenseMapInfo {
64	64	// static T getEmptyKey();
65		// static T getTombstoneKey();
66	65	// static unsigned getHashValue(const T &Val);
67	66	// static bool isEqual(const T &LHS, const T &RHS);
68	67	};
...	...	@@ -86,12 +85,6 @@ struct DenseMapInfo<T *> {
86	85	return reinterpret_cast<T *>(Val);
87	86	}
88	87
89		static constexpr T *getTombstoneKey() {
90		uptr Val = static_cast<uptr>(-2);
91		Val <<= Log2MaxAlign;
92		return reinterpret_cast<T *>(Val);
93		}
94
95	88	static constexpr unsigned getHashValue(const T *PtrVal) {
96	89	return (unsigned((uptr)PtrVal) >> 4) ^ (unsigned((uptr)PtrVal) >> 9);
97	90	}
...	...	@@ -105,7 +98,6 @@ struct DenseMapInfo<T *> {
105	98	template <>
106	99	struct DenseMapInfo<char> {
107	100	static constexpr char getEmptyKey() { return ~0; }
108		static constexpr char getTombstoneKey() { return ~0 - 1; }
109	101	static constexpr unsigned getHashValue(const char &Val) { return Val * 37U; }
110	102
111	103	static constexpr bool isEqual(const char &LHS, const char &RHS) {
...	...	@@ -117,7 +109,6 @@ struct DenseMapInfo<char> {
117	109	template <>
118	110	struct DenseMapInfo<unsigned char> {
119	111	static constexpr unsigned char getEmptyKey() { return ~0; }
120		static constexpr unsigned char getTombstoneKey() { return ~0 - 1; }
121	112	static constexpr unsigned getHashValue(const unsigned char &Val) {
122	113	return Val * 37U;
123	114	}
...	...	@@ -132,7 +123,6 @@ struct DenseMapInfo<unsigned char> {
132	123	template <>
133	124	struct DenseMapInfo<unsigned short> {
134	125	static constexpr unsigned short getEmptyKey() { return 0xFFFF; }
135		static constexpr unsigned short getTombstoneKey() { return 0xFFFF - 1; }
136	126	static constexpr unsigned getHashValue(const unsigned short &Val) {
137	127	return Val * 37U;
138	128	}
...	...	@@ -147,7 +137,6 @@ struct DenseMapInfo<unsigned short> {
147	137	template <>
148	138	struct DenseMapInfo<unsigned> {
149	139	static constexpr unsigned getEmptyKey() { return ~0U; }
150		static constexpr unsigned getTombstoneKey() { return ~0U - 1; }
151	140	static constexpr unsigned getHashValue(const unsigned &Val) {
152	141	return Val * 37U;
153	142	}
...	...	@@ -161,7 +150,6 @@ struct DenseMapInfo<unsigned> {
161	150	template <>
162	151	struct DenseMapInfo<unsigned long> {
163	152	static constexpr unsigned long getEmptyKey() { return ~0UL; }
164		static constexpr unsigned long getTombstoneKey() { return ~0UL - 1L; }
165	153
166	154	static constexpr unsigned getHashValue(const unsigned long &Val) {
167	155	return (unsigned)(Val * 37UL);
...	...	@@ -177,7 +165,6 @@ struct DenseMapInfo<unsigned long> {
177	165	template <>
178	166	struct DenseMapInfo<unsigned long long> {
179	167	static constexpr unsigned long long getEmptyKey() { return ~0ULL; }
180		static constexpr unsigned long long getTombstoneKey() { return ~0ULL - 1ULL; }
181	168
182	169	static constexpr unsigned getHashValue(const unsigned long long &Val) {
183	170	return (unsigned)(Val * 37ULL);
...	...	@@ -193,7 +180,6 @@ struct DenseMapInfo<unsigned long long> {
193	180	template <>
194	181	struct DenseMapInfo<short> {
195	182	static constexpr short getEmptyKey() { return 0x7FFF; }
196		static constexpr short getTombstoneKey() { return -0x7FFF - 1; }
197	183	static constexpr unsigned getHashValue(const short &Val) { return Val * 37U; }
198	184	static constexpr bool isEqual(const short &LHS, const short &RHS) {
199	185	return LHS == RHS;
...	...	@@ -204,7 +190,6 @@ struct DenseMapInfo<short> {
204	190	template <>
205	191	struct DenseMapInfo<int> {
206	192	static constexpr int getEmptyKey() { return 0x7fffffff; }
207		static constexpr int getTombstoneKey() { return -0x7fffffff - 1; }
208	193	static constexpr unsigned getHashValue(const int &Val) {
209	194	return (unsigned)(Val * 37U);
210	195	}
...	...	@@ -221,8 +206,6 @@ struct DenseMapInfo<long> {
221	206	return (1UL << (sizeof(long) * 8 - 1)) - 1UL;
222	207	}
223	208
224		static constexpr long getTombstoneKey() { return getEmptyKey() - 1L; }
225
226	209	static constexpr unsigned getHashValue(const long &Val) {
227	210	return (unsigned)(Val * 37UL);
228	211	}
...	...	@@ -236,9 +219,6 @@ struct DenseMapInfo<long> {
236	219	template <>
237	220	struct DenseMapInfo<long long> {
238	221	static constexpr long long getEmptyKey() { return 0x7fffffffffffffffLL; }
239		static constexpr long long getTombstoneKey() {
240		return -0x7fffffffffffffffLL - 1;
241		}
242	222
243	223	static constexpr unsigned getHashValue(const long long &Val) {
244	224	return (unsigned)(Val * 37ULL);
...	...	@@ -261,11 +241,6 @@ struct DenseMapInfo<detail::DenseMapPair<T, U>> {
261	241	SecondInfo::getEmptyKey());
262	242	}
263	243
264		static constexpr Pair getTombstoneKey() {
265		return detail::DenseMapPair<T, U>(FirstInfo::getTombstoneKey(),
266		SecondInfo::getTombstoneKey());
267		}
268
269	244	static constexpr unsigned getHashValue(const Pair &PairVal) {
270	245	return detail::combineHashValue(FirstInfo::getHashValue(PairVal.first),
271	246	SecondInfo::getHashValue(PairVal.second));

lib/libtsan/sanitizer_common/sanitizer_errno.h+2

...	...	@@ -31,6 +31,8 @@
31	31	# define __errno_location _errno
32	32	#elif SANITIZER_HAIKU
33	33	# define __errno_location _errnop
	34	#elif SANITIZER_AIX
	35	# define __errno_location _Errno
34	36	#endif
35	37
36	38	extern "C" int *__errno_location();

lib/libtsan/sanitizer_common/sanitizer_flag_parser.h+2-2

...	...	@@ -189,8 +189,8 @@ class FlagParser {
189	189	};
190	190
191	191	template <typename T>
192		static void RegisterFlag(FlagParser parser, const char name, const char *desc,
193		T *var) {
	192	void RegisterFlag(FlagParser* parser, const char* name, const char* desc,
	193	T* var) {
194	194	FlagHandler<T> *fh = new (GetGlobalLowLevelAllocator()) FlagHandler<T>(var);
195	195	parser->RegisterHandler(name, fh, desc);
196	196	}

lib/libtsan/sanitizer_common/sanitizer_fuchsia.cpp+2-2

...	...	@@ -93,8 +93,8 @@ void CheckMPROTECT() {}
93	93	void PlatformPrepareForSandboxing(void *args) {}
94	94	void DisableCoreDumperIfNecessary() {}
95	95	void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
96		void SetAlternateSignalStack() {}
97		void UnsetAlternateSignalStack() {}
	96	void* SetAlternateSignalStack() { return nullptr; }
	97	void UnsetAlternateSignalStack(void* altstack_base) {}
98	98
99	99	bool SignalContext::IsStackOverflow() const { return false; }
100	100	void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); }

lib/libtsan/sanitizer_common/sanitizer_haiku.cpp+4

...	...	@@ -128,6 +128,10 @@ uptr internal_close(fd_t fd) {
128	128	RETURN_AND_SET_ERRNO(_kern_close(fd));
129	129	}
130	130
	131	uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags) {
	132	return -1; // Not supported.
	133	}
	134
131	135	uptr internal_open(const char *filename, int flags) {
132	136	CHECK(&_kern_open);
133	137	RETURN_AND_SET_ERRNO(_kern_open(-1, filename, flags, 0));

lib/libtsan/sanitizer_common/sanitizer_interface_internal.h-5

...	...	@@ -76,11 +76,6 @@ void __sanitizer_annotate_double_ended_contiguous_container(
76	76	const void old_container_beg, const void old_container_end,
77	77	const void new_container_beg, const void new_container_end);
78	78	SANITIZER_INTERFACE_ATTRIBUTE
79		void __sanitizer_copy_contiguous_container_annotations(const void *src_begin,
80		const void *src_end,
81		const void *dst_begin,
82		const void *dst_end);
83		SANITIZER_INTERFACE_ATTRIBUTE
84	79	int __sanitizer_verify_contiguous_container(const void beg, const void mid,
85	80	const void *end);
86	81	SANITIZER_INTERFACE_ATTRIBUTE

lib/libtsan/sanitizer_common/sanitizer_internal_defs.h+16-12

...	...	@@ -29,20 +29,24 @@
29	29
30	30	// Only use SANITIZER_ATTRIBUTE before the function return type!
31	31	#if SANITIZER_WINDOWS
32		#if SANITIZER_IMPORT_INTERFACE
33		# define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllimport)
34		#else
35		# define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllexport)
36		#endif
37		# define SANITIZER_WEAK_ATTRIBUTE
38		# define SANITIZER_WEAK_IMPORT
39		#elif SANITIZER_GO
40		# define SANITIZER_INTERFACE_ATTRIBUTE
41		# define SANITIZER_WEAK_ATTRIBUTE
	32	# if SANITIZER_IMPORT_INTERFACE
	33	# define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllimport)
	34	# else
	35	# define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllexport)
	36	# endif
	37	# define SANITIZER_WEAK_ATTRIBUTE
42	38	# define SANITIZER_WEAK_IMPORT
43	39	#else
44		# define SANITIZER_INTERFACE_ATTRIBUTE __attribute__((visibility("default")))
45		# define SANITIZER_WEAK_ATTRIBUTE __attribute__((weak))
	40	# if SANITIZER_GO
	41	# define SANITIZER_INTERFACE_ATTRIBUTE
	42	# define SANITIZER_WEAK_ATTRIBUTE
	43	# elif SANITIZER_AMDGPU \|\| SANITIZER_NVPTX
	44	# define SANITIZER_INTERFACE_ATTRIBUTE __attribute__((visibility("hidden")))
	45	# define SANITIZER_WEAK_ATTRIBUTE __attribute__((weak))
	46	# else
	47	# define SANITIZER_INTERFACE_ATTRIBUTE __attribute__((visibility("default")))
	48	# define SANITIZER_WEAK_ATTRIBUTE __attribute__((weak))
	49	# endif // SANITIZER_GO
46	50	# if SANITIZER_APPLE
47	51	# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak_import))
48	52	# else

lib/libtsan/sanitizer_common/sanitizer_linux.cpp+81-12

...	...	@@ -90,10 +90,18 @@
90	90	extern "C" SANITIZER_WEAK_ATTRIBUTE const char *strerrorname_np(int);
91	91	# endif
92	92
93		# if SANITIZER_LINUX && defined(__loongarch__)
	93	# if SANITIZER_LINUX && \
	94	(defined(__loongarch__) \|\| defined(__hexagon__) \|\| defined(__alpha__))
94	95	# include <sys/sysmacros.h>
95	96	# endif
96	97
	98	// Hexagon uses statx() instead of stat64(). glibc provides struct statx
	99	// through <sys/stat.h>, but musl does not — pull it from <linux/stat.h>.
	100	// On this musl/hexagon combination the two headers coexist without conflict.
	101	# if SANITIZER_LINUX && defined(__hexagon__)
	102	# include <linux/stat.h>
	103	# endif
	104
97	105	# if SANITIZER_LINUX && defined(__powerpc64__)
98	106	# include <asm/ptrace.h>
99	107	# endif
...	...	@@ -254,6 +262,8 @@ ScopedBlockSignals::~ScopedBlockSignals() { SetSigProcMask(&saved_, nullptr); }
254	262	# include "sanitizer_syscall_linux_hexagon.inc"
255	263	# elif SANITIZER_LINUX && SANITIZER_LOONGARCH64
256	264	# include "sanitizer_syscall_linux_loongarch64.inc"
	265	# elif SANITIZER_LINUX && SANITIZER_ALPHA
	266	# include "sanitizer_syscall_linux_alpha.inc"
257	267	# else
258	268	# include "sanitizer_syscall_generic.inc"
259	269	# endif
...	...	@@ -296,11 +306,13 @@ int internal_madvise(uptr addr, uptr length, int advice) {
296	306	return internal_syscall(SYSCALL(madvise), addr, length, advice);
297	307	}
298	308
299		# if SANITIZER_FREEBSD
300	309	uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags) {
	310	# if SANITIZER_FREEBSD \|\| (SANITIZER_LINUX && defined(__NR_close_range))
301	311	return internal_syscall(SYSCALL(close_range), lowfd, highfd, flags);
302		}
303	312	# endif
	313	return -1; // Not supported.
	314	}
	315
304	316	uptr internal_close(fd_t fd) { return internal_syscall(SYSCALL(close), fd); }
305	317
306	318	uptr internal_open(const char *filename, int flags) {
...	...	@@ -341,7 +353,8 @@ uptr internal_ftruncate(fd_t fd, uptr size) {
341	353	return res;
342	354	}
343	355
344		# if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && SANITIZER_LINUX
	356	# if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && SANITIZER_LINUX && \
	357	!defined(__hexagon__)
345	358	static void stat64_to_stat(struct stat64 in, struct stat out) {
346	359	internal_memset(out, 0, sizeof(*out));
347	360	out->st_dev = in->st_dev;
...	...	@@ -360,7 +373,8 @@ static void stat64_to_stat(struct stat64 in, struct stat out) {
360	373	}
361	374	# endif
362	375
363		# if SANITIZER_LINUX && defined(__loongarch__)
	376	# if SANITIZER_LINUX && \
	377	(defined(__loongarch__) \|\| defined(__hexagon__) \|\| defined(__alpha__))
364	378	static void statx_to_stat(struct statx in, struct stat out) {
365	379	internal_memset(out, 0, sizeof(*out));
366	380	out->st_dev = makedev(in->stx_dev_major, in->stx_dev_minor);
...	...	@@ -440,7 +454,7 @@ uptr internal_stat(const char path, void buf) {
440	454	# if SANITIZER_FREEBSD
441	455	return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0);
442	456	# elif SANITIZER_LINUX
443		# if defined(__loongarch__)
	457	# if defined(__loongarch__) \|\| defined(__hexagon__) \|\| defined(__alpha__)
444	458	struct statx bufx;
445	459	int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path,
446	460	AT_NO_AUTOMOUNT, STATX_BASIC_STATS, (uptr)&bufx);
...	...	@@ -478,7 +492,7 @@ uptr internal_lstat(const char path, void buf) {
478	492	return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf,
479	493	AT_SYMLINK_NOFOLLOW);
480	494	# elif SANITIZER_LINUX
481		# if defined(__loongarch__)
	495	# if defined(__loongarch__) \|\| defined(__hexagon__) \|\| defined(__alpha__)
482	496	struct statx bufx;
483	497	int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path,
484	498	AT_SYMLINK_NOFOLLOW \| AT_NO_AUTOMOUNT,
...	...	@@ -526,7 +540,7 @@ uptr internal_fstat(fd_t fd, void *buf) {
526	540	int res = internal_syscall(SYSCALL(fstat64), fd, &kbuf);
527	541	kernel_stat_to_stat(&kbuf, (struct stat *)buf);
528	542	return res;
529		# elif SANITIZER_LINUX && defined(__loongarch__)
	543	# elif SANITIZER_LINUX && (defined(__loongarch__) \|\| defined(__alpha__))
530	544	struct statx bufx;
531	545	int res = internal_syscall(SYSCALL(statx), fd, "", AT_EMPTY_PATH,
532	546	STATX_BASIC_STATS, (uptr)&bufx);
...	...	@@ -535,6 +549,13 @@ uptr internal_fstat(fd_t fd, void *buf) {
535	549	# else
536	550	return internal_syscall(SYSCALL(fstat), fd, (uptr)buf);
537	551	# endif
	552	# elif SANITIZER_LINUX && defined(__hexagon__)
	553	// Hexagon musl lacks struct stat64; use statx() instead.
	554	struct statx bufx;
	555	int res = internal_syscall(SYSCALL(statx), fd, "", AT_EMPTY_PATH,
	556	STATX_BASIC_STATS, (uptr)&bufx);
	557	statx_to_stat(&bufx, (struct stat*)buf);
	558	return res;
538	559	# else
539	560	struct stat64 buf64;
540	561	int res = internal_syscall(SYSCALL(fstat64), fd, &buf64);
...	...	@@ -1003,7 +1024,7 @@ int internal_sigaction_norestorer(int signum, const void act, void oldact) {
1003	1024	// rt_sigaction, so we need to do the same (we'll need to reimplement the
1004	1025	// restorers; for x86_64 the restorer address can be obtained from
1005	1026	// oldact->sa_restorer upon a call to sigaction(xxx, NULL, oldact).
1006		# if !SANITIZER_ANDROID \|\| !SANITIZER_MIPS32
	1027	# if (!SANITIZER_ANDROID \|\| !SANITIZER_MIPS32) && !defined(__alpha__)
1007	1028	k_act.sa_restorer = u_act->sa_restorer;
1008	1029	# endif
1009	1030	}
...	...	@@ -1019,7 +1040,7 @@ int internal_sigaction_norestorer(int signum, const void act, void oldact) {
1019	1040	internal_memcpy(&u_oldact->sa_mask, &k_oldact.sa_mask,
1020	1041	sizeof(__sanitizer_kernel_sigset_t));
1021	1042	u_oldact->sa_flags = k_oldact.sa_flags;
1022		# if !SANITIZER_ANDROID \|\| !SANITIZER_MIPS32
	1043	# if (!SANITIZER_ANDROID \|\| !SANITIZER_MIPS32) && !defined(__alpha__)
1023	1044	u_oldact->sa_restorer = k_oldact.sa_restorer;
1024	1045	# endif
1025	1046	}
...	...	@@ -1228,6 +1249,16 @@ uptr GetMaxVirtualAddress() {
1228	1249	// loongarch64 also has multiple address space layouts: default is 47-bit.
1229	1250	// RISC-V 64 also has multiple address space layouts: 39, 48 and 57-bit.
1230	1251	return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;
	1252	# elif SANITIZER_ALPHA
	1253	// Linux/Alpha uses a 42-bit user VAS (TASK_SIZE = 0x40000000000). With
	1254	// fixed shadow offset 0x10000000000 (1 TiB) the layout is:
	1255	// LowMem: [0x000000000000, 0x00ffffffffff] (1 TiB)
	1256	// LowShadow: [0x010000000000, 0x011fffffffff] (128 GiB)
	1257	// ShadowGap: [0x012000000000, 0x012fffffffff]
	1258	// HighShadow:[0x013000000000, 0x017fffffffff] (256 GiB)
	1259	// HighMem: [0x018000000000, 0x03ffffffffff] (2.5 TiB, stack near top)
	1260	// Capping at TASK_SIZE - 1 avoids treating kernel addresses as HighMem.
	1261	return (1ULL << 42) - 1; // TASK_SIZE - 1
1231	1262	# elif SANITIZER_MIPS64
1232	1263	return (1ULL << 40) - 1; // 0x000000ffffffffffUL;
1233	1264	# elif defined(__s390x__)
...	...	@@ -1894,6 +1925,39 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1894	1925	: "memory");
1895	1926	return res;
1896	1927	}
	1928	# elif defined(__hexagon__)
	1929	uptr internal_clone(int (fn)(void), void* child_stack, int flags, void* arg,
	1930	int* parent_tidptr, void* newtls, int* child_tidptr) {
	1931	if (!fn \|\| !child_stack)
	1932	return -EINVAL;
	1933	child_stack = (char)child_stack - 2 sizeof(unsigned int);
	1934	((unsigned int*)child_stack)[0] = (uptr)fn;
	1935	((unsigned int*)child_stack)[1] = (uptr)arg;
	1936
	1937	// Hexagon clone syscall uses the generic argument order (no
	1938	// CONFIG_CLONE_BACKWARDS): flags, stack, ptid, ctid, tls.
	1939	register int r0 __asm__("r0") = flags;
	1940	register void* r1 __asm__("r1") = child_stack;
	1941	register int* r2 __asm__("r2") = parent_tidptr;
	1942	register int* r3 __asm__("r3") = child_tidptr;
	1943	register void* r4 __asm__("r4") = newtls;
	1944	register int r6 __asm__("r6") = __NR_clone;
	1945
	1946	__asm__ __volatile__(
	1947	"trap0(#1)\n" /* syscall */
	1948	"{ p0 = cmp.eq(r0, #0)\n" /* child? */
	1949	" if (!p0.new) jump:nt 1f }\n"
	1950	"r1 = memw(r29 + #0)\n" /* r1 = fn */
	1951	"r0 = memw(r29 + #4)\n" /* r0 = arg */
	1952	"callr r1\n" /* fn(arg) */
	1953	"r6 = #%7\n" /* __NR_exit */
	1954	"trap0(#1)\n"
	1955	"1:\n"
	1956	: "=r"(r0)
	1957	: "0"(r0), "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r6), "i"(__NR_exit)
	1958	: "memory", "p0", "r1", "lr");
	1959	return (uptr)r0;
	1960	}
1897	1961	# endif
1898	1962	# endif // SANITIZER_LINUX
1899	1963
...	...	@@ -2427,7 +2491,7 @@ static void DumpSingleReg(ucontext_t *ctx, int RegNum) {
2427	2491	# if SANITIZER_LINUX
2428	2492	ctx->uc_mcontext.gregs[RegNum]
2429	2493	# elif SANITIZER_NETBSD
2430		ctx->uc_mcontext.__gregs[RegNum]
	2494	(unsigned long long)ctx->uc_mcontext.__gregs[RegNum]
2431	2495	# endif
2432	2496	);
2433	2497	# elif defined(__i386__)
...	...	@@ -2729,6 +2793,11 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2729	2793	*pc = ucontext->uc_mcontext.__pc;
2730	2794	*bp = ucontext->uc_mcontext.__gregs[22];
2731	2795	*sp = ucontext->uc_mcontext.__gregs[3];
	2796	# elif defined(__alpha__)
	2797	ucontext_t* ucontext = (ucontext_t*)context;
	2798	*pc = ucontext->uc_mcontext.sc_pc;
	2799	*bp = ucontext->uc_mcontext.sc_regs[15]; // $fp / $s6
	2800	*sp = ucontext->uc_mcontext.sc_regs[30]; // $sp
2732	2801	# else
2733	2802	# error "Unsupported arch"
2734	2803	# endif
...	...	@@ -2819,7 +2888,7 @@ void CheckMPROTECT() {
2819	2888	# endif
2820	2889	}
2821	2890
2822		void CheckNoDeepBind(const char *filename, int flag) {
	2891	void OnDlOpen(const char* filename, int flag) {
2823	2892	# ifdef RTLD_DEEPBIND
2824	2893	if (flag & RTLD_DEEPBIND) {
2825	2894	Report(

lib/libtsan/sanitizer_common/sanitizer_linux.h+2-1

...	...	@@ -86,7 +86,8 @@ int internal_sigaction_norestorer(int signum, const void act, void oldact);
86	86	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
87	87	# if defined(__x86_64__) \|\| defined(__mips__) \|\| defined(__aarch64__) \|\| \
88	88	defined(__powerpc64__) \|\| defined(__s390__) \|\| defined(__i386__) \|\| \
89		defined(__arm__) \|\| SANITIZER_RISCV64 \|\| SANITIZER_LOONGARCH64
	89	defined(__arm__) \|\| defined(__hexagon__) \|\| SANITIZER_RISCV64 \|\| \
	90	SANITIZER_LOONGARCH64
90	91	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
91	92	int parent_tidptr, void newtls, int *child_tidptr);
92	93	# endif

lib/libtsan/sanitizer_common/sanitizer_linux_libcdep.cpp+6-2

...	...	@@ -284,6 +284,10 @@ static uptr ThreadDescriptorSizeFallback() {
284	284	# if defined(__powerpc64__)
285	285	return 1776; // from glibc.ppc64le 2.20-8.fc21
286	286	# endif
	287
	288	# if defined(__alpha__)
	289	return 1824; // from glibc 2.43
	290	# endif
287	291	}
288	292	# endif // SANITIZER_GLIBC && !SANITIZER_GO
289	293
...	...	@@ -494,10 +498,10 @@ __attribute__((unused)) static void GetStaticTlsBoundary(uptr addr, uptr size,
494	498	// loader places static TLS blocks this way not to waste space.
495	499	uptr l = one;
496	500	*align = ranges[l].align;
497		while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l].align)
	501	while (l != 0 && ranges[l].begin <= ranges[l - 1].end + ranges[l].align)
498	502	align = Max(align, ranges[--l].align);
499	503	uptr r = one + 1;
500		while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r].align)
	504	while (r != len && ranges[r].begin <= ranges[r - 1].end + ranges[r].align)
501	505	align = Max(align, ranges[r++].align);
502	506	*addr = ranges[l].begin;
503	507	*size = ranges[r - 1].end - ranges[l].begin;

lib/libtsan/sanitizer_common/sanitizer_lzw.h+1-3

...	...	@@ -26,9 +26,7 @@ ItOut LzwEncode(ItIn begin, ItIn end, ItOut out) {
26	26
27	27	// Sentinel value for substrings of len 1.
28	28	static constexpr LzwCodeType kNoPrefix =
29		Min(DenseMapInfo<Substring>::getEmptyKey().first,
30		DenseMapInfo<Substring>::getTombstoneKey().first) -
31		1;
	29	DenseMapInfo<Substring>::getEmptyKey().first - 1;
32	30	DenseMap<Substring, LzwCodeType> prefix_to_code;
33	31	{
34	32	// Add all substring of len 1 as initial dictionary.

lib/libtsan/sanitizer_common/sanitizer_mac.cpp+30-2

...	...	@@ -170,6 +170,10 @@ uptr internal_close(fd_t fd) {
170	170	return close(fd);
171	171	}
172	172
	173	uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags) {
	174	return -1; // Not supported.
	175	}
	176
173	177	uptr internal_open(const char *filename, int flags) {
174	178	return open(filename, flags);
175	179	}
...	...	@@ -609,7 +613,13 @@ static uptr ApproximateOSVersionViaKernelVersion(VersStr vers) {
609	613	u16 os_major = kernel_major - offset;
610	614
611	615	const char *format = "%d.0";
612		if (TARGET_OS_OSX) {
	616	if (kernel_major >= 27) {
	617	// with kernel major 27 <=> OSes 27.0, OS versions are aligned with kernel
	618	os_major = kernel_major;
	619	} else if (kernel_major >= 25) {
	620	// with kernel_major 25 <=> OSes 26.0, OS versions are aligned
	621	os_major = kernel_major + 1;
	622	} else if (TARGET_OS_OSX) {
613	623	if (os_major >= 16) { // macOS 11+
614	624	os_major -= 5;
615	625	} else { // macOS 10.15 and below
...	...	@@ -666,6 +676,24 @@ static void MapToMacos(u16 major, u16 minor) {
666	676	if (TARGET_OS_OSX)
667	677	return;
668	678
	679	// All supported platforms (including DriverKit) have
	680	// aligned version numbers in macOS 27+
	681	if (*major >= 27)
	682	return;
	683
	684	# if TARGET_OS_DRIVERKIT
	685	// Driverkit 25.0+ aligns with macOS 26+
	686	if (*major >= 25) {
	687	*major += 1;
	688	return;
	689	}
	690	# else
	691	// macOS 26 and later have aligned version strings.
	692	if (*major >= 26)
	693	return;
	694	# endif
	695
	696	// Below are mappings for pre-macOS-25-aligned releases
669	697	if (TARGET_OS_IOS \|\| TARGET_OS_TV)
670	698	*major += 2;
671	699	else if (TARGET_OS_WATCH)
...	...	@@ -1529,7 +1557,7 @@ void DumpProcessMap() {
1529	1557	Printf("End of module map.\n");
1530	1558	}
1531	1559
1532		void CheckNoDeepBind(const char *filename, int flag) {
	1560	void OnDlOpen(const char* filename, int flag) {
1533	1561	// Do nothing.
1534	1562	}
1535	1563

lib/libtsan/sanitizer_common/sanitizer_netbsd.cpp+4

...	...	@@ -126,6 +126,10 @@ uptr internal_close(fd_t fd) {
126	126	return _sys_close(fd);
127	127	}
128	128
	129	uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags) {
	130	return -1; // Not supported.
	131	}
	132
129	133	uptr internal_open(const char *filename, int flags) {
130	134	CHECK(&_sys_open);
131	135	return _sys_open(filename, flags);

lib/libtsan/sanitizer_common/sanitizer_platform.h+33-2

...	...	@@ -15,7 +15,8 @@
15	15	#if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__NetBSD__) && \
16	16	!defined(__APPLE__) && !defined(_WIN32) && !defined(__Fuchsia__) && \
17	17	!(defined(__sun__) && defined(__svr4__)) && !defined(__HAIKU__) && \
18		!defined(__wasi__)
	18	!defined(__wasi__) && !defined(__NVPTX__) && !defined(__AMDGPU__) && \
	19	!defined(__SPIRV__) && !defined(_AIX)
19	20	# error "This operating system is not supported"
20	21	#endif
21	22
...	...	@@ -32,6 +33,12 @@
32	33	# define SANITIZER_LINUX 0
33	34	#endif
34	35
	36	#if defined(_AIX)
	37	# define SANITIZER_AIX 1
	38	#else
	39	# define SANITIZER_AIX 0
	40	#endif
	41
35	42	#if defined(__GLIBC__)
36	43	# define SANITIZER_GLIBC 1
37	44	#else
...	...	@@ -151,7 +158,7 @@
151	158
152	159	#define SANITIZER_POSIX \
153	160	(SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_APPLE \|\| \
154		SANITIZER_NETBSD \|\| SANITIZER_SOLARIS \|\| SANITIZER_HAIKU)
	161	SANITIZER_NETBSD \|\| SANITIZER_SOLARIS \|\| SANITIZER_HAIKU \|\| SANITIZER_AIX)
155	162
156	163	#if __LP64__ \|\| defined(_WIN64)
157	164	# define SANITIZER_WORDSIZE 64
...	...	@@ -302,6 +309,30 @@
302	309	# define SANITIZER_LOONGARCH64 0
303	310	#endif
304	311
	312	#if defined(__alpha__)
	313	# define SANITIZER_ALPHA 1
	314	#else
	315	# define SANITIZER_ALPHA 0
	316	#endif
	317
	318	#if defined(__AMDGPU__)
	319	# define SANITIZER_AMDGPU 1
	320	#else
	321	# define SANITIZER_AMDGPU 0
	322	#endif
	323
	324	#if defined(__NVPTX__)
	325	# define SANITIZER_NVPTX 1
	326	#else
	327	# define SANITIZER_NVPTX 0
	328	#endif
	329
	330	#if defined(__SPIRV__)
	331	# define SANITIZER_SPIRV 1
	332	#else
	333	# define SANITIZER_SPIRV 0
	334	#endif
	335
305	336	// By default we allow to use SizeClassAllocator64 on 64-bit platform.
306	337	// But in some cases SizeClassAllocator64 does not work well and we need to
307	338	// fallback to SizeClassAllocator32.

lib/libtsan/sanitizer_common/sanitizer_platform_interceptors.h+6-1

...	...	@@ -201,6 +201,9 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
201	201
202	202	#define SANITIZER_INTERCEPT_READ SI_POSIX
203	203	#define SANITIZER_INTERCEPT_PREAD SI_POSIX
	204	#define SANITIZER_INTERCEPT___READ_CHK SI_GLIBC
	205	#define SANITIZER_INTERCEPT___PREAD_CHK SI_GLIBC
	206	#define SANITIZER_INTERCEPT___PREAD64_CHK SI_GLIBC
204	207	#define SANITIZER_INTERCEPT_WRITE SI_POSIX
205	208	#define SANITIZER_INTERCEPT_PWRITE SI_POSIX
206	209
...	...	@@ -393,6 +396,8 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
393	396	#define SANITIZER_INTERCEPT_SHMCTL \
394	397	(((SI_FREEBSD \|\| SI_LINUX_NOT_ANDROID) && SANITIZER_WORDSIZE == 64) \|\| \
395	398	SI_NETBSD \|\| SI_SOLARIS)
	399	// shmat calls REAL(shmctl), so it requires shmctl interception.
	400	#define SANITIZER_INTERCEPT_SHMAT SANITIZER_INTERCEPT_SHMCTL
396	401	#define SANITIZER_INTERCEPT_RANDOM_R SI_GLIBC
397	402	#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_POSIX
398	403	#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \
...	...	@@ -545,7 +550,7 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
545	550	#define SANITIZER_INTERCEPT___LIBC_MEMALIGN SI_GLIBC
546	551	#define SANITIZER_INTERCEPT_PVALLOC (SI_GLIBC \|\| SI_ANDROID)
547	552	#define SANITIZER_INTERCEPT_CFREE (SI_GLIBC && !SANITIZER_RISCV64)
548		#define SANITIZER_INTERCEPT_REALLOCARRAY SI_POSIX
	553	#define SANITIZER_INTERCEPT_REALLOCARRAY (SI_POSIX \|\| SI_FUCHSIA)
549	554	#define SANITIZER_INTERCEPT_ALIGNED_ALLOC \
550	555	(!SI_MAC \|\| SI_MAC_SDK_10_15_AVAILABLE)
551	556	#define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD)

lib/libtsan/sanitizer_common/sanitizer_platform_limits_linux.cpp+1-1

...	...	@@ -60,7 +60,7 @@ using namespace __sanitizer;
60	60	# if !defined(__powerpc64__) && !defined(__x86_64__) && \
61	61	!defined(__aarch64__) && !defined(__mips__) && !defined(__s390__) && \
62	62	!defined(__sparc__) && !defined(__riscv) && !defined(__hexagon__) && \
63		!defined(__loongarch__)
	63	!defined(__loongarch__) && !defined(__alpha__)
64	64	COMPILER_CHECK(struct___old_kernel_stat_sz == sizeof(struct __old_kernel_stat));
65	65	#endif
66	66

lib/libtsan/sanitizer_common/sanitizer_platform_limits_posix.cpp+57-25

...	...	@@ -24,7 +24,7 @@
24	24	// Must go after undef _FILE_OFFSET_BITS.
25	25	#include "sanitizer_platform.h"
26	26
27		#if SANITIZER_LINUX \|\| SANITIZER_APPLE \|\| SANITIZER_HAIKU
	27	#if SANITIZER_LINUX \|\| SANITIZER_APPLE \|\| SANITIZER_HAIKU \|\| SANITIZER_AIX
28	28	// Must go after undef _FILE_OFFSET_BITS.
29	29	#include "sanitizer_glibc_version.h"
30	30
...	...	@@ -61,11 +61,11 @@
61	61	#endif
62	62
63	63	#if !SANITIZER_ANDROID
64		#if !SANITIZER_HAIKU
65		#include <sys/mount.h>
66		#endif
67		#include <sys/timeb.h>
68		#include <utmpx.h>
	64	# if !SANITIZER_HAIKU && !SANITIZER_AIX
	65	# include <sys/mount.h>
	66	# endif
	67	# include <sys/timeb.h>
	68	# include <utmpx.h>
69	69	#endif
70	70
71	71	#if SANITIZER_LINUX
...	...	@@ -113,11 +113,15 @@ typedef struct user_fpregs elf_fpregset_t;
113	113	#endif
114	114
115	115	#if !SANITIZER_ANDROID
116		#include <ifaddrs.h>
117		#if !SANITIZER_HAIKU
118		#include <sys/ucontext.h>
119		#include <wordexp.h>
120		#endif
	116	# if !SANITIZER_AIX
	117	# include <ifaddrs.h>
	118	# else
	119	# include <netinet/in.h>
	120	# endif
	121	# if !SANITIZER_HAIKU
	122	# include <sys/ucontext.h>
	123	# include <wordexp.h>
	124	# endif
121	125	#endif
122	126
123	127	#if SANITIZER_LINUX
...	...	@@ -182,6 +186,17 @@ typedef struct user_fpregs elf_fpregset_t;
182	186	#include <sys/ioctl.h>
183	187	#endif
184	188
	189	# if SANITIZER_AIX
	190	# include <netinet/ip_mroute.h>
	191	# include <stropts.h>
	192	# include <sys/ioctl.h>
	193	# include <sys/statfs.h>
	194	# include <unistd.h>
	195	# if HAVE_RPC_XDR_H
	196	# include <tirpc/rpc/xdr.h>
	197	# endif
	198	# endif
	199
185	200	// Include these after system headers to avoid name clashes and ambiguities.
186	201	# include "sanitizer_common.h"
187	202	# include "sanitizer_internal_defs.h"
...	...	@@ -293,7 +308,7 @@ namespace __sanitizer {
293	308	#define SIZEOF_STRUCT_USTAT 32
294	309	# elif defined(__arm__) \|\| defined(__i386__) \|\| defined(__mips__) \|\| \
295	310	defined(__powerpc__) \|\| defined(__s390__) \|\| defined(__sparc__) \|\| \
296		defined(__hexagon__)
	311	defined(__hexagon__) \|\| defined(__alpha__)
297	312	# define SIZEOF_STRUCT_USTAT 20
298	313	# elif defined(__loongarch__)
299	314	// Not used. The minimum Glibc version available for LoongArch is 2.36
...	...	@@ -305,9 +320,12 @@ namespace __sanitizer {
305	320	unsigned struct_ustat_sz = SIZEOF_STRUCT_USTAT;
306	321	unsigned struct_rlimit64_sz = sizeof(struct rlimit64);
307	322	unsigned struct_statvfs64_sz = sizeof(struct statvfs64);
308		#endif // SANITIZER_GLIBC
	323	# elif SANITIZER_MUSL
	324	// On musl, rlimit64 is an alias for rlimit.
	325	unsigned struct_rlimit64_sz = sizeof(struct rlimit);
	326	# endif // SANITIZER_GLIBC
309	327
310		#if SANITIZER_LINUX && !SANITIZER_ANDROID
	328	# if SANITIZER_LINUX && !SANITIZER_ANDROID
311	329	unsigned struct_timex_sz = sizeof(struct timex);
312	330	unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds);
313	331	unsigned struct_mq_attr_sz = sizeof(struct mq_attr);
...	...	@@ -556,13 +574,13 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
556	574	const unsigned IOCTL_NOT_PRESENT = 0;
557	575
558	576	unsigned IOCTL_FIONBIO = FIONBIO;
559		#if !SANITIZER_HAIKU
	577	# if !SANITIZER_HAIKU
560	578	unsigned IOCTL_FIOASYNC = FIOASYNC;
561	579	unsigned IOCTL_FIOCLEX = FIOCLEX;
562	580	unsigned IOCTL_FIOGETOWN = FIOGETOWN;
563	581	unsigned IOCTL_FIONCLEX = FIONCLEX;
564	582	unsigned IOCTL_FIOSETOWN = FIOSETOWN;
565		#endif
	583	# endif
566	584	unsigned IOCTL_SIOCADDMULTI = SIOCADDMULTI;
567	585	unsigned IOCTL_SIOCATMARK = SIOCATMARK;
568	586	unsigned IOCTL_SIOCDELMULTI = SIOCDELMULTI;
...	...	@@ -584,14 +602,14 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
584	602	unsigned IOCTL_SIOCSIFNETMASK = SIOCSIFNETMASK;
585	603	unsigned IOCTL_SIOCSPGRP = SIOCSPGRP;
586	604
587		#if !SANITIZER_HAIKU
	605	# if !SANITIZER_HAIKU
588	606	unsigned IOCTL_TIOCCONS = TIOCCONS;
589	607	unsigned IOCTL_TIOCGETD = TIOCGETD;
590	608	unsigned IOCTL_TIOCNOTTY = TIOCNOTTY;
591	609	unsigned IOCTL_TIOCPKT = TIOCPKT;
592	610	unsigned IOCTL_TIOCSETD = TIOCSETD;
593	611	unsigned IOCTL_TIOCSTI = TIOCSTI;
594		#endif
	612	# endif
595	613
596	614	unsigned IOCTL_TIOCEXCL = TIOCEXCL;
597	615	unsigned IOCTL_TIOCGPGRP = TIOCGPGRP;
...	...	@@ -602,10 +620,12 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
602	620	unsigned IOCTL_TIOCMSET = TIOCMSET;
603	621	unsigned IOCTL_TIOCNXCL = TIOCNXCL;
604	622	unsigned IOCTL_TIOCOUTQ = TIOCOUTQ;
	623	# if !SANITIZER_AIX
605	624	unsigned IOCTL_TIOCSCTTY = TIOCSCTTY;
	625	# endif
606	626	unsigned IOCTL_TIOCSPGRP = TIOCSPGRP;
607	627	unsigned IOCTL_TIOCSWINSZ = TIOCSWINSZ;
608		#if SANITIZER_LINUX && !SANITIZER_ANDROID
	628	# if SANITIZER_LINUX && !SANITIZER_ANDROID
609	629	unsigned IOCTL_SIOCGETSGCNT = SIOCGETSGCNT;
610	630	unsigned IOCTL_SIOCGETVIFCNT = SIOCGETVIFCNT;
611	631	#endif
...	...	@@ -1067,6 +1087,9 @@ CHECK_SIZE_AND_OFFSET(addrinfo, ai_protocol);
1067	1087	CHECK_SIZE_AND_OFFSET(addrinfo, ai_addrlen);
1068	1088	CHECK_SIZE_AND_OFFSET(addrinfo, ai_canonname);
1069	1089	CHECK_SIZE_AND_OFFSET(addrinfo, ai_addr);
	1090	# if SANITIZER_AIX
	1091	CHECK_SIZE_AND_OFFSET(addrinfo, ai_eflags);
	1092	# endif
1070	1093
1071	1094	CHECK_TYPE_SIZE(hostent);
1072	1095	CHECK_SIZE_AND_OFFSET(hostent, h_name);
...	...	@@ -1113,11 +1136,13 @@ COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent));
1113	1136	CHECK_SIZE_AND_OFFSET(dirent, d_ino);
1114	1137	#if SANITIZER_APPLE
1115	1138	CHECK_SIZE_AND_OFFSET(dirent, d_seekoff);
1116		#elif SANITIZER_FREEBSD \|\| SANITIZER_HAIKU
	1139	# elif SANITIZER_AIX
	1140	CHECK_SIZE_AND_OFFSET(dirent, d_offset);
	1141	# elif SANITIZER_FREEBSD \|\| SANITIZER_HAIKU
1117	1142	// There is no 'd_off' field on FreeBSD.
1118		#else
	1143	# else
1119	1144	CHECK_SIZE_AND_OFFSET(dirent, d_off);
1120		#endif
	1145	# endif
1121	1146	CHECK_SIZE_AND_OFFSET(dirent, d_reclen);
1122	1147
1123	1148	#if SANITIZER_GLIBC
...	...	@@ -1151,7 +1176,8 @@ CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_mask);
1151	1176	// didn't exist.
1152	1177	CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_flags);
1153	1178	#endif
1154		#if SANITIZER_LINUX && (!SANITIZER_ANDROID \|\| !SANITIZER_MIPS32)
	1179	# if SANITIZER_LINUX && (!SANITIZER_ANDROID \|\| !SANITIZER_MIPS32) && \
	1180	!defined(__alpha__)
1155	1181	CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_restorer);
1156	1182	#endif
1157	1183
...	...	@@ -1192,6 +1218,10 @@ CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordc);
1192	1218	CHECK_SIZE_AND_OFFSET(wordexp_t, we_wordv);
1193	1219	CHECK_SIZE_AND_OFFSET(wordexp_t, we_offs);
1194	1220	#endif
	1221	# if SANITIZER_AIX
	1222	CHECK_SIZE_AND_OFFSET(wordexp_t, we_sflags);
	1223	CHECK_SIZE_AND_OFFSET(wordexp_t, we_soffs);
	1224	# endif
1195	1225
1196	1226	CHECK_TYPE_SIZE(tm);
1197	1227	CHECK_SIZE_AND_OFFSET(tm, tm_sec);
...	...	@@ -1203,10 +1233,12 @@ CHECK_SIZE_AND_OFFSET(tm, tm_year);
1203	1233	CHECK_SIZE_AND_OFFSET(tm, tm_wday);
1204	1234	CHECK_SIZE_AND_OFFSET(tm, tm_yday);
1205	1235	CHECK_SIZE_AND_OFFSET(tm, tm_isdst);
	1236	# if !SANITIZER_AIX
1206	1237	CHECK_SIZE_AND_OFFSET(tm, tm_gmtoff);
1207	1238	CHECK_SIZE_AND_OFFSET(tm, tm_zone);
	1239	# endif
1208	1240
1209		#if SANITIZER_LINUX
	1241	# if SANITIZER_LINUX
1210	1242	CHECK_TYPE_SIZE(mntent);
1211	1243	CHECK_SIZE_AND_OFFSET(mntent, mnt_fsname);
1212	1244	CHECK_SIZE_AND_OFFSET(mntent, mnt_dir);
...	...	@@ -1256,7 +1288,7 @@ CHECK_TYPE_SIZE(clock_t);
1256	1288	CHECK_TYPE_SIZE(clockid_t);
1257	1289	#endif
1258	1290
1259		#if !SANITIZER_ANDROID && !SANITIZER_HAIKU
	1291	# if !SANITIZER_ANDROID && !SANITIZER_HAIKU && !SANITIZER_AIX
1260	1292	CHECK_TYPE_SIZE(ifaddrs);
1261	1293	CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_next);
1262	1294	CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_name);

lib/libtsan/sanitizer_common/sanitizer_platform_limits_posix.h+69-13

...	...	@@ -14,7 +14,7 @@
14	14	#ifndef SANITIZER_PLATFORM_LIMITS_POSIX_H
15	15	#define SANITIZER_PLATFORM_LIMITS_POSIX_H
16	16
17		#if SANITIZER_LINUX \|\| SANITIZER_APPLE \|\| SANITIZER_HAIKU
	17	#if SANITIZER_LINUX \|\| SANITIZER_APPLE \|\| SANITIZER_HAIKU \|\| SANITIZER_AIX
18	18
19	19	# include "sanitizer_internal_defs.h"
20	20	# include "sanitizer_mallinfo.h"
...	...	@@ -29,7 +29,7 @@
29	29	# define SANITIZER_HAS_STAT64 0
30	30	# define SANITIZER_HAS_STATFS64 0
31	31	# endif
32		# elif SANITIZER_GLIBC \|\| SANITIZER_ANDROID
	32	# elif SANITIZER_GLIBC \|\| SANITIZER_ANDROID \|\| SANITIZER_AIX
33	33	# define SANITIZER_HAS_STAT64 1
34	34	# define SANITIZER_HAS_STATFS64 1
35	35	# elif SANITIZER_HAIKU
...	...	@@ -103,9 +103,15 @@ const unsigned struct_kernel_stat64_sz = 104;
103	103	const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID
104	104	? FIRST_32_SECOND_64(104, 128)
105	105	# if defined(_ABIN32) && _MIPS_SIM == _ABIN32
	106	# if defined(_TIME_BITS) && _TIME_BITS == 64
	107	: FIRST_32_SECOND_64(112, 216);
	108	# else
106	109	: FIRST_32_SECOND_64(176, 216);
	110	# endif
107	111	# elif SANITIZER_MUSL
108	112	: FIRST_32_SECOND_64(160, 208);
	113	# elif defined(_TIME_BITS) && _TIME_BITS == 64
	114	: FIRST_32_SECOND_64(112, 216);
109	115	# else
110	116	: FIRST_32_SECOND_64(160, 216);
111	117	# endif
...	...	@@ -133,6 +139,9 @@ const unsigned struct_kernel_stat64_sz = 0;
133	139	# elif defined(__loongarch__)
134	140	const unsigned struct_kernel_stat_sz = 128;
135	141	const unsigned struct_kernel_stat64_sz = 0;
	142	# elif defined(__alpha__)
	143	const unsigned struct_kernel_stat_sz = 80;
	144	const unsigned struct_kernel_stat64_sz = 136;
136	145	# endif
137	146	struct __sanitizer_perf_event_attr {
138	147	unsigned type;
...	...	@@ -323,7 +332,7 @@ struct __sanitizer_iovec {
323	332	usize iov_len;
324	333	};
325	334
326		# if !SANITIZER_ANDROID
	335	# if !SANITIZER_ANDROID && !SANITIZER_AIX
327	336	struct __sanitizer_ifaddrs {
328	337	struct __sanitizer_ifaddrs *ifa_next;
329	338	char *ifa_name;
...	...	@@ -337,7 +346,7 @@ struct __sanitizer_ifaddrs {
337	346	void ifa_dstaddr; // (struct sockaddr )
338	347	void *ifa_data;
339	348	};
340		# endif // !SANITIZER_ANDROID
	349	# endif // !SANITIZER_ANDROID && !SANITIZER_AIX
341	350
342	351	# if SANITIZER_APPLE
343	352	typedef unsigned long __sanitizer_pthread_key_t;
...	...	@@ -345,7 +354,7 @@ typedef unsigned long __sanitizer_pthread_key_t;
345	354	typedef unsigned __sanitizer_pthread_key_t;
346	355	# endif
347	356
348		# if SANITIZER_LINUX && !SANITIZER_ANDROID
	357	# if (SANITIZER_LINUX && !SANITIZER_ANDROID) \|\| SANITIZER_AIX
349	358
350	359	struct __sanitizer_XDR {
351	360	int x_op;
...	...	@@ -440,12 +449,14 @@ struct __sanitizer_tm {
440	449	int tm_wday;
441	450	int tm_yday;
442	451	int tm_isdst;
443		# if SANITIZER_HAIKU
	452	# if !SANITIZER_AIX
	453	# if SANITIZER_HAIKU
444	454	int tm_gmtoff;
445	455	# else
446	456	long int tm_gmtoff;
447	457	# endif
448	458	const char *tm_zone;
	459	# endif
449	460	};
450	461
451	462	# if SANITIZER_LINUX
...	...	@@ -513,11 +524,19 @@ struct __sanitizer_msghdr {
513	524	struct __sanitizer_iovec *msg_iov;
514	525	uptr msg_iovlen;
515	526	void *msg_control;
	527	# if !SANITIZER_AIX
516	528	uptr msg_controllen;
	529	# else
	530	unsigned msg_controllen;
	531	# endif
517	532	int msg_flags;
518	533	};
519	534	struct __sanitizer_cmsghdr {
	535	# if !SANITIZER_AIX
520	536	uptr cmsg_len;
	537	# else
	538	unsigned cmsg_len;
	539	# endif
521	540	int cmsg_level;
522	541	int cmsg_type;
523	542	};
...	...	@@ -554,10 +573,23 @@ struct __sanitizer_dirent {
554	573	unsigned short d_reclen;
555	574	// more fields that we don't care about
556	575	};
	576	# elif defined(__alpha__)
	577	struct __sanitizer_dirent {
	578	unsigned int d_ino; // ino_t is 32-bit on Alpha
	579	int __pad; // explicit padding before d_off
	580	unsigned long d_off;
	581	unsigned short d_reclen;
	582	// more fields that we don't care about
	583	};
557	584	# else
558	585	struct __sanitizer_dirent {
	586	# if SANITIZER_AIX
	587	uptr d_offset;
	588	uptr d_ino;
	589	# else
559	590	uptr d_ino;
560	591	uptr d_off;
	592	# endif
561	593	unsigned short d_reclen;
562	594	// more fields that we don't care about
563	595	};
...	...	@@ -573,7 +605,7 @@ struct __sanitizer_dirent64 {
573	605	extern unsigned struct_sock_fprog_sz;
574	606	# endif
575	607
576		# if SANITIZER_HAIKU
	608	# if SANITIZER_HAIKU \|\| SANITIZER_AIX
577	609	typedef int __sanitizer_clock_t;
578	610	# elif defined(__x86_64__) && !defined(_LP64)
579	611	typedef long long __sanitizer_clock_t;
...	...	@@ -581,8 +613,10 @@ typedef long long __sanitizer_clock_t;
581	613	typedef long __sanitizer_clock_t;
582	614	# endif
583	615
584		# if SANITIZER_LINUX \|\| SANITIZER_HAIKU
	616	# if SANITIZER_LINUX \|\| SANITIZER_HAIKU \|\| SANITIZER_AIX
585	617	typedef int __sanitizer_clockid_t;
	618	# endif
	619	# if SANITIZER_LINUX \|\| SANITIZER_HAIKU
586	620	typedef unsigned long long __sanitizer_eventfd_t;
587	621	# endif
588	622
...	...	@@ -637,6 +671,14 @@ struct __sanitizer_sigset_t {
637	671	// The size is determined by looking at sizeof of real sigset_t on linux.
638	672	uptr val[128 / sizeof(uptr)];
639	673	};
	674	# elif SANITIZER_AIX
	675	struct __sanitizer_sigset_t {
	676	# if SANITIZER_WORDSIZE == 64
	677	uptr val[4];
	678	# else
	679	uptr val[2];
	680	# endif
	681	};
640	682	# endif
641	683
642	684	struct __sanitizer_siginfo_pad {
...	...	@@ -741,7 +783,7 @@ struct __sanitizer_sigaction {
741	783	# endif
742	784	# endif
743	785	# endif
744		# if SANITIZER_LINUX \|\| SANITIZER_HAIKU
	786	# if (SANITIZER_LINUX \|\| SANITIZER_HAIKU) && !defined(__alpha__)
745	787	void (*sa_restorer)();
746	788	# endif
747	789	# if defined(__mips__) && (SANITIZER_WORDSIZE == 32) && !SANITIZER_MUSL
...	...	@@ -828,8 +870,12 @@ struct __sanitizer_addrinfo {
828	870	int ai_family;
829	871	int ai_socktype;
830	872	int ai_protocol;
831		# if SANITIZER_ANDROID \|\| SANITIZER_APPLE \|\| SANITIZER_HAIKU
	873	# if SANITIZER_ANDROID \|\| SANITIZER_APPLE \|\| SANITIZER_HAIKU \|\| SANITIZER_AIX
	874	# if SANITIZER_AIX // AIX ai_addrlen type is size_t
	875	uptr ai_addrlen;
	876	# else
832	877	unsigned ai_addrlen;
	878	# endif
833	879	char *ai_canonname;
834	880	void *ai_addr;
835	881	# else // LINUX
...	...	@@ -838,6 +884,9 @@ struct __sanitizer_addrinfo {
838	884	char *ai_canonname;
839	885	# endif
840	886	struct __sanitizer_addrinfo *ai_next;
	887	# if SANITIZER_AIX
	888	int ai_eflags;
	889	# endif
841	890	};
842	891
843	892	struct __sanitizer_hostent {
...	...	@@ -854,7 +903,7 @@ struct __sanitizer_pollfd {
854	903	short revents;
855	904	};
856	905
857		# if SANITIZER_ANDROID \|\| SANITIZER_APPLE
	906	# if SANITIZER_ANDROID \|\| SANITIZER_APPLE \|\| SANITIZER_AIX
858	907	typedef unsigned __sanitizer_nfds_t;
859	908	# else
860	909	typedef unsigned long __sanitizer_nfds_t;
...	...	@@ -892,6 +941,10 @@ struct __sanitizer_wordexp_t {
892	941	uptr we_wordc;
893	942	char **we_wordv;
894	943	uptr we_offs;
	944	# if SANITIZER_AIX
	945	int we_sflags;
	946	uptr we_soffs;
	947	# endif
895	948	};
896	949
897	950	# if SANITIZER_LINUX && !SANITIZER_ANDROID
...	...	@@ -1023,7 +1076,7 @@ struct __sanitizer_cookie_io_functions_t {
1023	1076	# define IOC_NRBITS 8
1024	1077	# define IOC_TYPEBITS 8
1025	1078	# if defined(__powerpc__) \|\| defined(__powerpc64__) \|\| defined(__mips__) \|\| \
1026		defined(__sparc__)
	1079	defined(__sparc__) \|\| defined(__alpha__)
1027	1080	# define IOC_SIZEBITS 13
1028	1081	# define IOC_DIRBITS 3
1029	1082	# define IOC_NONE 1U
...	...	@@ -1193,7 +1246,9 @@ extern unsigned IOCTL_TIOCMGET;
1193	1246	extern unsigned IOCTL_TIOCMSET;
1194	1247	extern unsigned IOCTL_TIOCNXCL;
1195	1248	extern unsigned IOCTL_TIOCOUTQ;
	1249	# if !SANITIZER_AIX
1196	1250	extern unsigned IOCTL_TIOCSCTTY;
	1251	# endif
1197	1252	extern unsigned IOCTL_TIOCSPGRP;
1198	1253	extern unsigned IOCTL_TIOCSWINSZ;
1199	1254	# if SANITIZER_LINUX && !SANITIZER_ANDROID
...	...	@@ -1593,6 +1648,7 @@ extern const int si_SEGV_ACCERR;
1593	1648	typedef void *__sanitizer_timer_t;
1594	1649	# endif
1595	1650
1596		#endif // SANITIZER_LINUX \|\| SANITIZER_APPLE \|\| SANITIZER_HAIKU
	1651	#endif // SANITIZER_LINUX \|\| SANITIZER_APPLE \|\| SANITIZER_HAIKU \|\|
	1652	// SANITIZER_AIX
1597	1653
1598	1654	#endif

lib/libtsan/sanitizer_common/sanitizer_platform_limits_solaris.cpp+1

...	...	@@ -51,6 +51,7 @@
51	51	#include <sys/timeb.h>
52	52	#include <sys/times.h>
53	53	#include <sys/types.h>
	54	#include <sys/ucontext.h>
54	55	#include <sys/utsname.h>
55	56	#include <termios.h>
56	57	#include <time.h>

lib/libtsan/sanitizer_common/sanitizer_posix.cpp+9-7

...	...	@@ -27,12 +27,13 @@
27	27	#include <signal.h>
28	28	#include <sys/mman.h>
29	29
30		#if SANITIZER_FREEBSD
	30	# if SANITIZER_FREEBSD \|\| SANITIZER_AIX
31	31	// The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before
32	32	// that, it was never implemented. So just define it to zero.
33		#undef MAP_NORESERVE
34		#define MAP_NORESERVE 0
35		#endif
	33	// Similarly, AIX does not define MAP_NORESERVE.
	34	# undef MAP_NORESERVE
	35	# define MAP_NORESERVE 0
	36	# endif
36	37
37	38	namespace __sanitizer {
38	39
...	...	@@ -357,9 +358,10 @@ int GetNamedMappingFd(const char name, uptr size, int flags) {
357	358	if (!common_flags()->decorate_proc_maps \|\| !name)
358	359	return -1;
359	360	char shmname[200];
360		CHECK(internal_strlen(name) < sizeof(shmname) - 10);
361		internal_snprintf(shmname, sizeof(shmname), "/dev/shm/%zu [%s]",
362		internal_getpid(), name);
	361	int len =
	362	internal_snprintf(shmname, sizeof(shmname), "/dev/shm/%zu.%llu [%s]",
	363	internal_getpid(), GetTid(), name);
	364	CHECK_LT(len, sizeof(shmname));
363	365	int o_cloexec = 0;
364	366	#if defined(O_CLOEXEC)
365	367	o_cloexec = O_CLOEXEC;

lib/libtsan/sanitizer_common/sanitizer_posix.h+2-2

...	...	@@ -28,9 +28,9 @@ namespace __sanitizer {
28	28	// Don't use directly, use __sanitizer::OpenFile() instead.
29	29	uptr internal_open(const char *filename, int flags);
30	30	uptr internal_open(const char *filename, int flags, u32 mode);
31		# if SANITIZER_FREEBSD
	31	// Closes all file descriptors from lowfd to highfd (inclusive).
	32	// Returns 0 on success or non-zero if not supported on this platform.
32	33	uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags);
33		# endif
34	34	uptr internal_close(fd_t fd);
35	35
36	36	uptr internal_read(fd_t fd, void *buf, uptr count);

lib/libtsan/sanitizer_common/sanitizer_posix_libcdep.cpp+12-9

...	...	@@ -188,12 +188,13 @@ static uptr GetAltStackSize() {
188	188	return SIGSTKSZ * 4;
189	189	}
190	190
191		void SetAlternateSignalStack() {
	191	void* SetAlternateSignalStack() {
192	192	stack_t altstack, oldstack;
193	193	CHECK_EQ(0, sigaltstack(nullptr, &oldstack));
194	194	// If the alternate stack is already in place, do nothing.
195	195	// Android always sets an alternate stack, but it's too small for us.
196		if (!SANITIZER_ANDROID && !(oldstack.ss_flags & SS_DISABLE)) return;
	196	if (!SANITIZER_ANDROID && !(oldstack.ss_flags & SS_DISABLE))
	197	return nullptr;
197	198	// TODO(glider): the mapped stack should have the MAP_STACK flag in the
198	199	// future. It is not required by man 2 sigaltstack now (they're using
199	200	// malloc()).
...	...	@@ -201,15 +202,18 @@ void SetAlternateSignalStack() {
201	202	altstack.ss_sp = (char *)MmapOrDie(altstack.ss_size, __func__);
202	203	altstack.ss_flags = 0;
203	204	CHECK_EQ(0, sigaltstack(&altstack, nullptr));
	205	return altstack.ss_sp;
204	206	}
205	207
206		void UnsetAlternateSignalStack() {
	208	void UnsetAlternateSignalStack(void* altstack_base) {
207	209	stack_t altstack, oldstack;
208	210	altstack.ss_sp = nullptr;
209	211	altstack.ss_flags = SS_DISABLE;
210	212	altstack.ss_size = GetAltStackSize(); // Some sane value required on Darwin.
211	213	CHECK_EQ(0, sigaltstack(&altstack, &oldstack));
212		UnmapOrDie(oldstack.ss_sp, oldstack.ss_size);
	214	if (altstack_base && altstack_base == oldstack.ss_sp) {
	215	UnmapOrDie(oldstack.ss_sp, oldstack.ss_size);
	216	}
213	217	}
214	218
215	219	bool IsSignalHandlerFromSanitizer(int signum) {
...	...	@@ -562,11 +566,10 @@ pid_t StartSubprocess(const char program, const char const argv[],
562	566	internal_close(stderr_fd);
563	567	}
564	568
565		# if SANITIZER_FREEBSD
566		internal_close_range(3, ~static_cast<fd_t>(0), 0);
567		# else
568		for (int fd = sysconf(_SC_OPEN_MAX); fd > 2; fd--) internal_close(fd);
569		# endif
	569	// Close all fds except stdin/stdout/stderr before exec.
	570	// Fallback to the loop if close_range is not supported.
	571	if (internal_close_range(3, ~static_cast<fd_t>(0), 0) != 0)
	572	for (int fd = sysconf(_SC_OPEN_MAX); fd > 2; fd--) internal_close(fd);
570	573
571	574	internal_execve(program, const_cast<char **>(&argv[0]),
572	575	const_cast<char const >(envp));

lib/libtsan/sanitizer_common/sanitizer_redefine_builtins.h+17-5

...	...	@@ -17,11 +17,23 @@
17	17	// The asm hack only works with GCC and Clang.
18	18	# if !defined(_WIN32) && !defined(_AIX) && !defined(__APPLE__)
19	19
20		asm(R"(
21		.set memcpy, __sanitizer_internal_memcpy
22		.set memmove, __sanitizer_internal_memmove
23		.set memset, __sanitizer_internal_memset
24		)");
	20	# if defined(__hexagon__)
	21
	22	# define SANITIZER_REDEFINE_BUILTIN_ASM(name) \
	23	asm(".set " #name ", __sanitizer_internal_" #name)
	24
	25	# else
	26
	27	# define SANITIZER_REDEFINE_BUILTIN_ASM(name) \
	28	asm(#name " = __sanitizer_internal_" #name)
	29
	30	# endif
	31
	32	SANITIZER_REDEFINE_BUILTIN_ASM(memcpy);
	33	SANITIZER_REDEFINE_BUILTIN_ASM(memmove);
	34	SANITIZER_REDEFINE_BUILTIN_ASM(memset);
	35
	36	# undef SANITIZER_REDEFINE_BUILTIN_ASM
25	37
26	38	# if defined(__cplusplus) && \
27	39	!defined(SANITIZER_COMMON_REDEFINE_BUILTINS_IN_STD)

lib/libtsan/sanitizer_common/sanitizer_solaris.cpp+4

...	...	@@ -102,6 +102,10 @@ uptr internal_open(const char *filename, int flags, u32 mode) {
102	102	return _REAL64(open)(filename, flags, mode);
103	103	}
104	104
	105	uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags) {
	106	return -1; // Not supported.
	107	}
	108
105	109	DECLARE__REAL_AND_INTERNAL(uptr, read, fd_t fd, void *buf, uptr count) {
106	110	return _REAL(read)(fd, buf, count);
107	111	}

lib/libtsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp+11-3

...	...	@@ -16,7 +16,8 @@
16	16	#if SANITIZER_LINUX && \
17	17	(defined(__x86_64__) \|\| defined(__mips__) \|\| defined(__aarch64__) \|\| \
18	18	defined(__powerpc64__) \|\| defined(__s390__) \|\| defined(__i386__) \|\| \
19		defined(__arm__) \|\| SANITIZER_RISCV64 \|\| SANITIZER_LOONGARCH64)
	19	defined(__arm__) \|\| defined(__hexagon__) \|\| SANITIZER_RISCV64 \|\| \
	20	SANITIZER_LOONGARCH64)
20	21
21	22	#include "sanitizer_stoptheworld.h"
22	23
...	...	@@ -32,8 +33,8 @@
32	33	#include <sys/uio.h> // for iovec
33	34	#include <elf.h> // for NT_PRSTATUS
34	35	#if (defined(__aarch64__) \|\| defined(__powerpc64__) \|\| \
35		SANITIZER_RISCV64 \|\| SANITIZER_LOONGARCH64) && \
36		!SANITIZER_ANDROID
	36	defined(__hexagon__) \|\| SANITIZER_RISCV64 \|\| \
	37	SANITIZER_LOONGARCH64) && !SANITIZER_ANDROID
37	38	// GLIBC 2.20+ sys/user does not include asm/ptrace.h
38	39	# include <asm/ptrace.h>
39	40	#endif
...	...	@@ -613,6 +614,13 @@ typedef _user_regs_struct regs_struct;
613	614	static constexpr uptr kExtraRegs[] = {0};
614	615	#define ARCH_IOVEC_FOR_GETREGSET
615	616
	617	#elif defined(__hexagon__)
	618	#include <asm/user.h>
	619	typedef struct user_regs_struct regs_struct;
	620	#define REG_SP r29
	621	static constexpr uptr kExtraRegs[] = {0};
	622	#define ARCH_IOVEC_FOR_GETREGSET
	623
616	624	#else
617	625	#error "Unsupported architecture"
618	626	#endif // SANITIZER_ANDROID && defined(__arm__)

lib/libtsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp+7-7

...	...	@@ -475,6 +475,13 @@ static SymbolizerTool ChooseExternalSymbolizer(LowLevelAllocator allocator) {
475	475	return new (*allocator) Addr2LinePool(found_path, allocator);
476	476	}
477	477	}
	478
	479	# if SANITIZER_APPLE
	480	Report(
	481	"WARN: No external symbolizers found. Symbols may be missing or "
	482	"unreliable.\n");
	483	Report("HINT: Is PATH set? Does sandbox allow file-read of /usr/bin/atos?\n");
	484	# endif
478	485	return nullptr;
479	486	# endif // SANITIZER_DISABLE_SYMBOLIZER_PATH_SEARCH
480	487	}
...	...	@@ -509,13 +516,6 @@ static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,
509	516	}
510	517
511	518	# if SANITIZER_APPLE
512		if (list->empty()) {
513		Report(
514		"WARN: No external symbolizers found. Symbols may be missing or "
515		"unreliable.\n");
516		Report(
517		"HINT: Is PATH set? Does sandbox allow file-read of /usr/bin/atos?\n");
518		}
519	519	VReport(2, "Using dladdr symbolizer.\n");
520	520	list->push_back(new (*allocator) DlAddrSymbolizer());
521	521	# endif // SANITIZER_APPLE

lib/libtsan/sanitizer_common/sanitizer_symbolizer_report.cpp+7-6

...	...	@@ -184,7 +184,7 @@ static void MaybeReportNonExecRegion(uptr pc) {
184	184	MemoryMappedSegment segment;
185	185	while (proc_maps.Next(&segment)) {
186	186	if (pc >= segment.start && pc < segment.end && !segment.IsExecutable())
187		Report("Hint: PC is at a non-executable region. Maybe a wild jump?\n");
	187	Report("HINT: PC is at a non-executable region. Maybe a wild jump?\n");
188	188	}
189	189	#endif
190	190	}
...	...	@@ -254,7 +254,7 @@ static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid,
254	254	(void )sig.bp, (void )sig.sp, tid);
255	255	Printf("%s", d.Default());
256	256	if (sig.pc < GetPageSizeCached())
257		Report("Hint: pc points to the zero page.\n");
	257	Report("HINT: pc points to the zero page.\n");
258	258	if (sig.is_memory_access) {
259	259	const char *access_type =
260	260	sig.write_flag == SignalContext::Write
...	...	@@ -262,11 +262,12 @@ static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid,
262	262	: (sig.write_flag == SignalContext::Read ? "READ" : "UNKNOWN");
263	263	Report("The signal is caused by a %s memory access.\n", access_type);
264	264	if (!sig.is_true_faulting_addr)
265		Report("Hint: this fault was caused by a dereference of a high value "
266		"address (see register values below). Disassemble the provided "
267		"pc to learn which register was used.\n");
	265	Report(
	266	"HINT: this fault was caused by a dereference of a high value "
	267	"address (see register values below). Disassemble the provided "
	268	"pc to learn which register was used.\n");
268	269	else if (sig.addr < GetPageSizeCached())
269		Report("Hint: address points to the zero page.\n");
	270	Report("HINT: address points to the zero page.\n");
270	271	}
271	272	MaybeReportNonExecRegion(sig.pc);
272	273	InternalMmapVector<BufferedStackTrace> stack_buffer(1);

lib/libtsan/sanitizer_common/sanitizer_unwind_win.cpp+43-6

...	...	@@ -43,10 +43,47 @@ void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
43	43	trace_buffer[0] = pc;
44	44	}
45	45
46		#ifdef __clang__
47		#pragma clang diagnostic push
48		#pragma clang diagnostic ignored "-Wframe-larger-than="
49		#endif
	46	PVOID CALLBACK FallbackFunctionTableAccess(HANDLE hProcess,
	47	DWORD64 dwAddrBase) {
	48	// First try DbgHelp's function.
	49	if (PVOID pResult =
	50	__sanitizer::SymFunctionTableAccess64(hProcess, dwAddrBase)) {
	51	return pResult;
	52	}
	53
	54	// Fall back to RtlLookupFunctionEntry for dynamic code.
	55	// Function registered with RtlAddFunctionTable is not necessarily registered
	56	// with DbgHelp, so this is required to cover some edge cases (e.g. JIT
	57	// compilers can use Rtl* functions).
	58	# if SANITIZER_WINDOWS64
	59	DWORD64 dw64ImageBase = 0;
	60	return RtlLookupFunctionEntry(dwAddrBase, &dw64ImageBase, nullptr);
	61	# else
	62	return nullptr;
	63	# endif
	64	}
	65
	66	DWORD64 CALLBACK FallbackGetModuleBase(HANDLE hProcess, DWORD64 dwAddr) {
	67	if (DWORD64 dwResult = __sanitizer::SymGetModuleBase64(hProcess, dwAddr)) {
	68	return dwResult;
	69	}
	70
	71	// Both GetModuleBase and FunctionTableAccess must provide this fallback,
	72	// otherwise dynamic functions won't be properly unwound.
	73	# if SANITIZER_WINDOWS64
	74	DWORD64 dw64ImageBase = 0;
	75	if (RtlLookupFunctionEntry(dwAddr, &dw64ImageBase, nullptr)) {
	76	return dw64ImageBase;
	77	}
	78	# endif
	79
	80	return 0;
	81	}
	82
	83	# ifdef __clang__
	84	# pragma clang diagnostic push
	85	# pragma clang diagnostic ignored "-Wframe-larger-than="
	86	# endif
50	87	void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
51	88	CHECK(context);
52	89	CHECK_GE(max_depth, 2);
...	...	@@ -91,8 +128,8 @@ void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
91	128	stack_frame.AddrFrame.Mode = AddrModeFlat;
92	129	stack_frame.AddrStack.Mode = AddrModeFlat;
93	130	while (StackWalk64(machine_type, GetCurrentProcess(), GetCurrentThread(),
94		&stack_frame, &ctx, NULL, SymFunctionTableAccess64,
95		SymGetModuleBase64, NULL) &&
	131	&stack_frame, &ctx, NULL, FallbackFunctionTableAccess,
	132	FallbackGetModuleBase, NULL) &&
96	133	size < Min(max_depth, kStackTraceMax)) {
97	134	trace_buffer[size++] = (uptr)stack_frame.AddrPC.Offset;
98	135	}

lib/libtsan/sanitizer_common/sanitizer_win.cpp+4-3

...	...	@@ -913,11 +913,12 @@ void ReportFile::Write(const char *buffer, uptr length) {
913	913	}
914	914	}
915	915
916		void SetAlternateSignalStack() {
	916	void* SetAlternateSignalStack() {
917	917	// FIXME: Decide what to do on Windows.
	918	return nullptr;
918	919	}
919	920
920		void UnsetAlternateSignalStack() {
	921	void UnsetAlternateSignalStack(void* altstack_base) {
921	922	// FIXME: Decide what to do on Windows.
922	923	}
923	924
...	...	@@ -1222,7 +1223,7 @@ int WaitForProcess(pid_t pid) { return -1; }
1222	1223	// FIXME implement on this platform.
1223	1224	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
1224	1225
1225		void CheckNoDeepBind(const char *filename, int flag) {
	1226	void OnDlOpen(const char* filename, int flag) {
1226	1227	// Do nothing.
1227	1228	}
1228	1229

lib/libtsan/tsan_adaptive_delay.cpp created+433

...	...	@@ -0,0 +1,433 @@
	1	//===-- tsan_adaptive_delay.h ------------------------------------ C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	//===----------------------------------------------------------------------===//
	12
	13	#include "tsan_adaptive_delay.h"
	14
	15	#include "interception/interception.h"
	16	#include "sanitizer_common/sanitizer_allocator_internal.h"
	17	#include "sanitizer_common/sanitizer_common.h"
	18	#include "sanitizer_common/sanitizer_errno_codes.h"
	19	#include "tsan_interface.h"
	20	#include "tsan_rtl.h"
	21
	22	namespace __tsan {
	23
	24	namespace {
	25
	26	// =============================================================================
	27	// DelaySpec: Represents a delay configuration parsed from flag strings
	28	// =============================================================================
	29	//
	30	// Delay can be specified as:
	31	// - "spin=N" : Spin for up to N cycles (very short delays)
	32	// - "yield" : Call sched_yield() once
	33	// - "sleep_us=N" : Sleep for up to N microseconds
	34
	35	enum class DelayType { Spin, Yield, SleepUs };
	36
	37	struct DelaySpec {
	38	DelayType type;
	39	int value; // spin cycles or sleep_us value; ignored for yield
	40
	41	// Both estimates below are used internally as a very rough estimate for
	42	// delay overhead calculation, to cap the overall delay to the
	43	// adaptive_delay_aggressiveness option. They're not intended to be 100%
	44	// accurate on any or all architectures/operating systems, or for use in any
	45	// other contexts.
	46	//
	47	// Estimated nanoseconds per spin cycle (volatile loop iteration).
	48	static constexpr u64 kNsPerSpinCycle = 1;
	49	// Estimated nanoseconds for a yield (context switch overhead)
	50	static constexpr u64 kNsPerYield = 500;
	51
	52	static DelaySpec Parse(const char* str) {
	53	DelaySpec spec;
	54	if (internal_strncmp(str, "spin=", 5) == 0) {
	55	spec.type = DelayType::Spin;
	56	spec.value = internal_atoll(str + 5);
	57	if (spec.value <= 0 \|\| spec.value > 10000) {
	58	Printf(
	59	"FATAL: Invalid TSAN_OPTIONS spin value '%s'; value must be "
	60	"between 1 and 10000\n",
	61	str);
	62	Die();
	63	}
	64	} else if (internal_strcmp(str, "yield") == 0) {
	65	spec.type = DelayType::Yield;
	66	spec.value = 0;
	67	} else if (internal_strncmp(str, "sleep_us=", 9) == 0) {
	68	spec.type = DelayType::SleepUs;
	69	spec.value = internal_atoll(str + 9);
	70	if (spec.value <= 0) {
	71	Printf(
	72	"FATAL: Invalid TSAN_OPTIONS sleep_us value '%s'; value must be a "
	73	"positive integer\n",
	74	str);
	75	Die();
	76	}
	77	} else {
	78	Printf("FATAL: Unrecognized delay spec '%s', check TSAN_OPTIONS\n", str);
	79	Die();
	80	}
	81	return spec;
	82	}
	83
	84	const char* TypeName() const {
	85	switch (type) {
	86	case DelayType::Spin:
	87	return "spin";
	88	case DelayType::Yield:
	89	return "yield";
	90	case DelayType::SleepUs:
	91	return "sleep_us";
	92	}
	93	return "unknown";
	94	}
	95	};
	96
	97	} // namespace
	98
	99	// =============================================================================
	100	// AdaptiveDelayImpl: Time-budget aware delay injection for race exposure
	101	// =============================================================================
	102	//
	103	// This implementation injects delays to expose data races while maintaining a
	104	// configurable overhead target. It uses several strategies:
	105	//
	106	// 1. Time-Budget Controller: Tracks cumulative delays vs wall-clock time
	107	// and adjusts delay probability to maintain target overhead.
	108	//
	109	// 2. Tiered Delays: Different delay strategies for different op types:
	110	// - Relaxed atomics: Very rare sampling, tiny spin delays
	111	// - Sync atomics (acq/rel/seq_cst): Moderate sampling, small usleep
	112	// - Mutex/CV ops: Higher sampling, larger delays
	113	// - Thread create/join: Always delay (rare but high value)
	114	//
	115	// 3. Address-based Sampling: Exponential backoff per address to avoid
	116	// repeatedly delaying hot atomics.
	117
	118	struct AdaptiveDelayImpl {
	119	ALWAYS_INLINE static AdaptiveDelayState* TLS() {
	120	return &cur_thread()->adaptive_delay_state;
	121	}
	122	ALWAYS_INLINE static unsigned int* GetRandomSeed() {
	123	return &TLS()->tls_random_seed_;
	124	}
	125	ALWAYS_INLINE static void SetRandomSeed(unsigned int seed) {
	126	TLS()->tls_random_seed_ = seed;
	127	}
	128
	129	// The public facing option is adaptive_delay_aggressiveness, which is an
	130	// opaque value for the user to tune the amount of delay injected into the
	131	// program. Internally, the implementation maps the aggressiveness to a target
	132	// percent delay for the overall program runtime. It's not easy to implement
	133	// a true wall clock delay target (e.g., 25% program wall time slowdown)
	134	// because 1) spin loops and yield are hard to calculate actual wall time
	135	// slowness and 2) usleep(N) is often slower than advertised. Thus, we keep
	136	// the user facing parameter opaque to not under deliver on a promise of
	137	// percent wall time slowdown.
	138	struct TimeBudget {
	139	int target_overhead_pct_;
	140	Percent target_low_;
	141	Percent target_high_;
	142
	143	void Init(int target_pct) {
	144	target_overhead_pct_ = target_pct;
	145	target_low_ = Percent::FromPct(
	146	target_overhead_pct_ >= 5 ? target_overhead_pct_ - 5 : 0);
	147	target_high_ = Percent::FromPct(target_overhead_pct_ + 5);
	148	}
	149
	150	static constexpr u64 BucketDurationNs = 30'000'000'000ULL;
	151
	152	void RecordDelay(u64 delay_ns) {
	153	u64 now = NanoTime();
	154	u64 elapsed_ns = now - TLS()->bucket_start_ns_;
	155
	156	if (elapsed_ns >= BucketDurationNs) {
	157	// Shift: old bucket is discarded, new becomes old, start fresh new
	158	TLS()->delay_buckets_ns_[0] = TLS()->delay_buckets_ns_[1];
	159	TLS()->delay_buckets_ns_[1] = 0;
	160	TLS()->bucket_start_ns_ = now;
	161	TLS()->bucket0_window_ns = BucketDurationNs;
	162	}
	163
	164	TLS()->delay_buckets_ns_[1] += delay_ns;
	165	}
	166
	167	Percent GetOverheadPercent() {
	168	u64 now = NanoTime();
	169	u64 elapsed_ns = now - TLS()->bucket_start_ns_;
	170
	171	// Need at least 1ms to calculate
	172	if (elapsed_ns < 1'000'000ULL)
	173	return Percent::FromPct(0);
	174
	175	if (elapsed_ns > BucketDurationNs * 2) {
	176	// Both buckets are stale
	177	return Percent::FromPct(0);
	178	} else if (elapsed_ns > BucketDurationNs) {
	179	// bucket[0] is stale, use only bucket[1] (current bucket)
	180	u64 total_delay_ns = TLS()->delay_buckets_ns_[1];
	181	return Percent::FromRatio(total_delay_ns, elapsed_ns);
	182	} else {
	183	u64 total_delay_ns =
	184	TLS()->delay_buckets_ns_[0] + TLS()->delay_buckets_ns_[1];
	185	u64 window_ns = TLS()->bucket0_window_ns + elapsed_ns;
	186	return Percent::FromRatio(total_delay_ns, window_ns);
	187	}
	188	}
	189
	190	bool ShouldDelay() {
	191	Percent ratio = GetOverheadPercent();
	192
	193	if (ratio < target_low_)
	194	return true;
	195	if (ratio > target_high_)
	196	return false;
	197
	198	// Linear interpolation: at target_low -> 100%, at target_high -> 0%
	199	Percent prob = (target_high_ - ratio) / (target_high_ - target_low_);
	200	return prob.RandomCheck(GetRandomSeed());
	201	}
	202	};
	203
	204	// Address Sampler with Exponential Backoff
	205	struct AddressSampler {
	206	static constexpr u64 TABLE_SIZE = 2048;
	207	struct Entry {
	208	atomic_uintptr_t addr_;
	209	atomic_uint32_t count_;
	210	};
	211	Entry table_[TABLE_SIZE];
	212	static constexpr u32 ExponentialBackoffCap = 64;
	213
	214	void Init() {
	215	for (u64 i = 0; i < TABLE_SIZE; ++i) {
	216	atomic_store(&table_[i].addr_, 0, memory_order_relaxed);
	217	atomic_store(&table_[i].count_, 0, memory_order_relaxed);
	218	}
	219	}
	220
	221	static ALWAYS_INLINE u64 splitmix64(u64 x) {
	222	x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9ULL;
	223	x = (x ^ (x >> 27)) * 0x94D049BB133111EBULL;
	224	x = x ^ (x >> 31);
	225	return x;
	226	}
	227
	228	// Uses exponential backoff: delay on 1st, 2nd, 4th, 8th, 16th, ...
	229	bool ShouldDelayAddr(uptr addr) {
	230	u64 idx = splitmix64(addr >> 3) & (TABLE_SIZE - 1);
	231	Entry& e = table_[idx];
	232
	233	// This function is not thread safe.
	234	// If two threads access the same hashed entry in parallel,
	235	// worst case, we may end up returning true too often. This is
	236	// acceptable...instead of full locking.
	237
	238	uptr stored_addr = atomic_load(&e.addr_, memory_order_relaxed);
	239	if (stored_addr != addr) {
	240	// Hash Collision - reset
	241	atomic_store(&e.addr_, addr, memory_order_relaxed);
	242	atomic_store(&e.count_, 1, memory_order_relaxed);
	243	return true;
	244	}
	245
	246	u32 count = atomic_fetch_add(&e.count_, 1, memory_order_relaxed) + 1;
	247
	248	if ((count & (count - 1)) == 0 && count <= ExponentialBackoffCap)
	249	return true;
	250	return false;
	251	}
	252	};
	253
	254	TimeBudget budget_;
	255	AddressSampler sampler_;
	256
	257	int relaxed_sample_rate_;
	258	int sync_atomic_sample_rate_;
	259	int mutex_sample_rate_;
	260	DelaySpec atomic_delay_;
	261	DelaySpec sync_delay_;
	262
	263	void Init() { InitTls(); }
	264
	265	void InitTls() {
	266	TLS()->bucket_start_ns_ = NanoTime();
	267	TLS()->delay_buckets_ns_[0] = 0;
	268	TLS()->delay_buckets_ns_[1] = 0;
	269	TLS()->bucket0_window_ns = 0;
	270
	271	SetRandomSeed(NanoTime());
	272	TLS()->tls_initialized_ = true;
	273	}
	274
	275	bool IsTlsInitialized() const { return TLS()->tls_initialized_; }
	276
	277	AdaptiveDelayImpl() {
	278	relaxed_sample_rate_ = flags()->adaptive_delay_relaxed_sample_rate;
	279	sync_atomic_sample_rate_ = flags()->adaptive_delay_sync_atomic_sample_rate;
	280	mutex_sample_rate_ = flags()->adaptive_delay_mutex_sample_rate;
	281	atomic_delay_ = DelaySpec::Parse(flags()->adaptive_delay_max_atomic);
	282	sync_delay_ = DelaySpec::Parse(flags()->adaptive_delay_max_sync);
	283
	284	int delay_aggressiveness = flags()->adaptive_delay_aggressiveness;
	285	if (delay_aggressiveness < 1)
	286	delay_aggressiveness = 1;
	287
	288	budget_.Init(delay_aggressiveness);
	289	sampler_.Init();
	290
	291	VPrintf(1, "INFO: ThreadSanitizer AdaptiveDelay initialized\n");
	292	VPrintf(1, " Delay aggressiveness: %d\n", delay_aggressiveness);
	293	VPrintf(1, " Relaxed atomic sample rate: 1/%d\n", relaxed_sample_rate_);
	294	VPrintf(1, " Sync atomic sample rate: 1/%d\n", sync_atomic_sample_rate_);
	295	VPrintf(1, " Mutex sample rate: 1/%d\n", mutex_sample_rate_);
	296	VPrintf(1, " Atomic delay: %s=%d\n", atomic_delay_.TypeName(),
	297	atomic_delay_.value);
	298	VPrintf(1, " Sync delay: %s=%d\n", sync_delay_.TypeName(),
	299	sync_delay_.value);
	300	}
	301
	302	void DoSpinDelay(int iters) {
	303	volatile int v = 0;
	304	for (int i = 0; i < iters; ++i) v = i;
	305	(void)v;
	306	budget_.RecordDelay(iters * DelaySpec::kNsPerSpinCycle);
	307	}
	308
	309	void DoYieldDelay() {
	310	internal_sched_yield();
	311	budget_.RecordDelay(DelaySpec::kNsPerYield);
	312	}
	313
	314	void DoSleepUsDelay(int max_us) {
	315	// Use two Rand() calls to get full 32-bit range for larger sleep values
	316	u32 rnd = ((u32)Rand(GetRandomSeed()) << 16) \| Rand(GetRandomSeed());
	317	int delay_us = 1 + (rnd % max_us);
	318	internal_usleep(delay_us);
	319	budget_.RecordDelay(delay_us * 1000ULL);
	320	}
	321
	322	void ExecuteDelay(const DelaySpec& spec) {
	323	switch (spec.type) {
	324	case DelayType::Spin: {
	325	int iters = 1 + (Rand(GetRandomSeed()) % spec.value);
	326	DoSpinDelay(iters);
	327	break;
	328	}
	329	case DelayType::Yield:
	330	DoYieldDelay();
	331	break;
	332	case DelayType::SleepUs:
	333	DoSleepUsDelay(spec.value);
	334	break;
	335	}
	336	}
	337
	338	void AtomicRelaxedOpDelay() {
	339	if ((Rand(GetRandomSeed()) % relaxed_sample_rate_) != 0)
	340	return;
	341	if (!budget_.ShouldDelay())
	342	return;
	343
	344	int iters = 10 + (Rand(GetRandomSeed()) % 10);
	345	DoSpinDelay(iters);
	346	}
	347
	348	void AtomicSyncOpDelay(uptr* addr) {
	349	if ((Rand(GetRandomSeed()) % sync_atomic_sample_rate_) != 0)
	350	return;
	351	if (!budget_.ShouldDelay())
	352	return;
	353
	354	if (addr && !sampler_.ShouldDelayAddr(*addr))
	355	return;
	356
	357	ExecuteDelay(atomic_delay_);
	358	}
	359
	360	void AtomicOpFence(int mo) {
	361	CHECK(IsTlsInitialized());
	362
	363	if (mo < mo_acquire)
	364	AtomicRelaxedOpDelay();
	365	else
	366	AtomicSyncOpDelay(nullptr);
	367	}
	368
	369	void AtomicOpAddr(uptr addr, int mo) {
	370	CHECK(IsTlsInitialized());
	371
	372	if (mo < mo_acquire)
	373	AtomicRelaxedOpDelay();
	374	else
	375	AtomicSyncOpDelay(&addr);
	376	}
	377
	378	void UnsampledDelay() {
	379	CHECK(IsTlsInitialized());
	380
	381	if (!budget_.ShouldDelay())
	382	return;
	383
	384	ExecuteDelay(sync_delay_);
	385	}
	386
	387	void SyncOp() {
	388	CHECK(IsTlsInitialized());
	389
	390	if ((Rand(GetRandomSeed()) % mutex_sample_rate_) != 0)
	391	return;
	392	if (!budget_.ShouldDelay())
	393	return;
	394
	395	ExecuteDelay(sync_delay_);
	396	}
	397
	398	void BeforeChildThreadRuns() {
	399	InitTls();
	400	UnsampledDelay();
	401	}
	402
	403	void AfterThreadCreation() { UnsampledDelay(); }
	404	};
	405
	406	AdaptiveDelayImpl& GetImpl() {
	407	static AdaptiveDelayImpl impl;
	408	return impl;
	409	}
	410
	411	bool AdaptiveDelay::is_adaptive_delay_enabled;
	412
	413	void AdaptiveDelay::InitImpl() {
	414	AdaptiveDelay::is_adaptive_delay_enabled = flags()->enable_adaptive_delay;
	415	if (!AdaptiveDelay::is_adaptive_delay_enabled)
	416	return;
	417
	418	GetImpl().Init();
	419	}
	420
	421	void AdaptiveDelay::SyncOpImpl() { GetImpl().SyncOp(); }
	422	void AdaptiveDelay::AtomicOpFenceImpl(int mo) { GetImpl().AtomicOpFence(mo); }
	423	void AdaptiveDelay::AtomicOpAddrImpl(__sanitizer::uptr addr, int mo) {
	424	GetImpl().AtomicOpAddr(addr, mo);
	425	}
	426	void AdaptiveDelay::AfterThreadCreationImpl() {
	427	GetImpl().AfterThreadCreation();
	428	}
	429	void AdaptiveDelay::BeforeChildThreadRunsImpl() {
	430	GetImpl().BeforeChildThreadRuns();
	431	}
	432
	433	} // namespace __tsan

lib/libtsan/tsan_adaptive_delay.h created+173

...	...	@@ -0,0 +1,173 @@
	1	//===-- tsan_adaptive_delay.h ------------------------------------ C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	//===----------------------------------------------------------------------===//
	12
	13	#ifndef TSAN_ADAPTIVE_DELAY_H
	14	#define TSAN_ADAPTIVE_DELAY_H
	15
	16	#include "sanitizer_common/sanitizer_common.h"
	17	#include "sanitizer_common/sanitizer_internal_defs.h"
	18
	19	namespace __tsan {
	20
	21	// AdaptiveDelay injects delays at synchronization points, atomic operations,
	22	// and thread lifecycle events to increase the likelihood of exposing data
	23	// races. The delay injection is controlled by an approximate time budget to
	24	// maintain a configurable overhead target.
	25	//
	26	// SyncOp() delays non-atomic synchronization points (those with clear
	27	// happens-before relationships):
	28	// - Acquire operations like locking a mutex delays before the mutex is locked.
	29	// - Release operations like unlocking a mutex delays after the mutex is
	30	// unlocked
	31	// These are more likely to expose interesting (rare) thread interleavings.
	32	// For example, delaying a thread that unlocks a mutex from running to allow
	33	// newly woken thread to execute before the unlocking thread would normally
	34	// execute.
	35	//
	36	// TODO:
	37	// - Move the adaptive delay implementation into sanitizer_common so that
	38	// ASAN can also leverage it in pthread_* interceptors
	39	// - Integrate into other interceptors like libdispatch.
	40	struct AdaptiveDelay {
	41	ALWAYS_INLINE static void Init() { InitImpl(); }
	42
	43	ALWAYS_INLINE static void SyncOp() {
	44	if (!is_adaptive_delay_enabled)
	45	return;
	46	SyncOpImpl();
	47	}
	48
	49	ALWAYS_INLINE static void AtomicOpFence(int mo) {
	50	if (!is_adaptive_delay_enabled)
	51	return;
	52	AtomicOpFenceImpl(mo);
	53	}
	54
	55	ALWAYS_INLINE static void AtomicOpAddr(__sanitizer::uptr addr, int mo) {
	56	if (!is_adaptive_delay_enabled)
	57	return;
	58	AtomicOpAddrImpl(addr, mo);
	59	}
	60
	61	ALWAYS_INLINE static void AfterThreadCreation() {
	62	if (!is_adaptive_delay_enabled)
	63	return;
	64	AfterThreadCreationImpl();
	65	}
	66
	67	ALWAYS_INLINE static void BeforeChildThreadRuns() {
	68	if (!is_adaptive_delay_enabled)
	69	return;
	70	BeforeChildThreadRunsImpl();
	71	}
	72
	73	private:
	74	static void InitImpl();
	75
	76	static void SyncOpImpl();
	77
	78	static void AtomicOpFenceImpl(int mo);
	79	static void AtomicOpAddrImpl(__sanitizer::uptr addr, int mo);
	80
	81	static void AfterThreadCreationImpl();
	82	static void BeforeChildThreadRunsImpl();
	83
	84	static bool is_adaptive_delay_enabled;
	85	};
	86
	87	// The runtime defines cur_thread() to retrieve TLS thread state, and it
	88	// takes care of platform specific implementation details. The AdaptiveDelay
	89	// implementation stores per-thread data in this struct, which is embedded
	90	// in cur_thread().
	91	struct AdaptiveDelayState {
	92	// For the adaptive delay implementation
	93	// Sliding window delay tracking: 2 buckets of 30 seconds each
	94	u64 delay_buckets_ns_[2]; // [0] = older 30s, [1] = newer 30s
	95	u64 bucket_start_ns_; // When current bucket (index 1) started
	96	u64 bucket0_window_ns; // 0ns before the first bucket has rolled, and set to
	97	// the bucket window time after This handles the case
	98	// where, before the program has ran one bucket window
	99	// duration, we should not include the previous bucket
	100	// duration in the overhead percent calculation.
	101	unsigned int tls_random_seed_;
	102	bool tls_initialized_;
	103	};
	104
	105	// Fixed-point arithmetic type that mimics floating point operations
	106	class Percent {
	107	using u32 = __sanitizer::u32;
	108	using u64 = __sanitizer::u64;
	109
	110	u32 bp_{}; // basis points (0-10000 represents 0.0-1.0)
	111	bool is_valid_{};
	112
	113	static constexpr u32 kBasisPointsPerUnit = 10000;
	114
	115	Percent(u32 bp, bool is_valid) : bp_(bp), is_valid_(is_valid) {}
	116
	117	public:
	118	Percent() = default;
	119	Percent(const Percent&) = default;
	120	Percent& operator=(const Percent&) = default;
	121	Percent(Percent&&) = default;
	122	Percent& operator=(Percent&&) = default;
	123
	124	static Percent FromPct(u32 pct) { return Percent{pct * 100, true}; }
	125	static Percent FromRatio(u64 numerator, u64 denominator) {
	126	if (denominator == 0)
	127	return Percent{0, false};
	128	// Avoid overflow: scale down if needed
	129	if (numerator > UINT64_MAX / kBasisPointsPerUnit) {
	130	return Percent{(u32)((numerator / denominator) * kBasisPointsPerUnit),
	131	true};
	132	}
	133	return Percent{(u32)((numerator * kBasisPointsPerUnit) / denominator),
	134	true};
	135	}
	136
	137	bool IsValid() const { return is_valid_; }
	138
	139	// Returns true with probability equal to the percentage.
	140	bool RandomCheck(u32* seed) const {
	141	return (Rand(seed) % kBasisPointsPerUnit) < bp_;
	142	}
	143
	144	int GetPct() const { return bp_ / 100; }
	145	int GetBasisPoints() const { return bp_; }
	146
	147	bool operator==(const Percent& other) const { return bp_ == other.bp_; }
	148	bool operator!=(const Percent& other) const { return bp_ != other.bp_; }
	149	bool operator<(const Percent& other) const { return bp_ < other.bp_; }
	150	bool operator>(const Percent& other) const { return bp_ > other.bp_; }
	151	bool operator<=(const Percent& other) const { return bp_ <= other.bp_; }
	152	bool operator>=(const Percent& other) const { return bp_ >= other.bp_; }
	153
	154	Percent operator-(const Percent& other) const {
	155	if (!is_valid_ \|\| !other.is_valid_)
	156	return Percent{0, false};
	157	if (bp_ < other.bp_)
	158	return Percent{0, false};
	159	return Percent{bp_ - other.bp_, true};
	160	}
	161
	162	Percent operator/(const Percent& other) const {
	163	if (!is_valid_ \|\| !other.is_valid_)
	164	return Percent{0, false};
	165	if (other.bp_ == 0)
	166	return Percent{0, false};
	167	return Percent{(bp_ * kBasisPointsPerUnit) / other.bp_, true};
	168	}
	169	};
	170
	171	} // namespace __tsan
	172
	173	#endif // TSAN_ADAPTIVE_DELAY_H

lib/libtsan/tsan_flags.cpp+1-1

...	...	@@ -37,7 +37,7 @@ inline bool FlagHandler<LockDuringWriteSetting>::Parse(const char *value) {
37	37	*t_ = kNoLockDuringWritesAllProcesses;
38	38	return true;
39	39	}
40		Printf("ERROR: Invalid value for signal handler option: '%s'\n", value);
	40	Printf("ERROR: Invalid value for lock_during_write option: '%s'\n", value);
41	41	return false;
42	42	}
43	43

lib/libtsan/tsan_flags.inc+27

...	...	@@ -92,3 +92,30 @@ TSAN_FLAG(LockDuringWriteSetting, lock_during_write, kLockDuringAllWrites,
92	92	"\"disable_for_all_processes\" - don't lock during all writes in "
93	93	"the current process and it's children processes.")
94	94	#endif
	95
	96	TSAN_FLAG(bool, enable_adaptive_delay, false,
	97	"Enable adaptive delay injection to expose data races. When "
	98	"enabled, delays are strategically injected at synchronization "
	99	"points, atomic operations, and thread lifecycle events to increase "
	100	"the likelihood of exposing races while maintaining a configurable "
	101	"overhead budget.")
	102
	103	TSAN_FLAG(
	104	int, adaptive_delay_aggressiveness, 25,
	105	"Controls delay injection intensity for race detection. Higher values "
	106	"inject more delays to expose races. Suggested values: 10 (minimal delay), "
	107	"50 (moderate delay), 200 (aggressive). "
	108	"This is a tuning parameter; actual overhead varies by workload and "
	109	"platform.")
	110	TSAN_FLAG(int, adaptive_delay_relaxed_sample_rate, 10000,
	111	"Sample 1 in N relaxed atomic operations for delay")
	112	TSAN_FLAG(int, adaptive_delay_sync_atomic_sample_rate, 100,
	113	"Sample 1 in N acquire/release/seq_cst atomic operations for delay")
	114	TSAN_FLAG(int, adaptive_delay_mutex_sample_rate, 10,
	115	"Sample 1 in N mutex/cv operations for delay")
	116	TSAN_FLAG(const char*, adaptive_delay_max_atomic, "sleep_us=50",
	117	"Delay for atomic operations: 'spin=N' (max N spins), 'yield', or "
	118	"'sleep_us=N' (max N>0 us sleep)")
	119	TSAN_FLAG(const char*, adaptive_delay_max_sync, "sleep_us=500",
	120	"Delay for sync operations: 'spin=N' (max N spins), 'yield', or "
	121	"'sleep_us=N' (max N>0 us sleep)")

lib/libtsan/tsan_interceptors_posix.cpp+24-2

...	...	@@ -34,6 +34,7 @@
34	34	#if SANITIZER_APPLE && !SANITIZER_GO
35	35	# include "tsan_flags.h"
36	36	#endif
	37	#include "tsan_adaptive_delay.h"
37	38	#include "tsan_interceptors.h"
38	39	#include "tsan_interface.h"
39	40	#include "tsan_mman.h"
...	...	@@ -1065,6 +1066,9 @@ extern "C" void __tsan_thread_start_func(void arg) {
1065	1066	ThreadStart(thr, p->tid, GetTid(), ThreadType::Regular);
1066	1067	p->started.Post();
1067	1068	}
	1069
	1070	AdaptiveDelay::BeforeChildThreadRuns();
	1071
1068	1072	void *res = callback(param);
1069	1073	// Prevent the callback from being tail called,
1070	1074	// it mixes up stack traces.
...	...	@@ -1128,6 +1132,7 @@ TSAN_INTERCEPTOR(int, pthread_create,
1128	1132	}
1129	1133	if (attr == &myattr)
1130	1134	pthread_attr_destroy(&myattr);
	1135	AdaptiveDelay::AfterThreadCreation();
1131	1136	return res;
1132	1137	}
1133	1138
...	...	@@ -1423,6 +1428,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
1423	1428	TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) {
1424	1429	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock, m);
1425	1430	MutexPreLock(thr, pc, (uptr)m);
	1431	AdaptiveDelay::SyncOp();
1426	1432	int res = BLOCK_REAL(pthread_mutex_lock)(m);
1427	1433	if (res == errno_EOWNERDEAD)
1428	1434	MutexRepair(thr, pc, (uptr)m);
...	...	@@ -1435,6 +1441,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) {
1435	1441
1436	1442	TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
1437	1443	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
	1444	AdaptiveDelay::SyncOp();
1438	1445	int res = REAL(pthread_mutex_trylock)(m);
1439	1446	if (res == errno_EOWNERDEAD)
1440	1447	MutexRepair(thr, pc, (uptr)m);
...	...	@@ -1446,6 +1453,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
1446	1453	#if !SANITIZER_APPLE
1447	1454	TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void m, void abstime) {
1448	1455	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime);
	1456	AdaptiveDelay::SyncOp();
1449	1457	int res = REAL(pthread_mutex_timedlock)(m, abstime);
1450	1458	if (res == 0) {
1451	1459	MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
...	...	@@ -1458,6 +1466,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) {
1458	1466	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_unlock, m);
1459	1467	MutexUnlock(thr, pc, (uptr)m);
1460	1468	int res = REAL(pthread_mutex_unlock)(m);
	1469	AdaptiveDelay::SyncOp();
1461	1470	if (res == errno_EINVAL)
1462	1471	MutexInvalidAccess(thr, pc, (uptr)m);
1463	1472	return res;
...	...	@@ -1468,6 +1477,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_clocklock, void *m,
1468	1477	__sanitizer_clockid_t clock, void *abstime) {
1469	1478	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_clocklock, m, clock, abstime);
1470	1479	MutexPreLock(thr, pc, (uptr)m);
	1480	AdaptiveDelay::SyncOp();
1471	1481	int res = BLOCK_REAL(pthread_mutex_clocklock)(m, clock, abstime);
1472	1482	if (res == errno_EOWNERDEAD)
1473	1483	MutexRepair(thr, pc, (uptr)m);
...	...	@@ -1486,6 +1496,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_clocklock, void *m,
1486	1496	TSAN_INTERCEPTOR(int, __pthread_mutex_lock, void *m) {
1487	1497	SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_lock, m);
1488	1498	MutexPreLock(thr, pc, (uptr)m);
	1499	AdaptiveDelay::SyncOp();
1489	1500	int res = BLOCK_REAL(__pthread_mutex_lock)(m);
1490	1501	if (res == errno_EOWNERDEAD)
1491	1502	MutexRepair(thr, pc, (uptr)m);
...	...	@@ -1500,6 +1511,7 @@ TSAN_INTERCEPTOR(int, __pthread_mutex_unlock, void *m) {
1500	1511	SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_unlock, m);
1501	1512	MutexUnlock(thr, pc, (uptr)m);
1502	1513	int res = REAL(__pthread_mutex_unlock)(m);
	1514	AdaptiveDelay::SyncOp();
1503	1515	if (res == errno_EINVAL)
1504	1516	MutexInvalidAccess(thr, pc, (uptr)m);
1505	1517	return res;
...	...	@@ -1529,6 +1541,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) {
1529	1541	TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) {
1530	1542	SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m);
1531	1543	MutexPreLock(thr, pc, (uptr)m);
	1544	AdaptiveDelay::SyncOp();
1532	1545	int res = BLOCK_REAL(pthread_spin_lock)(m);
1533	1546	if (res == 0) {
1534	1547	MutexPostLock(thr, pc, (uptr)m);
...	...	@@ -1538,6 +1551,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) {
1538	1551
1539	1552	TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) {
1540	1553	SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m);
	1554	AdaptiveDelay::SyncOp();
1541	1555	int res = REAL(pthread_spin_trylock)(m);
1542	1556	if (res == 0) {
1543	1557	MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
...	...	@@ -1549,6 +1563,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) {
1549	1563	SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m);
1550	1564	MutexUnlock(thr, pc, (uptr)m);
1551	1565	int res = REAL(pthread_spin_unlock)(m);
	1566	AdaptiveDelay::SyncOp();
1552	1567	return res;
1553	1568	}
1554	1569	#endif
...	...	@@ -1574,6 +1589,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) {
1574	1589	TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) {
1575	1590	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m);
1576	1591	MutexPreReadLock(thr, pc, (uptr)m);
	1592	AdaptiveDelay::SyncOp();
1577	1593	int res = REAL(pthread_rwlock_rdlock)(m);
1578	1594	if (res == 0) {
1579	1595	MutexPostReadLock(thr, pc, (uptr)m);
...	...	@@ -1583,6 +1599,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) {
1583	1599
1584	1600	TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) {
1585	1601	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m);
	1602	AdaptiveDelay::SyncOp();
1586	1603	int res = REAL(pthread_rwlock_tryrdlock)(m);
1587	1604	if (res == 0) {
1588	1605	MutexPostReadLock(thr, pc, (uptr)m, MutexFlagTryLock);
...	...	@@ -1593,6 +1610,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) {
1593	1610	#if !SANITIZER_APPLE
1594	1611	TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void m, void abstime) {
1595	1612	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime);
	1613	AdaptiveDelay::SyncOp();
1596	1614	int res = REAL(pthread_rwlock_timedrdlock)(m, abstime);
1597	1615	if (res == 0) {
1598	1616	MutexPostReadLock(thr, pc, (uptr)m);
...	...	@@ -1604,6 +1622,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void m, void abstime) {
1604	1622	TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) {
1605	1623	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m);
1606	1624	MutexPreLock(thr, pc, (uptr)m);
	1625	AdaptiveDelay::SyncOp();
1607	1626	int res = BLOCK_REAL(pthread_rwlock_wrlock)(m);
1608	1627	if (res == 0) {
1609	1628	MutexPostLock(thr, pc, (uptr)m);
...	...	@@ -1613,6 +1632,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) {
1613	1632
1614	1633	TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) {
1615	1634	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m);
	1635	AdaptiveDelay::SyncOp();
1616	1636	int res = REAL(pthread_rwlock_trywrlock)(m);
1617	1637	if (res == 0) {
1618	1638	MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
...	...	@@ -1623,6 +1643,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) {
1623	1643	#if !SANITIZER_APPLE
1624	1644	TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void m, void abstime) {
1625	1645	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime);
	1646	AdaptiveDelay::SyncOp();
1626	1647	int res = REAL(pthread_rwlock_timedwrlock)(m, abstime);
1627	1648	if (res == 0) {
1628	1649	MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
...	...	@@ -1635,6 +1656,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) {
1635	1656	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m);
1636	1657	MutexReadOrWriteUnlock(thr, pc, (uptr)m);
1637	1658	int res = REAL(pthread_rwlock_unlock)(m);
	1659	AdaptiveDelay::SyncOp();
1638	1660	return res;
1639	1661	}
1640	1662
...	...	@@ -2574,9 +2596,9 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
2574	2596
2575	2597	#define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \
2576	2598	({ \
2577		CheckNoDeepBind(filename, flag); \
	2599	OnDlOpen(filename, flag); \
2578	2600	ThreadIgnoreBegin(thr, 0); \
2579		void *res = REAL(dlopen)(filename, flag); \
	2601	void* res = REAL(dlopen)(filename, flag); \
2580	2602	ThreadIgnoreEnd(thr); \
2581	2603	res; \
2582	2604	})

lib/libtsan/tsan_interface_ann.cpp+9-5

...	...	@@ -9,17 +9,19 @@
9	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	10	//
11	11	//===----------------------------------------------------------------------===//
12		#include "sanitizer_common/sanitizer_libc.h"
	12	#include "tsan_interface_ann.h"
	13
13	14	#include "sanitizer_common/sanitizer_internal_defs.h"
	15	#include "sanitizer_common/sanitizer_libc.h"
14	16	#include "sanitizer_common/sanitizer_placement_new.h"
15	17	#include "sanitizer_common/sanitizer_stacktrace.h"
16	18	#include "sanitizer_common/sanitizer_vector.h"
17		#include "tsan_interface_ann.h"
18		#include "tsan_report.h"
19		#include "tsan_rtl.h"
20		#include "tsan_mman.h"
	19	#include "tsan_adaptive_delay.h"
21	20	#include "tsan_flags.h"
	21	#include "tsan_mman.h"
22	22	#include "tsan_platform.h"
	23	#include "tsan_report.h"
	24	#include "tsan_rtl.h"
23	25
24	26	#define CALLERPC ((uptr)__builtin_return_address(0))
25	27
...	...	@@ -370,6 +372,7 @@ void __tsan_mutex_pre_lock(void *m, unsigned flagz) {
370	372	}
371	373	ThreadIgnoreBegin(thr, 0);
372	374	ThreadIgnoreSyncBegin(thr, 0);
	375	AdaptiveDelay::SyncOp();
373	376	}
374	377
375	378	INTERFACE_ATTRIBUTE
...	...	@@ -402,6 +405,7 @@ int __tsan_mutex_pre_unlock(void *m, unsigned flagz) {
402	405
403	406	INTERFACE_ATTRIBUTE
404	407	void __tsan_mutex_post_unlock(void *m, unsigned flagz) {
	408	AdaptiveDelay::SyncOp();
405	409	SCOPED_ANNOTATION(__tsan_mutex_post_unlock);
406	410	ThreadIgnoreSyncEnd(thr);
407	411	ThreadIgnoreEnd(thr);

lib/libtsan/tsan_interface_atomic.cpp+12

...	...	@@ -21,6 +21,7 @@
21	21	#include "sanitizer_common/sanitizer_mutex.h"
22	22	#include "sanitizer_common/sanitizer_placement_new.h"
23	23	#include "sanitizer_common/sanitizer_stacktrace.h"
	24	#include "tsan_adaptive_delay.h"
24	25	#include "tsan_flags.h"
25	26	#include "tsan_interface.h"
26	27	#include "tsan_rtl.h"
...	...	@@ -520,8 +521,19 @@ static morder to_morder(int mo) {
520	521	return res;
521	522	}
522	523
	524	template <class... Types>
	525	ALWAYS_INLINE auto AtomicDelayImpl(morder mo, Types... args) {
	526	AdaptiveDelay::AtomicOpFence(mo);
	527	}
	528
	529	template <class AddrType, class... Types>
	530	ALWAYS_INLINE auto AtomicDelayImpl(morder mo, AddrType addr, Types... args) {
	531	AdaptiveDelay::AtomicOpAddr((uptr)addr, (int)mo);
	532	}
	533
523	534	template <class Op, class... Types>
524	535	ALWAYS_INLINE auto AtomicImpl(morder mo, Types... args) {
	536	AtomicDelayImpl(mo, args...);
525	537	ThreadState *const thr = cur_thread();
526	538	ProcessPendingSignals(thr);
527	539	if (UNLIKELY(thr->ignore_sync \|\| thr->ignore_interceptors))

lib/libtsan/tsan_platform.h+10-7

...	...	@@ -404,7 +404,7 @@ struct MappingRiscv64_39 {
404	404	static const uptr kHeapMemBeg = 0x2c00000000ull;
405	405	static const uptr kHeapMemEnd = 0x2c00000000ull;
406	406	static const uptr kHiAppMemBeg = 0x3c00000000ull;
407		static const uptr kHiAppMemEnd = 0x3fffffffffull;
	407	static const uptr kHiAppMemEnd = 0x4000000000ull;
408	408	static const uptr kShadowMsk = 0x3800000000ull;
409	409	static const uptr kShadowXor = 0x0800000000ull;
410	410	static const uptr kShadowAdd = 0x0000000000ull;
...	...	@@ -434,7 +434,7 @@ struct MappingRiscv64_48 {
434	434	static const uptr kHeapMemBeg = 0x5a0000000000ull;
435	435	static const uptr kHeapMemEnd = 0x5a0000000000ull;
436	436	static const uptr kHiAppMemBeg = 0x7a0000000000ull;
437		static const uptr kHiAppMemEnd = 0x7fffffffffffull;
	437	static const uptr kHiAppMemEnd = 0x800000000000ull;
438	438	static const uptr kShadowMsk = 0x700000000000ull;
439	439	static const uptr kShadowXor = 0x100000000000ull;
440	440	static const uptr kShadowAdd = 0x000000000000ull;
...	...	@@ -738,13 +738,16 @@ struct MappingGoRiscv64_48 {
738	738	Go on linux/s390x
739	739	0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
740	740	1000 0000 0000 - 4000 0000 0000: -
741		4000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app)
742		6000 0000 0000 - 9000 0000 0000: -
743		9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
	741	4000 0000 0000 - 6000 0000 0000: shadow - 32 TiB (2 * app)
	742	6000 0000 0000 - 7000 0000 0000: -
	743	7000 0000 0000 - 7800 0000 0000: metainfo - 8 TiB (0.5 * app)
	744	7800 0000 0000 - 8000 0000 0000: -
744	745	*/
745	746	struct MappingGoS390x {
746		static const uptr kMetaShadowBeg = 0x900000000000ull;
747		static const uptr kMetaShadowEnd = 0x980000000000ull;
	747	// Keep the mapping below 2^47 for QEMU linux-user on x86-64 hosts with
	748	// four-level page tables.
	749	static const uptr kMetaShadowBeg = 0x700000000000ull;
	750	static const uptr kMetaShadowEnd = 0x780000000000ull;
748	751	static const uptr kShadowBeg = 0x400000000000ull;
749	752	static const uptr kShadowEnd = 0x600000000000ull;
750	753	static const uptr kLoAppMemBeg = 0x000000001000ull;

lib/libtsan/tsan_platform_linux.cpp+12

...	...	@@ -27,6 +27,18 @@
27	27	#include "tsan_platform.h"
28	28	#include "tsan_rtl.h"
29	29
	30	#if SANITIZER_NETBSD
	31	# // for __lwp_gettcb_fast() / __lwp_getprivate_fast()
	32	# define _RTLD_SOURCE
	33	# include <sys/types.h>
	34	# include <machine/mcontext.h>
	35	# undef _RTLD_SOURCE
	36	# include <sys/param.h>
	37	# if __NetBSD_Version__ >= 1099001200
	38	# include <machine/lwp_private.h>
	39	# endif
	40	#endif
	41
30	42	#include <fcntl.h>
31	43	#include <pthread.h>
32	44	#include <signal.h>

lib/libtsan/tsan_report.cpp+3-2

...	...	@@ -317,8 +317,9 @@ void PrintReport(const ReportDesc *rep) {
317	317	} else {
318	318	PrintStack(rep->stacks[i]);
319	319	if (i == 0)
320		Printf(" Hint: use TSAN_OPTIONS=second_deadlock_stack=1 "
321		"to get more informative warning message\n\n");
	320	Printf(
	321	" HINT: use TSAN_OPTIONS=second_deadlock_stack=1 "
	322	"to get more informative warning message\n\n");
322	323	}
323	324	}
324	325	} else {

lib/libtsan/tsan_rtl.cpp+5

...	...	@@ -21,6 +21,7 @@
21	21	#include "sanitizer_common/sanitizer_placement_new.h"
22	22	#include "sanitizer_common/sanitizer_stackdepot.h"
23	23	#include "sanitizer_common/sanitizer_symbolizer.h"
	24	#include "tsan_adaptive_delay.h"
24	25	#include "tsan_defs.h"
25	26	#include "tsan_interface.h"
26	27	#include "tsan_mman.h"
...	...	@@ -775,6 +776,10 @@ void Initialize(ThreadState *thr) {
775	776	while (__tsan_resumed == 0) {}
776	777	}
777	778
	779	#if !SANITIZER_GO
	780	AdaptiveDelay::Init();
	781	#endif
	782
778	783	OnInitialize();
779	784	}
780	785

lib/libtsan/tsan_rtl.h+3

...	...	@@ -34,6 +34,7 @@
34	34	#include "sanitizer_common/sanitizer_suppressions.h"
35	35	#include "sanitizer_common/sanitizer_thread_registry.h"
36	36	#include "sanitizer_common/sanitizer_vector.h"
	37	#include "tsan_adaptive_delay.h"
37	38	#include "tsan_defs.h"
38	39	#include "tsan_flags.h"
39	40	#include "tsan_ignoreset.h"
...	...	@@ -240,6 +241,8 @@ struct alignas(SANITIZER_CACHE_LINE_SIZE) ThreadState {
240	241	bool in_internal_write_call;
241	242	#endif
242	243
	244	AdaptiveDelayState adaptive_delay_state;
	245
243	246	explicit ThreadState(Tid tid);
244	247	};
245	248

lib/libtsan/ubsan/ubsan_flags.h+4

...	...	@@ -41,6 +41,10 @@ extern "C" {
41	41	// override the default flag values.
42	42	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
43	43	const char *__ubsan_default_options();
	44	// Users may provide their own implementation of __ubsan_default_suppressions to
	45	// override the default suppression values.
	46	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char *
	47	__ubsan_default_suppressions();
44	48	} // extern "C"
45	49
46	50	#endif // UBSAN_FLAGS_H

src/libs/libtsan.zig+1

...	...	@@ -350,6 +350,7 @@ fn addCcArgs(target: const std.Target, args: std.array_list.Managed([]const u8
350	350	}
351	351
352	352	const tsan_sources = [_][]const u8{
	353	"tsan_adaptive_delay.cpp",
353	354	"tsan_debugging.cpp",
354	355	"tsan_external.cpp",
355	356	"tsan_fd.cpp",