Commit e99c11856df5dafb54e92af12c0858d187dc2ad2 | zig

lib/tsan/builtins/assembly.h+12

...	@@ -290,4 +290,16 @@	...	@@ -290,4 +290,16 @@
290	CFI_END	290	CFI_END
291	#endif	291	#endif
292		292
		293	#ifdef __arm__
		294	#include "int_endianness.h"
		295
		296	#if _YUGA_BIG_ENDIAN
		297	#define VMOV_TO_DOUBLE(dst, src0, src1) vmov dst, src1, src0 SEPARATOR
		298	#define VMOV_FROM_DOUBLE(dst0, dst1, src) vmov dst1, dst0, src SEPARATOR
		299	#else
		300	#define VMOV_TO_DOUBLE(dst, src0, src1) vmov dst, src0, src1 SEPARATOR
		301	#define VMOV_FROM_DOUBLE(dst0, dst1, src) vmov dst0, dst1, src SEPARATOR
		302	#endif
		303	#endif
		304
293	#endif // COMPILERRT_ASSEMBLY_H	305	#endif // COMPILERRT_ASSEMBLY_H

lib/tsan/interception/interception.h+14-11

 @@ -25,8 +25,19 @@
 // These typedefs should be used only in the interceptor definitions to replace
 // the standard system types (e.g. SSIZE_T instead of ssize_t)
-typedef __sanitizer::uptr    SIZE_T;
+// On Windows the system headers (basetsd.h) provide a conflicting definition
-typedef __sanitizer::sptr    SSIZE_T;
+// of SIZE_T/SSIZE_T that do not match the real size_t/ssize_t for 32-bit
+// systems (using long instead of the expected int). Work around the typedef
+// redefinition by #defining SIZE_T instead of using a typedef.
+// TODO: We should be using __sanitizer::usize (and a new ssize) instead of
+// these new macros as long as we ensure they match the real system definitions.
+#if SANITIZER_WINDOWS
+// Ensure that (S)SIZE_T were already defined as we are about to override them.
+#  include <basetsd.h>
+#endif
+#define SIZE_T __sanitizer::usize
+#define SSIZE_T __sanitizer::ssize
 typedef __sanitizer::sptr    PTRDIFF_T;
 typedef __sanitizer::s64     INTMAX_T;
 typedef __sanitizer::u64     UINTMAX_T;
 @@ -338,16 +349,8 @@ const interpose_substitution substitution_##func_name[]             \
 #endif
 // ISO C++ forbids casting between pointer-to-function and pointer-to-object,
-// so we use casting via an integral type __interception::uptr,
+// so we use casts via uintptr_t (the local __sanitizer::uptr equivalent).
-// assuming that system is POSIX-compliant. Using other hacks seem
-// challenging, as we don't even pass function type to
-// INTERCEPT_FUNCTION macro, only its name.
 namespace __interception {
-#if defined(_WIN64)
-typedef unsigned long long uptr;
-#else
-typedef unsigned long uptr;
-#endif  // _WIN64
 #if defined(__ELF__) && !SANITIZER_FUCHSIA
 // The use of interceptors makes many sanitizers unusable for static linking.

lib/tsan/interception/interception_type_test.cpp+19-12

...	@@ -12,28 +12,35 @@	...	@@ -12,28 +12,35 @@
12	//===----------------------------------------------------------------------===//	12	//===----------------------------------------------------------------------===//
13		13
14	#include "interception.h"	14	#include "interception.h"
		15	#include "sanitizer_common/sanitizer_type_traits.h"
15		16
16	#if SANITIZER_LINUX \|\| SANITIZER_APPLE	17	#if __has_include(<sys/types.h>)
17		18	# include <sys/types.h>
18	#include <sys/types.h>	19	#endif
19	#include <stddef.h>	20	#include <stddef.h>
20	#include <stdint.h>	21	#include <stdint.h>
21		22
22	COMPILER_CHECK(sizeof(::SIZE_T) == sizeof(size_t));	23	COMPILER_CHECK((__sanitizer::is_same<__sanitizer::uptr, ::uintptr_t>::value));
23	COMPILER_CHECK(sizeof(::SSIZE_T) == sizeof(ssize_t));	24	COMPILER_CHECK((__sanitizer::is_same<__sanitizer::sptr, ::intptr_t>::value));
24	COMPILER_CHECK(sizeof(::PTRDIFF_T) == sizeof(ptrdiff_t));	25	COMPILER_CHECK((__sanitizer::is_same<__sanitizer::usize, ::size_t>::value));
		26	COMPILER_CHECK((__sanitizer::is_same<::PTRDIFF_T, ::ptrdiff_t>::value));
		27	COMPILER_CHECK((__sanitizer::is_same<::SIZE_T, ::size_t>::value));
		28	#if !SANITIZER_WINDOWS
		29	// No ssize_t on Windows.
		30	COMPILER_CHECK((__sanitizer::is_same<::SSIZE_T, ::ssize_t>::value));
		31	#endif
		32	// TODO: These are not actually the same type on Linux (long vs long long)
25	COMPILER_CHECK(sizeof(::INTMAX_T) == sizeof(intmax_t));	33	COMPILER_CHECK(sizeof(::INTMAX_T) == sizeof(intmax_t));
		34	COMPILER_CHECK(sizeof(::UINTMAX_T) == sizeof(uintmax_t));
26		35
27	# if SANITIZER_GLIBC \|\| SANITIZER_ANDROID	36	#if SANITIZER_GLIBC \|\| SANITIZER_ANDROID
28	COMPILER_CHECK(sizeof(::OFF64_T) == sizeof(off64_t));	37	COMPILER_CHECK(sizeof(::OFF64_T) == sizeof(off64_t));
29	# endif	38	#endif
30		39
31	// The following are the cases when pread (and friends) is used instead of	40	// The following are the cases when pread (and friends) is used instead of
32	// pread64. In those cases we need OFF_T to match off_t. We don't care about the	41	// pread64. In those cases we need OFF_T to match off_t. We don't care about the
33	// rest (they depend on _FILE_OFFSET_BITS setting when building an application).	42	// rest (they depend on _FILE_OFFSET_BITS setting when building an application).
34	# if SANITIZER_ANDROID \|\| !defined _FILE_OFFSET_BITS \|\| \	43	#if !SANITIZER_WINDOWS && (SANITIZER_ANDROID \|\| !defined _FILE_OFFSET_BITS \|\| \
35	_FILE_OFFSET_BITS != 64	44	_FILE_OFFSET_BITS != 64)
36	COMPILER_CHECK(sizeof(::OFF_T) == sizeof(off_t));	45	COMPILER_CHECK(sizeof(::OFF_T) == sizeof(off_t));
37	# endif
38
39	#endif	46	#endif

lib/tsan/interception/interception_win.cpp+364-68

...	@@ -27,7 +27,7 @@	...	@@ -27,7 +27,7 @@
27	//	27	//
28	// 1) Detour	28	// 1) Detour
29	//	29	//
30	// The Detour hooking technique is assuming the presence of an header with	30	// The Detour hooking technique is assuming the presence of a header with
31	// padding and an overridable 2-bytes nop instruction (mov edi, edi). The	31	// padding and an overridable 2-bytes nop instruction (mov edi, edi). The
32	// nop instruction can safely be replaced by a 2-bytes jump without any need	32	// nop instruction can safely be replaced by a 2-bytes jump without any need
33	// to save the instruction. A jump to the target is encoded in the function	33	// to save the instruction. A jump to the target is encoded in the function
...	@@ -47,7 +47,7 @@	...	@@ -47,7 +47,7 @@
47	//	47	//
48	// func: jmp <label> --> func: jmp <hook>	48	// func: jmp <label> --> func: jmp <hook>
49	//	49	//
50	// On an 64-bit architecture, a trampoline is inserted.	50	// On a 64-bit architecture, a trampoline is inserted.
51	//	51	//
52	// func: jmp <label> --> func: jmp <tramp>	52	// func: jmp <label> --> func: jmp <tramp>
53	// [...]	53	// [...]
...	@@ -60,7 +60,7 @@	...	@@ -60,7 +60,7 @@
60	//	60	//
61	// 3) HotPatch	61	// 3) HotPatch
62	//	62	//
63	// The HotPatch hooking is assuming the presence of an header with padding	63	// The HotPatch hooking is assuming the presence of a header with padding
64	// and a first instruction with at least 2-bytes.	64	// and a first instruction with at least 2-bytes.
65	//	65	//
66	// The reason to enforce the 2-bytes limitation is to provide the minimal	66	// The reason to enforce the 2-bytes limitation is to provide the minimal
...	@@ -80,7 +80,7 @@	...	@@ -80,7 +80,7 @@
80	// real: <instr>	80	// real: <instr>
81	// jmp <body>	81	// jmp <body>
82	//	82	//
83	// On an 64-bit architecture:	83	// On a 64-bit architecture:
84	//	84	//
85	// head: 6 x nop head: jmp QWORD [addr1]	85	// head: 6 x nop head: jmp QWORD [addr1]
86	// func: <instr> --> func: jmp short <head>	86	// func: <instr> --> func: jmp short <head>
...	@@ -110,7 +110,7 @@	...	@@ -110,7 +110,7 @@
110	// <instr>	110	// <instr>
111	// jmp <body>	111	// jmp <body>
112	//	112	//
113	// On an 64-bit architecture:	113	// On a 64-bit architecture:
114	//	114	//
115	// func: <instr> --> func: jmp QWORD [addr1]	115	// func: <instr> --> func: jmp QWORD [addr1]
116	// <instr>	116	// <instr>
...	@@ -130,6 +130,7 @@	...	@@ -130,6 +130,7 @@
130	#include "sanitizer_common/sanitizer_platform.h"	130	#include "sanitizer_common/sanitizer_platform.h"
131	#define WIN32_LEAN_AND_MEAN	131	#define WIN32_LEAN_AND_MEAN
132	#include <windows.h>	132	#include <windows.h>
		133	#include <psapi.h>
133		134
134	namespace __interception {	135	namespace __interception {
135		136
...	@@ -186,8 +187,12 @@ static uptr GetMmapGranularity() {	...	@@ -186,8 +187,12 @@ static uptr GetMmapGranularity() {
186	return si.dwAllocationGranularity;	187	return si.dwAllocationGranularity;
187	}	188	}
188		189
		190	UNUSED static uptr RoundDownTo(uptr size, uptr boundary) {
		191	return size & ~(boundary - 1);
		192	}
		193
189	UNUSED static uptr RoundUpTo(uptr size, uptr boundary) {	194	UNUSED static uptr RoundUpTo(uptr size, uptr boundary) {
190	return (size + boundary - 1) & ~(boundary - 1);	195	return RoundDownTo(size + boundary - 1, boundary);
191	}	196	}
192		197
193	// FIXME: internal_str* and internal_mem* functions should be moved from the	198	// FIXME: internal_str* and internal_mem* functions should be moved from the
...	@@ -208,6 +213,18 @@ static char* _strchr(char* str, char c) {	...	@@ -208,6 +213,18 @@ static char* _strchr(char* str, char c) {
208	return nullptr;	213	return nullptr;
209	}	214	}
210		215
		216	static int _strcmp(const char s1, const char s2) {
		217	while (true) {
		218	unsigned c1 = *s1;
		219	unsigned c2 = *s2;
		220	if (c1 != c2) return (c1 < c2) ? -1 : 1;
		221	if (c1 == 0) break;
		222	s1++;
		223	s2++;
		224	}
		225	return 0;
		226	}
		227
211	static void _memset(void *p, int value, size_t sz) {	228	static void _memset(void *p, int value, size_t sz) {
212	for (size_t i = 0; i < sz; ++i)	229	for (size_t i = 0; i < sz; ++i)
213	((char*)p)[i] = (char)value;	230	((char*)p)[i] = (char)value;
...	@@ -284,8 +301,11 @@ static void WriteJumpInstruction(uptr from, uptr target) {	...	@@ -284,8 +301,11 @@ static void WriteJumpInstruction(uptr from, uptr target) {
284		301
285	static void WriteShortJumpInstruction(uptr from, uptr target) {	302	static void WriteShortJumpInstruction(uptr from, uptr target) {
286	sptr offset = target - from - kShortJumpInstructionLength;	303	sptr offset = target - from - kShortJumpInstructionLength;
287	if (offset < -128 \|\| offset > 127)	304	if (offset < -128 \|\| offset > 127) {
		305	ReportError("interception_win: cannot write short jmp from %p to %p\n",
		306	(void )from, (void )target);
288	InterceptionFailed();	307	InterceptionFailed();
		308	}
289	(u8)from = 0xEB;	309	(u8)from = 0xEB;
290	(u8)(from + 1) = (u8)offset;	310	(u8)(from + 1) = (u8)offset;
291	}	311	}
...	@@ -339,32 +359,78 @@ struct TrampolineMemoryRegion {	...	@@ -339,32 +359,78 @@ struct TrampolineMemoryRegion {
339	uptr max_size;	359	uptr max_size;
340	};	360	};
341		361
342	UNUSED static const uptr kTrampolineScanLimitRange = 1ull << 31; // 2 gig	362	UNUSED static const uptr kTrampolineRangeLimit = 1ull << 31; // 2 gig
343	static const int kMaxTrampolineRegion = 1024;	363	static const int kMaxTrampolineRegion = 1024;
344	static TrampolineMemoryRegion TrampolineRegions[kMaxTrampolineRegion];	364	static TrampolineMemoryRegion TrampolineRegions[kMaxTrampolineRegion];
345		365
346	static void *AllocateTrampolineRegion(uptr image_address, size_t granularity) {	366	static void *AllocateTrampolineRegion(uptr min_addr, uptr max_addr,
347	#if SANITIZER_WINDOWS64	367	uptr func_addr, size_t granularity) {
348	uptr address = image_address;	368	# if SANITIZER_WINDOWS64
349	uptr scanned = 0;	369	// Clamp {min,max}_addr to the accessible address space.
350	while (scanned < kTrampolineScanLimitRange) {	370	SYSTEM_INFO system_info;
		371	::GetSystemInfo(&system_info);
		372	uptr min_virtual_addr =
		373	RoundUpTo((uptr)system_info.lpMinimumApplicationAddress, granularity);
		374	uptr max_virtual_addr =
		375	RoundDownTo((uptr)system_info.lpMaximumApplicationAddress, granularity);
		376	if (min_addr < min_virtual_addr)
		377	min_addr = min_virtual_addr;
		378	if (max_addr > max_virtual_addr)
		379	max_addr = max_virtual_addr;
		380
		381	// This loop probes the virtual address space to find free memory in the
		382	// [min_addr, max_addr] interval. The search starts from func_addr and
		383	// proceeds "outwards" towards the interval bounds using two probes, lo_addr
		384	// and hi_addr, for addresses lower/higher than func_addr. At each step, it
		385	// considers the probe closest to func_addr. If that address is not free, the
		386	// probe is advanced (lower or higher depending on the probe) to the next
		387	// memory block and the search continues.
		388	uptr lo_addr = RoundDownTo(func_addr, granularity);
		389	uptr hi_addr = RoundUpTo(func_addr, granularity);
		390	while (lo_addr >= min_addr \|\| hi_addr <= max_addr) {
		391	// Consider the in-range address closest to func_addr.
		392	uptr addr;
		393	if (lo_addr < min_addr)
		394	addr = hi_addr;
		395	else if (hi_addr > max_addr)
		396	addr = lo_addr;
		397	else
		398	addr = (hi_addr - func_addr < func_addr - lo_addr) ? hi_addr : lo_addr;
		399
351	MEMORY_BASIC_INFORMATION info;	400	MEMORY_BASIC_INFORMATION info;
352	if (!::VirtualQuery((void*)address, &info, sizeof(info)))	401	if (!::VirtualQuery((void *)addr, &info, sizeof(info))) {
		402	ReportError(
		403	"interception_win: VirtualQuery in AllocateTrampolineRegion failed "
		404	"for %p\n",
		405	(void *)addr);
353	return nullptr;	406	return nullptr;
		407	}
354		408
355	// Check whether a region can be allocated at \|address\|.	409	// Check whether a region can be allocated at \|addr\|.
356	if (info.State == MEM_FREE && info.RegionSize >= granularity) {	410	if (info.State == MEM_FREE && info.RegionSize >= granularity) {
357	void page = ::VirtualAlloc((void)RoundUpTo(address, granularity),	411	void *page =
358	granularity,	412	::VirtualAlloc((void *)addr, granularity, MEM_RESERVE \| MEM_COMMIT,
359	MEM_RESERVE \| MEM_COMMIT,	413	PAGE_EXECUTE_READWRITE);
360	PAGE_EXECUTE_READWRITE);	414	if (page == nullptr)
		415	ReportError(
		416	"interception_win: VirtualAlloc in AllocateTrampolineRegion failed "
		417	"for %p\n",
		418	(void *)addr);
361	return page;	419	return page;
362	}	420	}
363		421
364	// Move to the next region.	422	if (addr == lo_addr)
365	address = (uptr)info.BaseAddress + info.RegionSize;	423	lo_addr =
366	scanned += info.RegionSize;	424	RoundDownTo((uptr)info.AllocationBase - granularity, granularity);
		425	if (addr == hi_addr)
		426	hi_addr =
		427	RoundUpTo((uptr)info.BaseAddress + info.RegionSize, granularity);
367	}	428	}
		429
		430	ReportError(
		431	"interception_win: AllocateTrampolineRegion failed to find free memory; "
		432	"min_addr: %p, max_addr: %p, func_addr: %p, granularity: %zu\n",
		433	(void )min_addr, (void )max_addr, (void *)func_addr, granularity);
368	return nullptr;	434	return nullptr;
369	#else	435	#else
370	return ::VirtualAlloc(nullptr,	436	return ::VirtualAlloc(nullptr,
...	@@ -385,15 +451,51 @@ void TestOnlyReleaseTrampolineRegions() {	...	@@ -385,15 +451,51 @@ void TestOnlyReleaseTrampolineRegions() {
385	}	451	}
386	}	452	}
387		453
388	static uptr AllocateMemoryForTrampoline(uptr image_address, size_t size) {	454	static uptr AllocateMemoryForTrampoline(uptr func_address, size_t size) {
389	// Find a region within 2G with enough space to allocate \|size\| bytes.	455	# if SANITIZER_WINDOWS64
		456	uptr min_addr = func_address - kTrampolineRangeLimit;
		457	uptr max_addr = func_address + kTrampolineRangeLimit - size;
		458
		459	// Allocate memory within 2GB of the module (DLL or EXE file) so that any
		460	// address within the module can be referenced with PC-relative operands.
		461	// This allows us to not just jump to the trampoline with a PC-relative
		462	// offset, but to relocate any instructions that we copy to the trampoline
		463	// which have references to the original module. If we can't find the base
		464	// address of the module (e.g. if func_address is in mmap'ed memory), just
		465	// stay within 2GB of func_address.
		466	HMODULE module;
		467	if (::GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS \|
		468	GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
		469	(LPCWSTR)func_address, &module)) {
		470	MODULEINFO module_info;
		471	if (::GetModuleInformation(::GetCurrentProcess(), module,
		472	&module_info, sizeof(module_info))) {
		473	min_addr = (uptr)module_info.lpBaseOfDll + module_info.SizeOfImage -
		474	kTrampolineRangeLimit;
		475	max_addr = (uptr)module_info.lpBaseOfDll + kTrampolineRangeLimit - size;
		476	}
		477	}
		478
		479	// Check for overflow.
		480	if (min_addr > func_address)
		481	min_addr = 0;
		482	if (max_addr < func_address)
		483	max_addr = ~(uptr)0;
		484	# else
		485	uptr min_addr = 0;
		486	uptr max_addr = ~min_addr;
		487	# endif
		488
		489	// Find a region within [min_addr,max_addr] with enough space to allocate
		490	// \|size\| bytes.
390	TrampolineMemoryRegion *region = nullptr;	491	TrampolineMemoryRegion *region = nullptr;
391	for (size_t bucket = 0; bucket < kMaxTrampolineRegion; ++bucket) {	492	for (size_t bucket = 0; bucket < kMaxTrampolineRegion; ++bucket) {
392	TrampolineMemoryRegion* current = &TrampolineRegions[bucket];	493	TrampolineMemoryRegion* current = &TrampolineRegions[bucket];
393	if (current->content == 0) {	494	if (current->content == 0) {
394	// No valid region found, allocate a new region.	495	// No valid region found, allocate a new region.
395	size_t bucket_size = GetMmapGranularity();	496	size_t bucket_size = GetMmapGranularity();
396	void *content = AllocateTrampolineRegion(image_address, bucket_size);	497	void *content = AllocateTrampolineRegion(min_addr, max_addr, func_address,
		498	bucket_size);
397	if (content == nullptr)	499	if (content == nullptr)
398	return 0U;	500	return 0U;
399		501
...	@@ -403,13 +505,9 @@ static uptr AllocateMemoryForTrampoline(uptr image_address, size_t size) {	...	@@ -403,13 +505,9 @@ static uptr AllocateMemoryForTrampoline(uptr image_address, size_t size) {
403	region = current;	505	region = current;
404	break;	506	break;
405	} else if (current->max_size - current->allocated_size > size) {	507	} else if (current->max_size - current->allocated_size > size) {
406	#if SANITIZER_WINDOWS64	508	uptr next_address = current->content + current->allocated_size;
407	// In 64-bits, the memory space must be allocated within 2G boundary.	509	if (next_address < min_addr \|\| next_address > max_addr)
408	uptr next_address = current->content + current->allocated_size;	510	continue;
409	if (next_address < image_address \|\|
410	next_address - image_address >= 0x7FFF0000)
411	continue;
412	#endif
413	// The space can be allocated in the current region.	511	// The space can be allocated in the current region.
414	region = current;	512	region = current;
415	break;	513	break;
...	@@ -458,6 +556,10 @@ static const u8 kPrologueWithShortJump2[] = {	...	@@ -458,6 +556,10 @@ static const u8 kPrologueWithShortJump2[] = {
458		556
459	// Returns 0 on error.	557	// Returns 0 on error.
460	static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	558	static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
		559	if (rel_offset) {
		560	*rel_offset = 0;
		561	}
		562
461	#if SANITIZER_ARM64	563	#if SANITIZER_ARM64
462	// An ARM64 instruction is 4 bytes long.	564	// An ARM64 instruction is 4 bytes long.
463	return 4;	565	return 4;
...	@@ -497,8 +599,14 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	...	@@ -497,8 +599,14 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
497	case 0x6A: // 6A XX = push XX	599	case 0x6A: // 6A XX = push XX
498	return 2;	600	return 2;
499		601
		602	// This instruction can be encoded with a 16-bit immediate but that is
		603	// incredibly unlikely.
		604	case 0x68: // 68 XX XX XX XX : push imm32
		605	return 5;
		606
500	case 0xb8: // b8 XX XX XX XX : mov eax, XX XX XX XX	607	case 0xb8: // b8 XX XX XX XX : mov eax, XX XX XX XX
501	case 0xB9: // b9 XX XX XX XX : mov ecx, XX XX XX XX	608	case 0xB9: // b9 XX XX XX XX : mov ecx, XX XX XX XX
		609	case 0xBA: // ba XX XX XX XX : mov edx, XX XX XX XX
502	return 5;	610	return 5;
503		611
504	// Cannot overwrite control-instruction. Return 0 to indicate failure.	612	// Cannot overwrite control-instruction. Return 0 to indicate failure.
...	@@ -529,19 +637,43 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	...	@@ -529,19 +637,43 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
529	case 0xFF8B: // 8B FF : mov edi, edi	637	case 0xFF8B: // 8B FF : mov edi, edi
530	case 0xEC8B: // 8B EC : mov ebp, esp	638	case 0xEC8B: // 8B EC : mov ebp, esp
531	case 0xc889: // 89 C8 : mov eax, ecx	639	case 0xc889: // 89 C8 : mov eax, ecx
		640	case 0xD189: // 89 D1 : mov ecx, edx
532	case 0xE589: // 89 E5 : mov ebp, esp	641	case 0xE589: // 89 E5 : mov ebp, esp
533	case 0xC18B: // 8B C1 : mov eax, ecx	642	case 0xC18B: // 8B C1 : mov eax, ecx
		643	case 0xC031: // 31 C0 : xor eax, eax
		644	case 0xC931: // 31 C9 : xor ecx, ecx
		645	case 0xD231: // 31 D2 : xor edx, edx
534	case 0xC033: // 33 C0 : xor eax, eax	646	case 0xC033: // 33 C0 : xor eax, eax
535	case 0xC933: // 33 C9 : xor ecx, ecx	647	case 0xC933: // 33 C9 : xor ecx, ecx
536	case 0xD233: // 33 D2 : xor edx, edx	648	case 0xD233: // 33 D2 : xor edx, edx
		649	case 0xDB84: // 84 DB : test bl,bl
		650	case 0xC084: // 84 C0 : test al,al
		651	case 0xC984: // 84 C9 : test cl,cl
		652	case 0xD284: // 84 D2 : test dl,dl
537	return 2;	653	return 2;
538		654
		655	case 0x3980: // 80 39 XX : cmp BYTE PTR [rcx], XX
		656	case 0x4D8B: // 8B 4D XX : mov XX(%ebp), ecx
		657	case 0x558B: // 8B 55 XX : mov XX(%ebp), edx
		658	case 0x758B: // 8B 75 XX : mov XX(%ebp), esp
		659	case 0xE483: // 83 E4 XX : and esp, XX
		660	case 0xEC83: // 83 EC XX : sub esp, XX
		661	case 0xC1F6: // F6 C1 XX : test cl, XX
		662	return 3;
		663
		664	case 0x89FF: // FF 89 XX XX XX XX : dec dword ptr [ecx + XX XX XX XX]
		665	case 0xEC81: // 81 EC XX XX XX XX : sub esp, XX XX XX XX
		666	return 6;
		667
539	// Cannot overwrite control-instruction. Return 0 to indicate failure.	668	// Cannot overwrite control-instruction. Return 0 to indicate failure.
540	case 0x25FF: // FF 25 XX XX XX XX : jmp [XXXXXXXX]	669	case 0x25FF: // FF 25 XX YY ZZ WW : jmp dword ptr ds:[WWZZYYXX]
541	return 0;	670	return 0;
542	}	671	}
543		672
544	switch (0x00FFFFFF & (u32)address) {	673	switch (0x00FFFFFF & (u32 )address) {
		674	case 0x244C8D: // 8D 4C 24 XX : lea ecx, [esp + XX]
		675	case 0x2474FF: // FF 74 24 XX : push qword ptr [rsp + XX]
		676	return 4;
545	case 0x24A48D: // 8D A4 24 XX XX XX XX : lea esp, [esp + XX XX XX XX]	677	case 0x24A48D: // 8D A4 24 XX XX XX XX : lea esp, [esp + XX XX XX XX]
546	return 7;	678	return 7;
547	}	679	}
...	@@ -556,6 +688,21 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	...	@@ -556,6 +688,21 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
556	case 0xA1: // A1 XX XX XX XX XX XX XX XX :	688	case 0xA1: // A1 XX XX XX XX XX XX XX XX :
557	// movabs eax, dword ptr ds:[XXXXXXXX]	689	// movabs eax, dword ptr ds:[XXXXXXXX]
558	return 9;	690	return 9;
		691	case 0xF2:
		692	switch ((u32 )(address + 1)) {
		693	case 0x2444110f: // f2 0f 11 44 24 XX movsd QWORD PTR
		694	// [rsp + XX], xmm0
		695	case 0x244c110f: // f2 0f 11 4c 24 XX movsd QWORD PTR
		696	// [rsp + XX], xmm1
		697	case 0x2454110f: // f2 0f 11 54 24 XX movsd QWORD PTR
		698	// [rsp + XX], xmm2
		699	case 0x245c110f: // f2 0f 11 5c 24 XX movsd QWORD PTR
		700	// [rsp + XX], xmm3
		701	case 0x2464110f: // f2 0f 11 64 24 XX movsd QWORD PTR
		702	// [rsp + XX], xmm4
		703	return 6;
		704	}
		705	break;
559		706
560	case 0x83:	707	case 0x83:
561	const u8 next_byte = (u8)(address + 1);	708	const u8 next_byte = (u8)(address + 1);
...	@@ -584,55 +731,155 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	...	@@ -584,55 +731,155 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
584	case 0x018a: // mov al, byte ptr [rcx]	731	case 0x018a: // mov al, byte ptr [rcx]
585	return 2;	732	return 2;
586		733
		734	case 0x7E80: // 80 7E YY XX cmp BYTE PTR [rsi+YY], XX
		735	case 0x7D80: // 80 7D YY XX cmp BYTE PTR [rbp+YY], XX
		736	case 0x7A80: // 80 7A YY XX cmp BYTE PTR [rdx+YY], XX
		737	case 0x7880: // 80 78 YY XX cmp BYTE PTR [rax+YY], XX
		738	case 0x7B80: // 80 7B YY XX cmp BYTE PTR [rbx+YY], XX
		739	case 0x7980: // 80 79 YY XX cmp BYTE ptr [rcx+YY], XX
		740	return 4;
		741
587	case 0x058A: // 8A 05 XX XX XX XX : mov al, byte ptr [XX XX XX XX]	742	case 0x058A: // 8A 05 XX XX XX XX : mov al, byte ptr [XX XX XX XX]
588	case 0x058B: // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX]	743	case 0x058B: // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX]
589	if (rel_offset)	744	if (rel_offset)
590	*rel_offset = 2;	745	*rel_offset = 2;
		746	case 0xB841: // 41 B8 XX XX XX XX : mov r8d, XX XX XX XX
591	return 6;	747	return 6;
		748
		749	case 0x7E81: // 81 7E YY XX XX XX XX cmp DWORD PTR [rsi+YY], XX XX XX XX
		750	case 0x7D81: // 81 7D YY XX XX XX XX cmp DWORD PTR [rbp+YY], XX XX XX XX
		751	case 0x7A81: // 81 7A YY XX XX XX XX cmp DWORD PTR [rdx+YY], XX XX XX XX
		752	case 0x7881: // 81 78 YY XX XX XX XX cmp DWORD PTR [rax+YY], XX XX XX XX
		753	case 0x7B81: // 81 7B YY XX XX XX XX cmp DWORD PTR [rbx+YY], XX XX XX XX
		754	case 0x7981: // 81 79 YY XX XX XX XX cmp dword ptr [rcx+YY], XX XX XX XX
		755	return 7;
592	}	756	}
593		757
594	switch (0x00FFFFFF & (u32)address) {	758	switch (0x00FFFFFF & (u32 )address) {
595	case 0xe58948: // 48 8b c4 : mov rbp, rsp	759	case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax]
596	case 0xc18b48: // 48 8b c1 : mov rax, rcx	760	case 0xc00b4d: // 4d 0b c0 : or r8, r8
597	case 0xc48b48: // 48 8b c4 : mov rax, rsp	761	case 0xc03345: // 45 33 c0 : xor r8d, r8d
598	case 0xd9f748: // 48 f7 d9 : neg rcx	762	case 0xc08548: // 48 85 c0 : test rax, rax
599	case 0xd12b48: // 48 2b d1 : sub rdx, rcx
600	case 0x07c1f6: // f6 c1 07 : test cl, 0x7
601	case 0xc98548: // 48 85 C9 : test rcx, rcx
602	case 0xd28548: // 48 85 d2 : test rdx, rdx
603	case 0xc0854d: // 4d 85 c0 : test r8, r8	763	case 0xc0854d: // 4d 85 c0 : test r8, r8
		764	case 0xc08b41: // 41 8b c0 : mov eax, r8d
		765	case 0xc0ff48: // 48 ff c0 : inc rax
		766	case 0xc0ff49: // 49 ff c0 : inc r8
		767	case 0xc18b41: // 41 8b c1 : mov eax, r9d
		768	case 0xc18b48: // 48 8b c1 : mov rax, rcx
		769	case 0xc18b4c: // 4c 8b c1 : mov r8, rcx
		770	case 0xc1ff48: // 48 ff c1 : inc rcx
		771	case 0xc1ff49: // 49 ff c1 : inc r9
		772	case 0xc28b41: // 41 8b c2 : mov eax, r10d
		773	case 0x01b60f: // 0f b6 01 : movzx eax, BYTE PTR [rcx]
		774	case 0x09b60f: // 0f b6 09 : movzx ecx, BYTE PTR [rcx]
		775	case 0x11b60f: // 0f b6 11 : movzx edx, BYTE PTR [rcx]
604	case 0xc2b60f: // 0f b6 c2 : movzx eax, dl	776	case 0xc2b60f: // 0f b6 c2 : movzx eax, dl
605	case 0xc03345: // 45 33 c0 : xor r8d, r8d	777	case 0xc2ff48: // 48 ff c2 : inc rdx
		778	case 0xc2ff49: // 49 ff c2 : inc r10
		779	case 0xc38b41: // 41 8b c3 : mov eax, r11d
		780	case 0xc3ff48: // 48 ff c3 : inc rbx
		781	case 0xc3ff49: // 49 ff c3 : inc r11
		782	case 0xc48b41: // 41 8b c4 : mov eax, r12d
		783	case 0xc48b48: // 48 8b c4 : mov rax, rsp
		784	case 0xc4ff49: // 49 ff c4 : inc r12
		785	case 0xc5ff49: // 49 ff c5 : inc r13
		786	case 0xc6ff48: // 48 ff c6 : inc rsi
		787	case 0xc6ff49: // 49 ff c6 : inc r14
		788	case 0xc7ff48: // 48 ff c7 : inc rdi
		789	case 0xc7ff49: // 49 ff c7 : inc r15
606	case 0xc93345: // 45 33 c9 : xor r9d, r9d	790	case 0xc93345: // 45 33 c9 : xor r9d, r9d
607	case 0xdb3345: // 45 33 DB : xor r11d, r11d	791	case 0xc98548: // 48 85 c9 : test rcx, rcx
608	case 0xd98b4c: // 4c 8b d9 : mov r11, rcx	792	case 0xc9854d: // 4d 85 c9 : test r9, r9
609	case 0xd28b4c: // 4c 8b d2 : mov r10, rdx	793	case 0xc98b4c: // 4c 8b c9 : mov r9, rcx
610	case 0xc98b4c: // 4C 8B C9 : mov r9, rcx	794	case 0xd12948: // 48 29 d1 : sub rcx, rdx
611	case 0xc18b4c: // 4C 8B C1 : mov r8, rcx
612	case 0xd2b60f: // 0f b6 d2 : movzx edx, dl
613	case 0xca2b48: // 48 2b ca : sub rcx, rdx	795	case 0xca2b48: // 48 2b ca : sub rcx, rdx
614	case 0xca3b48: // 48 3b ca : cmp rcx, rdx	796	case 0xca3b48: // 48 3b ca : cmp rcx, rdx
615	case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax]	797	case 0xd12b48: // 48 2b d1 : sub rdx, rcx
616	case 0xc00b4d: // 3d 0b c0 : or r8, r8
617	case 0xc08b41: // 41 8b c0 : mov eax, r8d
618	case 0xd18b48: // 48 8b d1 : mov rdx, rcx	798	case 0xd18b48: // 48 8b d1 : mov rdx, rcx
619	case 0xdc8b4c: // 4c 8b dc : mov r11, rsp
620	case 0xd18b4c: // 4c 8b d1 : mov r10, rcx	799	case 0xd18b4c: // 4c 8b d1 : mov r10, rcx
621	case 0xE0E483: // 83 E4 E0 : and esp, 0xFFFFFFE0	800	case 0xd28548: // 48 85 d2 : test rdx, rdx
		801	case 0xd2854d: // 4d 85 d2 : test r10, r10
		802	case 0xd28b4c: // 4c 8b d2 : mov r10, rdx
		803	case 0xd2b60f: // 0f b6 d2 : movzx edx, dl
		804	case 0xd2be0f: // 0f be d2 : movsx edx, dl
		805	case 0xd98b4c: // 4c 8b d9 : mov r11, rcx
		806	case 0xd9f748: // 48 f7 d9 : neg rcx
		807	case 0xc03145: // 45 31 c0 : xor r8d,r8d
		808	case 0xc93145: // 45 31 c9 : xor r9d,r9d
		809	case 0xdb3345: // 45 33 db : xor r11d, r11d
		810	case 0xc08445: // 45 84 c0 : test r8b,r8b
		811	case 0xd28445: // 45 84 d2 : test r10b,r10b
		812	case 0xdb8548: // 48 85 db : test rbx, rbx
		813	case 0xdb854d: // 4d 85 db : test r11, r11
		814	case 0xdc8b4c: // 4c 8b dc : mov r11, rsp
		815	case 0xe48548: // 48 85 e4 : test rsp, rsp
		816	case 0xe4854d: // 4d 85 e4 : test r12, r12
		817	case 0xc88948: // 48 89 c8 : mov rax,rcx
		818	case 0xcb8948: // 48 89 cb : mov rbx,rcx
		819	case 0xd08948: // 48 89 d0 : mov rax,rdx
		820	case 0xd18948: // 48 89 d1 : mov rcx,rdx
		821	case 0xd38948: // 48 89 d3 : mov rbx,rdx
		822	case 0xe58948: // 48 89 e5 : mov rbp, rsp
		823	case 0xed8548: // 48 85 ed : test rbp, rbp
		824	case 0xc88949: // 49 89 c8 : mov r8, rcx
		825	case 0xc98949: // 49 89 c9 : mov r9, rcx
		826	case 0xca8949: // 49 89 ca : mov r10,rcx
		827	case 0xd08949: // 49 89 d0 : mov r8, rdx
		828	case 0xd18949: // 49 89 d1 : mov r9, rdx
		829	case 0xd28949: // 49 89 d2 : mov r10, rdx
		830	case 0xd38949: // 49 89 d3 : mov r11, rdx
		831	case 0xed854d: // 4d 85 ed : test r13, r13
		832	case 0xf6854d: // 4d 85 f6 : test r14, r14
		833	case 0xff854d: // 4d 85 ff : test r15, r15
622	return 3;	834	return 3;
623		835
		836	case 0x245489: // 89 54 24 XX : mov DWORD PTR[rsp + XX], edx
		837	case 0x428d44: // 44 8d 42 XX : lea r8d , [rdx + XX]
		838	case 0x588948: // 48 89 58 XX : mov QWORD PTR[rax + XX], rbx
624	case 0xec8348: // 48 83 ec XX : sub rsp, XX	839	case 0xec8348: // 48 83 ec XX : sub rsp, XX
625	case 0xf88349: // 49 83 f8 XX : cmp r8, XX	840	case 0xf88349: // 49 83 f8 XX : cmp r8, XX
626	case 0x588948: // 48 89 58 XX : mov QWORD PTR[rax + XX], rbx	841	case 0x488d49: // 49 8d 48 XX : lea rcx, [...]
		842	case 0x048d4c: // 4c 8d 04 XX : lea r8, [...]
		843	case 0x148d4e: // 4e 8d 14 XX : lea r10, [...]
		844	case 0x398366: // 66 83 39 XX : cmp WORD PTR [rcx], XX
627	return 4;	845	return 4;
628		846
		847	case 0x441F0F: // 0F 1F 44 XX XX : nop DWORD PTR [...]
		848	case 0x246483: // 83 64 24 XX YY : and DWORD PTR [rsp+XX], YY
		849	return 5;
		850
		851	case 0x788166: // 66 81 78 XX YY YY cmp WORD PTR [rax+XX], YY YY
		852	case 0x798166: // 66 81 79 XX YY YY cmp WORD PTR [rcx+XX], YY YY
		853	case 0x7a8166: // 66 81 7a XX YY YY cmp WORD PTR [rdx+XX], YY YY
		854	case 0x7b8166: // 66 81 7b XX YY YY cmp WORD PTR [rbx+XX], YY YY
		855	case 0x7e8166: // 66 81 7e XX YY YY cmp WORD PTR [rsi+XX], YY YY
		856	case 0x7f8166: // 66 81 7f XX YY YY cmp WORD PTR [rdi+XX], YY YY
		857	return 6;
		858
629	case 0xec8148: // 48 81 EC XX XX XX XX : sub rsp, XXXXXXXX	859	case 0xec8148: // 48 81 EC XX XX XX XX : sub rsp, XXXXXXXX
		860	case 0xc0c748: // 48 C7 C0 XX XX XX XX : mov rax, XX XX XX XX
630	return 7;	861	return 7;
631		862
		863	// clang-format off
		864	case 0x788141: // 41 81 78 XX YY YY YY YY : cmp DWORD PTR [r8+YY], XX XX XX XX
		865	case 0x798141: // 41 81 79 XX YY YY YY YY : cmp DWORD PTR [r9+YY], XX XX XX XX
		866	case 0x7a8141: // 41 81 7a XX YY YY YY YY : cmp DWORD PTR [r10+YY], XX XX XX XX
		867	case 0x7b8141: // 41 81 7b XX YY YY YY YY : cmp DWORD PTR [r11+YY], XX XX XX XX
		868	case 0x7d8141: // 41 81 7d XX YY YY YY YY : cmp DWORD PTR [r13+YY], XX XX XX XX
		869	case 0x7e8141: // 41 81 7e XX YY YY YY YY : cmp DWORD PTR [r14+YY], XX XX XX XX
		870	case 0x7f8141: // 41 81 7f YY XX XX XX XX : cmp DWORD PTR [r15+YY], XX XX XX XX
		871	case 0x247c81: // 81 7c 24 YY XX XX XX XX : cmp DWORD PTR [rsp+YY], XX XX XX XX
		872	return 8;
		873	// clang-format on
		874
632	case 0x058b48: // 48 8b 05 XX XX XX XX :	875	case 0x058b48: // 48 8b 05 XX XX XX XX :
633	// mov rax, QWORD PTR [rip + XXXXXXXX]	876	// mov rax, QWORD PTR [rip + XXXXXXXX]
634	case 0x058d48: // 48 8d 05 XX XX XX XX :	877	case 0x058d48: // 48 8d 05 XX XX XX XX :
635	// lea rax, QWORD PTR [rip + XXXXXXXX]	878	// lea rax, QWORD PTR [rip + XXXXXXXX]
		879	case 0x0d8948: // 48 89 0d XX XX XX XX :
		880	// mov QWORD PTR [rip + XXXXXXXX], rcx
		881	case 0x158948: // 48 89 15 XX XX XX XX :
		882	// mov QWORD PTR [rip + XXXXXXXX], rdx
636	case 0x25ff48: // 48 ff 25 XX XX XX XX :	883	case 0x25ff48: // 48 ff 25 XX XX XX XX :
637	// rex.W jmp QWORD PTR [rip + XXXXXXXX]	884	// rex.W jmp QWORD PTR [rip + XXXXXXXX]
638	case 0x158D4C: // 4c 8d 15 XX XX XX XX : lea r10, [rip + XX]	885	case 0x158D4C: // 4c 8d 15 XX XX XX XX : lea r10, [rip + XX]
...	@@ -644,9 +891,19 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	...	@@ -644,9 +891,19 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
644	case 0x2444c7: // C7 44 24 XX YY YY YY YY	891	case 0x2444c7: // C7 44 24 XX YY YY YY YY
645	// mov dword ptr [rsp + XX], YYYYYYYY	892	// mov dword ptr [rsp + XX], YYYYYYYY
646	return 8;	893	return 8;
		894
		895	case 0x7c8141: // 41 81 7c ZZ YY XX XX XX XX
		896	// cmp DWORD PTR [reg+reg*n+YY], XX XX XX XX
		897	return 9;
647	}	898	}
648		899
649	switch ((u32)(address)) {	900	switch ((u32)(address)) {
		901	case 0x01b60f44: // 44 0f b6 01 : movzx r8d, BYTE PTR [rcx]
		902	case 0x09b60f44: // 44 0f b6 09 : movzx r9d, BYTE PTR [rcx]
		903	case 0x0ab60f44: // 44 0f b6 0a : movzx r8d, BYTE PTR [rdx]
		904	case 0x11b60f44: // 44 0f b6 11 : movzx r10d, BYTE PTR [rcx]
		905	case 0x1ab60f44: // 44 0f b6 1a : movzx r11d, BYTE PTR [rdx]
		906	return 4;
650	case 0x24448b48: // 48 8b 44 24 XX : mov rax, QWORD ptr [rsp + XX]	907	case 0x24448b48: // 48 8b 44 24 XX : mov rax, QWORD ptr [rsp + XX]
651	case 0x246c8948: // 48 89 6C 24 XX : mov QWORD ptr [rsp + XX], rbp	908	case 0x246c8948: // 48 89 6C 24 XX : mov QWORD ptr [rsp + XX], rbp
652	case 0x245c8948: // 48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx	909	case 0x245c8948: // 48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx
...	@@ -656,9 +913,19 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	...	@@ -656,9 +913,19 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
656	case 0x24548948: // 48 89 54 24 XX : mov QWORD PTR [rsp + XX], rdx	913	case 0x24548948: // 48 89 54 24 XX : mov QWORD PTR [rsp + XX], rdx
657	case 0x244c894c: // 4c 89 4c 24 XX : mov QWORD PTR [rsp + XX], r9	914	case 0x244c894c: // 4c 89 4c 24 XX : mov QWORD PTR [rsp + XX], r9
658	case 0x2444894c: // 4c 89 44 24 XX : mov QWORD PTR [rsp + XX], r8	915	case 0x2444894c: // 4c 89 44 24 XX : mov QWORD PTR [rsp + XX], r8
		916	case 0x244c8944: // 44 89 4c 24 XX mov DWORD PTR [rsp + XX], r9d
		917	case 0x24448944: // 44 89 44 24 XX mov DWORD PTR [rsp + XX], r8d
		918	case 0x246c8d48: // 48 8d 6c 24 XX : lea rbp, [rsp + XX]
659	return 5;	919	return 5;
660	case 0x24648348: // 48 83 64 24 XX : and QWORD PTR [rsp + XX], YY	920	case 0x24648348: // 48 83 64 24 XX YY : and QWORD PTR [rsp + XX], YY
661	return 6;	921	return 6;
		922	case 0x24A48D48: // 48 8D A4 24 XX XX XX XX : lea rsp, [rsp + XX XX XX XX]
		923	return 8;
		924	}
		925
		926	switch (0xFFFFFFFFFFULL & (u64 )(address)) {
		927	case 0xC07E0F4866: // 66 48 0F 7E C0 : movq rax, xmm0
		928	return 5;
662	}	929	}
663		930
664	#else	931	#else
...	@@ -671,11 +938,10 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	...	@@ -671,11 +938,10 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
671	case 0x458B: // 8B 45 XX : mov eax, dword ptr [ebp + XX]	938	case 0x458B: // 8B 45 XX : mov eax, dword ptr [ebp + XX]
672	case 0x5D8B: // 8B 5D XX : mov ebx, dword ptr [ebp + XX]	939	case 0x5D8B: // 8B 5D XX : mov ebx, dword ptr [ebp + XX]
673	case 0x7D8B: // 8B 7D XX : mov edi, dword ptr [ebp + XX]	940	case 0x7D8B: // 8B 7D XX : mov edi, dword ptr [ebp + XX]
674	case 0xEC83: // 83 EC XX : sub esp, XX	941	case 0x758B: // 8B 75 XX : mov esi, dword ptr [ebp + XX]
675	case 0x75FF: // FF 75 XX : push dword ptr [ebp + XX]	942	case 0x75FF: // FF 75 XX : push dword ptr [ebp + XX]
676	return 3;	943	return 3;
677	case 0xC1F7: // F7 C1 XX YY ZZ WW : test ecx, WWZZYYXX	944	case 0xC1F7: // F7 C1 XX YY ZZ WW : test ecx, WWZZYYXX
678	case 0x25FF: // FF 25 XX YY ZZ WW : jmp dword ptr ds:[WWZZYYXX]
679	return 6;	945	return 6;
680	case 0x3D83: // 83 3D XX YY ZZ WW TT : cmp TT, WWZZYYXX	946	case 0x3D83: // 83 3D XX YY ZZ WW TT : cmp TT, WWZZYYXX
681	return 7;	947	return 7;
...	@@ -718,6 +984,10 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {	...	@@ -718,6 +984,10 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
718	return 0;	984	return 0;
719	}	985	}
720		986
		987	size_t TestOnlyGetInstructionSize(uptr address, size_t *rel_offset) {
		988	return GetInstructionSize(address, rel_offset);
		989	}
		990
721	// Returns 0 on error.	991	// Returns 0 on error.
722	static size_t RoundUpToInstrBoundary(size_t size, uptr address) {	992	static size_t RoundUpToInstrBoundary(size_t size, uptr address) {
723	size_t cursor = 0;	993	size_t cursor = 0;
...	@@ -745,8 +1015,14 @@ static bool CopyInstructions(uptr to, uptr from, size_t size) {	...	@@ -745,8 +1015,14 @@ static bool CopyInstructions(uptr to, uptr from, size_t size) {
745	// this will be untrue if relocated_offset \notin [-231, 231)	1015	// this will be untrue if relocated_offset \notin [-231, 231)
746	s64 delta = to - from;	1016	s64 delta = to - from;
747	s64 relocated_offset = (s32 )(to + cursor + rel_offset) - delta;	1017	s64 relocated_offset = (s32 )(to + cursor + rel_offset) - delta;
748	if (-0x8000'0000ll > relocated_offset \|\| relocated_offset > 0x7FFF'FFFFll)	1018	if (-0x8000'0000ll > relocated_offset \|\|
		1019	relocated_offset > 0x7FFF'FFFFll) {
		1020	ReportError(
		1021	"interception_win: CopyInstructions relocated_offset %lld outside "
		1022	"32-bit range\n",
		1023	(long long)relocated_offset);
749	return false;	1024	return false;
		1025	}
750	# else	1026	# else
751	// on 32-bit, the relative offset will always be correct	1027	// on 32-bit, the relative offset will always be correct
752	s32 delta = to - from;	1028	s32 delta = to - from;
...	@@ -969,8 +1245,7 @@ static void **InterestingDLLsAvailable() {	...	@@ -969,8 +1245,7 @@ static void **InterestingDLLsAvailable() {
969	"libc++.dll", // libc++	1245	"libc++.dll", // libc++
970	"libunwind.dll", // libunwind	1246	"libunwind.dll", // libunwind
971	# endif	1247	# endif
972	// NTDLL should go last as it exports some functions that we should	1248	// NTDLL must go last as it gets special treatment in OverrideFunction.
973	// override in the CRT [presumably only used internally].
974	"ntdll.dll",	1249	"ntdll.dll",
975	NULL	1250	NULL
976	};	1251	};
...	@@ -1027,7 +1302,7 @@ uptr InternalGetProcAddress(void module, const char func_name) {	...	@@ -1027,7 +1302,7 @@ uptr InternalGetProcAddress(void module, const char func_name) {
1027		1302
1028	for (DWORD i = 0; i < exports->NumberOfNames; i++) {	1303	for (DWORD i = 0; i < exports->NumberOfNames; i++) {
1029	RVAPtr<char> name(module, names[i]);	1304	RVAPtr<char> name(module, names[i]);
1030	if (!strcmp(func_name, name)) {	1305	if (!_strcmp(func_name, name)) {
1031	DWORD index = ordinals[i];	1306	DWORD index = ordinals[i];
1032	RVAPtr<char> func(module, functions[index]);	1307	RVAPtr<char> func(module, functions[index]);
1033		1308
...	@@ -1040,19 +1315,27 @@ uptr InternalGetProcAddress(void module, const char func_name) {	...	@@ -1040,19 +1315,27 @@ uptr InternalGetProcAddress(void module, const char func_name) {
1040	// exported directory.	1315	// exported directory.
1041	char function_name[256];	1316	char function_name[256];
1042	size_t funtion_name_length = _strlen(func);	1317	size_t funtion_name_length = _strlen(func);
1043	if (funtion_name_length >= sizeof(function_name) - 1)	1318	if (funtion_name_length >= sizeof(function_name) - 1) {
		1319	ReportError("interception_win: func too long: '%s'\n", (char *)func);
1044	InterceptionFailed();	1320	InterceptionFailed();
		1321	}
1045		1322
1046	_memcpy(function_name, func, funtion_name_length);	1323	_memcpy(function_name, func, funtion_name_length);
1047	function_name[funtion_name_length] = '\0';	1324	function_name[funtion_name_length] = '\0';
1048	char* separator = _strchr(function_name, '.');	1325	char* separator = _strchr(function_name, '.');
1049	if (!separator)	1326	if (!separator) {
		1327	ReportError("interception_win: no separator in '%s'\n",
		1328	function_name);
1050	InterceptionFailed();	1329	InterceptionFailed();
		1330	}
1051	*separator = '\0';	1331	*separator = '\0';
1052		1332
1053	void* redirected_module = GetModuleHandleA(function_name);	1333	void* redirected_module = GetModuleHandleA(function_name);
1054	if (!redirected_module)	1334	if (!redirected_module) {
		1335	ReportError("interception_win: GetModuleHandleA failed for '%s'\n",
		1336	function_name);
1055	InterceptionFailed();	1337	InterceptionFailed();
		1338	}
1056	return InternalGetProcAddress(redirected_module, separator + 1);	1339	return InternalGetProcAddress(redirected_module, separator + 1);
1057	}	1340	}
1058		1341
...	@@ -1065,9 +1348,22 @@ uptr InternalGetProcAddress(void module, const char func_name) {	...	@@ -1065,9 +1348,22 @@ uptr InternalGetProcAddress(void module, const char func_name) {
1065		1348
1066	bool OverrideFunction(	1349	bool OverrideFunction(
1067	const char func_name, uptr new_func, uptr orig_old_func) {	1350	const char func_name, uptr new_func, uptr orig_old_func) {
		1351	static const char *kNtDllIgnore[] = {
		1352	"memcmp", "memcpy", "memmove", "memset"
		1353	};
		1354
1068	bool hooked = false;	1355	bool hooked = false;
1069	void **DLLs = InterestingDLLsAvailable();	1356	void **DLLs = InterestingDLLsAvailable();
1070	for (size_t i = 0; DLLs[i]; ++i) {	1357	for (size_t i = 0; DLLs[i]; ++i) {
		1358	if (DLLs[i + 1] == nullptr) {
		1359	// This is the last DLL, i.e. NTDLL. It exports some functions that
		1360	// we only want to override in the CRT.
		1361	for (const char *ignored : kNtDllIgnore) {
		1362	if (_strcmp(func_name, ignored) == 0)
		1363	return hooked;
		1364	}
		1365	}
		1366
1071	uptr func_addr = InternalGetProcAddress(DLLs[i], func_name);	1367	uptr func_addr = InternalGetProcAddress(DLLs[i], func_name);
1072	if (func_addr &&	1368	if (func_addr &&
1073	OverrideFunction(func_addr, new_func, orig_old_func)) {	1369	OverrideFunction(func_addr, new_func, orig_old_func)) {
...	@@ -1121,7 +1417,7 @@ bool OverrideImportedFunction(const char *module_to_patch,	...	@@ -1121,7 +1417,7 @@ bool OverrideImportedFunction(const char *module_to_patch,
1121	RVAPtr<IMAGE_IMPORT_BY_NAME> import_by_name(	1417	RVAPtr<IMAGE_IMPORT_BY_NAME> import_by_name(
1122	module, name_table->u1.ForwarderString);	1418	module, name_table->u1.ForwarderString);
1123	const char *funcname = &import_by_name->Name[0];	1419	const char *funcname = &import_by_name->Name[0];
1124	if (strcmp(funcname, function_name) == 0)	1420	if (_strcmp(funcname, function_name) == 0)
1125	break;	1421	break;
1126	}	1422	}
1127	}	1423	}
...	@@ -1144,4 +1440,4 @@ bool OverrideImportedFunction(const char *module_to_patch,	...	@@ -1144,4 +1440,4 @@ bool OverrideImportedFunction(const char *module_to_patch,
1144		1440
1145	} // namespace __interception	1441	} // namespace __interception
1146		1442
1147	#endif // SANITIZER_APPLE	1443	#endif // SANITIZER_WINDOWS

lib/tsan/interception/interception_win.h+3

...	@@ -63,6 +63,9 @@ bool OverrideFunctionWithTrampoline(	...	@@ -63,6 +63,9 @@ bool OverrideFunctionWithTrampoline(
63	// Exposed for unittests	63	// Exposed for unittests
64	void TestOnlyReleaseTrampolineRegions();	64	void TestOnlyReleaseTrampolineRegions();
65		65
		66	// Exposed for unittests
		67	SIZE_T TestOnlyGetInstructionSize(uptr address, SIZE_T *rel_offset);
		68
66	} // namespace __interception	69	} // namespace __interception
67		70
68	#if defined(INTERCEPTION_DYNAMIC_CRT)	71	#if defined(INTERCEPTION_DYNAMIC_CRT)

lib/tsan/sanitizer_common/sanitizer_allocator.cpp+3-2

...	@@ -59,7 +59,7 @@ static void RawInternalAlloc(uptr size, InternalAllocatorCache cache,	...	@@ -59,7 +59,7 @@ static void RawInternalAlloc(uptr size, InternalAllocatorCache cache,
59		59
60	static void RawInternalRealloc(void ptr, uptr size,	60	static void RawInternalRealloc(void ptr, uptr size,
61	InternalAllocatorCache *cache) {	61	InternalAllocatorCache *cache) {
62	uptr alignment = 8;	62	constexpr usize alignment = Max<usize>(8, sizeof(void *));
63	if (cache == 0) {	63	if (cache == 0) {
64	SpinMutexLock l(&internal_allocator_cache_mu);	64	SpinMutexLock l(&internal_allocator_cache_mu);
65	return internal_allocator()->Reallocate(&internal_allocator_cache, ptr,	65	return internal_allocator()->Reallocate(&internal_allocator_cache, ptr,
...	@@ -137,7 +137,8 @@ void InternalAllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {	...	@@ -137,7 +137,8 @@ void InternalAllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
137	}	137	}
138		138
139	// LowLevelAllocator	139	// LowLevelAllocator
140	constexpr uptr kLowLevelAllocatorDefaultAlignment = 8;	140	constexpr usize kLowLevelAllocatorDefaultAlignment =
		141	Max<usize>(8, sizeof(void *));
141	constexpr uptr kMinNumPagesRounded = 16;	142	constexpr uptr kMinNumPagesRounded = 16;
142	constexpr uptr kMinRoundedSize = 65536;	143	constexpr uptr kMinRoundedSize = 65536;
143	static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment;	144	static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment;

lib/tsan/sanitizer_common/sanitizer_allocator_dlsym.h+18-9

 @@ -15,6 +15,8 @@
 #define SANITIZER_ALLOCATOR_DLSYM_H
 #include "sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_allocator_checks.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
 namespace __sanitizer {
 @@ -31,24 +33,22 @@ struct DlSymAllocator {
            UNLIKELY(internal_allocator()->FromPrimary(ptr));
   }
-  static void *Allocate(uptr size_in_bytes) {
+  static void *Allocate(uptr size_in_bytes, uptr align = kWordSize) {
-    void *ptr = InternalAlloc(size_in_bytes, nullptr, kWordSize);
+    void *ptr = InternalAlloc(size_in_bytes, nullptr, align);
     CHECK(internal_allocator()->FromPrimary(ptr));
-    Details::OnAllocate(ptr,
+    Details::OnAllocate(ptr, GetSize(ptr));
-                        internal_allocator()->GetActuallyAllocatedSize(ptr));
     return ptr;
   }
-  static void *Callocate(SIZE_T nmemb, SIZE_T size) {
+  static void *Callocate(usize nmemb, usize size) {
     void *ptr = InternalCalloc(nmemb, size);
     CHECK(internal_allocator()->FromPrimary(ptr));
-    Details::OnAllocate(ptr,
+    Details::OnAllocate(ptr, GetSize(ptr));
-                        internal_allocator()->GetActuallyAllocatedSize(ptr));
     return ptr;
   }
   static void Free(void *ptr) {
-    uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
+    uptr size = GetSize(ptr);
     Details::OnFree(ptr, size);
     InternalFree(ptr);
   }
 @@ -61,7 +61,7 @@ struct DlSymAllocator {
       Free(ptr);
       return nullptr;
     }
-    uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
+    uptr size = GetSize(ptr);
     uptr memcpy_size = Min(new_size, size);
     void *new_ptr = Allocate(new_size);
     if (new_ptr)
 @@ -70,6 +70,15 @@ struct DlSymAllocator {
     return new_ptr;
   }
+  static void *ReallocArray(void *ptr, uptr count, uptr size) {
+    CHECK(!CheckForCallocOverflow(count, size));
+    return Realloc(ptr, count * size);
+  }
+  static uptr GetSize(void *ptr) {
+    return internal_allocator()->GetActuallyAllocatedSize(ptr);
+  }
   static void OnAllocate(const void *ptr, uptr size) {}
   static void OnFree(const void *ptr, uptr size) {}
 };

lib/tsan/sanitizer_common/sanitizer_allocator_primary64.h+7-5

 @@ -185,9 +185,10 @@ class SizeClassAllocator64 {
     // recoverable.
     if (UNLIKELY(!EnsureFreeArraySpace(region, region_beg,
                                        new_num_freed_chunks))) {
-      Report("FATAL: Internal error: %s's allocator exhausted the free list "
+      Report(
-             "space for size class %zd (%zd bytes).\n", SanitizerToolName,
+          "FATAL: Internal error: %s's allocator exhausted the free list "
-             class_id, ClassIdToSize(class_id));
+          "space for size class %zu (%zu bytes).\n",
+          SanitizerToolName, class_id, ClassIdToSize(class_id));
       Die();
     }
     for (uptr i = 0; i < n_chunks; i++)
 @@ -763,8 +764,9 @@ class SizeClassAllocator64 {
     if (!region->exhausted) {
       region->exhausted = true;
       Printf("%s: Out of memory. ", SanitizerToolName);
-      Printf("The process has exhausted %zuMB for size class %zu.\n",
+      Printf(
-             kRegionSize >> 20, ClassIdToSize(class_id));
+          "The process has exhausted %zu MB for size class %zu (%zu bytes).\n",
+          kRegionSize >> 20, class_id, ClassIdToSize(class_id));
     }
     return true;
   }

lib/tsan/sanitizer_common/sanitizer_common.h+18-9

 @@ -83,8 +83,8 @@ int TgKill(pid_t pid, tid_t tid, int sig);
 uptr GetThreadSelf();
 void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
                                 uptr *stack_bottom);
-void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size,
+void GetThreadStackAndTls(bool main, uptr *stk_begin, uptr *stk_end,
-                          uptr *tls_addr, uptr *tls_size);
+                          uptr *tls_begin, uptr *tls_end);
 // Memory management
 void *MmapOrDie(uptr size, const char *mem_type, bool raw_report = false);
 @@ -239,13 +239,15 @@ void RemoveANSIEscapeSequencesFromString(char *buffer);
 void Printf(const char *format, ...) FORMAT(1, 2);
 void Report(const char *format, ...) FORMAT(1, 2);
 void SetPrintfAndReportCallback(void (*callback)(const char *));
-#define VReport(level, ...)                                              \
+#define VReport(level, ...)                     \
-  do {                                                                   \
+  do {                                          \
-    if ((uptr)Verbosity() >= (level)) Report(__VA_ARGS__); \
+    if (UNLIKELY((uptr)Verbosity() >= (level))) \
+      Report(__VA_ARGS__);                      \
   } while (0)
-#define VPrintf(level, ...)                                              \
+#define VPrintf(level, ...)                     \
-  do {                                                                   \
+  do {                                          \
-    if ((uptr)Verbosity() >= (level)) Printf(__VA_ARGS__); \
+    if (UNLIKELY((uptr)Verbosity() >= (level))) \
+      Printf(__VA_ARGS__);                      \
   } while (0)
 // Lock sanitizer error reporting and protects against nested errors.
 @@ -266,7 +268,15 @@ class ScopedErrorReportLock {
 extern uptr stoptheworld_tracer_pid;
 extern uptr stoptheworld_tracer_ppid;
+// Returns true if the entire range can be read.
 bool IsAccessibleMemoryRange(uptr beg, uptr size);
+// Attempts to copy `n` bytes from memory range starting at `src` to `dest`.
+// Returns true if the entire range can be read. Returns `false` if any part of
+// the source range cannot be read, in which case the contents of `dest` are
+// undefined.
+bool TryMemCpy(void *dest, const void *src, uptr n);
+// Copies accessible memory, and zero fill inaccessible.
+void MemCpyAccessible(void *dest, const void *src, uptr n);
 // Error report formatting.
 const char *StripPathPrefix(const char *filepath,
 @@ -917,7 +927,6 @@ typedef void (*RangeIteratorCallback)(uptr begin, uptr end, void *arg);
 enum AndroidApiLevel {
   ANDROID_NOT_ANDROID = 0,
-  ANDROID_KITKAT = 19,
   ANDROID_LOLLIPOP_MR1 = 22,
   ANDROID_POST_LOLLIPOP = 23
 };

lib/tsan/sanitizer_common/sanitizer_common_interceptors.inc+189-221

 @@ -41,6 +41,7 @@
 #include "sanitizer_errno.h"
 #include "sanitizer_placement_new.h"
 #include "sanitizer_platform_interceptors.h"
+#include "sanitizer_platform_limits_posix.h"
 #include "sanitizer_symbolizer.h"
 #include "sanitizer_tls_get_addr.h"
 @@ -127,6 +128,39 @@ extern const short *_toupper_tab_;
 extern const short *_tolower_tab_;
 #endif
+#if SANITIZER_LINUX && SANITIZER_SPARC32
+// On 32-bit Linux/sparc64, double and long double are identical and glibc
+// uses a __nldbl_ (no long double) prefix for various stdio functions.
+#  define __isoc23_fscanf __nldbl___isoc23_fscanf
+#  define __isoc23_scanf __nldbl___isoc23_scanf
+#  define __isoc23_sscanf __nldbl___isoc23_sscanf
+#  define __isoc23_vfscanf __nldbl___isoc23_vfscanf
+#  define __isoc23_vscanf __nldbl___isoc23_vscanf
+#  define __isoc23_vsscanf __nldbl___isoc23_vsscanf
+#  define __isoc99_fscanf __nldbl___isoc99_fscanf
+#  define __isoc99_scanf __nldbl___isoc99_scanf
+#  define __isoc99_sscanf __nldbl___isoc99_sscanf
+#  define __isoc99_vfscanf __nldbl___isoc99_vfscanf
+#  define __isoc99_vscanf __nldbl___isoc99_vscanf
+#  define __isoc99_vsscanf __nldbl___isoc99_vsscanf
+#  define asprintf __nldbl_asprintf
+#  define fprintf __nldbl_fprintf
+#  define fscanf __nldbl_fscanf
+#  define printf __nldbl_printf
+#  define scanf __nldbl_scanf
+#  define snprintf __nldbl_snprintf
+#  define sprintf __nldbl_sprintf
+#  define sscanf __nldbl_sscanf
+#  define vasprintf __nldbl_vasprintf
+#  define vfprintf __nldbl_vfprintf
+#  define vfscanf __nldbl_vfscanf
+#  define vprintf __nldbl_vprintf
+#  define vscanf __nldbl_vscanf
+#  define vsnprintf __nldbl_vsnprintf
+#  define vsprintf __nldbl_vsprintf
+#  define vsscanf __nldbl_vsscanf
+#endif
 #if SANITIZER_MUSL && \
   (defined(__i386__) || defined(__arm__) || SANITIZER_MIPS32 || SANITIZER_PPC32)
 // musl 1.2.0 on existing 32-bit architectures uses new symbol names for the
 @@ -313,7 +347,7 @@ extern const short *_tolower_tab_;
   uptr copy_length = internal_strnlen(s, size);                               \
   char *new_mem = (char *)WRAP(malloc)(copy_length + 1);                      \
   if (common_flags()->intercept_strndup) {                                    \
-    COMMON_INTERCEPTOR_READ_STRING(ctx, s, Min(size, copy_length + 1));       \
+    COMMON_INTERCEPTOR_READ_STRING(ctx, s, Min<uptr>(size, copy_length + 1)); \
   }                                                                           \
   if (new_mem) {                                                              \
     COMMON_INTERCEPTOR_COPY_STRING(ctx, new_mem, s, copy_length);             \
 @@ -411,7 +445,7 @@ INTERCEPTOR(SIZE_T, strnlen, const char *s, SIZE_T maxlen) {
 #endif
 #if SANITIZER_INTERCEPT_STRNDUP
-INTERCEPTOR(char*, strndup, const char *s, uptr size) {
+INTERCEPTOR(char*, strndup, const char *s, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_STRNDUP_IMPL(ctx, s, size);
 }
 @@ -421,7 +455,7 @@ INTERCEPTOR(char*, strndup, const char *s, uptr size) {
 #endif // SANITIZER_INTERCEPT_STRNDUP
 #if SANITIZER_INTERCEPT___STRNDUP
-INTERCEPTOR(char*, __strndup, const char *s, uptr size) {
+INTERCEPTOR(char*, __strndup, const char *s, usize size) {
   void *ctx;
   COMMON_INTERCEPTOR_STRNDUP_IMPL(ctx, s, size);
 }
 @@ -477,23 +511,23 @@ INTERCEPTOR(int, strcmp, const char *s1, const char *s2) {
 }
 DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncmp, uptr called_pc,
-                              const char *s1, const char *s2, uptr n,
+                              const char *s1, const char *s2, usize n,
                               int result)
-INTERCEPTOR(int, strncmp, const char *s1, const char *s2, uptr size) {
+INTERCEPTOR(int, strncmp, const char *s1, const char *s2, usize size) {
   if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
     return internal_strncmp(s1, s2, size);
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, strncmp, s1, s2, size);
   unsigned char c1 = 0, c2 = 0;
-  uptr i;
+  usize i;
   for (i = 0; i < size; i++) {
     c1 = (unsigned char)s1[i];
     c2 = (unsigned char)s2[i];
     if (c1 != c2 || c1 == '\0') break;
   }
-  uptr i1 = i;
+  usize i1 = i;
-  uptr i2 = i;
+  usize i2 = i;
   if (common_flags()->strict_string_checks) {
     for (; i1 < size && s1[i1]; i1++) {}
     for (; i2 < size && s2[i2]; i2++) {}
 @@ -542,21 +576,21 @@ INTERCEPTOR(int, strcasecmp, const char *s1, const char *s2) {
 }
 DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncasecmp, uptr called_pc,
-                              const char *s1, const char *s2, uptr size,
+                              const char *s1, const char *s2, usize size,
                               int result)
 INTERCEPTOR(int, strncasecmp, const char *s1, const char *s2, SIZE_T size) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, strncasecmp, s1, s2, size);
   unsigned char c1 = 0, c2 = 0;
-  uptr i;
+  usize i;
   for (i = 0; i < size; i++) {
     c1 = (unsigned char)s1[i];
     c2 = (unsigned char)s2[i];
     if (CharCaseCmp(c1, c2) != 0 || c1 == '\0') break;
   }
-  uptr i1 = i;
+  usize i1 = i;
-  uptr i2 = i;
+  usize i2 = i;
   if (common_flags()->strict_string_checks) {
     for (; i1 < size && s1[i1]; i1++) {}
     for (; i2 < size && s2[i2]; i2++) {}
 @@ -799,13 +833,13 @@ INTERCEPTOR(char *, strpbrk, const char *s1, const char *s2) {
 #if SANITIZER_INTERCEPT_MEMCMP
 DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc,
-                              const void *s1, const void *s2, uptr n,
+                              const void *s1, const void *s2, usize n,
                               int result)
 // Common code for `memcmp` and `bcmp`.
 int MemcmpInterceptorCommon(void *ctx,
-                            int (*real_fn)(const void *, const void *, uptr),
+                            int (*real_fn)(const void *, const void *, usize),
-                            const void *a1, const void *a2, uptr size) {
+                            const void *a1, const void *a2, usize size) {
   if (common_flags()->intercept_memcmp) {
     if (common_flags()->strict_memcmp) {
       // Check the entire regions even if the first bytes of the buffers are
 @@ -817,7 +851,7 @@ int MemcmpInterceptorCommon(void *ctx,
       unsigned char c1 = 0, c2 = 0;
       const unsigned char *s1 = (const unsigned char*)a1;
       const unsigned char *s2 = (const unsigned char*)a2;
-      uptr i;
+      usize i;
       for (i = 0; i < size; i++) {
         c1 = s1[i];
         c2 = s2[i];
 @@ -837,7 +871,7 @@ int MemcmpInterceptorCommon(void *ctx,
   return result;
 }
-INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) {
+INTERCEPTOR(int, memcmp, const void *a1, const void *a2, usize size) {
   if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
     return internal_memcmp(a1, a2, size);
   void *ctx;
 @@ -851,7 +885,7 @@ INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) {
 #endif
 #if SANITIZER_INTERCEPT_BCMP
-INTERCEPTOR(int, bcmp, const void *a1, const void *a2, uptr size) {
+INTERCEPTOR(int, bcmp, const void *a1, const void *a2, usize size) {
   if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
     return internal_memcmp(a1, a2, size);
   void *ctx;
 @@ -1104,7 +1138,7 @@ INTERCEPTOR(SSIZE_T, write, int fd, void *ptr, SIZE_T count) {
 #endif
 #if SANITIZER_INTERCEPT_FWRITE
-INTERCEPTOR(SIZE_T, fwrite, const void *p, uptr size, uptr nmemb, void *file) {
+INTERCEPTOR(SIZE_T, fwrite, const void *p, usize size, usize nmemb, void *file) {
   // libc file streams can call user-supplied functions, see fopencookie.
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, fwrite, p, size, nmemb, file);
 @@ -1255,6 +1289,12 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
   static const int PR_SET_VMA = 0x53564d41;
   static const int PR_SCHED_CORE = 62;
   static const int PR_SCHED_CORE_GET = 0;
+  static const int PR_GET_PDEATHSIG = 2;
+#  if !SANITIZER_ANDROID
+  static const int PR_SET_SECCOMP = 22;
+  static const int SECCOMP_MODE_FILTER = 2;
+#  endif
   if (option == PR_SET_VMA && arg2 == 0UL) {
     char *name = (char *)arg5;
     COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
 @@ -1270,7 +1310,14 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
     COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
   } else if (res != -1 && option == PR_SCHED_CORE &&
              arg2 == PR_SCHED_CORE_GET) {
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64*)(arg5), sizeof(u64));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg5), sizeof(u64));
+  } else if (res != -1 && option == PR_GET_PDEATHSIG) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg2), sizeof(int));
+#  if SANITIZER_GLIBC
+  } else if (res != -1 && option == PR_SET_SECCOMP &&
+             arg2 == SECCOMP_MODE_FILTER) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg3), struct_sock_fprog_sz);
+#  endif
   }
   return res;
 }
 @@ -2242,6 +2289,61 @@ INTERCEPTOR(int, pthread_getcpuclockid, uptr thread,
 #define INIT_CLOCK_GETCPUCLOCKID
 #endif
+#if SANITIZER_INTERCEPT_TIMER_CREATE
+INTERCEPTOR(int, timer_create, __sanitizer_clockid_t clockid, void *sevp,
+            __sanitizer_timer_t *timer) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, timer_create, clockid, sevp, timer);
+  int res = REAL(timer_create)(clockid, sevp, timer);
+  if (!res && timer) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, timer, sizeof *timer);
+  }
+  return res;
+}
+INTERCEPTOR(int, timer_delete, __sanitizer_timer_t timer) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, timer_delete, timer);
+  int res = REAL(timer_delete)(timer);
+  return res;
+}
+INTERCEPTOR(int, timer_gettime, __sanitizer_timer_t timer,
+            struct __sanitizer_itimerspec *curr_value) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, timer_gettime, timer, curr_value);
+  int res = REAL(timer_gettime)(timer, curr_value);
+  if (!res && curr_value) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, curr_value, sizeof *curr_value);
+  }
+  return res;
+}
+INTERCEPTOR(int, timer_settime, __sanitizer_timer_t timer, int flags,
+            const struct __sanitizer_itimerspec *new_value,
+            struct __sanitizer_itimerspec *old_value) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, timer_settime, timer, flags, new_value,
+                           old_value);
+  int res = REAL(timer_settime)(timer, flags, new_value, old_value);
+  if (!res) {
+    if (new_value)
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, new_value, sizeof *new_value);
+    if (old_value)
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, old_value, sizeof *old_value);
+  }
+  return res;
+}
+#  define INIT_TIMER_CREATE                                                \
+    COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(timer_create, "GLIBC_2.3.3");  \
+    COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(timer_delete, "GLIBC_2.3.3");  \
+    COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(timer_gettime, "GLIBC_2.3.3"); \
+    COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(timer_settime, "GLIBC_2.3.3");
+#else
+#  define INIT_TIMER_CREATE
+#endif
 #if SANITIZER_INTERCEPT_GETITIMER
 INTERCEPTOR(int, getitimer, int which, void *curr_value) {
   void *ctx;
 @@ -2287,6 +2389,25 @@ INTERCEPTOR(int, setitimer, int which, const void *new_value, void *old_value) {
 #define INIT_GETITIMER
 #endif
+#if SANITIZER_INTERCEPT_TIMESPEC_GET
+INTERCEPTOR(int, timespec_get, struct __sanitizer_timespec *ts, int base) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, timespec_get, ts, base);
+  // We don't yet know if ts is addressable, so we use our own scratch buffer
+  struct __sanitizer_timespec ts_local;
+  int res = REAL(timespec_get)(&ts_local, base);
+  if (res) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ts,
+                                   sizeof(struct __sanitizer_timespec));
+    internal_memcpy(ts, &ts_local, sizeof(struct __sanitizer_timespec));
+  }
+  return res;
+}
+#  define INIT_TIMESPEC_GET COMMON_INTERCEPT_FUNCTION(timespec_get);
+#else
+#  define INIT_TIMESPEC_GET
+#endif
 #if SANITIZER_INTERCEPT_GLOB
 static void unpoison_glob_t(void *ctx, __sanitizer_glob_t *pglob) {
   COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pglob, sizeof(*pglob));
 @@ -3427,23 +3548,27 @@ INTERCEPTOR(uptr, ptrace, int request, int pid, void *addr, void *data) {
   COMMON_INTERCEPTOR_ENTER(ctx, ptrace, request, pid, addr, data);
   __sanitizer_iovec local_iovec;
-  if (data) {
+  void *data_arg = ptrace_data_arg(request, addr, data);
+  if (data_arg) {
     if (request == ptrace_setregs) {
-      COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_regs_struct_sz);
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, data_arg, struct_user_regs_struct_sz);
     } else if (request == ptrace_setfpregs) {
-      COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_fpregs_struct_sz);
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, data_arg,
+                                    struct_user_fpregs_struct_sz);
     } else if (request == ptrace_setfpxregs) {
-      COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_fpxregs_struct_sz);
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, data_arg,
+                                    struct_user_fpxregs_struct_sz);
     } else if (request == ptrace_setvfpregs) {
-      COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_vfpregs_struct_sz);
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, data_arg,
+                                    struct_user_vfpregs_struct_sz);
     } else if (request == ptrace_setsiginfo) {
-      COMMON_INTERCEPTOR_READ_RANGE(ctx, data, siginfo_t_sz);
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, data_arg, siginfo_t_sz);
-    // Some kernel might zero the iovec::iov_base in case of invalid
+      // Some kernel might zero the iovec::iov_base in case of invalid
-    // write access.  In this case copy the invalid address for further
+      // write access.  In this case copy the invalid address for further
-    // inspection.
+      // inspection.
     } else if (request == ptrace_setregset || request == ptrace_getregset) {
-      __sanitizer_iovec *iovec = (__sanitizer_iovec*)data;
+      __sanitizer_iovec *iovec = (__sanitizer_iovec *)data_arg;
       COMMON_INTERCEPTOR_READ_RANGE(ctx, iovec, sizeof(*iovec));
       local_iovec = *iovec;
       if (request == ptrace_setregset)
 @@ -3456,23 +3581,26 @@ INTERCEPTOR(uptr, ptrace, int request, int pid, void *addr, void *data) {
   // https://github.com/google/sanitizers/issues/321.
   uptr res = REAL(ptrace)(request, pid, addr, data);
-  if (!res && data) {
+  if (!res && data_arg) {
     // Note that PEEK* requests assign different meaning to the return value.
     // This function does not handle them (nor does it need to).
     if (request == ptrace_getregs) {
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_regs_struct_sz);
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg, struct_user_regs_struct_sz);
     } else if (request == ptrace_getfpregs) {
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_fpregs_struct_sz);
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg,
+                                     struct_user_fpregs_struct_sz);
     } else if (request == ptrace_getfpxregs) {
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_fpxregs_struct_sz);
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg,
+                                     struct_user_fpxregs_struct_sz);
     } else if (request == ptrace_getvfpregs) {
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_vfpregs_struct_sz);
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg,
+                                     struct_user_vfpregs_struct_sz);
     } else if (request == ptrace_getsiginfo) {
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, siginfo_t_sz);
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg, siginfo_t_sz);
     } else if (request == ptrace_geteventmsg) {
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, sizeof(unsigned long));
+      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg, sizeof(unsigned long));
     } else if (request == ptrace_getregset) {
-      __sanitizer_iovec *iovec = (__sanitizer_iovec*)data;
+      __sanitizer_iovec *iovec = (__sanitizer_iovec *)data_arg;
       COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iovec, sizeof(*iovec));
       COMMON_INTERCEPTOR_WRITE_RANGE(ctx, local_iovec.iov_base,
                                      local_iovec.iov_len);
 @@ -6425,12 +6553,12 @@ static void MlockIsUnsupported() {
           SanitizerToolName);
 }
-INTERCEPTOR(int, mlock, const void *addr, uptr len) {
+INTERCEPTOR(int, mlock, const void *addr, usize len) {
   MlockIsUnsupported();
   return 0;
 }
-INTERCEPTOR(int, munlock, const void *addr, uptr len) {
+INTERCEPTOR(int, munlock, const void *addr, usize len) {
   MlockIsUnsupported();
   return 0;
 }
 @@ -8823,83 +8951,6 @@ INTERCEPTOR(char *, RMD160Data, u8 *data, SIZE_T len, char *buf) {
 #define INIT_RMD160
 #endif
-#if SANITIZER_INTERCEPT_MD5
-INTERCEPTOR(void, MD5Init, void *context) {
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, MD5Init, context);
-  REAL(MD5Init)(context);
-  if (context)
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD5_CTX_sz);
-}
-INTERCEPTOR(void, MD5Update, void *context, const unsigned char *data,
-            unsigned int len) {
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, MD5Update, context, data, len);
-  if (data && len > 0)
-    COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len);
-  if (context)
-    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz);
-  REAL(MD5Update)(context, data, len);
-  if (context)
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD5_CTX_sz);
-}
-INTERCEPTOR(void, MD5Final, unsigned char digest[16], void *context) {
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, MD5Final, digest, context);
-  if (context)
-    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz);
-  REAL(MD5Final)(digest, context);
-  if (digest)
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(unsigned char) * 16);
-}
-INTERCEPTOR(char *, MD5End, void *context, char *buf) {
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, MD5End, context, buf);
-  if (context)
-    COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz);
-  char *ret = REAL(MD5End)(context, buf);
-  if (ret)
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length);
-  return ret;
-}
-INTERCEPTOR(char *, MD5File, const char *filename, char *buf) {
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, MD5File, filename, buf);
-  if (filename)
-    COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
-  char *ret = REAL(MD5File)(filename, buf);
-  if (ret)
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length);
-  return ret;
-}
-INTERCEPTOR(char *, MD5Data, const unsigned char *data, unsigned int len,
-            char *buf) {
-  void *ctx;
-  COMMON_INTERCEPTOR_ENTER(ctx, MD5Data, data, len, buf);
-  if (data && len > 0)
-    COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len);
-  char *ret = REAL(MD5Data)(data, len, buf);
-  if (ret)
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length);
-  return ret;
-}
-#define INIT_MD5                                                               \
-  COMMON_INTERCEPT_FUNCTION(MD5Init);                                          \
-  COMMON_INTERCEPT_FUNCTION(MD5Update);                                        \
-  COMMON_INTERCEPT_FUNCTION(MD5Final);                                         \
-  COMMON_INTERCEPT_FUNCTION(MD5End);                                           \
-  COMMON_INTERCEPT_FUNCTION(MD5File);                                          \
-  COMMON_INTERCEPT_FUNCTION(MD5Data)
-#else
-#define INIT_MD5
-#endif
 #if SANITIZER_INTERCEPT_FSEEK
 INTERCEPTOR(int, fseek, __sanitizer_FILE *stream, long int offset, int whence) {
   void *ctx;
 @@ -9030,107 +9081,6 @@ INTERCEPTOR(char *, MD2Data, const unsigned char *data, unsigned int len,
 #define INIT_MD2
 #endif
-#if SANITIZER_INTERCEPT_SHA2
-#define SHA2_INTERCEPTORS(LEN, SHA2_STATE_T) \
-  INTERCEPTOR(void, SHA##LEN##_Init, void *context) { \
-    void *ctx; \
-    COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Init, context); \
-    REAL(SHA##LEN##_Init)(context); \
-    if (context) \
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA##LEN##_CTX_sz); \
-  } \
-  INTERCEPTOR(void, SHA##LEN##_Update, void *context, \
-              const u8 *data, SIZE_T len) { \
-    void *ctx; \
-    COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Update, context, data, len); \
-    if (data && len > 0) \
-      COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); \
-    if (context) \
-      COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \
-    REAL(SHA##LEN##_Update)(context, data, len); \
-    if (context) \
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA##LEN##_CTX_sz); \
-  } \
-  INTERCEPTOR(void, SHA##LEN##_Final, u8 digest[LEN/8], \
-  void *context) { \
-    void *ctx; \
-    CHECK_EQ(SHA##LEN##_digest_length, LEN/8); \
-    COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Final, digest, context); \
-    if (context) \
-      COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \
-    REAL(SHA##LEN##_Final)(digest, context); \
-    if (digest) \
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, \
-                                     sizeof(digest[0]) * \
-  SHA##LEN##_digest_length); \
-  } \
-  INTERCEPTOR(char *, SHA##LEN##_End, void *context, char *buf) { \
-    void *ctx; \
-    COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_End, context, buf); \
-    if (context) \
-      COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \
-    char *ret = REAL(SHA##LEN##_End)(context, buf); \
-    if (ret) \
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
-    return ret; \
-  } \
-  INTERCEPTOR(char *, SHA##LEN##_File, const char *filename, char *buf) { \
-    void *ctx; \
-    COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_File, filename, buf); \
-    if (filename) \
-      COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);\
-    char *ret = REAL(SHA##LEN##_File)(filename, buf); \
-    if (ret) \
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
-    return ret; \
-  } \
-  INTERCEPTOR(char *, SHA##LEN##_FileChunk, const char *filename, char *buf, \
-              OFF_T offset, OFF_T length) { \
-    void *ctx; \
-    COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_FileChunk, filename, buf, offset, \
-  length); \
-    if (filename) \
-      COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);\
-    char *ret = REAL(SHA##LEN##_FileChunk)(filename, buf, offset, length); \
-    if (ret) \
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
-    return ret; \
-  } \
-  INTERCEPTOR(char *, SHA##LEN##_Data, u8 *data, SIZE_T len, char *buf) { \
-    void *ctx; \
-    COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Data, data, len, buf); \
-    if (data && len > 0) \
-      COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); \
-    char *ret = REAL(SHA##LEN##_Data)(data, len, buf); \
-    if (ret) \
-      COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
-    return ret; \
-  }
-SHA2_INTERCEPTORS(224, u32)
-SHA2_INTERCEPTORS(256, u32)
-SHA2_INTERCEPTORS(384, u64)
-SHA2_INTERCEPTORS(512, u64)
-#define INIT_SHA2_INTECEPTORS(LEN) \
-  COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Init); \
-  COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Update); \
-  COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Final); \
-  COMMON_INTERCEPT_FUNCTION(SHA##LEN##_End); \
-  COMMON_INTERCEPT_FUNCTION(SHA##LEN##_File); \
-  COMMON_INTERCEPT_FUNCTION(SHA##LEN##_FileChunk); \
-  COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Data)
-#define INIT_SHA2 \
-  INIT_SHA2_INTECEPTORS(224); \
-  INIT_SHA2_INTECEPTORS(256); \
-  INIT_SHA2_INTECEPTORS(384); \
-  INIT_SHA2_INTECEPTORS(512)
-#undef SHA2_INTERCEPTORS
-#else
-#define INIT_SHA2
-#endif
 #if SANITIZER_INTERCEPT_VIS
 INTERCEPTOR(char *, vis, char *dst, int c, int flag, int nextc) {
   void *ctx;
 @@ -9980,7 +9930,7 @@ INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, getrandom, buf, buflen, flags);
   // If GRND_NONBLOCK is set in the flags, it is non blocking.
   static const int grnd_nonblock = 1;
   SSIZE_T n;
   if ((flags & grnd_nonblock))
     n = REAL(getrandom)(buf, buflen, flags);
 @@ -10296,6 +10246,23 @@ INTERCEPTOR(SSIZE_T, pwritev2, int fd, __sanitizer_iovec *iov, int iovcnt,
 #define INIT_PWRITEV2
 #endif
+#if SANITIZER_INTERCEPT_FREADLINK
+INTERCEPTOR(SSIZE_T, freadlink, int fd, char *buf, SIZE_T bufsiz) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, freadlink, fd, buf, bufsiz);
+  COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
+  SSIZE_T res = REAL(freadlink)(fd, buf, bufsiz);
+  if (res > 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res);
+  if (res >= 0 && fd > 0)
+    COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
+  return res;
+}
+#  define INIT_FREADLINK COMMON_INTERCEPT_FUNCTION(freadlink)
+#else
+#  define INIT_FREADLINK
+#endif
 #include "sanitizer_common_interceptors_netbsd_compat.inc"
 namespace __sanitizer {
 @@ -10373,8 +10340,10 @@ static void InitializeCommonInterceptors() {
   INIT_SETPWENT;
   INIT_CLOCK_GETTIME;
   INIT_CLOCK_GETCPUCLOCKID;
+  INIT_TIMER_CREATE;
   INIT_GETITIMER;
   INIT_TIME;
+  INIT_TIMESPEC_GET;
   INIT_GLOB;
   INIT_GLOB64;
   INIT___B64_TO;
 @@ -10588,10 +10557,8 @@ static void InitializeCommonInterceptors() {
   INIT_SHA1;
   INIT_MD4;
   INIT_RMD160;
-  INIT_MD5;
   INIT_FSEEK;
   INIT_MD2;
-  INIT_SHA2;
   INIT_VIS;
   INIT_CDB;
   INIT_GETFSENT;
 @@ -10617,6 +10584,7 @@ static void InitializeCommonInterceptors() {
   INIT_CPUSET_GETAFFINITY;
   INIT_PREADV2;
   INIT_PWRITEV2;
+  INIT_FREADLINK;
   INIT___PRINTF_CHK;
 }

lib/tsan/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc+17-17

lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S created+48

...	@@ -0,0 +1,48 @@
1	#if defined(__aarch64__) && defined(__linux__)
2
3	#include "sanitizer_common/sanitizer_asm.h"
4	#include "builtins/assembly.h"
5
6	ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA)
7
8	.comm _ZN14__interception10real_vforkE,8,8
9	.globl ASM_WRAPPER_NAME(vfork)
10	ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
11	ASM_WRAPPER_NAME(vfork):
12	// Save x30 in the off-stack spill area.
13	hint #25 // paciasp
14	stp xzr, x30, [sp, #-16]!
15	bl COMMON_INTERCEPTOR_SPILL_AREA
16	ldp xzr, x30, [sp], 16
17	str x30, [x0]
18
19	// Call real vfork. This may return twice. User code that runs between the first and the second return
20	// may clobber the stack frame of the interceptor; that's why it does not have a frame.
21	adrp x0, _ZN14__interception10real_vforkE
22	ldr x0, [x0, :lo12:_ZN14__interception10real_vforkE]
23	blr x0
24
25	stp x0, xzr, [sp, #-16]!
26	cmp x0, #0
27	b.eq .L_exit
28
29	// x0 != 0 => parent process. Clear stack shadow.
30	add x0, sp, #16
31	bl COMMON_INTERCEPTOR_HANDLE_VFORK
32
33	.L_exit:
34	// Restore x30.
35	bl COMMON_INTERCEPTOR_SPILL_AREA
36	ldr x30, [x0]
37	ldp x0, xzr, [sp], 16
38	hint #29 // autiasp
39
40	ret
41	ASM_SIZE(vfork)
42
43	ASM_INTERCEPTOR_TRAMPOLINE(vfork)
44	ASM_TRAMPOLINE_ALIAS(vfork, vfork)
45
46	GNU_PROPERTY_BTI_PAC
47
48	#endif

lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S created+49

...	@@ -0,0 +1,49 @@
1	#if defined(__arm__) && defined(__linux__)
2
3	#include "sanitizer_common/sanitizer_asm.h"
4
5	ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA)
6
7	.comm _ZN14__interception10real_vforkE,4,4
8	.globl ASM_WRAPPER_NAME(vfork)
9	ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
10	ASM_WRAPPER_NAME(vfork):
11	// Save LR in the off-stack spill area.
12	push {r4, lr}
13	bl COMMON_INTERCEPTOR_SPILL_AREA
14	pop {r4, lr}
15	str lr, [r0]
16
17	// Call real vfork. This may return twice. User code that runs between the first and the second return
18	// may clobber the stack frame of the interceptor; that's why it does not have a frame.
19	ldr r0, .LCPI0_0
20	.LPC0_0:
21	ldr r0, [pc, r0]
22	mov lr, pc
23	bx r0
24
25	push {r0, r4}
26	cmp r0, #0
27	beq .L_exit
28
29	// r0 != 0 => parent process. Clear stack shadow.
30	add r0, sp, #8
31	bl COMMON_INTERCEPTOR_HANDLE_VFORK
32
33	.L_exit:
34	// Restore LR.
35	bl COMMON_INTERCEPTOR_SPILL_AREA
36	ldr lr, [r0]
37	pop {r0, r4}
38
39	mov pc, lr
40
41	.LCPI0_0:
42	.long _ZN14__interception10real_vforkE - (.LPC0_0+8)
43
44	ASM_SIZE(vfork)
45
46	ASM_INTERCEPTOR_TRAMPOLINE(vfork)
47	ASM_TRAMPOLINE_ALIAS(vfork, vfork)
48
49	#endif

lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S created+64

...	@@ -0,0 +1,64 @@
1	#if defined(__i386__) && defined(__linux__)
2
3	#include "sanitizer_common/sanitizer_asm.h"
4
5	.comm _ZN14__interception10real_vforkE,4,4
6	.globl ASM_WRAPPER_NAME(vfork)
7	ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
8	ASM_WRAPPER_NAME(vfork):
9	_CET_ENDBR
10	// Store return address in the spill area and tear down the stack frame.
11	sub $12, %esp
12	call COMMON_INTERCEPTOR_SPILL_AREA
13	mov 12(%esp), %ecx
14	mov %ecx, (%eax)
15	add $16, %esp
16
17	call .L0$pb
18	.L0$pb:
19	pop %eax
20	.Ltmp0:
21	add $_GLOBAL_OFFSET_TABLE_+(.Ltmp0-.L0$pb), %eax
22	call *_ZN14__interception10real_vforkE@GOTOFF(%eax)
23
24	// Restore the stack frame.
25	// 12(%esp) return address
26	// 8(%esp) spill %ebx
27	// 4(%esp) spill REAL(vfork) return value
28	// (%esp) call frame (arg0) for __*_handle_vfork
29	sub $16, %esp
30	mov %ebx, 8(%esp)
31	mov %eax, 4(%esp)
32
33	// Form GOT address in %ebx.
34	call .L1$pb
35	.L1$pb:
36	pop %ebx
37	.Ltmp1:
38	add $_GLOBAL_OFFSET_TABLE_+(.Ltmp1-.L1$pb), %ebx
39
40	// Restore original return address.
41	call COMMON_INTERCEPTOR_SPILL_AREA
42	mov (%eax), %ecx
43	mov %ecx, 12(%esp)
44	mov 4(%esp), %eax
45
46	// Call handle_vfork in the parent process (%rax != 0).
47	test %eax, %eax
48	je .L_exit
49
50	lea 16(%esp), %ecx
51	mov %ecx, (%esp)
52	call COMMON_INTERCEPTOR_HANDLE_VFORK@PLT
53
54	.L_exit:
55	mov 4(%esp), %eax
56	mov 8(%esp), %ebx
57	add $12, %esp
58	ret
59	ASM_SIZE(vfork)
60
61	ASM_INTERCEPTOR_TRAMPOLINE(vfork)
62	ASM_TRAMPOLINE_ALIAS(vfork, vfork)
63
64	#endif

lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S created+57

...	@@ -0,0 +1,57 @@
1	#if defined(__loongarch_lp64) && defined(__linux__)
2
3	#include "sanitizer_common/sanitizer_asm.h"
4
5	ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA)
6	ASM_HIDDEN(_ZN14__interception10real_vforkE)
7
8	.text
9	.globl ASM_WRAPPER_NAME(vfork)
10	ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
11	ASM_WRAPPER_NAME(vfork):
12	// Save ra in the off-stack spill area.
13	// allocate space on stack
14	addi.d $sp, $sp, -16
15	// store $ra value
16	st.d $ra, $sp, 8
17	bl COMMON_INTERCEPTOR_SPILL_AREA
18	// restore previous values from stack
19	ld.d $ra, $sp, 8
20	// adjust stack
21	addi.d $sp, $sp, 16
22	// store $ra by $a0
23	st.d $ra, $a0, 0
24
25	// Call real vfork. This may return twice. User code that runs between the first and the second return
26	// may clobber the stack frame of the interceptor; that's why it does not have a frame.
27	la.local $a0, _ZN14__interception10real_vforkE
28	ld.d $a0, $a0, 0
29	jirl $ra, $a0, 0
30
31	// adjust stack
32	addi.d $sp, $sp, -16
33	// store $a0 by adjusted stack
34	st.d $a0, $sp, 8
35	// jump to exit label if $a0 is 0
36	beqz $a0, .L_exit
37
38	// $a0 != 0 => parent process. Clear stack shadow.
39	// put old $sp to $a0
40	addi.d $a0, $sp, 16
41	bl %plt(COMMON_INTERCEPTOR_HANDLE_VFORK)
42
43	.L_exit:
44	// Restore $ra
45	bl COMMON_INTERCEPTOR_SPILL_AREA
46	ld.d $ra, $a0, 0
47	// load value by stack
48	ld.d $a0, $sp, 8
49	// adjust stack
50	addi.d $sp, $sp, 16
51	jr $ra
52	ASM_SIZE(vfork)
53
54	ASM_INTERCEPTOR_TRAMPOLINE(vfork)
55	ASM_TRAMPOLINE_ALIAS(vfork, vfork)
56
57	#endif

lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_riscv64.inc.S created+56

...	@@ -0,0 +1,56 @@
1	#if (defined(__riscv) && (__riscv_xlen == 64)) && defined(__linux__)
2
3	#include "sanitizer_common/sanitizer_asm.h"
4
5	ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA)
6
7	.comm _ZN14__interception10real_vforkE,8,8
8	.globl ASM_WRAPPER_NAME(vfork)
9	ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
10	ASM_WRAPPER_NAME(vfork):
11	// Save ra in the off-stack spill area.
12	// allocate space on stack
13	addi sp, sp, -16
14	// store ra value
15	sd ra, 8(sp)
16	call COMMON_INTERCEPTOR_SPILL_AREA
17	// restore previous values from stack
18	ld ra, 8(sp)
19	// adjust stack
20	addi sp, sp, 16
21	// store ra by x10
22	sd ra, 0(x10)
23
24	// Call real vfork. This may return twice. User code that runs between the first and the second return
25	// may clobber the stack frame of the interceptor; that's why it does not have a frame.
26	la x10, _ZN14__interception10real_vforkE
27	ld x10, 0(x10)
28	jalr x10
29
30	// adjust stack
31	addi sp, sp, -16
32	// store x10 by adjusted stack
33	sd x10, 8(sp)
34	// jump to exit label if x10 is 0
35	beqz x10, .L_exit
36
37	// x0 != 0 => parent process. Clear stack shadow.
38	// put old sp to x10
39	addi x10, sp, 16
40	call COMMON_INTERCEPTOR_HANDLE_VFORK
41
42	.L_exit:
43	// Restore ra
44	call COMMON_INTERCEPTOR_SPILL_AREA
45	ld ra, 0(x10)
46	// load value by stack
47	ld x10, 8(sp)
48	// adjust stack
49	addi sp, sp, 16
50	ret
51	ASM_SIZE(vfork)
52
53	ASM_INTERCEPTOR_TRAMPOLINE(vfork)
54	ASM_TRAMPOLINE_ALIAS(vfork, vfork)
55
56	#endif

lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S created+42

...	@@ -0,0 +1,42 @@
1	#if defined(__x86_64__) && defined(__linux__)
2
3	#include "sanitizer_common/sanitizer_asm.h"
4
5	.comm _ZN14__interception10real_vforkE,8,8
6	.globl ASM_WRAPPER_NAME(vfork)
7	ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
8	ASM_WRAPPER_NAME(vfork):
9	_CET_ENDBR
10	// Store return address in the spill area and tear down the stack frame.
11	push %rcx
12	call COMMON_INTERCEPTOR_SPILL_AREA
13	pop %rcx
14	pop %rdi
15	mov %rdi, (%rax)
16
17	call *_ZN14__interception10real_vforkE(%rip)
18
19	// Restore return address from the spill area.
20	push %rcx
21	push %rax
22	call COMMON_INTERCEPTOR_SPILL_AREA
23	mov (%rax), %rdx
24	mov %rdx, 8(%rsp)
25	mov (%rsp), %rax
26
27	// Call handle_vfork in the parent process (%rax != 0).
28	test %rax, %rax
29	je .L_exit
30
31	lea 16(%rsp), %rdi
32	call COMMON_INTERCEPTOR_HANDLE_VFORK@PLT
33
34	.L_exit:
35	pop %rax
36	ret
37	ASM_SIZE(ASM_WRAPPER_NAME(vfork))
38
39	ASM_INTERCEPTOR_TRAMPOLINE(vfork)
40	ASM_TRAMPOLINE_ALIAS(vfork, vfork)
41
42	#endif

lib/tsan/sanitizer_common/sanitizer_common_interface.inc+8

lib/tsan/sanitizer_common/sanitizer_common_libcdep.cpp+27-1

...	@@ -171,7 +171,7 @@ void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name,	...	@@ -171,7 +171,7 @@ void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name,
171	"ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "	171	"ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
172	"Perhaps you're using ulimit -v or ulimit -d\n",	172	"Perhaps you're using ulimit -v or ulimit -d\n",
173	size);	173	size);
174	Abort();	174	Die();
175	}	175	}
176	if (madvise_shadow && common_flags()->use_madv_dontdump)	176	if (madvise_shadow && common_flags()->use_madv_dontdump)
177	DontDumpShadowMemory(beg, size);	177	DontDumpShadowMemory(beg, size);
...	@@ -219,6 +219,32 @@ static void StopStackDepotBackgroundThread() {	...	@@ -219,6 +219,32 @@ static void StopStackDepotBackgroundThread() {
219	static void StopStackDepotBackgroundThread() {}	219	static void StopStackDepotBackgroundThread() {}
220	#endif	220	#endif
221		221
		222	void MemCpyAccessible(void dest, const void src, uptr n) {
		223	if (TryMemCpy(dest, src, n))
		224	return;
		225
		226	const uptr page_size = GetPageSize();
		227	uptr b = reinterpret_cast<uptr>(src);
		228	uptr b_up = RoundUpTo(b, page_size);
		229
		230	uptr e = reinterpret_cast<uptr>(src) + n;
		231	uptr e_down = RoundDownTo(e, page_size);
		232
		233	auto copy_or_zero = [dest, src](uptr beg, uptr end) {
		234	const uptr udest = reinterpret_cast<uptr>(dest);
		235	const uptr usrc = reinterpret_cast<uptr>(src);
		236	void d = reinterpret_cast<void >(udest + (beg - usrc));
		237	const uptr size = end - beg;
		238	if (!TryMemCpy(d, reinterpret_cast<void *>(beg), size))
		239	internal_memset(d, 0, size);
		240	};
		241
		242	copy_or_zero(b, b_up);
		243	for (uptr p = b_up; p < e_down; p += page_size)
		244	copy_or_zero(p, p + page_size);
		245	copy_or_zero(e_down, e);
		246	}
		247
222	} // namespace __sanitizer	248	} // namespace __sanitizer
223		249
224	SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify,	250	SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify,

lib/tsan/sanitizer_common/sanitizer_common_nolibc.cpp+4-3

 @@ -20,13 +20,14 @@ namespace __sanitizer {
 // The Windows implementations of these functions use the win32 API directly,
 // bypassing libc.
 #if !SANITIZER_WINDOWS
-#if SANITIZER_LINUX
+#  if SANITIZER_LINUX
 void LogMessageOnPrintf(const char *str) {}
-#endif
+void InitTlsSize() {}
+#  endif
 void WriteToSyslog(const char *buffer) {}
 void Abort() { internal__exit(1); }
 bool CreateDir(const char *pathname) { return false; }
-#endif // !SANITIZER_WINDOWS
+#endif  // !SANITIZER_WINDOWS
 #if !SANITIZER_WINDOWS && !SANITIZER_APPLE
 void ListOfModules::init() {}

lib/tsan/sanitizer_common/sanitizer_common_syscalls.inc+18-15

 @@ -48,6 +48,7 @@
 #if SANITIZER_LINUX
 #  include "sanitizer_libc.h"
+#  include "sanitizer_platform_limits_posix.h"
 #  define PRE_SYSCALL(name) \
     SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_pre_impl_##name
 @@ -2530,18 +2531,19 @@ PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) {
 #  if !SANITIZER_ANDROID &&                                                   \
       (defined(__i386) || defined(__x86_64) || defined(__mips64) ||           \
        defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \
-       defined(__loongarch__) || SANITIZER_RISCV64)
+       defined(__loongarch__) || SANITIZER_RISCV64 || defined(__sparc__))
-  if (data) {
+  long data_arg = ptrace_data_arg(request, addr, data);
+  if (data_arg) {
     if (request == ptrace_setregs) {
-      PRE_READ((void *)data, struct_user_regs_struct_sz);
+      PRE_READ((void *)data_arg, struct_user_regs_struct_sz);
     } else if (request == ptrace_setfpregs) {
-      PRE_READ((void *)data, struct_user_fpregs_struct_sz);
+      PRE_READ((void *)data_arg, struct_user_fpregs_struct_sz);
     } else if (request == ptrace_setfpxregs) {
-      PRE_READ((void *)data, struct_user_fpxregs_struct_sz);
+      PRE_READ((void *)data_arg, struct_user_fpxregs_struct_sz);
     } else if (request == ptrace_setsiginfo) {
-      PRE_READ((void *)data, siginfo_t_sz);
+      PRE_READ((void *)data_arg, siginfo_t_sz);
     } else if (request == ptrace_setregset) {
-      __sanitizer_iovec *iov = (__sanitizer_iovec *)data;
+      __sanitizer_iovec *iov = (__sanitizer_iovec *)data_arg;
       PRE_READ(iov->iov_base, iov->iov_len);
     }
   }
 @@ -2552,25 +2554,26 @@ POST_SYSCALL(ptrace)(long res, long request, long pid, long addr, long data) {
 #  if !SANITIZER_ANDROID &&                                                   \
       (defined(__i386) || defined(__x86_64) || defined(__mips64) ||           \
        defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \
-       defined(__loongarch__) || SANITIZER_RISCV64)
+       defined(__loongarch__) || SANITIZER_RISCV64 || defined(__sparc__))
-  if (res >= 0 && data) {
+  long data_arg = ptrace_data_arg(request, addr, data);
+  if (res >= 0 && data_arg) {
     // Note that this is different from the interceptor in
     // sanitizer_common_interceptors.inc.
     // PEEK* requests return resulting values through data pointer.
     if (request == ptrace_getregs) {
-      POST_WRITE((void *)data, struct_user_regs_struct_sz);
+      POST_WRITE((void *)data_arg, struct_user_regs_struct_sz);
     } else if (request == ptrace_getfpregs) {
-      POST_WRITE((void *)data, struct_user_fpregs_struct_sz);
+      POST_WRITE((void *)data_arg, struct_user_fpregs_struct_sz);
     } else if (request == ptrace_getfpxregs) {
-      POST_WRITE((void *)data, struct_user_fpxregs_struct_sz);
+      POST_WRITE((void *)data_arg, struct_user_fpxregs_struct_sz);
     } else if (request == ptrace_getsiginfo) {
-      POST_WRITE((void *)data, siginfo_t_sz);
+      POST_WRITE((void *)data_arg, siginfo_t_sz);
     } else if (request == ptrace_getregset) {
-      __sanitizer_iovec *iov = (__sanitizer_iovec *)data;
+      __sanitizer_iovec *iov = (__sanitizer_iovec *)data_arg;
       POST_WRITE(iov->iov_base, iov->iov_len);
     } else if (request == ptrace_peekdata || request == ptrace_peektext ||
                request == ptrace_peekuser) {
-      POST_WRITE((void *)data, sizeof(void *));
+      POST_WRITE((void *)data_arg, sizeof(void *));
     }
   }
 #  endif

lib/tsan/sanitizer_common/sanitizer_coverage_win_dll_thunk.cpp deleted-20

...	@@ -1,20 +0,0 @@
1	//===-- sanitizer_coverage_win_dll_thunk.cpp ------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file defines a family of thunks that should be statically linked into
10	// the DLLs that have instrumentation in order to delegate the calls to the
11	// shared runtime that lives in the main binary.
12	// See https://github.com/google/sanitizers/issues/209 for the details.
13	//===----------------------------------------------------------------------===//
14	#ifdef SANITIZER_DLL_THUNK
15	#include "sanitizer_win_dll_thunk.h"
16	// Sanitizer Coverage interface functions.
17	#define INTERFACE_FUNCTION(Name) INTERCEPT_SANITIZER_FUNCTION(Name)
18	#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
19	#include "sanitizer_coverage_interface.inc"
20	#endif // SANITIZER_DLL_THUNK

lib/tsan/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cpp deleted-26

...	@@ -1,26 +0,0 @@
1	//===-- sanitizer_coverage_win_dynamic_runtime_thunk.cpp ------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file defines things that need to be present in the application modules
10	// to interact with Sanitizer Coverage, when it is included in a dll.
11	//
12	//===----------------------------------------------------------------------===//
13	#ifdef SANITIZER_DYNAMIC_RUNTIME_THUNK
14	#define SANITIZER_IMPORT_INTERFACE 1
15	#include "sanitizer_win_defs.h"
16	// Define weak alias for all weak functions imported from sanitizer coverage.
17	#define INTERFACE_FUNCTION(Name)
18	#define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name)
19	#include "sanitizer_coverage_interface.inc"
20	#endif // SANITIZER_DYNAMIC_RUNTIME_THUNK
21
22	namespace __sanitizer {
23	// Add one, otherwise unused, external symbol to this object file so that the
24	// Visual C++ linker includes it and reads the .drective section.
25	void ForceWholeArchiveIncludeForSanCov() {}
26	}

lib/tsan/sanitizer_common/sanitizer_coverage_win_weak_interception.cpp deleted-23

...	@@ -1,23 +0,0 @@
1	//===-- sanitizer_coverage_win_weak_interception.cpp ----------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	// This module should be included in Sanitizer Coverage when it implemented as a
9	// shared library on Windows (dll), in order to delegate the calls of weak
10	// functions to the implementation in the main executable when a strong
11	// definition is provided.
12	//===----------------------------------------------------------------------===//
13	#ifdef SANITIZER_DYNAMIC
14	#include "sanitizer_win_weak_interception.h"
15	#include "sanitizer_interface_internal.h"
16	#include "sancov_flags.h"
17	// Check if strong definitions for weak functions are present in the main
18	// executable. If that is the case, override dll functions to point to strong
19	// implementations.
20	#define INTERFACE_FUNCTION(Name)
21	#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
22	#include "sanitizer_coverage_interface.inc"
23	#endif // SANITIZER_DYNAMIC

lib/tsan/sanitizer_common/sanitizer_deadlock_detector.h+1-1

...	@@ -120,7 +120,7 @@ class DeadlockDetectorTLS {	...	@@ -120,7 +120,7 @@ class DeadlockDetectorTLS {
120	u32 lock;	120	u32 lock;
121	u32 stk;	121	u32 stk;
122	};	122	};
123	LockWithContext all_locks_with_contexts_[64];	123	LockWithContext all_locks_with_contexts_[128];
124	uptr n_all_locks_;	124	uptr n_all_locks_;
125	};	125	};
126		126

lib/tsan/sanitizer_common/sanitizer_dense_map.h+41-29

 @@ -69,24 +69,14 @@ class DenseMapBase {
     setNumTombstones(0);
   }
+  /// Return true if the specified key is in the map, false otherwise.
+  bool contains(const KeyT &Key) const { return doFind(Key) != nullptr; }
   /// Return 1 if the specified key is in the map, 0 otherwise.
-  size_type count(const KeyT &Key) const {
+  size_type count(const KeyT &Key) const { return contains(Key) ? 1 : 0; }
-    const BucketT *TheBucket;
-    return LookupBucketFor(Key, TheBucket) ? 1 : 0;
-  }
-  value_type *find(const KeyT &Key) {
+  value_type *find(const KeyT &Key) { return doFind(Key); }
-    BucketT *TheBucket;
+  const value_type *find(const KeyT &Key) const { return doFind(Key); }
-    if (LookupBucketFor(Key, TheBucket))
-      return TheBucket;
-    return nullptr;
-  }
-  const value_type *find(const KeyT &Key) const {
-    const BucketT *TheBucket;
-    if (LookupBucketFor(Key, TheBucket))
-      return TheBucket;
-    return nullptr;
-  }
   /// Alternate version of find() which allows a different, and possibly
   /// less expensive, key type.
 @@ -95,25 +85,18 @@ class DenseMapBase {
   /// type used.
   template <class LookupKeyT>
   value_type *find_as(const LookupKeyT &Key) {
-    BucketT *TheBucket;
+    return doFind(Key);
-    if (LookupBucketFor(Key, TheBucket))
-      return TheBucket;
-    return nullptr;
   }
   template <class LookupKeyT>
   const value_type *find_as(const LookupKeyT &Key) const {
-    const BucketT *TheBucket;
+    return doFind(Key);
-    if (LookupBucketFor(Key, TheBucket))
-      return TheBucket;
-    return nullptr;
   }
   /// lookup - Return the entry for the specified key, or a default
   /// constructed value if no such entry exists.
   ValueT lookup(const KeyT &Key) const {
-    const BucketT *TheBucket;
+    if (const BucketT *Bucket = doFind(Key))
-    if (LookupBucketFor(Key, TheBucket))
+      return Bucket->getSecond();
-      return TheBucket->getSecond();
     return ValueT();
   }
 @@ -184,8 +167,8 @@ class DenseMapBase {
   }
   bool erase(const KeyT &Val) {
-    BucketT *TheBucket;
+    BucketT *TheBucket = doFind(Val);
-    if (!LookupBucketFor(Val, TheBucket))
+    if (!TheBucket)
       return false;  // not in map.
     TheBucket->getSecond().~ValueT();
 @@ -449,6 +432,35 @@ class DenseMapBase {
     return TheBucket;
   }
+  template <typename LookupKeyT>
+  BucketT *doFind(const LookupKeyT &Val) {
+    BucketT *BucketsPtr = getBuckets();
+    const unsigned NumBuckets = getNumBuckets();
+    if (NumBuckets == 0)
+      return nullptr;
+    const KeyT EmptyKey = getEmptyKey();
+    unsigned BucketNo = getHashValue(Val) & (NumBuckets - 1);
+    unsigned ProbeAmt = 1;
+    while (true) {
+      BucketT *Bucket = BucketsPtr + BucketNo;
+      if (LIKELY(KeyInfoT::isEqual(Val, Bucket->getFirst())))
+        return Bucket;
+      if (LIKELY(KeyInfoT::isEqual(Bucket->getFirst(), EmptyKey)))
+        return nullptr;
+      // Otherwise, it's a hash collision or a tombstone, continue quadratic
+      // probing.
+      BucketNo += ProbeAmt++;
+      BucketNo &= NumBuckets - 1;
+    }
+  }
+  template <typename LookupKeyT>
+  const BucketT *doFind(const LookupKeyT &Val) const {
+    return const_cast<DenseMapBase *>(this)->doFind(Val);
+  }
   /// LookupBucketFor - Lookup the appropriate bucket for Val, returning it in
   /// FoundBucket.  If the bucket contains the key and a value, this returns
   /// true, otherwise it returns a bucket with an empty marker or tombstone and

lib/tsan/sanitizer_common/sanitizer_errno.cpp+1

...	@@ -23,6 +23,7 @@ namespace __sanitizer {	...	@@ -23,6 +23,7 @@ namespace __sanitizer {
23	COMPILER_CHECK(errno_ENOMEM == ENOMEM);	23	COMPILER_CHECK(errno_ENOMEM == ENOMEM);
24	COMPILER_CHECK(errno_EBUSY == EBUSY);	24	COMPILER_CHECK(errno_EBUSY == EBUSY);
25	COMPILER_CHECK(errno_EINVAL == EINVAL);	25	COMPILER_CHECK(errno_EINVAL == EINVAL);
		26	COMPILER_CHECK(errno_ERANGE == ERANGE);
26		27
27	// EOWNERDEAD is not present in some older platforms.	28	// EOWNERDEAD is not present in some older platforms.
28	#if defined(EOWNERDEAD)	29	#if defined(EOWNERDEAD)

lib/tsan/sanitizer_common/sanitizer_errno_codes.h+1

...	@@ -24,6 +24,7 @@ namespace __sanitizer {	...	@@ -24,6 +24,7 @@ namespace __sanitizer {
24	#define errno_ENOMEM 12	24	#define errno_ENOMEM 12
25	#define errno_EBUSY 16	25	#define errno_EBUSY 16
26	#define errno_EINVAL 22	26	#define errno_EINVAL 22
		27	#define errno_ERANGE 34
27	#define errno_ENAMETOOLONG 36	28	#define errno_ENAMETOOLONG 36
28	#define errno_ENOSYS 38	29	#define errno_ENOSYS 38
29		30

lib/tsan/sanitizer_common/sanitizer_fuchsia.cpp+5-2

lib/tsan/sanitizer_common/sanitizer_getauxval.h+11-12

...	@@ -21,22 +21,21 @@	...	@@ -21,22 +21,21 @@
21		21
22	#if SANITIZER_LINUX \|\| SANITIZER_FUCHSIA	22	#if SANITIZER_LINUX \|\| SANITIZER_FUCHSIA
23		23
24	# if (__GLIBC_PREREQ(2, 16) \|\| (SANITIZER_ANDROID && __ANDROID_API__ >= 21) \|\| \	24	# if (__GLIBC_PREREQ(2, 16) \|\| SANITIZER_ANDROID \|\| SANITIZER_FUCHSIA) && \
25	SANITIZER_FUCHSIA) && \	25	!SANITIZER_GO
26	!SANITIZER_GO	26	# define SANITIZER_USE_GETAUXVAL 1
27	# define SANITIZER_USE_GETAUXVAL 1	27	# else
28	# else	28	# define SANITIZER_USE_GETAUXVAL 0
29	# define SANITIZER_USE_GETAUXVAL 0	29	# endif
30	# endif	30
31		31	# if SANITIZER_USE_GETAUXVAL
32	# if SANITIZER_USE_GETAUXVAL	32	# include <sys/auxv.h>
33	# include <sys/auxv.h>	33	# else
34	# else
35	// The weak getauxval definition allows to check for the function at runtime.	34	// The weak getauxval definition allows to check for the function at runtime.
36	// This is useful for Android, when compiled at a lower API level yet running	35	// This is useful for Android, when compiled at a lower API level yet running
37	// on a more recent platform that offers the function.	36	// on a more recent platform that offers the function.
38	extern "C" SANITIZER_WEAK_ATTRIBUTE unsigned long getauxval(unsigned long type);	37	extern "C" SANITIZER_WEAK_ATTRIBUTE unsigned long getauxval(unsigned long type);
39	# endif	38	# endif
40		39
41	#elif SANITIZER_NETBSD	40	#elif SANITIZER_NETBSD
42		41

lib/tsan/sanitizer_common/sanitizer_interface_internal.h+10

 @@ -49,6 +49,11 @@ __sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args);
 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
 __sanitizer_report_error_summary(const char *error_summary);
+// Returns size of dynamically allocated block. This function can be overridden
+// by the client.
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE __sanitizer::uptr
+__sanitizer_get_dtls_size(const void *tls_begin);
 SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump();
 SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage(
     const __sanitizer::uptr *pcs, const __sanitizer::uptr len);
 @@ -71,6 +76,11 @@ void __sanitizer_annotate_double_ended_contiguous_container(
     const void *old_container_beg, const void *old_container_end,
     const void *new_container_beg, const void *new_container_end);
 SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_copy_contiguous_container_annotations(const void *src_begin,
+                                                       const void *src_end,
+                                                       const void *dst_begin,
+                                                       const void *dst_end);
+SANITIZER_INTERFACE_ATTRIBUTE
 int __sanitizer_verify_contiguous_container(const void *beg, const void *mid,
                                             const void *end);
 SANITIZER_INTERFACE_ATTRIBUTE

lib/tsan/sanitizer_common/sanitizer_internal_defs.h+23-17

 @@ -138,19 +138,25 @@
 // in a portable way by the language itself.
 namespace __sanitizer {
-#if defined(_WIN64)
+#if defined(__UINTPTR_TYPE__)
+#  if defined(__arm__) && defined(__linux__)
+// Linux Arm headers redefine __UINTPTR_TYPE__ and disagree with clang/gcc.
+typedef unsigned int uptr;
+typedef int sptr;
+#  else
+typedef __UINTPTR_TYPE__ uptr;
+typedef __INTPTR_TYPE__ sptr;
+#  endif
+#elif defined(_WIN64)
 // 64-bit Windows uses LLP64 data model.
 typedef unsigned long long uptr;
 typedef signed long long sptr;
-#else
+#elif defined(_WIN32)
-#  if (SANITIZER_WORDSIZE == 64) || SANITIZER_APPLE || SANITIZER_WINDOWS
-typedef unsigned long uptr;
-typedef signed long sptr;
-#  else
 typedef unsigned int uptr;
 typedef signed int sptr;
-#  endif
+#else
-#endif  // defined(_WIN64)
+#  error Unsupported compiler, missing __UINTPTR_TYPE__
+#endif  // defined(__UINTPTR_TYPE__)
 #if defined(__x86_64__)
 // Since x32 uses ILP32 data model in 64-bit hardware mode, we must use
 // 64-bit pointer to unwind stack frame.
 @@ -194,16 +200,13 @@ typedef u64  OFF64_T;
 #ifdef __SIZE_TYPE__
 typedef __SIZE_TYPE__ usize;
 #else
-// Since we use this for operator new, usize must match the real size_t, but on
-// 32-bit Windows the definition of uptr does not actually match uintptr_t or
-// size_t because we are working around typedef mismatches for the (S)SIZE_T
-// types used in interception.h.
-// Until the definition of uptr has been fixed we have to special case Win32.
-#  if SANITIZER_WINDOWS && SANITIZER_WORDSIZE == 32
-typedef unsigned int usize;
-#  else
 typedef uptr usize;
-#  endif
+#endif
+#if defined(__s390__) && !defined(__s390x__)
+typedef long ssize;
+#else
+typedef sptr ssize;
 #endif
 typedef u64 tid_t;
 @@ -466,6 +469,9 @@ using namespace __sanitizer;
 namespace __msan {
 using namespace __sanitizer;
 }
+namespace __nsan {
+using namespace __sanitizer;
+}
 namespace __hwasan {
 using namespace __sanitizer;
 }

lib/tsan/sanitizer_common/sanitizer_libignore.cpp+15-13

 @@ -32,7 +32,7 @@ void LibIgnore::AddIgnoredLibrary(const char *name_templ) {
   lib->templ = internal_strdup(name_templ);
   lib->name = nullptr;
   lib->real_name = nullptr;
-  lib->loaded = false;
+  lib->range_id = kInvalidCodeRangeId;
 }
 void LibIgnore::OnLibraryLoaded(const char *name) {
 @@ -43,7 +43,7 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
       buf[0]) {
     for (uptr i = 0; i < count_; i++) {
       Lib *lib = &libs_[i];
-      if (!lib->loaded && (!lib->real_name) &&
+      if (!lib->loaded() && (!lib->real_name) &&
           TemplateMatch(lib->templ, name))
         lib->real_name = internal_strdup(buf.data());
     }
 @@ -70,28 +70,31 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
           Die();
         }
         loaded = true;
-        if (lib->loaded)
+        if (lib->loaded())
           continue;
         VReport(1,
                 "Matched called_from_lib suppression '%s' against library"
                 " '%s'\n",
                 lib->templ, mod.full_name());
-        lib->loaded = true;
         lib->name = internal_strdup(mod.full_name());
         const uptr idx =
             atomic_load(&ignored_ranges_count_, memory_order_relaxed);
         CHECK_LT(idx, ARRAY_SIZE(ignored_code_ranges_));
-        ignored_code_ranges_[idx].begin = range.beg;
+        ignored_code_ranges_[idx].OnLoad(range.beg, range.end);
-        ignored_code_ranges_[idx].end = range.end;
+        // Record the index of the ignored range.
+        lib->range_id = idx;
         atomic_store(&ignored_ranges_count_, idx + 1, memory_order_release);
         break;
       }
     }
-    if (lib->loaded && !loaded) {
+    if (lib->loaded() && !loaded) {
-      Report("%s: library '%s' that was matched against called_from_lib"
+      VReport(1,
-             " suppression '%s' is unloaded\n",
+              "%s: library '%s' that was matched against called_from_lib"
-             SanitizerToolName, lib->name, lib->templ);
+              " suppression '%s' is unloaded\n",
-      Die();
+              SanitizerToolName, lib->name, lib->templ);
+      // The library is unloaded so mark the ignored code range as unloaded.
+      ignored_code_ranges_[lib->range_id].OnUnload();
+      lib->range_id = kInvalidCodeRangeId;
     }
   }
 @@ -110,8 +113,7 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
         const uptr idx =
             atomic_load(&instrumented_ranges_count_, memory_order_relaxed);
         CHECK_LT(idx, ARRAY_SIZE(instrumented_code_ranges_));
-        instrumented_code_ranges_[idx].begin = range.beg;
+        instrumented_code_ranges_[idx].OnLoad(range.beg, range.end);
-        instrumented_code_ranges_[idx].end = range.end;
         atomic_store(&instrumented_ranges_count_, idx + 1,
                      memory_order_release);
       }

lib/tsan/sanitizer_common/sanitizer_libignore.h+23-12

...	@@ -49,25 +49,36 @@ class LibIgnore {	...	@@ -49,25 +49,36 @@ class LibIgnore {
49	bool IsPcInstrumented(uptr pc) const;	49	bool IsPcInstrumented(uptr pc) const;
50		50
51	private:	51	private:
		52	static const uptr kMaxIgnoredRanges = 128;
		53	static const uptr kMaxInstrumentedRanges = 1024;
		54	static const uptr kMaxLibs = 1024;
		55	static const uptr kInvalidCodeRangeId = -1;
		56
52	struct Lib {	57	struct Lib {
53	char *templ;	58	char *templ;
54	char *name;	59	char *name;
55	char *real_name; // target of symlink	60	char *real_name; // target of symlink
56	bool loaded;	61	uptr range_id;
		62	bool loaded() const { return range_id != kInvalidCodeRangeId; };
57	};	63	};
58		64
59	struct LibCodeRange {	65	struct LibCodeRange {
60	uptr begin;	66	bool IsInRange(uptr pc) const {
61	uptr end;	67	return (pc >= begin && pc < atomic_load(&end, memory_order_acquire));
62	};	68	}
63		69
64	inline bool IsInRange(uptr pc, const LibCodeRange &range) const {	70	void OnLoad(uptr b, uptr e) {
65	return (pc >= range.begin && pc < range.end);	71	begin = b;
66	}	72	atomic_store(&end, e, memory_order_release);
		73	}
67		74
68	static const uptr kMaxIgnoredRanges = 128;	75	void OnUnload() { atomic_store(&end, 0, memory_order_release); }
69	static const uptr kMaxInstrumentedRanges = 1024;	76
70	static const uptr kMaxLibs = 1024;	77	private:
		78	uptr begin;
		79	// A value of 0 means the associated module was unloaded.
		80	atomic_uintptr_t end;
		81	};
71		82
72	// Hot part:	83	// Hot part:
73	atomic_uintptr_t ignored_ranges_count_;	84	atomic_uintptr_t ignored_ranges_count_;
...	@@ -90,7 +101,7 @@ class LibIgnore {	...	@@ -90,7 +101,7 @@ class LibIgnore {
90	inline bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const {	101	inline bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const {
91	const uptr n = atomic_load(&ignored_ranges_count_, memory_order_acquire);	102	const uptr n = atomic_load(&ignored_ranges_count_, memory_order_acquire);
92	for (uptr i = 0; i < n; i++) {	103	for (uptr i = 0; i < n; i++) {
93	if (IsInRange(pc, ignored_code_ranges_[i])) {	104	if (ignored_code_ranges_[i].IsInRange(pc)) {
94	*pc_in_ignored_lib = true;	105	*pc_in_ignored_lib = true;
95	return true;	106	return true;
96	}	107	}
...	@@ -104,7 +115,7 @@ inline bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const {	...	@@ -104,7 +115,7 @@ inline bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const {
104	inline bool LibIgnore::IsPcInstrumented(uptr pc) const {	115	inline bool LibIgnore::IsPcInstrumented(uptr pc) const {
105	const uptr n = atomic_load(&instrumented_ranges_count_, memory_order_acquire);	116	const uptr n = atomic_load(&instrumented_ranges_count_, memory_order_acquire);
106	for (uptr i = 0; i < n; i++) {	117	for (uptr i = 0; i < n; i++) {
107	if (IsInRange(pc, instrumented_code_ranges_[i]))	118	if (instrumented_code_ranges_[i].IsInRange(pc))
108	return true;	119	return true;
109	}	120	}
110	return false;	121	return false;

lib/tsan/sanitizer_common/sanitizer_linux.cpp+325-71

...	@@ -107,7 +107,9 @@ extern struct ps_strings *__ps_strings;	...	@@ -107,7 +107,9 @@ extern struct ps_strings *__ps_strings;
107	# endif // SANITIZER_NETBSD	107	# endif // SANITIZER_NETBSD
108		108
109	# if SANITIZER_SOLARIS	109	# if SANITIZER_SOLARIS
		110	# include <stddef.h>
110	# include <stdlib.h>	111	# include <stdlib.h>
		112	# include <sys/frame.h>
111	# include <thread.h>	113	# include <thread.h>
112	# define environ _environ	114	# define environ _environ
113	# endif	115	# endif
...	@@ -132,9 +134,10 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE \| FUTEX_PRIVATE_FLAG;	...	@@ -132,9 +134,10 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE \| FUTEX_PRIVATE_FLAG;
132	// Are we using 32-bit or 64-bit Linux syscalls?	134	// Are we using 32-bit or 64-bit Linux syscalls?
133	// x32 (which defines __x86_64__) has SANITIZER_WORDSIZE == 32	135	// x32 (which defines __x86_64__) has SANITIZER_WORDSIZE == 32
134	// but it still needs to use 64-bit syscalls.	136	// but it still needs to use 64-bit syscalls.
135	# if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__powerpc64__) \|\| \	137	# if SANITIZER_LINUX && \
136	SANITIZER_WORDSIZE == 64 \|\| \	138	(defined(__x86_64__) \|\| defined(__powerpc64__) \|\| \
137	(defined(__mips__) && _MIPS_SIM == _ABIN32))	139	SANITIZER_WORDSIZE == 64 \|\| \
		140	(defined(__mips__) && defined(_ABIN32) && _MIPS_SIM == _ABIN32))
138	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1	141	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1
139	# else	142	# else
140	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0	143	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0
...	@@ -152,6 +155,8 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE \| FUTEX_PRIVATE_FLAG;	...	@@ -152,6 +155,8 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE \| FUTEX_PRIVATE_FLAG;
152		155
153	# if SANITIZER_FREEBSD	156	# if SANITIZER_FREEBSD
154	# define SANITIZER_USE_GETENTROPY 1	157	# define SANITIZER_USE_GETENTROPY 1
		158	extern "C" void __sys_mmap(void addr, size_t len, int prot, int flags, int fd,
		159	off_t offset);
155	# endif	160	# endif
156		161
157	namespace __sanitizer {	162	namespace __sanitizer {
...	@@ -160,33 +165,56 @@ void SetSigProcMask(__sanitizer_sigset_t set, __sanitizer_sigset_t oldset) {	...	@@ -160,33 +165,56 @@ void SetSigProcMask(__sanitizer_sigset_t set, __sanitizer_sigset_t oldset) {
160	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, oldset));	165	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, oldset));
161	}	166	}
162		167
		168	# if SANITIZER_LINUX
		169	// Deletes the specified signal from newset, if it is not present in oldset
		170	// Equivalently: newset[signum] = newset[signum] & oldset[signum]
		171	static void KeepUnblocked(__sanitizer_sigset_t &newset,
		172	__sanitizer_sigset_t &oldset, int signum) {
		173	// FIXME: https://github.com/google/sanitizers/issues/1816
		174	if (SANITIZER_ANDROID \|\| !internal_sigismember(&oldset, signum))
		175	internal_sigdelset(&newset, signum);
		176	}
		177	# endif
		178
163	// Block asynchronous signals	179	// Block asynchronous signals
164	void BlockSignals(__sanitizer_sigset_t *oldset) {	180	void BlockSignals(__sanitizer_sigset_t *oldset) {
165	__sanitizer_sigset_t set;	181	__sanitizer_sigset_t newset;
166	internal_sigfillset(&set);	182	internal_sigfillset(&newset);
167	# if SANITIZER_LINUX && !SANITIZER_ANDROID	183
		184	# if SANITIZER_LINUX
		185	__sanitizer_sigset_t currentset;
		186
		187	# if !SANITIZER_ANDROID
		188	// FIXME: https://github.com/google/sanitizers/issues/1816
		189	SetSigProcMask(NULL, &currentset);
		190
168	// Glibc uses SIGSETXID signal during setuid call. If this signal is blocked	191	// Glibc uses SIGSETXID signal during setuid call. If this signal is blocked
169	// on any thread, setuid call hangs.	192	// on any thread, setuid call hangs.
170	// See test/sanitizer_common/TestCases/Linux/setuid.c.	193	// See test/sanitizer_common/TestCases/Linux/setuid.c.
171	internal_sigdelset(&set, 33);	194	KeepUnblocked(newset, currentset, 33);
172	# endif	195	# endif // !SANITIZER_ANDROID
173	# if SANITIZER_LINUX	196
174	// Seccomp-BPF-sandboxed processes rely on SIGSYS to handle trapped syscalls.	197	// Seccomp-BPF-sandboxed processes rely on SIGSYS to handle trapped syscalls.
175	// If this signal is blocked, such calls cannot be handled and the process may	198	// If this signal is blocked, such calls cannot be handled and the process may
176	// hang.	199	// hang.
177	internal_sigdelset(&set, 31);	200	KeepUnblocked(newset, currentset, 31);
178		201
		202	# if !SANITIZER_ANDROID
179	// Don't block synchronous signals	203	// Don't block synchronous signals
180	internal_sigdelset(&set, SIGSEGV);	204	// but also don't unblock signals that the user had deliberately blocked.
181	internal_sigdelset(&set, SIGBUS);	205	// FIXME: https://github.com/google/sanitizers/issues/1816
182	internal_sigdelset(&set, SIGILL);	206	KeepUnblocked(newset, currentset, SIGSEGV);
183	internal_sigdelset(&set, SIGTRAP);	207	KeepUnblocked(newset, currentset, SIGBUS);
184	internal_sigdelset(&set, SIGABRT);	208	KeepUnblocked(newset, currentset, SIGILL);
185	internal_sigdelset(&set, SIGFPE);	209	KeepUnblocked(newset, currentset, SIGTRAP);
186	internal_sigdelset(&set, SIGPIPE);	210	KeepUnblocked(newset, currentset, SIGABRT);
187	# endif	211	KeepUnblocked(newset, currentset, SIGFPE);
		212	KeepUnblocked(newset, currentset, SIGPIPE);
		213	# endif //! SANITIZER_ANDROID
		214
		215	# endif // SANITIZER_LINUX
188		216
189	SetSigProcMask(&set, oldset);	217	SetSigProcMask(&newset, oldset);
190	}	218	}
191		219
192	ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) {	220	ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) {
...	@@ -218,7 +246,9 @@ ScopedBlockSignals::~ScopedBlockSignals() { SetSigProcMask(&saved_, nullptr); }	...	@@ -218,7 +246,9 @@ ScopedBlockSignals::~ScopedBlockSignals() { SetSigProcMask(&saved_, nullptr); }
218	# if !SANITIZER_S390	246	# if !SANITIZER_S390
219	uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,	247	uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
220	u64 offset) {	248	u64 offset) {
221	# if SANITIZER_FREEBSD \|\| SANITIZER_LINUX_USES_64BIT_SYSCALLS	249	# if SANITIZER_FREEBSD
		250	return (uptr)__sys_mmap(addr, length, prot, flags, fd, offset);
		251	# elif SANITIZER_LINUX_USES_64BIT_SYSCALLS
222	return internal_syscall(SYSCALL(mmap), (uptr)addr, length, prot, flags, fd,	252	return internal_syscall(SYSCALL(mmap), (uptr)addr, length, prot, flags, fd,
223	offset);	253	offset);
224	# else	254	# else
...	@@ -250,6 +280,11 @@ int internal_madvise(uptr addr, uptr length, int advice) {	...	@@ -250,6 +280,11 @@ int internal_madvise(uptr addr, uptr length, int advice) {
250	return internal_syscall(SYSCALL(madvise), addr, length, advice);	280	return internal_syscall(SYSCALL(madvise), addr, length, advice);
251	}	281	}
252		282
		283	# if SANITIZER_FREEBSD
		284	uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags) {
		285	return internal_syscall(SYSCALL(close_range), lowfd, highfd, flags);
		286	}
		287	# endif
253	uptr internal_close(fd_t fd) { return internal_syscall(SYSCALL(close), fd); }	288	uptr internal_close(fd_t fd) { return internal_syscall(SYSCALL(close), fd); }
254		289
255	uptr internal_open(const char *filename, int flags) {	290	uptr internal_open(const char *filename, int flags) {
...	@@ -395,8 +430,9 @@ uptr internal_stat(const char path, void buf) {	...	@@ -395,8 +430,9 @@ uptr internal_stat(const char path, void buf) {
395	AT_NO_AUTOMOUNT, STATX_BASIC_STATS, (uptr)&bufx);	430	AT_NO_AUTOMOUNT, STATX_BASIC_STATS, (uptr)&bufx);
396	statx_to_stat(&bufx, (struct stat *)buf);	431	statx_to_stat(&bufx, (struct stat *)buf);
397	return res;	432	return res;
398	# elif (SANITIZER_WORDSIZE == 64 \|\| SANITIZER_X32 \|\| \	433	# elif ( \
399	(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \	434	SANITIZER_WORDSIZE == 64 \|\| SANITIZER_X32 \|\| \
		435	(defined(__mips__) && defined(_ABIN32) && _MIPS_SIM == _ABIN32)) && \
400	!SANITIZER_SPARC	436	!SANITIZER_SPARC
401	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,	437	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
402	0);	438	0);
...	@@ -433,8 +469,9 @@ uptr internal_lstat(const char path, void buf) {	...	@@ -433,8 +469,9 @@ uptr internal_lstat(const char path, void buf) {
433	STATX_BASIC_STATS, (uptr)&bufx);	469	STATX_BASIC_STATS, (uptr)&bufx);
434	statx_to_stat(&bufx, (struct stat *)buf);	470	statx_to_stat(&bufx, (struct stat *)buf);
435	return res;	471	return res;
436	# elif (defined(_LP64) \|\| SANITIZER_X32 \|\| \	472	# elif ( \
437	(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \	473	defined(_LP64) \|\| SANITIZER_X32 \|\| \
		474	(defined(__mips__) && defined(_ABIN32) && _MIPS_SIM == _ABIN32)) && \
438	!SANITIZER_SPARC	475	!SANITIZER_SPARC
439	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,	476	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
440	AT_SYMLINK_NOFOLLOW);	477	AT_SYMLINK_NOFOLLOW);
...	@@ -721,6 +758,11 @@ static void GetArgsAndEnv(char *argv, char *envp) {	...	@@ -721,6 +758,11 @@ static void GetArgsAndEnv(char *argv, char *envp) {
721	# if !SANITIZER_GO	758	# if !SANITIZER_GO
722	if (&__libc_stack_end) {	759	if (&__libc_stack_end) {
723	uptr stack_end = (uptr )__libc_stack_end;	760	uptr stack_end = (uptr )__libc_stack_end;
		761	// Linux/sparc64 needs an adjustment, cf. glibc
		762	// sysdeps/sparc/sparc{32,64}/dl-machine.h (DL_STACK_END).
		763	# if SANITIZER_LINUX && defined(__sparc__)
		764	stack_end = &stack_end[16];
		765	# endif
724	// Normally argc can be obtained from *stack_end, however, on ARM glibc's	766	// Normally argc can be obtained from *stack_end, however, on ARM glibc's
725	// _start clobbers it:	767	// _start clobbers it:
726	// https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/arm/start.S;hb=refs/heads/release/2.31/master#l75	768	// https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/arm/start.S;hb=refs/heads/release/2.31/master#l75
...	@@ -1014,34 +1056,29 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {	...	@@ -1014,34 +1056,29 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {
1014		1056
1015	# if !SANITIZER_NETBSD	1057	# if !SANITIZER_NETBSD
1016	// ThreadLister implementation.	1058	// ThreadLister implementation.
1017	ThreadLister::ThreadLister(pid_t pid) : pid_(pid), buffer_(4096) {	1059	ThreadLister::ThreadLister(pid_t pid) : buffer_(4096) {
1018	char task_directory_path[80];	1060	task_path_.AppendF("/proc/%d/task", pid);
1019	internal_snprintf(task_directory_path, sizeof(task_directory_path),
1020	"/proc/%d/task/", pid);
1021	descriptor_ = internal_open(task_directory_path, O_RDONLY \| O_DIRECTORY);
1022	if (internal_iserror(descriptor_)) {
1023	Report("Can't open /proc/%d/task for reading.\n", pid);
1024	}
1025	}	1061	}
1026		1062
1027	ThreadLister::Result ThreadLister::ListThreads(	1063	ThreadLister::Result ThreadLister::ListThreads(
1028	InternalMmapVector<tid_t> *threads) {	1064	InternalMmapVector<tid_t> *threads) {
1029	if (internal_iserror(descriptor_))	1065	int descriptor = internal_open(task_path_.data(), O_RDONLY \| O_DIRECTORY);
		1066	if (internal_iserror(descriptor)) {
		1067	Report("Can't open %s for reading.\n", task_path_.data());
1030	return Error;	1068	return Error;
1031	internal_lseek(descriptor_, 0, SEEK_SET);	1069	}
		1070	auto cleanup = at_scope_exit([&] { internal_close(descriptor); });
1032	threads->clear();	1071	threads->clear();
1033		1072
1034	Result result = Ok;	1073	Result result = Ok;
1035	for (bool first_read = true;; first_read = false) {	1074	for (bool first_read = true;; first_read = false) {
1036	// Resize to max capacity if it was downsized by IsAlive.
1037	buffer_.resize(buffer_.capacity());
1038	CHECK_GE(buffer_.size(), 4096);	1075	CHECK_GE(buffer_.size(), 4096);
1039	uptr read = internal_getdents(	1076	uptr read = internal_getdents(
1040	descriptor_, (struct linux_dirent *)buffer_.data(), buffer_.size());	1077	descriptor, (struct linux_dirent *)buffer_.data(), buffer_.size());
1041	if (!read)	1078	if (!read)
1042	return result;	1079	return result;
1043	if (internal_iserror(read)) {	1080	if (internal_iserror(read)) {
1044	Report("Can't read directory entries from /proc/%d/task.\n", pid_);	1081	Report("Can't read directory entries from %s.\n", task_path_.data());
1045	return Error;	1082	return Error;
1046	}	1083	}
1047		1084
...	@@ -1079,26 +1116,33 @@ ThreadLister::Result ThreadLister::ListThreads(	...	@@ -1079,26 +1116,33 @@ ThreadLister::Result ThreadLister::ListThreads(
1079	}	1116	}
1080	}	1117	}
1081		1118
1082	bool ThreadLister::IsAlive(int tid) {	1119	const char *ThreadLister::LoadStatus(tid_t tid) {
		1120	status_path_.clear();
		1121	status_path_.AppendF("%s/%llu/status", task_path_.data(), tid);
		1122	auto cleanup = at_scope_exit([&] {
		1123	// Resize back to capacity if it is downsized by `ReadFileToVector`.
		1124	buffer_.resize(buffer_.capacity());
		1125	});
		1126	if (!ReadFileToVector(status_path_.data(), &buffer_) \|\| buffer_.empty())
		1127	return nullptr;
		1128	buffer_.push_back('\0');
		1129	return buffer_.data();
		1130	}
		1131
		1132	bool ThreadLister::IsAlive(tid_t tid) {
1083	// /proc/%d/task/%d/status uses same call to detect alive threads as	1133	// /proc/%d/task/%d/status uses same call to detect alive threads as
1084	// proc_task_readdir. See task_state implementation in Linux.	1134	// proc_task_readdir. See task_state implementation in Linux.
1085	char path[80];
1086	internal_snprintf(path, sizeof(path), "/proc/%d/task/%d/status", pid_, tid);
1087	if (!ReadFileToVector(path, &buffer_) \|\| buffer_.empty())
1088	return false;
1089	buffer_.push_back(0);
1090	static const char kPrefix[] = "\nPPid:";	1135	static const char kPrefix[] = "\nPPid:";
1091	const char *field = internal_strstr(buffer_.data(), kPrefix);	1136	const char *status = LoadStatus(tid);
		1137	if (!status)
		1138	return false;
		1139	const char *field = internal_strstr(status, kPrefix);
1092	if (!field)	1140	if (!field)
1093	return false;	1141	return false;
1094	field += internal_strlen(kPrefix);	1142	field += internal_strlen(kPrefix);
1095	return (int)internal_atoll(field) != 0;	1143	return (int)internal_atoll(field) != 0;
1096	}	1144	}
1097		1145
1098	ThreadLister::~ThreadLister() {
1099	if (!internal_iserror(descriptor_))
1100	internal_close(descriptor_);
1101	}
1102	# endif	1146	# endif
1103		1147
1104	# if SANITIZER_WORDSIZE == 32	1148	# if SANITIZER_WORDSIZE == 32
...	@@ -1808,11 +1852,6 @@ int internal_uname(struct utsname *buf) {	...	@@ -1808,11 +1852,6 @@ int internal_uname(struct utsname *buf) {
1808	# endif	1852	# endif
1809		1853
1810	# if SANITIZER_ANDROID	1854	# if SANITIZER_ANDROID
1811	# if __ANDROID_API__ < 21
1812	extern "C" __attribute__((weak)) int dl_iterate_phdr(
1813	int ()(struct dl_phdr_info , size_t, void ), void );
1814	# endif
1815
1816	static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,	1855	static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,
1817	void *data) {	1856	void *data) {
1818	// Any name starting with "lib" indicates a bug in L where library base names	1857	// Any name starting with "lib" indicates a bug in L where library base names
...	@@ -1828,9 +1867,7 @@ static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,	...	@@ -1828,9 +1867,7 @@ static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,
1828	static atomic_uint32_t android_api_level;	1867	static atomic_uint32_t android_api_level;
1829		1868
1830	static AndroidApiLevel AndroidDetectApiLevelStatic() {	1869	static AndroidApiLevel AndroidDetectApiLevelStatic() {
1831	# if __ANDROID_API__ <= 19	1870	# if __ANDROID_API__ <= 22
1832	return ANDROID_KITKAT;
1833	# elif __ANDROID_API__ <= 22
1834	return ANDROID_LOLLIPOP_MR1;	1871	return ANDROID_LOLLIPOP_MR1;
1835	# else	1872	# else
1836	return ANDROID_POST_LOLLIPOP;	1873	return ANDROID_POST_LOLLIPOP;
...	@@ -1838,8 +1875,6 @@ static AndroidApiLevel AndroidDetectApiLevelStatic() {	...	@@ -1838,8 +1875,6 @@ static AndroidApiLevel AndroidDetectApiLevelStatic() {
1838	}	1875	}
1839		1876
1840	static AndroidApiLevel AndroidDetectApiLevel() {	1877	static AndroidApiLevel AndroidDetectApiLevel() {
1841	if (!&dl_iterate_phdr)
1842	return ANDROID_KITKAT; // K or lower
1843	bool base_name_seen = false;	1878	bool base_name_seen = false;
1844	dl_iterate_phdr(dl_iterate_phdr_test_cb, &base_name_seen);	1879	dl_iterate_phdr(dl_iterate_phdr_test_cb, &base_name_seen);
1845	if (base_name_seen)	1880	if (base_name_seen)
...	@@ -2014,6 +2049,18 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -2014,6 +2049,18 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
2014	return Unknown;	2049	return Unknown;
2015	return esr & ESR_ELx_WNR ? Write : Read;	2050	return esr & ESR_ELx_WNR ? Write : Read;
2016	# elif defined(__loongarch__)	2051	# elif defined(__loongarch__)
		2052	// In the musl environment, the Linux kernel uapi sigcontext.h is not
		2053	// included in signal.h. To avoid missing the SC_ADDRERR_{RD,WR} macros,
		2054	// copy them here. The LoongArch Linux kernel uapi is already stable,
		2055	// so there's no need to worry about the value changing.
		2056	# ifndef SC_ADDRERR_RD
		2057	// Address error was due to memory load
		2058	# define SC_ADDRERR_RD (1 << 30)
		2059	# endif
		2060	# ifndef SC_ADDRERR_WR
		2061	// Address error was due to memory store
		2062	# define SC_ADDRERR_WR (1 << 31)
		2063	# endif
2017	u32 flags = ucontext->uc_mcontext.__flags;	2064	u32 flags = ucontext->uc_mcontext.__flags;
2018	if (flags & SC_ADDRERR_RD)	2065	if (flags & SC_ADDRERR_RD)
2019	return SignalContext::Read;	2066	return SignalContext::Read;
...	@@ -2154,8 +2201,26 @@ bool SignalContext::IsTrueFaultingAddress() const {	...	@@ -2154,8 +2201,26 @@ bool SignalContext::IsTrueFaultingAddress() const {
2154	UNUSED	2201	UNUSED
2155	static const char *RegNumToRegName(int reg) {	2202	static const char *RegNumToRegName(int reg) {
2156	switch (reg) {	2203	switch (reg) {
2157	# if SANITIZER_LINUX	2204	# if SANITIZER_LINUX && SANITIZER_GLIBC \|\| SANITIZER_NETBSD
2158	# if defined(__x86_64__)	2205	# if defined(__x86_64__)
		2206	# if SANITIZER_NETBSD
		2207	# define REG_RAX _REG_RAX
		2208	# define REG_RBX _REG_RBX
		2209	# define REG_RCX _REG_RCX
		2210	# define REG_RDX _REG_RDX
		2211	# define REG_RDI _REG_RDI
		2212	# define REG_RSI _REG_RSI
		2213	# define REG_RBP _REG_RBP
		2214	# define REG_RSP _REG_RSP
		2215	# define REG_R8 _REG_R8
		2216	# define REG_R9 _REG_R9
		2217	# define REG_R10 _REG_R10
		2218	# define REG_R11 _REG_R11
		2219	# define REG_R12 _REG_R12
		2220	# define REG_R13 _REG_R13
		2221	# define REG_R14 _REG_R14
		2222	# define REG_R15 _REG_R15
		2223	# endif
2159	case REG_RAX:	2224	case REG_RAX:
2160	return "rax";	2225	return "rax";
2161	case REG_RBX:	2226	case REG_RBX:
...	@@ -2189,6 +2254,16 @@ static const char *RegNumToRegName(int reg) {	...	@@ -2189,6 +2254,16 @@ static const char *RegNumToRegName(int reg) {
2189	case REG_R15:	2254	case REG_R15:
2190	return "r15";	2255	return "r15";
2191	# elif defined(__i386__)	2256	# elif defined(__i386__)
		2257	# if SANITIZER_NETBSD
		2258	# define REG_EAX _REG_EAX
		2259	# define REG_EBX _REG_EBX
		2260	# define REG_ECX _REG_ECX
		2261	# define REG_EDX _REG_EDX
		2262	# define REG_EDI _REG_EDI
		2263	# define REG_ESI _REG_ESI
		2264	# define REG_EBP _REG_EBP
		2265	# define REG_ESP _REG_ESP
		2266	# endif
2192	case REG_EAX:	2267	case REG_EAX:
2193	return "eax";	2268	return "eax";
2194	case REG_EBX:	2269	case REG_EBX:
...	@@ -2205,32 +2280,170 @@ static const char *RegNumToRegName(int reg) {	...	@@ -2205,32 +2280,170 @@ static const char *RegNumToRegName(int reg) {
2205	return "ebp";	2280	return "ebp";
2206	case REG_ESP:	2281	case REG_ESP:
2207	return "esp";	2282	return "esp";
		2283	# elif defined(__arm__)
		2284	# ifdef MAKE_CASE
		2285	# undef MAKE_CASE
		2286	# endif
		2287	# define REG_STR(reg) #reg
		2288	# define MAKE_CASE(N) \
		2289	case REG_R##N: \
		2290	return REG_STR(r##N)
		2291	MAKE_CASE(0);
		2292	MAKE_CASE(1);
		2293	MAKE_CASE(2);
		2294	MAKE_CASE(3);
		2295	MAKE_CASE(4);
		2296	MAKE_CASE(5);
		2297	MAKE_CASE(6);
		2298	MAKE_CASE(7);
		2299	MAKE_CASE(8);
		2300	MAKE_CASE(9);
		2301	MAKE_CASE(10);
		2302	MAKE_CASE(11);
		2303	MAKE_CASE(12);
		2304	case REG_R13:
		2305	return "sp";
		2306	case REG_R14:
		2307	return "lr";
		2308	case REG_R15:
		2309	return "pc";
		2310	# elif defined(__aarch64__)
		2311	# define REG_STR(reg) #reg
		2312	# define MAKE_CASE(N) \
		2313	case N: \
		2314	return REG_STR(x##N)
		2315	MAKE_CASE(0);
		2316	MAKE_CASE(1);
		2317	MAKE_CASE(2);
		2318	MAKE_CASE(3);
		2319	MAKE_CASE(4);
		2320	MAKE_CASE(5);
		2321	MAKE_CASE(6);
		2322	MAKE_CASE(7);
		2323	MAKE_CASE(8);
		2324	MAKE_CASE(9);
		2325	MAKE_CASE(10);
		2326	MAKE_CASE(11);
		2327	MAKE_CASE(12);
		2328	MAKE_CASE(13);
		2329	MAKE_CASE(14);
		2330	MAKE_CASE(15);
		2331	MAKE_CASE(16);
		2332	MAKE_CASE(17);
		2333	MAKE_CASE(18);
		2334	MAKE_CASE(19);
		2335	MAKE_CASE(20);
		2336	MAKE_CASE(21);
		2337	MAKE_CASE(22);
		2338	MAKE_CASE(23);
		2339	MAKE_CASE(24);
		2340	MAKE_CASE(25);
		2341	MAKE_CASE(26);
		2342	MAKE_CASE(27);
		2343	MAKE_CASE(28);
		2344	case 29:
		2345	return "fp";
		2346	case 30:
		2347	return "lr";
		2348	case 31:
		2349	return "sp";
2208	# endif	2350	# endif
2209	# endif	2351	# endif // SANITIZER_LINUX && SANITIZER_GLIBC
2210	default:	2352	default:
2211	return NULL;	2353	return NULL;
2212	}	2354	}
2213	return NULL;	2355	return NULL;
2214	}	2356	}
2215		2357
2216	# if SANITIZER_LINUX	2358	# if ((SANITIZER_LINUX && SANITIZER_GLIBC) \|\| SANITIZER_NETBSD) && \
		2359	(defined(__arm__) \|\| defined(__aarch64__))
		2360	static uptr GetArmRegister(ucontext_t *ctx, int RegNum) {
		2361	switch (RegNum) {
		2362	# if defined(__arm__) && !SANITIZER_NETBSD
		2363	# ifdef MAKE_CASE
		2364	# undef MAKE_CASE
		2365	# endif
		2366	# define MAKE_CASE(N) \
		2367	case REG_R##N: \
		2368	return ctx->uc_mcontext.arm_r##N
		2369	MAKE_CASE(0);
		2370	MAKE_CASE(1);
		2371	MAKE_CASE(2);
		2372	MAKE_CASE(3);
		2373	MAKE_CASE(4);
		2374	MAKE_CASE(5);
		2375	MAKE_CASE(6);
		2376	MAKE_CASE(7);
		2377	MAKE_CASE(8);
		2378	MAKE_CASE(9);
		2379	MAKE_CASE(10);
		2380	case REG_R11:
		2381	return ctx->uc_mcontext.arm_fp;
		2382	case REG_R12:
		2383	return ctx->uc_mcontext.arm_ip;
		2384	case REG_R13:
		2385	return ctx->uc_mcontext.arm_sp;
		2386	case REG_R14:
		2387	return ctx->uc_mcontext.arm_lr;
		2388	case REG_R15:
		2389	return ctx->uc_mcontext.arm_pc;
		2390	# elif defined(__aarch64__)
		2391	# if SANITIZER_LINUX
		2392	case 0 ... 30:
		2393	return ctx->uc_mcontext.regs[RegNum];
		2394	case 31:
		2395	return ctx->uc_mcontext.sp;
		2396	# elif SANITIZER_NETBSD
		2397	case 0 ... 31:
		2398	return ctx->uc_mcontext.__gregs[RegNum];
		2399	# endif
		2400	# endif
		2401	default:
		2402	return 0;
		2403	}
		2404	return 0;
		2405	}
		2406	# endif // SANITIZER_LINUX && SANITIZER_GLIBC && (defined(__arm__) \|\|
		2407	// defined(__aarch64__))
		2408
2217	UNUSED	2409	UNUSED
2218	static void DumpSingleReg(ucontext_t *ctx, int RegNum) {	2410	static void DumpSingleReg(ucontext_t *ctx, int RegNum) {
2219	const char *RegName = RegNumToRegName(RegNum);	2411	const char *RegName = RegNumToRegName(RegNum);
		2412	# if SANITIZER_LINUX && SANITIZER_GLIBC \|\| SANITIZER_NETBSD
2220	# if defined(__x86_64__)	2413	# if defined(__x86_64__)
2221	Printf("%s%s = 0x%016llx ", internal_strlen(RegName) == 2 ? " " : "",	2414	Printf("%s%s = 0x%016llx ", internal_strlen(RegName) == 2 ? " " : "",
2222	RegName, ctx->uc_mcontext.gregs[RegNum]);	2415	RegName,
		2416	# if SANITIZER_LINUX
		2417	ctx->uc_mcontext.gregs[RegNum]
		2418	# elif SANITIZER_NETBSD
		2419	ctx->uc_mcontext.__gregs[RegNum]
		2420	# endif
		2421	);
2223	# elif defined(__i386__)	2422	# elif defined(__i386__)
2224	Printf("%s = 0x%08x ", RegName, ctx->uc_mcontext.gregs[RegNum]);	2423	Printf("%s = 0x%08x ", RegName,
		2424	# if SANITIZER_LINUX
		2425	ctx->uc_mcontext.gregs[RegNum]
		2426	# elif SANITIZER_NETBSD
		2427	ctx->uc_mcontext.__gregs[RegNum]
		2428	# endif
		2429	);
		2430	# elif defined(__arm__)
		2431	Printf("%s%s = 0x%08zx ", internal_strlen(RegName) == 2 ? " " : "", RegName,
		2432	GetArmRegister(ctx, RegNum));
		2433	# elif defined(__aarch64__)
		2434	Printf("%s%s = 0x%016zx ", internal_strlen(RegName) == 2 ? " " : "", RegName,
		2435	GetArmRegister(ctx, RegNum));
2225	# else	2436	# else
2226	(void)RegName;	2437	(void)RegName;
2227	# endif	2438	# endif
2228	}	2439	# else
		2440	(void)RegName;
2229	# endif	2441	# endif
		2442	}
2230		2443
2231	void SignalContext::DumpAllRegisters(void *context) {	2444	void SignalContext::DumpAllRegisters(void *context) {
2232	ucontext_t ucontext = (ucontext_t )context;	2445	ucontext_t ucontext = (ucontext_t )context;
2233	# if SANITIZER_LINUX	2446	# if SANITIZER_LINUX && SANITIZER_GLIBC \|\| SANITIZER_NETBSD
2234	# if defined(__x86_64__)	2447	# if defined(__x86_64__)
2235	Report("Register values:\n");	2448	Report("Register values:\n");
2236	DumpSingleReg(ucontext, REG_RAX);	2449	DumpSingleReg(ucontext, REG_RAX);
...	@@ -2269,6 +2482,35 @@ void SignalContext::DumpAllRegisters(void *context) {	...	@@ -2269,6 +2482,35 @@ void SignalContext::DumpAllRegisters(void *context) {
2269	DumpSingleReg(ucontext, REG_EBP);	2482	DumpSingleReg(ucontext, REG_EBP);
2270	DumpSingleReg(ucontext, REG_ESP);	2483	DumpSingleReg(ucontext, REG_ESP);
2271	Printf("\n");	2484	Printf("\n");
		2485	# elif defined(__arm__) && !SANITIZER_NETBSD
		2486	Report("Register values:\n");
		2487	DumpSingleReg(ucontext, REG_R0);
		2488	DumpSingleReg(ucontext, REG_R1);
		2489	DumpSingleReg(ucontext, REG_R2);
		2490	DumpSingleReg(ucontext, REG_R3);
		2491	Printf("\n");
		2492	DumpSingleReg(ucontext, REG_R4);
		2493	DumpSingleReg(ucontext, REG_R5);
		2494	DumpSingleReg(ucontext, REG_R6);
		2495	DumpSingleReg(ucontext, REG_R7);
		2496	Printf("\n");
		2497	DumpSingleReg(ucontext, REG_R8);
		2498	DumpSingleReg(ucontext, REG_R9);
		2499	DumpSingleReg(ucontext, REG_R10);
		2500	DumpSingleReg(ucontext, REG_R11);
		2501	Printf("\n");
		2502	DumpSingleReg(ucontext, REG_R12);
		2503	DumpSingleReg(ucontext, REG_R13);
		2504	DumpSingleReg(ucontext, REG_R14);
		2505	DumpSingleReg(ucontext, REG_R15);
		2506	Printf("\n");
		2507	# elif defined(__aarch64__)
		2508	Report("Register values:\n");
		2509	for (int i = 0; i <= 31; ++i) {
		2510	DumpSingleReg(ucontext, i);
		2511	if (i % 4 == 3)
		2512	Printf("\n");
		2513	}
2272	# else	2514	# else
2273	(void)ucontext;	2515	(void)ucontext;
2274	# endif	2516	# endif
...	@@ -2310,6 +2552,8 @@ void SignalContext::DumpAllRegisters(void *context) {	...	@@ -2310,6 +2552,8 @@ void SignalContext::DumpAllRegisters(void *context) {
2310	# else	2552	# else
2311	(void)ucontext;	2553	(void)ucontext;
2312	# endif	2554	# endif
		2555	# else
		2556	(void)ucontext;
2313	# endif	2557	# endif
2314	// FIXME: Implement this for other OSes and architectures.	2558	// FIXME: Implement this for other OSes and architectures.
2315	}	2559	}
...	@@ -2404,7 +2648,19 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {	...	@@ -2404,7 +2648,19 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2404	# if SANITIZER_SOLARIS	2648	# if SANITIZER_SOLARIS
2405	ucontext_t ucontext = (ucontext_t )context;	2649	ucontext_t ucontext = (ucontext_t )context;
2406	*pc = ucontext->uc_mcontext.gregs[REG_PC];	2650	*pc = ucontext->uc_mcontext.gregs[REG_PC];
2407	*sp = ucontext->uc_mcontext.gregs[REG_O6] + STACK_BIAS;	2651	*sp = ucontext->uc_mcontext.gregs[REG_SP] + STACK_BIAS;
		2652	// Avoid SEGV when dereferencing sp on stack overflow with non-faulting load.
		2653	// This requires a SPARC V9 CPU. Cannot use #ASI_PNF here: only supported
		2654	// since clang-19.
		2655	# if defined(__sparcv9)
		2656	asm("ldxa [%[fp]] 0x82, %[bp]"
		2657	# else
		2658	asm("lduwa [%[fp]] 0x82, %[bp]"
		2659	# endif
		2660	: [bp] "=r"(*bp)
		2661	: [fp] "r"(&((struct frame )sp)->fr_savfp));
		2662	if (*bp)
		2663	*bp += STACK_BIAS;
2408	# else	2664	# else
2409	// Historical BSDism here.	2665	// Historical BSDism here.
2410	struct sigcontext scontext = (struct sigcontext )context;	2666	struct sigcontext scontext = (struct sigcontext )context;
...	@@ -2415,8 +2671,8 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {	...	@@ -2415,8 +2671,8 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2415	*pc = scontext->si_regs.pc;	2671	*pc = scontext->si_regs.pc;
2416	*sp = scontext->si_regs.u_regs[14];	2672	*sp = scontext->si_regs.u_regs[14];
2417	# endif	2673	# endif
2418	# endif
2419	bp = (uptr)((uhwptr )*sp)[14] + STACK_BIAS;	2674	bp = (uptr)((uhwptr )*sp)[14] + STACK_BIAS;
		2675	# endif
2420	# elif defined(__mips__)	2676	# elif defined(__mips__)
2421	ucontext_t ucontext = (ucontext_t )context;	2677	ucontext_t ucontext = (ucontext_t )context;
2422	*pc = ucontext->uc_mcontext.pc;	2678	*pc = ucontext->uc_mcontext.pc;
...	@@ -2459,9 +2715,7 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {	...	@@ -2459,9 +2715,7 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2459		2715
2460	void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); }	2716	void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); }
2461		2717
2462	void InitializePlatformEarly() {	2718	void InitializePlatformEarly() { InitTlsSize(); }
2463	// Do nothing.
2464	}
2465		2719
2466	void CheckASLR() {	2720	void CheckASLR() {
2467	# if SANITIZER_NETBSD	2721	# if SANITIZER_NETBSD

lib/tsan/sanitizer_common/sanitizer_linux.h+4-4

 @@ -97,19 +97,19 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg);
 class ThreadLister {
  public:
   explicit ThreadLister(pid_t pid);
-  ~ThreadLister();
   enum Result {
     Error,
     Incomplete,
     Ok,
   };
   Result ListThreads(InternalMmapVector<tid_t> *threads);
+  const char *LoadStatus(tid_t tid);
  private:
-  bool IsAlive(int tid);
+  bool IsAlive(tid_t tid);
-  pid_t pid_;
+  InternalScopedString task_path_;
-  int descriptor_ = -1;
+  InternalScopedString status_path_;
   InternalMmapVector<char> buffer_;
 };

lib/tsan/sanitizer_common/sanitizer_linux_libcdep.cpp+171-129

...	@@ -40,6 +40,10 @@	...	@@ -40,6 +40,10 @@
40	# include <sys/resource.h>	40	# include <sys/resource.h>
41	# include <syslog.h>	41	# include <syslog.h>
42		42
		43	# if SANITIZER_GLIBC
		44	# include <gnu/libc-version.h>
		45	# endif
		46
43	# if !defined(ElfW)	47	# if !defined(ElfW)
44	# define ElfW(type) Elf_##type	48	# define ElfW(type) Elf_##type
45	# endif	49	# endif
...	@@ -53,7 +57,7 @@	...	@@ -53,7 +57,7 @@
53	// that, it was never implemented. So just define it to zero.	57	// that, it was never implemented. So just define it to zero.
54	# undef MAP_NORESERVE	58	# undef MAP_NORESERVE
55	# define MAP_NORESERVE 0	59	# define MAP_NORESERVE 0
56	extern const Elf_Auxinfo *__elf_aux_vector;	60	extern const Elf_Auxinfo *__elf_aux_vector __attribute__((weak));
57	extern "C" int __sys_sigaction(int signum, const struct sigaction *act,	61	extern "C" int __sys_sigaction(int signum, const struct sigaction *act,
58	struct sigaction *oldact);	62	struct sigaction *oldact);
59	# endif	63	# endif
...	@@ -196,27 +200,6 @@ bool SetEnv(const char name, const char value) {	...	@@ -196,27 +200,6 @@ bool SetEnv(const char name, const char value) {
196	}	200	}
197	# endif	201	# endif
198		202
199	__attribute__((unused)) static bool GetLibcVersion(int major, int minor,
200	int *patch) {
201	# ifdef _CS_GNU_LIBC_VERSION
202	char buf[64];
203	uptr len = confstr(_CS_GNU_LIBC_VERSION, buf, sizeof(buf));
204	if (len >= sizeof(buf))
205	return false;
206	buf[len] = 0;
207	static const char kGLibC[] = "glibc ";
208	if (internal_strncmp(buf, kGLibC, sizeof(kGLibC) - 1) != 0)
209	return false;
210	const char *p = buf + sizeof(kGLibC) - 1;
211	*major = internal_simple_strtoll(p, &p, 10);
212	minor = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
213	patch = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
214	return true;
215	# else
216	return false;
217	# endif
218	}
219
220	// True if we can use dlpi_tls_data. glibc before 2.25 may leave NULL (BZ	203	// True if we can use dlpi_tls_data. glibc before 2.25 may leave NULL (BZ
221	// #19826) so dlpi_tls_data cannot be used.	204	// #19826) so dlpi_tls_data cannot be used.
222	//	205	//
...	@@ -226,112 +209,166 @@ __attribute__((unused)) static bool GetLibcVersion(int major, int minor,	...	@@ -226,112 +209,166 @@ __attribute__((unused)) static bool GetLibcVersion(int major, int minor,
226	__attribute__((unused)) static int g_use_dlpi_tls_data;	209	__attribute__((unused)) static int g_use_dlpi_tls_data;
227		210
228	# if SANITIZER_GLIBC && !SANITIZER_GO	211	# if SANITIZER_GLIBC && !SANITIZER_GO
229	__attribute__((unused)) static size_t g_tls_size;	212	static void GetGLibcVersion(int major, int minor, int *patch) {
230	void InitTlsSize() {	213	const char *p = gnu_get_libc_version();
231	int major, minor, patch;	214	*major = internal_simple_strtoll(p, &p, 10);
232	g_use_dlpi_tls_data =	215	// Caller does not expect anything else.
233	GetLibcVersion(&major, &minor, &patch) && major == 2 && minor >= 25;	216	CHECK_EQ(*major, 2);
234		217	minor = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
235	# if defined(__aarch64__) \|\| defined(__x86_64__) \|\| \	218	patch = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
236	defined(__powerpc64__) \|\| defined(__loongarch__)
237	void *get_tls_static_info = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");
238	size_t tls_align;
239	((void ()(size_t , size_t *))get_tls_static_info)(&g_tls_size, &tls_align);
240	# endif
241	}	219	}
242	# else
243	void InitTlsSize() {}
244	# endif // SANITIZER_GLIBC && !SANITIZER_GO
245
246	// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage
247	// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan
248	// to get the pointer to thread-specific data keys in the thread control block.
249	# if (SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_SOLARIS) && \
250	!SANITIZER_ANDROID && !SANITIZER_GO
251	// sizeof(struct pthread) from glibc.
252	static atomic_uintptr_t thread_descriptor_size;
253		220
254	static uptr ThreadDescriptorSizeFallback() {	221	static uptr ThreadDescriptorSizeFallback() {
255	uptr val = 0;	222	# if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__arm__) \|\| \
256	# if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__arm__)	223	SANITIZER_RISCV64
257	int major;	224	int major;
258	int minor;	225	int minor;
259	int patch;	226	int patch;
260	if (GetLibcVersion(&major, &minor, &patch) && major == 2) {	227	GetGLibcVersion(&major, &minor, &patch);
261	/* sizeof(struct pthread) values from various glibc versions. */	228	# endif
262	if (SANITIZER_X32)	229
263	val = 1728; // Assume only one particular version for x32.	230	# if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__arm__)
264	// For ARM sizeof(struct pthread) changed in Glibc 2.23.	231	/* sizeof(struct pthread) values from various glibc versions. */
265	else if (SANITIZER_ARM)	232	if (SANITIZER_X32)
266	val = minor <= 22 ? 1120 : 1216;	233	return 1728; // Assume only one particular version for x32.
267	else if (minor <= 3)	234	// For ARM sizeof(struct pthread) changed in Glibc 2.23.
268	val = FIRST_32_SECOND_64(1104, 1696);	235	if (SANITIZER_ARM)
269	else if (minor == 4)	236	return minor <= 22 ? 1120 : 1216;
270	val = FIRST_32_SECOND_64(1120, 1728);	237	if (minor <= 3)
271	else if (minor == 5)	238	return FIRST_32_SECOND_64(1104, 1696);
272	val = FIRST_32_SECOND_64(1136, 1728);	239	if (minor == 4)
273	else if (minor <= 9)	240	return FIRST_32_SECOND_64(1120, 1728);
274	val = FIRST_32_SECOND_64(1136, 1712);	241	if (minor == 5)
275	else if (minor == 10)	242	return FIRST_32_SECOND_64(1136, 1728);
276	val = FIRST_32_SECOND_64(1168, 1776);	243	if (minor <= 9)
277	else if (minor == 11 \|\| (minor == 12 && patch == 1))	244	return FIRST_32_SECOND_64(1136, 1712);
278	val = FIRST_32_SECOND_64(1168, 2288);	245	if (minor == 10)
279	else if (minor <= 14)	246	return FIRST_32_SECOND_64(1168, 1776);
280	val = FIRST_32_SECOND_64(1168, 2304);	247	if (minor == 11 \|\| (minor == 12 && patch == 1))
281	else if (minor < 32) // Unknown version	248	return FIRST_32_SECOND_64(1168, 2288);
282	val = FIRST_32_SECOND_64(1216, 2304);	249	if (minor <= 14)
283	else // minor == 32	250	return FIRST_32_SECOND_64(1168, 2304);
284	val = FIRST_32_SECOND_64(1344, 2496);	251	if (minor < 32) // Unknown version
285	}	252	return FIRST_32_SECOND_64(1216, 2304);
286	# elif defined(__s390__) \|\| defined(__sparc__)	253	// minor == 32
		254	return FIRST_32_SECOND_64(1344, 2496);
		255	# endif
		256
		257	# if SANITIZER_RISCV64
		258	// TODO: consider adding an optional runtime check for an unknown (untested)
		259	// glibc version
		260	if (minor <= 28) // WARNING: the highest tested version is 2.29
		261	return 1772; // no guarantees for this one
		262	if (minor <= 31)
		263	return 1772; // tested against glibc 2.29, 2.31
		264	return 1936; // tested against glibc 2.32
		265	# endif
		266
		267	# if defined(__s390__) \|\| defined(__sparc__)
287	// The size of a prefix of TCB including pthread::{specific_1stblock,specific}	268	// The size of a prefix of TCB including pthread::{specific_1stblock,specific}
288	// suffices. Just return offsetof(struct pthread, specific_used), which hasn't	269	// suffices. Just return offsetof(struct pthread, specific_used), which hasn't
289	// changed since 2007-05. Technically this applies to i386/x86_64 as well but	270	// changed since 2007-05. Technically this applies to i386/x86_64 as well but
290	// we call _dl_get_tls_static_info and need the precise size of struct	271	// we call _dl_get_tls_static_info and need the precise size of struct
291	// pthread.	272	// pthread.
292	return FIRST_32_SECOND_64(524, 1552);	273	return FIRST_32_SECOND_64(524, 1552);
293	# elif defined(__mips__)	274	# endif
		275
		276	# if defined(__mips__)
294	// TODO(sagarthakur): add more values as per different glibc versions.	277	// TODO(sagarthakur): add more values as per different glibc versions.
295	val = FIRST_32_SECOND_64(1152, 1776);	278	return FIRST_32_SECOND_64(1152, 1776);
296	# elif SANITIZER_LOONGARCH64	279	# endif
297	val = 1856; // from glibc 2.36	280
298	# elif SANITIZER_RISCV64	281	# if SANITIZER_LOONGARCH64
299	int major;	282	return 1856; // from glibc 2.36
300	int minor;	283	# endif
301	int patch;
302	if (GetLibcVersion(&major, &minor, &patch) && major == 2) {
303	// TODO: consider adding an optional runtime check for an unknown (untested)
304	// glibc version
305	if (minor <= 28) // WARNING: the highest tested version is 2.29
306	val = 1772; // no guarantees for this one
307	else if (minor <= 31)
308	val = 1772; // tested against glibc 2.29, 2.31
309	else
310	val = 1936; // tested against glibc 2.32
311	}
312		284
313	# elif defined(__aarch64__)	285	# if defined(__aarch64__)
314	// The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.	286	// The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
315	val = 1776;	287	return 1776;
316	# elif defined(__powerpc64__)	288	# endif
317	val = 1776; // from glibc.ppc64le 2.20-8.fc21	289
		290	# if defined(__powerpc64__)
		291	return 1776; // from glibc.ppc64le 2.20-8.fc21
318	# endif	292	# endif
319	return val;
320	}	293	}
		294	# endif // SANITIZER_GLIBC && !SANITIZER_GO
		295
		296	# if SANITIZER_FREEBSD && !SANITIZER_GO
		297	// FIXME: Implementation is very GLIBC specific, but it's used by FreeBSD.
		298	static uptr ThreadDescriptorSizeFallback() {
		299	# if defined(__s390__) \|\| defined(__sparc__)
		300	// The size of a prefix of TCB including pthread::{specific_1stblock,specific}
		301	// suffices. Just return offsetof(struct pthread, specific_used), which hasn't
		302	// changed since 2007-05. Technically this applies to i386/x86_64 as well but
		303	// we call _dl_get_tls_static_info and need the precise size of struct
		304	// pthread.
		305	return FIRST_32_SECOND_64(524, 1552);
		306	# endif
321		307
322	uptr ThreadDescriptorSize() {	308	# if defined(__mips__)
323	uptr val = atomic_load_relaxed(&thread_descriptor_size);	309	// TODO(sagarthakur): add more values as per different glibc versions.
324	if (val)	310	return FIRST_32_SECOND_64(1152, 1776);
325	return val;	311	# endif
326	// _thread_db_sizeof_pthread is a GLIBC_PRIVATE symbol that is exported in	312
327	// glibc 2.34 and later.	313	# if SANITIZER_LOONGARCH64
328	if (unsigned psizeof = static_cast<unsigned >(	314	return 1856; // from glibc 2.36
329	dlsym(RTLD_DEFAULT, "_thread_db_sizeof_pthread")))	315	# endif
330	val = *psizeof;	316
331	if (!val)	317	# if defined(__aarch64__)
332	val = ThreadDescriptorSizeFallback();	318	// The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
333	atomic_store_relaxed(&thread_descriptor_size, val);	319	return 1776;
334	return val;	320	# endif
		321
		322	# if defined(__powerpc64__)
		323	return 1776; // from glibc.ppc64le 2.20-8.fc21
		324	# endif
		325
		326	return 0;
		327	}
		328	# endif // SANITIZER_FREEBSD && !SANITIZER_GO
		329
		330	# if (SANITIZER_FREEBSD \|\| SANITIZER_GLIBC) && !SANITIZER_GO
		331	// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage
		332	// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan
		333	// to get the pointer to thread-specific data keys in the thread control block.
		334	// sizeof(struct pthread) from glibc.
		335	static uptr thread_descriptor_size;
		336
		337	uptr ThreadDescriptorSize() { return thread_descriptor_size; }
		338
		339	# if SANITIZER_GLIBC
		340	__attribute__((unused)) static size_t g_tls_size;
		341	# endif
		342
		343	void InitTlsSize() {
		344	# if SANITIZER_GLIBC
		345	int major, minor, patch;
		346	GetGLibcVersion(&major, &minor, &patch);
		347	g_use_dlpi_tls_data = major == 2 && minor >= 25;
		348
		349	if (major == 2 && minor >= 34) {
		350	// _thread_db_sizeof_pthread is a GLIBC_PRIVATE symbol that is exported in
		351	// glibc 2.34 and later.
		352	if (unsigned psizeof = static_cast<unsigned >(
		353	dlsym(RTLD_DEFAULT, "_thread_db_sizeof_pthread"))) {
		354	thread_descriptor_size = *psizeof;
		355	}
		356	}
		357
		358	# if defined(__aarch64__) \|\| defined(__x86_64__) \|\| \
		359	defined(__powerpc64__) \|\| defined(__loongarch__)
		360	auto get_tls_static_info = (void ()(size_t , size_t ))dlsym(
		361	RTLD_DEFAULT, "_dl_get_tls_static_info");
		362	size_t tls_align;
		363	// Can be null if static link.
		364	if (get_tls_static_info)
		365	get_tls_static_info(&g_tls_size, &tls_align);
		366	# endif
		367
		368	# endif // SANITIZER_GLIBC
		369
		370	if (!thread_descriptor_size)
		371	thread_descriptor_size = ThreadDescriptorSizeFallback();
335	}	372	}
336		373
337	# if defined(__mips__) \|\| defined(__powerpc64__) \|\| SANITIZER_RISCV64 \|\| \	374	# if defined(__mips__) \|\| defined(__powerpc64__) \|\| SANITIZER_RISCV64 \|\| \
...	@@ -354,7 +391,13 @@ static uptr TlsPreTcbSize() {	...	@@ -354,7 +391,13 @@ static uptr TlsPreTcbSize() {
354	return kTlsPreTcbSize;	391	return kTlsPreTcbSize;
355	}	392	}
356	# endif	393	# endif
		394	# else // (SANITIZER_FREEBSD \|\| SANITIZER_GLIBC) && !SANITIZER_GO
		395	void InitTlsSize() {}
		396	uptr ThreadDescriptorSize() { return 0; }
		397	# endif // (SANITIZER_FREEBSD \|\| SANITIZER_GLIBC) && !SANITIZER_GO
357		398
		399	# if (SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_SOLARIS) && \
		400	!SANITIZER_ANDROID && !SANITIZER_GO
358	namespace {	401	namespace {
359	struct TlsBlock {	402	struct TlsBlock {
360	uptr begin, end, align;	403	uptr begin, end, align;
...	@@ -626,25 +669,32 @@ uptr GetTlsSize() {	...	@@ -626,25 +669,32 @@ uptr GetTlsSize() {
626	}	669	}
627	# endif	670	# endif
628		671
629	void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,	672	void GetThreadStackAndTls(bool main, uptr stk_begin, uptr stk_end,
630	uptr tls_addr, uptr tls_size) {	673	uptr tls_begin, uptr tls_end) {
631	# if SANITIZER_GO	674	# if SANITIZER_GO
632	// Stub implementation for Go.	675	// Stub implementation for Go.
633	stk_addr = stk_size = tls_addr = tls_size = 0;	676	*stk_begin = 0;
		677	*stk_end = 0;
		678	*tls_begin = 0;
		679	*tls_end = 0;
634	# else	680	# else
635	GetTls(tls_addr, tls_size);	681	uptr tls_addr = 0;
		682	uptr tls_size = 0;
		683	GetTls(&tls_addr, &tls_size);
		684	*tls_begin = tls_addr;
		685	*tls_end = tls_addr + tls_size;
636		686
637	uptr stack_top, stack_bottom;	687	uptr stack_top, stack_bottom;
638	GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);	688	GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);
639	*stk_addr = stack_bottom;	689	*stk_begin = stack_bottom;
640	*stk_size = stack_top - stack_bottom;	690	*stk_end = stack_top;
641		691
642	if (!main) {	692	if (!main) {
643	// If stack and tls intersect, make them non-intersecting.	693	// If stack and tls intersect, make them non-intersecting.
644	if (tls_addr > stk_addr && tls_addr < stk_addr + *stk_size) {	694	if (tls_begin > stk_begin && tls_begin < stk_end) {
645	if (stk_addr + stk_size < tls_addr + tls_size)	695	if (stk_end < tls_end)
646	tls_size = stk_addr + stk_size - tls_addr;	696	tls_end = stk_end;
647	stk_size = tls_addr - *stk_addr;	697	stk_end = tls_begin;
648	}	698	}
649	}	699	}
650	# endif	700	# endif
...	@@ -723,11 +773,6 @@ static int dl_iterate_phdr_cb(dl_phdr_info info, size_t size, void arg) {	...	@@ -723,11 +773,6 @@ static int dl_iterate_phdr_cb(dl_phdr_info info, size_t size, void arg) {
723	return 0;	773	return 0;
724	}	774	}
725		775
726	# if SANITIZER_ANDROID && __ANDROID_API__ < 21
727	extern "C" __attribute__((weak)) int dl_iterate_phdr(
728	int ()(struct dl_phdr_info , size_t, void ), void );
729	# endif
730
731	static bool requiresProcmaps() {	776	static bool requiresProcmaps() {
732	# if SANITIZER_ANDROID && __ANDROID_API__ <= 22	777	# if SANITIZER_ANDROID && __ANDROID_API__ <= 22
733	// Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken.	778	// Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken.
...	@@ -890,11 +935,8 @@ extern "C" SANITIZER_WEAK_ATTRIBUTE int __android_log_write(int prio,	...	@@ -890,11 +935,8 @@ extern "C" SANITIZER_WEAK_ATTRIBUTE int __android_log_write(int prio,
890	void WriteOneLineToSyslog(const char *s) {	935	void WriteOneLineToSyslog(const char *s) {
891	if (&async_safe_write_log) {	936	if (&async_safe_write_log) {
892	async_safe_write_log(SANITIZER_ANDROID_LOG_INFO, GetProcessName(), s);	937	async_safe_write_log(SANITIZER_ANDROID_LOG_INFO, GetProcessName(), s);
893	} else if (AndroidGetApiLevel() > ANDROID_KITKAT) {
894	syslog(LOG_INFO, "%s", s);
895	} else {	938	} else {
896	CHECK(&__android_log_write);	939	syslog(LOG_INFO, "%s", s);
897	__android_log_write(SANITIZER_ANDROID_LOG_INFO, nullptr, s);
898	}	940	}
899	}	941	}
900		942

lib/tsan/sanitizer_common/sanitizer_mac.cpp+36-28

...	@@ -45,7 +45,7 @@ extern char **environ;	...	@@ -45,7 +45,7 @@ extern char **environ;
45	# define SANITIZER_OS_TRACE 0	45	# define SANITIZER_OS_TRACE 0
46	# endif	46	# endif
47		47
48	// import new crash reporting api	48	// Integrate with CrashReporter library if available
49	# if defined(__has_include) && __has_include(<CrashReporterClient.h>)	49	# if defined(__has_include) && __has_include(<CrashReporterClient.h>)
50	# define HAVE_CRASHREPORTERCLIENT_H 1	50	# define HAVE_CRASHREPORTERCLIENT_H 1
51	# include <CrashReporterClient.h>	51	# include <CrashReporterClient.h>
...	@@ -545,9 +545,6 @@ uptr GetTlsSize() {	...	@@ -545,9 +545,6 @@ uptr GetTlsSize() {
545	return 0;	545	return 0;
546	}	546	}
547		547
548	void InitTlsSize() {
549	}
550
551	uptr TlsBaseAddr() {	548	uptr TlsBaseAddr() {
552	uptr segbase = 0;	549	uptr segbase = 0;
553	#if defined(__x86_64__)	550	#if defined(__x86_64__)
...	@@ -572,21 +569,18 @@ uptr TlsSize() {	...	@@ -572,21 +569,18 @@ uptr TlsSize() {
572	#endif	569	#endif
573	}	570	}
574		571
575	void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,	572	void GetThreadStackAndTls(bool main, uptr stk_begin, uptr stk_end,
576	uptr tls_addr, uptr tls_size) {	573	uptr tls_begin, uptr tls_end) {
577	#if !SANITIZER_GO	574	# if !SANITIZER_GO
578	uptr stack_top, stack_bottom;	575	GetThreadStackTopAndBottom(main, stk_end, stk_begin);
579	GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);	576	*tls_begin = TlsBaseAddr();
580	*stk_addr = stack_bottom;	577	tls_end = tls_begin + TlsSize();
581	*stk_size = stack_top - stack_bottom;	578	# else
582	*tls_addr = TlsBaseAddr();	579	*stk_begin = 0;
583	*tls_size = TlsSize();	580	*stk_end = 0;
584	#else	581	*tls_begin = 0;
585	*stk_addr = 0;	582	*tls_end = 0;
586	*stk_size = 0;	583	# endif
587	*tls_addr = 0;
588	*tls_size = 0;
589	#endif
590	}	584	}
591		585
592	void ListOfModules::init() {	586	void ListOfModules::init() {
...	@@ -788,7 +782,11 @@ void WriteOneLineToSyslog(const char *s) {	...	@@ -788,7 +782,11 @@ void WriteOneLineToSyslog(const char *s) {
788	if (GetMacosAlignedVersion() >= MacosVersion(10, 12)) {	782	if (GetMacosAlignedVersion() >= MacosVersion(10, 12)) {
789	os_log_error(OS_LOG_DEFAULT, "%{public}s", s);	783	os_log_error(OS_LOG_DEFAULT, "%{public}s", s);
790	} else {	784	} else {
		785	#pragma clang diagnostic push
		786	// as_log is deprecated.
		787	#pragma clang diagnostic ignored "-Wdeprecated-declarations"
791	asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "%s", s);	788	asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "%s", s);
		789	#pragma clang diagnostic pop
792	}	790	}
793	#endif	791	#endif
794	}	792	}
...	@@ -798,8 +796,13 @@ static char crashreporter_info_buff[__sanitizer::kErrorMessageBufferSize] = {};	...	@@ -798,8 +796,13 @@ static char crashreporter_info_buff[__sanitizer::kErrorMessageBufferSize] = {};
798	static Mutex crashreporter_info_mutex;	796	static Mutex crashreporter_info_mutex;
799		797
800	extern "C" {	798	extern "C" {
801	// Integrate with crash reporter libraries.	799
802	#if HAVE_CRASHREPORTERCLIENT_H	800	#if HAVE_CRASHREPORTERCLIENT_H
		801	// Available in CRASHREPORTER_ANNOTATIONS_VERSION 5+
		802	# ifdef CRASHREPORTER_ANNOTATIONS_INITIALIZER
		803	CRASHREPORTER_ANNOTATIONS_INITIALIZER()
		804	# else
		805	// Support for older CrashRerporter annotiations
803	CRASH_REPORTER_CLIENT_HIDDEN	806	CRASH_REPORTER_CLIENT_HIDDEN
804	struct crashreporter_annotations_t gCRAnnotations	807	struct crashreporter_annotations_t gCRAnnotations
805	__attribute__((section("__DATA," CRASHREPORTER_ANNOTATIONS_SECTION))) = {	808	__attribute__((section("__DATA," CRASHREPORTER_ANNOTATIONS_SECTION))) = {
...	@@ -810,17 +813,17 @@ struct crashreporter_annotations_t gCRAnnotations	...	@@ -810,17 +813,17 @@ struct crashreporter_annotations_t gCRAnnotations
810	0,	813	0,
811	0,	814	0,
812	0,	815	0,
813	#if CRASHREPORTER_ANNOTATIONS_VERSION > 4	816	# if CRASHREPORTER_ANNOTATIONS_VERSION > 4
814	0,	817	0,
815	#endif	818	# endif
816	};	819	};
817		820	# endif
818	#else	821	# else
819	// fall back to old crashreporter api	822	// Revert to previous crash reporter API if client header is not available
820	static const char *__crashreporter_info__ __attribute__((__used__)) =	823	static const char *__crashreporter_info__ __attribute__((__used__)) =
821	&crashreporter_info_buff[0];	824	&crashreporter_info_buff[0];
822	asm(".desc ___crashreporter_info__, 0x10");	825	asm(".desc ___crashreporter_info__, 0x10");
823	#endif	826	#endif // HAVE_CRASHREPORTERCLIENT_H
824		827
825	} // extern "C"	828	} // extern "C"
826		829
...	@@ -843,6 +846,9 @@ void LogFullErrorReport(const char *buffer) {	...	@@ -843,6 +846,9 @@ void LogFullErrorReport(const char *buffer) {
843	#if !SANITIZER_GO	846	#if !SANITIZER_GO
844	// Log with os_trace. This will make it into the crash log.	847	// Log with os_trace. This will make it into the crash log.
845	#if SANITIZER_OS_TRACE	848	#if SANITIZER_OS_TRACE
		849	#pragma clang diagnostic push
		850	// os_trace is deprecated.
		851	#pragma clang diagnostic ignored "-Wdeprecated-declarations"
846	if (GetMacosAlignedVersion() >= MacosVersion(10, 10)) {	852	if (GetMacosAlignedVersion() >= MacosVersion(10, 10)) {
847	// os_trace requires the message (format parameter) to be a string literal.	853	// os_trace requires the message (format parameter) to be a string literal.
848	if (internal_strncmp(SanitizerToolName, "AddressSanitizer",	854	if (internal_strncmp(SanitizerToolName, "AddressSanitizer",
...	@@ -860,6 +866,7 @@ void LogFullErrorReport(const char *buffer) {	...	@@ -860,6 +866,7 @@ void LogFullErrorReport(const char *buffer) {
860	if (common_flags()->log_to_syslog)	866	if (common_flags()->log_to_syslog)
861	os_trace("Consult syslog for more information.");	867	os_trace("Consult syslog for more information.");
862	}	868	}
		869	#pragma clang diagnostic pop
863	#endif	870	#endif
864		871
865	// Log to syslog.	872	// Log to syslog.
...	@@ -970,8 +977,9 @@ static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES";	...	@@ -970,8 +977,9 @@ static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES";
970	LowLevelAllocator allocator_for_env;	977	LowLevelAllocator allocator_for_env;
971		978
972	static bool ShouldCheckInterceptors() {	979	static bool ShouldCheckInterceptors() {
973	// Restrict "interceptors working?" check to ASan and TSan.	980	// Restrict "interceptors working?" check
974	const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer"};	981	const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer",
		982	"RealtimeSanitizer"};
975	size_t count = sizeof(sanitizer_names) / sizeof(sanitizer_names[0]);	983	size_t count = sizeof(sanitizer_names) / sizeof(sanitizer_names[0]);
976	for (size_t i = 0; i < count; i++) {	984	for (size_t i = 0; i < count; i++) {
977	if (internal_strcmp(sanitizer_names[i], SanitizerToolName) == 0)	985	if (internal_strcmp(sanitizer_names[i], SanitizerToolName) == 0)

lib/tsan/sanitizer_common/sanitizer_platform_interceptors.h+45-7

...	@@ -84,6 +84,25 @@	...	@@ -84,6 +84,25 @@
84	#define SI_NOT_MAC 1	84	#define SI_NOT_MAC 1
85	#endif	85	#endif
86		86
		87	#if SANITIZER_APPLE
		88	# include <Availability.h>
		89
		90	// aligned_alloc was introduced in OSX 10.15
		91	// Linking will fail when using an older SDK
		92	# if defined(__MAC_10_15)
		93	// macOS 10.15 is greater than our minimal deployment target. To ensure we
		94	// generate a weak reference so the dylib continues to work on older
		95	// systems, we need to forward declare the intercepted function as "weak
		96	// imports".
		97	SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
		98	__sanitizer::usize __size);
		99	# define SI_MAC_SDK_10_15_AVAILABLE 1
		100	# else
		101	# define SI_MAC_SDK_10_15_AVAILABLE 0
		102	# endif // defined(__MAC_10_15)
		103
		104	#endif // SANITIZER_APPLE
		105
87	#if SANITIZER_IOS	106	#if SANITIZER_IOS
88	#define SI_IOS 1	107	#define SI_IOS 1
89	#else	108	#else
...	@@ -183,9 +202,16 @@	...	@@ -183,9 +202,16 @@
183	#define SANITIZER_INTERCEPT_FPUTS SI_POSIX	202	#define SANITIZER_INTERCEPT_FPUTS SI_POSIX
184	#define SANITIZER_INTERCEPT_PUTS SI_POSIX	203	#define SANITIZER_INTERCEPT_PUTS SI_POSIX
185		204
		205	#define SANITIZER_INTERCEPT_CREAT64 (SI_GLIBC \|\| SI_SOLARIS32)
		206	#define SANITIZER_INTERCEPT_FCNTL64 (SI_GLIBC \|\| SI_SOLARIS32)
		207	#define SANITIZER_INTERCEPT_OPEN64 (SI_GLIBC \|\| SI_SOLARIS32)
		208	#define SANITIZER_INTERCEPT_OPENAT64 (SI_GLIBC \|\| SI_SOLARIS32)
		209
186	#define SANITIZER_INTERCEPT_PREAD64 (SI_GLIBC \|\| SI_SOLARIS32)	210	#define SANITIZER_INTERCEPT_PREAD64 (SI_GLIBC \|\| SI_SOLARIS32)
187	#define SANITIZER_INTERCEPT_PWRITE64 (SI_GLIBC \|\| SI_SOLARIS32)	211	#define SANITIZER_INTERCEPT_PWRITE64 (SI_GLIBC \|\| SI_SOLARIS32)
188		212
		213	#define SANITIZER_INTERCEPT_LSEEK64 (SI_GLIBC \|\| SI_SOLARIS32)
		214
189	#define SANITIZER_INTERCEPT_READV SI_POSIX	215	#define SANITIZER_INTERCEPT_READV SI_POSIX
190	#define SANITIZER_INTERCEPT_WRITEV SI_POSIX	216	#define SANITIZER_INTERCEPT_WRITEV SI_POSIX
191		217
...	@@ -232,8 +258,12 @@	...	@@ -232,8 +258,12 @@
232	(SI_FREEBSD \|\| SI_NETBSD \|\| SI_LINUX \|\| SI_SOLARIS)	258	(SI_FREEBSD \|\| SI_NETBSD \|\| SI_LINUX \|\| SI_SOLARIS)
233	#define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID \	259	#define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID \
234	(SI_LINUX \|\| SI_FREEBSD \|\| SI_NETBSD)	260	(SI_LINUX \|\| SI_FREEBSD \|\| SI_NETBSD)
		261	// TODO: This should be SI_POSIX, adding glibc first until I have time
		262	// to verify all timer_t typedefs on other platforms.
		263	#define SANITIZER_INTERCEPT_TIMER_CREATE SI_GLIBC
235	#define SANITIZER_INTERCEPT_GETITIMER SI_POSIX	264	#define SANITIZER_INTERCEPT_GETITIMER SI_POSIX
236	#define SANITIZER_INTERCEPT_TIME SI_POSIX	265	#define SANITIZER_INTERCEPT_TIME SI_POSIX
		266	#define SANITIZER_INTERCEPT_TIMESPEC_GET SI_LINUX
237	#define SANITIZER_INTERCEPT_GLOB (SI_GLIBC \|\| SI_SOLARIS)	267	#define SANITIZER_INTERCEPT_GLOB (SI_GLIBC \|\| SI_SOLARIS)
238	#define SANITIZER_INTERCEPT_GLOB64 SI_GLIBC	268	#define SANITIZER_INTERCEPT_GLOB64 SI_GLIBC
239	#define SANITIZER_INTERCEPT___B64_TO SI_LINUX_NOT_ANDROID	269	#define SANITIZER_INTERCEPT___B64_TO SI_LINUX_NOT_ANDROID
...	@@ -274,8 +304,9 @@	...	@@ -274,8 +304,9 @@
274	#if SI_LINUX_NOT_ANDROID && \	304	#if SI_LINUX_NOT_ANDROID && \
275	(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \	305	(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \
276	defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__arm__) \|\| \	306	defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__arm__) \|\| \
277	defined(__s390__) \|\| defined(__loongarch__) \|\| SANITIZER_RISCV64)	307	defined(__s390__) \|\| defined(__loongarch__) \|\| SANITIZER_RISCV64 \|\| \
278	#define SANITIZER_INTERCEPT_PTRACE 1	308	defined(__sparc__))
		309	# define SANITIZER_INTERCEPT_PTRACE 1
279	#else	310	#else
280	#define SANITIZER_INTERCEPT_PTRACE 0	311	#define SANITIZER_INTERCEPT_PTRACE 0
281	#endif	312	#endif
...	@@ -314,6 +345,8 @@	...	@@ -314,6 +345,8 @@
314	#define SANITIZER_INTERCEPT_GETGROUPS SI_POSIX	345	#define SANITIZER_INTERCEPT_GETGROUPS SI_POSIX
315	#define SANITIZER_INTERCEPT_POLL SI_POSIX	346	#define SANITIZER_INTERCEPT_POLL SI_POSIX
316	#define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS	347	#define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS
		348	#define SANITIZER_INTERCEPT_EPOLL (SI_LINUX)
		349	#define SANITIZER_INTERCEPT_KQUEUE (SI_FREEBSD \|\| SI_NETBSD \|\| SI_MAC)
317	#define SANITIZER_INTERCEPT_WORDEXP \	350	#define SANITIZER_INTERCEPT_WORDEXP \
318	(SI_FREEBSD \|\| SI_NETBSD \|\| (SI_MAC && !SI_IOS) \|\| SI_LINUX_NOT_ANDROID \|\| \	351	(SI_FREEBSD \|\| SI_NETBSD \|\| (SI_MAC && !SI_IOS) \|\| SI_LINUX_NOT_ANDROID \|\| \
319	SI_SOLARIS)	352	SI_SOLARIS)
...	@@ -494,7 +527,8 @@	...	@@ -494,7 +527,8 @@
494	#define SANITIZER_INTERCEPT_PVALLOC (SI_GLIBC \|\| SI_ANDROID)	527	#define SANITIZER_INTERCEPT_PVALLOC (SI_GLIBC \|\| SI_ANDROID)
495	#define SANITIZER_INTERCEPT_CFREE (SI_GLIBC && !SANITIZER_RISCV64)	528	#define SANITIZER_INTERCEPT_CFREE (SI_GLIBC && !SANITIZER_RISCV64)
496	#define SANITIZER_INTERCEPT_REALLOCARRAY SI_POSIX	529	#define SANITIZER_INTERCEPT_REALLOCARRAY SI_POSIX
497	#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC)	530	#define SANITIZER_INTERCEPT_ALIGNED_ALLOC \
		531	(!SI_MAC \|\| SI_MAC_SDK_10_15_AVAILABLE)
498	#define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD)	532	#define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD)
499	#define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID	533	#define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID
500	#define SANITIZER_INTERCEPT_WCSLEN 1	534	#define SANITIZER_INTERCEPT_WCSLEN 1
...	@@ -559,10 +593,8 @@	...	@@ -559,10 +593,8 @@
559	#define SANITIZER_INTERCEPT_SHA1 SI_NETBSD	593	#define SANITIZER_INTERCEPT_SHA1 SI_NETBSD
560	#define SANITIZER_INTERCEPT_MD4 SI_NETBSD	594	#define SANITIZER_INTERCEPT_MD4 SI_NETBSD
561	#define SANITIZER_INTERCEPT_RMD160 SI_NETBSD	595	#define SANITIZER_INTERCEPT_RMD160 SI_NETBSD
562	#define SANITIZER_INTERCEPT_MD5 (SI_NETBSD \|\| SI_FREEBSD)	596	#define SANITIZER_INTERCEPT_FSEEK SI_POSIX
563	#define SANITIZER_INTERCEPT_FSEEK (SI_NETBSD \|\| SI_FREEBSD)
564	#define SANITIZER_INTERCEPT_MD2 SI_NETBSD	597	#define SANITIZER_INTERCEPT_MD2 SI_NETBSD
565	#define SANITIZER_INTERCEPT_SHA2 (SI_NETBSD \|\| SI_FREEBSD)
566	#define SANITIZER_INTERCEPT_CDB SI_NETBSD	598	#define SANITIZER_INTERCEPT_CDB SI_NETBSD
567	#define SANITIZER_INTERCEPT_VIS (SI_NETBSD \|\| SI_FREEBSD)	599	#define SANITIZER_INTERCEPT_VIS (SI_NETBSD \|\| SI_FREEBSD)
568	#define SANITIZER_INTERCEPT_POPEN SI_POSIX	600	#define SANITIZER_INTERCEPT_POPEN SI_POSIX
...	@@ -601,7 +633,13 @@	...	@@ -601,7 +633,13 @@
601	// FIXME: also available from musl 1.2.5	633	// FIXME: also available from musl 1.2.5
602	#define SANITIZER_INTERCEPT_PREADV2 (SI_LINUX && __GLIBC_PREREQ(2, 26))	634	#define SANITIZER_INTERCEPT_PREADV2 (SI_LINUX && __GLIBC_PREREQ(2, 26))
603	#define SANITIZER_INTERCEPT_PWRITEV2 (SI_LINUX && __GLIBC_PREREQ(2, 26))	635	#define SANITIZER_INTERCEPT_PWRITEV2 (SI_LINUX && __GLIBC_PREREQ(2, 26))
604		636	#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && \
		637	__MAC_OS_X_VERSION_MIN_REQUIRED >= 130000
		638	# define SI_MAC_OS_DEPLOYMENT_MIN_13_00 1
		639	#else
		640	# define SI_MAC_OS_DEPLOYMENT_MIN_13_00 0
		641	#endif
		642	#define SANITIZER_INTERCEPT_FREADLINK (SI_MAC && SI_MAC_OS_DEPLOYMENT_MIN_13_00)
605	// This macro gives a way for downstream users to override the above	643	// This macro gives a way for downstream users to override the above
606	// interceptor macros irrespective of the platform they are on. They have	644	// interceptor macros irrespective of the platform they are on. They have
607	// to do two things:	645	// to do two things:

lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.cpp+1-2

...	@@ -547,6 +547,7 @@ unsigned pid_t_sz = sizeof(pid_t);	...	@@ -547,6 +547,7 @@ unsigned pid_t_sz = sizeof(pid_t);
547	unsigned timeval_sz = sizeof(timeval);	547	unsigned timeval_sz = sizeof(timeval);
548	unsigned uid_t_sz = sizeof(uid_t);	548	unsigned uid_t_sz = sizeof(uid_t);
549	unsigned gid_t_sz = sizeof(gid_t);	549	unsigned gid_t_sz = sizeof(gid_t);
		550	unsigned fpos_t_sz = sizeof(fpos_t);
550	unsigned mbstate_t_sz = sizeof(mbstate_t);	551	unsigned mbstate_t_sz = sizeof(mbstate_t);
551	unsigned sigset_t_sz = sizeof(sigset_t);	552	unsigned sigset_t_sz = sizeof(sigset_t);
552	unsigned struct_timezone_sz = sizeof(struct timezone);	553	unsigned struct_timezone_sz = sizeof(struct timezone);
...	@@ -2487,8 +2488,6 @@ const unsigned RMD160_return_length = RMD160_DIGEST_STRING_LENGTH;	...	@@ -2487,8 +2488,6 @@ const unsigned RMD160_return_length = RMD160_DIGEST_STRING_LENGTH;
2487	const unsigned MD5_CTX_sz = sizeof(MD5_CTX);	2488	const unsigned MD5_CTX_sz = sizeof(MD5_CTX);
2488	const unsigned MD5_return_length = MD5_DIGEST_STRING_LENGTH;	2489	const unsigned MD5_return_length = MD5_DIGEST_STRING_LENGTH;
2489		2490
2490	const unsigned fpos_t_sz = sizeof(fpos_t);
2491
2492	const unsigned MD2_CTX_sz = sizeof(MD2_CTX);	2491	const unsigned MD2_CTX_sz = sizeof(MD2_CTX);
2493	const unsigned MD2_return_length = MD2_DIGEST_STRING_LENGTH;	2492	const unsigned MD2_return_length = MD2_DIGEST_STRING_LENGTH;
2494		2493

lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.h+1-2

...	@@ -36,6 +36,7 @@ extern unsigned pid_t_sz;	...	@@ -36,6 +36,7 @@ extern unsigned pid_t_sz;
36	extern unsigned timeval_sz;	36	extern unsigned timeval_sz;
37	extern unsigned uid_t_sz;	37	extern unsigned uid_t_sz;
38	extern unsigned gid_t_sz;	38	extern unsigned gid_t_sz;
		39	extern unsigned fpos_t_sz;
39	extern unsigned mbstate_t_sz;	40	extern unsigned mbstate_t_sz;
40	extern unsigned struct_timezone_sz;	41	extern unsigned struct_timezone_sz;
41	extern unsigned struct_tms_sz;	42	extern unsigned struct_tms_sz;
...	@@ -2335,8 +2336,6 @@ extern const unsigned RMD160_return_length;	...	@@ -2335,8 +2336,6 @@ extern const unsigned RMD160_return_length;
2335	extern const unsigned MD5_CTX_sz;	2336	extern const unsigned MD5_CTX_sz;
2336	extern const unsigned MD5_return_length;	2337	extern const unsigned MD5_return_length;
2337		2338
2338	extern const unsigned fpos_t_sz;
2339
2340	extern const unsigned MD2_CTX_sz;	2339	extern const unsigned MD2_CTX_sz;
2341	extern const unsigned MD2_return_length;	2340	extern const unsigned MD2_return_length;
2342		2341

lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.cpp+35-26

 @@ -94,8 +94,9 @@
 #if SANITIZER_LINUX
 # include <utime.h>
 # include <sys/ptrace.h>
-#    if defined(__mips64) || defined(__aarch64__) || defined(__arm__) || \
+#    if defined(__mips64) || defined(__aarch64__) || defined(__arm__) ||       \
-        defined(__hexagon__) || defined(__loongarch__) ||SANITIZER_RISCV64
+        defined(__hexagon__) || defined(__loongarch__) || SANITIZER_RISCV64 || \
+        defined(__sparc__)
 #      include <asm/ptrace.h>
 #      ifdef __arm__
 typedef struct user_fpregs elf_fpregset_t;
 @@ -117,15 +118,16 @@ typedef struct user_fpregs elf_fpregset_t;
 #if SANITIZER_LINUX
 #if SANITIZER_GLIBC
 #include <fstab.h>
-#include <net/if_ppp.h>
+#      include <linux/filter.h>
-#include <netax25/ax25.h>
+#      include <net/if_ppp.h>
-#include <netipx/ipx.h>
+#      include <netax25/ax25.h>
-#include <netrom/netrom.h>
+#      include <netipx/ipx.h>
-#include <obstack.h>
+#      include <netrom/netrom.h>
-#if HAVE_RPC_XDR_H
+#      include <obstack.h>
-# include <rpc/xdr.h>
+#      if HAVE_RPC_XDR_H
-#endif
+#        include <rpc/xdr.h>
-#include <scsi/scsi.h>
+#      endif
+#      include <scsi/scsi.h>
 #else
 #include <linux/if_ppp.h>
 #include <linux/kd.h>
 @@ -358,11 +360,12 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
   const int wordexp_wrde_dooffs = WRDE_DOOFFS;
 #  endif  // !SANITIZER_ANDROID
-#if SANITIZER_LINUX && !SANITIZER_ANDROID &&                               \
+#  if SANITIZER_LINUX && !SANITIZER_ANDROID &&                               \
-    (defined(__i386) || defined(__x86_64) || defined(__mips64) ||          \
+      (defined(__i386) || defined(__x86_64) || defined(__mips64) ||          \
-     defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
+       defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
-     defined(__s390__) || defined(__loongarch__)|| SANITIZER_RISCV64)
+       defined(__s390__) || defined(__loongarch__) || SANITIZER_RISCV64 ||   \
-#if defined(__mips64) || defined(__powerpc64__) || defined(__arm__)
+       defined(__sparc__))
+#    if defined(__mips64) || defined(__powerpc64__) || defined(__arm__)
   unsigned struct_user_regs_struct_sz = sizeof(struct pt_regs);
   unsigned struct_user_fpregs_struct_sz = sizeof(elf_fpregset_t);
 #elif SANITIZER_RISCV64
 @@ -377,19 +380,22 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
 #elif defined(__s390__)
   unsigned struct_user_regs_struct_sz = sizeof(struct _user_regs_struct);
   unsigned struct_user_fpregs_struct_sz = sizeof(struct _user_fpregs_struct);
-#else
+#    elif defined(__sparc__)
+  unsigned struct_user_regs_struct_sz = sizeof(struct sunos_regs);
+  unsigned struct_user_fpregs_struct_sz = sizeof(struct sunos_fp);
+#    else
   unsigned struct_user_regs_struct_sz = sizeof(struct user_regs_struct);
   unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpregs_struct);
-#endif // __mips64 || __powerpc64__ || __aarch64__ || __loongarch__
+#    endif  // __mips64 || __powerpc64__ || __aarch64__ || __loongarch__
-#if defined(__x86_64) || defined(__mips64) || defined(__powerpc64__) || \
+#    if defined(__x86_64) || defined(__mips64) || defined(__powerpc64__) || \
-    defined(__aarch64__) || defined(__arm__) || defined(__s390__) ||    \
+        defined(__aarch64__) || defined(__arm__) || defined(__s390__) ||    \
-    defined(__loongarch__) || SANITIZER_RISCV64
+        defined(__loongarch__) || SANITIZER_RISCV64 || defined(__sparc__)
   unsigned struct_user_fpxregs_struct_sz = 0;
 #else
   unsigned struct_user_fpxregs_struct_sz = sizeof(struct user_fpxregs_struct);
 #endif // __x86_64 || __mips64 || __powerpc64__ || __aarch64__ || __arm__
-// || __s390__ || __loongarch__
+  // || __s390__ || __loongarch__ || SANITIZER_RISCV64 || __sparc__
-#ifdef __arm__
+#    ifdef __arm__
   unsigned struct_user_vfpregs_struct_sz = ARM_VFPREGS_SIZE;
 #else
   unsigned struct_user_vfpregs_struct_sz = 0;
 @@ -531,13 +537,16 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
   unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info);
   unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats);
-#endif  // SANITIZER_GLIBC
+  unsigned struct_sock_fprog_sz = sizeof(struct sock_fprog);
+#  endif  // SANITIZER_GLIBC
-#if !SANITIZER_ANDROID && !SANITIZER_APPLE
+#  if !SANITIZER_ANDROID && !SANITIZER_APPLE
   unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
   unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
 #endif
+  unsigned fpos_t_sz = sizeof(fpos_t);
   const unsigned long __sanitizer_bufsiz = BUFSIZ;
   const unsigned IOCTL_NOT_PRESENT = 0;
 @@ -1084,7 +1093,7 @@ CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len);
 CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level);
 CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type);
-#if SANITIZER_LINUX && (__ANDROID_API__ >= 21 || __GLIBC_PREREQ (2, 14))
+#  if SANITIZER_LINUX && (SANITIZER_ANDROID || __GLIBC_PREREQ(2, 14))
 CHECK_TYPE_SIZE(mmsghdr);
 CHECK_SIZE_AND_OFFSET(mmsghdr, msg_hdr);
 CHECK_SIZE_AND_OFFSET(mmsghdr, msg_len);

lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.h+49-13

 @@ -98,10 +98,13 @@ const unsigned struct_kernel_stat64_sz = 104;
 const unsigned struct_kernel_stat_sz = 144;
 const unsigned struct_kernel_stat64_sz = 104;
 #elif defined(__mips__)
-const unsigned struct_kernel_stat_sz =
+const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID
-    SANITIZER_ANDROID
+                                           ? FIRST_32_SECOND_64(104, 128)
-        ? FIRST_32_SECOND_64(104, 128)
+#      if defined(_ABIN32) && _MIPS_SIM == _ABIN32
-        : FIRST_32_SECOND_64((_MIPS_SIM == _ABIN32) ? 176 : 160, 216);
+                                           : FIRST_32_SECOND_64(176, 216);
+#      else
+                                           : FIRST_32_SECOND_64(160, 216);
+#      endif
 const unsigned struct_kernel_stat64_sz = 104;
 #elif defined(__s390__) && !defined(__s390x__)
 const unsigned struct_kernel_stat_sz = 64;
 @@ -313,7 +316,7 @@ extern unsigned struct_statvfs_sz;
 struct __sanitizer_iovec {
   void *iov_base;
-  uptr iov_len;
+  usize iov_len;
 };
 #if !SANITIZER_ANDROID
 @@ -389,6 +392,16 @@ typedef long __sanitizer_time_t;
 typedef long __sanitizer_suseconds_t;
+struct __sanitizer_timespec {
+  __sanitizer_time_t tv_sec; /* seconds */
+  u64 tv_nsec;               /* nanoseconds */
+};
+struct __sanitizer_itimerspec {
+  struct __sanitizer_timespec it_interval; /* timer period */
+  struct __sanitizer_timespec it_value;    /* timer expiration */
+};
 struct __sanitizer_timeval {
   __sanitizer_time_t tv_sec;
   __sanitizer_suseconds_t tv_usec;
 @@ -513,6 +526,7 @@ struct __sanitizer_dirent64 {
   unsigned short d_reclen;
   // more fields that we don't care about
 };
+extern unsigned struct_sock_fprog_sz;
 #endif
 #if defined(__x86_64__) && !defined(_LP64)
 @@ -590,7 +604,7 @@ struct __sanitizer_siginfo_pad {
 #if SANITIZER_LINUX
 # define SANITIZER_HAS_SIGINFO 1
 union __sanitizer_siginfo {
-  struct {
+  __extension__ struct {
     int si_signo;
 # if SANITIZER_MIPS
     int si_code;
 @@ -855,10 +869,11 @@ typedef void __sanitizer_FILE;
 # define SANITIZER_HAS_STRUCT_FILE 0
 #endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID &&                               \
+#  if SANITIZER_LINUX && !SANITIZER_ANDROID &&                               \
-    (defined(__i386) || defined(__x86_64) || defined(__mips64) ||          \
+      (defined(__i386) || defined(__x86_64) || defined(__mips64) ||          \
-     defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
+       defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
-     defined(__s390__) || defined(__loongarch__) || SANITIZER_RISCV64)
+       defined(__s390__) || defined(__loongarch__) || SANITIZER_RISCV64 ||   \
+       defined(__sparc__))
 extern unsigned struct_user_regs_struct_sz;
 extern unsigned struct_user_fpregs_struct_sz;
 extern unsigned struct_user_fpxregs_struct_sz;
 @@ -880,9 +895,24 @@ extern int ptrace_setsiginfo;
 extern int ptrace_getregset;
 extern int ptrace_setregset;
 extern int ptrace_geteventmsg;
-#endif
-#if SANITIZER_LINUX  && !SANITIZER_ANDROID
+// Helper for the ptrace interceptor.
+template <class T>
+inline T ptrace_data_arg(int request, T addr, T data) {
+#    if SANITIZER_LINUX && SANITIZER_SPARC
+  // As described in ptrace(2), the meanings of addr and data are reversed
+  // for the PTRACE_GETREGS, PTRACE_GETFPREGS, PTRACE_GETREGS, and
+  // PTRACE_GETFPREGS requests on Linux/sparc64.
+  if (request == ptrace_getregs || request == ptrace_getfpregs ||
+      request == ptrace_setregs || request == ptrace_setfpregs)
+    return addr;
+  else
+#    endif
+    return data;
+}
+#  endif
+#  if SANITIZER_LINUX && !SANITIZER_ANDROID
 extern unsigned struct_shminfo_sz;
 extern unsigned struct_shm_info_sz;
 extern int shmctl_ipc_stat;
 @@ -1050,7 +1080,7 @@ extern unsigned struct_serial_struct_sz;
 extern unsigned struct_sockaddr_ax25_sz;
 extern unsigned struct_unimapdesc_sz;
 extern unsigned struct_unimapinit_sz;
-#endif  // SANITIZER_LINUX && !SANITIZER_ANDROID
+#  endif  // SANITIZER_LINUX && !SANITIZER_ANDROID
 extern const unsigned long __sanitizer_bufsiz;
 @@ -1064,6 +1094,8 @@ extern unsigned struct_sioc_sg_req_sz;
 extern unsigned struct_sioc_vif_req_sz;
 #endif
+extern unsigned fpos_t_sz;
 // ioctl request identifiers
 // A special value to mark ioctls that are not present on the target platform,
 @@ -1500,6 +1532,10 @@ extern const int si_SEGV_ACCERR;
 #define SIGACTION_SYMNAME sigaction
+#  if SANITIZER_LINUX
+typedef void *__sanitizer_timer_t;
+#  endif
 #endif  // SANITIZER_LINUX || SANITIZER_APPLE
 #endif

lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.cpp+3

...	@@ -32,6 +32,7 @@	...	@@ -32,6 +32,7 @@
32	#include <semaphore.h>	32	#include <semaphore.h>
33	#include <signal.h>	33	#include <signal.h>
34	#include <stddef.h>	34	#include <stddef.h>
		35	#include <stdio.h>
35	#include <sys/ethernet.h>	36	#include <sys/ethernet.h>
36	#include <sys/filio.h>	37	#include <sys/filio.h>
37	#include <sys/ipc.h>	38	#include <sys/ipc.h>
...	@@ -135,6 +136,8 @@ namespace __sanitizer {	...	@@ -135,6 +136,8 @@ namespace __sanitizer {
135	unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);	136	unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
136	unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);	137	unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
137		138
		139	unsigned fpos_t_sz = sizeof(fpos_t);
		140
138	const unsigned IOCTL_NOT_PRESENT = 0;	141	const unsigned IOCTL_NOT_PRESENT = 0;
139		142
140	unsigned IOCTL_FIOASYNC = FIOASYNC;	143	unsigned IOCTL_FIOASYNC = FIOASYNC;

lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.h+2

...	@@ -418,6 +418,8 @@ extern unsigned struct_winsize_sz;	...	@@ -418,6 +418,8 @@ extern unsigned struct_winsize_sz;
418	extern unsigned struct_sioc_sg_req_sz;	418	extern unsigned struct_sioc_sg_req_sz;
419	extern unsigned struct_sioc_vif_req_sz;	419	extern unsigned struct_sioc_vif_req_sz;
420		420
		421	extern unsigned fpos_t_sz;
		422
421	// ioctl request identifiers	423	// ioctl request identifiers
422		424
423	// A special value to mark ioctls that are not present on the target platform,	425	// A special value to mark ioctls that are not present on the target platform,

lib/tsan/sanitizer_common/sanitizer_posix.cpp+9-1

...	@@ -353,7 +353,15 @@ bool ShouldMockFailureToOpen(const char *path) {	...	@@ -353,7 +353,15 @@ bool ShouldMockFailureToOpen(const char *path) {
353	internal_strncmp(path, "/proc/", 6) == 0;	353	internal_strncmp(path, "/proc/", 6) == 0;
354	}	354	}
355		355
356	#if SANITIZER_LINUX && !SANITIZER_ANDROID && !SANITIZER_GO	356	bool OpenReadsVaArgs(int oflag) {
		357	# ifdef O_TMPFILE
		358	return (oflag & (O_CREAT \| O_TMPFILE)) != 0;
		359	# else
		360	return (oflag & O_CREAT) != 0;
		361	# endif
		362	}
		363
		364	# if SANITIZER_LINUX && !SANITIZER_ANDROID && !SANITIZER_GO
357	int GetNamedMappingFd(const char name, uptr size, int flags) {	365	int GetNamedMappingFd(const char name, uptr size, int flags) {
358	if (!common_flags()->decorate_proc_maps \|\| !name)	366	if (!common_flags()->decorate_proc_maps \|\| !name)
359	return -1;	367	return -1;

lib/tsan/sanitizer_common/sanitizer_posix.h+5-1

lib/tsan/sanitizer_common/sanitizer_posix_libcdep.cpp+82-18

...	@@ -288,26 +288,86 @@ bool SignalContext::IsStackOverflow() const {	...	@@ -288,26 +288,86 @@ bool SignalContext::IsStackOverflow() const {
288		288
289	#endif // SANITIZER_GO	289	#endif // SANITIZER_GO
290		290
		291	static void SetNonBlock(int fd) {
		292	int res = fcntl(fd, F_GETFL, 0);
		293	CHECK(!internal_iserror(res, nullptr));
		294
		295	res \|= O_NONBLOCK;
		296	res = fcntl(fd, F_SETFL, res);
		297	CHECK(!internal_iserror(res, nullptr));
		298	}
		299
291	bool IsAccessibleMemoryRange(uptr beg, uptr size) {	300	bool IsAccessibleMemoryRange(uptr beg, uptr size) {
292	uptr page_size = GetPageSizeCached();	301	while (size) {
293	// Checking too large memory ranges is slow.	302	// `read` from `fds[0]` into a dummy buffer to free up the pipe buffer for
294	CHECK_LT(size, page_size * 10);	303	// more `write` is slower than just recreating a pipe.
295	int sock_pair[2];	304	int fds[2];
296	if (pipe(sock_pair))	305	CHECK_EQ(0, pipe(fds));
297	return false;	306
298	uptr bytes_written =	307	auto cleanup = at_scope_exit([&]() {
299	internal_write(sock_pair[1], reinterpret_cast<void *>(beg), size);	308	internal_close(fds[0]);
300	int write_errno;	309	internal_close(fds[1]);
301	bool result;	310	});
302	if (internal_iserror(bytes_written, &write_errno)) {	311
303	CHECK_EQ(EFAULT, write_errno);	312	SetNonBlock(fds[1]);
304	result = false;	313
305	} else {	314	int write_errno;
306	result = (bytes_written == size);	315	uptr w = internal_write(fds[1], reinterpret_cast<char *>(beg), size);
		316	if (internal_iserror(w, &write_errno)) {
		317	if (write_errno == EINTR)
		318	continue;
		319	CHECK_EQ(EFAULT, write_errno);
		320	return false;
		321	}
		322	size -= w;
		323	beg += w;
		324	}
		325
		326	return true;
		327	}
		328
		329	bool TryMemCpy(void dest, const void src, uptr n) {
		330	if (!n)
		331	return true;
		332	int fds[2];
		333	CHECK_EQ(0, pipe(fds));
		334
		335	auto cleanup = at_scope_exit([&]() {
		336	internal_close(fds[0]);
		337	internal_close(fds[1]);
		338	});
		339
		340	SetNonBlock(fds[0]);
		341	SetNonBlock(fds[1]);
		342
		343	char d = static_cast<char >(dest);
		344	const char s = static_cast<const char >(src);
		345
		346	while (n) {
		347	int e;
		348	uptr w = internal_write(fds[1], s, n);
		349	if (internal_iserror(w, &e)) {
		350	if (e == EINTR)
		351	continue;
		352	CHECK_EQ(EFAULT, e);
		353	return false;
		354	}
		355	s += w;
		356	n -= w;
		357
		358	while (w) {
		359	uptr r = internal_read(fds[0], d, w);
		360	if (internal_iserror(r, &e)) {
		361	CHECK_EQ(EINTR, e);
		362	continue;
		363	}
		364
		365	d += r;
		366	w -= r;
		367	}
307	}	368	}
308	internal_close(sock_pair[0]);	369
309	internal_close(sock_pair[1]);	370	return true;
310	return result;
311	}	371	}
312		372
313	void PlatformPrepareForSandboxing(void *args) {	373	void PlatformPrepareForSandboxing(void *args) {
...	@@ -483,7 +543,11 @@ pid_t StartSubprocess(const char program, const char const argv[],	...	@@ -483,7 +543,11 @@ pid_t StartSubprocess(const char program, const char const argv[],
483	internal_close(stderr_fd);	543	internal_close(stderr_fd);
484	}	544	}
485		545
		546	# if SANITIZER_FREEBSD
		547	internal_close_range(3, ~static_cast<fd_t>(0), 0);
		548	# else
486	for (int fd = sysconf(_SC_OPEN_MAX); fd > 2; fd--) internal_close(fd);	549	for (int fd = sysconf(_SC_OPEN_MAX); fd > 2; fd--) internal_close(fd);
		550	# endif
487		551
488	internal_execve(program, const_cast<char **>(&argv[0]),	552	internal_execve(program, const_cast<char **>(&argv[0]),
489	const_cast<char const >(envp));	553	const_cast<char const >(envp));

lib/tsan/sanitizer_common/sanitizer_procmaps_mac.cpp+3-1

 @@ -433,7 +433,9 @@ void MemoryMappingLayout::DumpListOfModules(
   MemoryMappedSegmentData data;
   segment.data_ = &data;
   while (Next(&segment)) {
-    if (segment.filename[0] == '\0') continue;
+    // skip the __PAGEZERO segment, its vmsize is 0
+    if (segment.filename[0] == '\0' || (segment.start == segment.end))
+      continue;
     LoadedModule *cur_module = nullptr;
     if (!modules->empty() &&
         0 == internal_strcmp(segment.filename, modules->back().full_name())) {

lib/tsan/sanitizer_common/sanitizer_procmaps_solaris.cpp+4

 @@ -11,6 +11,10 @@
 // Before Solaris 11.4, <procfs.h> doesn't work in a largefile environment.
 #undef _FILE_OFFSET_BITS
+// Avoid conflict between `_TIME_BITS` defined vs. `_FILE_OFFSET_BITS`
+// undefined in some Linux configurations.
+#undef _TIME_BITS
 #include "sanitizer_platform.h"
 #if SANITIZER_SOLARIS
 #  include <fcntl.h>

lib/tsan/sanitizer_common/sanitizer_redefine_builtins.h+5-3

 @@ -17,9 +17,11 @@
 // The asm hack only works with GCC and Clang.
 #    if !defined(_WIN32)
-asm("memcpy = __sanitizer_internal_memcpy");
+asm(R"(
-asm("memmove = __sanitizer_internal_memmove");
+    .set memcpy, __sanitizer_internal_memcpy
-asm("memset = __sanitizer_internal_memset");
+    .set memmove, __sanitizer_internal_memmove
+    .set memset, __sanitizer_internal_memset
+    )");
 #      if defined(__cplusplus) && \
           !defined(SANITIZER_COMMON_REDEFINE_BUILTINS_IN_STD)

lib/tsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp+11-4

 @@ -137,10 +137,6 @@ class ThreadSuspender {
 };
 bool ThreadSuspender::SuspendThread(tid_t tid) {
-  // Are we already attached to this thread?
-  // Currently this check takes linear time, however the number of threads is
-  // usually small.
-  if (suspended_threads_list_.ContainsTid(tid)) return false;
   int pterrno;
   if (internal_iserror(internal_ptrace(PTRACE_ATTACH, tid, nullptr, nullptr),
                        &pterrno)) {
 @@ -217,17 +213,28 @@ bool ThreadSuspender::SuspendAllThreads() {
     switch (thread_lister.ListThreads(&threads)) {
       case ThreadLister::Error:
         ResumeAllThreads();
+        VReport(1, "Failed to list threads\n");
         return false;
       case ThreadLister::Incomplete:
+        VReport(1, "Incomplete list\n");
         retry = true;
         break;
       case ThreadLister::Ok:
         break;
     }
     for (tid_t tid : threads) {
+      // Are we already attached to this thread?
+      // Currently this check takes linear time, however the number of threads
+      // is usually small.
+      if (suspended_threads_list_.ContainsTid(tid))
+        continue;
       if (SuspendThread(tid))
         retry = true;
+      else
+        VReport(2, "%llu/status: %s\n", tid, thread_lister.LoadStatus(tid));
     }
+    if (retry)
+      VReport(1, "SuspendAllThreads retry: %d\n", i);
   }
   return suspended_threads_list_.ThreadCount();
 }

lib/tsan/sanitizer_common/sanitizer_symbolizer.h+4-4

 @@ -185,17 +185,17 @@ class Symbolizer final {
   class ModuleNameOwner {
    public:
     explicit ModuleNameOwner(Mutex *synchronized_by)
-        : last_match_(nullptr), mu_(synchronized_by) {
+        : mu_(synchronized_by), last_match_(nullptr) {
       storage_.reserve(kInitialCapacity);
     }
     const char *GetOwnedCopy(const char *str);
    private:
     static const uptr kInitialCapacity = 1000;
-    InternalMmapVector<const char*> storage_;
-    const char *last_match_;
     Mutex *mu_;
+    const char *last_match_ SANITIZER_GUARDED_BY(mu_);
+    InternalMmapVector<const char *> storage_ SANITIZER_GUARDED_BY(*mu_);
   } module_names_;
   /// Platform-specific function for creating a Symbolizer object.
 @@ -220,7 +220,7 @@ class Symbolizer final {
   // always synchronized.
   Mutex mu_;
-  IntrusiveList<SymbolizerTool> tools_;
+  IntrusiveList<SymbolizerTool> tools_ SANITIZER_GUARDED_BY(mu_);
   explicit Symbolizer(IntrusiveList<SymbolizerTool> tools);

lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.cpp+2-2

lib/tsan/sanitizer_common/sanitizer_symbolizer_report.cpp+9-6

 @@ -227,12 +227,15 @@ static void ReportStackOverflowImpl(const SignalContext &sig, u32 tid,
          SanitizerToolName, kDescription, (void *)sig.addr, (void *)sig.pc,
          (void *)sig.bp, (void *)sig.sp, tid);
   Printf("%s", d.Default());
-  InternalMmapVector<BufferedStackTrace> stack_buffer(1);
+  // Avoid SEGVs in the unwinder when bp couldn't be determined.
-  BufferedStackTrace *stack = stack_buffer.data();
+  if (sig.bp) {
-  stack->Reset();
+    InternalMmapVector<BufferedStackTrace> stack_buffer(1);
-  unwind(sig, unwind_context, stack);
+    BufferedStackTrace *stack = stack_buffer.data();
-  stack->Print();
+    stack->Reset();
-  ReportErrorSummary(kDescription, stack);
+    unwind(sig, unwind_context, stack);
+    stack->Print();
+    ReportErrorSummary(kDescription, stack);
+  }
 }
 static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid,

lib/tsan/sanitizer_common/sanitizer_symbolizer_win.cpp+7-6

...	@@ -65,12 +65,13 @@ void InitializeDbgHelpIfNeeded() {	...	@@ -65,12 +65,13 @@ void InitializeDbgHelpIfNeeded() {
65	HMODULE dbghelp = LoadLibraryA("dbghelp.dll");	65	HMODULE dbghelp = LoadLibraryA("dbghelp.dll");
66	CHECK(dbghelp && "failed to load dbghelp.dll");	66	CHECK(dbghelp && "failed to load dbghelp.dll");
67		67
68	#define DBGHELP_IMPORT(name) \	68	# define DBGHELP_IMPORT(name) \
69	do { \	69	do { \
70	name = \	70	name = reinterpret_cast<decltype(::name) *>( \
71	reinterpret_cast<decltype(::name) *>(GetProcAddress(dbghelp, #name)); \	71	(void *)GetProcAddress(dbghelp, #name)); \
72	CHECK(name != nullptr); \	72	CHECK(name != nullptr); \
73	} while (0)	73	} while (0)
		74
74	DBGHELP_IMPORT(StackWalk64);	75	DBGHELP_IMPORT(StackWalk64);
75	DBGHELP_IMPORT(SymCleanup);	76	DBGHELP_IMPORT(SymCleanup);
76	DBGHELP_IMPORT(SymFromAddr);	77	DBGHELP_IMPORT(SymFromAddr);

lib/tsan/sanitizer_common/sanitizer_thread_history.cpp created+72

...	@@ -0,0 +1,72 @@
1	//===-- sanitizer_thread_history.cpp --------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "sanitizer_thread_history.h"
10
11	#include "sanitizer_stackdepot.h"
12	namespace __sanitizer {
13
14	void PrintThreadHistory(ThreadRegistry &registry, InternalScopedString &out) {
15	ThreadRegistryLock l(&registry);
16	// Stack traces are largest part of printout and they often the same for
17	// multiple threads, so we will deduplicate them.
18	InternalMmapVector<const ThreadContextBase *> stacks;
19
20	registry.RunCallbackForEachThreadLocked(
21	[](ThreadContextBase context, void arg) {
22	static_cast<decltype(&stacks)>(arg)->push_back(context);
23	},
24	&stacks);
25
26	Sort(stacks.data(), stacks.size(),
27	[](const ThreadContextBase a, const ThreadContextBase b) {
28	if (a->stack_id < b->stack_id)
29	return true;
30	if (a->stack_id > b->stack_id)
31	return false;
32	return a->unique_id < b->unique_id;
33	});
34
35	auto describe_thread = [&](const ThreadContextBase *context) {
36	if (!context) {
37	out.Append("T-1");
38	return;
39	}
40	out.AppendF("T%llu/%llu", context->unique_id, context->os_id);
41	if (internal_strlen(context->name))
42	out.AppendF(" (%s)", context->name);
43	};
44
45	auto get_parent =
46	[&](const ThreadContextBase context) -> const ThreadContextBase {
47	if (!context)
48	return nullptr;
49	ThreadContextBase *parent = registry.GetThreadLocked(context->parent_tid);
50	if (!parent)
51	return nullptr;
52	if (parent->unique_id >= context->unique_id)
53	return nullptr;
54	return parent;
55	};
56
57	const ThreadContextBase *prev = nullptr;
58	for (const ThreadContextBase *context : stacks) {
59	if (prev && prev->stack_id != context->stack_id)
60	StackDepotGet(prev->stack_id).PrintTo(&out);
61	prev = context;
62	out.Append("Thread ");
63	describe_thread(context);
64	out.Append(" was created by ");
65	describe_thread(get_parent(context));
66	out.Append("\n");
67	}
68	if (prev)
69	StackDepotGet(prev->stack_id).PrintTo(&out);
70	}
71
72	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_thread_history.h created+24

...	@@ -0,0 +1,24 @@
1	//===-- sanitizer_thread_history.h ------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Utility to print thread histroy from ThreadRegistry.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#ifndef SANITIZER_THREAD_HISTORY_H
14	#define SANITIZER_THREAD_HISTORY_H
15
16	#include "sanitizer_thread_registry.h"
17
18	namespace __sanitizer {
19
20	void PrintThreadHistory(ThreadRegistry& registry, InternalScopedString& out);
21
22	} // namespace __sanitizer
23
24	#endif // SANITIZER_THREAD_HISTORY_H

lib/tsan/sanitizer_common/sanitizer_thread_registry.cpp+33-18

 @@ -18,9 +18,17 @@
 namespace __sanitizer {
 ThreadContextBase::ThreadContextBase(u32 tid)
-    : tid(tid), unique_id(0), reuse_count(), os_id(0), user_id(0),
+    : tid(tid),
-      status(ThreadStatusInvalid), detached(false),
+      unique_id(0),
-      thread_type(ThreadType::Regular), parent_tid(0), next(0) {
+      reuse_count(),
+      os_id(0),
+      user_id(0),
+      status(ThreadStatusInvalid),
+      detached(false),
+      thread_type(ThreadType::Regular),
+      parent_tid(0),
+      stack_id(0),
+      next(0) {
   name[0] = '\0';
   atomic_store(&thread_destroyed, 0, memory_order_release);
 }
 @@ -39,8 +47,7 @@ void ThreadContextBase::SetName(const char *new_name) {
 }
 void ThreadContextBase::SetDead() {
-  CHECK(status == ThreadStatusRunning ||
+  CHECK(status == ThreadStatusRunning || status == ThreadStatusFinished);
-        status == ThreadStatusFinished);
   status = ThreadStatusDead;
   user_id = 0;
   OnDead();
 @@ -68,7 +75,8 @@ void ThreadContextBase::SetFinished() {
   // for a thread that never actually started.  In that case the thread
   // should go to ThreadStatusFinished regardless of whether it was created
   // as detached.
-  if (!detached || status == ThreadStatusCreated) status = ThreadStatusFinished;
+  if (!detached || status == ThreadStatusCreated)
+    status = ThreadStatusFinished;
   OnFinished();
 }
 @@ -81,14 +89,17 @@ void ThreadContextBase::SetStarted(tid_t _os_id, ThreadType _thread_type,
 }
 void ThreadContextBase::SetCreated(uptr _user_id, u64 _unique_id,
-                                   bool _detached, u32 _parent_tid, void *arg) {
+                                   bool _detached, u32 _parent_tid,
+                                   u32 _stack_tid, void *arg) {
   status = ThreadStatusCreated;
   user_id = _user_id;
   unique_id = _unique_id;
   detached = _detached;
   // Parent tid makes no sense for the main thread.
-  if (tid != kMainTid)
+  if (tid != kMainTid) {
     parent_tid = _parent_tid;
+    stack_id = _stack_tid;
+  }
   OnCreated(arg);
 }
 @@ -124,8 +135,10 @@ void ThreadRegistry::GetNumberOfThreads(uptr *total, uptr *running,
   ThreadRegistryLock l(this);
   if (total)
     *total = threads_.size();
-  if (running) *running = running_threads_;
+  if (running)
-  if (alive) *alive = alive_threads_;
+    *running = running_threads_;
+  if (alive)
+    *alive = alive_threads_;
 }
 uptr ThreadRegistry::GetMaxAliveThreads() {
 @@ -134,7 +147,7 @@ uptr ThreadRegistry::GetMaxAliveThreads() {
 }
 u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
-                                 void *arg) {
+                                 u32 stack_tid, void *arg) {
   ThreadRegistryLock l(this);
   u32 tid = kInvalidTid;
   ThreadContextBase *tctx = QuarantinePop();
 @@ -150,8 +163,10 @@ u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
     Report("%s: Thread limit (%u threads) exceeded. Dying.\n",
            SanitizerToolName, max_threads_);
 #else
-    Printf("race: limit on %u simultaneously alive goroutines is exceeded,"
+    Printf(
-        " dying\n", max_threads_);
+        "race: limit on %u simultaneously alive goroutines is exceeded,"
+        " dying\n",
+        max_threads_);
 #endif
     Die();
   }
 @@ -170,8 +185,8 @@ u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
     // positives later (e.g. if we join a wrong thread).
     CHECK(live_.try_emplace(user_id, tid).second);
   }
-  tctx->SetCreated(user_id, total_threads_++, detached,
+  tctx->SetCreated(user_id, total_threads_++, detached, parent_tid, stack_tid,
-                   parent_tid, arg);
+                   arg);
   return tid;
 }
 @@ -196,8 +211,8 @@ u32 ThreadRegistry::FindThread(FindThreadCallback cb, void *arg) {
   return kInvalidTid;
 }
-ThreadContextBase *
+ThreadContextBase *ThreadRegistry::FindThreadContextLocked(
-ThreadRegistry::FindThreadContextLocked(FindThreadCallback cb, void *arg) {
+    FindThreadCallback cb, void *arg) {
   CheckLocked();
   for (u32 tid = 0; tid < threads_.size(); tid++) {
     ThreadContextBase *tctx = threads_[tid];
 @@ -210,7 +225,7 @@ ThreadRegistry::FindThreadContextLocked(FindThreadCallback cb, void *arg) {
 static bool FindThreadContextByOsIdCallback(ThreadContextBase *tctx,
                                             void *arg) {
   return (tctx->os_id == (uptr)arg && tctx->status != ThreadStatusInvalid &&
-      tctx->status != ThreadStatusDead);
+          tctx->status != ThreadStatusDead);
 }
 ThreadContextBase *ThreadRegistry::FindThreadContextByOsIDLocked(tid_t os_id) {

lib/tsan/sanitizer_common/sanitizer_thread_registry.h+8-3

 @@ -52,6 +52,7 @@ class ThreadContextBase {
   ThreadType thread_type;
   u32 parent_tid;
+  u32 stack_id;
   ThreadContextBase *next;  // For storing thread contexts in a list.
   atomic_uint32_t thread_destroyed; // To address race of Joined vs Finished
 @@ -63,7 +64,7 @@ class ThreadContextBase {
   void SetFinished();
   void SetStarted(tid_t _os_id, ThreadType _thread_type, void *arg);
   void SetCreated(uptr _user_id, u64 _unique_id, bool _detached,
-                  u32 _parent_tid, void *arg);
+                  u32 _parent_tid, u32 _stack_tid, void *arg);
   void Reset();
   void SetDestroyed();
 @@ -101,12 +102,16 @@ class SANITIZER_MUTEX ThreadRegistry {
   // Should be guarded by ThreadRegistryLock.
   ThreadContextBase *GetThreadLocked(u32 tid) {
-    return threads_.empty() ? nullptr : threads_[tid];
+    return tid < threads_.size() ? threads_[tid] : nullptr;
   }
   u32 NumThreadsLocked() const { return threads_.size(); }
-  u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg);
+  u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, u32 stack_tid,
+                   void *arg);
+  u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg) {
+    return CreateThread(user_id, detached, parent_tid, 0, arg);
+  }
   typedef void (*ThreadCallback)(ThreadContextBase *tctx, void *arg);
   // Invokes callback with a specified arg for each thread context.

lib/tsan/sanitizer_common/sanitizer_tls_get_addr.cpp+45-29

 @@ -14,6 +14,8 @@
 #include "sanitizer_allocator_interface.h"
 #include "sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_flags.h"
 #include "sanitizer_platform_interceptors.h"
 @@ -66,7 +68,7 @@ static DTLS::DTVBlock *DTLS_NextBlock(atomic_uintptr_t *cur) {
 }
 static DTLS::DTV *DTLS_Find(uptr id) {
-  VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", (void *)&dtls, id);
+  VReport(3, "__tls_get_addr: DTLS_Find %p %zd\n", (void *)&dtls, id);
   static constexpr uptr kPerBlock = ARRAY_SIZE(DTLS::DTVBlock::dtvs);
   DTLS::DTVBlock *cur = DTLS_NextBlock(&dtls.dtv_block);
   if (!cur)
 @@ -110,6 +112,21 @@ SANITIZER_WEAK_ATTRIBUTE
 const void *__sanitizer_get_allocated_begin(const void *p);
 }
+SANITIZER_INTERFACE_WEAK_DEF(uptr, __sanitizer_get_dtls_size,
+                             const void *tls_begin) {
+  const void *start = __sanitizer_get_allocated_begin(tls_begin);
+  if (!start)
+    return 0;
+  CHECK_LE(start, tls_begin);
+  uptr tls_size = __sanitizer_get_allocated_size(start);
+  VReport(2, "__tls_get_addr: glibc DTLS suspected; tls={%p,0x%zx}\n",
+          tls_begin, tls_size);
+  uptr offset =
+      (reinterpret_cast<uptr>(tls_begin) - reinterpret_cast<uptr>(start));
+  CHECK_LE(offset, tls_size);
+  return tls_size - offset;
+}
 DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
                                 uptr static_tls_begin, uptr static_tls_end) {
   if (!common_flags()->intercept_tls_get_addr) return 0;
 @@ -117,8 +134,8 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
   uptr dso_id = arg->dso_id;
   DTLS::DTV *dtv = DTLS_Find(dso_id);
   if (!dtv || dtv->beg)
-    return 0;
+    return nullptr;
-  uptr tls_size = 0;
+  CHECK_LE(static_tls_begin, static_tls_end);
   uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
   VReport(2,
           "__tls_get_addr: %p {0x%zx,0x%zx} => %p; tls_beg: %p; sp: %p "
 @@ -126,36 +143,33 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
           (void *)arg, arg->dso_id, arg->offset, res, (void *)tls_beg,
           (void *)&tls_beg,
           atomic_load(&number_of_live_dtls, memory_order_relaxed));
-  if (dtls.last_memalign_ptr == tls_beg) {
+  if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) {
-    tls_size = dtls.last_memalign_size;
-    VReport(2, "__tls_get_addr: glibc <=2.24 suspected; tls={%p,0x%zx}\n",
-            (void *)tls_beg, tls_size);
-  } else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) {
     // This is the static TLS block which was initialized / unpoisoned at thread
     // creation.
     VReport(2, "__tls_get_addr: static tls: %p\n", (void *)tls_beg);
-    tls_size = 0;
+    dtv->beg = tls_beg;
-  } else if (const void *start =
+    dtv->size = 0;
-                 __sanitizer_get_allocated_begin((void *)tls_beg)) {
+    return nullptr;
-    tls_beg = (uptr)start;
-    tls_size = __sanitizer_get_allocated_size(start);
-    VReport(2, "__tls_get_addr: glibc >=2.25 suspected; tls={%p,0x%zx}\n",
-            (void *)tls_beg, tls_size);
-  } else {
-    VReport(2, "__tls_get_addr: Can't guess glibc version\n");
-    // This may happen inside the DTOR of main thread, so just ignore it.
-    tls_size = 0;
   }
+  if (uptr tls_size =
+          __sanitizer_get_dtls_size(reinterpret_cast<void *>(tls_beg))) {
+    dtv->beg = tls_beg;
+    dtv->size = tls_size;
+    return dtv;
+  }
+  VReport(2, "__tls_get_addr: Can't guess glibc version\n");
+  // This may happen inside the DTOR a thread, or async signal handlers before
+  // thread initialization, so just ignore it.
+  //
+  // If the unknown block is dynamic TLS, unlikely we will be able to recognize
+  // it in future, mark it as done with '{tls_beg, 0}'.
+  //
+  // If the block is static TLS, possible reason of failed detection is nullptr
+  // in `static_tls_begin`. Regardless of reasons, the future handling of static
+  // TLS is still '{tls_beg, 0}'.
   dtv->beg = tls_beg;
-  dtv->size = tls_size;
+  dtv->size = 0;
-  return dtv;
+  return nullptr;
-}
-void DTLS_on_libc_memalign(void *ptr, uptr size) {
-  if (!common_flags()->intercept_tls_get_addr) return;
-  VReport(2, "DTLS_on_libc_memalign: %p 0x%zx\n", ptr, size);
-  dtls.last_memalign_ptr = reinterpret_cast<uptr>(ptr);
-  dtls.last_memalign_size = size;
 }
 DTLS *DTLS_Get() { return &dtls; }
 @@ -166,7 +180,9 @@ bool DTLSInDestruction(DTLS *dtls) {
 }
 #else
-void DTLS_on_libc_memalign(void *ptr, uptr size) {}
+SANITIZER_INTERFACE_WEAK_DEF(uptr, __sanitizer_get_dtls_size, const void *) {
+  return 0;
+}
 DTLS::DTV *DTLS_on_tls_get_addr(void *arg, void *res,
   unsigned long, unsigned long) { return 0; }
 DTLS *DTLS_Get() { return 0; }

lib/tsan/sanitizer_common/sanitizer_tls_get_addr.h-4

 @@ -55,10 +55,6 @@ struct DTLS {
   static_assert(sizeof(DTVBlock) <= 4096UL, "Unexpected block size");
   atomic_uintptr_t dtv_block;
-  // Auxiliary fields, don't access them outside sanitizer_tls_get_addr.cpp
-  uptr last_memalign_size;
-  uptr last_memalign_ptr;
 };
 template <typename Fn>

lib/tsan/sanitizer_common/sanitizer_win.cpp+81-20

...	@@ -164,7 +164,24 @@ void UnmapOrDie(void *addr, uptr size, bool raw_report) {	...	@@ -164,7 +164,24 @@ void UnmapOrDie(void *addr, uptr size, bool raw_report) {
164	static void ReturnNullptrOnOOMOrDie(uptr size, const char mem_type,	164	static void ReturnNullptrOnOOMOrDie(uptr size, const char mem_type,
165	const char *mmap_type) {	165	const char *mmap_type) {
166	error_t last_error = GetLastError();	166	error_t last_error = GetLastError();
167	if (last_error == ERROR_NOT_ENOUGH_MEMORY)	167
		168	// Assumption: VirtualAlloc is the last system call that was invoked before
		169	// this method.
		170	// VirtualAlloc emits one of 3 error codes when running out of memory
		171	// 1. ERROR_NOT_ENOUGH_MEMORY:
		172	// There's not enough memory to execute the command
		173	// 2. ERROR_INVALID_PARAMETER:
		174	// VirtualAlloc will return this if the request would allocate memory at an
		175	// address exceeding or being very close to the maximum application address
		176	// (the `lpMaximumApplicationAddress` field within the `SystemInfo` struct).
		177	// This does not seem to be officially documented, but is corroborated here:
		178	// https://stackoverflow.com/questions/45833674/why-does-virtualalloc-fail-for-lpaddress-greater-than-0x6ffffffffff
		179	// 3. ERROR_COMMITMENT_LIMIT:
		180	// VirtualAlloc will return this if e.g. the pagefile is too small to commit
		181	// the requested amount of memory.
		182	if (last_error == ERROR_NOT_ENOUGH_MEMORY \|\|
		183	last_error == ERROR_INVALID_PARAMETER \|\|
		184	last_error == ERROR_COMMITMENT_LIMIT)
168	return nullptr;	185	return nullptr;
169	ReportMmapFailureAndDie(size, mem_type, mmap_type, last_error);	186	ReportMmapFailureAndDie(size, mem_type, mmap_type, last_error);
170	}	187	}
...	@@ -873,24 +890,18 @@ uptr GetTlsSize() {	...	@@ -873,24 +890,18 @@ uptr GetTlsSize() {
873	return 0;	890	return 0;
874	}	891	}
875		892
876	void InitTlsSize() {	893	void GetThreadStackAndTls(bool main, uptr stk_begin, uptr stk_end,
877	}	894	uptr tls_begin, uptr tls_end) {
878		895	# if SANITIZER_GO
879	void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,	896	*stk_begin = 0;
880	uptr tls_addr, uptr tls_size) {	897	*stk_end = 0;
881	#if SANITIZER_GO	898	*tls_begin = 0;
882	*stk_addr = 0;	899	*tls_end = 0;
883	*stk_size = 0;	900	# else
884	*tls_addr = 0;	901	GetThreadStackTopAndBottom(main, stk_end, stk_begin);
885	*tls_size = 0;	902	*tls_begin = 0;
886	#else	903	*tls_end = 0;
887	uptr stack_top, stack_bottom;	904	# endif
888	GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);
889	*stk_addr = stack_bottom;
890	*stk_size = stack_top - stack_bottom;
891	*tls_addr = 0;
892	*tls_size = 0;
893	#endif
894	}	905	}
895		906
896	void ReportFile::Write(const char *buffer, uptr length) {	907	void ReportFile::Write(const char *buffer, uptr length) {
...	@@ -974,6 +985,11 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) {	...	@@ -974,6 +985,11 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) {
974	return true;	985	return true;
975	}	986	}
976		987
		988	bool TryMemCpy(void dest, const void src, uptr n) {
		989	// TODO: implement.
		990	return false;
		991	}
		992
977	bool SignalContext::IsStackOverflow() const {	993	bool SignalContext::IsStackOverflow() const {
978	return (DWORD)GetType() == EXCEPTION_STACK_OVERFLOW;	994	return (DWORD)GetType() == EXCEPTION_STACK_OVERFLOW;
979	}	995	}
...	@@ -1039,7 +1055,52 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -1039,7 +1055,52 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1039	}	1055	}
1040		1056
1041	void SignalContext::DumpAllRegisters(void *context) {	1057	void SignalContext::DumpAllRegisters(void *context) {
1042	// FIXME: Implement this.	1058	CONTEXT ctx = (CONTEXT )context;
		1059	# if defined(_M_X64)
		1060	Report("Register values:\n");
		1061	Printf("rax = %llx ", ctx->Rax);
		1062	Printf("rbx = %llx ", ctx->Rbx);
		1063	Printf("rcx = %llx ", ctx->Rcx);
		1064	Printf("rdx = %llx ", ctx->Rdx);
		1065	Printf("\n");
		1066	Printf("rdi = %llx ", ctx->Rdi);
		1067	Printf("rsi = %llx ", ctx->Rsi);
		1068	Printf("rbp = %llx ", ctx->Rbp);
		1069	Printf("rsp = %llx ", ctx->Rsp);
		1070	Printf("\n");
		1071	Printf("r8 = %llx ", ctx->R8);
		1072	Printf("r9 = %llx ", ctx->R9);
		1073	Printf("r10 = %llx ", ctx->R10);
		1074	Printf("r11 = %llx ", ctx->R11);
		1075	Printf("\n");
		1076	Printf("r12 = %llx ", ctx->R12);
		1077	Printf("r13 = %llx ", ctx->R13);
		1078	Printf("r14 = %llx ", ctx->R14);
		1079	Printf("r15 = %llx ", ctx->R15);
		1080	Printf("\n");
		1081	# elif defined(_M_IX86)
		1082	Report("Register values:\n");
		1083	Printf("eax = %lx ", ctx->Eax);
		1084	Printf("ebx = %lx ", ctx->Ebx);
		1085	Printf("ecx = %lx ", ctx->Ecx);
		1086	Printf("edx = %lx ", ctx->Edx);
		1087	Printf("\n");
		1088	Printf("edi = %lx ", ctx->Edi);
		1089	Printf("esi = %lx ", ctx->Esi);
		1090	Printf("ebp = %lx ", ctx->Ebp);
		1091	Printf("esp = %lx ", ctx->Esp);
		1092	Printf("\n");
		1093	# elif defined(_M_ARM64)
		1094	Report("Register values:\n");
		1095	for (int i = 0; i <= 30; i++) {
		1096	Printf("x%d%s = %llx", i < 10 ? " " : "", ctx->X[i]);
		1097	if (i % 4 == 3)
		1098	Printf("\n");
		1099	}
		1100	# else
		1101	// TODO
		1102	(void)ctx;
		1103	# endif
1043	}	1104	}
1044		1105
1045	int SignalContext::GetType() const {	1106	int SignalContext::GetType() const {

lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.cpp deleted-101

...	@@ -1,101 +0,0 @@
1	//===-- sanitizer_win_dll_thunk.cpp ---------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	// This file defines a family of thunks that should be statically linked into
9	// the DLLs that have instrumentation in order to delegate the calls to the
10	// shared runtime that lives in the main binary.
11	// See https://github.com/google/sanitizers/issues/209 for the details.
12	//===----------------------------------------------------------------------===//
13
14	#ifdef SANITIZER_DLL_THUNK
15	#include "sanitizer_win_defs.h"
16	#include "sanitizer_win_dll_thunk.h"
17	#include "interception/interception.h"
18
19	extern "C" {
20	void WINAPI GetModuleHandleA(const char module_name);
21	void abort();
22	}
23
24	namespace __sanitizer {
25	uptr dllThunkGetRealAddrOrDie(const char *name) {
26	uptr ret =
27	__interception::InternalGetProcAddress((void *)GetModuleHandleA(0), name);
28	if (!ret)
29	abort();
30	return ret;
31	}
32
33	int dllThunkIntercept(const char* main_function, uptr dll_function) {
34	uptr wrapper = dllThunkGetRealAddrOrDie(main_function);
35	if (!__interception::OverrideFunction(dll_function, wrapper, 0))
36	abort();
37	return 0;
38	}
39
40	int dllThunkInterceptWhenPossible(const char* main_function,
41	const char* default_function, uptr dll_function) {
42	uptr wrapper = __interception::InternalGetProcAddress(
43	(void *)GetModuleHandleA(0), main_function);
44	if (!wrapper)
45	wrapper = dllThunkGetRealAddrOrDie(default_function);
46	if (!__interception::OverrideFunction(dll_function, wrapper, 0))
47	abort();
48	return 0;
49	}
50	} // namespace __sanitizer
51
52	// Include Sanitizer Common interface.
53	#define INTERFACE_FUNCTION(Name) INTERCEPT_SANITIZER_FUNCTION(Name)
54	#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
55	#include "sanitizer_common_interface.inc"
56
57	#pragma section(".DLLTH$A", read)
58	#pragma section(".DLLTH$Z", read)
59
60	typedef void (*DllThunkCB)();
61	extern "C" {
62	__declspec(allocate(".DLLTH$A")) DllThunkCB __start_dll_thunk;
63	__declspec(allocate(".DLLTH$Z")) DllThunkCB __stop_dll_thunk;
64	}
65
66	// Disable compiler warnings that show up if we declare our own version
67	// of a compiler intrinsic (e.g. strlen).
68	#pragma warning(disable: 4391)
69	#pragma warning(disable: 4392)
70
71	extern "C" int __dll_thunk_init() {
72	static bool flag = false;
73	// __dll_thunk_init is expected to be called by only one thread.
74	if (flag) return 0;
75	flag = true;
76
77	for (DllThunkCB *it = &__start_dll_thunk; it < &__stop_dll_thunk; ++it)
78	if (*it)
79	(*it)();
80
81	// In DLLs, the callbacks are expected to return 0,
82	// otherwise CRT initialization fails.
83	return 0;
84	}
85
86	// We want to call dll_thunk_init before C/C++ initializers / constructors are
87	// executed, otherwise functions like memset might be invoked.
88	#pragma section(".CRT$XIB", long, read)
89	__declspec(allocate(".CRT$XIB")) int (*__dll_thunk_preinit)() =
90	__dll_thunk_init;
91
92	static void WINAPI dll_thunk_thread_init(void *mod, unsigned long reason,
93	void *reserved) {
94	if (reason == /DLL_PROCESS_ATTACH=/1) __dll_thunk_init();
95	}
96
97	#pragma section(".CRT$XLAB", long, read)
98	__declspec(allocate(".CRT$XLAB")) void (WINAPI __dll_thunk_tls_init)(void ,
99	unsigned long, void *) = dll_thunk_thread_init;
100
101	#endif // SANITIZER_DLL_THUNK

lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.h deleted-181

...	@@ -1,181 +0,0 @@
1	//===-- sanitizer_win_dll_thunk.h -----------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	// This header provide helper macros to delegate calls to the shared runtime
9	// that lives in the main executable. It should be included to dll_thunks that
10	// will be linked to the dlls, when the sanitizer is a static library included
11	// in the main executable.
12	//===----------------------------------------------------------------------===//
13	#ifndef SANITIZER_WIN_DLL_THUNK_H
14	#define SANITIZER_WIN_DLL_THUNK_H
15	#include "sanitizer_internal_defs.h"
16
17	namespace __sanitizer {
18	uptr dllThunkGetRealAddrOrDie(const char *name);
19
20	int dllThunkIntercept(const char* main_function, uptr dll_function);
21
22	int dllThunkInterceptWhenPossible(const char* main_function,
23	const char* default_function, uptr dll_function);
24	}
25
26	extern "C" int __dll_thunk_init();
27
28	// ----------------- Function interception helper macros -------------------- //
29	// Override dll_function with main_function from main executable.
30	#define INTERCEPT_OR_DIE(main_function, dll_function) \
31	static int intercept_##dll_function() { \
32	return __sanitizer::dllThunkIntercept(main_function, (__sanitizer::uptr) \
33	dll_function); \
34	} \
35	__pragma(section(".DLLTH$M", long, read)) \
36	__declspec(allocate(".DLLTH$M")) int (*__dll_thunk_##dll_function)() = \
37	intercept_##dll_function;
38
39	// Try to override dll_function with main_function from main executable.
40	// If main_function is not present, override dll_function with default_function.
41	#define INTERCEPT_WHEN_POSSIBLE(main_function, default_function, dll_function) \
42	static int intercept_##dll_function() { \
43	return __sanitizer::dllThunkInterceptWhenPossible(main_function, \
44	default_function, (__sanitizer::uptr)dll_function); \
45	} \
46	__pragma(section(".DLLTH$M", long, read)) \
47	__declspec(allocate(".DLLTH$M")) int (*__dll_thunk_##dll_function)() = \
48	intercept_##dll_function;
49
50	// -------------------- Function interception macros ------------------------ //
51	// Special case of hooks -- ASan own interface functions. Those are only called
52	// after __asan_init, thus an empty implementation is sufficient.
53	#define INTERCEPT_SANITIZER_FUNCTION(name) \
54	extern "C" __declspec(noinline) void name() { \
55	volatile int prevent_icf = (__LINE__ << 8) ^ __COUNTER__; \
56	static const char function_name[] = #name; \
57	for (const char* ptr = &function_name[0]; *ptr; ++ptr) \
58	prevent_icf ^= *ptr; \
59	(void)prevent_icf; \
60	__debugbreak(); \
61	} \
62	INTERCEPT_OR_DIE(#name, name)
63
64	// Special case of hooks -- Weak functions, could be redefined in the main
65	// executable, but that is not necessary, so we shouldn't die if we can not find
66	// a reference. Instead, when the function is not present in the main executable
67	// we consider the default impl provided by asan library.
68	#define INTERCEPT_SANITIZER_WEAK_FUNCTION(name) \
69	extern "C" __declspec(noinline) void name() { \
70	volatile int prevent_icf = (__LINE__ << 8) ^ __COUNTER__; \
71	static const char function_name[] = #name; \
72	for (const char* ptr = &function_name[0]; *ptr; ++ptr) \
73	prevent_icf ^= *ptr; \
74	(void)prevent_icf; \
75	__debugbreak(); \
76	} \
77	INTERCEPT_WHEN_POSSIBLE(#name, STRINGIFY(WEAK_EXPORT_NAME(name)), name)
78
79	// We can't define our own version of strlen etc. because that would lead to
80	// link-time or even type mismatch errors. Instead, we can declare a function
81	// just to be able to get its address. Me may miss the first few calls to the
82	// functions since it can be called before __dll_thunk_init, but that would lead
83	// to false negatives in the startup code before user's global initializers,
84	// which isn't a big deal.
85	#define INTERCEPT_LIBRARY_FUNCTION(name) \
86	extern "C" void name(); \
87	INTERCEPT_OR_DIE(STRINGIFY(WRAP(name)), name)
88
89	// Use these macros for functions that could be called before __dll_thunk_init()
90	// is executed and don't lead to errors if defined (free, malloc, etc).
91	#define INTERCEPT_WRAP_V_V(name) \
92	extern "C" void name() { \
93	typedef decltype(name) *fntype; \
94	static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
95	fn(); \
96	} \
97	INTERCEPT_OR_DIE(#name, name);
98
99	#define INTERCEPT_WRAP_V_W(name) \
100	extern "C" void name(void *arg) { \
101	typedef decltype(name) *fntype; \
102	static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
103	fn(arg); \
104	} \
105	INTERCEPT_OR_DIE(#name, name);
106
107	#define INTERCEPT_WRAP_V_WW(name) \
108	extern "C" void name(void arg1, void arg2) { \
109	typedef decltype(name) *fntype; \
110	static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
111	fn(arg1, arg2); \
112	} \
113	INTERCEPT_OR_DIE(#name, name);
114
115	#define INTERCEPT_WRAP_V_WWW(name) \
116	extern "C" void name(void arg1, void arg2, void *arg3) { \
117	typedef decltype(name) *fntype; \
118	static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
119	fn(arg1, arg2, arg3); \
120	} \
121	INTERCEPT_OR_DIE(#name, name);
122
123	#define INTERCEPT_WRAP_W_V(name) \
124	extern "C" void *name() { \
125	typedef decltype(name) *fntype; \
126	static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
127	return fn(); \
128	} \
129	INTERCEPT_OR_DIE(#name, name);
130
131	#define INTERCEPT_WRAP_W_W(name) \
132	extern "C" void name(void arg) { \
133	typedef decltype(name) *fntype; \
134	static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
135	return fn(arg); \
136	} \
137	INTERCEPT_OR_DIE(#name, name);
138
139	#define INTERCEPT_WRAP_W_WW(name) \
140	extern "C" void name(void arg1, void *arg2) { \
141	typedef decltype(name) *fntype; \
142	static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
143	return fn(arg1, arg2); \
144	} \
145	INTERCEPT_OR_DIE(#name, name);
146
147	#define INTERCEPT_WRAP_W_WWW(name) \
148	extern "C" void name(void arg1, void arg2, void arg3) { \
149	typedef decltype(name) *fntype; \
150	static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
151	return fn(arg1, arg2, arg3); \
152	} \
153	INTERCEPT_OR_DIE(#name, name);
154
155	#define INTERCEPT_WRAP_W_WWWW(name) \
156	extern "C" void name(void arg1, void arg2, void arg3, void *arg4) { \
157	typedef decltype(name) *fntype; \
158	static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
159	return fn(arg1, arg2, arg3, arg4); \
160	} \
161	INTERCEPT_OR_DIE(#name, name);
162
163	#define INTERCEPT_WRAP_W_WWWWW(name) \
164	extern "C" void name(void arg1, void arg2, void arg3, void *arg4, \
165	void *arg5) { \
166	typedef decltype(name) *fntype; \
167	static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
168	return fn(arg1, arg2, arg3, arg4, arg5); \
169	} \
170	INTERCEPT_OR_DIE(#name, name);
171
172	#define INTERCEPT_WRAP_W_WWWWWW(name) \
173	extern "C" void name(void arg1, void arg2, void arg3, void *arg4, \
174	void arg5, void arg6) { \
175	typedef decltype(name) *fntype; \
176	static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
177	return fn(arg1, arg2, arg3, arg4, arg5, arg6); \
178	} \
179	INTERCEPT_OR_DIE(#name, name);
180
181	#endif // SANITIZER_WIN_DLL_THUNK_H

lib/tsan/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cpp deleted-26

...	@@ -1,26 +0,0 @@
1	//===-- santizer_win_dynamic_runtime_thunk.cpp ----------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file defines things that need to be present in the application modules
10	// to interact with Sanitizer Common, when it is included in a dll.
11	//
12	//===----------------------------------------------------------------------===//
13	#ifdef SANITIZER_DYNAMIC_RUNTIME_THUNK
14	#define SANITIZER_IMPORT_INTERFACE 1
15	#include "sanitizer_win_defs.h"
16	// Define weak alias for all weak functions imported from sanitizer common.
17	#define INTERFACE_FUNCTION(Name)
18	#define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name)
19	#include "sanitizer_common_interface.inc"
20	#endif // SANITIZER_DYNAMIC_RUNTIME_THUNK
21
22	namespace __sanitizer {
23	// Add one, otherwise unused, external symbol to this object file so that the
24	// Visual C++ linker includes it and reads the .drective section.
25	void ForceWholeArchiveIncludeForSanitizerCommon() {}
26	}

lib/tsan/sanitizer_common/sanitizer_win_immortalize.h created+71

...	@@ -0,0 +1,71 @@
1	//===-- sanitizer_win_immortalize.h ---------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between AddressSanitizer, and interception.
10	//
11	// Windows-specific thread-safe and pre-CRT global initialization safe
12	// infrastructure to create an object whose destructor is never called.
13	//===----------------------------------------------------------------------===//
14	#if SANITIZER_WINDOWS
15	# pragma once
16	// Requires including sanitizer_placement_new.h (which is not allowed to be
17	// included in headers).
18
19	# include "sanitizer_win_defs.h"
20	// These types are required to satisfy XFG which requires that the names of the
21	// types for indirect calls to be correct as well as the name of the original
22	// type for any typedefs.
23
24	// TODO: There must be a better way to do this
25	# ifndef _WINDOWS_
26	typedef void* PVOID;
27	typedef int BOOL;
28	typedef union _RTL_RUN_ONCE {
29	PVOID ptr;
30	} INIT_ONCE, *PINIT_ONCE;
31
32	extern "C" {
33	__declspec(dllimport) int WINAPI InitOnceExecuteOnce(
34	PINIT_ONCE, BOOL(WINAPI)(PINIT_ONCE, PVOID, PVOID), void, void);
35	}
36	# endif
37
38	namespace __sanitizer {
39	template <class Ty>
40	BOOL WINAPI immortalize_impl(PINIT_ONCE, PVOID storage_ptr, PVOID*) noexcept {
41	// Ty must provide a placement new operator
42	new (storage_ptr) Ty();
43	return 1;
44	}
45
46	template <class Ty, typename Arg>
47	BOOL WINAPI immortalize_impl(PINIT_ONCE, PVOID storage_ptr,
48	PVOID* param) noexcept {
49	// Ty must provide a placement new operator
50	new (storage_ptr) Ty(((Arg)param));
51	return 1;
52	}
53
54	template <class Ty>
55	Ty& immortalize() { // return a reference to an object that will live forever
56	static INIT_ONCE flag;
57	alignas(Ty) static unsigned char storage[sizeof(Ty)];
58	InitOnceExecuteOnce(&flag, immortalize_impl<Ty>, &storage, nullptr);
59	return reinterpret_cast<Ty&>(storage);
60	}
61
62	template <class Ty, typename Arg>
63	Ty& immortalize(
64	Arg arg) { // return a reference to an object that will live forever
65	static INIT_ONCE flag;
66	alignas(Ty) static unsigned char storage[sizeof(Ty)];
67	InitOnceExecuteOnce(&flag, immortalize_impl<Ty, Arg>, &storage, &arg);
68	return reinterpret_cast<Ty&>(storage);
69	}
70	} // namespace __sanitizer
71	#endif // SANITIZER_WINDOWS

lib/tsan/sanitizer_common/sanitizer_win_interception.cpp created+156

...	@@ -0,0 +1,156 @@
1	//===-- sanitizer_win_interception.cpp -------------------- --- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Windows-specific export surface to provide interception for parts of the
10	// runtime that are always statically linked, both for overriding user-defined
11	// functions as well as registering weak functions that the ASAN runtime should
12	// use over defaults.
13	//
14	//===----------------------------------------------------------------------===//
15
16	#include "sanitizer_platform.h"
17	#if SANITIZER_WINDOWS
18	# include <stddef.h>
19
20	# include "interception/interception.h"
21	# include "sanitizer_addrhashmap.h"
22	# include "sanitizer_common.h"
23	# include "sanitizer_internal_defs.h"
24	# include "sanitizer_placement_new.h"
25	# include "sanitizer_win_immortalize.h"
26	# include "sanitizer_win_interception.h"
27
28	using namespace __sanitizer;
29
30	extern "C" void *__ImageBase;
31
32	namespace __sanitizer {
33
34	static uptr GetSanitizerDllExport(const char *export_name) {
35	const uptr function_address =
36	__interception::InternalGetProcAddress(&__ImageBase, export_name);
37	if (function_address == 0) {
38	Report("ERROR: Failed to find sanitizer DLL export '%s'\n", export_name);
39	CHECK("Failed to find sanitizer DLL export" && 0);
40	}
41	return function_address;
42	}
43
44	struct WeakCallbackList {
45	explicit constexpr WeakCallbackList(RegisterWeakFunctionCallback cb)
46	: callback(cb), next(nullptr) {}
47
48	static void *operator new(size_t size) { return InternalAlloc(size); }
49
50	static void operator delete(void *p) { InternalFree(p); }
51
52	RegisterWeakFunctionCallback callback;
53	WeakCallbackList *next;
54	};
55	using WeakCallbackMap = AddrHashMap<WeakCallbackList *, 11>;
56
57	static WeakCallbackMap *GetWeakCallbackMap() {
58	return &immortalize<WeakCallbackMap>();
59	}
60
61	void AddRegisterWeakFunctionCallback(uptr export_address,
62	RegisterWeakFunctionCallback cb) {
63	WeakCallbackMap::Handle h_find_or_create(GetWeakCallbackMap(), export_address,
64	false, true);
65	CHECK(h_find_or_create.exists());
66	if (h_find_or_create.created()) {
67	*h_find_or_create = new WeakCallbackList(cb);
68	} else {
69	(*h_find_or_create)->next = new WeakCallbackList(cb);
70	}
71	}
72
73	static void RunWeakFunctionCallbacks(uptr export_address) {
74	WeakCallbackMap::Handle h_find(GetWeakCallbackMap(), export_address, false,
75	false);
76	if (!h_find.exists()) {
77	return;
78	}
79
80	WeakCallbackList list = h_find;
81	do {
82	list->callback();
83	} while ((list = list->next));
84	}
85
86	} // namespace __sanitizer
87
88	extern "C" __declspec(dllexport) bool __cdecl __sanitizer_override_function(
89	const char *export_name, const uptr user_function,
90	uptr *const old_user_function) {
91	CHECK(export_name);
92	CHECK(user_function);
93
94	const uptr sanitizer_function = GetSanitizerDllExport(export_name);
95
96	const bool function_overridden = __interception::OverrideFunction(
97	user_function, sanitizer_function, old_user_function);
98	if (!function_overridden) {
99	Report(
100	"ERROR: Failed to override local function at '%p' with sanitizer "
101	"function '%s'\n",
102	user_function, export_name);
103	CHECK("Failed to replace local function with sanitizer version." && 0);
104	}
105
106	return function_overridden;
107	}
108
109	extern "C"
110	__declspec(dllexport) bool __cdecl __sanitizer_override_function_by_addr(
111	const uptr source_function, const uptr target_function,
112	uptr *const old_target_function) {
113	CHECK(source_function);
114	CHECK(target_function);
115
116	const bool function_overridden = __interception::OverrideFunction(
117	target_function, source_function, old_target_function);
118	if (!function_overridden) {
119	Report(
120	"ERROR: Failed to override function at '%p' with function at "
121	"'%p'\n",
122	target_function, source_function);
123	CHECK("Failed to apply function override." && 0);
124	}
125
126	return function_overridden;
127	}
128
129	extern "C"
130	__declspec(dllexport) bool __cdecl __sanitizer_register_weak_function(
131	const char *export_name, const uptr user_function,
132	uptr *const old_user_function) {
133	CHECK(export_name);
134	CHECK(user_function);
135
136	const uptr sanitizer_function = GetSanitizerDllExport(export_name);
137
138	const bool function_overridden = __interception::OverrideFunction(
139	sanitizer_function, user_function, old_user_function);
140	if (!function_overridden) {
141	Report(
142	"ERROR: Failed to register local function at '%p' to be used in "
143	"place of sanitizer function '%s'\n.",
144	user_function, export_name);
145	CHECK("Failed to register weak function." && 0);
146	}
147
148	// Note that thread-safety of RunWeakFunctionCallbacks in InitializeFlags
149	// depends on __sanitizer_register_weak_functions being called during the
150	// loader lock.
151	RunWeakFunctionCallbacks(sanitizer_function);
152
153	return function_overridden;
154	}
155
156	#endif // SANITIZER_WINDOWS

lib/tsan/sanitizer_common/sanitizer_win_interception.h created+32

...	@@ -0,0 +1,32 @@
1	//===-- sanitizer_win_interception.h ---------------------- --- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Windows-specific export surface to provide interception for parts of the
10	// runtime that are always statically linked, both for overriding user-defined
11	// functions as well as registering weak functions that the ASAN runtime should
12	// use over defaults.
13	//
14	//===----------------------------------------------------------------------===//
15
16	#ifndef SANITIZER_WIN_INTERCEPTION_H
17	#define SANITIZER_WIN_INTERCEPTION_H
18
19	#include "sanitizer_platform.h"
20	#if SANITIZER_WINDOWS
21
22	# include "sanitizer_common.h"
23	# include "sanitizer_internal_defs.h"
24
25	namespace __sanitizer {
26	using RegisterWeakFunctionCallback = void (*)();
27	void AddRegisterWeakFunctionCallback(uptr export_address,
28	RegisterWeakFunctionCallback cb);
29	} // namespace __sanitizer
30
31	#endif // SANITIZER_WINDOWS
32	#endif // SANITIZER_WIN_INTERCEPTION_H
	\ No newline at end of file

lib/tsan/sanitizer_common/sanitizer_win_weak_interception.cpp deleted-94

...	@@ -1,94 +0,0 @@
1	//===-- sanitizer_win_weak_interception.cpp -------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	// This module should be included in the sanitizer when it is implemented as a
9	// shared library on Windows (dll), in order to delegate the calls of weak
10	// functions to the implementation in the main executable when a strong
11	// definition is provided.
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_platform.h"
15	#if SANITIZER_WINDOWS && SANITIZER_DYNAMIC
16	#include "sanitizer_win_weak_interception.h"
17	#include "sanitizer_allocator_interface.h"
18	#include "sanitizer_interface_internal.h"
19	#include "sanitizer_win_defs.h"
20	#include "interception/interception.h"
21
22	extern "C" {
23	void WINAPI GetModuleHandleA(const char module_name);
24	void abort();
25	}
26
27	namespace __sanitizer {
28	// Try to get a pointer to real_function in the main module and override
29	// dll_function with that pointer. If the function isn't found, nothing changes.
30	int interceptWhenPossible(uptr dll_function, const char *real_function) {
31	uptr real = __interception::InternalGetProcAddress(
32	(void *)GetModuleHandleA(0), real_function);
33	if (real && !__interception::OverrideFunction((uptr)dll_function, real, 0))
34	abort();
35	return 0;
36	}
37	} // namespace __sanitizer
38
39	// Declare weak hooks.
40	extern "C" {
41	void __sanitizer_on_print(const char *str);
42	void __sanitizer_weak_hook_memcmp(uptr called_pc, const void *s1,
43	const void *s2, uptr n, int result);
44	void __sanitizer_weak_hook_strcmp(uptr called_pc, const char *s1,
45	const char *s2, int result);
46	void __sanitizer_weak_hook_strncmp(uptr called_pc, const char *s1,
47	const char *s2, uptr n, int result);
48	void __sanitizer_weak_hook_strstr(uptr called_pc, const char *s1,
49	const char s2, char result);
50	}
51
52	// Include Sanitizer Common interface.
53	#define INTERFACE_FUNCTION(Name)
54	#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
55	#include "sanitizer_common_interface.inc"
56
57	#pragma section(".WEAK$A", read)
58	#pragma section(".WEAK$Z", read)
59
60	typedef void (*InterceptCB)();
61	extern "C" {
62	__declspec(allocate(".WEAK$A")) InterceptCB __start_weak_list;
63	__declspec(allocate(".WEAK$Z")) InterceptCB __stop_weak_list;
64	}
65
66	static int weak_intercept_init() {
67	static bool flag = false;
68	// weak_interception_init is expected to be called by only one thread.
69	if (flag) return 0;
70	flag = true;
71
72	for (InterceptCB *it = &__start_weak_list; it < &__stop_weak_list; ++it)
73	if (*it)
74	(*it)();
75
76	// In DLLs, the callbacks are expected to return 0,
77	// otherwise CRT initialization fails.
78	return 0;
79	}
80
81	#pragma section(".CRT$XIB", long, read)
82	__declspec(allocate(".CRT$XIB")) int (*__weak_intercept_preinit)() =
83	weak_intercept_init;
84
85	static void WINAPI weak_intercept_thread_init(void *mod, unsigned long reason,
86	void *reserved) {
87	if (reason == /DLL_PROCESS_ATTACH=/1) weak_intercept_init();
88	}
89
90	#pragma section(".CRT$XLAB", long, read)
91	__declspec(allocate(".CRT$XLAB")) void(WINAPI *__weak_intercept_tls_init)(
92	void , unsigned long, void ) = weak_intercept_thread_init;
93
94	#endif // SANITIZER_WINDOWS && SANITIZER_DYNAMIC

lib/tsan/sanitizer_common/sanitizer_win_weak_interception.h deleted-32

...	@@ -1,32 +0,0 @@
1	//===-- sanitizer_win_weak_interception.h ---------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	// This header provide helper macros to delegate calls of weak functions to the
9	// implementation in the main executable when a strong definition is present.
10	//===----------------------------------------------------------------------===//
11	#ifndef SANITIZER_WIN_WEAK_INTERCEPTION_H
12	#define SANITIZER_WIN_WEAK_INTERCEPTION_H
13	#include "sanitizer_internal_defs.h"
14
15	namespace __sanitizer {
16	int interceptWhenPossible(uptr dll_function, const char *real_function);
17	}
18
19	// ----------------- Function interception helper macros -------------------- //
20	// Weak functions, could be redefined in the main executable, but that is not
21	// necessary, so we shouldn't die if we can not find a reference.
22	#define INTERCEPT_WEAK(Name) interceptWhenPossible((uptr) Name, #Name);
23
24	#define INTERCEPT_SANITIZER_WEAK_FUNCTION(Name) \
25	static int intercept_##Name() { \
26	return __sanitizer::interceptWhenPossible((__sanitizer::uptr) Name, #Name);\
27	} \
28	__pragma(section(".WEAK$M", long, read)) \
29	__declspec(allocate(".WEAK$M")) int (*__weak_intercept_##Name)() = \
30	intercept_##Name;
31
32	#endif // SANITIZER_WIN_WEAK_INTERCEPTION_H

lib/tsan/tsan_interceptors_mac.cpp+123-97

...	@@ -14,22 +14,21 @@	...	@@ -14,22 +14,21 @@
14	#include "sanitizer_common/sanitizer_platform.h"	14	#include "sanitizer_common/sanitizer_platform.h"
15	#if SANITIZER_APPLE	15	#if SANITIZER_APPLE
16		16
17	#include "interception/interception.h"	17	# include <errno.h>
18	#include "tsan_interceptors.h"	18	# include <libkern/OSAtomic.h>
19	#include "tsan_interface.h"	19	# include <objc/objc-sync.h>
20	#include "tsan_interface_ann.h"	20	# include <os/lock.h>
21	#include "tsan_spinlock_defs_mac.h"	21	# include <sys/ucontext.h>
22	#include "sanitizer_common/sanitizer_addrhashmap.h"	22
23		23	# include "interception/interception.h"
24	#include <errno.h>	24	# include "sanitizer_common/sanitizer_addrhashmap.h"
25	#include <libkern/OSAtomic.h>	25	# include "tsan_interceptors.h"
26	#include <objc/objc-sync.h>	26	# include "tsan_interface.h"
27	#include <os/lock.h>	27	# include "tsan_interface_ann.h"
28	#include <sys/ucontext.h>	28
29		29	# if defined(__has_include) && __has_include(<xpc/xpc.h>)
30	#if defined(__has_include) && __has_include(<xpc/xpc.h>)	30	# include <xpc/xpc.h>
31	#include <xpc/xpc.h>	31	# endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
32	#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
33		32
34	typedef long long_t;	33	typedef long long_t;
35		34
...	@@ -49,51 +48,55 @@ static constexpr morder kMacOrderBarrier = mo_acq_rel;	...	@@ -49,51 +48,55 @@ static constexpr morder kMacOrderBarrier = mo_acq_rel;
49	static constexpr morder kMacOrderNonBarrier = mo_acq_rel;	48	static constexpr morder kMacOrderNonBarrier = mo_acq_rel;
50	static constexpr morder kMacFailureOrder = mo_relaxed;	49	static constexpr morder kMacFailureOrder = mo_relaxed;
51		50
52	#define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \	51	# define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
53	TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \	52	TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
54	SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \	53	SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
55	return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \	54	return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \
56	}	55	}
57
58	#define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
59	TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
60	SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
61	return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \
62	}
63		56
64	#define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \	57	# define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, \
65	TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \	58	mo) \
66	SCOPED_TSAN_INTERCEPTOR(f, ptr); \	59	TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
67	return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \	60	SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
68	}	61	return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \
		62	}
69		63
70	#define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \	64	# define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
71	mo) \	65	mo) \
72	TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \	66	TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
73	SCOPED_TSAN_INTERCEPTOR(f, ptr); \	67	SCOPED_TSAN_INTERCEPTOR(f, ptr); \
74	return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \	68	return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \
75	}	69	}
76		70
77	#define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \	71	# define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
78	m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \	72	mo) \
79	kMacOrderNonBarrier) \	73	TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
80	m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \	74	SCOPED_TSAN_INTERCEPTOR(f, ptr); \
81	kMacOrderBarrier) \	75	return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \
82	m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \	76	}
83	kMacOrderNonBarrier) \
84	m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \
85	kMacOrderBarrier)
86
87	#define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \
88	m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
89	kMacOrderNonBarrier) \
90	m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
91	kMacOrderBarrier) \
92	m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \
93	kMacOrderNonBarrier) \
94	m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \
95	__tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
96		77
		78	# define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \
		79	m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
		80	kMacOrderNonBarrier) \
		81	m(int32_t, int32_t, a32, f##32##Barrier, \
		82	__tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) \
		83	m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \
		84	kMacOrderNonBarrier) \
		85	m(int64_t, int64_t, a64, f##64##Barrier, \
		86	__tsan_atomic64_##tsan_atomic_f, kMacOrderBarrier)
		87
		88	# define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \
		89	m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
		90	kMacOrderNonBarrier) \
		91	m(int32_t, uint32_t, a32, f##32##Barrier, \
		92	__tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) \
		93	m_orig(int32_t, uint32_t, a32, f##32##Orig, \
		94	__tsan_atomic32_##tsan_atomic_f, kMacOrderNonBarrier) \
		95	m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \
		96	__tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
		97
		98	# pragma clang diagnostic push // OSAtomic* deprecation
		99	# pragma clang diagnostic ignored "-Wdeprecated-declarations"
97	OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,	100	OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,
98	OSATOMIC_INTERCEPTOR_PLUS_X)	101	OSATOMIC_INTERCEPTOR_PLUS_X)
99	OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,	102	OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,
...	@@ -106,23 +109,26 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,	...	@@ -106,23 +109,26 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,
106	OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)	109	OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
107	OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,	110	OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
108	OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)	111	OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
		112	# pragma clang diagnostic pop // OSAtomic* deprecation
		113
		114	# define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \
		115	TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \
		116	SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
		117	return tsan_atomic_f##_compare_exchange_strong( \
		118	(volatile tsan_t )ptr, (tsan_t )&old_value, (tsan_t)new_value, \
		119	kMacOrderNonBarrier, kMacFailureOrder); \
		120	} \
		121	\
		122	TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \
		123	t volatile *ptr) { \
		124	SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
		125	return tsan_atomic_f##_compare_exchange_strong( \
		126	(volatile tsan_t )ptr, (tsan_t )&old_value, (tsan_t)new_value, \
		127	kMacOrderBarrier, kMacFailureOrder); \
		128	}
109		129
110	#define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \	130	# pragma clang diagnostic push // OSAtomicCompareAndSwap* deprecation
111	TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \	131	# pragma clang diagnostic ignored "-Wdeprecated-declarations"
112	SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
113	return tsan_atomic_f##_compare_exchange_strong( \
114	(volatile tsan_t )ptr, (tsan_t )&old_value, (tsan_t)new_value, \
115	kMacOrderNonBarrier, kMacFailureOrder); \
116	} \
117	\
118	TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \
119	t volatile *ptr) { \
120	SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
121	return tsan_atomic_f##_compare_exchange_strong( \
122	(volatile tsan_t )ptr, (tsan_t )&old_value, (tsan_t)new_value, \
123	kMacOrderBarrier, kMacFailureOrder); \
124	}
125
126	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)	132	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
127	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,	133	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,
128	long_t)	134	long_t)
...	@@ -132,24 +138,28 @@ OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,	...	@@ -132,24 +138,28 @@ OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
132	int32_t)	138	int32_t)
133	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,	139	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
134	int64_t)	140	int64_t)
		141	# pragma clang diagnostic pop // OSAtomicCompareAndSwap* deprecation
		142
		143	# define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \
		144	TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
		145	SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \
		146	volatile char byte_ptr = ((volatile char )ptr) + (n >> 3); \
		147	char bit = 0x80u >> (n & 7); \
		148	char mask = clear ? ~bit : bit; \
		149	char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \
		150	return orig_byte & bit; \
		151	}
135		152
136	#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \	153	# define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \
137	TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \	154	OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
138	SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \	155	OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
139	volatile char byte_ptr = ((volatile char )ptr) + (n >> 3); \
140	char bit = 0x80u >> (n & 7); \
141	char mask = clear ? ~bit : bit; \
142	char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \
143	return orig_byte & bit; \
144	}
145
146	#define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \
147	OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
148	OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
149		156
		157	# pragma clang diagnostic push // OSAtomicTestAnd* deprecation
		158	# pragma clang diagnostic ignored "-Wdeprecated-declarations"
150	OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)	159	OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)
151	OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,	160	OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,
152	true)	161	true)
		162	# pragma clang diagnostic pop // OSAtomicTestAnd* deprecation
153		163
154	TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead list, void item,	164	TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead list, void item,
155	size_t offset) {	165	size_t offset) {
...	@@ -161,12 +171,13 @@ TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead list, void item,	...	@@ -161,12 +171,13 @@ TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead list, void item,
161	TSAN_INTERCEPTOR(void , OSAtomicDequeue, OSQueueHead list, size_t offset) {	171	TSAN_INTERCEPTOR(void , OSAtomicDequeue, OSQueueHead list, size_t offset) {
162	SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);	172	SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);
163	void *item = REAL(OSAtomicDequeue)(list, offset);	173	void *item = REAL(OSAtomicDequeue)(list, offset);
164	if (item) __tsan_acquire(item);	174	if (item)
		175	__tsan_acquire(item);
165	return item;	176	return item;
166	}	177	}
167		178
168	// OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.	179	// OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.
169	#if !SANITIZER_IOS	180	# if !SANITIZER_IOS
170		181
171	TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead list, void item,	182	TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead list, void item,
172	size_t offset) {	183	size_t offset) {
...	@@ -179,11 +190,22 @@ TSAN_INTERCEPTOR(void , OSAtomicFifoDequeue, OSFifoQueueHead list,	...	@@ -179,11 +190,22 @@ TSAN_INTERCEPTOR(void , OSAtomicFifoDequeue, OSFifoQueueHead list,
179	size_t offset) {	190	size_t offset) {
180	SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);	191	SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);
181	void *item = REAL(OSAtomicFifoDequeue)(list, offset);	192	void *item = REAL(OSAtomicFifoDequeue)(list, offset);
182	if (item) __tsan_acquire(item);	193	if (item)
		194	__tsan_acquire(item);
183	return item;	195	return item;
184	}	196	}
185		197
186	#endif	198	# endif
		199
		200	// If `OSSPINLOCK_USE_INLINED=1` is set, then SDK headers don't declare these
		201	// as functions, but macros that call non-deprecated APIs. Undefine these
		202	// macros so they don't interfere with the interceptor machinery.
		203	# undef OSSpinLockLock
		204	# undef OSSpinLockTry
		205	# undef OSSpinLockUnlock
		206
		207	# pragma clang diagnostic push // OSSpinLock* deprecation
		208	# pragma clang diagnostic ignored "-Wdeprecated-declarations"
187		209
188	TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {	210	TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
189	CHECK(!cur_thread()->is_dead);	211	CHECK(!cur_thread()->is_dead);
...	@@ -216,6 +238,7 @@ TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {	...	@@ -216,6 +238,7 @@ TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {
216	Release(thr, pc, (uptr)lock);	238	Release(thr, pc, (uptr)lock);
217	REAL(OSSpinLockUnlock)(lock);	239	REAL(OSSpinLockUnlock)(lock);
218	}	240	}
		241	# pragma clang diagnostic pop // OSSpinLock* deprecation
219		242
220	TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {	243	TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {
221	CHECK(!cur_thread()->is_dead);	244	CHECK(!cur_thread()->is_dead);
...	@@ -288,7 +311,7 @@ TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) {	...	@@ -288,7 +311,7 @@ TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) {
288	REAL(os_unfair_lock_unlock)(lock);	311	REAL(os_unfair_lock_unlock)(lock);
289	}	312	}
290		313
291	#if defined(__has_include) && __has_include(<xpc/xpc.h>)	314	# if defined(__has_include) && __has_include(<xpc/xpc.h>)
292		315
293	TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,	316	TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
294	xpc_connection_t connection, xpc_handler_t handler) {	317	xpc_connection_t connection, xpc_handler_t handler) {
...	@@ -342,7 +365,7 @@ TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {	...	@@ -342,7 +365,7 @@ TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
342	REAL(xpc_connection_cancel)(connection);	365	REAL(xpc_connection_cancel)(connection);
343	}	366	}
344		367
345	#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)	368	# endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
346		369
347	// Determines whether the Obj-C object pointer is a tagged pointer. Tagged	370	// Determines whether the Obj-C object pointer is a tagged pointer. Tagged
348	// pointers encode the object data directly in their pointer bits and do not	371	// pointers encode the object data directly in their pointer bits and do not
...	@@ -365,7 +388,7 @@ static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) {	...	@@ -365,7 +388,7 @@ static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) {
365	Map::Handle h(&Addresses, addr);	388	Map::Handle h(&Addresses, addr);
366	if (h.created()) {	389	if (h.created()) {
367	ThreadIgnoreBegin(thr, pc);	390	ThreadIgnoreBegin(thr, pc);
368	h = (uptr) user_alloc(thr, pc, /size=*/1);	391	h = (uptr)user_alloc(thr, pc, /size=*/1);
369	ThreadIgnoreEnd(thr);	392	ThreadIgnoreEnd(thr);
370	}	393	}
371	return *h;	394	return *h;
...	@@ -383,7 +406,8 @@ static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) {	...	@@ -383,7 +406,8 @@ static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) {
383		406
384	TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) {	407	TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) {
385	SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj);	408	SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj);
386	if (!obj) return REAL(objc_sync_enter)(obj);	409	if (!obj)
		410	return REAL(objc_sync_enter)(obj);
387	uptr addr = SyncAddressForObjCObject(obj, thr, pc);	411	uptr addr = SyncAddressForObjCObject(obj, thr, pc);
388	MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant);	412	MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant);
389	int result = REAL(objc_sync_enter)(obj);	413	int result = REAL(objc_sync_enter)(obj);
...	@@ -394,11 +418,13 @@ TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) {	...	@@ -394,11 +418,13 @@ TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) {
394		418
395	TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) {	419	TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) {
396	SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj);	420	SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj);
397	if (!obj) return REAL(objc_sync_exit)(obj);	421	if (!obj)
		422	return REAL(objc_sync_exit)(obj);
398	uptr addr = SyncAddressForObjCObject(obj, thr, pc);	423	uptr addr = SyncAddressForObjCObject(obj, thr, pc);
399	MutexUnlock(thr, pc, addr);	424	MutexUnlock(thr, pc, addr);
400	int result = REAL(objc_sync_exit)(obj);	425	int result = REAL(objc_sync_exit)(obj);
401	if (result != OBJC_SYNC_SUCCESS) MutexInvalidAccess(thr, pc, addr);	426	if (result != OBJC_SYNC_SUCCESS)
		427	MutexInvalidAccess(thr, pc, addr);
402	return result;	428	return result;
403	}	429	}
404		430
...	@@ -429,7 +455,7 @@ TSAN_INTERCEPTOR(int, swapcontext, ucontext_t oucp, const ucontext_t ucp) {	...	@@ -429,7 +455,7 @@ TSAN_INTERCEPTOR(int, swapcontext, ucontext_t oucp, const ucontext_t ucp) {
429		455
430	// On macOS, libc++ is always linked dynamically, so intercepting works the	456	// On macOS, libc++ is always linked dynamically, so intercepting works the
431	// usual way.	457	// usual way.
432	#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR	458	# define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
433		459
434	namespace {	460	namespace {
435	struct fake_shared_weak_count {	461	struct fake_shared_weak_count {

lib/tsan/tsan_interceptors_posix.cpp+45-17

...	@@ -12,14 +12,15 @@	...	@@ -12,14 +12,15 @@
12	// sanitizer_common/sanitizer_common_interceptors.inc	12	// sanitizer_common/sanitizer_common_interceptors.inc
13	//===----------------------------------------------------------------------===//	13	//===----------------------------------------------------------------------===//
14		14
		15	#include "sanitizer_common/sanitizer_allocator_dlsym.h"
15	#include "sanitizer_common/sanitizer_atomic.h"	16	#include "sanitizer_common/sanitizer_atomic.h"
16	#include "sanitizer_common/sanitizer_errno.h"	17	#include "sanitizer_common/sanitizer_errno.h"
17	#include "sanitizer_common/sanitizer_glibc_version.h"	18	#include "sanitizer_common/sanitizer_glibc_version.h"
		19	#include "sanitizer_common/sanitizer_internal_defs.h"
18	#include "sanitizer_common/sanitizer_libc.h"	20	#include "sanitizer_common/sanitizer_libc.h"
19	#include "sanitizer_common/sanitizer_linux.h"	21	#include "sanitizer_common/sanitizer_linux.h"
20	#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"	22	#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
21	#include "sanitizer_common/sanitizer_platform_limits_posix.h"	23	#include "sanitizer_common/sanitizer_platform_limits_posix.h"
22	#include "sanitizer_common/sanitizer_placement_new.h"
23	#include "sanitizer_common/sanitizer_posix.h"	24	#include "sanitizer_common/sanitizer_posix.h"
24	#include "sanitizer_common/sanitizer_stacktrace.h"	25	#include "sanitizer_common/sanitizer_stacktrace.h"
25	#include "sanitizer_common/sanitizer_tls_get_addr.h"	26	#include "sanitizer_common/sanitizer_tls_get_addr.h"
...	@@ -96,7 +97,7 @@ extern "C" int pthread_key_create(unsigned key, void (destructor)(void* v));	...	@@ -96,7 +97,7 @@ extern "C" int pthread_key_create(unsigned key, void (destructor)(void* v));
96	extern "C" int pthread_setspecific(unsigned key, const void *v);	97	extern "C" int pthread_setspecific(unsigned key, const void *v);
97	DECLARE_REAL(int, pthread_mutexattr_gettype, void , void )	98	DECLARE_REAL(int, pthread_mutexattr_gettype, void , void )
98	DECLARE_REAL(int, fflush, __sanitizer_FILE *fp)	99	DECLARE_REAL(int, fflush, __sanitizer_FILE *fp)
99	DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size)	100	DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, usize size)
100	DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)	101	DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
101	extern "C" int pthread_equal(void t1, void t2);	102	extern "C" int pthread_equal(void t1, void t2);
102	extern "C" void *pthread_self();	103	extern "C" void *pthread_self();
...	@@ -252,6 +253,13 @@ SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionBegin() {}	...	@@ -252,6 +253,13 @@ SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionBegin() {}
252	SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {}	253	SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {}
253	#endif	254	#endif
254		255
		256	// FIXME: Use for `in_symbolizer()` as well. As-is we can't use
		257	// `DlSymAllocator`, because it uses the primary allocator only. Symbolizer
		258	// requires support of the secondary allocator for larger blocks.
		259	struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
		260	static bool UseImpl() { return (ctx && !ctx->initialized); }
		261	};
		262
255	} // namespace __tsan	263	} // namespace __tsan
256		264
257	static ThreadSignalContext SigCtx(ThreadState thr) {	265	static ThreadSignalContext SigCtx(ThreadState thr) {
...	@@ -661,6 +669,8 @@ TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) {	...	@@ -661,6 +669,8 @@ TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) {
661	TSAN_INTERCEPTOR(void*, malloc, uptr size) {	669	TSAN_INTERCEPTOR(void*, malloc, uptr size) {
662	if (in_symbolizer())	670	if (in_symbolizer())
663	return InternalAlloc(size);	671	return InternalAlloc(size);
		672	if (DlsymAlloc::Use())
		673	return DlsymAlloc::Allocate(size);
664	void *p = 0;	674	void *p = 0;
665	{	675	{
666	SCOPED_INTERCEPTOR_RAW(malloc, size);	676	SCOPED_INTERCEPTOR_RAW(malloc, size);
...	@@ -678,12 +688,14 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {	...	@@ -678,12 +688,14 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
678	return user_memalign(thr, pc, align, sz);	688	return user_memalign(thr, pc, align, sz);
679	}	689	}
680		690
681	TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {	691	TSAN_INTERCEPTOR(void *, calloc, uptr n, uptr size) {
682	if (in_symbolizer())	692	if (in_symbolizer())
683	return InternalCalloc(size, n);	693	return InternalCalloc(n, size);
		694	if (DlsymAlloc::Use())
		695	return DlsymAlloc::Callocate(n, size);
684	void *p = 0;	696	void *p = 0;
685	{	697	{
686	SCOPED_INTERCEPTOR_RAW(calloc, size, n);	698	SCOPED_INTERCEPTOR_RAW(calloc, n, size);
687	p = user_calloc(thr, pc, size, n);	699	p = user_calloc(thr, pc, size, n);
688	}	700	}
689	invoke_malloc_hook(p, n * size);	701	invoke_malloc_hook(p, n * size);
...	@@ -693,6 +705,8 @@ TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {	...	@@ -693,6 +705,8 @@ TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
693	TSAN_INTERCEPTOR(void, realloc, void p, uptr size) {	705	TSAN_INTERCEPTOR(void, realloc, void p, uptr size) {
694	if (in_symbolizer())	706	if (in_symbolizer())
695	return InternalRealloc(p, size);	707	return InternalRealloc(p, size);
		708	if (DlsymAlloc::Use() \|\| DlsymAlloc::PointerIsMine(p))
		709	return DlsymAlloc::Realloc(p, size);
696	if (p)	710	if (p)
697	invoke_free_hook(p);	711	invoke_free_hook(p);
698	{	712	{
...	@@ -703,13 +717,13 @@ TSAN_INTERCEPTOR(void, realloc, void p, uptr size) {	...	@@ -703,13 +717,13 @@ TSAN_INTERCEPTOR(void, realloc, void p, uptr size) {
703	return p;	717	return p;
704	}	718	}
705		719
706	TSAN_INTERCEPTOR(void, reallocarray, void p, uptr size, uptr n) {	720	TSAN_INTERCEPTOR(void , reallocarray, void p, uptr n, uptr size) {
707	if (in_symbolizer())	721	if (in_symbolizer())
708	return InternalReallocArray(p, size, n);	722	return InternalReallocArray(p, n, size);
709	if (p)	723	if (p)
710	invoke_free_hook(p);	724	invoke_free_hook(p);
711	{	725	{
712	SCOPED_INTERCEPTOR_RAW(reallocarray, p, size, n);	726	SCOPED_INTERCEPTOR_RAW(reallocarray, p, n, size);
713	p = user_reallocarray(thr, pc, p, size, n);	727	p = user_reallocarray(thr, pc, p, size, n);
714	}	728	}
715	invoke_malloc_hook(p, size);	729	invoke_malloc_hook(p, size);
...	@@ -717,20 +731,24 @@ TSAN_INTERCEPTOR(void, reallocarray, void p, uptr size, uptr n) {	...	@@ -717,20 +731,24 @@ TSAN_INTERCEPTOR(void, reallocarray, void p, uptr size, uptr n) {
717	}	731	}
718		732
719	TSAN_INTERCEPTOR(void, free, void *p) {	733	TSAN_INTERCEPTOR(void, free, void *p) {
720	if (p == 0)	734	if (UNLIKELY(!p))
721	return;	735	return;
722	if (in_symbolizer())	736	if (in_symbolizer())
723	return InternalFree(p);	737	return InternalFree(p);
		738	if (DlsymAlloc::PointerIsMine(p))
		739	return DlsymAlloc::Free(p);
724	invoke_free_hook(p);	740	invoke_free_hook(p);
725	SCOPED_INTERCEPTOR_RAW(free, p);	741	SCOPED_INTERCEPTOR_RAW(free, p);
726	user_free(thr, pc, p);	742	user_free(thr, pc, p);
727	}	743	}
728		744
729	TSAN_INTERCEPTOR(void, cfree, void *p) {	745	TSAN_INTERCEPTOR(void, cfree, void *p) {
730	if (p == 0)	746	if (UNLIKELY(!p))
731	return;	747	return;
732	if (in_symbolizer())	748	if (in_symbolizer())
733	return InternalFree(p);	749	return InternalFree(p);
		750	if (DlsymAlloc::PointerIsMine(p))
		751	return DlsymAlloc::Free(p);
734	invoke_free_hook(p);	752	invoke_free_hook(p);
735	SCOPED_INTERCEPTOR_RAW(cfree, p);	753	SCOPED_INTERCEPTOR_RAW(cfree, p);
736	user_free(thr, pc, p);	754	user_free(thr, pc, p);
...	@@ -750,7 +768,7 @@ TSAN_INTERCEPTOR(char , strcpy, char dst, const char *src) {	...	@@ -750,7 +768,7 @@ TSAN_INTERCEPTOR(char , strcpy, char dst, const char *src) {
750	return REAL(strcpy)(dst, src);	768	return REAL(strcpy)(dst, src);
751	}	769	}
752		770
753	TSAN_INTERCEPTOR(char, strncpy, char dst, char *src, uptr n) {	771	TSAN_INTERCEPTOR(char, strncpy, char dst, char *src, usize n) {
754	SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n);	772	SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n);
755	uptr srclen = internal_strnlen(src, n);	773	uptr srclen = internal_strnlen(src, n);
756	MemoryAccessRange(thr, pc, (uptr)dst, n, true);	774	MemoryAccessRange(thr, pc, (uptr)dst, n, true);
...	@@ -1097,7 +1115,7 @@ int internal_pthread_create(void th, void attr, void (callback)(void *),	...	@@ -1097,7 +1115,7 @@ int internal_pthread_create(void th, void attr, void (callback)(void *),
1097	}	1115	}
1098	int internal_pthread_join(void th, void *ret) {	1116	int internal_pthread_join(void th, void *ret) {
1099	ScopedIgnoreInterceptors ignore;	1117	ScopedIgnoreInterceptors ignore;
1100	return REAL(pthread_join(th, ret));	1118	return REAL(pthread_join)(th, ret);
1101	}	1119	}
1102	} // namespace __sanitizer	1120	} // namespace __sanitizer
1103		1121
...	@@ -1662,13 +1680,23 @@ TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {	...	@@ -1662,13 +1680,23 @@ TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
1662	#endif	1680	#endif
1663		1681
1664	TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) {	1682	TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) {
1665	va_list ap;	1683	mode_t mode = 0;
1666	va_start(ap, oflag);	1684	if (OpenReadsVaArgs(oflag)) {
1667	mode_t mode = va_arg(ap, int);	1685	va_list ap;
1668	va_end(ap);	1686	va_start(ap, oflag);
		1687	mode = va_arg(ap, int);
		1688	va_end(ap);
		1689	}
		1690
1669	SCOPED_TSAN_INTERCEPTOR(open, name, oflag, mode);	1691	SCOPED_TSAN_INTERCEPTOR(open, name, oflag, mode);
1670	READ_STRING(thr, pc, name, 0);	1692	READ_STRING(thr, pc, name, 0);
1671	int fd = REAL(open)(name, oflag, mode);	1693
		1694	int fd;
		1695	if (OpenReadsVaArgs(oflag))
		1696	fd = REAL(open)(name, oflag, mode);
		1697	else
		1698	fd = REAL(open)(name, oflag);
		1699
1672	if (fd >= 0)	1700	if (fd >= 0)
1673	FdFileCreate(thr, pc, fd);	1701	FdFileCreate(thr, pc, fd);
1674	return fd;	1702	return fd;

lib/tsan/tsan_interface.h+76-75

 @@ -16,8 +16,8 @@
 #define TSAN_INTERFACE_H
 #include <sanitizer_common/sanitizer_internal_defs.h>
-using __sanitizer::uptr;
 using __sanitizer::tid_t;
+using __sanitizer::uptr;
 // This header should NOT include any other headers.
 // All functions in this header are extern "C" and start with __tsan_.
 @@ -203,17 +203,18 @@ int __tsan_get_alloc_stack(uptr addr, uptr *trace, uptr size, int *thread_id,
 namespace __tsan {
 // These should match declarations from public tsan_interface_atomic.h header.
-typedef unsigned char      a8;
+typedef unsigned char a8;
 typedef unsigned short a16;
-typedef unsigned int       a32;
+typedef unsigned int a32;
 typedef unsigned long long a64;
-#if !SANITIZER_GO && (defined(__SIZEOF_INT128__) \
+#if !SANITIZER_GO &&                                      \
-    || (__clang_major__ * 100 + __clang_minor__ >= 302)) && \
+    (defined(__SIZEOF_INT128__) ||                        \
+     (__clang_major__ * 100 + __clang_minor__ >= 302)) && \
     !defined(__mips64) && !defined(__s390x__)
 __extension__ typedef __int128 a128;
-# define __TSAN_HAS_INT128 1
+#  define __TSAN_HAS_INT128 1
 #else
-# define __TSAN_HAS_INT128 0
+#  define __TSAN_HAS_INT128 0
 #endif
 // Part of ABI, do not change.
 @@ -231,180 +232,180 @@ struct ThreadState;
 extern "C" {
 SANITIZER_INTERFACE_ATTRIBUTE
-a8 __tsan_atomic8_load(const volatile a8 *a, morder mo);
+a8 __tsan_atomic8_load(const volatile a8 *a, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a16 __tsan_atomic16_load(const volatile a16 *a, morder mo);
+a16 __tsan_atomic16_load(const volatile a16 *a, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a32 __tsan_atomic32_load(const volatile a32 *a, morder mo);
+a32 __tsan_atomic32_load(const volatile a32 *a, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a64 __tsan_atomic64_load(const volatile a64 *a, morder mo);
+a64 __tsan_atomic64_load(const volatile a64 *a, int mo);
 #if __TSAN_HAS_INT128
 SANITIZER_INTERFACE_ATTRIBUTE
-a128 __tsan_atomic128_load(const volatile a128 *a, morder mo);
+a128 __tsan_atomic128_load(const volatile a128 *a, int mo);
 #endif
 SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo);
+void __tsan_atomic8_store(volatile a8 *a, a8 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo);
+void __tsan_atomic16_store(volatile a16 *a, a16 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo);
+void __tsan_atomic32_store(volatile a32 *a, a32 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo);
+void __tsan_atomic64_store(volatile a64 *a, a64 v, int mo);
 #if __TSAN_HAS_INT128
 SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo);
+void __tsan_atomic128_store(volatile a128 *a, a128 v, int mo);
 #endif
 SANITIZER_INTERFACE_ATTRIBUTE
-a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo);
+a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo);
+a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo);
+a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo);
+a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, int mo);
 #if __TSAN_HAS_INT128
 SANITIZER_INTERFACE_ATTRIBUTE
-a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo);
+a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, int mo);
 #endif
 SANITIZER_INTERFACE_ATTRIBUTE
-a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo);
+a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo);
+a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo);
+a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo);
+a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, int mo);
 #if __TSAN_HAS_INT128
 SANITIZER_INTERFACE_ATTRIBUTE
-a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo);
+a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, int mo);
 #endif
 SANITIZER_INTERFACE_ATTRIBUTE
-a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo);
+a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo);
+a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo);
+a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo);
+a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, int mo);
 #if __TSAN_HAS_INT128
 SANITIZER_INTERFACE_ATTRIBUTE
-a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo);
+a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, int mo);
 #endif
 SANITIZER_INTERFACE_ATTRIBUTE
-a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo);
+a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo);
+a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo);
+a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo);
+a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, int mo);
 #if __TSAN_HAS_INT128
 SANITIZER_INTERFACE_ATTRIBUTE
-a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo);
+a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, int mo);
 #endif
 SANITIZER_INTERFACE_ATTRIBUTE
-a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo);
+a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo);
+a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo);
+a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo);
+a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, int mo);
 #if __TSAN_HAS_INT128
 SANITIZER_INTERFACE_ATTRIBUTE
-a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo);
+a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, int mo);
 #endif
 SANITIZER_INTERFACE_ATTRIBUTE
-a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo);
+a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo);
+a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo);
+a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo);
+a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, int mo);
 #if __TSAN_HAS_INT128
 SANITIZER_INTERFACE_ATTRIBUTE
-a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo);
+a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, int mo);
 #endif
 SANITIZER_INTERFACE_ATTRIBUTE
-a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo);
+a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo);
+a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo);
+a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo);
+a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, int mo);
 #if __TSAN_HAS_INT128
 SANITIZER_INTERFACE_ATTRIBUTE
-a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo);
+a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, int mo);
 #endif
 SANITIZER_INTERFACE_ATTRIBUTE
-int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v,
+int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, int mo,
-                                           morder mo, morder fmo);
+                                           int fmo);
 SANITIZER_INTERFACE_ATTRIBUTE
 int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
-                                            morder mo, morder fmo);
+                                            int mo, int fmo);
 SANITIZER_INTERFACE_ATTRIBUTE
 int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
-                                            morder mo, morder fmo);
+                                            int mo, int fmo);
 SANITIZER_INTERFACE_ATTRIBUTE
 int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
-                                            morder mo, morder fmo);
+                                            int mo, int fmo);
 #if __TSAN_HAS_INT128
 SANITIZER_INTERFACE_ATTRIBUTE
 int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
-                                             morder mo, morder fmo);
+                                             int mo, int fmo);
 #endif
 SANITIZER_INTERFACE_ATTRIBUTE
-int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, morder mo,
+int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, int mo,
-                                         morder fmo);
+                                         int fmo);
 SANITIZER_INTERFACE_ATTRIBUTE
 int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
-                                          morder mo, morder fmo);
+                                          int mo, int fmo);
 SANITIZER_INTERFACE_ATTRIBUTE
 int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
-                                          morder mo, morder fmo);
+                                          int mo, int fmo);
 SANITIZER_INTERFACE_ATTRIBUTE
 int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
-                                          morder mo, morder fmo);
+                                          int mo, int fmo);
 #if __TSAN_HAS_INT128
 SANITIZER_INTERFACE_ATTRIBUTE
 int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
-                                           morder mo, morder fmo);
+                                           int mo, int fmo);
 #endif
 SANITIZER_INTERFACE_ATTRIBUTE
-a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, morder mo,
+a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, int mo,
-                                       morder fmo);
+                                       int fmo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
+a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, int mo,
-                                         morder mo, morder fmo);
+                                         int fmo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
+a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, int mo,
-                                         morder mo, morder fmo);
+                                         int fmo);
 SANITIZER_INTERFACE_ATTRIBUTE
-a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
+a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, int mo,
-                                         morder mo, morder fmo);
+                                         int fmo);
 #if __TSAN_HAS_INT128
 SANITIZER_INTERFACE_ATTRIBUTE
 a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
-                                           morder mo, morder fmo);
+                                           int mo, int fmo);
 #endif
 SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_atomic_thread_fence(morder mo);
+void __tsan_atomic_thread_fence(int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_atomic_signal_fence(morder mo);
+void __tsan_atomic_signal_fence(int mo);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a);

lib/tsan/tsan_interface_atomic.cpp+476-430

...	@@ -18,9 +18,9 @@	...	@@ -18,9 +18,9 @@
18	// The following page contains more background information:	18	// The following page contains more background information:
19	// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/	19	// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
20		20
		21	#include "sanitizer_common/sanitizer_mutex.h"
21	#include "sanitizer_common/sanitizer_placement_new.h"	22	#include "sanitizer_common/sanitizer_placement_new.h"
22	#include "sanitizer_common/sanitizer_stacktrace.h"	23	#include "sanitizer_common/sanitizer_stacktrace.h"
23	#include "sanitizer_common/sanitizer_mutex.h"
24	#include "tsan_flags.h"	24	#include "tsan_flags.h"
25	#include "tsan_interface.h"	25	#include "tsan_interface.h"
26	#include "tsan_rtl.h"	26	#include "tsan_rtl.h"
...	@@ -34,8 +34,8 @@ static StaticSpinMutex mutex128;	...	@@ -34,8 +34,8 @@ static StaticSpinMutex mutex128;
34		34
35	#if SANITIZER_DEBUG	35	#if SANITIZER_DEBUG
36	static bool IsLoadOrder(morder mo) {	36	static bool IsLoadOrder(morder mo) {
37	return mo == mo_relaxed \|\| mo == mo_consume	37	return mo == mo_relaxed \|\| mo == mo_consume \|\| mo == mo_acquire \|\|
38	\|\| mo == mo_acquire \|\| mo == mo_seq_cst;	38	mo == mo_seq_cst;
39	}	39	}
40		40
41	static bool IsStoreOrder(morder mo) {	41	static bool IsStoreOrder(morder mo) {
...	@@ -48,42 +48,49 @@ static bool IsReleaseOrder(morder mo) {	...	@@ -48,42 +48,49 @@ static bool IsReleaseOrder(morder mo) {
48	}	48	}
49		49
50	static bool IsAcquireOrder(morder mo) {	50	static bool IsAcquireOrder(morder mo) {
51	return mo == mo_consume \|\| mo == mo_acquire	51	return mo == mo_consume \|\| mo == mo_acquire \|\| mo == mo_acq_rel \|\|
52	\|\| mo == mo_acq_rel \|\| mo == mo_seq_cst;	52	mo == mo_seq_cst;
53	}	53	}
54		54
55	static bool IsAcqRelOrder(morder mo) {	55	static bool IsAcqRelOrder(morder mo) {
56	return mo == mo_acq_rel \|\| mo == mo_seq_cst;	56	return mo == mo_acq_rel \|\| mo == mo_seq_cst;
57	}	57	}
58		58
59	template<typename T> T func_xchg(volatile T *v, T op) {	59	template <typename T>
		60	T func_xchg(volatile T *v, T op) {
60	T res = __sync_lock_test_and_set(v, op);	61	T res = __sync_lock_test_and_set(v, op);
61	// __sync_lock_test_and_set does not contain full barrier.	62	// __sync_lock_test_and_set does not contain full barrier.
62	__sync_synchronize();	63	__sync_synchronize();
63	return res;	64	return res;
64	}	65	}
65		66
66	template<typename T> T func_add(volatile T *v, T op) {	67	template <typename T>
		68	T func_add(volatile T *v, T op) {
67	return __sync_fetch_and_add(v, op);	69	return __sync_fetch_and_add(v, op);
68	}	70	}
69		71
70	template<typename T> T func_sub(volatile T *v, T op) {	72	template <typename T>
		73	T func_sub(volatile T *v, T op) {
71	return __sync_fetch_and_sub(v, op);	74	return __sync_fetch_and_sub(v, op);
72	}	75	}
73		76
74	template<typename T> T func_and(volatile T *v, T op) {	77	template <typename T>
		78	T func_and(volatile T *v, T op) {
75	return __sync_fetch_and_and(v, op);	79	return __sync_fetch_and_and(v, op);
76	}	80	}
77		81
78	template<typename T> T func_or(volatile T *v, T op) {	82	template <typename T>
		83	T func_or(volatile T *v, T op) {
79	return __sync_fetch_and_or(v, op);	84	return __sync_fetch_and_or(v, op);
80	}	85	}
81		86
82	template<typename T> T func_xor(volatile T *v, T op) {	87	template <typename T>
		88	T func_xor(volatile T *v, T op) {
83	return __sync_fetch_and_xor(v, op);	89	return __sync_fetch_and_xor(v, op);
84	}	90	}
85		91
86	template<typename T> T func_nand(volatile T *v, T op) {	92	template <typename T>
		93	T func_nand(volatile T *v, T op) {
87	// clang does not support __sync_fetch_and_nand.	94	// clang does not support __sync_fetch_and_nand.
88	T cmp = *v;	95	T cmp = *v;
89	for (;;) {	96	for (;;) {
...	@@ -95,7 +102,8 @@ template<typename T> T func_nand(volatile T *v, T op) {	...	@@ -95,7 +102,8 @@ template<typename T> T func_nand(volatile T *v, T op) {
95	}	102	}
96	}	103	}
97		104
98	template<typename T> T func_cas(volatile T *v, T cmp, T xch) {	105	template <typename T>
		106	T func_cas(volatile T *v, T cmp, T xch) {
99	return __sync_val_compare_and_swap(v, cmp, xch);	107	return __sync_val_compare_and_swap(v, cmp, xch);
100	}	108	}
101		109
...	@@ -103,8 +111,8 @@ template<typename T> T func_cas(volatile T *v, T cmp, T xch) {	...	@@ -103,8 +111,8 @@ template<typename T> T func_cas(volatile T *v, T cmp, T xch) {
103	// Atomic ops are executed under tsan internal mutex,	111	// Atomic ops are executed under tsan internal mutex,
104	// here we assume that the atomic variables are not accessed	112	// here we assume that the atomic variables are not accessed
105	// from non-instrumented code.	113	// from non-instrumented code.
106	#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO \	114	#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO && \
107	&& __TSAN_HAS_INT128	115	__TSAN_HAS_INT128
108	a128 func_xchg(volatile a128 *v, a128 op) {	116	a128 func_xchg(volatile a128 *v, a128 op) {
109	SpinMutexLock lock(&mutex128);	117	SpinMutexLock lock(&mutex128);
110	a128 cmp = *v;	118	a128 cmp = *v;
...	@@ -197,89 +205,24 @@ static atomic_uint64_t to_atomic(const volatile a64 a) {	...	@@ -197,89 +205,24 @@ static atomic_uint64_t to_atomic(const volatile a64 a) {
197		205
198	static memory_order to_mo(morder mo) {	206	static memory_order to_mo(morder mo) {
199	switch (mo) {	207	switch (mo) {
200	case mo_relaxed: return memory_order_relaxed;	208	case mo_relaxed:
201	case mo_consume: return memory_order_consume;	209	return memory_order_relaxed;
202	case mo_acquire: return memory_order_acquire;	210	case mo_consume:
203	case mo_release: return memory_order_release;	211	return memory_order_consume;
204	case mo_acq_rel: return memory_order_acq_rel;	212	case mo_acquire:
205	case mo_seq_cst: return memory_order_seq_cst;	213	return memory_order_acquire;
		214	case mo_release:
		215	return memory_order_release;
		216	case mo_acq_rel:
		217	return memory_order_acq_rel;
		218	case mo_seq_cst:
		219	return memory_order_seq_cst;
206	}	220	}
207	DCHECK(0);	221	DCHECK(0);
208	return memory_order_seq_cst;	222	return memory_order_seq_cst;
209	}	223	}
210		224
211	template<typename T>	225	namespace {
212	static T NoTsanAtomicLoad(const volatile T *a, morder mo) {
213	return atomic_load(to_atomic(a), to_mo(mo));
214	}
215
216	#if __TSAN_HAS_INT128 && !SANITIZER_GO
217	static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
218	SpinMutexLock lock(&mutex128);
219	return *a;
220	}
221	#endif
222
223	template <typename T>
224	static T AtomicLoad(ThreadState thr, uptr pc, const volatile T a, morder mo) {
225	DCHECK(IsLoadOrder(mo));
226	// This fast-path is critical for performance.
227	// Assume the access is atomic.
228	if (!IsAcquireOrder(mo)) {
229	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
230	kAccessRead \| kAccessAtomic);
231	return NoTsanAtomicLoad(a, mo);
232	}
233	// Don't create sync object if it does not exist yet. For example, an atomic
234	// pointer is initialized to nullptr and then periodically acquire-loaded.
235	T v = NoTsanAtomicLoad(a, mo);
236	SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a);
237	if (s) {
238	SlotLocker locker(thr);
239	ReadLock lock(&s->mtx);
240	thr->clock.Acquire(s->clock);
241	// Re-read under sync mutex because we need a consistent snapshot
242	// of the value and the clock we acquire.
243	v = NoTsanAtomicLoad(a, mo);
244	}
245	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessRead \| kAccessAtomic);
246	return v;
247	}
248
249	template<typename T>
250	static void NoTsanAtomicStore(volatile T *a, T v, morder mo) {
251	atomic_store(to_atomic(a), v, to_mo(mo));
252	}
253
254	#if __TSAN_HAS_INT128 && !SANITIZER_GO
255	static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) {
256	SpinMutexLock lock(&mutex128);
257	*a = v;
258	}
259	#endif
260
261	template <typename T>
262	static void AtomicStore(ThreadState thr, uptr pc, volatile T a, T v,
263	morder mo) {
264	DCHECK(IsStoreOrder(mo));
265	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite \| kAccessAtomic);
266	// This fast-path is critical for performance.
267	// Assume the access is atomic.
268	// Strictly saying even relaxed store cuts off release sequence,
269	// so must reset the clock.
270	if (!IsReleaseOrder(mo)) {
271	NoTsanAtomicStore(a, v, mo);
272	return;
273	}
274	SlotLocker locker(thr);
275	{
276	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
277	Lock lock(&s->mtx);
278	thr->clock.ReleaseStore(&s->clock);
279	NoTsanAtomicStore(a, v, mo);
280	}
281	IncrementEpoch(thr);
282	}
283		226
284	template <typename T, T (F)(volatile T v, T op)>	227	template <typename T, T (F)(volatile T v, T op)>
285	static T AtomicRMW(ThreadState thr, uptr pc, volatile T a, T v, morder mo) {	228	static T AtomicRMW(ThreadState thr, uptr pc, volatile T a, T v, morder mo) {
...	@@ -303,175 +246,265 @@ static T AtomicRMW(ThreadState thr, uptr pc, volatile T a, T v, morder mo) {	...	@@ -303,175 +246,265 @@ static T AtomicRMW(ThreadState thr, uptr pc, volatile T a, T v, morder mo) {
303	return v;	246	return v;
304	}	247	}
305		248
306	template<typename T>	249	struct OpLoad {
307	static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) {	250	template <typename T>
308	return func_xchg(a, v);	251	static T NoTsanAtomic(morder mo, const volatile T *a) {
309	}	252	return atomic_load(to_atomic(a), to_mo(mo));
		253	}
310		254
311	template<typename T>	255	#if __TSAN_HAS_INT128 && !SANITIZER_GO
312	static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) {	256	static a128 NoTsanAtomic(morder mo, const volatile a128 *a) {
313	return func_add(a, v);	257	SpinMutexLock lock(&mutex128);
314	}	258	return *a;
		259	}
		260	#endif
315		261
316	template<typename T>	262	template <typename T>
317	static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) {	263	static T Atomic(ThreadState thr, uptr pc, morder mo, const volatile T a) {
318	return func_sub(a, v);	264	DCHECK(IsLoadOrder(mo));
319	}	265	// This fast-path is critical for performance.
		266	// Assume the access is atomic.
		267	if (!IsAcquireOrder(mo)) {
		268	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
		269	kAccessRead \| kAccessAtomic);
		270	return NoTsanAtomic(mo, a);
		271	}
		272	// Don't create sync object if it does not exist yet. For example, an atomic
		273	// pointer is initialized to nullptr and then periodically acquire-loaded.
		274	T v = NoTsanAtomic(mo, a);
		275	SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a);
		276	if (s) {
		277	SlotLocker locker(thr);
		278	ReadLock lock(&s->mtx);
		279	thr->clock.Acquire(s->clock);
		280	// Re-read under sync mutex because we need a consistent snapshot
		281	// of the value and the clock we acquire.
		282	v = NoTsanAtomic(mo, a);
		283	}
		284	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
		285	kAccessRead \| kAccessAtomic);
		286	return v;
		287	}
		288	};
320		289
321	template<typename T>	290	struct OpStore {
322	static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) {	291	template <typename T>
323	return func_and(a, v);	292	static void NoTsanAtomic(morder mo, volatile T *a, T v) {
324	}	293	atomic_store(to_atomic(a), v, to_mo(mo));
		294	}
325		295
326	template<typename T>	296	#if __TSAN_HAS_INT128 && !SANITIZER_GO
327	static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) {	297	static void NoTsanAtomic(morder mo, volatile a128 *a, a128 v) {
328	return func_or(a, v);	298	SpinMutexLock lock(&mutex128);
329	}	299	*a = v;
		300	}
		301	#endif
330		302
331	template<typename T>	303	template <typename T>
332	static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) {	304	static void Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
333	return func_xor(a, v);	305	DCHECK(IsStoreOrder(mo));
334	}	306	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
		307	kAccessWrite \| kAccessAtomic);
		308	// This fast-path is critical for performance.
		309	// Assume the access is atomic.
		310	// Strictly saying even relaxed store cuts off release sequence,
		311	// so must reset the clock.
		312	if (!IsReleaseOrder(mo)) {
		313	NoTsanAtomic(mo, a, v);
		314	return;
		315	}
		316	SlotLocker locker(thr);
		317	{
		318	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
		319	Lock lock(&s->mtx);
		320	thr->clock.ReleaseStore(&s->clock);
		321	NoTsanAtomic(mo, a, v);
		322	}
		323	IncrementEpoch(thr);
		324	}
		325	};
335		326
336	template<typename T>	327	struct OpExchange {
337	static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) {	328	template <typename T>
338	return func_nand(a, v);	329	static T NoTsanAtomic(morder mo, volatile T *a, T v) {
339	}	330	return func_xchg(a, v);
		331	}
		332	template <typename T>
		333	static T Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
		334	return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
		335	}
		336	};
340		337
341	template<typename T>	338	struct OpFetchAdd {
342	static T AtomicExchange(ThreadState thr, uptr pc, volatile T a, T v,	339	template <typename T>
343	morder mo) {	340	static T NoTsanAtomic(morder mo, volatile T *a, T v) {
344	return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);	341	return func_add(a, v);
345	}	342	}
346		343
347	template<typename T>	344	template <typename T>
348	static T AtomicFetchAdd(ThreadState thr, uptr pc, volatile T a, T v,	345	static T Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
349	morder mo) {	346	return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
350	return AtomicRMW<T, func_add>(thr, pc, a, v, mo);	347	}
351	}	348	};
352		349
353	template<typename T>	350	struct OpFetchSub {
354	static T AtomicFetchSub(ThreadState thr, uptr pc, volatile T a, T v,	351	template <typename T>
355	morder mo) {	352	static T NoTsanAtomic(morder mo, volatile T *a, T v) {
356	return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);	353	return func_sub(a, v);
357	}	354	}
358		355
359	template<typename T>	356	template <typename T>
360	static T AtomicFetchAnd(ThreadState thr, uptr pc, volatile T a, T v,	357	static T Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
361	morder mo) {	358	return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
362	return AtomicRMW<T, func_and>(thr, pc, a, v, mo);	359	}
363	}	360	};
364		361
365	template<typename T>	362	struct OpFetchAnd {
366	static T AtomicFetchOr(ThreadState thr, uptr pc, volatile T a, T v,	363	template <typename T>
367	morder mo) {	364	static T NoTsanAtomic(morder mo, volatile T *a, T v) {
368	return AtomicRMW<T, func_or>(thr, pc, a, v, mo);	365	return func_and(a, v);
369	}	366	}
370		367
371	template<typename T>	368	template <typename T>
372	static T AtomicFetchXor(ThreadState thr, uptr pc, volatile T a, T v,	369	static T Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
373	morder mo) {	370	return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
374	return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);	371	}
375	}	372	};
376		373
377	template<typename T>	374	struct OpFetchOr {
378	static T AtomicFetchNand(ThreadState thr, uptr pc, volatile T a, T v,	375	template <typename T>
379	morder mo) {	376	static T NoTsanAtomic(morder mo, volatile T *a, T v) {
380	return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);	377	return func_or(a, v);
381	}	378	}
382		379
383	template<typename T>	380	template <typename T>
384	static bool NoTsanAtomicCAS(volatile T a, T c, T v, morder mo, morder fmo) {	381	static T Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
385	return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo));	382	return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
386	}	383	}
		384	};
387		385
388	#if __TSAN_HAS_INT128	386	struct OpFetchXor {
389	static bool NoTsanAtomicCAS(volatile a128 a, a128 c, a128 v,	387	template <typename T>
390	morder mo, morder fmo) {	388	static T NoTsanAtomic(morder mo, volatile T *a, T v) {
391	a128 old = *c;	389	return func_xor(a, v);
392	a128 cur = func_cas(a, old, v);	390	}
393	if (cur == old)
394	return true;
395	*c = cur;
396	return false;
397	}
398	#endif
399		391
400	template<typename T>	392	template <typename T>
401	static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {	393	static T Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
402	NoTsanAtomicCAS(a, &c, v, mo, fmo);	394	return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
403	return c;	395	}
404	}	396	};
405		397
406	template <typename T>	398	struct OpFetchNand {
407	static bool AtomicCAS(ThreadState thr, uptr pc, volatile T a, T *c, T v,	399	template <typename T>
408	morder mo, morder fmo) {	400	static T NoTsanAtomic(morder mo, volatile T *a, T v) {
409	// 31.7.2.18: "The failure argument shall not be memory_order_release	401	return func_nand(a, v);
410	// nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic	402	}
411	// (mo_relaxed) when those are used.
412	DCHECK(IsLoadOrder(fmo));
413		403
414	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite \| kAccessAtomic);	404	template <typename T>
415	if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) {	405	static T Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
416	T cc = *c;	406	return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
417	T pr = func_cas(a, cc, v);	407	}
418	if (pr == cc)	408	};
		409
		410	struct OpCAS {
		411	template <typename T>
		412	static bool NoTsanAtomic(morder mo, morder fmo, volatile T a, T c, T v) {
		413	return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo));
		414	}
		415
		416	#if __TSAN_HAS_INT128
		417	static bool NoTsanAtomic(morder mo, morder fmo, volatile a128 a, a128 c,
		418	a128 v) {
		419	a128 old = *c;
		420	a128 cur = func_cas(a, old, v);
		421	if (cur == old)
419	return true;	422	return true;
420	*c = pr;	423	*c = cur;
421	return false;	424	return false;
422	}	425	}
423	SlotLocker locker(thr);	426	#endif
424	bool release = IsReleaseOrder(mo);	427
425	bool success;	428	template <typename T>
426	{	429	static T NoTsanAtomic(morder mo, morder fmo, volatile T *a, T c, T v) {
427	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);	430	NoTsanAtomic(mo, fmo, a, &c, v);
428	RWLock lock(&s->mtx, release);	431	return c;
429	T cc = *c;	432	}
430	T pr = func_cas(a, cc, v);	433
431	success = pr == cc;	434	template <typename T>
432	if (!success) {	435	static bool Atomic(ThreadState *thr, uptr pc, morder mo, morder fmo,
		436	volatile T a, T c, T v) {
		437	// 31.7.2.18: "The failure argument shall not be memory_order_release
		438	// nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic
		439	// (mo_relaxed) when those are used.
		440	DCHECK(IsLoadOrder(fmo));
		441
		442	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
		443	kAccessWrite \| kAccessAtomic);
		444	if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) {
		445	T cc = *c;
		446	T pr = func_cas(a, cc, v);
		447	if (pr == cc)
		448	return true;
433	*c = pr;	449	*c = pr;
434	mo = fmo;	450	return false;
435	}	451	}
436	if (success && IsAcqRelOrder(mo))	452	SlotLocker locker(thr);
437	thr->clock.ReleaseAcquire(&s->clock);	453	bool release = IsReleaseOrder(mo);
438	else if (success && IsReleaseOrder(mo))	454	bool success;
439	thr->clock.Release(&s->clock);	455	{
440	else if (IsAcquireOrder(mo))	456	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
441	thr->clock.Acquire(s->clock);	457	RWLock lock(&s->mtx, release);
		458	T cc = *c;
		459	T pr = func_cas(a, cc, v);
		460	success = pr == cc;
		461	if (!success) {
		462	*c = pr;
		463	mo = fmo;
		464	}
		465	if (success && IsAcqRelOrder(mo))
		466	thr->clock.ReleaseAcquire(&s->clock);
		467	else if (success && IsReleaseOrder(mo))
		468	thr->clock.Release(&s->clock);
		469	else if (IsAcquireOrder(mo))
		470	thr->clock.Acquire(s->clock);
		471	}
		472	if (success && release)
		473	IncrementEpoch(thr);
		474	return success;
442	}	475	}
443	if (success && release)
444	IncrementEpoch(thr);
445	return success;
446	}
447		476
448	template<typename T>	477	template <typename T>
449	static T AtomicCAS(ThreadState *thr, uptr pc,	478	static T Atomic(ThreadState *thr, uptr pc, morder mo, morder fmo,
450	volatile T *a, T c, T v, morder mo, morder fmo) {	479	volatile T *a, T c, T v) {
451	AtomicCAS(thr, pc, a, &c, v, mo, fmo);	480	Atomic(thr, pc, mo, fmo, a, &c, v);
452	return c;	481	return c;
453	}	482	}
		483	};
454		484
455	#if !SANITIZER_GO	485	#if !SANITIZER_GO
456	static void NoTsanAtomicFence(morder mo) {	486	struct OpFence {
457	__sync_synchronize();	487	static void NoTsanAtomic(morder mo) { __sync_synchronize(); }
458	}
459		488
460	static void AtomicFence(ThreadState *thr, uptr pc, morder mo) {	489	static void Atomic(ThreadState *thr, uptr pc, morder mo) {
461	// FIXME(dvyukov): not implemented.	490	// FIXME(dvyukov): not implemented.
462	__sync_synchronize();	491	__sync_synchronize();
463	}	492	}
		493	};
464	#endif	494	#endif
465		495
		496	} // namespace
		497
466	// Interface functions follow.	498	// Interface functions follow.
467	#if !SANITIZER_GO	499	#if !SANITIZER_GO
468		500
469	// C/C++	501	// C/C++
470		502
471	static morder convert_morder(morder mo) {	503	static morder convert_morder(morder mo) {
472	if (flags()->force_seq_cst_atomics)	504	return flags()->force_seq_cst_atomics ? mo_seq_cst : mo;
473	return (morder)mo_seq_cst;	505	}
474		506
		507	static morder to_morder(int mo) {
475	// Filter out additional memory order flags:	508	// Filter out additional memory order flags:
476	// MEMMODEL_SYNC = 1 << 15	509	// MEMMODEL_SYNC = 1 << 15
477	// __ATOMIC_HLE_ACQUIRE = 1 << 16	510	// __ATOMIC_HLE_ACQUIRE = 1 << 16
...	@@ -482,468 +515,481 @@ static morder convert_morder(morder mo) {	...	@@ -482,468 +515,481 @@ static morder convert_morder(morder mo) {
482	// since we use __sync_ atomics for actual atomic operations,	515	// since we use __sync_ atomics for actual atomic operations,
483	// we can safely ignore it as well. It also subtly affects semantics,	516	// we can safely ignore it as well. It also subtly affects semantics,
484	// but we don't model the difference.	517	// but we don't model the difference.
485	return (morder)(mo & 0x7fff);	518	morder res = static_cast<morder>(static_cast<u8>(mo));
		519	DCHECK_LE(res, mo_seq_cst);
		520	return res;
486	}	521	}
487		522
488	# define ATOMIC_IMPL(func, ...) \	523	template <class Op, class... Types>
489	ThreadState *const thr = cur_thread(); \	524	ALWAYS_INLINE auto AtomicImpl(morder mo, Types... args) {
490	ProcessPendingSignals(thr); \	525	ThreadState *const thr = cur_thread();
491	if (UNLIKELY(thr->ignore_sync \|\| thr->ignore_interceptors)) \	526	ProcessPendingSignals(thr);
492	return NoTsanAtomic##func(__VA_ARGS__); \	527	if (UNLIKELY(thr->ignore_sync \|\| thr->ignore_interceptors))
493	mo = convert_morder(mo); \	528	return Op::NoTsanAtomic(mo, args...);
494	return Atomic##func(thr, GET_CALLER_PC(), __VA_ARGS__);	529	return Op::Atomic(thr, GET_CALLER_PC(), convert_morder(mo), args...);
		530	}
495		531
496	extern "C" {	532	extern "C" {
497	SANITIZER_INTERFACE_ATTRIBUTE	533	SANITIZER_INTERFACE_ATTRIBUTE
498	a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {	534	a8 __tsan_atomic8_load(const volatile a8 *a, int mo) {
499	ATOMIC_IMPL(Load, a, mo);	535	return AtomicImpl<OpLoad>(to_morder(mo), a);
500	}	536	}
501		537
502	SANITIZER_INTERFACE_ATTRIBUTE	538	SANITIZER_INTERFACE_ATTRIBUTE
503	a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {	539	a16 __tsan_atomic16_load(const volatile a16 *a, int mo) {
504	ATOMIC_IMPL(Load, a, mo);	540	return AtomicImpl<OpLoad>(to_morder(mo), a);
505	}	541	}
506		542
507	SANITIZER_INTERFACE_ATTRIBUTE	543	SANITIZER_INTERFACE_ATTRIBUTE
508	a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {	544	a32 __tsan_atomic32_load(const volatile a32 *a, int mo) {
509	ATOMIC_IMPL(Load, a, mo);	545	return AtomicImpl<OpLoad>(to_morder(mo), a);
510	}	546	}
511		547
512	SANITIZER_INTERFACE_ATTRIBUTE	548	SANITIZER_INTERFACE_ATTRIBUTE
513	a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {	549	a64 __tsan_atomic64_load(const volatile a64 *a, int mo) {
514	ATOMIC_IMPL(Load, a, mo);	550	return AtomicImpl<OpLoad>(to_morder(mo), a);
515	}	551	}
516		552
517	#if __TSAN_HAS_INT128	553	# if __TSAN_HAS_INT128
518	SANITIZER_INTERFACE_ATTRIBUTE	554	SANITIZER_INTERFACE_ATTRIBUTE
519	a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {	555	a128 __tsan_atomic128_load(const volatile a128 *a, int mo) {
520	ATOMIC_IMPL(Load, a, mo);	556	return AtomicImpl<OpLoad>(to_morder(mo), a);
521	}	557	}
522	#endif	558	# endif
523		559
524	SANITIZER_INTERFACE_ATTRIBUTE	560	SANITIZER_INTERFACE_ATTRIBUTE
525	void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {	561	void __tsan_atomic8_store(volatile a8 *a, a8 v, int mo) {
526	ATOMIC_IMPL(Store, a, v, mo);	562	return AtomicImpl<OpStore>(to_morder(mo), a, v);
527	}	563	}
528		564
529	SANITIZER_INTERFACE_ATTRIBUTE	565	SANITIZER_INTERFACE_ATTRIBUTE
530	void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {	566	void __tsan_atomic16_store(volatile a16 *a, a16 v, int mo) {
531	ATOMIC_IMPL(Store, a, v, mo);	567	return AtomicImpl<OpStore>(to_morder(mo), a, v);
532	}	568	}
533		569
534	SANITIZER_INTERFACE_ATTRIBUTE	570	SANITIZER_INTERFACE_ATTRIBUTE
535	void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {	571	void __tsan_atomic32_store(volatile a32 *a, a32 v, int mo) {
536	ATOMIC_IMPL(Store, a, v, mo);	572	return AtomicImpl<OpStore>(to_morder(mo), a, v);
537	}	573	}
538		574
539	SANITIZER_INTERFACE_ATTRIBUTE	575	SANITIZER_INTERFACE_ATTRIBUTE
540	void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {	576	void __tsan_atomic64_store(volatile a64 *a, a64 v, int mo) {
541	ATOMIC_IMPL(Store, a, v, mo);	577	return AtomicImpl<OpStore>(to_morder(mo), a, v);
542	}	578	}
543		579
544	#if __TSAN_HAS_INT128	580	# if __TSAN_HAS_INT128
545	SANITIZER_INTERFACE_ATTRIBUTE	581	SANITIZER_INTERFACE_ATTRIBUTE
546	void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {	582	void __tsan_atomic128_store(volatile a128 *a, a128 v, int mo) {
547	ATOMIC_IMPL(Store, a, v, mo);	583	return AtomicImpl<OpStore>(to_morder(mo), a, v);
548	}	584	}
549	#endif	585	# endif
550		586
551	SANITIZER_INTERFACE_ATTRIBUTE	587	SANITIZER_INTERFACE_ATTRIBUTE
552	a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {	588	a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, int mo) {
553	ATOMIC_IMPL(Exchange, a, v, mo);	589	return AtomicImpl<OpExchange>(to_morder(mo), a, v);
554	}	590	}
555		591
556	SANITIZER_INTERFACE_ATTRIBUTE	592	SANITIZER_INTERFACE_ATTRIBUTE
557	a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {	593	a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, int mo) {
558	ATOMIC_IMPL(Exchange, a, v, mo);	594	return AtomicImpl<OpExchange>(to_morder(mo), a, v);
559	}	595	}
560		596
561	SANITIZER_INTERFACE_ATTRIBUTE	597	SANITIZER_INTERFACE_ATTRIBUTE
562	a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {	598	a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, int mo) {
563	ATOMIC_IMPL(Exchange, a, v, mo);	599	return AtomicImpl<OpExchange>(to_morder(mo), a, v);
564	}	600	}
565		601
566	SANITIZER_INTERFACE_ATTRIBUTE	602	SANITIZER_INTERFACE_ATTRIBUTE
567	a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {	603	a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, int mo) {
568	ATOMIC_IMPL(Exchange, a, v, mo);	604	return AtomicImpl<OpExchange>(to_morder(mo), a, v);
569	}	605	}
570		606
571	#if __TSAN_HAS_INT128	607	# if __TSAN_HAS_INT128
572	SANITIZER_INTERFACE_ATTRIBUTE	608	SANITIZER_INTERFACE_ATTRIBUTE
573	a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {	609	a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, int mo) {
574	ATOMIC_IMPL(Exchange, a, v, mo);	610	return AtomicImpl<OpExchange>(to_morder(mo), a, v);
575	}	611	}
576	#endif	612	# endif
577		613
578	SANITIZER_INTERFACE_ATTRIBUTE	614	SANITIZER_INTERFACE_ATTRIBUTE
579	a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {	615	a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, int mo) {
580	ATOMIC_IMPL(FetchAdd, a, v, mo);	616	return AtomicImpl<OpFetchAdd>(to_morder(mo), a, v);
581	}	617	}
582		618
583	SANITIZER_INTERFACE_ATTRIBUTE	619	SANITIZER_INTERFACE_ATTRIBUTE
584	a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {	620	a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, int mo) {
585	ATOMIC_IMPL(FetchAdd, a, v, mo);	621	return AtomicImpl<OpFetchAdd>(to_morder(mo), a, v);
586	}	622	}
587		623
588	SANITIZER_INTERFACE_ATTRIBUTE	624	SANITIZER_INTERFACE_ATTRIBUTE
589	a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {	625	a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, int mo) {
590	ATOMIC_IMPL(FetchAdd, a, v, mo);	626	return AtomicImpl<OpFetchAdd>(to_morder(mo), a, v);
591	}	627	}
592		628
593	SANITIZER_INTERFACE_ATTRIBUTE	629	SANITIZER_INTERFACE_ATTRIBUTE
594	a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {	630	a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, int mo) {
595	ATOMIC_IMPL(FetchAdd, a, v, mo);	631	return AtomicImpl<OpFetchAdd>(to_morder(mo), a, v);
596	}	632	}
597		633
598	#if __TSAN_HAS_INT128	634	# if __TSAN_HAS_INT128
599	SANITIZER_INTERFACE_ATTRIBUTE	635	SANITIZER_INTERFACE_ATTRIBUTE
600	a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {	636	a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, int mo) {
601	ATOMIC_IMPL(FetchAdd, a, v, mo);	637	return AtomicImpl<OpFetchAdd>(to_morder(mo), a, v);
602	}	638	}
603	#endif	639	# endif
604		640
605	SANITIZER_INTERFACE_ATTRIBUTE	641	SANITIZER_INTERFACE_ATTRIBUTE
606	a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {	642	a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, int mo) {
607	ATOMIC_IMPL(FetchSub, a, v, mo);	643	return AtomicImpl<OpFetchSub>(to_morder(mo), a, v);
608	}	644	}
609		645
610	SANITIZER_INTERFACE_ATTRIBUTE	646	SANITIZER_INTERFACE_ATTRIBUTE
611	a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {	647	a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, int mo) {
612	ATOMIC_IMPL(FetchSub, a, v, mo);	648	return AtomicImpl<OpFetchSub>(to_morder(mo), a, v);
613	}	649	}
614		650
615	SANITIZER_INTERFACE_ATTRIBUTE	651	SANITIZER_INTERFACE_ATTRIBUTE
616	a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {	652	a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, int mo) {
617	ATOMIC_IMPL(FetchSub, a, v, mo);	653	return AtomicImpl<OpFetchSub>(to_morder(mo), a, v);
618	}	654	}
619		655
620	SANITIZER_INTERFACE_ATTRIBUTE	656	SANITIZER_INTERFACE_ATTRIBUTE
621	a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {	657	a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, int mo) {
622	ATOMIC_IMPL(FetchSub, a, v, mo);	658	return AtomicImpl<OpFetchSub>(to_morder(mo), a, v);
623	}	659	}
624		660
625	#if __TSAN_HAS_INT128	661	# if __TSAN_HAS_INT128
626	SANITIZER_INTERFACE_ATTRIBUTE	662	SANITIZER_INTERFACE_ATTRIBUTE
627	a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {	663	a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, int mo) {
628	ATOMIC_IMPL(FetchSub, a, v, mo);	664	return AtomicImpl<OpFetchSub>(to_morder(mo), a, v);
629	}	665	}
630	#endif	666	# endif
631		667
632	SANITIZER_INTERFACE_ATTRIBUTE	668	SANITIZER_INTERFACE_ATTRIBUTE
633	a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {	669	a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, int mo) {
634	ATOMIC_IMPL(FetchAnd, a, v, mo);	670	return AtomicImpl<OpFetchAnd>(to_morder(mo), a, v);
635	}	671	}
636		672
637	SANITIZER_INTERFACE_ATTRIBUTE	673	SANITIZER_INTERFACE_ATTRIBUTE
638	a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {	674	a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, int mo) {
639	ATOMIC_IMPL(FetchAnd, a, v, mo);	675	return AtomicImpl<OpFetchAnd>(to_morder(mo), a, v);
640	}	676	}
641		677
642	SANITIZER_INTERFACE_ATTRIBUTE	678	SANITIZER_INTERFACE_ATTRIBUTE
643	a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {	679	a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, int mo) {
644	ATOMIC_IMPL(FetchAnd, a, v, mo);	680	return AtomicImpl<OpFetchAnd>(to_morder(mo), a, v);
645	}	681	}
646		682
647	SANITIZER_INTERFACE_ATTRIBUTE	683	SANITIZER_INTERFACE_ATTRIBUTE
648	a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {	684	a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, int mo) {
649	ATOMIC_IMPL(FetchAnd, a, v, mo);	685	return AtomicImpl<OpFetchAnd>(to_morder(mo), a, v);
650	}	686	}
651		687
652	#if __TSAN_HAS_INT128	688	# if __TSAN_HAS_INT128
653	SANITIZER_INTERFACE_ATTRIBUTE	689	SANITIZER_INTERFACE_ATTRIBUTE
654	a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {	690	a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, int mo) {
655	ATOMIC_IMPL(FetchAnd, a, v, mo);	691	return AtomicImpl<OpFetchAnd>(to_morder(mo), a, v);
656	}	692	}
657	#endif	693	# endif
658		694
659	SANITIZER_INTERFACE_ATTRIBUTE	695	SANITIZER_INTERFACE_ATTRIBUTE
660	a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {	696	a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, int mo) {
661	ATOMIC_IMPL(FetchOr, a, v, mo);	697	return AtomicImpl<OpFetchOr>(to_morder(mo), a, v);
662	}	698	}
663		699
664	SANITIZER_INTERFACE_ATTRIBUTE	700	SANITIZER_INTERFACE_ATTRIBUTE
665	a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {	701	a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, int mo) {
666	ATOMIC_IMPL(FetchOr, a, v, mo);	702	return AtomicImpl<OpFetchOr>(to_morder(mo), a, v);
667	}	703	}
668		704
669	SANITIZER_INTERFACE_ATTRIBUTE	705	SANITIZER_INTERFACE_ATTRIBUTE
670	a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {	706	a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, int mo) {
671	ATOMIC_IMPL(FetchOr, a, v, mo);	707	return AtomicImpl<OpFetchOr>(to_morder(mo), a, v);
672	}	708	}
673		709
674	SANITIZER_INTERFACE_ATTRIBUTE	710	SANITIZER_INTERFACE_ATTRIBUTE
675	a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {	711	a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, int mo) {
676	ATOMIC_IMPL(FetchOr, a, v, mo);	712	return AtomicImpl<OpFetchOr>(to_morder(mo), a, v);
677	}	713	}
678		714
679	#if __TSAN_HAS_INT128	715	# if __TSAN_HAS_INT128
680	SANITIZER_INTERFACE_ATTRIBUTE	716	SANITIZER_INTERFACE_ATTRIBUTE
681	a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {	717	a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, int mo) {
682	ATOMIC_IMPL(FetchOr, a, v, mo);	718	return AtomicImpl<OpFetchOr>(to_morder(mo), a, v);
683	}	719	}
684	#endif	720	# endif
685		721
686	SANITIZER_INTERFACE_ATTRIBUTE	722	SANITIZER_INTERFACE_ATTRIBUTE
687	a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {	723	a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, int mo) {
688	ATOMIC_IMPL(FetchXor, a, v, mo);	724	return AtomicImpl<OpFetchXor>(to_morder(mo), a, v);
689	}	725	}
690		726
691	SANITIZER_INTERFACE_ATTRIBUTE	727	SANITIZER_INTERFACE_ATTRIBUTE
692	a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {	728	a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, int mo) {
693	ATOMIC_IMPL(FetchXor, a, v, mo);	729	return AtomicImpl<OpFetchXor>(to_morder(mo), a, v);
694	}	730	}
695		731
696	SANITIZER_INTERFACE_ATTRIBUTE	732	SANITIZER_INTERFACE_ATTRIBUTE
697	a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {	733	a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, int mo) {
698	ATOMIC_IMPL(FetchXor, a, v, mo);	734	return AtomicImpl<OpFetchXor>(to_morder(mo), a, v);
699	}	735	}
700		736
701	SANITIZER_INTERFACE_ATTRIBUTE	737	SANITIZER_INTERFACE_ATTRIBUTE
702	a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {	738	a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, int mo) {
703	ATOMIC_IMPL(FetchXor, a, v, mo);	739	return AtomicImpl<OpFetchXor>(to_morder(mo), a, v);
704	}	740	}
705		741
706	#if __TSAN_HAS_INT128	742	# if __TSAN_HAS_INT128
707	SANITIZER_INTERFACE_ATTRIBUTE	743	SANITIZER_INTERFACE_ATTRIBUTE
708	a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {	744	a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, int mo) {
709	ATOMIC_IMPL(FetchXor, a, v, mo);	745	return AtomicImpl<OpFetchXor>(to_morder(mo), a, v);
710	}	746	}
711	#endif	747	# endif
712		748
713	SANITIZER_INTERFACE_ATTRIBUTE	749	SANITIZER_INTERFACE_ATTRIBUTE
714	a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {	750	a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, int mo) {
715	ATOMIC_IMPL(FetchNand, a, v, mo);	751	return AtomicImpl<OpFetchNand>(to_morder(mo), a, v);
716	}	752	}
717		753
718	SANITIZER_INTERFACE_ATTRIBUTE	754	SANITIZER_INTERFACE_ATTRIBUTE
719	a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {	755	a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, int mo) {
720	ATOMIC_IMPL(FetchNand, a, v, mo);	756	return AtomicImpl<OpFetchNand>(to_morder(mo), a, v);
721	}	757	}
722		758
723	SANITIZER_INTERFACE_ATTRIBUTE	759	SANITIZER_INTERFACE_ATTRIBUTE
724	a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {	760	a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, int mo) {
725	ATOMIC_IMPL(FetchNand, a, v, mo);	761	return AtomicImpl<OpFetchNand>(to_morder(mo), a, v);
726	}	762	}
727		763
728	SANITIZER_INTERFACE_ATTRIBUTE	764	SANITIZER_INTERFACE_ATTRIBUTE
729	a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {	765	a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, int mo) {
730	ATOMIC_IMPL(FetchNand, a, v, mo);	766	return AtomicImpl<OpFetchNand>(to_morder(mo), a, v);
731	}	767	}
732		768
733	#if __TSAN_HAS_INT128	769	# if __TSAN_HAS_INT128
734	SANITIZER_INTERFACE_ATTRIBUTE	770	SANITIZER_INTERFACE_ATTRIBUTE
735	a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {	771	a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, int mo) {
736	ATOMIC_IMPL(FetchNand, a, v, mo);	772	return AtomicImpl<OpFetchNand>(to_morder(mo), a, v);
737	}	773	}
738	#endif	774	# endif
739		775
740	SANITIZER_INTERFACE_ATTRIBUTE	776	SANITIZER_INTERFACE_ATTRIBUTE
741	int __tsan_atomic8_compare_exchange_strong(volatile a8 a, a8 c, a8 v,	777	int __tsan_atomic8_compare_exchange_strong(volatile a8 a, a8 c, a8 v, int mo,
742	morder mo, morder fmo) {	778	int fmo) {
743	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);	779	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
744	}	780	}
745		781
746	SANITIZER_INTERFACE_ATTRIBUTE	782	SANITIZER_INTERFACE_ATTRIBUTE
747	int __tsan_atomic16_compare_exchange_strong(volatile a16 a, a16 c, a16 v,	783	int __tsan_atomic16_compare_exchange_strong(volatile a16 a, a16 c, a16 v,
748	morder mo, morder fmo) {	784	int mo, int fmo) {
749	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);	785	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
750	}	786	}
751		787
752	SANITIZER_INTERFACE_ATTRIBUTE	788	SANITIZER_INTERFACE_ATTRIBUTE
753	int __tsan_atomic32_compare_exchange_strong(volatile a32 a, a32 c, a32 v,	789	int __tsan_atomic32_compare_exchange_strong(volatile a32 a, a32 c, a32 v,
754	morder mo, morder fmo) {	790	int mo, int fmo) {
755	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);	791	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
756	}	792	}
757		793
758	SANITIZER_INTERFACE_ATTRIBUTE	794	SANITIZER_INTERFACE_ATTRIBUTE
759	int __tsan_atomic64_compare_exchange_strong(volatile a64 a, a64 c, a64 v,	795	int __tsan_atomic64_compare_exchange_strong(volatile a64 a, a64 c, a64 v,
760	morder mo, morder fmo) {	796	int mo, int fmo) {
761	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);	797	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
762	}	798	}
763		799
764	#if __TSAN_HAS_INT128	800	# if __TSAN_HAS_INT128
765	SANITIZER_INTERFACE_ATTRIBUTE	801	SANITIZER_INTERFACE_ATTRIBUTE
766	int __tsan_atomic128_compare_exchange_strong(volatile a128 a, a128 c, a128 v,	802	int __tsan_atomic128_compare_exchange_strong(volatile a128 a, a128 c, a128 v,
767	morder mo, morder fmo) {	803	int mo, int fmo) {
768	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);	804	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
769	}	805	}
770	#endif	806	# endif
771		807
772	SANITIZER_INTERFACE_ATTRIBUTE	808	SANITIZER_INTERFACE_ATTRIBUTE
773	int __tsan_atomic8_compare_exchange_weak(volatile a8 a, a8 c, a8 v,	809	int __tsan_atomic8_compare_exchange_weak(volatile a8 a, a8 c, a8 v, int mo,
774	morder mo, morder fmo) {	810	int fmo) {
775	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);	811	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
776	}	812	}
777		813
778	SANITIZER_INTERFACE_ATTRIBUTE	814	SANITIZER_INTERFACE_ATTRIBUTE
779	int __tsan_atomic16_compare_exchange_weak(volatile a16 a, a16 c, a16 v,	815	int __tsan_atomic16_compare_exchange_weak(volatile a16 a, a16 c, a16 v,
780	morder mo, morder fmo) {	816	int mo, int fmo) {
781	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);	817	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
782	}	818	}
783		819
784	SANITIZER_INTERFACE_ATTRIBUTE	820	SANITIZER_INTERFACE_ATTRIBUTE
785	int __tsan_atomic32_compare_exchange_weak(volatile a32 a, a32 c, a32 v,	821	int __tsan_atomic32_compare_exchange_weak(volatile a32 a, a32 c, a32 v,
786	morder mo, morder fmo) {	822	int mo, int fmo) {
787	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);	823	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
788	}	824	}
789		825
790	SANITIZER_INTERFACE_ATTRIBUTE	826	SANITIZER_INTERFACE_ATTRIBUTE
791	int __tsan_atomic64_compare_exchange_weak(volatile a64 a, a64 c, a64 v,	827	int __tsan_atomic64_compare_exchange_weak(volatile a64 a, a64 c, a64 v,
792	morder mo, morder fmo) {	828	int mo, int fmo) {
793	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);	829	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
794	}	830	}
795		831
796	#if __TSAN_HAS_INT128	832	# if __TSAN_HAS_INT128
797	SANITIZER_INTERFACE_ATTRIBUTE	833	SANITIZER_INTERFACE_ATTRIBUTE
798	int __tsan_atomic128_compare_exchange_weak(volatile a128 a, a128 c, a128 v,	834	int __tsan_atomic128_compare_exchange_weak(volatile a128 a, a128 c, a128 v,
799	morder mo, morder fmo) {	835	int mo, int fmo) {
800	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);	836	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
801	}	837	}
802	#endif	838	# endif
803		839
804	SANITIZER_INTERFACE_ATTRIBUTE	840	SANITIZER_INTERFACE_ATTRIBUTE
805	a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,	841	a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, int mo,
806	morder mo, morder fmo) {	842	int fmo) {
807	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);	843	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
808	}	844	}
809		845
810	SANITIZER_INTERFACE_ATTRIBUTE	846	SANITIZER_INTERFACE_ATTRIBUTE
811	a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,	847	a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, int mo,
812	morder mo, morder fmo) {	848	int fmo) {
813	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);	849	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
814	}	850	}
815		851
816	SANITIZER_INTERFACE_ATTRIBUTE	852	SANITIZER_INTERFACE_ATTRIBUTE
817	a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,	853	a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, int mo,
818	morder mo, morder fmo) {	854	int fmo) {
819	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);	855	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
820	}	856	}
821		857
822	SANITIZER_INTERFACE_ATTRIBUTE	858	SANITIZER_INTERFACE_ATTRIBUTE
823	a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,	859	a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, int mo,
824	morder mo, morder fmo) {	860	int fmo) {
825	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);	861	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
826	}	862	}
827		863
828	#if __TSAN_HAS_INT128	864	# if __TSAN_HAS_INT128
829	SANITIZER_INTERFACE_ATTRIBUTE	865	SANITIZER_INTERFACE_ATTRIBUTE
830	a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,	866	a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
831	morder mo, morder fmo) {	867	int mo, int fmo) {
832	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);	868	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
833	}	869	}
834	#endif	870	# endif
835		871
836	SANITIZER_INTERFACE_ATTRIBUTE	872	SANITIZER_INTERFACE_ATTRIBUTE
837	void __tsan_atomic_thread_fence(morder mo) { ATOMIC_IMPL(Fence, mo); }	873	void __tsan_atomic_thread_fence(int mo) {
		874	return AtomicImpl<OpFence>(to_morder(mo));
		875	}
838		876
839	SANITIZER_INTERFACE_ATTRIBUTE	877	SANITIZER_INTERFACE_ATTRIBUTE
840	void __tsan_atomic_signal_fence(morder mo) {	878	void __tsan_atomic_signal_fence(int mo) {}
841	}
842	} // extern "C"	879	} // extern "C"
843		880
844	#else // #if !SANITIZER_GO	881	#else // #if !SANITIZER_GO
845		882
846	// Go	883	// Go
847		884
848	# define ATOMIC(func, ...) \	885	template <class Op, class... Types>
849	if (thr->ignore_sync) { \	886	void AtomicGo(ThreadState *thr, uptr cpc, uptr pc, Types... args) {
850	NoTsanAtomic##func(__VA_ARGS__); \	887	if (thr->ignore_sync) {
851	} else { \	888	(void)Op::NoTsanAtomic(args...);
852	FuncEntry(thr, cpc); \	889	} else {
853	Atomic##func(thr, pc, __VA_ARGS__); \	890	FuncEntry(thr, cpc);
854	FuncExit(thr); \	891	(void)Op::Atomic(thr, pc, args...);
855	}	892	FuncExit(thr);
		893	}
		894	}
856		895
857	# define ATOMIC_RET(func, ret, ...) \	896	template <class Op, class... Types>
858	if (thr->ignore_sync) { \	897	auto AtomicGoRet(ThreadState *thr, uptr cpc, uptr pc, Types... args) {
859	(ret) = NoTsanAtomic##func(__VA_ARGS__); \	898	if (thr->ignore_sync) {
860	} else { \	899	return Op::NoTsanAtomic(args...);
861	FuncEntry(thr, cpc); \	900	} else {
862	(ret) = Atomic##func(thr, pc, __VA_ARGS__); \	901	FuncEntry(thr, cpc);
863	FuncExit(thr); \	902	auto ret = Op::Atomic(thr, pc, args...);
864	}	903	FuncExit(thr);
		904	return ret;
		905	}
		906	}
865		907
866	extern "C" {	908	extern "C" {
867	SANITIZER_INTERFACE_ATTRIBUTE	909	SANITIZER_INTERFACE_ATTRIBUTE
868	void __tsan_go_atomic32_load(ThreadState thr, uptr cpc, uptr pc, u8 a) {	910	void __tsan_go_atomic32_load(ThreadState thr, uptr cpc, uptr pc, u8 a) {
869	ATOMIC_RET(Load, (a32)(a+8), (a32*)a, mo_acquire);	911	(a32 )(a + 8) = AtomicGoRet<OpLoad>(thr, cpc, pc, mo_acquire, (a32 *)a);
870	}	912	}
871		913
872	SANITIZER_INTERFACE_ATTRIBUTE	914	SANITIZER_INTERFACE_ATTRIBUTE
873	void __tsan_go_atomic64_load(ThreadState thr, uptr cpc, uptr pc, u8 a) {	915	void __tsan_go_atomic64_load(ThreadState thr, uptr cpc, uptr pc, u8 a) {
874	ATOMIC_RET(Load, (a64)(a+8), (a64*)a, mo_acquire);	916	(a64 )(a + 8) = AtomicGoRet<OpLoad>(thr, cpc, pc, mo_acquire, (a64 *)a);
875	}	917	}
876		918
877	SANITIZER_INTERFACE_ATTRIBUTE	919	SANITIZER_INTERFACE_ATTRIBUTE
878	void __tsan_go_atomic32_store(ThreadState thr, uptr cpc, uptr pc, u8 a) {	920	void __tsan_go_atomic32_store(ThreadState thr, uptr cpc, uptr pc, u8 a) {
879	ATOMIC(Store, (a32)a, (a32*)(a+8), mo_release);	921	AtomicGo<OpStore>(thr, cpc, pc, mo_release, (a32 )a, (a32 *)(a + 8));
880	}	922	}
881		923
882	SANITIZER_INTERFACE_ATTRIBUTE	924	SANITIZER_INTERFACE_ATTRIBUTE
883	void __tsan_go_atomic64_store(ThreadState thr, uptr cpc, uptr pc, u8 a) {	925	void __tsan_go_atomic64_store(ThreadState thr, uptr cpc, uptr pc, u8 a) {
884	ATOMIC(Store, (a64)a, (a64*)(a+8), mo_release);	926	AtomicGo<OpStore>(thr, cpc, pc, mo_release, (a64 )a, (a64 *)(a + 8));
885	}	927	}
886		928
887	SANITIZER_INTERFACE_ATTRIBUTE	929	SANITIZER_INTERFACE_ATTRIBUTE
888	void __tsan_go_atomic32_fetch_add(ThreadState thr, uptr cpc, uptr pc, u8 a) {	930	void __tsan_go_atomic32_fetch_add(ThreadState thr, uptr cpc, uptr pc, u8 a) {
889	ATOMIC_RET(FetchAdd, (a32)(a+16), (a32)a, (a32*)(a+8), mo_acq_rel);	931	(a32 )(a + 16) = AtomicGoRet<OpFetchAdd>(thr, cpc, pc, mo_acq_rel,
		932	(a32 )a, (a32 *)(a + 8));
890	}	933	}
891		934
892	SANITIZER_INTERFACE_ATTRIBUTE	935	SANITIZER_INTERFACE_ATTRIBUTE
893	void __tsan_go_atomic64_fetch_add(ThreadState thr, uptr cpc, uptr pc, u8 a) {	936	void __tsan_go_atomic64_fetch_add(ThreadState thr, uptr cpc, uptr pc, u8 a) {
894	ATOMIC_RET(FetchAdd, (a64)(a+16), (a64)a, (a64*)(a+8), mo_acq_rel);	937	(a64 )(a + 16) = AtomicGoRet<OpFetchAdd>(thr, cpc, pc, mo_acq_rel,
		938	(a64 )a, (a64 *)(a + 8));
895	}	939	}
896		940
897	SANITIZER_INTERFACE_ATTRIBUTE	941	SANITIZER_INTERFACE_ATTRIBUTE
898	void __tsan_go_atomic32_fetch_and(ThreadState thr, uptr cpc, uptr pc, u8 a) {	942	void __tsan_go_atomic32_fetch_and(ThreadState thr, uptr cpc, uptr pc, u8 a) {
899	ATOMIC_RET(FetchAnd, (a32 )(a + 16), (a32 )a, (a32 *)(a + 8),	943	(a32 )(a + 16) = AtomicGoRet<OpFetchAnd>(thr, cpc, pc, mo_acq_rel,
900	mo_acq_rel);	944	(a32 )a, (a32 *)(a + 8));
901	}	945	}
902		946
903	SANITIZER_INTERFACE_ATTRIBUTE	947	SANITIZER_INTERFACE_ATTRIBUTE
904	void __tsan_go_atomic64_fetch_and(ThreadState thr, uptr cpc, uptr pc, u8 a) {	948	void __tsan_go_atomic64_fetch_and(ThreadState thr, uptr cpc, uptr pc, u8 a) {
905	ATOMIC_RET(FetchAnd, (a64 )(a + 16), (a64 )a, (a64 *)(a + 8),	949	(a64 )(a + 16) = AtomicGoRet<OpFetchAnd>(thr, cpc, pc, mo_acq_rel,
906	mo_acq_rel);	950	(a64 )a, (a64 *)(a + 8));
907	}	951	}
908		952
909	SANITIZER_INTERFACE_ATTRIBUTE	953	SANITIZER_INTERFACE_ATTRIBUTE
910	void __tsan_go_atomic32_fetch_or(ThreadState thr, uptr cpc, uptr pc, u8 a) {	954	void __tsan_go_atomic32_fetch_or(ThreadState thr, uptr cpc, uptr pc, u8 a) {
911	ATOMIC_RET(FetchOr, (a32 )(a + 16), (a32 )a, (a32 *)(a + 8),	955	(a32 )(a + 16) = AtomicGoRet<OpFetchOr>(thr, cpc, pc, mo_acq_rel,
912	mo_acq_rel);	956	(a32 )a, (a32 *)(a + 8));
913	}	957	}
914		958
915	SANITIZER_INTERFACE_ATTRIBUTE	959	SANITIZER_INTERFACE_ATTRIBUTE
916	void __tsan_go_atomic64_fetch_or(ThreadState thr, uptr cpc, uptr pc, u8 a) {	960	void __tsan_go_atomic64_fetch_or(ThreadState thr, uptr cpc, uptr pc, u8 a) {
917	ATOMIC_RET(FetchOr, (a64 )(a + 16), (a64 )a, (a64 *)(a + 8),	961	(a64 )(a + 16) = AtomicGoRet<OpFetchOr>(thr, cpc, pc, mo_acq_rel,
918	mo_acq_rel);	962	(a64 )a, (a64 *)(a + 8));
919	}	963	}
920		964
921	SANITIZER_INTERFACE_ATTRIBUTE	965	SANITIZER_INTERFACE_ATTRIBUTE
922	void __tsan_go_atomic32_exchange(ThreadState thr, uptr cpc, uptr pc, u8 a) {	966	void __tsan_go_atomic32_exchange(ThreadState thr, uptr cpc, uptr pc, u8 a) {
923	ATOMIC_RET(Exchange, (a32)(a+16), (a32)a, (a32*)(a+8), mo_acq_rel);	967	(a32 )(a + 16) = AtomicGoRet<OpExchange>(thr, cpc, pc, mo_acq_rel,
		968	(a32 )a, (a32 *)(a + 8));
924	}	969	}
925		970
926	SANITIZER_INTERFACE_ATTRIBUTE	971	SANITIZER_INTERFACE_ATTRIBUTE
927	void __tsan_go_atomic64_exchange(ThreadState thr, uptr cpc, uptr pc, u8 a) {	972	void __tsan_go_atomic64_exchange(ThreadState thr, uptr cpc, uptr pc, u8 a) {
928	ATOMIC_RET(Exchange, (a64)(a+16), (a64)a, (a64*)(a+8), mo_acq_rel);	973	(a64 )(a + 16) = AtomicGoRet<OpExchange>(thr, cpc, pc, mo_acq_rel,
		974	(a64 )a, (a64 *)(a + 8));
929	}	975	}
930		976
931	SANITIZER_INTERFACE_ATTRIBUTE	977	SANITIZER_INTERFACE_ATTRIBUTE
932	void __tsan_go_atomic32_compare_exchange(	978	void __tsan_go_atomic32_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
933	ThreadState thr, uptr cpc, uptr pc, u8 a) {	979	u8 *a) {
934	a32 cur = 0;	980	a32 cmp = (a32 )(a + 8);
935	a32 cmp = (a32)(a+8);	981	a32 cur = AtomicGoRet<OpCAS>(thr, cpc, pc, mo_acq_rel, mo_acquire, (a32 *)a,
936	ATOMIC_RET(CAS, cur, (a32)a, cmp, (a32*)(a+12), mo_acq_rel, mo_acquire);	982	cmp, (a32 )(a + 12));
937	(bool)(a+16) = (cur == cmp);	983	(bool )(a + 16) = (cur == cmp);
938	}	984	}
939		985
940	SANITIZER_INTERFACE_ATTRIBUTE	986	SANITIZER_INTERFACE_ATTRIBUTE
941	void __tsan_go_atomic64_compare_exchange(	987	void __tsan_go_atomic64_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
942	ThreadState thr, uptr cpc, uptr pc, u8 a) {	988	u8 *a) {
943	a64 cur = 0;	989	a64 cmp = (a64 )(a + 8);
944	a64 cmp = (a64)(a+8);	990	a64 cur = AtomicGoRet<OpCAS>(thr, cpc, pc, mo_acq_rel, mo_acquire, (a64 *)a,
945	ATOMIC_RET(CAS, cur, (a64)a, cmp, (a64*)(a+16), mo_acq_rel, mo_acquire);	991	cmp, (a64 )(a + 16));
946	(bool)(a+24) = (cur == cmp);	992	(bool )(a + 24) = (cur == cmp);
947	}	993	}
948	} // extern "C"	994	} // extern "C"
949	#endif // #if !SANITIZER_GO	995	#endif // #if !SANITIZER_GO

lib/tsan/tsan_mman.cpp+1-1

...	@@ -252,7 +252,7 @@ void user_reallocarray(ThreadState thr, uptr pc, void *p, uptr size, uptr n) {	...	@@ -252,7 +252,7 @@ void user_reallocarray(ThreadState thr, uptr pc, void *p, uptr size, uptr n) {
252	if (AllocatorMayReturnNull())	252	if (AllocatorMayReturnNull())
253	return SetErrnoOnNull(nullptr);	253	return SetErrnoOnNull(nullptr);
254	GET_STACK_TRACE_FATAL(thr, pc);	254	GET_STACK_TRACE_FATAL(thr, pc);
255	ReportReallocArrayOverflow(size, n, &stack);	255	ReportReallocArrayOverflow(n, size, &stack);
256	}	256	}
257	return user_realloc(thr, pc, p, size * n);	257	return user_realloc(thr, pc, p, size * n);
258	}	258	}

lib/tsan/tsan_platform_linux.cpp-1

...	@@ -418,7 +418,6 @@ void InitializePlatform() {	...	@@ -418,7 +418,6 @@ void InitializePlatform() {
418	Die();	418	Die();
419	}	419	}
420		420
421	InitTlsSize();
422	#endif // !SANITIZER_GO	421	#endif // !SANITIZER_GO
423	}	422	}
424		423

lib/tsan/tsan_rtl.cpp+4-1

...	@@ -673,7 +673,8 @@ void CheckUnwind() {	...	@@ -673,7 +673,8 @@ void CheckUnwind() {
673	thr->ignore_reads_and_writes++;	673	thr->ignore_reads_and_writes++;
674	atomic_store_relaxed(&thr->in_signal_handler, 0);	674	atomic_store_relaxed(&thr->in_signal_handler, 0);
675	#endif	675	#endif
676	PrintCurrentStackSlow(StackTrace::GetCurrentPc());	676	PrintCurrentStack(StackTrace::GetCurrentPc(),
		677	common_flags()->fast_unwind_on_fatal);
677	}	678	}
678		679
679	bool is_initialized;	680	bool is_initialized;
...	@@ -806,6 +807,7 @@ int Finalize(ThreadState *thr) {	...	@@ -806,6 +807,7 @@ int Finalize(ThreadState *thr) {
806		807
807	#if !SANITIZER_GO	808	#if !SANITIZER_GO
808	void ForkBefore(ThreadState* thr, uptr pc) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {	809	void ForkBefore(ThreadState* thr, uptr pc) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
		810	VReport(2, "BeforeFork tid: %llu\n", GetTid());
809	GlobalProcessorLock();	811	GlobalProcessorLock();
810	// Detaching from the slot makes OnUserFree skip writing to the shadow.	812	// Detaching from the slot makes OnUserFree skip writing to the shadow.
811	// The slot will be locked so any attempts to use it will deadlock anyway.	813	// The slot will be locked so any attempts to use it will deadlock anyway.
...	@@ -847,6 +849,7 @@ static void ForkAfter(ThreadState* thr,	...	@@ -847,6 +849,7 @@ static void ForkAfter(ThreadState* thr,
847	SlotAttachAndLock(thr);	849	SlotAttachAndLock(thr);
848	SlotUnlock(thr);	850	SlotUnlock(thr);
849	GlobalProcessorUnlock();	851	GlobalProcessorUnlock();
		852	VReport(2, "AfterFork tid: %llu\n", GetTid());
850	}	853	}
851		854
852	void ForkParentAfter(ThreadState* thr, uptr pc) { ForkAfter(thr, false); }	855	void ForkParentAfter(ThreadState* thr, uptr pc) { ForkAfter(thr, false); }

lib/tsan/tsan_rtl.h+1-1

...	@@ -514,7 +514,7 @@ bool IsExpectedReport(uptr addr, uptr size);	...	@@ -514,7 +514,7 @@ bool IsExpectedReport(uptr addr, uptr size);
514	StackID CurrentStackId(ThreadState *thr, uptr pc);	514	StackID CurrentStackId(ThreadState *thr, uptr pc);
515	ReportStack *SymbolizeStackId(StackID stack_id);	515	ReportStack *SymbolizeStackId(StackID stack_id);
516	void PrintCurrentStack(ThreadState *thr, uptr pc);	516	void PrintCurrentStack(ThreadState *thr, uptr pc);
517	void PrintCurrentStackSlow(uptr pc); // uses libunwind	517	void PrintCurrentStack(uptr pc, bool fast); // may uses libunwind
518	MBlock JavaHeapBlock(uptr addr, uptr start);	518	MBlock JavaHeapBlock(uptr addr, uptr start);
519		519
520	void Initialize(ThreadState *thr);	520	void Initialize(ThreadState *thr);

lib/tsan/tsan_rtl_report.cpp+5-5

lib/tsan/tsan_rtl_thread.cpp+6-4

 @@ -165,14 +165,16 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
 #endif
   uptr stk_addr = 0;
-  uptr stk_size = 0;
+  uptr stk_end = 0;
   uptr tls_addr = 0;
-  uptr tls_size = 0;
+  uptr tls_end = 0;
 #if !SANITIZER_GO
   if (thread_type != ThreadType::Fiber)
-    GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_size, &tls_addr,
+    GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_end, &tls_addr,
-                         &tls_size);
+                         &tls_end);
 #endif
+  uptr stk_size = stk_end - stk_addr;
+  uptr tls_size = tls_end - tls_addr;
   thr->stk_addr = stk_addr;
   thr->stk_size = stk_size;
   thr->tls_addr = tls_addr;

lib/tsan/tsan_spinlock_defs_mac.h deleted-45

...	@@ -1,45 +0,0 @@
1	//===-- tsan_spinlock_defs_mac.h -------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	// Mac-specific forward-declared function defintions that may be
12	// deprecated in later versions of the OS.
13	// These are needed for interceptors.
14	//
15	//===----------------------------------------------------------------------===//
16
17	#if SANITIZER_APPLE
18
19	#ifndef TSAN_SPINLOCK_DEFS_MAC_H
20	#define TSAN_SPINLOCK_DEFS_MAC_H
21
22	#include <stdint.h>
23
24	extern "C" {
25
26	/*
27	Provides forward declarations related to OSSpinLocks on Darwin. These functions are
28	deprecated on macOS version 10.12 and later,
29	and are no longer included in the system headers.
30
31	However, the symbols are still available on the system, so we provide these forward
32	declarations to prevent compilation errors in tsan_interceptors_mac.cpp, which
33	references these functions when defining TSAN interceptor functions.
34	*/
35
36	typedef int32_t OSSpinLock;
37
38	void OSSpinLockLock(volatile OSSpinLock *__lock);
39	void OSSpinLockUnlock(volatile OSSpinLock *__lock);
40	bool OSSpinLockTry(volatile OSSpinLock *__lock);
41
42	}
43
44	#endif //TSAN_SPINLOCK_DEFS_MAC_H
45	#endif // SANITIZER_APPLE

src/libtsan.zig+2-6

...	@@ -410,9 +410,6 @@ const sanitizer_common_sources = [_][]const u8{	...	@@ -410,9 +410,6 @@ const sanitizer_common_sources = [_][]const u8{
410	"sanitizer_allocator.cpp",	410	"sanitizer_allocator.cpp",
411	"sanitizer_chained_origin_depot.cpp",	411	"sanitizer_chained_origin_depot.cpp",
412	"sanitizer_common.cpp",	412	"sanitizer_common.cpp",
413	"sanitizer_coverage_win_dll_thunk.cpp",
414	"sanitizer_coverage_win_dynamic_runtime_thunk.cpp",
415	"sanitizer_coverage_win_weak_interception.cpp",
416	"sanitizer_deadlock_detector1.cpp",	413	"sanitizer_deadlock_detector1.cpp",
417	"sanitizer_deadlock_detector2.cpp",	414	"sanitizer_deadlock_detector2.cpp",
418	"sanitizer_errno.cpp",	415	"sanitizer_errno.cpp",
...	@@ -452,9 +449,7 @@ const sanitizer_common_sources = [_][]const u8{	...	@@ -452,9 +449,7 @@ const sanitizer_common_sources = [_][]const u8{
452	"sanitizer_tls_get_addr.cpp",	449	"sanitizer_tls_get_addr.cpp",
453	"sanitizer_type_traits.cpp",	450	"sanitizer_type_traits.cpp",
454	"sanitizer_win.cpp",	451	"sanitizer_win.cpp",
455	"sanitizer_win_dll_thunk.cpp",	452	"sanitizer_win_interception.cpp",
456	"sanitizer_win_dynamic_runtime_thunk.cpp",
457	"sanitizer_win_weak_interception.cpp",
458	};	453	};
459		454
460	const sanitizer_nolibc_sources = [_][]const u8{	455	const sanitizer_nolibc_sources = [_][]const u8{
...	@@ -490,6 +485,7 @@ const sanitizer_symbolizer_sources = [_][]const u8{	...	@@ -490,6 +485,7 @@ const sanitizer_symbolizer_sources = [_][]const u8{
490	"sanitizer_symbolizer_report.cpp",	485	"sanitizer_symbolizer_report.cpp",
491	"sanitizer_symbolizer_report_fuchsia.cpp",	486	"sanitizer_symbolizer_report_fuchsia.cpp",
492	"sanitizer_symbolizer_win.cpp",	487	"sanitizer_symbolizer_win.cpp",
		488	"sanitizer_thread_history.cpp",
493	"sanitizer_unwind_linux_libcdep.cpp",	489	"sanitizer_unwind_linux_libcdep.cpp",
494	"sanitizer_unwind_fuchsia.cpp",	490	"sanitizer_unwind_fuchsia.cpp",
495	"sanitizer_unwind_win.cpp",	491	"sanitizer_unwind_win.cpp",

author	Alex Rønne Petersen <alex@alexrp.com> 2025-02-05 13:36:56+01:00
committer	Alex Rønne Petersen <alex@alexrp.com> 2025-04-04 06:08:09+02:00
log	e99c11856df5dafb54e92af12c0858d187dc2ad2
tree	47602df74eafa2c0d5e58d23ce7b4aef9756832a
parent	d67d52abe57b13517013e81562d64bb7c2f77775
signature	Signed by SSH key SHA256:7B/LJ7bpR1eX8aCXSr4mtd5M45VMPKcx9zY8e95b5QM

...	@@ -82,7 +82,7 @@	...	@@ -82,7 +82,7 @@
82	#endif	82	#endif
83		83
84	#if SANITIZER_INTERCEPT_MEMSET	84	#if SANITIZER_INTERCEPT_MEMSET
85	INTERCEPTOR(void , memset, void dst, int v, uptr size) {	85	INTERCEPTOR(void , memset, void dst, int v, usize size) {
86	void *ctx;	86	void *ctx;
87	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size);	87	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size);
88	}	88	}
...	@@ -93,7 +93,7 @@ INTERCEPTOR(void , memset, void dst, int v, uptr size) {	...	@@ -93,7 +93,7 @@ INTERCEPTOR(void , memset, void dst, int v, uptr size) {
93	#endif	93	#endif
94		94
95	#if SANITIZER_INTERCEPT_MEMMOVE	95	#if SANITIZER_INTERCEPT_MEMMOVE
96	INTERCEPTOR(void , memmove, void dst, const void *src, uptr size) {	96	INTERCEPTOR(void , memmove, void dst, const void *src, usize size) {
97	void *ctx;	97	void *ctx;
98	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);	98	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
99	}	99	}
...	@@ -104,7 +104,7 @@ INTERCEPTOR(void , memmove, void dst, const void *src, uptr size) {	...	@@ -104,7 +104,7 @@ INTERCEPTOR(void , memmove, void dst, const void *src, uptr size) {
104	#endif	104	#endif
105		105
106	#if SANITIZER_INTERCEPT_MEMCPY	106	#if SANITIZER_INTERCEPT_MEMCPY
107	INTERCEPTOR(void , memcpy, void dst, const void *src, uptr size) {	107	INTERCEPTOR(void , memcpy, void dst, const void *src, usize size) {
108	// On OS X, calling internal_memcpy here will cause memory corruptions,	108	// On OS X, calling internal_memcpy here will cause memory corruptions,
109	// because memcpy and memmove are actually aliases of the same	109	// because memcpy and memmove are actually aliases of the same
110	// implementation. We need to use internal_memmove here.	110	// implementation. We need to use internal_memmove here.
...	@@ -133,63 +133,63 @@ INTERCEPTOR(void , memcpy, void dst, const void *src, uptr size) {	...	@@ -133,63 +133,63 @@ INTERCEPTOR(void , memcpy, void dst, const void *src, uptr size) {
133	#endif	133	#endif
134		134
135	#if SANITIZER_INTERCEPT_AEABI_MEM	135	#if SANITIZER_INTERCEPT_AEABI_MEM
136	INTERCEPTOR(void , __aeabi_memmove, void to, const void *from, uptr size) {	136	INTERCEPTOR(void , __aeabi_memmove, void to, const void *from, usize size) {
137	void *ctx;	137	void *ctx;
138	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);	138	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
139	}	139	}
140		140
141	INTERCEPTOR(void , __aeabi_memmove4, void to, const void *from, uptr size) {	141	INTERCEPTOR(void , __aeabi_memmove4, void to, const void *from, usize size) {
142	void *ctx;	142	void *ctx;
143	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);	143	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
144	}	144	}
145		145
146	INTERCEPTOR(void , __aeabi_memmove8, void to, const void *from, uptr size) {	146	INTERCEPTOR(void , __aeabi_memmove8, void to, const void *from, usize size) {
147	void *ctx;	147	void *ctx;
148	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);	148	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
149	}	149	}
150		150
151	INTERCEPTOR(void , __aeabi_memcpy, void to, const void *from, uptr size) {	151	INTERCEPTOR(void , __aeabi_memcpy, void to, const void *from, usize size) {
152	void *ctx;	152	void *ctx;
153	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);	153	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
154	}	154	}
155		155
156	INTERCEPTOR(void , __aeabi_memcpy4, void to, const void *from, uptr size) {	156	INTERCEPTOR(void , __aeabi_memcpy4, void to, const void *from, usize size) {
157	void *ctx;	157	void *ctx;
158	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);	158	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
159	}	159	}
160		160
161	INTERCEPTOR(void , __aeabi_memcpy8, void to, const void *from, uptr size) {	161	INTERCEPTOR(void , __aeabi_memcpy8, void to, const void *from, usize size) {
162	void *ctx;	162	void *ctx;
163	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);	163	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
164	}	164	}
165		165
166	// Note the argument order.	166	// Note the argument order.
167	INTERCEPTOR(void , __aeabi_memset, void block, uptr size, int c) {	167	INTERCEPTOR(void , __aeabi_memset, void block, usize size, int c) {
168	void *ctx;	168	void *ctx;
169	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);	169	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
170	}	170	}
171		171
172	INTERCEPTOR(void , __aeabi_memset4, void block, uptr size, int c) {	172	INTERCEPTOR(void , __aeabi_memset4, void block, usize size, int c) {
173	void *ctx;	173	void *ctx;
174	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);	174	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
175	}	175	}
176		176
177	INTERCEPTOR(void , __aeabi_memset8, void block, uptr size, int c) {	177	INTERCEPTOR(void , __aeabi_memset8, void block, usize size, int c) {
178	void *ctx;	178	void *ctx;
179	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);	179	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
180	}	180	}
181		181
182	INTERCEPTOR(void , __aeabi_memclr, void block, uptr size) {	182	INTERCEPTOR(void , __aeabi_memclr, void block, usize size) {
183	void *ctx;	183	void *ctx;
184	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);	184	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
185	}	185	}
186		186
187	INTERCEPTOR(void , __aeabi_memclr4, void block, uptr size) {	187	INTERCEPTOR(void , __aeabi_memclr4, void block, usize size) {
188	void *ctx;	188	void *ctx;
189	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);	189	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
190	}	190	}
191		191
192	INTERCEPTOR(void , __aeabi_memclr8, void block, uptr size) {	192	INTERCEPTOR(void , __aeabi_memclr8, void block, usize size) {
193	void *ctx;	193	void *ctx;
194	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);	194	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
195	}	195	}
...	@@ -212,7 +212,7 @@ INTERCEPTOR(void , __aeabi_memclr8, void block, uptr size) {	...	@@ -212,7 +212,7 @@ INTERCEPTOR(void , __aeabi_memclr8, void block, uptr size) {
212	#endif // SANITIZER_INTERCEPT_AEABI_MEM	212	#endif // SANITIZER_INTERCEPT_AEABI_MEM
213		213
214	#if SANITIZER_INTERCEPT___BZERO	214	#if SANITIZER_INTERCEPT___BZERO
215	INTERCEPTOR(void , __bzero, void block, uptr size) {	215	INTERCEPTOR(void , __bzero, void block, usize size) {
216	void *ctx;	216	void *ctx;
217	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);	217	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
218	}	218	}
...	@@ -222,7 +222,7 @@ INTERCEPTOR(void , __bzero, void block, uptr size) {	...	@@ -222,7 +222,7 @@ INTERCEPTOR(void , __bzero, void block, uptr size) {
222	#endif // SANITIZER_INTERCEPT___BZERO	222	#endif // SANITIZER_INTERCEPT___BZERO
223		223
224	#if SANITIZER_INTERCEPT_BZERO	224	#if SANITIZER_INTERCEPT_BZERO
225	INTERCEPTOR(void , bzero, void block, uptr size) {	225	INTERCEPTOR(void , bzero, void block, usize size) {
226	void *ctx;	226	void *ctx;
227	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);	227	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
228	}	228	}

...	@@ -10,6 +10,7 @@	...	@@ -10,6 +10,7 @@
10	INTERFACE_FUNCTION(__sanitizer_acquire_crash_state)	10	INTERFACE_FUNCTION(__sanitizer_acquire_crash_state)
11	INTERFACE_FUNCTION(__sanitizer_annotate_contiguous_container)	11	INTERFACE_FUNCTION(__sanitizer_annotate_contiguous_container)
12	INTERFACE_FUNCTION(__sanitizer_annotate_double_ended_contiguous_container)	12	INTERFACE_FUNCTION(__sanitizer_annotate_double_ended_contiguous_container)
		13	INTERFACE_FUNCTION(__sanitizer_copy_contiguous_container_annotations)
13	INTERFACE_FUNCTION(__sanitizer_contiguous_container_find_bad_address)	14	INTERFACE_FUNCTION(__sanitizer_contiguous_container_find_bad_address)
14	INTERFACE_FUNCTION(	15	INTERFACE_FUNCTION(
15	__sanitizer_double_ended_contiguous_container_find_bad_address)	16	__sanitizer_double_ended_contiguous_container_find_bad_address)
...	@@ -22,6 +23,7 @@ INTERFACE_FUNCTION(__sanitizer_verify_double_ended_contiguous_container)	...	@@ -22,6 +23,7 @@ INTERFACE_FUNCTION(__sanitizer_verify_double_ended_contiguous_container)
22	INTERFACE_WEAK_FUNCTION(__sanitizer_on_print)	23	INTERFACE_WEAK_FUNCTION(__sanitizer_on_print)
23	INTERFACE_WEAK_FUNCTION(__sanitizer_report_error_summary)	24	INTERFACE_WEAK_FUNCTION(__sanitizer_report_error_summary)
24	INTERFACE_WEAK_FUNCTION(__sanitizer_sandbox_on_notify)	25	INTERFACE_WEAK_FUNCTION(__sanitizer_sandbox_on_notify)
		26	INTERFACE_WEAK_FUNCTION(__sanitizer_get_dtls_size)
25	// Sanitizer weak hooks	27	// Sanitizer weak hooks
26	INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_memcmp)	28	INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_memcmp)
27	INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_strcmp)	29	INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_strcmp)
...	@@ -51,3 +53,9 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_ignore_free_hook)	...	@@ -51,3 +53,9 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_ignore_free_hook)
51	INTERFACE_FUNCTION(__sanitizer_internal_memcpy)	53	INTERFACE_FUNCTION(__sanitizer_internal_memcpy)
52	INTERFACE_FUNCTION(__sanitizer_internal_memmove)	54	INTERFACE_FUNCTION(__sanitizer_internal_memmove)
53	INTERFACE_FUNCTION(__sanitizer_internal_memset)	55	INTERFACE_FUNCTION(__sanitizer_internal_memset)
		56
		57	#if SANITIZER_WINDOWS
		58	INTERFACE_FUNCTION(__sanitizer_override_function)
		59	INTERFACE_FUNCTION(__sanitizer_override_function_by_addr)
		60	INTERFACE_FUNCTION(__sanitizer_register_weak_function)
		61	#endif

...	@@ -94,7 +94,6 @@ void DisableCoreDumperIfNecessary() {}	...	@@ -94,7 +94,6 @@ void DisableCoreDumperIfNecessary() {}
94	void InstallDeadlySignalHandlers(SignalHandlerType handler) {}	94	void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
95	void SetAlternateSignalStack() {}	95	void SetAlternateSignalStack() {}
96	void UnsetAlternateSignalStack() {}	96	void UnsetAlternateSignalStack() {}
97	void InitTlsSize() {}
98		97
99	bool SignalContext::IsStackOverflow() const { return false; }	98	bool SignalContext::IsStackOverflow() const { return false; }
100	void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); }	99	void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); }
...	@@ -445,6 +444,11 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) {	...	@@ -445,6 +444,11 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) {
445	return status == ZX_OK;	444	return status == ZX_OK;
446	}	445	}
447		446
		447	bool TryMemCpy(void dest, const void src, uptr n) {
		448	// TODO: implement.
		449	return false;
		450	}
		451
448	// FIXME implement on this platform.	452	// FIXME implement on this platform.
449	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}	453	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
450		454
...	@@ -519,7 +523,6 @@ uptr ReadLongProcessName(/out/ char *buf, uptr buf_len) {	...	@@ -519,7 +523,6 @@ uptr ReadLongProcessName(/out/ char *buf, uptr buf_len) {
519	uptr MainThreadStackBase, MainThreadStackSize;	523	uptr MainThreadStackBase, MainThreadStackSize;
520		524
521	bool GetRandom(void *buffer, uptr length, bool blocking) {	525	bool GetRandom(void *buffer, uptr length, bool blocking) {
522	CHECK_LE(length, ZX_CPRNG_DRAW_MAX_LEN);
523	_zx_cprng_draw(buffer, length);	526	_zx_cprng_draw(buffer, length);
524	return true;	527	return true;
525	}	528	}

...	@@ -28,6 +28,9 @@ namespace __sanitizer {	...	@@ -28,6 +28,9 @@ namespace __sanitizer {
28	// Don't use directly, use __sanitizer::OpenFile() instead.	28	// Don't use directly, use __sanitizer::OpenFile() instead.
29	uptr internal_open(const char *filename, int flags);	29	uptr internal_open(const char *filename, int flags);
30	uptr internal_open(const char *filename, int flags, u32 mode);	30	uptr internal_open(const char *filename, int flags, u32 mode);
		31	# if SANITIZER_FREEBSD
		32	uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags);
		33	# endif
31	uptr internal_close(fd_t fd);	34	uptr internal_close(fd_t fd);
32		35
33	uptr internal_read(fd_t fd, void *buf, uptr count);	36	uptr internal_read(fd_t fd, void *buf, uptr count);
...	@@ -86,7 +89,7 @@ int internal_pthread_join(void th, void *ret);	...	@@ -86,7 +89,7 @@ int internal_pthread_join(void th, void *ret);
86	return REAL(pthread_create)(th, attr, callback, param); \	89	return REAL(pthread_create)(th, attr, callback, param); \
87	} \	90	} \
88	int internal_pthread_join(void th, void *ret) { \	91	int internal_pthread_join(void th, void *ret) { \
89	return REAL(pthread_join(th, ret)); \	92	return REAL(pthread_join)(th, ret); \
90	} \	93	} \
91	} // namespace __sanitizer	94	} // namespace __sanitizer
92		95
...	@@ -108,6 +111,7 @@ bool IsStateDetached(int state);	...	@@ -108,6 +111,7 @@ bool IsStateDetached(int state);
108	fd_t ReserveStandardFds(fd_t fd);	111	fd_t ReserveStandardFds(fd_t fd);
109		112
110	bool ShouldMockFailureToOpen(const char *path);	113	bool ShouldMockFailureToOpen(const char *path);
		114	bool OpenReadsVaArgs(int oflag);
111		115
112	// Create a non-file mapping with a given /proc/self/maps name.	116	// Create a non-file mapping with a given /proc/self/maps name.
113	uptr MmapNamed(void addr, uptr length, int prot, int flags, const char name);	117	uptr MmapNamed(void addr, uptr length, int prot, int flags, const char name);

...	@@ -30,7 +30,7 @@ namespace __sanitizer {	...	@@ -30,7 +30,7 @@ namespace __sanitizer {
30	bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {	30	bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
31	Dl_info info;	31	Dl_info info;
32	int result = dladdr((const void *)addr, &info);	32	int result = dladdr((const void *)addr, &info);
33	if (!result) return false;	33	if (!result \|\| !info.dli_sname) return false;
34		34
35	// Compute offset if possible. `dladdr()` doesn't always ensure that `addr >=	35	// Compute offset if possible. `dladdr()` doesn't always ensure that `addr >=
36	// sym_addr` so only compute the offset when this holds. Failure to find the	36	// sym_addr` so only compute the offset when this holds. Failure to find the
...	@@ -51,7 +51,7 @@ bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {	...	@@ -51,7 +51,7 @@ bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
51	bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) {	51	bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) {
52	Dl_info info;	52	Dl_info info;
53	int result = dladdr((const void *)addr, &info);	53	int result = dladdr((const void *)addr, &info);
54	if (!result) return false;	54	if (!result \|\| !info.dli_sname) return false;
55	const char *demangled = DemangleSwiftAndCXX(info.dli_sname);	55	const char *demangled = DemangleSwiftAndCXX(info.dli_sname);
56	if (!demangled)	56	if (!demangled)
57	demangled = info.dli_sname;	57	demangled = info.dli_sname;

...	@@ -828,18 +828,18 @@ void PrintCurrentStack(ThreadState *thr, uptr pc) {	...	@@ -828,18 +828,18 @@ void PrintCurrentStack(ThreadState *thr, uptr pc) {
828	PrintStack(SymbolizeStack(trace));	828	PrintStack(SymbolizeStack(trace));
829	}	829	}
830		830
831	// Always inlining PrintCurrentStackSlow, because LocatePcInTrace assumes	831	// Always inlining PrintCurrentStack, because LocatePcInTrace assumes
832	// __sanitizer_print_stack_trace exists in the actual unwinded stack, but	832	// __sanitizer_print_stack_trace exists in the actual unwinded stack, but
833	// tail-call to PrintCurrentStackSlow breaks this assumption because	833	// tail-call to PrintCurrentStack breaks this assumption because
834	// __sanitizer_print_stack_trace disappears after tail-call.	834	// __sanitizer_print_stack_trace disappears after tail-call.
835	// However, this solution is not reliable enough, please see dvyukov's comment	835	// However, this solution is not reliable enough, please see dvyukov's comment
836	// http://reviews.llvm.org/D19148#406208	836	// http://reviews.llvm.org/D19148#406208
837	// Also see PR27280 comment 2 and 3 for breaking examples and analysis.	837	// Also see PR27280 comment 2 and 3 for breaking examples and analysis.
838	ALWAYS_INLINE USED void PrintCurrentStackSlow(uptr pc) {	838	ALWAYS_INLINE USED void PrintCurrentStack(uptr pc, bool fast) {
839	#if !SANITIZER_GO	839	#if !SANITIZER_GO
840	uptr bp = GET_CURRENT_FRAME();	840	uptr bp = GET_CURRENT_FRAME();
841	auto *ptrace = New<BufferedStackTrace>();	841	auto *ptrace = New<BufferedStackTrace>();
842	ptrace->Unwind(pc, bp, nullptr, false);	842	ptrace->Unwind(pc, bp, nullptr, fast);
843		843
844	for (uptr i = 0; i < ptrace->size / 2; i++) {	844	for (uptr i = 0; i < ptrace->size / 2; i++) {
845	uptr tmp = ptrace->trace_buffer[i];	845	uptr tmp = ptrace->trace_buffer[i];
...	@@ -857,6 +857,6 @@ using namespace __tsan;	...	@@ -857,6 +857,6 @@ using namespace __tsan;
857	extern "C" {	857	extern "C" {
858	SANITIZER_INTERFACE_ATTRIBUTE	858	SANITIZER_INTERFACE_ATTRIBUTE
859	void __sanitizer_print_stack_trace() {	859	void __sanitizer_print_stack_trace() {
860	PrintCurrentStackSlow(StackTrace::GetCurrentPc());	860	PrintCurrentStack(StackTrace::GetCurrentPc(), false);
861	}	861	}
862	} // extern "C"	862	} // extern "C"

libtsan: Update to LLVM 20.