Commit e99c11856df5dafb54e92af12c0858d187dc2ad2 | zig

lib/tsan/builtins/assembly.h+12

...	...	@@ -290,4 +290,16 @@
290	290	CFI_END
291	291	#endif
292	292
	293	#ifdef __arm__
	294	#include "int_endianness.h"
	295
	296	#if _YUGA_BIG_ENDIAN
	297	#define VMOV_TO_DOUBLE(dst, src0, src1) vmov dst, src1, src0 SEPARATOR
	298	#define VMOV_FROM_DOUBLE(dst0, dst1, src) vmov dst1, dst0, src SEPARATOR
	299	#else
	300	#define VMOV_TO_DOUBLE(dst, src0, src1) vmov dst, src0, src1 SEPARATOR
	301	#define VMOV_FROM_DOUBLE(dst0, dst1, src) vmov dst0, dst1, src SEPARATOR
	302	#endif
	303	#endif
	304
293	305	#endif // COMPILERRT_ASSEMBLY_H

lib/tsan/interception/interception.h+14-11

...	...	@@ -25,8 +25,19 @@
25	25
26	26	// These typedefs should be used only in the interceptor definitions to replace
27	27	// the standard system types (e.g. SSIZE_T instead of ssize_t)
28		typedef __sanitizer::uptr SIZE_T;
29		typedef __sanitizer::sptr SSIZE_T;
	28	// On Windows the system headers (basetsd.h) provide a conflicting definition
	29	// of SIZE_T/SSIZE_T that do not match the real size_t/ssize_t for 32-bit
	30	// systems (using long instead of the expected int). Work around the typedef
	31	// redefinition by #defining SIZE_T instead of using a typedef.
	32	// TODO: We should be using __sanitizer::usize (and a new ssize) instead of
	33	// these new macros as long as we ensure they match the real system definitions.
	34	#if SANITIZER_WINDOWS
	35	// Ensure that (S)SIZE_T were already defined as we are about to override them.
	36	# include <basetsd.h>
	37	#endif
	38
	39	#define SIZE_T __sanitizer::usize
	40	#define SSIZE_T __sanitizer::ssize
30	41	typedef __sanitizer::sptr PTRDIFF_T;
31	42	typedef __sanitizer::s64 INTMAX_T;
32	43	typedef __sanitizer::u64 UINTMAX_T;
...	...	@@ -338,16 +349,8 @@ const interpose_substitution substitution_##func_name[] \
338	349	#endif
339	350
340	351	// ISO C++ forbids casting between pointer-to-function and pointer-to-object,
341		// so we use casting via an integral type __interception::uptr,
342		// assuming that system is POSIX-compliant. Using other hacks seem
343		// challenging, as we don't even pass function type to
344		// INTERCEPT_FUNCTION macro, only its name.
	352	// so we use casts via uintptr_t (the local __sanitizer::uptr equivalent).
345	353	namespace __interception {
346		#if defined(_WIN64)
347		typedef unsigned long long uptr;
348		#else
349		typedef unsigned long uptr;
350		#endif // _WIN64
351	354
352	355	#if defined(__ELF__) && !SANITIZER_FUCHSIA
353	356	// The use of interceptors makes many sanitizers unusable for static linking.

lib/tsan/interception/interception_type_test.cpp+19-12

...	...	@@ -12,28 +12,35 @@
12	12	//===----------------------------------------------------------------------===//
13	13
14	14	#include "interception.h"
	15	#include "sanitizer_common/sanitizer_type_traits.h"
15	16
16		#if SANITIZER_LINUX \|\| SANITIZER_APPLE
17
18		#include <sys/types.h>
	17	#if __has_include(<sys/types.h>)
	18	# include <sys/types.h>
	19	#endif
19	20	#include <stddef.h>
20	21	#include <stdint.h>
21	22
22		COMPILER_CHECK(sizeof(::SIZE_T) == sizeof(size_t));
23		COMPILER_CHECK(sizeof(::SSIZE_T) == sizeof(ssize_t));
24		COMPILER_CHECK(sizeof(::PTRDIFF_T) == sizeof(ptrdiff_t));
	23	COMPILER_CHECK((__sanitizer::is_same<__sanitizer::uptr, ::uintptr_t>::value));
	24	COMPILER_CHECK((__sanitizer::is_same<__sanitizer::sptr, ::intptr_t>::value));
	25	COMPILER_CHECK((__sanitizer::is_same<__sanitizer::usize, ::size_t>::value));
	26	COMPILER_CHECK((__sanitizer::is_same<::PTRDIFF_T, ::ptrdiff_t>::value));
	27	COMPILER_CHECK((__sanitizer::is_same<::SIZE_T, ::size_t>::value));
	28	#if !SANITIZER_WINDOWS
	29	// No ssize_t on Windows.
	30	COMPILER_CHECK((__sanitizer::is_same<::SSIZE_T, ::ssize_t>::value));
	31	#endif
	32	// TODO: These are not actually the same type on Linux (long vs long long)
25	33	COMPILER_CHECK(sizeof(::INTMAX_T) == sizeof(intmax_t));
	34	COMPILER_CHECK(sizeof(::UINTMAX_T) == sizeof(uintmax_t));
26	35
27		# if SANITIZER_GLIBC \|\| SANITIZER_ANDROID
	36	#if SANITIZER_GLIBC \|\| SANITIZER_ANDROID
28	37	COMPILER_CHECK(sizeof(::OFF64_T) == sizeof(off64_t));
29		# endif
	38	#endif
30	39
31	40	// The following are the cases when pread (and friends) is used instead of
32	41	// pread64. In those cases we need OFF_T to match off_t. We don't care about the
33	42	// rest (they depend on _FILE_OFFSET_BITS setting when building an application).
34		# if SANITIZER_ANDROID \|\| !defined _FILE_OFFSET_BITS \|\| \
35		_FILE_OFFSET_BITS != 64
	43	#if !SANITIZER_WINDOWS && (SANITIZER_ANDROID \|\| !defined _FILE_OFFSET_BITS \|\| \
	44	_FILE_OFFSET_BITS != 64)
36	45	COMPILER_CHECK(sizeof(::OFF_T) == sizeof(off_t));
37		# endif
38
39	46	#endif

lib/tsan/interception/interception_win.cpp+364-68

...	...	@@ -27,7 +27,7 @@
27	27	//
28	28	// 1) Detour
29	29	//
30		// The Detour hooking technique is assuming the presence of an header with
	30	// The Detour hooking technique is assuming the presence of a header with
31	31	// padding and an overridable 2-bytes nop instruction (mov edi, edi). The
32	32	// nop instruction can safely be replaced by a 2-bytes jump without any need
33	33	// to save the instruction. A jump to the target is encoded in the function
...	...	@@ -47,7 +47,7 @@
47	47	//
48	48	// func: jmp <label> --> func: jmp <hook>
49	49	//
50		// On an 64-bit architecture, a trampoline is inserted.
	50	// On a 64-bit architecture, a trampoline is inserted.
51	51	//
52	52	// func: jmp <label> --> func: jmp <tramp>
53	53	// [...]
...	...	@@ -60,7 +60,7 @@
60	60	//
61	61	// 3) HotPatch
62	62	//
63		// The HotPatch hooking is assuming the presence of an header with padding
	63	// The HotPatch hooking is assuming the presence of a header with padding
64	64	// and a first instruction with at least 2-bytes.
65	65	//
66	66	// The reason to enforce the 2-bytes limitation is to provide the minimal
...	...	@@ -80,7 +80,7 @@
80	80	// real: <instr>
81	81	// jmp <body>
82	82	//
83		// On an 64-bit architecture:
	83	// On a 64-bit architecture:
84	84	//
85	85	// head: 6 x nop head: jmp QWORD [addr1]
86	86	// func: <instr> --> func: jmp short <head>
...	...	@@ -110,7 +110,7 @@
110	110	// <instr>
111	111	// jmp <body>
112	112	//
113		// On an 64-bit architecture:
	113	// On a 64-bit architecture:
114	114	//
115	115	// func: <instr> --> func: jmp QWORD [addr1]
116	116	// <instr>
...	...	@@ -130,6 +130,7 @@
130	130	#include "sanitizer_common/sanitizer_platform.h"
131	131	#define WIN32_LEAN_AND_MEAN
132	132	#include <windows.h>
	133	#include <psapi.h>
133	134
134	135	namespace __interception {
135	136
...	...	@@ -186,8 +187,12 @@ static uptr GetMmapGranularity() {
186	187	return si.dwAllocationGranularity;
187	188	}
188	189
	190	UNUSED static uptr RoundDownTo(uptr size, uptr boundary) {
	191	return size & ~(boundary - 1);
	192	}
	193
189	194	UNUSED static uptr RoundUpTo(uptr size, uptr boundary) {
190		return (size + boundary - 1) & ~(boundary - 1);
	195	return RoundDownTo(size + boundary - 1, boundary);
191	196	}
192	197
193	198	// FIXME: internal_str* and internal_mem* functions should be moved from the
...	...	@@ -208,6 +213,18 @@ static char* _strchr(char* str, char c) {
208	213	return nullptr;
209	214	}
210	215
	216	static int _strcmp(const char s1, const char s2) {
	217	while (true) {
	218	unsigned c1 = *s1;
	219	unsigned c2 = *s2;
	220	if (c1 != c2) return (c1 < c2) ? -1 : 1;
	221	if (c1 == 0) break;
	222	s1++;
	223	s2++;
	224	}
	225	return 0;
	226	}
	227
211	228	static void _memset(void *p, int value, size_t sz) {
212	229	for (size_t i = 0; i < sz; ++i)
213	230	((char*)p)[i] = (char)value;
...	...	@@ -284,8 +301,11 @@ static void WriteJumpInstruction(uptr from, uptr target) {
284	301
285	302	static void WriteShortJumpInstruction(uptr from, uptr target) {
286	303	sptr offset = target - from - kShortJumpInstructionLength;
287		if (offset < -128 \|\| offset > 127)
	304	if (offset < -128 \|\| offset > 127) {
	305	ReportError("interception_win: cannot write short jmp from %p to %p\n",
	306	(void )from, (void )target);
288	307	InterceptionFailed();
	308	}
289	309	(u8)from = 0xEB;
290	310	(u8)(from + 1) = (u8)offset;
291	311	}
...	...	@@ -339,32 +359,78 @@ struct TrampolineMemoryRegion {
339	359	uptr max_size;
340	360	};
341	361
342		UNUSED static const uptr kTrampolineScanLimitRange = 1ull << 31; // 2 gig
	362	UNUSED static const uptr kTrampolineRangeLimit = 1ull << 31; // 2 gig
343	363	static const int kMaxTrampolineRegion = 1024;
344	364	static TrampolineMemoryRegion TrampolineRegions[kMaxTrampolineRegion];
345	365
346		static void *AllocateTrampolineRegion(uptr image_address, size_t granularity) {
347		#if SANITIZER_WINDOWS64
348		uptr address = image_address;
349		uptr scanned = 0;
350		while (scanned < kTrampolineScanLimitRange) {
	366	static void *AllocateTrampolineRegion(uptr min_addr, uptr max_addr,
	367	uptr func_addr, size_t granularity) {
	368	# if SANITIZER_WINDOWS64
	369	// Clamp {min,max}_addr to the accessible address space.
	370	SYSTEM_INFO system_info;
	371	::GetSystemInfo(&system_info);
	372	uptr min_virtual_addr =
	373	RoundUpTo((uptr)system_info.lpMinimumApplicationAddress, granularity);
	374	uptr max_virtual_addr =
	375	RoundDownTo((uptr)system_info.lpMaximumApplicationAddress, granularity);
	376	if (min_addr < min_virtual_addr)
	377	min_addr = min_virtual_addr;
	378	if (max_addr > max_virtual_addr)
	379	max_addr = max_virtual_addr;
	380
	381	// This loop probes the virtual address space to find free memory in the
	382	// [min_addr, max_addr] interval. The search starts from func_addr and
	383	// proceeds "outwards" towards the interval bounds using two probes, lo_addr
	384	// and hi_addr, for addresses lower/higher than func_addr. At each step, it
	385	// considers the probe closest to func_addr. If that address is not free, the
	386	// probe is advanced (lower or higher depending on the probe) to the next
	387	// memory block and the search continues.
	388	uptr lo_addr = RoundDownTo(func_addr, granularity);
	389	uptr hi_addr = RoundUpTo(func_addr, granularity);
	390	while (lo_addr >= min_addr \|\| hi_addr <= max_addr) {
	391	// Consider the in-range address closest to func_addr.
	392	uptr addr;
	393	if (lo_addr < min_addr)
	394	addr = hi_addr;
	395	else if (hi_addr > max_addr)
	396	addr = lo_addr;
	397	else
	398	addr = (hi_addr - func_addr < func_addr - lo_addr) ? hi_addr : lo_addr;
	399
351	400	MEMORY_BASIC_INFORMATION info;
352		if (!::VirtualQuery((void*)address, &info, sizeof(info)))
	401	if (!::VirtualQuery((void *)addr, &info, sizeof(info))) {
	402	ReportError(
	403	"interception_win: VirtualQuery in AllocateTrampolineRegion failed "
	404	"for %p\n",
	405	(void *)addr);
353	406	return nullptr;
	407	}
354	408
355		// Check whether a region can be allocated at \|address\|.
	409	// Check whether a region can be allocated at \|addr\|.
356	410	if (info.State == MEM_FREE && info.RegionSize >= granularity) {
357		void page = ::VirtualAlloc((void)RoundUpTo(address, granularity),
358		granularity,
359		MEM_RESERVE \| MEM_COMMIT,
360		PAGE_EXECUTE_READWRITE);
	411	void *page =
	412	::VirtualAlloc((void *)addr, granularity, MEM_RESERVE \| MEM_COMMIT,
	413	PAGE_EXECUTE_READWRITE);
	414	if (page == nullptr)
	415	ReportError(
	416	"interception_win: VirtualAlloc in AllocateTrampolineRegion failed "
	417	"for %p\n",
	418	(void *)addr);
361	419	return page;
362	420	}
363	421
364		// Move to the next region.
365		address = (uptr)info.BaseAddress + info.RegionSize;
366		scanned += info.RegionSize;
	422	if (addr == lo_addr)
	423	lo_addr =
	424	RoundDownTo((uptr)info.AllocationBase - granularity, granularity);
	425	if (addr == hi_addr)
	426	hi_addr =
	427	RoundUpTo((uptr)info.BaseAddress + info.RegionSize, granularity);
367	428	}
	429
	430	ReportError(
	431	"interception_win: AllocateTrampolineRegion failed to find free memory; "
	432	"min_addr: %p, max_addr: %p, func_addr: %p, granularity: %zu\n",
	433	(void )min_addr, (void )max_addr, (void *)func_addr, granularity);
368	434	return nullptr;
369	435	#else
370	436	return ::VirtualAlloc(nullptr,
...	...	@@ -385,15 +451,51 @@ void TestOnlyReleaseTrampolineRegions() {
385	451	}
386	452	}
387	453
388		static uptr AllocateMemoryForTrampoline(uptr image_address, size_t size) {
389		// Find a region within 2G with enough space to allocate \|size\| bytes.
	454	static uptr AllocateMemoryForTrampoline(uptr func_address, size_t size) {
	455	# if SANITIZER_WINDOWS64
	456	uptr min_addr = func_address - kTrampolineRangeLimit;
	457	uptr max_addr = func_address + kTrampolineRangeLimit - size;
	458
	459	// Allocate memory within 2GB of the module (DLL or EXE file) so that any
	460	// address within the module can be referenced with PC-relative operands.
	461	// This allows us to not just jump to the trampoline with a PC-relative
	462	// offset, but to relocate any instructions that we copy to the trampoline
	463	// which have references to the original module. If we can't find the base
	464	// address of the module (e.g. if func_address is in mmap'ed memory), just
	465	// stay within 2GB of func_address.
	466	HMODULE module;
	467	if (::GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS \|
	468	GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
	469	(LPCWSTR)func_address, &module)) {
	470	MODULEINFO module_info;
	471	if (::GetModuleInformation(::GetCurrentProcess(), module,
	472	&module_info, sizeof(module_info))) {
	473	min_addr = (uptr)module_info.lpBaseOfDll + module_info.SizeOfImage -
	474	kTrampolineRangeLimit;
	475	max_addr = (uptr)module_info.lpBaseOfDll + kTrampolineRangeLimit - size;
	476	}
	477	}
	478
	479	// Check for overflow.
	480	if (min_addr > func_address)
	481	min_addr = 0;
	482	if (max_addr < func_address)
	483	max_addr = ~(uptr)0;
	484	# else
	485	uptr min_addr = 0;
	486	uptr max_addr = ~min_addr;
	487	# endif
	488
	489	// Find a region within [min_addr,max_addr] with enough space to allocate
	490	// \|size\| bytes.
390	491	TrampolineMemoryRegion *region = nullptr;
391	492	for (size_t bucket = 0; bucket < kMaxTrampolineRegion; ++bucket) {
392	493	TrampolineMemoryRegion* current = &TrampolineRegions[bucket];
393	494	if (current->content == 0) {
394	495	// No valid region found, allocate a new region.
395	496	size_t bucket_size = GetMmapGranularity();
396		void *content = AllocateTrampolineRegion(image_address, bucket_size);
	497	void *content = AllocateTrampolineRegion(min_addr, max_addr, func_address,
	498	bucket_size);
397	499	if (content == nullptr)
398	500	return 0U;
399	501
...	...	@@ -403,13 +505,9 @@ static uptr AllocateMemoryForTrampoline(uptr image_address, size_t size) {
403	505	region = current;
404	506	break;
405	507	} else if (current->max_size - current->allocated_size > size) {
406		#if SANITIZER_WINDOWS64
407		// In 64-bits, the memory space must be allocated within 2G boundary.
408		uptr next_address = current->content + current->allocated_size;
409		if (next_address < image_address \|\|
410		next_address - image_address >= 0x7FFF0000)
411		continue;
412		#endif
	508	uptr next_address = current->content + current->allocated_size;
	509	if (next_address < min_addr \|\| next_address > max_addr)
	510	continue;
413	511	// The space can be allocated in the current region.
414	512	region = current;
415	513	break;
...	...	@@ -458,6 +556,10 @@ static const u8 kPrologueWithShortJump2[] = {
458	556
459	557	// Returns 0 on error.
460	558	static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
	559	if (rel_offset) {
	560	*rel_offset = 0;
	561	}
	562
461	563	#if SANITIZER_ARM64
462	564	// An ARM64 instruction is 4 bytes long.
463	565	return 4;
...	...	@@ -497,8 +599,14 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
497	599	case 0x6A: // 6A XX = push XX
498	600	return 2;
499	601
	602	// This instruction can be encoded with a 16-bit immediate but that is
	603	// incredibly unlikely.
	604	case 0x68: // 68 XX XX XX XX : push imm32
	605	return 5;
	606
500	607	case 0xb8: // b8 XX XX XX XX : mov eax, XX XX XX XX
501	608	case 0xB9: // b9 XX XX XX XX : mov ecx, XX XX XX XX
	609	case 0xBA: // ba XX XX XX XX : mov edx, XX XX XX XX
502	610	return 5;
503	611
504	612	// Cannot overwrite control-instruction. Return 0 to indicate failure.
...	...	@@ -529,19 +637,43 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
529	637	case 0xFF8B: // 8B FF : mov edi, edi
530	638	case 0xEC8B: // 8B EC : mov ebp, esp
531	639	case 0xc889: // 89 C8 : mov eax, ecx
	640	case 0xD189: // 89 D1 : mov ecx, edx
532	641	case 0xE589: // 89 E5 : mov ebp, esp
533	642	case 0xC18B: // 8B C1 : mov eax, ecx
	643	case 0xC031: // 31 C0 : xor eax, eax
	644	case 0xC931: // 31 C9 : xor ecx, ecx
	645	case 0xD231: // 31 D2 : xor edx, edx
534	646	case 0xC033: // 33 C0 : xor eax, eax
535	647	case 0xC933: // 33 C9 : xor ecx, ecx
536	648	case 0xD233: // 33 D2 : xor edx, edx
	649	case 0xDB84: // 84 DB : test bl,bl
	650	case 0xC084: // 84 C0 : test al,al
	651	case 0xC984: // 84 C9 : test cl,cl
	652	case 0xD284: // 84 D2 : test dl,dl
537	653	return 2;
538	654
	655	case 0x3980: // 80 39 XX : cmp BYTE PTR [rcx], XX
	656	case 0x4D8B: // 8B 4D XX : mov XX(%ebp), ecx
	657	case 0x558B: // 8B 55 XX : mov XX(%ebp), edx
	658	case 0x758B: // 8B 75 XX : mov XX(%ebp), esp
	659	case 0xE483: // 83 E4 XX : and esp, XX
	660	case 0xEC83: // 83 EC XX : sub esp, XX
	661	case 0xC1F6: // F6 C1 XX : test cl, XX
	662	return 3;
	663
	664	case 0x89FF: // FF 89 XX XX XX XX : dec dword ptr [ecx + XX XX XX XX]
	665	case 0xEC81: // 81 EC XX XX XX XX : sub esp, XX XX XX XX
	666	return 6;
	667
539	668	// Cannot overwrite control-instruction. Return 0 to indicate failure.
540		case 0x25FF: // FF 25 XX XX XX XX : jmp [XXXXXXXX]
	669	case 0x25FF: // FF 25 XX YY ZZ WW : jmp dword ptr ds:[WWZZYYXX]
541	670	return 0;
542	671	}
543	672
544		switch (0x00FFFFFF & (u32)address) {
	673	switch (0x00FFFFFF & (u32 )address) {
	674	case 0x244C8D: // 8D 4C 24 XX : lea ecx, [esp + XX]
	675	case 0x2474FF: // FF 74 24 XX : push qword ptr [rsp + XX]
	676	return 4;
545	677	case 0x24A48D: // 8D A4 24 XX XX XX XX : lea esp, [esp + XX XX XX XX]
546	678	return 7;
547	679	}
...	...	@@ -556,6 +688,21 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
556	688	case 0xA1: // A1 XX XX XX XX XX XX XX XX :
557	689	// movabs eax, dword ptr ds:[XXXXXXXX]
558	690	return 9;
	691	case 0xF2:
	692	switch ((u32 )(address + 1)) {
	693	case 0x2444110f: // f2 0f 11 44 24 XX movsd QWORD PTR
	694	// [rsp + XX], xmm0
	695	case 0x244c110f: // f2 0f 11 4c 24 XX movsd QWORD PTR
	696	// [rsp + XX], xmm1
	697	case 0x2454110f: // f2 0f 11 54 24 XX movsd QWORD PTR
	698	// [rsp + XX], xmm2
	699	case 0x245c110f: // f2 0f 11 5c 24 XX movsd QWORD PTR
	700	// [rsp + XX], xmm3
	701	case 0x2464110f: // f2 0f 11 64 24 XX movsd QWORD PTR
	702	// [rsp + XX], xmm4
	703	return 6;
	704	}
	705	break;
559	706
560	707	case 0x83:
561	708	const u8 next_byte = (u8)(address + 1);
...	...	@@ -584,55 +731,155 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
584	731	case 0x018a: // mov al, byte ptr [rcx]
585	732	return 2;
586	733
	734	case 0x7E80: // 80 7E YY XX cmp BYTE PTR [rsi+YY], XX
	735	case 0x7D80: // 80 7D YY XX cmp BYTE PTR [rbp+YY], XX
	736	case 0x7A80: // 80 7A YY XX cmp BYTE PTR [rdx+YY], XX
	737	case 0x7880: // 80 78 YY XX cmp BYTE PTR [rax+YY], XX
	738	case 0x7B80: // 80 7B YY XX cmp BYTE PTR [rbx+YY], XX
	739	case 0x7980: // 80 79 YY XX cmp BYTE ptr [rcx+YY], XX
	740	return 4;
	741
587	742	case 0x058A: // 8A 05 XX XX XX XX : mov al, byte ptr [XX XX XX XX]
588	743	case 0x058B: // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX]
589	744	if (rel_offset)
590	745	*rel_offset = 2;
	746	case 0xB841: // 41 B8 XX XX XX XX : mov r8d, XX XX XX XX
591	747	return 6;
	748
	749	case 0x7E81: // 81 7E YY XX XX XX XX cmp DWORD PTR [rsi+YY], XX XX XX XX
	750	case 0x7D81: // 81 7D YY XX XX XX XX cmp DWORD PTR [rbp+YY], XX XX XX XX
	751	case 0x7A81: // 81 7A YY XX XX XX XX cmp DWORD PTR [rdx+YY], XX XX XX XX
	752	case 0x7881: // 81 78 YY XX XX XX XX cmp DWORD PTR [rax+YY], XX XX XX XX
	753	case 0x7B81: // 81 7B YY XX XX XX XX cmp DWORD PTR [rbx+YY], XX XX XX XX
	754	case 0x7981: // 81 79 YY XX XX XX XX cmp dword ptr [rcx+YY], XX XX XX XX
	755	return 7;
592	756	}
593	757
594		switch (0x00FFFFFF & (u32)address) {
595		case 0xe58948: // 48 8b c4 : mov rbp, rsp
596		case 0xc18b48: // 48 8b c1 : mov rax, rcx
597		case 0xc48b48: // 48 8b c4 : mov rax, rsp
598		case 0xd9f748: // 48 f7 d9 : neg rcx
599		case 0xd12b48: // 48 2b d1 : sub rdx, rcx
600		case 0x07c1f6: // f6 c1 07 : test cl, 0x7
601		case 0xc98548: // 48 85 C9 : test rcx, rcx
602		case 0xd28548: // 48 85 d2 : test rdx, rdx
	758	switch (0x00FFFFFF & (u32 )address) {
	759	case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax]
	760	case 0xc00b4d: // 4d 0b c0 : or r8, r8
	761	case 0xc03345: // 45 33 c0 : xor r8d, r8d
	762	case 0xc08548: // 48 85 c0 : test rax, rax
603	763	case 0xc0854d: // 4d 85 c0 : test r8, r8
	764	case 0xc08b41: // 41 8b c0 : mov eax, r8d
	765	case 0xc0ff48: // 48 ff c0 : inc rax
	766	case 0xc0ff49: // 49 ff c0 : inc r8
	767	case 0xc18b41: // 41 8b c1 : mov eax, r9d
	768	case 0xc18b48: // 48 8b c1 : mov rax, rcx
	769	case 0xc18b4c: // 4c 8b c1 : mov r8, rcx
	770	case 0xc1ff48: // 48 ff c1 : inc rcx
	771	case 0xc1ff49: // 49 ff c1 : inc r9
	772	case 0xc28b41: // 41 8b c2 : mov eax, r10d
	773	case 0x01b60f: // 0f b6 01 : movzx eax, BYTE PTR [rcx]
	774	case 0x09b60f: // 0f b6 09 : movzx ecx, BYTE PTR [rcx]
	775	case 0x11b60f: // 0f b6 11 : movzx edx, BYTE PTR [rcx]
604	776	case 0xc2b60f: // 0f b6 c2 : movzx eax, dl
605		case 0xc03345: // 45 33 c0 : xor r8d, r8d
	777	case 0xc2ff48: // 48 ff c2 : inc rdx
	778	case 0xc2ff49: // 49 ff c2 : inc r10
	779	case 0xc38b41: // 41 8b c3 : mov eax, r11d
	780	case 0xc3ff48: // 48 ff c3 : inc rbx
	781	case 0xc3ff49: // 49 ff c3 : inc r11
	782	case 0xc48b41: // 41 8b c4 : mov eax, r12d
	783	case 0xc48b48: // 48 8b c4 : mov rax, rsp
	784	case 0xc4ff49: // 49 ff c4 : inc r12
	785	case 0xc5ff49: // 49 ff c5 : inc r13
	786	case 0xc6ff48: // 48 ff c6 : inc rsi
	787	case 0xc6ff49: // 49 ff c6 : inc r14
	788	case 0xc7ff48: // 48 ff c7 : inc rdi
	789	case 0xc7ff49: // 49 ff c7 : inc r15
606	790	case 0xc93345: // 45 33 c9 : xor r9d, r9d
607		case 0xdb3345: // 45 33 DB : xor r11d, r11d
608		case 0xd98b4c: // 4c 8b d9 : mov r11, rcx
609		case 0xd28b4c: // 4c 8b d2 : mov r10, rdx
610		case 0xc98b4c: // 4C 8B C9 : mov r9, rcx
611		case 0xc18b4c: // 4C 8B C1 : mov r8, rcx
612		case 0xd2b60f: // 0f b6 d2 : movzx edx, dl
	791	case 0xc98548: // 48 85 c9 : test rcx, rcx
	792	case 0xc9854d: // 4d 85 c9 : test r9, r9
	793	case 0xc98b4c: // 4c 8b c9 : mov r9, rcx
	794	case 0xd12948: // 48 29 d1 : sub rcx, rdx
613	795	case 0xca2b48: // 48 2b ca : sub rcx, rdx
614	796	case 0xca3b48: // 48 3b ca : cmp rcx, rdx
615		case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax]
616		case 0xc00b4d: // 3d 0b c0 : or r8, r8
617		case 0xc08b41: // 41 8b c0 : mov eax, r8d
	797	case 0xd12b48: // 48 2b d1 : sub rdx, rcx
618	798	case 0xd18b48: // 48 8b d1 : mov rdx, rcx
619		case 0xdc8b4c: // 4c 8b dc : mov r11, rsp
620	799	case 0xd18b4c: // 4c 8b d1 : mov r10, rcx
621		case 0xE0E483: // 83 E4 E0 : and esp, 0xFFFFFFE0
	800	case 0xd28548: // 48 85 d2 : test rdx, rdx
	801	case 0xd2854d: // 4d 85 d2 : test r10, r10
	802	case 0xd28b4c: // 4c 8b d2 : mov r10, rdx
	803	case 0xd2b60f: // 0f b6 d2 : movzx edx, dl
	804	case 0xd2be0f: // 0f be d2 : movsx edx, dl
	805	case 0xd98b4c: // 4c 8b d9 : mov r11, rcx
	806	case 0xd9f748: // 48 f7 d9 : neg rcx
	807	case 0xc03145: // 45 31 c0 : xor r8d,r8d
	808	case 0xc93145: // 45 31 c9 : xor r9d,r9d
	809	case 0xdb3345: // 45 33 db : xor r11d, r11d
	810	case 0xc08445: // 45 84 c0 : test r8b,r8b
	811	case 0xd28445: // 45 84 d2 : test r10b,r10b
	812	case 0xdb8548: // 48 85 db : test rbx, rbx
	813	case 0xdb854d: // 4d 85 db : test r11, r11
	814	case 0xdc8b4c: // 4c 8b dc : mov r11, rsp
	815	case 0xe48548: // 48 85 e4 : test rsp, rsp
	816	case 0xe4854d: // 4d 85 e4 : test r12, r12
	817	case 0xc88948: // 48 89 c8 : mov rax,rcx
	818	case 0xcb8948: // 48 89 cb : mov rbx,rcx
	819	case 0xd08948: // 48 89 d0 : mov rax,rdx
	820	case 0xd18948: // 48 89 d1 : mov rcx,rdx
	821	case 0xd38948: // 48 89 d3 : mov rbx,rdx
	822	case 0xe58948: // 48 89 e5 : mov rbp, rsp
	823	case 0xed8548: // 48 85 ed : test rbp, rbp
	824	case 0xc88949: // 49 89 c8 : mov r8, rcx
	825	case 0xc98949: // 49 89 c9 : mov r9, rcx
	826	case 0xca8949: // 49 89 ca : mov r10,rcx
	827	case 0xd08949: // 49 89 d0 : mov r8, rdx
	828	case 0xd18949: // 49 89 d1 : mov r9, rdx
	829	case 0xd28949: // 49 89 d2 : mov r10, rdx
	830	case 0xd38949: // 49 89 d3 : mov r11, rdx
	831	case 0xed854d: // 4d 85 ed : test r13, r13
	832	case 0xf6854d: // 4d 85 f6 : test r14, r14
	833	case 0xff854d: // 4d 85 ff : test r15, r15
622	834	return 3;
623	835
	836	case 0x245489: // 89 54 24 XX : mov DWORD PTR[rsp + XX], edx
	837	case 0x428d44: // 44 8d 42 XX : lea r8d , [rdx + XX]
	838	case 0x588948: // 48 89 58 XX : mov QWORD PTR[rax + XX], rbx
624	839	case 0xec8348: // 48 83 ec XX : sub rsp, XX
625	840	case 0xf88349: // 49 83 f8 XX : cmp r8, XX
626		case 0x588948: // 48 89 58 XX : mov QWORD PTR[rax + XX], rbx
	841	case 0x488d49: // 49 8d 48 XX : lea rcx, [...]
	842	case 0x048d4c: // 4c 8d 04 XX : lea r8, [...]
	843	case 0x148d4e: // 4e 8d 14 XX : lea r10, [...]
	844	case 0x398366: // 66 83 39 XX : cmp WORD PTR [rcx], XX
627	845	return 4;
628	846
	847	case 0x441F0F: // 0F 1F 44 XX XX : nop DWORD PTR [...]
	848	case 0x246483: // 83 64 24 XX YY : and DWORD PTR [rsp+XX], YY
	849	return 5;
	850
	851	case 0x788166: // 66 81 78 XX YY YY cmp WORD PTR [rax+XX], YY YY
	852	case 0x798166: // 66 81 79 XX YY YY cmp WORD PTR [rcx+XX], YY YY
	853	case 0x7a8166: // 66 81 7a XX YY YY cmp WORD PTR [rdx+XX], YY YY
	854	case 0x7b8166: // 66 81 7b XX YY YY cmp WORD PTR [rbx+XX], YY YY
	855	case 0x7e8166: // 66 81 7e XX YY YY cmp WORD PTR [rsi+XX], YY YY
	856	case 0x7f8166: // 66 81 7f XX YY YY cmp WORD PTR [rdi+XX], YY YY
	857	return 6;
	858
629	859	case 0xec8148: // 48 81 EC XX XX XX XX : sub rsp, XXXXXXXX
	860	case 0xc0c748: // 48 C7 C0 XX XX XX XX : mov rax, XX XX XX XX
630	861	return 7;
631	862
	863	// clang-format off
	864	case 0x788141: // 41 81 78 XX YY YY YY YY : cmp DWORD PTR [r8+YY], XX XX XX XX
	865	case 0x798141: // 41 81 79 XX YY YY YY YY : cmp DWORD PTR [r9+YY], XX XX XX XX
	866	case 0x7a8141: // 41 81 7a XX YY YY YY YY : cmp DWORD PTR [r10+YY], XX XX XX XX
	867	case 0x7b8141: // 41 81 7b XX YY YY YY YY : cmp DWORD PTR [r11+YY], XX XX XX XX
	868	case 0x7d8141: // 41 81 7d XX YY YY YY YY : cmp DWORD PTR [r13+YY], XX XX XX XX
	869	case 0x7e8141: // 41 81 7e XX YY YY YY YY : cmp DWORD PTR [r14+YY], XX XX XX XX
	870	case 0x7f8141: // 41 81 7f YY XX XX XX XX : cmp DWORD PTR [r15+YY], XX XX XX XX
	871	case 0x247c81: // 81 7c 24 YY XX XX XX XX : cmp DWORD PTR [rsp+YY], XX XX XX XX
	872	return 8;
	873	// clang-format on
	874
632	875	case 0x058b48: // 48 8b 05 XX XX XX XX :
633	876	// mov rax, QWORD PTR [rip + XXXXXXXX]
634	877	case 0x058d48: // 48 8d 05 XX XX XX XX :
635	878	// lea rax, QWORD PTR [rip + XXXXXXXX]
	879	case 0x0d8948: // 48 89 0d XX XX XX XX :
	880	// mov QWORD PTR [rip + XXXXXXXX], rcx
	881	case 0x158948: // 48 89 15 XX XX XX XX :
	882	// mov QWORD PTR [rip + XXXXXXXX], rdx
636	883	case 0x25ff48: // 48 ff 25 XX XX XX XX :
637	884	// rex.W jmp QWORD PTR [rip + XXXXXXXX]
638	885	case 0x158D4C: // 4c 8d 15 XX XX XX XX : lea r10, [rip + XX]
...	...	@@ -644,9 +891,19 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
644	891	case 0x2444c7: // C7 44 24 XX YY YY YY YY
645	892	// mov dword ptr [rsp + XX], YYYYYYYY
646	893	return 8;
	894
	895	case 0x7c8141: // 41 81 7c ZZ YY XX XX XX XX
	896	// cmp DWORD PTR [reg+reg*n+YY], XX XX XX XX
	897	return 9;
647	898	}
648	899
649	900	switch ((u32)(address)) {
	901	case 0x01b60f44: // 44 0f b6 01 : movzx r8d, BYTE PTR [rcx]
	902	case 0x09b60f44: // 44 0f b6 09 : movzx r9d, BYTE PTR [rcx]
	903	case 0x0ab60f44: // 44 0f b6 0a : movzx r8d, BYTE PTR [rdx]
	904	case 0x11b60f44: // 44 0f b6 11 : movzx r10d, BYTE PTR [rcx]
	905	case 0x1ab60f44: // 44 0f b6 1a : movzx r11d, BYTE PTR [rdx]
	906	return 4;
650	907	case 0x24448b48: // 48 8b 44 24 XX : mov rax, QWORD ptr [rsp + XX]
651	908	case 0x246c8948: // 48 89 6C 24 XX : mov QWORD ptr [rsp + XX], rbp
652	909	case 0x245c8948: // 48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx
...	...	@@ -656,9 +913,19 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
656	913	case 0x24548948: // 48 89 54 24 XX : mov QWORD PTR [rsp + XX], rdx
657	914	case 0x244c894c: // 4c 89 4c 24 XX : mov QWORD PTR [rsp + XX], r9
658	915	case 0x2444894c: // 4c 89 44 24 XX : mov QWORD PTR [rsp + XX], r8
	916	case 0x244c8944: // 44 89 4c 24 XX mov DWORD PTR [rsp + XX], r9d
	917	case 0x24448944: // 44 89 44 24 XX mov DWORD PTR [rsp + XX], r8d
	918	case 0x246c8d48: // 48 8d 6c 24 XX : lea rbp, [rsp + XX]
659	919	return 5;
660		case 0x24648348: // 48 83 64 24 XX : and QWORD PTR [rsp + XX], YY
	920	case 0x24648348: // 48 83 64 24 XX YY : and QWORD PTR [rsp + XX], YY
661	921	return 6;
	922	case 0x24A48D48: // 48 8D A4 24 XX XX XX XX : lea rsp, [rsp + XX XX XX XX]
	923	return 8;
	924	}
	925
	926	switch (0xFFFFFFFFFFULL & (u64 )(address)) {
	927	case 0xC07E0F4866: // 66 48 0F 7E C0 : movq rax, xmm0
	928	return 5;
662	929	}
663	930
664	931	#else
...	...	@@ -671,11 +938,10 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
671	938	case 0x458B: // 8B 45 XX : mov eax, dword ptr [ebp + XX]
672	939	case 0x5D8B: // 8B 5D XX : mov ebx, dword ptr [ebp + XX]
673	940	case 0x7D8B: // 8B 7D XX : mov edi, dword ptr [ebp + XX]
674		case 0xEC83: // 83 EC XX : sub esp, XX
	941	case 0x758B: // 8B 75 XX : mov esi, dword ptr [ebp + XX]
675	942	case 0x75FF: // FF 75 XX : push dword ptr [ebp + XX]
676	943	return 3;
677	944	case 0xC1F7: // F7 C1 XX YY ZZ WW : test ecx, WWZZYYXX
678		case 0x25FF: // FF 25 XX YY ZZ WW : jmp dword ptr ds:[WWZZYYXX]
679	945	return 6;
680	946	case 0x3D83: // 83 3D XX YY ZZ WW TT : cmp TT, WWZZYYXX
681	947	return 7;
...	...	@@ -718,6 +984,10 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
718	984	return 0;
719	985	}
720	986
	987	size_t TestOnlyGetInstructionSize(uptr address, size_t *rel_offset) {
	988	return GetInstructionSize(address, rel_offset);
	989	}
	990
721	991	// Returns 0 on error.
722	992	static size_t RoundUpToInstrBoundary(size_t size, uptr address) {
723	993	size_t cursor = 0;
...	...	@@ -745,8 +1015,14 @@ static bool CopyInstructions(uptr to, uptr from, size_t size) {
745	1015	// this will be untrue if relocated_offset \notin [-231, 231)
746	1016	s64 delta = to - from;
747	1017	s64 relocated_offset = (s32 )(to + cursor + rel_offset) - delta;
748		if (-0x8000'0000ll > relocated_offset \|\| relocated_offset > 0x7FFF'FFFFll)
	1018	if (-0x8000'0000ll > relocated_offset \|\|
	1019	relocated_offset > 0x7FFF'FFFFll) {
	1020	ReportError(
	1021	"interception_win: CopyInstructions relocated_offset %lld outside "
	1022	"32-bit range\n",
	1023	(long long)relocated_offset);
749	1024	return false;
	1025	}
750	1026	# else
751	1027	// on 32-bit, the relative offset will always be correct
752	1028	s32 delta = to - from;
...	...	@@ -969,8 +1245,7 @@ static void **InterestingDLLsAvailable() {
969	1245	"libc++.dll", // libc++
970	1246	"libunwind.dll", // libunwind
971	1247	# endif
972		// NTDLL should go last as it exports some functions that we should
973		// override in the CRT [presumably only used internally].
	1248	// NTDLL must go last as it gets special treatment in OverrideFunction.
974	1249	"ntdll.dll",
975	1250	NULL
976	1251	};
...	...	@@ -1027,7 +1302,7 @@ uptr InternalGetProcAddress(void module, const char func_name) {
1027	1302
1028	1303	for (DWORD i = 0; i < exports->NumberOfNames; i++) {
1029	1304	RVAPtr<char> name(module, names[i]);
1030		if (!strcmp(func_name, name)) {
	1305	if (!_strcmp(func_name, name)) {
1031	1306	DWORD index = ordinals[i];
1032	1307	RVAPtr<char> func(module, functions[index]);
1033	1308
...	...	@@ -1040,19 +1315,27 @@ uptr InternalGetProcAddress(void module, const char func_name) {
1040	1315	// exported directory.
1041	1316	char function_name[256];
1042	1317	size_t funtion_name_length = _strlen(func);
1043		if (funtion_name_length >= sizeof(function_name) - 1)
	1318	if (funtion_name_length >= sizeof(function_name) - 1) {
	1319	ReportError("interception_win: func too long: '%s'\n", (char *)func);
1044	1320	InterceptionFailed();
	1321	}
1045	1322
1046	1323	_memcpy(function_name, func, funtion_name_length);
1047	1324	function_name[funtion_name_length] = '\0';
1048	1325	char* separator = _strchr(function_name, '.');
1049		if (!separator)
	1326	if (!separator) {
	1327	ReportError("interception_win: no separator in '%s'\n",
	1328	function_name);
1050	1329	InterceptionFailed();
	1330	}
1051	1331	*separator = '\0';
1052	1332
1053	1333	void* redirected_module = GetModuleHandleA(function_name);
1054		if (!redirected_module)
	1334	if (!redirected_module) {
	1335	ReportError("interception_win: GetModuleHandleA failed for '%s'\n",
	1336	function_name);
1055	1337	InterceptionFailed();
	1338	}
1056	1339	return InternalGetProcAddress(redirected_module, separator + 1);
1057	1340	}
1058	1341
...	...	@@ -1065,9 +1348,22 @@ uptr InternalGetProcAddress(void module, const char func_name) {
1065	1348
1066	1349	bool OverrideFunction(
1067	1350	const char func_name, uptr new_func, uptr orig_old_func) {
	1351	static const char *kNtDllIgnore[] = {
	1352	"memcmp", "memcpy", "memmove", "memset"
	1353	};
	1354
1068	1355	bool hooked = false;
1069	1356	void **DLLs = InterestingDLLsAvailable();
1070	1357	for (size_t i = 0; DLLs[i]; ++i) {
	1358	if (DLLs[i + 1] == nullptr) {
	1359	// This is the last DLL, i.e. NTDLL. It exports some functions that
	1360	// we only want to override in the CRT.
	1361	for (const char *ignored : kNtDllIgnore) {
	1362	if (_strcmp(func_name, ignored) == 0)
	1363	return hooked;
	1364	}
	1365	}
	1366
1071	1367	uptr func_addr = InternalGetProcAddress(DLLs[i], func_name);
1072	1368	if (func_addr &&
1073	1369	OverrideFunction(func_addr, new_func, orig_old_func)) {
...	...	@@ -1121,7 +1417,7 @@ bool OverrideImportedFunction(const char *module_to_patch,
1121	1417	RVAPtr<IMAGE_IMPORT_BY_NAME> import_by_name(
1122	1418	module, name_table->u1.ForwarderString);
1123	1419	const char *funcname = &import_by_name->Name[0];
1124		if (strcmp(funcname, function_name) == 0)
	1420	if (_strcmp(funcname, function_name) == 0)
1125	1421	break;
1126	1422	}
1127	1423	}
...	...	@@ -1144,4 +1440,4 @@ bool OverrideImportedFunction(const char *module_to_patch,
1144	1440
1145	1441	} // namespace __interception
1146	1442
1147		#endif // SANITIZER_APPLE
	1443	#endif // SANITIZER_WINDOWS

lib/tsan/interception/interception_win.h+3

...	...	@@ -63,6 +63,9 @@ bool OverrideFunctionWithTrampoline(
63	63	// Exposed for unittests
64	64	void TestOnlyReleaseTrampolineRegions();
65	65
	66	// Exposed for unittests
	67	SIZE_T TestOnlyGetInstructionSize(uptr address, SIZE_T *rel_offset);
	68
66	69	} // namespace __interception
67	70
68	71	#if defined(INTERCEPTION_DYNAMIC_CRT)

lib/tsan/sanitizer_common/sanitizer_allocator.cpp+3-2

...	...	@@ -59,7 +59,7 @@ static void RawInternalAlloc(uptr size, InternalAllocatorCache cache,
59	59
60	60	static void RawInternalRealloc(void ptr, uptr size,
61	61	InternalAllocatorCache *cache) {
62		uptr alignment = 8;
	62	constexpr usize alignment = Max<usize>(8, sizeof(void *));
63	63	if (cache == 0) {
64	64	SpinMutexLock l(&internal_allocator_cache_mu);
65	65	return internal_allocator()->Reallocate(&internal_allocator_cache, ptr,
...	...	@@ -137,7 +137,8 @@ void InternalAllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
137	137	}
138	138
139	139	// LowLevelAllocator
140		constexpr uptr kLowLevelAllocatorDefaultAlignment = 8;
	140	constexpr usize kLowLevelAllocatorDefaultAlignment =
	141	Max<usize>(8, sizeof(void *));
141	142	constexpr uptr kMinNumPagesRounded = 16;
142	143	constexpr uptr kMinRoundedSize = 65536;
143	144	static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment;

lib/tsan/sanitizer_common/sanitizer_allocator_dlsym.h+18-9

...	...	@@ -15,6 +15,8 @@
15	15	#define SANITIZER_ALLOCATOR_DLSYM_H
16	16
17	17	#include "sanitizer_allocator_internal.h"
	18	#include "sanitizer_common/sanitizer_allocator_checks.h"
	19	#include "sanitizer_common/sanitizer_internal_defs.h"
18	20
19	21	namespace __sanitizer {
20	22
...	...	@@ -31,24 +33,22 @@ struct DlSymAllocator {
31	33	UNLIKELY(internal_allocator()->FromPrimary(ptr));
32	34	}
33	35
34		static void *Allocate(uptr size_in_bytes) {
35		void *ptr = InternalAlloc(size_in_bytes, nullptr, kWordSize);
	36	static void *Allocate(uptr size_in_bytes, uptr align = kWordSize) {
	37	void *ptr = InternalAlloc(size_in_bytes, nullptr, align);
36	38	CHECK(internal_allocator()->FromPrimary(ptr));
37		Details::OnAllocate(ptr,
38		internal_allocator()->GetActuallyAllocatedSize(ptr));
	39	Details::OnAllocate(ptr, GetSize(ptr));
39	40	return ptr;
40	41	}
41	42
42		static void *Callocate(SIZE_T nmemb, SIZE_T size) {
	43	static void *Callocate(usize nmemb, usize size) {
43	44	void *ptr = InternalCalloc(nmemb, size);
44	45	CHECK(internal_allocator()->FromPrimary(ptr));
45		Details::OnAllocate(ptr,
46		internal_allocator()->GetActuallyAllocatedSize(ptr));
	46	Details::OnAllocate(ptr, GetSize(ptr));
47	47	return ptr;
48	48	}
49	49
50	50	static void Free(void *ptr) {
51		uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
	51	uptr size = GetSize(ptr);
52	52	Details::OnFree(ptr, size);
53	53	InternalFree(ptr);
54	54	}
...	...	@@ -61,7 +61,7 @@ struct DlSymAllocator {
61	61	Free(ptr);
62	62	return nullptr;
63	63	}
64		uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
	64	uptr size = GetSize(ptr);
65	65	uptr memcpy_size = Min(new_size, size);
66	66	void *new_ptr = Allocate(new_size);
67	67	if (new_ptr)
...	...	@@ -70,6 +70,15 @@ struct DlSymAllocator {
70	70	return new_ptr;
71	71	}
72	72
	73	static void ReallocArray(void ptr, uptr count, uptr size) {
	74	CHECK(!CheckForCallocOverflow(count, size));
	75	return Realloc(ptr, count * size);
	76	}
	77
	78	static uptr GetSize(void *ptr) {
	79	return internal_allocator()->GetActuallyAllocatedSize(ptr);
	80	}
	81
73	82	static void OnAllocate(const void *ptr, uptr size) {}
74	83	static void OnFree(const void *ptr, uptr size) {}
75	84	};

lib/tsan/sanitizer_common/sanitizer_allocator_primary64.h+7-5

...	...	@@ -185,9 +185,10 @@ class SizeClassAllocator64 {
185	185	// recoverable.
186	186	if (UNLIKELY(!EnsureFreeArraySpace(region, region_beg,
187	187	new_num_freed_chunks))) {
188		Report("FATAL: Internal error: %s's allocator exhausted the free list "
189		"space for size class %zd (%zd bytes).\n", SanitizerToolName,
190		class_id, ClassIdToSize(class_id));
	188	Report(
	189	"FATAL: Internal error: %s's allocator exhausted the free list "
	190	"space for size class %zu (%zu bytes).\n",
	191	SanitizerToolName, class_id, ClassIdToSize(class_id));
191	192	Die();
192	193	}
193	194	for (uptr i = 0; i < n_chunks; i++)
...	...	@@ -763,8 +764,9 @@ class SizeClassAllocator64 {
763	764	if (!region->exhausted) {
764	765	region->exhausted = true;
765	766	Printf("%s: Out of memory. ", SanitizerToolName);
766		Printf("The process has exhausted %zuMB for size class %zu.\n",
767		kRegionSize >> 20, ClassIdToSize(class_id));
	767	Printf(
	768	"The process has exhausted %zu MB for size class %zu (%zu bytes).\n",
	769	kRegionSize >> 20, class_id, ClassIdToSize(class_id));
768	770	}
769	771	return true;
770	772	}

lib/tsan/sanitizer_common/sanitizer_common.h+18-9

...	...	@@ -83,8 +83,8 @@ int TgKill(pid_t pid, tid_t tid, int sig);
83	83	uptr GetThreadSelf();
84	84	void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
85	85	uptr *stack_bottom);
86		void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,
87		uptr tls_addr, uptr tls_size);
	86	void GetThreadStackAndTls(bool main, uptr stk_begin, uptr stk_end,
	87	uptr tls_begin, uptr tls_end);
88	88
89	89	// Memory management
90	90	void MmapOrDie(uptr size, const char mem_type, bool raw_report = false);
...	...	@@ -239,13 +239,15 @@ void RemoveANSIEscapeSequencesFromString(char *buffer);
239	239	void Printf(const char *format, ...) FORMAT(1, 2);
240	240	void Report(const char *format, ...) FORMAT(1, 2);
241	241	void SetPrintfAndReportCallback(void (callback)(const char ));
242		#define VReport(level, ...) \
243		do { \
244		if ((uptr)Verbosity() >= (level)) Report(__VA_ARGS__); \
	242	#define VReport(level, ...) \
	243	do { \
	244	if (UNLIKELY((uptr)Verbosity() >= (level))) \
	245	Report(__VA_ARGS__); \
245	246	} while (0)
246		#define VPrintf(level, ...) \
247		do { \
248		if ((uptr)Verbosity() >= (level)) Printf(__VA_ARGS__); \
	247	#define VPrintf(level, ...) \
	248	do { \
	249	if (UNLIKELY((uptr)Verbosity() >= (level))) \
	250	Printf(__VA_ARGS__); \
249	251	} while (0)
250	252
251	253	// Lock sanitizer error reporting and protects against nested errors.
...	...	@@ -266,7 +268,15 @@ class ScopedErrorReportLock {
266	268	extern uptr stoptheworld_tracer_pid;
267	269	extern uptr stoptheworld_tracer_ppid;
268	270
	271	// Returns true if the entire range can be read.
269	272	bool IsAccessibleMemoryRange(uptr beg, uptr size);
	273	// Attempts to copy `n` bytes from memory range starting at `src` to `dest`.
	274	// Returns true if the entire range can be read. Returns `false` if any part of
	275	// the source range cannot be read, in which case the contents of `dest` are
	276	// undefined.
	277	bool TryMemCpy(void dest, const void src, uptr n);
	278	// Copies accessible memory, and zero fill inaccessible.
	279	void MemCpyAccessible(void dest, const void src, uptr n);
270	280
271	281	// Error report formatting.
272	282	const char StripPathPrefix(const char filepath,
...	...	@@ -917,7 +927,6 @@ typedef void (RangeIteratorCallback)(uptr begin, uptr end, void arg);
917	927
918	928	enum AndroidApiLevel {
919	929	ANDROID_NOT_ANDROID = 0,
920		ANDROID_KITKAT = 19,
921	930	ANDROID_LOLLIPOP_MR1 = 22,
922	931	ANDROID_POST_LOLLIPOP = 23
923	932	};

lib/tsan/sanitizer_common/sanitizer_common_interceptors.inc+189-221

...	...	@@ -41,6 +41,7 @@
41	41	#include "sanitizer_errno.h"
42	42	#include "sanitizer_placement_new.h"
43	43	#include "sanitizer_platform_interceptors.h"
	44	#include "sanitizer_platform_limits_posix.h"
44	45	#include "sanitizer_symbolizer.h"
45	46	#include "sanitizer_tls_get_addr.h"
46	47
...	...	@@ -127,6 +128,39 @@ extern const short *_toupper_tab_;
127	128	extern const short *_tolower_tab_;
128	129	#endif
129	130
	131	#if SANITIZER_LINUX && SANITIZER_SPARC32
	132	// On 32-bit Linux/sparc64, double and long double are identical and glibc
	133	// uses a __nldbl_ (no long double) prefix for various stdio functions.
	134	# define __isoc23_fscanf __nldbl___isoc23_fscanf
	135	# define __isoc23_scanf __nldbl___isoc23_scanf
	136	# define __isoc23_sscanf __nldbl___isoc23_sscanf
	137	# define __isoc23_vfscanf __nldbl___isoc23_vfscanf
	138	# define __isoc23_vscanf __nldbl___isoc23_vscanf
	139	# define __isoc23_vsscanf __nldbl___isoc23_vsscanf
	140	# define __isoc99_fscanf __nldbl___isoc99_fscanf
	141	# define __isoc99_scanf __nldbl___isoc99_scanf
	142	# define __isoc99_sscanf __nldbl___isoc99_sscanf
	143	# define __isoc99_vfscanf __nldbl___isoc99_vfscanf
	144	# define __isoc99_vscanf __nldbl___isoc99_vscanf
	145	# define __isoc99_vsscanf __nldbl___isoc99_vsscanf
	146	# define asprintf __nldbl_asprintf
	147	# define fprintf __nldbl_fprintf
	148	# define fscanf __nldbl_fscanf
	149	# define printf __nldbl_printf
	150	# define scanf __nldbl_scanf
	151	# define snprintf __nldbl_snprintf
	152	# define sprintf __nldbl_sprintf
	153	# define sscanf __nldbl_sscanf
	154	# define vasprintf __nldbl_vasprintf
	155	# define vfprintf __nldbl_vfprintf
	156	# define vfscanf __nldbl_vfscanf
	157	# define vprintf __nldbl_vprintf
	158	# define vscanf __nldbl_vscanf
	159	# define vsnprintf __nldbl_vsnprintf
	160	# define vsprintf __nldbl_vsprintf
	161	# define vsscanf __nldbl_vsscanf
	162	#endif
	163
130	164	#if SANITIZER_MUSL && \
131	165	(defined(__i386__) \|\| defined(__arm__) \|\| SANITIZER_MIPS32 \|\| SANITIZER_PPC32)
132	166	// musl 1.2.0 on existing 32-bit architectures uses new symbol names for the
...	...	@@ -313,7 +347,7 @@ extern const short *_tolower_tab_;
313	347	uptr copy_length = internal_strnlen(s, size); \
314	348	char new_mem = (char )WRAP(malloc)(copy_length + 1); \
315	349	if (common_flags()->intercept_strndup) { \
316		COMMON_INTERCEPTOR_READ_STRING(ctx, s, Min(size, copy_length + 1)); \
	350	COMMON_INTERCEPTOR_READ_STRING(ctx, s, Min<uptr>(size, copy_length + 1)); \
317	351	} \
318	352	if (new_mem) { \
319	353	COMMON_INTERCEPTOR_COPY_STRING(ctx, new_mem, s, copy_length); \
...	...	@@ -411,7 +445,7 @@ INTERCEPTOR(SIZE_T, strnlen, const char *s, SIZE_T maxlen) {
411	445	#endif
412	446
413	447	#if SANITIZER_INTERCEPT_STRNDUP
414		INTERCEPTOR(char, strndup, const char s, uptr size) {
	448	INTERCEPTOR(char, strndup, const char s, usize size) {
415	449	void *ctx;
416	450	COMMON_INTERCEPTOR_STRNDUP_IMPL(ctx, s, size);
417	451	}
...	...	@@ -421,7 +455,7 @@ INTERCEPTOR(char, strndup, const char s, uptr size) {
421	455	#endif // SANITIZER_INTERCEPT_STRNDUP
422	456
423	457	#if SANITIZER_INTERCEPT___STRNDUP
424		INTERCEPTOR(char, __strndup, const char s, uptr size) {
	458	INTERCEPTOR(char, __strndup, const char s, usize size) {
425	459	void *ctx;
426	460	COMMON_INTERCEPTOR_STRNDUP_IMPL(ctx, s, size);
427	461	}
...	...	@@ -477,23 +511,23 @@ INTERCEPTOR(int, strcmp, const char s1, const char s2) {
477	511	}
478	512
479	513	DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncmp, uptr called_pc,
480		const char s1, const char s2, uptr n,
	514	const char s1, const char s2, usize n,
481	515	int result)
482	516
483		INTERCEPTOR(int, strncmp, const char s1, const char s2, uptr size) {
	517	INTERCEPTOR(int, strncmp, const char s1, const char s2, usize size) {
484	518	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
485	519	return internal_strncmp(s1, s2, size);
486	520	void *ctx;
487	521	COMMON_INTERCEPTOR_ENTER(ctx, strncmp, s1, s2, size);
488	522	unsigned char c1 = 0, c2 = 0;
489		uptr i;
	523	usize i;
490	524	for (i = 0; i < size; i++) {
491	525	c1 = (unsigned char)s1[i];
492	526	c2 = (unsigned char)s2[i];
493	527	if (c1 != c2 \|\| c1 == '\0') break;
494	528	}
495		uptr i1 = i;
496		uptr i2 = i;
	529	usize i1 = i;
	530	usize i2 = i;
497	531	if (common_flags()->strict_string_checks) {
498	532	for (; i1 < size && s1[i1]; i1++) {}
499	533	for (; i2 < size && s2[i2]; i2++) {}
...	...	@@ -542,21 +576,21 @@ INTERCEPTOR(int, strcasecmp, const char s1, const char s2) {
542	576	}
543	577
544	578	DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncasecmp, uptr called_pc,
545		const char s1, const char s2, uptr size,
	579	const char s1, const char s2, usize size,
546	580	int result)
547	581
548	582	INTERCEPTOR(int, strncasecmp, const char s1, const char s2, SIZE_T size) {
549	583	void *ctx;
550	584	COMMON_INTERCEPTOR_ENTER(ctx, strncasecmp, s1, s2, size);
551	585	unsigned char c1 = 0, c2 = 0;
552		uptr i;
	586	usize i;
553	587	for (i = 0; i < size; i++) {
554	588	c1 = (unsigned char)s1[i];
555	589	c2 = (unsigned char)s2[i];
556	590	if (CharCaseCmp(c1, c2) != 0 \|\| c1 == '\0') break;
557	591	}
558		uptr i1 = i;
559		uptr i2 = i;
	592	usize i1 = i;
	593	usize i2 = i;
560	594	if (common_flags()->strict_string_checks) {
561	595	for (; i1 < size && s1[i1]; i1++) {}
562	596	for (; i2 < size && s2[i2]; i2++) {}
...	...	@@ -799,13 +833,13 @@ INTERCEPTOR(char , strpbrk, const char s1, const char *s2) {
799	833
800	834	#if SANITIZER_INTERCEPT_MEMCMP
801	835	DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc,
802		const void s1, const void s2, uptr n,
	836	const void s1, const void s2, usize n,
803	837	int result)
804	838
805	839	// Common code for `memcmp` and `bcmp`.
806	840	int MemcmpInterceptorCommon(void *ctx,
807		int (real_fn)(const void , const void *, uptr),
808		const void a1, const void a2, uptr size) {
	841	int (real_fn)(const void , const void *, usize),
	842	const void a1, const void a2, usize size) {
809	843	if (common_flags()->intercept_memcmp) {
810	844	if (common_flags()->strict_memcmp) {
811	845	// Check the entire regions even if the first bytes of the buffers are
...	...	@@ -817,7 +851,7 @@ int MemcmpInterceptorCommon(void *ctx,
817	851	unsigned char c1 = 0, c2 = 0;
818	852	const unsigned char s1 = (const unsigned char)a1;
819	853	const unsigned char s2 = (const unsigned char)a2;
820		uptr i;
	854	usize i;
821	855	for (i = 0; i < size; i++) {
822	856	c1 = s1[i];
823	857	c2 = s2[i];
...	...	@@ -837,7 +871,7 @@ int MemcmpInterceptorCommon(void *ctx,
837	871	return result;
838	872	}
839	873
840		INTERCEPTOR(int, memcmp, const void a1, const void a2, uptr size) {
	874	INTERCEPTOR(int, memcmp, const void a1, const void a2, usize size) {
841	875	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
842	876	return internal_memcmp(a1, a2, size);
843	877	void *ctx;
...	...	@@ -851,7 +885,7 @@ INTERCEPTOR(int, memcmp, const void a1, const void a2, uptr size) {
851	885	#endif
852	886
853	887	#if SANITIZER_INTERCEPT_BCMP
854		INTERCEPTOR(int, bcmp, const void a1, const void a2, uptr size) {
	888	INTERCEPTOR(int, bcmp, const void a1, const void a2, usize size) {
855	889	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
856	890	return internal_memcmp(a1, a2, size);
857	891	void *ctx;
...	...	@@ -1104,7 +1138,7 @@ INTERCEPTOR(SSIZE_T, write, int fd, void *ptr, SIZE_T count) {
1104	1138	#endif
1105	1139
1106	1140	#if SANITIZER_INTERCEPT_FWRITE
1107		INTERCEPTOR(SIZE_T, fwrite, const void p, uptr size, uptr nmemb, void file) {
	1141	INTERCEPTOR(SIZE_T, fwrite, const void p, usize size, usize nmemb, void file) {
1108	1142	// libc file streams can call user-supplied functions, see fopencookie.
1109	1143	void *ctx;
1110	1144	COMMON_INTERCEPTOR_ENTER(ctx, fwrite, p, size, nmemb, file);
...	...	@@ -1255,6 +1289,12 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
1255	1289	static const int PR_SET_VMA = 0x53564d41;
1256	1290	static const int PR_SCHED_CORE = 62;
1257	1291	static const int PR_SCHED_CORE_GET = 0;
	1292	static const int PR_GET_PDEATHSIG = 2;
	1293
	1294	# if !SANITIZER_ANDROID
	1295	static const int PR_SET_SECCOMP = 22;
	1296	static const int SECCOMP_MODE_FILTER = 2;
	1297	# endif
1258	1298	if (option == PR_SET_VMA && arg2 == 0UL) {
1259	1299	char name = (char )arg5;
1260	1300	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
...	...	@@ -1270,7 +1310,14 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
1270	1310	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
1271	1311	} else if (res != -1 && option == PR_SCHED_CORE &&
1272	1312	arg2 == PR_SCHED_CORE_GET) {
1273		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64*)(arg5), sizeof(u64));
	1313	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg5), sizeof(u64));
	1314	} else if (res != -1 && option == PR_GET_PDEATHSIG) {
	1315	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg2), sizeof(int));
	1316	# if SANITIZER_GLIBC
	1317	} else if (res != -1 && option == PR_SET_SECCOMP &&
	1318	arg2 == SECCOMP_MODE_FILTER) {
	1319	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg3), struct_sock_fprog_sz);
	1320	# endif
1274	1321	}
1275	1322	return res;
1276	1323	}
...	...	@@ -2242,6 +2289,61 @@ INTERCEPTOR(int, pthread_getcpuclockid, uptr thread,
2242	2289	#define INIT_CLOCK_GETCPUCLOCKID
2243	2290	#endif
2244	2291
	2292	#if SANITIZER_INTERCEPT_TIMER_CREATE
	2293	INTERCEPTOR(int, timer_create, __sanitizer_clockid_t clockid, void *sevp,
	2294	__sanitizer_timer_t *timer) {
	2295	void *ctx;
	2296	COMMON_INTERCEPTOR_ENTER(ctx, timer_create, clockid, sevp, timer);
	2297	int res = REAL(timer_create)(clockid, sevp, timer);
	2298	if (!res && timer) {
	2299	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, timer, sizeof *timer);
	2300	}
	2301	return res;
	2302	}
	2303
	2304	INTERCEPTOR(int, timer_delete, __sanitizer_timer_t timer) {
	2305	void *ctx;
	2306	COMMON_INTERCEPTOR_ENTER(ctx, timer_delete, timer);
	2307	int res = REAL(timer_delete)(timer);
	2308	return res;
	2309	}
	2310
	2311	INTERCEPTOR(int, timer_gettime, __sanitizer_timer_t timer,
	2312	struct __sanitizer_itimerspec *curr_value) {
	2313	void *ctx;
	2314	COMMON_INTERCEPTOR_ENTER(ctx, timer_gettime, timer, curr_value);
	2315	int res = REAL(timer_gettime)(timer, curr_value);
	2316	if (!res && curr_value) {
	2317	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, curr_value, sizeof *curr_value);
	2318	}
	2319	return res;
	2320	}
	2321
	2322	INTERCEPTOR(int, timer_settime, __sanitizer_timer_t timer, int flags,
	2323	const struct __sanitizer_itimerspec *new_value,
	2324	struct __sanitizer_itimerspec *old_value) {
	2325	void *ctx;
	2326	COMMON_INTERCEPTOR_ENTER(ctx, timer_settime, timer, flags, new_value,
	2327	old_value);
	2328	int res = REAL(timer_settime)(timer, flags, new_value, old_value);
	2329	if (!res) {
	2330	if (new_value)
	2331	COMMON_INTERCEPTOR_READ_RANGE(ctx, new_value, sizeof *new_value);
	2332	if (old_value)
	2333	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, old_value, sizeof *old_value);
	2334	}
	2335	return res;
	2336	}
	2337
	2338	# define INIT_TIMER_CREATE \
	2339	COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(timer_create, "GLIBC_2.3.3"); \
	2340	COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(timer_delete, "GLIBC_2.3.3"); \
	2341	COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(timer_gettime, "GLIBC_2.3.3"); \
	2342	COMMON_INTERCEPT_FUNCTION_GLIBC_VER_MIN(timer_settime, "GLIBC_2.3.3");
	2343	#else
	2344	# define INIT_TIMER_CREATE
	2345	#endif
	2346
2245	2347	#if SANITIZER_INTERCEPT_GETITIMER
2246	2348	INTERCEPTOR(int, getitimer, int which, void *curr_value) {
2247	2349	void *ctx;
...	...	@@ -2287,6 +2389,25 @@ INTERCEPTOR(int, setitimer, int which, const void new_value, void old_value) {
2287	2389	#define INIT_GETITIMER
2288	2390	#endif
2289	2391
	2392	#if SANITIZER_INTERCEPT_TIMESPEC_GET
	2393	INTERCEPTOR(int, timespec_get, struct __sanitizer_timespec *ts, int base) {
	2394	void *ctx;
	2395	COMMON_INTERCEPTOR_ENTER(ctx, timespec_get, ts, base);
	2396	// We don't yet know if ts is addressable, so we use our own scratch buffer
	2397	struct __sanitizer_timespec ts_local;
	2398	int res = REAL(timespec_get)(&ts_local, base);
	2399	if (res) {
	2400	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ts,
	2401	sizeof(struct __sanitizer_timespec));
	2402	internal_memcpy(ts, &ts_local, sizeof(struct __sanitizer_timespec));
	2403	}
	2404	return res;
	2405	}
	2406	# define INIT_TIMESPEC_GET COMMON_INTERCEPT_FUNCTION(timespec_get);
	2407	#else
	2408	# define INIT_TIMESPEC_GET
	2409	#endif
	2410
2290	2411	#if SANITIZER_INTERCEPT_GLOB
2291	2412	static void unpoison_glob_t(void ctx, __sanitizer_glob_t pglob) {
2292	2413	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pglob, sizeof(*pglob));
...	...	@@ -3427,23 +3548,27 @@ INTERCEPTOR(uptr, ptrace, int request, int pid, void addr, void data) {
3427	3548	COMMON_INTERCEPTOR_ENTER(ctx, ptrace, request, pid, addr, data);
3428	3549	__sanitizer_iovec local_iovec;
3429	3550
3430		if (data) {
	3551	void *data_arg = ptrace_data_arg(request, addr, data);
	3552	if (data_arg) {
3431	3553	if (request == ptrace_setregs) {
3432		COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_regs_struct_sz);
	3554	COMMON_INTERCEPTOR_READ_RANGE(ctx, data_arg, struct_user_regs_struct_sz);
3433	3555	} else if (request == ptrace_setfpregs) {
3434		COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_fpregs_struct_sz);
	3556	COMMON_INTERCEPTOR_READ_RANGE(ctx, data_arg,
	3557	struct_user_fpregs_struct_sz);
3435	3558	} else if (request == ptrace_setfpxregs) {
3436		COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_fpxregs_struct_sz);
	3559	COMMON_INTERCEPTOR_READ_RANGE(ctx, data_arg,
	3560	struct_user_fpxregs_struct_sz);
3437	3561	} else if (request == ptrace_setvfpregs) {
3438		COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_vfpregs_struct_sz);
	3562	COMMON_INTERCEPTOR_READ_RANGE(ctx, data_arg,
	3563	struct_user_vfpregs_struct_sz);
3439	3564	} else if (request == ptrace_setsiginfo) {
3440		COMMON_INTERCEPTOR_READ_RANGE(ctx, data, siginfo_t_sz);
	3565	COMMON_INTERCEPTOR_READ_RANGE(ctx, data_arg, siginfo_t_sz);
3441	3566
3442		// Some kernel might zero the iovec::iov_base in case of invalid
3443		// write access. In this case copy the invalid address for further
3444		// inspection.
	3567	// Some kernel might zero the iovec::iov_base in case of invalid
	3568	// write access. In this case copy the invalid address for further
	3569	// inspection.
3445	3570	} else if (request == ptrace_setregset \|\| request == ptrace_getregset) {
3446		__sanitizer_iovec iovec = (__sanitizer_iovec)data;
	3571	__sanitizer_iovec iovec = (__sanitizer_iovec )data_arg;
3447	3572	COMMON_INTERCEPTOR_READ_RANGE(ctx, iovec, sizeof(*iovec));
3448	3573	local_iovec = *iovec;
3449	3574	if (request == ptrace_setregset)
...	...	@@ -3456,23 +3581,26 @@ INTERCEPTOR(uptr, ptrace, int request, int pid, void addr, void data) {
3456	3581	// https://github.com/google/sanitizers/issues/321.
3457	3582	uptr res = REAL(ptrace)(request, pid, addr, data);
3458	3583
3459		if (!res && data) {
	3584	if (!res && data_arg) {
3460	3585	// Note that PEEK* requests assign different meaning to the return value.
3461	3586	// This function does not handle them (nor does it need to).
3462	3587	if (request == ptrace_getregs) {
3463		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_regs_struct_sz);
	3588	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg, struct_user_regs_struct_sz);
3464	3589	} else if (request == ptrace_getfpregs) {
3465		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_fpregs_struct_sz);
	3590	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg,
	3591	struct_user_fpregs_struct_sz);
3466	3592	} else if (request == ptrace_getfpxregs) {
3467		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_fpxregs_struct_sz);
	3593	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg,
	3594	struct_user_fpxregs_struct_sz);
3468	3595	} else if (request == ptrace_getvfpregs) {
3469		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_vfpregs_struct_sz);
	3596	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg,
	3597	struct_user_vfpregs_struct_sz);
3470	3598	} else if (request == ptrace_getsiginfo) {
3471		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, siginfo_t_sz);
	3599	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg, siginfo_t_sz);
3472	3600	} else if (request == ptrace_geteventmsg) {
3473		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, sizeof(unsigned long));
	3601	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data_arg, sizeof(unsigned long));
3474	3602	} else if (request == ptrace_getregset) {
3475		__sanitizer_iovec iovec = (__sanitizer_iovec)data;
	3603	__sanitizer_iovec iovec = (__sanitizer_iovec )data_arg;
3476	3604	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iovec, sizeof(*iovec));
3477	3605	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, local_iovec.iov_base,
3478	3606	local_iovec.iov_len);
...	...	@@ -6425,12 +6553,12 @@ static void MlockIsUnsupported() {
6425	6553	SanitizerToolName);
6426	6554	}
6427	6555
6428		INTERCEPTOR(int, mlock, const void *addr, uptr len) {
	6556	INTERCEPTOR(int, mlock, const void *addr, usize len) {
6429	6557	MlockIsUnsupported();
6430	6558	return 0;
6431	6559	}
6432	6560
6433		INTERCEPTOR(int, munlock, const void *addr, uptr len) {
	6561	INTERCEPTOR(int, munlock, const void *addr, usize len) {
6434	6562	MlockIsUnsupported();
6435	6563	return 0;
6436	6564	}
...	...	@@ -8823,83 +8951,6 @@ INTERCEPTOR(char , RMD160Data, u8 data, SIZE_T len, char *buf) {
8823	8951	#define INIT_RMD160
8824	8952	#endif
8825	8953
8826		#if SANITIZER_INTERCEPT_MD5
8827		INTERCEPTOR(void, MD5Init, void *context) {
8828		void *ctx;
8829		COMMON_INTERCEPTOR_ENTER(ctx, MD5Init, context);
8830		REAL(MD5Init)(context);
8831		if (context)
8832		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD5_CTX_sz);
8833		}
8834
8835		INTERCEPTOR(void, MD5Update, void context, const unsigned char data,
8836		unsigned int len) {
8837		void *ctx;
8838		COMMON_INTERCEPTOR_ENTER(ctx, MD5Update, context, data, len);
8839		if (data && len > 0)
8840		COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len);
8841		if (context)
8842		COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz);
8843		REAL(MD5Update)(context, data, len);
8844		if (context)
8845		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, MD5_CTX_sz);
8846		}
8847
8848		INTERCEPTOR(void, MD5Final, unsigned char digest[16], void *context) {
8849		void *ctx;
8850		COMMON_INTERCEPTOR_ENTER(ctx, MD5Final, digest, context);
8851		if (context)
8852		COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz);
8853		REAL(MD5Final)(digest, context);
8854		if (digest)
8855		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, sizeof(unsigned char) * 16);
8856		}
8857
8858		INTERCEPTOR(char , MD5End, void context, char *buf) {
8859		void *ctx;
8860		COMMON_INTERCEPTOR_ENTER(ctx, MD5End, context, buf);
8861		if (context)
8862		COMMON_INTERCEPTOR_READ_RANGE(ctx, context, MD5_CTX_sz);
8863		char *ret = REAL(MD5End)(context, buf);
8864		if (ret)
8865		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length);
8866		return ret;
8867		}
8868
8869		INTERCEPTOR(char , MD5File, const char filename, char *buf) {
8870		void *ctx;
8871		COMMON_INTERCEPTOR_ENTER(ctx, MD5File, filename, buf);
8872		if (filename)
8873		COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
8874		char *ret = REAL(MD5File)(filename, buf);
8875		if (ret)
8876		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length);
8877		return ret;
8878		}
8879
8880		INTERCEPTOR(char , MD5Data, const unsigned char data, unsigned int len,
8881		char *buf) {
8882		void *ctx;
8883		COMMON_INTERCEPTOR_ENTER(ctx, MD5Data, data, len, buf);
8884		if (data && len > 0)
8885		COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len);
8886		char *ret = REAL(MD5Data)(data, len, buf);
8887		if (ret)
8888		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length);
8889		return ret;
8890		}
8891
8892		#define INIT_MD5 \
8893		COMMON_INTERCEPT_FUNCTION(MD5Init); \
8894		COMMON_INTERCEPT_FUNCTION(MD5Update); \
8895		COMMON_INTERCEPT_FUNCTION(MD5Final); \
8896		COMMON_INTERCEPT_FUNCTION(MD5End); \
8897		COMMON_INTERCEPT_FUNCTION(MD5File); \
8898		COMMON_INTERCEPT_FUNCTION(MD5Data)
8899		#else
8900		#define INIT_MD5
8901		#endif
8902
8903	8954	#if SANITIZER_INTERCEPT_FSEEK
8904	8955	INTERCEPTOR(int, fseek, __sanitizer_FILE *stream, long int offset, int whence) {
8905	8956	void *ctx;
...	...	@@ -9030,107 +9081,6 @@ INTERCEPTOR(char , MD2Data, const unsigned char data, unsigned int len,
9030	9081	#define INIT_MD2
9031	9082	#endif
9032	9083
9033		#if SANITIZER_INTERCEPT_SHA2
9034		#define SHA2_INTERCEPTORS(LEN, SHA2_STATE_T) \
9035		INTERCEPTOR(void, SHA##LEN##_Init, void *context) { \
9036		void *ctx; \
9037		COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Init, context); \
9038		REAL(SHA##LEN##_Init)(context); \
9039		if (context) \
9040		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA##LEN##_CTX_sz); \
9041		} \
9042		INTERCEPTOR(void, SHA##LEN##_Update, void *context, \
9043		const u8 *data, SIZE_T len) { \
9044		void *ctx; \
9045		COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Update, context, data, len); \
9046		if (data && len > 0) \
9047		COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); \
9048		if (context) \
9049		COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \
9050		REAL(SHA##LEN##_Update)(context, data, len); \
9051		if (context) \
9052		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, context, SHA##LEN##_CTX_sz); \
9053		} \
9054		INTERCEPTOR(void, SHA##LEN##_Final, u8 digest[LEN/8], \
9055		void *context) { \
9056		void *ctx; \
9057		CHECK_EQ(SHA##LEN##_digest_length, LEN/8); \
9058		COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Final, digest, context); \
9059		if (context) \
9060		COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \
9061		REAL(SHA##LEN##_Final)(digest, context); \
9062		if (digest) \
9063		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, digest, \
9064		sizeof(digest[0]) * \
9065		SHA##LEN##_digest_length); \
9066		} \
9067		INTERCEPTOR(char , SHA##LEN##_End, void context, char *buf) { \
9068		void *ctx; \
9069		COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_End, context, buf); \
9070		if (context) \
9071		COMMON_INTERCEPTOR_READ_RANGE(ctx, context, SHA##LEN##_CTX_sz); \
9072		char *ret = REAL(SHA##LEN##_End)(context, buf); \
9073		if (ret) \
9074		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
9075		return ret; \
9076		} \
9077		INTERCEPTOR(char , SHA##LEN##_File, const char filename, char *buf) { \
9078		void *ctx; \
9079		COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_File, filename, buf); \
9080		if (filename) \
9081		COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);\
9082		char *ret = REAL(SHA##LEN##_File)(filename, buf); \
9083		if (ret) \
9084		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
9085		return ret; \
9086		} \
9087		INTERCEPTOR(char , SHA##LEN##_FileChunk, const char filename, char *buf, \
9088		OFF_T offset, OFF_T length) { \
9089		void *ctx; \
9090		COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_FileChunk, filename, buf, offset, \
9091		length); \
9092		if (filename) \
9093		COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);\
9094		char *ret = REAL(SHA##LEN##_FileChunk)(filename, buf, offset, length); \
9095		if (ret) \
9096		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
9097		return ret; \
9098		} \
9099		INTERCEPTOR(char , SHA##LEN##_Data, u8 data, SIZE_T len, char *buf) { \
9100		void *ctx; \
9101		COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_Data, data, len, buf); \
9102		if (data && len > 0) \
9103		COMMON_INTERCEPTOR_READ_RANGE(ctx, data, len); \
9104		char *ret = REAL(SHA##LEN##_Data)(data, len, buf); \
9105		if (ret) \
9106		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
9107		return ret; \
9108		}
9109
9110		SHA2_INTERCEPTORS(224, u32)
9111		SHA2_INTERCEPTORS(256, u32)
9112		SHA2_INTERCEPTORS(384, u64)
9113		SHA2_INTERCEPTORS(512, u64)
9114
9115		#define INIT_SHA2_INTECEPTORS(LEN) \
9116		COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Init); \
9117		COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Update); \
9118		COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Final); \
9119		COMMON_INTERCEPT_FUNCTION(SHA##LEN##_End); \
9120		COMMON_INTERCEPT_FUNCTION(SHA##LEN##_File); \
9121		COMMON_INTERCEPT_FUNCTION(SHA##LEN##_FileChunk); \
9122		COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Data)
9123
9124		#define INIT_SHA2 \
9125		INIT_SHA2_INTECEPTORS(224); \
9126		INIT_SHA2_INTECEPTORS(256); \
9127		INIT_SHA2_INTECEPTORS(384); \
9128		INIT_SHA2_INTECEPTORS(512)
9129		#undef SHA2_INTERCEPTORS
9130		#else
9131		#define INIT_SHA2
9132		#endif
9133
9134	9084	#if SANITIZER_INTERCEPT_VIS
9135	9085	INTERCEPTOR(char , vis, char dst, int c, int flag, int nextc) {
9136	9086	void *ctx;
...	...	@@ -9980,7 +9930,7 @@ INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) {
9980	9930	void *ctx;
9981	9931	COMMON_INTERCEPTOR_ENTER(ctx, getrandom, buf, buflen, flags);
9982	9932	// If GRND_NONBLOCK is set in the flags, it is non blocking.
9983		static const int grnd_nonblock = 1;
	9933	static const int grnd_nonblock = 1;
9984	9934	SSIZE_T n;
9985	9935	if ((flags & grnd_nonblock))
9986	9936	n = REAL(getrandom)(buf, buflen, flags);
...	...	@@ -10296,6 +10246,23 @@ INTERCEPTOR(SSIZE_T, pwritev2, int fd, __sanitizer_iovec *iov, int iovcnt,
10296	10246	#define INIT_PWRITEV2
10297	10247	#endif
10298	10248
	10249	#if SANITIZER_INTERCEPT_FREADLINK
	10250	INTERCEPTOR(SSIZE_T, freadlink, int fd, char *buf, SIZE_T bufsiz) {
	10251	void *ctx;
	10252	COMMON_INTERCEPTOR_ENTER(ctx, freadlink, fd, buf, bufsiz);
	10253	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
	10254	SSIZE_T res = REAL(freadlink)(fd, buf, bufsiz);
	10255	if (res > 0)
	10256	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res);
	10257	if (res >= 0 && fd > 0)
	10258	COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
	10259	return res;
	10260	}
	10261	# define INIT_FREADLINK COMMON_INTERCEPT_FUNCTION(freadlink)
	10262	#else
	10263	# define INIT_FREADLINK
	10264	#endif
	10265
10299	10266	#include "sanitizer_common_interceptors_netbsd_compat.inc"
10300	10267
10301	10268	namespace __sanitizer {
...	...	@@ -10373,8 +10340,10 @@ static void InitializeCommonInterceptors() {
10373	10340	INIT_SETPWENT;
10374	10341	INIT_CLOCK_GETTIME;
10375	10342	INIT_CLOCK_GETCPUCLOCKID;
	10343	INIT_TIMER_CREATE;
10376	10344	INIT_GETITIMER;
10377	10345	INIT_TIME;
	10346	INIT_TIMESPEC_GET;
10378	10347	INIT_GLOB;
10379	10348	INIT_GLOB64;
10380	10349	INIT___B64_TO;
...	...	@@ -10588,10 +10557,8 @@ static void InitializeCommonInterceptors() {
10588	10557	INIT_SHA1;
10589	10558	INIT_MD4;
10590	10559	INIT_RMD160;
10591		INIT_MD5;
10592	10560	INIT_FSEEK;
10593	10561	INIT_MD2;
10594		INIT_SHA2;
10595	10562	INIT_VIS;
10596	10563	INIT_CDB;
10597	10564	INIT_GETFSENT;
...	...	@@ -10617,6 +10584,7 @@ static void InitializeCommonInterceptors() {
10617	10584	INIT_CPUSET_GETAFFINITY;
10618	10585	INIT_PREADV2;
10619	10586	INIT_PWRITEV2;
	10587	INIT_FREADLINK;
10620	10588
10621	10589	INIT___PRINTF_CHK;
10622	10590	}

lib/tsan/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc+17-17

...	...	@@ -82,7 +82,7 @@
82	82	#endif
83	83
84	84	#if SANITIZER_INTERCEPT_MEMSET
85		INTERCEPTOR(void , memset, void dst, int v, uptr size) {
	85	INTERCEPTOR(void , memset, void dst, int v, usize size) {
86	86	void *ctx;
87	87	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size);
88	88	}
...	...	@@ -93,7 +93,7 @@ INTERCEPTOR(void , memset, void dst, int v, uptr size) {
93	93	#endif
94	94
95	95	#if SANITIZER_INTERCEPT_MEMMOVE
96		INTERCEPTOR(void , memmove, void dst, const void *src, uptr size) {
	96	INTERCEPTOR(void , memmove, void dst, const void *src, usize size) {
97	97	void *ctx;
98	98	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
99	99	}
...	...	@@ -104,7 +104,7 @@ INTERCEPTOR(void , memmove, void dst, const void *src, uptr size) {
104	104	#endif
105	105
106	106	#if SANITIZER_INTERCEPT_MEMCPY
107		INTERCEPTOR(void , memcpy, void dst, const void *src, uptr size) {
	107	INTERCEPTOR(void , memcpy, void dst, const void *src, usize size) {
108	108	// On OS X, calling internal_memcpy here will cause memory corruptions,
109	109	// because memcpy and memmove are actually aliases of the same
110	110	// implementation. We need to use internal_memmove here.
...	...	@@ -133,63 +133,63 @@ INTERCEPTOR(void , memcpy, void dst, const void *src, uptr size) {
133	133	#endif
134	134
135	135	#if SANITIZER_INTERCEPT_AEABI_MEM
136		INTERCEPTOR(void , __aeabi_memmove, void to, const void *from, uptr size) {
	136	INTERCEPTOR(void , __aeabi_memmove, void to, const void *from, usize size) {
137	137	void *ctx;
138	138	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
139	139	}
140	140
141		INTERCEPTOR(void , __aeabi_memmove4, void to, const void *from, uptr size) {
	141	INTERCEPTOR(void , __aeabi_memmove4, void to, const void *from, usize size) {
142	142	void *ctx;
143	143	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
144	144	}
145	145
146		INTERCEPTOR(void , __aeabi_memmove8, void to, const void *from, uptr size) {
	146	INTERCEPTOR(void , __aeabi_memmove8, void to, const void *from, usize size) {
147	147	void *ctx;
148	148	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
149	149	}
150	150
151		INTERCEPTOR(void , __aeabi_memcpy, void to, const void *from, uptr size) {
	151	INTERCEPTOR(void , __aeabi_memcpy, void to, const void *from, usize size) {
152	152	void *ctx;
153	153	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
154	154	}
155	155
156		INTERCEPTOR(void , __aeabi_memcpy4, void to, const void *from, uptr size) {
	156	INTERCEPTOR(void , __aeabi_memcpy4, void to, const void *from, usize size) {
157	157	void *ctx;
158	158	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
159	159	}
160	160
161		INTERCEPTOR(void , __aeabi_memcpy8, void to, const void *from, uptr size) {
	161	INTERCEPTOR(void , __aeabi_memcpy8, void to, const void *from, usize size) {
162	162	void *ctx;
163	163	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
164	164	}
165	165
166	166	// Note the argument order.
167		INTERCEPTOR(void , __aeabi_memset, void block, uptr size, int c) {
	167	INTERCEPTOR(void , __aeabi_memset, void block, usize size, int c) {
168	168	void *ctx;
169	169	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
170	170	}
171	171
172		INTERCEPTOR(void , __aeabi_memset4, void block, uptr size, int c) {
	172	INTERCEPTOR(void , __aeabi_memset4, void block, usize size, int c) {
173	173	void *ctx;
174	174	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
175	175	}
176	176
177		INTERCEPTOR(void , __aeabi_memset8, void block, uptr size, int c) {
	177	INTERCEPTOR(void , __aeabi_memset8, void block, usize size, int c) {
178	178	void *ctx;
179	179	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
180	180	}
181	181
182		INTERCEPTOR(void , __aeabi_memclr, void block, uptr size) {
	182	INTERCEPTOR(void , __aeabi_memclr, void block, usize size) {
183	183	void *ctx;
184	184	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
185	185	}
186	186
187		INTERCEPTOR(void , __aeabi_memclr4, void block, uptr size) {
	187	INTERCEPTOR(void , __aeabi_memclr4, void block, usize size) {
188	188	void *ctx;
189	189	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
190	190	}
191	191
192		INTERCEPTOR(void , __aeabi_memclr8, void block, uptr size) {
	192	INTERCEPTOR(void , __aeabi_memclr8, void block, usize size) {
193	193	void *ctx;
194	194	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
195	195	}
...	...	@@ -212,7 +212,7 @@ INTERCEPTOR(void , __aeabi_memclr8, void block, uptr size) {
212	212	#endif // SANITIZER_INTERCEPT_AEABI_MEM
213	213
214	214	#if SANITIZER_INTERCEPT___BZERO
215		INTERCEPTOR(void , __bzero, void block, uptr size) {
	215	INTERCEPTOR(void , __bzero, void block, usize size) {
216	216	void *ctx;
217	217	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
218	218	}
...	...	@@ -222,7 +222,7 @@ INTERCEPTOR(void , __bzero, void block, uptr size) {
222	222	#endif // SANITIZER_INTERCEPT___BZERO
223	223
224	224	#if SANITIZER_INTERCEPT_BZERO
225		INTERCEPTOR(void , bzero, void block, uptr size) {
	225	INTERCEPTOR(void , bzero, void block, usize size) {
226	226	void *ctx;
227	227	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
228	228	}

lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S created+48

...	...	@@ -0,0 +1,48 @@
	1	#if defined(__aarch64__) && defined(__linux__)
	2
	3	#include "sanitizer_common/sanitizer_asm.h"
	4	#include "builtins/assembly.h"
	5
	6	ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA)
	7
	8	.comm _ZN14__interception10real_vforkE,8,8
	9	.globl ASM_WRAPPER_NAME(vfork)
	10	ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
	11	ASM_WRAPPER_NAME(vfork):
	12	// Save x30 in the off-stack spill area.
	13	hint #25 // paciasp
	14	stp xzr, x30, [sp, #-16]!
	15	bl COMMON_INTERCEPTOR_SPILL_AREA
	16	ldp xzr, x30, [sp], 16
	17	str x30, [x0]
	18
	19	// Call real vfork. This may return twice. User code that runs between the first and the second return
	20	// may clobber the stack frame of the interceptor; that's why it does not have a frame.
	21	adrp x0, _ZN14__interception10real_vforkE
	22	ldr x0, [x0, :lo12:_ZN14__interception10real_vforkE]
	23	blr x0
	24
	25	stp x0, xzr, [sp, #-16]!
	26	cmp x0, #0
	27	b.eq .L_exit
	28
	29	// x0 != 0 => parent process. Clear stack shadow.
	30	add x0, sp, #16
	31	bl COMMON_INTERCEPTOR_HANDLE_VFORK
	32
	33	.L_exit:
	34	// Restore x30.
	35	bl COMMON_INTERCEPTOR_SPILL_AREA
	36	ldr x30, [x0]
	37	ldp x0, xzr, [sp], 16
	38	hint #29 // autiasp
	39
	40	ret
	41	ASM_SIZE(vfork)
	42
	43	ASM_INTERCEPTOR_TRAMPOLINE(vfork)
	44	ASM_TRAMPOLINE_ALIAS(vfork, vfork)
	45
	46	GNU_PROPERTY_BTI_PAC
	47
	48	#endif

lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S created+49

...	...	@@ -0,0 +1,49 @@
	1	#if defined(__arm__) && defined(__linux__)
	2
	3	#include "sanitizer_common/sanitizer_asm.h"
	4
	5	ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA)
	6
	7	.comm _ZN14__interception10real_vforkE,4,4
	8	.globl ASM_WRAPPER_NAME(vfork)
	9	ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
	10	ASM_WRAPPER_NAME(vfork):
	11	// Save LR in the off-stack spill area.
	12	push {r4, lr}
	13	bl COMMON_INTERCEPTOR_SPILL_AREA
	14	pop {r4, lr}
	15	str lr, [r0]
	16
	17	// Call real vfork. This may return twice. User code that runs between the first and the second return
	18	// may clobber the stack frame of the interceptor; that's why it does not have a frame.
	19	ldr r0, .LCPI0_0
	20	.LPC0_0:
	21	ldr r0, [pc, r0]
	22	mov lr, pc
	23	bx r0
	24
	25	push {r0, r4}
	26	cmp r0, #0
	27	beq .L_exit
	28
	29	// r0 != 0 => parent process. Clear stack shadow.
	30	add r0, sp, #8
	31	bl COMMON_INTERCEPTOR_HANDLE_VFORK
	32
	33	.L_exit:
	34	// Restore LR.
	35	bl COMMON_INTERCEPTOR_SPILL_AREA
	36	ldr lr, [r0]
	37	pop {r0, r4}
	38
	39	mov pc, lr
	40
	41	.LCPI0_0:
	42	.long _ZN14__interception10real_vforkE - (.LPC0_0+8)
	43
	44	ASM_SIZE(vfork)
	45
	46	ASM_INTERCEPTOR_TRAMPOLINE(vfork)
	47	ASM_TRAMPOLINE_ALIAS(vfork, vfork)
	48
	49	#endif

lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S created+64

...	...	@@ -0,0 +1,64 @@
	1	#if defined(__i386__) && defined(__linux__)
	2
	3	#include "sanitizer_common/sanitizer_asm.h"
	4
	5	.comm _ZN14__interception10real_vforkE,4,4
	6	.globl ASM_WRAPPER_NAME(vfork)
	7	ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
	8	ASM_WRAPPER_NAME(vfork):
	9	_CET_ENDBR
	10	// Store return address in the spill area and tear down the stack frame.
	11	sub $12, %esp
	12	call COMMON_INTERCEPTOR_SPILL_AREA
	13	mov 12(%esp), %ecx
	14	mov %ecx, (%eax)
	15	add $16, %esp
	16
	17	call .L0$pb
	18	.L0$pb:
	19	pop %eax
	20	.Ltmp0:
	21	add $_GLOBAL_OFFSET_TABLE_+(.Ltmp0-.L0$pb), %eax
	22	call *_ZN14__interception10real_vforkE@GOTOFF(%eax)
	23
	24	// Restore the stack frame.
	25	// 12(%esp) return address
	26	// 8(%esp) spill %ebx
	27	// 4(%esp) spill REAL(vfork) return value
	28	// (%esp) call frame (arg0) for __*_handle_vfork
	29	sub $16, %esp
	30	mov %ebx, 8(%esp)
	31	mov %eax, 4(%esp)
	32
	33	// Form GOT address in %ebx.
	34	call .L1$pb
	35	.L1$pb:
	36	pop %ebx
	37	.Ltmp1:
	38	add $_GLOBAL_OFFSET_TABLE_+(.Ltmp1-.L1$pb), %ebx
	39
	40	// Restore original return address.
	41	call COMMON_INTERCEPTOR_SPILL_AREA
	42	mov (%eax), %ecx
	43	mov %ecx, 12(%esp)
	44	mov 4(%esp), %eax
	45
	46	// Call handle_vfork in the parent process (%rax != 0).
	47	test %eax, %eax
	48	je .L_exit
	49
	50	lea 16(%esp), %ecx
	51	mov %ecx, (%esp)
	52	call COMMON_INTERCEPTOR_HANDLE_VFORK@PLT
	53
	54	.L_exit:
	55	mov 4(%esp), %eax
	56	mov 8(%esp), %ebx
	57	add $12, %esp
	58	ret
	59	ASM_SIZE(vfork)
	60
	61	ASM_INTERCEPTOR_TRAMPOLINE(vfork)
	62	ASM_TRAMPOLINE_ALIAS(vfork, vfork)
	63
	64	#endif

lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S created+57

...	...	@@ -0,0 +1,57 @@
	1	#if defined(__loongarch_lp64) && defined(__linux__)
	2
	3	#include "sanitizer_common/sanitizer_asm.h"
	4
	5	ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA)
	6	ASM_HIDDEN(_ZN14__interception10real_vforkE)
	7
	8	.text
	9	.globl ASM_WRAPPER_NAME(vfork)
	10	ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
	11	ASM_WRAPPER_NAME(vfork):
	12	// Save ra in the off-stack spill area.
	13	// allocate space on stack
	14	addi.d $sp, $sp, -16
	15	// store $ra value
	16	st.d $ra, $sp, 8
	17	bl COMMON_INTERCEPTOR_SPILL_AREA
	18	// restore previous values from stack
	19	ld.d $ra, $sp, 8
	20	// adjust stack
	21	addi.d $sp, $sp, 16
	22	// store $ra by $a0
	23	st.d $ra, $a0, 0
	24
	25	// Call real vfork. This may return twice. User code that runs between the first and the second return
	26	// may clobber the stack frame of the interceptor; that's why it does not have a frame.
	27	la.local $a0, _ZN14__interception10real_vforkE
	28	ld.d $a0, $a0, 0
	29	jirl $ra, $a0, 0
	30
	31	// adjust stack
	32	addi.d $sp, $sp, -16
	33	// store $a0 by adjusted stack
	34	st.d $a0, $sp, 8
	35	// jump to exit label if $a0 is 0
	36	beqz $a0, .L_exit
	37
	38	// $a0 != 0 => parent process. Clear stack shadow.
	39	// put old $sp to $a0
	40	addi.d $a0, $sp, 16
	41	bl %plt(COMMON_INTERCEPTOR_HANDLE_VFORK)
	42
	43	.L_exit:
	44	// Restore $ra
	45	bl COMMON_INTERCEPTOR_SPILL_AREA
	46	ld.d $ra, $a0, 0
	47	// load value by stack
	48	ld.d $a0, $sp, 8
	49	// adjust stack
	50	addi.d $sp, $sp, 16
	51	jr $ra
	52	ASM_SIZE(vfork)
	53
	54	ASM_INTERCEPTOR_TRAMPOLINE(vfork)
	55	ASM_TRAMPOLINE_ALIAS(vfork, vfork)
	56
	57	#endif

lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_riscv64.inc.S created+56

...	...	@@ -0,0 +1,56 @@
	1	#if (defined(__riscv) && (__riscv_xlen == 64)) && defined(__linux__)
	2
	3	#include "sanitizer_common/sanitizer_asm.h"
	4
	5	ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA)
	6
	7	.comm _ZN14__interception10real_vforkE,8,8
	8	.globl ASM_WRAPPER_NAME(vfork)
	9	ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
	10	ASM_WRAPPER_NAME(vfork):
	11	// Save ra in the off-stack spill area.
	12	// allocate space on stack
	13	addi sp, sp, -16
	14	// store ra value
	15	sd ra, 8(sp)
	16	call COMMON_INTERCEPTOR_SPILL_AREA
	17	// restore previous values from stack
	18	ld ra, 8(sp)
	19	// adjust stack
	20	addi sp, sp, 16
	21	// store ra by x10
	22	sd ra, 0(x10)
	23
	24	// Call real vfork. This may return twice. User code that runs between the first and the second return
	25	// may clobber the stack frame of the interceptor; that's why it does not have a frame.
	26	la x10, _ZN14__interception10real_vforkE
	27	ld x10, 0(x10)
	28	jalr x10
	29
	30	// adjust stack
	31	addi sp, sp, -16
	32	// store x10 by adjusted stack
	33	sd x10, 8(sp)
	34	// jump to exit label if x10 is 0
	35	beqz x10, .L_exit
	36
	37	// x0 != 0 => parent process. Clear stack shadow.
	38	// put old sp to x10
	39	addi x10, sp, 16
	40	call COMMON_INTERCEPTOR_HANDLE_VFORK
	41
	42	.L_exit:
	43	// Restore ra
	44	call COMMON_INTERCEPTOR_SPILL_AREA
	45	ld ra, 0(x10)
	46	// load value by stack
	47	ld x10, 8(sp)
	48	// adjust stack
	49	addi sp, sp, 16
	50	ret
	51	ASM_SIZE(vfork)
	52
	53	ASM_INTERCEPTOR_TRAMPOLINE(vfork)
	54	ASM_TRAMPOLINE_ALIAS(vfork, vfork)
	55
	56	#endif

lib/tsan/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S created+42

...	...	@@ -0,0 +1,42 @@
	1	#if defined(__x86_64__) && defined(__linux__)
	2
	3	#include "sanitizer_common/sanitizer_asm.h"
	4
	5	.comm _ZN14__interception10real_vforkE,8,8
	6	.globl ASM_WRAPPER_NAME(vfork)
	7	ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork))
	8	ASM_WRAPPER_NAME(vfork):
	9	_CET_ENDBR
	10	// Store return address in the spill area and tear down the stack frame.
	11	push %rcx
	12	call COMMON_INTERCEPTOR_SPILL_AREA
	13	pop %rcx
	14	pop %rdi
	15	mov %rdi, (%rax)
	16
	17	call *_ZN14__interception10real_vforkE(%rip)
	18
	19	// Restore return address from the spill area.
	20	push %rcx
	21	push %rax
	22	call COMMON_INTERCEPTOR_SPILL_AREA
	23	mov (%rax), %rdx
	24	mov %rdx, 8(%rsp)
	25	mov (%rsp), %rax
	26
	27	// Call handle_vfork in the parent process (%rax != 0).
	28	test %rax, %rax
	29	je .L_exit
	30
	31	lea 16(%rsp), %rdi
	32	call COMMON_INTERCEPTOR_HANDLE_VFORK@PLT
	33
	34	.L_exit:
	35	pop %rax
	36	ret
	37	ASM_SIZE(ASM_WRAPPER_NAME(vfork))
	38
	39	ASM_INTERCEPTOR_TRAMPOLINE(vfork)
	40	ASM_TRAMPOLINE_ALIAS(vfork, vfork)
	41
	42	#endif

lib/tsan/sanitizer_common/sanitizer_common_interface.inc+8

...	...	@@ -10,6 +10,7 @@
10	10	INTERFACE_FUNCTION(__sanitizer_acquire_crash_state)
11	11	INTERFACE_FUNCTION(__sanitizer_annotate_contiguous_container)
12	12	INTERFACE_FUNCTION(__sanitizer_annotate_double_ended_contiguous_container)
	13	INTERFACE_FUNCTION(__sanitizer_copy_contiguous_container_annotations)
13	14	INTERFACE_FUNCTION(__sanitizer_contiguous_container_find_bad_address)
14	15	INTERFACE_FUNCTION(
15	16	__sanitizer_double_ended_contiguous_container_find_bad_address)
...	...	@@ -22,6 +23,7 @@ INTERFACE_FUNCTION(__sanitizer_verify_double_ended_contiguous_container)
22	23	INTERFACE_WEAK_FUNCTION(__sanitizer_on_print)
23	24	INTERFACE_WEAK_FUNCTION(__sanitizer_report_error_summary)
24	25	INTERFACE_WEAK_FUNCTION(__sanitizer_sandbox_on_notify)
	26	INTERFACE_WEAK_FUNCTION(__sanitizer_get_dtls_size)
25	27	// Sanitizer weak hooks
26	28	INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_memcmp)
27	29	INTERFACE_WEAK_FUNCTION(__sanitizer_weak_hook_strcmp)
...	...	@@ -51,3 +53,9 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_ignore_free_hook)
51	53	INTERFACE_FUNCTION(__sanitizer_internal_memcpy)
52	54	INTERFACE_FUNCTION(__sanitizer_internal_memmove)
53	55	INTERFACE_FUNCTION(__sanitizer_internal_memset)
	56
	57	#if SANITIZER_WINDOWS
	58	INTERFACE_FUNCTION(__sanitizer_override_function)
	59	INTERFACE_FUNCTION(__sanitizer_override_function_by_addr)
	60	INTERFACE_FUNCTION(__sanitizer_register_weak_function)
	61	#endif

lib/tsan/sanitizer_common/sanitizer_common_libcdep.cpp+27-1

...	...	@@ -171,7 +171,7 @@ void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name,
171	171	"ReserveShadowMemoryRange failed while trying to map 0x%zx bytes. "
172	172	"Perhaps you're using ulimit -v or ulimit -d\n",
173	173	size);
174		Abort();
	174	Die();
175	175	}
176	176	if (madvise_shadow && common_flags()->use_madv_dontdump)
177	177	DontDumpShadowMemory(beg, size);
...	...	@@ -219,6 +219,32 @@ static void StopStackDepotBackgroundThread() {
219	219	static void StopStackDepotBackgroundThread() {}
220	220	#endif
221	221
	222	void MemCpyAccessible(void dest, const void src, uptr n) {
	223	if (TryMemCpy(dest, src, n))
	224	return;
	225
	226	const uptr page_size = GetPageSize();
	227	uptr b = reinterpret_cast<uptr>(src);
	228	uptr b_up = RoundUpTo(b, page_size);
	229
	230	uptr e = reinterpret_cast<uptr>(src) + n;
	231	uptr e_down = RoundDownTo(e, page_size);
	232
	233	auto copy_or_zero = [dest, src](uptr beg, uptr end) {
	234	const uptr udest = reinterpret_cast<uptr>(dest);
	235	const uptr usrc = reinterpret_cast<uptr>(src);
	236	void d = reinterpret_cast<void >(udest + (beg - usrc));
	237	const uptr size = end - beg;
	238	if (!TryMemCpy(d, reinterpret_cast<void *>(beg), size))
	239	internal_memset(d, 0, size);
	240	};
	241
	242	copy_or_zero(b, b_up);
	243	for (uptr p = b_up; p < e_down; p += page_size)
	244	copy_or_zero(p, p + page_size);
	245	copy_or_zero(e_down, e);
	246	}
	247
222	248	} // namespace __sanitizer
223	249
224	250	SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify,

lib/tsan/sanitizer_common/sanitizer_common_nolibc.cpp+4-3

...	...	@@ -20,13 +20,14 @@ namespace __sanitizer {
20	20	// The Windows implementations of these functions use the win32 API directly,
21	21	// bypassing libc.
22	22	#if !SANITIZER_WINDOWS
23		#if SANITIZER_LINUX
	23	# if SANITIZER_LINUX
24	24	void LogMessageOnPrintf(const char *str) {}
25		#endif
	25	void InitTlsSize() {}
	26	# endif
26	27	void WriteToSyslog(const char *buffer) {}
27	28	void Abort() { internal__exit(1); }
28	29	bool CreateDir(const char *pathname) { return false; }
29		#endif // !SANITIZER_WINDOWS
	30	#endif // !SANITIZER_WINDOWS
30	31
31	32	#if !SANITIZER_WINDOWS && !SANITIZER_APPLE
32	33	void ListOfModules::init() {}

lib/tsan/sanitizer_common/sanitizer_common_syscalls.inc+18-15

...	...	@@ -48,6 +48,7 @@
48	48	#if SANITIZER_LINUX
49	49
50	50	# include "sanitizer_libc.h"
	51	# include "sanitizer_platform_limits_posix.h"
51	52
52	53	# define PRE_SYSCALL(name) \
53	54	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_pre_impl_##name
...	...	@@ -2530,18 +2531,19 @@ PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) {
2530	2531	# if !SANITIZER_ANDROID && \
2531	2532	(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \
2532	2533	defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__s390__) \|\| \
2533		defined(__loongarch__) \|\| SANITIZER_RISCV64)
2534		if (data) {
	2534	defined(__loongarch__) \|\| SANITIZER_RISCV64 \|\| defined(__sparc__))
	2535	long data_arg = ptrace_data_arg(request, addr, data);
	2536	if (data_arg) {
2535	2537	if (request == ptrace_setregs) {
2536		PRE_READ((void *)data, struct_user_regs_struct_sz);
	2538	PRE_READ((void *)data_arg, struct_user_regs_struct_sz);
2537	2539	} else if (request == ptrace_setfpregs) {
2538		PRE_READ((void *)data, struct_user_fpregs_struct_sz);
	2540	PRE_READ((void *)data_arg, struct_user_fpregs_struct_sz);
2539	2541	} else if (request == ptrace_setfpxregs) {
2540		PRE_READ((void *)data, struct_user_fpxregs_struct_sz);
	2542	PRE_READ((void *)data_arg, struct_user_fpxregs_struct_sz);
2541	2543	} else if (request == ptrace_setsiginfo) {
2542		PRE_READ((void *)data, siginfo_t_sz);
	2544	PRE_READ((void *)data_arg, siginfo_t_sz);
2543	2545	} else if (request == ptrace_setregset) {
2544		__sanitizer_iovec iov = (__sanitizer_iovec )data;
	2546	__sanitizer_iovec iov = (__sanitizer_iovec )data_arg;
2545	2547	PRE_READ(iov->iov_base, iov->iov_len);
2546	2548	}
2547	2549	}
...	...	@@ -2552,25 +2554,26 @@ POST_SYSCALL(ptrace)(long res, long request, long pid, long addr, long data) {
2552	2554	# if !SANITIZER_ANDROID && \
2553	2555	(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \
2554	2556	defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__s390__) \|\| \
2555		defined(__loongarch__) \|\| SANITIZER_RISCV64)
2556		if (res >= 0 && data) {
	2557	defined(__loongarch__) \|\| SANITIZER_RISCV64 \|\| defined(__sparc__))
	2558	long data_arg = ptrace_data_arg(request, addr, data);
	2559	if (res >= 0 && data_arg) {
2557	2560	// Note that this is different from the interceptor in
2558	2561	// sanitizer_common_interceptors.inc.
2559	2562	// PEEK* requests return resulting values through data pointer.
2560	2563	if (request == ptrace_getregs) {
2561		POST_WRITE((void *)data, struct_user_regs_struct_sz);
	2564	POST_WRITE((void *)data_arg, struct_user_regs_struct_sz);
2562	2565	} else if (request == ptrace_getfpregs) {
2563		POST_WRITE((void *)data, struct_user_fpregs_struct_sz);
	2566	POST_WRITE((void *)data_arg, struct_user_fpregs_struct_sz);
2564	2567	} else if (request == ptrace_getfpxregs) {
2565		POST_WRITE((void *)data, struct_user_fpxregs_struct_sz);
	2568	POST_WRITE((void *)data_arg, struct_user_fpxregs_struct_sz);
2566	2569	} else if (request == ptrace_getsiginfo) {
2567		POST_WRITE((void *)data, siginfo_t_sz);
	2570	POST_WRITE((void *)data_arg, siginfo_t_sz);
2568	2571	} else if (request == ptrace_getregset) {
2569		__sanitizer_iovec iov = (__sanitizer_iovec )data;
	2572	__sanitizer_iovec iov = (__sanitizer_iovec )data_arg;
2570	2573	POST_WRITE(iov->iov_base, iov->iov_len);
2571	2574	} else if (request == ptrace_peekdata \|\| request == ptrace_peektext \|\|
2572	2575	request == ptrace_peekuser) {
2573		POST_WRITE((void )data, sizeof(void ));
	2576	POST_WRITE((void )data_arg, sizeof(void ));
2574	2577	}
2575	2578	}
2576	2579	# endif

lib/tsan/sanitizer_common/sanitizer_coverage_win_dll_thunk.cpp deleted-20

...	...	@@ -1,20 +0,0 @@
1		//===-- sanitizer_coverage_win_dll_thunk.cpp ------------------------------===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// This file defines a family of thunks that should be statically linked into
10		// the DLLs that have instrumentation in order to delegate the calls to the
11		// shared runtime that lives in the main binary.
12		// See https://github.com/google/sanitizers/issues/209 for the details.
13		//===----------------------------------------------------------------------===//
14		#ifdef SANITIZER_DLL_THUNK
15		#include "sanitizer_win_dll_thunk.h"
16		// Sanitizer Coverage interface functions.
17		#define INTERFACE_FUNCTION(Name) INTERCEPT_SANITIZER_FUNCTION(Name)
18		#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
19		#include "sanitizer_coverage_interface.inc"
20		#endif // SANITIZER_DLL_THUNK

lib/tsan/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cpp deleted-26

...	...	@@ -1,26 +0,0 @@
1		//===-- sanitizer_coverage_win_dynamic_runtime_thunk.cpp ------------------===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// This file defines things that need to be present in the application modules
10		// to interact with Sanitizer Coverage, when it is included in a dll.
11		//
12		//===----------------------------------------------------------------------===//
13		#ifdef SANITIZER_DYNAMIC_RUNTIME_THUNK
14		#define SANITIZER_IMPORT_INTERFACE 1
15		#include "sanitizer_win_defs.h"
16		// Define weak alias for all weak functions imported from sanitizer coverage.
17		#define INTERFACE_FUNCTION(Name)
18		#define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name)
19		#include "sanitizer_coverage_interface.inc"
20		#endif // SANITIZER_DYNAMIC_RUNTIME_THUNK
21
22		namespace __sanitizer {
23		// Add one, otherwise unused, external symbol to this object file so that the
24		// Visual C++ linker includes it and reads the .drective section.
25		void ForceWholeArchiveIncludeForSanCov() {}
26		}

lib/tsan/sanitizer_common/sanitizer_coverage_win_weak_interception.cpp deleted-23

...	...	@@ -1,23 +0,0 @@
1		//===-- sanitizer_coverage_win_weak_interception.cpp ----------------------===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		// This module should be included in Sanitizer Coverage when it implemented as a
9		// shared library on Windows (dll), in order to delegate the calls of weak
10		// functions to the implementation in the main executable when a strong
11		// definition is provided.
12		//===----------------------------------------------------------------------===//
13		#ifdef SANITIZER_DYNAMIC
14		#include "sanitizer_win_weak_interception.h"
15		#include "sanitizer_interface_internal.h"
16		#include "sancov_flags.h"
17		// Check if strong definitions for weak functions are present in the main
18		// executable. If that is the case, override dll functions to point to strong
19		// implementations.
20		#define INTERFACE_FUNCTION(Name)
21		#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
22		#include "sanitizer_coverage_interface.inc"
23		#endif // SANITIZER_DYNAMIC

lib/tsan/sanitizer_common/sanitizer_deadlock_detector.h+1-1

...	...	@@ -120,7 +120,7 @@ class DeadlockDetectorTLS {
120	120	u32 lock;
121	121	u32 stk;
122	122	};
123		LockWithContext all_locks_with_contexts_[64];
	123	LockWithContext all_locks_with_contexts_[128];
124	124	uptr n_all_locks_;
125	125	};
126	126

lib/tsan/sanitizer_common/sanitizer_dense_map.h+41-29

...	...	@@ -69,24 +69,14 @@ class DenseMapBase {
69	69	setNumTombstones(0);
70	70	}
71	71
	72	/// Return true if the specified key is in the map, false otherwise.
	73	bool contains(const KeyT &Key) const { return doFind(Key) != nullptr; }
	74
72	75	/// Return 1 if the specified key is in the map, 0 otherwise.
73		size_type count(const KeyT &Key) const {
74		const BucketT *TheBucket;
75		return LookupBucketFor(Key, TheBucket) ? 1 : 0;
76		}
	76	size_type count(const KeyT &Key) const { return contains(Key) ? 1 : 0; }
77	77
78		value_type *find(const KeyT &Key) {
79		BucketT *TheBucket;
80		if (LookupBucketFor(Key, TheBucket))
81		return TheBucket;
82		return nullptr;
83		}
84		const value_type *find(const KeyT &Key) const {
85		const BucketT *TheBucket;
86		if (LookupBucketFor(Key, TheBucket))
87		return TheBucket;
88		return nullptr;
89		}
	78	value_type *find(const KeyT &Key) { return doFind(Key); }
	79	const value_type *find(const KeyT &Key) const { return doFind(Key); }
90	80
91	81	/// Alternate version of find() which allows a different, and possibly
92	82	/// less expensive, key type.
...	...	@@ -95,25 +85,18 @@ class DenseMapBase {
95	85	/// type used.
96	86	template <class LookupKeyT>
97	87	value_type *find_as(const LookupKeyT &Key) {
98		BucketT *TheBucket;
99		if (LookupBucketFor(Key, TheBucket))
100		return TheBucket;
101		return nullptr;
	88	return doFind(Key);
102	89	}
103	90	template <class LookupKeyT>
104	91	const value_type *find_as(const LookupKeyT &Key) const {
105		const BucketT *TheBucket;
106		if (LookupBucketFor(Key, TheBucket))
107		return TheBucket;
108		return nullptr;
	92	return doFind(Key);
109	93	}
110	94
111	95	/// lookup - Return the entry for the specified key, or a default
112	96	/// constructed value if no such entry exists.
113	97	ValueT lookup(const KeyT &Key) const {
114		const BucketT *TheBucket;
115		if (LookupBucketFor(Key, TheBucket))
116		return TheBucket->getSecond();
	98	if (const BucketT *Bucket = doFind(Key))
	99	return Bucket->getSecond();
117	100	return ValueT();
118	101	}
119	102
...	...	@@ -184,8 +167,8 @@ class DenseMapBase {
184	167	}
185	168
186	169	bool erase(const KeyT &Val) {
187		BucketT *TheBucket;
188		if (!LookupBucketFor(Val, TheBucket))
	170	BucketT *TheBucket = doFind(Val);
	171	if (!TheBucket)
189	172	return false; // not in map.
190	173
191	174	TheBucket->getSecond().~ValueT();
...	...	@@ -449,6 +432,35 @@ class DenseMapBase {
449	432	return TheBucket;
450	433	}
451	434
	435	template <typename LookupKeyT>
	436	BucketT *doFind(const LookupKeyT &Val) {
	437	BucketT *BucketsPtr = getBuckets();
	438	const unsigned NumBuckets = getNumBuckets();
	439	if (NumBuckets == 0)
	440	return nullptr;
	441
	442	const KeyT EmptyKey = getEmptyKey();
	443	unsigned BucketNo = getHashValue(Val) & (NumBuckets - 1);
	444	unsigned ProbeAmt = 1;
	445	while (true) {
	446	BucketT *Bucket = BucketsPtr + BucketNo;
	447	if (LIKELY(KeyInfoT::isEqual(Val, Bucket->getFirst())))
	448	return Bucket;
	449	if (LIKELY(KeyInfoT::isEqual(Bucket->getFirst(), EmptyKey)))
	450	return nullptr;
	451
	452	// Otherwise, it's a hash collision or a tombstone, continue quadratic
	453	// probing.
	454	BucketNo += ProbeAmt++;
	455	BucketNo &= NumBuckets - 1;
	456	}
	457	}
	458
	459	template <typename LookupKeyT>
	460	const BucketT *doFind(const LookupKeyT &Val) const {
	461	return const_cast<DenseMapBase *>(this)->doFind(Val);
	462	}
	463
452	464	/// LookupBucketFor - Lookup the appropriate bucket for Val, returning it in
453	465	/// FoundBucket. If the bucket contains the key and a value, this returns
454	466	/// true, otherwise it returns a bucket with an empty marker or tombstone and

lib/tsan/sanitizer_common/sanitizer_errno.cpp+1

...	...	@@ -23,6 +23,7 @@ namespace __sanitizer {
23	23	COMPILER_CHECK(errno_ENOMEM == ENOMEM);
24	24	COMPILER_CHECK(errno_EBUSY == EBUSY);
25	25	COMPILER_CHECK(errno_EINVAL == EINVAL);
	26	COMPILER_CHECK(errno_ERANGE == ERANGE);
26	27
27	28	// EOWNERDEAD is not present in some older platforms.
28	29	#if defined(EOWNERDEAD)

lib/tsan/sanitizer_common/sanitizer_errno_codes.h+1

...	...	@@ -24,6 +24,7 @@ namespace __sanitizer {
24	24	#define errno_ENOMEM 12
25	25	#define errno_EBUSY 16
26	26	#define errno_EINVAL 22
	27	#define errno_ERANGE 34
27	28	#define errno_ENAMETOOLONG 36
28	29	#define errno_ENOSYS 38
29	30

lib/tsan/sanitizer_common/sanitizer_fuchsia.cpp+5-2

...	...	@@ -94,7 +94,6 @@ void DisableCoreDumperIfNecessary() {}
94	94	void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
95	95	void SetAlternateSignalStack() {}
96	96	void UnsetAlternateSignalStack() {}
97		void InitTlsSize() {}
98	97
99	98	bool SignalContext::IsStackOverflow() const { return false; }
100	99	void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); }
...	...	@@ -445,6 +444,11 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) {
445	444	return status == ZX_OK;
446	445	}
447	446
	447	bool TryMemCpy(void dest, const void src, uptr n) {
	448	// TODO: implement.
	449	return false;
	450	}
	451
448	452	// FIXME implement on this platform.
449	453	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
450	454
...	...	@@ -519,7 +523,6 @@ uptr ReadLongProcessName(/out/ char *buf, uptr buf_len) {
519	523	uptr MainThreadStackBase, MainThreadStackSize;
520	524
521	525	bool GetRandom(void *buffer, uptr length, bool blocking) {
522		CHECK_LE(length, ZX_CPRNG_DRAW_MAX_LEN);
523	526	_zx_cprng_draw(buffer, length);
524	527	return true;
525	528	}

lib/tsan/sanitizer_common/sanitizer_getauxval.h+11-12

...	...	@@ -21,22 +21,21 @@
21	21
22	22	#if SANITIZER_LINUX \|\| SANITIZER_FUCHSIA
23	23
24		# if (__GLIBC_PREREQ(2, 16) \|\| (SANITIZER_ANDROID && __ANDROID_API__ >= 21) \|\| \
25		SANITIZER_FUCHSIA) && \
26		!SANITIZER_GO
27		# define SANITIZER_USE_GETAUXVAL 1
28		# else
29		# define SANITIZER_USE_GETAUXVAL 0
30		# endif
31
32		# if SANITIZER_USE_GETAUXVAL
33		# include <sys/auxv.h>
34		# else
	24	# if (__GLIBC_PREREQ(2, 16) \|\| SANITIZER_ANDROID \|\| SANITIZER_FUCHSIA) && \
	25	!SANITIZER_GO
	26	# define SANITIZER_USE_GETAUXVAL 1
	27	# else
	28	# define SANITIZER_USE_GETAUXVAL 0
	29	# endif
	30
	31	# if SANITIZER_USE_GETAUXVAL
	32	# include <sys/auxv.h>
	33	# else
35	34	// The weak getauxval definition allows to check for the function at runtime.
36	35	// This is useful for Android, when compiled at a lower API level yet running
37	36	// on a more recent platform that offers the function.
38	37	extern "C" SANITIZER_WEAK_ATTRIBUTE unsigned long getauxval(unsigned long type);
39		# endif
	38	# endif
40	39
41	40	#elif SANITIZER_NETBSD
42	41

lib/tsan/sanitizer_common/sanitizer_interface_internal.h+10

...	...	@@ -49,6 +49,11 @@ __sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args);
49	49	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
50	50	__sanitizer_report_error_summary(const char *error_summary);
51	51
	52	// Returns size of dynamically allocated block. This function can be overridden
	53	// by the client.
	54	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE __sanitizer::uptr
	55	__sanitizer_get_dtls_size(const void *tls_begin);
	56
52	57	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump();
53	58	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage(
54	59	const __sanitizer::uptr *pcs, const __sanitizer::uptr len);
...	...	@@ -71,6 +76,11 @@ void __sanitizer_annotate_double_ended_contiguous_container(
71	76	const void old_container_beg, const void old_container_end,
72	77	const void new_container_beg, const void new_container_end);
73	78	SANITIZER_INTERFACE_ATTRIBUTE
	79	void __sanitizer_copy_contiguous_container_annotations(const void *src_begin,
	80	const void *src_end,
	81	const void *dst_begin,
	82	const void *dst_end);
	83	SANITIZER_INTERFACE_ATTRIBUTE
74	84	int __sanitizer_verify_contiguous_container(const void beg, const void mid,
75	85	const void *end);
76	86	SANITIZER_INTERFACE_ATTRIBUTE

lib/tsan/sanitizer_common/sanitizer_internal_defs.h+23-17

...	...	@@ -138,19 +138,25 @@
138	138	// in a portable way by the language itself.
139	139	namespace __sanitizer {
140	140
141		#if defined(_WIN64)
	141	#if defined(__UINTPTR_TYPE__)
	142	# if defined(__arm__) && defined(__linux__)
	143	// Linux Arm headers redefine __UINTPTR_TYPE__ and disagree with clang/gcc.
	144	typedef unsigned int uptr;
	145	typedef int sptr;
	146	# else
	147	typedef __UINTPTR_TYPE__ uptr;
	148	typedef __INTPTR_TYPE__ sptr;
	149	# endif
	150	#elif defined(_WIN64)
142	151	// 64-bit Windows uses LLP64 data model.
143	152	typedef unsigned long long uptr;
144	153	typedef signed long long sptr;
145		#else
146		# if (SANITIZER_WORDSIZE == 64) \|\| SANITIZER_APPLE \|\| SANITIZER_WINDOWS
147		typedef unsigned long uptr;
148		typedef signed long sptr;
149		# else
	154	#elif defined(_WIN32)
150	155	typedef unsigned int uptr;
151	156	typedef signed int sptr;
152		# endif
153		#endif // defined(_WIN64)
	157	#else
	158	# error Unsupported compiler, missing __UINTPTR_TYPE__
	159	#endif // defined(__UINTPTR_TYPE__)
154	160	#if defined(__x86_64__)
155	161	// Since x32 uses ILP32 data model in 64-bit hardware mode, we must use
156	162	// 64-bit pointer to unwind stack frame.
...	...	@@ -194,16 +200,13 @@ typedef u64 OFF64_T;
194	200	#ifdef __SIZE_TYPE__
195	201	typedef __SIZE_TYPE__ usize;
196	202	#else
197		// Since we use this for operator new, usize must match the real size_t, but on
198		// 32-bit Windows the definition of uptr does not actually match uintptr_t or
199		// size_t because we are working around typedef mismatches for the (S)SIZE_T
200		// types used in interception.h.
201		// Until the definition of uptr has been fixed we have to special case Win32.
202		# if SANITIZER_WINDOWS && SANITIZER_WORDSIZE == 32
203		typedef unsigned int usize;
204		# else
205	203	typedef uptr usize;
206		# endif
	204	#endif
	205
	206	#if defined(__s390__) && !defined(__s390x__)
	207	typedef long ssize;
	208	#else
	209	typedef sptr ssize;
207	210	#endif
208	211
209	212	typedef u64 tid_t;
...	...	@@ -466,6 +469,9 @@ using namespace __sanitizer;
466	469	namespace __msan {
467	470	using namespace __sanitizer;
468	471	}
	472	namespace __nsan {
	473	using namespace __sanitizer;
	474	}
469	475	namespace __hwasan {
470	476	using namespace __sanitizer;
471	477	}

lib/tsan/sanitizer_common/sanitizer_libignore.cpp+15-13

...	...	@@ -32,7 +32,7 @@ void LibIgnore::AddIgnoredLibrary(const char *name_templ) {
32	32	lib->templ = internal_strdup(name_templ);
33	33	lib->name = nullptr;
34	34	lib->real_name = nullptr;
35		lib->loaded = false;
	35	lib->range_id = kInvalidCodeRangeId;
36	36	}
37	37
38	38	void LibIgnore::OnLibraryLoaded(const char *name) {
...	...	@@ -43,7 +43,7 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
43	43	buf[0]) {
44	44	for (uptr i = 0; i < count_; i++) {
45	45	Lib *lib = &libs_[i];
46		if (!lib->loaded && (!lib->real_name) &&
	46	if (!lib->loaded() && (!lib->real_name) &&
47	47	TemplateMatch(lib->templ, name))
48	48	lib->real_name = internal_strdup(buf.data());
49	49	}
...	...	@@ -70,28 +70,31 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
70	70	Die();
71	71	}
72	72	loaded = true;
73		if (lib->loaded)
	73	if (lib->loaded())
74	74	continue;
75	75	VReport(1,
76	76	"Matched called_from_lib suppression '%s' against library"
77	77	" '%s'\n",
78	78	lib->templ, mod.full_name());
79		lib->loaded = true;
80	79	lib->name = internal_strdup(mod.full_name());
81	80	const uptr idx =
82	81	atomic_load(&ignored_ranges_count_, memory_order_relaxed);
83	82	CHECK_LT(idx, ARRAY_SIZE(ignored_code_ranges_));
84		ignored_code_ranges_[idx].begin = range.beg;
85		ignored_code_ranges_[idx].end = range.end;
	83	ignored_code_ranges_[idx].OnLoad(range.beg, range.end);
	84	// Record the index of the ignored range.
	85	lib->range_id = idx;
86	86	atomic_store(&ignored_ranges_count_, idx + 1, memory_order_release);
87	87	break;
88	88	}
89	89	}
90		if (lib->loaded && !loaded) {
91		Report("%s: library '%s' that was matched against called_from_lib"
92		" suppression '%s' is unloaded\n",
93		SanitizerToolName, lib->name, lib->templ);
94		Die();
	90	if (lib->loaded() && !loaded) {
	91	VReport(1,
	92	"%s: library '%s' that was matched against called_from_lib"
	93	" suppression '%s' is unloaded\n",
	94	SanitizerToolName, lib->name, lib->templ);
	95	// The library is unloaded so mark the ignored code range as unloaded.
	96	ignored_code_ranges_[lib->range_id].OnUnload();
	97	lib->range_id = kInvalidCodeRangeId;
95	98	}
96	99	}
97	100
...	...	@@ -110,8 +113,7 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
110	113	const uptr idx =
111	114	atomic_load(&instrumented_ranges_count_, memory_order_relaxed);
112	115	CHECK_LT(idx, ARRAY_SIZE(instrumented_code_ranges_));
113		instrumented_code_ranges_[idx].begin = range.beg;
114		instrumented_code_ranges_[idx].end = range.end;
	116	instrumented_code_ranges_[idx].OnLoad(range.beg, range.end);
115	117	atomic_store(&instrumented_ranges_count_, idx + 1,
116	118	memory_order_release);
117	119	}

lib/tsan/sanitizer_common/sanitizer_libignore.h+23-12

...	...	@@ -49,25 +49,36 @@ class LibIgnore {
49	49	bool IsPcInstrumented(uptr pc) const;
50	50
51	51	private:
	52	static const uptr kMaxIgnoredRanges = 128;
	53	static const uptr kMaxInstrumentedRanges = 1024;
	54	static const uptr kMaxLibs = 1024;
	55	static const uptr kInvalidCodeRangeId = -1;
	56
52	57	struct Lib {
53	58	char *templ;
54	59	char *name;
55	60	char *real_name; // target of symlink
56		bool loaded;
	61	uptr range_id;
	62	bool loaded() const { return range_id != kInvalidCodeRangeId; };
57	63	};
58	64
59	65	struct LibCodeRange {
60		uptr begin;
61		uptr end;
62		};
	66	bool IsInRange(uptr pc) const {
	67	return (pc >= begin && pc < atomic_load(&end, memory_order_acquire));
	68	}
63	69
64		inline bool IsInRange(uptr pc, const LibCodeRange &range) const {
65		return (pc >= range.begin && pc < range.end);
66		}
	70	void OnLoad(uptr b, uptr e) {
	71	begin = b;
	72	atomic_store(&end, e, memory_order_release);
	73	}
67	74
68		static const uptr kMaxIgnoredRanges = 128;
69		static const uptr kMaxInstrumentedRanges = 1024;
70		static const uptr kMaxLibs = 1024;
	75	void OnUnload() { atomic_store(&end, 0, memory_order_release); }
	76
	77	private:
	78	uptr begin;
	79	// A value of 0 means the associated module was unloaded.
	80	atomic_uintptr_t end;
	81	};
71	82
72	83	// Hot part:
73	84	atomic_uintptr_t ignored_ranges_count_;
...	...	@@ -90,7 +101,7 @@ class LibIgnore {
90	101	inline bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const {
91	102	const uptr n = atomic_load(&ignored_ranges_count_, memory_order_acquire);
92	103	for (uptr i = 0; i < n; i++) {
93		if (IsInRange(pc, ignored_code_ranges_[i])) {
	104	if (ignored_code_ranges_[i].IsInRange(pc)) {
94	105	*pc_in_ignored_lib = true;
95	106	return true;
96	107	}
...	...	@@ -104,7 +115,7 @@ inline bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const {
104	115	inline bool LibIgnore::IsPcInstrumented(uptr pc) const {
105	116	const uptr n = atomic_load(&instrumented_ranges_count_, memory_order_acquire);
106	117	for (uptr i = 0; i < n; i++) {
107		if (IsInRange(pc, instrumented_code_ranges_[i]))
	118	if (instrumented_code_ranges_[i].IsInRange(pc))
108	119	return true;
109	120	}
110	121	return false;

lib/tsan/sanitizer_common/sanitizer_linux.cpp+325-71

...	...	@@ -107,7 +107,9 @@ extern struct ps_strings *__ps_strings;
107	107	# endif // SANITIZER_NETBSD
108	108
109	109	# if SANITIZER_SOLARIS
	110	# include <stddef.h>
110	111	# include <stdlib.h>
	112	# include <sys/frame.h>
111	113	# include <thread.h>
112	114	# define environ _environ
113	115	# endif
...	...	@@ -132,9 +134,10 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE \| FUTEX_PRIVATE_FLAG;
132	134	// Are we using 32-bit or 64-bit Linux syscalls?
133	135	// x32 (which defines __x86_64__) has SANITIZER_WORDSIZE == 32
134	136	// but it still needs to use 64-bit syscalls.
135		# if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__powerpc64__) \|\| \
136		SANITIZER_WORDSIZE == 64 \|\| \
137		(defined(__mips__) && _MIPS_SIM == _ABIN32))
	137	# if SANITIZER_LINUX && \
	138	(defined(__x86_64__) \|\| defined(__powerpc64__) \|\| \
	139	SANITIZER_WORDSIZE == 64 \|\| \
	140	(defined(__mips__) && defined(_ABIN32) && _MIPS_SIM == _ABIN32))
138	141	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1
139	142	# else
140	143	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0
...	...	@@ -152,6 +155,8 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE \| FUTEX_PRIVATE_FLAG;
152	155
153	156	# if SANITIZER_FREEBSD
154	157	# define SANITIZER_USE_GETENTROPY 1
	158	extern "C" void __sys_mmap(void addr, size_t len, int prot, int flags, int fd,
	159	off_t offset);
155	160	# endif
156	161
157	162	namespace __sanitizer {
...	...	@@ -160,33 +165,56 @@ void SetSigProcMask(__sanitizer_sigset_t set, __sanitizer_sigset_t oldset) {
160	165	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, oldset));
161	166	}
162	167
	168	# if SANITIZER_LINUX
	169	// Deletes the specified signal from newset, if it is not present in oldset
	170	// Equivalently: newset[signum] = newset[signum] & oldset[signum]
	171	static void KeepUnblocked(__sanitizer_sigset_t &newset,
	172	__sanitizer_sigset_t &oldset, int signum) {
	173	// FIXME: https://github.com/google/sanitizers/issues/1816
	174	if (SANITIZER_ANDROID \|\| !internal_sigismember(&oldset, signum))
	175	internal_sigdelset(&newset, signum);
	176	}
	177	# endif
	178
163	179	// Block asynchronous signals
164	180	void BlockSignals(__sanitizer_sigset_t *oldset) {
165		__sanitizer_sigset_t set;
166		internal_sigfillset(&set);
167		# if SANITIZER_LINUX && !SANITIZER_ANDROID
	181	__sanitizer_sigset_t newset;
	182	internal_sigfillset(&newset);
	183
	184	# if SANITIZER_LINUX
	185	__sanitizer_sigset_t currentset;
	186
	187	# if !SANITIZER_ANDROID
	188	// FIXME: https://github.com/google/sanitizers/issues/1816
	189	SetSigProcMask(NULL, &currentset);
	190
168	191	// Glibc uses SIGSETXID signal during setuid call. If this signal is blocked
169	192	// on any thread, setuid call hangs.
170	193	// See test/sanitizer_common/TestCases/Linux/setuid.c.
171		internal_sigdelset(&set, 33);
172		# endif
173		# if SANITIZER_LINUX
	194	KeepUnblocked(newset, currentset, 33);
	195	# endif // !SANITIZER_ANDROID
	196
174	197	// Seccomp-BPF-sandboxed processes rely on SIGSYS to handle trapped syscalls.
175	198	// If this signal is blocked, such calls cannot be handled and the process may
176	199	// hang.
177		internal_sigdelset(&set, 31);
	200	KeepUnblocked(newset, currentset, 31);
178	201
	202	# if !SANITIZER_ANDROID
179	203	// Don't block synchronous signals
180		internal_sigdelset(&set, SIGSEGV);
181		internal_sigdelset(&set, SIGBUS);
182		internal_sigdelset(&set, SIGILL);
183		internal_sigdelset(&set, SIGTRAP);
184		internal_sigdelset(&set, SIGABRT);
185		internal_sigdelset(&set, SIGFPE);
186		internal_sigdelset(&set, SIGPIPE);
187		# endif
	204	// but also don't unblock signals that the user had deliberately blocked.
	205	// FIXME: https://github.com/google/sanitizers/issues/1816
	206	KeepUnblocked(newset, currentset, SIGSEGV);
	207	KeepUnblocked(newset, currentset, SIGBUS);
	208	KeepUnblocked(newset, currentset, SIGILL);
	209	KeepUnblocked(newset, currentset, SIGTRAP);
	210	KeepUnblocked(newset, currentset, SIGABRT);
	211	KeepUnblocked(newset, currentset, SIGFPE);
	212	KeepUnblocked(newset, currentset, SIGPIPE);
	213	# endif //! SANITIZER_ANDROID
	214
	215	# endif // SANITIZER_LINUX
188	216
189		SetSigProcMask(&set, oldset);
	217	SetSigProcMask(&newset, oldset);
190	218	}
191	219
192	220	ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) {
...	...	@@ -218,7 +246,9 @@ ScopedBlockSignals::~ScopedBlockSignals() { SetSigProcMask(&saved_, nullptr); }
218	246	# if !SANITIZER_S390
219	247	uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
220	248	u64 offset) {
221		# if SANITIZER_FREEBSD \|\| SANITIZER_LINUX_USES_64BIT_SYSCALLS
	249	# if SANITIZER_FREEBSD
	250	return (uptr)__sys_mmap(addr, length, prot, flags, fd, offset);
	251	# elif SANITIZER_LINUX_USES_64BIT_SYSCALLS
222	252	return internal_syscall(SYSCALL(mmap), (uptr)addr, length, prot, flags, fd,
223	253	offset);
224	254	# else
...	...	@@ -250,6 +280,11 @@ int internal_madvise(uptr addr, uptr length, int advice) {
250	280	return internal_syscall(SYSCALL(madvise), addr, length, advice);
251	281	}
252	282
	283	# if SANITIZER_FREEBSD
	284	uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags) {
	285	return internal_syscall(SYSCALL(close_range), lowfd, highfd, flags);
	286	}
	287	# endif
253	288	uptr internal_close(fd_t fd) { return internal_syscall(SYSCALL(close), fd); }
254	289
255	290	uptr internal_open(const char *filename, int flags) {
...	...	@@ -395,8 +430,9 @@ uptr internal_stat(const char path, void buf) {
395	430	AT_NO_AUTOMOUNT, STATX_BASIC_STATS, (uptr)&bufx);
396	431	statx_to_stat(&bufx, (struct stat *)buf);
397	432	return res;
398		# elif (SANITIZER_WORDSIZE == 64 \|\| SANITIZER_X32 \|\| \
399		(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \
	433	# elif ( \
	434	SANITIZER_WORDSIZE == 64 \|\| SANITIZER_X32 \|\| \
	435	(defined(__mips__) && defined(_ABIN32) && _MIPS_SIM == _ABIN32)) && \
400	436	!SANITIZER_SPARC
401	437	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
402	438	0);
...	...	@@ -433,8 +469,9 @@ uptr internal_lstat(const char path, void buf) {
433	469	STATX_BASIC_STATS, (uptr)&bufx);
434	470	statx_to_stat(&bufx, (struct stat *)buf);
435	471	return res;
436		# elif (defined(_LP64) \|\| SANITIZER_X32 \|\| \
437		(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \
	472	# elif ( \
	473	defined(_LP64) \|\| SANITIZER_X32 \|\| \
	474	(defined(__mips__) && defined(_ABIN32) && _MIPS_SIM == _ABIN32)) && \
438	475	!SANITIZER_SPARC
439	476	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
440	477	AT_SYMLINK_NOFOLLOW);
...	...	@@ -721,6 +758,11 @@ static void GetArgsAndEnv(char *argv, char *envp) {
721	758	# if !SANITIZER_GO
722	759	if (&__libc_stack_end) {
723	760	uptr stack_end = (uptr )__libc_stack_end;
	761	// Linux/sparc64 needs an adjustment, cf. glibc
	762	// sysdeps/sparc/sparc{32,64}/dl-machine.h (DL_STACK_END).
	763	# if SANITIZER_LINUX && defined(__sparc__)
	764	stack_end = &stack_end[16];
	765	# endif
724	766	// Normally argc can be obtained from *stack_end, however, on ARM glibc's
725	767	// _start clobbers it:
726	768	// https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/arm/start.S;hb=refs/heads/release/2.31/master#l75
...	...	@@ -1014,34 +1056,29 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {
1014	1056
1015	1057	# if !SANITIZER_NETBSD
1016	1058	// ThreadLister implementation.
1017		ThreadLister::ThreadLister(pid_t pid) : pid_(pid), buffer_(4096) {
1018		char task_directory_path[80];
1019		internal_snprintf(task_directory_path, sizeof(task_directory_path),
1020		"/proc/%d/task/", pid);
1021		descriptor_ = internal_open(task_directory_path, O_RDONLY \| O_DIRECTORY);
1022		if (internal_iserror(descriptor_)) {
1023		Report("Can't open /proc/%d/task for reading.\n", pid);
1024		}
	1059	ThreadLister::ThreadLister(pid_t pid) : buffer_(4096) {
	1060	task_path_.AppendF("/proc/%d/task", pid);
1025	1061	}
1026	1062
1027	1063	ThreadLister::Result ThreadLister::ListThreads(
1028	1064	InternalMmapVector<tid_t> *threads) {
1029		if (internal_iserror(descriptor_))
	1065	int descriptor = internal_open(task_path_.data(), O_RDONLY \| O_DIRECTORY);
	1066	if (internal_iserror(descriptor)) {
	1067	Report("Can't open %s for reading.\n", task_path_.data());
1030	1068	return Error;
1031		internal_lseek(descriptor_, 0, SEEK_SET);
	1069	}
	1070	auto cleanup = at_scope_exit([&] { internal_close(descriptor); });
1032	1071	threads->clear();
1033	1072
1034	1073	Result result = Ok;
1035	1074	for (bool first_read = true;; first_read = false) {
1036		// Resize to max capacity if it was downsized by IsAlive.
1037		buffer_.resize(buffer_.capacity());
1038	1075	CHECK_GE(buffer_.size(), 4096);
1039	1076	uptr read = internal_getdents(
1040		descriptor_, (struct linux_dirent *)buffer_.data(), buffer_.size());
	1077	descriptor, (struct linux_dirent *)buffer_.data(), buffer_.size());
1041	1078	if (!read)
1042	1079	return result;
1043	1080	if (internal_iserror(read)) {
1044		Report("Can't read directory entries from /proc/%d/task.\n", pid_);
	1081	Report("Can't read directory entries from %s.\n", task_path_.data());
1045	1082	return Error;
1046	1083	}
1047	1084
...	...	@@ -1079,26 +1116,33 @@ ThreadLister::Result ThreadLister::ListThreads(
1079	1116	}
1080	1117	}
1081	1118
1082		bool ThreadLister::IsAlive(int tid) {
	1119	const char *ThreadLister::LoadStatus(tid_t tid) {
	1120	status_path_.clear();
	1121	status_path_.AppendF("%s/%llu/status", task_path_.data(), tid);
	1122	auto cleanup = at_scope_exit([&] {
	1123	// Resize back to capacity if it is downsized by `ReadFileToVector`.
	1124	buffer_.resize(buffer_.capacity());
	1125	});
	1126	if (!ReadFileToVector(status_path_.data(), &buffer_) \|\| buffer_.empty())
	1127	return nullptr;
	1128	buffer_.push_back('\0');
	1129	return buffer_.data();
	1130	}
	1131
	1132	bool ThreadLister::IsAlive(tid_t tid) {
1083	1133	// /proc/%d/task/%d/status uses same call to detect alive threads as
1084	1134	// proc_task_readdir. See task_state implementation in Linux.
1085		char path[80];
1086		internal_snprintf(path, sizeof(path), "/proc/%d/task/%d/status", pid_, tid);
1087		if (!ReadFileToVector(path, &buffer_) \|\| buffer_.empty())
1088		return false;
1089		buffer_.push_back(0);
1090	1135	static const char kPrefix[] = "\nPPid:";
1091		const char *field = internal_strstr(buffer_.data(), kPrefix);
	1136	const char *status = LoadStatus(tid);
	1137	if (!status)
	1138	return false;
	1139	const char *field = internal_strstr(status, kPrefix);
1092	1140	if (!field)
1093	1141	return false;
1094	1142	field += internal_strlen(kPrefix);
1095	1143	return (int)internal_atoll(field) != 0;
1096	1144	}
1097	1145
1098		ThreadLister::~ThreadLister() {
1099		if (!internal_iserror(descriptor_))
1100		internal_close(descriptor_);
1101		}
1102	1146	# endif
1103	1147
1104	1148	# if SANITIZER_WORDSIZE == 32
...	...	@@ -1808,11 +1852,6 @@ int internal_uname(struct utsname *buf) {
1808	1852	# endif
1809	1853
1810	1854	# if SANITIZER_ANDROID
1811		# if __ANDROID_API__ < 21
1812		extern "C" __attribute__((weak)) int dl_iterate_phdr(
1813		int ()(struct dl_phdr_info , size_t, void ), void );
1814		# endif
1815
1816	1855	static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,
1817	1856	void *data) {
1818	1857	// Any name starting with "lib" indicates a bug in L where library base names
...	...	@@ -1828,9 +1867,7 @@ static int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,
1828	1867	static atomic_uint32_t android_api_level;
1829	1868
1830	1869	static AndroidApiLevel AndroidDetectApiLevelStatic() {
1831		# if __ANDROID_API__ <= 19
1832		return ANDROID_KITKAT;
1833		# elif __ANDROID_API__ <= 22
	1870	# if __ANDROID_API__ <= 22
1834	1871	return ANDROID_LOLLIPOP_MR1;
1835	1872	# else
1836	1873	return ANDROID_POST_LOLLIPOP;
...	...	@@ -1838,8 +1875,6 @@ static AndroidApiLevel AndroidDetectApiLevelStatic() {
1838	1875	}
1839	1876
1840	1877	static AndroidApiLevel AndroidDetectApiLevel() {
1841		if (!&dl_iterate_phdr)
1842		return ANDROID_KITKAT; // K or lower
1843	1878	bool base_name_seen = false;
1844	1879	dl_iterate_phdr(dl_iterate_phdr_test_cb, &base_name_seen);
1845	1880	if (base_name_seen)
...	...	@@ -2014,6 +2049,18 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
2014	2049	return Unknown;
2015	2050	return esr & ESR_ELx_WNR ? Write : Read;
2016	2051	# elif defined(__loongarch__)
	2052	// In the musl environment, the Linux kernel uapi sigcontext.h is not
	2053	// included in signal.h. To avoid missing the SC_ADDRERR_{RD,WR} macros,
	2054	// copy them here. The LoongArch Linux kernel uapi is already stable,
	2055	// so there's no need to worry about the value changing.
	2056	# ifndef SC_ADDRERR_RD
	2057	// Address error was due to memory load
	2058	# define SC_ADDRERR_RD (1 << 30)
	2059	# endif
	2060	# ifndef SC_ADDRERR_WR
	2061	// Address error was due to memory store
	2062	# define SC_ADDRERR_WR (1 << 31)
	2063	# endif
2017	2064	u32 flags = ucontext->uc_mcontext.__flags;
2018	2065	if (flags & SC_ADDRERR_RD)
2019	2066	return SignalContext::Read;
...	...	@@ -2154,8 +2201,26 @@ bool SignalContext::IsTrueFaultingAddress() const {
2154	2201	UNUSED
2155	2202	static const char *RegNumToRegName(int reg) {
2156	2203	switch (reg) {
2157		# if SANITIZER_LINUX
	2204	# if SANITIZER_LINUX && SANITIZER_GLIBC \|\| SANITIZER_NETBSD
2158	2205	# if defined(__x86_64__)
	2206	# if SANITIZER_NETBSD
	2207	# define REG_RAX _REG_RAX
	2208	# define REG_RBX _REG_RBX
	2209	# define REG_RCX _REG_RCX
	2210	# define REG_RDX _REG_RDX
	2211	# define REG_RDI _REG_RDI
	2212	# define REG_RSI _REG_RSI
	2213	# define REG_RBP _REG_RBP
	2214	# define REG_RSP _REG_RSP
	2215	# define REG_R8 _REG_R8
	2216	# define REG_R9 _REG_R9
	2217	# define REG_R10 _REG_R10
	2218	# define REG_R11 _REG_R11
	2219	# define REG_R12 _REG_R12
	2220	# define REG_R13 _REG_R13
	2221	# define REG_R14 _REG_R14
	2222	# define REG_R15 _REG_R15
	2223	# endif
2159	2224	case REG_RAX:
2160	2225	return "rax";
2161	2226	case REG_RBX:
...	...	@@ -2189,6 +2254,16 @@ static const char *RegNumToRegName(int reg) {
2189	2254	case REG_R15:
2190	2255	return "r15";
2191	2256	# elif defined(__i386__)
	2257	# if SANITIZER_NETBSD
	2258	# define REG_EAX _REG_EAX
	2259	# define REG_EBX _REG_EBX
	2260	# define REG_ECX _REG_ECX
	2261	# define REG_EDX _REG_EDX
	2262	# define REG_EDI _REG_EDI
	2263	# define REG_ESI _REG_ESI
	2264	# define REG_EBP _REG_EBP
	2265	# define REG_ESP _REG_ESP
	2266	# endif
2192	2267	case REG_EAX:
2193	2268	return "eax";
2194	2269	case REG_EBX:
...	...	@@ -2205,32 +2280,170 @@ static const char *RegNumToRegName(int reg) {
2205	2280	return "ebp";
2206	2281	case REG_ESP:
2207	2282	return "esp";
	2283	# elif defined(__arm__)
	2284	# ifdef MAKE_CASE
	2285	# undef MAKE_CASE
	2286	# endif
	2287	# define REG_STR(reg) #reg
	2288	# define MAKE_CASE(N) \
	2289	case REG_R##N: \
	2290	return REG_STR(r##N)
	2291	MAKE_CASE(0);
	2292	MAKE_CASE(1);
	2293	MAKE_CASE(2);
	2294	MAKE_CASE(3);
	2295	MAKE_CASE(4);
	2296	MAKE_CASE(5);
	2297	MAKE_CASE(6);
	2298	MAKE_CASE(7);
	2299	MAKE_CASE(8);
	2300	MAKE_CASE(9);
	2301	MAKE_CASE(10);
	2302	MAKE_CASE(11);
	2303	MAKE_CASE(12);
	2304	case REG_R13:
	2305	return "sp";
	2306	case REG_R14:
	2307	return "lr";
	2308	case REG_R15:
	2309	return "pc";
	2310	# elif defined(__aarch64__)
	2311	# define REG_STR(reg) #reg
	2312	# define MAKE_CASE(N) \
	2313	case N: \
	2314	return REG_STR(x##N)
	2315	MAKE_CASE(0);
	2316	MAKE_CASE(1);
	2317	MAKE_CASE(2);
	2318	MAKE_CASE(3);
	2319	MAKE_CASE(4);
	2320	MAKE_CASE(5);
	2321	MAKE_CASE(6);
	2322	MAKE_CASE(7);
	2323	MAKE_CASE(8);
	2324	MAKE_CASE(9);
	2325	MAKE_CASE(10);
	2326	MAKE_CASE(11);
	2327	MAKE_CASE(12);
	2328	MAKE_CASE(13);
	2329	MAKE_CASE(14);
	2330	MAKE_CASE(15);
	2331	MAKE_CASE(16);
	2332	MAKE_CASE(17);
	2333	MAKE_CASE(18);
	2334	MAKE_CASE(19);
	2335	MAKE_CASE(20);
	2336	MAKE_CASE(21);
	2337	MAKE_CASE(22);
	2338	MAKE_CASE(23);
	2339	MAKE_CASE(24);
	2340	MAKE_CASE(25);
	2341	MAKE_CASE(26);
	2342	MAKE_CASE(27);
	2343	MAKE_CASE(28);
	2344	case 29:
	2345	return "fp";
	2346	case 30:
	2347	return "lr";
	2348	case 31:
	2349	return "sp";
2208	2350	# endif
2209		# endif
	2351	# endif // SANITIZER_LINUX && SANITIZER_GLIBC
2210	2352	default:
2211	2353	return NULL;
2212	2354	}
2213	2355	return NULL;
2214	2356	}
2215	2357
2216		# if SANITIZER_LINUX
	2358	# if ((SANITIZER_LINUX && SANITIZER_GLIBC) \|\| SANITIZER_NETBSD) && \
	2359	(defined(__arm__) \|\| defined(__aarch64__))
	2360	static uptr GetArmRegister(ucontext_t *ctx, int RegNum) {
	2361	switch (RegNum) {
	2362	# if defined(__arm__) && !SANITIZER_NETBSD
	2363	# ifdef MAKE_CASE
	2364	# undef MAKE_CASE
	2365	# endif
	2366	# define MAKE_CASE(N) \
	2367	case REG_R##N: \
	2368	return ctx->uc_mcontext.arm_r##N
	2369	MAKE_CASE(0);
	2370	MAKE_CASE(1);
	2371	MAKE_CASE(2);
	2372	MAKE_CASE(3);
	2373	MAKE_CASE(4);
	2374	MAKE_CASE(5);
	2375	MAKE_CASE(6);
	2376	MAKE_CASE(7);
	2377	MAKE_CASE(8);
	2378	MAKE_CASE(9);
	2379	MAKE_CASE(10);
	2380	case REG_R11:
	2381	return ctx->uc_mcontext.arm_fp;
	2382	case REG_R12:
	2383	return ctx->uc_mcontext.arm_ip;
	2384	case REG_R13:
	2385	return ctx->uc_mcontext.arm_sp;
	2386	case REG_R14:
	2387	return ctx->uc_mcontext.arm_lr;
	2388	case REG_R15:
	2389	return ctx->uc_mcontext.arm_pc;
	2390	# elif defined(__aarch64__)
	2391	# if SANITIZER_LINUX
	2392	case 0 ... 30:
	2393	return ctx->uc_mcontext.regs[RegNum];
	2394	case 31:
	2395	return ctx->uc_mcontext.sp;
	2396	# elif SANITIZER_NETBSD
	2397	case 0 ... 31:
	2398	return ctx->uc_mcontext.__gregs[RegNum];
	2399	# endif
	2400	# endif
	2401	default:
	2402	return 0;
	2403	}
	2404	return 0;
	2405	}
	2406	# endif // SANITIZER_LINUX && SANITIZER_GLIBC && (defined(__arm__) \|\|
	2407	// defined(__aarch64__))
	2408
2217	2409	UNUSED
2218	2410	static void DumpSingleReg(ucontext_t *ctx, int RegNum) {
2219	2411	const char *RegName = RegNumToRegName(RegNum);
	2412	# if SANITIZER_LINUX && SANITIZER_GLIBC \|\| SANITIZER_NETBSD
2220	2413	# if defined(__x86_64__)
2221	2414	Printf("%s%s = 0x%016llx ", internal_strlen(RegName) == 2 ? " " : "",
2222		RegName, ctx->uc_mcontext.gregs[RegNum]);
	2415	RegName,
	2416	# if SANITIZER_LINUX
	2417	ctx->uc_mcontext.gregs[RegNum]
	2418	# elif SANITIZER_NETBSD
	2419	ctx->uc_mcontext.__gregs[RegNum]
	2420	# endif
	2421	);
2223	2422	# elif defined(__i386__)
2224		Printf("%s = 0x%08x ", RegName, ctx->uc_mcontext.gregs[RegNum]);
	2423	Printf("%s = 0x%08x ", RegName,
	2424	# if SANITIZER_LINUX
	2425	ctx->uc_mcontext.gregs[RegNum]
	2426	# elif SANITIZER_NETBSD
	2427	ctx->uc_mcontext.__gregs[RegNum]
	2428	# endif
	2429	);
	2430	# elif defined(__arm__)
	2431	Printf("%s%s = 0x%08zx ", internal_strlen(RegName) == 2 ? " " : "", RegName,
	2432	GetArmRegister(ctx, RegNum));
	2433	# elif defined(__aarch64__)
	2434	Printf("%s%s = 0x%016zx ", internal_strlen(RegName) == 2 ? " " : "", RegName,
	2435	GetArmRegister(ctx, RegNum));
2225	2436	# else
2226	2437	(void)RegName;
2227	2438	# endif
2228		}
	2439	# else
	2440	(void)RegName;
2229	2441	# endif
	2442	}
2230	2443
2231	2444	void SignalContext::DumpAllRegisters(void *context) {
2232	2445	ucontext_t ucontext = (ucontext_t )context;
2233		# if SANITIZER_LINUX
	2446	# if SANITIZER_LINUX && SANITIZER_GLIBC \|\| SANITIZER_NETBSD
2234	2447	# if defined(__x86_64__)
2235	2448	Report("Register values:\n");
2236	2449	DumpSingleReg(ucontext, REG_RAX);
...	...	@@ -2269,6 +2482,35 @@ void SignalContext::DumpAllRegisters(void *context) {
2269	2482	DumpSingleReg(ucontext, REG_EBP);
2270	2483	DumpSingleReg(ucontext, REG_ESP);
2271	2484	Printf("\n");
	2485	# elif defined(__arm__) && !SANITIZER_NETBSD
	2486	Report("Register values:\n");
	2487	DumpSingleReg(ucontext, REG_R0);
	2488	DumpSingleReg(ucontext, REG_R1);
	2489	DumpSingleReg(ucontext, REG_R2);
	2490	DumpSingleReg(ucontext, REG_R3);
	2491	Printf("\n");
	2492	DumpSingleReg(ucontext, REG_R4);
	2493	DumpSingleReg(ucontext, REG_R5);
	2494	DumpSingleReg(ucontext, REG_R6);
	2495	DumpSingleReg(ucontext, REG_R7);
	2496	Printf("\n");
	2497	DumpSingleReg(ucontext, REG_R8);
	2498	DumpSingleReg(ucontext, REG_R9);
	2499	DumpSingleReg(ucontext, REG_R10);
	2500	DumpSingleReg(ucontext, REG_R11);
	2501	Printf("\n");
	2502	DumpSingleReg(ucontext, REG_R12);
	2503	DumpSingleReg(ucontext, REG_R13);
	2504	DumpSingleReg(ucontext, REG_R14);
	2505	DumpSingleReg(ucontext, REG_R15);
	2506	Printf("\n");
	2507	# elif defined(__aarch64__)
	2508	Report("Register values:\n");
	2509	for (int i = 0; i <= 31; ++i) {
	2510	DumpSingleReg(ucontext, i);
	2511	if (i % 4 == 3)
	2512	Printf("\n");
	2513	}
2272	2514	# else
2273	2515	(void)ucontext;
2274	2516	# endif
...	...	@@ -2310,6 +2552,8 @@ void SignalContext::DumpAllRegisters(void *context) {
2310	2552	# else
2311	2553	(void)ucontext;
2312	2554	# endif
	2555	# else
	2556	(void)ucontext;
2313	2557	# endif
2314	2558	// FIXME: Implement this for other OSes and architectures.
2315	2559	}
...	...	@@ -2404,7 +2648,19 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2404	2648	# if SANITIZER_SOLARIS
2405	2649	ucontext_t ucontext = (ucontext_t )context;
2406	2650	*pc = ucontext->uc_mcontext.gregs[REG_PC];
2407		*sp = ucontext->uc_mcontext.gregs[REG_O6] + STACK_BIAS;
	2651	*sp = ucontext->uc_mcontext.gregs[REG_SP] + STACK_BIAS;
	2652	// Avoid SEGV when dereferencing sp on stack overflow with non-faulting load.
	2653	// This requires a SPARC V9 CPU. Cannot use #ASI_PNF here: only supported
	2654	// since clang-19.
	2655	# if defined(__sparcv9)
	2656	asm("ldxa [%[fp]] 0x82, %[bp]"
	2657	# else
	2658	asm("lduwa [%[fp]] 0x82, %[bp]"
	2659	# endif
	2660	: [bp] "=r"(*bp)
	2661	: [fp] "r"(&((struct frame )sp)->fr_savfp));
	2662	if (*bp)
	2663	*bp += STACK_BIAS;
2408	2664	# else
2409	2665	// Historical BSDism here.
2410	2666	struct sigcontext scontext = (struct sigcontext )context;
...	...	@@ -2415,8 +2671,8 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2415	2671	*pc = scontext->si_regs.pc;
2416	2672	*sp = scontext->si_regs.u_regs[14];
2417	2673	# endif
2418		# endif
2419	2674	bp = (uptr)((uhwptr )*sp)[14] + STACK_BIAS;
	2675	# endif
2420	2676	# elif defined(__mips__)
2421	2677	ucontext_t ucontext = (ucontext_t )context;
2422	2678	*pc = ucontext->uc_mcontext.pc;
...	...	@@ -2459,9 +2715,7 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2459	2715
2460	2716	void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); }
2461	2717
2462		void InitializePlatformEarly() {
2463		// Do nothing.
2464		}
	2718	void InitializePlatformEarly() { InitTlsSize(); }
2465	2719
2466	2720	void CheckASLR() {
2467	2721	# if SANITIZER_NETBSD

lib/tsan/sanitizer_common/sanitizer_linux.h+4-4

...	...	@@ -97,19 +97,19 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg);
97	97	class ThreadLister {
98	98	public:
99	99	explicit ThreadLister(pid_t pid);
100		~ThreadLister();
101	100	enum Result {
102	101	Error,
103	102	Incomplete,
104	103	Ok,
105	104	};
106	105	Result ListThreads(InternalMmapVector<tid_t> *threads);
	106	const char *LoadStatus(tid_t tid);
107	107
108	108	private:
109		bool IsAlive(int tid);
	109	bool IsAlive(tid_t tid);
110	110
111		pid_t pid_;
112		int descriptor_ = -1;
	111	InternalScopedString task_path_;
	112	InternalScopedString status_path_;
113	113	InternalMmapVector<char> buffer_;
114	114	};
115	115

lib/tsan/sanitizer_common/sanitizer_linux_libcdep.cpp+171-129

...	...	@@ -40,6 +40,10 @@
40	40	# include <sys/resource.h>
41	41	# include <syslog.h>
42	42
	43	# if SANITIZER_GLIBC
	44	# include <gnu/libc-version.h>
	45	# endif
	46
43	47	# if !defined(ElfW)
44	48	# define ElfW(type) Elf_##type
45	49	# endif
...	...	@@ -53,7 +57,7 @@
53	57	// that, it was never implemented. So just define it to zero.
54	58	# undef MAP_NORESERVE
55	59	# define MAP_NORESERVE 0
56		extern const Elf_Auxinfo *__elf_aux_vector;
	60	extern const Elf_Auxinfo *__elf_aux_vector __attribute__((weak));
57	61	extern "C" int __sys_sigaction(int signum, const struct sigaction *act,
58	62	struct sigaction *oldact);
59	63	# endif
...	...	@@ -196,27 +200,6 @@ bool SetEnv(const char name, const char value) {
196	200	}
197	201	# endif
198	202
199		__attribute__((unused)) static bool GetLibcVersion(int major, int minor,
200		int *patch) {
201		# ifdef _CS_GNU_LIBC_VERSION
202		char buf[64];
203		uptr len = confstr(_CS_GNU_LIBC_VERSION, buf, sizeof(buf));
204		if (len >= sizeof(buf))
205		return false;
206		buf[len] = 0;
207		static const char kGLibC[] = "glibc ";
208		if (internal_strncmp(buf, kGLibC, sizeof(kGLibC) - 1) != 0)
209		return false;
210		const char *p = buf + sizeof(kGLibC) - 1;
211		*major = internal_simple_strtoll(p, &p, 10);
212		minor = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
213		patch = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
214		return true;
215		# else
216		return false;
217		# endif
218		}
219
220	203	// True if we can use dlpi_tls_data. glibc before 2.25 may leave NULL (BZ
221	204	// #19826) so dlpi_tls_data cannot be used.
222	205	//
...	...	@@ -226,112 +209,166 @@ __attribute__((unused)) static bool GetLibcVersion(int major, int minor,
226	209	__attribute__((unused)) static int g_use_dlpi_tls_data;
227	210
228	211	# if SANITIZER_GLIBC && !SANITIZER_GO
229		__attribute__((unused)) static size_t g_tls_size;
230		void InitTlsSize() {
231		int major, minor, patch;
232		g_use_dlpi_tls_data =
233		GetLibcVersion(&major, &minor, &patch) && major == 2 && minor >= 25;
234
235		# if defined(__aarch64__) \|\| defined(__x86_64__) \|\| \
236		defined(__powerpc64__) \|\| defined(__loongarch__)
237		void *get_tls_static_info = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");
238		size_t tls_align;
239		((void ()(size_t , size_t *))get_tls_static_info)(&g_tls_size, &tls_align);
240		# endif
	212	static void GetGLibcVersion(int major, int minor, int *patch) {
	213	const char *p = gnu_get_libc_version();
	214	*major = internal_simple_strtoll(p, &p, 10);
	215	// Caller does not expect anything else.
	216	CHECK_EQ(*major, 2);
	217	minor = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
	218	patch = (p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
241	219	}
242		# else
243		void InitTlsSize() {}
244		# endif // SANITIZER_GLIBC && !SANITIZER_GO
245
246		// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage
247		// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan
248		// to get the pointer to thread-specific data keys in the thread control block.
249		# if (SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_SOLARIS) && \
250		!SANITIZER_ANDROID && !SANITIZER_GO
251		// sizeof(struct pthread) from glibc.
252		static atomic_uintptr_t thread_descriptor_size;
253	220
254	221	static uptr ThreadDescriptorSizeFallback() {
255		uptr val = 0;
256		# if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__arm__)
	222	# if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__arm__) \|\| \
	223	SANITIZER_RISCV64
257	224	int major;
258	225	int minor;
259	226	int patch;
260		if (GetLibcVersion(&major, &minor, &patch) && major == 2) {
261		/* sizeof(struct pthread) values from various glibc versions. */
262		if (SANITIZER_X32)
263		val = 1728; // Assume only one particular version for x32.
264		// For ARM sizeof(struct pthread) changed in Glibc 2.23.
265		else if (SANITIZER_ARM)
266		val = minor <= 22 ? 1120 : 1216;
267		else if (minor <= 3)
268		val = FIRST_32_SECOND_64(1104, 1696);
269		else if (minor == 4)
270		val = FIRST_32_SECOND_64(1120, 1728);
271		else if (minor == 5)
272		val = FIRST_32_SECOND_64(1136, 1728);
273		else if (minor <= 9)
274		val = FIRST_32_SECOND_64(1136, 1712);
275		else if (minor == 10)
276		val = FIRST_32_SECOND_64(1168, 1776);
277		else if (minor == 11 \|\| (minor == 12 && patch == 1))
278		val = FIRST_32_SECOND_64(1168, 2288);
279		else if (minor <= 14)
280		val = FIRST_32_SECOND_64(1168, 2304);
281		else if (minor < 32) // Unknown version
282		val = FIRST_32_SECOND_64(1216, 2304);
283		else // minor == 32
284		val = FIRST_32_SECOND_64(1344, 2496);
285		}
286		# elif defined(__s390__) \|\| defined(__sparc__)
	227	GetGLibcVersion(&major, &minor, &patch);
	228	# endif
	229
	230	# if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__arm__)
	231	/* sizeof(struct pthread) values from various glibc versions. */
	232	if (SANITIZER_X32)
	233	return 1728; // Assume only one particular version for x32.
	234	// For ARM sizeof(struct pthread) changed in Glibc 2.23.
	235	if (SANITIZER_ARM)
	236	return minor <= 22 ? 1120 : 1216;
	237	if (minor <= 3)
	238	return FIRST_32_SECOND_64(1104, 1696);
	239	if (minor == 4)
	240	return FIRST_32_SECOND_64(1120, 1728);
	241	if (minor == 5)
	242	return FIRST_32_SECOND_64(1136, 1728);
	243	if (minor <= 9)
	244	return FIRST_32_SECOND_64(1136, 1712);
	245	if (minor == 10)
	246	return FIRST_32_SECOND_64(1168, 1776);
	247	if (minor == 11 \|\| (minor == 12 && patch == 1))
	248	return FIRST_32_SECOND_64(1168, 2288);
	249	if (minor <= 14)
	250	return FIRST_32_SECOND_64(1168, 2304);
	251	if (minor < 32) // Unknown version
	252	return FIRST_32_SECOND_64(1216, 2304);
	253	// minor == 32
	254	return FIRST_32_SECOND_64(1344, 2496);
	255	# endif
	256
	257	# if SANITIZER_RISCV64
	258	// TODO: consider adding an optional runtime check for an unknown (untested)
	259	// glibc version
	260	if (minor <= 28) // WARNING: the highest tested version is 2.29
	261	return 1772; // no guarantees for this one
	262	if (minor <= 31)
	263	return 1772; // tested against glibc 2.29, 2.31
	264	return 1936; // tested against glibc 2.32
	265	# endif
	266
	267	# if defined(__s390__) \|\| defined(__sparc__)
287	268	// The size of a prefix of TCB including pthread::{specific_1stblock,specific}
288	269	// suffices. Just return offsetof(struct pthread, specific_used), which hasn't
289	270	// changed since 2007-05. Technically this applies to i386/x86_64 as well but
290	271	// we call _dl_get_tls_static_info and need the precise size of struct
291	272	// pthread.
292	273	return FIRST_32_SECOND_64(524, 1552);
293		# elif defined(__mips__)
	274	# endif
	275
	276	# if defined(__mips__)
294	277	// TODO(sagarthakur): add more values as per different glibc versions.
295		val = FIRST_32_SECOND_64(1152, 1776);
296		# elif SANITIZER_LOONGARCH64
297		val = 1856; // from glibc 2.36
298		# elif SANITIZER_RISCV64
299		int major;
300		int minor;
301		int patch;
302		if (GetLibcVersion(&major, &minor, &patch) && major == 2) {
303		// TODO: consider adding an optional runtime check for an unknown (untested)
304		// glibc version
305		if (minor <= 28) // WARNING: the highest tested version is 2.29
306		val = 1772; // no guarantees for this one
307		else if (minor <= 31)
308		val = 1772; // tested against glibc 2.29, 2.31
309		else
310		val = 1936; // tested against glibc 2.32
311		}
	278	return FIRST_32_SECOND_64(1152, 1776);
	279	# endif
	280
	281	# if SANITIZER_LOONGARCH64
	282	return 1856; // from glibc 2.36
	283	# endif
312	284
313		# elif defined(__aarch64__)
	285	# if defined(__aarch64__)
314	286	// The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
315		val = 1776;
316		# elif defined(__powerpc64__)
317		val = 1776; // from glibc.ppc64le 2.20-8.fc21
	287	return 1776;
	288	# endif
	289
	290	# if defined(__powerpc64__)
	291	return 1776; // from glibc.ppc64le 2.20-8.fc21
318	292	# endif
319		return val;
320	293	}
	294	# endif // SANITIZER_GLIBC && !SANITIZER_GO
	295
	296	# if SANITIZER_FREEBSD && !SANITIZER_GO
	297	// FIXME: Implementation is very GLIBC specific, but it's used by FreeBSD.
	298	static uptr ThreadDescriptorSizeFallback() {
	299	# if defined(__s390__) \|\| defined(__sparc__)
	300	// The size of a prefix of TCB including pthread::{specific_1stblock,specific}
	301	// suffices. Just return offsetof(struct pthread, specific_used), which hasn't
	302	// changed since 2007-05. Technically this applies to i386/x86_64 as well but
	303	// we call _dl_get_tls_static_info and need the precise size of struct
	304	// pthread.
	305	return FIRST_32_SECOND_64(524, 1552);
	306	# endif
321	307
322		uptr ThreadDescriptorSize() {
323		uptr val = atomic_load_relaxed(&thread_descriptor_size);
324		if (val)
325		return val;
326		// _thread_db_sizeof_pthread is a GLIBC_PRIVATE symbol that is exported in
327		// glibc 2.34 and later.
328		if (unsigned psizeof = static_cast<unsigned >(
329		dlsym(RTLD_DEFAULT, "_thread_db_sizeof_pthread")))
330		val = *psizeof;
331		if (!val)
332		val = ThreadDescriptorSizeFallback();
333		atomic_store_relaxed(&thread_descriptor_size, val);
334		return val;
	308	# if defined(__mips__)
	309	// TODO(sagarthakur): add more values as per different glibc versions.
	310	return FIRST_32_SECOND_64(1152, 1776);
	311	# endif
	312
	313	# if SANITIZER_LOONGARCH64
	314	return 1856; // from glibc 2.36
	315	# endif
	316
	317	# if defined(__aarch64__)
	318	// The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
	319	return 1776;
	320	# endif
	321
	322	# if defined(__powerpc64__)
	323	return 1776; // from glibc.ppc64le 2.20-8.fc21
	324	# endif
	325
	326	return 0;
	327	}
	328	# endif // SANITIZER_FREEBSD && !SANITIZER_GO
	329
	330	# if (SANITIZER_FREEBSD \|\| SANITIZER_GLIBC) && !SANITIZER_GO
	331	// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage
	332	// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan
	333	// to get the pointer to thread-specific data keys in the thread control block.
	334	// sizeof(struct pthread) from glibc.
	335	static uptr thread_descriptor_size;
	336
	337	uptr ThreadDescriptorSize() { return thread_descriptor_size; }
	338
	339	# if SANITIZER_GLIBC
	340	__attribute__((unused)) static size_t g_tls_size;
	341	# endif
	342
	343	void InitTlsSize() {
	344	# if SANITIZER_GLIBC
	345	int major, minor, patch;
	346	GetGLibcVersion(&major, &minor, &patch);
	347	g_use_dlpi_tls_data = major == 2 && minor >= 25;
	348
	349	if (major == 2 && minor >= 34) {
	350	// _thread_db_sizeof_pthread is a GLIBC_PRIVATE symbol that is exported in
	351	// glibc 2.34 and later.
	352	if (unsigned psizeof = static_cast<unsigned >(
	353	dlsym(RTLD_DEFAULT, "_thread_db_sizeof_pthread"))) {
	354	thread_descriptor_size = *psizeof;
	355	}
	356	}
	357
	358	# if defined(__aarch64__) \|\| defined(__x86_64__) \|\| \
	359	defined(__powerpc64__) \|\| defined(__loongarch__)
	360	auto get_tls_static_info = (void ()(size_t , size_t ))dlsym(
	361	RTLD_DEFAULT, "_dl_get_tls_static_info");
	362	size_t tls_align;
	363	// Can be null if static link.
	364	if (get_tls_static_info)
	365	get_tls_static_info(&g_tls_size, &tls_align);
	366	# endif
	367
	368	# endif // SANITIZER_GLIBC
	369
	370	if (!thread_descriptor_size)
	371	thread_descriptor_size = ThreadDescriptorSizeFallback();
335	372	}
336	373
337	374	# if defined(__mips__) \|\| defined(__powerpc64__) \|\| SANITIZER_RISCV64 \|\| \
...	...	@@ -354,7 +391,13 @@ static uptr TlsPreTcbSize() {
354	391	return kTlsPreTcbSize;
355	392	}
356	393	# endif
	394	# else // (SANITIZER_FREEBSD \|\| SANITIZER_GLIBC) && !SANITIZER_GO
	395	void InitTlsSize() {}
	396	uptr ThreadDescriptorSize() { return 0; }
	397	# endif // (SANITIZER_FREEBSD \|\| SANITIZER_GLIBC) && !SANITIZER_GO
357	398
	399	# if (SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_SOLARIS) && \
	400	!SANITIZER_ANDROID && !SANITIZER_GO
358	401	namespace {
359	402	struct TlsBlock {
360	403	uptr begin, end, align;
...	...	@@ -626,25 +669,32 @@ uptr GetTlsSize() {
626	669	}
627	670	# endif
628	671
629		void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,
630		uptr tls_addr, uptr tls_size) {
	672	void GetThreadStackAndTls(bool main, uptr stk_begin, uptr stk_end,
	673	uptr tls_begin, uptr tls_end) {
631	674	# if SANITIZER_GO
632	675	// Stub implementation for Go.
633		stk_addr = stk_size = tls_addr = tls_size = 0;
	676	*stk_begin = 0;
	677	*stk_end = 0;
	678	*tls_begin = 0;
	679	*tls_end = 0;
634	680	# else
635		GetTls(tls_addr, tls_size);
	681	uptr tls_addr = 0;
	682	uptr tls_size = 0;
	683	GetTls(&tls_addr, &tls_size);
	684	*tls_begin = tls_addr;
	685	*tls_end = tls_addr + tls_size;
636	686
637	687	uptr stack_top, stack_bottom;
638	688	GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);
639		*stk_addr = stack_bottom;
640		*stk_size = stack_top - stack_bottom;
	689	*stk_begin = stack_bottom;
	690	*stk_end = stack_top;
641	691
642	692	if (!main) {
643	693	// If stack and tls intersect, make them non-intersecting.
644		if (tls_addr > stk_addr && tls_addr < stk_addr + *stk_size) {
645		if (stk_addr + stk_size < tls_addr + tls_size)
646		tls_size = stk_addr + stk_size - tls_addr;
647		stk_size = tls_addr - *stk_addr;
	694	if (tls_begin > stk_begin && tls_begin < stk_end) {
	695	if (stk_end < tls_end)
	696	tls_end = stk_end;
	697	stk_end = tls_begin;
648	698	}
649	699	}
650	700	# endif
...	...	@@ -723,11 +773,6 @@ static int dl_iterate_phdr_cb(dl_phdr_info info, size_t size, void arg) {
723	773	return 0;
724	774	}
725	775
726		# if SANITIZER_ANDROID && __ANDROID_API__ < 21
727		extern "C" __attribute__((weak)) int dl_iterate_phdr(
728		int ()(struct dl_phdr_info , size_t, void ), void );
729		# endif
730
731	776	static bool requiresProcmaps() {
732	777	# if SANITIZER_ANDROID && __ANDROID_API__ <= 22
733	778	// Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken.
...	...	@@ -890,11 +935,8 @@ extern "C" SANITIZER_WEAK_ATTRIBUTE int __android_log_write(int prio,
890	935	void WriteOneLineToSyslog(const char *s) {
891	936	if (&async_safe_write_log) {
892	937	async_safe_write_log(SANITIZER_ANDROID_LOG_INFO, GetProcessName(), s);
893		} else if (AndroidGetApiLevel() > ANDROID_KITKAT) {
894		syslog(LOG_INFO, "%s", s);
895	938	} else {
896		CHECK(&__android_log_write);
897		__android_log_write(SANITIZER_ANDROID_LOG_INFO, nullptr, s);
	939	syslog(LOG_INFO, "%s", s);
898	940	}
899	941	}
900	942

lib/tsan/sanitizer_common/sanitizer_mac.cpp+36-28

...	...	@@ -45,7 +45,7 @@ extern char **environ;
45	45	# define SANITIZER_OS_TRACE 0
46	46	# endif
47	47
48		// import new crash reporting api
	48	// Integrate with CrashReporter library if available
49	49	# if defined(__has_include) && __has_include(<CrashReporterClient.h>)
50	50	# define HAVE_CRASHREPORTERCLIENT_H 1
51	51	# include <CrashReporterClient.h>
...	...	@@ -545,9 +545,6 @@ uptr GetTlsSize() {
545	545	return 0;
546	546	}
547	547
548		void InitTlsSize() {
549		}
550
551	548	uptr TlsBaseAddr() {
552	549	uptr segbase = 0;
553	550	#if defined(__x86_64__)
...	...	@@ -572,21 +569,18 @@ uptr TlsSize() {
572	569	#endif
573	570	}
574	571
575		void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,
576		uptr tls_addr, uptr tls_size) {
577		#if !SANITIZER_GO
578		uptr stack_top, stack_bottom;
579		GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);
580		*stk_addr = stack_bottom;
581		*stk_size = stack_top - stack_bottom;
582		*tls_addr = TlsBaseAddr();
583		*tls_size = TlsSize();
584		#else
585		*stk_addr = 0;
586		*stk_size = 0;
587		*tls_addr = 0;
588		*tls_size = 0;
589		#endif
	572	void GetThreadStackAndTls(bool main, uptr stk_begin, uptr stk_end,
	573	uptr tls_begin, uptr tls_end) {
	574	# if !SANITIZER_GO
	575	GetThreadStackTopAndBottom(main, stk_end, stk_begin);
	576	*tls_begin = TlsBaseAddr();
	577	tls_end = tls_begin + TlsSize();
	578	# else
	579	*stk_begin = 0;
	580	*stk_end = 0;
	581	*tls_begin = 0;
	582	*tls_end = 0;
	583	# endif
590	584	}
591	585
592	586	void ListOfModules::init() {
...	...	@@ -788,7 +782,11 @@ void WriteOneLineToSyslog(const char *s) {
788	782	if (GetMacosAlignedVersion() >= MacosVersion(10, 12)) {
789	783	os_log_error(OS_LOG_DEFAULT, "%{public}s", s);
790	784	} else {
	785	#pragma clang diagnostic push
	786	// as_log is deprecated.
	787	#pragma clang diagnostic ignored "-Wdeprecated-declarations"
791	788	asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "%s", s);
	789	#pragma clang diagnostic pop
792	790	}
793	791	#endif
794	792	}
...	...	@@ -798,8 +796,13 @@ static char crashreporter_info_buff[__sanitizer::kErrorMessageBufferSize] = {};
798	796	static Mutex crashreporter_info_mutex;
799	797
800	798	extern "C" {
801		// Integrate with crash reporter libraries.
	799
802	800	#if HAVE_CRASHREPORTERCLIENT_H
	801	// Available in CRASHREPORTER_ANNOTATIONS_VERSION 5+
	802	# ifdef CRASHREPORTER_ANNOTATIONS_INITIALIZER
	803	CRASHREPORTER_ANNOTATIONS_INITIALIZER()
	804	# else
	805	// Support for older CrashRerporter annotiations
803	806	CRASH_REPORTER_CLIENT_HIDDEN
804	807	struct crashreporter_annotations_t gCRAnnotations
805	808	__attribute__((section("__DATA," CRASHREPORTER_ANNOTATIONS_SECTION))) = {
...	...	@@ -810,17 +813,17 @@ struct crashreporter_annotations_t gCRAnnotations
810	813	0,
811	814	0,
812	815	0,
813		#if CRASHREPORTER_ANNOTATIONS_VERSION > 4
	816	# if CRASHREPORTER_ANNOTATIONS_VERSION > 4
814	817	0,
815		#endif
	818	# endif
816	819	};
817
818		#else
819		// fall back to old crashreporter api
	820	# endif
	821	# else
	822	// Revert to previous crash reporter API if client header is not available
820	823	static const char *__crashreporter_info__ __attribute__((__used__)) =
821	824	&crashreporter_info_buff[0];
822	825	asm(".desc ___crashreporter_info__, 0x10");
823		#endif
	826	#endif // HAVE_CRASHREPORTERCLIENT_H
824	827
825	828	} // extern "C"
826	829
...	...	@@ -843,6 +846,9 @@ void LogFullErrorReport(const char *buffer) {
843	846	#if !SANITIZER_GO
844	847	// Log with os_trace. This will make it into the crash log.
845	848	#if SANITIZER_OS_TRACE
	849	#pragma clang diagnostic push
	850	// os_trace is deprecated.
	851	#pragma clang diagnostic ignored "-Wdeprecated-declarations"
846	852	if (GetMacosAlignedVersion() >= MacosVersion(10, 10)) {
847	853	// os_trace requires the message (format parameter) to be a string literal.
848	854	if (internal_strncmp(SanitizerToolName, "AddressSanitizer",
...	...	@@ -860,6 +866,7 @@ void LogFullErrorReport(const char *buffer) {
860	866	if (common_flags()->log_to_syslog)
861	867	os_trace("Consult syslog for more information.");
862	868	}
	869	#pragma clang diagnostic pop
863	870	#endif
864	871
865	872	// Log to syslog.
...	...	@@ -970,8 +977,9 @@ static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES";
970	977	LowLevelAllocator allocator_for_env;
971	978
972	979	static bool ShouldCheckInterceptors() {
973		// Restrict "interceptors working?" check to ASan and TSan.
974		const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer"};
	980	// Restrict "interceptors working?" check
	981	const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer",
	982	"RealtimeSanitizer"};
975	983	size_t count = sizeof(sanitizer_names) / sizeof(sanitizer_names[0]);
976	984	for (size_t i = 0; i < count; i++) {
977	985	if (internal_strcmp(sanitizer_names[i], SanitizerToolName) == 0)

lib/tsan/sanitizer_common/sanitizer_platform_interceptors.h+45-7

...	...	@@ -84,6 +84,25 @@
84	84	#define SI_NOT_MAC 1
85	85	#endif
86	86
	87	#if SANITIZER_APPLE
	88	# include <Availability.h>
	89
	90	// aligned_alloc was introduced in OSX 10.15
	91	// Linking will fail when using an older SDK
	92	# if defined(__MAC_10_15)
	93	// macOS 10.15 is greater than our minimal deployment target. To ensure we
	94	// generate a weak reference so the dylib continues to work on older
	95	// systems, we need to forward declare the intercepted function as "weak
	96	// imports".
	97	SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
	98	__sanitizer::usize __size);
	99	# define SI_MAC_SDK_10_15_AVAILABLE 1
	100	# else
	101	# define SI_MAC_SDK_10_15_AVAILABLE 0
	102	# endif // defined(__MAC_10_15)
	103
	104	#endif // SANITIZER_APPLE
	105
87	106	#if SANITIZER_IOS
88	107	#define SI_IOS 1
89	108	#else
...	...	@@ -183,9 +202,16 @@
183	202	#define SANITIZER_INTERCEPT_FPUTS SI_POSIX
184	203	#define SANITIZER_INTERCEPT_PUTS SI_POSIX
185	204
	205	#define SANITIZER_INTERCEPT_CREAT64 (SI_GLIBC \|\| SI_SOLARIS32)
	206	#define SANITIZER_INTERCEPT_FCNTL64 (SI_GLIBC \|\| SI_SOLARIS32)
	207	#define SANITIZER_INTERCEPT_OPEN64 (SI_GLIBC \|\| SI_SOLARIS32)
	208	#define SANITIZER_INTERCEPT_OPENAT64 (SI_GLIBC \|\| SI_SOLARIS32)
	209
186	210	#define SANITIZER_INTERCEPT_PREAD64 (SI_GLIBC \|\| SI_SOLARIS32)
187	211	#define SANITIZER_INTERCEPT_PWRITE64 (SI_GLIBC \|\| SI_SOLARIS32)
188	212
	213	#define SANITIZER_INTERCEPT_LSEEK64 (SI_GLIBC \|\| SI_SOLARIS32)
	214
189	215	#define SANITIZER_INTERCEPT_READV SI_POSIX
190	216	#define SANITIZER_INTERCEPT_WRITEV SI_POSIX
191	217
...	...	@@ -232,8 +258,12 @@
232	258	(SI_FREEBSD \|\| SI_NETBSD \|\| SI_LINUX \|\| SI_SOLARIS)
233	259	#define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID \
234	260	(SI_LINUX \|\| SI_FREEBSD \|\| SI_NETBSD)
	261	// TODO: This should be SI_POSIX, adding glibc first until I have time
	262	// to verify all timer_t typedefs on other platforms.
	263	#define SANITIZER_INTERCEPT_TIMER_CREATE SI_GLIBC
235	264	#define SANITIZER_INTERCEPT_GETITIMER SI_POSIX
236	265	#define SANITIZER_INTERCEPT_TIME SI_POSIX
	266	#define SANITIZER_INTERCEPT_TIMESPEC_GET SI_LINUX
237	267	#define SANITIZER_INTERCEPT_GLOB (SI_GLIBC \|\| SI_SOLARIS)
238	268	#define SANITIZER_INTERCEPT_GLOB64 SI_GLIBC
239	269	#define SANITIZER_INTERCEPT___B64_TO SI_LINUX_NOT_ANDROID
...	...	@@ -274,8 +304,9 @@
274	304	#if SI_LINUX_NOT_ANDROID && \
275	305	(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \
276	306	defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__arm__) \|\| \
277		defined(__s390__) \|\| defined(__loongarch__) \|\| SANITIZER_RISCV64)
278		#define SANITIZER_INTERCEPT_PTRACE 1
	307	defined(__s390__) \|\| defined(__loongarch__) \|\| SANITIZER_RISCV64 \|\| \
	308	defined(__sparc__))
	309	# define SANITIZER_INTERCEPT_PTRACE 1
279	310	#else
280	311	#define SANITIZER_INTERCEPT_PTRACE 0
281	312	#endif
...	...	@@ -314,6 +345,8 @@
314	345	#define SANITIZER_INTERCEPT_GETGROUPS SI_POSIX
315	346	#define SANITIZER_INTERCEPT_POLL SI_POSIX
316	347	#define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS
	348	#define SANITIZER_INTERCEPT_EPOLL (SI_LINUX)
	349	#define SANITIZER_INTERCEPT_KQUEUE (SI_FREEBSD \|\| SI_NETBSD \|\| SI_MAC)
317	350	#define SANITIZER_INTERCEPT_WORDEXP \
318	351	(SI_FREEBSD \|\| SI_NETBSD \|\| (SI_MAC && !SI_IOS) \|\| SI_LINUX_NOT_ANDROID \|\| \
319	352	SI_SOLARIS)
...	...	@@ -494,7 +527,8 @@
494	527	#define SANITIZER_INTERCEPT_PVALLOC (SI_GLIBC \|\| SI_ANDROID)
495	528	#define SANITIZER_INTERCEPT_CFREE (SI_GLIBC && !SANITIZER_RISCV64)
496	529	#define SANITIZER_INTERCEPT_REALLOCARRAY SI_POSIX
497		#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC)
	530	#define SANITIZER_INTERCEPT_ALIGNED_ALLOC \
	531	(!SI_MAC \|\| SI_MAC_SDK_10_15_AVAILABLE)
498	532	#define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD)
499	533	#define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID
500	534	#define SANITIZER_INTERCEPT_WCSLEN 1
...	...	@@ -559,10 +593,8 @@
559	593	#define SANITIZER_INTERCEPT_SHA1 SI_NETBSD
560	594	#define SANITIZER_INTERCEPT_MD4 SI_NETBSD
561	595	#define SANITIZER_INTERCEPT_RMD160 SI_NETBSD
562		#define SANITIZER_INTERCEPT_MD5 (SI_NETBSD \|\| SI_FREEBSD)
563		#define SANITIZER_INTERCEPT_FSEEK (SI_NETBSD \|\| SI_FREEBSD)
	596	#define SANITIZER_INTERCEPT_FSEEK SI_POSIX
564	597	#define SANITIZER_INTERCEPT_MD2 SI_NETBSD
565		#define SANITIZER_INTERCEPT_SHA2 (SI_NETBSD \|\| SI_FREEBSD)
566	598	#define SANITIZER_INTERCEPT_CDB SI_NETBSD
567	599	#define SANITIZER_INTERCEPT_VIS (SI_NETBSD \|\| SI_FREEBSD)
568	600	#define SANITIZER_INTERCEPT_POPEN SI_POSIX
...	...	@@ -601,7 +633,13 @@
601	633	// FIXME: also available from musl 1.2.5
602	634	#define SANITIZER_INTERCEPT_PREADV2 (SI_LINUX && __GLIBC_PREREQ(2, 26))
603	635	#define SANITIZER_INTERCEPT_PWRITEV2 (SI_LINUX && __GLIBC_PREREQ(2, 26))
604
	636	#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && \
	637	__MAC_OS_X_VERSION_MIN_REQUIRED >= 130000
	638	# define SI_MAC_OS_DEPLOYMENT_MIN_13_00 1
	639	#else
	640	# define SI_MAC_OS_DEPLOYMENT_MIN_13_00 0
	641	#endif
	642	#define SANITIZER_INTERCEPT_FREADLINK (SI_MAC && SI_MAC_OS_DEPLOYMENT_MIN_13_00)
605	643	// This macro gives a way for downstream users to override the above
606	644	// interceptor macros irrespective of the platform they are on. They have
607	645	// to do two things:

lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.cpp+1-2

...	...	@@ -547,6 +547,7 @@ unsigned pid_t_sz = sizeof(pid_t);
547	547	unsigned timeval_sz = sizeof(timeval);
548	548	unsigned uid_t_sz = sizeof(uid_t);
549	549	unsigned gid_t_sz = sizeof(gid_t);
	550	unsigned fpos_t_sz = sizeof(fpos_t);
550	551	unsigned mbstate_t_sz = sizeof(mbstate_t);
551	552	unsigned sigset_t_sz = sizeof(sigset_t);
552	553	unsigned struct_timezone_sz = sizeof(struct timezone);
...	...	@@ -2487,8 +2488,6 @@ const unsigned RMD160_return_length = RMD160_DIGEST_STRING_LENGTH;
2487	2488	const unsigned MD5_CTX_sz = sizeof(MD5_CTX);
2488	2489	const unsigned MD5_return_length = MD5_DIGEST_STRING_LENGTH;
2489	2490
2490		const unsigned fpos_t_sz = sizeof(fpos_t);
2491
2492	2491	const unsigned MD2_CTX_sz = sizeof(MD2_CTX);
2493	2492	const unsigned MD2_return_length = MD2_DIGEST_STRING_LENGTH;
2494	2493

lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.h+1-2

...	...	@@ -36,6 +36,7 @@ extern unsigned pid_t_sz;
36	36	extern unsigned timeval_sz;
37	37	extern unsigned uid_t_sz;
38	38	extern unsigned gid_t_sz;
	39	extern unsigned fpos_t_sz;
39	40	extern unsigned mbstate_t_sz;
40	41	extern unsigned struct_timezone_sz;
41	42	extern unsigned struct_tms_sz;
...	...	@@ -2335,8 +2336,6 @@ extern const unsigned RMD160_return_length;
2335	2336	extern const unsigned MD5_CTX_sz;
2336	2337	extern const unsigned MD5_return_length;
2337	2338
2338		extern const unsigned fpos_t_sz;
2339
2340	2339	extern const unsigned MD2_CTX_sz;
2341	2340	extern const unsigned MD2_return_length;
2342	2341

lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.cpp+35-26

...	...	@@ -94,8 +94,9 @@
94	94	#if SANITIZER_LINUX
95	95	# include <utime.h>
96	96	# include <sys/ptrace.h>
97		# if defined(__mips64) \|\| defined(__aarch64__) \|\| defined(__arm__) \|\| \
98		defined(__hexagon__) \|\| defined(__loongarch__) \|\|SANITIZER_RISCV64
	97	# if defined(__mips64) \|\| defined(__aarch64__) \|\| defined(__arm__) \|\| \
	98	defined(__hexagon__) \|\| defined(__loongarch__) \|\| SANITIZER_RISCV64 \|\| \
	99	defined(__sparc__)
99	100	# include <asm/ptrace.h>
100	101	# ifdef __arm__
101	102	typedef struct user_fpregs elf_fpregset_t;
...	...	@@ -117,15 +118,16 @@ typedef struct user_fpregs elf_fpregset_t;
117	118	#if SANITIZER_LINUX
118	119	#if SANITIZER_GLIBC
119	120	#include <fstab.h>
120		#include <net/if_ppp.h>
121		#include <netax25/ax25.h>
122		#include <netipx/ipx.h>
123		#include <netrom/netrom.h>
124		#include <obstack.h>
125		#if HAVE_RPC_XDR_H
126		# include <rpc/xdr.h>
127		#endif
128		#include <scsi/scsi.h>
	121	# include <linux/filter.h>
	122	# include <net/if_ppp.h>
	123	# include <netax25/ax25.h>
	124	# include <netipx/ipx.h>
	125	# include <netrom/netrom.h>
	126	# include <obstack.h>
	127	# if HAVE_RPC_XDR_H
	128	# include <rpc/xdr.h>
	129	# endif
	130	# include <scsi/scsi.h>
129	131	#else
130	132	#include <linux/if_ppp.h>
131	133	#include <linux/kd.h>
...	...	@@ -358,11 +360,12 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
358	360	const int wordexp_wrde_dooffs = WRDE_DOOFFS;
359	361	# endif // !SANITIZER_ANDROID
360	362
361		#if SANITIZER_LINUX && !SANITIZER_ANDROID && \
362		(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \
363		defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__arm__) \|\| \
364		defined(__s390__) \|\| defined(__loongarch__)\|\| SANITIZER_RISCV64)
365		#if defined(__mips64) \|\| defined(__powerpc64__) \|\| defined(__arm__)
	363	# if SANITIZER_LINUX && !SANITIZER_ANDROID && \
	364	(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \
	365	defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__arm__) \|\| \
	366	defined(__s390__) \|\| defined(__loongarch__) \|\| SANITIZER_RISCV64 \|\| \
	367	defined(__sparc__))
	368	# if defined(__mips64) \|\| defined(__powerpc64__) \|\| defined(__arm__)
366	369	unsigned struct_user_regs_struct_sz = sizeof(struct pt_regs);
367	370	unsigned struct_user_fpregs_struct_sz = sizeof(elf_fpregset_t);
368	371	#elif SANITIZER_RISCV64
...	...	@@ -377,19 +380,22 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
377	380	#elif defined(__s390__)
378	381	unsigned struct_user_regs_struct_sz = sizeof(struct _user_regs_struct);
379	382	unsigned struct_user_fpregs_struct_sz = sizeof(struct _user_fpregs_struct);
380		#else
	383	# elif defined(__sparc__)
	384	unsigned struct_user_regs_struct_sz = sizeof(struct sunos_regs);
	385	unsigned struct_user_fpregs_struct_sz = sizeof(struct sunos_fp);
	386	# else
381	387	unsigned struct_user_regs_struct_sz = sizeof(struct user_regs_struct);
382	388	unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpregs_struct);
383		#endif // __mips64 \|\| __powerpc64__ \|\| __aarch64__ \|\| __loongarch__
384		#if defined(__x86_64) \|\| defined(__mips64) \|\| defined(__powerpc64__) \|\| \
385		defined(__aarch64__) \|\| defined(__arm__) \|\| defined(__s390__) \|\| \
386		defined(__loongarch__) \|\| SANITIZER_RISCV64
	389	# endif // __mips64 \|\| __powerpc64__ \|\| __aarch64__ \|\| __loongarch__
	390	# if defined(__x86_64) \|\| defined(__mips64) \|\| defined(__powerpc64__) \|\| \
	391	defined(__aarch64__) \|\| defined(__arm__) \|\| defined(__s390__) \|\| \
	392	defined(__loongarch__) \|\| SANITIZER_RISCV64 \|\| defined(__sparc__)
387	393	unsigned struct_user_fpxregs_struct_sz = 0;
388	394	#else
389	395	unsigned struct_user_fpxregs_struct_sz = sizeof(struct user_fpxregs_struct);
390	396	#endif // __x86_64 \|\| __mips64 \|\| __powerpc64__ \|\| __aarch64__ \|\| __arm__
391		// \|\| __s390__ \|\| __loongarch__
392		#ifdef __arm__
	397	// \|\| __s390__ \|\| __loongarch__ \|\| SANITIZER_RISCV64 \|\| __sparc__
	398	# ifdef __arm__
393	399	unsigned struct_user_vfpregs_struct_sz = ARM_VFPREGS_SIZE;
394	400	#else
395	401	unsigned struct_user_vfpregs_struct_sz = 0;
...	...	@@ -531,13 +537,16 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
531	537
532	538	unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info);
533	539	unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats);
534		#endif // SANITIZER_GLIBC
	540	unsigned struct_sock_fprog_sz = sizeof(struct sock_fprog);
	541	# endif // SANITIZER_GLIBC
535	542
536		#if !SANITIZER_ANDROID && !SANITIZER_APPLE
	543	# if !SANITIZER_ANDROID && !SANITIZER_APPLE
537	544	unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
538	545	unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
539	546	#endif
540	547
	548	unsigned fpos_t_sz = sizeof(fpos_t);
	549
541	550	const unsigned long __sanitizer_bufsiz = BUFSIZ;
542	551
543	552	const unsigned IOCTL_NOT_PRESENT = 0;
...	...	@@ -1084,7 +1093,7 @@ CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_len);
1084	1093	CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_level);
1085	1094	CHECK_SIZE_AND_OFFSET(cmsghdr, cmsg_type);
1086	1095
1087		#if SANITIZER_LINUX && (__ANDROID_API__ >= 21 \|\| __GLIBC_PREREQ (2, 14))
	1096	# if SANITIZER_LINUX && (SANITIZER_ANDROID \|\| __GLIBC_PREREQ(2, 14))
1088	1097	CHECK_TYPE_SIZE(mmsghdr);
1089	1098	CHECK_SIZE_AND_OFFSET(mmsghdr, msg_hdr);
1090	1099	CHECK_SIZE_AND_OFFSET(mmsghdr, msg_len);

lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.h+49-13

...	...	@@ -98,10 +98,13 @@ const unsigned struct_kernel_stat64_sz = 104;
98	98	const unsigned struct_kernel_stat_sz = 144;
99	99	const unsigned struct_kernel_stat64_sz = 104;
100	100	#elif defined(__mips__)
101		const unsigned struct_kernel_stat_sz =
102		SANITIZER_ANDROID
103		? FIRST_32_SECOND_64(104, 128)
104		: FIRST_32_SECOND_64((_MIPS_SIM == _ABIN32) ? 176 : 160, 216);
	101	const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID
	102	? FIRST_32_SECOND_64(104, 128)
	103	# if defined(_ABIN32) && _MIPS_SIM == _ABIN32
	104	: FIRST_32_SECOND_64(176, 216);
	105	# else
	106	: FIRST_32_SECOND_64(160, 216);
	107	# endif
105	108	const unsigned struct_kernel_stat64_sz = 104;
106	109	#elif defined(__s390__) && !defined(__s390x__)
107	110	const unsigned struct_kernel_stat_sz = 64;
...	...	@@ -313,7 +316,7 @@ extern unsigned struct_statvfs_sz;
313	316
314	317	struct __sanitizer_iovec {
315	318	void *iov_base;
316		uptr iov_len;
	319	usize iov_len;
317	320	};
318	321
319	322	#if !SANITIZER_ANDROID
...	...	@@ -389,6 +392,16 @@ typedef long __sanitizer_time_t;
389	392
390	393	typedef long __sanitizer_suseconds_t;
391	394
	395	struct __sanitizer_timespec {
	396	__sanitizer_time_t tv_sec; /* seconds */
	397	u64 tv_nsec; /* nanoseconds */
	398	};
	399
	400	struct __sanitizer_itimerspec {
	401	struct __sanitizer_timespec it_interval; /* timer period */
	402	struct __sanitizer_timespec it_value; /* timer expiration */
	403	};
	404
392	405	struct __sanitizer_timeval {
393	406	__sanitizer_time_t tv_sec;
394	407	__sanitizer_suseconds_t tv_usec;
...	...	@@ -513,6 +526,7 @@ struct __sanitizer_dirent64 {
513	526	unsigned short d_reclen;
514	527	// more fields that we don't care about
515	528	};
	529	extern unsigned struct_sock_fprog_sz;
516	530	#endif
517	531
518	532	#if defined(__x86_64__) && !defined(_LP64)
...	...	@@ -590,7 +604,7 @@ struct __sanitizer_siginfo_pad {
590	604	#if SANITIZER_LINUX
591	605	# define SANITIZER_HAS_SIGINFO 1
592	606	union __sanitizer_siginfo {
593		struct {
	607	__extension__ struct {
594	608	int si_signo;
595	609	# if SANITIZER_MIPS
596	610	int si_code;
...	...	@@ -855,10 +869,11 @@ typedef void __sanitizer_FILE;
855	869	# define SANITIZER_HAS_STRUCT_FILE 0
856	870	#endif
857	871
858		#if SANITIZER_LINUX && !SANITIZER_ANDROID && \
859		(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \
860		defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__arm__) \|\| \
861		defined(__s390__) \|\| defined(__loongarch__) \|\| SANITIZER_RISCV64)
	872	# if SANITIZER_LINUX && !SANITIZER_ANDROID && \
	873	(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \
	874	defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__arm__) \|\| \
	875	defined(__s390__) \|\| defined(__loongarch__) \|\| SANITIZER_RISCV64 \|\| \
	876	defined(__sparc__))
862	877	extern unsigned struct_user_regs_struct_sz;
863	878	extern unsigned struct_user_fpregs_struct_sz;
864	879	extern unsigned struct_user_fpxregs_struct_sz;
...	...	@@ -880,9 +895,24 @@ extern int ptrace_setsiginfo;
880	895	extern int ptrace_getregset;
881	896	extern int ptrace_setregset;
882	897	extern int ptrace_geteventmsg;
883		#endif
884	898
885		#if SANITIZER_LINUX && !SANITIZER_ANDROID
	899	// Helper for the ptrace interceptor.
	900	template <class T>
	901	inline T ptrace_data_arg(int request, T addr, T data) {
	902	# if SANITIZER_LINUX && SANITIZER_SPARC
	903	// As described in ptrace(2), the meanings of addr and data are reversed
	904	// for the PTRACE_GETREGS, PTRACE_GETFPREGS, PTRACE_GETREGS, and
	905	// PTRACE_GETFPREGS requests on Linux/sparc64.
	906	if (request == ptrace_getregs \|\| request == ptrace_getfpregs \|\|
	907	request == ptrace_setregs \|\| request == ptrace_setfpregs)
	908	return addr;
	909	else
	910	# endif
	911	return data;
	912	}
	913	# endif
	914
	915	# if SANITIZER_LINUX && !SANITIZER_ANDROID
886	916	extern unsigned struct_shminfo_sz;
887	917	extern unsigned struct_shm_info_sz;
888	918	extern int shmctl_ipc_stat;
...	...	@@ -1050,7 +1080,7 @@ extern unsigned struct_serial_struct_sz;
1050	1080	extern unsigned struct_sockaddr_ax25_sz;
1051	1081	extern unsigned struct_unimapdesc_sz;
1052	1082	extern unsigned struct_unimapinit_sz;
1053		#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
	1083	# endif // SANITIZER_LINUX && !SANITIZER_ANDROID
1054	1084
1055	1085	extern const unsigned long __sanitizer_bufsiz;
1056	1086
...	...	@@ -1064,6 +1094,8 @@ extern unsigned struct_sioc_sg_req_sz;
1064	1094	extern unsigned struct_sioc_vif_req_sz;
1065	1095	#endif
1066	1096
	1097	extern unsigned fpos_t_sz;
	1098
1067	1099	// ioctl request identifiers
1068	1100
1069	1101	// A special value to mark ioctls that are not present on the target platform,
...	...	@@ -1500,6 +1532,10 @@ extern const int si_SEGV_ACCERR;
1500	1532
1501	1533	#define SIGACTION_SYMNAME sigaction
1502	1534
	1535	# if SANITIZER_LINUX
	1536	typedef void *__sanitizer_timer_t;
	1537	# endif
	1538
1503	1539	#endif // SANITIZER_LINUX \|\| SANITIZER_APPLE
1504	1540
1505	1541	#endif

lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.cpp+3

...	...	@@ -32,6 +32,7 @@
32	32	#include <semaphore.h>
33	33	#include <signal.h>
34	34	#include <stddef.h>
	35	#include <stdio.h>
35	36	#include <sys/ethernet.h>
36	37	#include <sys/filio.h>
37	38	#include <sys/ipc.h>
...	...	@@ -135,6 +136,8 @@ namespace __sanitizer {
135	136	unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
136	137	unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
137	138
	139	unsigned fpos_t_sz = sizeof(fpos_t);
	140
138	141	const unsigned IOCTL_NOT_PRESENT = 0;
139	142
140	143	unsigned IOCTL_FIOASYNC = FIOASYNC;

lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.h+2

...	...	@@ -418,6 +418,8 @@ extern unsigned struct_winsize_sz;
418	418	extern unsigned struct_sioc_sg_req_sz;
419	419	extern unsigned struct_sioc_vif_req_sz;
420	420
	421	extern unsigned fpos_t_sz;
	422
421	423	// ioctl request identifiers
422	424
423	425	// A special value to mark ioctls that are not present on the target platform,

lib/tsan/sanitizer_common/sanitizer_posix.cpp+9-1

...	...	@@ -353,7 +353,15 @@ bool ShouldMockFailureToOpen(const char *path) {
353	353	internal_strncmp(path, "/proc/", 6) == 0;
354	354	}
355	355
356		#if SANITIZER_LINUX && !SANITIZER_ANDROID && !SANITIZER_GO
	356	bool OpenReadsVaArgs(int oflag) {
	357	# ifdef O_TMPFILE
	358	return (oflag & (O_CREAT \| O_TMPFILE)) != 0;
	359	# else
	360	return (oflag & O_CREAT) != 0;
	361	# endif
	362	}
	363
	364	# if SANITIZER_LINUX && !SANITIZER_ANDROID && !SANITIZER_GO
357	365	int GetNamedMappingFd(const char name, uptr size, int flags) {
358	366	if (!common_flags()->decorate_proc_maps \|\| !name)
359	367	return -1;

lib/tsan/sanitizer_common/sanitizer_posix.h+5-1

...	...	@@ -28,6 +28,9 @@ namespace __sanitizer {
28	28	// Don't use directly, use __sanitizer::OpenFile() instead.
29	29	uptr internal_open(const char *filename, int flags);
30	30	uptr internal_open(const char *filename, int flags, u32 mode);
	31	# if SANITIZER_FREEBSD
	32	uptr internal_close_range(fd_t lowfd, fd_t highfd, int flags);
	33	# endif
31	34	uptr internal_close(fd_t fd);
32	35
33	36	uptr internal_read(fd_t fd, void *buf, uptr count);
...	...	@@ -86,7 +89,7 @@ int internal_pthread_join(void th, void *ret);
86	89	return REAL(pthread_create)(th, attr, callback, param); \
87	90	} \
88	91	int internal_pthread_join(void th, void *ret) { \
89		return REAL(pthread_join(th, ret)); \
	92	return REAL(pthread_join)(th, ret); \
90	93	} \
91	94	} // namespace __sanitizer
92	95
...	...	@@ -108,6 +111,7 @@ bool IsStateDetached(int state);
108	111	fd_t ReserveStandardFds(fd_t fd);
109	112
110	113	bool ShouldMockFailureToOpen(const char *path);
	114	bool OpenReadsVaArgs(int oflag);
111	115
112	116	// Create a non-file mapping with a given /proc/self/maps name.
113	117	uptr MmapNamed(void addr, uptr length, int prot, int flags, const char name);

lib/tsan/sanitizer_common/sanitizer_posix_libcdep.cpp+82-18

...	...	@@ -288,26 +288,86 @@ bool SignalContext::IsStackOverflow() const {
288	288
289	289	#endif // SANITIZER_GO
290	290
	291	static void SetNonBlock(int fd) {
	292	int res = fcntl(fd, F_GETFL, 0);
	293	CHECK(!internal_iserror(res, nullptr));
	294
	295	res \|= O_NONBLOCK;
	296	res = fcntl(fd, F_SETFL, res);
	297	CHECK(!internal_iserror(res, nullptr));
	298	}
	299
291	300	bool IsAccessibleMemoryRange(uptr beg, uptr size) {
292		uptr page_size = GetPageSizeCached();
293		// Checking too large memory ranges is slow.
294		CHECK_LT(size, page_size * 10);
295		int sock_pair[2];
296		if (pipe(sock_pair))
297		return false;
298		uptr bytes_written =
299		internal_write(sock_pair[1], reinterpret_cast<void *>(beg), size);
300		int write_errno;
301		bool result;
302		if (internal_iserror(bytes_written, &write_errno)) {
303		CHECK_EQ(EFAULT, write_errno);
304		result = false;
305		} else {
306		result = (bytes_written == size);
	301	while (size) {
	302	// `read` from `fds[0]` into a dummy buffer to free up the pipe buffer for
	303	// more `write` is slower than just recreating a pipe.
	304	int fds[2];
	305	CHECK_EQ(0, pipe(fds));
	306
	307	auto cleanup = at_scope_exit([&]() {
	308	internal_close(fds[0]);
	309	internal_close(fds[1]);
	310	});
	311
	312	SetNonBlock(fds[1]);
	313
	314	int write_errno;
	315	uptr w = internal_write(fds[1], reinterpret_cast<char *>(beg), size);
	316	if (internal_iserror(w, &write_errno)) {
	317	if (write_errno == EINTR)
	318	continue;
	319	CHECK_EQ(EFAULT, write_errno);
	320	return false;
	321	}
	322	size -= w;
	323	beg += w;
	324	}
	325
	326	return true;
	327	}
	328
	329	bool TryMemCpy(void dest, const void src, uptr n) {
	330	if (!n)
	331	return true;
	332	int fds[2];
	333	CHECK_EQ(0, pipe(fds));
	334
	335	auto cleanup = at_scope_exit([&]() {
	336	internal_close(fds[0]);
	337	internal_close(fds[1]);
	338	});
	339
	340	SetNonBlock(fds[0]);
	341	SetNonBlock(fds[1]);
	342
	343	char d = static_cast<char >(dest);
	344	const char s = static_cast<const char >(src);
	345
	346	while (n) {
	347	int e;
	348	uptr w = internal_write(fds[1], s, n);
	349	if (internal_iserror(w, &e)) {
	350	if (e == EINTR)
	351	continue;
	352	CHECK_EQ(EFAULT, e);
	353	return false;
	354	}
	355	s += w;
	356	n -= w;
	357
	358	while (w) {
	359	uptr r = internal_read(fds[0], d, w);
	360	if (internal_iserror(r, &e)) {
	361	CHECK_EQ(EINTR, e);
	362	continue;
	363	}
	364
	365	d += r;
	366	w -= r;
	367	}
307	368	}
308		internal_close(sock_pair[0]);
309		internal_close(sock_pair[1]);
310		return result;
	369
	370	return true;
311	371	}
312	372
313	373	void PlatformPrepareForSandboxing(void *args) {
...	...	@@ -483,7 +543,11 @@ pid_t StartSubprocess(const char program, const char const argv[],
483	543	internal_close(stderr_fd);
484	544	}
485	545
	546	# if SANITIZER_FREEBSD
	547	internal_close_range(3, ~static_cast<fd_t>(0), 0);
	548	# else
486	549	for (int fd = sysconf(_SC_OPEN_MAX); fd > 2; fd--) internal_close(fd);
	550	# endif
487	551
488	552	internal_execve(program, const_cast<char **>(&argv[0]),
489	553	const_cast<char const >(envp));

lib/tsan/sanitizer_common/sanitizer_procmaps_mac.cpp+3-1

...	...	@@ -433,7 +433,9 @@ void MemoryMappingLayout::DumpListOfModules(
433	433	MemoryMappedSegmentData data;
434	434	segment.data_ = &data;
435	435	while (Next(&segment)) {
436		if (segment.filename[0] == '\0') continue;
	436	// skip the __PAGEZERO segment, its vmsize is 0
	437	if (segment.filename[0] == '\0' \|\| (segment.start == segment.end))
	438	continue;
437	439	LoadedModule *cur_module = nullptr;
438	440	if (!modules->empty() &&
439	441	0 == internal_strcmp(segment.filename, modules->back().full_name())) {

lib/tsan/sanitizer_common/sanitizer_procmaps_solaris.cpp+4

...	...	@@ -11,6 +11,10 @@
11	11
12	12	// Before Solaris 11.4, <procfs.h> doesn't work in a largefile environment.
13	13	#undef _FILE_OFFSET_BITS
	14
	15	// Avoid conflict between `_TIME_BITS` defined vs. `_FILE_OFFSET_BITS`
	16	// undefined in some Linux configurations.
	17	#undef _TIME_BITS
14	18	#include "sanitizer_platform.h"
15	19	#if SANITIZER_SOLARIS
16	20	# include <fcntl.h>

lib/tsan/sanitizer_common/sanitizer_redefine_builtins.h+5-3

...	...	@@ -17,9 +17,11 @@
17	17	// The asm hack only works with GCC and Clang.
18	18	# if !defined(_WIN32)
19	19
20		asm("memcpy = __sanitizer_internal_memcpy");
21		asm("memmove = __sanitizer_internal_memmove");
22		asm("memset = __sanitizer_internal_memset");
	20	asm(R"(
	21	.set memcpy, __sanitizer_internal_memcpy
	22	.set memmove, __sanitizer_internal_memmove
	23	.set memset, __sanitizer_internal_memset
	24	)");
23	25
24	26	# if defined(__cplusplus) && \
25	27	!defined(SANITIZER_COMMON_REDEFINE_BUILTINS_IN_STD)

lib/tsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp+11-4

...	...	@@ -137,10 +137,6 @@ class ThreadSuspender {
137	137	};
138	138
139	139	bool ThreadSuspender::SuspendThread(tid_t tid) {
140		// Are we already attached to this thread?
141		// Currently this check takes linear time, however the number of threads is
142		// usually small.
143		if (suspended_threads_list_.ContainsTid(tid)) return false;
144	140	int pterrno;
145	141	if (internal_iserror(internal_ptrace(PTRACE_ATTACH, tid, nullptr, nullptr),
146	142	&pterrno)) {
...	...	@@ -217,17 +213,28 @@ bool ThreadSuspender::SuspendAllThreads() {
217	213	switch (thread_lister.ListThreads(&threads)) {
218	214	case ThreadLister::Error:
219	215	ResumeAllThreads();
	216	VReport(1, "Failed to list threads\n");
220	217	return false;
221	218	case ThreadLister::Incomplete:
	219	VReport(1, "Incomplete list\n");
222	220	retry = true;
223	221	break;
224	222	case ThreadLister::Ok:
225	223	break;
226	224	}
227	225	for (tid_t tid : threads) {
	226	// Are we already attached to this thread?
	227	// Currently this check takes linear time, however the number of threads
	228	// is usually small.
	229	if (suspended_threads_list_.ContainsTid(tid))
	230	continue;
228	231	if (SuspendThread(tid))
229	232	retry = true;
	233	else
	234	VReport(2, "%llu/status: %s\n", tid, thread_lister.LoadStatus(tid));
230	235	}
	236	if (retry)
	237	VReport(1, "SuspendAllThreads retry: %d\n", i);
231	238	}
232	239	return suspended_threads_list_.ThreadCount();
233	240	}

lib/tsan/sanitizer_common/sanitizer_symbolizer.h+4-4

...	...	@@ -185,17 +185,17 @@ class Symbolizer final {
185	185	class ModuleNameOwner {
186	186	public:
187	187	explicit ModuleNameOwner(Mutex *synchronized_by)
188		: last_match_(nullptr), mu_(synchronized_by) {
	188	: mu_(synchronized_by), last_match_(nullptr) {
189	189	storage_.reserve(kInitialCapacity);
190	190	}
191	191	const char GetOwnedCopy(const char str);
192	192
193	193	private:
194	194	static const uptr kInitialCapacity = 1000;
195		InternalMmapVector<const char*> storage_;
196		const char *last_match_;
197	195
198	196	Mutex *mu_;
	197	const char *last_match_ SANITIZER_GUARDED_BY(mu_);
	198	InternalMmapVector<const char > storage_ SANITIZER_GUARDED_BY(mu_);
199	199	} module_names_;
200	200
201	201	/// Platform-specific function for creating a Symbolizer object.
...	...	@@ -220,7 +220,7 @@ class Symbolizer final {
220	220	// always synchronized.
221	221	Mutex mu_;
222	222
223		IntrusiveList<SymbolizerTool> tools_;
	223	IntrusiveList<SymbolizerTool> tools_ SANITIZER_GUARDED_BY(mu_);
224	224
225	225	explicit Symbolizer(IntrusiveList<SymbolizerTool> tools);
226	226

lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.cpp+2-2

...	...	@@ -30,7 +30,7 @@ namespace __sanitizer {
30	30	bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
31	31	Dl_info info;
32	32	int result = dladdr((const void *)addr, &info);
33		if (!result) return false;
	33	if (!result \|\| !info.dli_sname) return false;
34	34
35	35	// Compute offset if possible. `dladdr()` doesn't always ensure that `addr >=
36	36	// sym_addr` so only compute the offset when this holds. Failure to find the
...	...	@@ -51,7 +51,7 @@ bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
51	51	bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) {
52	52	Dl_info info;
53	53	int result = dladdr((const void *)addr, &info);
54		if (!result) return false;
	54	if (!result \|\| !info.dli_sname) return false;
55	55	const char *demangled = DemangleSwiftAndCXX(info.dli_sname);
56	56	if (!demangled)
57	57	demangled = info.dli_sname;

lib/tsan/sanitizer_common/sanitizer_symbolizer_report.cpp+9-6

...	...	@@ -227,12 +227,15 @@ static void ReportStackOverflowImpl(const SignalContext &sig, u32 tid,
227	227	SanitizerToolName, kDescription, (void )sig.addr, (void )sig.pc,
228	228	(void )sig.bp, (void )sig.sp, tid);
229	229	Printf("%s", d.Default());
230		InternalMmapVector<BufferedStackTrace> stack_buffer(1);
231		BufferedStackTrace *stack = stack_buffer.data();
232		stack->Reset();
233		unwind(sig, unwind_context, stack);
234		stack->Print();
235		ReportErrorSummary(kDescription, stack);
	230	// Avoid SEGVs in the unwinder when bp couldn't be determined.
	231	if (sig.bp) {
	232	InternalMmapVector<BufferedStackTrace> stack_buffer(1);
	233	BufferedStackTrace *stack = stack_buffer.data();
	234	stack->Reset();
	235	unwind(sig, unwind_context, stack);
	236	stack->Print();
	237	ReportErrorSummary(kDescription, stack);
	238	}
236	239	}
237	240
238	241	static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid,

lib/tsan/sanitizer_common/sanitizer_symbolizer_win.cpp+7-6

...	...	@@ -65,12 +65,13 @@ void InitializeDbgHelpIfNeeded() {
65	65	HMODULE dbghelp = LoadLibraryA("dbghelp.dll");
66	66	CHECK(dbghelp && "failed to load dbghelp.dll");
67	67
68		#define DBGHELP_IMPORT(name) \
69		do { \
70		name = \
71		reinterpret_cast<decltype(::name) *>(GetProcAddress(dbghelp, #name)); \
72		CHECK(name != nullptr); \
73		} while (0)
	68	# define DBGHELP_IMPORT(name) \
	69	do { \
	70	name = reinterpret_cast<decltype(::name) *>( \
	71	(void *)GetProcAddress(dbghelp, #name)); \
	72	CHECK(name != nullptr); \
	73	} while (0)
	74
74	75	DBGHELP_IMPORT(StackWalk64);
75	76	DBGHELP_IMPORT(SymCleanup);
76	77	DBGHELP_IMPORT(SymFromAddr);

lib/tsan/sanitizer_common/sanitizer_thread_history.cpp created+72

...	...	@@ -0,0 +1,72 @@
	1	//===-- sanitizer_thread_history.cpp --------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#include "sanitizer_thread_history.h"
	10
	11	#include "sanitizer_stackdepot.h"
	12	namespace __sanitizer {
	13
	14	void PrintThreadHistory(ThreadRegistry &registry, InternalScopedString &out) {
	15	ThreadRegistryLock l(&registry);
	16	// Stack traces are largest part of printout and they often the same for
	17	// multiple threads, so we will deduplicate them.
	18	InternalMmapVector<const ThreadContextBase *> stacks;
	19
	20	registry.RunCallbackForEachThreadLocked(
	21	[](ThreadContextBase context, void arg) {
	22	static_cast<decltype(&stacks)>(arg)->push_back(context);
	23	},
	24	&stacks);
	25
	26	Sort(stacks.data(), stacks.size(),
	27	[](const ThreadContextBase a, const ThreadContextBase b) {
	28	if (a->stack_id < b->stack_id)
	29	return true;
	30	if (a->stack_id > b->stack_id)
	31	return false;
	32	return a->unique_id < b->unique_id;
	33	});
	34
	35	auto describe_thread = [&](const ThreadContextBase *context) {
	36	if (!context) {
	37	out.Append("T-1");
	38	return;
	39	}
	40	out.AppendF("T%llu/%llu", context->unique_id, context->os_id);
	41	if (internal_strlen(context->name))
	42	out.AppendF(" (%s)", context->name);
	43	};
	44
	45	auto get_parent =
	46	[&](const ThreadContextBase context) -> const ThreadContextBase {
	47	if (!context)
	48	return nullptr;
	49	ThreadContextBase *parent = registry.GetThreadLocked(context->parent_tid);
	50	if (!parent)
	51	return nullptr;
	52	if (parent->unique_id >= context->unique_id)
	53	return nullptr;
	54	return parent;
	55	};
	56
	57	const ThreadContextBase *prev = nullptr;
	58	for (const ThreadContextBase *context : stacks) {
	59	if (prev && prev->stack_id != context->stack_id)
	60	StackDepotGet(prev->stack_id).PrintTo(&out);
	61	prev = context;
	62	out.Append("Thread ");
	63	describe_thread(context);
	64	out.Append(" was created by ");
	65	describe_thread(get_parent(context));
	66	out.Append("\n");
	67	}
	68	if (prev)
	69	StackDepotGet(prev->stack_id).PrintTo(&out);
	70	}
	71
	72	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_thread_history.h created+24

...	...	@@ -0,0 +1,24 @@
	1	//===-- sanitizer_thread_history.h ------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// Utility to print thread histroy from ThreadRegistry.
	10	//
	11	//===----------------------------------------------------------------------===//
	12
	13	#ifndef SANITIZER_THREAD_HISTORY_H
	14	#define SANITIZER_THREAD_HISTORY_H
	15
	16	#include "sanitizer_thread_registry.h"
	17
	18	namespace __sanitizer {
	19
	20	void PrintThreadHistory(ThreadRegistry& registry, InternalScopedString& out);
	21
	22	} // namespace __sanitizer
	23
	24	#endif // SANITIZER_THREAD_HISTORY_H

lib/tsan/sanitizer_common/sanitizer_thread_registry.cpp+33-18

...	...	@@ -18,9 +18,17 @@
18	18	namespace __sanitizer {
19	19
20	20	ThreadContextBase::ThreadContextBase(u32 tid)
21		: tid(tid), unique_id(0), reuse_count(), os_id(0), user_id(0),
22		status(ThreadStatusInvalid), detached(false),
23		thread_type(ThreadType::Regular), parent_tid(0), next(0) {
	21	: tid(tid),
	22	unique_id(0),
	23	reuse_count(),
	24	os_id(0),
	25	user_id(0),
	26	status(ThreadStatusInvalid),
	27	detached(false),
	28	thread_type(ThreadType::Regular),
	29	parent_tid(0),
	30	stack_id(0),
	31	next(0) {
24	32	name[0] = '\0';
25	33	atomic_store(&thread_destroyed, 0, memory_order_release);
26	34	}
...	...	@@ -39,8 +47,7 @@ void ThreadContextBase::SetName(const char *new_name) {
39	47	}
40	48
41	49	void ThreadContextBase::SetDead() {
42		CHECK(status == ThreadStatusRunning \|\|
43		status == ThreadStatusFinished);
	50	CHECK(status == ThreadStatusRunning \|\| status == ThreadStatusFinished);
44	51	status = ThreadStatusDead;
45	52	user_id = 0;
46	53	OnDead();
...	...	@@ -68,7 +75,8 @@ void ThreadContextBase::SetFinished() {
68	75	// for a thread that never actually started. In that case the thread
69	76	// should go to ThreadStatusFinished regardless of whether it was created
70	77	// as detached.
71		if (!detached \|\| status == ThreadStatusCreated) status = ThreadStatusFinished;
	78	if (!detached \|\| status == ThreadStatusCreated)
	79	status = ThreadStatusFinished;
72	80	OnFinished();
73	81	}
74	82
...	...	@@ -81,14 +89,17 @@ void ThreadContextBase::SetStarted(tid_t _os_id, ThreadType _thread_type,
81	89	}
82	90
83	91	void ThreadContextBase::SetCreated(uptr _user_id, u64 _unique_id,
84		bool _detached, u32 _parent_tid, void *arg) {
	92	bool _detached, u32 _parent_tid,
	93	u32 _stack_tid, void *arg) {
85	94	status = ThreadStatusCreated;
86	95	user_id = _user_id;
87	96	unique_id = _unique_id;
88	97	detached = _detached;
89	98	// Parent tid makes no sense for the main thread.
90		if (tid != kMainTid)
	99	if (tid != kMainTid) {
91	100	parent_tid = _parent_tid;
	101	stack_id = _stack_tid;
	102	}
92	103	OnCreated(arg);
93	104	}
94	105
...	...	@@ -124,8 +135,10 @@ void ThreadRegistry::GetNumberOfThreads(uptr total, uptr running,
124	135	ThreadRegistryLock l(this);
125	136	if (total)
126	137	*total = threads_.size();
127		if (running) *running = running_threads_;
128		if (alive) *alive = alive_threads_;
	138	if (running)
	139	*running = running_threads_;
	140	if (alive)
	141	*alive = alive_threads_;
129	142	}
130	143
131	144	uptr ThreadRegistry::GetMaxAliveThreads() {
...	...	@@ -134,7 +147,7 @@ uptr ThreadRegistry::GetMaxAliveThreads() {
134	147	}
135	148
136	149	u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
137		void *arg) {
	150	u32 stack_tid, void *arg) {
138	151	ThreadRegistryLock l(this);
139	152	u32 tid = kInvalidTid;
140	153	ThreadContextBase *tctx = QuarantinePop();
...	...	@@ -150,8 +163,10 @@ u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
150	163	Report("%s: Thread limit (%u threads) exceeded. Dying.\n",
151	164	SanitizerToolName, max_threads_);
152	165	#else
153		Printf("race: limit on %u simultaneously alive goroutines is exceeded,"
154		" dying\n", max_threads_);
	166	Printf(
	167	"race: limit on %u simultaneously alive goroutines is exceeded,"
	168	" dying\n",
	169	max_threads_);
155	170	#endif
156	171	Die();
157	172	}
...	...	@@ -170,8 +185,8 @@ u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
170	185	// positives later (e.g. if we join a wrong thread).
171	186	CHECK(live_.try_emplace(user_id, tid).second);
172	187	}
173		tctx->SetCreated(user_id, total_threads_++, detached,
174		parent_tid, arg);
	188	tctx->SetCreated(user_id, total_threads_++, detached, parent_tid, stack_tid,
	189	arg);
175	190	return tid;
176	191	}
177	192
...	...	@@ -196,8 +211,8 @@ u32 ThreadRegistry::FindThread(FindThreadCallback cb, void *arg) {
196	211	return kInvalidTid;
197	212	}
198	213
199		ThreadContextBase *
200		ThreadRegistry::FindThreadContextLocked(FindThreadCallback cb, void *arg) {
	214	ThreadContextBase *ThreadRegistry::FindThreadContextLocked(
	215	FindThreadCallback cb, void *arg) {
201	216	CheckLocked();
202	217	for (u32 tid = 0; tid < threads_.size(); tid++) {
203	218	ThreadContextBase *tctx = threads_[tid];
...	...	@@ -210,7 +225,7 @@ ThreadRegistry::FindThreadContextLocked(FindThreadCallback cb, void *arg) {
210	225	static bool FindThreadContextByOsIdCallback(ThreadContextBase *tctx,
211	226	void *arg) {
212	227	return (tctx->os_id == (uptr)arg && tctx->status != ThreadStatusInvalid &&
213		tctx->status != ThreadStatusDead);
	228	tctx->status != ThreadStatusDead);
214	229	}
215	230
216	231	ThreadContextBase *ThreadRegistry::FindThreadContextByOsIDLocked(tid_t os_id) {

lib/tsan/sanitizer_common/sanitizer_thread_registry.h+8-3

...	...	@@ -52,6 +52,7 @@ class ThreadContextBase {
52	52	ThreadType thread_type;
53	53
54	54	u32 parent_tid;
	55	u32 stack_id;
55	56	ThreadContextBase *next; // For storing thread contexts in a list.
56	57
57	58	atomic_uint32_t thread_destroyed; // To address race of Joined vs Finished
...	...	@@ -63,7 +64,7 @@ class ThreadContextBase {
63	64	void SetFinished();
64	65	void SetStarted(tid_t _os_id, ThreadType _thread_type, void *arg);
65	66	void SetCreated(uptr _user_id, u64 _unique_id, bool _detached,
66		u32 _parent_tid, void *arg);
	67	u32 _parent_tid, u32 _stack_tid, void *arg);
67	68	void Reset();
68	69
69	70	void SetDestroyed();
...	...	@@ -101,12 +102,16 @@ class SANITIZER_MUTEX ThreadRegistry {
101	102
102	103	// Should be guarded by ThreadRegistryLock.
103	104	ThreadContextBase *GetThreadLocked(u32 tid) {
104		return threads_.empty() ? nullptr : threads_[tid];
	105	return tid < threads_.size() ? threads_[tid] : nullptr;
105	106	}
106	107
107	108	u32 NumThreadsLocked() const { return threads_.size(); }
108	109
109		u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg);
	110	u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, u32 stack_tid,
	111	void *arg);
	112	u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg) {
	113	return CreateThread(user_id, detached, parent_tid, 0, arg);
	114	}
110	115
111	116	typedef void (ThreadCallback)(ThreadContextBase tctx, void *arg);
112	117	// Invokes callback with a specified arg for each thread context.

lib/tsan/sanitizer_common/sanitizer_tls_get_addr.cpp+45-29

...	...	@@ -14,6 +14,8 @@
14	14
15	15	#include "sanitizer_allocator_interface.h"
16	16	#include "sanitizer_atomic.h"
	17	#include "sanitizer_common/sanitizer_common.h"
	18	#include "sanitizer_common/sanitizer_internal_defs.h"
17	19	#include "sanitizer_flags.h"
18	20	#include "sanitizer_platform_interceptors.h"
19	21
...	...	@@ -66,7 +68,7 @@ static DTLS::DTVBlock DTLS_NextBlock(atomic_uintptr_t cur) {
66	68	}
67	69
68	70	static DTLS::DTV *DTLS_Find(uptr id) {
69		VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", (void *)&dtls, id);
	71	VReport(3, "__tls_get_addr: DTLS_Find %p %zd\n", (void *)&dtls, id);
70	72	static constexpr uptr kPerBlock = ARRAY_SIZE(DTLS::DTVBlock::dtvs);
71	73	DTLS::DTVBlock *cur = DTLS_NextBlock(&dtls.dtv_block);
72	74	if (!cur)
...	...	@@ -110,6 +112,21 @@ SANITIZER_WEAK_ATTRIBUTE
110	112	const void __sanitizer_get_allocated_begin(const void p);
111	113	}
112	114
	115	SANITIZER_INTERFACE_WEAK_DEF(uptr, __sanitizer_get_dtls_size,
	116	const void *tls_begin) {
	117	const void *start = __sanitizer_get_allocated_begin(tls_begin);
	118	if (!start)
	119	return 0;
	120	CHECK_LE(start, tls_begin);
	121	uptr tls_size = __sanitizer_get_allocated_size(start);
	122	VReport(2, "__tls_get_addr: glibc DTLS suspected; tls={%p,0x%zx}\n",
	123	tls_begin, tls_size);
	124	uptr offset =
	125	(reinterpret_cast<uptr>(tls_begin) - reinterpret_cast<uptr>(start));
	126	CHECK_LE(offset, tls_size);
	127	return tls_size - offset;
	128	}
	129
113	130	DTLS::DTV DTLS_on_tls_get_addr(void arg_void, void *res,
114	131	uptr static_tls_begin, uptr static_tls_end) {
115	132	if (!common_flags()->intercept_tls_get_addr) return 0;
...	...	@@ -117,8 +134,8 @@ DTLS::DTV DTLS_on_tls_get_addr(void arg_void, void *res,
117	134	uptr dso_id = arg->dso_id;
118	135	DTLS::DTV *dtv = DTLS_Find(dso_id);
119	136	if (!dtv \|\| dtv->beg)
120		return 0;
121		uptr tls_size = 0;
	137	return nullptr;
	138	CHECK_LE(static_tls_begin, static_tls_end);
122	139	uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
123	140	VReport(2,
124	141	"__tls_get_addr: %p {0x%zx,0x%zx} => %p; tls_beg: %p; sp: %p "
...	...	@@ -126,36 +143,33 @@ DTLS::DTV DTLS_on_tls_get_addr(void arg_void, void *res,
126	143	(void )arg, arg->dso_id, arg->offset, res, (void )tls_beg,
127	144	(void *)&tls_beg,
128	145	atomic_load(&number_of_live_dtls, memory_order_relaxed));
129		if (dtls.last_memalign_ptr == tls_beg) {
130		tls_size = dtls.last_memalign_size;
131		VReport(2, "__tls_get_addr: glibc <=2.24 suspected; tls={%p,0x%zx}\n",
132		(void *)tls_beg, tls_size);
133		} else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) {
	146	if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) {
134	147	// This is the static TLS block which was initialized / unpoisoned at thread
135	148	// creation.
136	149	VReport(2, "__tls_get_addr: static tls: %p\n", (void *)tls_beg);
137		tls_size = 0;
138		} else if (const void *start =
139		__sanitizer_get_allocated_begin((void *)tls_beg)) {
140		tls_beg = (uptr)start;
141		tls_size = __sanitizer_get_allocated_size(start);
142		VReport(2, "__tls_get_addr: glibc >=2.25 suspected; tls={%p,0x%zx}\n",
143		(void *)tls_beg, tls_size);
144		} else {
145		VReport(2, "__tls_get_addr: Can't guess glibc version\n");
146		// This may happen inside the DTOR of main thread, so just ignore it.
147		tls_size = 0;
	150	dtv->beg = tls_beg;
	151	dtv->size = 0;
	152	return nullptr;
148	153	}
	154	if (uptr tls_size =
	155	__sanitizer_get_dtls_size(reinterpret_cast<void *>(tls_beg))) {
	156	dtv->beg = tls_beg;
	157	dtv->size = tls_size;
	158	return dtv;
	159	}
	160	VReport(2, "__tls_get_addr: Can't guess glibc version\n");
	161	// This may happen inside the DTOR a thread, or async signal handlers before
	162	// thread initialization, so just ignore it.
	163	//
	164	// If the unknown block is dynamic TLS, unlikely we will be able to recognize
	165	// it in future, mark it as done with '{tls_beg, 0}'.
	166	//
	167	// If the block is static TLS, possible reason of failed detection is nullptr
	168	// in `static_tls_begin`. Regardless of reasons, the future handling of static
	169	// TLS is still '{tls_beg, 0}'.
149	170	dtv->beg = tls_beg;
150		dtv->size = tls_size;
151		return dtv;
152		}
153
154		void DTLS_on_libc_memalign(void *ptr, uptr size) {
155		if (!common_flags()->intercept_tls_get_addr) return;
156		VReport(2, "DTLS_on_libc_memalign: %p 0x%zx\n", ptr, size);
157		dtls.last_memalign_ptr = reinterpret_cast<uptr>(ptr);
158		dtls.last_memalign_size = size;
	171	dtv->size = 0;
	172	return nullptr;
159	173	}
160	174
161	175	DTLS *DTLS_Get() { return &dtls; }
...	...	@@ -166,7 +180,9 @@ bool DTLSInDestruction(DTLS *dtls) {
166	180	}
167	181
168	182	#else
169		void DTLS_on_libc_memalign(void *ptr, uptr size) {}
	183	SANITIZER_INTERFACE_WEAK_DEF(uptr, __sanitizer_get_dtls_size, const void *) {
	184	return 0;
	185	}
170	186	DTLS::DTV DTLS_on_tls_get_addr(void arg, void *res,
171	187	unsigned long, unsigned long) { return 0; }
172	188	DTLS *DTLS_Get() { return 0; }

lib/tsan/sanitizer_common/sanitizer_tls_get_addr.h-4

...	...	@@ -55,10 +55,6 @@ struct DTLS {
55	55	static_assert(sizeof(DTVBlock) <= 4096UL, "Unexpected block size");
56	56
57	57	atomic_uintptr_t dtv_block;
58
59		// Auxiliary fields, don't access them outside sanitizer_tls_get_addr.cpp
60		uptr last_memalign_size;
61		uptr last_memalign_ptr;
62	58	};
63	59
64	60	template <typename Fn>

lib/tsan/sanitizer_common/sanitizer_win.cpp+81-20

...	...	@@ -164,7 +164,24 @@ void UnmapOrDie(void *addr, uptr size, bool raw_report) {
164	164	static void ReturnNullptrOnOOMOrDie(uptr size, const char mem_type,
165	165	const char *mmap_type) {
166	166	error_t last_error = GetLastError();
167		if (last_error == ERROR_NOT_ENOUGH_MEMORY)
	167
	168	// Assumption: VirtualAlloc is the last system call that was invoked before
	169	// this method.
	170	// VirtualAlloc emits one of 3 error codes when running out of memory
	171	// 1. ERROR_NOT_ENOUGH_MEMORY:
	172	// There's not enough memory to execute the command
	173	// 2. ERROR_INVALID_PARAMETER:
	174	// VirtualAlloc will return this if the request would allocate memory at an
	175	// address exceeding or being very close to the maximum application address
	176	// (the `lpMaximumApplicationAddress` field within the `SystemInfo` struct).
	177	// This does not seem to be officially documented, but is corroborated here:
	178	// https://stackoverflow.com/questions/45833674/why-does-virtualalloc-fail-for-lpaddress-greater-than-0x6ffffffffff
	179	// 3. ERROR_COMMITMENT_LIMIT:
	180	// VirtualAlloc will return this if e.g. the pagefile is too small to commit
	181	// the requested amount of memory.
	182	if (last_error == ERROR_NOT_ENOUGH_MEMORY \|\|
	183	last_error == ERROR_INVALID_PARAMETER \|\|
	184	last_error == ERROR_COMMITMENT_LIMIT)
168	185	return nullptr;
169	186	ReportMmapFailureAndDie(size, mem_type, mmap_type, last_error);
170	187	}
...	...	@@ -873,24 +890,18 @@ uptr GetTlsSize() {
873	890	return 0;
874	891	}
875	892
876		void InitTlsSize() {
877		}
878
879		void GetThreadStackAndTls(bool main, uptr stk_addr, uptr stk_size,
880		uptr tls_addr, uptr tls_size) {
881		#if SANITIZER_GO
882		*stk_addr = 0;
883		*stk_size = 0;
884		*tls_addr = 0;
885		*tls_size = 0;
886		#else
887		uptr stack_top, stack_bottom;
888		GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom);
889		*stk_addr = stack_bottom;
890		*stk_size = stack_top - stack_bottom;
891		*tls_addr = 0;
892		*tls_size = 0;
893		#endif
	893	void GetThreadStackAndTls(bool main, uptr stk_begin, uptr stk_end,
	894	uptr tls_begin, uptr tls_end) {
	895	# if SANITIZER_GO
	896	*stk_begin = 0;
	897	*stk_end = 0;
	898	*tls_begin = 0;
	899	*tls_end = 0;
	900	# else
	901	GetThreadStackTopAndBottom(main, stk_end, stk_begin);
	902	*tls_begin = 0;
	903	*tls_end = 0;
	904	# endif
894	905	}
895	906
896	907	void ReportFile::Write(const char *buffer, uptr length) {
...	...	@@ -974,6 +985,11 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) {
974	985	return true;
975	986	}
976	987
	988	bool TryMemCpy(void dest, const void src, uptr n) {
	989	// TODO: implement.
	990	return false;
	991	}
	992
977	993	bool SignalContext::IsStackOverflow() const {
978	994	return (DWORD)GetType() == EXCEPTION_STACK_OVERFLOW;
979	995	}
...	...	@@ -1039,7 +1055,52 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1039	1055	}
1040	1056
1041	1057	void SignalContext::DumpAllRegisters(void *context) {
1042		// FIXME: Implement this.
	1058	CONTEXT ctx = (CONTEXT )context;
	1059	# if defined(_M_X64)
	1060	Report("Register values:\n");
	1061	Printf("rax = %llx ", ctx->Rax);
	1062	Printf("rbx = %llx ", ctx->Rbx);
	1063	Printf("rcx = %llx ", ctx->Rcx);
	1064	Printf("rdx = %llx ", ctx->Rdx);
	1065	Printf("\n");
	1066	Printf("rdi = %llx ", ctx->Rdi);
	1067	Printf("rsi = %llx ", ctx->Rsi);
	1068	Printf("rbp = %llx ", ctx->Rbp);
	1069	Printf("rsp = %llx ", ctx->Rsp);
	1070	Printf("\n");
	1071	Printf("r8 = %llx ", ctx->R8);
	1072	Printf("r9 = %llx ", ctx->R9);
	1073	Printf("r10 = %llx ", ctx->R10);
	1074	Printf("r11 = %llx ", ctx->R11);
	1075	Printf("\n");
	1076	Printf("r12 = %llx ", ctx->R12);
	1077	Printf("r13 = %llx ", ctx->R13);
	1078	Printf("r14 = %llx ", ctx->R14);
	1079	Printf("r15 = %llx ", ctx->R15);
	1080	Printf("\n");
	1081	# elif defined(_M_IX86)
	1082	Report("Register values:\n");
	1083	Printf("eax = %lx ", ctx->Eax);
	1084	Printf("ebx = %lx ", ctx->Ebx);
	1085	Printf("ecx = %lx ", ctx->Ecx);
	1086	Printf("edx = %lx ", ctx->Edx);
	1087	Printf("\n");
	1088	Printf("edi = %lx ", ctx->Edi);
	1089	Printf("esi = %lx ", ctx->Esi);
	1090	Printf("ebp = %lx ", ctx->Ebp);
	1091	Printf("esp = %lx ", ctx->Esp);
	1092	Printf("\n");
	1093	# elif defined(_M_ARM64)
	1094	Report("Register values:\n");
	1095	for (int i = 0; i <= 30; i++) {
	1096	Printf("x%d%s = %llx", i < 10 ? " " : "", ctx->X[i]);
	1097	if (i % 4 == 3)
	1098	Printf("\n");
	1099	}
	1100	# else
	1101	// TODO
	1102	(void)ctx;
	1103	# endif
1043	1104	}
1044	1105
1045	1106	int SignalContext::GetType() const {

lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.cpp deleted-101

...	...	@@ -1,101 +0,0 @@
1		//===-- sanitizer_win_dll_thunk.cpp ---------------------------------------===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		// This file defines a family of thunks that should be statically linked into
9		// the DLLs that have instrumentation in order to delegate the calls to the
10		// shared runtime that lives in the main binary.
11		// See https://github.com/google/sanitizers/issues/209 for the details.
12		//===----------------------------------------------------------------------===//
13
14		#ifdef SANITIZER_DLL_THUNK
15		#include "sanitizer_win_defs.h"
16		#include "sanitizer_win_dll_thunk.h"
17		#include "interception/interception.h"
18
19		extern "C" {
20		void WINAPI GetModuleHandleA(const char module_name);
21		void abort();
22		}
23
24		namespace __sanitizer {
25		uptr dllThunkGetRealAddrOrDie(const char *name) {
26		uptr ret =
27		__interception::InternalGetProcAddress((void *)GetModuleHandleA(0), name);
28		if (!ret)
29		abort();
30		return ret;
31		}
32
33		int dllThunkIntercept(const char* main_function, uptr dll_function) {
34		uptr wrapper = dllThunkGetRealAddrOrDie(main_function);
35		if (!__interception::OverrideFunction(dll_function, wrapper, 0))
36		abort();
37		return 0;
38		}
39
40		int dllThunkInterceptWhenPossible(const char* main_function,
41		const char* default_function, uptr dll_function) {
42		uptr wrapper = __interception::InternalGetProcAddress(
43		(void *)GetModuleHandleA(0), main_function);
44		if (!wrapper)
45		wrapper = dllThunkGetRealAddrOrDie(default_function);
46		if (!__interception::OverrideFunction(dll_function, wrapper, 0))
47		abort();
48		return 0;
49		}
50		} // namespace __sanitizer
51
52		// Include Sanitizer Common interface.
53		#define INTERFACE_FUNCTION(Name) INTERCEPT_SANITIZER_FUNCTION(Name)
54		#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
55		#include "sanitizer_common_interface.inc"
56
57		#pragma section(".DLLTH$A", read)
58		#pragma section(".DLLTH$Z", read)
59
60		typedef void (*DllThunkCB)();
61		extern "C" {
62		__declspec(allocate(".DLLTH$A")) DllThunkCB __start_dll_thunk;
63		__declspec(allocate(".DLLTH$Z")) DllThunkCB __stop_dll_thunk;
64		}
65
66		// Disable compiler warnings that show up if we declare our own version
67		// of a compiler intrinsic (e.g. strlen).
68		#pragma warning(disable: 4391)
69		#pragma warning(disable: 4392)
70
71		extern "C" int __dll_thunk_init() {
72		static bool flag = false;
73		// __dll_thunk_init is expected to be called by only one thread.
74		if (flag) return 0;
75		flag = true;
76
77		for (DllThunkCB *it = &__start_dll_thunk; it < &__stop_dll_thunk; ++it)
78		if (*it)
79		(*it)();
80
81		// In DLLs, the callbacks are expected to return 0,
82		// otherwise CRT initialization fails.
83		return 0;
84		}
85
86		// We want to call dll_thunk_init before C/C++ initializers / constructors are
87		// executed, otherwise functions like memset might be invoked.
88		#pragma section(".CRT$XIB", long, read)
89		__declspec(allocate(".CRT$XIB")) int (*__dll_thunk_preinit)() =
90		__dll_thunk_init;
91
92		static void WINAPI dll_thunk_thread_init(void *mod, unsigned long reason,
93		void *reserved) {
94		if (reason == /DLL_PROCESS_ATTACH=/1) __dll_thunk_init();
95		}
96
97		#pragma section(".CRT$XLAB", long, read)
98		__declspec(allocate(".CRT$XLAB")) void (WINAPI __dll_thunk_tls_init)(void ,
99		unsigned long, void *) = dll_thunk_thread_init;
100
101		#endif // SANITIZER_DLL_THUNK

lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.h deleted-181

...	...	@@ -1,181 +0,0 @@
1		//===-- sanitizer_win_dll_thunk.h -----------------------------------------===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		// This header provide helper macros to delegate calls to the shared runtime
9		// that lives in the main executable. It should be included to dll_thunks that
10		// will be linked to the dlls, when the sanitizer is a static library included
11		// in the main executable.
12		//===----------------------------------------------------------------------===//
13		#ifndef SANITIZER_WIN_DLL_THUNK_H
14		#define SANITIZER_WIN_DLL_THUNK_H
15		#include "sanitizer_internal_defs.h"
16
17		namespace __sanitizer {
18		uptr dllThunkGetRealAddrOrDie(const char *name);
19
20		int dllThunkIntercept(const char* main_function, uptr dll_function);
21
22		int dllThunkInterceptWhenPossible(const char* main_function,
23		const char* default_function, uptr dll_function);
24		}
25
26		extern "C" int __dll_thunk_init();
27
28		// ----------------- Function interception helper macros -------------------- //
29		// Override dll_function with main_function from main executable.
30		#define INTERCEPT_OR_DIE(main_function, dll_function) \
31		static int intercept_##dll_function() { \
32		return __sanitizer::dllThunkIntercept(main_function, (__sanitizer::uptr) \
33		dll_function); \
34		} \
35		__pragma(section(".DLLTH$M", long, read)) \
36		__declspec(allocate(".DLLTH$M")) int (*__dll_thunk_##dll_function)() = \
37		intercept_##dll_function;
38
39		// Try to override dll_function with main_function from main executable.
40		// If main_function is not present, override dll_function with default_function.
41		#define INTERCEPT_WHEN_POSSIBLE(main_function, default_function, dll_function) \
42		static int intercept_##dll_function() { \
43		return __sanitizer::dllThunkInterceptWhenPossible(main_function, \
44		default_function, (__sanitizer::uptr)dll_function); \
45		} \
46		__pragma(section(".DLLTH$M", long, read)) \
47		__declspec(allocate(".DLLTH$M")) int (*__dll_thunk_##dll_function)() = \
48		intercept_##dll_function;
49
50		// -------------------- Function interception macros ------------------------ //
51		// Special case of hooks -- ASan own interface functions. Those are only called
52		// after __asan_init, thus an empty implementation is sufficient.
53		#define INTERCEPT_SANITIZER_FUNCTION(name) \
54		extern "C" __declspec(noinline) void name() { \
55		volatile int prevent_icf = (__LINE__ << 8) ^ __COUNTER__; \
56		static const char function_name[] = #name; \
57		for (const char* ptr = &function_name[0]; *ptr; ++ptr) \
58		prevent_icf ^= *ptr; \
59		(void)prevent_icf; \
60		__debugbreak(); \
61		} \
62		INTERCEPT_OR_DIE(#name, name)
63
64		// Special case of hooks -- Weak functions, could be redefined in the main
65		// executable, but that is not necessary, so we shouldn't die if we can not find
66		// a reference. Instead, when the function is not present in the main executable
67		// we consider the default impl provided by asan library.
68		#define INTERCEPT_SANITIZER_WEAK_FUNCTION(name) \
69		extern "C" __declspec(noinline) void name() { \
70		volatile int prevent_icf = (__LINE__ << 8) ^ __COUNTER__; \
71		static const char function_name[] = #name; \
72		for (const char* ptr = &function_name[0]; *ptr; ++ptr) \
73		prevent_icf ^= *ptr; \
74		(void)prevent_icf; \
75		__debugbreak(); \
76		} \
77		INTERCEPT_WHEN_POSSIBLE(#name, STRINGIFY(WEAK_EXPORT_NAME(name)), name)
78
79		// We can't define our own version of strlen etc. because that would lead to
80		// link-time or even type mismatch errors. Instead, we can declare a function
81		// just to be able to get its address. Me may miss the first few calls to the
82		// functions since it can be called before __dll_thunk_init, but that would lead
83		// to false negatives in the startup code before user's global initializers,
84		// which isn't a big deal.
85		#define INTERCEPT_LIBRARY_FUNCTION(name) \
86		extern "C" void name(); \
87		INTERCEPT_OR_DIE(STRINGIFY(WRAP(name)), name)
88
89		// Use these macros for functions that could be called before __dll_thunk_init()
90		// is executed and don't lead to errors if defined (free, malloc, etc).
91		#define INTERCEPT_WRAP_V_V(name) \
92		extern "C" void name() { \
93		typedef decltype(name) *fntype; \
94		static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
95		fn(); \
96		} \
97		INTERCEPT_OR_DIE(#name, name);
98
99		#define INTERCEPT_WRAP_V_W(name) \
100		extern "C" void name(void *arg) { \
101		typedef decltype(name) *fntype; \
102		static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
103		fn(arg); \
104		} \
105		INTERCEPT_OR_DIE(#name, name);
106
107		#define INTERCEPT_WRAP_V_WW(name) \
108		extern "C" void name(void arg1, void arg2) { \
109		typedef decltype(name) *fntype; \
110		static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
111		fn(arg1, arg2); \
112		} \
113		INTERCEPT_OR_DIE(#name, name);
114
115		#define INTERCEPT_WRAP_V_WWW(name) \
116		extern "C" void name(void arg1, void arg2, void *arg3) { \
117		typedef decltype(name) *fntype; \
118		static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
119		fn(arg1, arg2, arg3); \
120		} \
121		INTERCEPT_OR_DIE(#name, name);
122
123		#define INTERCEPT_WRAP_W_V(name) \
124		extern "C" void *name() { \
125		typedef decltype(name) *fntype; \
126		static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
127		return fn(); \
128		} \
129		INTERCEPT_OR_DIE(#name, name);
130
131		#define INTERCEPT_WRAP_W_W(name) \
132		extern "C" void name(void arg) { \
133		typedef decltype(name) *fntype; \
134		static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
135		return fn(arg); \
136		} \
137		INTERCEPT_OR_DIE(#name, name);
138
139		#define INTERCEPT_WRAP_W_WW(name) \
140		extern "C" void name(void arg1, void *arg2) { \
141		typedef decltype(name) *fntype; \
142		static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
143		return fn(arg1, arg2); \
144		} \
145		INTERCEPT_OR_DIE(#name, name);
146
147		#define INTERCEPT_WRAP_W_WWW(name) \
148		extern "C" void name(void arg1, void arg2, void arg3) { \
149		typedef decltype(name) *fntype; \
150		static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
151		return fn(arg1, arg2, arg3); \
152		} \
153		INTERCEPT_OR_DIE(#name, name);
154
155		#define INTERCEPT_WRAP_W_WWWW(name) \
156		extern "C" void name(void arg1, void arg2, void arg3, void *arg4) { \
157		typedef decltype(name) *fntype; \
158		static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
159		return fn(arg1, arg2, arg3, arg4); \
160		} \
161		INTERCEPT_OR_DIE(#name, name);
162
163		#define INTERCEPT_WRAP_W_WWWWW(name) \
164		extern "C" void name(void arg1, void arg2, void arg3, void *arg4, \
165		void *arg5) { \
166		typedef decltype(name) *fntype; \
167		static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
168		return fn(arg1, arg2, arg3, arg4, arg5); \
169		} \
170		INTERCEPT_OR_DIE(#name, name);
171
172		#define INTERCEPT_WRAP_W_WWWWWW(name) \
173		extern "C" void name(void arg1, void arg2, void arg3, void *arg4, \
174		void arg5, void arg6) { \
175		typedef decltype(name) *fntype; \
176		static fntype fn = (fntype)__sanitizer::dllThunkGetRealAddrOrDie(#name); \
177		return fn(arg1, arg2, arg3, arg4, arg5, arg6); \
178		} \
179		INTERCEPT_OR_DIE(#name, name);
180
181		#endif // SANITIZER_WIN_DLL_THUNK_H

lib/tsan/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cpp deleted-26

...	...	@@ -1,26 +0,0 @@
1		//===-- santizer_win_dynamic_runtime_thunk.cpp ----------------------------===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// This file defines things that need to be present in the application modules
10		// to interact with Sanitizer Common, when it is included in a dll.
11		//
12		//===----------------------------------------------------------------------===//
13		#ifdef SANITIZER_DYNAMIC_RUNTIME_THUNK
14		#define SANITIZER_IMPORT_INTERFACE 1
15		#include "sanitizer_win_defs.h"
16		// Define weak alias for all weak functions imported from sanitizer common.
17		#define INTERFACE_FUNCTION(Name)
18		#define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name)
19		#include "sanitizer_common_interface.inc"
20		#endif // SANITIZER_DYNAMIC_RUNTIME_THUNK
21
22		namespace __sanitizer {
23		// Add one, otherwise unused, external symbol to this object file so that the
24		// Visual C++ linker includes it and reads the .drective section.
25		void ForceWholeArchiveIncludeForSanitizerCommon() {}
26		}

lib/tsan/sanitizer_common/sanitizer_win_immortalize.h created+71

...	...	@@ -0,0 +1,71 @@
	1	//===-- sanitizer_win_immortalize.h ---------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between AddressSanitizer, and interception.
	10	//
	11	// Windows-specific thread-safe and pre-CRT global initialization safe
	12	// infrastructure to create an object whose destructor is never called.
	13	//===----------------------------------------------------------------------===//
	14	#if SANITIZER_WINDOWS
	15	# pragma once
	16	// Requires including sanitizer_placement_new.h (which is not allowed to be
	17	// included in headers).
	18
	19	# include "sanitizer_win_defs.h"
	20	// These types are required to satisfy XFG which requires that the names of the
	21	// types for indirect calls to be correct as well as the name of the original
	22	// type for any typedefs.
	23
	24	// TODO: There must be a better way to do this
	25	# ifndef _WINDOWS_
	26	typedef void* PVOID;
	27	typedef int BOOL;
	28	typedef union _RTL_RUN_ONCE {
	29	PVOID ptr;
	30	} INIT_ONCE, *PINIT_ONCE;
	31
	32	extern "C" {
	33	__declspec(dllimport) int WINAPI InitOnceExecuteOnce(
	34	PINIT_ONCE, BOOL(WINAPI)(PINIT_ONCE, PVOID, PVOID), void, void);
	35	}
	36	# endif
	37
	38	namespace __sanitizer {
	39	template <class Ty>
	40	BOOL WINAPI immortalize_impl(PINIT_ONCE, PVOID storage_ptr, PVOID*) noexcept {
	41	// Ty must provide a placement new operator
	42	new (storage_ptr) Ty();
	43	return 1;
	44	}
	45
	46	template <class Ty, typename Arg>
	47	BOOL WINAPI immortalize_impl(PINIT_ONCE, PVOID storage_ptr,
	48	PVOID* param) noexcept {
	49	// Ty must provide a placement new operator
	50	new (storage_ptr) Ty(((Arg)param));
	51	return 1;
	52	}
	53
	54	template <class Ty>
	55	Ty& immortalize() { // return a reference to an object that will live forever
	56	static INIT_ONCE flag;
	57	alignas(Ty) static unsigned char storage[sizeof(Ty)];
	58	InitOnceExecuteOnce(&flag, immortalize_impl<Ty>, &storage, nullptr);
	59	return reinterpret_cast<Ty&>(storage);
	60	}
	61
	62	template <class Ty, typename Arg>
	63	Ty& immortalize(
	64	Arg arg) { // return a reference to an object that will live forever
	65	static INIT_ONCE flag;
	66	alignas(Ty) static unsigned char storage[sizeof(Ty)];
	67	InitOnceExecuteOnce(&flag, immortalize_impl<Ty, Arg>, &storage, &arg);
	68	return reinterpret_cast<Ty&>(storage);
	69	}
	70	} // namespace __sanitizer
	71	#endif // SANITIZER_WINDOWS

lib/tsan/sanitizer_common/sanitizer_win_interception.cpp created+156

...	...	@@ -0,0 +1,156 @@
	1	//===-- sanitizer_win_interception.cpp -------------------- --- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// Windows-specific export surface to provide interception for parts of the
	10	// runtime that are always statically linked, both for overriding user-defined
	11	// functions as well as registering weak functions that the ASAN runtime should
	12	// use over defaults.
	13	//
	14	//===----------------------------------------------------------------------===//
	15
	16	#include "sanitizer_platform.h"
	17	#if SANITIZER_WINDOWS
	18	# include <stddef.h>
	19
	20	# include "interception/interception.h"
	21	# include "sanitizer_addrhashmap.h"
	22	# include "sanitizer_common.h"
	23	# include "sanitizer_internal_defs.h"
	24	# include "sanitizer_placement_new.h"
	25	# include "sanitizer_win_immortalize.h"
	26	# include "sanitizer_win_interception.h"
	27
	28	using namespace __sanitizer;
	29
	30	extern "C" void *__ImageBase;
	31
	32	namespace __sanitizer {
	33
	34	static uptr GetSanitizerDllExport(const char *export_name) {
	35	const uptr function_address =
	36	__interception::InternalGetProcAddress(&__ImageBase, export_name);
	37	if (function_address == 0) {
	38	Report("ERROR: Failed to find sanitizer DLL export '%s'\n", export_name);
	39	CHECK("Failed to find sanitizer DLL export" && 0);
	40	}
	41	return function_address;
	42	}
	43
	44	struct WeakCallbackList {
	45	explicit constexpr WeakCallbackList(RegisterWeakFunctionCallback cb)
	46	: callback(cb), next(nullptr) {}
	47
	48	static void *operator new(size_t size) { return InternalAlloc(size); }
	49
	50	static void operator delete(void *p) { InternalFree(p); }
	51
	52	RegisterWeakFunctionCallback callback;
	53	WeakCallbackList *next;
	54	};
	55	using WeakCallbackMap = AddrHashMap<WeakCallbackList *, 11>;
	56
	57	static WeakCallbackMap *GetWeakCallbackMap() {
	58	return &immortalize<WeakCallbackMap>();
	59	}
	60
	61	void AddRegisterWeakFunctionCallback(uptr export_address,
	62	RegisterWeakFunctionCallback cb) {
	63	WeakCallbackMap::Handle h_find_or_create(GetWeakCallbackMap(), export_address,
	64	false, true);
	65	CHECK(h_find_or_create.exists());
	66	if (h_find_or_create.created()) {
	67	*h_find_or_create = new WeakCallbackList(cb);
	68	} else {
	69	(*h_find_or_create)->next = new WeakCallbackList(cb);
	70	}
	71	}
	72
	73	static void RunWeakFunctionCallbacks(uptr export_address) {
	74	WeakCallbackMap::Handle h_find(GetWeakCallbackMap(), export_address, false,
	75	false);
	76	if (!h_find.exists()) {
	77	return;
	78	}
	79
	80	WeakCallbackList list = h_find;
	81	do {
	82	list->callback();
	83	} while ((list = list->next));
	84	}
	85
	86	} // namespace __sanitizer
	87
	88	extern "C" __declspec(dllexport) bool __cdecl __sanitizer_override_function(
	89	const char *export_name, const uptr user_function,
	90	uptr *const old_user_function) {
	91	CHECK(export_name);
	92	CHECK(user_function);
	93
	94	const uptr sanitizer_function = GetSanitizerDllExport(export_name);
	95
	96	const bool function_overridden = __interception::OverrideFunction(
	97	user_function, sanitizer_function, old_user_function);
	98	if (!function_overridden) {
	99	Report(
	100	"ERROR: Failed to override local function at '%p' with sanitizer "
	101	"function '%s'\n",
	102	user_function, export_name);
	103	CHECK("Failed to replace local function with sanitizer version." && 0);
	104	}
	105
	106	return function_overridden;
	107	}
	108
	109	extern "C"
	110	__declspec(dllexport) bool __cdecl __sanitizer_override_function_by_addr(
	111	const uptr source_function, const uptr target_function,
	112	uptr *const old_target_function) {
	113	CHECK(source_function);
	114	CHECK(target_function);
	115
	116	const bool function_overridden = __interception::OverrideFunction(
	117	target_function, source_function, old_target_function);
	118	if (!function_overridden) {
	119	Report(
	120	"ERROR: Failed to override function at '%p' with function at "
	121	"'%p'\n",
	122	target_function, source_function);
	123	CHECK("Failed to apply function override." && 0);
	124	}
	125
	126	return function_overridden;
	127	}
	128
	129	extern "C"
	130	__declspec(dllexport) bool __cdecl __sanitizer_register_weak_function(
	131	const char *export_name, const uptr user_function,
	132	uptr *const old_user_function) {
	133	CHECK(export_name);
	134	CHECK(user_function);
	135
	136	const uptr sanitizer_function = GetSanitizerDllExport(export_name);
	137
	138	const bool function_overridden = __interception::OverrideFunction(
	139	sanitizer_function, user_function, old_user_function);
	140	if (!function_overridden) {
	141	Report(
	142	"ERROR: Failed to register local function at '%p' to be used in "
	143	"place of sanitizer function '%s'\n.",
	144	user_function, export_name);
	145	CHECK("Failed to register weak function." && 0);
	146	}
	147
	148	// Note that thread-safety of RunWeakFunctionCallbacks in InitializeFlags
	149	// depends on __sanitizer_register_weak_functions being called during the
	150	// loader lock.
	151	RunWeakFunctionCallbacks(sanitizer_function);
	152
	153	return function_overridden;
	154	}
	155
	156	#endif // SANITIZER_WINDOWS

lib/tsan/sanitizer_common/sanitizer_win_interception.h created+32

...	...	@@ -0,0 +1,32 @@
	1	//===-- sanitizer_win_interception.h ---------------------- --- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// Windows-specific export surface to provide interception for parts of the
	10	// runtime that are always statically linked, both for overriding user-defined
	11	// functions as well as registering weak functions that the ASAN runtime should
	12	// use over defaults.
	13	//
	14	//===----------------------------------------------------------------------===//
	15
	16	#ifndef SANITIZER_WIN_INTERCEPTION_H
	17	#define SANITIZER_WIN_INTERCEPTION_H
	18
	19	#include "sanitizer_platform.h"
	20	#if SANITIZER_WINDOWS
	21
	22	# include "sanitizer_common.h"
	23	# include "sanitizer_internal_defs.h"
	24
	25	namespace __sanitizer {
	26	using RegisterWeakFunctionCallback = void (*)();
	27	void AddRegisterWeakFunctionCallback(uptr export_address,
	28	RegisterWeakFunctionCallback cb);
	29	} // namespace __sanitizer
	30
	31	#endif // SANITIZER_WINDOWS
	32	#endif // SANITIZER_WIN_INTERCEPTION_H
		\ No newline at end of file

lib/tsan/sanitizer_common/sanitizer_win_weak_interception.cpp deleted-94

...	...	@@ -1,94 +0,0 @@
1		//===-- sanitizer_win_weak_interception.cpp -------------------------------===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		// This module should be included in the sanitizer when it is implemented as a
9		// shared library on Windows (dll), in order to delegate the calls of weak
10		// functions to the implementation in the main executable when a strong
11		// definition is provided.
12		//===----------------------------------------------------------------------===//
13
14		#include "sanitizer_platform.h"
15		#if SANITIZER_WINDOWS && SANITIZER_DYNAMIC
16		#include "sanitizer_win_weak_interception.h"
17		#include "sanitizer_allocator_interface.h"
18		#include "sanitizer_interface_internal.h"
19		#include "sanitizer_win_defs.h"
20		#include "interception/interception.h"
21
22		extern "C" {
23		void WINAPI GetModuleHandleA(const char module_name);
24		void abort();
25		}
26
27		namespace __sanitizer {
28		// Try to get a pointer to real_function in the main module and override
29		// dll_function with that pointer. If the function isn't found, nothing changes.
30		int interceptWhenPossible(uptr dll_function, const char *real_function) {
31		uptr real = __interception::InternalGetProcAddress(
32		(void *)GetModuleHandleA(0), real_function);
33		if (real && !__interception::OverrideFunction((uptr)dll_function, real, 0))
34		abort();
35		return 0;
36		}
37		} // namespace __sanitizer
38
39		// Declare weak hooks.
40		extern "C" {
41		void __sanitizer_on_print(const char *str);
42		void __sanitizer_weak_hook_memcmp(uptr called_pc, const void *s1,
43		const void *s2, uptr n, int result);
44		void __sanitizer_weak_hook_strcmp(uptr called_pc, const char *s1,
45		const char *s2, int result);
46		void __sanitizer_weak_hook_strncmp(uptr called_pc, const char *s1,
47		const char *s2, uptr n, int result);
48		void __sanitizer_weak_hook_strstr(uptr called_pc, const char *s1,
49		const char s2, char result);
50		}
51
52		// Include Sanitizer Common interface.
53		#define INTERFACE_FUNCTION(Name)
54		#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
55		#include "sanitizer_common_interface.inc"
56
57		#pragma section(".WEAK$A", read)
58		#pragma section(".WEAK$Z", read)
59
60		typedef void (*InterceptCB)();
61		extern "C" {
62		__declspec(allocate(".WEAK$A")) InterceptCB __start_weak_list;
63		__declspec(allocate(".WEAK$Z")) InterceptCB __stop_weak_list;
64		}
65
66		static int weak_intercept_init() {
67		static bool flag = false;
68		// weak_interception_init is expected to be called by only one thread.
69		if (flag) return 0;
70		flag = true;
71
72		for (InterceptCB *it = &__start_weak_list; it < &__stop_weak_list; ++it)
73		if (*it)
74		(*it)();
75
76		// In DLLs, the callbacks are expected to return 0,
77		// otherwise CRT initialization fails.
78		return 0;
79		}
80
81		#pragma section(".CRT$XIB", long, read)
82		__declspec(allocate(".CRT$XIB")) int (*__weak_intercept_preinit)() =
83		weak_intercept_init;
84
85		static void WINAPI weak_intercept_thread_init(void *mod, unsigned long reason,
86		void *reserved) {
87		if (reason == /DLL_PROCESS_ATTACH=/1) weak_intercept_init();
88		}
89
90		#pragma section(".CRT$XLAB", long, read)
91		__declspec(allocate(".CRT$XLAB")) void(WINAPI *__weak_intercept_tls_init)(
92		void , unsigned long, void ) = weak_intercept_thread_init;
93
94		#endif // SANITIZER_WINDOWS && SANITIZER_DYNAMIC

lib/tsan/sanitizer_common/sanitizer_win_weak_interception.h deleted-32

...	...	@@ -1,32 +0,0 @@
1		//===-- sanitizer_win_weak_interception.h ---------------------------------===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		// This header provide helper macros to delegate calls of weak functions to the
9		// implementation in the main executable when a strong definition is present.
10		//===----------------------------------------------------------------------===//
11		#ifndef SANITIZER_WIN_WEAK_INTERCEPTION_H
12		#define SANITIZER_WIN_WEAK_INTERCEPTION_H
13		#include "sanitizer_internal_defs.h"
14
15		namespace __sanitizer {
16		int interceptWhenPossible(uptr dll_function, const char *real_function);
17		}
18
19		// ----------------- Function interception helper macros -------------------- //
20		// Weak functions, could be redefined in the main executable, but that is not
21		// necessary, so we shouldn't die if we can not find a reference.
22		#define INTERCEPT_WEAK(Name) interceptWhenPossible((uptr) Name, #Name);
23
24		#define INTERCEPT_SANITIZER_WEAK_FUNCTION(Name) \
25		static int intercept_##Name() { \
26		return __sanitizer::interceptWhenPossible((__sanitizer::uptr) Name, #Name);\
27		} \
28		__pragma(section(".WEAK$M", long, read)) \
29		__declspec(allocate(".WEAK$M")) int (*__weak_intercept_##Name)() = \
30		intercept_##Name;
31
32		#endif // SANITIZER_WIN_WEAK_INTERCEPTION_H

lib/tsan/tsan_interceptors_mac.cpp+123-97

...	...	@@ -14,22 +14,21 @@
14	14	#include "sanitizer_common/sanitizer_platform.h"
15	15	#if SANITIZER_APPLE
16	16
17		#include "interception/interception.h"
18		#include "tsan_interceptors.h"
19		#include "tsan_interface.h"
20		#include "tsan_interface_ann.h"
21		#include "tsan_spinlock_defs_mac.h"
22		#include "sanitizer_common/sanitizer_addrhashmap.h"
23
24		#include <errno.h>
25		#include <libkern/OSAtomic.h>
26		#include <objc/objc-sync.h>
27		#include <os/lock.h>
28		#include <sys/ucontext.h>
29
30		#if defined(__has_include) && __has_include(<xpc/xpc.h>)
31		#include <xpc/xpc.h>
32		#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
	17	# include <errno.h>
	18	# include <libkern/OSAtomic.h>
	19	# include <objc/objc-sync.h>
	20	# include <os/lock.h>
	21	# include <sys/ucontext.h>
	22
	23	# include "interception/interception.h"
	24	# include "sanitizer_common/sanitizer_addrhashmap.h"
	25	# include "tsan_interceptors.h"
	26	# include "tsan_interface.h"
	27	# include "tsan_interface_ann.h"
	28
	29	# if defined(__has_include) && __has_include(<xpc/xpc.h>)
	30	# include <xpc/xpc.h>
	31	# endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
33	32
34	33	typedef long long_t;
35	34
...	...	@@ -49,51 +48,55 @@ static constexpr morder kMacOrderBarrier = mo_acq_rel;
49	48	static constexpr morder kMacOrderNonBarrier = mo_acq_rel;
50	49	static constexpr morder kMacFailureOrder = mo_relaxed;
51	50
52		#define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
53		TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
54		SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
55		return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \
56		}
57
58		#define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
59		TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
60		SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
61		return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \
62		}
	51	# define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
	52	TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
	53	SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
	54	return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \
	55	}
63	56
64		#define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
65		TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
66		SCOPED_TSAN_INTERCEPTOR(f, ptr); \
67		return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \
68		}
	57	# define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, \
	58	mo) \
	59	TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
	60	SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
	61	return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \
	62	}
69	63
70		#define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
71		mo) \
72		TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
73		SCOPED_TSAN_INTERCEPTOR(f, ptr); \
74		return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \
75		}
	64	# define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
	65	mo) \
	66	TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
	67	SCOPED_TSAN_INTERCEPTOR(f, ptr); \
	68	return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \
	69	}
76	70
77		#define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \
78		m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
79		kMacOrderNonBarrier) \
80		m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
81		kMacOrderBarrier) \
82		m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \
83		kMacOrderNonBarrier) \
84		m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \
85		kMacOrderBarrier)
86
87		#define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \
88		m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
89		kMacOrderNonBarrier) \
90		m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
91		kMacOrderBarrier) \
92		m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \
93		kMacOrderNonBarrier) \
94		m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \
95		__tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
	71	# define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
	72	mo) \
	73	TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
	74	SCOPED_TSAN_INTERCEPTOR(f, ptr); \
	75	return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \
	76	}
96	77
	78	# define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \
	79	m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
	80	kMacOrderNonBarrier) \
	81	m(int32_t, int32_t, a32, f##32##Barrier, \
	82	__tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) \
	83	m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \
	84	kMacOrderNonBarrier) \
	85	m(int64_t, int64_t, a64, f##64##Barrier, \
	86	__tsan_atomic64_##tsan_atomic_f, kMacOrderBarrier)
	87
	88	# define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \
	89	m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
	90	kMacOrderNonBarrier) \
	91	m(int32_t, uint32_t, a32, f##32##Barrier, \
	92	__tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) \
	93	m_orig(int32_t, uint32_t, a32, f##32##Orig, \
	94	__tsan_atomic32_##tsan_atomic_f, kMacOrderNonBarrier) \
	95	m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \
	96	__tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
	97
	98	# pragma clang diagnostic push // OSAtomic* deprecation
	99	# pragma clang diagnostic ignored "-Wdeprecated-declarations"
97	100	OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,
98	101	OSATOMIC_INTERCEPTOR_PLUS_X)
99	102	OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,
...	...	@@ -106,23 +109,26 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,
106	109	OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
107	110	OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
108	111	OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
	112	# pragma clang diagnostic pop // OSAtomic* deprecation
	113
	114	# define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \
	115	TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \
	116	SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
	117	return tsan_atomic_f##_compare_exchange_strong( \
	118	(volatile tsan_t )ptr, (tsan_t )&old_value, (tsan_t)new_value, \
	119	kMacOrderNonBarrier, kMacFailureOrder); \
	120	} \
	121	\
	122	TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \
	123	t volatile *ptr) { \
	124	SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
	125	return tsan_atomic_f##_compare_exchange_strong( \
	126	(volatile tsan_t )ptr, (tsan_t )&old_value, (tsan_t)new_value, \
	127	kMacOrderBarrier, kMacFailureOrder); \
	128	}
109	129
110		#define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \
111		TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \
112		SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
113		return tsan_atomic_f##_compare_exchange_strong( \
114		(volatile tsan_t )ptr, (tsan_t )&old_value, (tsan_t)new_value, \
115		kMacOrderNonBarrier, kMacFailureOrder); \
116		} \
117		\
118		TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \
119		t volatile *ptr) { \
120		SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
121		return tsan_atomic_f##_compare_exchange_strong( \
122		(volatile tsan_t )ptr, (tsan_t )&old_value, (tsan_t)new_value, \
123		kMacOrderBarrier, kMacFailureOrder); \
124		}
125
	130	# pragma clang diagnostic push // OSAtomicCompareAndSwap* deprecation
	131	# pragma clang diagnostic ignored "-Wdeprecated-declarations"
126	132	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
127	133	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,
128	134	long_t)
...	...	@@ -132,24 +138,28 @@ OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
132	138	int32_t)
133	139	OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
134	140	int64_t)
	141	# pragma clang diagnostic pop // OSAtomicCompareAndSwap* deprecation
	142
	143	# define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \
	144	TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
	145	SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \
	146	volatile char byte_ptr = ((volatile char )ptr) + (n >> 3); \
	147	char bit = 0x80u >> (n & 7); \
	148	char mask = clear ? ~bit : bit; \
	149	char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \
	150	return orig_byte & bit; \
	151	}
135	152
136		#define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \
137		TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
138		SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \
139		volatile char byte_ptr = ((volatile char )ptr) + (n >> 3); \
140		char bit = 0x80u >> (n & 7); \
141		char mask = clear ? ~bit : bit; \
142		char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \
143		return orig_byte & bit; \
144		}
145
146		#define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \
147		OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
148		OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
	153	# define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \
	154	OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
	155	OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
149	156
	157	# pragma clang diagnostic push // OSAtomicTestAnd* deprecation
	158	# pragma clang diagnostic ignored "-Wdeprecated-declarations"
150	159	OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)
151	160	OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,
152	161	true)
	162	# pragma clang diagnostic pop // OSAtomicTestAnd* deprecation
153	163
154	164	TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead list, void item,
155	165	size_t offset) {
...	...	@@ -161,12 +171,13 @@ TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead list, void item,
161	171	TSAN_INTERCEPTOR(void , OSAtomicDequeue, OSQueueHead list, size_t offset) {
162	172	SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);
163	173	void *item = REAL(OSAtomicDequeue)(list, offset);
164		if (item) __tsan_acquire(item);
	174	if (item)
	175	__tsan_acquire(item);
165	176	return item;
166	177	}
167	178
168	179	// OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.
169		#if !SANITIZER_IOS
	180	# if !SANITIZER_IOS
170	181
171	182	TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead list, void item,
172	183	size_t offset) {
...	...	@@ -179,11 +190,22 @@ TSAN_INTERCEPTOR(void , OSAtomicFifoDequeue, OSFifoQueueHead list,
179	190	size_t offset) {
180	191	SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);
181	192	void *item = REAL(OSAtomicFifoDequeue)(list, offset);
182		if (item) __tsan_acquire(item);
	193	if (item)
	194	__tsan_acquire(item);
183	195	return item;
184	196	}
185	197
186		#endif
	198	# endif
	199
	200	// If `OSSPINLOCK_USE_INLINED=1` is set, then SDK headers don't declare these
	201	// as functions, but macros that call non-deprecated APIs. Undefine these
	202	// macros so they don't interfere with the interceptor machinery.
	203	# undef OSSpinLockLock
	204	# undef OSSpinLockTry
	205	# undef OSSpinLockUnlock
	206
	207	# pragma clang diagnostic push // OSSpinLock* deprecation
	208	# pragma clang diagnostic ignored "-Wdeprecated-declarations"
187	209
188	210	TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
189	211	CHECK(!cur_thread()->is_dead);
...	...	@@ -216,6 +238,7 @@ TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {
216	238	Release(thr, pc, (uptr)lock);
217	239	REAL(OSSpinLockUnlock)(lock);
218	240	}
	241	# pragma clang diagnostic pop // OSSpinLock* deprecation
219	242
220	243	TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {
221	244	CHECK(!cur_thread()->is_dead);
...	...	@@ -288,7 +311,7 @@ TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) {
288	311	REAL(os_unfair_lock_unlock)(lock);
289	312	}
290	313
291		#if defined(__has_include) && __has_include(<xpc/xpc.h>)
	314	# if defined(__has_include) && __has_include(<xpc/xpc.h>)
292	315
293	316	TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
294	317	xpc_connection_t connection, xpc_handler_t handler) {
...	...	@@ -342,7 +365,7 @@ TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
342	365	REAL(xpc_connection_cancel)(connection);
343	366	}
344	367
345		#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
	368	# endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
346	369
347	370	// Determines whether the Obj-C object pointer is a tagged pointer. Tagged
348	371	// pointers encode the object data directly in their pointer bits and do not
...	...	@@ -365,7 +388,7 @@ static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) {
365	388	Map::Handle h(&Addresses, addr);
366	389	if (h.created()) {
367	390	ThreadIgnoreBegin(thr, pc);
368		h = (uptr) user_alloc(thr, pc, /size=*/1);
	391	h = (uptr)user_alloc(thr, pc, /size=*/1);
369	392	ThreadIgnoreEnd(thr);
370	393	}
371	394	return *h;
...	...	@@ -383,7 +406,8 @@ static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) {
383	406
384	407	TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) {
385	408	SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj);
386		if (!obj) return REAL(objc_sync_enter)(obj);
	409	if (!obj)
	410	return REAL(objc_sync_enter)(obj);
387	411	uptr addr = SyncAddressForObjCObject(obj, thr, pc);
388	412	MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant);
389	413	int result = REAL(objc_sync_enter)(obj);
...	...	@@ -394,11 +418,13 @@ TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) {
394	418
395	419	TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) {
396	420	SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj);
397		if (!obj) return REAL(objc_sync_exit)(obj);
	421	if (!obj)
	422	return REAL(objc_sync_exit)(obj);
398	423	uptr addr = SyncAddressForObjCObject(obj, thr, pc);
399	424	MutexUnlock(thr, pc, addr);
400	425	int result = REAL(objc_sync_exit)(obj);
401		if (result != OBJC_SYNC_SUCCESS) MutexInvalidAccess(thr, pc, addr);
	426	if (result != OBJC_SYNC_SUCCESS)
	427	MutexInvalidAccess(thr, pc, addr);
402	428	return result;
403	429	}
404	430
...	...	@@ -429,7 +455,7 @@ TSAN_INTERCEPTOR(int, swapcontext, ucontext_t oucp, const ucontext_t ucp) {
429	455
430	456	// On macOS, libc++ is always linked dynamically, so intercepting works the
431	457	// usual way.
432		#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
	458	# define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
433	459
434	460	namespace {
435	461	struct fake_shared_weak_count {

lib/tsan/tsan_interceptors_posix.cpp+45-17

...	...	@@ -12,14 +12,15 @@
12	12	// sanitizer_common/sanitizer_common_interceptors.inc
13	13	//===----------------------------------------------------------------------===//
14	14
	15	#include "sanitizer_common/sanitizer_allocator_dlsym.h"
15	16	#include "sanitizer_common/sanitizer_atomic.h"
16	17	#include "sanitizer_common/sanitizer_errno.h"
17	18	#include "sanitizer_common/sanitizer_glibc_version.h"
	19	#include "sanitizer_common/sanitizer_internal_defs.h"
18	20	#include "sanitizer_common/sanitizer_libc.h"
19	21	#include "sanitizer_common/sanitizer_linux.h"
20	22	#include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
21	23	#include "sanitizer_common/sanitizer_platform_limits_posix.h"
22		#include "sanitizer_common/sanitizer_placement_new.h"
23	24	#include "sanitizer_common/sanitizer_posix.h"
24	25	#include "sanitizer_common/sanitizer_stacktrace.h"
25	26	#include "sanitizer_common/sanitizer_tls_get_addr.h"
...	...	@@ -96,7 +97,7 @@ extern "C" int pthread_key_create(unsigned key, void (destructor)(void* v));
96	97	extern "C" int pthread_setspecific(unsigned key, const void *v);
97	98	DECLARE_REAL(int, pthread_mutexattr_gettype, void , void )
98	99	DECLARE_REAL(int, fflush, __sanitizer_FILE *fp)
99		DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size)
	100	DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, usize size)
100	101	DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
101	102	extern "C" int pthread_equal(void t1, void t2);
102	103	extern "C" void *pthread_self();
...	...	@@ -252,6 +253,13 @@ SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionBegin() {}
252	253	SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {}
253	254	#endif
254	255
	256	// FIXME: Use for `in_symbolizer()` as well. As-is we can't use
	257	// `DlSymAllocator`, because it uses the primary allocator only. Symbolizer
	258	// requires support of the secondary allocator for larger blocks.
	259	struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
	260	static bool UseImpl() { return (ctx && !ctx->initialized); }
	261	};
	262
255	263	} // namespace __tsan
256	264
257	265	static ThreadSignalContext SigCtx(ThreadState thr) {
...	...	@@ -661,6 +669,8 @@ TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) {
661	669	TSAN_INTERCEPTOR(void*, malloc, uptr size) {
662	670	if (in_symbolizer())
663	671	return InternalAlloc(size);
	672	if (DlsymAlloc::Use())
	673	return DlsymAlloc::Allocate(size);
664	674	void *p = 0;
665	675	{
666	676	SCOPED_INTERCEPTOR_RAW(malloc, size);
...	...	@@ -678,12 +688,14 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
678	688	return user_memalign(thr, pc, align, sz);
679	689	}
680	690
681		TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
	691	TSAN_INTERCEPTOR(void *, calloc, uptr n, uptr size) {
682	692	if (in_symbolizer())
683		return InternalCalloc(size, n);
	693	return InternalCalloc(n, size);
	694	if (DlsymAlloc::Use())
	695	return DlsymAlloc::Callocate(n, size);
684	696	void *p = 0;
685	697	{
686		SCOPED_INTERCEPTOR_RAW(calloc, size, n);
	698	SCOPED_INTERCEPTOR_RAW(calloc, n, size);
687	699	p = user_calloc(thr, pc, size, n);
688	700	}
689	701	invoke_malloc_hook(p, n * size);
...	...	@@ -693,6 +705,8 @@ TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
693	705	TSAN_INTERCEPTOR(void, realloc, void p, uptr size) {
694	706	if (in_symbolizer())
695	707	return InternalRealloc(p, size);
	708	if (DlsymAlloc::Use() \|\| DlsymAlloc::PointerIsMine(p))
	709	return DlsymAlloc::Realloc(p, size);
696	710	if (p)
697	711	invoke_free_hook(p);
698	712	{
...	...	@@ -703,13 +717,13 @@ TSAN_INTERCEPTOR(void, realloc, void p, uptr size) {
703	717	return p;
704	718	}
705	719
706		TSAN_INTERCEPTOR(void, reallocarray, void p, uptr size, uptr n) {
	720	TSAN_INTERCEPTOR(void , reallocarray, void p, uptr n, uptr size) {
707	721	if (in_symbolizer())
708		return InternalReallocArray(p, size, n);
	722	return InternalReallocArray(p, n, size);
709	723	if (p)
710	724	invoke_free_hook(p);
711	725	{
712		SCOPED_INTERCEPTOR_RAW(reallocarray, p, size, n);
	726	SCOPED_INTERCEPTOR_RAW(reallocarray, p, n, size);
713	727	p = user_reallocarray(thr, pc, p, size, n);
714	728	}
715	729	invoke_malloc_hook(p, size);
...	...	@@ -717,20 +731,24 @@ TSAN_INTERCEPTOR(void, reallocarray, void p, uptr size, uptr n) {
717	731	}
718	732
719	733	TSAN_INTERCEPTOR(void, free, void *p) {
720		if (p == 0)
	734	if (UNLIKELY(!p))
721	735	return;
722	736	if (in_symbolizer())
723	737	return InternalFree(p);
	738	if (DlsymAlloc::PointerIsMine(p))
	739	return DlsymAlloc::Free(p);
724	740	invoke_free_hook(p);
725	741	SCOPED_INTERCEPTOR_RAW(free, p);
726	742	user_free(thr, pc, p);
727	743	}
728	744
729	745	TSAN_INTERCEPTOR(void, cfree, void *p) {
730		if (p == 0)
	746	if (UNLIKELY(!p))
731	747	return;
732	748	if (in_symbolizer())
733	749	return InternalFree(p);
	750	if (DlsymAlloc::PointerIsMine(p))
	751	return DlsymAlloc::Free(p);
734	752	invoke_free_hook(p);
735	753	SCOPED_INTERCEPTOR_RAW(cfree, p);
736	754	user_free(thr, pc, p);
...	...	@@ -750,7 +768,7 @@ TSAN_INTERCEPTOR(char , strcpy, char dst, const char *src) {
750	768	return REAL(strcpy)(dst, src);
751	769	}
752	770
753		TSAN_INTERCEPTOR(char, strncpy, char dst, char *src, uptr n) {
	771	TSAN_INTERCEPTOR(char, strncpy, char dst, char *src, usize n) {
754	772	SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n);
755	773	uptr srclen = internal_strnlen(src, n);
756	774	MemoryAccessRange(thr, pc, (uptr)dst, n, true);
...	...	@@ -1097,7 +1115,7 @@ int internal_pthread_create(void th, void attr, void (callback)(void *),
1097	1115	}
1098	1116	int internal_pthread_join(void th, void *ret) {
1099	1117	ScopedIgnoreInterceptors ignore;
1100		return REAL(pthread_join(th, ret));
	1118	return REAL(pthread_join)(th, ret);
1101	1119	}
1102	1120	} // namespace __sanitizer
1103	1121
...	...	@@ -1662,13 +1680,23 @@ TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
1662	1680	#endif
1663	1681
1664	1682	TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) {
1665		va_list ap;
1666		va_start(ap, oflag);
1667		mode_t mode = va_arg(ap, int);
1668		va_end(ap);
	1683	mode_t mode = 0;
	1684	if (OpenReadsVaArgs(oflag)) {
	1685	va_list ap;
	1686	va_start(ap, oflag);
	1687	mode = va_arg(ap, int);
	1688	va_end(ap);
	1689	}
	1690
1669	1691	SCOPED_TSAN_INTERCEPTOR(open, name, oflag, mode);
1670	1692	READ_STRING(thr, pc, name, 0);
1671		int fd = REAL(open)(name, oflag, mode);
	1693
	1694	int fd;
	1695	if (OpenReadsVaArgs(oflag))
	1696	fd = REAL(open)(name, oflag, mode);
	1697	else
	1698	fd = REAL(open)(name, oflag);
	1699
1672	1700	if (fd >= 0)
1673	1701	FdFileCreate(thr, pc, fd);
1674	1702	return fd;

lib/tsan/tsan_interface.h+76-75

...	...	@@ -16,8 +16,8 @@
16	16	#define TSAN_INTERFACE_H
17	17
18	18	#include <sanitizer_common/sanitizer_internal_defs.h>
19		using __sanitizer::uptr;
20	19	using __sanitizer::tid_t;
	20	using __sanitizer::uptr;
21	21
22	22	// This header should NOT include any other headers.
23	23	// All functions in this header are extern "C" and start with __tsan_.
...	...	@@ -203,17 +203,18 @@ int __tsan_get_alloc_stack(uptr addr, uptr trace, uptr size, int thread_id,
203	203	namespace __tsan {
204	204
205	205	// These should match declarations from public tsan_interface_atomic.h header.
206		typedef unsigned char a8;
	206	typedef unsigned char a8;
207	207	typedef unsigned short a16;
208		typedef unsigned int a32;
	208	typedef unsigned int a32;
209	209	typedef unsigned long long a64;
210		#if !SANITIZER_GO && (defined(__SIZEOF_INT128__) \
211		\|\| (__clang_major__ * 100 + __clang_minor__ >= 302)) && \
	210	#if !SANITIZER_GO && \
	211	(defined(__SIZEOF_INT128__) \|\| \
	212	(__clang_major__ * 100 + __clang_minor__ >= 302)) && \
212	213	!defined(__mips64) && !defined(__s390x__)
213	214	__extension__ typedef __int128 a128;
214		# define __TSAN_HAS_INT128 1
	215	# define __TSAN_HAS_INT128 1
215	216	#else
216		# define __TSAN_HAS_INT128 0
	217	# define __TSAN_HAS_INT128 0
217	218	#endif
218	219
219	220	// Part of ABI, do not change.
...	...	@@ -231,180 +232,180 @@ struct ThreadState;
231	232
232	233	extern "C" {
233	234	SANITIZER_INTERFACE_ATTRIBUTE
234		a8 __tsan_atomic8_load(const volatile a8 *a, morder mo);
	235	a8 __tsan_atomic8_load(const volatile a8 *a, int mo);
235	236	SANITIZER_INTERFACE_ATTRIBUTE
236		a16 __tsan_atomic16_load(const volatile a16 *a, morder mo);
	237	a16 __tsan_atomic16_load(const volatile a16 *a, int mo);
237	238	SANITIZER_INTERFACE_ATTRIBUTE
238		a32 __tsan_atomic32_load(const volatile a32 *a, morder mo);
	239	a32 __tsan_atomic32_load(const volatile a32 *a, int mo);
239	240	SANITIZER_INTERFACE_ATTRIBUTE
240		a64 __tsan_atomic64_load(const volatile a64 *a, morder mo);
	241	a64 __tsan_atomic64_load(const volatile a64 *a, int mo);
241	242	#if __TSAN_HAS_INT128
242	243	SANITIZER_INTERFACE_ATTRIBUTE
243		a128 __tsan_atomic128_load(const volatile a128 *a, morder mo);
	244	a128 __tsan_atomic128_load(const volatile a128 *a, int mo);
244	245	#endif
245	246
246	247	SANITIZER_INTERFACE_ATTRIBUTE
247		void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo);
	248	void __tsan_atomic8_store(volatile a8 *a, a8 v, int mo);
248	249	SANITIZER_INTERFACE_ATTRIBUTE
249		void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo);
	250	void __tsan_atomic16_store(volatile a16 *a, a16 v, int mo);
250	251	SANITIZER_INTERFACE_ATTRIBUTE
251		void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo);
	252	void __tsan_atomic32_store(volatile a32 *a, a32 v, int mo);
252	253	SANITIZER_INTERFACE_ATTRIBUTE
253		void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo);
	254	void __tsan_atomic64_store(volatile a64 *a, a64 v, int mo);
254	255	#if __TSAN_HAS_INT128
255	256	SANITIZER_INTERFACE_ATTRIBUTE
256		void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo);
	257	void __tsan_atomic128_store(volatile a128 *a, a128 v, int mo);
257	258	#endif
258	259
259	260	SANITIZER_INTERFACE_ATTRIBUTE
260		a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo);
	261	a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, int mo);
261	262	SANITIZER_INTERFACE_ATTRIBUTE
262		a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo);
	263	a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, int mo);
263	264	SANITIZER_INTERFACE_ATTRIBUTE
264		a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo);
	265	a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, int mo);
265	266	SANITIZER_INTERFACE_ATTRIBUTE
266		a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo);
	267	a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, int mo);
267	268	#if __TSAN_HAS_INT128
268	269	SANITIZER_INTERFACE_ATTRIBUTE
269		a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo);
	270	a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, int mo);
270	271	#endif
271	272
272	273	SANITIZER_INTERFACE_ATTRIBUTE
273		a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo);
	274	a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, int mo);
274	275	SANITIZER_INTERFACE_ATTRIBUTE
275		a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo);
	276	a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, int mo);
276	277	SANITIZER_INTERFACE_ATTRIBUTE
277		a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo);
	278	a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, int mo);
278	279	SANITIZER_INTERFACE_ATTRIBUTE
279		a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo);
	280	a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, int mo);
280	281	#if __TSAN_HAS_INT128
281	282	SANITIZER_INTERFACE_ATTRIBUTE
282		a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo);
	283	a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, int mo);
283	284	#endif
284	285
285	286	SANITIZER_INTERFACE_ATTRIBUTE
286		a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo);
	287	a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, int mo);
287	288	SANITIZER_INTERFACE_ATTRIBUTE
288		a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo);
	289	a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, int mo);
289	290	SANITIZER_INTERFACE_ATTRIBUTE
290		a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo);
	291	a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, int mo);
291	292	SANITIZER_INTERFACE_ATTRIBUTE
292		a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo);
	293	a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, int mo);
293	294	#if __TSAN_HAS_INT128
294	295	SANITIZER_INTERFACE_ATTRIBUTE
295		a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo);
	296	a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, int mo);
296	297	#endif
297	298
298	299	SANITIZER_INTERFACE_ATTRIBUTE
299		a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo);
	300	a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, int mo);
300	301	SANITIZER_INTERFACE_ATTRIBUTE
301		a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo);
	302	a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, int mo);
302	303	SANITIZER_INTERFACE_ATTRIBUTE
303		a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo);
	304	a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, int mo);
304	305	SANITIZER_INTERFACE_ATTRIBUTE
305		a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo);
	306	a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, int mo);
306	307	#if __TSAN_HAS_INT128
307	308	SANITIZER_INTERFACE_ATTRIBUTE
308		a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo);
	309	a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, int mo);
309	310	#endif
310	311
311	312	SANITIZER_INTERFACE_ATTRIBUTE
312		a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo);
	313	a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, int mo);
313	314	SANITIZER_INTERFACE_ATTRIBUTE
314		a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo);
	315	a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, int mo);
315	316	SANITIZER_INTERFACE_ATTRIBUTE
316		a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo);
	317	a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, int mo);
317	318	SANITIZER_INTERFACE_ATTRIBUTE
318		a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo);
	319	a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, int mo);
319	320	#if __TSAN_HAS_INT128
320	321	SANITIZER_INTERFACE_ATTRIBUTE
321		a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo);
	322	a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, int mo);
322	323	#endif
323	324
324	325	SANITIZER_INTERFACE_ATTRIBUTE
325		a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo);
	326	a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, int mo);
326	327	SANITIZER_INTERFACE_ATTRIBUTE
327		a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo);
	328	a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, int mo);
328	329	SANITIZER_INTERFACE_ATTRIBUTE
329		a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo);
	330	a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, int mo);
330	331	SANITIZER_INTERFACE_ATTRIBUTE
331		a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo);
	332	a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, int mo);
332	333	#if __TSAN_HAS_INT128
333	334	SANITIZER_INTERFACE_ATTRIBUTE
334		a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo);
	335	a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, int mo);
335	336	#endif
336	337
337	338	SANITIZER_INTERFACE_ATTRIBUTE
338		a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo);
	339	a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, int mo);
339	340	SANITIZER_INTERFACE_ATTRIBUTE
340		a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo);
	341	a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, int mo);
341	342	SANITIZER_INTERFACE_ATTRIBUTE
342		a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo);
	343	a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, int mo);
343	344	SANITIZER_INTERFACE_ATTRIBUTE
344		a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo);
	345	a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, int mo);
345	346	#if __TSAN_HAS_INT128
346	347	SANITIZER_INTERFACE_ATTRIBUTE
347		a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo);
	348	a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, int mo);
348	349	#endif
349	350
350	351	SANITIZER_INTERFACE_ATTRIBUTE
351		int __tsan_atomic8_compare_exchange_strong(volatile a8 a, a8 c, a8 v,
352		morder mo, morder fmo);
	352	int __tsan_atomic8_compare_exchange_strong(volatile a8 a, a8 c, a8 v, int mo,
	353	int fmo);
353	354	SANITIZER_INTERFACE_ATTRIBUTE
354	355	int __tsan_atomic16_compare_exchange_strong(volatile a16 a, a16 c, a16 v,
355		morder mo, morder fmo);
	356	int mo, int fmo);
356	357	SANITIZER_INTERFACE_ATTRIBUTE
357	358	int __tsan_atomic32_compare_exchange_strong(volatile a32 a, a32 c, a32 v,
358		morder mo, morder fmo);
	359	int mo, int fmo);
359	360	SANITIZER_INTERFACE_ATTRIBUTE
360	361	int __tsan_atomic64_compare_exchange_strong(volatile a64 a, a64 c, a64 v,
361		morder mo, morder fmo);
	362	int mo, int fmo);
362	363	#if __TSAN_HAS_INT128
363	364	SANITIZER_INTERFACE_ATTRIBUTE
364	365	int __tsan_atomic128_compare_exchange_strong(volatile a128 a, a128 c, a128 v,
365		morder mo, morder fmo);
	366	int mo, int fmo);
366	367	#endif
367	368
368	369	SANITIZER_INTERFACE_ATTRIBUTE
369		int __tsan_atomic8_compare_exchange_weak(volatile a8 a, a8 c, a8 v, morder mo,
370		morder fmo);
	370	int __tsan_atomic8_compare_exchange_weak(volatile a8 a, a8 c, a8 v, int mo,
	371	int fmo);
371	372	SANITIZER_INTERFACE_ATTRIBUTE
372	373	int __tsan_atomic16_compare_exchange_weak(volatile a16 a, a16 c, a16 v,
373		morder mo, morder fmo);
	374	int mo, int fmo);
374	375	SANITIZER_INTERFACE_ATTRIBUTE
375	376	int __tsan_atomic32_compare_exchange_weak(volatile a32 a, a32 c, a32 v,
376		morder mo, morder fmo);
	377	int mo, int fmo);
377	378	SANITIZER_INTERFACE_ATTRIBUTE
378	379	int __tsan_atomic64_compare_exchange_weak(volatile a64 a, a64 c, a64 v,
379		morder mo, morder fmo);
	380	int mo, int fmo);
380	381	#if __TSAN_HAS_INT128
381	382	SANITIZER_INTERFACE_ATTRIBUTE
382	383	int __tsan_atomic128_compare_exchange_weak(volatile a128 a, a128 c, a128 v,
383		morder mo, morder fmo);
	384	int mo, int fmo);
384	385	#endif
385	386
386	387	SANITIZER_INTERFACE_ATTRIBUTE
387		a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, morder mo,
388		morder fmo);
	388	a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, int mo,
	389	int fmo);
389	390	SANITIZER_INTERFACE_ATTRIBUTE
390		a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
391		morder mo, morder fmo);
	391	a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, int mo,
	392	int fmo);
392	393	SANITIZER_INTERFACE_ATTRIBUTE
393		a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
394		morder mo, morder fmo);
	394	a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, int mo,
	395	int fmo);
395	396	SANITIZER_INTERFACE_ATTRIBUTE
396		a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
397		morder mo, morder fmo);
	397	a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, int mo,
	398	int fmo);
398	399	#if __TSAN_HAS_INT128
399	400	SANITIZER_INTERFACE_ATTRIBUTE
400	401	a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
401		morder mo, morder fmo);
	402	int mo, int fmo);
402	403	#endif
403	404
404	405	SANITIZER_INTERFACE_ATTRIBUTE
405		void __tsan_atomic_thread_fence(morder mo);
	406	void __tsan_atomic_thread_fence(int mo);
406	407	SANITIZER_INTERFACE_ATTRIBUTE
407		void __tsan_atomic_signal_fence(morder mo);
	408	void __tsan_atomic_signal_fence(int mo);
408	409
409	410	SANITIZER_INTERFACE_ATTRIBUTE
410	411	void __tsan_go_atomic32_load(ThreadState thr, uptr cpc, uptr pc, u8 a);

lib/tsan/tsan_interface_atomic.cpp+476-430

...	...	@@ -18,9 +18,9 @@
18	18	// The following page contains more background information:
19	19	// http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
20	20
	21	#include "sanitizer_common/sanitizer_mutex.h"
21	22	#include "sanitizer_common/sanitizer_placement_new.h"
22	23	#include "sanitizer_common/sanitizer_stacktrace.h"
23		#include "sanitizer_common/sanitizer_mutex.h"
24	24	#include "tsan_flags.h"
25	25	#include "tsan_interface.h"
26	26	#include "tsan_rtl.h"
...	...	@@ -34,8 +34,8 @@ static StaticSpinMutex mutex128;
34	34
35	35	#if SANITIZER_DEBUG
36	36	static bool IsLoadOrder(morder mo) {
37		return mo == mo_relaxed \|\| mo == mo_consume
38		\|\| mo == mo_acquire \|\| mo == mo_seq_cst;
	37	return mo == mo_relaxed \|\| mo == mo_consume \|\| mo == mo_acquire \|\|
	38	mo == mo_seq_cst;
39	39	}
40	40
41	41	static bool IsStoreOrder(morder mo) {
...	...	@@ -48,42 +48,49 @@ static bool IsReleaseOrder(morder mo) {
48	48	}
49	49
50	50	static bool IsAcquireOrder(morder mo) {
51		return mo == mo_consume \|\| mo == mo_acquire
52		\|\| mo == mo_acq_rel \|\| mo == mo_seq_cst;
	51	return mo == mo_consume \|\| mo == mo_acquire \|\| mo == mo_acq_rel \|\|
	52	mo == mo_seq_cst;
53	53	}
54	54
55	55	static bool IsAcqRelOrder(morder mo) {
56	56	return mo == mo_acq_rel \|\| mo == mo_seq_cst;
57	57	}
58	58
59		template<typename T> T func_xchg(volatile T *v, T op) {
	59	template <typename T>
	60	T func_xchg(volatile T *v, T op) {
60	61	T res = __sync_lock_test_and_set(v, op);
61	62	// __sync_lock_test_and_set does not contain full barrier.
62	63	__sync_synchronize();
63	64	return res;
64	65	}
65	66
66		template<typename T> T func_add(volatile T *v, T op) {
	67	template <typename T>
	68	T func_add(volatile T *v, T op) {
67	69	return __sync_fetch_and_add(v, op);
68	70	}
69	71
70		template<typename T> T func_sub(volatile T *v, T op) {
	72	template <typename T>
	73	T func_sub(volatile T *v, T op) {
71	74	return __sync_fetch_and_sub(v, op);
72	75	}
73	76
74		template<typename T> T func_and(volatile T *v, T op) {
	77	template <typename T>
	78	T func_and(volatile T *v, T op) {
75	79	return __sync_fetch_and_and(v, op);
76	80	}
77	81
78		template<typename T> T func_or(volatile T *v, T op) {
	82	template <typename T>
	83	T func_or(volatile T *v, T op) {
79	84	return __sync_fetch_and_or(v, op);
80	85	}
81	86
82		template<typename T> T func_xor(volatile T *v, T op) {
	87	template <typename T>
	88	T func_xor(volatile T *v, T op) {
83	89	return __sync_fetch_and_xor(v, op);
84	90	}
85	91
86		template<typename T> T func_nand(volatile T *v, T op) {
	92	template <typename T>
	93	T func_nand(volatile T *v, T op) {
87	94	// clang does not support __sync_fetch_and_nand.
88	95	T cmp = *v;
89	96	for (;;) {
...	...	@@ -95,7 +102,8 @@ template<typename T> T func_nand(volatile T *v, T op) {
95	102	}
96	103	}
97	104
98		template<typename T> T func_cas(volatile T *v, T cmp, T xch) {
	105	template <typename T>
	106	T func_cas(volatile T *v, T cmp, T xch) {
99	107	return __sync_val_compare_and_swap(v, cmp, xch);
100	108	}
101	109
...	...	@@ -103,8 +111,8 @@ template<typename T> T func_cas(volatile T *v, T cmp, T xch) {
103	111	// Atomic ops are executed under tsan internal mutex,
104	112	// here we assume that the atomic variables are not accessed
105	113	// from non-instrumented code.
106		#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO \
107		&& __TSAN_HAS_INT128
	114	#if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO && \
	115	__TSAN_HAS_INT128
108	116	a128 func_xchg(volatile a128 *v, a128 op) {
109	117	SpinMutexLock lock(&mutex128);
110	118	a128 cmp = *v;
...	...	@@ -197,89 +205,24 @@ static atomic_uint64_t to_atomic(const volatile a64 a) {
197	205
198	206	static memory_order to_mo(morder mo) {
199	207	switch (mo) {
200		case mo_relaxed: return memory_order_relaxed;
201		case mo_consume: return memory_order_consume;
202		case mo_acquire: return memory_order_acquire;
203		case mo_release: return memory_order_release;
204		case mo_acq_rel: return memory_order_acq_rel;
205		case mo_seq_cst: return memory_order_seq_cst;
	208	case mo_relaxed:
	209	return memory_order_relaxed;
	210	case mo_consume:
	211	return memory_order_consume;
	212	case mo_acquire:
	213	return memory_order_acquire;
	214	case mo_release:
	215	return memory_order_release;
	216	case mo_acq_rel:
	217	return memory_order_acq_rel;
	218	case mo_seq_cst:
	219	return memory_order_seq_cst;
206	220	}
207	221	DCHECK(0);
208	222	return memory_order_seq_cst;
209	223	}
210	224
211		template<typename T>
212		static T NoTsanAtomicLoad(const volatile T *a, morder mo) {
213		return atomic_load(to_atomic(a), to_mo(mo));
214		}
215
216		#if __TSAN_HAS_INT128 && !SANITIZER_GO
217		static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
218		SpinMutexLock lock(&mutex128);
219		return *a;
220		}
221		#endif
222
223		template <typename T>
224		static T AtomicLoad(ThreadState thr, uptr pc, const volatile T a, morder mo) {
225		DCHECK(IsLoadOrder(mo));
226		// This fast-path is critical for performance.
227		// Assume the access is atomic.
228		if (!IsAcquireOrder(mo)) {
229		MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
230		kAccessRead \| kAccessAtomic);
231		return NoTsanAtomicLoad(a, mo);
232		}
233		// Don't create sync object if it does not exist yet. For example, an atomic
234		// pointer is initialized to nullptr and then periodically acquire-loaded.
235		T v = NoTsanAtomicLoad(a, mo);
236		SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a);
237		if (s) {
238		SlotLocker locker(thr);
239		ReadLock lock(&s->mtx);
240		thr->clock.Acquire(s->clock);
241		// Re-read under sync mutex because we need a consistent snapshot
242		// of the value and the clock we acquire.
243		v = NoTsanAtomicLoad(a, mo);
244		}
245		MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessRead \| kAccessAtomic);
246		return v;
247		}
248
249		template<typename T>
250		static void NoTsanAtomicStore(volatile T *a, T v, morder mo) {
251		atomic_store(to_atomic(a), v, to_mo(mo));
252		}
253
254		#if __TSAN_HAS_INT128 && !SANITIZER_GO
255		static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) {
256		SpinMutexLock lock(&mutex128);
257		*a = v;
258		}
259		#endif
260
261		template <typename T>
262		static void AtomicStore(ThreadState thr, uptr pc, volatile T a, T v,
263		morder mo) {
264		DCHECK(IsStoreOrder(mo));
265		MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite \| kAccessAtomic);
266		// This fast-path is critical for performance.
267		// Assume the access is atomic.
268		// Strictly saying even relaxed store cuts off release sequence,
269		// so must reset the clock.
270		if (!IsReleaseOrder(mo)) {
271		NoTsanAtomicStore(a, v, mo);
272		return;
273		}
274		SlotLocker locker(thr);
275		{
276		auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
277		Lock lock(&s->mtx);
278		thr->clock.ReleaseStore(&s->clock);
279		NoTsanAtomicStore(a, v, mo);
280		}
281		IncrementEpoch(thr);
282		}
	225	namespace {
283	226
284	227	template <typename T, T (F)(volatile T v, T op)>
285	228	static T AtomicRMW(ThreadState thr, uptr pc, volatile T a, T v, morder mo) {
...	...	@@ -303,175 +246,265 @@ static T AtomicRMW(ThreadState thr, uptr pc, volatile T a, T v, morder mo) {
303	246	return v;
304	247	}
305	248
306		template<typename T>
307		static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) {
308		return func_xchg(a, v);
309		}
	249	struct OpLoad {
	250	template <typename T>
	251	static T NoTsanAtomic(morder mo, const volatile T *a) {
	252	return atomic_load(to_atomic(a), to_mo(mo));
	253	}
310	254
311		template<typename T>
312		static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) {
313		return func_add(a, v);
314		}
	255	#if __TSAN_HAS_INT128 && !SANITIZER_GO
	256	static a128 NoTsanAtomic(morder mo, const volatile a128 *a) {
	257	SpinMutexLock lock(&mutex128);
	258	return *a;
	259	}
	260	#endif
315	261
316		template<typename T>
317		static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) {
318		return func_sub(a, v);
319		}
	262	template <typename T>
	263	static T Atomic(ThreadState thr, uptr pc, morder mo, const volatile T a) {
	264	DCHECK(IsLoadOrder(mo));
	265	// This fast-path is critical for performance.
	266	// Assume the access is atomic.
	267	if (!IsAcquireOrder(mo)) {
	268	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
	269	kAccessRead \| kAccessAtomic);
	270	return NoTsanAtomic(mo, a);
	271	}
	272	// Don't create sync object if it does not exist yet. For example, an atomic
	273	// pointer is initialized to nullptr and then periodically acquire-loaded.
	274	T v = NoTsanAtomic(mo, a);
	275	SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a);
	276	if (s) {
	277	SlotLocker locker(thr);
	278	ReadLock lock(&s->mtx);
	279	thr->clock.Acquire(s->clock);
	280	// Re-read under sync mutex because we need a consistent snapshot
	281	// of the value and the clock we acquire.
	282	v = NoTsanAtomic(mo, a);
	283	}
	284	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
	285	kAccessRead \| kAccessAtomic);
	286	return v;
	287	}
	288	};
320	289
321		template<typename T>
322		static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) {
323		return func_and(a, v);
324		}
	290	struct OpStore {
	291	template <typename T>
	292	static void NoTsanAtomic(morder mo, volatile T *a, T v) {
	293	atomic_store(to_atomic(a), v, to_mo(mo));
	294	}
325	295
326		template<typename T>
327		static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) {
328		return func_or(a, v);
329		}
	296	#if __TSAN_HAS_INT128 && !SANITIZER_GO
	297	static void NoTsanAtomic(morder mo, volatile a128 *a, a128 v) {
	298	SpinMutexLock lock(&mutex128);
	299	*a = v;
	300	}
	301	#endif
330	302
331		template<typename T>
332		static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) {
333		return func_xor(a, v);
334		}
	303	template <typename T>
	304	static void Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
	305	DCHECK(IsStoreOrder(mo));
	306	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
	307	kAccessWrite \| kAccessAtomic);
	308	// This fast-path is critical for performance.
	309	// Assume the access is atomic.
	310	// Strictly saying even relaxed store cuts off release sequence,
	311	// so must reset the clock.
	312	if (!IsReleaseOrder(mo)) {
	313	NoTsanAtomic(mo, a, v);
	314	return;
	315	}
	316	SlotLocker locker(thr);
	317	{
	318	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
	319	Lock lock(&s->mtx);
	320	thr->clock.ReleaseStore(&s->clock);
	321	NoTsanAtomic(mo, a, v);
	322	}
	323	IncrementEpoch(thr);
	324	}
	325	};
335	326
336		template<typename T>
337		static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) {
338		return func_nand(a, v);
339		}
	327	struct OpExchange {
	328	template <typename T>
	329	static T NoTsanAtomic(morder mo, volatile T *a, T v) {
	330	return func_xchg(a, v);
	331	}
	332	template <typename T>
	333	static T Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
	334	return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
	335	}
	336	};
340	337
341		template<typename T>
342		static T AtomicExchange(ThreadState thr, uptr pc, volatile T a, T v,
343		morder mo) {
344		return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
345		}
	338	struct OpFetchAdd {
	339	template <typename T>
	340	static T NoTsanAtomic(morder mo, volatile T *a, T v) {
	341	return func_add(a, v);
	342	}
346	343
347		template<typename T>
348		static T AtomicFetchAdd(ThreadState thr, uptr pc, volatile T a, T v,
349		morder mo) {
350		return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
351		}
	344	template <typename T>
	345	static T Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
	346	return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
	347	}
	348	};
352	349
353		template<typename T>
354		static T AtomicFetchSub(ThreadState thr, uptr pc, volatile T a, T v,
355		morder mo) {
356		return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
357		}
	350	struct OpFetchSub {
	351	template <typename T>
	352	static T NoTsanAtomic(morder mo, volatile T *a, T v) {
	353	return func_sub(a, v);
	354	}
358	355
359		template<typename T>
360		static T AtomicFetchAnd(ThreadState thr, uptr pc, volatile T a, T v,
361		morder mo) {
362		return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
363		}
	356	template <typename T>
	357	static T Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
	358	return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
	359	}
	360	};
364	361
365		template<typename T>
366		static T AtomicFetchOr(ThreadState thr, uptr pc, volatile T a, T v,
367		morder mo) {
368		return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
369		}
	362	struct OpFetchAnd {
	363	template <typename T>
	364	static T NoTsanAtomic(morder mo, volatile T *a, T v) {
	365	return func_and(a, v);
	366	}
370	367
371		template<typename T>
372		static T AtomicFetchXor(ThreadState thr, uptr pc, volatile T a, T v,
373		morder mo) {
374		return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
375		}
	368	template <typename T>
	369	static T Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
	370	return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
	371	}
	372	};
376	373
377		template<typename T>
378		static T AtomicFetchNand(ThreadState thr, uptr pc, volatile T a, T v,
379		morder mo) {
380		return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
381		}
	374	struct OpFetchOr {
	375	template <typename T>
	376	static T NoTsanAtomic(morder mo, volatile T *a, T v) {
	377	return func_or(a, v);
	378	}
382	379
383		template<typename T>
384		static bool NoTsanAtomicCAS(volatile T a, T c, T v, morder mo, morder fmo) {
385		return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo));
386		}
	380	template <typename T>
	381	static T Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
	382	return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
	383	}
	384	};
387	385
388		#if __TSAN_HAS_INT128
389		static bool NoTsanAtomicCAS(volatile a128 a, a128 c, a128 v,
390		morder mo, morder fmo) {
391		a128 old = *c;
392		a128 cur = func_cas(a, old, v);
393		if (cur == old)
394		return true;
395		*c = cur;
396		return false;
397		}
398		#endif
	386	struct OpFetchXor {
	387	template <typename T>
	388	static T NoTsanAtomic(morder mo, volatile T *a, T v) {
	389	return func_xor(a, v);
	390	}
399	391
400		template<typename T>
401		static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {
402		NoTsanAtomicCAS(a, &c, v, mo, fmo);
403		return c;
404		}
	392	template <typename T>
	393	static T Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
	394	return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
	395	}
	396	};
405	397
406		template <typename T>
407		static bool AtomicCAS(ThreadState thr, uptr pc, volatile T a, T *c, T v,
408		morder mo, morder fmo) {
409		// 31.7.2.18: "The failure argument shall not be memory_order_release
410		// nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic
411		// (mo_relaxed) when those are used.
412		DCHECK(IsLoadOrder(fmo));
	398	struct OpFetchNand {
	399	template <typename T>
	400	static T NoTsanAtomic(morder mo, volatile T *a, T v) {
	401	return func_nand(a, v);
	402	}
413	403
414		MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite \| kAccessAtomic);
415		if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) {
416		T cc = *c;
417		T pr = func_cas(a, cc, v);
418		if (pr == cc)
	404	template <typename T>
	405	static T Atomic(ThreadState thr, uptr pc, morder mo, volatile T a, T v) {
	406	return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
	407	}
	408	};
	409
	410	struct OpCAS {
	411	template <typename T>
	412	static bool NoTsanAtomic(morder mo, morder fmo, volatile T a, T c, T v) {
	413	return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo));
	414	}
	415
	416	#if __TSAN_HAS_INT128
	417	static bool NoTsanAtomic(morder mo, morder fmo, volatile a128 a, a128 c,
	418	a128 v) {
	419	a128 old = *c;
	420	a128 cur = func_cas(a, old, v);
	421	if (cur == old)
419	422	return true;
420		*c = pr;
	423	*c = cur;
421	424	return false;
422	425	}
423		SlotLocker locker(thr);
424		bool release = IsReleaseOrder(mo);
425		bool success;
426		{
427		auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
428		RWLock lock(&s->mtx, release);
429		T cc = *c;
430		T pr = func_cas(a, cc, v);
431		success = pr == cc;
432		if (!success) {
	426	#endif
	427
	428	template <typename T>
	429	static T NoTsanAtomic(morder mo, morder fmo, volatile T *a, T c, T v) {
	430	NoTsanAtomic(mo, fmo, a, &c, v);
	431	return c;
	432	}
	433
	434	template <typename T>
	435	static bool Atomic(ThreadState *thr, uptr pc, morder mo, morder fmo,
	436	volatile T a, T c, T v) {
	437	// 31.7.2.18: "The failure argument shall not be memory_order_release
	438	// nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic
	439	// (mo_relaxed) when those are used.
	440	DCHECK(IsLoadOrder(fmo));
	441
	442	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
	443	kAccessWrite \| kAccessAtomic);
	444	if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) {
	445	T cc = *c;
	446	T pr = func_cas(a, cc, v);
	447	if (pr == cc)
	448	return true;
433	449	*c = pr;
434		mo = fmo;
	450	return false;
435	451	}
436		if (success && IsAcqRelOrder(mo))
437		thr->clock.ReleaseAcquire(&s->clock);
438		else if (success && IsReleaseOrder(mo))
439		thr->clock.Release(&s->clock);
440		else if (IsAcquireOrder(mo))
441		thr->clock.Acquire(s->clock);
	452	SlotLocker locker(thr);
	453	bool release = IsReleaseOrder(mo);
	454	bool success;
	455	{
	456	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
	457	RWLock lock(&s->mtx, release);
	458	T cc = *c;
	459	T pr = func_cas(a, cc, v);
	460	success = pr == cc;
	461	if (!success) {
	462	*c = pr;
	463	mo = fmo;
	464	}
	465	if (success && IsAcqRelOrder(mo))
	466	thr->clock.ReleaseAcquire(&s->clock);
	467	else if (success && IsReleaseOrder(mo))
	468	thr->clock.Release(&s->clock);
	469	else if (IsAcquireOrder(mo))
	470	thr->clock.Acquire(s->clock);
	471	}
	472	if (success && release)
	473	IncrementEpoch(thr);
	474	return success;
442	475	}
443		if (success && release)
444		IncrementEpoch(thr);
445		return success;
446		}
447	476
448		template<typename T>
449		static T AtomicCAS(ThreadState *thr, uptr pc,
450		volatile T *a, T c, T v, morder mo, morder fmo) {
451		AtomicCAS(thr, pc, a, &c, v, mo, fmo);
452		return c;
453		}
	477	template <typename T>
	478	static T Atomic(ThreadState *thr, uptr pc, morder mo, morder fmo,
	479	volatile T *a, T c, T v) {
	480	Atomic(thr, pc, mo, fmo, a, &c, v);
	481	return c;
	482	}
	483	};
454	484
455	485	#if !SANITIZER_GO
456		static void NoTsanAtomicFence(morder mo) {
457		__sync_synchronize();
458		}
	486	struct OpFence {
	487	static void NoTsanAtomic(morder mo) { __sync_synchronize(); }
459	488
460		static void AtomicFence(ThreadState *thr, uptr pc, morder mo) {
461		// FIXME(dvyukov): not implemented.
462		__sync_synchronize();
463		}
	489	static void Atomic(ThreadState *thr, uptr pc, morder mo) {
	490	// FIXME(dvyukov): not implemented.
	491	__sync_synchronize();
	492	}
	493	};
464	494	#endif
465	495
	496	} // namespace
	497
466	498	// Interface functions follow.
467	499	#if !SANITIZER_GO
468	500
469	501	// C/C++
470	502
471	503	static morder convert_morder(morder mo) {
472		if (flags()->force_seq_cst_atomics)
473		return (morder)mo_seq_cst;
	504	return flags()->force_seq_cst_atomics ? mo_seq_cst : mo;
	505	}
474	506
	507	static morder to_morder(int mo) {
475	508	// Filter out additional memory order flags:
476	509	// MEMMODEL_SYNC = 1 << 15
477	510	// __ATOMIC_HLE_ACQUIRE = 1 << 16
...	...	@@ -482,468 +515,481 @@ static morder convert_morder(morder mo) {
482	515	// since we use __sync_ atomics for actual atomic operations,
483	516	// we can safely ignore it as well. It also subtly affects semantics,
484	517	// but we don't model the difference.
485		return (morder)(mo & 0x7fff);
	518	morder res = static_cast<morder>(static_cast<u8>(mo));
	519	DCHECK_LE(res, mo_seq_cst);
	520	return res;
486	521	}
487	522
488		# define ATOMIC_IMPL(func, ...) \
489		ThreadState *const thr = cur_thread(); \
490		ProcessPendingSignals(thr); \
491		if (UNLIKELY(thr->ignore_sync \|\| thr->ignore_interceptors)) \
492		return NoTsanAtomic##func(__VA_ARGS__); \
493		mo = convert_morder(mo); \
494		return Atomic##func(thr, GET_CALLER_PC(), __VA_ARGS__);
	523	template <class Op, class... Types>
	524	ALWAYS_INLINE auto AtomicImpl(morder mo, Types... args) {
	525	ThreadState *const thr = cur_thread();
	526	ProcessPendingSignals(thr);
	527	if (UNLIKELY(thr->ignore_sync \|\| thr->ignore_interceptors))
	528	return Op::NoTsanAtomic(mo, args...);
	529	return Op::Atomic(thr, GET_CALLER_PC(), convert_morder(mo), args...);
	530	}
495	531
496	532	extern "C" {
497	533	SANITIZER_INTERFACE_ATTRIBUTE
498		a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
499		ATOMIC_IMPL(Load, a, mo);
	534	a8 __tsan_atomic8_load(const volatile a8 *a, int mo) {
	535	return AtomicImpl<OpLoad>(to_morder(mo), a);
500	536	}
501	537
502	538	SANITIZER_INTERFACE_ATTRIBUTE
503		a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {
504		ATOMIC_IMPL(Load, a, mo);
	539	a16 __tsan_atomic16_load(const volatile a16 *a, int mo) {
	540	return AtomicImpl<OpLoad>(to_morder(mo), a);
505	541	}
506	542
507	543	SANITIZER_INTERFACE_ATTRIBUTE
508		a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {
509		ATOMIC_IMPL(Load, a, mo);
	544	a32 __tsan_atomic32_load(const volatile a32 *a, int mo) {
	545	return AtomicImpl<OpLoad>(to_morder(mo), a);
510	546	}
511	547
512	548	SANITIZER_INTERFACE_ATTRIBUTE
513		a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {
514		ATOMIC_IMPL(Load, a, mo);
	549	a64 __tsan_atomic64_load(const volatile a64 *a, int mo) {
	550	return AtomicImpl<OpLoad>(to_morder(mo), a);
515	551	}
516	552
517		#if __TSAN_HAS_INT128
	553	# if __TSAN_HAS_INT128
518	554	SANITIZER_INTERFACE_ATTRIBUTE
519		a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {
520		ATOMIC_IMPL(Load, a, mo);
	555	a128 __tsan_atomic128_load(const volatile a128 *a, int mo) {
	556	return AtomicImpl<OpLoad>(to_morder(mo), a);
521	557	}
522		#endif
	558	# endif
523	559
524	560	SANITIZER_INTERFACE_ATTRIBUTE
525		void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {
526		ATOMIC_IMPL(Store, a, v, mo);
	561	void __tsan_atomic8_store(volatile a8 *a, a8 v, int mo) {
	562	return AtomicImpl<OpStore>(to_morder(mo), a, v);
527	563	}
528	564
529	565	SANITIZER_INTERFACE_ATTRIBUTE
530		void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {
531		ATOMIC_IMPL(Store, a, v, mo);
	566	void __tsan_atomic16_store(volatile a16 *a, a16 v, int mo) {
	567	return AtomicImpl<OpStore>(to_morder(mo), a, v);
532	568	}
533	569
534	570	SANITIZER_INTERFACE_ATTRIBUTE
535		void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {
536		ATOMIC_IMPL(Store, a, v, mo);
	571	void __tsan_atomic32_store(volatile a32 *a, a32 v, int mo) {
	572	return AtomicImpl<OpStore>(to_morder(mo), a, v);
537	573	}
538	574
539	575	SANITIZER_INTERFACE_ATTRIBUTE
540		void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {
541		ATOMIC_IMPL(Store, a, v, mo);
	576	void __tsan_atomic64_store(volatile a64 *a, a64 v, int mo) {
	577	return AtomicImpl<OpStore>(to_morder(mo), a, v);
542	578	}
543	579
544		#if __TSAN_HAS_INT128
	580	# if __TSAN_HAS_INT128
545	581	SANITIZER_INTERFACE_ATTRIBUTE
546		void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {
547		ATOMIC_IMPL(Store, a, v, mo);
	582	void __tsan_atomic128_store(volatile a128 *a, a128 v, int mo) {
	583	return AtomicImpl<OpStore>(to_morder(mo), a, v);
548	584	}
549		#endif
	585	# endif
550	586
551	587	SANITIZER_INTERFACE_ATTRIBUTE
552		a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {
553		ATOMIC_IMPL(Exchange, a, v, mo);
	588	a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, int mo) {
	589	return AtomicImpl<OpExchange>(to_morder(mo), a, v);
554	590	}
555	591
556	592	SANITIZER_INTERFACE_ATTRIBUTE
557		a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {
558		ATOMIC_IMPL(Exchange, a, v, mo);
	593	a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, int mo) {
	594	return AtomicImpl<OpExchange>(to_morder(mo), a, v);
559	595	}
560	596
561	597	SANITIZER_INTERFACE_ATTRIBUTE
562		a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {
563		ATOMIC_IMPL(Exchange, a, v, mo);
	598	a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, int mo) {
	599	return AtomicImpl<OpExchange>(to_morder(mo), a, v);
564	600	}
565	601
566	602	SANITIZER_INTERFACE_ATTRIBUTE
567		a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {
568		ATOMIC_IMPL(Exchange, a, v, mo);
	603	a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, int mo) {
	604	return AtomicImpl<OpExchange>(to_morder(mo), a, v);
569	605	}
570	606
571		#if __TSAN_HAS_INT128
	607	# if __TSAN_HAS_INT128
572	608	SANITIZER_INTERFACE_ATTRIBUTE
573		a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {
574		ATOMIC_IMPL(Exchange, a, v, mo);
	609	a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, int mo) {
	610	return AtomicImpl<OpExchange>(to_morder(mo), a, v);
575	611	}
576		#endif
	612	# endif
577	613
578	614	SANITIZER_INTERFACE_ATTRIBUTE
579		a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {
580		ATOMIC_IMPL(FetchAdd, a, v, mo);
	615	a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, int mo) {
	616	return AtomicImpl<OpFetchAdd>(to_morder(mo), a, v);
581	617	}
582	618
583	619	SANITIZER_INTERFACE_ATTRIBUTE
584		a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {
585		ATOMIC_IMPL(FetchAdd, a, v, mo);
	620	a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, int mo) {
	621	return AtomicImpl<OpFetchAdd>(to_morder(mo), a, v);
586	622	}
587	623
588	624	SANITIZER_INTERFACE_ATTRIBUTE
589		a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {
590		ATOMIC_IMPL(FetchAdd, a, v, mo);
	625	a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, int mo) {
	626	return AtomicImpl<OpFetchAdd>(to_morder(mo), a, v);
591	627	}
592	628
593	629	SANITIZER_INTERFACE_ATTRIBUTE
594		a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {
595		ATOMIC_IMPL(FetchAdd, a, v, mo);
	630	a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, int mo) {
	631	return AtomicImpl<OpFetchAdd>(to_morder(mo), a, v);
596	632	}
597	633
598		#if __TSAN_HAS_INT128
	634	# if __TSAN_HAS_INT128
599	635	SANITIZER_INTERFACE_ATTRIBUTE
600		a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {
601		ATOMIC_IMPL(FetchAdd, a, v, mo);
	636	a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, int mo) {
	637	return AtomicImpl<OpFetchAdd>(to_morder(mo), a, v);
602	638	}
603		#endif
	639	# endif
604	640
605	641	SANITIZER_INTERFACE_ATTRIBUTE
606		a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {
607		ATOMIC_IMPL(FetchSub, a, v, mo);
	642	a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, int mo) {
	643	return AtomicImpl<OpFetchSub>(to_morder(mo), a, v);
608	644	}
609	645
610	646	SANITIZER_INTERFACE_ATTRIBUTE
611		a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {
612		ATOMIC_IMPL(FetchSub, a, v, mo);
	647	a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, int mo) {
	648	return AtomicImpl<OpFetchSub>(to_morder(mo), a, v);
613	649	}
614	650
615	651	SANITIZER_INTERFACE_ATTRIBUTE
616		a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {
617		ATOMIC_IMPL(FetchSub, a, v, mo);
	652	a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, int mo) {
	653	return AtomicImpl<OpFetchSub>(to_morder(mo), a, v);
618	654	}
619	655
620	656	SANITIZER_INTERFACE_ATTRIBUTE
621		a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {
622		ATOMIC_IMPL(FetchSub, a, v, mo);
	657	a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, int mo) {
	658	return AtomicImpl<OpFetchSub>(to_morder(mo), a, v);
623	659	}
624	660
625		#if __TSAN_HAS_INT128
	661	# if __TSAN_HAS_INT128
626	662	SANITIZER_INTERFACE_ATTRIBUTE
627		a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {
628		ATOMIC_IMPL(FetchSub, a, v, mo);
	663	a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, int mo) {
	664	return AtomicImpl<OpFetchSub>(to_morder(mo), a, v);
629	665	}
630		#endif
	666	# endif
631	667
632	668	SANITIZER_INTERFACE_ATTRIBUTE
633		a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {
634		ATOMIC_IMPL(FetchAnd, a, v, mo);
	669	a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, int mo) {
	670	return AtomicImpl<OpFetchAnd>(to_morder(mo), a, v);
635	671	}
636	672
637	673	SANITIZER_INTERFACE_ATTRIBUTE
638		a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {
639		ATOMIC_IMPL(FetchAnd, a, v, mo);
	674	a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, int mo) {
	675	return AtomicImpl<OpFetchAnd>(to_morder(mo), a, v);
640	676	}
641	677
642	678	SANITIZER_INTERFACE_ATTRIBUTE
643		a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {
644		ATOMIC_IMPL(FetchAnd, a, v, mo);
	679	a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, int mo) {
	680	return AtomicImpl<OpFetchAnd>(to_morder(mo), a, v);
645	681	}
646	682
647	683	SANITIZER_INTERFACE_ATTRIBUTE
648		a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {
649		ATOMIC_IMPL(FetchAnd, a, v, mo);
	684	a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, int mo) {
	685	return AtomicImpl<OpFetchAnd>(to_morder(mo), a, v);
650	686	}
651	687
652		#if __TSAN_HAS_INT128
	688	# if __TSAN_HAS_INT128
653	689	SANITIZER_INTERFACE_ATTRIBUTE
654		a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {
655		ATOMIC_IMPL(FetchAnd, a, v, mo);
	690	a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, int mo) {
	691	return AtomicImpl<OpFetchAnd>(to_morder(mo), a, v);
656	692	}
657		#endif
	693	# endif
658	694
659	695	SANITIZER_INTERFACE_ATTRIBUTE
660		a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {
661		ATOMIC_IMPL(FetchOr, a, v, mo);
	696	a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, int mo) {
	697	return AtomicImpl<OpFetchOr>(to_morder(mo), a, v);
662	698	}
663	699
664	700	SANITIZER_INTERFACE_ATTRIBUTE
665		a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {
666		ATOMIC_IMPL(FetchOr, a, v, mo);
	701	a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, int mo) {
	702	return AtomicImpl<OpFetchOr>(to_morder(mo), a, v);
667	703	}
668	704
669	705	SANITIZER_INTERFACE_ATTRIBUTE
670		a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {
671		ATOMIC_IMPL(FetchOr, a, v, mo);
	706	a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, int mo) {
	707	return AtomicImpl<OpFetchOr>(to_morder(mo), a, v);
672	708	}
673	709
674	710	SANITIZER_INTERFACE_ATTRIBUTE
675		a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {
676		ATOMIC_IMPL(FetchOr, a, v, mo);
	711	a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, int mo) {
	712	return AtomicImpl<OpFetchOr>(to_morder(mo), a, v);
677	713	}
678	714
679		#if __TSAN_HAS_INT128
	715	# if __TSAN_HAS_INT128
680	716	SANITIZER_INTERFACE_ATTRIBUTE
681		a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {
682		ATOMIC_IMPL(FetchOr, a, v, mo);
	717	a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, int mo) {
	718	return AtomicImpl<OpFetchOr>(to_morder(mo), a, v);
683	719	}
684		#endif
	720	# endif
685	721
686	722	SANITIZER_INTERFACE_ATTRIBUTE
687		a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {
688		ATOMIC_IMPL(FetchXor, a, v, mo);
	723	a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, int mo) {
	724	return AtomicImpl<OpFetchXor>(to_morder(mo), a, v);
689	725	}
690	726
691	727	SANITIZER_INTERFACE_ATTRIBUTE
692		a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {
693		ATOMIC_IMPL(FetchXor, a, v, mo);
	728	a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, int mo) {
	729	return AtomicImpl<OpFetchXor>(to_morder(mo), a, v);
694	730	}
695	731
696	732	SANITIZER_INTERFACE_ATTRIBUTE
697		a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {
698		ATOMIC_IMPL(FetchXor, a, v, mo);
	733	a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, int mo) {
	734	return AtomicImpl<OpFetchXor>(to_morder(mo), a, v);
699	735	}
700	736
701	737	SANITIZER_INTERFACE_ATTRIBUTE
702		a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {
703		ATOMIC_IMPL(FetchXor, a, v, mo);
	738	a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, int mo) {
	739	return AtomicImpl<OpFetchXor>(to_morder(mo), a, v);
704	740	}
705	741
706		#if __TSAN_HAS_INT128
	742	# if __TSAN_HAS_INT128
707	743	SANITIZER_INTERFACE_ATTRIBUTE
708		a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {
709		ATOMIC_IMPL(FetchXor, a, v, mo);
	744	a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, int mo) {
	745	return AtomicImpl<OpFetchXor>(to_morder(mo), a, v);
710	746	}
711		#endif
	747	# endif
712	748
713	749	SANITIZER_INTERFACE_ATTRIBUTE
714		a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {
715		ATOMIC_IMPL(FetchNand, a, v, mo);
	750	a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, int mo) {
	751	return AtomicImpl<OpFetchNand>(to_morder(mo), a, v);
716	752	}
717	753
718	754	SANITIZER_INTERFACE_ATTRIBUTE
719		a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {
720		ATOMIC_IMPL(FetchNand, a, v, mo);
	755	a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, int mo) {
	756	return AtomicImpl<OpFetchNand>(to_morder(mo), a, v);
721	757	}
722	758
723	759	SANITIZER_INTERFACE_ATTRIBUTE
724		a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {
725		ATOMIC_IMPL(FetchNand, a, v, mo);
	760	a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, int mo) {
	761	return AtomicImpl<OpFetchNand>(to_morder(mo), a, v);
726	762	}
727	763
728	764	SANITIZER_INTERFACE_ATTRIBUTE
729		a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {
730		ATOMIC_IMPL(FetchNand, a, v, mo);
	765	a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, int mo) {
	766	return AtomicImpl<OpFetchNand>(to_morder(mo), a, v);
731	767	}
732	768
733		#if __TSAN_HAS_INT128
	769	# if __TSAN_HAS_INT128
734	770	SANITIZER_INTERFACE_ATTRIBUTE
735		a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {
736		ATOMIC_IMPL(FetchNand, a, v, mo);
	771	a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, int mo) {
	772	return AtomicImpl<OpFetchNand>(to_morder(mo), a, v);
737	773	}
738		#endif
	774	# endif
739	775
740	776	SANITIZER_INTERFACE_ATTRIBUTE
741		int __tsan_atomic8_compare_exchange_strong(volatile a8 a, a8 c, a8 v,
742		morder mo, morder fmo) {
743		ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
	777	int __tsan_atomic8_compare_exchange_strong(volatile a8 a, a8 c, a8 v, int mo,
	778	int fmo) {
	779	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
744	780	}
745	781
746	782	SANITIZER_INTERFACE_ATTRIBUTE
747	783	int __tsan_atomic16_compare_exchange_strong(volatile a16 a, a16 c, a16 v,
748		morder mo, morder fmo) {
749		ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
	784	int mo, int fmo) {
	785	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
750	786	}
751	787
752	788	SANITIZER_INTERFACE_ATTRIBUTE
753	789	int __tsan_atomic32_compare_exchange_strong(volatile a32 a, a32 c, a32 v,
754		morder mo, morder fmo) {
755		ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
	790	int mo, int fmo) {
	791	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
756	792	}
757	793
758	794	SANITIZER_INTERFACE_ATTRIBUTE
759	795	int __tsan_atomic64_compare_exchange_strong(volatile a64 a, a64 c, a64 v,
760		morder mo, morder fmo) {
761		ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
	796	int mo, int fmo) {
	797	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
762	798	}
763	799
764		#if __TSAN_HAS_INT128
	800	# if __TSAN_HAS_INT128
765	801	SANITIZER_INTERFACE_ATTRIBUTE
766	802	int __tsan_atomic128_compare_exchange_strong(volatile a128 a, a128 c, a128 v,
767		morder mo, morder fmo) {
768		ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
	803	int mo, int fmo) {
	804	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
769	805	}
770		#endif
	806	# endif
771	807
772	808	SANITIZER_INTERFACE_ATTRIBUTE
773		int __tsan_atomic8_compare_exchange_weak(volatile a8 a, a8 c, a8 v,
774		morder mo, morder fmo) {
775		ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
	809	int __tsan_atomic8_compare_exchange_weak(volatile a8 a, a8 c, a8 v, int mo,
	810	int fmo) {
	811	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
776	812	}
777	813
778	814	SANITIZER_INTERFACE_ATTRIBUTE
779	815	int __tsan_atomic16_compare_exchange_weak(volatile a16 a, a16 c, a16 v,
780		morder mo, morder fmo) {
781		ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
	816	int mo, int fmo) {
	817	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
782	818	}
783	819
784	820	SANITIZER_INTERFACE_ATTRIBUTE
785	821	int __tsan_atomic32_compare_exchange_weak(volatile a32 a, a32 c, a32 v,
786		morder mo, morder fmo) {
787		ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
	822	int mo, int fmo) {
	823	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
788	824	}
789	825
790	826	SANITIZER_INTERFACE_ATTRIBUTE
791	827	int __tsan_atomic64_compare_exchange_weak(volatile a64 a, a64 c, a64 v,
792		morder mo, morder fmo) {
793		ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
	828	int mo, int fmo) {
	829	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
794	830	}
795	831
796		#if __TSAN_HAS_INT128
	832	# if __TSAN_HAS_INT128
797	833	SANITIZER_INTERFACE_ATTRIBUTE
798	834	int __tsan_atomic128_compare_exchange_weak(volatile a128 a, a128 c, a128 v,
799		morder mo, morder fmo) {
800		ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
	835	int mo, int fmo) {
	836	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
801	837	}
802		#endif
	838	# endif
803	839
804	840	SANITIZER_INTERFACE_ATTRIBUTE
805		a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,
806		morder mo, morder fmo) {
807		ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
	841	a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, int mo,
	842	int fmo) {
	843	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
808	844	}
809	845
810	846	SANITIZER_INTERFACE_ATTRIBUTE
811		a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
812		morder mo, morder fmo) {
813		ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
	847	a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, int mo,
	848	int fmo) {
	849	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
814	850	}
815	851
816	852	SANITIZER_INTERFACE_ATTRIBUTE
817		a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
818		morder mo, morder fmo) {
819		ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
	853	a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, int mo,
	854	int fmo) {
	855	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
820	856	}
821	857
822	858	SANITIZER_INTERFACE_ATTRIBUTE
823		a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
824		morder mo, morder fmo) {
825		ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
	859	a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, int mo,
	860	int fmo) {
	861	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
826	862	}
827	863
828		#if __TSAN_HAS_INT128
	864	# if __TSAN_HAS_INT128
829	865	SANITIZER_INTERFACE_ATTRIBUTE
830	866	a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
831		morder mo, morder fmo) {
832		ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
	867	int mo, int fmo) {
	868	return AtomicImpl<OpCAS>(to_morder(mo), to_morder(fmo), a, c, v);
833	869	}
834		#endif
	870	# endif
835	871
836	872	SANITIZER_INTERFACE_ATTRIBUTE
837		void __tsan_atomic_thread_fence(morder mo) { ATOMIC_IMPL(Fence, mo); }
	873	void __tsan_atomic_thread_fence(int mo) {
	874	return AtomicImpl<OpFence>(to_morder(mo));
	875	}
838	876
839	877	SANITIZER_INTERFACE_ATTRIBUTE
840		void __tsan_atomic_signal_fence(morder mo) {
841		}
	878	void __tsan_atomic_signal_fence(int mo) {}
842	879	} // extern "C"
843	880
844	881	#else // #if !SANITIZER_GO
845	882
846	883	// Go
847	884
848		# define ATOMIC(func, ...) \
849		if (thr->ignore_sync) { \
850		NoTsanAtomic##func(__VA_ARGS__); \
851		} else { \
852		FuncEntry(thr, cpc); \
853		Atomic##func(thr, pc, __VA_ARGS__); \
854		FuncExit(thr); \
855		}
	885	template <class Op, class... Types>
	886	void AtomicGo(ThreadState *thr, uptr cpc, uptr pc, Types... args) {
	887	if (thr->ignore_sync) {
	888	(void)Op::NoTsanAtomic(args...);
	889	} else {
	890	FuncEntry(thr, cpc);
	891	(void)Op::Atomic(thr, pc, args...);
	892	FuncExit(thr);
	893	}
	894	}
856	895
857		# define ATOMIC_RET(func, ret, ...) \
858		if (thr->ignore_sync) { \
859		(ret) = NoTsanAtomic##func(__VA_ARGS__); \
860		} else { \
861		FuncEntry(thr, cpc); \
862		(ret) = Atomic##func(thr, pc, __VA_ARGS__); \
863		FuncExit(thr); \
864		}
	896	template <class Op, class... Types>
	897	auto AtomicGoRet(ThreadState *thr, uptr cpc, uptr pc, Types... args) {
	898	if (thr->ignore_sync) {
	899	return Op::NoTsanAtomic(args...);
	900	} else {
	901	FuncEntry(thr, cpc);
	902	auto ret = Op::Atomic(thr, pc, args...);
	903	FuncExit(thr);
	904	return ret;
	905	}
	906	}
865	907
866	908	extern "C" {
867	909	SANITIZER_INTERFACE_ATTRIBUTE
868	910	void __tsan_go_atomic32_load(ThreadState thr, uptr cpc, uptr pc, u8 a) {
869		ATOMIC_RET(Load, (a32)(a+8), (a32*)a, mo_acquire);
	911	(a32 )(a + 8) = AtomicGoRet<OpLoad>(thr, cpc, pc, mo_acquire, (a32 *)a);
870	912	}
871	913
872	914	SANITIZER_INTERFACE_ATTRIBUTE
873	915	void __tsan_go_atomic64_load(ThreadState thr, uptr cpc, uptr pc, u8 a) {
874		ATOMIC_RET(Load, (a64)(a+8), (a64*)a, mo_acquire);
	916	(a64 )(a + 8) = AtomicGoRet<OpLoad>(thr, cpc, pc, mo_acquire, (a64 *)a);
875	917	}
876	918
877	919	SANITIZER_INTERFACE_ATTRIBUTE
878	920	void __tsan_go_atomic32_store(ThreadState thr, uptr cpc, uptr pc, u8 a) {
879		ATOMIC(Store, (a32)a, (a32*)(a+8), mo_release);
	921	AtomicGo<OpStore>(thr, cpc, pc, mo_release, (a32 )a, (a32 *)(a + 8));
880	922	}
881	923
882	924	SANITIZER_INTERFACE_ATTRIBUTE
883	925	void __tsan_go_atomic64_store(ThreadState thr, uptr cpc, uptr pc, u8 a) {
884		ATOMIC(Store, (a64)a, (a64*)(a+8), mo_release);
	926	AtomicGo<OpStore>(thr, cpc, pc, mo_release, (a64 )a, (a64 *)(a + 8));
885	927	}
886	928
887	929	SANITIZER_INTERFACE_ATTRIBUTE
888	930	void __tsan_go_atomic32_fetch_add(ThreadState thr, uptr cpc, uptr pc, u8 a) {
889		ATOMIC_RET(FetchAdd, (a32)(a+16), (a32)a, (a32*)(a+8), mo_acq_rel);
	931	(a32 )(a + 16) = AtomicGoRet<OpFetchAdd>(thr, cpc, pc, mo_acq_rel,
	932	(a32 )a, (a32 *)(a + 8));
890	933	}
891	934
892	935	SANITIZER_INTERFACE_ATTRIBUTE
893	936	void __tsan_go_atomic64_fetch_add(ThreadState thr, uptr cpc, uptr pc, u8 a) {
894		ATOMIC_RET(FetchAdd, (a64)(a+16), (a64)a, (a64*)(a+8), mo_acq_rel);
	937	(a64 )(a + 16) = AtomicGoRet<OpFetchAdd>(thr, cpc, pc, mo_acq_rel,
	938	(a64 )a, (a64 *)(a + 8));
895	939	}
896	940
897	941	SANITIZER_INTERFACE_ATTRIBUTE
898	942	void __tsan_go_atomic32_fetch_and(ThreadState thr, uptr cpc, uptr pc, u8 a) {
899		ATOMIC_RET(FetchAnd, (a32 )(a + 16), (a32 )a, (a32 *)(a + 8),
900		mo_acq_rel);
	943	(a32 )(a + 16) = AtomicGoRet<OpFetchAnd>(thr, cpc, pc, mo_acq_rel,
	944	(a32 )a, (a32 *)(a + 8));
901	945	}
902	946
903	947	SANITIZER_INTERFACE_ATTRIBUTE
904	948	void __tsan_go_atomic64_fetch_and(ThreadState thr, uptr cpc, uptr pc, u8 a) {
905		ATOMIC_RET(FetchAnd, (a64 )(a + 16), (a64 )a, (a64 *)(a + 8),
906		mo_acq_rel);
	949	(a64 )(a + 16) = AtomicGoRet<OpFetchAnd>(thr, cpc, pc, mo_acq_rel,
	950	(a64 )a, (a64 *)(a + 8));
907	951	}
908	952
909	953	SANITIZER_INTERFACE_ATTRIBUTE
910	954	void __tsan_go_atomic32_fetch_or(ThreadState thr, uptr cpc, uptr pc, u8 a) {
911		ATOMIC_RET(FetchOr, (a32 )(a + 16), (a32 )a, (a32 *)(a + 8),
912		mo_acq_rel);
	955	(a32 )(a + 16) = AtomicGoRet<OpFetchOr>(thr, cpc, pc, mo_acq_rel,
	956	(a32 )a, (a32 *)(a + 8));
913	957	}
914	958
915	959	SANITIZER_INTERFACE_ATTRIBUTE
916	960	void __tsan_go_atomic64_fetch_or(ThreadState thr, uptr cpc, uptr pc, u8 a) {
917		ATOMIC_RET(FetchOr, (a64 )(a + 16), (a64 )a, (a64 *)(a + 8),
918		mo_acq_rel);
	961	(a64 )(a + 16) = AtomicGoRet<OpFetchOr>(thr, cpc, pc, mo_acq_rel,
	962	(a64 )a, (a64 *)(a + 8));
919	963	}
920	964
921	965	SANITIZER_INTERFACE_ATTRIBUTE
922	966	void __tsan_go_atomic32_exchange(ThreadState thr, uptr cpc, uptr pc, u8 a) {
923		ATOMIC_RET(Exchange, (a32)(a+16), (a32)a, (a32*)(a+8), mo_acq_rel);
	967	(a32 )(a + 16) = AtomicGoRet<OpExchange>(thr, cpc, pc, mo_acq_rel,
	968	(a32 )a, (a32 *)(a + 8));
924	969	}
925	970
926	971	SANITIZER_INTERFACE_ATTRIBUTE
927	972	void __tsan_go_atomic64_exchange(ThreadState thr, uptr cpc, uptr pc, u8 a) {
928		ATOMIC_RET(Exchange, (a64)(a+16), (a64)a, (a64*)(a+8), mo_acq_rel);
	973	(a64 )(a + 16) = AtomicGoRet<OpExchange>(thr, cpc, pc, mo_acq_rel,
	974	(a64 )a, (a64 *)(a + 8));
929	975	}
930	976
931	977	SANITIZER_INTERFACE_ATTRIBUTE
932		void __tsan_go_atomic32_compare_exchange(
933		ThreadState thr, uptr cpc, uptr pc, u8 a) {
934		a32 cur = 0;
935		a32 cmp = (a32)(a+8);
936		ATOMIC_RET(CAS, cur, (a32)a, cmp, (a32*)(a+12), mo_acq_rel, mo_acquire);
937		(bool)(a+16) = (cur == cmp);
	978	void __tsan_go_atomic32_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
	979	u8 *a) {
	980	a32 cmp = (a32 )(a + 8);
	981	a32 cur = AtomicGoRet<OpCAS>(thr, cpc, pc, mo_acq_rel, mo_acquire, (a32 *)a,
	982	cmp, (a32 )(a + 12));
	983	(bool )(a + 16) = (cur == cmp);
938	984	}
939	985
940	986	SANITIZER_INTERFACE_ATTRIBUTE
941		void __tsan_go_atomic64_compare_exchange(
942		ThreadState thr, uptr cpc, uptr pc, u8 a) {
943		a64 cur = 0;
944		a64 cmp = (a64)(a+8);
945		ATOMIC_RET(CAS, cur, (a64)a, cmp, (a64*)(a+16), mo_acq_rel, mo_acquire);
946		(bool)(a+24) = (cur == cmp);
	987	void __tsan_go_atomic64_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
	988	u8 *a) {
	989	a64 cmp = (a64 )(a + 8);
	990	a64 cur = AtomicGoRet<OpCAS>(thr, cpc, pc, mo_acq_rel, mo_acquire, (a64 *)a,
	991	cmp, (a64 )(a + 16));
	992	(bool )(a + 24) = (cur == cmp);
947	993	}
948	994	} // extern "C"
949	995	#endif // #if !SANITIZER_GO

lib/tsan/tsan_mman.cpp+1-1

...	...	@@ -252,7 +252,7 @@ void user_reallocarray(ThreadState thr, uptr pc, void *p, uptr size, uptr n) {
252	252	if (AllocatorMayReturnNull())
253	253	return SetErrnoOnNull(nullptr);
254	254	GET_STACK_TRACE_FATAL(thr, pc);
255		ReportReallocArrayOverflow(size, n, &stack);
	255	ReportReallocArrayOverflow(n, size, &stack);
256	256	}
257	257	return user_realloc(thr, pc, p, size * n);
258	258	}

lib/tsan/tsan_platform_linux.cpp-1

...	...	@@ -418,7 +418,6 @@ void InitializePlatform() {
418	418	Die();
419	419	}
420	420
421		InitTlsSize();
422	421	#endif // !SANITIZER_GO
423	422	}
424	423

lib/tsan/tsan_rtl.cpp+4-1

...	...	@@ -673,7 +673,8 @@ void CheckUnwind() {
673	673	thr->ignore_reads_and_writes++;
674	674	atomic_store_relaxed(&thr->in_signal_handler, 0);
675	675	#endif
676		PrintCurrentStackSlow(StackTrace::GetCurrentPc());
	676	PrintCurrentStack(StackTrace::GetCurrentPc(),
	677	common_flags()->fast_unwind_on_fatal);
677	678	}
678	679
679	680	bool is_initialized;
...	...	@@ -806,6 +807,7 @@ int Finalize(ThreadState *thr) {
806	807
807	808	#if !SANITIZER_GO
808	809	void ForkBefore(ThreadState* thr, uptr pc) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	810	VReport(2, "BeforeFork tid: %llu\n", GetTid());
809	811	GlobalProcessorLock();
810	812	// Detaching from the slot makes OnUserFree skip writing to the shadow.
811	813	// The slot will be locked so any attempts to use it will deadlock anyway.
...	...	@@ -847,6 +849,7 @@ static void ForkAfter(ThreadState* thr,
847	849	SlotAttachAndLock(thr);
848	850	SlotUnlock(thr);
849	851	GlobalProcessorUnlock();
	852	VReport(2, "AfterFork tid: %llu\n", GetTid());
850	853	}
851	854
852	855	void ForkParentAfter(ThreadState* thr, uptr pc) { ForkAfter(thr, false); }

lib/tsan/tsan_rtl.h+1-1

...	...	@@ -514,7 +514,7 @@ bool IsExpectedReport(uptr addr, uptr size);
514	514	StackID CurrentStackId(ThreadState *thr, uptr pc);
515	515	ReportStack *SymbolizeStackId(StackID stack_id);
516	516	void PrintCurrentStack(ThreadState *thr, uptr pc);
517		void PrintCurrentStackSlow(uptr pc); // uses libunwind
	517	void PrintCurrentStack(uptr pc, bool fast); // may uses libunwind
518	518	MBlock JavaHeapBlock(uptr addr, uptr start);
519	519
520	520	void Initialize(ThreadState *thr);

lib/tsan/tsan_rtl_report.cpp+5-5

...	...	@@ -828,18 +828,18 @@ void PrintCurrentStack(ThreadState *thr, uptr pc) {
828	828	PrintStack(SymbolizeStack(trace));
829	829	}
830	830
831		// Always inlining PrintCurrentStackSlow, because LocatePcInTrace assumes
	831	// Always inlining PrintCurrentStack, because LocatePcInTrace assumes
832	832	// __sanitizer_print_stack_trace exists in the actual unwinded stack, but
833		// tail-call to PrintCurrentStackSlow breaks this assumption because
	833	// tail-call to PrintCurrentStack breaks this assumption because
834	834	// __sanitizer_print_stack_trace disappears after tail-call.
835	835	// However, this solution is not reliable enough, please see dvyukov's comment
836	836	// http://reviews.llvm.org/D19148#406208
837	837	// Also see PR27280 comment 2 and 3 for breaking examples and analysis.
838		ALWAYS_INLINE USED void PrintCurrentStackSlow(uptr pc) {
	838	ALWAYS_INLINE USED void PrintCurrentStack(uptr pc, bool fast) {
839	839	#if !SANITIZER_GO
840	840	uptr bp = GET_CURRENT_FRAME();
841	841	auto *ptrace = New<BufferedStackTrace>();
842		ptrace->Unwind(pc, bp, nullptr, false);
	842	ptrace->Unwind(pc, bp, nullptr, fast);
843	843
844	844	for (uptr i = 0; i < ptrace->size / 2; i++) {
845	845	uptr tmp = ptrace->trace_buffer[i];
...	...	@@ -857,6 +857,6 @@ using namespace __tsan;
857	857	extern "C" {
858	858	SANITIZER_INTERFACE_ATTRIBUTE
859	859	void __sanitizer_print_stack_trace() {
860		PrintCurrentStackSlow(StackTrace::GetCurrentPc());
	860	PrintCurrentStack(StackTrace::GetCurrentPc(), false);
861	861	}
862	862	} // extern "C"

lib/tsan/tsan_rtl_thread.cpp+6-4

...	...	@@ -165,14 +165,16 @@ void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
165	165	#endif
166	166
167	167	uptr stk_addr = 0;
168		uptr stk_size = 0;
	168	uptr stk_end = 0;
169	169	uptr tls_addr = 0;
170		uptr tls_size = 0;
	170	uptr tls_end = 0;
171	171	#if !SANITIZER_GO
172	172	if (thread_type != ThreadType::Fiber)
173		GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_size, &tls_addr,
174		&tls_size);
	173	GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_end, &tls_addr,
	174	&tls_end);
175	175	#endif
	176	uptr stk_size = stk_end - stk_addr;
	177	uptr tls_size = tls_end - tls_addr;
176	178	thr->stk_addr = stk_addr;
177	179	thr->stk_size = stk_size;
178	180	thr->tls_addr = tls_addr;

lib/tsan/tsan_spinlock_defs_mac.h deleted-45

...	...	@@ -1,45 +0,0 @@
1		//===-- tsan_spinlock_defs_mac.h -------------------------------===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// This file is a part of ThreadSanitizer (TSan), a race detector.
10		//
11		// Mac-specific forward-declared function defintions that may be
12		// deprecated in later versions of the OS.
13		// These are needed for interceptors.
14		//
15		//===----------------------------------------------------------------------===//
16
17		#if SANITIZER_APPLE
18
19		#ifndef TSAN_SPINLOCK_DEFS_MAC_H
20		#define TSAN_SPINLOCK_DEFS_MAC_H
21
22		#include <stdint.h>
23
24		extern "C" {
25
26		/*
27		Provides forward declarations related to OSSpinLocks on Darwin. These functions are
28		deprecated on macOS version 10.12 and later,
29		and are no longer included in the system headers.
30
31		However, the symbols are still available on the system, so we provide these forward
32		declarations to prevent compilation errors in tsan_interceptors_mac.cpp, which
33		references these functions when defining TSAN interceptor functions.
34		*/
35
36		typedef int32_t OSSpinLock;
37
38		void OSSpinLockLock(volatile OSSpinLock *__lock);
39		void OSSpinLockUnlock(volatile OSSpinLock *__lock);
40		bool OSSpinLockTry(volatile OSSpinLock *__lock);
41
42		}
43
44		#endif //TSAN_SPINLOCK_DEFS_MAC_H
45		#endif // SANITIZER_APPLE

src/libtsan.zig+2-6

...	...	@@ -410,9 +410,6 @@ const sanitizer_common_sources = [_][]const u8{
410	410	"sanitizer_allocator.cpp",
411	411	"sanitizer_chained_origin_depot.cpp",
412	412	"sanitizer_common.cpp",
413		"sanitizer_coverage_win_dll_thunk.cpp",
414		"sanitizer_coverage_win_dynamic_runtime_thunk.cpp",
415		"sanitizer_coverage_win_weak_interception.cpp",
416	413	"sanitizer_deadlock_detector1.cpp",
417	414	"sanitizer_deadlock_detector2.cpp",
418	415	"sanitizer_errno.cpp",
...	...	@@ -452,9 +449,7 @@ const sanitizer_common_sources = [_][]const u8{
452	449	"sanitizer_tls_get_addr.cpp",
453	450	"sanitizer_type_traits.cpp",
454	451	"sanitizer_win.cpp",
455		"sanitizer_win_dll_thunk.cpp",
456		"sanitizer_win_dynamic_runtime_thunk.cpp",
457		"sanitizer_win_weak_interception.cpp",
	452	"sanitizer_win_interception.cpp",
458	453	};
459	454
460	455	const sanitizer_nolibc_sources = [_][]const u8{
...	...	@@ -490,6 +485,7 @@ const sanitizer_symbolizer_sources = [_][]const u8{
490	485	"sanitizer_symbolizer_report.cpp",
491	486	"sanitizer_symbolizer_report_fuchsia.cpp",
492	487	"sanitizer_symbolizer_win.cpp",
	488	"sanitizer_thread_history.cpp",
493	489	"sanitizer_unwind_linux_libcdep.cpp",
494	490	"sanitizer_unwind_fuchsia.cpp",
495	491	"sanitizer_unwind_win.cpp",

author	Alex Rønne Petersen <alex@alexrp.com> 2025-02-05 13:36:56+01:00
committer	Alex Rønne Petersen <alex@alexrp.com> 2025-04-04 06:08:09+02:00
log	e99c11856df5dafb54e92af12c0858d187dc2ad2
tree	47602df74eafa2c0d5e58d23ce7b4aef9756832a
parent	d67d52abe57b13517013e81562d64bb7c2f77775
signature	Signed by SSH key SHA256:7B/LJ7bpR1eX8aCXSr4mtd5M45VMPKcx9zY8e95b5QM

libtsan: Update to LLVM 20.