Commit 854b88fda0dc6e92399c27f819bb2a0f5c090eb9 | zig

author	Andrew Kelley <andrew@ziglang.org> 2024-01-10 00:59:52-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2024-01-10 01:00:37-07:00
log	854b88fda0dc6e92399c27f819bb2a0f5c090eb9
tree	519ea4302d30255d6ca79337cc326261f4058a14
parent	036e9fd479219ed1116fed8c0da89fa5a2829cf9

tsan: update rtl files to LLVM 17.0.6

214 files changed, 15752 insertions(+), 9952 deletions(-)

lib
- tsan

lib/tsan/interception/interception.h+145-71

...	...	@@ -14,9 +14,10 @@
14	14	#ifndef INTERCEPTION_H
15	15	#define INTERCEPTION_H
16	16
	17	#include "sanitizer_common/sanitizer_asm.h"
17	18	#include "sanitizer_common/sanitizer_internal_defs.h"
18	19
19		#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_MAC && \
	20	#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_APPLE && \
20	21	!SANITIZER_NETBSD && !SANITIZER_WINDOWS && !SANITIZER_FUCHSIA && \
21	22	!SANITIZER_SOLARIS
22	23	# error "Interception doesn't work on this operating system."
...	...	@@ -67,28 +68,54 @@ typedef __sanitizer::OFF64_T OFF64_T;
67	68	// for more details). To intercept such functions you need to use the
68	69	// INTERCEPTOR_WITH_SUFFIX(...) macro.
69	70
70		// How it works:
71		// To replace system functions on Linux we just need to declare functions
72		// with same names in our library and then obtain the real function pointers
	71	// How it works on Linux
	72	// ---------------------
	73	//
	74	// To replace system functions on Linux we just need to declare functions with
	75	// the same names in our library and then obtain the real function pointers
73	76	// using dlsym().
74		// There is one complication. A user may also intercept some of the functions
75		// we intercept. To resolve this we declare our interceptors with __interceptor_
76		// prefix, and then make actual interceptors weak aliases to __interceptor_
77		// functions.
78	77	//
79		// This is not so on Mac OS, where the two-level namespace makes
80		// our replacement functions invisible to other libraries. This may be overcomed
81		// using the DYLD_FORCE_FLAT_NAMESPACE, but some errors loading the shared
82		// libraries in Chromium were noticed when doing so.
	78	// There is one complication: a user may also intercept some of the functions we
	79	// intercept. To allow for up to 3 interceptors (including ours) of a given
	80	// function "func", the interceptor implementation is in ___interceptor_func,
	81	// which is aliased by a weak function __interceptor_func, which in turn is
	82	// aliased (via a trampoline) by weak wrapper function "func".
	83	//
	84	// Most user interceptors should define a foreign interceptor as follows:
	85	//
	86	// - provide a non-weak function "func" that performs interception;
	87	// - if __interceptor_func exists, call it to perform the real functionality;
	88	// - if it does not exist, figure out the real function and call it instead.
	89	//
	90	// In rare cases, a foreign interceptor (of another dynamic analysis runtime)
	91	// may be defined as follows (on supported architectures):
	92	//
	93	// - provide a non-weak function __interceptor_func that performs interception;
	94	// - if ___interceptor_func exists, call it to perform the real functionality;
	95	// - if it does not exist, figure out the real function and call it instead;
	96	// - provide a weak function "func" that is an alias to __interceptor_func.
	97	//
	98	// With this protocol, sanitizer interceptors, foreign user interceptors, and
	99	// foreign interceptors of other dynamic analysis runtimes, or any combination
	100	// thereof, may co-exist simultaneously.
	101	//
	102	// How it works on Mac OS
	103	// ----------------------
	104	//
	105	// This is not so on Mac OS, where the two-level namespace makes our replacement
	106	// functions invisible to other libraries. This may be overcomed using the
	107	// DYLD_FORCE_FLAT_NAMESPACE, but some errors loading the shared libraries in
	108	// Chromium were noticed when doing so.
	109	//
83	110	// Instead we create a dylib containing a __DATA,__interpose section that
84	111	// associates library functions with their wrappers. When this dylib is
85		// preloaded before an executable using DYLD_INSERT_LIBRARIES, it routes all
86		// the calls to interposed functions done through stubs to the wrapper
87		// functions.
	112	// preloaded before an executable using DYLD_INSERT_LIBRARIES, it routes all the
	113	// calls to interposed functions done through stubs to the wrapper functions.
	114	//
88	115	// As it's decided at compile time which functions are to be intercepted on Mac,
89	116	// INTERCEPT_FUNCTION() is effectively a no-op on this system.
90	117
91		#if SANITIZER_MAC
	118	#if SANITIZER_APPLE
92	119	#include <sys/cdefs.h> // For __DARWIN_ALIAS_C().
93	120
94	121	// Just a pair of pointers.
...	...	@@ -100,53 +127,102 @@ struct interpose_substitution {
100	127	// For a function foo() create a global pair of pointers { wrap_foo, foo } in
101	128	// the __DATA,__interpose section.
102	129	// As a result all the calls to foo() will be routed to wrap_foo() at runtime.
103		#define INTERPOSER(func_name) __attribute__((used)) \
	130	#define INTERPOSER(func_name) __attribute__((used)) \
104	131	const interpose_substitution substitution_##func_name[] \
105	132	__attribute__((section("__DATA, __interpose"))) = { \
106		{ reinterpret_cast<const uptr>(WRAP(func_name)), \
107		reinterpret_cast<const uptr>(func_name) } \
	133	{ reinterpret_cast<const uptr>(WRAP(func_name)), \
	134	reinterpret_cast<const uptr>(func_name) } \
108	135	}
109	136
110	137	// For a function foo() and a wrapper function bar() create a global pair
111	138	// of pointers { bar, foo } in the __DATA,__interpose section.
112	139	// As a result all the calls to foo() will be routed to bar() at runtime.
113	140	#define INTERPOSER_2(func_name, wrapper_name) __attribute__((used)) \
114		const interpose_substitution substitution_##func_name[] \
115		__attribute__((section("__DATA, __interpose"))) = { \
116		{ reinterpret_cast<const uptr>(wrapper_name), \
117		reinterpret_cast<const uptr>(func_name) } \
	141	const interpose_substitution substitution_##func_name[] \
	142	__attribute__((section("__DATA, __interpose"))) = { \
	143	{ reinterpret_cast<const uptr>(wrapper_name), \
	144	reinterpret_cast<const uptr>(func_name) } \
118	145	}
119	146
120	147	# define WRAP(x) wrap_##x
121		# define WRAPPER_NAME(x) "wrap_"#x
	148	# define TRAMPOLINE(x) WRAP(x)
122	149	# define INTERCEPTOR_ATTRIBUTE
123	150	# define DECLARE_WRAPPER(ret_type, func, ...)
124	151
125	152	#elif SANITIZER_WINDOWS
126	153	# define WRAP(x) __asan_wrap_##x
127		# define WRAPPER_NAME(x) "__asan_wrap_"#x
	154	# define TRAMPOLINE(x) WRAP(x)
128	155	# define INTERCEPTOR_ATTRIBUTE __declspec(dllexport)
129		# define DECLARE_WRAPPER(ret_type, func, ...) \
	156	# define DECLARE_WRAPPER(ret_type, func, ...) \
130	157	extern "C" ret_type func(__VA_ARGS__);
131		# define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \
	158	# define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \
132	159	extern "C" __declspec(dllimport) ret_type __stdcall func(__VA_ARGS__);
133		#elif SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
134		# define WRAP(x) __interceptor_ ## x
135		# define WRAPPER_NAME(x) "__interceptor_" #x
	160	#elif !SANITIZER_FUCHSIA // LINUX, FREEBSD, NETBSD, SOLARIS
136	161	# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
	162	# if ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT
	163	// Weak aliases of weak aliases do not work, therefore we need to set up a
	164	// trampoline function. The function "func" is a weak alias to the trampoline
	165	// (so that we may check if "func" was overridden), which calls the weak
	166	// function __interceptor_func, which in turn aliases the actual interceptor
	167	// implementation ___interceptor_func:
	168	//
	169	// [wrapper "func": weak] --(alias)--> [TRAMPOLINE(func)]
	170	// \|
	171	// +--------(tail call)-------+
	172	// \|
	173	// v
	174	// [__interceptor_func: weak] --(alias)--> [WRAP(func)]
	175	//
	176	// We use inline assembly to define most of this, because not all compilers
	177	// support functions with the "naked" attribute with every architecture.
	178	# define WRAP(x) ___interceptor_ ## x
	179	# define TRAMPOLINE(x) __interceptor_trampoline_ ## x
	180	# if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
137	181	// FreeBSD's dynamic linker (incompliantly) gives non-weak symbols higher
138	182	// priority than weak ones so weak aliases won't work for indirect calls
139	183	// in position-independent (-fPIC / -fPIE) mode.
140		# define DECLARE_WRAPPER(ret_type, func, ...) \
141		extern "C" ret_type func(__VA_ARGS__) \
142		__attribute__((alias("__interceptor_" #func), visibility("default")));
143		#elif !SANITIZER_FUCHSIA
144		# define WRAP(x) __interceptor_ ## x
145		# define WRAPPER_NAME(x) "__interceptor_" #x
146		# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
147		# define DECLARE_WRAPPER(ret_type, func, ...) \
148		extern "C" ret_type func(__VA_ARGS__) \
149		__attribute__((weak, alias("__interceptor_" #func), visibility("default")));
	184	# define __ASM_WEAK_WRAPPER(func) ".globl " #func "\n"
	185	# else
	186	# define __ASM_WEAK_WRAPPER(func) ".weak " #func "\n"
	187	# endif // SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
	188	// Keep trampoline implementation in sync with sanitizer_common/sanitizer_asm.h
	189	# define DECLARE_WRAPPER(ret_type, func, ...) \
	190	extern "C" ret_type func(__VA_ARGS__); \
	191	extern "C" ret_type TRAMPOLINE(func)(__VA_ARGS__); \
	192	extern "C" ret_type __interceptor_##func(__VA_ARGS__) \
	193	INTERCEPTOR_ATTRIBUTE __attribute__((weak)) ALIAS(WRAP(func)); \
	194	asm( \
	195	".text\n" \
	196	__ASM_WEAK_WRAPPER(func) \
	197	".set " #func ", " SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \
	198	".globl " SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \
	199	".type " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", %function\n" \
	200	SANITIZER_STRINGIFY(TRAMPOLINE(func)) ":\n" \
	201	SANITIZER_STRINGIFY(CFI_STARTPROC) "\n" \
	202	SANITIZER_STRINGIFY(ASM_TAIL_CALL) " __interceptor_" \
	203	SANITIZER_STRINGIFY(ASM_PREEMPTIBLE_SYM(func)) "\n" \
	204	SANITIZER_STRINGIFY(CFI_ENDPROC) "\n" \
	205	".size " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", " \
	206	".-" SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \
	207	);
	208	# else // ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT
	209	// Some architectures cannot implement efficient interceptor trampolines with
	210	// just a plain jump due to complexities of resolving a preemptible symbol. In
	211	// those cases, revert to just this scheme:
	212	//
	213	// [wrapper "func": weak] --(alias)--> [WRAP(func)]
	214	//
	215	# define WRAP(x) __interceptor_ ## x
	216	# define TRAMPOLINE(x) WRAP(x)
	217	# if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
	218	# define __ATTRIBUTE_WEAK_WRAPPER
	219	# else
	220	# define __ATTRIBUTE_WEAK_WRAPPER __attribute__((weak))
	221	# endif // SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
	222	# define DECLARE_WRAPPER(ret_type, func, ...) \
	223	extern "C" ret_type func(__VA_ARGS__) \
	224	INTERCEPTOR_ATTRIBUTE __ATTRIBUTE_WEAK_WRAPPER ALIAS(WRAP(func));
	225	# endif // ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT
150	226	#endif
151	227
152	228	#if SANITIZER_FUCHSIA
...	...	@@ -157,33 +233,35 @@ const interpose_substitution substitution_##func_name[] \
157	233	# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
158	234	# define REAL(x) __unsanitized_##x
159	235	# define DECLARE_REAL(ret_type, func, ...)
160		#elif !SANITIZER_MAC
	236	#elif !SANITIZER_APPLE
161	237	# define PTR_TO_REAL(x) real_##x
162	238	# define REAL(x) __interception::PTR_TO_REAL(x)
163	239	# define FUNC_TYPE(x) x##_type
164	240
165		# define DECLARE_REAL(ret_type, func, ...) \
	241	# define DECLARE_REAL(ret_type, func, ...) \
166	242	typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \
167		namespace __interception { \
168		extern FUNC_TYPE(func) PTR_TO_REAL(func); \
	243	namespace __interception { \
	244	extern FUNC_TYPE(func) PTR_TO_REAL(func); \
169	245	}
170	246	# define ASSIGN_REAL(dst, src) REAL(dst) = REAL(src)
171		#else // SANITIZER_MAC
	247	#else // SANITIZER_APPLE
172	248	# define REAL(x) x
173	249	# define DECLARE_REAL(ret_type, func, ...) \
174	250	extern "C" ret_type func(__VA_ARGS__);
175	251	# define ASSIGN_REAL(x, y)
176		#endif // SANITIZER_MAC
	252	#endif // SANITIZER_APPLE
177	253
178	254	#if !SANITIZER_FUCHSIA
179		# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \
	255	# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \
180	256	DECLARE_REAL(ret_type, func, __VA_ARGS__) \
	257	extern "C" ret_type TRAMPOLINE(func)(__VA_ARGS__); \
181	258	extern "C" ret_type WRAP(func)(__VA_ARGS__);
182	259	// Declare an interceptor and its wrapper defined in a different translation
183	260	// unit (ex. asm).
184		# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) \
185		extern "C" ret_type WRAP(func)(__VA_ARGS__); \
186		extern "C" ret_type func(__VA_ARGS__);
	261	# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) \
	262	extern "C" ret_type TRAMPOLINE(func)(__VA_ARGS__); \
	263	extern "C" ret_type WRAP(func)(__VA_ARGS__); \
	264	extern "C" ret_type func(__VA_ARGS__);
187	265	#else
188	266	# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...)
189	267	# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...)
...	...	@@ -193,7 +271,7 @@ const interpose_substitution substitution_##func_name[] \
193	271	// macros does its job. In exceptional cases you may need to call REAL(foo)
194	272	// without defining INTERCEPTOR(..., foo, ...). For example, if you override
195	273	// foo with an interceptor for other function.
196		#if !SANITIZER_MAC && !SANITIZER_FUCHSIA
	274	#if !SANITIZER_APPLE && !SANITIZER_FUCHSIA
197	275	# define DEFINE_REAL(ret_type, func, ...) \
198	276	typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \
199	277	namespace __interception { \
...	...	@@ -213,25 +291,23 @@ const interpose_substitution substitution_##func_name[] \
213	291	__interceptor_##func(__VA_ARGS__); \
214	292	extern "C" INTERCEPTOR_ATTRIBUTE ret_type func(__VA_ARGS__)
215	293
216		#elif !SANITIZER_MAC
	294	#elif !SANITIZER_APPLE
217	295
218		#define INTERCEPTOR(ret_type, func, ...) \
219		DEFINE_REAL(ret_type, func, __VA_ARGS__) \
220		DECLARE_WRAPPER(ret_type, func, __VA_ARGS__) \
221		extern "C" \
222		INTERCEPTOR_ATTRIBUTE \
223		ret_type WRAP(func)(__VA_ARGS__)
	296	#define INTERCEPTOR(ret_type, func, ...) \
	297	DEFINE_REAL(ret_type, func, __VA_ARGS__) \
	298	DECLARE_WRAPPER(ret_type, func, __VA_ARGS__) \
	299	extern "C" INTERCEPTOR_ATTRIBUTE ret_type WRAP(func)(__VA_ARGS__)
224	300
225	301	// We don't need INTERCEPTOR_WITH_SUFFIX on non-Darwin for now.
226	302	#define INTERCEPTOR_WITH_SUFFIX(ret_type, func, ...) \
227	303	INTERCEPTOR(ret_type, func, __VA_ARGS__)
228	304
229		#else // SANITIZER_MAC
	305	#else // SANITIZER_APPLE
230	306
231		#define INTERCEPTOR_ZZZ(suffix, ret_type, func, ...) \
232		extern "C" ret_type func(__VA_ARGS__) suffix; \
233		extern "C" ret_type WRAP(func)(__VA_ARGS__); \
234		INTERPOSER(func); \
	307	#define INTERCEPTOR_ZZZ(suffix, ret_type, func, ...) \
	308	extern "C" ret_type func(__VA_ARGS__) suffix; \
	309	extern "C" ret_type WRAP(func)(__VA_ARGS__); \
	310	INTERPOSER(func); \
235	311	extern "C" INTERCEPTOR_ATTRIBUTE ret_type WRAP(func)(__VA_ARGS__)
236	312
237	313	#define INTERCEPTOR(ret_type, func, ...) \
...	...	@@ -246,14 +322,12 @@ const interpose_substitution substitution_##func_name[] \
246	322	#endif
247	323
248	324	#if SANITIZER_WINDOWS
249		# define INTERCEPTOR_WINAPI(ret_type, func, ...) \
	325	# define INTERCEPTOR_WINAPI(ret_type, func, ...) \
250	326	typedef ret_type (__stdcall *FUNC_TYPE(func))(__VA_ARGS__); \
251		namespace __interception { \
252		FUNC_TYPE(func) PTR_TO_REAL(func); \
253		} \
254		extern "C" \
255		INTERCEPTOR_ATTRIBUTE \
256		ret_type __stdcall WRAP(func)(__VA_ARGS__)
	327	namespace __interception { \
	328	FUNC_TYPE(func) PTR_TO_REAL(func); \
	329	} \
	330	extern "C" INTERCEPTOR_ATTRIBUTE ret_type __stdcall WRAP(func)(__VA_ARGS__)
257	331	#endif
258	332
259	333	// ISO C++ forbids casting between pointer-to-function and pointer-to-object,
...	...	@@ -278,7 +352,7 @@ typedef unsigned long uptr;
278	352	# define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)
279	353	# define INTERCEPT_FUNCTION_VER(func, symver) \
280	354	INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver)
281		#elif SANITIZER_MAC
	355	#elif SANITIZER_APPLE
282	356	# include "interception_mac.h"
283	357	# define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_MAC(func)
284	358	# define INTERCEPT_FUNCTION_VER(func, symver) \

lib/tsan/interception/interception_linux.cpp+8-8

...	...	@@ -33,7 +33,7 @@ static int StrCmp(const char s1, const char s2) {
33	33	}
34	34	#endif
35	35
36		static void GetFuncAddr(const char name, uptr wrapper_addr) {
	36	static void GetFuncAddr(const char name, uptr trampoline) {
37	37	#if SANITIZER_NETBSD
38	38	// FIXME: Find a better way to handle renames
39	39	if (StrCmp(name, "sigaction"))
...	...	@@ -50,17 +50,17 @@ static void GetFuncAddr(const char name, uptr wrapper_addr) {
50	50
51	51	// In case `name' is not loaded, dlsym ends up finding the actual wrapper.
52	52	// We don't want to intercept the wrapper and have it point to itself.
53		if ((uptr)addr == wrapper_addr)
	53	if ((uptr)addr == trampoline)
54	54	addr = nullptr;
55	55	}
56	56	return addr;
57	57	}
58	58
59	59	bool InterceptFunction(const char name, uptr ptr_to_real, uptr func,
60		uptr wrapper) {
61		void *addr = GetFuncAddr(name, wrapper);
	60	uptr trampoline) {
	61	void *addr = GetFuncAddr(name, trampoline);
62	62	*ptr_to_real = (uptr)addr;
63		return addr && (func == wrapper);
	63	return addr && (func == trampoline);
64	64	}
65	65
66	66	// dlvsym is a GNU extension supported by some other platforms.
...	...	@@ -70,12 +70,12 @@ static void GetFuncAddr(const char name, const char *ver) {
70	70	}
71	71
72	72	bool InterceptFunction(const char name, const char ver, uptr *ptr_to_real,
73		uptr func, uptr wrapper) {
	73	uptr func, uptr trampoline) {
74	74	void *addr = GetFuncAddr(name, ver);
75	75	*ptr_to_real = (uptr)addr;
76		return addr && (func == wrapper);
	76	return addr && (func == trampoline);
77	77	}
78		#endif // SANITIZER_GLIBC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
	78	# endif // SANITIZER_GLIBC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
79	79
80	80	} // namespace __interception
81	81

lib/tsan/interception/interception_linux.h+9-9

...	...	@@ -15,7 +15,7 @@
15	15	SANITIZER_SOLARIS
16	16
17	17	#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)
18		# error "interception_linux.h should be included from interception library only"
	18	# error interception_linux.h should be included from interception library only
19	19	#endif
20	20
21	21	#ifndef INTERCEPTION_LINUX_H
...	...	@@ -23,26 +23,26 @@
23	23
24	24	namespace __interception {
25	25	bool InterceptFunction(const char name, uptr ptr_to_real, uptr func,
26		uptr wrapper);
	26	uptr trampoline);
27	27	bool InterceptFunction(const char name, const char ver, uptr *ptr_to_real,
28		uptr func, uptr wrapper);
	28	uptr func, uptr trampoline);
29	29	} // namespace __interception
30	30
31	31	#define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \
32	32	::__interception::InterceptFunction( \
33	33	#func, \
34		(::__interception::uptr *) & REAL(func), \
35		(::__interception::uptr) & (func), \
36		(::__interception::uptr) & WRAP(func))
	34	(::__interception::uptr *)&REAL(func), \
	35	(::__interception::uptr)&(func), \
	36	(::__interception::uptr)&TRAMPOLINE(func))
37	37
38	38	// dlvsym is a GNU extension supported by some other platforms.
39	39	#if SANITIZER_GLIBC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
40	40	#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \
41	41	::__interception::InterceptFunction( \
42	42	#func, symver, \
43		(::__interception::uptr *) & REAL(func), \
44		(::__interception::uptr) & (func), \
45		(::__interception::uptr) & WRAP(func))
	43	(::__interception::uptr *)&REAL(func), \
	44	(::__interception::uptr)&(func), \
	45	(::__interception::uptr)&TRAMPOLINE(func))
46	46	#else
47	47	#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \
48	48	INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)

lib/tsan/interception/interception_mac.cpp+2-2

...	...	@@ -13,6 +13,6 @@
13	13
14	14	#include "interception.h"
15	15
16		#if SANITIZER_MAC
	16	#if SANITIZER_APPLE
17	17
18		#endif // SANITIZER_MAC
	18	#endif // SANITIZER_APPLE

lib/tsan/interception/interception_mac.h+2-2

...	...	@@ -11,7 +11,7 @@
11	11	// Mac-specific interception methods.
12	12	//===----------------------------------------------------------------------===//
13	13
14		#if SANITIZER_MAC
	14	#if SANITIZER_APPLE
15	15
16	16	#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)
17	17	# error "interception_mac.h should be included from interception.h only"
...	...	@@ -24,4 +24,4 @@
24	24	#define INTERCEPT_FUNCTION_VER_MAC(func, symver)
25	25
26	26	#endif // INTERCEPTION_MAC_H
27		#endif // SANITIZER_MAC
	27	#endif // SANITIZER_APPLE

lib/tsan/interception/interception_type_test.cpp+3-3

...	...	@@ -13,7 +13,7 @@
13	13
14	14	#include "interception.h"
15	15
16		#if SANITIZER_LINUX \|\| SANITIZER_MAC
	16	#if SANITIZER_LINUX \|\| SANITIZER_APPLE
17	17
18	18	#include <sys/types.h>
19	19	#include <stddef.h>
...	...	@@ -24,9 +24,9 @@ COMPILER_CHECK(sizeof(::SSIZE_T) == sizeof(ssize_t));
24	24	COMPILER_CHECK(sizeof(::PTRDIFF_T) == sizeof(ptrdiff_t));
25	25	COMPILER_CHECK(sizeof(::INTMAX_T) == sizeof(intmax_t));
26	26
27		#if !SANITIZER_MAC
	27	# if SANITIZER_GLIBC \|\| SANITIZER_ANDROID
28	28	COMPILER_CHECK(sizeof(::OFF64_T) == sizeof(off64_t));
29		#endif
	29	# endif
30	30
31	31	// The following are the cases when pread (and friends) is used instead of
32	32	// pread64. In those cases we need OFF_T to match off_t. We don't care about the

lib/tsan/interception/interception_win.cpp+107-11

...	...	@@ -56,7 +56,7 @@
56	56	// tramp: jmp QWORD [addr]
57	57	// addr: .bytes <hook>
58	58	//
59		// Note: <real> is equilavent to <label>.
	59	// Note: <real> is equivalent to <label>.
60	60	//
61	61	// 3) HotPatch
62	62	//
...	...	@@ -141,8 +141,29 @@ static const int kBranchLength =
141	141	FIRST_32_SECOND_64(kJumpInstructionLength, kIndirectJumpInstructionLength);
142	142	static const int kDirectBranchLength = kBranchLength + kAddressLength;
143	143
	144	# if defined(_MSC_VER)
	145	# define INTERCEPTION_FORMAT(f, a)
	146	# else
	147	# define INTERCEPTION_FORMAT(f, a) __attribute__((format(printf, f, a)))
	148	# endif
	149
	150	static void (ErrorReportCallback)(const char format, ...)
	151	INTERCEPTION_FORMAT(1, 2);
	152
	153	void SetErrorReportCallback(void (callback)(const char format, ...)) {
	154	ErrorReportCallback = callback;
	155	}
	156
	157	# define ReportError(...) \
	158	do { \
	159	if (ErrorReportCallback) \
	160	ErrorReportCallback(__VA_ARGS__); \
	161	} while (0)
	162
144	163	static void InterceptionFailed() {
145		// Do we have a good way to abort with an error message here?
	164	ReportError("interception_win: failed due to an unrecoverable error.\n");
	165	// This acts like an abort when no debugger is attached. According to an old
	166	// comment, calling abort() leads to an infinite recursion in CheckFailed.
146	167	__debugbreak();
147	168	}
148	169
...	...	@@ -249,8 +270,13 @@ static void WritePadding(uptr from, uptr size) {
249	270	}
250	271
251	272	static void WriteJumpInstruction(uptr from, uptr target) {
252		if (!DistanceIsWithin2Gig(from + kJumpInstructionLength, target))
	273	if (!DistanceIsWithin2Gig(from + kJumpInstructionLength, target)) {
	274	ReportError(
	275	"interception_win: cannot write jmp further than 2GB away, from %p to "
	276	"%p.\n",
	277	(void )from, (void )target);
253	278	InterceptionFailed();
	279	}
254	280	ptrdiff_t offset = target - from - kJumpInstructionLength;
255	281	(u8)from = 0xE9;
256	282	(u32)(from + 1) = offset;
...	...	@@ -274,6 +300,10 @@ static void WriteIndirectJumpInstruction(uptr from, uptr indirect_target) {
274	300	int offset = indirect_target - from - kIndirectJumpInstructionLength;
275	301	if (!DistanceIsWithin2Gig(from + kIndirectJumpInstructionLength,
276	302	indirect_target)) {
	303	ReportError(
	304	"interception_win: cannot write indirect jmp with target further than "
	305	"2GB away, from %p to %p.\n",
	306	(void )from, (void )indirect_target);
277	307	InterceptionFailed();
278	308	}
279	309	(u16)from = 0x25FF;
...	...	@@ -398,8 +428,44 @@ static uptr AllocateMemoryForTrampoline(uptr image_address, size_t size) {
398	428	return allocated_space;
399	429	}
400	430
	431	// The following prologues cannot be patched because of the short jump
	432	// jumping to the patching region.
	433
	434	#if SANITIZER_WINDOWS64
	435	// ntdll!wcslen in Win11
	436	// 488bc1 mov rax,rcx
	437	// 0fb710 movzx edx,word ptr [rax]
	438	// 4883c002 add rax,2
	439	// 6685d2 test dx,dx
	440	// 75f4 jne -12
	441	static const u8 kPrologueWithShortJump1[] = {
	442	0x48, 0x8b, 0xc1, 0x0f, 0xb7, 0x10, 0x48, 0x83,
	443	0xc0, 0x02, 0x66, 0x85, 0xd2, 0x75, 0xf4,
	444	};
	445
	446	// ntdll!strrchr in Win11
	447	// 4c8bc1 mov r8,rcx
	448	// 8a01 mov al,byte ptr [rcx]
	449	// 48ffc1 inc rcx
	450	// 84c0 test al,al
	451	// 75f7 jne -9
	452	static const u8 kPrologueWithShortJump2[] = {
	453	0x4c, 0x8b, 0xc1, 0x8a, 0x01, 0x48, 0xff, 0xc1,
	454	0x84, 0xc0, 0x75, 0xf7,
	455	};
	456	#endif
	457
401	458	// Returns 0 on error.
402	459	static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
	460	#if SANITIZER_WINDOWS64
	461	if (memcmp((u8*)address, kPrologueWithShortJump1,
	462	sizeof(kPrologueWithShortJump1)) == 0 \|\|
	463	memcmp((u8*)address, kPrologueWithShortJump2,
	464	sizeof(kPrologueWithShortJump2)) == 0) {
	465	return 0;
	466	}
	467	#endif
	468
403	469	switch ((u64)address) {
404	470	case 0x90909090909006EB: // stub: jmp over 6 x nop.
405	471	return 8;
...	...	@@ -456,6 +522,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
456	522	case 0xFF8B: // 8B FF : mov edi, edi
457	523	case 0xEC8B: // 8B EC : mov ebp, esp
458	524	case 0xc889: // 89 C8 : mov eax, ecx
	525	case 0xE589: // 89 E5 : mov ebp, esp
459	526	case 0xC18B: // 8B C1 : mov eax, ecx
460	527	case 0xC033: // 33 C0 : xor eax, eax
461	528	case 0xC933: // 33 C9 : xor ecx, ecx
...	...	@@ -477,6 +544,14 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
477	544	case 0xA1: // A1 XX XX XX XX XX XX XX XX :
478	545	// movabs eax, dword ptr ds:[XXXXXXXX]
479	546	return 9;
	547
	548	case 0x83:
	549	const u8 next_byte = (u8)(address + 1);
	550	const u8 mod = next_byte >> 6;
	551	const u8 rm = next_byte & 7;
	552	if (mod == 1 && rm == 4)
	553	return 5; // 83 ModR/M SIB Disp8 Imm8
	554	// add\|or\|adc\|sbb\|and\|sub\|xor\|cmp [r+disp8], imm8
480	555	}
481	556
482	557	switch ((u16)address) {
...	...	@@ -493,6 +568,8 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
493	568	case 0x5641: // push r14
494	569	case 0x5741: // push r15
495	570	case 0x9066: // Two-byte NOP
	571	case 0xc084: // test al, al
	572	case 0x018a: // mov al, byte ptr [rcx]
496	573	return 2;
497	574
498	575	case 0x058B: // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX]
...	...	@@ -509,6 +586,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
509	586	case 0xd12b48: // 48 2b d1 : sub rdx, rcx
510	587	case 0x07c1f6: // f6 c1 07 : test cl, 0x7
511	588	case 0xc98548: // 48 85 C9 : test rcx, rcx
	589	case 0xd28548: // 48 85 d2 : test rdx, rdx
512	590	case 0xc0854d: // 4d 85 c0 : test r8, r8
513	591	case 0xc2b60f: // 0f b6 c2 : movzx eax, dl
514	592	case 0xc03345: // 45 33 c0 : xor r8d, r8d
...	...	@@ -522,6 +600,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
522	600	case 0xca2b48: // 48 2b ca : sub rcx, rdx
523	601	case 0x10b70f: // 0f b7 10 : movzx edx, WORD PTR [rax]
524	602	case 0xc00b4d: // 3d 0b c0 : or r8, r8
	603	case 0xc08b41: // 41 8b c0 : mov eax, r8d
525	604	case 0xd18b48: // 48 8b d1 : mov rdx, rcx
526	605	case 0xdc8b4c: // 4c 8b dc : mov r11, rsp
527	606	case 0xd18b4c: // 4c 8b d1 : mov r10, rcx
...	...	@@ -556,6 +635,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
556	635	case 0x246c8948: // 48 89 6C 24 XX : mov QWORD ptr [rsp + XX], rbp
557	636	case 0x245c8948: // 48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx
558	637	case 0x24748948: // 48 89 74 24 XX : mov QWORD PTR [rsp + XX], rsi
	638	case 0x247c8948: // 48 89 7c 24 XX : mov QWORD PTR [rsp + XX], rdi
559	639	case 0x244C8948: // 48 89 4C 24 XX : mov QWORD PTR [rsp + XX], rcx
560	640	case 0x24548948: // 48 89 54 24 XX : mov QWORD PTR [rsp + XX], rdx
561	641	case 0x244c894c: // 4c 89 4c 24 XX : mov QWORD PTR [rsp + XX], r9
...	...	@@ -592,6 +672,8 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
592	672	case 0x24448B: // 8B 44 24 XX : mov eax, dword ptr [esp + XX]
593	673	case 0x244C8B: // 8B 4C 24 XX : mov ecx, dword ptr [esp + XX]
594	674	case 0x24548B: // 8B 54 24 XX : mov edx, dword ptr [esp + XX]
	675	case 0x245C8B: // 8B 5C 24 XX : mov ebx, dword ptr [esp + XX]
	676	case 0x246C8B: // 8B 6C 24 XX : mov ebp, dword ptr [esp + XX]
595	677	case 0x24748B: // 8B 74 24 XX : mov esi, dword ptr [esp + XX]
596	678	case 0x247C8B: // 8B 7C 24 XX : mov edi, dword ptr [esp + XX]
597	679	return 4;
...	...	@@ -603,12 +685,20 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
603	685	}
604	686	#endif
605	687
606		// Unknown instruction!
607		// FIXME: Unknown instruction failures might happen when we add a new
608		// interceptor or a new compiler version. In either case, they should result
609		// in visible and readable error messages. However, merely calling abort()
610		// leads to an infinite recursion in CheckFailed.
611		InterceptionFailed();
	688	// Unknown instruction! This might happen when we add a new interceptor, use
	689	// a new compiler version, or if Windows changed how some functions are
	690	// compiled. In either case, we print the address and 8 bytes of instructions
	691	// to notify the user about the error and to help identify the unknown
	692	// instruction. Don't treat this as a fatal error, though we can break the
	693	// debugger if one has been attached.
	694	u8 bytes = (u8 )address;
	695	ReportError(
	696	"interception_win: unhandled instruction at %p: %02x %02x %02x %02x %02x "
	697	"%02x %02x %02x\n",
	698	(void *)address, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4],
	699	bytes[5], bytes[6], bytes[7]);
	700	if (::IsDebuggerPresent())
	701	__debugbreak();
612	702	return 0;
613	703	}
614	704
...	...	@@ -629,6 +719,8 @@ static bool CopyInstructions(uptr to, uptr from, size_t size) {
629	719	while (cursor != size) {
630	720	size_t rel_offset = 0;
631	721	size_t instruction_size = GetInstructionSize(from + cursor, &rel_offset);
	722	if (!instruction_size)
	723	return false;
632	724	_memcpy((void)(to + cursor), (void)(from + cursor),
633	725	(size_t)instruction_size);
634	726	if (rel_offset) {
...	...	@@ -689,7 +781,7 @@ bool OverrideFunctionWithRedirectJump(
689	781	return false;
690	782
691	783	if (orig_old_func) {
692		uptr relative_offset = (u32)(old_func + 1);
	784	sptr relative_offset = (s32 )(old_func + 1);
693	785	uptr absolute_target = old_func + relative_offset + kJumpInstructionLength;
694	786	*orig_old_func = absolute_target;
695	787	}
...	...	@@ -846,6 +938,10 @@ static void **InterestingDLLsAvailable() {
846	938	"msvcr120.dll", // VS2013
847	939	"vcruntime140.dll", // VS2015
848	940	"ucrtbase.dll", // Universal CRT
	941	#if (defined(__MINGW32__) && defined(__i386__))
	942	"libc++.dll", // libc++
	943	"libunwind.dll", // libunwind
	944	#endif
849	945	// NTDLL should go last as it exports some functions that we should
850	946	// override in the CRT [presumably only used internally].
851	947	"ntdll.dll", NULL};
...	...	@@ -1019,4 +1115,4 @@ bool OverrideImportedFunction(const char *module_to_patch,
1019	1115
1020	1116	} // namespace __interception
1021	1117
1022		#endif // SANITIZER_MAC
	1118	#endif // SANITIZER_APPLE

lib/tsan/interception/interception_win.h+5

...	...	@@ -41,6 +41,11 @@ bool OverrideImportedFunction(const char *module_to_patch,
41	41	const char *function_name, uptr new_function,
42	42	uptr *orig_old_func);
43	43
	44	// Sets a callback to be used for reporting errors by interception_win. The
	45	// callback will be called with printf-like arguments. Intended to be used with
	46	// __sanitizer::Report. Pass nullptr to disable error reporting (default).
	47	void SetErrorReportCallback(void (callback)(const char format, ...));
	48
44	49	#if !SANITIZER_WINDOWS64
45	50	// Exposed for unittests
46	51	bool OverrideFunctionWithDetour(

lib/tsan/sanitizer_common/sancov_flags.inc+1-1

...	...	@@ -14,7 +14,7 @@
14	14	#endif
15	15
16	16	SANCOV_FLAG(bool, symbolize, true,
17		"If set, converage information will be symbolized by sancov tool "
	17	"If set, coverage information will be symbolized by sancov tool "
18	18	"after dumping.")
19	19
20	20	SANCOV_FLAG(bool, help, false, "Print flags help.")

lib/tsan/sanitizer_common/sanitizer_addrhashmap.h+43-3

...	...	@@ -39,6 +39,11 @@ namespace __sanitizer {
39	39	// the current thread has exclusive access to the data
40	40	// if !h.exists() then the element never existed
41	41	// }
	42	// {
	43	// Map::Handle h(&m, addr, false, true);
	44	// this will create a new element or return a handle to an existing element
	45	// if !h.created() this thread does not have exclusive access to the data
	46	// }
42	47	template<typename T, uptr kSize>
43	48	class AddrHashMap {
44	49	private:
...	...	@@ -56,7 +61,7 @@ class AddrHashMap {
56	61	static const uptr kBucketSize = 3;
57	62
58	63	struct Bucket {
59		RWMutex mtx;
	64	Mutex mtx;
60	65	atomic_uintptr_t add;
61	66	Cell cells[kBucketSize];
62	67	};
...	...	@@ -89,6 +94,12 @@ class AddrHashMap {
89	94	bool create_;
90	95	};
91	96
	97	typedef void (ForEachCallback)(const uptr key, const T &val, void arg);
	98	// ForEach acquires a lock on each bucket while iterating over
	99	// elements. Note that this only ensures that the structure of the hashmap is
	100	// unchanged, there may be a data race to the element itself.
	101	void ForEach(ForEachCallback cb, void *arg);
	102
92	103	private:
93	104	friend class Handle;
94	105	Bucket *table_;
...	...	@@ -98,6 +109,33 @@ class AddrHashMap {
98	109	uptr calcHash(uptr addr);
99	110	};
100	111
	112	template <typename T, uptr kSize>
	113	void AddrHashMap<T, kSize>::ForEach(ForEachCallback cb, void *arg) {
	114	for (uptr n = 0; n < kSize; n++) {
	115	Bucket *bucket = &table_[n];
	116
	117	ReadLock lock(&bucket->mtx);
	118
	119	for (uptr i = 0; i < kBucketSize; i++) {
	120	Cell *c = &bucket->cells[i];
	121	uptr addr1 = atomic_load(&c->addr, memory_order_acquire);
	122	if (addr1 != 0)
	123	cb(addr1, c->val, arg);
	124	}
	125
	126	// Iterate over any additional cells.
	127	if (AddBucket *add =
	128	(AddBucket *)atomic_load(&bucket->add, memory_order_acquire)) {
	129	for (uptr i = 0; i < add->size; i++) {
	130	Cell *c = &add->cells[i];
	131	uptr addr1 = atomic_load(&c->addr, memory_order_acquire);
	132	if (addr1 != 0)
	133	cb(addr1, c->val, arg);
	134	}
	135	}
	136	}
	137	}
	138
101	139	template<typename T, uptr kSize>
102	140	AddrHashMap<T, kSize>::Handle::Handle(AddrHashMap<T, kSize> *map, uptr addr) {
103	141	map_ = map;
...	...	@@ -163,7 +201,8 @@ AddrHashMap<T, kSize>::AddrHashMap() {
163	201	}
164	202
165	203	template <typename T, uptr kSize>
166		void AddrHashMap<T, kSize>::acquire(Handle *h) NO_THREAD_SAFETY_ANALYSIS {
	204	void AddrHashMap<T, kSize>::acquire(Handle *h)
	205	SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
167	206	uptr addr = h->addr_;
168	207	uptr hash = calcHash(addr);
169	208	Bucket *b = &table_[hash];
...	...	@@ -292,7 +331,8 @@ void AddrHashMap<T, kSize>::acquire(Handle *h) NO_THREAD_SAFETY_ANALYSIS {
292	331	}
293	332
294	333	template <typename T, uptr kSize>
295		void AddrHashMap<T, kSize>::release(Handle *h) NO_THREAD_SAFETY_ANALYSIS {
	334	void AddrHashMap<T, kSize>::release(Handle *h)
	335	SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
296	336	if (!h->cell_)
297	337	return;
298	338	Bucket *b = h->bucket_;

lib/tsan/sanitizer_common/sanitizer_allocator.cpp+23-66

...	...	@@ -17,6 +17,7 @@
17	17	#include "sanitizer_allocator_internal.h"
18	18	#include "sanitizer_atomic.h"
19	19	#include "sanitizer_common.h"
	20	#include "sanitizer_platform.h"
20	21
21	22	namespace __sanitizer {
22	23
...	...	@@ -24,66 +25,6 @@ namespace __sanitizer {
24	25	const char *PrimaryAllocatorName = "SizeClassAllocator";
25	26	const char *SecondaryAllocatorName = "LargeMmapAllocator";
26	27
27		// ThreadSanitizer for Go uses libc malloc/free.
28		#if defined(SANITIZER_USE_MALLOC)
29		# if SANITIZER_LINUX && !SANITIZER_ANDROID
30		extern "C" void *__libc_malloc(uptr size);
31		# if !SANITIZER_GO
32		extern "C" void *__libc_memalign(uptr alignment, uptr size);
33		# endif
34		extern "C" void __libc_realloc(void ptr, uptr size);
35		extern "C" void __libc_free(void *ptr);
36		# else
37		# include <stdlib.h>
38		# define __libc_malloc malloc
39		# if !SANITIZER_GO
40		static void *__libc_memalign(uptr alignment, uptr size) {
41		void *p;
42		uptr error = posix_memalign(&p, alignment, size);
43		if (error) return nullptr;
44		return p;
45		}
46		# endif
47		# define __libc_realloc realloc
48		# define __libc_free free
49		# endif
50
51		static void RawInternalAlloc(uptr size, InternalAllocatorCache cache,
52		uptr alignment) {
53		(void)cache;
54		#if !SANITIZER_GO
55		if (alignment == 0)
56		return __libc_malloc(size);
57		else
58		return __libc_memalign(alignment, size);
59		#else
60		// Windows does not provide __libc_memalign/posix_memalign. It provides
61		// __aligned_malloc, but the allocated blocks can't be passed to free,
62		// they need to be passed to __aligned_free. InternalAlloc interface does
63		// not account for such requirement. Alignemnt does not seem to be used
64		// anywhere in runtime, so just call __libc_malloc for now.
65		DCHECK_EQ(alignment, 0);
66		return __libc_malloc(size);
67		#endif
68		}
69
70		static void RawInternalRealloc(void ptr, uptr size,
71		InternalAllocatorCache *cache) {
72		(void)cache;
73		return __libc_realloc(ptr, size);
74		}
75
76		static void RawInternalFree(void ptr, InternalAllocatorCache cache) {
77		(void)cache;
78		__libc_free(ptr);
79		}
80
81		InternalAllocator *internal_allocator() {
82		return 0;
83		}
84
85		#else // SANITIZER_GO \|\| defined(SANITIZER_USE_MALLOC)
86
87	28	static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)];
88	29	static atomic_uint8_t internal_allocator_initialized;
89	30	static StaticSpinMutex internal_alloc_init_mu;
...	...	@@ -135,8 +76,6 @@ static void RawInternalFree(void ptr, InternalAllocatorCache cache) {
135	76	internal_allocator()->Deallocate(cache, ptr);
136	77	}
137	78
138		#endif // SANITIZER_GO \|\| defined(SANITIZER_USE_MALLOC)
139
140	79	static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) {
141	80	SetAllocatorOutOfMemory();
142	81	Report("FATAL: %s: internal allocator is out of memory trying to allocate "
...	...	@@ -187,6 +126,16 @@ void InternalFree(void addr, InternalAllocatorCache cache) {
187	126	RawInternalFree(addr, cache);
188	127	}
189	128
	129	void InternalAllocatorLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	130	internal_allocator_cache_mu.Lock();
	131	internal_allocator()->ForceLock();
	132	}
	133
	134	void InternalAllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	135	internal_allocator()->ForceUnlock();
	136	internal_allocator_cache_mu.Unlock();
	137	}
	138
190	139	// LowLevelAllocator
191	140	constexpr uptr kLowLevelAllocatorDefaultAlignment = 8;
192	141	static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment;
...	...	@@ -197,12 +146,10 @@ void *LowLevelAllocator::Allocate(uptr size) {
197	146	size = RoundUpTo(size, low_level_alloc_min_alignment);
198	147	if (allocated_end_ - allocated_current_ < (sptr)size) {
199	148	uptr size_to_allocate = RoundUpTo(size, GetPageSizeCached());
200		allocated_current_ =
201		(char*)MmapOrDie(size_to_allocate, __func__);
	149	allocated_current_ = (char *)MmapOrDie(size_to_allocate, __func__);
202	150	allocated_end_ = allocated_current_ + size_to_allocate;
203	151	if (low_level_alloc_callback) {
204		low_level_alloc_callback((uptr)allocated_current_,
205		size_to_allocate);
	152	low_level_alloc_callback((uptr)allocated_current_, size_to_allocate);
206	153	}
207	154	}
208	155	CHECK(allocated_end_ - allocated_current_ >= (sptr)size);
...	...	@@ -247,4 +194,14 @@ void PrintHintAllocatorCannotReturnNull() {
247	194	"allocator_may_return_null=1\n");
248	195	}
249	196
	197	static atomic_uint8_t rss_limit_exceeded;
	198
	199	bool IsRssLimitExceeded() {
	200	return atomic_load(&rss_limit_exceeded, memory_order_relaxed);
	201	}
	202
	203	void SetRssLimitExceeded(bool limit_exceeded) {
	204	atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed);
	205	}
	206
250	207	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_allocator.h+11-7

...	...	@@ -14,6 +14,7 @@
14	14	#define SANITIZER_ALLOCATOR_H
15	15
16	16	#include "sanitizer_common.h"
	17	#include "sanitizer_flat_map.h"
17	18	#include "sanitizer_internal_defs.h"
18	19	#include "sanitizer_lfstack.h"
19	20	#include "sanitizer_libc.h"
...	...	@@ -43,12 +44,6 @@ void SetAllocatorOutOfMemory();
43	44
44	45	void PrintHintAllocatorCannotReturnNull();
45	46
46		// Allocators call these callbacks on mmap/munmap.
47		struct NoOpMapUnmapCallback {
48		void OnMap(uptr p, uptr size) const { }
49		void OnUnmap(uptr p, uptr size) const { }
50		};
51
52	47	// Callback type for iterating over chunks.
53	48	typedef void (ForEachChunkCallback)(uptr chunk, void arg);
54	49
...	...	@@ -67,15 +62,24 @@ inline void RandomShuffle(T a, u32 n, u32 rand_state) {
67	62	*rand_state = state;
68	63	}
69	64
	65	struct NoOpMapUnmapCallback {
	66	void OnMap(uptr p, uptr size) const {}
	67	void OnMapSecondary(uptr p, uptr size, uptr user_begin,
	68	uptr user_size) const {}
	69	void OnUnmap(uptr p, uptr size) const {}
	70	};
	71
70	72	#include "sanitizer_allocator_size_class_map.h"
71	73	#include "sanitizer_allocator_stats.h"
72	74	#include "sanitizer_allocator_primary64.h"
73		#include "sanitizer_allocator_bytemap.h"
74	75	#include "sanitizer_allocator_primary32.h"
75	76	#include "sanitizer_allocator_local_cache.h"
76	77	#include "sanitizer_allocator_secondary.h"
77	78	#include "sanitizer_allocator_combined.h"
78	79
	80	bool IsRssLimitExceeded();
	81	void SetRssLimitExceeded(bool limit_exceeded);
	82
79	83	} // namespace __sanitizer
80	84
81	85	#endif // SANITIZER_ALLOCATOR_H

lib/tsan/sanitizer_common/sanitizer_allocator_bytemap.h deleted-107

...	...	@@ -1,107 +0,0 @@
1		//===-- sanitizer_allocator_bytemap.h ---------------------------- C++ --===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// Part of the Sanitizer Allocator.
10		//
11		//===----------------------------------------------------------------------===//
12		#ifndef SANITIZER_ALLOCATOR_H
13		#error This file must be included inside sanitizer_allocator.h
14		#endif
15
16		// Maps integers in rage [0, kSize) to u8 values.
17		template <u64 kSize, typename AddressSpaceViewTy = LocalAddressSpaceView>
18		class FlatByteMap {
19		public:
20		using AddressSpaceView = AddressSpaceViewTy;
21		void Init() {
22		internal_memset(map_, 0, sizeof(map_));
23		}
24
25		void set(uptr idx, u8 val) {
26		CHECK_LT(idx, kSize);
27		CHECK_EQ(0U, map_[idx]);
28		map_[idx] = val;
29		}
30		u8 operator[] (uptr idx) {
31		CHECK_LT(idx, kSize);
32		// FIXME: CHECK may be too expensive here.
33		return map_[idx];
34		}
35		private:
36		u8 map_[kSize];
37		};
38
39		// TwoLevelByteMap maps integers in range [0, kSize1*kSize2) to u8 values.
40		// It is implemented as a two-dimensional array: array of kSize1 pointers
41		// to kSize2-byte arrays. The secondary arrays are mmaped on demand.
42		// Each value is initially zero and can be set to something else only once.
43		// Setting and getting values from multiple threads is safe w/o extra locking.
44		template <u64 kSize1, u64 kSize2,
45		typename AddressSpaceViewTy = LocalAddressSpaceView,
46		class MapUnmapCallback = NoOpMapUnmapCallback>
47		class TwoLevelByteMap {
48		public:
49		using AddressSpaceView = AddressSpaceViewTy;
50		void Init() {
51		internal_memset(map1_, 0, sizeof(map1_));
52		mu_.Init();
53		}
54
55		void TestOnlyUnmap() {
56		for (uptr i = 0; i < kSize1; i++) {
57		u8 *p = Get(i);
58		if (!p) continue;
59		MapUnmapCallback().OnUnmap(reinterpret_cast<uptr>(p), kSize2);
60		UnmapOrDie(p, kSize2);
61		}
62		}
63
64		uptr size() const { return kSize1 * kSize2; }
65		uptr size1() const { return kSize1; }
66		uptr size2() const { return kSize2; }
67
68		void set(uptr idx, u8 val) {
69		CHECK_LT(idx, kSize1 * kSize2);
70		u8 *map2 = GetOrCreate(idx / kSize2);
71		CHECK_EQ(0U, map2[idx % kSize2]);
72		map2[idx % kSize2] = val;
73		}
74
75		u8 operator[] (uptr idx) const {
76		CHECK_LT(idx, kSize1 * kSize2);
77		u8 *map2 = Get(idx / kSize2);
78		if (!map2) return 0;
79		auto value_ptr = AddressSpaceView::Load(&map2[idx % kSize2]);
80		return *value_ptr;
81		}
82
83		private:
84		u8 *Get(uptr idx) const {
85		CHECK_LT(idx, kSize1);
86		return reinterpret_cast<u8 *>(
87		atomic_load(&map1_[idx], memory_order_acquire));
88		}
89
90		u8 *GetOrCreate(uptr idx) {
91		u8 *res = Get(idx);
92		if (!res) {
93		SpinMutexLock l(&mu_);
94		if (!(res = Get(idx))) {
95		res = (u8*)MmapOrDie(kSize2, "TwoLevelByteMap");
96		MapUnmapCallback().OnMap(reinterpret_cast<uptr>(res), kSize2);
97		atomic_store(&map1_[idx], reinterpret_cast<uptr>(res),
98		memory_order_release);
99		}
100		}
101		return res;
102		}
103
104		atomic_uintptr_t map1_[kSize1];
105		StaticSpinMutex mu_;
106		};
107

lib/tsan/sanitizer_common/sanitizer_allocator_combined.h+8-10

...	...	@@ -29,9 +29,9 @@ class CombinedAllocator {
29	29	LargeMmapAllocatorPtrArray,
30	30	typename PrimaryAllocator::AddressSpaceView>;
31	31
32		void InitLinkerInitialized(s32 release_to_os_interval_ms) {
33		stats_.InitLinkerInitialized();
34		primary_.Init(release_to_os_interval_ms);
	32	void InitLinkerInitialized(s32 release_to_os_interval_ms,
	33	uptr heap_start = 0) {
	34	primary_.Init(release_to_os_interval_ms, heap_start);
35	35	secondary_.InitLinkerInitialized();
36	36	}
37	37
...	...	@@ -112,15 +112,13 @@ class CombinedAllocator {
112	112	return new_p;
113	113	}
114	114
115		bool PointerIsMine(void *p) {
	115	bool PointerIsMine(const void *p) const {
116	116	if (primary_.PointerIsMine(p))
117	117	return true;
118	118	return secondary_.PointerIsMine(p);
119	119	}
120	120
121		bool FromPrimary(void *p) {
122		return primary_.PointerIsMine(p);
123		}
	121	bool FromPrimary(const void *p) const { return primary_.PointerIsMine(p); }
124	122
125	123	void GetMetaData(const void p) {
126	124	if (primary_.PointerIsMine(p))
...	...	@@ -136,7 +134,7 @@ class CombinedAllocator {
136	134
137	135	// This function does the same as GetBlockBegin, but is much faster.
138	136	// Must be called with the allocator locked.
139		void GetBlockBeginFastLocked(void p) {
	137	void GetBlockBeginFastLocked(const void p) {
140	138	if (primary_.PointerIsMine(p))
141	139	return primary_.GetBlockBegin(p);
142	140	return secondary_.GetBlockBeginFastLocked(p);
...	...	@@ -177,12 +175,12 @@ class CombinedAllocator {
177	175
178	176	// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
179	177	// introspection API.
180		void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
	178	void ForceLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
181	179	primary_.ForceLock();
182	180	secondary_.ForceLock();
183	181	}
184	182
185		void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
	183	void ForceUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
186	184	secondary_.ForceUnlock();
187	185	primary_.ForceUnlock();
188	186	}

lib/tsan/sanitizer_common/sanitizer_allocator_dlsym.h created+79

...	...	@@ -0,0 +1,79 @@
	1	//===-- sanitizer_allocator_dlsym.h ------------------------------ C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// Hack: Sanitizer initializer calls dlsym which may need to allocate and call
	10	// back into uninitialized sanitizer.
	11	//
	12	//===----------------------------------------------------------------------===//
	13
	14	#ifndef SANITIZER_ALLOCATOR_DLSYM_H
	15	#define SANITIZER_ALLOCATOR_DLSYM_H
	16
	17	#include "sanitizer_allocator_internal.h"
	18
	19	namespace __sanitizer {
	20
	21	template <typename Details>
	22	struct DlSymAllocator {
	23	static bool Use() {
	24	// Fuchsia doesn't use dlsym-based interceptors.
	25	return !SANITIZER_FUCHSIA && UNLIKELY(Details::UseImpl());
	26	}
	27
	28	static bool PointerIsMine(const void *ptr) {
	29	// Fuchsia doesn't use dlsym-based interceptors.
	30	return !SANITIZER_FUCHSIA &&
	31	UNLIKELY(internal_allocator()->FromPrimary(ptr));
	32	}
	33
	34	static void *Allocate(uptr size_in_bytes) {
	35	void *ptr = InternalAlloc(size_in_bytes, nullptr, kWordSize);
	36	CHECK(internal_allocator()->FromPrimary(ptr));
	37	Details::OnAllocate(ptr,
	38	internal_allocator()->GetActuallyAllocatedSize(ptr));
	39	return ptr;
	40	}
	41
	42	static void *Callocate(SIZE_T nmemb, SIZE_T size) {
	43	void *ptr = InternalCalloc(nmemb, size);
	44	CHECK(internal_allocator()->FromPrimary(ptr));
	45	Details::OnAllocate(ptr,
	46	internal_allocator()->GetActuallyAllocatedSize(ptr));
	47	return ptr;
	48	}
	49
	50	static void Free(void *ptr) {
	51	uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
	52	Details::OnFree(ptr, size);
	53	InternalFree(ptr);
	54	}
	55
	56	static void Realloc(void ptr, uptr new_size) {
	57	if (!ptr)
	58	return Allocate(new_size);
	59	CHECK(internal_allocator()->FromPrimary(ptr));
	60	if (!new_size) {
	61	Free(ptr);
	62	return nullptr;
	63	}
	64	uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
	65	uptr memcpy_size = Min(new_size, size);
	66	void *new_ptr = Allocate(new_size);
	67	if (new_ptr)
	68	internal_memcpy(new_ptr, ptr, memcpy_size);
	69	Free(ptr);
	70	return new_ptr;
	71	}
	72
	73	static void OnAllocate(const void *ptr, uptr size) {}
	74	static void OnFree(const void *ptr, uptr size) {}
	75	};
	76
	77	} // namespace __sanitizer
	78
	79	#endif // SANITIZER_ALLOCATOR_DLSYM_H

lib/tsan/sanitizer_common/sanitizer_allocator_interface.h+4

...	...	@@ -21,8 +21,12 @@ extern "C" {
21	21	SANITIZER_INTERFACE_ATTRIBUTE
22	22	uptr __sanitizer_get_estimated_allocated_size(uptr size);
23	23	SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_get_ownership(const void *p);
	24	SANITIZER_INTERFACE_ATTRIBUTE const void *__sanitizer_get_allocated_begin(
	25	const void *p);
24	26	SANITIZER_INTERFACE_ATTRIBUTE uptr
25	27	__sanitizer_get_allocated_size(const void *p);
	28	SANITIZER_INTERFACE_ATTRIBUTE uptr
	29	__sanitizer_get_allocated_size_fast(const void *p);
26	30	SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_current_allocated_bytes();
27	31	SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_heap_size();
28	32	SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_free_bytes();

lib/tsan/sanitizer_common/sanitizer_allocator_internal.h+2-1

...	...	@@ -48,8 +48,9 @@ void InternalReallocArray(void p, uptr count, uptr size,
48	48	void *InternalCalloc(uptr count, uptr size,
49	49	InternalAllocatorCache *cache = nullptr);
50	50	void InternalFree(void p, InternalAllocatorCache cache = nullptr);
	51	void InternalAllocatorLock();
	52	void InternalAllocatorUnlock();
51	53	InternalAllocator *internal_allocator();
52
53	54	} // namespace __sanitizer
54	55
55	56	#endif // SANITIZER_ALLOCATOR_INTERNAL_H

lib/tsan/sanitizer_common/sanitizer_allocator_primary32.h+11-12

...	...	@@ -189,7 +189,7 @@ class SizeClassAllocator32 {
189	189	sci->free_list.push_front(b);
190	190	}
191	191
192		bool PointerIsMine(const void *p) {
	192	bool PointerIsMine(const void *p) const {
193	193	uptr mem = reinterpret_cast<uptr>(p);
194	194	if (SANITIZER_SIGN_EXTENDED_ADDRESSES)
195	195	mem &= (kSpaceSize - 1);
...	...	@@ -198,8 +198,9 @@ class SizeClassAllocator32 {
198	198	return GetSizeClass(p) != 0;
199	199	}
200	200
201		uptr GetSizeClass(const void *p) {
202		return possible_regions[ComputeRegionId(reinterpret_cast<uptr>(p))];
	201	uptr GetSizeClass(const void *p) const {
	202	uptr id = ComputeRegionId(reinterpret_cast<uptr>(p));
	203	return possible_regions.contains(id) ? possible_regions[id] : 0;
203	204	}
204	205
205	206	void GetBlockBegin(const void p) {
...	...	@@ -237,13 +238,13 @@ class SizeClassAllocator32 {
237	238
238	239	// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
239	240	// introspection API.
240		void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
	241	void ForceLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
241	242	for (uptr i = 0; i < kNumClasses; i++) {
242	243	GetSizeClassInfo(i)->mutex.Lock();
243	244	}
244	245	}
245	246
246		void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
	247	void ForceUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
247	248	for (int i = kNumClasses - 1; i >= 0; i--) {
248	249	GetSizeClassInfo(i)->mutex.Unlock();
249	250	}
...	...	@@ -251,9 +252,9 @@ class SizeClassAllocator32 {
251	252
252	253	// Iterate over all existing chunks.
253	254	// The allocator must be locked when calling this function.
254		void ForEachChunk(ForEachChunkCallback callback, void *arg) {
	255	void ForEachChunk(ForEachChunkCallback callback, void *arg) const {
255	256	for (uptr region = 0; region < kNumPossibleRegions; region++)
256		if (possible_regions[region]) {
	257	if (possible_regions.contains(region) && possible_regions[region]) {
257	258	uptr chunk_size = ClassIdToSize(possible_regions[region]);
258	259	uptr max_chunks_in_region = kRegionSize / (chunk_size + kMetadataSize);
259	260	uptr region_beg = region * kRegionSize;
...	...	@@ -292,9 +293,7 @@ class SizeClassAllocator32 {
292	293	return res;
293	294	}
294	295
295		uptr ComputeRegionBeg(uptr mem) {
296		return mem & ~(kRegionSize - 1);
297		}
	296	uptr ComputeRegionBeg(uptr mem) const { return mem & ~(kRegionSize - 1); }
298	297
299	298	uptr AllocateRegion(AllocatorStats *stat, uptr class_id) {
300	299	DCHECK_LT(class_id, kNumClasses);
...	...	@@ -305,7 +304,7 @@ class SizeClassAllocator32 {
305	304	MapUnmapCallback().OnMap(res, kRegionSize);
306	305	stat->Add(AllocatorStatMapped, kRegionSize);
307	306	CHECK(IsAligned(res, kRegionSize));
308		possible_regions.set(ComputeRegionId(res), static_cast<u8>(class_id));
	307	possible_regions[ComputeRegionId(res)] = class_id;
309	308	return res;
310	309	}
311	310
...	...	@@ -354,7 +353,7 @@ class SizeClassAllocator32 {
354	353	DCHECK_GT(max_count, 0);
355	354	TransferBatch *b = nullptr;
356	355	constexpr uptr kShuffleArraySize = 48;
357		uptr shuffle_array[kShuffleArraySize];
	356	UNINITIALIZED uptr shuffle_array[kShuffleArraySize];
358	357	uptr count = 0;
359	358	for (uptr i = region; i < region + n_chunks * size; i += size) {
360	359	shuffle_array[count++] = i;

lib/tsan/sanitizer_common/sanitizer_allocator_primary64.h+15-14

...	...	@@ -161,7 +161,7 @@ class SizeClassAllocator64 {
161	161	void ForceReleaseToOS() {
162	162	MemoryMapperT memory_mapper(*this);
163	163	for (uptr class_id = 1; class_id < kNumClasses; class_id++) {
164		BlockingMutexLock l(&GetRegionInfo(class_id)->mutex);
	164	Lock l(&GetRegionInfo(class_id)->mutex);
165	165	MaybeReleaseToOS(&memory_mapper, class_id, true /force/);
166	166	}
167	167	}
...	...	@@ -178,7 +178,7 @@ class SizeClassAllocator64 {
178	178	uptr region_beg = GetRegionBeginBySizeClass(class_id);
179	179	CompactPtrT *free_array = GetFreeArray(region_beg);
180	180
181		BlockingMutexLock l(&region->mutex);
	181	Lock l(&region->mutex);
182	182	uptr old_num_chunks = region->num_freed_chunks;
183	183	uptr new_num_freed_chunks = old_num_chunks + n_chunks;
184	184	// Failure to allocate free array space while releasing memory is non
...	...	@@ -204,7 +204,7 @@ class SizeClassAllocator64 {
204	204	uptr region_beg = GetRegionBeginBySizeClass(class_id);
205	205	CompactPtrT *free_array = GetFreeArray(region_beg);
206	206
207		BlockingMutexLock l(&region->mutex);
	207	Lock l(&region->mutex);
208	208	#if SANITIZER_WINDOWS
209	209	/* On Windows unmapping of memory during __sanitizer_purge_allocator is
210	210	explicit and immediate, so unmapped regions must be explicitly mapped back
...	...	@@ -282,6 +282,8 @@ class SizeClassAllocator64 {
282	282	CHECK(kMetadataSize);
283	283	uptr class_id = GetSizeClass(p);
284	284	uptr size = ClassIdToSize(class_id);
	285	if (!size)
	286	return nullptr;
285	287	uptr chunk_idx = GetChunkIdx(reinterpret_cast<uptr>(p), size);
286	288	uptr region_beg = GetRegionBeginBySizeClass(class_id);
287	289	return reinterpret_cast<void *>(GetMetadataEnd(region_beg) -
...	...	@@ -300,9 +302,8 @@ class SizeClassAllocator64 {
300	302	UnmapWithCallbackOrDie((uptr)address_range.base(), address_range.size());
301	303	}
302	304
303		static void FillMemoryProfile(uptr start, uptr rss, bool file, uptr *stats,
304		uptr stats_size) {
305		for (uptr class_id = 0; class_id < stats_size; class_id++)
	305	static void FillMemoryProfile(uptr start, uptr rss, bool file, uptr *stats) {
	306	for (uptr class_id = 0; class_id < kNumClasses; class_id++)
306	307	if (stats[class_id] == start)
307	308	stats[class_id] = rss;
308	309	}
...	...	@@ -315,7 +316,7 @@ class SizeClassAllocator64 {
315	316	Printf(
316	317	"%s %02zd (%6zd): mapped: %6zdK allocs: %7zd frees: %7zd inuse: %6zd "
317	318	"num_freed_chunks %7zd avail: %6zd rss: %6zdK releases: %6zd "
318		"last released: %6zdK region: 0x%zx\n",
	319	"last released: %6lldK region: 0x%zx\n",
319	320	region->exhausted ? "F" : " ", class_id, ClassIdToSize(class_id),
320	321	region->mapped_user >> 10, region->stats.n_allocated,
321	322	region->stats.n_freed, in_use, region->num_freed_chunks, avail_chunks,
...	...	@@ -328,7 +329,7 @@ class SizeClassAllocator64 {
328	329	uptr rss_stats[kNumClasses];
329	330	for (uptr class_id = 0; class_id < kNumClasses; class_id++)
330	331	rss_stats[class_id] = SpaceBeg() + kRegionSize * class_id;
331		GetMemoryProfile(FillMemoryProfile, rss_stats, kNumClasses);
	332	GetMemoryProfile(FillMemoryProfile, rss_stats);
332	333
333	334	uptr total_mapped = 0;
334	335	uptr total_rss = 0;
...	...	@@ -353,13 +354,13 @@ class SizeClassAllocator64 {
353	354
354	355	// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
355	356	// introspection API.
356		void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
	357	void ForceLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
357	358	for (uptr i = 0; i < kNumClasses; i++) {
358	359	GetRegionInfo(i)->mutex.Lock();
359	360	}
360	361	}
361	362
362		void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
	363	void ForceUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
363	364	for (int i = (int)kNumClasses - 1; i >= 0; i--) {
364	365	GetRegionInfo(i)->mutex.Unlock();
365	366	}
...	...	@@ -623,7 +624,7 @@ class SizeClassAllocator64 {
623	624
624	625	static const uptr kRegionSize = kSpaceSize / kNumClassesRounded;
625	626	// FreeArray is the array of free-d chunks (stored as 4-byte offsets).
626		// In the worst case it may reguire kRegionSize/SizeClassMap::kMinSize
	627	// In the worst case it may require kRegionSize/SizeClassMap::kMinSize
627	628	// elements, but in reality this will not happen. For simplicity we
628	629	// dedicate 1/8 of the region's virtual space to FreeArray.
629	630	static const uptr kFreeArraySize = kRegionSize / 8;
...	...	@@ -634,8 +635,8 @@ class SizeClassAllocator64 {
634	635	return kUsingConstantSpaceBeg ? kSpaceBeg : NonConstSpaceBeg;
635	636	}
636	637	uptr SpaceEnd() const { return SpaceBeg() + kSpaceSize; }
637		// kRegionSize must be >= 2^32.
638		COMPILER_CHECK((kRegionSize) >= (1ULL << (SANITIZER_WORDSIZE / 2)));
	638	// kRegionSize should be able to satisfy the largest size class.
	639	static_assert(kRegionSize >= SizeClassMap::kMaxSize);
639	640	// kRegionSize must be <= 2^36, see CompactPtrT.
640	641	COMPILER_CHECK((kRegionSize) <= (1ULL << (SANITIZER_WORDSIZE / 2 + 4)));
641	642	// Call mmap for user memory with at least this size.
...	...	@@ -665,7 +666,7 @@ class SizeClassAllocator64 {
665	666	};
666	667
667	668	struct ALIGNED(SANITIZER_CACHE_LINE_SIZE) RegionInfo {
668		BlockingMutex mutex;
	669	Mutex mutex;
669	670	uptr num_freed_chunks; // Number of elements in the freearray.
670	671	uptr mapped_free_array; // Bytes mapped for freearray.
671	672	uptr allocated_user; // Bytes allocated for user memory.

lib/tsan/sanitizer_common/sanitizer_allocator_report.cpp+1-2

...	...	@@ -128,8 +128,7 @@ void NORETURN ReportAllocationSizeTooBig(uptr user_size, uptr max_size,
128	128	void NORETURN ReportOutOfMemory(uptr requested_size, const StackTrace *stack) {
129	129	{
130	130	ScopedAllocatorErrorReport report("out-of-memory", stack);
131		Report("ERROR: %s: allocator is out of memory trying to allocate 0x%zx "
132		"bytes\n", SanitizerToolName, requested_size);
	131	ERROR_OOM("allocator is trying to allocate 0x%zx bytes\n", requested_size);
133	132	}
134	133	Die();
135	134	}

lib/tsan/sanitizer_common/sanitizer_allocator_secondary.h+9-9

...	...	@@ -82,7 +82,7 @@ class LargeMmapAllocator {
82	82	InitLinkerInitialized();
83	83	}
84	84
85		void Allocate(AllocatorStats stat, uptr size, uptr alignment) {
	85	void Allocate(AllocatorStats stat, const uptr size, uptr alignment) {
86	86	CHECK(IsPowerOfTwo(alignment));
87	87	uptr map_size = RoundUpMapSize(size);
88	88	if (alignment > page_size_)
...	...	@@ -99,11 +99,11 @@ class LargeMmapAllocator {
99	99	if (!map_beg)
100	100	return nullptr;
101	101	CHECK(IsAligned(map_beg, page_size_));
102		MapUnmapCallback().OnMap(map_beg, map_size);
103	102	uptr map_end = map_beg + map_size;
104	103	uptr res = map_beg + page_size_;
105	104	if (res & (alignment - 1)) // Align.
106	105	res += alignment - (res & (alignment - 1));
	106	MapUnmapCallback().OnMapSecondary(map_beg, map_size, res, size);
107	107	CHECK(IsAligned(res, alignment));
108	108	CHECK(IsAligned(res, page_size_));
109	109	CHECK_GE(res + size, map_beg);
...	...	@@ -161,7 +161,7 @@ class LargeMmapAllocator {
161	161	return res;
162	162	}
163	163
164		bool PointerIsMine(const void *p) {
	164	bool PointerIsMine(const void *p) const {
165	165	return GetBlockBegin(p) != nullptr;
166	166	}
167	167
...	...	@@ -179,7 +179,7 @@ class LargeMmapAllocator {
179	179	return GetHeader(p) + 1;
180	180	}
181	181
182		void GetBlockBegin(const void ptr) {
	182	void GetBlockBegin(const void ptr) const {
183	183	uptr p = reinterpret_cast<uptr>(ptr);
184	184	SpinMutexLock l(&mutex_);
185	185	uptr nearest_chunk = 0;
...	...	@@ -215,7 +215,7 @@ class LargeMmapAllocator {
215	215
216	216	// This function does the same as GetBlockBegin, but is much faster.
217	217	// Must be called with the allocator locked.
218		void GetBlockBeginFastLocked(void ptr) {
	218	void GetBlockBeginFastLocked(const void ptr) {
219	219	mutex_.CheckLocked();
220	220	uptr p = reinterpret_cast<uptr>(ptr);
221	221	uptr n = n_chunks_;
...	...	@@ -267,9 +267,9 @@ class LargeMmapAllocator {
267	267
268	268	// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
269	269	// introspection API.
270		void ForceLock() ACQUIRE(mutex_) { mutex_.Lock(); }
	270	void ForceLock() SANITIZER_ACQUIRE(mutex_) { mutex_.Lock(); }
271	271
272		void ForceUnlock() RELEASE(mutex_) { mutex_.Unlock(); }
	272	void ForceUnlock() SANITIZER_RELEASE(mutex_) { mutex_.Unlock(); }
273	273
274	274	// Iterate over all existing chunks.
275	275	// The allocator must be locked when calling this function.
...	...	@@ -301,7 +301,7 @@ class LargeMmapAllocator {
301	301	return GetHeader(reinterpret_cast<uptr>(p));
302	302	}
303	303
304		void GetUser(const Header h) {
	304	void GetUser(const Header h) const {
305	305	CHECK(IsAligned((uptr)h, page_size_));
306	306	return reinterpret_cast<void*>(reinterpret_cast<uptr>(h) + page_size_);
307	307	}
...	...	@@ -318,5 +318,5 @@ class LargeMmapAllocator {
318	318	struct Stats {
319	319	uptr n_allocs, n_frees, currently_allocated, max_allocated, by_size_log[64];
320	320	} stats;
321		StaticSpinMutex mutex_;
	321	mutable StaticSpinMutex mutex_;
322	322	};

lib/tsan/sanitizer_common/sanitizer_allocator_size_class_map.h+4-4

...	...	@@ -193,13 +193,13 @@ class SizeClassMap {
193	193	uptr cached = MaxCachedHint(s) * s;
194	194	if (i == kBatchClassID)
195	195	d = p = l = 0;
196		Printf("c%02zd => s: %zd diff: +%zd %02zd%% l %zd "
197		"cached: %zd %zd; id %zd\n",
198		i, Size(i), d, p, l, MaxCachedHint(s), cached, ClassID(s));
	196	Printf(
	197	"c%02zu => s: %zu diff: +%zu %02zu%% l %zu cached: %zu %zu; id %zu\n",
	198	i, Size(i), d, p, l, MaxCachedHint(s), cached, ClassID(s));
199	199	total_cached += cached;
200	200	prev_s = s;
201	201	}
202		Printf("Total cached: %zd\n", total_cached);
	202	Printf("Total cached: %zu\n", total_cached);
203	203	}
204	204
205	205	static void Validate() {

lib/tsan/sanitizer_common/sanitizer_allocator_stats.h+12-15

...	...	@@ -25,19 +25,13 @@ typedef uptr AllocatorStatCounters[AllocatorStatCount];
25	25	// Per-thread stats, live in per-thread cache.
26	26	class AllocatorStats {
27	27	public:
28		void Init() {
29		internal_memset(this, 0, sizeof(*this));
30		}
31		void InitLinkerInitialized() {}
32
	28	void Init() { internal_memset(this, 0, sizeof(*this)); }
33	29	void Add(AllocatorStat i, uptr v) {
34		v += atomic_load(&stats_[i], memory_order_relaxed);
35		atomic_store(&stats_[i], v, memory_order_relaxed);
	30	atomic_fetch_add(&stats_[i], v, memory_order_relaxed);
36	31	}
37	32
38	33	void Sub(AllocatorStat i, uptr v) {
39		v = atomic_load(&stats_[i], memory_order_relaxed) - v;
40		atomic_store(&stats_[i], v, memory_order_relaxed);
	34	atomic_fetch_sub(&stats_[i], v, memory_order_relaxed);
41	35	}
42	36
43	37	void Set(AllocatorStat i, uptr v) {
...	...	@@ -58,17 +52,13 @@ class AllocatorStats {
58	52	// Global stats, used for aggregation and querying.
59	53	class AllocatorGlobalStats : public AllocatorStats {
60	54	public:
61		void InitLinkerInitialized() {
62		next_ = this;
63		prev_ = this;
64		}
65	55	void Init() {
66	56	internal_memset(this, 0, sizeof(*this));
67		InitLinkerInitialized();
68	57	}
69	58
70	59	void Register(AllocatorStats *s) {
71	60	SpinMutexLock l(&mu_);
	61	LazyInit();
72	62	s->next_ = next_;
73	63	s->prev_ = this;
74	64	next_->prev_ = s;
...	...	@@ -87,7 +77,7 @@ class AllocatorGlobalStats : public AllocatorStats {
87	77	internal_memset(s, 0, AllocatorStatCount * sizeof(uptr));
88	78	SpinMutexLock l(&mu_);
89	79	const AllocatorStats *stats = this;
90		for (;;) {
	80	for (; stats;) {
91	81	for (int i = 0; i < AllocatorStatCount; i++)
92	82	s[i] += stats->Get(AllocatorStat(i));
93	83	stats = stats->next_;
...	...	@@ -100,6 +90,13 @@ class AllocatorGlobalStats : public AllocatorStats {
100	90	}
101	91
102	92	private:
	93	void LazyInit() {
	94	if (!next_) {
	95	next_ = this;
	96	prev_ = this;
	97	}
	98	}
	99
103	100	mutable StaticSpinMutex mu_;
104	101	};
105	102

lib/tsan/sanitizer_common/sanitizer_array_ref.h created+123

...	...	@@ -0,0 +1,123 @@
	1	//===-- sanitizer_array_ref.h ------------------------------------ C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef SANITIZER_ARRAY_REF_H
	10	#define SANITIZER_ARRAY_REF_H
	11
	12	#include "sanitizer_internal_defs.h"
	13
	14	namespace __sanitizer {
	15
	16	/// ArrayRef - Represent a constant reference to an array (0 or more elements
	17	/// consecutively in memory), i.e. a start pointer and a length. It allows
	18	/// various APIs to take consecutive elements easily and conveniently.
	19	///
	20	/// This class does not own the underlying data, it is expected to be used in
	21	/// situations where the data resides in some other buffer, whose lifetime
	22	/// extends past that of the ArrayRef. For this reason, it is not in general
	23	/// safe to store an ArrayRef.
	24	///
	25	/// This is intended to be trivially copyable, so it should be passed by
	26	/// value.
	27	template <typename T>
	28	class ArrayRef {
	29	public:
	30	constexpr ArrayRef() {}
	31	constexpr ArrayRef(const T begin, const T end) : begin_(begin), end_(end) {
	32	DCHECK(empty() \|\| begin);
	33	}
	34	constexpr ArrayRef(const T *data, uptr length)
	35	: ArrayRef(data, data + length) {}
	36	template <uptr N>
	37	constexpr ArrayRef(const T (&src)[N]) : ArrayRef(src, src + N) {}
	38	template <typename C>
	39	constexpr ArrayRef(const C &src)
	40	: ArrayRef(src.data(), src.data() + src.size()) {}
	41	ArrayRef(const T &one_elt) : ArrayRef(&one_elt, &one_elt + 1) {}
	42
	43	const T *data() const { return empty() ? nullptr : begin_; }
	44
	45	const T *begin() const { return begin_; }
	46	const T *end() const { return end_; }
	47
	48	bool empty() const { return begin_ == end_; }
	49
	50	uptr size() const { return end_ - begin_; }
	51
	52	/// equals - Check for element-wise equality.
	53	bool equals(ArrayRef rhs) const {
	54	if (size() != rhs.size())
	55	return false;
	56	auto r = rhs.begin();
	57	for (auto &l : *this) {
	58	if (!(l == *r))
	59	return false;
	60	++r;
	61	}
	62	return true;
	63	}
	64
	65	/// slice(n, m) - Chop off the first N elements of the array, and keep M
	66	/// elements in the array.
	67	ArrayRef<T> slice(uptr N, uptr M) const {
	68	DCHECK_LE(N + M, size());
	69	return ArrayRef<T>(data() + N, M);
	70	}
	71
	72	/// slice(n) - Chop off the first N elements of the array.
	73	ArrayRef<T> slice(uptr N) const { return slice(N, size() - N); }
	74
	75	/// Drop the first \p N elements of the array.
	76	ArrayRef<T> drop_front(uptr N = 1) const {
	77	DCHECK_GE(size(), N);
	78	return slice(N, size() - N);
	79	}
	80
	81	/// Drop the last \p N elements of the array.
	82	ArrayRef<T> drop_back(uptr N = 1) const {
	83	DCHECK_GE(size(), N);
	84	return slice(0, size() - N);
	85	}
	86
	87	/// Return a copy of *this with only the first \p N elements.
	88	ArrayRef<T> take_front(uptr N = 1) const {
	89	if (N >= size())
	90	return *this;
	91	return drop_back(size() - N);
	92	}
	93
	94	/// Return a copy of *this with only the last \p N elements.
	95	ArrayRef<T> take_back(uptr N = 1) const {
	96	if (N >= size())
	97	return *this;
	98	return drop_front(size() - N);
	99	}
	100
	101	const T &operator[](uptr index) const {
	102	DCHECK_LT(index, size());
	103	return begin_[index];
	104	}
	105
	106	private:
	107	const T *begin_ = nullptr;
	108	const T *end_ = nullptr;
	109	};
	110
	111	template <typename T>
	112	inline bool operator==(ArrayRef<T> lhs, ArrayRef<T> rhs) {
	113	return lhs.equals(rhs);
	114	}
	115
	116	template <typename T>
	117	inline bool operator!=(ArrayRef<T> lhs, ArrayRef<T> rhs) {
	118	return !(lhs == rhs);
	119	}
	120
	121	} // namespace __sanitizer
	122
	123	#endif // SANITIZER_ARRAY_REF_H

lib/tsan/sanitizer_common/sanitizer_asm.h+54-3

...	...	@@ -6,7 +6,7 @@
6	6	//
7	7	//===----------------------------------------------------------------------===//
8	8	//
9		// Various support for assemebler.
	9	// Various support for assembler.
10	10	//
11	11	//===----------------------------------------------------------------------===//
12	12
...	...	@@ -42,13 +42,57 @@
42	42	# define CFI_RESTORE(reg)
43	43	#endif
44	44
	45	#if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__sparc__)
	46	# define ASM_TAIL_CALL jmp
	47	#elif defined(__arm__) \|\| defined(__aarch64__) \|\| defined(__mips__) \|\| \
	48	defined(__powerpc__) \|\| defined(__loongarch_lp64)
	49	# define ASM_TAIL_CALL b
	50	#elif defined(__s390__)
	51	# define ASM_TAIL_CALL jg
	52	#elif defined(__riscv)
	53	# define ASM_TAIL_CALL tail
	54	#endif
	55
	56	#if defined(__ELF__) && defined(__x86_64__) \|\| defined(__i386__) \|\| \
	57	defined(__riscv)
	58	# define ASM_PREEMPTIBLE_SYM(sym) sym@plt
	59	#else
	60	# define ASM_PREEMPTIBLE_SYM(sym) sym
	61	#endif
	62
45	63	#if !defined(__APPLE__)
46	64	# define ASM_HIDDEN(symbol) .hidden symbol
47	65	# define ASM_TYPE_FUNCTION(symbol) .type symbol, %function
48	66	# define ASM_SIZE(symbol) .size symbol, .-symbol
49	67	# define ASM_SYMBOL(symbol) symbol
50	68	# define ASM_SYMBOL_INTERCEPTOR(symbol) symbol
51		# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol
	69	# if defined(__i386__) \|\| defined(__powerpc__) \|\| defined(__s390__) \|\| \
	70	defined(__sparc__)
	71	// For details, see interception.h
	72	# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol
	73	# define ASM_TRAMPOLINE_ALIAS(symbol, name) \
	74	.weak symbol; \
	75	.set symbol, ASM_WRAPPER_NAME(name)
	76	# define ASM_INTERCEPTOR_TRAMPOLINE(name)
	77	# define ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT 0
	78	# else // Architecture supports interceptor trampoline
	79	// Keep trampoline implementation in sync with interception/interception.h
	80	# define ASM_WRAPPER_NAME(symbol) ___interceptor_##symbol
	81	# define ASM_TRAMPOLINE_ALIAS(symbol, name) \
	82	.weak symbol; \
	83	.set symbol, __interceptor_trampoline_##name
	84	# define ASM_INTERCEPTOR_TRAMPOLINE(name) \
	85	.weak __interceptor_##name; \
	86	.set __interceptor_##name, ASM_WRAPPER_NAME(name); \
	87	.globl __interceptor_trampoline_##name; \
	88	ASM_TYPE_FUNCTION(__interceptor_trampoline_##name); \
	89	__interceptor_trampoline_##name: \
	90	CFI_STARTPROC; \
	91	ASM_TAIL_CALL ASM_PREEMPTIBLE_SYM(__interceptor_##name); \
	92	CFI_ENDPROC; \
	93	ASM_SIZE(__interceptor_trampoline_##name)
	94	# define ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT 1
	95	# endif // Architecture supports interceptor trampoline
52	96	#else
53	97	# define ASM_HIDDEN(symbol)
54	98	# define ASM_TYPE_FUNCTION(symbol)
...	...	@@ -61,8 +105,15 @@
61	105	#if defined(__ELF__) && (defined(__GNU__) \|\| defined(__FreeBSD__) \|\| \
62	106	defined(__Fuchsia__) \|\| defined(__linux__))
63	107	// clang-format off
64		#define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits // NOLINT
	108	#define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits
65	109	// clang-format on
66	110	#else
67	111	#define NO_EXEC_STACK_DIRECTIVE
68	112	#endif
	113
	114	#if (defined(__x86_64__) \|\| defined(__i386__)) && defined(__has_include) && __has_include(<cet.h>)
	115	#include <cet.h>
	116	#endif
	117	#ifndef _CET_ENDBR
	118	#define _CET_ENDBR
	119	#endif

lib/tsan/sanitizer_common/sanitizer_atomic_clang.h+8-9

...	...	@@ -74,13 +74,12 @@ template <typename T>
74	74	inline bool atomic_compare_exchange_strong(volatile T a, typename T::Type cmp,
75	75	typename T::Type xchg,
76	76	memory_order mo) {
77		typedef typename T::Type Type;
78		Type cmpv = *cmp;
79		Type prev;
80		prev = __sync_val_compare_and_swap(&a->val_dont_use, cmpv, xchg);
81		if (prev == cmpv) return true;
82		*cmp = prev;
83		return false;
	77	// Transitioned from __sync_val_compare_and_swap to support targets like
	78	// SPARC V8 that cannot inline atomic cmpxchg. __atomic_compare_exchange
	79	// can then be resolved from libatomic. __ATOMIC_SEQ_CST is used to best
	80	// match the __sync builtin memory order.
	81	return __atomic_compare_exchange(&a->val_dont_use, cmp, &xchg, false,
	82	__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
84	83	}
85	84
86	85	template<typename T>
...	...	@@ -96,8 +95,8 @@ inline bool atomic_compare_exchange_weak(volatile T *a,
96	95	// This include provides explicit template instantiations for atomic_uint64_t
97	96	// on MIPS32, which does not directly support 8 byte atomics. It has to
98	97	// proceed the template definitions above.
99		#if defined(_MIPS_SIM) && defined(_ABIO32)
100		#include "sanitizer_atomic_clang_mips.h"
	98	#if defined(_MIPS_SIM) && defined(_ABIO32) && _MIPS_SIM == _ABIO32
	99	# include "sanitizer_atomic_clang_mips.h"
101	100	#endif
102	101
103	102	#undef ATOMIC_ORDER

lib/tsan/sanitizer_common/sanitizer_atomic_clang_mips.h+1-1

...	...	@@ -18,7 +18,7 @@ namespace __sanitizer {
18	18
19	19	// MIPS32 does not support atomics > 4 bytes. To address this lack of
20	20	// functionality, the sanitizer library provides helper methods which use an
21		// internal spin lock mechanism to emulate atomic oprations when the size is
	21	// internal spin lock mechanism to emulate atomic operations when the size is
22	22	// 8 bytes.
23	23	static void __spin_lock(volatile int *lock) {
24	24	while (__sync_lock_test_and_set(lock, 1))

lib/tsan/sanitizer_common/sanitizer_chained_origin_depot.cpp created+148

...	...	@@ -0,0 +1,148 @@
	1	//===-- sanitizer_chained_origin_depot.cpp --------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// A storage for chained origins.
	10	//===----------------------------------------------------------------------===//
	11
	12	#include "sanitizer_chained_origin_depot.h"
	13
	14	#include "sanitizer_stackdepotbase.h"
	15
	16	namespace __sanitizer {
	17
	18	namespace {
	19	struct ChainedOriginDepotDesc {
	20	u32 here_id;
	21	u32 prev_id;
	22	};
	23
	24	struct ChainedOriginDepotNode {
	25	using hash_type = u32;
	26	u32 link;
	27	u32 here_id;
	28	u32 prev_id;
	29
	30	typedef ChainedOriginDepotDesc args_type;
	31
	32	bool eq(hash_type hash, const args_type &args) const;
	33
	34	static uptr allocated() { return 0; }
	35
	36	static hash_type hash(const args_type &args);
	37
	38	static bool is_valid(const args_type &args);
	39
	40	void store(u32 id, const args_type &args, hash_type other_hash);
	41
	42	args_type load(u32 id) const;
	43
	44	struct Handle {
	45	const ChainedOriginDepotNode *node_ = nullptr;
	46	u32 id_ = 0;
	47	Handle(const ChainedOriginDepotNode *node, u32 id) : node_(node), id_(id) {}
	48	bool valid() const { return node_; }
	49	u32 id() const { return id_; }
	50	int here_id() const { return node_->here_id; }
	51	int prev_id() const { return node_->prev_id; }
	52	};
	53
	54	static Handle get_handle(u32 id);
	55
	56	typedef Handle handle_type;
	57	};
	58
	59	} // namespace
	60
	61	static StackDepotBase<ChainedOriginDepotNode, 4, 20> depot;
	62
	63	bool ChainedOriginDepotNode::eq(hash_type hash, const args_type &args) const {
	64	return here_id == args.here_id && prev_id == args.prev_id;
	65	}
	66
	67	/* This is murmur2 hash for the 64->32 bit case.
	68	It does not behave all that well because the keys have a very biased
	69	distribution (I've seen 7-element buckets with the table only 14% full).
	70
	71	here_id is built of
	72	* (1 bits) Reserved, zero.
	73	* (8 bits) Part id = bits 13..20 of the hash value of here_id's key.
	74	* (23 bits) Sequential number (each part has each own sequence).
	75
	76	prev_id has either the same distribution as here_id (but with 3:8:21)
	77	split, or one of two reserved values (-1) or (-2). Either case can
	78	dominate depending on the workload.
	79	*/
	80	ChainedOriginDepotNode::hash_type ChainedOriginDepotNode::hash(
	81	const args_type &args) {
	82	const u32 m = 0x5bd1e995;
	83	const u32 seed = 0x9747b28c;
	84	const u32 r = 24;
	85	u32 h = seed;
	86	u32 k = args.here_id;
	87	k *= m;
	88	k ^= k >> r;
	89	k *= m;
	90	h *= m;
	91	h ^= k;
	92
	93	k = args.prev_id;
	94	k *= m;
	95	k ^= k >> r;
	96	k *= m;
	97	h *= m;
	98	h ^= k;
	99
	100	h ^= h >> 13;
	101	h *= m;
	102	h ^= h >> 15;
	103	return h;
	104	}
	105
	106	bool ChainedOriginDepotNode::is_valid(const args_type &args) { return true; }
	107
	108	void ChainedOriginDepotNode::store(u32 id, const args_type &args,
	109	hash_type other_hash) {
	110	here_id = args.here_id;
	111	prev_id = args.prev_id;
	112	}
	113
	114	ChainedOriginDepotNode::args_type ChainedOriginDepotNode::load(u32 id) const {
	115	args_type ret = {here_id, prev_id};
	116	return ret;
	117	}
	118
	119	ChainedOriginDepotNode::Handle ChainedOriginDepotNode::get_handle(u32 id) {
	120	return Handle(&depot.nodes[id], id);
	121	}
	122
	123	ChainedOriginDepot::ChainedOriginDepot() {}
	124
	125	StackDepotStats ChainedOriginDepot::GetStats() const {
	126	return depot.GetStats();
	127	}
	128
	129	bool ChainedOriginDepot::Put(u32 here_id, u32 prev_id, u32 *new_id) {
	130	ChainedOriginDepotDesc desc = {here_id, prev_id};
	131	bool inserted;
	132	*new_id = depot.Put(desc, &inserted);
	133	return inserted;
	134	}
	135
	136	u32 ChainedOriginDepot::Get(u32 id, u32 *other) {
	137	ChainedOriginDepotDesc desc = depot.Get(id);
	138	*other = desc.prev_id;
	139	return desc.here_id;
	140	}
	141
	142	void ChainedOriginDepot::LockAll() { depot.LockAll(); }
	143
	144	void ChainedOriginDepot::UnlockAll() { depot.UnlockAll(); }
	145
	146	void ChainedOriginDepot::TestOnlyUnmap() { depot.TestOnlyUnmap(); }
	147
	148	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_chained_origin_depot.h created+46

...	...	@@ -0,0 +1,46 @@
	1	//===-- sanitizer_chained_origin_depot.h ------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// A storage for chained origins.
	10	//===----------------------------------------------------------------------===//
	11
	12	#ifndef SANITIZER_CHAINED_ORIGIN_DEPOT_H
	13	#define SANITIZER_CHAINED_ORIGIN_DEPOT_H
	14
	15	#include "sanitizer_common.h"
	16
	17	namespace __sanitizer {
	18
	19	class ChainedOriginDepot {
	20	public:
	21	ChainedOriginDepot();
	22
	23	// Gets the statistic of the origin chain storage.
	24	StackDepotStats GetStats() const;
	25
	26	// Stores a chain with StackDepot ID here_id and previous chain ID prev_id.
	27	// If successful, returns true and the new chain id new_id.
	28	// If the same element already exists, returns false and sets new_id to the
	29	// existing ID.
	30	bool Put(u32 here_id, u32 prev_id, u32 *new_id);
	31
	32	// Retrieves the stored StackDepot ID for the given origin ID.
	33	u32 Get(u32 id, u32 *other);
	34
	35	void LockAll();
	36	void UnlockAll();
	37	void TestOnlyUnmap();
	38
	39	private:
	40	ChainedOriginDepot(const ChainedOriginDepot &) = delete;
	41	void operator=(const ChainedOriginDepot &) = delete;
	42	};
	43
	44	} // namespace __sanitizer
	45
	46	#endif // SANITIZER_CHAINED_ORIGIN_DEPOT_H

lib/tsan/sanitizer_common/sanitizer_common.cpp+68-10

...	...	@@ -11,10 +11,12 @@
11	11	//===----------------------------------------------------------------------===//
12	12
13	13	#include "sanitizer_common.h"
	14
14	15	#include "sanitizer_allocator_interface.h"
15	16	#include "sanitizer_allocator_internal.h"
16	17	#include "sanitizer_atomic.h"
17	18	#include "sanitizer_flags.h"
	19	#include "sanitizer_interface_internal.h"
18	20	#include "sanitizer_libc.h"
19	21	#include "sanitizer_placement_new.h"
20	22
...	...	@@ -44,15 +46,41 @@ void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
44	46	Die();
45	47	}
46	48	recursion_count++;
47		Report("ERROR: %s failed to "
48		"%s 0x%zx (%zd) bytes of %s (error code: %d)\n",
49		SanitizerToolName, mmap_type, size, size, mem_type, err);
	49	if (ErrorIsOOM(err)) {
	50	ERROR_OOM("failed to %s 0x%zx (%zd) bytes of %s (error code: %d)\n",
	51	mmap_type, size, size, mem_type, err);
	52	} else {
	53	Report(
	54	"ERROR: %s failed to "
	55	"%s 0x%zx (%zd) bytes of %s (error code: %d)\n",
	56	SanitizerToolName, mmap_type, size, size, mem_type, err);
	57	}
50	58	#if !SANITIZER_GO
51	59	DumpProcessMap();
52	60	#endif
53	61	UNREACHABLE("unable to mmap");
54	62	}
55	63
	64	void NORETURN ReportMunmapFailureAndDie(void *addr, uptr size, error_t err,
	65	bool raw_report) {
	66	static int recursion_count;
	67	if (raw_report \|\| recursion_count) {
	68	// If raw report is requested or we went into recursion just die. The
	69	// Report() and CHECK calls below may call munmap recursively and fail.
	70	RawWrite("ERROR: Failed to munmap\n");
	71	Die();
	72	}
	73	recursion_count++;
	74	Report(
	75	"ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p (error "
	76	"code: %d)\n",
	77	SanitizerToolName, size, size, addr, err);
	78	#if !SANITIZER_GO
	79	DumpProcessMap();
	80	#endif
	81	UNREACHABLE("unable to unmmap");
	82	}
	83
56	84	typedef bool UptrComparisonFunction(const uptr &a, const uptr &b);
57	85	typedef bool U32ComparisonFunction(const u32 &a, const u32 &b);
58	86
...	...	@@ -138,13 +166,21 @@ void LoadedModule::set(const char *module_name, uptr base_address,
138	166	set(module_name, base_address);
139	167	arch_ = arch;
140	168	internal_memcpy(uuid_, uuid, sizeof(uuid_));
	169	uuid_size_ = kModuleUUIDSize;
141	170	instrumented_ = instrumented;
142	171	}
143	172
	173	void LoadedModule::setUuid(const char *uuid, uptr size) {
	174	if (size > kModuleUUIDSize)
	175	size = kModuleUUIDSize;
	176	internal_memcpy(uuid_, uuid, size);
	177	uuid_size_ = size;
	178	}
	179
144	180	void LoadedModule::clear() {
145	181	InternalFree(full_name_);
146	182	base_address_ = 0;
147		max_executable_address_ = 0;
	183	max_address_ = 0;
148	184	full_name_ = nullptr;
149	185	arch_ = kModuleArchUnknown;
150	186	internal_memset(uuid_, 0, kModuleUUIDSize);
...	...	@@ -162,8 +198,7 @@ void LoadedModule::addAddressRange(uptr beg, uptr end, bool executable,
162	198	AddressRange *r =
163	199	new(mem) AddressRange(beg, end, executable, writable, name);
164	200	ranges_.push_back(r);
165		if (executable && end > max_executable_address_)
166		max_executable_address_ = end;
	201	max_address_ = Max(max_address_, end);
167	202	}
168	203
169	204	bool LoadedModule::containsAddress(uptr address) const {
...	...	@@ -301,18 +336,22 @@ struct MallocFreeHook {
301	336
302	337	static MallocFreeHook MFHooks[kMaxMallocFreeHooks];
303	338
304		void RunMallocHooks(const void *ptr, uptr size) {
	339	void RunMallocHooks(void *ptr, uptr size) {
	340	__sanitizer_malloc_hook(ptr, size);
305	341	for (int i = 0; i < kMaxMallocFreeHooks; i++) {
306	342	auto hook = MFHooks[i].malloc_hook;
307		if (!hook) return;
	343	if (!hook)
	344	break;
308	345	hook(ptr, size);
309	346	}
310	347	}
311	348
312		void RunFreeHooks(const void *ptr) {
	349	void RunFreeHooks(void *ptr) {
	350	__sanitizer_free_hook(ptr);
313	351	for (int i = 0; i < kMaxMallocFreeHooks; i++) {
314	352	auto hook = MFHooks[i].free_hook;
315		if (!hook) return;
	353	if (!hook)
	354	break;
316	355	hook(ptr);
317	356	}
318	357	}
...	...	@@ -338,6 +377,13 @@ void SleepForSeconds(unsigned seconds) {
338	377	}
339	378	void SleepForMillis(unsigned millis) { internal_usleep((u64)millis * 1000); }
340	379
	380	void WaitForDebugger(unsigned seconds, const char *label) {
	381	if (seconds) {
	382	Report("Sleeping for %u second(s) %s\n", seconds, label);
	383	SleepForSeconds(seconds);
	384	}
	385	}
	386
341	387	} // namespace __sanitizer
342	388
343	389	using namespace __sanitizer;
...	...	@@ -360,4 +406,16 @@ int __sanitizer_install_malloc_and_free_hooks(void (malloc_hook)(const void ,
360	406	void (free_hook)(const void )) {
361	407	return InstallMallocFreeHooks(malloc_hook, free_hook);
362	408	}
	409
	410	// Provide default (no-op) implementation of malloc hooks.
	411	SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook, void *ptr,
	412	uptr size) {
	413	(void)ptr;
	414	(void)size;
	415	}
	416
	417	SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *ptr) {
	418	(void)ptr;
	419	}
	420
363	421	} // extern "C"

lib/tsan/sanitizer_common/sanitizer_common.h+82-68

...	...	@@ -16,7 +16,6 @@
16	16	#define SANITIZER_COMMON_H
17	17
18	18	#include "sanitizer_flags.h"
19		#include "sanitizer_interface_internal.h"
20	19	#include "sanitizer_internal_defs.h"
21	20	#include "sanitizer_libc.h"
22	21	#include "sanitizer_list.h"
...	...	@@ -118,9 +117,15 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
118	117	// unaccessible memory.
119	118	bool MprotectNoAccess(uptr addr, uptr size);
120	119	bool MprotectReadOnly(uptr addr, uptr size);
	120	bool MprotectReadWrite(uptr addr, uptr size);
121	121
122	122	void MprotectMallocZones(void *addr, int prot);
123	123
	124	#if SANITIZER_WINDOWS
	125	// Zero previously mmap'd memory. Currently used only on Windows.
	126	bool ZeroMmapFixedRegion(uptr fixed_addr, uptr size) WARN_UNUSED_RESULT;
	127	#endif
	128
124	129	#if SANITIZER_LINUX
125	130	// Unmap memory. Currently only used on Linux.
126	131	void UnmapFromTo(uptr from, uptr to);
...	...	@@ -171,8 +176,8 @@ void SetShadowRegionHugePageMode(uptr addr, uptr length);
171	176	bool DontDumpShadowMemory(uptr addr, uptr length);
172	177	// Check if the built VMA size matches the runtime one.
173	178	void CheckVMASize();
174		void RunMallocHooks(const void *ptr, uptr size);
175		void RunFreeHooks(const void *ptr);
	179	void RunMallocHooks(void *ptr, uptr size);
	180	void RunFreeHooks(void *ptr);
176	181
177	182	class ReservedAddressRange {
178	183	public:
...	...	@@ -192,12 +197,13 @@ class ReservedAddressRange {
192	197	};
193	198
194	199	typedef void (*fill_profile_f)(uptr start, uptr rss, bool file,
195		/out/uptr *stats, uptr stats_size);
	200	/out/ uptr *stats);
196	201
197	202	// Parse the contents of /proc/self/smaps and generate a memory profile.
198		// \|cb\| is a tool-specific callback that fills the \|stats\| array containing
199		// \|stats_size\| elements.
200		void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size);
	203	// \|cb\| is a tool-specific callback that fills the \|stats\| array.
	204	void GetMemoryProfile(fill_profile_f cb, uptr *stats);
	205	void ParseUnixMemoryProfile(fill_profile_f cb, uptr stats, char smaps,
	206	uptr smaps_len);
201	207
202	208	// Simple low-level (mmap-based) allocator for internal use. Doesn't have
203	209	// constructor, so all instances of LowLevelAllocator should be
...	...	@@ -206,6 +212,7 @@ class LowLevelAllocator {
206	212	public:
207	213	// Requires an external lock.
208	214	void *Allocate(uptr size);
	215
209	216	private:
210	217	char *allocated_end_;
211	218	char *allocated_current_;
...	...	@@ -222,8 +229,8 @@ void CatastrophicErrorWrite(const char *buffer, uptr length);
222	229	void RawWrite(const char *buffer);
223	230	bool ColorizeReports();
224	231	void RemoveANSIEscapeSequencesFromString(char *buffer);
225		void Printf(const char *format, ...);
226		void Report(const char *format, ...);
	232	void Printf(const char *format, ...) FORMAT(1, 2);
	233	void Report(const char *format, ...) FORMAT(1, 2);
227	234	void SetPrintfAndReportCallback(void (callback)(const char ));
228	235	#define VReport(level, ...) \
229	236	do { \
...	...	@@ -237,12 +244,12 @@ void SetPrintfAndReportCallback(void (callback)(const char ));
237	244	// Lock sanitizer error reporting and protects against nested errors.
238	245	class ScopedErrorReportLock {
239	246	public:
240		ScopedErrorReportLock() ACQUIRE(mutex_) { Lock(); }
241		~ScopedErrorReportLock() RELEASE(mutex_) { Unlock(); }
	247	ScopedErrorReportLock() SANITIZER_ACQUIRE(mutex_) { Lock(); }
	248	~ScopedErrorReportLock() SANITIZER_RELEASE(mutex_) { Unlock(); }
242	249
243		static void Lock() ACQUIRE(mutex_);
244		static void Unlock() RELEASE(mutex_);
245		static void CheckLocked() CHECK_LOCKED(mutex_);
	250	static void Lock() SANITIZER_ACQUIRE(mutex_);
	251	static void Unlock() SANITIZER_RELEASE(mutex_);
	252	static void CheckLocked() SANITIZER_CHECK_LOCKED(mutex_);
246	253
247	254	private:
248	255	static atomic_uintptr_t reporting_thread_;
...	...	@@ -285,7 +292,7 @@ void SetStackSizeLimitInBytes(uptr limit);
285	292	bool AddressSpaceIsUnlimited();
286	293	void SetAddressSpaceUnlimited();
287	294	void AdjustStackSize(void *attr);
288		void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args);
	295	void PlatformPrepareForSandboxing(void *args);
289	296	void SetSandboxingCallback(void (*f)());
290	297
291	298	void InitializeCoverage(bool enabled, const char *coverage_dir);
...	...	@@ -294,6 +301,7 @@ void InitTlsSize();
294	301	uptr GetTlsSize();
295	302
296	303	// Other
	304	void WaitForDebugger(unsigned seconds, const char *label);
297	305	void SleepForSeconds(unsigned seconds);
298	306	void SleepForMillis(unsigned millis);
299	307	u64 NanoTime();
...	...	@@ -309,6 +317,20 @@ CheckFailed(const char file, int line, const char cond, u64 v1, u64 v2);
309	317	void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
310	318	const char *mmap_type, error_t err,
311	319	bool raw_report = false);
	320	void NORETURN ReportMunmapFailureAndDie(void *ptr, uptr size, error_t err,
	321	bool raw_report = false);
	322
	323	// Returns true if the platform-specific error reported is an OOM error.
	324	bool ErrorIsOOM(error_t err);
	325
	326	// This reports an error in the form:
	327	//
	328	// `ERROR: {{SanitizerToolName}}: out of memory: {{err_msg}}`
	329	//
	330	// Downstream tools that read sanitizer output will know that errors starting
	331	// in this format are specifically OOM errors.
	332	#define ERROR_OOM(err_msg, ...) \
	333	Report("ERROR: %s: out of memory: " err_msg, SanitizerToolName, __VA_ARGS__)
312	334
313	335	// Specific tools may override behavior of "Die" function to do tool-specific
314	336	// job.
...	...	@@ -325,12 +347,6 @@ void SetUserDieCallback(DieCallbackType callback);
325	347
326	348	void SetCheckUnwindCallback(void (*callback)());
327	349
328		// Callback will be called if soft_rss_limit_mb is given and the limit is
329		// exceeded (exceeded==true) or if rss went down below the limit
330		// (exceeded==false).
331		// The callback should be registered once at the tool init time.
332		void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded));
333
334	350	// Functions related to signal handling.
335	351	typedef void (SignalHandlerType)(int, void , void *);
336	352	HandleSignalMode GetHandleSignalMode(int signum);
...	...	@@ -371,7 +387,7 @@ void ReportErrorSummary(const char *error_type, const AddressInfo &info,
371	387	void ReportErrorSummary(const char error_type, const StackTrace trace,
372	388	const char *alt_tool_name = nullptr);
373	389
374		void ReportMmapWriteExec(int prot);
	390	void ReportMmapWriteExec(int prot, int mflags);
375	391
376	392	// Math
377	393	#if SANITIZER_WINDOWS && !defined(__clang__) && !defined(__GNUC__)
...	...	@@ -419,9 +435,7 @@ inline uptr LeastSignificantSetBitIndex(uptr x) {
419	435	return up;
420	436	}
421	437
422		inline bool IsPowerOfTwo(uptr x) {
423		return (x & (x - 1)) == 0;
424		}
	438	inline constexpr bool IsPowerOfTwo(uptr x) { return (x & (x - 1)) == 0; }
425	439
426	440	inline uptr RoundUpToPowerOfTwo(uptr size) {
427	441	CHECK(size);
...	...	@@ -433,16 +447,16 @@ inline uptr RoundUpToPowerOfTwo(uptr size) {
433	447	return 1ULL << (up + 1);
434	448	}
435	449
436		inline uptr RoundUpTo(uptr size, uptr boundary) {
	450	inline constexpr uptr RoundUpTo(uptr size, uptr boundary) {
437	451	RAW_CHECK(IsPowerOfTwo(boundary));
438	452	return (size + boundary - 1) & ~(boundary - 1);
439	453	}
440	454
441		inline uptr RoundDownTo(uptr x, uptr boundary) {
	455	inline constexpr uptr RoundDownTo(uptr x, uptr boundary) {
442	456	return x & ~(boundary - 1);
443	457	}
444	458
445		inline bool IsAligned(uptr a, uptr alignment) {
	459	inline constexpr bool IsAligned(uptr a, uptr alignment) {
446	460	return (a & (alignment - 1)) == 0;
447	461	}
448	462
...	...	@@ -461,6 +475,10 @@ template <class T>
461	475	constexpr T Max(T a, T b) {
462	476	return a > b ? a : b;
463	477	}
	478	template <class T>
	479	constexpr T Abs(T a) {
	480	return a < 0 ? -a : a;
	481	}
464	482	template<class T> void Swap(T& a, T& b) {
465	483	T tmp = a;
466	484	a = b;
...	...	@@ -502,8 +520,8 @@ class InternalMmapVectorNoCtor {
502	520	return data_[i];
503	521	}
504	522	void push_back(const T &element) {
505		CHECK_LE(size_, capacity());
506		if (size_ == capacity()) {
	523	if (UNLIKELY(size_ >= capacity())) {
	524	CHECK_EQ(size_, capacity());
507	525	uptr new_capacity = RoundUpToPowerOfTwo(size_ + 1);
508	526	Realloc(new_capacity);
509	527	}
...	...	@@ -563,7 +581,7 @@ class InternalMmapVectorNoCtor {
563	581	}
564	582
565	583	private:
566		void Realloc(uptr new_capacity) {
	584	NOINLINE void Realloc(uptr new_capacity) {
567	585	CHECK_GT(new_capacity, 0);
568	586	CHECK_LE(size_, new_capacity);
569	587	uptr new_capacity_bytes =
...	...	@@ -618,7 +636,7 @@ class InternalScopedString {
618	636	buffer_.resize(1);
619	637	buffer_[0] = '\0';
620	638	}
621		void append(const char *format, ...);
	639	void append(const char *format, ...) FORMAT(2, 3);
622	640	const char *data() const { return buffer_.data(); }
623	641	char *data() { return buffer_.data(); }
624	642
...	...	@@ -670,11 +688,9 @@ void Sort(T *v, uptr size, Compare comp = {}) {
670	688
671	689	// Works like std::lower_bound: finds the first element that is not less
672	690	// than the val.
673		template <class Container,
	691	template <class Container, class T,
674	692	class Compare = CompareLess<typename Container::value_type>>
675		uptr InternalLowerBound(const Container &v,
676		const typename Container::value_type &val,
677		Compare comp = {}) {
	693	uptr InternalLowerBound(const Container &v, const T &val, Compare comp = {}) {
678	694	uptr first = 0;
679	695	uptr last = v.size();
680	696	while (last > first) {
...	...	@@ -697,7 +713,9 @@ enum ModuleArch {
697	713	kModuleArchARMV7S,
698	714	kModuleArchARMV7K,
699	715	kModuleArchARM64,
700		kModuleArchRISCV64
	716	kModuleArchLoongArch64,
	717	kModuleArchRISCV64,
	718	kModuleArchHexagon
701	719	};
702	720
703	721	// Sorts and removes duplicates from the container.
...	...	@@ -721,12 +739,15 @@ void SortAndDedup(Container &v, Compare comp = {}) {
721	739	v.resize(last + 1);
722	740	}
723	741
	742	constexpr uptr kDefaultFileMaxSize = FIRST_32_SECOND_64(1 << 26, 1 << 28);
	743
724	744	// Opens the file 'file_name" and reads up to 'max_len' bytes.
725	745	// The resulting buffer is mmaped and stored in '*buff'.
726	746	// Returns true if file was successfully opened and read.
727	747	bool ReadFileToVector(const char *file_name,
728	748	InternalMmapVectorNoCtor<char> *buff,
729		uptr max_len = 1 << 26, error_t *errno_p = nullptr);
	749	uptr max_len = kDefaultFileMaxSize,
	750	error_t *errno_p = nullptr);
730	751
731	752	// Opens the file 'file_name" and reads up to 'max_len' bytes.
732	753	// This function is less I/O efficient than ReadFileToVector as it may reread
...	...	@@ -737,9 +758,12 @@ bool ReadFileToVector(const char *file_name,
737	758	// The total number of read bytes is stored in '*read_len'.
738	759	// Returns true if file was successfully opened and read.
739	760	bool ReadFileToBuffer(const char file_name, char buff, uptr buff_size,
740		uptr *read_len, uptr max_len = 1 << 26,
	761	uptr *read_len, uptr max_len = kDefaultFileMaxSize,
741	762	error_t *errno_p = nullptr);
742	763
	764	int GetModuleAndOffsetForPc(uptr pc, char *module_name, uptr module_name_len,
	765	uptr *pc_offset);
	766
743	767	// When adding a new architecture, don't forget to also update
744	768	// script/asan_symbolize.py and sanitizer_symbolizer_libcdep.cpp.
745	769	inline const char *ModuleArchToString(ModuleArch arch) {
...	...	@@ -762,14 +786,22 @@ inline const char *ModuleArchToString(ModuleArch arch) {
762	786	return "armv7k";
763	787	case kModuleArchARM64:
764	788	return "arm64";
	789	case kModuleArchLoongArch64:
	790	return "loongarch64";
765	791	case kModuleArchRISCV64:
766	792	return "riscv64";
	793	case kModuleArchHexagon:
	794	return "hexagon";
767	795	}
768	796	CHECK(0 && "Invalid module arch");
769	797	return "";
770	798	}
771	799
	800	#if SANITIZER_APPLE
772	801	const uptr kModuleUUIDSize = 16;
	802	#else
	803	const uptr kModuleUUIDSize = 32;
	804	#endif
773	805	const uptr kMaxSegName = 16;
774	806
775	807	// Represents a binary loaded into virtual memory (e.g. this can be an
...	...	@@ -779,8 +811,9 @@ class LoadedModule {
779	811	LoadedModule()
780	812	: full_name_(nullptr),
781	813	base_address_(0),
782		max_executable_address_(0),
	814	max_address_(0),
783	815	arch_(kModuleArchUnknown),
	816	uuid_size_(0),
784	817	instrumented_(false) {
785	818	internal_memset(uuid_, 0, kModuleUUIDSize);
786	819	ranges_.clear();
...	...	@@ -788,6 +821,7 @@ class LoadedModule {
788	821	void set(const char *module_name, uptr base_address);
789	822	void set(const char *module_name, uptr base_address, ModuleArch arch,
790	823	u8 uuid[kModuleUUIDSize], bool instrumented);
	824	void setUuid(const char *uuid, uptr size);
791	825	void clear();
792	826	void addAddressRange(uptr beg, uptr end, bool executable, bool writable,
793	827	const char *name = nullptr);
...	...	@@ -795,9 +829,10 @@ class LoadedModule {
795	829
796	830	const char *full_name() const { return full_name_; }
797	831	uptr base_address() const { return base_address_; }
798		uptr max_executable_address() const { return max_executable_address_; }
	832	uptr max_address() const { return max_address_; }
799	833	ModuleArch arch() const { return arch_; }
800	834	const u8 *uuid() const { return uuid_; }
	835	uptr uuid_size() const { return uuid_size_; }
801	836	bool instrumented() const { return instrumented_; }
802	837
803	838	struct AddressRange {
...	...	@@ -824,8 +859,9 @@ class LoadedModule {
824	859	private:
825	860	char *full_name_; // Owned.
826	861	uptr base_address_;
827		uptr max_executable_address_;
	862	uptr max_address_;
828	863	ModuleArch arch_;
	864	uptr uuid_size_;
829	865	u8 uuid_[kModuleUUIDSize];
830	866	bool instrumented_;
831	867	IntrusiveList<AddressRange> ranges_;
...	...	@@ -883,13 +919,13 @@ void WriteToSyslog(const char *buffer);
883	919	#define SANITIZER_WIN_TRACE 0
884	920	#endif
885	921
886		#if SANITIZER_MAC \|\| SANITIZER_WIN_TRACE
	922	#if SANITIZER_APPLE \|\| SANITIZER_WIN_TRACE
887	923	void LogFullErrorReport(const char *buffer);
888	924	#else
889	925	inline void LogFullErrorReport(const char *buffer) {}
890	926	#endif
891	927
892		#if SANITIZER_LINUX \|\| SANITIZER_MAC
	928	#if SANITIZER_LINUX \|\| SANITIZER_APPLE
893	929	void WriteOneLineToSyslog(const char *s);
894	930	void LogMessageOnPrintf(const char *str);
895	931	#else
...	...	@@ -951,7 +987,7 @@ struct SignalContext {
951	987	uptr sp;
952	988	uptr bp;
953	989	bool is_memory_access;
954		enum WriteFlag { UNKNOWN, READ, WRITE } write_flag;
	990	enum WriteFlag { Unknown, Read, Write } write_flag;
955	991
956	992	// In some cases the kernel cannot provide the true faulting address; `addr`
957	993	// will be zero then. This field allows to distinguish between these cases
...	...	@@ -996,7 +1032,6 @@ struct SignalContext {
996	1032	};
997	1033
998	1034	void InitializePlatformEarly();
999		void MaybeReexec();
1000	1035
1001	1036	template <typename Fn>
1002	1037	class RunOnDestruction {
...	...	@@ -1049,31 +1084,10 @@ inline u32 GetNumberOfCPUsCached() {
1049	1084	return NumberOfCPUsCached;
1050	1085	}
1051	1086
1052		template <typename T>
1053		class ArrayRef {
1054		public:
1055		ArrayRef() {}
1056		ArrayRef(T begin, T end) : begin_(begin), end_(end) {}
1057
1058		T *begin() { return begin_; }
1059		T *end() { return end_; }
1060
1061		private:
1062		T *begin_ = nullptr;
1063		T *end_ = nullptr;
1064		};
1065
1066		#define PRINTF_128(v) \
1067		(((u8 )&v + 0)), (((u8 )&v + 1)), (((u8 )&v + 2)), (((u8 )&v + 3)), \
1068		(((u8 )&v + 4)), (((u8 )&v + 5)), (((u8 )&v + 6)), \
1069		(((u8 )&v + 7)), (((u8 )&v + 8)), (((u8 )&v + 9)), \
1070		(((u8 )&v + 10)), (((u8 )&v + 11)), (((u8 )&v + 12)), \
1071		(((u8 )&v + 13)), (((u8 )&v + 14)), (((u8 )&v + 15))
1072
1073	1087	} // namespace __sanitizer
1074	1088
1075	1089	inline void *operator new(__sanitizer::operator_new_size_type size,
1076		__sanitizer::LowLevelAllocator &alloc) { // NOLINT
	1090	__sanitizer::LowLevelAllocator &alloc) {
1077	1091	return alloc.Allocate(size);
1078	1092	}
1079	1093

lib/tsan/sanitizer_common/sanitizer_common_interceptors.inc+878-736

...	...	@@ -21,19 +21,13 @@
21	21	// COMMON_INTERCEPTOR_FD_RELEASE
22	22	// COMMON_INTERCEPTOR_FD_ACCESS
23	23	// COMMON_INTERCEPTOR_SET_THREAD_NAME
24		// COMMON_INTERCEPTOR_ON_DLOPEN
	24	// COMMON_INTERCEPTOR_DLOPEN
25	25	// COMMON_INTERCEPTOR_ON_EXIT
26		// COMMON_INTERCEPTOR_MUTEX_PRE_LOCK
27		// COMMON_INTERCEPTOR_MUTEX_POST_LOCK
28		// COMMON_INTERCEPTOR_MUTEX_UNLOCK
29		// COMMON_INTERCEPTOR_MUTEX_REPAIR
30	26	// COMMON_INTERCEPTOR_SET_PTHREAD_NAME
31	27	// COMMON_INTERCEPTOR_HANDLE_RECVMSG
32	28	// COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED
33		// COMMON_INTERCEPTOR_MEMSET_IMPL
34		// COMMON_INTERCEPTOR_MEMMOVE_IMPL
35		// COMMON_INTERCEPTOR_MEMCPY_IMPL
36	29	// COMMON_INTERCEPTOR_MMAP_IMPL
	30	// COMMON_INTERCEPTOR_MUNMAP_IMPL
37	31	// COMMON_INTERCEPTOR_COPY_STRING
38	32	// COMMON_INTERCEPTOR_STRNDUP_IMPL
39	33	// COMMON_INTERCEPTOR_STRERROR
...	...	@@ -132,14 +126,75 @@ extern const short *_toupper_tab_;
132	126	extern const short *_tolower_tab_;
133	127	#endif
134	128
135		// Platform-specific options.
136		#if SANITIZER_MAC
137		#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
138		#elif SANITIZER_WINDOWS64
139		#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
140		#else
141		#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1
142		#endif // SANITIZER_MAC
	129	#if SANITIZER_MUSL && \
	130	(defined(__i386__) \|\| defined(__arm__) \|\| SANITIZER_MIPS32 \|\| SANITIZER_PPC32)
	131	// musl 1.2.0 on existing 32-bit architectures uses new symbol names for the
	132	// time-related functions that take 64-bit time_t values. See
	133	// https://musl.libc.org/time64.html
	134	#define adjtime __adjtime64
	135	#define adjtimex __adjtimex_time64
	136	#define aio_suspend __aio_suspend_time64
	137	#define clock_adjtime __clock_adjtime64
	138	#define clock_getres __clock_getres_time64
	139	#define clock_gettime __clock_gettime64
	140	#define clock_nanosleep __clock_nanosleep_time64
	141	#define clock_settime __clock_settime64
	142	#define cnd_timedwait __cnd_timedwait_time64
	143	#define ctime __ctime64
	144	#define ctime_r __ctime64_r
	145	#define difftime __difftime64
	146	#define dlsym __dlsym_time64
	147	#define fstatat __fstatat_time64
	148	#define fstat __fstat_time64
	149	#define ftime __ftime64
	150	#define futimens __futimens_time64
	151	#define futimesat __futimesat_time64
	152	#define futimes __futimes_time64
	153	#define getitimer __getitimer_time64
	154	#define getrusage __getrusage_time64
	155	#define gettimeofday __gettimeofday_time64
	156	#define gmtime __gmtime64
	157	#define gmtime_r __gmtime64_r
	158	#define localtime __localtime64
	159	#define localtime_r __localtime64_r
	160	#define lstat __lstat_time64
	161	#define lutimes __lutimes_time64
	162	#define mktime __mktime64
	163	#define mq_timedreceive __mq_timedreceive_time64
	164	#define mq_timedsend __mq_timedsend_time64
	165	#define mtx_timedlock __mtx_timedlock_time64
	166	#define nanosleep __nanosleep_time64
	167	#define ppoll __ppoll_time64
	168	#define pselect __pselect_time64
	169	#define pthread_cond_timedwait __pthread_cond_timedwait_time64
	170	#define pthread_mutex_timedlock __pthread_mutex_timedlock_time64
	171	#define pthread_rwlock_timedrdlock __pthread_rwlock_timedrdlock_time64
	172	#define pthread_rwlock_timedwrlock __pthread_rwlock_timedwrlock_time64
	173	#define pthread_timedjoin_np __pthread_timedjoin_np_time64
	174	#define recvmmsg __recvmmsg_time64
	175	#define sched_rr_get_interval __sched_rr_get_interval_time64
	176	#define select __select_time64
	177	#define semtimedop __semtimedop_time64
	178	#define sem_timedwait __sem_timedwait_time64
	179	#define setitimer __setitimer_time64
	180	#define settimeofday __settimeofday_time64
	181	#define sigtimedwait __sigtimedwait_time64
	182	#define stat __stat_time64
	183	#define stime __stime64
	184	#define thrd_sleep __thrd_sleep_time64
	185	#define timegm __timegm_time64
	186	#define timerfd_gettime __timerfd_gettime64
	187	#define timerfd_settime __timerfd_settime64
	188	#define timer_gettime __timer_gettime64
	189	#define timer_settime __timer_settime64
	190	#define timespec_get __timespec_get_time64
	191	#define time __time64
	192	#define utimensat __utimensat_time64
	193	#define utimes __utimes_time64
	194	#define utime __utime64
	195	#define wait3 __wait3_time64
	196	#define wait4 __wait4_time64
	197	#endif
143	198
144	199	#ifndef COMMON_INTERCEPTOR_INITIALIZE_RANGE
145	200	#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(p, size) {}
...	...	@@ -153,26 +208,6 @@ extern const short *_tolower_tab_;
153	208	#define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) {}
154	209	#endif
155	210
156		#ifndef COMMON_INTERCEPTOR_MUTEX_PRE_LOCK
157		#define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) {}
158		#endif
159
160		#ifndef COMMON_INTERCEPTOR_MUTEX_POST_LOCK
161		#define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) {}
162		#endif
163
164		#ifndef COMMON_INTERCEPTOR_MUTEX_UNLOCK
165		#define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) {}
166		#endif
167
168		#ifndef COMMON_INTERCEPTOR_MUTEX_REPAIR
169		#define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) {}
170		#endif
171
172		#ifndef COMMON_INTERCEPTOR_MUTEX_INVALID
173		#define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) {}
174		#endif
175
176	211	#ifndef COMMON_INTERCEPTOR_HANDLE_RECVMSG
177	212	#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) ((void)(msg))
178	213	#endif
...	...	@@ -204,11 +239,11 @@ extern const short *_tolower_tab_;
204	239
205	240	#define COMMON_INTERCEPTOR_READ_STRING(ctx, s, n) \
206	241	COMMON_INTERCEPTOR_READ_RANGE((ctx), (s), \
207		common_flags()->strict_string_checks ? (REAL(strlen)(s)) + 1 : (n) )
	242	common_flags()->strict_string_checks ? (internal_strlen(s)) + 1 : (n) )
208	243
209		#ifndef COMMON_INTERCEPTOR_ON_DLOPEN
210		#define COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag) \
211		CheckNoDeepBind(filename, flag);
	244	#ifndef COMMON_INTERCEPTOR_DLOPEN
	245	#define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \
	246	({ CheckNoDeepBind(filename, flag); REAL(dlopen)(filename, flag); })
212	247	#endif
213	248
214	249	#ifndef COMMON_INTERCEPTOR_GET_TLS_RANGE
...	...	@@ -256,53 +291,17 @@ extern const short *_tolower_tab_;
256	291	COMMON_INTERCEPT_FUNCTION(fn)
257	292	#endif
258	293
259		#ifndef COMMON_INTERCEPTOR_MEMSET_IMPL
260		#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \
261		{ \
262		if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \
263		return internal_memset(dst, v, size); \
264		COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \
265		if (common_flags()->intercept_intrin) \
266		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
267		return REAL(memset)(dst, v, size); \
268		}
269		#endif
270
271		#ifndef COMMON_INTERCEPTOR_MEMMOVE_IMPL
272		#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size) \
273		{ \
274		if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \
275		return internal_memmove(dst, src, size); \
276		COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size); \
277		if (common_flags()->intercept_intrin) { \
278		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
279		COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \
280		} \
281		return REAL(memmove)(dst, src, size); \
282		}
283		#endif
284
285		#ifndef COMMON_INTERCEPTOR_MEMCPY_IMPL
286		#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size) \
287		{ \
288		if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) { \
289		return internal_memmove(dst, src, size); \
290		} \
291		COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size); \
292		if (common_flags()->intercept_intrin) { \
293		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
294		COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \
295		} \
296		return REAL(memcpy)(dst, src, size); \
297		}
298		#endif
299
300	294	#ifndef COMMON_INTERCEPTOR_MMAP_IMPL
301	295	#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \
302	296	off) \
303	297	{ return REAL(mmap)(addr, sz, prot, flags, fd, off); }
304	298	#endif
305	299
	300	#ifndef COMMON_INTERCEPTOR_MUNMAP_IMPL
	301	#define COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz) \
	302	{ return REAL(munmap)(addr, sz); }
	303	#endif
	304
306	305	#ifndef COMMON_INTERCEPTOR_COPY_STRING
307	306	#define COMMON_INTERCEPTOR_COPY_STRING(ctx, to, from, size) {}
308	307	#endif
...	...	@@ -315,9 +314,11 @@ extern const short *_tolower_tab_;
315	314	if (common_flags()->intercept_strndup) { \
316	315	COMMON_INTERCEPTOR_READ_STRING(ctx, s, Min(size, copy_length + 1)); \
317	316	} \
318		COMMON_INTERCEPTOR_COPY_STRING(ctx, new_mem, s, copy_length); \
319		internal_memcpy(new_mem, s, copy_length); \
320		new_mem[copy_length] = '\0'; \
	317	if (new_mem) { \
	318	COMMON_INTERCEPTOR_COPY_STRING(ctx, new_mem, s, copy_length); \
	319	internal_memcpy(new_mem, s, copy_length); \
	320	new_mem[copy_length] = '\0'; \
	321	} \
321	322	return new_mem;
322	323	#endif
323	324
...	...	@@ -435,7 +436,7 @@ INTERCEPTOR(char, textdomain, const char domainname) {
435	436	if (domainname) COMMON_INTERCEPTOR_READ_STRING(ctx, domainname, 0);
436	437	char *domain = REAL(textdomain)(domainname);
437	438	if (domain) {
438		COMMON_INTERCEPTOR_INITIALIZE_RANGE(domain, REAL(strlen)(domain) + 1);
	439	COMMON_INTERCEPTOR_INITIALIZE_RANGE(domain, internal_strlen(domain) + 1);
439	440	}
440	441	return domain;
441	442	}
...	...	@@ -575,8 +576,8 @@ INTERCEPTOR(int, strncasecmp, const char s1, const char s2, SIZE_T size) {
575	576	#if SANITIZER_INTERCEPT_STRSTR \|\| SANITIZER_INTERCEPT_STRCASESTR
576	577	static inline void StrstrCheck(void ctx, char r, const char *s1,
577	578	const char *s2) {
578		uptr len1 = REAL(strlen)(s1);
579		uptr len2 = REAL(strlen)(s2);
	579	uptr len1 = internal_strlen(s1);
	580	uptr len2 = internal_strlen(s2);
580	581	COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r ? r - s1 + len2 : len1 + 1);
581	582	COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, len2 + 1);
582	583	}
...	...	@@ -640,10 +641,10 @@ INTERCEPTOR(char, strtok, char str, const char *delimiters) {
640	641	// for subsequent calls). We do not need to check strtok's result.
641	642	// As the delimiters can change, we check them every call.
642	643	if (str != nullptr) {
643		COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1);
	644	COMMON_INTERCEPTOR_READ_RANGE(ctx, str, internal_strlen(str) + 1);
644	645	}
645	646	COMMON_INTERCEPTOR_READ_RANGE(ctx, delimiters,
646		REAL(strlen)(delimiters) + 1);
	647	internal_strlen(delimiters) + 1);
647	648	return REAL(strtok)(str, delimiters);
648	649	} else {
649	650	// However, when strict_string_checks is disabled we cannot check the
...	...	@@ -657,11 +658,11 @@ INTERCEPTOR(char, strtok, char str, const char *delimiters) {
657	658	COMMON_INTERCEPTOR_READ_RANGE(ctx, delimiters, 1);
658	659	char *result = REAL(strtok)(str, delimiters);
659	660	if (result != nullptr) {
660		COMMON_INTERCEPTOR_READ_RANGE(ctx, result, REAL(strlen)(result) + 1);
	661	COMMON_INTERCEPTOR_READ_RANGE(ctx, result, internal_strlen(result) + 1);
661	662	} else if (str != nullptr) {
662	663	// No delimiter were found, it's safe to assume that the entire str was
663	664	// scanned.
664		COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1);
	665	COMMON_INTERCEPTOR_READ_RANGE(ctx, str, internal_strlen(str) + 1);
665	666	}
666	667	return result;
667	668	}
...	...	@@ -706,7 +707,7 @@ INTERCEPTOR(char, strchr, const char s, int c) {
706	707	if (common_flags()->intercept_strchr) {
707	708	// Keep strlen as macro argument, as macro may ignore it.
708	709	COMMON_INTERCEPTOR_READ_STRING(ctx, s,
709		(result ? result - s : REAL(strlen)(s)) + 1);
	710	(result ? result - s : internal_strlen(s)) + 1);
710	711	}
711	712	return result;
712	713	}
...	...	@@ -737,7 +738,7 @@ INTERCEPTOR(char, strrchr, const char s, int c) {
737	738	return internal_strrchr(s, c);
738	739	COMMON_INTERCEPTOR_ENTER(ctx, strrchr, s, c);
739	740	if (common_flags()->intercept_strchr)
740		COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);
	741	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
741	742	return REAL(strrchr)(s, c);
742	743	}
743	744	#define INIT_STRRCHR COMMON_INTERCEPT_FUNCTION(strrchr)
...	...	@@ -751,7 +752,7 @@ INTERCEPTOR(SIZE_T, strspn, const char s1, const char s2) {
751	752	COMMON_INTERCEPTOR_ENTER(ctx, strspn, s1, s2);
752	753	SIZE_T r = REAL(strspn)(s1, s2);
753	754	if (common_flags()->intercept_strspn) {
754		COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, REAL(strlen)(s2) + 1);
	755	COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, internal_strlen(s2) + 1);
755	756	COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r + 1);
756	757	}
757	758	return r;
...	...	@@ -762,7 +763,7 @@ INTERCEPTOR(SIZE_T, strcspn, const char s1, const char s2) {
762	763	COMMON_INTERCEPTOR_ENTER(ctx, strcspn, s1, s2);
763	764	SIZE_T r = REAL(strcspn)(s1, s2);
764	765	if (common_flags()->intercept_strspn) {
765		COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, REAL(strlen)(s2) + 1);
	766	COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, internal_strlen(s2) + 1);
766	767	COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r + 1);
767	768	}
768	769	return r;
...	...	@@ -781,9 +782,9 @@ INTERCEPTOR(char , strpbrk, const char s1, const char *s2) {
781	782	COMMON_INTERCEPTOR_ENTER(ctx, strpbrk, s1, s2);
782	783	char *r = REAL(strpbrk)(s1, s2);
783	784	if (common_flags()->intercept_strpbrk) {
784		COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, REAL(strlen)(s2) + 1);
	785	COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, internal_strlen(s2) + 1);
785	786	COMMON_INTERCEPTOR_READ_STRING(ctx, s1,
786		r ? r - s1 + 1 : REAL(strlen)(s1) + 1);
	787	r ? r - s1 + 1 : internal_strlen(s1) + 1);
787	788	}
788	789	return r;
789	790	}
...	...	@@ -793,57 +794,6 @@ INTERCEPTOR(char , strpbrk, const char s1, const char *s2) {
793	794	#define INIT_STRPBRK
794	795	#endif
795	796
796		#if SANITIZER_INTERCEPT_MEMSET
797		INTERCEPTOR(void , memset, void dst, int v, uptr size) {
798		void *ctx;
799		COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size);
800		}
801
802		#define INIT_MEMSET COMMON_INTERCEPT_FUNCTION(memset)
803		#else
804		#define INIT_MEMSET
805		#endif
806
807		#if SANITIZER_INTERCEPT_MEMMOVE
808		INTERCEPTOR(void , memmove, void dst, const void *src, uptr size) {
809		void *ctx;
810		COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
811		}
812
813		#define INIT_MEMMOVE COMMON_INTERCEPT_FUNCTION(memmove)
814		#else
815		#define INIT_MEMMOVE
816		#endif
817
818		#if SANITIZER_INTERCEPT_MEMCPY
819		INTERCEPTOR(void , memcpy, void dst, const void *src, uptr size) {
820		// On OS X, calling internal_memcpy here will cause memory corruptions,
821		// because memcpy and memmove are actually aliases of the same
822		// implementation. We need to use internal_memmove here.
823		// N.B.: If we switch this to internal_ we'll have to use internal_memmove
824		// due to memcpy being an alias of memmove on OS X.
825		void *ctx;
826		#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE
827		COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
828		#else
829		COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
830		#endif
831		}
832
833		#define INIT_MEMCPY \
834		do { \
835		if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { \
836		COMMON_INTERCEPT_FUNCTION(memcpy); \
837		} else { \
838		ASSIGN_REAL(memcpy, memmove); \
839		} \
840		CHECK(REAL(memcpy)); \
841		} while (false)
842
843		#else
844		#define INIT_MEMCPY
845		#endif
846
847	797	#if SANITIZER_INTERCEPT_MEMCMP
848	798	DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc,
849	799	const void s1, const void s2, uptr n,
...	...	@@ -1251,7 +1201,7 @@ INTERCEPTOR(char , fgets, char s, SIZE_T size, void *file) {
1251	1201	// https://github.com/google/sanitizers/issues/321.
1252	1202	char *res = REAL(fgets)(s, size, file);
1253	1203	if (res)
1254		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1);
	1204	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, internal_strlen(s) + 1);
1255	1205	return res;
1256	1206	}
1257	1207	#define INIT_FGETS COMMON_INTERCEPT_FUNCTION(fgets)
...	...	@@ -1264,8 +1214,8 @@ INTERCEPTOR_WITH_SUFFIX(int, fputs, char s, void file) {
1264	1214	// libc file streams can call user-supplied functions, see fopencookie.
1265	1215	void *ctx;
1266	1216	COMMON_INTERCEPTOR_ENTER(ctx, fputs, s, file);
1267		if (!SANITIZER_MAC \|\| s) { // `fputs(NULL, file)` is supported on Darwin.
1268		COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);
	1217	if (!SANITIZER_APPLE \|\| s) { // `fputs(NULL, file)` is supported on Darwin.
	1218	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
1269	1219	}
1270	1220	return REAL(fputs)(s, file);
1271	1221	}
...	...	@@ -1279,8 +1229,8 @@ INTERCEPTOR(int, puts, char *s) {
1279	1229	// libc file streams can call user-supplied functions, see fopencookie.
1280	1230	void *ctx;
1281	1231	COMMON_INTERCEPTOR_ENTER(ctx, puts, s);
1282		if (!SANITIZER_MAC \|\| s) { // `puts(NULL)` is supported on Darwin.
1283		COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);
	1232	if (!SANITIZER_APPLE \|\| s) { // `puts(NULL)` is supported on Darwin.
	1233	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
1284	1234	}
1285	1235	return REAL(puts)(s);
1286	1236	}
...	...	@@ -1295,12 +1245,21 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
1295	1245	void *ctx;
1296	1246	COMMON_INTERCEPTOR_ENTER(ctx, prctl, option, arg2, arg3, arg4, arg5);
1297	1247	static const int PR_SET_NAME = 15;
1298		int res = REAL(prctl(option, arg2, arg3, arg4, arg5));
	1248	static const int PR_SET_VMA = 0x53564d41;
	1249	static const int PR_SCHED_CORE = 62;
	1250	static const int PR_SCHED_CORE_GET = 0;
	1251	if (option == PR_SET_VMA && arg2 == 0UL) {
	1252	char name = (char )arg5;
	1253	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
	1254	}
	1255	int res = REAL(prctl)(option, arg2, arg3, arg4, arg5);
1299	1256	if (option == PR_SET_NAME) {
1300	1257	char buff[16];
1301	1258	internal_strncpy(buff, (char *)arg2, 15);
1302	1259	buff[15] = 0;
1303	1260	COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, buff);
	1261	} else if (res != -1 && option == PR_SCHED_CORE && arg2 == PR_SCHED_CORE_GET) {
	1262	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64*)(arg5), sizeof(u64));
1304	1263	}
1305	1264	return res;
1306	1265	}
...	...	@@ -1334,7 +1293,7 @@ static void unpoison_tm(void ctx, __sanitizer_tm tm) {
1334	1293	// Can not use COMMON_INTERCEPTOR_WRITE_RANGE here, because tm->tm_zone
1335	1294	// can point to shared memory and tsan would report a data race.
1336	1295	COMMON_INTERCEPTOR_INITIALIZE_RANGE(tm->tm_zone,
1337		REAL(strlen(tm->tm_zone)) + 1);
	1296	internal_strlen(tm->tm_zone) + 1);
1338	1297	}
1339	1298	#endif
1340	1299	}
...	...	@@ -1387,7 +1346,7 @@ INTERCEPTOR(char , ctime, unsigned long timep) {
1387	1346	char *res = REAL(ctime)(timep);
1388	1347	if (res) {
1389	1348	COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep));
1390		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
	1349	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
1391	1350	}
1392	1351	return res;
1393	1352	}
...	...	@@ -1400,7 +1359,7 @@ INTERCEPTOR(char , ctime_r, unsigned long timep, char *result) {
1400	1359	char *res = REAL(ctime_r)(timep, result);
1401	1360	if (res) {
1402	1361	COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep));
1403		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
	1362	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
1404	1363	}
1405	1364	return res;
1406	1365	}
...	...	@@ -1413,7 +1372,7 @@ INTERCEPTOR(char , asctime, __sanitizer_tm tm) {
1413	1372	char *res = REAL(asctime)(tm);
1414	1373	if (res) {
1415	1374	COMMON_INTERCEPTOR_READ_RANGE(ctx, tm, sizeof(*tm));
1416		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
	1375	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
1417	1376	}
1418	1377	return res;
1419	1378	}
...	...	@@ -1426,7 +1385,7 @@ INTERCEPTOR(char , asctime_r, __sanitizer_tm tm, char *result) {
1426	1385	char *res = REAL(asctime_r)(tm, result);
1427	1386	if (res) {
1428	1387	COMMON_INTERCEPTOR_READ_RANGE(ctx, tm, sizeof(*tm));
1429		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
	1388	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
1430	1389	}
1431	1390	return res;
1432	1391	}
...	...	@@ -1463,7 +1422,7 @@ INTERCEPTOR(char , strptime, char s, char format, __sanitizer_tm tm) {
1463	1422	void *ctx;
1464	1423	COMMON_INTERCEPTOR_ENTER(ctx, strptime, s, format, tm);
1465	1424	if (format)
1466		COMMON_INTERCEPTOR_READ_RANGE(ctx, format, REAL(strlen)(format) + 1);
	1425	COMMON_INTERCEPTOR_READ_RANGE(ctx, format, internal_strlen(format) + 1);
1467	1426	// FIXME: under ASan the call below may write to freed memory and corrupt
1468	1427	// its metadata. See
1469	1428	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -1532,6 +1491,16 @@ VSCANF_INTERCEPTOR_IMPL(__isoc99_vsscanf, false, str, format, ap)
1532	1491
1533	1492	INTERCEPTOR(int, __isoc99_vfscanf, void stream, const char format, va_list ap)
1534	1493	VSCANF_INTERCEPTOR_IMPL(__isoc99_vfscanf, false, stream, format, ap)
	1494
	1495	INTERCEPTOR(int, __isoc23_vscanf, const char *format, va_list ap)
	1496	VSCANF_INTERCEPTOR_IMPL(__isoc23_vscanf, false, format, ap)
	1497
	1498	INTERCEPTOR(int, __isoc23_vsscanf, const char str, const char format,
	1499	va_list ap)
	1500	VSCANF_INTERCEPTOR_IMPL(__isoc23_vsscanf, false, str, format, ap)
	1501
	1502	INTERCEPTOR(int, __isoc23_vfscanf, void stream, const char format, va_list ap)
	1503	VSCANF_INTERCEPTOR_IMPL(__isoc23_vfscanf, false, stream, format, ap)
1535	1504	#endif // SANITIZER_INTERCEPT_ISOC99_SCANF
1536	1505
1537	1506	INTERCEPTOR(int, scanf, const char *format, ...)
...	...	@@ -1552,6 +1521,15 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_fscanf, __isoc99_vfscanf, stream, format)
1552	1521
1553	1522	INTERCEPTOR(int, __isoc99_sscanf, const char str, const char format, ...)
1554	1523	FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format)
	1524
	1525	INTERCEPTOR(int, __isoc23_scanf, const char *format, ...)
	1526	FORMAT_INTERCEPTOR_IMPL(__isoc23_scanf, __isoc23_vscanf, format)
	1527
	1528	INTERCEPTOR(int, __isoc23_fscanf, void stream, const char format, ...)
	1529	FORMAT_INTERCEPTOR_IMPL(__isoc23_fscanf, __isoc23_vfscanf, stream, format)
	1530
	1531	INTERCEPTOR(int, __isoc23_sscanf, const char str, const char format, ...)
	1532	FORMAT_INTERCEPTOR_IMPL(__isoc23_sscanf, __isoc23_vsscanf, str, format)
1555	1533	#endif
1556	1534
1557	1535	#endif
...	...	@@ -1575,7 +1553,13 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format)
1575	1553	COMMON_INTERCEPT_FUNCTION(__isoc99_fscanf); \
1576	1554	COMMON_INTERCEPT_FUNCTION(__isoc99_vscanf); \
1577	1555	COMMON_INTERCEPT_FUNCTION(__isoc99_vsscanf); \
1578		COMMON_INTERCEPT_FUNCTION(__isoc99_vfscanf);
	1556	COMMON_INTERCEPT_FUNCTION(__isoc99_vfscanf); \
	1557	COMMON_INTERCEPT_FUNCTION(__isoc23_scanf); \
	1558	COMMON_INTERCEPT_FUNCTION(__isoc23_sscanf); \
	1559	COMMON_INTERCEPT_FUNCTION(__isoc23_fscanf); \
	1560	COMMON_INTERCEPT_FUNCTION(__isoc23_vscanf); \
	1561	COMMON_INTERCEPT_FUNCTION(__isoc23_vsscanf); \
	1562	COMMON_INTERCEPT_FUNCTION(__isoc23_vfscanf);
1579	1563	#else
1580	1564	#define INIT_ISOC99_SCANF
1581	1565	#endif
...	...	@@ -1843,9 +1827,9 @@ INTERCEPTOR(int, ioctl, int d, unsigned long request, ...) {
1843	1827	const ioctl_desc *desc = ioctl_lookup(request);
1844	1828	ioctl_desc decoded_desc;
1845	1829	if (!desc) {
1846		VPrintf(2, "Decoding unknown ioctl 0x%x\n", request);
	1830	VPrintf(2, "Decoding unknown ioctl 0x%lx\n", request);
1847	1831	if (!ioctl_decode(request, &decoded_desc))
1848		Printf("WARNING: failed decoding unknown ioctl 0x%x\n", request);
	1832	Printf("WARNING: failed decoding unknown ioctl 0x%lx\n", request);
1849	1833	else
1850	1834	desc = &decoded_desc;
1851	1835	}
...	...	@@ -1869,26 +1853,26 @@ UNUSED static void unpoison_passwd(void ctx, __sanitizer_passwd pwd) {
1869	1853	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd, sizeof(*pwd));
1870	1854	if (pwd->pw_name)
1871	1855	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_name,
1872		REAL(strlen)(pwd->pw_name) + 1);
	1856	internal_strlen(pwd->pw_name) + 1);
1873	1857	if (pwd->pw_passwd)
1874	1858	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_passwd,
1875		REAL(strlen)(pwd->pw_passwd) + 1);
	1859	internal_strlen(pwd->pw_passwd) + 1);
1876	1860	#if !SANITIZER_ANDROID
1877	1861	if (pwd->pw_gecos)
1878	1862	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_gecos,
1879		REAL(strlen)(pwd->pw_gecos) + 1);
	1863	internal_strlen(pwd->pw_gecos) + 1);
1880	1864	#endif
1881		#if SANITIZER_MAC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
	1865	#if SANITIZER_APPLE \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
1882	1866	if (pwd->pw_class)
1883	1867	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_class,
1884		REAL(strlen)(pwd->pw_class) + 1);
	1868	internal_strlen(pwd->pw_class) + 1);
1885	1869	#endif
1886	1870	if (pwd->pw_dir)
1887	1871	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_dir,
1888		REAL(strlen)(pwd->pw_dir) + 1);
	1872	internal_strlen(pwd->pw_dir) + 1);
1889	1873	if (pwd->pw_shell)
1890	1874	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_shell,
1891		REAL(strlen)(pwd->pw_shell) + 1);
	1875	internal_strlen(pwd->pw_shell) + 1);
1892	1876	}
1893	1877	}
1894	1878
...	...	@@ -1897,13 +1881,13 @@ UNUSED static void unpoison_group(void ctx, __sanitizer_group grp) {
1897	1881	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp, sizeof(*grp));
1898	1882	if (grp->gr_name)
1899	1883	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_name,
1900		REAL(strlen)(grp->gr_name) + 1);
	1884	internal_strlen(grp->gr_name) + 1);
1901	1885	if (grp->gr_passwd)
1902	1886	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_passwd,
1903		REAL(strlen)(grp->gr_passwd) + 1);
	1887	internal_strlen(grp->gr_passwd) + 1);
1904	1888	char **p = grp->gr_mem;
1905	1889	for (; *p; ++p) {
1906		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, REAL(strlen)(p) + 1);
	1890	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, internal_strlen(p) + 1);
1907	1891	}
1908	1892	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_mem,
1909	1893	(p - grp->gr_mem + 1) * sizeof(*p));
...	...	@@ -1916,7 +1900,7 @@ INTERCEPTOR(__sanitizer_passwd , getpwnam, const char name) {
1916	1900	void *ctx;
1917	1901	COMMON_INTERCEPTOR_ENTER(ctx, getpwnam, name);
1918	1902	if (name)
1919		COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
	1903	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
1920	1904	__sanitizer_passwd *res = REAL(getpwnam)(name);
1921	1905	unpoison_passwd(ctx, res);
1922	1906	return res;
...	...	@@ -1931,7 +1915,7 @@ INTERCEPTOR(__sanitizer_passwd *, getpwuid, u32 uid) {
1931	1915	INTERCEPTOR(__sanitizer_group , getgrnam, const char name) {
1932	1916	void *ctx;
1933	1917	COMMON_INTERCEPTOR_ENTER(ctx, getgrnam, name);
1934		COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
	1918	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
1935	1919	__sanitizer_group *res = REAL(getgrnam)(name);
1936	1920	unpoison_group(ctx, res);
1937	1921	return res;
...	...	@@ -1957,7 +1941,7 @@ INTERCEPTOR(int, getpwnam_r, const char name, __sanitizer_passwd pwd,
1957	1941	char buf, SIZE_T buflen, __sanitizer_passwd *result) {
1958	1942	void *ctx;
1959	1943	COMMON_INTERCEPTOR_ENTER(ctx, getpwnam_r, name, pwd, buf, buflen, result);
1960		COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
	1944	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
1961	1945	// FIXME: under ASan the call below may write to freed memory and corrupt
1962	1946	// its metadata. See
1963	1947	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -1984,7 +1968,7 @@ INTERCEPTOR(int, getgrnam_r, const char name, __sanitizer_group grp,
1984	1968	char buf, SIZE_T buflen, __sanitizer_group *result) {
1985	1969	void *ctx;
1986	1970	COMMON_INTERCEPTOR_ENTER(ctx, getgrnam_r, name, grp, buf, buflen, result);
1987		COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
	1971	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
1988	1972	// FIXME: under ASan the call below may write to freed memory and corrupt
1989	1973	// its metadata. See
1990	1974	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -2229,8 +2213,20 @@ INTERCEPTOR(int, clock_getcpuclockid, pid_t pid,
2229	2213	return res;
2230	2214	}
2231	2215
2232		#define INIT_CLOCK_GETCPUCLOCKID \
2233		COMMON_INTERCEPT_FUNCTION(clock_getcpuclockid);
	2216	INTERCEPTOR(int, pthread_getcpuclockid, uptr thread,
	2217	__sanitizer_clockid_t *clockid) {
	2218	void *ctx;
	2219	COMMON_INTERCEPTOR_ENTER(ctx, pthread_getcpuclockid, thread, clockid);
	2220	int res = REAL(pthread_getcpuclockid)(thread, clockid);
	2221	if (!res && clockid) {
	2222	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, clockid, sizeof *clockid);
	2223	}
	2224	return res;
	2225	}
	2226
	2227	#define INIT_CLOCK_GETCPUCLOCKID \
	2228	COMMON_INTERCEPT_FUNCTION(clock_getcpuclockid); \
	2229	COMMON_INTERCEPT_FUNCTION(pthread_getcpuclockid);
2234	2230	#else
2235	2231	#define INIT_CLOCK_GETCPUCLOCKID
2236	2232	#endif
...	...	@@ -2289,7 +2285,7 @@ static void unpoison_glob_t(void ctx, __sanitizer_glob_t pglob) {
2289	2285	ctx, pglob->gl_pathv, (pglob->gl_pathc + 1) * sizeof(*pglob->gl_pathv));
2290	2286	for (SIZE_T i = 0; i < pglob->gl_pathc; ++i) {
2291	2287	char *p = pglob->gl_pathv[i];
2292		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, REAL(strlen)(p) + 1);
	2288	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, internal_strlen(p) + 1);
2293	2289	}
2294	2290	}
2295	2291
...	...	@@ -2319,19 +2315,19 @@ static void wrapped_gl_readdir(void dir) {
2319	2315
2320	2316	static void wrapped_gl_opendir(const char s) {
2321	2317	COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
2322		COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, REAL(strlen)(s) + 1);
	2318	COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, internal_strlen(s) + 1);
2323	2319	return pglob_copy->gl_opendir(s);
2324	2320	}
2325	2321
2326	2322	static int wrapped_gl_lstat(const char s, void st) {
2327	2323	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
2328		COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, REAL(strlen)(s) + 1);
	2324	COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, internal_strlen(s) + 1);
2329	2325	return pglob_copy->gl_lstat(s, st);
2330	2326	}
2331	2327
2332	2328	static int wrapped_gl_stat(const char s, void st) {
2333	2329	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
2334		COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, REAL(strlen)(s) + 1);
	2330	COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, internal_strlen(s) + 1);
2335	2331	return pglob_copy->gl_stat(s, st);
2336	2332	}
2337	2333
...	...	@@ -2410,6 +2406,136 @@ INTERCEPTOR(int, glob64, const char *pattern, int flags,
2410	2406	#define INIT_GLOB64
2411	2407	#endif // SANITIZER_INTERCEPT_GLOB64
2412	2408
	2409	#if SANITIZER_INTERCEPT___B64_TO
	2410	INTERCEPTOR(int, __b64_ntop, unsigned char const *src, SIZE_T srclength,
	2411	char *target, SIZE_T targsize) {
	2412	void *ctx;
	2413	COMMON_INTERCEPTOR_ENTER(ctx, __b64_ntop, src, srclength, target, targsize);
	2414	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, srclength);
	2415	int res = REAL(__b64_ntop)(src, srclength, target, targsize);
	2416	if (res >= 0)
	2417	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, target, res + 1);
	2418	return res;
	2419	}
	2420	INTERCEPTOR(int, __b64_pton, char const src, char target, SIZE_T targsize) {
	2421	void *ctx;
	2422	COMMON_INTERCEPTOR_ENTER(ctx, __b64_pton, src, target, targsize);
	2423	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
	2424	int res = REAL(__b64_pton)(src, target, targsize);
	2425	if (res >= 0)
	2426	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, target, res);
	2427	return res;
	2428	}
	2429	#define INIT___B64_TO \
	2430	COMMON_INTERCEPT_FUNCTION(__b64_ntop); \
	2431	COMMON_INTERCEPT_FUNCTION(__b64_pton);
	2432	#else // SANITIZER_INTERCEPT___B64_TO
	2433	#define INIT___B64_TO
	2434	#endif // SANITIZER_INTERCEPT___B64_TO
	2435
	2436	#if SANITIZER_INTERCEPT_DN_COMP_EXPAND
	2437	# if __GLIBC_PREREQ(2, 34)
	2438	// Changed with https://sourceware.org/git/?p=glibc.git;h=640bbdf
	2439	# define DN_COMP_INTERCEPTOR_NAME dn_comp
	2440	# define DN_EXPAND_INTERCEPTOR_NAME dn_expand
	2441	# else
	2442	# define DN_COMP_INTERCEPTOR_NAME __dn_comp
	2443	# define DN_EXPAND_INTERCEPTOR_NAME __dn_expand
	2444	# endif
	2445	INTERCEPTOR(int, DN_COMP_INTERCEPTOR_NAME, unsigned char *exp_dn,
	2446	unsigned char comp_dn, int length, unsigned char *dnptrs,
	2447	unsigned char **lastdnptr) {
	2448	void *ctx;
	2449	COMMON_INTERCEPTOR_ENTER(ctx, DN_COMP_INTERCEPTOR_NAME, exp_dn, comp_dn,
	2450	length, dnptrs, lastdnptr);
	2451	int res = REAL(DN_COMP_INTERCEPTOR_NAME)(exp_dn, comp_dn, length, dnptrs,
	2452	lastdnptr);
	2453	if (res >= 0) {
	2454	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, comp_dn, res);
	2455	if (dnptrs && lastdnptr) {
	2456	unsigned char **p = dnptrs;
	2457	for (; p != lastdnptr && *p; ++p)
	2458	;
	2459	if (p != lastdnptr)
	2460	++p;
	2461	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dnptrs, (p - dnptrs) * sizeof(*p));
	2462	}
	2463	}
	2464	return res;
	2465	}
	2466	INTERCEPTOR(int, DN_EXPAND_INTERCEPTOR_NAME, unsigned char const *base,
	2467	unsigned char const end, unsigned char const src, char *dest,
	2468	int space) {
	2469	void *ctx;
	2470	COMMON_INTERCEPTOR_ENTER(ctx, DN_EXPAND_INTERCEPTOR_NAME, base, end, src,
	2471	dest, space);
	2472	// TODO: add read check if __dn_comp intercept added
	2473	int res = REAL(DN_EXPAND_INTERCEPTOR_NAME)(base, end, src, dest, space);
	2474	if (res >= 0)
	2475	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, internal_strlen(dest) + 1);
	2476	return res;
	2477	}
	2478	# define INIT_DN_COMP_EXPAND \
	2479	COMMON_INTERCEPT_FUNCTION(DN_COMP_INTERCEPTOR_NAME); \
	2480	COMMON_INTERCEPT_FUNCTION(DN_EXPAND_INTERCEPTOR_NAME);
	2481	#else // SANITIZER_INTERCEPT_DN_COMP_EXPAND
	2482	# define INIT_DN_COMP_EXPAND
	2483	#endif // SANITIZER_INTERCEPT_DN_COMP_EXPAND
	2484
	2485	#if SANITIZER_INTERCEPT_POSIX_SPAWN
	2486
	2487	template <class RealSpawnPtr>
	2488	static int PosixSpawnImpl(void ctx, RealSpawnPtr real_posix_spawn, pid_t *pid,
	2489	const char file_or_path, const void file_actions,
	2490	const void attrp, char const argv[],
	2491	char *const envp[]) {
	2492	COMMON_INTERCEPTOR_READ_RANGE(ctx, file_or_path,
	2493	internal_strlen(file_or_path) + 1);
	2494	if (argv) {
	2495	for (char const s = argv; ; ++s) {
	2496	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(*s));
	2497	if (!*s) break;
	2498	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
	2499	}
	2500	}
	2501	if (envp) {
	2502	for (char const s = envp; ; ++s) {
	2503	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(*s));
	2504	if (!*s) break;
	2505	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
	2506	}
	2507	}
	2508	int res =
	2509	real_posix_spawn(pid, file_or_path, file_actions, attrp, argv, envp);
	2510	if (res == 0)
	2511	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pid, sizeof(*pid));
	2512	return res;
	2513	}
	2514	INTERCEPTOR(int, posix_spawn, pid_t pid, const char path,
	2515	const void file_actions, const void attrp, char *const argv[],
	2516	char *const envp[]) {
	2517	void *ctx;
	2518	COMMON_INTERCEPTOR_ENTER(ctx, posix_spawn, pid, path, file_actions, attrp,
	2519	argv, envp);
	2520	return PosixSpawnImpl(ctx, REAL(posix_spawn), pid, path, file_actions, attrp,
	2521	argv, envp);
	2522	}
	2523	INTERCEPTOR(int, posix_spawnp, pid_t pid, const char file,
	2524	const void file_actions, const void attrp, char *const argv[],
	2525	char *const envp[]) {
	2526	void *ctx;
	2527	COMMON_INTERCEPTOR_ENTER(ctx, posix_spawnp, pid, file, file_actions, attrp,
	2528	argv, envp);
	2529	return PosixSpawnImpl(ctx, REAL(posix_spawnp), pid, file, file_actions, attrp,
	2530	argv, envp);
	2531	}
	2532	# define INIT_POSIX_SPAWN \
	2533	COMMON_INTERCEPT_FUNCTION(posix_spawn); \
	2534	COMMON_INTERCEPT_FUNCTION(posix_spawnp);
	2535	#else // SANITIZER_INTERCEPT_POSIX_SPAWN
	2536	# define INIT_POSIX_SPAWN
	2537	#endif // SANITIZER_INTERCEPT_POSIX_SPAWN
	2538
2413	2539	#if SANITIZER_INTERCEPT_WAIT
2414	2540	// According to sys/wait.h, wait(), waitid(), waitpid() may have symbol version
2415	2541	// suffixes on Darwin. See the declaration of INTERCEPTOR_WITH_SUFFIX for
...	...	@@ -2519,7 +2645,7 @@ INTERCEPTOR(char , inet_ntop, int af, const void src, char *dst, u32 size) {
2519	2645	// its metadata. See
2520	2646	// https://github.com/google/sanitizers/issues/321.
2521	2647	char *res = REAL(inet_ntop)(af, src, dst, size);
2522		if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
	2648	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
2523	2649	return res;
2524	2650	}
2525	2651	INTERCEPTOR(int, inet_pton, int af, const char src, void dst) {
...	...	@@ -2548,7 +2674,7 @@ INTERCEPTOR(int, inet_pton, int af, const char src, void dst) {
2548	2674	INTERCEPTOR(int, inet_aton, const char cp, void dst) {
2549	2675	void *ctx;
2550	2676	COMMON_INTERCEPTOR_ENTER(ctx, inet_aton, cp, dst);
2551		if (cp) COMMON_INTERCEPTOR_READ_RANGE(ctx, cp, REAL(strlen)(cp) + 1);
	2677	if (cp) COMMON_INTERCEPTOR_READ_RANGE(ctx, cp, internal_strlen(cp) + 1);
2552	2678	// FIXME: under ASan the call below may write to freed memory and corrupt
2553	2679	// its metadata. See
2554	2680	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -2590,9 +2716,9 @@ INTERCEPTOR(int, getaddrinfo, char node, char service,
2590	2716	struct __sanitizer_addrinfo **out) {
2591	2717	void *ctx;
2592	2718	COMMON_INTERCEPTOR_ENTER(ctx, getaddrinfo, node, service, hints, out);
2593		if (node) COMMON_INTERCEPTOR_READ_RANGE(ctx, node, REAL(strlen)(node) + 1);
	2719	if (node) COMMON_INTERCEPTOR_READ_RANGE(ctx, node, internal_strlen(node) + 1);
2594	2720	if (service)
2595		COMMON_INTERCEPTOR_READ_RANGE(ctx, service, REAL(strlen)(service) + 1);
	2721	COMMON_INTERCEPTOR_READ_RANGE(ctx, service, internal_strlen(service) + 1);
2596	2722	if (hints)
2597	2723	COMMON_INTERCEPTOR_READ_RANGE(ctx, hints, sizeof(__sanitizer_addrinfo));
2598	2724	// FIXME: under ASan the call below may write to freed memory and corrupt
...	...	@@ -2608,7 +2734,7 @@ INTERCEPTOR(int, getaddrinfo, char node, char service,
2608	2734	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ai_addr, p->ai_addrlen);
2609	2735	if (p->ai_canonname)
2610	2736	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ai_canonname,
2611		REAL(strlen)(p->ai_canonname) + 1);
	2737	internal_strlen(p->ai_canonname) + 1);
2612	2738	p = p->ai_next;
2613	2739	}
2614	2740	}
...	...	@@ -2634,9 +2760,9 @@ INTERCEPTOR(int, getnameinfo, void sockaddr, unsigned salen, char host,
2634	2760	REAL(getnameinfo)(sockaddr, salen, host, hostlen, serv, servlen, flags);
2635	2761	if (res == 0) {
2636	2762	if (host && hostlen)
2637		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, host, REAL(strlen)(host) + 1);
	2763	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, host, internal_strlen(host) + 1);
2638	2764	if (serv && servlen)
2639		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, serv, REAL(strlen)(serv) + 1);
	2765	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, serv, internal_strlen(serv) + 1);
2640	2766	}
2641	2767	return res;
2642	2768	}
...	...	@@ -2646,17 +2772,20 @@ INTERCEPTOR(int, getnameinfo, void sockaddr, unsigned salen, char host,
2646	2772	#endif
2647	2773
2648	2774	#if SANITIZER_INTERCEPT_GETSOCKNAME
2649		INTERCEPTOR(int, getsockname, int sock_fd, void addr, int addrlen) {
	2775	INTERCEPTOR(int, getsockname, int sock_fd, void addr, unsigned addrlen) {
2650	2776	void *ctx;
2651	2777	COMMON_INTERCEPTOR_ENTER(ctx, getsockname, sock_fd, addr, addrlen);
2652		COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
2653		int addrlen_in = *addrlen;
	2778	unsigned addr_sz;
	2779	if (addrlen) {
	2780	COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
	2781	addr_sz = *addrlen;
	2782	}
2654	2783	// FIXME: under ASan the call below may write to freed memory and corrupt
2655	2784	// its metadata. See
2656	2785	// https://github.com/google/sanitizers/issues/321.
2657	2786	int res = REAL(getsockname)(sock_fd, addr, addrlen);
2658		if (res == 0) {
2659		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addrlen_in, *addrlen));
	2787	if (!res && addr && addrlen) {
	2788	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addr_sz, *addrlen));
2660	2789	}
2661	2790	return res;
2662	2791	}
...	...	@@ -2669,10 +2798,10 @@ INTERCEPTOR(int, getsockname, int sock_fd, void addr, int addrlen) {
2669	2798	static void write_hostent(void ctx, struct __sanitizer_hostent h) {
2670	2799	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h, sizeof(__sanitizer_hostent));
2671	2800	if (h->h_name)
2672		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h->h_name, REAL(strlen)(h->h_name) + 1);
	2801	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h->h_name, internal_strlen(h->h_name) + 1);
2673	2802	char **p = h->h_aliases;
2674	2803	while (*p) {
2675		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, REAL(strlen)(p) + 1);
	2804	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, internal_strlen(p) + 1);
2676	2805	++p;
2677	2806	}
2678	2807	COMMON_INTERCEPTOR_WRITE_RANGE(
...	...	@@ -3161,13 +3290,17 @@ INTERCEPTOR(int, getpeername, int sockfd, void addr, unsigned addrlen) {
3161	3290	void *ctx;
3162	3291	COMMON_INTERCEPTOR_ENTER(ctx, getpeername, sockfd, addr, addrlen);
3163	3292	unsigned addr_sz;
3164		if (addrlen) addr_sz = *addrlen;
	3293	if (addrlen) {
	3294	COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
	3295	addr_sz = *addrlen;
	3296	}
3165	3297	// FIXME: under ASan the call below may write to freed memory and corrupt
3166	3298	// its metadata. See
3167	3299	// https://github.com/google/sanitizers/issues/321.
3168	3300	int res = REAL(getpeername)(sockfd, addr, addrlen);
3169		if (!res && addr && addrlen)
	3301	if (!res && addr && addrlen) {
3170	3302	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addr_sz, *addrlen));
	3303	}
3171	3304	return res;
3172	3305	}
3173	3306	#define INIT_GETPEERNAME COMMON_INTERCEPT_FUNCTION(getpeername);
...	...	@@ -3196,7 +3329,7 @@ INTERCEPTOR(int, sysinfo, void *info) {
3196	3329	INTERCEPTOR(__sanitizer_dirent , opendir, const char path) {
3197	3330	void *ctx;
3198	3331	COMMON_INTERCEPTOR_ENTER(ctx, opendir, path);
3199		COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	3332	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
3200	3333	__sanitizer_dirent *res = REAL(opendir)(path);
3201	3334	if (res)
3202	3335	COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path);
...	...	@@ -3210,7 +3343,8 @@ INTERCEPTOR(__sanitizer_dirent , readdir, void dirp) {
3210	3343	// its metadata. See
3211	3344	// https://github.com/google/sanitizers/issues/321.
3212	3345	__sanitizer_dirent *res = REAL(readdir)(dirp);
3213		if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen);
	3346	if (res)
	3347	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer_dirsiz(res));
3214	3348	return res;
3215	3349	}
3216	3350
...	...	@@ -3225,7 +3359,7 @@ INTERCEPTOR(int, readdir_r, void dirp, __sanitizer_dirent entry,
3225	3359	if (!res) {
3226	3360	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
3227	3361	if (*result)
3228		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, (result)->d_reclen);
	3362	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, __sanitizer_dirsiz(result));
3229	3363	}
3230	3364	return res;
3231	3365	}
...	...	@@ -3246,7 +3380,8 @@ INTERCEPTOR(__sanitizer_dirent64 , readdir64, void dirp) {
3246	3380	// its metadata. See
3247	3381	// https://github.com/google/sanitizers/issues/321.
3248	3382	__sanitizer_dirent64 *res = REAL(readdir64)(dirp);
3249		if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen);
	3383	if (res)
	3384	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer_dirsiz(res));
3250	3385	return res;
3251	3386	}
3252	3387
...	...	@@ -3261,7 +3396,7 @@ INTERCEPTOR(int, readdir64_r, void dirp, __sanitizer_dirent64 entry,
3261	3396	if (!res) {
3262	3397	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
3263	3398	if (*result)
3264		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, (result)->d_reclen);
	3399	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, __sanitizer_dirsiz(result));
3265	3400	}
3266	3401	return res;
3267	3402	}
...	...	@@ -3351,10 +3486,10 @@ INTERCEPTOR(char , setlocale, int category, char locale) {
3351	3486	void *ctx;
3352	3487	COMMON_INTERCEPTOR_ENTER(ctx, setlocale, category, locale);
3353	3488	if (locale)
3354		COMMON_INTERCEPTOR_READ_RANGE(ctx, locale, REAL(strlen)(locale) + 1);
	3489	COMMON_INTERCEPTOR_READ_RANGE(ctx, locale, internal_strlen(locale) + 1);
3355	3490	char *res = REAL(setlocale)(category, locale);
3356	3491	if (res) {
3357		COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
	3492	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
3358	3493	unpoison_ctype_arrays(ctx);
3359	3494	}
3360	3495	return res;
...	...	@@ -3373,7 +3508,7 @@ INTERCEPTOR(char , getcwd, char buf, SIZE_T size) {
3373	3508	// its metadata. See
3374	3509	// https://github.com/google/sanitizers/issues/321.
3375	3510	char *res = REAL(getcwd)(buf, size);
3376		if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
	3511	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
3377	3512	return res;
3378	3513	}
3379	3514	#define INIT_GETCWD COMMON_INTERCEPT_FUNCTION(getcwd);
...	...	@@ -3389,7 +3524,7 @@ INTERCEPTOR(char *, get_current_dir_name, int fake) {
3389	3524	// its metadata. See
3390	3525	// https://github.com/google/sanitizers/issues/321.
3391	3526	char *res = REAL(get_current_dir_name)(fake);
3392		if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
	3527	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
3393	3528	return res;
3394	3529	}
3395	3530
...	...	@@ -3429,30 +3564,26 @@ UNUSED static inline void StrtolFixAndCheck(void ctx, const char nptr,
3429	3564	(real_endptr - nptr) + 1 : 0);
3430	3565	}
3431	3566
3432
3433	3567	#if SANITIZER_INTERCEPT_STRTOIMAX
3434		INTERCEPTOR(INTMAX_T, strtoimax, const char nptr, char *endptr, int base) {
3435		void *ctx;
3436		COMMON_INTERCEPTOR_ENTER(ctx, strtoimax, nptr, endptr, base);
3437		// FIXME: under ASan the call below may write to freed memory and corrupt
3438		// its metadata. See
3439		// https://github.com/google/sanitizers/issues/321.
	3568	template <typename Fn>
	3569	static ALWAYS_INLINE auto StrtoimaxImpl(void ctx, Fn real, const char nptr,
	3570	char **endptr, int base)
	3571	-> decltype(real(nullptr, nullptr, 0)) {
3440	3572	char *real_endptr;
3441		INTMAX_T res = REAL(strtoimax)(nptr, &real_endptr, base);
	3573	auto res = real(nptr, &real_endptr, base);
3442	3574	StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base);
3443	3575	return res;
3444	3576	}
3445	3577
	3578	INTERCEPTOR(INTMAX_T, strtoimax, const char nptr, char *endptr, int base) {
	3579	void *ctx;
	3580	COMMON_INTERCEPTOR_ENTER(ctx, strtoimax, nptr, endptr, base);
	3581	return StrtoimaxImpl(ctx, REAL(strtoimax), nptr, endptr, base);
	3582	}
3446	3583	INTERCEPTOR(UINTMAX_T, strtoumax, const char nptr, char *endptr, int base) {
3447	3584	void *ctx;
3448	3585	COMMON_INTERCEPTOR_ENTER(ctx, strtoumax, nptr, endptr, base);
3449		// FIXME: under ASan the call below may write to freed memory and corrupt
3450		// its metadata. See
3451		// https://github.com/google/sanitizers/issues/321.
3452		char *real_endptr;
3453		UINTMAX_T res = REAL(strtoumax)(nptr, &real_endptr, base);
3454		StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base);
3455		return res;
	3586	return StrtoimaxImpl(ctx, REAL(strtoumax), nptr, endptr, base);
3456	3587	}
3457	3588
3458	3589	#define INIT_STRTOIMAX \
...	...	@@ -3462,6 +3593,25 @@ INTERCEPTOR(UINTMAX_T, strtoumax, const char nptr, char *endptr, int base) {
3462	3593	#define INIT_STRTOIMAX
3463	3594	#endif
3464	3595
	3596	#if SANITIZER_INTERCEPT_STRTOIMAX && SANITIZER_GLIBC
	3597	INTERCEPTOR(INTMAX_T, __isoc23_strtoimax, const char nptr, char *endptr, int base) {
	3598	void *ctx;
	3599	COMMON_INTERCEPTOR_ENTER(ctx, __isoc23_strtoimax, nptr, endptr, base);
	3600	return StrtoimaxImpl(ctx, REAL(__isoc23_strtoimax), nptr, endptr, base);
	3601	}
	3602	INTERCEPTOR(UINTMAX_T, __isoc23_strtoumax, const char nptr, char *endptr, int base) {
	3603	void *ctx;
	3604	COMMON_INTERCEPTOR_ENTER(ctx, __isoc23_strtoumax, nptr, endptr, base);
	3605	return StrtoimaxImpl(ctx, REAL(__isoc23_strtoumax), nptr, endptr, base);
	3606	}
	3607
	3608	# define INIT_STRTOIMAX_C23 \
	3609	COMMON_INTERCEPT_FUNCTION(__isoc23_strtoimax); \
	3610	COMMON_INTERCEPT_FUNCTION(__isoc23_strtoumax);
	3611	#else
	3612	# define INIT_STRTOIMAX_C23
	3613	#endif
	3614
3465	3615	#if SANITIZER_INTERCEPT_MBSTOWCS
3466	3616	INTERCEPTOR(SIZE_T, mbstowcs, wchar_t dest, const char src, SIZE_T len) {
3467	3617	void *ctx;
...	...	@@ -3663,7 +3813,7 @@ INTERCEPTOR(int, tcgetattr, int fd, void *termios_p) {
3663	3813	INTERCEPTOR(char , realpath, const char path, char *resolved_path) {
3664	3814	void *ctx;
3665	3815	COMMON_INTERCEPTOR_ENTER(ctx, realpath, path, resolved_path);
3666		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	3816	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
3667	3817
3668	3818	// Workaround a bug in glibc where dlsym(RTLD_NEXT, ...) returns the oldest
3669	3819	// version of a versioned symbol. For realpath(), this gives us something
...	...	@@ -3674,11 +3824,12 @@ INTERCEPTOR(char , realpath, const char path, char *resolved_path) {
3674	3824	allocated_path = resolved_path = (char *)WRAP(malloc)(path_max + 1);
3675	3825
3676	3826	char *res = REAL(realpath)(path, resolved_path);
3677		if (allocated_path && !res) WRAP(free)(allocated_path);
3678		if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
	3827	if (allocated_path && !res)
	3828	WRAP(free)(allocated_path);
	3829	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
3679	3830	return res;
3680	3831	}
3681		#define INIT_REALPATH COMMON_INTERCEPT_FUNCTION(realpath);
	3832	# define INIT_REALPATH COMMON_INTERCEPT_FUNCTION(realpath);
3682	3833	#else
3683	3834	#define INIT_REALPATH
3684	3835	#endif
...	...	@@ -3687,9 +3838,9 @@ INTERCEPTOR(char , realpath, const char path, char *resolved_path) {
3687	3838	INTERCEPTOR(char , canonicalize_file_name, const char path) {
3688	3839	void *ctx;
3689	3840	COMMON_INTERCEPTOR_ENTER(ctx, canonicalize_file_name, path);
3690		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	3841	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
3691	3842	char *res = REAL(canonicalize_file_name)(path);
3692		if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
	3843	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
3693	3844	return res;
3694	3845	}
3695	3846	#define INIT_CANONICALIZE_FILE_NAME \
...	...	@@ -3750,7 +3901,7 @@ INTERCEPTOR(char *, strerror, int errnum) {
3750	3901	COMMON_INTERCEPTOR_ENTER(ctx, strerror, errnum);
3751	3902	COMMON_INTERCEPTOR_STRERROR();
3752	3903	char *res = REAL(strerror)(errnum);
3753		if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
	3904	if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
3754	3905	return res;
3755	3906	}
3756	3907	#define INIT_STRERROR COMMON_INTERCEPT_FUNCTION(strerror);
...	...	@@ -3765,7 +3916,7 @@ INTERCEPTOR(char *, strerror, int errnum) {
3765	3916	// * GNU version returns message pointer, which points to either buf or some
3766	3917	// static storage.
3767	3918	#if ((_POSIX_C_SOURCE >= 200112L \|\| _XOPEN_SOURCE >= 600) && !_GNU_SOURCE) \|\| \
3768		SANITIZER_MAC \|\| SANITIZER_ANDROID \|\| SANITIZER_NETBSD \|\| \
	3919	SANITIZER_APPLE \|\| SANITIZER_ANDROID \|\| SANITIZER_NETBSD \|\| \
3769	3920	SANITIZER_FREEBSD
3770	3921	// POSIX version. Spec is not clear on whether buf is NULL-terminated.
3771	3922	// At least on OSX, buf contents are valid even when the call fails.
...	...	@@ -3792,13 +3943,13 @@ INTERCEPTOR(char , strerror_r, int errnum, char buf, SIZE_T buflen) {
3792	3943	// https://github.com/google/sanitizers/issues/321.
3793	3944	char *res = REAL(strerror_r)(errnum, buf, buflen);
3794	3945	if (res == buf)
3795		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
	3946	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
3796	3947	else
3797		COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
	3948	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
3798	3949	return res;
3799	3950	}
3800	3951	#endif //(_POSIX_C_SOURCE >= 200112L \|\| _XOPEN_SOURCE >= 600) && !_GNU_SOURCE \|\|
3801		//SANITIZER_MAC
	3952	//SANITIZER_APPLE
3802	3953	#define INIT_STRERROR_R COMMON_INTERCEPT_FUNCTION(strerror_r);
3803	3954	#else
3804	3955	#define INIT_STRERROR_R
...	...	@@ -3814,7 +3965,7 @@ INTERCEPTOR(int, __xpg_strerror_r, int errnum, char *buf, SIZE_T buflen) {
3814	3965	int res = REAL(__xpg_strerror_r)(errnum, buf, buflen);
3815	3966	// This version always returns a null-terminated string.
3816	3967	if (buf && buflen)
3817		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
	3968	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
3818	3969	return res;
3819	3970	}
3820	3971	#define INIT_XPG_STRERROR_R COMMON_INTERCEPT_FUNCTION(__xpg_strerror_r);
...	...	@@ -3832,7 +3983,7 @@ static THREADLOCAL scandir_compar_f scandir_compar;
3832	3983
3833	3984	static int wrapped_scandir_filter(const struct __sanitizer_dirent *dir) {
3834	3985	COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
3835		COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, dir->d_reclen);
	3986	COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, __sanitizer_dirsiz(dir));
3836	3987	return scandir_filter(dir);
3837	3988	}
3838	3989
...	...	@@ -3840,9 +3991,9 @@ static int wrapped_scandir_compar(const struct __sanitizer_dirent **a,
3840	3991	const struct __sanitizer_dirent **b) {
3841	3992	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
3842	3993	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, sizeof(*a));
3843		COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, (a)->d_reclen);
	3994	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, __sanitizer_dirsiz(a));
3844	3995	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, sizeof(*b));
3845		COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, (b)->d_reclen);
	3996	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, __sanitizer_dirsiz(b));
3846	3997	return scandir_compar(a, b);
3847	3998	}
3848	3999
...	...	@@ -3850,7 +4001,7 @@ INTERCEPTOR(int, scandir, char dirp, __sanitizer_dirent **namelist,
3850	4001	scandir_filter_f filter, scandir_compar_f compar) {
3851	4002	void *ctx;
3852	4003	COMMON_INTERCEPTOR_ENTER(ctx, scandir, dirp, namelist, filter, compar);
3853		if (dirp) COMMON_INTERCEPTOR_READ_RANGE(ctx, dirp, REAL(strlen)(dirp) + 1);
	4004	if (dirp) COMMON_INTERCEPTOR_READ_RANGE(ctx, dirp, internal_strlen(dirp) + 1);
3854	4005	scandir_filter = filter;
3855	4006	scandir_compar = compar;
3856	4007	// FIXME: under ASan the call below may write to freed memory and corrupt
...	...	@@ -3866,7 +4017,7 @@ INTERCEPTOR(int, scandir, char dirp, __sanitizer_dirent **namelist,
3866	4017	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelist, sizeof(namelist) res);
3867	4018	for (int i = 0; i < res; ++i)
3868	4019	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (*namelist)[i],
3869		(*namelist)[i]->d_reclen);
	4020	__sanitizer_dirsiz((*namelist)[i]));
3870	4021	}
3871	4022	return res;
3872	4023	}
...	...	@@ -3885,7 +4036,7 @@ static THREADLOCAL scandir64_compar_f scandir64_compar;
3885	4036
3886	4037	static int wrapped_scandir64_filter(const struct __sanitizer_dirent64 *dir) {
3887	4038	COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
3888		COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, dir->d_reclen);
	4039	COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, __sanitizer_dirsiz(dir));
3889	4040	return scandir64_filter(dir);
3890	4041	}
3891	4042
...	...	@@ -3893,9 +4044,9 @@ static int wrapped_scandir64_compar(const struct __sanitizer_dirent64 **a,
3893	4044	const struct __sanitizer_dirent64 **b) {
3894	4045	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
3895	4046	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, sizeof(*a));
3896		COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, (a)->d_reclen);
	4047	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, __sanitizer_dirsiz(a));
3897	4048	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, sizeof(*b));
3898		COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, (b)->d_reclen);
	4049	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, __sanitizer_dirsiz(b));
3899	4050	return scandir64_compar(a, b);
3900	4051	}
3901	4052
...	...	@@ -3903,7 +4054,7 @@ INTERCEPTOR(int, scandir64, char dirp, __sanitizer_dirent64 **namelist,
3903	4054	scandir64_filter_f filter, scandir64_compar_f compar) {
3904	4055	void *ctx;
3905	4056	COMMON_INTERCEPTOR_ENTER(ctx, scandir64, dirp, namelist, filter, compar);
3906		if (dirp) COMMON_INTERCEPTOR_READ_RANGE(ctx, dirp, REAL(strlen)(dirp) + 1);
	4057	if (dirp) COMMON_INTERCEPTOR_READ_RANGE(ctx, dirp, internal_strlen(dirp) + 1);
3907	4058	scandir64_filter = filter;
3908	4059	scandir64_compar = compar;
3909	4060	// FIXME: under ASan the call below may write to freed memory and corrupt
...	...	@@ -3920,7 +4071,7 @@ INTERCEPTOR(int, scandir64, char dirp, __sanitizer_dirent64 **namelist,
3920	4071	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelist, sizeof(namelist) res);
3921	4072	for (int i = 0; i < res; ++i)
3922	4073	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (*namelist)[i],
3923		(*namelist)[i]->d_reclen);
	4074	__sanitizer_dirsiz((*namelist)[i]));
3924	4075	}
3925	4076	return res;
3926	4077	}
...	...	@@ -3999,19 +4150,20 @@ INTERCEPTOR(int, ppoll, __sanitizer_pollfd *fds, __sanitizer_nfds_t nfds,
3999	4150	INTERCEPTOR(int, wordexp, char s, __sanitizer_wordexp_t p, int flags) {
4000	4151	void *ctx;
4001	4152	COMMON_INTERCEPTOR_ENTER(ctx, wordexp, s, p, flags);
4002		if (s) COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);
	4153	if (s) COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
4003	4154	// FIXME: under ASan the call below may write to freed memory and corrupt
4004	4155	// its metadata. See
4005	4156	// https://github.com/google/sanitizers/issues/321.
4006	4157	int res = REAL(wordexp)(s, p, flags);
4007	4158	if (!res && p) {
4008	4159	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
4009		if (p->we_wordc)
4010		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->we_wordv,
4011		sizeof(p->we_wordv) p->we_wordc);
4012		for (uptr i = 0; i < p->we_wordc; ++i) {
	4160	uptr we_wordc =
	4161	((flags & wordexp_wrde_dooffs) ? p->we_offs : 0) + p->we_wordc;
	4162	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->we_wordv,
	4163	sizeof(p->we_wordv) (we_wordc + 1));
	4164	for (uptr i = 0; i < we_wordc; ++i) {
4013	4165	char *w = p->we_wordv[i];
4014		if (w) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, w, REAL(strlen)(w) + 1);
	4166	if (w) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, w, internal_strlen(w) + 1);
4015	4167	}
4016	4168	}
4017	4169	return res;
...	...	@@ -4196,12 +4348,16 @@ INTERCEPTOR(int, pthread_sigmask, int how, __sanitizer_sigset_t *set,
4196	4348	INTERCEPTOR(int, backtrace, void **buffer, int size) {
4197	4349	void *ctx;
4198	4350	COMMON_INTERCEPTOR_ENTER(ctx, backtrace, buffer, size);
4199		// FIXME: under ASan the call below may write to freed memory and corrupt
4200		// its metadata. See
4201		// https://github.com/google/sanitizers/issues/321.
4202		int res = REAL(backtrace)(buffer, size);
4203		if (res && buffer)
	4351	// 'buffer' might be freed memory, hence it is unsafe to directly call
	4352	// REAL(backtrace)(buffer, size). Instead, we use our own known-good
	4353	// scratch buffer.
	4354	void scratch = (void)InternalAlloc(sizeof(void) size);
	4355	int res = REAL(backtrace)(scratch, size);
	4356	if (res && buffer) {
4204	4357	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buffer, res * sizeof(*buffer));
	4358	internal_memcpy(buffer, scratch, res * sizeof(*buffer));
	4359	}
	4360	InternalFree(scratch);
4205	4361	return res;
4206	4362	}
4207	4363
...	...	@@ -4210,14 +4366,13 @@ INTERCEPTOR(char , backtrace_symbols, void buffer, int size) {
4210	4366	COMMON_INTERCEPTOR_ENTER(ctx, backtrace_symbols, buffer, size);
4211	4367	if (buffer && size)
4212	4368	COMMON_INTERCEPTOR_READ_RANGE(ctx, buffer, size * sizeof(*buffer));
4213		// FIXME: under ASan the call below may write to freed memory and corrupt
4214		// its metadata. See
4215		// https://github.com/google/sanitizers/issues/321.
	4369	// The COMMON_INTERCEPTOR_READ_RANGE above ensures that 'buffer' is
	4370	// valid for reading.
4216	4371	char **res = REAL(backtrace_symbols)(buffer, size);
4217	4372	if (res && size) {
4218	4373	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, size * sizeof(*res));
4219	4374	for (int i = 0; i < size; ++i)
4220		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res[i], REAL(strlen(res[i])) + 1);
	4375	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res[i], internal_strlen(res[i]) + 1);
4221	4376	}
4222	4377	return res;
4223	4378	}
...	...	@@ -4243,90 +4398,13 @@ INTERCEPTOR(void, _exit, int status) {
4243	4398	#define INIT__EXIT
4244	4399	#endif
4245	4400
4246		#if SANITIZER_INTERCEPT_PTHREAD_MUTEX
4247		INTERCEPTOR(int, pthread_mutex_lock, void *m) {
4248		void *ctx;
4249		COMMON_INTERCEPTOR_ENTER(ctx, pthread_mutex_lock, m);
4250		COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m);
4251		int res = REAL(pthread_mutex_lock)(m);
4252		if (res == errno_EOWNERDEAD)
4253		COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m);
4254		if (res == 0 \|\| res == errno_EOWNERDEAD)
4255		COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m);
4256		if (res == errno_EINVAL)
4257		COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
4258		return res;
4259		}
4260
4261		INTERCEPTOR(int, pthread_mutex_unlock, void *m) {
4262		void *ctx;
4263		COMMON_INTERCEPTOR_ENTER(ctx, pthread_mutex_unlock, m);
4264		COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m);
4265		int res = REAL(pthread_mutex_unlock)(m);
4266		if (res == errno_EINVAL)
4267		COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
4268		return res;
4269		}
4270
4271		#define INIT_PTHREAD_MUTEX_LOCK COMMON_INTERCEPT_FUNCTION(pthread_mutex_lock)
4272		#define INIT_PTHREAD_MUTEX_UNLOCK \
4273		COMMON_INTERCEPT_FUNCTION(pthread_mutex_unlock)
4274		#else
4275		#define INIT_PTHREAD_MUTEX_LOCK
4276		#define INIT_PTHREAD_MUTEX_UNLOCK
4277		#endif
4278
4279		#if SANITIZER_INTERCEPT___PTHREAD_MUTEX
4280		INTERCEPTOR(int, __pthread_mutex_lock, void *m) {
4281		void *ctx;
4282		COMMON_INTERCEPTOR_ENTER(ctx, __pthread_mutex_lock, m);
4283		COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m);
4284		int res = REAL(__pthread_mutex_lock)(m);
4285		if (res == errno_EOWNERDEAD)
4286		COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m);
4287		if (res == 0 \|\| res == errno_EOWNERDEAD)
4288		COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m);
4289		if (res == errno_EINVAL)
4290		COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
4291		return res;
4292		}
4293
4294		INTERCEPTOR(int, __pthread_mutex_unlock, void *m) {
4295		void *ctx;
4296		COMMON_INTERCEPTOR_ENTER(ctx, __pthread_mutex_unlock, m);
4297		COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m);
4298		int res = REAL(__pthread_mutex_unlock)(m);
4299		if (res == errno_EINVAL)
4300		COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
4301		return res;
4302		}
4303
4304		#define INIT___PTHREAD_MUTEX_LOCK \
4305		COMMON_INTERCEPT_FUNCTION(__pthread_mutex_lock)
4306		#define INIT___PTHREAD_MUTEX_UNLOCK \
4307		COMMON_INTERCEPT_FUNCTION(__pthread_mutex_unlock)
4308		#else
4309		#define INIT___PTHREAD_MUTEX_LOCK
4310		#define INIT___PTHREAD_MUTEX_UNLOCK
4311		#endif
4312
4313	4401	#if SANITIZER_INTERCEPT___LIBC_MUTEX
4314		INTERCEPTOR(int, __libc_mutex_lock, void *m)
4315		ALIAS(WRAPPER_NAME(pthread_mutex_lock));
4316
4317		INTERCEPTOR(int, __libc_mutex_unlock, void *m)
4318		ALIAS(WRAPPER_NAME(pthread_mutex_unlock));
4319
4320	4402	INTERCEPTOR(int, __libc_thr_setcancelstate, int state, int *oldstate)
4321		ALIAS(WRAPPER_NAME(pthread_setcancelstate));
	4403	ALIAS(WRAP(pthread_setcancelstate));
4322	4404
4323		#define INIT___LIBC_MUTEX_LOCK COMMON_INTERCEPT_FUNCTION(__libc_mutex_lock)
4324		#define INIT___LIBC_MUTEX_UNLOCK COMMON_INTERCEPT_FUNCTION(__libc_mutex_unlock)
4325	4405	#define INIT___LIBC_THR_SETCANCELSTATE \
4326	4406	COMMON_INTERCEPT_FUNCTION(__libc_thr_setcancelstate)
4327	4407	#else
4328		#define INIT___LIBC_MUTEX_LOCK
4329		#define INIT___LIBC_MUTEX_UNLOCK
4330	4408	#define INIT___LIBC_THR_SETCANCELSTATE
4331	4409	#endif
4332	4410
...	...	@@ -4335,16 +4413,16 @@ static void write_mntent(void ctx, __sanitizer_mntent mnt) {
4335	4413	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt, sizeof(*mnt));
4336	4414	if (mnt->mnt_fsname)
4337	4415	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_fsname,
4338		REAL(strlen)(mnt->mnt_fsname) + 1);
	4416	internal_strlen(mnt->mnt_fsname) + 1);
4339	4417	if (mnt->mnt_dir)
4340	4418	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_dir,
4341		REAL(strlen)(mnt->mnt_dir) + 1);
	4419	internal_strlen(mnt->mnt_dir) + 1);
4342	4420	if (mnt->mnt_type)
4343	4421	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_type,
4344		REAL(strlen)(mnt->mnt_type) + 1);
	4422	internal_strlen(mnt->mnt_type) + 1);
4345	4423	if (mnt->mnt_opts)
4346	4424	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_opts,
4347		REAL(strlen)(mnt->mnt_opts) + 1);
	4425	internal_strlen(mnt->mnt_opts) + 1);
4348	4426	}
4349	4427	#endif
4350	4428
...	...	@@ -4379,7 +4457,7 @@ INTERCEPTOR(__sanitizer_mntent , getmntent_r, void fp,
4379	4457	INTERCEPTOR(int, statfs, char path, void buf) {
4380	4458	void *ctx;
4381	4459	COMMON_INTERCEPTOR_ENTER(ctx, statfs, path, buf);
4382		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	4460	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
4383	4461	// FIXME: under ASan the call below may write to freed memory and corrupt
4384	4462	// its metadata. See
4385	4463	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -4408,7 +4486,7 @@ INTERCEPTOR(int, fstatfs, int fd, void *buf) {
4408	4486	INTERCEPTOR(int, statfs64, char path, void buf) {
4409	4487	void *ctx;
4410	4488	COMMON_INTERCEPTOR_ENTER(ctx, statfs64, path, buf);
4411		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	4489	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
4412	4490	// FIXME: under ASan the call below may write to freed memory and corrupt
4413	4491	// its metadata. See
4414	4492	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -4437,7 +4515,7 @@ INTERCEPTOR(int, fstatfs64, int fd, void *buf) {
4437	4515	INTERCEPTOR(int, statvfs, char path, void buf) {
4438	4516	void *ctx;
4439	4517	COMMON_INTERCEPTOR_ENTER(ctx, statvfs, path, buf);
4440		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	4518	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
4441	4519	// FIXME: under ASan the call below may write to freed memory and corrupt
4442	4520	// its metadata. See
4443	4521	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -4471,7 +4549,7 @@ INTERCEPTOR(int, fstatvfs, int fd, void *buf) {
4471	4549	INTERCEPTOR(int, statvfs64, char path, void buf) {
4472	4550	void *ctx;
4473	4551	COMMON_INTERCEPTOR_ENTER(ctx, statvfs64, path, buf);
4474		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	4552	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
4475	4553	// FIXME: under ASan the call below may write to freed memory and corrupt
4476	4554	// its metadata. See
4477	4555	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -4500,7 +4578,7 @@ INTERCEPTOR(int, fstatvfs64, int fd, void *buf) {
4500	4578	INTERCEPTOR(int, initgroups, char *user, u32 group) {
4501	4579	void *ctx;
4502	4580	COMMON_INTERCEPTOR_ENTER(ctx, initgroups, user, group);
4503		if (user) COMMON_INTERCEPTOR_READ_RANGE(ctx, user, REAL(strlen)(user) + 1);
	4581	if (user) COMMON_INTERCEPTOR_READ_RANGE(ctx, user, internal_strlen(user) + 1);
4504	4582	int res = REAL(initgroups)(user, group);
4505	4583	return res;
4506	4584	}
...	...	@@ -4515,13 +4593,13 @@ INTERCEPTOR(char , ether_ntoa, __sanitizer_ether_addr addr) {
4515	4593	COMMON_INTERCEPTOR_ENTER(ctx, ether_ntoa, addr);
4516	4594	if (addr) COMMON_INTERCEPTOR_READ_RANGE(ctx, addr, sizeof(*addr));
4517	4595	char *res = REAL(ether_ntoa)(addr);
4518		if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
	4596	if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
4519	4597	return res;
4520	4598	}
4521	4599	INTERCEPTOR(__sanitizer_ether_addr , ether_aton, char buf) {
4522	4600	void *ctx;
4523	4601	COMMON_INTERCEPTOR_ENTER(ctx, ether_aton, buf);
4524		if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
	4602	if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, internal_strlen(buf) + 1);
4525	4603	__sanitizer_ether_addr *res = REAL(ether_aton)(buf);
4526	4604	if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, sizeof(*res));
4527	4605	return res;
...	...	@@ -4543,14 +4621,14 @@ INTERCEPTOR(int, ether_ntohost, char hostname, __sanitizer_ether_addr addr) {
4543	4621	// https://github.com/google/sanitizers/issues/321.
4544	4622	int res = REAL(ether_ntohost)(hostname, addr);
4545	4623	if (!res && hostname)
4546		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1);
	4624	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, internal_strlen(hostname) + 1);
4547	4625	return res;
4548	4626	}
4549	4627	INTERCEPTOR(int, ether_hostton, char hostname, __sanitizer_ether_addr addr) {
4550	4628	void *ctx;
4551	4629	COMMON_INTERCEPTOR_ENTER(ctx, ether_hostton, hostname, addr);
4552	4630	if (hostname)
4553		COMMON_INTERCEPTOR_READ_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1);
	4631	COMMON_INTERCEPTOR_READ_RANGE(ctx, hostname, internal_strlen(hostname) + 1);
4554	4632	// FIXME: under ASan the call below may write to freed memory and corrupt
4555	4633	// its metadata. See
4556	4634	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -4562,7 +4640,7 @@ INTERCEPTOR(int, ether_line, char line, __sanitizer_ether_addr addr,
4562	4640	char *hostname) {
4563	4641	void *ctx;
4564	4642	COMMON_INTERCEPTOR_ENTER(ctx, ether_line, line, addr, hostname);
4565		if (line) COMMON_INTERCEPTOR_READ_RANGE(ctx, line, REAL(strlen)(line) + 1);
	4643	if (line) COMMON_INTERCEPTOR_READ_RANGE(ctx, line, internal_strlen(line) + 1);
4566	4644	// FIXME: under ASan the call below may write to freed memory and corrupt
4567	4645	// its metadata. See
4568	4646	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -4570,7 +4648,7 @@ INTERCEPTOR(int, ether_line, char line, __sanitizer_ether_addr addr,
4570	4648	if (!res) {
4571	4649	if (addr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, sizeof(*addr));
4572	4650	if (hostname)
4573		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1);
	4651	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, internal_strlen(hostname) + 1);
4574	4652	}
4575	4653	return res;
4576	4654	}
...	...	@@ -4591,14 +4669,14 @@ INTERCEPTOR(char , ether_ntoa_r, __sanitizer_ether_addr addr, char *buf) {
4591	4669	// its metadata. See
4592	4670	// https://github.com/google/sanitizers/issues/321.
4593	4671	char *res = REAL(ether_ntoa_r)(addr, buf);
4594		if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
	4672	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
4595	4673	return res;
4596	4674	}
4597	4675	INTERCEPTOR(__sanitizer_ether_addr , ether_aton_r, char buf,
4598	4676	__sanitizer_ether_addr *addr) {
4599	4677	void *ctx;
4600	4678	COMMON_INTERCEPTOR_ENTER(ctx, ether_aton_r, buf, addr);
4601		if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
	4679	if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, internal_strlen(buf) + 1);
4602	4680	// FIXME: under ASan the call below may write to freed memory and corrupt
4603	4681	// its metadata. See
4604	4682	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -4766,6 +4844,27 @@ INTERCEPTOR(int, pthread_attr_getaffinity_np, void *attr, SIZE_T cpusetsize,
4766	4844	#define INIT_PTHREAD_ATTR_GETAFFINITY_NP
4767	4845	#endif
4768	4846
	4847	#if SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP
	4848	INTERCEPTOR(int, pthread_getaffinity_np, void *attr, SIZE_T cpusetsize,
	4849	void *cpuset) {
	4850	void *ctx;
	4851	COMMON_INTERCEPTOR_ENTER(ctx, pthread_getaffinity_np, attr, cpusetsize,
	4852	cpuset);
	4853	// FIXME: under ASan the call below may write to freed memory and corrupt
	4854	// its metadata. See
	4855	// https://github.com/google/sanitizers/issues/321.
	4856	int res = REAL(pthread_getaffinity_np)(attr, cpusetsize, cpuset);
	4857	if (!res && cpusetsize && cpuset)
	4858	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cpuset, cpusetsize);
	4859	return res;
	4860	}
	4861
	4862	#define INIT_PTHREAD_GETAFFINITY_NP \
	4863	COMMON_INTERCEPT_FUNCTION(pthread_getaffinity_np);
	4864	#else
	4865	#define INIT_PTHREAD_GETAFFINITY_NP
	4866	#endif
	4867
4769	4868	#if SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED
4770	4869	INTERCEPTOR_PTHREAD_MUTEXATTR_GET(pshared, sizeof(int))
4771	4870	#define INIT_PTHREAD_MUTEXATTR_GETPSHARED \
...	...	@@ -4864,9 +4963,9 @@ INTERCEPTOR(char , tmpnam, char s) {
4864	4963	// FIXME: under ASan the call below may write to freed memory and corrupt
4865	4964	// its metadata. See
4866	4965	// https://github.com/google/sanitizers/issues/321.
4867		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1);
	4966	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, internal_strlen(s) + 1);
4868	4967	else
4869		COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
	4968	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
4870	4969	}
4871	4970	return res;
4872	4971	}
...	...	@@ -4883,7 +4982,7 @@ INTERCEPTOR(char , tmpnam_r, char s) {
4883	4982	// its metadata. See
4884	4983	// https://github.com/google/sanitizers/issues/321.
4885	4984	char *res = REAL(tmpnam_r)(s);
4886		if (res && s) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1);
	4985	if (res && s) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, internal_strlen(s) + 1);
4887	4986	return res;
4888	4987	}
4889	4988	#define INIT_TMPNAM_R COMMON_INTERCEPT_FUNCTION(tmpnam_r);
...	...	@@ -4897,7 +4996,7 @@ INTERCEPTOR(char *, ptsname, int fd) {
4897	4996	COMMON_INTERCEPTOR_ENTER(ctx, ptsname, fd);
4898	4997	char *res = REAL(ptsname)(fd);
4899	4998	if (res != nullptr)
4900		COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
	4999	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
4901	5000	return res;
4902	5001	}
4903	5002	#define INIT_PTSNAME COMMON_INTERCEPT_FUNCTION(ptsname);
...	...	@@ -4911,7 +5010,7 @@ INTERCEPTOR(int, ptsname_r, int fd, char *name, SIZE_T namesize) {
4911	5010	COMMON_INTERCEPTOR_ENTER(ctx, ptsname_r, fd, name, namesize);
4912	5011	int res = REAL(ptsname_r)(fd, name, namesize);
4913	5012	if (res == 0)
4914		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);
	5013	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
4915	5014	return res;
4916	5015	}
4917	5016	#define INIT_PTSNAME_R COMMON_INTERCEPT_FUNCTION(ptsname_r);
...	...	@@ -4925,7 +5024,7 @@ INTERCEPTOR(char *, ttyname, int fd) {
4925	5024	COMMON_INTERCEPTOR_ENTER(ctx, ttyname, fd);
4926	5025	char *res = REAL(ttyname)(fd);
4927	5026	if (res != nullptr)
4928		COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
	5027	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
4929	5028	return res;
4930	5029	}
4931	5030	#define INIT_TTYNAME COMMON_INTERCEPT_FUNCTION(ttyname);
...	...	@@ -4939,7 +5038,7 @@ INTERCEPTOR(int, ttyname_r, int fd, char *name, SIZE_T namesize) {
4939	5038	COMMON_INTERCEPTOR_ENTER(ctx, ttyname_r, fd, name, namesize);
4940	5039	int res = REAL(ttyname_r)(fd, name, namesize);
4941	5040	if (res == 0)
4942		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);
	5041	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
4943	5042	return res;
4944	5043	}
4945	5044	#define INIT_TTYNAME_R COMMON_INTERCEPT_FUNCTION(ttyname_r);
...	...	@@ -4951,10 +5050,10 @@ INTERCEPTOR(int, ttyname_r, int fd, char *name, SIZE_T namesize) {
4951	5050	INTERCEPTOR(char , tempnam, char dir, char *pfx) {
4952	5051	void *ctx;
4953	5052	COMMON_INTERCEPTOR_ENTER(ctx, tempnam, dir, pfx);
4954		if (dir) COMMON_INTERCEPTOR_READ_RANGE(ctx, dir, REAL(strlen)(dir) + 1);
4955		if (pfx) COMMON_INTERCEPTOR_READ_RANGE(ctx, pfx, REAL(strlen)(pfx) + 1);
	5053	if (dir) COMMON_INTERCEPTOR_READ_RANGE(ctx, dir, internal_strlen(dir) + 1);
	5054	if (pfx) COMMON_INTERCEPTOR_READ_RANGE(ctx, pfx, internal_strlen(pfx) + 1);
4956	5055	char *res = REAL(tempnam)(dir, pfx);
4957		if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
	5056	if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
4958	5057	return res;
4959	5058	}
4960	5059	#define INIT_TEMPNAM COMMON_INTERCEPT_FUNCTION(tempnam);
...	...	@@ -5332,9 +5431,7 @@ INTERCEPTOR(void , __tls_get_addr, void arg) {
5332	5431	// On PowerPC, we also need to intercept __tls_get_addr_opt, which has
5333	5432	// mostly the same semantics as __tls_get_addr, but its presence enables
5334	5433	// some optimizations in linker (which are safe to ignore here).
5335		extern "C" __attribute__((alias("__interceptor___tls_get_addr"),
5336		visibility("default")))
5337		void __tls_get_addr_opt(void arg);
	5434	INTERCEPTOR(void , __tls_get_addr_opt, void arg) ALIAS(WRAP(__tls_get_addr));
5338	5435	#endif
5339	5436	#else // SANITIZER_S390
5340	5437	// On s390, we have to intercept two functions here:
...	...	@@ -5368,21 +5465,20 @@ INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) {
5368	5465
5369	5466	#if SANITIZER_S390 && \
5370	5467	(SANITIZER_INTERCEPT_TLS_GET_ADDR \|\| SANITIZER_INTERCEPT_TLS_GET_OFFSET)
5371		extern "C" uptr __tls_get_offset(void *arg);
5372		extern "C" uptr __interceptor___tls_get_offset(void *arg);
5373	5468	// We need a hidden symbol aliasing the above, so that we can jump
5374	5469	// directly to it from the assembly below.
5375		extern "C" __attribute__((alias("__interceptor___tls_get_addr_internal"),
5376		visibility("hidden")))
5377		uptr __tls_get_addr_hidden(void *arg);
	5470	extern "C" __attribute__((visibility("hidden"))) uptr __tls_get_addr_hidden(
	5471	void *arg) ALIAS(WRAP(__tls_get_addr_internal));
	5472	extern "C" uptr __tls_get_offset(void *arg);
	5473	extern "C" uptr TRAMPOLINE(__tls_get_offset)(void *arg);
	5474	extern "C" uptr WRAP(__tls_get_offset)(void *arg);
5378	5475	// Now carefully intercept __tls_get_offset.
5379	5476	asm(
5380	5477	".text\n"
5381	5478	// The __intercept_ version has to exist, so that gen_dynamic_list.py
5382	5479	// exports our symbol.
5383	5480	".weak __tls_get_offset\n"
5384		".type __tls_get_offset, @function\n"
5385		"__tls_get_offset:\n"
	5481	".set __tls_get_offset, __interceptor___tls_get_offset\n"
5386	5482	".global __interceptor___tls_get_offset\n"
5387	5483	".type __interceptor___tls_get_offset, @function\n"
5388	5484	"__interceptor___tls_get_offset:\n"
...	...	@@ -5414,7 +5510,7 @@ asm(
5414	5510	INTERCEPTOR(SSIZE_T, listxattr, const char path, char list, SIZE_T size) {
5415	5511	void *ctx;
5416	5512	COMMON_INTERCEPTOR_ENTER(ctx, listxattr, path, list, size);
5417		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	5513	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
5418	5514	// FIXME: under ASan the call below may write to freed memory and corrupt
5419	5515	// its metadata. See
5420	5516	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -5427,7 +5523,7 @@ INTERCEPTOR(SSIZE_T, listxattr, const char path, char list, SIZE_T size) {
5427	5523	INTERCEPTOR(SSIZE_T, llistxattr, const char path, char list, SIZE_T size) {
5428	5524	void *ctx;
5429	5525	COMMON_INTERCEPTOR_ENTER(ctx, llistxattr, path, list, size);
5430		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	5526	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
5431	5527	// FIXME: under ASan the call below may write to freed memory and corrupt
5432	5528	// its metadata. See
5433	5529	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -5458,8 +5554,8 @@ INTERCEPTOR(SSIZE_T, getxattr, const char path, const char name, char *value,
5458	5554	SIZE_T size) {
5459	5555	void *ctx;
5460	5556	COMMON_INTERCEPTOR_ENTER(ctx, getxattr, path, name, value, size);
5461		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
5462		if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
	5557	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
	5558	if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
5463	5559	// FIXME: under ASan the call below may write to freed memory and corrupt
5464	5560	// its metadata. See
5465	5561	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -5471,8 +5567,8 @@ INTERCEPTOR(SSIZE_T, lgetxattr, const char path, const char name, char *value,
5471	5567	SIZE_T size) {
5472	5568	void *ctx;
5473	5569	COMMON_INTERCEPTOR_ENTER(ctx, lgetxattr, path, name, value, size);
5474		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
5475		if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
	5570	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
	5571	if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
5476	5572	// FIXME: under ASan the call below may write to freed memory and corrupt
5477	5573	// its metadata. See
5478	5574	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -5484,7 +5580,7 @@ INTERCEPTOR(SSIZE_T, fgetxattr, int fd, const char name, char value,
5484	5580	SIZE_T size) {
5485	5581	void *ctx;
5486	5582	COMMON_INTERCEPTOR_ENTER(ctx, fgetxattr, fd, name, value, size);
5487		if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
	5583	if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
5488	5584	// FIXME: under ASan the call below may write to freed memory and corrupt
5489	5585	// its metadata. See
5490	5586	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -5554,7 +5650,7 @@ INTERCEPTOR(int, getifaddrs, __sanitizer_ifaddrs **ifap) {
5554	5650	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(__sanitizer_ifaddrs));
5555	5651	if (p->ifa_name)
5556	5652	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ifa_name,
5557		REAL(strlen)(p->ifa_name) + 1);
	5653	internal_strlen(p->ifa_name) + 1);
5558	5654	if (p->ifa_addr)
5559	5655	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ifa_addr, struct_sockaddr_sz);
5560	5656	if (p->ifa_netmask)
...	...	@@ -5584,14 +5680,14 @@ INTERCEPTOR(char , if_indextoname, unsigned int ifindex, char ifname) {
5584	5680	// https://github.com/google/sanitizers/issues/321.
5585	5681	char *res = REAL(if_indextoname)(ifindex, ifname);
5586	5682	if (res && ifname)
5587		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ifname, REAL(strlen)(ifname) + 1);
	5683	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ifname, internal_strlen(ifname) + 1);
5588	5684	return res;
5589	5685	}
5590	5686	INTERCEPTOR(unsigned int, if_nametoindex, const char* ifname) {
5591	5687	void *ctx;
5592	5688	COMMON_INTERCEPTOR_ENTER(ctx, if_nametoindex, ifname);
5593	5689	if (ifname)
5594		COMMON_INTERCEPTOR_READ_RANGE(ctx, ifname, REAL(strlen)(ifname) + 1);
	5690	COMMON_INTERCEPTOR_READ_RANGE(ctx, ifname, internal_strlen(ifname) + 1);
5595	5691	return REAL(if_nametoindex)(ifname);
5596	5692	}
5597	5693	#define INIT_IF_INDEXTONAME \
...	...	@@ -5611,8 +5707,10 @@ INTERCEPTOR(int, capget, void hdrp, void datap) {
5611	5707	// its metadata. See
5612	5708	// https://github.com/google/sanitizers/issues/321.
5613	5709	int res = REAL(capget)(hdrp, datap);
5614		if (res == 0 && datap)
5615		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datap, __user_cap_data_struct_sz);
	5710	if (res == 0 && datap) {
	5711	unsigned datasz = __user_cap_data_struct_sz(hdrp);
	5712	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datap, datasz);
	5713	}
5616	5714	// We can also return -1 and write to hdrp->version if the version passed in
5617	5715	// hdrp->version is unsupported. But that's not a trivial condition to check,
5618	5716	// and anyway COMMON_INTERCEPTOR_READ_RANGE protects us to some extent.
...	...	@@ -5623,8 +5721,10 @@ INTERCEPTOR(int, capset, void hdrp, const void datap) {
5623	5721	COMMON_INTERCEPTOR_ENTER(ctx, capset, hdrp, datap);
5624	5722	if (hdrp)
5625	5723	COMMON_INTERCEPTOR_READ_RANGE(ctx, hdrp, __user_cap_header_struct_sz);
5626		if (datap)
5627		COMMON_INTERCEPTOR_READ_RANGE(ctx, datap, __user_cap_data_struct_sz);
	5724	if (datap) {
	5725	unsigned datasz = __user_cap_data_struct_sz(hdrp);
	5726	COMMON_INTERCEPTOR_READ_RANGE(ctx, datap, datasz);
	5727	}
5628	5728	return REAL(capset)(hdrp, datap);
5629	5729	}
5630	5730	#define INIT_CAPGET \
...	...	@@ -5634,105 +5734,6 @@ INTERCEPTOR(int, capset, void hdrp, const void datap) {
5634	5734	#define INIT_CAPGET
5635	5735	#endif
5636	5736
5637		#if SANITIZER_INTERCEPT_AEABI_MEM
5638		INTERCEPTOR(void , __aeabi_memmove, void to, const void *from, uptr size) {
5639		void *ctx;
5640		COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
5641		}
5642
5643		INTERCEPTOR(void , __aeabi_memmove4, void to, const void *from, uptr size) {
5644		void *ctx;
5645		COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
5646		}
5647
5648		INTERCEPTOR(void , __aeabi_memmove8, void to, const void *from, uptr size) {
5649		void *ctx;
5650		COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
5651		}
5652
5653		INTERCEPTOR(void , __aeabi_memcpy, void to, const void *from, uptr size) {
5654		void *ctx;
5655		COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
5656		}
5657
5658		INTERCEPTOR(void , __aeabi_memcpy4, void to, const void *from, uptr size) {
5659		void *ctx;
5660		COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
5661		}
5662
5663		INTERCEPTOR(void , __aeabi_memcpy8, void to, const void *from, uptr size) {
5664		void *ctx;
5665		COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
5666		}
5667
5668		// Note the argument order.
5669		INTERCEPTOR(void , __aeabi_memset, void block, uptr size, int c) {
5670		void *ctx;
5671		COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
5672		}
5673
5674		INTERCEPTOR(void , __aeabi_memset4, void block, uptr size, int c) {
5675		void *ctx;
5676		COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
5677		}
5678
5679		INTERCEPTOR(void , __aeabi_memset8, void block, uptr size, int c) {
5680		void *ctx;
5681		COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
5682		}
5683
5684		INTERCEPTOR(void , __aeabi_memclr, void block, uptr size) {
5685		void *ctx;
5686		COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
5687		}
5688
5689		INTERCEPTOR(void , __aeabi_memclr4, void block, uptr size) {
5690		void *ctx;
5691		COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
5692		}
5693
5694		INTERCEPTOR(void , __aeabi_memclr8, void block, uptr size) {
5695		void *ctx;
5696		COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
5697		}
5698
5699		#define INIT_AEABI_MEM \
5700		COMMON_INTERCEPT_FUNCTION(__aeabi_memmove); \
5701		COMMON_INTERCEPT_FUNCTION(__aeabi_memmove4); \
5702		COMMON_INTERCEPT_FUNCTION(__aeabi_memmove8); \
5703		COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy); \
5704		COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy4); \
5705		COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy8); \
5706		COMMON_INTERCEPT_FUNCTION(__aeabi_memset); \
5707		COMMON_INTERCEPT_FUNCTION(__aeabi_memset4); \
5708		COMMON_INTERCEPT_FUNCTION(__aeabi_memset8); \
5709		COMMON_INTERCEPT_FUNCTION(__aeabi_memclr); \
5710		COMMON_INTERCEPT_FUNCTION(__aeabi_memclr4); \
5711		COMMON_INTERCEPT_FUNCTION(__aeabi_memclr8);
5712		#else
5713		#define INIT_AEABI_MEM
5714		#endif // SANITIZER_INTERCEPT_AEABI_MEM
5715
5716		#if SANITIZER_INTERCEPT___BZERO
5717		INTERCEPTOR(void , __bzero, void block, uptr size) {
5718		void *ctx;
5719		COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
5720		}
5721		#define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero);
5722		#else
5723		#define INIT___BZERO
5724		#endif // SANITIZER_INTERCEPT___BZERO
5725
5726		#if SANITIZER_INTERCEPT_BZERO
5727		INTERCEPTOR(void , bzero, void block, uptr size) {
5728		void *ctx;
5729		COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
5730		}
5731		#define INIT_BZERO COMMON_INTERCEPT_FUNCTION(bzero);
5732		#else
5733		#define INIT_BZERO
5734		#endif // SANITIZER_INTERCEPT_BZERO
5735
5736	5737	#if SANITIZER_INTERCEPT_FTIME
5737	5738	INTERCEPTOR(int, ftime, __sanitizer_timeb *tp) {
5738	5739	void *ctx;
...	...	@@ -5849,7 +5850,7 @@ INTERCEPTOR(int, xdr_string, __sanitizer_XDR xdrs, char *p,
5849	5850	COMMON_INTERCEPTOR_ENTER(ctx, xdr_string, xdrs, p, maxsize);
5850	5851	if (p && xdrs->x_op == __sanitizer_XDR_ENCODE) {
5851	5852	COMMON_INTERCEPTOR_READ_RANGE(ctx, p, sizeof(*p));
5852		COMMON_INTERCEPTOR_READ_RANGE(ctx, p, REAL(strlen)(p) + 1);
	5853	COMMON_INTERCEPTOR_READ_RANGE(ctx, p, internal_strlen(p) + 1);
5853	5854	}
5854	5855	// FIXME: under ASan the call below may write to freed memory and corrupt
5855	5856	// its metadata. See
...	...	@@ -5858,7 +5859,7 @@ INTERCEPTOR(int, xdr_string, __sanitizer_XDR xdrs, char *p,
5858	5859	if (p && xdrs->x_op == __sanitizer_XDR_DECODE) {
5859	5860	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
5860	5861	if (res && *p)
5861		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, REAL(strlen)(p) + 1);
	5862	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, internal_strlen(p) + 1);
5862	5863	}
5863	5864	return res;
5864	5865	}
...	...	@@ -6069,8 +6070,8 @@ INTERCEPTOR(int, __woverflow, __sanitizer_FILE *fp, int ch) {
6069	6070	INTERCEPTOR(__sanitizer_FILE , fopen, const char path, const char *mode) {
6070	6071	void *ctx;
6071	6072	COMMON_INTERCEPTOR_ENTER(ctx, fopen, path, mode);
6072		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
6073		COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);
	6073	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
	6074	COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, internal_strlen(mode) + 1);
6074	6075	__sanitizer_FILE *res = REAL(fopen)(path, mode);
6075	6076	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
6076	6077	if (res) unpoison_file(res);
...	...	@@ -6079,7 +6080,7 @@ INTERCEPTOR(__sanitizer_FILE , fopen, const char path, const char *mode) {
6079	6080	INTERCEPTOR(__sanitizer_FILE , fdopen, int fd, const char mode) {
6080	6081	void *ctx;
6081	6082	COMMON_INTERCEPTOR_ENTER(ctx, fdopen, fd, mode);
6082		COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);
	6083	COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, internal_strlen(mode) + 1);
6083	6084	__sanitizer_FILE *res = REAL(fdopen)(fd, mode);
6084	6085	if (res) unpoison_file(res);
6085	6086	return res;
...	...	@@ -6088,8 +6089,8 @@ INTERCEPTOR(__sanitizer_FILE , freopen, const char path, const char *mode,
6088	6089	__sanitizer_FILE *fp) {
6089	6090	void *ctx;
6090	6091	COMMON_INTERCEPTOR_ENTER(ctx, freopen, path, mode, fp);
6091		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
6092		COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);
	6092	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
	6093	COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, internal_strlen(mode) + 1);
6093	6094	COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp);
6094	6095	__sanitizer_FILE *res = REAL(freopen)(path, mode, fp);
6095	6096	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
...	...	@@ -6113,7 +6114,7 @@ INTERCEPTOR(int, flopen, const char *path, int flags, ...) {
6113	6114	va_end(ap);
6114	6115	COMMON_INTERCEPTOR_ENTER(ctx, flopen, path, flags, mode);
6115	6116	if (path) {
6116		COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	6117	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
6117	6118	}
6118	6119	return REAL(flopen)(path, flags, mode);
6119	6120	}
...	...	@@ -6126,7 +6127,7 @@ INTERCEPTOR(int, flopenat, int dirfd, const char *path, int flags, ...) {
6126	6127	va_end(ap);
6127	6128	COMMON_INTERCEPTOR_ENTER(ctx, flopen, path, flags, mode);
6128	6129	if (path) {
6129		COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	6130	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
6130	6131	}
6131	6132	return REAL(flopenat)(dirfd, path, flags, mode);
6132	6133	}
...	...	@@ -6142,8 +6143,8 @@ INTERCEPTOR(int, flopenat, int dirfd, const char *path, int flags, ...) {
6142	6143	INTERCEPTOR(__sanitizer_FILE , fopen64, const char path, const char *mode) {
6143	6144	void *ctx;
6144	6145	COMMON_INTERCEPTOR_ENTER(ctx, fopen64, path, mode);
6145		COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
6146		COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);
	6146	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
	6147	COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, internal_strlen(mode) + 1);
6147	6148	__sanitizer_FILE *res = REAL(fopen64)(path, mode);
6148	6149	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
6149	6150	if (res) unpoison_file(res);
...	...	@@ -6153,8 +6154,8 @@ INTERCEPTOR(__sanitizer_FILE , freopen64, const char path, const char *mode,
6153	6154	__sanitizer_FILE *fp) {
6154	6155	void *ctx;
6155	6156	COMMON_INTERCEPTOR_ENTER(ctx, freopen64, path, mode, fp);
6156		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
6157		COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);
	6157	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
	6158	COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, internal_strlen(mode) + 1);
6158	6159	COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp);
6159	6160	__sanitizer_FILE *res = REAL(freopen64)(path, mode, fp);
6160	6161	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
...	...	@@ -6305,8 +6306,7 @@ INTERCEPTOR(void, dlopen, const char filename, int flag) {
6305	6306	void *ctx;
6306	6307	COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlopen, filename, flag);
6307	6308	if (filename) COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0);
6308		COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag);
6309		void *res = REAL(dlopen)(filename, flag);
	6309	void *res = COMMON_INTERCEPTOR_DLOPEN(filename, flag);
6310	6310	Symbolizer::GetOrInit()->InvalidateModuleList();
6311	6311	COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res);
6312	6312	return res;
...	...	@@ -6332,9 +6332,9 @@ INTERCEPTOR(char , getpass, const char prompt) {
6332	6332	void *ctx;
6333	6333	COMMON_INTERCEPTOR_ENTER(ctx, getpass, prompt);
6334	6334	if (prompt)
6335		COMMON_INTERCEPTOR_READ_RANGE(ctx, prompt, REAL(strlen)(prompt)+1);
	6335	COMMON_INTERCEPTOR_READ_RANGE(ctx, prompt, internal_strlen(prompt)+1);
6336	6336	char *res = REAL(getpass)(prompt);
6337		if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res)+1);
	6337	if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res)+1);
6338	6338	return res;
6339	6339	}
6340	6340
...	...	@@ -6475,7 +6475,7 @@ INTERCEPTOR(int, sem_init, __sanitizer_sem_t *s, int pshared, unsigned value) {
6475	6475	COMMON_INTERCEPTOR_ENTER(ctx, sem_init, s, pshared, value);
6476	6476	// Workaround a bug in glibc's "old" semaphore implementation by
6477	6477	// zero-initializing the sem_t contents. This has to be done here because
6478		// interceptors bind to the lowest symbols version by default, hitting the
	6478	// interceptors bind to the lowest version before glibc 2.36, hitting the
6479	6479	// buggy code path while the non-sanitized build of the same code works fine.
6480	6480	REAL(memset)(s, 0, sizeof(*s));
6481	6481	int res = REAL(sem_init)(s, pshared, value);
...	...	@@ -6538,17 +6538,42 @@ INTERCEPTOR(int, sem_getvalue, __sanitizer_sem_t s, int sval) {
6538	6538	}
6539	6539	return res;
6540	6540	}
6541		#define INIT_SEM \
6542		COMMON_INTERCEPT_FUNCTION(sem_init); \
6543		COMMON_INTERCEPT_FUNCTION(sem_destroy); \
6544		COMMON_INTERCEPT_FUNCTION(sem_wait); \
6545		COMMON_INTERCEPT_FUNCTION(sem_trywait); \
6546		COMMON_INTERCEPT_FUNCTION(sem_timedwait); \
6547		COMMON_INTERCEPT_FUNCTION(sem_post); \
6548		COMMON_INTERCEPT_FUNCTION(sem_getvalue);
	6541
	6542	INTERCEPTOR(__sanitizer_sem_t , sem_open, const char name, int oflag, ...) {
	6543	void *ctx;
	6544	va_list ap;
	6545	va_start(ap, oflag);
	6546	u32 mode = va_arg(ap, u32);
	6547	u32 value = va_arg(ap, u32);
	6548	COMMON_INTERCEPTOR_ENTER(ctx, sem_open, name, oflag, mode, value);
	6549	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
	6550	__sanitizer_sem_t *s = REAL(sem_open)(name, oflag, mode, value);
	6551	if (s)
	6552	COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, sizeof(*s));
	6553	va_end(ap);
	6554	return s;
	6555	}
	6556
	6557	INTERCEPTOR(int, sem_unlink, const char *name) {
	6558	void *ctx;
	6559	COMMON_INTERCEPTOR_ENTER(ctx, sem_unlink, name);
	6560	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
	6561	return REAL(sem_unlink)(name);
	6562	}
	6563
	6564	# define INIT_SEM \
	6565	COMMON_INTERCEPT_FUNCTION(sem_init); \
	6566	COMMON_INTERCEPT_FUNCTION(sem_destroy); \
	6567	COMMON_INTERCEPT_FUNCTION(sem_wait); \
	6568	COMMON_INTERCEPT_FUNCTION(sem_trywait); \
	6569	COMMON_INTERCEPT_FUNCTION(sem_timedwait); \
	6570	COMMON_INTERCEPT_FUNCTION(sem_post); \
	6571	COMMON_INTERCEPT_FUNCTION(sem_getvalue); \
	6572	COMMON_INTERCEPT_FUNCTION(sem_open); \
	6573	COMMON_INTERCEPT_FUNCTION(sem_unlink);
6549	6574	#else
6550		#define INIT_SEM
6551		#endif // SANITIZER_INTERCEPT_SEM
	6575	# define INIT_SEM
	6576	#endif // SANITIZER_INTERCEPT_SEM
6552	6577
6553	6578	#if SANITIZER_INTERCEPT_PTHREAD_SETCANCEL
6554	6579	INTERCEPTOR(int, pthread_setcancelstate, int state, int *oldstate) {
...	...	@@ -6631,7 +6656,7 @@ INTERCEPTOR(char , ctermid, char s) {
6631	6656	COMMON_INTERCEPTOR_ENTER(ctx, ctermid, s);
6632	6657	char *res = REAL(ctermid)(s);
6633	6658	if (res) {
6634		COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
	6659	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
6635	6660	}
6636	6661	return res;
6637	6662	}
...	...	@@ -6646,7 +6671,7 @@ INTERCEPTOR(char , ctermid_r, char s) {
6646	6671	COMMON_INTERCEPTOR_ENTER(ctx, ctermid_r, s);
6647	6672	char *res = REAL(ctermid_r)(s);
6648	6673	if (res) {
6649		COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
	6674	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
6650	6675	}
6651	6676	return res;
6652	6677	}
...	...	@@ -6772,6 +6797,23 @@ INTERCEPTOR(int, stat, const char path, void buf) {
6772	6797	#define INIT_STAT
6773	6798	#endif
6774	6799
	6800	#if SANITIZER_INTERCEPT_STAT64
	6801	INTERCEPTOR(int, stat64, const char path, void buf) {
	6802	void *ctx;
	6803	COMMON_INTERCEPTOR_ENTER(ctx, stat64, path, buf);
	6804	if (common_flags()->intercept_stat)
	6805	COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0);
	6806	int res = REAL(stat64)(path, buf);
	6807	if (!res)
	6808	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat64_sz);
	6809	return res;
	6810	}
	6811	#define INIT_STAT64 COMMON_INTERCEPT_FUNCTION(stat64)
	6812	#else
	6813	#define INIT_STAT64
	6814	#endif
	6815
	6816
6775	6817	#if SANITIZER_INTERCEPT_LSTAT
6776	6818	INTERCEPTOR(int, lstat, const char path, void buf) {
6777	6819	void *ctx;
...	...	@@ -6788,6 +6830,22 @@ INTERCEPTOR(int, lstat, const char path, void buf) {
6788	6830	#define INIT_LSTAT
6789	6831	#endif
6790	6832
	6833	#if SANITIZER_INTERCEPT_STAT64
	6834	INTERCEPTOR(int, lstat64, const char path, void buf) {
	6835	void *ctx;
	6836	COMMON_INTERCEPTOR_ENTER(ctx, lstat64, path, buf);
	6837	if (common_flags()->intercept_stat)
	6838	COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0);
	6839	int res = REAL(lstat64)(path, buf);
	6840	if (!res)
	6841	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat64_sz);
	6842	return res;
	6843	}
	6844	#define INIT_LSTAT64 COMMON_INTERCEPT_FUNCTION(lstat64)
	6845	#else
	6846	#define INIT_LSTAT64
	6847	#endif
	6848
6791	6849	#if SANITIZER_INTERCEPT___XSTAT
6792	6850	INTERCEPTOR(int, __xstat, int version, const char path, void buf) {
6793	6851	void *ctx;
...	...	@@ -6960,6 +7018,7 @@ INTERCEPTOR(int, mprobe, void *ptr) {
6960	7018	}
6961	7019	#endif
6962	7020
	7021	#if SANITIZER_INTERCEPT_WCSLEN
6963	7022	INTERCEPTOR(SIZE_T, wcslen, const wchar_t *s) {
6964	7023	void *ctx;
6965	7024	COMMON_INTERCEPTOR_ENTER(ctx, wcslen, s);
...	...	@@ -6978,13 +7037,16 @@ INTERCEPTOR(SIZE_T, wcsnlen, const wchar_t *s, SIZE_T n) {
6978	7037	#define INIT_WCSLEN \
6979	7038	COMMON_INTERCEPT_FUNCTION(wcslen); \
6980	7039	COMMON_INTERCEPT_FUNCTION(wcsnlen);
	7040	#else
	7041	#define INIT_WCSLEN
	7042	#endif
6981	7043
6982	7044	#if SANITIZER_INTERCEPT_WCSCAT
6983	7045	INTERCEPTOR(wchar_t , wcscat, wchar_t dst, const wchar_t *src) {
6984	7046	void *ctx;
6985	7047	COMMON_INTERCEPTOR_ENTER(ctx, wcscat, dst, src);
6986		SIZE_T src_size = REAL(wcslen)(src);
6987		SIZE_T dst_size = REAL(wcslen)(dst);
	7048	SIZE_T src_size = internal_wcslen(src);
	7049	SIZE_T dst_size = internal_wcslen(dst);
6988	7050	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, (src_size + 1) * sizeof(wchar_t));
6989	7051	COMMON_INTERCEPTOR_READ_RANGE(ctx, dst, (dst_size + 1) * sizeof(wchar_t));
6990	7052	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst + dst_size,
...	...	@@ -6995,8 +7057,8 @@ INTERCEPTOR(wchar_t , wcscat, wchar_t dst, const wchar_t *src) {
6995	7057	INTERCEPTOR(wchar_t , wcsncat, wchar_t dst, const wchar_t *src, SIZE_T n) {
6996	7058	void *ctx;
6997	7059	COMMON_INTERCEPTOR_ENTER(ctx, wcsncat, dst, src, n);
6998		SIZE_T src_size = REAL(wcsnlen)(src, n);
6999		SIZE_T dst_size = REAL(wcslen)(dst);
	7060	SIZE_T src_size = internal_wcsnlen(src, n);
	7061	SIZE_T dst_size = internal_wcslen(dst);
7000	7062	COMMON_INTERCEPTOR_READ_RANGE(ctx, src,
7001	7063	Min(src_size + 1, n) * sizeof(wchar_t));
7002	7064	COMMON_INTERCEPTOR_READ_RANGE(ctx, dst, (dst_size + 1) * sizeof(wchar_t));
...	...	@@ -7015,7 +7077,7 @@ INTERCEPTOR(wchar_t , wcsncat, wchar_t dst, const wchar_t *src, SIZE_T n) {
7015	7077	INTERCEPTOR(wchar_t , wcsdup, wchar_t s) {
7016	7078	void *ctx;
7017	7079	COMMON_INTERCEPTOR_ENTER(ctx, wcsdup, s);
7018		SIZE_T len = REAL(wcslen)(s);
	7080	SIZE_T len = internal_wcslen(s);
7019	7081	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(wchar_t) * (len + 1));
7020	7082	wchar_t *result = REAL(wcsdup)(s);
7021	7083	if (result)
...	...	@@ -7029,9 +7091,9 @@ INTERCEPTOR(wchar_t , wcsdup, wchar_t s) {
7029	7091	#endif
7030	7092
7031	7093	#if SANITIZER_INTERCEPT_STRXFRM
7032		static SIZE_T RealStrLen(const char *str) { return REAL(strlen)(str); }
	7094	static SIZE_T RealStrLen(const char *str) { return internal_strlen(str); }
7033	7095
7034		static SIZE_T RealStrLen(const wchar_t *str) { return REAL(wcslen)(str); }
	7096	static SIZE_T RealStrLen(const wchar_t *str) { return internal_wcslen(str); }
7035	7097
7036	7098	#define STRXFRM_INTERCEPTOR_IMPL(strxfrm, dest, src, len, ...) \
7037	7099	{ \
...	...	@@ -7105,7 +7167,7 @@ INTERCEPTOR(int, acct, const char *file) {
7105	7167	void *ctx;
7106	7168	COMMON_INTERCEPTOR_ENTER(ctx, acct, file);
7107	7169	if (file)
7108		COMMON_INTERCEPTOR_READ_RANGE(ctx, file, REAL(strlen)(file) + 1);
	7170	COMMON_INTERCEPTOR_READ_RANGE(ctx, file, internal_strlen(file) + 1);
7109	7171	return REAL(acct)(file);
7110	7172	}
7111	7173	#define INIT_ACCT COMMON_INTERCEPT_FUNCTION(acct)
...	...	@@ -7120,7 +7182,7 @@ INTERCEPTOR(const char *, user_from_uid, u32 uid, int nouser) {
7120	7182	COMMON_INTERCEPTOR_ENTER(ctx, user_from_uid, uid, nouser);
7121	7183	user = REAL(user_from_uid)(uid, nouser);
7122	7184	if (user)
7123		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, user, REAL(strlen)(user) + 1);
	7185	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, user, internal_strlen(user) + 1);
7124	7186	return user;
7125	7187	}
7126	7188	#define INIT_USER_FROM_UID COMMON_INTERCEPT_FUNCTION(user_from_uid)
...	...	@@ -7134,7 +7196,7 @@ INTERCEPTOR(int, uid_from_user, const char name, u32 uid) {
7134	7196	int res;
7135	7197	COMMON_INTERCEPTOR_ENTER(ctx, uid_from_user, name, uid);
7136	7198	if (name)
7137		COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
	7199	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
7138	7200	res = REAL(uid_from_user)(name, uid);
7139	7201	if (uid)
7140	7202	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, uid, sizeof(*uid));
...	...	@@ -7152,7 +7214,7 @@ INTERCEPTOR(const char *, group_from_gid, u32 gid, int nogroup) {
7152	7214	COMMON_INTERCEPTOR_ENTER(ctx, group_from_gid, gid, nogroup);
7153	7215	group = REAL(group_from_gid)(gid, nogroup);
7154	7216	if (group)
7155		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, group, REAL(strlen)(group) + 1);
	7217	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, group, internal_strlen(group) + 1);
7156	7218	return group;
7157	7219	}
7158	7220	#define INIT_GROUP_FROM_GID COMMON_INTERCEPT_FUNCTION(group_from_gid)
...	...	@@ -7166,7 +7228,7 @@ INTERCEPTOR(int, gid_from_group, const char group, u32 gid) {
7166	7228	int res;
7167	7229	COMMON_INTERCEPTOR_ENTER(ctx, gid_from_group, group, gid);
7168	7230	if (group)
7169		COMMON_INTERCEPTOR_READ_RANGE(ctx, group, REAL(strlen)(group) + 1);
	7231	COMMON_INTERCEPTOR_READ_RANGE(ctx, group, internal_strlen(group) + 1);
7170	7232	res = REAL(gid_from_group)(group, gid);
7171	7233	if (gid)
7172	7234	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, gid, sizeof(*gid));
...	...	@@ -7182,7 +7244,7 @@ INTERCEPTOR(int, access, const char *path, int mode) {
7182	7244	void *ctx;
7183	7245	COMMON_INTERCEPTOR_ENTER(ctx, access, path, mode);
7184	7246	if (path)
7185		COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	7247	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
7186	7248	return REAL(access)(path, mode);
7187	7249	}
7188	7250	#define INIT_ACCESS COMMON_INTERCEPT_FUNCTION(access)
...	...	@@ -7195,7 +7257,7 @@ INTERCEPTOR(int, faccessat, int fd, const char *path, int mode, int flags) {
7195	7257	void *ctx;
7196	7258	COMMON_INTERCEPTOR_ENTER(ctx, faccessat, fd, path, mode, flags);
7197	7259	if (path)
7198		COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	7260	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
7199	7261	return REAL(faccessat)(fd, path, mode, flags);
7200	7262	}
7201	7263	#define INIT_FACCESSAT COMMON_INTERCEPT_FUNCTION(faccessat)
...	...	@@ -7210,7 +7272,7 @@ INTERCEPTOR(int, getgrouplist, const char name, u32 basegid, u32 groups,
7210	7272	int res;
7211	7273	COMMON_INTERCEPTOR_ENTER(ctx, getgrouplist, name, basegid, groups, ngroups);
7212	7274	if (name)
7213		COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
	7275	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
7214	7276	if (ngroups)
7215	7277	COMMON_INTERCEPTOR_READ_RANGE(ctx, ngroups, sizeof(*ngroups));
7216	7278	res = REAL(getgrouplist)(name, basegid, groups, ngroups);
...	...	@@ -7234,7 +7296,7 @@ INTERCEPTOR(int, getgroupmembership, const char name, u32 basegid, u32 groups,
7234	7296	COMMON_INTERCEPTOR_ENTER(ctx, getgroupmembership, name, basegid, groups,
7235	7297	maxgrp, ngroups);
7236	7298	if (name)
7237		COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
	7299	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
7238	7300	res = REAL(getgroupmembership)(name, basegid, groups, maxgrp, ngroups);
7239	7301	if (!res && groups && ngroups) {
7240	7302	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, groups, sizeof(groups) (*ngroups));
...	...	@@ -7252,7 +7314,7 @@ INTERCEPTOR(int, getgroupmembership, const char name, u32 basegid, u32 groups,
7252	7314	INTERCEPTOR(SSIZE_T, readlink, const char path, char buf, SIZE_T bufsiz) {
7253	7315	void* ctx;
7254	7316	COMMON_INTERCEPTOR_ENTER(ctx, readlink, path, buf, bufsiz);
7255		COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	7317	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
7256	7318	SSIZE_T res = REAL(readlink)(path, buf, bufsiz);
7257	7319	if (res > 0)
7258	7320	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res);
...	...	@@ -7269,7 +7331,7 @@ INTERCEPTOR(SSIZE_T, readlinkat, int dirfd, const char path, char buf,
7269	7331	SIZE_T bufsiz) {
7270	7332	void* ctx;
7271	7333	COMMON_INTERCEPTOR_ENTER(ctx, readlinkat, dirfd, path, buf, bufsiz);
7272		COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	7334	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
7273	7335	SSIZE_T res = REAL(readlinkat)(dirfd, path, buf, bufsiz);
7274	7336	if (res > 0)
7275	7337	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res);
...	...	@@ -7287,7 +7349,7 @@ INTERCEPTOR(int, name_to_handle_at, int dirfd, const char *pathname,
7287	7349	void* ctx;
7288	7350	COMMON_INTERCEPTOR_ENTER(ctx, name_to_handle_at, dirfd, pathname, handle,
7289	7351	mount_id, flags);
7290		COMMON_INTERCEPTOR_READ_RANGE(ctx, pathname, REAL(strlen)(pathname) + 1);
	7352	COMMON_INTERCEPTOR_READ_RANGE(ctx, pathname, internal_strlen(pathname) + 1);
7291	7353
7292	7354	__sanitizer_file_handle *sanitizer_handle =
7293	7355	reinterpret_cast<__sanitizer_file_handle*>(handle);
...	...	@@ -7351,7 +7413,7 @@ INTERCEPTOR(SIZE_T, strlcpy, char dst, char src, SIZE_T size) {
7351	7413	ctx, src, Min(internal_strnlen(src, size), size - 1) + 1);
7352	7414	}
7353	7415	res = REAL(strlcpy)(dst, src, size);
7354		COMMON_INTERCEPTOR_COPY_STRING(ctx, dst, src, REAL(strlen)(dst) + 1);
	7416	COMMON_INTERCEPTOR_COPY_STRING(ctx, dst, src, internal_strlen(dst) + 1);
7355	7417	return res;
7356	7418	}
7357	7419
...	...	@@ -7379,17 +7441,25 @@ INTERCEPTOR(void , mmap, void addr, SIZE_T sz, int prot, int flags, int fd,
7379	7441	OFF_T off) {
7380	7442	void *ctx;
7381	7443	if (common_flags()->detect_write_exec)
7382		ReportMmapWriteExec(prot);
	7444	ReportMmapWriteExec(prot, flags);
7383	7445	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
7384	7446	return (void *)internal_mmap(addr, sz, prot, flags, fd, off);
7385	7447	COMMON_INTERCEPTOR_ENTER(ctx, mmap, addr, sz, prot, flags, fd, off);
7386	7448	COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, off);
7387	7449	}
7388	7450
	7451	INTERCEPTOR(int, munmap, void *addr, SIZE_T sz) {
	7452	void *ctx;
	7453	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
	7454	return (int)internal_munmap(addr, sz);
	7455	COMMON_INTERCEPTOR_ENTER(ctx, munmap, addr, sz);
	7456	COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz);
	7457	}
	7458
7389	7459	INTERCEPTOR(int, mprotect, void *addr, SIZE_T sz, int prot) {
7390	7460	void *ctx;
7391	7461	if (common_flags()->detect_write_exec)
7392		ReportMmapWriteExec(prot);
	7462	ReportMmapWriteExec(prot, 0);
7393	7463	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
7394	7464	return (int)internal_mprotect(addr, sz, prot);
7395	7465	COMMON_INTERCEPTOR_ENTER(ctx, mprotect, addr, sz, prot);
...	...	@@ -7398,6 +7468,7 @@ INTERCEPTOR(int, mprotect, void *addr, SIZE_T sz, int prot) {
7398	7468	}
7399	7469	#define INIT_MMAP \
7400	7470	COMMON_INTERCEPT_FUNCTION(mmap); \
	7471	COMMON_INTERCEPT_FUNCTION(munmap); \
7401	7472	COMMON_INTERCEPT_FUNCTION(mprotect);
7402	7473	#else
7403	7474	#define INIT_MMAP
...	...	@@ -7408,7 +7479,7 @@ INTERCEPTOR(void , mmap64, void addr, SIZE_T sz, int prot, int flags, int fd,
7408	7479	OFF64_T off) {
7409	7480	void *ctx;
7410	7481	if (common_flags()->detect_write_exec)
7411		ReportMmapWriteExec(prot);
	7482	ReportMmapWriteExec(prot, flags);
7412	7483	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
7413	7484	return (void *)internal_mmap(addr, sz, prot, flags, fd, off);
7414	7485	COMMON_INTERCEPTOR_ENTER(ctx, mmap64, addr, sz, prot, flags, fd, off);
...	...	@@ -7426,7 +7497,7 @@ INTERCEPTOR(char *, devname, u64 dev, u32 type) {
7426	7497	COMMON_INTERCEPTOR_ENTER(ctx, devname, dev, type);
7427	7498	name = REAL(devname)(dev, type);
7428	7499	if (name)
7429		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);
	7500	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
7430	7501	return name;
7431	7502	}
7432	7503	#define INIT_DEVNAME COMMON_INTERCEPT_FUNCTION(devname);
...	...	@@ -7448,7 +7519,7 @@ INTERCEPTOR(DEVNAME_R_RETTYPE, devname_r, u64 dev, u32 type, char *path,
7448	7519	COMMON_INTERCEPTOR_ENTER(ctx, devname_r, dev, type, path, len);
7449	7520	DEVNAME_R_RETTYPE res = REAL(devname_r)(dev, type, path, len);
7450	7521	if (DEVNAME_R_SUCCESS(res))
7451		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, path, REAL(strlen)(path) + 1);
	7522	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, path, internal_strlen(path) + 1);
7452	7523	return res;
7453	7524	}
7454	7525	#define INIT_DEVNAME_R COMMON_INTERCEPT_FUNCTION(devname_r);
...	...	@@ -7478,7 +7549,7 @@ INTERCEPTOR(void, strmode, u32 mode, char *bp) {
7478	7549	COMMON_INTERCEPTOR_ENTER(ctx, strmode, mode, bp);
7479	7550	REAL(strmode)(mode, bp);
7480	7551	if (bp)
7481		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, bp, REAL(strlen)(bp) + 1);
	7552	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, bp, internal_strlen(bp) + 1);
7482	7553	}
7483	7554	#define INIT_STRMODE COMMON_INTERCEPT_FUNCTION(strmode)
7484	7555	#else
...	...	@@ -7498,40 +7569,44 @@ INTERCEPTOR(struct __sanitizer_ttyent , getttynam, char name) {
7498	7569	void *ctx;
7499	7570	COMMON_INTERCEPTOR_ENTER(ctx, getttynam, name);
7500	7571	if (name)
7501		COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
	7572	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
7502	7573	struct __sanitizer_ttyent *ttyent = REAL(getttynam)(name);
7503	7574	if (ttyent)
7504	7575	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ttyent, struct_ttyent_sz);
7505	7576	return ttyent;
7506	7577	}
	7578	#define INIT_TTYENT \
	7579	COMMON_INTERCEPT_FUNCTION(getttyent); \
	7580	COMMON_INTERCEPT_FUNCTION(getttynam);
	7581	#else
	7582	#define INIT_TTYENT
	7583	#endif
	7584
	7585	#if SANITIZER_INTERCEPT_TTYENTPATH
7507	7586	INTERCEPTOR(int, setttyentpath, char *path) {
7508	7587	void *ctx;
7509	7588	COMMON_INTERCEPTOR_ENTER(ctx, setttyentpath, path);
7510	7589	if (path)
7511		COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	7590	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
7512	7591	return REAL(setttyentpath)(path);
7513	7592	}
7514		#define INIT_TTYENT \
7515		COMMON_INTERCEPT_FUNCTION(getttyent); \
7516		COMMON_INTERCEPT_FUNCTION(getttynam); \
7517		COMMON_INTERCEPT_FUNCTION(setttyentpath)
	7593	#define INIT_TTYENTPATH COMMON_INTERCEPT_FUNCTION(setttyentpath);
7518	7594	#else
7519		#define INIT_TTYENT
	7595	#define INIT_TTYENTPATH
7520	7596	#endif
7521	7597
7522	7598	#if SANITIZER_INTERCEPT_PROTOENT
7523	7599	static void write_protoent(void ctx, struct __sanitizer_protoent p) {
7524	7600	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
7525	7601
7526		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_name, REAL(strlen)(p->p_name) + 1);
	7602	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_name, internal_strlen(p->p_name) + 1);
7527	7603
7528	7604	SIZE_T pp_size = 1; // One handles the trailing \0
7529	7605
7530	7606	for (char *pp = p->p_aliases; pp; ++pp, ++pp_size)
7531		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pp, REAL(strlen)(pp) + 1);
	7607	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pp, internal_strlen(pp) + 1);
7532	7608
7533		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases,
7534		pp_size * sizeof(char **));
	7609	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases, pp_size * sizeof(char *));
7535	7610	}
7536	7611
7537	7612	INTERCEPTOR(struct __sanitizer_protoent *, getprotoent) {
...	...	@@ -7547,7 +7622,7 @@ INTERCEPTOR(struct __sanitizer_protoent , getprotobyname, const char name) {
7547	7622	void *ctx;
7548	7623	COMMON_INTERCEPTOR_ENTER(ctx, getprotobyname, name);
7549	7624	if (name)
7550		COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
	7625	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
7551	7626	struct __sanitizer_protoent *p = REAL(getprotobyname)(name);
7552	7627	if (p)
7553	7628	write_protoent(ctx, p);
...	...	@@ -7591,7 +7666,7 @@ INTERCEPTOR(int, getprotobyname_r, const char *name,
7591	7666	COMMON_INTERCEPTOR_ENTER(ctx, getprotobyname_r, name, result_buf, buf,
7592	7667	buflen, result);
7593	7668	if (name)
7594		COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
	7669	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
7595	7670	int res = REAL(getprotobyname_r)(name, result_buf, buf, buflen, result);
7596	7671
7597	7672	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof *result);
...	...	@@ -7630,15 +7705,14 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetent) {
7630	7705	if (n) {
7631	7706	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
7632	7707
7633		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, REAL(strlen)(n->n_name) + 1);
	7708	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, internal_strlen(n->n_name) + 1);
7634	7709
7635	7710	SIZE_T nn_size = 1; // One handles the trailing \0
7636	7711
7637	7712	for (char *nn = n->n_aliases; nn; ++nn, ++nn_size)
7638		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, nn, REAL(strlen)(nn) + 1);
	7713	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, nn, internal_strlen(nn) + 1);
7639	7714
7640		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,
7641		nn_size * sizeof(char **));
	7715	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, nn_size * sizeof(char *));
7642	7716	}
7643	7717	return n;
7644	7718	}
...	...	@@ -7647,20 +7721,19 @@ INTERCEPTOR(struct __sanitizer_netent , getnetbyname, const char name) {
7647	7721	void *ctx;
7648	7722	COMMON_INTERCEPTOR_ENTER(ctx, getnetbyname, name);
7649	7723	if (name)
7650		COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
	7724	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
7651	7725	struct __sanitizer_netent *n = REAL(getnetbyname)(name);
7652	7726	if (n) {
7653	7727	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
7654	7728
7655		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, REAL(strlen)(n->n_name) + 1);
	7729	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, internal_strlen(n->n_name) + 1);
7656	7730
7657	7731	SIZE_T nn_size = 1; // One handles the trailing \0
7658	7732
7659	7733	for (char *nn = n->n_aliases; nn; ++nn, ++nn_size)
7660		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, nn, REAL(strlen)(nn) + 1);
	7734	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, nn, internal_strlen(nn) + 1);
7661	7735
7662		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,
7663		nn_size * sizeof(char **));
	7736	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, nn_size * sizeof(char *));
7664	7737	}
7665	7738	return n;
7666	7739	}
...	...	@@ -7672,15 +7745,14 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetbyaddr, u32 net, int type) {
7672	7745	if (n) {
7673	7746	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
7674	7747
7675		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, REAL(strlen)(n->n_name) + 1);
	7748	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, internal_strlen(n->n_name) + 1);
7676	7749
7677	7750	SIZE_T nn_size = 1; // One handles the trailing \0
7678	7751
7679	7752	for (char *nn = n->n_aliases; nn; ++nn, ++nn_size)
7680		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, nn, REAL(strlen)(nn) + 1);
	7753	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, nn, internal_strlen(nn) + 1);
7681	7754
7682		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,
7683		nn_size * sizeof(char **));
	7755	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, nn_size * sizeof(char *));
7684	7756	}
7685	7757	return n;
7686	7758	}
...	...	@@ -7753,12 +7825,12 @@ INTERCEPTOR(void, setbuf, __sanitizer_FILE stream, char buf) {
7753	7825	unpoison_file(stream);
7754	7826	}
7755	7827
7756		INTERCEPTOR(void, setbuffer, __sanitizer_FILE stream, char buf, int mode) {
	7828	INTERCEPTOR(void, setbuffer, __sanitizer_FILE stream, char buf, SIZE_T size) {
7757	7829	void *ctx;
7758		COMMON_INTERCEPTOR_ENTER(ctx, setbuffer, stream, buf, mode);
7759		REAL(setbuffer)(stream, buf, mode);
	7830	COMMON_INTERCEPTOR_ENTER(ctx, setbuffer, stream, buf, size);
	7831	REAL(setbuffer)(stream, buf, size);
7760	7832	if (buf) {
7761		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer_bufsiz);
	7833	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, size);
7762	7834	}
7763	7835	if (stream)
7764	7836	unpoison_file(stream);
...	...	@@ -7798,9 +7870,9 @@ INTERCEPTOR(int, regcomp, void preg, const char pattern, int cflags) {
7798	7870	void *ctx;
7799	7871	COMMON_INTERCEPTOR_ENTER(ctx, regcomp, preg, pattern, cflags);
7800	7872	if (pattern)
7801		COMMON_INTERCEPTOR_READ_RANGE(ctx, pattern, REAL(strlen)(pattern) + 1);
	7873	COMMON_INTERCEPTOR_READ_RANGE(ctx, pattern, internal_strlen(pattern) + 1);
7802	7874	int res = REAL(regcomp)(preg, pattern, cflags);
7803		if (!res)
	7875	if (preg)
7804	7876	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, preg, struct_regex_sz);
7805	7877	return res;
7806	7878	}
...	...	@@ -7811,7 +7883,7 @@ INTERCEPTOR(int, regexec, const void preg, const char string, SIZE_T nmatch,
7811	7883	if (preg)
7812	7884	COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz);
7813	7885	if (string)
7814		COMMON_INTERCEPTOR_READ_RANGE(ctx, string, REAL(strlen)(string) + 1);
	7886	COMMON_INTERCEPTOR_READ_RANGE(ctx, string, internal_strlen(string) + 1);
7815	7887	int res = REAL(regexec)(preg, string, nmatch, pmatch, eflags);
7816	7888	if (!res && pmatch)
7817	7889	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pmatch, nmatch * struct_regmatch_sz);
...	...	@@ -7825,7 +7897,7 @@ INTERCEPTOR(SIZE_T, regerror, int errcode, const void preg, char errbuf,
7825	7897	COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz);
7826	7898	SIZE_T res = REAL(regerror)(errcode, preg, errbuf, errbuf_size);
7827	7899	if (errbuf)
7828		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errbuf, REAL(strlen)(errbuf) + 1);
	7900	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errbuf, internal_strlen(errbuf) + 1);
7829	7901	return res;
7830	7902	}
7831	7903	INTERCEPTOR(void, regfree, const void *preg) {
...	...	@@ -7850,15 +7922,15 @@ INTERCEPTOR(SSIZE_T, regnsub, char buf, SIZE_T bufsiz, const char sub,
7850	7922	void *ctx;
7851	7923	COMMON_INTERCEPTOR_ENTER(ctx, regnsub, buf, bufsiz, sub, rm, str);
7852	7924	if (sub)
7853		COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, REAL(strlen)(sub) + 1);
	7925	COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, internal_strlen(sub) + 1);
7854	7926	// The implementation demands and hardcodes 10 elements
7855	7927	if (rm)
7856	7928	COMMON_INTERCEPTOR_READ_RANGE(ctx, rm, 10 * struct_regmatch_sz);
7857	7929	if (str)
7858		COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1);
	7930	COMMON_INTERCEPTOR_READ_RANGE(ctx, str, internal_strlen(str) + 1);
7859	7931	SSIZE_T res = REAL(regnsub)(buf, bufsiz, sub, rm, str);
7860	7932	if (res > 0 && buf)
7861		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
	7933	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
7862	7934	return res;
7863	7935	}
7864	7936	INTERCEPTOR(SSIZE_T, regasub, char *buf, const char sub,
...	...	@@ -7866,16 +7938,16 @@ INTERCEPTOR(SSIZE_T, regasub, char *buf, const char sub,
7866	7938	void *ctx;
7867	7939	COMMON_INTERCEPTOR_ENTER(ctx, regasub, buf, sub, rm, sstr);
7868	7940	if (sub)
7869		COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, REAL(strlen)(sub) + 1);
	7941	COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, internal_strlen(sub) + 1);
7870	7942	// Hardcode 10 elements as this is hardcoded size
7871	7943	if (rm)
7872	7944	COMMON_INTERCEPTOR_READ_RANGE(ctx, rm, 10 * struct_regmatch_sz);
7873	7945	if (sstr)
7874		COMMON_INTERCEPTOR_READ_RANGE(ctx, sstr, REAL(strlen)(sstr) + 1);
	7946	COMMON_INTERCEPTOR_READ_RANGE(ctx, sstr, internal_strlen(sstr) + 1);
7875	7947	SSIZE_T res = REAL(regasub)(buf, sub, rm, sstr);
7876	7948	if (res > 0 && buf) {
7877	7949	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, sizeof(char *));
7878		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
	7950	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
7879	7951	}
7880	7952	return res;
7881	7953	}
...	...	@@ -7897,7 +7969,7 @@ INTERCEPTOR(void , fts_open, char const *path_argv, int options,
7897	7969	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **));
7898	7970	if (!*pa)
7899	7971	break;
7900		COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, REAL(strlen)(pa) + 1);
	7972	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, internal_strlen(pa) + 1);
7901	7973	}
7902	7974	}
7903	7975	// TODO(kamil): handle compar callback
...	...	@@ -7989,7 +8061,7 @@ INTERCEPTOR(int, sysctlbyname, char sname, void oldp, SIZE_T *oldlenp,
7989	8061	COMMON_INTERCEPTOR_ENTER(ctx, sysctlbyname, sname, oldp, oldlenp, newp,
7990	8062	newlen);
7991	8063	if (sname)
7992		COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);
	8064	COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, internal_strlen(sname) + 1);
7993	8065	if (oldlenp)
7994	8066	COMMON_INTERCEPTOR_READ_RANGE(ctx, oldlenp, sizeof(*oldlenp));
7995	8067	if (newp && newlen)
...	...	@@ -8010,7 +8082,7 @@ INTERCEPTOR(int, sysctlnametomib, const char sname, int name,
8010	8082	void *ctx;
8011	8083	COMMON_INTERCEPTOR_ENTER(ctx, sysctlnametomib, sname, name, namelenp);
8012	8084	if (sname)
8013		COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);
	8085	COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, internal_strlen(sname) + 1);
8014	8086	if (namelenp)
8015	8087	COMMON_INTERCEPTOR_READ_RANGE(ctx, namelenp, sizeof(*namelenp));
8016	8088	int res = REAL(sysctlnametomib)(sname, name, namelenp);
...	...	@@ -8050,7 +8122,7 @@ INTERCEPTOR(void , asysctlbyname, const char sname, SIZE_T *len) {
8050	8122	void *ctx;
8051	8123	COMMON_INTERCEPTOR_ENTER(ctx, asysctlbyname, sname, len);
8052	8124	if (sname)
8053		COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);
	8125	COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, internal_strlen(sname) + 1);
8054	8126	void *res = REAL(asysctlbyname)(sname, len);
8055	8127	if (res && len) {
8056	8128	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len));
...	...	@@ -8073,7 +8145,7 @@ INTERCEPTOR(int, sysctlgetmibinfo, char sname, int name,
8073	8145	COMMON_INTERCEPTOR_ENTER(ctx, sysctlgetmibinfo, sname, name, namelenp, cname,
8074	8146	csz, rnode, v);
8075	8147	if (sname)
8076		COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);
	8148	COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, internal_strlen(sname) + 1);
8077	8149	if (namelenp)
8078	8150	COMMON_INTERCEPTOR_READ_RANGE(ctx, namelenp, sizeof(*namelenp));
8079	8151	if (csz)
...	...	@@ -8107,7 +8179,7 @@ INTERCEPTOR(char *, nl_langinfo, long item) {
8107	8179	COMMON_INTERCEPTOR_ENTER(ctx, nl_langinfo, item);
8108	8180	char *ret = REAL(nl_langinfo)(item);
8109	8181	if (ret)
8110		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, REAL(strlen)(ret) + 1);
	8182	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, internal_strlen(ret) + 1);
8111	8183	return ret;
8112	8184	}
8113	8185	#define INIT_NL_LANGINFO COMMON_INTERCEPT_FUNCTION(nl_langinfo)
...	...	@@ -8127,7 +8199,7 @@ INTERCEPTOR(int, modctl, int operation, void *argp) {
8127	8199	COMMON_INTERCEPTOR_READ_RANGE(ctx, ml, sizeof(*ml));
8128	8200	if (ml->ml_filename)
8129	8201	COMMON_INTERCEPTOR_READ_RANGE(ctx, ml->ml_filename,
8130		REAL(strlen)(ml->ml_filename) + 1);
	8202	internal_strlen(ml->ml_filename) + 1);
8131	8203	if (ml->ml_props)
8132	8204	COMMON_INTERCEPTOR_READ_RANGE(ctx, ml->ml_props, ml->ml_propslen);
8133	8205	}
...	...	@@ -8135,7 +8207,7 @@ INTERCEPTOR(int, modctl, int operation, void *argp) {
8135	8207	} else if (operation == modctl_unload) {
8136	8208	if (argp) {
8137	8209	const char name = (const char )argp;
8138		COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);
	8210	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
8139	8211	}
8140	8212	ret = REAL(modctl)(operation, argp);
8141	8213	} else if (operation == modctl_stat) {
...	...	@@ -8177,7 +8249,7 @@ INTERCEPTOR(long long, strtonum, const char *nptr, long long minval,
8177	8249	if (errstr) {
8178	8250	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errstr, sizeof(const char *));
8179	8251	if (*errstr)
8180		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errstr, REAL(strlen)(errstr) + 1);
	8252	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errstr, internal_strlen(errstr) + 1);
8181	8253	}
8182	8254	return ret;
8183	8255	}
...	...	@@ -8197,7 +8269,7 @@ INTERCEPTOR(char , fparseln, __sanitizer_FILE stream, SIZE_T *len,
8197	8269	COMMON_INTERCEPTOR_READ_RANGE(ctx, delim, sizeof(delim[0]) * 3);
8198	8270	char *ret = REAL(fparseln)(stream, len, lineno, delim, flags);
8199	8271	if (ret) {
8200		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, REAL(strlen)(ret) + 1);
	8272	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, internal_strlen(ret) + 1);
8201	8273	if (len)
8202	8274	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len));
8203	8275	if (lineno)
...	...	@@ -8214,7 +8286,7 @@ INTERCEPTOR(char , fparseln, __sanitizer_FILE stream, SIZE_T *len,
8214	8286	INTERCEPTOR(int, statvfs1, const char path, void buf, int flags) {
8215	8287	void *ctx;
8216	8288	COMMON_INTERCEPTOR_ENTER(ctx, statvfs1, path, buf, flags);
8217		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	8289	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
8218	8290	int res = REAL(statvfs1)(path, buf, flags);
8219	8291	if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz);
8220	8292	return res;
...	...	@@ -8495,7 +8567,7 @@ INTERCEPTOR(char , SHA1File, char filename, char *buf) {
8495	8567	void *ctx;
8496	8568	COMMON_INTERCEPTOR_ENTER(ctx, SHA1File, filename, buf);
8497	8569	if (filename)
8498		COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
	8570	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
8499	8571	char *ret = REAL(SHA1File)(filename, buf);
8500	8572	if (ret)
8501	8573	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length);
...	...	@@ -8506,7 +8578,7 @@ INTERCEPTOR(char , SHA1FileChunk, char filename, char *buf, OFF_T offset,
8506	8578	void *ctx;
8507	8579	COMMON_INTERCEPTOR_ENTER(ctx, SHA1FileChunk, filename, buf, offset, length);
8508	8580	if (filename)
8509		COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
	8581	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
8510	8582	char *ret = REAL(SHA1FileChunk)(filename, buf, offset, length);
8511	8583	if (ret)
8512	8584	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length);
...	...	@@ -8582,7 +8654,7 @@ INTERCEPTOR(char , MD4File, const char filename, char *buf) {
8582	8654	void *ctx;
8583	8655	COMMON_INTERCEPTOR_ENTER(ctx, MD4File, filename, buf);
8584	8656	if (filename)
8585		COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
	8657	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
8586	8658	char *ret = REAL(MD4File)(filename, buf);
8587	8659	if (ret)
8588	8660	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD4_return_length);
...	...	@@ -8665,7 +8737,7 @@ INTERCEPTOR(char , RMD160File, char filename, char *buf) {
8665	8737	void *ctx;
8666	8738	COMMON_INTERCEPTOR_ENTER(ctx, RMD160File, filename, buf);
8667	8739	if (filename)
8668		COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
	8740	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
8669	8741	char *ret = REAL(RMD160File)(filename, buf);
8670	8742	if (ret)
8671	8743	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length);
...	...	@@ -8676,7 +8748,7 @@ INTERCEPTOR(char , RMD160FileChunk, char filename, char *buf, OFF_T offset,
8676	8748	void *ctx;
8677	8749	COMMON_INTERCEPTOR_ENTER(ctx, RMD160FileChunk, filename, buf, offset, length);
8678	8750	if (filename)
8679		COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
	8751	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
8680	8752	char *ret = REAL(RMD160FileChunk)(filename, buf, offset, length);
8681	8753	if (ret)
8682	8754	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length);
...	...	@@ -8752,7 +8824,7 @@ INTERCEPTOR(char , MD5File, const char filename, char *buf) {
8752	8824	void *ctx;
8753	8825	COMMON_INTERCEPTOR_ENTER(ctx, MD5File, filename, buf);
8754	8826	if (filename)
8755		COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
	8827	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
8756	8828	char *ret = REAL(MD5File)(filename, buf);
8757	8829	if (ret)
8758	8830	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length);
...	...	@@ -8882,7 +8954,7 @@ INTERCEPTOR(char , MD2File, const char filename, char *buf) {
8882	8954	void *ctx;
8883	8955	COMMON_INTERCEPTOR_ENTER(ctx, MD2File, filename, buf);
8884	8956	if (filename)
8885		COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);
	8957	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
8886	8958	char *ret = REAL(MD2File)(filename, buf);
8887	8959	if (ret)
8888	8960	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD2_return_length);
...	...	@@ -8960,7 +9032,7 @@ INTERCEPTOR(char , MD2Data, const unsigned char data, unsigned int len,
8960	9032	void *ctx; \
8961	9033	COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_File, filename, buf); \
8962	9034	if (filename) \
8963		COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);\
	9035	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);\
8964	9036	char *ret = REAL(SHA##LEN##_File)(filename, buf); \
8965	9037	if (ret) \
8966	9038	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
...	...	@@ -8972,7 +9044,7 @@ INTERCEPTOR(char , MD2Data, const unsigned char data, unsigned int len,
8972	9044	COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_FileChunk, filename, buf, offset, \
8973	9045	length); \
8974	9046	if (filename) \
8975		COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);\
	9047	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);\
8976	9048	char *ret = REAL(SHA##LEN##_FileChunk)(filename, buf, offset, length); \
8977	9049	if (ret) \
8978	9050	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
...	...	@@ -8989,10 +9061,10 @@ INTERCEPTOR(char , MD2Data, const unsigned char data, unsigned int len,
8989	9061	return ret; \
8990	9062	}
8991	9063
8992		SHA2_INTERCEPTORS(224, u32);
8993		SHA2_INTERCEPTORS(256, u32);
8994		SHA2_INTERCEPTORS(384, u64);
8995		SHA2_INTERCEPTORS(512, u64);
	9064	SHA2_INTERCEPTORS(224, u32)
	9065	SHA2_INTERCEPTORS(256, u32)
	9066	SHA2_INTERCEPTORS(384, u64)
	9067	SHA2_INTERCEPTORS(512, u64)
8996	9068
8997	9069	#define INIT_SHA2_INTECEPTORS(LEN) \
8998	9070	COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Init); \
...	...	@@ -9036,7 +9108,7 @@ INTERCEPTOR(int, strvis, char dst, const char src, int flag) {
9036	9108	void *ctx;
9037	9109	COMMON_INTERCEPTOR_ENTER(ctx, strvis, dst, src, flag);
9038	9110	if (src)
9039		COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
	9111	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9040	9112	int len = REAL(strvis)(dst, src, flag);
9041	9113	if (dst)
9042	9114	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1);
...	...	@@ -9046,7 +9118,7 @@ INTERCEPTOR(int, stravis, char *dst, const char src, int flag) {
9046	9118	void *ctx;
9047	9119	COMMON_INTERCEPTOR_ENTER(ctx, stravis, dst, src, flag);
9048	9120	if (src)
9049		COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
	9121	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9050	9122	int len = REAL(stravis)(dst, src, flag);
9051	9123	if (dst) {
9052	9124	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(char *));
...	...	@@ -9059,7 +9131,7 @@ INTERCEPTOR(int, strnvis, char dst, SIZE_T dlen, const char src, int flag) {
9059	9131	void *ctx;
9060	9132	COMMON_INTERCEPTOR_ENTER(ctx, strnvis, dst, dlen, src, flag);
9061	9133	if (src)
9062		COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
	9134	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9063	9135	int len = REAL(strnvis)(dst, dlen, src, flag);
9064	9136	// The interface will be valid even if there is no space for NULL char
9065	9137	if (dst && len > 0)
...	...	@@ -9109,7 +9181,7 @@ INTERCEPTOR(char , svis, char dst, int c, int flag, int nextc,
9109	9181	void *ctx;
9110	9182	COMMON_INTERCEPTOR_ENTER(ctx, svis, dst, c, flag, nextc, extra);
9111	9183	if (extra)
9112		COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
	9184	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
9113	9185	char *end = REAL(svis)(dst, c, flag, nextc, extra);
9114	9186	if (dst && end)
9115	9187	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, end - dst + 1);
...	...	@@ -9120,7 +9192,7 @@ INTERCEPTOR(char , snvis, char dst, SIZE_T dlen, int c, int flag, int nextc,
9120	9192	void *ctx;
9121	9193	COMMON_INTERCEPTOR_ENTER(ctx, snvis, dst, dlen, c, flag, nextc, extra);
9122	9194	if (extra)
9123		COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
	9195	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
9124	9196	char *end = REAL(snvis)(dst, dlen, c, flag, nextc, extra);
9125	9197	if (dst && end)
9126	9198	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst,
...	...	@@ -9132,9 +9204,9 @@ INTERCEPTOR(int, strsvis, char dst, const char src, int flag,
9132	9204	void *ctx;
9133	9205	COMMON_INTERCEPTOR_ENTER(ctx, strsvis, dst, src, flag, extra);
9134	9206	if (src)
9135		COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
	9207	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9136	9208	if (extra)
9137		COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
	9209	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
9138	9210	int len = REAL(strsvis)(dst, src, flag, extra);
9139	9211	if (dst)
9140	9212	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1);
...	...	@@ -9145,9 +9217,9 @@ INTERCEPTOR(int, strsnvis, char dst, SIZE_T dlen, const char src, int flag,
9145	9217	void *ctx;
9146	9218	COMMON_INTERCEPTOR_ENTER(ctx, strsnvis, dst, dlen, src, flag, extra);
9147	9219	if (src)
9148		COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
	9220	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9149	9221	if (extra)
9150		COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
	9222	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
9151	9223	int len = REAL(strsnvis)(dst, dlen, src, flag, extra);
9152	9224	// The interface will be valid even if there is no space for NULL char
9153	9225	if (dst && len >= 0)
...	...	@@ -9161,7 +9233,7 @@ INTERCEPTOR(int, strsvisx, char dst, const char src, SIZE_T len, int flag,
9161	9233	if (src)
9162	9234	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);
9163	9235	if (extra)
9164		COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
	9236	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
9165	9237	int ret = REAL(strsvisx)(dst, src, len, flag, extra);
9166	9238	if (dst)
9167	9239	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
...	...	@@ -9174,7 +9246,7 @@ INTERCEPTOR(int, strsnvisx, char dst, SIZE_T dlen, const char src, SIZE_T len,
9174	9246	if (src)
9175	9247	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);
9176	9248	if (extra)
9177		COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
	9249	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
9178	9250	int ret = REAL(strsnvisx)(dst, dlen, src, len, flag, extra);
9179	9251	if (dst && ret >= 0)
9180	9252	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
...	...	@@ -9188,7 +9260,7 @@ INTERCEPTOR(int, strsenvisx, char dst, SIZE_T dlen, const char src,
9188	9260	if (src)
9189	9261	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);
9190	9262	if (extra)
9191		COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);
	9263	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
9192	9264	// FIXME: only need to be checked when "flag \| VIS_NOLOCALE" doesn't hold
9193	9265	// according to the implementation
9194	9266	if (cerr_ptr)
...	...	@@ -9215,7 +9287,7 @@ INTERCEPTOR(int, strunvis, char dst, const char src) {
9215	9287	void *ctx;
9216	9288	COMMON_INTERCEPTOR_ENTER(ctx, strunvis, dst, src);
9217	9289	if (src)
9218		COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
	9290	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9219	9291	int ret = REAL(strunvis)(dst, src);
9220	9292	if (ret != -1)
9221	9293	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
...	...	@@ -9225,7 +9297,7 @@ INTERCEPTOR(int, strnunvis, char dst, SIZE_T dlen, const char src) {
9225	9297	void *ctx;
9226	9298	COMMON_INTERCEPTOR_ENTER(ctx, strnunvis, dst, dlen, src);
9227	9299	if (src)
9228		COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
	9300	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9229	9301	int ret = REAL(strnunvis)(dst, dlen, src);
9230	9302	if (ret != -1)
9231	9303	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
...	...	@@ -9235,7 +9307,7 @@ INTERCEPTOR(int, strunvisx, char dst, const char src, int flag) {
9235	9307	void *ctx;
9236	9308	COMMON_INTERCEPTOR_ENTER(ctx, strunvisx, dst, src, flag);
9237	9309	if (src)
9238		COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
	9310	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9239	9311	int ret = REAL(strunvisx)(dst, src, flag);
9240	9312	if (ret != -1)
9241	9313	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
...	...	@@ -9246,7 +9318,7 @@ INTERCEPTOR(int, strnunvisx, char dst, SIZE_T dlen, const char src,
9246	9318	void *ctx;
9247	9319	COMMON_INTERCEPTOR_ENTER(ctx, strnunvisx, dst, dlen, src, flag);
9248	9320	if (src)
9249		COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);
	9321	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9250	9322	int ret = REAL(strnunvisx)(dst, dlen, src, flag);
9251	9323	if (ret != -1)
9252	9324	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
...	...	@@ -9282,7 +9354,7 @@ INTERCEPTOR(struct __sanitizer_cdbr , cdbr_open, const char path, int flags) {
9282	9354	void *ctx;
9283	9355	COMMON_INTERCEPTOR_ENTER(ctx, cdbr_open, path, flags);
9284	9356	if (path)
9285		COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	9357	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
9286	9358	struct __sanitizer_cdbr *cdbr = REAL(cdbr_open)(path, flags);
9287	9359	if (cdbr)
9288	9360	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbr, sizeof(*cdbr));
...	...	@@ -9474,7 +9546,7 @@ INTERCEPTOR(void , getfsspec, const char spec) {
9474	9546	void *ctx;
9475	9547	COMMON_INTERCEPTOR_ENTER(ctx, getfsspec, spec);
9476	9548	if (spec)
9477		COMMON_INTERCEPTOR_READ_RANGE(ctx, spec, REAL(strlen)(spec) + 1);
	9549	COMMON_INTERCEPTOR_READ_RANGE(ctx, spec, internal_strlen(spec) + 1);
9478	9550	void *ret = REAL(getfsspec)(spec);
9479	9551	if (ret)
9480	9552	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz);
...	...	@@ -9485,7 +9557,7 @@ INTERCEPTOR(void , getfsfile, const char file) {
9485	9557	void *ctx;
9486	9558	COMMON_INTERCEPTOR_ENTER(ctx, getfsfile, file);
9487	9559	if (file)
9488		COMMON_INTERCEPTOR_READ_RANGE(ctx, file, REAL(strlen)(file) + 1);
	9560	COMMON_INTERCEPTOR_READ_RANGE(ctx, file, internal_strlen(file) + 1);
9489	9561	void *ret = REAL(getfsfile)(file);
9490	9562	if (ret)
9491	9563	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz);
...	...	@@ -9529,9 +9601,9 @@ INTERCEPTOR(__sanitizer_FILE , popen, const char command, const char *type) {
9529	9601	void *ctx;
9530	9602	COMMON_INTERCEPTOR_ENTER(ctx, popen, command, type);
9531	9603	if (command)
9532		COMMON_INTERCEPTOR_READ_RANGE(ctx, command, REAL(strlen)(command) + 1);
	9604	COMMON_INTERCEPTOR_READ_RANGE(ctx, command, internal_strlen(command) + 1);
9533	9605	if (type)
9534		COMMON_INTERCEPTOR_READ_RANGE(ctx, type, REAL(strlen)(type) + 1);
	9606	COMMON_INTERCEPTOR_READ_RANGE(ctx, type, internal_strlen(type) + 1);
9535	9607	__sanitizer_FILE *res = REAL(popen)(command, type);
9536	9608	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, nullptr);
9537	9609	if (res) unpoison_file(res);
...	...	@@ -9548,13 +9620,13 @@ INTERCEPTOR(__sanitizer_FILE , popenve, const char path,
9548	9620	void *ctx;
9549	9621	COMMON_INTERCEPTOR_ENTER(ctx, popenve, path, argv, envp, type);
9550	9622	if (path)
9551		COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	9623	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
9552	9624	if (argv) {
9553	9625	for (char const pa = argv; ; ++pa) {
9554	9626	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **));
9555	9627	if (!*pa)
9556	9628	break;
9557		COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, REAL(strlen)(pa) + 1);
	9629	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, internal_strlen(pa) + 1);
9558	9630	}
9559	9631	}
9560	9632	if (envp) {
...	...	@@ -9562,11 +9634,11 @@ INTERCEPTOR(__sanitizer_FILE , popenve, const char path,
9562	9634	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **));
9563	9635	if (!*pa)
9564	9636	break;
9565		COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, REAL(strlen)(pa) + 1);
	9637	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, internal_strlen(pa) + 1);
9566	9638	}
9567	9639	}
9568	9640	if (type)
9569		COMMON_INTERCEPTOR_READ_RANGE(ctx, type, REAL(strlen)(type) + 1);
	9641	COMMON_INTERCEPTOR_READ_RANGE(ctx, type, internal_strlen(type) + 1);
9570	9642	__sanitizer_FILE *res = REAL(popenve)(path, argv, envp, type);
9571	9643	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, nullptr);
9572	9644	if (res) unpoison_file(res);
...	...	@@ -9762,7 +9834,7 @@ INTERCEPTOR(char *, fdevname, int fd) {
9762	9834	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
9763	9835	char *name = REAL(fdevname)(fd);
9764	9836	if (name) {
9765		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);
	9837	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
9766	9838	if (fd > 0)
9767	9839	COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
9768	9840	}
...	...	@@ -9775,7 +9847,7 @@ INTERCEPTOR(char , fdevname_r, int fd, char buf, SIZE_T len) {
9775	9847	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
9776	9848	char *name = REAL(fdevname_r)(fd, buf, len);
9777	9849	if (name && buf && len > 0) {
9778		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1);
	9850	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
9779	9851	if (fd > 0)
9780	9852	COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
9781	9853	}
...	...	@@ -9795,7 +9867,7 @@ INTERCEPTOR(char *, getusershell) {
9795	9867	COMMON_INTERCEPTOR_ENTER(ctx, getusershell);
9796	9868	char *res = REAL(getusershell)();
9797	9869	if (res)
9798		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
	9870	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
9799	9871	return res;
9800	9872	}
9801	9873
...	...	@@ -9820,7 +9892,7 @@ INTERCEPTOR(int, sl_add, void sl, char item) {
9820	9892	if (sl)
9821	9893	COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);
9822	9894	if (item)
9823		COMMON_INTERCEPTOR_READ_RANGE(ctx, item, REAL(strlen)(item) + 1);
	9895	COMMON_INTERCEPTOR_READ_RANGE(ctx, item, internal_strlen(item) + 1);
9824	9896	int res = REAL(sl_add)(sl, item);
9825	9897	if (!res)
9826	9898	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);
...	...	@@ -9833,10 +9905,10 @@ INTERCEPTOR(char , sl_find, void sl, const char *item) {
9833	9905	if (sl)
9834	9906	COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);
9835	9907	if (item)
9836		COMMON_INTERCEPTOR_READ_RANGE(ctx, item, REAL(strlen)(item) + 1);
	9908	COMMON_INTERCEPTOR_READ_RANGE(ctx, item, internal_strlen(item) + 1);
9837	9909	char *res = REAL(sl_find)(sl, item);
9838	9910	if (res)
9839		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);
	9911	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
9840	9912	return res;
9841	9913	}
9842	9914
...	...	@@ -9872,41 +9944,6 @@ INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) {
9872	9944	#define INIT_GETRANDOM
9873	9945	#endif
9874	9946
9875		#if SANITIZER_INTERCEPT_CRYPT
9876		INTERCEPTOR(char , crypt, char key, char *salt) {
9877		void *ctx;
9878		COMMON_INTERCEPTOR_ENTER(ctx, crypt, key, salt);
9879		COMMON_INTERCEPTOR_READ_RANGE(ctx, key, internal_strlen(key) + 1);
9880		COMMON_INTERCEPTOR_READ_RANGE(ctx, salt, internal_strlen(salt) + 1);
9881		char *res = REAL(crypt)(key, salt);
9882		if (res != nullptr)
9883		COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
9884		return res;
9885		}
9886		#define INIT_CRYPT COMMON_INTERCEPT_FUNCTION(crypt);
9887		#else
9888		#define INIT_CRYPT
9889		#endif
9890
9891		#if SANITIZER_INTERCEPT_CRYPT_R
9892		INTERCEPTOR(char , crypt_r, char key, char salt, void data) {
9893		void *ctx;
9894		COMMON_INTERCEPTOR_ENTER(ctx, crypt_r, key, salt, data);
9895		COMMON_INTERCEPTOR_READ_RANGE(ctx, key, internal_strlen(key) + 1);
9896		COMMON_INTERCEPTOR_READ_RANGE(ctx, salt, internal_strlen(salt) + 1);
9897		char *res = REAL(crypt_r)(key, salt, data);
9898		if (res != nullptr) {
9899		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data,
9900		__sanitizer::struct_crypt_data_sz);
9901		COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
9902		}
9903		return res;
9904		}
9905		#define INIT_CRYPT_R COMMON_INTERCEPT_FUNCTION(crypt_r);
9906		#else
9907		#define INIT_CRYPT_R
9908		#endif
9909
9910	9947	#if SANITIZER_INTERCEPT_GETENTROPY
9911	9948	INTERCEPTOR(int, getentropy, void *buf, SIZE_T buflen) {
9912	9949	void *ctx;
...	...	@@ -9922,7 +9959,52 @@ INTERCEPTOR(int, getentropy, void *buf, SIZE_T buflen) {
9922	9959	#define INIT_GETENTROPY
9923	9960	#endif
9924	9961
9925		#if SANITIZER_INTERCEPT_QSORT
	9962	#if SANITIZER_INTERCEPT_QSORT_R
	9963	typedef int (qsort_r_compar_f)(const void , const void , void );
	9964	struct qsort_r_compar_params {
	9965	SIZE_T size;
	9966	qsort_r_compar_f compar;
	9967	void *arg;
	9968	};
	9969	static int wrapped_qsort_r_compar(const void a, const void b, void *arg) {
	9970	qsort_r_compar_params params = (qsort_r_compar_params )arg;
	9971	COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
	9972	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, params->size);
	9973	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, params->size);
	9974	return params->compar(a, b, params->arg);
	9975	}
	9976
	9977	INTERCEPTOR(void, qsort_r, void *base, SIZE_T nmemb, SIZE_T size,
	9978	qsort_r_compar_f compar, void *arg) {
	9979	void *ctx;
	9980	COMMON_INTERCEPTOR_ENTER(ctx, qsort_r, base, nmemb, size, compar, arg);
	9981	// Run the comparator over all array elements to detect any memory issues.
	9982	if (nmemb > 1) {
	9983	for (SIZE_T i = 0; i < nmemb - 1; ++i) {
	9984	void p = (void )((char )base + i size);
	9985	void q = (void )((char )base + (i + 1) size);
	9986	COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
	9987	compar(p, q, arg);
	9988	}
	9989	}
	9990	qsort_r_compar_params params = {size, compar, arg};
	9991	REAL(qsort_r)(base, nmemb, size, wrapped_qsort_r_compar, &params);
	9992	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);
	9993	}
	9994	# define INIT_QSORT_R COMMON_INTERCEPT_FUNCTION(qsort_r)
	9995	#else
	9996	# define INIT_QSORT_R
	9997	#endif
	9998
	9999	#if SANITIZER_INTERCEPT_QSORT && SANITIZER_INTERCEPT_QSORT_R
	10000	INTERCEPTOR(void, qsort, void *base, SIZE_T nmemb, SIZE_T size,
	10001	qsort_r_compar_f compar) {
	10002	void *ctx;
	10003	COMMON_INTERCEPTOR_ENTER(ctx, qsort, base, nmemb, size, compar);
	10004	WRAP(qsort_r)(base, nmemb, size, compar, nullptr);
	10005	}
	10006	# define INIT_QSORT COMMON_INTERCEPT_FUNCTION(qsort)
	10007	#elif SANITIZER_INTERCEPT_QSORT && !SANITIZER_INTERCEPT_QSORT_R
9926	10008	// Glibc qsort uses a temporary buffer allocated either on stack or on heap.
9927	10009	// Poisoned memory from there may get copied into the comparator arguments,
9928	10010	// where it needs to be dealt with. But even that is not enough - the results of
...	...	@@ -9937,7 +10019,7 @@ INTERCEPTOR(int, getentropy, void *buf, SIZE_T buflen) {
9937	10019	typedef int (qsort_compar_f)(const void , const void *);
9938	10020	static THREADLOCAL qsort_compar_f qsort_compar;
9939	10021	static THREADLOCAL SIZE_T qsort_size;
9940		int wrapped_qsort_compar(const void a, const void b) {
	10022	static int wrapped_qsort_compar(const void a, const void b) {
9941	10023	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
9942	10024	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, qsort_size);
9943	10025	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, qsort_size);
...	...	@@ -9979,60 +10061,34 @@ INTERCEPTOR(void, qsort, void *base, SIZE_T nmemb, SIZE_T size,
9979	10061	}
9980	10062	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);
9981	10063	}
9982		#define INIT_QSORT COMMON_INTERCEPT_FUNCTION(qsort)
	10064	# define INIT_QSORT COMMON_INTERCEPT_FUNCTION(qsort)
9983	10065	#else
9984		#define INIT_QSORT
	10066	# define INIT_QSORT
9985	10067	#endif
9986	10068
9987		#if SANITIZER_INTERCEPT_QSORT_R
9988		typedef int (qsort_r_compar_f)(const void , const void , void );
9989		static THREADLOCAL qsort_r_compar_f qsort_r_compar;
9990		static THREADLOCAL SIZE_T qsort_r_size;
9991		int wrapped_qsort_r_compar(const void a, const void b, void *arg) {
9992		COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
9993		COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, qsort_r_size);
9994		COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, qsort_r_size);
9995		return qsort_r_compar(a, b, arg);
	10069	#if SANITIZER_INTERCEPT_BSEARCH
	10070	typedef int (bsearch_compar_f)(const void , const void *);
	10071	struct bsearch_compar_params {
	10072	const void *key;
	10073	bsearch_compar_f compar;
	10074	};
	10075
	10076	static int wrapped_bsearch_compar(const void key, const void b) {
	10077	const bsearch_compar_params params = (const bsearch_compar_params )key;
	10078	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
	10079	return params->compar(params->key, b);
9996	10080	}
9997	10081
9998		INTERCEPTOR(void, qsort_r, void *base, SIZE_T nmemb, SIZE_T size,
9999		qsort_r_compar_f compar, void *arg) {
	10082	INTERCEPTOR(void , bsearch, const void key, const void *base, SIZE_T nmemb,
	10083	SIZE_T size, bsearch_compar_f compar) {
10000	10084	void *ctx;
10001		COMMON_INTERCEPTOR_ENTER(ctx, qsort_r, base, nmemb, size, compar, arg);
10002		// Run the comparator over all array elements to detect any memory issues.
10003		if (nmemb > 1) {
10004		for (SIZE_T i = 0; i < nmemb - 1; ++i) {
10005		void p = (void )((char )base + i size);
10006		void q = (void )((char )base + (i + 1) size);
10007		COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
10008		compar(p, q, arg);
10009		}
10010		}
10011		qsort_r_compar_f old_compar = qsort_r_compar;
10012		SIZE_T old_size = qsort_r_size;
10013		// Handle qsort_r() implementations that recurse using an
10014		// interposable function call:
10015		bool already_wrapped = compar == wrapped_qsort_r_compar;
10016		if (already_wrapped) {
10017		// This case should only happen if the qsort() implementation calls itself
10018		// using a preemptible function call (e.g. the FreeBSD libc version).
10019		// Check that the size and comparator arguments are as expected.
10020		CHECK_NE(compar, qsort_r_compar);
10021		CHECK_EQ(qsort_r_size, size);
10022		} else {
10023		qsort_r_compar = compar;
10024		qsort_r_size = size;
10025		}
10026		REAL(qsort_r)(base, nmemb, size, wrapped_qsort_r_compar, arg);
10027		if (!already_wrapped) {
10028		qsort_r_compar = old_compar;
10029		qsort_r_size = old_size;
10030		}
10031		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);
	10085	COMMON_INTERCEPTOR_ENTER(ctx, bsearch, key, base, nmemb, size, compar);
	10086	bsearch_compar_params params = {key, compar};
	10087	return REAL(bsearch)(&params, base, nmemb, size, wrapped_bsearch_compar);
10032	10088	}
10033		#define INIT_QSORT_R COMMON_INTERCEPT_FUNCTION(qsort_r)
	10089	# define INIT_BSEARCH COMMON_INTERCEPT_FUNCTION(bsearch)
10034	10090	#else
10035		#define INIT_QSORT_R
	10091	# define INIT_BSEARCH
10036	10092	#endif
10037	10093
10038	10094	#if SANITIZER_INTERCEPT_SIGALTSTACK
...	...	@@ -10050,6 +10106,42 @@ INTERCEPTOR(int, sigaltstack, void ss, void oss) {
10050	10106	#define INIT_SIGALTSTACK
10051	10107	#endif
10052	10108
	10109	#if SANITIZER_INTERCEPT_PROCCTL
	10110	INTERCEPTOR(int, procctl, int idtype, u64 id, int cmd, uptr data) {
	10111	void *ctx;
	10112	COMMON_INTERCEPTOR_ENTER(ctx, procctl, idtype, id, cmd, data);
	10113	static const int PROC_REAP_ACQUIRE = 2;
	10114	static const int PROC_REAP_RELEASE = 3;
	10115	static const int PROC_REAP_STATUS = 4;
	10116	static const int PROC_REAP_GETPIDS = 5;
	10117	static const int PROC_REAP_KILL = 6;
	10118	if (cmd < PROC_REAP_ACQUIRE \|\| cmd > PROC_REAP_KILL) {
	10119	COMMON_INTERCEPTOR_READ_RANGE(ctx, (void *)data, sizeof(int));
	10120	} else {
	10121	// reap_acquire/reap_release bears no arguments.
	10122	if (cmd > PROC_REAP_RELEASE) {
	10123	unsigned int reapsz;
	10124	switch (cmd) {
	10125	case PROC_REAP_STATUS:
	10126	reapsz = struct_procctl_reaper_status_sz;
	10127	break;
	10128	case PROC_REAP_GETPIDS:
	10129	reapsz = struct_procctl_reaper_pids_sz;
	10130	break;
	10131	case PROC_REAP_KILL:
	10132	reapsz = struct_procctl_reaper_kill_sz;
	10133	break;
	10134	}
	10135	COMMON_INTERCEPTOR_READ_RANGE(ctx, (void *)data, reapsz);
	10136	}
	10137	}
	10138	return REAL(procctl)(idtype, id, cmd, data);
	10139	}
	10140	#define INIT_PROCCTL COMMON_INTERCEPT_FUNCTION(procctl)
	10141	#else
	10142	#define INIT_PROCCTL
	10143	#endif
	10144
10053	10145	#if SANITIZER_INTERCEPT_UNAME
10054	10146	INTERCEPTOR(int, uname, struct utsname *utsname) {
10055	10147	#if SANITIZER_LINUX
...	...	@@ -10088,14 +10180,66 @@ INTERCEPTOR(int, __xuname, int size, void *utsname) {
10088	10180	#define INIT___XUNAME
10089	10181	#endif
10090	10182
	10183	#if SANITIZER_INTERCEPT_HEXDUMP
	10184	INTERCEPTOR(void, hexdump, const void ptr, int length, const char header, int flags) {
	10185	void *ctx;
	10186	COMMON_INTERCEPTOR_ENTER(ctx, hexdump, ptr, length, header, flags);
	10187	COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, length);
	10188	COMMON_INTERCEPTOR_READ_RANGE(ctx, header, internal_strlen(header) + 1);
	10189	REAL(hexdump)(ptr, length, header, flags);
	10190	}
	10191
	10192	#define INIT_HEXDUMP COMMON_INTERCEPT_FUNCTION(hexdump);
	10193	#else
	10194	#define INIT_HEXDUMP
	10195	#endif
	10196
	10197	#if SANITIZER_INTERCEPT_ARGP_PARSE
	10198	INTERCEPTOR(int, argp_parse, const struct argp argp, int argc, char *argv,
	10199	unsigned flags, int arg_index, void input) {
	10200	void *ctx;
	10201	COMMON_INTERCEPTOR_ENTER(ctx, argp_parse, argp, argc, argv, flags, arg_index,
	10202	input);
	10203	for (int i = 0; i < argc; i++)
	10204	COMMON_INTERCEPTOR_READ_RANGE(ctx, argv[i], internal_strlen(argv[i]) + 1);
	10205	int res = REAL(argp_parse)(argp, argc, argv, flags, arg_index, input);
	10206	if (!res && arg_index)
	10207	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, arg_index, sizeof(int));
	10208	return res;
	10209	}
	10210
	10211	#define INIT_ARGP_PARSE COMMON_INTERCEPT_FUNCTION(argp_parse);
	10212	#else
	10213	#define INIT_ARGP_PARSE
	10214	#endif
	10215
	10216	#if SANITIZER_INTERCEPT_CPUSET_GETAFFINITY
	10217	INTERCEPTOR(int, cpuset_getaffinity, int level, int which, __int64_t id, SIZE_T cpusetsize, __sanitizer_cpuset_t *mask) {
	10218	void *ctx;
	10219	COMMON_INTERCEPTOR_ENTER(ctx, cpuset_getaffinity, level, which, id, cpusetsize, mask);
	10220	int res = REAL(cpuset_getaffinity)(level, which, id, cpusetsize, mask);
	10221	if (mask && !res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mask, cpusetsize);
	10222	return res;
	10223	}
	10224	#define INIT_CPUSET_GETAFFINITY COMMON_INTERCEPT_FUNCTION(cpuset_getaffinity);
	10225	#else
	10226	#define INIT_CPUSET_GETAFFINITY
	10227	#endif
	10228
10091	10229	#include "sanitizer_common_interceptors_netbsd_compat.inc"
10092	10230
	10231	namespace __sanitizer {
	10232	void InitializeMemintrinsicInterceptors();
	10233	} // namespace __sanitizer
	10234
10093	10235	static void InitializeCommonInterceptors() {
10094	10236	#if SI_POSIX
10095	10237	static u64 metadata_mem[sizeof(MetadataHashMap) / sizeof(u64) + 1];
10096	10238	interceptor_metadata_map = new ((void *)&metadata_mem) MetadataHashMap();
10097	10239	#endif
10098	10240
	10241	__sanitizer::InitializeMemintrinsicInterceptors();
	10242
10099	10243	INIT_MMAP;
10100	10244	INIT_MMAP64;
10101	10245	INIT_TEXTDOMAIN;
...	...	@@ -10117,9 +10261,6 @@ static void InitializeCommonInterceptors() {
10117	10261	INIT_STRPBRK;
10118	10262	INIT_STRXFRM;
10119	10263	INIT___STRXFRM_L;
10120		INIT_MEMSET;
10121		INIT_MEMMOVE;
10122		INIT_MEMCPY;
10123	10264	INIT_MEMCHR;
10124	10265	INIT_MEMCMP;
10125	10266	INIT_BCMP;
...	...	@@ -10166,6 +10307,9 @@ static void InitializeCommonInterceptors() {
10166	10307	INIT_TIME;
10167	10308	INIT_GLOB;
10168	10309	INIT_GLOB64;
	10310	INIT___B64_TO;
	10311	INIT_DN_COMP_EXPAND;
	10312	INIT_POSIX_SPAWN;
10169	10313	INIT_WAIT;
10170	10314	INIT_WAIT4;
10171	10315	INIT_INET;
...	...	@@ -10200,6 +10344,7 @@ static void InitializeCommonInterceptors() {
10200	10344	INIT_GETCWD;
10201	10345	INIT_GET_CURRENT_DIR_NAME;
10202	10346	INIT_STRTOIMAX;
	10347	INIT_STRTOIMAX_C23;
10203	10348	INIT_MBSTOWCS;
10204	10349	INIT_MBSNRTOWCS;
10205	10350	INIT_WCSTOMBS;
...	...	@@ -10231,12 +10376,6 @@ static void InitializeCommonInterceptors() {
10231	10376	INIT_PTHREAD_SIGMASK;
10232	10377	INIT_BACKTRACE;
10233	10378	INIT__EXIT;
10234		INIT_PTHREAD_MUTEX_LOCK;
10235		INIT_PTHREAD_MUTEX_UNLOCK;
10236		INIT___PTHREAD_MUTEX_LOCK;
10237		INIT___PTHREAD_MUTEX_UNLOCK;
10238		INIT___LIBC_MUTEX_LOCK;
10239		INIT___LIBC_MUTEX_UNLOCK;
10240	10379	INIT___LIBC_THR_SETCANCELSTATE;
10241	10380	INIT_GETMNTENT;
10242	10381	INIT_GETMNTENT_R;
...	...	@@ -10254,6 +10393,7 @@ static void InitializeCommonInterceptors() {
10254	10393	INIT_PTHREAD_ATTR_GET_SCHED;
10255	10394	INIT_PTHREAD_ATTR_GETINHERITSCHED;
10256	10395	INIT_PTHREAD_ATTR_GETAFFINITY_NP;
	10396	INIT_PTHREAD_GETAFFINITY_NP;
10257	10397	INIT_PTHREAD_MUTEXATTR_GETPSHARED;
10258	10398	INIT_PTHREAD_MUTEXATTR_GETTYPE;
10259	10399	INIT_PTHREAD_MUTEXATTR_GETPROTOCOL;
...	...	@@ -10293,9 +10433,6 @@ static void InitializeCommonInterceptors() {
10293	10433	INIT_GETIFADDRS;
10294	10434	INIT_IF_INDEXTONAME;
10295	10435	INIT_CAPGET;
10296		INIT_AEABI_MEM;
10297		INIT___BZERO;
10298		INIT_BZERO;
10299	10436	INIT_FTIME;
10300	10437	INIT_XDR;
10301	10438	INIT_XDRREC_LINUX;
...	...	@@ -10322,8 +10459,10 @@ static void InitializeCommonInterceptors() {
10322	10459	INIT_RECV_RECVFROM;
10323	10460	INIT_SEND_SENDTO;
10324	10461	INIT_STAT;
	10462	INIT_STAT64;
10325	10463	INIT_EVENTFD_READ_WRITE;
10326	10464	INIT_LSTAT;
	10465	INIT_LSTAT64;
10327	10466	INIT___XSTAT;
10328	10467	INIT___XSTAT64;
10329	10468	INIT___LXSTAT;
...	...	@@ -10396,14 +10535,17 @@ static void InitializeCommonInterceptors() {
10396	10535	INIT_GETUSERSHELL;
10397	10536	INIT_SL_INIT;
10398	10537	INIT_GETRANDOM;
10399		INIT_CRYPT;
10400		INIT_CRYPT_R;
10401	10538	INIT_GETENTROPY;
10402	10539	INIT_QSORT;
10403	10540	INIT_QSORT_R;
	10541	INIT_BSEARCH;
10404	10542	INIT_SIGALTSTACK;
	10543	INIT_PROCCTL
10405	10544	INIT_UNAME;
10406	10545	INIT___XUNAME;
	10546	INIT_HEXDUMP;
	10547	INIT_ARGP_PARSE;
	10548	INIT_CPUSET_GETAFFINITY;
10407	10549
10408	10550	INIT___PRINTF_CHK;
10409	10551	}

lib/tsan/sanitizer_common/sanitizer_common_interceptors_format.inc+17-9

...	...	@@ -324,8 +324,8 @@ static void scanf_common(void *ctx, int n_inputs, bool allowGnuMalloc,
324	324	continue;
325	325	int size = scanf_get_value_size(&dir);
326	326	if (size == FSS_INVALID) {
327		Report("%s: WARNING: unexpected format specifier in scanf interceptor: ",
328		SanitizerToolName, "%.*s\n", dir.end - dir.begin, dir.begin);
	327	Report("%s: WARNING: unexpected format specifier in scanf interceptor: %.*s\n",
	328	SanitizerToolName, static_cast<int>(dir.end - dir.begin), dir.begin);
329	329	break;
330	330	}
331	331	void argp = va_arg(aq, void );
...	...	@@ -340,11 +340,19 @@ static void scanf_common(void *ctx, int n_inputs, bool allowGnuMalloc,
340	340	size = 0;
341	341	}
342	342	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, argp, size);
343		// For %ms/%mc, write the allocated output buffer as well.
	343	// For %mc/%mC/%ms/%m[/%mS, write the allocated output buffer as well.
344	344	if (dir.allocate) {
345		char buf = (char **)argp;
346		if (buf)
347		COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
	345	if (char buf = (char **)argp) {
	346	if (dir.convSpecifier == 'c')
	347	size = 1;
	348	else if (dir.convSpecifier == 'C')
	349	size = sizeof(wchar_t);
	350	else if (dir.convSpecifier == 'S')
	351	size = (internal_wcslen((wchar_t )buf) + 1) sizeof(wchar_t);
	352	else // 's' or '['
	353	size = internal_strlen(buf) + 1;
	354	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, size);
	355	}
348	356	}
349	357	}
350	358	}
...	...	@@ -469,7 +477,7 @@ static int printf_get_value_size(PrintfDirective *dir) {
469	477	break; \
470	478	default: \
471	479	Report("WARNING: unexpected floating-point arg size" \
472		" in printf interceptor: %d\n", size); \
	480	" in printf interceptor: %zu\n", static_cast<uptr>(size)); \
473	481	return; \
474	482	} \
475	483	} else { \
...	...	@@ -484,7 +492,7 @@ static int printf_get_value_size(PrintfDirective *dir) {
484	492	break; \
485	493	default: \
486	494	Report("WARNING: unexpected arg size" \
487		" in printf interceptor: %d\n", size); \
	495	" in printf interceptor: %zu\n", static_cast<uptr>(size)); \
488	496	return; \
489	497	} \
490	498	} \
...	...	@@ -530,7 +538,7 @@ static void printf_common(void ctx, const char format, va_list aq) {
530	538	Report(
531	539	"%s: WARNING: unexpected format specifier in printf "
532	540	"interceptor: %.*s (reported once per process)\n",
533		SanitizerToolName, dir.end - dir.begin, dir.begin);
	541	SanitizerToolName, static_cast<int>(dir.end - dir.begin), dir.begin);
534	542	break;
535	543	}
536	544	if (dir.convSpecifier == 'n') {

lib/tsan/sanitizer_common/sanitizer_common_interceptors_ioctl.inc+9-1

...	...	@@ -115,11 +115,19 @@ static void ioctl_table_fill() {
115	115	// _(SOUND_MIXER_WRITE_MUTE, WRITE, sizeof(int)); // same as ...WRITE_ENHANCE
116	116	_(BLKFLSBUF, NONE, 0);
117	117	_(BLKGETSIZE, WRITE, sizeof(uptr));
118		_(BLKRAGET, WRITE, sizeof(int));
	118	_(BLKRAGET, WRITE, sizeof(uptr));
119	119	_(BLKRASET, NONE, 0);
120	120	_(BLKROGET, WRITE, sizeof(int));
121	121	_(BLKROSET, READ, sizeof(int));
122	122	_(BLKRRPART, NONE, 0);
	123	_(BLKFRASET, NONE, 0);
	124	_(BLKFRAGET, WRITE, sizeof(uptr));
	125	_(BLKSECTSET, READ, sizeof(short));
	126	_(BLKSECTGET, WRITE, sizeof(short));
	127	_(BLKSSZGET, WRITE, sizeof(int));
	128	_(BLKBSZGET, WRITE, sizeof(int));
	129	_(BLKBSZSET, READ, sizeof(uptr));
	130	_(BLKGETSIZE64, WRITE, sizeof(u64));
123	131	_(CDROMEJECT, NONE, 0);
124	132	_(CDROMEJECT_SW, NONE, 0);
125	133	_(CDROMMULTISESSION, WRITE, struct_cdrom_multisession_sz);

lib/tsan/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc created+244

...	...	@@ -0,0 +1,244 @@
	1	//===-- sanitizer_common_interceptors_memintrinsics.inc ---------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// Memintrinsic function interceptors for tools like AddressSanitizer,
	10	// ThreadSanitizer, MemorySanitizer, etc.
	11	//
	12	// These interceptors are part of the common interceptors, but separated out so
	13	// that implementations may add them, if necessary, to a separate source file
	14	// that should define SANITIZER_COMMON_NO_REDEFINE_BUILTINS at the top.
	15	//
	16	// This file should be included into the tool's memintrinsic interceptor file,
	17	// which has to define its own macros:
	18	// COMMON_INTERCEPTOR_ENTER
	19	// COMMON_INTERCEPTOR_READ_RANGE
	20	// COMMON_INTERCEPTOR_WRITE_RANGE
	21	// COMMON_INTERCEPTOR_MEMSET_IMPL
	22	// COMMON_INTERCEPTOR_MEMMOVE_IMPL
	23	// COMMON_INTERCEPTOR_MEMCPY_IMPL
	24	// COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED
	25	//===----------------------------------------------------------------------===//
	26
	27	#ifdef SANITIZER_REDEFINE_BUILTINS_H
	28	#error "Define SANITIZER_COMMON_NO_REDEFINE_BUILTINS in .cpp file"
	29	#endif
	30
	31	#include "interception/interception.h"
	32	#include "sanitizer_platform_interceptors.h"
	33
	34	// Platform-specific options.
	35	#if SANITIZER_APPLE
	36	#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
	37	#elif SANITIZER_WINDOWS64
	38	#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
	39	#else
	40	#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1
	41	#endif // SANITIZER_APPLE
	42
	43	#ifndef COMMON_INTERCEPTOR_MEMSET_IMPL
	44	#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \
	45	{ \
	46	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \
	47	return internal_memset(dst, v, size); \
	48	COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \
	49	if (common_flags()->intercept_intrin) \
	50	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
	51	return REAL(memset)(dst, v, size); \
	52	}
	53	#endif
	54
	55	#ifndef COMMON_INTERCEPTOR_MEMMOVE_IMPL
	56	#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size) \
	57	{ \
	58	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \
	59	return internal_memmove(dst, src, size); \
	60	COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size); \
	61	if (common_flags()->intercept_intrin) { \
	62	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
	63	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \
	64	} \
	65	return REAL(memmove)(dst, src, size); \
	66	}
	67	#endif
	68
	69	#ifndef COMMON_INTERCEPTOR_MEMCPY_IMPL
	70	#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size) \
	71	{ \
	72	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) { \
	73	return internal_memmove(dst, src, size); \
	74	} \
	75	COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size); \
	76	if (common_flags()->intercept_intrin) { \
	77	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
	78	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \
	79	} \
	80	return REAL(memcpy)(dst, src, size); \
	81	}
	82	#endif
	83
	84	#if SANITIZER_INTERCEPT_MEMSET
	85	INTERCEPTOR(void , memset, void dst, int v, uptr size) {
	86	void *ctx;
	87	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size);
	88	}
	89
	90	#define INIT_MEMSET COMMON_INTERCEPT_FUNCTION(memset)
	91	#else
	92	#define INIT_MEMSET
	93	#endif
	94
	95	#if SANITIZER_INTERCEPT_MEMMOVE
	96	INTERCEPTOR(void , memmove, void dst, const void *src, uptr size) {
	97	void *ctx;
	98	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
	99	}
	100
	101	#define INIT_MEMMOVE COMMON_INTERCEPT_FUNCTION(memmove)
	102	#else
	103	#define INIT_MEMMOVE
	104	#endif
	105
	106	#if SANITIZER_INTERCEPT_MEMCPY
	107	INTERCEPTOR(void , memcpy, void dst, const void *src, uptr size) {
	108	// On OS X, calling internal_memcpy here will cause memory corruptions,
	109	// because memcpy and memmove are actually aliases of the same
	110	// implementation. We need to use internal_memmove here.
	111	// N.B.: If we switch this to internal_ we'll have to use internal_memmove
	112	// due to memcpy being an alias of memmove on OS X.
	113	void *ctx;
	114	#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE
	115	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
	116	#else
	117	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
	118	#endif
	119	}
	120
	121	#define INIT_MEMCPY \
	122	do { \
	123	if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { \
	124	COMMON_INTERCEPT_FUNCTION(memcpy); \
	125	} else { \
	126	ASSIGN_REAL(memcpy, memmove); \
	127	} \
	128	CHECK(REAL(memcpy)); \
	129	} while (false)
	130
	131	#else
	132	#define INIT_MEMCPY
	133	#endif
	134
	135	#if SANITIZER_INTERCEPT_AEABI_MEM
	136	INTERCEPTOR(void , __aeabi_memmove, void to, const void *from, uptr size) {
	137	void *ctx;
	138	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
	139	}
	140
	141	INTERCEPTOR(void , __aeabi_memmove4, void to, const void *from, uptr size) {
	142	void *ctx;
	143	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
	144	}
	145
	146	INTERCEPTOR(void , __aeabi_memmove8, void to, const void *from, uptr size) {
	147	void *ctx;
	148	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
	149	}
	150
	151	INTERCEPTOR(void , __aeabi_memcpy, void to, const void *from, uptr size) {
	152	void *ctx;
	153	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
	154	}
	155
	156	INTERCEPTOR(void , __aeabi_memcpy4, void to, const void *from, uptr size) {
	157	void *ctx;
	158	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
	159	}
	160
	161	INTERCEPTOR(void , __aeabi_memcpy8, void to, const void *from, uptr size) {
	162	void *ctx;
	163	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
	164	}
	165
	166	// Note the argument order.
	167	INTERCEPTOR(void , __aeabi_memset, void block, uptr size, int c) {
	168	void *ctx;
	169	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
	170	}
	171
	172	INTERCEPTOR(void , __aeabi_memset4, void block, uptr size, int c) {
	173	void *ctx;
	174	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
	175	}
	176
	177	INTERCEPTOR(void , __aeabi_memset8, void block, uptr size, int c) {
	178	void *ctx;
	179	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
	180	}
	181
	182	INTERCEPTOR(void , __aeabi_memclr, void block, uptr size) {
	183	void *ctx;
	184	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
	185	}
	186
	187	INTERCEPTOR(void , __aeabi_memclr4, void block, uptr size) {
	188	void *ctx;
	189	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
	190	}
	191
	192	INTERCEPTOR(void , __aeabi_memclr8, void block, uptr size) {
	193	void *ctx;
	194	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
	195	}
	196
	197	#define INIT_AEABI_MEM \
	198	COMMON_INTERCEPT_FUNCTION(__aeabi_memmove); \
	199	COMMON_INTERCEPT_FUNCTION(__aeabi_memmove4); \
	200	COMMON_INTERCEPT_FUNCTION(__aeabi_memmove8); \
	201	COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy); \
	202	COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy4); \
	203	COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy8); \
	204	COMMON_INTERCEPT_FUNCTION(__aeabi_memset); \
	205	COMMON_INTERCEPT_FUNCTION(__aeabi_memset4); \
	206	COMMON_INTERCEPT_FUNCTION(__aeabi_memset8); \
	207	COMMON_INTERCEPT_FUNCTION(__aeabi_memclr); \
	208	COMMON_INTERCEPT_FUNCTION(__aeabi_memclr4); \
	209	COMMON_INTERCEPT_FUNCTION(__aeabi_memclr8);
	210	#else
	211	#define INIT_AEABI_MEM
	212	#endif // SANITIZER_INTERCEPT_AEABI_MEM
	213
	214	#if SANITIZER_INTERCEPT___BZERO
	215	INTERCEPTOR(void , __bzero, void block, uptr size) {
	216	void *ctx;
	217	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
	218	}
	219	#define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero);
	220	#else
	221	#define INIT___BZERO
	222	#endif // SANITIZER_INTERCEPT___BZERO
	223
	224	#if SANITIZER_INTERCEPT_BZERO
	225	INTERCEPTOR(void , bzero, void block, uptr size) {
	226	void *ctx;
	227	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
	228	}
	229	#define INIT_BZERO COMMON_INTERCEPT_FUNCTION(bzero);
	230	#else
	231	#define INIT_BZERO
	232	#endif // SANITIZER_INTERCEPT_BZERO
	233
	234	namespace __sanitizer {
	235	// This does not need to be called if InitializeCommonInterceptors() is called.
	236	void InitializeMemintrinsicInterceptors() {
	237	INIT_MEMSET;
	238	INIT_MEMMOVE;
	239	INIT_MEMCPY;
	240	INIT_AEABI_MEM;
	241	INIT___BZERO;
	242	INIT_BZERO;
	243	}
	244	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_common_interceptors_netbsd_compat.inc+2-2

...	...	@@ -33,7 +33,7 @@
33	33	INTERCEPTOR(int, statvfs, char path, void buf) {
34	34	void *ctx;
35	35	COMMON_INTERCEPTOR_ENTER(ctx, statvfs, path, buf);
36		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	36	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
37	37	// FIXME: under ASan the call below may write to freed memory and corrupt
38	38	// its metadata. See
39	39	// https://github.com/google/sanitizers/issues/321.
...	...	@@ -99,7 +99,7 @@ INTERCEPTOR(int, getvfsstat, void *buf, SIZE_T bufsize, int flags) {
99	99	INTERCEPTOR(int, statvfs1, const char path, void buf, int flags) {
100	100	void *ctx;
101	101	COMMON_INTERCEPTOR_ENTER(ctx, statvfs1, path, buf, flags);
102		if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);
	102	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
103	103	int res = REAL(statvfs1)(path, buf, flags);
104	104	if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs90_sz);
105	105	return res;

lib/tsan/sanitizer_common/sanitizer_common_interface.inc+10

...	...	@@ -9,12 +9,16 @@
9	9	//===----------------------------------------------------------------------===//
10	10	INTERFACE_FUNCTION(__sanitizer_acquire_crash_state)
11	11	INTERFACE_FUNCTION(__sanitizer_annotate_contiguous_container)
	12	INTERFACE_FUNCTION(__sanitizer_annotate_double_ended_contiguous_container)
12	13	INTERFACE_FUNCTION(__sanitizer_contiguous_container_find_bad_address)
	14	INTERFACE_FUNCTION(
	15	__sanitizer_double_ended_contiguous_container_find_bad_address)
13	16	INTERFACE_FUNCTION(__sanitizer_set_death_callback)
14	17	INTERFACE_FUNCTION(__sanitizer_set_report_path)
15	18	INTERFACE_FUNCTION(__sanitizer_set_report_fd)
16	19	INTERFACE_FUNCTION(__sanitizer_get_report_path)
17	20	INTERFACE_FUNCTION(__sanitizer_verify_contiguous_container)
	21	INTERFACE_FUNCTION(__sanitizer_verify_double_ended_contiguous_container)
18	22	INTERFACE_WEAK_FUNCTION(__sanitizer_on_print)
19	23	INTERFACE_WEAK_FUNCTION(__sanitizer_report_error_summary)
20	24	INTERFACE_WEAK_FUNCTION(__sanitizer_sandbox_on_notify)
...	...	@@ -28,7 +32,9 @@ INTERFACE_FUNCTION(__sanitizer_get_module_and_offset_for_pc)
28	32	INTERFACE_FUNCTION(__sanitizer_symbolize_global)
29	33	INTERFACE_FUNCTION(__sanitizer_symbolize_pc)
30	34	// Allocator interface.
	35	INTERFACE_FUNCTION(__sanitizer_get_allocated_begin)
31	36	INTERFACE_FUNCTION(__sanitizer_get_allocated_size)
	37	INTERFACE_FUNCTION(__sanitizer_get_allocated_size_fast)
32	38	INTERFACE_FUNCTION(__sanitizer_get_current_allocated_bytes)
33	39	INTERFACE_FUNCTION(__sanitizer_get_estimated_allocated_size)
34	40	INTERFACE_FUNCTION(__sanitizer_get_free_bytes)
...	...	@@ -40,3 +46,7 @@ INTERFACE_FUNCTION(__sanitizer_purge_allocator)
40	46	INTERFACE_FUNCTION(__sanitizer_print_memory_profile)
41	47	INTERFACE_WEAK_FUNCTION(__sanitizer_free_hook)
42	48	INTERFACE_WEAK_FUNCTION(__sanitizer_malloc_hook)
	49	// Memintrinsic functions.
	50	INTERFACE_FUNCTION(__sanitizer_internal_memcpy)
	51	INTERFACE_FUNCTION(__sanitizer_internal_memmove)
	52	INTERFACE_FUNCTION(__sanitizer_internal_memset)

lib/tsan/sanitizer_common/sanitizer_common_interface_posix.inc+2

...	...	@@ -11,3 +11,5 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_code)
11	11	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_data)
12	12	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_demangle)
13	13	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_flush)
	14	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_set_demangle)
	15	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_set_inline_frames)

lib/tsan/sanitizer_common/sanitizer_common_libcdep.cpp+63-36

...	...	@@ -10,27 +10,22 @@
10	10	// run-time libraries.
11	11	//===----------------------------------------------------------------------===//
12	12
	13	#include "sanitizer_allocator.h"
13	14	#include "sanitizer_allocator_interface.h"
14	15	#include "sanitizer_common.h"
15	16	#include "sanitizer_flags.h"
	17	#include "sanitizer_interface_internal.h"
16	18	#include "sanitizer_procmaps.h"
17
	19	#include "sanitizer_stackdepot.h"
18	20
19	21	namespace __sanitizer {
20	22
21		static void (*SoftRssLimitExceededCallback)(bool exceeded);
22		void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)) {
23		CHECK_EQ(SoftRssLimitExceededCallback, nullptr);
24		SoftRssLimitExceededCallback = Callback;
25		}
26
27	23	#if (SANITIZER_LINUX \|\| SANITIZER_NETBSD) && !SANITIZER_GO
28	24	// Weak default implementation for when sanitizer_stackdepot is not linked in.
29		SANITIZER_WEAK_ATTRIBUTE StackDepotStats *StackDepotGetStats() {
30		return nullptr;
31		}
	25	SANITIZER_WEAK_ATTRIBUTE StackDepotStats StackDepotGetStats() { return {}; }
32	26
33	27	void BackgroundThread(void arg) {
	28	VPrintf(1, "%s: Started BackgroundThread\n", SanitizerToolName);
34	29	const uptr hard_rss_limit_mb = common_flags()->hard_rss_limit_mb;
35	30	const uptr soft_rss_limit_mb = common_flags()->soft_rss_limit_mb;
36	31	const bool heap_profile = common_flags()->heap_profile;
...	...	@@ -48,16 +43,12 @@ void BackgroundThread(void arg) {
48	43	prev_reported_rss = current_rss_mb;
49	44	}
50	45	// If stack depot has grown 10% since last time, print it too.
51		StackDepotStats *stack_depot_stats = StackDepotGetStats();
52		if (stack_depot_stats) {
53		if (prev_reported_stack_depot_size * 11 / 10 <
54		stack_depot_stats->allocated) {
55		Printf("%s: StackDepot: %zd ids; %zdM allocated\n",
56		SanitizerToolName,
57		stack_depot_stats->n_uniq_ids,
58		stack_depot_stats->allocated >> 20);
59		prev_reported_stack_depot_size = stack_depot_stats->allocated;
60		}
	46	StackDepotStats stack_depot_stats = StackDepotGetStats();
	47	if (prev_reported_stack_depot_size * 11 / 10 <
	48	stack_depot_stats.allocated) {
	49	Printf("%s: StackDepot: %zd ids; %zdM allocated\n", SanitizerToolName,
	50	stack_depot_stats.n_uniq_ids, stack_depot_stats.allocated >> 20);
	51	prev_reported_stack_depot_size = stack_depot_stats.allocated;
61	52	}
62	53	}
63	54	// Check RSS against the limit.
...	...	@@ -72,13 +63,13 @@ void BackgroundThread(void arg) {
72	63	reached_soft_rss_limit = true;
73	64	Report("%s: soft rss limit exhausted (%zdMb vs %zdMb)\n",
74	65	SanitizerToolName, soft_rss_limit_mb, current_rss_mb);
75		if (SoftRssLimitExceededCallback)
76		SoftRssLimitExceededCallback(true);
	66	SetRssLimitExceeded(true);
77	67	} else if (soft_rss_limit_mb >= current_rss_mb &&
78	68	reached_soft_rss_limit) {
79	69	reached_soft_rss_limit = false;
80		if (SoftRssLimitExceededCallback)
81		SoftRssLimitExceededCallback(false);
	70	Report("%s: soft rss limit unexhausted (%zdMb vs %zdMb)\n",
	71	SanitizerToolName, soft_rss_limit_mb, current_rss_mb);
	72	SetRssLimitExceeded(false);
82	73	}
83	74	}
84	75	if (heap_profile &&
...	...	@@ -89,6 +80,42 @@ void BackgroundThread(void arg) {
89	80	}
90	81	}
91	82	}
	83
	84	void MaybeStartBackgroudThread() {
	85	// Need to implement/test on other platforms.
	86	// Start the background thread if one of the rss limits is given.
	87	if (!common_flags()->hard_rss_limit_mb &&
	88	!common_flags()->soft_rss_limit_mb &&
	89	!common_flags()->heap_profile) return;
	90	if (!&real_pthread_create) {
	91	VPrintf(1, "%s: real_pthread_create undefined\n", SanitizerToolName);
	92	return; // Can't spawn the thread anyway.
	93	}
	94
	95	static bool started = false;
	96	if (!started) {
	97	started = true;
	98	internal_start_thread(BackgroundThread, nullptr);
	99	}
	100	}
	101
	102	# if !SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL
	103	# ifdef __clang__
	104	# pragma clang diagnostic push
	105	// We avoid global-constructors to be sure that globals are ready when
	106	// sanitizers need them. This can happend before global constructors executed.
	107	// Here we don't mind if thread is started on later stages.
	108	# pragma clang diagnostic ignored "-Wglobal-constructors"
	109	# endif
	110	static struct BackgroudThreadStarted {
	111	BackgroudThreadStarted() { MaybeStartBackgroudThread(); }
	112	} background_thread_strarter UNUSED;
	113	# ifdef __clang__
	114	# pragma clang diagnostic pop
	115	# endif
	116	# endif
	117	#else
	118	void MaybeStartBackgroudThread() {}
92	119	#endif
93	120
94	121	void WriteToSyslog(const char *msg) {
...	...	@@ -111,18 +138,6 @@ void WriteToSyslog(const char *msg) {
111	138	WriteOneLineToSyslog(p);
112	139	}
113	140
114		void MaybeStartBackgroudThread() {
115		#if (SANITIZER_LINUX \|\| SANITIZER_NETBSD) && \
116		!SANITIZER_GO // Need to implement/test on other platforms.
117		// Start the background thread if one of the rss limits is given.
118		if (!common_flags()->hard_rss_limit_mb &&
119		!common_flags()->soft_rss_limit_mb &&
120		!common_flags()->heap_profile) return;
121		if (!&real_pthread_create) return; // Can't spawn the thread anyway.
122		internal_start_thread(BackgroundThread, nullptr);
123		#endif
124		}
125
126	141	static void (*sandboxing_callback)();
127	142	void SetSandboxingCallback(void (*f)()) {
128	143	sandboxing_callback = f;
...	...	@@ -191,10 +206,22 @@ void ProtectGap(uptr addr, uptr size, uptr zero_base_shadow_start,
191	206
192	207	#endif // !SANITIZER_FUCHSIA
193	208
	209	#if !SANITIZER_WINDOWS && !SANITIZER_GO
	210	// Weak default implementation for when sanitizer_stackdepot is not linked in.
	211	SANITIZER_WEAK_ATTRIBUTE void StackDepotStopBackgroundThread() {}
	212	static void StopStackDepotBackgroundThread() {
	213	StackDepotStopBackgroundThread();
	214	}
	215	#else
	216	// SANITIZER_WEAK_ATTRIBUTE is unsupported.
	217	static void StopStackDepotBackgroundThread() {}
	218	#endif
	219
194	220	} // namespace __sanitizer
195	221
196	222	SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify,
197	223	__sanitizer_sandbox_arguments *args) {
	224	__sanitizer::StopStackDepotBackgroundThread();
198	225	__sanitizer::PlatformPrepareForSandboxing(args);
199	226	if (__sanitizer::sandboxing_callback)
200	227	__sanitizer::sandboxing_callback();

lib/tsan/sanitizer_common/sanitizer_common_nolibc.cpp+2-1

...	...	@@ -25,9 +25,10 @@ void LogMessageOnPrintf(const char *str) {}
25	25	#endif
26	26	void WriteToSyslog(const char *buffer) {}
27	27	void Abort() { internal__exit(1); }
	28	bool CreateDir(const char *pathname) { return false; }
28	29	#endif // !SANITIZER_WINDOWS
29	30
30		#if !SANITIZER_WINDOWS && !SANITIZER_MAC
	31	#if !SANITIZER_WINDOWS && !SANITIZER_APPLE
31	32	void ListOfModules::init() {}
32	33	void InitializePlatformCommonFlags(CommonFlags *cf) {}
33	34	#endif

lib/tsan/sanitizer_common/sanitizer_common_syscalls.inc+921-658

...	...	@@ -43,45 +43,47 @@
43	43	#include "sanitizer_platform.h"
44	44	#if SANITIZER_LINUX
45	45
46		#include "sanitizer_libc.h"
	46	# include "sanitizer_libc.h"
47	47
48		#define PRE_SYSCALL(name) \
49		SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_pre_impl_##name
50		#define PRE_READ(p, s) COMMON_SYSCALL_PRE_READ_RANGE(p, s)
51		#define PRE_WRITE(p, s) COMMON_SYSCALL_PRE_WRITE_RANGE(p, s)
	48	# define PRE_SYSCALL(name) \
	49	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_pre_impl_##name
	50	# define PRE_READ(p, s) COMMON_SYSCALL_PRE_READ_RANGE(p, s)
	51	# define PRE_WRITE(p, s) COMMON_SYSCALL_PRE_WRITE_RANGE(p, s)
52	52
53		#define POST_SYSCALL(name) \
54		SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_post_impl_##name
55		#define POST_READ(p, s) COMMON_SYSCALL_POST_READ_RANGE(p, s)
56		#define POST_WRITE(p, s) COMMON_SYSCALL_POST_WRITE_RANGE(p, s)
	53	# define POST_SYSCALL(name) \
	54	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_post_impl_##name
	55	# define POST_READ(p, s) COMMON_SYSCALL_POST_READ_RANGE(p, s)
	56	# define POST_WRITE(p, s) COMMON_SYSCALL_POST_WRITE_RANGE(p, s)
57	57
58		#ifndef COMMON_SYSCALL_ACQUIRE
59		# define COMMON_SYSCALL_ACQUIRE(addr) ((void)(addr))
60		#endif
	58	# ifndef COMMON_SYSCALL_ACQUIRE
	59	# define COMMON_SYSCALL_ACQUIRE(addr) ((void)(addr))
	60	# endif
61	61
62		#ifndef COMMON_SYSCALL_RELEASE
63		# define COMMON_SYSCALL_RELEASE(addr) ((void)(addr))
64		#endif
	62	# ifndef COMMON_SYSCALL_RELEASE
	63	# define COMMON_SYSCALL_RELEASE(addr) ((void)(addr))
	64	# endif
65	65
66		#ifndef COMMON_SYSCALL_FD_CLOSE
67		# define COMMON_SYSCALL_FD_CLOSE(fd) ((void)(fd))
68		#endif
	66	# ifndef COMMON_SYSCALL_FD_CLOSE
	67	# define COMMON_SYSCALL_FD_CLOSE(fd) ((void)(fd))
	68	# endif
69	69
70		#ifndef COMMON_SYSCALL_FD_ACQUIRE
71		# define COMMON_SYSCALL_FD_ACQUIRE(fd) ((void)(fd))
72		#endif
	70	# ifndef COMMON_SYSCALL_FD_ACQUIRE
	71	# define COMMON_SYSCALL_FD_ACQUIRE(fd) ((void)(fd))
	72	# endif
73	73
74		#ifndef COMMON_SYSCALL_FD_RELEASE
75		# define COMMON_SYSCALL_FD_RELEASE(fd) ((void)(fd))
76		#endif
	74	# ifndef COMMON_SYSCALL_FD_RELEASE
	75	# define COMMON_SYSCALL_FD_RELEASE(fd) ((void)(fd))
	76	# endif
77	77
78		#ifndef COMMON_SYSCALL_PRE_FORK
79		# define COMMON_SYSCALL_PRE_FORK() {}
80		#endif
	78	# ifndef COMMON_SYSCALL_PRE_FORK
	79	# define COMMON_SYSCALL_PRE_FORK() \
	80	{}
	81	# endif
81	82
82		#ifndef COMMON_SYSCALL_POST_FORK
83		# define COMMON_SYSCALL_POST_FORK(res) {}
84		#endif
	83	# ifndef COMMON_SYSCALL_POST_FORK
	84	# define COMMON_SYSCALL_POST_FORK(res) \
	85	{}
	86	# endif
85	87
86	88	// FIXME: do some kind of PRE_READ for all syscall arguments (int(s) and such).
87	89
...	...	@@ -130,8 +132,8 @@ struct sanitizer_kernel_sockaddr {
130	132	// Declare it "void" to catch sizeof(kernel_sigset_t).
131	133	typedef void kernel_sigset_t;
132	134
133		static void kernel_write_iovec(const __sanitizer_iovec *iovec,
134		SIZE_T iovlen, SIZE_T maxlen) {
	135	static void kernel_write_iovec(const __sanitizer_iovec *iovec, SIZE_T iovlen,
	136	SIZE_T maxlen) {
135	137	for (SIZE_T i = 0; i < iovlen && maxlen; ++i) {
136	138	SSIZE_T sz = Min(iovec[i].iov_len, maxlen);
137	139	POST_WRITE(iovec[i].iov_base, sz);
...	...	@@ -141,8 +143,8 @@ static void kernel_write_iovec(const __sanitizer_iovec *iovec,
141	143
142	144	// This functions uses POST_READ, because it needs to run after syscall to know
143	145	// the real read range.
144		static void kernel_read_iovec(const __sanitizer_iovec *iovec,
145		SIZE_T iovlen, SIZE_T maxlen) {
	146	static void kernel_read_iovec(const __sanitizer_iovec *iovec, SIZE_T iovlen,
	147	SIZE_T maxlen) {
146	148	POST_READ(iovec, sizeof(iovec) iovlen);
147	149	for (SIZE_T i = 0; i < iovlen && maxlen; ++i) {
148	150	SSIZE_T sz = Min(iovec[i].iov_len, maxlen);
...	...	@@ -155,8 +157,8 @@ PRE_SYSCALL(recvmsg)(long sockfd, sanitizer_kernel_msghdr *msg, long flags) {
155	157	PRE_READ(msg, sizeof(*msg));
156	158	}
157	159
158		POST_SYSCALL(recvmsg)(long res, long sockfd, sanitizer_kernel_msghdr *msg,
159		long flags) {
	160	POST_SYSCALL(recvmsg)
	161	(long res, long sockfd, sanitizer_kernel_msghdr *msg, long flags) {
160	162	if (res >= 0) {
161	163	if (msg) {
162	164	for (unsigned long i = 0; i < msg->msg_iovlen; ++i) {
...	...	@@ -167,13 +169,14 @@ POST_SYSCALL(recvmsg)(long res, long sockfd, sanitizer_kernel_msghdr *msg,
167	169	}
168	170	}
169	171
170		PRE_SYSCALL(recvmmsg)(long fd, sanitizer_kernel_mmsghdr *msg, long vlen,
171		long flags, void *timeout) {
	172	PRE_SYSCALL(recvmmsg)
	173	(long fd, sanitizer_kernel_mmsghdr msg, long vlen, long flags, void timeout) {
172	174	PRE_READ(msg, vlen * sizeof(*msg));
173	175	}
174	176
175		POST_SYSCALL(recvmmsg)(long res, long fd, sanitizer_kernel_mmsghdr *msg,
176		long vlen, long flags, void *timeout) {
	177	POST_SYSCALL(recvmmsg)
	178	(long res, long fd, sanitizer_kernel_mmsghdr *msg, long vlen, long flags,
	179	void *timeout) {
177	180	if (res >= 0) {
178	181	if (msg) {
179	182	for (unsigned long i = 0; i < msg->msg_hdr.msg_iovlen; ++i) {
...	...	@@ -183,7 +186,8 @@ POST_SYSCALL(recvmmsg)(long res, long fd, sanitizer_kernel_mmsghdr *msg,
183	186	POST_WRITE(msg->msg_hdr.msg_control, msg->msg_hdr.msg_controllen);
184	187	POST_WRITE(&msg->msg_len, sizeof(msg->msg_len));
185	188	}
186		if (timeout) POST_WRITE(timeout, struct_timespec_sz);
	189	if (timeout)
	190	POST_WRITE(timeout, struct_timespec_sz);
187	191	}
188	192	}
189	193
...	...	@@ -203,7 +207,8 @@ PRE_SYSCALL(time)(void *tloc) {}
203	207
204	208	POST_SYSCALL(time)(long res, void *tloc) {
205	209	if (res >= 0) {
206		if (tloc) POST_WRITE(tloc, sizeof(long));
	210	if (tloc)
	211	POST_WRITE(tloc, sizeof(long));
207	212	}
208	213	}
209	214
...	...	@@ -211,7 +216,8 @@ PRE_SYSCALL(stime)(void *tptr) {}
211	216
212	217	POST_SYSCALL(stime)(long res, void *tptr) {
213	218	if (res >= 0) {
214		if (tptr) POST_WRITE(tptr, sizeof(long));
	219	if (tptr)
	220	POST_WRITE(tptr, sizeof(long));
215	221	}
216	222	}
217	223
...	...	@@ -219,8 +225,10 @@ PRE_SYSCALL(gettimeofday)(void tv, void tz) {}
219	225
220	226	POST_SYSCALL(gettimeofday)(long res, void tv, void tz) {
221	227	if (res >= 0) {
222		if (tv) POST_WRITE(tv, timeval_sz);
223		if (tz) POST_WRITE(tz, struct_timezone_sz);
	228	if (tv)
	229	POST_WRITE(tv, timeval_sz);
	230	if (tz)
	231	POST_WRITE(tz, struct_timezone_sz);
224	232	}
225	233	}
226	234
...	...	@@ -228,26 +236,30 @@ PRE_SYSCALL(settimeofday)(void tv, void tz) {}
228	236
229	237	POST_SYSCALL(settimeofday)(long res, void tv, void tz) {
230	238	if (res >= 0) {
231		if (tv) POST_WRITE(tv, timeval_sz);
232		if (tz) POST_WRITE(tz, struct_timezone_sz);
	239	if (tv)
	240	POST_WRITE(tv, timeval_sz);
	241	if (tz)
	242	POST_WRITE(tz, struct_timezone_sz);
233	243	}
234	244	}
235	245
236		#if !SANITIZER_ANDROID
	246	# if !SANITIZER_ANDROID
237	247	PRE_SYSCALL(adjtimex)(void *txc_p) {}
238	248
239	249	POST_SYSCALL(adjtimex)(long res, void *txc_p) {
240	250	if (res >= 0) {
241		if (txc_p) POST_WRITE(txc_p, struct_timex_sz);
	251	if (txc_p)
	252	POST_WRITE(txc_p, struct_timex_sz);
242	253	}
243	254	}
244		#endif
	255	# endif
245	256
246	257	PRE_SYSCALL(times)(void *tbuf) {}
247	258
248	259	POST_SYSCALL(times)(long res, void *tbuf) {
249	260	if (res >= 0) {
250		if (tbuf) POST_WRITE(tbuf, struct_tms_sz);
	261	if (tbuf)
	262	POST_WRITE(tbuf, struct_tms_sz);
251	263	}
252	264	}
253	265
...	...	@@ -259,8 +271,10 @@ PRE_SYSCALL(nanosleep)(void rqtp, void rmtp) {}
259	271
260	272	POST_SYSCALL(nanosleep)(long res, void rqtp, void rmtp) {
261	273	if (res >= 0) {
262		if (rqtp) POST_WRITE(rqtp, struct_timespec_sz);
263		if (rmtp) POST_WRITE(rmtp, struct_timespec_sz);
	274	if (rqtp)
	275	POST_WRITE(rqtp, struct_timespec_sz);
	276	if (rmtp)
	277	POST_WRITE(rmtp, struct_timespec_sz);
264	278	}
265	279	}
266	280
...	...	@@ -296,9 +310,12 @@ PRE_SYSCALL(getresuid)(void ruid, void euid, void *suid) {}
296	310
297	311	POST_SYSCALL(getresuid)(long res, void ruid, void euid, void *suid) {
298	312	if (res >= 0) {
299		if (ruid) POST_WRITE(ruid, sizeof(unsigned));
300		if (euid) POST_WRITE(euid, sizeof(unsigned));
301		if (suid) POST_WRITE(suid, sizeof(unsigned));
	313	if (ruid)
	314	POST_WRITE(ruid, sizeof(unsigned));
	315	if (euid)
	316	POST_WRITE(euid, sizeof(unsigned));
	317	if (suid)
	318	POST_WRITE(suid, sizeof(unsigned));
302	319	}
303	320	}
304	321
...	...	@@ -306,9 +323,12 @@ PRE_SYSCALL(getresgid)(void rgid, void egid, void *sgid) {}
306	323
307	324	POST_SYSCALL(getresgid)(long res, void rgid, void egid, void *sgid) {
308	325	if (res >= 0) {
309		if (rgid) POST_WRITE(rgid, sizeof(unsigned));
310		if (egid) POST_WRITE(egid, sizeof(unsigned));
311		if (sgid) POST_WRITE(sgid, sizeof(unsigned));
	326	if (rgid)
	327	POST_WRITE(rgid, sizeof(unsigned));
	328	if (egid)
	329	POST_WRITE(egid, sizeof(unsigned));
	330	if (sgid)
	331	POST_WRITE(sgid, sizeof(unsigned));
312	332	}
313	333	}
314	334
...	...	@@ -326,10 +346,11 @@ POST_SYSCALL(getsid)(long res, long pid) {}
326	346
327	347	PRE_SYSCALL(getgroups)(long gidsetsize, void *grouplist) {}
328	348
329		POST_SYSCALL(getgroups)(long res, long gidsetsize,
330		__sanitizer___kernel_gid_t *grouplist) {
	349	POST_SYSCALL(getgroups)
	350	(long res, long gidsetsize, __sanitizer___kernel_gid_t *grouplist) {
331	351	if (res >= 0) {
332		if (grouplist) POST_WRITE(grouplist, res * sizeof(*grouplist));
	352	if (grouplist)
	353	POST_WRITE(grouplist, res * sizeof(*grouplist));
333	354	}
334	355	}
335	356
...	...	@@ -374,11 +395,12 @@ PRE_SYSCALL(setsid)() {}
374	395	POST_SYSCALL(setsid)(long res) {}
375	396
376	397	PRE_SYSCALL(setgroups)(long gidsetsize, __sanitizer___kernel_gid_t *grouplist) {
377		if (grouplist) POST_WRITE(grouplist, gidsetsize * sizeof(*grouplist));
	398	if (grouplist)
	399	POST_WRITE(grouplist, gidsetsize * sizeof(*grouplist));
378	400	}
379	401
380		POST_SYSCALL(setgroups)(long res, long gidsetsize,
381		__sanitizer___kernel_gid_t *grouplist) {}
	402	POST_SYSCALL(setgroups)
	403	(long res, long gidsetsize, __sanitizer___kernel_gid_t *grouplist) {}
382	404
383	405	PRE_SYSCALL(acct)(const void *name) {
384	406	if (name)
...	...	@@ -388,17 +410,21 @@ PRE_SYSCALL(acct)(const void *name) {
388	410	POST_SYSCALL(acct)(long res, const void *name) {}
389	411
390	412	PRE_SYSCALL(capget)(void header, void dataptr) {
391		if (header) PRE_READ(header, __user_cap_header_struct_sz);
	413	if (header)
	414	PRE_READ(header, __user_cap_header_struct_sz);
392	415	}
393	416
394	417	POST_SYSCALL(capget)(long res, void header, void dataptr) {
395	418	if (res >= 0)
396		if (dataptr) POST_WRITE(dataptr, __user_cap_data_struct_sz);
	419	if (dataptr)
	420	POST_WRITE(dataptr, __user_cap_data_struct_sz(header));
397	421	}
398	422
399	423	PRE_SYSCALL(capset)(void header, const void data) {
400		if (header) PRE_READ(header, __user_cap_header_struct_sz);
401		if (data) PRE_READ(data, __user_cap_data_struct_sz);
	424	if (header)
	425	PRE_READ(header, __user_cap_header_struct_sz);
	426	if (data)
	427	PRE_READ(data, __user_cap_data_struct_sz(header));
402	428	}
403	429
404	430	POST_SYSCALL(capset)(long res, void header, const void data) {}
...	...	@@ -411,7 +437,8 @@ PRE_SYSCALL(sigpending)(void *set) {}
411	437
412	438	POST_SYSCALL(sigpending)(long res, void *set) {
413	439	if (res >= 0) {
414		if (set) POST_WRITE(set, old_sigset_t_sz);
	440	if (set)
	441	POST_WRITE(set, old_sigset_t_sz);
415	442	}
416	443	}
417	444
...	...	@@ -419,8 +446,10 @@ PRE_SYSCALL(sigprocmask)(long how, void set, void oset) {}
419	446
420	447	POST_SYSCALL(sigprocmask)(long res, long how, void set, void oset) {
421	448	if (res >= 0) {
422		if (set) POST_WRITE(set, old_sigset_t_sz);
423		if (oset) POST_WRITE(oset, old_sigset_t_sz);
	449	if (set)
	450	POST_WRITE(set, old_sigset_t_sz);
	451	if (oset)
	452	POST_WRITE(oset, old_sigset_t_sz);
424	453	}
425	454	}
426	455
...	...	@@ -428,7 +457,8 @@ PRE_SYSCALL(getitimer)(long which, void *value) {}
428	457
429	458	POST_SYSCALL(getitimer)(long res, long which, void *value) {
430	459	if (res >= 0) {
431		if (value) POST_WRITE(value, struct_itimerval_sz);
	460	if (value)
	461	POST_WRITE(value, struct_itimerval_sz);
432	462	}
433	463	}
434	464
...	...	@@ -436,19 +466,23 @@ PRE_SYSCALL(setitimer)(long which, void value, void ovalue) {}
436	466
437	467	POST_SYSCALL(setitimer)(long res, long which, void value, void ovalue) {
438	468	if (res >= 0) {
439		if (value) POST_WRITE(value, struct_itimerval_sz);
440		if (ovalue) POST_WRITE(ovalue, struct_itimerval_sz);
	469	if (value)
	470	POST_WRITE(value, struct_itimerval_sz);
	471	if (ovalue)
	472	POST_WRITE(ovalue, struct_itimerval_sz);
441	473	}
442	474	}
443	475
444		PRE_SYSCALL(timer_create)(long which_clock, void *timer_event_spec,
445		void *created_timer_id) {}
	476	PRE_SYSCALL(timer_create)
	477	(long which_clock, void timer_event_spec, void created_timer_id) {}
446	478
447		POST_SYSCALL(timer_create)(long res, long which_clock, void *timer_event_spec,
448		void *created_timer_id) {
	479	POST_SYSCALL(timer_create)
	480	(long res, long which_clock, void timer_event_spec, void created_timer_id) {
449	481	if (res >= 0) {
450		if (timer_event_spec) POST_WRITE(timer_event_spec, struct_sigevent_sz);
451		if (created_timer_id) POST_WRITE(created_timer_id, sizeof(long));
	482	if (timer_event_spec)
	483	POST_WRITE(timer_event_spec, struct_sigevent_sz);
	484	if (created_timer_id)
	485	POST_WRITE(created_timer_id, sizeof(long));
452	486	}
453	487	}
454	488
...	...	@@ -456,7 +490,8 @@ PRE_SYSCALL(timer_gettime)(long timer_id, void *setting) {}
456	490
457	491	POST_SYSCALL(timer_gettime)(long res, long timer_id, void *setting) {
458	492	if (res >= 0) {
459		if (setting) POST_WRITE(setting, struct_itimerspec_sz);
	493	if (setting)
	494	POST_WRITE(setting, struct_itimerspec_sz);
460	495	}
461	496	}
462	497
...	...	@@ -464,15 +499,18 @@ PRE_SYSCALL(timer_getoverrun)(long timer_id) {}
464	499
465	500	POST_SYSCALL(timer_getoverrun)(long res, long timer_id) {}
466	501
467		PRE_SYSCALL(timer_settime)(long timer_id, long flags, const void *new_setting,
468		void *old_setting) {
469		if (new_setting) PRE_READ(new_setting, struct_itimerspec_sz);
	502	PRE_SYSCALL(timer_settime)
	503	(long timer_id, long flags, const void new_setting, void old_setting) {
	504	if (new_setting)
	505	PRE_READ(new_setting, struct_itimerspec_sz);
470	506	}
471	507
472		POST_SYSCALL(timer_settime)(long res, long timer_id, long flags,
473		const void new_setting, void old_setting) {
	508	POST_SYSCALL(timer_settime)
	509	(long res, long timer_id, long flags, const void *new_setting,
	510	void *old_setting) {
474	511	if (res >= 0) {
475		if (old_setting) POST_WRITE(old_setting, struct_itimerspec_sz);
	512	if (old_setting)
	513	POST_WRITE(old_setting, struct_itimerspec_sz);
476	514	}
477	515	}
478	516
...	...	@@ -481,7 +519,8 @@ PRE_SYSCALL(timer_delete)(long timer_id) {}
481	519	POST_SYSCALL(timer_delete)(long res, long timer_id) {}
482	520
483	521	PRE_SYSCALL(clock_settime)(long which_clock, const void *tp) {
484		if (tp) PRE_READ(tp, struct_timespec_sz);
	522	if (tp)
	523	PRE_READ(tp, struct_timespec_sz);
485	524	}
486	525
487	526	POST_SYSCALL(clock_settime)(long res, long which_clock, const void *tp) {}
...	...	@@ -490,37 +529,42 @@ PRE_SYSCALL(clock_gettime)(long which_clock, void *tp) {}
490	529
491	530	POST_SYSCALL(clock_gettime)(long res, long which_clock, void *tp) {
492	531	if (res >= 0) {
493		if (tp) POST_WRITE(tp, struct_timespec_sz);
	532	if (tp)
	533	POST_WRITE(tp, struct_timespec_sz);
494	534	}
495	535	}
496	536
497		#if !SANITIZER_ANDROID
	537	# if !SANITIZER_ANDROID
498	538	PRE_SYSCALL(clock_adjtime)(long which_clock, void *tx) {}
499	539
500	540	POST_SYSCALL(clock_adjtime)(long res, long which_clock, void *tx) {
501	541	if (res >= 0) {
502		if (tx) POST_WRITE(tx, struct_timex_sz);
	542	if (tx)
	543	POST_WRITE(tx, struct_timex_sz);
503	544	}
504	545	}
505		#endif
	546	# endif
506	547
507	548	PRE_SYSCALL(clock_getres)(long which_clock, void *tp) {}
508	549
509	550	POST_SYSCALL(clock_getres)(long res, long which_clock, void *tp) {
510	551	if (res >= 0) {
511		if (tp) POST_WRITE(tp, struct_timespec_sz);
	552	if (tp)
	553	POST_WRITE(tp, struct_timespec_sz);
512	554	}
513	555	}
514	556
515		PRE_SYSCALL(clock_nanosleep)(long which_clock, long flags, const void *rqtp,
516		void *rmtp) {
517		if (rqtp) PRE_READ(rqtp, struct_timespec_sz);
	557	PRE_SYSCALL(clock_nanosleep)
	558	(long which_clock, long flags, const void rqtp, void rmtp) {
	559	if (rqtp)
	560	PRE_READ(rqtp, struct_timespec_sz);
518	561	}
519	562
520		POST_SYSCALL(clock_nanosleep)(long res, long which_clock, long flags,
521		const void rqtp, void rmtp) {
	563	POST_SYSCALL(clock_nanosleep)
	564	(long res, long which_clock, long flags, const void rqtp, void rmtp) {
522	565	if (res >= 0) {
523		if (rmtp) POST_WRITE(rmtp, struct_timespec_sz);
	566	if (rmtp)
	567	POST_WRITE(rmtp, struct_timespec_sz);
524	568	}
525	569	}
526	570
...	...	@@ -532,12 +576,14 @@ PRE_SYSCALL(sched_setscheduler)(long pid, long policy, void *param) {}
532	576
533	577	POST_SYSCALL(sched_setscheduler)(long res, long pid, long policy, void *param) {
534	578	if (res >= 0) {
535		if (param) POST_WRITE(param, struct_sched_param_sz);
	579	if (param)
	580	POST_WRITE(param, struct_sched_param_sz);
536	581	}
537	582	}
538	583
539	584	PRE_SYSCALL(sched_setparam)(long pid, void *param) {
540		if (param) PRE_READ(param, struct_sched_param_sz);
	585	if (param)
	586	PRE_READ(param, struct_sched_param_sz);
541	587	}
542	588
543	589	POST_SYSCALL(sched_setparam)(long res, long pid, void *param) {}
...	...	@@ -550,23 +596,26 @@ PRE_SYSCALL(sched_getparam)(long pid, void *param) {}
550	596
551	597	POST_SYSCALL(sched_getparam)(long res, long pid, void *param) {
552	598	if (res >= 0) {
553		if (param) POST_WRITE(param, struct_sched_param_sz);
	599	if (param)
	600	POST_WRITE(param, struct_sched_param_sz);
554	601	}
555	602	}
556	603
557	604	PRE_SYSCALL(sched_setaffinity)(long pid, long len, void *user_mask_ptr) {
558		if (user_mask_ptr) PRE_READ(user_mask_ptr, len);
	605	if (user_mask_ptr)
	606	PRE_READ(user_mask_ptr, len);
559	607	}
560	608
561		POST_SYSCALL(sched_setaffinity)(long res, long pid, long len,
562		void *user_mask_ptr) {}
	609	POST_SYSCALL(sched_setaffinity)
	610	(long res, long pid, long len, void *user_mask_ptr) {}
563	611
564	612	PRE_SYSCALL(sched_getaffinity)(long pid, long len, void *user_mask_ptr) {}
565	613
566		POST_SYSCALL(sched_getaffinity)(long res, long pid, long len,
567		void *user_mask_ptr) {
	614	POST_SYSCALL(sched_getaffinity)
	615	(long res, long pid, long len, void *user_mask_ptr) {
568	616	if (res >= 0) {
569		if (user_mask_ptr) POST_WRITE(user_mask_ptr, len);
	617	if (user_mask_ptr)
	618	POST_WRITE(user_mask_ptr, len);
570	619	}
571	620	}
572	621
...	...	@@ -586,7 +635,8 @@ PRE_SYSCALL(sched_rr_get_interval)(long pid, void *interval) {}
586	635
587	636	POST_SYSCALL(sched_rr_get_interval)(long res, long pid, void *interval) {
588	637	if (res >= 0) {
589		if (interval) POST_WRITE(interval, struct_timespec_sz);
	638	if (interval)
	639	POST_WRITE(interval, struct_timespec_sz);
590	640	}
591	641	}
592	642
...	...	@@ -610,13 +660,14 @@ PRE_SYSCALL(restart_syscall)() {}
610	660
611	661	POST_SYSCALL(restart_syscall)(long res) {}
612	662
613		PRE_SYSCALL(kexec_load)(long entry, long nr_segments, void *segments,
614		long flags) {}
	663	PRE_SYSCALL(kexec_load)
	664	(long entry, long nr_segments, void *segments, long flags) {}
615	665
616		POST_SYSCALL(kexec_load)(long res, long entry, long nr_segments, void *segments,
617		long flags) {
	666	POST_SYSCALL(kexec_load)
	667	(long res, long entry, long nr_segments, void *segments, long flags) {
618	668	if (res >= 0) {
619		if (segments) POST_WRITE(segments, struct_kexec_segment_sz);
	669	if (segments)
	670	POST_WRITE(segments, struct_kexec_segment_sz);
620	671	}
621	672	}
622	673
...	...	@@ -630,22 +681,26 @@ POST_SYSCALL(exit_group)(long res, long error_code) {}
630	681
631	682	PRE_SYSCALL(wait4)(long pid, void stat_addr, long options, void ru) {}
632	683
633		POST_SYSCALL(wait4)(long res, long pid, void *stat_addr, long options,
634		void *ru) {
	684	POST_SYSCALL(wait4)
	685	(long res, long pid, void stat_addr, long options, void ru) {
635	686	if (res >= 0) {
636		if (stat_addr) POST_WRITE(stat_addr, sizeof(int));
637		if (ru) POST_WRITE(ru, struct_rusage_sz);
	687	if (stat_addr)
	688	POST_WRITE(stat_addr, sizeof(int));
	689	if (ru)
	690	POST_WRITE(ru, struct_rusage_sz);
638	691	}
639	692	}
640	693
641		PRE_SYSCALL(waitid)(long which, long pid, void infop, long options, void ru) {
642		}
	694	PRE_SYSCALL(waitid)
	695	(long which, long pid, void infop, long options, void ru) {}
643	696
644		POST_SYSCALL(waitid)(long res, long which, long pid, void *infop, long options,
645		void *ru) {
	697	POST_SYSCALL(waitid)
	698	(long res, long which, long pid, void infop, long options, void ru) {
646	699	if (res >= 0) {
647		if (infop) POST_WRITE(infop, siginfo_t_sz);
648		if (ru) POST_WRITE(ru, struct_rusage_sz);
	700	if (infop)
	701	POST_WRITE(infop, siginfo_t_sz);
	702	if (ru)
	703	POST_WRITE(ru, struct_rusage_sz);
649	704	}
650	705	}
651	706
...	...	@@ -653,7 +708,8 @@ PRE_SYSCALL(waitpid)(long pid, void *stat_addr, long options) {}
653	708
654	709	POST_SYSCALL(waitpid)(long res, long pid, void *stat_addr, long options) {
655	710	if (res >= 0) {
656		if (stat_addr) POST_WRITE(stat_addr, sizeof(int));
	711	if (stat_addr)
	712	POST_WRITE(stat_addr, sizeof(int));
657	713	}
658	714	}
659	715
...	...	@@ -661,7 +717,8 @@ PRE_SYSCALL(set_tid_address)(void *tidptr) {}
661	717
662	718	POST_SYSCALL(set_tid_address)(long res, void *tidptr) {
663	719	if (res >= 0) {
664		if (tidptr) POST_WRITE(tidptr, sizeof(int));
	720	if (tidptr)
	721	POST_WRITE(tidptr, sizeof(int));
665	722	}
666	723	}
667	724
...	...	@@ -682,11 +739,14 @@ POST_SYSCALL(delete_module)(long res, const void *name_user, long flags) {}
682	739
683	740	PRE_SYSCALL(rt_sigprocmask)(long how, void set, void oset, long sigsetsize) {}
684	741
685		POST_SYSCALL(rt_sigprocmask)(long res, long how, kernel_sigset_t *set,
686		kernel_sigset_t *oset, long sigsetsize) {
	742	POST_SYSCALL(rt_sigprocmask)
	743	(long res, long how, kernel_sigset_t set, kernel_sigset_t oset,
	744	long sigsetsize) {
687	745	if (res >= 0) {
688		if (set) POST_WRITE(set, sigsetsize);
689		if (oset) POST_WRITE(oset, sigsetsize);
	746	if (set)
	747	POST_WRITE(set, sigsetsize);
	748	if (oset)
	749	POST_WRITE(oset, sigsetsize);
690	750	}
691	751	}
692	752
...	...	@@ -694,29 +754,34 @@ PRE_SYSCALL(rt_sigpending)(void *set, long sigsetsize) {}
694	754
695	755	POST_SYSCALL(rt_sigpending)(long res, kernel_sigset_t *set, long sigsetsize) {
696	756	if (res >= 0) {
697		if (set) POST_WRITE(set, sigsetsize);
	757	if (set)
	758	POST_WRITE(set, sigsetsize);
698	759	}
699	760	}
700	761
701		PRE_SYSCALL(rt_sigtimedwait)(const kernel_sigset_t uthese, void uinfo,
702		const void *uts, long sigsetsize) {
703		if (uthese) PRE_READ(uthese, sigsetsize);
704		if (uts) PRE_READ(uts, struct_timespec_sz);
	762	PRE_SYSCALL(rt_sigtimedwait)
	763	(const kernel_sigset_t uthese, void uinfo, const void *uts, long sigsetsize) {
	764	if (uthese)
	765	PRE_READ(uthese, sigsetsize);
	766	if (uts)
	767	PRE_READ(uts, struct_timespec_sz);
705	768	}
706	769
707		POST_SYSCALL(rt_sigtimedwait)(long res, const void uthese, void uinfo,
708		const void *uts, long sigsetsize) {
	770	POST_SYSCALL(rt_sigtimedwait)
	771	(long res, const void uthese, void uinfo, const void *uts, long sigsetsize) {
709	772	if (res >= 0) {
710		if (uinfo) POST_WRITE(uinfo, siginfo_t_sz);
	773	if (uinfo)
	774	POST_WRITE(uinfo, siginfo_t_sz);
711	775	}
712	776	}
713	777
714	778	PRE_SYSCALL(rt_tgsigqueueinfo)(long tgid, long pid, long sig, void *uinfo) {}
715	779
716		POST_SYSCALL(rt_tgsigqueueinfo)(long res, long tgid, long pid, long sig,
717		void *uinfo) {
	780	POST_SYSCALL(rt_tgsigqueueinfo)
	781	(long res, long tgid, long pid, long sig, void *uinfo) {
718	782	if (res >= 0) {
719		if (uinfo) POST_WRITE(uinfo, siginfo_t_sz);
	783	if (uinfo)
	784	POST_WRITE(uinfo, siginfo_t_sz);
720	785	}
721	786	}
722	787
...	...	@@ -736,7 +801,8 @@ PRE_SYSCALL(rt_sigqueueinfo)(long pid, long sig, void *uinfo) {}
736	801
737	802	POST_SYSCALL(rt_sigqueueinfo)(long res, long pid, long sig, void *uinfo) {
738	803	if (res >= 0) {
739		if (uinfo) POST_WRITE(uinfo, siginfo_t_sz);
	804	if (uinfo)
	805	POST_WRITE(uinfo, siginfo_t_sz);
740	806	}
741	807	}
742	808
...	...	@@ -772,11 +838,11 @@ PRE_SYSCALL(bdflush)(long func, long data) {}
772	838
773	839	POST_SYSCALL(bdflush)(long res, long func, long data) {}
774	840
775		PRE_SYSCALL(mount)(void dev_name, void dir_name, void *type, long flags,
776		void *data) {}
	841	PRE_SYSCALL(mount)
	842	(void dev_name, void dir_name, void type, long flags, void data) {}
777	843
778		POST_SYSCALL(mount)(long res, void dev_name, void dir_name, void *type,
779		long flags, void *data) {
	844	POST_SYSCALL(mount)
	845	(long res, void dev_name, void dir_name, void type, long flags, void data) {
780	846	if (res >= 0) {
781	847	if (dev_name)
782	848	POST_WRITE(dev_name,
...	...	@@ -826,11 +892,12 @@ PRE_SYSCALL(stat)(const void filename, void statbuf) {
826	892
827	893	POST_SYSCALL(stat)(long res, const void filename, void statbuf) {
828	894	if (res >= 0) {
829		if (statbuf) POST_WRITE(statbuf, struct___old_kernel_stat_sz);
	895	if (statbuf)
	896	POST_WRITE(statbuf, struct___old_kernel_stat_sz);
830	897	}
831	898	}
832	899
833		#if !SANITIZER_ANDROID
	900	# if !SANITIZER_ANDROID
834	901	PRE_SYSCALL(statfs)(const void path, void buf) {
835	902	if (path)
836	903	PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
...	...	@@ -838,26 +905,31 @@ PRE_SYSCALL(statfs)(const void path, void buf) {
838	905
839	906	POST_SYSCALL(statfs)(long res, const void path, void buf) {
840	907	if (res >= 0) {
841		if (buf) POST_WRITE(buf, struct_statfs_sz);
	908	if (buf)
	909	POST_WRITE(buf, struct_statfs_sz);
842	910	}
843	911	}
844	912
845		PRE_SYSCALL(statfs64)(const void path, long sz, void buf) {
846		if (path)
847		PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
848		}
	913	PRE_SYSCALL(fstatfs)(long fd, void *buf) {}
849	914
850		POST_SYSCALL(statfs64)(long res, const void path, long sz, void buf) {
	915	POST_SYSCALL(fstatfs)(long res, long fd, void *buf) {
851	916	if (res >= 0) {
852		if (buf) POST_WRITE(buf, struct_statfs64_sz);
	917	if (buf)
	918	POST_WRITE(buf, struct_statfs_sz);
853	919	}
854	920	}
	921	# endif // !SANITIZER_ANDROID
855	922
856		PRE_SYSCALL(fstatfs)(long fd, void *buf) {}
	923	# if SANITIZER_GLIBC
	924	PRE_SYSCALL(statfs64)(const void path, long sz, void buf) {
	925	if (path)
	926	PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
	927	}
857	928
858		POST_SYSCALL(fstatfs)(long res, long fd, void *buf) {
	929	POST_SYSCALL(statfs64)(long res, const void path, long sz, void buf) {
859	930	if (res >= 0) {
860		if (buf) POST_WRITE(buf, struct_statfs_sz);
	931	if (buf)
	932	POST_WRITE(buf, struct_statfs64_sz);
861	933	}
862	934	}
863	935
...	...	@@ -865,10 +937,11 @@ PRE_SYSCALL(fstatfs64)(long fd, long sz, void *buf) {}
865	937
866	938	POST_SYSCALL(fstatfs64)(long res, long fd, long sz, void *buf) {
867	939	if (res >= 0) {
868		if (buf) POST_WRITE(buf, struct_statfs64_sz);
	940	if (buf)
	941	POST_WRITE(buf, struct_statfs64_sz);
869	942	}
870	943	}
871		#endif // !SANITIZER_ANDROID
	944	# endif // SANITIZER_GLIBC
872	945
873	946	PRE_SYSCALL(lstat)(const void filename, void statbuf) {
874	947	if (filename)
...	...	@@ -878,7 +951,8 @@ PRE_SYSCALL(lstat)(const void filename, void statbuf) {
878	951
879	952	POST_SYSCALL(lstat)(long res, const void filename, void statbuf) {
880	953	if (res >= 0) {
881		if (statbuf) POST_WRITE(statbuf, struct___old_kernel_stat_sz);
	954	if (statbuf)
	955	POST_WRITE(statbuf, struct___old_kernel_stat_sz);
882	956	}
883	957	}
884	958
...	...	@@ -886,7 +960,8 @@ PRE_SYSCALL(fstat)(long fd, void *statbuf) {}
886	960
887	961	POST_SYSCALL(fstat)(long res, long fd, void *statbuf) {
888	962	if (res >= 0) {
889		if (statbuf) POST_WRITE(statbuf, struct___old_kernel_stat_sz);
	963	if (statbuf)
	964	POST_WRITE(statbuf, struct___old_kernel_stat_sz);
890	965	}
891	966	}
892	967
...	...	@@ -898,7 +973,8 @@ PRE_SYSCALL(newstat)(const void filename, void statbuf) {
898	973
899	974	POST_SYSCALL(newstat)(long res, const void filename, void statbuf) {
900	975	if (res >= 0) {
901		if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
	976	if (statbuf)
	977	POST_WRITE(statbuf, struct_kernel_stat_sz);
902	978	}
903	979	}
904	980
...	...	@@ -910,7 +986,8 @@ PRE_SYSCALL(newlstat)(const void filename, void statbuf) {
910	986
911	987	POST_SYSCALL(newlstat)(long res, const void filename, void statbuf) {
912	988	if (res >= 0) {
913		if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
	989	if (statbuf)
	990	POST_WRITE(statbuf, struct_kernel_stat_sz);
914	991	}
915	992	}
916	993
...	...	@@ -918,19 +995,21 @@ PRE_SYSCALL(newfstat)(long fd, void *statbuf) {}
918	995
919	996	POST_SYSCALL(newfstat)(long res, long fd, void *statbuf) {
920	997	if (res >= 0) {
921		if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
	998	if (statbuf)
	999	POST_WRITE(statbuf, struct_kernel_stat_sz);
922	1000	}
923	1001	}
924	1002
925		#if !SANITIZER_ANDROID
	1003	# if SANITIZER_GLIBC
926	1004	PRE_SYSCALL(ustat)(long dev, void *ubuf) {}
927	1005
928	1006	POST_SYSCALL(ustat)(long res, long dev, void *ubuf) {
929	1007	if (res >= 0) {
930		if (ubuf) POST_WRITE(ubuf, struct_ustat_sz);
	1008	if (ubuf)
	1009	POST_WRITE(ubuf, struct_ustat_sz);
931	1010	}
932	1011	}
933		#endif // !SANITIZER_ANDROID
	1012	# endif // SANITIZER_GLIBC
934	1013
935	1014	PRE_SYSCALL(stat64)(const void filename, void statbuf) {
936	1015	if (filename)
...	...	@@ -940,7 +1019,8 @@ PRE_SYSCALL(stat64)(const void filename, void statbuf) {
940	1019
941	1020	POST_SYSCALL(stat64)(long res, const void filename, void statbuf) {
942	1021	if (res >= 0) {
943		if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
	1022	if (statbuf)
	1023	POST_WRITE(statbuf, struct_kernel_stat64_sz);
944	1024	}
945	1025	}
946	1026
...	...	@@ -948,7 +1028,8 @@ PRE_SYSCALL(fstat64)(long fd, void *statbuf) {}
948	1028
949	1029	POST_SYSCALL(fstat64)(long res, long fd, void *statbuf) {
950	1030	if (res >= 0) {
951		if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
	1031	if (statbuf)
	1032	POST_WRITE(statbuf, struct_kernel_stat64_sz);
952	1033	}
953	1034	}
954	1035
...	...	@@ -960,71 +1041,80 @@ PRE_SYSCALL(lstat64)(const void filename, void statbuf) {
960	1041
961	1042	POST_SYSCALL(lstat64)(long res, const void filename, void statbuf) {
962	1043	if (res >= 0) {
963		if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
	1044	if (statbuf)
	1045	POST_WRITE(statbuf, struct_kernel_stat64_sz);
964	1046	}
965	1047	}
966	1048
967		PRE_SYSCALL(setxattr)(const void path, const void name, const void *value,
968		long size, long flags) {
	1049	PRE_SYSCALL(setxattr)
	1050	(const void path, const void name, const void *value, long size, long flags) {
969	1051	if (path)
970	1052	PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
971	1053	if (name)
972	1054	PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
973		if (value) PRE_READ(value, size);
	1055	if (value)
	1056	PRE_READ(value, size);
974	1057	}
975	1058
976		POST_SYSCALL(setxattr)(long res, const void path, const void name,
977		const void *value, long size, long flags) {}
	1059	POST_SYSCALL(setxattr)
	1060	(long res, const void path, const void name, const void *value, long size,
	1061	long flags) {}
978	1062
979		PRE_SYSCALL(lsetxattr)(const void path, const void name, const void *value,
980		long size, long flags) {
	1063	PRE_SYSCALL(lsetxattr)
	1064	(const void path, const void name, const void *value, long size, long flags) {
981	1065	if (path)
982	1066	PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
983	1067	if (name)
984	1068	PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
985		if (value) PRE_READ(value, size);
	1069	if (value)
	1070	PRE_READ(value, size);
986	1071	}
987	1072
988		POST_SYSCALL(lsetxattr)(long res, const void path, const void name,
989		const void *value, long size, long flags) {}
	1073	POST_SYSCALL(lsetxattr)
	1074	(long res, const void path, const void name, const void *value, long size,
	1075	long flags) {}
990	1076
991		PRE_SYSCALL(fsetxattr)(long fd, const void name, const void value, long size,
992		long flags) {
	1077	PRE_SYSCALL(fsetxattr)
	1078	(long fd, const void name, const void value, long size, long flags) {
993	1079	if (name)
994	1080	PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
995		if (value) PRE_READ(value, size);
	1081	if (value)
	1082	PRE_READ(value, size);
996	1083	}
997	1084
998		POST_SYSCALL(fsetxattr)(long res, long fd, const void name, const void value,
999		long size, long flags) {}
	1085	POST_SYSCALL(fsetxattr)
	1086	(long res, long fd, const void name, const void value, long size,
	1087	long flags) {}
1000	1088
1001		PRE_SYSCALL(getxattr)(const void path, const void name, void *value,
1002		long size) {
	1089	PRE_SYSCALL(getxattr)
	1090	(const void path, const void name, void *value, long size) {
1003	1091	if (path)
1004	1092	PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
1005	1093	if (name)
1006	1094	PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
1007	1095	}
1008	1096
1009		POST_SYSCALL(getxattr)(long res, const void path, const void name,
1010		void *value, long size) {
	1097	POST_SYSCALL(getxattr)
	1098	(long res, const void path, const void name, void *value, long size) {
1011	1099	if (size && res > 0) {
1012		if (value) POST_WRITE(value, res);
	1100	if (value)
	1101	POST_WRITE(value, res);
1013	1102	}
1014	1103	}
1015	1104
1016		PRE_SYSCALL(lgetxattr)(const void path, const void name, void *value,
1017		long size) {
	1105	PRE_SYSCALL(lgetxattr)
	1106	(const void path, const void name, void *value, long size) {
1018	1107	if (path)
1019	1108	PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
1020	1109	if (name)
1021	1110	PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
1022	1111	}
1023	1112
1024		POST_SYSCALL(lgetxattr)(long res, const void path, const void name,
1025		void *value, long size) {
	1113	POST_SYSCALL(lgetxattr)
	1114	(long res, const void path, const void name, void *value, long size) {
1026	1115	if (size && res > 0) {
1027		if (value) POST_WRITE(value, res);
	1116	if (value)
	1117	POST_WRITE(value, res);
1028	1118	}
1029	1119	}
1030	1120
...	...	@@ -1033,10 +1123,11 @@ PRE_SYSCALL(fgetxattr)(long fd, const void name, void value, long size) {
1033	1123	PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
1034	1124	}
1035	1125
1036		POST_SYSCALL(fgetxattr)(long res, long fd, const void name, void value,
1037		long size) {
	1126	POST_SYSCALL(fgetxattr)
	1127	(long res, long fd, const void name, void value, long size) {
1038	1128	if (size && res > 0) {
1039		if (value) POST_WRITE(value, res);
	1129	if (value)
	1130	POST_WRITE(value, res);
1040	1131	}
1041	1132	}
1042	1133
...	...	@@ -1047,7 +1138,8 @@ PRE_SYSCALL(listxattr)(const void path, void list, long size) {
1047	1138
1048	1139	POST_SYSCALL(listxattr)(long res, const void path, void list, long size) {
1049	1140	if (size && res > 0) {
1050		if (list) POST_WRITE(list, res);
	1141	if (list)
	1142	POST_WRITE(list, res);
1051	1143	}
1052	1144	}
1053	1145
...	...	@@ -1058,7 +1150,8 @@ PRE_SYSCALL(llistxattr)(const void path, void list, long size) {
1058	1150
1059	1151	POST_SYSCALL(llistxattr)(long res, const void path, void list, long size) {
1060	1152	if (size && res > 0) {
1061		if (list) POST_WRITE(list, res);
	1153	if (list)
	1154	POST_WRITE(list, res);
1062	1155	}
1063	1156	}
1064	1157
...	...	@@ -1066,7 +1159,8 @@ PRE_SYSCALL(flistxattr)(long fd, void *list, long size) {}
1066	1159
1067	1160	POST_SYSCALL(flistxattr)(long res, long fd, void *list, long size) {
1068	1161	if (size && res > 0) {
1069		if (list) POST_WRITE(list, res);
	1162	if (list)
	1163	POST_WRITE(list, res);
1070	1164	}
1071	1165	}
1072	1166
...	...	@@ -1103,17 +1197,17 @@ PRE_SYSCALL(mprotect)(long start, long len, long prot) {}
1103	1197
1104	1198	POST_SYSCALL(mprotect)(long res, long start, long len, long prot) {}
1105	1199
1106		PRE_SYSCALL(mremap)(long addr, long old_len, long new_len, long flags,
1107		long new_addr) {}
	1200	PRE_SYSCALL(mremap)
	1201	(long addr, long old_len, long new_len, long flags, long new_addr) {}
1108	1202
1109		POST_SYSCALL(mremap)(long res, long addr, long old_len, long new_len,
1110		long flags, long new_addr) {}
	1203	POST_SYSCALL(mremap)
	1204	(long res, long addr, long old_len, long new_len, long flags, long new_addr) {}
1111	1205
1112		PRE_SYSCALL(remap_file_pages)(long start, long size, long prot, long pgoff,
1113		long flags) {}
	1206	PRE_SYSCALL(remap_file_pages)
	1207	(long start, long size, long prot, long pgoff, long flags) {}
1114	1208
1115		POST_SYSCALL(remap_file_pages)(long res, long start, long size, long prot,
1116		long pgoff, long flags) {}
	1209	POST_SYSCALL(remap_file_pages)
	1210	(long res, long start, long size, long prot, long pgoff, long flags) {}
1117	1211
1118	1212	PRE_SYSCALL(msync)(long start, long len, long flags) {}
1119	1213
...	...	@@ -1189,7 +1283,8 @@ PRE_SYSCALL(link)(const void oldname, const void newname) {
1189	1283	POST_SYSCALL(link)(long res, const void oldname, const void newname) {}
1190	1284
1191	1285	PRE_SYSCALL(symlink)(const void old, const void new_) {
1192		if (old) PRE_READ(old, __sanitizer::internal_strlen((const char *)old) + 1);
	1286	if (old)
	1287	PRE_READ(old, __sanitizer::internal_strlen((const char *)old) + 1);
1193	1288	if (new_)
1194	1289	PRE_READ(new_, __sanitizer::internal_strlen((const char *)new_) + 1);
1195	1290	}
...	...	@@ -1237,14 +1332,16 @@ PRE_SYSCALL(pipe)(void *fildes) {}
1237	1332
1238	1333	POST_SYSCALL(pipe)(long res, void *fildes) {
1239	1334	if (res >= 0)
1240		if (fildes) POST_WRITE(fildes, sizeof(int) * 2);
	1335	if (fildes)
	1336	POST_WRITE(fildes, sizeof(int) * 2);
1241	1337	}
1242	1338
1243	1339	PRE_SYSCALL(pipe2)(void *fildes, long flags) {}
1244	1340
1245	1341	POST_SYSCALL(pipe2)(long res, void *fildes, long flags) {
1246	1342	if (res >= 0)
1247		if (fildes) POST_WRITE(fildes, sizeof(int) * 2);
	1343	if (fildes)
	1344	POST_WRITE(fildes, sizeof(int) * 2);
1248	1345	}
1249	1346
1250	1347	PRE_SYSCALL(dup)(long fildes) {}
...	...	@@ -1272,16 +1369,18 @@ PRE_SYSCALL(flock)(long fd, long cmd) {}
1272	1369	POST_SYSCALL(flock)(long res, long fd, long cmd) {}
1273	1370
1274	1371	PRE_SYSCALL(io_setup)(long nr_reqs, void **ctx) {
1275		if (ctx) PRE_WRITE(ctx, sizeof(*ctx));
	1372	if (ctx)
	1373	PRE_WRITE(ctx, sizeof(*ctx));
1276	1374	}
1277	1375
1278	1376	POST_SYSCALL(io_setup)(long res, long nr_reqs, void **ctx) {
1279		if (res >= 0) {
1280		if (ctx) POST_WRITE(ctx, sizeof(*ctx));
	1377	if (res >= 0 && ctx) {
	1378	POST_WRITE(ctx, sizeof(*ctx));
1281	1379	// (*ctx) is actually a pointer to a kernel mapped page, and there are
1282	1380	// people out there who are crazy enough to peek into that page's 32-byte
1283	1381	// header.
1284		if (ctx) POST_WRITE(ctx, 32);
	1382	if (*ctx)
	1383	POST_WRITE(*ctx, 32);
1285	1384	}
1286	1385	}
1287	1386
...	...	@@ -1289,16 +1388,21 @@ PRE_SYSCALL(io_destroy)(long ctx) {}
1289	1388
1290	1389	POST_SYSCALL(io_destroy)(long res, long ctx) {}
1291	1390
1292		PRE_SYSCALL(io_getevents)(long ctx_id, long min_nr, long nr,
1293		__sanitizer_io_event ioevpp, void timeout) {
1294		if (timeout) PRE_READ(timeout, struct_timespec_sz);
	1391	PRE_SYSCALL(io_getevents)
	1392	(long ctx_id, long min_nr, long nr, __sanitizer_io_event *ioevpp,
	1393	void *timeout) {
	1394	if (timeout)
	1395	PRE_READ(timeout, struct_timespec_sz);
1295	1396	}
1296	1397
1297		POST_SYSCALL(io_getevents)(long res, long ctx_id, long min_nr, long nr,
1298		__sanitizer_io_event ioevpp, void timeout) {
	1398	POST_SYSCALL(io_getevents)
	1399	(long res, long ctx_id, long min_nr, long nr, __sanitizer_io_event *ioevpp,
	1400	void *timeout) {
1299	1401	if (res >= 0) {
1300		if (ioevpp) POST_WRITE(ioevpp, res * sizeof(*ioevpp));
1301		if (timeout) POST_WRITE(timeout, struct_timespec_sz);
	1402	if (ioevpp)
	1403	POST_WRITE(ioevpp, res * sizeof(*ioevpp));
	1404	if (timeout)
	1405	POST_WRITE(timeout, struct_timespec_sz);
1302	1406	}
1303	1407	for (long i = 0; i < res; i++) {
1304	1408	// We synchronize io_submit -> io_getevents/io_cancel using the
...	...	@@ -1308,26 +1412,26 @@ POST_SYSCALL(io_getevents)(long res, long ctx_id, long min_nr, long nr,
1308	1412	// synchronize on 0. But there does not seem to be a better solution
1309	1413	// (except wrapping all operations in own context, which is unreliable).
1310	1414	// We can not reliably extract fildes in io_getevents.
1311		COMMON_SYSCALL_ACQUIRE((void*)ioevpp[i].data);
	1415	COMMON_SYSCALL_ACQUIRE((void *)ioevpp[i].data);
1312	1416	}
1313	1417	}
1314	1418
1315	1419	PRE_SYSCALL(io_submit)(long ctx_id, long nr, __sanitizer_iocb **iocbpp) {
1316	1420	for (long i = 0; i < nr; ++i) {
1317	1421	uptr op = iocbpp[i]->aio_lio_opcode;
1318		void data = (void)iocbpp[i]->aio_data;
1319		void buf = (void)iocbpp[i]->aio_buf;
	1422	void data = (void )iocbpp[i]->aio_data;
	1423	void buf = (void )iocbpp[i]->aio_buf;
1320	1424	uptr len = (uptr)iocbpp[i]->aio_nbytes;
1321	1425	if (op == iocb_cmd_pwrite && buf && len) {
1322	1426	PRE_READ(buf, len);
1323	1427	} else if (op == iocb_cmd_pread && buf && len) {
1324	1428	POST_WRITE(buf, len);
1325	1429	} else if (op == iocb_cmd_pwritev) {
1326		__sanitizer_iovec iovec = (__sanitizer_iovec)buf;
	1430	__sanitizer_iovec iovec = (__sanitizer_iovec )buf;
1327	1431	for (uptr v = 0; v < len; v++)
1328	1432	PRE_READ(iovec[v].iov_base, iovec[v].iov_len);
1329	1433	} else if (op == iocb_cmd_preadv) {
1330		__sanitizer_iovec iovec = (__sanitizer_iovec)buf;
	1434	__sanitizer_iovec iovec = (__sanitizer_iovec )buf;
1331	1435	for (uptr v = 0; v < len; v++)
1332	1436	POST_WRITE(iovec[v].iov_base, iovec[v].iov_len);
1333	1437	}
...	...	@@ -1336,19 +1440,18 @@ PRE_SYSCALL(io_submit)(long ctx_id, long nr, __sanitizer_iocb **iocbpp) {
1336	1440	}
1337	1441	}
1338	1442
1339		POST_SYSCALL(io_submit)(long res, long ctx_id, long nr,
1340		__sanitizer_iocb **iocbpp) {}
	1443	POST_SYSCALL(io_submit)
	1444	(long res, long ctx_id, long nr, __sanitizer_iocb **iocbpp) {}
1341	1445
1342		PRE_SYSCALL(io_cancel)(long ctx_id, __sanitizer_iocb *iocb,
1343		__sanitizer_io_event *result) {
1344		}
	1446	PRE_SYSCALL(io_cancel)
	1447	(long ctx_id, __sanitizer_iocb iocb, __sanitizer_io_event result) {}
1345	1448
1346		POST_SYSCALL(io_cancel)(long res, long ctx_id, __sanitizer_iocb *iocb,
1347		__sanitizer_io_event *result) {
	1449	POST_SYSCALL(io_cancel)
	1450	(long res, long ctx_id, __sanitizer_iocb iocb, __sanitizer_io_event result) {
1348	1451	if (res == 0) {
1349	1452	if (result) {
1350	1453	// See comment in io_getevents.
1351		COMMON_SYSCALL_ACQUIRE((void*)result->data);
	1454	COMMON_SYSCALL_ACQUIRE((void *)result->data);
1352	1455	POST_WRITE(result, sizeof(*result));
1353	1456	}
1354	1457	if (iocb)
...	...	@@ -1358,19 +1461,23 @@ POST_SYSCALL(io_cancel)(long res, long ctx_id, __sanitizer_iocb *iocb,
1358	1461
1359	1462	PRE_SYSCALL(sendfile)(long out_fd, long in_fd, void *offset, long count) {}
1360	1463
1361		POST_SYSCALL(sendfile)(long res, long out_fd, long in_fd,
1362		__sanitizer___kernel_off_t *offset, long count) {
	1464	POST_SYSCALL(sendfile)
	1465	(long res, long out_fd, long in_fd, __sanitizer___kernel_off_t *offset,
	1466	long count) {
1363	1467	if (res >= 0) {
1364		if (offset) POST_WRITE(offset, sizeof(*offset));
	1468	if (offset)
	1469	POST_WRITE(offset, sizeof(*offset));
1365	1470	}
1366	1471	}
1367	1472
1368	1473	PRE_SYSCALL(sendfile64)(long out_fd, long in_fd, void *offset, long count) {}
1369	1474
1370		POST_SYSCALL(sendfile64)(long res, long out_fd, long in_fd,
1371		__sanitizer___kernel_loff_t *offset, long count) {
	1475	POST_SYSCALL(sendfile64)
	1476	(long res, long out_fd, long in_fd, __sanitizer___kernel_loff_t *offset,
	1477	long count) {
1372	1478	if (res >= 0) {
1373		if (offset) POST_WRITE(offset, sizeof(*offset));
	1479	if (offset)
	1480	POST_WRITE(offset, sizeof(*offset));
1374	1481	}
1375	1482	}
1376	1483
...	...	@@ -1402,9 +1509,7 @@ PRE_SYSCALL(open)(const void *filename, long flags, long mode) {
1402	1509
1403	1510	POST_SYSCALL(open)(long res, const void *filename, long flags, long mode) {}
1404	1511
1405		PRE_SYSCALL(close)(long fd) {
1406		COMMON_SYSCALL_FD_CLOSE((int)fd);
1407		}
	1512	PRE_SYSCALL(close)(long fd) { COMMON_SYSCALL_FD_CLOSE((int)fd); }
1408	1513
1409	1514	POST_SYSCALL(close)(long res, long fd) {}
1410	1515
...	...	@@ -1440,7 +1545,7 @@ PRE_SYSCALL(fchown)(long fd, long user, long group) {}
1440	1545
1441	1546	POST_SYSCALL(fchown)(long res, long fd, long user, long group) {}
1442	1547
1443		#if SANITIZER_USES_UID16_SYSCALLS
	1548	# if SANITIZER_USES_UID16_SYSCALLS
1444	1549	PRE_SYSCALL(chown16)(const void *filename, long user, long group) {
1445	1550	if (filename)
1446	1551	PRE_READ(filename,
...	...	@@ -1483,13 +1588,16 @@ POST_SYSCALL(setresuid16)(long res, long ruid, long euid, long suid) {}
1483	1588
1484	1589	PRE_SYSCALL(getresuid16)(void ruid, void euid, void *suid) {}
1485	1590
1486		POST_SYSCALL(getresuid16)(long res, __sanitizer___kernel_old_uid_t *ruid,
1487		__sanitizer___kernel_old_uid_t *euid,
1488		__sanitizer___kernel_old_uid_t *suid) {
	1591	POST_SYSCALL(getresuid16)
	1592	(long res, __sanitizer___kernel_old_uid_t *ruid,
	1593	__sanitizer___kernel_old_uid_t euid, __sanitizer___kernel_old_uid_t suid) {
1489	1594	if (res >= 0) {
1490		if (ruid) POST_WRITE(ruid, sizeof(*ruid));
1491		if (euid) POST_WRITE(euid, sizeof(*euid));
1492		if (suid) POST_WRITE(suid, sizeof(*suid));
	1595	if (ruid)
	1596	POST_WRITE(ruid, sizeof(*ruid));
	1597	if (euid)
	1598	POST_WRITE(euid, sizeof(*euid));
	1599	if (suid)
	1600	POST_WRITE(suid, sizeof(*suid));
1493	1601	}
1494	1602	}
1495	1603
...	...	@@ -1499,13 +1607,16 @@ POST_SYSCALL(setresgid16)(long res, long rgid, long egid, long sgid) {}
1499	1607
1500	1608	PRE_SYSCALL(getresgid16)(void rgid, void egid, void *sgid) {}
1501	1609
1502		POST_SYSCALL(getresgid16)(long res, __sanitizer___kernel_old_gid_t *rgid,
1503		__sanitizer___kernel_old_gid_t *egid,
1504		__sanitizer___kernel_old_gid_t *sgid) {
	1610	POST_SYSCALL(getresgid16)
	1611	(long res, __sanitizer___kernel_old_gid_t *rgid,
	1612	__sanitizer___kernel_old_gid_t egid, __sanitizer___kernel_old_gid_t sgid) {
1505	1613	if (res >= 0) {
1506		if (rgid) POST_WRITE(rgid, sizeof(*rgid));
1507		if (egid) POST_WRITE(egid, sizeof(*egid));
1508		if (sgid) POST_WRITE(sgid, sizeof(*sgid));
	1614	if (rgid)
	1615	POST_WRITE(rgid, sizeof(*rgid));
	1616	if (egid)
	1617	POST_WRITE(egid, sizeof(*egid));
	1618	if (sgid)
	1619	POST_WRITE(sgid, sizeof(*sgid));
1509	1620	}
1510	1621	}
1511	1622
...	...	@@ -1517,23 +1628,25 @@ PRE_SYSCALL(setfsgid16)(long gid) {}
1517	1628
1518	1629	POST_SYSCALL(setfsgid16)(long res, long gid) {}
1519	1630
1520		PRE_SYSCALL(getgroups16)(long gidsetsize,
1521		__sanitizer___kernel_old_gid_t *grouplist) {}
	1631	PRE_SYSCALL(getgroups16)
	1632	(long gidsetsize, __sanitizer___kernel_old_gid_t *grouplist) {}
1522	1633
1523		POST_SYSCALL(getgroups16)(long res, long gidsetsize,
1524		__sanitizer___kernel_old_gid_t *grouplist) {
	1634	POST_SYSCALL(getgroups16)
	1635	(long res, long gidsetsize, __sanitizer___kernel_old_gid_t *grouplist) {
1525	1636	if (res >= 0) {
1526		if (grouplist) POST_WRITE(grouplist, res * sizeof(*grouplist));
	1637	if (grouplist)
	1638	POST_WRITE(grouplist, res * sizeof(*grouplist));
1527	1639	}
1528	1640	}
1529	1641
1530		PRE_SYSCALL(setgroups16)(long gidsetsize,
1531		__sanitizer___kernel_old_gid_t *grouplist) {
1532		if (grouplist) POST_WRITE(grouplist, gidsetsize * sizeof(*grouplist));
	1642	PRE_SYSCALL(setgroups16)
	1643	(long gidsetsize, __sanitizer___kernel_old_gid_t *grouplist) {
	1644	if (grouplist)
	1645	POST_WRITE(grouplist, gidsetsize * sizeof(*grouplist));
1533	1646	}
1534	1647
1535		POST_SYSCALL(setgroups16)(long res, long gidsetsize,
1536		__sanitizer___kernel_old_gid_t *grouplist) {}
	1648	POST_SYSCALL(setgroups16)
	1649	(long res, long gidsetsize, __sanitizer___kernel_old_gid_t *grouplist) {}
1537	1650
1538	1651	PRE_SYSCALL(getuid16)() {}
1539	1652
...	...	@@ -1550,7 +1663,7 @@ POST_SYSCALL(getgid16)(long res) {}
1550	1663	PRE_SYSCALL(getegid16)() {}
1551	1664
1552	1665	POST_SYSCALL(getegid16)(long res) {}
1553		#endif // SANITIZER_USES_UID16_SYSCALLS
	1666	# endif // SANITIZER_USES_UID16_SYSCALLS
1554	1667
1555	1668	PRE_SYSCALL(utime)(void filename, void times) {}
1556	1669
...	...	@@ -1559,7 +1672,8 @@ POST_SYSCALL(utime)(long res, void filename, void times) {
1559	1672	if (filename)
1560	1673	POST_WRITE(filename,
1561	1674	__sanitizer::internal_strlen((const char *)filename) + 1);
1562		if (times) POST_WRITE(times, struct_utimbuf_sz);
	1675	if (times)
	1676	POST_WRITE(times, struct_utimbuf_sz);
1563	1677	}
1564	1678	}
1565	1679
...	...	@@ -1570,7 +1684,8 @@ POST_SYSCALL(utimes)(long res, void filename, void utimes) {
1570	1684	if (filename)
1571	1685	POST_WRITE(filename,
1572	1686	__sanitizer::internal_strlen((const char *)filename) + 1);
1573		if (utimes) POST_WRITE(utimes, timeval_sz);
	1687	if (utimes)
	1688	POST_WRITE(utimes, timeval_sz);
1574	1689	}
1575	1690	}
1576	1691
...	...	@@ -1578,91 +1693,104 @@ PRE_SYSCALL(lseek)(long fd, long offset, long origin) {}
1578	1693
1579	1694	POST_SYSCALL(lseek)(long res, long fd, long offset, long origin) {}
1580	1695
1581		PRE_SYSCALL(llseek)(long fd, long offset_high, long offset_low, void *result,
1582		long origin) {}
	1696	PRE_SYSCALL(llseek)
	1697	(long fd, long offset_high, long offset_low, void *result, long origin) {}
1583	1698
1584		POST_SYSCALL(llseek)(long res, long fd, long offset_high, long offset_low,
1585		void *result, long origin) {
	1699	POST_SYSCALL(llseek)
	1700	(long res, long fd, long offset_high, long offset_low, void *result,
	1701	long origin) {
1586	1702	if (res >= 0) {
1587		if (result) POST_WRITE(result, sizeof(long long));
	1703	if (result)
	1704	POST_WRITE(result, sizeof(long long));
1588	1705	}
1589	1706	}
1590	1707
1591	1708	PRE_SYSCALL(readv)(long fd, const __sanitizer_iovec *vec, long vlen) {}
1592	1709
1593		POST_SYSCALL(readv)(long res, long fd, const __sanitizer_iovec *vec,
1594		long vlen) {
	1710	POST_SYSCALL(readv)
	1711	(long res, long fd, const __sanitizer_iovec *vec, long vlen) {
1595	1712	if (res >= 0) {
1596		if (vec) kernel_write_iovec(vec, vlen, res);
	1713	if (vec)
	1714	kernel_write_iovec(vec, vlen, res);
1597	1715	}
1598	1716	}
1599	1717
1600	1718	PRE_SYSCALL(write)(long fd, const void *buf, long count) {
1601		if (buf) PRE_READ(buf, count);
	1719	if (buf)
	1720	PRE_READ(buf, count);
1602	1721	}
1603	1722
1604	1723	POST_SYSCALL(write)(long res, long fd, const void *buf, long count) {}
1605	1724
1606	1725	PRE_SYSCALL(writev)(long fd, const __sanitizer_iovec *vec, long vlen) {}
1607	1726
1608		POST_SYSCALL(writev)(long res, long fd, const __sanitizer_iovec *vec,
1609		long vlen) {
	1727	POST_SYSCALL(writev)
	1728	(long res, long fd, const __sanitizer_iovec *vec, long vlen) {
1610	1729	if (res >= 0) {
1611		if (vec) kernel_read_iovec(vec, vlen, res);
	1730	if (vec)
	1731	kernel_read_iovec(vec, vlen, res);
1612	1732	}
1613	1733	}
1614	1734
1615		#ifdef _LP64
	1735	# ifdef _LP64
1616	1736	PRE_SYSCALL(pread64)(long fd, void *buf, long count, long pos) {}
1617	1737
1618	1738	POST_SYSCALL(pread64)(long res, long fd, void *buf, long count, long pos) {
1619	1739	if (res >= 0) {
1620		if (buf) POST_WRITE(buf, res);
	1740	if (buf)
	1741	POST_WRITE(buf, res);
1621	1742	}
1622	1743	}
1623	1744
1624	1745	PRE_SYSCALL(pwrite64)(long fd, const void *buf, long count, long pos) {
1625		if (buf) PRE_READ(buf, count);
	1746	if (buf)
	1747	PRE_READ(buf, count);
1626	1748	}
1627	1749
1628		POST_SYSCALL(pwrite64)(long res, long fd, const void *buf, long count,
1629		long pos) {}
1630		#else
	1750	POST_SYSCALL(pwrite64)
	1751	(long res, long fd, const void *buf, long count, long pos) {}
	1752	# else
1631	1753	PRE_SYSCALL(pread64)(long fd, void *buf, long count, long pos0, long pos1) {}
1632	1754
1633		POST_SYSCALL(pread64)(long res, long fd, void *buf, long count, long pos0,
1634		long pos1) {
	1755	POST_SYSCALL(pread64)
	1756	(long res, long fd, void *buf, long count, long pos0, long pos1) {
1635	1757	if (res >= 0) {
1636		if (buf) POST_WRITE(buf, res);
	1758	if (buf)
	1759	POST_WRITE(buf, res);
1637	1760	}
1638	1761	}
1639	1762
1640		PRE_SYSCALL(pwrite64)(long fd, const void *buf, long count, long pos0,
1641		long pos1) {
1642		if (buf) PRE_READ(buf, count);
	1763	PRE_SYSCALL(pwrite64)
	1764	(long fd, const void *buf, long count, long pos0, long pos1) {
	1765	if (buf)
	1766	PRE_READ(buf, count);
1643	1767	}
1644	1768
1645		POST_SYSCALL(pwrite64)(long res, long fd, const void *buf, long count,
1646		long pos0, long pos1) {}
1647		#endif
	1769	POST_SYSCALL(pwrite64)
	1770	(long res, long fd, const void *buf, long count, long pos0, long pos1) {}
	1771	# endif
1648	1772
1649		PRE_SYSCALL(preadv)(long fd, const __sanitizer_iovec *vec, long vlen,
1650		long pos_l, long pos_h) {}
	1773	PRE_SYSCALL(preadv)
	1774	(long fd, const __sanitizer_iovec *vec, long vlen, long pos_l, long pos_h) {}
1651	1775
1652		POST_SYSCALL(preadv)(long res, long fd, const __sanitizer_iovec *vec, long vlen,
1653		long pos_l, long pos_h) {
	1776	POST_SYSCALL(preadv)
	1777	(long res, long fd, const __sanitizer_iovec *vec, long vlen, long pos_l,
	1778	long pos_h) {
1654	1779	if (res >= 0) {
1655		if (vec) kernel_write_iovec(vec, vlen, res);
	1780	if (vec)
	1781	kernel_write_iovec(vec, vlen, res);
1656	1782	}
1657	1783	}
1658	1784
1659		PRE_SYSCALL(pwritev)(long fd, const __sanitizer_iovec *vec, long vlen,
1660		long pos_l, long pos_h) {}
	1785	PRE_SYSCALL(pwritev)
	1786	(long fd, const __sanitizer_iovec *vec, long vlen, long pos_l, long pos_h) {}
1661	1787
1662		POST_SYSCALL(pwritev)(long res, long fd, const __sanitizer_iovec *vec,
1663		long vlen, long pos_l, long pos_h) {
	1788	POST_SYSCALL(pwritev)
	1789	(long res, long fd, const __sanitizer_iovec *vec, long vlen, long pos_l,
	1790	long pos_h) {
1664	1791	if (res >= 0) {
1665		if (vec) kernel_read_iovec(vec, vlen, res);
	1792	if (vec)
	1793	kernel_read_iovec(vec, vlen, res);
1666	1794	}
1667	1795	}
1668	1796
...	...	@@ -1717,14 +1845,15 @@ PRE_SYSCALL(quotactl)(long cmd, const void special, long id, void addr) {
1717	1845	PRE_READ(special, __sanitizer::internal_strlen((const char *)special) + 1);
1718	1846	}
1719	1847
1720		POST_SYSCALL(quotactl)(long res, long cmd, const void *special, long id,
1721		void *addr) {}
	1848	POST_SYSCALL(quotactl)
	1849	(long res, long cmd, const void special, long id, void addr) {}
1722	1850
1723	1851	PRE_SYSCALL(getdents)(long fd, void *dirent, long count) {}
1724	1852
1725	1853	POST_SYSCALL(getdents)(long res, long fd, void *dirent, long count) {
1726	1854	if (res >= 0) {
1727		if (dirent) POST_WRITE(dirent, res);
	1855	if (dirent)
	1856	POST_WRITE(dirent, res);
1728	1857	}
1729	1858	}
1730	1859
...	...	@@ -1732,15 +1861,16 @@ PRE_SYSCALL(getdents64)(long fd, void *dirent, long count) {}
1732	1861
1733	1862	POST_SYSCALL(getdents64)(long res, long fd, void *dirent, long count) {
1734	1863	if (res >= 0) {
1735		if (dirent) POST_WRITE(dirent, res);
	1864	if (dirent)
	1865	POST_WRITE(dirent, res);
1736	1866	}
1737	1867	}
1738	1868
1739		PRE_SYSCALL(setsockopt)(long fd, long level, long optname, void *optval,
1740		long optlen) {}
	1869	PRE_SYSCALL(setsockopt)
	1870	(long fd, long level, long optname, void *optval, long optlen) {}
1741	1871
1742		POST_SYSCALL(setsockopt)(long res, long fd, long level, long optname,
1743		void *optval, long optlen) {
	1872	POST_SYSCALL(setsockopt)
	1873	(long res, long fd, long level, long optname, void *optval, long optlen) {
1744	1874	if (res >= 0) {
1745	1875	if (optval)
1746	1876	POST_WRITE(optval,
...	...	@@ -1748,77 +1878,88 @@ POST_SYSCALL(setsockopt)(long res, long fd, long level, long optname,
1748	1878	}
1749	1879	}
1750	1880
1751		PRE_SYSCALL(getsockopt)(long fd, long level, long optname, void *optval,
1752		void *optlen) {}
	1881	PRE_SYSCALL(getsockopt)
	1882	(long fd, long level, long optname, void optval, void optlen) {}
1753	1883
1754		POST_SYSCALL(getsockopt)(long res, long fd, long level, long optname,
1755		void optval, void optlen) {
	1884	POST_SYSCALL(getsockopt)
	1885	(long res, long fd, long level, long optname, void optval, void optlen) {
1756	1886	if (res >= 0) {
1757	1887	if (optval)
1758	1888	POST_WRITE(optval,
1759	1889	__sanitizer::internal_strlen((const char *)optval) + 1);
1760		if (optlen) POST_WRITE(optlen, sizeof(int));
	1890	if (optlen)
	1891	POST_WRITE(optlen, sizeof(int));
1761	1892	}
1762	1893	}
1763	1894
1764	1895	PRE_SYSCALL(bind)(long arg0, sanitizer_kernel_sockaddr *arg1, long arg2) {}
1765	1896
1766		POST_SYSCALL(bind)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
1767		long arg2) {
	1897	POST_SYSCALL(bind)
	1898	(long res, long arg0, sanitizer_kernel_sockaddr *arg1, long arg2) {
1768	1899	if (res >= 0) {
1769		if (arg1) POST_WRITE(arg1, sizeof(*arg1));
	1900	if (arg1)
	1901	POST_WRITE(arg1, sizeof(*arg1));
1770	1902	}
1771	1903	}
1772	1904
1773	1905	PRE_SYSCALL(connect)(long arg0, sanitizer_kernel_sockaddr *arg1, long arg2) {}
1774	1906
1775		POST_SYSCALL(connect)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
1776		long arg2) {
	1907	POST_SYSCALL(connect)
	1908	(long res, long arg0, sanitizer_kernel_sockaddr *arg1, long arg2) {
1777	1909	if (res >= 0) {
1778		if (arg1) POST_WRITE(arg1, sizeof(*arg1));
	1910	if (arg1)
	1911	POST_WRITE(arg1, sizeof(*arg1));
1779	1912	}
1780	1913	}
1781	1914
1782	1915	PRE_SYSCALL(accept)(long arg0, sanitizer_kernel_sockaddr arg1, void arg2) {}
1783	1916
1784		POST_SYSCALL(accept)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
1785		void *arg2) {
	1917	POST_SYSCALL(accept)
	1918	(long res, long arg0, sanitizer_kernel_sockaddr arg1, void arg2) {
1786	1919	if (res >= 0) {
1787		if (arg1) POST_WRITE(arg1, sizeof(*arg1));
1788		if (arg2) POST_WRITE(arg2, sizeof(unsigned));
	1920	if (arg1)
	1921	POST_WRITE(arg1, sizeof(*arg1));
	1922	if (arg2)
	1923	POST_WRITE(arg2, sizeof(unsigned));
1789	1924	}
1790	1925	}
1791	1926
1792		PRE_SYSCALL(accept4)(long arg0, sanitizer_kernel_sockaddr arg1, void arg2,
1793		long arg3) {}
	1927	PRE_SYSCALL(accept4)
	1928	(long arg0, sanitizer_kernel_sockaddr arg1, void arg2, long arg3) {}
1794	1929
1795		POST_SYSCALL(accept4)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
1796		void *arg2, long arg3) {
	1930	POST_SYSCALL(accept4)
	1931	(long res, long arg0, sanitizer_kernel_sockaddr arg1, void arg2, long arg3) {
1797	1932	if (res >= 0) {
1798		if (arg1) POST_WRITE(arg1, sizeof(*arg1));
1799		if (arg2) POST_WRITE(arg2, sizeof(unsigned));
	1933	if (arg1)
	1934	POST_WRITE(arg1, sizeof(*arg1));
	1935	if (arg2)
	1936	POST_WRITE(arg2, sizeof(unsigned));
1800	1937	}
1801	1938	}
1802	1939
1803		PRE_SYSCALL(getsockname)(long arg0, sanitizer_kernel_sockaddr *arg1,
1804		void *arg2) {}
	1940	PRE_SYSCALL(getsockname)
	1941	(long arg0, sanitizer_kernel_sockaddr arg1, void arg2) {}
1805	1942
1806		POST_SYSCALL(getsockname)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
1807		void *arg2) {
	1943	POST_SYSCALL(getsockname)
	1944	(long res, long arg0, sanitizer_kernel_sockaddr arg1, void arg2) {
1808	1945	if (res >= 0) {
1809		if (arg1) POST_WRITE(arg1, sizeof(*arg1));
1810		if (arg2) POST_WRITE(arg2, sizeof(unsigned));
	1946	if (arg1)
	1947	POST_WRITE(arg1, sizeof(*arg1));
	1948	if (arg2)
	1949	POST_WRITE(arg2, sizeof(unsigned));
1811	1950	}
1812	1951	}
1813	1952
1814		PRE_SYSCALL(getpeername)(long arg0, sanitizer_kernel_sockaddr *arg1,
1815		void *arg2) {}
	1953	PRE_SYSCALL(getpeername)
	1954	(long arg0, sanitizer_kernel_sockaddr arg1, void arg2) {}
1816	1955
1817		POST_SYSCALL(getpeername)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
1818		void *arg2) {
	1956	POST_SYSCALL(getpeername)
	1957	(long res, long arg0, sanitizer_kernel_sockaddr arg1, void arg2) {
1819	1958	if (res >= 0) {
1820		if (arg1) POST_WRITE(arg1, sizeof(*arg1));
1821		if (arg2) POST_WRITE(arg2, sizeof(unsigned));
	1959	if (arg1)
	1960	POST_WRITE(arg1, sizeof(*arg1));
	1961	if (arg2)
	1962	POST_WRITE(arg2, sizeof(unsigned));
1822	1963	}
1823	1964	}
1824	1965
...	...	@@ -1826,18 +1967,23 @@ PRE_SYSCALL(send)(long arg0, void *arg1, long arg2, long arg3) {}
1826	1967
1827	1968	POST_SYSCALL(send)(long res, long arg0, void *arg1, long arg2, long arg3) {
1828	1969	if (res) {
1829		if (arg1) POST_READ(arg1, res);
	1970	if (arg1)
	1971	POST_READ(arg1, res);
1830	1972	}
1831	1973	}
1832	1974
1833		PRE_SYSCALL(sendto)(long arg0, void *arg1, long arg2, long arg3,
1834		sanitizer_kernel_sockaddr *arg4, long arg5) {}
	1975	PRE_SYSCALL(sendto)
	1976	(long arg0, void arg1, long arg2, long arg3, sanitizer_kernel_sockaddr arg4,
	1977	long arg5) {}
1835	1978
1836		POST_SYSCALL(sendto)(long res, long arg0, void *arg1, long arg2, long arg3,
1837		sanitizer_kernel_sockaddr *arg4, long arg5) {
	1979	POST_SYSCALL(sendto)
	1980	(long res, long arg0, void *arg1, long arg2, long arg3,
	1981	sanitizer_kernel_sockaddr *arg4, long arg5) {
1838	1982	if (res >= 0) {
1839		if (arg1) POST_READ(arg1, res);
1840		if (arg4) POST_WRITE(arg4, sizeof(*arg4));
	1983	if (arg1)
	1984	POST_READ(arg1, res);
	1985	if (arg4)
	1986	POST_WRITE(arg4, sizeof(*arg4));
1841	1987	}
1842	1988	}
1843	1989
...	...	@@ -1857,19 +2003,25 @@ PRE_SYSCALL(recv)(long arg0, void *buf, long len, long flags) {}
1857	2003
1858	2004	POST_SYSCALL(recv)(long res, void *buf, long len, long flags) {
1859	2005	if (res >= 0) {
1860		if (buf) POST_WRITE(buf, res);
	2006	if (buf)
	2007	POST_WRITE(buf, res);
1861	2008	}
1862	2009	}
1863	2010
1864		PRE_SYSCALL(recvfrom)(long arg0, void *buf, long len, long flags,
1865		sanitizer_kernel_sockaddr arg4, void arg5) {}
	2011	PRE_SYSCALL(recvfrom)
	2012	(long arg0, void buf, long len, long flags, sanitizer_kernel_sockaddr arg4,
	2013	void *arg5) {}
1866	2014
1867		POST_SYSCALL(recvfrom)(long res, long arg0, void *buf, long len, long flags,
1868		sanitizer_kernel_sockaddr arg4, void arg5) {
	2015	POST_SYSCALL(recvfrom)
	2016	(long res, long arg0, void *buf, long len, long flags,
	2017	sanitizer_kernel_sockaddr arg4, void arg5) {
1869	2018	if (res >= 0) {
1870		if (buf) POST_WRITE(buf, res);
1871		if (arg4) POST_WRITE(arg4, sizeof(*arg4));
1872		if (arg5) POST_WRITE(arg5, sizeof(int));
	2019	if (buf)
	2020	POST_WRITE(buf, res);
	2021	if (arg4)
	2022	POST_WRITE(arg4, sizeof(*arg4));
	2023	if (arg5)
	2024	POST_WRITE(arg5, sizeof(int));
1873	2025	}
1874	2026	}
1875	2027
...	...	@@ -1881,14 +2033,16 @@ PRE_SYSCALL(socketpair)(long arg0, long arg1, long arg2, int *sv) {}
1881	2033
1882	2034	POST_SYSCALL(socketpair)(long res, long arg0, long arg1, long arg2, int *sv) {
1883	2035	if (res >= 0)
1884		if (sv) POST_WRITE(sv, sizeof(int) * 2);
	2036	if (sv)
	2037	POST_WRITE(sv, sizeof(int) * 2);
1885	2038	}
1886	2039
1887	2040	PRE_SYSCALL(socketcall)(long call, void *args) {}
1888	2041
1889	2042	POST_SYSCALL(socketcall)(long res, long call, void *args) {
1890	2043	if (res >= 0) {
1891		if (args) POST_WRITE(args, sizeof(long));
	2044	if (args)
	2045	POST_WRITE(args, sizeof(long));
1892	2046	}
1893	2047	}
1894	2048
...	...	@@ -1898,25 +2052,31 @@ POST_SYSCALL(listen)(long res, long arg0, long arg1) {}
1898	2052
1899	2053	PRE_SYSCALL(poll)(void *ufds, long nfds, long timeout) {}
1900	2054
1901		POST_SYSCALL(poll)(long res, __sanitizer_pollfd *ufds, long nfds,
1902		long timeout) {
	2055	POST_SYSCALL(poll)
	2056	(long res, __sanitizer_pollfd *ufds, long nfds, long timeout) {
1903	2057	if (res >= 0) {
1904		if (ufds) POST_WRITE(ufds, nfds * sizeof(*ufds));
	2058	if (ufds)
	2059	POST_WRITE(ufds, nfds * sizeof(*ufds));
1905	2060	}
1906	2061	}
1907	2062
1908		PRE_SYSCALL(select)(long n, __sanitizer___kernel_fd_set *inp,
1909		__sanitizer___kernel_fd_set *outp,
1910		__sanitizer___kernel_fd_set exp, void tvp) {}
	2063	PRE_SYSCALL(select)
	2064	(long n, __sanitizer___kernel_fd_set inp, __sanitizer___kernel_fd_set outp,
	2065	__sanitizer___kernel_fd_set exp, void tvp) {}
1911	2066
1912		POST_SYSCALL(select)(long res, long n, __sanitizer___kernel_fd_set *inp,
1913		__sanitizer___kernel_fd_set *outp,
1914		__sanitizer___kernel_fd_set exp, void tvp) {
	2067	POST_SYSCALL(select)
	2068	(long res, long n, __sanitizer___kernel_fd_set *inp,
	2069	__sanitizer___kernel_fd_set outp, __sanitizer___kernel_fd_set exp,
	2070	void *tvp) {
1915	2071	if (res >= 0) {
1916		if (inp) POST_WRITE(inp, sizeof(*inp));
1917		if (outp) POST_WRITE(outp, sizeof(*outp));
1918		if (exp) POST_WRITE(exp, sizeof(*exp));
1919		if (tvp) POST_WRITE(tvp, timeval_sz);
	2072	if (inp)
	2073	POST_WRITE(inp, sizeof(*inp));
	2074	if (outp)
	2075	POST_WRITE(outp, sizeof(*outp));
	2076	if (exp)
	2077	POST_WRITE(exp, sizeof(*exp));
	2078	if (tvp)
	2079	POST_WRITE(tvp, timeval_sz);
1920	2080	}
1921	2081	}
1922	2082
...	...	@@ -1936,29 +2096,58 @@ PRE_SYSCALL(epoll_ctl)(long epfd, long op, long fd, void *event) {}
1936	2096
1937	2097	POST_SYSCALL(epoll_ctl)(long res, long epfd, long op, long fd, void *event) {
1938	2098	if (res >= 0) {
1939		if (event) POST_WRITE(event, struct_epoll_event_sz);
	2099	if (event)
	2100	POST_WRITE(event, struct_epoll_event_sz);
	2101	}
	2102	}
	2103
	2104	PRE_SYSCALL(epoll_wait)
	2105	(long epfd, void *events, long maxevents, long timeout) {}
	2106
	2107	POST_SYSCALL(epoll_wait)
	2108	(long res, long epfd, void *events, long maxevents, long timeout) {
	2109	if (res >= 0) {
	2110	COMMON_SYSCALL_FD_ACQUIRE(epfd);
	2111	if (events)
	2112	POST_WRITE(events, res * struct_epoll_event_sz);
1940	2113	}
1941	2114	}
1942	2115
1943		PRE_SYSCALL(epoll_wait)(long epfd, void *events, long maxevents, long timeout) {
	2116	PRE_SYSCALL(epoll_pwait)
	2117	(long epfd, void *events, long maxevents, long timeout,
	2118	const kernel_sigset_t *sigmask, long sigsetsize) {
	2119	if (sigmask)
	2120	PRE_READ(sigmask, sigsetsize);
1944	2121	}
1945	2122
1946		POST_SYSCALL(epoll_wait)(long res, long epfd, void *events, long maxevents,
1947		long timeout) {
	2123	POST_SYSCALL(epoll_pwait)
	2124	(long res, long epfd, void *events, long maxevents, long timeout,
	2125	const void *sigmask, long sigsetsize) {
1948	2126	if (res >= 0) {
1949		if (events) POST_WRITE(events, struct_epoll_event_sz);
	2127	COMMON_SYSCALL_FD_ACQUIRE(epfd);
	2128	if (events)
	2129	POST_WRITE(events, res * struct_epoll_event_sz);
1950	2130	}
1951	2131	}
1952	2132
1953		PRE_SYSCALL(epoll_pwait)(long epfd, void *events, long maxevents, long timeout,
1954		const kernel_sigset_t *sigmask, long sigsetsize) {
1955		if (sigmask) PRE_READ(sigmask, sigsetsize);
	2133	PRE_SYSCALL(epoll_pwait2)
	2134	(long epfd, void *events, long maxevents,
	2135	const sanitizer_kernel_timespec timeout, const kernel_sigset_t sigmask,
	2136	long sigsetsize) {
	2137	if (timeout)
	2138	PRE_READ(timeout, sizeof(*timeout));
	2139	if (sigmask)
	2140	PRE_READ(sigmask, sigsetsize);
1956	2141	}
1957	2142
1958		POST_SYSCALL(epoll_pwait)(long res, long epfd, void *events, long maxevents,
1959		long timeout, const void *sigmask, long sigsetsize) {
	2143	POST_SYSCALL(epoll_pwait2)
	2144	(long res, long epfd, void *events, long maxevents,
	2145	const sanitizer_kernel_timespec timeout, const void sigmask,
	2146	long sigsetsize) {
1960	2147	if (res >= 0) {
1961		if (events) POST_WRITE(events, struct_epoll_event_sz);
	2148	COMMON_SYSCALL_FD_ACQUIRE(epfd);
	2149	if (events)
	2150	POST_WRITE(events, res * struct_epoll_event_sz);
1962	2151	}
1963	2152	}
1964	2153
...	...	@@ -1993,7 +2182,8 @@ PRE_SYSCALL(newuname)(void *name) {}
1993	2182
1994	2183	POST_SYSCALL(newuname)(long res, void *name) {
1995	2184	if (res >= 0) {
1996		if (name) POST_WRITE(name, struct_new_utsname_sz);
	2185	if (name)
	2186	POST_WRITE(name, struct_new_utsname_sz);
1997	2187	}
1998	2188	}
1999	2189
...	...	@@ -2001,7 +2191,8 @@ PRE_SYSCALL(uname)(void *arg0) {}
2001	2191
2002	2192	POST_SYSCALL(uname)(long res, void *arg0) {
2003	2193	if (res >= 0) {
2004		if (arg0) POST_WRITE(arg0, struct_old_utsname_sz);
	2194	if (arg0)
	2195	POST_WRITE(arg0, struct_old_utsname_sz);
2005	2196	}
2006	2197	}
2007	2198
...	...	@@ -2009,7 +2200,8 @@ PRE_SYSCALL(olduname)(void *arg0) {}
2009	2200
2010	2201	POST_SYSCALL(olduname)(long res, void *arg0) {
2011	2202	if (res >= 0) {
2012		if (arg0) POST_WRITE(arg0, struct_oldold_utsname_sz);
	2203	if (arg0)
	2204	POST_WRITE(arg0, struct_oldold_utsname_sz);
2013	2205	}
2014	2206	}
2015	2207
...	...	@@ -2017,7 +2209,8 @@ PRE_SYSCALL(getrlimit)(long resource, void *rlim) {}
2017	2209
2018	2210	POST_SYSCALL(getrlimit)(long res, long resource, void *rlim) {
2019	2211	if (res >= 0) {
2020		if (rlim) POST_WRITE(rlim, struct_rlimit_sz);
	2212	if (rlim)
	2213	POST_WRITE(rlim, struct_rlimit_sz);
2021	2214	}
2022	2215	}
2023	2216
...	...	@@ -2025,7 +2218,8 @@ PRE_SYSCALL(old_getrlimit)(long resource, void *rlim) {}
2025	2218
2026	2219	POST_SYSCALL(old_getrlimit)(long res, long resource, void *rlim) {
2027	2220	if (res >= 0) {
2028		if (rlim) POST_WRITE(rlim, struct_rlimit_sz);
	2221	if (rlim)
	2222	POST_WRITE(rlim, struct_rlimit_sz);
2029	2223	}
2030	2224	}
2031	2225
...	...	@@ -2033,29 +2227,33 @@ PRE_SYSCALL(setrlimit)(long resource, void *rlim) {}
2033	2227
2034	2228	POST_SYSCALL(setrlimit)(long res, long resource, void *rlim) {
2035	2229	if (res >= 0) {
2036		if (rlim) POST_WRITE(rlim, struct_rlimit_sz);
	2230	if (rlim)
	2231	POST_WRITE(rlim, struct_rlimit_sz);
2037	2232	}
2038	2233	}
2039	2234
2040		#if !SANITIZER_ANDROID
2041		PRE_SYSCALL(prlimit64)(long pid, long resource, const void *new_rlim,
2042		void *old_rlim) {
2043		if (new_rlim) PRE_READ(new_rlim, struct_rlimit64_sz);
	2235	# if SANITIZER_GLIBC
	2236	PRE_SYSCALL(prlimit64)
	2237	(long pid, long resource, const void new_rlim, void old_rlim) {
	2238	if (new_rlim)
	2239	PRE_READ(new_rlim, struct_rlimit64_sz);
2044	2240	}
2045	2241
2046		POST_SYSCALL(prlimit64)(long res, long pid, long resource, const void *new_rlim,
2047		void *old_rlim) {
	2242	POST_SYSCALL(prlimit64)
	2243	(long res, long pid, long resource, const void new_rlim, void old_rlim) {
2048	2244	if (res >= 0) {
2049		if (old_rlim) POST_WRITE(old_rlim, struct_rlimit64_sz);
	2245	if (old_rlim)
	2246	POST_WRITE(old_rlim, struct_rlimit64_sz);
2050	2247	}
2051	2248	}
2052		#endif
	2249	# endif
2053	2250
2054	2251	PRE_SYSCALL(getrusage)(long who, void *ru) {}
2055	2252
2056	2253	POST_SYSCALL(getrusage)(long res, long who, void *ru) {
2057	2254	if (res >= 0) {
2058		if (ru) POST_WRITE(ru, struct_rusage_sz);
	2255	if (ru)
	2256	POST_WRITE(ru, struct_rusage_sz);
2059	2257	}
2060	2258	}
2061	2259
...	...	@@ -2068,31 +2266,34 @@ PRE_SYSCALL(msgget)(long key, long msgflg) {}
2068	2266	POST_SYSCALL(msgget)(long res, long key, long msgflg) {}
2069	2267
2070	2268	PRE_SYSCALL(msgsnd)(long msqid, void *msgp, long msgsz, long msgflg) {
2071		if (msgp) PRE_READ(msgp, msgsz);
	2269	if (msgp)
	2270	PRE_READ(msgp, msgsz);
2072	2271	}
2073	2272
2074		POST_SYSCALL(msgsnd)(long res, long msqid, void *msgp, long msgsz,
2075		long msgflg) {}
	2273	POST_SYSCALL(msgsnd)
	2274	(long res, long msqid, void *msgp, long msgsz, long msgflg) {}
2076	2275
2077		PRE_SYSCALL(msgrcv)(long msqid, void *msgp, long msgsz, long msgtyp,
2078		long msgflg) {}
	2276	PRE_SYSCALL(msgrcv)
	2277	(long msqid, void *msgp, long msgsz, long msgtyp, long msgflg) {}
2079	2278
2080		POST_SYSCALL(msgrcv)(long res, long msqid, void *msgp, long msgsz, long msgtyp,
2081		long msgflg) {
	2279	POST_SYSCALL(msgrcv)
	2280	(long res, long msqid, void *msgp, long msgsz, long msgtyp, long msgflg) {
2082	2281	if (res >= 0) {
2083		if (msgp) POST_WRITE(msgp, res);
	2282	if (msgp)
	2283	POST_WRITE(msgp, res);
2084	2284	}
2085	2285	}
2086	2286
2087		#if !SANITIZER_ANDROID
	2287	# if !SANITIZER_ANDROID
2088	2288	PRE_SYSCALL(msgctl)(long msqid, long cmd, void *buf) {}
2089	2289
2090	2290	POST_SYSCALL(msgctl)(long res, long msqid, long cmd, void *buf) {
2091	2291	if (res >= 0) {
2092		if (buf) POST_WRITE(buf, struct_msqid_ds_sz);
	2292	if (buf)
	2293	POST_WRITE(buf, struct_msqid_ds_sz);
2093	2294	}
2094	2295	}
2095		#endif
	2296	# endif
2096	2297
2097	2298	PRE_SYSCALL(semget)(long key, long nsems, long semflg) {}
2098	2299
...	...	@@ -2106,13 +2307,14 @@ PRE_SYSCALL(semctl)(long semid, long semnum, long cmd, void *arg) {}
2106	2307
2107	2308	POST_SYSCALL(semctl)(long res, long semid, long semnum, long cmd, void *arg) {}
2108	2309
2109		PRE_SYSCALL(semtimedop)(long semid, void *sops, long nsops,
2110		const void *timeout) {
2111		if (timeout) PRE_READ(timeout, struct_timespec_sz);
	2310	PRE_SYSCALL(semtimedop)
	2311	(long semid, void sops, long nsops, const void timeout) {
	2312	if (timeout)
	2313	PRE_READ(timeout, struct_timespec_sz);
2112	2314	}
2113	2315
2114		POST_SYSCALL(semtimedop)(long res, long semid, void *sops, long nsops,
2115		const void *timeout) {}
	2316	POST_SYSCALL(semtimedop)
	2317	(long res, long semid, void sops, long nsops, const void timeout) {}
2116	2318
2117	2319	PRE_SYSCALL(shmat)(long shmid, void *shmaddr, long shmflg) {}
2118	2320
...	...	@@ -2138,18 +2340,20 @@ POST_SYSCALL(shmdt)(long res, void *shmaddr) {
2138	2340	}
2139	2341	}
2140	2342
2141		PRE_SYSCALL(ipc)(long call, long first, long second, long third, void *ptr,
2142		long fifth) {}
	2343	PRE_SYSCALL(ipc)
	2344	(long call, long first, long second, long third, void *ptr, long fifth) {}
2143	2345
2144		POST_SYSCALL(ipc)(long res, long call, long first, long second, long third,
2145		void *ptr, long fifth) {}
	2346	POST_SYSCALL(ipc)
	2347	(long res, long call, long first, long second, long third, void *ptr,
	2348	long fifth) {}
2146	2349
2147		#if !SANITIZER_ANDROID
	2350	# if !SANITIZER_ANDROID
2148	2351	PRE_SYSCALL(shmctl)(long shmid, long cmd, void *buf) {}
2149	2352
2150	2353	POST_SYSCALL(shmctl)(long res, long shmid, long cmd, void *buf) {
2151	2354	if (res >= 0) {
2152		if (buf) POST_WRITE(buf, sizeof(__sanitizer_shmid_ds));
	2355	if (buf)
	2356	POST_WRITE(buf, sizeof(__sanitizer_shmid_ds));
2153	2357	}
2154	2358	}
2155	2359
...	...	@@ -2158,10 +2362,11 @@ PRE_SYSCALL(mq_open)(const void name, long oflag, long mode, void attr) {
2158	2362	PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
2159	2363	}
2160	2364
2161		POST_SYSCALL(mq_open)(long res, const void *name, long oflag, long mode,
2162		void *attr) {
	2365	POST_SYSCALL(mq_open)
	2366	(long res, const void name, long oflag, long mode, void attr) {
2163	2367	if (res >= 0) {
2164		if (attr) POST_WRITE(attr, struct_mq_attr_sz);
	2368	if (attr)
	2369	POST_WRITE(attr, struct_mq_attr_sz);
2165	2370	}
2166	2371	}
2167	2372
...	...	@@ -2172,62 +2377,73 @@ PRE_SYSCALL(mq_unlink)(const void *name) {
2172	2377
2173	2378	POST_SYSCALL(mq_unlink)(long res, const void *name) {}
2174	2379
2175		PRE_SYSCALL(mq_timedsend)(long mqdes, const void *msg_ptr, long msg_len,
2176		long msg_prio, const void *abs_timeout) {
2177		if (msg_ptr) PRE_READ(msg_ptr, msg_len);
2178		if (abs_timeout) PRE_READ(abs_timeout, struct_timespec_sz);
	2380	PRE_SYSCALL(mq_timedsend)
	2381	(long mqdes, const void *msg_ptr, long msg_len, long msg_prio,
	2382	const void *abs_timeout) {
	2383	if (msg_ptr)
	2384	PRE_READ(msg_ptr, msg_len);
	2385	if (abs_timeout)
	2386	PRE_READ(abs_timeout, struct_timespec_sz);
2179	2387	}
2180	2388
2181		POST_SYSCALL(mq_timedsend)(long res, long mqdes, const void *msg_ptr,
2182		long msg_len, long msg_prio,
2183		const void *abs_timeout) {}
	2389	POST_SYSCALL(mq_timedsend)
	2390	(long res, long mqdes, const void *msg_ptr, long msg_len, long msg_prio,
	2391	const void *abs_timeout) {}
2184	2392
2185		PRE_SYSCALL(mq_timedreceive)(long mqdes, void *msg_ptr, long msg_len,
2186		void msg_prio, const void abs_timeout) {
2187		if (abs_timeout) PRE_READ(abs_timeout, struct_timespec_sz);
	2393	PRE_SYSCALL(mq_timedreceive)
	2394	(long mqdes, void msg_ptr, long msg_len, void msg_prio,
	2395	const void *abs_timeout) {
	2396	if (abs_timeout)
	2397	PRE_READ(abs_timeout, struct_timespec_sz);
2188	2398	}
2189	2399
2190		POST_SYSCALL(mq_timedreceive)(long res, long mqdes, void *msg_ptr, long msg_len,
2191		int msg_prio, const void abs_timeout) {
	2400	POST_SYSCALL(mq_timedreceive)
	2401	(long res, long mqdes, void msg_ptr, long msg_len, int msg_prio,
	2402	const void *abs_timeout) {
2192	2403	if (res >= 0) {
2193		if (msg_ptr) POST_WRITE(msg_ptr, res);
2194		if (msg_prio) POST_WRITE(msg_prio, sizeof(*msg_prio));
	2404	if (msg_ptr)
	2405	POST_WRITE(msg_ptr, res);
	2406	if (msg_prio)
	2407	POST_WRITE(msg_prio, sizeof(*msg_prio));
2195	2408	}
2196	2409	}
2197	2410
2198	2411	PRE_SYSCALL(mq_notify)(long mqdes, const void *notification) {
2199		if (notification) PRE_READ(notification, struct_sigevent_sz);
	2412	if (notification)
	2413	PRE_READ(notification, struct_sigevent_sz);
2200	2414	}
2201	2415
2202	2416	POST_SYSCALL(mq_notify)(long res, long mqdes, const void *notification) {}
2203	2417
2204	2418	PRE_SYSCALL(mq_getsetattr)(long mqdes, const void mqstat, void omqstat) {
2205		if (mqstat) PRE_READ(mqstat, struct_mq_attr_sz);
	2419	if (mqstat)
	2420	PRE_READ(mqstat, struct_mq_attr_sz);
2206	2421	}
2207	2422
2208		POST_SYSCALL(mq_getsetattr)(long res, long mqdes, const void *mqstat,
2209		void *omqstat) {
	2423	POST_SYSCALL(mq_getsetattr)
	2424	(long res, long mqdes, const void mqstat, void omqstat) {
2210	2425	if (res >= 0) {
2211		if (omqstat) POST_WRITE(omqstat, struct_mq_attr_sz);
	2426	if (omqstat)
	2427	POST_WRITE(omqstat, struct_mq_attr_sz);
2212	2428	}
2213	2429	}
2214		#endif // SANITIZER_ANDROID
	2430	# endif // SANITIZER_ANDROID
2215	2431
2216	2432	PRE_SYSCALL(pciconfig_iobase)(long which, long bus, long devfn) {}
2217	2433
2218	2434	POST_SYSCALL(pciconfig_iobase)(long res, long which, long bus, long devfn) {}
2219	2435
2220		PRE_SYSCALL(pciconfig_read)(long bus, long dfn, long off, long len, void *buf) {
2221		}
	2436	PRE_SYSCALL(pciconfig_read)
	2437	(long bus, long dfn, long off, long len, void *buf) {}
2222	2438
2223		POST_SYSCALL(pciconfig_read)(long res, long bus, long dfn, long off, long len,
2224		void *buf) {}
	2439	POST_SYSCALL(pciconfig_read)
	2440	(long res, long bus, long dfn, long off, long len, void *buf) {}
2225	2441
2226		PRE_SYSCALL(pciconfig_write)(long bus, long dfn, long off, long len,
2227		void *buf) {}
	2442	PRE_SYSCALL(pciconfig_write)
	2443	(long bus, long dfn, long off, long len, void *buf) {}
2228	2444
2229		POST_SYSCALL(pciconfig_write)(long res, long bus, long dfn, long off, long len,
2230		void *buf) {}
	2445	POST_SYSCALL(pciconfig_write)
	2446	(long res, long bus, long dfn, long off, long len, void *buf) {}
2231	2447
2232	2448	PRE_SYSCALL(swapon)(const void *specialfile, long swap_flags) {
2233	2449	if (specialfile)
...	...	@@ -2247,8 +2463,10 @@ POST_SYSCALL(swapoff)(long res, const void *specialfile) {}
2247	2463
2248	2464	PRE_SYSCALL(sysctl)(__sanitizer___sysctl_args *args) {
2249	2465	if (args) {
2250		if (args->name) PRE_READ(args->name, args->nlen * sizeof(*args->name));
2251		if (args->newval) PRE_READ(args->name, args->newlen);
	2466	if (args->name)
	2467	PRE_READ(args->name, args->nlen * sizeof(*args->name));
	2468	if (args->newval)
	2469	PRE_READ(args->name, args->newlen);
2252	2470	}
2253	2471	}
2254	2472
...	...	@@ -2265,7 +2483,8 @@ PRE_SYSCALL(sysinfo)(void *info) {}
2265	2483
2266	2484	POST_SYSCALL(sysinfo)(long res, void *info) {
2267	2485	if (res >= 0) {
2268		if (info) POST_WRITE(info, struct_sysinfo_sz);
	2486	if (info)
	2487	POST_WRITE(info, struct_sysinfo_sz);
2269	2488	}
2270	2489	}
2271	2490
...	...	@@ -2294,10 +2513,10 @@ PRE_SYSCALL(ni_syscall)() {}
2294	2513	POST_SYSCALL(ni_syscall)(long res) {}
2295	2514
2296	2515	PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) {
2297		#if !SANITIZER_ANDROID && \
2298		(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \
2299		defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__s390__) \|\| \
2300		SANITIZER_RISCV64)
	2516	# if !SANITIZER_ANDROID && \
	2517	(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \
	2518	defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__s390__) \|\| \
	2519	defined(__loongarch__) \|\| SANITIZER_RISCV64)
2301	2520	if (data) {
2302	2521	if (request == ptrace_setregs) {
2303	2522	PRE_READ((void *)data, struct_user_regs_struct_sz);
...	...	@@ -2312,14 +2531,14 @@ PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) {
2312	2531	PRE_READ(iov->iov_base, iov->iov_len);
2313	2532	}
2314	2533	}
2315		#endif
	2534	# endif
2316	2535	}
2317	2536
2318	2537	POST_SYSCALL(ptrace)(long res, long request, long pid, long addr, long data) {
2319		#if !SANITIZER_ANDROID && \
2320		(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \
2321		defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__s390__) \|\| \
2322		SANITIZER_RISCV64)
	2538	# if !SANITIZER_ANDROID && \
	2539	(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \
	2540	defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__s390__) \|\| \
	2541	defined(__loongarch__) \|\| SANITIZER_RISCV64)
2323	2542	if (res >= 0 && data) {
2324	2543	// Note that this is different from the interceptor in
2325	2544	// sanitizer_common_interceptors.inc.
...	...	@@ -2340,11 +2559,12 @@ POST_SYSCALL(ptrace)(long res, long request, long pid, long addr, long data) {
2340	2559	POST_WRITE((void )data, sizeof(void ));
2341	2560	}
2342	2561	}
2343		#endif
	2562	# endif
2344	2563	}
2345	2564
2346		PRE_SYSCALL(add_key)(const void _type, const void _description,
2347		const void *_payload, long plen, long destringid) {
	2565	PRE_SYSCALL(add_key)
	2566	(const void _type, const void _description, const void *_payload, long plen,
	2567	long destringid) {
2348	2568	if (_type)
2349	2569	PRE_READ(_type, __sanitizer::internal_strlen((const char *)_type) + 1);
2350	2570	if (_description)
...	...	@@ -2352,11 +2572,13 @@ PRE_SYSCALL(add_key)(const void _type, const void _description,
2352	2572	__sanitizer::internal_strlen((const char *)_description) + 1);
2353	2573	}
2354	2574
2355		POST_SYSCALL(add_key)(long res, const void _type, const void _description,
2356		const void *_payload, long plen, long destringid) {}
	2575	POST_SYSCALL(add_key)
	2576	(long res, const void _type, const void _description, const void *_payload,
	2577	long plen, long destringid) {}
2357	2578
2358		PRE_SYSCALL(request_key)(const void _type, const void _description,
2359		const void *_callout_info, long destringid) {
	2579	PRE_SYSCALL(request_key)
	2580	(const void _type, const void _description, const void *_callout_info,
	2581	long destringid) {
2360	2582	if (_type)
2361	2583	PRE_READ(_type, __sanitizer::internal_strlen((const char *)_type) + 1);
2362	2584	if (_description)
...	...	@@ -2367,13 +2589,14 @@ PRE_SYSCALL(request_key)(const void _type, const void _description,
2367	2589	__sanitizer::internal_strlen((const char *)_callout_info) + 1);
2368	2590	}
2369	2591
2370		POST_SYSCALL(request_key)(long res, const void _type, const void _description,
2371		const void *_callout_info, long destringid) {}
	2592	POST_SYSCALL(request_key)
	2593	(long res, const void _type, const void _description,
	2594	const void *_callout_info, long destringid) {}
2372	2595
2373	2596	PRE_SYSCALL(keyctl)(long cmd, long arg2, long arg3, long arg4, long arg5) {}
2374	2597
2375		POST_SYSCALL(keyctl)(long res, long cmd, long arg2, long arg3, long arg4,
2376		long arg5) {}
	2598	POST_SYSCALL(keyctl)
	2599	(long res, long cmd, long arg2, long arg3, long arg4, long arg5) {}
2377	2600
2378	2601	PRE_SYSCALL(ioprio_set)(long which, long who, long ioprio) {}
2379	2602
...	...	@@ -2387,50 +2610,62 @@ PRE_SYSCALL(set_mempolicy)(long mode, void *nmask, long maxnode) {}
2387	2610
2388	2611	POST_SYSCALL(set_mempolicy)(long res, long mode, void *nmask, long maxnode) {
2389	2612	if (res >= 0) {
2390		if (nmask) POST_WRITE(nmask, sizeof(long));
	2613	if (nmask)
	2614	POST_WRITE(nmask, sizeof(long));
2391	2615	}
2392	2616	}
2393	2617
2394		PRE_SYSCALL(migrate_pages)(long pid, long maxnode, const void *from,
2395		const void *to) {
2396		if (from) PRE_READ(from, sizeof(long));
2397		if (to) PRE_READ(to, sizeof(long));
	2618	PRE_SYSCALL(migrate_pages)
	2619	(long pid, long maxnode, const void from, const void to) {
	2620	if (from)
	2621	PRE_READ(from, sizeof(long));
	2622	if (to)
	2623	PRE_READ(to, sizeof(long));
2398	2624	}
2399	2625
2400		POST_SYSCALL(migrate_pages)(long res, long pid, long maxnode, const void *from,
2401		const void *to) {}
	2626	POST_SYSCALL(migrate_pages)
	2627	(long res, long pid, long maxnode, const void from, const void to) {}
2402	2628
2403		PRE_SYSCALL(move_pages)(long pid, long nr_pages, const void **pages,
2404		const int nodes, int status, long flags) {
2405		if (pages) PRE_READ(pages, nr_pages * sizeof(*pages));
2406		if (nodes) PRE_READ(nodes, nr_pages * sizeof(*nodes));
	2629	PRE_SYSCALL(move_pages)
	2630	(long pid, long nr_pages, const void *pages, const int nodes, int *status,
	2631	long flags) {
	2632	if (pages)
	2633	PRE_READ(pages, nr_pages * sizeof(*pages));
	2634	if (nodes)
	2635	PRE_READ(nodes, nr_pages * sizeof(*nodes));
2407	2636	}
2408	2637
2409		POST_SYSCALL(move_pages)(long res, long pid, long nr_pages, const void **pages,
2410		const int nodes, int status, long flags) {
	2638	POST_SYSCALL(move_pages)
	2639	(long res, long pid, long nr_pages, const void *pages, const int nodes,
	2640	int *status, long flags) {
2411	2641	if (res >= 0) {
2412		if (status) POST_WRITE(status, nr_pages * sizeof(*status));
	2642	if (status)
	2643	POST_WRITE(status, nr_pages * sizeof(*status));
2413	2644	}
2414	2645	}
2415	2646
2416		PRE_SYSCALL(mbind)(long start, long len, long mode, void *nmask, long maxnode,
2417		long flags) {}
	2647	PRE_SYSCALL(mbind)
	2648	(long start, long len, long mode, void *nmask, long maxnode, long flags) {}
2418	2649
2419		POST_SYSCALL(mbind)(long res, long start, long len, long mode, void *nmask,
2420		long maxnode, long flags) {
	2650	POST_SYSCALL(mbind)
	2651	(long res, long start, long len, long mode, void *nmask, long maxnode,
	2652	long flags) {
2421	2653	if (res >= 0) {
2422		if (nmask) POST_WRITE(nmask, sizeof(long));
	2654	if (nmask)
	2655	POST_WRITE(nmask, sizeof(long));
2423	2656	}
2424	2657	}
2425	2658
2426		PRE_SYSCALL(get_mempolicy)(void policy, void nmask, long maxnode, long addr,
2427		long flags) {}
	2659	PRE_SYSCALL(get_mempolicy)
	2660	(void policy, void nmask, long maxnode, long addr, long flags) {}
2428	2661
2429		POST_SYSCALL(get_mempolicy)(long res, void policy, void nmask, long maxnode,
2430		long addr, long flags) {
	2662	POST_SYSCALL(get_mempolicy)
	2663	(long res, void policy, void nmask, long maxnode, long addr, long flags) {
2431	2664	if (res >= 0) {
2432		if (policy) POST_WRITE(policy, sizeof(int));
2433		if (nmask) POST_WRITE(nmask, sizeof(long));
	2665	if (policy)
	2666	POST_WRITE(policy, sizeof(int));
	2667	if (nmask)
	2668	POST_WRITE(nmask, sizeof(long));
2434	2669	}
2435	2670	}
2436	2671
...	...	@@ -2447,8 +2682,8 @@ PRE_SYSCALL(inotify_add_watch)(long fd, const void *path, long mask) {
2447	2682	PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
2448	2683	}
2449	2684
2450		POST_SYSCALL(inotify_add_watch)(long res, long fd, const void *path,
2451		long mask) {}
	2685	POST_SYSCALL(inotify_add_watch)
	2686	(long res, long fd, const void *path, long mask) {}
2452	2687
2453	2688	PRE_SYSCALL(inotify_rm_watch)(long fd, long wd) {}
2454	2689
...	...	@@ -2458,8 +2693,10 @@ PRE_SYSCALL(spu_run)(long fd, void unpc, void ustatus) {}
2458	2693
2459	2694	POST_SYSCALL(spu_run)(long res, long fd, unsigned unpc, unsigned ustatus) {
2460	2695	if (res >= 0) {
2461		if (unpc) POST_WRITE(unpc, sizeof(*unpc));
2462		if (ustatus) POST_WRITE(ustatus, sizeof(*ustatus));
	2696	if (unpc)
	2697	POST_WRITE(unpc, sizeof(*unpc));
	2698	if (ustatus)
	2699	POST_WRITE(ustatus, sizeof(*ustatus));
2463	2700	}
2464	2701	}
2465	2702
...	...	@@ -2468,8 +2705,8 @@ PRE_SYSCALL(spu_create)(const void *name, long flags, long mode, long fd) {
2468	2705	PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
2469	2706	}
2470	2707
2471		POST_SYSCALL(spu_create)(long res, const void *name, long flags, long mode,
2472		long fd) {}
	2708	POST_SYSCALL(spu_create)
	2709	(long res, const void *name, long flags, long mode, long fd) {}
2473	2710
2474	2711	PRE_SYSCALL(mknodat)(long dfd, const void *filename, long mode, long dev) {
2475	2712	if (filename)
...	...	@@ -2477,8 +2714,8 @@ PRE_SYSCALL(mknodat)(long dfd, const void *filename, long mode, long dev) {
2477	2714	__sanitizer::internal_strlen((const char *)filename) + 1);
2478	2715	}
2479	2716
2480		POST_SYSCALL(mknodat)(long res, long dfd, const void *filename, long mode,
2481		long dev) {}
	2717	POST_SYSCALL(mknodat)
	2718	(long res, long dfd, const void *filename, long mode, long dev) {}
2482	2719
2483	2720	PRE_SYSCALL(mkdirat)(long dfd, const void *pathname, long mode) {
2484	2721	if (pathname)
...	...	@@ -2503,30 +2740,33 @@ PRE_SYSCALL(symlinkat)(const void oldname, long newdfd, const void newname) {
2503	2740	PRE_READ(newname, __sanitizer::internal_strlen((const char *)newname) + 1);
2504	2741	}
2505	2742
2506		POST_SYSCALL(symlinkat)(long res, const void *oldname, long newdfd,
2507		const void *newname) {}
	2743	POST_SYSCALL(symlinkat)
	2744	(long res, const void oldname, long newdfd, const void newname) {}
2508	2745
2509		PRE_SYSCALL(linkat)(long olddfd, const void *oldname, long newdfd,
2510		const void *newname, long flags) {
	2746	PRE_SYSCALL(linkat)
	2747	(long olddfd, const void oldname, long newdfd, const void newname,
	2748	long flags) {
2511	2749	if (oldname)
2512	2750	PRE_READ(oldname, __sanitizer::internal_strlen((const char *)oldname) + 1);
2513	2751	if (newname)
2514	2752	PRE_READ(newname, __sanitizer::internal_strlen((const char *)newname) + 1);
2515	2753	}
2516	2754
2517		POST_SYSCALL(linkat)(long res, long olddfd, const void *oldname, long newdfd,
2518		const void *newname, long flags) {}
	2755	POST_SYSCALL(linkat)
	2756	(long res, long olddfd, const void oldname, long newdfd, const void newname,
	2757	long flags) {}
2519	2758
2520		PRE_SYSCALL(renameat)(long olddfd, const void *oldname, long newdfd,
2521		const void *newname) {
	2759	PRE_SYSCALL(renameat)
	2760	(long olddfd, const void oldname, long newdfd, const void newname) {
2522	2761	if (oldname)
2523	2762	PRE_READ(oldname, __sanitizer::internal_strlen((const char *)oldname) + 1);
2524	2763	if (newname)
2525	2764	PRE_READ(newname, __sanitizer::internal_strlen((const char *)newname) + 1);
2526	2765	}
2527	2766
2528		POST_SYSCALL(renameat)(long res, long olddfd, const void *oldname, long newdfd,
2529		const void *newname) {}
	2767	POST_SYSCALL(renameat)
	2768	(long res, long olddfd, const void oldname, long newdfd, const void newname) {
	2769	}
2530	2770
2531	2771	PRE_SYSCALL(futimesat)(long dfd, const void filename, void utimes) {
2532	2772	if (filename)
...	...	@@ -2534,10 +2774,11 @@ PRE_SYSCALL(futimesat)(long dfd, const void filename, void utimes) {
2534	2774	__sanitizer::internal_strlen((const char *)filename) + 1);
2535	2775	}
2536	2776
2537		POST_SYSCALL(futimesat)(long res, long dfd, const void *filename,
2538		void *utimes) {
	2777	POST_SYSCALL(futimesat)
	2778	(long res, long dfd, const void filename, void utimes) {
2539	2779	if (res >= 0) {
2540		if (utimes) POST_WRITE(utimes, timeval_sz);
	2780	if (utimes)
	2781	POST_WRITE(utimes, timeval_sz);
2541	2782	}
2542	2783	}
2543	2784
...	...	@@ -2557,15 +2798,15 @@ PRE_SYSCALL(fchmodat)(long dfd, const void *filename, long mode) {
2557	2798
2558	2799	POST_SYSCALL(fchmodat)(long res, long dfd, const void *filename, long mode) {}
2559	2800
2560		PRE_SYSCALL(fchownat)(long dfd, const void *filename, long user, long group,
2561		long flag) {
	2801	PRE_SYSCALL(fchownat)
	2802	(long dfd, const void *filename, long user, long group, long flag) {
2562	2803	if (filename)
2563	2804	PRE_READ(filename,
2564	2805	__sanitizer::internal_strlen((const char *)filename) + 1);
2565	2806	}
2566	2807
2567		POST_SYSCALL(fchownat)(long res, long dfd, const void *filename, long user,
2568		long group, long flag) {}
	2808	POST_SYSCALL(fchownat)
	2809	(long res, long dfd, const void *filename, long user, long group, long flag) {}
2569	2810
2570	2811	PRE_SYSCALL(openat)(long dfd, const void *filename, long flags, long mode) {
2571	2812	if (filename)
...	...	@@ -2573,34 +2814,36 @@ PRE_SYSCALL(openat)(long dfd, const void *filename, long flags, long mode) {
2573	2814	__sanitizer::internal_strlen((const char *)filename) + 1);
2574	2815	}
2575	2816
2576		POST_SYSCALL(openat)(long res, long dfd, const void *filename, long flags,
2577		long mode) {}
	2817	POST_SYSCALL(openat)
	2818	(long res, long dfd, const void *filename, long flags, long mode) {}
2578	2819
2579		PRE_SYSCALL(newfstatat)(long dfd, const void filename, void statbuf,
2580		long flag) {
	2820	PRE_SYSCALL(newfstatat)
	2821	(long dfd, const void filename, void statbuf, long flag) {
2581	2822	if (filename)
2582	2823	PRE_READ(filename,
2583	2824	__sanitizer::internal_strlen((const char *)filename) + 1);
2584	2825	}
2585	2826
2586		POST_SYSCALL(newfstatat)(long res, long dfd, const void *filename,
2587		void *statbuf, long flag) {
	2827	POST_SYSCALL(newfstatat)
	2828	(long res, long dfd, const void filename, void statbuf, long flag) {
2588	2829	if (res >= 0) {
2589		if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
	2830	if (statbuf)
	2831	POST_WRITE(statbuf, struct_kernel_stat_sz);
2590	2832	}
2591	2833	}
2592	2834
2593		PRE_SYSCALL(fstatat64)(long dfd, const void filename, void statbuf,
2594		long flag) {
	2835	PRE_SYSCALL(fstatat64)
	2836	(long dfd, const void filename, void statbuf, long flag) {
2595	2837	if (filename)
2596	2838	PRE_READ(filename,
2597	2839	__sanitizer::internal_strlen((const char *)filename) + 1);
2598	2840	}
2599	2841
2600		POST_SYSCALL(fstatat64)(long res, long dfd, const void filename, void statbuf,
2601		long flag) {
	2842	POST_SYSCALL(fstatat64)
	2843	(long res, long dfd, const void filename, void statbuf, long flag) {
2602	2844	if (res >= 0) {
2603		if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
	2845	if (statbuf)
	2846	POST_WRITE(statbuf, struct_kernel_stat64_sz);
2604	2847	}
2605	2848	}
2606	2849
...	...	@@ -2609,25 +2852,26 @@ PRE_SYSCALL(readlinkat)(long dfd, const void path, void buf, long bufsiz) {
2609	2852	PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
2610	2853	}
2611	2854
2612		POST_SYSCALL(readlinkat)(long res, long dfd, const void path, void buf,
2613		long bufsiz) {
	2855	POST_SYSCALL(readlinkat)
	2856	(long res, long dfd, const void path, void buf, long bufsiz) {
2614	2857	if (res >= 0) {
2615	2858	if (buf)
2616	2859	POST_WRITE(buf, __sanitizer::internal_strlen((const char *)buf) + 1);
2617	2860	}
2618	2861	}
2619	2862
2620		PRE_SYSCALL(utimensat)(long dfd, const void filename, void utimes,
2621		long flags) {
	2863	PRE_SYSCALL(utimensat)
	2864	(long dfd, const void filename, void utimes, long flags) {
2622	2865	if (filename)
2623	2866	PRE_READ(filename,
2624	2867	__sanitizer::internal_strlen((const char *)filename) + 1);
2625	2868	}
2626	2869
2627		POST_SYSCALL(utimensat)(long res, long dfd, const void filename, void utimes,
2628		long flags) {
	2870	POST_SYSCALL(utimensat)
	2871	(long res, long dfd, const void filename, void utimes, long flags) {
2629	2872	if (res >= 0) {
2630		if (utimes) POST_WRITE(utimes, struct_timespec_sz);
	2873	if (utimes)
	2874	POST_WRITE(utimes, struct_timespec_sz);
2631	2875	}
2632	2876	}
2633	2877
...	...	@@ -2635,24 +2879,28 @@ PRE_SYSCALL(unshare)(long unshare_flags) {}
2635	2879
2636	2880	POST_SYSCALL(unshare)(long res, long unshare_flags) {}
2637	2881
2638		PRE_SYSCALL(splice)(long fd_in, void off_in, long fd_out, void off_out,
2639		long len, long flags) {}
	2882	PRE_SYSCALL(splice)
	2883	(long fd_in, void off_in, long fd_out, void off_out, long len, long flags) {}
2640	2884
2641		POST_SYSCALL(splice)(long res, long fd_in, void *off_in, long fd_out,
2642		void *off_out, long len, long flags) {
	2885	POST_SYSCALL(splice)
	2886	(long res, long fd_in, void off_in, long fd_out, void off_out, long len,
	2887	long flags) {
2643	2888	if (res >= 0) {
2644		if (off_in) POST_WRITE(off_in, sizeof(long long));
2645		if (off_out) POST_WRITE(off_out, sizeof(long long));
	2889	if (off_in)
	2890	POST_WRITE(off_in, sizeof(long long));
	2891	if (off_out)
	2892	POST_WRITE(off_out, sizeof(long long));
2646	2893	}
2647	2894	}
2648	2895
2649		PRE_SYSCALL(vmsplice)(long fd, const __sanitizer_iovec *iov, long nr_segs,
2650		long flags) {}
	2896	PRE_SYSCALL(vmsplice)
	2897	(long fd, const __sanitizer_iovec *iov, long nr_segs, long flags) {}
2651	2898
2652		POST_SYSCALL(vmsplice)(long res, long fd, const __sanitizer_iovec *iov,
2653		long nr_segs, long flags) {
	2899	POST_SYSCALL(vmsplice)
	2900	(long res, long fd, const __sanitizer_iovec *iov, long nr_segs, long flags) {
2654	2901	if (res >= 0) {
2655		if (iov) kernel_read_iovec(iov, nr_segs, res);
	2902	if (iov)
	2903	kernel_read_iovec(iov, nr_segs, res);
2656	2904	}
2657	2905	}
2658	2906
...	...	@@ -2662,8 +2910,8 @@ POST_SYSCALL(tee)(long res, long fdin, long fdout, long len, long flags) {}
2662	2910
2663	2911	PRE_SYSCALL(get_robust_list)(long pid, void head_ptr, void len_ptr) {}
2664	2912
2665		POST_SYSCALL(get_robust_list)(long res, long pid, void *head_ptr,
2666		void *len_ptr) {}
	2913	POST_SYSCALL(get_robust_list)
	2914	(long res, long pid, void head_ptr, void len_ptr) {}
2667	2915
2668	2916	PRE_SYSCALL(set_robust_list)(void *head, long len) {}
2669	2917
...	...	@@ -2673,27 +2921,31 @@ PRE_SYSCALL(getcpu)(void cpu, void node, void *cache) {}
2673	2921
2674	2922	POST_SYSCALL(getcpu)(long res, void cpu, void node, void *cache) {
2675	2923	if (res >= 0) {
2676		if (cpu) POST_WRITE(cpu, sizeof(unsigned));
2677		if (node) POST_WRITE(node, sizeof(unsigned));
	2924	if (cpu)
	2925	POST_WRITE(cpu, sizeof(unsigned));
	2926	if (node)
	2927	POST_WRITE(node, sizeof(unsigned));
2678	2928	// The third argument to this system call is nowadays unused.
2679	2929	}
2680	2930	}
2681	2931
2682	2932	PRE_SYSCALL(signalfd)(long ufd, void *user_mask, long sizemask) {}
2683	2933
2684		POST_SYSCALL(signalfd)(long res, long ufd, kernel_sigset_t *user_mask,
2685		long sizemask) {
	2934	POST_SYSCALL(signalfd)
	2935	(long res, long ufd, kernel_sigset_t *user_mask, long sizemask) {
2686	2936	if (res >= 0) {
2687		if (user_mask) POST_WRITE(user_mask, sizemask);
	2937	if (user_mask)
	2938	POST_WRITE(user_mask, sizemask);
2688	2939	}
2689	2940	}
2690	2941
2691	2942	PRE_SYSCALL(signalfd4)(long ufd, void *user_mask, long sizemask, long flags) {}
2692	2943
2693		POST_SYSCALL(signalfd4)(long res, long ufd, kernel_sigset_t *user_mask,
2694		long sizemask, long flags) {
	2944	POST_SYSCALL(signalfd4)
	2945	(long res, long ufd, kernel_sigset_t *user_mask, long sizemask, long flags) {
2695	2946	if (res >= 0) {
2696		if (user_mask) POST_WRITE(user_mask, sizemask);
	2947	if (user_mask)
	2948	POST_WRITE(user_mask, sizemask);
2697	2949	}
2698	2950	}
2699	2951
...	...	@@ -2701,15 +2953,17 @@ PRE_SYSCALL(timerfd_create)(long clockid, long flags) {}
2701	2953
2702	2954	POST_SYSCALL(timerfd_create)(long res, long clockid, long flags) {}
2703	2955
2704		PRE_SYSCALL(timerfd_settime)(long ufd, long flags, const void *utmr,
2705		void *otmr) {
2706		if (utmr) PRE_READ(utmr, struct_itimerspec_sz);
	2956	PRE_SYSCALL(timerfd_settime)
	2957	(long ufd, long flags, const void utmr, void otmr) {
	2958	if (utmr)
	2959	PRE_READ(utmr, struct_itimerspec_sz);
2707	2960	}
2708	2961
2709		POST_SYSCALL(timerfd_settime)(long res, long ufd, long flags, const void *utmr,
2710		void *otmr) {
	2962	POST_SYSCALL(timerfd_settime)
	2963	(long res, long ufd, long flags, const void utmr, void otmr) {
2711	2964	if (res >= 0) {
2712		if (otmr) POST_WRITE(otmr, struct_itimerspec_sz);
	2965	if (otmr)
	2966	POST_WRITE(otmr, struct_itimerspec_sz);
2713	2967	}
2714	2968	}
2715	2969
...	...	@@ -2717,7 +2971,8 @@ PRE_SYSCALL(timerfd_gettime)(long ufd, void *otmr) {}
2717	2971
2718	2972	POST_SYSCALL(timerfd_gettime)(long res, long ufd, void *otmr) {
2719	2973	if (res >= 0) {
2720		if (otmr) POST_WRITE(otmr, struct_itimerspec_sz);
	2974	if (otmr)
	2975	POST_WRITE(otmr, struct_itimerspec_sz);
2721	2976	}
2722	2977	}
2723	2978
...	...	@@ -2735,33 +2990,42 @@ POST_SYSCALL(old_readdir)(long res, long arg0, void *arg1, long arg2) {
2735	2990	// Missing definition of 'struct old_linux_dirent'.
2736	2991	}
2737	2992
2738		PRE_SYSCALL(pselect6)(long arg0, __sanitizer___kernel_fd_set *arg1,
2739		__sanitizer___kernel_fd_set *arg2,
2740		__sanitizer___kernel_fd_set arg3, void arg4,
2741		void *arg5) {}
	2993	PRE_SYSCALL(pselect6)
	2994	(long arg0, __sanitizer___kernel_fd_set *arg1,
	2995	__sanitizer___kernel_fd_set arg2, __sanitizer___kernel_fd_set arg3,
	2996	void arg4, void arg5) {}
2742	2997
2743		POST_SYSCALL(pselect6)(long res, long arg0, __sanitizer___kernel_fd_set *arg1,
2744		__sanitizer___kernel_fd_set *arg2,
2745		__sanitizer___kernel_fd_set arg3, void arg4,
2746		void *arg5) {
	2998	POST_SYSCALL(pselect6)
	2999	(long res, long arg0, __sanitizer___kernel_fd_set *arg1,
	3000	__sanitizer___kernel_fd_set arg2, __sanitizer___kernel_fd_set arg3,
	3001	void arg4, void arg5) {
2747	3002	if (res >= 0) {
2748		if (arg1) POST_WRITE(arg1, sizeof(*arg1));
2749		if (arg2) POST_WRITE(arg2, sizeof(*arg2));
2750		if (arg3) POST_WRITE(arg3, sizeof(*arg3));
2751		if (arg4) POST_WRITE(arg4, struct_timespec_sz);
	3003	if (arg1)
	3004	POST_WRITE(arg1, sizeof(*arg1));
	3005	if (arg2)
	3006	POST_WRITE(arg2, sizeof(*arg2));
	3007	if (arg3)
	3008	POST_WRITE(arg3, sizeof(*arg3));
	3009	if (arg4)
	3010	POST_WRITE(arg4, struct_timespec_sz);
2752	3011	}
2753	3012	}
2754	3013
2755		PRE_SYSCALL(ppoll)(__sanitizer_pollfd arg0, long arg1, void arg2,
2756		const kernel_sigset_t *arg3, long arg4) {
2757		if (arg3) PRE_READ(arg3, arg4);
	3014	PRE_SYSCALL(ppoll)
	3015	(__sanitizer_pollfd arg0, long arg1, void arg2, const kernel_sigset_t *arg3,
	3016	long arg4) {
	3017	if (arg3)
	3018	PRE_READ(arg3, arg4);
2758	3019	}
2759	3020
2760		POST_SYSCALL(ppoll)(long res, __sanitizer_pollfd arg0, long arg1, void arg2,
2761		const void *arg3, long arg4) {
	3021	POST_SYSCALL(ppoll)
	3022	(long res, __sanitizer_pollfd arg0, long arg1, void arg2, const void *arg3,
	3023	long arg4) {
2762	3024	if (res >= 0) {
2763		if (arg0) POST_WRITE(arg0, sizeof(*arg0));
2764		if (arg2) POST_WRITE(arg2, struct_timespec_sz);
	3025	if (arg0)
	3026	POST_WRITE(arg0, sizeof(*arg0));
	3027	if (arg2)
	3028	POST_WRITE(arg2, struct_timespec_sz);
2765	3029	}
2766	3030	}
2767	3031
...	...	@@ -2769,81 +3033,79 @@ PRE_SYSCALL(syncfs)(long fd) {}
2769	3033
2770	3034	POST_SYSCALL(syncfs)(long res, long fd) {}
2771	3035
2772		PRE_SYSCALL(perf_event_open)(__sanitizer_perf_event_attr *attr_uptr, long pid,
2773		long cpu, long group_fd, long flags) {
2774		if (attr_uptr) PRE_READ(attr_uptr, attr_uptr->size);
	3036	PRE_SYSCALL(perf_event_open)
	3037	(__sanitizer_perf_event_attr *attr_uptr, long pid, long cpu, long group_fd,
	3038	long flags) {
	3039	if (attr_uptr)
	3040	PRE_READ(attr_uptr, attr_uptr->size);
2775	3041	}
2776	3042
2777		POST_SYSCALL(perf_event_open)(long res, __sanitizer_perf_event_attr *attr_uptr,
2778		long pid, long cpu, long group_fd, long flags) {}
	3043	POST_SYSCALL(perf_event_open)
	3044	(long res, __sanitizer_perf_event_attr *attr_uptr, long pid, long cpu,
	3045	long group_fd, long flags) {}
2779	3046
2780		PRE_SYSCALL(mmap_pgoff)(long addr, long len, long prot, long flags, long fd,
2781		long pgoff) {}
	3047	PRE_SYSCALL(mmap_pgoff)
	3048	(long addr, long len, long prot, long flags, long fd, long pgoff) {}
2782	3049
2783		POST_SYSCALL(mmap_pgoff)(long res, long addr, long len, long prot, long flags,
2784		long fd, long pgoff) {}
	3050	POST_SYSCALL(mmap_pgoff)
	3051	(long res, long addr, long len, long prot, long flags, long fd, long pgoff) {}
2785	3052
2786	3053	PRE_SYSCALL(old_mmap)(void *arg) {}
2787	3054
2788	3055	POST_SYSCALL(old_mmap)(long res, void *arg) {}
2789	3056
2790		PRE_SYSCALL(name_to_handle_at)(long dfd, const void name, void handle,
2791		void *mnt_id, long flag) {}
	3057	PRE_SYSCALL(name_to_handle_at)
	3058	(long dfd, const void name, void handle, void *mnt_id, long flag) {}
2792	3059
2793		POST_SYSCALL(name_to_handle_at)(long res, long dfd, const void *name,
2794		void handle, void mnt_id, long flag) {}
	3060	POST_SYSCALL(name_to_handle_at)
	3061	(long res, long dfd, const void name, void handle, void *mnt_id, long flag) {}
2795	3062
2796	3063	PRE_SYSCALL(open_by_handle_at)(long mountdirfd, void *handle, long flags) {}
2797	3064
2798		POST_SYSCALL(open_by_handle_at)(long res, long mountdirfd, void *handle,
2799		long flags) {}
	3065	POST_SYSCALL(open_by_handle_at)
	3066	(long res, long mountdirfd, void *handle, long flags) {}
2800	3067
2801	3068	PRE_SYSCALL(setns)(long fd, long nstype) {}
2802	3069
2803	3070	POST_SYSCALL(setns)(long res, long fd, long nstype) {}
2804	3071
2805		PRE_SYSCALL(process_vm_readv)(long pid, const __sanitizer_iovec *lvec,
2806		long liovcnt, const void *rvec, long riovcnt,
2807		long flags) {}
	3072	PRE_SYSCALL(process_vm_readv)
	3073	(long pid, const __sanitizer_iovec lvec, long liovcnt, const void rvec,
	3074	long riovcnt, long flags) {}
2808	3075
2809		POST_SYSCALL(process_vm_readv)(long res, long pid,
2810		const __sanitizer_iovec *lvec, long liovcnt,
2811		const void *rvec, long riovcnt, long flags) {
	3076	POST_SYSCALL(process_vm_readv)
	3077	(long res, long pid, const __sanitizer_iovec *lvec, long liovcnt,
	3078	const void *rvec, long riovcnt, long flags) {
2812	3079	if (res >= 0) {
2813		if (lvec) kernel_write_iovec(lvec, liovcnt, res);
	3080	if (lvec)
	3081	kernel_write_iovec(lvec, liovcnt, res);
2814	3082	}
2815	3083	}
2816	3084
2817		PRE_SYSCALL(process_vm_writev)(long pid, const __sanitizer_iovec *lvec,
2818		long liovcnt, const void *rvec, long riovcnt,
2819		long flags) {}
	3085	PRE_SYSCALL(process_vm_writev)
	3086	(long pid, const __sanitizer_iovec lvec, long liovcnt, const void rvec,
	3087	long riovcnt, long flags) {}
2820	3088
2821		POST_SYSCALL(process_vm_writev)(long res, long pid,
2822		const __sanitizer_iovec *lvec, long liovcnt,
2823		const void *rvec, long riovcnt, long flags) {
	3089	POST_SYSCALL(process_vm_writev)
	3090	(long res, long pid, const __sanitizer_iovec *lvec, long liovcnt,
	3091	const void *rvec, long riovcnt, long flags) {
2824	3092	if (res >= 0) {
2825		if (lvec) kernel_read_iovec(lvec, liovcnt, res);
	3093	if (lvec)
	3094	kernel_read_iovec(lvec, liovcnt, res);
2826	3095	}
2827	3096	}
2828	3097
2829		PRE_SYSCALL(fork)() {
2830		COMMON_SYSCALL_PRE_FORK();
2831		}
	3098	PRE_SYSCALL(fork)() { COMMON_SYSCALL_PRE_FORK(); }
2832	3099
2833		POST_SYSCALL(fork)(long res) {
2834		COMMON_SYSCALL_POST_FORK(res);
2835		}
	3100	POST_SYSCALL(fork)(long res) { COMMON_SYSCALL_POST_FORK(res); }
2836	3101
2837		PRE_SYSCALL(vfork)() {
2838		COMMON_SYSCALL_PRE_FORK();
2839		}
	3102	PRE_SYSCALL(vfork)() { COMMON_SYSCALL_PRE_FORK(); }
2840	3103
2841		POST_SYSCALL(vfork)(long res) {
2842		COMMON_SYSCALL_POST_FORK(res);
2843		}
	3104	POST_SYSCALL(vfork)(long res) { COMMON_SYSCALL_POST_FORK(res); }
2844	3105
2845		PRE_SYSCALL(sigaction)(long signum, const __sanitizer_kernel_sigaction_t *act,
2846		__sanitizer_kernel_sigaction_t *oldact) {
	3106	PRE_SYSCALL(sigaction)
	3107	(long signum, const __sanitizer_kernel_sigaction_t *act,
	3108	__sanitizer_kernel_sigaction_t *oldact) {
2847	3109	if (act) {
2848	3110	PRE_READ(&act->sigaction, sizeof(act->sigaction));
2849	3111	PRE_READ(&act->sa_flags, sizeof(act->sa_flags));
...	...	@@ -2851,15 +3113,16 @@ PRE_SYSCALL(sigaction)(long signum, const __sanitizer_kernel_sigaction_t *act,
2851	3113	}
2852	3114	}
2853	3115
2854		POST_SYSCALL(sigaction)(long res, long signum,
2855		const __sanitizer_kernel_sigaction_t *act,
2856		__sanitizer_kernel_sigaction_t *oldact) {
2857		if (res >= 0 && oldact) POST_WRITE(oldact, sizeof(*oldact));
	3116	POST_SYSCALL(sigaction)
	3117	(long res, long signum, const __sanitizer_kernel_sigaction_t *act,
	3118	__sanitizer_kernel_sigaction_t *oldact) {
	3119	if (res >= 0 && oldact)
	3120	POST_WRITE(oldact, sizeof(*oldact));
2858	3121	}
2859	3122
2860		PRE_SYSCALL(rt_sigaction)(long signum,
2861		const __sanitizer_kernel_sigaction_t *act,
2862		__sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) {
	3123	PRE_SYSCALL(rt_sigaction)
	3124	(long signum, const __sanitizer_kernel_sigaction_t *act,
	3125	__sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) {
2863	3126	if (act) {
2864	3127	PRE_READ(&act->sigaction, sizeof(act->sigaction));
2865	3128	PRE_READ(&act->sa_flags, sizeof(act->sa_flags));
...	...	@@ -2867,9 +3130,9 @@ PRE_SYSCALL(rt_sigaction)(long signum,
2867	3130	}
2868	3131	}
2869	3132
2870		POST_SYSCALL(rt_sigaction)(long res, long signum,
2871		const __sanitizer_kernel_sigaction_t *act,
2872		__sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) {
	3133	POST_SYSCALL(rt_sigaction)
	3134	(long res, long signum, const __sanitizer_kernel_sigaction_t *act,
	3135	__sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) {
2873	3136	if (res >= 0 && oldact) {
2874	3137	SIZE_T oldact_sz = ((char )&oldact->sa_mask) - ((char )oldact) + sz;
2875	3138	POST_WRITE(oldact, oldact_sz);
...	...	@@ -2906,11 +3169,11 @@ POST_SYSCALL(sigaltstack)(long res, void ss, void oss) {
2906	3169	}
2907	3170	} // extern "C"
2908	3171
2909		#undef PRE_SYSCALL
2910		#undef PRE_READ
2911		#undef PRE_WRITE
2912		#undef POST_SYSCALL
2913		#undef POST_READ
2914		#undef POST_WRITE
	3172	# undef PRE_SYSCALL
	3173	# undef PRE_READ
	3174	# undef PRE_WRITE
	3175	# undef POST_SYSCALL
	3176	# undef POST_READ
	3177	# undef POST_WRITE
2915	3178
2916	3179	#endif // SANITIZER_LINUX

lib/tsan/sanitizer_common/sanitizer_coverage_interface.inc+10

...	...	@@ -27,6 +27,16 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_gep)
27	27	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard)
28	28	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard_init)
29	29	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_indir)
	30	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load1)
	31	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load2)
	32	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load4)
	33	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load8)
	34	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load16)
	35	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store1)
	36	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store2)
	37	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store4)
	38	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store8)
	39	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store16)
30	40	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_switch)
31	41	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_8bit_counters_init)
32	42	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_bool_flag_init)

lib/tsan/sanitizer_common/sanitizer_coverage_win_dll_thunk.cpp created+20

...	...	@@ -0,0 +1,20 @@
	1	//===-- sanitizer_coverage_win_dll_thunk.cpp ------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file defines a family of thunks that should be statically linked into
	10	// the DLLs that have instrumentation in order to delegate the calls to the
	11	// shared runtime that lives in the main binary.
	12	// See https://github.com/google/sanitizers/issues/209 for the details.
	13	//===----------------------------------------------------------------------===//
	14	#ifdef SANITIZER_DLL_THUNK
	15	#include "sanitizer_win_dll_thunk.h"
	16	// Sanitizer Coverage interface functions.
	17	#define INTERFACE_FUNCTION(Name) INTERCEPT_SANITIZER_FUNCTION(Name)
	18	#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
	19	#include "sanitizer_coverage_interface.inc"
	20	#endif // SANITIZER_DLL_THUNK

lib/tsan/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cpp created+26

...	...	@@ -0,0 +1,26 @@
	1	//===-- sanitizer_coverage_win_dynamic_runtime_thunk.cpp ------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file defines things that need to be present in the application modules
	10	// to interact with Sanitizer Coverage, when it is included in a dll.
	11	//
	12	//===----------------------------------------------------------------------===//
	13	#ifdef SANITIZER_DYNAMIC_RUNTIME_THUNK
	14	#define SANITIZER_IMPORT_INTERFACE 1
	15	#include "sanitizer_win_defs.h"
	16	// Define weak alias for all weak functions imported from sanitizer coverage.
	17	#define INTERFACE_FUNCTION(Name)
	18	#define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name)
	19	#include "sanitizer_coverage_interface.inc"
	20	#endif // SANITIZER_DYNAMIC_RUNTIME_THUNK
	21
	22	namespace __sanitizer {
	23	// Add one, otherwise unused, external symbol to this object file so that the
	24	// Visual C++ linker includes it and reads the .drective section.
	25	void ForceWholeArchiveIncludeForSanCov() {}
	26	}

lib/tsan/sanitizer_common/sanitizer_coverage_win_weak_interception.cpp created+23

...	...	@@ -0,0 +1,23 @@
	1	//===-- sanitizer_coverage_win_weak_interception.cpp ----------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	// This module should be included in Sanitizer Coverage when it implemented as a
	9	// shared library on Windows (dll), in order to delegate the calls of weak
	10	// functions to the implementation in the main executable when a strong
	11	// definition is provided.
	12	//===----------------------------------------------------------------------===//
	13	#ifdef SANITIZER_DYNAMIC
	14	#include "sanitizer_win_weak_interception.h"
	15	#include "sanitizer_interface_internal.h"
	16	#include "sancov_flags.h"
	17	// Check if strong definitions for weak functions are present in the main
	18	// executable. If that is the case, override dll functions to point to strong
	19	// implementations.
	20	#define INTERFACE_FUNCTION(Name)
	21	#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
	22	#include "sanitizer_coverage_interface.inc"
	23	#endif // SANITIZER_DYNAMIC

lib/tsan/sanitizer_common/sanitizer_deadlock_detector.h+1-1

...	...	@@ -293,7 +293,7 @@ class DeadlockDetector {
293	293	}
294	294
295	295	// Returns true iff dtls is empty (no locks are currently held) and we can
296		// add the node to the currently held locks w/o chanding the global state.
	296	// add the node to the currently held locks w/o changing the global state.
297	297	// This operation is thread-safe as it only touches the dtls.
298	298	bool onFirstLock(DeadlockDetectorTLS<BV> *dtls, uptr node, u32 stk = 0) {
299	299	if (!dtls->empty()) return false;

lib/tsan/sanitizer_common/sanitizer_dense_map.h created+705

...	...	@@ -0,0 +1,705 @@
	1	//===- sanitizer_dense_map.h - Dense probed hash table ----------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This is fork of llvm/ADT/DenseMap.h class with the following changes:
	10	// * Use mmap to allocate.
	11	// * No iterators.
	12	// * Does not shrink.
	13	//
	14	//===----------------------------------------------------------------------===//
	15
	16	#ifndef SANITIZER_DENSE_MAP_H
	17	#define SANITIZER_DENSE_MAP_H
	18
	19	#include "sanitizer_common.h"
	20	#include "sanitizer_dense_map_info.h"
	21	#include "sanitizer_internal_defs.h"
	22	#include "sanitizer_type_traits.h"
	23
	24	namespace __sanitizer {
	25
	26	template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT,
	27	typename BucketT>
	28	class DenseMapBase {
	29	public:
	30	using size_type = unsigned;
	31	using key_type = KeyT;
	32	using mapped_type = ValueT;
	33	using value_type = BucketT;
	34
	35	WARN_UNUSED_RESULT bool empty() const { return getNumEntries() == 0; }
	36	unsigned size() const { return getNumEntries(); }
	37
	38	/// Grow the densemap so that it can contain at least \p NumEntries items
	39	/// before resizing again.
	40	void reserve(size_type NumEntries) {
	41	auto NumBuckets = getMinBucketToReserveForEntries(NumEntries);
	42	if (NumBuckets > getNumBuckets())
	43	grow(NumBuckets);
	44	}
	45
	46	void clear() {
	47	if (getNumEntries() == 0 && getNumTombstones() == 0)
	48	return;
	49
	50	const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey();
	51	if (__sanitizer::is_trivially_destructible<ValueT>::value) {
	52	// Use a simpler loop when values don't need destruction.
	53	for (BucketT P = getBuckets(), E = getBucketsEnd(); P != E; ++P)
	54	P->getFirst() = EmptyKey;
	55	} else {
	56	unsigned NumEntries = getNumEntries();
	57	for (BucketT P = getBuckets(), E = getBucketsEnd(); P != E; ++P) {
	58	if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey)) {
	59	if (!KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) {
	60	P->getSecond().~ValueT();
	61	--NumEntries;
	62	}
	63	P->getFirst() = EmptyKey;
	64	}
	65	}
	66	CHECK_EQ(NumEntries, 0);
	67	}
	68	setNumEntries(0);
	69	setNumTombstones(0);
	70	}
	71
	72	/// Return 1 if the specified key is in the map, 0 otherwise.
	73	size_type count(const KeyT &Key) const {
	74	const BucketT *TheBucket;
	75	return LookupBucketFor(Key, TheBucket) ? 1 : 0;
	76	}
	77
	78	value_type *find(const KeyT &Key) {
	79	BucketT *TheBucket;
	80	if (LookupBucketFor(Key, TheBucket))
	81	return TheBucket;
	82	return nullptr;
	83	}
	84	const value_type *find(const KeyT &Key) const {
	85	const BucketT *TheBucket;
	86	if (LookupBucketFor(Key, TheBucket))
	87	return TheBucket;
	88	return nullptr;
	89	}
	90
	91	/// Alternate version of find() which allows a different, and possibly
	92	/// less expensive, key type.
	93	/// The DenseMapInfo is responsible for supplying methods
	94	/// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key
	95	/// type used.
	96	template <class LookupKeyT>
	97	value_type *find_as(const LookupKeyT &Key) {
	98	BucketT *TheBucket;
	99	if (LookupBucketFor(Key, TheBucket))
	100	return TheBucket;
	101	return nullptr;
	102	}
	103	template <class LookupKeyT>
	104	const value_type *find_as(const LookupKeyT &Key) const {
	105	const BucketT *TheBucket;
	106	if (LookupBucketFor(Key, TheBucket))
	107	return TheBucket;
	108	return nullptr;
	109	}
	110
	111	/// lookup - Return the entry for the specified key, or a default
	112	/// constructed value if no such entry exists.
	113	ValueT lookup(const KeyT &Key) const {
	114	const BucketT *TheBucket;
	115	if (LookupBucketFor(Key, TheBucket))
	116	return TheBucket->getSecond();
	117	return ValueT();
	118	}
	119
	120	// Inserts key,value pair into the map if the key isn't already in the map.
	121	// If the key is already in the map, it returns false and doesn't update the
	122	// value.
	123	detail::DenseMapPair<value_type *, bool> insert(const value_type &KV) {
	124	return try_emplace(KV.first, KV.second);
	125	}
	126
	127	// Inserts key,value pair into the map if the key isn't already in the map.
	128	// If the key is already in the map, it returns false and doesn't update the
	129	// value.
	130	detail::DenseMapPair<value_type *, bool> insert(value_type &&KV) {
	131	return try_emplace(__sanitizer::move(KV.first),
	132	__sanitizer::move(KV.second));
	133	}
	134
	135	// Inserts key,value pair into the map if the key isn't already in the map.
	136	// The value is constructed in-place if the key is not in the map, otherwise
	137	// it is not moved.
	138	template <typename... Ts>
	139	detail::DenseMapPair<value_type *, bool> try_emplace(KeyT &&Key,
	140	Ts &&...Args) {
	141	BucketT *TheBucket;
	142	if (LookupBucketFor(Key, TheBucket))
	143	return {TheBucket, false}; // Already in map.
	144
	145	// Otherwise, insert the new element.
	146	TheBucket = InsertIntoBucket(TheBucket, __sanitizer::move(Key),
	147	__sanitizer::forward<Ts>(Args)...);
	148	return {TheBucket, true};
	149	}
	150
	151	// Inserts key,value pair into the map if the key isn't already in the map.
	152	// The value is constructed in-place if the key is not in the map, otherwise
	153	// it is not moved.
	154	template <typename... Ts>
	155	detail::DenseMapPair<value_type *, bool> try_emplace(const KeyT &Key,
	156	Ts &&...Args) {
	157	BucketT *TheBucket;
	158	if (LookupBucketFor(Key, TheBucket))
	159	return {TheBucket, false}; // Already in map.
	160
	161	// Otherwise, insert the new element.
	162	TheBucket =
	163	InsertIntoBucket(TheBucket, Key, __sanitizer::forward<Ts>(Args)...);
	164	return {TheBucket, true};
	165	}
	166
	167	/// Alternate version of insert() which allows a different, and possibly
	168	/// less expensive, key type.
	169	/// The DenseMapInfo is responsible for supplying methods
	170	/// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key
	171	/// type used.
	172	template <typename LookupKeyT>
	173	detail::DenseMapPair<value_type *, bool> insert_as(value_type &&KV,
	174	const LookupKeyT &Val) {
	175	BucketT *TheBucket;
	176	if (LookupBucketFor(Val, TheBucket))
	177	return {TheBucket, false}; // Already in map.
	178
	179	// Otherwise, insert the new element.
	180	TheBucket =
	181	InsertIntoBucketWithLookup(TheBucket, __sanitizer::move(KV.first),
	182	__sanitizer::move(KV.second), Val);
	183	return {TheBucket, true};
	184	}
	185
	186	bool erase(const KeyT &Val) {
	187	BucketT *TheBucket;
	188	if (!LookupBucketFor(Val, TheBucket))
	189	return false; // not in map.
	190
	191	TheBucket->getSecond().~ValueT();
	192	TheBucket->getFirst() = getTombstoneKey();
	193	decrementNumEntries();
	194	incrementNumTombstones();
	195	return true;
	196	}
	197
	198	void erase(value_type *I) {
	199	CHECK_NE(I, nullptr);
	200	BucketT TheBucket = &I;
	201	TheBucket->getSecond().~ValueT();
	202	TheBucket->getFirst() = getTombstoneKey();
	203	decrementNumEntries();
	204	incrementNumTombstones();
	205	}
	206
	207	value_type &FindAndConstruct(const KeyT &Key) {
	208	BucketT *TheBucket;
	209	if (LookupBucketFor(Key, TheBucket))
	210	return *TheBucket;
	211
	212	return *InsertIntoBucket(TheBucket, Key);
	213	}
	214
	215	ValueT &operator[](const KeyT &Key) { return FindAndConstruct(Key).second; }
	216
	217	value_type &FindAndConstruct(KeyT &&Key) {
	218	BucketT *TheBucket;
	219	if (LookupBucketFor(Key, TheBucket))
	220	return *TheBucket;
	221
	222	return *InsertIntoBucket(TheBucket, __sanitizer::move(Key));
	223	}
	224
	225	ValueT &operator[](KeyT &&Key) {
	226	return FindAndConstruct(__sanitizer::move(Key)).second;
	227	}
	228
	229	/// Iterate over active entries of the container.
	230	///
	231	/// Function can return fast to stop the process.
	232	template <class Fn>
	233	void forEach(Fn fn) {
	234	const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey();
	235	for (auto P = getBuckets(), E = getBucketsEnd(); P != E; ++P) {
	236	const KeyT K = P->getFirst();
	237	if (!KeyInfoT::isEqual(K, EmptyKey) &&
	238	!KeyInfoT::isEqual(K, TombstoneKey)) {
	239	if (!fn(*P))
	240	return;
	241	}
	242	}
	243	}
	244
	245	template <class Fn>
	246	void forEach(Fn fn) const {
	247	const_cast<DenseMapBase *>(this)->forEach(
	248	[&](const value_type &KV) { return fn(KV); });
	249	}
	250
	251	protected:
	252	DenseMapBase() = default;
	253
	254	void destroyAll() {
	255	if (getNumBuckets() == 0) // Nothing to do.
	256	return;
	257
	258	const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey();
	259	for (BucketT P = getBuckets(), E = getBucketsEnd(); P != E; ++P) {
	260	if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey) &&
	261	!KeyInfoT::isEqual(P->getFirst(), TombstoneKey))
	262	P->getSecond().~ValueT();
	263	P->getFirst().~KeyT();
	264	}
	265	}
	266
	267	void initEmpty() {
	268	setNumEntries(0);
	269	setNumTombstones(0);
	270
	271	CHECK_EQ((getNumBuckets() & (getNumBuckets() - 1)), 0);
	272	const KeyT EmptyKey = getEmptyKey();
	273	for (BucketT B = getBuckets(), E = getBucketsEnd(); B != E; ++B)
	274	::new (&B->getFirst()) KeyT(EmptyKey);
	275	}
	276
	277	/// Returns the number of buckets to allocate to ensure that the DenseMap can
	278	/// accommodate \p NumEntries without need to grow().
	279	unsigned getMinBucketToReserveForEntries(unsigned NumEntries) {
	280	// Ensure that "NumEntries * 4 < NumBuckets * 3"
	281	if (NumEntries == 0)
	282	return 0;
	283	// +1 is required because of the strict equality.
	284	// For example if NumEntries is 48, we need to return 401.
	285	return RoundUpToPowerOfTwo((NumEntries * 4 / 3 + 1) + /* NextPowerOf2 */ 1);
	286	}
	287
	288	void moveFromOldBuckets(BucketT OldBucketsBegin, BucketT OldBucketsEnd) {
	289	initEmpty();
	290
	291	// Insert all the old elements.
	292	const KeyT EmptyKey = getEmptyKey();
	293	const KeyT TombstoneKey = getTombstoneKey();
	294	for (BucketT B = OldBucketsBegin, E = OldBucketsEnd; B != E; ++B) {
	295	if (!KeyInfoT::isEqual(B->getFirst(), EmptyKey) &&
	296	!KeyInfoT::isEqual(B->getFirst(), TombstoneKey)) {
	297	// Insert the key/value into the new table.
	298	BucketT *DestBucket;
	299	bool FoundVal = LookupBucketFor(B->getFirst(), DestBucket);
	300	(void)FoundVal; // silence warning.
	301	CHECK(!FoundVal);
	302	DestBucket->getFirst() = __sanitizer::move(B->getFirst());
	303	::new (&DestBucket->getSecond())
	304	ValueT(__sanitizer::move(B->getSecond()));
	305	incrementNumEntries();
	306
	307	// Free the value.
	308	B->getSecond().~ValueT();
	309	}
	310	B->getFirst().~KeyT();
	311	}
	312	}
	313
	314	template <typename OtherBaseT>
	315	void copyFrom(
	316	const DenseMapBase<OtherBaseT, KeyT, ValueT, KeyInfoT, BucketT> &other) {
	317	CHECK_NE(&other, this);
	318	CHECK_EQ(getNumBuckets(), other.getNumBuckets());
	319
	320	setNumEntries(other.getNumEntries());
	321	setNumTombstones(other.getNumTombstones());
	322
	323	if (__sanitizer::is_trivially_copyable<KeyT>::value &&
	324	__sanitizer::is_trivially_copyable<ValueT>::value)
	325	internal_memcpy(reinterpret_cast<void *>(getBuckets()),
	326	other.getBuckets(), getNumBuckets() * sizeof(BucketT));
	327	else
	328	for (uptr i = 0; i < getNumBuckets(); ++i) {
	329	::new (&getBuckets()[i].getFirst())
	330	KeyT(other.getBuckets()[i].getFirst());
	331	if (!KeyInfoT::isEqual(getBuckets()[i].getFirst(), getEmptyKey()) &&
	332	!KeyInfoT::isEqual(getBuckets()[i].getFirst(), getTombstoneKey()))
	333	::new (&getBuckets()[i].getSecond())
	334	ValueT(other.getBuckets()[i].getSecond());
	335	}
	336	}
	337
	338	static unsigned getHashValue(const KeyT &Val) {
	339	return KeyInfoT::getHashValue(Val);
	340	}
	341
	342	template <typename LookupKeyT>
	343	static unsigned getHashValue(const LookupKeyT &Val) {
	344	return KeyInfoT::getHashValue(Val);
	345	}
	346
	347	static const KeyT getEmptyKey() { return KeyInfoT::getEmptyKey(); }
	348
	349	static const KeyT getTombstoneKey() { return KeyInfoT::getTombstoneKey(); }
	350
	351	private:
	352	unsigned getNumEntries() const {
	353	return static_cast<const DerivedT *>(this)->getNumEntries();
	354	}
	355
	356	void setNumEntries(unsigned Num) {
	357	static_cast<DerivedT *>(this)->setNumEntries(Num);
	358	}
	359
	360	void incrementNumEntries() { setNumEntries(getNumEntries() + 1); }
	361
	362	void decrementNumEntries() { setNumEntries(getNumEntries() - 1); }
	363
	364	unsigned getNumTombstones() const {
	365	return static_cast<const DerivedT *>(this)->getNumTombstones();
	366	}
	367
	368	void setNumTombstones(unsigned Num) {
	369	static_cast<DerivedT *>(this)->setNumTombstones(Num);
	370	}
	371
	372	void incrementNumTombstones() { setNumTombstones(getNumTombstones() + 1); }
	373
	374	void decrementNumTombstones() { setNumTombstones(getNumTombstones() - 1); }
	375
	376	const BucketT *getBuckets() const {
	377	return static_cast<const DerivedT *>(this)->getBuckets();
	378	}
	379
	380	BucketT getBuckets() { return static_cast<DerivedT >(this)->getBuckets(); }
	381
	382	unsigned getNumBuckets() const {
	383	return static_cast<const DerivedT *>(this)->getNumBuckets();
	384	}
	385
	386	BucketT *getBucketsEnd() { return getBuckets() + getNumBuckets(); }
	387
	388	const BucketT *getBucketsEnd() const {
	389	return getBuckets() + getNumBuckets();
	390	}
	391
	392	void grow(unsigned AtLeast) { static_cast<DerivedT *>(this)->grow(AtLeast); }
	393
	394	template <typename KeyArg, typename... ValueArgs>
	395	BucketT InsertIntoBucket(BucketT TheBucket, KeyArg &&Key,
	396	ValueArgs &&...Values) {
	397	TheBucket = InsertIntoBucketImpl(Key, Key, TheBucket);
	398
	399	TheBucket->getFirst() = __sanitizer::forward<KeyArg>(Key);
	400	::new (&TheBucket->getSecond())
	401	ValueT(__sanitizer::forward<ValueArgs>(Values)...);
	402	return TheBucket;
	403	}
	404
	405	template <typename LookupKeyT>
	406	BucketT InsertIntoBucketWithLookup(BucketT TheBucket, KeyT &&Key,
	407	ValueT &&Value, LookupKeyT &Lookup) {
	408	TheBucket = InsertIntoBucketImpl(Key, Lookup, TheBucket);
	409
	410	TheBucket->getFirst() = __sanitizer::move(Key);
	411	::new (&TheBucket->getSecond()) ValueT(__sanitizer::move(Value));
	412	return TheBucket;
	413	}
	414
	415	template <typename LookupKeyT>
	416	BucketT *InsertIntoBucketImpl(const KeyT &Key, const LookupKeyT &Lookup,
	417	BucketT *TheBucket) {
	418	// If the load of the hash table is more than 3/4, or if fewer than 1/8 of
	419	// the buckets are empty (meaning that many are filled with tombstones),
	420	// grow the table.
	421	//
	422	// The later case is tricky. For example, if we had one empty bucket with
	423	// tons of tombstones, failing lookups (e.g. for insertion) would have to
	424	// probe almost the entire table until it found the empty bucket. If the
	425	// table completely filled with tombstones, no lookup would ever succeed,
	426	// causing infinite loops in lookup.
	427	unsigned NewNumEntries = getNumEntries() + 1;
	428	unsigned NumBuckets = getNumBuckets();
	429	if (UNLIKELY(NewNumEntries * 4 >= NumBuckets * 3)) {
	430	this->grow(NumBuckets * 2);
	431	LookupBucketFor(Lookup, TheBucket);
	432	NumBuckets = getNumBuckets();
	433	} else if (UNLIKELY(NumBuckets - (NewNumEntries + getNumTombstones()) <=
	434	NumBuckets / 8)) {
	435	this->grow(NumBuckets);
	436	LookupBucketFor(Lookup, TheBucket);
	437	}
	438	CHECK(TheBucket);
	439
	440	// Only update the state after we've grown our bucket space appropriately
	441	// so that when growing buckets we have self-consistent entry count.
	442	incrementNumEntries();
	443
	444	// If we are writing over a tombstone, remember this.
	445	const KeyT EmptyKey = getEmptyKey();
	446	if (!KeyInfoT::isEqual(TheBucket->getFirst(), EmptyKey))
	447	decrementNumTombstones();
	448
	449	return TheBucket;
	450	}
	451
	452	/// LookupBucketFor - Lookup the appropriate bucket for Val, returning it in
	453	/// FoundBucket. If the bucket contains the key and a value, this returns
	454	/// true, otherwise it returns a bucket with an empty marker or tombstone and
	455	/// returns false.
	456	template <typename LookupKeyT>
	457	bool LookupBucketFor(const LookupKeyT &Val,
	458	const BucketT *&FoundBucket) const {
	459	const BucketT *BucketsPtr = getBuckets();
	460	const unsigned NumBuckets = getNumBuckets();
	461
	462	if (NumBuckets == 0) {
	463	FoundBucket = nullptr;
	464	return false;
	465	}
	466
	467	// FoundTombstone - Keep track of whether we find a tombstone while probing.
	468	const BucketT *FoundTombstone = nullptr;
	469	const KeyT EmptyKey = getEmptyKey();
	470	const KeyT TombstoneKey = getTombstoneKey();
	471	CHECK(!KeyInfoT::isEqual(Val, EmptyKey));
	472	CHECK(!KeyInfoT::isEqual(Val, TombstoneKey));
	473
	474	unsigned BucketNo = getHashValue(Val) & (NumBuckets - 1);
	475	unsigned ProbeAmt = 1;
	476	while (true) {
	477	const BucketT *ThisBucket = BucketsPtr + BucketNo;
	478	// Found Val's bucket? If so, return it.
	479	if (LIKELY(KeyInfoT::isEqual(Val, ThisBucket->getFirst()))) {
	480	FoundBucket = ThisBucket;
	481	return true;
	482	}
	483
	484	// If we found an empty bucket, the key doesn't exist in the set.
	485	// Insert it and return the default value.
	486	if (LIKELY(KeyInfoT::isEqual(ThisBucket->getFirst(), EmptyKey))) {
	487	// If we've already seen a tombstone while probing, fill it in instead
	488	// of the empty bucket we eventually probed to.
	489	FoundBucket = FoundTombstone ? FoundTombstone : ThisBucket;
	490	return false;
	491	}
	492
	493	// If this is a tombstone, remember it. If Val ends up not in the map, we
	494	// prefer to return it than something that would require more probing.
	495	if (KeyInfoT::isEqual(ThisBucket->getFirst(), TombstoneKey) &&
	496	!FoundTombstone)
	497	FoundTombstone = ThisBucket; // Remember the first tombstone found.
	498
	499	// Otherwise, it's a hash collision or a tombstone, continue quadratic
	500	// probing.
	501	BucketNo += ProbeAmt++;
	502	BucketNo &= (NumBuckets - 1);
	503	}
	504	}
	505
	506	template <typename LookupKeyT>
	507	bool LookupBucketFor(const LookupKeyT &Val, BucketT *&FoundBucket) {
	508	const BucketT *ConstFoundBucket;
	509	bool Result = const_cast<const DenseMapBase *>(this)->LookupBucketFor(
	510	Val, ConstFoundBucket);
	511	FoundBucket = const_cast<BucketT *>(ConstFoundBucket);
	512	return Result;
	513	}
	514
	515	public:
	516	/// Return the approximate size (in bytes) of the actual map.
	517	/// This is just the raw memory used by DenseMap.
	518	/// If entries are pointers to objects, the size of the referenced objects
	519	/// are not included.
	520	uptr getMemorySize() const {
	521	return RoundUpTo(getNumBuckets() * sizeof(BucketT), GetPageSizeCached());
	522	}
	523	};
	524
	525	/// Equality comparison for DenseMap.
	526	///
	527	/// Iterates over elements of LHS confirming that each (key, value) pair in LHS
	528	/// is also in RHS, and that no additional pairs are in RHS.
	529	/// Equivalent to N calls to RHS.find and N value comparisons. Amortized
	530	/// complexity is linear, worst case is O(N^2) (if every hash collides).
	531	template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT,
	532	typename BucketT>
	533	bool operator==(
	534	const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &LHS,
	535	const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &RHS) {
	536	if (LHS.size() != RHS.size())
	537	return false;
	538
	539	bool R = true;
	540	LHS.forEach(
	541	[&](const typename DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT,
	542	BucketT>::value_type &KV) -> bool {
	543	const auto *I = RHS.find(KV.first);
	544	if (!I \|\| I->second != KV.second) {
	545	R = false;
	546	return false;
	547	}
	548	return true;
	549	});
	550
	551	return R;
	552	}
	553
	554	/// Inequality comparison for DenseMap.
	555	///
	556	/// Equivalent to !(LHS == RHS). See operator== for performance notes.
	557	template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT,
	558	typename BucketT>
	559	bool operator!=(
	560	const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &LHS,
	561	const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &RHS) {
	562	return !(LHS == RHS);
	563	}
	564
	565	template <typename KeyT, typename ValueT,
	566	typename KeyInfoT = DenseMapInfo<KeyT>,
	567	typename BucketT = detail::DenseMapPair<KeyT, ValueT>>
	568	class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
	569	KeyT, ValueT, KeyInfoT, BucketT> {
	570	friend class DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
	571
	572	// Lift some types from the dependent base class into this class for
	573	// simplicity of referring to them.
	574	using BaseT = DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
	575
	576	BucketT *Buckets = nullptr;
	577	unsigned NumEntries = 0;
	578	unsigned NumTombstones = 0;
	579	unsigned NumBuckets = 0;
	580
	581	public:
	582	/// Create a DenseMap with an optional \p InitialReserve that guarantee that
	583	/// this number of elements can be inserted in the map without grow()
	584	explicit DenseMap(unsigned InitialReserve) { init(InitialReserve); }
	585	constexpr DenseMap() = default;
	586
	587	DenseMap(const DenseMap &other) : BaseT() {
	588	init(0);
	589	copyFrom(other);
	590	}
	591
	592	DenseMap(DenseMap &&other) : BaseT() {
	593	init(0);
	594	swap(other);
	595	}
	596
	597	~DenseMap() {
	598	this->destroyAll();
	599	deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets);
	600	}
	601
	602	void swap(DenseMap &RHS) {
	603	Swap(Buckets, RHS.Buckets);
	604	Swap(NumEntries, RHS.NumEntries);
	605	Swap(NumTombstones, RHS.NumTombstones);
	606	Swap(NumBuckets, RHS.NumBuckets);
	607	}
	608
	609	DenseMap &operator=(const DenseMap &other) {
	610	if (&other != this)
	611	copyFrom(other);
	612	return *this;
	613	}
	614
	615	DenseMap &operator=(DenseMap &&other) {
	616	this->destroyAll();
	617	deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets, alignof(BucketT));
	618	init(0);
	619	swap(other);
	620	return *this;
	621	}
	622
	623	void copyFrom(const DenseMap &other) {
	624	this->destroyAll();
	625	deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets);
	626	if (allocateBuckets(other.NumBuckets)) {
	627	this->BaseT::copyFrom(other);
	628	} else {
	629	NumEntries = 0;
	630	NumTombstones = 0;
	631	}
	632	}
	633
	634	void init(unsigned InitNumEntries) {
	635	auto InitBuckets = BaseT::getMinBucketToReserveForEntries(InitNumEntries);
	636	if (allocateBuckets(InitBuckets)) {
	637	this->BaseT::initEmpty();
	638	} else {
	639	NumEntries = 0;
	640	NumTombstones = 0;
	641	}
	642	}
	643
	644	void grow(unsigned AtLeast) {
	645	unsigned OldNumBuckets = NumBuckets;
	646	BucketT *OldBuckets = Buckets;
	647
	648	allocateBuckets(RoundUpToPowerOfTwo(Max<unsigned>(64, AtLeast)));
	649	CHECK(Buckets);
	650	if (!OldBuckets) {
	651	this->BaseT::initEmpty();
	652	return;
	653	}
	654
	655	this->moveFromOldBuckets(OldBuckets, OldBuckets + OldNumBuckets);
	656
	657	// Free the old table.
	658	deallocate_buffer(OldBuckets, sizeof(BucketT) * OldNumBuckets);
	659	}
	660
	661	private:
	662	unsigned getNumEntries() const { return NumEntries; }
	663
	664	void setNumEntries(unsigned Num) { NumEntries = Num; }
	665
	666	unsigned getNumTombstones() const { return NumTombstones; }
	667
	668	void setNumTombstones(unsigned Num) { NumTombstones = Num; }
	669
	670	BucketT *getBuckets() const { return Buckets; }
	671
	672	unsigned getNumBuckets() const { return NumBuckets; }
	673
	674	bool allocateBuckets(unsigned Num) {
	675	NumBuckets = Num;
	676	if (NumBuckets == 0) {
	677	Buckets = nullptr;
	678	return false;
	679	}
	680
	681	uptr Size = sizeof(BucketT) * NumBuckets;
	682	if (Size * 2 <= GetPageSizeCached()) {
	683	// We always allocate at least a page, so use entire space.
	684	unsigned Log2 = MostSignificantSetBitIndex(GetPageSizeCached() / Size);
	685	Size <<= Log2;
	686	NumBuckets <<= Log2;
	687	CHECK_EQ(Size, sizeof(BucketT) * NumBuckets);
	688	CHECK_GT(Size * 2, GetPageSizeCached());
	689	}
	690	Buckets = static_cast<BucketT *>(allocate_buffer(Size));
	691	return true;
	692	}
	693
	694	static void *allocate_buffer(uptr Size) {
	695	return MmapOrDie(RoundUpTo(Size, GetPageSizeCached()), "DenseMap");
	696	}
	697
	698	static void deallocate_buffer(void *Ptr, uptr Size) {
	699	UnmapOrDie(Ptr, RoundUpTo(Size, GetPageSizeCached()));
	700	}
	701	};
	702
	703	} // namespace __sanitizer
	704
	705	#endif // SANITIZER_DENSE_MAP_H

lib/tsan/sanitizer_common/sanitizer_dense_map_info.h created+282

...	...	@@ -0,0 +1,282 @@
	1	//===- sanitizer_dense_map_info.h - Type traits for DenseMap ----- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef SANITIZER_DENSE_MAP_INFO_H
	10	#define SANITIZER_DENSE_MAP_INFO_H
	11
	12	#include "sanitizer_common.h"
	13	#include "sanitizer_internal_defs.h"
	14	#include "sanitizer_type_traits.h"
	15
	16	namespace __sanitizer {
	17
	18	namespace detail {
	19
	20	/// Simplistic combination of 32-bit hash values into 32-bit hash values.
	21	static constexpr unsigned combineHashValue(unsigned a, unsigned b) {
	22	u64 key = (u64)a << 32 \| (u64)b;
	23	key += ~(key << 32);
	24	key ^= (key >> 22);
	25	key += ~(key << 13);
	26	key ^= (key >> 8);
	27	key += (key << 3);
	28	key ^= (key >> 15);
	29	key += ~(key << 27);
	30	key ^= (key >> 31);
	31	return (unsigned)key;
	32	}
	33
	34	// We extend a pair to allow users to override the bucket type with their own
	35	// implementation without requiring two members.
	36	template <typename KeyT, typename ValueT>
	37	struct DenseMapPair {
	38	KeyT first = {};
	39	ValueT second = {};
	40	constexpr DenseMapPair() = default;
	41	constexpr DenseMapPair(const KeyT &f, const ValueT &s)
	42	: first(f), second(s) {}
	43
	44	template <typename KeyT2, typename ValueT2>
	45	constexpr DenseMapPair(KeyT2 &&f, ValueT2 &&s)
	46	: first(__sanitizer::forward<KeyT2>(f)),
	47	second(__sanitizer::forward<ValueT2>(s)) {}
	48
	49	constexpr DenseMapPair(const DenseMapPair &other) = default;
	50	constexpr DenseMapPair &operator=(const DenseMapPair &other) = default;
	51	constexpr DenseMapPair(DenseMapPair &&other) = default;
	52	constexpr DenseMapPair &operator=(DenseMapPair &&other) = default;
	53
	54	KeyT &getFirst() { return first; }
	55	const KeyT &getFirst() const { return first; }
	56	ValueT &getSecond() { return second; }
	57	const ValueT &getSecond() const { return second; }
	58	};
	59
	60	} // end namespace detail
	61
	62	template <typename T>
	63	struct DenseMapInfo {
	64	// static T getEmptyKey();
	65	// static T getTombstoneKey();
	66	// static unsigned getHashValue(const T &Val);
	67	// static bool isEqual(const T &LHS, const T &RHS);
	68	};
	69
	70	// Provide DenseMapInfo for all pointers. Come up with sentinel pointer values
	71	// that are aligned to alignof(T) bytes, but try to avoid requiring T to be
	72	// complete. This allows clients to instantiate DenseMap<T*, ...> with forward
	73	// declared key types. Assume that no pointer key type requires more than 4096
	74	// bytes of alignment.
	75	template <typename T>
	76	struct DenseMapInfo<T *> {
	77	// The following should hold, but it would require T to be complete:
	78	// static_assert(alignof(T) <= (1 << Log2MaxAlign),
	79	// "DenseMap does not support pointer keys requiring more than "
	80	// "Log2MaxAlign bits of alignment");
	81	static constexpr uptr Log2MaxAlign = 12;
	82
	83	static constexpr T *getEmptyKey() {
	84	uptr Val = static_cast<uptr>(-1);
	85	Val <<= Log2MaxAlign;
	86	return reinterpret_cast<T *>(Val);
	87	}
	88
	89	static constexpr T *getTombstoneKey() {
	90	uptr Val = static_cast<uptr>(-2);
	91	Val <<= Log2MaxAlign;
	92	return reinterpret_cast<T *>(Val);
	93	}
	94
	95	static constexpr unsigned getHashValue(const T *PtrVal) {
	96	return (unsigned((uptr)PtrVal) >> 4) ^ (unsigned((uptr)PtrVal) >> 9);
	97	}
	98
	99	static constexpr bool isEqual(const T LHS, const T RHS) {
	100	return LHS == RHS;
	101	}
	102	};
	103
	104	// Provide DenseMapInfo for chars.
	105	template <>
	106	struct DenseMapInfo<char> {
	107	static constexpr char getEmptyKey() { return ~0; }
	108	static constexpr char getTombstoneKey() { return ~0 - 1; }
	109	static constexpr unsigned getHashValue(const char &Val) { return Val * 37U; }
	110
	111	static constexpr bool isEqual(const char &LHS, const char &RHS) {
	112	return LHS == RHS;
	113	}
	114	};
	115
	116	// Provide DenseMapInfo for unsigned chars.
	117	template <>
	118	struct DenseMapInfo<unsigned char> {
	119	static constexpr unsigned char getEmptyKey() { return ~0; }
	120	static constexpr unsigned char getTombstoneKey() { return ~0 - 1; }
	121	static constexpr unsigned getHashValue(const unsigned char &Val) {
	122	return Val * 37U;
	123	}
	124
	125	static constexpr bool isEqual(const unsigned char &LHS,
	126	const unsigned char &RHS) {
	127	return LHS == RHS;
	128	}
	129	};
	130
	131	// Provide DenseMapInfo for unsigned shorts.
	132	template <>
	133	struct DenseMapInfo<unsigned short> {
	134	static constexpr unsigned short getEmptyKey() { return 0xFFFF; }
	135	static constexpr unsigned short getTombstoneKey() { return 0xFFFF - 1; }
	136	static constexpr unsigned getHashValue(const unsigned short &Val) {
	137	return Val * 37U;
	138	}
	139
	140	static constexpr bool isEqual(const unsigned short &LHS,
	141	const unsigned short &RHS) {
	142	return LHS == RHS;
	143	}
	144	};
	145
	146	// Provide DenseMapInfo for unsigned ints.
	147	template <>
	148	struct DenseMapInfo<unsigned> {
	149	static constexpr unsigned getEmptyKey() { return ~0U; }
	150	static constexpr unsigned getTombstoneKey() { return ~0U - 1; }
	151	static constexpr unsigned getHashValue(const unsigned &Val) {
	152	return Val * 37U;
	153	}
	154
	155	static constexpr bool isEqual(const unsigned &LHS, const unsigned &RHS) {
	156	return LHS == RHS;
	157	}
	158	};
	159
	160	// Provide DenseMapInfo for unsigned longs.
	161	template <>
	162	struct DenseMapInfo<unsigned long> {
	163	static constexpr unsigned long getEmptyKey() { return ~0UL; }
	164	static constexpr unsigned long getTombstoneKey() { return ~0UL - 1L; }
	165
	166	static constexpr unsigned getHashValue(const unsigned long &Val) {
	167	return (unsigned)(Val * 37UL);
	168	}
	169
	170	static constexpr bool isEqual(const unsigned long &LHS,
	171	const unsigned long &RHS) {
	172	return LHS == RHS;
	173	}
	174	};
	175
	176	// Provide DenseMapInfo for unsigned long longs.
	177	template <>
	178	struct DenseMapInfo<unsigned long long> {
	179	static constexpr unsigned long long getEmptyKey() { return ~0ULL; }
	180	static constexpr unsigned long long getTombstoneKey() { return ~0ULL - 1ULL; }
	181
	182	static constexpr unsigned getHashValue(const unsigned long long &Val) {
	183	return (unsigned)(Val * 37ULL);
	184	}
	185
	186	static constexpr bool isEqual(const unsigned long long &LHS,
	187	const unsigned long long &RHS) {
	188	return LHS == RHS;
	189	}
	190	};
	191
	192	// Provide DenseMapInfo for shorts.
	193	template <>
	194	struct DenseMapInfo<short> {
	195	static constexpr short getEmptyKey() { return 0x7FFF; }
	196	static constexpr short getTombstoneKey() { return -0x7FFF - 1; }
	197	static constexpr unsigned getHashValue(const short &Val) { return Val * 37U; }
	198	static constexpr bool isEqual(const short &LHS, const short &RHS) {
	199	return LHS == RHS;
	200	}
	201	};
	202
	203	// Provide DenseMapInfo for ints.
	204	template <>
	205	struct DenseMapInfo<int> {
	206	static constexpr int getEmptyKey() { return 0x7fffffff; }
	207	static constexpr int getTombstoneKey() { return -0x7fffffff - 1; }
	208	static constexpr unsigned getHashValue(const int &Val) {
	209	return (unsigned)(Val * 37U);
	210	}
	211
	212	static constexpr bool isEqual(const int &LHS, const int &RHS) {
	213	return LHS == RHS;
	214	}
	215	};
	216
	217	// Provide DenseMapInfo for longs.
	218	template <>
	219	struct DenseMapInfo<long> {
	220	static constexpr long getEmptyKey() {
	221	return (1UL << (sizeof(long) * 8 - 1)) - 1UL;
	222	}
	223
	224	static constexpr long getTombstoneKey() { return getEmptyKey() - 1L; }
	225
	226	static constexpr unsigned getHashValue(const long &Val) {
	227	return (unsigned)(Val * 37UL);
	228	}
	229
	230	static constexpr bool isEqual(const long &LHS, const long &RHS) {
	231	return LHS == RHS;
	232	}
	233	};
	234
	235	// Provide DenseMapInfo for long longs.
	236	template <>
	237	struct DenseMapInfo<long long> {
	238	static constexpr long long getEmptyKey() { return 0x7fffffffffffffffLL; }
	239	static constexpr long long getTombstoneKey() {
	240	return -0x7fffffffffffffffLL - 1;
	241	}
	242
	243	static constexpr unsigned getHashValue(const long long &Val) {
	244	return (unsigned)(Val * 37ULL);
	245	}
	246
	247	static constexpr bool isEqual(const long long &LHS, const long long &RHS) {
	248	return LHS == RHS;
	249	}
	250	};
	251
	252	// Provide DenseMapInfo for all pairs whose members have info.
	253	template <typename T, typename U>
	254	struct DenseMapInfo<detail::DenseMapPair<T, U>> {
	255	using Pair = detail::DenseMapPair<T, U>;
	256	using FirstInfo = DenseMapInfo<T>;
	257	using SecondInfo = DenseMapInfo<U>;
	258
	259	static constexpr Pair getEmptyKey() {
	260	return detail::DenseMapPair<T, U>(FirstInfo::getEmptyKey(),
	261	SecondInfo::getEmptyKey());
	262	}
	263
	264	static constexpr Pair getTombstoneKey() {
	265	return detail::DenseMapPair<T, U>(FirstInfo::getTombstoneKey(),
	266	SecondInfo::getTombstoneKey());
	267	}
	268
	269	static constexpr unsigned getHashValue(const Pair &PairVal) {
	270	return detail::combineHashValue(FirstInfo::getHashValue(PairVal.first),
	271	SecondInfo::getHashValue(PairVal.second));
	272	}
	273
	274	static constexpr bool isEqual(const Pair &LHS, const Pair &RHS) {
	275	return FirstInfo::isEqual(LHS.first, RHS.first) &&
	276	SecondInfo::isEqual(LHS.second, RHS.second);
	277	}
	278	};
	279
	280	} // namespace __sanitizer
	281
	282	#endif // SANITIZER_DENSE_MAP_INFO_H

lib/tsan/sanitizer_common/sanitizer_errno.h+1-1

...	...	@@ -21,7 +21,7 @@
21	21	#include "sanitizer_errno_codes.h"
22	22	#include "sanitizer_platform.h"
23	23
24		#if SANITIZER_FREEBSD \|\| SANITIZER_MAC
	24	#if SANITIZER_FREEBSD \|\| SANITIZER_APPLE
25	25	# define __errno_location __error
26	26	#elif SANITIZER_ANDROID \|\| SANITIZER_NETBSD
27	27	# define __errno_location __errno

lib/tsan/sanitizer_common/sanitizer_errno_codes.h+1

...	...	@@ -25,6 +25,7 @@ namespace __sanitizer {
25	25	#define errno_EBUSY 16
26	26	#define errno_EINVAL 22
27	27	#define errno_ENAMETOOLONG 36
	28	#define errno_ENOSYS 38
28	29
29	30	// Those might not present or their value differ on different platforms.
30	31	extern const int errno_EOWNERDEAD;

lib/tsan/sanitizer_common/sanitizer_file.cpp+20

...	...	@@ -19,6 +19,7 @@
19	19
20	20	#include "sanitizer_common.h"
21	21	#include "sanitizer_file.h"
	22	# include "sanitizer_interface_internal.h"
22	23
23	24	namespace __sanitizer {
24	25
...	...	@@ -75,6 +76,24 @@ void ReportFile::ReopenIfNecessary() {
75	76	fd_pid = pid;
76	77	}
77	78
	79	static void RecursiveCreateParentDirs(char *path) {
	80	if (path[0] == '\0')
	81	return;
	82	for (int i = 1; path[i] != '\0'; ++i) {
	83	char save = path[i];
	84	if (!IsPathSeparator(path[i]))
	85	continue;
	86	path[i] = '\0';
	87	if (!DirExists(path) && !CreateDir(path)) {
	88	const char *ErrorMsgPrefix = "ERROR: Can't create directory: ";
	89	WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
	90	WriteToFile(kStderrFd, path, internal_strlen(path));
	91	Die();
	92	}
	93	path[i] = save;
	94	}
	95	}
	96
78	97	void ReportFile::SetReportPath(const char *path) {
79	98	if (path) {
80	99	uptr len = internal_strlen(path);
...	...	@@ -95,6 +114,7 @@ void ReportFile::SetReportPath(const char *path) {
95	114	fd = kStdoutFd;
96	115	} else {
97	116	internal_snprintf(path_prefix, kMaxPathLength, "%s", path);
	117	RecursiveCreateParentDirs(path_prefix);
98	118	}
99	119	}
100	120

lib/tsan/sanitizer_common/sanitizer_file.h+4-1

...	...	@@ -15,7 +15,7 @@
15	15	#ifndef SANITIZER_FILE_H
16	16	#define SANITIZER_FILE_H
17	17
18		#include "sanitizer_interface_internal.h"
	18	#include "sanitizer_common.h"
19	19	#include "sanitizer_internal_defs.h"
20	20	#include "sanitizer_libc.h"
21	21	#include "sanitizer_mutex.h"
...	...	@@ -78,9 +78,12 @@ bool SupportsColoredOutput(fd_t fd);
78	78	// OS
79	79	const char *GetPwd();
80	80	bool FileExists(const char *filename);
	81	bool DirExists(const char *path);
81	82	char FindPathToBinary(const char name);
82	83	bool IsPathSeparator(const char c);
83	84	bool IsAbsolutePath(const char *path);
	85	// Returns true on success, false on failure.
	86	bool CreateDir(const char *pathname);
84	87	// Starts a subprocess and returs its pid.
85	88	// If *_fd parameters are not kInvalidFd their corresponding input/output
86	89	// streams will be redirect to the file. The files will always be closed

lib/tsan/sanitizer_common/sanitizer_flag_parser.cpp+2-2

...	...	@@ -13,9 +13,9 @@
13	13	#include "sanitizer_flag_parser.h"
14	14
15	15	#include "sanitizer_common.h"
16		#include "sanitizer_libc.h"
17		#include "sanitizer_flags.h"
18	16	#include "sanitizer_flag_parser.h"
	17	#include "sanitizer_flags.h"
	18	#include "sanitizer_libc.h"
19	19
20	20	namespace __sanitizer {
21	21

lib/tsan/sanitizer_common/sanitizer_flag_parser.h+2-2

...	...	@@ -13,9 +13,9 @@
13	13	#ifndef SANITIZER_FLAG_REGISTRY_H
14	14	#define SANITIZER_FLAG_REGISTRY_H
15	15
	16	#include "sanitizer_common.h"
16	17	#include "sanitizer_internal_defs.h"
17	18	#include "sanitizer_libc.h"
18		#include "sanitizer_common.h"
19	19
20	20	namespace __sanitizer {
21	21
...	...	@@ -138,7 +138,7 @@ inline bool FlagHandler<uptr>::Parse(const char *value) {
138	138
139	139	template <>
140	140	inline bool FlagHandler<uptr>::Format(char *buffer, uptr size) {
141		uptr num_symbols_should_write = internal_snprintf(buffer, size, "%p", *t_);
	141	uptr num_symbols_should_write = internal_snprintf(buffer, size, "0x%zx", *t_);
142	142	return num_symbols_should_write < size;
143	143	}
144	144

lib/tsan/sanitizer_common/sanitizer_flags.inc+15-5

...	...	@@ -62,16 +62,19 @@ COMMON_FLAG(
62	62	COMMON_FLAG(const char *, log_suffix, nullptr,
63	63	"String to append to log file name, e.g. \".txt\".")
64	64	COMMON_FLAG(
65		bool, log_to_syslog, (bool)SANITIZER_ANDROID \|\| (bool)SANITIZER_MAC,
	65	bool, log_to_syslog, (bool)SANITIZER_ANDROID \|\| (bool)SANITIZER_APPLE,
66	66	"Write all sanitizer output to syslog in addition to other means of "
67	67	"logging.")
68	68	COMMON_FLAG(
69	69	int, verbosity, 0,
70	70	"Verbosity level (0 - silent, 1 - a bit of output, 2+ - more output).")
71		COMMON_FLAG(bool, strip_env, 1,
	71	COMMON_FLAG(bool, strip_env, true,
72	72	"Whether to remove the sanitizer from DYLD_INSERT_LIBRARIES to "
73		"avoid passing it to children. Default is true.")
74		COMMON_FLAG(bool, detect_leaks, !SANITIZER_MAC, "Enable memory leak detection.")
	73	"avoid passing it to children on Apple platforms. Default is true.")
	74	COMMON_FLAG(bool, verify_interceptors, true,
	75	"Verify that interceptors are working on Apple platforms. Default "
	76	"is true.")
	77	COMMON_FLAG(bool, detect_leaks, !SANITIZER_APPLE, "Enable memory leak detection.")
75	78	COMMON_FLAG(
76	79	bool, leak_check_at_exit, true,
77	80	"Invoke leak checking in an atexit handler. Has no effect if "
...	...	@@ -160,6 +163,10 @@ COMMON_FLAG(
160	163	COMMON_FLAG(const char *, coverage_dir, ".",
161	164	"Target directory for coverage dumps. Defaults to the current "
162	165	"directory.")
	166	COMMON_FLAG(const char *, cov_8bit_counters_out, "",
	167	"If non-empty, write 8bit counters to this file. ")
	168	COMMON_FLAG(const char *, cov_pcs_out, "",
	169	"If non-empty, write the coverage pc table to this file. ")
163	170	COMMON_FLAG(bool, full_address_space, false,
164	171	"Sanitize complete address space; "
165	172	"by default kernel area on 32-bit platforms will not be sanitized")
...	...	@@ -175,6 +182,7 @@ COMMON_FLAG(bool, use_madv_dontdump, true,
175	182	"in core file.")
176	183	COMMON_FLAG(bool, symbolize_inline_frames, true,
177	184	"Print inlined frames in stacktraces. Defaults to true.")
	185	COMMON_FLAG(bool, demangle, true, "Print demangled symbols.")
178	186	COMMON_FLAG(bool, symbolize_vs_style, false,
179	187	"Print file locations in Visual Studio style (e.g: "
180	188	" file(10,42): ...")
...	...	@@ -187,6 +195,8 @@ COMMON_FLAG(const char *, stack_trace_format, "DEFAULT",
187	195	"Format string used to render stack frames. "
188	196	"See sanitizer_stacktrace_printer.h for the format description. "
189	197	"Use DEFAULT to get default format.")
	198	COMMON_FLAG(int, compress_stack_depot, 0,
	199	"Compress stack depot to save memory.")
190	200	COMMON_FLAG(bool, no_huge_pages_for_shadow, true,
191	201	"If true, the shadow is not allowed to use huge pages. ")
192	202	COMMON_FLAG(bool, strict_string_checks, false,
...	...	@@ -238,7 +248,7 @@ COMMON_FLAG(bool, decorate_proc_maps, (bool)SANITIZER_ANDROID,
238	248	COMMON_FLAG(int, exitcode, 1, "Override the program exit status if the tool "
239	249	"found an error")
240	250	COMMON_FLAG(
241		bool, abort_on_error, (bool)SANITIZER_ANDROID \|\| (bool)SANITIZER_MAC,
	251	bool, abort_on_error, (bool)SANITIZER_ANDROID \|\| (bool)SANITIZER_APPLE,
242	252	"If set, the tool calls abort() instead of _exit() after printing the "
243	253	"error report.")
244	254	COMMON_FLAG(bool, suppress_equal_pcs, true,

lib/tsan/sanitizer_common/sanitizer_flat_map.h created+162

...	...	@@ -0,0 +1,162 @@
	1	//===-- sanitizer_flat_map.h ------------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// Part of the Sanitizer Allocator.
	10	//
	11	//===----------------------------------------------------------------------===//
	12
	13	#ifndef SANITIZER_FLAT_MAP_H
	14	#define SANITIZER_FLAT_MAP_H
	15
	16	#include "sanitizer_atomic.h"
	17	#include "sanitizer_common.h"
	18	#include "sanitizer_internal_defs.h"
	19	#include "sanitizer_local_address_space_view.h"
	20	#include "sanitizer_mutex.h"
	21
	22	namespace __sanitizer {
	23
	24	// Maps integers in rage [0, kSize) to values.
	25	template <typename T, u64 kSize,
	26	typename AddressSpaceViewTy = LocalAddressSpaceView>
	27	class FlatMap {
	28	public:
	29	using AddressSpaceView = AddressSpaceViewTy;
	30	void Init() { internal_memset(map_, 0, sizeof(map_)); }
	31
	32	constexpr uptr size() const { return kSize; }
	33
	34	bool contains(uptr idx) const {
	35	CHECK_LT(idx, kSize);
	36	return true;
	37	}
	38
	39	T &operator[](uptr idx) {
	40	DCHECK_LT(idx, kSize);
	41	return map_[idx];
	42	}
	43
	44	const T &operator[](uptr idx) const {
	45	DCHECK_LT(idx, kSize);
	46	return map_[idx];
	47	}
	48
	49	private:
	50	T map_[kSize];
	51	};
	52
	53	// TwoLevelMap maps integers in range [0, kSize1*kSize2) to values.
	54	// It is implemented as a two-dimensional array: array of kSize1 pointers
	55	// to kSize2-byte arrays. The secondary arrays are mmaped on demand.
	56	// Each value is initially zero and can be set to something else only once.
	57	// Setting and getting values from multiple threads is safe w/o extra locking.
	58	template <typename T, u64 kSize1, u64 kSize2,
	59	typename AddressSpaceViewTy = LocalAddressSpaceView>
	60	class TwoLevelMap {
	61	static_assert(IsPowerOfTwo(kSize2), "Use a power of two for performance.");
	62
	63	public:
	64	using AddressSpaceView = AddressSpaceViewTy;
	65	void Init() {
	66	mu_.Init();
	67	internal_memset(map1_, 0, sizeof(map1_));
	68	}
	69
	70	void TestOnlyUnmap() {
	71	for (uptr i = 0; i < kSize1; i++) {
	72	T *p = Get(i);
	73	if (!p)
	74	continue;
	75	UnmapOrDie(p, kSize2);
	76	}
	77	Init();
	78	}
	79
	80	uptr MemoryUsage() const {
	81	uptr res = 0;
	82	for (uptr i = 0; i < kSize1; i++) {
	83	T *p = Get(i);
	84	if (!p)
	85	continue;
	86	res += MmapSize();
	87	}
	88	return res;
	89	}
	90
	91	constexpr uptr size() const { return kSize1 * kSize2; }
	92	constexpr uptr size1() const { return kSize1; }
	93	constexpr uptr size2() const { return kSize2; }
	94
	95	bool contains(uptr idx) const {
	96	CHECK_LT(idx, kSize1 * kSize2);
	97	return Get(idx / kSize2);
	98	}
	99
	100	const T &operator[](uptr idx) const {
	101	DCHECK_LT(idx, kSize1 * kSize2);
	102	T *map2 = GetOrCreate(idx / kSize2);
	103	return *AddressSpaceView::Load(&map2[idx % kSize2]);
	104	}
	105
	106	T &operator[](uptr idx) {
	107	DCHECK_LT(idx, kSize1 * kSize2);
	108	T *map2 = GetOrCreate(idx / kSize2);
	109	return *AddressSpaceView::LoadWritable(&map2[idx % kSize2]);
	110	}
	111
	112	private:
	113	constexpr uptr MmapSize() const {
	114	return RoundUpTo(kSize2 * sizeof(T), GetPageSizeCached());
	115	}
	116
	117	T *Get(uptr idx) const {
	118	DCHECK_LT(idx, kSize1);
	119	return reinterpret_cast<T *>(
	120	atomic_load(&map1_[idx], memory_order_acquire));
	121	}
	122
	123	T *GetOrCreate(uptr idx) const {
	124	DCHECK_LT(idx, kSize1);
	125	// This code needs to use memory_order_acquire/consume, but we use
	126	// memory_order_relaxed for performance reasons (matters for arm64). We
	127	// expect memory_order_relaxed to be effectively equivalent to
	128	// memory_order_consume in this case for all relevant architectures: all
	129	// dependent data is reachable only by dereferencing the resulting pointer.
	130	// If relaxed load fails to see stored ptr, the code will fall back to
	131	// Create() and reload the value again with locked mutex as a memory
	132	// barrier.
	133	T res = reinterpret_cast<T >(atomic_load_relaxed(&map1_[idx]));
	134	if (LIKELY(res))
	135	return res;
	136	return Create(idx);
	137	}
	138
	139	NOINLINE T *Create(uptr idx) const {
	140	SpinMutexLock l(&mu_);
	141	T *res = Get(idx);
	142	if (!res) {
	143	res = reinterpret_cast<T *>(MmapOrDie(MmapSize(), "TwoLevelMap"));
	144	atomic_store(&map1_[idx], reinterpret_cast<uptr>(res),
	145	memory_order_release);
	146	}
	147	return res;
	148	}
	149
	150	mutable StaticSpinMutex mu_;
	151	mutable atomic_uintptr_t map1_[kSize1];
	152	};
	153
	154	template <u64 kSize, typename AddressSpaceViewTy = LocalAddressSpaceView>
	155	using FlatByteMap = FlatMap<u8, kSize, AddressSpaceViewTy>;
	156
	157	template <u64 kSize1, u64 kSize2,
	158	typename AddressSpaceViewTy = LocalAddressSpaceView>
	159	using TwoLevelByteMap = TwoLevelMap<u8, kSize1, kSize2, AddressSpaceViewTy>;
	160	} // namespace __sanitizer
	161
	162	#endif

lib/tsan/sanitizer_common/sanitizer_fuchsia.cpp+37-79

...	...	@@ -14,24 +14,25 @@
14	14	#include "sanitizer_fuchsia.h"
15	15	#if SANITIZER_FUCHSIA
16	16
17		#include <pthread.h>
18		#include <stdlib.h>
19		#include <unistd.h>
20		#include <zircon/errors.h>
21		#include <zircon/process.h>
22		#include <zircon/syscalls.h>
23		#include <zircon/utc.h>
24
25		#include "sanitizer_common.h"
26		#include "sanitizer_libc.h"
27		#include "sanitizer_mutex.h"
	17	# include <pthread.h>
	18	# include <stdlib.h>
	19	# include <unistd.h>
	20	# include <zircon/errors.h>
	21	# include <zircon/process.h>
	22	# include <zircon/syscalls.h>
	23	# include <zircon/utc.h>
	24
	25	# include "sanitizer_common.h"
	26	# include "sanitizer_interface_internal.h"
	27	# include "sanitizer_libc.h"
	28	# include "sanitizer_mutex.h"
28	29
29	30	namespace __sanitizer {
30	31
31	32	void NORETURN internal__exit(int exitcode) { _zx_process_exit(exitcode); }
32	33
33	34	uptr internal_sched_yield() {
34		zx_status_t status = _zx_nanosleep(0);
	35	zx_status_t status = _zx_thread_legacy_yield(0u);
35	36	CHECK_EQ(status, ZX_OK);
36	37	return 0; // Why doesn't this return void?
37	38	}
...	...	@@ -86,10 +87,9 @@ void GetThreadStackTopAndBottom(bool, uptr stack_top, uptr stack_bottom) {
86	87	}
87	88
88	89	void InitializePlatformEarly() {}
89		void MaybeReexec() {}
90	90	void CheckASLR() {}
91	91	void CheckMPROTECT() {}
92		void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {}
	92	void PlatformPrepareForSandboxing(void *args) {}
93	93	void DisableCoreDumperIfNecessary() {}
94	94	void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
95	95	void SetAlternateSignalStack() {}
...	...	@@ -112,47 +112,6 @@ void FutexWake(atomic_uint32_t *p, u32 count) {
112	112	CHECK_EQ(status, ZX_OK);
113	113	}
114	114
115		enum MutexState : int { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 };
116
117		BlockingMutex::BlockingMutex() {
118		// NOTE! It's important that this use internal_memset, because plain
119		// memset might be intercepted (e.g., actually be __asan_memset).
120		// Defining this so the compiler initializes each field, e.g.:
121		// BlockingMutex::BlockingMutex() : BlockingMutex(LINKER_INITIALIZED) {}
122		// might result in the compiler generating a call to memset, which would
123		// have the same problem.
124		internal_memset(this, 0, sizeof(*this));
125		}
126
127		void BlockingMutex::Lock() {
128		CHECK_EQ(owner_, 0);
129		atomic_uint32_t m = reinterpret_cast<atomic_uint32_t >(&opaque_storage_);
130		if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked)
131		return;
132		while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) {
133		zx_status_t status =
134		_zx_futex_wait(reinterpret_cast<zx_futex_t *>(m), MtxSleeping,
135		ZX_HANDLE_INVALID, ZX_TIME_INFINITE);
136		if (status != ZX_ERR_BAD_STATE) // Normal race.
137		CHECK_EQ(status, ZX_OK);
138		}
139		}
140
141		void BlockingMutex::Unlock() {
142		atomic_uint32_t m = reinterpret_cast<atomic_uint32_t >(&opaque_storage_);
143		u32 v = atomic_exchange(m, MtxUnlocked, memory_order_release);
144		CHECK_NE(v, MtxUnlocked);
145		if (v == MtxSleeping) {
146		zx_status_t status = _zx_futex_wake(reinterpret_cast<zx_futex_t *>(m), 1);
147		CHECK_EQ(status, ZX_OK);
148		}
149		}
150
151		void BlockingMutex::CheckLocked() const {
152		auto m = reinterpret_cast<atomic_uint32_t const *>(&opaque_storage_);
153		CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
154		}
155
156	115	uptr GetPageSize() { return _zx_system_get_page_size(); }
157	116
158	117	uptr GetMmapGranularity() { return _zx_system_get_page_size(); }
...	...	@@ -168,6 +127,8 @@ uptr GetMaxUserVirtualAddress() {
168	127
169	128	uptr GetMaxVirtualAddress() { return GetMaxUserVirtualAddress(); }
170	129
	130	bool ErrorIsOOM(error_t err) { return err == ZX_ERR_NO_MEMORY; }
	131
171	132	static void DoAnonymousMmapOrDie(uptr size, const char mem_type,
172	133	bool raw_report, bool die_for_nomem) {
173	134	size = RoundUpTo(size, GetPageSize());
...	...	@@ -315,6 +276,21 @@ void MmapFixedNoAccess(uptr fixed_addr, uptr size, const char name) {
315	276	UNIMPLEMENTED();
316	277	}
317	278
	279	bool MprotectNoAccess(uptr addr, uptr size) {
	280	return _zx_vmar_protect(_zx_vmar_root_self(), 0, addr, size) == ZX_OK;
	281	}
	282
	283	bool MprotectReadOnly(uptr addr, uptr size) {
	284	return _zx_vmar_protect(_zx_vmar_root_self(), ZX_VM_PERM_READ, addr, size) ==
	285	ZX_OK;
	286	}
	287
	288	bool MprotectReadWrite(uptr addr, uptr size) {
	289	return _zx_vmar_protect(_zx_vmar_root_self(),
	290	ZX_VM_PERM_READ \| ZX_VM_PERM_WRITE, addr,
	291	size) == ZX_OK;
	292	}
	293
318	294	void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
319	295	const char *mem_type) {
320	296	CHECK_GE(size, GetPageSize());
...	...	@@ -413,33 +389,12 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) {
413	389	}
414	390
415	391	// FIXME implement on this platform.
416		void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {}
	392	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
417	393
418	394	bool ReadFileToBuffer(const char file_name, char buff, uptr buff_size,
419	395	uptr read_len, uptr max_len, error_t errno_p) {
420		zx_handle_t vmo;
421		zx_status_t status = __sanitizer_get_configuration(file_name, &vmo);
422		if (status == ZX_OK) {
423		uint64_t vmo_size;
424		status = _zx_vmo_get_size(vmo, &vmo_size);
425		if (status == ZX_OK) {
426		if (vmo_size < max_len)
427		max_len = vmo_size;
428		size_t map_size = RoundUpTo(max_len, GetPageSize());
429		uintptr_t addr;
430		status = _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ, 0, vmo, 0,
431		map_size, &addr);
432		if (status == ZX_OK) {
433		buff = reinterpret_cast<char >(addr);
434		*buff_size = map_size;
435		*read_len = max_len;
436		}
437		}
438		_zx_handle_close(vmo);
439		}
440		if (status != ZX_OK && errno_p)
441		*errno_p = status;
442		return status == ZX_OK;
	396	*errno_p = ZX_ERR_NOT_SUPPORTED;
	397	return false;
443	398	}
444	399
445	400	void RawWrite(const char *buffer) {
...	...	@@ -516,6 +471,9 @@ u32 GetNumberOfCPUs() { return zx_system_get_num_cpus(); }
516	471
517	472	uptr GetRSS() { UNIMPLEMENTED(); }
518	473
	474	void internal_start_thread(void (func)(void arg), void *arg) { return 0; }
	475	void internal_join_thread(void *th) {}
	476
519	477	void InitializePlatformCommonFlags(CommonFlags *cf) {}
520	478
521	479	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_hash.h+24

...	...	@@ -38,6 +38,30 @@ class MurMur2HashBuilder {
38	38	return x;
39	39	}
40	40	};
	41
	42	class MurMur2Hash64Builder {
	43	static const u64 m = 0xc6a4a7935bd1e995ull;
	44	static const u64 seed = 0x9747b28c9747b28cull;
	45	static const u64 r = 47;
	46	u64 h;
	47
	48	public:
	49	explicit MurMur2Hash64Builder(u64 init = 0) { h = seed ^ (init * m); }
	50	void add(u64 k) {
	51	k *= m;
	52	k ^= k >> r;
	53	k *= m;
	54	h ^= k;
	55	h *= m;
	56	}
	57	u64 get() {
	58	u64 x = h;
	59	x ^= x >> r;
	60	x *= m;
	61	x ^= x >> r;
	62	return x;
	63	}
	64	};
41	65	} //namespace __sanitizer
42	66
43	67	#endif // SANITIZER_HASH_H

lib/tsan/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc+1-3

...	...	@@ -1267,8 +1267,6 @@ static void ioctl_table_fill() {
1267	1267	_(TIOCGFLAGS, WRITE, sizeof(int));
1268	1268	_(TIOCSFLAGS, READ, sizeof(int));
1269	1269	_(TIOCDCDTIMESTAMP, WRITE, struct_timeval_sz);
1270		_(TIOCRCVFRAME, READ, sizeof(uptr));
1271		_(TIOCXMTFRAME, READ, sizeof(uptr));
1272	1270	_(TIOCPTMGET, WRITE, struct_ptmget_sz);
1273	1271	_(TIOCGRANTPT, NONE, 0);
1274	1272	_(TIOCPTSNAME, WRITE, struct_ptmget_sz);
...	...	@@ -1406,7 +1404,7 @@ static void ioctl_table_fill() {
1406	1404	_(URIO_SEND_COMMAND, READWRITE, struct_urio_command_sz);
1407	1405	_(URIO_RECV_COMMAND, READWRITE, struct_urio_command_sz);
1408	1406	#undef _
1409		} // NOLINT
	1407	}
1410	1408
1411	1409	static bool ioctl_initialized = false;
1412	1410

lib/tsan/sanitizer_common/sanitizer_interface_internal.h+121-90

...	...	@@ -20,103 +20,134 @@
20	20	#include "sanitizer_internal_defs.h"
21	21
22	22	extern "C" {
23		// Tell the tools to write their reports to "path.<pid>" instead of stderr.
24		// The special values are "stdout" and "stderr".
25		SANITIZER_INTERFACE_ATTRIBUTE
26		void __sanitizer_set_report_path(const char *path);
27		// Tell the tools to write their reports to the provided file descriptor
28		// (casted to void *).
29		SANITIZER_INTERFACE_ATTRIBUTE
30		void __sanitizer_set_report_fd(void *fd);
31		// Get the current full report file path, if a path was specified by
32		// an earlier call to __sanitizer_set_report_path. Returns null otherwise.
33		SANITIZER_INTERFACE_ATTRIBUTE
34		const char *__sanitizer_get_report_path();
	23	// Tell the tools to write their reports to "path.<pid>" instead of stderr.
	24	// The special values are "stdout" and "stderr".
	25	SANITIZER_INTERFACE_ATTRIBUTE
	26	void __sanitizer_set_report_path(const char *path);
	27	// Tell the tools to write their reports to the provided file descriptor
	28	// (casted to void *).
	29	SANITIZER_INTERFACE_ATTRIBUTE
	30	void __sanitizer_set_report_fd(void *fd);
	31	// Get the current full report file path, if a path was specified by
	32	// an earlier call to __sanitizer_set_report_path. Returns null otherwise.
	33	SANITIZER_INTERFACE_ATTRIBUTE
	34	const char *__sanitizer_get_report_path();
35	35
36		typedef struct {
37		int coverage_sandboxed;
38		__sanitizer::sptr coverage_fd;
39		unsigned int coverage_max_block_size;
40		} __sanitizer_sandbox_arguments;
	36	typedef struct {
	37	int coverage_sandboxed;
	38	__sanitizer::sptr coverage_fd;
	39	unsigned int coverage_max_block_size;
	40	} __sanitizer_sandbox_arguments;
41	41
42		// Notify the tools that the sandbox is going to be turned on.
43		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
44		__sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args);
	42	// Notify the tools that the sandbox is going to be turned on.
	43	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	44	__sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args);
45	45
46		// This function is called by the tool when it has just finished reporting
47		// an error. 'error_summary' is a one-line string that summarizes
48		// the error message. This function can be overridden by the client.
49		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
50		void __sanitizer_report_error_summary(const char *error_summary);
	46	// This function is called by the tool when it has just finished reporting
	47	// an error. 'error_summary' is a one-line string that summarizes
	48	// the error message. This function can be overridden by the client.
	49	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	50	__sanitizer_report_error_summary(const char *error_summary);
51	51
52		SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump();
53		SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage(
54		const __sanitizer::uptr *pcs, const __sanitizer::uptr len);
55		SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage();
	52	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump();
	53	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage(
	54	const __sanitizer::uptr *pcs, const __sanitizer::uptr len);
	55	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage();
56	56
57		SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(__sanitizer::u32 *guard);
	57	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(__sanitizer::u32 *guard);
58	58
59		// Returns 1 on the first call, then returns 0 thereafter. Called by the tool
60		// to ensure only one report is printed when multiple errors occur
61		// simultaneously.
62		SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_acquire_crash_state();
	59	// Returns 1 on the first call, then returns 0 thereafter. Called by the tool
	60	// to ensure only one report is printed when multiple errors occur
	61	// simultaneously.
	62	SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_acquire_crash_state();
63	63
64		SANITIZER_INTERFACE_ATTRIBUTE
65		void __sanitizer_annotate_contiguous_container(const void *beg,
66		const void *end,
67		const void *old_mid,
68		const void *new_mid);
69		SANITIZER_INTERFACE_ATTRIBUTE
70		int __sanitizer_verify_contiguous_container(const void beg, const void mid,
71		const void *end);
72		SANITIZER_INTERFACE_ATTRIBUTE
73		const void *__sanitizer_contiguous_container_find_bad_address(
74		const void beg, const void mid, const void *end);
	64	SANITIZER_INTERFACE_ATTRIBUTE
	65	void __sanitizer_annotate_contiguous_container(const void beg, const void end,
	66	const void *old_mid,
	67	const void *new_mid);
	68	SANITIZER_INTERFACE_ATTRIBUTE
	69	void __sanitizer_annotate_double_ended_contiguous_container(
	70	const void storage_beg, const void storage_end,
	71	const void old_container_beg, const void old_container_end,
	72	const void new_container_beg, const void new_container_end);
	73	SANITIZER_INTERFACE_ATTRIBUTE
	74	int __sanitizer_verify_contiguous_container(const void beg, const void mid,
	75	const void *end);
	76	SANITIZER_INTERFACE_ATTRIBUTE
	77	int __sanitizer_verify_double_ended_contiguous_container(
	78	const void storage_beg, const void container_beg,
	79	const void container_end, const void storage_end);
	80	SANITIZER_INTERFACE_ATTRIBUTE
	81	const void __sanitizer_contiguous_container_find_bad_address(const void beg,
	82	const void *mid,
	83	const void *end);
	84	SANITIZER_INTERFACE_ATTRIBUTE
	85	const void *__sanitizer_double_ended_contiguous_container_find_bad_address(
	86	const void storage_beg, const void container_beg,
	87	const void container_end, const void storage_end);
75	88
76		SANITIZER_INTERFACE_ATTRIBUTE
77		int __sanitizer_get_module_and_offset_for_pc(
78		__sanitizer::uptr pc, char *module_path,
79		__sanitizer::uptr module_path_len, __sanitizer::uptr *pc_offset);
80
81		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
82		void __sanitizer_cov_trace_cmp();
83		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
84		void __sanitizer_cov_trace_cmp1();
85		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
86		void __sanitizer_cov_trace_cmp2();
87		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
88		void __sanitizer_cov_trace_cmp4();
89		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
90		void __sanitizer_cov_trace_cmp8();
91		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
92		void __sanitizer_cov_trace_const_cmp1();
93		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
94		void __sanitizer_cov_trace_const_cmp2();
95		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
96		void __sanitizer_cov_trace_const_cmp4();
97		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
98		void __sanitizer_cov_trace_const_cmp8();
99		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
100		void __sanitizer_cov_trace_switch();
101		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
102		void __sanitizer_cov_trace_div4();
103		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
104		void __sanitizer_cov_trace_div8();
105		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
106		void __sanitizer_cov_trace_gep();
107		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
108		void __sanitizer_cov_trace_pc_indir();
109		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
110		void __sanitizer_cov_trace_pc_guard(__sanitizer::u32*);
111		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
112		void __sanitizer_cov_trace_pc_guard_init(__sanitizer::u32*,
113		__sanitizer::u32*);
114		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
115		void __sanitizer_cov_8bit_counters_init();
116		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
117		__sanitizer_cov_bool_flag_init();
118		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
119		__sanitizer_cov_pcs_init();
120		} // extern "C"
	89	SANITIZER_INTERFACE_ATTRIBUTE
	90	int __sanitizer_get_module_and_offset_for_pc(void pc, char module_path,
	91	__sanitizer::uptr module_path_len,
	92	void **pc_offset);
	93	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	94	__sanitizer_cov_trace_cmp();
	95	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	96	__sanitizer_cov_trace_cmp1();
	97	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	98	__sanitizer_cov_trace_cmp2();
	99	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	100	__sanitizer_cov_trace_cmp4();
	101	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	102	__sanitizer_cov_trace_cmp8();
	103	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	104	__sanitizer_cov_trace_const_cmp1();
	105	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	106	__sanitizer_cov_trace_const_cmp2();
	107	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	108	__sanitizer_cov_trace_const_cmp4();
	109	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	110	__sanitizer_cov_trace_const_cmp8();
	111	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	112	__sanitizer_cov_trace_switch();
	113	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	114	__sanitizer_cov_trace_div4();
	115	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	116	__sanitizer_cov_trace_div8();
	117	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	118	__sanitizer_cov_trace_gep();
	119	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	120	__sanitizer_cov_trace_pc_indir();
	121	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	122	__sanitizer_cov_load1();
	123	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	124	__sanitizer_cov_load2();
	125	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	126	__sanitizer_cov_load4();
	127	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	128	__sanitizer_cov_load8();
	129	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	130	__sanitizer_cov_load16();
	131	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	132	__sanitizer_cov_store1();
	133	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	134	__sanitizer_cov_store2();
	135	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	136	__sanitizer_cov_store4();
	137	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	138	__sanitizer_cov_store8();
	139	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	140	__sanitizer_cov_store16();
	141	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	142	__sanitizer_cov_trace_pc_guard(__sanitizer::u32 *);
	143	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	144	__sanitizer_cov_trace_pc_guard_init(__sanitizer::u32 , __sanitizer::u32 );
	145	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	146	__sanitizer_cov_8bit_counters_init(char , char );
	147	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	148	__sanitizer_cov_bool_flag_init();
	149	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	150	__sanitizer_cov_pcs_init(const __sanitizer::uptr , const __sanitizer::uptr );
	151	} // extern "C"
121	152
122	153	#endif // SANITIZER_INTERFACE_INTERNAL_H

lib/tsan/sanitizer_common/sanitizer_internal_defs.h+56-35

...	...	@@ -13,6 +13,7 @@
13	13	#define SANITIZER_DEFS_H
14	14
15	15	#include "sanitizer_platform.h"
	16	#include "sanitizer_redefine_builtins.h"
16	17
17	18	#ifndef SANITIZER_DEBUG
18	19	# define SANITIZER_DEBUG 0
...	...	@@ -37,15 +38,6 @@
37	38	# define SANITIZER_WEAK_ATTRIBUTE __attribute__((weak))
38	39	#endif
39	40
40		// TLS is handled differently on different platforms
41		#if SANITIZER_LINUX \|\| SANITIZER_NETBSD \|\| \
42		SANITIZER_FREEBSD
43		# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE \
44		__attribute__((tls_model("initial-exec"))) thread_local
45		#else
46		# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE
47		#endif
48
49	41	//--------------------------- WEAK FUNCTIONS ---------------------------------//
50	42	// When working with weak functions, to simplify the code and make it more
51	43	// portable, when possible define a default implementation using this macro:
...	...	@@ -73,7 +65,7 @@
73	65	// Before Xcode 4.5, the Darwin linker doesn't reliably support undefined
74	66	// weak symbols. Mac OS X 10.9/Darwin 13 is the first release only supported
75	67	// by Xcode >= 4.5.
76		#elif SANITIZER_MAC && \
	68	#elif SANITIZER_APPLE && \
77	69	__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1090 && !SANITIZER_GO
78	70	# define SANITIZER_SUPPORTS_WEAK_HOOKS 1
79	71	#else
...	...	@@ -125,6 +117,10 @@
125	117	# define __has_attribute(x) 0
126	118	#endif
127	119
	120	#if !defined(__has_cpp_attribute)
	121	# define __has_cpp_attribute(x) 0
	122	#endif
	123
128	124	// For portability reasons we do not include stddef.h, stdint.h or any other
129	125	// system header, but we do need some basic types that are not defined
130	126	// in a portable way by the language itself.
...	...	@@ -135,8 +131,13 @@ namespace __sanitizer {
135	131	typedef unsigned long long uptr;
136	132	typedef signed long long sptr;
137	133	#else
	134	# if (SANITIZER_WORDSIZE == 64) \|\| SANITIZER_APPLE \|\| SANITIZER_WINDOWS
138	135	typedef unsigned long uptr;
139	136	typedef signed long sptr;
	137	# else
	138	typedef unsigned int uptr;
	139	typedef signed int sptr;
	140	# endif
140	141	#endif // defined(_WIN64)
141	142	#if defined(__x86_64__)
142	143	// Since x32 uses ILP32 data model in 64-bit hardware mode, we must use
...	...	@@ -168,17 +169,17 @@ typedef long pid_t;
168	169	typedef int pid_t;
169	170	#endif
170	171
171		#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| \
172		SANITIZER_MAC \|\| \
	172	#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| SANITIZER_APPLE \|\| \
173	173	(SANITIZER_SOLARIS && (defined(_LP64) \|\| _FILE_OFFSET_BITS == 64)) \|\| \
174		(SANITIZER_LINUX && defined(__x86_64__))
	174	(SANITIZER_LINUX && !SANITIZER_GLIBC && !SANITIZER_ANDROID) \|\| \
	175	(SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__hexagon__)))
175	176	typedef u64 OFF_T;
176	177	#else
177	178	typedef uptr OFF_T;
178	179	#endif
179	180	typedef u64 OFF64_T;
180	181
181		#if (SANITIZER_WORDSIZE == 64) \|\| SANITIZER_MAC
	182	#if (SANITIZER_WORDSIZE == 64) \|\| SANITIZER_APPLE
182	183	typedef uptr operator_new_size_type;
183	184	#else
184	185	# if defined(__s390__) && !defined(__s390x__)
...	...	@@ -217,7 +218,7 @@ typedef u64 tid_t;
217	218	# define WARN_UNUSED_RESULT
218	219	#else // _MSC_VER
219	220	# define ALWAYS_INLINE inline __attribute__((always_inline))
220		# define ALIAS(x) __attribute__((alias(x)))
	221	# define ALIAS(x) __attribute__((alias(SANITIZER_STRINGIFY(x))))
221	222	// Please only use the ALIGNED macro before the type.
222	223	// Using ALIGNED after the variable declaration is not portable!
223	224	# define ALIGNED(x) __attribute__((aligned(x)))
...	...	@@ -250,6 +251,20 @@ typedef u64 tid_t;
250	251	# define NOEXCEPT throw()
251	252	#endif
252	253
	254	#if __has_cpp_attribute(clang::fallthrough)
	255	# define FALLTHROUGH [[clang::fallthrough]]
	256	#elif __has_cpp_attribute(fallthrough)
	257	# define FALLTHROUGH [[fallthrough]]
	258	#else
	259	# define FALLTHROUGH
	260	#endif
	261
	262	#if __has_attribute(uninitialized)
	263	# define UNINITIALIZED __attribute__((uninitialized))
	264	#else
	265	# define UNINITIALIZED
	266	#endif
	267
253	268	// Unaligned versions of basic types.
254	269	typedef ALIGNED(1) u16 uu16;
255	270	typedef ALIGNED(1) u32 uu32;
...	...	@@ -277,14 +292,17 @@ void NORETURN CheckFailed(const char file, int line, const char cond,
277	292	u64 v1, u64 v2);
278	293
279	294	// Check macro
280		#define RAW_CHECK_MSG(expr, msg) do { \
281		if (UNLIKELY(!(expr))) { \
282		RawWrite(msg); \
283		Die(); \
284		} \
285		} while (0)
	295	#define RAW_CHECK_MSG(expr, msg, ...) \
	296	do { \
	297	if (UNLIKELY(!(expr))) { \
	298	const char* msgs[] = {msg, __VA_ARGS__}; \
	299	for (const char* m : msgs) RawWrite(m); \
	300	Die(); \
	301	} \
	302	} while (0)
286	303
287		#define RAW_CHECK(expr) RAW_CHECK_MSG(expr, #expr)
	304	#define RAW_CHECK(expr) RAW_CHECK_MSG(expr, #expr "\n", )
	305	#define RAW_CHECK_VA(expr, ...) RAW_CHECK_MSG(expr, #expr "\n", __VA_ARGS__)
288	306
289	307	#define CHECK_IMPL(c1, op, c2) \
290	308	do { \
...	...	@@ -366,13 +384,10 @@ void NORETURN CheckFailed(const char file, int line, const char cond,
366	384	enum LinkerInitialized { LINKER_INITIALIZED = 0 };
367	385
368	386	#if !defined(_MSC_VER) \|\| defined(__clang__)
369		#if SANITIZER_S390_31
370		#define GET_CALLER_PC() \
371		(__sanitizer::uptr) __builtin_extract_return_addr(__builtin_return_address(0))
372		#else
373		#define GET_CALLER_PC() (__sanitizer::uptr) __builtin_return_address(0)
374		#endif
375		#define GET_CURRENT_FRAME() (__sanitizer::uptr) __builtin_frame_address(0)
	387	# define GET_CALLER_PC() \
	388	((__sanitizer::uptr)__builtin_extract_return_addr( \
	389	__builtin_return_address(0)))
	390	# define GET_CURRENT_FRAME() ((__sanitizer::uptr)__builtin_frame_address(0))
376	391	inline void Trap() {
377	392	__builtin_trap();
378	393	}
...	...	@@ -381,13 +396,13 @@ extern "C" void* _ReturnAddress(void);
381	396	extern "C" void* _AddressOfReturnAddress(void);
382	397	# pragma intrinsic(_ReturnAddress)
383	398	# pragma intrinsic(_AddressOfReturnAddress)
384		#define GET_CALLER_PC() (__sanitizer::uptr) _ReturnAddress()
	399	# define GET_CALLER_PC() ((__sanitizer::uptr)_ReturnAddress())
385	400	// CaptureStackBackTrace doesn't need to know BP on Windows.
386		#define GET_CURRENT_FRAME() \
387		(((__sanitizer::uptr)_AddressOfReturnAddress()) + sizeof(__sanitizer::uptr))
	401	# define GET_CURRENT_FRAME() \
	402	(((__sanitizer::uptr)_AddressOfReturnAddress()) + sizeof(__sanitizer::uptr))
388	403
389	404	extern "C" void __ud2(void);
390		# pragma intrinsic(__ud2)
	405	# pragma intrinsic(__ud2)
391	406	inline void Trap() {
392	407	__ud2();
393	408	}
...	...	@@ -409,8 +424,14 @@ inline void Trap() {
409	424	(void)enable_fp; \
410	425	} while (0)
411	426
412		constexpr u32 kInvalidTid = -1;
413		constexpr u32 kMainTid = 0;
	427	// Internal thread identifier allocated by ThreadRegistry.
	428	typedef u32 Tid;
	429	constexpr Tid kInvalidTid = -1;
	430	constexpr Tid kMainTid = 0;
	431
	432	// Stack depot stack identifier.
	433	typedef u32 StackID;
	434	const StackID kInvalidStackID = 0;
414	435
415	436	} // namespace __sanitizer
416	437

lib/tsan/sanitizer_common/sanitizer_leb128.h created+87

...	...	@@ -0,0 +1,87 @@
	1	//===-- sanitizer_leb128.h --------------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef SANITIZER_LEB128_H
	10	#define SANITIZER_LEB128_H
	11
	12	#include "sanitizer_common.h"
	13	#include "sanitizer_internal_defs.h"
	14
	15	namespace __sanitizer {
	16
	17	template <typename T, typename It>
	18	It EncodeSLEB128(T value, It begin, It end) {
	19	bool more;
	20	do {
	21	u8 byte = value & 0x7f;
	22	// NOTE: this assumes that this signed shift is an arithmetic right shift.
	23	value >>= 7;
	24	more = !((((value == 0) && ((byte & 0x40) == 0)) \|\|
	25	((value == -1) && ((byte & 0x40) != 0))));
	26	if (more)
	27	byte \|= 0x80;
	28	if (UNLIKELY(begin == end))
	29	break;
	30	*(begin++) = byte;
	31	} while (more);
	32	return begin;
	33	}
	34
	35	template <typename T, typename It>
	36	It DecodeSLEB128(It begin, It end, T* v) {
	37	T value = 0;
	38	unsigned shift = 0;
	39	u8 byte;
	40	do {
	41	if (UNLIKELY(begin == end))
	42	return begin;
	43	byte = *(begin++);
	44	T slice = byte & 0x7f;
	45	value \|= slice << shift;
	46	shift += 7;
	47	} while (byte >= 128);
	48	if (shift < 64 && (byte & 0x40))
	49	value \|= (-1ULL) << shift;
	50	*v = value;
	51	return begin;
	52	}
	53
	54	template <typename T, typename It>
	55	It EncodeULEB128(T value, It begin, It end) {
	56	do {
	57	u8 byte = value & 0x7f;
	58	value >>= 7;
	59	if (value)
	60	byte \|= 0x80;
	61	if (UNLIKELY(begin == end))
	62	break;
	63	*(begin++) = byte;
	64	} while (value);
	65	return begin;
	66	}
	67
	68	template <typename T, typename It>
	69	It DecodeULEB128(It begin, It end, T* v) {
	70	T value = 0;
	71	unsigned shift = 0;
	72	u8 byte;
	73	do {
	74	if (UNLIKELY(begin == end))
	75	return begin;
	76	byte = *(begin++);
	77	T slice = byte & 0x7f;
	78	value += slice << shift;
	79	shift += 7;
	80	} while (byte >= 128);
	81	*v = value;
	82	return begin;
	83	}
	84
	85	} // namespace __sanitizer
	86
	87	#endif // SANITIZER_LEB128_H

lib/tsan/sanitizer_common/sanitizer_libc.cpp+24-3

...	...	@@ -10,6 +10,9 @@
10	10	// run-time libraries. See sanitizer_libc.h for details.
11	11	//===----------------------------------------------------------------------===//
12	12
	13	// Do not redefine builtins; this file is defining the builtin replacements.
	14	#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS
	15
13	16	#include "sanitizer_allocator_internal.h"
14	17	#include "sanitizer_common.h"
15	18	#include "sanitizer_libc.h"
...	...	@@ -46,7 +49,10 @@ int internal_memcmp(const void* s1, const void* s2, uptr n) {
46	49	return 0;
47	50	}
48	51
49		void internal_memcpy(void dest, const void *src, uptr n) {
	52	extern "C" {
	53	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_internal_memcpy(void dest,
	54	const void *src,
	55	uptr n) {
50	56	char d = (char)dest;
51	57	const char s = (const char )src;
52	58	for (uptr i = 0; i < n; ++i)
...	...	@@ -54,7 +60,8 @@ void internal_memcpy(void dest, const void *src, uptr n) {
54	60	return dest;
55	61	}
56	62
57		void internal_memmove(void dest, const void *src, uptr n) {
	63	SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memmove(
	64	void dest, const void src, uptr n) {
58	65	char d = (char)dest;
59	66	const char s = (const char )src;
60	67	sptr i, signed_n = (sptr)n;
...	...	@@ -72,7 +79,8 @@ void internal_memmove(void dest, const void *src, uptr n) {
72	79	return dest;
73	80	}
74	81
75		void internal_memset(void s, int c, uptr n) {
	82	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_internal_memset(void s, int c,
	83	uptr n) {
76	84	// Optimize for the most performance-critical case:
77	85	if ((reinterpret_cast<uptr>(s) % 16) == 0 && (n % 16) == 0) {
78	86	u64 p = reinterpret_cast<u64>(s);
...	...	@@ -95,6 +103,7 @@ void internal_memset(void s, int c, uptr n) {
95	103	}
96	104	return s;
97	105	}
	106	} // extern "C"
98	107
99	108	uptr internal_strcspn(const char s, const char reject) {
100	109	uptr i;
...	...	@@ -258,6 +267,18 @@ s64 internal_simple_strtoll(const char nptr, const char *endptr, int base) {
258	267	}
259	268	}
260	269
	270	uptr internal_wcslen(const wchar_t *s) {
	271	uptr i = 0;
	272	while (s[i]) i++;
	273	return i;
	274	}
	275
	276	uptr internal_wcsnlen(const wchar_t *s, uptr maxlen) {
	277	uptr i = 0;
	278	while (i < maxlen && s[i]) i++;
	279	return i;
	280	}
	281
261	282	bool mem_is_zero(const char *beg, uptr size) {
262	283	CHECK_LE(size, 1ULL << FIRST_32_SECOND_64(30, 40)); // Sanity check.
263	284	const char *end = beg + size;

lib/tsan/sanitizer_common/sanitizer_libc.h+25-4

...	...	@@ -24,15 +24,33 @@ namespace __sanitizer {
24	24
25	25	// internal_X() is a custom implementation of X() for use in RTL.
26	26
	27	extern "C" {
	28	// These are used as builtin replacements; see sanitizer_redefine_builtins.h.
	29	// In normal runtime code, use the __sanitizer::internal_X() aliases instead.
	30	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_internal_memcpy(void dest,
	31	const void *src,
	32	uptr n);
	33	SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memmove(
	34	void dest, const void src, uptr n);
	35	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_internal_memset(void s, int c,
	36	uptr n);
	37	} // extern "C"
	38
27	39	// String functions
28	40	s64 internal_atoll(const char *nptr);
29	41	void internal_memchr(const void s, int c, uptr n);
30	42	void internal_memrchr(const void s, int c, uptr n);
31	43	int internal_memcmp(const void* s1, const void* s2, uptr n);
32		void internal_memcpy(void dest, const void *src, uptr n);
33		void internal_memmove(void dest, const void *src, uptr n);
	44	ALWAYS_INLINE void internal_memcpy(void dest, const void *src, uptr n) {
	45	return __sanitizer_internal_memcpy(dest, src, n);
	46	}
	47	ALWAYS_INLINE void internal_memmove(void dest, const void *src, uptr n) {
	48	return __sanitizer_internal_memmove(dest, src, n);
	49	}
34	50	// Should not be used in performance-critical places.
35		void internal_memset(void s, int c, uptr n);
	51	ALWAYS_INLINE void internal_memset(void s, int c, uptr n) {
	52	return __sanitizer_internal_memset(s, c, n);
	53	}
36	54	char* internal_strchr(const char *s, int c);
37	55	char internal_strchrnul(const char s, int c);
38	56	int internal_strcmp(const char s1, const char s2);
...	...	@@ -49,7 +67,10 @@ char internal_strrchr(const char s, int c);
49	67	char internal_strstr(const char haystack, const char *needle);
50	68	// Works only for base=10 and doesn't set errno.
51	69	s64 internal_simple_strtoll(const char nptr, const char *endptr, int base);
52		int internal_snprintf(char buffer, uptr length, const char format, ...);
	70	int internal_snprintf(char buffer, uptr length, const char format, ...)
	71	FORMAT(3, 4);
	72	uptr internal_wcslen(const wchar_t *s);
	73	uptr internal_wcsnlen(const wchar_t *s, uptr maxlen);
53	74
54	75	// Return true if all bytes in [mem, mem+size) are zero.
55	76	// Optimized for the case when the result is true.

lib/tsan/sanitizer_common/sanitizer_libignore.cpp+6-6

...	...	@@ -8,7 +8,7 @@
8	8
9	9	#include "sanitizer_platform.h"
10	10
11		#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_MAC \|\| \
	11	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_APPLE \|\| \
12	12	SANITIZER_NETBSD
13	13
14	14	#include "sanitizer_libignore.h"
...	...	@@ -22,9 +22,9 @@ LibIgnore::LibIgnore(LinkerInitialized) {
22	22	}
23	23
24	24	void LibIgnore::AddIgnoredLibrary(const char *name_templ) {
25		BlockingMutexLock lock(&mutex_);
	25	Lock lock(&mutex_);
26	26	if (count_ >= kMaxLibs) {
27		Report("%s: too many ignored libraries (max: %d)\n", SanitizerToolName,
	27	Report("%s: too many ignored libraries (max: %zu)\n", SanitizerToolName,
28	28	kMaxLibs);
29	29	Die();
30	30	}
...	...	@@ -36,7 +36,7 @@ void LibIgnore::AddIgnoredLibrary(const char *name_templ) {
36	36	}
37	37
38	38	void LibIgnore::OnLibraryLoaded(const char *name) {
39		BlockingMutexLock lock(&mutex_);
	39	Lock lock(&mutex_);
40	40	// Try to match suppressions with symlink target.
41	41	InternalMmapVector<char> buf(kMaxPathLength);
42	42	if (name && internal_readlink(name, buf.data(), buf.size() - 1) > 0 &&
...	...	@@ -105,7 +105,7 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
105	105	continue;
106	106	if (IsPcInstrumented(range.beg) && IsPcInstrumented(range.end - 1))
107	107	continue;
108		VReport(1, "Adding instrumented range %p-%p from library '%s'\n",
	108	VReport(1, "Adding instrumented range 0x%zx-0x%zx from library '%s'\n",
109	109	range.beg, range.end, mod.full_name());
110	110	const uptr idx =
111	111	atomic_load(&instrumented_ranges_count_, memory_order_relaxed);
...	...	@@ -125,5 +125,5 @@ void LibIgnore::OnLibraryUnloaded() {
125	125
126	126	} // namespace __sanitizer
127	127
128		#endif // SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_MAC \|\|
	128	#endif // SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_APPLE \|\|
129	129	// SANITIZER_NETBSD

lib/tsan/sanitizer_common/sanitizer_libignore.h+1-1

...	...	@@ -77,7 +77,7 @@ class LibIgnore {
77	77	LibCodeRange instrumented_code_ranges_[kMaxInstrumentedRanges];
78	78
79	79	// Cold part:
80		BlockingMutex mutex_;
	80	Mutex mutex_;
81	81	uptr count_;
82	82	Lib libs_[kMaxLibs];
83	83	bool track_instrumented_libs_;

lib/tsan/sanitizer_common/sanitizer_linux.cpp+317-193

...	...	@@ -34,7 +34,7 @@
34	34	// format. Struct kernel_stat is defined as 'struct stat' in asm/stat.h. To
35	35	// access stat from asm/stat.h, without conflicting with definition in
36	36	// sys/stat.h, we use this trick.
37		#if defined(__mips64)
	37	#if SANITIZER_MIPS64
38	38	#include <asm/unistd.h>
39	39	#include <sys/types.h>
40	40	#define stat kernel_stat
...	...	@@ -78,8 +78,13 @@
78	78	#include <sys/personality.h>
79	79	#endif
80	80
	81	#if SANITIZER_LINUX && defined(__loongarch__)
	82	# include <sys/sysmacros.h>
	83	#endif
	84
81	85	#if SANITIZER_FREEBSD
82	86	#include <sys/exec.h>
	87	#include <sys/procctl.h>
83	88	#include <sys/sysctl.h>
84	89	#include <machine/atomic.h>
85	90	extern "C" {
...	...	@@ -123,8 +128,9 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE \| FUTEX_PRIVATE_FLAG;
123	128	// Are we using 32-bit or 64-bit Linux syscalls?
124	129	// x32 (which defines __x86_64__) has SANITIZER_WORDSIZE == 32
125	130	// but it still needs to use 64-bit syscalls.
126		#if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__powerpc64__) \|\| \
127		SANITIZER_WORDSIZE == 64)
	131	#if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__powerpc64__) \|\| \
	132	SANITIZER_WORDSIZE == 64 \|\| \
	133	(defined(__mips__) && _MIPS_SIM == _ABIN32))
128	134	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1
129	135	#else
130	136	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0
...	...	@@ -150,17 +156,51 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE \| FUTEX_PRIVATE_FLAG;
150	156
151	157	namespace __sanitizer {
152	158
153		#if SANITIZER_LINUX && defined(__x86_64__)
154		#include "sanitizer_syscall_linux_x86_64.inc"
155		#elif SANITIZER_LINUX && SANITIZER_RISCV64
156		#include "sanitizer_syscall_linux_riscv64.inc"
157		#elif SANITIZER_LINUX && defined(__aarch64__)
158		#include "sanitizer_syscall_linux_aarch64.inc"
159		#elif SANITIZER_LINUX && defined(__arm__)
160		#include "sanitizer_syscall_linux_arm.inc"
161		#else
162		#include "sanitizer_syscall_generic.inc"
163		#endif
	159	void SetSigProcMask(__sanitizer_sigset_t set, __sanitizer_sigset_t oldset) {
	160	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, oldset));
	161	}
	162
	163	void BlockSignals(__sanitizer_sigset_t *oldset) {
	164	__sanitizer_sigset_t set;
	165	internal_sigfillset(&set);
	166	# if SANITIZER_LINUX && !SANITIZER_ANDROID
	167	// Glibc uses SIGSETXID signal during setuid call. If this signal is blocked
	168	// on any thread, setuid call hangs.
	169	// See test/sanitizer_common/TestCases/Linux/setuid.c.
	170	internal_sigdelset(&set, 33);
	171	# endif
	172	# if SANITIZER_LINUX
	173	// Seccomp-BPF-sandboxed processes rely on SIGSYS to handle trapped syscalls.
	174	// If this signal is blocked, such calls cannot be handled and the process may
	175	// hang.
	176	internal_sigdelset(&set, 31);
	177	# endif
	178	SetSigProcMask(&set, oldset);
	179	}
	180
	181	ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) {
	182	BlockSignals(&saved_);
	183	if (copy)
	184	internal_memcpy(copy, &saved_, sizeof(saved_));
	185	}
	186
	187	ScopedBlockSignals::~ScopedBlockSignals() { SetSigProcMask(&saved_, nullptr); }
	188
	189	# if SANITIZER_LINUX && defined(__x86_64__)
	190	# include "sanitizer_syscall_linux_x86_64.inc"
	191	# elif SANITIZER_LINUX && SANITIZER_RISCV64
	192	# include "sanitizer_syscall_linux_riscv64.inc"
	193	# elif SANITIZER_LINUX && defined(__aarch64__)
	194	# include "sanitizer_syscall_linux_aarch64.inc"
	195	# elif SANITIZER_LINUX && defined(__arm__)
	196	# include "sanitizer_syscall_linux_arm.inc"
	197	# elif SANITIZER_LINUX && defined(__hexagon__)
	198	# include "sanitizer_syscall_linux_hexagon.inc"
	199	# elif SANITIZER_LINUX && SANITIZER_LOONGARCH64
	200	# include "sanitizer_syscall_linux_loongarch64.inc"
	201	# else
	202	# include "sanitizer_syscall_generic.inc"
	203	# endif
164	204
165	205	// --------------- sanitizer_libc.h
166	206	#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD
...	...	@@ -204,7 +244,7 @@ uptr internal_close(fd_t fd) {
204	244	}
205	245
206	246	uptr internal_open(const char *filename, int flags) {
207		#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
	247	# if SANITIZER_LINUX
208	248	return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags);
209	249	#else
210	250	return internal_syscall(SYSCALL(open), (uptr)filename, flags);
...	...	@@ -212,7 +252,7 @@ uptr internal_open(const char *filename, int flags) {
212	252	}
213	253
214	254	uptr internal_open(const char *filename, int flags, u32 mode) {
215		#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
	255	# if SANITIZER_LINUX
216	256	return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags,
217	257	mode);
218	258	#else
...	...	@@ -241,7 +281,7 @@ uptr internal_ftruncate(fd_t fd, uptr size) {
241	281	return res;
242	282	}
243	283
244		#if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && SANITIZER_LINUX
	284	#if (!SANITIZER_LINUX_USES_64BIT_SYSCALLS \|\| SANITIZER_SPARC) && SANITIZER_LINUX
245	285	static void stat64_to_stat(struct stat64 in, struct stat out) {
246	286	internal_memset(out, 0, sizeof(*out));
247	287	out->st_dev = in->st_dev;
...	...	@@ -260,7 +300,29 @@ static void stat64_to_stat(struct stat64 in, struct stat out) {
260	300	}
261	301	#endif
262	302
263		#if defined(__mips64)
	303	#if SANITIZER_LINUX && defined(__loongarch__)
	304	static void statx_to_stat(struct statx in, struct stat out) {
	305	internal_memset(out, 0, sizeof(*out));
	306	out->st_dev = makedev(in->stx_dev_major, in->stx_dev_minor);
	307	out->st_ino = in->stx_ino;
	308	out->st_mode = in->stx_mode;
	309	out->st_nlink = in->stx_nlink;
	310	out->st_uid = in->stx_uid;
	311	out->st_gid = in->stx_gid;
	312	out->st_rdev = makedev(in->stx_rdev_major, in->stx_rdev_minor);
	313	out->st_size = in->stx_size;
	314	out->st_blksize = in->stx_blksize;
	315	out->st_blocks = in->stx_blocks;
	316	out->st_atime = in->stx_atime.tv_sec;
	317	out->st_atim.tv_nsec = in->stx_atime.tv_nsec;
	318	out->st_mtime = in->stx_mtime.tv_sec;
	319	out->st_mtim.tv_nsec = in->stx_mtime.tv_nsec;
	320	out->st_ctime = in->stx_ctime.tv_sec;
	321	out->st_ctim.tv_nsec = in->stx_ctime.tv_nsec;
	322	}
	323	#endif
	324
	325	#if SANITIZER_MIPS64
264	326	// Undefine compatibility macros from <sys/stat.h>
265	327	// so that they would not clash with the kernel_stat
266	328	// st_[a\|m\|c]time fields
...	...	@@ -311,52 +373,65 @@ static void kernel_stat_to_stat(struct kernel_stat in, struct stat out) {
311	373	#endif
312	374
313	375	uptr internal_stat(const char path, void buf) {
314		#if SANITIZER_FREEBSD
	376	# if SANITIZER_FREEBSD
315	377	return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0);
316		#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
	378	# elif SANITIZER_LINUX
	379	# if defined(__loongarch__)
	380	struct statx bufx;
	381	int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path,
	382	AT_NO_AUTOMOUNT, STATX_BASIC_STATS, (uptr)&bufx);
	383	statx_to_stat(&bufx, (struct stat *)buf);
	384	return res;
	385	# elif (SANITIZER_WORDSIZE == 64 \|\| SANITIZER_X32 \|\| \
	386	(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \
	387	!SANITIZER_SPARC
317	388	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
318	389	0);
319		#elif SANITIZER_LINUX_USES_64BIT_SYSCALLS
320		# if defined(__mips64)
321		// For mips64, stat syscall fills buffer in the format of kernel_stat
322		struct kernel_stat kbuf;
323		int res = internal_syscall(SYSCALL(stat), path, &kbuf);
324		kernel_stat_to_stat(&kbuf, (struct stat *)buf);
	390	# else
	391	struct stat64 buf64;
	392	int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path,
	393	(uptr)&buf64, 0);
	394	stat64_to_stat(&buf64, (struct stat *)buf);
325	395	return res;
326		# else
327		return internal_syscall(SYSCALL(stat), (uptr)path, (uptr)buf);
328		# endif
329		#else
	396	# endif
	397	# else
330	398	struct stat64 buf64;
331	399	int res = internal_syscall(SYSCALL(stat64), path, &buf64);
332	400	stat64_to_stat(&buf64, (struct stat *)buf);
333	401	return res;
334		#endif
	402	# endif
335	403	}
336	404
337	405	uptr internal_lstat(const char path, void buf) {
338		#if SANITIZER_FREEBSD
	406	# if SANITIZER_FREEBSD
339	407	return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf,
340	408	AT_SYMLINK_NOFOLLOW);
341		#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
	409	# elif SANITIZER_LINUX
	410	# if defined(__loongarch__)
	411	struct statx bufx;
	412	int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path,
	413	AT_SYMLINK_NOFOLLOW \| AT_NO_AUTOMOUNT,
	414	STATX_BASIC_STATS, (uptr)&bufx);
	415	statx_to_stat(&bufx, (struct stat *)buf);
	416	return res;
	417	# elif (defined(_LP64) \|\| SANITIZER_X32 \|\| \
	418	(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \
	419	!SANITIZER_SPARC
342	420	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
343	421	AT_SYMLINK_NOFOLLOW);
344		#elif SANITIZER_LINUX_USES_64BIT_SYSCALLS
345		# if SANITIZER_MIPS64
346		// For mips64, lstat syscall fills buffer in the format of kernel_stat
347		struct kernel_stat kbuf;
348		int res = internal_syscall(SYSCALL(lstat), path, &kbuf);
349		kernel_stat_to_stat(&kbuf, (struct stat *)buf);
	422	# else
	423	struct stat64 buf64;
	424	int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path,
	425	(uptr)&buf64, AT_SYMLINK_NOFOLLOW);
	426	stat64_to_stat(&buf64, (struct stat *)buf);
350	427	return res;
351		# else
352		return internal_syscall(SYSCALL(lstat), (uptr)path, (uptr)buf);
353		# endif
354		#else
	428	# endif
	429	# else
355	430	struct stat64 buf64;
356	431	int res = internal_syscall(SYSCALL(lstat64), path, &buf64);
357	432	stat64_to_stat(&buf64, (struct stat *)buf);
358	433	return res;
359		#endif
	434	# endif
360	435	}
361	436
362	437	uptr internal_fstat(fd_t fd, void *buf) {
...	...	@@ -367,9 +442,15 @@ uptr internal_fstat(fd_t fd, void *buf) {
367	442	int res = internal_syscall(SYSCALL(fstat), fd, &kbuf);
368	443	kernel_stat_to_stat(&kbuf, (struct stat *)buf);
369	444	return res;
370		# else
	445	# elif SANITIZER_LINUX && defined(__loongarch__)
	446	struct statx bufx;
	447	int res = internal_syscall(SYSCALL(statx), fd, "", AT_EMPTY_PATH,
	448	STATX_BASIC_STATS, (uptr)&bufx);
	449	statx_to_stat(&bufx, (struct stat *)buf);
	450	return res;
	451	# else
371	452	return internal_syscall(SYSCALL(fstat), fd, (uptr)buf);
372		# endif
	453	# endif
373	454	#else
374	455	struct stat64 buf64;
375	456	int res = internal_syscall(SYSCALL(fstat64), fd, &buf64);
...	...	@@ -390,7 +471,7 @@ uptr internal_dup(int oldfd) {
390	471	}
391	472
392	473	uptr internal_dup2(int oldfd, int newfd) {
393		#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
	474	# if SANITIZER_LINUX
394	475	return internal_syscall(SYSCALL(dup3), oldfd, newfd, 0);
395	476	#else
396	477	return internal_syscall(SYSCALL(dup2), oldfd, newfd);
...	...	@@ -398,7 +479,7 @@ uptr internal_dup2(int oldfd, int newfd) {
398	479	}
399	480
400	481	uptr internal_readlink(const char path, char buf, uptr bufsize) {
401		#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
	482	# if SANITIZER_LINUX
402	483	return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf,
403	484	bufsize);
404	485	#else
...	...	@@ -407,7 +488,7 @@ uptr internal_readlink(const char path, char buf, uptr bufsize) {
407	488	}
408	489
409	490	uptr internal_unlink(const char *path) {
410		#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
	491	# if SANITIZER_LINUX
411	492	return internal_syscall(SYSCALL(unlinkat), AT_FDCWD, (uptr)path, 0);
412	493	#else
413	494	return internal_syscall(SYSCALL(unlink), (uptr)path);
...	...	@@ -415,15 +496,15 @@ uptr internal_unlink(const char *path) {
415	496	}
416	497
417	498	uptr internal_rename(const char oldpath, const char newpath) {
418		#if defined(__riscv)
	499	# if (defined(__riscv) \|\| defined(__loongarch__)) && defined(__linux__)
419	500	return internal_syscall(SYSCALL(renameat2), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
420	501	(uptr)newpath, 0);
421		#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
	502	# elif SANITIZER_LINUX
422	503	return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
423	504	(uptr)newpath);
424		#else
	505	# else
425	506	return internal_syscall(SYSCALL(rename), (uptr)oldpath, (uptr)newpath);
426		#endif
	507	# endif
427	508	}
428	509
429	510	uptr internal_sched_yield() {
...	...	@@ -460,17 +541,20 @@ bool FileExists(const char *filename) {
460	541	if (ShouldMockFailureToOpen(filename))
461	542	return false;
462	543	struct stat st;
463		#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
464		if (internal_syscall(SYSCALL(newfstatat), AT_FDCWD, filename, &st, 0))
465		#else
466	544	if (internal_stat(filename, &st))
467		#endif
468	545	return false;
469	546	// Sanity check: filename is a regular file.
470	547	return S_ISREG(st.st_mode);
471	548	}
472	549
473		#if !SANITIZER_NETBSD
	550	bool DirExists(const char *path) {
	551	struct stat st;
	552	if (internal_stat(path, &st))
	553	return false;
	554	return S_ISDIR(st.st_mode);
	555	}
	556
	557	# if !SANITIZER_NETBSD
474	558	tid_t GetTid() {
475	559	#if SANITIZER_FREEBSD
476	560	long Tid;
...	...	@@ -659,48 +743,6 @@ void FutexWake(atomic_uint32_t *p, u32 count) {
659	743	# endif
660	744	}
661	745
662		enum { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 };
663
664		BlockingMutex::BlockingMutex() {
665		internal_memset(this, 0, sizeof(*this));
666		}
667
668		void BlockingMutex::Lock() {
669		CHECK_EQ(owner_, 0);
670		atomic_uint32_t m = reinterpret_cast<atomic_uint32_t >(&opaque_storage_);
671		if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked)
672		return;
673		while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) {
674		#if SANITIZER_FREEBSD
675		_umtx_op(m, UMTX_OP_WAIT_UINT, MtxSleeping, 0, 0);
676		#elif SANITIZER_NETBSD
677		sched_yield(); /* No userspace futex-like synchronization */
678		#else
679		internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAIT_PRIVATE, MtxSleeping,
680		0, 0, 0);
681		#endif
682		}
683		}
684
685		void BlockingMutex::Unlock() {
686		atomic_uint32_t m = reinterpret_cast<atomic_uint32_t >(&opaque_storage_);
687		u32 v = atomic_exchange(m, MtxUnlocked, memory_order_release);
688		CHECK_NE(v, MtxUnlocked);
689		if (v == MtxSleeping) {
690		#if SANITIZER_FREEBSD
691		_umtx_op(m, UMTX_OP_WAKE, 1, 0, 0);
692		#elif SANITIZER_NETBSD
693		/* No userspace futex-like synchronization */
694		#else
695		internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAKE_PRIVATE, 1, 0, 0, 0);
696		#endif
697		}
698		}
699
700		void BlockingMutex::CheckLocked() const {
701		auto m = reinterpret_cast<atomic_uint32_t const *>(&opaque_storage_);
702		CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
703		}
704	746	# endif // !SANITIZER_SOLARIS
705	747
706	748	// ----------------- sanitizer_linux.h
...	...	@@ -711,17 +753,17 @@ void BlockingMutex::CheckLocked() const {
711	753	// Not used
712	754	#else
713	755	struct linux_dirent {
714		#if SANITIZER_X32 \|\| defined(__aarch64__) \|\| SANITIZER_RISCV64
	756	# if SANITIZER_X32 \|\| SANITIZER_LINUX
715	757	u64 d_ino;
716	758	u64 d_off;
717		#else
	759	# else
718	760	unsigned long d_ino;
719	761	unsigned long d_off;
720		#endif
	762	# endif
721	763	unsigned short d_reclen;
722		#if defined(__aarch64__) \|\| SANITIZER_RISCV64
	764	# if SANITIZER_LINUX
723	765	unsigned char d_type;
724		#endif
	766	# endif
725	767	char d_name[256];
726	768	};
727	769	#endif
...	...	@@ -757,11 +799,11 @@ int internal_dlinfo(void handle, int request, void p) {
757	799	uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count) {
758	800	#if SANITIZER_FREEBSD
759	801	return internal_syscall(SYSCALL(getdirentries), fd, (uptr)dirp, count, NULL);
760		#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
	802	# elif SANITIZER_LINUX
761	803	return internal_syscall(SYSCALL(getdents64), fd, (uptr)dirp, count);
762		#else
	804	# else
763	805	return internal_syscall(SYSCALL(getdents), fd, (uptr)dirp, count);
764		#endif
	806	# endif
765	807	}
766	808
767	809	uptr internal_lseek(fd_t fd, OFF_T offset, int whence) {
...	...	@@ -772,18 +814,29 @@ uptr internal_lseek(fd_t fd, OFF_T offset, int whence) {
772	814	uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) {
773	815	return internal_syscall(SYSCALL(prctl), option, arg2, arg3, arg4, arg5);
774	816	}
775		#endif
	817	# if defined(__x86_64__)
	818	# include <asm/unistd_64.h>
	819	// Currently internal_arch_prctl() is only needed on x86_64.
	820	uptr internal_arch_prctl(int option, uptr arg2) {
	821	return internal_syscall(__NR_arch_prctl, option, arg2);
	822	}
	823	# endif
	824	# endif
776	825
777	826	uptr internal_sigaltstack(const void ss, void oss) {
778	827	return internal_syscall(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss);
779	828	}
780	829
781	830	int internal_fork() {
782		#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
	831	# if SANITIZER_LINUX
	832	# if SANITIZER_S390
	833	return internal_syscall(SYSCALL(clone), 0, SIGCHLD);
	834	# else
783	835	return internal_syscall(SYSCALL(clone), SIGCHLD, 0);
784		#else
	836	# endif
	837	# else
785	838	return internal_syscall(SYSCALL(fork));
786		#endif
	839	# endif
787	840	}
788	841
789	842	#if SANITIZER_FREEBSD
...	...	@@ -911,6 +964,10 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {
911	964	return k_set->sig[idx] & ((uptr)1 << bit);
912	965	}
913	966	#elif SANITIZER_FREEBSD
	967	uptr internal_procctl(int type, int id, int cmd, void *data) {
	968	return internal_syscall(SYSCALL(procctl), type, id, cmd, data);
	969	}
	970
914	971	void internal_sigdelset(__sanitizer_sigset_t *set, int signum) {
915	972	sigset_t rset = reinterpret_cast<sigset_t >(set);
916	973	sigdelset(rset, signum);
...	...	@@ -1052,7 +1109,7 @@ uptr GetMaxVirtualAddress() {
1052	1109	#if SANITIZER_NETBSD && defined(__x86_64__)
1053	1110	return 0x7f7ffffff000ULL; // (0x00007f8000000000 - PAGE_SIZE)
1054	1111	#elif SANITIZER_WORDSIZE == 64
1055		# if defined(__powerpc64__) \|\| defined(__aarch64__)
	1112	# if defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__loongarch__)
1056	1113	// On PowerPC64 we have two different address space layouts: 44- and 46-bit.
1057	1114	// We somehow need to figure out which one we are using now and choose
1058	1115	// one of 0x00000fffffffffffUL and 0x00003fffffffffffUL.
...	...	@@ -1060,10 +1117,11 @@ uptr GetMaxVirtualAddress() {
1060	1117	// of the address space, so simply checking the stack address is not enough.
1061	1118	// This should (does) work for both PowerPC64 Endian modes.
1062	1119	// Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit.
	1120	// loongarch64 also has multiple address space layouts: default is 47-bit.
1063	1121	return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;
1064	1122	#elif SANITIZER_RISCV64
1065	1123	return (1ULL << 38) - 1;
1066		# elif defined(__mips64)
	1124	# elif SANITIZER_MIPS64
1067	1125	return (1ULL << 40) - 1; // 0x000000ffffffffffUL;
1068	1126	# elif defined(__s390x__)
1069	1127	return (1ULL << 53) - 1; // 0x001fffffffffffffUL;
...	...	@@ -1217,7 +1275,8 @@ void ForEachMappedRegion(link_map map, void (cb)(const void *, uptr)) {
1217	1275	}
1218	1276	#endif
1219	1277
1220		#if defined(__x86_64__) && SANITIZER_LINUX
	1278	#if SANITIZER_LINUX
	1279	#if defined(__x86_64__)
1221	1280	// We cannot use glibc's clone wrapper, because it messes with the child
1222	1281	// task's TLS. It writes the PID and TID of the child task to its thread
1223	1282	// descriptor, but in our case the child task shares the thread descriptor with
...	...	@@ -1399,7 +1458,7 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1399	1458	#elif defined(__aarch64__)
1400	1459	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1401	1460	int parent_tidptr, void newtls, int *child_tidptr) {
1402		long long res;
	1461	register long long res __asm__("x0");
1403	1462	if (!fn \|\| !child_stack)
1404	1463	return -EINVAL;
1405	1464	CHECK_EQ(0, (uptr)child_stack % 16);
...	...	@@ -1447,6 +1506,47 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1447	1506	: "x30", "memory");
1448	1507	return res;
1449	1508	}
	1509	#elif SANITIZER_LOONGARCH64
	1510	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
	1511	int parent_tidptr, void newtls, int *child_tidptr) {
	1512	if (!fn \|\| !child_stack)
	1513	return -EINVAL;
	1514
	1515	CHECK_EQ(0, (uptr)child_stack % 16);
	1516
	1517	register int res __asm__("$a0");
	1518	register int __flags __asm__("$a0") = flags;
	1519	register void *__stack __asm__("$a1") = child_stack;
	1520	register int *__ptid __asm__("$a2") = parent_tidptr;
	1521	register int *__ctid __asm__("$a3") = child_tidptr;
	1522	register void *__tls __asm__("$a4") = newtls;
	1523	register int (__fn)(void ) __asm__("$a5") = fn;
	1524	register void *__arg __asm__("$a6") = arg;
	1525	register int nr_clone __asm__("$a7") = __NR_clone;
	1526
	1527	__asm__ __volatile__(
	1528	"syscall 0\n"
	1529
	1530	// if ($a0 != 0)
	1531	// return $a0;
	1532	"bnez $a0, 1f\n"
	1533
	1534	// In the child, now. Call "fn(arg)".
	1535	"move $a0, $a6\n"
	1536	"jirl $ra, $a5, 0\n"
	1537
	1538	// Call _exit($a0).
	1539	"addi.d $a7, $zero, %9\n"
	1540	"syscall 0\n"
	1541
	1542	"1:\n"
	1543
	1544	: "=r"(res)
	1545	: "0"(__flags), "r"(__stack), "r"(__ptid), "r"(__ctid), "r"(__tls),
	1546	"r"(__fn), "r"(__arg), "r"(nr_clone), "i"(__NR_exit)
	1547	: "memory", "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", "$t8");
	1548	return res;
	1549	}
1450	1550	#elif defined(__powerpc64__)
1451	1551	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1452	1552	int parent_tidptr, void newtls, int *child_tidptr) {
...	...	@@ -1556,7 +1656,7 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1556	1656	: "cr0", "cr1", "memory", "ctr", "r0", "r27", "r28", "r29");
1557	1657	return res;
1558	1658	}
1559		#elif defined(__i386__) && SANITIZER_LINUX
	1659	#elif defined(__i386__)
1560	1660	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1561	1661	int parent_tidptr, void newtls, int *child_tidptr) {
1562	1662	int res;
...	...	@@ -1621,7 +1721,7 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1621	1721	: "memory");
1622	1722	return res;
1623	1723	}
1624		#elif defined(__arm__) && SANITIZER_LINUX
	1724	#elif defined(__arm__)
1625	1725	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1626	1726	int parent_tidptr, void newtls, int *child_tidptr) {
1627	1727	unsigned int res;
...	...	@@ -1687,7 +1787,8 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1687	1787	: "memory");
1688	1788	return res;
1689	1789	}
1690		#endif // defined(__x86_64__) && SANITIZER_LINUX
	1790	#endif
	1791	#endif // SANITIZER_LINUX
1691	1792
1692	1793	#if SANITIZER_LINUX
1693	1794	int internal_uname(struct utsname *buf) {
...	...	@@ -1778,23 +1879,18 @@ HandleSignalMode GetHandleSignalMode(int signum) {
1778	1879
1779	1880	#if !SANITIZER_GO
1780	1881	void internal_start_thread(void (func)(void arg), void *arg) {
	1882	if (&real_pthread_create == 0)
	1883	return nullptr;
1781	1884	// Start the thread with signals blocked, otherwise it can steal user signals.
1782		__sanitizer_sigset_t set, old;
1783		internal_sigfillset(&set);
1784		#if SANITIZER_LINUX && !SANITIZER_ANDROID
1785		// Glibc uses SIGSETXID signal during setuid call. If this signal is blocked
1786		// on any thread, setuid call hangs (see test/tsan/setuid.c).
1787		internal_sigdelset(&set, 33);
1788		#endif
1789		internal_sigprocmask(SIG_SETMASK, &set, &old);
	1885	ScopedBlockSignals block(nullptr);
1790	1886	void *th;
1791	1887	real_pthread_create(&th, nullptr, func, arg);
1792		internal_sigprocmask(SIG_SETMASK, &old, nullptr);
1793	1888	return th;
1794	1889	}
1795	1890
1796	1891	void internal_join_thread(void *th) {
1797		real_pthread_join(th, nullptr);
	1892	if (&real_pthread_join)
	1893	real_pthread_join(th, nullptr);
1798	1894	}
1799	1895	#else
1800	1896	void internal_start_thread(void (func)(void ), void *arg) { return 0; }
...	...	@@ -1802,7 +1898,7 @@ void internal_start_thread(void (func)(void ), void *arg) { return 0; }
1802	1898	void internal_join_thread(void *th) {}
1803	1899	#endif
1804	1900
1805		#if defined(__aarch64__)
	1901	#if SANITIZER_LINUX && defined(__aarch64__)
1806	1902	// Android headers in the older NDK releases miss this definition.
1807	1903	struct __sanitizer_esr_context {
1808	1904	struct _aarch64_ctx head;
...	...	@@ -1811,7 +1907,7 @@ struct __sanitizer_esr_context {
1811	1907
1812	1908	static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {
1813	1909	static const u32 kEsrMagic = 0x45535201;
1814		u8 *aux = ucontext->uc_mcontext.__reserved;
	1910	u8 aux = reinterpret_cast<u8 >(ucontext->uc_mcontext.__reserved);
1815	1911	while (true) {
1816	1912	_aarch64_ctx ctx = (_aarch64_ctx )aux;
1817	1913	if (ctx->size == 0) break;
...	...	@@ -1823,6 +1919,11 @@ static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {
1823	1919	}
1824	1920	return false;
1825	1921	}
	1922	#elif SANITIZER_FREEBSD && defined(__aarch64__)
	1923	// FreeBSD doesn't provide ESR in the ucontext.
	1924	static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {
	1925	return false;
	1926	}
1826	1927	#endif
1827	1928
1828	1929	using Context = ucontext_t;
...	...	@@ -1841,7 +1942,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1841	1942	#else
1842	1943	uptr err = ucontext->uc_mcontext.gregs[REG_ERR];
1843	1944	#endif // SANITIZER_FREEBSD
1844		return err & PF_WRITE ? WRITE : READ;
	1945	return err & PF_WRITE ? Write : Read;
1845	1946	#elif defined(__mips__)
1846	1947	uint32_t *exception_source;
1847	1948	uint32_t faulty_instruction;
...	...	@@ -1864,7 +1965,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1864	1965	case 0x2a: // swl
1865	1966	case 0x2e: // swr
1866	1967	#endif
1867		return SignalContext::WRITE;
	1968	return SignalContext::Write;
1868	1969
1869	1970	case 0x20: // lb
1870	1971	case 0x24: // lbu
...	...	@@ -1879,27 +1980,34 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1879	1980	case 0x22: // lwl
1880	1981	case 0x26: // lwr
1881	1982	#endif
1882		return SignalContext::READ;
	1983	return SignalContext::Read;
1883	1984	#if __mips_isa_rev == 6
1884	1985	case 0x3b: // pcrel
1885	1986	op_code = (faulty_instruction >> 19) & 0x3;
1886	1987	switch (op_code) {
1887	1988	case 0x1: // lwpc
1888	1989	case 0x2: // lwupc
1889		return SignalContext::READ;
	1990	return SignalContext::Read;
1890	1991	}
1891	1992	#endif
1892	1993	}
1893		return SignalContext::UNKNOWN;
	1994	return SignalContext::Unknown;
1894	1995	#elif defined(__arm__)
1895	1996	static const uptr FSR_WRITE = 1U << 11;
1896	1997	uptr fsr = ucontext->uc_mcontext.error_code;
1897		return fsr & FSR_WRITE ? WRITE : READ;
	1998	return fsr & FSR_WRITE ? Write : Read;
1898	1999	#elif defined(__aarch64__)
1899	2000	static const u64 ESR_ELx_WNR = 1U << 6;
1900	2001	u64 esr;
1901		if (!Aarch64GetESR(ucontext, &esr)) return UNKNOWN;
1902		return esr & ESR_ELx_WNR ? WRITE : READ;
	2002	if (!Aarch64GetESR(ucontext, &esr)) return Unknown;
	2003	return esr & ESR_ELx_WNR ? Write : Read;
	2004	#elif defined(__loongarch__)
	2005	u32 flags = ucontext->uc_mcontext.__flags;
	2006	if (flags & SC_ADDRERR_RD)
	2007	return SignalContext::Read;
	2008	if (flags & SC_ADDRERR_WR)
	2009	return SignalContext::Write;
	2010	return SignalContext::Unknown;
1903	2011	#elif defined(__sparc__)
1904	2012	// Decode the instruction to determine the access type.
1905	2013	// From OpenSolaris $SRC/uts/sun4/os/trap.c (get_accesstype).
...	...	@@ -1915,9 +2023,13 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1915	2023	#endif
1916	2024	#endif
1917	2025	u32 instr = (u32 )pc;
1918		return (instr >> 21) & 1 ? WRITE: READ;
	2026	return (instr >> 21) & 1 ? Write: Read;
1919	2027	#elif defined(__riscv)
	2028	#if SANITIZER_FREEBSD
	2029	unsigned long pc = ucontext->uc_mcontext.mc_gpregs.gp_sepc;
	2030	#else
1920	2031	unsigned long pc = ucontext->uc_mcontext.__gregs[REG_PC];
	2032	#endif
1921	2033	unsigned faulty_instruction = (uint16_t )pc;
1922	2034
1923	2035	#if defined(__riscv_compressed)
...	...	@@ -1931,7 +2043,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1931	2043	#if __riscv_xlen == 64
1932	2044	case 0b10'011: // c.ldsp (rd != x0)
1933	2045	#endif
1934		return rd ? SignalContext::READ : SignalContext::UNKNOWN;
	2046	return rd ? SignalContext::Read : SignalContext::Unknown;
1935	2047	case 0b00'010: // c.lw
1936	2048	#if __riscv_flen >= 32 && __riscv_xlen == 32
1937	2049	case 0b10'011: // c.flwsp
...	...	@@ -1943,7 +2055,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1943	2055	case 0b00'001: // c.fld
1944	2056	case 0b10'001: // c.fldsp
1945	2057	#endif
1946		return SignalContext::READ;
	2058	return SignalContext::Read;
1947	2059	case 0b00'110: // c.sw
1948	2060	case 0b10'110: // c.swsp
1949	2061	#if __riscv_flen >= 32 \|\| __riscv_xlen == 64
...	...	@@ -1954,9 +2066,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1954	2066	case 0b00'101: // c.fsd
1955	2067	case 0b10'101: // c.fsdsp
1956	2068	#endif
1957		return SignalContext::WRITE;
	2069	return SignalContext::Write;
1958	2070	default:
1959		return SignalContext::UNKNOWN;
	2071	return SignalContext::Unknown;
1960	2072	}
1961	2073	}
1962	2074	#endif
...	...	@@ -1974,9 +2086,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1974	2086	#endif
1975	2087	case 0b100: // lbu
1976	2088	case 0b101: // lhu
1977		return SignalContext::READ;
	2089	return SignalContext::Read;
1978	2090	default:
1979		return SignalContext::UNKNOWN;
	2091	return SignalContext::Unknown;
1980	2092	}
1981	2093	case 0b0100011: // stores
1982	2094	switch (funct3) {
...	...	@@ -1986,9 +2098,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1986	2098	#if __riscv_xlen == 64
1987	2099	case 0b011: // sd
1988	2100	#endif
1989		return SignalContext::WRITE;
	2101	return SignalContext::Write;
1990	2102	default:
1991		return SignalContext::UNKNOWN;
	2103	return SignalContext::Unknown;
1992	2104	}
1993	2105	#if __riscv_flen >= 32
1994	2106	case 0b0000111: // floating-point loads
...	...	@@ -1997,9 +2109,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1997	2109	#if __riscv_flen == 64
1998	2110	case 0b011: // fld
1999	2111	#endif
2000		return SignalContext::READ;
	2112	return SignalContext::Read;
2001	2113	default:
2002		return SignalContext::UNKNOWN;
	2114	return SignalContext::Unknown;
2003	2115	}
2004	2116	case 0b0100111: // floating-point stores
2005	2117	switch (funct3) {
...	...	@@ -2007,17 +2119,17 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
2007	2119	#if __riscv_flen == 64
2008	2120	case 0b011: // fsd
2009	2121	#endif
2010		return SignalContext::WRITE;
	2122	return SignalContext::Write;
2011	2123	default:
2012		return SignalContext::UNKNOWN;
	2124	return SignalContext::Unknown;
2013	2125	}
2014	2126	#endif
2015	2127	default:
2016		return SignalContext::UNKNOWN;
	2128	return SignalContext::Unknown;
2017	2129	}
2018	2130	#else
2019	2131	(void)ucontext;
2020		return UNKNOWN; // FIXME: Implement.
	2132	return Unknown; // FIXME: Implement.
2021	2133	#endif
2022	2134	}
2023	2135
...	...	@@ -2044,10 +2156,17 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2044	2156	*bp = ucontext->uc_mcontext.arm_fp;
2045	2157	*sp = ucontext->uc_mcontext.arm_sp;
2046	2158	#elif defined(__aarch64__)
	2159	# if SANITIZER_FREEBSD
	2160	ucontext_t ucontext = (ucontext_t)context;
	2161	*pc = ucontext->uc_mcontext.mc_gpregs.gp_elr;
	2162	*bp = ucontext->uc_mcontext.mc_gpregs.gp_x[29];
	2163	*sp = ucontext->uc_mcontext.mc_gpregs.gp_sp;
	2164	# else
2047	2165	ucontext_t ucontext = (ucontext_t)context;
2048	2166	*pc = ucontext->uc_mcontext.pc;
2049	2167	*bp = ucontext->uc_mcontext.regs[29];
2050	2168	*sp = ucontext->uc_mcontext.sp;
	2169	# endif
2051	2170	#elif defined(__hppa__)
2052	2171	ucontext_t ucontext = (ucontext_t)context;
2053	2172	*pc = ucontext->uc_mcontext.sc_iaoq[0];
...	...	@@ -2092,12 +2211,19 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2092	2211	*sp = ucontext->uc_mcontext.gregs[REG_UESP];
2093	2212	# endif
2094	2213	#elif defined(__powerpc__) \|\| defined(__powerpc64__)
	2214	# if SANITIZER_FREEBSD
	2215	ucontext_t ucontext = (ucontext_t )context;
	2216	*pc = ucontext->uc_mcontext.mc_srr0;
	2217	*sp = ucontext->uc_mcontext.mc_frame[1];
	2218	*bp = ucontext->uc_mcontext.mc_frame[31];
	2219	# else
2095	2220	ucontext_t ucontext = (ucontext_t)context;
2096	2221	*pc = ucontext->uc_mcontext.regs->nip;
2097	2222	*sp = ucontext->uc_mcontext.regs->gpr[PT_R1];
2098	2223	// The powerpc{,64}-linux ABIs do not specify r31 as the frame
2099	2224	// pointer, but GCC always uses r31 when we need a frame pointer.
2100	2225	*bp = ucontext->uc_mcontext.regs->gpr[PT_R31];
	2226	# endif
2101	2227	#elif defined(__sparc__)
2102	2228	#if defined(__arch64__) \|\| defined(__sparcv9)
2103	2229	#define STACK_BIAS 2047
...	...	@@ -2136,12 +2262,28 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2136	2262	*sp = ucontext->uc_mcontext.gregs[15];
2137	2263	#elif defined(__riscv)
2138	2264	ucontext_t ucontext = (ucontext_t)context;
	2265	# if SANITIZER_FREEBSD
	2266	*pc = ucontext->uc_mcontext.mc_gpregs.gp_sepc;
	2267	*bp = ucontext->uc_mcontext.mc_gpregs.gp_s[0];
	2268	*sp = ucontext->uc_mcontext.mc_gpregs.gp_sp;
	2269	# else
2139	2270	*pc = ucontext->uc_mcontext.__gregs[REG_PC];
2140	2271	*bp = ucontext->uc_mcontext.__gregs[REG_S0];
2141	2272	*sp = ucontext->uc_mcontext.__gregs[REG_SP];
2142		#else
2143		# error "Unsupported arch"
2144		#endif
	2273	# endif
	2274	# elif defined(__hexagon__)
	2275	ucontext_t ucontext = (ucontext_t )context;
	2276	*pc = ucontext->uc_mcontext.pc;
	2277	*bp = ucontext->uc_mcontext.r30;
	2278	*sp = ucontext->uc_mcontext.r29;
	2279	# elif defined(__loongarch__)
	2280	ucontext_t ucontext = (ucontext_t )context;
	2281	*pc = ucontext->uc_mcontext.__pc;
	2282	*bp = ucontext->uc_mcontext.__gregs[22];
	2283	*sp = ucontext->uc_mcontext.__gregs[3];
	2284	# else
	2285	# error "Unsupported arch"
	2286	# endif
2145	2287	}
2146	2288
2147	2289	void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); }
...	...	@@ -2150,10 +2292,6 @@ void InitializePlatformEarly() {
2150	2292	// Do nothing.
2151	2293	}
2152	2294
2153		void MaybeReexec() {
2154		// No need to re-exec on Linux.
2155		}
2156
2157	2295	void CheckASLR() {
2158	2296	#if SANITIZER_NETBSD
2159	2297	int mib[3];
...	...	@@ -2175,49 +2313,35 @@ void CheckASLR() {
2175	2313	GetArgv()[0]);
2176	2314	Die();
2177	2315	}
2178		#elif SANITIZER_PPC64V2
2179		// Disable ASLR for Linux PPC64LE.
2180		int old_personality = personality(0xffffffff);
2181		if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) {
2182		VReport(1, "WARNING: Program is being run with address space layout "
2183		"randomization (ASLR) enabled which prevents the thread and "
2184		"memory sanitizers from working on powerpc64le.\n"
2185		"ASLR will be disabled and the program re-executed.\n");
2186		CHECK_NE(personality(old_personality \| ADDR_NO_RANDOMIZE), -1);
2187		ReExec();
2188		}
2189	2316	#elif SANITIZER_FREEBSD
2190		int aslr_pie;
2191		uptr len = sizeof(aslr_pie);
2192		#if SANITIZER_WORDSIZE == 64
2193		if (UNLIKELY(internal_sysctlbyname("kern.elf64.aslr.pie_enable",
2194		&aslr_pie, &len, NULL, 0) == -1)) {
	2317	int aslr_status;
	2318	int r = internal_procctl(P_PID, 0, PROC_ASLR_STATUS, &aslr_status);
	2319	if (UNLIKELY(r == -1)) {
2195	2320	// We're making things less 'dramatic' here since
2196		// the OID is not necessarily guaranteed to be here
	2321	// the cmd is not necessarily guaranteed to be here
2197	2322	// just yet regarding FreeBSD release
2198	2323	return;
2199	2324	}
2200
2201		if (aslr_pie > 0) {
	2325	if ((aslr_status & PROC_ASLR_ACTIVE) != 0) {
2202	2326	Printf("This sanitizer is not compatible with enabled ASLR "
2203	2327	"and binaries compiled with PIE\n");
2204	2328	Die();
2205	2329	}
2206		#endif
2207		// there might be 32 bits compat for 64 bits
2208		if (UNLIKELY(internal_sysctlbyname("kern.elf32.aslr.pie_enable",
2209		&aslr_pie, &len, NULL, 0) == -1)) {
2210		return;
2211		}
2212
2213		if (aslr_pie > 0) {
2214		Printf("This sanitizer is not compatible with enabled ASLR "
2215		"and binaries compiled with PIE\n");
2216		Die();
	2330	# elif SANITIZER_PPC64V2
	2331	// Disable ASLR for Linux PPC64LE.
	2332	int old_personality = personality(0xffffffff);
	2333	if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) {
	2334	VReport(1,
	2335	"WARNING: Program is being run with address space layout "
	2336	"randomization (ASLR) enabled which prevents the thread and "
	2337	"memory sanitizers from working on powerpc64le.\n"
	2338	"ASLR will be disabled and the program re-executed.\n");
	2339	CHECK_NE(personality(old_personality \| ADDR_NO_RANDOMIZE), -1);
	2340	ReExec();
2217	2341	}
2218		#else
	2342	# else
2219	2343	// Do nothing
2220		#endif
	2344	# endif
2221	2345	}
2222	2346
2223	2347	void CheckMPROTECT() {

lib/tsan/sanitizer_common/sanitizer_linux.h+25-4

...	...	@@ -49,26 +49,44 @@ uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count);
49	49	uptr internal_sigaltstack(const void* ss, void* oss);
50	50	uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
51	51	__sanitizer_sigset_t *oldset);
52		#if SANITIZER_GLIBC
	52
	53	void SetSigProcMask(__sanitizer_sigset_t set, __sanitizer_sigset_t oldset);
	54	void BlockSignals(__sanitizer_sigset_t *oldset = nullptr);
	55	struct ScopedBlockSignals {
	56	explicit ScopedBlockSignals(__sanitizer_sigset_t *copy);
	57	~ScopedBlockSignals();
	58
	59	ScopedBlockSignals &operator=(const ScopedBlockSignals &) = delete;
	60	ScopedBlockSignals(const ScopedBlockSignals &) = delete;
	61
	62	private:
	63	__sanitizer_sigset_t saved_;
	64	};
	65
	66	# if SANITIZER_GLIBC
53	67	uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp);
54	68	#endif
55	69
56	70	// Linux-only syscalls.
57	71	#if SANITIZER_LINUX
58	72	uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5);
	73	# if defined(__x86_64__)
	74	uptr internal_arch_prctl(int option, uptr arg2);
	75	# endif
59	76	// Used only by sanitizer_stoptheworld. Signal handlers that are actually used
60	77	// (like the process-wide error reporting SEGV handler) must use
61	78	// internal_sigaction instead.
62	79	int internal_sigaction_norestorer(int signum, const void act, void oldact);
63	80	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
64		#if defined(__x86_64__) \|\| defined(__mips__) \|\| defined(__aarch64__) \|\| \
65		defined(__powerpc64__) \|\| defined(__s390__) \|\| defined(__i386__) \|\| \
66		defined(__arm__) \|\| SANITIZER_RISCV64
	81	# if defined(__x86_64__) \|\| defined(__mips__) \|\| defined(__aarch64__) \|\| \
	82	defined(__powerpc64__) \|\| defined(__s390__) \|\| defined(__i386__) \|\| \
	83	defined(__arm__) \|\| SANITIZER_RISCV64 \|\| SANITIZER_LOONGARCH64
67	84	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
68	85	int parent_tidptr, void newtls, int *child_tidptr);
69	86	#endif
70	87	int internal_uname(struct utsname *buf);
71	88	#elif SANITIZER_FREEBSD
	89	uptr internal_procctl(int type, int id, int cmd, void *data);
72	90	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
73	91	#elif SANITIZER_NETBSD
74	92	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
...	...	@@ -135,6 +153,9 @@ inline void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) {
135	153	"rdhwr %0,$29\n" \
136	154	".set pop\n" : "=r"(__v)); \
137	155	__v; })
	156	#elif defined (__riscv)
	157	# define __get_tls() \
	158	({ void** __v; __asm__("mv %0, tp" : "=r"(__v)); __v; })
138	159	#elif defined(__i386__)
139	160	# define __get_tls() \
140	161	({ void** __v; __asm__("movl %%gs:0, %0" : "=r"(__v)); __v; })

lib/tsan/sanitizer_common/sanitizer_linux_libcdep.cpp+109-30

...	...	@@ -27,6 +27,7 @@
27	27	#include "sanitizer_linux.h"
28	28	#include "sanitizer_placement_new.h"
29	29	#include "sanitizer_procmaps.h"
	30	#include "sanitizer_solaris.h"
30	31
31	32	#if SANITIZER_NETBSD
32	33	#define _RTLD_SOURCE // for __lwp_gettcb_fast() / __lwp_getprivate_fast()
...	...	@@ -62,6 +63,7 @@
62	63	#endif
63	64
64	65	#if SANITIZER_SOLARIS
	66	#include <stddef.h>
65	67	#include <stdlib.h>
66	68	#include <thread.h>
67	69	#endif
...	...	@@ -146,7 +148,7 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
146	148	pthread_attr_t attr;
147	149	pthread_attr_init(&attr);
148	150	CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
149		my_pthread_attr_getstack(&attr, &stackaddr, &stacksize);
	151	internal_pthread_attr_getstack(&attr, &stackaddr, &stacksize);
150	152	pthread_attr_destroy(&attr);
151	153	#endif // SANITIZER_SOLARIS
152	154
...	...	@@ -203,7 +205,8 @@ void InitTlsSize() {
203	205	g_use_dlpi_tls_data =
204	206	GetLibcVersion(&major, &minor, &patch) && major == 2 && minor >= 25;
205	207
206		#if defined(__aarch64__) \|\| defined(__x86_64__) \|\| defined(__powerpc64__)
	208	#if defined(__aarch64__) \|\| defined(__x86_64__) \|\| defined(__powerpc64__) \|\| \
	209	defined(__loongarch__)
207	210	void *get_tls_static_info = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");
208	211	size_t tls_align;
209	212	((void ()(size_t , size_t *))get_tls_static_info)(&g_tls_size, &tls_align);
...	...	@@ -216,14 +219,13 @@ void InitTlsSize() { }
216	219	// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage
217	220	// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan
218	221	// to get the pointer to thread-specific data keys in the thread control block.
219		#if (SANITIZER_FREEBSD \|\| SANITIZER_LINUX) && !SANITIZER_ANDROID
	222	#if (SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_SOLARIS) && \
	223	!SANITIZER_ANDROID && !SANITIZER_GO
220	224	// sizeof(struct pthread) from glibc.
221	225	static atomic_uintptr_t thread_descriptor_size;
222	226
223		uptr ThreadDescriptorSize() {
224		uptr val = atomic_load_relaxed(&thread_descriptor_size);
225		if (val)
226		return val;
	227	static uptr ThreadDescriptorSizeFallback() {
	228	uptr val = 0;
227	229	#if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__arm__)
228	230	int major;
229	231	int minor;
...	...	@@ -264,6 +266,8 @@ uptr ThreadDescriptorSize() {
264	266	#elif defined(__mips__)
265	267	// TODO(sagarthakur): add more values as per different glibc versions.
266	268	val = FIRST_32_SECOND_64(1152, 1776);
	269	#elif SANITIZER_LOONGARCH64
	270	val = 1856; // from glibc 2.36
267	271	#elif SANITIZER_RISCV64
268	272	int major;
269	273	int minor;
...	...	@@ -285,12 +289,26 @@ uptr ThreadDescriptorSize() {
285	289	#elif defined(__powerpc64__)
286	290	val = 1776; // from glibc.ppc64le 2.20-8.fc21
287	291	#endif
	292	return val;
	293	}
	294
	295	uptr ThreadDescriptorSize() {
	296	uptr val = atomic_load_relaxed(&thread_descriptor_size);
288	297	if (val)
289		atomic_store_relaxed(&thread_descriptor_size, val);
	298	return val;
	299	// _thread_db_sizeof_pthread is a GLIBC_PRIVATE symbol that is exported in
	300	// glibc 2.34 and later.
	301	if (unsigned psizeof = static_cast<unsigned >(
	302	dlsym(RTLD_DEFAULT, "_thread_db_sizeof_pthread")))
	303	val = *psizeof;
	304	if (!val)
	305	val = ThreadDescriptorSizeFallback();
	306	atomic_store_relaxed(&thread_descriptor_size, val);
290	307	return val;
291	308	}
292	309
293		#if defined(__mips__) \|\| defined(__powerpc64__) \|\| SANITIZER_RISCV64
	310	#if defined(__mips__) \|\| defined(__powerpc64__) \|\| SANITIZER_RISCV64 \|\| \
	311	SANITIZER_LOONGARCH64
294	312	// TlsPreTcbSize includes size of struct pthread_descr and size of tcb
295	313	// head structure. It lies before the static tls blocks.
296	314	static uptr TlsPreTcbSize() {
...	...	@@ -300,6 +318,8 @@ static uptr TlsPreTcbSize() {
300	318	const uptr kTcbHead = 88; // sizeof (tcbhead_t)
301	319	#elif SANITIZER_RISCV64
302	320	const uptr kTcbHead = 16; // sizeof (tcbhead_t)
	321	#elif SANITIZER_LOONGARCH64
	322	const uptr kTcbHead = 16; // sizeof (tcbhead_t)
303	323	#endif
304	324	const uptr kTlsAlign = 16;
305	325	const uptr kTlsPreTcbSize =
...	...	@@ -308,7 +328,6 @@ static uptr TlsPreTcbSize() {
308	328	}
309	329	#endif
310	330
311		#if !SANITIZER_GO
312	331	namespace {
313	332	struct TlsBlock {
314	333	uptr begin, end, align;
...	...	@@ -339,19 +358,43 @@ static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) {
339	358	extern "C" void __tls_get_addr(size_t );
340	359	#endif
341	360
	361	static size_t main_tls_modid;
	362
342	363	static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,
343	364	void *data) {
344		if (!info->dlpi_tls_modid)
	365	size_t tls_modid;
	366	#if SANITIZER_SOLARIS
	367	// dlpi_tls_modid is only available since Solaris 11.4 SRU 10. Use
	368	// dlinfo(RTLD_DI_LINKMAP) instead which works on all of Solaris 11.3,
	369	// 11.4, and Illumos. The tlsmodid of the executable was changed to 1 in
	370	// 11.4 to match other implementations.
	371	if (size >= offsetof(dl_phdr_info_test, dlpi_tls_modid))
	372	main_tls_modid = 1;
	373	else
	374	main_tls_modid = 0;
	375	g_use_dlpi_tls_data = 0;
	376	Rt_map *map;
	377	dlinfo(RTLD_SELF, RTLD_DI_LINKMAP, &map);
	378	tls_modid = map->rt_tlsmodid;
	379	#else
	380	main_tls_modid = 1;
	381	tls_modid = info->dlpi_tls_modid;
	382	#endif
	383
	384	if (tls_modid < main_tls_modid)
345	385	return 0;
346		uptr begin = (uptr)info->dlpi_tls_data;
	386	uptr begin;
	387	#if !SANITIZER_SOLARIS
	388	begin = (uptr)info->dlpi_tls_data;
	389	#endif
347	390	if (!g_use_dlpi_tls_data) {
348	391	// Call __tls_get_addr as a fallback. This forces TLS allocation on glibc
349	392	// and FreeBSD.
350	393	#ifdef __s390__
351	394	begin = (uptr)__builtin_thread_pointer() +
352		TlsGetOffset(info->dlpi_tls_modid, 0);
	395	TlsGetOffset(tls_modid, 0);
353	396	#else
354		size_t mod_and_off[2] = {info->dlpi_tls_modid, 0};
	397	size_t mod_and_off[2] = {tls_modid, 0};
355	398	begin = (uptr)__tls_get_addr(mod_and_off);
356	399	#endif
357	400	}
...	...	@@ -359,7 +402,7 @@ static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,
359	402	if (info->dlpi_phdr[i].p_type == PT_TLS) {
360	403	static_cast<InternalMmapVector<TlsBlock> *>(data)->push_back(
361	404	TlsBlock{begin, begin + info->dlpi_phdr[i].p_memsz,
362		info->dlpi_phdr[i].p_align, info->dlpi_tls_modid});
	405	info->dlpi_phdr[i].p_align, tls_modid});
363	406	break;
364	407	}
365	408	return 0;
...	...	@@ -371,11 +414,11 @@ __attribute__((unused)) static void GetStaticTlsBoundary(uptr addr, uptr size,
371	414	dl_iterate_phdr(CollectStaticTlsBlocks, &ranges);
372	415	uptr len = ranges.size();
373	416	Sort(ranges.begin(), len);
374		// Find the range with tls_modid=1. For glibc, because libc.so uses PT_TLS,
375		// this module is guaranteed to exist and is one of the initially loaded
376		// modules.
	417	// Find the range with tls_modid == main_tls_modid. For glibc, because
	418	// libc.so uses PT_TLS, this module is guaranteed to exist and is one of
	419	// the initially loaded modules.
377	420	uptr one = 0;
378		while (one != len && ranges[one].tls_modid != 1) ++one;
	421	while (one != len && ranges[one].tls_modid != main_tls_modid) ++one;
379	422	if (one == len) {
380	423	// This may happen with musl if no module uses PT_TLS.
381	424	*addr = 0;
...	...	@@ -384,21 +427,20 @@ __attribute__((unused)) static void GetStaticTlsBoundary(uptr addr, uptr size,
384	427	return;
385	428	}
386	429	// Find the maximum consecutive ranges. We consider two modules consecutive if
387		// the gap is smaller than the alignment. The dynamic loader places static TLS
388		// blocks this way not to waste space.
	430	// the gap is smaller than the alignment of the latter range. The dynamic
	431	// loader places static TLS blocks this way not to waste space.
389	432	uptr l = one;
390	433	*align = ranges[l].align;
391		while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l - 1].align)
	434	while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l].align)
392	435	align = Max(align, ranges[--l].align);
393	436	uptr r = one + 1;
394		while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r - 1].align)
	437	while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r].align)
395	438	align = Max(align, ranges[r++].align);
396	439	*addr = ranges[l].begin;
397	440	*size = ranges[r - 1].end - ranges[l].begin;
398	441	}
399		#endif // !SANITIZER_GO
400	442	#endif // (x86_64 \|\| i386 \|\| mips \|\| ...) && (SANITIZER_FREEBSD \|\|
401		// SANITIZER_LINUX) && !SANITIZER_ANDROID
	443	// SANITIZER_LINUX) && !SANITIZER_ANDROID && !SANITIZER_GO
402	444
403	445	#if SANITIZER_NETBSD
404	446	static struct tls_tcb * ThreadSelfTlsTcb() {
...	...	@@ -452,7 +494,11 @@ static void GetTls(uptr addr, uptr size) {
452	494	#elif SANITIZER_GLIBC && defined(__x86_64__)
453	495	// For aarch64 and x86-64, use an O(1) approach which requires relatively
454	496	// precise ThreadDescriptorSize. g_tls_size was initialized in InitTlsSize.
	497	# if SANITIZER_X32
	498	asm("mov %%fs:8,%0" : "=r"(*addr));
	499	# else
455	500	asm("mov %%fs:16,%0" : "=r"(*addr));
	501	# endif
456	502	*size = g_tls_size;
457	503	addr -= size;
458	504	*addr += ThreadDescriptorSize();
...	...	@@ -460,6 +506,15 @@ static void GetTls(uptr addr, uptr size) {
460	506	*addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
461	507	ThreadDescriptorSize();
462	508	*size = g_tls_size + ThreadDescriptorSize();
	509	#elif SANITIZER_GLIBC && defined(__loongarch__)
	510	# ifdef __clang__
	511	*addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
	512	ThreadDescriptorSize();
	513	# else
	514	asm("or %0,$tp,$zero" : "=r"(*addr));
	515	*addr -= ThreadDescriptorSize();
	516	# endif
	517	*size = g_tls_size + ThreadDescriptorSize();
463	518	#elif SANITIZER_GLIBC && defined(__powerpc64__)
464	519	// Workaround for glibc<2.25(?). 2.27 is known to not need this.
465	520	uptr tp;
...	...	@@ -467,7 +522,7 @@ static void GetTls(uptr addr, uptr size) {
467	522	const uptr pre_tcb_size = TlsPreTcbSize();
468	523	*addr = tp - pre_tcb_size;
469	524	*size = g_tls_size + pre_tcb_size;
470		#elif SANITIZER_FREEBSD \|\| SANITIZER_LINUX
	525	#elif SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_SOLARIS
471	526	uptr align;
472	527	GetStaticTlsBoundary(addr, size, &align);
473	528	#if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__s390__) \|\| \
...	...	@@ -528,10 +583,6 @@ static void GetTls(uptr addr, uptr size) {
528	583	*addr = (uptr)tcb->tcb_dtv[1];
529	584	}
530	585	}
531		#elif SANITIZER_SOLARIS
532		// FIXME
533		*addr = 0;
534		*size = 0;
535	586	#else
536	587	#error "Unknown OS"
537	588	#endif
...	...	@@ -603,6 +654,34 @@ static int AddModuleSegments(const char module_name, dl_phdr_info info,
603	654	bool writable = phdr->p_flags & PF_W;
604	655	cur_module.addAddressRange(cur_beg, cur_end, executable,
605	656	writable);
	657	} else if (phdr->p_type == PT_NOTE) {
	658	# ifdef NT_GNU_BUILD_ID
	659	uptr off = 0;
	660	while (off + sizeof(ElfW(Nhdr)) < phdr->p_memsz) {
	661	auto nhdr = reinterpret_cast<const ElfW(Nhdr) >(info->dlpi_addr +
	662	phdr->p_vaddr + off);
	663	constexpr auto kGnuNamesz = 4; // "GNU" with NUL-byte.
	664	static_assert(kGnuNamesz % 4 == 0, "kGnuNameSize is aligned to 4.");
	665	if (nhdr->n_type == NT_GNU_BUILD_ID && nhdr->n_namesz == kGnuNamesz) {
	666	if (off + sizeof(ElfW(Nhdr)) + nhdr->n_namesz + nhdr->n_descsz >
	667	phdr->p_memsz) {
	668	// Something is very wrong, bail out instead of reading potentially
	669	// arbitrary memory.
	670	break;
	671	}
	672	const char *name =
	673	reinterpret_cast<const char >(nhdr) + sizeof(nhdr);
	674	if (internal_memcmp(name, "GNU", 3) == 0) {
	675	const char value = reinterpret_cast<const char >(nhdr) +
	676	sizeof(*nhdr) + kGnuNamesz;
	677	cur_module.setUuid(value, nhdr->n_descsz);
	678	break;
	679	}
	680	}
	681	off += sizeof(*nhdr) + RoundUpTo(nhdr->n_namesz, 4) +
	682	RoundUpTo(nhdr->n_descsz, 4);
	683	}
	684	# endif
606	685	}
607	686	}
608	687	modules->push_back(cur_module);

lib/tsan/sanitizer_common/sanitizer_linux_s390.cpp+10-4

...	...	@@ -57,8 +57,10 @@ uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
57	57
58	58	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
59	59	int parent_tidptr, void newtls, int *child_tidptr) {
60		if (!fn \|\| !child_stack)
61		return -EINVAL;
	60	if (!fn \|\| !child_stack) {
	61	errno = EINVAL;
	62	return -1;
	63	}
62	64	CHECK_EQ(0, (uptr)child_stack % 16);
63	65	// Minimum frame size.
64	66	#ifdef __s390x__
...	...	@@ -71,9 +73,9 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
71	73	// And pass parameters.
72	74	((unsigned long *)child_stack)[1] = (uptr)fn;
73	75	((unsigned long *)child_stack)[2] = (uptr)arg;
74		register long res __asm__("r2");
	76	register uptr res __asm__("r2");
75	77	register void *__cstack __asm__("r2") = child_stack;
76		register int __flags __asm__("r3") = flags;
	78	register long __flags __asm__("r3") = flags;
77	79	register int * __ptidptr __asm__("r4") = parent_tidptr;
78	80	register int * __ctidptr __asm__("r5") = child_tidptr;
79	81	register void * __newtls __asm__("r6") = newtls;
...	...	@@ -113,6 +115,10 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
113	115	"r"(__ctidptr),
114	116	"r"(__newtls)
115	117	: "memory", "cc");
	118	if (res >= (uptr)-4095) {
	119	errno = -res;
	120	return -1;
	121	}
116	122	return res;
117	123	}
118	124

lib/tsan/sanitizer_common/sanitizer_local_address_space_view.h+1-1

...	...	@@ -17,7 +17,7 @@
17	17	// instantiated with the `LocalAddressSpaceView` type. This type is used to
18	18	// load any pointers in instance methods. This implementation is effectively
19	19	// a no-op. When an object is to be used in an out-of-process manner it is
20		// instansiated with the `RemoteAddressSpaceView` type.
	20	// instantiated with the `RemoteAddressSpaceView` type.
21	21	//
22	22	// By making `AddressSpaceView` a template parameter of an object, it can
23	23	// change its implementation at compile time which has no run time overhead.

lib/tsan/sanitizer_common/sanitizer_lzw.h created+159

...	...	@@ -0,0 +1,159 @@
	1	//===-- sanitizer_lzw.h ------------------------------------------ C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// Lempel–Ziv–Welch encoding/decoding
	10	//
	11	//===----------------------------------------------------------------------===//
	12
	13	#ifndef SANITIZER_LZW_H
	14	#define SANITIZER_LZW_H
	15
	16	#include "sanitizer_dense_map.h"
	17
	18	namespace __sanitizer {
	19
	20	using LzwCodeType = u32;
	21
	22	template <class T, class ItIn, class ItOut>
	23	ItOut LzwEncode(ItIn begin, ItIn end, ItOut out) {
	24	using Substring =
	25	detail::DenseMapPair<LzwCodeType /* Prefix /, T / Next input */>;
	26
	27	// Sentinel value for substrings of len 1.
	28	static constexpr LzwCodeType kNoPrefix =
	29	Min(DenseMapInfo<Substring>::getEmptyKey().first,
	30	DenseMapInfo<Substring>::getTombstoneKey().first) -
	31	1;
	32	DenseMap<Substring, LzwCodeType> prefix_to_code;
	33	{
	34	// Add all substring of len 1 as initial dictionary.
	35	InternalMmapVector<T> dict_len1;
	36	for (auto it = begin; it != end; ++it)
	37	if (prefix_to_code.try_emplace({kNoPrefix, *it}, 0).second)
	38	dict_len1.push_back(*it);
	39
	40	// Slightly helps with later delta encoding.
	41	Sort(dict_len1.data(), dict_len1.size());
	42
	43	// For large sizeof(T) we have to store dict_len1. Smaller types like u8 can
	44	// just generate them.
	45	*out = dict_len1.size();
	46	++out;
	47
	48	for (uptr i = 0; i != dict_len1.size(); ++i) {
	49	// Remap after the Sort.
	50	prefix_to_code[{kNoPrefix, dict_len1[i]}] = i;
	51	*out = dict_len1[i];
	52	++out;
	53	}
	54	CHECK_EQ(prefix_to_code.size(), dict_len1.size());
	55	}
	56
	57	if (begin == end)
	58	return out;
	59
	60	// Main LZW encoding loop.
	61	LzwCodeType match = prefix_to_code.find({kNoPrefix, *begin})->second;
	62	++begin;
	63	for (auto it = begin; it != end; ++it) {
	64	// Extend match with the new item.
	65	auto ins = prefix_to_code.try_emplace({match, *it}, prefix_to_code.size());
	66	if (ins.second) {
	67	// This is a new substring, but emit the code for the current match
	68	// (before extend). This allows LZW decoder to recover the dictionary.
	69	*out = match;
	70	++out;
	71	// Reset the match to a single item, which must be already in the map.
	72	match = prefix_to_code.find({kNoPrefix, *it})->second;
	73	} else {
	74	// Already known, use as the current match.
	75	match = ins.first->second;
	76	}
	77	}
	78
	79	*out = match;
	80	++out;
	81
	82	return out;
	83	}
	84
	85	template <class T, class ItIn, class ItOut>
	86	ItOut LzwDecode(ItIn begin, ItIn end, ItOut out) {
	87	if (begin == end)
	88	return out;
	89
	90	// Load dictionary of len 1 substrings. Theses correspont to lowest codes.
	91	InternalMmapVector<T> dict_len1(*begin);
	92	++begin;
	93
	94	if (begin == end)
	95	return out;
	96
	97	for (auto& v : dict_len1) {
	98	v = *begin;
	99	++begin;
	100	}
	101
	102	// Substrings of len 2 and up. Indexes are shifted because [0,
	103	// dict_len1.size()) stored in dict_len1. Substings get here after being
	104	// emitted to the output, so we can use output position.
	105	InternalMmapVector<detail::DenseMapPair<ItOut /* begin. /, ItOut / end */>>
	106	code_to_substr;
	107
	108	// Copies already emitted substrings into the output again.
	109	auto copy = [&code_to_substr, &dict_len1](LzwCodeType code, ItOut out) {
	110	if (code < dict_len1.size()) {
	111	*out = dict_len1[code];
	112	++out;
	113	return out;
	114	}
	115	const auto& s = code_to_substr[code - dict_len1.size()];
	116
	117	for (ItOut it = s.first; it != s.second; ++it, ++out) out = it;
	118	return out;
	119	};
	120
	121	// Returns lens of the substring with the given code.
	122	auto code_to_len = [&code_to_substr, &dict_len1](LzwCodeType code) -> uptr {
	123	if (code < dict_len1.size())
	124	return 1;
	125	const auto& s = code_to_substr[code - dict_len1.size()];
	126	return s.second - s.first;
	127	};
	128
	129	// Main LZW decoding loop.
	130	LzwCodeType prev_code = *begin;
	131	++begin;
	132	out = copy(prev_code, out);
	133	for (auto it = begin; it != end; ++it) {
	134	LzwCodeType code = *it;
	135	auto start = out;
	136	if (code == dict_len1.size() + code_to_substr.size()) {
	137	// Special LZW case. The code is not in the dictionary yet. This is
	138	// possible only when the new substring is the same as previous one plus
	139	// the first item of the previous substring. We can emit that in two
	140	// steps.
	141	out = copy(prev_code, out);
	142	out = start;
	143	++out;
	144	} else {
	145	out = copy(code, out);
	146	}
	147
	148	// Every time encoded emits the code, it also creates substing of len + 1
	149	// including the first item of the just emmited substring. Do the same here.
	150	uptr len = code_to_len(prev_code);
	151	code_to_substr.push_back({start - len, start + 1});
	152
	153	prev_code = code;
	154	}
	155	return out;
	156	}
	157
	158	} // namespace __sanitizer
	159	#endif

lib/tsan/sanitizer_common/sanitizer_mac.cpp+229-210

...	...	@@ -11,80 +11,82 @@
11	11	//===----------------------------------------------------------------------===//
12	12
13	13	#include "sanitizer_platform.h"
14		#if SANITIZER_MAC
15		#include "sanitizer_mac.h"
16		#include "interception/interception.h"
	14	#if SANITIZER_APPLE
	15	# include "interception/interception.h"
	16	# include "sanitizer_mac.h"
17	17
18	18	// Use 64-bit inodes in file operations. ASan does not support OS X 10.5, so
19	19	// the clients will most certainly use 64-bit ones as well.
20		#ifndef _DARWIN_USE_64_BIT_INODE
21		#define _DARWIN_USE_64_BIT_INODE 1
22		#endif
23		#include <stdio.h>
24
25		#include "sanitizer_common.h"
26		#include "sanitizer_file.h"
27		#include "sanitizer_flags.h"
28		#include "sanitizer_internal_defs.h"
29		#include "sanitizer_libc.h"
30		#include "sanitizer_platform_limits_posix.h"
31		#include "sanitizer_procmaps.h"
32		#include "sanitizer_ptrauth.h"
33
34		#if !SANITIZER_IOS
35		#include <crt_externs.h> // for _NSGetEnviron
36		#else
	20	# ifndef _DARWIN_USE_64_BIT_INODE
	21	# define _DARWIN_USE_64_BIT_INODE 1
	22	# endif
	23	# include <stdio.h>
	24
	25	# include "sanitizer_common.h"
	26	# include "sanitizer_file.h"
	27	# include "sanitizer_flags.h"
	28	# include "sanitizer_interface_internal.h"
	29	# include "sanitizer_internal_defs.h"
	30	# include "sanitizer_libc.h"
	31	# include "sanitizer_platform_limits_posix.h"
	32	# include "sanitizer_procmaps.h"
	33	# include "sanitizer_ptrauth.h"
	34
	35	# if !SANITIZER_IOS
	36	# include <crt_externs.h> // for _NSGetEnviron
	37	# else
37	38	extern char **environ;
38		#endif
	39	# endif
39	40
40		#if defined(__has_include) && __has_include(<os/trace.h>)
41		#define SANITIZER_OS_TRACE 1
42		#include <os/trace.h>
43		#else
44		#define SANITIZER_OS_TRACE 0
45		#endif
	41	# if defined(__has_include) && __has_include(<os/trace.h>)
	42	# define SANITIZER_OS_TRACE 1
	43	# include <os/trace.h>
	44	# else
	45	# define SANITIZER_OS_TRACE 0
	46	# endif
46	47
47	48	// import new crash reporting api
48		#if defined(__has_include) && __has_include(<CrashReporterClient.h>)
49		#define HAVE_CRASHREPORTERCLIENT_H 1
50		#include <CrashReporterClient.h>
51		#else
52		#define HAVE_CRASHREPORTERCLIENT_H 0
53		#endif
54
55		#if !SANITIZER_IOS
56		#include <crt_externs.h> // for _NSGetArgv and _NSGetEnviron
57		#else
	49	# if defined(__has_include) && __has_include(<CrashReporterClient.h>)
	50	# define HAVE_CRASHREPORTERCLIENT_H 1
	51	# include <CrashReporterClient.h>
	52	# else
	53	# define HAVE_CRASHREPORTERCLIENT_H 0
	54	# endif
	55
	56	# if !SANITIZER_IOS
	57	# include <crt_externs.h> // for _NSGetArgv and _NSGetEnviron
	58	# else
58	59	extern "C" {
59		extern char ***_NSGetArgv(void);
60		}
61		#endif
62
63		#include <asl.h>
64		#include <dlfcn.h> // for dladdr()
65		#include <errno.h>
66		#include <fcntl.h>
67		#include <libkern/OSAtomic.h>
68		#include <mach-o/dyld.h>
69		#include <mach/mach.h>
70		#include <mach/mach_time.h>
71		#include <mach/vm_statistics.h>
72		#include <malloc/malloc.h>
73		#include <os/log.h>
74		#include <pthread.h>
75		#include <sched.h>
76		#include <signal.h>
77		#include <spawn.h>
78		#include <stdlib.h>
79		#include <sys/ioctl.h>
80		#include <sys/mman.h>
81		#include <sys/resource.h>
82		#include <sys/stat.h>
83		#include <sys/sysctl.h>
84		#include <sys/types.h>
85		#include <sys/wait.h>
86		#include <unistd.h>
87		#include <util.h>
	60	extern char ***_NSGetArgv(void);
	61	}
	62	# endif
	63
	64	# include <asl.h>
	65	# include <dlfcn.h> // for dladdr()
	66	# include <errno.h>
	67	# include <fcntl.h>
	68	# include <libkern/OSAtomic.h>
	69	# include <mach-o/dyld.h>
	70	# include <mach/mach.h>
	71	# include <mach/mach_time.h>
	72	# include <mach/vm_statistics.h>
	73	# include <malloc/malloc.h>
	74	# include <os/log.h>
	75	# include <pthread.h>
	76	# include <pthread/introspection.h>
	77	# include <sched.h>
	78	# include <signal.h>
	79	# include <spawn.h>
	80	# include <stdlib.h>
	81	# include <sys/ioctl.h>
	82	# include <sys/mman.h>
	83	# include <sys/resource.h>
	84	# include <sys/stat.h>
	85	# include <sys/sysctl.h>
	86	# include <sys/types.h>
	87	# include <sys/wait.h>
	88	# include <unistd.h>
	89	# include <util.h>
88	90
89	91	// From <crt_externs.h>, but we don't have that file on iOS.
90	92	extern "C" {
...	...	@@ -265,30 +267,32 @@ int internal_sysctlbyname(const char sname, void oldp, uptr *oldlenp,
265	267
266	268	static fd_t internal_spawn_impl(const char argv[], const char envp[],
267	269	pid_t *pid) {
268		fd_t master_fd = kInvalidFd;
269		fd_t slave_fd = kInvalidFd;
	270	fd_t primary_fd = kInvalidFd;
	271	fd_t secondary_fd = kInvalidFd;
270	272
271	273	auto fd_closer = at_scope_exit([&] {
272		internal_close(master_fd);
273		internal_close(slave_fd);
	274	internal_close(primary_fd);
	275	internal_close(secondary_fd);
274	276	});
275	277
276	278	// We need a new pseudoterminal to avoid buffering problems. The 'atos' tool
277	279	// in particular detects when it's talking to a pipe and forgets to flush the
278	280	// output stream after sending a response.
279		master_fd = posix_openpt(O_RDWR);
280		if (master_fd == kInvalidFd) return kInvalidFd;
	281	primary_fd = posix_openpt(O_RDWR);
	282	if (primary_fd == kInvalidFd)
	283	return kInvalidFd;
281	284
282		int res = grantpt(master_fd) \|\| unlockpt(master_fd);
	285	int res = grantpt(primary_fd) \|\| unlockpt(primary_fd);
283	286	if (res != 0) return kInvalidFd;
284	287
285	288	// Use TIOCPTYGNAME instead of ptsname() to avoid threading problems.
286		char slave_pty_name[128];
287		res = ioctl(master_fd, TIOCPTYGNAME, slave_pty_name);
	289	char secondary_pty_name[128];
	290	res = ioctl(primary_fd, TIOCPTYGNAME, secondary_pty_name);
288	291	if (res == -1) return kInvalidFd;
289	292
290		slave_fd = internal_open(slave_pty_name, O_RDWR);
291		if (slave_fd == kInvalidFd) return kInvalidFd;
	293	secondary_fd = internal_open(secondary_pty_name, O_RDWR);
	294	if (secondary_fd == kInvalidFd)
	295	return kInvalidFd;
292	296
293	297	// File descriptor actions
294	298	posix_spawn_file_actions_t acts;
...	...	@@ -299,9 +303,9 @@ static fd_t internal_spawn_impl(const char argv[], const char envp[],
299	303	posix_spawn_file_actions_destroy(&acts);
300	304	});
301	305
302		res = posix_spawn_file_actions_adddup2(&acts, slave_fd, STDIN_FILENO) \|\|
303		posix_spawn_file_actions_adddup2(&acts, slave_fd, STDOUT_FILENO) \|\|
304		posix_spawn_file_actions_addclose(&acts, slave_fd);
	306	res = posix_spawn_file_actions_adddup2(&acts, secondary_fd, STDIN_FILENO) \|\|
	307	posix_spawn_file_actions_adddup2(&acts, secondary_fd, STDOUT_FILENO) \|\|
	308	posix_spawn_file_actions_addclose(&acts, secondary_fd);
305	309	if (res != 0) return kInvalidFd;
306	310
307	311	// Spawn attributes
...	...	@@ -326,14 +330,14 @@ static fd_t internal_spawn_impl(const char argv[], const char envp[],
326	330
327	331	// Disable echo in the new terminal, disable CR.
328	332	struct termios termflags;
329		tcgetattr(master_fd, &termflags);
	333	tcgetattr(primary_fd, &termflags);
330	334	termflags.c_oflag &= ~ONLCR;
331	335	termflags.c_lflag &= ~ECHO;
332		tcsetattr(master_fd, TCSANOW, &termflags);
	336	tcsetattr(primary_fd, TCSANOW, &termflags);
333	337
334		// On success, do not close master_fd on scope exit.
335		fd_t fd = master_fd;
336		master_fd = kInvalidFd;
	338	// On success, do not close primary_fd on scope exit.
	339	fd_t fd = primary_fd;
	340	primary_fd = kInvalidFd;
337	341
338	342	return fd;
339	343	}
...	...	@@ -390,6 +394,13 @@ bool FileExists(const char *filename) {
390	394	return S_ISREG(st.st_mode);
391	395	}
392	396
	397	bool DirExists(const char *path) {
	398	struct stat st;
	399	if (stat(path, &st))
	400	return false;
	401	return S_ISDIR(st.st_mode);
	402	}
	403
393	404	tid_t GetTid() {
394	405	tid_t tid;
395	406	pthread_threadid_np(nullptr, &tid);
...	...	@@ -516,25 +527,6 @@ void FutexWait(atomic_uint32_t *p, u32 cmp) {
516	527
517	528	void FutexWake(atomic_uint32_t *p, u32 count) {}
518	529
519		BlockingMutex::BlockingMutex() {
520		internal_memset(this, 0, sizeof(*this));
521		}
522
523		void BlockingMutex::Lock() {
524		CHECK(sizeof(OSSpinLock) <= sizeof(opaque_storage_));
525		CHECK_EQ(OS_SPINLOCK_INIT, 0);
526		CHECK_EQ(owner_, 0);
527		OSSpinLockLock((OSSpinLock*)&opaque_storage_);
528		}
529
530		void BlockingMutex::Unlock() {
531		OSSpinLockUnlock((OSSpinLock*)&opaque_storage_);
532		}
533
534		void BlockingMutex::CheckLocked() const {
535		CHECK_NE((const OSSpinLock)&opaque_storage_, 0);
536		}
537
538	530	u64 NanoTime() {
539	531	timeval tv;
540	532	internal_memset(&tv, 0, sizeof(tv));
...	...	@@ -562,6 +554,9 @@ uptr TlsBaseAddr() {
562	554	asm("movq %%gs:0,%0" : "=r"(segbase));
563	555	#elif defined(__i386__)
564	556	asm("movl %%gs:0,%0" : "=r"(segbase));
	557	#elif defined(__aarch64__)
	558	asm("mrs %x0, tpidrro_el0" : "=r"(segbase));
	559	segbase &= 0x07ul; // clearing lower bits, cpu id stored there
565	560	#endif
566	561	return segbase;
567	562	}
...	...	@@ -784,8 +779,8 @@ void internal_start_thread(void (func)(void arg), void *arg) {
784	779	void internal_join_thread(void *th) { pthread_join((pthread_t)th, 0); }
785	780
786	781	#if !SANITIZER_GO
787		static BlockingMutex syslog_lock(LINKER_INITIALIZED);
788		#endif
	782	static Mutex syslog_lock;
	783	# endif
789	784
790	785	void WriteOneLineToSyslog(const char *s) {
791	786	#if !SANITIZER_GO
...	...	@@ -800,7 +795,7 @@ void WriteOneLineToSyslog(const char *s) {
800	795
801	796	// buffer to store crash report application information
802	797	static char crashreporter_info_buff[__sanitizer::kErrorMessageBufferSize] = {};
803		static BlockingMutex crashreporter_info_mutex(LINKER_INITIALIZED);
	798	static Mutex crashreporter_info_mutex;
804	799
805	800	extern "C" {
806	801	// Integrate with crash reporter libraries.
...	...	@@ -830,7 +825,7 @@ asm(".desc ___crashreporter_info__, 0x10");
830	825	} // extern "C"
831	826
832	827	static void CRAppendCrashLogMessage(const char *msg) {
833		BlockingMutexLock l(&crashreporter_info_mutex);
	828	Lock l(&crashreporter_info_mutex);
834	829	internal_strlcat(crashreporter_info_buff, msg,
835	830	sizeof(crashreporter_info_buff));
836	831	#if HAVE_CRASHREPORTERCLIENT_H
...	...	@@ -874,7 +869,7 @@ void LogFullErrorReport(const char *buffer) {
874	869	// the reporting thread holds the thread registry mutex, and asl_log waits
875	870	// for GCD to dispatch a new thread, the process will deadlock, because the
876	871	// pthread_create wrapper needs to acquire the lock as well.
877		BlockingMutexLock l(&syslog_lock);
	872	Lock l(&syslog_lock);
878	873	if (common_flags()->log_to_syslog)
879	874	WriteToSyslog(buffer);
880	875
...	...	@@ -885,9 +880,12 @@ void LogFullErrorReport(const char *buffer) {
885	880	SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
886	881	#if defined(__x86_64__) \|\| defined(__i386__)
887	882	ucontext_t ucontext = static_cast<ucontext_t>(context);
888		return ucontext->uc_mcontext->__es.__err & 2 /T_PF_WRITE/ ? WRITE : READ;
	883	return ucontext->uc_mcontext->__es.__err & 2 /T_PF_WRITE/ ? Write : Read;
	884	#elif defined(__arm64__)
	885	ucontext_t ucontext = static_cast<ucontext_t>(context);
	886	return ucontext->uc_mcontext->__es.__esr & 0x40 /ISS_DA_WNR/ ? Write : Read;
889	887	#else
890		return UNKNOWN;
	888	return Unknown;
891	889	#endif
892	890	}
893	891
...	...	@@ -902,18 +900,14 @@ bool SignalContext::IsTrueFaultingAddress() const {
902	900	(uptr)ptrauth_strip( \
903	901	(void *)arm_thread_state64_get_##r(ucontext->uc_mcontext->__ss), 0)
904	902	#else
905		#define AARCH64_GET_REG(r) ucontext->uc_mcontext->__ss.__##r
	903	#define AARCH64_GET_REG(r) (uptr)ucontext->uc_mcontext->__ss.__##r
906	904	#endif
907	905
908	906	static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
909	907	ucontext_t ucontext = (ucontext_t)context;
910	908	# if defined(__aarch64__)
911	909	*pc = AARCH64_GET_REG(pc);
912		# if defined(__IPHONE_8_0) && __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_8_0
913	910	*bp = AARCH64_GET_REG(fp);
914		# else
915		*bp = AARCH64_GET_REG(lr);
916		# endif
917	911	*sp = AARCH64_GET_REG(sp);
918	912	# elif defined(__x86_64__)
919	913	*pc = ucontext->uc_mcontext->__ss.__rip;
...	...	@@ -950,6 +944,9 @@ static void DisableMmapExcGuardExceptions() {
950	944	set_behavior(mach_task_self(), task_exc_guard_none);
951	945	}
952	946
	947	static void VerifyInterceptorsWorking();
	948	static void StripEnv();
	949
953	950	void InitializePlatformEarly() {
954	951	// Only use xnu_fast_mmap when on x86_64 and the kernel supports it.
955	952	use_xnu_fast_mmap =
...	...	@@ -960,17 +957,54 @@ void InitializePlatformEarly() {
960	957	#endif
961	958	if (GetDarwinKernelVersion() >= DarwinKernelVersion(19, 0))
962	959	DisableMmapExcGuardExceptions();
	960
	961	# if !SANITIZER_GO
	962	MonotonicNanoTime(); // Call to initialize mach_timebase_info
	963	VerifyInterceptorsWorking();
	964	StripEnv();
	965	# endif
963	966	}
964	967
965	968	#if !SANITIZER_GO
966	969	static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES";
967	970	LowLevelAllocator allocator_for_env;
968	971
	972	static bool ShouldCheckInterceptors() {
	973	// Restrict "interceptors working?" check to ASan and TSan.
	974	const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer"};
	975	size_t count = sizeof(sanitizer_names) / sizeof(sanitizer_names[0]);
	976	for (size_t i = 0; i < count; i++) {
	977	if (internal_strcmp(sanitizer_names[i], SanitizerToolName) == 0)
	978	return true;
	979	}
	980	return false;
	981	}
	982
	983	static void VerifyInterceptorsWorking() {
	984	if (!common_flags()->verify_interceptors \|\| !ShouldCheckInterceptors())
	985	return;
	986
	987	// Verify that interceptors really work. We'll use dlsym to locate
	988	// "puts", if interceptors are working, it should really point to
	989	// "wrap_puts" within our own dylib.
	990	Dl_info info_puts, info_runtime;
	991	RAW_CHECK(dladdr(dlsym(RTLD_DEFAULT, "puts"), &info_puts));
	992	RAW_CHECK(dladdr((void *)&VerifyInterceptorsWorking, &info_runtime));
	993	if (internal_strcmp(info_puts.dli_fname, info_runtime.dli_fname) != 0) {
	994	Report(
	995	"ERROR: Interceptors are not working. This may be because %s is "
	996	"loaded too late (e.g. via dlopen). Please launch the executable "
	997	"with:\n%s=%s\n",
	998	SanitizerToolName, kDyldInsertLibraries, info_runtime.dli_fname);
	999	RAW_CHECK("interceptors not installed" && 0);
	1000	}
	1001	}
	1002
969	1003	// Change the value of the env var \|name\|, leaking the original value.
970	1004	// If \|name_value\| is NULL, the variable is deleted from the environment,
971	1005	// otherwise the corresponding "NAME=value" string is replaced with
972	1006	// \|name_value\|.
973		void LeakyResetEnv(const char name, const char name_value) {
	1007	static void LeakyResetEnv(const char name, const char name_value) {
974	1008	char **env = GetEnviron();
975	1009	uptr name_len = internal_strlen(name);
976	1010	while (*env != 0) {
...	...	@@ -995,100 +1029,28 @@ void LeakyResetEnv(const char name, const char name_value) {
995	1029	}
996	1030	}
997	1031
998		SANITIZER_WEAK_CXX_DEFAULT_IMPL
999		bool ReexecDisabled() {
1000		return false;
1001		}
1002
1003		static bool DyldNeedsEnvVariable() {
1004		// If running on OS X 10.11+ or iOS 9.0+, dyld will interpose even if
1005		// DYLD_INSERT_LIBRARIES is not set.
1006		return GetMacosAlignedVersion() < MacosVersion(10, 11);
1007		}
1008
1009		void MaybeReexec() {
1010		// FIXME: This should really live in some "InitializePlatform" method.
1011		MonotonicNanoTime();
	1032	static void StripEnv() {
	1033	if (!common_flags()->strip_env)
	1034	return;
1012	1035
1013		if (ReexecDisabled()) return;
	1036	char *dyld_insert_libraries =
	1037	const_cast<char *>(GetEnv(kDyldInsertLibraries));
	1038	if (!dyld_insert_libraries)
	1039	return;
1014	1040
1015		// Make sure the dynamic runtime library is preloaded so that the
1016		// wrappers work. If it is not, set DYLD_INSERT_LIBRARIES and re-exec
1017		// ourselves.
1018	1041	Dl_info info;
1019		RAW_CHECK(dladdr((void*)((uptr)&__sanitizer_report_error_summary), &info));
1020		char *dyld_insert_libraries =
1021		const_cast<char*>(GetEnv(kDyldInsertLibraries));
1022		uptr old_env_len = dyld_insert_libraries ?
1023		internal_strlen(dyld_insert_libraries) : 0;
1024		uptr fname_len = internal_strlen(info.dli_fname);
	1042	RAW_CHECK(dladdr((void *)&StripEnv, &info));
1025	1043	const char *dylib_name = StripModuleName(info.dli_fname);
1026		uptr dylib_name_len = internal_strlen(dylib_name);
1027
1028		bool lib_is_in_env = dyld_insert_libraries &&
1029		internal_strstr(dyld_insert_libraries, dylib_name);
1030		if (DyldNeedsEnvVariable() && !lib_is_in_env) {
1031		// DYLD_INSERT_LIBRARIES is not set or does not contain the runtime
1032		// library.
1033		InternalMmapVector<char> program_name(1024);
1034		uint32_t buf_size = program_name.size();
1035		_NSGetExecutablePath(program_name.data(), &buf_size);
1036		char new_env = const_cast<char>(info.dli_fname);
1037		if (dyld_insert_libraries) {
1038		// Append the runtime dylib name to the existing value of
1039		// DYLD_INSERT_LIBRARIES.
1040		new_env = (char*)allocator_for_env.Allocate(old_env_len + fname_len + 2);
1041		internal_strncpy(new_env, dyld_insert_libraries, old_env_len);
1042		new_env[old_env_len] = ':';
1043		// Copy fname_len and add a trailing zero.
1044		internal_strncpy(new_env + old_env_len + 1, info.dli_fname,
1045		fname_len + 1);
1046		// Ok to use setenv() since the wrappers don't depend on the value of
1047		// asan_inited.
1048		setenv(kDyldInsertLibraries, new_env, /overwrite/1);
1049		} else {
1050		// Set DYLD_INSERT_LIBRARIES equal to the runtime dylib name.
1051		setenv(kDyldInsertLibraries, info.dli_fname, /overwrite/0);
1052		}
1053		VReport(1, "exec()-ing the program with\n");
1054		VReport(1, "%s=%s\n", kDyldInsertLibraries, new_env);
1055		VReport(1, "to enable wrappers.\n");
1056		execv(program_name.data(), *_NSGetArgv());
1057
1058		// We get here only if execv() failed.
1059		Report("ERROR: The process is launched without DYLD_INSERT_LIBRARIES, "
1060		"which is required for the sanitizer to work. We tried to set the "
1061		"environment variable and re-execute itself, but execv() failed, "
1062		"possibly because of sandbox restrictions. Make sure to launch the "
1063		"executable with:\n%s=%s\n", kDyldInsertLibraries, new_env);
1064		RAW_CHECK("execv failed" && 0);
1065		}
1066
1067		// Verify that interceptors really work. We'll use dlsym to locate
1068		// "pthread_create", if interceptors are working, it should really point to
1069		// "wrap_pthread_create" within our own dylib.
1070		Dl_info info_pthread_create;
1071		void *dlopen_addr = dlsym(RTLD_DEFAULT, "pthread_create");
1072		RAW_CHECK(dladdr(dlopen_addr, &info_pthread_create));
1073		if (internal_strcmp(info.dli_fname, info_pthread_create.dli_fname) != 0) {
1074		Report(
1075		"ERROR: Interceptors are not working. This may be because %s is "
1076		"loaded too late (e.g. via dlopen). Please launch the executable "
1077		"with:\n%s=%s\n",
1078		SanitizerToolName, kDyldInsertLibraries, info.dli_fname);
1079		RAW_CHECK("interceptors not installed" && 0);
1080		}
1081
	1044	bool lib_is_in_env = internal_strstr(dyld_insert_libraries, dylib_name);
1082	1045	if (!lib_is_in_env)
1083	1046	return;
1084	1047
1085		if (!common_flags()->strip_env)
1086		return;
1087
1088	1048	// DYLD_INSERT_LIBRARIES is set and contains the runtime library. Let's remove
1089	1049	// the dylib from the environment variable, because interceptors are installed
1090	1050	// and we don't want our children to inherit the variable.
1091	1051
	1052	uptr old_env_len = internal_strlen(dyld_insert_libraries);
	1053	uptr dylib_name_len = internal_strlen(dylib_name);
1092	1054	uptr env_name_len = internal_strlen(kDyldInsertLibraries);
1093	1055	// Allocate memory to hold the previous env var name, its value, the '='
1094	1056	// sign and the '\0' char.
...	...	@@ -1237,7 +1199,7 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
1237	1199
1238	1200	uptr largest_gap_found = 0;
1239	1201	uptr max_occupied_addr = 0;
1240		VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
	1202	VReport(2, "FindDynamicShadowStart, space_size = %p\n", (void *)space_size);
1241	1203	uptr shadow_start =
1242	1204	FindAvailableMemoryRange(space_size, alignment, granularity,
1243	1205	&largest_gap_found, &max_occupied_addr);
...	...	@@ -1246,20 +1208,21 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
1246	1208	VReport(
1247	1209	2,
1248	1210	"Shadow doesn't fit, largest_gap_found = %p, max_occupied_addr = %p\n",
1249		largest_gap_found, max_occupied_addr);
	1211	(void )largest_gap_found, (void )max_occupied_addr);
1250	1212	uptr new_max_vm = RoundDownTo(largest_gap_found << shadow_scale, alignment);
1251	1213	if (new_max_vm < max_occupied_addr) {
1252	1214	Report("Unable to find a memory range for dynamic shadow.\n");
1253	1215	Report(
1254	1216	"space_size = %p, largest_gap_found = %p, max_occupied_addr = %p, "
1255	1217	"new_max_vm = %p\n",
1256		space_size, largest_gap_found, max_occupied_addr, new_max_vm);
	1218	(void )space_size, (void )largest_gap_found,
	1219	(void )max_occupied_addr, (void )new_max_vm);
1257	1220	CHECK(0 && "cannot place shadow");
1258	1221	}
1259	1222	RestrictMemoryToMaxAddress(new_max_vm);
1260	1223	high_mem_end = new_max_vm - 1;
1261	1224	space_size = (high_mem_end >> shadow_scale) + left_padding;
1262		VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);
	1225	VReport(2, "FindDynamicShadowStart, space_size = %p\n", (void *)space_size);
1263	1226	shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity,
1264	1227	nullptr, nullptr);
1265	1228	if (shadow_start == 0) {
...	...	@@ -1288,6 +1251,7 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
1288	1251	mach_vm_address_t start_address =
1289	1252	(SANITIZER_WORDSIZE == 32) ? 0x000000001000 : 0x000100000000;
1290	1253
	1254	const mach_vm_address_t max_vm_address = GetMaxVirtualAddress() + 1;
1291	1255	mach_vm_address_t address = start_address;
1292	1256	mach_vm_address_t free_begin = start_address;
1293	1257	kern_return_t kr = KERN_SUCCESS;
...	...	@@ -1302,7 +1266,7 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
1302	1266	(vm_region_info_t)&vminfo, &count);
1303	1267	if (kr == KERN_INVALID_ADDRESS) {
1304	1268	// No more regions beyond "address", consider the gap at the end of VM.
1305		address = GetMaxVirtualAddress() + 1;
	1269	address = max_vm_address;
1306	1270	vmsize = 0;
1307	1271	} else {
1308	1272	if (max_occupied_addr) *max_occupied_addr = address + vmsize;
...	...	@@ -1310,7 +1274,7 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
1310	1274	if (free_begin != address) {
1311	1275	// We found a free region [free_begin..address-1].
1312	1276	uptr gap_start = RoundUpTo((uptr)free_begin + left_padding, alignment);
1313		uptr gap_end = RoundDownTo((uptr)address, alignment);
	1277	uptr gap_end = RoundDownTo((uptr)Min(address, max_vm_address), alignment);
1314	1278	uptr gap_size = gap_end > gap_start ? gap_end - gap_start : 0;
1315	1279	if (size < gap_size) {
1316	1280	return gap_start;
...	...	@@ -1330,7 +1294,7 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
1330	1294	}
1331	1295
1332	1296	// FIXME implement on this platform.
1333		void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { }
	1297	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
1334	1298
1335	1299	void SignalContext::DumpAllRegisters(void *context) {
1336	1300	Report("Register values:\n");
...	...	@@ -1339,7 +1303,7 @@ void SignalContext::DumpAllRegisters(void *context) {
1339	1303	# define DUMPREG64(r) \
1340	1304	Printf("%s = 0x%016llx ", #r, ucontext->uc_mcontext->__ss.__ ## r);
1341	1305	# define DUMPREGA64(r) \
1342		Printf(" %s = 0x%016llx ", #r, AARCH64_GET_REG(r));
	1306	Printf(" %s = 0x%016lx ", #r, AARCH64_GET_REG(r));
1343	1307	# define DUMPREG32(r) \
1344	1308	Printf("%s = 0x%08x ", #r, ucontext->uc_mcontext->__ss.__ ## r);
1345	1309	# define DUMPREG_(r) Printf(" "); DUMPREG(r);
...	...	@@ -1409,7 +1373,7 @@ void DumpProcessMap() {
1409	1373	char uuid_str[128];
1410	1374	FormatUUID(uuid_str, sizeof(uuid_str), modules[i].uuid());
1411	1375	Printf("0x%zx-0x%zx %s (%s) %s\n", modules[i].base_address(),
1412		modules[i].max_executable_address(), modules[i].full_name(),
	1376	modules[i].max_address(), modules[i].full_name(),
1413	1377	ModuleArchToString(modules[i].arch()), uuid_str);
1414	1378	}
1415	1379	Printf("End of module map.\n");
...	...	@@ -1433,6 +1397,61 @@ u32 GetNumberOfCPUs() {
1433	1397
1434	1398	void InitializePlatformCommonFlags(CommonFlags *cf) {}
1435	1399
	1400	// Pthread introspection hook
	1401	//
	1402	// * GCD worker threads are created without a call to pthread_create(), but we
	1403	// still need to register these threads (with ThreadCreate/Start()).
	1404	// * We use the "pthread introspection hook" below to observe the creation of
	1405	// such threads.
	1406	// * GCD worker threads don't have parent threads and the CREATE event is
	1407	// delivered in the context of the thread itself. CREATE events for regular
	1408	// threads, are delivered on the parent. We use this to tell apart which
	1409	// threads are GCD workers with `thread == pthread_self()`.
	1410	//
	1411	static pthread_introspection_hook_t prev_pthread_introspection_hook;
	1412	static ThreadEventCallbacks thread_event_callbacks;
	1413
	1414	static void sanitizer_pthread_introspection_hook(unsigned int event,
	1415	pthread_t thread, void *addr,
	1416	size_t size) {
	1417	// create -> start -> terminate -> destroy
	1418	// * create/destroy are usually (not guaranteed) delivered on the parent and
	1419	// track resource allocation/reclamation
	1420	// * start/terminate are guaranteed to be delivered in the context of the
	1421	// thread and give hooks into "just after (before) thread starts (stops)
	1422	// executing"
	1423	DCHECK(event >= PTHREAD_INTROSPECTION_THREAD_CREATE &&
	1424	event <= PTHREAD_INTROSPECTION_THREAD_DESTROY);
	1425
	1426	if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) {
	1427	bool gcd_worker = (thread == pthread_self());
	1428	if (thread_event_callbacks.create)
	1429	thread_event_callbacks.create((uptr)thread, gcd_worker);
	1430	} else if (event == PTHREAD_INTROSPECTION_THREAD_START) {
	1431	CHECK_EQ(thread, pthread_self());
	1432	if (thread_event_callbacks.start)
	1433	thread_event_callbacks.start((uptr)thread);
	1434	}
	1435
	1436	if (prev_pthread_introspection_hook)
	1437	prev_pthread_introspection_hook(event, thread, addr, size);
	1438
	1439	if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) {
	1440	CHECK_EQ(thread, pthread_self());
	1441	if (thread_event_callbacks.terminate)
	1442	thread_event_callbacks.terminate((uptr)thread);
	1443	} else if (event == PTHREAD_INTROSPECTION_THREAD_DESTROY) {
	1444	if (thread_event_callbacks.destroy)
	1445	thread_event_callbacks.destroy((uptr)thread);
	1446	}
	1447	}
	1448
	1449	void InstallPthreadIntrospectionHook(const ThreadEventCallbacks &callbacks) {
	1450	thread_event_callbacks = callbacks;
	1451	prev_pthread_introspection_hook =
	1452	pthread_introspection_hook_install(&sanitizer_pthread_introspection_hook);
	1453	}
	1454
1436	1455	} // namespace __sanitizer
1437	1456
1438		#endif // SANITIZER_MAC
	1457	#endif // SANITIZER_APPLE

lib/tsan/sanitizer_common/sanitizer_mac.h+16-5

...	...	@@ -9,12 +9,12 @@
9	9	// This file is shared between various sanitizers' runtime libraries and
10	10	// provides definitions for OSX-specific functions.
11	11	//===----------------------------------------------------------------------===//
12		#ifndef SANITIZER_MAC_H
13		#define SANITIZER_MAC_H
	12	#ifndef SANITIZER_APPLE_H
	13	#define SANITIZER_APPLE_H
14	14
15	15	#include "sanitizer_common.h"
16	16	#include "sanitizer_platform.h"
17		#if SANITIZER_MAC
	17	#if SANITIZER_APPLE
18	18	#include "sanitizer_posix.h"
19	19
20	20	namespace __sanitizer {
...	...	@@ -62,7 +62,18 @@ char **GetEnviron();
62	62
63	63	void RestrictMemoryToMaxAddress(uptr max_address);
64	64
	65	using ThreadEventCallback = void (*)(uptr thread);
	66	using ThreadCreateEventCallback = void (*)(uptr thread, bool gcd_worker);
	67	struct ThreadEventCallbacks {
	68	ThreadCreateEventCallback create;
	69	ThreadEventCallback start;
	70	ThreadEventCallback terminate;
	71	ThreadEventCallback destroy;
	72	};
	73
	74	void InstallPthreadIntrospectionHook(const ThreadEventCallbacks &callbacks);
	75
65	76	} // namespace __sanitizer
66	77
67		#endif // SANITIZER_MAC
68		#endif // SANITIZER_MAC_H
	78	#endif // SANITIZER_APPLE
	79	#endif // SANITIZER_APPLE_H

lib/tsan/sanitizer_common/sanitizer_mac_libcdep.cpp+2-2

...	...	@@ -11,7 +11,7 @@
11	11	//===----------------------------------------------------------------------===//
12	12
13	13	#include "sanitizer_platform.h"
14		#if SANITIZER_MAC
	14	#if SANITIZER_APPLE
15	15	#include "sanitizer_mac.h"
16	16
17	17	#include <sys/mman.h>
...	...	@@ -26,4 +26,4 @@ void RestrictMemoryToMaxAddress(uptr max_address) {
26	26
27	27	} // namespace __sanitizer
28	28
29		#endif // SANITIZER_MAC
	29	#endif // SANITIZER_APPLE

lib/tsan/sanitizer_common/sanitizer_mallinfo.h created+38

...	...	@@ -0,0 +1,38 @@
	1	//===-- sanitizer_mallinfo.h ----------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of Sanitizer common code.
	10	//
	11	// Definition for mallinfo on different platforms.
	12	//===----------------------------------------------------------------------===//
	13
	14	#ifndef SANITIZER_MALLINFO_H
	15	#define SANITIZER_MALLINFO_H
	16
	17	#include "sanitizer_internal_defs.h"
	18	#include "sanitizer_platform.h"
	19
	20	namespace __sanitizer {
	21
	22	#if SANITIZER_ANDROID
	23
	24	struct __sanitizer_struct_mallinfo {
	25	uptr v[10];
	26	};
	27
	28	#elif SANITIZER_LINUX \|\| SANITIZER_APPLE \|\| SANITIZER_FUCHSIA
	29
	30	struct __sanitizer_struct_mallinfo {
	31	int v[10];
	32	};
	33
	34	#endif
	35
	36	} // namespace __sanitizer
	37
	38	#endif // SANITIZER_MALLINFO_H

lib/tsan/sanitizer_common/sanitizer_malloc_mac.inc+10-12

...	...	@@ -12,7 +12,7 @@
12	12	//===----------------------------------------------------------------------===//
13	13
14	14	#include "sanitizer_common/sanitizer_platform.h"
15		#if !SANITIZER_MAC
	15	#if !SANITIZER_APPLE
16	16	#error "This file should only be compiled on Darwin."
17	17	#endif
18	18
...	...	@@ -23,6 +23,7 @@
23	23	#include <sys/mman.h>
24	24
25	25	#include "interception/interception.h"
	26	#include "sanitizer_common/sanitizer_allocator_dlsym.h"
26	27	#include "sanitizer_common/sanitizer_mac.h"
27	28
28	29	// Similar code is used in Google Perftools,
...	...	@@ -192,20 +193,15 @@ void __sanitizer_mz_malloc(malloc_zone_t zone, uptr size) {
192	193	return p;
193	194	}
194	195
	196	struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
	197	static bool UseImpl() { return !COMMON_MALLOC_SANITIZER_INITIALIZED; }
	198	};
	199
195	200	extern "C"
196	201	SANITIZER_INTERFACE_ATTRIBUTE
197	202	void __sanitizer_mz_calloc(malloc_zone_t zone, size_t nmemb, size_t size) {
198		if (UNLIKELY(!COMMON_MALLOC_SANITIZER_INITIALIZED)) {
199		// Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
200		const size_t kCallocPoolSize = 1024;
201		static uptr calloc_memory_for_dlsym[kCallocPoolSize];
202		static size_t allocated;
203		size_t size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize;
204		void mem = (void)&calloc_memory_for_dlsym[allocated];
205		allocated += size_in_words;
206		CHECK(allocated < kCallocPoolSize);
207		return mem;
208		}
	203	if (DlsymAlloc::Use())
	204	return DlsymAlloc::Callocate(nmemb, size);
209	205	COMMON_MALLOC_CALLOC(nmemb, size);
210	206	return p;
211	207	}
...	...	@@ -223,6 +219,8 @@ extern "C"
223	219	SANITIZER_INTERFACE_ATTRIBUTE
224	220	void __sanitizer_mz_free(malloc_zone_t zone, void ptr) {
225	221	if (!ptr) return;
	222	if (DlsymAlloc::PointerIsMine(ptr))
	223	return DlsymAlloc::Free(ptr);
226	224	COMMON_MALLOC_FREE(ptr);
227	225	}
228	226

lib/tsan/sanitizer_common/sanitizer_mutex.cpp+2-2

...	...	@@ -73,7 +73,7 @@ void DebugMutexInit() {
73	73	// Build adjacency matrix.
74	74	bool leaf[kMutexTypeMax];
75	75	internal_memset(&leaf, 0, sizeof(leaf));
76		int cnt[kMutexTypeMax] = {};
	76	int cnt[kMutexTypeMax];
77	77	internal_memset(&cnt, 0, sizeof(cnt));
78	78	for (int t = 0; t < kMutexTypeMax; t++) {
79	79	mutex_type_count = t;
...	...	@@ -174,7 +174,7 @@ struct InternalDeadlockDetector {
174	174	if (max_idx != MutexInvalid && !mutex_can_lock[max_idx][type]) {
175	175	Printf("%s: internal deadlock: can't lock %s under %s mutex\n", SanitizerToolName,
176	176	mutex_meta[type].name, mutex_meta[max_idx].name);
177		PrintMutexPC(pc);
	177	PrintMutexPC(locked[max_idx].pc);
178	178	CHECK(0);
179	179	}
180	180	locked[type].seq = ++sequence;

lib/tsan/sanitizer_common/sanitizer_mutex.h+120-161

...	...	@@ -20,25 +20,27 @@
20	20
21	21	namespace __sanitizer {
22	22
23		class MUTEX StaticSpinMutex {
	23	class SANITIZER_MUTEX StaticSpinMutex {
24	24	public:
25	25	void Init() {
26	26	atomic_store(&state_, 0, memory_order_relaxed);
27	27	}
28	28
29		void Lock() ACQUIRE() {
	29	void Lock() SANITIZER_ACQUIRE() {
30	30	if (LIKELY(TryLock()))
31	31	return;
32	32	LockSlow();
33	33	}
34	34
35		bool TryLock() TRY_ACQUIRE(true) {
	35	bool TryLock() SANITIZER_TRY_ACQUIRE(true) {
36	36	return atomic_exchange(&state_, 1, memory_order_acquire) == 0;
37	37	}
38	38
39		void Unlock() RELEASE() { atomic_store(&state_, 0, memory_order_release); }
	39	void Unlock() SANITIZER_RELEASE() {
	40	atomic_store(&state_, 0, memory_order_release);
	41	}
40	42
41		void CheckLocked() const CHECK_LOCKED() {
	43	void CheckLocked() const SANITIZER_CHECK_LOCKED() {
42	44	CHECK_EQ(atomic_load(&state_, memory_order_relaxed), 1);
43	45	}
44	46
...	...	@@ -48,7 +50,7 @@ class MUTEX StaticSpinMutex {
48	50	void LockSlow();
49	51	};
50	52
51		class MUTEX SpinMutex : public StaticSpinMutex {
	53	class SANITIZER_MUTEX SpinMutex : public StaticSpinMutex {
52	54	public:
53	55	SpinMutex() {
54	56	Init();
...	...	@@ -95,7 +97,11 @@ enum {
95	97
96	98	// Go linker does not support THREADLOCAL variables,
97	99	// so we can't use per-thread state.
98		#define SANITIZER_CHECK_DEADLOCKS (SANITIZER_DEBUG && !SANITIZER_GO)
	100	// Disable checked locks on Darwin. Although Darwin platforms support
	101	// THREADLOCAL variables they are not usable early on during process init when
	102	// `__sanitizer::Mutex` is used.
	103	#define SANITIZER_CHECK_DEADLOCKS \
	104	(SANITIZER_DEBUG && !SANITIZER_GO && SANITIZER_SUPPORTS_THREADLOCAL && !SANITIZER_APPLE)
99	105
100	106	#if SANITIZER_CHECK_DEADLOCKS
101	107	struct MutexMeta {
...	...	@@ -111,7 +117,7 @@ struct MutexMeta {
111	117
112	118	class CheckedMutex {
113	119	public:
114		constexpr CheckedMutex(MutexType type)
	120	explicit constexpr CheckedMutex(MutexType type)
115	121	#if SANITIZER_CHECK_DEADLOCKS
116	122	: type_(type)
117	123	#endif
...	...	@@ -152,15 +158,15 @@ class CheckedMutex {
152	158	// Derive from CheckedMutex for the purposes of EBO.
153	159	// We could make it a field marked with [[no_unique_address]],
154	160	// but this attribute is not supported by some older compilers.
155		class MUTEX Mutex : CheckedMutex {
	161	class SANITIZER_MUTEX Mutex : CheckedMutex {
156	162	public:
157		constexpr Mutex(MutexType type = MutexUnchecked) : CheckedMutex(type) {}
	163	explicit constexpr Mutex(MutexType type = MutexUnchecked)
	164	: CheckedMutex(type) {}
158	165
159		void Lock() ACQUIRE() {
	166	void Lock() SANITIZER_ACQUIRE() {
160	167	CheckedMutex::Lock();
161	168	u64 reset_mask = ~0ull;
162	169	u64 state = atomic_load_relaxed(&state_);
163		const uptr kMaxSpinIters = 1500;
164	170	for (uptr spin_iters = 0;; spin_iters++) {
165	171	u64 new_state;
166	172	bool locked = (state & (kWriterLock \| kReaderLockMask)) != 0;
...	...	@@ -189,8 +195,6 @@ class MUTEX Mutex : CheckedMutex {
189	195	// We've incremented waiting writers, so now block.
190	196	writers_.Wait();
191	197	spin_iters = 0;
192		state = atomic_load(&state_, memory_order_relaxed);
193		DCHECK_NE(state & kWriterSpinWait, 0);
194	198	} else {
195	199	// We've set kWriterSpinWait, but we are still in active spinning.
196	200	}
...	...	@@ -199,10 +203,26 @@ class MUTEX Mutex : CheckedMutex {
199	203	// Either way we need to reset kWriterSpinWait
200	204	// next time we take the lock or block again.
201	205	reset_mask = ~kWriterSpinWait;
	206	state = atomic_load(&state_, memory_order_relaxed);
	207	DCHECK_NE(state & kWriterSpinWait, 0);
	208	}
	209	}
	210
	211	bool TryLock() SANITIZER_TRY_ACQUIRE(true) {
	212	u64 state = atomic_load_relaxed(&state_);
	213	for (;;) {
	214	if (UNLIKELY(state & (kWriterLock \| kReaderLockMask)))
	215	return false;
	216	// The mutex is not read-/write-locked, try to lock.
	217	if (LIKELY(atomic_compare_exchange_weak(
	218	&state_, &state, state \| kWriterLock, memory_order_acquire))) {
	219	CheckedMutex::Lock();
	220	return true;
	221	}
202	222	}
203	223	}
204	224
205		void Unlock() RELEASE() {
	225	void Unlock() SANITIZER_RELEASE() {
206	226	CheckedMutex::Unlock();
207	227	bool wake_writer;
208	228	u64 wake_readers;
...	...	@@ -212,17 +232,16 @@ class MUTEX Mutex : CheckedMutex {
212	232	DCHECK_NE(state & kWriterLock, 0);
213	233	DCHECK_EQ(state & kReaderLockMask, 0);
214	234	new_state = state & ~kWriterLock;
215		wake_writer =
216		(state & kWriterSpinWait) == 0 && (state & kWaitingWriterMask) != 0;
	235	wake_writer = (state & (kWriterSpinWait \| kReaderSpinWait)) == 0 &&
	236	(state & kWaitingWriterMask) != 0;
217	237	if (wake_writer)
218	238	new_state = (new_state - kWaitingWriterInc) \| kWriterSpinWait;
219	239	wake_readers =
220		(state & (kWriterSpinWait \| kWaitingWriterMask)) != 0
	240	wake_writer \|\| (state & kWriterSpinWait) != 0
221	241	? 0
222	242	: ((state & kWaitingReaderMask) >> kWaitingReaderShift);
223	243	if (wake_readers)
224		new_state = (new_state & ~kWaitingReaderMask) +
225		(wake_readers << kReaderLockShift);
	244	new_state = (new_state & ~kWaitingReaderMask) \| kReaderSpinWait;
226	245	} while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
227	246	memory_order_release)));
228	247	if (UNLIKELY(wake_writer))
...	...	@@ -231,37 +250,54 @@ class MUTEX Mutex : CheckedMutex {
231	250	readers_.Post(wake_readers);
232	251	}
233	252
234		void ReadLock() ACQUIRE_SHARED() {
	253	void ReadLock() SANITIZER_ACQUIRE_SHARED() {
235	254	CheckedMutex::Lock();
236		bool locked;
237		u64 new_state;
	255	u64 reset_mask = ~0ull;
238	256	u64 state = atomic_load_relaxed(&state_);
239		do {
240		locked =
241		(state & kReaderLockMask) == 0 &&
242		(state & (kWriterLock \| kWriterSpinWait \| kWaitingWriterMask)) != 0;
	257	for (uptr spin_iters = 0;; spin_iters++) {
	258	bool locked = (state & kWriterLock) != 0;
	259	u64 new_state;
	260	if (LIKELY(!locked)) {
	261	new_state = (state + kReaderLockInc) & reset_mask;
	262	} else if (spin_iters > kMaxSpinIters) {
	263	new_state = (state + kWaitingReaderInc) & reset_mask;
	264	} else if ((state & kReaderSpinWait) == 0) {
	265	// Active spinning, but denote our presence so that unlocking
	266	// thread does not wake up other threads.
	267	new_state = state \| kReaderSpinWait;
	268	} else {
	269	// Active spinning.
	270	state = atomic_load(&state_, memory_order_relaxed);
	271	continue;
	272	}
	273	if (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
	274	memory_order_acquire)))
	275	continue;
243	276	if (LIKELY(!locked))
244		new_state = state + kReaderLockInc;
245		else
246		new_state = state + kWaitingReaderInc;
247		} while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
248		memory_order_acquire)));
249		if (UNLIKELY(locked))
250		readers_.Wait();
251		DCHECK_EQ(atomic_load_relaxed(&state_) & kWriterLock, 0);
252		DCHECK_NE(atomic_load_relaxed(&state_) & kReaderLockMask, 0);
	277	return; // We've locked the mutex.
	278	if (spin_iters > kMaxSpinIters) {
	279	// We've incremented waiting readers, so now block.
	280	readers_.Wait();
	281	spin_iters = 0;
	282	} else {
	283	// We've set kReaderSpinWait, but we are still in active spinning.
	284	}
	285	reset_mask = ~kReaderSpinWait;
	286	state = atomic_load(&state_, memory_order_relaxed);
	287	}
253	288	}
254	289
255		void ReadUnlock() RELEASE_SHARED() {
	290	void ReadUnlock() SANITIZER_RELEASE_SHARED() {
256	291	CheckedMutex::Unlock();
257	292	bool wake;
258	293	u64 new_state;
259	294	u64 state = atomic_load_relaxed(&state_);
260	295	do {
261	296	DCHECK_NE(state & kReaderLockMask, 0);
262		DCHECK_EQ(state & (kWaitingReaderMask \| kWriterLock), 0);
	297	DCHECK_EQ(state & kWriterLock, 0);
263	298	new_state = state - kReaderLockInc;
264		wake = (new_state & (kReaderLockMask \| kWriterSpinWait)) == 0 &&
	299	wake = (new_state &
	300	(kReaderLockMask \| kWriterSpinWait \| kReaderSpinWait)) == 0 &&
265	301	(new_state & kWaitingWriterMask) != 0;
266	302	if (wake)
267	303	new_state = (new_state - kWaitingWriterInc) \| kWriterSpinWait;
...	...	@@ -277,13 +313,13 @@ class MUTEX Mutex : CheckedMutex {
277	313	// owns the mutex but a child checks that it is locked. Rather than
278	314	// maintaining complex state to work around those situations, the check only
279	315	// checks that the mutex is owned.
280		void CheckWriteLocked() const CHECK_LOCKED() {
	316	void CheckWriteLocked() const SANITIZER_CHECK_LOCKED() {
281	317	CHECK(atomic_load(&state_, memory_order_relaxed) & kWriterLock);
282	318	}
283	319
284		void CheckLocked() const CHECK_LOCKED() { CheckWriteLocked(); }
	320	void CheckLocked() const SANITIZER_CHECK_LOCKED() { CheckWriteLocked(); }
285	321
286		void CheckReadLocked() const CHECK_LOCKED() {
	322	void CheckReadLocked() const SANITIZER_CHECK_LOCKED() {
287	323	CHECK(atomic_load(&state_, memory_order_relaxed) & kReaderLockMask);
288	324	}
289	325
...	...	@@ -305,16 +341,14 @@ class MUTEX Mutex : CheckedMutex {
305	341	// - a writer is awake and spin-waiting
306	342	// the flag is used to prevent thundering herd problem
307	343	// (new writers are not woken if this flag is set)
	344	// - a reader is awake and spin-waiting
308	345	//
309		// Writer support active spinning, readers does not.
	346	// Both writers and readers use active spinning before blocking.
310	347	// But readers are more aggressive and always take the mutex
311	348	// if there are any other readers.
312		// Writers hand off the mutex to readers: after wake up readers
313		// already assume ownership of the mutex (don't need to do any
314		// state updates). But the mutex is not handed off to writers,
315		// after wake up writers compete to lock the mutex again.
316		// This is needed to allow repeated write locks even in presence
317		// of other blocked writers.
	349	// After wake up both writers and readers compete to lock the
	350	// mutex again. This is needed to allow repeated locks even in presence
	351	// of other blocked threads.
318	352	static constexpr u64 kCounterWidth = 20;
319	353	static constexpr u64 kReaderLockShift = 0;
320	354	static constexpr u64 kReaderLockInc = 1ull << kReaderLockShift;
...	...	@@ -330,7 +364,11 @@ class MUTEX Mutex : CheckedMutex {
330	364	<< kWaitingWriterShift;
331	365	static constexpr u64 kWriterLock = 1ull << (3 * kCounterWidth);
332	366	static constexpr u64 kWriterSpinWait = 1ull << (3 * kCounterWidth + 1);
	367	static constexpr u64 kReaderSpinWait = 1ull << (3 * kCounterWidth + 2);
	368
	369	static constexpr uptr kMaxSpinIters = 1500;
333	370
	371	Mutex(LinkerInitialized) = delete;
334	372	Mutex(const Mutex &) = delete;
335	373	void operator=(const Mutex &) = delete;
336	374	};
...	...	@@ -338,149 +376,70 @@ class MUTEX Mutex : CheckedMutex {
338	376	void FutexWait(atomic_uint32_t *p, u32 cmp);
339	377	void FutexWake(atomic_uint32_t *p, u32 count);
340	378
341		class MUTEX BlockingMutex {
342		public:
343		explicit constexpr BlockingMutex(LinkerInitialized)
344		: opaque_storage_ {0, }, owner_ {0} {}
345		BlockingMutex();
346		void Lock() ACQUIRE();
347		void Unlock() RELEASE();
348
349		// This function does not guarantee an explicit check that the calling thread
350		// is the thread which owns the mutex. This behavior, while more strictly
351		// correct, causes problems in cases like StopTheWorld, where a parent thread
352		// owns the mutex but a child checks that it is locked. Rather than
353		// maintaining complex state to work around those situations, the check only
354		// checks that the mutex is owned, and assumes callers to be generally
355		// well-behaved.
356		void CheckLocked() const CHECK_LOCKED();
357
358		private:
359		// Solaris mutex_t has a member that requires 64-bit alignment.
360		ALIGNED(8) uptr opaque_storage_[10];
361		uptr owner_; // for debugging
362		};
363
364		// Reader-writer spin mutex.
365		class MUTEX RWMutex {
	379	template <typename MutexType>
	380	class SANITIZER_SCOPED_LOCK GenericScopedLock {
366	381	public:
367		RWMutex() {
368		atomic_store(&state_, kUnlocked, memory_order_relaxed);
369		}
370
371		~RWMutex() {
372		CHECK_EQ(atomic_load(&state_, memory_order_relaxed), kUnlocked);
373		}
374
375		void Lock() ACQUIRE() {
376		u32 cmp = kUnlocked;
377		if (atomic_compare_exchange_strong(&state_, &cmp, kWriteLock,
378		memory_order_acquire))
379		return;
380		LockSlow();
381		}
382
383		void Unlock() RELEASE() {
384		u32 prev = atomic_fetch_sub(&state_, kWriteLock, memory_order_release);
385		DCHECK_NE(prev & kWriteLock, 0);
386		(void)prev;
387		}
388
389		void ReadLock() ACQUIRE_SHARED() {
390		u32 prev = atomic_fetch_add(&state_, kReadLock, memory_order_acquire);
391		if ((prev & kWriteLock) == 0)
392		return;
393		ReadLockSlow();
394		}
395
396		void ReadUnlock() RELEASE_SHARED() {
397		u32 prev = atomic_fetch_sub(&state_, kReadLock, memory_order_release);
398		DCHECK_EQ(prev & kWriteLock, 0);
399		DCHECK_GT(prev & ~kWriteLock, 0);
400		(void)prev;
	382	explicit GenericScopedLock(MutexType *mu) SANITIZER_ACQUIRE(mu) : mu_(mu) {
	383	mu_->Lock();
401	384	}
402	385
403		void CheckLocked() const CHECK_LOCKED() {
404		CHECK_NE(atomic_load(&state_, memory_order_relaxed), kUnlocked);
405		}
	386	~GenericScopedLock() SANITIZER_RELEASE() { mu_->Unlock(); }
406	387
407	388	private:
408		atomic_uint32_t state_;
409
410		enum {
411		kUnlocked = 0,
412		kWriteLock = 1,
413		kReadLock = 2
414		};
415
416		void NOINLINE LockSlow() {
417		for (int i = 0;; i++) {
418		if (i < 10)
419		proc_yield(10);
420		else
421		internal_sched_yield();
422		u32 cmp = atomic_load(&state_, memory_order_relaxed);
423		if (cmp == kUnlocked &&
424		atomic_compare_exchange_weak(&state_, &cmp, kWriteLock,
425		memory_order_acquire))
426		return;
427		}
428		}
429
430		void NOINLINE ReadLockSlow() {
431		for (int i = 0;; i++) {
432		if (i < 10)
433		proc_yield(10);
434		else
435		internal_sched_yield();
436		u32 prev = atomic_load(&state_, memory_order_acquire);
437		if ((prev & kWriteLock) == 0)
438		return;
439		}
440		}
	389	MutexType *mu_;
441	390
442		RWMutex(const RWMutex &) = delete;
443		void operator=(const RWMutex &) = delete;
	391	GenericScopedLock(const GenericScopedLock &) = delete;
	392	void operator=(const GenericScopedLock &) = delete;
444	393	};
445	394
446	395	template <typename MutexType>
447		class SCOPED_LOCK GenericScopedLock {
	396	class SANITIZER_SCOPED_LOCK GenericScopedReadLock {
448	397	public:
449		explicit GenericScopedLock(MutexType *mu) ACQUIRE(mu) : mu_(mu) {
450		mu_->Lock();
	398	explicit GenericScopedReadLock(MutexType *mu) SANITIZER_ACQUIRE(mu)
	399	: mu_(mu) {
	400	mu_->ReadLock();
451	401	}
452	402
453		~GenericScopedLock() RELEASE() { mu_->Unlock(); }
	403	~GenericScopedReadLock() SANITIZER_RELEASE() { mu_->ReadUnlock(); }
454	404
455	405	private:
456	406	MutexType *mu_;
457	407
458		GenericScopedLock(const GenericScopedLock &) = delete;
459		void operator=(const GenericScopedLock &) = delete;
	408	GenericScopedReadLock(const GenericScopedReadLock &) = delete;
	409	void operator=(const GenericScopedReadLock &) = delete;
460	410	};
461	411
462	412	template <typename MutexType>
463		class SCOPED_LOCK GenericScopedReadLock {
	413	class SANITIZER_SCOPED_LOCK GenericScopedRWLock {
464	414	public:
465		explicit GenericScopedReadLock(MutexType *mu) ACQUIRE(mu) : mu_(mu) {
466		mu_->ReadLock();
	415	ALWAYS_INLINE explicit GenericScopedRWLock(MutexType *mu, bool write)
	416	SANITIZER_ACQUIRE(mu)
	417	: mu_(mu), write_(write) {
	418	if (write_)
	419	mu_->Lock();
	420	else
	421	mu_->ReadLock();
467	422	}
468	423
469		~GenericScopedReadLock() RELEASE() { mu_->ReadUnlock(); }
	424	ALWAYS_INLINE ~GenericScopedRWLock() SANITIZER_RELEASE() {
	425	if (write_)
	426	mu_->Unlock();
	427	else
	428	mu_->ReadUnlock();
	429	}
470	430
471	431	private:
472	432	MutexType *mu_;
	433	bool write_;
473	434
474		GenericScopedReadLock(const GenericScopedReadLock &) = delete;
475		void operator=(const GenericScopedReadLock &) = delete;
	435	GenericScopedRWLock(const GenericScopedRWLock &) = delete;
	436	void operator=(const GenericScopedRWLock &) = delete;
476	437	};
477	438
478	439	typedef GenericScopedLock<StaticSpinMutex> SpinMutexLock;
479		typedef GenericScopedLock<BlockingMutex> BlockingMutexLock;
480		typedef GenericScopedLock<RWMutex> RWMutexLock;
481		typedef GenericScopedReadLock<RWMutex> RWMutexReadLock;
482	440	typedef GenericScopedLock<Mutex> Lock;
483	441	typedef GenericScopedReadLock<Mutex> ReadLock;
	442	typedef GenericScopedRWLock<Mutex> RWLock;
484	443
485	444	} // namespace __sanitizer
486	445

lib/tsan/sanitizer_common/sanitizer_openbsd.cpp deleted

lib/tsan/sanitizer_common/sanitizer_persistent_allocator.cpp deleted-18

...	...	@@ -1,18 +0,0 @@
1		//===-- sanitizer_persistent_allocator.cpp ----------------------- C++ --===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// This file is shared between AddressSanitizer and ThreadSanitizer
10		// run-time libraries.
11		//===----------------------------------------------------------------------===//
12		#include "sanitizer_persistent_allocator.h"
13
14		namespace __sanitizer {
15
16		PersistentAllocator thePersistentAllocator;
17
18		} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_persistent_allocator.h deleted-71

...	...	@@ -1,71 +0,0 @@
1		//===-- sanitizer_persistent_allocator.h ------------------------- C++ --===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// A fast memory allocator that does not support free() nor realloc().
10		// All allocations are forever.
11		//===----------------------------------------------------------------------===//
12
13		#ifndef SANITIZER_PERSISTENT_ALLOCATOR_H
14		#define SANITIZER_PERSISTENT_ALLOCATOR_H
15
16		#include "sanitizer_internal_defs.h"
17		#include "sanitizer_mutex.h"
18		#include "sanitizer_atomic.h"
19		#include "sanitizer_common.h"
20
21		namespace __sanitizer {
22
23		class PersistentAllocator {
24		public:
25		void *alloc(uptr size);
26
27		private:
28		void *tryAlloc(uptr size);
29		StaticSpinMutex mtx; // Protects alloc of new blocks for region allocator.
30		atomic_uintptr_t region_pos; // Region allocator for Node's.
31		atomic_uintptr_t region_end;
32		};
33
34		inline void *PersistentAllocator::tryAlloc(uptr size) {
35		// Optimisic lock-free allocation, essentially try to bump the region ptr.
36		for (;;) {
37		uptr cmp = atomic_load(&region_pos, memory_order_acquire);
38		uptr end = atomic_load(&region_end, memory_order_acquire);
39		if (cmp == 0 \|\| cmp + size > end) return nullptr;
40		if (atomic_compare_exchange_weak(&region_pos, &cmp, cmp + size,
41		memory_order_acquire))
42		return (void *)cmp;
43		}
44		}
45
46		inline void *PersistentAllocator::alloc(uptr size) {
47		// First, try to allocate optimisitically.
48		void *s = tryAlloc(size);
49		if (s) return s;
50		// If failed, lock, retry and alloc new superblock.
51		SpinMutexLock l(&mtx);
52		for (;;) {
53		s = tryAlloc(size);
54		if (s) return s;
55		atomic_store(&region_pos, 0, memory_order_relaxed);
56		uptr allocsz = 64 * 1024;
57		if (allocsz < size) allocsz = size;
58		uptr mem = (uptr)MmapOrDie(allocsz, "stack depot");
59		atomic_store(&region_end, mem + allocsz, memory_order_release);
60		atomic_store(&region_pos, mem, memory_order_release);
61		}
62		}
63
64		extern PersistentAllocator thePersistentAllocator;
65		inline void *PersistentAlloc(uptr sz) {
66		return thePersistentAllocator.alloc(sz);
67		}
68
69		} // namespace __sanitizer
70
71		#endif // SANITIZER_PERSISTENT_ALLOCATOR_H

lib/tsan/sanitizer_common/sanitizer_platform.h+234-166

...	...	@@ -22,103 +22,123 @@
22	22	// function declarations into a .S file which doesn't compile.
23	23	// https://crbug.com/1162741
24	24	#if __has_include(<features.h>) && !defined(__ANDROID__)
25		#include <features.h>
	25	# include <features.h>
26	26	#endif
27	27
28	28	#if defined(__linux__)
29		# define SANITIZER_LINUX 1
	29	# define SANITIZER_LINUX 1
30	30	#else
31		# define SANITIZER_LINUX 0
	31	# define SANITIZER_LINUX 0
32	32	#endif
33	33
34	34	#if defined(__GLIBC__)
35		# define SANITIZER_GLIBC 1
	35	# define SANITIZER_GLIBC 1
36	36	#else
37		# define SANITIZER_GLIBC 0
	37	# define SANITIZER_GLIBC 0
38	38	#endif
39	39
40	40	#if defined(__FreeBSD__)
41		# define SANITIZER_FREEBSD 1
	41	# define SANITIZER_FREEBSD 1
42	42	#else
43		# define SANITIZER_FREEBSD 0
	43	# define SANITIZER_FREEBSD 0
44	44	#endif
45	45
46	46	#if defined(__NetBSD__)
47		# define SANITIZER_NETBSD 1
	47	# define SANITIZER_NETBSD 1
48	48	#else
49		# define SANITIZER_NETBSD 0
	49	# define SANITIZER_NETBSD 0
50	50	#endif
51	51
52	52	#if defined(__sun__) && defined(__svr4__)
53		# define SANITIZER_SOLARIS 1
	53	# define SANITIZER_SOLARIS 1
54	54	#else
55		# define SANITIZER_SOLARIS 0
	55	# define SANITIZER_SOLARIS 0
56	56	#endif
57	57
	58	// - SANITIZER_APPLE: all Apple code
	59	// - TARGET_OS_OSX: macOS
	60	// - SANITIZER_IOS: devices (iOS and iOS-like)
	61	// - SANITIZER_WATCHOS
	62	// - SANITIZER_TVOS
	63	// - SANITIZER_IOSSIM: simulators (iOS and iOS-like)
	64	// - SANITIZER_DRIVERKIT
58	65	#if defined(__APPLE__)
59		# define SANITIZER_MAC 1
60		# include <TargetConditionals.h>
61		# if TARGET_OS_OSX
62		# define SANITIZER_OSX 1
63		# else
64		# define SANITIZER_OSX 0
65		# endif
66		# if TARGET_OS_IPHONE
67		# define SANITIZER_IOS 1
68		# else
69		# define SANITIZER_IOS 0
70		# endif
71		# if TARGET_OS_SIMULATOR
72		# define SANITIZER_IOSSIM 1
73		# else
74		# define SANITIZER_IOSSIM 0
75		# endif
76		#else
77		# define SANITIZER_MAC 0
78		# define SANITIZER_IOS 0
79		# define SANITIZER_IOSSIM 0
80		# define SANITIZER_OSX 0
81		#endif
82
83		#if defined(__APPLE__) && TARGET_OS_IPHONE && TARGET_OS_WATCH
84		# define SANITIZER_WATCHOS 1
85		#else
86		# define SANITIZER_WATCHOS 0
87		#endif
88
89		#if defined(__APPLE__) && TARGET_OS_IPHONE && TARGET_OS_TV
90		# define SANITIZER_TVOS 1
	66	# define SANITIZER_APPLE 1
	67	# include <TargetConditionals.h>
	68	# if TARGET_OS_OSX
	69	# define SANITIZER_OSX 1
	70	# else
	71	# define SANITIZER_OSX 0
	72	# endif
	73	# if TARGET_OS_IPHONE
	74	# define SANITIZER_IOS 1
	75	# else
	76	# define SANITIZER_IOS 0
	77	# endif
	78	# if TARGET_OS_WATCH
	79	# define SANITIZER_WATCHOS 1
	80	# else
	81	# define SANITIZER_WATCHOS 0
	82	# endif
	83	# if TARGET_OS_TV
	84	# define SANITIZER_TVOS 1
	85	# else
	86	# define SANITIZER_TVOS 0
	87	# endif
	88	# if TARGET_OS_SIMULATOR
	89	# define SANITIZER_IOSSIM 1
	90	# else
	91	# define SANITIZER_IOSSIM 0
	92	# endif
	93	# if defined(TARGET_OS_DRIVERKIT) && TARGET_OS_DRIVERKIT
	94	# define SANITIZER_DRIVERKIT 1
	95	# else
	96	# define SANITIZER_DRIVERKIT 0
	97	# endif
91	98	#else
92		# define SANITIZER_TVOS 0
	99	# define SANITIZER_APPLE 0
	100	# define SANITIZER_OSX 0
	101	# define SANITIZER_IOS 0
	102	# define SANITIZER_WATCHOS 0
	103	# define SANITIZER_TVOS 0
	104	# define SANITIZER_IOSSIM 0
	105	# define SANITIZER_DRIVERKIT 0
93	106	#endif
94	107
95	108	#if defined(_WIN32)
96		# define SANITIZER_WINDOWS 1
	109	# define SANITIZER_WINDOWS 1
97	110	#else
98		# define SANITIZER_WINDOWS 0
	111	# define SANITIZER_WINDOWS 0
99	112	#endif
100	113
101	114	#if defined(_WIN64)
102		# define SANITIZER_WINDOWS64 1
	115	# define SANITIZER_WINDOWS64 1
103	116	#else
104		# define SANITIZER_WINDOWS64 0
	117	# define SANITIZER_WINDOWS64 0
105	118	#endif
106	119
107	120	#if defined(__ANDROID__)
108		# define SANITIZER_ANDROID 1
	121	# define SANITIZER_ANDROID 1
109	122	#else
110		# define SANITIZER_ANDROID 0
	123	# define SANITIZER_ANDROID 0
111	124	#endif
112	125
113	126	#if defined(__Fuchsia__)
114		# define SANITIZER_FUCHSIA 1
	127	# define SANITIZER_FUCHSIA 1
115	128	#else
116		# define SANITIZER_FUCHSIA 0
	129	# define SANITIZER_FUCHSIA 0
117	130	#endif
118	131
119		#define SANITIZER_POSIX \
120		(SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_MAC \|\| \
121		SANITIZER_NETBSD \|\| SANITIZER_SOLARIS)
	132	// Assume linux that is not glibc or android is musl libc.
	133	#if SANITIZER_LINUX && !SANITIZER_GLIBC && !SANITIZER_ANDROID
	134	# define SANITIZER_MUSL 1
	135	#else
	136	# define SANITIZER_MUSL 0
	137	#endif
	138
	139	#define SANITIZER_POSIX \
	140	(SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_APPLE \|\| \
	141	SANITIZER_NETBSD \|\| SANITIZER_SOLARIS)
122	142
123	143	#if __LP64__ \|\| defined(_WIN64)
124	144	# define SANITIZER_WORDSIZE 64
...	...	@@ -127,58 +147,79 @@
127	147	#endif
128	148
129	149	#if SANITIZER_WORDSIZE == 64
130		# define FIRST_32_SECOND_64(a, b) (b)
	150	# define FIRST_32_SECOND_64(a, b) (b)
131	151	#else
132		# define FIRST_32_SECOND_64(a, b) (a)
	152	# define FIRST_32_SECOND_64(a, b) (a)
133	153	#endif
134	154
135	155	#if defined(__x86_64__) && !defined(_LP64)
136		# define SANITIZER_X32 1
	156	# define SANITIZER_X32 1
137	157	#else
138		# define SANITIZER_X32 0
	158	# define SANITIZER_X32 0
	159	#endif
	160
	161	#if defined(__x86_64__) \|\| defined(_M_X64)
	162	# define SANITIZER_X64 1
	163	#else
	164	# define SANITIZER_X64 0
139	165	#endif
140	166
141	167	#if defined(__i386__) \|\| defined(_M_IX86)
142		# define SANITIZER_I386 1
	168	# define SANITIZER_I386 1
143	169	#else
144		# define SANITIZER_I386 0
	170	# define SANITIZER_I386 0
145	171	#endif
146	172
147	173	#if defined(__mips__)
148		# define SANITIZER_MIPS 1
149		# if defined(__mips64)
	174	# define SANITIZER_MIPS 1
	175	# if defined(__mips64) && _MIPS_SIM == _ABI64
	176	# define SANITIZER_MIPS32 0
	177	# define SANITIZER_MIPS64 1
	178	# else
	179	# define SANITIZER_MIPS32 1
	180	# define SANITIZER_MIPS64 0
	181	# endif
	182	#else
	183	# define SANITIZER_MIPS 0
150	184	# define SANITIZER_MIPS32 0
151		# define SANITIZER_MIPS64 1
152		# else
153		# define SANITIZER_MIPS32 1
154	185	# define SANITIZER_MIPS64 0
155		# endif
156		#else
157		# define SANITIZER_MIPS 0
158		# define SANITIZER_MIPS32 0
159		# define SANITIZER_MIPS64 0
160	186	#endif
161	187
162	188	#if defined(__s390__)
163		# define SANITIZER_S390 1
164		# if defined(__s390x__)
	189	# define SANITIZER_S390 1
	190	# if defined(__s390x__)
	191	# define SANITIZER_S390_31 0
	192	# define SANITIZER_S390_64 1
	193	# else
	194	# define SANITIZER_S390_31 1
	195	# define SANITIZER_S390_64 0
	196	# endif
	197	#else
	198	# define SANITIZER_S390 0
165	199	# define SANITIZER_S390_31 0
166		# define SANITIZER_S390_64 1
167		# else
168		# define SANITIZER_S390_31 1
169	200	# define SANITIZER_S390_64 0
170		# endif
	201	#endif
	202
	203	#if defined(__sparc__)
	204	# define SANITIZER_SPARC 1
	205	# if defined(__arch64__)
	206	# define SANITIZER_SPARC32 0
	207	# define SANITIZER_SPARC64 1
	208	# else
	209	# define SANITIZER_SPARC32 1
	210	# define SANITIZER_SPARC64 0
	211	# endif
171	212	#else
172		# define SANITIZER_S390 0
173		# define SANITIZER_S390_31 0
174		# define SANITIZER_S390_64 0
	213	# define SANITIZER_SPARC 0
	214	# define SANITIZER_SPARC32 0
	215	# define SANITIZER_SPARC64 0
175	216	#endif
176	217
177	218	#if defined(__powerpc__)
178		# define SANITIZER_PPC 1
179		# if defined(__powerpc64__)
180		# define SANITIZER_PPC32 0
181		# define SANITIZER_PPC64 1
	219	# define SANITIZER_PPC 1
	220	# if defined(__powerpc64__)
	221	# define SANITIZER_PPC32 0
	222	# define SANITIZER_PPC64 1
182	223	// 64-bit PPC has two ABIs (v1 and v2). The old powerpc64 target is
183	224	// big-endian, and uses v1 ABI (known for its function descriptors),
184	225	// while the new powerpc64le target is little-endian and uses v2.
...	...	@@ -186,106 +227,109 @@
186	227	// (eg. big-endian v2), but you won't find such combinations in the wild
187	228	// (it'd require bootstrapping a whole system, which would be quite painful
188	229	// - there's no target triple for that). LLVM doesn't support them either.
189		# if _CALL_ELF == 2
190		# define SANITIZER_PPC64V1 0
191		# define SANITIZER_PPC64V2 1
	230	# if _CALL_ELF == 2
	231	# define SANITIZER_PPC64V1 0
	232	# define SANITIZER_PPC64V2 1
	233	# else
	234	# define SANITIZER_PPC64V1 1
	235	# define SANITIZER_PPC64V2 0
	236	# endif
192	237	# else
193		# define SANITIZER_PPC64V1 1
194		# define SANITIZER_PPC64V2 0
	238	# define SANITIZER_PPC32 1
	239	# define SANITIZER_PPC64 0
	240	# define SANITIZER_PPC64V1 0
	241	# define SANITIZER_PPC64V2 0
195	242	# endif
196		# else
197		# define SANITIZER_PPC32 1
	243	#else
	244	# define SANITIZER_PPC 0
	245	# define SANITIZER_PPC32 0
198	246	# define SANITIZER_PPC64 0
199	247	# define SANITIZER_PPC64V1 0
200	248	# define SANITIZER_PPC64V2 0
201		# endif
	249	#endif
	250
	251	#if defined(__arm__) \|\| defined(_M_ARM)
	252	# define SANITIZER_ARM 1
202	253	#else
203		# define SANITIZER_PPC 0
204		# define SANITIZER_PPC32 0
205		# define SANITIZER_PPC64 0
206		# define SANITIZER_PPC64V1 0
207		# define SANITIZER_PPC64V2 0
	254	# define SANITIZER_ARM 0
208	255	#endif
209	256
210		#if defined(__arm__)
211		# define SANITIZER_ARM 1
	257	#if defined(__aarch64__) \|\| defined(_M_ARM64)
	258	# define SANITIZER_ARM64 1
212	259	#else
213		# define SANITIZER_ARM 0
	260	# define SANITIZER_ARM64 0
214	261	#endif
215	262
216	263	#if SANITIZER_SOLARIS && SANITIZER_WORDSIZE == 32
217		# define SANITIZER_SOLARIS32 1
	264	# define SANITIZER_SOLARIS32 1
218	265	#else
219		# define SANITIZER_SOLARIS32 0
	266	# define SANITIZER_SOLARIS32 0
220	267	#endif
221	268
222	269	#if defined(__riscv) && (__riscv_xlen == 64)
223		#define SANITIZER_RISCV64 1
	270	# define SANITIZER_RISCV64 1
224	271	#else
225		#define SANITIZER_RISCV64 0
	272	# define SANITIZER_RISCV64 0
	273	#endif
	274
	275	#if defined(__loongarch_lp64)
	276	# define SANITIZER_LOONGARCH64 1
	277	#else
	278	# define SANITIZER_LOONGARCH64 0
226	279	#endif
227	280
228	281	// By default we allow to use SizeClassAllocator64 on 64-bit platform.
229		// But in some cases (e.g. AArch64's 39-bit address space) SizeClassAllocator64
230		// does not work well and we need to fallback to SizeClassAllocator32.
	282	// But in some cases SizeClassAllocator64 does not work well and we need to
	283	// fallback to SizeClassAllocator32.
231	284	// For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or
232	285	// change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here.
233	286	#ifndef SANITIZER_CAN_USE_ALLOCATOR64
234		# if (SANITIZER_ANDROID && defined(__aarch64__)) \|\| SANITIZER_FUCHSIA
235		# define SANITIZER_CAN_USE_ALLOCATOR64 1
236		# elif defined(__mips64) \|\| defined(__aarch64__)
237		# define SANITIZER_CAN_USE_ALLOCATOR64 0
238		# else
239		# define SANITIZER_CAN_USE_ALLOCATOR64 (SANITIZER_WORDSIZE == 64)
240		# endif
	287	# if SANITIZER_RISCV64 \|\| SANITIZER_IOS
	288	# define SANITIZER_CAN_USE_ALLOCATOR64 0
	289	# elif defined(__mips64) \|\| defined(__hexagon__)
	290	# define SANITIZER_CAN_USE_ALLOCATOR64 0
	291	# else
	292	# define SANITIZER_CAN_USE_ALLOCATOR64 (SANITIZER_WORDSIZE == 64)
	293	# endif
241	294	#endif
242	295
243	296	// The range of addresses which can be returned my mmap.
244	297	// FIXME: this value should be different on different platforms. Larger values
245	298	// will still work but will consume more memory for TwoLevelByteMap.
246	299	#if defined(__mips__)
247		#if SANITIZER_GO && defined(__mips64)
248		#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
249		#else
250		# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40)
251		#endif
	300	# if SANITIZER_GO && defined(__mips64)
	301	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
	302	# else
	303	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40)
	304	# endif
252	305	#elif SANITIZER_RISCV64
253		#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 38)
	306	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 38)
254	307	#elif defined(__aarch64__)
255		# if SANITIZER_MAC
256		# if SANITIZER_OSX \|\| SANITIZER_IOSSIM
257		# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
	308	# if SANITIZER_APPLE
	309	# if SANITIZER_OSX \|\| SANITIZER_IOSSIM
	310	# define SANITIZER_MMAP_RANGE_SIZE \
	311	FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
	312	# else
	313	// Darwin iOS/ARM64 has a 36-bit VMA, 64GiB VM
	314	# define SANITIZER_MMAP_RANGE_SIZE \
	315	FIRST_32_SECOND_64(1ULL << 32, 1ULL << 36)
	316	# endif
258	317	# else
259		// Darwin iOS/ARM64 has a 36-bit VMA, 64GiB VM
260		# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 36)
	318	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48)
261	319	# endif
262		# else
263		# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48)
264		# endif
265	320	#elif defined(__sparc__)
266		#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 52)
	321	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 52)
267	322	#else
268		# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
	323	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
269	324	#endif
270	325
271	326	// Whether the addresses are sign-extended from the VMA range to the word.
272	327	// The SPARC64 Linux port implements this to split the VMA space into two
273	328	// non-contiguous halves with a huge hole in the middle.
274	329	#if defined(__sparc__) && SANITIZER_WORDSIZE == 64
275		#define SANITIZER_SIGN_EXTENDED_ADDRESSES 1
	330	# define SANITIZER_SIGN_EXTENDED_ADDRESSES 1
276	331	#else
277		#define SANITIZER_SIGN_EXTENDED_ADDRESSES 0
278		#endif
279
280		// The AArch64 and RISC-V linux ports use the canonical syscall set as
281		// mandated by the upstream linux community for all new ports. Other ports
282		// may still use legacy syscalls.
283		#ifndef SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
284		# if (defined(__aarch64__) \|\| defined(__riscv)) && SANITIZER_LINUX
285		# define SANITIZER_USES_CANONICAL_LINUX_SYSCALLS 1
286		# else
287		# define SANITIZER_USES_CANONICAL_LINUX_SYSCALLS 0
288		# endif
	332	# define SANITIZER_SIGN_EXTENDED_ADDRESSES 0
289	333	#endif
290	334
291	335	// udi16 syscalls can only be used when the following conditions are
...	...	@@ -296,15 +340,15 @@
296	340	// Since we don't want to include libc headers here, we check the
297	341	// target only.
298	342	#if defined(__arm__) \|\| SANITIZER_X32 \|\| defined(__sparc__)
299		#define SANITIZER_USES_UID16_SYSCALLS 1
	343	# define SANITIZER_USES_UID16_SYSCALLS 1
300	344	#else
301		#define SANITIZER_USES_UID16_SYSCALLS 0
	345	# define SANITIZER_USES_UID16_SYSCALLS 0
302	346	#endif
303	347
304	348	#if defined(__mips__)
305		# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 10)
	349	# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 10)
306	350	#else
307		# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 12)
	351	# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 12)
308	352	#endif
309	353
310	354	/// \macro MSC_PREREQ
...	...	@@ -313,15 +357,15 @@
313	357	/// * 1800: Microsoft Visual Studio 2013 / 12.0
314	358	/// * 1900: Microsoft Visual Studio 2015 / 14.0
315	359	#ifdef _MSC_VER
316		# define MSC_PREREQ(version) (_MSC_VER >= (version))
	360	# define MSC_PREREQ(version) (_MSC_VER >= (version))
317	361	#else
318		# define MSC_PREREQ(version) 0
	362	# define MSC_PREREQ(version) 0
319	363	#endif
320	364
321		#if SANITIZER_MAC && !(defined(__arm64__) && SANITIZER_IOS)
322		# define SANITIZER_NON_UNIQUE_TYPEINFO 0
	365	#if SANITIZER_APPLE && defined(__x86_64__)
	366	# define SANITIZER_NON_UNIQUE_TYPEINFO 0
323	367	#else
324		# define SANITIZER_NON_UNIQUE_TYPEINFO 1
	368	# define SANITIZER_NON_UNIQUE_TYPEINFO 1
325	369	#endif
326	370
327	371	// On linux, some architectures had an ABI transition from 64-bit long double
...	...	@@ -329,11 +373,11 @@
329	373	// involving long doubles come in two versions, and we need to pass the
330	374	// correct one to dlvsym when intercepting them.
331	375	#if SANITIZER_LINUX && (SANITIZER_S390 \|\| SANITIZER_PPC32 \|\| SANITIZER_PPC64V1)
332		#define SANITIZER_NLDBL_VERSION "GLIBC_2.4"
	376	# define SANITIZER_NLDBL_VERSION "GLIBC_2.4"
333	377	#endif
334	378
335	379	#if SANITIZER_GO == 0
336		# define SANITIZER_GO 0
	380	# define SANITIZER_GO 0
337	381	#endif
338	382
339	383	// On PowerPC and ARM Thumb, calling pthread_exit() causes LSan to detect leaks.
...	...	@@ -341,40 +385,64 @@
341	385	// dlopen mallocs "libgcc_s.so" string which confuses LSan, it fails to realize
342	386	// that this allocation happens in dynamic linker and should be ignored.
343	387	#if SANITIZER_PPC \|\| defined(__thumb__)
344		# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 1
	388	# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 1
345	389	#else
346		# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 0
	390	# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 0
347	391	#endif
348	392
349		#if SANITIZER_FREEBSD \|\| SANITIZER_MAC \|\| SANITIZER_NETBSD \|\| \
350		SANITIZER_SOLARIS
351		# define SANITIZER_MADVISE_DONTNEED MADV_FREE
	393	#if SANITIZER_FREEBSD \|\| SANITIZER_APPLE \|\| SANITIZER_NETBSD \|\| SANITIZER_SOLARIS
	394	# define SANITIZER_MADVISE_DONTNEED MADV_FREE
352	395	#else
353		# define SANITIZER_MADVISE_DONTNEED MADV_DONTNEED
	396	# define SANITIZER_MADVISE_DONTNEED MADV_DONTNEED
354	397	#endif
355	398
356	399	// Older gcc have issues aligning to a constexpr, and require an integer.
357	400	// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=56859 among others.
358	401	#if defined(__powerpc__) \|\| defined(__powerpc64__)
359		# define SANITIZER_CACHE_LINE_SIZE 128
	402	# define SANITIZER_CACHE_LINE_SIZE 128
360	403	#else
361		# define SANITIZER_CACHE_LINE_SIZE 64
	404	# define SANITIZER_CACHE_LINE_SIZE 64
362	405	#endif
363	406
364	407	// Enable offline markup symbolizer for Fuchsia.
365	408	#if SANITIZER_FUCHSIA
366	409	# define SANITIZER_SYMBOLIZER_MARKUP 1
367	410	#else
368		#define SANITIZER_SYMBOLIZER_MARKUP 0
	411	# define SANITIZER_SYMBOLIZER_MARKUP 0
369	412	#endif
370	413
371	414	// Enable ability to support sanitizer initialization that is
372	415	// compatible with the sanitizer library being loaded via
373	416	// `dlopen()`.
374		#if SANITIZER_MAC
375		#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 1
	417	#if SANITIZER_APPLE
	418	# define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 1
	419	#else
	420	# define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 0
	421	#endif
	422
	423	// SANITIZER_SUPPORTS_THREADLOCAL
	424	// 1 - THREADLOCAL macro is supported by target
	425	// 0 - THREADLOCAL macro is not supported by target
	426	#ifndef __has_feature
	427	// TODO: Support other compilers here
	428	# define SANITIZER_SUPPORTS_THREADLOCAL 1
	429	#else
	430	# if __has_feature(tls)
	431	# define SANITIZER_SUPPORTS_THREADLOCAL 1
	432	# else
	433	# define SANITIZER_SUPPORTS_THREADLOCAL 0
	434	# endif
	435	#endif
	436
	437	#if defined(__thumb__) && defined(__linux__)
	438	// Workaround for
	439	// https://lab.llvm.org/buildbot/#/builders/clang-thumbv7-full-2stage
	440	// or
	441	// https://lab.llvm.org/staging/#/builders/clang-thumbv7-full-2stage
	442	// It fails rss_limit_mb_test without meaningful errors.
	443	# define SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL 1
376	444	#else
377		#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 0
	445	# define SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL 0
378	446	#endif
379	447
380		#endif // SANITIZER_PLATFORM_H
	448	#endif // SANITIZER_PLATFORM_H

lib/tsan/sanitizer_common/sanitizer_platform_interceptors.h+53-34

...	...	@@ -76,7 +76,7 @@
76	76	#define SI_LINUX 0
77	77	#endif
78	78
79		#if SANITIZER_MAC
	79	#if SANITIZER_APPLE
80	80	#define SI_MAC 1
81	81	#define SI_NOT_MAC 0
82	82	#else
...	...	@@ -126,7 +126,7 @@
126	126	#define SI_SOLARIS32 0
127	127	#endif
128	128
129		#if SANITIZER_POSIX && !SANITIZER_MAC
	129	#if SANITIZER_POSIX && !SANITIZER_APPLE
130	130	#define SI_POSIX_NOT_MAC 1
131	131	#else
132	132	#define SI_POSIX_NOT_MAC 0
...	...	@@ -229,11 +229,15 @@
229	229	(SI_MAC \|\| SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS)
230	230	#define SANITIZER_INTERCEPT_CLOCK_GETTIME \
231	231	(SI_FREEBSD \|\| SI_NETBSD \|\| SI_LINUX \|\| SI_SOLARIS)
232		#define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID SI_LINUX
	232	#define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID \
	233	(SI_LINUX \|\| SI_FREEBSD \|\| SI_NETBSD)
233	234	#define SANITIZER_INTERCEPT_GETITIMER SI_POSIX
234	235	#define SANITIZER_INTERCEPT_TIME SI_POSIX
235	236	#define SANITIZER_INTERCEPT_GLOB (SI_GLIBC \|\| SI_SOLARIS)
236	237	#define SANITIZER_INTERCEPT_GLOB64 SI_GLIBC
	238	#define SANITIZER_INTERCEPT___B64_TO SI_LINUX_NOT_ANDROID
	239	#define SANITIZER_INTERCEPT_DN_COMP_EXPAND SI_LINUX_NOT_ANDROID
	240	#define SANITIZER_INTERCEPT_POSIX_SPAWN SI_POSIX
237	241	#define SANITIZER_INTERCEPT_WAIT SI_POSIX
238	242	#define SANITIZER_INTERCEPT_INET SI_POSIX
239	243	#define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM SI_POSIX
...	...	@@ -251,7 +255,8 @@
251	255	#define SANITIZER_INTERCEPT_GETHOSTENT_R (SI_FREEBSD \|\| SI_GLIBC \|\| SI_SOLARIS)
252	256	#define SANITIZER_INTERCEPT_GETSOCKOPT SI_POSIX
253	257	#define SANITIZER_INTERCEPT_ACCEPT SI_POSIX
254		#define SANITIZER_INTERCEPT_ACCEPT4 (SI_LINUX_NOT_ANDROID \|\| SI_NETBSD)
	258	#define SANITIZER_INTERCEPT_ACCEPT4 \
	259	(SI_LINUX_NOT_ANDROID \|\| SI_NETBSD \|\| SI_FREEBSD)
255	260	#define SANITIZER_INTERCEPT_PACCEPT SI_NETBSD
256	261	#define SANITIZER_INTERCEPT_MODF SI_POSIX
257	262	#define SANITIZER_INTERCEPT_RECVMSG SI_POSIX
...	...	@@ -264,11 +269,11 @@
264	269	#define SANITIZER_INTERCEPT_INET_ATON SI_POSIX
265	270	#define SANITIZER_INTERCEPT_SYSINFO SI_LINUX
266	271	#define SANITIZER_INTERCEPT_READDIR SI_POSIX
267		#define SANITIZER_INTERCEPT_READDIR64 SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS32
	272	#define SANITIZER_INTERCEPT_READDIR64 SI_GLIBC \|\| SI_SOLARIS32
268	273	#if SI_LINUX_NOT_ANDROID && \
269	274	(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \
270	275	defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__arm__) \|\| \
271		defined(__s390__) \|\| SANITIZER_RISCV64)
	276	defined(__s390__) \|\| defined(__loongarch__) \|\| SANITIZER_RISCV64)
272	277	#define SANITIZER_INTERCEPT_PTRACE 1
273	278	#else
274	279	#define SANITIZER_INTERCEPT_PTRACE 0
...	...	@@ -303,13 +308,13 @@
303	308	#define SANITIZER_INTERCEPT_XPG_STRERROR_R SI_LINUX_NOT_ANDROID
304	309	#define SANITIZER_INTERCEPT_SCANDIR \
305	310	(SI_FREEBSD \|\| SI_NETBSD \|\| SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS)
306		#define SANITIZER_INTERCEPT_SCANDIR64 SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS32
	311	#define SANITIZER_INTERCEPT_SCANDIR64 SI_GLIBC \|\| SI_SOLARIS32
307	312	#define SANITIZER_INTERCEPT_GETGROUPS SI_POSIX
308	313	#define SANITIZER_INTERCEPT_POLL SI_POSIX
309	314	#define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS
310	315	#define SANITIZER_INTERCEPT_WORDEXP \
311	316	(SI_FREEBSD \|\| SI_NETBSD \|\| (SI_MAC && !SI_IOS) \|\| SI_LINUX_NOT_ANDROID \|\| \
312		SI_SOLARIS) // NOLINT
	317	SI_SOLARIS)
313	318	#define SANITIZER_INTERCEPT_SIGWAIT SI_POSIX
314	319	#define SANITIZER_INTERCEPT_SIGWAITINFO SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS
315	320	#define SANITIZER_INTERCEPT_SIGTIMEDWAIT SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS
...	...	@@ -325,11 +330,10 @@
325	330	#define SANITIZER_INTERCEPT_GETMNTENT_R SI_LINUX_NOT_ANDROID
326	331	#define SANITIZER_INTERCEPT_STATFS \
327	332	(SI_FREEBSD \|\| SI_MAC \|\| SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS)
328		#define SANITIZER_INTERCEPT_STATFS64 \
329		(((SI_MAC && !TARGET_CPU_ARM64) && !SI_IOS) \|\| SI_LINUX_NOT_ANDROID)
	333	#define SANITIZER_INTERCEPT_STATFS64 SI_GLIBC && SANITIZER_HAS_STATFS64
330	334	#define SANITIZER_INTERCEPT_STATVFS \
331	335	(SI_FREEBSD \|\| SI_NETBSD \|\| SI_LINUX_NOT_ANDROID)
332		#define SANITIZER_INTERCEPT_STATVFS64 SI_LINUX_NOT_ANDROID
	336	#define SANITIZER_INTERCEPT_STATVFS64 SI_GLIBC
333	337	#define SANITIZER_INTERCEPT_INITGROUPS SI_POSIX
334	338	#define SANITIZER_INTERCEPT_ETHER_NTOA_ATON SI_POSIX
335	339	#define SANITIZER_INTERCEPT_ETHER_HOST \
...	...	@@ -337,12 +341,14 @@
337	341	#define SANITIZER_INTERCEPT_ETHER_R (SI_FREEBSD \|\| SI_LINUX_NOT_ANDROID)
338	342	#define SANITIZER_INTERCEPT_SHMCTL \
339	343	(((SI_FREEBSD \|\| SI_LINUX_NOT_ANDROID) && SANITIZER_WORDSIZE == 64) \|\| \
340		SI_NETBSD \|\| SI_SOLARIS) // NOLINT
	344	SI_NETBSD \|\| SI_SOLARIS)
341	345	#define SANITIZER_INTERCEPT_RANDOM_R SI_GLIBC
342	346	#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_POSIX
343	347	#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \
344	348	(SI_FREEBSD \|\| SI_NETBSD \|\| SI_MAC \|\| SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS)
345	349	#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP SI_GLIBC
	350	#define SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP \
	351	(SI_LINUX_NOT_ANDROID \|\| SI_FREEBSD)
346	352	#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET_SCHED SI_POSIX
347	353	#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED \
348	354	(SI_POSIX && !SI_NETBSD)
...	...	@@ -362,6 +368,8 @@
362	368	(SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS)
363	369	#define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED \
364	370	(SI_LINUX_NOT_ANDROID && !SI_NETBSD)
	371	#define SANITIZER_INTERCEPT_TRYJOIN SI_GLIBC
	372	#define SANITIZER_INTERCEPT_TIMEDJOIN SI_GLIBC
365	373	#define SANITIZER_INTERCEPT_THR_EXIT SI_FREEBSD
366	374	#define SANITIZER_INTERCEPT_TMPNAM SI_POSIX
367	375	#define SANITIZER_INTERCEPT_TMPNAM_R (SI_GLIBC \|\| SI_SOLARIS)
...	...	@@ -391,8 +399,6 @@
391	399	#define SANITIZER_INTERCEPT__EXIT \
392	400	(SI_LINUX \|\| SI_FREEBSD \|\| SI_NETBSD \|\| SI_MAC \|\| SI_SOLARIS)
393	401
394		#define SANITIZER_INTERCEPT_PTHREAD_MUTEX SI_POSIX
395		#define SANITIZER_INTERCEPT___PTHREAD_MUTEX SI_GLIBC
396	402	#define SANITIZER_INTERCEPT___LIBC_MUTEX SI_NETBSD
397	403	#define SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP \
398	404	(SI_FREEBSD \|\| SI_NETBSD \|\| SI_GLIBC \|\| SI_SOLARIS)
...	...	@@ -400,7 +406,7 @@
400	406	(SI_FREEBSD \|\| SI_NETBSD \|\| SI_GLIBC \|\| SI_SOLARIS)
401	407
402	408	#define SANITIZER_INTERCEPT_TLS_GET_ADDR \
403		(SI_FREEBSD \|\| SI_NETBSD \|\| SI_LINUX_NOT_ANDROID \|\| SI_SOLARIS)
	409	(SI_FREEBSD \|\| SI_NETBSD \|\| SI_LINUX_NOT_ANDROID)
404	410
405	411	#define SANITIZER_INTERCEPT_LISTXATTR SI_LINUX
406	412	#define SANITIZER_INTERCEPT_GETXATTR SI_LINUX
...	...	@@ -445,7 +451,8 @@
445	451	#define SANITIZER_INTERCEPT_SEM \
446	452	(SI_LINUX \|\| SI_FREEBSD \|\| SI_NETBSD \|\| SI_SOLARIS)
447	453	#define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL SI_POSIX
448		#define SANITIZER_INTERCEPT_MINCORE (SI_LINUX \|\| SI_NETBSD \|\| SI_SOLARIS)
	454	#define SANITIZER_INTERCEPT_MINCORE \
	455	(SI_LINUX \|\| SI_NETBSD \|\| SI_FREEBSD \|\| SI_SOLARIS)
449	456	#define SANITIZER_INTERCEPT_PROCESS_VM_READV SI_LINUX
450	457	#define SANITIZER_INTERCEPT_CTERMID \
451	458	(SI_LINUX \|\| SI_MAC \|\| SI_FREEBSD \|\| SI_NETBSD \|\| SI_SOLARIS)
...	...	@@ -457,13 +464,17 @@
457	464	#define SANITIZER_INTERCEPT_SEND_SENDTO SI_POSIX
458	465	#define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE SI_LINUX
459	466
460		#define SANITIZER_INTERCEPT_STAT \
461		(SI_FREEBSD \|\| SI_MAC \|\| SI_ANDROID \|\| SI_NETBSD \|\| SI_SOLARIS)
462		#define SANITIZER_INTERCEPT_LSTAT (SI_NETBSD \|\| SI_FREEBSD)
463		#define SANITIZER_INTERCEPT___XSTAT (!SANITIZER_INTERCEPT_STAT && SI_POSIX)
464		#define SANITIZER_INTERCEPT___XSTAT64 SI_LINUX_NOT_ANDROID
	467	#define SI_STAT_LINUX (SI_LINUX && __GLIBC_PREREQ(2, 33))
	468	#define SANITIZER_INTERCEPT_STAT \
	469	(SI_FREEBSD \|\| SI_MAC \|\| SI_ANDROID \|\| SI_NETBSD \|\| SI_SOLARIS \|\| \
	470	SI_STAT_LINUX)
	471	#define SANITIZER_INTERCEPT_STAT64 SI_STAT_LINUX && SANITIZER_HAS_STAT64
	472	#define SANITIZER_INTERCEPT_LSTAT (SI_NETBSD \|\| SI_FREEBSD \|\| SI_STAT_LINUX)
	473	#define SANITIZER_INTERCEPT___XSTAT \
	474	((!SANITIZER_INTERCEPT_STAT && SI_POSIX) \|\| SI_STAT_LINUX)
	475	#define SANITIZER_INTERCEPT___XSTAT64 SI_GLIBC
465	476	#define SANITIZER_INTERCEPT___LXSTAT SANITIZER_INTERCEPT___XSTAT
466		#define SANITIZER_INTERCEPT___LXSTAT64 SI_LINUX_NOT_ANDROID
	477	#define SANITIZER_INTERCEPT___LXSTAT64 SI_GLIBC
467	478
468	479	#define SANITIZER_INTERCEPT_UTMP \
469	480	(SI_POSIX && !SI_MAC && !SI_FREEBSD && !SI_NETBSD)
...	...	@@ -474,7 +485,7 @@
474	485	(SI_LINUX_NOT_ANDROID \|\| SI_MAC \|\| SI_FREEBSD \|\| SI_NETBSD)
475	486
476	487	#define SANITIZER_INTERCEPT_MMAP SI_POSIX
477		#define SANITIZER_INTERCEPT_MMAP64 SI_LINUX_NOT_ANDROID
	488	#define SANITIZER_INTERCEPT_MMAP64 SI_GLIBC \|\| SI_SOLARIS
478	489	#define SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO (SI_GLIBC \|\| SI_ANDROID)
479	490	#define SANITIZER_INTERCEPT_MEMALIGN (!SI_FREEBSD && !SI_MAC && !SI_NETBSD)
480	491	#define SANITIZER_INTERCEPT___LIBC_MEMALIGN SI_GLIBC
...	...	@@ -484,6 +495,7 @@
484	495	#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC)
485	496	#define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD)
486	497	#define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID
	498	#define SANITIZER_INTERCEPT_WCSLEN 1
487	499	#define SANITIZER_INTERCEPT_WCSCAT SI_POSIX
488	500	#define SANITIZER_INTERCEPT_WCSDUP SI_POSIX
489	501	#define SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION (!SI_WINDOWS && SI_NOT_FUCHSIA)
...	...	@@ -496,7 +508,8 @@
496	508	#define SANITIZER_INTERCEPT_GID_FROM_GROUP SI_NETBSD
497	509	#define SANITIZER_INTERCEPT_ACCESS (SI_NETBSD \|\| SI_FREEBSD)
498	510	#define SANITIZER_INTERCEPT_FACCESSAT (SI_NETBSD \|\| SI_FREEBSD)
499		#define SANITIZER_INTERCEPT_GETGROUPLIST SI_NETBSD
	511	#define SANITIZER_INTERCEPT_GETGROUPLIST \
	512	(SI_NETBSD \|\| SI_FREEBSD \|\| SI_LINUX)
500	513	#define SANITIZER_INTERCEPT_STRLCPY \
501	514	(SI_NETBSD \|\| SI_FREEBSD \|\| SI_MAC \|\| SI_ANDROID)
502	515
...	...	@@ -517,10 +530,11 @@
517	530	#define SANITIZER_INTERCEPT_DEVNAME_R (SI_NETBSD \|\| SI_FREEBSD)
518	531	#define SANITIZER_INTERCEPT_FGETLN (SI_NETBSD \|\| SI_FREEBSD)
519	532	#define SANITIZER_INTERCEPT_STRMODE (SI_NETBSD \|\| SI_FREEBSD)
520		#define SANITIZER_INTERCEPT_TTYENT SI_NETBSD
521		#define SANITIZER_INTERCEPT_PROTOENT (SI_NETBSD \|\| SI_LINUX)
	533	#define SANITIZER_INTERCEPT_TTYENT (SI_NETBSD \|\| SI_FREEBSD)
	534	#define SANITIZER_INTERCEPT_TTYENTPATH SI_NETBSD
	535	#define SANITIZER_INTERCEPT_PROTOENT (SI_LINUX \|\| SI_NETBSD \|\| SI_FREEBSD)
522	536	#define SANITIZER_INTERCEPT_PROTOENT_R SI_GLIBC
523		#define SANITIZER_INTERCEPT_NETENT SI_NETBSD
	537	#define SANITIZER_INTERCEPT_NETENT (SI_LINUX \|\| SI_NETBSD \|\| SI_FREEBSD)
524	538	#define SANITIZER_INTERCEPT_SETVBUF \
525	539	(SI_NETBSD \|\| SI_FREEBSD \|\| SI_LINUX \|\| SI_MAC)
526	540	#define SANITIZER_INTERCEPT_GETMNTINFO (SI_NETBSD \|\| SI_FREEBSD \|\| SI_MAC)
...	...	@@ -536,17 +550,17 @@
536	550	#define SANITIZER_INTERCEPT_MODCTL SI_NETBSD
537	551	#define SANITIZER_INTERCEPT_CAPSICUM SI_FREEBSD
538	552	#define SANITIZER_INTERCEPT_STRTONUM (SI_NETBSD \|\| SI_FREEBSD)
539		#define SANITIZER_INTERCEPT_FPARSELN SI_NETBSD
	553	#define SANITIZER_INTERCEPT_FPARSELN (SI_NETBSD \|\| SI_FREEBSD)
540	554	#define SANITIZER_INTERCEPT_STATVFS1 SI_NETBSD
541	555	#define SANITIZER_INTERCEPT_STRTOI SI_NETBSD
542	556	#define SANITIZER_INTERCEPT_CAPSICUM SI_FREEBSD
543	557	#define SANITIZER_INTERCEPT_SHA1 SI_NETBSD
544	558	#define SANITIZER_INTERCEPT_MD4 SI_NETBSD
545	559	#define SANITIZER_INTERCEPT_RMD160 SI_NETBSD
546		#define SANITIZER_INTERCEPT_MD5 SI_NETBSD
	560	#define SANITIZER_INTERCEPT_MD5 (SI_NETBSD \|\| SI_FREEBSD)
547	561	#define SANITIZER_INTERCEPT_FSEEK (SI_NETBSD \|\| SI_FREEBSD)
548	562	#define SANITIZER_INTERCEPT_MD2 SI_NETBSD
549		#define SANITIZER_INTERCEPT_SHA2 SI_NETBSD
	563	#define SANITIZER_INTERCEPT_SHA2 (SI_NETBSD \|\| SI_FREEBSD)
550	564	#define SANITIZER_INTERCEPT_CDB SI_NETBSD
551	565	#define SANITIZER_INTERCEPT_VIS (SI_NETBSD \|\| SI_FREEBSD)
552	566	#define SANITIZER_INTERCEPT_POPEN SI_POSIX
...	...	@@ -559,25 +573,30 @@
559	573	#define SANITIZER_INTERCEPT_FDEVNAME SI_FREEBSD
560	574	#define SANITIZER_INTERCEPT_GETUSERSHELL (SI_POSIX && !SI_ANDROID)
561	575	#define SANITIZER_INTERCEPT_SL_INIT (SI_FREEBSD \|\| SI_NETBSD)
562		#define SANITIZER_INTERCEPT_CRYPT (SI_POSIX && !SI_ANDROID)
563		#define SANITIZER_INTERCEPT_CRYPT_R (SI_LINUX && !SI_ANDROID)
564	576
565	577	#define SANITIZER_INTERCEPT_GETRANDOM \
566	578	((SI_LINUX && __GLIBC_PREREQ(2, 25)) \|\| SI_FREEBSD)
567	579	#define SANITIZER_INTERCEPT___CXA_ATEXIT SI_NETBSD
568	580	#define SANITIZER_INTERCEPT_ATEXIT SI_NETBSD
569	581	#define SANITIZER_INTERCEPT_PTHREAD_ATFORK SI_NETBSD
570		#define SANITIZER_INTERCEPT_GETENTROPY SI_FREEBSD
	582	#define SANITIZER_INTERCEPT_GETENTROPY \
	583	((SI_LINUX && __GLIBC_PREREQ(2, 25)) \|\| SI_FREEBSD)
571	584	#define SANITIZER_INTERCEPT_QSORT \
572	585	(SI_POSIX && !SI_IOSSIM && !SI_WATCHOS && !SI_TVOS && !SI_ANDROID)
573	586	#define SANITIZER_INTERCEPT_QSORT_R SI_GLIBC
	587	#define SANITIZER_INTERCEPT_BSEARCH \
	588	(SI_POSIX && !SI_IOSSIM && !SI_WATCHOS && !SI_TVOS && !SI_ANDROID)
574	589	// sigaltstack on i386 macOS cannot be intercepted due to setjmp()
575	590	// calling it and assuming that it does not clobber registers.
576	591	#define SANITIZER_INTERCEPT_SIGALTSTACK \
577		(SI_POSIX && !(SANITIZER_MAC && SANITIZER_I386))
	592	(SI_POSIX && !(SANITIZER_APPLE && SANITIZER_I386))
578	593	#define SANITIZER_INTERCEPT_UNAME (SI_POSIX && !SI_FREEBSD)
579	594	#define SANITIZER_INTERCEPT___XUNAME SI_FREEBSD
580	595	#define SANITIZER_INTERCEPT_FLOPEN SI_FREEBSD
	596	#define SANITIZER_INTERCEPT_PROCCTL SI_FREEBSD
	597	#define SANITIZER_INTERCEPT_HEXDUMP SI_FREEBSD
	598	#define SANITIZER_INTERCEPT_ARGP_PARSE SI_GLIBC
	599	#define SANITIZER_INTERCEPT_CPUSET_GETAFFINITY SI_FREEBSD
581	600
582	601	// This macro gives a way for downstream users to override the above
583	602	// interceptor macros irrespective of the platform they are on. They have

lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.cpp+39-1

...	...	@@ -17,6 +17,7 @@
17	17
18	18	#include <sys/capsicum.h>
19	19	#include <sys/consio.h>
	20	#include <sys/cpuset.h>
20	21	#include <sys/filio.h>
21	22	#include <sys/ipc.h>
22	23	#include <sys/kbio.h>
...	...	@@ -69,11 +70,17 @@
69	70	#include <semaphore.h>
70	71	#include <signal.h>
71	72	#include <stddef.h>
	73	#include <md5.h>
	74	#include <sha224.h>
	75	#include <sha256.h>
	76	#include <sha384.h>
	77	#include <sha512.h>
72	78	#include <stdio.h>
73	79	#include <stringlist.h>
74	80	#include <term.h>
75	81	#include <termios.h>
76	82	#include <time.h>
	83	#include <ttyent.h>
77	84	#include <utime.h>
78	85	#include <utmpx.h>
79	86	#include <vis.h>
...	...	@@ -97,6 +104,7 @@ void __sanitizer_get_link_map_by_dlopen_handle(void handle) {
97	104	return internal_dlinfo(handle, RTLD_DI_LINKMAP, &p) == 0 ? p : nullptr;
98	105	}
99	106
	107	unsigned struct_cpuset_sz = sizeof(cpuset_t);
100	108	unsigned struct_cap_rights_sz = sizeof(cap_rights_t);
101	109	unsigned struct_utsname_sz = sizeof(struct utsname);
102	110	unsigned struct_stat_sz = sizeof(struct stat);
...	...	@@ -124,7 +132,7 @@ unsigned struct_sigevent_sz = sizeof(struct sigevent);
124	132	unsigned struct_sched_param_sz = sizeof(struct sched_param);
125	133	unsigned struct_statfs_sz = sizeof(struct statfs);
126	134	unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
127		unsigned ucontext_t_sz = sizeof(ucontext_t);
	135	unsigned ucontext_t_sz(void *ctx) { return sizeof(ucontext_t); }
128	136	unsigned struct_rlimit_sz = sizeof(struct rlimit);
129	137	unsigned struct_timespec_sz = sizeof(struct timespec);
130	138	unsigned struct_utimbuf_sz = sizeof(struct utimbuf);
...	...	@@ -167,12 +175,21 @@ uptr __sanitizer_in_addr_sz(int af) {
167	175	return 0;
168	176	}
169	177
	178	// For FreeBSD the actual size of a directory entry is not always in d_reclen.
	179	// Use the appropriate macro to get the correct size for all cases (e.g. NFS).
	180	u16 __sanitizer_dirsiz(const __sanitizer_dirent *dp) {
	181	return _GENERIC_DIRSIZ(dp);
	182	}
	183
170	184	unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
171	185	int glob_nomatch = GLOB_NOMATCH;
172	186	int glob_altdirfunc = GLOB_ALTDIRFUNC;
	187	const int wordexp_wrde_dooffs = WRDE_DOOFFS;
173	188
174	189	unsigned path_max = PATH_MAX;
175	190
	191	int struct_ttyent_sz = sizeof(struct ttyent);
	192
176	193	// ioctl arguments
177	194	unsigned struct_ifreq_sz = sizeof(struct ifreq);
178	195	unsigned struct_termios_sz = sizeof(struct termios);
...	...	@@ -196,6 +213,10 @@ unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info);
196	213	unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats);
197	214	unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
198	215	unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
	216	unsigned struct_procctl_reaper_status_sz = sizeof(struct __sanitizer_procctl_reaper_status);
	217	unsigned struct_procctl_reaper_pidinfo_sz = sizeof(struct __sanitizer_procctl_reaper_pidinfo);
	218	unsigned struct_procctl_reaper_pids_sz = sizeof(struct __sanitizer_procctl_reaper_pids);
	219	unsigned struct_procctl_reaper_kill_sz = sizeof(struct __sanitizer_procctl_reaper_kill);
199	220	const unsigned long __sanitizer_bufsiz = BUFSIZ;
200	221
201	222	const unsigned IOCTL_NOT_PRESENT = 0;
...	...	@@ -357,6 +378,22 @@ const int si_SEGV_MAPERR = SEGV_MAPERR;
357	378	const int si_SEGV_ACCERR = SEGV_ACCERR;
358	379	const int unvis_valid = UNVIS_VALID;
359	380	const int unvis_validpush = UNVIS_VALIDPUSH;
	381
	382	const unsigned MD5_CTX_sz = sizeof(MD5_CTX);
	383	const unsigned MD5_return_length = MD5_DIGEST_STRING_LENGTH;
	384
	385	#define SHA2_CONST(LEN) \
	386	const unsigned SHA##LEN##_CTX_sz = sizeof(SHA##LEN##_CTX); \
	387	const unsigned SHA##LEN##_return_length = SHA##LEN##_DIGEST_STRING_LENGTH; \
	388	const unsigned SHA##LEN##_block_length = SHA##LEN##_BLOCK_LENGTH; \
	389	const unsigned SHA##LEN##_digest_length = SHA##LEN##_DIGEST_LENGTH
	390
	391	SHA2_CONST(224);
	392	SHA2_CONST(256);
	393	SHA2_CONST(384);
	394	SHA2_CONST(512);
	395
	396	#undef SHA2_CONST
360	397	} // namespace __sanitizer
361	398
362	399	using namespace __sanitizer;
...	...	@@ -529,4 +566,5 @@ COMPILER_CHECK(__sanitizer_XDR_FREE == XDR_FREE);
529	566	CHECK_TYPE_SIZE(sem_t);
530	567
531	568	COMPILER_CHECK(sizeof(__sanitizer_cap_rights_t) >= sizeof(cap_rights_t));
	569	COMPILER_CHECK(sizeof(__sanitizer_cpuset_t) >= sizeof(cpuset_t));
532	570	#endif // SANITIZER_FREEBSD

lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.h+167-71

...	...	@@ -16,26 +16,26 @@
16	16
17	17	#if SANITIZER_FREEBSD
18	18
19		#include "sanitizer_internal_defs.h"
20		#include "sanitizer_platform.h"
21		#include "sanitizer_platform_limits_posix.h"
	19	# include "sanitizer_internal_defs.h"
	20	# include "sanitizer_platform.h"
	21	# include "sanitizer_platform_limits_posix.h"
22	22
23	23	// Get sys/_types.h, because that tells us whether 64-bit inodes are
24	24	// used in struct dirent below.
25		#include <sys/_types.h>
	25	# include <sys/_types.h>
26	26
27	27	namespace __sanitizer {
28	28	void __sanitizer_get_link_map_by_dlopen_handle(void handle);
29		#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
30		(link_map *)__sanitizer_get_link_map_by_dlopen_handle(handle)
	29	# define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
	30	(link_map *)__sanitizer_get_link_map_by_dlopen_handle(handle)
31	31
32	32	extern unsigned struct_utsname_sz;
33	33	extern unsigned struct_stat_sz;
34		#if defined(__powerpc64__)
	34	# if defined(__powerpc64__)
35	35	const unsigned struct___old_kernel_stat_sz = 0;
36		#else
	36	# else
37	37	const unsigned struct___old_kernel_stat_sz = 32;
38		#endif
	38	# endif
39	39	extern unsigned struct_rusage_sz;
40	40	extern unsigned siginfo_t_sz;
41	41	extern unsigned struct_itimerval_sz;
...	...	@@ -57,7 +57,7 @@ extern unsigned struct_sched_param_sz;
57	57	extern unsigned struct_statfs64_sz;
58	58	extern unsigned struct_statfs_sz;
59	59	extern unsigned struct_sockaddr_sz;
60		extern unsigned ucontext_t_sz;
	60	unsigned ucontext_t_sz(void *ctx);
61	61	extern unsigned struct_rlimit_sz;
62	62	extern unsigned struct_utimbuf_sz;
63	63	extern unsigned struct_timespec_sz;
...	...	@@ -114,11 +114,24 @@ struct __sanitizer_ipc_perm {
114	114	long key;
115	115	};
116	116
117		#if !defined(__i386__)
	117	struct __sanitizer_protoent {
	118	char *p_name;
	119	char **p_aliases;
	120	int p_proto;
	121	};
	122
	123	struct __sanitizer_netent {
	124	char *n_name;
	125	char **n_aliases;
	126	int n_addrtype;
	127	u32 n_net;
	128	};
	129
	130	# if !defined(__i386__)
118	131	typedef long long __sanitizer_time_t;
119		#else
	132	# else
120	133	typedef long __sanitizer_time_t;
121		#endif
	134	# endif
122	135
123	136	struct __sanitizer_shmid_ds {
124	137	__sanitizer_ipc_perm shm_perm;
...	...	@@ -147,7 +160,7 @@ struct __sanitizer_ifaddrs {
147	160	unsigned int ifa_flags;
148	161	void ifa_addr; // (struct sockaddr )
149	162	void ifa_netmask; // (struct sockaddr )
150		#undef ifa_dstaddr
	163	# undef ifa_dstaddr
151	164	void ifa_dstaddr; // (struct sockaddr )
152	165	void *ifa_data;
153	166	};
...	...	@@ -229,37 +242,43 @@ struct __sanitizer_cmsghdr {
229	242	};
230	243
231	244	struct __sanitizer_dirent {
232		#if defined(__INO64)
	245	# if defined(__INO64)
233	246	unsigned long long d_fileno;
234	247	unsigned long long d_off;
235		#else
	248	# else
236	249	unsigned int d_fileno;
237		#endif
	250	# endif
238	251	unsigned short d_reclen;
239		// more fields that we don't care about
	252	u8 d_type;
	253	u8 d_pad0;
	254	u16 d_namlen;
	255	u16 d_pad1;
	256	char d_name[256];
240	257	};
241	258
	259	u16 __sanitizer_dirsiz(const __sanitizer_dirent *dp);
	260
242	261	// 'clock_t' is 32 bits wide on x64 FreeBSD
243	262	typedef int __sanitizer_clock_t;
244	263	typedef int __sanitizer_clockid_t;
245	264
246		#if defined(_LP64) \|\| defined(__x86_64__) \|\| defined(__powerpc__) \|\| \
247		defined(__mips__)
	265	# if defined(_LP64) \|\| defined(__x86_64__) \|\| defined(__powerpc__) \|\| \
	266	defined(__mips__)
248	267	typedef unsigned __sanitizer___kernel_uid_t;
249	268	typedef unsigned __sanitizer___kernel_gid_t;
250		#else
	269	# else
251	270	typedef unsigned short __sanitizer___kernel_uid_t;
252	271	typedef unsigned short __sanitizer___kernel_gid_t;
253		#endif
	272	# endif
254	273	typedef long long __sanitizer___kernel_off_t;
255	274
256		#if defined(__powerpc__) \|\| defined(__mips__)
	275	# if defined(__powerpc__) \|\| defined(__mips__)
257	276	typedef unsigned int __sanitizer___kernel_old_uid_t;
258	277	typedef unsigned int __sanitizer___kernel_old_gid_t;
259		#else
	278	# else
260	279	typedef unsigned short __sanitizer___kernel_old_uid_t;
261	280	typedef unsigned short __sanitizer___kernel_old_gid_t;
262		#endif
	281	# endif
263	282
264	283	typedef long long __sanitizer___kernel_loff_t;
265	284	typedef struct {
...	...	@@ -366,9 +385,12 @@ struct __sanitizer_glob_t {
366	385
367	386	extern int glob_nomatch;
368	387	extern int glob_altdirfunc;
	388	extern const int wordexp_wrde_dooffs;
369	389
370	390	extern unsigned path_max;
371	391
	392	extern int struct_ttyent_sz;
	393
372	394	struct __sanitizer_wordexp_t {
373	395	uptr we_wordc;
374	396	char **we_wordv;
...	...	@@ -398,39 +420,81 @@ struct __sanitizer_ifconf {
398	420	} ifc_ifcu;
399	421	};
400	422
401		#define IOC_NRBITS 8
402		#define IOC_TYPEBITS 8
403		#if defined(__powerpc__) \|\| defined(__powerpc64__) \|\| defined(__mips__)
404		#define IOC_SIZEBITS 13
405		#define IOC_DIRBITS 3
406		#define IOC_NONE 1U
407		#define IOC_WRITE 4U
408		#define IOC_READ 2U
409		#else
410		#define IOC_SIZEBITS 14
411		#define IOC_DIRBITS 2
412		#define IOC_NONE 0U
413		#define IOC_WRITE 1U
414		#define IOC_READ 2U
415		#endif
416		#define IOC_NRMASK ((1 << IOC_NRBITS) - 1)
417		#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1)
418		#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1)
419		#if defined(IOC_DIRMASK)
420		#undef IOC_DIRMASK
421		#endif
422		#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1)
423		#define IOC_NRSHIFT 0
424		#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS)
425		#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS)
426		#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS)
427		#define EVIOC_EV_MAX 0x1f
428		#define EVIOC_ABS_MAX 0x3f
429
430		#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK)
431		#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK)
432		#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK)
433		#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK)
	423	struct __sanitizer__ttyent {
	424	char *ty_name;
	425	char *ty_getty;
	426	char *ty_type;
	427	int ty_status;
	428	char *ty_window;
	429	char *ty_comment;
	430	char *ty_group;
	431	};
	432
	433	// procctl reaper data for PROCCTL_REAPER flags
	434	struct __sanitizer_procctl_reaper_status {
	435	unsigned int rs_flags;
	436	unsigned int rs_children;
	437	unsigned int rs_descendants;
	438	pid_t rs_reaper;
	439	pid_t rs_pid;
	440	unsigned int rs_pad0[15];
	441	};
	442
	443	struct __sanitizer_procctl_reaper_pidinfo {
	444	pid_t pi_pid;
	445	pid_t pi_subtree;
	446	unsigned int pi_flags;
	447	unsigned int pi_pad0[15];
	448	};
	449
	450	struct __sanitizer_procctl_reaper_pids {
	451	unsigned int rp_count;
	452	unsigned int rp_pad0[15];
	453	struct __sanitize_procctl_reapper_pidinfo *rp_pids;
	454	};
	455
	456	struct __sanitizer_procctl_reaper_kill {
	457	int rk_sig;
	458	unsigned int rk_flags;
	459	pid_t rk_subtree;
	460	unsigned int rk_killed;
	461	pid_t rk_fpid;
	462	unsigned int rk_pad[15];
	463	};
	464
	465	# define IOC_NRBITS 8
	466	# define IOC_TYPEBITS 8
	467	# if defined(__powerpc__) \|\| defined(__powerpc64__) \|\| defined(__mips__)
	468	# define IOC_SIZEBITS 13
	469	# define IOC_DIRBITS 3
	470	# define IOC_NONE 1U
	471	# define IOC_WRITE 4U
	472	# define IOC_READ 2U
	473	# else
	474	# define IOC_SIZEBITS 14
	475	# define IOC_DIRBITS 2
	476	# define IOC_NONE 0U
	477	# define IOC_WRITE 1U
	478	# define IOC_READ 2U
	479	# endif
	480	# define IOC_NRMASK ((1 << IOC_NRBITS) - 1)
	481	# define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1)
	482	# define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1)
	483	# if defined(IOC_DIRMASK)
	484	# undef IOC_DIRMASK
	485	# endif
	486	# define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1)
	487	# define IOC_NRSHIFT 0
	488	# define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS)
	489	# define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS)
	490	# define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS)
	491	# define EVIOC_EV_MAX 0x1f
	492	# define EVIOC_ABS_MAX 0x3f
	493
	494	# define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK)
	495	# define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK)
	496	# define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK)
	497	# define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK)
434	498
435	499	extern unsigned struct_ifreq_sz;
436	500	extern unsigned struct_termios_sz;
...	...	@@ -454,6 +518,11 @@ extern unsigned struct_ppp_stats_sz;
454	518	extern unsigned struct_sioc_sg_req_sz;
455	519	extern unsigned struct_sioc_vif_req_sz;
456	520
	521	extern unsigned struct_procctl_reaper_status_sz;
	522	extern unsigned struct_procctl_reaper_pidinfo_sz;
	523	extern unsigned struct_procctl_reaper_pids_sz;
	524	extern unsigned struct_procctl_reaper_kill_sz;
	525
457	526	// ioctl request identifiers
458	527
459	528	// A special value to mark ioctls that are not present on the target platform,
...	...	@@ -621,6 +690,22 @@ extern unsigned IOCTL_KDSKBMODE;
621	690	extern const int si_SEGV_MAPERR;
622	691	extern const int si_SEGV_ACCERR;
623	692
	693	extern const unsigned MD5_CTX_sz;
	694	extern const unsigned MD5_return_length;
	695
	696	#define SHA2_EXTERN(LEN) \
	697	extern const unsigned SHA##LEN##_CTX_sz; \
	698	extern const unsigned SHA##LEN##_return_length; \
	699	extern const unsigned SHA##LEN##_block_length; \
	700	extern const unsigned SHA##LEN##_digest_length
	701
	702	SHA2_EXTERN(224);
	703	SHA2_EXTERN(256);
	704	SHA2_EXTERN(384);
	705	SHA2_EXTERN(512);
	706
	707	#undef SHA2_EXTERN
	708
624	709	struct __sanitizer_cap_rights {
625	710	u64 cr_rights[2];
626	711	};
...	...	@@ -630,26 +715,37 @@ extern unsigned struct_cap_rights_sz;
630	715
631	716	extern unsigned struct_fstab_sz;
632	717	extern unsigned struct_StringList_sz;
	718
	719	struct __sanitizer_cpuset {
	720	#if __FreeBSD_version >= 1400090
	721	long __bits[(1024 + (sizeof(long) * 8) - 1) / (sizeof(long) * 8)];
	722	#else
	723	long __bits[(256 + (sizeof(long) * 8) - 1) / (sizeof(long) * 8)];
	724	#endif
	725	};
	726
	727	typedef struct __sanitizer_cpuset __sanitizer_cpuset_t;
	728	extern unsigned struct_cpuset_sz;
633	729	} // namespace __sanitizer
634	730
635		#define CHECK_TYPE_SIZE(TYPE) \
636		COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE))
	731	# define CHECK_TYPE_SIZE(TYPE) \
	732	COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE))
637	733
638		#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \
639		COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *)NULL)->MEMBER) == \
640		sizeof(((CLASS *)NULL)->MEMBER)); \
641		COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \
642		offsetof(CLASS, MEMBER))
	734	# define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \
	735	COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *)NULL)->MEMBER) == \
	736	sizeof(((CLASS *)NULL)->MEMBER)); \
	737	COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \
	738	offsetof(CLASS, MEMBER))
643	739
644	740	// For sigaction, which is a function and struct at the same time,
645	741	// and thus requires explicit "struct" in sizeof() expression.
646		#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \
647		COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *)NULL)->MEMBER) == \
648		sizeof(((struct CLASS *)NULL)->MEMBER)); \
649		COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \
650		offsetof(struct CLASS, MEMBER))
	742	# define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \
	743	COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *)NULL)->MEMBER) == \
	744	sizeof(((struct CLASS *)NULL)->MEMBER)); \
	745	COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \
	746	offsetof(struct CLASS, MEMBER))
651	747
652		#define SIGACTION_SYMNAME sigaction
	748	# define SIGACTION_SYMNAME sigaction
653	749
654	750	#endif
655	751

lib/tsan/sanitizer_common/sanitizer_platform_limits_linux.cpp+29-34

...	...	@@ -28,44 +28,39 @@
28	28	// are not defined anywhere in userspace headers. Fake them. This seems to work
29	29	// fine with newer headers, too.
30	30	#include <linux/posix_types.h>
31		#if defined(__x86_64__) \|\| defined(__mips__)
32		#include <sys/stat.h>
33		#else
34		#define ino_t __kernel_ino_t
35		#define mode_t __kernel_mode_t
36		#define nlink_t __kernel_nlink_t
37		#define uid_t __kernel_uid_t
38		#define gid_t __kernel_gid_t
39		#define off_t __kernel_off_t
40		#define time_t __kernel_time_t
	31	# if defined(__x86_64__) \|\| defined(__mips__) \|\| defined(__hexagon__)
	32	# include <sys/stat.h>
	33	# else
	34	# define ino_t __kernel_ino_t
	35	# define mode_t __kernel_mode_t
	36	# define nlink_t __kernel_nlink_t
	37	# define uid_t __kernel_uid_t
	38	# define gid_t __kernel_gid_t
	39	# define off_t __kernel_off_t
	40	# define time_t __kernel_time_t
41	41	// This header seems to contain the definitions of _kernel_ stat* structs.
42		#include <asm/stat.h>
43		#undef ino_t
44		#undef mode_t
45		#undef nlink_t
46		#undef uid_t
47		#undef gid_t
48		#undef off_t
49		#endif
50
51		#include <linux/aio_abi.h>
52
53		#if !SANITIZER_ANDROID
54		#include <sys/statfs.h>
55		#include <linux/perf_event.h>
56		#endif
	42	# include <asm/stat.h>
	43	# undef ino_t
	44	# undef mode_t
	45	# undef nlink_t
	46	# undef uid_t
	47	# undef gid_t
	48	# undef off_t
	49	# endif
	50
	51	# include <linux/aio_abi.h>
	52
	53	# if !SANITIZER_ANDROID
	54	# include <sys/statfs.h>
	55	# include <linux/perf_event.h>
	56	# endif
57	57
58	58	using namespace __sanitizer;
59	59
60		namespace __sanitizer {
61		#if !SANITIZER_ANDROID
62		unsigned struct_statfs64_sz = sizeof(struct statfs64);
63		#endif
64		} // namespace __sanitizer
65
66		#if !defined(__powerpc64__) && !defined(__x86_64__) && !defined(__aarch64__)\
67		&& !defined(__mips__) && !defined(__s390__)\
68		&& !defined(__sparc__) && !defined(__riscv)
	60	# if !defined(__powerpc64__) && !defined(__x86_64__) && \
	61	!defined(__aarch64__) && !defined(__mips__) && !defined(__s390__) && \
	62	!defined(__sparc__) && !defined(__riscv) && !defined(__hexagon__) && \
	63	!defined(__loongarch__)
69	64	COMPILER_CHECK(struct___old_kernel_stat_sz == sizeof(struct __old_kernel_stat));
70	65	#endif
71	66

lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.cpp+2-3

...	...	@@ -554,7 +554,7 @@ unsigned struct_tms_sz = sizeof(struct tms);
554	554	unsigned struct_sigevent_sz = sizeof(struct sigevent);
555	555	unsigned struct_sched_param_sz = sizeof(struct sched_param);
556	556	unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
557		unsigned ucontext_t_sz = sizeof(ucontext_t);
	557	unsigned ucontext_t_sz(void *ctx) { return sizeof(ucontext_t); }
558	558	unsigned struct_rlimit_sz = sizeof(struct rlimit);
559	559	unsigned struct_timespec_sz = sizeof(struct timespec);
560	560	unsigned struct_sembuf_sz = sizeof(struct sembuf);
...	...	@@ -666,6 +666,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
666	666
667	667	int glob_nomatch = GLOB_NOMATCH;
668	668	int glob_altdirfunc = GLOB_ALTDIRFUNC;
	669	const int wordexp_wrde_dooffs = WRDE_DOOFFS;
669	670
670	671	unsigned path_max = PATH_MAX;
671	672
...	...	@@ -2341,8 +2342,6 @@ unsigned IOCTL_TIOCDRAIN = TIOCDRAIN;
2341	2342	unsigned IOCTL_TIOCGFLAGS = TIOCGFLAGS;
2342	2343	unsigned IOCTL_TIOCSFLAGS = TIOCSFLAGS;
2343	2344	unsigned IOCTL_TIOCDCDTIMESTAMP = TIOCDCDTIMESTAMP;
2344		unsigned IOCTL_TIOCRCVFRAME = TIOCRCVFRAME;
2345		unsigned IOCTL_TIOCXMTFRAME = TIOCXMTFRAME;
2346	2345	unsigned IOCTL_TIOCPTMGET = TIOCPTMGET;
2347	2346	unsigned IOCTL_TIOCGRANTPT = TIOCGRANTPT;
2348	2347	unsigned IOCTL_TIOCPTSNAME = TIOCPTSNAME;

lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.h+2-3

...	...	@@ -45,7 +45,7 @@ extern unsigned struct_stack_t_sz;
45	45	extern unsigned struct_sched_param_sz;
46	46	extern unsigned struct_statfs_sz;
47	47	extern unsigned struct_sockaddr_sz;
48		extern unsigned ucontext_t_sz;
	48	unsigned ucontext_t_sz(void *ctx);
49	49
50	50	extern unsigned struct_rlimit_sz;
51	51	extern unsigned struct_utimbuf_sz;
...	...	@@ -394,6 +394,7 @@ struct __sanitizer_glob_t {
394	394
395	395	extern int glob_nomatch;
396	396	extern int glob_altdirfunc;
	397	extern const int wordexp_wrde_dooffs;
397	398
398	399	extern unsigned path_max;
399	400
...	...	@@ -2194,8 +2195,6 @@ extern unsigned IOCTL_TIOCDRAIN;
2194	2195	extern unsigned IOCTL_TIOCGFLAGS;
2195	2196	extern unsigned IOCTL_TIOCSFLAGS;
2196	2197	extern unsigned IOCTL_TIOCDCDTIMESTAMP;
2197		extern unsigned IOCTL_TIOCRCVFRAME;
2198		extern unsigned IOCTL_TIOCXMTFRAME;
2199	2198	extern unsigned IOCTL_TIOCPTMGET;
2200	2199	extern unsigned IOCTL_TIOCGRANTPT;
2201	2200	extern unsigned IOCTL_TIOCPTSNAME;

lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.cpp+110-44

...	...	@@ -18,12 +18,13 @@
18	18	// depends on _FILE_OFFSET_BITS setting.
19	19	// To get this "true" dirent definition, we undefine _FILE_OFFSET_BITS below.
20	20	#undef _FILE_OFFSET_BITS
	21	#undef _TIME_BITS
21	22	#endif
22	23
23	24	// Must go after undef _FILE_OFFSET_BITS.
24	25	#include "sanitizer_platform.h"
25	26
26		#if SANITIZER_LINUX \|\| SANITIZER_MAC
	27	#if SANITIZER_LINUX \|\| SANITIZER_APPLE
27	28	// Must go after undef _FILE_OFFSET_BITS.
28	29	#include "sanitizer_glibc_version.h"
29	30
...	...	@@ -51,7 +52,7 @@
51	52	#include <time.h>
52	53	#include <wchar.h>
53	54	#include <regex.h>
54		#if !SANITIZER_MAC
	55	#if !SANITIZER_APPLE
55	56	#include <utmp.h>
56	57	#endif
57	58
...	...	@@ -73,7 +74,9 @@
73	74	#include <sys/vt.h>
74	75	#include <linux/cdrom.h>
75	76	#include <linux/fd.h>
	77	#if SANITIZER_ANDROID
76	78	#include <linux/fs.h>
	79	#endif
77	80	#include <linux/hdreg.h>
78	81	#include <linux/input.h>
79	82	#include <linux/ioctl.h>
...	...	@@ -91,10 +94,10 @@
91	94	#if SANITIZER_LINUX
92	95	# include <utime.h>
93	96	# include <sys/ptrace.h>
94		#if defined(__mips64) \|\| defined(__aarch64__) \|\| defined(__arm__) \|\| \
95		SANITIZER_RISCV64
96		# include <asm/ptrace.h>
97		# ifdef __arm__
	97	# if defined(__mips64) \|\| defined(__aarch64__) \|\| defined(__arm__) \|\| \
	98	defined(__hexagon__) \|\| defined(__loongarch__) \|\|SANITIZER_RISCV64
	99	# include <asm/ptrace.h>
	100	# ifdef __arm__
98	101	typedef struct user_fpregs elf_fpregset_t;
99	102	# define ARM_VFPREGS_SIZE_ASAN (32 * 8 /fpregs/ + 4 /fpscr/)
100	103	# if !defined(ARM_VFPREGS_SIZE)
...	...	@@ -152,7 +155,6 @@ typedef struct user_fpregs elf_fpregset_t;
152	155	#include <linux/serial.h>
153	156	#include <sys/msg.h>
154	157	#include <sys/ipc.h>
155		#include <crypt.h>
156	158	#endif // SANITIZER_ANDROID
157	159
158	160	#include <link.h>
...	...	@@ -163,22 +165,24 @@ typedef struct user_fpregs elf_fpregset_t;
163	165	#include <fstab.h>
164	166	#endif // SANITIZER_LINUX
165	167
166		#if SANITIZER_MAC
	168	#if SANITIZER_APPLE
167	169	#include <net/ethernet.h>
168	170	#include <sys/filio.h>
169	171	#include <sys/sockio.h>
170	172	#endif
171	173
172	174	// Include these after system headers to avoid name clashes and ambiguities.
173		#include "sanitizer_internal_defs.h"
174		#include "sanitizer_platform_limits_posix.h"
	175	# include "sanitizer_common.h"
	176	# include "sanitizer_internal_defs.h"
	177	# include "sanitizer_platform_interceptors.h"
	178	# include "sanitizer_platform_limits_posix.h"
175	179
176	180	namespace __sanitizer {
177	181	unsigned struct_utsname_sz = sizeof(struct utsname);
178	182	unsigned struct_stat_sz = sizeof(struct stat);
179		#if !SANITIZER_IOS && !(SANITIZER_MAC && TARGET_CPU_ARM64)
	183	#if SANITIZER_HAS_STAT64
180	184	unsigned struct_stat64_sz = sizeof(struct stat64);
181		#endif // !SANITIZER_IOS && !(SANITIZER_MAC && TARGET_CPU_ARM64)
	185	#endif // SANITIZER_HAS_STAT64
182	186	unsigned struct_rusage_sz = sizeof(struct rusage);
183	187	unsigned struct_tm_sz = sizeof(struct tm);
184	188	unsigned struct_passwd_sz = sizeof(struct passwd);
...	...	@@ -203,26 +207,60 @@ namespace __sanitizer {
203	207	unsigned struct_regex_sz = sizeof(regex_t);
204	208	unsigned struct_regmatch_sz = sizeof(regmatch_t);
205	209
206		#if (SANITIZER_MAC && !TARGET_CPU_ARM64) && !SANITIZER_IOS
	210	#if SANITIZER_HAS_STATFS64
207	211	unsigned struct_statfs64_sz = sizeof(struct statfs64);
208		#endif // (SANITIZER_MAC && !TARGET_CPU_ARM64) && !SANITIZER_IOS
	212	#endif // SANITIZER_HAS_STATFS64
209	213
210		#if SANITIZER_GLIBC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| SANITIZER_MAC
	214	#if SANITIZER_GLIBC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| SANITIZER_APPLE
211	215	unsigned struct_fstab_sz = sizeof(struct fstab);
212	216	#endif // SANITIZER_GLIBC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\|
213		// SANITIZER_MAC
	217	// SANITIZER_APPLE
214	218	#if !SANITIZER_ANDROID
215	219	unsigned struct_statfs_sz = sizeof(struct statfs);
216	220	unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
217		unsigned ucontext_t_sz = sizeof(ucontext_t);
218		#endif // !SANITIZER_ANDROID
219	221
220		#if SANITIZER_LINUX
	222	unsigned ucontext_t_sz(void *ctx) {
	223	# if SANITIZER_GLIBC && SANITIZER_X64
	224	// Added in Linux kernel 3.4.0, merged to glibc in 2.16
	225	# ifndef FP_XSTATE_MAGIC1
	226	# define FP_XSTATE_MAGIC1 0x46505853U
	227	# endif
	228	// See kernel arch/x86/kernel/fpu/signal.c for details.
	229	const auto fpregs = static_cast<ucontext_t >(ctx)->uc_mcontext.fpregs;
	230	// The member names differ across header versions, but the actual layout
	231	// is always the same. So avoid using members, just use arithmetic.
	232	const uint32_t *after_xmm =
	233	reinterpret_cast<const uint32_t *>(fpregs + 1) - 24;
	234	if (after_xmm[12] == FP_XSTATE_MAGIC1)
	235	return reinterpret_cast<const char *>(fpregs) + after_xmm[13] -
	236	static_cast<const char *>(ctx);
	237	# endif
	238	return sizeof(ucontext_t);
	239	}
	240	# endif // !SANITIZER_ANDROID
	241
	242	# if SANITIZER_LINUX
221	243	unsigned struct_epoll_event_sz = sizeof(struct epoll_event);
222	244	unsigned struct_sysinfo_sz = sizeof(struct sysinfo);
223	245	unsigned __user_cap_header_struct_sz =
224	246	sizeof(struct __user_cap_header_struct);
225		unsigned __user_cap_data_struct_sz = sizeof(struct __user_cap_data_struct);
	247	unsigned __user_cap_data_struct_sz(void *hdrp) {
	248	int u32s = 0;
	249	if (hdrp) {
	250	switch (((struct __user_cap_header_struct *)hdrp)->version) {
	251	case _LINUX_CAPABILITY_VERSION_1:
	252	u32s = _LINUX_CAPABILITY_U32S_1;
	253	break;
	254	case _LINUX_CAPABILITY_VERSION_2:
	255	u32s = _LINUX_CAPABILITY_U32S_2;
	256	break;
	257	case _LINUX_CAPABILITY_VERSION_3:
	258	u32s = _LINUX_CAPABILITY_U32S_3;
	259	break;
	260	}
	261	}
	262	return sizeof(struct __user_cap_data_struct) * u32s;
	263	}
226	264	unsigned struct_new_utsname_sz = sizeof(struct new_utsname);
227	265	unsigned struct_old_utsname_sz = sizeof(struct old_utsname);
228	266	unsigned struct_oldold_utsname_sz = sizeof(struct oldold_utsname);
...	...	@@ -235,24 +273,28 @@ namespace __sanitizer {
235	273	unsigned struct_itimerspec_sz = sizeof(struct itimerspec);
236	274	#endif // SANITIZER_LINUX
237	275
238		#if SANITIZER_LINUX && !SANITIZER_ANDROID
	276	#if SANITIZER_GLIBC
239	277	// Use pre-computed size of struct ustat to avoid <sys/ustat.h> which
240	278	// has been removed from glibc 2.28.
241	279	#if defined(__aarch64__) \|\| defined(__s390x__) \|\| defined(__mips64) \|\| \
242	280	defined(__powerpc64__) \|\| defined(__arch64__) \|\| defined(__sparcv9) \|\| \
243	281	defined(__x86_64__) \|\| SANITIZER_RISCV64
244	282	#define SIZEOF_STRUCT_USTAT 32
245		#elif defined(__arm__) \|\| defined(__i386__) \|\| defined(__mips__) \
246		\|\| defined(__powerpc__) \|\| defined(__s390__) \|\| defined(__sparc__)
247		#define SIZEOF_STRUCT_USTAT 20
248		#else
249		#error Unknown size of struct ustat
250		#endif
	283	# elif defined(__arm__) \|\| defined(__i386__) \|\| defined(__mips__) \|\| \
	284	defined(__powerpc__) \|\| defined(__s390__) \|\| defined(__sparc__) \|\| \
	285	defined(__hexagon__)
	286	# define SIZEOF_STRUCT_USTAT 20
	287	# elif defined(__loongarch__)
	288	// Not used. The minimum Glibc version available for LoongArch is 2.36
	289	// so ustat() wrapper is already gone.
	290	# define SIZEOF_STRUCT_USTAT 0
	291	# else
	292	# error Unknown size of struct ustat
	293	# endif
251	294	unsigned struct_ustat_sz = SIZEOF_STRUCT_USTAT;
252	295	unsigned struct_rlimit64_sz = sizeof(struct rlimit64);
253	296	unsigned struct_statvfs64_sz = sizeof(struct statvfs64);
254		unsigned struct_crypt_data_sz = sizeof(struct crypt_data);
255		#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
	297	#endif // SANITIZER_GLIBC
256	298
257	299	#if SANITIZER_LINUX && !SANITIZER_ANDROID
258	300	unsigned struct_timex_sz = sizeof(struct timex);
...	...	@@ -280,7 +322,7 @@ namespace __sanitizer {
280	322	int shmctl_shm_stat = (int)SHM_STAT;
281	323	#endif
282	324
283		#if !SANITIZER_MAC && !SANITIZER_FREEBSD
	325	#if !SANITIZER_APPLE && !SANITIZER_FREEBSD
284	326	unsigned struct_utmp_sz = sizeof(struct utmp);
285	327	#endif
286	328	#if !SANITIZER_ANDROID
...	...	@@ -312,10 +354,14 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
312	354	int glob_altdirfunc = GLOB_ALTDIRFUNC;
313	355	#endif
314	356
	357	# if !SANITIZER_ANDROID
	358	const int wordexp_wrde_dooffs = WRDE_DOOFFS;
	359	# endif // !SANITIZER_ANDROID
	360
315	361	#if SANITIZER_LINUX && !SANITIZER_ANDROID && \
316	362	(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \
317	363	defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__arm__) \|\| \
318		defined(__s390__) \|\| SANITIZER_RISCV64)
	364	defined(__s390__) \|\| defined(__loongarch__)\|\| SANITIZER_RISCV64)
319	365	#if defined(__mips64) \|\| defined(__powerpc64__) \|\| defined(__arm__)
320	366	unsigned struct_user_regs_struct_sz = sizeof(struct pt_regs);
321	367	unsigned struct_user_fpregs_struct_sz = sizeof(elf_fpregset_t);
...	...	@@ -325,21 +371,24 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
325	371	#elif defined(__aarch64__)
326	372	unsigned struct_user_regs_struct_sz = sizeof(struct user_pt_regs);
327	373	unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpsimd_state);
	374	#elif defined(__loongarch__)
	375	unsigned struct_user_regs_struct_sz = sizeof(struct user_pt_regs);
	376	unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fp_state);
328	377	#elif defined(__s390__)
329	378	unsigned struct_user_regs_struct_sz = sizeof(struct _user_regs_struct);
330	379	unsigned struct_user_fpregs_struct_sz = sizeof(struct _user_fpregs_struct);
331	380	#else
332	381	unsigned struct_user_regs_struct_sz = sizeof(struct user_regs_struct);
333	382	unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpregs_struct);
334		#endif // __mips64 \|\| __powerpc64__ \|\| __aarch64__
	383	#endif // __mips64 \|\| __powerpc64__ \|\| __aarch64__ \|\| __loongarch__
335	384	#if defined(__x86_64) \|\| defined(__mips64) \|\| defined(__powerpc64__) \|\| \
336	385	defined(__aarch64__) \|\| defined(__arm__) \|\| defined(__s390__) \|\| \
337		SANITIZER_RISCV64
	386	defined(__loongarch__) \|\| SANITIZER_RISCV64
338	387	unsigned struct_user_fpxregs_struct_sz = 0;
339	388	#else
340	389	unsigned struct_user_fpxregs_struct_sz = sizeof(struct user_fpxregs_struct);
341	390	#endif // __x86_64 \|\| __mips64 \|\| __powerpc64__ \|\| __aarch64__ \|\| __arm__
342		// \|\| __s390__
	391	// \|\| __s390__ \|\| __loongarch__
343	392	#ifdef __arm__
344	393	unsigned struct_user_vfpregs_struct_sz = ARM_VFPREGS_SIZE;
345	394	#else
...	...	@@ -484,7 +533,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
484	533	unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats);
485	534	#endif // SANITIZER_GLIBC
486	535
487		#if !SANITIZER_ANDROID && !SANITIZER_MAC
	536	#if !SANITIZER_ANDROID && !SANITIZER_APPLE
488	537	unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
489	538	unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
490	539	#endif
...	...	@@ -570,6 +619,14 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
570	619	unsigned IOCTL_BLKROGET = BLKROGET;
571	620	unsigned IOCTL_BLKROSET = BLKROSET;
572	621	unsigned IOCTL_BLKRRPART = BLKRRPART;
	622	unsigned IOCTL_BLKFRASET = BLKFRASET;
	623	unsigned IOCTL_BLKFRAGET = BLKFRAGET;
	624	unsigned IOCTL_BLKSECTSET = BLKSECTSET;
	625	unsigned IOCTL_BLKSECTGET = BLKSECTGET;
	626	unsigned IOCTL_BLKSSZGET = BLKSSZGET;
	627	unsigned IOCTL_BLKBSZGET = BLKBSZGET;
	628	unsigned IOCTL_BLKBSZSET = BLKBSZSET;
	629	unsigned IOCTL_BLKGETSIZE64 = BLKGETSIZE64;
573	630	unsigned IOCTL_CDROMAUDIOBUFSIZ = CDROMAUDIOBUFSIZ;
574	631	unsigned IOCTL_CDROMEJECT = CDROMEJECT;
575	632	unsigned IOCTL_CDROMEJECT_SW = CDROMEJECT_SW;
...	...	@@ -837,10 +894,10 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
837	894	unsigned IOCTL_EVIOCGPROP = IOCTL_NOT_PRESENT;
838	895	unsigned IOCTL_EVIOCSKEYCODE_V2 = IOCTL_NOT_PRESENT;
839	896	#endif
840		unsigned IOCTL_FS_IOC_GETFLAGS = FS_IOC_GETFLAGS;
841		unsigned IOCTL_FS_IOC_GETVERSION = FS_IOC_GETVERSION;
842		unsigned IOCTL_FS_IOC_SETFLAGS = FS_IOC_SETFLAGS;
843		unsigned IOCTL_FS_IOC_SETVERSION = FS_IOC_SETVERSION;
	897	unsigned IOCTL_FS_IOC_GETFLAGS = _IOR('f', 1, long);
	898	unsigned IOCTL_FS_IOC_GETVERSION = _IOR('v', 1, long);
	899	unsigned IOCTL_FS_IOC_SETFLAGS = _IOW('f', 2, long);
	900	unsigned IOCTL_FS_IOC_SETVERSION = _IOW('v', 2, long);
844	901	unsigned IOCTL_GIO_CMAP = GIO_CMAP;
845	902	unsigned IOCTL_GIO_FONT = GIO_FONT;
846	903	unsigned IOCTL_GIO_UNIMAP = GIO_UNIMAP;
...	...	@@ -1035,7 +1092,7 @@ CHECK_SIZE_AND_OFFSET(mmsghdr, msg_len);
1035	1092
1036	1093	COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent));
1037	1094	CHECK_SIZE_AND_OFFSET(dirent, d_ino);
1038		#if SANITIZER_MAC
	1095	#if SANITIZER_APPLE
1039	1096	CHECK_SIZE_AND_OFFSET(dirent, d_seekoff);
1040	1097	#elif SANITIZER_FREEBSD
1041	1098	// There is no 'd_off' field on FreeBSD.
...	...	@@ -1044,7 +1101,7 @@ CHECK_SIZE_AND_OFFSET(dirent, d_off);
1044	1101	#endif
1045	1102	CHECK_SIZE_AND_OFFSET(dirent, d_reclen);
1046	1103
1047		#if SANITIZER_LINUX && !SANITIZER_ANDROID
	1104	#if SANITIZER_GLIBC
1048	1105	COMPILER_CHECK(sizeof(__sanitizer_dirent64) <= sizeof(dirent64));
1049	1106	CHECK_SIZE_AND_OFFSET(dirent64, d_ino);
1050	1107	CHECK_SIZE_AND_OFFSET(dirent64, d_off);
...	...	@@ -1077,6 +1134,15 @@ CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_flags);
1077	1134	CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_restorer);
1078	1135	#endif
1079	1136
	1137	#if SANITIZER_HAS_SIGINFO
	1138	COMPILER_CHECK(alignof(siginfo_t) == alignof(__sanitizer_siginfo));
	1139	using __sanitizer_siginfo_t = __sanitizer_siginfo;
	1140	CHECK_TYPE_SIZE(siginfo_t);
	1141	CHECK_SIZE_AND_OFFSET(siginfo_t, si_signo);
	1142	CHECK_SIZE_AND_OFFSET(siginfo_t, si_errno);
	1143	CHECK_SIZE_AND_OFFSET(siginfo_t, si_code);
	1144	#endif
	1145
1080	1146	#if SANITIZER_LINUX
1081	1147	CHECK_TYPE_SIZE(__sysctl_args);
1082	1148	CHECK_SIZE_AND_OFFSET(__sysctl_args, name);
...	...	@@ -1217,7 +1283,7 @@ CHECK_SIZE_AND_OFFSET(passwd, pw_shell);
1217	1283	CHECK_SIZE_AND_OFFSET(passwd, pw_gecos);
1218	1284	#endif
1219	1285
1220		#if SANITIZER_MAC
	1286	#if SANITIZER_APPLE
1221	1287	CHECK_SIZE_AND_OFFSET(passwd, pw_change);
1222	1288	CHECK_SIZE_AND_OFFSET(passwd, pw_expire);
1223	1289	CHECK_SIZE_AND_OFFSET(passwd, pw_class);
...	...	@@ -1230,7 +1296,7 @@ CHECK_SIZE_AND_OFFSET(group, gr_passwd);
1230	1296	CHECK_SIZE_AND_OFFSET(group, gr_gid);
1231	1297	CHECK_SIZE_AND_OFFSET(group, gr_mem);
1232	1298
1233		#if HAVE_RPC_XDR_H
	1299	#if HAVE_RPC_XDR_H && !SANITIZER_APPLE
1234	1300	CHECK_TYPE_SIZE(XDR);
1235	1301	CHECK_SIZE_AND_OFFSET(XDR, x_op);
1236	1302	CHECK_SIZE_AND_OFFSET(XDR, x_ops);
...	...	@@ -1285,4 +1351,4 @@ CHECK_TYPE_SIZE(sem_t);
1285	1351	COMPILER_CHECK(ARM_VFPREGS_SIZE == ARM_VFPREGS_SIZE_ASAN);
1286	1352	#endif
1287	1353
1288		#endif // SANITIZER_LINUX \|\| SANITIZER_FREEBSD \|\| SANITIZER_MAC
	1354	#endif // SANITIZER_LINUX \|\| SANITIZER_FREEBSD \|\| SANITIZER_APPLE

lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.h+105-46

...	...	@@ -14,10 +14,25 @@
14	14	#ifndef SANITIZER_PLATFORM_LIMITS_POSIX_H
15	15	#define SANITIZER_PLATFORM_LIMITS_POSIX_H
16	16
17		#if SANITIZER_LINUX \|\| SANITIZER_MAC
	17	#if SANITIZER_LINUX \|\| SANITIZER_APPLE
18	18
19	19	#include "sanitizer_internal_defs.h"
20	20	#include "sanitizer_platform.h"
	21	#include "sanitizer_mallinfo.h"
	22
	23	#if SANITIZER_APPLE
	24	#include <sys/cdefs.h>
	25	#if !__DARWIN_ONLY_64_BIT_INO_T
	26	#define SANITIZER_HAS_STAT64 1
	27	#define SANITIZER_HAS_STATFS64 1
	28	#else
	29	#define SANITIZER_HAS_STAT64 0
	30	#define SANITIZER_HAS_STATFS64 0
	31	#endif
	32	#elif SANITIZER_GLIBC \|\| SANITIZER_ANDROID
	33	#define SANITIZER_HAS_STAT64 1
	34	#define SANITIZER_HAS_STATFS64 1
	35	#endif
21	36
22	37	#if defined(__sparc__)
23	38	// FIXME: This can't be included from tsan which does not support sparc yet.
...	...	@@ -29,7 +44,7 @@
29	44	namespace __sanitizer {
30	45	extern unsigned struct_utsname_sz;
31	46	extern unsigned struct_stat_sz;
32		#if !SANITIZER_IOS
	47	#if SANITIZER_HAS_STAT64
33	48	extern unsigned struct_stat64_sz;
34	49	#endif
35	50	extern unsigned struct_rusage_sz;
...	...	@@ -49,7 +64,9 @@ extern unsigned struct_itimerspec_sz;
49	64	extern unsigned struct_sigevent_sz;
50	65	extern unsigned struct_stack_t_sz;
51	66	extern unsigned struct_sched_param_sz;
	67	#if SANITIZER_HAS_STATFS64
52	68	extern unsigned struct_statfs64_sz;
	69	#endif
53	70	extern unsigned struct_regex_sz;
54	71	extern unsigned struct_regmatch_sz;
55	72
...	...	@@ -57,12 +74,12 @@ extern unsigned struct_regmatch_sz;
57	74	extern unsigned struct_fstab_sz;
58	75	extern unsigned struct_statfs_sz;
59	76	extern unsigned struct_sockaddr_sz;
60		extern unsigned ucontext_t_sz;
61		#endif // !SANITIZER_ANDROID
	77	unsigned ucontext_t_sz(void *uctx);
	78	# endif // !SANITIZER_ANDROID
62	79
63		#if SANITIZER_LINUX
	80	# if SANITIZER_LINUX
64	81
65		#if defined(__x86_64__)
	82	# if defined(__x86_64__)
66	83	const unsigned struct_kernel_stat_sz = 144;
67	84	const unsigned struct_kernel_stat64_sz = 0;
68	85	#elif defined(__i386__)
...	...	@@ -81,9 +98,10 @@ const unsigned struct_kernel_stat64_sz = 104;
81	98	const unsigned struct_kernel_stat_sz = 144;
82	99	const unsigned struct_kernel_stat64_sz = 104;
83	100	#elif defined(__mips__)
84		const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID
85		? FIRST_32_SECOND_64(104, 128)
86		: FIRST_32_SECOND_64(160, 216);
	101	const unsigned struct_kernel_stat_sz =
	102	SANITIZER_ANDROID
	103	? FIRST_32_SECOND_64(104, 128)
	104	: FIRST_32_SECOND_64((_MIPS_SIM == _ABIN32) ? 176 : 160, 216);
87	105	const unsigned struct_kernel_stat64_sz = 104;
88	106	#elif defined(__s390__) && !defined(__s390x__)
89	107	const unsigned struct_kernel_stat_sz = 64;
...	...	@@ -102,7 +120,13 @@ const unsigned struct_kernel_stat64_sz = 104;
102	120	#elif SANITIZER_RISCV64
103	121	const unsigned struct_kernel_stat_sz = 128;
104	122	const unsigned struct_kernel_stat64_sz = 0; // RISCV64 does not use stat64
105		#endif
	123	# elif defined(__hexagon__)
	124	const unsigned struct_kernel_stat_sz = 128;
	125	const unsigned struct_kernel_stat64_sz = 0;
	126	# elif defined(__loongarch__)
	127	const unsigned struct_kernel_stat_sz = 128;
	128	const unsigned struct_kernel_stat64_sz = 0;
	129	# endif
106	130	struct __sanitizer_perf_event_attr {
107	131	unsigned type;
108	132	unsigned size;
...	...	@@ -112,7 +136,7 @@ struct __sanitizer_perf_event_attr {
112	136	extern unsigned struct_epoll_event_sz;
113	137	extern unsigned struct_sysinfo_sz;
114	138	extern unsigned __user_cap_header_struct_sz;
115		extern unsigned __user_cap_data_struct_sz;
	139	extern unsigned __user_cap_data_struct_sz(void *hdrp);
116	140	extern unsigned struct_new_utsname_sz;
117	141	extern unsigned struct_old_utsname_sz;
118	142	extern unsigned struct_oldold_utsname_sz;
...	...	@@ -122,7 +146,7 @@ const unsigned struct_kexec_segment_sz = 4 * sizeof(unsigned long);
122	146
123	147	#if SANITIZER_LINUX
124	148
125		#if defined(__powerpc64__) \|\| defined(__s390__)
	149	#if defined(__powerpc64__) \|\| defined(__s390__) \|\| defined(__loongarch__)
126	150	const unsigned struct___old_kernel_stat_sz = 0;
127	151	#elif !defined(__sparc__)
128	152	const unsigned struct___old_kernel_stat_sz = 32;
...	...	@@ -181,17 +205,7 @@ struct __sanitizer_sem_t {
181	205	};
182	206	#endif // SANITIZER_LINUX
183	207
184		#if SANITIZER_ANDROID
185		struct __sanitizer_struct_mallinfo {
186		uptr v[10];
187		};
188		#endif
189
190	208	#if SANITIZER_LINUX && !SANITIZER_ANDROID
191		struct __sanitizer_struct_mallinfo {
192		int v[10];
193		};
194
195	209	extern unsigned struct_ustat_sz;
196	210	extern unsigned struct_rlimit64_sz;
197	211	extern unsigned struct_statvfs64_sz;
...	...	@@ -295,7 +309,6 @@ extern unsigned struct_msqid_ds_sz;
295	309	extern unsigned struct_mq_attr_sz;
296	310	extern unsigned struct_timex_sz;
297	311	extern unsigned struct_statvfs_sz;
298		extern unsigned struct_crypt_data_sz;
299	312	#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
300	313
301	314	struct __sanitizer_iovec {
...	...	@@ -319,7 +332,7 @@ struct __sanitizer_ifaddrs {
319	332	};
320	333	#endif // !SANITIZER_ANDROID
321	334
322		#if SANITIZER_MAC
	335	#if SANITIZER_APPLE
323	336	typedef unsigned long __sanitizer_pthread_key_t;
324	337	#else
325	338	typedef unsigned __sanitizer_pthread_key_t;
...	...	@@ -346,7 +359,7 @@ struct __sanitizer_passwd {
346	359	char *pw_passwd;
347	360	int pw_uid;
348	361	int pw_gid;
349		#if SANITIZER_MAC
	362	#if SANITIZER_APPLE
350	363	long pw_change;
351	364	char *pw_class;
352	365	#endif
...	...	@@ -355,7 +368,7 @@ struct __sanitizer_passwd {
355	368	#endif
356	369	char *pw_dir;
357	370	char *pw_shell;
358		#if SANITIZER_MAC
	371	#if SANITIZER_APPLE
359	372	long pw_expire;
360	373	#endif
361	374	};
...	...	@@ -367,7 +380,8 @@ struct __sanitizer_group {
367	380	char **gr_mem;
368	381	};
369	382
370		#if defined(__x86_64__) && !defined(_LP64)
	383	# if (SANITIZER_LINUX && !SANITIZER_GLIBC && !SANITIZER_ANDROID) \|\| \
	384	(defined(__x86_64__) && !defined(_LP64)) \|\| defined(__hexagon__)
371	385	typedef long long __sanitizer_time_t;
372	386	#else
373	387	typedef long __sanitizer_time_t;
...	...	@@ -427,7 +441,7 @@ struct __sanitizer_file_handle {
427	441	};
428	442	#endif
429	443
430		#if SANITIZER_MAC
	444	#if SANITIZER_APPLE
431	445	struct __sanitizer_msghdr {
432	446	void *msg_name;
433	447	unsigned msg_namelen;
...	...	@@ -468,30 +482,31 @@ struct __sanitizer_mmsghdr {
468	482	};
469	483	#endif
470	484
471		#if SANITIZER_MAC
	485	#if SANITIZER_APPLE
472	486	struct __sanitizer_dirent {
473	487	unsigned long long d_ino;
474	488	unsigned long long d_seekoff;
475	489	unsigned short d_reclen;
476	490	// more fields that we don't care about
477	491	};
478		#elif SANITIZER_ANDROID \|\| defined(__x86_64__)
	492	# elif (SANITIZER_LINUX && !SANITIZER_GLIBC) \|\| defined(__x86_64__) \|\| \
	493	defined(__hexagon__)
479	494	struct __sanitizer_dirent {
480	495	unsigned long long d_ino;
481	496	unsigned long long d_off;
482	497	unsigned short d_reclen;
483	498	// more fields that we don't care about
484	499	};
485		#else
	500	# else
486	501	struct __sanitizer_dirent {
487	502	uptr d_ino;
488	503	uptr d_off;
489	504	unsigned short d_reclen;
490	505	// more fields that we don't care about
491	506	};
492		#endif
	507	# endif
493	508
494		#if SANITIZER_LINUX && !SANITIZER_ANDROID
	509	# if SANITIZER_GLIBC
495	510	struct __sanitizer_dirent64 {
496	511	unsigned long long d_ino;
497	512	unsigned long long d_off;
...	...	@@ -511,8 +526,8 @@ typedef int __sanitizer_clockid_t;
511	526	#endif
512	527
513	528	#if SANITIZER_LINUX
514		#if defined(_LP64) \|\| defined(__x86_64__) \|\| defined(__powerpc__) \|\| \
515		defined(__mips__)
	529	# if defined(_LP64) \|\| defined(__x86_64__) \|\| defined(__powerpc__) \|\| \
	530	defined(__mips__) \|\| defined(__hexagon__)
516	531	typedef unsigned __sanitizer___kernel_uid_t;
517	532	typedef unsigned __sanitizer___kernel_gid_t;
518	533	#else
...	...	@@ -552,7 +567,7 @@ typedef unsigned long __sanitizer_sigset_t[16 / sizeof(unsigned long)];
552	567	# else
553	568	typedef unsigned long __sanitizer_sigset_t;
554	569	# endif
555		#elif SANITIZER_MAC
	570	#elif SANITIZER_APPLE
556	571	typedef unsigned __sanitizer_sigset_t;
557	572	#elif SANITIZER_LINUX
558	573	struct __sanitizer_sigset_t {
...	...	@@ -561,10 +576,35 @@ struct __sanitizer_sigset_t {
561	576	};
562	577	#endif
563	578
564		struct __sanitizer_siginfo {
565		// The size is determined by looking at sizeof of real siginfo_t on linux.
566		u64 opaque[128 / sizeof(u64)];
	579	struct __sanitizer_siginfo_pad {
	580	#if SANITIZER_X32
	581	// x32 siginfo_t is aligned to 8 bytes.
	582	u64 pad[128 / sizeof(u64)];
	583	#else
	584	// Require uptr, because siginfo_t is always pointer-size aligned on Linux.
	585	uptr pad[128 / sizeof(uptr)];
	586	#endif
	587	};
	588
	589	#if SANITIZER_LINUX
	590	# define SANITIZER_HAS_SIGINFO 1
	591	union __sanitizer_siginfo {
	592	struct {
	593	int si_signo;
	594	# if SANITIZER_MIPS
	595	int si_code;
	596	int si_errno;
	597	# else
	598	int si_errno;
	599	int si_code;
	600	# endif
	601	};
	602	__sanitizer_siginfo_pad pad;
567	603	};
	604	#else
	605	# define SANITIZER_HAS_SIGINFO 0
	606	typedef __sanitizer_siginfo_pad __sanitizer_siginfo;
	607	#endif
568	608
569	609	using __sanitizer_sighandler_ptr = void (*)(int sig);
570	610	using __sanitizer_sigactionhandler_ptr = void (*)(int sig,
...	...	@@ -712,12 +752,19 @@ struct __sanitizer_protoent {
712	752	int p_proto;
713	753	};
714	754
	755	struct __sanitizer_netent {
	756	char *n_name;
	757	char **n_aliases;
	758	int n_addrtype;
	759	u32 n_net;
	760	};
	761
715	762	struct __sanitizer_addrinfo {
716	763	int ai_flags;
717	764	int ai_family;
718	765	int ai_socktype;
719	766	int ai_protocol;
720		#if SANITIZER_ANDROID \|\| SANITIZER_MAC
	767	#if SANITIZER_ANDROID \|\| SANITIZER_APPLE
721	768	unsigned ai_addrlen;
722	769	char *ai_canonname;
723	770	void *ai_addr;
...	...	@@ -743,7 +790,7 @@ struct __sanitizer_pollfd {
743	790	short revents;
744	791	};
745	792
746		#if SANITIZER_ANDROID \|\| SANITIZER_MAC
	793	#if SANITIZER_ANDROID \|\| SANITIZER_APPLE
747	794	typedef unsigned __sanitizer_nfds_t;
748	795	#else
749	796	typedef unsigned long __sanitizer_nfds_t;
...	...	@@ -773,6 +820,10 @@ extern int glob_altdirfunc;
773	820
774	821	extern unsigned path_max;
775	822
	823	# if !SANITIZER_ANDROID
	824	extern const int wordexp_wrde_dooffs;
	825	# endif // !SANITIZER_ANDROID
	826
776	827	struct __sanitizer_wordexp_t {
777	828	uptr we_wordc;
778	829	char **we_wordv;
...	...	@@ -806,7 +857,7 @@ typedef void __sanitizer_FILE;
806	857	#if SANITIZER_LINUX && !SANITIZER_ANDROID && \
807	858	(defined(__i386) \|\| defined(__x86_64) \|\| defined(__mips64) \|\| \
808	859	defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__arm__) \|\| \
809		defined(__s390__) \|\| SANITIZER_RISCV64)
	860	defined(__s390__) \|\| defined(__loongarch__) \|\| SANITIZER_RISCV64)
810	861	extern unsigned struct_user_regs_struct_sz;
811	862	extern unsigned struct_user_fpregs_struct_sz;
812	863	extern unsigned struct_user_fpxregs_struct_sz;
...	...	@@ -839,7 +890,7 @@ extern int shmctl_shm_info;
839	890	extern int shmctl_shm_stat;
840	891	#endif
841	892
842		#if !SANITIZER_MAC && !SANITIZER_FREEBSD
	893	#if !SANITIZER_APPLE && !SANITIZER_FREEBSD
843	894	extern unsigned struct_utmp_sz;
844	895	#endif
845	896	#if !SANITIZER_ANDROID
...	...	@@ -854,7 +905,7 @@ struct __sanitizer_ifconf {
854	905	union {
855	906	void *ifcu_req;
856	907	} ifc_ifcu;
857		#if SANITIZER_MAC
	908	#if SANITIZER_APPLE
858	909	} __attribute__((packed));
859	910	#else
860	911	};
...	...	@@ -1007,7 +1058,7 @@ extern unsigned struct_audio_buf_info_sz;
1007	1058	extern unsigned struct_ppp_stats_sz;
1008	1059	#endif // (SANITIZER_LINUX \|\| SANITIZER_FREEBSD) && !SANITIZER_ANDROID
1009	1060
1010		#if !SANITIZER_ANDROID && !SANITIZER_MAC
	1061	#if !SANITIZER_ANDROID && !SANITIZER_APPLE
1011	1062	extern unsigned struct_sioc_sg_req_sz;
1012	1063	extern unsigned struct_sioc_vif_req_sz;
1013	1064	#endif
...	...	@@ -1094,6 +1145,14 @@ extern unsigned IOCTL_BLKRASET;
1094	1145	extern unsigned IOCTL_BLKROGET;
1095	1146	extern unsigned IOCTL_BLKROSET;
1096	1147	extern unsigned IOCTL_BLKRRPART;
	1148	extern unsigned IOCTL_BLKFRASET;
	1149	extern unsigned IOCTL_BLKFRAGET;
	1150	extern unsigned IOCTL_BLKSECTSET;
	1151	extern unsigned IOCTL_BLKSECTGET;
	1152	extern unsigned IOCTL_BLKSSZGET;
	1153	extern unsigned IOCTL_BLKBSZGET;
	1154	extern unsigned IOCTL_BLKBSZSET;
	1155	extern unsigned IOCTL_BLKGETSIZE64;
1097	1156	extern unsigned IOCTL_CDROMAUDIOBUFSIZ;
1098	1157	extern unsigned IOCTL_CDROMEJECT;
1099	1158	extern unsigned IOCTL_CDROMEJECT_SW;
...	...	@@ -1440,6 +1499,6 @@ extern const int si_SEGV_ACCERR;
1440	1499
1441	1500	#define SIGACTION_SYMNAME sigaction
1442	1501
1443		#endif // SANITIZER_LINUX \|\| SANITIZER_MAC
	1502	#endif // SANITIZER_LINUX \|\| SANITIZER_APPLE
1444	1503
1445	1504	#endif

lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.cpp+2-1

...	...	@@ -89,7 +89,7 @@ namespace __sanitizer {
89	89	unsigned struct_sched_param_sz = sizeof(struct sched_param);
90	90	unsigned struct_statfs_sz = sizeof(struct statfs);
91	91	unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
92		unsigned ucontext_t_sz = sizeof(ucontext_t);
	92	unsigned ucontext_t_sz(void *ctx) { return sizeof(ucontext_t); }
93	93	unsigned struct_timespec_sz = sizeof(struct timespec);
94	94	#if SANITIZER_SOLARIS32
95	95	unsigned struct_statvfs64_sz = sizeof(struct statvfs64);
...	...	@@ -123,6 +123,7 @@ namespace __sanitizer {
123	123	unsigned struct_ElfW_Phdr_sz = sizeof(ElfW(Phdr));
124	124
125	125	int glob_nomatch = GLOB_NOMATCH;
	126	const int wordexp_wrde_dooffs = WRDE_DOOFFS;
126	127
127	128	unsigned path_max = PATH_MAX;
128	129

lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.h+2-1

...	...	@@ -43,7 +43,7 @@ extern unsigned struct_sched_param_sz;
43	43	extern unsigned struct_statfs64_sz;
44	44	extern unsigned struct_statfs_sz;
45	45	extern unsigned struct_sockaddr_sz;
46		extern unsigned ucontext_t_sz;
	46	unsigned ucontext_t_sz(void *ctx);
47	47
48	48	extern unsigned struct_timespec_sz;
49	49	extern unsigned struct_rlimit_sz;
...	...	@@ -341,6 +341,7 @@ struct __sanitizer_glob_t {
341	341
342	342	extern int glob_nomatch;
343	343	extern int glob_altdirfunc;
	344	extern const int wordexp_wrde_dooffs;
344	345
345	346	extern unsigned path_max;
346	347

lib/tsan/sanitizer_common/sanitizer_posix.cpp+21-9

...	...	@@ -41,6 +41,8 @@ uptr GetMmapGranularity() {
41	41	return GetPageSize();
42	42	}
43	43
	44	bool ErrorIsOOM(error_t err) { return err == ENOMEM; }
	45
44	46	void MmapOrDie(uptr size, const char mem_type, bool raw_report) {
45	47	size = RoundUpTo(size, GetPageSizeCached());
46	48	uptr res = MmapNamed(nullptr, size, PROT_READ \| PROT_WRITE,
...	...	@@ -55,11 +57,9 @@ void MmapOrDie(uptr size, const char mem_type, bool raw_report) {
55	57	void UnmapOrDie(void *addr, uptr size) {
56	58	if (!addr \|\| !size) return;
57	59	uptr res = internal_munmap(addr, size);
58		if (UNLIKELY(internal_iserror(res))) {
59		Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n",
60		SanitizerToolName, size, size, addr);
61		CHECK("unable to unmap" && 0);
62		}
	60	int reserrno;
	61	if (UNLIKELY(internal_iserror(res, &reserrno)))
	62	ReportMunmapFailureAndDie(addr, size, reserrno);
63	63	DecreaseTotalMmap(size);
64	64	}
65	65
...	...	@@ -85,18 +85,26 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
85	85	CHECK(IsPowerOfTwo(size));
86	86	CHECK(IsPowerOfTwo(alignment));
87	87	uptr map_size = size + alignment;
	88	// mmap maps entire pages and rounds up map_size needs to be a an integral
	89	// number of pages.
	90	// We need to be aware of this size for calculating end and for unmapping
	91	// fragments before and after the alignment region.
	92	map_size = RoundUpTo(map_size, GetPageSizeCached());
88	93	uptr map_res = (uptr)MmapOrDieOnFatalError(map_size, mem_type);
89	94	if (UNLIKELY(!map_res))
90	95	return nullptr;
91		uptr map_end = map_res + map_size;
92	96	uptr res = map_res;
93	97	if (!IsAligned(res, alignment)) {
94	98	res = (map_res + alignment - 1) & ~(alignment - 1);
95	99	UnmapOrDie((void*)map_res, res - map_res);
96	100	}
	101	uptr map_end = map_res + map_size;
97	102	uptr end = res + size;
98		if (end != map_end)
	103	end = RoundUpTo(end, GetPageSizeCached());
	104	if (end != map_end) {
	105	CHECK_LT(end, map_end);
99	106	UnmapOrDie((void*)end, map_end - end);
	107	}
100	108	return (void*)res;
101	109	}
102	110
...	...	@@ -146,7 +154,11 @@ bool MprotectReadOnly(uptr addr, uptr size) {
146	154	return 0 == internal_mprotect((void *)addr, size, PROT_READ);
147	155	}
148	156
149		#if !SANITIZER_MAC
	157	bool MprotectReadWrite(uptr addr, uptr size) {
	158	return 0 == internal_mprotect((void *)addr, size, PROT_READ \| PROT_WRITE);
	159	}
	160
	161	#if !SANITIZER_APPLE
150	162	void MprotectMallocZones(void *addr, int prot) {}
151	163	#endif
152	164
...	...	@@ -239,7 +251,7 @@ bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
239	251	return true;
240	252	}
241	253
242		#if !SANITIZER_MAC
	254	#if !SANITIZER_APPLE
243	255	void DumpProcessMap() {
244	256	MemoryMappingLayout proc_maps(/cache_enabled/true);
245	257	const sptr kBufSize = 4095;

lib/tsan/sanitizer_common/sanitizer_posix.h+7-5

...	...	@@ -20,10 +20,7 @@
20	20	#include "sanitizer_platform_limits_posix.h"
21	21	#include "sanitizer_platform_limits_solaris.h"
22	22
23		#if !SANITIZER_POSIX
24		// Make it hard to accidentally use any of functions declared in this file:
25		#error This file should only be included on POSIX
26		#endif
	23	#if SANITIZER_POSIX
27	24
28	25	namespace __sanitizer {
29	26
...	...	@@ -93,7 +90,7 @@ int real_pthread_join(void th, void *ret);
93	90	} \
94	91	} // namespace __sanitizer
95	92
96		int my_pthread_attr_getstack(void attr, void addr, uptr size);
	93	int internal_pthread_attr_getstack(void attr, void addr, uptr size);
97	94
98	95	// A routine named real_sigaction() must be implemented by each sanitizer in
99	96	// order for internal_sigaction() to bypass interceptors.
...	...	@@ -123,7 +120,12 @@ int GetNamedMappingFd(const char name, uptr size, int flags);
123	120	// alive at least as long as the mapping exists.
124	121	void DecorateMapping(uptr addr, uptr size, const char *name);
125	122
	123	# if !SANITIZER_FREEBSD
	124	# define __sanitizer_dirsiz(dp) ((dp)->d_reclen)
	125	# endif
126	126
127	127	} // namespace __sanitizer
128	128
	129	#endif // SANITIZER_POSIX
	130
129	131	#endif // SANITIZER_POSIX_H

lib/tsan/sanitizer_common/sanitizer_posix_libcdep.cpp+6-4

...	...	@@ -151,6 +151,8 @@ int Atexit(void (*function)(void)) {
151	151	#endif
152	152	}
153	153
	154	bool CreateDir(const char *pathname) { return mkdir(pathname, 0755) == 0; }
	155
154	156	bool SupportsColoredOutput(fd_t fd) {
155	157	return isatty(fd) != 0;
156	158	}
...	...	@@ -288,7 +290,7 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) {
288	290	return result;
289	291	}
290	292
291		void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {
	293	void PlatformPrepareForSandboxing(void *args) {
292	294	// Some kinds of sandboxes may forbid filesystem access, so we won't be able
293	295	// to read the file mappings from /proc/self/maps. Luckily, neither the
294	296	// process will be able to load additional libraries, so it's fine to use the
...	...	@@ -381,8 +383,8 @@ SANITIZER_WEAK_ATTRIBUTE int
381	383	real_pthread_attr_getstack(void attr, void addr, size_t size);
382	384	} // extern "C"
383	385
384		int my_pthread_attr_getstack(void attr, void addr, uptr size) {
385		#if !SANITIZER_GO && !SANITIZER_MAC
	386	int internal_pthread_attr_getstack(void attr, void addr, uptr size) {
	387	#if !SANITIZER_GO && !SANITIZER_APPLE
386	388	if (&real_pthread_attr_getstack)
387	389	return real_pthread_attr_getstack((pthread_attr_t *)attr, addr,
388	390	(size_t *)size);
...	...	@@ -395,7 +397,7 @@ void AdjustStackSize(void *attr_) {
395	397	pthread_attr_t attr = (pthread_attr_t )attr_;
396	398	uptr stackaddr = 0;
397	399	uptr stacksize = 0;
398		my_pthread_attr_getstack(attr, (void**)&stackaddr, &stacksize);
	400	internal_pthread_attr_getstack(attr, (void **)&stackaddr, &stacksize);
399	401	// GLibC will return (0 - stacksize) as the stack address in the case when
400	402	// stacksize is set, but stackaddr is not.
401	403	bool stack_set = (stackaddr != 0) && (stackaddr + stacksize != 0);

lib/tsan/sanitizer_common/sanitizer_printf.cpp+13-24

...	...	@@ -20,10 +20,6 @@
20	20	#include <stdio.h>
21	21	#include <stdarg.h>
22	22
23		#if defined(__x86_64__)
24		# include <emmintrin.h>
25		#endif
26
27	23	#if SANITIZER_WINDOWS && defined(_MSC_VER) && _MSC_VER < 1800 && \
28	24	!defined(va_copy)
29	25	# define va_copy(dst, src) ((dst) = (src))
...	...	@@ -132,8 +128,8 @@ static int AppendPointer(char *buff, const char buff_end, u64 ptr_value) {
132	128	int VSNPrintf(char *buff, int buff_length,
133	129	const char *format, va_list args) {
134	130	static const char *kPrintfFormatsHelp =
135		"Supported Printf formats: %([0-9]*)?(z\|ll)?{d,u,x,X,V}; %p; "
136		"%[-]([0-9])?(\\.\\)?s; %c\n";
	131	"Supported Printf formats: %([0-9]*)?(z\|l\|ll)?{d,u,x,X}; %p; "
	132	"%[-]([0-9])?(\\.\\)?s; %c\nProvided format: ";
137	133	RAW_CHECK(format);
138	134	RAW_CHECK(buff_length > 0);
139	135	const char *buff_end = &buff[buff_length - 1];
...	...	@@ -164,9 +160,11 @@ int VSNPrintf(char *buff, int buff_length,
164	160	}
165	161	bool have_z = (*cur == 'z');
166	162	cur += have_z;
167		bool have_ll = !have_z && (cur[0] == 'l' && cur[1] == 'l');
	163	bool have_l = cur[0] == 'l' && cur[1] != 'l';
	164	cur += have_l;
	165	bool have_ll = cur[0] == 'l' && cur[1] == 'l';
168	166	cur += have_ll * 2;
169		const bool have_length = have_z \|\| have_ll;
	167	const bool have_length = have_z \|\| have_l \|\| have_ll;
170	168	const bool have_flags = have_width \|\| have_length;
171	169	// At the moment only %s supports precision and left-justification.
172	170	CHECK(!((precision >= 0 \|\| left_justified) && *cur != 's'));
...	...	@@ -174,6 +172,7 @@ int VSNPrintf(char *buff, int buff_length,
174	172	case 'd': {
175	173	s64 dval = have_ll ? va_arg(args, s64)
176	174	: have_z ? va_arg(args, sptr)
	175	: have_l ? va_arg(args, long)
177	176	: va_arg(args, int);
178	177	result += AppendSignedDecimal(&buff, buff_end, dval, width,
179	178	pad_with_zero);
...	...	@@ -184,26 +183,20 @@ int VSNPrintf(char *buff, int buff_length,
184	183	case 'X': {
185	184	u64 uval = have_ll ? va_arg(args, u64)
186	185	: have_z ? va_arg(args, uptr)
	186	: have_l ? va_arg(args, unsigned long)
187	187	: va_arg(args, unsigned);
188	188	bool uppercase = (*cur == 'X');
189	189	result += AppendUnsigned(&buff, buff_end, uval, (*cur == 'u') ? 10 : 16,
190	190	width, pad_with_zero, uppercase);
191	191	break;
192	192	}
193		case 'V': {
194		for (uptr i = 0; i < 16; i++) {
195		unsigned x = va_arg(args, unsigned);
196		result += AppendUnsigned(&buff, buff_end, x, 16, 2, true, false);
197		}
198		break;
199		}
200	193	case 'p': {
201		RAW_CHECK_MSG(!have_flags, kPrintfFormatsHelp);
	194	RAW_CHECK_VA(!have_flags, kPrintfFormatsHelp, format);
202	195	result += AppendPointer(&buff, buff_end, va_arg(args, uptr));
203	196	break;
204	197	}
205	198	case 's': {
206		RAW_CHECK_MSG(!have_length, kPrintfFormatsHelp);
	199	RAW_CHECK_VA(!have_length, kPrintfFormatsHelp, format);
207	200	// Only left-justified width is supported.
208	201	CHECK(!have_width \|\| left_justified);
209	202	result += AppendString(&buff, buff_end, left_justified ? -width : width,
...	...	@@ -211,17 +204,17 @@ int VSNPrintf(char *buff, int buff_length,
211	204	break;
212	205	}
213	206	case 'c': {
214		RAW_CHECK_MSG(!have_flags, kPrintfFormatsHelp);
	207	RAW_CHECK_VA(!have_flags, kPrintfFormatsHelp, format);
215	208	result += AppendChar(&buff, buff_end, va_arg(args, int));
216	209	break;
217	210	}
218	211	case '%' : {
219		RAW_CHECK_MSG(!have_flags, kPrintfFormatsHelp);
	212	RAW_CHECK_VA(!have_flags, kPrintfFormatsHelp, format);
220	213	result += AppendChar(&buff, buff_end, '%');
221	214	break;
222	215	}
223	216	default: {
224		RAW_CHECK_MSG(false, kPrintfFormatsHelp);
	217	RAW_CHECK_VA(false, kPrintfFormatsHelp, format);
225	218	}
226	219	}
227	220	}
...	...	@@ -317,7 +310,6 @@ static void NOINLINE SharedPrintfCode(bool append_pid, const char *format,
317	310	format, args);
318	311	}
319	312
320		FORMAT(1, 2)
321	313	void Printf(const char *format, ...) {
322	314	va_list args;
323	315	va_start(args, format);
...	...	@@ -326,7 +318,6 @@ void Printf(const char *format, ...) {
326	318	}
327	319
328	320	// Like Printf, but prints the current PID before the output string.
329		FORMAT(1, 2)
330	321	void Report(const char *format, ...) {
331	322	va_list args;
332	323	va_start(args, format);
...	...	@@ -338,7 +329,6 @@ void Report(const char *format, ...) {
338	329	// Returns the number of symbols that should have been written to buffer
339	330	// (not including trailing '\0'). Thus, the string is truncated
340	331	// iff return value is not less than "length".
341		FORMAT(3, 4)
342	332	int internal_snprintf(char buffer, uptr length, const char format, ...) {
343	333	va_list args;
344	334	va_start(args, format);
...	...	@@ -347,7 +337,6 @@ int internal_snprintf(char buffer, uptr length, const char format, ...) {
347	337	return needed_length;
348	338	}
349	339
350		FORMAT(2, 3)
351	340	void InternalScopedString::append(const char *format, ...) {
352	341	uptr prev_len = length();
353	342

lib/tsan/sanitizer_common/sanitizer_procmaps.h+35-7

...	...	@@ -16,7 +16,7 @@
16	16	#include "sanitizer_platform.h"
17	17
18	18	#if SANITIZER_LINUX \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| \
19		SANITIZER_MAC \|\| SANITIZER_SOLARIS \|\| \
	19	SANITIZER_APPLE \|\| SANITIZER_SOLARIS \|\| \
20	20	SANITIZER_FUCHSIA
21	21
22	22	#include "sanitizer_common.h"
...	...	@@ -65,13 +65,37 @@ class MemoryMappedSegment {
65	65	MemoryMappedSegmentData *data_;
66	66	};
67	67
68		class MemoryMappingLayout {
	68	struct ImageHeader;
	69
	70	class MemoryMappingLayoutBase {
	71	public:
	72	virtual bool Next(MemoryMappedSegment *segment) { UNIMPLEMENTED(); }
	73	virtual bool Error() const { UNIMPLEMENTED(); };
	74	virtual void Reset() { UNIMPLEMENTED(); }
	75
	76	protected:
	77	~MemoryMappingLayoutBase() {}
	78	};
	79
	80	class MemoryMappingLayout : public MemoryMappingLayoutBase {
69	81	public:
70	82	explicit MemoryMappingLayout(bool cache_enabled);
	83
	84	// This destructor cannot be virtual, as it would cause an operator new() linking
	85	// failures in hwasan test cases. However non-virtual destructors emit warnings
	86	// in macOS build, hence disabling those
	87	#ifdef __clang__
	88	#pragma clang diagnostic push
	89	#pragma clang diagnostic ignored "-Wnon-virtual-dtor"
	90	#endif
71	91	~MemoryMappingLayout();
72		bool Next(MemoryMappedSegment *segment);
73		bool Error() const;
74		void Reset();
	92	#ifdef __clang__
	93	#pragma clang diagnostic pop
	94	#endif
	95
	96	virtual bool Next(MemoryMappedSegment *segment) override;
	97	virtual bool Error() const override;
	98	virtual void Reset() override;
75	99	// In some cases, e.g. when running under a sandbox on Linux, ASan is unable
76	100	// to obtain the memory mappings. It should fall back to pre-cached data
77	101	// instead of aborting.
...	...	@@ -80,10 +104,14 @@ class MemoryMappingLayout {
80	104	// Adds all mapped objects into a vector.
81	105	void DumpListOfModules(InternalMmapVectorNoCtor<LoadedModule> *modules);
82	106
	107	protected:
	108	#if SANITIZER_APPLE
	109	virtual const ImageHeader *CurrentImageHeader();
	110	#endif
	111	MemoryMappingLayoutData data_;
	112
83	113	private:
84	114	void LoadFromCache();
85
86		MemoryMappingLayoutData data_;
87	115	};
88	116
89	117	// Returns code range for the specified module.

lib/tsan/sanitizer_common/sanitizer_procmaps_bsd.cpp+16

...	...	@@ -39,6 +39,22 @@
39	39
40	40	namespace __sanitizer {
41	41
	42	#if SANITIZER_FREEBSD
	43	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {
	44	const int Mib[] = {
	45	CTL_KERN,
	46	KERN_PROC,
	47	KERN_PROC_PID,
	48	getpid()
	49	};
	50
	51	struct kinfo_proc InfoProc;
	52	uptr Len = sizeof(InfoProc);
	53	CHECK_EQ(internal_sysctl(Mib, ARRAY_SIZE(Mib), nullptr, (uptr *)&InfoProc, &Len, 0), 0);
	54	cb(0, InfoProc.ki_rssize * GetPageSizeCached(), false, stats);
	55	}
	56	#endif
	57
42	58	void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
43	59	const int Mib[] = {
44	60	#if SANITIZER_FREEBSD

lib/tsan/sanitizer_common/sanitizer_procmaps_common.cpp+23-5

...	...	@@ -145,29 +145,47 @@ void MemoryMappingLayout::DumpListOfModules(
145	145	}
146	146	}
147	147
148		void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {
	148	#if SANITIZER_LINUX \|\| SANITIZER_ANDROID \|\| SANITIZER_SOLARIS \|\| SANITIZER_NETBSD
	149	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {
149	150	char *smaps = nullptr;
150	151	uptr smaps_cap = 0;
151	152	uptr smaps_len = 0;
152	153	if (!ReadFileToBuffer("/proc/self/smaps", &smaps, &smaps_cap, &smaps_len))
153	154	return;
	155	ParseUnixMemoryProfile(cb, stats, smaps, smaps_len);
	156	UnmapOrDie(smaps, smaps_cap);
	157	}
	158
	159	void ParseUnixMemoryProfile(fill_profile_f cb, uptr stats, char smaps,
	160	uptr smaps_len) {
154	161	uptr start = 0;
155	162	bool file = false;
156	163	const char *pos = smaps;
157		while (pos < smaps + smaps_len) {
	164	char *end = smaps + smaps_len;
	165	if (smaps_len < 2)
	166	return;
	167	// The following parsing can crash on almost every line
	168	// in the case of malformed/truncated input.
	169	// Fixing that is hard b/c e.g. ParseDecimal does not
	170	// even accept end of the buffer and assumes well-formed input.
	171	// So instead we patch end of the input a bit,
	172	// it does not affect well-formed complete inputs.
	173	*--end = 0;
	174	*--end = '\n';
	175	while (pos < end) {
158	176	if (IsHex(pos[0])) {
159	177	start = ParseHex(&pos);
160	178	for (; pos != '/' && pos > '\n'; pos++) {}
161	179	file = *pos == '/';
162	180	} else if (internal_strncmp(pos, "Rss:", 4) == 0) {
163		while (!IsDecimal(*pos)) pos++;
	181	while (pos < end && !IsDecimal(*pos)) pos++;
164	182	uptr rss = ParseDecimal(&pos) * 1024;
165		cb(start, rss, file, stats, stats_size);
	183	cb(start, rss, file, stats);
166	184	}
167	185	while (*pos++ != '\n') {}
168	186	}
169		UnmapOrDie(smaps, smaps_cap);
170	187	}
	188	#endif
171	189
172	190	} // namespace __sanitizer
173	191

lib/tsan/sanitizer_common/sanitizer_procmaps_mac.cpp+91-17

...	...	@@ -10,7 +10,7 @@
10	10	//===----------------------------------------------------------------------===//
11	11
12	12	#include "sanitizer_platform.h"
13		#if SANITIZER_MAC
	13	#if SANITIZER_APPLE
14	14	#include "sanitizer_common.h"
15	15	#include "sanitizer_placement_new.h"
16	16	#include "sanitizer_procmaps.h"
...	...	@@ -136,29 +136,34 @@ void MemoryMappingLayout::LoadFromCache() {
136	136	// No-op on Mac for now.
137	137	}
138	138
	139	static bool IsDyldHdr(const mach_header *hdr) {
	140	return (hdr->magic == MH_MAGIC \|\| hdr->magic == MH_MAGIC_64) &&
	141	hdr->filetype == MH_DYLINKER;
	142	}
	143
139	144	// _dyld_get_image_header() and related APIs don't report dyld itself.
140	145	// We work around this by manually recursing through the memory map
141	146	// until we hit a Mach header matching dyld instead. These recurse
142	147	// calls are expensive, but the first memory map generation occurs
143	148	// early in the process, when dyld is one of the only images loaded,
144		// so it will be hit after only a few iterations.
145		static mach_header *get_dyld_image_header() {
146		unsigned depth = 1;
147		vm_size_t size = 0;
	149	// so it will be hit after only a few iterations. These assumptions don't hold
	150	// on macOS 13+ anymore (dyld itself has moved into the shared cache).
	151	static mach_header *GetDyldImageHeaderViaVMRegion() {
148	152	vm_address_t address = 0;
149		kern_return_t err = KERN_SUCCESS;
150		mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
151	153
152	154	while (true) {
	155	vm_size_t size = 0;
	156	unsigned depth = 1;
153	157	struct vm_region_submap_info_64 info;
154		err = vm_region_recurse_64(mach_task_self(), &address, &size, &depth,
155		(vm_region_info_t)&info, &count);
	158	mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
	159	kern_return_t err =
	160	vm_region_recurse_64(mach_task_self(), &address, &size, &depth,
	161	(vm_region_info_t)&info, &count);
156	162	if (err != KERN_SUCCESS) return nullptr;
157	163
158	164	if (size >= sizeof(mach_header) && info.protection & kProtectionRead) {
159	165	mach_header hdr = (mach_header )address;
160		if ((hdr->magic == MH_MAGIC \|\| hdr->magic == MH_MAGIC_64) &&
161		hdr->filetype == MH_DYLINKER) {
	166	if (IsDyldHdr(hdr)) {
162	167	return hdr;
163	168	}
164	169	}
...	...	@@ -166,8 +171,69 @@ static mach_header *get_dyld_image_header() {
166	171	}
167	172	}
168	173
	174	extern "C" {
	175	struct dyld_shared_cache_dylib_text_info {
	176	uint64_t version; // current version 2
	177	// following fields all exist in version 1
	178	uint64_t loadAddressUnslid;
	179	uint64_t textSegmentSize;
	180	uuid_t dylibUuid;
	181	const char *path; // pointer invalid at end of iterations
	182	// following fields all exist in version 2
	183	uint64_t textSegmentOffset; // offset from start of cache
	184	};
	185	typedef struct dyld_shared_cache_dylib_text_info
	186	dyld_shared_cache_dylib_text_info;
	187
	188	extern bool _dyld_get_shared_cache_uuid(uuid_t uuid);
	189	extern const void _dyld_get_shared_cache_range(size_t length);
	190	extern int dyld_shared_cache_iterate_text(
	191	const uuid_t cacheUuid,
	192	void (^callback)(const dyld_shared_cache_dylib_text_info *info));
	193	} // extern "C"
	194
	195	static mach_header *GetDyldImageHeaderViaSharedCache() {
	196	uuid_t uuid;
	197	bool hasCache = _dyld_get_shared_cache_uuid(uuid);
	198	if (!hasCache)
	199	return nullptr;
	200
	201	size_t cacheLength;
	202	__block uptr cacheStart = (uptr)_dyld_get_shared_cache_range(&cacheLength);
	203	CHECK(cacheStart && cacheLength);
	204
	205	__block mach_header *dyldHdr = nullptr;
	206	int res = dyld_shared_cache_iterate_text(
	207	uuid, ^(const dyld_shared_cache_dylib_text_info *info) {
	208	CHECK_GE(info->version, 2);
	209	mach_header *hdr =
	210	(mach_header *)(cacheStart + info->textSegmentOffset);
	211	if (IsDyldHdr(hdr))
	212	dyldHdr = hdr;
	213	});
	214	CHECK_EQ(res, 0);
	215
	216	return dyldHdr;
	217	}
	218
169	219	const mach_header *get_dyld_hdr() {
170		if (!dyld_hdr) dyld_hdr = get_dyld_image_header();
	220	if (!dyld_hdr) {
	221	// On macOS 13+, dyld itself has moved into the shared cache. Looking it up
	222	// via vm_region_recurse_64() causes spins/hangs/crashes.
	223	if (GetMacosAlignedVersion() >= MacosVersion(13, 0)) {
	224	dyld_hdr = GetDyldImageHeaderViaSharedCache();
	225	if (!dyld_hdr) {
	226	VReport(1,
	227	"Failed to lookup the dyld image header in the shared cache on "
	228	"macOS 13+ (or no shared cache in use). Falling back to "
	229	"lookup via vm_region_recurse_64().\n");
	230	dyld_hdr = GetDyldImageHeaderViaVMRegion();
	231	}
	232	} else {
	233	dyld_hdr = GetDyldImageHeaderViaVMRegion();
	234	}
	235	CHECK(dyld_hdr);
	236	}
171	237
172	238	return dyld_hdr;
173	239	}
...	...	@@ -184,7 +250,9 @@ static bool NextSegmentLoad(MemoryMappedSegment *segment,
184	250	MemoryMappedSegmentData *seg_data,
185	251	MemoryMappingLayoutData *layout_data) {
186	252	const char *lc = layout_data->current_load_cmd_addr;
	253
187	254	layout_data->current_load_cmd_addr += ((const load_command *)lc)->cmdsize;
	255	layout_data->current_load_cmd_count--;
188	256	if (((const load_command *)lc)->cmd == kLCSegment) {
189	257	const SegmentCommand* sc = (const SegmentCommand *)lc;
190	258	uptr base_virt_addr, addr_mask;
...	...	@@ -292,11 +360,16 @@ static bool IsModuleInstrumented(const load_command *first_lc) {
292	360	return false;
293	361	}
294	362
	363	const ImageHeader *MemoryMappingLayout::CurrentImageHeader() {
	364	const mach_header *hdr = (data_.current_image == kDyldImageIdx)
	365	? get_dyld_hdr()
	366	: _dyld_get_image_header(data_.current_image);
	367	return (const ImageHeader *)hdr;
	368	}
	369
295	370	bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
296	371	for (; data_.current_image >= kDyldImageIdx; data_.current_image--) {
297		const mach_header *hdr = (data_.current_image == kDyldImageIdx)
298		? get_dyld_hdr()
299		: _dyld_get_image_header(data_.current_image);
	372	const mach_header hdr = (const mach_header )CurrentImageHeader();
300	373	if (!hdr) continue;
301	374	if (data_.current_load_cmd_count < 0) {
302	375	// Set up for this image;
...	...	@@ -326,7 +399,7 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
326	399	(const load_command *)data_.current_load_cmd_addr);
327	400	}
328	401
329		for (; data_.current_load_cmd_count >= 0; data_.current_load_cmd_count--) {
	402	while (data_.current_load_cmd_count > 0) {
330	403	switch (data_.current_magic) {
331	404	// data_.current_magic may be only one of MH_MAGIC, MH_MAGIC_64.
332	405	#ifdef MH_MAGIC_64
...	...	@@ -347,6 +420,7 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
347	420	}
348	421	// If we get here, no more load_cmd's in this image talk about
349	422	// segments. Go on to the next image.
	423	data_.current_load_cmd_count = -1; // This will trigger loading next image
350	424	}
351	425	return false;
352	426	}
...	...	@@ -376,4 +450,4 @@ void MemoryMappingLayout::DumpListOfModules(
376	450
377	451	} // namespace __sanitizer
378	452
379		#endif // SANITIZER_MAC
	453	#endif // SANITIZER_APPLE

lib/tsan/sanitizer_common/sanitizer_procmaps_solaris.cpp+37-13

...	...	@@ -13,21 +13,30 @@
13	13	#undef _FILE_OFFSET_BITS
14	14	#include "sanitizer_platform.h"
15	15	#if SANITIZER_SOLARIS
16		#include "sanitizer_common.h"
17		#include "sanitizer_procmaps.h"
	16	# include <fcntl.h>
	17	# include <limits.h>
	18	# include <procfs.h>
18	19
19		#include <procfs.h>
20		#include <limits.h>
	20	# include "sanitizer_common.h"
	21	# include "sanitizer_procmaps.h"
21	22
22	23	namespace __sanitizer {
23	24
24	25	void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
25		if (!ReadFileToBuffer("/proc/self/xmap", &proc_maps->data,
26		&proc_maps->mmaped_size, &proc_maps->len)) {
27		proc_maps->data = nullptr;
28		proc_maps->mmaped_size = 0;
29		proc_maps->len = 0;
30		}
	26	uptr fd = internal_open("/proc/self/xmap", O_RDONLY);
	27	CHECK_NE(fd, -1);
	28	uptr Size = internal_filesize(fd);
	29	CHECK_GT(Size, 0);
	30
	31	// Allow for additional entries by following mmap.
	32	size_t MmapedSize = Size * 4 / 3;
	33	void *VmMap = MmapOrDie(MmapedSize, "ReadProcMaps()");
	34	Size = internal_read(fd, VmMap, MmapedSize);
	35	CHECK_NE(Size, -1);
	36	internal_close(fd);
	37	proc_maps->data = (char *)VmMap;
	38	proc_maps->mmaped_size = MmapedSize;
	39	proc_maps->len = Size;
31	40	}
32	41
33	42	bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
...	...	@@ -49,13 +58,28 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
49	58	segment->protection \|= kProtectionWrite;
50	59	if ((xmapentry->pr_mflags & MA_EXEC) != 0)
51	60	segment->protection \|= kProtectionExecute;
	61	if ((xmapentry->pr_mflags & MA_SHARED) != 0)
	62	segment->protection \|= kProtectionShared;
52	63
53	64	if (segment->filename != NULL && segment->filename_size > 0) {
54	65	char proc_path[PATH_MAX + 1];
55	66
56		internal_snprintf(proc_path, sizeof(proc_path), "/proc/self/path/%s",
57		xmapentry->pr_mapname);
58		internal_readlink(proc_path, segment->filename, segment->filename_size);
	67	// Avoid unnecessary readlink on unnamed entires.
	68	if (xmapentry->pr_mapname[0] == '\0')
	69	segment->filename[0] = '\0';
	70	else {
	71	internal_snprintf(proc_path, sizeof(proc_path), "/proc/self/path/%s",
	72	xmapentry->pr_mapname);
	73	ssize_t sz = internal_readlink(proc_path, segment->filename,
	74	segment->filename_size - 1);
	75
	76	// If readlink failed, the map is anonymous.
	77	if (sz == -1)
	78	segment->filename[0] = '\0';
	79	else if ((size_t)sz < segment->filename_size)
	80	// readlink doesn't NUL-terminate.
	81	segment->filename[sz] = '\0';
	82	}
59	83	}
60	84
61	85	data_.current += sizeof(prxmap_t);

lib/tsan/sanitizer_common/sanitizer_quarantine.h+12-17

...	...	@@ -68,10 +68,6 @@ struct QuarantineBatch {
68	68
69	69	COMPILER_CHECK(sizeof(QuarantineBatch) <= (1 << 13)); // 8Kb.
70	70
71		// The callback interface is:
72		// void Callback::Recycle(Node *ptr);
73		// void *cb.Allocate(uptr size);
74		// void cb.Deallocate(void *ptr);
75	71	template<typename Callback, typename Node>
76	72	class Quarantine {
77	73	public:
...	...	@@ -94,21 +90,20 @@ class Quarantine {
94	90	recycle_mutex_.Init();
95	91	}
96	92
97		uptr GetSize() const { return atomic_load_relaxed(&max_size_); }
98		uptr GetCacheSize() const {
99		return atomic_load_relaxed(&max_cache_size_);
100		}
	93	uptr GetMaxSize() const { return atomic_load_relaxed(&max_size_); }
	94	uptr GetMaxCacheSize() const { return atomic_load_relaxed(&max_cache_size_); }
101	95
102	96	void Put(Cache c, Callback cb, Node ptr, uptr size) {
103		uptr cache_size = GetCacheSize();
104		if (cache_size) {
	97	uptr max_cache_size = GetMaxCacheSize();
	98	if (max_cache_size && size <= GetMaxSize()) {
	99	cb.PreQuarantine(ptr);
105	100	c->Enqueue(cb, ptr, size);
106	101	} else {
107		// GetCacheSize() == 0 only when GetSize() == 0 (see Init).
108		cb.Recycle(ptr);
	102	// GetMaxCacheSize() == 0 only when GetMaxSize() == 0 (see Init).
	103	cb.RecyclePassThrough(ptr);
109	104	}
110	105	// Check cache size anyway to accommodate for runtime cache_size change.
111		if (c->Size() > cache_size)
	106	if (c->Size() > max_cache_size)
112	107	Drain(c, cb);
113	108	}
114	109
...	...	@@ -117,7 +112,7 @@ class Quarantine {
117	112	SpinMutexLock l(&cache_mutex_);
118	113	cache_.Transfer(c);
119	114	}
120		if (cache_.Size() > GetSize() && recycle_mutex_.TryLock())
	115	if (cache_.Size() > GetMaxSize() && recycle_mutex_.TryLock())
121	116	Recycle(atomic_load_relaxed(&min_size_), cb);
122	117	}
123	118
...	...	@@ -133,7 +128,7 @@ class Quarantine {
133	128	void PrintStats() const {
134	129	// It assumes that the world is stopped, just as the allocator's PrintStats.
135	130	Printf("Quarantine limits: global: %zdMb; thread local: %zdKb\n",
136		GetSize() >> 20, GetCacheSize() >> 10);
	131	GetMaxSize() >> 20, GetMaxCacheSize() >> 10);
137	132	cache_.PrintStats();
138	133	}
139	134
...	...	@@ -149,8 +144,8 @@ class Quarantine {
149	144	Cache cache_;
150	145	char pad2_[kCacheLineSize];
151	146
152		void NOINLINE Recycle(uptr min_size, Callback cb) REQUIRES(recycle_mutex_)
153		RELEASE(recycle_mutex_) {
	147	void NOINLINE Recycle(uptr min_size, Callback cb)
	148	SANITIZER_REQUIRES(recycle_mutex_) SANITIZER_RELEASE(recycle_mutex_) {
154	149	Cache tmp;
155	150	{
156	151	SpinMutexLock l(&cache_mutex_);

lib/tsan/sanitizer_common/sanitizer_range.cpp created+62

...	...	@@ -0,0 +1,62 @@
	1	//===-- sanitizer_range.cpp -----------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#include "sanitizer_range.h"
	10
	11	#include "sanitizer_common/sanitizer_array_ref.h"
	12
	13	namespace __sanitizer {
	14
	15	void Intersect(ArrayRef<Range> a, ArrayRef<Range> b,
	16	InternalMmapVectorNoCtor<Range> &output) {
	17	output.clear();
	18
	19	struct Event {
	20	uptr val;
	21	s8 diff1;
	22	s8 diff2;
	23	};
	24
	25	InternalMmapVector<Event> events;
	26	for (const Range &r : a) {
	27	CHECK_LE(r.begin, r.end);
	28	events.push_back({r.begin, 1, 0});
	29	events.push_back({r.end, -1, 0});
	30	}
	31
	32	for (const Range &r : b) {
	33	CHECK_LE(r.begin, r.end);
	34	events.push_back({r.begin, 0, 1});
	35	events.push_back({r.end, 0, -1});
	36	}
	37
	38	Sort(events.data(), events.size(),
	39	[](const Event &lh, const Event &rh) { return lh.val < rh.val; });
	40
	41	uptr start = 0;
	42	sptr state1 = 0;
	43	sptr state2 = 0;
	44	for (const auto &e : events) {
	45	if (e.val != start) {
	46	DCHECK_GE(state1, 0);
	47	DCHECK_GE(state2, 0);
	48	if (state1 && state2) {
	49	if (!output.empty() && start == output.back().end)
	50	output.back().end = e.val;
	51	else
	52	output.push_back({start, e.val});
	53	}
	54	start = e.val;
	55	}
	56
	57	state1 += e.diff1;
	58	state2 += e.diff2;
	59	}
	60	}
	61
	62	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_range.h created+40

...	...	@@ -0,0 +1,40 @@
	1	//===-- sanitizer_range.h ---------------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// Contais Range and related utilities.
	10	//
	11	//===----------------------------------------------------------------------===//
	12
	13	#ifndef SANITIZER_RANGE_H
	14	#define SANITIZER_RANGE_H
	15
	16	#include "sanitizer_common.h"
	17	#include "sanitizer_common/sanitizer_array_ref.h"
	18
	19	namespace __sanitizer {
	20
	21	struct Range {
	22	uptr begin;
	23	uptr end;
	24	};
	25
	26	inline bool operator==(const Range &lhs, const Range &rhs) {
	27	return lhs.begin == rhs.begin && lhs.end == rhs.end;
	28	}
	29
	30	inline bool operator!=(const Range &lhs, const Range &rhs) {
	31	return !(lhs == rhs);
	32	}
	33
	34	// Calculates intersection of two sets of regions in O(N log N) time.
	35	void Intersect(ArrayRef<Range> a, ArrayRef<Range> b,
	36	InternalMmapVectorNoCtor<Range> &output);
	37
	38	} // namespace __sanitizer
	39
	40	#endif // SANITIZER_RANGE_H

lib/tsan/sanitizer_common/sanitizer_redefine_builtins.h created+52

...	...	@@ -0,0 +1,52 @@
	1	//===-- sanitizer_redefine_builtins.h ---------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// Redefine builtin functions to use internal versions. This is needed where
	10	// compiler optimizations end up producing unwanted libcalls!
	11	//
	12	//===----------------------------------------------------------------------===//
	13	#ifndef SANITIZER_COMMON_NO_REDEFINE_BUILTINS
	14	#ifndef SANITIZER_REDEFINE_BUILTINS_H
	15	#define SANITIZER_REDEFINE_BUILTINS_H
	16
	17	// The asm hack only works with GCC and Clang.
	18	#if !defined(_WIN32)
	19
	20	asm("memcpy = __sanitizer_internal_memcpy");
	21	asm("memmove = __sanitizer_internal_memmove");
	22	asm("memset = __sanitizer_internal_memset");
	23
	24	// The builtins should not be redefined in source files that make use of C++
	25	// standard libraries, in particular where C++STL headers with inline functions
	26	// are used. The redefinition in such cases would lead to ODR violations.
	27	//
	28	// Try to break the build in common cases where builtins shouldn't be redefined.
	29	namespace std {
	30	class Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file {
	31	Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file(
	32	const Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file&) = delete;
	33	Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file& operator=(
	34	const Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file&) = delete;
	35	};
	36	using array = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
	37	using atomic = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
	38	using function = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
	39	using map = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
	40	using set = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
	41	using shared_ptr = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
	42	using string = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
	43	using unique_ptr = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
	44	using unordered_map = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
	45	using unordered_set = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
	46	using vector = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
	47	} // namespace std
	48
	49	#endif // !_WIN32
	50
	51	#endif // SANITIZER_REDEFINE_BUILTINS_H
	52	#endif // SANITIZER_COMMON_NO_REDEFINE_BUILTINS

lib/tsan/sanitizer_common/sanitizer_ring_buffer.h+9-4

...	...	@@ -86,10 +86,13 @@ class CompactRingBuffer {
86	86	// Lower bytes store the address of the next buffer element.
87	87	static constexpr int kPageSizeBits = 12;
88	88	static constexpr int kSizeShift = 56;
	89	static constexpr int kSizeBits = 64 - kSizeShift;
89	90	static constexpr uptr kNextMask = (1ULL << kSizeShift) - 1;
90	91
91	92	uptr GetStorageSize() const { return (long_ >> kSizeShift) << kPageSizeBits; }
92	93
	94	static uptr SignExtend(uptr x) { return ((sptr)x) << kSizeBits >> kSizeBits; }
	95
93	96	void Init(void *storage, uptr size) {
94	97	CHECK_EQ(sizeof(CompactRingBuffer<T>), sizeof(void *));
95	98	CHECK(IsPowerOfTwo(size));
...	...	@@ -97,12 +100,14 @@ class CompactRingBuffer {
97	100	CHECK_LE(size, 128 << kPageSizeBits);
98	101	CHECK_EQ(size % 4096, 0);
99	102	CHECK_EQ(size % sizeof(T), 0);
100		CHECK_EQ((uptr)storage % (size * 2), 0);
101		long_ = (uptr)storage \| ((size >> kPageSizeBits) << kSizeShift);
	103	uptr st = (uptr)storage;
	104	CHECK_EQ(st % (size * 2), 0);
	105	CHECK_EQ(st, SignExtend(st & kNextMask));
	106	long_ = (st & kNextMask) \| ((size >> kPageSizeBits) << kSizeShift);
102	107	}
103	108
104	109	void SetNext(const T *next) {
105		long_ = (long_ & ~kNextMask) \| (uptr)next;
	110	long_ = (long_ & ~kNextMask) \| ((uptr)next & kNextMask);
106	111	}
107	112
108	113	public:
...	...	@@ -119,7 +124,7 @@ class CompactRingBuffer {
119	124	SetNext((const T *)storage + Idx);
120	125	}
121	126
122		T Next() const { return (T )(long_ & kNextMask); }
	127	T Next() const { return (T )(SignExtend(long_ & kNextMask)); }
123	128
124	129	void *StartOfStorage() const {
125	130	return (void *)((uptr)Next() & ~(GetStorageSize() - 1));

lib/tsan/sanitizer_common/sanitizer_signal_interceptors.inc+13-2

...	...	@@ -29,12 +29,21 @@ using namespace __sanitizer;
29	29	#endif
30	30
31	31	#ifndef SIGNAL_INTERCEPTOR_SIGACTION_IMPL
32		#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact) \
33		{ return REAL(sigaction_symname)(signum, act, oldact); }
	32	# define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact) \
	33	{ \
	34	if (!REAL(sigaction_symname)) { \
	35	Printf( \
	36	"Warning: REAL(sigaction_symname) == nullptr. This may happen " \
	37	"if you link with ubsan statically. Sigaction will not work.\n"); \
	38	return -1; \
	39	} \
	40	return REAL(sigaction_symname)(signum, act, oldact); \
	41	}
34	42	#endif
35	43
36	44	#if SANITIZER_INTERCEPT_BSD_SIGNAL
37	45	INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) {
	46	SIGNAL_INTERCEPTOR_ENTER();
38	47	if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0;
39	48	SIGNAL_INTERCEPTOR_SIGNAL_IMPL(bsd_signal, signum, handler);
40	49	}
...	...	@@ -45,6 +54,7 @@ INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) {
45	54
46	55	#if SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION
47	56	INTERCEPTOR(uptr, signal, int signum, uptr handler) {
	57	SIGNAL_INTERCEPTOR_ENTER();
48	58	if (GetHandleSignalMode(signum) == kHandleSignalExclusive)
49	59	return (uptr) nullptr;
50	60	SIGNAL_INTERCEPTOR_SIGNAL_IMPL(signal, signum, handler);
...	...	@@ -53,6 +63,7 @@ INTERCEPTOR(uptr, signal, int signum, uptr handler) {
53	63
54	64	INTERCEPTOR(int, sigaction_symname, int signum,
55	65	const __sanitizer_sigaction act, __sanitizer_sigaction oldact) {
	66	SIGNAL_INTERCEPTOR_ENTER();
56	67	if (GetHandleSignalMode(signum) == kHandleSignalExclusive) {
57	68	if (!oldact) return 0;
58	69	act = nullptr;

lib/tsan/sanitizer_common/sanitizer_solaris.cpp-22

...	...	@@ -225,28 +225,6 @@ void FutexWait(atomic_uint32_t *p, u32 cmp) {
225	225
226	226	void FutexWake(atomic_uint32_t *p, u32 count) {}
227	227
228		BlockingMutex::BlockingMutex() {
229		CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_));
230		internal_memset(this, 0, sizeof(*this));
231		CHECK_EQ(mutex_init((mutex_t *)&opaque_storage_, USYNC_THREAD, NULL), 0);
232		}
233
234		void BlockingMutex::Lock() {
235		CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_));
236		CHECK_NE(owner_, (uptr)thr_self());
237		CHECK_EQ(mutex_lock((mutex_t *)&opaque_storage_), 0);
238		CHECK(!owner_);
239		owner_ = (uptr)thr_self();
240		}
241
242		void BlockingMutex::Unlock() {
243		CHECK(owner_ == (uptr)thr_self());
244		owner_ = 0;
245		CHECK_EQ(mutex_unlock((mutex_t *)&opaque_storage_), 0);
246		}
247
248		void BlockingMutex::CheckLocked() const { CHECK_EQ((uptr)thr_self(), owner_); }
249
250	228	} // namespace __sanitizer
251	229
252	230	#endif // SANITIZER_SOLARIS

lib/tsan/sanitizer_common/sanitizer_solaris.h created+56

...	...	@@ -0,0 +1,56 @@
	1	//===-- sanitizer_solaris.h -------------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of Sanitizer runtime. It contains Solaris-specific
	10	// definitions.
	11	//
	12	//===----------------------------------------------------------------------===//
	13
	14	#ifndef SANITIZER_SOLARIS_H
	15	#define SANITIZER_SOLARIS_H
	16
	17	#include "sanitizer_internal_defs.h"
	18
	19	#if SANITIZER_SOLARIS
	20
	21	#include <link.h>
	22
	23	namespace __sanitizer {
	24
	25	// Beginning of declaration from OpenSolaris/Illumos
	26	// $SRC/cmd/sgs/include/rtld.h.
	27	struct Rt_map {
	28	Link_map rt_public;
	29	const char *rt_pathname;
	30	ulong_t rt_padstart;
	31	ulong_t rt_padimlen;
	32	ulong_t rt_msize;
	33	uint_t rt_flags;
	34	uint_t rt_flags1;
	35	ulong_t rt_tlsmodid;
	36	};
	37
	38	// Structure matching the Solaris 11.4 struct dl_phdr_info used to determine
	39	// presence of dlpi_tls_modid field at runtime. Cf. Solaris 11.4
	40	// dl_iterate_phdr(3C), Example 2.
	41	struct dl_phdr_info_test {
	42	ElfW(Addr) dlpi_addr;
	43	const char *dlpi_name;
	44	const ElfW(Phdr) * dlpi_phdr;
	45	ElfW(Half) dlpi_phnum;
	46	u_longlong_t dlpi_adds;
	47	u_longlong_t dlpi_subs;
	48	size_t dlpi_tls_modid;
	49	void *dlpi_tls_data;
	50	};
	51
	52	} // namespace __sanitizer
	53
	54	#endif // SANITIZER_SOLARIS
	55
	56	#endif // SANITIZER_SOLARIS_H

lib/tsan/sanitizer_common/sanitizer_stack_store.cpp created+379

...	...	@@ -0,0 +1,379 @@
	1	//===-- sanitizer_stack_store.cpp -------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#include "sanitizer_stack_store.h"
	10
	11	#include "sanitizer_atomic.h"
	12	#include "sanitizer_common.h"
	13	#include "sanitizer_internal_defs.h"
	14	#include "sanitizer_leb128.h"
	15	#include "sanitizer_lzw.h"
	16	#include "sanitizer_placement_new.h"
	17	#include "sanitizer_stacktrace.h"
	18
	19	namespace __sanitizer {
	20
	21	namespace {
	22	struct StackTraceHeader {
	23	static constexpr u32 kStackSizeBits = 8;
	24
	25	u8 size;
	26	u8 tag;
	27	explicit StackTraceHeader(const StackTrace &trace)
	28	: size(Min<uptr>(trace.size, (1u << 8) - 1)), tag(trace.tag) {
	29	CHECK_EQ(trace.tag, static_cast<uptr>(tag));
	30	}
	31	explicit StackTraceHeader(uptr h)
	32	: size(h & ((1 << kStackSizeBits) - 1)), tag(h >> kStackSizeBits) {}
	33
	34	uptr ToUptr() const {
	35	return static_cast<uptr>(size) \| (static_cast<uptr>(tag) << kStackSizeBits);
	36	}
	37	};
	38	} // namespace
	39
	40	StackStore::Id StackStore::Store(const StackTrace &trace, uptr *pack) {
	41	if (!trace.size && !trace.tag)
	42	return 0;
	43	StackTraceHeader h(trace);
	44	uptr idx = 0;
	45	*pack = 0;
	46	uptr *stack_trace = Alloc(h.size + 1, &idx, pack);
	47	*stack_trace = h.ToUptr();
	48	internal_memcpy(stack_trace + 1, trace.trace, h.size * sizeof(uptr));
	49	*pack += blocks_[GetBlockIdx(idx)].Stored(h.size + 1);
	50	return OffsetToId(idx);
	51	}
	52
	53	StackTrace StackStore::Load(Id id) {
	54	if (!id)
	55	return {};
	56	uptr idx = IdToOffset(id);
	57	uptr block_idx = GetBlockIdx(idx);
	58	CHECK_LT(block_idx, ARRAY_SIZE(blocks_));
	59	const uptr *stack_trace = blocks_[block_idx].GetOrUnpack(this);
	60	if (!stack_trace)
	61	return {};
	62	stack_trace += GetInBlockIdx(idx);
	63	StackTraceHeader h(*stack_trace);
	64	return StackTrace(stack_trace + 1, h.size, h.tag);
	65	}
	66
	67	uptr StackStore::Allocated() const {
	68	return atomic_load_relaxed(&allocated_) + sizeof(*this);
	69	}
	70
	71	uptr StackStore::Alloc(uptr count, uptr idx, uptr *pack) {
	72	for (;;) {
	73	// Optimisic lock-free allocation, essentially try to bump the
	74	// total_frames_.
	75	uptr start = atomic_fetch_add(&total_frames_, count, memory_order_relaxed);
	76	uptr block_idx = GetBlockIdx(start);
	77	uptr last_idx = GetBlockIdx(start + count - 1);
	78	if (LIKELY(block_idx == last_idx)) {
	79	// Fits into the a single block.
	80	CHECK_LT(block_idx, ARRAY_SIZE(blocks_));
	81	*idx = start;
	82	return blocks_[block_idx].GetOrCreate(this) + GetInBlockIdx(start);
	83	}
	84
	85	// Retry. We can't use range allocated in two different blocks.
	86	CHECK_LE(count, kBlockSizeFrames);
	87	uptr in_first = kBlockSizeFrames - GetInBlockIdx(start);
	88	// Mark tail/head of these blocks as "stored".to avoid waiting before we can
	89	// Pack().
	90	*pack += blocks_[block_idx].Stored(in_first);
	91	*pack += blocks_[last_idx].Stored(count - in_first);
	92	}
	93	}
	94
	95	void StackStore::Map(uptr size, const char mem_type) {
	96	atomic_fetch_add(&allocated_, size, memory_order_relaxed);
	97	return MmapNoReserveOrDie(size, mem_type);
	98	}
	99
	100	void StackStore::Unmap(void *addr, uptr size) {
	101	atomic_fetch_sub(&allocated_, size, memory_order_relaxed);
	102	UnmapOrDie(addr, size);
	103	}
	104
	105	uptr StackStore::Pack(Compression type) {
	106	uptr res = 0;
	107	for (BlockInfo &b : blocks_) res += b.Pack(type, this);
	108	return res;
	109	}
	110
	111	void StackStore::LockAll() {
	112	for (BlockInfo &b : blocks_) b.Lock();
	113	}
	114
	115	void StackStore::UnlockAll() {
	116	for (BlockInfo &b : blocks_) b.Unlock();
	117	}
	118
	119	void StackStore::TestOnlyUnmap() {
	120	for (BlockInfo &b : blocks_) b.TestOnlyUnmap(this);
	121	internal_memset(this, 0, sizeof(*this));
	122	}
	123
	124	uptr *StackStore::BlockInfo::Get() const {
	125	// Idiomatic double-checked locking uses memory_order_acquire here. But
	126	// relaxed is fine for us, justification is similar to
	127	// TwoLevelMap::GetOrCreate.
	128	return reinterpret_cast<uptr *>(atomic_load_relaxed(&data_));
	129	}
	130
	131	uptr StackStore::BlockInfo::Create(StackStore store) {
	132	SpinMutexLock l(&mtx_);
	133	uptr *ptr = Get();
	134	if (!ptr) {
	135	ptr = reinterpret_cast<uptr *>(store->Map(kBlockSizeBytes, "StackStore"));
	136	atomic_store(&data_, reinterpret_cast<uptr>(ptr), memory_order_release);
	137	}
	138	return ptr;
	139	}
	140
	141	uptr StackStore::BlockInfo::GetOrCreate(StackStore store) {
	142	uptr *ptr = Get();
	143	if (LIKELY(ptr))
	144	return ptr;
	145	return Create(store);
	146	}
	147
	148	class SLeb128Encoder {
	149	public:
	150	SLeb128Encoder(u8 begin, u8 end) : begin(begin), end(end) {}
	151
	152	bool operator==(const SLeb128Encoder &other) const {
	153	return begin == other.begin;
	154	}
	155
	156	bool operator!=(const SLeb128Encoder &other) const {
	157	return begin != other.begin;
	158	}
	159
	160	SLeb128Encoder &operator=(uptr v) {
	161	sptr diff = v - previous;
	162	begin = EncodeSLEB128(diff, begin, end);
	163	previous = v;
	164	return *this;
	165	}
	166	SLeb128Encoder &operator() { return this; }
	167	SLeb128Encoder &operator++() { return *this; }
	168
	169	u8 *base() const { return begin; }
	170
	171	private:
	172	u8 *begin;
	173	u8 *end;
	174	uptr previous = 0;
	175	};
	176
	177	class SLeb128Decoder {
	178	public:
	179	SLeb128Decoder(const u8 begin, const u8 end) : begin(begin), end(end) {}
	180
	181	bool operator==(const SLeb128Decoder &other) const {
	182	return begin == other.begin;
	183	}
	184
	185	bool operator!=(const SLeb128Decoder &other) const {
	186	return begin != other.begin;
	187	}
	188
	189	uptr operator*() {
	190	sptr diff;
	191	begin = DecodeSLEB128(begin, end, &diff);
	192	previous += diff;
	193	return previous;
	194	}
	195	SLeb128Decoder &operator++() { return *this; }
	196
	197	SLeb128Decoder operator++(int) { return *this; }
	198
	199	private:
	200	const u8 *begin;
	201	const u8 *end;
	202	uptr previous = 0;
	203	};
	204
	205	static u8 CompressDelta(const uptr from, const uptr from_end, u8 to,
	206	u8 *to_end) {
	207	SLeb128Encoder encoder(to, to_end);
	208	for (; from != from_end; ++from, ++encoder) encoder = from;
	209	return encoder.base();
	210	}
	211
	212	static uptr UncompressDelta(const u8 from, const u8 from_end, uptr to,
	213	uptr *to_end) {
	214	SLeb128Decoder decoder(from, from_end);
	215	SLeb128Decoder end(from_end, from_end);
	216	for (; decoder != end; ++to, ++decoder) to = decoder;
	217	CHECK_EQ(to, to_end);
	218	return to;
	219	}
	220
	221	static u8 CompressLzw(const uptr from, const uptr from_end, u8 to,
	222	u8 *to_end) {
	223	SLeb128Encoder encoder(to, to_end);
	224	encoder = LzwEncode<uptr>(from, from_end, encoder);
	225	return encoder.base();
	226	}
	227
	228	static uptr UncompressLzw(const u8 from, const u8 from_end, uptr to,
	229	uptr *to_end) {
	230	SLeb128Decoder decoder(from, from_end);
	231	SLeb128Decoder end(from_end, from_end);
	232	to = LzwDecode<uptr>(decoder, end, to);
	233	CHECK_EQ(to, to_end);
	234	return to;
	235	}
	236
	237	#if defined(_MSC_VER) && !defined(__clang__)
	238	# pragma warning(push)
	239	// Disable 'nonstandard extension used: zero-sized array in struct/union'.
	240	# pragma warning(disable : 4200)
	241	#endif
	242	namespace {
	243	struct PackedHeader {
	244	uptr size;
	245	StackStore::Compression type;
	246	u8 data[];
	247	};
	248	} // namespace
	249	#if defined(_MSC_VER) && !defined(__clang__)
	250	# pragma warning(pop)
	251	#endif
	252
	253	uptr StackStore::BlockInfo::GetOrUnpack(StackStore store) {
	254	SpinMutexLock l(&mtx_);
	255	switch (state) {
	256	case State::Storing:
	257	state = State::Unpacked;
	258	FALLTHROUGH;
	259	case State::Unpacked:
	260	return Get();
	261	case State::Packed:
	262	break;
	263	}
	264
	265	u8 ptr = reinterpret_cast<u8 >(Get());
	266	CHECK_NE(nullptr, ptr);
	267	const PackedHeader header = reinterpret_cast<const PackedHeader >(ptr);
	268	CHECK_LE(header->size, kBlockSizeBytes);
	269	CHECK_GE(header->size, sizeof(PackedHeader));
	270
	271	uptr packed_size_aligned = RoundUpTo(header->size, GetPageSizeCached());
	272
	273	uptr *unpacked =
	274	reinterpret_cast<uptr *>(store->Map(kBlockSizeBytes, "StackStoreUnpack"));
	275
	276	uptr *unpacked_end;
	277	switch (header->type) {
	278	case Compression::Delta:
	279	unpacked_end = UncompressDelta(header->data, ptr + header->size, unpacked,
	280	unpacked + kBlockSizeFrames);
	281	break;
	282	case Compression::LZW:
	283	unpacked_end = UncompressLzw(header->data, ptr + header->size, unpacked,
	284	unpacked + kBlockSizeFrames);
	285	break;
	286	default:
	287	UNREACHABLE("Unexpected type");
	288	break;
	289	}
	290
	291	CHECK_EQ(kBlockSizeFrames, unpacked_end - unpacked);
	292
	293	MprotectReadOnly(reinterpret_cast<uptr>(unpacked), kBlockSizeBytes);
	294	atomic_store(&data_, reinterpret_cast<uptr>(unpacked), memory_order_release);
	295	store->Unmap(ptr, packed_size_aligned);
	296
	297	state = State::Unpacked;
	298	return Get();
	299	}
	300
	301	uptr StackStore::BlockInfo::Pack(Compression type, StackStore *store) {
	302	if (type == Compression::None)
	303	return 0;
	304
	305	SpinMutexLock l(&mtx_);
	306	switch (state) {
	307	case State::Unpacked:
	308	case State::Packed:
	309	return 0;
	310	case State::Storing:
	311	break;
	312	}
	313
	314	uptr *ptr = Get();
	315	if (!ptr \|\| !Stored(0))
	316	return 0;
	317
	318	u8 *packed =
	319	reinterpret_cast<u8 *>(store->Map(kBlockSizeBytes, "StackStorePack"));
	320	PackedHeader header = reinterpret_cast<PackedHeader >(packed);
	321	u8 *alloc_end = packed + kBlockSizeBytes;
	322
	323	u8 *packed_end = nullptr;
	324	switch (type) {
	325	case Compression::Delta:
	326	packed_end =
	327	CompressDelta(ptr, ptr + kBlockSizeFrames, header->data, alloc_end);
	328	break;
	329	case Compression::LZW:
	330	packed_end =
	331	CompressLzw(ptr, ptr + kBlockSizeFrames, header->data, alloc_end);
	332	break;
	333	default:
	334	UNREACHABLE("Unexpected type");
	335	break;
	336	}
	337
	338	header->type = type;
	339	header->size = packed_end - packed;
	340
	341	VPrintf(1, "Packed block of %zu KiB to %zu KiB\n", kBlockSizeBytes >> 10,
	342	header->size >> 10);
	343
	344	if (kBlockSizeBytes - header->size < kBlockSizeBytes / 8) {
	345	VPrintf(1, "Undo and keep block unpacked\n");
	346	MprotectReadOnly(reinterpret_cast<uptr>(ptr), kBlockSizeBytes);
	347	store->Unmap(packed, kBlockSizeBytes);
	348	state = State::Unpacked;
	349	return 0;
	350	}
	351
	352	uptr packed_size_aligned = RoundUpTo(header->size, GetPageSizeCached());
	353	store->Unmap(packed + packed_size_aligned,
	354	kBlockSizeBytes - packed_size_aligned);
	355	MprotectReadOnly(reinterpret_cast<uptr>(packed), packed_size_aligned);
	356
	357	atomic_store(&data_, reinterpret_cast<uptr>(packed), memory_order_release);
	358	store->Unmap(ptr, kBlockSizeBytes);
	359
	360	state = State::Packed;
	361	return kBlockSizeBytes - packed_size_aligned;
	362	}
	363
	364	void StackStore::BlockInfo::TestOnlyUnmap(StackStore *store) {
	365	if (uptr *ptr = Get())
	366	store->Unmap(ptr, kBlockSizeBytes);
	367	}
	368
	369	bool StackStore::BlockInfo::Stored(uptr n) {
	370	return n + atomic_fetch_add(&stored_, n, memory_order_release) ==
	371	kBlockSizeFrames;
	372	}
	373
	374	bool StackStore::BlockInfo::IsPacked() const {
	375	SpinMutexLock l(&mtx_);
	376	return state == State::Packed;
	377	}
	378
	379	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_stack_store.h created+121

...	...	@@ -0,0 +1,121 @@
	1	//===-- sanitizer_stack_store.h ---------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef SANITIZER_STACK_STORE_H
	10	#define SANITIZER_STACK_STORE_H
	11
	12	#include "sanitizer_atomic.h"
	13	#include "sanitizer_common.h"
	14	#include "sanitizer_internal_defs.h"
	15	#include "sanitizer_mutex.h"
	16	#include "sanitizer_stacktrace.h"
	17
	18	namespace __sanitizer {
	19
	20	class StackStore {
	21	static constexpr uptr kBlockSizeFrames = 0x100000;
	22	static constexpr uptr kBlockCount = 0x1000;
	23	static constexpr uptr kBlockSizeBytes = kBlockSizeFrames * sizeof(uptr);
	24
	25	public:
	26	enum class Compression : u8 {
	27	None = 0,
	28	Delta,
	29	LZW,
	30	};
	31
	32	constexpr StackStore() = default;
	33
	34	using Id = u32; // Enough for 2^32 * sizeof(uptr) bytes of traces.
	35	static_assert(u64(kBlockCount) * kBlockSizeFrames == 1ull << (sizeof(Id) * 8),
	36	"");
	37
	38	Id Store(const StackTrace &trace,
	39	uptr pack / number of blocks completed by this call */);
	40	StackTrace Load(Id id);
	41	uptr Allocated() const;
	42
	43	// Packs all blocks which don't expect any more writes. A block is going to be
	44	// packed once. As soon trace from that block was requested, it will unpack
	45	// and stay unpacked after that.
	46	// Returns the number of released bytes.
	47	uptr Pack(Compression type);
	48
	49	void LockAll();
	50	void UnlockAll();
	51
	52	void TestOnlyUnmap();
	53
	54	private:
	55	friend class StackStoreTest;
	56	static constexpr uptr GetBlockIdx(uptr frame_idx) {
	57	return frame_idx / kBlockSizeFrames;
	58	}
	59
	60	static constexpr uptr GetInBlockIdx(uptr frame_idx) {
	61	return frame_idx % kBlockSizeFrames;
	62	}
	63
	64	static constexpr uptr IdToOffset(Id id) {
	65	CHECK_NE(id, 0);
	66	return id - 1; // Avoid zero as id.
	67	}
	68
	69	static constexpr uptr OffsetToId(Id id) {
	70	// This makes UINT32_MAX to 0 and it will be retrived as and empty stack.
	71	// But this is not a problem as we will not be able to store anything after
	72	// that anyway.
	73	return id + 1; // Avoid zero as id.
	74	}
	75
	76	uptr Alloc(uptr count, uptr idx, uptr *pack);
	77
	78	void Map(uptr size, const char mem_type);
	79	void Unmap(void *addr, uptr size);
	80
	81	// Total number of allocated frames.
	82	atomic_uintptr_t total_frames_ = {};
	83
	84	// Tracks total allocated memory in bytes.
	85	atomic_uintptr_t allocated_ = {};
	86
	87	// Each block will hold pointer to exactly kBlockSizeFrames.
	88	class BlockInfo {
	89	atomic_uintptr_t data_;
	90	// Counter to track store progress to know when we can Pack() the block.
	91	atomic_uint32_t stored_;
	92	// Protects alloc of new blocks.
	93	mutable StaticSpinMutex mtx_;
	94
	95	enum class State : u8 {
	96	Storing = 0,
	97	Packed,
	98	Unpacked,
	99	};
	100	State state SANITIZER_GUARDED_BY(mtx_);
	101
	102	uptr Create(StackStore store);
	103
	104	public:
	105	uptr *Get() const;
	106	uptr GetOrCreate(StackStore store);
	107	uptr GetOrUnpack(StackStore store);
	108	uptr Pack(Compression type, StackStore *store);
	109	void TestOnlyUnmap(StackStore *store);
	110	bool Stored(uptr n);
	111	bool IsPacked() const;
	112	void Lock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { mtx_.Lock(); }
	113	void Unlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { mtx_.Unlock(); }
	114	};
	115
	116	BlockInfo blocks_[kBlockCount] = {};
	117	};
	118
	119	} // namespace __sanitizer
	120
	121	#endif // SANITIZER_STACK_STORE_H

lib/tsan/sanitizer_common/sanitizer_stackdepot.cpp+174-81

...	...	@@ -12,95 +12,203 @@
12	12
13	13	#include "sanitizer_stackdepot.h"
14	14
	15	#include "sanitizer_atomic.h"
15	16	#include "sanitizer_common.h"
16	17	#include "sanitizer_hash.h"
	18	#include "sanitizer_mutex.h"
	19	#include "sanitizer_stack_store.h"
17	20	#include "sanitizer_stackdepotbase.h"
18	21
19	22	namespace __sanitizer {
20	23
21	24	struct StackDepotNode {
22		StackDepotNode *link;
23		u32 id;
24		atomic_uint32_t hash_and_use_count; // hash_bits : 12; use_count : 20;
25		u32 size;
26		u32 tag;
27		uptr stack[1]; // [size]
	25	using hash_type = u64;
	26	hash_type stack_hash;
	27	u32 link;
	28	StackStore::Id store_id;
28	29
29	30	static const u32 kTabSizeLog = SANITIZER_ANDROID ? 16 : 20;
30		// Lower kTabSizeLog bits are equal for all items in one bucket.
31		// We use these bits to store the per-stack use counter.
32		static const u32 kUseCountBits = kTabSizeLog;
33		static const u32 kMaxUseCount = 1 << kUseCountBits;
34		static const u32 kUseCountMask = (1 << kUseCountBits) - 1;
35		static const u32 kHashMask = ~kUseCountMask;
36	31
37	32	typedef StackTrace args_type;
38		bool eq(u32 hash, const args_type &args) const {
39		u32 hash_bits =
40		atomic_load(&hash_and_use_count, memory_order_relaxed) & kHashMask;
41		if ((hash & kHashMask) != hash_bits \|\| args.size != size \|\| args.tag != tag)
42		return false;
43		uptr i = 0;
44		for (; i < size; i++) {
45		if (stack[i] != args.trace[i]) return false;
46		}
47		return true;
48		}
49		static uptr storage_size(const args_type &args) {
50		return sizeof(StackDepotNode) + (args.size - 1) * sizeof(uptr);
	33	bool eq(hash_type hash, const args_type &args) const {
	34	return hash == stack_hash;
51	35	}
52		static u32 hash(const args_type &args) {
53		MurMur2HashBuilder H(args.size * sizeof(uptr));
	36	static uptr allocated();
	37	static hash_type hash(const args_type &args) {
	38	MurMur2Hash64Builder H(args.size * sizeof(uptr));
54	39	for (uptr i = 0; i < args.size; i++) H.add(args.trace[i]);
	40	H.add(args.tag);
55	41	return H.get();
56	42	}
57	43	static bool is_valid(const args_type &args) {
58	44	return args.size > 0 && args.trace;
59	45	}
60		void store(const args_type &args, u32 hash) {
61		atomic_store(&hash_and_use_count, hash & kHashMask, memory_order_relaxed);
62		size = args.size;
63		tag = args.tag;
64		internal_memcpy(stack, args.trace, size * sizeof(uptr));
65		}
66		args_type load() const {
67		return args_type(&stack[0], size, tag);
68		}
69		StackDepotHandle get_handle() { return StackDepotHandle(this); }
	46	void store(u32 id, const args_type &args, hash_type hash);
	47	args_type load(u32 id) const;
	48	static StackDepotHandle get_handle(u32 id);
70	49
71	50	typedef StackDepotHandle handle_type;
72	51	};
73	52
74		COMPILER_CHECK(StackDepotNode::kMaxUseCount == (u32)kStackDepotMaxUseCount);
75
76		u32 StackDepotHandle::id() { return node_->id; }
77		int StackDepotHandle::use_count() {
78		return atomic_load(&node_->hash_and_use_count, memory_order_relaxed) &
79		StackDepotNode::kUseCountMask;
80		}
81		void StackDepotHandle::inc_use_count_unsafe() {
82		u32 prev =
83		atomic_fetch_add(&node_->hash_and_use_count, 1, memory_order_relaxed) &
84		StackDepotNode::kUseCountMask;
85		CHECK_LT(prev + 1, StackDepotNode::kMaxUseCount);
86		}
	53	static StackStore stackStore;
87	54
88	55	// FIXME(dvyukov): this single reserved bit is used in TSan.
89	56	typedef StackDepotBase<StackDepotNode, 1, StackDepotNode::kTabSizeLog>
90	57	StackDepot;
91	58	static StackDepot theDepot;
	59	// Keep mutable data out of frequently access nodes to improve caching
	60	// efficiency.
	61	static TwoLevelMap<atomic_uint32_t, StackDepot::kNodesSize1,
	62	StackDepot::kNodesSize2>
	63	useCounts;
	64
	65	int StackDepotHandle::use_count() const {
	66	return atomic_load_relaxed(&useCounts[id_]);
	67	}
	68
	69	void StackDepotHandle::inc_use_count_unsafe() {
	70	atomic_fetch_add(&useCounts[id_], 1, memory_order_relaxed);
	71	}
	72
	73	uptr StackDepotNode::allocated() {
	74	return stackStore.Allocated() + useCounts.MemoryUsage();
	75	}
	76
	77	static void CompressStackStore() {
	78	u64 start = Verbosity() >= 1 ? MonotonicNanoTime() : 0;
	79	uptr diff = stackStore.Pack(static_cast<StackStore::Compression>(
	80	Abs(common_flags()->compress_stack_depot)));
	81	if (!diff)
	82	return;
	83	if (Verbosity() >= 1) {
	84	u64 finish = MonotonicNanoTime();
	85	uptr total_before = theDepot.GetStats().allocated + diff;
	86	VPrintf(1, "%s: StackDepot released %zu KiB out of %zu KiB in %llu ms\n",
	87	SanitizerToolName, diff >> 10, total_before >> 10,
	88	(finish - start) / 1000000);
	89	}
	90	}
	91
	92	namespace {
	93
	94	class CompressThread {
	95	public:
	96	constexpr CompressThread() = default;
	97	void NewWorkNotify();
	98	void Stop();
	99	void LockAndStop() SANITIZER_NO_THREAD_SAFETY_ANALYSIS;
	100	void Unlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS;
	101
	102	private:
	103	enum class State {
	104	NotStarted = 0,
	105	Started,
	106	Failed,
	107	Stopped,
	108	};
	109
	110	void Run();
	111
	112	bool WaitForWork() {
	113	semaphore_.Wait();
	114	return atomic_load(&run_, memory_order_acquire);
	115	}
	116
	117	Semaphore semaphore_ = {};
	118	StaticSpinMutex mutex_ = {};
	119	State state_ SANITIZER_GUARDED_BY(mutex_) = State::NotStarted;
	120	void *thread_ SANITIZER_GUARDED_BY(mutex_) = nullptr;
	121	atomic_uint8_t run_ = {};
	122	};
	123
	124	static CompressThread compress_thread;
	125
	126	void CompressThread::NewWorkNotify() {
	127	int compress = common_flags()->compress_stack_depot;
	128	if (!compress)
	129	return;
	130	if (compress > 0 /* for testing or debugging */) {
	131	SpinMutexLock l(&mutex_);
	132	if (state_ == State::NotStarted) {
	133	atomic_store(&run_, 1, memory_order_release);
	134	CHECK_EQ(nullptr, thread_);
	135	thread_ = internal_start_thread(
	136	[](void arg) -> void {
	137	reinterpret_cast<CompressThread *>(arg)->Run();
	138	return nullptr;
	139	},
	140	this);
	141	state_ = thread_ ? State::Started : State::Failed;
	142	}
	143	if (state_ == State::Started) {
	144	semaphore_.Post();
	145	return;
	146	}
	147	}
	148	CompressStackStore();
	149	}
	150
	151	void CompressThread::Run() {
	152	VPrintf(1, "%s: StackDepot compression thread started\n", SanitizerToolName);
	153	while (WaitForWork()) CompressStackStore();
	154	VPrintf(1, "%s: StackDepot compression thread stopped\n", SanitizerToolName);
	155	}
	156
	157	void CompressThread::Stop() {
	158	void *t = nullptr;
	159	{
	160	SpinMutexLock l(&mutex_);
	161	if (state_ != State::Started)
	162	return;
	163	state_ = State::Stopped;
	164	CHECK_NE(nullptr, thread_);
	165	t = thread_;
	166	thread_ = nullptr;
	167	}
	168	atomic_store(&run_, 0, memory_order_release);
	169	semaphore_.Post();
	170	internal_join_thread(t);
	171	}
	172
	173	void CompressThread::LockAndStop() {
	174	mutex_.Lock();
	175	if (state_ != State::Started)
	176	return;
	177	CHECK_NE(nullptr, thread_);
	178
	179	atomic_store(&run_, 0, memory_order_release);
	180	semaphore_.Post();
	181	internal_join_thread(thread_);
	182	// Allow to restart after Unlock() if needed.
	183	state_ = State::NotStarted;
	184	thread_ = nullptr;
	185	}
	186
	187	void CompressThread::Unlock() { mutex_.Unlock(); }
	188
	189	} // namespace
92	190
93		StackDepotStats *StackDepotGetStats() {
94		return theDepot.GetStats();
	191	void StackDepotNode::store(u32 id, const args_type &args, hash_type hash) {
	192	stack_hash = hash;
	193	uptr pack = 0;
	194	store_id = stackStore.Store(args, &pack);
	195	if (LIKELY(!pack))
	196	return;
	197	compress_thread.NewWorkNotify();
95	198	}
96	199
97		u32 StackDepotPut(StackTrace stack) {
98		StackDepotHandle h = theDepot.Put(stack);
99		return h.valid() ? h.id() : 0;
	200	StackDepotNode::args_type StackDepotNode::load(u32 id) const {
	201	if (!store_id)
	202	return {};
	203	return stackStore.Load(store_id);
100	204	}
101	205
	206	StackDepotStats StackDepotGetStats() { return theDepot.GetStats(); }
	207
	208	u32 StackDepotPut(StackTrace stack) { return theDepot.Put(stack); }
	209
102	210	StackDepotHandle StackDepotPut_WithHandle(StackTrace stack) {
103		return theDepot.Put(stack);
	211	return StackDepotNode::get_handle(theDepot.Put(stack));
104	212	}
105	213
106	214	StackTrace StackDepotGet(u32 id) {
...	...	@@ -109,9 +217,13 @@ StackTrace StackDepotGet(u32 id) {
109	217
110	218	void StackDepotLockAll() {
111	219	theDepot.LockAll();
	220	compress_thread.LockAndStop();
	221	stackStore.LockAll();
112	222	}
113	223
114	224	void StackDepotUnlockAll() {
	225	stackStore.UnlockAll();
	226	compress_thread.Unlock();
115	227	theDepot.UnlockAll();
116	228	}
117	229
...	...	@@ -121,34 +233,15 @@ void StackDepotPrintAll() {
121	233	#endif
122	234	}
123	235
124		bool StackDepotReverseMap::IdDescPair::IdComparator(
125		const StackDepotReverseMap::IdDescPair &a,
126		const StackDepotReverseMap::IdDescPair &b) {
127		return a.id < b.id;
128		}
	236	void StackDepotStopBackgroundThread() { compress_thread.Stop(); }
129	237
130		StackDepotReverseMap::StackDepotReverseMap() {
131		map_.reserve(StackDepotGetStats()->n_uniq_ids + 100);
132		for (int idx = 0; idx < StackDepot::kTabSize; idx++) {
133		atomic_uintptr_t *p = &theDepot.tab[idx];
134		uptr v = atomic_load(p, memory_order_consume);
135		StackDepotNode s = (StackDepotNode)(v & ~1);
136		for (; s; s = s->link) {
137		IdDescPair pair = {s->id, s};
138		map_.push_back(pair);
139		}
140		}
141		Sort(map_.data(), map_.size(), &IdDescPair::IdComparator);
	238	StackDepotHandle StackDepotNode::get_handle(u32 id) {
	239	return StackDepotHandle(&theDepot.nodes[id], id);
142	240	}
143	241
144		StackTrace StackDepotReverseMap::Get(u32 id) {
145		if (!map_.size())
146		return StackTrace();
147		IdDescPair pair = {id, nullptr};
148		uptr idx = InternalLowerBound(map_, pair, IdDescPair::IdComparator);
149		if (idx > map_.size() \|\| map_[idx].id != id)
150		return StackTrace();
151		return map_[idx].desc->load();
	242	void StackDepotTestOnlyUnmap() {
	243	theDepot.TestOnlyUnmap();
	244	stackStore.TestOnlyUnmap();
152	245	}
153	246
154	247	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_stackdepot.h+9-30

...	...	@@ -22,18 +22,18 @@ namespace __sanitizer {
22	22	// StackDepot efficiently stores huge amounts of stack traces.
23	23	struct StackDepotNode;
24	24	struct StackDepotHandle {
25		StackDepotNode *node_;
26		StackDepotHandle() : node_(nullptr) {}
27		explicit StackDepotHandle(StackDepotNode *node) : node_(node) {}
28		bool valid() { return node_; }
29		u32 id();
30		int use_count();
	25	StackDepotNode *node_ = nullptr;
	26	u32 id_ = 0;
	27	StackDepotHandle(StackDepotNode *node, u32 id) : node_(node), id_(id) {}
	28	bool valid() const { return node_; }
	29	u32 id() const { return id_; }
	30	int use_count() const;
31	31	void inc_use_count_unsafe();
32	32	};
33	33
34	34	const int kStackDepotMaxUseCount = 1U << (SANITIZER_ANDROID ? 16 : 20);
35	35
36		StackDepotStats *StackDepotGetStats();
	36	StackDepotStats StackDepotGetStats();
37	37	u32 StackDepotPut(StackTrace stack);
38	38	StackDepotHandle StackDepotPut_WithHandle(StackTrace stack);
39	39	// Retrieves a stored stack trace by the id.
...	...	@@ -42,30 +42,9 @@ StackTrace StackDepotGet(u32 id);
42	42	void StackDepotLockAll();
43	43	void StackDepotUnlockAll();
44	44	void StackDepotPrintAll();
	45	void StackDepotStopBackgroundThread();
45	46
46		// Instantiating this class creates a snapshot of StackDepot which can be
47		// efficiently queried with StackDepotGet(). You can use it concurrently with
48		// StackDepot, but the snapshot is only guaranteed to contain those stack traces
49		// which were stored before it was instantiated.
50		class StackDepotReverseMap {
51		public:
52		StackDepotReverseMap();
53		StackTrace Get(u32 id);
54
55		private:
56		struct IdDescPair {
57		u32 id;
58		StackDepotNode *desc;
59
60		static bool IdComparator(const IdDescPair &a, const IdDescPair &b);
61		};
62
63		InternalMmapVector<IdDescPair> map_;
64
65		// Disallow evil constructors.
66		StackDepotReverseMap(const StackDepotReverseMap&);
67		void operator=(const StackDepotReverseMap&);
68		};
	47	void StackDepotTestOnlyUnmap();
69	48
70	49	} // namespace __sanitizer
71	50

lib/tsan/sanitizer_common/sanitizer_stackdepotbase.h+86-87

...	...	@@ -16,71 +16,87 @@
16	16	#include <stdio.h>
17	17
18	18	#include "sanitizer_atomic.h"
	19	#include "sanitizer_flat_map.h"
19	20	#include "sanitizer_internal_defs.h"
20	21	#include "sanitizer_mutex.h"
21		#include "sanitizer_persistent_allocator.h"
22	22
23	23	namespace __sanitizer {
24	24
25	25	template <class Node, int kReservedBits, int kTabSizeLog>
26	26	class StackDepotBase {
	27	static constexpr u32 kIdSizeLog =
	28	sizeof(u32) * 8 - Max(kReservedBits, 1 /* At least 1 bit for locking. */);
	29	static constexpr u32 kNodesSize1Log = kIdSizeLog / 2;
	30	static constexpr u32 kNodesSize2Log = kIdSizeLog - kNodesSize1Log;
	31	static constexpr int kTabSize = 1 << kTabSizeLog; // Hash table size.
	32	static constexpr u32 kUnlockMask = (1ull << kIdSizeLog) - 1;
	33	static constexpr u32 kLockMask = ~kUnlockMask;
	34
27	35	public:
28	36	typedef typename Node::args_type args_type;
29	37	typedef typename Node::handle_type handle_type;
	38	typedef typename Node::hash_type hash_type;
	39
	40	static constexpr u64 kNodesSize1 = 1ull << kNodesSize1Log;
	41	static constexpr u64 kNodesSize2 = 1ull << kNodesSize2Log;
	42
30	43	// Maps stack trace to an unique id.
31		handle_type Put(args_type args, bool *inserted = nullptr);
	44	u32 Put(args_type args, bool *inserted = nullptr);
32	45	// Retrieves a stored stack trace by the id.
33	46	args_type Get(u32 id);
34	47
35		StackDepotStats *GetStats() { return &stats; }
	48	StackDepotStats GetStats() const {
	49	return {
	50	atomic_load_relaxed(&n_uniq_ids),
	51	nodes.MemoryUsage() + Node::allocated(),
	52	};
	53	}
36	54
37	55	void LockAll();
38	56	void UnlockAll();
39	57	void PrintAll();
40	58
41		private:
42		static Node find(Node s, args_type args, u32 hash);
43		static Node lock(atomic_uintptr_t p);
44		static void unlock(atomic_uintptr_t p, Node s);
	59	void TestOnlyUnmap() {
	60	nodes.TestOnlyUnmap();
	61	internal_memset(this, 0, sizeof(*this));
	62	}
45	63
46		static const int kTabSize = 1 << kTabSizeLog; // Hash table size.
47		static const int kPartBits = 8;
48		static const int kPartShift = sizeof(u32) * 8 - kPartBits - kReservedBits;
49		static const int kPartCount =
50		1 << kPartBits; // Number of subparts in the table.
51		static const int kPartSize = kTabSize / kPartCount;
52		static const int kMaxId = 1 << kPartShift;
	64	private:
	65	friend Node;
	66	u32 find(u32 s, args_type args, hash_type hash) const;
	67	static u32 lock(atomic_uint32_t *p);
	68	static void unlock(atomic_uint32_t *p, u32 s);
	69	atomic_uint32_t tab[kTabSize]; // Hash table of Node's.
53	70
54		atomic_uintptr_t tab[kTabSize]; // Hash table of Node's.
55		atomic_uint32_t seq[kPartCount]; // Unique id generators.
	71	atomic_uint32_t n_uniq_ids;
56	72
57		StackDepotStats stats;
	73	TwoLevelMap<Node, kNodesSize1, kNodesSize2> nodes;
58	74
59	75	friend class StackDepotReverseMap;
60	76	};
61	77
62	78	template <class Node, int kReservedBits, int kTabSizeLog>
63		Node StackDepotBase<Node, kReservedBits, kTabSizeLog>::find(Node s,
64		args_type args,
65		u32 hash) {
	79	u32 StackDepotBase<Node, kReservedBits, kTabSizeLog>::find(
	80	u32 s, args_type args, hash_type hash) const {
66	81	// Searches linked list s for the stack, returns its id.
67		for (; s; s = s->link) {
68		if (s->eq(hash, args)) {
	82	for (; s;) {
	83	const Node &node = nodes[s];
	84	if (node.eq(hash, args))
69	85	return s;
70		}
	86	s = node.link;
71	87	}
72		return nullptr;
	88	return 0;
73	89	}
74	90
75	91	template <class Node, int kReservedBits, int kTabSizeLog>
76		Node *StackDepotBase<Node, kReservedBits, kTabSizeLog>::lock(
77		atomic_uintptr_t *p) {
	92	u32 StackDepotBase<Node, kReservedBits, kTabSizeLog>::lock(atomic_uint32_t *p) {
78	93	// Uses the pointer lsb as mutex.
79	94	for (int i = 0;; i++) {
80		uptr cmp = atomic_load(p, memory_order_relaxed);
81		if ((cmp & 1) == 0 &&
82		atomic_compare_exchange_weak(p, &cmp, cmp \| 1, memory_order_acquire))
83		return (Node *)cmp;
	95	u32 cmp = atomic_load(p, memory_order_relaxed);
	96	if ((cmp & kLockMask) == 0 &&
	97	atomic_compare_exchange_weak(p, &cmp, cmp \| kLockMask,
	98	memory_order_acquire))
	99	return cmp;
84	100	if (i < 10)
85	101	proc_yield(10);
86	102	else
...	...	@@ -90,73 +106,57 @@ Node *StackDepotBase<Node, kReservedBits, kTabSizeLog>::lock(
90	106
91	107	template <class Node, int kReservedBits, int kTabSizeLog>
92	108	void StackDepotBase<Node, kReservedBits, kTabSizeLog>::unlock(
93		atomic_uintptr_t p, Node s) {
94		DCHECK_EQ((uptr)s & 1, 0);
95		atomic_store(p, (uptr)s, memory_order_release);
	109	atomic_uint32_t *p, u32 s) {
	110	DCHECK_EQ(s & kLockMask, 0);
	111	atomic_store(p, s, memory_order_release);
96	112	}
97	113
98	114	template <class Node, int kReservedBits, int kTabSizeLog>
99		typename StackDepotBase<Node, kReservedBits, kTabSizeLog>::handle_type
100		StackDepotBase<Node, kReservedBits, kTabSizeLog>::Put(args_type args,
101		bool *inserted) {
102		if (inserted) *inserted = false;
103		if (!Node::is_valid(args)) return handle_type();
104		uptr h = Node::hash(args);
105		atomic_uintptr_t *p = &tab[h % kTabSize];
106		uptr v = atomic_load(p, memory_order_consume);
107		Node s = (Node )(v & ~1);
	115	u32 StackDepotBase<Node, kReservedBits, kTabSizeLog>::Put(args_type args,
	116	bool *inserted) {
	117	if (inserted)
	118	*inserted = false;
	119	if (!LIKELY(Node::is_valid(args)))
	120	return 0;
	121	hash_type h = Node::hash(args);
	122	atomic_uint32_t *p = &tab[h % kTabSize];
	123	u32 v = atomic_load(p, memory_order_consume);
	124	u32 s = v & kUnlockMask;
108	125	// First, try to find the existing stack.
109		Node *node = find(s, args, h);
110		if (node) return node->get_handle();
	126	u32 node = find(s, args, h);
	127	if (LIKELY(node))
	128	return node;
	129
111	130	// If failed, lock, retry and insert new.
112		Node *s2 = lock(p);
	131	u32 s2 = lock(p);
113	132	if (s2 != s) {
114	133	node = find(s2, args, h);
115	134	if (node) {
116	135	unlock(p, s2);
117		return node->get_handle();
	136	return node;
118	137	}
119	138	}
120		uptr part = (h % kTabSize) / kPartSize;
121		u32 id = atomic_fetch_add(&seq[part], 1, memory_order_relaxed) + 1;
122		stats.n_uniq_ids++;
123		CHECK_LT(id, kMaxId);
124		id \|= part << kPartShift;
125		CHECK_NE(id, 0);
126		CHECK_EQ(id & (((u32)-1) >> kReservedBits), id);
127		uptr memsz = Node::storage_size(args);
128		s = (Node *)PersistentAlloc(memsz);
129		stats.allocated += memsz;
130		s->id = id;
131		s->store(args, h);
132		s->link = s2;
	139	s = atomic_fetch_add(&n_uniq_ids, 1, memory_order_relaxed) + 1;
	140	CHECK_EQ(s & kUnlockMask, s);
	141	CHECK_EQ(s & (((u32)-1) >> kReservedBits), s);
	142	Node &new_node = nodes[s];
	143	new_node.store(s, args, h);
	144	new_node.link = s2;
133	145	unlock(p, s);
134	146	if (inserted) *inserted = true;
135		return s->get_handle();
	147	return s;
136	148	}
137	149
138	150	template <class Node, int kReservedBits, int kTabSizeLog>
139	151	typename StackDepotBase<Node, kReservedBits, kTabSizeLog>::args_type
140	152	StackDepotBase<Node, kReservedBits, kTabSizeLog>::Get(u32 id) {
141		if (id == 0) {
	153	if (id == 0)
142	154	return args_type();
143		}
144	155	CHECK_EQ(id & (((u32)-1) >> kReservedBits), id);
145		// High kPartBits contain part id, so we need to scan at most kPartSize lists.
146		uptr part = id >> kPartShift;
147		for (int i = 0; i != kPartSize; i++) {
148		uptr idx = part * kPartSize + i;
149		CHECK_LT(idx, kTabSize);
150		atomic_uintptr_t *p = &tab[idx];
151		uptr v = atomic_load(p, memory_order_consume);
152		Node s = (Node )(v & ~1);
153		for (; s; s = s->link) {
154		if (s->id == id) {
155		return s->load();
156		}
157		}
158		}
159		return args_type();
	156	if (!nodes.contains(id))
	157	return args_type();
	158	const Node &node = nodes[id];
	159	return node.load(id);
160	160	}
161	161
162	162	template <class Node, int kReservedBits, int kTabSizeLog>
...	...	@@ -169,24 +169,23 @@ void StackDepotBase<Node, kReservedBits, kTabSizeLog>::LockAll() {
169	169	template <class Node, int kReservedBits, int kTabSizeLog>
170	170	void StackDepotBase<Node, kReservedBits, kTabSizeLog>::UnlockAll() {
171	171	for (int i = 0; i < kTabSize; ++i) {
172		atomic_uintptr_t *p = &tab[i];
	172	atomic_uint32_t *p = &tab[i];
173	173	uptr s = atomic_load(p, memory_order_relaxed);
174		unlock(p, (Node *)(s & ~1UL));
	174	unlock(p, s & kUnlockMask);
175	175	}
176	176	}
177	177
178	178	template <class Node, int kReservedBits, int kTabSizeLog>
179	179	void StackDepotBase<Node, kReservedBits, kTabSizeLog>::PrintAll() {
180	180	for (int i = 0; i < kTabSize; ++i) {
181		atomic_uintptr_t *p = &tab[i];
182		lock(p);
183		uptr v = atomic_load(p, memory_order_relaxed);
184		Node s = (Node )(v & ~1UL);
185		for (; s; s = s->link) {
186		Printf("Stack for id %u:\n", s->id);
187		s->load().Print();
	181	atomic_uint32_t *p = &tab[i];
	182	u32 s = atomic_load(p, memory_order_consume) & kUnlockMask;
	183	for (; s;) {
	184	const Node &node = nodes[s];
	185	Printf("Stack for id %u:\n", s);
	186	node.load(s).Print();
	187	s = node.link;
188	188	}
189		unlock(p, s);
190	189	}
191	190	}
192	191

lib/tsan/sanitizer_common/sanitizer_stacktrace.cpp+9-7

...	...	@@ -20,10 +20,10 @@
20	20	namespace __sanitizer {
21	21
22	22	uptr StackTrace::GetNextInstructionPc(uptr pc) {
23		#if defined(__sparc__) \|\| defined(__mips__)
	23	#if defined(__aarch64__)
	24	return STRIP_PAC_PC((void *)pc) + 4;
	25	#elif defined(__sparc__) \|\| defined(__mips__)
24	26	return pc + 8;
25		#elif defined(__powerpc__) \|\| defined(__arm__) \|\| defined(__aarch64__)
26		return pc + 4;
27	27	#elif SANITIZER_RISCV64
28	28	// Current check order is 4 -> 2 -> 6 -> 8
29	29	u8 InsnByte = (u8 )(pc);
...	...	@@ -46,8 +46,10 @@ uptr StackTrace::GetNextInstructionPc(uptr pc) {
46	46	}
47	47	// bail-out if could not figure out the instruction size
48	48	return 0;
49		#else
	49	#elif SANITIZER_S390 \|\| SANITIZER_I386 \|\| SANITIZER_X32 \|\| SANITIZER_X64
50	50	return pc + 1;
	51	#else
	52	return pc + 4;
51	53	#endif
52	54	}
53	55
...	...	@@ -64,7 +66,7 @@ void BufferedStackTrace::Init(const uptr *pcs, uptr cnt, uptr extra_top_pc) {
64	66	top_frame_bp = 0;
65	67	}
66	68
67		// Sparc implemention is in its own file.
	69	// Sparc implementation is in its own file.
68	70	#if !defined(__sparc__)
69	71
70	72	// In GCC on ARM bp points to saved lr, not fp, so we should check the next
...	...	@@ -119,7 +121,7 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
119	121	uhwptr pc1 = caller_frame[2];
120	122	#elif defined(__s390__)
121	123	uhwptr pc1 = frame[14];
122		#elif defined(__riscv)
	124	#elif defined(__loongarch__) \|\| defined(__riscv)
123	125	// frame[-1] contains the return address
124	126	uhwptr pc1 = frame[-1];
125	127	#else
...	...	@@ -134,7 +136,7 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
134	136	trace_buffer[size++] = (uptr) pc1;
135	137	}
136	138	bottom = (uptr)frame;
137		#if defined(__riscv)
	139	#if defined(__loongarch__) \|\| defined(__riscv)
138	140	// frame[-2] contain fp of the previous frame
139	141	uptr new_bp = (uptr)frame[-2];
140	142	#else

lib/tsan/sanitizer_common/sanitizer_stacktrace.h+9-10

...	...	@@ -20,7 +20,7 @@ namespace __sanitizer {
20	20
21	21	struct BufferedStackTrace;
22	22
23		static const u32 kStackTraceMax = 256;
	23	static const u32 kStackTraceMax = 255;
24	24
25	25	#if SANITIZER_LINUX && defined(__mips__)
26	26	# define SANITIZER_CAN_FAST_UNWIND 0
...	...	@@ -33,7 +33,7 @@ static const u32 kStackTraceMax = 256;
33	33	// Fast unwind is the only option on Mac for now; we will need to
34	34	// revisit this macro when slow unwind works on Mac, see
35	35	// https://github.com/google/sanitizers/issues/137
36		#if SANITIZER_MAC
	36	#if SANITIZER_APPLE
37	37	# define SANITIZER_CAN_SLOW_UNWIND 0
38	38	#else
39	39	# define SANITIZER_CAN_SLOW_UNWIND 1
...	...	@@ -88,21 +88,20 @@ uptr StackTrace::GetPreviousInstructionPc(uptr pc) {
88	88	// so we return (pc-2) in that case in order to be safe.
89	89	// For A32 mode we return (pc-4) because all instructions are 32 bit long.
90	90	return (pc - 3) & (~1);
91		#elif defined(__powerpc__) \|\| defined(__powerpc64__) \|\| defined(__aarch64__)
92		// PCs are always 4 byte aligned.
93		return pc - 4;
94	91	#elif defined(__sparc__) \|\| defined(__mips__)
95	92	return pc - 8;
96	93	#elif SANITIZER_RISCV64
97		// RV-64 has variable instruciton length...
	94	// RV-64 has variable instruction length...
98	95	// C extentions gives us 2-byte instructoins
99	96	// RV-64 has 4-byte instructions
100		// + RISCV architecture allows instructions up to 8 bytes
	97	// + RISC-V architecture allows instructions up to 8 bytes
101	98	// It seems difficult to figure out the exact instruction length -
102	99	// pc - 2 seems like a safe option for the purposes of stack tracing
103	100	return pc - 2;
104		#else
	101	#elif SANITIZER_S390 \|\| SANITIZER_I386 \|\| SANITIZER_X32 \|\| SANITIZER_X64
105	102	return pc - 1;
	103	#else
	104	return pc - 4;
106	105	#endif
107	106	}
108	107
...	...	@@ -209,11 +208,11 @@ static inline bool IsValidFrame(uptr frame, uptr stack_top, uptr stack_bottom) {
209	208	// StackTrace::GetCurrentPc() faster.
210	209	#if defined(__x86_64__)
211	210	# define GET_CURRENT_PC() \
212		({ \
	211	(__extension__({ \
213	212	uptr pc; \
214	213	asm("lea 0(%%rip), %0" : "=r"(pc)); \
215	214	pc; \
216		})
	215	}))
217	216	#else
218	217	# define GET_CURRENT_PC() StackTrace::GetCurrentPc()
219	218	#endif

lib/tsan/sanitizer_common/sanitizer_stacktrace_libcdep.cpp+8-7

...	...	@@ -64,7 +64,7 @@ class StackTraceTextPrinter {
64	64	if (dedup_token_->length())
65	65	dedup_token_->append("--");
66	66	if (stack->info.function != nullptr)
67		dedup_token_->append(stack->info.function);
	67	dedup_token_->append("%s", stack->info.function);
68	68	}
69	69	}
70	70
...	...	@@ -166,8 +166,8 @@ void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context,
166	166	UnwindFast(pc, bp, stack_top, stack_bottom, max_depth);
167	167	}
168	168
169		static int GetModuleAndOffsetForPc(uptr pc, char *module_name,
170		uptr module_name_len, uptr *pc_offset) {
	169	int GetModuleAndOffsetForPc(uptr pc, char *module_name, uptr module_name_len,
	170	uptr *pc_offset) {
171	171	const char *found_module_name = nullptr;
172	172	bool ok = Symbolizer::GetOrInit()->GetModuleNameAndOffsetForPC(
173	173	pc, &found_module_name, pc_offset);
...	...	@@ -216,10 +216,11 @@ void __sanitizer_symbolize_global(uptr data_addr, const char *fmt,
216	216	}
217	217
218	218	SANITIZER_INTERFACE_ATTRIBUTE
219		int __sanitizer_get_module_and_offset_for_pc(uptr pc, char *module_name,
	219	int __sanitizer_get_module_and_offset_for_pc(void pc, char module_name,
220	220	uptr module_name_len,
221		uptr *pc_offset) {
222		return __sanitizer::GetModuleAndOffsetForPc(pc, module_name, module_name_len,
223		pc_offset);
	221	void **pc_offset) {
	222	return __sanitizer::GetModuleAndOffsetForPc(
	223	reinterpret_cast<uptr>(pc), module_name, module_name_len,
	224	reinterpret_cast<uptr *>(pc_offset));
224	225	}
225	226	} // extern "C"

lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.cpp+66-21

...	...	@@ -11,25 +11,47 @@
11	11	//===----------------------------------------------------------------------===//
12	12
13	13	#include "sanitizer_stacktrace_printer.h"
	14
14	15	#include "sanitizer_file.h"
	16	#include "sanitizer_flags.h"
15	17	#include "sanitizer_fuchsia.h"
16	18
17	19	namespace __sanitizer {
18	20
19		// sanitizer_symbolizer_markup.cpp implements these differently.
20		#if !SANITIZER_SYMBOLIZER_MARKUP
21
22		static const char StripFunctionName(const char function, const char *prefix) {
23		if (!function) return nullptr;
24		if (!prefix) return function;
25		uptr prefix_len = internal_strlen(prefix);
26		if (0 == internal_strncmp(function, prefix, prefix_len))
27		return function + prefix_len;
	21	const char StripFunctionName(const char function) {
	22	if (!common_flags()->demangle)
	23	return function;
	24	if (!function)
	25	return nullptr;
	26	auto try_strip = [function](const char prefix) -> const char {
	27	const uptr prefix_len = internal_strlen(prefix);
	28	if (!internal_strncmp(function, prefix, prefix_len))
	29	return function + prefix_len;
	30	return nullptr;
	31	};
	32	if (SANITIZER_APPLE) {
	33	if (const char *s = try_strip("wrap_"))
	34	return s;
	35	} else if (SANITIZER_WINDOWS) {
	36	if (const char *s = try_strip("__asan_wrap_"))
	37	return s;
	38	} else {
	39	if (const char *s = try_strip("___interceptor_"))
	40	return s;
	41	if (const char *s = try_strip("__interceptor_"))
	42	return s;
	43	}
28	44	return function;
29	45	}
30	46
	47	// sanitizer_symbolizer_markup.cpp implements these differently.
	48	#if !SANITIZER_SYMBOLIZER_MARKUP
	49
31	50	static const char DemangleFunctionName(const char function) {
32		if (!function) return nullptr;
	51	if (!common_flags()->demangle)
	52	return function;
	53	if (!function)
	54	return nullptr;
33	55
34	56	// NetBSD uses indirection for old threading functions for historical reasons
35	57	// The mangled names are internal implementation detail and should not be
...	...	@@ -104,11 +126,24 @@ static const char DemangleFunctionName(const char function) {
104	126	return function;
105	127	}
106	128
	129	static void MaybeBuildIdToBuffer(const AddressInfo &info, bool PrefixSpace,
	130	InternalScopedString *buffer) {
	131	if (info.uuid_size) {
	132	if (PrefixSpace)
	133	buffer->append(" ");
	134	buffer->append("(BuildId: ");
	135	for (uptr i = 0; i < info.uuid_size; ++i) {
	136	buffer->append("%02x", info.uuid[i]);
	137	}
	138	buffer->append(")");
	139	}
	140	}
	141
107	142	static const char kDefaultFormat[] = " #%n %p %F %L";
108	143
109	144	void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
110	145	uptr address, const AddressInfo *info, bool vs_style,
111		const char strip_path_prefix, const char strip_func_prefix) {
	146	const char *strip_path_prefix) {
112	147	// info will be null in the case where symbolization is not needed for the
113	148	// given format. This ensures that the code below will get a hard failure
114	149	// rather than print incorrect information in case RenderNeedsSymbolization
...	...	@@ -129,7 +164,7 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
129	164	break;
130	165	// Frame number and all fields of AddressInfo structure.
131	166	case 'n':
132		buffer->append("%zu", frame_no);
	167	buffer->append("%u", frame_no);
133	168	break;
134	169	case 'p':
135	170	buffer->append("0x%zx", address);
...	...	@@ -140,9 +175,12 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
140	175	case 'o':
141	176	buffer->append("0x%zx", info->module_offset);
142	177	break;
	178	case 'b':
	179	MaybeBuildIdToBuffer(info, /PrefixSpace=*/false, buffer);
	180	break;
143	181	case 'f':
144		buffer->append("%s", DemangleFunctionName(StripFunctionName(
145		info->function, strip_func_prefix)));
	182	buffer->append("%s",
	183	DemangleFunctionName(StripFunctionName(info->function)));
146	184	break;
147	185	case 'q':
148	186	buffer->append("0x%zx", info->function_offset != AddressInfo::kUnknown
...	...	@@ -162,8 +200,8 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
162	200	case 'F':
163	201	// Function name and offset, if file is unknown.
164	202	if (info->function) {
165		buffer->append("in %s", DemangleFunctionName(StripFunctionName(
166		info->function, strip_func_prefix)));
	203	buffer->append("in %s",
	204	DemangleFunctionName(StripFunctionName(info->function)));
167	205	if (!info->file && info->function_offset != AddressInfo::kUnknown)
168	206	buffer->append("+0x%zx", info->function_offset);
169	207	}
...	...	@@ -181,6 +219,10 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
181	219	} else if (info->module) {
182	220	RenderModuleLocation(buffer, info->module, info->module_offset,
183	221	info->module_arch, strip_path_prefix);
	222
	223	#if !SANITIZER_APPLE
	224	MaybeBuildIdToBuffer(info, /PrefixSpace=*/true, buffer);
	225	#endif
184	226	} else {
185	227	buffer->append("(<unknown module>)");
186	228	}
...	...	@@ -193,13 +235,16 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
193	235	// Always strip the module name for %M.
194	236	RenderModuleLocation(buffer, StripModuleName(info->module),
195	237	info->module_offset, info->module_arch, "");
	238	#if !SANITIZER_APPLE
	239	MaybeBuildIdToBuffer(info, /PrefixSpace=*/true, buffer);
	240	#endif
196	241	} else {
197	242	buffer->append("(%p)", (void *)address);
198	243	}
199	244	break;
200	245	default:
201		Report("Unsupported specifier in stack frame format: %c (0x%zx)!\n", *p,
202		*p);
	246	Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p,
	247	(void *)p);
203	248	Die();
204	249	}
205	250	}
...	...	@@ -244,14 +289,14 @@ void RenderData(InternalScopedString buffer, const char format,
244	289	buffer->append("%s", StripPathPrefix(DI->file, strip_path_prefix));
245	290	break;
246	291	case 'l':
247		buffer->append("%d", DI->line);
	292	buffer->append("%zu", DI->line);
248	293	break;
249	294	case 'g':
250	295	buffer->append("%s", DI->name);
251	296	break;
252	297	default:
253		Report("Unsupported specifier in stack frame format: %c (0x%zx)!\n", *p,
254		*p);
	298	Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p,
	299	(void *)p);
255	300	Die();
256	301	}
257	302	}

lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.h+5-4

...	...	@@ -17,6 +17,9 @@
17	17
18	18	namespace __sanitizer {
19	19
	20	// Strip interceptor prefixes from function name.
	21	const char StripFunctionName(const char function);
	22
20	23	// Render the contents of "info" structure, which represents the contents of
21	24	// stack frame "frame_no" and appends it to the "buffer". "format" is a
22	25	// string with placeholders, which is copied to the output with
...	...	@@ -26,8 +29,7 @@ namespace __sanitizer {
26	29	// will be turned into
27	30	// " frame 10: function foo::bar() at my/file.cc:10"
28	31	// You may additionally pass "strip_path_prefix" to strip prefixes of paths to
29		// source files and modules, and "strip_func_prefix" to strip prefixes of
30		// function names.
	32	// source files and modules.
31	33	// Here's the full list of available placeholders:
32	34	// %% - represents a '%' character;
33	35	// %n - frame number (copy of frame_no);
...	...	@@ -48,8 +50,7 @@ namespace __sanitizer {
48	50	// %M - prints module basename and offset, if it is known, or PC.
49	51	void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
50	52	uptr address, const AddressInfo *info, bool vs_style,
51		const char *strip_path_prefix = "",
52		const char *strip_func_prefix = "");
	53	const char *strip_path_prefix = "");
53	54
54	55	bool RenderNeedsSymbolization(const char *format);
55	56

lib/tsan/sanitizer_common/sanitizer_stacktrace_sparc.cpp+1-7

...	...	@@ -9,7 +9,7 @@
9	9	// This file is shared between AddressSanitizer and ThreadSanitizer
10	10	// run-time libraries.
11	11	//
12		// Implemention of fast stack unwinding for Sparc.
	12	// Implementation of fast stack unwinding for Sparc.
13	13	//===----------------------------------------------------------------------===//
14	14
15	15	#if defined(__sparc__)
...	...	@@ -30,13 +30,7 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
30	30	// TODO(yln): add arg sanity check for stack_top/stack_bottom
31	31	CHECK_GE(max_depth, 2);
32	32	const uptr kPageSize = GetPageSizeCached();
33		#if defined(__GNUC__)
34		// __builtin_return_address returns the address of the call instruction
35		// on the SPARC and not the return address, so we need to compensate.
36		trace_buffer[0] = GetNextInstructionPc(pc);
37		#else
38	33	trace_buffer[0] = pc;
39		#endif
40	34	size = 1;
41	35	if (stack_top < 4096) return; // Sanity check for stack top.
42	36	// Flush register windows to memory

lib/tsan/sanitizer_common/sanitizer_stoptheworld_fuchsia.h created+20

...	...	@@ -0,0 +1,20 @@
	1	//===-- sanitizer_stoptheworld_fuchsia.h ------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef SANITIZER_STOPTHEWORLD_FUCHSIA_H
	10	#define SANITIZER_STOPTHEWORLD_FUCHSIA_H
	11
	12	#include "sanitizer_stoptheworld.h"
	13
	14	namespace __sanitizer {
	15
	16	class SuspendedThreadsListFuchsia final : public SuspendedThreadsList {};
	17
	18	} // namespace __sanitizer
	19
	20	#endif // SANITIZER_STOPTHEWORLD_FUCHSIA_H

lib/tsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp+11-3

...	...	@@ -16,7 +16,7 @@
16	16	#if SANITIZER_LINUX && \
17	17	(defined(__x86_64__) \|\| defined(__mips__) \|\| defined(__aarch64__) \|\| \
18	18	defined(__powerpc64__) \|\| defined(__s390__) \|\| defined(__i386__) \|\| \
19		defined(__arm__) \|\| SANITIZER_RISCV64)
	19	defined(__arm__) \|\| SANITIZER_RISCV64 \|\| SANITIZER_LOONGARCH64)
20	20
21	21	#include "sanitizer_stoptheworld.h"
22	22
...	...	@@ -31,7 +31,8 @@
31	31	#include <sys/types.h> // for pid_t
32	32	#include <sys/uio.h> // for iovec
33	33	#include <elf.h> // for NT_PRSTATUS
34		#if (defined(__aarch64__) \|\| SANITIZER_RISCV64) && !SANITIZER_ANDROID
	34	#if (defined(__aarch64__) \|\| SANITIZER_RISCV64 \|\| SANITIZER_LOONGARCH64) && \
	35	!SANITIZER_ANDROID
35	36	// GLIBC 2.20+ sys/user does not include asm/ptrace.h
36	37	# include <asm/ptrace.h>
37	38	#endif
...	...	@@ -108,7 +109,7 @@ struct TracerThreadArgument {
108	109	void *callback_argument;
109	110	// The tracer thread waits on this mutex while the parent finishes its
110	111	// preparations.
111		BlockingMutex mutex;
	112	Mutex mutex;
112	113	// Tracer thread signals its completion by setting done.
113	114	atomic_uintptr_t done;
114	115	uptr parent_pid;
...	...	@@ -514,6 +515,12 @@ typedef struct user_pt_regs regs_struct;
514	515	static constexpr uptr kExtraRegs[] = {0};
515	516	#define ARCH_IOVEC_FOR_GETREGSET
516	517
	518	#elif defined(__loongarch__)
	519	typedef struct user_pt_regs regs_struct;
	520	#define REG_SP regs[3]
	521	static constexpr uptr kExtraRegs[] = {0};
	522	#define ARCH_IOVEC_FOR_GETREGSET
	523
517	524	#elif SANITIZER_RISCV64
518	525	typedef struct user_regs_struct regs_struct;
519	526	// sys/ucontext.h already defines REG_SP as 2. Undefine it first.
...	...	@@ -621,3 +628,4 @@ PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP(
621	628	#endif // SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__mips__)
622	629	// \|\| defined(__aarch64__) \|\| defined(__powerpc64__)
623	630	// \|\| defined(__s390__) \|\| defined(__i386__) \|\| defined(__arm__)
	631	// \|\| SANITIZER_LOONGARCH64

lib/tsan/sanitizer_common/sanitizer_stoptheworld_mac.cpp+10-9

...	...	@@ -12,7 +12,7 @@
12	12
13	13	#include "sanitizer_platform.h"
14	14
15		#if SANITIZER_MAC && (defined(__x86_64__) \|\| defined(__aarch64__) \|\| \
	15	#if SANITIZER_APPLE && (defined(__x86_64__) \|\| defined(__aarch64__) \|\| \
16	16	defined(__i386))
17	17
18	18	#include <mach/mach.h>
...	...	@@ -29,7 +29,7 @@ typedef struct {
29	29
30	30	class SuspendedThreadsListMac final : public SuspendedThreadsList {
31	31	public:
32		SuspendedThreadsListMac() : threads_(1024) {}
	32	SuspendedThreadsListMac() = default;
33	33
34	34	tid_t GetThreadID(uptr index) const override;
35	35	thread_t GetThread(uptr index) const;
...	...	@@ -87,11 +87,13 @@ void StopTheWorld(StopTheWorldCallback callback, void *argument) {
87	87
88	88	#if defined(__x86_64__)
89	89	typedef x86_thread_state64_t regs_struct;
	90	#define regs_flavor x86_THREAD_STATE64
90	91
91	92	#define SP_REG __rsp
92	93
93	94	#elif defined(__aarch64__)
94	95	typedef arm_thread_state64_t regs_struct;
	96	#define regs_flavor ARM_THREAD_STATE64
95	97
96	98	# if __DARWIN_UNIX03
97	99	# define SP_REG __sp
...	...	@@ -101,6 +103,7 @@ typedef arm_thread_state64_t regs_struct;
101	103
102	104	#elif defined(__i386)
103	105	typedef x86_thread_state32_t regs_struct;
	106	#define regs_flavor x86_THREAD_STATE32
104	107
105	108	#define SP_REG __esp
106	109
...	...	@@ -146,17 +149,15 @@ PtraceRegistersStatus SuspendedThreadsListMac::GetRegistersAndSP(
146	149	thread_t thread = GetThread(index);
147	150	regs_struct regs;
148	151	int err;
149		mach_msg_type_number_t reg_count = MACHINE_THREAD_STATE_COUNT;
150		err = thread_get_state(thread, MACHINE_THREAD_STATE, (thread_state_t)&regs,
	152	mach_msg_type_number_t reg_count = sizeof(regs) / sizeof(natural_t);
	153	err = thread_get_state(thread, regs_flavor, (thread_state_t)&regs,
151	154	&reg_count);
152	155	if (err != KERN_SUCCESS) {
153	156	VReport(1, "Error - unable to get registers for a thread\n");
154		// KERN_INVALID_ARGUMENT indicates that either the flavor is invalid,
155		// or the thread does not exist. The other possible error case,
156	157	// MIG_ARRAY_TOO_LARGE, means that the state is too large, but it's
157	158	// still safe to proceed.
158		return err == KERN_INVALID_ARGUMENT ? REGISTERS_UNAVAILABLE_FATAL
159		: REGISTERS_UNAVAILABLE;
	159	return err == MIG_ARRAY_TOO_LARGE ? REGISTERS_UNAVAILABLE
	160	: REGISTERS_UNAVAILABLE_FATAL;
160	161	}
161	162
162	163	buffer->resize(RoundUpTo(sizeof(regs), sizeof(uptr)) / sizeof(uptr));
...	...	@@ -176,5 +177,5 @@ PtraceRegistersStatus SuspendedThreadsListMac::GetRegistersAndSP(
176	177
177	178	} // namespace __sanitizer
178	179
179		#endif // SANITIZER_MAC && (defined(__x86_64__) \|\| defined(__aarch64__)) \|\|
	180	#endif // SANITIZER_APPLE && (defined(__x86_64__) \|\| defined(__aarch64__)) \|\|
180	181	// defined(__i386))

lib/tsan/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp+1-1

...	...	@@ -68,7 +68,7 @@ class SuspendedThreadsListNetBSD final : public SuspendedThreadsList {
68	68	struct TracerThreadArgument {
69	69	StopTheWorldCallback callback;
70	70	void *callback_argument;
71		BlockingMutex mutex;
	71	Mutex mutex;
72	72	atomic_uintptr_t done;
73	73	uptr parent_pid;
74	74	};

lib/tsan/sanitizer_common/sanitizer_stoptheworld_win.cpp created+175

...	...	@@ -0,0 +1,175 @@
	1	//===-- sanitizer_stoptheworld_win.cpp ------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// See sanitizer_stoptheworld.h for details.
	10	//
	11	//===----------------------------------------------------------------------===//
	12
	13	#include "sanitizer_platform.h"
	14
	15	#if SANITIZER_WINDOWS
	16
	17	# define WIN32_LEAN_AND_MEAN
	18	# include <windows.h>
	19	// windows.h needs to be included before tlhelp32.h
	20	# include <tlhelp32.h>
	21
	22	# include "sanitizer_stoptheworld.h"
	23
	24	namespace __sanitizer {
	25
	26	namespace {
	27
	28	struct SuspendedThreadsListWindows final : public SuspendedThreadsList {
	29	InternalMmapVector<HANDLE> threadHandles;
	30	InternalMmapVector<DWORD> threadIds;
	31
	32	SuspendedThreadsListWindows() {
	33	threadIds.reserve(1024);
	34	threadHandles.reserve(1024);
	35	}
	36
	37	PtraceRegistersStatus GetRegistersAndSP(uptr index,
	38	InternalMmapVector<uptr> *buffer,
	39	uptr *sp) const override;
	40
	41	tid_t GetThreadID(uptr index) const override;
	42	uptr ThreadCount() const override;
	43	};
	44
	45	// Stack Pointer register names on different architectures
	46	# if SANITIZER_X64
	47	# define SP_REG Rsp
	48	# elif SANITIZER_I386
	49	# define SP_REG Esp
	50	# elif SANITIZER_ARM \| SANITIZER_ARM64
	51	# define SP_REG Sp
	52	# else
	53	# error Architecture not supported!
	54	# endif
	55
	56	PtraceRegistersStatus SuspendedThreadsListWindows::GetRegistersAndSP(
	57	uptr index, InternalMmapVector<uptr> buffer, uptr sp) const {
	58	CHECK_LT(index, threadHandles.size());
	59
	60	buffer->resize(RoundUpTo(sizeof(CONTEXT), sizeof(uptr)) / sizeof(uptr));
	61	CONTEXT thread_context = reinterpret_cast<CONTEXT >(buffer->data());
	62	thread_context->ContextFlags = CONTEXT_ALL;
	63	CHECK(GetThreadContext(threadHandles[index], thread_context));
	64	*sp = thread_context->SP_REG;
	65
	66	return REGISTERS_AVAILABLE;
	67	}
	68
	69	tid_t SuspendedThreadsListWindows::GetThreadID(uptr index) const {
	70	CHECK_LT(index, threadIds.size());
	71	return threadIds[index];
	72	}
	73
	74	uptr SuspendedThreadsListWindows::ThreadCount() const {
	75	return threadIds.size();
	76	}
	77
	78	struct RunThreadArgs {
	79	StopTheWorldCallback callback;
	80	void *argument;
	81	};
	82
	83	DWORD WINAPI RunThread(void *argument) {
	84	RunThreadArgs run_args = (RunThreadArgs )argument;
	85
	86	const DWORD this_thread = GetCurrentThreadId();
	87	const DWORD this_process = GetCurrentProcessId();
	88
	89	SuspendedThreadsListWindows suspended_threads_list;
	90	bool new_thread_found;
	91
	92	do {
	93	// Take a snapshot of all Threads
	94	const HANDLE threads = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0);
	95	CHECK(threads != INVALID_HANDLE_VALUE);
	96
	97	THREADENTRY32 thread_entry;
	98	thread_entry.dwSize = sizeof(thread_entry);
	99	new_thread_found = false;
	100
	101	if (!Thread32First(threads, &thread_entry))
	102	break;
	103
	104	do {
	105	if (thread_entry.th32ThreadID == this_thread \|\|
	106	thread_entry.th32OwnerProcessID != this_process)
	107	continue;
	108
	109	bool suspended_thread = false;
	110	for (const auto thread_id : suspended_threads_list.threadIds) {
	111	if (thread_id == thread_entry.th32ThreadID) {
	112	suspended_thread = true;
	113	break;
	114	}
	115	}
	116
	117	// Skip the Thread if it was already suspended
	118	if (suspended_thread)
	119	continue;
	120
	121	const HANDLE thread =
	122	OpenThread(THREAD_ALL_ACCESS, FALSE, thread_entry.th32ThreadID);
	123	CHECK(thread);
	124
	125	if (SuspendThread(thread) == (DWORD)-1) {
	126	DWORD last_error = GetLastError();
	127
	128	VPrintf(1, "Could not suspend thread %lu (error %lu)",
	129	thread_entry.th32ThreadID, last_error);
	130	continue;
	131	}
	132
	133	suspended_threads_list.threadIds.push_back(thread_entry.th32ThreadID);
	134	suspended_threads_list.threadHandles.push_back(thread);
	135	new_thread_found = true;
	136	} while (Thread32Next(threads, &thread_entry));
	137
	138	CloseHandle(threads);
	139
	140	// Between the call to `CreateToolhelp32Snapshot` and suspending the
	141	// relevant Threads, new Threads could have potentially been created. So
	142	// continue to find and suspend new Threads until we don't find any.
	143	} while (new_thread_found);
	144
	145	// Now all Threads of this Process except of this Thread should be suspended.
	146	// Execute the callback function.
	147	run_args->callback(suspended_threads_list, run_args->argument);
	148
	149	// Resume all Threads
	150	for (const auto suspended_thread_handle :
	151	suspended_threads_list.threadHandles) {
	152	CHECK_NE(ResumeThread(suspended_thread_handle), -1);
	153	CloseHandle(suspended_thread_handle);
	154	}
	155
	156	return 0;
	157	}
	158
	159	} // namespace
	160
	161	void StopTheWorld(StopTheWorldCallback callback, void *argument) {
	162	struct RunThreadArgs arg = {callback, argument};
	163	DWORD trace_thread_id;
	164
	165	auto trace_thread =
	166	CreateThread(nullptr, 0, RunThread, &arg, 0, &trace_thread_id);
	167	CHECK(trace_thread);
	168
	169	WaitForSingleObject(trace_thread, INFINITE);
	170	CloseHandle(trace_thread);
	171	}
	172
	173	} // namespace __sanitizer
	174
	175	#endif // SANITIZER_WINDOWS

lib/tsan/sanitizer_common/sanitizer_suppressions.cpp+1

...	...	@@ -86,6 +86,7 @@ void SuppressionContext::ParseFromFile(const char *filename) {
86	86	}
87	87
88	88	Parse(file_contents);
	89	UnmapOrDie(file_contents, contents_size);
89	90	}
90	91
91	92	bool SuppressionContext::Match(const char str, const char type,

lib/tsan/sanitizer_common/sanitizer_symbolizer.cpp+13-7

...	...	@@ -11,10 +11,11 @@
11	11	//===----------------------------------------------------------------------===//
12	12
13	13	#include "sanitizer_allocator_internal.h"
14		#include "sanitizer_platform.h"
	14	#include "sanitizer_common.h"
15	15	#include "sanitizer_internal_defs.h"
16	16	#include "sanitizer_libc.h"
17	17	#include "sanitizer_placement_new.h"
	18	#include "sanitizer_platform.h"
18	19	#include "sanitizer_symbolizer_internal.h"
19	20
20	21	namespace __sanitizer {
...	...	@@ -30,6 +31,7 @@ void AddressInfo::Clear() {
30	31	InternalFree(file);
31	32	internal_memset(this, 0, sizeof(AddressInfo));
32	33	function_offset = kUnknown;
	34	uuid_size = 0;
33	35	}
34	36
35	37	void AddressInfo::FillModuleInfo(const char *mod_name, uptr mod_offset,
...	...	@@ -37,6 +39,16 @@ void AddressInfo::FillModuleInfo(const char *mod_name, uptr mod_offset,
37	39	module = internal_strdup(mod_name);
38	40	module_offset = mod_offset;
39	41	module_arch = mod_arch;
	42	uuid_size = 0;
	43	}
	44
	45	void AddressInfo::FillModuleInfo(const LoadedModule &mod) {
	46	module = internal_strdup(mod.full_name());
	47	module_offset = address - mod.base_address();
	48	module_arch = mod.arch();
	49	if (mod.uuid_size())
	50	internal_memcpy(uuid, mod.uuid(), mod.uuid_size());
	51	uuid_size = mod.uuid_size();
40	52	}
41	53
42	54	SymbolizedStack::SymbolizedStack() : next(nullptr), info() {}
...	...	@@ -126,10 +138,4 @@ Symbolizer::SymbolizerScope::~SymbolizerScope() {
126	138	sym_->end_hook_();
127	139	}
128	140
129		void Symbolizer::LateInitializeTools() {
130		for (auto &tool : tools_) {
131		tool.LateInitialize();
132		}
133		}
134
135	141	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_symbolizer.h+7-6

...	...	@@ -32,6 +32,8 @@ struct AddressInfo {
32	32	char *module;
33	33	uptr module_offset;
34	34	ModuleArch module_arch;
	35	u8 uuid[kModuleUUIDSize];
	36	uptr uuid_size;
35	37
36	38	static const uptr kUnknown = ~(uptr)0;
37	39	char *function;
...	...	@@ -45,6 +47,8 @@ struct AddressInfo {
45	47	// Deletes all strings and resets all fields.
46	48	void Clear();
47	49	void FillModuleInfo(const char *mod_name, uptr mod_offset, ModuleArch arch);
	50	void FillModuleInfo(const LoadedModule &mod);
	51	uptr module_base() const { return address - module_offset; }
48	52	};
49	53
50	54	// Linked list of symbolized frames (each frame is described by AddressInfo).
...	...	@@ -158,7 +162,7 @@ class Symbolizer final {
158	162	// its method should be protected by \|mu_\|.
159	163	class ModuleNameOwner {
160	164	public:
161		explicit ModuleNameOwner(BlockingMutex *synchronized_by)
	165	explicit ModuleNameOwner(Mutex *synchronized_by)
162	166	: last_match_(nullptr), mu_(synchronized_by) {
163	167	storage_.reserve(kInitialCapacity);
164	168	}
...	...	@@ -169,7 +173,7 @@ class Symbolizer final {
169	173	InternalMmapVector<const char*> storage_;
170	174	const char *last_match_;
171	175
172		BlockingMutex *mu_;
	176	Mutex *mu_;
173	177	} module_names_;
174	178
175	179	/// Platform-specific function for creating a Symbolizer object.
...	...	@@ -192,7 +196,7 @@ class Symbolizer final {
192	196	// Mutex locked from public methods of \|Symbolizer\|, so that the internals
193	197	// (including individual symbolizer tools and platform-specific methods) are
194	198	// always synchronized.
195		BlockingMutex mu_;
	199	Mutex mu_;
196	200
197	201	IntrusiveList<SymbolizerTool> tools_;
198	202
...	...	@@ -209,9 +213,6 @@ class Symbolizer final {
209	213	private:
210	214	const Symbolizer *sym_;
211	215	};
212
213		// Calls `LateInitialize()` on all items in `tools_`.
214		void LateInitializeTools();
215	216	};
216	217
217	218	#ifdef SANITIZER_WINDOWS

lib/tsan/sanitizer_common/sanitizer_symbolizer_internal.h+6-11

...	...	@@ -13,15 +13,15 @@
13	13	#ifndef SANITIZER_SYMBOLIZER_INTERNAL_H
14	14	#define SANITIZER_SYMBOLIZER_INTERNAL_H
15	15
16		#include "sanitizer_symbolizer.h"
17	16	#include "sanitizer_file.h"
	17	#include "sanitizer_symbolizer.h"
18	18	#include "sanitizer_vector.h"
19	19
20	20	namespace __sanitizer {
21	21
22	22	// Parsing helpers, 'str' is searched for delimiter(s) and a string or uptr
23	23	// is extracted. When extracting a string, a newly allocated (using
24		// InternalAlloc) and null-terminataed buffer is returned. They return a pointer
	24	// InternalAlloc) and null-terminated buffer is returned. They return a pointer
25	25	// to the next characted after the found delimiter.
26	26	const char ExtractToken(const char str, const char delims, char *result);
27	27	const char ExtractInt(const char str, const char delims, int result);
...	...	@@ -70,11 +70,6 @@ class SymbolizerTool {
70	70	return nullptr;
71	71	}
72	72
73		// Called during the LateInitialize phase of Sanitizer initialization.
74		// Usually this is a safe place to call code that might need to use user
75		// memory allocators.
76		virtual void LateInitialize() {}
77
78	73	protected:
79	74	~SymbolizerTool() {}
80	75	};
...	...	@@ -91,13 +86,14 @@ class SymbolizerProcess {
91	86	~SymbolizerProcess() {}
92	87
93	88	/// The maximum number of arguments required to invoke a tool process.
94		static const unsigned kArgVMax = 6;
	89	static const unsigned kArgVMax = 16;
95	90
96	91	// Customizable by subclasses.
97	92	virtual bool StartSymbolizerSubprocess();
98		virtual bool ReadFromSymbolizer(char *buffer, uptr max_length);
	93	virtual bool ReadFromSymbolizer();
99	94	// Return the environment to run the symbolizer in.
100	95	virtual char **GetEnvP() { return GetEnviron(); }
	96	InternalMmapVector<char> &GetBuff() { return buffer_; }
101	97
102	98	private:
103	99	virtual bool ReachedEndOfOutput(const char *buffer, uptr length) const {
...	...	@@ -118,8 +114,7 @@ class SymbolizerProcess {
118	114	fd_t input_fd_;
119	115	fd_t output_fd_;
120	116
121		static const uptr kBufferSize = 16 * 1024;
122		char buffer_[kBufferSize];
	117	InternalMmapVector<char> buffer_;
123	118
124	119	static const uptr kMaxTimesRestarted = 5;
125	120	static const int kSymbolizerStartupTimeMillis = 10;

lib/tsan/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp+2-2

...	...	@@ -11,11 +11,11 @@
11	11	// Libbacktrace implementation of symbolizer parts.
12	12	//===----------------------------------------------------------------------===//
13	13
14		#include "sanitizer_platform.h"
	14	#include "sanitizer_symbolizer_libbacktrace.h"
15	15
16	16	#include "sanitizer_internal_defs.h"
	17	#include "sanitizer_platform.h"
17	18	#include "sanitizer_symbolizer.h"
18		#include "sanitizer_symbolizer_libbacktrace.h"
19	19
20	20	#if SANITIZER_LIBBACKTRACE
21	21	# include "backtrace-supported.h"

lib/tsan/sanitizer_common/sanitizer_symbolizer_libcdep.cpp+51-40

...	...	@@ -83,16 +83,13 @@ const char ExtractTokenUpToDelimiter(const char str, const char *delimiter,
83	83	}
84	84
85	85	SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
86		BlockingMutexLock l(&mu_);
87		const char *module_name = nullptr;
88		uptr module_offset;
89		ModuleArch arch;
	86	Lock l(&mu_);
90	87	SymbolizedStack *res = SymbolizedStack::New(addr);
91		if (!FindModuleNameAndOffsetForAddress(addr, &module_name, &module_offset,
92		&arch))
	88	auto *mod = FindModuleForAddress(addr);
	89	if (!mod)
93	90	return res;
94	91	// Always fill data about module name and offset.
95		res->info.FillModuleInfo(module_name, module_offset, arch);
	92	res->info.FillModuleInfo(*mod);
96	93	for (auto &tool : tools_) {
97	94	SymbolizerScope sym_scope(this);
98	95	if (tool.SymbolizePC(addr, res)) {
...	...	@@ -103,7 +100,7 @@ SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
103	100	}
104	101
105	102	bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
106		BlockingMutexLock l(&mu_);
	103	Lock l(&mu_);
107	104	const char *module_name = nullptr;
108	105	uptr module_offset;
109	106	ModuleArch arch;
...	...	@@ -124,7 +121,7 @@ bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
124	121	}
125	122
126	123	bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) {
127		BlockingMutexLock l(&mu_);
	124	Lock l(&mu_);
128	125	const char *module_name = nullptr;
129	126	if (!FindModuleNameAndOffsetForAddress(
130	127	addr, &module_name, &info->module_offset, &info->module_arch))
...	...	@@ -141,7 +138,7 @@ bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) {
141	138
142	139	bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
143	140	uptr *module_address) {
144		BlockingMutexLock l(&mu_);
	141	Lock l(&mu_);
145	142	const char *internal_module_name = nullptr;
146	143	ModuleArch arch;
147	144	if (!FindModuleNameAndOffsetForAddress(pc, &internal_module_name,
...	...	@@ -154,7 +151,7 @@ bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
154	151	}
155	152
156	153	void Symbolizer::Flush() {
157		BlockingMutexLock l(&mu_);
	154	Lock l(&mu_);
158	155	for (auto &tool : tools_) {
159	156	SymbolizerScope sym_scope(this);
160	157	tool.Flush();
...	...	@@ -162,7 +159,7 @@ void Symbolizer::Flush() {
162	159	}
163	160
164	161	const char Symbolizer::Demangle(const char name) {
165		BlockingMutexLock l(&mu_);
	162	Lock l(&mu_);
166	163	for (auto &tool : tools_) {
167	164	SymbolizerScope sym_scope(this);
168	165	if (const char *demangled = tool.Demangle(name))
...	...	@@ -240,7 +237,7 @@ const LoadedModule *Symbolizer::FindModuleForAddress(uptr address) {
240	237	class LLVMSymbolizerProcess final : public SymbolizerProcess {
241	238	public:
242	239	explicit LLVMSymbolizerProcess(const char *path)
243		: SymbolizerProcess(path, /use_posix_spawn=/SANITIZER_MAC) {}
	240	: SymbolizerProcess(path, /use_posix_spawn=/SANITIZER_APPLE) {}
244	241
245	242	private:
246	243	bool ReachedEndOfOutput(const char *buffer, uptr length) const override {
...	...	@@ -259,6 +256,8 @@ class LLVMSymbolizerProcess final : public SymbolizerProcess {
259	256	const char* const kSymbolizerArch = "--default-arch=x86_64";
260	257	#elif defined(__i386__)
261	258	const char* const kSymbolizerArch = "--default-arch=i386";
	259	#elif SANITIZER_LOONGARCH64
	260	const char *const kSymbolizerArch = "--default-arch=loongarch64";
262	261	#elif SANITIZER_RISCV64
263	262	const char *const kSymbolizerArch = "--default-arch=riscv64";
264	263	#elif defined(__aarch64__)
...	...	@@ -277,14 +276,17 @@ class LLVMSymbolizerProcess final : public SymbolizerProcess {
277	276	const char* const kSymbolizerArch = "--default-arch=unknown";
278	277	#endif
279	278
280		const char *const inline_flag = common_flags()->symbolize_inline_frames
281		? "--inlines"
282		: "--no-inlines";
	279	const char *const demangle_flag =
	280	common_flags()->demangle ? "--demangle" : "--no-demangle";
	281	const char *const inline_flag =
	282	common_flags()->symbolize_inline_frames ? "--inlines" : "--no-inlines";
283	283	int i = 0;
284	284	argv[i++] = path_to_binary;
	285	argv[i++] = demangle_flag;
285	286	argv[i++] = inline_flag;
286	287	argv[i++] = kSymbolizerArch;
287	288	argv[i++] = nullptr;
	289	CHECK_LE(i, kArgVMax);
288	290	}
289	291	};
290	292
...	...	@@ -363,14 +365,21 @@ void ParseSymbolizePCOutput(const char str, SymbolizedStack res) {
363	365	}
364	366	}
365	367
366		// Parses a two-line string in the following format:
	368	// Parses a two- or three-line string in the following format:
367	369	// <symbol_name>
368	370	// <start_address> <size>
369		// Used by LLVMSymbolizer and InternalSymbolizer.
	371	// <filename>:<column>
	372	// Used by LLVMSymbolizer and InternalSymbolizer. LLVMSymbolizer added support
	373	// for symbolizing the third line in D123538, but we support the older two-line
	374	// information as well.
370	375	void ParseSymbolizeDataOutput(const char str, DataInfo info) {
371	376	str = ExtractToken(str, "\n", &info->name);
372	377	str = ExtractUptr(str, " ", &info->start);
373	378	str = ExtractUptr(str, "\n", &info->size);
	379	// Note: If the third line isn't present, these calls will set info.{file,
	380	// line} to empty strings.
	381	str = ExtractToken(str, ":", &info->file);
	382	str = ExtractUptr(str, "\n", &info->line);
374	383	}
375	384
376	385	static void ParseSymbolizeFrameOutput(const char *str,
...	...	@@ -500,9 +509,9 @@ const char SymbolizerProcess::SendCommandImpl(const char command) {
500	509	return nullptr;
501	510	if (!WriteToSymbolizer(command, internal_strlen(command)))
502	511	return nullptr;
503		if (!ReadFromSymbolizer(buffer_, kBufferSize))
504		return nullptr;
505		return buffer_;
	512	if (!ReadFromSymbolizer())
	513	return nullptr;
	514	return buffer_.data();
506	515	}
507	516
508	517	bool SymbolizerProcess::Restart() {
...	...	@@ -513,31 +522,33 @@ bool SymbolizerProcess::Restart() {
513	522	return StartSymbolizerSubprocess();
514	523	}
515	524
516		bool SymbolizerProcess::ReadFromSymbolizer(char *buffer, uptr max_length) {
517		if (max_length == 0)
518		return true;
519		uptr read_len = 0;
520		while (true) {
	525	bool SymbolizerProcess::ReadFromSymbolizer() {
	526	buffer_.clear();
	527	constexpr uptr max_length = 1024;
	528	bool ret = true;
	529	do {
521	530	uptr just_read = 0;
522		bool success = ReadFromFile(input_fd_, buffer + read_len,
523		max_length - read_len - 1, &just_read);
	531	uptr size_before = buffer_.size();
	532	buffer_.resize(size_before + max_length);
	533	buffer_.resize(buffer_.capacity());
	534	bool ret = ReadFromFile(input_fd_, &buffer_[size_before],
	535	buffer_.size() - size_before, &just_read);
	536
	537	if (!ret)
	538	just_read = 0;
	539
	540	buffer_.resize(size_before + just_read);
	541
524	542	// We can't read 0 bytes, as we don't expect external symbolizer to close
525	543	// its stdout.
526		if (!success \|\| just_read == 0) {
	544	if (just_read == 0) {
527	545	Report("WARNING: Can't read from symbolizer at fd %d\n", input_fd_);
528		return false;
529		}
530		read_len += just_read;
531		if (ReachedEndOfOutput(buffer, read_len))
532		break;
533		if (read_len + 1 == max_length) {
534		Report("WARNING: Symbolizer buffer too small\n");
535		read_len = 0;
	546	ret = false;
536	547	break;
537	548	}
538		}
539		buffer[read_len] = '\0';
540		return true;
	549	} while (!ReachedEndOfOutput(buffer_.data(), buffer_.size()));
	550	buffer_.push_back('\0');
	551	return ret;
541	552	}
542	553
543	554	bool SymbolizerProcess::WriteToSymbolizer(const char *buffer, uptr length) {

lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.cpp+15-66

...	...	@@ -12,19 +12,18 @@
12	12	//===----------------------------------------------------------------------===//
13	13
14	14	#include "sanitizer_platform.h"
15		#if SANITIZER_MAC
	15	#if SANITIZER_APPLE
16	16
17		#include "sanitizer_allocator_internal.h"
18		#include "sanitizer_mac.h"
19		#include "sanitizer_symbolizer_mac.h"
	17	# include <dlfcn.h>
	18	# include <errno.h>
	19	# include <stdlib.h>
	20	# include <sys/wait.h>
	21	# include <unistd.h>
	22	# include <util.h>
20	23
21		#include <dlfcn.h>
22		#include <errno.h>
23		#include <mach/mach.h>
24		#include <stdlib.h>
25		#include <sys/wait.h>
26		#include <unistd.h>
27		#include <util.h>
	24	# include "sanitizer_allocator_internal.h"
	25	# include "sanitizer_mac.h"
	26	# include "sanitizer_symbolizer_mac.h"
28	27
29	28	namespace __sanitizer {
30	29
...	...	@@ -58,13 +57,6 @@ bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) {
58	57	return true;
59	58	}
60	59
61		#define K_ATOS_ENV_VAR "__check_mach_ports_lookup"
62
63		// This cannot live in `AtosSymbolizerProcess` because instances of that object
64		// are allocated by the internal allocator which under ASan is poisoned with
65		// kAsanInternalHeapMagic.
66		static char kAtosMachPortEnvEntry[] = K_ATOS_ENV_VAR "=000000000000000";
67
68	60	class AtosSymbolizerProcess final : public SymbolizerProcess {
69	61	public:
70	62	explicit AtosSymbolizerProcess(const char *path)
...	...	@@ -72,51 +64,13 @@ class AtosSymbolizerProcess final : public SymbolizerProcess {
72	64	pid_str_[0] = '\0';
73	65	}
74	66
75		void LateInitialize() {
76		if (SANITIZER_IOSSIM) {
77		// `putenv()` may call malloc/realloc so it is only safe to do this
78		// during LateInitialize() or later (i.e. we can't do this in the
79		// constructor). We also can't do this in `StartSymbolizerSubprocess()`
80		// because in TSan we switch allocators when we're symbolizing.
81		// We use `putenv()` rather than `setenv()` so that we can later directly
82		// write into the storage without LibC getting involved to change what the
83		// variable is set to
84		int result = putenv(kAtosMachPortEnvEntry);
85		CHECK_EQ(result, 0);
86		}
87		}
88
89	67	private:
90	68	bool StartSymbolizerSubprocess() override {
91		// Configure sandbox before starting atos process.
92
93	69	// Put the string command line argument in the object so that it outlives
94	70	// the call to GetArgV.
95		internal_snprintf(pid_str_, sizeof(pid_str_), "%d", internal_getpid());
96
97		if (SANITIZER_IOSSIM) {
98		// `atos` in the simulator is restricted in its ability to retrieve the
99		// task port for the target process (us) so we need to do extra work
100		// to pass our task port to it.
101		mach_port_t ports[]{mach_task_self()};
102		kern_return_t ret =
103		mach_ports_register(mach_task_self(), ports, /count=/1);
104		CHECK_EQ(ret, KERN_SUCCESS);
105
106		// Set environment variable that signals to `atos` that it should look
107		// for our task port. We can't call `setenv()` here because it might call
108		// malloc/realloc. To avoid that we instead update the
109		// `mach_port_env_var_entry_` variable with our current PID.
110		uptr count = internal_snprintf(kAtosMachPortEnvEntry,
111		sizeof(kAtosMachPortEnvEntry),
112		K_ATOS_ENV_VAR "=%s", pid_str_);
113		CHECK_GE(count, sizeof(K_ATOS_ENV_VAR) + internal_strlen(pid_str_));
114		// Document our assumption but without calling `getenv()` in normal
115		// builds.
116		DCHECK(getenv(K_ATOS_ENV_VAR));
117		DCHECK_EQ(internal_strcmp(getenv(K_ATOS_ENV_VAR), pid_str_), 0);
118		}
	71	internal_snprintf(pid_str_, sizeof(pid_str_), "%d", (int)internal_getpid());
119	72
	73	// Configure sandbox before starting atos process.
120	74	return SymbolizerProcess::StartSymbolizerSubprocess();
121	75	}
122	76
...	...	@@ -137,13 +91,10 @@ class AtosSymbolizerProcess final : public SymbolizerProcess {
137	91	argv[i++] = "-d";
138	92	}
139	93	argv[i++] = nullptr;
	94	CHECK_LE(i, kArgVMax);
140	95	}
141	96
142	97	char pid_str_[16];
143		// Space for `\0` in `K_ATOS_ENV_VAR` is reused for `=`.
144		static_assert(sizeof(kAtosMachPortEnvEntry) ==
145		(sizeof(K_ATOS_ENV_VAR) + sizeof(pid_str_)),
146		"sizes should match");
147	98	};
148	99
149	100	#undef K_ATOS_ENV_VAR
...	...	@@ -212,7 +163,7 @@ bool AtosSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
212	163	uptr start_address = AddressInfo::kUnknown;
213	164	if (!ParseCommandOutput(buf, addr, &stack->info.function, &stack->info.module,
214	165	&stack->info.file, &line, &start_address)) {
215		process_ = nullptr;
	166	Report("WARNING: atos failed to symbolize address \"0x%zx\"\n", addr);
216	167	return false;
217	168	}
218	169	stack->info.line = (int)line;
...	...	@@ -249,8 +200,6 @@ bool AtosSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
249	200	return true;
250	201	}
251	202
252		void AtosSymbolizer::LateInitialize() { process_->LateInitialize(); }
253
254	203	} // namespace __sanitizer
255	204
256		#endif // SANITIZER_MAC
	205	#endif // SANITIZER_APPLE

lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.h+2-3

...	...	@@ -15,7 +15,7 @@
15	15	#define SANITIZER_SYMBOLIZER_MAC_H
16	16
17	17	#include "sanitizer_platform.h"
18		#if SANITIZER_MAC
	18	#if SANITIZER_APPLE
19	19
20	20	#include "sanitizer_symbolizer_internal.h"
21	21
...	...	@@ -35,7 +35,6 @@ class AtosSymbolizer final : public SymbolizerTool {
35	35
36	36	bool SymbolizePC(uptr addr, SymbolizedStack *stack) override;
37	37	bool SymbolizeData(uptr addr, DataInfo *info) override;
38		void LateInitialize() override;
39	38
40	39	private:
41	40	AtosSymbolizerProcess *process_;
...	...	@@ -43,6 +42,6 @@ class AtosSymbolizer final : public SymbolizerTool {
43	42
44	43	} // namespace __sanitizer
45	44
46		#endif // SANITIZER_MAC
	45	#endif // SANITIZER_APPLE
47	46
48	47	#endif // SANITIZER_SYMBOLIZER_MAC_H

lib/tsan/sanitizer_common/sanitizer_symbolizer_markup.cpp+2-4

...	...	@@ -91,7 +91,7 @@ bool RenderNeedsSymbolization(const char *format) { return false; }
91	91	// We don't support the stack_trace_format flag at all.
92	92	void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
93	93	uptr address, const AddressInfo *info, bool vs_style,
94		const char strip_path_prefix, const char strip_func_prefix) {
	94	const char *strip_path_prefix) {
95	95	CHECK(!RenderNeedsSymbolization(format));
96	96	buffer->append(kFormatFrame, frame_no, address);
97	97	}
...	...	@@ -100,9 +100,7 @@ Symbolizer *Symbolizer::PlatformInit() {
100	100	return new (symbolizer_allocator_) Symbolizer({});
101	101	}
102	102
103		void Symbolizer::LateInitialize() {
104		Symbolizer::GetOrInit()->LateInitializeTools();
105		}
	103	void Symbolizer::LateInitialize() { Symbolizer::GetOrInit(); }
106	104
107	105	void StartReportDeadlySignal() {}
108	106	void ReportDeadlySignal(const SignalContext &sig, u32 tid,

lib/tsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp+71-62

...	...	@@ -13,25 +13,25 @@
13	13
14	14	#include "sanitizer_platform.h"
15	15	#if SANITIZER_POSIX
16		#include "sanitizer_allocator_internal.h"
17		#include "sanitizer_common.h"
18		#include "sanitizer_file.h"
19		#include "sanitizer_flags.h"
20		#include "sanitizer_internal_defs.h"
21		#include "sanitizer_linux.h"
22		#include "sanitizer_placement_new.h"
23		#include "sanitizer_posix.h"
24		#include "sanitizer_procmaps.h"
25		#include "sanitizer_symbolizer_internal.h"
26		#include "sanitizer_symbolizer_libbacktrace.h"
27		#include "sanitizer_symbolizer_mac.h"
28
29		#include <dlfcn.h> // for dlsym()
30		#include <errno.h>
31		#include <stdint.h>
32		#include <stdlib.h>
33		#include <sys/wait.h>
34		#include <unistd.h>
	16	# include <dlfcn.h> // for dlsym()
	17	# include <errno.h>
	18	# include <stdint.h>
	19	# include <stdlib.h>
	20	# include <sys/wait.h>
	21	# include <unistd.h>
	22
	23	# include "sanitizer_allocator_internal.h"
	24	# include "sanitizer_common.h"
	25	# include "sanitizer_file.h"
	26	# include "sanitizer_flags.h"
	27	# include "sanitizer_internal_defs.h"
	28	# include "sanitizer_linux.h"
	29	# include "sanitizer_placement_new.h"
	30	# include "sanitizer_posix.h"
	31	# include "sanitizer_procmaps.h"
	32	# include "sanitizer_symbolizer_internal.h"
	33	# include "sanitizer_symbolizer_libbacktrace.h"
	34	# include "sanitizer_symbolizer_mac.h"
35	35
36	36	// C++ demangling function, as required by Itanium C++ ABI. This is weak,
37	37	// because we do not require a C++ ABI library to be linked to a program
...	...	@@ -72,7 +72,6 @@ static swift_demangle_ft swift_demangle_f;
72	72	// symbolication.
73	73	static void InitializeSwiftDemangler() {
74	74	swift_demangle_f = (swift_demangle_ft)dlsym(RTLD_DEFAULT, "swift_demangle");
75		(void)dlerror(); // Cleanup error message in case of failure
76	75	}
77	76
78	77	// Attempts to demangle a Swift name. The demangler will return nullptr if a
...	...	@@ -155,7 +154,7 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
155	154	}
156	155
157	156	if (use_posix_spawn_) {
158		#if SANITIZER_MAC
	157	#if SANITIZER_APPLE
159	158	fd_t fd = internal_spawn(argv, const_cast<const char **>(GetEnvP()), &pid);
160	159	if (fd == kInvalidFd) {
161	160	Report("WARNING: failed to spawn external symbolizer (errno: %d)\n",
...	...	@@ -165,9 +164,9 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
165	164
166	165	input_fd_ = fd;
167	166	output_fd_ = fd;
168		#else // SANITIZER_MAC
	167	#else // SANITIZER_APPLE
169	168	UNIMPLEMENTED();
170		#endif // SANITIZER_MAC
	169	#endif // SANITIZER_APPLE
171	170	} else {
172	171	fd_t infd[2] = {}, outfd[2] = {};
173	172	if (!CreateTwoHighNumberedPipes(infd, outfd)) {
...	...	@@ -213,31 +212,36 @@ class Addr2LineProcess final : public SymbolizerProcess {
213	212	const char *(&argv)[kArgVMax]) const override {
214	213	int i = 0;
215	214	argv[i++] = path_to_binary;
216		argv[i++] = "-iCfe";
	215	if (common_flags()->demangle)
	216	argv[i++] = "-C";
	217	if (common_flags()->symbolize_inline_frames)
	218	argv[i++] = "-i";
	219	argv[i++] = "-fe";
217	220	argv[i++] = module_name_;
218	221	argv[i++] = nullptr;
	222	CHECK_LE(i, kArgVMax);
219	223	}
220	224
221	225	bool ReachedEndOfOutput(const char *buffer, uptr length) const override;
222	226
223		bool ReadFromSymbolizer(char *buffer, uptr max_length) override {
224		if (!SymbolizerProcess::ReadFromSymbolizer(buffer, max_length))
	227	bool ReadFromSymbolizer() override {
	228	if (!SymbolizerProcess::ReadFromSymbolizer())
225	229	return false;
226		// The returned buffer is empty when output is valid, but exceeds
227		// max_length.
228		if (*buffer == '\0')
229		return true;
	230	auto &buff = GetBuff();
230	231	// We should cut out output_terminator_ at the end of given buffer,
231	232	// appended by addr2line to mark the end of its meaningful output.
232	233	// We cannot scan buffer from it's beginning, because it is legal for it
233	234	// to start with output_terminator_ in case given offset is invalid. So,
234	235	// scanning from second character.
235		char *garbage = internal_strstr(buffer + 1, output_terminator_);
	236	char *garbage = internal_strstr(buff.data() + 1, output_terminator_);
236	237	// This should never be NULL since buffer must end up with
237	238	// output_terminator_.
238	239	CHECK(garbage);
	240
239	241	// Trim the buffer.
240		garbage[0] = '\0';
	242	uintptr_t new_size = garbage - buff.data();
	243	GetBuff().resize(new_size);
	244	GetBuff().push_back('\0');
241	245	return true;
242	246	}
243	247
...	...	@@ -312,37 +316,42 @@ class Addr2LinePool final : public SymbolizerTool {
312	316	FIRST_32_SECOND_64(UINT32_MAX, UINT64_MAX);
313	317	};
314	318
315		#if SANITIZER_SUPPORTS_WEAK_HOOKS
	319	# if SANITIZER_SUPPORTS_WEAK_HOOKS
316	320	extern "C" {
317	321	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
318	322	__sanitizer_symbolize_code(const char *ModuleName, u64 ModuleOffset,
319		char *Buffer, int MaxLength,
320		bool SymbolizeInlineFrames);
321		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
322		bool __sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset,
323		char *Buffer, int MaxLength);
324		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
325		void __sanitizer_symbolize_flush();
326		SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
327		int __sanitizer_symbolize_demangle(const char Name, char Buffer,
328		int MaxLength);
	323	char *Buffer, int MaxLength);
	324	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
	325	__sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset,
	326	char *Buffer, int MaxLength);
	327	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
	328	__sanitizer_symbolize_flush();
	329	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int
	330	__sanitizer_symbolize_demangle(const char Name, char Buffer, int MaxLength);
	331	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
	332	__sanitizer_symbolize_set_demangle(bool Demangle);
	333	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
	334	__sanitizer_symbolize_set_inline_frames(bool InlineFrames);
329	335	} // extern "C"
330	336
331	337	class InternalSymbolizer final : public SymbolizerTool {
332	338	public:
333	339	static InternalSymbolizer get(LowLevelAllocator alloc) {
334		if (__sanitizer_symbolize_code != 0 &&
335		__sanitizer_symbolize_data != 0) {
336		return new(*alloc) InternalSymbolizer();
337		}
	340	if (__sanitizer_symbolize_set_demangle)
	341	CHECK(__sanitizer_symbolize_set_demangle(common_flags()->demangle));
	342	if (__sanitizer_symbolize_set_inline_frames)
	343	CHECK(__sanitizer_symbolize_set_inline_frames(
	344	common_flags()->symbolize_inline_frames));
	345	if (__sanitizer_symbolize_code && __sanitizer_symbolize_data)
	346	return new (*alloc) InternalSymbolizer();
338	347	return 0;
339	348	}
340	349
341	350	bool SymbolizePC(uptr addr, SymbolizedStack *stack) override {
342	351	bool result = __sanitizer_symbolize_code(
343		stack->info.module, stack->info.module_offset, buffer_, kBufferSize,
344		common_flags()->symbolize_inline_frames);
345		if (result) ParseSymbolizePCOutput(buffer_, stack);
	352	stack->info.module, stack->info.module_offset, buffer_, kBufferSize);
	353	if (result)
	354	ParseSymbolizePCOutput(buffer_, stack);
346	355	return result;
347	356	}
348	357
...	...	@@ -365,7 +374,7 @@ class InternalSymbolizer final : public SymbolizerTool {
365	374	if (__sanitizer_symbolize_demangle) {
366	375	for (uptr res_length = 1024;
367	376	res_length <= InternalSizeClassMap::kMaxSize;) {
368		char res_buff = static_cast<char>(InternalAlloc(res_length));
	377	char res_buff = static_cast<char >(InternalAlloc(res_length));
369	378	uptr req_length =
370	379	__sanitizer_symbolize_demangle(name, res_buff, res_length);
371	380	if (req_length > res_length) {
...	...	@@ -380,19 +389,19 @@ class InternalSymbolizer final : public SymbolizerTool {
380	389	}
381	390
382	391	private:
383		InternalSymbolizer() { }
	392	InternalSymbolizer() {}
384	393
385	394	static const int kBufferSize = 16 * 1024;
386	395	char buffer_[kBufferSize];
387	396	};
388		#else // SANITIZER_SUPPORTS_WEAK_HOOKS
	397	# else // SANITIZER_SUPPORTS_WEAK_HOOKS
389	398
390	399	class InternalSymbolizer final : public SymbolizerTool {
391	400	public:
392	401	static InternalSymbolizer get(LowLevelAllocator alloc) { return 0; }
393	402	};
394	403
395		#endif // SANITIZER_SUPPORTS_WEAK_HOOKS
	404	# endif // SANITIZER_SUPPORTS_WEAK_HOOKS
396	405
397	406	const char Symbolizer::PlatformDemangle(const char name) {
398	407	return DemangleSwiftAndCXX(name);
...	...	@@ -417,13 +426,13 @@ static SymbolizerTool ChooseExternalSymbolizer(LowLevelAllocator allocator) {
417	426	VReport(2, "Using llvm-symbolizer at user-specified path: %s\n", path);
418	427	return new(*allocator) LLVMSymbolizer(path, allocator);
419	428	} else if (!internal_strcmp(binary_name, "atos")) {
420		#if SANITIZER_MAC
	429	#if SANITIZER_APPLE
421	430	VReport(2, "Using atos at user-specified path: %s\n", path);
422	431	return new(*allocator) AtosSymbolizer(path, allocator);
423		#else // SANITIZER_MAC
	432	#else // SANITIZER_APPLE
424	433	Report("ERROR: Using `atos` is only supported on Darwin.\n");
425	434	Die();
426		#endif // SANITIZER_MAC
	435	#endif // SANITIZER_APPLE
427	436	} else if (!internal_strcmp(binary_name, "addr2line")) {
428	437	VReport(2, "Using addr2line at user-specified path: %s\n", path);
429	438	return new(*allocator) Addr2LinePool(path, allocator);
...	...	@@ -436,12 +445,12 @@ static SymbolizerTool ChooseExternalSymbolizer(LowLevelAllocator allocator) {
436	445
437	446	// Otherwise symbolizer program is unknown, let's search $PATH
438	447	CHECK(path == nullptr);
439		#if SANITIZER_MAC
	448	#if SANITIZER_APPLE
440	449	if (const char *found_path = FindPathToBinary("atos")) {
441	450	VReport(2, "Using atos found at: %s\n", found_path);
442	451	return new(*allocator) AtosSymbolizer(found_path, allocator);
443	452	}
444		#endif // SANITIZER_MAC
	453	#endif // SANITIZER_APPLE
445	454	if (const char *found_path = FindPathToBinary("llvm-symbolizer")) {
446	455	VReport(2, "Using llvm-symbolizer found at: %s\n", found_path);
447	456	return new(*allocator) LLVMSymbolizer(found_path, allocator);
...	...	@@ -478,10 +487,10 @@ static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,
478	487	list->push_back(tool);
479	488	}
480	489
481		#if SANITIZER_MAC
	490	#if SANITIZER_APPLE
482	491	VReport(2, "Using dladdr symbolizer.\n");
483	492	list->push_back(new(*allocator) DlAddrSymbolizer());
484		#endif // SANITIZER_MAC
	493	#endif // SANITIZER_APPLE
485	494	}
486	495
487	496	Symbolizer *Symbolizer::PlatformInit() {
...	...	@@ -492,7 +501,7 @@ Symbolizer *Symbolizer::PlatformInit() {
492	501	}
493	502
494	503	void Symbolizer::LateInitialize() {
495		Symbolizer::GetOrInit()->LateInitializeTools();
	504	Symbolizer::GetOrInit();
496	505	InitializeSwiftDemangler();
497	506	}
498	507

lib/tsan/sanitizer_common/sanitizer_symbolizer_report.cpp+11-6

...	...	@@ -88,11 +88,17 @@ void ReportErrorSummary(const char error_type, const StackTrace stack,
88	88	#endif
89	89	}
90	90
91		void ReportMmapWriteExec(int prot) {
	91	void ReportMmapWriteExec(int prot, int flags) {
92	92	#if SANITIZER_POSIX && (!SANITIZER_GO && !SANITIZER_ANDROID)
93		if ((prot & (PROT_WRITE \| PROT_EXEC)) != (PROT_WRITE \| PROT_EXEC))
	93	int pflags = (PROT_WRITE \| PROT_EXEC);
	94	if ((prot & pflags) != pflags)
94	95	return;
95	96
	97	# if SANITIZER_APPLE && defined(MAP_JIT)
	98	if ((flags & MAP_JIT) == MAP_JIT)
	99	return;
	100	# endif
	101
96	102	ScopedErrorReportLock l;
97	103	SanitizerCommonDecorator d;
98	104
...	...	@@ -101,8 +107,7 @@ void ReportMmapWriteExec(int prot) {
101	107	stack->Reset();
102	108	uptr top = 0;
103	109	uptr bottom = 0;
104		GET_CALLER_PC_BP_SP;
105		(void)sp;
	110	GET_CALLER_PC_BP;
106	111	bool fast = common_flags()->fast_unwind_on_fatal;
107	112	if (StackTrace::WillUseFastUnwind(fast)) {
108	113	GetThreadStackTopAndBottom(false, &top, &bottom);
...	...	@@ -205,9 +210,9 @@ static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid,
205	210	Report("Hint: pc points to the zero page.\n");
206	211	if (sig.is_memory_access) {
207	212	const char *access_type =
208		sig.write_flag == SignalContext::WRITE
	213	sig.write_flag == SignalContext::Write
209	214	? "WRITE"
210		: (sig.write_flag == SignalContext::READ ? "READ" : "UNKNOWN");
	215	: (sig.write_flag == SignalContext::Read ? "READ" : "UNKNOWN");
211	216	Report("The signal is caused by a %s memory access.\n", access_type);
212	217	if (!sig.is_true_faulting_addr)
213	218	Report("Hint: this fault was caused by a dereference of a high value "

lib/tsan/sanitizer_common/sanitizer_symbolizer_win.cpp+8-10

...	...	@@ -14,8 +14,8 @@
14	14	#include "sanitizer_platform.h"
15	15	#if SANITIZER_WINDOWS
16	16
17		#include "sanitizer_dbghelp.h"
18		#include "sanitizer_symbolizer_internal.h"
	17	# include "sanitizer_dbghelp.h"
	18	# include "sanitizer_symbolizer_internal.h"
19	19
20	20	namespace __sanitizer {
21	21
...	...	@@ -231,8 +231,6 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
231	231	// Check that tool command lines are simple and that complete escaping is
232	232	// unnecessary.
233	233	CHECK(!internal_strchr(arg, '"') && "quotes in args unsupported");
234		CHECK(!internal_strstr(arg, "\\\\") &&
235		"double backslashes in args unsupported");
236	234	CHECK(arglen > 0 && arg[arglen - 1] != '\\' &&
237	235	"args ending in backslash and empty args unsupported");
238	236	command_line.append("\"%s\" ", arg);
...	...	@@ -294,15 +292,15 @@ static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,
294	292	const char *path =
295	293	user_path ? user_path : FindPathToBinary("llvm-symbolizer.exe");
296	294	if (path) {
297		VReport(2, "Using llvm-symbolizer at %spath: %s\n",
298		user_path ? "user-specified " : "", path);
299		list->push_back(new(*allocator) LLVMSymbolizer(path, allocator));
300		} else {
301	295	if (user_path && user_path[0] == '\0') {
302	296	VReport(2, "External symbolizer is explicitly disabled.\n");
303	297	} else {
304		VReport(2, "External symbolizer is not present.\n");
	298	VReport(2, "Using llvm-symbolizer at %spath: %s\n",
	299	user_path ? "user-specified " : "", path);
	300	list->push_back(new (*allocator) LLVMSymbolizer(path, allocator));
305	301	}
	302	} else {
	303	VReport(2, "External symbolizer is not present.\n");
306	304	}
307	305
308	306	// Add the dbghelp based symbolizer.
...	...	@@ -318,7 +316,7 @@ Symbolizer *Symbolizer::PlatformInit() {
318	316	}
319	317
320	318	void Symbolizer::LateInitialize() {
321		Symbolizer::GetOrInit()->LateInitializeTools();
	319	Symbolizer::GetOrInit();
322	320	}
323	321
324	322	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_syscall_generic.inc+3-2

...	...	@@ -13,13 +13,14 @@
13	13	// NetBSD uses libc calls directly
14	14	#if !SANITIZER_NETBSD
15	15
16		#if SANITIZER_FREEBSD \|\| SANITIZER_MAC \|\| SANITIZER_SOLARIS
	16	#if SANITIZER_FREEBSD \|\| SANITIZER_APPLE \|\| SANITIZER_SOLARIS
17	17	# define SYSCALL(name) SYS_ ## name
18	18	#else
19	19	# define SYSCALL(name) __NR_ ## name
20	20	#endif
21	21
22		#if defined(__x86_64__) && (SANITIZER_FREEBSD \|\| SANITIZER_MAC)
	22	#if (defined(__x86_64__) && (SANITIZER_FREEBSD \|\| SANITIZER_APPLE)) \|\| \
	23	(defined(__aarch64__) && SANITIZER_FREEBSD)
23	24	# define internal_syscall __syscall
24	25	# else
25	26	# define internal_syscall syscall

lib/tsan/sanitizer_common/sanitizer_syscalls_netbsd.inc+2-2

...	...	@@ -2255,13 +2255,13 @@ PRE_SYSCALL(getcontext)(void ucp_) { / Nothing to do */ }
2255	2255	POST_SYSCALL(getcontext)(long long res, void ucp_) { / Nothing to do */ }
2256	2256	PRE_SYSCALL(setcontext)(void *ucp_) {
2257	2257	if (ucp_) {
2258		PRE_READ(ucp_, ucontext_t_sz);
	2258	PRE_READ(ucp_, ucontext_t_sz(ucp_));
2259	2259	}
2260	2260	}
2261	2261	POST_SYSCALL(setcontext)(long long res, void *ucp_) {}
2262	2262	PRE_SYSCALL(_lwp_create)(void ucp_, long long flags_, void new_lwp_) {
2263	2263	if (ucp_) {
2264		PRE_READ(ucp_, ucontext_t_sz);
	2264	PRE_READ(ucp_, ucontext_t_sz(ucp_));
2265	2265	}
2266	2266	}
2267	2267	POST_SYSCALL(_lwp_create)

lib/tsan/sanitizer_common/sanitizer_thread_arg_retval.cpp created+94

...	...	@@ -0,0 +1,94 @@
	1	//===-- sanitizer_thread_arg_retval.cpp -------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between sanitizer tools.
	10	//
	11	// Tracks thread arguments and return value for leak checking.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_thread_arg_retval.h"
	15
	16	#include "sanitizer_placement_new.h"
	17
	18	namespace __sanitizer {
	19
	20	void ThreadArgRetval::CreateLocked(uptr thread, bool detached,
	21	const Args& args) {
	22	CheckLocked();
	23	Data& t = data_[thread];
	24	t = {};
	25	t.gen = gen_++;
	26	t.detached = detached;
	27	t.args = args;
	28	}
	29
	30	ThreadArgRetval::Args ThreadArgRetval::GetArgs(uptr thread) const {
	31	__sanitizer::Lock lock(&mtx_);
	32	auto t = data_.find(thread);
	33	CHECK(t);
	34	if (t->second.done)
	35	return {};
	36	return t->second.args;
	37	}
	38
	39	void ThreadArgRetval::Finish(uptr thread, void* retval) {
	40	__sanitizer::Lock lock(&mtx_);
	41	auto t = data_.find(thread);
	42	if (!t)
	43	return;
	44	if (t->second.detached) {
	45	// Retval of detached thread connot be retrieved.
	46	data_.erase(t);
	47	return;
	48	}
	49	t->second.done = true;
	50	t->second.args.arg_retval = retval;
	51	}
	52
	53	u32 ThreadArgRetval::BeforeJoin(uptr thread) const {
	54	__sanitizer::Lock lock(&mtx_);
	55	auto t = data_.find(thread);
	56	CHECK(t);
	57	CHECK(!t->second.detached);
	58	return t->second.gen;
	59	}
	60
	61	void ThreadArgRetval::AfterJoin(uptr thread, u32 gen) {
	62	__sanitizer::Lock lock(&mtx_);
	63	auto t = data_.find(thread);
	64	if (!t \|\| gen != t->second.gen) {
	65	// Thread was reused and erased by any other event.
	66	return;
	67	}
	68	CHECK(!t->second.detached);
	69	data_.erase(t);
	70	}
	71
	72	void ThreadArgRetval::DetachLocked(uptr thread) {
	73	CheckLocked();
	74	auto t = data_.find(thread);
	75	CHECK(t);
	76	CHECK(!t->second.detached);
	77	if (t->second.done) {
	78	// We can't retrive retval after detached thread finished.
	79	data_.erase(t);
	80	return;
	81	}
	82	t->second.detached = true;
	83	}
	84
	85	void ThreadArgRetval::GetAllPtrsLocked(InternalMmapVector<uptr>* ptrs) {
	86	CheckLocked();
	87	CHECK(ptrs);
	88	data_.forEach([&](DenseMap<uptr, Data>::value_type& kv) -> bool {
	89	ptrs->push_back((uptr)kv.second.args.arg_retval);
	90	return true;
	91	});
	92	}
	93
	94	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_thread_arg_retval.h created+116

...	...	@@ -0,0 +1,116 @@
	1	//===-- sanitizer_thread_arg_retval.h ---------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is shared between sanitizer tools.
	10	//
	11	// Tracks thread arguments and return value for leak checking.
	12	//===----------------------------------------------------------------------===//
	13
	14	#ifndef SANITIZER_THREAD_ARG_RETVAL_H
	15	#define SANITIZER_THREAD_ARG_RETVAL_H
	16
	17	#include "sanitizer_common.h"
	18	#include "sanitizer_dense_map.h"
	19	#include "sanitizer_list.h"
	20	#include "sanitizer_mutex.h"
	21
	22	namespace __sanitizer {
	23
	24	// Primary goal of the class is to keep alive arg and retval pointer for leak
	25	// checking. However it can be used to pass those pointer into wrappers used by
	26	// interceptors. The difference from ThreadRegistry/ThreadList is that this
	27	// class keeps data up to the detach or join, as exited thread still can be
	28	// joined to retrive retval. ThreadRegistry/ThreadList can discard exited
	29	// threads immediately.
	30	class SANITIZER_MUTEX ThreadArgRetval {
	31	public:
	32	struct Args {
	33	void* (routine)(void);
	34	void* arg_retval; // Either arg or retval.
	35	};
	36	void Lock() SANITIZER_ACQUIRE() { mtx_.Lock(); }
	37	void CheckLocked() const SANITIZER_CHECK_LOCKED() { mtx_.CheckLocked(); }
	38	void Unlock() SANITIZER_RELEASE() { mtx_.Unlock(); }
	39
	40	// Wraps pthread_create or similar. We need to keep object locked, to
	41	// prevent child thread from proceeding without thread handle.
	42	template <typename CreateFn /* returns thread id on success, or 0 */>
	43	void Create(bool detached, const Args& args, const CreateFn& fn) {
	44	// No need to track detached threads with no args, but we will to do as it's
	45	// not expensive and less edge-cases.
	46	__sanitizer::Lock lock(&mtx_);
	47	if (uptr thread = fn())
	48	CreateLocked(thread, detached, args);
	49	}
	50
	51	// Returns thread arg and routine.
	52	Args GetArgs(uptr thread) const;
	53
	54	// Mark thread as done and stores retval or remove if detached. Should be
	55	// called by the thread.
	56	void Finish(uptr thread, void* retval);
	57
	58	// Mark thread as detached or remove if done.
	59	template <typename DetachFn /* returns true on success */>
	60	void Detach(uptr thread, const DetachFn& fn) {
	61	// Lock to prevent re-use of the thread between fn() and DetachLocked()
	62	// calls.
	63	__sanitizer::Lock lock(&mtx_);
	64	if (fn())
	65	DetachLocked(thread);
	66	}
	67
	68	// Joins the thread.
	69	template <typename JoinFn /* returns true on success */>
	70	void Join(uptr thread, const JoinFn& fn) {
	71	// Remember internal id of the thread to prevent re-use of the thread
	72	// between fn() and AfterJoin() calls. Locking JoinFn, like in
	73	// Detach(), implementation can cause deadlock.
	74	auto gen = BeforeJoin(thread);
	75	if (fn())
	76	AfterJoin(thread, gen);
	77	}
	78
	79	// Returns all arg and retval which are considered alive.
	80	void GetAllPtrsLocked(InternalMmapVector<uptr>* ptrs);
	81
	82	uptr size() const {
	83	__sanitizer::Lock lock(&mtx_);
	84	return data_.size();
	85	}
	86
	87	// FIXME: Add fork support. Expected users of the class are sloppy with forks
	88	// anyway. We likely should lock/unlock the object to avoid deadlocks, and
	89	// erase all but the current threads, so we can detect leaked arg or retval in
	90	// child process.
	91
	92	// FIXME: Add cancelation support. Now if a thread was canceled, the class
	93	// will keep pointers alive forever, missing leaks caused by cancelation.
	94
	95	private:
	96	struct Data {
	97	Args args;
	98	u32 gen; // Avoid collision if thread id re-used.
	99	bool detached;
	100	bool done;
	101	};
	102
	103	void CreateLocked(uptr thread, bool detached, const Args& args);
	104	u32 BeforeJoin(uptr thread) const;
	105	void AfterJoin(uptr thread, u32 gen);
	106	void DetachLocked(uptr thread);
	107
	108	mutable Mutex mtx_;
	109
	110	DenseMap<uptr, Data> data_;
	111	u32 gen_ = 0;
	112	};
	113
	114	} // namespace __sanitizer
	115
	116	#endif // SANITIZER_THREAD_ARG_RETVAL_H

lib/tsan/sanitizer_common/sanitizer_thread_registry.cpp+59-21

...	...	@@ -13,6 +13,8 @@
13	13
14	14	#include "sanitizer_thread_registry.h"
15	15
	16	#include "sanitizer_placement_new.h"
	17
16	18	namespace __sanitizer {
17	19
18	20	ThreadContextBase::ThreadContextBase(u32 tid)
...	...	@@ -108,7 +110,7 @@ ThreadRegistry::ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
108	110	max_threads_(max_threads),
109	111	thread_quarantine_size_(thread_quarantine_size),
110	112	max_reuse_(max_reuse),
111		mtx_(),
	113	mtx_(MutexThreadRegistry),
112	114	total_threads_(0),
113	115	alive_threads_(0),
114	116	max_alive_threads_(0),
...	...	@@ -119,7 +121,7 @@ ThreadRegistry::ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
119	121
120	122	void ThreadRegistry::GetNumberOfThreads(uptr total, uptr running,
121	123	uptr *alive) {
122		BlockingMutexLock l(&mtx_);
	124	ThreadRegistryLock l(this);
123	125	if (total)
124	126	*total = threads_.size();
125	127	if (running) *running = running_threads_;
...	...	@@ -127,13 +129,13 @@ void ThreadRegistry::GetNumberOfThreads(uptr total, uptr running,
127	129	}
128	130
129	131	uptr ThreadRegistry::GetMaxAliveThreads() {
130		BlockingMutexLock l(&mtx_);
	132	ThreadRegistryLock l(this);
131	133	return max_alive_threads_;
132	134	}
133	135
134	136	u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
135	137	void *arg) {
136		BlockingMutexLock l(&mtx_);
	138	ThreadRegistryLock l(this);
137	139	u32 tid = kInvalidTid;
138	140	ThreadContextBase *tctx = QuarantinePop();
139	141	if (tctx) {
...	...	@@ -162,6 +164,12 @@ u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
162	164	max_alive_threads_++;
163	165	CHECK_EQ(alive_threads_, max_alive_threads_);
164	166	}
	167	if (user_id) {
	168	// Ensure that user_id is unique. If it's not the case we are screwed.
	169	// Ignoring this situation may lead to very hard to debug false
	170	// positives later (e.g. if we join a wrong thread).
	171	CHECK(live_.try_emplace(user_id, tid).second);
	172	}
165	173	tctx->SetCreated(user_id, total_threads_++, detached,
166	174	parent_tid, arg);
167	175	return tid;
...	...	@@ -179,7 +187,7 @@ void ThreadRegistry::RunCallbackForEachThreadLocked(ThreadCallback cb,
179	187	}
180	188
181	189	u32 ThreadRegistry::FindThread(FindThreadCallback cb, void *arg) {
182		BlockingMutexLock l(&mtx_);
	190	ThreadRegistryLock l(this);
183	191	for (u32 tid = 0; tid < threads_.size(); tid++) {
184	192	ThreadContextBase *tctx = threads_[tid];
185	193	if (tctx != 0 && cb(tctx, arg))
...	...	@@ -211,7 +219,7 @@ ThreadContextBase *ThreadRegistry::FindThreadContextByOsIDLocked(tid_t os_id) {
211	219	}
212	220
213	221	void ThreadRegistry::SetThreadName(u32 tid, const char *name) {
214		BlockingMutexLock l(&mtx_);
	222	ThreadRegistryLock l(this);
215	223	ThreadContextBase *tctx = threads_[tid];
216	224	CHECK_NE(tctx, 0);
217	225	CHECK_EQ(SANITIZER_FUCHSIA ? ThreadStatusCreated : ThreadStatusRunning,
...	...	@@ -220,19 +228,13 @@ void ThreadRegistry::SetThreadName(u32 tid, const char *name) {
220	228	}
221	229
222	230	void ThreadRegistry::SetThreadNameByUserId(uptr user_id, const char *name) {
223		BlockingMutexLock l(&mtx_);
224		for (u32 tid = 0; tid < threads_.size(); tid++) {
225		ThreadContextBase *tctx = threads_[tid];
226		if (tctx != 0 && tctx->user_id == user_id &&
227		tctx->status != ThreadStatusInvalid) {
228		tctx->SetName(name);
229		return;
230		}
231		}
	231	ThreadRegistryLock l(this);
	232	if (const auto *tid = live_.find(user_id))
	233	threads_[tid->second]->SetName(name);
232	234	}
233	235
234	236	void ThreadRegistry::DetachThread(u32 tid, void *arg) {
235		BlockingMutexLock l(&mtx_);
	237	ThreadRegistryLock l(this);
236	238	ThreadContextBase *tctx = threads_[tid];
237	239	CHECK_NE(tctx, 0);
238	240	if (tctx->status == ThreadStatusInvalid) {
...	...	@@ -241,6 +243,8 @@ void ThreadRegistry::DetachThread(u32 tid, void *arg) {
241	243	}
242	244	tctx->OnDetached(arg);
243	245	if (tctx->status == ThreadStatusFinished) {
	246	if (tctx->user_id)
	247	live_.erase(tctx->user_id);
244	248	tctx->SetDead();
245	249	QuarantinePush(tctx);
246	250	} else {
...	...	@@ -252,7 +256,7 @@ void ThreadRegistry::JoinThread(u32 tid, void *arg) {
252	256	bool destroyed = false;
253	257	do {
254	258	{
255		BlockingMutexLock l(&mtx_);
	259	ThreadRegistryLock l(this);
256	260	ThreadContextBase *tctx = threads_[tid];
257	261	CHECK_NE(tctx, 0);
258	262	if (tctx->status == ThreadStatusInvalid) {
...	...	@@ -260,6 +264,8 @@ void ThreadRegistry::JoinThread(u32 tid, void *arg) {
260	264	return;
261	265	}
262	266	if ((destroyed = tctx->GetDestroyed())) {
	267	if (tctx->user_id)
	268	live_.erase(tctx->user_id);
263	269	tctx->SetJoined(arg);
264	270	QuarantinePush(tctx);
265	271	}
...	...	@@ -275,7 +281,7 @@ void ThreadRegistry::JoinThread(u32 tid, void *arg) {
275	281	// thread before trying to create it, and then failed to actually
276	282	// create it, and so never called StartThread.
277	283	ThreadStatus ThreadRegistry::FinishThread(u32 tid) {
278		BlockingMutexLock l(&mtx_);
	284	ThreadRegistryLock l(this);
279	285	CHECK_GT(alive_threads_, 0);
280	286	alive_threads_--;
281	287	ThreadContextBase *tctx = threads_[tid];
...	...	@@ -292,6 +298,8 @@ ThreadStatus ThreadRegistry::FinishThread(u32 tid) {
292	298	}
293	299	tctx->SetFinished();
294	300	if (dead) {
	301	if (tctx->user_id)
	302	live_.erase(tctx->user_id);
295	303	tctx->SetDead();
296	304	QuarantinePush(tctx);
297	305	}
...	...	@@ -301,7 +309,7 @@ ThreadStatus ThreadRegistry::FinishThread(u32 tid) {
301	309
302	310	void ThreadRegistry::StartThread(u32 tid, tid_t os_id, ThreadType thread_type,
303	311	void *arg) {
304		BlockingMutexLock l(&mtx_);
	312	ThreadRegistryLock l(this);
305	313	running_threads_++;
306	314	ThreadContextBase *tctx = threads_[tid];
307	315	CHECK_NE(tctx, 0);
...	...	@@ -327,20 +335,50 @@ void ThreadRegistry::QuarantinePush(ThreadContextBase *tctx) {
327	335
328	336	ThreadContextBase *ThreadRegistry::QuarantinePop() {
329	337	if (invalid_threads_.size() == 0)
330		return 0;
	338	return nullptr;
331	339	ThreadContextBase *tctx = invalid_threads_.front();
332	340	invalid_threads_.pop_front();
333	341	return tctx;
334	342	}
335	343
	344	u32 ThreadRegistry::ConsumeThreadUserId(uptr user_id) {
	345	ThreadRegistryLock l(this);
	346	u32 tid;
	347	auto *t = live_.find(user_id);
	348	CHECK(t);
	349	tid = t->second;
	350	live_.erase(t);
	351	auto *tctx = threads_[tid];
	352	CHECK_EQ(tctx->user_id, user_id);
	353	tctx->user_id = 0;
	354	return tid;
	355	}
	356
336	357	void ThreadRegistry::SetThreadUserId(u32 tid, uptr user_id) {
337		BlockingMutexLock l(&mtx_);
	358	ThreadRegistryLock l(this);
338	359	ThreadContextBase *tctx = threads_[tid];
339	360	CHECK_NE(tctx, 0);
340	361	CHECK_NE(tctx->status, ThreadStatusInvalid);
341	362	CHECK_NE(tctx->status, ThreadStatusDead);
342	363	CHECK_EQ(tctx->user_id, 0);
343	364	tctx->user_id = user_id;
	365	CHECK(live_.try_emplace(user_id, tctx->tid).second);
	366	}
	367
	368	u32 ThreadRegistry::OnFork(u32 tid) {
	369	ThreadRegistryLock l(this);
	370	// We only purge user_id (pthread_t) of live threads because
	371	// they cause CHECK failures if new threads with matching pthread_t
	372	// created after fork.
	373	// Potentially we could purge more info (ThreadContextBase themselves),
	374	// but it's hard to test and easy to introduce new issues by doing this.
	375	for (auto *tctx : threads_) {
	376	if (tctx->tid == tid \|\| !tctx->user_id)
	377	continue;
	378	CHECK(live_.erase(tctx->user_id));
	379	tctx->user_id = 0;
	380	}
	381	return alive_threads_;
344	382	}
345	383
346	384	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_thread_registry.h+15-5

...	...	@@ -15,6 +15,7 @@
15	15	#define SANITIZER_THREAD_REGISTRY_H
16	16
17	17	#include "sanitizer_common.h"
	18	#include "sanitizer_dense_map.h"
18	19	#include "sanitizer_list.h"
19	20	#include "sanitizer_mutex.h"
20	21
...	...	@@ -85,7 +86,7 @@ class ThreadContextBase {
85	86
86	87	typedef ThreadContextBase* (*ThreadContextFactory)(u32 tid);
87	88
88		class MUTEX ThreadRegistry {
	89	class SANITIZER_MUTEX ThreadRegistry {
89	90	public:
90	91	ThreadRegistry(ThreadContextFactory factory);
91	92	ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
...	...	@@ -94,15 +95,17 @@ class MUTEX ThreadRegistry {
94	95	uptr *alive = nullptr);
95	96	uptr GetMaxAliveThreads();
96	97
97		void Lock() ACQUIRE() { mtx_.Lock(); }
98		void CheckLocked() const CHECK_LOCKED() { mtx_.CheckLocked(); }
99		void Unlock() RELEASE() { mtx_.Unlock(); }
	98	void Lock() SANITIZER_ACQUIRE() { mtx_.Lock(); }
	99	void CheckLocked() const SANITIZER_CHECK_LOCKED() { mtx_.CheckLocked(); }
	100	void Unlock() SANITIZER_RELEASE() { mtx_.Unlock(); }
100	101
101	102	// Should be guarded by ThreadRegistryLock.
102	103	ThreadContextBase *GetThreadLocked(u32 tid) {
103	104	return threads_.empty() ? nullptr : threads_[tid];
104	105	}
105	106
	107	u32 NumThreadsLocked() const { return threads_.size(); }
	108
106	109	u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg);
107	110
108	111	typedef void (ThreadCallback)(ThreadContextBase tctx, void *arg);
...	...	@@ -127,15 +130,21 @@ class MUTEX ThreadRegistry {
127	130	// Finishes thread and returns previous status.
128	131	ThreadStatus FinishThread(u32 tid);
129	132	void StartThread(u32 tid, tid_t os_id, ThreadType thread_type, void *arg);
	133	u32 ConsumeThreadUserId(uptr user_id);
130	134	void SetThreadUserId(u32 tid, uptr user_id);
131	135
	136	// OnFork must be called in the child process after fork to purge old
	137	// threads that don't exist anymore (except for the current thread tid).
	138	// Returns number of alive threads before fork.
	139	u32 OnFork(u32 tid);
	140
132	141	private:
133	142	const ThreadContextFactory context_factory_;
134	143	const u32 max_threads_;
135	144	const u32 thread_quarantine_size_;
136	145	const u32 max_reuse_;
137	146
138		BlockingMutex mtx_;
	147	Mutex mtx_;
139	148
140	149	u64 total_threads_; // Total number of created threads. May be greater than
141	150	// max_threads_ if contexts were reused.
...	...	@@ -146,6 +155,7 @@ class MUTEX ThreadRegistry {
146	155	InternalMmapVector<ThreadContextBase *> threads_;
147	156	IntrusiveList<ThreadContextBase> dead_threads_;
148	157	IntrusiveList<ThreadContextBase> invalid_threads_;
	158	DenseMap<uptr, Tid> live_;
149	159
150	160	void QuarantinePush(ThreadContextBase *tctx);
151	161	ThreadContextBase *QuarantinePop();

lib/tsan/sanitizer_common/sanitizer_thread_safety.h+26-19

...	...	@@ -16,27 +16,34 @@
16	16	#define SANITIZER_THREAD_SAFETY_H
17	17
18	18	#if defined(__clang__)
19		# define THREAD_ANNOTATION(x) __attribute__((x))
	19	# define SANITIZER_THREAD_ANNOTATION(x) __attribute__((x))
20	20	#else
21		# define THREAD_ANNOTATION(x)
	21	# define SANITIZER_THREAD_ANNOTATION(x)
22	22	#endif
23	23
24		#define MUTEX THREAD_ANNOTATION(capability("mutex"))
25		#define SCOPED_LOCK THREAD_ANNOTATION(scoped_lockable)
26		#define GUARDED_BY(x) THREAD_ANNOTATION(guarded_by(x))
27		#define PT_GUARDED_BY(x) THREAD_ANNOTATION(pt_guarded_by(x))
28		#define REQUIRES(...) THREAD_ANNOTATION(requires_capability(__VA_ARGS__))
29		#define REQUIRES_SHARED(...) \
30		THREAD_ANNOTATION(requires_shared_capability(__VA_ARGS__))
31		#define ACQUIRE(...) THREAD_ANNOTATION(acquire_capability(__VA_ARGS__))
32		#define ACQUIRE_SHARED(...) \
33		THREAD_ANNOTATION(acquire_shared_capability(__VA_ARGS__))
34		#define TRY_ACQUIRE(...) THREAD_ANNOTATION(try_acquire_capability(__VA_ARGS__))
35		#define RELEASE(...) THREAD_ANNOTATION(release_capability(__VA_ARGS__))
36		#define RELEASE_SHARED(...) \
37		THREAD_ANNOTATION(release_shared_capability(__VA_ARGS__))
38		#define EXCLUDES(...) THREAD_ANNOTATION(locks_excluded(__VA_ARGS__))
39		#define CHECK_LOCKED(...) THREAD_ANNOTATION(assert_capability(__VA_ARGS__))
40		#define NO_THREAD_SAFETY_ANALYSIS THREAD_ANNOTATION(no_thread_safety_analysis)
	24	#define SANITIZER_MUTEX SANITIZER_THREAD_ANNOTATION(capability("mutex"))
	25	#define SANITIZER_SCOPED_LOCK SANITIZER_THREAD_ANNOTATION(scoped_lockable)
	26	#define SANITIZER_GUARDED_BY(x) SANITIZER_THREAD_ANNOTATION(guarded_by(x))
	27	#define SANITIZER_PT_GUARDED_BY(x) SANITIZER_THREAD_ANNOTATION(pt_guarded_by(x))
	28	#define SANITIZER_REQUIRES(...) \
	29	SANITIZER_THREAD_ANNOTATION(requires_capability(__VA_ARGS__))
	30	#define SANITIZER_REQUIRES_SHARED(...) \
	31	SANITIZER_THREAD_ANNOTATION(requires_shared_capability(__VA_ARGS__))
	32	#define SANITIZER_ACQUIRE(...) \
	33	SANITIZER_THREAD_ANNOTATION(acquire_capability(__VA_ARGS__))
	34	#define SANITIZER_ACQUIRE_SHARED(...) \
	35	SANITIZER_THREAD_ANNOTATION(acquire_shared_capability(__VA_ARGS__))
	36	#define SANITIZER_TRY_ACQUIRE(...) \
	37	SANITIZER_THREAD_ANNOTATION(try_acquire_capability(__VA_ARGS__))
	38	#define SANITIZER_RELEASE(...) \
	39	SANITIZER_THREAD_ANNOTATION(release_capability(__VA_ARGS__))
	40	#define SANITIZER_RELEASE_SHARED(...) \
	41	SANITIZER_THREAD_ANNOTATION(release_shared_capability(__VA_ARGS__))
	42	#define SANITIZER_EXCLUDES(...) \
	43	SANITIZER_THREAD_ANNOTATION(locks_excluded(__VA_ARGS__))
	44	#define SANITIZER_CHECK_LOCKED(...) \
	45	SANITIZER_THREAD_ANNOTATION(assert_capability(__VA_ARGS__))
	46	#define SANITIZER_NO_THREAD_SAFETY_ANALYSIS \
	47	SANITIZER_THREAD_ANNOTATION(no_thread_safety_analysis)
41	48
42	49	#endif

lib/tsan/sanitizer_common/sanitizer_tls_get_addr.cpp+28-25

...	...	@@ -12,6 +12,7 @@
12	12
13	13	#include "sanitizer_tls_get_addr.h"
14	14
	15	#include "sanitizer_allocator_interface.h"
15	16	#include "sanitizer_atomic.h"
16	17	#include "sanitizer_flags.h"
17	18	#include "sanitizer_platform_interceptors.h"
...	...	@@ -26,13 +27,6 @@ struct TlsGetAddrParam {
26	27	uptr offset;
27	28	};
28	29
29		// Glibc starting from 2.19 allocates tls using __signal_safe_memalign,
30		// which has such header.
31		struct Glibc_2_19_tls_header {
32		uptr size;
33		uptr start;
34		};
35
36	30	// This must be static TLS
37	31	__attribute__((tls_model("initial-exec")))
38	32	static __thread DTLS dtls;
...	...	@@ -44,7 +38,7 @@ static atomic_uintptr_t number_of_live_dtls;
44	38	static const uptr kDestroyedThread = -1;
45	39
46	40	static void DTLS_Deallocate(DTLS::DTVBlock *block) {
47		VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", block);
	41	VReport(2, "__tls_get_addr: DTLS_Deallocate %p\n", (void *)block);
48	42	UnmapOrDie(block, sizeof(DTLS::DTVBlock));
49	43	atomic_fetch_sub(&number_of_live_dtls, 1, memory_order_relaxed);
50	44	}
...	...	@@ -66,12 +60,13 @@ static DTLS::DTVBlock DTLS_NextBlock(atomic_uintptr_t cur) {
66	60	}
67	61	uptr num_live_dtls =
68	62	atomic_fetch_add(&number_of_live_dtls, 1, memory_order_relaxed);
69		VReport(2, "__tls_get_addr: DTLS_NextBlock %p %zd\n", &dtls, num_live_dtls);
	63	VReport(2, "__tls_get_addr: DTLS_NextBlock %p %zd\n", (void *)&dtls,
	64	num_live_dtls);
70	65	return new_dtv;
71	66	}
72	67
73	68	static DTLS::DTV *DTLS_Find(uptr id) {
74		VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", &dtls, id);
	69	VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", (void *)&dtls, id);
75	70	static constexpr uptr kPerBlock = ARRAY_SIZE(DTLS::DTVBlock::dtvs);
76	71	DTLS::DTVBlock *cur = DTLS_NextBlock(&dtls.dtv_block);
77	72	if (!cur)
...	...	@@ -82,7 +77,7 @@ static DTLS::DTV *DTLS_Find(uptr id) {
82	77
83	78	void DTLS_Destroy() {
84	79	if (!common_flags()->intercept_tls_get_addr) return;
85		VReport(2, "__tls_get_addr: DTLS_Destroy %p\n", &dtls);
	80	VReport(2, "__tls_get_addr: DTLS_Destroy %p\n", (void *)&dtls);
86	81	DTLS::DTVBlock block = (DTLS::DTVBlock )atomic_exchange(
87	82	&dtls.dtv_block, kDestroyedThread, memory_order_release);
88	83	while (block) {
...	...	@@ -107,6 +102,14 @@ static const uptr kDtvOffset = 0x800;
107	102	static const uptr kDtvOffset = 0;
108	103	#endif
109	104
	105	extern "C" {
	106	SANITIZER_WEAK_ATTRIBUTE
	107	uptr __sanitizer_get_allocated_size(const void *p);
	108
	109	SANITIZER_WEAK_ATTRIBUTE
	110	const void __sanitizer_get_allocated_begin(const void p);
	111	}
	112
110	113	DTLS::DTV DTLS_on_tls_get_addr(void arg_void, void *res,
111	114	uptr static_tls_begin, uptr static_tls_end) {
112	115	if (!common_flags()->intercept_tls_get_addr) return 0;
...	...	@@ -117,26 +120,26 @@ DTLS::DTV DTLS_on_tls_get_addr(void arg_void, void *res,
117	120	return 0;
118	121	uptr tls_size = 0;
119	122	uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
120		VReport(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p "
121		"num_live_dtls %zd\n",
122		arg, arg->dso_id, arg->offset, res, tls_beg, &tls_beg,
	123	VReport(2,
	124	"__tls_get_addr: %p {0x%zx,0x%zx} => %p; tls_beg: 0x%zx; sp: %p "
	125	"num_live_dtls %zd\n",
	126	(void )arg, arg->dso_id, arg->offset, res, tls_beg, (void )&tls_beg,
123	127	atomic_load(&number_of_live_dtls, memory_order_relaxed));
124	128	if (dtls.last_memalign_ptr == tls_beg) {
125	129	tls_size = dtls.last_memalign_size;
126		VReport(2, "__tls_get_addr: glibc <=2.18 suspected; tls={%p,%p}\n",
127		tls_beg, tls_size);
	130	VReport(2, "__tls_get_addr: glibc <=2.24 suspected; tls={0x%zx,0x%zx}\n",
	131	tls_beg, tls_size);
128	132	} else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) {
129	133	// This is the static TLS block which was initialized / unpoisoned at thread
130	134	// creation.
131		VReport(2, "__tls_get_addr: static tls: %p\n", tls_beg);
	135	VReport(2, "__tls_get_addr: static tls: 0x%zx\n", tls_beg);
132	136	tls_size = 0;
133		} else if ((tls_beg % 4096) == sizeof(Glibc_2_19_tls_header)) {
134		// We may want to check gnu_get_libc_version().
135		Glibc_2_19_tls_header header = (Glibc_2_19_tls_header )tls_beg - 1;
136		tls_size = header->size;
137		tls_beg = header->start;
138		VReport(2, "__tls_get_addr: glibc >=2.19 suspected; tls={%p %p}\n",
139		tls_beg, tls_size);
	137	} else if (const void *start =
	138	__sanitizer_get_allocated_begin((void *)tls_beg)) {
	139	tls_beg = (uptr)start;
	140	tls_size = __sanitizer_get_allocated_size(start);
	141	VReport(2, "__tls_get_addr: glibc >=2.25 suspected; tls={0x%zx,0x%zx}\n",
	142	tls_beg, tls_size);
140	143	} else {
141	144	VReport(2, "__tls_get_addr: Can't guess glibc version\n");
142	145	// This may happen inside the DTOR of main thread, so just ignore it.
...	...	@@ -149,7 +152,7 @@ DTLS::DTV DTLS_on_tls_get_addr(void arg_void, void *res,
149	152
150	153	void DTLS_on_libc_memalign(void *ptr, uptr size) {
151	154	if (!common_flags()->intercept_tls_get_addr) return;
152		VReport(2, "DTLS_on_libc_memalign: %p %p\n", ptr, size);
	155	VReport(2, "DTLS_on_libc_memalign: %p 0x%zx\n", ptr, size);
153	156	dtls.last_memalign_ptr = reinterpret_cast<uptr>(ptr);
154	157	dtls.last_memalign_size = size;
155	158	}

lib/tsan/sanitizer_common/sanitizer_tls_get_addr.h+17-9

...	...	@@ -12,16 +12,24 @@
12	12	// the lack of interface that would tell us about the Dynamic TLS (DTLS).
13	13	// https://sourceware.org/bugzilla/show_bug.cgi?id=16291
14	14	//
15		// The matters get worse because the glibc implementation changed between
16		// 2.18 and 2.19:
17		// https://groups.google.com/forum/#!topic/address-sanitizer/BfwYD8HMxTM
18		//
19		// Before 2.19, every DTLS chunk is allocated with __libc_memalign,
	15	// Before 2.25: every DTLS chunk is allocated with __libc_memalign,
20	16	// which we intercept and thus know where is the DTLS.
21		// Since 2.19, DTLS chunks are allocated with __signal_safe_memalign,
22		// which is an internal function that wraps a mmap call, neither of which
23		// we can intercept. Luckily, __signal_safe_memalign has a simple parseable
24		// header which we can use.
	17	//
	18	// Since 2.25: DTLS chunks are allocated with malloc. We could co-opt
	19	// the malloc interceptor to keep track of the last allocation, similar
	20	// to how we handle __libc_memalign; however, this adds some overhead
	21	// (since malloc, unlike __libc_memalign, is commonly called), and
	22	// requires care to avoid false negatives for LeakSanitizer.
	23	// Instead, we rely on our internal allocators - which keep track of all
	24	// its allocations - to determine if an address points to a malloc
	25	// allocation.
	26	//
	27	// There exists a since-deprecated version of Google's internal glibc fork
	28	// that used __signal_safe_memalign. DTLS_on_tls_get_addr relied on a
	29	// heuristic check (is the allocation 16 bytes from the start of a page
	30	// boundary?), which was sometimes erroneous:
	31	// https://bugs.chromium.org/p/chromium/issues/detail?id=1275223#c15
	32	// Since that check has no practical use anymore, we have removed it.
25	33	//
26	34	//===----------------------------------------------------------------------===//
27	35

lib/tsan/sanitizer_common/sanitizer_type_traits.h+79

...	...	@@ -13,6 +13,8 @@
13	13	#ifndef SANITIZER_TYPE_TRAITS_H
14	14	#define SANITIZER_TYPE_TRAITS_H
15	15
	16	#include "sanitizer_common/sanitizer_internal_defs.h"
	17
16	18	namespace __sanitizer {
17	19
18	20	struct true_type {
...	...	@@ -57,6 +59,83 @@ struct conditional<false, T, F> {
57	59	using type = F;
58	60	};
59	61
	62	template <class T>
	63	struct remove_reference {
	64	using type = T;
	65	};
	66	template <class T>
	67	struct remove_reference<T&> {
	68	using type = T;
	69	};
	70	template <class T>
	71	struct remove_reference<T&&> {
	72	using type = T;
	73	};
	74
	75	template <class T>
	76	WARN_UNUSED_RESULT inline typename remove_reference<T>::type&& move(T&& t) {
	77	return static_cast<typename remove_reference<T>::type&&>(t);
	78	}
	79
	80	template <class T>
	81	WARN_UNUSED_RESULT inline constexpr T&& forward(
	82	typename remove_reference<T>::type& t) {
	83	return static_cast<T&&>(t);
	84	}
	85
	86	template <class T>
	87	WARN_UNUSED_RESULT inline constexpr T&& forward(
	88	typename remove_reference<T>::type&& t) {
	89	return static_cast<T&&>(t);
	90	}
	91
	92	template <class T, T v>
	93	struct integral_constant {
	94	static constexpr const T value = v;
	95	typedef T value_type;
	96	typedef integral_constant type;
	97	constexpr operator value_type() const { return value; }
	98	constexpr value_type operator()() const { return value; }
	99	};
	100
	101	#ifndef __has_builtin
	102	# define __has_builtin(x) 0
	103	#endif
	104
	105	#if __has_builtin(__is_trivially_destructible)
	106
	107	template <class T>
	108	struct is_trivially_destructible
	109	: public integral_constant<bool, __is_trivially_destructible(T)> {};
	110
	111	#elif __has_builtin(__has_trivial_destructor)
	112
	113	template <class T>
	114	struct is_trivially_destructible
	115	: public integral_constant<bool, __has_trivial_destructor(T)> {};
	116
	117	#else
	118
	119	template <class T>
	120	struct is_trivially_destructible
	121	: public integral_constant<bool, /* less efficient fallback */ false> {};
	122
	123	#endif
	124
	125	#if __has_builtin(__is_trivially_copyable)
	126
	127	template <class T>
	128	struct is_trivially_copyable
	129	: public integral_constant<bool, __is_trivially_copyable(T)> {};
	130
	131	#else
	132
	133	template <class T>
	134	struct is_trivially_copyable
	135	: public integral_constant<bool, /* less efficient fallback */ false> {};
	136
	137	#endif
	138
60	139	} // namespace __sanitizer
61	140
62	141	#endif

lib/tsan/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp+1-7

...	...	@@ -58,7 +58,7 @@ unwind_backtrace_signal_arch_func unwind_backtrace_signal_arch;
58	58	#endif
59	59
60	60	uptr Unwind_GetIP(struct _Unwind_Context *ctx) {
61		#if defined(__arm__) && !SANITIZER_MAC
	61	#if defined(__arm__) && !SANITIZER_APPLE
62	62	uptr val;
63	63	_Unwind_VRS_Result res = _Unwind_VRS_Get(ctx, _UVRSC_CORE,
64	64	15 /* r15 = PC */, _UVRSD_UINT32, &val);
...	...	@@ -139,13 +139,7 @@ void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
139	139	if (to_pop == 0 && size > 1)
140	140	to_pop = 1;
141	141	PopStackFrames(to_pop);
142		#if defined(__GNUC__) && defined(__sparc__)
143		// __builtin_return_address returns the address of the call instruction
144		// on the SPARC and not the return address, so we need to compensate.
145		trace_buffer[0] = GetNextInstructionPc(pc);
146		#else
147	142	trace_buffer[0] = pc;
148		#endif
149	143	}
150	144
151	145	void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {

lib/tsan/sanitizer_common/sanitizer_unwind_win.cpp+17-10

...	...	@@ -57,30 +57,37 @@ void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
57	57	InitializeDbgHelpIfNeeded();
58	58
59	59	size = 0;
60		#if defined(_WIN64)
	60	# if SANITIZER_WINDOWS64
	61	# if SANITIZER_ARM64
	62	int machine_type = IMAGE_FILE_MACHINE_ARM64;
	63	stack_frame.AddrPC.Offset = ctx.Pc;
	64	stack_frame.AddrFrame.Offset = ctx.Fp;
	65	stack_frame.AddrStack.Offset = ctx.Sp;
	66	# else
61	67	int machine_type = IMAGE_FILE_MACHINE_AMD64;
62	68	stack_frame.AddrPC.Offset = ctx.Rip;
63	69	stack_frame.AddrFrame.Offset = ctx.Rbp;
64	70	stack_frame.AddrStack.Offset = ctx.Rsp;
65		#else
	71	# endif
	72	# else
66	73	int machine_type = IMAGE_FILE_MACHINE_I386;
67	74	stack_frame.AddrPC.Offset = ctx.Eip;
68	75	stack_frame.AddrFrame.Offset = ctx.Ebp;
69	76	stack_frame.AddrStack.Offset = ctx.Esp;
70		#endif
	77	# endif
71	78	stack_frame.AddrPC.Mode = AddrModeFlat;
72	79	stack_frame.AddrFrame.Mode = AddrModeFlat;
73	80	stack_frame.AddrStack.Mode = AddrModeFlat;
74	81	while (StackWalk64(machine_type, GetCurrentProcess(), GetCurrentThread(),
75		&stack_frame, &ctx, NULL, SymFunctionTableAccess64,
76		SymGetModuleBase64, NULL) &&
77		size < Min(max_depth, kStackTraceMax)) {
	82	&stack_frame, &ctx, NULL, SymFunctionTableAccess64,
	83	SymGetModuleBase64, NULL) &&
	84	size < Min(max_depth, kStackTraceMax)) {
78	85	trace_buffer[size++] = (uptr)stack_frame.AddrPC.Offset;
79	86	}
80	87	}
81		#ifdef __clang__
82		#pragma clang diagnostic pop
83		#endif
84		#endif // #if !SANITIZER_GO
	88	# ifdef __clang__
	89	# pragma clang diagnostic pop
	90	# endif
	91	# endif // #if !SANITIZER_GO
85	92
86	93	#endif // SANITIZER_WINDOWS

lib/tsan/sanitizer_common/sanitizer_vector.h+2-2

...	...	@@ -83,8 +83,8 @@ class Vector {
83	83	}
84	84	EnsureSize(size);
85	85	if (old_size < size) {
86		for (uptr i = old_size; i < size; i++)
87		internal_memset(&begin_[i], 0, sizeof(begin_[i]));
	86	internal_memset(&begin_[old_size], 0,
	87	sizeof(begin_[old_size]) * (size - old_size));
88	88	}
89	89	}
90	90

lib/tsan/sanitizer_common/sanitizer_win.cpp+62-36

...	...	@@ -93,6 +93,11 @@ bool FileExists(const char *filename) {
93	93	return ::GetFileAttributesA(filename) != INVALID_FILE_ATTRIBUTES;
94	94	}
95	95
	96	bool DirExists(const char *path) {
	97	auto attr = ::GetFileAttributesA(path);
	98	return (attr != INVALID_FILE_ATTRIBUTES) && (attr & FILE_ATTRIBUTE_DIRECTORY);
	99	}
	100
96	101	uptr internal_getpid() {
97	102	return GetProcessId(GetCurrentProcess());
98	103	}
...	...	@@ -126,6 +131,11 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
126	131	}
127	132	#endif // #if !SANITIZER_GO
128	133
	134	bool ErrorIsOOM(error_t err) {
	135	// TODO: This should check which `err`s correspond to OOM.
	136	return false;
	137	}
	138
129	139	void MmapOrDie(uptr size, const char mem_type, bool raw_report) {
130	140	void *rv = VirtualAlloc(0, size, MEM_RESERVE \| MEM_COMMIT, PAGE_READWRITE);
131	141	if (rv == 0)
...	...	@@ -224,6 +234,17 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
224	234	return (void *)mapped_addr;
225	235	}
226	236
	237	// ZeroMmapFixedRegion zero's out a region of memory previously returned from a
	238	// call to one of the MmapFixed* helpers. On non-windows systems this would be
	239	// done with another mmap, but on windows remapping is not an option.
	240	// VirtualFree(DECOMMIT)+VirtualAlloc(RECOMMIT) would also be a way to zero the
	241	// memory, but we can't do this atomically, so instead we fall back to using
	242	// internal_memset.
	243	bool ZeroMmapFixedRegion(uptr fixed_addr, uptr size) {
	244	internal_memset((void*) fixed_addr, 0, size);
	245	return true;
	246	}
	247
227	248	bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) {
228	249	// FIXME: is this really "NoReserve"? On Win32 this does not matter much,
229	250	// but on Win64 it does.
...	...	@@ -336,6 +357,16 @@ bool MprotectNoAccess(uptr addr, uptr size) {
336	357	return VirtualProtect((LPVOID)addr, size, PAGE_NOACCESS, &old_protection);
337	358	}
338	359
	360	bool MprotectReadOnly(uptr addr, uptr size) {
	361	DWORD old_protection;
	362	return VirtualProtect((LPVOID)addr, size, PAGE_READONLY, &old_protection);
	363	}
	364
	365	bool MprotectReadWrite(uptr addr, uptr size) {
	366	DWORD old_protection;
	367	return VirtualProtect((LPVOID)addr, size, PAGE_READWRITE, &old_protection);
	368	}
	369
339	370	void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
340	371	uptr beg_aligned = RoundDownTo(beg, GetPageSizeCached()),
341	372	end_aligned = RoundDownTo(end, GetPageSizeCached());
...	...	@@ -512,7 +543,7 @@ void ReExec() {
512	543	UNIMPLEMENTED();
513	544	}
514	545
515		void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {}
	546	void PlatformPrepareForSandboxing(void *args) {}
516	547
517	548	bool StackSizeIsUnlimited() {
518	549	UNIMPLEMENTED();
...	...	@@ -565,6 +596,10 @@ void Abort() {
565	596	internal__exit(3);
566	597	}
567	598
	599	bool CreateDir(const char *pathname) {
	600	return CreateDirectoryA(pathname, nullptr) != 0;
	601	}
	602
568	603	#if !SANITIZER_GO
569	604	// Read the file to extract the ImageBase field from the PE header. If ASLR is
570	605	// disabled and this virtual address is available, the loader will typically
...	...	@@ -688,13 +723,24 @@ void ListOfModules::fallbackInit() { clear(); }
688	723	// atexit() as soon as it is ready for use (i.e. after .CRT$XIC initializers).
689	724	InternalMmapVectorNoCtor<void (*)(void)> atexit_functions;
690	725
691		int Atexit(void (*function)(void)) {
	726	static int queueAtexit(void (*function)(void)) {
692	727	atexit_functions.push_back(function);
693	728	return 0;
694	729	}
695	730
	731	// If Atexit() is being called after RunAtexit() has already been run, it needs
	732	// to be able to call atexit() directly. Here we use a function ponter to
	733	// switch out its behaviour.
	734	// An example of where this is needed is the asan_dynamic runtime on MinGW-w64.
	735	// On this environment, __asan_init is called during global constructor phase,
	736	// way after calling the .CRT$XID initializer.
	737	static int (volatile queueOrCallAtExit)(void ()(void)) = &queueAtexit;
	738
	739	int Atexit(void (*function)(void)) { return queueOrCallAtExit(function); }
	740
696	741	static int RunAtexit() {
697	742	TraceLoggingUnregister(g_asan_provider);
	743	queueOrCallAtExit = &atexit;
698	744	int ret = 0;
699	745	for (uptr i = 0; i < atexit_functions.size(); ++i) {
700	746	ret \|= atexit(atexit_functions[i]);
...	...	@@ -827,27 +873,6 @@ void FutexWake(atomic_uint32_t *p, u32 count) {
827	873	WakeByAddressAll(p);
828	874	}
829	875
830		// ---------------------- BlockingMutex ---------------- {{{1
831
832		BlockingMutex::BlockingMutex() {
833		CHECK(sizeof(SRWLOCK) <= sizeof(opaque_storage_));
834		internal_memset(this, 0, sizeof(*this));
835		}
836
837		void BlockingMutex::Lock() {
838		AcquireSRWLockExclusive((PSRWLOCK)opaque_storage_);
839		CHECK_EQ(owner_, 0);
840		owner_ = GetThreadSelf();
841		}
842
843		void BlockingMutex::Unlock() {
844		CheckLocked();
845		owner_ = 0;
846		ReleaseSRWLockExclusive((PSRWLOCK)opaque_storage_);
847		}
848
849		void BlockingMutex::CheckLocked() const { CHECK_EQ(owner_, GetThreadSelf()); }
850
851	876	uptr GetTlsSize() {
852	877	return 0;
853	878	}
...	...	@@ -962,13 +987,18 @@ void SignalContext::InitPcSpBp() {
962	987	CONTEXT context_record = (CONTEXT )context;
963	988
964	989	pc = (uptr)exception_record->ExceptionAddress;
965		#ifdef _WIN64
	990	# if SANITIZER_WINDOWS64
	991	# if SANITIZER_ARM64
	992	bp = (uptr)context_record->Fp;
	993	sp = (uptr)context_record->Sp;
	994	# else
966	995	bp = (uptr)context_record->Rbp;
967	996	sp = (uptr)context_record->Rsp;
968		#else
	997	# endif
	998	# else
969	999	bp = (uptr)context_record->Ebp;
970	1000	sp = (uptr)context_record->Esp;
971		#endif
	1001	# endif
972	1002	}
973	1003
974	1004	uptr SignalContext::GetAddress() const {
...	...	@@ -990,7 +1020,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
990	1020
991	1021	// The write flag is only available for access violation exceptions.
992	1022	if (exception_record->ExceptionCode != EXCEPTION_ACCESS_VIOLATION)
993		return SignalContext::UNKNOWN;
	1023	return SignalContext::Unknown;
994	1024
995	1025	// The contents of this array are documented at
996	1026	// https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record
...	...	@@ -998,13 +1028,13 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
998	1028	// second element is the faulting address.
999	1029	switch (exception_record->ExceptionInformation[0]) {
1000	1030	case 0:
1001		return SignalContext::READ;
	1031	return SignalContext::Read;
1002	1032	case 1:
1003		return SignalContext::WRITE;
	1033	return SignalContext::Write;
1004	1034	case 8:
1005		return SignalContext::UNKNOWN;
	1035	return SignalContext::Unknown;
1006	1036	}
1007		return SignalContext::UNKNOWN;
	1037	return SignalContext::Unknown;
1008	1038	}
1009	1039
1010	1040	void SignalContext::DumpAllRegisters(void *context) {
...	...	@@ -1091,10 +1121,6 @@ void InitializePlatformEarly() {
1091	1121	// Do nothing.
1092	1122	}
1093	1123
1094		void MaybeReexec() {
1095		// No need to re-exec on Windows.
1096		}
1097
1098	1124	void CheckASLR() {
1099	1125	// Do nothing
1100	1126	}
...	...	@@ -1131,7 +1157,7 @@ bool IsProcessRunning(pid_t pid) {
1131	1157	int WaitForProcess(pid_t pid) { return -1; }
1132	1158
1133	1159	// FIXME implement on this platform.
1134		void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { }
	1160	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
1135	1161
1136	1162	void CheckNoDeepBind(const char *filename, int flag) {
1137	1163	// Do nothing.

lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.cpp created+101

...	...	@@ -0,0 +1,101 @@
	1	//===-- sanitizer_win_dll_thunk.cpp ---------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	// This file defines a family of thunks that should be statically linked into
	9	// the DLLs that have instrumentation in order to delegate the calls to the
	10	// shared runtime that lives in the main binary.
	11	// See https://github.com/google/sanitizers/issues/209 for the details.
	12	//===----------------------------------------------------------------------===//
	13
	14	#ifdef SANITIZER_DLL_THUNK
	15	#include "sanitizer_win_defs.h"
	16	#include "sanitizer_win_dll_thunk.h"
	17	#include "interception/interception.h"
	18
	19	extern "C" {
	20	void WINAPI GetModuleHandleA(const char module_name);
	21	void abort();
	22	}
	23
	24	namespace __sanitizer {
	25	uptr dllThunkGetRealAddrOrDie(const char *name) {
	26	uptr ret =
	27	__interception::InternalGetProcAddress((void *)GetModuleHandleA(0), name);
	28	if (!ret)
	29	abort();
	30	return ret;
	31	}
	32
	33	int dllThunkIntercept(const char* main_function, uptr dll_function) {
	34	uptr wrapper = dllThunkGetRealAddrOrDie(main_function);
	35	if (!__interception::OverrideFunction(dll_function, wrapper, 0))
	36	abort();
	37	return 0;
	38	}
	39
	40	int dllThunkInterceptWhenPossible(const char* main_function,
	41	const char* default_function, uptr dll_function) {
	42	uptr wrapper = __interception::InternalGetProcAddress(
	43	(void *)GetModuleHandleA(0), main_function);
	44	if (!wrapper)
	45	wrapper = dllThunkGetRealAddrOrDie(default_function);
	46	if (!__interception::OverrideFunction(dll_function, wrapper, 0))
	47	abort();
	48	return 0;
	49	}
	50	} // namespace __sanitizer
	51
	52	// Include Sanitizer Common interface.
	53	#define INTERFACE_FUNCTION(Name) INTERCEPT_SANITIZER_FUNCTION(Name)
	54	#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
	55	#include "sanitizer_common_interface.inc"
	56
	57	#pragma section(".DLLTH$A", read)
	58	#pragma section(".DLLTH$Z", read)
	59
	60	typedef void (*DllThunkCB)();
	61	extern "C" {
	62	__declspec(allocate(".DLLTH$A")) DllThunkCB __start_dll_thunk;
	63	__declspec(allocate(".DLLTH$Z")) DllThunkCB __stop_dll_thunk;
	64	}
	65
	66	// Disable compiler warnings that show up if we declare our own version
	67	// of a compiler intrinsic (e.g. strlen).
	68	#pragma warning(disable: 4391)
	69	#pragma warning(disable: 4392)
	70
	71	extern "C" int __dll_thunk_init() {
	72	static bool flag = false;
	73	// __dll_thunk_init is expected to be called by only one thread.
	74	if (flag) return 0;
	75	flag = true;
	76
	77	for (DllThunkCB *it = &__start_dll_thunk; it < &__stop_dll_thunk; ++it)
	78	if (*it)
	79	(*it)();
	80
	81	// In DLLs, the callbacks are expected to return 0,
	82	// otherwise CRT initialization fails.
	83	return 0;
	84	}
	85
	86	// We want to call dll_thunk_init before C/C++ initializers / constructors are
	87	// executed, otherwise functions like memset might be invoked.
	88	#pragma section(".CRT$XIB", long, read)
	89	__declspec(allocate(".CRT$XIB")) int (*__dll_thunk_preinit)() =
	90	__dll_thunk_init;
	91
	92	static void WINAPI dll_thunk_thread_init(void *mod, unsigned long reason,
	93	void *reserved) {
	94	if (reason == /DLL_PROCESS_ATTACH=/1) __dll_thunk_init();
	95	}
	96
	97	#pragma section(".CRT$XLAB", long, read)
	98	__declspec(allocate(".CRT$XLAB")) void (WINAPI __dll_thunk_tls_init)(void ,
	99	unsigned long, void *) = dll_thunk_thread_init;
	100
	101	#endif // SANITIZER_DLL_THUNK

lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.h+1-1

...	...	@@ -84,7 +84,7 @@ extern "C" int __dll_thunk_init();
84	84	// which isn't a big deal.
85	85	#define INTERCEPT_LIBRARY_FUNCTION(name) \
86	86	extern "C" void name(); \
87		INTERCEPT_OR_DIE(WRAPPER_NAME(name), name)
	87	INTERCEPT_OR_DIE(STRINGIFY(WRAP(name)), name)
88	88
89	89	// Use these macros for functions that could be called before __dll_thunk_init()
90	90	// is executed and don't lead to errors if defined (free, malloc, etc).

lib/tsan/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cpp created+26

...	...	@@ -0,0 +1,26 @@
	1	//===-- santizer_win_dynamic_runtime_thunk.cpp ----------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file defines things that need to be present in the application modules
	10	// to interact with Sanitizer Common, when it is included in a dll.
	11	//
	12	//===----------------------------------------------------------------------===//
	13	#ifdef SANITIZER_DYNAMIC_RUNTIME_THUNK
	14	#define SANITIZER_IMPORT_INTERFACE 1
	15	#include "sanitizer_win_defs.h"
	16	// Define weak alias for all weak functions imported from sanitizer common.
	17	#define INTERFACE_FUNCTION(Name)
	18	#define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name)
	19	#include "sanitizer_common_interface.inc"
	20	#endif // SANITIZER_DYNAMIC_RUNTIME_THUNK
	21
	22	namespace __sanitizer {
	23	// Add one, otherwise unused, external symbol to this object file so that the
	24	// Visual C++ linker includes it and reads the .drective section.
	25	void ForceWholeArchiveIncludeForSanitizerCommon() {}
	26	}

lib/tsan/sanitizer_common/sanitizer_win_weak_interception.cpp created+94

...	...	@@ -0,0 +1,94 @@
	1	//===-- sanitizer_win_weak_interception.cpp -------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	// This module should be included in the sanitizer when it is implemented as a
	9	// shared library on Windows (dll), in order to delegate the calls of weak
	10	// functions to the implementation in the main executable when a strong
	11	// definition is provided.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_platform.h"
	15	#if SANITIZER_WINDOWS && SANITIZER_DYNAMIC
	16	#include "sanitizer_win_weak_interception.h"
	17	#include "sanitizer_allocator_interface.h"
	18	#include "sanitizer_interface_internal.h"
	19	#include "sanitizer_win_defs.h"
	20	#include "interception/interception.h"
	21
	22	extern "C" {
	23	void WINAPI GetModuleHandleA(const char module_name);
	24	void abort();
	25	}
	26
	27	namespace __sanitizer {
	28	// Try to get a pointer to real_function in the main module and override
	29	// dll_function with that pointer. If the function isn't found, nothing changes.
	30	int interceptWhenPossible(uptr dll_function, const char *real_function) {
	31	uptr real = __interception::InternalGetProcAddress(
	32	(void *)GetModuleHandleA(0), real_function);
	33	if (real && !__interception::OverrideFunction((uptr)dll_function, real, 0))
	34	abort();
	35	return 0;
	36	}
	37	} // namespace __sanitizer
	38
	39	// Declare weak hooks.
	40	extern "C" {
	41	void __sanitizer_on_print(const char *str);
	42	void __sanitizer_weak_hook_memcmp(uptr called_pc, const void *s1,
	43	const void *s2, uptr n, int result);
	44	void __sanitizer_weak_hook_strcmp(uptr called_pc, const char *s1,
	45	const char *s2, int result);
	46	void __sanitizer_weak_hook_strncmp(uptr called_pc, const char *s1,
	47	const char *s2, uptr n, int result);
	48	void __sanitizer_weak_hook_strstr(uptr called_pc, const char *s1,
	49	const char s2, char result);
	50	}
	51
	52	// Include Sanitizer Common interface.
	53	#define INTERFACE_FUNCTION(Name)
	54	#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
	55	#include "sanitizer_common_interface.inc"
	56
	57	#pragma section(".WEAK$A", read)
	58	#pragma section(".WEAK$Z", read)
	59
	60	typedef void (*InterceptCB)();
	61	extern "C" {
	62	__declspec(allocate(".WEAK$A")) InterceptCB __start_weak_list;
	63	__declspec(allocate(".WEAK$Z")) InterceptCB __stop_weak_list;
	64	}
	65
	66	static int weak_intercept_init() {
	67	static bool flag = false;
	68	// weak_interception_init is expected to be called by only one thread.
	69	if (flag) return 0;
	70	flag = true;
	71
	72	for (InterceptCB *it = &__start_weak_list; it < &__stop_weak_list; ++it)
	73	if (*it)
	74	(*it)();
	75
	76	// In DLLs, the callbacks are expected to return 0,
	77	// otherwise CRT initialization fails.
	78	return 0;
	79	}
	80
	81	#pragma section(".CRT$XIB", long, read)
	82	__declspec(allocate(".CRT$XIB")) int (*__weak_intercept_preinit)() =
	83	weak_intercept_init;
	84
	85	static void WINAPI weak_intercept_thread_init(void *mod, unsigned long reason,
	86	void *reserved) {
	87	if (reason == /DLL_PROCESS_ATTACH=/1) weak_intercept_init();
	88	}
	89
	90	#pragma section(".CRT$XLAB", long, read)
	91	__declspec(allocate(".CRT$XLAB")) void(WINAPI *__weak_intercept_tls_init)(
	92	void , unsigned long, void ) = weak_intercept_thread_init;
	93
	94	#endif // SANITIZER_WINDOWS && SANITIZER_DYNAMIC

lib/tsan/tsan_clock.cpp deleted-625

...	...	@@ -1,625 +0,0 @@
1		//===-- tsan_clock.cpp ----------------------------------------------------===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// This file is a part of ThreadSanitizer (TSan), a race detector.
10		//
11		//===----------------------------------------------------------------------===//
12		#include "tsan_clock.h"
13		#include "tsan_rtl.h"
14		#include "sanitizer_common/sanitizer_placement_new.h"
15
16		// SyncClock and ThreadClock implement vector clocks for sync variables
17		// (mutexes, atomic variables, file descriptors, etc) and threads, respectively.
18		// ThreadClock contains fixed-size vector clock for maximum number of threads.
19		// SyncClock contains growable vector clock for currently necessary number of
20		// threads.
21		// Together they implement very simple model of operations, namely:
22		//
23		// void ThreadClock::acquire(const SyncClock *src) {
24		// for (int i = 0; i < kMaxThreads; i++)
25		// clock[i] = max(clock[i], src->clock[i]);
26		// }
27		//
28		// void ThreadClock::release(SyncClock *dst) const {
29		// for (int i = 0; i < kMaxThreads; i++)
30		// dst->clock[i] = max(dst->clock[i], clock[i]);
31		// }
32		//
33		// void ThreadClock::releaseStoreAcquire(SyncClock *sc) const {
34		// for (int i = 0; i < kMaxThreads; i++) {
35		// tmp = clock[i];
36		// clock[i] = max(clock[i], sc->clock[i]);
37		// sc->clock[i] = tmp;
38		// }
39		// }
40		//
41		// void ThreadClock::ReleaseStore(SyncClock *dst) const {
42		// for (int i = 0; i < kMaxThreads; i++)
43		// dst->clock[i] = clock[i];
44		// }
45		//
46		// void ThreadClock::acq_rel(SyncClock *dst) {
47		// acquire(dst);
48		// release(dst);
49		// }
50		//
51		// Conformance to this model is extensively verified in tsan_clock_test.cpp.
52		// However, the implementation is significantly more complex. The complexity
53		// allows to implement important classes of use cases in O(1) instead of O(N).
54		//
55		// The use cases are:
56		// 1. Singleton/once atomic that has a single release-store operation followed
57		// by zillions of acquire-loads (the acquire-load is O(1)).
58		// 2. Thread-local mutex (both lock and unlock can be O(1)).
59		// 3. Leaf mutex (unlock is O(1)).
60		// 4. A mutex shared by 2 threads (both lock and unlock can be O(1)).
61		// 5. An atomic with a single writer (writes can be O(1)).
62		// The implementation dynamically adopts to workload. So if an atomic is in
63		// read-only phase, these reads will be O(1); if it later switches to read/write
64		// phase, the implementation will correctly handle that by switching to O(N).
65		//
66		// Thread-safety note: all const operations on SyncClock's are conducted under
67		// a shared lock; all non-const operations on SyncClock's are conducted under
68		// an exclusive lock; ThreadClock's are private to respective threads and so
69		// do not need any protection.
70		//
71		// Description of SyncClock state:
72		// clk_ - variable size vector clock, low kClkBits hold timestamp,
73		// the remaining bits hold "acquired" flag (the actual value is thread's
74		// reused counter);
75		// if acquried == thr->reused_, then the respective thread has already
76		// acquired this clock (except possibly for dirty elements).
77		// dirty_ - holds up to two indeces in the vector clock that other threads
78		// need to acquire regardless of "acquired" flag value;
79		// release_store_tid_ - denotes that the clock state is a result of
80		// release-store operation by the thread with release_store_tid_ index.
81		// release_store_reused_ - reuse count of release_store_tid_.
82
83		namespace __tsan {
84
85		static atomic_uint32_t ref_ptr(ClockBlock cb) {
86		return reinterpret_cast<atomic_uint32_t *>(&cb->table[ClockBlock::kRefIdx]);
87		}
88
89		// Drop reference to the first level block idx.
90		static void UnrefClockBlock(ClockCache *c, u32 idx, uptr blocks) {
91		ClockBlock *cb = ctx->clock_alloc.Map(idx);
92		atomic_uint32_t *ref = ref_ptr(cb);
93		u32 v = atomic_load(ref, memory_order_acquire);
94		for (;;) {
95		CHECK_GT(v, 0);
96		if (v == 1)
97		break;
98		if (atomic_compare_exchange_strong(ref, &v, v - 1, memory_order_acq_rel))
99		return;
100		}
101		// First level block owns second level blocks, so them as well.
102		for (uptr i = 0; i < blocks; i++)
103		ctx->clock_alloc.Free(c, cb->table[ClockBlock::kBlockIdx - i]);
104		ctx->clock_alloc.Free(c, idx);
105		}
106
107		ThreadClock::ThreadClock(unsigned tid, unsigned reused)
108		: tid_(tid)
109		, reused_(reused + 1) // 0 has special meaning
110		, last_acquire_()
111		, global_acquire_()
112		, cached_idx_()
113		, cached_size_()
114		, cached_blocks_() {
115		CHECK_LT(tid, kMaxTidInClock);
116		CHECK_EQ(reused_, ((u64)reused_ << kClkBits) >> kClkBits);
117		nclk_ = tid_ + 1;
118		internal_memset(clk_, 0, sizeof(clk_));
119		}
120
121		void ThreadClock::ResetCached(ClockCache *c) {
122		if (cached_idx_) {
123		UnrefClockBlock(c, cached_idx_, cached_blocks_);
124		cached_idx_ = 0;
125		cached_size_ = 0;
126		cached_blocks_ = 0;
127		}
128		}
129
130		void ThreadClock::acquire(ClockCache c, SyncClock src) {
131		DCHECK_LE(nclk_, kMaxTid);
132		DCHECK_LE(src->size_, kMaxTid);
133
134		// Check if it's empty -> no need to do anything.
135		const uptr nclk = src->size_;
136		if (nclk == 0)
137		return;
138
139		bool acquired = false;
140		for (unsigned i = 0; i < kDirtyTids; i++) {
141		SyncClock::Dirty dirty = src->dirty_[i];
142		unsigned tid = dirty.tid();
143		if (tid != kInvalidTid) {
144		if (clk_[tid] < dirty.epoch) {
145		clk_[tid] = dirty.epoch;
146		acquired = true;
147		}
148		}
149		}
150
151		// Check if we've already acquired src after the last release operation on src
152		if (tid_ >= nclk \|\| src->elem(tid_).reused != reused_) {
153		// O(N) acquire.
154		nclk_ = max(nclk_, nclk);
155		u64 *dst_pos = &clk_[0];
156		for (ClockElem &src_elem : *src) {
157		u64 epoch = src_elem.epoch;
158		if (*dst_pos < epoch) {
159		*dst_pos = epoch;
160		acquired = true;
161		}
162		dst_pos++;
163		}
164
165		// Remember that this thread has acquired this clock.
166		if (nclk > tid_)
167		src->elem(tid_).reused = reused_;
168		}
169
170		if (acquired) {
171		last_acquire_ = clk_[tid_];
172		ResetCached(c);
173		}
174		}
175
176		void ThreadClock::releaseStoreAcquire(ClockCache c, SyncClock sc) {
177		DCHECK_LE(nclk_, kMaxTid);
178		DCHECK_LE(sc->size_, kMaxTid);
179
180		if (sc->size_ == 0) {
181		// ReleaseStore will correctly set release_store_tid_,
182		// which can be important for future operations.
183		ReleaseStore(c, sc);
184		return;
185		}
186
187		nclk_ = max(nclk_, (uptr) sc->size_);
188
189		// Check if we need to resize sc.
190		if (sc->size_ < nclk_)
191		sc->Resize(c, nclk_);
192
193		bool acquired = false;
194
195		sc->Unshare(c);
196		// Update sc->clk_.
197		sc->FlushDirty();
198		uptr i = 0;
199		for (ClockElem &ce : *sc) {
200		u64 tmp = clk_[i];
201		if (clk_[i] < ce.epoch) {
202		clk_[i] = ce.epoch;
203		acquired = true;
204		}
205		ce.epoch = tmp;
206		ce.reused = 0;
207		i++;
208		}
209		sc->release_store_tid_ = kInvalidTid;
210		sc->release_store_reused_ = 0;
211
212		if (acquired) {
213		last_acquire_ = clk_[tid_];
214		ResetCached(c);
215		}
216		}
217
218		void ThreadClock::release(ClockCache c, SyncClock dst) {
219		DCHECK_LE(nclk_, kMaxTid);
220		DCHECK_LE(dst->size_, kMaxTid);
221
222		if (dst->size_ == 0) {
223		// ReleaseStore will correctly set release_store_tid_,
224		// which can be important for future operations.
225		ReleaseStore(c, dst);
226		return;
227		}
228
229		// Check if we need to resize dst.
230		if (dst->size_ < nclk_)
231		dst->Resize(c, nclk_);
232
233		// Check if we had not acquired anything from other threads
234		// since the last release on dst. If so, we need to update
235		// only dst->elem(tid_).
236		if (!HasAcquiredAfterRelease(dst)) {
237		UpdateCurrentThread(c, dst);
238		if (dst->release_store_tid_ != tid_ \|\|
239		dst->release_store_reused_ != reused_)
240		dst->release_store_tid_ = kInvalidTid;
241		return;
242		}
243
244		// O(N) release.
245		dst->Unshare(c);
246		// First, remember whether we've acquired dst.
247		bool acquired = IsAlreadyAcquired(dst);
248		// Update dst->clk_.
249		dst->FlushDirty();
250		uptr i = 0;
251		for (ClockElem &ce : *dst) {
252		ce.epoch = max(ce.epoch, clk_[i]);
253		ce.reused = 0;
254		i++;
255		}
256		// Clear 'acquired' flag in the remaining elements.
257		dst->release_store_tid_ = kInvalidTid;
258		dst->release_store_reused_ = 0;
259		// If we've acquired dst, remember this fact,
260		// so that we don't need to acquire it on next acquire.
261		if (acquired)
262		dst->elem(tid_).reused = reused_;
263		}
264
265		void ThreadClock::ReleaseStore(ClockCache c, SyncClock dst) {
266		DCHECK_LE(nclk_, kMaxTid);
267		DCHECK_LE(dst->size_, kMaxTid);
268
269		if (dst->size_ == 0 && cached_idx_ != 0) {
270		// Reuse the cached clock.
271		// Note: we could reuse/cache the cached clock in more cases:
272		// we could update the existing clock and cache it, or replace it with the
273		// currently cached clock and release the old one. And for a shared
274		// existing clock, we could replace it with the currently cached;
275		// or unshare, update and cache. But, for simplicity, we currnetly reuse
276		// cached clock only when the target clock is empty.
277		dst->tab_ = ctx->clock_alloc.Map(cached_idx_);
278		dst->tab_idx_ = cached_idx_;
279		dst->size_ = cached_size_;
280		dst->blocks_ = cached_blocks_;
281		CHECK_EQ(dst->dirty_[0].tid(), kInvalidTid);
282		// The cached clock is shared (immutable),
283		// so this is where we store the current clock.
284		dst->dirty_[0].set_tid(tid_);
285		dst->dirty_[0].epoch = clk_[tid_];
286		dst->release_store_tid_ = tid_;
287		dst->release_store_reused_ = reused_;
288		// Rememeber that we don't need to acquire it in future.
289		dst->elem(tid_).reused = reused_;
290		// Grab a reference.
291		atomic_fetch_add(ref_ptr(dst->tab_), 1, memory_order_relaxed);
292		return;
293		}
294
295		// Check if we need to resize dst.
296		if (dst->size_ < nclk_)
297		dst->Resize(c, nclk_);
298
299		if (dst->release_store_tid_ == tid_ &&
300		dst->release_store_reused_ == reused_ &&
301		!HasAcquiredAfterRelease(dst)) {
302		UpdateCurrentThread(c, dst);
303		return;
304		}
305
306		// O(N) release-store.
307		dst->Unshare(c);
308		// Note: dst can be larger than this ThreadClock.
309		// This is fine since clk_ beyond size is all zeros.
310		uptr i = 0;
311		for (ClockElem &ce : *dst) {
312		ce.epoch = clk_[i];
313		ce.reused = 0;
314		i++;
315		}
316		for (uptr i = 0; i < kDirtyTids; i++) dst->dirty_[i].set_tid(kInvalidTid);
317		dst->release_store_tid_ = tid_;
318		dst->release_store_reused_ = reused_;
319		// Rememeber that we don't need to acquire it in future.
320		dst->elem(tid_).reused = reused_;
321
322		// If the resulting clock is cachable, cache it for future release operations.
323		// The clock is always cachable if we released to an empty sync object.
324		if (cached_idx_ == 0 && dst->Cachable()) {
325		// Grab a reference to the ClockBlock.
326		atomic_uint32_t *ref = ref_ptr(dst->tab_);
327		if (atomic_load(ref, memory_order_acquire) == 1)
328		atomic_store_relaxed(ref, 2);
329		else
330		atomic_fetch_add(ref_ptr(dst->tab_), 1, memory_order_relaxed);
331		cached_idx_ = dst->tab_idx_;
332		cached_size_ = dst->size_;
333		cached_blocks_ = dst->blocks_;
334		}
335		}
336
337		void ThreadClock::acq_rel(ClockCache c, SyncClock dst) {
338		acquire(c, dst);
339		ReleaseStore(c, dst);
340		}
341
342		// Updates only single element related to the current thread in dst->clk_.
343		void ThreadClock::UpdateCurrentThread(ClockCache c, SyncClock dst) const {
344		// Update the threads time, but preserve 'acquired' flag.
345		for (unsigned i = 0; i < kDirtyTids; i++) {
346		SyncClock::Dirty *dirty = &dst->dirty_[i];
347		const unsigned tid = dirty->tid();
348		if (tid == tid_ \|\| tid == kInvalidTid) {
349		dirty->set_tid(tid_);
350		dirty->epoch = clk_[tid_];
351		return;
352		}
353		}
354		// Reset all 'acquired' flags, O(N).
355		// We are going to touch dst elements, so we need to unshare it.
356		dst->Unshare(c);
357		dst->elem(tid_).epoch = clk_[tid_];
358		for (uptr i = 0; i < dst->size_; i++)
359		dst->elem(i).reused = 0;
360		dst->FlushDirty();
361		}
362
363		// Checks whether the current thread has already acquired src.
364		bool ThreadClock::IsAlreadyAcquired(const SyncClock *src) const {
365		if (src->elem(tid_).reused != reused_)
366		return false;
367		for (unsigned i = 0; i < kDirtyTids; i++) {
368		SyncClock::Dirty dirty = src->dirty_[i];
369		if (dirty.tid() != kInvalidTid) {
370		if (clk_[dirty.tid()] < dirty.epoch)
371		return false;
372		}
373		}
374		return true;
375		}
376
377		// Checks whether the current thread has acquired anything
378		// from other clocks after releasing to dst (directly or indirectly).
379		bool ThreadClock::HasAcquiredAfterRelease(const SyncClock *dst) const {
380		const u64 my_epoch = dst->elem(tid_).epoch;
381		return my_epoch <= last_acquire_ \|\|
382		my_epoch <= atomic_load_relaxed(&global_acquire_);
383		}
384
385		// Sets a single element in the vector clock.
386		// This function is called only from weird places like AcquireGlobal.
387		void ThreadClock::set(ClockCache *c, unsigned tid, u64 v) {
388		DCHECK_LT(tid, kMaxTid);
389		DCHECK_GE(v, clk_[tid]);
390		clk_[tid] = v;
391		if (nclk_ <= tid)
392		nclk_ = tid + 1;
393		last_acquire_ = clk_[tid_];
394		ResetCached(c);
395		}
396
397		void ThreadClock::DebugDump(int(printf)(const char s, ...)) {
398		printf("clock=[");
399		for (uptr i = 0; i < nclk_; i++)
400		printf("%s%llu", i == 0 ? "" : ",", clk_[i]);
401		printf("] tid=%u/%u last_acq=%llu", tid_, reused_, last_acquire_);
402		}
403
404		SyncClock::SyncClock() {
405		ResetImpl();
406		}
407
408		SyncClock::~SyncClock() {
409		// Reset must be called before dtor.
410		CHECK_EQ(size_, 0);
411		CHECK_EQ(blocks_, 0);
412		CHECK_EQ(tab_, 0);
413		CHECK_EQ(tab_idx_, 0);
414		}
415
416		void SyncClock::Reset(ClockCache *c) {
417		if (size_)
418		UnrefClockBlock(c, tab_idx_, blocks_);
419		ResetImpl();
420		}
421
422		void SyncClock::ResetImpl() {
423		tab_ = 0;
424		tab_idx_ = 0;
425		size_ = 0;
426		blocks_ = 0;
427		release_store_tid_ = kInvalidTid;
428		release_store_reused_ = 0;
429		for (uptr i = 0; i < kDirtyTids; i++) dirty_[i].set_tid(kInvalidTid);
430		}
431
432		void SyncClock::Resize(ClockCache *c, uptr nclk) {
433		Unshare(c);
434		if (nclk <= capacity()) {
435		// Memory is already allocated, just increase the size.
436		size_ = nclk;
437		return;
438		}
439		if (size_ == 0) {
440		// Grow from 0 to one-level table.
441		CHECK_EQ(size_, 0);
442		CHECK_EQ(blocks_, 0);
443		CHECK_EQ(tab_, 0);
444		CHECK_EQ(tab_idx_, 0);
445		tab_idx_ = ctx->clock_alloc.Alloc(c);
446		tab_ = ctx->clock_alloc.Map(tab_idx_);
447		internal_memset(tab_, 0, sizeof(*tab_));
448		atomic_store_relaxed(ref_ptr(tab_), 1);
449		size_ = 1;
450		} else if (size_ > blocks_ * ClockBlock::kClockCount) {
451		u32 idx = ctx->clock_alloc.Alloc(c);
452		ClockBlock *new_cb = ctx->clock_alloc.Map(idx);
453		uptr top = size_ - blocks_ * ClockBlock::kClockCount;
454		CHECK_LT(top, ClockBlock::kClockCount);
455		const uptr move = top * sizeof(tab_->clock[0]);
456		internal_memcpy(&new_cb->clock[0], tab_->clock, move);
457		internal_memset(&new_cb->clock[top], 0, sizeof(*new_cb) - move);
458		internal_memset(tab_->clock, 0, move);
459		append_block(idx);
460		}
461		// At this point we have first level table allocated and all clock elements
462		// are evacuated from it to a second level block.
463		// Add second level tables as necessary.
464		while (nclk > capacity()) {
465		u32 idx = ctx->clock_alloc.Alloc(c);
466		ClockBlock *cb = ctx->clock_alloc.Map(idx);
467		internal_memset(cb, 0, sizeof(*cb));
468		append_block(idx);
469		}
470		size_ = nclk;
471		}
472
473		// Flushes all dirty elements into the main clock array.
474		void SyncClock::FlushDirty() {
475		for (unsigned i = 0; i < kDirtyTids; i++) {
476		Dirty *dirty = &dirty_[i];
477		if (dirty->tid() != kInvalidTid) {
478		CHECK_LT(dirty->tid(), size_);
479		elem(dirty->tid()).epoch = dirty->epoch;
480		dirty->set_tid(kInvalidTid);
481		}
482		}
483		}
484
485		bool SyncClock::IsShared() const {
486		if (size_ == 0)
487		return false;
488		atomic_uint32_t *ref = ref_ptr(tab_);
489		u32 v = atomic_load(ref, memory_order_acquire);
490		CHECK_GT(v, 0);
491		return v > 1;
492		}
493
494		// Unshares the current clock if it's shared.
495		// Shared clocks are immutable, so they need to be unshared before any updates.
496		// Note: this does not apply to dirty entries as they are not shared.
497		void SyncClock::Unshare(ClockCache *c) {
498		if (!IsShared())
499		return;
500		// First, copy current state into old.
501		SyncClock old;
502		old.tab_ = tab_;
503		old.tab_idx_ = tab_idx_;
504		old.size_ = size_;
505		old.blocks_ = blocks_;
506		old.release_store_tid_ = release_store_tid_;
507		old.release_store_reused_ = release_store_reused_;
508		for (unsigned i = 0; i < kDirtyTids; i++)
509		old.dirty_[i] = dirty_[i];
510		// Then, clear current object.
511		ResetImpl();
512		// Allocate brand new clock in the current object.
513		Resize(c, old.size_);
514		// Now copy state back into this object.
515		Iter old_iter(&old);
516		for (ClockElem &ce : *this) {
517		ce = *old_iter;
518		++old_iter;
519		}
520		release_store_tid_ = old.release_store_tid_;
521		release_store_reused_ = old.release_store_reused_;
522		for (unsigned i = 0; i < kDirtyTids; i++)
523		dirty_[i] = old.dirty_[i];
524		// Drop reference to old and delete if necessary.
525		old.Reset(c);
526		}
527
528		// Can we cache this clock for future release operations?
529		ALWAYS_INLINE bool SyncClock::Cachable() const {
530		if (size_ == 0)
531		return false;
532		for (unsigned i = 0; i < kDirtyTids; i++) {
533		if (dirty_[i].tid() != kInvalidTid)
534		return false;
535		}
536		return atomic_load_relaxed(ref_ptr(tab_)) == 1;
537		}
538
539		// elem linearizes the two-level structure into linear array.
540		// Note: this is used only for one time accesses, vector operations use
541		// the iterator as it is much faster.
542		ALWAYS_INLINE ClockElem &SyncClock::elem(unsigned tid) const {
543		DCHECK_LT(tid, size_);
544		const uptr block = tid / ClockBlock::kClockCount;
545		DCHECK_LE(block, blocks_);
546		tid %= ClockBlock::kClockCount;
547		if (block == blocks_)
548		return tab_->clock[tid];
549		u32 idx = get_block(block);
550		ClockBlock *cb = ctx->clock_alloc.Map(idx);
551		return cb->clock[tid];
552		}
553
554		ALWAYS_INLINE uptr SyncClock::capacity() const {
555		if (size_ == 0)
556		return 0;
557		uptr ratio = sizeof(ClockBlock::clock[0]) / sizeof(ClockBlock::table[0]);
558		// How many clock elements we can fit into the first level block.
559		// +1 for ref counter.
560		uptr top = ClockBlock::kClockCount - RoundUpTo(blocks_ + 1, ratio) / ratio;
561		return blocks_ * ClockBlock::kClockCount + top;
562		}
563
564		ALWAYS_INLINE u32 SyncClock::get_block(uptr bi) const {
565		DCHECK(size_);
566		DCHECK_LT(bi, blocks_);
567		return tab_->table[ClockBlock::kBlockIdx - bi];
568		}
569
570		ALWAYS_INLINE void SyncClock::append_block(u32 idx) {
571		uptr bi = blocks_++;
572		CHECK_EQ(get_block(bi), 0);
573		tab_->table[ClockBlock::kBlockIdx - bi] = idx;
574		}
575
576		// Used only by tests.
577		u64 SyncClock::get(unsigned tid) const {
578		for (unsigned i = 0; i < kDirtyTids; i++) {
579		Dirty dirty = dirty_[i];
580		if (dirty.tid() == tid)
581		return dirty.epoch;
582		}
583		return elem(tid).epoch;
584		}
585
586		// Used only by Iter test.
587		u64 SyncClock::get_clean(unsigned tid) const {
588		return elem(tid).epoch;
589		}
590
591		void SyncClock::DebugDump(int(printf)(const char s, ...)) {
592		printf("clock=[");
593		for (uptr i = 0; i < size_; i++)
594		printf("%s%llu", i == 0 ? "" : ",", elem(i).epoch);
595		printf("] reused=[");
596		for (uptr i = 0; i < size_; i++)
597		printf("%s%llu", i == 0 ? "" : ",", elem(i).reused);
598		printf("] release_store_tid=%d/%d dirty_tids=%d[%llu]/%d[%llu]",
599		release_store_tid_, release_store_reused_, dirty_[0].tid(),
600		dirty_[0].epoch, dirty_[1].tid(), dirty_[1].epoch);
601		}
602
603		void SyncClock::Iter::Next() {
604		// Finished with the current block, move on to the next one.
605		block_++;
606		if (block_ < parent_->blocks_) {
607		// Iterate over the next second level block.
608		u32 idx = parent_->get_block(block_);
609		ClockBlock *cb = ctx->clock_alloc.Map(idx);
610		pos_ = &cb->clock[0];
611		end_ = pos_ + min(parent_->size_ - block_ * ClockBlock::kClockCount,
612		ClockBlock::kClockCount);
613		return;
614		}
615		if (block_ == parent_->blocks_ &&
616		parent_->size_ > parent_->blocks_ * ClockBlock::kClockCount) {
617		// Iterate over elements in the first level block.
618		pos_ = &parent_->tab_->clock[0];
619		end_ = pos_ + min(parent_->size_ - block_ * ClockBlock::kClockCount,
620		ClockBlock::kClockCount);
621		return;
622		}
623		parent_ = nullptr; // denotes end
624		}
625		} // namespace __tsan

lib/tsan/tsan_clock.h deleted-293

...	...	@@ -1,293 +0,0 @@
1		//===-- tsan_clock.h --------------------------------------------- C++ --===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// This file is a part of ThreadSanitizer (TSan), a race detector.
10		//
11		//===----------------------------------------------------------------------===//
12		#ifndef TSAN_CLOCK_H
13		#define TSAN_CLOCK_H
14
15		#include "tsan_defs.h"
16		#include "tsan_dense_alloc.h"
17
18		namespace __tsan {
19
20		typedef DenseSlabAlloc<ClockBlock, 1 << 22, 1 << 10> ClockAlloc;
21		typedef DenseSlabAllocCache ClockCache;
22
23		// The clock that lives in sync variables (mutexes, atomics, etc).
24		class SyncClock {
25		public:
26		SyncClock();
27		~SyncClock();
28
29		uptr size() const;
30
31		// These are used only in tests.
32		u64 get(unsigned tid) const;
33		u64 get_clean(unsigned tid) const;
34
35		void Resize(ClockCache *c, uptr nclk);
36		void Reset(ClockCache *c);
37
38		void DebugDump(int(printf)(const char s, ...));
39
40		// Clock element iterator.
41		// Note: it iterates only over the table without regard to dirty entries.
42		class Iter {
43		public:
44		explicit Iter(SyncClock* parent);
45		Iter& operator++();
46		bool operator!=(const Iter& other);
47		ClockElem &operator*();
48
49		private:
50		SyncClock *parent_;
51		// [pos_, end_) is the current continuous range of clock elements.
52		ClockElem *pos_;
53		ClockElem *end_;
54		int block_; // Current number of second level block.
55
56		NOINLINE void Next();
57		};
58
59		Iter begin();
60		Iter end();
61
62		private:
63		friend class ThreadClock;
64		friend class Iter;
65		static const uptr kDirtyTids = 2;
66
67		struct Dirty {
68		u32 tid() const { return tid_ == kShortInvalidTid ? kInvalidTid : tid_; }
69		void set_tid(u32 tid) {
70		tid_ = tid == kInvalidTid ? kShortInvalidTid : tid;
71		}
72		u64 epoch : kClkBits;
73
74		private:
75		// Full kInvalidTid won't fit into Dirty::tid.
76		static const u64 kShortInvalidTid = (1ull << (64 - kClkBits)) - 1;
77		u64 tid_ : 64 - kClkBits; // kInvalidId if not active
78		};
79
80		static_assert(sizeof(Dirty) == 8, "Dirty is not 64bit");
81
82		unsigned release_store_tid_;
83		unsigned release_store_reused_;
84		Dirty dirty_[kDirtyTids];
85		// If size_ is 0, tab_ is nullptr.
86		// If size <= 64 (kClockCount), tab_ contains pointer to an array with
87		// 64 ClockElem's (ClockBlock::clock).
88		// Otherwise, tab_ points to an array with up to 127 u32 elements,
89		// each pointing to the second-level 512b block with 64 ClockElem's.
90		// Unused space in the first level ClockBlock is used to store additional
91		// clock elements.
92		// The last u32 element in the first level ClockBlock is always used as
93		// reference counter.
94		//
95		// See the following scheme for details.
96		// All memory blocks are 512 bytes (allocated from ClockAlloc).
97		// Clock (clk) elements are 64 bits.
98		// Idx and ref are 32 bits.
99		//
100		// tab_
101		// \|
102		// \/
103		// +----------------------------------------------------+
104		// \| clk128 \| clk129 \| ...unused... \| idx1 \| idx0 \| ref \|
105		// +----------------------------------------------------+
106		// \| \|
107		// \| \/
108		// \| +----------------+
109		// \| \| clk0 ... clk63 \|
110		// \| +----------------+
111		// \/
112		// +------------------+
113		// \| clk64 ... clk127 \|
114		// +------------------+
115		//
116		// Note: dirty entries, if active, always override what's stored in the clock.
117		ClockBlock *tab_;
118		u32 tab_idx_;
119		u16 size_;
120		u16 blocks_; // Number of second level blocks.
121
122		void Unshare(ClockCache *c);
123		bool IsShared() const;
124		bool Cachable() const;
125		void ResetImpl();
126		void FlushDirty();
127		uptr capacity() const;
128		u32 get_block(uptr bi) const;
129		void append_block(u32 idx);
130		ClockElem &elem(unsigned tid) const;
131		};
132
133		// The clock that lives in threads.
134		class ThreadClock {
135		public:
136		typedef DenseSlabAllocCache Cache;
137
138		explicit ThreadClock(unsigned tid, unsigned reused = 0);
139
140		u64 get(unsigned tid) const;
141		void set(ClockCache *c, unsigned tid, u64 v);
142		void set(u64 v);
143		void tick();
144		uptr size() const;
145
146		void acquire(ClockCache c, SyncClock src);
147		void releaseStoreAcquire(ClockCache c, SyncClock src);
148		void release(ClockCache c, SyncClock dst);
149		void acq_rel(ClockCache c, SyncClock dst);
150		void ReleaseStore(ClockCache c, SyncClock dst);
151		void ResetCached(ClockCache *c);
152		void NoteGlobalAcquire(u64 v);
153
154		void DebugReset();
155		void DebugDump(int(printf)(const char s, ...));
156
157		private:
158		static const uptr kDirtyTids = SyncClock::kDirtyTids;
159		// Index of the thread associated with he clock ("current thread").
160		const unsigned tid_;
161		const unsigned reused_; // tid_ reuse count.
162		// Current thread time when it acquired something from other threads.
163		u64 last_acquire_;
164
165		// Last time another thread has done a global acquire of this thread's clock.
166		// It helps to avoid problem described in:
167		// https://github.com/golang/go/issues/39186
168		// See test/tsan/java_finalizer2.cpp for a regression test.
169		// Note the failuire is _extremely_ hard to hit, so if you are trying
170		// to reproduce it, you may want to run something like:
171		// $ go get golang.org/x/tools/cmd/stress
172		// $ stress -p=64 ./a.out
173		//
174		// The crux of the problem is roughly as follows.
175		// A number of O(1) optimizations in the clocks algorithm assume proper
176		// transitive cumulative propagation of clock values. The AcquireGlobal
177		// operation may produce an inconsistent non-linearazable view of
178		// thread clocks. Namely, it may acquire a later value from a thread
179		// with a higher ID, but fail to acquire an earlier value from a thread
180		// with a lower ID. If a thread that executed AcquireGlobal then releases
181		// to a sync clock, it will spoil the sync clock with the inconsistent
182		// values. If another thread later releases to the sync clock, the optimized
183		// algorithm may break.
184		//
185		// The exact sequence of events that leads to the failure.
186		// - thread 1 executes AcquireGlobal
187		// - thread 1 acquires value 1 for thread 2
188		// - thread 2 increments clock to 2
189		// - thread 2 releases to sync object 1
190		// - thread 3 at time 1
191		// - thread 3 acquires from sync object 1
192		// - thread 3 increments clock to 2
193		// - thread 1 acquires value 2 for thread 3
194		// - thread 1 releases to sync object 2
195		// - sync object 2 clock has 1 for thread 2 and 2 for thread 3
196		// - thread 3 releases to sync object 2
197		// - thread 3 sees value 2 in the clock for itself
198		// and decides that it has already released to the clock
199		// and did not acquire anything from other threads after that
200		// (the last_acquire_ check in release operation)
201		// - thread 3 does not update the value for thread 2 in the clock from 1 to 2
202		// - thread 4 acquires from sync object 2
203		// - thread 4 detects a false race with thread 2
204		// as it should have been synchronized with thread 2 up to time 2,
205		// but because of the broken clock it is now synchronized only up to time 1
206		//
207		// The global_acquire_ value helps to prevent this scenario.
208		// Namely, thread 3 will not trust any own clock values up to global_acquire_
209		// for the purposes of the last_acquire_ optimization.
210		atomic_uint64_t global_acquire_;
211
212		// Cached SyncClock (without dirty entries and release_store_tid_).
213		// We reuse it for subsequent store-release operations without intervening
214		// acquire operations. Since it is shared (and thus constant), clock value
215		// for the current thread is then stored in dirty entries in the SyncClock.
216		// We host a refernece to the table while it is cached here.
217		u32 cached_idx_;
218		u16 cached_size_;
219		u16 cached_blocks_;
220
221		// Number of active elements in the clk_ table (the rest is zeros).
222		uptr nclk_;
223		u64 clk_[kMaxTidInClock]; // Fixed size vector clock.
224
225		bool IsAlreadyAcquired(const SyncClock *src) const;
226		bool HasAcquiredAfterRelease(const SyncClock *dst) const;
227		void UpdateCurrentThread(ClockCache c, SyncClock dst) const;
228		};
229
230		ALWAYS_INLINE u64 ThreadClock::get(unsigned tid) const {
231		DCHECK_LT(tid, kMaxTidInClock);
232		return clk_[tid];
233		}
234
235		ALWAYS_INLINE void ThreadClock::set(u64 v) {
236		DCHECK_GE(v, clk_[tid_]);
237		clk_[tid_] = v;
238		}
239
240		ALWAYS_INLINE void ThreadClock::tick() {
241		clk_[tid_]++;
242		}
243
244		ALWAYS_INLINE uptr ThreadClock::size() const {
245		return nclk_;
246		}
247
248		ALWAYS_INLINE void ThreadClock::NoteGlobalAcquire(u64 v) {
249		// Here we rely on the fact that AcquireGlobal is protected by
250		// ThreadRegistryLock, thus only one thread at a time executes it
251		// and values passed to this function should not go backwards.
252		CHECK_LE(atomic_load_relaxed(&global_acquire_), v);
253		atomic_store_relaxed(&global_acquire_, v);
254		}
255
256		ALWAYS_INLINE SyncClock::Iter SyncClock::begin() {
257		return Iter(this);
258		}
259
260		ALWAYS_INLINE SyncClock::Iter SyncClock::end() {
261		return Iter(nullptr);
262		}
263
264		ALWAYS_INLINE uptr SyncClock::size() const {
265		return size_;
266		}
267
268		ALWAYS_INLINE SyncClock::Iter::Iter(SyncClock* parent)
269		: parent_(parent)
270		, pos_(nullptr)
271		, end_(nullptr)
272		, block_(-1) {
273		if (parent)
274		Next();
275		}
276
277		ALWAYS_INLINE SyncClock::Iter& SyncClock::Iter::operator++() {
278		pos_++;
279		if (UNLIKELY(pos_ >= end_))
280		Next();
281		return *this;
282		}
283
284		ALWAYS_INLINE bool SyncClock::Iter::operator!=(const SyncClock::Iter& other) {
285		return parent_ != other.parent_;
286		}
287
288		ALWAYS_INLINE ClockElem &SyncClock::Iter::operator*() {
289		return *pos_;
290		}
291		} // namespace __tsan
292
293		#endif // TSAN_CLOCK_H

lib/tsan/tsan_debugging.cpp+6-6

...	...	@@ -157,7 +157,7 @@ int __tsan_get_report_mutex(void report, uptr idx, uptr mutex_id, void **addr,
157	157	ReportMutex *mutex = rep->mutexes[idx];
158	158	*mutex_id = mutex->id;
159	159	addr = (void )mutex->addr;
160		*destroyed = mutex->destroyed;
	160	*destroyed = false;
161	161	if (mutex->stack) CopyTrace(mutex->stack->frames, trace, trace_size);
162	162	return 1;
163	163	}
...	...	@@ -195,9 +195,9 @@ const char __tsan_locate_address(uptr addr, char name, uptr name_size,
195	195	const char *region_kind = nullptr;
196	196	if (name && name_size > 0) name[0] = 0;
197	197
198		if (IsMetaMem(addr)) {
	198	if (IsMetaMem(reinterpret_cast<u32 *>(addr))) {
199	199	region_kind = "meta shadow";
200		} else if (IsShadowMem(addr)) {
	200	} else if (IsShadowMem(reinterpret_cast<RawShadow *>(addr))) {
201	201	region_kind = "shadow";
202	202	} else {
203	203	bool is_stack = false;
...	...	@@ -215,9 +215,9 @@ const char __tsan_locate_address(uptr addr, char name, uptr name_size,
215	215	} else {
216	216	// TODO(kuba.brecka): We should not lock. This is supposed to be called
217	217	// from within the debugger when other threads are stopped.
218		ctx->thread_registry->Lock();
	218	ctx->thread_registry.Lock();
219	219	ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack);
220		ctx->thread_registry->Unlock();
	220	ctx->thread_registry.Unlock();
221	221	if (tctx) {
222	222	region_kind = is_stack ? "stack" : "tls";
223	223	} else {
...	...	@@ -252,7 +252,7 @@ int __tsan_get_alloc_stack(uptr addr, uptr trace, uptr size, int thread_id,
252	252	*thread_id = b->tid;
253	253	// No locking. This is supposed to be called from within the debugger when
254	254	// other threads are stopped.
255		ThreadContextBase *tctx = ctx->thread_registry->GetThreadLocked(b->tid);
	255	ThreadContextBase *tctx = ctx->thread_registry.GetThreadLocked(b->tid);
256	256	*os_id = tctx->os_id;
257	257
258	258	StackTrace stack = StackDepotGet(b->stk);

lib/tsan/tsan_defs.h+64-36

...	...	@@ -18,6 +18,24 @@
18	18	#include "sanitizer_common/sanitizer_mutex.h"
19	19	#include "ubsan/ubsan_platform.h"
20	20
	21	#ifndef TSAN_VECTORIZE
	22	# define TSAN_VECTORIZE __SSE4_2__
	23	#endif
	24
	25	#if TSAN_VECTORIZE
	26	// <emmintrin.h> transitively includes <stdlib.h>,
	27	// and it's prohibited to include std headers into tsan runtime.
	28	// So we do this dirty trick.
	29	# define _MM_MALLOC_H_INCLUDED
	30	# define __MM_MALLOC_H
	31	# include <emmintrin.h>
	32	# include <smmintrin.h>
	33	# define VECTOR_ALIGNED ALIGNED(16)
	34	typedef __m128i m128;
	35	#else
	36	# define VECTOR_ALIGNED
	37	#endif
	38
21	39	// Setup defaults for compile definitions.
22	40	#ifndef TSAN_NO_HISTORY
23	41	# define TSAN_NO_HISTORY 0
...	...	@@ -33,40 +51,26 @@
33	51
34	52	namespace __tsan {
35	53
36		const int kClkBits = 42;
37		const unsigned kMaxTidReuse = (1 << (64 - kClkBits)) - 1;
	54	constexpr uptr kByteBits = 8;
38	55
39		struct ClockElem {
40		u64 epoch : kClkBits;
41		u64 reused : 64 - kClkBits; // tid reuse count
42		};
	56	// Thread slot ID.
	57	enum class Sid : u8 {};
	58	constexpr uptr kThreadSlotCount = 256;
	59	constexpr Sid kFreeSid = static_cast<Sid>(255);
43	60
44		struct ClockBlock {
45		static const uptr kSize = 512;
46		static const uptr kTableSize = kSize / sizeof(u32);
47		static const uptr kClockCount = kSize / sizeof(ClockElem);
48		static const uptr kRefIdx = kTableSize - 1;
49		static const uptr kBlockIdx = kTableSize - 2;
	61	// Abstract time unit, vector clock element.
	62	enum class Epoch : u16 {};
	63	constexpr uptr kEpochBits = 14;
	64	constexpr Epoch kEpochZero = static_cast<Epoch>(0);
	65	constexpr Epoch kEpochOver = static_cast<Epoch>(1 << kEpochBits);
	66	constexpr Epoch kEpochLast = static_cast<Epoch>((1 << kEpochBits) - 1);
50	67
51		union {
52		u32 table[kTableSize];
53		ClockElem clock[kClockCount];
54		};
	68	inline Epoch EpochInc(Epoch epoch) {
	69	return static_cast<Epoch>(static_cast<u16>(epoch) + 1);
	70	}
55	71
56		ClockBlock() {
57		}
58		};
	72	inline bool EpochOverflow(Epoch epoch) { return epoch == kEpochOver; }
59	73
60		const int kTidBits = 13;
61		// Reduce kMaxTid by kClockCount because one slot in ClockBlock table is
62		// occupied by reference counter, so total number of elements we can store
63		// in SyncClock is kClockCount * (kTableSize - 1).
64		const unsigned kMaxTid = (1 << kTidBits) - ClockBlock::kClockCount;
65		#if !SANITIZER_GO
66		const unsigned kMaxTidInClock = kMaxTid * 2; // This includes msb 'freed' bit.
67		#else
68		const unsigned kMaxTidInClock = kMaxTid; // Go does not track freed memory.
69		#endif
70	74	const uptr kShadowStackSize = 64 * 1024;
71	75
72	76	// Count of shadow values in a shadow cell.
...	...	@@ -75,8 +79,9 @@ const uptr kShadowCnt = 4;
75	79	// That many user bytes are mapped onto a single shadow cell.
76	80	const uptr kShadowCell = 8;
77	81
78		// Size of a single shadow value (u64).
79		const uptr kShadowSize = 8;
	82	// Single shadow value.
	83	enum class RawShadow : u32 {};
	84	const uptr kShadowSize = sizeof(RawShadow);
80	85
81	86	// Shadow memory is kShadowMultiplier times larger than user memory.
82	87	const uptr kShadowMultiplier = kShadowSize * kShadowCnt / kShadowCell;
...	...	@@ -88,6 +93,9 @@ const uptr kMetaShadowCell = 8;
88	93	// Size of a single meta shadow value (u32).
89	94	const uptr kMetaShadowSize = 4;
90	95
	96	// All addresses and PCs are assumed to be compressable to that many bits.
	97	const uptr kCompressedAddrBits = 44;
	98
91	99	#if TSAN_NO_HISTORY
92	100	const bool kCollectHistory = false;
93	101	#else
...	...	@@ -149,17 +157,34 @@ MD5Hash md5_hash(const void *data, uptr size);
149	157	struct Processor;
150	158	struct ThreadState;
151	159	class ThreadContext;
	160	struct TidSlot;
152	161	struct Context;
153	162	struct ReportStack;
154	163	class ReportDesc;
155	164	class RegionAlloc;
	165	struct Trace;
	166	struct TracePart;
	167
	168	typedef uptr AccessType;
	169
	170	enum : AccessType {
	171	kAccessWrite = 0,
	172	kAccessRead = 1 << 0,
	173	kAccessAtomic = 1 << 1,
	174	kAccessVptr = 1 << 2, // read or write of an object virtual table pointer
	175	kAccessFree = 1 << 3, // synthetic memory access during memory freeing
	176	kAccessExternalPC = 1 << 4, // access PC can have kExternalPCBit set
	177	kAccessCheckOnly = 1 << 5, // check for races, but don't store
	178	kAccessNoRodata = 1 << 6, // don't check for .rodata marker
	179	kAccessSlotLocked = 1 << 7, // memory access with TidSlot locked
	180	};
156	181
157	182	// Descriptor of user's memory block.
158	183	struct MBlock {
159	184	u64 siz : 48;
160	185	u64 tag : 16;
161		u32 stk;
162		u16 tid;
	186	StackID stk;
	187	Tid tid;
163	188	};
164	189
165	190	COMPILER_CHECK(sizeof(MBlock) == 16);
...	...	@@ -173,15 +198,18 @@ enum ExternalTag : uptr {
173	198	// as 16-bit values, see tsan_defs.h.
174	199	};
175	200
176		enum MutexType {
177		MutexTypeTrace = MutexLastCommon,
178		MutexTypeReport,
	201	enum {
	202	MutexTypeReport = MutexLastCommon,
179	203	MutexTypeSyncVar,
180	204	MutexTypeAnnotations,
181	205	MutexTypeAtExit,
182	206	MutexTypeFired,
183	207	MutexTypeRacy,
184	208	MutexTypeGlobalProc,
	209	MutexTypeInternalAlloc,
	210	MutexTypeTrace,
	211	MutexTypeSlot,
	212	MutexTypeSlots,
185	213	};
186	214
187	215	} // namespace __tsan

lib/tsan/tsan_dense_alloc.h+89-45

...	...	@@ -49,11 +49,7 @@ class DenseSlabAlloc {
49	49	static_assert(sizeof(T) > sizeof(IndexT),
50	50	"it doesn't make sense to use dense alloc");
51	51
52		explicit DenseSlabAlloc(LinkerInitialized, const char *name) {
53		freelist_ = 0;
54		fillpos_ = 0;
55		name_ = name;
56		}
	52	DenseSlabAlloc(LinkerInitialized, const char *name) : name_(name) {}
57	53
58	54	explicit DenseSlabAlloc(const char *name)
59	55	: DenseSlabAlloc(LINKER_INITIALIZED, name) {
...	...	@@ -89,12 +85,7 @@ class DenseSlabAlloc {
89	85	}
90	86
91	87	void FlushCache(Cache *c) {
92		SpinMutexLock lock(&mtx_);
93		while (c->pos) {
94		IndexT idx = c->cache[--c->pos];
95		(IndexT)Map(idx) = freelist_;
96		freelist_ = idx;
97		}
	88	while (c->pos) Drain(c);
98	89	}
99	90
100	91	void InitCache(Cache *c) {
...	...	@@ -102,48 +93,101 @@ class DenseSlabAlloc {
102	93	internal_memset(c->cache, 0, sizeof(c->cache));
103	94	}
104	95
	96	uptr AllocatedMemory() const {
	97	return atomic_load_relaxed(&fillpos_) * kL2Size * sizeof(T);
	98	}
	99
	100	template <typename Func>
	101	void ForEach(Func func) {
	102	Lock lock(&mtx_);
	103	uptr fillpos = atomic_load_relaxed(&fillpos_);
	104	for (uptr l1 = 0; l1 < fillpos; l1++) {
	105	for (IndexT l2 = l1 == 0 ? 1 : 0; l2 < kL2Size; l2++) func(&map_[l1][l2]);
	106	}
	107	}
	108
105	109	private:
106	110	T *map_[kL1Size];
107		SpinMutex mtx_;
108		IndexT freelist_;
109		uptr fillpos_;
110		const char *name_;
111
112		void Refill(Cache *c) {
113		SpinMutexLock lock(&mtx_);
114		if (freelist_ == 0) {
115		if (fillpos_ == kL1Size) {
116		Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n",
117		name_, kL1Size, kL2Size);
118		Die();
119		}
120		VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n",
121		name_, fillpos_, kL1Size, kL2Size);
122		T batch = (T)MmapOrDie(kL2Size * sizeof(T), name_);
123		// Reserve 0 as invalid index.
124		IndexT start = fillpos_ == 0 ? 1 : 0;
125		for (IndexT i = start; i < kL2Size; i++) {
126		new(batch + i) T;
127		(IndexT)(batch + i) = i + 1 + fillpos_ * kL2Size;
128		}
129		(IndexT)(batch + kL2Size - 1) = 0;
130		freelist_ = fillpos_ * kL2Size + start;
131		map_[fillpos_++] = batch;
132		}
133		for (uptr i = 0; i < Cache::kSize / 2 && freelist_ != 0; i++) {
134		IndexT idx = freelist_;
	111	Mutex mtx_;
	112	// The freelist is organized as a lock-free stack of batches of nodes.
	113	// The stack itself uses Block::next links, while the batch within each
	114	// stack node uses Block::batch links.
	115	// Low 32-bits of freelist_ is the node index, top 32-bits is ABA-counter.
	116	atomic_uint64_t freelist_ = {0};
	117	atomic_uintptr_t fillpos_ = {0};
	118	const char *const name_;
	119
	120	struct Block {
	121	IndexT next;
	122	IndexT batch;
	123	};
	124
	125	Block MapBlock(IndexT idx) { return reinterpret_cast<Block >(Map(idx)); }
	126
	127	static constexpr u64 kCounterInc = 1ull << 32;
	128	static constexpr u64 kCounterMask = ~(kCounterInc - 1);
	129
	130	NOINLINE void Refill(Cache *c) {
	131	// Pop 1 batch of nodes from the freelist.
	132	IndexT idx;
	133	u64 xchg;
	134	u64 cmp = atomic_load(&freelist_, memory_order_acquire);
	135	do {
	136	idx = static_cast<IndexT>(cmp);
	137	if (!idx)
	138	return AllocSuperBlock(c);
	139	Block *ptr = MapBlock(idx);
	140	xchg = ptr->next \| (cmp & kCounterMask);
	141	} while (!atomic_compare_exchange_weak(&freelist_, &cmp, xchg,
	142	memory_order_acq_rel));
	143	// Unpack it into c->cache.
	144	while (idx) {
135	145	c->cache[c->pos++] = idx;
136		freelist_ = (IndexT)Map(idx);
	146	idx = MapBlock(idx)->batch;
137	147	}
138	148	}
139	149
140		void Drain(Cache *c) {
141		SpinMutexLock lock(&mtx_);
142		for (uptr i = 0; i < Cache::kSize / 2; i++) {
	150	NOINLINE void Drain(Cache *c) {
	151	// Build a batch of at most Cache::kSize / 2 nodes linked by Block::batch.
	152	IndexT head_idx = 0;
	153	for (uptr i = 0; i < Cache::kSize / 2 && c->pos; i++) {
143	154	IndexT idx = c->cache[--c->pos];
144		(IndexT)Map(idx) = freelist_;
145		freelist_ = idx;
	155	Block *ptr = MapBlock(idx);
	156	ptr->batch = head_idx;
	157	head_idx = idx;
	158	}
	159	// Push it onto the freelist stack.
	160	Block *head = MapBlock(head_idx);
	161	u64 xchg;
	162	u64 cmp = atomic_load(&freelist_, memory_order_acquire);
	163	do {
	164	head->next = static_cast<IndexT>(cmp);
	165	xchg = head_idx \| (cmp & kCounterMask) + kCounterInc;
	166	} while (!atomic_compare_exchange_weak(&freelist_, &cmp, xchg,
	167	memory_order_acq_rel));
	168	}
	169
	170	NOINLINE void AllocSuperBlock(Cache *c) {
	171	Lock lock(&mtx_);
	172	uptr fillpos = atomic_load_relaxed(&fillpos_);
	173	if (fillpos == kL1Size) {
	174	Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n", name_, kL1Size,
	175	kL2Size);
	176	Die();
	177	}
	178	VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n", name_,
	179	fillpos, kL1Size, kL2Size);
	180	T batch = (T )MmapOrDie(kL2Size * sizeof(T), name_);
	181	map_[fillpos] = batch;
	182	// Reserve 0 as invalid index.
	183	for (IndexT i = fillpos ? 0 : 1; i < kL2Size; i++) {
	184	new (batch + i) T;
	185	c->cache[c->pos++] = i + fillpos * kL2Size;
	186	if (c->pos == Cache::kSize)
	187	Drain(c);
146	188	}
	189	atomic_store_relaxed(&fillpos_, fillpos + 1);
	190	CHECK(c->pos);
147	191	}
148	192	};
149	193

lib/tsan/tsan_dispatch_defs.h created+73

...	...	@@ -0,0 +1,73 @@
	1	//===-- tsan_dispatch_defs.h ------------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	//===----------------------------------------------------------------------===//
	12	#ifndef TSAN_DISPATCH_DEFS_H
	13	#define TSAN_DISPATCH_DEFS_H
	14
	15	#include "sanitizer_common/sanitizer_internal_defs.h"
	16
	17	typedef struct dispatch_object_s {} *dispatch_object_t;
	18
	19	#define DISPATCH_DECL(name) \
	20	typedef struct name##_s : public dispatch_object_s {} *name##_t
	21
	22	DISPATCH_DECL(dispatch_queue);
	23	DISPATCH_DECL(dispatch_source);
	24	DISPATCH_DECL(dispatch_group);
	25	DISPATCH_DECL(dispatch_data);
	26	DISPATCH_DECL(dispatch_semaphore);
	27	DISPATCH_DECL(dispatch_io);
	28
	29	typedef void (dispatch_function_t)(void arg);
	30	typedef void (^dispatch_block_t)(void);
	31	typedef void (^dispatch_io_handler_t)(bool done, dispatch_data_t data,
	32	int error);
	33
	34	typedef long dispatch_once_t;
	35	typedef __sanitizer::u64 dispatch_time_t;
	36	typedef int dispatch_fd_t;
	37	typedef unsigned long dispatch_io_type_t;
	38	typedef unsigned long dispatch_io_close_flags_t;
	39
	40	extern "C" {
	41	void *dispatch_get_context(dispatch_object_t object);
	42	void dispatch_retain(dispatch_object_t object);
	43	void dispatch_release(dispatch_object_t object);
	44
	45	extern const dispatch_block_t _dispatch_data_destructor_free;
	46	extern const dispatch_block_t _dispatch_data_destructor_munmap;
	47	} // extern "C"
	48
	49	#define DISPATCH_DATA_DESTRUCTOR_DEFAULT nullptr
	50	#define DISPATCH_DATA_DESTRUCTOR_FREE _dispatch_data_destructor_free
	51	#define DISPATCH_DATA_DESTRUCTOR_MUNMAP _dispatch_data_destructor_munmap
	52
	53	#if __has_attribute(noescape)
	54	# define DISPATCH_NOESCAPE __attribute__((__noescape__))
	55	#else
	56	# define DISPATCH_NOESCAPE
	57	#endif
	58
	59	#if SANITIZER_APPLE
	60	# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak_import))
	61	#else
	62	# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak))
	63	#endif
	64
	65
	66	// Data types used in dispatch APIs
	67	typedef unsigned long size_t;
	68	typedef unsigned long uintptr_t;
	69	typedef __sanitizer::s64 off_t;
	70	typedef __sanitizer::u16 mode_t;
	71	typedef long long_t;
	72
	73	#endif // TSAN_DISPATCH_DEFS_H

lib/tsan/tsan_external.cpp+24-16

...	...	@@ -10,9 +10,12 @@
10	10	//
11	11	//===----------------------------------------------------------------------===//
12	12	#include "tsan_rtl.h"
13		#include "tsan_interceptors.h"
14	13	#include "sanitizer_common/sanitizer_ptrauth.h"
15	14
	15	#if !SANITIZER_GO
	16	# include "tsan_interceptors.h"
	17	#endif
	18
16	19	namespace __tsan {
17	20
18	21	#define CALLERPC ((uptr)__builtin_return_address(0))
...	...	@@ -43,10 +46,6 @@ const char *GetReportHeaderFromTag(uptr tag) {
43	46	return tag_data ? tag_data->header : nullptr;
44	47	}
45	48
46		void InsertShadowStackFrameForTag(ThreadState *thr, uptr tag) {
47		FuncEntry(thr, (uptr)&registered_tags[tag]);
48		}
49
50	49	uptr TagFromShadowStackFrame(uptr pc) {
51	50	uptr tag_count = atomic_load(&used_tags, memory_order_relaxed);
52	51	void pc_ptr = (void )pc;
...	...	@@ -57,17 +56,26 @@ uptr TagFromShadowStackFrame(uptr pc) {
57	56
58	57	#if !SANITIZER_GO
59	58
60		typedef void(AccessFunc)(ThreadState , uptr, uptr, int);
61		void ExternalAccess(void addr, uptr caller_pc, void tag, AccessFunc access) {
	59	// We need to track tags for individual memory accesses, but there is no space
	60	// in the shadow cells for them. Instead we push/pop them onto the thread
	61	// traces and ignore the extra tag frames when printing reports.
	62	static void PushTag(ThreadState *thr, uptr tag) {
	63	FuncEntry(thr, (uptr)&registered_tags[tag]);
	64	}
	65	static void PopTag(ThreadState *thr) { FuncExit(thr); }
	66
	67	static void ExternalAccess(void *addr, uptr caller_pc, uptr tsan_caller_pc,
	68	void *tag, AccessType typ) {
62	69	CHECK_LT(tag, atomic_load(&used_tags, memory_order_relaxed));
	70	bool in_ignored_lib;
	71	if (caller_pc && libignore()->IsIgnored(caller_pc, &in_ignored_lib))
	72	return;
	73
63	74	ThreadState *thr = cur_thread();
64	75	if (caller_pc) FuncEntry(thr, caller_pc);
65		InsertShadowStackFrameForTag(thr, (uptr)tag);
66		bool in_ignored_lib;
67		if (!caller_pc \|\| !libignore()->IsIgnored(caller_pc, &in_ignored_lib)) {
68		access(thr, CALLERPC, (uptr)addr, kSizeLog1);
69		}
70		FuncExit(thr);
	76	PushTag(thr, (uptr)tag);
	77	MemoryAccess(thr, tsan_caller_pc, (uptr)addr, 1, typ);
	78	PopTag(thr);
71	79	if (caller_pc) FuncExit(thr);
72	80	}
73	81
...	...	@@ -92,7 +100,7 @@ void __tsan_external_register_header(void tag, const char header) {
92	100	header = internal_strdup(header);
93	101	char *old_header =
94	102	(char *)atomic_exchange(header_ptr, (uptr)header, memory_order_seq_cst);
95		if (old_header) internal_free(old_header);
	103	Free(old_header);
96	104	}
97	105
98	106	SANITIZER_INTERFACE_ATTRIBUTE
...	...	@@ -111,12 +119,12 @@ void __tsan_external_assign_tag(void addr, void tag) {
111	119
112	120	SANITIZER_INTERFACE_ATTRIBUTE
113	121	void __tsan_external_read(void addr, void caller_pc, void *tag) {
114		ExternalAccess(addr, STRIP_PAC_PC(caller_pc), tag, MemoryRead);
	122	ExternalAccess(addr, STRIP_PAC_PC(caller_pc), CALLERPC, tag, kAccessRead);
115	123	}
116	124
117	125	SANITIZER_INTERFACE_ATTRIBUTE
118	126	void __tsan_external_write(void addr, void caller_pc, void *tag) {
119		ExternalAccess(addr, STRIP_PAC_PC(caller_pc), tag, MemoryWrite);
	127	ExternalAccess(addr, STRIP_PAC_PC(caller_pc), CALLERPC, tag, kAccessWrite);
120	128	}
121	129	} // extern "C"
122	130

lib/tsan/tsan_fd.cpp+86-27

...	...	@@ -11,9 +11,12 @@
11	11	//===----------------------------------------------------------------------===//
12	12
13	13	#include "tsan_fd.h"
14		#include "tsan_rtl.h"
	14
15	15	#include <sanitizer_common/sanitizer_atomic.h>
16	16
	17	#include "tsan_interceptors.h"
	18	#include "tsan_rtl.h"
	19
17	20	namespace __tsan {
18	21
19	22	const int kTableSizeL1 = 1024;
...	...	@@ -26,8 +29,12 @@ struct FdSync {
26	29
27	30	struct FdDesc {
28	31	FdSync *sync;
29		int creation_tid;
30		u32 creation_stack;
	32	// This is used to establish write -> epoll_wait synchronization
	33	// where epoll_wait receives notification about the write.
	34	atomic_uintptr_t aux_sync; // FdSync*
	35	Tid creation_tid;
	36	StackID creation_stack;
	37	bool closed;
31	38	};
32	39
33	40	struct FdContext {
...	...	@@ -100,6 +107,10 @@ static void init(ThreadState thr, uptr pc, int fd, FdSync s,
100	107	unref(thr, pc, d->sync);
101	108	d->sync = 0;
102	109	}
	110	unref(thr, pc,
	111	reinterpret_cast<FdSync *>(
	112	atomic_load(&d->aux_sync, memory_order_relaxed)));
	113	atomic_store(&d->aux_sync, 0, memory_order_relaxed);
103	114	if (flags()->io_sync == 0) {
104	115	unref(thr, pc, s);
105	116	} else if (flags()->io_sync == 1) {
...	...	@@ -110,12 +121,18 @@ static void init(ThreadState thr, uptr pc, int fd, FdSync s,
110	121	}
111	122	d->creation_tid = thr->tid;
112	123	d->creation_stack = CurrentStackId(thr, pc);
	124	d->closed = false;
	125	// This prevents false positives on fd_close_norace3.cpp test.
	126	// The mechanics of the false positive are not completely clear,
	127	// but it happens only if global reset is enabled (flush_memory_ms=1)
	128	// and may be related to lost writes during asynchronous MADV_DONTNEED.
	129	SlotLocker locker(thr);
113	130	if (write) {
114	131	// To catch races between fd usage and open.
115	132	MemoryRangeImitateWrite(thr, pc, (uptr)d, 8);
116	133	} else {
117	134	// See the dup-related comment in FdClose.
118		MemoryRead(thr, pc, (uptr)d, kSizeLog8);
	135	MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead \| kAccessSlotLocked);
119	136	}
120	137	}
121	138
...	...	@@ -140,7 +157,7 @@ void FdOnFork(ThreadState *thr, uptr pc) {
140	157	}
141	158	}
142	159
143		bool FdLocation(uptr addr, int fd, int tid, u32 *stack) {
	160	bool FdLocation(uptr addr, int fd, Tid tid, StackID stack, bool closed) {
144	161	for (int l1 = 0; l1 < kTableSizeL1; l1++) {
145	162	FdDesc tab = (FdDesc)atomic_load(&fdctx.tab[l1], memory_order_relaxed);
146	163	if (tab == 0)
...	...	@@ -151,6 +168,7 @@ bool FdLocation(uptr addr, int fd, int tid, u32 *stack) {
151	168	fd = l1 kTableSizeL1 + l2;
152	169	*tid = d->creation_tid;
153	170	*stack = d->creation_stack;
	171	*closed = d->closed;
154	172	return true;
155	173	}
156	174	}
...	...	@@ -163,7 +181,7 @@ void FdAcquire(ThreadState *thr, uptr pc, int fd) {
163	181	FdDesc *d = fddesc(thr, pc, fd);
164	182	FdSync *s = d->sync;
165	183	DPrintf("#%d: FdAcquire(%d) -> %p\n", thr->tid, fd, s);
166		MemoryRead(thr, pc, (uptr)d, kSizeLog8);
	184	MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
167	185	if (s)
168	186	Acquire(thr, pc, (uptr)s);
169	187	}
...	...	@@ -174,9 +192,11 @@ void FdRelease(ThreadState *thr, uptr pc, int fd) {
174	192	FdDesc *d = fddesc(thr, pc, fd);
175	193	FdSync *s = d->sync;
176	194	DPrintf("#%d: FdRelease(%d) -> %p\n", thr->tid, fd, s);
177		MemoryRead(thr, pc, (uptr)d, kSizeLog8);
	195	MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
178	196	if (s)
179	197	Release(thr, pc, (uptr)s);
	198	if (uptr aux_sync = atomic_load(&d->aux_sync, memory_order_acquire))
	199	Release(thr, pc, aux_sync);
180	200	}
181	201
182	202	void FdAccess(ThreadState *thr, uptr pc, int fd) {
...	...	@@ -184,7 +204,7 @@ void FdAccess(ThreadState *thr, uptr pc, int fd) {
184	204	if (bogusfd(fd))
185	205	return;
186	206	FdDesc *d = fddesc(thr, pc, fd);
187		MemoryRead(thr, pc, (uptr)d, kSizeLog8);
	207	MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
188	208	}
189	209
190	210	void FdClose(ThreadState *thr, uptr pc, int fd, bool write) {
...	...	@@ -192,27 +212,42 @@ void FdClose(ThreadState *thr, uptr pc, int fd, bool write) {
192	212	if (bogusfd(fd))
193	213	return;
194	214	FdDesc *d = fddesc(thr, pc, fd);
195		if (write) {
196		// To catch races between fd usage and close.
197		MemoryWrite(thr, pc, (uptr)d, kSizeLog8);
198		} else {
199		// This path is used only by dup2/dup3 calls.
200		// We do read instead of write because there is a number of legitimate
201		// cases where write would lead to false positives:
202		// 1. Some software dups a closed pipe in place of a socket before closing
203		// the socket (to prevent races actually).
204		// 2. Some daemons dup /dev/null in place of stdin/stdout.
205		// On the other hand we have not seen cases when write here catches real
206		// bugs.
207		MemoryRead(thr, pc, (uptr)d, kSizeLog8);
	215	{
	216	// Need to lock the slot to make MemoryAccess and MemoryResetRange atomic
	217	// with respect to global reset. See the comment in MemoryRangeFreed.
	218	SlotLocker locker(thr);
	219	if (!MustIgnoreInterceptor(thr)) {
	220	if (write) {
	221	// To catch races between fd usage and close.
	222	MemoryAccess(thr, pc, (uptr)d, 8,
	223	kAccessWrite \| kAccessCheckOnly \| kAccessSlotLocked);
	224	} else {
	225	// This path is used only by dup2/dup3 calls.
	226	// We do read instead of write because there is a number of legitimate
	227	// cases where write would lead to false positives:
	228	// 1. Some software dups a closed pipe in place of a socket before
	229	// closing
	230	// the socket (to prevent races actually).
	231	// 2. Some daemons dup /dev/null in place of stdin/stdout.
	232	// On the other hand we have not seen cases when write here catches real
	233	// bugs.
	234	MemoryAccess(thr, pc, (uptr)d, 8,
	235	kAccessRead \| kAccessCheckOnly \| kAccessSlotLocked);
	236	}
	237	}
	238	// We need to clear it, because if we do not intercept any call out there
	239	// that creates fd, we will hit false postives.
	240	MemoryResetRange(thr, pc, (uptr)d, 8);
208	241	}
209		// We need to clear it, because if we do not intercept any call out there
210		// that creates fd, we will hit false postives.
211		MemoryResetRange(thr, pc, (uptr)d, 8);
212	242	unref(thr, pc, d->sync);
213	243	d->sync = 0;
214		d->creation_tid = 0;
215		d->creation_stack = 0;
	244	unref(thr, pc,
	245	reinterpret_cast<FdSync *>(
	246	atomic_load(&d->aux_sync, memory_order_relaxed)));
	247	atomic_store(&d->aux_sync, 0, memory_order_relaxed);
	248	d->closed = true;
	249	d->creation_tid = thr->tid;
	250	d->creation_stack = CurrentStackId(thr, pc);
216	251	}
217	252
218	253	void FdFileCreate(ThreadState *thr, uptr pc, int fd) {
...	...	@@ -228,7 +263,7 @@ void FdDup(ThreadState *thr, uptr pc, int oldfd, int newfd, bool write) {
228	263	return;
229	264	// Ignore the case when user dups not yet connected socket.
230	265	FdDesc *od = fddesc(thr, pc, oldfd);
231		MemoryRead(thr, pc, (uptr)od, kSizeLog8);
	266	MemoryAccess(thr, pc, (uptr)od, 8, kAccessRead);
232	267	FdClose(thr, pc, newfd, write);
233	268	init(thr, pc, newfd, ref(od->sync), write);
234	269	}
...	...	@@ -269,6 +304,30 @@ void FdPollCreate(ThreadState *thr, uptr pc, int fd) {
269	304	init(thr, pc, fd, allocsync(thr, pc));
270	305	}
271	306
	307	void FdPollAdd(ThreadState *thr, uptr pc, int epfd, int fd) {
	308	DPrintf("#%d: FdPollAdd(%d, %d)\n", thr->tid, epfd, fd);
	309	if (bogusfd(epfd) \|\| bogusfd(fd))
	310	return;
	311	FdDesc *d = fddesc(thr, pc, fd);
	312	// Associate fd with epoll fd only once.
	313	// While an fd can be associated with multiple epolls at the same time,
	314	// or with different epolls during different phases of lifetime,
	315	// synchronization semantics (and examples) of this are unclear.
	316	// So we don't support this for now.
	317	// If we change the association, it will also create lifetime management
	318	// problem for FdRelease which accesses the aux_sync.
	319	if (atomic_load(&d->aux_sync, memory_order_relaxed))
	320	return;
	321	FdDesc *epd = fddesc(thr, pc, epfd);
	322	FdSync *s = epd->sync;
	323	if (!s)
	324	return;
	325	uptr cmp = 0;
	326	if (atomic_compare_exchange_strong(
	327	&d->aux_sync, &cmp, reinterpret_cast<uptr>(s), memory_order_release))
	328	ref(s);
	329	}
	330
272	331	void FdSocketCreate(ThreadState *thr, uptr pc, int fd) {
273	332	DPrintf("#%d: FdSocketCreate(%d)\n", thr->tid, fd);
274	333	if (bogusfd(fd))

lib/tsan/tsan_fd.h+2-1

...	...	@@ -49,11 +49,12 @@ void FdEventCreate(ThreadState *thr, uptr pc, int fd);
49	49	void FdSignalCreate(ThreadState *thr, uptr pc, int fd);
50	50	void FdInotifyCreate(ThreadState *thr, uptr pc, int fd);
51	51	void FdPollCreate(ThreadState *thr, uptr pc, int fd);
	52	void FdPollAdd(ThreadState *thr, uptr pc, int epfd, int fd);
52	53	void FdSocketCreate(ThreadState *thr, uptr pc, int fd);
53	54	void FdSocketAccept(ThreadState *thr, uptr pc, int fd, int newfd);
54	55	void FdSocketConnecting(ThreadState *thr, uptr pc, int fd);
55	56	void FdSocketConnect(ThreadState *thr, uptr pc, int fd);
56		bool FdLocation(uptr addr, int fd, int tid, u32 *stack);
	57	bool FdLocation(uptr addr, int fd, Tid tid, StackID stack, bool closed);
57	58	void FdOnFork(ThreadState *thr, uptr pc);
58	59
59	60	uptr File2addr(const char *path);

lib/tsan/tsan_flags.cpp+8-11

...	...	@@ -10,19 +10,21 @@
10	10	//
11	11	//===----------------------------------------------------------------------===//
12	12
13		#include "sanitizer_common/sanitizer_flags.h"
	13	#include "tsan_flags.h"
	14
14	15	#include "sanitizer_common/sanitizer_flag_parser.h"
	16	#include "sanitizer_common/sanitizer_flags.h"
15	17	#include "sanitizer_common/sanitizer_libc.h"
16		#include "tsan_flags.h"
17		#include "tsan_rtl.h"
	18	#include "tsan_interface.h"
18	19	#include "tsan_mman.h"
	20	#include "tsan_rtl.h"
19	21	#include "ubsan/ubsan_flags.h"
20	22
21	23	namespace __tsan {
22	24
23	25	// Can be overriden in frontend.
24	26	#ifdef TSAN_EXTERNAL_HOOKS
25		extern "C" const char* __tsan_default_options();
	27	extern "C" const char *__tsan_default_options();
26	28	#else
27	29	SANITIZER_WEAK_DEFAULT_IMPL
28	30	const char *__tsan_default_options() {
...	...	@@ -55,6 +57,7 @@ void InitializeFlags(Flags f, const char env, const char *env_option_name) {
55	57	// Override some common flags defaults.
56	58	CommonFlags cf;
57	59	cf.CopyFrom(*common_flags());
	60	cf.external_symbolizer_path = GetEnv("TSAN_SYMBOLIZER_PATH");
58	61	cf.allow_addr2line = true;
59	62	if (SANITIZER_GO) {
60	63	// Does not work as expected for Go: runtime handles SIGABRT and crashes.
...	...	@@ -96,7 +99,7 @@ void InitializeFlags(Flags f, const char env, const char *env_option_name) {
96	99	ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");
97	100	#endif
98	101
99		// Sanity check.
	102	// Check flags.
100	103	if (!f->report_bugs) {
101	104	f->report_thread_leaks = false;
102	105	f->report_destroy_locked = false;
...	...	@@ -109,12 +112,6 @@ void InitializeFlags(Flags f, const char env, const char *env_option_name) {
109	112
110	113	if (common_flags()->help) parser.PrintFlagDescriptions();
111	114
112		if (f->history_size < 0 \|\| f->history_size > 7) {
113		Printf("ThreadSanitizer: incorrect value for history_size"
114		" (must be [0..7])\n");
115		Die();
116		}
117
118	115	if (f->io_sync < 0 \|\| f->io_sync > 2) {
119	116	Printf("ThreadSanitizer: incorrect value for io_sync"
120	117	" (must be [0..2])\n");

lib/tsan/tsan_flags.inc+11-14

...	...	@@ -23,10 +23,6 @@ TSAN_FLAG(bool, enable_annotations, true,
23	23	TSAN_FLAG(bool, suppress_equal_stacks, true,
24	24	"Suppress a race report if we've already output another race report "
25	25	"with the same stack.")
26		TSAN_FLAG(bool, suppress_equal_addresses, true,
27		"Suppress a race report if we've already output another race report "
28		"on the same address.")
29
30	26	TSAN_FLAG(bool, report_bugs, true,
31	27	"Turns off bug reporting entirely (useful for benchmarking).")
32	28	TSAN_FLAG(bool, report_thread_leaks, true, "Report thread leaks at exit?")
...	...	@@ -43,7 +39,9 @@ TSAN_FLAG(
43	39	bool, force_seq_cst_atomics, false,
44	40	"If set, all atomics are effectively sequentially consistent (seq_cst), "
45	41	"regardless of what user actually specified.")
46		TSAN_FLAG(bool, print_benign, false, "Print matched \"benign\" races at exit.")
	42	TSAN_FLAG(bool, force_background_thread, false,
	43	"If set, eagerly launch a background thread for memory reclamation "
	44	"instead of waiting for a user call to pthread_create.")
47	45	TSAN_FLAG(bool, halt_on_error, false, "Exit after first reported error.")
48	46	TSAN_FLAG(int, atexit_sleep_ms, 1000,
49	47	"Sleep in main thread before exiting for that many ms "
...	...	@@ -60,14 +58,10 @@ TSAN_FLAG(bool, stop_on_start, false,
60	58	"Stops on start until __tsan_resume() is called (for debugging).")
61	59	TSAN_FLAG(bool, running_on_valgrind, false,
62	60	"Controls whether RunningOnValgrind() returns true or false.")
63		// There are a lot of goroutines in Go, so we use smaller history.
64	61	TSAN_FLAG(
65		int, history_size, SANITIZER_GO ? 1 : 3,
66		"Per-thread history size, controls how many previous memory accesses "
67		"are remembered per thread. Possible values are [0..7]. "
68		"history_size=0 amounts to 32K memory accesses. Each next value doubles "
69		"the amount of memory accesses, up to history_size=7 that amounts to "
70		"4M memory accesses. The default value is 2 (128K memory accesses).")
	62	uptr, history_size, 0,
	63	"Per-thread history size,"
	64	" controls how many extra previous memory accesses are remembered per thread.")
71	65	TSAN_FLAG(int, io_sync, 1,
72	66	"Controls level of synchronization implied by IO operations. "
73	67	"0 - no synchronization "
...	...	@@ -76,10 +70,13 @@ TSAN_FLAG(int, io_sync, 1,
76	70	TSAN_FLAG(bool, die_after_fork, true,
77	71	"Die after multi-threaded fork if the child creates new threads.")
78	72	TSAN_FLAG(const char *, suppressions, "", "Suppressions file name.")
79		TSAN_FLAG(bool, ignore_interceptors_accesses, SANITIZER_MAC ? true : false,
	73	TSAN_FLAG(bool, ignore_interceptors_accesses, SANITIZER_APPLE ? true : false,
80	74	"Ignore reads and writes from all interceptors.")
81		TSAN_FLAG(bool, ignore_noninstrumented_modules, SANITIZER_MAC ? true : false,
	75	TSAN_FLAG(bool, ignore_noninstrumented_modules, SANITIZER_APPLE ? true : false,
82	76	"Interceptors should only detect races when called from instrumented "
83	77	"modules.")
84	78	TSAN_FLAG(bool, shared_ptr_interceptor, true,
85	79	"Track atomic reference counting in libc++ shared_ptr and weak_ptr.")
	80	TSAN_FLAG(bool, print_full_thread_history, false,
	81	"If set, prints thread creation stacks for the threads involved in "
	82	"the report and their ancestors up to the main thread.")

lib/tsan/tsan_ignoreset.cpp+2-10

...	...	@@ -19,7 +19,7 @@ IgnoreSet::IgnoreSet()
19	19	: size_() {
20	20	}
21	21
22		void IgnoreSet::Add(u32 stack_id) {
	22	void IgnoreSet::Add(StackID stack_id) {
23	23	if (size_ == kMaxSize)
24	24	return;
25	25	for (uptr i = 0; i < size_; i++) {
...	...	@@ -29,15 +29,7 @@ void IgnoreSet::Add(u32 stack_id) {
29	29	stacks_[size_++] = stack_id;
30	30	}
31	31
32		void IgnoreSet::Reset() {
33		size_ = 0;
34		}
35
36		uptr IgnoreSet::Size() const {
37		return size_;
38		}
39
40		u32 IgnoreSet::At(uptr i) const {
	32	StackID IgnoreSet::At(uptr i) const {
41	33	CHECK_LT(i, size_);
42	34	CHECK_LE(size_, kMaxSize);
43	35	return stacks_[i];

lib/tsan/tsan_ignoreset.h+6-7

...	...	@@ -19,17 +19,16 @@ namespace __tsan {
19	19
20	20	class IgnoreSet {
21	21	public:
22		static const uptr kMaxSize = 16;
23
24	22	IgnoreSet();
25		void Add(u32 stack_id);
26		void Reset();
27		uptr Size() const;
28		u32 At(uptr i) const;
	23	void Add(StackID stack_id);
	24	void Reset() { size_ = 0; }
	25	uptr Size() const { return size_; }
	26	StackID At(uptr i) const;
29	27
30	28	private:
	29	static constexpr uptr kMaxSize = 16;
31	30	uptr size_;
32		u32 stacks_[kMaxSize];
	31	StackID stacks_[kMaxSize];
33	32	};
34	33
35	34	} // namespace __tsan

lib/tsan/tsan_ilist.h created+189

...	...	@@ -0,0 +1,189 @@
	1	//===-- tsan_ilist.h --------------------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	//===----------------------------------------------------------------------===//
	12	#ifndef TSAN_ILIST_H
	13	#define TSAN_ILIST_H
	14
	15	#include "sanitizer_common/sanitizer_internal_defs.h"
	16
	17	namespace __tsan {
	18
	19	class INode {
	20	public:
	21	INode() = default;
	22
	23	private:
	24	INode* next_ = nullptr;
	25	INode* prev_ = nullptr;
	26
	27	template <typename Base, INode Base::*Node, typename Elem>
	28	friend class IList;
	29	INode(const INode&) = delete;
	30	void operator=(const INode&) = delete;
	31	};
	32
	33	// Intrusive doubly-linked list.
	34	//
	35	// The node class (MyNode) needs to include "INode foo" field,
	36	// then the list can be declared as IList<MyNode, &MyNode::foo>.
	37	// This design allows to link MyNode into multiple lists using
	38	// different INode fields.
	39	// The optional Elem template argument allows to specify node MDT
	40	// (most derived type) if it's different from MyNode.
	41	template <typename Base, INode Base::*Node, typename Elem = Base>
	42	class IList {
	43	public:
	44	IList();
	45
	46	void PushFront(Elem* e);
	47	void PushBack(Elem* e);
	48	void Remove(Elem* e);
	49
	50	Elem* PopFront();
	51	Elem* PopBack();
	52	Elem* Front();
	53	Elem* Back();
	54
	55	// Prev links point towards front of the queue.
	56	Elem* Prev(Elem* e);
	57	// Next links point towards back of the queue.
	58	Elem* Next(Elem* e);
	59
	60	uptr Size() const;
	61	bool Empty() const;
	62	bool Queued(Elem* e) const;
	63
	64	private:
	65	INode node_;
	66	uptr size_ = 0;
	67
	68	void Push(Elem* e, INode* after);
	69	static INode* ToNode(Elem* e);
	70	static Elem* ToElem(INode* n);
	71
	72	IList(const IList&) = delete;
	73	void operator=(const IList&) = delete;
	74	};
	75
	76	template <typename Base, INode Base::*Node, typename Elem>
	77	IList<Base, Node, Elem>::IList() {
	78	node_.next_ = node_.prev_ = &node_;
	79	}
	80
	81	template <typename Base, INode Base::*Node, typename Elem>
	82	void IList<Base, Node, Elem>::PushFront(Elem* e) {
	83	Push(e, &node_);
	84	}
	85
	86	template <typename Base, INode Base::*Node, typename Elem>
	87	void IList<Base, Node, Elem>::PushBack(Elem* e) {
	88	Push(e, node_.prev_);
	89	}
	90
	91	template <typename Base, INode Base::*Node, typename Elem>
	92	void IList<Base, Node, Elem>::Push(Elem* e, INode* after) {
	93	INode* n = ToNode(e);
	94	DCHECK_EQ(n->next_, nullptr);
	95	DCHECK_EQ(n->prev_, nullptr);
	96	INode* next = after->next_;
	97	n->next_ = next;
	98	n->prev_ = after;
	99	next->prev_ = n;
	100	after->next_ = n;
	101	size_++;
	102	}
	103
	104	template <typename Base, INode Base::*Node, typename Elem>
	105	void IList<Base, Node, Elem>::Remove(Elem* e) {
	106	INode* n = ToNode(e);
	107	INode* next = n->next_;
	108	INode* prev = n->prev_;
	109	DCHECK(next);
	110	DCHECK(prev);
	111	DCHECK(size_);
	112	next->prev_ = prev;
	113	prev->next_ = next;
	114	n->prev_ = n->next_ = nullptr;
	115	size_--;
	116	}
	117
	118	template <typename Base, INode Base::*Node, typename Elem>
	119	Elem* IList<Base, Node, Elem>::PopFront() {
	120	Elem* e = Front();
	121	if (e)
	122	Remove(e);
	123	return e;
	124	}
	125
	126	template <typename Base, INode Base::*Node, typename Elem>
	127	Elem* IList<Base, Node, Elem>::PopBack() {
	128	Elem* e = Back();
	129	if (e)
	130	Remove(e);
	131	return e;
	132	}
	133
	134	template <typename Base, INode Base::*Node, typename Elem>
	135	Elem* IList<Base, Node, Elem>::Front() {
	136	return size_ ? ToElem(node_.next_) : nullptr;
	137	}
	138
	139	template <typename Base, INode Base::*Node, typename Elem>
	140	Elem* IList<Base, Node, Elem>::Back() {
	141	return size_ ? ToElem(node_.prev_) : nullptr;
	142	}
	143
	144	template <typename Base, INode Base::*Node, typename Elem>
	145	Elem* IList<Base, Node, Elem>::Prev(Elem* e) {
	146	INode* n = ToNode(e);
	147	DCHECK(n->prev_);
	148	return n->prev_ != &node_ ? ToElem(n->prev_) : nullptr;
	149	}
	150
	151	template <typename Base, INode Base::*Node, typename Elem>
	152	Elem* IList<Base, Node, Elem>::Next(Elem* e) {
	153	INode* n = ToNode(e);
	154	DCHECK(n->next_);
	155	return n->next_ != &node_ ? ToElem(n->next_) : nullptr;
	156	}
	157
	158	template <typename Base, INode Base::*Node, typename Elem>
	159	uptr IList<Base, Node, Elem>::Size() const {
	160	return size_;
	161	}
	162
	163	template <typename Base, INode Base::*Node, typename Elem>
	164	bool IList<Base, Node, Elem>::Empty() const {
	165	return size_ == 0;
	166	}
	167
	168	template <typename Base, INode Base::*Node, typename Elem>
	169	bool IList<Base, Node, Elem>::Queued(Elem* e) const {
	170	INode* n = ToNode(e);
	171	DCHECK_EQ(!n->next_, !n->prev_);
	172	return n->next_;
	173	}
	174
	175	template <typename Base, INode Base::*Node, typename Elem>
	176	INode* IList<Base, Node, Elem>::ToNode(Elem* e) {
	177	return &(e->*Node);
	178	}
	179
	180	template <typename Base, INode Base::*Node, typename Elem>
	181	Elem* IList<Base, Node, Elem>::ToElem(INode* n) {
	182	return static_cast<Elem>(reinterpret_cast<Base>(
	183	reinterpret_cast<uptr>(n) -
	184	reinterpret_cast<uptr>(&(reinterpret_cast<Elem>(0)->Node))));
	185	}
	186
	187	} // namespace __tsan
	188
	189	#endif

lib/tsan/tsan_interceptors.h+83-27

...	...	@@ -10,44 +10,71 @@ class ScopedInterceptor {
10	10	public:
11	11	ScopedInterceptor(ThreadState thr, const char fname, uptr pc);
12	12	~ScopedInterceptor();
13		void DisableIgnores();
14		void EnableIgnores();
	13	void DisableIgnores() {
	14	if (UNLIKELY(ignoring_))
	15	DisableIgnoresImpl();
	16	}
	17	void EnableIgnores() {
	18	if (UNLIKELY(ignoring_))
	19	EnableIgnoresImpl();
	20	}
	21
15	22	private:
16	23	ThreadState *const thr_;
17		const uptr pc_;
18		bool in_ignored_lib_;
19		bool ignoring_;
	24	bool in_ignored_lib_ = false;
	25	bool in_blocking_func_ = false;
	26	bool ignoring_ = false;
	27
	28	void DisableIgnoresImpl();
	29	void EnableIgnoresImpl();
	30	};
	31
	32	struct TsanInterceptorContext {
	33	ThreadState *thr;
	34	const uptr pc;
20	35	};
21	36
22	37	LibIgnore *libignore();
23	38
24	39	#if !SANITIZER_GO
25	40	inline bool in_symbolizer() {
26		cur_thread_init();
27		return UNLIKELY(cur_thread()->in_symbolizer);
	41	return UNLIKELY(cur_thread_init()->in_symbolizer);
28	42	}
29	43	#endif
30	44
	45	inline bool MustIgnoreInterceptor(ThreadState *thr) {
	46	return !thr->is_inited \|\| thr->ignore_interceptors \|\| thr->in_ignored_lib;
	47	}
	48
31	49	} // namespace __tsan
32	50
33		#define SCOPED_INTERCEPTOR_RAW(func, ...) \
34		cur_thread_init(); \
35		ThreadState *thr = cur_thread(); \
36		const uptr caller_pc = GET_CALLER_PC(); \
37		ScopedInterceptor si(thr, #func, caller_pc); \
38		const uptr pc = GET_CURRENT_PC(); \
39		(void)pc; \
40		/**/
41
42		#define SCOPED_TSAN_INTERCEPTOR(func, ...) \
43		SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
44		if (REAL(func) == 0) { \
	51	#define SCOPED_INTERCEPTOR_RAW(func, ...) \
	52	ThreadState *thr = cur_thread_init(); \
	53	ScopedInterceptor si(thr, #func, GET_CALLER_PC()); \
	54	UNUSED const uptr pc = GET_CURRENT_PC();
	55
	56	#ifdef __powerpc64__
	57	// Debugging of crashes on powerpc after commit:
	58	// c80604f7a3 ("tsan: remove real func check from interceptors")
	59	// Somehow replacing if with DCHECK leads to strange failures in:
	60	// SanitizerCommon-tsan-powerpc64le-Linux :: Linux/ptrace.cpp
	61	// https://lab.llvm.org/buildbot/#/builders/105
	62	// https://lab.llvm.org/buildbot/#/builders/121
	63	// https://lab.llvm.org/buildbot/#/builders/57
	64	# define CHECK_REAL_FUNC(func) \
	65	if (REAL(func) == 0) { \
45	66	Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \
46		Die(); \
47		} \
48		if (!thr->is_inited \|\| thr->ignore_interceptors \|\| thr->in_ignored_lib) \
49		return REAL(func)(__VA_ARGS__); \
50		/**/
	67	Die(); \
	68	}
	69	#else
	70	# define CHECK_REAL_FUNC(func) DCHECK(REAL(func))
	71	#endif
	72
	73	#define SCOPED_TSAN_INTERCEPTOR(func, ...) \
	74	SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
	75	CHECK_REAL_FUNC(func); \
	76	if (MustIgnoreInterceptor(thr)) \
	77	return REAL(func)(__VA_ARGS__);
51	78
52	79	#define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() \
53	80	si.DisableIgnores();
...	...	@@ -57,20 +84,49 @@ inline bool in_symbolizer() {
57	84
58	85	#define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)
59	86
	87	#if SANITIZER_FREEBSD
	88	# define TSAN_INTERCEPTOR_FREEBSD_ALIAS(ret, func, ...) \
	89	TSAN_INTERCEPTOR(ret, _pthread_##func, __VA_ARGS__) \
	90	ALIAS(WRAP(pthread_##func));
	91	#else
	92	# define TSAN_INTERCEPTOR_FREEBSD_ALIAS(ret, func, ...)
	93	#endif
	94
60	95	#if SANITIZER_NETBSD
61	96	# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) \
62	97	TSAN_INTERCEPTOR(ret, __libc_##func, __VA_ARGS__) \
63		ALIAS(WRAPPER_NAME(pthread_##func));
	98	ALIAS(WRAP(pthread_##func));
64	99	# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) \
65	100	TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \
66		ALIAS(WRAPPER_NAME(pthread_##func));
	101	ALIAS(WRAP(pthread_##func));
67	102	# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...) \
68	103	TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \
69		ALIAS(WRAPPER_NAME(pthread_##func2));
	104	ALIAS(WRAP(pthread_##func2));
70	105	#else
71	106	# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...)
72	107	# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...)
73	108	# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...)
74	109	#endif
75	110
	111	#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
	112
	113	#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \
	114	(!cur_thread_init()->is_inited)
	115
	116	#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
	117	MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \
	118	((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
	119	true)
	120
	121	#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
	122	MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \
	123	((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
	124	false)
	125
	126	#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
	127	SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \
	128	TsanInterceptorContext _ctx = {thr, pc}; \
	129	ctx = (void *)&_ctx; \
	130	(void)ctx;
	131
76	132	#endif // TSAN_INTERCEPTORS_H

lib/tsan/tsan_interceptors_mac.cpp+6-5

...	...	@@ -12,12 +12,13 @@
12	12	//===----------------------------------------------------------------------===//
13	13
14	14	#include "sanitizer_common/sanitizer_platform.h"
15		#if SANITIZER_MAC
	15	#if SANITIZER_APPLE
16	16
17	17	#include "interception/interception.h"
18	18	#include "tsan_interceptors.h"
19	19	#include "tsan_interface.h"
20	20	#include "tsan_interface_ann.h"
	21	#include "tsan_spinlock_defs_mac.h"
21	22	#include "sanitizer_common/sanitizer_addrhashmap.h"
22	23
23	24	#include <errno.h>
...	...	@@ -365,7 +366,7 @@ static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) {
365	366	if (h.created()) {
366	367	ThreadIgnoreBegin(thr, pc);
367	368	h = (uptr) user_alloc(thr, pc, /size=*/1);
368		ThreadIgnoreEnd(thr, pc);
	369	ThreadIgnoreEnd(thr);
369	370	}
370	371	return *h;
371	372	}
...	...	@@ -405,8 +406,8 @@ TSAN_INTERCEPTOR(int, swapcontext, ucontext_t oucp, const ucontext_t ucp) {
405	406	{
406	407	SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp);
407	408	}
408		// Bacause of swapcontext() semantics we have no option but to copy its
409		// impementation here
	409	// Because of swapcontext() semantics we have no option but to copy its
	410	// implementation here
410	411	if (!oucp \|\| !ucp) {
411	412	errno = EINVAL;
412	413	return -1;
...	...	@@ -518,4 +519,4 @@ STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag,
518	519
519	520	} // namespace __tsan
520	521
521		#endif // SANITIZER_MAC
	522	#endif // SANITIZER_APPLE

lib/tsan/tsan_interceptors_memintrinsics.cpp created+43

...	...	@@ -0,0 +1,43 @@
	1	//===-- tsan_interceptors_posix.cpp ---------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	//===----------------------------------------------------------------------===//
	12
	13	#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS
	14
	15	#include "tsan_interceptors.h"
	16	#include "tsan_interface.h"
	17
	18	using namespace __tsan;
	19
	20	#include "sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc"
	21
	22	extern "C" {
	23
	24	void __tsan_memcpy(void dst, const void *src, uptr size) {
	25	void *ctx;
	26	#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE
	27	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
	28	#else
	29	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
	30	#endif
	31	}
	32
	33	void __tsan_memset(void dst, int c, uptr size) {
	34	void *ctx;
	35	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, c, size);
	36	}
	37
	38	void __tsan_memmove(void dst, const void *src, uptr size) {
	39	void *ctx;
	40	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
	41	}
	42
	43	} // extern "C"

lib/tsan/tsan_interceptors_posix.cpp+616-440

...	...	@@ -35,7 +35,10 @@
35	35
36	36	using namespace __tsan;
37	37
38		#if SANITIZER_FREEBSD \|\| SANITIZER_MAC
	38	DECLARE_REAL(void , memcpy, void to, const void *from, SIZE_T size)
	39	DECLARE_REAL(void , memset, void block, int c, SIZE_T size)
	40
	41	#if SANITIZER_FREEBSD \|\| SANITIZER_APPLE
39	42	#define stdout __stdoutp
40	43	#define stderr __stderrp
41	44	#endif
...	...	@@ -76,6 +79,8 @@ struct ucontext_t {
76	79	#define PTHREAD_ABI_BASE "GLIBC_2.3.2"
77	80	#elif defined(__aarch64__) \|\| SANITIZER_PPC64V2
78	81	#define PTHREAD_ABI_BASE "GLIBC_2.17"
	82	#elif SANITIZER_LOONGARCH64
	83	#define PTHREAD_ABI_BASE "GLIBC_2.36"
79	84	#endif
80	85
81	86	extern "C" int pthread_attr_init(void *attr);
...	...	@@ -90,28 +95,26 @@ DECLARE_REAL(int, pthread_mutexattr_gettype, void , void )
90	95	DECLARE_REAL(int, fflush, __sanitizer_FILE *fp)
91	96	DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size)
92	97	DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
	98	extern "C" int pthread_equal(void t1, void t2);
93	99	extern "C" void *pthread_self();
94	100	extern "C" void _exit(int status);
95	101	#if !SANITIZER_NETBSD
96	102	extern "C" int fileno_unlocked(void *stream);
97	103	extern "C" int dirfd(void *dirp);
98	104	#endif
99		#if SANITIZER_GLIBC
100		extern "C" int mallopt(int param, int value);
101		#endif
102	105	#if SANITIZER_NETBSD
103	106	extern __sanitizer_FILE __sF[];
104	107	#else
105	108	extern __sanitizer_FILE stdout, stderr;
106	109	#endif
107		#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
	110	#if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD
108	111	const int PTHREAD_MUTEX_RECURSIVE = 1;
109	112	const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
110	113	#else
111	114	const int PTHREAD_MUTEX_RECURSIVE = 2;
112	115	const int PTHREAD_MUTEX_RECURSIVE_NP = 2;
113	116	#endif
114		#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
	117	#if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD
115	118	const int EPOLL_CTL_ADD = 1;
116	119	#endif
117	120	const int SIGILL = 4;
...	...	@@ -121,17 +124,20 @@ const int SIGFPE = 8;
121	124	const int SIGSEGV = 11;
122	125	const int SIGPIPE = 13;
123	126	const int SIGTERM = 15;
124		#if defined(__mips__) \|\| SANITIZER_FREEBSD \|\| SANITIZER_MAC \|\| SANITIZER_NETBSD
	127	#if defined(__mips__) \|\| SANITIZER_FREEBSD \|\| SANITIZER_APPLE \|\| SANITIZER_NETBSD
125	128	const int SIGBUS = 10;
126	129	const int SIGSYS = 12;
127	130	#else
128	131	const int SIGBUS = 7;
129	132	const int SIGSYS = 31;
130	133	#endif
	134	#if SANITIZER_HAS_SIGINFO
	135	const int SI_TIMER = -2;
	136	#endif
131	137	void const MAP_FAILED = (void)-1;
132	138	#if SANITIZER_NETBSD
133	139	const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567;
134		#elif !SANITIZER_MAC
	140	#elif !SANITIZER_APPLE
135	141	const int PTHREAD_BARRIER_SERIAL_THREAD = -1;
136	142	#endif
137	143	const int MAP_FIXED = 0x10;
...	...	@@ -144,7 +150,7 @@ typedef __sanitizer::u16 mode_t;
144	150	# define F_TLOCK 2 /* Test and lock a region for exclusive use. */
145	151	# define F_TEST 3 /* Test a region for other processes locks. */
146	152
147		#if SANITIZER_FREEBSD \|\| SANITIZER_MAC \|\| SANITIZER_NETBSD
	153	#if SANITIZER_FREEBSD \|\| SANITIZER_APPLE \|\| SANITIZER_NETBSD
148	154	const int SA_SIGINFO = 0x40;
149	155	const int SIG_SETMASK = 3;
150	156	#elif defined(__mips__)
...	...	@@ -155,32 +161,41 @@ const int SA_SIGINFO = 4;
155	161	const int SIG_SETMASK = 2;
156	162	#endif
157	163
158		#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \
159		(cur_thread_init(), !cur_thread()->is_inited)
160
161	164	namespace __tsan {
162	165	struct SignalDesc {
163	166	bool armed;
164		bool sigaction;
165	167	__sanitizer_siginfo siginfo;
166	168	ucontext_t ctx;
167	169	};
168	170
169	171	struct ThreadSignalContext {
170	172	int int_signal_send;
171		atomic_uintptr_t in_blocking_func;
172		atomic_uintptr_t have_pending_signals;
173	173	SignalDesc pending_signals[kSigCount];
174	174	// emptyset and oldset are too big for stack.
175	175	__sanitizer_sigset_t emptyset;
176	176	__sanitizer_sigset_t oldset;
177	177	};
178	178
	179	void EnterBlockingFunc(ThreadState *thr) {
	180	for (;;) {
	181	// The order is important to not delay a signal infinitely if it's
	182	// delivered right before we set in_blocking_func. Note: we can't call
	183	// ProcessPendingSignals when in_blocking_func is set, or we can handle
	184	// a signal synchronously when we are already handling a signal.
	185	atomic_store(&thr->in_blocking_func, 1, memory_order_relaxed);
	186	if (atomic_load(&thr->pending_signals, memory_order_relaxed) == 0)
	187	break;
	188	atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
	189	ProcessPendingSignals(thr);
	190	}
	191	}
	192
179	193	// The sole reason tsan wraps atexit callbacks is to establish synchronization
180	194	// between callback setup and callback execution.
181	195	struct AtExitCtx {
182	196	void (*f)();
183	197	void *arg;
	198	uptr pc;
184	199	};
185	200
186	201	// InterceptorContext holds all global data required for interceptors.
...	...	@@ -192,7 +207,7 @@ struct InterceptorContext {
192	207	// in a single cache line if possible (it's accessed in every interceptor).
193	208	ALIGNED(64) LibIgnore libignore;
194	209	__sanitizer_sigaction sigactions[kSigCount];
195		#if !SANITIZER_MAC && !SANITIZER_NETBSD
	210	#if !SANITIZER_APPLE && !SANITIZER_NETBSD
196	211	unsigned finalize_key;
197	212	#endif
198	213
...	...	@@ -237,19 +252,37 @@ SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {}
237	252	} // namespace __tsan
238	253
239	254	static ThreadSignalContext SigCtx(ThreadState thr) {
240		ThreadSignalContext ctx = (ThreadSignalContext)thr->signal_ctx;
	255	// This function may be called reentrantly if it is interrupted by a signal
	256	// handler. Use CAS to handle the race.
	257	uptr ctx = atomic_load(&thr->signal_ctx, memory_order_relaxed);
241	258	if (ctx == 0 && !thr->is_dead) {
242		ctx = (ThreadSignalContext)MmapOrDie(sizeof(ctx), "ThreadSignalContext");
243		MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx));
244		thr->signal_ctx = ctx;
	259	uptr pctx =
	260	(uptr)MmapOrDie(sizeof(ThreadSignalContext), "ThreadSignalContext");
	261	MemoryResetRange(thr, (uptr)&SigCtx, pctx, sizeof(ThreadSignalContext));
	262	if (atomic_compare_exchange_strong(&thr->signal_ctx, &ctx, pctx,
	263	memory_order_relaxed)) {
	264	ctx = pctx;
	265	} else {
	266	UnmapOrDie((ThreadSignalContext *)pctx, sizeof(ThreadSignalContext));
	267	}
245	268	}
246		return ctx;
	269	return (ThreadSignalContext *)ctx;
247	270	}
248	271
249	272	ScopedInterceptor::ScopedInterceptor(ThreadState thr, const char fname,
250	273	uptr pc)
251		: thr_(thr), pc_(pc), in_ignored_lib_(false), ignoring_(false) {
252		Initialize(thr);
	274	: thr_(thr) {
	275	LazyInitialize(thr);
	276	if (UNLIKELY(atomic_load(&thr->in_blocking_func, memory_order_relaxed))) {
	277	// pthread_join is marked as blocking, but it's also known to call other
	278	// intercepted functions (mmap, free). If we don't reset in_blocking_func
	279	// we can get deadlocks and memory corruptions if we deliver a synchronous
	280	// signal inside of an mmap/free interceptor.
	281	// So reset it and restore it back in the destructor.
	282	// See https://github.com/google/sanitizers/issues/1540
	283	atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
	284	in_blocking_func_ = true;
	285	}
253	286	if (!thr_->is_inited) return;
254	287	if (!thr_->ignore_interceptors) FuncEntry(thr, pc);
255	288	DPrintf("#%d: intercept %s()\n", thr_->tid, fname);
...	...	@@ -262,6 +295,8 @@ ScopedInterceptor::ScopedInterceptor(ThreadState thr, const char fname,
262	295	ScopedInterceptor::~ScopedInterceptor() {
263	296	if (!thr_->is_inited) return;
264	297	DisableIgnores();
	298	if (UNLIKELY(in_blocking_func_))
	299	EnterBlockingFunc(thr_);
265	300	if (!thr_->ignore_interceptors) {
266	301	ProcessPendingSignals(thr_);
267	302	FuncExit(thr_);
...	...	@@ -269,43 +304,48 @@ ScopedInterceptor::~ScopedInterceptor() {
269	304	}
270	305	}
271	306
272		void ScopedInterceptor::EnableIgnores() {
273		if (ignoring_) {
274		ThreadIgnoreBegin(thr_, pc_, /save_stack=/false);
275		if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports++;
276		if (in_ignored_lib_) {
277		DCHECK(!thr_->in_ignored_lib);
278		thr_->in_ignored_lib = true;
279		}
	307	NOINLINE
	308	void ScopedInterceptor::EnableIgnoresImpl() {
	309	ThreadIgnoreBegin(thr_, 0);
	310	if (flags()->ignore_noninstrumented_modules)
	311	thr_->suppress_reports++;
	312	if (in_ignored_lib_) {
	313	DCHECK(!thr_->in_ignored_lib);
	314	thr_->in_ignored_lib = true;
280	315	}
281	316	}
282	317
283		void ScopedInterceptor::DisableIgnores() {
284		if (ignoring_) {
285		ThreadIgnoreEnd(thr_, pc_);
286		if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports--;
287		if (in_ignored_lib_) {
288		DCHECK(thr_->in_ignored_lib);
289		thr_->in_ignored_lib = false;
290		}
	318	NOINLINE
	319	void ScopedInterceptor::DisableIgnoresImpl() {
	320	ThreadIgnoreEnd(thr_);
	321	if (flags()->ignore_noninstrumented_modules)
	322	thr_->suppress_reports--;
	323	if (in_ignored_lib_) {
	324	DCHECK(thr_->in_ignored_lib);
	325	thr_->in_ignored_lib = false;
291	326	}
292	327	}
293	328
294	329	#define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
	330	#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
	331	# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
	332	#else
	333	# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
	334	#endif
295	335	#if SANITIZER_FREEBSD
296		# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
297		# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
298		# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
299		#elif SANITIZER_NETBSD
300		# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
301		# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \
302		INTERCEPT_FUNCTION(__libc_##func)
303		# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \
304		INTERCEPT_FUNCTION(__libc_thr_##func)
	336	# define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func) \
	337	INTERCEPT_FUNCTION(_pthread_##func)
305	338	#else
306		# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
307		# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
308		# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
	339	# define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func)
	340	#endif
	341	#if SANITIZER_NETBSD
	342	# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \
	343	INTERCEPT_FUNCTION(__libc_##func)
	344	# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \
	345	INTERCEPT_FUNCTION(__libc_thr_##func)
	346	#else
	347	# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
	348	# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
309	349	#endif
310	350
311	351	#define READ_STRING_OF_LEN(thr, pc, s, len, n) \
...	...	@@ -319,15 +359,8 @@ void ScopedInterceptor::DisableIgnores() {
319	359
320	360	struct BlockingCall {
321	361	explicit BlockingCall(ThreadState *thr)
322		: thr(thr)
323		, ctx(SigCtx(thr)) {
324		for (;;) {
325		atomic_store(&ctx->in_blocking_func, 1, memory_order_relaxed);
326		if (atomic_load(&ctx->have_pending_signals, memory_order_relaxed) == 0)
327		break;
328		atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
329		ProcessPendingSignals(thr);
330		}
	362	: thr(thr) {
	363	EnterBlockingFunc(thr);
331	364	// When we are in a "blocking call", we process signals asynchronously
332	365	// (right when they arrive). In this context we do not expect to be
333	366	// executing any user/runtime code. The known interceptor sequence when
...	...	@@ -338,11 +371,10 @@ struct BlockingCall {
338	371
339	372	~BlockingCall() {
340	373	thr->ignore_interceptors--;
341		atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
	374	atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
342	375	}
343	376
344	377	ThreadState *thr;
345		ThreadSignalContext *ctx;
346	378	};
347	379
348	380	TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) {
...	...	@@ -371,7 +403,10 @@ TSAN_INTERCEPTOR(int, pause, int fake) {
371	403	return BLOCK_REAL(pause)(fake);
372	404	}
373	405
374		static void at_exit_wrapper() {
	406	// Note: we specifically call the function in such strange way
	407	// with "installed_at" because in reports it will appear between
	408	// callback frames and the frame that installed the callback.
	409	static void at_exit_callback_installed_at() {
375	410	AtExitCtx *ctx;
376	411	{
377	412	// Ensure thread-safety.
...	...	@@ -383,16 +418,22 @@ static void at_exit_wrapper() {
383	418	interceptor_ctx()->AtExitStack.PopBack();
384	419	}
385	420
386		Acquire(cur_thread(), (uptr)0, (uptr)ctx);
	421	ThreadState *thr = cur_thread();
	422	Acquire(thr, ctx->pc, (uptr)ctx);
	423	FuncEntry(thr, ctx->pc);
387	424	((void(*)())ctx->f)();
388		InternalFree(ctx);
	425	FuncExit(thr);
	426	Free(ctx);
389	427	}
390	428
391		static void cxa_at_exit_wrapper(void *arg) {
392		Acquire(cur_thread(), 0, (uptr)arg);
	429	static void cxa_at_exit_callback_installed_at(void *arg) {
	430	ThreadState *thr = cur_thread();
393	431	AtExitCtx ctx = (AtExitCtx)arg;
	432	Acquire(thr, ctx->pc, (uptr)arg);
	433	FuncEntry(thr, ctx->pc);
394	434	((void()(void arg))ctx->f)(ctx->arg);
395		InternalFree(ctx);
	435	FuncExit(thr);
	436	Free(ctx);
396	437	}
397	438
398	439	static int setup_at_exit_wrapper(ThreadState thr, uptr pc, void(f)(),
...	...	@@ -405,7 +446,7 @@ TSAN_INTERCEPTOR(int, atexit, void (*f)()) {
405	446	// We want to setup the atexit callback even if we are in ignored lib
406	447	// or after fork.
407	448	SCOPED_INTERCEPTOR_RAW(atexit, f);
408		return setup_at_exit_wrapper(thr, pc, (void(*)())f, 0, 0);
	449	return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, 0, 0);
409	450	}
410	451	#endif
411	452
...	...	@@ -413,14 +454,15 @@ TSAN_INTERCEPTOR(int, __cxa_atexit, void (f)(void a), void arg, void dso) {
413	454	if (in_symbolizer())
414	455	return 0;
415	456	SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso);
416		return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso);
	457	return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, arg, dso);
417	458	}
418	459
419	460	static int setup_at_exit_wrapper(ThreadState thr, uptr pc, void(f)(),
420	461	void arg, void dso) {
421		AtExitCtx ctx = (AtExitCtx)InternalAlloc(sizeof(AtExitCtx));
	462	auto *ctx = New<AtExitCtx>();
422	463	ctx->f = f;
423	464	ctx->arg = arg;
	465	ctx->pc = pc;
424	466	Release(thr, pc, (uptr)ctx);
425	467	// Memory allocation in __cxa_atexit will race with free during exit,
426	468	// because we do not see synchronization around atexit callback list.
...	...	@@ -436,41 +478,44 @@ static int setup_at_exit_wrapper(ThreadState thr, uptr pc, void(f)(),
436	478	// due to atexit_mu held on exit from the calloc interceptor.
437	479	ScopedIgnoreInterceptors ignore;
438	480
439		res = REAL(__cxa_atexit)((void ()(void a))at_exit_wrapper, 0, 0);
	481	res = REAL(__cxa_atexit)((void ()(void a))at_exit_callback_installed_at,
	482	0, 0);
440	483	// Push AtExitCtx on the top of the stack of callback functions
441	484	if (!res) {
442	485	interceptor_ctx()->AtExitStack.PushBack(ctx);
443	486	}
444	487	} else {
445		res = REAL(__cxa_atexit)(cxa_at_exit_wrapper, ctx, dso);
	488	res = REAL(__cxa_atexit)(cxa_at_exit_callback_installed_at, ctx, dso);
446	489	}
447		ThreadIgnoreEnd(thr, pc);
	490	ThreadIgnoreEnd(thr);
448	491	return res;
449	492	}
450	493
451		#if !SANITIZER_MAC && !SANITIZER_NETBSD
452		static void on_exit_wrapper(int status, void *arg) {
	494	#if !SANITIZER_APPLE && !SANITIZER_NETBSD
	495	static void on_exit_callback_installed_at(int status, void *arg) {
453	496	ThreadState *thr = cur_thread();
454		uptr pc = 0;
455		Acquire(thr, pc, (uptr)arg);
456	497	AtExitCtx ctx = (AtExitCtx)arg;
	498	Acquire(thr, ctx->pc, (uptr)arg);
	499	FuncEntry(thr, ctx->pc);
457	500	((void()(int status, void arg))ctx->f)(status, ctx->arg);
458		InternalFree(ctx);
	501	FuncExit(thr);
	502	Free(ctx);
459	503	}
460	504
461	505	TSAN_INTERCEPTOR(int, on_exit, void(f)(int, void), void *arg) {
462	506	if (in_symbolizer())
463	507	return 0;
464	508	SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg);
465		AtExitCtx ctx = (AtExitCtx)InternalAlloc(sizeof(AtExitCtx));
	509	auto *ctx = New<AtExitCtx>();
466	510	ctx->f = (void(*)())f;
467	511	ctx->arg = arg;
	512	ctx->pc = GET_CALLER_PC();
468	513	Release(thr, pc, (uptr)ctx);
469	514	// Memory allocation in __cxa_atexit will race with free during exit,
470	515	// because we do not see synchronization around atexit callback list.
471	516	ThreadIgnoreBegin(thr, pc);
472		int res = REAL(on_exit)(on_exit_wrapper, ctx);
473		ThreadIgnoreEnd(thr, pc);
	517	int res = REAL(on_exit)(on_exit_callback_installed_at, ctx);
	518	ThreadIgnoreEnd(thr);
474	519	return res;
475	520	}
476	521	#define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit)
...	...	@@ -502,9 +547,7 @@ static void SetJmp(ThreadState *thr, uptr sp) {
502	547	buf->shadow_stack_pos = thr->shadow_stack_pos;
503	548	ThreadSignalContext *sctx = SigCtx(thr);
504	549	buf->int_signal_send = sctx ? sctx->int_signal_send : 0;
505		buf->in_blocking_func = sctx ?
506		atomic_load(&sctx->in_blocking_func, memory_order_relaxed) :
507		false;
	550	buf->in_blocking_func = atomic_load(&thr->in_blocking_func, memory_order_relaxed);
508	551	buf->in_signal_handler = atomic_load(&thr->in_signal_handler,
509	552	memory_order_relaxed);
510	553	}
...	...	@@ -520,11 +563,10 @@ static void LongJmp(ThreadState thr, uptr env) {
520	563	while (thr->shadow_stack_pos > buf->shadow_stack_pos)
521	564	FuncExit(thr);
522	565	ThreadSignalContext *sctx = SigCtx(thr);
523		if (sctx) {
	566	if (sctx)
524	567	sctx->int_signal_send = buf->int_signal_send;
525		atomic_store(&sctx->in_blocking_func, buf->in_blocking_func,
526		memory_order_relaxed);
527		}
	568	atomic_store(&thr->in_blocking_func, buf->in_blocking_func,
	569	memory_order_relaxed);
528	570	atomic_store(&thr->in_signal_handler, buf->in_signal_handler,
529	571	memory_order_relaxed);
530	572	JmpBufGarbageCollect(thr, buf->sp - 1); // do not collect buf->sp
...	...	@@ -536,16 +578,13 @@ static void LongJmp(ThreadState thr, uptr env) {
536	578	}
537	579
538	580	// FIXME: put everything below into a common extern "C" block?
539		extern "C" void __tsan_setjmp(uptr sp) {
540		cur_thread_init();
541		SetJmp(cur_thread(), sp);
542		}
	581	extern "C" void __tsan_setjmp(uptr sp) { SetJmp(cur_thread_init(), sp); }
543	582
544		#if SANITIZER_MAC
	583	#if SANITIZER_APPLE
545	584	TSAN_INTERCEPTOR(int, setjmp, void *env);
546	585	TSAN_INTERCEPTOR(int, _setjmp, void *env);
547	586	TSAN_INTERCEPTOR(int, sigsetjmp, void *env);
548		#else // SANITIZER_MAC
	587	#else // SANITIZER_APPLE
549	588
550	589	#if SANITIZER_NETBSD
551	590	#define setjmp_symname __setjmp14
...	...	@@ -555,59 +594,28 @@ TSAN_INTERCEPTOR(int, sigsetjmp, void *env);
555	594	#define sigsetjmp_symname sigsetjmp
556	595	#endif
557	596
558		#define TSAN_INTERCEPTOR_SETJMP_(x) __interceptor_ ## x
559		#define TSAN_INTERCEPTOR_SETJMP__(x) TSAN_INTERCEPTOR_SETJMP_(x)
560		#define TSAN_INTERCEPTOR_SETJMP TSAN_INTERCEPTOR_SETJMP__(setjmp_symname)
561		#define TSAN_INTERCEPTOR_SIGSETJMP TSAN_INTERCEPTOR_SETJMP__(sigsetjmp_symname)
562
563		#define TSAN_STRING_SETJMP SANITIZER_STRINGIFY(setjmp_symname)
564		#define TSAN_STRING_SIGSETJMP SANITIZER_STRINGIFY(sigsetjmp_symname)
565
566		// Not called. Merely to satisfy TSAN_INTERCEPT().
567		extern "C" SANITIZER_INTERFACE_ATTRIBUTE
568		int TSAN_INTERCEPTOR_SETJMP(void *env);
569		extern "C" int TSAN_INTERCEPTOR_SETJMP(void *env) {
570		CHECK(0);
571		return 0;
572		}
573
574		// FIXME: any reason to have a separate declaration?
575		extern "C" SANITIZER_INTERFACE_ATTRIBUTE
576		int __interceptor__setjmp(void *env);
577		extern "C" int __interceptor__setjmp(void *env) {
578		CHECK(0);
579		return 0;
580		}
581
582		extern "C" SANITIZER_INTERFACE_ATTRIBUTE
583		int TSAN_INTERCEPTOR_SIGSETJMP(void *env);
584		extern "C" int TSAN_INTERCEPTOR_SIGSETJMP(void *env) {
585		CHECK(0);
586		return 0;
587		}
588
589		#if !SANITIZER_NETBSD
590		extern "C" SANITIZER_INTERFACE_ATTRIBUTE
591		int __interceptor___sigsetjmp(void *env);
592		extern "C" int __interceptor___sigsetjmp(void *env) {
593		CHECK(0);
594		return 0;
595		}
596		#endif
597
598		extern "C" int setjmp_symname(void *env);
599		extern "C" int _setjmp(void *env);
600		extern "C" int sigsetjmp_symname(void *env);
601		#if !SANITIZER_NETBSD
602		extern "C" int __sigsetjmp(void *env);
603		#endif
604	597	DEFINE_REAL(int, setjmp_symname, void *env)
605	598	DEFINE_REAL(int, _setjmp, void *env)
606	599	DEFINE_REAL(int, sigsetjmp_symname, void *env)
607	600	#if !SANITIZER_NETBSD
608	601	DEFINE_REAL(int, __sigsetjmp, void *env)
609	602	#endif
610		#endif // SANITIZER_MAC
	603
	604	// The real interceptor for setjmp is special, and implemented in pure asm. We
	605	// just need to initialize the REAL functions so that they can be used in asm.
	606	static void InitializeSetjmpInterceptors() {
	607	// We can not use TSAN_INTERCEPT to get setjmp addr, because it does &setjmp and
	608	// setjmp is not present in some versions of libc.
	609	using __interception::InterceptFunction;
	610	InterceptFunction(SANITIZER_STRINGIFY(setjmp_symname), (uptr*)&REAL(setjmp_symname), 0, 0);
	611	InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0);
	612	InterceptFunction(SANITIZER_STRINGIFY(sigsetjmp_symname), (uptr*)&REAL(sigsetjmp_symname), 0,
	613	0);
	614	#if !SANITIZER_NETBSD
	615	InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0);
	616	#endif
	617	}
	618	#endif // SANITIZER_APPLE
611	619
612	620	#if SANITIZER_NETBSD
613	621	#define longjmp_symname __longjmp14
...	...	@@ -646,7 +654,7 @@ TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) {
646	654	}
647	655	#endif
648	656
649		#if !SANITIZER_MAC
	657	#if !SANITIZER_APPLE
650	658	TSAN_INTERCEPTOR(void*, malloc, uptr size) {
651	659	if (in_symbolizer())
652	660	return InternalAlloc(size);
...	...	@@ -787,10 +795,11 @@ static void mmap_interceptor(ThreadState thr, uptr pc, Mmap real_mmap,
787	795	return res;
788	796	}
789	797
790		TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) {
791		SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz);
	798	template <class Munmap>
	799	static int munmap_interceptor(ThreadState *thr, uptr pc, Munmap real_munmap,
	800	void *addr, SIZE_T sz) {
792	801	UnmapShadow(thr, (uptr)addr, sz);
793		int res = REAL(munmap)(addr, sz);
	802	int res = real_munmap(addr, sz);
794	803	return res;
795	804	}
796	805
...	...	@@ -804,7 +813,7 @@ TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
804	813	#define TSAN_MAYBE_INTERCEPT_MEMALIGN
805	814	#endif
806	815
807		#if !SANITIZER_MAC
	816	#if !SANITIZER_APPLE
808	817	TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) {
809	818	if (in_symbolizer())
810	819	return InternalAlloc(sz, nullptr, align);
...	...	@@ -835,7 +844,7 @@ TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
835	844	#define TSAN_MAYBE_INTERCEPT_PVALLOC
836	845	#endif
837	846
838		#if !SANITIZER_MAC
	847	#if !SANITIZER_APPLE
839	848	TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
840	849	if (in_symbolizer()) {
841	850	void *p = InternalAlloc(sz, nullptr, align);
...	...	@@ -849,6 +858,54 @@ TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
849	858	}
850	859	#endif
851	860
	861	// Both __cxa_guard_acquire and pthread_once 0-initialize
	862	// the object initially. pthread_once does not have any
	863	// other ABI requirements. __cxa_guard_acquire assumes
	864	// that any non-0 value in the first byte means that
	865	// initialization is completed. Contents of the remaining
	866	// bytes are up to us.
	867	constexpr u32 kGuardInit = 0;
	868	constexpr u32 kGuardDone = 1;
	869	constexpr u32 kGuardRunning = 1 << 16;
	870	constexpr u32 kGuardWaiter = 1 << 17;
	871
	872	static int guard_acquire(ThreadState thr, uptr pc, atomic_uint32_t g,
	873	bool blocking_hooks = true) {
	874	if (blocking_hooks)
	875	OnPotentiallyBlockingRegionBegin();
	876	auto on_exit = at_scope_exit([blocking_hooks] {
	877	if (blocking_hooks)
	878	OnPotentiallyBlockingRegionEnd();
	879	});
	880
	881	for (;;) {
	882	u32 cmp = atomic_load(g, memory_order_acquire);
	883	if (cmp == kGuardInit) {
	884	if (atomic_compare_exchange_strong(g, &cmp, kGuardRunning,
	885	memory_order_relaxed))
	886	return 1;
	887	} else if (cmp == kGuardDone) {
	888	if (!thr->in_ignored_lib)
	889	Acquire(thr, pc, (uptr)g);
	890	return 0;
	891	} else {
	892	if ((cmp & kGuardWaiter) \|\|
	893	atomic_compare_exchange_strong(g, &cmp, cmp \| kGuardWaiter,
	894	memory_order_relaxed))
	895	FutexWait(g, cmp \| kGuardWaiter);
	896	}
	897	}
	898	}
	899
	900	static void guard_release(ThreadState thr, uptr pc, atomic_uint32_t g,
	901	u32 v) {
	902	if (!thr->in_ignored_lib)
	903	Release(thr, pc, (uptr)g);
	904	u32 old = atomic_exchange(g, v, memory_order_release);
	905	if (old & kGuardWaiter)
	906	FutexWake(g, 1 << 30);
	907	}
	908
852	909	// __cxa_guard_acquire and friends need to be intercepted in a special way -
853	910	// regular interceptors will break statically-linked libstdc++. Linux
854	911	// interceptors are especially defined as weak functions (so that they don't
...	...	@@ -859,7 +916,7 @@ TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
859	916	// these interceptors with INTERFACE_ATTRIBUTE.
860	917	// On OS X, we don't support statically linking, so we just use a regular
861	918	// interceptor.
862		#if SANITIZER_MAC
	919	#if SANITIZER_APPLE
863	920	#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
864	921	#else
865	922	#define STDCXX_INTERCEPTOR(rettype, name, ...) \
...	...	@@ -869,31 +926,17 @@ TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
869	926	// Used in thread-safe function static initialization.
870	927	STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) {
871	928	SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g);
872		OnPotentiallyBlockingRegionBegin();
873		auto on_exit = at_scope_exit(&OnPotentiallyBlockingRegionEnd);
874		for (;;) {
875		u32 cmp = atomic_load(g, memory_order_acquire);
876		if (cmp == 0) {
877		if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed))
878		return 1;
879		} else if (cmp == 1) {
880		Acquire(thr, pc, (uptr)g);
881		return 0;
882		} else {
883		internal_sched_yield();
884		}
885		}
	929	return guard_acquire(thr, pc, g);
886	930	}
887	931
888	932	STDCXX_INTERCEPTOR(void, __cxa_guard_release, atomic_uint32_t *g) {
889	933	SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g);
890		Release(thr, pc, (uptr)g);
891		atomic_store(g, 1, memory_order_release);
	934	guard_release(thr, pc, g, kGuardDone);
892	935	}
893	936
894	937	STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) {
895	938	SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g);
896		atomic_store(g, 0, memory_order_relaxed);
	939	guard_release(thr, pc, g, kGuardInit);
897	940	}
898	941
899	942	namespace __tsan {
...	...	@@ -908,15 +951,16 @@ void DestroyThreadState() {
908	951	}
909	952
910	953	void PlatformCleanUpThreadState(ThreadState *thr) {
911		ThreadSignalContext *sctx = thr->signal_ctx;
	954	ThreadSignalContext sctx = (ThreadSignalContext )atomic_load(
	955	&thr->signal_ctx, memory_order_relaxed);
912	956	if (sctx) {
913		thr->signal_ctx = 0;
	957	atomic_store(&thr->signal_ctx, 0, memory_order_relaxed);
914	958	UnmapOrDie(sctx, sizeof(*sctx));
915	959	}
916	960	}
917	961	} // namespace __tsan
918	962
919		#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
	963	#if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
920	964	static void thread_finalize(void *v) {
921	965	uptr iter = (uptr)v;
922	966	if (iter > 1) {
...	...	@@ -935,34 +979,33 @@ static void thread_finalize(void *v) {
935	979	struct ThreadParam {
936	980	void* (callback)(void arg);
937	981	void *param;
938		atomic_uintptr_t tid;
	982	Tid tid;
	983	Semaphore created;
	984	Semaphore started;
939	985	};
940	986
941	987	extern "C" void __tsan_thread_start_func(void arg) {
942	988	ThreadParam p = (ThreadParam)arg;
943	989	void* (callback)(void arg) = p->callback;
944	990	void *param = p->param;
945		int tid = 0;
946	991	{
947		cur_thread_init();
948		ThreadState *thr = cur_thread();
	992	ThreadState *thr = cur_thread_init();
949	993	// Thread-local state is not initialized yet.
950	994	ScopedIgnoreInterceptors ignore;
951		#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
	995	#if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
952	996	ThreadIgnoreBegin(thr, 0);
953	997	if (pthread_setspecific(interceptor_ctx()->finalize_key,
954	998	(void *)GetPthreadDestructorIterations())) {
955	999	Printf("ThreadSanitizer: failed to set thread key\n");
956	1000	Die();
957	1001	}
958		ThreadIgnoreEnd(thr, 0);
	1002	ThreadIgnoreEnd(thr);
959	1003	#endif
960		while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0)
961		internal_sched_yield();
	1004	p->created.Wait();
962	1005	Processor *proc = ProcCreate();
963	1006	ProcWire(proc, thr);
964		ThreadStart(thr, tid, GetTid(), ThreadType::Regular);
965		atomic_store(&p->tid, 0, memory_order_release);
	1007	ThreadStart(thr, p->tid, GetTid(), ThreadType::Regular);
	1008	p->started.Post();
966	1009	}
967	1010	void *res = callback(param);
968	1011	// Prevent the callback from being tail called,
...	...	@@ -984,9 +1027,11 @@ TSAN_INTERCEPTOR(int, pthread_create,
984	1027	"fork is not supported. Dying (set die_after_fork=0 to override)\n");
985	1028	Die();
986	1029	} else {
987		VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded "
988		"fork is not supported (pid %d). Continuing because of "
989		"die_after_fork=0, but you are on your own\n", internal_getpid());
	1030	VPrintf(1,
	1031	"ThreadSanitizer: starting new threads after multi-threaded "
	1032	"fork is not supported (pid %lu). Continuing because of "
	1033	"die_after_fork=0, but you are on your own\n",
	1034	internal_getpid());
990	1035	}
991	1036	}
992	1037	__sanitizer_pthread_attr_t myattr;
...	...	@@ -1001,18 +1046,18 @@ TSAN_INTERCEPTOR(int, pthread_create,
1001	1046	ThreadParam p;
1002	1047	p.callback = callback;
1003	1048	p.param = param;
1004		atomic_store(&p.tid, 0, memory_order_relaxed);
	1049	p.tid = kMainTid;
1005	1050	int res = -1;
1006	1051	{
1007	1052	// Otherwise we see false positives in pthread stack manipulation.
1008	1053	ScopedIgnoreInterceptors ignore;
1009	1054	ThreadIgnoreBegin(thr, pc);
1010	1055	res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p);
1011		ThreadIgnoreEnd(thr, pc);
	1056	ThreadIgnoreEnd(thr);
1012	1057	}
1013	1058	if (res == 0) {
1014		int tid = ThreadCreate(thr, pc, (uptr)th, IsStateDetached(detached));
1015		CHECK_NE(tid, 0);
	1059	p.tid = ThreadCreate(thr, pc, (uptr )th, IsStateDetached(detached));
	1060	CHECK_NE(p.tid, kMainTid);
1016	1061	// Synchronization on p.tid serves two purposes:
1017	1062	// 1. ThreadCreate must finish before the new thread starts.
1018	1063	// Otherwise the new thread can call pthread_detach, but the pthread_t
...	...	@@ -1020,9 +1065,8 @@ TSAN_INTERCEPTOR(int, pthread_create,
1020	1065	// 2. ThreadStart must finish before this thread continues.
1021	1066	// Otherwise, this thread can call pthread_detach and reset thr->sync
1022	1067	// before the new thread got a chance to acquire from it in ThreadStart.
1023		atomic_store(&p.tid, tid, memory_order_release);
1024		while (atomic_load(&p.tid, memory_order_acquire) != 0)
1025		internal_sched_yield();
	1068	p.created.Post();
	1069	p.started.Wait();
1026	1070	}
1027	1071	if (attr == &myattr)
1028	1072	pthread_attr_destroy(&myattr);
...	...	@@ -1031,10 +1075,10 @@ TSAN_INTERCEPTOR(int, pthread_create,
1031	1075
1032	1076	TSAN_INTERCEPTOR(int, pthread_join, void th, void *ret) {
1033	1077	SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret);
1034		int tid = ThreadConsumeTid(thr, pc, (uptr)th);
	1078	Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
1035	1079	ThreadIgnoreBegin(thr, pc);
1036	1080	int res = BLOCK_REAL(pthread_join)(th, ret);
1037		ThreadIgnoreEnd(thr, pc);
	1081	ThreadIgnoreEnd(thr);
1038	1082	if (res == 0) {
1039	1083	ThreadJoin(thr, pc, tid);
1040	1084	}
...	...	@@ -1045,7 +1089,7 @@ DEFINE_REAL_PTHREAD_FUNCTIONS
1045	1089
1046	1090	TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
1047	1091	SCOPED_INTERCEPTOR_RAW(pthread_detach, th);
1048		int tid = ThreadConsumeTid(thr, pc, (uptr)th);
	1092	Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
1049	1093	int res = REAL(pthread_detach)(th);
1050	1094	if (res == 0) {
1051	1095	ThreadDetach(thr, pc, tid);
...	...	@@ -1056,7 +1100,7 @@ TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
1056	1100	TSAN_INTERCEPTOR(void, pthread_exit, void *retval) {
1057	1101	{
1058	1102	SCOPED_INTERCEPTOR_RAW(pthread_exit, retval);
1059		#if !SANITIZER_MAC && !SANITIZER_ANDROID
	1103	#if !SANITIZER_APPLE && !SANITIZER_ANDROID
1060	1104	CHECK_EQ(thr, &cur_thread_placeholder);
1061	1105	#endif
1062	1106	}
...	...	@@ -1066,10 +1110,10 @@ TSAN_INTERCEPTOR(void, pthread_exit, void *retval) {
1066	1110	#if SANITIZER_LINUX
1067	1111	TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void th, void *ret) {
1068	1112	SCOPED_INTERCEPTOR_RAW(pthread_tryjoin_np, th, ret);
1069		int tid = ThreadConsumeTid(thr, pc, (uptr)th);
	1113	Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
1070	1114	ThreadIgnoreBegin(thr, pc);
1071	1115	int res = REAL(pthread_tryjoin_np)(th, ret);
1072		ThreadIgnoreEnd(thr, pc);
	1116	ThreadIgnoreEnd(thr);
1073	1117	if (res == 0)
1074	1118	ThreadJoin(thr, pc, tid);
1075	1119	else
...	...	@@ -1080,10 +1124,10 @@ TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void th, void *ret) {
1080	1124	TSAN_INTERCEPTOR(int, pthread_timedjoin_np, void th, void *ret,
1081	1125	const struct timespec *abstime) {
1082	1126	SCOPED_INTERCEPTOR_RAW(pthread_timedjoin_np, th, ret, abstime);
1083		int tid = ThreadConsumeTid(thr, pc, (uptr)th);
	1127	Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
1084	1128	ThreadIgnoreBegin(thr, pc);
1085	1129	int res = BLOCK_REAL(pthread_timedjoin_np)(th, ret, abstime);
1086		ThreadIgnoreEnd(thr, pc);
	1130	ThreadIgnoreEnd(thr);
1087	1131	if (res == 0)
1088	1132	ThreadJoin(thr, pc, tid);
1089	1133	else
...	...	@@ -1152,9 +1196,8 @@ void CondMutexUnlockCtx<Fn>::Unlock() const {
1152	1196	// tsan code. Also ScopedInterceptor and BlockingCall destructors won't run
1153	1197	// since the thread is cancelled, so we have to manually execute them
1154	1198	// (the thread still can run some user code due to pthread_cleanup_push).
1155		ThreadSignalContext *ctx = SigCtx(thr);
1156		CHECK_EQ(atomic_load(&ctx->in_blocking_func, memory_order_relaxed), 1);
1157		atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
	1199	CHECK_EQ(atomic_load(&thr->in_blocking_func, memory_order_relaxed), 1);
	1200	atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
1158	1201	MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock);
1159	1202	// Undo BlockingCall ctor effects.
1160	1203	thr->ignore_interceptors--;
...	...	@@ -1225,7 +1268,7 @@ INTERCEPTOR(int, pthread_cond_clockwait, void c, void m,
1225	1268	#define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT
1226	1269	#endif
1227	1270
1228		#if SANITIZER_MAC
	1271	#if SANITIZER_APPLE
1229	1272	INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void c, void m,
1230	1273	void *reltime) {
1231	1274	void *cond = init_cond(c);
...	...	@@ -1292,6 +1335,19 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
1292	1335	return res;
1293	1336	}
1294	1337
	1338	TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) {
	1339	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock, m);
	1340	MutexPreLock(thr, pc, (uptr)m);
	1341	int res = REAL(pthread_mutex_lock)(m);
	1342	if (res == errno_EOWNERDEAD)
	1343	MutexRepair(thr, pc, (uptr)m);
	1344	if (res == 0 \|\| res == errno_EOWNERDEAD)
	1345	MutexPostLock(thr, pc, (uptr)m);
	1346	if (res == errno_EINVAL)
	1347	MutexInvalidAccess(thr, pc, (uptr)m);
	1348	return res;
	1349	}
	1350
1295	1351	TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
1296	1352	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
1297	1353	int res = REAL(pthread_mutex_trylock)(m);
...	...	@@ -1302,7 +1358,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
1302	1358	return res;
1303	1359	}
1304	1360
1305		#if !SANITIZER_MAC
	1361	#if !SANITIZER_APPLE
1306	1362	TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void m, void abstime) {
1307	1363	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime);
1308	1364	int res = REAL(pthread_mutex_timedlock)(m, abstime);
...	...	@@ -1313,7 +1369,44 @@ TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void m, void abstime) {
1313	1369	}
1314	1370	#endif
1315	1371
1316		#if !SANITIZER_MAC
	1372	TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) {
	1373	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_unlock, m);
	1374	MutexUnlock(thr, pc, (uptr)m);
	1375	int res = REAL(pthread_mutex_unlock)(m);
	1376	if (res == errno_EINVAL)
	1377	MutexInvalidAccess(thr, pc, (uptr)m);
	1378	return res;
	1379	}
	1380
	1381	#if SANITIZER_GLIBC
	1382	# if !__GLIBC_PREREQ(2, 34)
	1383	// glibc 2.34 applies a non-default version for the two functions. They are no
	1384	// longer expected to be intercepted by programs.
	1385	TSAN_INTERCEPTOR(int, __pthread_mutex_lock, void *m) {
	1386	SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_lock, m);
	1387	MutexPreLock(thr, pc, (uptr)m);
	1388	int res = REAL(__pthread_mutex_lock)(m);
	1389	if (res == errno_EOWNERDEAD)
	1390	MutexRepair(thr, pc, (uptr)m);
	1391	if (res == 0 \|\| res == errno_EOWNERDEAD)
	1392	MutexPostLock(thr, pc, (uptr)m);
	1393	if (res == errno_EINVAL)
	1394	MutexInvalidAccess(thr, pc, (uptr)m);
	1395	return res;
	1396	}
	1397
	1398	TSAN_INTERCEPTOR(int, __pthread_mutex_unlock, void *m) {
	1399	SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_unlock, m);
	1400	MutexUnlock(thr, pc, (uptr)m);
	1401	int res = REAL(__pthread_mutex_unlock)(m);
	1402	if (res == errno_EINVAL)
	1403	MutexInvalidAccess(thr, pc, (uptr)m);
	1404	return res;
	1405	}
	1406	# endif
	1407	#endif
	1408
	1409	#if !SANITIZER_APPLE
1317	1410	TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) {
1318	1411	SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared);
1319	1412	int res = REAL(pthread_spin_init)(m, pshared);
...	...	@@ -1396,7 +1489,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) {
1396	1489	return res;
1397	1490	}
1398	1491
1399		#if !SANITIZER_MAC
	1492	#if !SANITIZER_APPLE
1400	1493	TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void m, void abstime) {
1401	1494	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime);
1402	1495	int res = REAL(pthread_rwlock_timedrdlock)(m, abstime);
...	...	@@ -1426,7 +1519,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) {
1426	1519	return res;
1427	1520	}
1428	1521
1429		#if !SANITIZER_MAC
	1522	#if !SANITIZER_APPLE
1430	1523	TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void m, void abstime) {
1431	1524	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime);
1432	1525	int res = REAL(pthread_rwlock_timedwrlock)(m, abstime);
...	...	@@ -1444,17 +1537,17 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) {
1444	1537	return res;
1445	1538	}
1446	1539
1447		#if !SANITIZER_MAC
	1540	#if !SANITIZER_APPLE
1448	1541	TSAN_INTERCEPTOR(int, pthread_barrier_init, void b, void a, unsigned count) {
1449	1542	SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count);
1450		MemoryWrite(thr, pc, (uptr)b, kSizeLog1);
	1543	MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite);
1451	1544	int res = REAL(pthread_barrier_init)(b, a, count);
1452	1545	return res;
1453	1546	}
1454	1547
1455	1548	TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) {
1456	1549	SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b);
1457		MemoryWrite(thr, pc, (uptr)b, kSizeLog1);
	1550	MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite);
1458	1551	int res = REAL(pthread_barrier_destroy)(b);
1459	1552	return res;
1460	1553	}
...	...	@@ -1462,9 +1555,9 @@ TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) {
1462	1555	TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
1463	1556	SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b);
1464	1557	Release(thr, pc, (uptr)b);
1465		MemoryRead(thr, pc, (uptr)b, kSizeLog1);
	1558	MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead);
1466	1559	int res = REAL(pthread_barrier_wait)(b);
1467		MemoryRead(thr, pc, (uptr)b, kSizeLog1);
	1560	MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead);
1468	1561	if (res == 0 \|\| res == PTHREAD_BARRIER_SERIAL_THREAD) {
1469	1562	Acquire(thr, pc, (uptr)b);
1470	1563	}
...	...	@@ -1478,7 +1571,7 @@ TSAN_INTERCEPTOR(int, pthread_once, void o, void (f)()) {
1478	1571	return errno_EINVAL;
1479	1572	atomic_uint32_t *a;
1480	1573
1481		if (SANITIZER_MAC)
	1574	if (SANITIZER_APPLE)
1482	1575	a = static_cast<atomic_uint32_t>((void )((char *)o + sizeof(long_t)));
1483	1576	else if (SANITIZER_NETBSD)
1484	1577	a = static_cast<atomic_uint32_t*>
...	...	@@ -1486,25 +1579,16 @@ TSAN_INTERCEPTOR(int, pthread_once, void o, void (f)()) {
1486	1579	else
1487	1580	a = static_cast<atomic_uint32_t*>(o);
1488	1581
1489		u32 v = atomic_load(a, memory_order_acquire);
1490		if (v == 0 && atomic_compare_exchange_strong(a, &v, 1,
1491		memory_order_relaxed)) {
	1582	// Mac OS X appears to use pthread_once() where calling BlockingRegion hooks
	1583	// result in crashes due to too little stack space.
	1584	if (guard_acquire(thr, pc, a, !SANITIZER_APPLE)) {
1492	1585	(*f)();
1493		if (!thr->in_ignored_lib)
1494		Release(thr, pc, (uptr)o);
1495		atomic_store(a, 2, memory_order_release);
1496		} else {
1497		while (v != 2) {
1498		internal_sched_yield();
1499		v = atomic_load(a, memory_order_acquire);
1500		}
1501		if (!thr->in_ignored_lib)
1502		Acquire(thr, pc, (uptr)o);
	1586	guard_release(thr, pc, a, kGuardDone);
1503	1587	}
1504	1588	return 0;
1505	1589	}
1506	1590
1507		#if SANITIZER_LINUX && !SANITIZER_ANDROID
	1591	#if SANITIZER_GLIBC
1508	1592	TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
1509	1593	SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf);
1510	1594	if (fd > 0)
...	...	@@ -1517,20 +1601,20 @@ TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
1517	1601	#endif
1518	1602
1519	1603	TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {
1520		#if SANITIZER_FREEBSD \|\| SANITIZER_MAC \|\| SANITIZER_ANDROID \|\| SANITIZER_NETBSD
1521		SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf);
	1604	#if SANITIZER_GLIBC
	1605	SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);
1522	1606	if (fd > 0)
1523	1607	FdAccess(thr, pc, fd);
1524		return REAL(fstat)(fd, buf);
	1608	return REAL(__fxstat)(0, fd, buf);
1525	1609	#else
1526		SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);
	1610	SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf);
1527	1611	if (fd > 0)
1528	1612	FdAccess(thr, pc, fd);
1529		return REAL(__fxstat)(0, fd, buf);
	1613	return REAL(fstat)(fd, buf);
1530	1614	#endif
1531	1615	}
1532	1616
1533		#if SANITIZER_LINUX && !SANITIZER_ANDROID
	1617	#if SANITIZER_GLIBC
1534	1618	TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
1535	1619	SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
1536	1620	if (fd > 0)
...	...	@@ -1542,7 +1626,7 @@ TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
1542	1626	#define TSAN_MAYBE_INTERCEPT___FXSTAT64
1543	1627	#endif
1544	1628
1545		#if SANITIZER_LINUX && !SANITIZER_ANDROID
	1629	#if SANITIZER_GLIBC
1546	1630	TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
1547	1631	SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf);
1548	1632	if (fd > 0)
...	...	@@ -1624,7 +1708,7 @@ TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) {
1624	1708	return newfd2;
1625	1709	}
1626	1710
1627		#if !SANITIZER_MAC
	1711	#if !SANITIZER_APPLE
1628	1712	TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) {
1629	1713	SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags);
1630	1714	int newfd2 = REAL(dup3)(oldfd, newfd, flags);
...	...	@@ -1649,11 +1733,10 @@ TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) {
1649	1733
1650	1734	#if SANITIZER_LINUX
1651	1735	TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) {
1652		SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags);
1653		if (fd >= 0)
1654		FdClose(thr, pc, fd);
	1736	SCOPED_INTERCEPTOR_RAW(signalfd, fd, mask, flags);
	1737	FdClose(thr, pc, fd);
1655	1738	fd = REAL(signalfd)(fd, mask, flags);
1656		if (fd >= 0)
	1739	if (!MustIgnoreInterceptor(thr))
1657	1740	FdSignalCreate(thr, pc, fd);
1658	1741	return fd;
1659	1742	}
...	...	@@ -1730,17 +1813,16 @@ TSAN_INTERCEPTOR(int, listen, int fd, int backlog) {
1730	1813	}
1731	1814
1732	1815	TSAN_INTERCEPTOR(int, close, int fd) {
1733		SCOPED_TSAN_INTERCEPTOR(close, fd);
1734		if (fd >= 0)
	1816	SCOPED_INTERCEPTOR_RAW(close, fd);
	1817	if (!in_symbolizer())
1735	1818	FdClose(thr, pc, fd);
1736	1819	return REAL(close)(fd);
1737	1820	}
1738	1821
1739	1822	#if SANITIZER_LINUX
1740	1823	TSAN_INTERCEPTOR(int, __close, int fd) {
1741		SCOPED_TSAN_INTERCEPTOR(__close, fd);
1742		if (fd >= 0)
1743		FdClose(thr, pc, fd);
	1824	SCOPED_INTERCEPTOR_RAW(__close, fd);
	1825	FdClose(thr, pc, fd);
1744	1826	return REAL(__close)(fd);
1745	1827	}
1746	1828	#define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close)
...	...	@@ -1751,13 +1833,10 @@ TSAN_INTERCEPTOR(int, __close, int fd) {
1751	1833	// glibc guts
1752	1834	#if SANITIZER_LINUX && !SANITIZER_ANDROID
1753	1835	TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) {
1754		SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr);
	1836	SCOPED_INTERCEPTOR_RAW(__res_iclose, state, free_addr);
1755	1837	int fds[64];
1756	1838	int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds));
1757		for (int i = 0; i < cnt; i++) {
1758		if (fds[i] > 0)
1759		FdClose(thr, pc, fds[i]);
1760		}
	1839	for (int i = 0; i < cnt; i++) FdClose(thr, pc, fds[i]);
1761	1840	REAL(__res_iclose)(state, free_addr);
1762	1841	}
1763	1842	#define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose)
...	...	@@ -1773,7 +1852,7 @@ TSAN_INTERCEPTOR(int, pipe, int *pipefd) {
1773	1852	return res;
1774	1853	}
1775	1854
1776		#if !SANITIZER_MAC
	1855	#if !SANITIZER_APPLE
1777	1856	TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) {
1778	1857	SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags);
1779	1858	int res = REAL(pipe2)(pipefd, flags);
...	...	@@ -1838,7 +1917,7 @@ TSAN_INTERCEPTOR(int, rmdir, char *path) {
1838	1917	}
1839	1918
1840	1919	TSAN_INTERCEPTOR(int, closedir, void *dirp) {
1841		SCOPED_TSAN_INTERCEPTOR(closedir, dirp);
	1920	SCOPED_INTERCEPTOR_RAW(closedir, dirp);
1842	1921	if (dirp) {
1843	1922	int fd = dirfd(dirp);
1844	1923	FdClose(thr, pc, fd);
...	...	@@ -1869,8 +1948,10 @@ TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) {
1869	1948	FdAccess(thr, pc, epfd);
1870	1949	if (epfd >= 0 && fd >= 0)
1871	1950	FdAccess(thr, pc, fd);
1872		if (op == EPOLL_CTL_ADD && epfd >= 0)
	1951	if (op == EPOLL_CTL_ADD && epfd >= 0) {
	1952	FdPollAdd(thr, pc, epfd, fd);
1873	1953	FdRelease(thr, pc, epfd);
	1954	}
1874	1955	int res = REAL(epoll_ctl)(epfd, op, fd, ev);
1875	1956	return res;
1876	1957	}
...	...	@@ -1896,12 +1977,34 @@ TSAN_INTERCEPTOR(int, epoll_pwait, int epfd, void *ev, int cnt, int timeout,
1896	1977	return res;
1897	1978	}
1898	1979
1899		#define TSAN_MAYBE_INTERCEPT_EPOLL \
1900		TSAN_INTERCEPT(epoll_create); \
1901		TSAN_INTERCEPT(epoll_create1); \
1902		TSAN_INTERCEPT(epoll_ctl); \
1903		TSAN_INTERCEPT(epoll_wait); \
1904		TSAN_INTERCEPT(epoll_pwait)
	1980	TSAN_INTERCEPTOR(int, epoll_pwait2, int epfd, void ev, int cnt, void timeout,
	1981	void *sigmask) {
	1982	SCOPED_INTERCEPTOR_RAW(epoll_pwait2, epfd, ev, cnt, timeout, sigmask);
	1983	// This function is new and may not be present in libc and/or kernel.
	1984	// Since we effectively add it to libc (as will be probed by the program
	1985	// using dlsym or a weak function pointer) we need to handle the case
	1986	// when it's not present in the actual libc.
	1987	if (!REAL(epoll_pwait2)) {
	1988	errno = errno_ENOSYS;
	1989	return -1;
	1990	}
	1991	if (MustIgnoreInterceptor(thr))
	1992	REAL(epoll_pwait2)(epfd, ev, cnt, timeout, sigmask);
	1993	if (epfd >= 0)
	1994	FdAccess(thr, pc, epfd);
	1995	int res = BLOCK_REAL(epoll_pwait2)(epfd, ev, cnt, timeout, sigmask);
	1996	if (res > 0 && epfd >= 0)
	1997	FdAcquire(thr, pc, epfd);
	1998	return res;
	1999	}
	2000
	2001	# define TSAN_MAYBE_INTERCEPT_EPOLL \
	2002	TSAN_INTERCEPT(epoll_create); \
	2003	TSAN_INTERCEPT(epoll_create1); \
	2004	TSAN_INTERCEPT(epoll_ctl); \
	2005	TSAN_INTERCEPT(epoll_wait); \
	2006	TSAN_INTERCEPT(epoll_pwait); \
	2007	TSAN_INTERCEPT(epoll_pwait2)
1905	2008	#else
1906	2009	#define TSAN_MAYBE_INTERCEPT_EPOLL
1907	2010	#endif
...	...	@@ -1933,24 +2036,47 @@ TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set,
1933	2036
1934	2037	namespace __tsan {
1935	2038
	2039	static void ReportErrnoSpoiling(ThreadState *thr, uptr pc, int sig) {
	2040	VarSizeStackTrace stack;
	2041	// StackTrace::GetNestInstructionPc(pc) is used because return address is
	2042	// expected, OutputReport() will undo this.
	2043	ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack);
	2044	ThreadRegistryLock l(&ctx->thread_registry);
	2045	ScopedReport rep(ReportTypeErrnoInSignal);
	2046	rep.SetSigNum(sig);
	2047	if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) {
	2048	rep.AddStack(stack, true);
	2049	OutputReport(thr, rep);
	2050	}
	2051	}
	2052
1936	2053	static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
1937		bool sigact, int sig,
1938		__sanitizer_siginfo info, void uctx) {
	2054	int sig, __sanitizer_siginfo *info,
	2055	void *uctx) {
	2056	CHECK(thr->slot);
1939	2057	__sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions;
1940	2058	if (acquire)
1941	2059	Acquire(thr, 0, (uptr)&sigactions[sig]);
1942	2060	// Signals are generally asynchronous, so if we receive a signals when
1943	2061	// ignores are enabled we should disable ignores. This is critical for sync
1944		// and interceptors, because otherwise we can miss syncronization and report
	2062	// and interceptors, because otherwise we can miss synchronization and report
1945	2063	// false races.
1946	2064	int ignore_reads_and_writes = thr->ignore_reads_and_writes;
1947	2065	int ignore_interceptors = thr->ignore_interceptors;
1948	2066	int ignore_sync = thr->ignore_sync;
	2067	// For symbolizer we only process SIGSEGVs synchronously
	2068	// (bug in symbolizer or in tsan). But we want to reset
	2069	// in_symbolizer to fail gracefully. Symbolizer and user code
	2070	// use different memory allocators, so if we don't reset
	2071	// in_symbolizer we can get memory allocated with one being
	2072	// feed with another, which can cause more crashes.
	2073	int in_symbolizer = thr->in_symbolizer;
1949	2074	if (!ctx->after_multithreaded_fork) {
1950	2075	thr->ignore_reads_and_writes = 0;
1951	2076	thr->fast_state.ClearIgnoreBit();
1952	2077	thr->ignore_interceptors = 0;
1953	2078	thr->ignore_sync = 0;
	2079	thr->in_symbolizer = 0;
1954	2080	}
1955	2081	// Ensure that the handler does not spoil errno.
1956	2082	const int saved_errno = errno;
...	...	@@ -1958,13 +2084,14 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
1958	2084	// This code races with sigaction. Be careful to not read sa_sigaction twice.
1959	2085	// Also need to remember pc for reporting before the call,
1960	2086	// because the handler can reset it.
1961		volatile uptr pc =
1962		sigact ? (uptr)sigactions[sig].sigaction : (uptr)sigactions[sig].handler;
	2087	volatile uptr pc = (sigactions[sig].sa_flags & SA_SIGINFO)
	2088	? (uptr)sigactions[sig].sigaction
	2089	: (uptr)sigactions[sig].handler;
1963	2090	if (pc != sig_dfl && pc != sig_ign) {
1964		if (sigact)
1965		((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx);
1966		else
1967		((__sanitizer_sighandler_ptr)pc)(sig);
	2091	// The callback can be either sa_handler or sa_sigaction.
	2092	// They have different signatures, but we assume that passing
	2093	// additional arguments to sa_handler works and is harmless.
	2094	((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx);
1968	2095	}
1969	2096	if (!ctx->after_multithreaded_fork) {
1970	2097	thr->ignore_reads_and_writes = ignore_reads_and_writes;
...	...	@@ -1972,6 +2099,7 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
1972	2099	thr->fast_state.SetIgnoreBit();
1973	2100	thr->ignore_interceptors = ignore_interceptors;
1974	2101	thr->ignore_sync = ignore_sync;
	2102	thr->in_symbolizer = in_symbolizer;
1975	2103	}
1976	2104	// We do not detect errno spoiling for SIGTERM,
1977	2105	// because some SIGTERM handlers do spoil errno but reraise SIGTERM,
...	...	@@ -1981,27 +2109,16 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
1981	2109	// from rtl_generic_sighandler) we have not yet received the reraised
1982	2110	// signal; and it looks too fragile to intercept all ways to reraise a signal.
1983	2111	if (ShouldReport(thr, ReportTypeErrnoInSignal) && !sync && sig != SIGTERM &&
1984		errno != 99) {
1985		VarSizeStackTrace stack;
1986		// StackTrace::GetNestInstructionPc(pc) is used because return address is
1987		// expected, OutputReport() will undo this.
1988		ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack);
1989		ThreadRegistryLock l(ctx->thread_registry);
1990		ScopedReport rep(ReportTypeErrnoInSignal);
1991		if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) {
1992		rep.AddStack(stack, true);
1993		OutputReport(thr, rep);
1994		}
1995		}
	2112	errno != 99)
	2113	ReportErrnoSpoiling(thr, pc, sig);
1996	2114	errno = saved_errno;
1997	2115	}
1998	2116
1999		void ProcessPendingSignals(ThreadState *thr) {
	2117	void ProcessPendingSignalsImpl(ThreadState *thr) {
	2118	atomic_store(&thr->pending_signals, 0, memory_order_relaxed);
2000	2119	ThreadSignalContext *sctx = SigCtx(thr);
2001		if (sctx == 0 \|\|
2002		atomic_load(&sctx->have_pending_signals, memory_order_relaxed) == 0)
	2120	if (sctx == 0)
2003	2121	return;
2004		atomic_store(&sctx->have_pending_signals, 0, memory_order_relaxed);
2005	2122	atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed);
2006	2123	internal_sigfillset(&sctx->emptyset);
2007	2124	int res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->emptyset, &sctx->oldset);
...	...	@@ -2010,8 +2127,8 @@ void ProcessPendingSignals(ThreadState *thr) {
2010	2127	SignalDesc *signal = &sctx->pending_signals[sig];
2011	2128	if (signal->armed) {
2012	2129	signal->armed = false;
2013		CallUserSignalHandler(thr, false, true, signal->sigaction, sig,
2014		&signal->siginfo, &signal->ctx);
	2130	CallUserSignalHandler(thr, false, true, sig, &signal->siginfo,
	2131	&signal->ctx);
2015	2132	}
2016	2133	}
2017	2134	res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->oldset, 0);
...	...	@@ -2021,35 +2138,40 @@ void ProcessPendingSignals(ThreadState *thr) {
2021	2138
2022	2139	} // namespace __tsan
2023	2140
2024		static bool is_sync_signal(ThreadSignalContext *sctx, int sig) {
	2141	static bool is_sync_signal(ThreadSignalContext *sctx, int sig,
	2142	__sanitizer_siginfo *info) {
	2143	// If we are sending signal to ourselves, we must process it now.
	2144	if (sctx && sig == sctx->int_signal_send)
	2145	return true;
	2146	#if SANITIZER_HAS_SIGINFO
	2147	// POSIX timers can be configured to send any kind of signal; however, it
	2148	// doesn't make any sense to consider a timer signal as synchronous!
	2149	if (info->si_code == SI_TIMER)
	2150	return false;
	2151	#endif
2025	2152	return sig == SIGSEGV \|\| sig == SIGBUS \|\| sig == SIGILL \|\| sig == SIGTRAP \|\|
2026		sig == SIGABRT \|\| sig == SIGFPE \|\| sig == SIGPIPE \|\| sig == SIGSYS \|\|
2027		// If we are sending signal to ourselves, we must process it now.
2028		(sctx && sig == sctx->int_signal_send);
	2153	sig == SIGABRT \|\| sig == SIGFPE \|\| sig == SIGPIPE \|\| sig == SIGSYS;
2029	2154	}
2030	2155
2031		void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,
2032		__sanitizer_siginfo *info,
2033		void *ctx) {
2034		cur_thread_init();
2035		ThreadState *thr = cur_thread();
	2156	void sighandler(int sig, __sanitizer_siginfo info, void ctx) {
	2157	ThreadState *thr = cur_thread_init();
2036	2158	ThreadSignalContext *sctx = SigCtx(thr);
2037	2159	if (sig < 0 \|\| sig >= kSigCount) {
2038	2160	VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig);
2039	2161	return;
2040	2162	}
2041	2163	// Don't mess with synchronous signals.
2042		const bool sync = is_sync_signal(sctx, sig);
	2164	const bool sync = is_sync_signal(sctx, sig, info);
2043	2165	if (sync \|\|
2044	2166	// If we are in blocking function, we can safely process it now
2045	2167	// (but check if we are in a recursive interceptor,
2046	2168	// i.e. pthread_join()->munmap()).
2047		(sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed))) {
	2169	atomic_load(&thr->in_blocking_func, memory_order_relaxed)) {
2048	2170	atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed);
2049		if (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed)) {
2050		atomic_store(&sctx->in_blocking_func, 0, memory_order_relaxed);
2051		CallUserSignalHandler(thr, sync, true, sigact, sig, info, ctx);
2052		atomic_store(&sctx->in_blocking_func, 1, memory_order_relaxed);
	2171	if (atomic_load(&thr->in_blocking_func, memory_order_relaxed)) {
	2172	atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
	2173	CallUserSignalHandler(thr, sync, true, sig, info, ctx);
	2174	atomic_store(&thr->in_blocking_func, 1, memory_order_relaxed);
2053	2175	} else {
2054	2176	// Be very conservative with when we do acquire in this case.
2055	2177	// It's unsafe to do acquire in async handlers, because ThreadState
...	...	@@ -2057,7 +2179,7 @@ void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,
2057	2179	// SIGSYS looks relatively safe -- it's synchronous and can actually
2058	2180	// need some global state.
2059	2181	bool acq = (sig == SIGSYS);
2060		CallUserSignalHandler(thr, sync, acq, sigact, sig, info, ctx);
	2182	CallUserSignalHandler(thr, sync, acq, sig, info, ctx);
2061	2183	}
2062	2184	atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed);
2063	2185	return;
...	...	@@ -2068,23 +2190,12 @@ void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,
2068	2190	SignalDesc *signal = &sctx->pending_signals[sig];
2069	2191	if (signal->armed == false) {
2070	2192	signal->armed = true;
2071		signal->sigaction = sigact;
2072		if (info)
2073		internal_memcpy(&signal->siginfo, info, sizeof(*info));
2074		if (ctx)
2075		internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx));
2076		atomic_store(&sctx->have_pending_signals, 1, memory_order_relaxed);
	2193	internal_memcpy(&signal->siginfo, info, sizeof(*info));
	2194	internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx));
	2195	atomic_store(&thr->pending_signals, 1, memory_order_relaxed);
2077	2196	}
2078	2197	}
2079	2198
2080		static void rtl_sighandler(int sig) {
2081		rtl_generic_sighandler(false, sig, 0, 0);
2082		}
2083
2084		static void rtl_sigaction(int sig, __sanitizer_siginfo info, void ctx) {
2085		rtl_generic_sighandler(true, sig, info, ctx);
2086		}
2087
2088	2199	TSAN_INTERCEPTOR(int, raise, int sig) {
2089	2200	SCOPED_TSAN_INTERCEPTOR(raise, sig);
2090	2201	ThreadSignalContext *sctx = SigCtx(thr);
...	...	@@ -2118,11 +2229,11 @@ TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) {
2118	2229	ThreadSignalContext *sctx = SigCtx(thr);
2119	2230	CHECK_NE(sctx, 0);
2120	2231	int prev = sctx->int_signal_send;
2121		if (tid == pthread_self()) {
	2232	bool self = pthread_equal(tid, pthread_self());
	2233	if (self)
2122	2234	sctx->int_signal_send = sig;
2123		}
2124	2235	int res = REAL(pthread_kill)(tid, sig);
2125		if (tid == pthread_self()) {
	2236	if (self) {
2126	2237	CHECK_EQ(sctx->int_signal_send, sig);
2127	2238	sctx->int_signal_send = prev;
2128	2239	}
...	...	@@ -2143,7 +2254,7 @@ TSAN_INTERCEPTOR(int, getaddrinfo, void node, void service,
2143	2254	// inside of getaddrinfo. So ignore memory accesses.
2144	2255	ThreadIgnoreBegin(thr, pc);
2145	2256	int res = REAL(getaddrinfo)(node, service, hints, rv);
2146		ThreadIgnoreEnd(thr, pc);
	2257	ThreadIgnoreEnd(thr);
2147	2258	return res;
2148	2259	}
2149	2260
...	...	@@ -2175,10 +2286,11 @@ void atfork_child() {
2175	2286	return;
2176	2287	ThreadState *thr = cur_thread();
2177	2288	const uptr pc = StackTrace::GetCurrentPc();
2178		ForkChildAfter(thr, pc);
	2289	ForkChildAfter(thr, pc, true);
2179	2290	FdOnFork(thr, pc);
2180	2291	}
2181	2292
	2293	#if !SANITIZER_IOS
2182	2294	TSAN_INTERCEPTOR(int, vfork, int fake) {
2183	2295	// Some programs (e.g. openjdk) call close for all file descriptors
2184	2296	// in the child process. Under tsan it leads to false positives, because
...	...	@@ -2195,8 +2307,40 @@ TSAN_INTERCEPTOR(int, vfork, int fake) {
2195	2307	// Instead we simply turn vfork into fork.
2196	2308	return WRAP(fork)(fake);
2197	2309	}
	2310	#endif
	2311
	2312	#if SANITIZER_LINUX
	2313	TSAN_INTERCEPTOR(int, clone, int (fn)(void ), void *stack, int flags,
	2314	void arg, int parent_tid, void tls, pid_t child_tid) {
	2315	SCOPED_INTERCEPTOR_RAW(clone, fn, stack, flags, arg, parent_tid, tls,
	2316	child_tid);
	2317	struct Arg {
	2318	int (fn)(void );
	2319	void *arg;
	2320	};
	2321	auto wrapper = +[](void *p) -> int {
	2322	auto *thr = cur_thread();
	2323	uptr pc = GET_CURRENT_PC();
	2324	// Start the background thread for fork, but not for clone.
	2325	// For fork we did this always and it's known to work (or user code has
	2326	// adopted). But if we do this for the new clone interceptor some code
	2327	// (sandbox2) fails. So model we used to do for years and don't start the
	2328	// background thread after clone.
	2329	ForkChildAfter(thr, pc, false);
	2330	FdOnFork(thr, pc);
	2331	auto arg = static_cast<Arg >(p);
	2332	return arg->fn(arg->arg);
	2333	};
	2334	ForkBefore(thr, pc);
	2335	Arg arg_wrapper = {fn, arg};
	2336	int pid = REAL(clone)(wrapper, stack, flags, &arg_wrapper, parent_tid, tls,
	2337	child_tid);
	2338	ForkParentAfter(thr, pc);
	2339	return pid;
	2340	}
	2341	#endif
2198	2342
2199		#if !SANITIZER_MAC && !SANITIZER_ANDROID
	2343	#if !SANITIZER_APPLE && !SANITIZER_ANDROID
2200	2344	typedef int (dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info info, SIZE_T size,
2201	2345	void *data);
2202	2346	struct dl_iterate_phdr_data {
...	...	@@ -2207,7 +2351,7 @@ struct dl_iterate_phdr_data {
2207	2351	};
2208	2352
2209	2353	static bool IsAppNotRodata(uptr addr) {
2210		return IsAppMem(addr) && (u64)MemToShadow(addr) != kShadowRodata;
	2354	return IsAppMem(addr) && *MemToShadow(addr) != Shadow::kRodata;
2211	2355	}
2212	2356
2213	2357	static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size,
...	...	@@ -2248,13 +2392,7 @@ static int OnExit(ThreadState *thr) {
2248	2392	return status;
2249	2393	}
2250	2394
2251		struct TsanInterceptorContext {
2252		ThreadState *thr;
2253		const uptr caller_pc;
2254		const uptr pc;
2255		};
2256
2257		#if !SANITIZER_MAC
	2395	#if !SANITIZER_APPLE
2258	2396	static void HandleRecvmsg(ThreadState *thr, uptr pc,
2259	2397	__sanitizer_msghdr *msg) {
2260	2398	int fds[64];
...	...	@@ -2275,33 +2413,16 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
2275	2413	#define SANITIZER_INTERCEPT_TLS_GET_OFFSET 1
2276	2414	#undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK
2277	2415
2278		#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
2279	2416	#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
2280	2417	INTERCEPT_FUNCTION_VER(name, ver)
2281	2418	#define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \
2282	2419	(INTERCEPT_FUNCTION_VER(name, ver) \|\| INTERCEPT_FUNCTION(name))
2283	2420
2284		#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
2285		MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \
2286		((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
2287		true)
2288
2289		#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
2290		MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \
2291		((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
2292		false)
2293
2294		#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
2295		SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \
2296		TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \
2297		ctx = (void *)&_ctx; \
2298		(void) ctx;
2299
2300	2421	#define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \
2301	2422	SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
2302		TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \
	2423	TsanInterceptorContext _ctx = {thr, pc}; \
2303	2424	ctx = (void *)&_ctx; \
2304		(void) ctx;
	2425	(void)ctx;
2305	2426
2306	2427	#define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \
2307	2428	if (path) \
...	...	@@ -2314,14 +2435,33 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
2314	2435	#define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \
2315	2436	if (file) { \
2316	2437	int fd = fileno_unlocked(file); \
2317		if (fd >= 0) FdClose(thr, pc, fd); \
2318		}
2319
	2438	FdClose(thr, pc, fd); \
	2439	}
	2440
	2441	#define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \
	2442	({ \
	2443	CheckNoDeepBind(filename, flag); \
	2444	ThreadIgnoreBegin(thr, 0); \
	2445	void *res = REAL(dlopen)(filename, flag); \
	2446	ThreadIgnoreEnd(thr); \
	2447	res; \
	2448	})
	2449
	2450	// Ignore interceptors in OnLibraryLoaded()/Unloaded(). These hooks use code
	2451	// (ListOfModules::init, MemoryMappingLayout::DumpListOfModules) that make
	2452	// intercepted calls, which can cause deadlockes with ReportRace() which also
	2453	// uses this code.
2320	2454	#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \
2321		libignore()->OnLibraryLoaded(filename)
	2455	({ \
	2456	ScopedIgnoreInterceptors ignore_interceptors; \
	2457	libignore()->OnLibraryLoaded(filename); \
	2458	})
2322	2459
2323		#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \
2324		libignore()->OnLibraryUnloaded()
	2460	#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \
	2461	({ \
	2462	ScopedIgnoreInterceptors ignore_interceptors; \
	2463	libignore()->OnLibraryUnloaded(); \
	2464	})
2325	2465
2326	2466	#define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \
2327	2467	Acquire(((TsanInterceptorContext *) ctx)->thr, pc, u)
...	...	@@ -2347,34 +2487,17 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
2347	2487	#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
2348	2488	ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name)
2349	2489
2350		#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
2351		__tsan::ctx->thread_registry->SetThreadNameByUserId(thread, name)
	2490	#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
	2491	if (pthread_equal(pthread_self(), reinterpret_cast<void *>(thread))) \
	2492	COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name); \
	2493	else \
	2494	__tsan::ctx->thread_registry.SetThreadNameByUserId(thread, name)
2352	2495
2353	2496	#define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name)
2354	2497
2355	2498	#define COMMON_INTERCEPTOR_ON_EXIT(ctx) \
2356	2499	OnExit(((TsanInterceptorContext *) ctx)->thr)
2357	2500
2358		#define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) \
2359		MutexPreLock(((TsanInterceptorContext *)ctx)->thr, \
2360		((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2361
2362		#define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) \
2363		MutexPostLock(((TsanInterceptorContext *)ctx)->thr, \
2364		((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2365
2366		#define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \
2367		MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \
2368		((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2369
2370		#define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \
2371		MutexRepair(((TsanInterceptorContext *)ctx)->thr, \
2372		((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2373
2374		#define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) \
2375		MutexInvalidAccess(((TsanInterceptorContext *)ctx)->thr, \
2376		((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2377
2378	2501	#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \
2379	2502	off) \
2380	2503	do { \
...	...	@@ -2382,7 +2505,12 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
2382	2505	off); \
2383	2506	} while (false)
2384	2507
2385		#if !SANITIZER_MAC
	2508	#define COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz) \
	2509	do { \
	2510	return munmap_interceptor(thr, pc, REAL(munmap), addr, sz); \
	2511	} while (false)
	2512
	2513	#if !SANITIZER_APPLE
2386	2514	#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \
2387	2515	HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \
2388	2516	((TsanInterceptorContext *)ctx)->pc, msg)
...	...	@@ -2415,12 +2543,14 @@ static __sanitizer_sighandler_ptr signal_impl(int sig,
2415	2543	#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \
2416	2544	{ return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); }
2417	2545
	2546	#define SIGNAL_INTERCEPTOR_ENTER() LazyInitialize(cur_thread_init())
	2547
2418	2548	#include "sanitizer_common/sanitizer_signal_interceptors.inc"
2419	2549
2420	2550	int sigaction_impl(int sig, const __sanitizer_sigaction *act,
2421	2551	__sanitizer_sigaction *old) {
2422	2552	// Note: if we call REAL(sigaction) directly for any reason without proxying
2423		// the signal handler through rtl_sigaction, very bad things will happen.
	2553	// the signal handler through sighandler, very bad things will happen.
2424	2554	// The handler will run synchronously and corrupt tsan per-thread state.
2425	2555	SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old);
2426	2556	if (sig <= 0 \|\| sig >= kSigCount) {
...	...	@@ -2443,27 +2573,22 @@ int sigaction_impl(int sig, const __sanitizer_sigaction *act,
2443	2573	sigactions[sig].sa_flags = (volatile int const )&act->sa_flags;
2444	2574	internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask,
2445	2575	sizeof(sigactions[sig].sa_mask));
2446		#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
	2576	#if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD
2447	2577	sigactions[sig].sa_restorer = act->sa_restorer;
2448	2578	#endif
2449	2579	internal_memcpy(&newact, act, sizeof(newact));
2450	2580	internal_sigfillset(&newact.sa_mask);
2451		if ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl) {
2452		if (newact.sa_flags & SA_SIGINFO)
2453		newact.sigaction = rtl_sigaction;
2454		else
2455		newact.handler = rtl_sighandler;
	2581	if ((act->sa_flags & SA_SIGINFO) \|\|
	2582	((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl)) {
	2583	newact.sa_flags \|= SA_SIGINFO;
	2584	newact.sigaction = sighandler;
2456	2585	}
2457	2586	ReleaseStore(thr, pc, (uptr)&sigactions[sig]);
2458	2587	act = &newact;
2459	2588	}
2460	2589	int res = REAL(sigaction)(sig, act, old);
2461		if (res == 0 && old) {
2462		uptr cb = (uptr)old->sigaction;
2463		if (cb == (uptr)rtl_sigaction \|\| cb == (uptr)rtl_sighandler) {
2464		internal_memcpy(old, &old_stored, sizeof(*old));
2465		}
2466		}
	2590	if (res == 0 && old && old->sigaction == sighandler)
	2591	internal_memcpy(old, &old_stored, sizeof(*old));
2467	2592	return res;
2468	2593	}
2469	2594
...	...	@@ -2479,27 +2604,23 @@ static __sanitizer_sighandler_ptr signal_impl(int sig,
2479	2604	return old.handler;
2480	2605	}
2481	2606
2482		#define TSAN_SYSCALL() \
	2607	#define TSAN_SYSCALL() \
2483	2608	ThreadState *thr = cur_thread(); \
2484		if (thr->ignore_interceptors) \
2485		return; \
2486		ScopedSyscall scoped_syscall(thr) \
2487		/**/
	2609	if (thr->ignore_interceptors) \
	2610	return; \
	2611	ScopedSyscall scoped_syscall(thr)
2488	2612
2489	2613	struct ScopedSyscall {
2490	2614	ThreadState *thr;
2491	2615
2492		explicit ScopedSyscall(ThreadState *thr)
2493		: thr(thr) {
2494		Initialize(thr);
2495		}
	2616	explicit ScopedSyscall(ThreadState *thr) : thr(thr) { LazyInitialize(thr); }
2496	2617
2497	2618	~ScopedSyscall() {
2498	2619	ProcessPendingSignals(thr);
2499	2620	}
2500	2621	};
2501	2622
2502		#if !SANITIZER_FREEBSD && !SANITIZER_MAC
	2623	#if !SANITIZER_FREEBSD && !SANITIZER_APPLE
2503	2624	static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {
2504	2625	TSAN_SYSCALL();
2505	2626	MemoryAccessRange(thr, pc, p, s, write);
...	...	@@ -2508,29 +2629,29 @@ static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {
2508	2629	static USED void syscall_acquire(uptr pc, uptr addr) {
2509	2630	TSAN_SYSCALL();
2510	2631	Acquire(thr, pc, addr);
2511		DPrintf("syscall_acquire(%p)\n", addr);
	2632	DPrintf("syscall_acquire(0x%zx))\n", addr);
2512	2633	}
2513	2634
2514	2635	static USED void syscall_release(uptr pc, uptr addr) {
2515	2636	TSAN_SYSCALL();
2516		DPrintf("syscall_release(%p)\n", addr);
	2637	DPrintf("syscall_release(0x%zx)\n", addr);
2517	2638	Release(thr, pc, addr);
2518	2639	}
2519	2640
2520	2641	static void syscall_fd_close(uptr pc, int fd) {
2521		TSAN_SYSCALL();
	2642	auto *thr = cur_thread();
2522	2643	FdClose(thr, pc, fd);
2523	2644	}
2524	2645
2525	2646	static USED void syscall_fd_acquire(uptr pc, int fd) {
2526	2647	TSAN_SYSCALL();
2527	2648	FdAcquire(thr, pc, fd);
2528		DPrintf("syscall_fd_acquire(%p)\n", fd);
	2649	DPrintf("syscall_fd_acquire(%d)\n", fd);
2529	2650	}
2530	2651
2531	2652	static USED void syscall_fd_release(uptr pc, int fd) {
2532	2653	TSAN_SYSCALL();
2533		DPrintf("syscall_fd_release(%p)\n", fd);
	2654	DPrintf("syscall_fd_release(%d)\n", fd);
2534	2655	FdRelease(thr, pc, fd);
2535	2656	}
2536	2657
...	...	@@ -2540,7 +2661,7 @@ static void syscall_post_fork(uptr pc, int pid) {
2540	2661	ThreadState *thr = cur_thread();
2541	2662	if (pid == 0) {
2542	2663	// child
2543		ForkChildAfter(thr, pc);
	2664	ForkChildAfter(thr, pc, true);
2544	2665	FdOnFork(thr, pc);
2545	2666	} else if (pid > 0) {
2546	2667	// parent
...	...	@@ -2653,6 +2774,26 @@ TSAN_INTERCEPTOR(void, thr_exit, tid_t *state) {
2653	2774	#define TSAN_MAYBE_INTERCEPT_THR_EXIT
2654	2775	#endif
2655	2776
	2777	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_init, void c, void a)
	2778	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_destroy, void *c)
	2779	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_signal, void *c)
	2780	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_broadcast, void *c)
	2781	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_wait, void c, void m)
	2782	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_init, void m, void a)
	2783	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_destroy, void *m)
	2784	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_lock, void *m)
	2785	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_trylock, void *m)
	2786	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_unlock, void *m)
	2787	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_init, void l, void a)
	2788	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_destroy, void *l)
	2789	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_rdlock, void *l)
	2790	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_tryrdlock, void *l)
	2791	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_wrlock, void *l)
	2792	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_trywrlock, void *l)
	2793	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_unlock, void *l)
	2794	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, once, void o, void (i)())
	2795	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, sigmask, int f, void n, void o)
	2796
2656	2797	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init, void c, void a)
2657	2798	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal, void *c)
2658	2799	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast, void *c)
...	...	@@ -2660,7 +2801,9 @@ TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_wait, void c, void m)
2660	2801	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy, void *c)
2661	2802	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init, void m, void a)
2662	2803	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy, void *m)
	2804	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_lock, void *m)
2663	2805	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock, void *m)
	2806	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_unlock, void *m)
2664	2807	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init, void m, void a)
2665	2808	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy, void *m)
2666	2809	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock, void *m)
...	...	@@ -2683,7 +2826,7 @@ static void finalize(void *arg) {
2683	2826	Die();
2684	2827	}
2685	2828
2686		#if !SANITIZER_MAC && !SANITIZER_ANDROID
	2829	#if !SANITIZER_APPLE && !SANITIZER_ANDROID
2687	2830	static void unreachable() {
2688	2831	Report("FATAL: ThreadSanitizer: unreachable called\n");
2689	2832	Die();
...	...	@@ -2694,35 +2837,20 @@ static void unreachable() {
2694	2837	SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {}
2695	2838
2696	2839	void InitializeInterceptors() {
2697		#if !SANITIZER_MAC
	2840	#if !SANITIZER_APPLE
2698	2841	// We need to setup it early, because functions like dlsym() can call it.
2699	2842	REAL(memset) = internal_memset;
2700	2843	REAL(memcpy) = internal_memcpy;
2701	2844	#endif
2702	2845
2703		// Instruct libc malloc to consume less memory.
2704		#if SANITIZER_GLIBC
2705		mallopt(1, 0); // M_MXFAST
2706		mallopt(-3, 32*1024); // M_MMAP_THRESHOLD
2707		#endif
2708
2709	2846	new(interceptor_ctx()) InterceptorContext();
2710	2847
2711	2848	InitializeCommonInterceptors();
2712	2849	InitializeSignalInterceptors();
2713	2850	InitializeLibdispatchInterceptors();
2714	2851
2715		#if !SANITIZER_MAC
2716		// We can not use TSAN_INTERCEPT to get setjmp addr,
2717		// because it does &setjmp and setjmp is not present in some versions of libc.
2718		using __interception::InterceptFunction;
2719		InterceptFunction(TSAN_STRING_SETJMP, (uptr*)&REAL(setjmp_symname), 0, 0);
2720		InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0);
2721		InterceptFunction(TSAN_STRING_SIGSETJMP, (uptr*)&REAL(sigsetjmp_symname), 0,
2722		0);
2723		#if !SANITIZER_NETBSD
2724		InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0);
2725		#endif
	2852	#if !SANITIZER_APPLE
	2853	InitializeSetjmpInterceptors();
2726	2854	#endif
2727	2855
2728	2856	TSAN_INTERCEPT(longjmp_symname);
...	...	@@ -2768,8 +2896,16 @@ void InitializeInterceptors() {
2768	2896
2769	2897	TSAN_INTERCEPT(pthread_mutex_init);
2770	2898	TSAN_INTERCEPT(pthread_mutex_destroy);
	2899	TSAN_INTERCEPT(pthread_mutex_lock);
2771	2900	TSAN_INTERCEPT(pthread_mutex_trylock);
2772	2901	TSAN_INTERCEPT(pthread_mutex_timedlock);
	2902	TSAN_INTERCEPT(pthread_mutex_unlock);
	2903	#if SANITIZER_GLIBC
	2904	# if !__GLIBC_PREREQ(2, 34)
	2905	TSAN_INTERCEPT(__pthread_mutex_lock);
	2906	TSAN_INTERCEPT(__pthread_mutex_unlock);
	2907	# endif
	2908	#endif
2773	2909
2774	2910	TSAN_INTERCEPT(pthread_spin_init);
2775	2911	TSAN_INTERCEPT(pthread_spin_destroy);
...	...	@@ -2843,6 +2979,9 @@ void InitializeInterceptors() {
2843	2979
2844	2980	TSAN_INTERCEPT(fork);
2845	2981	TSAN_INTERCEPT(vfork);
	2982	#if SANITIZER_LINUX
	2983	TSAN_INTERCEPT(clone);
	2984	#endif
2846	2985	#if !SANITIZER_ANDROID
2847	2986	TSAN_INTERCEPT(dl_iterate_phdr);
2848	2987	#endif
...	...	@@ -2862,7 +3001,7 @@ void InitializeInterceptors() {
2862	3001	TSAN_MAYBE_INTERCEPT__LWP_EXIT;
2863	3002	TSAN_MAYBE_INTERCEPT_THR_EXIT;
2864	3003
2865		#if !SANITIZER_MAC && !SANITIZER_ANDROID
	3004	#if !SANITIZER_APPLE && !SANITIZER_ANDROID
2866	3005	// Need to setup it, because interceptors check that the function is resolved.
2867	3006	// But atexit is emitted directly into the module, so can't be resolved.
2868	3007	REAL(atexit) = (int()(void()()))unreachable;
...	...	@@ -2877,13 +3016,33 @@ void InitializeInterceptors() {
2877	3016	Die();
2878	3017	}
2879	3018
2880		#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
	3019	#if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
2881	3020	if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) {
2882	3021	Printf("ThreadSanitizer: failed to create thread key\n");
2883	3022	Die();
2884	3023	}
2885	3024	#endif
2886	3025
	3026	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_init);
	3027	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_destroy);
	3028	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_signal);
	3029	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_broadcast);
	3030	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_wait);
	3031	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_init);
	3032	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_destroy);
	3033	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_lock);
	3034	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_trylock);
	3035	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_unlock);
	3036	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_init);
	3037	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_destroy);
	3038	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_rdlock);
	3039	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_tryrdlock);
	3040	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_wrlock);
	3041	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_trywrlock);
	3042	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_unlock);
	3043	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(once);
	3044	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(sigmask);
	3045
2887	3046	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init);
2888	3047	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal);
2889	3048	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast);
...	...	@@ -2891,7 +3050,9 @@ void InitializeInterceptors() {
2891	3050	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy);
2892	3051	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init);
2893	3052	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy);
	3053	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_lock);
2894	3054	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock);
	3055	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_unlock);
2895	3056	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init);
2896	3057	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy);
2897	3058	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock);
...	...	@@ -2920,25 +3081,40 @@ void InitializeInterceptors() {
2920	3081	// Note that no_sanitize_thread attribute does not turn off atomic interception
2921	3082	// so attaching it to the function defined in user code does not help.
2922	3083	// That's why we now have what we have.
2923		extern "C" SANITIZER_INTERFACE_ATTRIBUTE
2924		void __tsan_testonly_barrier_init(u64 *barrier, u32 count) {
2925		if (count >= (1 << 8)) {
2926		Printf("barrier_init: count is too large (%d)\n", count);
2927		Die();
	3084	constexpr u32 kBarrierThreadBits = 10;
	3085	constexpr u32 kBarrierThreads = 1 << kBarrierThreadBits;
	3086
	3087	extern "C" {
	3088
	3089	SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_init(
	3090	atomic_uint32_t *barrier, u32 num_threads) {
	3091	if (num_threads >= kBarrierThreads) {
	3092	Printf("barrier_init: count is too large (%d)\n", num_threads);
	3093	Die();
2928	3094	}
2929		// 8 lsb is thread count, the remaining are count of entered threads.
2930		*barrier = count;
	3095	// kBarrierThreadBits lsb is thread count,
	3096	// the remaining are count of entered threads.
	3097	atomic_store(barrier, num_threads, memory_order_relaxed);
	3098	}
	3099
	3100	static u32 barrier_epoch(u32 value) {
	3101	return (value >> kBarrierThreadBits) / (value & (kBarrierThreads - 1));
2931	3102	}
2932	3103
2933		extern "C" SANITIZER_INTERFACE_ATTRIBUTE
2934		void __tsan_testonly_barrier_wait(u64 *barrier) {
2935		unsigned old = __atomic_fetch_add(barrier, 1 << 8, __ATOMIC_RELAXED);
2936		unsigned old_epoch = (old >> 8) / (old & 0xff);
	3104	SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait(
	3105	atomic_uint32_t *barrier) {
	3106	u32 old = atomic_fetch_add(barrier, kBarrierThreads, memory_order_relaxed);
	3107	u32 old_epoch = barrier_epoch(old);
	3108	if (barrier_epoch(old + kBarrierThreads) != old_epoch) {
	3109	FutexWake(barrier, (1 << 30));
	3110	return;
	3111	}
2937	3112	for (;;) {
2938		unsigned cur = __atomic_load_n(barrier, __ATOMIC_RELAXED);
2939		unsigned cur_epoch = (cur >> 8) / (cur & 0xff);
2940		if (cur_epoch != old_epoch)
	3113	u32 cur = atomic_load(barrier, memory_order_relaxed);
	3114	if (barrier_epoch(cur) != old_epoch)
2941	3115	return;
2942		internal_sched_yield();
	3116	FutexWait(barrier, cur);
2943	3117	}
2944	3118	}
	3119
	3120	} // extern "C"

lib/tsan/tsan_interface.cpp+17-82

...	...	@@ -20,108 +20,43 @@
20	20
21	21	using namespace __tsan;
22	22
23		void __tsan_init() {
24		cur_thread_init();
25		Initialize(cur_thread());
26		}
	23	void __tsan_init() { Initialize(cur_thread_init()); }
27	24
28	25	void __tsan_flush_memory() {
29	26	FlushShadowMemory();
30	27	}
31	28
32		void __tsan_read16(void *addr) {
33		MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
34		MemoryRead(cur_thread(), CALLERPC, (uptr)addr + 8, kSizeLog8);
35		}
36
37		void __tsan_write16(void *addr) {
38		MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
39		MemoryWrite(cur_thread(), CALLERPC, (uptr)addr + 8, kSizeLog8);
40		}
41
42	29	void __tsan_read16_pc(void addr, void pc) {
43		MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog8);
44		MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr + 8, kSizeLog8);
	30	uptr pc_no_pac = STRIP_PAC_PC(pc);
	31	ThreadState *thr = cur_thread();
	32	MemoryAccess(thr, pc_no_pac, (uptr)addr, 8, kAccessRead);
	33	MemoryAccess(thr, pc_no_pac, (uptr)addr + 8, 8, kAccessRead);
45	34	}
46	35
47	36	void __tsan_write16_pc(void addr, void pc) {
48		MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog8);
49		MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr + 8, kSizeLog8);
	37	uptr pc_no_pac = STRIP_PAC_PC(pc);
	38	ThreadState *thr = cur_thread();
	39	MemoryAccess(thr, pc_no_pac, (uptr)addr, 8, kAccessWrite);
	40	MemoryAccess(thr, pc_no_pac, (uptr)addr + 8, 8, kAccessWrite);
50	41	}
51	42
52	43	// __tsan_unaligned_read/write calls are emitted by compiler.
53	44
54		void __tsan_unaligned_read2(const void *addr) {
55		UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, false, false);
56		}
57
58		void __tsan_unaligned_read4(const void *addr) {
59		UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, false, false);
60		}
61
62		void __tsan_unaligned_read8(const void *addr) {
63		UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, false, false);
64		}
65
66	45	void __tsan_unaligned_read16(const void *addr) {
67		UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 16, false, false);
68		}
69
70		void __tsan_unaligned_write2(void *addr) {
71		UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, true, false);
72		}
73
74		void __tsan_unaligned_write4(void *addr) {
75		UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, true, false);
76		}
77
78		void __tsan_unaligned_write8(void *addr) {
79		UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, true, false);
	46	uptr pc = CALLERPC;
	47	ThreadState *thr = cur_thread();
	48	UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessRead);
	49	UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessRead);
80	50	}
81	51
82	52	void __tsan_unaligned_write16(void *addr) {
83		UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 16, true, false);
	53	uptr pc = CALLERPC;
	54	ThreadState *thr = cur_thread();
	55	UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessWrite);
	56	UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessWrite);
84	57	}
85	58
86		// __sanitizer_unaligned_load/store are for user instrumentation.
87
88	59	extern "C" {
89		SANITIZER_INTERFACE_ATTRIBUTE
90		u16 __sanitizer_unaligned_load16(const uu16 *addr) {
91		__tsan_unaligned_read2(addr);
92		return *addr;
93		}
94
95		SANITIZER_INTERFACE_ATTRIBUTE
96		u32 __sanitizer_unaligned_load32(const uu32 *addr) {
97		__tsan_unaligned_read4(addr);
98		return *addr;
99		}
100
101		SANITIZER_INTERFACE_ATTRIBUTE
102		u64 __sanitizer_unaligned_load64(const uu64 *addr) {
103		__tsan_unaligned_read8(addr);
104		return *addr;
105		}
106
107		SANITIZER_INTERFACE_ATTRIBUTE
108		void __sanitizer_unaligned_store16(uu16 *addr, u16 v) {
109		__tsan_unaligned_write2(addr);
110		*addr = v;
111		}
112
113		SANITIZER_INTERFACE_ATTRIBUTE
114		void __sanitizer_unaligned_store32(uu32 *addr, u32 v) {
115		__tsan_unaligned_write4(addr);
116		*addr = v;
117		}
118
119		SANITIZER_INTERFACE_ATTRIBUTE
120		void __sanitizer_unaligned_store64(uu64 *addr, u64 v) {
121		__tsan_unaligned_write8(addr);
122		*addr = v;
123		}
124
125	60	SANITIZER_INTERFACE_ATTRIBUTE
126	61	void *__tsan_get_current_fiber() {
127	62	return cur_thread();

lib/tsan/tsan_interface.h+14-8

...	...	@@ -32,6 +32,9 @@ extern "C" {
32	32	// before any instrumented code is executed and before any call to malloc.
33	33	SANITIZER_INTERFACE_ATTRIBUTE void __tsan_init();
34	34
	35	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char *
	36	__tsan_default_options();
	37
35	38	SANITIZER_INTERFACE_ATTRIBUTE void __tsan_flush_memory();
36	39
37	40	SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read1(void *addr);
...	...	@@ -72,12 +75,21 @@ SANITIZER_INTERFACE_ATTRIBUTE void __tsan_vptr_read(void **vptr_p);
72	75	SANITIZER_INTERFACE_ATTRIBUTE
73	76	void __tsan_vptr_update(void *vptr_p, void new_val);
74	77
	78	SANITIZER_INTERFACE_ATTRIBUTE
	79	void __tsan_memcpy(void dest, const void *src, uptr count);
	80	SANITIZER_INTERFACE_ATTRIBUTE
	81	void __tsan_memset(void dest, int ch, uptr count);
	82	SANITIZER_INTERFACE_ATTRIBUTE
	83	void __tsan_memmove(void dest, const void *src, uptr count);
	84
75	85	SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_entry(void *call_pc);
76	86	SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_exit();
77	87
78	88	SANITIZER_INTERFACE_ATTRIBUTE void __tsan_ignore_thread_begin();
79	89	SANITIZER_INTERFACE_ATTRIBUTE void __tsan_ignore_thread_end();
80	90
	91	SANITIZER_INTERFACE_ATTRIBUTE void __tsan_on_thread_idle();
	92
81	93	SANITIZER_INTERFACE_ATTRIBUTE
82	94	void __tsan_external_register_tag(const char object_type);
83	95	SANITIZER_INTERFACE_ATTRIBUTE
...	...	@@ -95,9 +107,9 @@ SANITIZER_INTERFACE_ATTRIBUTE
95	107	void __tsan_write_range(void *addr, unsigned long size);
96	108
97	109	SANITIZER_INTERFACE_ATTRIBUTE
98		void __tsan_read_range_pc(void addr, unsigned long size, void pc); // NOLINT
	110	void __tsan_read_range_pc(void addr, unsigned long size, void pc);
99	111	SANITIZER_INTERFACE_ATTRIBUTE
100		void __tsan_write_range_pc(void addr, unsigned long size, void pc); // NOLINT
	112	void __tsan_write_range_pc(void addr, unsigned long size, void pc);
101	113
102	114	// User may provide function that would be called right when TSan detects
103	115	// an error. The argument 'report' is an opaque pointer that can be used to
...	...	@@ -417,12 +429,6 @@ SANITIZER_INTERFACE_ATTRIBUTE
417	429	void __tsan_go_atomic64_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
418	430	u8 *a);
419	431
420		SANITIZER_INTERFACE_ATTRIBUTE
421		void __tsan_on_initialize();
422
423		SANITIZER_INTERFACE_ATTRIBUTE
424		int __tsan_on_finalize(int failed);
425
426	432	} // extern "C"
427	433
428	434	} // namespace __tsan

lib/tsan/tsan_interface.inc created+190

...	...	@@ -0,0 +1,190 @@
	1	//===-- tsan_interface.inc --------------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	//===----------------------------------------------------------------------===//
	12
	13	#include "sanitizer_common/sanitizer_ptrauth.h"
	14	#include "tsan_interface.h"
	15	#include "tsan_rtl.h"
	16
	17	#define CALLERPC ((uptr)__builtin_return_address(0))
	18
	19	using namespace __tsan;
	20
	21	void __tsan_read1(void *addr) {
	22	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessRead);
	23	}
	24
	25	void __tsan_read2(void *addr) {
	26	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessRead);
	27	}
	28
	29	void __tsan_read4(void *addr) {
	30	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessRead);
	31	}
	32
	33	void __tsan_read8(void *addr) {
	34	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead);
	35	}
	36
	37	void __tsan_read16(void *addr) {
	38	MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessRead);
	39	}
	40
	41	void __tsan_write1(void *addr) {
	42	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessWrite);
	43	}
	44
	45	void __tsan_write2(void *addr) {
	46	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessWrite);
	47	}
	48
	49	void __tsan_write4(void *addr) {
	50	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessWrite);
	51	}
	52
	53	void __tsan_write8(void *addr) {
	54	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite);
	55	}
	56
	57	void __tsan_write16(void *addr) {
	58	MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessWrite);
	59	}
	60
	61	void __tsan_read1_pc(void addr, void pc) {
	62	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessRead \| kAccessExternalPC);
	63	}
	64
	65	void __tsan_read2_pc(void addr, void pc) {
	66	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 2, kAccessRead \| kAccessExternalPC);
	67	}
	68
	69	void __tsan_read4_pc(void addr, void pc) {
	70	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 4, kAccessRead \| kAccessExternalPC);
	71	}
	72
	73	void __tsan_read8_pc(void addr, void pc) {
	74	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 8, kAccessRead \| kAccessExternalPC);
	75	}
	76
	77	void __tsan_write1_pc(void addr, void pc) {
	78	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessWrite \| kAccessExternalPC);
	79	}
	80
	81	void __tsan_write2_pc(void addr, void pc) {
	82	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 2, kAccessWrite \| kAccessExternalPC);
	83	}
	84
	85	void __tsan_write4_pc(void addr, void pc) {
	86	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 4, kAccessWrite \| kAccessExternalPC);
	87	}
	88
	89	void __tsan_write8_pc(void addr, void pc) {
	90	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 8, kAccessWrite \| kAccessExternalPC);
	91	}
	92
	93	ALWAYS_INLINE USED void __tsan_unaligned_read2(const void *addr) {
	94	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessRead);
	95	}
	96
	97	ALWAYS_INLINE USED void __tsan_unaligned_read4(const void *addr) {
	98	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessRead);
	99	}
	100
	101	ALWAYS_INLINE USED void __tsan_unaligned_read8(const void *addr) {
	102	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead);
	103	}
	104
	105	ALWAYS_INLINE USED void __tsan_unaligned_write2(void *addr) {
	106	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessWrite);
	107	}
	108
	109	ALWAYS_INLINE USED void __tsan_unaligned_write4(void *addr) {
	110	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessWrite);
	111	}
	112
	113	ALWAYS_INLINE USED void __tsan_unaligned_write8(void *addr) {
	114	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite);
	115	}
	116
	117	extern "C" {
	118	// __sanitizer_unaligned_load/store are for user instrumentation.
	119	SANITIZER_INTERFACE_ATTRIBUTE
	120	u16 __sanitizer_unaligned_load16(const uu16 *addr) {
	121	__tsan_unaligned_read2(addr);
	122	return *addr;
	123	}
	124
	125	SANITIZER_INTERFACE_ATTRIBUTE
	126	u32 __sanitizer_unaligned_load32(const uu32 *addr) {
	127	__tsan_unaligned_read4(addr);
	128	return *addr;
	129	}
	130
	131	SANITIZER_INTERFACE_ATTRIBUTE
	132	u64 __sanitizer_unaligned_load64(const uu64 *addr) {
	133	__tsan_unaligned_read8(addr);
	134	return *addr;
	135	}
	136
	137	SANITIZER_INTERFACE_ATTRIBUTE
	138	void __sanitizer_unaligned_store16(uu16 *addr, u16 v) {
	139	*addr = v;
	140	__tsan_unaligned_write2(addr);
	141	}
	142
	143	SANITIZER_INTERFACE_ATTRIBUTE
	144	void __sanitizer_unaligned_store32(uu32 *addr, u32 v) {
	145	*addr = v;
	146	__tsan_unaligned_write4(addr);
	147	}
	148
	149	SANITIZER_INTERFACE_ATTRIBUTE
	150	void __sanitizer_unaligned_store64(uu64 *addr, u64 v) {
	151	*addr = v;
	152	__tsan_unaligned_write8(addr);
	153	}
	154	}
	155
	156	void __tsan_vptr_update(void *vptr_p, void new_val) {
	157	if (*vptr_p == new_val)
	158	return;
	159	MemoryAccess(cur_thread(), CALLERPC, (uptr)vptr_p, sizeof(*vptr_p),
	160	kAccessWrite \| kAccessVptr);
	161	}
	162
	163	void __tsan_vptr_read(void **vptr_p) {
	164	MemoryAccess(cur_thread(), CALLERPC, (uptr)vptr_p, sizeof(*vptr_p),
	165	kAccessRead \| kAccessVptr);
	166	}
	167
	168	void __tsan_func_entry(void *pc) { FuncEntry(cur_thread(), STRIP_PAC_PC(pc)); }
	169
	170	void __tsan_func_exit() { FuncExit(cur_thread()); }
	171
	172	void __tsan_ignore_thread_begin() { ThreadIgnoreBegin(cur_thread(), CALLERPC); }
	173
	174	void __tsan_ignore_thread_end() { ThreadIgnoreEnd(cur_thread()); }
	175
	176	void __tsan_read_range(void *addr, uptr size) {
	177	MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, false);
	178	}
	179
	180	void __tsan_write_range(void *addr, uptr size) {
	181	MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, true);
	182	}
	183
	184	void __tsan_read_range_pc(void addr, uptr size, void pc) {
	185	MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, false);
	186	}
	187
	188	void __tsan_write_range_pc(void addr, uptr size, void pc) {
	189	MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, true);
	190	}

lib/tsan/tsan_interface_ann.cpp+30-139

...	...	@@ -43,15 +43,14 @@ class ScopedAnnotation {
43	43	ThreadState *const thr_;
44	44	};
45	45
46		#define SCOPED_ANNOTATION_RET(typ, ret) \
47		if (!flags()->enable_annotations) \
48		return ret; \
49		ThreadState *thr = cur_thread(); \
50		const uptr caller_pc = (uptr)__builtin_return_address(0); \
51		ScopedAnnotation sa(thr, __func__, caller_pc); \
52		const uptr pc = StackTrace::GetCurrentPc(); \
53		(void)pc; \
54		/**/
	46	#define SCOPED_ANNOTATION_RET(typ, ret) \
	47	if (!flags()->enable_annotations) \
	48	return ret; \
	49	ThreadState *thr = cur_thread(); \
	50	const uptr caller_pc = (uptr)__builtin_return_address(0); \
	51	ScopedAnnotation sa(thr, __func__, caller_pc); \
	52	const uptr pc = StackTrace::GetCurrentPc(); \
	53	(void)pc;
55	54
56	55	#define SCOPED_ANNOTATION(typ) SCOPED_ANNOTATION_RET(typ, )
57	56
...	...	@@ -71,7 +70,6 @@ struct ExpectRace {
71	70
72	71	struct DynamicAnnContext {
73	72	Mutex mtx;
74		ExpectRace expect;
75	73	ExpectRace benign;
76	74
77	75	DynamicAnnContext() : mtx(MutexTypeAnnotations) {}
...	...	@@ -90,7 +88,7 @@ static void AddExpectRace(ExpectRace *list,
90	88	return;
91	89	}
92	90	}
93		race = (ExpectRace*)internal_alloc(MBlockExpectRace, sizeof(ExpectRace));
	91	race = static_cast<ExpectRace *>(Alloc(sizeof(ExpectRace)));
94	92	race->addr = addr;
95	93	race->size = size;
96	94	race->file = f;
...	...	@@ -137,81 +135,12 @@ static void InitList(ExpectRace *list) {
137	135
138	136	void InitializeDynamicAnnotations() {
139	137	dyn_ann_ctx = new(dyn_ann_ctx_placeholder) DynamicAnnContext;
140		InitList(&dyn_ann_ctx->expect);
141	138	InitList(&dyn_ann_ctx->benign);
142	139	}
143	140
144	141	bool IsExpectedReport(uptr addr, uptr size) {
145	142	ReadLock lock(&dyn_ann_ctx->mtx);
146		if (CheckContains(&dyn_ann_ctx->expect, addr, size))
147		return true;
148		if (CheckContains(&dyn_ann_ctx->benign, addr, size))
149		return true;
150		return false;
151		}
152
153		static void CollectMatchedBenignRaces(Vector<ExpectRace> *matched,
154		int unique_count, int hit_count, atomic_uintptr_t ExpectRace::*counter) {
155		ExpectRace *list = &dyn_ann_ctx->benign;
156		for (ExpectRace *race = list->next; race != list; race = race->next) {
157		(*unique_count)++;
158		const uptr cnt = atomic_load_relaxed(&(race->*counter));
159		if (cnt == 0)
160		continue;
161		*hit_count += cnt;
162		uptr i = 0;
163		for (; i < matched->Size(); i++) {
164		ExpectRace race0 = &(matched)[i];
165		if (race->line == race0->line
166		&& internal_strcmp(race->file, race0->file) == 0
167		&& internal_strcmp(race->desc, race0->desc) == 0) {
168		atomic_fetch_add(&(race0->*counter), cnt, memory_order_relaxed);
169		break;
170		}
171		}
172		if (i == matched->Size())
173		matched->PushBack(*race);
174		}
175		}
176
177		void PrintMatchedBenignRaces() {
178		Lock lock(&dyn_ann_ctx->mtx);
179		int unique_count = 0;
180		int hit_count = 0;
181		int add_count = 0;
182		Vector<ExpectRace> hit_matched;
183		CollectMatchedBenignRaces(&hit_matched, &unique_count, &hit_count,
184		&ExpectRace::hitcount);
185		Vector<ExpectRace> add_matched;
186		CollectMatchedBenignRaces(&add_matched, &unique_count, &add_count,
187		&ExpectRace::addcount);
188		if (hit_matched.Size()) {
189		Printf("ThreadSanitizer: Matched %d \"benign\" races (pid=%d):\n",
190		hit_count, (int)internal_getpid());
191		for (uptr i = 0; i < hit_matched.Size(); i++) {
192		Printf("%d %s:%d %s\n",
193		atomic_load_relaxed(&hit_matched[i].hitcount),
194		hit_matched[i].file, hit_matched[i].line, hit_matched[i].desc);
195		}
196		}
197		if (hit_matched.Size()) {
198		Printf("ThreadSanitizer: Annotated %d \"benign\" races, %d unique"
199		" (pid=%d):\n",
200		add_count, unique_count, (int)internal_getpid());
201		for (uptr i = 0; i < add_matched.Size(); i++) {
202		Printf("%d %s:%d %s\n",
203		atomic_load_relaxed(&add_matched[i].addcount),
204		add_matched[i].file, add_matched[i].line, add_matched[i].desc);
205		}
206		}
207		}
208
209		static void ReportMissedExpectedRace(ExpectRace *race) {
210		Printf("==================\n");
211		Printf("WARNING: ThreadSanitizer: missed expected data race\n");
212		Printf(" %s addr=%zx %s:%d\n",
213		race->desc, race->addr, race->file, race->line);
214		Printf("==================\n");
	143	return CheckContains(&dyn_ann_ctx->benign, addr, size);
215	144	}
216	145	} // namespace __tsan
217	146
...	...	@@ -229,20 +158,16 @@ void INTERFACE_ATTRIBUTE AnnotateHappensAfter(char *f, int l, uptr addr) {
229	158	}
230	159
231	160	void INTERFACE_ATTRIBUTE AnnotateCondVarSignal(char *f, int l, uptr cv) {
232		SCOPED_ANNOTATION(AnnotateCondVarSignal);
233	161	}
234	162
235	163	void INTERFACE_ATTRIBUTE AnnotateCondVarSignalAll(char *f, int l, uptr cv) {
236		SCOPED_ANNOTATION(AnnotateCondVarSignalAll);
237	164	}
238	165
239	166	void INTERFACE_ATTRIBUTE AnnotateMutexIsNotPHB(char *f, int l, uptr mu) {
240		SCOPED_ANNOTATION(AnnotateMutexIsNotPHB);
241	167	}
242	168
243	169	void INTERFACE_ATTRIBUTE AnnotateCondVarWait(char *f, int l, uptr cv,
244	170	uptr lock) {
245		SCOPED_ANNOTATION(AnnotateCondVarWait);
246	171	}
247	172
248	173	void INTERFACE_ATTRIBUTE AnnotateRWLockCreate(char *f, int l, uptr m) {
...	...	@@ -279,86 +204,56 @@ void INTERFACE_ATTRIBUTE AnnotateRWLockReleased(char *f, int l, uptr m,
279	204	}
280	205
281	206	void INTERFACE_ATTRIBUTE AnnotateTraceMemory(char *f, int l, uptr mem) {
282		SCOPED_ANNOTATION(AnnotateTraceMemory);
283	207	}
284	208
285	209	void INTERFACE_ATTRIBUTE AnnotateFlushState(char *f, int l) {
286		SCOPED_ANNOTATION(AnnotateFlushState);
287	210	}
288	211
289	212	void INTERFACE_ATTRIBUTE AnnotateNewMemory(char *f, int l, uptr mem,
290	213	uptr size) {
291		SCOPED_ANNOTATION(AnnotateNewMemory);
292	214	}
293	215
294	216	void INTERFACE_ATTRIBUTE AnnotateNoOp(char *f, int l, uptr mem) {
295		SCOPED_ANNOTATION(AnnotateNoOp);
296	217	}
297	218
298	219	void INTERFACE_ATTRIBUTE AnnotateFlushExpectedRaces(char *f, int l) {
299		SCOPED_ANNOTATION(AnnotateFlushExpectedRaces);
300		Lock lock(&dyn_ann_ctx->mtx);
301		while (dyn_ann_ctx->expect.next != &dyn_ann_ctx->expect) {
302		ExpectRace *race = dyn_ann_ctx->expect.next;
303		if (atomic_load_relaxed(&race->hitcount) == 0) {
304		ctx->nmissed_expected++;
305		ReportMissedExpectedRace(race);
306		}
307		race->prev->next = race->next;
308		race->next->prev = race->prev;
309		internal_free(race);
310		}
311	220	}
312	221
313	222	void INTERFACE_ATTRIBUTE AnnotateEnableRaceDetection(
314	223	char *f, int l, int enable) {
315		SCOPED_ANNOTATION(AnnotateEnableRaceDetection);
316		// FIXME: Reconsider this functionality later. It may be irrelevant.
317	224	}
318	225
319	226	void INTERFACE_ATTRIBUTE AnnotateMutexIsUsedAsCondVar(
320	227	char *f, int l, uptr mu) {
321		SCOPED_ANNOTATION(AnnotateMutexIsUsedAsCondVar);
322	228	}
323	229
324	230	void INTERFACE_ATTRIBUTE AnnotatePCQGet(
325	231	char *f, int l, uptr pcq) {
326		SCOPED_ANNOTATION(AnnotatePCQGet);
327	232	}
328	233
329	234	void INTERFACE_ATTRIBUTE AnnotatePCQPut(
330	235	char *f, int l, uptr pcq) {
331		SCOPED_ANNOTATION(AnnotatePCQPut);
332	236	}
333	237
334	238	void INTERFACE_ATTRIBUTE AnnotatePCQDestroy(
335	239	char *f, int l, uptr pcq) {
336		SCOPED_ANNOTATION(AnnotatePCQDestroy);
337	240	}
338	241
339	242	void INTERFACE_ATTRIBUTE AnnotatePCQCreate(
340	243	char *f, int l, uptr pcq) {
341		SCOPED_ANNOTATION(AnnotatePCQCreate);
342	244	}
343	245
344	246	void INTERFACE_ATTRIBUTE AnnotateExpectRace(
345	247	char f, int l, uptr mem, char desc) {
346		SCOPED_ANNOTATION(AnnotateExpectRace);
347		Lock lock(&dyn_ann_ctx->mtx);
348		AddExpectRace(&dyn_ann_ctx->expect,
349		f, l, mem, 1, desc);
350		DPrintf("Add expected race: %s addr=%zx %s:%d\n", desc, mem, f, l);
351	248	}
352	249
353		static void BenignRaceImpl(
354		char f, int l, uptr mem, uptr size, char desc) {
	250	static void BenignRaceImpl(char f, int l, uptr mem, uptr size, char desc) {
355	251	Lock lock(&dyn_ann_ctx->mtx);
356	252	AddExpectRace(&dyn_ann_ctx->benign,
357	253	f, l, mem, size, desc);
358	254	DPrintf("Add benign race: %s addr=%zx %s:%d\n", desc, mem, f, l);
359	255	}
360	256
361		// FIXME: Turn it off later. WTF is benign race?1?? Go talk to Hans Boehm.
362	257	void INTERFACE_ATTRIBUTE AnnotateBenignRaceSized(
363	258	char f, int l, uptr mem, uptr size, char desc) {
364	259	SCOPED_ANNOTATION(AnnotateBenignRaceSized);
...	...	@@ -378,7 +273,7 @@ void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsBegin(char *f, int l) {
378	273
379	274	void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsEnd(char *f, int l) {
380	275	SCOPED_ANNOTATION(AnnotateIgnoreReadsEnd);
381		ThreadIgnoreEnd(thr, pc);
	276	ThreadIgnoreEnd(thr);
382	277	}
383	278
384	279	void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesBegin(char *f, int l) {
...	...	@@ -388,7 +283,7 @@ void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesBegin(char *f, int l) {
388	283
389	284	void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesEnd(char *f, int l) {
390	285	SCOPED_ANNOTATION(AnnotateIgnoreWritesEnd);
391		ThreadIgnoreEnd(thr, pc);
	286	ThreadIgnoreEnd(thr);
392	287	}
393	288
394	289	void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncBegin(char *f, int l) {
...	...	@@ -398,17 +293,15 @@ void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncBegin(char *f, int l) {
398	293
399	294	void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncEnd(char *f, int l) {
400	295	SCOPED_ANNOTATION(AnnotateIgnoreSyncEnd);
401		ThreadIgnoreSyncEnd(thr, pc);
	296	ThreadIgnoreSyncEnd(thr);
402	297	}
403	298
404	299	void INTERFACE_ATTRIBUTE AnnotatePublishMemoryRange(
405	300	char *f, int l, uptr addr, uptr size) {
406		SCOPED_ANNOTATION(AnnotatePublishMemoryRange);
407	301	}
408	302
409	303	void INTERFACE_ATTRIBUTE AnnotateUnpublishMemoryRange(
410	304	char *f, int l, uptr addr, uptr size) {
411		SCOPED_ANNOTATION(AnnotateUnpublishMemoryRange);
412	305	}
413	306
414	307	void INTERFACE_ATTRIBUTE AnnotateThreadName(
...	...	@@ -421,11 +314,9 @@ void INTERFACE_ATTRIBUTE AnnotateThreadName(
421	314	// WTFAnnotateHappensAfter(). Those are being used by Webkit to annotate
422	315	// atomic operations, which should be handled by ThreadSanitizer correctly.
423	316	void INTERFACE_ATTRIBUTE WTFAnnotateHappensBefore(char *f, int l, uptr addr) {
424		SCOPED_ANNOTATION(AnnotateHappensBefore);
425	317	}
426	318
427	319	void INTERFACE_ATTRIBUTE WTFAnnotateHappensAfter(char *f, int l, uptr addr) {
428		SCOPED_ANNOTATION(AnnotateHappensAfter);
429	320	}
430	321
431	322	void INTERFACE_ATTRIBUTE WTFAnnotateBenignRaceSized(
...	...	@@ -477,15 +368,15 @@ void __tsan_mutex_pre_lock(void *m, unsigned flagz) {
477	368	else
478	369	MutexPreLock(thr, pc, (uptr)m);
479	370	}
480		ThreadIgnoreBegin(thr, pc, /save_stack=/false);
481		ThreadIgnoreSyncBegin(thr, pc, /save_stack=/false);
	371	ThreadIgnoreBegin(thr, 0);
	372	ThreadIgnoreSyncBegin(thr, 0);
482	373	}
483	374
484	375	INTERFACE_ATTRIBUTE
485	376	void __tsan_mutex_post_lock(void *m, unsigned flagz, int rec) {
486	377	SCOPED_ANNOTATION(__tsan_mutex_post_lock);
487		ThreadIgnoreSyncEnd(thr, pc);
488		ThreadIgnoreEnd(thr, pc);
	378	ThreadIgnoreSyncEnd(thr);
	379	ThreadIgnoreEnd(thr);
489	380	if (!(flagz & MutexFlagTryLockFailed)) {
490	381	if (flagz & MutexFlagReadLock)
491	382	MutexPostReadLock(thr, pc, (uptr)m, flagz);
...	...	@@ -504,44 +395,44 @@ int __tsan_mutex_pre_unlock(void *m, unsigned flagz) {
504	395	} else {
505	396	ret = MutexUnlock(thr, pc, (uptr)m, flagz);
506	397	}
507		ThreadIgnoreBegin(thr, pc, /save_stack=/false);
508		ThreadIgnoreSyncBegin(thr, pc, /save_stack=/false);
	398	ThreadIgnoreBegin(thr, 0);
	399	ThreadIgnoreSyncBegin(thr, 0);
509	400	return ret;
510	401	}
511	402
512	403	INTERFACE_ATTRIBUTE
513	404	void __tsan_mutex_post_unlock(void *m, unsigned flagz) {
514	405	SCOPED_ANNOTATION(__tsan_mutex_post_unlock);
515		ThreadIgnoreSyncEnd(thr, pc);
516		ThreadIgnoreEnd(thr, pc);
	406	ThreadIgnoreSyncEnd(thr);
	407	ThreadIgnoreEnd(thr);
517	408	}
518	409
519	410	INTERFACE_ATTRIBUTE
520	411	void __tsan_mutex_pre_signal(void *addr, unsigned flagz) {
521	412	SCOPED_ANNOTATION(__tsan_mutex_pre_signal);
522		ThreadIgnoreBegin(thr, pc, /save_stack=/false);
523		ThreadIgnoreSyncBegin(thr, pc, /save_stack=/false);
	413	ThreadIgnoreBegin(thr, 0);
	414	ThreadIgnoreSyncBegin(thr, 0);
524	415	}
525	416
526	417	INTERFACE_ATTRIBUTE
527	418	void __tsan_mutex_post_signal(void *addr, unsigned flagz) {
528	419	SCOPED_ANNOTATION(__tsan_mutex_post_signal);
529		ThreadIgnoreSyncEnd(thr, pc);
530		ThreadIgnoreEnd(thr, pc);
	420	ThreadIgnoreSyncEnd(thr);
	421	ThreadIgnoreEnd(thr);
531	422	}
532	423
533	424	INTERFACE_ATTRIBUTE
534	425	void __tsan_mutex_pre_divert(void *addr, unsigned flagz) {
535	426	SCOPED_ANNOTATION(__tsan_mutex_pre_divert);
536	427	// Exit from ignore region started in __tsan_mutex_pre_lock/unlock/signal.
537		ThreadIgnoreSyncEnd(thr, pc);
538		ThreadIgnoreEnd(thr, pc);
	428	ThreadIgnoreSyncEnd(thr);
	429	ThreadIgnoreEnd(thr);
539	430	}
540	431
541	432	INTERFACE_ATTRIBUTE
542	433	void __tsan_mutex_post_divert(void *addr, unsigned flagz) {
543	434	SCOPED_ANNOTATION(__tsan_mutex_post_divert);
544		ThreadIgnoreBegin(thr, pc, /save_stack=/false);
545		ThreadIgnoreSyncBegin(thr, pc, /save_stack=/false);
	435	ThreadIgnoreBegin(thr, 0);
	436	ThreadIgnoreSyncBegin(thr, 0);
546	437	}
547	438	} // extern "C"

lib/tsan/tsan_interface_atomic.cpp+155-181

...	...	@@ -32,6 +32,7 @@ using namespace __tsan;
32	32	static StaticSpinMutex mutex128;
33	33	#endif
34	34
	35	#if SANITIZER_DEBUG
35	36	static bool IsLoadOrder(morder mo) {
36	37	return mo == mo_relaxed \|\| mo == mo_consume
37	38	\|\| mo == mo_acquire \|\| mo == mo_seq_cst;
...	...	@@ -40,6 +41,7 @@ static bool IsLoadOrder(morder mo) {
40	41	static bool IsStoreOrder(morder mo) {
41	42	return mo == mo_relaxed \|\| mo == mo_release \|\| mo == mo_seq_cst;
42	43	}
	44	#endif
43	45
44	46	static bool IsReleaseOrder(morder mo) {
45	47	return mo == mo_release \|\| mo == mo_acq_rel \|\| mo == mo_seq_cst;
...	...	@@ -161,16 +163,16 @@ a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {
161	163	}
162	164	#endif
163	165
164		template<typename T>
165		static int SizeLog() {
	166	template <typename T>
	167	static int AccessSize() {
166	168	if (sizeof(T) <= 1)
167		return kSizeLog1;
	169	return 1;
168	170	else if (sizeof(T) <= 2)
169		return kSizeLog2;
	171	return 2;
170	172	else if (sizeof(T) <= 4)
171		return kSizeLog4;
	173	return 4;
172	174	else
173		return kSizeLog8;
	175	return 8;
174	176	// For 16-byte atomics we also use 8-byte memory access,
175	177	// this leads to false negatives only in very obscure cases.
176	178	}
...	...	@@ -202,7 +204,7 @@ static memory_order to_mo(morder mo) {
202	204	case mo_acq_rel: return memory_order_acq_rel;
203	205	case mo_seq_cst: return memory_order_seq_cst;
204	206	}
205		CHECK(0);
	207	DCHECK(0);
206	208	return memory_order_seq_cst;
207	209	}
208	210
...	...	@@ -219,27 +221,28 @@ static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
219	221	#endif
220	222
221	223	template <typename T>
222		static T AtomicLoad(ThreadState thr, uptr pc, const volatile T a,
223		morder mo) NO_THREAD_SAFETY_ANALYSIS {
224		CHECK(IsLoadOrder(mo));
	224	static T AtomicLoad(ThreadState thr, uptr pc, const volatile T a, morder mo) {
	225	DCHECK(IsLoadOrder(mo));
225	226	// This fast-path is critical for performance.
226	227	// Assume the access is atomic.
227	228	if (!IsAcquireOrder(mo)) {
228		MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
	229	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
	230	kAccessRead \| kAccessAtomic);
229	231	return NoTsanAtomicLoad(a, mo);
230	232	}
231	233	// Don't create sync object if it does not exist yet. For example, an atomic
232	234	// pointer is initialized to nullptr and then periodically acquire-loaded.
233	235	T v = NoTsanAtomicLoad(a, mo);
234		SyncVar *s = ctx->metamap.GetIfExistsAndLock((uptr)a, false);
	236	SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a);
235	237	if (s) {
236		AcquireImpl(thr, pc, &s->clock);
	238	SlotLocker locker(thr);
	239	ReadLock lock(&s->mtx);
	240	thr->clock.Acquire(s->clock);
237	241	// Re-read under sync mutex because we need a consistent snapshot
238	242	// of the value and the clock we acquire.
239	243	v = NoTsanAtomicLoad(a, mo);
240		s->mtx.ReadUnlock();
241	244	}
242		MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
	245	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessRead \| kAccessAtomic);
243	246	return v;
244	247	}
245	248
...	...	@@ -257,9 +260,9 @@ static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) {
257	260
258	261	template <typename T>
259	262	static void AtomicStore(ThreadState thr, uptr pc, volatile T a, T v,
260		morder mo) NO_THREAD_SAFETY_ANALYSIS {
261		CHECK(IsStoreOrder(mo));
262		MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
	263	morder mo) {
	264	DCHECK(IsStoreOrder(mo));
	265	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite \| kAccessAtomic);
263	266	// This fast-path is critical for performance.
264	267	// Assume the access is atomic.
265	268	// Strictly saying even relaxed store cuts off release sequence,
...	...	@@ -268,36 +271,35 @@ static void AtomicStore(ThreadState thr, uptr pc, volatile T a, T v,
268	271	NoTsanAtomicStore(a, v, mo);
269	272	return;
270	273	}
271		__sync_synchronize();
272		SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
273		thr->fast_state.IncrementEpoch();
274		// Can't increment epoch w/o writing to the trace as well.
275		TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
276		ReleaseStoreImpl(thr, pc, &s->clock);
277		NoTsanAtomicStore(a, v, mo);
278		s->mtx.Unlock();
	274	SlotLocker locker(thr);
	275	{
	276	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
	277	Lock lock(&s->mtx);
	278	thr->clock.ReleaseStore(&s->clock);
	279	NoTsanAtomicStore(a, v, mo);
	280	}
	281	IncrementEpoch(thr);
279	282	}
280	283
281	284	template <typename T, T (F)(volatile T v, T op)>
282		static T AtomicRMW(ThreadState thr, uptr pc, volatile T a, T v,
283		morder mo) NO_THREAD_SAFETY_ANALYSIS {
284		MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
285		SyncVar *s = 0;
286		if (mo != mo_relaxed) {
287		s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
288		thr->fast_state.IncrementEpoch();
289		// Can't increment epoch w/o writing to the trace as well.
290		TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
	285	static T AtomicRMW(ThreadState thr, uptr pc, volatile T a, T v, morder mo) {
	286	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite \| kAccessAtomic);
	287	if (LIKELY(mo == mo_relaxed))
	288	return F(a, v);
	289	SlotLocker locker(thr);
	290	{
	291	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
	292	RWLock lock(&s->mtx, IsReleaseOrder(mo));
291	293	if (IsAcqRelOrder(mo))
292		AcquireReleaseImpl(thr, pc, &s->clock);
	294	thr->clock.ReleaseAcquire(&s->clock);
293	295	else if (IsReleaseOrder(mo))
294		ReleaseImpl(thr, pc, &s->clock);
	296	thr->clock.Release(&s->clock);
295	297	else if (IsAcquireOrder(mo))
296		AcquireImpl(thr, pc, &s->clock);
	298	thr->clock.Acquire(s->clock);
	299	v = F(a, v);
297	300	}
298		v = F(a, v);
299		if (s)
300		s->mtx.Unlock();
	301	if (IsReleaseOrder(mo))
	302	IncrementEpoch(thr);
301	303	return v;
302	304	}
303	305
...	...	@@ -402,46 +404,44 @@ static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {
402	404	}
403	405
404	406	template <typename T>
405		static bool AtomicCAS(ThreadState thr, uptr pc, volatile T a, T *c, T v, morder mo,
406		morder fmo) NO_THREAD_SAFETY_ANALYSIS {
	407	static bool AtomicCAS(ThreadState thr, uptr pc, volatile T a, T *c, T v,
	408	morder mo, morder fmo) {
407	409	// 31.7.2.18: "The failure argument shall not be memory_order_release
408	410	// nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic
409	411	// (mo_relaxed) when those are used.
410		CHECK(IsLoadOrder(fmo));
411
412		MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
413		SyncVar *s = 0;
414		bool write_lock = IsReleaseOrder(mo);
415
416		if (mo != mo_relaxed \|\| fmo != mo_relaxed)
417		s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, write_lock);
418
419		T cc = *c;
420		T pr = func_cas(a, cc, v);
421		bool success = pr == cc;
422		if (!success) {
	412	DCHECK(IsLoadOrder(fmo));
	413
	414	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite \| kAccessAtomic);
	415	if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) {
	416	T cc = *c;
	417	T pr = func_cas(a, cc, v);
	418	if (pr == cc)
	419	return true;
423	420	*c = pr;
424		mo = fmo;
	421	return false;
425	422	}
426
427		if (s) {
428		thr->fast_state.IncrementEpoch();
429		// Can't increment epoch w/o writing to the trace as well.
430		TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
431
	423	SlotLocker locker(thr);
	424	bool release = IsReleaseOrder(mo);
	425	bool success;
	426	{
	427	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
	428	RWLock lock(&s->mtx, release);
	429	T cc = *c;
	430	T pr = func_cas(a, cc, v);
	431	success = pr == cc;
	432	if (!success) {
	433	*c = pr;
	434	mo = fmo;
	435	}
432	436	if (success && IsAcqRelOrder(mo))
433		AcquireReleaseImpl(thr, pc, &s->clock);
	437	thr->clock.ReleaseAcquire(&s->clock);
434	438	else if (success && IsReleaseOrder(mo))
435		ReleaseImpl(thr, pc, &s->clock);
	439	thr->clock.Release(&s->clock);
436	440	else if (IsAcquireOrder(mo))
437		AcquireImpl(thr, pc, &s->clock);
438
439		if (write_lock)
440		s->mtx.Unlock();
441		else
442		s->mtx.ReadUnlock();
	441	thr->clock.Acquire(s->clock);
443	442	}
444
	443	if (success && release)
	444	IncrementEpoch(thr);
445	445	return success;
446	446	}
447	447
...	...	@@ -485,380 +485,356 @@ static morder convert_morder(morder mo) {
485	485	return (morder)(mo & 0x7fff);
486	486	}
487	487
488		#define SCOPED_ATOMIC(func, ...) \
489		ThreadState *const thr = cur_thread(); \
490		if (UNLIKELY(thr->ignore_sync \|\| thr->ignore_interceptors)) { \
491		ProcessPendingSignals(thr); \
492		return NoTsanAtomic##func(__VA_ARGS__); \
493		} \
494		const uptr callpc = (uptr)__builtin_return_address(0); \
495		uptr pc = StackTrace::GetCurrentPc(); \
496		mo = convert_morder(mo); \
497		ScopedAtomic sa(thr, callpc, a, mo, __func__); \
498		return Atomic##func(thr, pc, __VA_ARGS__); \
499		/**/
500
501		class ScopedAtomic {
502		public:
503		ScopedAtomic(ThreadState thr, uptr pc, const volatile void a,
504		morder mo, const char *func)
505		: thr_(thr) {
506		FuncEntry(thr_, pc);
507		DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo);
508		}
509		~ScopedAtomic() {
510		ProcessPendingSignals(thr_);
511		FuncExit(thr_);
512		}
513		private:
514		ThreadState *thr_;
515		};
	488	# define ATOMIC_IMPL(func, ...) \
	489	ThreadState *const thr = cur_thread(); \
	490	ProcessPendingSignals(thr); \
	491	if (UNLIKELY(thr->ignore_sync \|\| thr->ignore_interceptors)) \
	492	return NoTsanAtomic##func(__VA_ARGS__); \
	493	mo = convert_morder(mo); \
	494	return Atomic##func(thr, GET_CALLER_PC(), __VA_ARGS__);
516	495
517	496	extern "C" {
518	497	SANITIZER_INTERFACE_ATTRIBUTE
519	498	a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
520		SCOPED_ATOMIC(Load, a, mo);
	499	ATOMIC_IMPL(Load, a, mo);
521	500	}
522	501
523	502	SANITIZER_INTERFACE_ATTRIBUTE
524	503	a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {
525		SCOPED_ATOMIC(Load, a, mo);
	504	ATOMIC_IMPL(Load, a, mo);
526	505	}
527	506
528	507	SANITIZER_INTERFACE_ATTRIBUTE
529	508	a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {
530		SCOPED_ATOMIC(Load, a, mo);
	509	ATOMIC_IMPL(Load, a, mo);
531	510	}
532	511
533	512	SANITIZER_INTERFACE_ATTRIBUTE
534	513	a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {
535		SCOPED_ATOMIC(Load, a, mo);
	514	ATOMIC_IMPL(Load, a, mo);
536	515	}
537	516
538	517	#if __TSAN_HAS_INT128
539	518	SANITIZER_INTERFACE_ATTRIBUTE
540	519	a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {
541		SCOPED_ATOMIC(Load, a, mo);
	520	ATOMIC_IMPL(Load, a, mo);
542	521	}
543	522	#endif
544	523
545	524	SANITIZER_INTERFACE_ATTRIBUTE
546	525	void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {
547		SCOPED_ATOMIC(Store, a, v, mo);
	526	ATOMIC_IMPL(Store, a, v, mo);
548	527	}
549	528
550	529	SANITIZER_INTERFACE_ATTRIBUTE
551	530	void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {
552		SCOPED_ATOMIC(Store, a, v, mo);
	531	ATOMIC_IMPL(Store, a, v, mo);
553	532	}
554	533
555	534	SANITIZER_INTERFACE_ATTRIBUTE
556	535	void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {
557		SCOPED_ATOMIC(Store, a, v, mo);
	536	ATOMIC_IMPL(Store, a, v, mo);
558	537	}
559	538
560	539	SANITIZER_INTERFACE_ATTRIBUTE
561	540	void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {
562		SCOPED_ATOMIC(Store, a, v, mo);
	541	ATOMIC_IMPL(Store, a, v, mo);
563	542	}
564	543
565	544	#if __TSAN_HAS_INT128
566	545	SANITIZER_INTERFACE_ATTRIBUTE
567	546	void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {
568		SCOPED_ATOMIC(Store, a, v, mo);
	547	ATOMIC_IMPL(Store, a, v, mo);
569	548	}
570	549	#endif
571	550
572	551	SANITIZER_INTERFACE_ATTRIBUTE
573	552	a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {
574		SCOPED_ATOMIC(Exchange, a, v, mo);
	553	ATOMIC_IMPL(Exchange, a, v, mo);
575	554	}
576	555
577	556	SANITIZER_INTERFACE_ATTRIBUTE
578	557	a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {
579		SCOPED_ATOMIC(Exchange, a, v, mo);
	558	ATOMIC_IMPL(Exchange, a, v, mo);
580	559	}
581	560
582	561	SANITIZER_INTERFACE_ATTRIBUTE
583	562	a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {
584		SCOPED_ATOMIC(Exchange, a, v, mo);
	563	ATOMIC_IMPL(Exchange, a, v, mo);
585	564	}
586	565
587	566	SANITIZER_INTERFACE_ATTRIBUTE
588	567	a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {
589		SCOPED_ATOMIC(Exchange, a, v, mo);
	568	ATOMIC_IMPL(Exchange, a, v, mo);
590	569	}
591	570
592	571	#if __TSAN_HAS_INT128
593	572	SANITIZER_INTERFACE_ATTRIBUTE
594	573	a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {
595		SCOPED_ATOMIC(Exchange, a, v, mo);
	574	ATOMIC_IMPL(Exchange, a, v, mo);
596	575	}
597	576	#endif
598	577
599	578	SANITIZER_INTERFACE_ATTRIBUTE
600	579	a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {
601		SCOPED_ATOMIC(FetchAdd, a, v, mo);
	580	ATOMIC_IMPL(FetchAdd, a, v, mo);
602	581	}
603	582
604	583	SANITIZER_INTERFACE_ATTRIBUTE
605	584	a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {
606		SCOPED_ATOMIC(FetchAdd, a, v, mo);
	585	ATOMIC_IMPL(FetchAdd, a, v, mo);
607	586	}
608	587
609	588	SANITIZER_INTERFACE_ATTRIBUTE
610	589	a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {
611		SCOPED_ATOMIC(FetchAdd, a, v, mo);
	590	ATOMIC_IMPL(FetchAdd, a, v, mo);
612	591	}
613	592
614	593	SANITIZER_INTERFACE_ATTRIBUTE
615	594	a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {
616		SCOPED_ATOMIC(FetchAdd, a, v, mo);
	595	ATOMIC_IMPL(FetchAdd, a, v, mo);
617	596	}
618	597
619	598	#if __TSAN_HAS_INT128
620	599	SANITIZER_INTERFACE_ATTRIBUTE
621	600	a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {
622		SCOPED_ATOMIC(FetchAdd, a, v, mo);
	601	ATOMIC_IMPL(FetchAdd, a, v, mo);
623	602	}
624	603	#endif
625	604
626	605	SANITIZER_INTERFACE_ATTRIBUTE
627	606	a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {
628		SCOPED_ATOMIC(FetchSub, a, v, mo);
	607	ATOMIC_IMPL(FetchSub, a, v, mo);
629	608	}
630	609
631	610	SANITIZER_INTERFACE_ATTRIBUTE
632	611	a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {
633		SCOPED_ATOMIC(FetchSub, a, v, mo);
	612	ATOMIC_IMPL(FetchSub, a, v, mo);
634	613	}
635	614
636	615	SANITIZER_INTERFACE_ATTRIBUTE
637	616	a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {
638		SCOPED_ATOMIC(FetchSub, a, v, mo);
	617	ATOMIC_IMPL(FetchSub, a, v, mo);
639	618	}
640	619
641	620	SANITIZER_INTERFACE_ATTRIBUTE
642	621	a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {
643		SCOPED_ATOMIC(FetchSub, a, v, mo);
	622	ATOMIC_IMPL(FetchSub, a, v, mo);
644	623	}
645	624
646	625	#if __TSAN_HAS_INT128
647	626	SANITIZER_INTERFACE_ATTRIBUTE
648	627	a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {
649		SCOPED_ATOMIC(FetchSub, a, v, mo);
	628	ATOMIC_IMPL(FetchSub, a, v, mo);
650	629	}
651	630	#endif
652	631
653	632	SANITIZER_INTERFACE_ATTRIBUTE
654	633	a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {
655		SCOPED_ATOMIC(FetchAnd, a, v, mo);
	634	ATOMIC_IMPL(FetchAnd, a, v, mo);
656	635	}
657	636
658	637	SANITIZER_INTERFACE_ATTRIBUTE
659	638	a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {
660		SCOPED_ATOMIC(FetchAnd, a, v, mo);
	639	ATOMIC_IMPL(FetchAnd, a, v, mo);
661	640	}
662	641
663	642	SANITIZER_INTERFACE_ATTRIBUTE
664	643	a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {
665		SCOPED_ATOMIC(FetchAnd, a, v, mo);
	644	ATOMIC_IMPL(FetchAnd, a, v, mo);
666	645	}
667	646
668	647	SANITIZER_INTERFACE_ATTRIBUTE
669	648	a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {
670		SCOPED_ATOMIC(FetchAnd, a, v, mo);
	649	ATOMIC_IMPL(FetchAnd, a, v, mo);
671	650	}
672	651
673	652	#if __TSAN_HAS_INT128
674	653	SANITIZER_INTERFACE_ATTRIBUTE
675	654	a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {
676		SCOPED_ATOMIC(FetchAnd, a, v, mo);
	655	ATOMIC_IMPL(FetchAnd, a, v, mo);
677	656	}
678	657	#endif
679	658
680	659	SANITIZER_INTERFACE_ATTRIBUTE
681	660	a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {
682		SCOPED_ATOMIC(FetchOr, a, v, mo);
	661	ATOMIC_IMPL(FetchOr, a, v, mo);
683	662	}
684	663
685	664	SANITIZER_INTERFACE_ATTRIBUTE
686	665	a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {
687		SCOPED_ATOMIC(FetchOr, a, v, mo);
	666	ATOMIC_IMPL(FetchOr, a, v, mo);
688	667	}
689	668
690	669	SANITIZER_INTERFACE_ATTRIBUTE
691	670	a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {
692		SCOPED_ATOMIC(FetchOr, a, v, mo);
	671	ATOMIC_IMPL(FetchOr, a, v, mo);
693	672	}
694	673
695	674	SANITIZER_INTERFACE_ATTRIBUTE
696	675	a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {
697		SCOPED_ATOMIC(FetchOr, a, v, mo);
	676	ATOMIC_IMPL(FetchOr, a, v, mo);
698	677	}
699	678
700	679	#if __TSAN_HAS_INT128
701	680	SANITIZER_INTERFACE_ATTRIBUTE
702	681	a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {
703		SCOPED_ATOMIC(FetchOr, a, v, mo);
	682	ATOMIC_IMPL(FetchOr, a, v, mo);
704	683	}
705	684	#endif
706	685
707	686	SANITIZER_INTERFACE_ATTRIBUTE
708	687	a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {
709		SCOPED_ATOMIC(FetchXor, a, v, mo);
	688	ATOMIC_IMPL(FetchXor, a, v, mo);
710	689	}
711	690
712	691	SANITIZER_INTERFACE_ATTRIBUTE
713	692	a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {
714		SCOPED_ATOMIC(FetchXor, a, v, mo);
	693	ATOMIC_IMPL(FetchXor, a, v, mo);
715	694	}
716	695
717	696	SANITIZER_INTERFACE_ATTRIBUTE
718	697	a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {
719		SCOPED_ATOMIC(FetchXor, a, v, mo);
	698	ATOMIC_IMPL(FetchXor, a, v, mo);
720	699	}
721	700
722	701	SANITIZER_INTERFACE_ATTRIBUTE
723	702	a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {
724		SCOPED_ATOMIC(FetchXor, a, v, mo);
	703	ATOMIC_IMPL(FetchXor, a, v, mo);
725	704	}
726	705
727	706	#if __TSAN_HAS_INT128
728	707	SANITIZER_INTERFACE_ATTRIBUTE
729	708	a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {
730		SCOPED_ATOMIC(FetchXor, a, v, mo);
	709	ATOMIC_IMPL(FetchXor, a, v, mo);
731	710	}
732	711	#endif
733	712
734	713	SANITIZER_INTERFACE_ATTRIBUTE
735	714	a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {
736		SCOPED_ATOMIC(FetchNand, a, v, mo);
	715	ATOMIC_IMPL(FetchNand, a, v, mo);
737	716	}
738	717
739	718	SANITIZER_INTERFACE_ATTRIBUTE
740	719	a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {
741		SCOPED_ATOMIC(FetchNand, a, v, mo);
	720	ATOMIC_IMPL(FetchNand, a, v, mo);
742	721	}
743	722
744	723	SANITIZER_INTERFACE_ATTRIBUTE
745	724	a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {
746		SCOPED_ATOMIC(FetchNand, a, v, mo);
	725	ATOMIC_IMPL(FetchNand, a, v, mo);
747	726	}
748	727
749	728	SANITIZER_INTERFACE_ATTRIBUTE
750	729	a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {
751		SCOPED_ATOMIC(FetchNand, a, v, mo);
	730	ATOMIC_IMPL(FetchNand, a, v, mo);
752	731	}
753	732
754	733	#if __TSAN_HAS_INT128
755	734	SANITIZER_INTERFACE_ATTRIBUTE
756	735	a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {
757		SCOPED_ATOMIC(FetchNand, a, v, mo);
	736	ATOMIC_IMPL(FetchNand, a, v, mo);
758	737	}
759	738	#endif
760	739
761	740	SANITIZER_INTERFACE_ATTRIBUTE
762	741	int __tsan_atomic8_compare_exchange_strong(volatile a8 a, a8 c, a8 v,
763	742	morder mo, morder fmo) {
764		SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
	743	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
765	744	}
766	745
767	746	SANITIZER_INTERFACE_ATTRIBUTE
768	747	int __tsan_atomic16_compare_exchange_strong(volatile a16 a, a16 c, a16 v,
769	748	morder mo, morder fmo) {
770		SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
	749	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
771	750	}
772	751
773	752	SANITIZER_INTERFACE_ATTRIBUTE
774	753	int __tsan_atomic32_compare_exchange_strong(volatile a32 a, a32 c, a32 v,
775	754	morder mo, morder fmo) {
776		SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
	755	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
777	756	}
778	757
779	758	SANITIZER_INTERFACE_ATTRIBUTE
780	759	int __tsan_atomic64_compare_exchange_strong(volatile a64 a, a64 c, a64 v,
781	760	morder mo, morder fmo) {
782		SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
	761	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
783	762	}
784	763
785	764	#if __TSAN_HAS_INT128
786	765	SANITIZER_INTERFACE_ATTRIBUTE
787	766	int __tsan_atomic128_compare_exchange_strong(volatile a128 a, a128 c, a128 v,
788	767	morder mo, morder fmo) {
789		SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
	768	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
790	769	}
791	770	#endif
792	771
793	772	SANITIZER_INTERFACE_ATTRIBUTE
794	773	int __tsan_atomic8_compare_exchange_weak(volatile a8 a, a8 c, a8 v,
795	774	morder mo, morder fmo) {
796		SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
	775	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
797	776	}
798	777
799	778	SANITIZER_INTERFACE_ATTRIBUTE
800	779	int __tsan_atomic16_compare_exchange_weak(volatile a16 a, a16 c, a16 v,
801	780	morder mo, morder fmo) {
802		SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
	781	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
803	782	}
804	783
805	784	SANITIZER_INTERFACE_ATTRIBUTE
806	785	int __tsan_atomic32_compare_exchange_weak(volatile a32 a, a32 c, a32 v,
807	786	morder mo, morder fmo) {
808		SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
	787	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
809	788	}
810	789
811	790	SANITIZER_INTERFACE_ATTRIBUTE
812	791	int __tsan_atomic64_compare_exchange_weak(volatile a64 a, a64 c, a64 v,
813	792	morder mo, morder fmo) {
814		SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
	793	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
815	794	}
816	795
817	796	#if __TSAN_HAS_INT128
818	797	SANITIZER_INTERFACE_ATTRIBUTE
819	798	int __tsan_atomic128_compare_exchange_weak(volatile a128 a, a128 c, a128 v,
820	799	morder mo, morder fmo) {
821		SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
	800	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
822	801	}
823	802	#endif
824	803
825	804	SANITIZER_INTERFACE_ATTRIBUTE
826	805	a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,
827	806	morder mo, morder fmo) {
828		SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
	807	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
829	808	}
830	809
831	810	SANITIZER_INTERFACE_ATTRIBUTE
832	811	a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
833	812	morder mo, morder fmo) {
834		SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
	813	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
835	814	}
836	815
837	816	SANITIZER_INTERFACE_ATTRIBUTE
838	817	a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
839	818	morder mo, morder fmo) {
840		SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
	819	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
841	820	}
842	821
843	822	SANITIZER_INTERFACE_ATTRIBUTE
844	823	a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
845	824	morder mo, morder fmo) {
846		SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
	825	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
847	826	}
848	827
849	828	#if __TSAN_HAS_INT128
850	829	SANITIZER_INTERFACE_ATTRIBUTE
851	830	a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
852	831	morder mo, morder fmo) {
853		SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
	832	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
854	833	}
855	834	#endif
856	835
857	836	SANITIZER_INTERFACE_ATTRIBUTE
858		void __tsan_atomic_thread_fence(morder mo) {
859		char* a = 0;
860		SCOPED_ATOMIC(Fence, mo);
861		}
	837	void __tsan_atomic_thread_fence(morder mo) { ATOMIC_IMPL(Fence, mo); }
862	838
863	839	SANITIZER_INTERFACE_ATTRIBUTE
864	840	void __tsan_atomic_signal_fence(morder mo) {
...	...	@@ -869,25 +845,23 @@ void __tsan_atomic_signal_fence(morder mo) {
869	845
870	846	// Go
871	847
872		#define ATOMIC(func, ...) \
873		if (thr->ignore_sync) { \
874		NoTsanAtomic##func(__VA_ARGS__); \
875		} else { \
876		FuncEntry(thr, cpc); \
	848	# define ATOMIC(func, ...) \
	849	if (thr->ignore_sync) { \
	850	NoTsanAtomic##func(__VA_ARGS__); \
	851	} else { \
	852	FuncEntry(thr, cpc); \
877	853	Atomic##func(thr, pc, __VA_ARGS__); \
878		FuncExit(thr); \
879		} \
880		/**/
881
882		#define ATOMIC_RET(func, ret, ...) \
883		if (thr->ignore_sync) { \
884		(ret) = NoTsanAtomic##func(__VA_ARGS__); \
885		} else { \
886		FuncEntry(thr, cpc); \
	854	FuncExit(thr); \
	855	}
	856
	857	# define ATOMIC_RET(func, ret, ...) \
	858	if (thr->ignore_sync) { \
	859	(ret) = NoTsanAtomic##func(__VA_ARGS__); \
	860	} else { \
	861	FuncEntry(thr, cpc); \
887	862	(ret) = Atomic##func(thr, pc, __VA_ARGS__); \
888		FuncExit(thr); \
889		} \
890		/**/
	863	FuncExit(thr); \
	864	}
891	865
892	866	extern "C" {
893	867	SANITIZER_INTERFACE_ATTRIBUTE

lib/tsan/tsan_interface_inl.h deleted-133

...	...	@@ -1,133 +0,0 @@
1		//===-- tsan_interface_inl.h ------------------------------------- C++ --===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// This file is a part of ThreadSanitizer (TSan), a race detector.
10		//
11		//===----------------------------------------------------------------------===//
12
13		#include "tsan_interface.h"
14		#include "tsan_rtl.h"
15		#include "sanitizer_common/sanitizer_ptrauth.h"
16
17		#define CALLERPC ((uptr)__builtin_return_address(0))
18
19		using namespace __tsan;
20
21		void __tsan_read1(void *addr) {
22		MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog1);
23		}
24
25		void __tsan_read2(void *addr) {
26		MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog2);
27		}
28
29		void __tsan_read4(void *addr) {
30		MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog4);
31		}
32
33		void __tsan_read8(void *addr) {
34		MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
35		}
36
37		void __tsan_write1(void *addr) {
38		MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog1);
39		}
40
41		void __tsan_write2(void *addr) {
42		MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog2);
43		}
44
45		void __tsan_write4(void *addr) {
46		MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog4);
47		}
48
49		void __tsan_write8(void *addr) {
50		MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
51		}
52
53		void __tsan_read1_pc(void addr, void pc) {
54		MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog1);
55		}
56
57		void __tsan_read2_pc(void addr, void pc) {
58		MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog2);
59		}
60
61		void __tsan_read4_pc(void addr, void pc) {
62		MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog4);
63		}
64
65		void __tsan_read8_pc(void addr, void pc) {
66		MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog8);
67		}
68
69		void __tsan_write1_pc(void addr, void pc) {
70		MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog1);
71		}
72
73		void __tsan_write2_pc(void addr, void pc) {
74		MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog2);
75		}
76
77		void __tsan_write4_pc(void addr, void pc) {
78		MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog4);
79		}
80
81		void __tsan_write8_pc(void addr, void pc) {
82		MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog8);
83		}
84
85		void __tsan_vptr_update(void *vptr_p, void new_val) {
86		CHECK_EQ(sizeof(vptr_p), 8);
87		if (*vptr_p != new_val) {
88		ThreadState *thr = cur_thread();
89		thr->is_vptr_access = true;
90		MemoryWrite(thr, CALLERPC, (uptr)vptr_p, kSizeLog8);
91		thr->is_vptr_access = false;
92		}
93		}
94
95		void __tsan_vptr_read(void **vptr_p) {
96		CHECK_EQ(sizeof(vptr_p), 8);
97		ThreadState *thr = cur_thread();
98		thr->is_vptr_access = true;
99		MemoryRead(thr, CALLERPC, (uptr)vptr_p, kSizeLog8);
100		thr->is_vptr_access = false;
101		}
102
103		void __tsan_func_entry(void *pc) {
104		FuncEntry(cur_thread(), STRIP_PAC_PC(pc));
105		}
106
107		void __tsan_func_exit() {
108		FuncExit(cur_thread());
109		}
110
111		void __tsan_ignore_thread_begin() {
112		ThreadIgnoreBegin(cur_thread(), CALLERPC);
113		}
114
115		void __tsan_ignore_thread_end() {
116		ThreadIgnoreEnd(cur_thread(), CALLERPC);
117		}
118
119		void __tsan_read_range(void *addr, uptr size) {
120		MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, false);
121		}
122
123		void __tsan_write_range(void *addr, uptr size) {
124		MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, true);
125		}
126
127		void __tsan_read_range_pc(void addr, uptr size, void pc) {
128		MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, false);
129		}
130
131		void __tsan_write_range_pc(void addr, uptr size, void pc) {
132		MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, true);
133		}

lib/tsan/tsan_interface_java.cpp+143-151

...	...	@@ -34,52 +34,49 @@ struct JavaContext {
34	34	}
35	35	};
36	36
37		class ScopedJavaFunc {
38		public:
39		ScopedJavaFunc(ThreadState *thr, uptr pc)
40		: thr_(thr) {
41		Initialize(thr_);
42		FuncEntry(thr, pc);
43		}
44
45		~ScopedJavaFunc() {
46		FuncExit(thr_);
47		// FIXME(dvyukov): process pending signals.
48		}
49
50		private:
51		ThreadState *thr_;
52		};
53
54	37	static u64 jctx_buf[sizeof(JavaContext) / sizeof(u64) + 1];
55	38	static JavaContext *jctx;
56	39
	40	MBlock JavaHeapBlock(uptr addr, uptr start) {
	41	if (!jctx \|\| addr < jctx->heap_begin \|\|
	42	addr >= jctx->heap_begin + jctx->heap_size)
	43	return nullptr;
	44	for (uptr p = RoundDown(addr, kMetaShadowCell); p >= jctx->heap_begin;
	45	p -= kMetaShadowCell) {
	46	MBlock *b = ctx->metamap.GetBlock(p);
	47	if (!b)
	48	continue;
	49	if (p + b->siz <= addr)
	50	return nullptr;
	51	*start = p;
	52	return b;
	53	}
	54	return nullptr;
	55	}
	56
57	57	} // namespace __tsan
58	58
59		#define SCOPED_JAVA_FUNC(func) \
	59	#define JAVA_FUNC_ENTER(func) \
60	60	ThreadState *thr = cur_thread(); \
61		const uptr caller_pc = GET_CALLER_PC(); \
62		const uptr pc = StackTrace::GetCurrentPc(); \
63		(void)pc; \
64		ScopedJavaFunc scoped(thr, caller_pc); \
65		/**/
	61	(void)thr;
66	62
67	63	void __tsan_java_init(jptr heap_begin, jptr heap_size) {
68		SCOPED_JAVA_FUNC(__tsan_java_init);
69		DPrintf("#%d: java_init(%p, %p)\n", thr->tid, heap_begin, heap_size);
70		CHECK_EQ(jctx, 0);
71		CHECK_GT(heap_begin, 0);
72		CHECK_GT(heap_size, 0);
73		CHECK_EQ(heap_begin % kHeapAlignment, 0);
74		CHECK_EQ(heap_size % kHeapAlignment, 0);
75		CHECK_LT(heap_begin, heap_begin + heap_size);
	64	JAVA_FUNC_ENTER(__tsan_java_init);
	65	Initialize(thr);
	66	DPrintf("#%d: java_init(0x%zx, 0x%zx)\n", thr->tid, heap_begin, heap_size);
	67	DCHECK_EQ(jctx, 0);
	68	DCHECK_GT(heap_begin, 0);
	69	DCHECK_GT(heap_size, 0);
	70	DCHECK_EQ(heap_begin % kHeapAlignment, 0);
	71	DCHECK_EQ(heap_size % kHeapAlignment, 0);
	72	DCHECK_LT(heap_begin, heap_begin + heap_size);
76	73	jctx = new(jctx_buf) JavaContext(heap_begin, heap_size);
77	74	}
78	75
79	76	int __tsan_java_fini() {
80		SCOPED_JAVA_FUNC(__tsan_java_fini);
	77	JAVA_FUNC_ENTER(__tsan_java_fini);
81	78	DPrintf("#%d: java_fini()\n", thr->tid);
82		CHECK_NE(jctx, 0);
	79	DCHECK_NE(jctx, 0);
83	80	// FIXME(dvyukov): this does not call atexit() callbacks.
84	81	int status = Finalize(thr);
85	82	DPrintf("#%d: java_fini() = %d\n", thr->tid, status);
...	...	@@ -87,74 +84,65 @@ int __tsan_java_fini() {
87	84	}
88	85
89	86	void __tsan_java_alloc(jptr ptr, jptr size) {
90		SCOPED_JAVA_FUNC(__tsan_java_alloc);
91		DPrintf("#%d: java_alloc(%p, %p)\n", thr->tid, ptr, size);
92		CHECK_NE(jctx, 0);
93		CHECK_NE(size, 0);
94		CHECK_EQ(ptr % kHeapAlignment, 0);
95		CHECK_EQ(size % kHeapAlignment, 0);
96		CHECK_GE(ptr, jctx->heap_begin);
97		CHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
98
99		OnUserAlloc(thr, pc, ptr, size, false);
	87	JAVA_FUNC_ENTER(__tsan_java_alloc);
	88	DPrintf("#%d: java_alloc(0x%zx, 0x%zx)\n", thr->tid, ptr, size);
	89	DCHECK_NE(jctx, 0);
	90	DCHECK_NE(size, 0);
	91	DCHECK_EQ(ptr % kHeapAlignment, 0);
	92	DCHECK_EQ(size % kHeapAlignment, 0);
	93	DCHECK_GE(ptr, jctx->heap_begin);
	94	DCHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
	95
	96	OnUserAlloc(thr, 0, ptr, size, false);
100	97	}
101	98
102	99	void __tsan_java_free(jptr ptr, jptr size) {
103		SCOPED_JAVA_FUNC(__tsan_java_free);
104		DPrintf("#%d: java_free(%p, %p)\n", thr->tid, ptr, size);
105		CHECK_NE(jctx, 0);
106		CHECK_NE(size, 0);
107		CHECK_EQ(ptr % kHeapAlignment, 0);
108		CHECK_EQ(size % kHeapAlignment, 0);
109		CHECK_GE(ptr, jctx->heap_begin);
110		CHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
111
112		ctx->metamap.FreeRange(thr->proc(), ptr, size);
	100	JAVA_FUNC_ENTER(__tsan_java_free);
	101	DPrintf("#%d: java_free(0x%zx, 0x%zx)\n", thr->tid, ptr, size);
	102	DCHECK_NE(jctx, 0);
	103	DCHECK_NE(size, 0);
	104	DCHECK_EQ(ptr % kHeapAlignment, 0);
	105	DCHECK_EQ(size % kHeapAlignment, 0);
	106	DCHECK_GE(ptr, jctx->heap_begin);
	107	DCHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
	108
	109	ctx->metamap.FreeRange(thr->proc(), ptr, size, false);
113	110	}
114	111
115	112	void __tsan_java_move(jptr src, jptr dst, jptr size) {
116		SCOPED_JAVA_FUNC(__tsan_java_move);
117		DPrintf("#%d: java_move(%p, %p, %p)\n", thr->tid, src, dst, size);
118		CHECK_NE(jctx, 0);
119		CHECK_NE(size, 0);
120		CHECK_EQ(src % kHeapAlignment, 0);
121		CHECK_EQ(dst % kHeapAlignment, 0);
122		CHECK_EQ(size % kHeapAlignment, 0);
123		CHECK_GE(src, jctx->heap_begin);
124		CHECK_LE(src + size, jctx->heap_begin + jctx->heap_size);
125		CHECK_GE(dst, jctx->heap_begin);
126		CHECK_LE(dst + size, jctx->heap_begin + jctx->heap_size);
127		CHECK_NE(dst, src);
128		CHECK_NE(size, 0);
	113	JAVA_FUNC_ENTER(__tsan_java_move);
	114	DPrintf("#%d: java_move(0x%zx, 0x%zx, 0x%zx)\n", thr->tid, src, dst, size);
	115	DCHECK_NE(jctx, 0);
	116	DCHECK_NE(size, 0);
	117	DCHECK_EQ(src % kHeapAlignment, 0);
	118	DCHECK_EQ(dst % kHeapAlignment, 0);
	119	DCHECK_EQ(size % kHeapAlignment, 0);
	120	DCHECK_GE(src, jctx->heap_begin);
	121	DCHECK_LE(src + size, jctx->heap_begin + jctx->heap_size);
	122	DCHECK_GE(dst, jctx->heap_begin);
	123	DCHECK_LE(dst + size, jctx->heap_begin + jctx->heap_size);
	124	DCHECK_NE(dst, src);
	125	DCHECK_NE(size, 0);
129	126
130	127	// Assuming it's not running concurrently with threads that do
131	128	// memory accesses and mutex operations (stop-the-world phase).
132	129	ctx->metamap.MoveMemory(src, dst, size);
133	130
134		// Move shadow.
135		u64 s = (u64)MemToShadow(src);
136		u64 d = (u64)MemToShadow(dst);
137		u64 send = (u64)MemToShadow(src + size);
138		uptr inc = 1;
139		if (dst > src) {
140		s = (u64*)MemToShadow(src + size) - 1;
141		d = (u64*)MemToShadow(dst + size) - 1;
142		send = (u64*)MemToShadow(src) - 1;
143		inc = -1;
144		}
145		for (; s != send; s += inc, d += inc) {
146		d = s;
147		*s = 0;
148		}
	131	// Clear the destination shadow range.
	132	// We used to move shadow from src to dst, but the trace format does not
	133	// support that anymore as it contains addresses of accesses.
	134	RawShadow *d = MemToShadow(dst);
	135	RawShadow *dend = MemToShadow(dst + size);
	136	ShadowSet(d, dend, Shadow::kEmpty);
149	137	}
150	138
151	139	jptr __tsan_java_find(jptr *from_ptr, jptr to) {
152		SCOPED_JAVA_FUNC(__tsan_java_find);
153		DPrintf("#%d: java_find(&%p, %p)\n", *from_ptr, to);
154		CHECK_EQ((*from_ptr) % kHeapAlignment, 0);
155		CHECK_EQ(to % kHeapAlignment, 0);
156		CHECK_GE(*from_ptr, jctx->heap_begin);
157		CHECK_LE(to, jctx->heap_begin + jctx->heap_size);
	140	JAVA_FUNC_ENTER(__tsan_java_find);
	141	DPrintf("#%d: java_find(&0x%zx, 0x%zx)\n", thr->tid, *from_ptr, to);
	142	DCHECK_EQ((*from_ptr) % kHeapAlignment, 0);
	143	DCHECK_EQ(to % kHeapAlignment, 0);
	144	DCHECK_GE(*from_ptr, jctx->heap_begin);
	145	DCHECK_LE(to, jctx->heap_begin + jctx->heap_size);
158	146	for (uptr from = *from_ptr; from < to; from += kHeapAlignment) {
159	147	MBlock *b = ctx->metamap.GetBlock(from);
160	148	if (b) {
...	...	@@ -166,101 +154,105 @@ jptr __tsan_java_find(jptr *from_ptr, jptr to) {
166	154	}
167	155
168	156	void __tsan_java_finalize() {
169		SCOPED_JAVA_FUNC(__tsan_java_finalize);
170		DPrintf("#%d: java_mutex_finalize()\n", thr->tid);
171		AcquireGlobal(thr, 0);
	157	JAVA_FUNC_ENTER(__tsan_java_finalize);
	158	DPrintf("#%d: java_finalize()\n", thr->tid);
	159	AcquireGlobal(thr);
172	160	}
173	161
174	162	void __tsan_java_mutex_lock(jptr addr) {
175		SCOPED_JAVA_FUNC(__tsan_java_mutex_lock);
176		DPrintf("#%d: java_mutex_lock(%p)\n", thr->tid, addr);
177		CHECK_NE(jctx, 0);
178		CHECK_GE(addr, jctx->heap_begin);
179		CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
180
181		MutexPostLock(thr, pc, addr, MutexFlagLinkerInit \| MutexFlagWriteReentrant \|
182		MutexFlagDoPreLockOnPostLock);
	163	JAVA_FUNC_ENTER(__tsan_java_mutex_lock);
	164	DPrintf("#%d: java_mutex_lock(0x%zx)\n", thr->tid, addr);
	165	DCHECK_NE(jctx, 0);
	166	DCHECK_GE(addr, jctx->heap_begin);
	167	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
	168
	169	MutexPostLock(thr, 0, addr,
	170	MutexFlagLinkerInit \| MutexFlagWriteReentrant \|
	171	MutexFlagDoPreLockOnPostLock);
183	172	}
184	173
185	174	void __tsan_java_mutex_unlock(jptr addr) {
186		SCOPED_JAVA_FUNC(__tsan_java_mutex_unlock);
187		DPrintf("#%d: java_mutex_unlock(%p)\n", thr->tid, addr);
188		CHECK_NE(jctx, 0);
189		CHECK_GE(addr, jctx->heap_begin);
190		CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
	175	JAVA_FUNC_ENTER(__tsan_java_mutex_unlock);
	176	DPrintf("#%d: java_mutex_unlock(0x%zx)\n", thr->tid, addr);
	177	DCHECK_NE(jctx, 0);
	178	DCHECK_GE(addr, jctx->heap_begin);
	179	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
191	180
192		MutexUnlock(thr, pc, addr);
	181	MutexUnlock(thr, 0, addr);
193	182	}
194	183
195	184	void __tsan_java_mutex_read_lock(jptr addr) {
196		SCOPED_JAVA_FUNC(__tsan_java_mutex_read_lock);
197		DPrintf("#%d: java_mutex_read_lock(%p)\n", thr->tid, addr);
198		CHECK_NE(jctx, 0);
199		CHECK_GE(addr, jctx->heap_begin);
200		CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
201
202		MutexPostReadLock(thr, pc, addr, MutexFlagLinkerInit \|
203		MutexFlagWriteReentrant \| MutexFlagDoPreLockOnPostLock);
	185	JAVA_FUNC_ENTER(__tsan_java_mutex_read_lock);
	186	DPrintf("#%d: java_mutex_read_lock(0x%zx)\n", thr->tid, addr);
	187	DCHECK_NE(jctx, 0);
	188	DCHECK_GE(addr, jctx->heap_begin);
	189	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
	190
	191	MutexPostReadLock(thr, 0, addr,
	192	MutexFlagLinkerInit \| MutexFlagWriteReentrant \|
	193	MutexFlagDoPreLockOnPostLock);
204	194	}
205	195
206	196	void __tsan_java_mutex_read_unlock(jptr addr) {
207		SCOPED_JAVA_FUNC(__tsan_java_mutex_read_unlock);
208		DPrintf("#%d: java_mutex_read_unlock(%p)\n", thr->tid, addr);
209		CHECK_NE(jctx, 0);
210		CHECK_GE(addr, jctx->heap_begin);
211		CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
	197	JAVA_FUNC_ENTER(__tsan_java_mutex_read_unlock);
	198	DPrintf("#%d: java_mutex_read_unlock(0x%zx)\n", thr->tid, addr);
	199	DCHECK_NE(jctx, 0);
	200	DCHECK_GE(addr, jctx->heap_begin);
	201	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
212	202
213		MutexReadUnlock(thr, pc, addr);
	203	MutexReadUnlock(thr, 0, addr);
214	204	}
215	205
216	206	void __tsan_java_mutex_lock_rec(jptr addr, int rec) {
217		SCOPED_JAVA_FUNC(__tsan_java_mutex_lock_rec);
218		DPrintf("#%d: java_mutex_lock_rec(%p, %d)\n", thr->tid, addr, rec);
219		CHECK_NE(jctx, 0);
220		CHECK_GE(addr, jctx->heap_begin);
221		CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
222		CHECK_GT(rec, 0);
223
224		MutexPostLock(thr, pc, addr, MutexFlagLinkerInit \| MutexFlagWriteReentrant \|
225		MutexFlagDoPreLockOnPostLock \| MutexFlagRecursiveLock, rec);
	207	JAVA_FUNC_ENTER(__tsan_java_mutex_lock_rec);
	208	DPrintf("#%d: java_mutex_lock_rec(0x%zx, %d)\n", thr->tid, addr, rec);
	209	DCHECK_NE(jctx, 0);
	210	DCHECK_GE(addr, jctx->heap_begin);
	211	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
	212	DCHECK_GT(rec, 0);
	213
	214	MutexPostLock(thr, 0, addr,
	215	MutexFlagLinkerInit \| MutexFlagWriteReentrant \|
	216	MutexFlagDoPreLockOnPostLock \| MutexFlagRecursiveLock,
	217	rec);
226	218	}
227	219
228	220	int __tsan_java_mutex_unlock_rec(jptr addr) {
229		SCOPED_JAVA_FUNC(__tsan_java_mutex_unlock_rec);
230		DPrintf("#%d: java_mutex_unlock_rec(%p)\n", thr->tid, addr);
231		CHECK_NE(jctx, 0);
232		CHECK_GE(addr, jctx->heap_begin);
233		CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
	221	JAVA_FUNC_ENTER(__tsan_java_mutex_unlock_rec);
	222	DPrintf("#%d: java_mutex_unlock_rec(0x%zx)\n", thr->tid, addr);
	223	DCHECK_NE(jctx, 0);
	224	DCHECK_GE(addr, jctx->heap_begin);
	225	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
234	226
235		return MutexUnlock(thr, pc, addr, MutexFlagRecursiveUnlock);
	227	return MutexUnlock(thr, 0, addr, MutexFlagRecursiveUnlock);
236	228	}
237	229
238	230	void __tsan_java_acquire(jptr addr) {
239		SCOPED_JAVA_FUNC(__tsan_java_acquire);
240		DPrintf("#%d: java_acquire(%p)\n", thr->tid, addr);
241		CHECK_NE(jctx, 0);
242		CHECK_GE(addr, jctx->heap_begin);
243		CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
	231	JAVA_FUNC_ENTER(__tsan_java_acquire);
	232	DPrintf("#%d: java_acquire(0x%zx)\n", thr->tid, addr);
	233	DCHECK_NE(jctx, 0);
	234	DCHECK_GE(addr, jctx->heap_begin);
	235	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
244	236
245		Acquire(thr, caller_pc, addr);
	237	Acquire(thr, 0, addr);
246	238	}
247	239
248	240	void __tsan_java_release(jptr addr) {
249		SCOPED_JAVA_FUNC(__tsan_java_release);
250		DPrintf("#%d: java_release(%p)\n", thr->tid, addr);
251		CHECK_NE(jctx, 0);
252		CHECK_GE(addr, jctx->heap_begin);
253		CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
	241	JAVA_FUNC_ENTER(__tsan_java_release);
	242	DPrintf("#%d: java_release(0x%zx)\n", thr->tid, addr);
	243	DCHECK_NE(jctx, 0);
	244	DCHECK_GE(addr, jctx->heap_begin);
	245	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
254	246
255		Release(thr, caller_pc, addr);
	247	Release(thr, 0, addr);
256	248	}
257	249
258	250	void __tsan_java_release_store(jptr addr) {
259		SCOPED_JAVA_FUNC(__tsan_java_release);
260		DPrintf("#%d: java_release_store(%p)\n", thr->tid, addr);
261		CHECK_NE(jctx, 0);
262		CHECK_GE(addr, jctx->heap_begin);
263		CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
	251	JAVA_FUNC_ENTER(__tsan_java_release);
	252	DPrintf("#%d: java_release_store(0x%zx)\n", thr->tid, addr);
	253	DCHECK_NE(jctx, 0);
	254	DCHECK_GE(addr, jctx->heap_begin);
	255	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
264	256
265		ReleaseStore(thr, caller_pc, addr);
	257	ReleaseStore(thr, 0, addr);
266	258	}

lib/tsan/tsan_malloc_mac.cpp+23-7

...	...	@@ -12,11 +12,12 @@
12	12	//===----------------------------------------------------------------------===//
13	13
14	14	#include "sanitizer_common/sanitizer_platform.h"
15		#if SANITIZER_MAC
	15	#if SANITIZER_APPLE
16	16
17	17	#include "sanitizer_common/sanitizer_errno.h"
18	18	#include "tsan_interceptors.h"
19	19	#include "tsan_stack_trace.h"
	20	#include "tsan_mman.h"
20	21
21	22	using namespace __tsan;
22	23	#define COMMON_MALLOC_ZONE_NAME "tsan"
...	...	@@ -29,16 +30,30 @@ using namespace __tsan;
29	30	user_memalign(cur_thread(), StackTrace::GetCurrentPc(), alignment, size)
30	31	#define COMMON_MALLOC_MALLOC(size) \
31	32	if (in_symbolizer()) return InternalAlloc(size); \
32		SCOPED_INTERCEPTOR_RAW(malloc, size); \
33		void *p = user_alloc(thr, pc, size)
	33	void *p = 0; \
	34	{ \
	35	SCOPED_INTERCEPTOR_RAW(malloc, size); \
	36	p = user_alloc(thr, pc, size); \
	37	} \
	38	invoke_malloc_hook(p, size)
34	39	#define COMMON_MALLOC_REALLOC(ptr, size) \
35	40	if (in_symbolizer()) return InternalRealloc(ptr, size); \
36		SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \
37		void *p = user_realloc(thr, pc, ptr, size)
	41	if (ptr) \
	42	invoke_free_hook(ptr); \
	43	void *p = 0; \
	44	{ \
	45	SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \
	46	p = user_realloc(thr, pc, ptr, size); \
	47	} \
	48	invoke_malloc_hook(p, size)
38	49	#define COMMON_MALLOC_CALLOC(count, size) \
39	50	if (in_symbolizer()) return InternalCalloc(count, size); \
40		SCOPED_INTERCEPTOR_RAW(calloc, size, count); \
41		void *p = user_calloc(thr, pc, size, count)
	51	void *p = 0; \
	52	{ \
	53	SCOPED_INTERCEPTOR_RAW(calloc, size, count); \
	54	p = user_calloc(thr, pc, size, count); \
	55	} \
	56	invoke_malloc_hook(p, size * count)
42	57	#define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size) \
43	58	if (in_symbolizer()) { \
44	59	void *p = InternalAlloc(size, nullptr, alignment); \
...	...	@@ -55,6 +70,7 @@ using namespace __tsan;
55	70	void *p = user_valloc(thr, pc, size)
56	71	#define COMMON_MALLOC_FREE(ptr) \
57	72	if (in_symbolizer()) return InternalFree(ptr); \
	73	invoke_free_hook(ptr); \
58	74	SCOPED_INTERCEPTOR_RAW(free, ptr); \
59	75	user_free(thr, pc, ptr)
60	76	#define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr);

lib/tsan/tsan_mman.cpp+109-26

...	...	@@ -15,27 +15,18 @@
15	15	#include "sanitizer_common/sanitizer_common.h"
16	16	#include "sanitizer_common/sanitizer_errno.h"
17	17	#include "sanitizer_common/sanitizer_placement_new.h"
	18	#include "tsan_interface.h"
18	19	#include "tsan_mman.h"
19	20	#include "tsan_rtl.h"
20	21	#include "tsan_report.h"
21	22	#include "tsan_flags.h"
22	23
23		// May be overriden by front-end.
24		SANITIZER_WEAK_DEFAULT_IMPL
25		void __sanitizer_malloc_hook(void *ptr, uptr size) {
26		(void)ptr;
27		(void)size;
28		}
29
30		SANITIZER_WEAK_DEFAULT_IMPL
31		void __sanitizer_free_hook(void *ptr) {
32		(void)ptr;
33		}
34
35	24	namespace __tsan {
36	25
37	26	struct MapUnmapCallback {
38	27	void OnMap(uptr p, uptr size) const { }
	28	void OnMapSecondary(uptr p, uptr size, uptr user_begin,
	29	uptr user_size) const {};
39	30	void OnUnmap(uptr p, uptr size) const {
40	31	// We are about to unmap a chunk of user memory.
41	32	// Mark the corresponding shadow memory as not needed.
...	...	@@ -69,8 +60,17 @@ Allocator *allocator() {
69	60	struct GlobalProc {
70	61	Mutex mtx;
71	62	Processor *proc;
72
73		GlobalProc() : mtx(MutexTypeGlobalProc), proc(ProcCreate()) {}
	63	// This mutex represents the internal allocator combined for
	64	// the purposes of deadlock detection. The internal allocator
	65	// uses multiple mutexes, moreover they are locked only occasionally
	66	// and they are spin mutexes which don't support deadlock detection.
	67	// So we use this fake mutex to serve as a substitute for these mutexes.
	68	CheckedMutex internal_alloc_mtx;
	69
	70	GlobalProc()
	71	: mtx(MutexTypeGlobalProc),
	72	proc(ProcCreate()),
	73	internal_alloc_mtx(MutexTypeInternalAlloc) {}
74	74	};
75	75
76	76	static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64);
...	...	@@ -78,6 +78,11 @@ GlobalProc *global_proc() {
78	78	return reinterpret_cast<GlobalProc*>(&global_proc_placeholder);
79	79	}
80	80
	81	static void InternalAllocAccess() {
	82	global_proc()->internal_alloc_mtx.Lock();
	83	global_proc()->internal_alloc_mtx.Unlock();
	84	}
	85
81	86	ScopedGlobalProcessor::ScopedGlobalProcessor() {
82	87	GlobalProc *gp = global_proc();
83	88	ThreadState *thr = cur_thread();
...	...	@@ -110,6 +115,24 @@ ScopedGlobalProcessor::~ScopedGlobalProcessor() {
110	115	gp->mtx.Unlock();
111	116	}
112	117
	118	void AllocatorLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	119	global_proc()->internal_alloc_mtx.Lock();
	120	InternalAllocatorLock();
	121	}
	122
	123	void AllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	124	InternalAllocatorUnlock();
	125	global_proc()->internal_alloc_mtx.Unlock();
	126	}
	127
	128	void GlobalProcessorLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	129	global_proc()->mtx.Lock();
	130	}
	131
	132	void GlobalProcessorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	133	global_proc()->mtx.Unlock();
	134	}
	135
113	136	static constexpr uptr kMaxAllowedMallocSize = 1ull << 40;
114	137	static uptr max_user_defined_malloc_size;
115	138
...	...	@@ -148,7 +171,7 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
148	171	ObtainCurrentStack(thr, pc, &stack);
149	172	if (IsFiredSuppression(ctx, ReportTypeSignalUnsafe, stack))
150	173	return;
151		ThreadRegistryLock l(ctx->thread_registry);
	174	ThreadRegistryLock l(&ctx->thread_registry);
152	175	ScopedReport rep(ReportTypeSignalUnsafe);
153	176	rep.AddStack(stack, true);
154	177	OutputReport(thr, rep);
...	...	@@ -166,6 +189,12 @@ void user_alloc_internal(ThreadState thr, uptr pc, uptr sz, uptr align,
166	189	GET_STACK_TRACE_FATAL(thr, pc);
167	190	ReportAllocationSizeTooBig(sz, malloc_limit, &stack);
168	191	}
	192	if (UNLIKELY(IsRssLimitExceeded())) {
	193	if (AllocatorMayReturnNull())
	194	return nullptr;
	195	GET_STACK_TRACE_FATAL(thr, pc);
	196	ReportRssLimitExceeded(&stack);
	197	}
169	198	void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align);
170	199	if (UNLIKELY(!p)) {
171	200	SetAllocatorOutOfMemory();
...	...	@@ -218,9 +247,18 @@ void user_reallocarray(ThreadState thr, uptr pc, void *p, uptr size, uptr n) {
218	247	}
219	248
220	249	void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {
221		DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p);
	250	DPrintf("#%d: alloc(%zu) = 0x%zx\n", thr->tid, sz, p);
	251	// Note: this can run before thread initialization/after finalization.
	252	// As a result this is not necessarily synchronized with DoReset,
	253	// which iterates over and resets all sync objects,
	254	// but it is fine to create new MBlocks in this context.
222	255	ctx->metamap.AllocBlock(thr, pc, p, sz);
223		if (write && thr->ignore_reads_and_writes == 0)
	256	// If this runs before thread initialization/after finalization
	257	// and we don't have trace initialized, we can't imitate writes.
	258	// In such case just reset the shadow range, it is fine since
	259	// it affects only a small fraction of special objects.
	260	if (write && thr->ignore_reads_and_writes == 0 &&
	261	atomic_load_relaxed(&thr->trace_pos))
224	262	MemoryRangeImitateWrite(thr, pc, (uptr)p, sz);
225	263	else
226	264	MemoryResetRange(thr, pc, (uptr)p, sz);
...	...	@@ -228,8 +266,15 @@ void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {
228	266
229	267	void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) {
230	268	CHECK_NE(p, (void*)0);
231		uptr sz = ctx->metamap.FreeBlock(thr->proc(), p);
232		DPrintf("#%d: free(%p, %zu)\n", thr->tid, p, sz);
	269	if (!thr->slot) {
	270	// Very early/late in thread lifetime, or during fork.
	271	UNUSED uptr sz = ctx->metamap.FreeBlock(thr->proc(), p, false);
	272	DPrintf("#%d: free(0x%zx, %zu) (no slot)\n", thr->tid, p, sz);
	273	return;
	274	}
	275	SlotLocker locker(thr);
	276	uptr sz = ctx->metamap.FreeBlock(thr->proc(), p, true);
	277	DPrintf("#%d: free(0x%zx, %zu)\n", thr->tid, p, sz);
233	278	if (write && thr->ignore_reads_and_writes == 0)
234	279	MemoryRangeFreed(thr, pc, (uptr)p, sz);
235	280	}
...	...	@@ -309,8 +354,22 @@ void user_pvalloc(ThreadState thr, uptr pc, uptr sz) {
309	354	return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, PageSize));
310	355	}
311	356
	357	static const void user_alloc_begin(const void p) {
	358	if (p == nullptr \|\| !IsAppMem((uptr)p))
	359	return nullptr;
	360	void *beg = allocator()->GetBlockBegin(p);
	361	if (!beg)
	362	return nullptr;
	363
	364	MBlock *b = ctx->metamap.GetBlock((uptr)beg);
	365	if (!b)
	366	return nullptr; // Not a valid pointer.
	367
	368	return (const void *)beg;
	369	}
	370
312	371	uptr user_alloc_usable_size(const void *p) {
313		if (p == 0)
	372	if (p == 0 \|\| !IsAppMem((uptr)p))
314	373	return 0;
315	374	MBlock *b = ctx->metamap.GetBlock((uptr)p);
316	375	if (!b)
...	...	@@ -320,11 +379,21 @@ uptr user_alloc_usable_size(const void *p) {
320	379	return b->siz;
321	380	}
322	381
	382	uptr user_alloc_usable_size_fast(const void *p) {
	383	MBlock *b = ctx->metamap.GetBlock((uptr)p);
	384	// Static objects may have malloc'd before tsan completes
	385	// initialization, and may believe returned ptrs to be valid.
	386	if (!b)
	387	return 0; // Not a valid pointer.
	388	if (b->siz == 0)
	389	return 1; // Zero-sized allocations are actually 1 byte.
	390	return b->siz;
	391	}
	392
323	393	void invoke_malloc_hook(void *ptr, uptr size) {
324	394	ThreadState *thr = cur_thread();
325	395	if (ctx == 0 \|\| !ctx->initialized \|\| thr->ignore_interceptors)
326	396	return;
327		__sanitizer_malloc_hook(ptr, size);
328	397	RunMallocHooks(ptr, size);
329	398	}
330	399
...	...	@@ -332,25 +401,26 @@ void invoke_free_hook(void *ptr) {
332	401	ThreadState *thr = cur_thread();
333	402	if (ctx == 0 \|\| !ctx->initialized \|\| thr->ignore_interceptors)
334	403	return;
335		__sanitizer_free_hook(ptr);
336	404	RunFreeHooks(ptr);
337	405	}
338	406
339		void *internal_alloc(MBlockType typ, uptr sz) {
	407	void *Alloc(uptr sz) {
340	408	ThreadState *thr = cur_thread();
341	409	if (thr->nomalloc) {
342	410	thr->nomalloc = 0; // CHECK calls internal_malloc().
343	411	CHECK(0);
344	412	}
	413	InternalAllocAccess();
345	414	return InternalAlloc(sz, &thr->proc()->internal_alloc_cache);
346	415	}
347	416
348		void internal_free(void *p) {
	417	void FreeImpl(void *p) {
349	418	ThreadState *thr = cur_thread();
350	419	if (thr->nomalloc) {
351	420	thr->nomalloc = 0; // CHECK calls internal_malloc().
352	421	CHECK(0);
353	422	}
	423	InternalAllocAccess();
354	424	InternalFree(p, &thr->proc()->internal_alloc_cache);
355	425	}
356	426
...	...	@@ -387,14 +457,27 @@ int __sanitizer_get_ownership(const void *p) {
387	457	return allocator()->GetBlockBegin(p) != 0;
388	458	}
389	459
	460	const void __sanitizer_get_allocated_begin(const void p) {
	461	return user_alloc_begin(p);
	462	}
	463
390	464	uptr __sanitizer_get_allocated_size(const void *p) {
391	465	return user_alloc_usable_size(p);
392	466	}
393	467
	468	uptr __sanitizer_get_allocated_size_fast(const void *p) {
	469	DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
	470	uptr ret = user_alloc_usable_size_fast(p);
	471	DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
	472	return ret;
	473	}
	474
	475	void __sanitizer_purge_allocator() {
	476	allocator()->ForceReleaseToOS();
	477	}
	478
394	479	void __tsan_on_thread_idle() {
395	480	ThreadState *thr = cur_thread();
396		thr->clock.ResetCached(&thr->proc()->clock_cache);
397		thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
398	481	allocator()->SwallowCache(&thr->proc()->alloc_cache);
399	482	internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache);
400	483	ctx->metamap.OnProcIdle(thr->proc());

lib/tsan/tsan_mman.h+22-31

...	...	@@ -24,6 +24,10 @@ void ReplaceSystemMalloc();
24	24	void AllocatorProcStart(Processor *proc);
25	25	void AllocatorProcFinish(Processor *proc);
26	26	void AllocatorPrintStats();
	27	void AllocatorLock();
	28	void AllocatorUnlock();
	29	void GlobalProcessorLock();
	30	void GlobalProcessorUnlock();
27	31
28	32	// For user allocations.
29	33	void user_alloc_internal(ThreadState thr, uptr pc, uptr sz,
...	...	@@ -47,42 +51,29 @@ uptr user_alloc_usable_size(const void *p);
47	51	void invoke_malloc_hook(void *ptr, uptr size);
48	52	void invoke_free_hook(void *ptr);
49	53
50		enum MBlockType {
51		MBlockScopedBuf,
52		MBlockString,
53		MBlockStackTrace,
54		MBlockShadowStack,
55		MBlockSync,
56		MBlockClock,
57		MBlockThreadContex,
58		MBlockDeadInfo,
59		MBlockRacyStacks,
60		MBlockRacyAddresses,
61		MBlockAtExit,
62		MBlockFlag,
63		MBlockReport,
64		MBlockReportMop,
65		MBlockReportThread,
66		MBlockReportMutex,
67		MBlockReportLoc,
68		MBlockReportStack,
69		MBlockSuppression,
70		MBlockExpectRace,
71		MBlockSignal,
72		MBlockJmpBuf,
	54	// For internal data structures.
	55	void *Alloc(uptr sz);
	56	void FreeImpl(void *p);
73	57
74		// This must be the last.
75		MBlockTypeCount
76		};
	58	template <typename T, typename... Args>
	59	T *New(Args &&...args) {
	60	return new (Alloc(sizeof(T))) T(static_cast<Args &&>(args)...);
	61	}
77	62
78		// For internal data structures.
79		void *internal_alloc(MBlockType typ, uptr sz);
80		void internal_free(void *p);
	63	template <typename T>
	64	void Free(T *&p) {
	65	if (p == nullptr)
	66	return;
	67	FreeImpl(p);
	68	p = nullptr;
	69	}
81	70
82	71	template <typename T>
83		void DestroyAndFree(T *p) {
	72	void DestroyAndFree(T *&p) {
	73	if (p == nullptr)
	74	return;
84	75	p->~T();
85		internal_free(p);
	76	Free(p);
86	77	}
87	78
88	79	} // namespace __tsan

lib/tsan/tsan_mutexset.cpp+20-28

...	...	@@ -10,66 +10,55 @@
10	10	//
11	11	//===----------------------------------------------------------------------===//
12	12	#include "tsan_mutexset.h"
	13
	14	#include "sanitizer_common/sanitizer_placement_new.h"
13	15	#include "tsan_rtl.h"
14	16
15	17	namespace __tsan {
16	18
17		const uptr MutexSet::kMaxSize;
18
19	19	MutexSet::MutexSet() {
20		size_ = 0;
21		internal_memset(&descs_, 0, sizeof(descs_));
22	20	}
23	21
24		void MutexSet::Add(u64 id, bool write, u64 epoch) {
	22	void MutexSet::Reset() { internal_memset(this, 0, sizeof(*this)); }
	23
	24	void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) {
25	25	// Look up existing mutex with the same id.
26	26	for (uptr i = 0; i < size_; i++) {
27		if (descs_[i].id == id) {
	27	if (descs_[i].addr == addr) {
28	28	descs_[i].count++;
29		descs_[i].epoch = epoch;
	29	descs_[i].seq = seq_++;
30	30	return;
31	31	}
32	32	}
33	33	// On overflow, find the oldest mutex and drop it.
34	34	if (size_ == kMaxSize) {
35		u64 minepoch = (u64)-1;
36		u64 mini = (u64)-1;
	35	uptr min = 0;
37	36	for (uptr i = 0; i < size_; i++) {
38		if (descs_[i].epoch < minepoch) {
39		minepoch = descs_[i].epoch;
40		mini = i;
41		}
	37	if (descs_[i].seq < descs_[min].seq)
	38	min = i;
42	39	}
43		RemovePos(mini);
	40	RemovePos(min);
44	41	CHECK_EQ(size_, kMaxSize - 1);
45	42	}
46	43	// Add new mutex descriptor.
47		descs_[size_].id = id;
	44	descs_[size_].addr = addr;
	45	descs_[size_].stack_id = stack_id;
48	46	descs_[size_].write = write;
49		descs_[size_].epoch = epoch;
	47	descs_[size_].seq = seq_++;
50	48	descs_[size_].count = 1;
51	49	size_++;
52	50	}
53	51
54		void MutexSet::Del(u64 id, bool write) {
	52	void MutexSet::DelAddr(uptr addr, bool destroy) {
55	53	for (uptr i = 0; i < size_; i++) {
56		if (descs_[i].id == id) {
57		if (--descs_[i].count == 0)
	54	if (descs_[i].addr == addr) {
	55	if (destroy \|\| --descs_[i].count == 0)
58	56	RemovePos(i);
59	57	return;
60	58	}
61	59	}
62	60	}
63	61
64		void MutexSet::Remove(u64 id) {
65		for (uptr i = 0; i < size_; i++) {
66		if (descs_[i].id == id) {
67		RemovePos(i);
68		return;
69		}
70		}
71		}
72
73	62	void MutexSet::RemovePos(uptr i) {
74	63	CHECK_LT(i, size_);
75	64	descs_[i] = descs_[size_ - 1];
...	...	@@ -85,4 +74,7 @@ MutexSet::Desc MutexSet::Get(uptr i) const {
85	74	return descs_[i];
86	75	}
87	76
	77	DynamicMutexSet::DynamicMutexSet() : ptr_(New<MutexSet>()) {}
	78	DynamicMutexSet::~DynamicMutexSet() { DestroyAndFree(ptr_); }
	79
88	80	} // namespace __tsan

lib/tsan/tsan_mutexset.h+41-19

...	...	@@ -21,34 +21,55 @@ class MutexSet {
21	21	public:
22	22	// Holds limited number of mutexes.
23	23	// The oldest mutexes are discarded on overflow.
24		static const uptr kMaxSize = 16;
	24	static constexpr uptr kMaxSize = 16;
25	25	struct Desc {
26		u64 id;
27		u64 epoch;
28		int count;
	26	uptr addr;
	27	StackID stack_id;
	28	u32 seq;
	29	u32 count;
29	30	bool write;
	31
	32	Desc() { internal_memset(this, 0, sizeof(*this)); }
	33	Desc(const Desc& other) { *this = other; }
	34	Desc& operator=(const MutexSet::Desc& other) {
	35	internal_memcpy(this, &other, sizeof(*this));
	36	return *this;
	37	}
30	38	};
31	39
32	40	MutexSet();
33		// The 'id' is obtained from SyncVar::GetId().
34		void Add(u64 id, bool write, u64 epoch);
35		void Del(u64 id, bool write);
36		void Remove(u64 id); // Removes the mutex completely (if it's destroyed).
	41	void Reset();
	42	void AddAddr(uptr addr, StackID stack_id, bool write);
	43	void DelAddr(uptr addr, bool destroy = false);
37	44	uptr Size() const;
38	45	Desc Get(uptr i) const;
39	46
40		void operator=(const MutexSet &other) {
41		internal_memcpy(this, &other, sizeof(*this));
42		}
43
44	47	private:
45	48	#if !SANITIZER_GO
46		uptr size_;
	49	u32 seq_ = 0;
	50	uptr size_ = 0;
47	51	Desc descs_[kMaxSize];
48		#endif
49	52
50	53	void RemovePos(uptr i);
51		MutexSet(const MutexSet&);
	54	#endif
	55	};
	56
	57	// MutexSet is too large to live on stack.
	58	// DynamicMutexSet can be use used to create local MutexSet's.
	59	class DynamicMutexSet {
	60	public:
	61	DynamicMutexSet();
	62	~DynamicMutexSet();
	63	MutexSet* operator->() { return ptr_; }
	64	operator MutexSet*() { return ptr_; }
	65	DynamicMutexSet(const DynamicMutexSet&) = delete;
	66	DynamicMutexSet& operator=(const DynamicMutexSet&) = delete;
	67
	68	private:
	69	MutexSet* ptr_;
	70	#if SANITIZER_GO
	71	MutexSet set_;
	72	#endif
52	73	};
53	74
54	75	// Go does not have mutexes, so do not spend memory and time.
...	...	@@ -56,12 +77,13 @@ class MutexSet {
56	77	// in different goroutine).
57	78	#if SANITIZER_GO
58	79	MutexSet::MutexSet() {}
59		void MutexSet::Add(u64 id, bool write, u64 epoch) {}
60		void MutexSet::Del(u64 id, bool write) {}
61		void MutexSet::Remove(u64 id) {}
62		void MutexSet::RemovePos(uptr i) {}
	80	void MutexSet::Reset() {}
	81	void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) {}
	82	void MutexSet::DelAddr(uptr addr, bool destroy) {}
63	83	uptr MutexSet::Size() const { return 0; }
64	84	MutexSet::Desc MutexSet::Get(uptr i) const { return Desc(); }
	85	DynamicMutexSet::DynamicMutexSet() : ptr_(&set_) {}
	86	DynamicMutexSet::~DynamicMutexSet() {}
65	87	#endif
66	88
67	89	} // namespace __tsan

lib/tsan/tsan_new_delete.cpp created+199

...	...	@@ -0,0 +1,199 @@
	1	//===-- tsan_new_delete.cpp ---------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	// Interceptors for operators new and delete.
	12	//===----------------------------------------------------------------------===//
	13	#include "interception/interception.h"
	14	#include "sanitizer_common/sanitizer_allocator.h"
	15	#include "sanitizer_common/sanitizer_allocator_report.h"
	16	#include "sanitizer_common/sanitizer_internal_defs.h"
	17	#include "tsan_interceptors.h"
	18	#include "tsan_rtl.h"
	19
	20	using namespace __tsan;
	21
	22	namespace std {
	23	struct nothrow_t {};
	24	enum class align_val_t: __sanitizer::uptr {};
	25	} // namespace std
	26
	27	DECLARE_REAL(void *, malloc, uptr size)
	28	DECLARE_REAL(void, free, void *ptr)
	29
	30	// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
	31	#define OPERATOR_NEW_BODY(mangled_name, nothrow) \
	32	if (in_symbolizer()) \
	33	return InternalAlloc(size); \
	34	void *p = 0; \
	35	{ \
	36	SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
	37	p = user_alloc(thr, pc, size); \
	38	if (!nothrow && UNLIKELY(!p)) { \
	39	GET_STACK_TRACE_FATAL(thr, pc); \
	40	ReportOutOfMemory(size, &stack); \
	41	} \
	42	} \
	43	invoke_malloc_hook(p, size); \
	44	return p;
	45
	46	#define OPERATOR_NEW_BODY_ALIGN(mangled_name, nothrow) \
	47	if (in_symbolizer()) \
	48	return InternalAlloc(size, nullptr, (uptr)align); \
	49	void *p = 0; \
	50	{ \
	51	SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
	52	p = user_memalign(thr, pc, (uptr)align, size); \
	53	if (!nothrow && UNLIKELY(!p)) { \
	54	GET_STACK_TRACE_FATAL(thr, pc); \
	55	ReportOutOfMemory(size, &stack); \
	56	} \
	57	} \
	58	invoke_malloc_hook(p, size); \
	59	return p;
	60
	61	SANITIZER_INTERFACE_ATTRIBUTE
	62	void *operator new(__sanitizer::uptr size);
	63	void *operator new(__sanitizer::uptr size) {
	64	OPERATOR_NEW_BODY(_Znwm, false /nothrow/);
	65	}
	66
	67	SANITIZER_INTERFACE_ATTRIBUTE
	68	void *operator new[](__sanitizer::uptr size);
	69	void *operator new[](__sanitizer::uptr size) {
	70	OPERATOR_NEW_BODY(_Znam, false /nothrow/);
	71	}
	72
	73	SANITIZER_INTERFACE_ATTRIBUTE
	74	void *operator new(__sanitizer::uptr size, std::nothrow_t const&);
	75	void *operator new(__sanitizer::uptr size, std::nothrow_t const&) {
	76	OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t, true /nothrow/);
	77	}
	78
	79	SANITIZER_INTERFACE_ATTRIBUTE
	80	void *operator new[](__sanitizer::uptr size, std::nothrow_t const&);
	81	void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) {
	82	OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t, true /nothrow/);
	83	}
	84
	85	SANITIZER_INTERFACE_ATTRIBUTE
	86	void *operator new(__sanitizer::uptr size, std::align_val_t align);
	87	void *operator new(__sanitizer::uptr size, std::align_val_t align) {
	88	OPERATOR_NEW_BODY_ALIGN(_ZnwmSt11align_val_t, false /nothrow/);
	89	}
	90
	91	SANITIZER_INTERFACE_ATTRIBUTE
	92	void *operator new[](__sanitizer::uptr size, std::align_val_t align);
	93	void *operator new[](__sanitizer::uptr size, std::align_val_t align) {
	94	OPERATOR_NEW_BODY_ALIGN(_ZnamSt11align_val_t, false /nothrow/);
	95	}
	96
	97	SANITIZER_INTERFACE_ATTRIBUTE
	98	void *operator new(__sanitizer::uptr size, std::align_val_t align,
	99	std::nothrow_t const&);
	100	void *operator new(__sanitizer::uptr size, std::align_val_t align,
	101	std::nothrow_t const&) {
	102	OPERATOR_NEW_BODY_ALIGN(_ZnwmSt11align_val_tRKSt9nothrow_t,
	103	true /nothrow/);
	104	}
	105
	106	SANITIZER_INTERFACE_ATTRIBUTE
	107	void *operator new[](__sanitizer::uptr size, std::align_val_t align,
	108	std::nothrow_t const&);
	109	void *operator new[](__sanitizer::uptr size, std::align_val_t align,
	110	std::nothrow_t const&) {
	111	OPERATOR_NEW_BODY_ALIGN(_ZnamSt11align_val_tRKSt9nothrow_t,
	112	true /nothrow/);
	113	}
	114
	115	#define OPERATOR_DELETE_BODY(mangled_name) \
	116	if (ptr == 0) return; \
	117	if (in_symbolizer()) \
	118	return InternalFree(ptr); \
	119	invoke_free_hook(ptr); \
	120	SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \
	121	user_free(thr, pc, ptr);
	122
	123	SANITIZER_INTERFACE_ATTRIBUTE
	124	void operator delete(void *ptr) NOEXCEPT;
	125	void operator delete(void *ptr) NOEXCEPT {
	126	OPERATOR_DELETE_BODY(_ZdlPv);
	127	}
	128
	129	SANITIZER_INTERFACE_ATTRIBUTE
	130	void operator delete[](void *ptr) NOEXCEPT;
	131	void operator delete[](void *ptr) NOEXCEPT {
	132	OPERATOR_DELETE_BODY(_ZdaPv);
	133	}
	134
	135	SANITIZER_INTERFACE_ATTRIBUTE
	136	void operator delete(void *ptr, std::nothrow_t const&);
	137	void operator delete(void *ptr, std::nothrow_t const&) {
	138	OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t);
	139	}
	140
	141	SANITIZER_INTERFACE_ATTRIBUTE
	142	void operator delete[](void *ptr, std::nothrow_t const&);
	143	void operator delete[](void *ptr, std::nothrow_t const&) {
	144	OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t);
	145	}
	146
	147	SANITIZER_INTERFACE_ATTRIBUTE
	148	void operator delete(void *ptr, __sanitizer::uptr size) NOEXCEPT;
	149	void operator delete(void *ptr, __sanitizer::uptr size) NOEXCEPT {
	150	OPERATOR_DELETE_BODY(_ZdlPvm);
	151	}
	152
	153	SANITIZER_INTERFACE_ATTRIBUTE
	154	void operator delete[](void *ptr, __sanitizer::uptr size) NOEXCEPT;
	155	void operator delete[](void *ptr, __sanitizer::uptr size) NOEXCEPT {
	156	OPERATOR_DELETE_BODY(_ZdaPvm);
	157	}
	158
	159	SANITIZER_INTERFACE_ATTRIBUTE
	160	void operator delete(void *ptr, std::align_val_t align) NOEXCEPT;
	161	void operator delete(void *ptr, std::align_val_t align) NOEXCEPT {
	162	OPERATOR_DELETE_BODY(_ZdlPvSt11align_val_t);
	163	}
	164
	165	SANITIZER_INTERFACE_ATTRIBUTE
	166	void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT;
	167	void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT {
	168	OPERATOR_DELETE_BODY(_ZdaPvSt11align_val_t);
	169	}
	170
	171	SANITIZER_INTERFACE_ATTRIBUTE
	172	void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&);
	173	void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&) {
	174	OPERATOR_DELETE_BODY(_ZdlPvSt11align_val_tRKSt9nothrow_t);
	175	}
	176
	177	SANITIZER_INTERFACE_ATTRIBUTE
	178	void operator delete[](void *ptr, std::align_val_t align,
	179	std::nothrow_t const&);
	180	void operator delete[](void *ptr, std::align_val_t align,
	181	std::nothrow_t const&) {
	182	OPERATOR_DELETE_BODY(_ZdaPvSt11align_val_tRKSt9nothrow_t);
	183	}
	184
	185	SANITIZER_INTERFACE_ATTRIBUTE
	186	void operator delete(void *ptr, __sanitizer::uptr size,
	187	std::align_val_t align) NOEXCEPT;
	188	void operator delete(void *ptr, __sanitizer::uptr size,
	189	std::align_val_t align) NOEXCEPT {
	190	OPERATOR_DELETE_BODY(_ZdlPvmSt11align_val_t);
	191	}
	192
	193	SANITIZER_INTERFACE_ATTRIBUTE
	194	void operator delete[](void *ptr, __sanitizer::uptr size,
	195	std::align_val_t align) NOEXCEPT;
	196	void operator delete[](void *ptr, __sanitizer::uptr size,
	197	std::align_val_t align) NOEXCEPT {
	198	OPERATOR_DELETE_BODY(_ZdaPvmSt11align_val_t);
	199	}

lib/tsan/tsan_platform.h+568-841

...	...	@@ -18,38 +18,42 @@
18	18	# error "Only 64-bit is supported"
19	19	#endif
20	20
	21	#include "sanitizer_common/sanitizer_common.h"
21	22	#include "tsan_defs.h"
22		#include "tsan_trace.h"
23	23
24	24	namespace __tsan {
25	25
26		#if defined(__x86_64__)
27		#define HAS_48_BIT_ADDRESS_SPACE 1
28		#elif SANITIZER_IOSSIM // arm64 iOS simulators (order of #if matters)
29		#define HAS_48_BIT_ADDRESS_SPACE 1
30		#elif SANITIZER_IOS // arm64 iOS devices (order of #if matters)
31		#define HAS_48_BIT_ADDRESS_SPACE 0
32		#elif SANITIZER_MAC // arm64 macOS (order of #if matters)
33		#define HAS_48_BIT_ADDRESS_SPACE 1
34		#else
35		#define HAS_48_BIT_ADDRESS_SPACE 0
36		#endif
37
38		#if !SANITIZER_GO
	26	enum {
	27	// App memory is not mapped onto shadow memory range.
	28	kBrokenMapping = 1 << 0,
	29	// Mapping app memory and back does not produce the same address,
	30	// this can lead to wrong addresses in reports and potentially
	31	// other bad consequences.
	32	kBrokenReverseMapping = 1 << 1,
	33	// Mapping is non-linear for linear user range.
	34	// This is bad and can lead to unpredictable memory corruptions, etc
	35	// because range access functions assume linearity.
	36	kBrokenLinearity = 1 << 2,
	37	// Meta for an app region overlaps with the meta of another app region.
	38	// This is determined by recomputing the individual meta regions for
	39	// each app region.
	40	//
	41	// N.B. There is no "kBrokenReverseMetaMapping" constant because there
	42	// is no MetaToMem function. However, note that (!kBrokenLinearity
	43	// && !kBrokenAliasedMetas) implies that MemToMeta is invertible.
	44	kBrokenAliasedMetas = 1 << 3,
	45	};
39	46
40		#if HAS_48_BIT_ADDRESS_SPACE
41	47	/*
42	48	C/C++ on linux/x86_64 and freebsd/x86_64
43	49	0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
44	50	0040 0000 0000 - 0100 0000 0000: -
45		0100 0000 0000 - 2000 0000 0000: shadow
46		2000 0000 0000 - 3000 0000 0000: -
47		3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
48		4000 0000 0000 - 5500 0000 0000: -
	51	0100 0000 0000 - 1000 0000 0000: shadow
	52	1000 0000 0000 - 3000 0000 0000: -
	53	3000 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects)
	54	3400 0000 0000 - 5500 0000 0000: -
49	55	5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels
50		5680 0000 0000 - 6000 0000 0000: -
51		6000 0000 0000 - 6200 0000 0000: traces
52		6200 0000 0000 - 7d00 0000 0000: -
	56	5680 0000 0000 - 7d00 0000 0000: -
53	57	7b00 0000 0000 - 7c00 0000 0000: heap
54	58	7c00 0000 0000 - 7e80 0000 0000: -
55	59	7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
...	...	@@ -65,15 +69,12 @@ C/C++ on netbsd/amd64 can reuse the same mapping:
65	69	* Stack on NetBSD/amd64 has prereserved 128MB.
66	70	* Heap grows downwards (top-down).
67	71	* ASLR must be disabled per-process or globally.
68
69	72	*/
70		struct Mapping {
	73	struct Mapping48AddressSpace {
71	74	static const uptr kMetaShadowBeg = 0x300000000000ull;
72	75	static const uptr kMetaShadowEnd = 0x340000000000ull;
73		static const uptr kTraceMemBeg = 0x600000000000ull;
74		static const uptr kTraceMemEnd = 0x620000000000ull;
75	76	static const uptr kShadowBeg = 0x010000000000ull;
76		static const uptr kShadowEnd = 0x200000000000ull;
	77	static const uptr kShadowEnd = 0x100000000000ull;
77	78	static const uptr kHeapMemBeg = 0x7b0000000000ull;
78	79	static const uptr kHeapMemEnd = 0x7c0000000000ull;
79	80	static const uptr kLoAppMemBeg = 0x000000001000ull;
...	...	@@ -82,36 +83,32 @@ struct Mapping {
82	83	static const uptr kMidAppMemEnd = 0x568000000000ull;
83	84	static const uptr kHiAppMemBeg = 0x7e8000000000ull;
84	85	static const uptr kHiAppMemEnd = 0x800000000000ull;
85		static const uptr kAppMemMsk = 0x780000000000ull;
86		static const uptr kAppMemXor = 0x040000000000ull;
	86	static const uptr kShadowMsk = 0x780000000000ull;
	87	static const uptr kShadowXor = 0x040000000000ull;
	88	static const uptr kShadowAdd = 0x000000000000ull;
87	89	static const uptr kVdsoBeg = 0xf000000000000000ull;
88	90	};
89	91
90		#define TSAN_MID_APP_RANGE 1
91		#elif defined(__mips64)
92	92	/*
93	93	C/C++ on linux/mips64 (40-bit VMA)
94	94	0000 0000 00 - 0100 0000 00: - (4 GB)
95	95	0100 0000 00 - 0200 0000 00: main binary (4 GB)
96		0200 0000 00 - 2000 0000 00: - (120 GB)
97		2000 0000 00 - 4000 0000 00: shadow (128 GB)
	96	0200 0000 00 - 1200 0000 00: - (64 GB)
	97	1200 0000 00 - 2200 0000 00: shadow (64 GB)
	98	2200 0000 00 - 4000 0000 00: - (120 GB)
98	99	4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects) (64 GB)
99	100	5000 0000 00 - aa00 0000 00: - (360 GB)
100	101	aa00 0000 00 - ab00 0000 00: main binary (PIE) (4 GB)
101		ab00 0000 00 - b000 0000 00: - (20 GB)
102		b000 0000 00 - b200 0000 00: traces (8 GB)
103		b200 0000 00 - fe00 0000 00: - (304 GB)
	102	ab00 0000 00 - fe00 0000 00: - (332 GB)
104	103	fe00 0000 00 - ff00 0000 00: heap (4 GB)
105	104	ff00 0000 00 - ff80 0000 00: - (2 GB)
106	105	ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB)
107	106	*/
108		struct Mapping40 {
	107	struct MappingMips64_40 {
109	108	static const uptr kMetaShadowBeg = 0x4000000000ull;
110	109	static const uptr kMetaShadowEnd = 0x5000000000ull;
111		static const uptr kTraceMemBeg = 0xb000000000ull;
112		static const uptr kTraceMemEnd = 0xb200000000ull;
113		static const uptr kShadowBeg = 0x2000000000ull;
114		static const uptr kShadowEnd = 0x4000000000ull;
	110	static const uptr kShadowBeg = 0x1200000000ull;
	111	static const uptr kShadowEnd = 0x2200000000ull;
115	112	static const uptr kHeapMemBeg = 0xfe00000000ull;
116	113	static const uptr kHeapMemEnd = 0xff00000000ull;
117	114	static const uptr kLoAppMemBeg = 0x0100000000ull;
...	...	@@ -120,152 +117,170 @@ struct Mapping40 {
120	117	static const uptr kMidAppMemEnd = 0xab00000000ull;
121	118	static const uptr kHiAppMemBeg = 0xff80000000ull;
122	119	static const uptr kHiAppMemEnd = 0xffffffffffull;
123		static const uptr kAppMemMsk = 0xf800000000ull;
124		static const uptr kAppMemXor = 0x0800000000ull;
	120	static const uptr kShadowMsk = 0xf800000000ull;
	121	static const uptr kShadowXor = 0x0800000000ull;
	122	static const uptr kShadowAdd = 0x0000000000ull;
125	123	static const uptr kVdsoBeg = 0xfffff00000ull;
126	124	};
127	125
128		#define TSAN_MID_APP_RANGE 1
129		#define TSAN_RUNTIME_VMA 1
130		#elif defined(__aarch64__) && defined(__APPLE__)
131	126	/*
132	127	C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
133	128	0000 0000 00 - 0100 0000 00: - (4 GB)
134	129	0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB)
135	130	0200 0000 00 - 0300 0000 00: heap (4 GB)
136	131	0300 0000 00 - 0400 0000 00: - (4 GB)
137		0400 0000 00 - 0c00 0000 00: shadow memory (32 GB)
138		0c00 0000 00 - 0d00 0000 00: - (4 GB)
	132	0400 0000 00 - 0800 0000 00: shadow memory (16 GB)
	133	0800 0000 00 - 0d00 0000 00: - (20 GB)
139	134	0d00 0000 00 - 0e00 0000 00: metainfo (4 GB)
140		0e00 0000 00 - 0f00 0000 00: - (4 GB)
141		0f00 0000 00 - 0fc0 0000 00: traces (3 GB)
142		0fc0 0000 00 - 1000 0000 00: -
	135	0e00 0000 00 - 1000 0000 00: -
143	136	*/
144		struct Mapping {
	137	struct MappingAppleAarch64 {
145	138	static const uptr kLoAppMemBeg = 0x0100000000ull;
146	139	static const uptr kLoAppMemEnd = 0x0200000000ull;
147	140	static const uptr kHeapMemBeg = 0x0200000000ull;
148	141	static const uptr kHeapMemEnd = 0x0300000000ull;
149	142	static const uptr kShadowBeg = 0x0400000000ull;
150		static const uptr kShadowEnd = 0x0c00000000ull;
	143	static const uptr kShadowEnd = 0x0800000000ull;
151	144	static const uptr kMetaShadowBeg = 0x0d00000000ull;
152	145	static const uptr kMetaShadowEnd = 0x0e00000000ull;
153		static const uptr kTraceMemBeg = 0x0f00000000ull;
154		static const uptr kTraceMemEnd = 0x0fc0000000ull;
155	146	static const uptr kHiAppMemBeg = 0x0fc0000000ull;
156	147	static const uptr kHiAppMemEnd = 0x0fc0000000ull;
157		static const uptr kAppMemMsk = 0x0ull;
158		static const uptr kAppMemXor = 0x0ull;
	148	static const uptr kShadowMsk = 0x0ull;
	149	static const uptr kShadowXor = 0x0ull;
	150	static const uptr kShadowAdd = 0x0200000000ull;
159	151	static const uptr kVdsoBeg = 0x7000000000000000ull;
	152	static const uptr kMidAppMemBeg = 0;
	153	static const uptr kMidAppMemEnd = 0;
160	154	};
161	155
162		#elif defined(__aarch64__) && !defined(__APPLE__)
163		// AArch64 supports multiple VMA which leads to multiple address transformation
164		// functions. To support these multiple VMAS transformations and mappings TSAN
165		// runtime for AArch64 uses an external memory read (vmaSize) to select which
166		// mapping to use. Although slower, it make a same instrumented binary run on
167		// multiple kernels.
168
169	156	/*
170	157	C/C++ on linux/aarch64 (39-bit VMA)
171		0000 0010 00 - 0100 0000 00: main binary
172		0100 0000 00 - 0800 0000 00: -
173		0800 0000 00 - 2000 0000 00: shadow memory
174		2000 0000 00 - 3100 0000 00: -
175		3100 0000 00 - 3400 0000 00: metainfo
176		3400 0000 00 - 5500 0000 00: -
177		5500 0000 00 - 5600 0000 00: main binary (PIE)
178		5600 0000 00 - 6000 0000 00: -
179		6000 0000 00 - 6200 0000 00: traces
180		6200 0000 00 - 7d00 0000 00: -
181		7c00 0000 00 - 7d00 0000 00: heap
182		7d00 0000 00 - 7fff ffff ff: modules and main thread stack
	158	0000 0010 00 - 0500 0000 00: main binary (20 GB)
	159	0100 0000 00 - 2000 0000 00: -
	160	2000 0000 00 - 4000 0000 00: shadow memory (128 GB)
	161	4000 0000 00 - 4800 0000 00: metainfo (32 GB)
	162	4800 0000 00 - 5500 0000 00: -
	163	5500 0000 00 - 5a00 0000 00: main binary (PIE) (20 GB)
	164	5600 0000 00 - 7c00 0000 00: -
	165	7a00 0000 00 - 7d00 0000 00: heap (12 GB)
	166	7d00 0000 00 - 7fff ffff ff: modules and main thread stack (12 GB)
183	167	*/
184		struct Mapping39 {
	168	struct MappingAarch64_39 {
185	169	static const uptr kLoAppMemBeg = 0x0000001000ull;
186		static const uptr kLoAppMemEnd = 0x0100000000ull;
187		static const uptr kShadowBeg = 0x0800000000ull;
188		static const uptr kShadowEnd = 0x2000000000ull;
189		static const uptr kMetaShadowBeg = 0x3100000000ull;
190		static const uptr kMetaShadowEnd = 0x3400000000ull;
	170	static const uptr kLoAppMemEnd = 0x0500000000ull;
	171	static const uptr kShadowBeg = 0x2000000000ull;
	172	static const uptr kShadowEnd = 0x4000000000ull;
	173	static const uptr kMetaShadowBeg = 0x4000000000ull;
	174	static const uptr kMetaShadowEnd = 0x4800000000ull;
191	175	static const uptr kMidAppMemBeg = 0x5500000000ull;
192		static const uptr kMidAppMemEnd = 0x5600000000ull;
193		static const uptr kTraceMemBeg = 0x6000000000ull;
194		static const uptr kTraceMemEnd = 0x6200000000ull;
195		static const uptr kHeapMemBeg = 0x7c00000000ull;
	176	static const uptr kMidAppMemEnd = 0x5a00000000ull;
	177	static const uptr kHeapMemBeg = 0x7a00000000ull;
196	178	static const uptr kHeapMemEnd = 0x7d00000000ull;
197		static const uptr kHiAppMemBeg = 0x7e00000000ull;
	179	static const uptr kHiAppMemBeg = 0x7d00000000ull;
198	180	static const uptr kHiAppMemEnd = 0x7fffffffffull;
199		static const uptr kAppMemMsk = 0x7800000000ull;
200		static const uptr kAppMemXor = 0x0200000000ull;
	181	static const uptr kShadowMsk = 0x7000000000ull;
	182	static const uptr kShadowXor = 0x1000000000ull;
	183	static const uptr kShadowAdd = 0x0000000000ull;
201	184	static const uptr kVdsoBeg = 0x7f00000000ull;
202	185	};
203	186
204	187	/*
205	188	C/C++ on linux/aarch64 (42-bit VMA)
206		00000 0010 00 - 01000 0000 00: main binary
207		01000 0000 00 - 10000 0000 00: -
208		10000 0000 00 - 20000 0000 00: shadow memory
209		20000 0000 00 - 26000 0000 00: -
210		26000 0000 00 - 28000 0000 00: metainfo
211		28000 0000 00 - 2aa00 0000 00: -
212		2aa00 0000 00 - 2ab00 0000 00: main binary (PIE)
213		2ab00 0000 00 - 36200 0000 00: -
214		36200 0000 00 - 36240 0000 00: traces
215		36240 0000 00 - 3e000 0000 00: -
216		3e000 0000 00 - 3f000 0000 00: heap
217		3f000 0000 00 - 3ffff ffff ff: modules and main thread stack
	189	00000 0010 00 - 02000 0000 00: main binary (128 GB)
	190	02000 0000 00 - 08000 0000 00: -
	191	10000 0000 00 - 20000 0000 00: shadow memory (1024 GB)
	192	20000 0000 00 - 24000 0000 00: metainfo (256 GB)
	193	24000 0000 00 - 2aa00 0000 00: -
	194	2aa00 0000 00 - 2c000 0000 00: main binary (PIE) (88 GB)
	195	2c000 0000 00 - 3c000 0000 00: -
	196	3c000 0000 00 - 3f000 0000 00: heap (192 GB)
	197	3f000 0000 00 - 3ffff ffff ff: modules and main thread stack (64 GB)
218	198	*/
219		struct Mapping42 {
	199	struct MappingAarch64_42 {
220	200	static const uptr kLoAppMemBeg = 0x00000001000ull;
221		static const uptr kLoAppMemEnd = 0x01000000000ull;
	201	static const uptr kLoAppMemEnd = 0x02000000000ull;
222	202	static const uptr kShadowBeg = 0x10000000000ull;
223	203	static const uptr kShadowEnd = 0x20000000000ull;
224		static const uptr kMetaShadowBeg = 0x26000000000ull;
225		static const uptr kMetaShadowEnd = 0x28000000000ull;
	204	static const uptr kMetaShadowBeg = 0x20000000000ull;
	205	static const uptr kMetaShadowEnd = 0x24000000000ull;
226	206	static const uptr kMidAppMemBeg = 0x2aa00000000ull;
227		static const uptr kMidAppMemEnd = 0x2ab00000000ull;
228		static const uptr kTraceMemBeg = 0x36200000000ull;
229		static const uptr kTraceMemEnd = 0x36400000000ull;
230		static const uptr kHeapMemBeg = 0x3e000000000ull;
	207	static const uptr kMidAppMemEnd = 0x2c000000000ull;
	208	static const uptr kHeapMemBeg = 0x3c000000000ull;
231	209	static const uptr kHeapMemEnd = 0x3f000000000ull;
232	210	static const uptr kHiAppMemBeg = 0x3f000000000ull;
233	211	static const uptr kHiAppMemEnd = 0x3ffffffffffull;
234		static const uptr kAppMemMsk = 0x3c000000000ull;
235		static const uptr kAppMemXor = 0x04000000000ull;
	212	static const uptr kShadowMsk = 0x38000000000ull;
	213	static const uptr kShadowXor = 0x08000000000ull;
	214	static const uptr kShadowAdd = 0x00000000000ull;
236	215	static const uptr kVdsoBeg = 0x37f00000000ull;
237	216	};
238	217
239		struct Mapping48 {
	218	/*
	219	C/C++ on linux/aarch64 (48-bit VMA)
	220	0000 0000 1000 - 0a00 0000 0000: main binary (10240 GB)
	221	0a00 0000 1000 - 1554 0000 0000: -
	222	1554 0000 1000 - 5400 0000 0000: shadow memory (64176 GB)
	223	5400 0000 1000 - 8000 0000 0000: -
	224	8000 0000 1000 - 0a00 0000 0000: metainfo (32768 GB)
	225	a000 0000 1000 - aaaa 0000 0000: -
	226	aaaa 0000 1000 - ac00 0000 0000: main binary (PIE) (1368 GB)
	227	ac00 0000 1000 - fc00 0000 0000: -
	228	fc00 0000 1000 - ffff ffff ffff: modules and main thread stack (4096 GB)
	229
	230	N.B. the shadow memory region has a strange start address, because it
	231	contains the shadows for the mid, high and low app regions (in this
	232	unusual order).
	233	*/
	234	struct MappingAarch64_48 {
240	235	static const uptr kLoAppMemBeg = 0x0000000001000ull;
241		static const uptr kLoAppMemEnd = 0x0000200000000ull;
242		static const uptr kShadowBeg = 0x0002000000000ull;
243		static const uptr kShadowEnd = 0x0004000000000ull;
244		static const uptr kMetaShadowBeg = 0x0005000000000ull;
245		static const uptr kMetaShadowEnd = 0x0006000000000ull;
	236	static const uptr kLoAppMemEnd = 0x00a0000000000ull;
	237	static const uptr kShadowBeg = 0x0155400000000ull;
	238	static const uptr kShadowEnd = 0x0540000000000ull;
	239	static const uptr kMetaShadowBeg = 0x0800000000000ull;
	240	static const uptr kMetaShadowEnd = 0x0a00000000000ull;
246	241	static const uptr kMidAppMemBeg = 0x0aaaa00000000ull;
247		static const uptr kMidAppMemEnd = 0x0aaaf00000000ull;
248		static const uptr kTraceMemBeg = 0x0f06000000000ull;
249		static const uptr kTraceMemEnd = 0x0f06200000000ull;
250		static const uptr kHeapMemBeg = 0x0ffff00000000ull;
251		static const uptr kHeapMemEnd = 0x0ffff00000000ull;
252		static const uptr kHiAppMemBeg = 0x0ffff00000000ull;
	242	static const uptr kMidAppMemEnd = 0x0ac0000000000ull;
	243	static const uptr kHiAppMemBeg = 0x0fc0000000000ull;
253	244	static const uptr kHiAppMemEnd = 0x1000000000000ull;
254		static const uptr kAppMemMsk = 0x0fff800000000ull;
255		static const uptr kAppMemXor = 0x0000800000000ull;
	245	static const uptr kHeapMemBeg = 0x0fc0000000000ull;
	246	static const uptr kHeapMemEnd = 0x0fc0000000000ull;
	247	static const uptr kShadowMsk = 0x0c00000000000ull;
	248	static const uptr kShadowXor = 0x0200000000000ull;
	249	static const uptr kShadowAdd = 0x0000000000000ull;
256	250	static const uptr kVdsoBeg = 0xffff000000000ull;
257	251	};
258	252
259		// Indicates the runtime will define the memory regions at runtime.
260		#define TSAN_RUNTIME_VMA 1
261		// Indicates that mapping defines a mid range memory segment.
262		#define TSAN_MID_APP_RANGE 1
263		#elif defined(__powerpc64__)
264		// PPC64 supports multiple VMA which leads to multiple address transformation
265		// functions. To support these multiple VMAS transformations and mappings TSAN
266		// runtime for PPC64 uses an external memory read (vmaSize) to select which
267		// mapping to use. Although slower, it make a same instrumented binary run on
268		// multiple kernels.
	253	/* C/C++ on linux/loongarch64 (47-bit VMA)
	254	0000 0000 4000 - 0080 0000 0000: main binary
	255	0080 0000 0000 - 0100 0000 0000: -
	256	0100 0000 0000 - 1000 0000 0000: shadow memory
	257	1000 0000 0000 - 3000 0000 0000: -
	258	3000 0000 0000 - 3400 0000 0000: metainfo
	259	3400 0000 0000 - 5555 0000 0000: -
	260	5555 0000 0000 - 5556 0000 0000: main binary (PIE)
	261	5556 0000 0000 - 7ffe 0000 0000: -
	262	7ffe 0000 0000 - 7fff 0000 0000: heap
	263	7fff 0000 0000 - 7fff 8000 0000: -
	264	7fff 8000 0000 - 8000 0000 0000: modules and main thread stack
	265	*/
	266	struct MappingLoongArch64_47 {
	267	static const uptr kMetaShadowBeg = 0x300000000000ull;
	268	static const uptr kMetaShadowEnd = 0x340000000000ull;
	269	static const uptr kShadowBeg = 0x010000000000ull;
	270	static const uptr kShadowEnd = 0x100000000000ull;
	271	static const uptr kHeapMemBeg = 0x7ffe00000000ull;
	272	static const uptr kHeapMemEnd = 0x7fff00000000ull;
	273	static const uptr kLoAppMemBeg = 0x000000004000ull;
	274	static const uptr kLoAppMemEnd = 0x008000000000ull;
	275	static const uptr kMidAppMemBeg = 0x555500000000ull;
	276	static const uptr kMidAppMemEnd = 0x555600000000ull;
	277	static const uptr kHiAppMemBeg = 0x7fff80000000ull;
	278	static const uptr kHiAppMemEnd = 0x800000000000ull;
	279	static const uptr kShadowMsk = 0x780000000000ull;
	280	static const uptr kShadowXor = 0x040000000000ull;
	281	static const uptr kShadowAdd = 0x000000000000ull;
	282	static const uptr kVdsoBeg = 0x7fffffffc000ull;
	283	};
269	284
270	285	/*
271	286	C/C++ on linux/powerpc64 (44-bit VMA)
...	...	@@ -274,18 +289,16 @@ C/C++ on linux/powerpc64 (44-bit VMA)
274	289	0001 0000 0000 - 0b00 0000 0000: shadow
275	290	0b00 0000 0000 - 0b00 0000 0000: -
276	291	0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
277		0d00 0000 0000 - 0d00 0000 0000: -
278		0d00 0000 0000 - 0f00 0000 0000: traces
279		0f00 0000 0000 - 0f00 0000 0000: -
	292	0d00 0000 0000 - 0f00 0000 0000: -
280	293	0f00 0000 0000 - 0f50 0000 0000: heap
281	294	0f50 0000 0000 - 0f60 0000 0000: -
282	295	0f60 0000 0000 - 1000 0000 0000: modules and main thread stack
283	296	*/
284		struct Mapping44 {
	297	struct MappingPPC64_44 {
	298	static const uptr kBroken = kBrokenMapping \| kBrokenReverseMapping \|
	299	kBrokenLinearity \| kBrokenAliasedMetas;
285	300	static const uptr kMetaShadowBeg = 0x0b0000000000ull;
286	301	static const uptr kMetaShadowEnd = 0x0d0000000000ull;
287		static const uptr kTraceMemBeg = 0x0d0000000000ull;
288		static const uptr kTraceMemEnd = 0x0f0000000000ull;
289	302	static const uptr kShadowBeg = 0x000100000000ull;
290	303	static const uptr kShadowEnd = 0x0b0000000000ull;
291	304	static const uptr kLoAppMemBeg = 0x000000000100ull;
...	...	@@ -294,188 +307,196 @@ struct Mapping44 {
294	307	static const uptr kHeapMemEnd = 0x0f5000000000ull;
295	308	static const uptr kHiAppMemBeg = 0x0f6000000000ull;
296	309	static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits
297		static const uptr kAppMemMsk = 0x0f0000000000ull;
298		static const uptr kAppMemXor = 0x002100000000ull;
	310	static const uptr kShadowMsk = 0x0f0000000000ull;
	311	static const uptr kShadowXor = 0x002100000000ull;
	312	static const uptr kShadowAdd = 0x000000000000ull;
299	313	static const uptr kVdsoBeg = 0x3c0000000000000ull;
	314	static const uptr kMidAppMemBeg = 0;
	315	static const uptr kMidAppMemEnd = 0;
300	316	};
301	317
302	318	/*
303	319	C/C++ on linux/powerpc64 (46-bit VMA)
304	320	0000 0000 1000 - 0100 0000 0000: main binary
305	321	0100 0000 0000 - 0200 0000 0000: -
306		0100 0000 0000 - 1000 0000 0000: shadow
307		1000 0000 0000 - 1000 0000 0000: -
308		1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
309		2000 0000 0000 - 2000 0000 0000: -
310		2000 0000 0000 - 2200 0000 0000: traces
311		2200 0000 0000 - 3d00 0000 0000: -
	322	0100 0000 0000 - 0800 0000 0000: shadow
	323	0800 0000 0000 - 1000 0000 0000: -
	324	1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
	325	1200 0000 0000 - 3d00 0000 0000: -
312	326	3d00 0000 0000 - 3e00 0000 0000: heap
313	327	3e00 0000 0000 - 3e80 0000 0000: -
314	328	3e80 0000 0000 - 4000 0000 0000: modules and main thread stack
315	329	*/
316		struct Mapping46 {
	330	struct MappingPPC64_46 {
317	331	static const uptr kMetaShadowBeg = 0x100000000000ull;
318		static const uptr kMetaShadowEnd = 0x200000000000ull;
319		static const uptr kTraceMemBeg = 0x200000000000ull;
320		static const uptr kTraceMemEnd = 0x220000000000ull;
	332	static const uptr kMetaShadowEnd = 0x120000000000ull;
321	333	static const uptr kShadowBeg = 0x010000000000ull;
322		static const uptr kShadowEnd = 0x100000000000ull;
	334	static const uptr kShadowEnd = 0x080000000000ull;
323	335	static const uptr kHeapMemBeg = 0x3d0000000000ull;
324	336	static const uptr kHeapMemEnd = 0x3e0000000000ull;
325	337	static const uptr kLoAppMemBeg = 0x000000001000ull;
326	338	static const uptr kLoAppMemEnd = 0x010000000000ull;
327	339	static const uptr kHiAppMemBeg = 0x3e8000000000ull;
328	340	static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits
329		static const uptr kAppMemMsk = 0x3c0000000000ull;
330		static const uptr kAppMemXor = 0x020000000000ull;
	341	static const uptr kShadowMsk = 0x3c0000000000ull;
	342	static const uptr kShadowXor = 0x020000000000ull;
	343	static const uptr kShadowAdd = 0x000000000000ull;
331	344	static const uptr kVdsoBeg = 0x7800000000000000ull;
	345	static const uptr kMidAppMemBeg = 0;
	346	static const uptr kMidAppMemEnd = 0;
332	347	};
333	348
334	349	/*
335	350	C/C++ on linux/powerpc64 (47-bit VMA)
336	351	0000 0000 1000 - 0100 0000 0000: main binary
337	352	0100 0000 0000 - 0200 0000 0000: -
338		0100 0000 0000 - 1000 0000 0000: shadow
339		1000 0000 0000 - 1000 0000 0000: -
340		1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
341		2000 0000 0000 - 2000 0000 0000: -
342		2000 0000 0000 - 2200 0000 0000: traces
343		2200 0000 0000 - 7d00 0000 0000: -
	353	0100 0000 0000 - 0800 0000 0000: shadow
	354	0800 0000 0000 - 1000 0000 0000: -
	355	1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
	356	1200 0000 0000 - 7d00 0000 0000: -
344	357	7d00 0000 0000 - 7e00 0000 0000: heap
345	358	7e00 0000 0000 - 7e80 0000 0000: -
346	359	7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
347	360	*/
348		struct Mapping47 {
	361	struct MappingPPC64_47 {
349	362	static const uptr kMetaShadowBeg = 0x100000000000ull;
350		static const uptr kMetaShadowEnd = 0x200000000000ull;
351		static const uptr kTraceMemBeg = 0x200000000000ull;
352		static const uptr kTraceMemEnd = 0x220000000000ull;
	363	static const uptr kMetaShadowEnd = 0x120000000000ull;
353	364	static const uptr kShadowBeg = 0x010000000000ull;
354		static const uptr kShadowEnd = 0x100000000000ull;
	365	static const uptr kShadowEnd = 0x080000000000ull;
355	366	static const uptr kHeapMemBeg = 0x7d0000000000ull;
356	367	static const uptr kHeapMemEnd = 0x7e0000000000ull;
357	368	static const uptr kLoAppMemBeg = 0x000000001000ull;
358	369	static const uptr kLoAppMemEnd = 0x010000000000ull;
359	370	static const uptr kHiAppMemBeg = 0x7e8000000000ull;
360	371	static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits
361		static const uptr kAppMemMsk = 0x7c0000000000ull;
362		static const uptr kAppMemXor = 0x020000000000ull;
	372	static const uptr kShadowMsk = 0x7c0000000000ull;
	373	static const uptr kShadowXor = 0x020000000000ull;
	374	static const uptr kShadowAdd = 0x000000000000ull;
363	375	static const uptr kVdsoBeg = 0x7800000000000000ull;
	376	static const uptr kMidAppMemBeg = 0;
	377	static const uptr kMidAppMemEnd = 0;
364	378	};
365	379
366		// Indicates the runtime will define the memory regions at runtime.
367		#define TSAN_RUNTIME_VMA 1
368		#elif defined(__s390x__)
369	380	/*
370	381	C/C++ on linux/s390x
371	382	While the kernel provides a 64-bit address space, we have to restrict ourselves
372	383	to 48 bits due to how e.g. SyncVar::GetId() works.
373	384	0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
374		0e00 0000 0000 - 4000 0000 0000: -
375		4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
376		8000 0000 0000 - 9000 0000 0000: -
	385	0e00 0000 0000 - 2000 0000 0000: -
	386	2000 0000 0000 - 4000 0000 0000: shadow - 32TiB (2 * app)
	387	4000 0000 0000 - 9000 0000 0000: -
377	388	9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
378		9800 0000 0000 - a000 0000 0000: -
379		a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
380		b000 0000 0000 - be00 0000 0000: -
	389	9800 0000 0000 - be00 0000 0000: -
381	390	be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
382	391	*/
383		struct Mapping {
	392	struct MappingS390x {
384	393	static const uptr kMetaShadowBeg = 0x900000000000ull;
385	394	static const uptr kMetaShadowEnd = 0x980000000000ull;
386		static const uptr kTraceMemBeg = 0xa00000000000ull;
387		static const uptr kTraceMemEnd = 0xb00000000000ull;
388		static const uptr kShadowBeg = 0x400000000000ull;
389		static const uptr kShadowEnd = 0x800000000000ull;
	395	static const uptr kShadowBeg = 0x200000000000ull;
	396	static const uptr kShadowEnd = 0x400000000000ull;
390	397	static const uptr kHeapMemBeg = 0xbe0000000000ull;
391	398	static const uptr kHeapMemEnd = 0xc00000000000ull;
392	399	static const uptr kLoAppMemBeg = 0x000000001000ull;
393	400	static const uptr kLoAppMemEnd = 0x0e0000000000ull;
394	401	static const uptr kHiAppMemBeg = 0xc00000004000ull;
395	402	static const uptr kHiAppMemEnd = 0xc00000004000ull;
396		static const uptr kAppMemMsk = 0xb00000000000ull;
397		static const uptr kAppMemXor = 0x100000000000ull;
	403	static const uptr kShadowMsk = 0xb00000000000ull;
	404	static const uptr kShadowXor = 0x100000000000ull;
	405	static const uptr kShadowAdd = 0x000000000000ull;
398	406	static const uptr kVdsoBeg = 0xfffffffff000ull;
	407	static const uptr kMidAppMemBeg = 0;
	408	static const uptr kMidAppMemEnd = 0;
399	409	};
400		#endif
401
402		#elif SANITIZER_GO && !SANITIZER_WINDOWS && HAS_48_BIT_ADDRESS_SPACE
403	410
404	411	/* Go on linux, darwin and freebsd on x86_64
405	412	0000 0000 1000 - 0000 1000 0000: executable
406	413	0000 1000 0000 - 00c0 0000 0000: -
407	414	00c0 0000 0000 - 00e0 0000 0000: heap
408	415	00e0 0000 0000 - 2000 0000 0000: -
409		2000 0000 0000 - 2380 0000 0000: shadow
410		2380 0000 0000 - 3000 0000 0000: -
	416	2000 0000 0000 - 21c0 0000 0000: shadow
	417	21c0 0000 0000 - 3000 0000 0000: -
411	418	3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
412		4000 0000 0000 - 6000 0000 0000: -
413		6000 0000 0000 - 6200 0000 0000: traces
414		6200 0000 0000 - 8000 0000 0000: -
	419	4000 0000 0000 - 8000 0000 0000: -
415	420	*/
416	421
417		struct Mapping {
	422	struct MappingGo48 {
418	423	static const uptr kMetaShadowBeg = 0x300000000000ull;
419	424	static const uptr kMetaShadowEnd = 0x400000000000ull;
420		static const uptr kTraceMemBeg = 0x600000000000ull;
421		static const uptr kTraceMemEnd = 0x620000000000ull;
422	425	static const uptr kShadowBeg = 0x200000000000ull;
423		static const uptr kShadowEnd = 0x238000000000ull;
424		static const uptr kAppMemBeg = 0x000000001000ull;
425		static const uptr kAppMemEnd = 0x00e000000000ull;
	426	static const uptr kShadowEnd = 0x21c000000000ull;
	427	static const uptr kLoAppMemBeg = 0x000000001000ull;
	428	static const uptr kLoAppMemEnd = 0x00e000000000ull;
	429	static const uptr kMidAppMemBeg = 0;
	430	static const uptr kMidAppMemEnd = 0;
	431	static const uptr kHiAppMemBeg = 0;
	432	static const uptr kHiAppMemEnd = 0;
	433	static const uptr kHeapMemBeg = 0;
	434	static const uptr kHeapMemEnd = 0;
	435	static const uptr kVdsoBeg = 0;
	436	static const uptr kShadowMsk = 0;
	437	static const uptr kShadowXor = 0;
	438	static const uptr kShadowAdd = 0x200000000000ull;
426	439	};
427	440
428		#elif SANITIZER_GO && SANITIZER_WINDOWS
429
430	441	/* Go on windows
431	442	0000 0000 1000 - 0000 1000 0000: executable
432	443	0000 1000 0000 - 00f8 0000 0000: -
433	444	00c0 0000 0000 - 00e0 0000 0000: heap
434	445	00e0 0000 0000 - 0100 0000 0000: -
435		0100 0000 0000 - 0500 0000 0000: shadow
436		0500 0000 0000 - 0560 0000 0000: -
437		0560 0000 0000 - 0760 0000 0000: traces
438		0760 0000 0000 - 07d0 0000 0000: metainfo (memory blocks and sync objects)
	446	0100 0000 0000 - 0300 0000 0000: shadow
	447	0300 0000 0000 - 0700 0000 0000: -
	448	0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects)
439	449	07d0 0000 0000 - 8000 0000 0000: -
	450	PIE binaries currently not supported, but it should be theoretically possible.
440	451	*/
441	452
442		struct Mapping {
443		static const uptr kMetaShadowBeg = 0x076000000000ull;
444		static const uptr kMetaShadowEnd = 0x07d000000000ull;
445		static const uptr kTraceMemBeg = 0x056000000000ull;
446		static const uptr kTraceMemEnd = 0x076000000000ull;
	453	struct MappingGoWindows {
	454	static const uptr kMetaShadowBeg = 0x070000000000ull;
	455	static const uptr kMetaShadowEnd = 0x077000000000ull;
447	456	static const uptr kShadowBeg = 0x010000000000ull;
448		static const uptr kShadowEnd = 0x050000000000ull;
449		static const uptr kAppMemBeg = 0x000000001000ull;
450		static const uptr kAppMemEnd = 0x00e000000000ull;
	457	static const uptr kShadowEnd = 0x030000000000ull;
	458	static const uptr kLoAppMemBeg = 0x000000001000ull;
	459	static const uptr kLoAppMemEnd = 0x00e000000000ull;
	460	static const uptr kMidAppMemBeg = 0;
	461	static const uptr kMidAppMemEnd = 0;
	462	static const uptr kHiAppMemBeg = 0;
	463	static const uptr kHiAppMemEnd = 0;
	464	static const uptr kHeapMemBeg = 0;
	465	static const uptr kHeapMemEnd = 0;
	466	static const uptr kVdsoBeg = 0;
	467	static const uptr kShadowMsk = 0;
	468	static const uptr kShadowXor = 0;
	469	static const uptr kShadowAdd = 0x010000000000ull;
451	470	};
452	471
453		#elif SANITIZER_GO && defined(__powerpc64__)
454
455		/* Only Mapping46 and Mapping47 are currently supported for powercp64 on Go. */
456
457	472	/* Go on linux/powerpc64 (46-bit VMA)
458	473	0000 0000 1000 - 0000 1000 0000: executable
459	474	0000 1000 0000 - 00c0 0000 0000: -
460	475	00c0 0000 0000 - 00e0 0000 0000: heap
461	476	00e0 0000 0000 - 2000 0000 0000: -
462		2000 0000 0000 - 2380 0000 0000: shadow
463		2380 0000 0000 - 2400 0000 0000: -
464		2400 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects)
465		3400 0000 0000 - 3600 0000 0000: -
466		3600 0000 0000 - 3800 0000 0000: traces
467		3800 0000 0000 - 4000 0000 0000: -
	477	2000 0000 0000 - 21c0 0000 0000: shadow
	478	21c0 0000 0000 - 2400 0000 0000: -
	479	2400 0000 0000 - 2470 0000 0000: metainfo (memory blocks and sync objects)
	480	2470 0000 0000 - 4000 0000 0000: -
468	481	*/
469	482
470		struct Mapping46 {
	483	struct MappingGoPPC64_46 {
471	484	static const uptr kMetaShadowBeg = 0x240000000000ull;
472		static const uptr kMetaShadowEnd = 0x340000000000ull;
473		static const uptr kTraceMemBeg = 0x360000000000ull;
474		static const uptr kTraceMemEnd = 0x380000000000ull;
	485	static const uptr kMetaShadowEnd = 0x247000000000ull;
475	486	static const uptr kShadowBeg = 0x200000000000ull;
476		static const uptr kShadowEnd = 0x238000000000ull;
477		static const uptr kAppMemBeg = 0x000000001000ull;
478		static const uptr kAppMemEnd = 0x00e000000000ull;
	487	static const uptr kShadowEnd = 0x21c000000000ull;
	488	static const uptr kLoAppMemBeg = 0x000000001000ull;
	489	static const uptr kLoAppMemEnd = 0x00e000000000ull;
	490	static const uptr kMidAppMemBeg = 0;
	491	static const uptr kMidAppMemEnd = 0;
	492	static const uptr kHiAppMemBeg = 0;
	493	static const uptr kHiAppMemEnd = 0;
	494	static const uptr kHeapMemBeg = 0;
	495	static const uptr kHeapMemEnd = 0;
	496	static const uptr kVdsoBeg = 0;
	497	static const uptr kShadowMsk = 0;
	498	static const uptr kShadowXor = 0;
	499	static const uptr kShadowAdd = 0x200000000000ull;
479	500	};
480	501
481	502	/* Go on linux/powerpc64 (47-bit VMA)
...	...	@@ -483,718 +504,424 @@ struct Mapping46 {
483	504	0000 1000 0000 - 00c0 0000 0000: -
484	505	00c0 0000 0000 - 00e0 0000 0000: heap
485	506	00e0 0000 0000 - 2000 0000 0000: -
486		2000 0000 0000 - 3000 0000 0000: shadow
487		3000 0000 0000 - 3000 0000 0000: -
488		3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
489		4000 0000 0000 - 6000 0000 0000: -
490		6000 0000 0000 - 6200 0000 0000: traces
491		6200 0000 0000 - 8000 0000 0000: -
	507	2000 0000 0000 - 2800 0000 0000: shadow
	508	2800 0000 0000 - 3000 0000 0000: -
	509	3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
	510	3200 0000 0000 - 8000 0000 0000: -
492	511	*/
493	512
494		struct Mapping47 {
	513	struct MappingGoPPC64_47 {
495	514	static const uptr kMetaShadowBeg = 0x300000000000ull;
496		static const uptr kMetaShadowEnd = 0x400000000000ull;
497		static const uptr kTraceMemBeg = 0x600000000000ull;
498		static const uptr kTraceMemEnd = 0x620000000000ull;
	515	static const uptr kMetaShadowEnd = 0x320000000000ull;
499	516	static const uptr kShadowBeg = 0x200000000000ull;
500		static const uptr kShadowEnd = 0x300000000000ull;
501		static const uptr kAppMemBeg = 0x000000001000ull;
502		static const uptr kAppMemEnd = 0x00e000000000ull;
	517	static const uptr kShadowEnd = 0x280000000000ull;
	518	static const uptr kLoAppMemBeg = 0x000000001000ull;
	519	static const uptr kLoAppMemEnd = 0x00e000000000ull;
	520	static const uptr kMidAppMemBeg = 0;
	521	static const uptr kMidAppMemEnd = 0;
	522	static const uptr kHiAppMemBeg = 0;
	523	static const uptr kHiAppMemEnd = 0;
	524	static const uptr kHeapMemBeg = 0;
	525	static const uptr kHeapMemEnd = 0;
	526	static const uptr kVdsoBeg = 0;
	527	static const uptr kShadowMsk = 0;
	528	static const uptr kShadowXor = 0;
	529	static const uptr kShadowAdd = 0x200000000000ull;
503	530	};
504	531
505		#define TSAN_RUNTIME_VMA 1
506
507		#elif SANITIZER_GO && defined(__aarch64__)
508
509	532	/* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA)
510	533	0000 0000 1000 - 0000 1000 0000: executable
511	534	0000 1000 0000 - 00c0 0000 0000: -
512	535	00c0 0000 0000 - 00e0 0000 0000: heap
513	536	00e0 0000 0000 - 2000 0000 0000: -
514		2000 0000 0000 - 3000 0000 0000: shadow
515		3000 0000 0000 - 3000 0000 0000: -
516		3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
517		4000 0000 0000 - 6000 0000 0000: -
518		6000 0000 0000 - 6200 0000 0000: traces
519		6200 0000 0000 - 8000 0000 0000: -
	537	2000 0000 0000 - 2800 0000 0000: shadow
	538	2800 0000 0000 - 3000 0000 0000: -
	539	3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
	540	3200 0000 0000 - 8000 0000 0000: -
520	541	*/
521
522		struct Mapping {
	542	struct MappingGoAarch64 {
523	543	static const uptr kMetaShadowBeg = 0x300000000000ull;
524		static const uptr kMetaShadowEnd = 0x400000000000ull;
525		static const uptr kTraceMemBeg = 0x600000000000ull;
526		static const uptr kTraceMemEnd = 0x620000000000ull;
	544	static const uptr kMetaShadowEnd = 0x320000000000ull;
527	545	static const uptr kShadowBeg = 0x200000000000ull;
528		static const uptr kShadowEnd = 0x300000000000ull;
529		static const uptr kAppMemBeg = 0x000000001000ull;
530		static const uptr kAppMemEnd = 0x00e000000000ull;
	546	static const uptr kShadowEnd = 0x280000000000ull;
	547	static const uptr kLoAppMemBeg = 0x000000001000ull;
	548	static const uptr kLoAppMemEnd = 0x00e000000000ull;
	549	static const uptr kMidAppMemBeg = 0;
	550	static const uptr kMidAppMemEnd = 0;
	551	static const uptr kHiAppMemBeg = 0;
	552	static const uptr kHiAppMemEnd = 0;
	553	static const uptr kHeapMemBeg = 0;
	554	static const uptr kHeapMemEnd = 0;
	555	static const uptr kVdsoBeg = 0;
	556	static const uptr kShadowMsk = 0;
	557	static const uptr kShadowXor = 0;
	558	static const uptr kShadowAdd = 0x200000000000ull;
531	559	};
532	560
533		// Indicates the runtime will define the memory regions at runtime.
534		#define TSAN_RUNTIME_VMA 1
535
536		#elif SANITIZER_GO && defined(__mips64)
537	561	/*
538	562	Go on linux/mips64 (47-bit VMA)
539	563	0000 0000 1000 - 0000 1000 0000: executable
540	564	0000 1000 0000 - 00c0 0000 0000: -
541	565	00c0 0000 0000 - 00e0 0000 0000: heap
542	566	00e0 0000 0000 - 2000 0000 0000: -
543		2000 0000 0000 - 3000 0000 0000: shadow
544		3000 0000 0000 - 3000 0000 0000: -
545		3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
546		4000 0000 0000 - 6000 0000 0000: -
547		6000 0000 0000 - 6200 0000 0000: traces
548		6200 0000 0000 - 8000 0000 0000: -
	567	2000 0000 0000 - 2800 0000 0000: shadow
	568	2800 0000 0000 - 3000 0000 0000: -
	569	3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
	570	3200 0000 0000 - 8000 0000 0000: -
549	571	*/
550		struct Mapping47 {
	572	struct MappingGoMips64_47 {
551	573	static const uptr kMetaShadowBeg = 0x300000000000ull;
552		static const uptr kMetaShadowEnd = 0x400000000000ull;
553		static const uptr kTraceMemBeg = 0x600000000000ull;
554		static const uptr kTraceMemEnd = 0x620000000000ull;
	574	static const uptr kMetaShadowEnd = 0x320000000000ull;
555	575	static const uptr kShadowBeg = 0x200000000000ull;
556		static const uptr kShadowEnd = 0x300000000000ull;
557		static const uptr kAppMemBeg = 0x000000001000ull;
558		static const uptr kAppMemEnd = 0x00e000000000ull;
	576	static const uptr kShadowEnd = 0x280000000000ull;
	577	static const uptr kLoAppMemBeg = 0x000000001000ull;
	578	static const uptr kLoAppMemEnd = 0x00e000000000ull;
	579	static const uptr kMidAppMemBeg = 0;
	580	static const uptr kMidAppMemEnd = 0;
	581	static const uptr kHiAppMemBeg = 0;
	582	static const uptr kHiAppMemEnd = 0;
	583	static const uptr kHeapMemBeg = 0;
	584	static const uptr kHeapMemEnd = 0;
	585	static const uptr kVdsoBeg = 0;
	586	static const uptr kShadowMsk = 0;
	587	static const uptr kShadowXor = 0;
	588	static const uptr kShadowAdd = 0x200000000000ull;
559	589	};
560	590
561		#define TSAN_RUNTIME_VMA 1
562
563		#elif SANITIZER_GO && defined(__s390x__)
564	591	/*
565	592	Go on linux/s390x
566	593	0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
567	594	1000 0000 0000 - 4000 0000 0000: -
568		4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
569		8000 0000 0000 - 9000 0000 0000: -
	595	4000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app)
	596	6000 0000 0000 - 9000 0000 0000: -
570	597	9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
571		9800 0000 0000 - a000 0000 0000: -
572		a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
573	598	*/
574		struct Mapping {
	599	struct MappingGoS390x {
575	600	static const uptr kMetaShadowBeg = 0x900000000000ull;
576	601	static const uptr kMetaShadowEnd = 0x980000000000ull;
577		static const uptr kTraceMemBeg = 0xa00000000000ull;
578		static const uptr kTraceMemEnd = 0xb00000000000ull;
579	602	static const uptr kShadowBeg = 0x400000000000ull;
580		static const uptr kShadowEnd = 0x800000000000ull;
581		static const uptr kAppMemBeg = 0x000000001000ull;
582		static const uptr kAppMemEnd = 0x100000000000ull;
	603	static const uptr kShadowEnd = 0x600000000000ull;
	604	static const uptr kLoAppMemBeg = 0x000000001000ull;
	605	static const uptr kLoAppMemEnd = 0x100000000000ull;
	606	static const uptr kMidAppMemBeg = 0;
	607	static const uptr kMidAppMemEnd = 0;
	608	static const uptr kHiAppMemBeg = 0;
	609	static const uptr kHiAppMemEnd = 0;
	610	static const uptr kHeapMemBeg = 0;
	611	static const uptr kHeapMemEnd = 0;
	612	static const uptr kVdsoBeg = 0;
	613	static const uptr kShadowMsk = 0;
	614	static const uptr kShadowXor = 0;
	615	static const uptr kShadowAdd = 0x400000000000ull;
583	616	};
584	617
585		#else
586		# error "Unknown platform"
587		#endif
588
589
590		#ifdef TSAN_RUNTIME_VMA
591	618	extern uptr vmaSize;
592		#endif
593
594
595		enum MappingType {
596		MAPPING_LO_APP_BEG,
597		MAPPING_LO_APP_END,
598		MAPPING_HI_APP_BEG,
599		MAPPING_HI_APP_END,
600		#ifdef TSAN_MID_APP_RANGE
601		MAPPING_MID_APP_BEG,
602		MAPPING_MID_APP_END,
603		#endif
604		MAPPING_HEAP_BEG,
605		MAPPING_HEAP_END,
606		MAPPING_APP_BEG,
607		MAPPING_APP_END,
608		MAPPING_SHADOW_BEG,
609		MAPPING_SHADOW_END,
610		MAPPING_META_SHADOW_BEG,
611		MAPPING_META_SHADOW_END,
612		MAPPING_TRACE_BEG,
613		MAPPING_TRACE_END,
614		MAPPING_VDSO_BEG,
615		};
616	619
617		template<typename Mapping, int Type>
618		uptr MappingImpl(void) {
619		switch (Type) {
620		#if !SANITIZER_GO
621		case MAPPING_LO_APP_BEG: return Mapping::kLoAppMemBeg;
622		case MAPPING_LO_APP_END: return Mapping::kLoAppMemEnd;
623		# ifdef TSAN_MID_APP_RANGE
624		case MAPPING_MID_APP_BEG: return Mapping::kMidAppMemBeg;
625		case MAPPING_MID_APP_END: return Mapping::kMidAppMemEnd;
626		# endif
627		case MAPPING_HI_APP_BEG: return Mapping::kHiAppMemBeg;
628		case MAPPING_HI_APP_END: return Mapping::kHiAppMemEnd;
629		case MAPPING_HEAP_BEG: return Mapping::kHeapMemBeg;
630		case MAPPING_HEAP_END: return Mapping::kHeapMemEnd;
631		case MAPPING_VDSO_BEG: return Mapping::kVdsoBeg;
632		#else
633		case MAPPING_APP_BEG: return Mapping::kAppMemBeg;
634		case MAPPING_APP_END: return Mapping::kAppMemEnd;
635		#endif
636		case MAPPING_SHADOW_BEG: return Mapping::kShadowBeg;
637		case MAPPING_SHADOW_END: return Mapping::kShadowEnd;
638		case MAPPING_META_SHADOW_BEG: return Mapping::kMetaShadowBeg;
639		case MAPPING_META_SHADOW_END: return Mapping::kMetaShadowEnd;
640		case MAPPING_TRACE_BEG: return Mapping::kTraceMemBeg;
641		case MAPPING_TRACE_END: return Mapping::kTraceMemEnd;
642		}
643		}
644
645		template<int Type>
646		uptr MappingArchImpl(void) {
647		#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
	620	template <typename Func, typename Arg>
	621	ALWAYS_INLINE auto SelectMapping(Arg arg) {
	622	#if SANITIZER_GO
	623	# if defined(__powerpc64__)
648	624	switch (vmaSize) {
649		case 39: return MappingImpl<Mapping39, Type>();
650		case 42: return MappingImpl<Mapping42, Type>();
651		case 48: return MappingImpl<Mapping48, Type>();
	625	case 46:
	626	return Func::template Apply<MappingGoPPC64_46>(arg);
	627	case 47:
	628	return Func::template Apply<MappingGoPPC64_47>(arg);
652	629	}
653		DCHECK(0);
654		return 0;
655		#elif defined(__powerpc64__)
	630	# elif defined(__mips64)
	631	return Func::template Apply<MappingGoMips64_47>(arg);
	632	# elif defined(__s390x__)
	633	return Func::template Apply<MappingGoS390x>(arg);
	634	# elif defined(__aarch64__)
	635	return Func::template Apply<MappingGoAarch64>(arg);
	636	# elif SANITIZER_WINDOWS
	637	return Func::template Apply<MappingGoWindows>(arg);
	638	# else
	639	return Func::template Apply<MappingGo48>(arg);
	640	# endif
	641	#else // SANITIZER_GO
	642	# if SANITIZER_IOS && !SANITIZER_IOSSIM
	643	return Func::template Apply<MappingAppleAarch64>(arg);
	644	# elif defined(__x86_64__) \|\| SANITIZER_APPLE
	645	return Func::template Apply<Mapping48AddressSpace>(arg);
	646	# elif defined(__aarch64__)
656	647	switch (vmaSize) {
657		#if !SANITIZER_GO
658		case 44: return MappingImpl<Mapping44, Type>();
659		#endif
660		case 46: return MappingImpl<Mapping46, Type>();
661		case 47: return MappingImpl<Mapping47, Type>();
	648	case 39:
	649	return Func::template Apply<MappingAarch64_39>(arg);
	650	case 42:
	651	return Func::template Apply<MappingAarch64_42>(arg);
	652	case 48:
	653	return Func::template Apply<MappingAarch64_48>(arg);
662	654	}
663		DCHECK(0);
664		return 0;
665		#elif defined(__mips64)
	655	# elif SANITIZER_LOONGARCH64
	656	return Func::template Apply<MappingLoongArch64_47>(arg);
	657	# elif defined(__powerpc64__)
666	658	switch (vmaSize) {
667		#if !SANITIZER_GO
668		case 40: return MappingImpl<Mapping40, Type>();
669		#else
670		case 47: return MappingImpl<Mapping47, Type>();
671		#endif
	659	case 44:
	660	return Func::template Apply<MappingPPC64_44>(arg);
	661	case 46:
	662	return Func::template Apply<MappingPPC64_46>(arg);
	663	case 47:
	664	return Func::template Apply<MappingPPC64_47>(arg);
672	665	}
673		DCHECK(0);
674		return 0;
675		#else
676		return MappingImpl<Mapping, Type>();
677		#endif
	666	# elif defined(__mips64)
	667	return Func::template Apply<MappingMips64_40>(arg);
	668	# elif defined(__s390x__)
	669	return Func::template Apply<MappingS390x>(arg);
	670	# else
	671	# error "unsupported platform"
	672	# endif
	673	#endif
	674	Die();
	675	}
	676
	677	template <typename Func>
	678	void ForEachMapping() {
	679	Func::template Apply<Mapping48AddressSpace>();
	680	Func::template Apply<MappingMips64_40>();
	681	Func::template Apply<MappingAppleAarch64>();
	682	Func::template Apply<MappingAarch64_39>();
	683	Func::template Apply<MappingAarch64_42>();
	684	Func::template Apply<MappingAarch64_48>();
	685	Func::template Apply<MappingLoongArch64_47>();
	686	Func::template Apply<MappingPPC64_44>();
	687	Func::template Apply<MappingPPC64_46>();
	688	Func::template Apply<MappingPPC64_47>();
	689	Func::template Apply<MappingS390x>();
	690	Func::template Apply<MappingGo48>();
	691	Func::template Apply<MappingGoWindows>();
	692	Func::template Apply<MappingGoPPC64_46>();
	693	Func::template Apply<MappingGoPPC64_47>();
	694	Func::template Apply<MappingGoAarch64>();
	695	Func::template Apply<MappingGoMips64_47>();
	696	Func::template Apply<MappingGoS390x>();
678	697	}
679	698
680		#if !SANITIZER_GO
681		ALWAYS_INLINE
682		uptr LoAppMemBeg(void) {
683		return MappingArchImpl<MAPPING_LO_APP_BEG>();
684		}
685		ALWAYS_INLINE
686		uptr LoAppMemEnd(void) {
687		return MappingArchImpl<MAPPING_LO_APP_END>();
688		}
	699	enum MappingType {
	700	kLoAppMemBeg,
	701	kLoAppMemEnd,
	702	kHiAppMemBeg,
	703	kHiAppMemEnd,
	704	kMidAppMemBeg,
	705	kMidAppMemEnd,
	706	kHeapMemBeg,
	707	kHeapMemEnd,
	708	kShadowBeg,
	709	kShadowEnd,
	710	kMetaShadowBeg,
	711	kMetaShadowEnd,
	712	kVdsoBeg,
	713	};
689	714
690		#ifdef TSAN_MID_APP_RANGE
691		ALWAYS_INLINE
692		uptr MidAppMemBeg(void) {
693		return MappingArchImpl<MAPPING_MID_APP_BEG>();
694		}
695		ALWAYS_INLINE
696		uptr MidAppMemEnd(void) {
697		return MappingArchImpl<MAPPING_MID_APP_END>();
698		}
699		#endif
	715	struct MappingField {
	716	template <typename Mapping>
	717	static uptr Apply(MappingType type) {
	718	switch (type) {
	719	case kLoAppMemBeg:
	720	return Mapping::kLoAppMemBeg;
	721	case kLoAppMemEnd:
	722	return Mapping::kLoAppMemEnd;
	723	case kMidAppMemBeg:
	724	return Mapping::kMidAppMemBeg;
	725	case kMidAppMemEnd:
	726	return Mapping::kMidAppMemEnd;
	727	case kHiAppMemBeg:
	728	return Mapping::kHiAppMemBeg;
	729	case kHiAppMemEnd:
	730	return Mapping::kHiAppMemEnd;
	731	case kHeapMemBeg:
	732	return Mapping::kHeapMemBeg;
	733	case kHeapMemEnd:
	734	return Mapping::kHeapMemEnd;
	735	case kVdsoBeg:
	736	return Mapping::kVdsoBeg;
	737	case kShadowBeg:
	738	return Mapping::kShadowBeg;
	739	case kShadowEnd:
	740	return Mapping::kShadowEnd;
	741	case kMetaShadowBeg:
	742	return Mapping::kMetaShadowBeg;
	743	case kMetaShadowEnd:
	744	return Mapping::kMetaShadowEnd;
	745	}
	746	Die();
	747	}
	748	};
700	749
701	750	ALWAYS_INLINE
702		uptr HeapMemBeg(void) {
703		return MappingArchImpl<MAPPING_HEAP_BEG>();
704		}
	751	uptr LoAppMemBeg(void) { return SelectMapping<MappingField>(kLoAppMemBeg); }
705	752	ALWAYS_INLINE
706		uptr HeapMemEnd(void) {
707		return MappingArchImpl<MAPPING_HEAP_END>();
708		}
	753	uptr LoAppMemEnd(void) { return SelectMapping<MappingField>(kLoAppMemEnd); }
709	754
710	755	ALWAYS_INLINE
711		uptr HiAppMemBeg(void) {
712		return MappingArchImpl<MAPPING_HI_APP_BEG>();
713		}
	756	uptr MidAppMemBeg(void) { return SelectMapping<MappingField>(kMidAppMemBeg); }
714	757	ALWAYS_INLINE
715		uptr HiAppMemEnd(void) {
716		return MappingArchImpl<MAPPING_HI_APP_END>();
717		}
	758	uptr MidAppMemEnd(void) { return SelectMapping<MappingField>(kMidAppMemEnd); }
718	759
719	760	ALWAYS_INLINE
720		uptr VdsoBeg(void) {
721		return MappingArchImpl<MAPPING_VDSO_BEG>();
722		}
723
724		#else
	761	uptr HeapMemBeg(void) { return SelectMapping<MappingField>(kHeapMemBeg); }
	762	ALWAYS_INLINE
	763	uptr HeapMemEnd(void) { return SelectMapping<MappingField>(kHeapMemEnd); }
725	764
726	765	ALWAYS_INLINE
727		uptr AppMemBeg(void) {
728		return MappingArchImpl<MAPPING_APP_BEG>();
729		}
	766	uptr HiAppMemBeg(void) { return SelectMapping<MappingField>(kHiAppMemBeg); }
730	767	ALWAYS_INLINE
731		uptr AppMemEnd(void) {
732		return MappingArchImpl<MAPPING_APP_END>();
733		}
734
735		#endif
	768	uptr HiAppMemEnd(void) { return SelectMapping<MappingField>(kHiAppMemEnd); }
736	769
737		static inline
738		bool GetUserRegion(int i, uptr start, uptr end) {
739		switch (i) {
740		default:
741		return false;
742		#if !SANITIZER_GO
743		case 0:
744		*start = LoAppMemBeg();
745		*end = LoAppMemEnd();
746		return true;
747		case 1:
748		*start = HiAppMemBeg();
749		*end = HiAppMemEnd();
750		return true;
751		case 2:
752		*start = HeapMemBeg();
753		*end = HeapMemEnd();
754		return true;
755		# ifdef TSAN_MID_APP_RANGE
756		case 3:
757		*start = MidAppMemBeg();
758		*end = MidAppMemEnd();
759		return true;
760		# endif
761		#else
762		case 0:
763		*start = AppMemBeg();
764		*end = AppMemEnd();
765		return true;
766		#endif
767		}
768		}
769
770		ALWAYS_INLINE
771		uptr ShadowBeg(void) {
772		return MappingArchImpl<MAPPING_SHADOW_BEG>();
773		}
774	770	ALWAYS_INLINE
775		uptr ShadowEnd(void) {
776		return MappingArchImpl<MAPPING_SHADOW_END>();
777		}
	771	uptr VdsoBeg(void) { return SelectMapping<MappingField>(kVdsoBeg); }
778	772
779	773	ALWAYS_INLINE
780		uptr MetaShadowBeg(void) {
781		return MappingArchImpl<MAPPING_META_SHADOW_BEG>();
782		}
	774	uptr ShadowBeg(void) { return SelectMapping<MappingField>(kShadowBeg); }
783	775	ALWAYS_INLINE
784		uptr MetaShadowEnd(void) {
785		return MappingArchImpl<MAPPING_META_SHADOW_END>();
786		}
	776	uptr ShadowEnd(void) { return SelectMapping<MappingField>(kShadowEnd); }
787	777
788	778	ALWAYS_INLINE
789		uptr TraceMemBeg(void) {
790		return MappingArchImpl<MAPPING_TRACE_BEG>();
791		}
	779	uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(kMetaShadowBeg); }
792	780	ALWAYS_INLINE
793		uptr TraceMemEnd(void) {
794		return MappingArchImpl<MAPPING_TRACE_END>();
795		}
796
	781	uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(kMetaShadowEnd); }
797	782
798		template<typename Mapping>
799		bool IsAppMemImpl(uptr mem) {
800		#if !SANITIZER_GO
	783	struct IsAppMemImpl {
	784	template <typename Mapping>
	785	static bool Apply(uptr mem) {
801	786	return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) \|\|
802		# ifdef TSAN_MID_APP_RANGE
803	787	(mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) \|\|
804		# endif
805	788	(mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) \|\|
806	789	(mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);
807		#else
808		return mem >= Mapping::kAppMemBeg && mem < Mapping::kAppMemEnd;
809		#endif
810		}
811
812		ALWAYS_INLINE
813		bool IsAppMem(uptr mem) {
814		#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
815		switch (vmaSize) {
816		case 39: return IsAppMemImpl<Mapping39>(mem);
817		case 42: return IsAppMemImpl<Mapping42>(mem);
818		case 48: return IsAppMemImpl<Mapping48>(mem);
819		}
820		DCHECK(0);
821		return false;
822		#elif defined(__powerpc64__)
823		switch (vmaSize) {
824		#if !SANITIZER_GO
825		case 44: return IsAppMemImpl<Mapping44>(mem);
826		#endif
827		case 46: return IsAppMemImpl<Mapping46>(mem);
828		case 47: return IsAppMemImpl<Mapping47>(mem);
829		}
830		DCHECK(0);
831		return false;
832		#elif defined(__mips64)
833		switch (vmaSize) {
834		#if !SANITIZER_GO
835		case 40: return IsAppMemImpl<Mapping40>(mem);
836		#else
837		case 47: return IsAppMemImpl<Mapping47>(mem);
838		#endif
839	790	}
840		DCHECK(0);
841		return false;
842		#else
843		return IsAppMemImpl<Mapping>(mem);
844		#endif
845		}
846
847
848		template<typename Mapping>
849		bool IsShadowMemImpl(uptr mem) {
850		return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
851		}
	791	};
852	792
853	793	ALWAYS_INLINE
854		bool IsShadowMem(uptr mem) {
855		#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
856		switch (vmaSize) {
857		case 39: return IsShadowMemImpl<Mapping39>(mem);
858		case 42: return IsShadowMemImpl<Mapping42>(mem);
859		case 48: return IsShadowMemImpl<Mapping48>(mem);
860		}
861		DCHECK(0);
862		return false;
863		#elif defined(__powerpc64__)
864		switch (vmaSize) {
865		#if !SANITIZER_GO
866		case 44: return IsShadowMemImpl<Mapping44>(mem);
867		#endif
868		case 46: return IsShadowMemImpl<Mapping46>(mem);
869		case 47: return IsShadowMemImpl<Mapping47>(mem);
870		}
871		DCHECK(0);
872		return false;
873		#elif defined(__mips64)
874		switch (vmaSize) {
875		#if !SANITIZER_GO
876		case 40: return IsShadowMemImpl<Mapping40>(mem);
877		#else
878		case 47: return IsShadowMemImpl<Mapping47>(mem);
879		#endif
880		}
881		DCHECK(0);
882		return false;
883		#else
884		return IsShadowMemImpl<Mapping>(mem);
885		#endif
886		}
	794	bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(mem); }
887	795
888
889		template<typename Mapping>
890		bool IsMetaMemImpl(uptr mem) {
891		return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
892		}
893
894		ALWAYS_INLINE
895		bool IsMetaMem(uptr mem) {
896		#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
897		switch (vmaSize) {
898		case 39: return IsMetaMemImpl<Mapping39>(mem);
899		case 42: return IsMetaMemImpl<Mapping42>(mem);
900		case 48: return IsMetaMemImpl<Mapping48>(mem);
901		}
902		DCHECK(0);
903		return false;
904		#elif defined(__powerpc64__)
905		switch (vmaSize) {
906		#if !SANITIZER_GO
907		case 44: return IsMetaMemImpl<Mapping44>(mem);
908		#endif
909		case 46: return IsMetaMemImpl<Mapping46>(mem);
910		case 47: return IsMetaMemImpl<Mapping47>(mem);
	796	struct IsShadowMemImpl {
	797	template <typename Mapping>
	798	static bool Apply(uptr mem) {
	799	return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
911	800	}
912		DCHECK(0);
913		return false;
914		#elif defined(__mips64)
915		switch (vmaSize) {
916		#if !SANITIZER_GO
917		case 40: return IsMetaMemImpl<Mapping40>(mem);
918		#else
919		case 47: return IsMetaMemImpl<Mapping47>(mem);
920		#endif
921		}
922		DCHECK(0);
923		return false;
924		#else
925		return IsMetaMemImpl<Mapping>(mem);
926		#endif
927		}
928
929
930		template<typename Mapping>
931		uptr MemToShadowImpl(uptr x) {
932		DCHECK(IsAppMem(x));
933		#if !SANITIZER_GO
934		return (((x) & ~(Mapping::kAppMemMsk \| (kShadowCell - 1)))
935		^ Mapping::kAppMemXor) * kShadowCnt;
936		#else
937		# ifndef SANITIZER_WINDOWS
938		return ((x & ~(kShadowCell - 1)) * kShadowCnt) \| Mapping::kShadowBeg;
939		# else
940		return ((x & ~(kShadowCell - 1)) * kShadowCnt) + Mapping::kShadowBeg;
941		# endif
942		#endif
943		}
	801	};
944	802
945	803	ALWAYS_INLINE
946		uptr MemToShadow(uptr x) {
947		#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
948		switch (vmaSize) {
949		case 39: return MemToShadowImpl<Mapping39>(x);
950		case 42: return MemToShadowImpl<Mapping42>(x);
951		case 48: return MemToShadowImpl<Mapping48>(x);
952		}
953		DCHECK(0);
954		return 0;
955		#elif defined(__powerpc64__)
956		switch (vmaSize) {
957		#if !SANITIZER_GO
958		case 44: return MemToShadowImpl<Mapping44>(x);
959		#endif
960		case 46: return MemToShadowImpl<Mapping46>(x);
961		case 47: return MemToShadowImpl<Mapping47>(x);
962		}
963		DCHECK(0);
964		return 0;
965		#elif defined(__mips64)
966		switch (vmaSize) {
967		#if !SANITIZER_GO
968		case 40: return MemToShadowImpl<Mapping40>(x);
969		#else
970		case 47: return MemToShadowImpl<Mapping47>(x);
971		#endif
972		}
973		DCHECK(0);
974		return 0;
975		#else
976		return MemToShadowImpl<Mapping>(x);
977		#endif
	804	bool IsShadowMem(RawShadow *p) {
	805	return SelectMapping<IsShadowMemImpl>(reinterpret_cast<uptr>(p));
978	806	}
979	807
980
981		template<typename Mapping>
982		u32 *MemToMetaImpl(uptr x) {
983		DCHECK(IsAppMem(x));
984		#if !SANITIZER_GO
985		return (u32*)(((((x) & ~(Mapping::kAppMemMsk \| (kMetaShadowCell - 1)))) /
986		kMetaShadowCell * kMetaShadowSize) \| Mapping::kMetaShadowBeg);
987		#else
988		# ifndef SANITIZER_WINDOWS
989		return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
990		kMetaShadowCell * kMetaShadowSize) \| Mapping::kMetaShadowBeg);
991		# else
992		return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
993		kMetaShadowCell * kMetaShadowSize) + Mapping::kMetaShadowBeg);
994		# endif
995		#endif
996		}
	808	struct IsMetaMemImpl {
	809	template <typename Mapping>
	810	static bool Apply(uptr mem) {
	811	return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
	812	}
	813	};
997	814
998	815	ALWAYS_INLINE
999		u32 *MemToMeta(uptr x) {
1000		#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
1001		switch (vmaSize) {
1002		case 39: return MemToMetaImpl<Mapping39>(x);
1003		case 42: return MemToMetaImpl<Mapping42>(x);
1004		case 48: return MemToMetaImpl<Mapping48>(x);
1005		}
1006		DCHECK(0);
1007		return 0;
1008		#elif defined(__powerpc64__)
1009		switch (vmaSize) {
1010		#if !SANITIZER_GO
1011		case 44: return MemToMetaImpl<Mapping44>(x);
1012		#endif
1013		case 46: return MemToMetaImpl<Mapping46>(x);
1014		case 47: return MemToMetaImpl<Mapping47>(x);
1015		}
1016		DCHECK(0);
1017		return 0;
1018		#elif defined(__mips64)
1019		switch (vmaSize) {
1020		#if !SANITIZER_GO
1021		case 40: return MemToMetaImpl<Mapping40>(x);
1022		#else
1023		case 47: return MemToMetaImpl<Mapping47>(x);
1024		#endif
	816	bool IsMetaMem(const u32 *p) {
	817	return SelectMapping<IsMetaMemImpl>(reinterpret_cast<uptr>(p));
	818	}
	819
	820	struct MemToShadowImpl {
	821	template <typename Mapping>
	822	static uptr Apply(uptr x) {
	823	DCHECK(IsAppMemImpl::Apply<Mapping>(x));
	824	return (((x) & ~(Mapping::kShadowMsk \| (kShadowCell - 1))) ^
	825	Mapping::kShadowXor) *
	826	kShadowMultiplier +
	827	Mapping::kShadowAdd;
1025	828	}
1026		DCHECK(0);
1027		return 0;
1028		#else
1029		return MemToMetaImpl<Mapping>(x);
1030		#endif
1031		}
1032
1033
1034		template<typename Mapping>
1035		uptr ShadowToMemImpl(uptr s) {
1036		DCHECK(IsShadowMem(s));
1037		#if !SANITIZER_GO
1038		// The shadow mapping is non-linear and we've lost some bits, so we don't have
1039		// an easy way to restore the original app address. But the mapping is a
1040		// bijection, so we try to restore the address as belonging to low/mid/high
1041		// range consecutively and see if shadow->app->shadow mapping gives us the
1042		// same address.
1043		uptr p = (s / kShadowCnt) ^ Mapping::kAppMemXor;
1044		if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
1045		MemToShadow(p) == s)
1046		return p;
1047		# ifdef TSAN_MID_APP_RANGE
1048		p = ((s / kShadowCnt) ^ Mapping::kAppMemXor) +
1049		(Mapping::kMidAppMemBeg & Mapping::kAppMemMsk);
1050		if (p >= Mapping::kMidAppMemBeg && p < Mapping::kMidAppMemEnd &&
1051		MemToShadow(p) == s)
1052		return p;
1053		# endif
1054		return ((s / kShadowCnt) ^ Mapping::kAppMemXor) \| Mapping::kAppMemMsk;
1055		#else // #if !SANITIZER_GO
1056		# ifndef SANITIZER_WINDOWS
1057		return (s & ~Mapping::kShadowBeg) / kShadowCnt;
1058		# else
1059		return (s - Mapping::kShadowBeg) / kShadowCnt;
1060		# endif // SANITIZER_WINDOWS
1061		#endif
1062		}
	829	};
1063	830
1064	831	ALWAYS_INLINE
1065		uptr ShadowToMem(uptr s) {
1066		#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
1067		switch (vmaSize) {
1068		case 39: return ShadowToMemImpl<Mapping39>(s);
1069		case 42: return ShadowToMemImpl<Mapping42>(s);
1070		case 48: return ShadowToMemImpl<Mapping48>(s);
1071		}
1072		DCHECK(0);
1073		return 0;
1074		#elif defined(__powerpc64__)
1075		switch (vmaSize) {
1076		#if !SANITIZER_GO
1077		case 44: return ShadowToMemImpl<Mapping44>(s);
1078		#endif
1079		case 46: return ShadowToMemImpl<Mapping46>(s);
1080		case 47: return ShadowToMemImpl<Mapping47>(s);
1081		}
1082		DCHECK(0);
1083		return 0;
1084		#elif defined(__mips64)
1085		switch (vmaSize) {
1086		#if !SANITIZER_GO
1087		case 40: return ShadowToMemImpl<Mapping40>(s);
1088		#else
1089		case 47: return ShadowToMemImpl<Mapping47>(s);
1090		#endif
1091		}
1092		DCHECK(0);
1093		return 0;
1094		#else
1095		return ShadowToMemImpl<Mapping>(s);
1096		#endif
	832	RawShadow *MemToShadow(uptr x) {
	833	return reinterpret_cast<RawShadow *>(SelectMapping<MemToShadowImpl>(x));
1097	834	}
1098	835
1099
1100
1101		// The additional page is to catch shadow stack overflow as paging fault.
1102		// Windows wants 64K alignment for mmaps.
1103		const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace)
1104		+ (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1);
1105
1106		template<typename Mapping>
1107		uptr GetThreadTraceImpl(int tid) {
1108		uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize;
1109		DCHECK_LT(p, Mapping::kTraceMemEnd);
1110		return p;
1111		}
	836	struct MemToMetaImpl {
	837	template <typename Mapping>
	838	static u32 *Apply(uptr x) {
	839	DCHECK(IsAppMemImpl::Apply<Mapping>(x));
	840	return (u32 *)(((((x) & ~(Mapping::kShadowMsk \| (kMetaShadowCell - 1)))) /
	841	kMetaShadowCell * kMetaShadowSize) \|
	842	Mapping::kMetaShadowBeg);
	843	}
	844	};
1112	845
1113	846	ALWAYS_INLINE
1114		uptr GetThreadTrace(int tid) {
1115		#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
1116		switch (vmaSize) {
1117		case 39: return GetThreadTraceImpl<Mapping39>(tid);
1118		case 42: return GetThreadTraceImpl<Mapping42>(tid);
1119		case 48: return GetThreadTraceImpl<Mapping48>(tid);
1120		}
1121		DCHECK(0);
1122		return 0;
1123		#elif defined(__powerpc64__)
1124		switch (vmaSize) {
1125		#if !SANITIZER_GO
1126		case 44: return GetThreadTraceImpl<Mapping44>(tid);
1127		#endif
1128		case 46: return GetThreadTraceImpl<Mapping46>(tid);
1129		case 47: return GetThreadTraceImpl<Mapping47>(tid);
	847	u32 *MemToMeta(uptr x) { return SelectMapping<MemToMetaImpl>(x); }
	848
	849	struct ShadowToMemImpl {
	850	template <typename Mapping>
	851	static uptr Apply(uptr sp) {
	852	if (!IsShadowMemImpl::Apply<Mapping>(sp))
	853	return 0;
	854	// The shadow mapping is non-linear and we've lost some bits, so we don't
	855	// have an easy way to restore the original app address. But the mapping is
	856	// a bijection, so we try to restore the address as belonging to
	857	// low/mid/high range consecutively and see if shadow->app->shadow mapping
	858	// gives us the same address.
	859	uptr p =
	860	((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor;
	861	if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
	862	MemToShadowImpl::Apply<Mapping>(p) == sp)
	863	return p;
	864	if (Mapping::kMidAppMemBeg) {
	865	uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk);
	866	if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd &&
	867	MemToShadowImpl::Apply<Mapping>(p_mid) == sp)
	868	return p_mid;
	869	}
	870	return p \| Mapping::kShadowMsk;
1130	871	}
1131		DCHECK(0);
1132		return 0;
1133		#elif defined(__mips64)
1134		switch (vmaSize) {
1135		#if !SANITIZER_GO
1136		case 40: return GetThreadTraceImpl<Mapping40>(tid);
1137		#else
1138		case 47: return GetThreadTraceImpl<Mapping47>(tid);
1139		#endif
1140		}
1141		DCHECK(0);
1142		return 0;
1143		#else
1144		return GetThreadTraceImpl<Mapping>(tid);
1145		#endif
1146		}
1147
1148
1149		template<typename Mapping>
1150		uptr GetThreadTraceHeaderImpl(int tid) {
1151		uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize
1152		+ kTraceSize * sizeof(Event);
1153		DCHECK_LT(p, Mapping::kTraceMemEnd);
1154		return p;
1155		}
	872	};
1156	873
1157	874	ALWAYS_INLINE
1158		uptr GetThreadTraceHeader(int tid) {
1159		#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
1160		switch (vmaSize) {
1161		case 39: return GetThreadTraceHeaderImpl<Mapping39>(tid);
1162		case 42: return GetThreadTraceHeaderImpl<Mapping42>(tid);
1163		case 48: return GetThreadTraceHeaderImpl<Mapping48>(tid);
1164		}
1165		DCHECK(0);
1166		return 0;
1167		#elif defined(__powerpc64__)
1168		switch (vmaSize) {
1169		#if !SANITIZER_GO
1170		case 44: return GetThreadTraceHeaderImpl<Mapping44>(tid);
1171		#endif
1172		case 46: return GetThreadTraceHeaderImpl<Mapping46>(tid);
1173		case 47: return GetThreadTraceHeaderImpl<Mapping47>(tid);
	875	uptr ShadowToMem(RawShadow *s) {
	876	return SelectMapping<ShadowToMemImpl>(reinterpret_cast<uptr>(s));
	877	}
	878
	879	// Compresses addr to kCompressedAddrBits stored in least significant bits.
	880	ALWAYS_INLINE uptr CompressAddr(uptr addr) {
	881	return addr & ((1ull << kCompressedAddrBits) - 1);
	882	}
	883
	884	struct RestoreAddrImpl {
	885	typedef uptr Result;
	886	template <typename Mapping>
	887	static Result Apply(uptr addr) {
	888	// To restore the address we go over all app memory ranges and check if top
	889	// 3 bits of the compressed addr match that of the app range. If yes, we
	890	// assume that the compressed address come from that range and restore the
	891	// missing top bits to match the app range address.
	892	const uptr ranges[] = {
	893	Mapping::kLoAppMemBeg, Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg,
	894	Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd,
	895	Mapping::kHeapMemBeg, Mapping::kHeapMemEnd,
	896	};
	897	const uptr indicator = 0x0e0000000000ull;
	898	const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator);
	899	for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) {
	900	uptr beg = ranges[i];
	901	uptr end = ranges[i + 1];
	902	if (beg == end)
	903	continue;
	904	for (uptr p = beg; p < end; p = RoundDown(p + ind_lsb, ind_lsb)) {
	905	if ((addr & indicator) == (p & indicator))
	906	return addr \| (p & ~(ind_lsb - 1));
	907	}
	908	}
	909	Printf("ThreadSanitizer: failed to restore address 0x%zx\n", addr);
	910	Die();
1174	911	}
1175		DCHECK(0);
1176		return 0;
1177		#elif defined(__mips64)
1178		switch (vmaSize) {
1179		#if !SANITIZER_GO
1180		case 40: return GetThreadTraceHeaderImpl<Mapping40>(tid);
1181		#else
1182		case 47: return GetThreadTraceHeaderImpl<Mapping47>(tid);
1183		#endif
1184		}
1185		DCHECK(0);
1186		return 0;
1187		#else
1188		return GetThreadTraceHeaderImpl<Mapping>(tid);
1189		#endif
	912	};
	913
	914	// Restores compressed addr from kCompressedAddrBits to full representation.
	915	// This is called only during reporting and is not performance-critical.
	916	inline uptr RestoreAddr(uptr addr) {
	917	return SelectMapping<RestoreAddrImpl>(addr);
1190	918	}
1191	919
1192	920	void InitializePlatform();
1193	921	void InitializePlatformEarly();
1194	922	void CheckAndProtect();
1195	923	void InitializeShadowMemoryPlatform();
1196		void FlushShadowMemory();
1197		void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive);
	924	void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);
1198	925	int ExtractResolvFDs(void state, int fds, int nfd);
1199	926	int ExtractRecvmsgFDs(void msg, int fds, int nfd);
1200	927	uptr ExtractLongJmpSp(uptr *env);

lib/tsan/tsan_platform_linux.cpp+89-68

...	...	@@ -66,7 +66,8 @@ extern "C" void *__libc_stack_end;
66	66	void *__libc_stack_end = 0;
67	67	#endif
68	68
69		#if SANITIZER_LINUX && defined(__aarch64__) && !SANITIZER_GO
	69	#if SANITIZER_LINUX && (defined(__aarch64__) \|\| defined(__loongarch_lp64)) && \
	70	!SANITIZER_GO
70	71	# define INIT_LONGJMP_XOR_KEY 1
71	72	#else
72	73	# define INIT_LONGJMP_XOR_KEY 0
...	...	@@ -85,78 +86,71 @@ static void InitializeLongjmpXorKey();
85	86	static uptr longjmp_xor_key;
86	87	#endif
87	88
88		#ifdef TSAN_RUNTIME_VMA
89	89	// Runtime detected VMA size.
90	90	uptr vmaSize;
91		#endif
92	91
93	92	enum {
94		MemTotal = 0,
95		MemShadow = 1,
96		MemMeta = 2,
97		MemFile = 3,
98		MemMmap = 4,
99		MemTrace = 5,
100		MemHeap = 6,
101		MemOther = 7,
102		MemCount = 8,
	93	MemTotal,
	94	MemShadow,
	95	MemMeta,
	96	MemFile,
	97	MemMmap,
	98	MemHeap,
	99	MemOther,
	100	MemCount,
103	101	};
104	102
105		void FillProfileCallback(uptr p, uptr rss, bool file,
106		uptr *mem, uptr stats_size) {
	103	void FillProfileCallback(uptr p, uptr rss, bool file, uptr *mem) {
107	104	mem[MemTotal] += rss;
108	105	if (p >= ShadowBeg() && p < ShadowEnd())
109	106	mem[MemShadow] += rss;
110	107	else if (p >= MetaShadowBeg() && p < MetaShadowEnd())
111	108	mem[MemMeta] += rss;
112		#if !SANITIZER_GO
	109	else if ((p >= LoAppMemBeg() && p < LoAppMemEnd()) \|\|
	110	(p >= MidAppMemBeg() && p < MidAppMemEnd()) \|\|
	111	(p >= HiAppMemBeg() && p < HiAppMemEnd()))
	112	mem[file ? MemFile : MemMmap] += rss;
113	113	else if (p >= HeapMemBeg() && p < HeapMemEnd())
114	114	mem[MemHeap] += rss;
115		else if (p >= LoAppMemBeg() && p < LoAppMemEnd())
116		mem[file ? MemFile : MemMmap] += rss;
117		else if (p >= HiAppMemBeg() && p < HiAppMemEnd())
118		mem[file ? MemFile : MemMmap] += rss;
119		#else
120		else if (p >= AppMemBeg() && p < AppMemEnd())
121		mem[file ? MemFile : MemMmap] += rss;
122		#endif
123		else if (p >= TraceMemBeg() && p < TraceMemEnd())
124		mem[MemTrace] += rss;
125	115	else
126	116	mem[MemOther] += rss;
127	117	}
128	118
129		void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
	119	void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
130	120	uptr mem[MemCount];
131		internal_memset(mem, 0, sizeof(mem[0]) * MemCount);
132		__sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7);
133		StackDepotStats *stacks = StackDepotGetStats();
134		internal_snprintf(buf, buf_size,
135		"RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd"
136		" trace:%zd heap:%zd other:%zd stacks=%zd[%zd] nthr=%zd/%zd\n",
	121	internal_memset(mem, 0, sizeof(mem));
	122	GetMemoryProfile(FillProfileCallback, mem);
	123	auto meta = ctx->metamap.GetMemoryStats();
	124	StackDepotStats stacks = StackDepotGetStats();
	125	uptr nthread, nlive;
	126	ctx->thread_registry.GetNumberOfThreads(&nthread, &nlive);
	127	uptr trace_mem;
	128	{
	129	Lock l(&ctx->slot_mtx);
	130	trace_mem = ctx->trace_part_total_allocated * sizeof(TracePart);
	131	}
	132	uptr internal_stats[AllocatorStatCount];
	133	internal_allocator()->GetStats(internal_stats);
	134	// All these are allocated from the common mmap region.
	135	mem[MemMmap] -= meta.mem_block + meta.sync_obj + trace_mem +
	136	stacks.allocated + internal_stats[AllocatorStatMapped];
	137	if (s64(mem[MemMmap]) < 0)
	138	mem[MemMmap] = 0;
	139	internal_snprintf(
	140	buf, buf_size,
	141	"==%zu== %llus [%zu]: RSS %zd MB: shadow:%zd meta:%zd file:%zd"
	142	" mmap:%zd heap:%zd other:%zd intalloc:%zd memblocks:%zd syncobj:%zu"
	143	" trace:%zu stacks=%zd threads=%zu/%zu\n",
	144	internal_getpid(), uptime_ns / (1000 * 1000 * 1000), ctx->global_epoch,
137	145	mem[MemTotal] >> 20, mem[MemShadow] >> 20, mem[MemMeta] >> 20,
138		mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemTrace] >> 20,
139		mem[MemHeap] >> 20, mem[MemOther] >> 20,
140		stacks->allocated >> 20, stacks->n_uniq_ids,
141		nlive, nthread);
142		}
143
144		#if SANITIZER_LINUX
145		void FlushShadowMemoryCallback(
146		const SuspendedThreadsList &suspended_threads_list,
147		void *argument) {
148		ReleaseMemoryPagesToOS(ShadowBeg(), ShadowEnd());
149		}
150		#endif
151
152		void FlushShadowMemory() {
153		#if SANITIZER_LINUX
154		StopTheWorld(FlushShadowMemoryCallback, 0);
155		#endif
	146	mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemHeap] >> 20,
	147	mem[MemOther] >> 20, internal_stats[AllocatorStatMapped] >> 20,
	148	meta.mem_block >> 20, meta.sync_obj >> 20, trace_mem >> 20,
	149	stacks.allocated >> 20, nlive, nthread);
156	150	}
157	151
158	152	#if !SANITIZER_GO
159		// Mark shadow for .rodata sections with the special kShadowRodata marker.
	153	// Mark shadow for .rodata sections with the special Shadow::kRodata marker.
160	154	// Accesses to .rodata can't race, so this saves time, memory and trace space.
161	155	static void MapRodata() {
162	156	// First create temp file.
...	...	@@ -177,13 +171,14 @@ static void MapRodata() {
177	171	return;
178	172	internal_unlink(name); // Unlink it now, so that we can reuse the buffer.
179	173	fd_t fd = openrv;
180		// Fill the file with kShadowRodata.
181		const uptr kMarkerSize = 512 * 1024 / sizeof(u64);
182		InternalMmapVector<u64> marker(kMarkerSize);
	174	// Fill the file with Shadow::kRodata.
	175	const uptr kMarkerSize = 512 * 1024 / sizeof(RawShadow);
	176	InternalMmapVector<RawShadow> marker(kMarkerSize);
183	177	// volatile to prevent insertion of memset
184		for (volatile u64 *p = marker.data(); p < marker.data() + kMarkerSize; p++)
185		*p = kShadowRodata;
186		internal_write(fd, marker.data(), marker.size() * sizeof(u64));
	178	for (volatile RawShadow *p = marker.data(); p < marker.data() + kMarkerSize;
	179	p++)
	180	*p = Shadow::kRodata;
	181	internal_write(fd, marker.data(), marker.size() * sizeof(RawShadow));
187	182	// Map the file into memory.
188	183	uptr page = internal_mmap(0, GetPageSizeCached(), PROT_READ \| PROT_WRITE,
189	184	MAP_PRIVATE \| MAP_ANONYMOUS, fd, 0);
...	...	@@ -203,9 +198,10 @@ static void MapRodata() {
203	198	char shadow_start = (char )MemToShadow(segment.start);
204	199	char shadow_end = (char )MemToShadow(segment.end);
205	200	for (char *p = shadow_start; p < shadow_end;
206		p += marker.size() * sizeof(u64)) {
207		internal_mmap(p, Min<uptr>(marker.size() * sizeof(u64), shadow_end - p),
208		PROT_READ, MAP_PRIVATE \| MAP_FIXED, fd, 0);
	201	p += marker.size() * sizeof(RawShadow)) {
	202	internal_mmap(
	203	p, Min<uptr>(marker.size() * sizeof(RawShadow), shadow_end - p),
	204	PROT_READ, MAP_PRIVATE \| MAP_FIXED, fd, 0);
209	205	}
210	206	}
211	207	}
...	...	@@ -219,7 +215,6 @@ void InitializeShadowMemoryPlatform() {
219	215	#endif // #if !SANITIZER_GO
220	216
221	217	void InitializePlatformEarly() {
222		#ifdef TSAN_RUNTIME_VMA
223	218	vmaSize =
224	219	(MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
225	220	#if defined(__aarch64__)
...	...	@@ -236,6 +231,14 @@ void InitializePlatformEarly() {
236	231	Die();
237	232	}
238	233	#endif
	234	#elif SANITIZER_LOONGARCH64
	235	# if !SANITIZER_GO
	236	if (vmaSize != 47) {
	237	Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
	238	Printf("FATAL: Found %zd - Supported 47\n", vmaSize);
	239	Die();
	240	}
	241	# endif
239	242	#elif defined(__powerpc64__)
240	243	# if !SANITIZER_GO
241	244	if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) {
...	...	@@ -265,7 +268,6 @@ void InitializePlatformEarly() {
265	268	}
266	269	# endif
267	270	#endif
268		#endif
269	271	}
270	272
271	273	void InitializePlatform() {
...	...	@@ -297,11 +299,12 @@ void InitializePlatform() {
297	299	SetAddressSpaceUnlimited();
298	300	reexec = true;
299	301	}
300		#if SANITIZER_LINUX && defined(__aarch64__)
	302	#if SANITIZER_ANDROID && (defined(__aarch64__) \|\| defined(__x86_64__))
301	303	// After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in
302	304	// linux kernel, the random gap between stack and mapped area is increased
303	305	// from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover
304	306	// this big range, we should disable randomized virtual space on aarch64.
	307	// ASLR personality check.
305	308	int old_personality = personality(0xffffffff);
306	309	if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) {
307	310	VReport(1, "WARNING: Program is run with randomized virtual address "
...	...	@@ -310,6 +313,9 @@ void InitializePlatform() {
310	313	CHECK_NE(personality(old_personality \| ADDR_NO_RANDOMIZE), -1);
311	314	reexec = true;
312	315	}
	316
	317	#endif
	318	#if SANITIZER_LINUX && (defined(__aarch64__) \|\| defined(__loongarch_lp64))
313	319	// Initialize the xor key used in {sig}{set,long}jump.
314	320	InitializeLongjmpXorKey();
315	321	#endif
...	...	@@ -341,7 +347,7 @@ int ExtractResolvFDs(void state, int fds, int nfd) {
341	347	}
342	348
343	349	// Extract file descriptors passed via UNIX domain sockets.
344		// This is requried to properly handle "open" of these fds.
	350	// This is required to properly handle "open" of these fds.
345	351	// see 'man recvmsg' and 'man 3 cmsg'.
346	352	int ExtractRecvmsgFDs(void msgp, int fds, int nfd) {
347	353	int res = 0;
...	...	@@ -382,6 +388,8 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {
382	388	# else
383	389	return mangled_sp;
384	390	# endif
	391	#elif defined(__loongarch_lp64)
	392	return mangled_sp ^ longjmp_xor_key;
385	393	#elif defined(__powerpc64__)
386	394	// Reverse of:
387	395	// ld r4, -28696(r13)
...	...	@@ -409,10 +417,16 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {
409	417	#elif defined(__powerpc__)
410	418	# define LONG_JMP_SP_ENV_SLOT 0
411	419	#elif SANITIZER_FREEBSD
412		# define LONG_JMP_SP_ENV_SLOT 2
	420	# ifdef __aarch64__
	421	# define LONG_JMP_SP_ENV_SLOT 1
	422	# else
	423	# define LONG_JMP_SP_ENV_SLOT 2
	424	# endif
413	425	#elif SANITIZER_LINUX
414	426	# ifdef __aarch64__
415	427	# define LONG_JMP_SP_ENV_SLOT 13
	428	# elif defined(__loongarch__)
	429	# define LONG_JMP_SP_ENV_SLOT 1
416	430	# elif defined(__mips64)
417	431	# define LONG_JMP_SP_ENV_SLOT 1
418	432	# elif defined(__s390x__)
...	...	@@ -439,7 +453,11 @@ static void InitializeLongjmpXorKey() {
439	453
440	454	// 2. Retrieve vanilla/mangled SP.
441	455	uptr sp;
	456	#ifdef __loongarch__
	457	asm("move %0, $sp" : "=r" (sp));
	458	#else
442	459	asm("mov %0, sp" : "=r" (sp));
	460	#endif
443	461	uptr mangled_sp = ((uptr *)&env)[LONG_JMP_SP_ENV_SLOT];
444	462
445	463	// 3. xor SPs to obtain key.
...	...	@@ -447,6 +465,8 @@ static void InitializeLongjmpXorKey() {
447	465	}
448	466	#endif
449	467
	468	extern "C" void __tsan_tls_initialization() {}
	469
450	470	void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
451	471	// Check that the thr object is in tls;
452	472	const uptr thr_beg = (uptr)thr;
...	...	@@ -456,9 +476,10 @@ void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
456	476	CHECK_GE(thr_end, tls_addr);
457	477	CHECK_LE(thr_end, tls_addr + tls_size);
458	478	// Since the thr object is huge, skip it.
459		MemoryRangeImitateWrite(thr, /pc=/2, tls_addr, thr_beg - tls_addr);
460		MemoryRangeImitateWrite(thr, /pc=/2, thr_end,
461		tls_addr + tls_size - thr_end);
	479	const uptr pc = StackTrace::GetNextInstructionPc(
	480	reinterpret_cast<uptr>(__tsan_tls_initialization));
	481	MemoryRangeImitateWrite(thr, pc, tls_addr, thr_beg - tls_addr);
	482	MemoryRangeImitateWrite(thr, pc, thr_end, tls_addr + tls_size - thr_end);
462	483	}
463	484
464	485	// Note: this function runs with async signals enabled,

lib/tsan/tsan_platform_mac.cpp+130-149

...	...	@@ -12,7 +12,7 @@
12	12	//===----------------------------------------------------------------------===//
13	13
14	14	#include "sanitizer_common/sanitizer_platform.h"
15		#if SANITIZER_MAC
	15	#if SANITIZER_APPLE
16	16
17	17	#include "sanitizer_common/sanitizer_atomic.h"
18	18	#include "sanitizer_common/sanitizer_common.h"
...	...	@@ -25,6 +25,7 @@
25	25	#include "tsan_rtl.h"
26	26	#include "tsan_flags.h"
27	27
	28	#include <limits.h>
28	29	#include <mach/mach.h>
29	30	#include <pthread.h>
30	31	#include <signal.h>
...	...	@@ -45,76 +46,86 @@
45	46	namespace __tsan {
46	47
47	48	#if !SANITIZER_GO
48		static void SignalSafeGetOrAllocate(uptr dst, uptr size) {
49		atomic_uintptr_t a = (atomic_uintptr_t )dst;
50		void val = (void )atomic_load_relaxed(a);
51		atomic_signal_fence(memory_order_acquire); // Turns the previous load into
52		// acquire wrt signals.
53		if (UNLIKELY(val == nullptr)) {
54		val = (void *)internal_mmap(nullptr, size, PROT_READ \| PROT_WRITE,
55		MAP_PRIVATE \| MAP_ANON, -1, 0);
56		CHECK(val);
57		void *cmp = nullptr;
58		if (!atomic_compare_exchange_strong(a, (uintptr_t *)&cmp, (uintptr_t)val,
59		memory_order_acq_rel)) {
60		internal_munmap(val, size);
61		val = cmp;
62		}
63		}
64		return val;
	49	static char main_thread_state[sizeof(ThreadState)] ALIGNED(
	50	SANITIZER_CACHE_LINE_SIZE);
	51	static ThreadState *dead_thread_state;
	52	static pthread_key_t thread_state_key;
	53
	54	// We rely on the following documented, but Darwin-specific behavior to keep the
	55	// reference to the ThreadState object alive in TLS:
	56	// pthread_key_create man page:
	57	// If, after all the destructors have been called for all non-NULL values with
	58	// associated destructors, there are still some non-NULL values with
	59	// associated destructors, then the process is repeated. If, after at least
	60	// [PTHREAD_DESTRUCTOR_ITERATIONS] iterations of destructor calls for
	61	// outstanding non-NULL values, there are still some non-NULL values with
	62	// associated destructors, the implementation stops calling destructors.
	63	static_assert(PTHREAD_DESTRUCTOR_ITERATIONS == 4, "Small number of iterations");
	64	static void ThreadStateDestructor(void *thr) {
	65	int res = pthread_setspecific(thread_state_key, thr);
	66	CHECK_EQ(res, 0);
65	67	}
66	68
67		// On OS X, accessing TLVs via __thread or manually by using pthread_key_* is
68		// problematic, because there are several places where interceptors are called
69		// when TLVs are not accessible (early process startup, thread cleanup, ...).
70		// The following provides a "poor man's TLV" implementation, where we use the
71		// shadow memory of the pointer returned by pthread_self() to store a pointer to
72		// the ThreadState object. The main thread's ThreadState is stored separately
73		// in a static variable, because we need to access it even before the
74		// shadow memory is set up.
75		static uptr main_thread_identity = 0;
76		ALIGNED(64) static char main_thread_state[sizeof(ThreadState)];
77		static ThreadState main_thread_state_loc = (ThreadState )main_thread_state;
78
79		// We cannot use pthread_self() before libpthread has been initialized. Our
80		// current heuristic for guarding this is checking `main_thread_identity` which
81		// is only assigned in `__tsan::InitializePlatform`.
82		static ThreadState **cur_thread_location() {
83		if (main_thread_identity == 0)
84		return &main_thread_state_loc;
85		uptr thread_identity = (uptr)pthread_self();
86		if (thread_identity == main_thread_identity)
87		return &main_thread_state_loc;
88		return (ThreadState **)MemToShadow(thread_identity);
	69	static void InitializeThreadStateStorage() {
	70	int res;
	71	CHECK_EQ(thread_state_key, 0);
	72	res = pthread_key_create(&thread_state_key, ThreadStateDestructor);
	73	CHECK_EQ(res, 0);
	74	res = pthread_setspecific(thread_state_key, main_thread_state);
	75	CHECK_EQ(res, 0);
	76
	77	auto dts = (ThreadState *)MmapOrDie(sizeof(ThreadState), "ThreadState");
	78	dts->fast_state.SetIgnoreBit();
	79	dts->ignore_interceptors = 1;
	80	dts->is_dead = true;
	81	const_cast<Tid &>(dts->tid) = kInvalidTid;
	82	res = internal_mprotect(dts, sizeof(ThreadState), PROT_READ); // immutable
	83	CHECK_EQ(res, 0);
	84	dead_thread_state = dts;
89	85	}
90	86
91	87	ThreadState *cur_thread() {
92		return (ThreadState *)SignalSafeGetOrAllocate(
93		(uptr *)cur_thread_location(), sizeof(ThreadState));
	88	// Some interceptors get called before libpthread has been initialized and in
	89	// these cases we must avoid calling any pthread APIs.
	90	if (UNLIKELY(!thread_state_key)) {
	91	return (ThreadState *)main_thread_state;
	92	}
	93
	94	// We only reach this line after InitializeThreadStateStorage() ran, i.e,
	95	// after TSan (and therefore libpthread) have been initialized.
	96	ThreadState thr = (ThreadState )pthread_getspecific(thread_state_key);
	97	if (UNLIKELY(!thr)) {
	98	thr = (ThreadState *)MmapOrDie(sizeof(ThreadState), "ThreadState");
	99	int res = pthread_setspecific(thread_state_key, thr);
	100	CHECK_EQ(res, 0);
	101	}
	102	return thr;
94	103	}
95	104
96	105	void set_cur_thread(ThreadState *thr) {
97		*cur_thread_location() = thr;
	106	int res = pthread_setspecific(thread_state_key, thr);
	107	CHECK_EQ(res, 0);
98	108	}
99	109
100		// TODO(kuba.brecka): This is not async-signal-safe. In particular, we call
101		// munmap first and then clear `fake_tls`; if we receive a signal in between,
102		// handler will try to access the unmapped ThreadState.
103	110	void cur_thread_finalize() {
104		ThreadState **thr_state_loc = cur_thread_location();
105		if (thr_state_loc == &main_thread_state_loc) {
	111	ThreadState thr = (ThreadState )pthread_getspecific(thread_state_key);
	112	CHECK(thr);
	113	if (thr == (ThreadState *)main_thread_state) {
106	114	// Calling dispatch_main() or xpc_main() actually invokes pthread_exit to
107	115	// exit the main thread. Let's keep the main thread's ThreadState.
108	116	return;
109	117	}
110		internal_munmap(*thr_state_loc, sizeof(ThreadState));
111		*thr_state_loc = nullptr;
	118	// Intercepted functions can still get called after cur_thread_finalize()
	119	// (called from DestroyThreadState()), so put a fake thread state for "dead"
	120	// threads. An alternative solution would be to release the ThreadState
	121	// object from THREAD_DESTROY (which is delivered later and on the parent
	122	// thread) instead of THREAD_TERMINATE.
	123	int res = pthread_setspecific(thread_state_key, dead_thread_state);
	124	CHECK_EQ(res, 0);
	125	UnmapOrDie(thr, sizeof(ThreadState));
112	126	}
113	127	#endif
114	128
115		void FlushShadowMemory() {
116		}
117
118	129	static void RegionMemUsage(uptr start, uptr end, uptr res, uptr dirty) {
119	130	vm_address_t address = start;
120	131	vm_address_t end_address = end;
...	...	@@ -139,15 +150,13 @@ static void RegionMemUsage(uptr start, uptr end, uptr res, uptr dirty) {
139	150	dirty = dirty_pages GetPageSizeCached();
140	151	}
141	152
142		void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
	153	void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
143	154	uptr shadow_res, shadow_dirty;
144	155	uptr meta_res, meta_dirty;
145		uptr trace_res, trace_dirty;
146	156	RegionMemUsage(ShadowBeg(), ShadowEnd(), &shadow_res, &shadow_dirty);
147	157	RegionMemUsage(MetaShadowBeg(), MetaShadowEnd(), &meta_res, &meta_dirty);
148		RegionMemUsage(TraceMemBeg(), TraceMemEnd(), &trace_res, &trace_dirty);
149	158
150		#if !SANITIZER_GO
	159	# if !SANITIZER_GO
151	160	uptr low_res, low_dirty;
152	161	uptr high_res, high_dirty;
153	162	uptr heap_res, heap_dirty;
...	...	@@ -156,89 +165,70 @@ void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
156	165	RegionMemUsage(HeapMemBeg(), HeapMemEnd(), &heap_res, &heap_dirty);
157	166	#else // !SANITIZER_GO
158	167	uptr app_res, app_dirty;
159		RegionMemUsage(AppMemBeg(), AppMemEnd(), &app_res, &app_dirty);
	168	RegionMemUsage(LoAppMemBeg(), LoAppMemEnd(), &app_res, &app_dirty);
160	169	#endif
161	170
162		StackDepotStats *stacks = StackDepotGetStats();
163		internal_snprintf(buf, buf_size,
164		"shadow (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
165		"meta (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
166		"traces (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
167		#if !SANITIZER_GO
168		"low app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
169		"high app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
170		"heap (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
171		#else // !SANITIZER_GO
172		"app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
173		#endif
174		"stacks: %zd unique IDs, %zd kB allocated\n"
175		"threads: %zd total, %zd live\n"
176		"------------------------------\n",
177		ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024,
178		MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024,
179		TraceMemBeg(), TraceMemEnd(), trace_res / 1024, trace_dirty / 1024,
180		#if !SANITIZER_GO
181		LoAppMemBeg(), LoAppMemEnd(), low_res / 1024, low_dirty / 1024,
182		HiAppMemBeg(), HiAppMemEnd(), high_res / 1024, high_dirty / 1024,
183		HeapMemBeg(), HeapMemEnd(), heap_res / 1024, heap_dirty / 1024,
184		#else // !SANITIZER_GO
185		AppMemBeg(), AppMemEnd(), app_res / 1024, app_dirty / 1024,
186		#endif
187		stacks->n_uniq_ids, stacks->allocated / 1024,
188		nthread, nlive);
	171	StackDepotStats stacks = StackDepotGetStats();
	172	uptr nthread, nlive;
	173	ctx->thread_registry.GetNumberOfThreads(&nthread, &nlive);
	174	internal_snprintf(
	175	buf, buf_size,
	176	"shadow (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
	177	"meta (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
	178	# if !SANITIZER_GO
	179	"low app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
	180	"high app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
	181	"heap (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
	182	# else // !SANITIZER_GO
	183	"app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
	184	# endif
	185	"stacks: %zd unique IDs, %zd kB allocated\n"
	186	"threads: %zd total, %zd live\n"
	187	"------------------------------\n",
	188	ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024,
	189	MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024,
	190	# if !SANITIZER_GO
	191	LoAppMemBeg(), LoAppMemEnd(), low_res / 1024, low_dirty / 1024,
	192	HiAppMemBeg(), HiAppMemEnd(), high_res / 1024, high_dirty / 1024,
	193	HeapMemBeg(), HeapMemEnd(), heap_res / 1024, heap_dirty / 1024,
	194	# else // !SANITIZER_GO
	195	LoAppMemBeg(), LoAppMemEnd(), app_res / 1024, app_dirty / 1024,
	196	# endif
	197	stacks.n_uniq_ids, stacks.allocated / 1024, nthread, nlive);
189	198	}
190	199
191		#if !SANITIZER_GO
	200	# if !SANITIZER_GO
192	201	void InitializeShadowMemoryPlatform() { }
193	202
194		// On OS X, GCD worker threads are created without a call to pthread_create. We
195		// need to properly register these threads with ThreadCreate and ThreadStart.
196		// These threads don't have a parent thread, as they are created "spuriously".
197		// We're using a libpthread API that notifies us about a newly created thread.
198		// The `thread == pthread_self()` check indicates this is actually a worker
199		// thread. If it's just a regular thread, this hook is called on the parent
200		// thread.
201		typedef void (*pthread_introspection_hook_t)(unsigned int event,
202		pthread_t thread, void *addr,
203		size_t size);
204		extern "C" pthread_introspection_hook_t pthread_introspection_hook_install(
205		pthread_introspection_hook_t hook);
206		static const uptr PTHREAD_INTROSPECTION_THREAD_CREATE = 1;
207		static const uptr PTHREAD_INTROSPECTION_THREAD_TERMINATE = 3;
208		static pthread_introspection_hook_t prev_pthread_introspection_hook;
209		static void my_pthread_introspection_hook(unsigned int event, pthread_t thread,
210		void *addr, size_t size) {
211		if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) {
212		if (thread == pthread_self()) {
213		// The current thread is a newly created GCD worker thread.
214		ThreadState *thr = cur_thread();
215		Processor *proc = ProcCreate();
216		ProcWire(proc, thr);
217		ThreadState *parent_thread_state = nullptr; // No parent.
218		int tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true);
219		CHECK_NE(tid, 0);
220		ThreadStart(thr, tid, GetTid(), ThreadType::Worker);
221		}
222		} else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) {
223		if (thread == pthread_self()) {
224		ThreadState *thr = cur_thread();
225		if (thr->tctx) {
226		DestroyThreadState();
227		}
228		}
	203	// Register GCD worker threads, which are created without an observable call to
	204	// pthread_create().
	205	static void ThreadCreateCallback(uptr thread, bool gcd_worker) {
	206	if (gcd_worker) {
	207	ThreadState *thr = cur_thread();
	208	Processor *proc = ProcCreate();
	209	ProcWire(proc, thr);
	210	ThreadState *parent_thread_state = nullptr; // No parent.
	211	Tid tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true);
	212	CHECK_NE(tid, kMainTid);
	213	ThreadStart(thr, tid, GetTid(), ThreadType::Worker);
229	214	}
	215	}
230	216
231		if (prev_pthread_introspection_hook != nullptr)
232		prev_pthread_introspection_hook(event, thread, addr, size);
	217	// Destroy thread state for all threads.
	218	static void ThreadTerminateCallback(uptr thread) {
	219	ThreadState *thr = cur_thread();
	220	if (thr->tctx) {
	221	DestroyThreadState();
	222	}
233	223	}
234	224	#endif
235	225
236	226	void InitializePlatformEarly() {
237		#if !SANITIZER_GO && !HAS_48_BIT_ADDRESS_SPACE
	227	# if !SANITIZER_GO && SANITIZER_IOS
238	228	uptr max_vm = GetMaxUserVirtualAddress() + 1;
239		if (max_vm != Mapping::kHiAppMemEnd) {
	229	if (max_vm != HiAppMemEnd()) {
240	230	Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
241		max_vm, Mapping::kHiAppMemEnd);
	231	(void )max_vm, (void )HiAppMemEnd());
242	232	Die();
243	233	}
244	234	#endif
...	...	@@ -251,11 +241,13 @@ void InitializePlatform() {
251	241	#if !SANITIZER_GO
252	242	CheckAndProtect();
253	243
254		CHECK_EQ(main_thread_identity, 0);
255		main_thread_identity = (uptr)pthread_self();
	244	InitializeThreadStateStorage();
256	245
257		prev_pthread_introspection_hook =
258		pthread_introspection_hook_install(&my_pthread_introspection_hook);
	246	ThreadEventCallbacks callbacks = {
	247	.create = ThreadCreateCallback,
	248	.terminate = ThreadTerminateCallback,
	249	};
	250	InstallPthreadIntrospectionHook(callbacks);
259	251	#endif
260	252
261	253	if (GetMacosAlignedVersion() >= MacosVersion(10, 14)) {
...	...	@@ -281,25 +273,14 @@ uptr ExtractLongJmpSp(uptr *env) {
281	273	}
282	274
283	275	#if !SANITIZER_GO
	276	extern "C" void __tsan_tls_initialization() {}
	277
284	278	void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
285		// The pointer to the ThreadState object is stored in the shadow memory
286		// of the tls.
287		uptr tls_end = tls_addr + tls_size;
288		uptr thread_identity = (uptr)pthread_self();
289		if (thread_identity == main_thread_identity) {
290		MemoryRangeImitateWrite(thr, /pc=/2, tls_addr, tls_size);
291		} else {
292		uptr thr_state_start = thread_identity;
293		uptr thr_state_end = thr_state_start + sizeof(uptr);
294		CHECK_GE(thr_state_start, tls_addr);
295		CHECK_LE(thr_state_start, tls_addr + tls_size);
296		CHECK_GE(thr_state_end, tls_addr);
297		CHECK_LE(thr_state_end, tls_addr + tls_size);
298		MemoryRangeImitateWrite(thr, /pc=/2, tls_addr,
299		thr_state_start - tls_addr);
300		MemoryRangeImitateWrite(thr, /pc=/2, thr_state_end,
301		tls_end - thr_state_end);
302		}
	279	const uptr pc = StackTrace::GetNextInstructionPc(
	280	reinterpret_cast<uptr>(__tsan_tls_initialization));
	281	// Unlike Linux, we only store a pointer to the ThreadState object in TLS;
	282	// just mark the entire range as written to.
	283	MemoryRangeImitateWrite(thr, pc, tls_addr, tls_size);
303	284	}
304	285	#endif
305	286
...	...	@@ -320,4 +301,4 @@ int call_pthread_cancel_with_cleanup(int (fn)(void arg),
320	301
321	302	} // namespace __tsan
322	303
323		#endif // SANITIZER_MAC
	304	#endif // SANITIZER_APPLE

lib/tsan/tsan_platform_posix.cpp+27-24

...	...	@@ -14,12 +14,14 @@
14	14	#include "sanitizer_common/sanitizer_platform.h"
15	15	#if SANITIZER_POSIX
16	16
17		#include "sanitizer_common/sanitizer_common.h"
18		#include "sanitizer_common/sanitizer_errno.h"
19		#include "sanitizer_common/sanitizer_libc.h"
20		#include "sanitizer_common/sanitizer_procmaps.h"
21		#include "tsan_platform.h"
22		#include "tsan_rtl.h"
	17	# include <dlfcn.h>
	18
	19	# include "sanitizer_common/sanitizer_common.h"
	20	# include "sanitizer_common/sanitizer_errno.h"
	21	# include "sanitizer_common/sanitizer_libc.h"
	22	# include "sanitizer_common/sanitizer_procmaps.h"
	23	# include "tsan_platform.h"
	24	# include "tsan_rtl.h"
23	25
24	26	namespace __tsan {
25	27
...	...	@@ -29,7 +31,8 @@ static const char kShadowMemoryMappingHint[] =
29	31	"HINT: if %s is not supported in your environment, you may set "
30	32	"TSAN_OPTIONS=%s=0\n";
31	33
32		static void DontDumpShadow(uptr addr, uptr size) {
	34	# if !SANITIZER_GO
	35	void DontDumpShadow(uptr addr, uptr size) {
33	36	if (common_flags()->use_madv_dontdump)
34	37	if (!DontDumpShadowMemory(addr, size)) {
35	38	Printf(kShadowMemoryMappingWarning, SanitizerToolName, addr, addr + size,
...	...	@@ -39,7 +42,6 @@ static void DontDumpShadow(uptr addr, uptr size) {
39	42	}
40	43	}
41	44
42		#if !SANITIZER_GO
43	45	void InitializeShadowMemory() {
44	46	// Map memory shadow.
45	47	if (!MmapFixedSuperNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(),
...	...	@@ -70,6 +72,11 @@ void InitializeShadowMemory() {
70	72	meta, meta + meta_size, meta_size >> 30);
71	73
72	74	InitializeShadowMemoryPlatform();
	75
	76	on_initialize = reinterpret_cast<void (*)(void)>(
	77	dlsym(RTLD_DEFAULT, "__tsan_on_initialize"));
	78	on_finalize =
	79	reinterpret_cast<int (*)(int)>(dlsym(RTLD_DEFAULT, "__tsan_on_finalize"));
73	80	}
74	81
75	82	static bool TryProtectRange(uptr beg, uptr end) {
...	...	@@ -98,32 +105,28 @@ void CheckAndProtect() {
98	105	continue;
99	106	if (segment.start >= VdsoBeg()) // vdso
100	107	break;
101		Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n",
	108	Printf("FATAL: ThreadSanitizer: unexpected memory mapping 0x%zx-0x%zx\n",
102	109	segment.start, segment.end);
103	110	Die();
104	111	}
105	112
106		#if defined(__aarch64__) && defined(__APPLE__) && !HAS_48_BIT_ADDRESS_SPACE
	113	# if SANITIZER_IOS && !SANITIZER_IOSSIM
107	114	ProtectRange(HeapMemEnd(), ShadowBeg());
108	115	ProtectRange(ShadowEnd(), MetaShadowBeg());
109		ProtectRange(MetaShadowEnd(), TraceMemBeg());
110		#else
	116	ProtectRange(MetaShadowEnd(), HiAppMemBeg());
	117	# else
111	118	ProtectRange(LoAppMemEnd(), ShadowBeg());
112	119	ProtectRange(ShadowEnd(), MetaShadowBeg());
113		#ifdef TSAN_MID_APP_RANGE
114		ProtectRange(MetaShadowEnd(), MidAppMemBeg());
115		ProtectRange(MidAppMemEnd(), TraceMemBeg());
116		#else
117		ProtectRange(MetaShadowEnd(), TraceMemBeg());
118		#endif
119		// Memory for traces is mapped lazily in MapThreadTrace.
120		// Protect the whole range for now, so that user does not map something here.
121		ProtectRange(TraceMemBeg(), TraceMemEnd());
122		ProtectRange(TraceMemEnd(), HeapMemBeg());
	120	if (MidAppMemBeg()) {
	121	ProtectRange(MetaShadowEnd(), MidAppMemBeg());
	122	ProtectRange(MidAppMemEnd(), HeapMemBeg());
	123	} else {
	124	ProtectRange(MetaShadowEnd(), HeapMemBeg());
	125	}
123	126	ProtectRange(HeapEnd(), HiAppMemBeg());
124		#endif
	127	# endif
125	128
126		#if defined(__s390x__)
	129	# if defined(__s390x__)
127	130	// Protect the rest of the address space.
128	131	const uptr user_addr_max_l4 = 0x0020000000000000ull;
129	132	const uptr user_addr_max_l5 = 0xfffffffffffff000ull;

lib/tsan/tsan_platform_windows.cpp created+33

...	...	@@ -0,0 +1,33 @@
	1	//===-- tsan_platform_windows.cpp -----------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	// Windows-specific code.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "sanitizer_common/sanitizer_platform.h"
	15	#if SANITIZER_WINDOWS
	16
	17	#include "tsan_platform.h"
	18
	19	#include <stdlib.h>
	20
	21	namespace __tsan {
	22
	23	void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {}
	24
	25	void InitializePlatformEarly() {
	26	}
	27
	28	void InitializePlatform() {
	29	}
	30
	31	} // namespace __tsan
	32
	33	#endif // SANITIZER_WINDOWS

lib/tsan/tsan_report.cpp+46-62

...	...	@@ -19,22 +19,6 @@
19	19
20	20	namespace __tsan {
21	21
22		ReportStack::ReportStack() : frames(nullptr), suppressable(false) {}
23
24		ReportStack *ReportStack::New() {
25		void *mem = internal_alloc(MBlockReportStack, sizeof(ReportStack));
26		return new(mem) ReportStack();
27		}
28
29		ReportLocation::ReportLocation(ReportLocationType type)
30		: type(type), global(), heap_chunk_start(0), heap_chunk_size(0), tid(0),
31		fd(0), suppressable(false), stack(nullptr) {}
32
33		ReportLocation *ReportLocation::New(ReportLocationType type) {
34		void *mem = internal_alloc(MBlockReportStack, sizeof(ReportLocation));
35		return new(mem) ReportLocation(type);
36		}
37
38	22	class Decorator: public __sanitizer::SanitizerCommonDecorator {
39	23	public:
40	24	Decorator() : SanitizerCommonDecorator() { }
...	...	@@ -68,7 +52,7 @@ ReportDesc::~ReportDesc() {
68	52	#if !SANITIZER_GO
69	53
70	54	const int kThreadBufSize = 32;
71		const char thread_name(char buf, int tid) {
	55	const char thread_name(char buf, Tid tid) {
72	56	if (tid == kMainTid)
73	57	return "main thread";
74	58	internal_snprintf(buf, kThreadBufSize, "thread T%d", tid);
...	...	@@ -114,12 +98,6 @@ static const char *ReportTypeString(ReportType typ, uptr tag) {
114	98	UNREACHABLE("missing case");
115	99	}
116	100
117		#if SANITIZER_MAC
118		static const char *const kInterposedFunctionPrefix = "wrap_";
119		#else
120		static const char *const kInterposedFunctionPrefix = "__interceptor_";
121		#endif
122
123	101	void PrintStack(const ReportStack *ent) {
124	102	if (ent == 0 \|\| ent->frames == 0) {
125	103	Printf(" [failed to restore the stack]\n\n");
...	...	@@ -131,7 +109,7 @@ void PrintStack(const ReportStack *ent) {
131	109	RenderFrame(&res, common_flags()->stack_trace_format, i,
132	110	frame->info.address, &frame->info,
133	111	common_flags()->symbolize_vs_style,
134		common_flags()->strip_path_prefix, kInterposedFunctionPrefix);
	112	common_flags()->strip_path_prefix);
135	113	Printf("%s\n", res.data());
136	114	}
137	115	Printf("\n");
...	...	@@ -142,7 +120,7 @@ static void PrintMutexSet(Vector<ReportMopMutex> const& mset) {
142	120	if (i == 0)
143	121	Printf(" (mutexes:");
144	122	const ReportMopMutex m = mset[i];
145		Printf(" %s M%llu", m.write ? "write" : "read", m.id);
	123	Printf(" %s M%u", m.write ? "write" : "read", m.id);
146	124	Printf(i == mset.Size() - 1 ? ")" : ",");
147	125	}
148	126	}
...	...	@@ -189,23 +167,25 @@ static void PrintLocation(const ReportLocation *loc) {
189	167	if (loc->type == ReportLocationGlobal) {
190	168	const DataInfo &global = loc->global;
191	169	if (global.size != 0)
192		Printf(" Location is global '%s' of size %zu at %p (%s+%p)\n\n",
193		global.name, global.size, global.start,
	170	Printf(" Location is global '%s' of size %zu at %p (%s+0x%zx)\n\n",
	171	global.name, global.size, reinterpret_cast<void *>(global.start),
194	172	StripModuleName(global.module), global.module_offset);
195	173	else
196		Printf(" Location is global '%s' at %p (%s+%p)\n\n", global.name,
197		global.start, StripModuleName(global.module),
198		global.module_offset);
	174	Printf(" Location is global '%s' at %p (%s+0x%zx)\n\n", global.name,
	175	reinterpret_cast<void *>(global.start),
	176	StripModuleName(global.module), global.module_offset);
199	177	} else if (loc->type == ReportLocationHeap) {
200	178	char thrbuf[kThreadBufSize];
201	179	const char *object_type = GetObjectTypeFromTag(loc->external_tag);
202	180	if (!object_type) {
203	181	Printf(" Location is heap block of size %zu at %p allocated by %s:\n",
204		loc->heap_chunk_size, loc->heap_chunk_start,
	182	loc->heap_chunk_size,
	183	reinterpret_cast<void *>(loc->heap_chunk_start),
205	184	thread_name(thrbuf, loc->tid));
206	185	} else {
207	186	Printf(" Location is %s of size %zu at %p allocated by %s:\n",
208		object_type, loc->heap_chunk_size, loc->heap_chunk_start,
	187	object_type, loc->heap_chunk_size,
	188	reinterpret_cast<void *>(loc->heap_chunk_start),
209	189	thread_name(thrbuf, loc->tid));
210	190	}
211	191	print_stack = true;
...	...	@@ -214,8 +194,9 @@ static void PrintLocation(const ReportLocation *loc) {
214	194	} else if (loc->type == ReportLocationTLS) {
215	195	Printf(" Location is TLS of %s.\n\n", thread_name(thrbuf, loc->tid));
216	196	} else if (loc->type == ReportLocationFD) {
217		Printf(" Location is file descriptor %d created by %s at:\n",
218		loc->fd, thread_name(thrbuf, loc->tid));
	197	Printf(" Location is file descriptor %d %s by %s at:\n", loc->fd,
	198	loc->fd_closed ? "destroyed" : "created",
	199	thread_name(thrbuf, loc->tid));
219	200	print_stack = true;
220	201	}
221	202	Printf("%s", d.Default());
...	...	@@ -225,27 +206,23 @@ static void PrintLocation(const ReportLocation *loc) {
225	206
226	207	static void PrintMutexShort(const ReportMutex rm, const char after) {
227	208	Decorator d;
228		Printf("%sM%zd%s%s", d.Mutex(), rm->id, d.Default(), after);
	209	Printf("%sM%d%s%s", d.Mutex(), rm->id, d.Default(), after);
229	210	}
230	211
231	212	static void PrintMutexShortWithAddress(const ReportMutex *rm,
232	213	const char *after) {
233	214	Decorator d;
234		Printf("%sM%zd (%p)%s%s", d.Mutex(), rm->id, rm->addr, d.Default(), after);
	215	Printf("%sM%d (%p)%s%s", d.Mutex(), rm->id,
	216	reinterpret_cast<void *>(rm->addr), d.Default(), after);
235	217	}
236	218
237	219	static void PrintMutex(const ReportMutex *rm) {
238	220	Decorator d;
239		if (rm->destroyed) {
240		Printf("%s", d.Mutex());
241		Printf(" Mutex M%llu is already destroyed.\n\n", rm->id);
242		Printf("%s", d.Default());
243		} else {
244		Printf("%s", d.Mutex());
245		Printf(" Mutex M%llu (%p) created at:\n", rm->id, rm->addr);
246		Printf("%s", d.Default());
247		PrintStack(rm->stack);
248		}
	221	Printf("%s", d.Mutex());
	222	Printf(" Mutex M%u (%p) created at:\n", rm->id,
	223	reinterpret_cast<void *>(rm->addr));
	224	Printf("%s", d.Default());
	225	PrintStack(rm->stack);
249	226	}
250	227
251	228	static void PrintThread(const ReportThread *rt) {
...	...	@@ -259,12 +236,13 @@ static void PrintThread(const ReportThread *rt) {
259	236	char thrbuf[kThreadBufSize];
260	237	const char *thread_status = rt->running ? "running" : "finished";
261	238	if (rt->thread_type == ThreadType::Worker) {
262		Printf(" (tid=%zu, %s) is a GCD worker thread\n", rt->os_id, thread_status);
	239	Printf(" (tid=%llu, %s) is a GCD worker thread\n", rt->os_id,
	240	thread_status);
263	241	Printf("\n");
264	242	Printf("%s", d.Default());
265	243	return;
266	244	}
267		Printf(" (tid=%zu, %s) created by %s", rt->os_id, thread_status,
	245	Printf(" (tid=%llu, %s) created by %s", rt->os_id, thread_status,
268	246	thread_name(thrbuf, rt->parent_tid));
269	247	if (rt->stack)
270	248	Printf(" at:");
...	...	@@ -300,6 +278,7 @@ static bool FrameIsInternal(const SymbolizedStack *frame) {
300	278	const char *module = frame->info.module;
301	279	if (file != 0 &&
302	280	(internal_strstr(file, "tsan_interceptors_posix.cpp") \|\|
	281	internal_strstr(file, "tsan_interceptors_memintrinsics.cpp") \|\|
303	282	internal_strstr(file, "sanitizer_common_interceptors.inc") \|\|
304	283	internal_strstr(file, "tsan_interface_")))
305	284	return true;
...	...	@@ -323,6 +302,9 @@ void PrintReport(const ReportDesc *rep) {
323	302	(int)internal_getpid());
324	303	Printf("%s", d.Default());
325	304
	305	if (rep->typ == ReportTypeErrnoInSignal)
	306	Printf(" Signal %u handler invoked at:\n", rep->signum);
	307
326	308	if (rep->typ == ReportTypeDeadlock) {
327	309	char thrbuf[kThreadBufSize];
328	310	Printf(" Cycle in lock order graph: ");
...	...	@@ -394,7 +376,7 @@ void PrintReport(const ReportDesc *rep) {
394	376
395	377	#else // #if !SANITIZER_GO
396	378
397		const u32 kMainGoroutineId = 1;
	379	const Tid kMainGoroutineId = 1;
398	380
399	381	void PrintStack(const ReportStack *ent) {
400	382	if (ent == 0 \|\| ent->frames == 0) {
...	...	@@ -405,16 +387,17 @@ void PrintStack(const ReportStack *ent) {
405	387	for (int i = 0; frame; frame = frame->next, i++) {
406	388	const AddressInfo &info = frame->info;
407	389	Printf(" %s()\n %s:%d +0x%zx\n", info.function,
408		StripPathPrefix(info.file, common_flags()->strip_path_prefix),
409		info.line, (void *)info.module_offset);
	390	StripPathPrefix(info.file, common_flags()->strip_path_prefix),
	391	info.line, info.module_offset);
410	392	}
411	393	}
412	394
413	395	static void PrintMop(const ReportMop *mop, bool first) {
414	396	Printf("\n");
415	397	Printf("%s at %p by ",
416		(first ? (mop->write ? "Write" : "Read")
417		: (mop->write ? "Previous write" : "Previous read")), mop->addr);
	398	(first ? (mop->write ? "Write" : "Read")
	399	: (mop->write ? "Previous write" : "Previous read")),
	400	reinterpret_cast<void *>(mop->addr));
418	401	if (mop->tid == kMainGoroutineId)
419	402	Printf("main goroutine:\n");
420	403	else
...	...	@@ -426,8 +409,8 @@ static void PrintLocation(const ReportLocation *loc) {
426	409	switch (loc->type) {
427	410	case ReportLocationHeap: {
428	411	Printf("\n");
429		Printf("Heap block of size %zu at %p allocated by ",
430		loc->heap_chunk_size, loc->heap_chunk_start);
	412	Printf("Heap block of size %zu at %p allocated by ", loc->heap_chunk_size,
	413	reinterpret_cast<void *>(loc->heap_chunk_start));
431	414	if (loc->tid == kMainGoroutineId)
432	415	Printf("main goroutine:\n");
433	416	else
...	...	@@ -438,8 +421,9 @@ static void PrintLocation(const ReportLocation *loc) {
438	421	case ReportLocationGlobal: {
439	422	Printf("\n");
440	423	Printf("Global var %s of size %zu at %p declared at %s:%zu\n",
441		loc->global.name, loc->global.size, loc->global.start,
442		loc->global.file, loc->global.line);
	424	loc->global.name, loc->global.size,
	425	reinterpret_cast<void *>(loc->global.start), loc->global.file,
	426	loc->global.line);
443	427	break;
444	428	}
445	429	default:
...	...	@@ -469,13 +453,13 @@ void PrintReport(const ReportDesc *rep) {
469	453	} else if (rep->typ == ReportTypeDeadlock) {
470	454	Printf("WARNING: DEADLOCK\n");
471	455	for (uptr i = 0; i < rep->mutexes.Size(); i++) {
472		Printf("Goroutine %d lock mutex %d while holding mutex %d:\n",
473		999, rep->mutexes[i]->id,
474		rep->mutexes[(i+1) % rep->mutexes.Size()]->id);
	456	Printf("Goroutine %d lock mutex %u while holding mutex %u:\n", 999,
	457	rep->mutexes[i]->id,
	458	rep->mutexes[(i + 1) % rep->mutexes.Size()]->id);
475	459	PrintStack(rep->stacks[2*i]);
476	460	Printf("\n");
477		Printf("Mutex %d was previously locked here:\n",
478		rep->mutexes[(i+1) % rep->mutexes.Size()]->id);
	461	Printf("Mutex %u was previously locked here:\n",
	462	rep->mutexes[(i + 1) % rep->mutexes.Size()]->id);
479	463	PrintStack(rep->stacks[2*i + 1]);
480	464	Printf("\n");
481	465	}

lib/tsan/tsan_report.h+18-25

...	...	@@ -38,16 +38,12 @@ enum ReportType {
38	38	};
39	39
40	40	struct ReportStack {
41		SymbolizedStack *frames;
42		bool suppressable;
43		static ReportStack *New();
44
45		private:
46		ReportStack();
	41	SymbolizedStack *frames = nullptr;
	42	bool suppressable = false;
47	43	};
48	44
49	45	struct ReportMopMutex {
50		u64 id;
	46	int id;
51	47	bool write;
52	48	};
53	49
...	...	@@ -73,35 +69,31 @@ enum ReportLocationType {
73	69	};
74	70
75	71	struct ReportLocation {
76		ReportLocationType type;
77		DataInfo global;
78		uptr heap_chunk_start;
79		uptr heap_chunk_size;
80		uptr external_tag;
81		int tid;
82		int fd;
83		bool suppressable;
84		ReportStack *stack;
85
86		static ReportLocation *New(ReportLocationType type);
87		private:
88		explicit ReportLocation(ReportLocationType type);
	72	ReportLocationType type = ReportLocationGlobal;
	73	DataInfo global = {};
	74	uptr heap_chunk_start = 0;
	75	uptr heap_chunk_size = 0;
	76	uptr external_tag = 0;
	77	Tid tid = kInvalidTid;
	78	int fd = 0;
	79	bool fd_closed = false;
	80	bool suppressable = false;
	81	ReportStack *stack = nullptr;
89	82	};
90	83
91	84	struct ReportThread {
92		int id;
	85	Tid id;
93	86	tid_t os_id;
94	87	bool running;
95	88	ThreadType thread_type;
96	89	char *name;
97		u32 parent_tid;
	90	Tid parent_tid;
98	91	ReportStack *stack;
99	92	};
100	93
101	94	struct ReportMutex {
102		u64 id;
	95	int id;
103	96	uptr addr;
104		bool destroyed;
105	97	ReportStack *stack;
106	98	};
107	99
...	...	@@ -114,9 +106,10 @@ class ReportDesc {
114	106	Vector<ReportLocation*> locs;
115	107	Vector<ReportMutex*> mutexes;
116	108	Vector<ReportThread*> threads;
117		Vector<int> unique_tids;
	109	Vector<Tid> unique_tids;
118	110	ReportStack *sleep;
119	111	int count;
	112	int signum = 0;
120	113
121	114	ReportDesc();
122	115	~ReportDesc();

lib/tsan/tsan_rtl.cpp+657-693

...	...	@@ -16,6 +16,7 @@
16	16	#include "sanitizer_common/sanitizer_atomic.h"
17	17	#include "sanitizer_common/sanitizer_common.h"
18	18	#include "sanitizer_common/sanitizer_file.h"
	19	#include "sanitizer_common/sanitizer_interface_internal.h"
19	20	#include "sanitizer_common/sanitizer_libc.h"
20	21	#include "sanitizer_common/sanitizer_placement_new.h"
21	22	#include "sanitizer_common/sanitizer_stackdepot.h"
...	...	@@ -28,29 +29,28 @@
28	29	#include "tsan_symbolize.h"
29	30	#include "ubsan/ubsan_init.h"
30	31
31		#ifdef __SSE3__
32		// <emmintrin.h> transitively includes <stdlib.h>,
33		// and it's prohibited to include std headers into tsan runtime.
34		// So we do this dirty trick.
35		#define _MM_MALLOC_H_INCLUDED
36		#define __MM_MALLOC_H
37		#include <emmintrin.h>
38		typedef __m128i m128;
39		#endif
40
41	32	volatile int __tsan_resumed = 0;
42	33
43	34	extern "C" void __tsan_resume() {
44	35	__tsan_resumed = 1;
45	36	}
46	37
	38	SANITIZER_WEAK_DEFAULT_IMPL
	39	void __tsan_test_only_on_fork() {}
	40
47	41	namespace __tsan {
48	42
49		#if !SANITIZER_GO && !SANITIZER_MAC
	43	#if !SANITIZER_GO
	44	void (*on_initialize)(void);
	45	int (*on_finalize)(int);
	46	#endif
	47
	48	#if !SANITIZER_GO && !SANITIZER_APPLE
50	49	__attribute__((tls_model("initial-exec")))
51		THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64);
	50	THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(
	51	SANITIZER_CACHE_LINE_SIZE);
52	52	#endif
53		static char ctx_placeholder[sizeof(Context)] ALIGNED(64);
	53	static char ctx_placeholder[sizeof(Context)] ALIGNED(SANITIZER_CACHE_LINE_SIZE);
54	54	Context *ctx;
55	55
56	56	// Can be overriden by a front-end.
...	...	@@ -58,113 +58,404 @@ Context *ctx;
58	58	bool OnFinalize(bool failed);
59	59	void OnInitialize();
60	60	#else
61		#include <dlfcn.h>
62	61	SANITIZER_WEAK_CXX_DEFAULT_IMPL
63	62	bool OnFinalize(bool failed) {
64		#if !SANITIZER_GO
65		if (auto *ptr = dlsym(RTLD_DEFAULT, "__tsan_on_finalize"))
66		return reinterpret_cast<decltype(&__tsan_on_finalize)>(ptr)(failed);
67		#endif
	63	# if !SANITIZER_GO
	64	if (on_finalize)
	65	return on_finalize(failed);
	66	# endif
68	67	return failed;
69	68	}
	69
70	70	SANITIZER_WEAK_CXX_DEFAULT_IMPL
71	71	void OnInitialize() {
72		#if !SANITIZER_GO
73		if (auto *ptr = dlsym(RTLD_DEFAULT, "__tsan_on_initialize")) {
74		return reinterpret_cast<decltype(&__tsan_on_initialize)>(ptr)();
75		}
	72	# if !SANITIZER_GO
	73	if (on_initialize)
	74	on_initialize();
	75	# endif
	76	}
76	77	#endif
	78
	79	static TracePart* TracePartAlloc(ThreadState* thr) {
	80	TracePart* part = nullptr;
	81	{
	82	Lock lock(&ctx->slot_mtx);
	83	uptr max_parts = Trace::kMinParts + flags()->history_size;
	84	Trace* trace = &thr->tctx->trace;
	85	if (trace->parts_allocated == max_parts \|\|
	86	ctx->trace_part_finished_excess) {
	87	part = ctx->trace_part_recycle.PopFront();
	88	DPrintf("#%d: TracePartAlloc: part=%p\n", thr->tid, part);
	89	if (part && part->trace) {
	90	Trace* trace1 = part->trace;
	91	Lock trace_lock(&trace1->mtx);
	92	part->trace = nullptr;
	93	TracePart* part1 = trace1->parts.PopFront();
	94	CHECK_EQ(part, part1);
	95	if (trace1->parts_allocated > trace1->parts.Size()) {
	96	ctx->trace_part_finished_excess +=
	97	trace1->parts_allocated - trace1->parts.Size();
	98	trace1->parts_allocated = trace1->parts.Size();
	99	}
	100	}
	101	}
	102	if (trace->parts_allocated < max_parts) {
	103	trace->parts_allocated++;
	104	if (ctx->trace_part_finished_excess)
	105	ctx->trace_part_finished_excess--;
	106	}
	107	if (!part)
	108	ctx->trace_part_total_allocated++;
	109	else if (ctx->trace_part_recycle_finished)
	110	ctx->trace_part_recycle_finished--;
	111	}
	112	if (!part)
	113	part = new (MmapOrDie(sizeof(*part), "TracePart")) TracePart();
	114	return part;
	115	}
	116
	117	static void TracePartFree(TracePart* part) SANITIZER_REQUIRES(ctx->slot_mtx) {
	118	DCHECK(part->trace);
	119	part->trace = nullptr;
	120	ctx->trace_part_recycle.PushFront(part);
	121	}
	122
	123	void TraceResetForTesting() {
	124	Lock lock(&ctx->slot_mtx);
	125	while (auto* part = ctx->trace_part_recycle.PopFront()) {
	126	if (auto trace = part->trace)
	127	CHECK_EQ(trace->parts.PopFront(), part);
	128	UnmapOrDie(part, sizeof(*part));
	129	}
	130	ctx->trace_part_total_allocated = 0;
	131	ctx->trace_part_recycle_finished = 0;
	132	ctx->trace_part_finished_excess = 0;
77	133	}
	134
	135	static void DoResetImpl(uptr epoch) {
	136	ThreadRegistryLock lock0(&ctx->thread_registry);
	137	Lock lock1(&ctx->slot_mtx);
	138	CHECK_EQ(ctx->global_epoch, epoch);
	139	ctx->global_epoch++;
	140	CHECK(!ctx->resetting);
	141	ctx->resetting = true;
	142	for (u32 i = ctx->thread_registry.NumThreadsLocked(); i--;) {
	143	ThreadContext* tctx = (ThreadContext*)ctx->thread_registry.GetThreadLocked(
	144	static_cast<Tid>(i));
	145	// Potentially we could purge all ThreadStatusDead threads from the
	146	// registry. Since we reset all shadow, they can't race with anything
	147	// anymore. However, their tid's can still be stored in some aux places
	148	// (e.g. tid of thread that created something).
	149	auto trace = &tctx->trace;
	150	Lock lock(&trace->mtx);
	151	bool attached = tctx->thr && tctx->thr->slot;
	152	auto parts = &trace->parts;
	153	bool local = false;
	154	while (!parts->Empty()) {
	155	auto part = parts->Front();
	156	local = local \|\| part == trace->local_head;
	157	if (local)
	158	CHECK(!ctx->trace_part_recycle.Queued(part));
	159	else
	160	ctx->trace_part_recycle.Remove(part);
	161	if (attached && parts->Size() == 1) {
	162	// The thread is running and this is the last/current part.
	163	// Set the trace position to the end of the current part
	164	// to force the thread to call SwitchTracePart and re-attach
	165	// to a new slot and allocate a new trace part.
	166	// Note: the thread is concurrently modifying the position as well,
	167	// so this is only best-effort. The thread can only modify position
	168	// within this part, because switching parts is protected by
	169	// slot/trace mutexes that we hold here.
	170	atomic_store_relaxed(
	171	&tctx->thr->trace_pos,
	172	reinterpret_cast<uptr>(&part->events[TracePart::kSize]));
	173	break;
	174	}
	175	parts->Remove(part);
	176	TracePartFree(part);
	177	}
	178	CHECK_LE(parts->Size(), 1);
	179	trace->local_head = parts->Front();
	180	if (tctx->thr && !tctx->thr->slot) {
	181	atomic_store_relaxed(&tctx->thr->trace_pos, 0);
	182	tctx->thr->trace_prev_pc = 0;
	183	}
	184	if (trace->parts_allocated > trace->parts.Size()) {
	185	ctx->trace_part_finished_excess +=
	186	trace->parts_allocated - trace->parts.Size();
	187	trace->parts_allocated = trace->parts.Size();
	188	}
	189	}
	190	while (ctx->slot_queue.PopFront()) {
	191	}
	192	for (auto& slot : ctx->slots) {
	193	slot.SetEpoch(kEpochZero);
	194	slot.journal.Reset();
	195	slot.thr = nullptr;
	196	ctx->slot_queue.PushBack(&slot);
	197	}
	198
	199	DPrintf("Resetting shadow...\n");
	200	auto shadow_begin = ShadowBeg();
	201	auto shadow_end = ShadowEnd();
	202	#if SANITIZER_GO
	203	CHECK_NE(0, ctx->mapped_shadow_begin);
	204	shadow_begin = ctx->mapped_shadow_begin;
	205	shadow_end = ctx->mapped_shadow_end;
	206	VPrintf(2, "shadow_begin-shadow_end: (0x%zx-0x%zx)\n",
	207	shadow_begin, shadow_end);
78	208	#endif
79	209
80		static ALIGNED(64) char thread_registry_placeholder[sizeof(ThreadRegistry)];
81
82		static ThreadContextBase *CreateThreadContext(u32 tid) {
83		// Map thread trace when context is created.
84		char name[50];
85		internal_snprintf(name, sizeof(name), "trace %u", tid);
86		MapThreadTrace(GetThreadTrace(tid), TraceSize() * sizeof(Event), name);
87		const uptr hdr = GetThreadTraceHeader(tid);
88		internal_snprintf(name, sizeof(name), "trace header %u", tid);
89		MapThreadTrace(hdr, sizeof(Trace), name);
90		new((void*)hdr) Trace();
91		// We are going to use only a small part of the trace with the default
92		// value of history_size. However, the constructor writes to the whole trace.
93		// Release the unused part.
94		uptr hdr_end = hdr + sizeof(Trace);
95		hdr_end -= sizeof(TraceHeader) * (kTraceParts - TraceParts());
96		hdr_end = RoundUp(hdr_end, GetPageSizeCached());
97		if (hdr_end < hdr + sizeof(Trace)) {
98		ReleaseMemoryPagesToOS(hdr_end, hdr + sizeof(Trace));
99		uptr unused = hdr + sizeof(Trace) - hdr_end;
100		if (hdr_end != (uptr)MmapFixedNoAccess(hdr_end, unused)) {
101		Report("ThreadSanitizer: failed to mprotect(%p, %p)\n",
102		hdr_end, unused);
103		CHECK("unable to mprotect" && 0);
	210	#if SANITIZER_WINDOWS
	211	auto resetFailed =
	212	!ZeroMmapFixedRegion(shadow_begin, shadow_end - shadow_begin);
	213	#else
	214	auto resetFailed =
	215	!MmapFixedSuperNoReserve(shadow_begin, shadow_end-shadow_begin, "shadow");
	216	# if !SANITIZER_GO
	217	DontDumpShadow(shadow_begin, shadow_end - shadow_begin);
	218	# endif
	219	#endif
	220	if (resetFailed) {
	221	Printf("failed to reset shadow memory\n");
	222	Die();
	223	}
	224	DPrintf("Resetting meta shadow...\n");
	225	ctx->metamap.ResetClocks();
	226	StoreShadow(&ctx->last_spurious_race, Shadow::kEmpty);
	227	ctx->resetting = false;
	228	}
	229
	230	// Clang does not understand locking all slots in the loop:
	231	// error: expecting mutex 'slot.mtx' to be held at start of each loop
	232	void DoReset(ThreadState* thr, uptr epoch) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	233	for (auto& slot : ctx->slots) {
	234	slot.mtx.Lock();
	235	if (UNLIKELY(epoch == 0))
	236	epoch = ctx->global_epoch;
	237	if (UNLIKELY(epoch != ctx->global_epoch)) {
	238	// Epoch can't change once we've locked the first slot.
	239	CHECK_EQ(slot.sid, 0);
	240	slot.mtx.Unlock();
	241	return;
	242	}
	243	}
	244	DPrintf("#%d: DoReset epoch=%lu\n", thr ? thr->tid : -1, epoch);
	245	DoResetImpl(epoch);
	246	for (auto& slot : ctx->slots) slot.mtx.Unlock();
	247	}
	248
	249	void FlushShadowMemory() { DoReset(nullptr, 0); }
	250
	251	static TidSlot* FindSlotAndLock(ThreadState* thr)
	252	SANITIZER_ACQUIRE(thr->slot->mtx) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	253	CHECK(!thr->slot);
	254	TidSlot* slot = nullptr;
	255	for (;;) {
	256	uptr epoch;
	257	{
	258	Lock lock(&ctx->slot_mtx);
	259	epoch = ctx->global_epoch;
	260	if (slot) {
	261	// This is an exhausted slot from the previous iteration.
	262	if (ctx->slot_queue.Queued(slot))
	263	ctx->slot_queue.Remove(slot);
	264	thr->slot_locked = false;
	265	slot->mtx.Unlock();
	266	}
	267	for (;;) {
	268	slot = ctx->slot_queue.PopFront();
	269	if (!slot)
	270	break;
	271	if (slot->epoch() != kEpochLast) {
	272	ctx->slot_queue.PushBack(slot);
	273	break;
	274	}
	275	}
	276	}
	277	if (!slot) {
	278	DoReset(thr, epoch);
	279	continue;
104	280	}
	281	slot->mtx.Lock();
	282	CHECK(!thr->slot_locked);
	283	thr->slot_locked = true;
	284	if (slot->thr) {
	285	DPrintf("#%d: preempting sid=%d tid=%d\n", thr->tid, (u32)slot->sid,
	286	slot->thr->tid);
	287	slot->SetEpoch(slot->thr->fast_state.epoch());
	288	slot->thr = nullptr;
	289	}
	290	if (slot->epoch() != kEpochLast)
	291	return slot;
105	292	}
106		void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadContext));
107		return new(mem) ThreadContext(tid);
108	293	}
109	294
	295	void SlotAttachAndLock(ThreadState* thr) {
	296	TidSlot* slot = FindSlotAndLock(thr);
	297	DPrintf("#%d: SlotAttach: slot=%u\n", thr->tid, static_cast<int>(slot->sid));
	298	CHECK(!slot->thr);
	299	CHECK(!thr->slot);
	300	slot->thr = thr;
	301	thr->slot = slot;
	302	Epoch epoch = EpochInc(slot->epoch());
	303	CHECK(!EpochOverflow(epoch));
	304	slot->SetEpoch(epoch);
	305	thr->fast_state.SetSid(slot->sid);
	306	thr->fast_state.SetEpoch(epoch);
	307	if (thr->slot_epoch != ctx->global_epoch) {
	308	thr->slot_epoch = ctx->global_epoch;
	309	thr->clock.Reset();
110	310	#if !SANITIZER_GO
111		static const u32 kThreadQuarantineSize = 16;
112		#else
113		static const u32 kThreadQuarantineSize = 64;
	311	thr->last_sleep_stack_id = kInvalidStackID;
	312	thr->last_sleep_clock.Reset();
	313	#endif
	314	}
	315	thr->clock.Set(slot->sid, epoch);
	316	slot->journal.PushBack({thr->tid, epoch});
	317	}
	318
	319	static void SlotDetachImpl(ThreadState* thr, bool exiting) {
	320	TidSlot* slot = thr->slot;
	321	thr->slot = nullptr;
	322	if (thr != slot->thr) {
	323	slot = nullptr; // we don't own the slot anymore
	324	if (thr->slot_epoch != ctx->global_epoch) {
	325	TracePart* part = nullptr;
	326	auto* trace = &thr->tctx->trace;
	327	{
	328	Lock l(&trace->mtx);
	329	auto* parts = &trace->parts;
	330	// The trace can be completely empty in an unlikely event
	331	// the thread is preempted right after it acquired the slot
	332	// in ThreadStart and did not trace any events yet.
	333	CHECK_LE(parts->Size(), 1);
	334	part = parts->PopFront();
	335	thr->tctx->trace.local_head = nullptr;
	336	atomic_store_relaxed(&thr->trace_pos, 0);
	337	thr->trace_prev_pc = 0;
	338	}
	339	if (part) {
	340	Lock l(&ctx->slot_mtx);
	341	TracePartFree(part);
	342	}
	343	}
	344	return;
	345	}
	346	CHECK(exiting \|\| thr->fast_state.epoch() == kEpochLast);
	347	slot->SetEpoch(thr->fast_state.epoch());
	348	slot->thr = nullptr;
	349	}
	350
	351	void SlotDetach(ThreadState* thr) {
	352	Lock lock(&thr->slot->mtx);
	353	SlotDetachImpl(thr, true);
	354	}
	355
	356	void SlotLock(ThreadState* thr) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	357	DCHECK(!thr->slot_locked);
	358	#if SANITIZER_DEBUG
	359	// Check these mutexes are not locked.
	360	// We can call DoReset from SlotAttachAndLock, which will lock
	361	// these mutexes, but it happens only every once in a while.
	362	{ ThreadRegistryLock lock(&ctx->thread_registry); }
	363	{ Lock lock(&ctx->slot_mtx); }
114	364	#endif
	365	TidSlot* slot = thr->slot;
	366	slot->mtx.Lock();
	367	thr->slot_locked = true;
	368	if (LIKELY(thr == slot->thr && thr->fast_state.epoch() != kEpochLast))
	369	return;
	370	SlotDetachImpl(thr, false);
	371	thr->slot_locked = false;
	372	slot->mtx.Unlock();
	373	SlotAttachAndLock(thr);
	374	}
	375
	376	void SlotUnlock(ThreadState* thr) {
	377	DCHECK(thr->slot_locked);
	378	thr->slot_locked = false;
	379	thr->slot->mtx.Unlock();
	380	}
115	381
116	382	Context::Context()
117	383	: initialized(),
118	384	report_mtx(MutexTypeReport),
119	385	nreported(),
120		nmissed_expected(),
121		thread_registry(new (thread_registry_placeholder) ThreadRegistry(
122		CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse)),
	386	thread_registry([](Tid tid) -> ThreadContextBase* {
	387	return new (Alloc(sizeof(ThreadContext))) ThreadContext(tid);
	388	}),
123	389	racy_mtx(MutexTypeRacy),
124	390	racy_stacks(),
125		racy_addresses(),
126	391	fired_suppressions_mtx(MutexTypeFired),
127		clock_alloc(LINKER_INITIALIZED, "clock allocator") {
	392	slot_mtx(MutexTypeSlots),
	393	resetting() {
128	394	fired_suppressions.reserve(8);
	395	for (uptr i = 0; i < ARRAY_SIZE(slots); i++) {
	396	TidSlot* slot = &slots[i];
	397	slot->sid = static_cast<Sid>(i);
	398	slot_queue.PushBack(slot);
	399	}
	400	global_epoch = 1;
129	401	}
130	402
	403	TidSlot::TidSlot() : mtx(MutexTypeSlot) {}
	404
131	405	// The objects are allocated in TLS, so one may rely on zero-initialization.
132		ThreadState::ThreadState(Context *ctx, u32 tid, int unique_id, u64 epoch,
133		unsigned reuse_count, uptr stk_addr, uptr stk_size,
134		uptr tls_addr, uptr tls_size)
135		: fast_state(tid, epoch)
136		// Do not touch these, rely on zero initialization,
137		// they may be accessed before the ctor.
138		// , ignore_reads_and_writes()
139		// , ignore_interceptors()
140		,
141		clock(tid, reuse_count)
142		#if !SANITIZER_GO
143		,
144		jmp_bufs()
145		#endif
146		,
147		tid(tid),
148		unique_id(unique_id),
149		stk_addr(stk_addr),
150		stk_size(stk_size),
151		tls_addr(tls_addr),
152		tls_size(tls_size)
	406	ThreadState::ThreadState(Tid tid)
	407	// Do not touch these, rely on zero initialization,
	408	// they may be accessed before the ctor.
	409	// ignore_reads_and_writes()
	410	// ignore_interceptors()
	411	: tid(tid) {
	412	CHECK_EQ(reinterpret_cast<uptr>(this) % SANITIZER_CACHE_LINE_SIZE, 0);
153	413	#if !SANITIZER_GO
154		,
155		last_sleep_clock(tid)
	414	// C/C++ uses fixed size shadow stack.
	415	const int kInitStackSize = kShadowStackSize;
	416	shadow_stack = static_cast<uptr*>(
	417	MmapNoReserveOrDie(kInitStackSize * sizeof(uptr), "shadow stack"));
	418	SetShadowRegionHugePageMode(reinterpret_cast<uptr>(shadow_stack),
	419	kInitStackSize * sizeof(uptr));
	420	#else
	421	// Go uses malloc-allocated shadow stack with dynamic size.
	422	const int kInitStackSize = 8;
	423	shadow_stack = static_cast<uptr>(Alloc(kInitStackSize sizeof(uptr)));
156	424	#endif
157		{
	425	shadow_stack_pos = shadow_stack;
	426	shadow_stack_end = shadow_stack + kInitStackSize;
158	427	}
159	428
160	429	#if !SANITIZER_GO
161		static void MemoryProfiler(Context *ctx, fd_t fd, int i) {
162		uptr n_threads;
163		uptr n_running_threads;
164		ctx->thread_registry->GetNumberOfThreads(&n_threads, &n_running_threads);
	430	void MemoryProfiler(u64 uptime) {
	431	if (ctx->memprof_fd == kInvalidFd)
	432	return;
165	433	InternalMmapVector<char> buf(4096);
166		WriteMemoryProfile(buf.data(), buf.size(), n_threads, n_running_threads);
167		WriteToFile(fd, buf.data(), internal_strlen(buf.data()));
	434	WriteMemoryProfile(buf.data(), buf.size(), uptime);
	435	WriteToFile(ctx->memprof_fd, buf.data(), internal_strlen(buf.data()));
	436	}
	437
	438	static bool InitializeMemoryProfiler() {
	439	ctx->memprof_fd = kInvalidFd;
	440	const char *fname = flags()->profile_memory;
	441	if (!fname \|\| !fname[0])
	442	return false;
	443	if (internal_strcmp(fname, "stdout") == 0) {
	444	ctx->memprof_fd = 1;
	445	} else if (internal_strcmp(fname, "stderr") == 0) {
	446	ctx->memprof_fd = 2;
	447	} else {
	448	InternalScopedString filename;
	449	filename.append("%s.%d", fname, (int)internal_getpid());
	450	ctx->memprof_fd = OpenFile(filename.data(), WrOnly);
	451	if (ctx->memprof_fd == kInvalidFd) {
	452	Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
	453	filename.data());
	454	return false;
	455	}
	456	}
	457	MemoryProfiler(0);
	458	return true;
168	459	}
169	460
170	461	static void BackgroundThread(void arg) {
...	...	@@ -172,64 +463,43 @@ static void BackgroundThread(void arg) {
172	463	// We don't use ScopedIgnoreInterceptors, because we want ignores to be
173	464	// enabled even when the thread function exits (e.g. during pthread thread
174	465	// shutdown code).
175		cur_thread_init();
176		cur_thread()->ignore_interceptors++;
	466	cur_thread_init()->ignore_interceptors++;
177	467	const u64 kMs2Ns = 1000 * 1000;
	468	const u64 start = NanoTime();
178	469
179		fd_t mprof_fd = kInvalidFd;
180		if (flags()->profile_memory && flags()->profile_memory[0]) {
181		if (internal_strcmp(flags()->profile_memory, "stdout") == 0) {
182		mprof_fd = 1;
183		} else if (internal_strcmp(flags()->profile_memory, "stderr") == 0) {
184		mprof_fd = 2;
185		} else {
186		InternalScopedString filename;
187		filename.append("%s.%d", flags()->profile_memory, (int)internal_getpid());
188		fd_t fd = OpenFile(filename.data(), WrOnly);
189		if (fd == kInvalidFd) {
190		Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
191		filename.data());
192		} else {
193		mprof_fd = fd;
194		}
195		}
196		}
197
198		u64 last_flush = NanoTime();
	470	u64 last_flush = start;
199	471	uptr last_rss = 0;
200		for (int i = 0;
201		atomic_load(&ctx->stop_background_thread, memory_order_relaxed) == 0;
202		i++) {
	472	while (!atomic_load_relaxed(&ctx->stop_background_thread)) {
203	473	SleepForMillis(100);
204	474	u64 now = NanoTime();
205	475
206	476	// Flush memory if requested.
207	477	if (flags()->flush_memory_ms > 0) {
208	478	if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) {
209		VPrintf(1, "ThreadSanitizer: periodic memory flush\n");
	479	VReport(1, "ThreadSanitizer: periodic memory flush\n");
210	480	FlushShadowMemory();
211		last_flush = NanoTime();
	481	now = last_flush = NanoTime();
212	482	}
213	483	}
214		// GetRSS can be expensive on huge programs, so don't do it every 100ms.
215	484	if (flags()->memory_limit_mb > 0) {
216	485	uptr rss = GetRSS();
217	486	uptr limit = uptr(flags()->memory_limit_mb) << 20;
218		VPrintf(1, "ThreadSanitizer: memory flush check"
219		" RSS=%llu LAST=%llu LIMIT=%llu\n",
	487	VReport(1,
	488	"ThreadSanitizer: memory flush check"
	489	" RSS=%llu LAST=%llu LIMIT=%llu\n",
220	490	(u64)rss >> 20, (u64)last_rss >> 20, (u64)limit >> 20);
221	491	if (2 * rss > limit + last_rss) {
222		VPrintf(1, "ThreadSanitizer: flushing memory due to RSS\n");
	492	VReport(1, "ThreadSanitizer: flushing memory due to RSS\n");
223	493	FlushShadowMemory();
224	494	rss = GetRSS();
225		VPrintf(1, "ThreadSanitizer: memory flushed RSS=%llu\n", (u64)rss>>20);
	495	now = NanoTime();
	496	VReport(1, "ThreadSanitizer: memory flushed RSS=%llu\n",
	497	(u64)rss >> 20);
226	498	}
227	499	last_rss = rss;
228	500	}
229	501
230		// Write memory profile if requested.
231		if (mprof_fd != kInvalidFd)
232		MemoryProfiler(ctx, mprof_fd, i);
	502	MemoryProfiler(now - start);
233	503
234	504	// Flush symbolizer cache if requested.
235	505	if (flags()->flush_symbolizer_ms > 0) {
...	...	@@ -260,31 +530,96 @@ static void StopBackgroundThread() {
260	530	#endif
261	531
262	532	void DontNeedShadowFor(uptr addr, uptr size) {
263		ReleaseMemoryPagesToOS(MemToShadow(addr), MemToShadow(addr + size));
	533	ReleaseMemoryPagesToOS(reinterpret_cast<uptr>(MemToShadow(addr)),
	534	reinterpret_cast<uptr>(MemToShadow(addr + size)));
264	535	}
265	536
266	537	#if !SANITIZER_GO
	538	// We call UnmapShadow before the actual munmap, at that point we don't yet
	539	// know if the provided address/size are sane. We can't call UnmapShadow
	540	// after the actual munmap becuase at that point the memory range can
	541	// already be reused for something else, so we can't rely on the munmap
	542	// return value to understand is the values are sane.
	543	// While calling munmap with insane values (non-canonical address, negative
	544	// size, etc) is an error, the kernel won't crash. We must also try to not
	545	// crash as the failure mode is very confusing (paging fault inside of the
	546	// runtime on some derived shadow address).
	547	static bool IsValidMmapRange(uptr addr, uptr size) {
	548	if (size == 0)
	549	return true;
	550	if (static_cast<sptr>(size) < 0)
	551	return false;
	552	if (!IsAppMem(addr) \|\| !IsAppMem(addr + size - 1))
	553	return false;
	554	// Check that if the start of the region belongs to one of app ranges,
	555	// end of the region belongs to the same region.
	556	const uptr ranges[][2] = {
	557	{LoAppMemBeg(), LoAppMemEnd()},
	558	{MidAppMemBeg(), MidAppMemEnd()},
	559	{HiAppMemBeg(), HiAppMemEnd()},
	560	};
	561	for (auto range : ranges) {
	562	if (addr >= range[0] && addr < range[1])
	563	return addr + size <= range[1];
	564	}
	565	return false;
	566	}
	567
267	568	void UnmapShadow(ThreadState *thr, uptr addr, uptr size) {
268		if (size == 0) return;
	569	if (size == 0 \|\| !IsValidMmapRange(addr, size))
	570	return;
269	571	DontNeedShadowFor(addr, size);
270	572	ScopedGlobalProcessor sgp;
271		ctx->metamap.ResetRange(thr->proc(), addr, size);
	573	SlotLocker locker(thr, true);
	574	ctx->metamap.ResetRange(thr->proc(), addr, size, true);
272	575	}
273	576	#endif
274	577
275	578	void MapShadow(uptr addr, uptr size) {
	579	// Ensure thead registry lock held, so as to synchronize
	580	// with DoReset, which also access the mapped_shadow_* ctxt fields.
	581	ThreadRegistryLock lock0(&ctx->thread_registry);
	582	static bool data_mapped = false;
	583
	584	#if !SANITIZER_GO
276	585	// Global data is not 64K aligned, but there are no adjacent mappings,
277	586	// so we can get away with unaligned mapping.
278	587	// CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment
279	588	const uptr kPageSize = GetPageSizeCached();
280	589	uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize);
281	590	uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize);
282		if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin,
283		"shadow"))
	591	if (!MmapFixedNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow"))
284	592	Die();
	593	#else
	594	uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), (64 << 10));
	595	uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), (64 << 10));
	596	VPrintf(2, "MapShadow for (0x%zx-0x%zx), begin/end: (0x%zx-0x%zx)\n",
	597	addr, addr + size, shadow_begin, shadow_end);
	598
	599	if (!data_mapped) {
	600	// First call maps data+bss.
	601	if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow"))
	602	Die();
	603	} else {
	604	VPrintf(2, "ctx->mapped_shadow_{begin,end} = (0x%zx-0x%zx)\n",
	605	ctx->mapped_shadow_begin, ctx->mapped_shadow_end);
	606	// Second and subsequent calls map heap.
	607	if (shadow_end <= ctx->mapped_shadow_end)
	608	return;
	609	if (!ctx->mapped_shadow_begin \|\| ctx->mapped_shadow_begin > shadow_begin)
	610	ctx->mapped_shadow_begin = shadow_begin;
	611	if (shadow_begin < ctx->mapped_shadow_end)
	612	shadow_begin = ctx->mapped_shadow_end;
	613	VPrintf(2, "MapShadow begin/end = (0x%zx-0x%zx)\n",
	614	shadow_begin, shadow_end);
	615	if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin,
	616	"shadow"))
	617	Die();
	618	ctx->mapped_shadow_end = shadow_end;
	619	}
	620	#endif
285	621
286	622	// Meta shadow is 2:1, so tread carefully.
287		static bool data_mapped = false;
288	623	static uptr mapped_meta_end = 0;
289	624	uptr meta_begin = (uptr)MemToMeta(addr);
290	625	uptr meta_end = (uptr)MemToMeta(addr + size);
...	...	@@ -297,12 +632,11 @@ void MapShadow(uptr addr, uptr size) {
297	632	"meta shadow"))
298	633	Die();
299	634	} else {
300		// Mapping continous heap.
	635	// Mapping continuous heap.
301	636	// Windows wants 64K alignment.
302	637	meta_begin = RoundDownTo(meta_begin, 64 << 10);
303	638	meta_end = RoundUpTo(meta_end, 64 << 10);
304		if (meta_end <= mapped_meta_end)
305		return;
	639	CHECK_GT(meta_end, mapped_meta_end);
306	640	if (meta_begin < mapped_meta_end)
307	641	meta_begin = mapped_meta_end;
308	642	if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin,
...	...	@@ -310,56 +644,8 @@ void MapShadow(uptr addr, uptr size) {
310	644	Die();
311	645	mapped_meta_end = meta_end;
312	646	}
313		VPrintf(2, "mapped meta shadow for (%p-%p) at (%p-%p)\n",
314		addr, addr+size, meta_begin, meta_end);
315		}
316
317		void MapThreadTrace(uptr addr, uptr size, const char *name) {
318		DPrintf("#0: Mapping trace at %p-%p(0x%zx)\n", addr, addr + size, size);
319		CHECK_GE(addr, TraceMemBeg());
320		CHECK_LE(addr + size, TraceMemEnd());
321		CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment
322		if (!MmapFixedSuperNoReserve(addr, size, name)) {
323		Printf("FATAL: ThreadSanitizer can not mmap thread trace (%p/%p)\n",
324		addr, size);
325		Die();
326		}
327		}
328
329		static void CheckShadowMapping() {
330		uptr beg, end;
331		for (int i = 0; GetUserRegion(i, &beg, &end); i++) {
332		// Skip cases for empty regions (heap definition for architectures that
333		// do not use 64-bit allocator).
334		if (beg == end)
335		continue;
336		VPrintf(3, "checking shadow region %p-%p\n", beg, end);
337		uptr prev = 0;
338		for (uptr p0 = beg; p0 <= end; p0 += (end - beg) / 4) {
339		for (int x = -(int)kShadowCell; x <= (int)kShadowCell; x += kShadowCell) {
340		const uptr p = RoundDown(p0 + x, kShadowCell);
341		if (p < beg \|\| p >= end)
342		continue;
343		const uptr s = MemToShadow(p);
344		const uptr m = (uptr)MemToMeta(p);
345		VPrintf(3, " checking pointer %p: shadow=%p meta=%p\n", p, s, m);
346		CHECK(IsAppMem(p));
347		CHECK(IsShadowMem(s));
348		CHECK_EQ(p, ShadowToMem(s));
349		CHECK(IsMetaMem(m));
350		if (prev) {
351		// Ensure that shadow and meta mappings are linear within a single
352		// user range. Lots of code that processes memory ranges assumes it.
353		const uptr prev_s = MemToShadow(prev);
354		const uptr prev_m = (uptr)MemToMeta(prev);
355		CHECK_EQ(s - prev_s, (p - prev) * kShadowMultiplier);
356		CHECK_EQ((m - prev_m) / kMetaShadowSize,
357		(p - prev) / kMetaShadowCell);
358		}
359		prev = p;
360		}
361		}
362		}
	647	VPrintf(2, "mapped meta shadow for (0x%zx-0x%zx) at (0x%zx-0x%zx)\n", addr,
	648	addr + size, meta_begin, meta_end);
363	649	}
364	650
365	651	#if !SANITIZER_GO
...	...	@@ -380,15 +666,19 @@ void CheckUnwind() {
380	666	// since we are going to die soon.
381	667	ScopedIgnoreInterceptors ignore;
382	668	#if !SANITIZER_GO
383		cur_thread()->ignore_sync++;
384		cur_thread()->ignore_reads_and_writes++;
	669	ThreadState* thr = cur_thread();
	670	thr->nomalloc = false;
	671	thr->ignore_sync++;
	672	thr->ignore_reads_and_writes++;
	673	atomic_store_relaxed(&thr->in_signal_handler, 0);
385	674	#endif
386	675	PrintCurrentStackSlow(StackTrace::GetCurrentPc());
387	676	}
388	677
	678	bool is_initialized;
	679
389	680	void Initialize(ThreadState *thr) {
390	681	// Thread safe because done before all threads exist.
391		static bool is_initialized = false;
392	682	if (is_initialized)
393	683	return;
394	684	is_initialized = true;
...	...	@@ -409,9 +699,6 @@ void Initialize(ThreadState *thr) {
409	699	__tsan::InitializePlatformEarly();
410	700
411	701	#if !SANITIZER_GO
412		// Re-exec ourselves if we need to set additional env or command line args.
413		MaybeReexec();
414
415	702	InitializeAllocator();
416	703	ReplaceSystemMalloc();
417	704	#endif
...	...	@@ -420,7 +707,6 @@ void Initialize(ThreadState *thr) {
420	707	Processor *proc = ProcCreate();
421	708	ProcWire(proc, thr);
422	709	InitializeInterceptors();
423		CheckShadowMapping();
424	710	InitializePlatform();
425	711	InitializeDynamicAnnotations();
426	712	#if !SANITIZER_GO
...	...	@@ -436,21 +722,23 @@ void Initialize(ThreadState *thr) {
436	722	Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
437	723	#endif
438	724
439		VPrintf(1, "*** Running under ThreadSanitizer v2 (pid %d) ***\n",
	725	VPrintf(1, "*** Running under ThreadSanitizer v3 (pid %d) ***\n",
440	726	(int)internal_getpid());
441	727
442	728	// Initialize thread 0.
443		int tid = ThreadCreate(thr, 0, 0, true);
444		CHECK_EQ(tid, 0);
	729	Tid tid = ThreadCreate(nullptr, 0, 0, true);
	730	CHECK_EQ(tid, kMainTid);
445	731	ThreadStart(thr, tid, GetTid(), ThreadType::Regular);
446	732	#if TSAN_CONTAINS_UBSAN
447	733	__ubsan::InitAsPlugin();
448	734	#endif
449		ctx->initialized = true;
450	735
451	736	#if !SANITIZER_GO
452	737	Symbolizer::LateInitialize();
	738	if (InitializeMemoryProfiler() \|\| flags()->force_background_thread)
	739	MaybeSpawnBackgroundThread();
453	740	#endif
	741	ctx->initialized = true;
454	742
455	743	if (flags()->stop_on_start) {
456	744	Printf("ThreadSanitizer is suspended at startup (pid %d)."
...	...	@@ -476,20 +764,21 @@ void MaybeSpawnBackgroundThread() {
476	764	#endif
477	765	}
478	766
479
480	767	int Finalize(ThreadState *thr) {
481	768	bool failed = false;
482	769
	770	#if !SANITIZER_GO
483	771	if (common_flags()->print_module_map == 1)
484	772	DumpProcessMap();
	773	#endif
485	774
486	775	if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1)
487		SleepForMillis(flags()->atexit_sleep_ms);
	776	internal_usleep(u64(flags()->atexit_sleep_ms) * 1000);
488	777
489		// Wait for pending reports.
490		ctx->report_mtx.Lock();
491		{ ScopedErrorReportLock l; }
492		ctx->report_mtx.Unlock();
	778	{
	779	// Wait for pending reports.
	780	ScopedErrorReportLock lock;
	781	}
493	782
494	783	#if !SANITIZER_GO
495	784	if (Verbosity()) AllocatorPrintStats();
...	...	@@ -506,18 +795,8 @@ int Finalize(ThreadState *thr) {
506	795	#endif
507	796	}
508	797
509		if (ctx->nmissed_expected) {
510		failed = true;
511		Printf("ThreadSanitizer: missed %d expected races\n",
512		ctx->nmissed_expected);
513		}
514
515	798	if (common_flags()->print_suppressions)
516	799	PrintMatchedSuppressions();
517		#if !SANITIZER_GO
518		if (flags()->print_benign)
519		PrintMatchedBenignRaces();
520		#endif
521	800
522	801	failed = OnFinalize(failed);
523	802
...	...	@@ -525,10 +804,16 @@ int Finalize(ThreadState *thr) {
525	804	}
526	805
527	806	#if !SANITIZER_GO
528		void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
529		ctx->thread_registry->Lock();
530		ctx->report_mtx.Lock();
	807	void ForkBefore(ThreadState* thr, uptr pc) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	808	GlobalProcessorLock();
	809	// Detaching from the slot makes OnUserFree skip writing to the shadow.
	810	// The slot will be locked so any attempts to use it will deadlock anyway.
	811	SlotDetach(thr);
	812	for (auto& slot : ctx->slots) slot.mtx.Lock();
	813	ctx->thread_registry.Lock();
	814	ctx->slot_mtx.Lock();
531	815	ScopedErrorReportLock::Lock();
	816	AllocatorLock();
532	817	// Suppress all reports in the pthread_atfork callbacks.
533	818	// Reports will deadlock on the report_mtx.
534	819	// We could ignore sync operations as well,
...	...	@@ -537,36 +822,48 @@ void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
537	822	thr->suppress_reports++;
538	823	// On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and
539	824	// we'll assert in CheckNoLocks() unless we ignore interceptors.
	825	// On OS X libSystem_atfork_prepare/parent/child callbacks are called
	826	// after/before our callbacks and they call free.
540	827	thr->ignore_interceptors++;
	828	// Disables memory write in OnUserAlloc/Free.
	829	thr->ignore_reads_and_writes++;
	830
	831	__tsan_test_only_on_fork();
541	832	}
542	833
543		void ForkParentAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
	834	static void ForkAfter(ThreadState* thr) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
544	835	thr->suppress_reports--; // Enabled in ForkBefore.
545	836	thr->ignore_interceptors--;
	837	thr->ignore_reads_and_writes--;
	838	AllocatorUnlock();
546	839	ScopedErrorReportLock::Unlock();
547		ctx->report_mtx.Unlock();
548		ctx->thread_registry->Unlock();
	840	ctx->slot_mtx.Unlock();
	841	ctx->thread_registry.Unlock();
	842	for (auto& slot : ctx->slots) slot.mtx.Unlock();
	843	SlotAttachAndLock(thr);
	844	SlotUnlock(thr);
	845	GlobalProcessorUnlock();
549	846	}
550	847
551		void ForkChildAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
552		thr->suppress_reports--; // Enabled in ForkBefore.
553		thr->ignore_interceptors--;
554		ScopedErrorReportLock::Unlock();
555		ctx->report_mtx.Unlock();
556		ctx->thread_registry->Unlock();
	848	void ForkParentAfter(ThreadState* thr, uptr pc) { ForkAfter(thr); }
557	849
558		uptr nthread = 0;
559		ctx->thread_registry->GetNumberOfThreads(0, 0, &nthread /* alive threads */);
560		VPrintf(1, "ThreadSanitizer: forked new process with pid %d,"
561		" parent had %d threads\n", (int)internal_getpid(), (int)nthread);
	850	void ForkChildAfter(ThreadState* thr, uptr pc, bool start_thread) {
	851	ForkAfter(thr);
	852	u32 nthread = ctx->thread_registry.OnFork(thr->tid);
	853	VPrintf(1,
	854	"ThreadSanitizer: forked new process with pid %d,"
	855	" parent had %d threads\n",
	856	(int)internal_getpid(), (int)nthread);
562	857	if (nthread == 1) {
563		StartBackgroundThread();
	858	if (start_thread)
	859	StartBackgroundThread();
564	860	} else {
565	861	// We've just forked a multi-threaded process. We cannot reasonably function
566	862	// after that (some mutexes may be locked before fork). So just enable
567	863	// ignores for everything in the hope that we will exec soon.
568	864	ctx->after_multithreaded_fork = true;
569	865	thr->ignore_interceptors++;
	866	thr->suppress_reports++;
570	867	ThreadIgnoreBegin(thr, pc);
571	868	ThreadIgnoreSyncBegin(thr, pc);
572	869	}
...	...	@@ -578,19 +875,20 @@ NOINLINE
578	875	void GrowShadowStack(ThreadState *thr) {
579	876	const int sz = thr->shadow_stack_end - thr->shadow_stack;
580	877	const int newsz = 2 * sz;
581		uptr newstack = (uptr)internal_alloc(MBlockShadowStack,
582		newsz * sizeof(uptr));
	878	auto newstack = (uptr )Alloc(newsz * sizeof(uptr));
583	879	internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr));
584		internal_free(thr->shadow_stack);
	880	Free(thr->shadow_stack);
585	881	thr->shadow_stack = newstack;
586	882	thr->shadow_stack_pos = newstack + sz;
587	883	thr->shadow_stack_end = newstack + newsz;
588	884	}
589	885	#endif
590	886
591		u32 CurrentStackId(ThreadState *thr, uptr pc) {
	887	StackID CurrentStackId(ThreadState *thr, uptr pc) {
	888	#if !SANITIZER_GO
592	889	if (!thr->is_inited) // May happen during bootstrap.
593		return 0;
	890	return kInvalidStackID;
	891	#endif
594	892	if (pc != 0) {
595	893	#if !SANITIZER_GO
596	894	DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
...	...	@@ -601,486 +899,149 @@ u32 CurrentStackId(ThreadState *thr, uptr pc) {
601	899	thr->shadow_stack_pos[0] = pc;
602	900	thr->shadow_stack_pos++;
603	901	}
604		u32 id = StackDepotPut(
	902	StackID id = StackDepotPut(
605	903	StackTrace(thr->shadow_stack, thr->shadow_stack_pos - thr->shadow_stack));
606	904	if (pc != 0)
607	905	thr->shadow_stack_pos--;
608	906	return id;
609	907	}
610	908
611		void TraceSwitch(ThreadState *thr) {
612		#if !SANITIZER_GO
613		if (ctx->after_multithreaded_fork)
614		return;
615		#endif
616		thr->nomalloc++;
617		Trace *thr_trace = ThreadTrace(thr->tid);
618		Lock l(&thr_trace->mtx);
619		unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % TraceParts();
620		TraceHeader *hdr = &thr_trace->headers[trace];
621		hdr->epoch0 = thr->fast_state.epoch();
622		ObtainCurrentStack(thr, 0, &hdr->stack0);
623		hdr->mset0 = thr->mset;
624		thr->nomalloc--;
625		}
626
627		Trace *ThreadTrace(int tid) {
628		return (Trace*)GetThreadTraceHeader(tid);
629		}
630
631		uptr TraceTopPC(ThreadState *thr) {
632		Event events = (Event)GetThreadTrace(thr->tid);
633		uptr pc = events[thr->fast_state.GetTracePos()];
634		return pc;
635		}
636
637		uptr TraceSize() {
638		return (uptr)(1ull << (kTracePartSizeBits + flags()->history_size + 1));
639		}
640
641		uptr TraceParts() {
642		return TraceSize() / kTracePartSize;
643		}
644
645		#if !SANITIZER_GO
646		extern "C" void __tsan_trace_switch() {
647		TraceSwitch(cur_thread());
648		}
649
650		extern "C" void __tsan_report_race() {
651		ReportRace(cur_thread());
652		}
653		#endif
654
655		ALWAYS_INLINE
656		Shadow LoadShadow(u64 *p) {
657		u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed);
658		return Shadow(raw);
659		}
660
661		ALWAYS_INLINE
662		void StoreShadow(u64 *sp, u64 s) {
663		atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed);
664		}
665
666		ALWAYS_INLINE
667		void StoreIfNotYetStored(u64 sp, u64 s) {
668		StoreShadow(sp, *s);
669		*s = 0;
670		}
671
672		ALWAYS_INLINE
673		void HandleRace(ThreadState thr, u64 shadow_mem,
674		Shadow cur, Shadow old) {
675		thr->racy_state[0] = cur.raw();
676		thr->racy_state[1] = old.raw();
677		thr->racy_shadow_addr = shadow_mem;
678		#if !SANITIZER_GO
679		HACKY_CALL(__tsan_report_race);
680		#else
681		ReportRace(thr);
682		#endif
683		}
684
685		static inline bool HappensBefore(Shadow old, ThreadState *thr) {
686		return thr->clock.get(old.TidWithIgnore()) >= old.epoch();
687		}
688
689		ALWAYS_INLINE
690		void MemoryAccessImpl1(ThreadState *thr, uptr addr,
691		int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
692		u64 *shadow_mem, Shadow cur) {
693
694		// This potentially can live in an MMX/SSE scratch register.
695		// The required intrinsics are:
696		// __m128i _mm_move_epi64(__m128i*);
697		// _mm_storel_epi64(u64*, __m128i);
698		u64 store_word = cur.raw();
699		bool stored = false;
700
701		// scan all the shadow values and dispatch to 4 categories:
702		// same, replace, candidate and race (see comments below).
703		// we consider only 3 cases regarding access sizes:
704		// equal, intersect and not intersect. initially I considered
705		// larger and smaller as well, it allowed to replace some
706		// 'candidates' with 'same' or 'replace', but I think
707		// it's just not worth it (performance- and complexity-wise).
708
709		Shadow old(0);
710
711		// It release mode we manually unroll the loop,
712		// because empirically gcc generates better code this way.
713		// However, we can't afford unrolling in debug mode, because the function
714		// consumes almost 4K of stack. Gtest gives only 4K of stack to death test
715		// threads, which is not enough for the unrolled loop.
716		#if SANITIZER_DEBUG
717		for (int idx = 0; idx < 4; idx++) {
718		#include "tsan_update_shadow_word_inl.h"
719		}
720		#else
721		int idx = 0;
722		#include "tsan_update_shadow_word_inl.h"
723		idx = 1;
724		if (stored) {
725		#include "tsan_update_shadow_word_inl.h"
726		} else {
727		#include "tsan_update_shadow_word_inl.h"
728		}
729		idx = 2;
730		if (stored) {
731		#include "tsan_update_shadow_word_inl.h"
732		} else {
733		#include "tsan_update_shadow_word_inl.h"
734		}
735		idx = 3;
736		if (stored) {
737		#include "tsan_update_shadow_word_inl.h"
738		} else {
739		#include "tsan_update_shadow_word_inl.h"
740		}
741		#endif
742
743		// we did not find any races and had already stored
744		// the current access info, so we are done
745		if (LIKELY(stored))
746		return;
747		// choose a random candidate slot and replace it
748		StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word);
749		return;
750		RACE:
751		HandleRace(thr, shadow_mem, cur, old);
752		return;
753		}
754
755		void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr,
756		int size, bool kAccessIsWrite, bool kIsAtomic) {
757		while (size) {
758		int size1 = 1;
759		int kAccessSizeLog = kSizeLog1;
760		if (size >= 8 && (addr & ~7) == ((addr + 7) & ~7)) {
761		size1 = 8;
762		kAccessSizeLog = kSizeLog8;
763		} else if (size >= 4 && (addr & ~7) == ((addr + 3) & ~7)) {
764		size1 = 4;
765		kAccessSizeLog = kSizeLog4;
766		} else if (size >= 2 && (addr & ~7) == ((addr + 1) & ~7)) {
767		size1 = 2;
768		kAccessSizeLog = kSizeLog2;
769		}
770		MemoryAccess(thr, pc, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic);
771		addr += size1;
772		size -= size1;
773		}
774		}
775
776		ALWAYS_INLINE
777		bool ContainsSameAccessSlow(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
778		Shadow cur(a);
779		for (uptr i = 0; i < kShadowCnt; i++) {
780		Shadow old(LoadShadow(&s[i]));
781		if (Shadow::Addr0AndSizeAreEqual(cur, old) &&
782		old.TidWithIgnore() == cur.TidWithIgnore() &&
783		old.epoch() > sync_epoch &&
784		old.IsAtomic() == cur.IsAtomic() &&
785		old.IsRead() <= cur.IsRead())
786		return true;
	909	static bool TraceSkipGap(ThreadState* thr) {
	910	Trace *trace = &thr->tctx->trace;
	911	Event pos = reinterpret_cast<Event >(atomic_load_relaxed(&thr->trace_pos));
	912	DCHECK_EQ(reinterpret_cast<uptr>(pos + 1) & TracePart::kAlignment, 0);
	913	auto *part = trace->parts.Back();
	914	DPrintf("#%d: TraceSwitchPart enter trace=%p parts=%p-%p pos=%p\n", thr->tid,
	915	trace, trace->parts.Front(), part, pos);
	916	if (!part)
	917	return false;
	918	// We can get here when we still have space in the current trace part.
	919	// The fast-path check in TraceAcquire has false positives in the middle of
	920	// the part. Check if we are indeed at the end of the current part or not,
	921	// and fill any gaps with NopEvent's.
	922	Event* end = &part->events[TracePart::kSize];
	923	DCHECK_GE(pos, &part->events[0]);
	924	DCHECK_LE(pos, end);
	925	if (pos + 1 < end) {
	926	if ((reinterpret_cast<uptr>(pos) & TracePart::kAlignment) ==
	927	TracePart::kAlignment)
	928	*pos++ = NopEvent;
	929	*pos++ = NopEvent;
	930	DCHECK_LE(pos + 2, end);
	931	atomic_store_relaxed(&thr->trace_pos, reinterpret_cast<uptr>(pos));
	932	return true;
787	933	}
	934	// We are indeed at the end.
	935	for (; pos < end; pos++) *pos = NopEvent;
788	936	return false;
789	937	}
790	938
791		#if defined(__SSE3__)
792		#define SHUF(v0, v1, i0, i1, i2, i3) _mm_castps_si128(_mm_shuffle_ps( \
793		_mm_castsi128_ps(v0), _mm_castsi128_ps(v1), \
794		(i0)1 + (i1)4 + (i2)16 + (i3)64))
795		ALWAYS_INLINE
796		bool ContainsSameAccessFast(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
797		// This is an optimized version of ContainsSameAccessSlow.
798		// load current access into access[0:63]
799		const m128 access = _mm_cvtsi64_si128(a);
800		// duplicate high part of access in addr0:
801		// addr0[0:31] = access[32:63]
802		// addr0[32:63] = access[32:63]
803		// addr0[64:95] = access[32:63]
804		// addr0[96:127] = access[32:63]
805		const m128 addr0 = SHUF(access, access, 1, 1, 1, 1);
806		// load 4 shadow slots
807		const m128 shadow0 = _mm_load_si128((__m128i*)s);
808		const m128 shadow1 = _mm_load_si128((__m128i*)s + 1);
809		// load high parts of 4 shadow slots into addr_vect:
810		// addr_vect[0:31] = shadow0[32:63]
811		// addr_vect[32:63] = shadow0[96:127]
812		// addr_vect[64:95] = shadow1[32:63]
813		// addr_vect[96:127] = shadow1[96:127]
814		m128 addr_vect = SHUF(shadow0, shadow1, 1, 3, 1, 3);
815		if (!is_write) {
816		// set IsRead bit in addr_vect
817		const m128 rw_mask1 = _mm_cvtsi64_si128(1<<15);
818		const m128 rw_mask = SHUF(rw_mask1, rw_mask1, 0, 0, 0, 0);
819		addr_vect = _mm_or_si128(addr_vect, rw_mask);
820		}
821		// addr0 == addr_vect?
822		const m128 addr_res = _mm_cmpeq_epi32(addr0, addr_vect);
823		// epoch1[0:63] = sync_epoch
824		const m128 epoch1 = _mm_cvtsi64_si128(sync_epoch);
825		// epoch[0:31] = sync_epoch[0:31]
826		// epoch[32:63] = sync_epoch[0:31]
827		// epoch[64:95] = sync_epoch[0:31]
828		// epoch[96:127] = sync_epoch[0:31]
829		const m128 epoch = SHUF(epoch1, epoch1, 0, 0, 0, 0);
830		// load low parts of shadow cell epochs into epoch_vect:
831		// epoch_vect[0:31] = shadow0[0:31]
832		// epoch_vect[32:63] = shadow0[64:95]
833		// epoch_vect[64:95] = shadow1[0:31]
834		// epoch_vect[96:127] = shadow1[64:95]
835		const m128 epoch_vect = SHUF(shadow0, shadow1, 0, 2, 0, 2);
836		// epoch_vect >= sync_epoch?
837		const m128 epoch_res = _mm_cmpgt_epi32(epoch_vect, epoch);
838		// addr_res & epoch_res
839		const m128 res = _mm_and_si128(addr_res, epoch_res);
840		// mask[0] = res[7]
841		// mask[1] = res[15]
842		// ...
843		// mask[15] = res[127]
844		const int mask = _mm_movemask_epi8(res);
845		return mask != 0;
846		}
847		#endif
848
849		ALWAYS_INLINE
850		bool ContainsSameAccess(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
851		#if defined(__SSE3__)
852		bool res = ContainsSameAccessFast(s, a, sync_epoch, is_write);
853		// NOTE: this check can fail if the shadow is concurrently mutated
854		// by other threads. But it still can be useful if you modify
855		// ContainsSameAccessFast and want to ensure that it's not completely broken.
856		// DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write));
857		return res;
858		#else
859		return ContainsSameAccessSlow(s, a, sync_epoch, is_write);
860		#endif
861		}
862
863		ALWAYS_INLINE USED
864		void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
865		int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic) {
866		u64 shadow_mem = (u64)MemToShadow(addr);
867		DPrintf2("#%d: MemoryAccess: @%p %p size=%d"
868		" is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n",
869		(int)thr->fast_state.tid(), (void)pc, (void)addr,
870		(int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem,
871		(uptr)shadow_mem[0], (uptr)shadow_mem[1],
872		(uptr)shadow_mem[2], (uptr)shadow_mem[3]);
873		#if SANITIZER_DEBUG
874		if (!IsAppMem(addr)) {
875		Printf("Access to non app mem %zx\n", addr);
876		DCHECK(IsAppMem(addr));
877		}
878		if (!IsShadowMem((uptr)shadow_mem)) {
879		Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
880		DCHECK(IsShadowMem((uptr)shadow_mem));
881		}
882		#endif
883
884		if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) {
885		// Access to .rodata section, no races here.
886		// Measurements show that it can be 10-20% of all memory accesses.
887		return;
888		}
889
890		FastState fast_state = thr->fast_state;
891		if (UNLIKELY(fast_state.GetIgnoreBit())) {
892		return;
893		}
894
895		Shadow cur(fast_state);
896		cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog);
897		cur.SetWrite(kAccessIsWrite);
898		cur.SetAtomic(kIsAtomic);
899
900		if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
901		thr->fast_synch_epoch, kAccessIsWrite))) {
902		return;
903		}
904
905		if (kCollectHistory) {
906		fast_state.IncrementEpoch();
907		thr->fast_state = fast_state;
908		TraceAddEvent(thr, fast_state, EventTypeMop, pc);
909		cur.IncrementEpoch();
910		}
911
912		MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
913		shadow_mem, cur);
914		}
915
916		// Called by MemoryAccessRange in tsan_rtl_thread.cpp
917		ALWAYS_INLINE USED
918		void MemoryAccessImpl(ThreadState *thr, uptr addr,
919		int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
920		u64 *shadow_mem, Shadow cur) {
921		if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
922		thr->fast_synch_epoch, kAccessIsWrite))) {
923		return;
924		}
925
926		MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
927		shadow_mem, cur);
928		}
929
930		static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
931		u64 val) {
932		(void)thr;
933		(void)pc;
934		if (size == 0)
	939	NOINLINE
	940	void TraceSwitchPart(ThreadState* thr) {
	941	if (TraceSkipGap(thr))
935	942	return;
936		// FIXME: fix me.
937		uptr offset = addr % kShadowCell;
938		if (offset) {
939		offset = kShadowCell - offset;
940		if (size <= offset)
	943	#if !SANITIZER_GO
	944	if (ctx->after_multithreaded_fork) {
	945	// We just need to survive till exec.
	946	TracePart* part = thr->tctx->trace.parts.Back();
	947	if (part) {
	948	atomic_store_relaxed(&thr->trace_pos,
	949	reinterpret_cast<uptr>(&part->events[0]));
941	950	return;
942		addr += offset;
943		size -= offset;
944		}
945		DCHECK_EQ(addr % 8, 0);
946		// If a user passes some insane arguments (memset(0)),
947		// let it just crash as usual.
948		if (!IsAppMem(addr) \|\| !IsAppMem(addr + size - 1))
949		return;
950		// Don't want to touch lots of shadow memory.
951		// If a program maps 10MB stack, there is no need reset the whole range.
952		size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1);
953		// UnmapOrDie/MmapFixedNoReserve does not work on Windows.
954		if (SANITIZER_WINDOWS \|\| size < common_flags()->clear_shadow_mmap_threshold) {
955		u64 p = (u64)MemToShadow(addr);
956		CHECK(IsShadowMem((uptr)p));
957		CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1)));
958		// FIXME: may overwrite a part outside the region
959		for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) {
960		p[i++] = val;
961		for (uptr j = 1; j < kShadowCnt; j++)
962		p[i++] = 0;
963		}
964		} else {
965		// The region is big, reset only beginning and end.
966		const uptr kPageSize = GetPageSizeCached();
967		u64 begin = (u64)MemToShadow(addr);
968		u64 end = begin + size / kShadowCell kShadowCnt;
969		u64 *p = begin;
970		// Set at least first kPageSize/2 to page boundary.
971		while ((p < begin + kPageSize / kShadowSize / 2) \|\| ((uptr)p % kPageSize)) {
972		*p++ = val;
973		for (uptr j = 1; j < kShadowCnt; j++)
974		*p++ = 0;
975		}
976		// Reset middle part.
977		u64 *p1 = p;
978		p = RoundDown(end, kPageSize);
979		if (!MmapFixedSuperNoReserve((uptr)p1, (uptr)p - (uptr)p1))
980		Die();
981		// Set the ending.
982		while (p < end) {
983		*p++ = val;
984		for (uptr j = 1; j < kShadowCnt; j++)
985		*p++ = 0;
986	951	}
987	952	}
	953	#endif
	954	TraceSwitchPartImpl(thr);
988	955	}
989	956
990		void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) {
991		MemoryRangeSet(thr, pc, addr, size, 0);
992		}
993
994		void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) {
995		// Processing more than 1k (4k of shadow) is expensive,
996		// can cause excessive memory consumption (user does not necessary touch
997		// the whole range) and most likely unnecessary.
998		if (size > 1024)
999		size = 1024;
1000		CHECK_EQ(thr->is_freeing, false);
1001		thr->is_freeing = true;
1002		MemoryAccessRange(thr, pc, addr, size, true);
1003		thr->is_freeing = false;
1004		if (kCollectHistory) {
1005		thr->fast_state.IncrementEpoch();
1006		TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
	957	void TraceSwitchPartImpl(ThreadState* thr) {
	958	SlotLocker locker(thr, true);
	959	Trace* trace = &thr->tctx->trace;
	960	TracePart* part = TracePartAlloc(thr);
	961	part->trace = trace;
	962	thr->trace_prev_pc = 0;
	963	TracePart* recycle = nullptr;
	964	// Keep roughly half of parts local to the thread
	965	// (not queued into the recycle queue).
	966	uptr local_parts = (Trace::kMinParts + flags()->history_size + 1) / 2;
	967	{
	968	Lock lock(&trace->mtx);
	969	if (trace->parts.Empty())
	970	trace->local_head = part;
	971	if (trace->parts.Size() >= local_parts) {
	972	recycle = trace->local_head;
	973	trace->local_head = trace->parts.Next(recycle);
	974	}
	975	trace->parts.PushBack(part);
	976	atomic_store_relaxed(&thr->trace_pos,
	977	reinterpret_cast<uptr>(&part->events[0]));
1007	978	}
1008		Shadow s(thr->fast_state);
1009		s.ClearIgnoreBit();
1010		s.MarkAsFreed();
1011		s.SetWrite(true);
1012		s.SetAddr0AndSizeLog(0, 3);
1013		MemoryRangeSet(thr, pc, addr, size, s.raw());
1014		}
1015
1016		void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) {
1017		if (kCollectHistory) {
1018		thr->fast_state.IncrementEpoch();
1019		TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
	979	// Make this part self-sufficient by restoring the current stack
	980	// and mutex set in the beginning of the trace.
	981	TraceTime(thr);
	982	{
	983	// Pathologically large stacks may not fit into the part.
	984	// In these cases we log only fixed number of top frames.
	985	const uptr kMaxFrames = 1000;
	986	// Check that kMaxFrames won't consume the whole part.
	987	static_assert(kMaxFrames < TracePart::kSize / 2, "kMaxFrames is too big");
	988	uptr* pos = Max(&thr->shadow_stack[0], thr->shadow_stack_pos - kMaxFrames);
	989	for (; pos < thr->shadow_stack_pos; pos++) {
	990	if (TryTraceFunc(thr, *pos))
	991	continue;
	992	CHECK(TraceSkipGap(thr));
	993	CHECK(TryTraceFunc(thr, *pos));
	994	}
1020	995	}
1021		Shadow s(thr->fast_state);
1022		s.ClearIgnoreBit();
1023		s.SetWrite(true);
1024		s.SetAddr0AndSizeLog(0, 3);
1025		MemoryRangeSet(thr, pc, addr, size, s.raw());
1026		}
1027
1028		void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
1029		uptr size) {
1030		if (thr->ignore_reads_and_writes == 0)
1031		MemoryRangeImitateWrite(thr, pc, addr, size);
1032		else
1033		MemoryResetRange(thr, pc, addr, size);
1034		}
1035
1036		ALWAYS_INLINE USED
1037		void FuncEntry(ThreadState *thr, uptr pc) {
1038		DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc);
1039		if (kCollectHistory) {
1040		thr->fast_state.IncrementEpoch();
1041		TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc);
	996	for (uptr i = 0; i < thr->mset.Size(); i++) {
	997	MutexSet::Desc d = thr->mset.Get(i);
	998	for (uptr i = 0; i < d.count; i++)
	999	TraceMutexLock(thr, d.write ? EventType::kLock : EventType::kRLock, 0,
	1000	d.addr, d.stack_id);
1042	1001	}
1043
1044		// Shadow stack maintenance can be replaced with
1045		// stack unwinding during trace switch (which presumably must be faster).
1046		DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
1047		#if !SANITIZER_GO
1048		DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
1049		#else
1050		if (thr->shadow_stack_pos == thr->shadow_stack_end)
1051		GrowShadowStack(thr);
1052		#endif
1053		thr->shadow_stack_pos[0] = pc;
1054		thr->shadow_stack_pos++;
1055		}
1056
1057		ALWAYS_INLINE USED
1058		void FuncExit(ThreadState *thr) {
1059		DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid());
1060		if (kCollectHistory) {
1061		thr->fast_state.IncrementEpoch();
1062		TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0);
	1002	// Callers of TraceSwitchPart expect that TraceAcquire will always succeed
	1003	// after the call. It's possible that TryTraceFunc/TraceMutexLock above
	1004	// filled the trace part exactly up to the TracePart::kAlignment gap
	1005	// and the next TraceAcquire won't succeed. Skip the gap to avoid that.
	1006	EventFunc *ev;
	1007	if (!TraceAcquire(thr, &ev)) {
	1008	CHECK(TraceSkipGap(thr));
	1009	CHECK(TraceAcquire(thr, &ev));
1063	1010	}
1064
1065		DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
1066		#if !SANITIZER_GO
1067		DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
1068		#endif
1069		thr->shadow_stack_pos--;
	1011	{
	1012	Lock lock(&ctx->slot_mtx);
	1013	// There is a small chance that the slot may be not queued at this point.
	1014	// This can happen if the slot has kEpochLast epoch and another thread
	1015	// in FindSlotAndLock discovered that it's exhausted and removed it from
	1016	// the slot queue. kEpochLast can happen in 2 cases: (1) if TraceSwitchPart
	1017	// was called with the slot locked and epoch already at kEpochLast,
	1018	// or (2) if we've acquired a new slot in SlotLock in the beginning
	1019	// of the function and the slot was at kEpochLast - 1, so after increment
	1020	// in SlotAttachAndLock it become kEpochLast.
	1021	if (ctx->slot_queue.Queued(thr->slot)) {
	1022	ctx->slot_queue.Remove(thr->slot);
	1023	ctx->slot_queue.PushBack(thr->slot);
	1024	}
	1025	if (recycle)
	1026	ctx->trace_part_recycle.PushBack(recycle);
	1027	}
	1028	DPrintf("#%d: TraceSwitchPart exit parts=%p-%p pos=0x%zx\n", thr->tid,
	1029	trace->parts.Front(), trace->parts.Back(),
	1030	atomic_load_relaxed(&thr->trace_pos));
1070	1031	}
1071	1032
1072		void ThreadIgnoreBegin(ThreadState *thr, uptr pc, bool save_stack) {
	1033	void ThreadIgnoreBegin(ThreadState* thr, uptr pc) {
1073	1034	DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid);
1074	1035	thr->ignore_reads_and_writes++;
1075	1036	CHECK_GT(thr->ignore_reads_and_writes, 0);
1076	1037	thr->fast_state.SetIgnoreBit();
1077	1038	#if !SANITIZER_GO
1078		if (save_stack && !ctx->after_multithreaded_fork)
	1039	if (pc && !ctx->after_multithreaded_fork)
1079	1040	thr->mop_ignore_set.Add(CurrentStackId(thr, pc));
1080	1041	#endif
1081	1042	}
1082	1043
1083		void ThreadIgnoreEnd(ThreadState *thr, uptr pc) {
	1044	void ThreadIgnoreEnd(ThreadState *thr) {
1084	1045	DPrintf("#%d: ThreadIgnoreEnd\n", thr->tid);
1085	1046	CHECK_GT(thr->ignore_reads_and_writes, 0);
1086	1047	thr->ignore_reads_and_writes--;
...	...	@@ -1100,17 +1061,17 @@ uptr __tsan_testonly_shadow_stack_current_size() {
1100	1061	}
1101	1062	#endif
1102	1063
1103		void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc, bool save_stack) {
	1064	void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc) {
1104	1065	DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr->tid);
1105	1066	thr->ignore_sync++;
1106	1067	CHECK_GT(thr->ignore_sync, 0);
1107	1068	#if !SANITIZER_GO
1108		if (save_stack && !ctx->after_multithreaded_fork)
	1069	if (pc && !ctx->after_multithreaded_fork)
1109	1070	thr->sync_ignore_set.Add(CurrentStackId(thr, pc));
1110	1071	#endif
1111	1072	}
1112	1073
1113		void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc) {
	1074	void ThreadIgnoreSyncEnd(ThreadState *thr) {
1114	1075	DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr->tid);
1115	1076	CHECK_GT(thr->ignore_sync, 0);
1116	1077	thr->ignore_sync--;
...	...	@@ -1129,7 +1090,6 @@ void build_consistency_debug() {}
1129	1090	#else
1130	1091	void build_consistency_release() {}
1131	1092	#endif
1132
1133	1093	} // namespace __tsan
1134	1094
1135	1095	#if SANITIZER_CHECK_DEADLOCKS
...	...	@@ -1137,23 +1097,27 @@ namespace __sanitizer {
1137	1097	using namespace __tsan;
1138	1098	MutexMeta mutex_meta[] = {
1139	1099	{MutexInvalid, "Invalid", {}},
1140		{MutexThreadRegistry, "ThreadRegistry", {}},
1141		{MutexTypeTrace, "Trace", {MutexLeaf}},
1142		{MutexTypeReport, "Report", {MutexTypeSyncVar}},
1143		{MutexTypeSyncVar, "SyncVar", {}},
	1100	{MutexThreadRegistry,
	1101	"ThreadRegistry",
	1102	{MutexTypeSlots, MutexTypeTrace, MutexTypeReport}},
	1103	{MutexTypeReport, "Report", {MutexTypeTrace}},
	1104	{MutexTypeSyncVar, "SyncVar", {MutexTypeReport, MutexTypeTrace}},
1144	1105	{MutexTypeAnnotations, "Annotations", {}},
1145		{MutexTypeAtExit, "AtExit", {MutexTypeSyncVar}},
	1106	{MutexTypeAtExit, "AtExit", {}},
1146	1107	{MutexTypeFired, "Fired", {MutexLeaf}},
1147	1108	{MutexTypeRacy, "Racy", {MutexLeaf}},
1148		{MutexTypeGlobalProc, "GlobalProc", {}},
	1109	{MutexTypeGlobalProc, "GlobalProc", {MutexTypeSlot, MutexTypeSlots}},
	1110	{MutexTypeInternalAlloc, "InternalAlloc", {MutexLeaf}},
	1111	{MutexTypeTrace, "Trace", {}},
	1112	{MutexTypeSlot,
	1113	"Slot",
	1114	{MutexMulti, MutexTypeTrace, MutexTypeSyncVar, MutexThreadRegistry,
	1115	MutexTypeSlots}},
	1116	{MutexTypeSlots, "Slots", {MutexTypeTrace, MutexTypeReport}},
1149	1117	{},
1150	1118	};
1151	1119
1152	1120	void PrintMutexPC(uptr pc) { StackTrace(&pc, 1).Print(); }
1153		} // namespace __sanitizer
1154		#endif
1155	1121
1156		#if !SANITIZER_GO
1157		// Must be included in this file to make sure everything is inlined.
1158		# include "tsan_interface_inl.h"
	1122	} // namespace __sanitizer
1159	1123	#endif

lib/tsan/tsan_rtl.h+376-429

...	...	@@ -34,17 +34,19 @@
34	34	#include "sanitizer_common/sanitizer_suppressions.h"
35	35	#include "sanitizer_common/sanitizer_thread_registry.h"
36	36	#include "sanitizer_common/sanitizer_vector.h"
37		#include "tsan_clock.h"
38	37	#include "tsan_defs.h"
39	38	#include "tsan_flags.h"
	39	#include "tsan_ignoreset.h"
	40	#include "tsan_ilist.h"
40	41	#include "tsan_mman.h"
41		#include "tsan_sync.h"
42		#include "tsan_trace.h"
43		#include "tsan_report.h"
44		#include "tsan_platform.h"
45	42	#include "tsan_mutexset.h"
46		#include "tsan_ignoreset.h"
	43	#include "tsan_platform.h"
	44	#include "tsan_report.h"
	45	#include "tsan_shadow.h"
47	46	#include "tsan_stack_trace.h"
	47	#include "tsan_sync.h"
	48	#include "tsan_trace.h"
	49	#include "tsan_vector_clock.h"
48	50
49	51	#if SANITIZER_WORDSIZE != 64
50	52	# error "ThreadSanitizer is supported only on 64-bit platforms"
...	...	@@ -54,7 +56,8 @@ namespace __tsan {
54	56
55	57	#if !SANITIZER_GO
56	58	struct MapUnmapCallback;
57		#if defined(__mips64) \|\| defined(__aarch64__) \|\| defined(__powerpc__)
	59	#if defined(__mips64) \|\| defined(__aarch64__) \|\| defined(__loongarch__) \|\| \
	60	defined(__powerpc__)
58	61
59	62	struct AP32 {
60	63	static const uptr kSpaceBeg = 0;
...	...	@@ -69,6 +72,11 @@ struct AP32 {
69	72	typedef SizeClassAllocator32<AP32> PrimaryAllocator;
70	73	#else
71	74	struct AP64 { // Allocator64 parameters. Deliberately using a short name.
	75	# if defined(__s390x__)
	76	typedef MappingS390x Mapping;
	77	# else
	78	typedef Mapping48AddressSpace Mapping;
	79	# endif
72	80	static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
73	81	static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg;
74	82	static const uptr kMetadataSize = 0;
...	...	@@ -84,240 +92,6 @@ typedef Allocator::AllocatorCache AllocatorCache;
84	92	Allocator *allocator();
85	93	#endif
86	94
87		const u64 kShadowRodata = (u64)-1; // .rodata shadow marker
88
89		// FastState (from most significant bit):
90		// ignore : 1
91		// tid : kTidBits
92		// unused : -
93		// history_size : 3
94		// epoch : kClkBits
95		class FastState {
96		public:
97		FastState(u64 tid, u64 epoch) {
98		x_ = tid << kTidShift;
99		x_ \|= epoch;
100		DCHECK_EQ(tid, this->tid());
101		DCHECK_EQ(epoch, this->epoch());
102		DCHECK_EQ(GetIgnoreBit(), false);
103		}
104
105		explicit FastState(u64 x)
106		: x_(x) {
107		}
108
109		u64 raw() const {
110		return x_;
111		}
112
113		u64 tid() const {
114		u64 res = (x_ & ~kIgnoreBit) >> kTidShift;
115		return res;
116		}
117
118		u64 TidWithIgnore() const {
119		u64 res = x_ >> kTidShift;
120		return res;
121		}
122
123		u64 epoch() const {
124		u64 res = x_ & ((1ull << kClkBits) - 1);
125		return res;
126		}
127
128		void IncrementEpoch() {
129		u64 old_epoch = epoch();
130		x_ += 1;
131		DCHECK_EQ(old_epoch + 1, epoch());
132		(void)old_epoch;
133		}
134
135		void SetIgnoreBit() { x_ \|= kIgnoreBit; }
136		void ClearIgnoreBit() { x_ &= ~kIgnoreBit; }
137		bool GetIgnoreBit() const { return (s64)x_ < 0; }
138
139		void SetHistorySize(int hs) {
140		CHECK_GE(hs, 0);
141		CHECK_LE(hs, 7);
142		x_ = (x_ & ~(kHistoryMask << kHistoryShift)) \| (u64(hs) << kHistoryShift);
143		}
144
145		ALWAYS_INLINE
146		int GetHistorySize() const {
147		return (int)((x_ >> kHistoryShift) & kHistoryMask);
148		}
149
150		void ClearHistorySize() {
151		SetHistorySize(0);
152		}
153
154		ALWAYS_INLINE
155		u64 GetTracePos() const {
156		const int hs = GetHistorySize();
157		// When hs == 0, the trace consists of 2 parts.
158		const u64 mask = (1ull << (kTracePartSizeBits + hs + 1)) - 1;
159		return epoch() & mask;
160		}
161
162		private:
163		friend class Shadow;
164		static const int kTidShift = 64 - kTidBits - 1;
165		static const u64 kIgnoreBit = 1ull << 63;
166		static const u64 kFreedBit = 1ull << 63;
167		static const u64 kHistoryShift = kClkBits;
168		static const u64 kHistoryMask = 7;
169		u64 x_;
170		};
171
172		// Shadow (from most significant bit):
173		// freed : 1
174		// tid : kTidBits
175		// is_atomic : 1
176		// is_read : 1
177		// size_log : 2
178		// addr0 : 3
179		// epoch : kClkBits
180		class Shadow : public FastState {
181		public:
182		explicit Shadow(u64 x)
183		: FastState(x) {
184		}
185
186		explicit Shadow(const FastState &s)
187		: FastState(s.x_) {
188		ClearHistorySize();
189		}
190
191		void SetAddr0AndSizeLog(u64 addr0, unsigned kAccessSizeLog) {
192		DCHECK_EQ((x_ >> kClkBits) & 31, 0);
193		DCHECK_LE(addr0, 7);
194		DCHECK_LE(kAccessSizeLog, 3);
195		x_ \|= ((kAccessSizeLog << 3) \| addr0) << kClkBits;
196		DCHECK_EQ(kAccessSizeLog, size_log());
197		DCHECK_EQ(addr0, this->addr0());
198		}
199
200		void SetWrite(unsigned kAccessIsWrite) {
201		DCHECK_EQ(x_ & kReadBit, 0);
202		if (!kAccessIsWrite)
203		x_ \|= kReadBit;
204		DCHECK_EQ(kAccessIsWrite, IsWrite());
205		}
206
207		void SetAtomic(bool kIsAtomic) {
208		DCHECK(!IsAtomic());
209		if (kIsAtomic)
210		x_ \|= kAtomicBit;
211		DCHECK_EQ(IsAtomic(), kIsAtomic);
212		}
213
214		bool IsAtomic() const {
215		return x_ & kAtomicBit;
216		}
217
218		bool IsZero() const {
219		return x_ == 0;
220		}
221
222		static inline bool TidsAreEqual(const Shadow s1, const Shadow s2) {
223		u64 shifted_xor = (s1.x_ ^ s2.x_) >> kTidShift;
224		DCHECK_EQ(shifted_xor == 0, s1.TidWithIgnore() == s2.TidWithIgnore());
225		return shifted_xor == 0;
226		}
227
228		static ALWAYS_INLINE
229		bool Addr0AndSizeAreEqual(const Shadow s1, const Shadow s2) {
230		u64 masked_xor = ((s1.x_ ^ s2.x_) >> kClkBits) & 31;
231		return masked_xor == 0;
232		}
233
234		static ALWAYS_INLINE bool TwoRangesIntersect(Shadow s1, Shadow s2,
235		unsigned kS2AccessSize) {
236		bool res = false;
237		u64 diff = s1.addr0() - s2.addr0();
238		if ((s64)diff < 0) { // s1.addr0 < s2.addr0
239		// if (s1.addr0() + size1) > s2.addr0()) return true;
240		if (s1.size() > -diff)
241		res = true;
242		} else {
243		// if (s2.addr0() + kS2AccessSize > s1.addr0()) return true;
244		if (kS2AccessSize > diff)
245		res = true;
246		}
247		DCHECK_EQ(res, TwoRangesIntersectSlow(s1, s2));
248		DCHECK_EQ(res, TwoRangesIntersectSlow(s2, s1));
249		return res;
250		}
251
252		u64 ALWAYS_INLINE addr0() const { return (x_ >> kClkBits) & 7; }
253		u64 ALWAYS_INLINE size() const { return 1ull << size_log(); }
254		bool ALWAYS_INLINE IsWrite() const { return !IsRead(); }
255		bool ALWAYS_INLINE IsRead() const { return x_ & kReadBit; }
256
257		// The idea behind the freed bit is as follows.
258		// When the memory is freed (or otherwise unaccessible) we write to the shadow
259		// values with tid/epoch related to the free and the freed bit set.
260		// During memory accesses processing the freed bit is considered
261		// as msb of tid. So any access races with shadow with freed bit set
262		// (it is as if write from a thread with which we never synchronized before).
263		// This allows us to detect accesses to freed memory w/o additional
264		// overheads in memory access processing and at the same time restore
265		// tid/epoch of free.
266		void MarkAsFreed() {
267		x_ \|= kFreedBit;
268		}
269
270		bool IsFreed() const {
271		return x_ & kFreedBit;
272		}
273
274		bool GetFreedAndReset() {
275		bool res = x_ & kFreedBit;
276		x_ &= ~kFreedBit;
277		return res;
278		}
279
280		bool ALWAYS_INLINE IsBothReadsOrAtomic(bool kIsWrite, bool kIsAtomic) const {
281		bool v = x_ & ((u64(kIsWrite ^ 1) << kReadShift)
282		\| (u64(kIsAtomic) << kAtomicShift));
283		DCHECK_EQ(v, (!IsWrite() && !kIsWrite) \|\| (IsAtomic() && kIsAtomic));
284		return v;
285		}
286
287		bool ALWAYS_INLINE IsRWNotWeaker(bool kIsWrite, bool kIsAtomic) const {
288		bool v = ((x_ >> kReadShift) & 3)
289		<= u64((kIsWrite ^ 1) \| (kIsAtomic << 1));
290		DCHECK_EQ(v, (IsAtomic() < kIsAtomic) \|\|
291		(IsAtomic() == kIsAtomic && !IsWrite() <= !kIsWrite));
292		return v;
293		}
294
295		bool ALWAYS_INLINE IsRWWeakerOrEqual(bool kIsWrite, bool kIsAtomic) const {
296		bool v = ((x_ >> kReadShift) & 3)
297		>= u64((kIsWrite ^ 1) \| (kIsAtomic << 1));
298		DCHECK_EQ(v, (IsAtomic() > kIsAtomic) \|\|
299		(IsAtomic() == kIsAtomic && !IsWrite() >= !kIsWrite));
300		return v;
301		}
302
303		private:
304		static const u64 kReadShift = 5 + kClkBits;
305		static const u64 kReadBit = 1ull << kReadShift;
306		static const u64 kAtomicShift = 6 + kClkBits;
307		static const u64 kAtomicBit = 1ull << kAtomicShift;
308
309		u64 size_log() const { return (x_ >> (3 + kClkBits)) & 3; }
310
311		static bool TwoRangesIntersectSlow(const Shadow s1, const Shadow s2) {
312		if (s1.addr0() == s2.addr0()) return true;
313		if (s1.addr0() < s2.addr0() && s1.addr0() + s1.size() > s2.addr0())
314		return true;
315		if (s2.addr0() < s1.addr0() && s2.addr0() + s2.size() > s1.addr0())
316		return true;
317		return false;
318		}
319		};
320
321	95	struct ThreadSignalContext;
322	96
323	97	struct JmpBuf {
...	...	@@ -344,7 +118,6 @@ struct Processor {
344	118	#endif
345	119	DenseSlabAllocCache block_cache;
346	120	DenseSlabAllocCache sync_cache;
347		DenseSlabAllocCache clock_cache;
348	121	DDPhysicalThread *dd_pt;
349	122	};
350	123
...	...	@@ -358,64 +131,86 @@ struct ScopedGlobalProcessor {
358	131	};
359	132	#endif
360	133
	134	struct TidEpoch {
	135	Tid tid;
	136	Epoch epoch;
	137	};
	138
	139	struct TidSlot {
	140	Mutex mtx;
	141	Sid sid;
	142	atomic_uint32_t raw_epoch;
	143	ThreadState *thr;
	144	Vector<TidEpoch> journal;
	145	INode node;
	146
	147	Epoch epoch() const {
	148	return static_cast<Epoch>(atomic_load(&raw_epoch, memory_order_relaxed));
	149	}
	150
	151	void SetEpoch(Epoch v) {
	152	atomic_store(&raw_epoch, static_cast<u32>(v), memory_order_relaxed);
	153	}
	154
	155	TidSlot();
	156	} ALIGNED(SANITIZER_CACHE_LINE_SIZE);
	157
361	158	// This struct is stored in TLS.
362	159	struct ThreadState {
363	160	FastState fast_state;
364		// Synch epoch represents the threads's epoch before the last synchronization
365		// action. It allows to reduce number of shadow state updates.
366		// For example, fast_synch_epoch=100, last write to addr X was at epoch=150,
367		// if we are processing write to X from the same thread at epoch=200,
368		// we do nothing, because both writes happen in the same 'synch epoch'.
369		// That is, if another memory access does not race with the former write,
370		// it does not race with the latter as well.
371		// QUESTION: can we can squeeze this into ThreadState::Fast?
372		// E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are
373		// taken by epoch between synchs.
374		// This way we can save one load from tls.
375		u64 fast_synch_epoch;
	161	int ignore_sync;
	162	#if !SANITIZER_GO
	163	int ignore_interceptors;
	164	#endif
	165	uptr *shadow_stack_pos;
	166
	167	// Current position in tctx->trace.Back()->events (Event*).
	168	atomic_uintptr_t trace_pos;
	169	// PC of the last memory access, used to compute PC deltas in the trace.
	170	uptr trace_prev_pc;
	171
376	172	// Technically `current` should be a separate THREADLOCAL variable;
377	173	// but it is placed here in order to share cache line with previous fields.
378	174	ThreadState* current;
	175
	176	atomic_sint32_t pending_signals;
	177
	178	VectorClock clock;
	179
379	180	// This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
380	181	// We do not distinguish beteween ignoring reads and writes
381	182	// for better performance.
382	183	int ignore_reads_and_writes;
383		int ignore_sync;
384	184	int suppress_reports;
385	185	// Go does not support ignores.
386	186	#if !SANITIZER_GO
387	187	IgnoreSet mop_ignore_set;
388	188	IgnoreSet sync_ignore_set;
389	189	#endif
390		// C/C++ uses fixed size shadow stack embed into Trace.
391		// Go uses malloc-allocated shadow stack with dynamic size.
392	190	uptr *shadow_stack;
393	191	uptr *shadow_stack_end;
394		uptr *shadow_stack_pos;
395		u64 *racy_shadow_addr;
396		u64 racy_state[2];
397		MutexSet mset;
398		ThreadClock clock;
399	192	#if !SANITIZER_GO
400	193	Vector<JmpBuf> jmp_bufs;
401		int ignore_interceptors;
402		#endif
403		const u32 tid;
404		const int unique_id;
405		bool in_symbolizer;
	194	int in_symbolizer;
	195	atomic_uintptr_t in_blocking_func;
406	196	bool in_ignored_lib;
407	197	bool is_inited;
	198	#endif
	199	MutexSet mset;
408	200	bool is_dead;
409		bool is_freeing;
410		bool is_vptr_access;
411		const uptr stk_addr;
412		const uptr stk_size;
413		const uptr tls_addr;
414		const uptr tls_size;
	201	const Tid tid;
	202	uptr stk_addr;
	203	uptr stk_size;
	204	uptr tls_addr;
	205	uptr tls_size;
415	206	ThreadContext *tctx;
416	207
417	208	DDLogicalThread *dd_lt;
418	209
	210	TidSlot *slot;
	211	uptr slot_epoch;
	212	bool slot_locked;
	213
419	214	// Current wired Processor, or nullptr. Required to handle any events.
420	215	Processor *proc1;
421	216	#if !SANITIZER_GO
...	...	@@ -425,11 +220,11 @@ struct ThreadState {
425	220	#endif
426	221
427	222	atomic_uintptr_t in_signal_handler;
428		ThreadSignalContext *signal_ctx;
	223	atomic_uintptr_t signal_ctx;
429	224
430	225	#if !SANITIZER_GO
431		u32 last_sleep_stack_id;
432		ThreadClock last_sleep_clock;
	226	StackID last_sleep_stack_id;
	227	VectorClock last_sleep_clock;
433	228	#endif
434	229
435	230	// Set in regions of runtime that must be signal-safe and fork-safe.
...	...	@@ -438,47 +233,43 @@ struct ThreadState {
438	233
439	234	const ReportDesc *current_report;
440	235
441		explicit ThreadState(Context *ctx, u32 tid, int unique_id, u64 epoch,
442		unsigned reuse_count, uptr stk_addr, uptr stk_size,
443		uptr tls_addr, uptr tls_size);
444		};
	236	explicit ThreadState(Tid tid);
	237	} ALIGNED(SANITIZER_CACHE_LINE_SIZE);
445	238
446	239	#if !SANITIZER_GO
447		#if SANITIZER_MAC \|\| SANITIZER_ANDROID
	240	#if SANITIZER_APPLE \|\| SANITIZER_ANDROID
448	241	ThreadState *cur_thread();
449	242	void set_cur_thread(ThreadState *thr);
450	243	void cur_thread_finalize();
451		inline void cur_thread_init() { }
452		#else
	244	inline ThreadState *cur_thread_init() { return cur_thread(); }
	245	# else
453	246	__attribute__((tls_model("initial-exec")))
454	247	extern THREADLOCAL char cur_thread_placeholder[];
455	248	inline ThreadState *cur_thread() {
456	249	return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current;
457	250	}
458		inline void cur_thread_init() {
	251	inline ThreadState *cur_thread_init() {
459	252	ThreadState thr = reinterpret_cast<ThreadState >(cur_thread_placeholder);
460	253	if (UNLIKELY(!thr->current))
461	254	thr->current = thr;
	255	return thr->current;
462	256	}
463	257	inline void set_cur_thread(ThreadState *thr) {
464	258	reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr;
465	259	}
466	260	inline void cur_thread_finalize() { }
467		#endif // SANITIZER_MAC \|\| SANITIZER_ANDROID
	261	# endif // SANITIZER_APPLE \|\| SANITIZER_ANDROID
468	262	#endif // SANITIZER_GO
469	263
470	264	class ThreadContext final : public ThreadContextBase {
471	265	public:
472		explicit ThreadContext(int tid);
	266	explicit ThreadContext(Tid tid);
473	267	~ThreadContext();
474	268	ThreadState *thr;
475		u32 creation_stack_id;
476		SyncClock sync;
477		// Epoch at which the thread had started.
478		// If we see an event from the thread stamped by an older epoch,
479		// the event is from a dead thread that shared tid with this thread.
480		u64 epoch0;
481		u64 epoch1;
	269	StackID creation_stack_id;
	270	VectorClock *sync;
	271	uptr sync_epoch;
	272	Trace trace;
482	273
483	274	// Override superclass callbacks.
484	275	void OnDead() override;
...	...	@@ -492,13 +283,7 @@ class ThreadContext final : public ThreadContextBase {
492	283
493	284	struct RacyStacks {
494	285	MD5Hash hash[2];
495		bool operator==(const RacyStacks &other) const {
496		if (hash[0] == other.hash[0] && hash[1] == other.hash[1])
497		return true;
498		if (hash[0] == other.hash[1] && hash[1] == other.hash[0])
499		return true;
500		return false;
501		}
	286	bool operator==(const RacyStacks &other) const;
502	287	};
503	288
504	289	struct RacyAddress {
...	...	@@ -524,28 +309,75 @@ struct Context {
524	309
525	310	Mutex report_mtx;
526	311	int nreported;
527		int nmissed_expected;
528	312	atomic_uint64_t last_symbolize_time_ns;
529	313
530	314	void *background_thread;
531	315	atomic_uint32_t stop_background_thread;
532	316
533		ThreadRegistry *thread_registry;
	317	ThreadRegistry thread_registry;
	318
	319	// This is used to prevent a very unlikely but very pathological behavior.
	320	// Since memory access handling is not synchronized with DoReset,
	321	// a thread running concurrently with DoReset can leave a bogus shadow value
	322	// that will be later falsely detected as a race. For such false races
	323	// RestoreStack will return false and we will not report it.
	324	// However, consider that a thread leaves a whole lot of such bogus values
	325	// and these values are later read by a whole lot of threads.
	326	// This will cause massive amounts of ReportRace calls and lots of
	327	// serialization. In very pathological cases the resulting slowdown
	328	// can be >100x. This is very unlikely, but it was presumably observed
	329	// in practice: https://github.com/google/sanitizers/issues/1552
	330	// If this happens, previous access sid+epoch will be the same for all of
	331	// these false races b/c if the thread will try to increment epoch, it will
	332	// notice that DoReset has happened and will stop producing bogus shadow
	333	// values. So, last_spurious_race is used to remember the last sid+epoch
	334	// for which RestoreStack returned false. Then it is used to filter out
	335	// races with the same sid+epoch very early and quickly.
	336	// It is of course possible that multiple threads left multiple bogus shadow
	337	// values and all of them are read by lots of threads at the same time.
	338	// In such case last_spurious_race will only be able to deduplicate a few
	339	// races from one thread, then few from another and so on. An alternative
	340	// would be to hold an array of such sid+epoch, but we consider such scenario
	341	// as even less likely.
	342	// Note: this can lead to some rare false negatives as well:
	343	// 1. When a legit access with the same sid+epoch participates in a race
	344	// as the "previous" memory access, it will be wrongly filtered out.
	345	// 2. When RestoreStack returns false for a legit memory access because it
	346	// was already evicted from the thread trace, we will still remember it in
	347	// last_spurious_race. Then if there is another racing memory access from
	348	// the same thread that happened in the same epoch, but was stored in the
	349	// next thread trace part (which is still preserved in the thread trace),
	350	// we will also wrongly filter it out while RestoreStack would actually
	351	// succeed for that second memory access.
	352	RawShadow last_spurious_race;
534	353
535	354	Mutex racy_mtx;
536	355	Vector<RacyStacks> racy_stacks;
537		Vector<RacyAddress> racy_addresses;
538	356	// Number of fired suppressions may be large enough.
539	357	Mutex fired_suppressions_mtx;
540	358	InternalMmapVector<FiredSuppression> fired_suppressions;
541	359	DDetector *dd;
542	360
543		ClockAlloc clock_alloc;
544
545	361	Flags flags;
546
547		u64 int_alloc_cnt[MBlockTypeCount];
548		u64 int_alloc_siz[MBlockTypeCount];
	362	fd_t memprof_fd;
	363
	364	// The last slot index (kFreeSid) is used to denote freed memory.
	365	TidSlot slots[kThreadSlotCount - 1];
	366
	367	// Protects global_epoch, slot_queue, trace_part_recycle.
	368	Mutex slot_mtx;
	369	uptr global_epoch; // guarded by slot_mtx and by all slot mutexes
	370	bool resetting; // global reset is in progress
	371	IList<TidSlot, &TidSlot::node> slot_queue SANITIZER_GUARDED_BY(slot_mtx);
	372	IList<TraceHeader, &TraceHeader::global, TracePart> trace_part_recycle
	373	SANITIZER_GUARDED_BY(slot_mtx);
	374	uptr trace_part_total_allocated SANITIZER_GUARDED_BY(slot_mtx);
	375	uptr trace_part_recycle_finished SANITIZER_GUARDED_BY(slot_mtx);
	376	uptr trace_part_finished_excess SANITIZER_GUARDED_BY(slot_mtx);
	377	#if SANITIZER_GO
	378	uptr mapped_shadow_begin;
	379	uptr mapped_shadow_end;
	380	#endif
549	381	};
550	382
551	383	extern Context *ctx; // The one and the only global runtime context.
...	...	@@ -574,17 +406,17 @@ uptr TagFromShadowStackFrame(uptr pc);
574	406
575	407	class ScopedReportBase {
576	408	public:
577		void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, StackTrace stack,
578		const MutexSet *mset);
	409	void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, Tid tid,
	410	StackTrace stack, const MutexSet *mset);
579	411	void AddStack(StackTrace stack, bool suppressable = false);
580	412	void AddThread(const ThreadContext *tctx, bool suppressable = false);
581		void AddThread(int unique_tid, bool suppressable = false);
582		void AddUniqueTid(int unique_tid);
583		void AddMutex(const SyncVar *s);
584		u64 AddMutex(u64 id);
	413	void AddThread(Tid tid, bool suppressable = false);
	414	void AddUniqueTid(Tid unique_tid);
	415	int AddMutex(uptr addr, StackID creation_stack_id);
585	416	void AddLocation(uptr addr, uptr size);
586		void AddSleep(u32 stack_id);
	417	void AddSleep(StackID stack_id);
587	418	void SetCount(int count);
	419	void SetSigNum(int sig);
588	420
589	421	const ReportDesc *GetReport() const;
590	422
...	...	@@ -598,8 +430,6 @@ class ScopedReportBase {
598	430	// at best it will cause deadlocks on internal mutexes.
599	431	ScopedIgnoreInterceptors ignore_interceptors_;
600	432
601		void AddDeadMutex(u64 id);
602
603	433	ScopedReportBase(const ScopedReportBase &) = delete;
604	434	void operator=(const ScopedReportBase &) = delete;
605	435	};
...	...	@@ -615,8 +445,6 @@ class ScopedReport : public ScopedReportBase {
615	445
616	446	bool ShouldReport(ThreadState *thr, ReportType typ);
617	447	ThreadContext IsThreadStackOrTls(uptr addr, bool is_stack);
618		void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
619		MutexSet mset, uptr tag = nullptr);
620	448
621	449	// The stack could look like:
622	450	// <start> \| <main> \| <foo> \| tag \| <bar>
...	...	@@ -656,19 +484,20 @@ void MapThreadTrace(uptr addr, uptr size, const char *name);
656	484	void DontNeedShadowFor(uptr addr, uptr size);
657	485	void UnmapShadow(ThreadState *thr, uptr addr, uptr size);
658	486	void InitializeShadowMemory();
	487	void DontDumpShadow(uptr addr, uptr size);
659	488	void InitializeInterceptors();
660	489	void InitializeLibIgnore();
661	490	void InitializeDynamicAnnotations();
662	491
663	492	void ForkBefore(ThreadState *thr, uptr pc);
664	493	void ForkParentAfter(ThreadState *thr, uptr pc);
665		void ForkChildAfter(ThreadState *thr, uptr pc);
	494	void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread);
666	495
667		void ReportRace(ThreadState *thr);
	496	void ReportRace(ThreadState thr, RawShadow shadow_mem, Shadow cur, Shadow old,
	497	AccessType typ);
668	498	bool OutputReport(ThreadState *thr, const ScopedReport &srep);
669	499	bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);
670	500	bool IsExpectedReport(uptr addr, uptr size);
671		void PrintMatchedBenignRaces();
672	501
673	502	#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
674	503	# define DPrintf Printf
...	...	@@ -682,10 +511,11 @@ void PrintMatchedBenignRaces();
682	511	# define DPrintf2(...)
683	512	#endif
684	513
685		u32 CurrentStackId(ThreadState *thr, uptr pc);
686		ReportStack *SymbolizeStackId(u32 stack_id);
	514	StackID CurrentStackId(ThreadState *thr, uptr pc);
	515	ReportStack *SymbolizeStackId(StackID stack_id);
687	516	void PrintCurrentStack(ThreadState *thr, uptr pc);
688	517	void PrintCurrentStackSlow(uptr pc); // uses libunwind
	518	MBlock JavaHeapBlock(uptr addr, uptr start);
689	519
690	520	void Initialize(ThreadState *thr);
691	521	void MaybeSpawnBackgroundThread();
...	...	@@ -694,69 +524,49 @@ int Finalize(ThreadState *thr);
694	524	void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
695	525	void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);
696	526
697		void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
698		int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic);
699		void MemoryAccessImpl(ThreadState *thr, uptr addr,
700		int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
701		u64 *shadow_mem, Shadow cur);
702		void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
703		uptr size, bool is_write);
704		void MemoryAccessRangeStep(ThreadState *thr, uptr pc, uptr addr,
705		uptr size, uptr step, bool is_write);
706		void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr,
707		int size, bool kAccessIsWrite, bool kIsAtomic);
708
709		const int kSizeLog1 = 0;
710		const int kSizeLog2 = 1;
711		const int kSizeLog4 = 2;
712		const int kSizeLog8 = 3;
713
714		void ALWAYS_INLINE MemoryRead(ThreadState *thr, uptr pc,
715		uptr addr, int kAccessSizeLog) {
716		MemoryAccess(thr, pc, addr, kAccessSizeLog, false, false);
717		}
718
719		void ALWAYS_INLINE MemoryWrite(ThreadState *thr, uptr pc,
720		uptr addr, int kAccessSizeLog) {
721		MemoryAccess(thr, pc, addr, kAccessSizeLog, true, false);
722		}
723
724		void ALWAYS_INLINE MemoryReadAtomic(ThreadState *thr, uptr pc,
725		uptr addr, int kAccessSizeLog) {
726		MemoryAccess(thr, pc, addr, kAccessSizeLog, false, true);
727		}
728
729		void ALWAYS_INLINE MemoryWriteAtomic(ThreadState *thr, uptr pc,
730		uptr addr, int kAccessSizeLog) {
731		MemoryAccess(thr, pc, addr, kAccessSizeLog, true, true);
	527	void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
	528	AccessType typ);
	529	void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
	530	AccessType typ);
	531	// This creates 2 non-inlined specialized versions of MemoryAccessRange.
	532	template <bool is_read>
	533	void MemoryAccessRangeT(ThreadState *thr, uptr pc, uptr addr, uptr size);
	534
	535	ALWAYS_INLINE
	536	void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
	537	bool is_write) {
	538	if (size == 0)
	539	return;
	540	if (is_write)
	541	MemoryAccessRangeT<false>(thr, pc, addr, size);
	542	else
	543	MemoryAccessRangeT<true>(thr, pc, addr, size);
732	544	}
733	545
734		void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
	546	void ShadowSet(RawShadow p, RawShadow end, RawShadow v);
735	547	void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
	548	void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
736	549	void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
737	550	void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
738	551	uptr size);
739	552
740		void ThreadIgnoreBegin(ThreadState *thr, uptr pc, bool save_stack = true);
741		void ThreadIgnoreEnd(ThreadState *thr, uptr pc);
742		void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc, bool save_stack = true);
743		void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc);
744
745		void FuncEntry(ThreadState *thr, uptr pc);
746		void FuncExit(ThreadState *thr);
	553	void ThreadIgnoreBegin(ThreadState *thr, uptr pc);
	554	void ThreadIgnoreEnd(ThreadState *thr);
	555	void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc);
	556	void ThreadIgnoreSyncEnd(ThreadState *thr);
747	557
748		int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
749		void ThreadStart(ThreadState *thr, int tid, tid_t os_id,
	558	Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
	559	void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
750	560	ThreadType thread_type);
751	561	void ThreadFinish(ThreadState *thr);
752		int ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid);
753		void ThreadJoin(ThreadState *thr, uptr pc, int tid);
754		void ThreadDetach(ThreadState *thr, uptr pc, int tid);
	562	Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid);
	563	void ThreadJoin(ThreadState *thr, uptr pc, Tid tid);
	564	void ThreadDetach(ThreadState *thr, uptr pc, Tid tid);
755	565	void ThreadFinalize(ThreadState *thr);
756	566	void ThreadSetName(ThreadState thr, const char name);
757	567	int ThreadCount(ThreadState *thr);
758		void ProcessPendingSignals(ThreadState *thr);
759		void ThreadNotJoined(ThreadState *thr, uptr pc, int tid, uptr uid);
	568	void ProcessPendingSignalsImpl(ThreadState *thr);
	569	void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid);
760	570
761	571	Processor *ProcCreate();
762	572	void ProcDestroy(Processor *proc);
...	...	@@ -785,65 +595,12 @@ void Acquire(ThreadState *thr, uptr pc, uptr addr);
785	595	// handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
786	596	// right before executing finalizers. This provides a coarse, but simple
787	597	// approximation of the actual required synchronization.
788		void AcquireGlobal(ThreadState *thr, uptr pc);
	598	void AcquireGlobal(ThreadState *thr);
789	599	void Release(ThreadState *thr, uptr pc, uptr addr);
790	600	void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr);
791	601	void ReleaseStore(ThreadState *thr, uptr pc, uptr addr);
792	602	void AfterSleep(ThreadState *thr, uptr pc);
793		void AcquireImpl(ThreadState thr, uptr pc, SyncClock c);
794		void ReleaseImpl(ThreadState thr, uptr pc, SyncClock c);
795		void ReleaseStoreAcquireImpl(ThreadState thr, uptr pc, SyncClock c);
796		void ReleaseStoreImpl(ThreadState thr, uptr pc, SyncClock c);
797		void AcquireReleaseImpl(ThreadState thr, uptr pc, SyncClock c);
798
799		// The hacky call uses custom calling convention and an assembly thunk.
800		// It is considerably faster that a normal call for the caller
801		// if it is not executed (it is intended for slow paths from hot functions).
802		// The trick is that the call preserves all registers and the compiler
803		// does not treat it as a call.
804		// If it does not work for you, use normal call.
805		#if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC
806		// The caller may not create the stack frame for itself at all,
807		// so we create a reserve stack frame for it (1024b must be enough).
808		#define HACKY_CALL(f) \
809		__asm__ __volatile__("sub $1024, %%rsp;" \
810		CFI_INL_ADJUST_CFA_OFFSET(1024) \
811		".hidden " #f "_thunk;" \
812		"call " #f "_thunk;" \
813		"add $1024, %%rsp;" \
814		CFI_INL_ADJUST_CFA_OFFSET(-1024) \
815		::: "memory", "cc");
816		#else
817		#define HACKY_CALL(f) f()
818		#endif
819
820		void TraceSwitch(ThreadState *thr);
821		uptr TraceTopPC(ThreadState *thr);
822		uptr TraceSize();
823		uptr TraceParts();
824		Trace *ThreadTrace(int tid);
825
826		extern "C" void __tsan_trace_switch();
827		void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs,
828		EventType typ, u64 addr) {
829		if (!kCollectHistory)
830		return;
831		DCHECK_GE((int)typ, 0);
832		DCHECK_LE((int)typ, 7);
833		DCHECK_EQ(GetLsb(addr, kEventPCBits), addr);
834		u64 pos = fs.GetTracePos();
835		if (UNLIKELY((pos % kTracePartSize) == 0)) {
836		#if !SANITIZER_GO
837		HACKY_CALL(__tsan_trace_switch);
838		#else
839		TraceSwitch(thr);
840		#endif
841		}
842		Event trace = (Event)GetThreadTrace(fs.tid());
843		Event *evp = &trace[pos];
844		Event ev = (u64)addr \| ((u64)typ << kEventPCBits);
845		*evp = ev;
846		}
	603	void IncrementEpoch(ThreadState *thr);
847	604
848	605	#if !SANITIZER_GO
849	606	uptr ALWAYS_INLINE HeapEnd() {
...	...	@@ -851,6 +608,13 @@ uptr ALWAYS_INLINE HeapEnd() {
851	608	}
852	609	#endif
853	610
	611	void SlotAttachAndLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx);
	612	void SlotDetach(ThreadState *thr);
	613	void SlotLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx);
	614	void SlotUnlock(ThreadState *thr) SANITIZER_RELEASE(thr->slot->mtx);
	615	void DoReset(ThreadState *thr, uptr epoch);
	616	void FlushShadowMemory();
	617
854	618	ThreadState FiberCreate(ThreadState thr, uptr pc, unsigned flags);
855	619	void FiberDestroy(ThreadState thr, uptr pc, ThreadState fiber);
856	620	void FiberSwitch(ThreadState thr, uptr pc, ThreadState fiber, unsigned flags);
...	...	@@ -861,6 +625,189 @@ enum FiberSwitchFlags {
861	625	FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync
862	626	};
863	627
	628	class SlotLocker {
	629	public:
	630	ALWAYS_INLINE
	631	SlotLocker(ThreadState *thr, bool recursive = false)
	632	: thr_(thr), locked_(recursive ? thr->slot_locked : false) {
	633	#if !SANITIZER_GO
	634	// We are in trouble if we are here with in_blocking_func set.
	635	// If in_blocking_func is set, all signals will be delivered synchronously,
	636	// which means we can't lock slots since the signal handler will try
	637	// to lock it recursively and deadlock.
	638	DCHECK(!atomic_load(&thr->in_blocking_func, memory_order_relaxed));
	639	#endif
	640	if (!locked_)
	641	SlotLock(thr_);
	642	}
	643
	644	ALWAYS_INLINE
	645	~SlotLocker() {
	646	if (!locked_)
	647	SlotUnlock(thr_);
	648	}
	649
	650	private:
	651	ThreadState *thr_;
	652	bool locked_;
	653	};
	654
	655	class SlotUnlocker {
	656	public:
	657	SlotUnlocker(ThreadState *thr) : thr_(thr), locked_(thr->slot_locked) {
	658	if (locked_)
	659	SlotUnlock(thr_);
	660	}
	661
	662	~SlotUnlocker() {
	663	if (locked_)
	664	SlotLock(thr_);
	665	}
	666
	667	private:
	668	ThreadState *thr_;
	669	bool locked_;
	670	};
	671
	672	ALWAYS_INLINE void ProcessPendingSignals(ThreadState *thr) {
	673	if (UNLIKELY(atomic_load_relaxed(&thr->pending_signals)))
	674	ProcessPendingSignalsImpl(thr);
	675	}
	676
	677	extern bool is_initialized;
	678
	679	ALWAYS_INLINE
	680	void LazyInitialize(ThreadState *thr) {
	681	// If we can use .preinit_array, assume that __tsan_init
	682	// called from .preinit_array initializes runtime before
	683	// any instrumented code except when tsan is used as a
	684	// shared library.
	685	#if (!SANITIZER_CAN_USE_PREINIT_ARRAY \|\| defined(SANITIZER_SHARED))
	686	if (UNLIKELY(!is_initialized))
	687	Initialize(thr);
	688	#endif
	689	}
	690
	691	void TraceResetForTesting();
	692	void TraceSwitchPart(ThreadState *thr);
	693	void TraceSwitchPartImpl(ThreadState *thr);
	694	bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size,
	695	AccessType typ, Tid ptid, VarSizeStackTrace pstk,
	696	MutexSet pmset, uptr ptag);
	697
	698	template <typename EventT>
	699	ALWAYS_INLINE WARN_UNUSED_RESULT bool TraceAcquire(ThreadState *thr,
	700	EventT **ev) {
	701	// TraceSwitchPart accesses shadow_stack, but it's called infrequently,
	702	// so we check it here proactively.
	703	DCHECK(thr->shadow_stack);
	704	Event pos = reinterpret_cast<Event >(atomic_load_relaxed(&thr->trace_pos));
	705	#if SANITIZER_DEBUG
	706	// TraceSwitch acquires these mutexes,
	707	// so we lock them here to detect deadlocks more reliably.
	708	{ Lock lock(&ctx->slot_mtx); }
	709	{ Lock lock(&thr->tctx->trace.mtx); }
	710	TracePart *current = thr->tctx->trace.parts.Back();
	711	if (current) {
	712	DCHECK_GE(pos, &current->events[0]);
	713	DCHECK_LE(pos, &current->events[TracePart::kSize]);
	714	} else {
	715	DCHECK_EQ(pos, nullptr);
	716	}
	717	#endif
	718	// TracePart is allocated with mmap and is at least 4K aligned.
	719	// So the following check is a faster way to check for part end.
	720	// It may have false positives in the middle of the trace,
	721	// they are filtered out in TraceSwitch.
	722	if (UNLIKELY(((uptr)(pos + 1) & TracePart::kAlignment) == 0))
	723	return false;
	724	ev = reinterpret_cast<EventT >(pos);
	725	return true;
	726	}
	727
	728	template <typename EventT>
	729	ALWAYS_INLINE void TraceRelease(ThreadState thr, EventT evp) {
	730	DCHECK_LE(evp + 1, &thr->tctx->trace.parts.Back()->events[TracePart::kSize]);
	731	atomic_store_relaxed(&thr->trace_pos, (uptr)(evp + 1));
	732	}
	733
	734	template <typename EventT>
	735	void TraceEvent(ThreadState *thr, EventT ev) {
	736	EventT *evp;
	737	if (!TraceAcquire(thr, &evp)) {
	738	TraceSwitchPart(thr);
	739	UNUSED bool res = TraceAcquire(thr, &evp);
	740	DCHECK(res);
	741	}
	742	*evp = ev;
	743	TraceRelease(thr, evp);
	744	}
	745
	746	ALWAYS_INLINE WARN_UNUSED_RESULT bool TryTraceFunc(ThreadState *thr,
	747	uptr pc = 0) {
	748	if (!kCollectHistory)
	749	return true;
	750	EventFunc *ev;
	751	if (UNLIKELY(!TraceAcquire(thr, &ev)))
	752	return false;
	753	ev->is_access = 0;
	754	ev->is_func = 1;
	755	ev->pc = pc;
	756	TraceRelease(thr, ev);
	757	return true;
	758	}
	759
	760	WARN_UNUSED_RESULT
	761	bool TryTraceMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
	762	AccessType typ);
	763	WARN_UNUSED_RESULT
	764	bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
	765	AccessType typ);
	766	void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
	767	AccessType typ);
	768	void TraceFunc(ThreadState *thr, uptr pc = 0);
	769	void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr,
	770	StackID stk);
	771	void TraceMutexUnlock(ThreadState *thr, uptr addr);
	772	void TraceTime(ThreadState *thr);
	773
	774	void TraceRestartFuncExit(ThreadState *thr);
	775	void TraceRestartFuncEntry(ThreadState *thr, uptr pc);
	776
	777	void GrowShadowStack(ThreadState *thr);
	778
	779	ALWAYS_INLINE
	780	void FuncEntry(ThreadState *thr, uptr pc) {
	781	DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.sid(), (void *)pc);
	782	if (UNLIKELY(!TryTraceFunc(thr, pc)))
	783	return TraceRestartFuncEntry(thr, pc);
	784	DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
	785	#if !SANITIZER_GO
	786	DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
	787	#else
	788	if (thr->shadow_stack_pos == thr->shadow_stack_end)
	789	GrowShadowStack(thr);
	790	#endif
	791	thr->shadow_stack_pos[0] = pc;
	792	thr->shadow_stack_pos++;
	793	}
	794
	795	ALWAYS_INLINE
	796	void FuncExit(ThreadState *thr) {
	797	DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.sid());
	798	if (UNLIKELY(!TryTraceFunc(thr, 0)))
	799	return TraceRestartFuncExit(thr);
	800	DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
	801	#if !SANITIZER_GO
	802	DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
	803	#endif
	804	thr->shadow_stack_pos--;
	805	}
	806
	807	#if !SANITIZER_GO
	808	extern void (*on_initialize)(void);
	809	extern int (*on_finalize)(int);
	810	#endif
864	811	} // namespace __tsan
865	812
866	813	#endif // TSAN_RTL_H

lib/tsan/tsan_rtl_aarch64.S+6-31

...	...	@@ -3,28 +3,6 @@
3	3
4	4	#include "sanitizer_common/sanitizer_asm.h"
5	5
6		#if defined(__APPLE__)
7		.align 2
8
9		.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
10		.long _setjmp$non_lazy_ptr
11		_setjmp$non_lazy_ptr:
12		.indirect_symbol _setjmp
13		.long 0
14
15		.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
16		.long __setjmp$non_lazy_ptr
17		__setjmp$non_lazy_ptr:
18		.indirect_symbol __setjmp
19		.long 0
20
21		.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
22		.long _sigsetjmp$non_lazy_ptr
23		_sigsetjmp$non_lazy_ptr:
24		.indirect_symbol _sigsetjmp
25		.long 0
26		#endif
27
28	6	#if !defined(__APPLE__)
29	7	.section .text
30	8	#else
...	...	@@ -75,9 +53,8 @@ ASM_SYMBOL_INTERCEPTOR(setjmp):
75	53	ldr x1, [x1, #:got_lo12:_ZN14__interception11real_setjmpE]
76	54	ldr x1, [x1]
77	55	#else
78		adrp x1, _setjmp$non_lazy_ptr@page
79		add x1, x1, _setjmp$non_lazy_ptr@pageoff
80		ldr x1, [x1]
	56	adrp x1, _setjmp@GOTPAGE
	57	ldr x1, [x1, _setjmp@GOTPAGEOFF]
81	58	#endif
82	59	br x1
83	60
...	...	@@ -126,9 +103,8 @@ ASM_SYMBOL_INTERCEPTOR(_setjmp):
126	103	ldr x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE]
127	104	ldr x1, [x1]
128	105	#else
129		adrp x1, __setjmp$non_lazy_ptr@page
130		add x1, x1, __setjmp$non_lazy_ptr@pageoff
131		ldr x1, [x1]
	106	adrp x1, __setjmp@GOTPAGE
	107	ldr x1, [x1, __setjmp@GOTPAGEOFF]
132	108	#endif
133	109	br x1
134	110
...	...	@@ -179,9 +155,8 @@ ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
179	155	ldr x2, [x2, #:got_lo12:_ZN14__interception14real_sigsetjmpE]
180	156	ldr x2, [x2]
181	157	#else
182		adrp x2, _sigsetjmp$non_lazy_ptr@page
183		add x2, x2, _sigsetjmp$non_lazy_ptr@pageoff
184		ldr x2, [x2]
	158	adrp x2, _sigsetjmp@GOTPAGE
	159	ldr x2, [x2, _sigsetjmp@GOTPAGEOFF]
185	160	#endif
186	161	br x2
187	162	CFI_ENDPROC

lib/tsan/tsan_rtl_access.cpp created+744

...	...	@@ -0,0 +1,744 @@
	1	//===-- tsan_rtl_access.cpp -----------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	// Definitions of memory access and function entry/exit entry points.
	12	//===----------------------------------------------------------------------===//
	13
	14	#include "tsan_rtl.h"
	15
	16	namespace __tsan {
	17
	18	ALWAYS_INLINE USED bool TryTraceMemoryAccess(ThreadState* thr, uptr pc,
	19	uptr addr, uptr size,
	20	AccessType typ) {
	21	DCHECK(size == 1 \|\| size == 2 \|\| size == 4 \|\| size == 8);
	22	if (!kCollectHistory)
	23	return true;
	24	EventAccess* ev;
	25	if (UNLIKELY(!TraceAcquire(thr, &ev)))
	26	return false;
	27	u64 size_log = size == 1 ? 0 : size == 2 ? 1 : size == 4 ? 2 : 3;
	28	uptr pc_delta = pc - thr->trace_prev_pc + (1 << (EventAccess::kPCBits - 1));
	29	thr->trace_prev_pc = pc;
	30	if (LIKELY(pc_delta < (1 << EventAccess::kPCBits))) {
	31	ev->is_access = 1;
	32	ev->is_read = !!(typ & kAccessRead);
	33	ev->is_atomic = !!(typ & kAccessAtomic);
	34	ev->size_log = size_log;
	35	ev->pc_delta = pc_delta;
	36	DCHECK_EQ(ev->pc_delta, pc_delta);
	37	ev->addr = CompressAddr(addr);
	38	TraceRelease(thr, ev);
	39	return true;
	40	}
	41	auto* evex = reinterpret_cast<EventAccessExt*>(ev);
	42	evex->is_access = 0;
	43	evex->is_func = 0;
	44	evex->type = EventType::kAccessExt;
	45	evex->is_read = !!(typ & kAccessRead);
	46	evex->is_atomic = !!(typ & kAccessAtomic);
	47	evex->size_log = size_log;
	48	// Note: this is important, see comment in EventAccessExt.
	49	evex->_ = 0;
	50	evex->addr = CompressAddr(addr);
	51	evex->pc = pc;
	52	TraceRelease(thr, evex);
	53	return true;
	54	}
	55
	56	ALWAYS_INLINE
	57	bool TryTraceMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr, uptr size,
	58	AccessType typ) {
	59	if (!kCollectHistory)
	60	return true;
	61	EventAccessRange* ev;
	62	if (UNLIKELY(!TraceAcquire(thr, &ev)))
	63	return false;
	64	thr->trace_prev_pc = pc;
	65	ev->is_access = 0;
	66	ev->is_func = 0;
	67	ev->type = EventType::kAccessRange;
	68	ev->is_read = !!(typ & kAccessRead);
	69	ev->is_free = !!(typ & kAccessFree);
	70	ev->size_lo = size;
	71	ev->pc = CompressAddr(pc);
	72	ev->addr = CompressAddr(addr);
	73	ev->size_hi = size >> EventAccessRange::kSizeLoBits;
	74	TraceRelease(thr, ev);
	75	return true;
	76	}
	77
	78	void TraceMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr, uptr size,
	79	AccessType typ) {
	80	if (LIKELY(TryTraceMemoryAccessRange(thr, pc, addr, size, typ)))
	81	return;
	82	TraceSwitchPart(thr);
	83	UNUSED bool res = TryTraceMemoryAccessRange(thr, pc, addr, size, typ);
	84	DCHECK(res);
	85	}
	86
	87	void TraceFunc(ThreadState* thr, uptr pc) {
	88	if (LIKELY(TryTraceFunc(thr, pc)))
	89	return;
	90	TraceSwitchPart(thr);
	91	UNUSED bool res = TryTraceFunc(thr, pc);
	92	DCHECK(res);
	93	}
	94
	95	NOINLINE void TraceRestartFuncEntry(ThreadState* thr, uptr pc) {
	96	TraceSwitchPart(thr);
	97	FuncEntry(thr, pc);
	98	}
	99
	100	NOINLINE void TraceRestartFuncExit(ThreadState* thr) {
	101	TraceSwitchPart(thr);
	102	FuncExit(thr);
	103	}
	104
	105	void TraceMutexLock(ThreadState* thr, EventType type, uptr pc, uptr addr,
	106	StackID stk) {
	107	DCHECK(type == EventType::kLock \|\| type == EventType::kRLock);
	108	if (!kCollectHistory)
	109	return;
	110	EventLock ev;
	111	ev.is_access = 0;
	112	ev.is_func = 0;
	113	ev.type = type;
	114	ev.pc = CompressAddr(pc);
	115	ev.stack_lo = stk;
	116	ev.stack_hi = stk >> EventLock::kStackIDLoBits;
	117	ev._ = 0;
	118	ev.addr = CompressAddr(addr);
	119	TraceEvent(thr, ev);
	120	}
	121
	122	void TraceMutexUnlock(ThreadState* thr, uptr addr) {
	123	if (!kCollectHistory)
	124	return;
	125	EventUnlock ev;
	126	ev.is_access = 0;
	127	ev.is_func = 0;
	128	ev.type = EventType::kUnlock;
	129	ev._ = 0;
	130	ev.addr = CompressAddr(addr);
	131	TraceEvent(thr, ev);
	132	}
	133
	134	void TraceTime(ThreadState* thr) {
	135	if (!kCollectHistory)
	136	return;
	137	FastState fast_state = thr->fast_state;
	138	EventTime ev;
	139	ev.is_access = 0;
	140	ev.is_func = 0;
	141	ev.type = EventType::kTime;
	142	ev.sid = static_cast<u64>(fast_state.sid());
	143	ev.epoch = static_cast<u64>(fast_state.epoch());
	144	ev._ = 0;
	145	TraceEvent(thr, ev);
	146	}
	147
	148	NOINLINE void DoReportRace(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
	149	Shadow old,
	150	AccessType typ) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
	151	// For the free shadow markers the first element (that contains kFreeSid)
	152	// triggers the race, but the second element contains info about the freeing
	153	// thread, take it.
	154	if (old.sid() == kFreeSid)
	155	old = Shadow(LoadShadow(&shadow_mem[1]));
	156	// This prevents trapping on this address in future.
	157	for (uptr i = 0; i < kShadowCnt; i++)
	158	StoreShadow(&shadow_mem[i], i == 0 ? Shadow::kRodata : Shadow::kEmpty);
	159	// See the comment in MemoryRangeFreed as to why the slot is locked
	160	// for free memory accesses. ReportRace must not be called with
	161	// the slot locked because of the fork. But MemoryRangeFreed is not
	162	// called during fork because fork sets ignore_reads_and_writes,
	163	// so simply unlocking the slot should be fine.
	164	if (typ & kAccessSlotLocked)
	165	SlotUnlock(thr);
	166	ReportRace(thr, shadow_mem, cur, Shadow(old), typ);
	167	if (typ & kAccessSlotLocked)
	168	SlotLock(thr);
	169	}
	170
	171	#if !TSAN_VECTORIZE
	172	ALWAYS_INLINE
	173	bool ContainsSameAccess(RawShadow* s, Shadow cur, int unused0, int unused1,
	174	AccessType typ) {
	175	for (uptr i = 0; i < kShadowCnt; i++) {
	176	auto old = LoadShadow(&s[i]);
	177	if (!(typ & kAccessRead)) {
	178	if (old == cur.raw())
	179	return true;
	180	continue;
	181	}
	182	auto masked = static_cast<RawShadow>(static_cast<u32>(old) \|
	183	static_cast<u32>(Shadow::kRodata));
	184	if (masked == cur.raw())
	185	return true;
	186	if (!(typ & kAccessNoRodata) && !SANITIZER_GO) {
	187	if (old == Shadow::kRodata)
	188	return true;
	189	}
	190	}
	191	return false;
	192	}
	193
	194	ALWAYS_INLINE
	195	bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
	196	int unused0, int unused1, AccessType typ) {
	197	bool stored = false;
	198	for (uptr idx = 0; idx < kShadowCnt; idx++) {
	199	RawShadow* sp = &shadow_mem[idx];
	200	Shadow old(LoadShadow(sp));
	201	if (LIKELY(old.raw() == Shadow::kEmpty)) {
	202	if (!(typ & kAccessCheckOnly) && !stored)
	203	StoreShadow(sp, cur.raw());
	204	return false;
	205	}
	206	if (LIKELY(!(cur.access() & old.access())))
	207	continue;
	208	if (LIKELY(cur.sid() == old.sid())) {
	209	if (!(typ & kAccessCheckOnly) &&
	210	LIKELY(cur.access() == old.access() && old.IsRWWeakerOrEqual(typ))) {
	211	StoreShadow(sp, cur.raw());
	212	stored = true;
	213	}
	214	continue;
	215	}
	216	if (LIKELY(old.IsBothReadsOrAtomic(typ)))
	217	continue;
	218	if (LIKELY(thr->clock.Get(old.sid()) >= old.epoch()))
	219	continue;
	220	DoReportRace(thr, shadow_mem, cur, old, typ);
	221	return true;
	222	}
	223	// We did not find any races and had already stored
	224	// the current access info, so we are done.
	225	if (LIKELY(stored))
	226	return false;
	227	// Choose a random candidate slot and replace it.
	228	uptr index =
	229	atomic_load_relaxed(&thr->trace_pos) / sizeof(Event) % kShadowCnt;
	230	StoreShadow(&shadow_mem[index], cur.raw());
	231	return false;
	232	}
	233
	234	# define LOAD_CURRENT_SHADOW(cur, shadow_mem) UNUSED int access = 0, shadow = 0
	235
	236	#else /* !TSAN_VECTORIZE */
	237
	238	ALWAYS_INLINE
	239	bool ContainsSameAccess(RawShadow* unused0, Shadow unused1, m128 shadow,
	240	m128 access, AccessType typ) {
	241	// Note: we could check if there is a larger access of the same type,
	242	// e.g. we just allocated/memset-ed a block (so it contains 8 byte writes)
	243	// and now do smaller reads/writes, these can also be considered as "same
	244	// access". However, it will make the check more expensive, so it's unclear
	245	// if it's worth it. But this would conserve trace space, so it's useful
	246	// besides potential speed up.
	247	if (!(typ & kAccessRead)) {
	248	const m128 same = _mm_cmpeq_epi32(shadow, access);
	249	return _mm_movemask_epi8(same);
	250	}
	251	// For reads we need to reset read bit in the shadow,
	252	// because we need to match read with both reads and writes.
	253	// Shadow::kRodata has only read bit set, so it does what we want.
	254	// We also abuse it for rodata check to save few cycles
	255	// since we already loaded Shadow::kRodata into a register.
	256	// Reads from rodata can't race.
	257	// Measurements show that they can be 10-20% of all memory accesses.
	258	// Shadow::kRodata has epoch 0 which cannot appear in shadow normally
	259	// (thread epochs start from 1). So the same read bit mask
	260	// serves as rodata indicator.
	261	const m128 read_mask = _mm_set1_epi32(static_cast<u32>(Shadow::kRodata));
	262	const m128 masked_shadow = _mm_or_si128(shadow, read_mask);
	263	m128 same = _mm_cmpeq_epi32(masked_shadow, access);
	264	// Range memory accesses check Shadow::kRodata before calling this,
	265	// Shadow::kRodatas is not possible for free memory access
	266	// and Go does not use Shadow::kRodata.
	267	if (!(typ & kAccessNoRodata) && !SANITIZER_GO) {
	268	const m128 ro = _mm_cmpeq_epi32(shadow, read_mask);
	269	same = _mm_or_si128(ro, same);
	270	}
	271	return _mm_movemask_epi8(same);
	272	}
	273
	274	NOINLINE void DoReportRaceV(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
	275	u32 race_mask, m128 shadow, AccessType typ) {
	276	// race_mask points which of the shadow elements raced with the current
	277	// access. Extract that element.
	278	CHECK_NE(race_mask, 0);
	279	u32 old;
	280	// Note: _mm_extract_epi32 index must be a constant value.
	281	switch (__builtin_ffs(race_mask) / 4) {
	282	case 0:
	283	old = _mm_extract_epi32(shadow, 0);
	284	break;
	285	case 1:
	286	old = _mm_extract_epi32(shadow, 1);
	287	break;
	288	case 2:
	289	old = _mm_extract_epi32(shadow, 2);
	290	break;
	291	case 3:
	292	old = _mm_extract_epi32(shadow, 3);
	293	break;
	294	}
	295	Shadow prev(static_cast<RawShadow>(old));
	296	// For the free shadow markers the first element (that contains kFreeSid)
	297	// triggers the race, but the second element contains info about the freeing
	298	// thread, take it.
	299	if (prev.sid() == kFreeSid)
	300	prev = Shadow(static_cast<RawShadow>(_mm_extract_epi32(shadow, 1)));
	301	DoReportRace(thr, shadow_mem, cur, prev, typ);
	302	}
	303
	304	ALWAYS_INLINE
	305	bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
	306	m128 shadow, m128 access, AccessType typ) {
	307	// Note: empty/zero slots don't intersect with any access.
	308	const m128 zero = _mm_setzero_si128();
	309	const m128 mask_access = _mm_set1_epi32(0x000000ff);
	310	const m128 mask_sid = _mm_set1_epi32(0x0000ff00);
	311	const m128 mask_read_atomic = _mm_set1_epi32(0xc0000000);
	312	const m128 access_and = _mm_and_si128(access, shadow);
	313	const m128 access_xor = _mm_xor_si128(access, shadow);
	314	const m128 intersect = _mm_and_si128(access_and, mask_access);
	315	const m128 not_intersect = _mm_cmpeq_epi32(intersect, zero);
	316	const m128 not_same_sid = _mm_and_si128(access_xor, mask_sid);
	317	const m128 same_sid = _mm_cmpeq_epi32(not_same_sid, zero);
	318	const m128 both_read_or_atomic = _mm_and_si128(access_and, mask_read_atomic);
	319	const m128 no_race =
	320	_mm_or_si128(_mm_or_si128(not_intersect, same_sid), both_read_or_atomic);
	321	const int race_mask = _mm_movemask_epi8(_mm_cmpeq_epi32(no_race, zero));
	322	if (UNLIKELY(race_mask))
	323	goto SHARED;
	324
	325	STORE : {
	326	if (typ & kAccessCheckOnly)
	327	return false;
	328	// We could also replace different sid's if access is the same,
	329	// rw weaker and happens before. However, just checking access below
	330	// is not enough because we also need to check that !both_read_or_atomic
	331	// (reads from different sids can be concurrent).
	332	// Theoretically we could replace smaller accesses with larger accesses,
	333	// but it's unclear if it's worth doing.
	334	const m128 mask_access_sid = _mm_set1_epi32(0x0000ffff);
	335	const m128 not_same_sid_access = _mm_and_si128(access_xor, mask_access_sid);
	336	const m128 same_sid_access = _mm_cmpeq_epi32(not_same_sid_access, zero);
	337	const m128 access_read_atomic =
	338	_mm_set1_epi32((typ & (kAccessRead \| kAccessAtomic)) << 30);
	339	const m128 rw_weaker =
	340	_mm_cmpeq_epi32(_mm_max_epu32(shadow, access_read_atomic), shadow);
	341	const m128 rewrite = _mm_and_si128(same_sid_access, rw_weaker);
	342	const int rewrite_mask = _mm_movemask_epi8(rewrite);
	343	int index = __builtin_ffs(rewrite_mask);
	344	if (UNLIKELY(index == 0)) {
	345	const m128 empty = _mm_cmpeq_epi32(shadow, zero);
	346	const int empty_mask = _mm_movemask_epi8(empty);
	347	index = __builtin_ffs(empty_mask);
	348	if (UNLIKELY(index == 0))
	349	index = (atomic_load_relaxed(&thr->trace_pos) / 2) % 16;
	350	}
	351	StoreShadow(&shadow_mem[index / 4], cur.raw());
	352	// We could zero other slots determined by rewrite_mask.
	353	// That would help other threads to evict better slots,
	354	// but it's unclear if it's worth it.
	355	return false;
	356	}
	357
	358	SHARED:
	359	m128 thread_epochs = _mm_set1_epi32(0x7fffffff);
	360	// Need to unwind this because _mm_extract_epi8/_mm_insert_epi32
	361	// indexes must be constants.
	362	# define LOAD_EPOCH(idx) \
	363	if (LIKELY(race_mask & (1 << (idx * 4)))) { \
	364	u8 sid = _mm_extract_epi8(shadow, idx * 4 + 1); \
	365	u16 epoch = static_cast<u16>(thr->clock.Get(static_cast<Sid>(sid))); \
	366	thread_epochs = _mm_insert_epi32(thread_epochs, u32(epoch) << 16, idx); \
	367	}
	368	LOAD_EPOCH(0);
	369	LOAD_EPOCH(1);
	370	LOAD_EPOCH(2);
	371	LOAD_EPOCH(3);
	372	# undef LOAD_EPOCH
	373	const m128 mask_epoch = _mm_set1_epi32(0x3fff0000);
	374	const m128 shadow_epochs = _mm_and_si128(shadow, mask_epoch);
	375	const m128 concurrent = _mm_cmplt_epi32(thread_epochs, shadow_epochs);
	376	const int concurrent_mask = _mm_movemask_epi8(concurrent);
	377	if (LIKELY(concurrent_mask == 0))
	378	goto STORE;
	379
	380	DoReportRaceV(thr, shadow_mem, cur, concurrent_mask, shadow, typ);
	381	return true;
	382	}
	383
	384	# define LOAD_CURRENT_SHADOW(cur, shadow_mem) \
	385	const m128 access = _mm_set1_epi32(static_cast<u32>((cur).raw())); \
	386	const m128 shadow = _mm_load_si128(reinterpret_cast<m128*>(shadow_mem))
	387	#endif
	388
	389	char* DumpShadow(char* buf, RawShadow raw) {
	390	if (raw == Shadow::kEmpty) {
	391	internal_snprintf(buf, 64, "0");
	392	return buf;
	393	}
	394	Shadow s(raw);
	395	AccessType typ;
	396	s.GetAccess(nullptr, nullptr, &typ);
	397	internal_snprintf(buf, 64, "{tid=%u@%u access=0x%x typ=%x}",
	398	static_cast<u32>(s.sid()), static_cast<u32>(s.epoch()),
	399	s.access(), static_cast<u32>(typ));
	400	return buf;
	401	}
	402
	403	// TryTrace* and TraceRestart* functions allow to turn memory access and func
	404	// entry/exit callbacks into leaf functions with all associated performance
	405	// benefits. These hottest callbacks do only 2 slow path calls: report a race
	406	// and trace part switching. Race reporting is easy to turn into a tail call, we
	407	// just always return from the runtime after reporting a race. But trace part
	408	// switching is harder because it needs to be in the middle of callbacks. To
	409	// turn it into a tail call we immidiately return after TraceRestart* functions,
	410	// but TraceRestart* functions themselves recurse into the callback after
	411	// switching trace part. As the result the hottest callbacks contain only tail
	412	// calls, which effectively makes them leaf functions (can use all registers,
	413	// no frame setup, etc).
	414	NOINLINE void TraceRestartMemoryAccess(ThreadState* thr, uptr pc, uptr addr,
	415	uptr size, AccessType typ) {
	416	TraceSwitchPart(thr);
	417	MemoryAccess(thr, pc, addr, size, typ);
	418	}
	419
	420	ALWAYS_INLINE USED void MemoryAccess(ThreadState* thr, uptr pc, uptr addr,
	421	uptr size, AccessType typ) {
	422	RawShadow* shadow_mem = MemToShadow(addr);
	423	UNUSED char memBuf[4][64];
	424	DPrintf2("#%d: Access: %d@%d %p/%zd typ=0x%x {%s, %s, %s, %s}\n", thr->tid,
	425	static_cast<int>(thr->fast_state.sid()),
	426	static_cast<int>(thr->fast_state.epoch()), (void*)addr, size,
	427	static_cast<int>(typ), DumpShadow(memBuf[0], shadow_mem[0]),
	428	DumpShadow(memBuf[1], shadow_mem[1]),
	429	DumpShadow(memBuf[2], shadow_mem[2]),
	430	DumpShadow(memBuf[3], shadow_mem[3]));
	431
	432	FastState fast_state = thr->fast_state;
	433	Shadow cur(fast_state, addr, size, typ);
	434
	435	LOAD_CURRENT_SHADOW(cur, shadow_mem);
	436	if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
	437	return;
	438	if (UNLIKELY(fast_state.GetIgnoreBit()))
	439	return;
	440	if (!TryTraceMemoryAccess(thr, pc, addr, size, typ))
	441	return TraceRestartMemoryAccess(thr, pc, addr, size, typ);
	442	CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
	443	}
	444
	445	void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr, AccessType typ);
	446
	447	NOINLINE
	448	void RestartMemoryAccess16(ThreadState* thr, uptr pc, uptr addr,
	449	AccessType typ) {
	450	TraceSwitchPart(thr);
	451	MemoryAccess16(thr, pc, addr, typ);
	452	}
	453
	454	ALWAYS_INLINE USED void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr,
	455	AccessType typ) {
	456	const uptr size = 16;
	457	FastState fast_state = thr->fast_state;
	458	if (UNLIKELY(fast_state.GetIgnoreBit()))
	459	return;
	460	Shadow cur(fast_state, 0, 8, typ);
	461	RawShadow* shadow_mem = MemToShadow(addr);
	462	bool traced = false;
	463	{
	464	LOAD_CURRENT_SHADOW(cur, shadow_mem);
	465	if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
	466	goto SECOND;
	467	if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
	468	return RestartMemoryAccess16(thr, pc, addr, typ);
	469	traced = true;
	470	if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
	471	return;
	472	}
	473	SECOND:
	474	shadow_mem += kShadowCnt;
	475	LOAD_CURRENT_SHADOW(cur, shadow_mem);
	476	if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
	477	return;
	478	if (!traced && !TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
	479	return RestartMemoryAccess16(thr, pc, addr, typ);
	480	CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
	481	}
	482
	483	NOINLINE
	484	void RestartUnalignedMemoryAccess(ThreadState* thr, uptr pc, uptr addr,
	485	uptr size, AccessType typ) {
	486	TraceSwitchPart(thr);
	487	UnalignedMemoryAccess(thr, pc, addr, size, typ);
	488	}
	489
	490	ALWAYS_INLINE USED void UnalignedMemoryAccess(ThreadState* thr, uptr pc,
	491	uptr addr, uptr size,
	492	AccessType typ) {
	493	DCHECK_LE(size, 8);
	494	FastState fast_state = thr->fast_state;
	495	if (UNLIKELY(fast_state.GetIgnoreBit()))
	496	return;
	497	RawShadow* shadow_mem = MemToShadow(addr);
	498	bool traced = false;
	499	uptr size1 = Min<uptr>(size, RoundUp(addr + 1, kShadowCell) - addr);
	500	{
	501	Shadow cur(fast_state, addr, size1, typ);
	502	LOAD_CURRENT_SHADOW(cur, shadow_mem);
	503	if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
	504	goto SECOND;
	505	if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
	506	return RestartUnalignedMemoryAccess(thr, pc, addr, size, typ);
	507	traced = true;
	508	if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
	509	return;
	510	}
	511	SECOND:
	512	uptr size2 = size - size1;
	513	if (LIKELY(size2 == 0))
	514	return;
	515	shadow_mem += kShadowCnt;
	516	Shadow cur(fast_state, 0, size2, typ);
	517	LOAD_CURRENT_SHADOW(cur, shadow_mem);
	518	if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
	519	return;
	520	if (!traced && !TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
	521	return RestartUnalignedMemoryAccess(thr, pc, addr, size, typ);
	522	CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
	523	}
	524
	525	void ShadowSet(RawShadow* p, RawShadow* end, RawShadow v) {
	526	DCHECK_LE(p, end);
	527	DCHECK(IsShadowMem(p));
	528	DCHECK(IsShadowMem(end));
	529	UNUSED const uptr kAlign = kShadowCnt * kShadowSize;
	530	DCHECK_EQ(reinterpret_cast<uptr>(p) % kAlign, 0);
	531	DCHECK_EQ(reinterpret_cast<uptr>(end) % kAlign, 0);
	532	#if !TSAN_VECTORIZE
	533	for (; p < end; p += kShadowCnt) {
	534	p[0] = v;
	535	for (uptr i = 1; i < kShadowCnt; i++) p[i] = Shadow::kEmpty;
	536	}
	537	#else
	538	m128 vv = _mm_setr_epi32(
	539	static_cast<u32>(v), static_cast<u32>(Shadow::kEmpty),
	540	static_cast<u32>(Shadow::kEmpty), static_cast<u32>(Shadow::kEmpty));
	541	m128* vp = reinterpret_cast<m128*>(p);
	542	m128* vend = reinterpret_cast<m128*>(end);
	543	for (; vp < vend; vp++) _mm_store_si128(vp, vv);
	544	#endif
	545	}
	546
	547	static void MemoryRangeSet(uptr addr, uptr size, RawShadow val) {
	548	if (size == 0)
	549	return;
	550	DCHECK_EQ(addr % kShadowCell, 0);
	551	DCHECK_EQ(size % kShadowCell, 0);
	552	// If a user passes some insane arguments (memset(0)),
	553	// let it just crash as usual.
	554	if (!IsAppMem(addr) \|\| !IsAppMem(addr + size - 1))
	555	return;
	556	RawShadow* begin = MemToShadow(addr);
	557	RawShadow* end = begin + size / kShadowCell * kShadowCnt;
	558	// Don't want to touch lots of shadow memory.
	559	// If a program maps 10MB stack, there is no need reset the whole range.
	560	// UnmapOrDie/MmapFixedNoReserve does not work on Windows.
	561	if (SANITIZER_WINDOWS \|\|
	562	size <= common_flags()->clear_shadow_mmap_threshold) {
	563	ShadowSet(begin, end, val);
	564	return;
	565	}
	566	// The region is big, reset only beginning and end.
	567	const uptr kPageSize = GetPageSizeCached();
	568	// Set at least first kPageSize/2 to page boundary.
	569	RawShadow* mid1 =
	570	Min(end, reinterpret_cast<RawShadow*>(RoundUp(
	571	reinterpret_cast<uptr>(begin) + kPageSize / 2, kPageSize)));
	572	ShadowSet(begin, mid1, val);
	573	// Reset middle part.
	574	RawShadow* mid2 = RoundDown(end, kPageSize);
	575	if (mid2 > mid1) {
	576	if (!MmapFixedSuperNoReserve((uptr)mid1, (uptr)mid2 - (uptr)mid1))
	577	Die();
	578	}
	579	// Set the ending.
	580	ShadowSet(mid2, end, val);
	581	}
	582
	583	void MemoryResetRange(ThreadState* thr, uptr pc, uptr addr, uptr size) {
	584	uptr addr1 = RoundDown(addr, kShadowCell);
	585	uptr size1 = RoundUp(size + addr - addr1, kShadowCell);
	586	MemoryRangeSet(addr1, size1, Shadow::kEmpty);
	587	}
	588
	589	void MemoryRangeFreed(ThreadState* thr, uptr pc, uptr addr, uptr size) {
	590	// Callers must lock the slot to ensure synchronization with the reset.
	591	// The problem with "freed" memory is that it's not "monotonic"
	592	// with respect to bug detection: freed memory is bad to access,
	593	// but then if the heap block is reallocated later, it's good to access.
	594	// As the result a garbage "freed" shadow can lead to a false positive
	595	// if it happens to match a real free in the thread trace,
	596	// but the heap block was reallocated before the current memory access,
	597	// so it's still good to access. It's not the case with data races.
	598	DCHECK(thr->slot_locked);
	599	DCHECK_EQ(addr % kShadowCell, 0);
	600	size = RoundUp(size, kShadowCell);
	601	// Processing more than 1k (2k of shadow) is expensive,
	602	// can cause excessive memory consumption (user does not necessary touch
	603	// the whole range) and most likely unnecessary.
	604	size = Min<uptr>(size, 1024);
	605	const AccessType typ = kAccessWrite \| kAccessFree \| kAccessSlotLocked \|
	606	kAccessCheckOnly \| kAccessNoRodata;
	607	TraceMemoryAccessRange(thr, pc, addr, size, typ);
	608	RawShadow* shadow_mem = MemToShadow(addr);
	609	Shadow cur(thr->fast_state, 0, kShadowCell, typ);
	610	#if TSAN_VECTORIZE
	611	const m128 access = _mm_set1_epi32(static_cast<u32>(cur.raw()));
	612	const m128 freed = _mm_setr_epi32(
	613	static_cast<u32>(Shadow::FreedMarker()),
	614	static_cast<u32>(Shadow::FreedInfo(cur.sid(), cur.epoch())), 0, 0);
	615	for (; size; size -= kShadowCell, shadow_mem += kShadowCnt) {
	616	const m128 shadow = _mm_load_si128((m128*)shadow_mem);
	617	if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
	618	return;
	619	_mm_store_si128((m128*)shadow_mem, freed);
	620	}
	621	#else
	622	for (; size; size -= kShadowCell, shadow_mem += kShadowCnt) {
	623	if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, 0, 0, typ)))
	624	return;
	625	StoreShadow(&shadow_mem[0], Shadow::FreedMarker());
	626	StoreShadow(&shadow_mem[1], Shadow::FreedInfo(cur.sid(), cur.epoch()));
	627	StoreShadow(&shadow_mem[2], Shadow::kEmpty);
	628	StoreShadow(&shadow_mem[3], Shadow::kEmpty);
	629	}
	630	#endif
	631	}
	632
	633	void MemoryRangeImitateWrite(ThreadState* thr, uptr pc, uptr addr, uptr size) {
	634	DCHECK_EQ(addr % kShadowCell, 0);
	635	size = RoundUp(size, kShadowCell);
	636	TraceMemoryAccessRange(thr, pc, addr, size, kAccessWrite);
	637	Shadow cur(thr->fast_state, 0, 8, kAccessWrite);
	638	MemoryRangeSet(addr, size, cur.raw());
	639	}
	640
	641	void MemoryRangeImitateWriteOrResetRange(ThreadState* thr, uptr pc, uptr addr,
	642	uptr size) {
	643	if (thr->ignore_reads_and_writes == 0)
	644	MemoryRangeImitateWrite(thr, pc, addr, size);
	645	else
	646	MemoryResetRange(thr, pc, addr, size);
	647	}
	648
	649	ALWAYS_INLINE
	650	bool MemoryAccessRangeOne(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
	651	AccessType typ) {
	652	LOAD_CURRENT_SHADOW(cur, shadow_mem);
	653	if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
	654	return false;
	655	return CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
	656	}
	657
	658	template <bool is_read>
	659	NOINLINE void RestartMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr,
	660	uptr size) {
	661	TraceSwitchPart(thr);
	662	MemoryAccessRangeT<is_read>(thr, pc, addr, size);
	663	}
	664
	665	template <bool is_read>
	666	void MemoryAccessRangeT(ThreadState* thr, uptr pc, uptr addr, uptr size) {
	667	const AccessType typ =
	668	(is_read ? kAccessRead : kAccessWrite) \| kAccessNoRodata;
	669	RawShadow* shadow_mem = MemToShadow(addr);
	670	DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_read=%d\n", thr->tid,
	671	(void)pc, (void)addr, (int)size, is_read);
	672
	673	#if SANITIZER_DEBUG
	674	if (!IsAppMem(addr)) {
	675	Printf("Access to non app mem %zx\n", addr);
	676	DCHECK(IsAppMem(addr));
	677	}
	678	if (!IsAppMem(addr + size - 1)) {
	679	Printf("Access to non app mem %zx\n", addr + size - 1);
	680	DCHECK(IsAppMem(addr + size - 1));
	681	}
	682	if (!IsShadowMem(shadow_mem)) {
	683	Printf("Bad shadow addr %p (%zx)\n", static_cast<void*>(shadow_mem), addr);
	684	DCHECK(IsShadowMem(shadow_mem));
	685	}
	686	if (!IsShadowMem(shadow_mem + size * kShadowCnt - 1)) {
	687	Printf("Bad shadow addr %p (%zx)\n",
	688	static_cast<void>(shadow_mem + size kShadowCnt - 1),
	689	addr + size - 1);
	690	DCHECK(IsShadowMem(shadow_mem + size * kShadowCnt - 1));
	691	}
	692	#endif
	693
	694	// Access to .rodata section, no races here.
	695	// Measurements show that it can be 10-20% of all memory accesses.
	696	// Check here once to not check for every access separately.
	697	// Note: we could (and should) do this only for the is_read case
	698	// (writes shouldn't go to .rodata). But it happens in Chromium tests:
	699	// https://bugs.chromium.org/p/chromium/issues/detail?id=1275581#c19
	700	// Details are unknown since it happens only on CI machines.
	701	if (*shadow_mem == Shadow::kRodata)
	702	return;
	703
	704	FastState fast_state = thr->fast_state;
	705	if (UNLIKELY(fast_state.GetIgnoreBit()))
	706	return;
	707
	708	if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
	709	return RestartMemoryAccessRange<is_read>(thr, pc, addr, size);
	710
	711	if (UNLIKELY(addr % kShadowCell)) {
	712	// Handle unaligned beginning, if any.
	713	uptr size1 = Min(size, RoundUp(addr, kShadowCell) - addr);
	714	size -= size1;
	715	Shadow cur(fast_state, addr, size1, typ);
	716	if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ)))
	717	return;
	718	shadow_mem += kShadowCnt;
	719	}
	720	// Handle middle part, if any.
	721	Shadow cur(fast_state, 0, kShadowCell, typ);
	722	for (; size >= kShadowCell; size -= kShadowCell, shadow_mem += kShadowCnt) {
	723	if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ)))
	724	return;
	725	}
	726	// Handle ending, if any.
	727	if (UNLIKELY(size)) {
	728	Shadow cur(fast_state, 0, size, typ);
	729	if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ)))
	730	return;
	731	}
	732	}
	733
	734	template void MemoryAccessRangeT<true>(ThreadState* thr, uptr pc, uptr addr,
	735	uptr size);
	736	template void MemoryAccessRangeT<false>(ThreadState* thr, uptr pc, uptr addr,
	737	uptr size);
	738
	739	} // namespace __tsan
	740
	741	#if !SANITIZER_GO
	742	// Must be included in this file to make sure everything is inlined.
	743	# include "tsan_interface.inc"
	744	#endif

lib/tsan/tsan_rtl_amd64.S+4-160

...	...	@@ -9,166 +9,6 @@
9	9	.section __TEXT,__text
10	10	#endif
11	11
12		ASM_HIDDEN(__tsan_trace_switch)
13		.globl ASM_SYMBOL(__tsan_trace_switch_thunk)
14		ASM_SYMBOL(__tsan_trace_switch_thunk):
15		CFI_STARTPROC
16		# Save scratch registers.
17		push %rax
18		CFI_ADJUST_CFA_OFFSET(8)
19		CFI_REL_OFFSET(%rax, 0)
20		push %rcx
21		CFI_ADJUST_CFA_OFFSET(8)
22		CFI_REL_OFFSET(%rcx, 0)
23		push %rdx
24		CFI_ADJUST_CFA_OFFSET(8)
25		CFI_REL_OFFSET(%rdx, 0)
26		push %rsi
27		CFI_ADJUST_CFA_OFFSET(8)
28		CFI_REL_OFFSET(%rsi, 0)
29		push %rdi
30		CFI_ADJUST_CFA_OFFSET(8)
31		CFI_REL_OFFSET(%rdi, 0)
32		push %r8
33		CFI_ADJUST_CFA_OFFSET(8)
34		CFI_REL_OFFSET(%r8, 0)
35		push %r9
36		CFI_ADJUST_CFA_OFFSET(8)
37		CFI_REL_OFFSET(%r9, 0)
38		push %r10
39		CFI_ADJUST_CFA_OFFSET(8)
40		CFI_REL_OFFSET(%r10, 0)
41		push %r11
42		CFI_ADJUST_CFA_OFFSET(8)
43		CFI_REL_OFFSET(%r11, 0)
44		# Align stack frame.
45		push %rbx # non-scratch
46		CFI_ADJUST_CFA_OFFSET(8)
47		CFI_REL_OFFSET(%rbx, 0)
48		mov %rsp, %rbx # save current rsp
49		CFI_DEF_CFA_REGISTER(%rbx)
50		shr $4, %rsp # clear 4 lsb, align to 16
51		shl $4, %rsp
52
53		call ASM_SYMBOL(__tsan_trace_switch)
54
55		# Unalign stack frame back.
56		mov %rbx, %rsp # restore the original rsp
57		CFI_DEF_CFA_REGISTER(%rsp)
58		pop %rbx
59		CFI_ADJUST_CFA_OFFSET(-8)
60		# Restore scratch registers.
61		pop %r11
62		CFI_ADJUST_CFA_OFFSET(-8)
63		pop %r10
64		CFI_ADJUST_CFA_OFFSET(-8)
65		pop %r9
66		CFI_ADJUST_CFA_OFFSET(-8)
67		pop %r8
68		CFI_ADJUST_CFA_OFFSET(-8)
69		pop %rdi
70		CFI_ADJUST_CFA_OFFSET(-8)
71		pop %rsi
72		CFI_ADJUST_CFA_OFFSET(-8)
73		pop %rdx
74		CFI_ADJUST_CFA_OFFSET(-8)
75		pop %rcx
76		CFI_ADJUST_CFA_OFFSET(-8)
77		pop %rax
78		CFI_ADJUST_CFA_OFFSET(-8)
79		CFI_RESTORE(%rax)
80		CFI_RESTORE(%rbx)
81		CFI_RESTORE(%rcx)
82		CFI_RESTORE(%rdx)
83		CFI_RESTORE(%rsi)
84		CFI_RESTORE(%rdi)
85		CFI_RESTORE(%r8)
86		CFI_RESTORE(%r9)
87		CFI_RESTORE(%r10)
88		CFI_RESTORE(%r11)
89		ret
90		CFI_ENDPROC
91
92		ASM_HIDDEN(__tsan_report_race)
93		.globl ASM_SYMBOL(__tsan_report_race_thunk)
94		ASM_SYMBOL(__tsan_report_race_thunk):
95		CFI_STARTPROC
96		# Save scratch registers.
97		push %rax
98		CFI_ADJUST_CFA_OFFSET(8)
99		CFI_REL_OFFSET(%rax, 0)
100		push %rcx
101		CFI_ADJUST_CFA_OFFSET(8)
102		CFI_REL_OFFSET(%rcx, 0)
103		push %rdx
104		CFI_ADJUST_CFA_OFFSET(8)
105		CFI_REL_OFFSET(%rdx, 0)
106		push %rsi
107		CFI_ADJUST_CFA_OFFSET(8)
108		CFI_REL_OFFSET(%rsi, 0)
109		push %rdi
110		CFI_ADJUST_CFA_OFFSET(8)
111		CFI_REL_OFFSET(%rdi, 0)
112		push %r8
113		CFI_ADJUST_CFA_OFFSET(8)
114		CFI_REL_OFFSET(%r8, 0)
115		push %r9
116		CFI_ADJUST_CFA_OFFSET(8)
117		CFI_REL_OFFSET(%r9, 0)
118		push %r10
119		CFI_ADJUST_CFA_OFFSET(8)
120		CFI_REL_OFFSET(%r10, 0)
121		push %r11
122		CFI_ADJUST_CFA_OFFSET(8)
123		CFI_REL_OFFSET(%r11, 0)
124		# Align stack frame.
125		push %rbx # non-scratch
126		CFI_ADJUST_CFA_OFFSET(8)
127		CFI_REL_OFFSET(%rbx, 0)
128		mov %rsp, %rbx # save current rsp
129		CFI_DEF_CFA_REGISTER(%rbx)
130		shr $4, %rsp # clear 4 lsb, align to 16
131		shl $4, %rsp
132
133		call ASM_SYMBOL(__tsan_report_race)
134
135		# Unalign stack frame back.
136		mov %rbx, %rsp # restore the original rsp
137		CFI_DEF_CFA_REGISTER(%rsp)
138		pop %rbx
139		CFI_ADJUST_CFA_OFFSET(-8)
140		# Restore scratch registers.
141		pop %r11
142		CFI_ADJUST_CFA_OFFSET(-8)
143		pop %r10
144		CFI_ADJUST_CFA_OFFSET(-8)
145		pop %r9
146		CFI_ADJUST_CFA_OFFSET(-8)
147		pop %r8
148		CFI_ADJUST_CFA_OFFSET(-8)
149		pop %rdi
150		CFI_ADJUST_CFA_OFFSET(-8)
151		pop %rsi
152		CFI_ADJUST_CFA_OFFSET(-8)
153		pop %rdx
154		CFI_ADJUST_CFA_OFFSET(-8)
155		pop %rcx
156		CFI_ADJUST_CFA_OFFSET(-8)
157		pop %rax
158		CFI_ADJUST_CFA_OFFSET(-8)
159		CFI_RESTORE(%rax)
160		CFI_RESTORE(%rbx)
161		CFI_RESTORE(%rcx)
162		CFI_RESTORE(%rdx)
163		CFI_RESTORE(%rsi)
164		CFI_RESTORE(%rdi)
165		CFI_RESTORE(%r8)
166		CFI_RESTORE(%r9)
167		CFI_RESTORE(%r10)
168		CFI_RESTORE(%r11)
169		ret
170		CFI_ENDPROC
171
172	12	ASM_HIDDEN(__tsan_setjmp)
173	13	#if defined(__NetBSD__)
174	14	.comm _ZN14__interception15real___setjmp14E,8,8
...	...	@@ -185,6 +25,7 @@ ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
185	25	ASM_SYMBOL_INTERCEPTOR(setjmp):
186	26	#endif
187	27	CFI_STARTPROC
	28	_CET_ENDBR
188	29	// save env parameter
189	30	push %rdi
190	31	CFI_ADJUST_CFA_OFFSET(8)
...	...	@@ -226,6 +67,7 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
226	67	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
227	68	ASM_SYMBOL_INTERCEPTOR(_setjmp):
228	69	CFI_STARTPROC
	70	_CET_ENDBR
229	71	// save env parameter
230	72	push %rdi
231	73	CFI_ADJUST_CFA_OFFSET(8)
...	...	@@ -267,6 +109,7 @@ ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
267	109	ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
268	110	#endif
269	111	CFI_STARTPROC
	112	_CET_ENDBR
270	113	// save env parameter
271	114	push %rdi
272	115	CFI_ADJUST_CFA_OFFSET(8)
...	...	@@ -323,6 +166,7 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
323	166	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
324	167	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
325	168	CFI_STARTPROC
	169	_CET_ENDBR
326	170	// save env parameter
327	171	push %rdi
328	172	CFI_ADJUST_CFA_OFFSET(8)

lib/tsan/tsan_rtl_mutex.cpp+369-338

...	...	@@ -23,6 +23,8 @@
23	23	namespace __tsan {
24	24
25	25	void ReportDeadlock(ThreadState thr, uptr pc, DDReport r);
	26	void ReportDestroyLocked(ThreadState *thr, uptr pc, uptr addr,
	27	FastState last_lock, StackID creation_stack_id);
26	28
27	29	struct Callback final : public DDCallback {
28	30	ThreadState *thr;
...	...	@@ -35,27 +37,27 @@ struct Callback final : public DDCallback {
35	37	DDCallback::lt = thr->dd_lt;
36	38	}
37	39
38		u32 Unwind() override { return CurrentStackId(thr, pc); }
39		int UniqueTid() override { return thr->unique_id; }
	40	StackID Unwind() override { return CurrentStackId(thr, pc); }
	41	int UniqueTid() override { return thr->tid; }
40	42	};
41	43
42	44	void DDMutexInit(ThreadState thr, uptr pc, SyncVar s) {
43	45	Callback cb(thr, pc);
44	46	ctx->dd->MutexInit(&cb, &s->dd);
45		s->dd.ctx = s->GetId();
	47	s->dd.ctx = s->addr;
46	48	}
47	49
48	50	static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,
49		uptr addr, u64 mid) {
	51	uptr addr, StackID creation_stack_id) {
50	52	// In Go, these misuses are either impossible, or detected by std lib,
51	53	// or false positives (e.g. unlock in a different thread).
52	54	if (SANITIZER_GO)
53	55	return;
54	56	if (!ShouldReport(thr, typ))
55	57	return;
56		ThreadRegistryLock l(ctx->thread_registry);
	58	ThreadRegistryLock l(&ctx->thread_registry);
57	59	ScopedReport rep(typ);
58		rep.AddMutex(mid);
	60	rep.AddMutex(addr, creation_stack_id);
59	61	VarSizeStackTrace trace;
60	62	ObtainCurrentStack(thr, pc, &trace);
61	63	rep.AddStack(trace, true);
...	...	@@ -63,185 +65,197 @@ static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,
63	65	OutputReport(thr, rep);
64	66	}
65	67
66		void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
	68	static void RecordMutexLock(ThreadState *thr, uptr pc, uptr addr,
	69	StackID stack_id, bool write) {
	70	auto typ = write ? EventType::kLock : EventType::kRLock;
	71	// Note: it's important to trace before modifying mutex set
	72	// because tracing can switch trace part and we write the current
	73	// mutex set in the beginning of each part.
	74	// If we do it in the opposite order, we will write already reduced
	75	// mutex set in the beginning of the part and then trace unlock again.
	76	TraceMutexLock(thr, typ, pc, addr, stack_id);
	77	thr->mset.AddAddr(addr, stack_id, write);
	78	}
	79
	80	static void RecordMutexUnlock(ThreadState *thr, uptr addr) {
	81	// See the comment in RecordMutexLock re order of operations.
	82	TraceMutexUnlock(thr, addr);
	83	thr->mset.DelAddr(addr);
	84	}
	85
	86	void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
67	87	DPrintf("#%d: MutexCreate %zx flagz=0x%x\n", thr->tid, addr, flagz);
68		if (!(flagz & MutexFlagLinkerInit) && IsAppMem(addr)) {
69		CHECK(!thr->is_freeing);
70		thr->is_freeing = true;
71		MemoryWrite(thr, pc, addr, kSizeLog1);
72		thr->is_freeing = false;
73		}
74		SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
	88	if (!(flagz & MutexFlagLinkerInit) && pc && IsAppMem(addr))
	89	MemoryAccess(thr, pc, addr, 1, kAccessWrite);
	90	SlotLocker locker(thr);
	91	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
75	92	s->SetFlags(flagz & MutexCreationFlagMask);
76		if (!SANITIZER_GO && s->creation_stack_id == 0)
	93	// Save stack in the case the sync object was created before as atomic.
	94	if (!SANITIZER_GO && s->creation_stack_id == kInvalidStackID)
77	95	s->creation_stack_id = CurrentStackId(thr, pc);
78		s->mtx.Unlock();
79	96	}
80	97
81		void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
	98	void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
82	99	DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr);
83		SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
84		if (s == 0)
85		return;
86		if ((flagz & MutexFlagLinkerInit)
87		\|\| s->IsFlagSet(MutexFlagLinkerInit)
88		\|\| ((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) {
89		// Destroy is no-op for linker-initialized mutexes.
90		s->mtx.Unlock();
91		return;
92		}
93		if (common_flags()->detect_deadlocks) {
94		Callback cb(thr, pc);
95		ctx->dd->MutexDestroy(&cb, &s->dd);
96		ctx->dd->MutexInit(&cb, &s->dd);
97		}
98	100	bool unlock_locked = false;
99		if (flags()->report_destroy_locked && s->owner_tid != kInvalidTid &&
100		!s->IsFlagSet(MutexFlagBroken)) {
101		s->SetFlags(MutexFlagBroken);
102		unlock_locked = true;
103		}
104		u64 mid = s->GetId();
105		u64 last_lock = s->last_lock;
106		if (!unlock_locked)
107		s->Reset(thr->proc()); // must not reset it before the report is printed
108		s->mtx.Unlock();
109		if (unlock_locked && ShouldReport(thr, ReportTypeMutexDestroyLocked)) {
110		ThreadRegistryLock l(ctx->thread_registry);
111		ScopedReport rep(ReportTypeMutexDestroyLocked);
112		rep.AddMutex(mid);
113		VarSizeStackTrace trace;
114		ObtainCurrentStack(thr, pc, &trace);
115		rep.AddStack(trace, true);
116		FastState last(last_lock);
117		RestoreStack(last.tid(), last.epoch(), &trace, 0);
118		rep.AddStack(trace, true);
119		rep.AddLocation(addr, 1);
120		OutputReport(thr, rep);
121
122		SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
123		if (s != 0) {
124		s->Reset(thr->proc());
125		s->mtx.Unlock();
	101	StackID creation_stack_id;
	102	FastState last_lock;
	103	{
	104	auto s = ctx->metamap.GetSyncIfExists(addr);
	105	if (!s)
	106	return;
	107	SlotLocker locker(thr);
	108	{
	109	Lock lock(&s->mtx);
	110	creation_stack_id = s->creation_stack_id;
	111	last_lock = s->last_lock;
	112	if ((flagz & MutexFlagLinkerInit) \|\| s->IsFlagSet(MutexFlagLinkerInit) \|\|
	113	((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) {
	114	// Destroy is no-op for linker-initialized mutexes.
	115	return;
	116	}
	117	if (common_flags()->detect_deadlocks) {
	118	Callback cb(thr, pc);
	119	ctx->dd->MutexDestroy(&cb, &s->dd);
	120	ctx->dd->MutexInit(&cb, &s->dd);
	121	}
	122	if (flags()->report_destroy_locked && s->owner_tid != kInvalidTid &&
	123	!s->IsFlagSet(MutexFlagBroken)) {
	124	s->SetFlags(MutexFlagBroken);
	125	unlock_locked = true;
	126	}
	127	s->Reset();
126	128	}
	129	// Imitate a memory write to catch unlock-destroy races.
	130	if (pc && IsAppMem(addr))
	131	MemoryAccess(thr, pc, addr, 1,
	132	kAccessWrite \| kAccessFree \| kAccessSlotLocked);
127	133	}
128		thr->mset.Remove(mid);
129		// Imitate a memory write to catch unlock-destroy races.
130		// Do this outside of sync mutex, because it can report a race which locks
131		// sync mutexes.
132		if (IsAppMem(addr)) {
133		CHECK(!thr->is_freeing);
134		thr->is_freeing = true;
135		MemoryWrite(thr, pc, addr, kSizeLog1);
136		thr->is_freeing = false;
137		}
	134	if (unlock_locked && ShouldReport(thr, ReportTypeMutexDestroyLocked))
	135	ReportDestroyLocked(thr, pc, addr, last_lock, creation_stack_id);
	136	thr->mset.DelAddr(addr, true);
138	137	// s will be destroyed and freed in MetaMap::FreeBlock.
139	138	}
140	139
141		void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
	140	void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
142	141	DPrintf("#%d: MutexPreLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
143		if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
144		SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
	142	if (flagz & MutexFlagTryLock)
	143	return;
	144	if (!common_flags()->detect_deadlocks)
	145	return;
	146	Callback cb(thr, pc);
	147	{
	148	SlotLocker locker(thr);
	149	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
	150	ReadLock lock(&s->mtx);
145	151	s->UpdateFlags(flagz);
146		if (s->owner_tid != thr->tid) {
147		Callback cb(thr, pc);
	152	if (s->owner_tid != thr->tid)
148	153	ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
149		s->mtx.ReadUnlock();
150		ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
151		} else {
152		s->mtx.ReadUnlock();
153		}
154	154	}
	155	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
155	156	}
156	157
157		void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz,
158		int rec) NO_THREAD_SAFETY_ANALYSIS {
	158	void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) {
159	159	DPrintf("#%d: MutexPostLock %zx flag=0x%x rec=%d\n",
160	160	thr->tid, addr, flagz, rec);
161	161	if (flagz & MutexFlagRecursiveLock)
162	162	CHECK_GT(rec, 0);
163	163	else
164	164	rec = 1;
165		if (IsAppMem(addr))
166		MemoryReadAtomic(thr, pc, addr, kSizeLog1);
167		SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
168		s->UpdateFlags(flagz);
169		thr->fast_state.IncrementEpoch();
170		TraceAddEvent(thr, thr->fast_state, EventTypeLock, s->GetId());
	165	if (pc && IsAppMem(addr))
	166	MemoryAccess(thr, pc, addr, 1, kAccessRead \| kAccessAtomic);
171	167	bool report_double_lock = false;
172		if (s->owner_tid == kInvalidTid) {
173		CHECK_EQ(s->recursion, 0);
174		s->owner_tid = thr->tid;
175		s->last_lock = thr->fast_state.raw();
176		} else if (s->owner_tid == thr->tid) {
177		CHECK_GT(s->recursion, 0);
178		} else if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
179		s->SetFlags(MutexFlagBroken);
180		report_double_lock = true;
181		}
182		const bool first = s->recursion == 0;
183		s->recursion += rec;
184		if (first) {
185		AcquireImpl(thr, pc, &s->clock);
186		AcquireImpl(thr, pc, &s->read_clock);
187		} else if (!s->IsFlagSet(MutexFlagWriteReentrant)) {
188		}
189		thr->mset.Add(s->GetId(), true, thr->fast_state.epoch());
190	168	bool pre_lock = false;
191		if (first && common_flags()->detect_deadlocks) {
192		pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
193		!(flagz & MutexFlagTryLock);
194		Callback cb(thr, pc);
195		if (pre_lock)
196		ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
197		ctx->dd->MutexAfterLock(&cb, &s->dd, true, flagz & MutexFlagTryLock);
	169	bool first = false;
	170	StackID creation_stack_id = kInvalidStackID;
	171	{
	172	SlotLocker locker(thr);
	173	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
	174	creation_stack_id = s->creation_stack_id;
	175	RecordMutexLock(thr, pc, addr, creation_stack_id, true);
	176	{
	177	Lock lock(&s->mtx);
	178	first = s->recursion == 0;
	179	s->UpdateFlags(flagz);
	180	if (s->owner_tid == kInvalidTid) {
	181	CHECK_EQ(s->recursion, 0);
	182	s->owner_tid = thr->tid;
	183	s->last_lock = thr->fast_state;
	184	} else if (s->owner_tid == thr->tid) {
	185	CHECK_GT(s->recursion, 0);
	186	} else if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
	187	s->SetFlags(MutexFlagBroken);
	188	report_double_lock = true;
	189	}
	190	s->recursion += rec;
	191	if (first) {
	192	if (!thr->ignore_sync) {
	193	thr->clock.Acquire(s->clock);
	194	thr->clock.Acquire(s->read_clock);
	195	}
	196	}
	197	if (first && common_flags()->detect_deadlocks) {
	198	pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
	199	!(flagz & MutexFlagTryLock);
	200	Callback cb(thr, pc);
	201	if (pre_lock)
	202	ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
	203	ctx->dd->MutexAfterLock(&cb, &s->dd, true, flagz & MutexFlagTryLock);
	204	}
	205	}
198	206	}
199		u64 mid = s->GetId();
200		s->mtx.Unlock();
201		// Can't touch s after this point.
202		s = 0;
203	207	if (report_double_lock)
204		ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr, mid);
	208	ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr,
	209	creation_stack_id);
205	210	if (first && pre_lock && common_flags()->detect_deadlocks) {
206	211	Callback cb(thr, pc);
207	212	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
208	213	}
209	214	}
210	215
211		int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
	216	int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
212	217	DPrintf("#%d: MutexUnlock %zx flagz=0x%x\n", thr->tid, addr, flagz);
213		if (IsAppMem(addr))
214		MemoryReadAtomic(thr, pc, addr, kSizeLog1);
215		SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
216		thr->fast_state.IncrementEpoch();
217		TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId());
218		int rec = 0;
	218	if (pc && IsAppMem(addr))
	219	MemoryAccess(thr, pc, addr, 1, kAccessRead \| kAccessAtomic);
	220	StackID creation_stack_id;
	221	RecordMutexUnlock(thr, addr);
219	222	bool report_bad_unlock = false;
220		if (!SANITIZER_GO && (s->recursion == 0 \|\| s->owner_tid != thr->tid)) {
221		if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
222		s->SetFlags(MutexFlagBroken);
223		report_bad_unlock = true;
224		}
225		} else {
226		rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1;
227		s->recursion -= rec;
228		if (s->recursion == 0) {
229		s->owner_tid = kInvalidTid;
230		ReleaseStoreImpl(thr, pc, &s->clock);
231		} else {
	223	int rec = 0;
	224	{
	225	SlotLocker locker(thr);
	226	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
	227	bool released = false;
	228	{
	229	Lock lock(&s->mtx);
	230	creation_stack_id = s->creation_stack_id;
	231	if (!SANITIZER_GO && (s->recursion == 0 \|\| s->owner_tid != thr->tid)) {
	232	if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
	233	s->SetFlags(MutexFlagBroken);
	234	report_bad_unlock = true;
	235	}
	236	} else {
	237	rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1;
	238	s->recursion -= rec;
	239	if (s->recursion == 0) {
	240	s->owner_tid = kInvalidTid;
	241	if (!thr->ignore_sync) {
	242	thr->clock.ReleaseStore(&s->clock);
	243	released = true;
	244	}
	245	}
	246	}
	247	if (common_flags()->detect_deadlocks && s->recursion == 0 &&
	248	!report_bad_unlock) {
	249	Callback cb(thr, pc);
	250	ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true);
	251	}
232	252	}
	253	if (released)
	254	IncrementEpoch(thr);
233	255	}
234		thr->mset.Del(s->GetId(), true);
235		if (common_flags()->detect_deadlocks && s->recursion == 0 &&
236		!report_bad_unlock) {
237		Callback cb(thr, pc);
238		ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true);
239		}
240		u64 mid = s->GetId();
241		s->mtx.Unlock();
242		// Can't touch s after this point.
243	256	if (report_bad_unlock)
244		ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid);
	257	ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr,
	258	creation_stack_id);
245	259	if (common_flags()->detect_deadlocks && !report_bad_unlock) {
246	260	Callback cb(thr, pc);
247	261	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
...	...	@@ -249,282 +263,275 @@ int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFET
249	263	return rec;
250	264	}
251	265
252		void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
	266	void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
253	267	DPrintf("#%d: MutexPreReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
254		if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {
255		SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
	268	if ((flagz & MutexFlagTryLock) \|\| !common_flags()->detect_deadlocks)
	269	return;
	270	Callback cb(thr, pc);
	271	{
	272	SlotLocker locker(thr);
	273	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
	274	ReadLock lock(&s->mtx);
256	275	s->UpdateFlags(flagz);
257		Callback cb(thr, pc);
258	276	ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
259		s->mtx.ReadUnlock();
260		ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
261	277	}
	278	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
262	279	}
263	280
264		void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {
	281	void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
265	282	DPrintf("#%d: MutexPostReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
266		if (IsAppMem(addr))
267		MemoryReadAtomic(thr, pc, addr, kSizeLog1);
268		SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
269		s->UpdateFlags(flagz);
270		thr->fast_state.IncrementEpoch();
271		TraceAddEvent(thr, thr->fast_state, EventTypeRLock, s->GetId());
	283	if (pc && IsAppMem(addr))
	284	MemoryAccess(thr, pc, addr, 1, kAccessRead \| kAccessAtomic);
272	285	bool report_bad_lock = false;
273		if (s->owner_tid != kInvalidTid) {
274		if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
275		s->SetFlags(MutexFlagBroken);
276		report_bad_lock = true;
277		}
278		}
279		AcquireImpl(thr, pc, &s->clock);
280		s->last_lock = thr->fast_state.raw();
281		thr->mset.Add(s->GetId(), false, thr->fast_state.epoch());
282	286	bool pre_lock = false;
283		if (common_flags()->detect_deadlocks) {
284		pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
285		!(flagz & MutexFlagTryLock);
286		Callback cb(thr, pc);
287		if (pre_lock)
288		ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
289		ctx->dd->MutexAfterLock(&cb, &s->dd, false, flagz & MutexFlagTryLock);
	287	StackID creation_stack_id = kInvalidStackID;
	288	{
	289	SlotLocker locker(thr);
	290	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
	291	creation_stack_id = s->creation_stack_id;
	292	RecordMutexLock(thr, pc, addr, creation_stack_id, false);
	293	{
	294	ReadLock lock(&s->mtx);
	295	s->UpdateFlags(flagz);
	296	if (s->owner_tid != kInvalidTid) {
	297	if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
	298	s->SetFlags(MutexFlagBroken);
	299	report_bad_lock = true;
	300	}
	301	}
	302	if (!thr->ignore_sync)
	303	thr->clock.Acquire(s->clock);
	304	s->last_lock = thr->fast_state;
	305	if (common_flags()->detect_deadlocks) {
	306	pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
	307	!(flagz & MutexFlagTryLock);
	308	Callback cb(thr, pc);
	309	if (pre_lock)
	310	ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
	311	ctx->dd->MutexAfterLock(&cb, &s->dd, false, flagz & MutexFlagTryLock);
	312	}
	313	}
290	314	}
291		u64 mid = s->GetId();
292		s->mtx.ReadUnlock();
293		// Can't touch s after this point.
294		s = 0;
295	315	if (report_bad_lock)
296		ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr, mid);
	316	ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr,
	317	creation_stack_id);
297	318	if (pre_lock && common_flags()->detect_deadlocks) {
298	319	Callback cb(thr, pc);
299	320	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
300	321	}
301	322	}
302	323
303		void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
	324	void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) {
304	325	DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr);
305		if (IsAppMem(addr))
306		MemoryReadAtomic(thr, pc, addr, kSizeLog1);
307		SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
308		thr->fast_state.IncrementEpoch();
309		TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
	326	if (pc && IsAppMem(addr))
	327	MemoryAccess(thr, pc, addr, 1, kAccessRead \| kAccessAtomic);
	328	RecordMutexUnlock(thr, addr);
	329	StackID creation_stack_id;
310	330	bool report_bad_unlock = false;
311		if (s->owner_tid != kInvalidTid) {
312		if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
313		s->SetFlags(MutexFlagBroken);
314		report_bad_unlock = true;
	331	{
	332	SlotLocker locker(thr);
	333	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
	334	bool released = false;
	335	{
	336	Lock lock(&s->mtx);
	337	creation_stack_id = s->creation_stack_id;
	338	if (s->owner_tid != kInvalidTid) {
	339	if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
	340	s->SetFlags(MutexFlagBroken);
	341	report_bad_unlock = true;
	342	}
	343	}
	344	if (!thr->ignore_sync) {
	345	thr->clock.Release(&s->read_clock);
	346	released = true;
	347	}
	348	if (common_flags()->detect_deadlocks && s->recursion == 0) {
	349	Callback cb(thr, pc);
	350	ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false);
	351	}
315	352	}
	353	if (released)
	354	IncrementEpoch(thr);
316	355	}
317		ReleaseImpl(thr, pc, &s->read_clock);
318		if (common_flags()->detect_deadlocks && s->recursion == 0) {
319		Callback cb(thr, pc);
320		ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false);
321		}
322		u64 mid = s->GetId();
323		s->mtx.Unlock();
324		// Can't touch s after this point.
325		thr->mset.Del(mid, false);
326	356	if (report_bad_unlock)
327		ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr, mid);
	357	ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr,
	358	creation_stack_id);
328	359	if (common_flags()->detect_deadlocks) {
329	360	Callback cb(thr, pc);
330	361	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
331	362	}
332	363	}
333	364
334		void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
	365	void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) {
335	366	DPrintf("#%d: MutexReadOrWriteUnlock %zx\n", thr->tid, addr);
336		if (IsAppMem(addr))
337		MemoryReadAtomic(thr, pc, addr, kSizeLog1);
338		SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
339		bool write = true;
	367	if (pc && IsAppMem(addr))
	368	MemoryAccess(thr, pc, addr, 1, kAccessRead \| kAccessAtomic);
	369	RecordMutexUnlock(thr, addr);
	370	StackID creation_stack_id;
340	371	bool report_bad_unlock = false;
341		if (s->owner_tid == kInvalidTid) {
342		// Seems to be read unlock.
343		write = false;
344		thr->fast_state.IncrementEpoch();
345		TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
346		ReleaseImpl(thr, pc, &s->read_clock);
347		} else if (s->owner_tid == thr->tid) {
348		// Seems to be write unlock.
349		thr->fast_state.IncrementEpoch();
350		TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId());
351		CHECK_GT(s->recursion, 0);
352		s->recursion--;
353		if (s->recursion == 0) {
354		s->owner_tid = kInvalidTid;
355		ReleaseStoreImpl(thr, pc, &s->clock);
356		} else {
	372	bool write = true;
	373	{
	374	SlotLocker locker(thr);
	375	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
	376	bool released = false;
	377	{
	378	Lock lock(&s->mtx);
	379	creation_stack_id = s->creation_stack_id;
	380	if (s->owner_tid == kInvalidTid) {
	381	// Seems to be read unlock.
	382	write = false;
	383	if (!thr->ignore_sync) {
	384	thr->clock.Release(&s->read_clock);
	385	released = true;
	386	}
	387	} else if (s->owner_tid == thr->tid) {
	388	// Seems to be write unlock.
	389	CHECK_GT(s->recursion, 0);
	390	s->recursion--;
	391	if (s->recursion == 0) {
	392	s->owner_tid = kInvalidTid;
	393	if (!thr->ignore_sync) {
	394	thr->clock.ReleaseStore(&s->clock);
	395	released = true;
	396	}
	397	}
	398	} else if (!s->IsFlagSet(MutexFlagBroken)) {
	399	s->SetFlags(MutexFlagBroken);
	400	report_bad_unlock = true;
	401	}
	402	if (common_flags()->detect_deadlocks && s->recursion == 0) {
	403	Callback cb(thr, pc);
	404	ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write);
	405	}
357	406	}
358		} else if (!s->IsFlagSet(MutexFlagBroken)) {
359		s->SetFlags(MutexFlagBroken);
360		report_bad_unlock = true;
	407	if (released)
	408	IncrementEpoch(thr);
361	409	}
362		thr->mset.Del(s->GetId(), write);
363		if (common_flags()->detect_deadlocks && s->recursion == 0) {
364		Callback cb(thr, pc);
365		ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write);
366		}
367		u64 mid = s->GetId();
368		s->mtx.Unlock();
369		// Can't touch s after this point.
370	410	if (report_bad_unlock)
371		ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid);
	411	ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr,
	412	creation_stack_id);
372	413	if (common_flags()->detect_deadlocks) {
373	414	Callback cb(thr, pc);
374	415	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
375	416	}
376	417	}
377	418
378		void MutexRepair(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
	419	void MutexRepair(ThreadState *thr, uptr pc, uptr addr) {
379	420	DPrintf("#%d: MutexRepair %zx\n", thr->tid, addr);
380		SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
	421	SlotLocker locker(thr);
	422	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
	423	Lock lock(&s->mtx);
381	424	s->owner_tid = kInvalidTid;
382	425	s->recursion = 0;
383		s->mtx.Unlock();
384	426	}
385	427
386		void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
	428	void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) {
387	429	DPrintf("#%d: MutexInvalidAccess %zx\n", thr->tid, addr);
388		SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
389		u64 mid = s->GetId();
390		s->mtx.Unlock();
391		ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr, mid);
	430	StackID creation_stack_id = kInvalidStackID;
	431	{
	432	SlotLocker locker(thr);
	433	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
	434	if (s)
	435	creation_stack_id = s->creation_stack_id;
	436	}
	437	ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr,
	438	creation_stack_id);
392	439	}
393	440
394		void Acquire(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
	441	void Acquire(ThreadState *thr, uptr pc, uptr addr) {
395	442	DPrintf("#%d: Acquire %zx\n", thr->tid, addr);
396	443	if (thr->ignore_sync)
397	444	return;
398		SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, false);
	445	auto s = ctx->metamap.GetSyncIfExists(addr);
399	446	if (!s)
400	447	return;
401		AcquireImpl(thr, pc, &s->clock);
402		s->mtx.ReadUnlock();
403		}
404
405		static void UpdateClockCallback(ThreadContextBase tctx_base, void arg) {
406		ThreadState thr = reinterpret_cast<ThreadState>(arg);
407		ThreadContext tctx = static_cast<ThreadContext>(tctx_base);
408		u64 epoch = tctx->epoch1;
409		if (tctx->status == ThreadStatusRunning) {
410		epoch = tctx->thr->fast_state.epoch();
411		tctx->thr->clock.NoteGlobalAcquire(epoch);
412		}
413		thr->clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
	448	SlotLocker locker(thr);
	449	if (!s->clock)
	450	return;
	451	ReadLock lock(&s->mtx);
	452	thr->clock.Acquire(s->clock);
414	453	}
415	454
416		void AcquireGlobal(ThreadState *thr, uptr pc) {
	455	void AcquireGlobal(ThreadState *thr) {
417	456	DPrintf("#%d: AcquireGlobal\n", thr->tid);
418	457	if (thr->ignore_sync)
419	458	return;
420		ThreadRegistryLock l(ctx->thread_registry);
421		ctx->thread_registry->RunCallbackForEachThreadLocked(
422		UpdateClockCallback, thr);
	459	SlotLocker locker(thr);
	460	for (auto &slot : ctx->slots) thr->clock.Set(slot.sid, slot.epoch());
423	461	}
424	462
425		void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
426		DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr);
	463	void Release(ThreadState *thr, uptr pc, uptr addr) {
	464	DPrintf("#%d: Release %zx\n", thr->tid, addr);
427	465	if (thr->ignore_sync)
428	466	return;
429		SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
430		thr->fast_state.IncrementEpoch();
431		// Can't increment epoch w/o writing to the trace as well.
432		TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
433		ReleaseStoreAcquireImpl(thr, pc, &s->clock);
434		s->mtx.Unlock();
	467	SlotLocker locker(thr);
	468	{
	469	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
	470	Lock lock(&s->mtx);
	471	thr->clock.Release(&s->clock);
	472	}
	473	IncrementEpoch(thr);
435	474	}
436	475
437		void Release(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
438		DPrintf("#%d: Release %zx\n", thr->tid, addr);
	476	void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) {
	477	DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr);
439	478	if (thr->ignore_sync)
440	479	return;
441		SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
442		thr->fast_state.IncrementEpoch();
443		// Can't increment epoch w/o writing to the trace as well.
444		TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
445		ReleaseImpl(thr, pc, &s->clock);
446		s->mtx.Unlock();
	480	SlotLocker locker(thr);
	481	{
	482	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
	483	Lock lock(&s->mtx);
	484	thr->clock.ReleaseStore(&s->clock);
	485	}
	486	IncrementEpoch(thr);
447	487	}
448	488
449		void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {
450		DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr);
	489	void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) {
	490	DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr);
451	491	if (thr->ignore_sync)
452	492	return;
453		SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
454		thr->fast_state.IncrementEpoch();
455		// Can't increment epoch w/o writing to the trace as well.
456		TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
457		ReleaseStoreImpl(thr, pc, &s->clock);
458		s->mtx.Unlock();
	493	SlotLocker locker(thr);
	494	{
	495	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
	496	Lock lock(&s->mtx);
	497	thr->clock.ReleaseStoreAcquire(&s->clock);
	498	}
	499	IncrementEpoch(thr);
459	500	}
460	501
461		#if !SANITIZER_GO
462		static void UpdateSleepClockCallback(ThreadContextBase tctx_base, void arg) {
463		ThreadState thr = reinterpret_cast<ThreadState>(arg);
464		ThreadContext tctx = static_cast<ThreadContext>(tctx_base);
465		u64 epoch = tctx->epoch1;
466		if (tctx->status == ThreadStatusRunning)
467		epoch = tctx->thr->fast_state.epoch();
468		thr->last_sleep_clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
	502	void IncrementEpoch(ThreadState *thr) {
	503	DCHECK(!thr->ignore_sync);
	504	DCHECK(thr->slot_locked);
	505	Epoch epoch = EpochInc(thr->fast_state.epoch());
	506	if (!EpochOverflow(epoch)) {
	507	Sid sid = thr->fast_state.sid();
	508	thr->clock.Set(sid, epoch);
	509	thr->fast_state.SetEpoch(epoch);
	510	thr->slot->SetEpoch(epoch);
	511	TraceTime(thr);
	512	}
469	513	}
470	514
	515	#if !SANITIZER_GO
471	516	void AfterSleep(ThreadState *thr, uptr pc) {
472		DPrintf("#%d: AfterSleep %zx\n", thr->tid);
	517	DPrintf("#%d: AfterSleep\n", thr->tid);
473	518	if (thr->ignore_sync)
474	519	return;
475	520	thr->last_sleep_stack_id = CurrentStackId(thr, pc);
476		ThreadRegistryLock l(ctx->thread_registry);
477		ctx->thread_registry->RunCallbackForEachThreadLocked(
478		UpdateSleepClockCallback, thr);
	521	thr->last_sleep_clock.Reset();
	522	SlotLocker locker(thr);
	523	for (auto &slot : ctx->slots)
	524	thr->last_sleep_clock.Set(slot.sid, slot.epoch());
479	525	}
480	526	#endif
481	527
482		void AcquireImpl(ThreadState thr, uptr pc, SyncClock c) {
483		if (thr->ignore_sync)
484		return;
485		thr->clock.set(thr->fast_state.epoch());
486		thr->clock.acquire(&thr->proc()->clock_cache, c);
487		}
488
489		void ReleaseStoreAcquireImpl(ThreadState thr, uptr pc, SyncClock c) {
490		if (thr->ignore_sync)
491		return;
492		thr->clock.set(thr->fast_state.epoch());
493		thr->fast_synch_epoch = thr->fast_state.epoch();
494		thr->clock.releaseStoreAcquire(&thr->proc()->clock_cache, c);
495		}
496
497		void ReleaseImpl(ThreadState thr, uptr pc, SyncClock c) {
498		if (thr->ignore_sync)
499		return;
500		thr->clock.set(thr->fast_state.epoch());
501		thr->fast_synch_epoch = thr->fast_state.epoch();
502		thr->clock.release(&thr->proc()->clock_cache, c);
503		}
504
505		void ReleaseStoreImpl(ThreadState thr, uptr pc, SyncClock c) {
506		if (thr->ignore_sync)
507		return;
508		thr->clock.set(thr->fast_state.epoch());
509		thr->fast_synch_epoch = thr->fast_state.epoch();
510		thr->clock.ReleaseStore(&thr->proc()->clock_cache, c);
511		}
512
513		void AcquireReleaseImpl(ThreadState thr, uptr pc, SyncClock c) {
514		if (thr->ignore_sync)
515		return;
516		thr->clock.set(thr->fast_state.epoch());
517		thr->fast_synch_epoch = thr->fast_state.epoch();
518		thr->clock.acq_rel(&thr->proc()->clock_cache, c);
519		}
520
521	528	void ReportDeadlock(ThreadState thr, uptr pc, DDReport r) {
522	529	if (r == 0 \|\| !ShouldReport(thr, ReportTypeDeadlock))
523	530	return;
524		ThreadRegistryLock l(ctx->thread_registry);
	531	ThreadRegistryLock l(&ctx->thread_registry);
525	532	ScopedReport rep(ReportTypeDeadlock);
526	533	for (int i = 0; i < r->n; i++) {
527		rep.AddMutex(r->loop[i].mtx_ctx0);
	534	rep.AddMutex(r->loop[i].mtx_ctx0, r->loop[i].stk[0]);
528	535	rep.AddUniqueTid((int)r->loop[i].thr_ctx);
529	536	rep.AddThread((int)r->loop[i].thr_ctx);
530	537	}
...	...	@@ -532,7 +539,7 @@ void ReportDeadlock(ThreadState thr, uptr pc, DDReport r) {
532	539	for (int i = 0; i < r->n; i++) {
533	540	for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) {
534	541	u32 stk = r->loop[i].stk[j];
535		if (stk && stk != 0xffffffff) {
	542	if (stk && stk != kInvalidStackID) {
536	543	rep.AddStack(StackDepotGet(stk), true);
537	544	} else {
538	545	// Sometimes we fail to extract the stack trace (FIXME: investigate),
...	...	@@ -544,4 +551,28 @@ void ReportDeadlock(ThreadState thr, uptr pc, DDReport r) {
544	551	OutputReport(thr, rep);
545	552	}
546	553
	554	void ReportDestroyLocked(ThreadState *thr, uptr pc, uptr addr,
	555	FastState last_lock, StackID creation_stack_id) {
	556	// We need to lock the slot during RestoreStack because it protects
	557	// the slot journal.
	558	Lock slot_lock(&ctx->slots[static_cast<uptr>(last_lock.sid())].mtx);
	559	ThreadRegistryLock l0(&ctx->thread_registry);
	560	Lock slots_lock(&ctx->slot_mtx);
	561	ScopedReport rep(ReportTypeMutexDestroyLocked);
	562	rep.AddMutex(addr, creation_stack_id);
	563	VarSizeStackTrace trace;
	564	ObtainCurrentStack(thr, pc, &trace);
	565	rep.AddStack(trace, true);
	566
	567	Tid tid;
	568	DynamicMutexSet mset;
	569	uptr tag;
	570	if (!RestoreStack(EventType::kLock, last_lock.sid(), last_lock.epoch(), addr,
	571	0, kAccessWrite, &tid, &trace, mset, &tag))
	572	return;
	573	rep.AddStack(trace, true);
	574	rep.AddLocation(addr, 1);
	575	OutputReport(thr, rep);
	576	}
	577
547	578	} // namespace __tsan

lib/tsan/tsan_rtl_proc.cpp-1

...	...	@@ -35,7 +35,6 @@ void ProcDestroy(Processor *proc) {
35	35	#if !SANITIZER_GO
36	36	AllocatorProcFinish(proc);
37	37	#endif
38		ctx->clock_alloc.FlushCache(&proc->clock_cache);
39	38	ctx->metamap.OnProcIdle(proc);
40	39	if (common_flags()->detect_deadlocks)
41	40	ctx->dd->DestroyPhysicalThread(proc->dd_pt);

lib/tsan/tsan_rtl_report.cpp+388-290

...	...	@@ -10,20 +10,20 @@
10	10	//
11	11	//===----------------------------------------------------------------------===//
12	12
	13	#include "sanitizer_common/sanitizer_common.h"
13	14	#include "sanitizer_common/sanitizer_libc.h"
14	15	#include "sanitizer_common/sanitizer_placement_new.h"
15	16	#include "sanitizer_common/sanitizer_stackdepot.h"
16		#include "sanitizer_common/sanitizer_common.h"
17	17	#include "sanitizer_common/sanitizer_stacktrace.h"
	18	#include "tsan_fd.h"
	19	#include "tsan_flags.h"
	20	#include "tsan_mman.h"
18	21	#include "tsan_platform.h"
	22	#include "tsan_report.h"
19	23	#include "tsan_rtl.h"
20	24	#include "tsan_suppressions.h"
21	25	#include "tsan_symbolize.h"
22		#include "tsan_report.h"
23	26	#include "tsan_sync.h"
24		#include "tsan_mman.h"
25		#include "tsan_flags.h"
26		#include "tsan_fd.h"
27	27
28	28	namespace __tsan {
29	29
...	...	@@ -68,8 +68,10 @@ static void StackStripMain(SymbolizedStack *frames) {
68	68	} else if (last && 0 == internal_strcmp(last, "__tsan_thread_start_func")) {
69	69	last_frame->ClearAll();
70	70	last_frame2->next = nullptr;
71		// Strip global ctors init.
72		} else if (last && 0 == internal_strcmp(last, "__do_global_ctors_aux")) {
	71	// Strip global ctors init, .preinit_array and main caller.
	72	} else if (last && (0 == internal_strcmp(last, "__do_global_ctors_aux") \|\|
	73	0 == internal_strcmp(last, "__libc_csu_init") \|\|
	74	0 == internal_strcmp(last, "__libc_start_main"))) {
73	75	last_frame->ClearAll();
74	76	last_frame2->next = nullptr;
75	77	// If both are 0, then we probably just failed to symbolize.
...	...	@@ -120,7 +122,7 @@ static ReportStack *SymbolizeStack(StackTrace trace) {
120	122	}
121	123	StackStripMain(top);
122	124
123		ReportStack *stack = ReportStack::New();
	125	auto *stack = New<ReportStack>();
124	126	stack->frames = top;
125	127	return stack;
126	128	}
...	...	@@ -132,7 +134,7 @@ bool ShouldReport(ThreadState *thr, ReportType typ) {
132	134	CheckedMutex::CheckNoLocks();
133	135	// For the same reason check we didn't lock thread_registry yet.
134	136	if (SANITIZER_DEBUG)
135		ThreadRegistryLock l(ctx->thread_registry);
	137	ThreadRegistryLock l(&ctx->thread_registry);
136	138	if (!flags()->report_bugs \|\| thr->suppress_reports)
137	139	return false;
138	140	switch (typ) {
...	...	@@ -154,9 +156,8 @@ bool ShouldReport(ThreadState *thr, ReportType typ) {
154	156	}
155	157
156	158	ScopedReportBase::ScopedReportBase(ReportType typ, uptr tag) {
157		ctx->thread_registry->CheckLocked();
158		void *mem = internal_alloc(MBlockReport, sizeof(ReportDesc));
159		rep_ = new(mem) ReportDesc;
	159	ctx->thread_registry.CheckLocked();
	160	rep_ = New<ReportDesc>();
160	161	rep_->typ = typ;
161	162	rep_->tag = tag;
162	163	ctx->report_mtx.Lock();
...	...	@@ -165,7 +166,6 @@ ScopedReportBase::ScopedReportBase(ReportType typ, uptr tag) {
165	166	ScopedReportBase::~ScopedReportBase() {
166	167	ctx->report_mtx.Unlock();
167	168	DestroyAndFree(rep_);
168		rep_ = nullptr;
169	169	}
170	170
171	171	void ScopedReportBase::AddStack(StackTrace stack, bool suppressable) {
...	...	@@ -175,28 +175,31 @@ void ScopedReportBase::AddStack(StackTrace stack, bool suppressable) {
175	175	}
176	176
177	177	void ScopedReportBase::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s,
178		StackTrace stack, const MutexSet *mset) {
179		void *mem = internal_alloc(MBlockReportMop, sizeof(ReportMop));
180		ReportMop *mop = new(mem) ReportMop;
	178	Tid tid, StackTrace stack,
	179	const MutexSet *mset) {
	180	uptr addr0, size;
	181	AccessType typ;
	182	s.GetAccess(&addr0, &size, &typ);
	183	auto *mop = New<ReportMop>();
181	184	rep_->mops.PushBack(mop);
182		mop->tid = s.tid();
183		mop->addr = addr + s.addr0();
184		mop->size = s.size();
185		mop->write = s.IsWrite();
186		mop->atomic = s.IsAtomic();
	185	mop->tid = tid;
	186	mop->addr = addr + addr0;
	187	mop->size = size;
	188	mop->write = !(typ & kAccessRead);
	189	mop->atomic = typ & kAccessAtomic;
187	190	mop->stack = SymbolizeStack(stack);
188	191	mop->external_tag = external_tag;
189	192	if (mop->stack)
190	193	mop->stack->suppressable = true;
191	194	for (uptr i = 0; i < mset->Size(); i++) {
192	195	MutexSet::Desc d = mset->Get(i);
193		u64 mid = this->AddMutex(d.id);
194		ReportMopMutex mtx = {mid, d.write};
	196	int id = this->AddMutex(d.addr, d.stack_id);
	197	ReportMopMutex mtx = {id, d.write};
195	198	mop->mset.PushBack(mtx);
196	199	}
197	200	}
198	201
199		void ScopedReportBase::AddUniqueTid(int unique_tid) {
	202	void ScopedReportBase::AddUniqueTid(Tid unique_tid) {
200	203	rep_->unique_tids.PushBack(unique_tid);
201	204	}
202	205
...	...	@@ -205,8 +208,7 @@ void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) {
205	208	if ((u32)rep_->threads[i]->id == tctx->tid)
206	209	return;
207	210	}
208		void *mem = internal_alloc(MBlockReportThread, sizeof(ReportThread));
209		ReportThread *rt = new(mem) ReportThread;
	211	auto *rt = New<ReportThread>();
210	212	rep_->threads.PushBack(rt);
211	213	rt->id = tctx->tid;
212	214	rt->os_id = tctx->os_id;
...	...	@@ -221,22 +223,10 @@ void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) {
221	223	}
222	224
223	225	#if !SANITIZER_GO
224		static bool FindThreadByUidLockedCallback(ThreadContextBase tctx, void arg) {
225		int unique_id = (int )arg;
226		return tctx->unique_id == (u32)unique_id;
227		}
228
229		static ThreadContext *FindThreadByUidLocked(int unique_id) {
230		ctx->thread_registry->CheckLocked();
	226	static ThreadContext *FindThreadByTidLocked(Tid tid) {
	227	ctx->thread_registry.CheckLocked();
231	228	return static_cast<ThreadContext *>(
232		ctx->thread_registry->FindThreadContextLocked(
233		FindThreadByUidLockedCallback, &unique_id));
234		}
235
236		static ThreadContext *FindThreadByTidLocked(int tid) {
237		ctx->thread_registry->CheckLocked();
238		return static_cast<ThreadContext*>(
239		ctx->thread_registry->GetThreadLocked(tid));
	229	ctx->thread_registry.GetThreadLocked(tid));
240	230	}
241	231
242	232	static bool IsInStackOrTls(ThreadContextBase tctx_base, void arg) {
...	...	@@ -251,10 +241,10 @@ static bool IsInStackOrTls(ThreadContextBase tctx_base, void arg) {
251	241	}
252	242
253	243	ThreadContext IsThreadStackOrTls(uptr addr, bool is_stack) {
254		ctx->thread_registry->CheckLocked();
255		ThreadContext tctx = static_cast<ThreadContext>(
256		ctx->thread_registry->FindThreadContextLocked(IsInStackOrTls,
257		(void*)addr));
	244	ctx->thread_registry.CheckLocked();
	245	ThreadContext *tctx =
	246	static_cast<ThreadContext *>(ctx->thread_registry.FindThreadContextLocked(
	247	IsInStackOrTls, (void *)addr));
258	248	if (!tctx)
259	249	return 0;
260	250	ThreadState *thr = tctx->thr;
...	...	@@ -264,58 +254,24 @@ ThreadContext IsThreadStackOrTls(uptr addr, bool is_stack) {
264	254	}
265	255	#endif
266	256
267		void ScopedReportBase::AddThread(int unique_tid, bool suppressable) {
	257	void ScopedReportBase::AddThread(Tid tid, bool suppressable) {
268	258	#if !SANITIZER_GO
269		if (const ThreadContext *tctx = FindThreadByUidLocked(unique_tid))
	259	if (const ThreadContext *tctx = FindThreadByTidLocked(tid))
270	260	AddThread(tctx, suppressable);
271	261	#endif
272	262	}
273	263
274		void ScopedReportBase::AddMutex(const SyncVar *s) {
275		for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
276		if (rep_->mutexes[i]->id == s->uid)
277		return;
278		}
279		void *mem = internal_alloc(MBlockReportMutex, sizeof(ReportMutex));
280		ReportMutex *rm = new(mem) ReportMutex;
281		rep_->mutexes.PushBack(rm);
282		rm->id = s->uid;
283		rm->addr = s->addr;
284		rm->destroyed = false;
285		rm->stack = SymbolizeStackId(s->creation_stack_id);
286		}
287
288		u64 ScopedReportBase::AddMutex(u64 id) NO_THREAD_SAFETY_ANALYSIS {
289		u64 uid = 0;
290		u64 mid = id;
291		uptr addr = SyncVar::SplitId(id, &uid);
292		SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
293		// Check that the mutex is still alive.
294		// Another mutex can be created at the same address,
295		// so check uid as well.
296		if (s && s->CheckId(uid)) {
297		mid = s->uid;
298		AddMutex(s);
299		} else {
300		AddDeadMutex(id);
301		}
302		if (s)
303		s->mtx.Unlock();
304		return mid;
305		}
306
307		void ScopedReportBase::AddDeadMutex(u64 id) {
	264	int ScopedReportBase::AddMutex(uptr addr, StackID creation_stack_id) {
308	265	for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
309		if (rep_->mutexes[i]->id == id)
310		return;
	266	if (rep_->mutexes[i]->addr == addr)
	267	return rep_->mutexes[i]->id;
311	268	}
312		void *mem = internal_alloc(MBlockReportMutex, sizeof(ReportMutex));
313		ReportMutex *rm = new(mem) ReportMutex;
	269	auto *rm = New<ReportMutex>();
314	270	rep_->mutexes.PushBack(rm);
315		rm->id = id;
316		rm->addr = 0;
317		rm->destroyed = true;
318		rm->stack = 0;
	271	rm->id = rep_->mutexes.Size() - 1;
	272	rm->addr = addr;
	273	rm->stack = SymbolizeStackId(creation_stack_id);
	274	return rm->id;
319	275	}
320	276
321	277	void ScopedReportBase::AddLocation(uptr addr, uptr size) {
...	...	@@ -323,43 +279,46 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) {
323	279	return;
324	280	#if !SANITIZER_GO
325	281	int fd = -1;
326		int creat_tid = kInvalidTid;
327		u32 creat_stack = 0;
328		if (FdLocation(addr, &fd, &creat_tid, &creat_stack)) {
329		ReportLocation *loc = ReportLocation::New(ReportLocationFD);
	282	Tid creat_tid = kInvalidTid;
	283	StackID creat_stack = 0;
	284	bool closed = false;
	285	if (FdLocation(addr, &fd, &creat_tid, &creat_stack, &closed)) {
	286	auto *loc = New<ReportLocation>();
	287	loc->type = ReportLocationFD;
	288	loc->fd_closed = closed;
330	289	loc->fd = fd;
331	290	loc->tid = creat_tid;
332	291	loc->stack = SymbolizeStackId(creat_stack);
333	292	rep_->locs.PushBack(loc);
334		ThreadContext *tctx = FindThreadByUidLocked(creat_tid);
335		if (tctx)
336		AddThread(tctx);
	293	AddThread(creat_tid);
337	294	return;
338	295	}
339	296	MBlock *b = 0;
	297	uptr block_begin = 0;
340	298	Allocator *a = allocator();
341	299	if (a->PointerIsMine((void*)addr)) {
342		void block_begin = a->GetBlockBegin((void)addr);
	300	block_begin = (uptr)a->GetBlockBegin((void *)addr);
343	301	if (block_begin)
344		b = ctx->metamap.GetBlock((uptr)block_begin);
	302	b = ctx->metamap.GetBlock(block_begin);
345	303	}
	304	if (!b)
	305	b = JavaHeapBlock(addr, &block_begin);
346	306	if (b != 0) {
347		ThreadContext *tctx = FindThreadByTidLocked(b->tid);
348		ReportLocation *loc = ReportLocation::New(ReportLocationHeap);
349		loc->heap_chunk_start = (uptr)allocator()->GetBlockBegin((void *)addr);
	307	auto *loc = New<ReportLocation>();
	308	loc->type = ReportLocationHeap;
	309	loc->heap_chunk_start = block_begin;
350	310	loc->heap_chunk_size = b->siz;
351	311	loc->external_tag = b->tag;
352		loc->tid = tctx ? tctx->tid : b->tid;
	312	loc->tid = b->tid;
353	313	loc->stack = SymbolizeStackId(b->stk);
354	314	rep_->locs.PushBack(loc);
355		if (tctx)
356		AddThread(tctx);
	315	AddThread(b->tid);
357	316	return;
358	317	}
359	318	bool is_stack = false;
360	319	if (ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack)) {
361		ReportLocation *loc =
362		ReportLocation::New(is_stack ? ReportLocationStack : ReportLocationTLS);
	320	auto *loc = New<ReportLocation>();
	321	loc->type = is_stack ? ReportLocationStack : ReportLocationTLS;
363	322	loc->tid = tctx->tid;
364	323	rep_->locs.PushBack(loc);
365	324	AddThread(tctx);
...	...	@@ -373,13 +332,15 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) {
373	332	}
374	333
375	334	#if !SANITIZER_GO
376		void ScopedReportBase::AddSleep(u32 stack_id) {
	335	void ScopedReportBase::AddSleep(StackID stack_id) {
377	336	rep_->sleep = SymbolizeStackId(stack_id);
378	337	}
379	338	#endif
380	339
381	340	void ScopedReportBase::SetCount(int count) { rep_->count = count; }
382	341
	342	void ScopedReportBase::SetSigNum(int sig) { rep_->signum = sig; }
	343
383	344	const ReportDesc *ScopedReportBase::GetReport() const { return rep_; }
384	345
385	346	ScopedReport::ScopedReport(ReportType typ, uptr tag)
...	...	@@ -387,67 +348,256 @@ ScopedReport::ScopedReport(ReportType typ, uptr tag)
387	348
388	349	ScopedReport::~ScopedReport() {}
389	350
390		void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
391		MutexSet mset, uptr tag) {
	351	// Replays the trace up to last_pos position in the last part
	352	// or up to the provided epoch/sid (whichever is earlier)
	353	// and calls the provided function f for each event.
	354	template <typename Func>
	355	void TraceReplay(Trace trace, TracePart last, Event *last_pos, Sid sid,
	356	Epoch epoch, Func f) {
	357	TracePart *part = trace->parts.Front();
	358	Sid ev_sid = kFreeSid;
	359	Epoch ev_epoch = kEpochOver;
	360	for (;;) {
	361	DCHECK_EQ(part->trace, trace);
	362	// Note: an event can't start in the last element.
	363	// Since an event can take up to 2 elements,
	364	// we ensure we have at least 2 before adding an event.
	365	Event *end = &part->events[TracePart::kSize - 1];
	366	if (part == last)
	367	end = last_pos;
	368	f(kFreeSid, kEpochOver, nullptr); // notify about part start
	369	for (Event *evp = &part->events[0]; evp < end; evp++) {
	370	Event *evp0 = evp;
	371	if (!evp->is_access && !evp->is_func) {
	372	switch (evp->type) {
	373	case EventType::kTime: {
	374	auto ev = reinterpret_cast<EventTime >(evp);
	375	ev_sid = static_cast<Sid>(ev->sid);
	376	ev_epoch = static_cast<Epoch>(ev->epoch);
	377	if (ev_sid == sid && ev_epoch > epoch)
	378	return;
	379	break;
	380	}
	381	case EventType::kAccessExt:
	382	FALLTHROUGH;
	383	case EventType::kAccessRange:
	384	FALLTHROUGH;
	385	case EventType::kLock:
	386	FALLTHROUGH;
	387	case EventType::kRLock:
	388	// These take 2 Event elements.
	389	evp++;
	390	break;
	391	case EventType::kUnlock:
	392	// This takes 1 Event element.
	393	break;
	394	}
	395	}
	396	CHECK_NE(ev_sid, kFreeSid);
	397	CHECK_NE(ev_epoch, kEpochOver);
	398	f(ev_sid, ev_epoch, evp0);
	399	}
	400	if (part == last)
	401	return;
	402	part = trace->parts.Next(part);
	403	CHECK(part);
	404	}
	405	CHECK(0);
	406	}
	407
	408	static void RestoreStackMatch(VarSizeStackTrace pstk, MutexSet pmset,
	409	Vector<uptr> stack, MutexSet mset, uptr pc,
	410	bool *found) {
	411	DPrintf2(" MATCHED\n");
	412	pmset = mset;
	413	stack->PushBack(pc);
	414	pstk->Init(&(*stack)[0], stack->Size());
	415	stack->PopBack();
	416	*found = true;
	417	}
	418
	419	// Checks if addr1\|size1 is fully contained in addr2\|size2.
	420	// We check for fully contained instread of just overlapping
	421	// because a memory access is always traced once, but can be
	422	// split into multiple accesses in the shadow.
	423	static constexpr bool IsWithinAccess(uptr addr1, uptr size1, uptr addr2,
	424	uptr size2) {
	425	return addr1 >= addr2 && addr1 + size1 <= addr2 + size2;
	426	}
	427
	428	// Replays the trace of slot sid up to the target event identified
	429	// by epoch/addr/size/typ and restores and returns tid, stack, mutex set
	430	// and tag for that event. If there are multiple such events, it returns
	431	// the last one. Returns false if the event is not present in the trace.
	432	bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size,
	433	AccessType typ, Tid ptid, VarSizeStackTrace pstk,
	434	MutexSet pmset, uptr ptag) {
392	435	// This function restores stack trace and mutex set for the thread/epoch.
393	436	// It does so by getting stack trace and mutex set at the beginning of
394	437	// trace part, and then replaying the trace till the given epoch.
395		Trace* trace = ThreadTrace(tid);
396		ReadLock l(&trace->mtx);
397		const int partidx = (epoch / kTracePartSize) % TraceParts();
398		TraceHeader* hdr = &trace->headers[partidx];
399		if (epoch < hdr->epoch0 \|\| epoch >= hdr->epoch0 + kTracePartSize)
400		return;
401		CHECK_EQ(RoundDown(epoch, kTracePartSize), hdr->epoch0);
402		const u64 epoch0 = RoundDown(epoch, TraceSize());
403		const u64 eend = epoch % TraceSize();
404		const u64 ebegin = RoundDown(eend, kTracePartSize);
405		DPrintf("#%d: RestoreStack epoch=%zu ebegin=%zu eend=%zu partidx=%d\n",
406		tid, (uptr)epoch, (uptr)ebegin, (uptr)eend, partidx);
407		Vector<uptr> stack;
408		stack.Resize(hdr->stack0.size + 64);
409		for (uptr i = 0; i < hdr->stack0.size; i++) {
410		stack[i] = hdr->stack0.trace[i];
411		DPrintf2(" #%02zu: pc=%zx\n", i, stack[i]);
412		}
413		if (mset)
414		*mset = hdr->mset0;
415		uptr pos = hdr->stack0.size;
416		Event events = (Event)GetThreadTrace(tid);
417		for (uptr i = ebegin; i <= eend; i++) {
418		Event ev = events[i];
419		EventType typ = (EventType)(ev >> kEventPCBits);
420		uptr pc = (uptr)(ev & ((1ull << kEventPCBits) - 1));
421		DPrintf2(" %zu typ=%d pc=%zx\n", i, typ, pc);
422		if (typ == EventTypeMop) {
423		stack[pos] = pc;
424		} else if (typ == EventTypeFuncEnter) {
425		if (stack.Size() < pos + 2)
426		stack.Resize(pos + 2);
427		stack[pos++] = pc;
428		} else if (typ == EventTypeFuncExit) {
429		if (pos > 0)
430		pos--;
	438	DPrintf2("RestoreStack: sid=%u@%u addr=0x%zx/%zu typ=%x\n",
	439	static_cast<int>(sid), static_cast<int>(epoch), addr, size,
	440	static_cast<int>(typ));
	441	ctx->slot_mtx.CheckLocked(); // needed to prevent trace part recycling
	442	ctx->thread_registry.CheckLocked();
	443	TidSlot *slot = &ctx->slots[static_cast<uptr>(sid)];
	444	Tid tid = kInvalidTid;
	445	// Need to lock the slot mutex as it protects slot->journal.
	446	slot->mtx.CheckLocked();
	447	for (uptr i = 0; i < slot->journal.Size(); i++) {
	448	DPrintf2(" journal: epoch=%d tid=%d\n",
	449	static_cast<int>(slot->journal[i].epoch), slot->journal[i].tid);
	450	if (i == slot->journal.Size() - 1 \|\| slot->journal[i + 1].epoch > epoch) {
	451	tid = slot->journal[i].tid;
	452	break;
431	453	}
432		if (mset) {
433		if (typ == EventTypeLock) {
434		mset->Add(pc, true, epoch0 + i);
435		} else if (typ == EventTypeUnlock) {
436		mset->Del(pc, true);
437		} else if (typ == EventTypeRLock) {
438		mset->Add(pc, false, epoch0 + i);
439		} else if (typ == EventTypeRUnlock) {
440		mset->Del(pc, false);
441		}
	454	}
	455	if (tid == kInvalidTid)
	456	return false;
	457	*ptid = tid;
	458	ThreadContext *tctx =
	459	static_cast<ThreadContext *>(ctx->thread_registry.GetThreadLocked(tid));
	460	Trace *trace = &tctx->trace;
	461	// Snapshot first/last parts and the current position in the last part.
	462	TracePart *first_part;
	463	TracePart *last_part;
	464	Event *last_pos;
	465	{
	466	Lock lock(&trace->mtx);
	467	first_part = trace->parts.Front();
	468	if (!first_part) {
	469	DPrintf2("RestoreStack: tid=%d trace=%p no trace parts\n", tid, trace);
	470	return false;
442	471	}
443		for (uptr j = 0; j <= pos; j++)
444		DPrintf2(" #%zu: %zx\n", j, stack[j]);
	472	last_part = trace->parts.Back();
	473	last_pos = trace->final_pos;
	474	if (tctx->thr)
	475	last_pos = (Event *)atomic_load_relaxed(&tctx->thr->trace_pos);
445	476	}
446		if (pos == 0 && stack[0] == 0)
447		return;
448		pos++;
449		stk->Init(&stack[0], pos);
450		ExtractTagFromStack(stk, tag);
	477	DynamicMutexSet mset;
	478	Vector<uptr> stack;
	479	uptr prev_pc = 0;
	480	bool found = false;
	481	bool is_read = typ & kAccessRead;
	482	bool is_atomic = typ & kAccessAtomic;
	483	bool is_free = typ & kAccessFree;
	484	DPrintf2("RestoreStack: tid=%d parts=[%p-%p] last_pos=%p\n", tid,
	485	trace->parts.Front(), last_part, last_pos);
	486	TraceReplay(
	487	trace, last_part, last_pos, sid, epoch,
	488	[&](Sid ev_sid, Epoch ev_epoch, Event *evp) {
	489	if (evp == nullptr) {
	490	// Each trace part is self-consistent, so we reset state.
	491	stack.Resize(0);
	492	mset->Reset();
	493	prev_pc = 0;
	494	return;
	495	}
	496	bool match = ev_sid == sid && ev_epoch == epoch;
	497	if (evp->is_access) {
	498	if (evp->is_func == 0 && evp->type == EventType::kAccessExt &&
	499	evp->_ == 0) // NopEvent
	500	return;
	501	auto ev = reinterpret_cast<EventAccess >(evp);
	502	uptr ev_addr = RestoreAddr(ev->addr);
	503	uptr ev_size = 1 << ev->size_log;
	504	uptr ev_pc =
	505	prev_pc + ev->pc_delta - (1 << (EventAccess::kPCBits - 1));
	506	prev_pc = ev_pc;
	507	DPrintf2(" Access: pc=0x%zx addr=0x%zx/%zu type=%u/%u\n", ev_pc,
	508	ev_addr, ev_size, ev->is_read, ev->is_atomic);
	509	if (match && type == EventType::kAccessExt &&
	510	IsWithinAccess(addr, size, ev_addr, ev_size) &&
	511	is_read == ev->is_read && is_atomic == ev->is_atomic && !is_free)
	512	RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found);
	513	return;
	514	}
	515	if (evp->is_func) {
	516	auto ev = reinterpret_cast<EventFunc >(evp);
	517	if (ev->pc) {
	518	DPrintf2(" FuncEnter: pc=0x%llx\n", ev->pc);
	519	stack.PushBack(ev->pc);
	520	} else {
	521	DPrintf2(" FuncExit\n");
	522	// We don't log pathologically large stacks in each part,
	523	// if the stack was truncated we can have more func exits than
	524	// entries.
	525	if (stack.Size())
	526	stack.PopBack();
	527	}
	528	return;
	529	}
	530	switch (evp->type) {
	531	case EventType::kAccessExt: {
	532	auto ev = reinterpret_cast<EventAccessExt >(evp);
	533	uptr ev_addr = RestoreAddr(ev->addr);
	534	uptr ev_size = 1 << ev->size_log;
	535	prev_pc = ev->pc;
	536	DPrintf2(" AccessExt: pc=0x%llx addr=0x%zx/%zu type=%u/%u\n",
	537	ev->pc, ev_addr, ev_size, ev->is_read, ev->is_atomic);
	538	if (match && type == EventType::kAccessExt &&
	539	IsWithinAccess(addr, size, ev_addr, ev_size) &&
	540	is_read == ev->is_read && is_atomic == ev->is_atomic &&
	541	!is_free)
	542	RestoreStackMatch(pstk, pmset, &stack, mset, ev->pc, &found);
	543	break;
	544	}
	545	case EventType::kAccessRange: {
	546	auto ev = reinterpret_cast<EventAccessRange >(evp);
	547	uptr ev_addr = RestoreAddr(ev->addr);
	548	uptr ev_size =
	549	(ev->size_hi << EventAccessRange::kSizeLoBits) + ev->size_lo;
	550	uptr ev_pc = RestoreAddr(ev->pc);
	551	prev_pc = ev_pc;
	552	DPrintf2(" Range: pc=0x%zx addr=0x%zx/%zu type=%u/%u\n", ev_pc,
	553	ev_addr, ev_size, ev->is_read, ev->is_free);
	554	if (match && type == EventType::kAccessExt &&
	555	IsWithinAccess(addr, size, ev_addr, ev_size) &&
	556	is_read == ev->is_read && !is_atomic && is_free == ev->is_free)
	557	RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found);
	558	break;
	559	}
	560	case EventType::kLock:
	561	FALLTHROUGH;
	562	case EventType::kRLock: {
	563	auto ev = reinterpret_cast<EventLock >(evp);
	564	bool is_write = ev->type == EventType::kLock;
	565	uptr ev_addr = RestoreAddr(ev->addr);
	566	uptr ev_pc = RestoreAddr(ev->pc);
	567	StackID stack_id =
	568	(ev->stack_hi << EventLock::kStackIDLoBits) + ev->stack_lo;
	569	DPrintf2(" Lock: pc=0x%zx addr=0x%zx stack=%u write=%d\n", ev_pc,
	570	ev_addr, stack_id, is_write);
	571	mset->AddAddr(ev_addr, stack_id, is_write);
	572	// Events with ev_pc == 0 are written to the beginning of trace
	573	// part as initial mutex set (are not real).
	574	if (match && type == EventType::kLock && addr == ev_addr && ev_pc)
	575	RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found);
	576	break;
	577	}
	578	case EventType::kUnlock: {
	579	auto ev = reinterpret_cast<EventUnlock >(evp);
	580	uptr ev_addr = RestoreAddr(ev->addr);
	581	DPrintf2(" Unlock: addr=0x%zx\n", ev_addr);
	582	mset->DelAddr(ev_addr);
	583	break;
	584	}
	585	case EventType::kTime:
	586	// TraceReplay already extracted sid/epoch from it,
	587	// nothing else to do here.
	588	break;
	589	}
	590	});
	591	ExtractTagFromStack(pstk, ptag);
	592	return found;
	593	}
	594
	595	bool RacyStacks::operator==(const RacyStacks &other) const {
	596	if (hash[0] == other.hash[0] && hash[1] == other.hash[1])
	597	return true;
	598	if (hash[0] == other.hash[1] && hash[1] == other.hash[0])
	599	return true;
	600	return false;
451	601	}
452	602
453	603	static bool FindRacyStacks(const RacyStacks &hash) {
...	...	@@ -478,35 +628,6 @@ static bool HandleRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2]) {
478	628	return false;
479	629	}
480	630
481		static bool FindRacyAddress(const RacyAddress &ra0) {
482		for (uptr i = 0; i < ctx->racy_addresses.Size(); i++) {
483		RacyAddress ra2 = ctx->racy_addresses[i];
484		uptr maxbeg = max(ra0.addr_min, ra2.addr_min);
485		uptr minend = min(ra0.addr_max, ra2.addr_max);
486		if (maxbeg < minend) {
487		VPrintf(2, "ThreadSanitizer: suppressing report as doubled (addr)\n");
488		return true;
489		}
490		}
491		return false;
492		}
493
494		static bool HandleRacyAddress(ThreadState *thr, uptr addr_min, uptr addr_max) {
495		if (!flags()->suppress_equal_addresses)
496		return false;
497		RacyAddress ra0 = {addr_min, addr_max};
498		{
499		ReadLock lock(&ctx->racy_mtx);
500		if (FindRacyAddress(ra0))
501		return true;
502		}
503		Lock lock(&ctx->racy_mtx);
504		if (FindRacyAddress(ra0))
505		return true;
506		ctx->racy_addresses.PushBack(ra0);
507		return false;
508		}
509
510	631	bool OutputReport(ThreadState *thr, const ScopedReport &srep) {
511	632	// These should have been checked in ShouldReport.
512	633	// It's too late to check them here, we have already taken locks.
...	...	@@ -532,10 +653,7 @@ bool OutputReport(ThreadState *thr, const ScopedReport &srep) {
532	653	ctx->fired_suppressions.push_back(s);
533	654	}
534	655	{
535		bool old_is_freeing = thr->is_freeing;
536		thr->is_freeing = false;
537	656	bool suppressed = OnReport(rep, pc_or_addr != 0);
538		thr->is_freeing = old_is_freeing;
539	657	if (suppressed) {
540	658	thr->current_report = nullptr;
541	659	return false;
...	...	@@ -582,101 +700,81 @@ static bool IsFiredSuppression(Context *ctx, ReportType type, uptr addr) {
582	700	return false;
583	701	}
584	702
585		static bool RaceBetweenAtomicAndFree(ThreadState *thr) {
586		Shadow s0(thr->racy_state[0]);
587		Shadow s1(thr->racy_state[1]);
588		CHECK(!(s0.IsAtomic() && s1.IsAtomic()));
589		if (!s0.IsAtomic() && !s1.IsAtomic())
590		return true;
591		if (s0.IsAtomic() && s1.IsFreed())
592		return true;
593		if (s1.IsAtomic() && thr->is_freeing)
594		return true;
595		return false;
	703	static bool SpuriousRace(Shadow old) {
	704	Shadow last(LoadShadow(&ctx->last_spurious_race));
	705	return last.sid() == old.sid() && last.epoch() == old.epoch();
596	706	}
597	707
598		void ReportRace(ThreadState *thr) {
	708	void ReportRace(ThreadState thr, RawShadow shadow_mem, Shadow cur, Shadow old,
	709	AccessType typ0) {
599	710	CheckedMutex::CheckNoLocks();
600	711
601	712	// Symbolizer makes lots of intercepted calls. If we try to process them,
602	713	// at best it will cause deadlocks on internal mutexes.
603	714	ScopedIgnoreInterceptors ignore;
604	715
	716	uptr addr = ShadowToMem(shadow_mem);
	717	DPrintf("#%d: ReportRace %p\n", thr->tid, (void *)addr);
605	718	if (!ShouldReport(thr, ReportTypeRace))
606	719	return;
607		if (!flags()->report_atomic_races && !RaceBetweenAtomicAndFree(thr))
	720	uptr addr_off0, size0;
	721	cur.GetAccess(&addr_off0, &size0, nullptr);
	722	uptr addr_off1, size1, typ1;
	723	old.GetAccess(&addr_off1, &size1, &typ1);
	724	if (!flags()->report_atomic_races &&
	725	((typ0 & kAccessAtomic) \|\| (typ1 & kAccessAtomic)) &&
	726	!(typ0 & kAccessFree) && !(typ1 & kAccessFree))
	727	return;
	728	if (SpuriousRace(old))
608	729	return;
609	730
610		bool freed = false;
611		{
612		Shadow s(thr->racy_state[1]);
613		freed = s.GetFreedAndReset();
614		thr->racy_state[1] = s.raw();
615		}
616
617		uptr addr = ShadowToMem((uptr)thr->racy_shadow_addr);
618		uptr addr_min = 0;
619		uptr addr_max = 0;
620		{
621		uptr a0 = addr + Shadow(thr->racy_state[0]).addr0();
622		uptr a1 = addr + Shadow(thr->racy_state[1]).addr0();
623		uptr e0 = a0 + Shadow(thr->racy_state[0]).size();
624		uptr e1 = a1 + Shadow(thr->racy_state[1]).size();
625		addr_min = min(a0, a1);
626		addr_max = max(e0, e1);
627		if (IsExpectedReport(addr_min, addr_max - addr_min))
628		return;
629		}
630		if (HandleRacyAddress(thr, addr_min, addr_max))
	731	const uptr kMop = 2;
	732	Shadow s[kMop] = {cur, old};
	733	uptr addr0 = addr + addr_off0;
	734	uptr addr1 = addr + addr_off1;
	735	uptr end0 = addr0 + size0;
	736	uptr end1 = addr1 + size1;
	737	uptr addr_min = min(addr0, addr1);
	738	uptr addr_max = max(end0, end1);
	739	if (IsExpectedReport(addr_min, addr_max - addr_min))
631	740	return;
632	741
633		ReportType typ = ReportTypeRace;
634		if (thr->is_vptr_access && freed)
635		typ = ReportTypeVptrUseAfterFree;
636		else if (thr->is_vptr_access)
637		typ = ReportTypeVptrRace;
638		else if (freed)
639		typ = ReportTypeUseAfterFree;
	742	ReportType rep_typ = ReportTypeRace;
	743	if ((typ0 & kAccessVptr) && (typ1 & kAccessFree))
	744	rep_typ = ReportTypeVptrUseAfterFree;
	745	else if (typ0 & kAccessVptr)
	746	rep_typ = ReportTypeVptrRace;
	747	else if (typ1 & kAccessFree)
	748	rep_typ = ReportTypeUseAfterFree;
640	749
641		if (IsFiredSuppression(ctx, typ, addr))
	750	if (IsFiredSuppression(ctx, rep_typ, addr))
642	751	return;
643	752
644		const uptr kMop = 2;
645	753	VarSizeStackTrace traces[kMop];
646		uptr tags[kMop] = {kExternalTagNone};
647		uptr toppc = TraceTopPC(thr);
648		if (toppc >> kEventPCBits) {
649		// This is a work-around for a known issue.
650		// The scenario where this happens is rather elaborate and requires
651		// an instrumented __sanitizer_report_error_summary callback and
652		// a __tsan_symbolize_external callback and a race during a range memory
653		// access larger than 8 bytes. MemoryAccessRange adds the current PC to
654		// the trace and starts processing memory accesses. A first memory access
655		// triggers a race, we report it and call the instrumented
656		// __sanitizer_report_error_summary, which adds more stuff to the trace
657		// since it is intrumented. Then a second memory access in MemoryAccessRange
658		// also triggers a race and we get here and call TraceTopPC to get the
659		// current PC, however now it contains some unrelated events from the
660		// callback. Most likely, TraceTopPC will now return a EventTypeFuncExit
661		// event. Later we subtract -1 from it (in GetPreviousInstructionPc)
662		// and the resulting PC has kExternalPCBit set, so we pass it to
663		// __tsan_symbolize_external_ex. __tsan_symbolize_external_ex is within its
664		// rights to crash since the PC is completely bogus.
665		// test/tsan/double_race.cpp contains a test case for this.
666		toppc = 0;
667		}
668		ObtainCurrentStack(thr, toppc, &traces[0], &tags[0]);
669		if (IsFiredSuppression(ctx, typ, traces[0]))
	754	Tid tids[kMop] = {thr->tid, kInvalidTid};
	755	uptr tags[kMop] = {kExternalTagNone, kExternalTagNone};
	756
	757	ObtainCurrentStack(thr, thr->trace_prev_pc, &traces[0], &tags[0]);
	758	if (IsFiredSuppression(ctx, rep_typ, traces[0]))
670	759	return;
671	760
672		// MutexSet is too large to live on stack.
673		Vector<u64> mset_buffer;
674		mset_buffer.Resize(sizeof(MutexSet) / sizeof(u64) + 1);
675		MutexSet *mset2 = new(&mset_buffer[0]) MutexSet();
	761	DynamicMutexSet mset1;
	762	MutexSet *mset[kMop] = {&thr->mset, mset1};
676	763
677		Shadow s2(thr->racy_state[1]);
678		RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2, &tags[1]);
679		if (IsFiredSuppression(ctx, typ, traces[1]))
	764	// We need to lock the slot during RestoreStack because it protects
	765	// the slot journal.
	766	Lock slot_lock(&ctx->slots[static_cast<uptr>(s[1].sid())].mtx);
	767	ThreadRegistryLock l0(&ctx->thread_registry);
	768	Lock slots_lock(&ctx->slot_mtx);
	769	if (SpuriousRace(old))
	770	return;
	771	if (!RestoreStack(EventType::kAccessExt, s[1].sid(), s[1].epoch(), addr1,
	772	size1, typ1, &tids[1], &traces[1], mset[1], &tags[1])) {
	773	StoreShadow(&ctx->last_spurious_race, old.raw());
	774	return;
	775	}
	776
	777	if (IsFiredSuppression(ctx, rep_typ, traces[1]))
680	778	return;
681	779
682	780	if (HandleRacyStacks(thr, traces))
...	...	@@ -686,39 +784,41 @@ void ReportRace(ThreadState *thr) {
686	784	uptr tag = kExternalTagNone;
687	785	for (uptr i = 0; i < kMop; i++) {
688	786	if (tags[i] != kExternalTagNone) {
689		typ = ReportTypeExternalRace;
	787	rep_typ = ReportTypeExternalRace;
690	788	tag = tags[i];
691	789	break;
692	790	}
693	791	}
694	792
695		ThreadRegistryLock l0(ctx->thread_registry);
696		ScopedReport rep(typ, tag);
697		for (uptr i = 0; i < kMop; i++) {
698		Shadow s(thr->racy_state[i]);
699		rep.AddMemoryAccess(addr, tags[i], s, traces[i],
700		i == 0 ? &thr->mset : mset2);
701		}
	793	ScopedReport rep(rep_typ, tag);
	794	for (uptr i = 0; i < kMop; i++)
	795	rep.AddMemoryAccess(addr, tags[i], s[i], tids[i], traces[i], mset[i]);
702	796
703	797	for (uptr i = 0; i < kMop; i++) {
704		FastState s(thr->racy_state[i]);
705		ThreadContext tctx = static_cast<ThreadContext>(
706		ctx->thread_registry->GetThreadLocked(s.tid()));
707		if (s.epoch() < tctx->epoch0 \|\| s.epoch() > tctx->epoch1)
708		continue;
	798	ThreadContext tctx = static_cast<ThreadContext >(
	799	ctx->thread_registry.GetThreadLocked(tids[i]));
709	800	rep.AddThread(tctx);
710	801	}
711	802
712	803	rep.AddLocation(addr_min, addr_max - addr_min);
713	804
714		#if !SANITIZER_GO
715		{
716		Shadow s(thr->racy_state[1]);
717		if (s.epoch() <= thr->last_sleep_clock.get(s.tid()))
718		rep.AddSleep(thr->last_sleep_stack_id);
	805	if (flags()->print_full_thread_history) {
	806	const ReportDesc *rep_desc = rep.GetReport();
	807	for (uptr i = 0; i < rep_desc->threads.Size(); i++) {
	808	Tid parent_tid = rep_desc->threads[i]->parent_tid;
	809	if (parent_tid == kMainTid \|\| parent_tid == kInvalidTid)
	810	continue;
	811	ThreadContext parent_tctx = static_cast<ThreadContext >(
	812	ctx->thread_registry.GetThreadLocked(parent_tid));
	813	rep.AddThread(parent_tctx);
	814	}
719	815	}
720		#endif
721	816
	817	#if !SANITIZER_GO
	818	if (!((typ0 \| typ1) & kAccessFree) &&
	819	s[1].epoch() <= thr->last_sleep_clock.Get(s[1].sid()))
	820	rep.AddSleep(thr->last_sleep_stack_id);
	821	#endif
722	822	OutputReport(thr, rep);
723	823	}
724	824
...	...	@@ -738,9 +838,7 @@ void PrintCurrentStack(ThreadState *thr, uptr pc) {
738	838	ALWAYS_INLINE USED void PrintCurrentStackSlow(uptr pc) {
739	839	#if !SANITIZER_GO
740	840	uptr bp = GET_CURRENT_FRAME();
741		BufferedStackTrace *ptrace =
742		new(internal_alloc(MBlockStackTrace, sizeof(BufferedStackTrace)))
743		BufferedStackTrace();
	841	auto *ptrace = New<BufferedStackTrace>();
744	842	ptrace->Unwind(pc, bp, nullptr, false);
745	843
746	844	for (uptr i = 0; i < ptrace->size / 2; i++) {

lib/tsan/tsan_rtl_thread.cpp+185-268

...	...	@@ -21,133 +21,14 @@ namespace __tsan {
21	21
22	22	// ThreadContext implementation.
23	23
24		ThreadContext::ThreadContext(int tid)
25		: ThreadContextBase(tid)
26		, thr()
27		, sync()
28		, epoch0()
29		, epoch1() {
30		}
	24	ThreadContext::ThreadContext(Tid tid) : ThreadContextBase(tid), thr(), sync() {}
31	25
32	26	#if !SANITIZER_GO
33	27	ThreadContext::~ThreadContext() {
34	28	}
35	29	#endif
36	30
37		void ThreadContext::OnDead() {
38		CHECK_EQ(sync.size(), 0);
39		}
40
41		void ThreadContext::OnJoined(void *arg) {
42		ThreadState caller_thr = static_cast<ThreadState >(arg);
43		AcquireImpl(caller_thr, 0, &sync);
44		sync.Reset(&caller_thr->proc()->clock_cache);
45		}
46
47		struct OnCreatedArgs {
48		ThreadState *thr;
49		uptr pc;
50		};
51
52		void ThreadContext::OnCreated(void *arg) {
53		thr = 0;
54		if (tid == kMainTid)
55		return;
56		OnCreatedArgs args = static_cast<OnCreatedArgs >(arg);
57		if (!args->thr) // GCD workers don't have a parent thread.
58		return;
59		args->thr->fast_state.IncrementEpoch();
60		// Can't increment epoch w/o writing to the trace as well.
61		TraceAddEvent(args->thr, args->thr->fast_state, EventTypeMop, 0);
62		ReleaseImpl(args->thr, 0, &sync);
63		creation_stack_id = CurrentStackId(args->thr, args->pc);
64		}
65
66		void ThreadContext::OnReset() {
67		CHECK_EQ(sync.size(), 0);
68		uptr trace_p = GetThreadTrace(tid);
69		ReleaseMemoryPagesToOS(trace_p, trace_p + TraceSize() * sizeof(Event));
70		//!!! ReleaseMemoryToOS(GetThreadTraceHeader(tid), sizeof(Trace));
71		}
72
73		void ThreadContext::OnDetached(void *arg) {
74		ThreadState thr1 = static_cast<ThreadState>(arg);
75		sync.Reset(&thr1->proc()->clock_cache);
76		}
77
78		struct OnStartedArgs {
79		ThreadState *thr;
80		uptr stk_addr;
81		uptr stk_size;
82		uptr tls_addr;
83		uptr tls_size;
84		};
85
86		void ThreadContext::OnStarted(void *arg) {
87		OnStartedArgs args = static_cast<OnStartedArgs>(arg);
88		thr = args->thr;
89		// RoundUp so that one trace part does not contain events
90		// from different threads.
91		epoch0 = RoundUp(epoch1 + 1, kTracePartSize);
92		epoch1 = (u64)-1;
93		new(thr) ThreadState(ctx, tid, unique_id, epoch0, reuse_count,
94		args->stk_addr, args->stk_size, args->tls_addr, args->tls_size);
95		#if !SANITIZER_GO
96		thr->shadow_stack = &ThreadTrace(thr->tid)->shadow_stack[0];
97		thr->shadow_stack_pos = thr->shadow_stack;
98		thr->shadow_stack_end = thr->shadow_stack + kShadowStackSize;
99		#else
100		// Setup dynamic shadow stack.
101		const int kInitStackSize = 8;
102		thr->shadow_stack = (uptr*)internal_alloc(MBlockShadowStack,
103		kInitStackSize * sizeof(uptr));
104		thr->shadow_stack_pos = thr->shadow_stack;
105		thr->shadow_stack_end = thr->shadow_stack + kInitStackSize;
106		#endif
107		if (common_flags()->detect_deadlocks)
108		thr->dd_lt = ctx->dd->CreateLogicalThread(unique_id);
109		thr->fast_state.SetHistorySize(flags()->history_size);
110		// Commit switch to the new part of the trace.
111		// TraceAddEvent will reset stack0/mset0 in the new part for us.
112		TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
113
114		thr->fast_synch_epoch = epoch0;
115		AcquireImpl(thr, 0, &sync);
116		sync.Reset(&thr->proc()->clock_cache);
117		thr->is_inited = true;
118		DPrintf("#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx "
119		"tls_addr=%zx tls_size=%zx\n",
120		tid, (uptr)epoch0, args->stk_addr, args->stk_size,
121		args->tls_addr, args->tls_size);
122		}
123
124		void ThreadContext::OnFinished() {
125		#if SANITIZER_GO
126		internal_free(thr->shadow_stack);
127		thr->shadow_stack = nullptr;
128		thr->shadow_stack_pos = nullptr;
129		thr->shadow_stack_end = nullptr;
130		#endif
131		if (!detached) {
132		thr->fast_state.IncrementEpoch();
133		// Can't increment epoch w/o writing to the trace as well.
134		TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
135		ReleaseImpl(thr, 0, &sync);
136		}
137		epoch1 = thr->fast_state.epoch();
138
139		if (common_flags()->detect_deadlocks)
140		ctx->dd->DestroyLogicalThread(thr->dd_lt);
141		thr->clock.ResetCached(&thr->proc()->clock_cache);
142		#if !SANITIZER_GO
143		thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
144		#endif
145		#if !SANITIZER_GO
146		PlatformCleanUpThreadState(thr);
147		#endif
148		thr->~ThreadState();
149		thr = 0;
150		}
	31	void ThreadContext::OnReset() { CHECK(!sync); }
151	32
152	33	#if !SANITIZER_GO
153	34	struct ThreadLeak {
...	...	@@ -155,9 +36,9 @@ struct ThreadLeak {
155	36	int count;
156	37	};
157	38
158		static void MaybeReportThreadLeak(ThreadContextBase tctx_base, void arg) {
159		Vector<ThreadLeak> &leaks = (Vector<ThreadLeak>)arg;
160		ThreadContext tctx = static_cast<ThreadContext>(tctx_base);
	39	static void CollectThreadLeaks(ThreadContextBase tctx_base, void arg) {
	40	auto &leaks = static_cast<Vector<ThreadLeak> >(arg);
	41	auto tctx = static_cast<ThreadContext >(tctx_base);
161	42	if (tctx->detached \|\| tctx->status != ThreadStatusFinished)
162	43	return;
163	44	for (uptr i = 0; i < leaks.Size(); i++) {
...	...	@@ -166,12 +47,13 @@ static void MaybeReportThreadLeak(ThreadContextBase tctx_base, void arg) {
166	47	return;
167	48	}
168	49	}
169		ThreadLeak leak = {tctx, 1};
170		leaks.PushBack(leak);
	50	leaks.PushBack({tctx, 1});
171	51	}
172	52	#endif
173	53
174		#if !SANITIZER_GO
	54	// Disabled on Mac because lldb test TestTsanBasic fails:
	55	// https://reviews.llvm.org/D112603#3163158
	56	#if !SANITIZER_GO && !SANITIZER_APPLE
175	57	static void ReportIgnoresEnabled(ThreadContext tctx, IgnoreSet set) {
176	58	if (tctx->tid == kMainTid) {
177	59	Printf("ThreadSanitizer: main thread finished with ignores enabled\n");
...	...	@@ -206,10 +88,10 @@ void ThreadFinalize(ThreadState *thr) {
206	88	#if !SANITIZER_GO
207	89	if (!ShouldReport(thr, ReportTypeThreadLeak))
208	90	return;
209		ThreadRegistryLock l(ctx->thread_registry);
	91	ThreadRegistryLock l(&ctx->thread_registry);
210	92	Vector<ThreadLeak> leaks;
211		ctx->thread_registry->RunCallbackForEachThreadLocked(
212		MaybeReportThreadLeak, &leaks);
	93	ctx->thread_registry.RunCallbackForEachThreadLocked(CollectThreadLeaks,
	94	&leaks);
213	95	for (uptr i = 0; i < leaks.Size(); i++) {
214	96	ScopedReport rep(ReportTypeThreadLeak);
215	97	rep.AddThread(leaks[i].tctx, true);
...	...	@@ -221,21 +103,63 @@ void ThreadFinalize(ThreadState *thr) {
221	103
222	104	int ThreadCount(ThreadState *thr) {
223	105	uptr result;
224		ctx->thread_registry->GetNumberOfThreads(0, 0, &result);
	106	ctx->thread_registry.GetNumberOfThreads(0, 0, &result);
225	107	return (int)result;
226	108	}
227	109
228		int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) {
229		OnCreatedArgs args = { thr, pc };
230		u32 parent_tid = thr ? thr->tid : kInvalidTid; // No parent for GCD workers.
231		int tid =
232		ctx->thread_registry->CreateThread(uid, detached, parent_tid, &args);
233		DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent_tid, tid, uid);
	110	struct OnCreatedArgs {
	111	VectorClock *sync;
	112	uptr sync_epoch;
	113	StackID stack;
	114	};
	115
	116	Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) {
	117	// The main thread and GCD workers don't have a parent thread.
	118	Tid parent = kInvalidTid;
	119	OnCreatedArgs arg = {nullptr, 0, kInvalidStackID};
	120	if (thr) {
	121	parent = thr->tid;
	122	arg.stack = CurrentStackId(thr, pc);
	123	if (!thr->ignore_sync) {
	124	SlotLocker locker(thr);
	125	thr->clock.ReleaseStore(&arg.sync);
	126	arg.sync_epoch = ctx->global_epoch;
	127	IncrementEpoch(thr);
	128	}
	129	}
	130	Tid tid = ctx->thread_registry.CreateThread(uid, detached, parent, &arg);
	131	DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent, tid, uid);
234	132	return tid;
235	133	}
236	134
237		void ThreadStart(ThreadState *thr, int tid, tid_t os_id,
	135	void ThreadContext::OnCreated(void *arg) {
	136	OnCreatedArgs args = static_cast<OnCreatedArgs >(arg);
	137	sync = args->sync;
	138	sync_epoch = args->sync_epoch;
	139	creation_stack_id = args->stack;
	140	}
	141
	142	extern "C" void __tsan_stack_initialization() {}
	143
	144	struct OnStartedArgs {
	145	ThreadState *thr;
	146	uptr stk_addr;
	147	uptr stk_size;
	148	uptr tls_addr;
	149	uptr tls_size;
	150	};
	151
	152	void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
238	153	ThreadType thread_type) {
	154	ctx->thread_registry.StartThread(tid, os_id, thread_type, thr);
	155	if (!thr->ignore_sync) {
	156	SlotAttachAndLock(thr);
	157	if (thr->tctx->sync_epoch == ctx->global_epoch)
	158	thr->clock.Acquire(thr->tctx->sync);
	159	SlotUnlock(thr);
	160	}
	161	Free(thr->tctx->sync);
	162
239	163	uptr stk_addr = 0;
240	164	uptr stk_size = 0;
241	165	uptr tls_addr = 0;
...	...	@@ -244,22 +168,11 @@ void ThreadStart(ThreadState *thr, int tid, tid_t os_id,
244	168	if (thread_type != ThreadType::Fiber)
245	169	GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_size, &tls_addr,
246	170	&tls_size);
247
248		if (tid != kMainTid) {
249		if (stk_addr && stk_size)
250		MemoryRangeImitateWrite(thr, /pc=/ 1, stk_addr, stk_size);
251
252		if (tls_addr && tls_size) ImitateTlsWrite(thr, tls_addr, tls_size);
253		}
254	171	#endif
255
256		ThreadRegistry *tr = ctx->thread_registry;
257		OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size };
258		tr->StartThread(tid, os_id, thread_type, &args);
259
260		tr->Lock();
261		thr->tctx = (ThreadContext*)tr->GetThreadLocked(tid);
262		tr->Unlock();
	172	thr->stk_addr = stk_addr;
	173	thr->stk_size = stk_size;
	174	thr->tls_addr = tls_addr;
	175	thr->tls_size = tls_size;
263	176
264	177	#if !SANITIZER_GO
265	178	if (ctx->after_multithreaded_fork) {
...	...	@@ -268,16 +181,99 @@ void ThreadStart(ThreadState *thr, int tid, tid_t os_id,
268	181	ThreadIgnoreSyncBegin(thr, 0);
269	182	}
270	183	#endif
	184
	185	#if !SANITIZER_GO
	186	// Don't imitate stack/TLS writes for the main thread,
	187	// because its initialization is synchronized with all
	188	// subsequent threads anyway.
	189	if (tid != kMainTid) {
	190	if (stk_addr && stk_size) {
	191	const uptr pc = StackTrace::GetNextInstructionPc(
	192	reinterpret_cast<uptr>(__tsan_stack_initialization));
	193	MemoryRangeImitateWrite(thr, pc, stk_addr, stk_size);
	194	}
	195
	196	if (tls_addr && tls_size)
	197	ImitateTlsWrite(thr, tls_addr, tls_size);
	198	}
	199	#endif
	200	}
	201
	202	void ThreadContext::OnStarted(void *arg) {
	203	thr = static_cast<ThreadState *>(arg);
	204	DPrintf("#%d: ThreadStart\n", tid);
	205	new (thr) ThreadState(tid);
	206	if (common_flags()->detect_deadlocks)
	207	thr->dd_lt = ctx->dd->CreateLogicalThread(tid);
	208	thr->tctx = this;
	209	#if !SANITIZER_GO
	210	thr->is_inited = true;
	211	#endif
271	212	}
272	213
273	214	void ThreadFinish(ThreadState *thr) {
	215	DPrintf("#%d: ThreadFinish\n", thr->tid);
274	216	ThreadCheckIgnore(thr);
275	217	if (thr->stk_addr && thr->stk_size)
276	218	DontNeedShadowFor(thr->stk_addr, thr->stk_size);
277	219	if (thr->tls_addr && thr->tls_size)
278	220	DontNeedShadowFor(thr->tls_addr, thr->tls_size);
279	221	thr->is_dead = true;
280		ctx->thread_registry->FinishThread(thr->tid);
	222	#if !SANITIZER_GO
	223	thr->is_inited = false;
	224	thr->ignore_interceptors++;
	225	PlatformCleanUpThreadState(thr);
	226	#endif
	227	if (!thr->ignore_sync) {
	228	SlotLocker locker(thr);
	229	ThreadRegistryLock lock(&ctx->thread_registry);
	230	// Note: detached is protected by the thread registry mutex,
	231	// the thread may be detaching concurrently in another thread.
	232	if (!thr->tctx->detached) {
	233	thr->clock.ReleaseStore(&thr->tctx->sync);
	234	thr->tctx->sync_epoch = ctx->global_epoch;
	235	IncrementEpoch(thr);
	236	}
	237	}
	238	#if !SANITIZER_GO
	239	UnmapOrDie(thr->shadow_stack, kShadowStackSize * sizeof(uptr));
	240	#else
	241	Free(thr->shadow_stack);
	242	#endif
	243	thr->shadow_stack = nullptr;
	244	thr->shadow_stack_pos = nullptr;
	245	thr->shadow_stack_end = nullptr;
	246	if (common_flags()->detect_deadlocks)
	247	ctx->dd->DestroyLogicalThread(thr->dd_lt);
	248	SlotDetach(thr);
	249	ctx->thread_registry.FinishThread(thr->tid);
	250	thr->~ThreadState();
	251	}
	252
	253	void ThreadContext::OnFinished() {
	254	Lock lock(&ctx->slot_mtx);
	255	Lock lock1(&trace.mtx);
	256	// Queue all trace parts into the global recycle queue.
	257	auto parts = &trace.parts;
	258	while (trace.local_head) {
	259	CHECK(parts->Queued(trace.local_head));
	260	ctx->trace_part_recycle.PushBack(trace.local_head);
	261	trace.local_head = parts->Next(trace.local_head);
	262	}
	263	ctx->trace_part_recycle_finished += parts->Size();
	264	if (ctx->trace_part_recycle_finished > Trace::kFinishedThreadHi) {
	265	ctx->trace_part_finished_excess += parts->Size();
	266	trace.parts_allocated = 0;
	267	} else if (ctx->trace_part_recycle_finished > Trace::kFinishedThreadLo &&
	268	parts->Size() > 1) {
	269	ctx->trace_part_finished_excess += parts->Size() - 1;
	270	trace.parts_allocated = 1;
	271	}
	272	// From now on replay will use trace->final_pos.
	273	trace.final_pos = (Event *)atomic_load_relaxed(&thr->trace_pos);
	274	atomic_store_relaxed(&thr->trace_pos, 0);
	275	thr->tctx = nullptr;
	276	thr = nullptr;
281	277	}
282	278
283	279	struct ConsumeThreadContext {
...	...	@@ -285,131 +281,52 @@ struct ConsumeThreadContext {
285	281	ThreadContextBase *tctx;
286	282	};
287	283
288		static bool ConsumeThreadByUid(ThreadContextBase tctx, void arg) {
289		ConsumeThreadContext findCtx = (ConsumeThreadContext )arg;
290		if (tctx->user_id == findCtx->uid && tctx->status != ThreadStatusInvalid) {
291		if (findCtx->tctx) {
292		// Ensure that user_id is unique. If it's not the case we are screwed.
293		// Something went wrong before, but now there is no way to recover.
294		// Returning a wrong thread is not an option, it may lead to very hard
295		// to debug false positives (e.g. if we join a wrong thread).
296		Report("ThreadSanitizer: dup thread with used id 0x%zx\n", findCtx->uid);
297		Die();
298		}
299		findCtx->tctx = tctx;
300		tctx->user_id = 0;
301		}
302		return false;
	284	Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid) {
	285	return ctx->thread_registry.ConsumeThreadUserId(uid);
303	286	}
304	287
305		int ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid) {
306		ConsumeThreadContext findCtx = {uid, nullptr};
307		ctx->thread_registry->FindThread(ConsumeThreadByUid, &findCtx);
308		int tid = findCtx.tctx ? findCtx.tctx->tid : kInvalidTid;
309		DPrintf("#%d: ThreadTid uid=%zu tid=%d\n", thr->tid, uid, tid);
310		return tid;
311		}
	288	struct JoinArg {
	289	VectorClock *sync;
	290	uptr sync_epoch;
	291	};
312	292
313		void ThreadJoin(ThreadState *thr, uptr pc, int tid) {
	293	void ThreadJoin(ThreadState *thr, uptr pc, Tid tid) {
314	294	CHECK_GT(tid, 0);
315		CHECK_LT(tid, kMaxTid);
316	295	DPrintf("#%d: ThreadJoin tid=%d\n", thr->tid, tid);
317		ctx->thread_registry->JoinThread(tid, thr);
	296	JoinArg arg = {};
	297	ctx->thread_registry.JoinThread(tid, &arg);
	298	if (!thr->ignore_sync) {
	299	SlotLocker locker(thr);
	300	if (arg.sync_epoch == ctx->global_epoch)
	301	thr->clock.Acquire(arg.sync);
	302	}
	303	Free(arg.sync);
318	304	}
319	305
320		void ThreadDetach(ThreadState *thr, uptr pc, int tid) {
321		CHECK_GT(tid, 0);
322		CHECK_LT(tid, kMaxTid);
323		ctx->thread_registry->DetachThread(tid, thr);
	306	void ThreadContext::OnJoined(void *ptr) {
	307	auto arg = static_cast<JoinArg *>(ptr);
	308	arg->sync = sync;
	309	arg->sync_epoch = sync_epoch;
	310	sync = nullptr;
	311	sync_epoch = 0;
324	312	}
325	313
326		void ThreadNotJoined(ThreadState *thr, uptr pc, int tid, uptr uid) {
327		CHECK_GT(tid, 0);
328		CHECK_LT(tid, kMaxTid);
329		ctx->thread_registry->SetThreadUserId(tid, uid);
330		}
	314	void ThreadContext::OnDead() { CHECK_EQ(sync, nullptr); }
331	315
332		void ThreadSetName(ThreadState thr, const char name) {
333		ctx->thread_registry->SetThreadName(thr->tid, name);
	316	void ThreadDetach(ThreadState *thr, uptr pc, Tid tid) {
	317	CHECK_GT(tid, 0);
	318	ctx->thread_registry.DetachThread(tid, thr);
334	319	}
335	320
336		void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
337		uptr size, bool is_write) {
338		if (size == 0)
339		return;
340
341		u64 shadow_mem = (u64)MemToShadow(addr);
342		DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_write=%d\n",
343		thr->tid, (void)pc, (void)addr,
344		(int)size, is_write);
	321	void ThreadContext::OnDetached(void *arg) { Free(sync); }
345	322
346		#if SANITIZER_DEBUG
347		if (!IsAppMem(addr)) {
348		Printf("Access to non app mem %zx\n", addr);
349		DCHECK(IsAppMem(addr));
350		}
351		if (!IsAppMem(addr + size - 1)) {
352		Printf("Access to non app mem %zx\n", addr + size - 1);
353		DCHECK(IsAppMem(addr + size - 1));
354		}
355		if (!IsShadowMem((uptr)shadow_mem)) {
356		Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
357		DCHECK(IsShadowMem((uptr)shadow_mem));
358		}
359		if (!IsShadowMem((uptr)(shadow_mem + size * kShadowCnt / 8 - 1))) {
360		Printf("Bad shadow addr %p (%zx)\n",
361		shadow_mem + size * kShadowCnt / 8 - 1, addr + size - 1);
362		DCHECK(IsShadowMem((uptr)(shadow_mem + size * kShadowCnt / 8 - 1)));
363		}
364		#endif
365
366		if (*shadow_mem == kShadowRodata) {
367		DCHECK(!is_write);
368		// Access to .rodata section, no races here.
369		// Measurements show that it can be 10-20% of all memory accesses.
370		return;
371		}
372
373		FastState fast_state = thr->fast_state;
374		if (fast_state.GetIgnoreBit())
375		return;
376
377		fast_state.IncrementEpoch();
378		thr->fast_state = fast_state;
379		TraceAddEvent(thr, fast_state, EventTypeMop, pc);
380
381		bool unaligned = (addr % kShadowCell) != 0;
	323	void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid) {
	324	CHECK_GT(tid, 0);
	325	ctx->thread_registry.SetThreadUserId(tid, uid);
	326	}
382	327
383		// Handle unaligned beginning, if any.
384		for (; addr % kShadowCell && size; addr++, size--) {
385		int const kAccessSizeLog = 0;
386		Shadow cur(fast_state);
387		cur.SetWrite(is_write);
388		cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
389		MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false,
390		shadow_mem, cur);
391		}
392		if (unaligned)
393		shadow_mem += kShadowCnt;
394		// Handle middle part, if any.
395		for (; size >= kShadowCell; addr += kShadowCell, size -= kShadowCell) {
396		int const kAccessSizeLog = 3;
397		Shadow cur(fast_state);
398		cur.SetWrite(is_write);
399		cur.SetAddr0AndSizeLog(0, kAccessSizeLog);
400		MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false,
401		shadow_mem, cur);
402		shadow_mem += kShadowCnt;
403		}
404		// Handle ending, if any.
405		for (; size; addr++, size--) {
406		int const kAccessSizeLog = 0;
407		Shadow cur(fast_state);
408		cur.SetWrite(is_write);
409		cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
410		MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false,
411		shadow_mem, cur);
412		}
	328	void ThreadSetName(ThreadState thr, const char name) {
	329	ctx->thread_registry.SetThreadName(thr->tid, name);
413	330	}
414	331
415	332	#if !SANITIZER_GO
...	...	@@ -421,10 +338,10 @@ void FiberSwitchImpl(ThreadState from, ThreadState to) {
421	338	}
422	339
423	340	ThreadState FiberCreate(ThreadState thr, uptr pc, unsigned flags) {
424		void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadState));
	341	void *mem = Alloc(sizeof(ThreadState));
425	342	ThreadState fiber = static_cast<ThreadState >(mem);
426	343	internal_memset(fiber, 0, sizeof(*fiber));
427		int tid = ThreadCreate(thr, pc, 0, true);
	344	Tid tid = ThreadCreate(thr, pc, 0, true);
428	345	FiberSwitchImpl(thr, fiber);
429	346	ThreadStart(fiber, tid, 0, ThreadType::Fiber);
430	347	FiberSwitchImpl(fiber, thr);
...	...	@@ -435,7 +352,7 @@ void FiberDestroy(ThreadState thr, uptr pc, ThreadState fiber) {
435	352	FiberSwitchImpl(thr, fiber);
436	353	ThreadFinish(fiber);
437	354	FiberSwitchImpl(fiber, thr);
438		internal_free(fiber);
	355	Free(fiber);
439	356	}
440	357
441	358	void FiberSwitch(ThreadState *thr, uptr pc,

lib/tsan/tsan_shadow.h created+193

...	...	@@ -0,0 +1,193 @@
	1	//===-- tsan_shadow.h -------------------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8
	9	#ifndef TSAN_SHADOW_H
	10	#define TSAN_SHADOW_H
	11
	12	#include "tsan_defs.h"
	13
	14	namespace __tsan {
	15
	16	class FastState {
	17	public:
	18	FastState() { Reset(); }
	19
	20	void Reset() {
	21	part_.unused0_ = 0;
	22	part_.sid_ = static_cast<u8>(kFreeSid);
	23	part_.epoch_ = static_cast<u16>(kEpochLast);
	24	part_.unused1_ = 0;
	25	part_.ignore_accesses_ = false;
	26	}
	27
	28	void SetSid(Sid sid) { part_.sid_ = static_cast<u8>(sid); }
	29
	30	Sid sid() const { return static_cast<Sid>(part_.sid_); }
	31
	32	Epoch epoch() const { return static_cast<Epoch>(part_.epoch_); }
	33
	34	void SetEpoch(Epoch epoch) { part_.epoch_ = static_cast<u16>(epoch); }
	35
	36	void SetIgnoreBit() { part_.ignore_accesses_ = 1; }
	37	void ClearIgnoreBit() { part_.ignore_accesses_ = 0; }
	38	bool GetIgnoreBit() const { return part_.ignore_accesses_; }
	39
	40	private:
	41	friend class Shadow;
	42	struct Parts {
	43	u32 unused0_ : 8;
	44	u32 sid_ : 8;
	45	u32 epoch_ : kEpochBits;
	46	u32 unused1_ : 1;
	47	u32 ignore_accesses_ : 1;
	48	};
	49	union {
	50	Parts part_;
	51	u32 raw_;
	52	};
	53	};
	54
	55	static_assert(sizeof(FastState) == kShadowSize, "bad FastState size");
	56
	57	class Shadow {
	58	public:
	59	static constexpr RawShadow kEmpty = static_cast<RawShadow>(0);
	60
	61	Shadow(FastState state, u32 addr, u32 size, AccessType typ) {
	62	raw_ = state.raw_;
	63	DCHECK_GT(size, 0);
	64	DCHECK_LE(size, 8);
	65	UNUSED Sid sid0 = part_.sid_;
	66	UNUSED u16 epoch0 = part_.epoch_;
	67	raw_ \|= (!!(typ & kAccessAtomic) << kIsAtomicShift) \|
	68	(!!(typ & kAccessRead) << kIsReadShift) \|
	69	(((((1u << size) - 1) << (addr & 0x7)) & 0xff) << kAccessShift);
	70	// Note: we don't check kAccessAtomic because it overlaps with
	71	// FastState::ignore_accesses_ and it may be set spuriously.
	72	DCHECK_EQ(part_.is_read_, !!(typ & kAccessRead));
	73	DCHECK_EQ(sid(), sid0);
	74	DCHECK_EQ(epoch(), epoch0);
	75	}
	76
	77	explicit Shadow(RawShadow x = Shadow::kEmpty) { raw_ = static_cast<u32>(x); }
	78
	79	RawShadow raw() const { return static_cast<RawShadow>(raw_); }
	80	Sid sid() const { return part_.sid_; }
	81	Epoch epoch() const { return static_cast<Epoch>(part_.epoch_); }
	82	u8 access() const { return part_.access_; }
	83
	84	void GetAccess(uptr addr, uptr size, AccessType *typ) const {
	85	DCHECK(part_.access_ != 0 \|\| raw_ == static_cast<u32>(Shadow::kRodata));
	86	if (addr)
	87	*addr = part_.access_ ? __builtin_ffs(part_.access_) - 1 : 0;
	88	if (size)
	89	*size = part_.access_ == kFreeAccess ? kShadowCell
	90	: __builtin_popcount(part_.access_);
	91	if (typ) {
	92	*typ = part_.is_read_ ? kAccessRead : kAccessWrite;
	93	if (part_.is_atomic_)
	94	*typ \|= kAccessAtomic;
	95	if (part_.access_ == kFreeAccess)
	96	*typ \|= kAccessFree;
	97	}
	98	}
	99
	100	ALWAYS_INLINE
	101	bool IsBothReadsOrAtomic(AccessType typ) const {
	102	u32 is_read = !!(typ & kAccessRead);
	103	u32 is_atomic = !!(typ & kAccessAtomic);
	104	bool res =
	105	raw_ & ((is_atomic << kIsAtomicShift) \| (is_read << kIsReadShift));
	106	DCHECK_EQ(res,
	107	(part_.is_read_ && is_read) \|\| (part_.is_atomic_ && is_atomic));
	108	return res;
	109	}
	110
	111	ALWAYS_INLINE
	112	bool IsRWWeakerOrEqual(AccessType typ) const {
	113	u32 is_read = !!(typ & kAccessRead);
	114	u32 is_atomic = !!(typ & kAccessAtomic);
	115	UNUSED u32 res0 =
	116	(part_.is_atomic_ > is_atomic) \|\|
	117	(part_.is_atomic_ == is_atomic && part_.is_read_ >= is_read);
	118	#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
	119	const u32 kAtomicReadMask = (1 << kIsAtomicShift) \| (1 << kIsReadShift);
	120	bool res = (raw_ & kAtomicReadMask) >=
	121	((is_atomic << kIsAtomicShift) \| (is_read << kIsReadShift));
	122
	123	DCHECK_EQ(res, res0);
	124	return res;
	125	#else
	126	return res0;
	127	#endif
	128	}
	129
	130	// The FreedMarker must not pass "the same access check" so that we don't
	131	// return from the race detection algorithm early.
	132	static RawShadow FreedMarker() {
	133	FastState fs;
	134	fs.SetSid(kFreeSid);
	135	fs.SetEpoch(kEpochLast);
	136	Shadow s(fs, 0, 8, kAccessWrite);
	137	return s.raw();
	138	}
	139
	140	static RawShadow FreedInfo(Sid sid, Epoch epoch) {
	141	Shadow s;
	142	s.part_.sid_ = sid;
	143	s.part_.epoch_ = static_cast<u16>(epoch);
	144	s.part_.access_ = kFreeAccess;
	145	return s.raw();
	146	}
	147
	148	private:
	149	struct Parts {
	150	u8 access_;
	151	Sid sid_;
	152	u16 epoch_ : kEpochBits;
	153	u16 is_read_ : 1;
	154	u16 is_atomic_ : 1;
	155	};
	156	union {
	157	Parts part_;
	158	u32 raw_;
	159	};
	160
	161	static constexpr u8 kFreeAccess = 0x81;
	162
	163	#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
	164	static constexpr uptr kAccessShift = 0;
	165	static constexpr uptr kIsReadShift = 30;
	166	static constexpr uptr kIsAtomicShift = 31;
	167	#else
	168	static constexpr uptr kAccessShift = 24;
	169	static constexpr uptr kIsReadShift = 1;
	170	static constexpr uptr kIsAtomicShift = 0;
	171	#endif
	172
	173	public:
	174	// .rodata shadow marker, see MapRodata and ContainsSameAccessFast.
	175	static constexpr RawShadow kRodata =
	176	static_cast<RawShadow>(1 << kIsReadShift);
	177	};
	178
	179	static_assert(sizeof(Shadow) == kShadowSize, "bad Shadow size");
	180
	181	ALWAYS_INLINE RawShadow LoadShadow(RawShadow *p) {
	182	return static_cast<RawShadow>(
	183	atomic_load((atomic_uint32_t *)p, memory_order_relaxed));
	184	}
	185
	186	ALWAYS_INLINE void StoreShadow(RawShadow *sp, RawShadow s) {
	187	atomic_store((atomic_uint32_t *)sp, static_cast<u32>(s),
	188	memory_order_relaxed);
	189	}
	190
	191	} // namespace __tsan
	192
	193	#endif

lib/tsan/tsan_spinlock_defs_mac.h created+45

...	...	@@ -0,0 +1,45 @@
	1	//===-- tsan_spinlock_defs_mac.h -------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	// Mac-specific forward-declared function defintions that may be
	12	// deprecated in later versions of the OS.
	13	// These are needed for interceptors.
	14	//
	15	//===----------------------------------------------------------------------===//
	16
	17	#if SANITIZER_APPLE
	18
	19	#ifndef TSAN_SPINLOCK_DEFS_MAC_H
	20	#define TSAN_SPINLOCK_DEFS_MAC_H
	21
	22	#include <stdint.h>
	23
	24	extern "C" {
	25
	26	/*
	27	Provides forward declarations related to OSSpinLocks on Darwin. These functions are
	28	deprecated on macOS version 10.12 and later,
	29	and are no longer included in the system headers.
	30
	31	However, the symbols are still available on the system, so we provide these forward
	32	declarations to prevent compilation errors in tsan_interceptors_mac.cpp, which
	33	references these functions when defining TSAN interceptor functions.
	34	*/
	35
	36	typedef int32_t OSSpinLock;
	37
	38	void OSSpinLockLock(volatile OSSpinLock *__lock);
	39	void OSSpinLockUnlock(volatile OSSpinLock *__lock);
	40	bool OSSpinLockTry(volatile OSSpinLock *__lock);
	41
	42	}
	43
	44	#endif //TSAN_SPINLOCK_DEFS_MAC_H
	45	#endif // SANITIZER_APPLE

lib/tsan/tsan_stack_trace.cpp+4-8

...	...	@@ -23,14 +23,10 @@ VarSizeStackTrace::~VarSizeStackTrace() {
23	23	}
24	24
25	25	void VarSizeStackTrace::ResizeBuffer(uptr new_size) {
26		if (trace_buffer) {
27		internal_free(trace_buffer);
28		}
29		trace_buffer =
30		(new_size > 0)
31		? (uptr *)internal_alloc(MBlockStackTrace,
32		new_size * sizeof(trace_buffer[0]))
33		: nullptr;
	26	Free(trace_buffer);
	27	trace_buffer = (new_size > 0)
	28	? (uptr )Alloc(new_size sizeof(trace_buffer[0]))
	29	: nullptr;
34	30	trace = trace_buffer;
35	31	size = new_size;
36	32	}

lib/tsan/tsan_suppressions.cpp+3-2

...	...	@@ -10,15 +10,16 @@
10	10	//
11	11	//===----------------------------------------------------------------------===//
12	12
	13	#include "tsan_suppressions.h"
	14
13	15	#include "sanitizer_common/sanitizer_common.h"
14	16	#include "sanitizer_common/sanitizer_libc.h"
15	17	#include "sanitizer_common/sanitizer_placement_new.h"
16	18	#include "sanitizer_common/sanitizer_suppressions.h"
17		#include "tsan_suppressions.h"
18		#include "tsan_rtl.h"
19	19	#include "tsan_flags.h"
20	20	#include "tsan_mman.h"
21	21	#include "tsan_platform.h"
	22	#include "tsan_rtl.h"
22	23
23	24	#if !SANITIZER_GO
24	25	// Suppressions for true/false positives in standard libraries.

lib/tsan/tsan_symbolize.cpp+2-1

...	...	@@ -110,7 +110,8 @@ ReportLocation *SymbolizeData(uptr addr) {
110	110	DataInfo info;
111	111	if (!Symbolizer::GetOrInit()->SymbolizeData(addr, &info))
112	112	return 0;
113		ReportLocation *ent = ReportLocation::New(ReportLocationGlobal);
	113	auto *ent = New<ReportLocation>();
	114	ent->type = ReportLocationGlobal;
114	115	internal_memcpy(&ent->global, &info, sizeof(info));
115	116	return ent;
116	117	}

lib/tsan/tsan_sync.cpp+64-68

...	...	@@ -18,42 +18,31 @@ namespace __tsan {
18	18
19	19	void DDMutexInit(ThreadState thr, uptr pc, SyncVar s);
20	20
21		SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(0); }
	21	SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(); }
22	22
23		void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, u64 uid) {
	23	void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, bool save_stack) {
	24	Reset();
24	25	this->addr = addr;
25		this->uid = uid;
26		this->next = 0;
27
28		creation_stack_id = 0;
29		if (!SANITIZER_GO) // Go does not use them
	26	next = 0;
	27	if (save_stack && !SANITIZER_GO) // Go does not use them
30	28	creation_stack_id = CurrentStackId(thr, pc);
31	29	if (common_flags()->detect_deadlocks)
32	30	DDMutexInit(thr, pc, this);
33	31	}
34	32
35		void SyncVar::Reset(Processor *proc) {
36		uid = 0;
37		creation_stack_id = 0;
	33	void SyncVar::Reset() {
	34	CHECK(!ctx->resetting);
	35	creation_stack_id = kInvalidStackID;
38	36	owner_tid = kInvalidTid;
39		last_lock = 0;
	37	last_lock.Reset();
40	38	recursion = 0;
41	39	atomic_store_relaxed(&flags, 0);
42
43		if (proc == 0) {
44		CHECK_EQ(clock.size(), 0);
45		CHECK_EQ(read_clock.size(), 0);
46		} else {
47		clock.Reset(&proc->clock_cache);
48		read_clock.Reset(&proc->clock_cache);
49		}
	40	Free(clock);
	41	Free(read_clock);
50	42	}
51	43
52	44	MetaMap::MetaMap()
53		: block_alloc_(LINKER_INITIALIZED, "heap block allocator"),
54		sync_alloc_(LINKER_INITIALIZED, "sync allocator") {
55		atomic_store(&uid_gen_, 0, memory_order_relaxed);
56		}
	45	: block_alloc_("heap block allocator"), sync_alloc_("sync allocator") {}
57	46
58	47	void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) {
59	48	u32 idx = block_alloc_.Alloc(&thr->proc()->block_cache);
...	...	@@ -67,16 +56,16 @@ void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) {
67	56	*meta = idx \| kFlagBlock;
68	57	}
69	58
70		uptr MetaMap::FreeBlock(Processor *proc, uptr p) {
	59	uptr MetaMap::FreeBlock(Processor *proc, uptr p, bool reset) {
71	60	MBlock* b = GetBlock(p);
72	61	if (b == 0)
73	62	return 0;
74	63	uptr sz = RoundUpTo(b->siz, kMetaShadowCell);
75		FreeRange(proc, p, sz);
	64	FreeRange(proc, p, sz, reset);
76	65	return sz;
77	66	}
78	67
79		bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) {
	68	bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz, bool reset) {
80	69	bool has_something = false;
81	70	u32 *meta = MemToMeta(p);
82	71	u32 *end = MemToMeta(p + sz);
...	...	@@ -98,7 +87,8 @@ bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) {
98	87	DCHECK(idx & kFlagSync);
99	88	SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask);
100	89	u32 next = s->next;
101		s->Reset(proc);
	90	if (reset)
	91	s->Reset();
102	92	sync_alloc_.Free(&proc->sync_cache, idx & ~kFlagMask);
103	93	idx = next;
104	94	} else {
...	...	@@ -115,30 +105,30 @@ bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) {
115	105	// which can be huge. The function probes pages one-by-one until it finds a page
116	106	// without meta objects, at this point it stops freeing meta objects. Because
117	107	// thread stacks grow top-down, we do the same starting from end as well.
118		void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
	108	void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz, bool reset) {
119	109	if (SANITIZER_GO) {
120	110	// UnmapOrDie/MmapFixedNoReserve does not work on Windows,
121	111	// so we do the optimization only for C/C++.
122		FreeRange(proc, p, sz);
	112	FreeRange(proc, p, sz, reset);
123	113	return;
124	114	}
125	115	const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize;
126	116	const uptr kPageSize = GetPageSizeCached() * kMetaRatio;
127	117	if (sz <= 4 * kPageSize) {
128	118	// If the range is small, just do the normal free procedure.
129		FreeRange(proc, p, sz);
	119	FreeRange(proc, p, sz, reset);
130	120	return;
131	121	}
132	122	// First, round both ends of the range to page size.
133	123	uptr diff = RoundUp(p, kPageSize) - p;
134	124	if (diff != 0) {
135		FreeRange(proc, p, diff);
	125	FreeRange(proc, p, diff, reset);
136	126	p += diff;
137	127	sz -= diff;
138	128	}
139	129	diff = p + sz - RoundDown(p + sz, kPageSize);
140	130	if (diff != 0) {
141		FreeRange(proc, p + sz - diff, diff);
	131	FreeRange(proc, p + sz - diff, diff, reset);
142	132	sz -= diff;
143	133	}
144	134	// Now we must have a non-empty page-aligned range.
...	...	@@ -149,7 +139,7 @@ void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
149	139	const uptr sz0 = sz;
150	140	// Probe start of the range.
151	141	for (uptr checked = 0; sz > 0; checked += kPageSize) {
152		bool has_something = FreeRange(proc, p, kPageSize);
	142	bool has_something = FreeRange(proc, p, kPageSize, reset);
153	143	p += kPageSize;
154	144	sz -= kPageSize;
155	145	if (!has_something && checked > (128 << 10))
...	...	@@ -157,7 +147,7 @@ void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
157	147	}
158	148	// Probe end of the range.
159	149	for (uptr checked = 0; sz > 0; checked += kPageSize) {
160		bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize);
	150	bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize, reset);
161	151	sz -= kPageSize;
162	152	// Stacks grow down, so sync object are most likely at the end of the region
163	153	// (if it is a stack). The very end of the stack is TLS and tsan increases
...	...	@@ -176,6 +166,27 @@ void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
176	166	Die();
177	167	}
178	168
	169	void MetaMap::ResetClocks() {
	170	// This can be called from the background thread
	171	// which does not have proc/cache.
	172	// The cache is too large for stack.
	173	static InternalAllocatorCache cache;
	174	internal_memset(&cache, 0, sizeof(cache));
	175	internal_allocator()->InitCache(&cache);
	176	sync_alloc_.ForEach([&](SyncVar *s) {
	177	if (s->clock) {
	178	InternalFree(s->clock, &cache);
	179	s->clock = nullptr;
	180	}
	181	if (s->read_clock) {
	182	InternalFree(s->read_clock, &cache);
	183	s->read_clock = nullptr;
	184	}
	185	s->last_lock.Reset();
	186	});
	187	internal_allocator()->DestroyCache(&cache);
	188	}
	189
179	190	MBlock* MetaMap::GetBlock(uptr p) {
180	191	u32 *meta = MemToMeta(p);
181	192	u32 idx = *meta;
...	...	@@ -190,63 +201,41 @@ MBlock* MetaMap::GetBlock(uptr p) {
190	201	}
191	202	}
192	203
193		SyncVar* MetaMap::GetOrCreateAndLock(ThreadState *thr, uptr pc,
194		uptr addr, bool write_lock) {
195		return GetAndLock(thr, pc, addr, write_lock, true);
196		}
197
198		SyncVar* MetaMap::GetIfExistsAndLock(uptr addr, bool write_lock) {
199		return GetAndLock(0, 0, addr, write_lock, false);
200		}
201
202		SyncVar MetaMap::GetAndLock(ThreadState thr, uptr pc, uptr addr, bool write_lock,
203		bool create) NO_THREAD_SAFETY_ANALYSIS {
	204	SyncVar MetaMap::GetSync(ThreadState thr, uptr pc, uptr addr, bool create,
	205	bool save_stack) {
	206	DCHECK(!create \|\| thr->slot_locked);
204	207	u32 *meta = MemToMeta(addr);
205	208	u32 idx0 = *meta;
206	209	u32 myidx = 0;
207		SyncVar *mys = 0;
	210	SyncVar *mys = nullptr;
208	211	for (;;) {
209		u32 idx = idx0;
210		for (;;) {
211		if (idx == 0)
212		break;
213		if (idx & kFlagBlock)
214		break;
	212	for (u32 idx = idx0; idx && !(idx & kFlagBlock);) {
215	213	DCHECK(idx & kFlagSync);
216	214	SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask);
217		if (s->addr == addr) {
218		if (myidx != 0) {
219		mys->Reset(thr->proc());
	215	if (LIKELY(s->addr == addr)) {
	216	if (UNLIKELY(myidx != 0)) {
	217	mys->Reset();
220	218	sync_alloc_.Free(&thr->proc()->sync_cache, myidx);
221	219	}
222		if (write_lock)
223		s->mtx.Lock();
224		else
225		s->mtx.ReadLock();
226	220	return s;
227	221	}
228	222	idx = s->next;
229	223	}
230	224	if (!create)
231		return 0;
232		if (*meta != idx0) {
	225	return nullptr;
	226	if (UNLIKELY(*meta != idx0)) {
233	227	idx0 = *meta;
234	228	continue;
235	229	}
236	230
237		if (myidx == 0) {
238		const u64 uid = atomic_fetch_add(&uid_gen_, 1, memory_order_relaxed);
	231	if (LIKELY(myidx == 0)) {
239	232	myidx = sync_alloc_.Alloc(&thr->proc()->sync_cache);
240	233	mys = sync_alloc_.Map(myidx);
241		mys->Init(thr, pc, addr, uid);
	234	mys->Init(thr, pc, addr, save_stack);
242	235	}
243	236	mys->next = idx0;
244	237	if (atomic_compare_exchange_strong((atomic_uint32_t*)meta, &idx0,
245	238	myidx \| kFlagSync, memory_order_release)) {
246		if (write_lock)
247		mys->mtx.Lock();
248		else
249		mys->mtx.ReadLock();
250	239	return mys;
251	240	}
252	241	}
...	...	@@ -290,4 +279,11 @@ void MetaMap::OnProcIdle(Processor *proc) {
290	279	sync_alloc_.FlushCache(&proc->sync_cache);
291	280	}
292	281
	282	MetaMap::MemoryStats MetaMap::GetMemoryStats() const {
	283	MemoryStats stats;
	284	stats.mem_block = block_alloc_.AllocatedMemory();
	285	stats.sync_obj = sync_alloc_.AllocatedMemory();
	286	return stats;
	287	}
	288
293	289	} // namespace __tsan

lib/tsan/tsan_sync.h+43-37

...	...	@@ -16,8 +16,9 @@
16	16	#include "sanitizer_common/sanitizer_common.h"
17	17	#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
18	18	#include "tsan_defs.h"
19		#include "tsan_clock.h"
20	19	#include "tsan_dense_alloc.h"
	20	#include "tsan_shadow.h"
	21	#include "tsan_vector_clock.h"
21	22
22	23	namespace __tsan {
23	24
...	...	@@ -46,39 +47,25 @@ enum MutexFlags {
46	47	MutexFlagNotStatic,
47	48	};
48	49
	50	// SyncVar is a descriptor of a user synchronization object
	51	// (mutex or an atomic variable).
49	52	struct SyncVar {
50	53	SyncVar();
51	54
52	55	uptr addr; // overwritten by DenseSlabAlloc freelist
53	56	Mutex mtx;
54		u64 uid; // Globally unique id.
55		u32 creation_stack_id;
56		u32 owner_tid; // Set only by exclusive owners.
57		u64 last_lock;
	57	StackID creation_stack_id;
	58	Tid owner_tid; // Set only by exclusive owners.
	59	FastState last_lock;
58	60	int recursion;
59	61	atomic_uint32_t flags;
60	62	u32 next; // in MetaMap
61	63	DDMutex dd;
62		SyncClock read_clock; // Used for rw mutexes only.
63		// The clock is placed last, so that it is situated on a different cache line
64		// with the mtx. This reduces contention for hot sync objects.
65		SyncClock clock;
	64	VectorClock *read_clock; // Used for rw mutexes only.
	65	VectorClock *clock;
66	66
67		void Init(ThreadState *thr, uptr pc, uptr addr, u64 uid);
68		void Reset(Processor *proc);
69
70		u64 GetId() const {
71		// 48 lsb is addr, then 14 bits is low part of uid, then 2 zero bits.
72		return GetLsb((u64)addr \| (uid << 48), 60);
73		}
74		bool CheckId(u64 uid) const {
75		CHECK_EQ(uid, GetLsb(uid, 14));
76		return GetLsb(this->uid, 14) == uid;
77		}
78		static uptr SplitId(u64 id, u64 *uid) {
79		*uid = id >> 48;
80		return (uptr)GetLsb(id, 48);
81		}
	67	void Init(ThreadState *thr, uptr pc, uptr addr, bool save_stack);
	68	void Reset();
82	69
83	70	bool IsFlagSet(u32 f) const {
84	71	return atomic_load_relaxed(&flags) & f;
...	...	@@ -101,28 +88,48 @@ struct SyncVar {
101	88	}
102	89	};
103	90
104		/* MetaMap allows to map arbitrary user pointers onto various descriptors.
105		Currently it maps pointers to heap block descriptors and sync var descs.
106		It uses 1/2 direct shadow, see tsan_platform.h.
107		*/
	91	// MetaMap maps app addresses to heap block (MBlock) and sync var (SyncVar)
	92	// descriptors. It uses 1/2 direct shadow, see tsan_platform.h for the mapping.
108	93	class MetaMap {
109	94	public:
110	95	MetaMap();
111	96
112	97	void AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz);
113		uptr FreeBlock(Processor *proc, uptr p);
114		bool FreeRange(Processor *proc, uptr p, uptr sz);
115		void ResetRange(Processor *proc, uptr p, uptr sz);
	98
	99	// FreeBlock resets all sync objects in the range if reset=true and must not
	100	// run concurrently with ResetClocks which resets all sync objects
	101	// w/o any synchronization (as part of DoReset).
	102	// If we don't have a thread slot (very early/late in thread lifetime or
	103	// Go/Java callbacks) or the slot is not locked, then reset must be set to
	104	// false. In such case sync object clocks will be reset later (when it's
	105	// reused or during the next ResetClocks).
	106	uptr FreeBlock(Processor *proc, uptr p, bool reset);
	107	bool FreeRange(Processor *proc, uptr p, uptr sz, bool reset);
	108	void ResetRange(Processor *proc, uptr p, uptr sz, bool reset);
	109	// Reset vector clocks of all sync objects.
	110	// Must be called when no other threads access sync objects.
	111	void ResetClocks();
116	112	MBlock* GetBlock(uptr p);
117	113
118		SyncVar* GetOrCreateAndLock(ThreadState *thr, uptr pc,
119		uptr addr, bool write_lock);
120		SyncVar* GetIfExistsAndLock(uptr addr, bool write_lock);
	114	SyncVar GetSyncOrCreate(ThreadState thr, uptr pc, uptr addr,
	115	bool save_stack) {
	116	return GetSync(thr, pc, addr, true, save_stack);
	117	}
	118	SyncVar *GetSyncIfExists(uptr addr) {
	119	return GetSync(nullptr, 0, addr, false, false);
	120	}
121	121
122	122	void MoveMemory(uptr src, uptr dst, uptr sz);
123	123
124	124	void OnProcIdle(Processor *proc);
125	125
	126	struct MemoryStats {
	127	uptr mem_block;
	128	uptr sync_obj;
	129	};
	130
	131	MemoryStats GetMemoryStats() const;
	132
126	133	private:
127	134	static const u32 kFlagMask = 3u << 30;
128	135	static const u32 kFlagBlock = 1u << 30;
...	...	@@ -131,10 +138,9 @@ class MetaMap {
131	138	typedef DenseSlabAlloc<SyncVar, 1 << 20, 1 << 10, kFlagMask> SyncAlloc;
132	139	BlockAlloc block_alloc_;
133	140	SyncAlloc sync_alloc_;
134		atomic_uint64_t uid_gen_;
135	141
136		SyncVar* GetAndLock(ThreadState *thr, uptr pc, uptr addr, bool write_lock,
137		bool create);
	142	SyncVar GetSync(ThreadState thr, uptr pc, uptr addr, bool create,
	143	bool save_stack);
138	144	};
139	145
140	146	} // namespace __tsan

lib/tsan/tsan_trace.h+180-37

...	...	@@ -13,58 +13,201 @@
13	13	#define TSAN_TRACE_H
14	14
15	15	#include "tsan_defs.h"
16		#include "tsan_stack_trace.h"
	16	#include "tsan_ilist.h"
17	17	#include "tsan_mutexset.h"
	18	#include "tsan_stack_trace.h"
18	19
19	20	namespace __tsan {
20	21
21		const int kTracePartSizeBits = 13;
22		const int kTracePartSize = 1 << kTracePartSizeBits;
23		const int kTraceParts = 2 * 1024 * 1024 / kTracePartSize;
24		const int kTraceSize = kTracePartSize * kTraceParts;
25
26		// Must fit into 3 bits.
27		enum EventType {
28		EventTypeMop,
29		EventTypeFuncEnter,
30		EventTypeFuncExit,
31		EventTypeLock,
32		EventTypeUnlock,
33		EventTypeRLock,
34		EventTypeRUnlock
	22	enum class EventType : u64 {
	23	kAccessExt,
	24	kAccessRange,
	25	kLock,
	26	kRLock,
	27	kUnlock,
	28	kTime,
	29	};
	30
	31	// "Base" type for all events for type dispatch.
	32	struct Event {
	33	// We use variable-length type encoding to give more bits to some event
	34	// types that need them. If is_access is set, this is EventAccess.
	35	// Otherwise, if is_func is set, this is EventFunc.
	36	// Otherwise type denotes the type.
	37	u64 is_access : 1;
	38	u64 is_func : 1;
	39	EventType type : 3;
	40	u64 _ : 59;
	41	};
	42	static_assert(sizeof(Event) == 8, "bad Event size");
	43
	44	// Nop event used as padding and does not affect state during replay.
	45	static constexpr Event NopEvent = {1, 0, EventType::kAccessExt, 0};
	46
	47	// Compressed memory access can represent only some events with PCs
	48	// close enough to each other. Otherwise we fall back to EventAccessExt.
	49	struct EventAccess {
	50	static constexpr uptr kPCBits = 15;
	51	static_assert(kPCBits + kCompressedAddrBits + 5 == 64,
	52	"unused bits in EventAccess");
	53
	54	u64 is_access : 1; // = 1
	55	u64 is_read : 1;
	56	u64 is_atomic : 1;
	57	u64 size_log : 2;
	58	u64 pc_delta : kPCBits; // signed delta from the previous memory access PC
	59	u64 addr : kCompressedAddrBits;
35	60	};
	61	static_assert(sizeof(EventAccess) == 8, "bad EventAccess size");
36	62
37		// Represents a thread event (from most significant bit):
38		// u64 typ : 3; // EventType.
39		// u64 addr : 61; // Associated pc.
40		typedef u64 Event;
	63	// Function entry (pc != 0) or exit (pc == 0).
	64	struct EventFunc {
	65	u64 is_access : 1; // = 0
	66	u64 is_func : 1; // = 1
	67	u64 pc : 62;
	68	};
	69	static_assert(sizeof(EventFunc) == 8, "bad EventFunc size");
	70
	71	// Extended memory access with full PC.
	72	struct EventAccessExt {
	73	// Note: precisely specifying the unused parts of the bitfield is critical for
	74	// performance. If we don't specify them, compiler will generate code to load
	75	// the old value and shuffle it to extract the unused bits to apply to the new
	76	// value. If we specify the unused part and store 0 in there, all that
	77	// unnecessary code goes away (store of the 0 const is combined with other
	78	// constant parts).
	79	static constexpr uptr kUnusedBits = 11;
	80	static_assert(kCompressedAddrBits + kUnusedBits + 9 == 64,
	81	"unused bits in EventAccessExt");
	82
	83	u64 is_access : 1; // = 0
	84	u64 is_func : 1; // = 0
	85	EventType type : 3; // = EventType::kAccessExt
	86	u64 is_read : 1;
	87	u64 is_atomic : 1;
	88	u64 size_log : 2;
	89	u64 _ : kUnusedBits;
	90	u64 addr : kCompressedAddrBits;
	91	u64 pc;
	92	};
	93	static_assert(sizeof(EventAccessExt) == 16, "bad EventAccessExt size");
	94
	95	// Access to a memory range.
	96	struct EventAccessRange {
	97	static constexpr uptr kSizeLoBits = 13;
	98	static_assert(kCompressedAddrBits + kSizeLoBits + 7 == 64,
	99	"unused bits in EventAccessRange");
	100
	101	u64 is_access : 1; // = 0
	102	u64 is_func : 1; // = 0
	103	EventType type : 3; // = EventType::kAccessRange
	104	u64 is_read : 1;
	105	u64 is_free : 1;
	106	u64 size_lo : kSizeLoBits;
	107	u64 pc : kCompressedAddrBits;
	108	u64 addr : kCompressedAddrBits;
	109	u64 size_hi : 64 - kCompressedAddrBits;
	110	};
	111	static_assert(sizeof(EventAccessRange) == 16, "bad EventAccessRange size");
41	112
42		const uptr kEventPCBits = 61;
	113	// Mutex lock.
	114	struct EventLock {
	115	static constexpr uptr kStackIDLoBits = 15;
	116	static constexpr uptr kStackIDHiBits =
	117	sizeof(StackID) * kByteBits - kStackIDLoBits;
	118	static constexpr uptr kUnusedBits = 3;
	119	static_assert(kCompressedAddrBits + kStackIDLoBits + 5 == 64,
	120	"unused bits in EventLock");
	121	static_assert(kCompressedAddrBits + kStackIDHiBits + kUnusedBits == 64,
	122	"unused bits in EventLock");
	123
	124	u64 is_access : 1; // = 0
	125	u64 is_func : 1; // = 0
	126	EventType type : 3; // = EventType::kLock or EventType::kRLock
	127	u64 pc : kCompressedAddrBits;
	128	u64 stack_lo : kStackIDLoBits;
	129	u64 stack_hi : sizeof(StackID) * kByteBits - kStackIDLoBits;
	130	u64 _ : kUnusedBits;
	131	u64 addr : kCompressedAddrBits;
	132	};
	133	static_assert(sizeof(EventLock) == 16, "bad EventLock size");
	134
	135	// Mutex unlock.
	136	struct EventUnlock {
	137	static constexpr uptr kUnusedBits = 15;
	138	static_assert(kCompressedAddrBits + kUnusedBits + 5 == 64,
	139	"unused bits in EventUnlock");
	140
	141	u64 is_access : 1; // = 0
	142	u64 is_func : 1; // = 0
	143	EventType type : 3; // = EventType::kUnlock
	144	u64 _ : kUnusedBits;
	145	u64 addr : kCompressedAddrBits;
	146	};
	147	static_assert(sizeof(EventUnlock) == 8, "bad EventUnlock size");
	148
	149	// Time change event.
	150	struct EventTime {
	151	static constexpr uptr kUnusedBits = 37;
	152	static_assert(kUnusedBits + sizeof(Sid) * kByteBits + kEpochBits + 5 == 64,
	153	"unused bits in EventTime");
	154
	155	u64 is_access : 1; // = 0
	156	u64 is_func : 1; // = 0
	157	EventType type : 3; // = EventType::kTime
	158	u64 sid : sizeof(Sid) * kByteBits;
	159	u64 epoch : kEpochBits;
	160	u64 _ : kUnusedBits;
	161	};
	162	static_assert(sizeof(EventTime) == 8, "bad EventTime size");
	163
	164	struct Trace;
43	165
44	166	struct TraceHeader {
45		#if !SANITIZER_GO
46		BufferedStackTrace stack0; // Start stack for the trace.
47		#else
48		VarSizeStackTrace stack0;
49		#endif
50		u64 epoch0; // Start epoch for the trace.
51		MutexSet mset0;
52
53		TraceHeader() : stack0(), epoch0() {}
	167	Trace* trace = nullptr; // back-pointer to Trace containing this part
	168	INode trace_parts; // in Trace::parts
	169	INode global; // in Contex::trace_part_recycle
54	170	};
55	171
	172	struct TracePart : TraceHeader {
	173	// There are a lot of goroutines in Go, so we use smaller parts.
	174	static constexpr uptr kByteSize = (SANITIZER_GO ? 128 : 256) << 10;
	175	static constexpr uptr kSize =
	176	(kByteSize - sizeof(TraceHeader)) / sizeof(Event);
	177	// TraceAcquire does a fast event pointer overflow check by comparing
	178	// pointer into TracePart::events with kAlignment mask. Since TracePart's
	179	// are allocated page-aligned, this check detects end of the array
	180	// (it also have false positives in the middle that are filtered separately).
	181	// This also requires events to be the last field.
	182	static constexpr uptr kAlignment = 0xff0;
	183	Event events[kSize];
	184
	185	TracePart() {}
	186	};
	187	static_assert(sizeof(TracePart) == TracePart::kByteSize, "bad TracePart size");
	188
56	189	struct Trace {
57	190	Mutex mtx;
58		#if !SANITIZER_GO
59		// Must be last to catch overflow as paging fault.
60		// Go shadow stack is dynamically allocated.
61		uptr shadow_stack[kShadowStackSize];
62		#endif
63		// Must be the last field, because we unmap the unused part in
64		// CreateThreadContext.
65		TraceHeader headers[kTraceParts];
	191	IList<TraceHeader, &TraceHeader::trace_parts, TracePart> parts;
	192	// First node non-queued into ctx->trace_part_recycle.
	193	TracePart* local_head;
	194	// Final position in the last part for finished threads.
	195	Event* final_pos = nullptr;
	196	// Number of trace parts allocated on behalf of this trace specifically.
	197	// Total number of parts in this trace can be larger if we retake some
	198	// parts from other traces.
	199	uptr parts_allocated = 0;
66	200
67	201	Trace() : mtx(MutexTypeTrace) {}
	202
	203	// We need at least 3 parts per thread, because we want to keep at last
	204	// 2 parts per thread that are not queued into ctx->trace_part_recycle
	205	// (the current one being filled and one full part that ensures that
	206	// we always have at least one part worth of previous memory accesses).
	207	static constexpr uptr kMinParts = 3;
	208
	209	static constexpr uptr kFinishedThreadLo = 16;
	210	static constexpr uptr kFinishedThreadHi = 64;
68	211	};
69	212
70	213	} // namespace __tsan

lib/tsan/tsan_update_shadow_word_inl.h deleted-59

...	...	@@ -1,59 +0,0 @@
1		//===-- tsan_update_shadow_word_inl.h ---------------------------- C++ --===//
2		//
3		// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4		// See https://llvm.org/LICENSE.txt for license information.
5		// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6		//
7		//===----------------------------------------------------------------------===//
8		//
9		// This file is a part of ThreadSanitizer (TSan), a race detector.
10		//
11		// Body of the hottest inner loop.
12		// If we wrap this body into a function, compilers (both gcc and clang)
13		// produce sligtly less efficient code.
14		//===----------------------------------------------------------------------===//
15		do {
16		const unsigned kAccessSize = 1 << kAccessSizeLog;
17		u64 *sp = &shadow_mem[idx];
18		old = LoadShadow(sp);
19		if (LIKELY(old.IsZero())) {
20		if (!stored) {
21		StoreIfNotYetStored(sp, &store_word);
22		stored = true;
23		}
24		break;
25		}
26		// is the memory access equal to the previous?
27		if (LIKELY(Shadow::Addr0AndSizeAreEqual(cur, old))) {
28		// same thread?
29		if (LIKELY(Shadow::TidsAreEqual(old, cur))) {
30		if (LIKELY(old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))) {
31		StoreIfNotYetStored(sp, &store_word);
32		stored = true;
33		}
34		break;
35		}
36		if (HappensBefore(old, thr)) {
37		if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) {
38		StoreIfNotYetStored(sp, &store_word);
39		stored = true;
40		}
41		break;
42		}
43		if (LIKELY(old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)))
44		break;
45		goto RACE;
46		}
47		// Do the memory access intersect?
48		if (Shadow::TwoRangesIntersect(old, cur, kAccessSize)) {
49		if (Shadow::TidsAreEqual(old, cur))
50		break;
51		if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
52		break;
53		if (LIKELY(HappensBefore(old, thr)))
54		break;
55		goto RACE;
56		}
57		// The accesses do not intersect.
58		break;
59		} while (0);

lib/tsan/tsan_vector_clock.cpp created+126

...	...	@@ -0,0 +1,126 @@
	1	//===-- tsan_vector_clock.cpp ---------------------------------------------===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	//===----------------------------------------------------------------------===//
	12	#include "tsan_vector_clock.h"
	13
	14	#include "sanitizer_common/sanitizer_placement_new.h"
	15	#include "tsan_mman.h"
	16
	17	namespace __tsan {
	18
	19	#if TSAN_VECTORIZE
	20	const uptr kVectorClockSize = kThreadSlotCount * sizeof(Epoch) / sizeof(m128);
	21	#endif
	22
	23	VectorClock::VectorClock() { Reset(); }
	24
	25	void VectorClock::Reset() {
	26	#if !TSAN_VECTORIZE
	27	for (uptr i = 0; i < kThreadSlotCount; i++)
	28	clk_[i] = kEpochZero;
	29	#else
	30	m128 z = _mm_setzero_si128();
	31	m128* vclk = reinterpret_cast<m128*>(clk_);
	32	for (uptr i = 0; i < kVectorClockSize; i++) _mm_store_si128(&vclk[i], z);
	33	#endif
	34	}
	35
	36	void VectorClock::Acquire(const VectorClock* src) {
	37	if (!src)
	38	return;
	39	#if !TSAN_VECTORIZE
	40	for (uptr i = 0; i < kThreadSlotCount; i++)
	41	clk_[i] = max(clk_[i], src->clk_[i]);
	42	#else
	43	m128* __restrict vdst = reinterpret_cast<m128*>(clk_);
	44	m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(src->clk_);
	45	for (uptr i = 0; i < kVectorClockSize; i++) {
	46	m128 s = _mm_load_si128(&vsrc[i]);
	47	m128 d = _mm_load_si128(&vdst[i]);
	48	m128 m = _mm_max_epu16(s, d);
	49	_mm_store_si128(&vdst[i], m);
	50	}
	51	#endif
	52	}
	53
	54	static VectorClock* AllocClock(VectorClock** dstp) {
	55	if (UNLIKELY(!*dstp))
	56	*dstp = New<VectorClock>();
	57	return *dstp;
	58	}
	59
	60	void VectorClock::Release(VectorClock** dstp) const {
	61	VectorClock* dst = AllocClock(dstp);
	62	dst->Acquire(this);
	63	}
	64
	65	void VectorClock::ReleaseStore(VectorClock** dstp) const {
	66	VectorClock* dst = AllocClock(dstp);
	67	dst = this;
	68	}
	69
	70	VectorClock& VectorClock::operator=(const VectorClock& other) {
	71	#if !TSAN_VECTORIZE
	72	for (uptr i = 0; i < kThreadSlotCount; i++)
	73	clk_[i] = other.clk_[i];
	74	#else
	75	m128* __restrict vdst = reinterpret_cast<m128*>(clk_);
	76	m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(other.clk_);
	77	for (uptr i = 0; i < kVectorClockSize; i++) {
	78	m128 s = _mm_load_si128(&vsrc[i]);
	79	_mm_store_si128(&vdst[i], s);
	80	}
	81	#endif
	82	return *this;
	83	}
	84
	85	void VectorClock::ReleaseStoreAcquire(VectorClock** dstp) {
	86	VectorClock* dst = AllocClock(dstp);
	87	#if !TSAN_VECTORIZE
	88	for (uptr i = 0; i < kThreadSlotCount; i++) {
	89	Epoch tmp = dst->clk_[i];
	90	dst->clk_[i] = clk_[i];
	91	clk_[i] = max(clk_[i], tmp);
	92	}
	93	#else
	94	m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);
	95	m128* __restrict vclk = reinterpret_cast<m128*>(clk_);
	96	for (uptr i = 0; i < kVectorClockSize; i++) {
	97	m128 t = _mm_load_si128(&vdst[i]);
	98	m128 c = _mm_load_si128(&vclk[i]);
	99	m128 m = _mm_max_epu16(c, t);
	100	_mm_store_si128(&vdst[i], c);
	101	_mm_store_si128(&vclk[i], m);
	102	}
	103	#endif
	104	}
	105
	106	void VectorClock::ReleaseAcquire(VectorClock** dstp) {
	107	VectorClock* dst = AllocClock(dstp);
	108	#if !TSAN_VECTORIZE
	109	for (uptr i = 0; i < kThreadSlotCount; i++) {
	110	dst->clk_[i] = max(dst->clk_[i], clk_[i]);
	111	clk_[i] = dst->clk_[i];
	112	}
	113	#else
	114	m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);
	115	m128* __restrict vclk = reinterpret_cast<m128*>(clk_);
	116	for (uptr i = 0; i < kVectorClockSize; i++) {
	117	m128 c = _mm_load_si128(&vclk[i]);
	118	m128 d = _mm_load_si128(&vdst[i]);
	119	m128 m = _mm_max_epu16(c, d);
	120	_mm_store_si128(&vdst[i], m);
	121	_mm_store_si128(&vclk[i], m);
	122	}
	123	#endif
	124	}
	125
	126	} // namespace __tsan

lib/tsan/tsan_vector_clock.h created+51

...	...	@@ -0,0 +1,51 @@
	1	//===-- tsan_vector_clock.h -------------------------------------- C++ --===//
	2	//
	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer (TSan), a race detector.
	10	//
	11	//===----------------------------------------------------------------------===//
	12	#ifndef TSAN_VECTOR_CLOCK_H
	13	#define TSAN_VECTOR_CLOCK_H
	14
	15	#include "tsan_defs.h"
	16
	17	namespace __tsan {
	18
	19	// Fixed-size vector clock, used both for threads and sync objects.
	20	class VectorClock {
	21	public:
	22	VectorClock();
	23
	24	Epoch Get(Sid sid) const;
	25	void Set(Sid sid, Epoch v);
	26
	27	void Reset();
	28	void Acquire(const VectorClock* src);
	29	void Release(VectorClock** dstp) const;
	30	void ReleaseStore(VectorClock** dstp) const;
	31	void ReleaseStoreAcquire(VectorClock** dstp);
	32	void ReleaseAcquire(VectorClock** dstp);
	33
	34	VectorClock& operator=(const VectorClock& other);
	35
	36	private:
	37	Epoch clk_[kThreadSlotCount] VECTOR_ALIGNED;
	38	};
	39
	40	ALWAYS_INLINE Epoch VectorClock::Get(Sid sid) const {
	41	return clk_[static_cast<u8>(sid)];
	42	}
	43
	44	ALWAYS_INLINE void VectorClock::Set(Sid sid, Epoch v) {
	45	DCHECK_GE(v, clk_[static_cast<u8>(sid)]);
	46	clk_[static_cast<u8>(sid)] = v;
	47	}
	48
	49	} // namespace __tsan
	50
	51	#endif // TSAN_VECTOR_CLOCK_H