Commit df6aed0fc3a85285ad9bf15c942a66255c2b0b31 | zig

lib/tsan/interception/interception.h+145-71

 @@ -14,9 +14,10 @@
 #ifndef INTERCEPTION_H
 #define INTERCEPTION_H
+#include "sanitizer_common/sanitizer_asm.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
-#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_MAC &&      \
+#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_APPLE &&    \
     !SANITIZER_NETBSD && !SANITIZER_WINDOWS && !SANITIZER_FUCHSIA && \
     !SANITIZER_SOLARIS
 #  error "Interception doesn't work on this operating system."
 @@ -67,28 +68,54 @@ typedef __sanitizer::OFF64_T OFF64_T;
 //           for more details). To intercept such functions you need to use the
 //           INTERCEPTOR_WITH_SUFFIX(...) macro.
-// How it works:
+// How it works on Linux
-// To replace system functions on Linux we just need to declare functions
+// ---------------------
-// with same names in our library and then obtain the real function pointers
+//
+// To replace system functions on Linux we just need to declare functions with
+// the same names in our library and then obtain the real function pointers
 // using dlsym().
-// There is one complication. A user may also intercept some of the functions
-// we intercept. To resolve this we declare our interceptors with __interceptor_
-// prefix, and then make actual interceptors weak aliases to __interceptor_
-// functions.
 //
-// This is not so on Mac OS, where the two-level namespace makes
+// There is one complication: a user may also intercept some of the functions we
-// our replacement functions invisible to other libraries. This may be overcomed
+// intercept. To allow for up to 3 interceptors (including ours) of a given
-// using the DYLD_FORCE_FLAT_NAMESPACE, but some errors loading the shared
+// function "func", the interceptor implementation is in ___interceptor_func,
-// libraries in Chromium were noticed when doing so.
+// which is aliased by a weak function __interceptor_func, which in turn is
+// aliased (via a trampoline) by weak wrapper function "func".
+//
+// Most user interceptors should define a foreign interceptor as follows:
+//
+//  - provide a non-weak function "func" that performs interception;
+//  - if __interceptor_func exists, call it to perform the real functionality;
+//  - if it does not exist, figure out the real function and call it instead.
+//
+// In rare cases, a foreign interceptor (of another dynamic analysis runtime)
+// may be defined as follows (on supported architectures):
+//
+//  - provide a non-weak function __interceptor_func that performs interception;
+//  - if ___interceptor_func exists, call it to perform the real functionality;
+//  - if it does not exist, figure out the real function and call it instead;
+//  - provide a weak function "func" that is an alias to __interceptor_func.
+//
+// With this protocol, sanitizer interceptors, foreign user interceptors, and
+// foreign interceptors of other dynamic analysis runtimes, or any combination
+// thereof, may co-exist simultaneously.
+//
+// How it works on Mac OS
+// ----------------------
+//
+// This is not so on Mac OS, where the two-level namespace makes our replacement
+// functions invisible to other libraries. This may be overcomed using the
+// DYLD_FORCE_FLAT_NAMESPACE, but some errors loading the shared libraries in
+// Chromium were noticed when doing so.
+//
 // Instead we create a dylib containing a __DATA,__interpose section that
 // associates library functions with their wrappers. When this dylib is
-// preloaded before an executable using DYLD_INSERT_LIBRARIES, it routes all
+// preloaded before an executable using DYLD_INSERT_LIBRARIES, it routes all the
-// the calls to interposed functions done through stubs to the wrapper
+// calls to interposed functions done through stubs to the wrapper functions.
-// functions.
+//
 // As it's decided at compile time which functions are to be intercepted on Mac,
 // INTERCEPT_FUNCTION() is effectively a no-op on this system.
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
 #include <sys/cdefs.h>  // For __DARWIN_ALIAS_C().
 // Just a pair of pointers.
 @@ -100,53 +127,102 @@ struct interpose_substitution {
 // For a function foo() create a global pair of pointers { wrap_foo, foo } in
 // the __DATA,__interpose section.
 // As a result all the calls to foo() will be routed to wrap_foo() at runtime.
-#define INTERPOSER(func_name) __attribute__((used)) \
+#define INTERPOSER(func_name) __attribute__((used))     \
 const interpose_substitution substitution_##func_name[] \
     __attribute__((section("__DATA, __interpose"))) = { \
-    { reinterpret_cast<const uptr>(WRAP(func_name)), \
+    { reinterpret_cast<const uptr>(WRAP(func_name)),    \
-      reinterpret_cast<const uptr>(func_name) } \
+      reinterpret_cast<const uptr>(func_name) }         \
 }
 // For a function foo() and a wrapper function bar() create a global pair
 // of pointers { bar, foo } in the __DATA,__interpose section.
 // As a result all the calls to foo() will be routed to bar() at runtime.
 #define INTERPOSER_2(func_name, wrapper_name) __attribute__((used)) \
-const interpose_substitution substitution_##func_name[] \
+const interpose_substitution substitution_##func_name[]             \
-    __attribute__((section("__DATA, __interpose"))) = { \
+    __attribute__((section("__DATA, __interpose"))) = {             \
-    { reinterpret_cast<const uptr>(wrapper_name), \
+    { reinterpret_cast<const uptr>(wrapper_name),                   \
-      reinterpret_cast<const uptr>(func_name) } \
+      reinterpret_cast<const uptr>(func_name) }                     \
 }
 # define WRAP(x) wrap_##x
-# define WRAPPER_NAME(x) "wrap_"#x
+# define TRAMPOLINE(x) WRAP(x)
 # define INTERCEPTOR_ATTRIBUTE
 # define DECLARE_WRAPPER(ret_type, func, ...)
 #elif SANITIZER_WINDOWS
 # define WRAP(x) __asan_wrap_##x
-# define WRAPPER_NAME(x) "__asan_wrap_"#x
+# define TRAMPOLINE(x) WRAP(x)
 # define INTERCEPTOR_ATTRIBUTE __declspec(dllexport)
-# define DECLARE_WRAPPER(ret_type, func, ...) \
+# define DECLARE_WRAPPER(ret_type, func, ...)         \
     extern "C" ret_type func(__VA_ARGS__);
-# define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \
+# define DECLARE_WRAPPER_WINAPI(ret_type, func, ...)  \
     extern "C" __declspec(dllimport) ret_type __stdcall func(__VA_ARGS__);
-#elif SANITIZER_FREEBSD || SANITIZER_NETBSD
+#elif !SANITIZER_FUCHSIA  // LINUX, FREEBSD, NETBSD, SOLARIS
-# define WRAP(x) __interceptor_ ## x
-# define WRAPPER_NAME(x) "__interceptor_" #x
 # define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
+# if ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT
+// Weak aliases of weak aliases do not work, therefore we need to set up a
+// trampoline function. The function "func" is a weak alias to the trampoline
+// (so that we may check if "func" was overridden), which calls the weak
+// function __interceptor_func, which in turn aliases the actual interceptor
+// implementation ___interceptor_func:
+//
+//    [wrapper "func": weak] --(alias)--> [TRAMPOLINE(func)]
+//                                                |
+//                     +--------(tail call)-------+
+//                     |
+//                     v
+//      [__interceptor_func: weak] --(alias)--> [WRAP(func)]
+//
+// We use inline assembly to define most of this, because not all compilers
+// support functions with the "naked" attribute with every architecture.
+#  define WRAP(x) ___interceptor_ ## x
+#  define TRAMPOLINE(x) __interceptor_trampoline_ ## x
+#  if SANITIZER_FREEBSD || SANITIZER_NETBSD
 // FreeBSD's dynamic linker (incompliantly) gives non-weak symbols higher
 // priority than weak ones so weak aliases won't work for indirect calls
 // in position-independent (-fPIC / -fPIE) mode.
-# define DECLARE_WRAPPER(ret_type, func, ...) \
+#   define __ASM_WEAK_WRAPPER(func) ".globl " #func "\n"
-     extern "C" ret_type func(__VA_ARGS__) \
+#  else
-     __attribute__((alias("__interceptor_" #func), visibility("default")));
+#   define __ASM_WEAK_WRAPPER(func) ".weak " #func "\n"
-#elif !SANITIZER_FUCHSIA
+#  endif  // SANITIZER_FREEBSD || SANITIZER_NETBSD
-# define WRAP(x) __interceptor_ ## x
+// Keep trampoline implementation in sync with sanitizer_common/sanitizer_asm.h
-# define WRAPPER_NAME(x) "__interceptor_" #x
+#  define DECLARE_WRAPPER(ret_type, func, ...)                                 \
-# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
+     extern "C" ret_type func(__VA_ARGS__);                                    \
-# define DECLARE_WRAPPER(ret_type, func, ...) \
+     extern "C" ret_type TRAMPOLINE(func)(__VA_ARGS__);                        \
-    extern "C" ret_type func(__VA_ARGS__) \
+     extern "C" ret_type __interceptor_##func(__VA_ARGS__)                     \
-    __attribute__((weak, alias("__interceptor_" #func), visibility("default")));
+       INTERCEPTOR_ATTRIBUTE __attribute__((weak)) ALIAS(WRAP(func));          \
+     asm(                                                                      \
+       ".text\n"                                                               \
+       __ASM_WEAK_WRAPPER(func)                                                \
+       ".set " #func ", " SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n"           \
+       ".globl " SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n"                    \
+       ".type  " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", %function\n"         \
+       SANITIZER_STRINGIFY(TRAMPOLINE(func)) ":\n"                             \
+       SANITIZER_STRINGIFY(CFI_STARTPROC) "\n"                                 \
+       SANITIZER_STRINGIFY(ASM_TAIL_CALL) " __interceptor_"                    \
+         SANITIZER_STRINGIFY(ASM_PREEMPTIBLE_SYM(func)) "\n"                   \
+       SANITIZER_STRINGIFY(CFI_ENDPROC) "\n"                                   \
+       ".size  " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", "                    \
+            ".-" SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n"                    \
+     );
+# else  // ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT
+// Some architectures cannot implement efficient interceptor trampolines with
+// just a plain jump due to complexities of resolving a preemptible symbol. In
+// those cases, revert to just this scheme:
+//
+//    [wrapper "func": weak] --(alias)--> [WRAP(func)]
+//
+#  define WRAP(x) __interceptor_ ## x
+#  define TRAMPOLINE(x) WRAP(x)
+#  if SANITIZER_FREEBSD || SANITIZER_NETBSD
+#   define __ATTRIBUTE_WEAK_WRAPPER
+#  else
+#   define __ATTRIBUTE_WEAK_WRAPPER __attribute__((weak))
+#  endif  // SANITIZER_FREEBSD || SANITIZER_NETBSD
+#  define DECLARE_WRAPPER(ret_type, func, ...)                                 \
+     extern "C" ret_type func(__VA_ARGS__)                                     \
+       INTERCEPTOR_ATTRIBUTE __ATTRIBUTE_WEAK_WRAPPER ALIAS(WRAP(func));
+# endif  // ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT
 #endif
 #if SANITIZER_FUCHSIA
 @@ -157,33 +233,35 @@ const interpose_substitution substitution_##func_name[] \
 # define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
 # define REAL(x) __unsanitized_##x
 # define DECLARE_REAL(ret_type, func, ...)
-#elif !SANITIZER_MAC
+#elif !SANITIZER_APPLE
 # define PTR_TO_REAL(x) real_##x
 # define REAL(x) __interception::PTR_TO_REAL(x)
 # define FUNC_TYPE(x) x##_type
-# define DECLARE_REAL(ret_type, func, ...) \
+# define DECLARE_REAL(ret_type, func, ...)            \
     typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \
-    namespace __interception { \
+    namespace __interception {                        \
-      extern FUNC_TYPE(func) PTR_TO_REAL(func); \
+    extern FUNC_TYPE(func) PTR_TO_REAL(func);         \
     }
 # define ASSIGN_REAL(dst, src) REAL(dst) = REAL(src)
-#else  // SANITIZER_MAC
+#else  // SANITIZER_APPLE
 # define REAL(x) x
 # define DECLARE_REAL(ret_type, func, ...) \
     extern "C" ret_type func(__VA_ARGS__);
 # define ASSIGN_REAL(x, y)
-#endif  // SANITIZER_MAC
+#endif  // SANITIZER_APPLE
 #if !SANITIZER_FUCHSIA
-#  define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \
+# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...)  \
     DECLARE_REAL(ret_type, func, __VA_ARGS__)               \
+    extern "C" ret_type TRAMPOLINE(func)(__VA_ARGS__);      \
     extern "C" ret_type WRAP(func)(__VA_ARGS__);
 // Declare an interceptor and its wrapper defined in a different translation
 // unit (ex. asm).
-# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...)    \
+# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...)  \
-  extern "C" ret_type WRAP(func)(__VA_ARGS__); \
+    extern "C" ret_type TRAMPOLINE(func)(__VA_ARGS__);                \
-  extern "C" ret_type func(__VA_ARGS__);
+    extern "C" ret_type WRAP(func)(__VA_ARGS__);                      \
+    extern "C" ret_type func(__VA_ARGS__);
 #else
 # define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...)
 # define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...)
 @@ -193,7 +271,7 @@ const interpose_substitution substitution_##func_name[] \
 // macros does its job. In exceptional cases you may need to call REAL(foo)
 // without defining INTERCEPTOR(..., foo, ...). For example, if you override
 // foo with an interceptor for other function.
-#if !SANITIZER_MAC && !SANITIZER_FUCHSIA
+#if !SANITIZER_APPLE && !SANITIZER_FUCHSIA
 #  define DEFINE_REAL(ret_type, func, ...)            \
     typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \
     namespace __interception {                        \
 @@ -213,25 +291,23 @@ const interpose_substitution substitution_##func_name[] \
       __interceptor_##func(__VA_ARGS__);                                \
   extern "C" INTERCEPTOR_ATTRIBUTE ret_type func(__VA_ARGS__)
-#elif !SANITIZER_MAC
+#elif !SANITIZER_APPLE
-#define INTERCEPTOR(ret_type, func, ...) \
+#define INTERCEPTOR(ret_type, func, ...)        \
-  DEFINE_REAL(ret_type, func, __VA_ARGS__) \
+  DEFINE_REAL(ret_type, func, __VA_ARGS__)      \
-  DECLARE_WRAPPER(ret_type, func, __VA_ARGS__) \
+  DECLARE_WRAPPER(ret_type, func, __VA_ARGS__)  \
-  extern "C" \
+  extern "C" INTERCEPTOR_ATTRIBUTE ret_type WRAP(func)(__VA_ARGS__)
-  INTERCEPTOR_ATTRIBUTE \
-  ret_type WRAP(func)(__VA_ARGS__)
 // We don't need INTERCEPTOR_WITH_SUFFIX on non-Darwin for now.
 #define INTERCEPTOR_WITH_SUFFIX(ret_type, func, ...) \
   INTERCEPTOR(ret_type, func, __VA_ARGS__)
-#else  // SANITIZER_MAC
+#else  // SANITIZER_APPLE
-#define INTERCEPTOR_ZZZ(suffix, ret_type, func, ...) \
+#define INTERCEPTOR_ZZZ(suffix, ret_type, func, ...)  \
-  extern "C" ret_type func(__VA_ARGS__) suffix; \
+  extern "C" ret_type func(__VA_ARGS__) suffix;       \
-  extern "C" ret_type WRAP(func)(__VA_ARGS__); \
+  extern "C" ret_type WRAP(func)(__VA_ARGS__);        \
-  INTERPOSER(func); \
+  INTERPOSER(func);                                   \
   extern "C" INTERCEPTOR_ATTRIBUTE ret_type WRAP(func)(__VA_ARGS__)
 #define INTERCEPTOR(ret_type, func, ...) \
 @@ -246,14 +322,12 @@ const interpose_substitution substitution_##func_name[] \
 #endif
 #if SANITIZER_WINDOWS
-# define INTERCEPTOR_WINAPI(ret_type, func, ...) \
+# define INTERCEPTOR_WINAPI(ret_type, func, ...)                \
     typedef ret_type (__stdcall *FUNC_TYPE(func))(__VA_ARGS__); \
-    namespace __interception { \
+    namespace __interception {                                  \
-      FUNC_TYPE(func) PTR_TO_REAL(func); \
+      FUNC_TYPE(func) PTR_TO_REAL(func);                        \
-    } \
+    }                                                           \
-    extern "C" \
+    extern "C" INTERCEPTOR_ATTRIBUTE ret_type __stdcall WRAP(func)(__VA_ARGS__)
-    INTERCEPTOR_ATTRIBUTE \
-    ret_type __stdcall WRAP(func)(__VA_ARGS__)
 #endif
 // ISO C++ forbids casting between pointer-to-function and pointer-to-object,
 @@ -278,7 +352,7 @@ typedef unsigned long uptr;
 # define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)
 # define INTERCEPT_FUNCTION_VER(func, symver) \
     INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver)
-#elif SANITIZER_MAC
+#elif SANITIZER_APPLE
 # include "interception_mac.h"
 # define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_MAC(func)
 # define INTERCEPT_FUNCTION_VER(func, symver) \

lib/tsan/interception/interception_linux.cpp+8-8

...	@@ -33,7 +33,7 @@ static int StrCmp(const char s1, const char s2) {	...	@@ -33,7 +33,7 @@ static int StrCmp(const char s1, const char s2) {
33	}	33	}
34	#endif	34	#endif
35		35
36	static void GetFuncAddr(const char name, uptr wrapper_addr) {	36	static void GetFuncAddr(const char name, uptr trampoline) {
37	#if SANITIZER_NETBSD	37	#if SANITIZER_NETBSD
38	// FIXME: Find a better way to handle renames	38	// FIXME: Find a better way to handle renames
39	if (StrCmp(name, "sigaction"))	39	if (StrCmp(name, "sigaction"))
...	@@ -50,17 +50,17 @@ static void GetFuncAddr(const char name, uptr wrapper_addr) {	...	@@ -50,17 +50,17 @@ static void GetFuncAddr(const char name, uptr wrapper_addr) {
50		50
51	// In case `name' is not loaded, dlsym ends up finding the actual wrapper.	51	// In case `name' is not loaded, dlsym ends up finding the actual wrapper.
52	// We don't want to intercept the wrapper and have it point to itself.	52	// We don't want to intercept the wrapper and have it point to itself.
53	if ((uptr)addr == wrapper_addr)	53	if ((uptr)addr == trampoline)
54	addr = nullptr;	54	addr = nullptr;
55	}	55	}
56	return addr;	56	return addr;
57	}	57	}
58		58
59	bool InterceptFunction(const char name, uptr ptr_to_real, uptr func,	59	bool InterceptFunction(const char name, uptr ptr_to_real, uptr func,
60	uptr wrapper) {	60	uptr trampoline) {
61	void *addr = GetFuncAddr(name, wrapper);	61	void *addr = GetFuncAddr(name, trampoline);
62	*ptr_to_real = (uptr)addr;	62	*ptr_to_real = (uptr)addr;
63	return addr && (func == wrapper);	63	return addr && (func == trampoline);
64	}	64	}
65		65
66	// dlvsym is a GNU extension supported by some other platforms.	66	// dlvsym is a GNU extension supported by some other platforms.
...	@@ -70,12 +70,12 @@ static void GetFuncAddr(const char name, const char *ver) {	...	@@ -70,12 +70,12 @@ static void GetFuncAddr(const char name, const char *ver) {
70	}	70	}
71		71
72	bool InterceptFunction(const char name, const char ver, uptr *ptr_to_real,	72	bool InterceptFunction(const char name, const char ver, uptr *ptr_to_real,
73	uptr func, uptr wrapper) {	73	uptr func, uptr trampoline) {
74	void *addr = GetFuncAddr(name, ver);	74	void *addr = GetFuncAddr(name, ver);
75	*ptr_to_real = (uptr)addr;	75	*ptr_to_real = (uptr)addr;
76	return addr && (func == wrapper);	76	return addr && (func == trampoline);
77	}	77	}
78	#endif // SANITIZER_GLIBC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD	78	# endif // SANITIZER_GLIBC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
79		79
80	} // namespace __interception	80	} // namespace __interception
81		81

lib/tsan/interception/interception_linux.h+9-9

...	@@ -15,7 +15,7 @@	...	@@ -15,7 +15,7 @@
15	SANITIZER_SOLARIS	15	SANITIZER_SOLARIS
16		16
17	#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)	17	#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)
18	# error "interception_linux.h should be included from interception library only"	18	# error interception_linux.h should be included from interception library only
19	#endif	19	#endif
20		20
21	#ifndef INTERCEPTION_LINUX_H	21	#ifndef INTERCEPTION_LINUX_H
...	@@ -23,26 +23,26 @@	...	@@ -23,26 +23,26 @@
23		23
24	namespace __interception {	24	namespace __interception {
25	bool InterceptFunction(const char name, uptr ptr_to_real, uptr func,	25	bool InterceptFunction(const char name, uptr ptr_to_real, uptr func,
26	uptr wrapper);	26	uptr trampoline);
27	bool InterceptFunction(const char name, const char ver, uptr *ptr_to_real,	27	bool InterceptFunction(const char name, const char ver, uptr *ptr_to_real,
28	uptr func, uptr wrapper);	28	uptr func, uptr trampoline);
29	} // namespace __interception	29	} // namespace __interception
30		30
31	#define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \	31	#define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \
32	::__interception::InterceptFunction( \	32	::__interception::InterceptFunction( \
33	#func, \	33	#func, \
34	(::__interception::uptr *) & REAL(func), \	34	(::__interception::uptr *)&REAL(func), \
35	(::__interception::uptr) & (func), \	35	(::__interception::uptr)&(func), \
36	(::__interception::uptr) & WRAP(func))	36	(::__interception::uptr)&TRAMPOLINE(func))
37		37
38	// dlvsym is a GNU extension supported by some other platforms.	38	// dlvsym is a GNU extension supported by some other platforms.
39	#if SANITIZER_GLIBC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD	39	#if SANITIZER_GLIBC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
40	#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \	40	#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \
41	::__interception::InterceptFunction( \	41	::__interception::InterceptFunction( \
42	#func, symver, \	42	#func, symver, \
43	(::__interception::uptr *) & REAL(func), \	43	(::__interception::uptr *)&REAL(func), \
44	(::__interception::uptr) & (func), \	44	(::__interception::uptr)&(func), \
45	(::__interception::uptr) & WRAP(func))	45	(::__interception::uptr)&TRAMPOLINE(func))
46	#else	46	#else
47	#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \	47	#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \
48	INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)	48	INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)

lib/tsan/interception/interception_mac.cpp+2-2

...	@@ -13,6 +13,6 @@	...	@@ -13,6 +13,6 @@
13		13
14	#include "interception.h"	14	#include "interception.h"
15		15
16	#if SANITIZER_MAC	16	#if SANITIZER_APPLE
17		17
18	#endif // SANITIZER_MAC	18	#endif // SANITIZER_APPLE

lib/tsan/interception/interception_mac.h+2-2

...	@@ -11,7 +11,7 @@	...	@@ -11,7 +11,7 @@
11	// Mac-specific interception methods.	11	// Mac-specific interception methods.
12	//===----------------------------------------------------------------------===//	12	//===----------------------------------------------------------------------===//
13		13
14	#if SANITIZER_MAC	14	#if SANITIZER_APPLE
15		15
16	#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)	16	#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)
17	# error "interception_mac.h should be included from interception.h only"	17	# error "interception_mac.h should be included from interception.h only"
...	@@ -24,4 +24,4 @@	...	@@ -24,4 +24,4 @@
24	#define INTERCEPT_FUNCTION_VER_MAC(func, symver)	24	#define INTERCEPT_FUNCTION_VER_MAC(func, symver)
25		25
26	#endif // INTERCEPTION_MAC_H	26	#endif // INTERCEPTION_MAC_H
27	#endif // SANITIZER_MAC	27	#endif // SANITIZER_APPLE

lib/tsan/interception/interception_type_test.cpp+3-3

...	@@ -13,7 +13,7 @@	...	@@ -13,7 +13,7 @@
13		13
14	#include "interception.h"	14	#include "interception.h"
15		15
16	#if SANITIZER_LINUX \|\| SANITIZER_MAC	16	#if SANITIZER_LINUX \|\| SANITIZER_APPLE
17		17
18	#include <sys/types.h>	18	#include <sys/types.h>
19	#include <stddef.h>	19	#include <stddef.h>
...	@@ -24,9 +24,9 @@ COMPILER_CHECK(sizeof(::SSIZE_T) == sizeof(ssize_t));	...	@@ -24,9 +24,9 @@ COMPILER_CHECK(sizeof(::SSIZE_T) == sizeof(ssize_t));
24	COMPILER_CHECK(sizeof(::PTRDIFF_T) == sizeof(ptrdiff_t));	24	COMPILER_CHECK(sizeof(::PTRDIFF_T) == sizeof(ptrdiff_t));
25	COMPILER_CHECK(sizeof(::INTMAX_T) == sizeof(intmax_t));	25	COMPILER_CHECK(sizeof(::INTMAX_T) == sizeof(intmax_t));
26		26
27	#if !SANITIZER_MAC	27	# if SANITIZER_GLIBC \|\| SANITIZER_ANDROID
28	COMPILER_CHECK(sizeof(::OFF64_T) == sizeof(off64_t));	28	COMPILER_CHECK(sizeof(::OFF64_T) == sizeof(off64_t));
29	#endif	29	# endif
30		30
31	// The following are the cases when pread (and friends) is used instead of	31	// The following are the cases when pread (and friends) is used instead of
32	// pread64. In those cases we need OFF_T to match off_t. We don't care about the	32	// pread64. In those cases we need OFF_T to match off_t. We don't care about the

lib/tsan/interception/interception_win.cpp+107-11

 @@ -56,7 +56,7 @@
 //                                      tramp:  jmp QWORD [addr]
 //                                       addr:  .bytes <hook>
 //
-//    Note: <real> is equilavent to <label>.
+//    Note: <real> is equivalent to <label>.
 //
 // 3) HotPatch
 //
 @@ -141,8 +141,29 @@ static const int kBranchLength =
     FIRST_32_SECOND_64(kJumpInstructionLength, kIndirectJumpInstructionLength);
 static const int kDirectBranchLength = kBranchLength + kAddressLength;
+#  if defined(_MSC_VER)
+#    define INTERCEPTION_FORMAT(f, a)
+#  else
+#    define INTERCEPTION_FORMAT(f, a) __attribute__((format(printf, f, a)))
+#  endif
+static void (*ErrorReportCallback)(const char *format, ...)
+    INTERCEPTION_FORMAT(1, 2);
+void SetErrorReportCallback(void (*callback)(const char *format, ...)) {
+  ErrorReportCallback = callback;
+}
+#  define ReportError(...)                \
+    do {                                  \
+      if (ErrorReportCallback)            \
+        ErrorReportCallback(__VA_ARGS__); \
+    } while (0)
 static void InterceptionFailed() {
-  // Do we have a good way to abort with an error message here?
+  ReportError("interception_win: failed due to an unrecoverable error.\n");
+  // This acts like an abort when no debugger is attached. According to an old
+  // comment, calling abort() leads to an infinite recursion in CheckFailed.
   __debugbreak();
 }
 @@ -249,8 +270,13 @@ static void WritePadding(uptr from, uptr size) {
 }
 static void WriteJumpInstruction(uptr from, uptr target) {
-  if (!DistanceIsWithin2Gig(from + kJumpInstructionLength, target))
+  if (!DistanceIsWithin2Gig(from + kJumpInstructionLength, target)) {
+    ReportError(
+        "interception_win: cannot write jmp further than 2GB away, from %p to "
+        "%p.\n",
+        (void *)from, (void *)target);
     InterceptionFailed();
+  }
   ptrdiff_t offset = target - from - kJumpInstructionLength;
   *(u8*)from = 0xE9;
   *(u32*)(from + 1) = offset;
 @@ -274,6 +300,10 @@ static void WriteIndirectJumpInstruction(uptr from, uptr indirect_target) {
   int offset = indirect_target - from - kIndirectJumpInstructionLength;
   if (!DistanceIsWithin2Gig(from + kIndirectJumpInstructionLength,
                             indirect_target)) {
+    ReportError(
+        "interception_win: cannot write indirect jmp with target further than "
+        "2GB away, from %p to %p.\n",
+        (void *)from, (void *)indirect_target);
     InterceptionFailed();
   }
   *(u16*)from = 0x25FF;
 @@ -398,8 +428,44 @@ static uptr AllocateMemoryForTrampoline(uptr image_address, size_t size) {
   return allocated_space;
 }
+// The following prologues cannot be patched because of the short jump
+// jumping to the patching region.
+#if SANITIZER_WINDOWS64
+// ntdll!wcslen in Win11
+//   488bc1          mov     rax,rcx
+//   0fb710          movzx   edx,word ptr [rax]
+//   4883c002        add     rax,2
+//   6685d2          test    dx,dx
+//   75f4            jne     -12
+static const u8 kPrologueWithShortJump1[] = {
+    0x48, 0x8b, 0xc1, 0x0f, 0xb7, 0x10, 0x48, 0x83,
+    0xc0, 0x02, 0x66, 0x85, 0xd2, 0x75, 0xf4,
+};
+// ntdll!strrchr in Win11
+//   4c8bc1          mov     r8,rcx
+//   8a01            mov     al,byte ptr [rcx]
+//   48ffc1          inc     rcx
+//   84c0            test    al,al
+//   75f7            jne     -9
+static const u8 kPrologueWithShortJump2[] = {
+    0x4c, 0x8b, 0xc1, 0x8a, 0x01, 0x48, 0xff, 0xc1,
+    0x84, 0xc0, 0x75, 0xf7,
+};
+#endif
 // Returns 0 on error.
 static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
+#if SANITIZER_WINDOWS64
+  if (memcmp((u8*)address, kPrologueWithShortJump1,
+             sizeof(kPrologueWithShortJump1)) == 0 ||
+      memcmp((u8*)address, kPrologueWithShortJump2,
+             sizeof(kPrologueWithShortJump2)) == 0) {
+    return 0;
+  }
+#endif
   switch (*(u64*)address) {
     case 0x90909090909006EB:  // stub: jmp over 6 x nop.
       return 8;
 @@ -456,6 +522,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0xFF8B:  // 8B FF : mov edi, edi
     case 0xEC8B:  // 8B EC : mov ebp, esp
     case 0xc889:  // 89 C8 : mov eax, ecx
+    case 0xE589:  // 89 E5 : mov ebp, esp
     case 0xC18B:  // 8B C1 : mov eax, ecx
     case 0xC033:  // 33 C0 : xor eax, eax
     case 0xC933:  // 33 C9 : xor ecx, ecx
 @@ -477,6 +544,14 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0xA1:  // A1 XX XX XX XX XX XX XX XX :
                 //   movabs eax, dword ptr ds:[XXXXXXXX]
       return 9;
+    case 0x83:
+      const u8 next_byte = *(u8*)(address + 1);
+      const u8 mod = next_byte >> 6;
+      const u8 rm = next_byte & 7;
+      if (mod == 1 && rm == 4)
+        return 5;  // 83 ModR/M SIB Disp8 Imm8
+                   //   add|or|adc|sbb|and|sub|xor|cmp [r+disp8], imm8
   }
   switch (*(u16*)address) {
 @@ -493,6 +568,8 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0x5641:  // push r14
     case 0x5741:  // push r15
     case 0x9066:  // Two-byte NOP
+    case 0xc084:  // test al, al
+    case 0x018a:  // mov al, byte ptr [rcx]
       return 2;
     case 0x058B:  // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX]
 @@ -509,6 +586,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0xd12b48:    // 48 2b d1 : sub rdx, rcx
     case 0x07c1f6:    // f6 c1 07 : test cl, 0x7
     case 0xc98548:    // 48 85 C9 : test rcx, rcx
+    case 0xd28548:    // 48 85 d2 : test rdx, rdx
     case 0xc0854d:    // 4d 85 c0 : test r8, r8
     case 0xc2b60f:    // 0f b6 c2 : movzx eax, dl
     case 0xc03345:    // 45 33 c0 : xor r8d, r8d
 @@ -522,6 +600,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0xca2b48:    // 48 2b ca : sub rcx, rdx
     case 0x10b70f:    // 0f b7 10 : movzx edx, WORD PTR [rax]
     case 0xc00b4d:    // 3d 0b c0 : or r8, r8
+    case 0xc08b41:    // 41 8b c0 : mov eax, r8d
     case 0xd18b48:    // 48 8b d1 : mov rdx, rcx
     case 0xdc8b4c:    // 4c 8b dc : mov r11, rsp
     case 0xd18b4c:    // 4c 8b d1 : mov r10, rcx
 @@ -556,6 +635,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0x246c8948:  // 48 89 6C 24 XX : mov QWORD ptr [rsp + XX], rbp
     case 0x245c8948:  // 48 89 5c 24 XX : mov QWORD PTR [rsp + XX], rbx
     case 0x24748948:  // 48 89 74 24 XX : mov QWORD PTR [rsp + XX], rsi
+    case 0x247c8948:  // 48 89 7c 24 XX : mov QWORD PTR [rsp + XX], rdi
     case 0x244C8948:  // 48 89 4C 24 XX : mov QWORD PTR [rsp + XX], rcx
     case 0x24548948:  // 48 89 54 24 XX : mov QWORD PTR [rsp + XX], rdx
     case 0x244c894c:  // 4c 89 4c 24 XX : mov QWORD PTR [rsp + XX], r9
 @@ -592,6 +672,8 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
     case 0x24448B:  // 8B 44 24 XX : mov eax, dword ptr [esp + XX]
     case 0x244C8B:  // 8B 4C 24 XX : mov ecx, dword ptr [esp + XX]
     case 0x24548B:  // 8B 54 24 XX : mov edx, dword ptr [esp + XX]
+    case 0x245C8B:  // 8B 5C 24 XX : mov ebx, dword ptr [esp + XX]
+    case 0x246C8B:  // 8B 6C 24 XX : mov ebp, dword ptr [esp + XX]
     case 0x24748B:  // 8B 74 24 XX : mov esi, dword ptr [esp + XX]
     case 0x247C8B:  // 8B 7C 24 XX : mov edi, dword ptr [esp + XX]
       return 4;
 @@ -603,12 +685,20 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
   }
 #endif
-  // Unknown instruction!
+  // Unknown instruction! This might happen when we add a new interceptor, use
-  // FIXME: Unknown instruction failures might happen when we add a new
+  // a new compiler version, or if Windows changed how some functions are
-  // interceptor or a new compiler version. In either case, they should result
+  // compiled. In either case, we print the address and 8 bytes of instructions
-  // in visible and readable error messages. However, merely calling abort()
+  // to notify the user about the error and to help identify the unknown
-  // leads to an infinite recursion in CheckFailed.
+  // instruction. Don't treat this as a fatal error, though we can break the
-  InterceptionFailed();
+  // debugger if one has been attached.
+  u8 *bytes = (u8 *)address;
+  ReportError(
+      "interception_win: unhandled instruction at %p: %02x %02x %02x %02x %02x "
+      "%02x %02x %02x\n",
+      (void *)address, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4],
+      bytes[5], bytes[6], bytes[7]);
+  if (::IsDebuggerPresent())
+    __debugbreak();
   return 0;
 }
 @@ -629,6 +719,8 @@ static bool CopyInstructions(uptr to, uptr from, size_t size) {
   while (cursor != size) {
     size_t rel_offset = 0;
     size_t instruction_size = GetInstructionSize(from + cursor, &rel_offset);
+    if (!instruction_size)
+      return false;
     _memcpy((void*)(to + cursor), (void*)(from + cursor),
             (size_t)instruction_size);
     if (rel_offset) {
 @@ -689,7 +781,7 @@ bool OverrideFunctionWithRedirectJump(
     return false;
   if (orig_old_func) {
-    uptr relative_offset = *(u32*)(old_func + 1);
+    sptr relative_offset = *(s32 *)(old_func + 1);
     uptr absolute_target = old_func + relative_offset + kJumpInstructionLength;
     *orig_old_func = absolute_target;
   }
 @@ -846,6 +938,10 @@ static void **InterestingDLLsAvailable() {
       "msvcr120.dll",      // VS2013
       "vcruntime140.dll",  // VS2015
       "ucrtbase.dll",      // Universal CRT
+#if (defined(__MINGW32__) && defined(__i386__))
+      "libc++.dll",        // libc++
+      "libunwind.dll",     // libunwind
+#endif
       // NTDLL should go last as it exports some functions that we should
       // override in the CRT [presumably only used internally].
       "ntdll.dll", NULL};
 @@ -1019,4 +1115,4 @@ bool OverrideImportedFunction(const char *module_to_patch,
 }  // namespace __interception
-#endif  // SANITIZER_MAC
+#endif  // SANITIZER_APPLE

lib/tsan/interception/interception_win.h+5

 @@ -41,6 +41,11 @@ bool OverrideImportedFunction(const char *module_to_patch,
                               const char *function_name, uptr new_function,
                               uptr *orig_old_func);
+// Sets a callback to be used for reporting errors by interception_win. The
+// callback will be called with printf-like arguments. Intended to be used with
+// __sanitizer::Report. Pass nullptr to disable error reporting (default).
+void SetErrorReportCallback(void (*callback)(const char *format, ...));
 #if !SANITIZER_WINDOWS64
 // Exposed for unittests
 bool OverrideFunctionWithDetour(

lib/tsan/sanitizer_common/sancov_flags.inc+1-1

...	@@ -14,7 +14,7 @@	...	@@ -14,7 +14,7 @@
14	#endif	14	#endif
15		15
16	SANCOV_FLAG(bool, symbolize, true,	16	SANCOV_FLAG(bool, symbolize, true,
17	"If set, converage information will be symbolized by sancov tool "	17	"If set, coverage information will be symbolized by sancov tool "
18	"after dumping.")	18	"after dumping.")
19		19
20	SANCOV_FLAG(bool, help, false, "Print flags help.")	20	SANCOV_FLAG(bool, help, false, "Print flags help.")

lib/tsan/sanitizer_common/sanitizer_addrhashmap.h+43-3

...	@@ -39,6 +39,11 @@ namespace __sanitizer {	...	@@ -39,6 +39,11 @@ namespace __sanitizer {
39	// the current thread has exclusive access to the data	39	// the current thread has exclusive access to the data
40	// if !h.exists() then the element never existed	40	// if !h.exists() then the element never existed
41	// }	41	// }
		42	// {
		43	// Map::Handle h(&m, addr, false, true);
		44	// this will create a new element or return a handle to an existing element
		45	// if !h.created() this thread does not have exclusive access to the data
		46	// }
42	template<typename T, uptr kSize>	47	template<typename T, uptr kSize>
43	class AddrHashMap {	48	class AddrHashMap {
44	private:	49	private:
...	@@ -56,7 +61,7 @@ class AddrHashMap {	...	@@ -56,7 +61,7 @@ class AddrHashMap {
56	static const uptr kBucketSize = 3;	61	static const uptr kBucketSize = 3;
57		62
58	struct Bucket {	63	struct Bucket {
59	RWMutex mtx;	64	Mutex mtx;
60	atomic_uintptr_t add;	65	atomic_uintptr_t add;
61	Cell cells[kBucketSize];	66	Cell cells[kBucketSize];
62	};	67	};
...	@@ -89,6 +94,12 @@ class AddrHashMap {	...	@@ -89,6 +94,12 @@ class AddrHashMap {
89	bool create_;	94	bool create_;
90	};	95	};
91		96
		97	typedef void (ForEachCallback)(const uptr key, const T &val, void arg);
		98	// ForEach acquires a lock on each bucket while iterating over
		99	// elements. Note that this only ensures that the structure of the hashmap is
		100	// unchanged, there may be a data race to the element itself.
		101	void ForEach(ForEachCallback cb, void *arg);
		102
92	private:	103	private:
93	friend class Handle;	104	friend class Handle;
94	Bucket *table_;	105	Bucket *table_;
...	@@ -98,6 +109,33 @@ class AddrHashMap {	...	@@ -98,6 +109,33 @@ class AddrHashMap {
98	uptr calcHash(uptr addr);	109	uptr calcHash(uptr addr);
99	};	110	};
100		111
		112	template <typename T, uptr kSize>
		113	void AddrHashMap<T, kSize>::ForEach(ForEachCallback cb, void *arg) {
		114	for (uptr n = 0; n < kSize; n++) {
		115	Bucket *bucket = &table_[n];
		116
		117	ReadLock lock(&bucket->mtx);
		118
		119	for (uptr i = 0; i < kBucketSize; i++) {
		120	Cell *c = &bucket->cells[i];
		121	uptr addr1 = atomic_load(&c->addr, memory_order_acquire);
		122	if (addr1 != 0)
		123	cb(addr1, c->val, arg);
		124	}
		125
		126	// Iterate over any additional cells.
		127	if (AddBucket *add =
		128	(AddBucket *)atomic_load(&bucket->add, memory_order_acquire)) {
		129	for (uptr i = 0; i < add->size; i++) {
		130	Cell *c = &add->cells[i];
		131	uptr addr1 = atomic_load(&c->addr, memory_order_acquire);
		132	if (addr1 != 0)
		133	cb(addr1, c->val, arg);
		134	}
		135	}
		136	}
		137	}
		138
101	template<typename T, uptr kSize>	139	template<typename T, uptr kSize>
102	AddrHashMap<T, kSize>::Handle::Handle(AddrHashMap<T, kSize> *map, uptr addr) {	140	AddrHashMap<T, kSize>::Handle::Handle(AddrHashMap<T, kSize> *map, uptr addr) {
103	map_ = map;	141	map_ = map;
...	@@ -163,7 +201,8 @@ AddrHashMap<T, kSize>::AddrHashMap() {	...	@@ -163,7 +201,8 @@ AddrHashMap<T, kSize>::AddrHashMap() {
163	}	201	}
164		202
165	template <typename T, uptr kSize>	203	template <typename T, uptr kSize>
166	void AddrHashMap<T, kSize>::acquire(Handle *h) NO_THREAD_SAFETY_ANALYSIS {	204	void AddrHashMap<T, kSize>::acquire(Handle *h)
		205	SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
167	uptr addr = h->addr_;	206	uptr addr = h->addr_;
168	uptr hash = calcHash(addr);	207	uptr hash = calcHash(addr);
169	Bucket *b = &table_[hash];	208	Bucket *b = &table_[hash];
...	@@ -292,7 +331,8 @@ void AddrHashMap<T, kSize>::acquire(Handle *h) NO_THREAD_SAFETY_ANALYSIS {	...	@@ -292,7 +331,8 @@ void AddrHashMap<T, kSize>::acquire(Handle *h) NO_THREAD_SAFETY_ANALYSIS {
292	}	331	}
293		332
294	template <typename T, uptr kSize>	333	template <typename T, uptr kSize>
295	void AddrHashMap<T, kSize>::release(Handle *h) NO_THREAD_SAFETY_ANALYSIS {	334	void AddrHashMap<T, kSize>::release(Handle *h)
		335	SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
296	if (!h->cell_)	336	if (!h->cell_)
297	return;	337	return;
298	Bucket *b = h->bucket_;	338	Bucket *b = h->bucket_;

lib/tsan/sanitizer_common/sanitizer_allocator.cpp+23-66

...	@@ -17,6 +17,7 @@	...	@@ -17,6 +17,7 @@
17	#include "sanitizer_allocator_internal.h"	17	#include "sanitizer_allocator_internal.h"
18	#include "sanitizer_atomic.h"	18	#include "sanitizer_atomic.h"
19	#include "sanitizer_common.h"	19	#include "sanitizer_common.h"
		20	#include "sanitizer_platform.h"
20		21
21	namespace __sanitizer {	22	namespace __sanitizer {
22		23
...	@@ -24,66 +25,6 @@ namespace __sanitizer {	...	@@ -24,66 +25,6 @@ namespace __sanitizer {
24	const char *PrimaryAllocatorName = "SizeClassAllocator";	25	const char *PrimaryAllocatorName = "SizeClassAllocator";
25	const char *SecondaryAllocatorName = "LargeMmapAllocator";	26	const char *SecondaryAllocatorName = "LargeMmapAllocator";
26		27
27	// ThreadSanitizer for Go uses libc malloc/free.
28	#if defined(SANITIZER_USE_MALLOC)
29	# if SANITIZER_LINUX && !SANITIZER_ANDROID
30	extern "C" void *__libc_malloc(uptr size);
31	# if !SANITIZER_GO
32	extern "C" void *__libc_memalign(uptr alignment, uptr size);
33	# endif
34	extern "C" void __libc_realloc(void ptr, uptr size);
35	extern "C" void __libc_free(void *ptr);
36	# else
37	# include <stdlib.h>
38	# define __libc_malloc malloc
39	# if !SANITIZER_GO
40	static void *__libc_memalign(uptr alignment, uptr size) {
41	void *p;
42	uptr error = posix_memalign(&p, alignment, size);
43	if (error) return nullptr;
44	return p;
45	}
46	# endif
47	# define __libc_realloc realloc
48	# define __libc_free free
49	# endif
50
51	static void RawInternalAlloc(uptr size, InternalAllocatorCache cache,
52	uptr alignment) {
53	(void)cache;
54	#if !SANITIZER_GO
55	if (alignment == 0)
56	return __libc_malloc(size);
57	else
58	return __libc_memalign(alignment, size);
59	#else
60	// Windows does not provide __libc_memalign/posix_memalign. It provides
61	// __aligned_malloc, but the allocated blocks can't be passed to free,
62	// they need to be passed to __aligned_free. InternalAlloc interface does
63	// not account for such requirement. Alignemnt does not seem to be used
64	// anywhere in runtime, so just call __libc_malloc for now.
65	DCHECK_EQ(alignment, 0);
66	return __libc_malloc(size);
67	#endif
68	}
69
70	static void RawInternalRealloc(void ptr, uptr size,
71	InternalAllocatorCache *cache) {
72	(void)cache;
73	return __libc_realloc(ptr, size);
74	}
75
76	static void RawInternalFree(void ptr, InternalAllocatorCache cache) {
77	(void)cache;
78	__libc_free(ptr);
79	}
80
81	InternalAllocator *internal_allocator() {
82	return 0;
83	}
84
85	#else // SANITIZER_GO \|\| defined(SANITIZER_USE_MALLOC)
86
87	static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)];	28	static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)];
88	static atomic_uint8_t internal_allocator_initialized;	29	static atomic_uint8_t internal_allocator_initialized;
89	static StaticSpinMutex internal_alloc_init_mu;	30	static StaticSpinMutex internal_alloc_init_mu;
...	@@ -135,8 +76,6 @@ static void RawInternalFree(void ptr, InternalAllocatorCache cache) {	...	@@ -135,8 +76,6 @@ static void RawInternalFree(void ptr, InternalAllocatorCache cache) {
135	internal_allocator()->Deallocate(cache, ptr);	76	internal_allocator()->Deallocate(cache, ptr);
136	}	77	}
137		78
138	#endif // SANITIZER_GO \|\| defined(SANITIZER_USE_MALLOC)
139
140	static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) {	79	static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) {
141	SetAllocatorOutOfMemory();	80	SetAllocatorOutOfMemory();
142	Report("FATAL: %s: internal allocator is out of memory trying to allocate "	81	Report("FATAL: %s: internal allocator is out of memory trying to allocate "
...	@@ -187,6 +126,16 @@ void InternalFree(void addr, InternalAllocatorCache cache) {	...	@@ -187,6 +126,16 @@ void InternalFree(void addr, InternalAllocatorCache cache) {
187	RawInternalFree(addr, cache);	126	RawInternalFree(addr, cache);
188	}	127	}
189		128
		129	void InternalAllocatorLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
		130	internal_allocator_cache_mu.Lock();
		131	internal_allocator()->ForceLock();
		132	}
		133
		134	void InternalAllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
		135	internal_allocator()->ForceUnlock();
		136	internal_allocator_cache_mu.Unlock();
		137	}
		138
190	// LowLevelAllocator	139	// LowLevelAllocator
191	constexpr uptr kLowLevelAllocatorDefaultAlignment = 8;	140	constexpr uptr kLowLevelAllocatorDefaultAlignment = 8;
192	static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment;	141	static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment;
...	@@ -197,12 +146,10 @@ void *LowLevelAllocator::Allocate(uptr size) {	...	@@ -197,12 +146,10 @@ void *LowLevelAllocator::Allocate(uptr size) {
197	size = RoundUpTo(size, low_level_alloc_min_alignment);	146	size = RoundUpTo(size, low_level_alloc_min_alignment);
198	if (allocated_end_ - allocated_current_ < (sptr)size) {	147	if (allocated_end_ - allocated_current_ < (sptr)size) {
199	uptr size_to_allocate = RoundUpTo(size, GetPageSizeCached());	148	uptr size_to_allocate = RoundUpTo(size, GetPageSizeCached());
200	allocated_current_ =	149	allocated_current_ = (char *)MmapOrDie(size_to_allocate, __func__);
201	(char*)MmapOrDie(size_to_allocate, __func__);
202	allocated_end_ = allocated_current_ + size_to_allocate;	150	allocated_end_ = allocated_current_ + size_to_allocate;
203	if (low_level_alloc_callback) {	151	if (low_level_alloc_callback) {
204	low_level_alloc_callback((uptr)allocated_current_,	152	low_level_alloc_callback((uptr)allocated_current_, size_to_allocate);
205	size_to_allocate);
206	}	153	}
207	}	154	}
208	CHECK(allocated_end_ - allocated_current_ >= (sptr)size);	155	CHECK(allocated_end_ - allocated_current_ >= (sptr)size);
...	@@ -247,4 +194,14 @@ void PrintHintAllocatorCannotReturnNull() {	...	@@ -247,4 +194,14 @@ void PrintHintAllocatorCannotReturnNull() {
247	"allocator_may_return_null=1\n");	194	"allocator_may_return_null=1\n");
248	}	195	}
249		196
		197	static atomic_uint8_t rss_limit_exceeded;
		198
		199	bool IsRssLimitExceeded() {
		200	return atomic_load(&rss_limit_exceeded, memory_order_relaxed);
		201	}
		202
		203	void SetRssLimitExceeded(bool limit_exceeded) {
		204	atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed);
		205	}
		206
250	} // namespace __sanitizer	207	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_allocator.h+11-7

 @@ -14,6 +14,7 @@
 #define SANITIZER_ALLOCATOR_H
 #include "sanitizer_common.h"
+#include "sanitizer_flat_map.h"
 #include "sanitizer_internal_defs.h"
 #include "sanitizer_lfstack.h"
 #include "sanitizer_libc.h"
 @@ -43,12 +44,6 @@ void SetAllocatorOutOfMemory();
 void PrintHintAllocatorCannotReturnNull();
-// Allocators call these callbacks on mmap/munmap.
-struct NoOpMapUnmapCallback {
-  void OnMap(uptr p, uptr size) const { }
-  void OnUnmap(uptr p, uptr size) const { }
-};
 // Callback type for iterating over chunks.
 typedef void (*ForEachChunkCallback)(uptr chunk, void *arg);
 @@ -67,15 +62,24 @@ inline void RandomShuffle(T *a, u32 n, u32 *rand_state) {
   *rand_state = state;
 }
+struct NoOpMapUnmapCallback {
+  void OnMap(uptr p, uptr size) const {}
+  void OnMapSecondary(uptr p, uptr size, uptr user_begin,
+                      uptr user_size) const {}
+  void OnUnmap(uptr p, uptr size) const {}
+};
 #include "sanitizer_allocator_size_class_map.h"
 #include "sanitizer_allocator_stats.h"
 #include "sanitizer_allocator_primary64.h"
-#include "sanitizer_allocator_bytemap.h"
 #include "sanitizer_allocator_primary32.h"
 #include "sanitizer_allocator_local_cache.h"
 #include "sanitizer_allocator_secondary.h"
 #include "sanitizer_allocator_combined.h"
+bool IsRssLimitExceeded();
+void SetRssLimitExceeded(bool limit_exceeded);
 } // namespace __sanitizer
 #endif // SANITIZER_ALLOCATOR_H

lib/tsan/sanitizer_common/sanitizer_allocator_bytemap.h deleted-107

...	@@ -1,107 +0,0 @@
1	//===-- sanitizer_allocator_bytemap.h ---------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Part of the Sanitizer Allocator.
10	//
11	//===----------------------------------------------------------------------===//
12	#ifndef SANITIZER_ALLOCATOR_H
13	#error This file must be included inside sanitizer_allocator.h
14	#endif
15
16	// Maps integers in rage [0, kSize) to u8 values.
17	template <u64 kSize, typename AddressSpaceViewTy = LocalAddressSpaceView>
18	class FlatByteMap {
19	public:
20	using AddressSpaceView = AddressSpaceViewTy;
21	void Init() {
22	internal_memset(map_, 0, sizeof(map_));
23	}
24
25	void set(uptr idx, u8 val) {
26	CHECK_LT(idx, kSize);
27	CHECK_EQ(0U, map_[idx]);
28	map_[idx] = val;
29	}
30	u8 operator[] (uptr idx) {
31	CHECK_LT(idx, kSize);
32	// FIXME: CHECK may be too expensive here.
33	return map_[idx];
34	}
35	private:
36	u8 map_[kSize];
37	};
38
39	// TwoLevelByteMap maps integers in range [0, kSize1*kSize2) to u8 values.
40	// It is implemented as a two-dimensional array: array of kSize1 pointers
41	// to kSize2-byte arrays. The secondary arrays are mmaped on demand.
42	// Each value is initially zero and can be set to something else only once.
43	// Setting and getting values from multiple threads is safe w/o extra locking.
44	template <u64 kSize1, u64 kSize2,
45	typename AddressSpaceViewTy = LocalAddressSpaceView,
46	class MapUnmapCallback = NoOpMapUnmapCallback>
47	class TwoLevelByteMap {
48	public:
49	using AddressSpaceView = AddressSpaceViewTy;
50	void Init() {
51	internal_memset(map1_, 0, sizeof(map1_));
52	mu_.Init();
53	}
54
55	void TestOnlyUnmap() {
56	for (uptr i = 0; i < kSize1; i++) {
57	u8 *p = Get(i);
58	if (!p) continue;
59	MapUnmapCallback().OnUnmap(reinterpret_cast<uptr>(p), kSize2);
60	UnmapOrDie(p, kSize2);
61	}
62	}
63
64	uptr size() const { return kSize1 * kSize2; }
65	uptr size1() const { return kSize1; }
66	uptr size2() const { return kSize2; }
67
68	void set(uptr idx, u8 val) {
69	CHECK_LT(idx, kSize1 * kSize2);
70	u8 *map2 = GetOrCreate(idx / kSize2);
71	CHECK_EQ(0U, map2[idx % kSize2]);
72	map2[idx % kSize2] = val;
73	}
74
75	u8 operator[] (uptr idx) const {
76	CHECK_LT(idx, kSize1 * kSize2);
77	u8 *map2 = Get(idx / kSize2);
78	if (!map2) return 0;
79	auto value_ptr = AddressSpaceView::Load(&map2[idx % kSize2]);
80	return *value_ptr;
81	}
82
83	private:
84	u8 *Get(uptr idx) const {
85	CHECK_LT(idx, kSize1);
86	return reinterpret_cast<u8 *>(
87	atomic_load(&map1_[idx], memory_order_acquire));
88	}
89
90	u8 *GetOrCreate(uptr idx) {
91	u8 *res = Get(idx);
92	if (!res) {
93	SpinMutexLock l(&mu_);
94	if (!(res = Get(idx))) {
95	res = (u8*)MmapOrDie(kSize2, "TwoLevelByteMap");
96	MapUnmapCallback().OnMap(reinterpret_cast<uptr>(res), kSize2);
97	atomic_store(&map1_[idx], reinterpret_cast<uptr>(res),
98	memory_order_release);
99	}
100	}
101	return res;
102	}
103
104	atomic_uintptr_t map1_[kSize1];
105	StaticSpinMutex mu_;
106	};
107

lib/tsan/sanitizer_common/sanitizer_allocator_combined.h+8-10

 @@ -29,9 +29,9 @@ class CombinedAllocator {
                          LargeMmapAllocatorPtrArray,
                          typename PrimaryAllocator::AddressSpaceView>;
-  void InitLinkerInitialized(s32 release_to_os_interval_ms) {
+  void InitLinkerInitialized(s32 release_to_os_interval_ms,
-    stats_.InitLinkerInitialized();
+                             uptr heap_start = 0) {
-    primary_.Init(release_to_os_interval_ms);
+    primary_.Init(release_to_os_interval_ms, heap_start);
     secondary_.InitLinkerInitialized();
   }
 @@ -112,15 +112,13 @@ class CombinedAllocator {
     return new_p;
   }
-  bool PointerIsMine(void *p) {
+  bool PointerIsMine(const void *p) const {
     if (primary_.PointerIsMine(p))
       return true;
     return secondary_.PointerIsMine(p);
   }
-  bool FromPrimary(void *p) {
+  bool FromPrimary(const void *p) const { return primary_.PointerIsMine(p); }
-    return primary_.PointerIsMine(p);
-  }
   void *GetMetaData(const void *p) {
     if (primary_.PointerIsMine(p))
 @@ -136,7 +134,7 @@ class CombinedAllocator {
   // This function does the same as GetBlockBegin, but is much faster.
   // Must be called with the allocator locked.
-  void *GetBlockBeginFastLocked(void *p) {
+  void *GetBlockBeginFastLocked(const void *p) {
     if (primary_.PointerIsMine(p))
       return primary_.GetBlockBegin(p);
     return secondary_.GetBlockBeginFastLocked(p);
 @@ -177,12 +175,12 @@ class CombinedAllocator {
   // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
   // introspection API.
-  void ForceLock() NO_THREAD_SAFETY_ANALYSIS {
+  void ForceLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
     primary_.ForceLock();
     secondary_.ForceLock();
   }
-  void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {
+  void ForceUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
     secondary_.ForceUnlock();
     primary_.ForceUnlock();
   }

lib/tsan/sanitizer_common/sanitizer_allocator_dlsym.h created+79

...	@@ -0,0 +1,79 @@
1	//===-- sanitizer_allocator_dlsym.h ------------------------------ C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Hack: Sanitizer initializer calls dlsym which may need to allocate and call
10	// back into uninitialized sanitizer.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#ifndef SANITIZER_ALLOCATOR_DLSYM_H
15	#define SANITIZER_ALLOCATOR_DLSYM_H
16
17	#include "sanitizer_allocator_internal.h"
18
19	namespace __sanitizer {
20
21	template <typename Details>
22	struct DlSymAllocator {
23	static bool Use() {
24	// Fuchsia doesn't use dlsym-based interceptors.
25	return !SANITIZER_FUCHSIA && UNLIKELY(Details::UseImpl());
26	}
27
28	static bool PointerIsMine(const void *ptr) {
29	// Fuchsia doesn't use dlsym-based interceptors.
30	return !SANITIZER_FUCHSIA &&
31	UNLIKELY(internal_allocator()->FromPrimary(ptr));
32	}
33
34	static void *Allocate(uptr size_in_bytes) {
35	void *ptr = InternalAlloc(size_in_bytes, nullptr, kWordSize);
36	CHECK(internal_allocator()->FromPrimary(ptr));
37	Details::OnAllocate(ptr,
38	internal_allocator()->GetActuallyAllocatedSize(ptr));
39	return ptr;
40	}
41
42	static void *Callocate(SIZE_T nmemb, SIZE_T size) {
43	void *ptr = InternalCalloc(nmemb, size);
44	CHECK(internal_allocator()->FromPrimary(ptr));
45	Details::OnAllocate(ptr,
46	internal_allocator()->GetActuallyAllocatedSize(ptr));
47	return ptr;
48	}
49
50	static void Free(void *ptr) {
51	uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
52	Details::OnFree(ptr, size);
53	InternalFree(ptr);
54	}
55
56	static void Realloc(void ptr, uptr new_size) {
57	if (!ptr)
58	return Allocate(new_size);
59	CHECK(internal_allocator()->FromPrimary(ptr));
60	if (!new_size) {
61	Free(ptr);
62	return nullptr;
63	}
64	uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
65	uptr memcpy_size = Min(new_size, size);
66	void *new_ptr = Allocate(new_size);
67	if (new_ptr)
68	internal_memcpy(new_ptr, ptr, memcpy_size);
69	Free(ptr);
70	return new_ptr;
71	}
72
73	static void OnAllocate(const void *ptr, uptr size) {}
74	static void OnFree(const void *ptr, uptr size) {}
75	};
76
77	} // namespace __sanitizer
78
79	#endif // SANITIZER_ALLOCATOR_DLSYM_H

lib/tsan/sanitizer_common/sanitizer_allocator_interface.h+4

 @@ -21,8 +21,12 @@ extern "C" {
 SANITIZER_INTERFACE_ATTRIBUTE
 uptr __sanitizer_get_estimated_allocated_size(uptr size);
 SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_get_ownership(const void *p);
+SANITIZER_INTERFACE_ATTRIBUTE const void *__sanitizer_get_allocated_begin(
+    const void *p);
 SANITIZER_INTERFACE_ATTRIBUTE uptr
 __sanitizer_get_allocated_size(const void *p);
+SANITIZER_INTERFACE_ATTRIBUTE uptr
+__sanitizer_get_allocated_size_fast(const void *p);
 SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_current_allocated_bytes();
 SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_heap_size();
 SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_free_bytes();

lib/tsan/sanitizer_common/sanitizer_allocator_internal.h+2-1

 @@ -48,8 +48,9 @@ void *InternalReallocArray(void *p, uptr count, uptr size,
 void *InternalCalloc(uptr count, uptr size,
                      InternalAllocatorCache *cache = nullptr);
 void InternalFree(void *p, InternalAllocatorCache *cache = nullptr);
+void InternalAllocatorLock();
+void InternalAllocatorUnlock();
 InternalAllocator *internal_allocator();
 } // namespace __sanitizer
 #endif // SANITIZER_ALLOCATOR_INTERNAL_H

lib/tsan/sanitizer_common/sanitizer_allocator_primary32.h+11-12

lib/tsan/sanitizer_common/sanitizer_allocator_primary64.h+15-14

lib/tsan/sanitizer_common/sanitizer_allocator_report.cpp+1-2

...	@@ -128,8 +128,7 @@ void NORETURN ReportAllocationSizeTooBig(uptr user_size, uptr max_size,	...	@@ -128,8 +128,7 @@ void NORETURN ReportAllocationSizeTooBig(uptr user_size, uptr max_size,
128	void NORETURN ReportOutOfMemory(uptr requested_size, const StackTrace *stack) {	128	void NORETURN ReportOutOfMemory(uptr requested_size, const StackTrace *stack) {
129	{	129	{
130	ScopedAllocatorErrorReport report("out-of-memory", stack);	130	ScopedAllocatorErrorReport report("out-of-memory", stack);
131	Report("ERROR: %s: allocator is out of memory trying to allocate 0x%zx "	131	ERROR_OOM("allocator is trying to allocate 0x%zx bytes\n", requested_size);
132	"bytes\n", SanitizerToolName, requested_size);
133	}	132	}
134	Die();	133	Die();
135	}	134	}

lib/tsan/sanitizer_common/sanitizer_allocator_secondary.h+9-9

...	@@ -82,7 +82,7 @@ class LargeMmapAllocator {	...	@@ -82,7 +82,7 @@ class LargeMmapAllocator {
82	InitLinkerInitialized();	82	InitLinkerInitialized();
83	}	83	}
84		84
85	void Allocate(AllocatorStats stat, uptr size, uptr alignment) {	85	void Allocate(AllocatorStats stat, const uptr size, uptr alignment) {
86	CHECK(IsPowerOfTwo(alignment));	86	CHECK(IsPowerOfTwo(alignment));
87	uptr map_size = RoundUpMapSize(size);	87	uptr map_size = RoundUpMapSize(size);
88	if (alignment > page_size_)	88	if (alignment > page_size_)
...	@@ -99,11 +99,11 @@ class LargeMmapAllocator {	...	@@ -99,11 +99,11 @@ class LargeMmapAllocator {
99	if (!map_beg)	99	if (!map_beg)
100	return nullptr;	100	return nullptr;
101	CHECK(IsAligned(map_beg, page_size_));	101	CHECK(IsAligned(map_beg, page_size_));
102	MapUnmapCallback().OnMap(map_beg, map_size);
103	uptr map_end = map_beg + map_size;	102	uptr map_end = map_beg + map_size;
104	uptr res = map_beg + page_size_;	103	uptr res = map_beg + page_size_;
105	if (res & (alignment - 1)) // Align.	104	if (res & (alignment - 1)) // Align.
106	res += alignment - (res & (alignment - 1));	105	res += alignment - (res & (alignment - 1));
		106	MapUnmapCallback().OnMapSecondary(map_beg, map_size, res, size);
107	CHECK(IsAligned(res, alignment));	107	CHECK(IsAligned(res, alignment));
108	CHECK(IsAligned(res, page_size_));	108	CHECK(IsAligned(res, page_size_));
109	CHECK_GE(res + size, map_beg);	109	CHECK_GE(res + size, map_beg);
...	@@ -161,7 +161,7 @@ class LargeMmapAllocator {	...	@@ -161,7 +161,7 @@ class LargeMmapAllocator {
161	return res;	161	return res;
162	}	162	}
163		163
164	bool PointerIsMine(const void *p) {	164	bool PointerIsMine(const void *p) const {
165	return GetBlockBegin(p) != nullptr;	165	return GetBlockBegin(p) != nullptr;
166	}	166	}
167		167
...	@@ -179,7 +179,7 @@ class LargeMmapAllocator {	...	@@ -179,7 +179,7 @@ class LargeMmapAllocator {
179	return GetHeader(p) + 1;	179	return GetHeader(p) + 1;
180	}	180	}
181		181
182	void GetBlockBegin(const void ptr) {	182	void GetBlockBegin(const void ptr) const {
183	uptr p = reinterpret_cast<uptr>(ptr);	183	uptr p = reinterpret_cast<uptr>(ptr);
184	SpinMutexLock l(&mutex_);	184	SpinMutexLock l(&mutex_);
185	uptr nearest_chunk = 0;	185	uptr nearest_chunk = 0;
...	@@ -215,7 +215,7 @@ class LargeMmapAllocator {	...	@@ -215,7 +215,7 @@ class LargeMmapAllocator {
215		215
216	// This function does the same as GetBlockBegin, but is much faster.	216	// This function does the same as GetBlockBegin, but is much faster.
217	// Must be called with the allocator locked.	217	// Must be called with the allocator locked.
218	void GetBlockBeginFastLocked(void ptr) {	218	void GetBlockBeginFastLocked(const void ptr) {
219	mutex_.CheckLocked();	219	mutex_.CheckLocked();
220	uptr p = reinterpret_cast<uptr>(ptr);	220	uptr p = reinterpret_cast<uptr>(ptr);
221	uptr n = n_chunks_;	221	uptr n = n_chunks_;
...	@@ -267,9 +267,9 @@ class LargeMmapAllocator {	...	@@ -267,9 +267,9 @@ class LargeMmapAllocator {
267		267
268	// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone	268	// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
269	// introspection API.	269	// introspection API.
270	void ForceLock() ACQUIRE(mutex_) { mutex_.Lock(); }	270	void ForceLock() SANITIZER_ACQUIRE(mutex_) { mutex_.Lock(); }
271		271
272	void ForceUnlock() RELEASE(mutex_) { mutex_.Unlock(); }	272	void ForceUnlock() SANITIZER_RELEASE(mutex_) { mutex_.Unlock(); }
273		273
274	// Iterate over all existing chunks.	274	// Iterate over all existing chunks.
275	// The allocator must be locked when calling this function.	275	// The allocator must be locked when calling this function.
...	@@ -301,7 +301,7 @@ class LargeMmapAllocator {	...	@@ -301,7 +301,7 @@ class LargeMmapAllocator {
301	return GetHeader(reinterpret_cast<uptr>(p));	301	return GetHeader(reinterpret_cast<uptr>(p));
302	}	302	}
303		303
304	void GetUser(const Header h) {	304	void GetUser(const Header h) const {
305	CHECK(IsAligned((uptr)h, page_size_));	305	CHECK(IsAligned((uptr)h, page_size_));
306	return reinterpret_cast<void*>(reinterpret_cast<uptr>(h) + page_size_);	306	return reinterpret_cast<void*>(reinterpret_cast<uptr>(h) + page_size_);
307	}	307	}
...	@@ -318,5 +318,5 @@ class LargeMmapAllocator {	...	@@ -318,5 +318,5 @@ class LargeMmapAllocator {
318	struct Stats {	318	struct Stats {
319	uptr n_allocs, n_frees, currently_allocated, max_allocated, by_size_log[64];	319	uptr n_allocs, n_frees, currently_allocated, max_allocated, by_size_log[64];
320	} stats;	320	} stats;
321	StaticSpinMutex mutex_;	321	mutable StaticSpinMutex mutex_;
322	};	322	};

lib/tsan/sanitizer_common/sanitizer_allocator_size_class_map.h+4-4

...	@@ -193,13 +193,13 @@ class SizeClassMap {	...	@@ -193,13 +193,13 @@ class SizeClassMap {
193	uptr cached = MaxCachedHint(s) * s;	193	uptr cached = MaxCachedHint(s) * s;
194	if (i == kBatchClassID)	194	if (i == kBatchClassID)
195	d = p = l = 0;	195	d = p = l = 0;
196	Printf("c%02zd => s: %zd diff: +%zd %02zd%% l %zd "	196	Printf(
197	"cached: %zd %zd; id %zd\n",	197	"c%02zu => s: %zu diff: +%zu %02zu%% l %zu cached: %zu %zu; id %zu\n",
198	i, Size(i), d, p, l, MaxCachedHint(s), cached, ClassID(s));	198	i, Size(i), d, p, l, MaxCachedHint(s), cached, ClassID(s));
199	total_cached += cached;	199	total_cached += cached;
200	prev_s = s;	200	prev_s = s;
201	}	201	}
202	Printf("Total cached: %zd\n", total_cached);	202	Printf("Total cached: %zu\n", total_cached);
203	}	203	}
204		204
205	static void Validate() {	205	static void Validate() {

lib/tsan/sanitizer_common/sanitizer_allocator_stats.h+12-15

 @@ -25,19 +25,13 @@ typedef uptr AllocatorStatCounters[AllocatorStatCount];
 // Per-thread stats, live in per-thread cache.
 class AllocatorStats {
  public:
-  void Init() {
+  void Init() { internal_memset(this, 0, sizeof(*this)); }
-    internal_memset(this, 0, sizeof(*this));
-  }
-  void InitLinkerInitialized() {}
   void Add(AllocatorStat i, uptr v) {
-    v += atomic_load(&stats_[i], memory_order_relaxed);
+    atomic_fetch_add(&stats_[i], v, memory_order_relaxed);
-    atomic_store(&stats_[i], v, memory_order_relaxed);
   }
   void Sub(AllocatorStat i, uptr v) {
-    v = atomic_load(&stats_[i], memory_order_relaxed) - v;
+    atomic_fetch_sub(&stats_[i], v, memory_order_relaxed);
-    atomic_store(&stats_[i], v, memory_order_relaxed);
   }
   void Set(AllocatorStat i, uptr v) {
 @@ -58,17 +52,13 @@ class AllocatorStats {
 // Global stats, used for aggregation and querying.
 class AllocatorGlobalStats : public AllocatorStats {
  public:
-  void InitLinkerInitialized() {
-    next_ = this;
-    prev_ = this;
-  }
   void Init() {
     internal_memset(this, 0, sizeof(*this));
-    InitLinkerInitialized();
   }
   void Register(AllocatorStats *s) {
     SpinMutexLock l(&mu_);
+    LazyInit();
     s->next_ = next_;
     s->prev_ = this;
     next_->prev_ = s;
 @@ -87,7 +77,7 @@ class AllocatorGlobalStats : public AllocatorStats {
     internal_memset(s, 0, AllocatorStatCount * sizeof(uptr));
     SpinMutexLock l(&mu_);
     const AllocatorStats *stats = this;
-    for (;;) {
+    for (; stats;) {
       for (int i = 0; i < AllocatorStatCount; i++)
         s[i] += stats->Get(AllocatorStat(i));
       stats = stats->next_;
 @@ -100,6 +90,13 @@ class AllocatorGlobalStats : public AllocatorStats {
   }
  private:
+  void LazyInit() {
+    if (!next_) {
+      next_ = this;
+      prev_ = this;
+    }
+  }
   mutable StaticSpinMutex mu_;
 };

lib/tsan/sanitizer_common/sanitizer_array_ref.h created+123

...	@@ -0,0 +1,123 @@
1	//===-- sanitizer_array_ref.h ------------------------------------ C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#ifndef SANITIZER_ARRAY_REF_H
10	#define SANITIZER_ARRAY_REF_H
11
12	#include "sanitizer_internal_defs.h"
13
14	namespace __sanitizer {
15
16	/// ArrayRef - Represent a constant reference to an array (0 or more elements
17	/// consecutively in memory), i.e. a start pointer and a length. It allows
18	/// various APIs to take consecutive elements easily and conveniently.
19	///
20	/// This class does not own the underlying data, it is expected to be used in
21	/// situations where the data resides in some other buffer, whose lifetime
22	/// extends past that of the ArrayRef. For this reason, it is not in general
23	/// safe to store an ArrayRef.
24	///
25	/// This is intended to be trivially copyable, so it should be passed by
26	/// value.
27	template <typename T>
28	class ArrayRef {
29	public:
30	constexpr ArrayRef() {}
31	constexpr ArrayRef(const T begin, const T end) : begin_(begin), end_(end) {
32	DCHECK(empty() \|\| begin);
33	}
34	constexpr ArrayRef(const T *data, uptr length)
35	: ArrayRef(data, data + length) {}
36	template <uptr N>
37	constexpr ArrayRef(const T (&src)[N]) : ArrayRef(src, src + N) {}
38	template <typename C>
39	constexpr ArrayRef(const C &src)
40	: ArrayRef(src.data(), src.data() + src.size()) {}
41	ArrayRef(const T &one_elt) : ArrayRef(&one_elt, &one_elt + 1) {}
42
43	const T *data() const { return empty() ? nullptr : begin_; }
44
45	const T *begin() const { return begin_; }
46	const T *end() const { return end_; }
47
48	bool empty() const { return begin_ == end_; }
49
50	uptr size() const { return end_ - begin_; }
51
52	/// equals - Check for element-wise equality.
53	bool equals(ArrayRef rhs) const {
54	if (size() != rhs.size())
55	return false;
56	auto r = rhs.begin();
57	for (auto &l : *this) {
58	if (!(l == *r))
59	return false;
60	++r;
61	}
62	return true;
63	}
64
65	/// slice(n, m) - Chop off the first N elements of the array, and keep M
66	/// elements in the array.
67	ArrayRef<T> slice(uptr N, uptr M) const {
68	DCHECK_LE(N + M, size());
69	return ArrayRef<T>(data() + N, M);
70	}
71
72	/// slice(n) - Chop off the first N elements of the array.
73	ArrayRef<T> slice(uptr N) const { return slice(N, size() - N); }
74
75	/// Drop the first \p N elements of the array.
76	ArrayRef<T> drop_front(uptr N = 1) const {
77	DCHECK_GE(size(), N);
78	return slice(N, size() - N);
79	}
80
81	/// Drop the last \p N elements of the array.
82	ArrayRef<T> drop_back(uptr N = 1) const {
83	DCHECK_GE(size(), N);
84	return slice(0, size() - N);
85	}
86
87	/// Return a copy of *this with only the first \p N elements.
88	ArrayRef<T> take_front(uptr N = 1) const {
89	if (N >= size())
90	return *this;
91	return drop_back(size() - N);
92	}
93
94	/// Return a copy of *this with only the last \p N elements.
95	ArrayRef<T> take_back(uptr N = 1) const {
96	if (N >= size())
97	return *this;
98	return drop_front(size() - N);
99	}
100
101	const T &operator[](uptr index) const {
102	DCHECK_LT(index, size());
103	return begin_[index];
104	}
105
106	private:
107	const T *begin_ = nullptr;
108	const T *end_ = nullptr;
109	};
110
111	template <typename T>
112	inline bool operator==(ArrayRef<T> lhs, ArrayRef<T> rhs) {
113	return lhs.equals(rhs);
114	}
115
116	template <typename T>
117	inline bool operator!=(ArrayRef<T> lhs, ArrayRef<T> rhs) {
118	return !(lhs == rhs);
119	}
120
121	} // namespace __sanitizer
122
123	#endif // SANITIZER_ARRAY_REF_H

lib/tsan/sanitizer_common/sanitizer_asm.h+54-3

...	@@ -6,7 +6,7 @@	...	@@ -6,7 +6,7 @@
6	//	6	//
7	//===----------------------------------------------------------------------===//	7	//===----------------------------------------------------------------------===//
8	//	8	//
9	// Various support for assemebler.	9	// Various support for assembler.
10	//	10	//
11	//===----------------------------------------------------------------------===//	11	//===----------------------------------------------------------------------===//
12		12
...	@@ -42,13 +42,57 @@	...	@@ -42,13 +42,57 @@
42	# define CFI_RESTORE(reg)	42	# define CFI_RESTORE(reg)
43	#endif	43	#endif
44		44
		45	#if defined(__x86_64__) \|\| defined(__i386__) \|\| defined(__sparc__)
		46	# define ASM_TAIL_CALL jmp
		47	#elif defined(__arm__) \|\| defined(__aarch64__) \|\| defined(__mips__) \|\| \
		48	defined(__powerpc__) \|\| defined(__loongarch_lp64)
		49	# define ASM_TAIL_CALL b
		50	#elif defined(__s390__)
		51	# define ASM_TAIL_CALL jg
		52	#elif defined(__riscv)
		53	# define ASM_TAIL_CALL tail
		54	#endif
		55
		56	#if defined(__ELF__) && defined(__x86_64__) \|\| defined(__i386__) \|\| \
		57	defined(__riscv)
		58	# define ASM_PREEMPTIBLE_SYM(sym) sym@plt
		59	#else
		60	# define ASM_PREEMPTIBLE_SYM(sym) sym
		61	#endif
		62
45	#if !defined(__APPLE__)	63	#if !defined(__APPLE__)
46	# define ASM_HIDDEN(symbol) .hidden symbol	64	# define ASM_HIDDEN(symbol) .hidden symbol
47	# define ASM_TYPE_FUNCTION(symbol) .type symbol, %function	65	# define ASM_TYPE_FUNCTION(symbol) .type symbol, %function
48	# define ASM_SIZE(symbol) .size symbol, .-symbol	66	# define ASM_SIZE(symbol) .size symbol, .-symbol
49	# define ASM_SYMBOL(symbol) symbol	67	# define ASM_SYMBOL(symbol) symbol
50	# define ASM_SYMBOL_INTERCEPTOR(symbol) symbol	68	# define ASM_SYMBOL_INTERCEPTOR(symbol) symbol
51	# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol	69	# if defined(__i386__) \|\| defined(__powerpc__) \|\| defined(__s390__) \|\| \
		70	defined(__sparc__)
		71	// For details, see interception.h
		72	# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol
		73	# define ASM_TRAMPOLINE_ALIAS(symbol, name) \
		74	.weak symbol; \
		75	.set symbol, ASM_WRAPPER_NAME(name)
		76	# define ASM_INTERCEPTOR_TRAMPOLINE(name)
		77	# define ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT 0
		78	# else // Architecture supports interceptor trampoline
		79	// Keep trampoline implementation in sync with interception/interception.h
		80	# define ASM_WRAPPER_NAME(symbol) ___interceptor_##symbol
		81	# define ASM_TRAMPOLINE_ALIAS(symbol, name) \
		82	.weak symbol; \
		83	.set symbol, __interceptor_trampoline_##name
		84	# define ASM_INTERCEPTOR_TRAMPOLINE(name) \
		85	.weak __interceptor_##name; \
		86	.set __interceptor_##name, ASM_WRAPPER_NAME(name); \
		87	.globl __interceptor_trampoline_##name; \
		88	ASM_TYPE_FUNCTION(__interceptor_trampoline_##name); \
		89	__interceptor_trampoline_##name: \
		90	CFI_STARTPROC; \
		91	ASM_TAIL_CALL ASM_PREEMPTIBLE_SYM(__interceptor_##name); \
		92	CFI_ENDPROC; \
		93	ASM_SIZE(__interceptor_trampoline_##name)
		94	# define ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT 1
		95	# endif // Architecture supports interceptor trampoline
52	#else	96	#else
53	# define ASM_HIDDEN(symbol)	97	# define ASM_HIDDEN(symbol)
54	# define ASM_TYPE_FUNCTION(symbol)	98	# define ASM_TYPE_FUNCTION(symbol)
...	@@ -61,8 +105,15 @@	...	@@ -61,8 +105,15 @@
61	#if defined(__ELF__) && (defined(__GNU__) \|\| defined(__FreeBSD__) \|\| \	105	#if defined(__ELF__) && (defined(__GNU__) \|\| defined(__FreeBSD__) \|\| \
62	defined(__Fuchsia__) \|\| defined(__linux__))	106	defined(__Fuchsia__) \|\| defined(__linux__))
63	// clang-format off	107	// clang-format off
64	#define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits // NOLINT	108	#define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits
65	// clang-format on	109	// clang-format on
66	#else	110	#else
67	#define NO_EXEC_STACK_DIRECTIVE	111	#define NO_EXEC_STACK_DIRECTIVE
68	#endif	112	#endif
		113
		114	#if (defined(__x86_64__) \|\| defined(__i386__)) && defined(__has_include) && __has_include(<cet.h>)
		115	#include <cet.h>
		116	#endif
		117	#ifndef _CET_ENDBR
		118	#define _CET_ENDBR
		119	#endif

lib/tsan/sanitizer_common/sanitizer_atomic_clang.h+8-9

 @@ -74,13 +74,12 @@ template <typename T>
 inline bool atomic_compare_exchange_strong(volatile T *a, typename T::Type *cmp,
                                            typename T::Type xchg,
                                            memory_order mo) {
-  typedef typename T::Type Type;
+  // Transitioned from __sync_val_compare_and_swap to support targets like
-  Type cmpv = *cmp;
+  // SPARC V8 that cannot inline atomic cmpxchg.  __atomic_compare_exchange
-  Type prev;
+  // can then be resolved from libatomic.  __ATOMIC_SEQ_CST is used to best
-  prev = __sync_val_compare_and_swap(&a->val_dont_use, cmpv, xchg);
+  // match the __sync builtin memory order.
-  if (prev == cmpv) return true;
+  return __atomic_compare_exchange(&a->val_dont_use, cmp, &xchg, false,
-  *cmp = prev;
+                                   __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
-  return false;
 }
 template<typename T>
 @@ -96,8 +95,8 @@ inline bool atomic_compare_exchange_weak(volatile T *a,
 // This include provides explicit template instantiations for atomic_uint64_t
 // on MIPS32, which does not directly support 8 byte atomics. It has to
 // proceed the template definitions above.
-#if defined(_MIPS_SIM) && defined(_ABIO32)
+#if defined(_MIPS_SIM) && defined(_ABIO32) && _MIPS_SIM == _ABIO32
-  #include "sanitizer_atomic_clang_mips.h"
+#  include "sanitizer_atomic_clang_mips.h"
 #endif
 #undef ATOMIC_ORDER

lib/tsan/sanitizer_common/sanitizer_atomic_clang_mips.h+1-1

...	@@ -18,7 +18,7 @@ namespace __sanitizer {	...	@@ -18,7 +18,7 @@ namespace __sanitizer {
18		18
19	// MIPS32 does not support atomics > 4 bytes. To address this lack of	19	// MIPS32 does not support atomics > 4 bytes. To address this lack of
20	// functionality, the sanitizer library provides helper methods which use an	20	// functionality, the sanitizer library provides helper methods which use an
21	// internal spin lock mechanism to emulate atomic oprations when the size is	21	// internal spin lock mechanism to emulate atomic operations when the size is
22	// 8 bytes.	22	// 8 bytes.
23	static void __spin_lock(volatile int *lock) {	23	static void __spin_lock(volatile int *lock) {
24	while (__sync_lock_test_and_set(lock, 1))	24	while (__sync_lock_test_and_set(lock, 1))

lib/tsan/sanitizer_common/sanitizer_chained_origin_depot.cpp created+148

...	@@ -0,0 +1,148 @@
1	//===-- sanitizer_chained_origin_depot.cpp --------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// A storage for chained origins.
10	//===----------------------------------------------------------------------===//
11
12	#include "sanitizer_chained_origin_depot.h"
13
14	#include "sanitizer_stackdepotbase.h"
15
16	namespace __sanitizer {
17
18	namespace {
19	struct ChainedOriginDepotDesc {
20	u32 here_id;
21	u32 prev_id;
22	};
23
24	struct ChainedOriginDepotNode {
25	using hash_type = u32;
26	u32 link;
27	u32 here_id;
28	u32 prev_id;
29
30	typedef ChainedOriginDepotDesc args_type;
31
32	bool eq(hash_type hash, const args_type &args) const;
33
34	static uptr allocated() { return 0; }
35
36	static hash_type hash(const args_type &args);
37
38	static bool is_valid(const args_type &args);
39
40	void store(u32 id, const args_type &args, hash_type other_hash);
41
42	args_type load(u32 id) const;
43
44	struct Handle {
45	const ChainedOriginDepotNode *node_ = nullptr;
46	u32 id_ = 0;
47	Handle(const ChainedOriginDepotNode *node, u32 id) : node_(node), id_(id) {}
48	bool valid() const { return node_; }
49	u32 id() const { return id_; }
50	int here_id() const { return node_->here_id; }
51	int prev_id() const { return node_->prev_id; }
52	};
53
54	static Handle get_handle(u32 id);
55
56	typedef Handle handle_type;
57	};
58
59	} // namespace
60
61	static StackDepotBase<ChainedOriginDepotNode, 4, 20> depot;
62
63	bool ChainedOriginDepotNode::eq(hash_type hash, const args_type &args) const {
64	return here_id == args.here_id && prev_id == args.prev_id;
65	}
66
67	/* This is murmur2 hash for the 64->32 bit case.
68	It does not behave all that well because the keys have a very biased
69	distribution (I've seen 7-element buckets with the table only 14% full).
70
71	here_id is built of
72	* (1 bits) Reserved, zero.
73	* (8 bits) Part id = bits 13..20 of the hash value of here_id's key.
74	* (23 bits) Sequential number (each part has each own sequence).
75
76	prev_id has either the same distribution as here_id (but with 3:8:21)
77	split, or one of two reserved values (-1) or (-2). Either case can
78	dominate depending on the workload.
79	*/
80	ChainedOriginDepotNode::hash_type ChainedOriginDepotNode::hash(
81	const args_type &args) {
82	const u32 m = 0x5bd1e995;
83	const u32 seed = 0x9747b28c;
84	const u32 r = 24;
85	u32 h = seed;
86	u32 k = args.here_id;
87	k *= m;
88	k ^= k >> r;
89	k *= m;
90	h *= m;
91	h ^= k;
92
93	k = args.prev_id;
94	k *= m;
95	k ^= k >> r;
96	k *= m;
97	h *= m;
98	h ^= k;
99
100	h ^= h >> 13;
101	h *= m;
102	h ^= h >> 15;
103	return h;
104	}
105
106	bool ChainedOriginDepotNode::is_valid(const args_type &args) { return true; }
107
108	void ChainedOriginDepotNode::store(u32 id, const args_type &args,
109	hash_type other_hash) {
110	here_id = args.here_id;
111	prev_id = args.prev_id;
112	}
113
114	ChainedOriginDepotNode::args_type ChainedOriginDepotNode::load(u32 id) const {
115	args_type ret = {here_id, prev_id};
116	return ret;
117	}
118
119	ChainedOriginDepotNode::Handle ChainedOriginDepotNode::get_handle(u32 id) {
120	return Handle(&depot.nodes[id], id);
121	}
122
123	ChainedOriginDepot::ChainedOriginDepot() {}
124
125	StackDepotStats ChainedOriginDepot::GetStats() const {
126	return depot.GetStats();
127	}
128
129	bool ChainedOriginDepot::Put(u32 here_id, u32 prev_id, u32 *new_id) {
130	ChainedOriginDepotDesc desc = {here_id, prev_id};
131	bool inserted;
132	*new_id = depot.Put(desc, &inserted);
133	return inserted;
134	}
135
136	u32 ChainedOriginDepot::Get(u32 id, u32 *other) {
137	ChainedOriginDepotDesc desc = depot.Get(id);
138	*other = desc.prev_id;
139	return desc.here_id;
140	}
141
142	void ChainedOriginDepot::LockAll() { depot.LockAll(); }
143
144	void ChainedOriginDepot::UnlockAll() { depot.UnlockAll(); }
145
146	void ChainedOriginDepot::TestOnlyUnmap() { depot.TestOnlyUnmap(); }
147
148	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_chained_origin_depot.h created+46

...	@@ -0,0 +1,46 @@
1	//===-- sanitizer_chained_origin_depot.h ------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// A storage for chained origins.
10	//===----------------------------------------------------------------------===//
11
12	#ifndef SANITIZER_CHAINED_ORIGIN_DEPOT_H
13	#define SANITIZER_CHAINED_ORIGIN_DEPOT_H
14
15	#include "sanitizer_common.h"
16
17	namespace __sanitizer {
18
19	class ChainedOriginDepot {
20	public:
21	ChainedOriginDepot();
22
23	// Gets the statistic of the origin chain storage.
24	StackDepotStats GetStats() const;
25
26	// Stores a chain with StackDepot ID here_id and previous chain ID prev_id.
27	// If successful, returns true and the new chain id new_id.
28	// If the same element already exists, returns false and sets new_id to the
29	// existing ID.
30	bool Put(u32 here_id, u32 prev_id, u32 *new_id);
31
32	// Retrieves the stored StackDepot ID for the given origin ID.
33	u32 Get(u32 id, u32 *other);
34
35	void LockAll();
36	void UnlockAll();
37	void TestOnlyUnmap();
38
39	private:
40	ChainedOriginDepot(const ChainedOriginDepot &) = delete;
41	void operator=(const ChainedOriginDepot &) = delete;
42	};
43
44	} // namespace __sanitizer
45
46	#endif // SANITIZER_CHAINED_ORIGIN_DEPOT_H

lib/tsan/sanitizer_common/sanitizer_common.cpp+68-10

 @@ -11,10 +11,12 @@
 //===----------------------------------------------------------------------===//
 #include "sanitizer_common.h"
 #include "sanitizer_allocator_interface.h"
 #include "sanitizer_allocator_internal.h"
 #include "sanitizer_atomic.h"
 #include "sanitizer_flags.h"
+#include "sanitizer_interface_internal.h"
 #include "sanitizer_libc.h"
 #include "sanitizer_placement_new.h"
 @@ -44,15 +46,41 @@ void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
     Die();
   }
   recursion_count++;
-  Report("ERROR: %s failed to "
+  if (ErrorIsOOM(err)) {
-         "%s 0x%zx (%zd) bytes of %s (error code: %d)\n",
+    ERROR_OOM("failed to %s 0x%zx (%zd) bytes of %s (error code: %d)\n",
-         SanitizerToolName, mmap_type, size, size, mem_type, err);
+              mmap_type, size, size, mem_type, err);
+  } else {
+    Report(
+        "ERROR: %s failed to "
+        "%s 0x%zx (%zd) bytes of %s (error code: %d)\n",
+        SanitizerToolName, mmap_type, size, size, mem_type, err);
+  }
 #if !SANITIZER_GO
   DumpProcessMap();
 #endif
   UNREACHABLE("unable to mmap");
 }
+void NORETURN ReportMunmapFailureAndDie(void *addr, uptr size, error_t err,
+                                        bool raw_report) {
+  static int recursion_count;
+  if (raw_report || recursion_count) {
+    // If raw report is requested or we went into recursion just die.  The
+    // Report() and CHECK calls below may call munmap recursively and fail.
+    RawWrite("ERROR: Failed to munmap\n");
+    Die();
+  }
+  recursion_count++;
+  Report(
+      "ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p (error "
+      "code: %d)\n",
+      SanitizerToolName, size, size, addr, err);
+#if !SANITIZER_GO
+  DumpProcessMap();
+#endif
+  UNREACHABLE("unable to unmmap");
+}
 typedef bool UptrComparisonFunction(const uptr &a, const uptr &b);
 typedef bool U32ComparisonFunction(const u32 &a, const u32 &b);
 @@ -138,13 +166,21 @@ void LoadedModule::set(const char *module_name, uptr base_address,
   set(module_name, base_address);
   arch_ = arch;
   internal_memcpy(uuid_, uuid, sizeof(uuid_));
+  uuid_size_ = kModuleUUIDSize;
   instrumented_ = instrumented;
 }
+void LoadedModule::setUuid(const char *uuid, uptr size) {
+  if (size > kModuleUUIDSize)
+    size = kModuleUUIDSize;
+  internal_memcpy(uuid_, uuid, size);
+  uuid_size_ = size;
+}
 void LoadedModule::clear() {
   InternalFree(full_name_);
   base_address_ = 0;
-  max_executable_address_ = 0;
+  max_address_ = 0;
   full_name_ = nullptr;
   arch_ = kModuleArchUnknown;
   internal_memset(uuid_, 0, kModuleUUIDSize);
 @@ -162,8 +198,7 @@ void LoadedModule::addAddressRange(uptr beg, uptr end, bool executable,
   AddressRange *r =
       new(mem) AddressRange(beg, end, executable, writable, name);
   ranges_.push_back(r);
-  if (executable && end > max_executable_address_)
+  max_address_ = Max(max_address_, end);
-    max_executable_address_ = end;
 }
 bool LoadedModule::containsAddress(uptr address) const {
 @@ -301,18 +336,22 @@ struct MallocFreeHook {
 static MallocFreeHook MFHooks[kMaxMallocFreeHooks];
-void RunMallocHooks(const void *ptr, uptr size) {
+void RunMallocHooks(void *ptr, uptr size) {
+  __sanitizer_malloc_hook(ptr, size);
   for (int i = 0; i < kMaxMallocFreeHooks; i++) {
     auto hook = MFHooks[i].malloc_hook;
-    if (!hook) return;
+    if (!hook)
+      break;
     hook(ptr, size);
   }
 }
-void RunFreeHooks(const void *ptr) {
+void RunFreeHooks(void *ptr) {
+  __sanitizer_free_hook(ptr);
   for (int i = 0; i < kMaxMallocFreeHooks; i++) {
     auto hook = MFHooks[i].free_hook;
-    if (!hook) return;
+    if (!hook)
+      break;
     hook(ptr);
   }
 }
 @@ -338,6 +377,13 @@ void SleepForSeconds(unsigned seconds) {
 }
 void SleepForMillis(unsigned millis) { internal_usleep((u64)millis * 1000); }
+void WaitForDebugger(unsigned seconds, const char *label) {
+  if (seconds) {
+    Report("Sleeping for %u second(s) %s\n", seconds, label);
+    SleepForSeconds(seconds);
+  }
+}
 } // namespace __sanitizer
 using namespace __sanitizer;
 @@ -360,4 +406,16 @@ int __sanitizer_install_malloc_and_free_hooks(void (*malloc_hook)(const void *,
                                               void (*free_hook)(const void *)) {
   return InstallMallocFreeHooks(malloc_hook, free_hook);
 }
+// Provide default (no-op) implementation of malloc hooks.
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_malloc_hook, void *ptr,
+                             uptr size) {
+  (void)ptr;
+  (void)size;
+}
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_free_hook, void *ptr) {
+  (void)ptr;
+}
 } // extern "C"

lib/tsan/sanitizer_common/sanitizer_common.h+82-68

 @@ -16,7 +16,6 @@
 #define SANITIZER_COMMON_H
 #include "sanitizer_flags.h"
-#include "sanitizer_interface_internal.h"
 #include "sanitizer_internal_defs.h"
 #include "sanitizer_libc.h"
 #include "sanitizer_list.h"
 @@ -118,9 +117,15 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
 // unaccessible memory.
 bool MprotectNoAccess(uptr addr, uptr size);
 bool MprotectReadOnly(uptr addr, uptr size);
+bool MprotectReadWrite(uptr addr, uptr size);
 void MprotectMallocZones(void *addr, int prot);
+#if SANITIZER_WINDOWS
+// Zero previously mmap'd memory. Currently used only on Windows.
+bool ZeroMmapFixedRegion(uptr fixed_addr, uptr size) WARN_UNUSED_RESULT;
+#endif
 #if SANITIZER_LINUX
 // Unmap memory. Currently only used on Linux.
 void UnmapFromTo(uptr from, uptr to);
 @@ -171,8 +176,8 @@ void SetShadowRegionHugePageMode(uptr addr, uptr length);
 bool DontDumpShadowMemory(uptr addr, uptr length);
 // Check if the built VMA size matches the runtime one.
 void CheckVMASize();
-void RunMallocHooks(const void *ptr, uptr size);
+void RunMallocHooks(void *ptr, uptr size);
-void RunFreeHooks(const void *ptr);
+void RunFreeHooks(void *ptr);
 class ReservedAddressRange {
  public:
 @@ -192,12 +197,13 @@ class ReservedAddressRange {
 };
 typedef void (*fill_profile_f)(uptr start, uptr rss, bool file,
-                               /*out*/uptr *stats, uptr stats_size);
+                               /*out*/ uptr *stats);
 // Parse the contents of /proc/self/smaps and generate a memory profile.
-// |cb| is a tool-specific callback that fills the |stats| array containing
+// |cb| is a tool-specific callback that fills the |stats| array.
-// |stats_size| elements.
+void GetMemoryProfile(fill_profile_f cb, uptr *stats);
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size);
+void ParseUnixMemoryProfile(fill_profile_f cb, uptr *stats, char *smaps,
+                            uptr smaps_len);
 // Simple low-level (mmap-based) allocator for internal use. Doesn't have
 // constructor, so all instances of LowLevelAllocator should be
 @@ -206,6 +212,7 @@ class LowLevelAllocator {
  public:
   // Requires an external lock.
   void *Allocate(uptr size);
  private:
   char *allocated_end_;
   char *allocated_current_;
 @@ -222,8 +229,8 @@ void CatastrophicErrorWrite(const char *buffer, uptr length);
 void RawWrite(const char *buffer);
 bool ColorizeReports();
 void RemoveANSIEscapeSequencesFromString(char *buffer);
-void Printf(const char *format, ...);
+void Printf(const char *format, ...) FORMAT(1, 2);
-void Report(const char *format, ...);
+void Report(const char *format, ...) FORMAT(1, 2);
 void SetPrintfAndReportCallback(void (*callback)(const char *));
 #define VReport(level, ...)                                              \
   do {                                                                   \
 @@ -237,12 +244,12 @@ void SetPrintfAndReportCallback(void (*callback)(const char *));
 // Lock sanitizer error reporting and protects against nested errors.
 class ScopedErrorReportLock {
  public:
-  ScopedErrorReportLock() ACQUIRE(mutex_) { Lock(); }
+  ScopedErrorReportLock() SANITIZER_ACQUIRE(mutex_) { Lock(); }
-  ~ScopedErrorReportLock() RELEASE(mutex_) { Unlock(); }
+  ~ScopedErrorReportLock() SANITIZER_RELEASE(mutex_) { Unlock(); }
-  static void Lock() ACQUIRE(mutex_);
+  static void Lock() SANITIZER_ACQUIRE(mutex_);
-  static void Unlock() RELEASE(mutex_);
+  static void Unlock() SANITIZER_RELEASE(mutex_);
-  static void CheckLocked() CHECK_LOCKED(mutex_);
+  static void CheckLocked() SANITIZER_CHECK_LOCKED(mutex_);
  private:
   static atomic_uintptr_t reporting_thread_;
 @@ -285,7 +292,7 @@ void SetStackSizeLimitInBytes(uptr limit);
 bool AddressSpaceIsUnlimited();
 void SetAddressSpaceUnlimited();
 void AdjustStackSize(void *attr);
-void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args);
+void PlatformPrepareForSandboxing(void *args);
 void SetSandboxingCallback(void (*f)());
 void InitializeCoverage(bool enabled, const char *coverage_dir);
 @@ -294,6 +301,7 @@ void InitTlsSize();
 uptr GetTlsSize();
 // Other
+void WaitForDebugger(unsigned seconds, const char *label);
 void SleepForSeconds(unsigned seconds);
 void SleepForMillis(unsigned millis);
 u64 NanoTime();
 @@ -309,6 +317,20 @@ CheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2);
 void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
                                       const char *mmap_type, error_t err,
                                       bool raw_report = false);
+void NORETURN ReportMunmapFailureAndDie(void *ptr, uptr size, error_t err,
+                                        bool raw_report = false);
+// Returns true if the platform-specific error reported is an OOM error.
+bool ErrorIsOOM(error_t err);
+// This reports an error in the form:
+//
+//   `ERROR: {{SanitizerToolName}}: out of memory: {{err_msg}}`
+//
+// Downstream tools that read sanitizer output will know that errors starting
+// in this format are specifically OOM errors.
+#define ERROR_OOM(err_msg, ...) \
+  Report("ERROR: %s: out of memory: " err_msg, SanitizerToolName, __VA_ARGS__)
 // Specific tools may override behavior of "Die" function to do tool-specific
 // job.
 @@ -325,12 +347,6 @@ void SetUserDieCallback(DieCallbackType callback);
 void SetCheckUnwindCallback(void (*callback)());
-// Callback will be called if soft_rss_limit_mb is given and the limit is
-// exceeded (exceeded==true) or if rss went down below the limit
-// (exceeded==false).
-// The callback should be registered once at the tool init time.
-void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded));
 // Functions related to signal handling.
 typedef void (*SignalHandlerType)(int, void *, void *);
 HandleSignalMode GetHandleSignalMode(int signum);
 @@ -371,7 +387,7 @@ void ReportErrorSummary(const char *error_type, const AddressInfo &info,
 void ReportErrorSummary(const char *error_type, const StackTrace *trace,
                         const char *alt_tool_name = nullptr);
-void ReportMmapWriteExec(int prot);
+void ReportMmapWriteExec(int prot, int mflags);
 // Math
 #if SANITIZER_WINDOWS && !defined(__clang__) && !defined(__GNUC__)
 @@ -419,9 +435,7 @@ inline uptr LeastSignificantSetBitIndex(uptr x) {
   return up;
 }
-inline bool IsPowerOfTwo(uptr x) {
+inline constexpr bool IsPowerOfTwo(uptr x) { return (x & (x - 1)) == 0; }
-  return (x & (x - 1)) == 0;
-}
 inline uptr RoundUpToPowerOfTwo(uptr size) {
   CHECK(size);
 @@ -433,16 +447,16 @@ inline uptr RoundUpToPowerOfTwo(uptr size) {
   return 1ULL << (up + 1);
 }
-inline uptr RoundUpTo(uptr size, uptr boundary) {
+inline constexpr uptr RoundUpTo(uptr size, uptr boundary) {
   RAW_CHECK(IsPowerOfTwo(boundary));
   return (size + boundary - 1) & ~(boundary - 1);
 }
-inline uptr RoundDownTo(uptr x, uptr boundary) {
+inline constexpr uptr RoundDownTo(uptr x, uptr boundary) {
   return x & ~(boundary - 1);
 }
-inline bool IsAligned(uptr a, uptr alignment) {
+inline constexpr bool IsAligned(uptr a, uptr alignment) {
   return (a & (alignment - 1)) == 0;
 }
 @@ -461,6 +475,10 @@ template <class T>
 constexpr T Max(T a, T b) {
   return a > b ? a : b;
 }
+template <class T>
+constexpr T Abs(T a) {
+  return a < 0 ? -a : a;
+}
 template<class T> void Swap(T& a, T& b) {
   T tmp = a;
   a = b;
 @@ -502,8 +520,8 @@ class InternalMmapVectorNoCtor {
     return data_[i];
   }
   void push_back(const T &element) {
-    CHECK_LE(size_, capacity());
+    if (UNLIKELY(size_ >= capacity())) {
-    if (size_ == capacity()) {
+      CHECK_EQ(size_, capacity());
       uptr new_capacity = RoundUpToPowerOfTwo(size_ + 1);
       Realloc(new_capacity);
     }
 @@ -563,7 +581,7 @@ class InternalMmapVectorNoCtor {
   }
  private:
-  void Realloc(uptr new_capacity) {
+  NOINLINE void Realloc(uptr new_capacity) {
     CHECK_GT(new_capacity, 0);
     CHECK_LE(size_, new_capacity);
     uptr new_capacity_bytes =
 @@ -618,7 +636,7 @@ class InternalScopedString {
     buffer_.resize(1);
     buffer_[0] = '\0';
   }
-  void append(const char *format, ...);
+  void append(const char *format, ...) FORMAT(2, 3);
   const char *data() const { return buffer_.data(); }
   char *data() { return buffer_.data(); }
 @@ -670,11 +688,9 @@ void Sort(T *v, uptr size, Compare comp = {}) {
 // Works like std::lower_bound: finds the first element that is not less
 // than the val.
-template <class Container,
+template <class Container, class T,
           class Compare = CompareLess<typename Container::value_type>>
-uptr InternalLowerBound(const Container &v,
+uptr InternalLowerBound(const Container &v, const T &val, Compare comp = {}) {
-                        const typename Container::value_type &val,
-                        Compare comp = {}) {
   uptr first = 0;
   uptr last = v.size();
   while (last > first) {
 @@ -697,7 +713,9 @@ enum ModuleArch {
   kModuleArchARMV7S,
   kModuleArchARMV7K,
   kModuleArchARM64,
-  kModuleArchRISCV64
+  kModuleArchLoongArch64,
+  kModuleArchRISCV64,
+  kModuleArchHexagon
 };
 // Sorts and removes duplicates from the container.
 @@ -721,12 +739,15 @@ void SortAndDedup(Container &v, Compare comp = {}) {
   v.resize(last + 1);
 }
+constexpr uptr kDefaultFileMaxSize = FIRST_32_SECOND_64(1 << 26, 1 << 28);
 // Opens the file 'file_name" and reads up to 'max_len' bytes.
 // The resulting buffer is mmaped and stored in '*buff'.
 // Returns true if file was successfully opened and read.
 bool ReadFileToVector(const char *file_name,
                       InternalMmapVectorNoCtor<char> *buff,
-                      uptr max_len = 1 << 26, error_t *errno_p = nullptr);
+                      uptr max_len = kDefaultFileMaxSize,
+                      error_t *errno_p = nullptr);
 // Opens the file 'file_name" and reads up to 'max_len' bytes.
 // This function is less I/O efficient than ReadFileToVector as it may reread
 @@ -737,9 +758,12 @@ bool ReadFileToVector(const char *file_name,
 // The total number of read bytes is stored in '*read_len'.
 // Returns true if file was successfully opened and read.
 bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
-                      uptr *read_len, uptr max_len = 1 << 26,
+                      uptr *read_len, uptr max_len = kDefaultFileMaxSize,
                       error_t *errno_p = nullptr);
+int GetModuleAndOffsetForPc(uptr pc, char *module_name, uptr module_name_len,
+                            uptr *pc_offset);
 // When adding a new architecture, don't forget to also update
 // script/asan_symbolize.py and sanitizer_symbolizer_libcdep.cpp.
 inline const char *ModuleArchToString(ModuleArch arch) {
 @@ -762,14 +786,22 @@ inline const char *ModuleArchToString(ModuleArch arch) {
       return "armv7k";
     case kModuleArchARM64:
       return "arm64";
+    case kModuleArchLoongArch64:
+      return "loongarch64";
     case kModuleArchRISCV64:
       return "riscv64";
+    case kModuleArchHexagon:
+      return "hexagon";
   }
   CHECK(0 && "Invalid module arch");
   return "";
 }
+#if SANITIZER_APPLE
 const uptr kModuleUUIDSize = 16;
+#else
+const uptr kModuleUUIDSize = 32;
+#endif
 const uptr kMaxSegName = 16;
 // Represents a binary loaded into virtual memory (e.g. this can be an
 @@ -779,8 +811,9 @@ class LoadedModule {
   LoadedModule()
       : full_name_(nullptr),
         base_address_(0),
-        max_executable_address_(0),
+        max_address_(0),
         arch_(kModuleArchUnknown),
+        uuid_size_(0),
         instrumented_(false) {
     internal_memset(uuid_, 0, kModuleUUIDSize);
     ranges_.clear();
 @@ -788,6 +821,7 @@ class LoadedModule {
   void set(const char *module_name, uptr base_address);
   void set(const char *module_name, uptr base_address, ModuleArch arch,
            u8 uuid[kModuleUUIDSize], bool instrumented);
+  void setUuid(const char *uuid, uptr size);
   void clear();
   void addAddressRange(uptr beg, uptr end, bool executable, bool writable,
                        const char *name = nullptr);
 @@ -795,9 +829,10 @@ class LoadedModule {
   const char *full_name() const { return full_name_; }
   uptr base_address() const { return base_address_; }
-  uptr max_executable_address() const { return max_executable_address_; }
+  uptr max_address() const { return max_address_; }
   ModuleArch arch() const { return arch_; }
   const u8 *uuid() const { return uuid_; }
+  uptr uuid_size() const { return uuid_size_; }
   bool instrumented() const { return instrumented_; }
   struct AddressRange {
 @@ -824,8 +859,9 @@ class LoadedModule {
  private:
   char *full_name_;  // Owned.
   uptr base_address_;
-  uptr max_executable_address_;
+  uptr max_address_;
   ModuleArch arch_;
+  uptr uuid_size_;
   u8 uuid_[kModuleUUIDSize];
   bool instrumented_;
   IntrusiveList<AddressRange> ranges_;
 @@ -883,13 +919,13 @@ void WriteToSyslog(const char *buffer);
 #define SANITIZER_WIN_TRACE 0
 #endif
-#if SANITIZER_MAC || SANITIZER_WIN_TRACE
+#if SANITIZER_APPLE || SANITIZER_WIN_TRACE
 void LogFullErrorReport(const char *buffer);
 #else
 inline void LogFullErrorReport(const char *buffer) {}
 #endif
-#if SANITIZER_LINUX || SANITIZER_MAC
+#if SANITIZER_LINUX || SANITIZER_APPLE
 void WriteOneLineToSyslog(const char *s);
 void LogMessageOnPrintf(const char *str);
 #else
 @@ -951,7 +987,7 @@ struct SignalContext {
   uptr sp;
   uptr bp;
   bool is_memory_access;
-  enum WriteFlag { UNKNOWN, READ, WRITE } write_flag;
+  enum WriteFlag { Unknown, Read, Write } write_flag;
   // In some cases the kernel cannot provide the true faulting address; `addr`
   // will be zero then.  This field allows to distinguish between these cases
 @@ -996,7 +1032,6 @@ struct SignalContext {
 };
 void InitializePlatformEarly();
-void MaybeReexec();
 template <typename Fn>
 class RunOnDestruction {
 @@ -1049,31 +1084,10 @@ inline u32 GetNumberOfCPUsCached() {
   return NumberOfCPUsCached;
 }
-template <typename T>
-class ArrayRef {
- public:
-  ArrayRef() {}
-  ArrayRef(T *begin, T *end) : begin_(begin), end_(end) {}
-  T *begin() { return begin_; }
-  T *end() { return end_; }
- private:
-  T *begin_ = nullptr;
-  T *end_ = nullptr;
-};
-#define PRINTF_128(v)                                                         \
-  (*((u8 *)&v + 0)), (*((u8 *)&v + 1)), (*((u8 *)&v + 2)), (*((u8 *)&v + 3)), \
-      (*((u8 *)&v + 4)), (*((u8 *)&v + 5)), (*((u8 *)&v + 6)),                \
-      (*((u8 *)&v + 7)), (*((u8 *)&v + 8)), (*((u8 *)&v + 9)),                \
-      (*((u8 *)&v + 10)), (*((u8 *)&v + 11)), (*((u8 *)&v + 12)),             \
-      (*((u8 *)&v + 13)), (*((u8 *)&v + 14)), (*((u8 *)&v + 15))
 }  // namespace __sanitizer
 inline void *operator new(__sanitizer::operator_new_size_type size,
-                          __sanitizer::LowLevelAllocator &alloc) {  // NOLINT
+                          __sanitizer::LowLevelAllocator &alloc) {
   return alloc.Allocate(size);
 }

lib/tsan/sanitizer_common/sanitizer_common_interceptors.inc+878-736

...	@@ -21,19 +21,13 @@	...	@@ -21,19 +21,13 @@
21	// COMMON_INTERCEPTOR_FD_RELEASE	21	// COMMON_INTERCEPTOR_FD_RELEASE
22	// COMMON_INTERCEPTOR_FD_ACCESS	22	// COMMON_INTERCEPTOR_FD_ACCESS
23	// COMMON_INTERCEPTOR_SET_THREAD_NAME	23	// COMMON_INTERCEPTOR_SET_THREAD_NAME
24	// COMMON_INTERCEPTOR_ON_DLOPEN	24	// COMMON_INTERCEPTOR_DLOPEN
25	// COMMON_INTERCEPTOR_ON_EXIT	25	// COMMON_INTERCEPTOR_ON_EXIT
26	// COMMON_INTERCEPTOR_MUTEX_PRE_LOCK
27	// COMMON_INTERCEPTOR_MUTEX_POST_LOCK
28	// COMMON_INTERCEPTOR_MUTEX_UNLOCK
29	// COMMON_INTERCEPTOR_MUTEX_REPAIR
30	// COMMON_INTERCEPTOR_SET_PTHREAD_NAME	26	// COMMON_INTERCEPTOR_SET_PTHREAD_NAME
31	// COMMON_INTERCEPTOR_HANDLE_RECVMSG	27	// COMMON_INTERCEPTOR_HANDLE_RECVMSG
32	// COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED	28	// COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED
33	// COMMON_INTERCEPTOR_MEMSET_IMPL
34	// COMMON_INTERCEPTOR_MEMMOVE_IMPL
35	// COMMON_INTERCEPTOR_MEMCPY_IMPL
36	// COMMON_INTERCEPTOR_MMAP_IMPL	29	// COMMON_INTERCEPTOR_MMAP_IMPL
		30	// COMMON_INTERCEPTOR_MUNMAP_IMPL
37	// COMMON_INTERCEPTOR_COPY_STRING	31	// COMMON_INTERCEPTOR_COPY_STRING
38	// COMMON_INTERCEPTOR_STRNDUP_IMPL	32	// COMMON_INTERCEPTOR_STRNDUP_IMPL
39	// COMMON_INTERCEPTOR_STRERROR	33	// COMMON_INTERCEPTOR_STRERROR
...	@@ -132,14 +126,75 @@ extern const short *_toupper_tab_;	...	@@ -132,14 +126,75 @@ extern const short *_toupper_tab_;
132	extern const short *_tolower_tab_;	126	extern const short *_tolower_tab_;
133	#endif	127	#endif
134		128
135	// Platform-specific options.	129	#if SANITIZER_MUSL && \
136	#if SANITIZER_MAC	130	(defined(__i386__) \|\| defined(__arm__) \|\| SANITIZER_MIPS32 \|\| SANITIZER_PPC32)
137	#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0	131	// musl 1.2.0 on existing 32-bit architectures uses new symbol names for the
138	#elif SANITIZER_WINDOWS64	132	// time-related functions that take 64-bit time_t values. See
139	#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0	133	// https://musl.libc.org/time64.html
140	#else	134	#define adjtime __adjtime64
141	#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1	135	#define adjtimex __adjtimex_time64
142	#endif // SANITIZER_MAC	136	#define aio_suspend __aio_suspend_time64
		137	#define clock_adjtime __clock_adjtime64
		138	#define clock_getres __clock_getres_time64
		139	#define clock_gettime __clock_gettime64
		140	#define clock_nanosleep __clock_nanosleep_time64
		141	#define clock_settime __clock_settime64
		142	#define cnd_timedwait __cnd_timedwait_time64
		143	#define ctime __ctime64
		144	#define ctime_r __ctime64_r
		145	#define difftime __difftime64
		146	#define dlsym __dlsym_time64
		147	#define fstatat __fstatat_time64
		148	#define fstat __fstat_time64
		149	#define ftime __ftime64
		150	#define futimens __futimens_time64
		151	#define futimesat __futimesat_time64
		152	#define futimes __futimes_time64
		153	#define getitimer __getitimer_time64
		154	#define getrusage __getrusage_time64
		155	#define gettimeofday __gettimeofday_time64
		156	#define gmtime __gmtime64
		157	#define gmtime_r __gmtime64_r
		158	#define localtime __localtime64
		159	#define localtime_r __localtime64_r
		160	#define lstat __lstat_time64
		161	#define lutimes __lutimes_time64
		162	#define mktime __mktime64
		163	#define mq_timedreceive __mq_timedreceive_time64
		164	#define mq_timedsend __mq_timedsend_time64
		165	#define mtx_timedlock __mtx_timedlock_time64
		166	#define nanosleep __nanosleep_time64
		167	#define ppoll __ppoll_time64
		168	#define pselect __pselect_time64
		169	#define pthread_cond_timedwait __pthread_cond_timedwait_time64
		170	#define pthread_mutex_timedlock __pthread_mutex_timedlock_time64
		171	#define pthread_rwlock_timedrdlock __pthread_rwlock_timedrdlock_time64
		172	#define pthread_rwlock_timedwrlock __pthread_rwlock_timedwrlock_time64
		173	#define pthread_timedjoin_np __pthread_timedjoin_np_time64
		174	#define recvmmsg __recvmmsg_time64
		175	#define sched_rr_get_interval __sched_rr_get_interval_time64
		176	#define select __select_time64
		177	#define semtimedop __semtimedop_time64
		178	#define sem_timedwait __sem_timedwait_time64
		179	#define setitimer __setitimer_time64
		180	#define settimeofday __settimeofday_time64
		181	#define sigtimedwait __sigtimedwait_time64
		182	#define stat __stat_time64
		183	#define stime __stime64
		184	#define thrd_sleep __thrd_sleep_time64
		185	#define timegm __timegm_time64
		186	#define timerfd_gettime __timerfd_gettime64
		187	#define timerfd_settime __timerfd_settime64
		188	#define timer_gettime __timer_gettime64
		189	#define timer_settime __timer_settime64
		190	#define timespec_get __timespec_get_time64
		191	#define time __time64
		192	#define utimensat __utimensat_time64
		193	#define utimes __utimes_time64
		194	#define utime __utime64
		195	#define wait3 __wait3_time64
		196	#define wait4 __wait4_time64
		197	#endif
143		198
144	#ifndef COMMON_INTERCEPTOR_INITIALIZE_RANGE	199	#ifndef COMMON_INTERCEPTOR_INITIALIZE_RANGE
145	#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(p, size) {}	200	#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(p, size) {}
...	@@ -153,26 +208,6 @@ extern const short *_tolower_tab_;	...	@@ -153,26 +208,6 @@ extern const short *_tolower_tab_;
153	#define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) {}	208	#define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) {}
154	#endif	209	#endif
155		210
156	#ifndef COMMON_INTERCEPTOR_MUTEX_PRE_LOCK
157	#define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) {}
158	#endif
159
160	#ifndef COMMON_INTERCEPTOR_MUTEX_POST_LOCK
161	#define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) {}
162	#endif
163
164	#ifndef COMMON_INTERCEPTOR_MUTEX_UNLOCK
165	#define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) {}
166	#endif
167
168	#ifndef COMMON_INTERCEPTOR_MUTEX_REPAIR
169	#define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) {}
170	#endif
171
172	#ifndef COMMON_INTERCEPTOR_MUTEX_INVALID
173	#define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) {}
174	#endif
175
176	#ifndef COMMON_INTERCEPTOR_HANDLE_RECVMSG	211	#ifndef COMMON_INTERCEPTOR_HANDLE_RECVMSG
177	#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) ((void)(msg))	212	#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) ((void)(msg))
178	#endif	213	#endif
...	@@ -204,11 +239,11 @@ extern const short *_tolower_tab_;	...	@@ -204,11 +239,11 @@ extern const short *_tolower_tab_;
204		239
205	#define COMMON_INTERCEPTOR_READ_STRING(ctx, s, n) \	240	#define COMMON_INTERCEPTOR_READ_STRING(ctx, s, n) \
206	COMMON_INTERCEPTOR_READ_RANGE((ctx), (s), \	241	COMMON_INTERCEPTOR_READ_RANGE((ctx), (s), \
207	common_flags()->strict_string_checks ? (REAL(strlen)(s)) + 1 : (n) )	242	common_flags()->strict_string_checks ? (internal_strlen(s)) + 1 : (n) )
208		243
209	#ifndef COMMON_INTERCEPTOR_ON_DLOPEN	244	#ifndef COMMON_INTERCEPTOR_DLOPEN
210	#define COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag) \	245	#define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \
211	CheckNoDeepBind(filename, flag);	246	({ CheckNoDeepBind(filename, flag); REAL(dlopen)(filename, flag); })
212	#endif	247	#endif
213		248
214	#ifndef COMMON_INTERCEPTOR_GET_TLS_RANGE	249	#ifndef COMMON_INTERCEPTOR_GET_TLS_RANGE
...	@@ -256,53 +291,17 @@ extern const short *_tolower_tab_;	...	@@ -256,53 +291,17 @@ extern const short *_tolower_tab_;
256	COMMON_INTERCEPT_FUNCTION(fn)	291	COMMON_INTERCEPT_FUNCTION(fn)
257	#endif	292	#endif
258		293
259	#ifndef COMMON_INTERCEPTOR_MEMSET_IMPL
260	#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \
261	{ \
262	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \
263	return internal_memset(dst, v, size); \
264	COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \
265	if (common_flags()->intercept_intrin) \
266	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
267	return REAL(memset)(dst, v, size); \
268	}
269	#endif
270
271	#ifndef COMMON_INTERCEPTOR_MEMMOVE_IMPL
272	#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size) \
273	{ \
274	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \
275	return internal_memmove(dst, src, size); \
276	COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size); \
277	if (common_flags()->intercept_intrin) { \
278	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
279	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \
280	} \
281	return REAL(memmove)(dst, src, size); \
282	}
283	#endif
284
285	#ifndef COMMON_INTERCEPTOR_MEMCPY_IMPL
286	#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size) \
287	{ \
288	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) { \
289	return internal_memmove(dst, src, size); \
290	} \
291	COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size); \
292	if (common_flags()->intercept_intrin) { \
293	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
294	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \
295	} \
296	return REAL(memcpy)(dst, src, size); \
297	}
298	#endif
299
300	#ifndef COMMON_INTERCEPTOR_MMAP_IMPL	294	#ifndef COMMON_INTERCEPTOR_MMAP_IMPL
301	#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \	295	#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \
302	off) \	296	off) \
303	{ return REAL(mmap)(addr, sz, prot, flags, fd, off); }	297	{ return REAL(mmap)(addr, sz, prot, flags, fd, off); }
304	#endif	298	#endif
305		299
		300	#ifndef COMMON_INTERCEPTOR_MUNMAP_IMPL
		301	#define COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz) \
		302	{ return REAL(munmap)(addr, sz); }
		303	#endif
		304
306	#ifndef COMMON_INTERCEPTOR_COPY_STRING	305	#ifndef COMMON_INTERCEPTOR_COPY_STRING
307	#define COMMON_INTERCEPTOR_COPY_STRING(ctx, to, from, size) {}	306	#define COMMON_INTERCEPTOR_COPY_STRING(ctx, to, from, size) {}
308	#endif	307	#endif
...	@@ -315,9 +314,11 @@ extern const short *_tolower_tab_;	...	@@ -315,9 +314,11 @@ extern const short *_tolower_tab_;
315	if (common_flags()->intercept_strndup) { \	314	if (common_flags()->intercept_strndup) { \
316	COMMON_INTERCEPTOR_READ_STRING(ctx, s, Min(size, copy_length + 1)); \	315	COMMON_INTERCEPTOR_READ_STRING(ctx, s, Min(size, copy_length + 1)); \
317	} \	316	} \
318	COMMON_INTERCEPTOR_COPY_STRING(ctx, new_mem, s, copy_length); \	317	if (new_mem) { \
319	internal_memcpy(new_mem, s, copy_length); \	318	COMMON_INTERCEPTOR_COPY_STRING(ctx, new_mem, s, copy_length); \
320	new_mem[copy_length] = '\0'; \	319	internal_memcpy(new_mem, s, copy_length); \
		320	new_mem[copy_length] = '\0'; \
		321	} \
321	return new_mem;	322	return new_mem;
322	#endif	323	#endif
323		324
...	@@ -435,7 +436,7 @@ INTERCEPTOR(char, textdomain, const char domainname) {	...	@@ -435,7 +436,7 @@ INTERCEPTOR(char, textdomain, const char domainname) {
435	if (domainname) COMMON_INTERCEPTOR_READ_STRING(ctx, domainname, 0);	436	if (domainname) COMMON_INTERCEPTOR_READ_STRING(ctx, domainname, 0);
436	char *domain = REAL(textdomain)(domainname);	437	char *domain = REAL(textdomain)(domainname);
437	if (domain) {	438	if (domain) {
438	COMMON_INTERCEPTOR_INITIALIZE_RANGE(domain, REAL(strlen)(domain) + 1);	439	COMMON_INTERCEPTOR_INITIALIZE_RANGE(domain, internal_strlen(domain) + 1);
439	}	440	}
440	return domain;	441	return domain;
441	}	442	}
...	@@ -575,8 +576,8 @@ INTERCEPTOR(int, strncasecmp, const char s1, const char s2, SIZE_T size) {	...	@@ -575,8 +576,8 @@ INTERCEPTOR(int, strncasecmp, const char s1, const char s2, SIZE_T size) {
575	#if SANITIZER_INTERCEPT_STRSTR \|\| SANITIZER_INTERCEPT_STRCASESTR	576	#if SANITIZER_INTERCEPT_STRSTR \|\| SANITIZER_INTERCEPT_STRCASESTR
576	static inline void StrstrCheck(void ctx, char r, const char *s1,	577	static inline void StrstrCheck(void ctx, char r, const char *s1,
577	const char *s2) {	578	const char *s2) {
578	uptr len1 = REAL(strlen)(s1);	579	uptr len1 = internal_strlen(s1);
579	uptr len2 = REAL(strlen)(s2);	580	uptr len2 = internal_strlen(s2);
580	COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r ? r - s1 + len2 : len1 + 1);	581	COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r ? r - s1 + len2 : len1 + 1);
581	COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, len2 + 1);	582	COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, len2 + 1);
582	}	583	}
...	@@ -640,10 +641,10 @@ INTERCEPTOR(char, strtok, char str, const char *delimiters) {	...	@@ -640,10 +641,10 @@ INTERCEPTOR(char, strtok, char str, const char *delimiters) {
640	// for subsequent calls). We do not need to check strtok's result.	641	// for subsequent calls). We do not need to check strtok's result.
641	// As the delimiters can change, we check them every call.	642	// As the delimiters can change, we check them every call.
642	if (str != nullptr) {	643	if (str != nullptr) {
643	COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1);	644	COMMON_INTERCEPTOR_READ_RANGE(ctx, str, internal_strlen(str) + 1);
644	}	645	}
645	COMMON_INTERCEPTOR_READ_RANGE(ctx, delimiters,	646	COMMON_INTERCEPTOR_READ_RANGE(ctx, delimiters,
646	REAL(strlen)(delimiters) + 1);	647	internal_strlen(delimiters) + 1);
647	return REAL(strtok)(str, delimiters);	648	return REAL(strtok)(str, delimiters);
648	} else {	649	} else {
649	// However, when strict_string_checks is disabled we cannot check the	650	// However, when strict_string_checks is disabled we cannot check the
...	@@ -657,11 +658,11 @@ INTERCEPTOR(char, strtok, char str, const char *delimiters) {	...	@@ -657,11 +658,11 @@ INTERCEPTOR(char, strtok, char str, const char *delimiters) {
657	COMMON_INTERCEPTOR_READ_RANGE(ctx, delimiters, 1);	658	COMMON_INTERCEPTOR_READ_RANGE(ctx, delimiters, 1);
658	char *result = REAL(strtok)(str, delimiters);	659	char *result = REAL(strtok)(str, delimiters);
659	if (result != nullptr) {	660	if (result != nullptr) {
660	COMMON_INTERCEPTOR_READ_RANGE(ctx, result, REAL(strlen)(result) + 1);	661	COMMON_INTERCEPTOR_READ_RANGE(ctx, result, internal_strlen(result) + 1);
661	} else if (str != nullptr) {	662	} else if (str != nullptr) {
662	// No delimiter were found, it's safe to assume that the entire str was	663	// No delimiter were found, it's safe to assume that the entire str was
663	// scanned.	664	// scanned.
664	COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1);	665	COMMON_INTERCEPTOR_READ_RANGE(ctx, str, internal_strlen(str) + 1);
665	}	666	}
666	return result;	667	return result;
667	}	668	}
...	@@ -706,7 +707,7 @@ INTERCEPTOR(char, strchr, const char s, int c) {	...	@@ -706,7 +707,7 @@ INTERCEPTOR(char, strchr, const char s, int c) {
706	if (common_flags()->intercept_strchr) {	707	if (common_flags()->intercept_strchr) {
707	// Keep strlen as macro argument, as macro may ignore it.	708	// Keep strlen as macro argument, as macro may ignore it.
708	COMMON_INTERCEPTOR_READ_STRING(ctx, s,	709	COMMON_INTERCEPTOR_READ_STRING(ctx, s,
709	(result ? result - s : REAL(strlen)(s)) + 1);	710	(result ? result - s : internal_strlen(s)) + 1);
710	}	711	}
711	return result;	712	return result;
712	}	713	}
...	@@ -737,7 +738,7 @@ INTERCEPTOR(char, strrchr, const char s, int c) {	...	@@ -737,7 +738,7 @@ INTERCEPTOR(char, strrchr, const char s, int c) {
737	return internal_strrchr(s, c);	738	return internal_strrchr(s, c);
738	COMMON_INTERCEPTOR_ENTER(ctx, strrchr, s, c);	739	COMMON_INTERCEPTOR_ENTER(ctx, strrchr, s, c);
739	if (common_flags()->intercept_strchr)	740	if (common_flags()->intercept_strchr)
740	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);	741	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
741	return REAL(strrchr)(s, c);	742	return REAL(strrchr)(s, c);
742	}	743	}
743	#define INIT_STRRCHR COMMON_INTERCEPT_FUNCTION(strrchr)	744	#define INIT_STRRCHR COMMON_INTERCEPT_FUNCTION(strrchr)
...	@@ -751,7 +752,7 @@ INTERCEPTOR(SIZE_T, strspn, const char s1, const char s2) {	...	@@ -751,7 +752,7 @@ INTERCEPTOR(SIZE_T, strspn, const char s1, const char s2) {
751	COMMON_INTERCEPTOR_ENTER(ctx, strspn, s1, s2);	752	COMMON_INTERCEPTOR_ENTER(ctx, strspn, s1, s2);
752	SIZE_T r = REAL(strspn)(s1, s2);	753	SIZE_T r = REAL(strspn)(s1, s2);
753	if (common_flags()->intercept_strspn) {	754	if (common_flags()->intercept_strspn) {
754	COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, REAL(strlen)(s2) + 1);	755	COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, internal_strlen(s2) + 1);
755	COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r + 1);	756	COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r + 1);
756	}	757	}
757	return r;	758	return r;
...	@@ -762,7 +763,7 @@ INTERCEPTOR(SIZE_T, strcspn, const char s1, const char s2) {	...	@@ -762,7 +763,7 @@ INTERCEPTOR(SIZE_T, strcspn, const char s1, const char s2) {
762	COMMON_INTERCEPTOR_ENTER(ctx, strcspn, s1, s2);	763	COMMON_INTERCEPTOR_ENTER(ctx, strcspn, s1, s2);
763	SIZE_T r = REAL(strcspn)(s1, s2);	764	SIZE_T r = REAL(strcspn)(s1, s2);
764	if (common_flags()->intercept_strspn) {	765	if (common_flags()->intercept_strspn) {
765	COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, REAL(strlen)(s2) + 1);	766	COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, internal_strlen(s2) + 1);
766	COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r + 1);	767	COMMON_INTERCEPTOR_READ_STRING(ctx, s1, r + 1);
767	}	768	}
768	return r;	769	return r;
...	@@ -781,9 +782,9 @@ INTERCEPTOR(char , strpbrk, const char s1, const char *s2) {	...	@@ -781,9 +782,9 @@ INTERCEPTOR(char , strpbrk, const char s1, const char *s2) {
781	COMMON_INTERCEPTOR_ENTER(ctx, strpbrk, s1, s2);	782	COMMON_INTERCEPTOR_ENTER(ctx, strpbrk, s1, s2);
782	char *r = REAL(strpbrk)(s1, s2);	783	char *r = REAL(strpbrk)(s1, s2);
783	if (common_flags()->intercept_strpbrk) {	784	if (common_flags()->intercept_strpbrk) {
784	COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, REAL(strlen)(s2) + 1);	785	COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, internal_strlen(s2) + 1);
785	COMMON_INTERCEPTOR_READ_STRING(ctx, s1,	786	COMMON_INTERCEPTOR_READ_STRING(ctx, s1,
786	r ? r - s1 + 1 : REAL(strlen)(s1) + 1);	787	r ? r - s1 + 1 : internal_strlen(s1) + 1);
787	}	788	}
788	return r;	789	return r;
789	}	790	}
...	@@ -793,57 +794,6 @@ INTERCEPTOR(char , strpbrk, const char s1, const char *s2) {	...	@@ -793,57 +794,6 @@ INTERCEPTOR(char , strpbrk, const char s1, const char *s2) {
793	#define INIT_STRPBRK	794	#define INIT_STRPBRK
794	#endif	795	#endif
795		796
796	#if SANITIZER_INTERCEPT_MEMSET
797	INTERCEPTOR(void , memset, void dst, int v, uptr size) {
798	void *ctx;
799	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size);
800	}
801
802	#define INIT_MEMSET COMMON_INTERCEPT_FUNCTION(memset)
803	#else
804	#define INIT_MEMSET
805	#endif
806
807	#if SANITIZER_INTERCEPT_MEMMOVE
808	INTERCEPTOR(void , memmove, void dst, const void *src, uptr size) {
809	void *ctx;
810	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
811	}
812
813	#define INIT_MEMMOVE COMMON_INTERCEPT_FUNCTION(memmove)
814	#else
815	#define INIT_MEMMOVE
816	#endif
817
818	#if SANITIZER_INTERCEPT_MEMCPY
819	INTERCEPTOR(void , memcpy, void dst, const void *src, uptr size) {
820	// On OS X, calling internal_memcpy here will cause memory corruptions,
821	// because memcpy and memmove are actually aliases of the same
822	// implementation. We need to use internal_memmove here.
823	// N.B.: If we switch this to internal_ we'll have to use internal_memmove
824	// due to memcpy being an alias of memmove on OS X.
825	void *ctx;
826	#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE
827	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
828	#else
829	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
830	#endif
831	}
832
833	#define INIT_MEMCPY \
834	do { \
835	if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { \
836	COMMON_INTERCEPT_FUNCTION(memcpy); \
837	} else { \
838	ASSIGN_REAL(memcpy, memmove); \
839	} \
840	CHECK(REAL(memcpy)); \
841	} while (false)
842
843	#else
844	#define INIT_MEMCPY
845	#endif
846
847	#if SANITIZER_INTERCEPT_MEMCMP	797	#if SANITIZER_INTERCEPT_MEMCMP
848	DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc,	798	DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc,
849	const void s1, const void s2, uptr n,	799	const void s1, const void s2, uptr n,
...	@@ -1251,7 +1201,7 @@ INTERCEPTOR(char , fgets, char s, SIZE_T size, void *file) {	...	@@ -1251,7 +1201,7 @@ INTERCEPTOR(char , fgets, char s, SIZE_T size, void *file) {
1251	// https://github.com/google/sanitizers/issues/321.	1201	// https://github.com/google/sanitizers/issues/321.
1252	char *res = REAL(fgets)(s, size, file);	1202	char *res = REAL(fgets)(s, size, file);
1253	if (res)	1203	if (res)
1254	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1);	1204	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, internal_strlen(s) + 1);
1255	return res;	1205	return res;
1256	}	1206	}
1257	#define INIT_FGETS COMMON_INTERCEPT_FUNCTION(fgets)	1207	#define INIT_FGETS COMMON_INTERCEPT_FUNCTION(fgets)
...	@@ -1264,8 +1214,8 @@ INTERCEPTOR_WITH_SUFFIX(int, fputs, char s, void file) {	...	@@ -1264,8 +1214,8 @@ INTERCEPTOR_WITH_SUFFIX(int, fputs, char s, void file) {
1264	// libc file streams can call user-supplied functions, see fopencookie.	1214	// libc file streams can call user-supplied functions, see fopencookie.
1265	void *ctx;	1215	void *ctx;
1266	COMMON_INTERCEPTOR_ENTER(ctx, fputs, s, file);	1216	COMMON_INTERCEPTOR_ENTER(ctx, fputs, s, file);
1267	if (!SANITIZER_MAC \|\| s) { // `fputs(NULL, file)` is supported on Darwin.	1217	if (!SANITIZER_APPLE \|\| s) { // `fputs(NULL, file)` is supported on Darwin.
1268	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);	1218	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
1269	}	1219	}
1270	return REAL(fputs)(s, file);	1220	return REAL(fputs)(s, file);
1271	}	1221	}
...	@@ -1279,8 +1229,8 @@ INTERCEPTOR(int, puts, char *s) {	...	@@ -1279,8 +1229,8 @@ INTERCEPTOR(int, puts, char *s) {
1279	// libc file streams can call user-supplied functions, see fopencookie.	1229	// libc file streams can call user-supplied functions, see fopencookie.
1280	void *ctx;	1230	void *ctx;
1281	COMMON_INTERCEPTOR_ENTER(ctx, puts, s);	1231	COMMON_INTERCEPTOR_ENTER(ctx, puts, s);
1282	if (!SANITIZER_MAC \|\| s) { // `puts(NULL)` is supported on Darwin.	1232	if (!SANITIZER_APPLE \|\| s) { // `puts(NULL)` is supported on Darwin.
1283	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);	1233	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
1284	}	1234	}
1285	return REAL(puts)(s);	1235	return REAL(puts)(s);
1286	}	1236	}
...	@@ -1295,12 +1245,21 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,	...	@@ -1295,12 +1245,21 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
1295	void *ctx;	1245	void *ctx;
1296	COMMON_INTERCEPTOR_ENTER(ctx, prctl, option, arg2, arg3, arg4, arg5);	1246	COMMON_INTERCEPTOR_ENTER(ctx, prctl, option, arg2, arg3, arg4, arg5);
1297	static const int PR_SET_NAME = 15;	1247	static const int PR_SET_NAME = 15;
1298	int res = REAL(prctl(option, arg2, arg3, arg4, arg5));	1248	static const int PR_SET_VMA = 0x53564d41;
		1249	static const int PR_SCHED_CORE = 62;
		1250	static const int PR_SCHED_CORE_GET = 0;
		1251	if (option == PR_SET_VMA && arg2 == 0UL) {
		1252	char name = (char )arg5;
		1253	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
		1254	}
		1255	int res = REAL(prctl)(option, arg2, arg3, arg4, arg5);
1299	if (option == PR_SET_NAME) {	1256	if (option == PR_SET_NAME) {
1300	char buff[16];	1257	char buff[16];
1301	internal_strncpy(buff, (char *)arg2, 15);	1258	internal_strncpy(buff, (char *)arg2, 15);
1302	buff[15] = 0;	1259	buff[15] = 0;
1303	COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, buff);	1260	COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, buff);
		1261	} else if (res != -1 && option == PR_SCHED_CORE && arg2 == PR_SCHED_CORE_GET) {
		1262	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64*)(arg5), sizeof(u64));
1304	}	1263	}
1305	return res;	1264	return res;
1306	}	1265	}
...	@@ -1334,7 +1293,7 @@ static void unpoison_tm(void ctx, __sanitizer_tm tm) {	...	@@ -1334,7 +1293,7 @@ static void unpoison_tm(void ctx, __sanitizer_tm tm) {
1334	// Can not use COMMON_INTERCEPTOR_WRITE_RANGE here, because tm->tm_zone	1293	// Can not use COMMON_INTERCEPTOR_WRITE_RANGE here, because tm->tm_zone
1335	// can point to shared memory and tsan would report a data race.	1294	// can point to shared memory and tsan would report a data race.
1336	COMMON_INTERCEPTOR_INITIALIZE_RANGE(tm->tm_zone,	1295	COMMON_INTERCEPTOR_INITIALIZE_RANGE(tm->tm_zone,
1337	REAL(strlen(tm->tm_zone)) + 1);	1296	internal_strlen(tm->tm_zone) + 1);
1338	}	1297	}
1339	#endif	1298	#endif
1340	}	1299	}
...	@@ -1387,7 +1346,7 @@ INTERCEPTOR(char , ctime, unsigned long timep) {	...	@@ -1387,7 +1346,7 @@ INTERCEPTOR(char , ctime, unsigned long timep) {
1387	char *res = REAL(ctime)(timep);	1346	char *res = REAL(ctime)(timep);
1388	if (res) {	1347	if (res) {
1389	COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep));	1348	COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep));
1390	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);	1349	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
1391	}	1350	}
1392	return res;	1351	return res;
1393	}	1352	}
...	@@ -1400,7 +1359,7 @@ INTERCEPTOR(char , ctime_r, unsigned long timep, char *result) {	...	@@ -1400,7 +1359,7 @@ INTERCEPTOR(char , ctime_r, unsigned long timep, char *result) {
1400	char *res = REAL(ctime_r)(timep, result);	1359	char *res = REAL(ctime_r)(timep, result);
1401	if (res) {	1360	if (res) {
1402	COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep));	1361	COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep));
1403	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);	1362	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
1404	}	1363	}
1405	return res;	1364	return res;
1406	}	1365	}
...	@@ -1413,7 +1372,7 @@ INTERCEPTOR(char , asctime, __sanitizer_tm tm) {	...	@@ -1413,7 +1372,7 @@ INTERCEPTOR(char , asctime, __sanitizer_tm tm) {
1413	char *res = REAL(asctime)(tm);	1372	char *res = REAL(asctime)(tm);
1414	if (res) {	1373	if (res) {
1415	COMMON_INTERCEPTOR_READ_RANGE(ctx, tm, sizeof(*tm));	1374	COMMON_INTERCEPTOR_READ_RANGE(ctx, tm, sizeof(*tm));
1416	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);	1375	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
1417	}	1376	}
1418	return res;	1377	return res;
1419	}	1378	}
...	@@ -1426,7 +1385,7 @@ INTERCEPTOR(char , asctime_r, __sanitizer_tm tm, char *result) {	...	@@ -1426,7 +1385,7 @@ INTERCEPTOR(char , asctime_r, __sanitizer_tm tm, char *result) {
1426	char *res = REAL(asctime_r)(tm, result);	1385	char *res = REAL(asctime_r)(tm, result);
1427	if (res) {	1386	if (res) {
1428	COMMON_INTERCEPTOR_READ_RANGE(ctx, tm, sizeof(*tm));	1387	COMMON_INTERCEPTOR_READ_RANGE(ctx, tm, sizeof(*tm));
1429	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);	1388	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
1430	}	1389	}
1431	return res;	1390	return res;
1432	}	1391	}
...	@@ -1463,7 +1422,7 @@ INTERCEPTOR(char , strptime, char s, char format, __sanitizer_tm tm) {	...	@@ -1463,7 +1422,7 @@ INTERCEPTOR(char , strptime, char s, char format, __sanitizer_tm tm) {
1463	void *ctx;	1422	void *ctx;
1464	COMMON_INTERCEPTOR_ENTER(ctx, strptime, s, format, tm);	1423	COMMON_INTERCEPTOR_ENTER(ctx, strptime, s, format, tm);
1465	if (format)	1424	if (format)
1466	COMMON_INTERCEPTOR_READ_RANGE(ctx, format, REAL(strlen)(format) + 1);	1425	COMMON_INTERCEPTOR_READ_RANGE(ctx, format, internal_strlen(format) + 1);
1467	// FIXME: under ASan the call below may write to freed memory and corrupt	1426	// FIXME: under ASan the call below may write to freed memory and corrupt
1468	// its metadata. See	1427	// its metadata. See
1469	// https://github.com/google/sanitizers/issues/321.	1428	// https://github.com/google/sanitizers/issues/321.
...	@@ -1532,6 +1491,16 @@ VSCANF_INTERCEPTOR_IMPL(__isoc99_vsscanf, false, str, format, ap)	...	@@ -1532,6 +1491,16 @@ VSCANF_INTERCEPTOR_IMPL(__isoc99_vsscanf, false, str, format, ap)
1532		1491
1533	INTERCEPTOR(int, __isoc99_vfscanf, void stream, const char format, va_list ap)	1492	INTERCEPTOR(int, __isoc99_vfscanf, void stream, const char format, va_list ap)
1534	VSCANF_INTERCEPTOR_IMPL(__isoc99_vfscanf, false, stream, format, ap)	1493	VSCANF_INTERCEPTOR_IMPL(__isoc99_vfscanf, false, stream, format, ap)
		1494
		1495	INTERCEPTOR(int, __isoc23_vscanf, const char *format, va_list ap)
		1496	VSCANF_INTERCEPTOR_IMPL(__isoc23_vscanf, false, format, ap)
		1497
		1498	INTERCEPTOR(int, __isoc23_vsscanf, const char str, const char format,
		1499	va_list ap)
		1500	VSCANF_INTERCEPTOR_IMPL(__isoc23_vsscanf, false, str, format, ap)
		1501
		1502	INTERCEPTOR(int, __isoc23_vfscanf, void stream, const char format, va_list ap)
		1503	VSCANF_INTERCEPTOR_IMPL(__isoc23_vfscanf, false, stream, format, ap)
1535	#endif // SANITIZER_INTERCEPT_ISOC99_SCANF	1504	#endif // SANITIZER_INTERCEPT_ISOC99_SCANF
1536		1505
1537	INTERCEPTOR(int, scanf, const char *format, ...)	1506	INTERCEPTOR(int, scanf, const char *format, ...)
...	@@ -1552,6 +1521,15 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_fscanf, __isoc99_vfscanf, stream, format)	...	@@ -1552,6 +1521,15 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_fscanf, __isoc99_vfscanf, stream, format)
1552		1521
1553	INTERCEPTOR(int, __isoc99_sscanf, const char str, const char format, ...)	1522	INTERCEPTOR(int, __isoc99_sscanf, const char str, const char format, ...)
1554	FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format)	1523	FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format)
		1524
		1525	INTERCEPTOR(int, __isoc23_scanf, const char *format, ...)
		1526	FORMAT_INTERCEPTOR_IMPL(__isoc23_scanf, __isoc23_vscanf, format)
		1527
		1528	INTERCEPTOR(int, __isoc23_fscanf, void stream, const char format, ...)
		1529	FORMAT_INTERCEPTOR_IMPL(__isoc23_fscanf, __isoc23_vfscanf, stream, format)
		1530
		1531	INTERCEPTOR(int, __isoc23_sscanf, const char str, const char format, ...)
		1532	FORMAT_INTERCEPTOR_IMPL(__isoc23_sscanf, __isoc23_vsscanf, str, format)
1555	#endif	1533	#endif
1556		1534
1557	#endif	1535	#endif
...	@@ -1575,7 +1553,13 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format)	...	@@ -1575,7 +1553,13 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format)
1575	COMMON_INTERCEPT_FUNCTION(__isoc99_fscanf); \	1553	COMMON_INTERCEPT_FUNCTION(__isoc99_fscanf); \
1576	COMMON_INTERCEPT_FUNCTION(__isoc99_vscanf); \	1554	COMMON_INTERCEPT_FUNCTION(__isoc99_vscanf); \
1577	COMMON_INTERCEPT_FUNCTION(__isoc99_vsscanf); \	1555	COMMON_INTERCEPT_FUNCTION(__isoc99_vsscanf); \
1578	COMMON_INTERCEPT_FUNCTION(__isoc99_vfscanf);	1556	COMMON_INTERCEPT_FUNCTION(__isoc99_vfscanf); \
		1557	COMMON_INTERCEPT_FUNCTION(__isoc23_scanf); \
		1558	COMMON_INTERCEPT_FUNCTION(__isoc23_sscanf); \
		1559	COMMON_INTERCEPT_FUNCTION(__isoc23_fscanf); \
		1560	COMMON_INTERCEPT_FUNCTION(__isoc23_vscanf); \
		1561	COMMON_INTERCEPT_FUNCTION(__isoc23_vsscanf); \
		1562	COMMON_INTERCEPT_FUNCTION(__isoc23_vfscanf);
1579	#else	1563	#else
1580	#define INIT_ISOC99_SCANF	1564	#define INIT_ISOC99_SCANF
1581	#endif	1565	#endif
...	@@ -1843,9 +1827,9 @@ INTERCEPTOR(int, ioctl, int d, unsigned long request, ...) {	...	@@ -1843,9 +1827,9 @@ INTERCEPTOR(int, ioctl, int d, unsigned long request, ...) {
1843	const ioctl_desc *desc = ioctl_lookup(request);	1827	const ioctl_desc *desc = ioctl_lookup(request);
1844	ioctl_desc decoded_desc;	1828	ioctl_desc decoded_desc;
1845	if (!desc) {	1829	if (!desc) {
1846	VPrintf(2, "Decoding unknown ioctl 0x%x\n", request);	1830	VPrintf(2, "Decoding unknown ioctl 0x%lx\n", request);
1847	if (!ioctl_decode(request, &decoded_desc))	1831	if (!ioctl_decode(request, &decoded_desc))
1848	Printf("WARNING: failed decoding unknown ioctl 0x%x\n", request);	1832	Printf("WARNING: failed decoding unknown ioctl 0x%lx\n", request);
1849	else	1833	else
1850	desc = &decoded_desc;	1834	desc = &decoded_desc;
1851	}	1835	}
...	@@ -1869,26 +1853,26 @@ UNUSED static void unpoison_passwd(void ctx, __sanitizer_passwd pwd) {	...	@@ -1869,26 +1853,26 @@ UNUSED static void unpoison_passwd(void ctx, __sanitizer_passwd pwd) {
1869	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd, sizeof(*pwd));	1853	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd, sizeof(*pwd));
1870	if (pwd->pw_name)	1854	if (pwd->pw_name)
1871	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_name,	1855	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_name,
1872	REAL(strlen)(pwd->pw_name) + 1);	1856	internal_strlen(pwd->pw_name) + 1);
1873	if (pwd->pw_passwd)	1857	if (pwd->pw_passwd)
1874	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_passwd,	1858	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_passwd,
1875	REAL(strlen)(pwd->pw_passwd) + 1);	1859	internal_strlen(pwd->pw_passwd) + 1);
1876	#if !SANITIZER_ANDROID	1860	#if !SANITIZER_ANDROID
1877	if (pwd->pw_gecos)	1861	if (pwd->pw_gecos)
1878	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_gecos,	1862	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_gecos,
1879	REAL(strlen)(pwd->pw_gecos) + 1);	1863	internal_strlen(pwd->pw_gecos) + 1);
1880	#endif	1864	#endif
1881	#if SANITIZER_MAC \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD	1865	#if SANITIZER_APPLE \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
1882	if (pwd->pw_class)	1866	if (pwd->pw_class)
1883	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_class,	1867	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_class,
1884	REAL(strlen)(pwd->pw_class) + 1);	1868	internal_strlen(pwd->pw_class) + 1);
1885	#endif	1869	#endif
1886	if (pwd->pw_dir)	1870	if (pwd->pw_dir)
1887	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_dir,	1871	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_dir,
1888	REAL(strlen)(pwd->pw_dir) + 1);	1872	internal_strlen(pwd->pw_dir) + 1);
1889	if (pwd->pw_shell)	1873	if (pwd->pw_shell)
1890	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_shell,	1874	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_shell,
1891	REAL(strlen)(pwd->pw_shell) + 1);	1875	internal_strlen(pwd->pw_shell) + 1);
1892	}	1876	}
1893	}	1877	}
1894		1878
...	@@ -1897,13 +1881,13 @@ UNUSED static void unpoison_group(void ctx, __sanitizer_group grp) {	...	@@ -1897,13 +1881,13 @@ UNUSED static void unpoison_group(void ctx, __sanitizer_group grp) {
1897	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp, sizeof(*grp));	1881	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp, sizeof(*grp));
1898	if (grp->gr_name)	1882	if (grp->gr_name)
1899	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_name,	1883	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_name,
1900	REAL(strlen)(grp->gr_name) + 1);	1884	internal_strlen(grp->gr_name) + 1);
1901	if (grp->gr_passwd)	1885	if (grp->gr_passwd)
1902	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_passwd,	1886	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_passwd,
1903	REAL(strlen)(grp->gr_passwd) + 1);	1887	internal_strlen(grp->gr_passwd) + 1);
1904	char **p = grp->gr_mem;	1888	char **p = grp->gr_mem;
1905	for (; *p; ++p) {	1889	for (; *p; ++p) {
1906	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, REAL(strlen)(p) + 1);	1890	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, internal_strlen(p) + 1);
1907	}	1891	}
1908	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_mem,	1892	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, grp->gr_mem,
1909	(p - grp->gr_mem + 1) * sizeof(*p));	1893	(p - grp->gr_mem + 1) * sizeof(*p));
...	@@ -1916,7 +1900,7 @@ INTERCEPTOR(__sanitizer_passwd , getpwnam, const char name) {	...	@@ -1916,7 +1900,7 @@ INTERCEPTOR(__sanitizer_passwd , getpwnam, const char name) {
1916	void *ctx;	1900	void *ctx;
1917	COMMON_INTERCEPTOR_ENTER(ctx, getpwnam, name);	1901	COMMON_INTERCEPTOR_ENTER(ctx, getpwnam, name);
1918	if (name)	1902	if (name)
1919	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);	1903	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
1920	__sanitizer_passwd *res = REAL(getpwnam)(name);	1904	__sanitizer_passwd *res = REAL(getpwnam)(name);
1921	unpoison_passwd(ctx, res);	1905	unpoison_passwd(ctx, res);
1922	return res;	1906	return res;
...	@@ -1931,7 +1915,7 @@ INTERCEPTOR(__sanitizer_passwd *, getpwuid, u32 uid) {	...	@@ -1931,7 +1915,7 @@ INTERCEPTOR(__sanitizer_passwd *, getpwuid, u32 uid) {
1931	INTERCEPTOR(__sanitizer_group , getgrnam, const char name) {	1915	INTERCEPTOR(__sanitizer_group , getgrnam, const char name) {
1932	void *ctx;	1916	void *ctx;
1933	COMMON_INTERCEPTOR_ENTER(ctx, getgrnam, name);	1917	COMMON_INTERCEPTOR_ENTER(ctx, getgrnam, name);
1934	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);	1918	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
1935	__sanitizer_group *res = REAL(getgrnam)(name);	1919	__sanitizer_group *res = REAL(getgrnam)(name);
1936	unpoison_group(ctx, res);	1920	unpoison_group(ctx, res);
1937	return res;	1921	return res;
...	@@ -1957,7 +1941,7 @@ INTERCEPTOR(int, getpwnam_r, const char name, __sanitizer_passwd pwd,	...	@@ -1957,7 +1941,7 @@ INTERCEPTOR(int, getpwnam_r, const char name, __sanitizer_passwd pwd,
1957	char buf, SIZE_T buflen, __sanitizer_passwd *result) {	1941	char buf, SIZE_T buflen, __sanitizer_passwd *result) {
1958	void *ctx;	1942	void *ctx;
1959	COMMON_INTERCEPTOR_ENTER(ctx, getpwnam_r, name, pwd, buf, buflen, result);	1943	COMMON_INTERCEPTOR_ENTER(ctx, getpwnam_r, name, pwd, buf, buflen, result);
1960	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);	1944	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
1961	// FIXME: under ASan the call below may write to freed memory and corrupt	1945	// FIXME: under ASan the call below may write to freed memory and corrupt
1962	// its metadata. See	1946	// its metadata. See
1963	// https://github.com/google/sanitizers/issues/321.	1947	// https://github.com/google/sanitizers/issues/321.
...	@@ -1984,7 +1968,7 @@ INTERCEPTOR(int, getgrnam_r, const char name, __sanitizer_group grp,	...	@@ -1984,7 +1968,7 @@ INTERCEPTOR(int, getgrnam_r, const char name, __sanitizer_group grp,
1984	char buf, SIZE_T buflen, __sanitizer_group *result) {	1968	char buf, SIZE_T buflen, __sanitizer_group *result) {
1985	void *ctx;	1969	void *ctx;
1986	COMMON_INTERCEPTOR_ENTER(ctx, getgrnam_r, name, grp, buf, buflen, result);	1970	COMMON_INTERCEPTOR_ENTER(ctx, getgrnam_r, name, grp, buf, buflen, result);
1987	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);	1971	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
1988	// FIXME: under ASan the call below may write to freed memory and corrupt	1972	// FIXME: under ASan the call below may write to freed memory and corrupt
1989	// its metadata. See	1973	// its metadata. See
1990	// https://github.com/google/sanitizers/issues/321.	1974	// https://github.com/google/sanitizers/issues/321.
...	@@ -2229,8 +2213,20 @@ INTERCEPTOR(int, clock_getcpuclockid, pid_t pid,	...	@@ -2229,8 +2213,20 @@ INTERCEPTOR(int, clock_getcpuclockid, pid_t pid,
2229	return res;	2213	return res;
2230	}	2214	}
2231		2215
2232	#define INIT_CLOCK_GETCPUCLOCKID \	2216	INTERCEPTOR(int, pthread_getcpuclockid, uptr thread,
2233	COMMON_INTERCEPT_FUNCTION(clock_getcpuclockid);	2217	__sanitizer_clockid_t *clockid) {
		2218	void *ctx;
		2219	COMMON_INTERCEPTOR_ENTER(ctx, pthread_getcpuclockid, thread, clockid);
		2220	int res = REAL(pthread_getcpuclockid)(thread, clockid);
		2221	if (!res && clockid) {
		2222	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, clockid, sizeof *clockid);
		2223	}
		2224	return res;
		2225	}
		2226
		2227	#define INIT_CLOCK_GETCPUCLOCKID \
		2228	COMMON_INTERCEPT_FUNCTION(clock_getcpuclockid); \
		2229	COMMON_INTERCEPT_FUNCTION(pthread_getcpuclockid);
2234	#else	2230	#else
2235	#define INIT_CLOCK_GETCPUCLOCKID	2231	#define INIT_CLOCK_GETCPUCLOCKID
2236	#endif	2232	#endif
...	@@ -2289,7 +2285,7 @@ static void unpoison_glob_t(void ctx, __sanitizer_glob_t pglob) {	...	@@ -2289,7 +2285,7 @@ static void unpoison_glob_t(void ctx, __sanitizer_glob_t pglob) {
2289	ctx, pglob->gl_pathv, (pglob->gl_pathc + 1) * sizeof(*pglob->gl_pathv));	2285	ctx, pglob->gl_pathv, (pglob->gl_pathc + 1) * sizeof(*pglob->gl_pathv));
2290	for (SIZE_T i = 0; i < pglob->gl_pathc; ++i) {	2286	for (SIZE_T i = 0; i < pglob->gl_pathc; ++i) {
2291	char *p = pglob->gl_pathv[i];	2287	char *p = pglob->gl_pathv[i];
2292	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, REAL(strlen)(p) + 1);	2288	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, internal_strlen(p) + 1);
2293	}	2289	}
2294	}	2290	}
2295		2291
...	@@ -2319,19 +2315,19 @@ static void wrapped_gl_readdir(void dir) {	...	@@ -2319,19 +2315,19 @@ static void wrapped_gl_readdir(void dir) {
2319		2315
2320	static void wrapped_gl_opendir(const char s) {	2316	static void wrapped_gl_opendir(const char s) {
2321	COMMON_INTERCEPTOR_UNPOISON_PARAM(1);	2317	COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
2322	COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, REAL(strlen)(s) + 1);	2318	COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, internal_strlen(s) + 1);
2323	return pglob_copy->gl_opendir(s);	2319	return pglob_copy->gl_opendir(s);
2324	}	2320	}
2325		2321
2326	static int wrapped_gl_lstat(const char s, void st) {	2322	static int wrapped_gl_lstat(const char s, void st) {
2327	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);	2323	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
2328	COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, REAL(strlen)(s) + 1);	2324	COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, internal_strlen(s) + 1);
2329	return pglob_copy->gl_lstat(s, st);	2325	return pglob_copy->gl_lstat(s, st);
2330	}	2326	}
2331		2327
2332	static int wrapped_gl_stat(const char s, void st) {	2328	static int wrapped_gl_stat(const char s, void st) {
2333	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);	2329	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
2334	COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, REAL(strlen)(s) + 1);	2330	COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, internal_strlen(s) + 1);
2335	return pglob_copy->gl_stat(s, st);	2331	return pglob_copy->gl_stat(s, st);
2336	}	2332	}
2337		2333
...	@@ -2410,6 +2406,136 @@ INTERCEPTOR(int, glob64, const char *pattern, int flags,	...	@@ -2410,6 +2406,136 @@ INTERCEPTOR(int, glob64, const char *pattern, int flags,
2410	#define INIT_GLOB64	2406	#define INIT_GLOB64
2411	#endif // SANITIZER_INTERCEPT_GLOB64	2407	#endif // SANITIZER_INTERCEPT_GLOB64
2412		2408
		2409	#if SANITIZER_INTERCEPT___B64_TO
		2410	INTERCEPTOR(int, __b64_ntop, unsigned char const *src, SIZE_T srclength,
		2411	char *target, SIZE_T targsize) {
		2412	void *ctx;
		2413	COMMON_INTERCEPTOR_ENTER(ctx, __b64_ntop, src, srclength, target, targsize);
		2414	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, srclength);
		2415	int res = REAL(__b64_ntop)(src, srclength, target, targsize);
		2416	if (res >= 0)
		2417	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, target, res + 1);
		2418	return res;
		2419	}
		2420	INTERCEPTOR(int, __b64_pton, char const src, char target, SIZE_T targsize) {
		2421	void *ctx;
		2422	COMMON_INTERCEPTOR_ENTER(ctx, __b64_pton, src, target, targsize);
		2423	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
		2424	int res = REAL(__b64_pton)(src, target, targsize);
		2425	if (res >= 0)
		2426	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, target, res);
		2427	return res;
		2428	}
		2429	#define INIT___B64_TO \
		2430	COMMON_INTERCEPT_FUNCTION(__b64_ntop); \
		2431	COMMON_INTERCEPT_FUNCTION(__b64_pton);
		2432	#else // SANITIZER_INTERCEPT___B64_TO
		2433	#define INIT___B64_TO
		2434	#endif // SANITIZER_INTERCEPT___B64_TO
		2435
		2436	#if SANITIZER_INTERCEPT_DN_COMP_EXPAND
		2437	# if __GLIBC_PREREQ(2, 34)
		2438	// Changed with https://sourceware.org/git/?p=glibc.git;h=640bbdf
		2439	# define DN_COMP_INTERCEPTOR_NAME dn_comp
		2440	# define DN_EXPAND_INTERCEPTOR_NAME dn_expand
		2441	# else
		2442	# define DN_COMP_INTERCEPTOR_NAME __dn_comp
		2443	# define DN_EXPAND_INTERCEPTOR_NAME __dn_expand
		2444	# endif
		2445	INTERCEPTOR(int, DN_COMP_INTERCEPTOR_NAME, unsigned char *exp_dn,
		2446	unsigned char comp_dn, int length, unsigned char *dnptrs,
		2447	unsigned char **lastdnptr) {
		2448	void *ctx;
		2449	COMMON_INTERCEPTOR_ENTER(ctx, DN_COMP_INTERCEPTOR_NAME, exp_dn, comp_dn,
		2450	length, dnptrs, lastdnptr);
		2451	int res = REAL(DN_COMP_INTERCEPTOR_NAME)(exp_dn, comp_dn, length, dnptrs,
		2452	lastdnptr);
		2453	if (res >= 0) {
		2454	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, comp_dn, res);
		2455	if (dnptrs && lastdnptr) {
		2456	unsigned char **p = dnptrs;
		2457	for (; p != lastdnptr && *p; ++p)
		2458	;
		2459	if (p != lastdnptr)
		2460	++p;
		2461	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dnptrs, (p - dnptrs) * sizeof(*p));
		2462	}
		2463	}
		2464	return res;
		2465	}
		2466	INTERCEPTOR(int, DN_EXPAND_INTERCEPTOR_NAME, unsigned char const *base,
		2467	unsigned char const end, unsigned char const src, char *dest,
		2468	int space) {
		2469	void *ctx;
		2470	COMMON_INTERCEPTOR_ENTER(ctx, DN_EXPAND_INTERCEPTOR_NAME, base, end, src,
		2471	dest, space);
		2472	// TODO: add read check if __dn_comp intercept added
		2473	int res = REAL(DN_EXPAND_INTERCEPTOR_NAME)(base, end, src, dest, space);
		2474	if (res >= 0)
		2475	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, internal_strlen(dest) + 1);
		2476	return res;
		2477	}
		2478	# define INIT_DN_COMP_EXPAND \
		2479	COMMON_INTERCEPT_FUNCTION(DN_COMP_INTERCEPTOR_NAME); \
		2480	COMMON_INTERCEPT_FUNCTION(DN_EXPAND_INTERCEPTOR_NAME);
		2481	#else // SANITIZER_INTERCEPT_DN_COMP_EXPAND
		2482	# define INIT_DN_COMP_EXPAND
		2483	#endif // SANITIZER_INTERCEPT_DN_COMP_EXPAND
		2484
		2485	#if SANITIZER_INTERCEPT_POSIX_SPAWN
		2486
		2487	template <class RealSpawnPtr>
		2488	static int PosixSpawnImpl(void ctx, RealSpawnPtr real_posix_spawn, pid_t *pid,
		2489	const char file_or_path, const void file_actions,
		2490	const void attrp, char const argv[],
		2491	char *const envp[]) {
		2492	COMMON_INTERCEPTOR_READ_RANGE(ctx, file_or_path,
		2493	internal_strlen(file_or_path) + 1);
		2494	if (argv) {
		2495	for (char const s = argv; ; ++s) {
		2496	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(*s));
		2497	if (!*s) break;
		2498	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
		2499	}
		2500	}
		2501	if (envp) {
		2502	for (char const s = envp; ; ++s) {
		2503	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(*s));
		2504	if (!*s) break;
		2505	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
		2506	}
		2507	}
		2508	int res =
		2509	real_posix_spawn(pid, file_or_path, file_actions, attrp, argv, envp);
		2510	if (res == 0)
		2511	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pid, sizeof(*pid));
		2512	return res;
		2513	}
		2514	INTERCEPTOR(int, posix_spawn, pid_t pid, const char path,
		2515	const void file_actions, const void attrp, char *const argv[],
		2516	char *const envp[]) {
		2517	void *ctx;
		2518	COMMON_INTERCEPTOR_ENTER(ctx, posix_spawn, pid, path, file_actions, attrp,
		2519	argv, envp);
		2520	return PosixSpawnImpl(ctx, REAL(posix_spawn), pid, path, file_actions, attrp,
		2521	argv, envp);
		2522	}
		2523	INTERCEPTOR(int, posix_spawnp, pid_t pid, const char file,
		2524	const void file_actions, const void attrp, char *const argv[],
		2525	char *const envp[]) {
		2526	void *ctx;
		2527	COMMON_INTERCEPTOR_ENTER(ctx, posix_spawnp, pid, file, file_actions, attrp,
		2528	argv, envp);
		2529	return PosixSpawnImpl(ctx, REAL(posix_spawnp), pid, file, file_actions, attrp,
		2530	argv, envp);
		2531	}
		2532	# define INIT_POSIX_SPAWN \
		2533	COMMON_INTERCEPT_FUNCTION(posix_spawn); \
		2534	COMMON_INTERCEPT_FUNCTION(posix_spawnp);
		2535	#else // SANITIZER_INTERCEPT_POSIX_SPAWN
		2536	# define INIT_POSIX_SPAWN
		2537	#endif // SANITIZER_INTERCEPT_POSIX_SPAWN
		2538
2413	#if SANITIZER_INTERCEPT_WAIT	2539	#if SANITIZER_INTERCEPT_WAIT
2414	// According to sys/wait.h, wait(), waitid(), waitpid() may have symbol version	2540	// According to sys/wait.h, wait(), waitid(), waitpid() may have symbol version
2415	// suffixes on Darwin. See the declaration of INTERCEPTOR_WITH_SUFFIX for	2541	// suffixes on Darwin. See the declaration of INTERCEPTOR_WITH_SUFFIX for
...	@@ -2519,7 +2645,7 @@ INTERCEPTOR(char , inet_ntop, int af, const void src, char *dst, u32 size) {	...	@@ -2519,7 +2645,7 @@ INTERCEPTOR(char , inet_ntop, int af, const void src, char *dst, u32 size) {
2519	// its metadata. See	2645	// its metadata. See
2520	// https://github.com/google/sanitizers/issues/321.	2646	// https://github.com/google/sanitizers/issues/321.
2521	char *res = REAL(inet_ntop)(af, src, dst, size);	2647	char *res = REAL(inet_ntop)(af, src, dst, size);
2522	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);	2648	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
2523	return res;	2649	return res;
2524	}	2650	}
2525	INTERCEPTOR(int, inet_pton, int af, const char src, void dst) {	2651	INTERCEPTOR(int, inet_pton, int af, const char src, void dst) {
...	@@ -2548,7 +2674,7 @@ INTERCEPTOR(int, inet_pton, int af, const char src, void dst) {	...	@@ -2548,7 +2674,7 @@ INTERCEPTOR(int, inet_pton, int af, const char src, void dst) {
2548	INTERCEPTOR(int, inet_aton, const char cp, void dst) {	2674	INTERCEPTOR(int, inet_aton, const char cp, void dst) {
2549	void *ctx;	2675	void *ctx;
2550	COMMON_INTERCEPTOR_ENTER(ctx, inet_aton, cp, dst);	2676	COMMON_INTERCEPTOR_ENTER(ctx, inet_aton, cp, dst);
2551	if (cp) COMMON_INTERCEPTOR_READ_RANGE(ctx, cp, REAL(strlen)(cp) + 1);	2677	if (cp) COMMON_INTERCEPTOR_READ_RANGE(ctx, cp, internal_strlen(cp) + 1);
2552	// FIXME: under ASan the call below may write to freed memory and corrupt	2678	// FIXME: under ASan the call below may write to freed memory and corrupt
2553	// its metadata. See	2679	// its metadata. See
2554	// https://github.com/google/sanitizers/issues/321.	2680	// https://github.com/google/sanitizers/issues/321.
...	@@ -2590,9 +2716,9 @@ INTERCEPTOR(int, getaddrinfo, char node, char service,	...	@@ -2590,9 +2716,9 @@ INTERCEPTOR(int, getaddrinfo, char node, char service,
2590	struct __sanitizer_addrinfo **out) {	2716	struct __sanitizer_addrinfo **out) {
2591	void *ctx;	2717	void *ctx;
2592	COMMON_INTERCEPTOR_ENTER(ctx, getaddrinfo, node, service, hints, out);	2718	COMMON_INTERCEPTOR_ENTER(ctx, getaddrinfo, node, service, hints, out);
2593	if (node) COMMON_INTERCEPTOR_READ_RANGE(ctx, node, REAL(strlen)(node) + 1);	2719	if (node) COMMON_INTERCEPTOR_READ_RANGE(ctx, node, internal_strlen(node) + 1);
2594	if (service)	2720	if (service)
2595	COMMON_INTERCEPTOR_READ_RANGE(ctx, service, REAL(strlen)(service) + 1);	2721	COMMON_INTERCEPTOR_READ_RANGE(ctx, service, internal_strlen(service) + 1);
2596	if (hints)	2722	if (hints)
2597	COMMON_INTERCEPTOR_READ_RANGE(ctx, hints, sizeof(__sanitizer_addrinfo));	2723	COMMON_INTERCEPTOR_READ_RANGE(ctx, hints, sizeof(__sanitizer_addrinfo));
2598	// FIXME: under ASan the call below may write to freed memory and corrupt	2724	// FIXME: under ASan the call below may write to freed memory and corrupt
...	@@ -2608,7 +2734,7 @@ INTERCEPTOR(int, getaddrinfo, char node, char service,	...	@@ -2608,7 +2734,7 @@ INTERCEPTOR(int, getaddrinfo, char node, char service,
2608	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ai_addr, p->ai_addrlen);	2734	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ai_addr, p->ai_addrlen);
2609	if (p->ai_canonname)	2735	if (p->ai_canonname)
2610	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ai_canonname,	2736	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ai_canonname,
2611	REAL(strlen)(p->ai_canonname) + 1);	2737	internal_strlen(p->ai_canonname) + 1);
2612	p = p->ai_next;	2738	p = p->ai_next;
2613	}	2739	}
2614	}	2740	}
...	@@ -2634,9 +2760,9 @@ INTERCEPTOR(int, getnameinfo, void sockaddr, unsigned salen, char host,	...	@@ -2634,9 +2760,9 @@ INTERCEPTOR(int, getnameinfo, void sockaddr, unsigned salen, char host,
2634	REAL(getnameinfo)(sockaddr, salen, host, hostlen, serv, servlen, flags);	2760	REAL(getnameinfo)(sockaddr, salen, host, hostlen, serv, servlen, flags);
2635	if (res == 0) {	2761	if (res == 0) {
2636	if (host && hostlen)	2762	if (host && hostlen)
2637	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, host, REAL(strlen)(host) + 1);	2763	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, host, internal_strlen(host) + 1);
2638	if (serv && servlen)	2764	if (serv && servlen)
2639	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, serv, REAL(strlen)(serv) + 1);	2765	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, serv, internal_strlen(serv) + 1);
2640	}	2766	}
2641	return res;	2767	return res;
2642	}	2768	}
...	@@ -2646,17 +2772,20 @@ INTERCEPTOR(int, getnameinfo, void sockaddr, unsigned salen, char host,	...	@@ -2646,17 +2772,20 @@ INTERCEPTOR(int, getnameinfo, void sockaddr, unsigned salen, char host,
2646	#endif	2772	#endif
2647		2773
2648	#if SANITIZER_INTERCEPT_GETSOCKNAME	2774	#if SANITIZER_INTERCEPT_GETSOCKNAME
2649	INTERCEPTOR(int, getsockname, int sock_fd, void addr, int addrlen) {	2775	INTERCEPTOR(int, getsockname, int sock_fd, void addr, unsigned addrlen) {
2650	void *ctx;	2776	void *ctx;
2651	COMMON_INTERCEPTOR_ENTER(ctx, getsockname, sock_fd, addr, addrlen);	2777	COMMON_INTERCEPTOR_ENTER(ctx, getsockname, sock_fd, addr, addrlen);
2652	COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));	2778	unsigned addr_sz;
2653	int addrlen_in = *addrlen;	2779	if (addrlen) {
		2780	COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
		2781	addr_sz = *addrlen;
		2782	}
2654	// FIXME: under ASan the call below may write to freed memory and corrupt	2783	// FIXME: under ASan the call below may write to freed memory and corrupt
2655	// its metadata. See	2784	// its metadata. See
2656	// https://github.com/google/sanitizers/issues/321.	2785	// https://github.com/google/sanitizers/issues/321.
2657	int res = REAL(getsockname)(sock_fd, addr, addrlen);	2786	int res = REAL(getsockname)(sock_fd, addr, addrlen);
2658	if (res == 0) {	2787	if (!res && addr && addrlen) {
2659	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addrlen_in, *addrlen));	2788	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addr_sz, *addrlen));
2660	}	2789	}
2661	return res;	2790	return res;
2662	}	2791	}
...	@@ -2669,10 +2798,10 @@ INTERCEPTOR(int, getsockname, int sock_fd, void addr, int addrlen) {	...	@@ -2669,10 +2798,10 @@ INTERCEPTOR(int, getsockname, int sock_fd, void addr, int addrlen) {
2669	static void write_hostent(void ctx, struct __sanitizer_hostent h) {	2798	static void write_hostent(void ctx, struct __sanitizer_hostent h) {
2670	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h, sizeof(__sanitizer_hostent));	2799	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h, sizeof(__sanitizer_hostent));
2671	if (h->h_name)	2800	if (h->h_name)
2672	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h->h_name, REAL(strlen)(h->h_name) + 1);	2801	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, h->h_name, internal_strlen(h->h_name) + 1);
2673	char **p = h->h_aliases;	2802	char **p = h->h_aliases;
2674	while (*p) {	2803	while (*p) {
2675	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, REAL(strlen)(p) + 1);	2804	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, internal_strlen(p) + 1);
2676	++p;	2805	++p;
2677	}	2806	}
2678	COMMON_INTERCEPTOR_WRITE_RANGE(	2807	COMMON_INTERCEPTOR_WRITE_RANGE(
...	@@ -3161,13 +3290,17 @@ INTERCEPTOR(int, getpeername, int sockfd, void addr, unsigned addrlen) {	...	@@ -3161,13 +3290,17 @@ INTERCEPTOR(int, getpeername, int sockfd, void addr, unsigned addrlen) {
3161	void *ctx;	3290	void *ctx;
3162	COMMON_INTERCEPTOR_ENTER(ctx, getpeername, sockfd, addr, addrlen);	3291	COMMON_INTERCEPTOR_ENTER(ctx, getpeername, sockfd, addr, addrlen);
3163	unsigned addr_sz;	3292	unsigned addr_sz;
3164	if (addrlen) addr_sz = *addrlen;	3293	if (addrlen) {
		3294	COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
		3295	addr_sz = *addrlen;
		3296	}
3165	// FIXME: under ASan the call below may write to freed memory and corrupt	3297	// FIXME: under ASan the call below may write to freed memory and corrupt
3166	// its metadata. See	3298	// its metadata. See
3167	// https://github.com/google/sanitizers/issues/321.	3299	// https://github.com/google/sanitizers/issues/321.
3168	int res = REAL(getpeername)(sockfd, addr, addrlen);	3300	int res = REAL(getpeername)(sockfd, addr, addrlen);
3169	if (!res && addr && addrlen)	3301	if (!res && addr && addrlen) {
3170	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addr_sz, *addrlen));	3302	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addr_sz, *addrlen));
		3303	}
3171	return res;	3304	return res;
3172	}	3305	}
3173	#define INIT_GETPEERNAME COMMON_INTERCEPT_FUNCTION(getpeername);	3306	#define INIT_GETPEERNAME COMMON_INTERCEPT_FUNCTION(getpeername);
...	@@ -3196,7 +3329,7 @@ INTERCEPTOR(int, sysinfo, void *info) {	...	@@ -3196,7 +3329,7 @@ INTERCEPTOR(int, sysinfo, void *info) {
3196	INTERCEPTOR(__sanitizer_dirent , opendir, const char path) {	3329	INTERCEPTOR(__sanitizer_dirent , opendir, const char path) {
3197	void *ctx;	3330	void *ctx;
3198	COMMON_INTERCEPTOR_ENTER(ctx, opendir, path);	3331	COMMON_INTERCEPTOR_ENTER(ctx, opendir, path);
3199	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	3332	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
3200	__sanitizer_dirent *res = REAL(opendir)(path);	3333	__sanitizer_dirent *res = REAL(opendir)(path);
3201	if (res)	3334	if (res)
3202	COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path);	3335	COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path);
...	@@ -3210,7 +3343,8 @@ INTERCEPTOR(__sanitizer_dirent , readdir, void dirp) {	...	@@ -3210,7 +3343,8 @@ INTERCEPTOR(__sanitizer_dirent , readdir, void dirp) {
3210	// its metadata. See	3343	// its metadata. See
3211	// https://github.com/google/sanitizers/issues/321.	3344	// https://github.com/google/sanitizers/issues/321.
3212	__sanitizer_dirent *res = REAL(readdir)(dirp);	3345	__sanitizer_dirent *res = REAL(readdir)(dirp);
3213	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen);	3346	if (res)
		3347	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer_dirsiz(res));
3214	return res;	3348	return res;
3215	}	3349	}
3216		3350
...	@@ -3225,7 +3359,7 @@ INTERCEPTOR(int, readdir_r, void dirp, __sanitizer_dirent entry,	...	@@ -3225,7 +3359,7 @@ INTERCEPTOR(int, readdir_r, void dirp, __sanitizer_dirent entry,
3225	if (!res) {	3359	if (!res) {
3226	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));	3360	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
3227	if (*result)	3361	if (*result)
3228	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, (result)->d_reclen);	3362	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, __sanitizer_dirsiz(result));
3229	}	3363	}
3230	return res;	3364	return res;
3231	}	3365	}
...	@@ -3246,7 +3380,8 @@ INTERCEPTOR(__sanitizer_dirent64 , readdir64, void dirp) {	...	@@ -3246,7 +3380,8 @@ INTERCEPTOR(__sanitizer_dirent64 , readdir64, void dirp) {
3246	// its metadata. See	3380	// its metadata. See
3247	// https://github.com/google/sanitizers/issues/321.	3381	// https://github.com/google/sanitizers/issues/321.
3248	__sanitizer_dirent64 *res = REAL(readdir64)(dirp);	3382	__sanitizer_dirent64 *res = REAL(readdir64)(dirp);
3249	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen);	3383	if (res)
		3384	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer_dirsiz(res));
3250	return res;	3385	return res;
3251	}	3386	}
3252		3387
...	@@ -3261,7 +3396,7 @@ INTERCEPTOR(int, readdir64_r, void dirp, __sanitizer_dirent64 entry,	...	@@ -3261,7 +3396,7 @@ INTERCEPTOR(int, readdir64_r, void dirp, __sanitizer_dirent64 entry,
3261	if (!res) {	3396	if (!res) {
3262	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));	3397	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
3263	if (*result)	3398	if (*result)
3264	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, (result)->d_reclen);	3399	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, __sanitizer_dirsiz(result));
3265	}	3400	}
3266	return res;	3401	return res;
3267	}	3402	}
...	@@ -3351,10 +3486,10 @@ INTERCEPTOR(char , setlocale, int category, char locale) {	...	@@ -3351,10 +3486,10 @@ INTERCEPTOR(char , setlocale, int category, char locale) {
3351	void *ctx;	3486	void *ctx;
3352	COMMON_INTERCEPTOR_ENTER(ctx, setlocale, category, locale);	3487	COMMON_INTERCEPTOR_ENTER(ctx, setlocale, category, locale);
3353	if (locale)	3488	if (locale)
3354	COMMON_INTERCEPTOR_READ_RANGE(ctx, locale, REAL(strlen)(locale) + 1);	3489	COMMON_INTERCEPTOR_READ_RANGE(ctx, locale, internal_strlen(locale) + 1);
3355	char *res = REAL(setlocale)(category, locale);	3490	char *res = REAL(setlocale)(category, locale);
3356	if (res) {	3491	if (res) {
3357	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);	3492	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
3358	unpoison_ctype_arrays(ctx);	3493	unpoison_ctype_arrays(ctx);
3359	}	3494	}
3360	return res;	3495	return res;
...	@@ -3373,7 +3508,7 @@ INTERCEPTOR(char , getcwd, char buf, SIZE_T size) {	...	@@ -3373,7 +3508,7 @@ INTERCEPTOR(char , getcwd, char buf, SIZE_T size) {
3373	// its metadata. See	3508	// its metadata. See
3374	// https://github.com/google/sanitizers/issues/321.	3509	// https://github.com/google/sanitizers/issues/321.
3375	char *res = REAL(getcwd)(buf, size);	3510	char *res = REAL(getcwd)(buf, size);
3376	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);	3511	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
3377	return res;	3512	return res;
3378	}	3513	}
3379	#define INIT_GETCWD COMMON_INTERCEPT_FUNCTION(getcwd);	3514	#define INIT_GETCWD COMMON_INTERCEPT_FUNCTION(getcwd);
...	@@ -3389,7 +3524,7 @@ INTERCEPTOR(char *, get_current_dir_name, int fake) {	...	@@ -3389,7 +3524,7 @@ INTERCEPTOR(char *, get_current_dir_name, int fake) {
3389	// its metadata. See	3524	// its metadata. See
3390	// https://github.com/google/sanitizers/issues/321.	3525	// https://github.com/google/sanitizers/issues/321.
3391	char *res = REAL(get_current_dir_name)(fake);	3526	char *res = REAL(get_current_dir_name)(fake);
3392	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);	3527	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
3393	return res;	3528	return res;
3394	}	3529	}
3395		3530
...	@@ -3429,30 +3564,26 @@ UNUSED static inline void StrtolFixAndCheck(void ctx, const char nptr,	...	@@ -3429,30 +3564,26 @@ UNUSED static inline void StrtolFixAndCheck(void ctx, const char nptr,
3429	(real_endptr - nptr) + 1 : 0);	3564	(real_endptr - nptr) + 1 : 0);
3430	}	3565	}
3431		3566
3432
3433	#if SANITIZER_INTERCEPT_STRTOIMAX	3567	#if SANITIZER_INTERCEPT_STRTOIMAX
3434	INTERCEPTOR(INTMAX_T, strtoimax, const char nptr, char *endptr, int base) {	3568	template <typename Fn>
3435	void *ctx;	3569	static ALWAYS_INLINE auto StrtoimaxImpl(void ctx, Fn real, const char nptr,
3436	COMMON_INTERCEPTOR_ENTER(ctx, strtoimax, nptr, endptr, base);	3570	char **endptr, int base)
3437	// FIXME: under ASan the call below may write to freed memory and corrupt	3571	-> decltype(real(nullptr, nullptr, 0)) {
3438	// its metadata. See
3439	// https://github.com/google/sanitizers/issues/321.
3440	char *real_endptr;	3572	char *real_endptr;
3441	INTMAX_T res = REAL(strtoimax)(nptr, &real_endptr, base);	3573	auto res = real(nptr, &real_endptr, base);
3442	StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base);	3574	StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base);
3443	return res;	3575	return res;
3444	}	3576	}
3445		3577
		3578	INTERCEPTOR(INTMAX_T, strtoimax, const char nptr, char *endptr, int base) {
		3579	void *ctx;
		3580	COMMON_INTERCEPTOR_ENTER(ctx, strtoimax, nptr, endptr, base);
		3581	return StrtoimaxImpl(ctx, REAL(strtoimax), nptr, endptr, base);
		3582	}
3446	INTERCEPTOR(UINTMAX_T, strtoumax, const char nptr, char *endptr, int base) {	3583	INTERCEPTOR(UINTMAX_T, strtoumax, const char nptr, char *endptr, int base) {
3447	void *ctx;	3584	void *ctx;
3448	COMMON_INTERCEPTOR_ENTER(ctx, strtoumax, nptr, endptr, base);	3585	COMMON_INTERCEPTOR_ENTER(ctx, strtoumax, nptr, endptr, base);
3449	// FIXME: under ASan the call below may write to freed memory and corrupt	3586	return StrtoimaxImpl(ctx, REAL(strtoumax), nptr, endptr, base);
3450	// its metadata. See
3451	// https://github.com/google/sanitizers/issues/321.
3452	char *real_endptr;
3453	UINTMAX_T res = REAL(strtoumax)(nptr, &real_endptr, base);
3454	StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base);
3455	return res;
3456	}	3587	}
3457		3588
3458	#define INIT_STRTOIMAX \	3589	#define INIT_STRTOIMAX \
...	@@ -3462,6 +3593,25 @@ INTERCEPTOR(UINTMAX_T, strtoumax, const char nptr, char *endptr, int base) {	...	@@ -3462,6 +3593,25 @@ INTERCEPTOR(UINTMAX_T, strtoumax, const char nptr, char *endptr, int base) {
3462	#define INIT_STRTOIMAX	3593	#define INIT_STRTOIMAX
3463	#endif	3594	#endif
3464		3595
		3596	#if SANITIZER_INTERCEPT_STRTOIMAX && SANITIZER_GLIBC
		3597	INTERCEPTOR(INTMAX_T, __isoc23_strtoimax, const char nptr, char *endptr, int base) {
		3598	void *ctx;
		3599	COMMON_INTERCEPTOR_ENTER(ctx, __isoc23_strtoimax, nptr, endptr, base);
		3600	return StrtoimaxImpl(ctx, REAL(__isoc23_strtoimax), nptr, endptr, base);
		3601	}
		3602	INTERCEPTOR(UINTMAX_T, __isoc23_strtoumax, const char nptr, char *endptr, int base) {
		3603	void *ctx;
		3604	COMMON_INTERCEPTOR_ENTER(ctx, __isoc23_strtoumax, nptr, endptr, base);
		3605	return StrtoimaxImpl(ctx, REAL(__isoc23_strtoumax), nptr, endptr, base);
		3606	}
		3607
		3608	# define INIT_STRTOIMAX_C23 \
		3609	COMMON_INTERCEPT_FUNCTION(__isoc23_strtoimax); \
		3610	COMMON_INTERCEPT_FUNCTION(__isoc23_strtoumax);
		3611	#else
		3612	# define INIT_STRTOIMAX_C23
		3613	#endif
		3614
3465	#if SANITIZER_INTERCEPT_MBSTOWCS	3615	#if SANITIZER_INTERCEPT_MBSTOWCS
3466	INTERCEPTOR(SIZE_T, mbstowcs, wchar_t dest, const char src, SIZE_T len) {	3616	INTERCEPTOR(SIZE_T, mbstowcs, wchar_t dest, const char src, SIZE_T len) {
3467	void *ctx;	3617	void *ctx;
...	@@ -3663,7 +3813,7 @@ INTERCEPTOR(int, tcgetattr, int fd, void *termios_p) {	...	@@ -3663,7 +3813,7 @@ INTERCEPTOR(int, tcgetattr, int fd, void *termios_p) {
3663	INTERCEPTOR(char , realpath, const char path, char *resolved_path) {	3813	INTERCEPTOR(char , realpath, const char path, char *resolved_path) {
3664	void *ctx;	3814	void *ctx;
3665	COMMON_INTERCEPTOR_ENTER(ctx, realpath, path, resolved_path);	3815	COMMON_INTERCEPTOR_ENTER(ctx, realpath, path, resolved_path);
3666	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	3816	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
3667		3817
3668	// Workaround a bug in glibc where dlsym(RTLD_NEXT, ...) returns the oldest	3818	// Workaround a bug in glibc where dlsym(RTLD_NEXT, ...) returns the oldest
3669	// version of a versioned symbol. For realpath(), this gives us something	3819	// version of a versioned symbol. For realpath(), this gives us something
...	@@ -3674,11 +3824,12 @@ INTERCEPTOR(char , realpath, const char path, char *resolved_path) {	...	@@ -3674,11 +3824,12 @@ INTERCEPTOR(char , realpath, const char path, char *resolved_path) {
3674	allocated_path = resolved_path = (char *)WRAP(malloc)(path_max + 1);	3824	allocated_path = resolved_path = (char *)WRAP(malloc)(path_max + 1);
3675		3825
3676	char *res = REAL(realpath)(path, resolved_path);	3826	char *res = REAL(realpath)(path, resolved_path);
3677	if (allocated_path && !res) WRAP(free)(allocated_path);	3827	if (allocated_path && !res)
3678	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);	3828	WRAP(free)(allocated_path);
		3829	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
3679	return res;	3830	return res;
3680	}	3831	}
3681	#define INIT_REALPATH COMMON_INTERCEPT_FUNCTION(realpath);	3832	# define INIT_REALPATH COMMON_INTERCEPT_FUNCTION(realpath);
3682	#else	3833	#else
3683	#define INIT_REALPATH	3834	#define INIT_REALPATH
3684	#endif	3835	#endif
...	@@ -3687,9 +3838,9 @@ INTERCEPTOR(char , realpath, const char path, char *resolved_path) {	...	@@ -3687,9 +3838,9 @@ INTERCEPTOR(char , realpath, const char path, char *resolved_path) {
3687	INTERCEPTOR(char , canonicalize_file_name, const char path) {	3838	INTERCEPTOR(char , canonicalize_file_name, const char path) {
3688	void *ctx;	3839	void *ctx;
3689	COMMON_INTERCEPTOR_ENTER(ctx, canonicalize_file_name, path);	3840	COMMON_INTERCEPTOR_ENTER(ctx, canonicalize_file_name, path);
3690	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	3841	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
3691	char *res = REAL(canonicalize_file_name)(path);	3842	char *res = REAL(canonicalize_file_name)(path);
3692	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);	3843	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
3693	return res;	3844	return res;
3694	}	3845	}
3695	#define INIT_CANONICALIZE_FILE_NAME \	3846	#define INIT_CANONICALIZE_FILE_NAME \
...	@@ -3750,7 +3901,7 @@ INTERCEPTOR(char *, strerror, int errnum) {	...	@@ -3750,7 +3901,7 @@ INTERCEPTOR(char *, strerror, int errnum) {
3750	COMMON_INTERCEPTOR_ENTER(ctx, strerror, errnum);	3901	COMMON_INTERCEPTOR_ENTER(ctx, strerror, errnum);
3751	COMMON_INTERCEPTOR_STRERROR();	3902	COMMON_INTERCEPTOR_STRERROR();
3752	char *res = REAL(strerror)(errnum);	3903	char *res = REAL(strerror)(errnum);
3753	if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);	3904	if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
3754	return res;	3905	return res;
3755	}	3906	}
3756	#define INIT_STRERROR COMMON_INTERCEPT_FUNCTION(strerror);	3907	#define INIT_STRERROR COMMON_INTERCEPT_FUNCTION(strerror);
...	@@ -3765,7 +3916,7 @@ INTERCEPTOR(char *, strerror, int errnum) {	...	@@ -3765,7 +3916,7 @@ INTERCEPTOR(char *, strerror, int errnum) {
3765	// * GNU version returns message pointer, which points to either buf or some	3916	// * GNU version returns message pointer, which points to either buf or some
3766	// static storage.	3917	// static storage.
3767	#if ((_POSIX_C_SOURCE >= 200112L \|\| _XOPEN_SOURCE >= 600) && !_GNU_SOURCE) \|\| \	3918	#if ((_POSIX_C_SOURCE >= 200112L \|\| _XOPEN_SOURCE >= 600) && !_GNU_SOURCE) \|\| \
3768	SANITIZER_MAC \|\| SANITIZER_ANDROID \|\| SANITIZER_NETBSD \|\| \	3919	SANITIZER_APPLE \|\| SANITIZER_ANDROID \|\| SANITIZER_NETBSD \|\| \
3769	SANITIZER_FREEBSD	3920	SANITIZER_FREEBSD
3770	// POSIX version. Spec is not clear on whether buf is NULL-terminated.	3921	// POSIX version. Spec is not clear on whether buf is NULL-terminated.
3771	// At least on OSX, buf contents are valid even when the call fails.	3922	// At least on OSX, buf contents are valid even when the call fails.
...	@@ -3792,13 +3943,13 @@ INTERCEPTOR(char , strerror_r, int errnum, char buf, SIZE_T buflen) {	...	@@ -3792,13 +3943,13 @@ INTERCEPTOR(char , strerror_r, int errnum, char buf, SIZE_T buflen) {
3792	// https://github.com/google/sanitizers/issues/321.	3943	// https://github.com/google/sanitizers/issues/321.
3793	char *res = REAL(strerror_r)(errnum, buf, buflen);	3944	char *res = REAL(strerror_r)(errnum, buf, buflen);
3794	if (res == buf)	3945	if (res == buf)
3795	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);	3946	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
3796	else	3947	else
3797	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);	3948	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
3798	return res;	3949	return res;
3799	}	3950	}
3800	#endif //(_POSIX_C_SOURCE >= 200112L \|\| _XOPEN_SOURCE >= 600) && !_GNU_SOURCE \|\|	3951	#endif //(_POSIX_C_SOURCE >= 200112L \|\| _XOPEN_SOURCE >= 600) && !_GNU_SOURCE \|\|
3801	//SANITIZER_MAC	3952	//SANITIZER_APPLE
3802	#define INIT_STRERROR_R COMMON_INTERCEPT_FUNCTION(strerror_r);	3953	#define INIT_STRERROR_R COMMON_INTERCEPT_FUNCTION(strerror_r);
3803	#else	3954	#else
3804	#define INIT_STRERROR_R	3955	#define INIT_STRERROR_R
...	@@ -3814,7 +3965,7 @@ INTERCEPTOR(int, __xpg_strerror_r, int errnum, char *buf, SIZE_T buflen) {	...	@@ -3814,7 +3965,7 @@ INTERCEPTOR(int, __xpg_strerror_r, int errnum, char *buf, SIZE_T buflen) {
3814	int res = REAL(__xpg_strerror_r)(errnum, buf, buflen);	3965	int res = REAL(__xpg_strerror_r)(errnum, buf, buflen);
3815	// This version always returns a null-terminated string.	3966	// This version always returns a null-terminated string.
3816	if (buf && buflen)	3967	if (buf && buflen)
3817	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1);	3968	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
3818	return res;	3969	return res;
3819	}	3970	}
3820	#define INIT_XPG_STRERROR_R COMMON_INTERCEPT_FUNCTION(__xpg_strerror_r);	3971	#define INIT_XPG_STRERROR_R COMMON_INTERCEPT_FUNCTION(__xpg_strerror_r);
...	@@ -3832,7 +3983,7 @@ static THREADLOCAL scandir_compar_f scandir_compar;	...	@@ -3832,7 +3983,7 @@ static THREADLOCAL scandir_compar_f scandir_compar;
3832		3983
3833	static int wrapped_scandir_filter(const struct __sanitizer_dirent *dir) {	3984	static int wrapped_scandir_filter(const struct __sanitizer_dirent *dir) {
3834	COMMON_INTERCEPTOR_UNPOISON_PARAM(1);	3985	COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
3835	COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, dir->d_reclen);	3986	COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, __sanitizer_dirsiz(dir));
3836	return scandir_filter(dir);	3987	return scandir_filter(dir);
3837	}	3988	}
3838		3989
...	@@ -3840,9 +3991,9 @@ static int wrapped_scandir_compar(const struct __sanitizer_dirent **a,	...	@@ -3840,9 +3991,9 @@ static int wrapped_scandir_compar(const struct __sanitizer_dirent **a,
3840	const struct __sanitizer_dirent **b) {	3991	const struct __sanitizer_dirent **b) {
3841	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);	3992	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
3842	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, sizeof(*a));	3993	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, sizeof(*a));
3843	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, (a)->d_reclen);	3994	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, __sanitizer_dirsiz(a));
3844	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, sizeof(*b));	3995	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, sizeof(*b));
3845	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, (b)->d_reclen);	3996	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, __sanitizer_dirsiz(b));
3846	return scandir_compar(a, b);	3997	return scandir_compar(a, b);
3847	}	3998	}
3848		3999
...	@@ -3850,7 +4001,7 @@ INTERCEPTOR(int, scandir, char dirp, __sanitizer_dirent **namelist,	...	@@ -3850,7 +4001,7 @@ INTERCEPTOR(int, scandir, char dirp, __sanitizer_dirent **namelist,
3850	scandir_filter_f filter, scandir_compar_f compar) {	4001	scandir_filter_f filter, scandir_compar_f compar) {
3851	void *ctx;	4002	void *ctx;
3852	COMMON_INTERCEPTOR_ENTER(ctx, scandir, dirp, namelist, filter, compar);	4003	COMMON_INTERCEPTOR_ENTER(ctx, scandir, dirp, namelist, filter, compar);
3853	if (dirp) COMMON_INTERCEPTOR_READ_RANGE(ctx, dirp, REAL(strlen)(dirp) + 1);	4004	if (dirp) COMMON_INTERCEPTOR_READ_RANGE(ctx, dirp, internal_strlen(dirp) + 1);
3854	scandir_filter = filter;	4005	scandir_filter = filter;
3855	scandir_compar = compar;	4006	scandir_compar = compar;
3856	// FIXME: under ASan the call below may write to freed memory and corrupt	4007	// FIXME: under ASan the call below may write to freed memory and corrupt
...	@@ -3866,7 +4017,7 @@ INTERCEPTOR(int, scandir, char dirp, __sanitizer_dirent **namelist,	...	@@ -3866,7 +4017,7 @@ INTERCEPTOR(int, scandir, char dirp, __sanitizer_dirent **namelist,
3866	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelist, sizeof(namelist) res);	4017	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelist, sizeof(namelist) res);
3867	for (int i = 0; i < res; ++i)	4018	for (int i = 0; i < res; ++i)
3868	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (*namelist)[i],	4019	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (*namelist)[i],
3869	(*namelist)[i]->d_reclen);	4020	__sanitizer_dirsiz((*namelist)[i]));
3870	}	4021	}
3871	return res;	4022	return res;
3872	}	4023	}
...	@@ -3885,7 +4036,7 @@ static THREADLOCAL scandir64_compar_f scandir64_compar;	...	@@ -3885,7 +4036,7 @@ static THREADLOCAL scandir64_compar_f scandir64_compar;
3885		4036
3886	static int wrapped_scandir64_filter(const struct __sanitizer_dirent64 *dir) {	4037	static int wrapped_scandir64_filter(const struct __sanitizer_dirent64 *dir) {
3887	COMMON_INTERCEPTOR_UNPOISON_PARAM(1);	4038	COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
3888	COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, dir->d_reclen);	4039	COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, __sanitizer_dirsiz(dir));
3889	return scandir64_filter(dir);	4040	return scandir64_filter(dir);
3890	}	4041	}
3891		4042
...	@@ -3893,9 +4044,9 @@ static int wrapped_scandir64_compar(const struct __sanitizer_dirent64 **a,	...	@@ -3893,9 +4044,9 @@ static int wrapped_scandir64_compar(const struct __sanitizer_dirent64 **a,
3893	const struct __sanitizer_dirent64 **b) {	4044	const struct __sanitizer_dirent64 **b) {
3894	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);	4045	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
3895	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, sizeof(*a));	4046	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, sizeof(*a));
3896	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, (a)->d_reclen);	4047	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, __sanitizer_dirsiz(a));
3897	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, sizeof(*b));	4048	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, sizeof(*b));
3898	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, (b)->d_reclen);	4049	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, __sanitizer_dirsiz(b));
3899	return scandir64_compar(a, b);	4050	return scandir64_compar(a, b);
3900	}	4051	}
3901		4052
...	@@ -3903,7 +4054,7 @@ INTERCEPTOR(int, scandir64, char dirp, __sanitizer_dirent64 **namelist,	...	@@ -3903,7 +4054,7 @@ INTERCEPTOR(int, scandir64, char dirp, __sanitizer_dirent64 **namelist,
3903	scandir64_filter_f filter, scandir64_compar_f compar) {	4054	scandir64_filter_f filter, scandir64_compar_f compar) {
3904	void *ctx;	4055	void *ctx;
3905	COMMON_INTERCEPTOR_ENTER(ctx, scandir64, dirp, namelist, filter, compar);	4056	COMMON_INTERCEPTOR_ENTER(ctx, scandir64, dirp, namelist, filter, compar);
3906	if (dirp) COMMON_INTERCEPTOR_READ_RANGE(ctx, dirp, REAL(strlen)(dirp) + 1);	4057	if (dirp) COMMON_INTERCEPTOR_READ_RANGE(ctx, dirp, internal_strlen(dirp) + 1);
3907	scandir64_filter = filter;	4058	scandir64_filter = filter;
3908	scandir64_compar = compar;	4059	scandir64_compar = compar;
3909	// FIXME: under ASan the call below may write to freed memory and corrupt	4060	// FIXME: under ASan the call below may write to freed memory and corrupt
...	@@ -3920,7 +4071,7 @@ INTERCEPTOR(int, scandir64, char dirp, __sanitizer_dirent64 **namelist,	...	@@ -3920,7 +4071,7 @@ INTERCEPTOR(int, scandir64, char dirp, __sanitizer_dirent64 **namelist,
3920	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelist, sizeof(namelist) res);	4071	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelist, sizeof(namelist) res);
3921	for (int i = 0; i < res; ++i)	4072	for (int i = 0; i < res; ++i)
3922	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (*namelist)[i],	4073	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (*namelist)[i],
3923	(*namelist)[i]->d_reclen);	4074	__sanitizer_dirsiz((*namelist)[i]));
3924	}	4075	}
3925	return res;	4076	return res;
3926	}	4077	}
...	@@ -3999,19 +4150,20 @@ INTERCEPTOR(int, ppoll, __sanitizer_pollfd *fds, __sanitizer_nfds_t nfds,	...	@@ -3999,19 +4150,20 @@ INTERCEPTOR(int, ppoll, __sanitizer_pollfd *fds, __sanitizer_nfds_t nfds,
3999	INTERCEPTOR(int, wordexp, char s, __sanitizer_wordexp_t p, int flags) {	4150	INTERCEPTOR(int, wordexp, char s, __sanitizer_wordexp_t p, int flags) {
4000	void *ctx;	4151	void *ctx;
4001	COMMON_INTERCEPTOR_ENTER(ctx, wordexp, s, p, flags);	4152	COMMON_INTERCEPTOR_ENTER(ctx, wordexp, s, p, flags);
4002	if (s) COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1);	4153	if (s) COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1);
4003	// FIXME: under ASan the call below may write to freed memory and corrupt	4154	// FIXME: under ASan the call below may write to freed memory and corrupt
4004	// its metadata. See	4155	// its metadata. See
4005	// https://github.com/google/sanitizers/issues/321.	4156	// https://github.com/google/sanitizers/issues/321.
4006	int res = REAL(wordexp)(s, p, flags);	4157	int res = REAL(wordexp)(s, p, flags);
4007	if (!res && p) {	4158	if (!res && p) {
4008	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));	4159	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
4009	if (p->we_wordc)	4160	uptr we_wordc =
4010	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->we_wordv,	4161	((flags & wordexp_wrde_dooffs) ? p->we_offs : 0) + p->we_wordc;
4011	sizeof(p->we_wordv) p->we_wordc);	4162	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->we_wordv,
4012	for (uptr i = 0; i < p->we_wordc; ++i) {	4163	sizeof(p->we_wordv) (we_wordc + 1));
		4164	for (uptr i = 0; i < we_wordc; ++i) {
4013	char *w = p->we_wordv[i];	4165	char *w = p->we_wordv[i];
4014	if (w) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, w, REAL(strlen)(w) + 1);	4166	if (w) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, w, internal_strlen(w) + 1);
4015	}	4167	}
4016	}	4168	}
4017	return res;	4169	return res;
...	@@ -4196,12 +4348,16 @@ INTERCEPTOR(int, pthread_sigmask, int how, __sanitizer_sigset_t *set,	...	@@ -4196,12 +4348,16 @@ INTERCEPTOR(int, pthread_sigmask, int how, __sanitizer_sigset_t *set,
4196	INTERCEPTOR(int, backtrace, void **buffer, int size) {	4348	INTERCEPTOR(int, backtrace, void **buffer, int size) {
4197	void *ctx;	4349	void *ctx;
4198	COMMON_INTERCEPTOR_ENTER(ctx, backtrace, buffer, size);	4350	COMMON_INTERCEPTOR_ENTER(ctx, backtrace, buffer, size);
4199	// FIXME: under ASan the call below may write to freed memory and corrupt	4351	// 'buffer' might be freed memory, hence it is unsafe to directly call
4200	// its metadata. See	4352	// REAL(backtrace)(buffer, size). Instead, we use our own known-good
4201	// https://github.com/google/sanitizers/issues/321.	4353	// scratch buffer.
4202	int res = REAL(backtrace)(buffer, size);	4354	void scratch = (void)InternalAlloc(sizeof(void) size);
4203	if (res && buffer)	4355	int res = REAL(backtrace)(scratch, size);
		4356	if (res && buffer) {
4204	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buffer, res * sizeof(*buffer));	4357	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buffer, res * sizeof(*buffer));
		4358	internal_memcpy(buffer, scratch, res * sizeof(*buffer));
		4359	}
		4360	InternalFree(scratch);
4205	return res;	4361	return res;
4206	}	4362	}
4207		4363
...	@@ -4210,14 +4366,13 @@ INTERCEPTOR(char , backtrace_symbols, void buffer, int size) {	...	@@ -4210,14 +4366,13 @@ INTERCEPTOR(char , backtrace_symbols, void buffer, int size) {
4210	COMMON_INTERCEPTOR_ENTER(ctx, backtrace_symbols, buffer, size);	4366	COMMON_INTERCEPTOR_ENTER(ctx, backtrace_symbols, buffer, size);
4211	if (buffer && size)	4367	if (buffer && size)
4212	COMMON_INTERCEPTOR_READ_RANGE(ctx, buffer, size * sizeof(*buffer));	4368	COMMON_INTERCEPTOR_READ_RANGE(ctx, buffer, size * sizeof(*buffer));
4213	// FIXME: under ASan the call below may write to freed memory and corrupt	4369	// The COMMON_INTERCEPTOR_READ_RANGE above ensures that 'buffer' is
4214	// its metadata. See	4370	// valid for reading.
4215	// https://github.com/google/sanitizers/issues/321.
4216	char **res = REAL(backtrace_symbols)(buffer, size);	4371	char **res = REAL(backtrace_symbols)(buffer, size);
4217	if (res && size) {	4372	if (res && size) {
4218	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, size * sizeof(*res));	4373	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, size * sizeof(*res));
4219	for (int i = 0; i < size; ++i)	4374	for (int i = 0; i < size; ++i)
4220	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res[i], REAL(strlen(res[i])) + 1);	4375	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res[i], internal_strlen(res[i]) + 1);
4221	}	4376	}
4222	return res;	4377	return res;
4223	}	4378	}
...	@@ -4243,90 +4398,13 @@ INTERCEPTOR(void, _exit, int status) {	...	@@ -4243,90 +4398,13 @@ INTERCEPTOR(void, _exit, int status) {
4243	#define INIT__EXIT	4398	#define INIT__EXIT
4244	#endif	4399	#endif
4245		4400
4246	#if SANITIZER_INTERCEPT_PTHREAD_MUTEX
4247	INTERCEPTOR(int, pthread_mutex_lock, void *m) {
4248	void *ctx;
4249	COMMON_INTERCEPTOR_ENTER(ctx, pthread_mutex_lock, m);
4250	COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m);
4251	int res = REAL(pthread_mutex_lock)(m);
4252	if (res == errno_EOWNERDEAD)
4253	COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m);
4254	if (res == 0 \|\| res == errno_EOWNERDEAD)
4255	COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m);
4256	if (res == errno_EINVAL)
4257	COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
4258	return res;
4259	}
4260
4261	INTERCEPTOR(int, pthread_mutex_unlock, void *m) {
4262	void *ctx;
4263	COMMON_INTERCEPTOR_ENTER(ctx, pthread_mutex_unlock, m);
4264	COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m);
4265	int res = REAL(pthread_mutex_unlock)(m);
4266	if (res == errno_EINVAL)
4267	COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
4268	return res;
4269	}
4270
4271	#define INIT_PTHREAD_MUTEX_LOCK COMMON_INTERCEPT_FUNCTION(pthread_mutex_lock)
4272	#define INIT_PTHREAD_MUTEX_UNLOCK \
4273	COMMON_INTERCEPT_FUNCTION(pthread_mutex_unlock)
4274	#else
4275	#define INIT_PTHREAD_MUTEX_LOCK
4276	#define INIT_PTHREAD_MUTEX_UNLOCK
4277	#endif
4278
4279	#if SANITIZER_INTERCEPT___PTHREAD_MUTEX
4280	INTERCEPTOR(int, __pthread_mutex_lock, void *m) {
4281	void *ctx;
4282	COMMON_INTERCEPTOR_ENTER(ctx, __pthread_mutex_lock, m);
4283	COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m);
4284	int res = REAL(__pthread_mutex_lock)(m);
4285	if (res == errno_EOWNERDEAD)
4286	COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m);
4287	if (res == 0 \|\| res == errno_EOWNERDEAD)
4288	COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m);
4289	if (res == errno_EINVAL)
4290	COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
4291	return res;
4292	}
4293
4294	INTERCEPTOR(int, __pthread_mutex_unlock, void *m) {
4295	void *ctx;
4296	COMMON_INTERCEPTOR_ENTER(ctx, __pthread_mutex_unlock, m);
4297	COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m);
4298	int res = REAL(__pthread_mutex_unlock)(m);
4299	if (res == errno_EINVAL)
4300	COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m);
4301	return res;
4302	}
4303
4304	#define INIT___PTHREAD_MUTEX_LOCK \
4305	COMMON_INTERCEPT_FUNCTION(__pthread_mutex_lock)
4306	#define INIT___PTHREAD_MUTEX_UNLOCK \
4307	COMMON_INTERCEPT_FUNCTION(__pthread_mutex_unlock)
4308	#else
4309	#define INIT___PTHREAD_MUTEX_LOCK
4310	#define INIT___PTHREAD_MUTEX_UNLOCK
4311	#endif
4312
4313	#if SANITIZER_INTERCEPT___LIBC_MUTEX	4401	#if SANITIZER_INTERCEPT___LIBC_MUTEX
4314	INTERCEPTOR(int, __libc_mutex_lock, void *m)
4315	ALIAS(WRAPPER_NAME(pthread_mutex_lock));
4316
4317	INTERCEPTOR(int, __libc_mutex_unlock, void *m)
4318	ALIAS(WRAPPER_NAME(pthread_mutex_unlock));
4319
4320	INTERCEPTOR(int, __libc_thr_setcancelstate, int state, int *oldstate)	4402	INTERCEPTOR(int, __libc_thr_setcancelstate, int state, int *oldstate)
4321	ALIAS(WRAPPER_NAME(pthread_setcancelstate));	4403	ALIAS(WRAP(pthread_setcancelstate));
4322		4404
4323	#define INIT___LIBC_MUTEX_LOCK COMMON_INTERCEPT_FUNCTION(__libc_mutex_lock)
4324	#define INIT___LIBC_MUTEX_UNLOCK COMMON_INTERCEPT_FUNCTION(__libc_mutex_unlock)
4325	#define INIT___LIBC_THR_SETCANCELSTATE \	4405	#define INIT___LIBC_THR_SETCANCELSTATE \
4326	COMMON_INTERCEPT_FUNCTION(__libc_thr_setcancelstate)	4406	COMMON_INTERCEPT_FUNCTION(__libc_thr_setcancelstate)
4327	#else	4407	#else
4328	#define INIT___LIBC_MUTEX_LOCK
4329	#define INIT___LIBC_MUTEX_UNLOCK
4330	#define INIT___LIBC_THR_SETCANCELSTATE	4408	#define INIT___LIBC_THR_SETCANCELSTATE
4331	#endif	4409	#endif
4332		4410
...	@@ -4335,16 +4413,16 @@ static void write_mntent(void ctx, __sanitizer_mntent mnt) {	...	@@ -4335,16 +4413,16 @@ static void write_mntent(void ctx, __sanitizer_mntent mnt) {
4335	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt, sizeof(*mnt));	4413	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt, sizeof(*mnt));
4336	if (mnt->mnt_fsname)	4414	if (mnt->mnt_fsname)
4337	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_fsname,	4415	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_fsname,
4338	REAL(strlen)(mnt->mnt_fsname) + 1);	4416	internal_strlen(mnt->mnt_fsname) + 1);
4339	if (mnt->mnt_dir)	4417	if (mnt->mnt_dir)
4340	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_dir,	4418	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_dir,
4341	REAL(strlen)(mnt->mnt_dir) + 1);	4419	internal_strlen(mnt->mnt_dir) + 1);
4342	if (mnt->mnt_type)	4420	if (mnt->mnt_type)
4343	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_type,	4421	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_type,
4344	REAL(strlen)(mnt->mnt_type) + 1);	4422	internal_strlen(mnt->mnt_type) + 1);
4345	if (mnt->mnt_opts)	4423	if (mnt->mnt_opts)
4346	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_opts,	4424	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mnt->mnt_opts,
4347	REAL(strlen)(mnt->mnt_opts) + 1);	4425	internal_strlen(mnt->mnt_opts) + 1);
4348	}	4426	}
4349	#endif	4427	#endif
4350		4428
...	@@ -4379,7 +4457,7 @@ INTERCEPTOR(__sanitizer_mntent , getmntent_r, void fp,	...	@@ -4379,7 +4457,7 @@ INTERCEPTOR(__sanitizer_mntent , getmntent_r, void fp,
4379	INTERCEPTOR(int, statfs, char path, void buf) {	4457	INTERCEPTOR(int, statfs, char path, void buf) {
4380	void *ctx;	4458	void *ctx;
4381	COMMON_INTERCEPTOR_ENTER(ctx, statfs, path, buf);	4459	COMMON_INTERCEPTOR_ENTER(ctx, statfs, path, buf);
4382	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	4460	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
4383	// FIXME: under ASan the call below may write to freed memory and corrupt	4461	// FIXME: under ASan the call below may write to freed memory and corrupt
4384	// its metadata. See	4462	// its metadata. See
4385	// https://github.com/google/sanitizers/issues/321.	4463	// https://github.com/google/sanitizers/issues/321.
...	@@ -4408,7 +4486,7 @@ INTERCEPTOR(int, fstatfs, int fd, void *buf) {	...	@@ -4408,7 +4486,7 @@ INTERCEPTOR(int, fstatfs, int fd, void *buf) {
4408	INTERCEPTOR(int, statfs64, char path, void buf) {	4486	INTERCEPTOR(int, statfs64, char path, void buf) {
4409	void *ctx;	4487	void *ctx;
4410	COMMON_INTERCEPTOR_ENTER(ctx, statfs64, path, buf);	4488	COMMON_INTERCEPTOR_ENTER(ctx, statfs64, path, buf);
4411	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	4489	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
4412	// FIXME: under ASan the call below may write to freed memory and corrupt	4490	// FIXME: under ASan the call below may write to freed memory and corrupt
4413	// its metadata. See	4491	// its metadata. See
4414	// https://github.com/google/sanitizers/issues/321.	4492	// https://github.com/google/sanitizers/issues/321.
...	@@ -4437,7 +4515,7 @@ INTERCEPTOR(int, fstatfs64, int fd, void *buf) {	...	@@ -4437,7 +4515,7 @@ INTERCEPTOR(int, fstatfs64, int fd, void *buf) {
4437	INTERCEPTOR(int, statvfs, char path, void buf) {	4515	INTERCEPTOR(int, statvfs, char path, void buf) {
4438	void *ctx;	4516	void *ctx;
4439	COMMON_INTERCEPTOR_ENTER(ctx, statvfs, path, buf);	4517	COMMON_INTERCEPTOR_ENTER(ctx, statvfs, path, buf);
4440	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	4518	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
4441	// FIXME: under ASan the call below may write to freed memory and corrupt	4519	// FIXME: under ASan the call below may write to freed memory and corrupt
4442	// its metadata. See	4520	// its metadata. See
4443	// https://github.com/google/sanitizers/issues/321.	4521	// https://github.com/google/sanitizers/issues/321.
...	@@ -4471,7 +4549,7 @@ INTERCEPTOR(int, fstatvfs, int fd, void *buf) {	...	@@ -4471,7 +4549,7 @@ INTERCEPTOR(int, fstatvfs, int fd, void *buf) {
4471	INTERCEPTOR(int, statvfs64, char path, void buf) {	4549	INTERCEPTOR(int, statvfs64, char path, void buf) {
4472	void *ctx;	4550	void *ctx;
4473	COMMON_INTERCEPTOR_ENTER(ctx, statvfs64, path, buf);	4551	COMMON_INTERCEPTOR_ENTER(ctx, statvfs64, path, buf);
4474	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	4552	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
4475	// FIXME: under ASan the call below may write to freed memory and corrupt	4553	// FIXME: under ASan the call below may write to freed memory and corrupt
4476	// its metadata. See	4554	// its metadata. See
4477	// https://github.com/google/sanitizers/issues/321.	4555	// https://github.com/google/sanitizers/issues/321.
...	@@ -4500,7 +4578,7 @@ INTERCEPTOR(int, fstatvfs64, int fd, void *buf) {	...	@@ -4500,7 +4578,7 @@ INTERCEPTOR(int, fstatvfs64, int fd, void *buf) {
4500	INTERCEPTOR(int, initgroups, char *user, u32 group) {	4578	INTERCEPTOR(int, initgroups, char *user, u32 group) {
4501	void *ctx;	4579	void *ctx;
4502	COMMON_INTERCEPTOR_ENTER(ctx, initgroups, user, group);	4580	COMMON_INTERCEPTOR_ENTER(ctx, initgroups, user, group);
4503	if (user) COMMON_INTERCEPTOR_READ_RANGE(ctx, user, REAL(strlen)(user) + 1);	4581	if (user) COMMON_INTERCEPTOR_READ_RANGE(ctx, user, internal_strlen(user) + 1);
4504	int res = REAL(initgroups)(user, group);	4582	int res = REAL(initgroups)(user, group);
4505	return res;	4583	return res;
4506	}	4584	}
...	@@ -4515,13 +4593,13 @@ INTERCEPTOR(char , ether_ntoa, __sanitizer_ether_addr addr) {	...	@@ -4515,13 +4593,13 @@ INTERCEPTOR(char , ether_ntoa, __sanitizer_ether_addr addr) {
4515	COMMON_INTERCEPTOR_ENTER(ctx, ether_ntoa, addr);	4593	COMMON_INTERCEPTOR_ENTER(ctx, ether_ntoa, addr);
4516	if (addr) COMMON_INTERCEPTOR_READ_RANGE(ctx, addr, sizeof(*addr));	4594	if (addr) COMMON_INTERCEPTOR_READ_RANGE(ctx, addr, sizeof(*addr));
4517	char *res = REAL(ether_ntoa)(addr);	4595	char *res = REAL(ether_ntoa)(addr);
4518	if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);	4596	if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
4519	return res;	4597	return res;
4520	}	4598	}
4521	INTERCEPTOR(__sanitizer_ether_addr , ether_aton, char buf) {	4599	INTERCEPTOR(__sanitizer_ether_addr , ether_aton, char buf) {
4522	void *ctx;	4600	void *ctx;
4523	COMMON_INTERCEPTOR_ENTER(ctx, ether_aton, buf);	4601	COMMON_INTERCEPTOR_ENTER(ctx, ether_aton, buf);
4524	if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, REAL(strlen)(buf) + 1);	4602	if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, internal_strlen(buf) + 1);
4525	__sanitizer_ether_addr *res = REAL(ether_aton)(buf);	4603	__sanitizer_ether_addr *res = REAL(ether_aton)(buf);
4526	if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, sizeof(*res));	4604	if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, sizeof(*res));
4527	return res;	4605	return res;
...	@@ -4543,14 +4621,14 @@ INTERCEPTOR(int, ether_ntohost, char hostname, __sanitizer_ether_addr addr) {	...	@@ -4543,14 +4621,14 @@ INTERCEPTOR(int, ether_ntohost, char hostname, __sanitizer_ether_addr addr) {
4543	// https://github.com/google/sanitizers/issues/321.	4621	// https://github.com/google/sanitizers/issues/321.
4544	int res = REAL(ether_ntohost)(hostname, addr);	4622	int res = REAL(ether_ntohost)(hostname, addr);
4545	if (!res && hostname)	4623	if (!res && hostname)
4546	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1);	4624	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, internal_strlen(hostname) + 1);
4547	return res;	4625	return res;
4548	}	4626	}
4549	INTERCEPTOR(int, ether_hostton, char hostname, __sanitizer_ether_addr addr) {	4627	INTERCEPTOR(int, ether_hostton, char hostname, __sanitizer_ether_addr addr) {
4550	void *ctx;	4628	void *ctx;
4551	COMMON_INTERCEPTOR_ENTER(ctx, ether_hostton, hostname, addr);	4629	COMMON_INTERCEPTOR_ENTER(ctx, ether_hostton, hostname, addr);
4552	if (hostname)	4630	if (hostname)
4553	COMMON_INTERCEPTOR_READ_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1);	4631	COMMON_INTERCEPTOR_READ_RANGE(ctx, hostname, internal_strlen(hostname) + 1);
4554	// FIXME: under ASan the call below may write to freed memory and corrupt	4632	// FIXME: under ASan the call below may write to freed memory and corrupt
4555	// its metadata. See	4633	// its metadata. See
4556	// https://github.com/google/sanitizers/issues/321.	4634	// https://github.com/google/sanitizers/issues/321.
...	@@ -4562,7 +4640,7 @@ INTERCEPTOR(int, ether_line, char line, __sanitizer_ether_addr addr,	...	@@ -4562,7 +4640,7 @@ INTERCEPTOR(int, ether_line, char line, __sanitizer_ether_addr addr,
4562	char *hostname) {	4640	char *hostname) {
4563	void *ctx;	4641	void *ctx;
4564	COMMON_INTERCEPTOR_ENTER(ctx, ether_line, line, addr, hostname);	4642	COMMON_INTERCEPTOR_ENTER(ctx, ether_line, line, addr, hostname);
4565	if (line) COMMON_INTERCEPTOR_READ_RANGE(ctx, line, REAL(strlen)(line) + 1);	4643	if (line) COMMON_INTERCEPTOR_READ_RANGE(ctx, line, internal_strlen(line) + 1);
4566	// FIXME: under ASan the call below may write to freed memory and corrupt	4644	// FIXME: under ASan the call below may write to freed memory and corrupt
4567	// its metadata. See	4645	// its metadata. See
4568	// https://github.com/google/sanitizers/issues/321.	4646	// https://github.com/google/sanitizers/issues/321.
...	@@ -4570,7 +4648,7 @@ INTERCEPTOR(int, ether_line, char line, __sanitizer_ether_addr addr,	...	@@ -4570,7 +4648,7 @@ INTERCEPTOR(int, ether_line, char line, __sanitizer_ether_addr addr,
4570	if (!res) {	4648	if (!res) {
4571	if (addr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, sizeof(*addr));	4649	if (addr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, sizeof(*addr));
4572	if (hostname)	4650	if (hostname)
4573	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1);	4651	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, internal_strlen(hostname) + 1);
4574	}	4652	}
4575	return res;	4653	return res;
4576	}	4654	}
...	@@ -4591,14 +4669,14 @@ INTERCEPTOR(char , ether_ntoa_r, __sanitizer_ether_addr addr, char *buf) {	...	@@ -4591,14 +4669,14 @@ INTERCEPTOR(char , ether_ntoa_r, __sanitizer_ether_addr addr, char *buf) {
4591	// its metadata. See	4669	// its metadata. See
4592	// https://github.com/google/sanitizers/issues/321.	4670	// https://github.com/google/sanitizers/issues/321.
4593	char *res = REAL(ether_ntoa_r)(addr, buf);	4671	char *res = REAL(ether_ntoa_r)(addr, buf);
4594	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);	4672	if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
4595	return res;	4673	return res;
4596	}	4674	}
4597	INTERCEPTOR(__sanitizer_ether_addr , ether_aton_r, char buf,	4675	INTERCEPTOR(__sanitizer_ether_addr , ether_aton_r, char buf,
4598	__sanitizer_ether_addr *addr) {	4676	__sanitizer_ether_addr *addr) {
4599	void *ctx;	4677	void *ctx;
4600	COMMON_INTERCEPTOR_ENTER(ctx, ether_aton_r, buf, addr);	4678	COMMON_INTERCEPTOR_ENTER(ctx, ether_aton_r, buf, addr);
4601	if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, REAL(strlen)(buf) + 1);	4679	if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, internal_strlen(buf) + 1);
4602	// FIXME: under ASan the call below may write to freed memory and corrupt	4680	// FIXME: under ASan the call below may write to freed memory and corrupt
4603	// its metadata. See	4681	// its metadata. See
4604	// https://github.com/google/sanitizers/issues/321.	4682	// https://github.com/google/sanitizers/issues/321.
...	@@ -4766,6 +4844,27 @@ INTERCEPTOR(int, pthread_attr_getaffinity_np, void *attr, SIZE_T cpusetsize,	...	@@ -4766,6 +4844,27 @@ INTERCEPTOR(int, pthread_attr_getaffinity_np, void *attr, SIZE_T cpusetsize,
4766	#define INIT_PTHREAD_ATTR_GETAFFINITY_NP	4844	#define INIT_PTHREAD_ATTR_GETAFFINITY_NP
4767	#endif	4845	#endif
4768		4846
		4847	#if SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP
		4848	INTERCEPTOR(int, pthread_getaffinity_np, void *attr, SIZE_T cpusetsize,
		4849	void *cpuset) {
		4850	void *ctx;
		4851	COMMON_INTERCEPTOR_ENTER(ctx, pthread_getaffinity_np, attr, cpusetsize,
		4852	cpuset);
		4853	// FIXME: under ASan the call below may write to freed memory and corrupt
		4854	// its metadata. See
		4855	// https://github.com/google/sanitizers/issues/321.
		4856	int res = REAL(pthread_getaffinity_np)(attr, cpusetsize, cpuset);
		4857	if (!res && cpusetsize && cpuset)
		4858	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cpuset, cpusetsize);
		4859	return res;
		4860	}
		4861
		4862	#define INIT_PTHREAD_GETAFFINITY_NP \
		4863	COMMON_INTERCEPT_FUNCTION(pthread_getaffinity_np);
		4864	#else
		4865	#define INIT_PTHREAD_GETAFFINITY_NP
		4866	#endif
		4867
4769	#if SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED	4868	#if SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED
4770	INTERCEPTOR_PTHREAD_MUTEXATTR_GET(pshared, sizeof(int))	4869	INTERCEPTOR_PTHREAD_MUTEXATTR_GET(pshared, sizeof(int))
4771	#define INIT_PTHREAD_MUTEXATTR_GETPSHARED \	4870	#define INIT_PTHREAD_MUTEXATTR_GETPSHARED \
...	@@ -4864,9 +4963,9 @@ INTERCEPTOR(char , tmpnam, char s) {	...	@@ -4864,9 +4963,9 @@ INTERCEPTOR(char , tmpnam, char s) {
4864	// FIXME: under ASan the call below may write to freed memory and corrupt	4963	// FIXME: under ASan the call below may write to freed memory and corrupt
4865	// its metadata. See	4964	// its metadata. See
4866	// https://github.com/google/sanitizers/issues/321.	4965	// https://github.com/google/sanitizers/issues/321.
4867	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1);	4966	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, internal_strlen(s) + 1);
4868	else	4967	else
4869	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);	4968	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
4870	}	4969	}
4871	return res;	4970	return res;
4872	}	4971	}
...	@@ -4883,7 +4982,7 @@ INTERCEPTOR(char , tmpnam_r, char s) {	...	@@ -4883,7 +4982,7 @@ INTERCEPTOR(char , tmpnam_r, char s) {
4883	// its metadata. See	4982	// its metadata. See
4884	// https://github.com/google/sanitizers/issues/321.	4983	// https://github.com/google/sanitizers/issues/321.
4885	char *res = REAL(tmpnam_r)(s);	4984	char *res = REAL(tmpnam_r)(s);
4886	if (res && s) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1);	4985	if (res && s) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, internal_strlen(s) + 1);
4887	return res;	4986	return res;
4888	}	4987	}
4889	#define INIT_TMPNAM_R COMMON_INTERCEPT_FUNCTION(tmpnam_r);	4988	#define INIT_TMPNAM_R COMMON_INTERCEPT_FUNCTION(tmpnam_r);
...	@@ -4897,7 +4996,7 @@ INTERCEPTOR(char *, ptsname, int fd) {	...	@@ -4897,7 +4996,7 @@ INTERCEPTOR(char *, ptsname, int fd) {
4897	COMMON_INTERCEPTOR_ENTER(ctx, ptsname, fd);	4996	COMMON_INTERCEPTOR_ENTER(ctx, ptsname, fd);
4898	char *res = REAL(ptsname)(fd);	4997	char *res = REAL(ptsname)(fd);
4899	if (res != nullptr)	4998	if (res != nullptr)
4900	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);	4999	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
4901	return res;	5000	return res;
4902	}	5001	}
4903	#define INIT_PTSNAME COMMON_INTERCEPT_FUNCTION(ptsname);	5002	#define INIT_PTSNAME COMMON_INTERCEPT_FUNCTION(ptsname);
...	@@ -4911,7 +5010,7 @@ INTERCEPTOR(int, ptsname_r, int fd, char *name, SIZE_T namesize) {	...	@@ -4911,7 +5010,7 @@ INTERCEPTOR(int, ptsname_r, int fd, char *name, SIZE_T namesize) {
4911	COMMON_INTERCEPTOR_ENTER(ctx, ptsname_r, fd, name, namesize);	5010	COMMON_INTERCEPTOR_ENTER(ctx, ptsname_r, fd, name, namesize);
4912	int res = REAL(ptsname_r)(fd, name, namesize);	5011	int res = REAL(ptsname_r)(fd, name, namesize);
4913	if (res == 0)	5012	if (res == 0)
4914	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);	5013	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
4915	return res;	5014	return res;
4916	}	5015	}
4917	#define INIT_PTSNAME_R COMMON_INTERCEPT_FUNCTION(ptsname_r);	5016	#define INIT_PTSNAME_R COMMON_INTERCEPT_FUNCTION(ptsname_r);
...	@@ -4925,7 +5024,7 @@ INTERCEPTOR(char *, ttyname, int fd) {	...	@@ -4925,7 +5024,7 @@ INTERCEPTOR(char *, ttyname, int fd) {
4925	COMMON_INTERCEPTOR_ENTER(ctx, ttyname, fd);	5024	COMMON_INTERCEPTOR_ENTER(ctx, ttyname, fd);
4926	char *res = REAL(ttyname)(fd);	5025	char *res = REAL(ttyname)(fd);
4927	if (res != nullptr)	5026	if (res != nullptr)
4928	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);	5027	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
4929	return res;	5028	return res;
4930	}	5029	}
4931	#define INIT_TTYNAME COMMON_INTERCEPT_FUNCTION(ttyname);	5030	#define INIT_TTYNAME COMMON_INTERCEPT_FUNCTION(ttyname);
...	@@ -4939,7 +5038,7 @@ INTERCEPTOR(int, ttyname_r, int fd, char *name, SIZE_T namesize) {	...	@@ -4939,7 +5038,7 @@ INTERCEPTOR(int, ttyname_r, int fd, char *name, SIZE_T namesize) {
4939	COMMON_INTERCEPTOR_ENTER(ctx, ttyname_r, fd, name, namesize);	5038	COMMON_INTERCEPTOR_ENTER(ctx, ttyname_r, fd, name, namesize);
4940	int res = REAL(ttyname_r)(fd, name, namesize);	5039	int res = REAL(ttyname_r)(fd, name, namesize);
4941	if (res == 0)	5040	if (res == 0)
4942	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);	5041	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
4943	return res;	5042	return res;
4944	}	5043	}
4945	#define INIT_TTYNAME_R COMMON_INTERCEPT_FUNCTION(ttyname_r);	5044	#define INIT_TTYNAME_R COMMON_INTERCEPT_FUNCTION(ttyname_r);
...	@@ -4951,10 +5050,10 @@ INTERCEPTOR(int, ttyname_r, int fd, char *name, SIZE_T namesize) {	...	@@ -4951,10 +5050,10 @@ INTERCEPTOR(int, ttyname_r, int fd, char *name, SIZE_T namesize) {
4951	INTERCEPTOR(char , tempnam, char dir, char *pfx) {	5050	INTERCEPTOR(char , tempnam, char dir, char *pfx) {
4952	void *ctx;	5051	void *ctx;
4953	COMMON_INTERCEPTOR_ENTER(ctx, tempnam, dir, pfx);	5052	COMMON_INTERCEPTOR_ENTER(ctx, tempnam, dir, pfx);
4954	if (dir) COMMON_INTERCEPTOR_READ_RANGE(ctx, dir, REAL(strlen)(dir) + 1);	5053	if (dir) COMMON_INTERCEPTOR_READ_RANGE(ctx, dir, internal_strlen(dir) + 1);
4955	if (pfx) COMMON_INTERCEPTOR_READ_RANGE(ctx, pfx, REAL(strlen)(pfx) + 1);	5054	if (pfx) COMMON_INTERCEPTOR_READ_RANGE(ctx, pfx, internal_strlen(pfx) + 1);
4956	char *res = REAL(tempnam)(dir, pfx);	5055	char *res = REAL(tempnam)(dir, pfx);
4957	if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);	5056	if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
4958	return res;	5057	return res;
4959	}	5058	}
4960	#define INIT_TEMPNAM COMMON_INTERCEPT_FUNCTION(tempnam);	5059	#define INIT_TEMPNAM COMMON_INTERCEPT_FUNCTION(tempnam);
...	@@ -5332,9 +5431,7 @@ INTERCEPTOR(void , __tls_get_addr, void arg) {	...	@@ -5332,9 +5431,7 @@ INTERCEPTOR(void , __tls_get_addr, void arg) {
5332	// On PowerPC, we also need to intercept __tls_get_addr_opt, which has	5431	// On PowerPC, we also need to intercept __tls_get_addr_opt, which has
5333	// mostly the same semantics as __tls_get_addr, but its presence enables	5432	// mostly the same semantics as __tls_get_addr, but its presence enables
5334	// some optimizations in linker (which are safe to ignore here).	5433	// some optimizations in linker (which are safe to ignore here).
5335	extern "C" __attribute__((alias("__interceptor___tls_get_addr"),	5434	INTERCEPTOR(void , __tls_get_addr_opt, void arg) ALIAS(WRAP(__tls_get_addr));
5336	visibility("default")))
5337	void __tls_get_addr_opt(void arg);
5338	#endif	5435	#endif
5339	#else // SANITIZER_S390	5436	#else // SANITIZER_S390
5340	// On s390, we have to intercept two functions here:	5437	// On s390, we have to intercept two functions here:
...	@@ -5368,21 +5465,20 @@ INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) {	...	@@ -5368,21 +5465,20 @@ INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) {
5368		5465
5369	#if SANITIZER_S390 && \	5466	#if SANITIZER_S390 && \
5370	(SANITIZER_INTERCEPT_TLS_GET_ADDR \|\| SANITIZER_INTERCEPT_TLS_GET_OFFSET)	5467	(SANITIZER_INTERCEPT_TLS_GET_ADDR \|\| SANITIZER_INTERCEPT_TLS_GET_OFFSET)
5371	extern "C" uptr __tls_get_offset(void *arg);
5372	extern "C" uptr __interceptor___tls_get_offset(void *arg);
5373	// We need a hidden symbol aliasing the above, so that we can jump	5468	// We need a hidden symbol aliasing the above, so that we can jump
5374	// directly to it from the assembly below.	5469	// directly to it from the assembly below.
5375	extern "C" __attribute__((alias("__interceptor___tls_get_addr_internal"),	5470	extern "C" __attribute__((visibility("hidden"))) uptr __tls_get_addr_hidden(
5376	visibility("hidden")))	5471	void *arg) ALIAS(WRAP(__tls_get_addr_internal));
5377	uptr __tls_get_addr_hidden(void *arg);	5472	extern "C" uptr __tls_get_offset(void *arg);
		5473	extern "C" uptr TRAMPOLINE(__tls_get_offset)(void *arg);
		5474	extern "C" uptr WRAP(__tls_get_offset)(void *arg);
5378	// Now carefully intercept __tls_get_offset.	5475	// Now carefully intercept __tls_get_offset.
5379	asm(	5476	asm(
5380	".text\n"	5477	".text\n"
5381	// The __intercept_ version has to exist, so that gen_dynamic_list.py	5478	// The __intercept_ version has to exist, so that gen_dynamic_list.py
5382	// exports our symbol.	5479	// exports our symbol.
5383	".weak __tls_get_offset\n"	5480	".weak __tls_get_offset\n"
5384	".type __tls_get_offset, @function\n"	5481	".set __tls_get_offset, __interceptor___tls_get_offset\n"
5385	"__tls_get_offset:\n"
5386	".global __interceptor___tls_get_offset\n"	5482	".global __interceptor___tls_get_offset\n"
5387	".type __interceptor___tls_get_offset, @function\n"	5483	".type __interceptor___tls_get_offset, @function\n"
5388	"__interceptor___tls_get_offset:\n"	5484	"__interceptor___tls_get_offset:\n"
...	@@ -5414,7 +5510,7 @@ asm(	...	@@ -5414,7 +5510,7 @@ asm(
5414	INTERCEPTOR(SSIZE_T, listxattr, const char path, char list, SIZE_T size) {	5510	INTERCEPTOR(SSIZE_T, listxattr, const char path, char list, SIZE_T size) {
5415	void *ctx;	5511	void *ctx;
5416	COMMON_INTERCEPTOR_ENTER(ctx, listxattr, path, list, size);	5512	COMMON_INTERCEPTOR_ENTER(ctx, listxattr, path, list, size);
5417	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	5513	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
5418	// FIXME: under ASan the call below may write to freed memory and corrupt	5514	// FIXME: under ASan the call below may write to freed memory and corrupt
5419	// its metadata. See	5515	// its metadata. See
5420	// https://github.com/google/sanitizers/issues/321.	5516	// https://github.com/google/sanitizers/issues/321.
...	@@ -5427,7 +5523,7 @@ INTERCEPTOR(SSIZE_T, listxattr, const char path, char list, SIZE_T size) {	...	@@ -5427,7 +5523,7 @@ INTERCEPTOR(SSIZE_T, listxattr, const char path, char list, SIZE_T size) {
5427	INTERCEPTOR(SSIZE_T, llistxattr, const char path, char list, SIZE_T size) {	5523	INTERCEPTOR(SSIZE_T, llistxattr, const char path, char list, SIZE_T size) {
5428	void *ctx;	5524	void *ctx;
5429	COMMON_INTERCEPTOR_ENTER(ctx, llistxattr, path, list, size);	5525	COMMON_INTERCEPTOR_ENTER(ctx, llistxattr, path, list, size);
5430	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	5526	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
5431	// FIXME: under ASan the call below may write to freed memory and corrupt	5527	// FIXME: under ASan the call below may write to freed memory and corrupt
5432	// its metadata. See	5528	// its metadata. See
5433	// https://github.com/google/sanitizers/issues/321.	5529	// https://github.com/google/sanitizers/issues/321.
...	@@ -5458,8 +5554,8 @@ INTERCEPTOR(SSIZE_T, getxattr, const char path, const char name, char *value,	...	@@ -5458,8 +5554,8 @@ INTERCEPTOR(SSIZE_T, getxattr, const char path, const char name, char *value,
5458	SIZE_T size) {	5554	SIZE_T size) {
5459	void *ctx;	5555	void *ctx;
5460	COMMON_INTERCEPTOR_ENTER(ctx, getxattr, path, name, value, size);	5556	COMMON_INTERCEPTOR_ENTER(ctx, getxattr, path, name, value, size);
5461	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	5557	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
5462	if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);	5558	if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
5463	// FIXME: under ASan the call below may write to freed memory and corrupt	5559	// FIXME: under ASan the call below may write to freed memory and corrupt
5464	// its metadata. See	5560	// its metadata. See
5465	// https://github.com/google/sanitizers/issues/321.	5561	// https://github.com/google/sanitizers/issues/321.
...	@@ -5471,8 +5567,8 @@ INTERCEPTOR(SSIZE_T, lgetxattr, const char path, const char name, char *value,	...	@@ -5471,8 +5567,8 @@ INTERCEPTOR(SSIZE_T, lgetxattr, const char path, const char name, char *value,
5471	SIZE_T size) {	5567	SIZE_T size) {
5472	void *ctx;	5568	void *ctx;
5473	COMMON_INTERCEPTOR_ENTER(ctx, lgetxattr, path, name, value, size);	5569	COMMON_INTERCEPTOR_ENTER(ctx, lgetxattr, path, name, value, size);
5474	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	5570	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
5475	if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);	5571	if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
5476	// FIXME: under ASan the call below may write to freed memory and corrupt	5572	// FIXME: under ASan the call below may write to freed memory and corrupt
5477	// its metadata. See	5573	// its metadata. See
5478	// https://github.com/google/sanitizers/issues/321.	5574	// https://github.com/google/sanitizers/issues/321.
...	@@ -5484,7 +5580,7 @@ INTERCEPTOR(SSIZE_T, fgetxattr, int fd, const char name, char value,	...	@@ -5484,7 +5580,7 @@ INTERCEPTOR(SSIZE_T, fgetxattr, int fd, const char name, char value,
5484	SIZE_T size) {	5580	SIZE_T size) {
5485	void *ctx;	5581	void *ctx;
5486	COMMON_INTERCEPTOR_ENTER(ctx, fgetxattr, fd, name, value, size);	5582	COMMON_INTERCEPTOR_ENTER(ctx, fgetxattr, fd, name, value, size);
5487	if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);	5583	if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
5488	// FIXME: under ASan the call below may write to freed memory and corrupt	5584	// FIXME: under ASan the call below may write to freed memory and corrupt
5489	// its metadata. See	5585	// its metadata. See
5490	// https://github.com/google/sanitizers/issues/321.	5586	// https://github.com/google/sanitizers/issues/321.
...	@@ -5554,7 +5650,7 @@ INTERCEPTOR(int, getifaddrs, __sanitizer_ifaddrs **ifap) {	...	@@ -5554,7 +5650,7 @@ INTERCEPTOR(int, getifaddrs, __sanitizer_ifaddrs **ifap) {
5554	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(__sanitizer_ifaddrs));	5650	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(__sanitizer_ifaddrs));
5555	if (p->ifa_name)	5651	if (p->ifa_name)
5556	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ifa_name,	5652	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ifa_name,
5557	REAL(strlen)(p->ifa_name) + 1);	5653	internal_strlen(p->ifa_name) + 1);
5558	if (p->ifa_addr)	5654	if (p->ifa_addr)
5559	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ifa_addr, struct_sockaddr_sz);	5655	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->ifa_addr, struct_sockaddr_sz);
5560	if (p->ifa_netmask)	5656	if (p->ifa_netmask)
...	@@ -5584,14 +5680,14 @@ INTERCEPTOR(char , if_indextoname, unsigned int ifindex, char ifname) {	...	@@ -5584,14 +5680,14 @@ INTERCEPTOR(char , if_indextoname, unsigned int ifindex, char ifname) {
5584	// https://github.com/google/sanitizers/issues/321.	5680	// https://github.com/google/sanitizers/issues/321.
5585	char *res = REAL(if_indextoname)(ifindex, ifname);	5681	char *res = REAL(if_indextoname)(ifindex, ifname);
5586	if (res && ifname)	5682	if (res && ifname)
5587	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ifname, REAL(strlen)(ifname) + 1);	5683	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ifname, internal_strlen(ifname) + 1);
5588	return res;	5684	return res;
5589	}	5685	}
5590	INTERCEPTOR(unsigned int, if_nametoindex, const char* ifname) {	5686	INTERCEPTOR(unsigned int, if_nametoindex, const char* ifname) {
5591	void *ctx;	5687	void *ctx;
5592	COMMON_INTERCEPTOR_ENTER(ctx, if_nametoindex, ifname);	5688	COMMON_INTERCEPTOR_ENTER(ctx, if_nametoindex, ifname);
5593	if (ifname)	5689	if (ifname)
5594	COMMON_INTERCEPTOR_READ_RANGE(ctx, ifname, REAL(strlen)(ifname) + 1);	5690	COMMON_INTERCEPTOR_READ_RANGE(ctx, ifname, internal_strlen(ifname) + 1);
5595	return REAL(if_nametoindex)(ifname);	5691	return REAL(if_nametoindex)(ifname);
5596	}	5692	}
5597	#define INIT_IF_INDEXTONAME \	5693	#define INIT_IF_INDEXTONAME \
...	@@ -5611,8 +5707,10 @@ INTERCEPTOR(int, capget, void hdrp, void datap) {	...	@@ -5611,8 +5707,10 @@ INTERCEPTOR(int, capget, void hdrp, void datap) {
5611	// its metadata. See	5707	// its metadata. See
5612	// https://github.com/google/sanitizers/issues/321.	5708	// https://github.com/google/sanitizers/issues/321.
5613	int res = REAL(capget)(hdrp, datap);	5709	int res = REAL(capget)(hdrp, datap);
5614	if (res == 0 && datap)	5710	if (res == 0 && datap) {
5615	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datap, __user_cap_data_struct_sz);	5711	unsigned datasz = __user_cap_data_struct_sz(hdrp);
		5712	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datap, datasz);
		5713	}
5616	// We can also return -1 and write to hdrp->version if the version passed in	5714	// We can also return -1 and write to hdrp->version if the version passed in
5617	// hdrp->version is unsupported. But that's not a trivial condition to check,	5715	// hdrp->version is unsupported. But that's not a trivial condition to check,
5618	// and anyway COMMON_INTERCEPTOR_READ_RANGE protects us to some extent.	5716	// and anyway COMMON_INTERCEPTOR_READ_RANGE protects us to some extent.
...	@@ -5623,8 +5721,10 @@ INTERCEPTOR(int, capset, void hdrp, const void datap) {	...	@@ -5623,8 +5721,10 @@ INTERCEPTOR(int, capset, void hdrp, const void datap) {
5623	COMMON_INTERCEPTOR_ENTER(ctx, capset, hdrp, datap);	5721	COMMON_INTERCEPTOR_ENTER(ctx, capset, hdrp, datap);
5624	if (hdrp)	5722	if (hdrp)
5625	COMMON_INTERCEPTOR_READ_RANGE(ctx, hdrp, __user_cap_header_struct_sz);	5723	COMMON_INTERCEPTOR_READ_RANGE(ctx, hdrp, __user_cap_header_struct_sz);
5626	if (datap)	5724	if (datap) {
5627	COMMON_INTERCEPTOR_READ_RANGE(ctx, datap, __user_cap_data_struct_sz);	5725	unsigned datasz = __user_cap_data_struct_sz(hdrp);
		5726	COMMON_INTERCEPTOR_READ_RANGE(ctx, datap, datasz);
		5727	}
5628	return REAL(capset)(hdrp, datap);	5728	return REAL(capset)(hdrp, datap);
5629	}	5729	}
5630	#define INIT_CAPGET \	5730	#define INIT_CAPGET \
...	@@ -5634,105 +5734,6 @@ INTERCEPTOR(int, capset, void hdrp, const void datap) {	...	@@ -5634,105 +5734,6 @@ INTERCEPTOR(int, capset, void hdrp, const void datap) {
5634	#define INIT_CAPGET	5734	#define INIT_CAPGET
5635	#endif	5735	#endif
5636		5736
5637	#if SANITIZER_INTERCEPT_AEABI_MEM
5638	INTERCEPTOR(void , __aeabi_memmove, void to, const void *from, uptr size) {
5639	void *ctx;
5640	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
5641	}
5642
5643	INTERCEPTOR(void , __aeabi_memmove4, void to, const void *from, uptr size) {
5644	void *ctx;
5645	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
5646	}
5647
5648	INTERCEPTOR(void , __aeabi_memmove8, void to, const void *from, uptr size) {
5649	void *ctx;
5650	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
5651	}
5652
5653	INTERCEPTOR(void , __aeabi_memcpy, void to, const void *from, uptr size) {
5654	void *ctx;
5655	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
5656	}
5657
5658	INTERCEPTOR(void , __aeabi_memcpy4, void to, const void *from, uptr size) {
5659	void *ctx;
5660	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
5661	}
5662
5663	INTERCEPTOR(void , __aeabi_memcpy8, void to, const void *from, uptr size) {
5664	void *ctx;
5665	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
5666	}
5667
5668	// Note the argument order.
5669	INTERCEPTOR(void , __aeabi_memset, void block, uptr size, int c) {
5670	void *ctx;
5671	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
5672	}
5673
5674	INTERCEPTOR(void , __aeabi_memset4, void block, uptr size, int c) {
5675	void *ctx;
5676	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
5677	}
5678
5679	INTERCEPTOR(void , __aeabi_memset8, void block, uptr size, int c) {
5680	void *ctx;
5681	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
5682	}
5683
5684	INTERCEPTOR(void , __aeabi_memclr, void block, uptr size) {
5685	void *ctx;
5686	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
5687	}
5688
5689	INTERCEPTOR(void , __aeabi_memclr4, void block, uptr size) {
5690	void *ctx;
5691	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
5692	}
5693
5694	INTERCEPTOR(void , __aeabi_memclr8, void block, uptr size) {
5695	void *ctx;
5696	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
5697	}
5698
5699	#define INIT_AEABI_MEM \
5700	COMMON_INTERCEPT_FUNCTION(__aeabi_memmove); \
5701	COMMON_INTERCEPT_FUNCTION(__aeabi_memmove4); \
5702	COMMON_INTERCEPT_FUNCTION(__aeabi_memmove8); \
5703	COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy); \
5704	COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy4); \
5705	COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy8); \
5706	COMMON_INTERCEPT_FUNCTION(__aeabi_memset); \
5707	COMMON_INTERCEPT_FUNCTION(__aeabi_memset4); \
5708	COMMON_INTERCEPT_FUNCTION(__aeabi_memset8); \
5709	COMMON_INTERCEPT_FUNCTION(__aeabi_memclr); \
5710	COMMON_INTERCEPT_FUNCTION(__aeabi_memclr4); \
5711	COMMON_INTERCEPT_FUNCTION(__aeabi_memclr8);
5712	#else
5713	#define INIT_AEABI_MEM
5714	#endif // SANITIZER_INTERCEPT_AEABI_MEM
5715
5716	#if SANITIZER_INTERCEPT___BZERO
5717	INTERCEPTOR(void , __bzero, void block, uptr size) {
5718	void *ctx;
5719	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
5720	}
5721	#define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero);
5722	#else
5723	#define INIT___BZERO
5724	#endif // SANITIZER_INTERCEPT___BZERO
5725
5726	#if SANITIZER_INTERCEPT_BZERO
5727	INTERCEPTOR(void , bzero, void block, uptr size) {
5728	void *ctx;
5729	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
5730	}
5731	#define INIT_BZERO COMMON_INTERCEPT_FUNCTION(bzero);
5732	#else
5733	#define INIT_BZERO
5734	#endif // SANITIZER_INTERCEPT_BZERO
5735
5736	#if SANITIZER_INTERCEPT_FTIME	5737	#if SANITIZER_INTERCEPT_FTIME
5737	INTERCEPTOR(int, ftime, __sanitizer_timeb *tp) {	5738	INTERCEPTOR(int, ftime, __sanitizer_timeb *tp) {
5738	void *ctx;	5739	void *ctx;
...	@@ -5849,7 +5850,7 @@ INTERCEPTOR(int, xdr_string, __sanitizer_XDR xdrs, char *p,	...	@@ -5849,7 +5850,7 @@ INTERCEPTOR(int, xdr_string, __sanitizer_XDR xdrs, char *p,
5849	COMMON_INTERCEPTOR_ENTER(ctx, xdr_string, xdrs, p, maxsize);	5850	COMMON_INTERCEPTOR_ENTER(ctx, xdr_string, xdrs, p, maxsize);
5850	if (p && xdrs->x_op == __sanitizer_XDR_ENCODE) {	5851	if (p && xdrs->x_op == __sanitizer_XDR_ENCODE) {
5851	COMMON_INTERCEPTOR_READ_RANGE(ctx, p, sizeof(*p));	5852	COMMON_INTERCEPTOR_READ_RANGE(ctx, p, sizeof(*p));
5852	COMMON_INTERCEPTOR_READ_RANGE(ctx, p, REAL(strlen)(p) + 1);	5853	COMMON_INTERCEPTOR_READ_RANGE(ctx, p, internal_strlen(p) + 1);
5853	}	5854	}
5854	// FIXME: under ASan the call below may write to freed memory and corrupt	5855	// FIXME: under ASan the call below may write to freed memory and corrupt
5855	// its metadata. See	5856	// its metadata. See
...	@@ -5858,7 +5859,7 @@ INTERCEPTOR(int, xdr_string, __sanitizer_XDR xdrs, char *p,	...	@@ -5858,7 +5859,7 @@ INTERCEPTOR(int, xdr_string, __sanitizer_XDR xdrs, char *p,
5858	if (p && xdrs->x_op == __sanitizer_XDR_DECODE) {	5859	if (p && xdrs->x_op == __sanitizer_XDR_DECODE) {
5859	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));	5860	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
5860	if (res && *p)	5861	if (res && *p)
5861	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, REAL(strlen)(p) + 1);	5862	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, internal_strlen(p) + 1);
5862	}	5863	}
5863	return res;	5864	return res;
5864	}	5865	}
...	@@ -6069,8 +6070,8 @@ INTERCEPTOR(int, __woverflow, __sanitizer_FILE *fp, int ch) {	...	@@ -6069,8 +6070,8 @@ INTERCEPTOR(int, __woverflow, __sanitizer_FILE *fp, int ch) {
6069	INTERCEPTOR(__sanitizer_FILE , fopen, const char path, const char *mode) {	6070	INTERCEPTOR(__sanitizer_FILE , fopen, const char path, const char *mode) {
6070	void *ctx;	6071	void *ctx;
6071	COMMON_INTERCEPTOR_ENTER(ctx, fopen, path, mode);	6072	COMMON_INTERCEPTOR_ENTER(ctx, fopen, path, mode);
6072	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	6073	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
6073	COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);	6074	COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, internal_strlen(mode) + 1);
6074	__sanitizer_FILE *res = REAL(fopen)(path, mode);	6075	__sanitizer_FILE *res = REAL(fopen)(path, mode);
6075	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);	6076	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
6076	if (res) unpoison_file(res);	6077	if (res) unpoison_file(res);
...	@@ -6079,7 +6080,7 @@ INTERCEPTOR(__sanitizer_FILE , fopen, const char path, const char *mode) {	...	@@ -6079,7 +6080,7 @@ INTERCEPTOR(__sanitizer_FILE , fopen, const char path, const char *mode) {
6079	INTERCEPTOR(__sanitizer_FILE , fdopen, int fd, const char mode) {	6080	INTERCEPTOR(__sanitizer_FILE , fdopen, int fd, const char mode) {
6080	void *ctx;	6081	void *ctx;
6081	COMMON_INTERCEPTOR_ENTER(ctx, fdopen, fd, mode);	6082	COMMON_INTERCEPTOR_ENTER(ctx, fdopen, fd, mode);
6082	COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);	6083	COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, internal_strlen(mode) + 1);
6083	__sanitizer_FILE *res = REAL(fdopen)(fd, mode);	6084	__sanitizer_FILE *res = REAL(fdopen)(fd, mode);
6084	if (res) unpoison_file(res);	6085	if (res) unpoison_file(res);
6085	return res;	6086	return res;
...	@@ -6088,8 +6089,8 @@ INTERCEPTOR(__sanitizer_FILE , freopen, const char path, const char *mode,	...	@@ -6088,8 +6089,8 @@ INTERCEPTOR(__sanitizer_FILE , freopen, const char path, const char *mode,
6088	__sanitizer_FILE *fp) {	6089	__sanitizer_FILE *fp) {
6089	void *ctx;	6090	void *ctx;
6090	COMMON_INTERCEPTOR_ENTER(ctx, freopen, path, mode, fp);	6091	COMMON_INTERCEPTOR_ENTER(ctx, freopen, path, mode, fp);
6091	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	6092	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
6092	COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);	6093	COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, internal_strlen(mode) + 1);
6093	COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp);	6094	COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp);
6094	__sanitizer_FILE *res = REAL(freopen)(path, mode, fp);	6095	__sanitizer_FILE *res = REAL(freopen)(path, mode, fp);
6095	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);	6096	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
...	@@ -6113,7 +6114,7 @@ INTERCEPTOR(int, flopen, const char *path, int flags, ...) {	...	@@ -6113,7 +6114,7 @@ INTERCEPTOR(int, flopen, const char *path, int flags, ...) {
6113	va_end(ap);	6114	va_end(ap);
6114	COMMON_INTERCEPTOR_ENTER(ctx, flopen, path, flags, mode);	6115	COMMON_INTERCEPTOR_ENTER(ctx, flopen, path, flags, mode);
6115	if (path) {	6116	if (path) {
6116	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	6117	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
6117	}	6118	}
6118	return REAL(flopen)(path, flags, mode);	6119	return REAL(flopen)(path, flags, mode);
6119	}	6120	}
...	@@ -6126,7 +6127,7 @@ INTERCEPTOR(int, flopenat, int dirfd, const char *path, int flags, ...) {	...	@@ -6126,7 +6127,7 @@ INTERCEPTOR(int, flopenat, int dirfd, const char *path, int flags, ...) {
6126	va_end(ap);	6127	va_end(ap);
6127	COMMON_INTERCEPTOR_ENTER(ctx, flopen, path, flags, mode);	6128	COMMON_INTERCEPTOR_ENTER(ctx, flopen, path, flags, mode);
6128	if (path) {	6129	if (path) {
6129	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	6130	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
6130	}	6131	}
6131	return REAL(flopenat)(dirfd, path, flags, mode);	6132	return REAL(flopenat)(dirfd, path, flags, mode);
6132	}	6133	}
...	@@ -6142,8 +6143,8 @@ INTERCEPTOR(int, flopenat, int dirfd, const char *path, int flags, ...) {	...	@@ -6142,8 +6143,8 @@ INTERCEPTOR(int, flopenat, int dirfd, const char *path, int flags, ...) {
6142	INTERCEPTOR(__sanitizer_FILE , fopen64, const char path, const char *mode) {	6143	INTERCEPTOR(__sanitizer_FILE , fopen64, const char path, const char *mode) {
6143	void *ctx;	6144	void *ctx;
6144	COMMON_INTERCEPTOR_ENTER(ctx, fopen64, path, mode);	6145	COMMON_INTERCEPTOR_ENTER(ctx, fopen64, path, mode);
6145	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	6146	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
6146	COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);	6147	COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, internal_strlen(mode) + 1);
6147	__sanitizer_FILE *res = REAL(fopen64)(path, mode);	6148	__sanitizer_FILE *res = REAL(fopen64)(path, mode);
6148	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);	6149	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
6149	if (res) unpoison_file(res);	6150	if (res) unpoison_file(res);
...	@@ -6153,8 +6154,8 @@ INTERCEPTOR(__sanitizer_FILE , freopen64, const char path, const char *mode,	...	@@ -6153,8 +6154,8 @@ INTERCEPTOR(__sanitizer_FILE , freopen64, const char path, const char *mode,
6153	__sanitizer_FILE *fp) {	6154	__sanitizer_FILE *fp) {
6154	void *ctx;	6155	void *ctx;
6155	COMMON_INTERCEPTOR_ENTER(ctx, freopen64, path, mode, fp);	6156	COMMON_INTERCEPTOR_ENTER(ctx, freopen64, path, mode, fp);
6156	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	6157	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
6157	COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1);	6158	COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, internal_strlen(mode) + 1);
6158	COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp);	6159	COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp);
6159	__sanitizer_FILE *res = REAL(freopen64)(path, mode, fp);	6160	__sanitizer_FILE *res = REAL(freopen64)(path, mode, fp);
6160	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);	6161	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path);
...	@@ -6305,8 +6306,7 @@ INTERCEPTOR(void, dlopen, const char filename, int flag) {	...	@@ -6305,8 +6306,7 @@ INTERCEPTOR(void, dlopen, const char filename, int flag) {
6305	void *ctx;	6306	void *ctx;
6306	COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlopen, filename, flag);	6307	COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlopen, filename, flag);
6307	if (filename) COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0);	6308	if (filename) COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0);
6308	COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag);	6309	void *res = COMMON_INTERCEPTOR_DLOPEN(filename, flag);
6309	void *res = REAL(dlopen)(filename, flag);
6310	Symbolizer::GetOrInit()->InvalidateModuleList();	6310	Symbolizer::GetOrInit()->InvalidateModuleList();
6311	COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res);	6311	COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res);
6312	return res;	6312	return res;
...	@@ -6332,9 +6332,9 @@ INTERCEPTOR(char , getpass, const char prompt) {	...	@@ -6332,9 +6332,9 @@ INTERCEPTOR(char , getpass, const char prompt) {
6332	void *ctx;	6332	void *ctx;
6333	COMMON_INTERCEPTOR_ENTER(ctx, getpass, prompt);	6333	COMMON_INTERCEPTOR_ENTER(ctx, getpass, prompt);
6334	if (prompt)	6334	if (prompt)
6335	COMMON_INTERCEPTOR_READ_RANGE(ctx, prompt, REAL(strlen)(prompt)+1);	6335	COMMON_INTERCEPTOR_READ_RANGE(ctx, prompt, internal_strlen(prompt)+1);
6336	char *res = REAL(getpass)(prompt);	6336	char *res = REAL(getpass)(prompt);
6337	if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res)+1);	6337	if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res)+1);
6338	return res;	6338	return res;
6339	}	6339	}
6340		6340
...	@@ -6475,7 +6475,7 @@ INTERCEPTOR(int, sem_init, __sanitizer_sem_t *s, int pshared, unsigned value) {	...	@@ -6475,7 +6475,7 @@ INTERCEPTOR(int, sem_init, __sanitizer_sem_t *s, int pshared, unsigned value) {
6475	COMMON_INTERCEPTOR_ENTER(ctx, sem_init, s, pshared, value);	6475	COMMON_INTERCEPTOR_ENTER(ctx, sem_init, s, pshared, value);
6476	// Workaround a bug in glibc's "old" semaphore implementation by	6476	// Workaround a bug in glibc's "old" semaphore implementation by
6477	// zero-initializing the sem_t contents. This has to be done here because	6477	// zero-initializing the sem_t contents. This has to be done here because
6478	// interceptors bind to the lowest symbols version by default, hitting the	6478	// interceptors bind to the lowest version before glibc 2.36, hitting the
6479	// buggy code path while the non-sanitized build of the same code works fine.	6479	// buggy code path while the non-sanitized build of the same code works fine.
6480	REAL(memset)(s, 0, sizeof(*s));	6480	REAL(memset)(s, 0, sizeof(*s));
6481	int res = REAL(sem_init)(s, pshared, value);	6481	int res = REAL(sem_init)(s, pshared, value);
...	@@ -6538,17 +6538,42 @@ INTERCEPTOR(int, sem_getvalue, __sanitizer_sem_t s, int sval) {	...	@@ -6538,17 +6538,42 @@ INTERCEPTOR(int, sem_getvalue, __sanitizer_sem_t s, int sval) {
6538	}	6538	}
6539	return res;	6539	return res;
6540	}	6540	}
6541	#define INIT_SEM \	6541
6542	COMMON_INTERCEPT_FUNCTION(sem_init); \	6542	INTERCEPTOR(__sanitizer_sem_t , sem_open, const char name, int oflag, ...) {
6543	COMMON_INTERCEPT_FUNCTION(sem_destroy); \	6543	void *ctx;
6544	COMMON_INTERCEPT_FUNCTION(sem_wait); \	6544	va_list ap;
6545	COMMON_INTERCEPT_FUNCTION(sem_trywait); \	6545	va_start(ap, oflag);
6546	COMMON_INTERCEPT_FUNCTION(sem_timedwait); \	6546	u32 mode = va_arg(ap, u32);
6547	COMMON_INTERCEPT_FUNCTION(sem_post); \	6547	u32 value = va_arg(ap, u32);
6548	COMMON_INTERCEPT_FUNCTION(sem_getvalue);	6548	COMMON_INTERCEPTOR_ENTER(ctx, sem_open, name, oflag, mode, value);
		6549	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
		6550	__sanitizer_sem_t *s = REAL(sem_open)(name, oflag, mode, value);
		6551	if (s)
		6552	COMMON_INTERCEPTOR_INITIALIZE_RANGE(s, sizeof(*s));
		6553	va_end(ap);
		6554	return s;
		6555	}
		6556
		6557	INTERCEPTOR(int, sem_unlink, const char *name) {
		6558	void *ctx;
		6559	COMMON_INTERCEPTOR_ENTER(ctx, sem_unlink, name);
		6560	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
		6561	return REAL(sem_unlink)(name);
		6562	}
		6563
		6564	# define INIT_SEM \
		6565	COMMON_INTERCEPT_FUNCTION(sem_init); \
		6566	COMMON_INTERCEPT_FUNCTION(sem_destroy); \
		6567	COMMON_INTERCEPT_FUNCTION(sem_wait); \
		6568	COMMON_INTERCEPT_FUNCTION(sem_trywait); \
		6569	COMMON_INTERCEPT_FUNCTION(sem_timedwait); \
		6570	COMMON_INTERCEPT_FUNCTION(sem_post); \
		6571	COMMON_INTERCEPT_FUNCTION(sem_getvalue); \
		6572	COMMON_INTERCEPT_FUNCTION(sem_open); \
		6573	COMMON_INTERCEPT_FUNCTION(sem_unlink);
6549	#else	6574	#else
6550	#define INIT_SEM	6575	# define INIT_SEM
6551	#endif // SANITIZER_INTERCEPT_SEM	6576	#endif // SANITIZER_INTERCEPT_SEM
6552		6577
6553	#if SANITIZER_INTERCEPT_PTHREAD_SETCANCEL	6578	#if SANITIZER_INTERCEPT_PTHREAD_SETCANCEL
6554	INTERCEPTOR(int, pthread_setcancelstate, int state, int *oldstate) {	6579	INTERCEPTOR(int, pthread_setcancelstate, int state, int *oldstate) {
...	@@ -6631,7 +6656,7 @@ INTERCEPTOR(char , ctermid, char s) {	...	@@ -6631,7 +6656,7 @@ INTERCEPTOR(char , ctermid, char s) {
6631	COMMON_INTERCEPTOR_ENTER(ctx, ctermid, s);	6656	COMMON_INTERCEPTOR_ENTER(ctx, ctermid, s);
6632	char *res = REAL(ctermid)(s);	6657	char *res = REAL(ctermid)(s);
6633	if (res) {	6658	if (res) {
6634	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);	6659	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
6635	}	6660	}
6636	return res;	6661	return res;
6637	}	6662	}
...	@@ -6646,7 +6671,7 @@ INTERCEPTOR(char , ctermid_r, char s) {	...	@@ -6646,7 +6671,7 @@ INTERCEPTOR(char , ctermid_r, char s) {
6646	COMMON_INTERCEPTOR_ENTER(ctx, ctermid_r, s);	6671	COMMON_INTERCEPTOR_ENTER(ctx, ctermid_r, s);
6647	char *res = REAL(ctermid_r)(s);	6672	char *res = REAL(ctermid_r)(s);
6648	if (res) {	6673	if (res) {
6649	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);	6674	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
6650	}	6675	}
6651	return res;	6676	return res;
6652	}	6677	}
...	@@ -6772,6 +6797,23 @@ INTERCEPTOR(int, stat, const char path, void buf) {	...	@@ -6772,6 +6797,23 @@ INTERCEPTOR(int, stat, const char path, void buf) {
6772	#define INIT_STAT	6797	#define INIT_STAT
6773	#endif	6798	#endif
6774		6799
		6800	#if SANITIZER_INTERCEPT_STAT64
		6801	INTERCEPTOR(int, stat64, const char path, void buf) {
		6802	void *ctx;
		6803	COMMON_INTERCEPTOR_ENTER(ctx, stat64, path, buf);
		6804	if (common_flags()->intercept_stat)
		6805	COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0);
		6806	int res = REAL(stat64)(path, buf);
		6807	if (!res)
		6808	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat64_sz);
		6809	return res;
		6810	}
		6811	#define INIT_STAT64 COMMON_INTERCEPT_FUNCTION(stat64)
		6812	#else
		6813	#define INIT_STAT64
		6814	#endif
		6815
		6816
6775	#if SANITIZER_INTERCEPT_LSTAT	6817	#if SANITIZER_INTERCEPT_LSTAT
6776	INTERCEPTOR(int, lstat, const char path, void buf) {	6818	INTERCEPTOR(int, lstat, const char path, void buf) {
6777	void *ctx;	6819	void *ctx;
...	@@ -6788,6 +6830,22 @@ INTERCEPTOR(int, lstat, const char path, void buf) {	...	@@ -6788,6 +6830,22 @@ INTERCEPTOR(int, lstat, const char path, void buf) {
6788	#define INIT_LSTAT	6830	#define INIT_LSTAT
6789	#endif	6831	#endif
6790		6832
		6833	#if SANITIZER_INTERCEPT_STAT64
		6834	INTERCEPTOR(int, lstat64, const char path, void buf) {
		6835	void *ctx;
		6836	COMMON_INTERCEPTOR_ENTER(ctx, lstat64, path, buf);
		6837	if (common_flags()->intercept_stat)
		6838	COMMON_INTERCEPTOR_READ_STRING(ctx, path, 0);
		6839	int res = REAL(lstat64)(path, buf);
		6840	if (!res)
		6841	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer::struct_stat64_sz);
		6842	return res;
		6843	}
		6844	#define INIT_LSTAT64 COMMON_INTERCEPT_FUNCTION(lstat64)
		6845	#else
		6846	#define INIT_LSTAT64
		6847	#endif
		6848
6791	#if SANITIZER_INTERCEPT___XSTAT	6849	#if SANITIZER_INTERCEPT___XSTAT
6792	INTERCEPTOR(int, __xstat, int version, const char path, void buf) {	6850	INTERCEPTOR(int, __xstat, int version, const char path, void buf) {
6793	void *ctx;	6851	void *ctx;
...	@@ -6960,6 +7018,7 @@ INTERCEPTOR(int, mprobe, void *ptr) {	...	@@ -6960,6 +7018,7 @@ INTERCEPTOR(int, mprobe, void *ptr) {
6960	}	7018	}
6961	#endif	7019	#endif
6962		7020
		7021	#if SANITIZER_INTERCEPT_WCSLEN
6963	INTERCEPTOR(SIZE_T, wcslen, const wchar_t *s) {	7022	INTERCEPTOR(SIZE_T, wcslen, const wchar_t *s) {
6964	void *ctx;	7023	void *ctx;
6965	COMMON_INTERCEPTOR_ENTER(ctx, wcslen, s);	7024	COMMON_INTERCEPTOR_ENTER(ctx, wcslen, s);
...	@@ -6978,13 +7037,16 @@ INTERCEPTOR(SIZE_T, wcsnlen, const wchar_t *s, SIZE_T n) {	...	@@ -6978,13 +7037,16 @@ INTERCEPTOR(SIZE_T, wcsnlen, const wchar_t *s, SIZE_T n) {
6978	#define INIT_WCSLEN \	7037	#define INIT_WCSLEN \
6979	COMMON_INTERCEPT_FUNCTION(wcslen); \	7038	COMMON_INTERCEPT_FUNCTION(wcslen); \
6980	COMMON_INTERCEPT_FUNCTION(wcsnlen);	7039	COMMON_INTERCEPT_FUNCTION(wcsnlen);
		7040	#else
		7041	#define INIT_WCSLEN
		7042	#endif
6981		7043
6982	#if SANITIZER_INTERCEPT_WCSCAT	7044	#if SANITIZER_INTERCEPT_WCSCAT
6983	INTERCEPTOR(wchar_t , wcscat, wchar_t dst, const wchar_t *src) {	7045	INTERCEPTOR(wchar_t , wcscat, wchar_t dst, const wchar_t *src) {
6984	void *ctx;	7046	void *ctx;
6985	COMMON_INTERCEPTOR_ENTER(ctx, wcscat, dst, src);	7047	COMMON_INTERCEPTOR_ENTER(ctx, wcscat, dst, src);
6986	SIZE_T src_size = REAL(wcslen)(src);	7048	SIZE_T src_size = internal_wcslen(src);
6987	SIZE_T dst_size = REAL(wcslen)(dst);	7049	SIZE_T dst_size = internal_wcslen(dst);
6988	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, (src_size + 1) * sizeof(wchar_t));	7050	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, (src_size + 1) * sizeof(wchar_t));
6989	COMMON_INTERCEPTOR_READ_RANGE(ctx, dst, (dst_size + 1) * sizeof(wchar_t));	7051	COMMON_INTERCEPTOR_READ_RANGE(ctx, dst, (dst_size + 1) * sizeof(wchar_t));
6990	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst + dst_size,	7052	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst + dst_size,
...	@@ -6995,8 +7057,8 @@ INTERCEPTOR(wchar_t , wcscat, wchar_t dst, const wchar_t *src) {	...	@@ -6995,8 +7057,8 @@ INTERCEPTOR(wchar_t , wcscat, wchar_t dst, const wchar_t *src) {
6995	INTERCEPTOR(wchar_t , wcsncat, wchar_t dst, const wchar_t *src, SIZE_T n) {	7057	INTERCEPTOR(wchar_t , wcsncat, wchar_t dst, const wchar_t *src, SIZE_T n) {
6996	void *ctx;	7058	void *ctx;
6997	COMMON_INTERCEPTOR_ENTER(ctx, wcsncat, dst, src, n);	7059	COMMON_INTERCEPTOR_ENTER(ctx, wcsncat, dst, src, n);
6998	SIZE_T src_size = REAL(wcsnlen)(src, n);	7060	SIZE_T src_size = internal_wcsnlen(src, n);
6999	SIZE_T dst_size = REAL(wcslen)(dst);	7061	SIZE_T dst_size = internal_wcslen(dst);
7000	COMMON_INTERCEPTOR_READ_RANGE(ctx, src,	7062	COMMON_INTERCEPTOR_READ_RANGE(ctx, src,
7001	Min(src_size + 1, n) * sizeof(wchar_t));	7063	Min(src_size + 1, n) * sizeof(wchar_t));
7002	COMMON_INTERCEPTOR_READ_RANGE(ctx, dst, (dst_size + 1) * sizeof(wchar_t));	7064	COMMON_INTERCEPTOR_READ_RANGE(ctx, dst, (dst_size + 1) * sizeof(wchar_t));
...	@@ -7015,7 +7077,7 @@ INTERCEPTOR(wchar_t , wcsncat, wchar_t dst, const wchar_t *src, SIZE_T n) {	...	@@ -7015,7 +7077,7 @@ INTERCEPTOR(wchar_t , wcsncat, wchar_t dst, const wchar_t *src, SIZE_T n) {
7015	INTERCEPTOR(wchar_t , wcsdup, wchar_t s) {	7077	INTERCEPTOR(wchar_t , wcsdup, wchar_t s) {
7016	void *ctx;	7078	void *ctx;
7017	COMMON_INTERCEPTOR_ENTER(ctx, wcsdup, s);	7079	COMMON_INTERCEPTOR_ENTER(ctx, wcsdup, s);
7018	SIZE_T len = REAL(wcslen)(s);	7080	SIZE_T len = internal_wcslen(s);
7019	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(wchar_t) * (len + 1));	7081	COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(wchar_t) * (len + 1));
7020	wchar_t *result = REAL(wcsdup)(s);	7082	wchar_t *result = REAL(wcsdup)(s);
7021	if (result)	7083	if (result)
...	@@ -7029,9 +7091,9 @@ INTERCEPTOR(wchar_t , wcsdup, wchar_t s) {	...	@@ -7029,9 +7091,9 @@ INTERCEPTOR(wchar_t , wcsdup, wchar_t s) {
7029	#endif	7091	#endif
7030		7092
7031	#if SANITIZER_INTERCEPT_STRXFRM	7093	#if SANITIZER_INTERCEPT_STRXFRM
7032	static SIZE_T RealStrLen(const char *str) { return REAL(strlen)(str); }	7094	static SIZE_T RealStrLen(const char *str) { return internal_strlen(str); }
7033		7095
7034	static SIZE_T RealStrLen(const wchar_t *str) { return REAL(wcslen)(str); }	7096	static SIZE_T RealStrLen(const wchar_t *str) { return internal_wcslen(str); }
7035		7097
7036	#define STRXFRM_INTERCEPTOR_IMPL(strxfrm, dest, src, len, ...) \	7098	#define STRXFRM_INTERCEPTOR_IMPL(strxfrm, dest, src, len, ...) \
7037	{ \	7099	{ \
...	@@ -7105,7 +7167,7 @@ INTERCEPTOR(int, acct, const char *file) {	...	@@ -7105,7 +7167,7 @@ INTERCEPTOR(int, acct, const char *file) {
7105	void *ctx;	7167	void *ctx;
7106	COMMON_INTERCEPTOR_ENTER(ctx, acct, file);	7168	COMMON_INTERCEPTOR_ENTER(ctx, acct, file);
7107	if (file)	7169	if (file)
7108	COMMON_INTERCEPTOR_READ_RANGE(ctx, file, REAL(strlen)(file) + 1);	7170	COMMON_INTERCEPTOR_READ_RANGE(ctx, file, internal_strlen(file) + 1);
7109	return REAL(acct)(file);	7171	return REAL(acct)(file);
7110	}	7172	}
7111	#define INIT_ACCT COMMON_INTERCEPT_FUNCTION(acct)	7173	#define INIT_ACCT COMMON_INTERCEPT_FUNCTION(acct)
...	@@ -7120,7 +7182,7 @@ INTERCEPTOR(const char *, user_from_uid, u32 uid, int nouser) {	...	@@ -7120,7 +7182,7 @@ INTERCEPTOR(const char *, user_from_uid, u32 uid, int nouser) {
7120	COMMON_INTERCEPTOR_ENTER(ctx, user_from_uid, uid, nouser);	7182	COMMON_INTERCEPTOR_ENTER(ctx, user_from_uid, uid, nouser);
7121	user = REAL(user_from_uid)(uid, nouser);	7183	user = REAL(user_from_uid)(uid, nouser);
7122	if (user)	7184	if (user)
7123	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, user, REAL(strlen)(user) + 1);	7185	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, user, internal_strlen(user) + 1);
7124	return user;	7186	return user;
7125	}	7187	}
7126	#define INIT_USER_FROM_UID COMMON_INTERCEPT_FUNCTION(user_from_uid)	7188	#define INIT_USER_FROM_UID COMMON_INTERCEPT_FUNCTION(user_from_uid)
...	@@ -7134,7 +7196,7 @@ INTERCEPTOR(int, uid_from_user, const char name, u32 uid) {	...	@@ -7134,7 +7196,7 @@ INTERCEPTOR(int, uid_from_user, const char name, u32 uid) {
7134	int res;	7196	int res;
7135	COMMON_INTERCEPTOR_ENTER(ctx, uid_from_user, name, uid);	7197	COMMON_INTERCEPTOR_ENTER(ctx, uid_from_user, name, uid);
7136	if (name)	7198	if (name)
7137	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);	7199	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
7138	res = REAL(uid_from_user)(name, uid);	7200	res = REAL(uid_from_user)(name, uid);
7139	if (uid)	7201	if (uid)
7140	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, uid, sizeof(*uid));	7202	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, uid, sizeof(*uid));
...	@@ -7152,7 +7214,7 @@ INTERCEPTOR(const char *, group_from_gid, u32 gid, int nogroup) {	...	@@ -7152,7 +7214,7 @@ INTERCEPTOR(const char *, group_from_gid, u32 gid, int nogroup) {
7152	COMMON_INTERCEPTOR_ENTER(ctx, group_from_gid, gid, nogroup);	7214	COMMON_INTERCEPTOR_ENTER(ctx, group_from_gid, gid, nogroup);
7153	group = REAL(group_from_gid)(gid, nogroup);	7215	group = REAL(group_from_gid)(gid, nogroup);
7154	if (group)	7216	if (group)
7155	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, group, REAL(strlen)(group) + 1);	7217	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, group, internal_strlen(group) + 1);
7156	return group;	7218	return group;
7157	}	7219	}
7158	#define INIT_GROUP_FROM_GID COMMON_INTERCEPT_FUNCTION(group_from_gid)	7220	#define INIT_GROUP_FROM_GID COMMON_INTERCEPT_FUNCTION(group_from_gid)
...	@@ -7166,7 +7228,7 @@ INTERCEPTOR(int, gid_from_group, const char group, u32 gid) {	...	@@ -7166,7 +7228,7 @@ INTERCEPTOR(int, gid_from_group, const char group, u32 gid) {
7166	int res;	7228	int res;
7167	COMMON_INTERCEPTOR_ENTER(ctx, gid_from_group, group, gid);	7229	COMMON_INTERCEPTOR_ENTER(ctx, gid_from_group, group, gid);
7168	if (group)	7230	if (group)
7169	COMMON_INTERCEPTOR_READ_RANGE(ctx, group, REAL(strlen)(group) + 1);	7231	COMMON_INTERCEPTOR_READ_RANGE(ctx, group, internal_strlen(group) + 1);
7170	res = REAL(gid_from_group)(group, gid);	7232	res = REAL(gid_from_group)(group, gid);
7171	if (gid)	7233	if (gid)
7172	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, gid, sizeof(*gid));	7234	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, gid, sizeof(*gid));
...	@@ -7182,7 +7244,7 @@ INTERCEPTOR(int, access, const char *path, int mode) {	...	@@ -7182,7 +7244,7 @@ INTERCEPTOR(int, access, const char *path, int mode) {
7182	void *ctx;	7244	void *ctx;
7183	COMMON_INTERCEPTOR_ENTER(ctx, access, path, mode);	7245	COMMON_INTERCEPTOR_ENTER(ctx, access, path, mode);
7184	if (path)	7246	if (path)
7185	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	7247	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
7186	return REAL(access)(path, mode);	7248	return REAL(access)(path, mode);
7187	}	7249	}
7188	#define INIT_ACCESS COMMON_INTERCEPT_FUNCTION(access)	7250	#define INIT_ACCESS COMMON_INTERCEPT_FUNCTION(access)
...	@@ -7195,7 +7257,7 @@ INTERCEPTOR(int, faccessat, int fd, const char *path, int mode, int flags) {	...	@@ -7195,7 +7257,7 @@ INTERCEPTOR(int, faccessat, int fd, const char *path, int mode, int flags) {
7195	void *ctx;	7257	void *ctx;
7196	COMMON_INTERCEPTOR_ENTER(ctx, faccessat, fd, path, mode, flags);	7258	COMMON_INTERCEPTOR_ENTER(ctx, faccessat, fd, path, mode, flags);
7197	if (path)	7259	if (path)
7198	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	7260	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
7199	return REAL(faccessat)(fd, path, mode, flags);	7261	return REAL(faccessat)(fd, path, mode, flags);
7200	}	7262	}
7201	#define INIT_FACCESSAT COMMON_INTERCEPT_FUNCTION(faccessat)	7263	#define INIT_FACCESSAT COMMON_INTERCEPT_FUNCTION(faccessat)
...	@@ -7210,7 +7272,7 @@ INTERCEPTOR(int, getgrouplist, const char name, u32 basegid, u32 groups,	...	@@ -7210,7 +7272,7 @@ INTERCEPTOR(int, getgrouplist, const char name, u32 basegid, u32 groups,
7210	int res;	7272	int res;
7211	COMMON_INTERCEPTOR_ENTER(ctx, getgrouplist, name, basegid, groups, ngroups);	7273	COMMON_INTERCEPTOR_ENTER(ctx, getgrouplist, name, basegid, groups, ngroups);
7212	if (name)	7274	if (name)
7213	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);	7275	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
7214	if (ngroups)	7276	if (ngroups)
7215	COMMON_INTERCEPTOR_READ_RANGE(ctx, ngroups, sizeof(*ngroups));	7277	COMMON_INTERCEPTOR_READ_RANGE(ctx, ngroups, sizeof(*ngroups));
7216	res = REAL(getgrouplist)(name, basegid, groups, ngroups);	7278	res = REAL(getgrouplist)(name, basegid, groups, ngroups);
...	@@ -7234,7 +7296,7 @@ INTERCEPTOR(int, getgroupmembership, const char name, u32 basegid, u32 groups,	...	@@ -7234,7 +7296,7 @@ INTERCEPTOR(int, getgroupmembership, const char name, u32 basegid, u32 groups,
7234	COMMON_INTERCEPTOR_ENTER(ctx, getgroupmembership, name, basegid, groups,	7296	COMMON_INTERCEPTOR_ENTER(ctx, getgroupmembership, name, basegid, groups,
7235	maxgrp, ngroups);	7297	maxgrp, ngroups);
7236	if (name)	7298	if (name)
7237	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);	7299	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
7238	res = REAL(getgroupmembership)(name, basegid, groups, maxgrp, ngroups);	7300	res = REAL(getgroupmembership)(name, basegid, groups, maxgrp, ngroups);
7239	if (!res && groups && ngroups) {	7301	if (!res && groups && ngroups) {
7240	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, groups, sizeof(groups) (*ngroups));	7302	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, groups, sizeof(groups) (*ngroups));
...	@@ -7252,7 +7314,7 @@ INTERCEPTOR(int, getgroupmembership, const char name, u32 basegid, u32 groups,	...	@@ -7252,7 +7314,7 @@ INTERCEPTOR(int, getgroupmembership, const char name, u32 basegid, u32 groups,
7252	INTERCEPTOR(SSIZE_T, readlink, const char path, char buf, SIZE_T bufsiz) {	7314	INTERCEPTOR(SSIZE_T, readlink, const char path, char buf, SIZE_T bufsiz) {
7253	void* ctx;	7315	void* ctx;
7254	COMMON_INTERCEPTOR_ENTER(ctx, readlink, path, buf, bufsiz);	7316	COMMON_INTERCEPTOR_ENTER(ctx, readlink, path, buf, bufsiz);
7255	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	7317	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
7256	SSIZE_T res = REAL(readlink)(path, buf, bufsiz);	7318	SSIZE_T res = REAL(readlink)(path, buf, bufsiz);
7257	if (res > 0)	7319	if (res > 0)
7258	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res);	7320	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res);
...	@@ -7269,7 +7331,7 @@ INTERCEPTOR(SSIZE_T, readlinkat, int dirfd, const char path, char buf,	...	@@ -7269,7 +7331,7 @@ INTERCEPTOR(SSIZE_T, readlinkat, int dirfd, const char path, char buf,
7269	SIZE_T bufsiz) {	7331	SIZE_T bufsiz) {
7270	void* ctx;	7332	void* ctx;
7271	COMMON_INTERCEPTOR_ENTER(ctx, readlinkat, dirfd, path, buf, bufsiz);	7333	COMMON_INTERCEPTOR_ENTER(ctx, readlinkat, dirfd, path, buf, bufsiz);
7272	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	7334	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
7273	SSIZE_T res = REAL(readlinkat)(dirfd, path, buf, bufsiz);	7335	SSIZE_T res = REAL(readlinkat)(dirfd, path, buf, bufsiz);
7274	if (res > 0)	7336	if (res > 0)
7275	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res);	7337	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res);
...	@@ -7287,7 +7349,7 @@ INTERCEPTOR(int, name_to_handle_at, int dirfd, const char *pathname,	...	@@ -7287,7 +7349,7 @@ INTERCEPTOR(int, name_to_handle_at, int dirfd, const char *pathname,
7287	void* ctx;	7349	void* ctx;
7288	COMMON_INTERCEPTOR_ENTER(ctx, name_to_handle_at, dirfd, pathname, handle,	7350	COMMON_INTERCEPTOR_ENTER(ctx, name_to_handle_at, dirfd, pathname, handle,
7289	mount_id, flags);	7351	mount_id, flags);
7290	COMMON_INTERCEPTOR_READ_RANGE(ctx, pathname, REAL(strlen)(pathname) + 1);	7352	COMMON_INTERCEPTOR_READ_RANGE(ctx, pathname, internal_strlen(pathname) + 1);
7291		7353
7292	__sanitizer_file_handle *sanitizer_handle =	7354	__sanitizer_file_handle *sanitizer_handle =
7293	reinterpret_cast<__sanitizer_file_handle*>(handle);	7355	reinterpret_cast<__sanitizer_file_handle*>(handle);
...	@@ -7351,7 +7413,7 @@ INTERCEPTOR(SIZE_T, strlcpy, char dst, char src, SIZE_T size) {	...	@@ -7351,7 +7413,7 @@ INTERCEPTOR(SIZE_T, strlcpy, char dst, char src, SIZE_T size) {
7351	ctx, src, Min(internal_strnlen(src, size), size - 1) + 1);	7413	ctx, src, Min(internal_strnlen(src, size), size - 1) + 1);
7352	}	7414	}
7353	res = REAL(strlcpy)(dst, src, size);	7415	res = REAL(strlcpy)(dst, src, size);
7354	COMMON_INTERCEPTOR_COPY_STRING(ctx, dst, src, REAL(strlen)(dst) + 1);	7416	COMMON_INTERCEPTOR_COPY_STRING(ctx, dst, src, internal_strlen(dst) + 1);
7355	return res;	7417	return res;
7356	}	7418	}
7357		7419
...	@@ -7379,17 +7441,25 @@ INTERCEPTOR(void , mmap, void addr, SIZE_T sz, int prot, int flags, int fd,	...	@@ -7379,17 +7441,25 @@ INTERCEPTOR(void , mmap, void addr, SIZE_T sz, int prot, int flags, int fd,
7379	OFF_T off) {	7441	OFF_T off) {
7380	void *ctx;	7442	void *ctx;
7381	if (common_flags()->detect_write_exec)	7443	if (common_flags()->detect_write_exec)
7382	ReportMmapWriteExec(prot);	7444	ReportMmapWriteExec(prot, flags);
7383	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)	7445	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
7384	return (void *)internal_mmap(addr, sz, prot, flags, fd, off);	7446	return (void *)internal_mmap(addr, sz, prot, flags, fd, off);
7385	COMMON_INTERCEPTOR_ENTER(ctx, mmap, addr, sz, prot, flags, fd, off);	7447	COMMON_INTERCEPTOR_ENTER(ctx, mmap, addr, sz, prot, flags, fd, off);
7386	COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, off);	7448	COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, off);
7387	}	7449	}
7388		7450
		7451	INTERCEPTOR(int, munmap, void *addr, SIZE_T sz) {
		7452	void *ctx;
		7453	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
		7454	return (int)internal_munmap(addr, sz);
		7455	COMMON_INTERCEPTOR_ENTER(ctx, munmap, addr, sz);
		7456	COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz);
		7457	}
		7458
7389	INTERCEPTOR(int, mprotect, void *addr, SIZE_T sz, int prot) {	7459	INTERCEPTOR(int, mprotect, void *addr, SIZE_T sz, int prot) {
7390	void *ctx;	7460	void *ctx;
7391	if (common_flags()->detect_write_exec)	7461	if (common_flags()->detect_write_exec)
7392	ReportMmapWriteExec(prot);	7462	ReportMmapWriteExec(prot, 0);
7393	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)	7463	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
7394	return (int)internal_mprotect(addr, sz, prot);	7464	return (int)internal_mprotect(addr, sz, prot);
7395	COMMON_INTERCEPTOR_ENTER(ctx, mprotect, addr, sz, prot);	7465	COMMON_INTERCEPTOR_ENTER(ctx, mprotect, addr, sz, prot);
...	@@ -7398,6 +7468,7 @@ INTERCEPTOR(int, mprotect, void *addr, SIZE_T sz, int prot) {	...	@@ -7398,6 +7468,7 @@ INTERCEPTOR(int, mprotect, void *addr, SIZE_T sz, int prot) {
7398	}	7468	}
7399	#define INIT_MMAP \	7469	#define INIT_MMAP \
7400	COMMON_INTERCEPT_FUNCTION(mmap); \	7470	COMMON_INTERCEPT_FUNCTION(mmap); \
		7471	COMMON_INTERCEPT_FUNCTION(munmap); \
7401	COMMON_INTERCEPT_FUNCTION(mprotect);	7472	COMMON_INTERCEPT_FUNCTION(mprotect);
7402	#else	7473	#else
7403	#define INIT_MMAP	7474	#define INIT_MMAP
...	@@ -7408,7 +7479,7 @@ INTERCEPTOR(void , mmap64, void addr, SIZE_T sz, int prot, int flags, int fd,	...	@@ -7408,7 +7479,7 @@ INTERCEPTOR(void , mmap64, void addr, SIZE_T sz, int prot, int flags, int fd,
7408	OFF64_T off) {	7479	OFF64_T off) {
7409	void *ctx;	7480	void *ctx;
7410	if (common_flags()->detect_write_exec)	7481	if (common_flags()->detect_write_exec)
7411	ReportMmapWriteExec(prot);	7482	ReportMmapWriteExec(prot, flags);
7412	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)	7483	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
7413	return (void *)internal_mmap(addr, sz, prot, flags, fd, off);	7484	return (void *)internal_mmap(addr, sz, prot, flags, fd, off);
7414	COMMON_INTERCEPTOR_ENTER(ctx, mmap64, addr, sz, prot, flags, fd, off);	7485	COMMON_INTERCEPTOR_ENTER(ctx, mmap64, addr, sz, prot, flags, fd, off);
...	@@ -7426,7 +7497,7 @@ INTERCEPTOR(char *, devname, u64 dev, u32 type) {	...	@@ -7426,7 +7497,7 @@ INTERCEPTOR(char *, devname, u64 dev, u32 type) {
7426	COMMON_INTERCEPTOR_ENTER(ctx, devname, dev, type);	7497	COMMON_INTERCEPTOR_ENTER(ctx, devname, dev, type);
7427	name = REAL(devname)(dev, type);	7498	name = REAL(devname)(dev, type);
7428	if (name)	7499	if (name)
7429	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);	7500	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
7430	return name;	7501	return name;
7431	}	7502	}
7432	#define INIT_DEVNAME COMMON_INTERCEPT_FUNCTION(devname);	7503	#define INIT_DEVNAME COMMON_INTERCEPT_FUNCTION(devname);
...	@@ -7448,7 +7519,7 @@ INTERCEPTOR(DEVNAME_R_RETTYPE, devname_r, u64 dev, u32 type, char *path,	...	@@ -7448,7 +7519,7 @@ INTERCEPTOR(DEVNAME_R_RETTYPE, devname_r, u64 dev, u32 type, char *path,
7448	COMMON_INTERCEPTOR_ENTER(ctx, devname_r, dev, type, path, len);	7519	COMMON_INTERCEPTOR_ENTER(ctx, devname_r, dev, type, path, len);
7449	DEVNAME_R_RETTYPE res = REAL(devname_r)(dev, type, path, len);	7520	DEVNAME_R_RETTYPE res = REAL(devname_r)(dev, type, path, len);
7450	if (DEVNAME_R_SUCCESS(res))	7521	if (DEVNAME_R_SUCCESS(res))
7451	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, path, REAL(strlen)(path) + 1);	7522	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, path, internal_strlen(path) + 1);
7452	return res;	7523	return res;
7453	}	7524	}
7454	#define INIT_DEVNAME_R COMMON_INTERCEPT_FUNCTION(devname_r);	7525	#define INIT_DEVNAME_R COMMON_INTERCEPT_FUNCTION(devname_r);
...	@@ -7478,7 +7549,7 @@ INTERCEPTOR(void, strmode, u32 mode, char *bp) {	...	@@ -7478,7 +7549,7 @@ INTERCEPTOR(void, strmode, u32 mode, char *bp) {
7478	COMMON_INTERCEPTOR_ENTER(ctx, strmode, mode, bp);	7549	COMMON_INTERCEPTOR_ENTER(ctx, strmode, mode, bp);
7479	REAL(strmode)(mode, bp);	7550	REAL(strmode)(mode, bp);
7480	if (bp)	7551	if (bp)
7481	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, bp, REAL(strlen)(bp) + 1);	7552	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, bp, internal_strlen(bp) + 1);
7482	}	7553	}
7483	#define INIT_STRMODE COMMON_INTERCEPT_FUNCTION(strmode)	7554	#define INIT_STRMODE COMMON_INTERCEPT_FUNCTION(strmode)
7484	#else	7555	#else
...	@@ -7498,40 +7569,44 @@ INTERCEPTOR(struct __sanitizer_ttyent , getttynam, char name) {	...	@@ -7498,40 +7569,44 @@ INTERCEPTOR(struct __sanitizer_ttyent , getttynam, char name) {
7498	void *ctx;	7569	void *ctx;
7499	COMMON_INTERCEPTOR_ENTER(ctx, getttynam, name);	7570	COMMON_INTERCEPTOR_ENTER(ctx, getttynam, name);
7500	if (name)	7571	if (name)
7501	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);	7572	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
7502	struct __sanitizer_ttyent *ttyent = REAL(getttynam)(name);	7573	struct __sanitizer_ttyent *ttyent = REAL(getttynam)(name);
7503	if (ttyent)	7574	if (ttyent)
7504	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ttyent, struct_ttyent_sz);	7575	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ttyent, struct_ttyent_sz);
7505	return ttyent;	7576	return ttyent;
7506	}	7577	}
		7578	#define INIT_TTYENT \
		7579	COMMON_INTERCEPT_FUNCTION(getttyent); \
		7580	COMMON_INTERCEPT_FUNCTION(getttynam);
		7581	#else
		7582	#define INIT_TTYENT
		7583	#endif
		7584
		7585	#if SANITIZER_INTERCEPT_TTYENTPATH
7507	INTERCEPTOR(int, setttyentpath, char *path) {	7586	INTERCEPTOR(int, setttyentpath, char *path) {
7508	void *ctx;	7587	void *ctx;
7509	COMMON_INTERCEPTOR_ENTER(ctx, setttyentpath, path);	7588	COMMON_INTERCEPTOR_ENTER(ctx, setttyentpath, path);
7510	if (path)	7589	if (path)
7511	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	7590	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
7512	return REAL(setttyentpath)(path);	7591	return REAL(setttyentpath)(path);
7513	}	7592	}
7514	#define INIT_TTYENT \	7593	#define INIT_TTYENTPATH COMMON_INTERCEPT_FUNCTION(setttyentpath);
7515	COMMON_INTERCEPT_FUNCTION(getttyent); \
7516	COMMON_INTERCEPT_FUNCTION(getttynam); \
7517	COMMON_INTERCEPT_FUNCTION(setttyentpath)
7518	#else	7594	#else
7519	#define INIT_TTYENT	7595	#define INIT_TTYENTPATH
7520	#endif	7596	#endif
7521		7597
7522	#if SANITIZER_INTERCEPT_PROTOENT	7598	#if SANITIZER_INTERCEPT_PROTOENT
7523	static void write_protoent(void ctx, struct __sanitizer_protoent p) {	7599	static void write_protoent(void ctx, struct __sanitizer_protoent p) {
7524	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));	7600	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p));
7525		7601
7526	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_name, REAL(strlen)(p->p_name) + 1);	7602	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_name, internal_strlen(p->p_name) + 1);
7527		7603
7528	SIZE_T pp_size = 1; // One handles the trailing \0	7604	SIZE_T pp_size = 1; // One handles the trailing \0
7529		7605
7530	for (char *pp = p->p_aliases; pp; ++pp, ++pp_size)	7606	for (char *pp = p->p_aliases; pp; ++pp, ++pp_size)
7531	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pp, REAL(strlen)(pp) + 1);	7607	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pp, internal_strlen(pp) + 1);
7532		7608
7533	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases,	7609	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases, pp_size * sizeof(char *));
7534	pp_size * sizeof(char **));
7535	}	7610	}
7536		7611
7537	INTERCEPTOR(struct __sanitizer_protoent *, getprotoent) {	7612	INTERCEPTOR(struct __sanitizer_protoent *, getprotoent) {
...	@@ -7547,7 +7622,7 @@ INTERCEPTOR(struct __sanitizer_protoent , getprotobyname, const char name) {	...	@@ -7547,7 +7622,7 @@ INTERCEPTOR(struct __sanitizer_protoent , getprotobyname, const char name) {
7547	void *ctx;	7622	void *ctx;
7548	COMMON_INTERCEPTOR_ENTER(ctx, getprotobyname, name);	7623	COMMON_INTERCEPTOR_ENTER(ctx, getprotobyname, name);
7549	if (name)	7624	if (name)
7550	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);	7625	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
7551	struct __sanitizer_protoent *p = REAL(getprotobyname)(name);	7626	struct __sanitizer_protoent *p = REAL(getprotobyname)(name);
7552	if (p)	7627	if (p)
7553	write_protoent(ctx, p);	7628	write_protoent(ctx, p);
...	@@ -7591,7 +7666,7 @@ INTERCEPTOR(int, getprotobyname_r, const char *name,	...	@@ -7591,7 +7666,7 @@ INTERCEPTOR(int, getprotobyname_r, const char *name,
7591	COMMON_INTERCEPTOR_ENTER(ctx, getprotobyname_r, name, result_buf, buf,	7666	COMMON_INTERCEPTOR_ENTER(ctx, getprotobyname_r, name, result_buf, buf,
7592	buflen, result);	7667	buflen, result);
7593	if (name)	7668	if (name)
7594	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);	7669	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
7595	int res = REAL(getprotobyname_r)(name, result_buf, buf, buflen, result);	7670	int res = REAL(getprotobyname_r)(name, result_buf, buf, buflen, result);
7596		7671
7597	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof *result);	7672	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof *result);
...	@@ -7630,15 +7705,14 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetent) {	...	@@ -7630,15 +7705,14 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetent) {
7630	if (n) {	7705	if (n) {
7631	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));	7706	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
7632		7707
7633	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, REAL(strlen)(n->n_name) + 1);	7708	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, internal_strlen(n->n_name) + 1);
7634		7709
7635	SIZE_T nn_size = 1; // One handles the trailing \0	7710	SIZE_T nn_size = 1; // One handles the trailing \0
7636		7711
7637	for (char *nn = n->n_aliases; nn; ++nn, ++nn_size)	7712	for (char *nn = n->n_aliases; nn; ++nn, ++nn_size)
7638	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, nn, REAL(strlen)(nn) + 1);	7713	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, nn, internal_strlen(nn) + 1);
7639		7714
7640	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,	7715	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, nn_size * sizeof(char *));
7641	nn_size * sizeof(char **));
7642	}	7716	}
7643	return n;	7717	return n;
7644	}	7718	}
...	@@ -7647,20 +7721,19 @@ INTERCEPTOR(struct __sanitizer_netent , getnetbyname, const char name) {	...	@@ -7647,20 +7721,19 @@ INTERCEPTOR(struct __sanitizer_netent , getnetbyname, const char name) {
7647	void *ctx;	7721	void *ctx;
7648	COMMON_INTERCEPTOR_ENTER(ctx, getnetbyname, name);	7722	COMMON_INTERCEPTOR_ENTER(ctx, getnetbyname, name);
7649	if (name)	7723	if (name)
7650	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);	7724	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
7651	struct __sanitizer_netent *n = REAL(getnetbyname)(name);	7725	struct __sanitizer_netent *n = REAL(getnetbyname)(name);
7652	if (n) {	7726	if (n) {
7653	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));	7727	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
7654		7728
7655	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, REAL(strlen)(n->n_name) + 1);	7729	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, internal_strlen(n->n_name) + 1);
7656		7730
7657	SIZE_T nn_size = 1; // One handles the trailing \0	7731	SIZE_T nn_size = 1; // One handles the trailing \0
7658		7732
7659	for (char *nn = n->n_aliases; nn; ++nn, ++nn_size)	7733	for (char *nn = n->n_aliases; nn; ++nn, ++nn_size)
7660	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, nn, REAL(strlen)(nn) + 1);	7734	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, nn, internal_strlen(nn) + 1);
7661		7735
7662	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,	7736	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, nn_size * sizeof(char *));
7663	nn_size * sizeof(char **));
7664	}	7737	}
7665	return n;	7738	return n;
7666	}	7739	}
...	@@ -7672,15 +7745,14 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetbyaddr, u32 net, int type) {	...	@@ -7672,15 +7745,14 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetbyaddr, u32 net, int type) {
7672	if (n) {	7745	if (n) {
7673	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));	7746	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n, sizeof(*n));
7674		7747
7675	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, REAL(strlen)(n->n_name) + 1);	7748	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_name, internal_strlen(n->n_name) + 1);
7676		7749
7677	SIZE_T nn_size = 1; // One handles the trailing \0	7750	SIZE_T nn_size = 1; // One handles the trailing \0
7678		7751
7679	for (char *nn = n->n_aliases; nn; ++nn, ++nn_size)	7752	for (char *nn = n->n_aliases; nn; ++nn, ++nn_size)
7680	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, nn, REAL(strlen)(nn) + 1);	7753	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, nn, internal_strlen(nn) + 1);
7681		7754
7682	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,	7755	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, nn_size * sizeof(char *));
7683	nn_size * sizeof(char **));
7684	}	7756	}
7685	return n;	7757	return n;
7686	}	7758	}
...	@@ -7753,12 +7825,12 @@ INTERCEPTOR(void, setbuf, __sanitizer_FILE stream, char buf) {	...	@@ -7753,12 +7825,12 @@ INTERCEPTOR(void, setbuf, __sanitizer_FILE stream, char buf) {
7753	unpoison_file(stream);	7825	unpoison_file(stream);
7754	}	7826	}
7755		7827
7756	INTERCEPTOR(void, setbuffer, __sanitizer_FILE stream, char buf, int mode) {	7828	INTERCEPTOR(void, setbuffer, __sanitizer_FILE stream, char buf, SIZE_T size) {
7757	void *ctx;	7829	void *ctx;
7758	COMMON_INTERCEPTOR_ENTER(ctx, setbuffer, stream, buf, mode);	7830	COMMON_INTERCEPTOR_ENTER(ctx, setbuffer, stream, buf, size);
7759	REAL(setbuffer)(stream, buf, mode);	7831	REAL(setbuffer)(stream, buf, size);
7760	if (buf) {	7832	if (buf) {
7761	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, __sanitizer_bufsiz);	7833	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, size);
7762	}	7834	}
7763	if (stream)	7835	if (stream)
7764	unpoison_file(stream);	7836	unpoison_file(stream);
...	@@ -7798,9 +7870,9 @@ INTERCEPTOR(int, regcomp, void preg, const char pattern, int cflags) {	...	@@ -7798,9 +7870,9 @@ INTERCEPTOR(int, regcomp, void preg, const char pattern, int cflags) {
7798	void *ctx;	7870	void *ctx;
7799	COMMON_INTERCEPTOR_ENTER(ctx, regcomp, preg, pattern, cflags);	7871	COMMON_INTERCEPTOR_ENTER(ctx, regcomp, preg, pattern, cflags);
7800	if (pattern)	7872	if (pattern)
7801	COMMON_INTERCEPTOR_READ_RANGE(ctx, pattern, REAL(strlen)(pattern) + 1);	7873	COMMON_INTERCEPTOR_READ_RANGE(ctx, pattern, internal_strlen(pattern) + 1);
7802	int res = REAL(regcomp)(preg, pattern, cflags);	7874	int res = REAL(regcomp)(preg, pattern, cflags);
7803	if (!res)	7875	if (preg)
7804	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, preg, struct_regex_sz);	7876	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, preg, struct_regex_sz);
7805	return res;	7877	return res;
7806	}	7878	}
...	@@ -7811,7 +7883,7 @@ INTERCEPTOR(int, regexec, const void preg, const char string, SIZE_T nmatch,	...	@@ -7811,7 +7883,7 @@ INTERCEPTOR(int, regexec, const void preg, const char string, SIZE_T nmatch,
7811	if (preg)	7883	if (preg)
7812	COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz);	7884	COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz);
7813	if (string)	7885	if (string)
7814	COMMON_INTERCEPTOR_READ_RANGE(ctx, string, REAL(strlen)(string) + 1);	7886	COMMON_INTERCEPTOR_READ_RANGE(ctx, string, internal_strlen(string) + 1);
7815	int res = REAL(regexec)(preg, string, nmatch, pmatch, eflags);	7887	int res = REAL(regexec)(preg, string, nmatch, pmatch, eflags);
7816	if (!res && pmatch)	7888	if (!res && pmatch)
7817	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pmatch, nmatch * struct_regmatch_sz);	7889	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pmatch, nmatch * struct_regmatch_sz);
...	@@ -7825,7 +7897,7 @@ INTERCEPTOR(SIZE_T, regerror, int errcode, const void preg, char errbuf,	...	@@ -7825,7 +7897,7 @@ INTERCEPTOR(SIZE_T, regerror, int errcode, const void preg, char errbuf,
7825	COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz);	7897	COMMON_INTERCEPTOR_READ_RANGE(ctx, preg, struct_regex_sz);
7826	SIZE_T res = REAL(regerror)(errcode, preg, errbuf, errbuf_size);	7898	SIZE_T res = REAL(regerror)(errcode, preg, errbuf, errbuf_size);
7827	if (errbuf)	7899	if (errbuf)
7828	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errbuf, REAL(strlen)(errbuf) + 1);	7900	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errbuf, internal_strlen(errbuf) + 1);
7829	return res;	7901	return res;
7830	}	7902	}
7831	INTERCEPTOR(void, regfree, const void *preg) {	7903	INTERCEPTOR(void, regfree, const void *preg) {
...	@@ -7850,15 +7922,15 @@ INTERCEPTOR(SSIZE_T, regnsub, char buf, SIZE_T bufsiz, const char sub,	...	@@ -7850,15 +7922,15 @@ INTERCEPTOR(SSIZE_T, regnsub, char buf, SIZE_T bufsiz, const char sub,
7850	void *ctx;	7922	void *ctx;
7851	COMMON_INTERCEPTOR_ENTER(ctx, regnsub, buf, bufsiz, sub, rm, str);	7923	COMMON_INTERCEPTOR_ENTER(ctx, regnsub, buf, bufsiz, sub, rm, str);
7852	if (sub)	7924	if (sub)
7853	COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, REAL(strlen)(sub) + 1);	7925	COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, internal_strlen(sub) + 1);
7854	// The implementation demands and hardcodes 10 elements	7926	// The implementation demands and hardcodes 10 elements
7855	if (rm)	7927	if (rm)
7856	COMMON_INTERCEPTOR_READ_RANGE(ctx, rm, 10 * struct_regmatch_sz);	7928	COMMON_INTERCEPTOR_READ_RANGE(ctx, rm, 10 * struct_regmatch_sz);
7857	if (str)	7929	if (str)
7858	COMMON_INTERCEPTOR_READ_RANGE(ctx, str, REAL(strlen)(str) + 1);	7930	COMMON_INTERCEPTOR_READ_RANGE(ctx, str, internal_strlen(str) + 1);
7859	SSIZE_T res = REAL(regnsub)(buf, bufsiz, sub, rm, str);	7931	SSIZE_T res = REAL(regnsub)(buf, bufsiz, sub, rm, str);
7860	if (res > 0 && buf)	7932	if (res > 0 && buf)
7861	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1);	7933	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
7862	return res;	7934	return res;
7863	}	7935	}
7864	INTERCEPTOR(SSIZE_T, regasub, char *buf, const char sub,	7936	INTERCEPTOR(SSIZE_T, regasub, char *buf, const char sub,
...	@@ -7866,16 +7938,16 @@ INTERCEPTOR(SSIZE_T, regasub, char *buf, const char sub,	...	@@ -7866,16 +7938,16 @@ INTERCEPTOR(SSIZE_T, regasub, char *buf, const char sub,
7866	void *ctx;	7938	void *ctx;
7867	COMMON_INTERCEPTOR_ENTER(ctx, regasub, buf, sub, rm, sstr);	7939	COMMON_INTERCEPTOR_ENTER(ctx, regasub, buf, sub, rm, sstr);
7868	if (sub)	7940	if (sub)
7869	COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, REAL(strlen)(sub) + 1);	7941	COMMON_INTERCEPTOR_READ_RANGE(ctx, sub, internal_strlen(sub) + 1);
7870	// Hardcode 10 elements as this is hardcoded size	7942	// Hardcode 10 elements as this is hardcoded size
7871	if (rm)	7943	if (rm)
7872	COMMON_INTERCEPTOR_READ_RANGE(ctx, rm, 10 * struct_regmatch_sz);	7944	COMMON_INTERCEPTOR_READ_RANGE(ctx, rm, 10 * struct_regmatch_sz);
7873	if (sstr)	7945	if (sstr)
7874	COMMON_INTERCEPTOR_READ_RANGE(ctx, sstr, REAL(strlen)(sstr) + 1);	7946	COMMON_INTERCEPTOR_READ_RANGE(ctx, sstr, internal_strlen(sstr) + 1);
7875	SSIZE_T res = REAL(regasub)(buf, sub, rm, sstr);	7947	SSIZE_T res = REAL(regasub)(buf, sub, rm, sstr);
7876	if (res > 0 && buf) {	7948	if (res > 0 && buf) {
7877	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, sizeof(char *));	7949	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, sizeof(char *));
7878	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1);	7950	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
7879	}	7951	}
7880	return res;	7952	return res;
7881	}	7953	}
...	@@ -7897,7 +7969,7 @@ INTERCEPTOR(void , fts_open, char const *path_argv, int options,	...	@@ -7897,7 +7969,7 @@ INTERCEPTOR(void , fts_open, char const *path_argv, int options,
7897	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **));	7969	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **));
7898	if (!*pa)	7970	if (!*pa)
7899	break;	7971	break;
7900	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, REAL(strlen)(pa) + 1);	7972	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, internal_strlen(pa) + 1);
7901	}	7973	}
7902	}	7974	}
7903	// TODO(kamil): handle compar callback	7975	// TODO(kamil): handle compar callback
...	@@ -7989,7 +8061,7 @@ INTERCEPTOR(int, sysctlbyname, char sname, void oldp, SIZE_T *oldlenp,	...	@@ -7989,7 +8061,7 @@ INTERCEPTOR(int, sysctlbyname, char sname, void oldp, SIZE_T *oldlenp,
7989	COMMON_INTERCEPTOR_ENTER(ctx, sysctlbyname, sname, oldp, oldlenp, newp,	8061	COMMON_INTERCEPTOR_ENTER(ctx, sysctlbyname, sname, oldp, oldlenp, newp,
7990	newlen);	8062	newlen);
7991	if (sname)	8063	if (sname)
7992	COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);	8064	COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, internal_strlen(sname) + 1);
7993	if (oldlenp)	8065	if (oldlenp)
7994	COMMON_INTERCEPTOR_READ_RANGE(ctx, oldlenp, sizeof(*oldlenp));	8066	COMMON_INTERCEPTOR_READ_RANGE(ctx, oldlenp, sizeof(*oldlenp));
7995	if (newp && newlen)	8067	if (newp && newlen)
...	@@ -8010,7 +8082,7 @@ INTERCEPTOR(int, sysctlnametomib, const char sname, int name,	...	@@ -8010,7 +8082,7 @@ INTERCEPTOR(int, sysctlnametomib, const char sname, int name,
8010	void *ctx;	8082	void *ctx;
8011	COMMON_INTERCEPTOR_ENTER(ctx, sysctlnametomib, sname, name, namelenp);	8083	COMMON_INTERCEPTOR_ENTER(ctx, sysctlnametomib, sname, name, namelenp);
8012	if (sname)	8084	if (sname)
8013	COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);	8085	COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, internal_strlen(sname) + 1);
8014	if (namelenp)	8086	if (namelenp)
8015	COMMON_INTERCEPTOR_READ_RANGE(ctx, namelenp, sizeof(*namelenp));	8087	COMMON_INTERCEPTOR_READ_RANGE(ctx, namelenp, sizeof(*namelenp));
8016	int res = REAL(sysctlnametomib)(sname, name, namelenp);	8088	int res = REAL(sysctlnametomib)(sname, name, namelenp);
...	@@ -8050,7 +8122,7 @@ INTERCEPTOR(void , asysctlbyname, const char sname, SIZE_T *len) {	...	@@ -8050,7 +8122,7 @@ INTERCEPTOR(void , asysctlbyname, const char sname, SIZE_T *len) {
8050	void *ctx;	8122	void *ctx;
8051	COMMON_INTERCEPTOR_ENTER(ctx, asysctlbyname, sname, len);	8123	COMMON_INTERCEPTOR_ENTER(ctx, asysctlbyname, sname, len);
8052	if (sname)	8124	if (sname)
8053	COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);	8125	COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, internal_strlen(sname) + 1);
8054	void *res = REAL(asysctlbyname)(sname, len);	8126	void *res = REAL(asysctlbyname)(sname, len);
8055	if (res && len) {	8127	if (res && len) {
8056	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len));	8128	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len));
...	@@ -8073,7 +8145,7 @@ INTERCEPTOR(int, sysctlgetmibinfo, char sname, int name,	...	@@ -8073,7 +8145,7 @@ INTERCEPTOR(int, sysctlgetmibinfo, char sname, int name,
8073	COMMON_INTERCEPTOR_ENTER(ctx, sysctlgetmibinfo, sname, name, namelenp, cname,	8145	COMMON_INTERCEPTOR_ENTER(ctx, sysctlgetmibinfo, sname, name, namelenp, cname,
8074	csz, rnode, v);	8146	csz, rnode, v);
8075	if (sname)	8147	if (sname)
8076	COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, REAL(strlen)(sname) + 1);	8148	COMMON_INTERCEPTOR_READ_RANGE(ctx, sname, internal_strlen(sname) + 1);
8077	if (namelenp)	8149	if (namelenp)
8078	COMMON_INTERCEPTOR_READ_RANGE(ctx, namelenp, sizeof(*namelenp));	8150	COMMON_INTERCEPTOR_READ_RANGE(ctx, namelenp, sizeof(*namelenp));
8079	if (csz)	8151	if (csz)
...	@@ -8107,7 +8179,7 @@ INTERCEPTOR(char *, nl_langinfo, long item) {	...	@@ -8107,7 +8179,7 @@ INTERCEPTOR(char *, nl_langinfo, long item) {
8107	COMMON_INTERCEPTOR_ENTER(ctx, nl_langinfo, item);	8179	COMMON_INTERCEPTOR_ENTER(ctx, nl_langinfo, item);
8108	char *ret = REAL(nl_langinfo)(item);	8180	char *ret = REAL(nl_langinfo)(item);
8109	if (ret)	8181	if (ret)
8110	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, REAL(strlen)(ret) + 1);	8182	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, internal_strlen(ret) + 1);
8111	return ret;	8183	return ret;
8112	}	8184	}
8113	#define INIT_NL_LANGINFO COMMON_INTERCEPT_FUNCTION(nl_langinfo)	8185	#define INIT_NL_LANGINFO COMMON_INTERCEPT_FUNCTION(nl_langinfo)
...	@@ -8127,7 +8199,7 @@ INTERCEPTOR(int, modctl, int operation, void *argp) {	...	@@ -8127,7 +8199,7 @@ INTERCEPTOR(int, modctl, int operation, void *argp) {
8127	COMMON_INTERCEPTOR_READ_RANGE(ctx, ml, sizeof(*ml));	8199	COMMON_INTERCEPTOR_READ_RANGE(ctx, ml, sizeof(*ml));
8128	if (ml->ml_filename)	8200	if (ml->ml_filename)
8129	COMMON_INTERCEPTOR_READ_RANGE(ctx, ml->ml_filename,	8201	COMMON_INTERCEPTOR_READ_RANGE(ctx, ml->ml_filename,
8130	REAL(strlen)(ml->ml_filename) + 1);	8202	internal_strlen(ml->ml_filename) + 1);
8131	if (ml->ml_props)	8203	if (ml->ml_props)
8132	COMMON_INTERCEPTOR_READ_RANGE(ctx, ml->ml_props, ml->ml_propslen);	8204	COMMON_INTERCEPTOR_READ_RANGE(ctx, ml->ml_props, ml->ml_propslen);
8133	}	8205	}
...	@@ -8135,7 +8207,7 @@ INTERCEPTOR(int, modctl, int operation, void *argp) {	...	@@ -8135,7 +8207,7 @@ INTERCEPTOR(int, modctl, int operation, void *argp) {
8135	} else if (operation == modctl_unload) {	8207	} else if (operation == modctl_unload) {
8136	if (argp) {	8208	if (argp) {
8137	const char name = (const char )argp;	8209	const char name = (const char )argp;
8138	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1);	8210	COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
8139	}	8211	}
8140	ret = REAL(modctl)(operation, argp);	8212	ret = REAL(modctl)(operation, argp);
8141	} else if (operation == modctl_stat) {	8213	} else if (operation == modctl_stat) {
...	@@ -8177,7 +8249,7 @@ INTERCEPTOR(long long, strtonum, const char *nptr, long long minval,	...	@@ -8177,7 +8249,7 @@ INTERCEPTOR(long long, strtonum, const char *nptr, long long minval,
8177	if (errstr) {	8249	if (errstr) {
8178	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errstr, sizeof(const char *));	8250	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errstr, sizeof(const char *));
8179	if (*errstr)	8251	if (*errstr)
8180	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errstr, REAL(strlen)(errstr) + 1);	8252	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, errstr, internal_strlen(errstr) + 1);
8181	}	8253	}
8182	return ret;	8254	return ret;
8183	}	8255	}
...	@@ -8197,7 +8269,7 @@ INTERCEPTOR(char , fparseln, __sanitizer_FILE stream, SIZE_T *len,	...	@@ -8197,7 +8269,7 @@ INTERCEPTOR(char , fparseln, __sanitizer_FILE stream, SIZE_T *len,
8197	COMMON_INTERCEPTOR_READ_RANGE(ctx, delim, sizeof(delim[0]) * 3);	8269	COMMON_INTERCEPTOR_READ_RANGE(ctx, delim, sizeof(delim[0]) * 3);
8198	char *ret = REAL(fparseln)(stream, len, lineno, delim, flags);	8270	char *ret = REAL(fparseln)(stream, len, lineno, delim, flags);
8199	if (ret) {	8271	if (ret) {
8200	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, REAL(strlen)(ret) + 1);	8272	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, internal_strlen(ret) + 1);
8201	if (len)	8273	if (len)
8202	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len));	8274	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, len, sizeof(*len));
8203	if (lineno)	8275	if (lineno)
...	@@ -8214,7 +8286,7 @@ INTERCEPTOR(char , fparseln, __sanitizer_FILE stream, SIZE_T *len,	...	@@ -8214,7 +8286,7 @@ INTERCEPTOR(char , fparseln, __sanitizer_FILE stream, SIZE_T *len,
8214	INTERCEPTOR(int, statvfs1, const char path, void buf, int flags) {	8286	INTERCEPTOR(int, statvfs1, const char path, void buf, int flags) {
8215	void *ctx;	8287	void *ctx;
8216	COMMON_INTERCEPTOR_ENTER(ctx, statvfs1, path, buf, flags);	8288	COMMON_INTERCEPTOR_ENTER(ctx, statvfs1, path, buf, flags);
8217	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	8289	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
8218	int res = REAL(statvfs1)(path, buf, flags);	8290	int res = REAL(statvfs1)(path, buf, flags);
8219	if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz);	8291	if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz);
8220	return res;	8292	return res;
...	@@ -8495,7 +8567,7 @@ INTERCEPTOR(char , SHA1File, char filename, char *buf) {	...	@@ -8495,7 +8567,7 @@ INTERCEPTOR(char , SHA1File, char filename, char *buf) {
8495	void *ctx;	8567	void *ctx;
8496	COMMON_INTERCEPTOR_ENTER(ctx, SHA1File, filename, buf);	8568	COMMON_INTERCEPTOR_ENTER(ctx, SHA1File, filename, buf);
8497	if (filename)	8569	if (filename)
8498	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);	8570	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
8499	char *ret = REAL(SHA1File)(filename, buf);	8571	char *ret = REAL(SHA1File)(filename, buf);
8500	if (ret)	8572	if (ret)
8501	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length);	8573	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length);
...	@@ -8506,7 +8578,7 @@ INTERCEPTOR(char , SHA1FileChunk, char filename, char *buf, OFF_T offset,	...	@@ -8506,7 +8578,7 @@ INTERCEPTOR(char , SHA1FileChunk, char filename, char *buf, OFF_T offset,
8506	void *ctx;	8578	void *ctx;
8507	COMMON_INTERCEPTOR_ENTER(ctx, SHA1FileChunk, filename, buf, offset, length);	8579	COMMON_INTERCEPTOR_ENTER(ctx, SHA1FileChunk, filename, buf, offset, length);
8508	if (filename)	8580	if (filename)
8509	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);	8581	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
8510	char *ret = REAL(SHA1FileChunk)(filename, buf, offset, length);	8582	char *ret = REAL(SHA1FileChunk)(filename, buf, offset, length);
8511	if (ret)	8583	if (ret)
8512	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length);	8584	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA1_return_length);
...	@@ -8582,7 +8654,7 @@ INTERCEPTOR(char , MD4File, const char filename, char *buf) {	...	@@ -8582,7 +8654,7 @@ INTERCEPTOR(char , MD4File, const char filename, char *buf) {
8582	void *ctx;	8654	void *ctx;
8583	COMMON_INTERCEPTOR_ENTER(ctx, MD4File, filename, buf);	8655	COMMON_INTERCEPTOR_ENTER(ctx, MD4File, filename, buf);
8584	if (filename)	8656	if (filename)
8585	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);	8657	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
8586	char *ret = REAL(MD4File)(filename, buf);	8658	char *ret = REAL(MD4File)(filename, buf);
8587	if (ret)	8659	if (ret)
8588	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD4_return_length);	8660	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD4_return_length);
...	@@ -8665,7 +8737,7 @@ INTERCEPTOR(char , RMD160File, char filename, char *buf) {	...	@@ -8665,7 +8737,7 @@ INTERCEPTOR(char , RMD160File, char filename, char *buf) {
8665	void *ctx;	8737	void *ctx;
8666	COMMON_INTERCEPTOR_ENTER(ctx, RMD160File, filename, buf);	8738	COMMON_INTERCEPTOR_ENTER(ctx, RMD160File, filename, buf);
8667	if (filename)	8739	if (filename)
8668	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);	8740	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
8669	char *ret = REAL(RMD160File)(filename, buf);	8741	char *ret = REAL(RMD160File)(filename, buf);
8670	if (ret)	8742	if (ret)
8671	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length);	8743	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length);
...	@@ -8676,7 +8748,7 @@ INTERCEPTOR(char , RMD160FileChunk, char filename, char *buf, OFF_T offset,	...	@@ -8676,7 +8748,7 @@ INTERCEPTOR(char , RMD160FileChunk, char filename, char *buf, OFF_T offset,
8676	void *ctx;	8748	void *ctx;
8677	COMMON_INTERCEPTOR_ENTER(ctx, RMD160FileChunk, filename, buf, offset, length);	8749	COMMON_INTERCEPTOR_ENTER(ctx, RMD160FileChunk, filename, buf, offset, length);
8678	if (filename)	8750	if (filename)
8679	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);	8751	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
8680	char *ret = REAL(RMD160FileChunk)(filename, buf, offset, length);	8752	char *ret = REAL(RMD160FileChunk)(filename, buf, offset, length);
8681	if (ret)	8753	if (ret)
8682	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length);	8754	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, RMD160_return_length);
...	@@ -8752,7 +8824,7 @@ INTERCEPTOR(char , MD5File, const char filename, char *buf) {	...	@@ -8752,7 +8824,7 @@ INTERCEPTOR(char , MD5File, const char filename, char *buf) {
8752	void *ctx;	8824	void *ctx;
8753	COMMON_INTERCEPTOR_ENTER(ctx, MD5File, filename, buf);	8825	COMMON_INTERCEPTOR_ENTER(ctx, MD5File, filename, buf);
8754	if (filename)	8826	if (filename)
8755	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);	8827	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
8756	char *ret = REAL(MD5File)(filename, buf);	8828	char *ret = REAL(MD5File)(filename, buf);
8757	if (ret)	8829	if (ret)
8758	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length);	8830	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD5_return_length);
...	@@ -8882,7 +8954,7 @@ INTERCEPTOR(char , MD2File, const char filename, char *buf) {	...	@@ -8882,7 +8954,7 @@ INTERCEPTOR(char , MD2File, const char filename, char *buf) {
8882	void *ctx;	8954	void *ctx;
8883	COMMON_INTERCEPTOR_ENTER(ctx, MD2File, filename, buf);	8955	COMMON_INTERCEPTOR_ENTER(ctx, MD2File, filename, buf);
8884	if (filename)	8956	if (filename)
8885	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);	8957	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);
8886	char *ret = REAL(MD2File)(filename, buf);	8958	char *ret = REAL(MD2File)(filename, buf);
8887	if (ret)	8959	if (ret)
8888	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD2_return_length);	8960	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, MD2_return_length);
...	@@ -8960,7 +9032,7 @@ INTERCEPTOR(char , MD2Data, const unsigned char data, unsigned int len,	...	@@ -8960,7 +9032,7 @@ INTERCEPTOR(char , MD2Data, const unsigned char data, unsigned int len,
8960	void *ctx; \	9032	void *ctx; \
8961	COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_File, filename, buf); \	9033	COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_File, filename, buf); \
8962	if (filename) \	9034	if (filename) \
8963	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);\	9035	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);\
8964	char *ret = REAL(SHA##LEN##_File)(filename, buf); \	9036	char *ret = REAL(SHA##LEN##_File)(filename, buf); \
8965	if (ret) \	9037	if (ret) \
8966	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \	9038	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
...	@@ -8972,7 +9044,7 @@ INTERCEPTOR(char , MD2Data, const unsigned char data, unsigned int len,	...	@@ -8972,7 +9044,7 @@ INTERCEPTOR(char , MD2Data, const unsigned char data, unsigned int len,
8972	COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_FileChunk, filename, buf, offset, \	9044	COMMON_INTERCEPTOR_ENTER(ctx, SHA##LEN##_FileChunk, filename, buf, offset, \
8973	length); \	9045	length); \
8974	if (filename) \	9046	if (filename) \
8975	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, REAL(strlen)(filename) + 1);\	9047	COMMON_INTERCEPTOR_READ_RANGE(ctx, filename, internal_strlen(filename) + 1);\
8976	char *ret = REAL(SHA##LEN##_FileChunk)(filename, buf, offset, length); \	9048	char *ret = REAL(SHA##LEN##_FileChunk)(filename, buf, offset, length); \
8977	if (ret) \	9049	if (ret) \
8978	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \	9050	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, SHA##LEN##_return_length); \
...	@@ -8989,10 +9061,10 @@ INTERCEPTOR(char , MD2Data, const unsigned char data, unsigned int len,	...	@@ -8989,10 +9061,10 @@ INTERCEPTOR(char , MD2Data, const unsigned char data, unsigned int len,
8989	return ret; \	9061	return ret; \
8990	}	9062	}
8991		9063
8992	SHA2_INTERCEPTORS(224, u32);	9064	SHA2_INTERCEPTORS(224, u32)
8993	SHA2_INTERCEPTORS(256, u32);	9065	SHA2_INTERCEPTORS(256, u32)
8994	SHA2_INTERCEPTORS(384, u64);	9066	SHA2_INTERCEPTORS(384, u64)
8995	SHA2_INTERCEPTORS(512, u64);	9067	SHA2_INTERCEPTORS(512, u64)
8996		9068
8997	#define INIT_SHA2_INTECEPTORS(LEN) \	9069	#define INIT_SHA2_INTECEPTORS(LEN) \
8998	COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Init); \	9070	COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Init); \
...	@@ -9036,7 +9108,7 @@ INTERCEPTOR(int, strvis, char dst, const char src, int flag) {	...	@@ -9036,7 +9108,7 @@ INTERCEPTOR(int, strvis, char dst, const char src, int flag) {
9036	void *ctx;	9108	void *ctx;
9037	COMMON_INTERCEPTOR_ENTER(ctx, strvis, dst, src, flag);	9109	COMMON_INTERCEPTOR_ENTER(ctx, strvis, dst, src, flag);
9038	if (src)	9110	if (src)
9039	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);	9111	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9040	int len = REAL(strvis)(dst, src, flag);	9112	int len = REAL(strvis)(dst, src, flag);
9041	if (dst)	9113	if (dst)
9042	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1);	9114	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1);
...	@@ -9046,7 +9118,7 @@ INTERCEPTOR(int, stravis, char *dst, const char src, int flag) {	...	@@ -9046,7 +9118,7 @@ INTERCEPTOR(int, stravis, char *dst, const char src, int flag) {
9046	void *ctx;	9118	void *ctx;
9047	COMMON_INTERCEPTOR_ENTER(ctx, stravis, dst, src, flag);	9119	COMMON_INTERCEPTOR_ENTER(ctx, stravis, dst, src, flag);
9048	if (src)	9120	if (src)
9049	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);	9121	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9050	int len = REAL(stravis)(dst, src, flag);	9122	int len = REAL(stravis)(dst, src, flag);
9051	if (dst) {	9123	if (dst) {
9052	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(char *));	9124	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, sizeof(char *));
...	@@ -9059,7 +9131,7 @@ INTERCEPTOR(int, strnvis, char dst, SIZE_T dlen, const char src, int flag) {	...	@@ -9059,7 +9131,7 @@ INTERCEPTOR(int, strnvis, char dst, SIZE_T dlen, const char src, int flag) {
9059	void *ctx;	9131	void *ctx;
9060	COMMON_INTERCEPTOR_ENTER(ctx, strnvis, dst, dlen, src, flag);	9132	COMMON_INTERCEPTOR_ENTER(ctx, strnvis, dst, dlen, src, flag);
9061	if (src)	9133	if (src)
9062	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);	9134	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9063	int len = REAL(strnvis)(dst, dlen, src, flag);	9135	int len = REAL(strnvis)(dst, dlen, src, flag);
9064	// The interface will be valid even if there is no space for NULL char	9136	// The interface will be valid even if there is no space for NULL char
9065	if (dst && len > 0)	9137	if (dst && len > 0)
...	@@ -9109,7 +9181,7 @@ INTERCEPTOR(char , svis, char dst, int c, int flag, int nextc,	...	@@ -9109,7 +9181,7 @@ INTERCEPTOR(char , svis, char dst, int c, int flag, int nextc,
9109	void *ctx;	9181	void *ctx;
9110	COMMON_INTERCEPTOR_ENTER(ctx, svis, dst, c, flag, nextc, extra);	9182	COMMON_INTERCEPTOR_ENTER(ctx, svis, dst, c, flag, nextc, extra);
9111	if (extra)	9183	if (extra)
9112	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);	9184	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
9113	char *end = REAL(svis)(dst, c, flag, nextc, extra);	9185	char *end = REAL(svis)(dst, c, flag, nextc, extra);
9114	if (dst && end)	9186	if (dst && end)
9115	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, end - dst + 1);	9187	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, end - dst + 1);
...	@@ -9120,7 +9192,7 @@ INTERCEPTOR(char , snvis, char dst, SIZE_T dlen, int c, int flag, int nextc,	...	@@ -9120,7 +9192,7 @@ INTERCEPTOR(char , snvis, char dst, SIZE_T dlen, int c, int flag, int nextc,
9120	void *ctx;	9192	void *ctx;
9121	COMMON_INTERCEPTOR_ENTER(ctx, snvis, dst, dlen, c, flag, nextc, extra);	9193	COMMON_INTERCEPTOR_ENTER(ctx, snvis, dst, dlen, c, flag, nextc, extra);
9122	if (extra)	9194	if (extra)
9123	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);	9195	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
9124	char *end = REAL(snvis)(dst, dlen, c, flag, nextc, extra);	9196	char *end = REAL(snvis)(dst, dlen, c, flag, nextc, extra);
9125	if (dst && end)	9197	if (dst && end)
9126	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst,	9198	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst,
...	@@ -9132,9 +9204,9 @@ INTERCEPTOR(int, strsvis, char dst, const char src, int flag,	...	@@ -9132,9 +9204,9 @@ INTERCEPTOR(int, strsvis, char dst, const char src, int flag,
9132	void *ctx;	9204	void *ctx;
9133	COMMON_INTERCEPTOR_ENTER(ctx, strsvis, dst, src, flag, extra);	9205	COMMON_INTERCEPTOR_ENTER(ctx, strsvis, dst, src, flag, extra);
9134	if (src)	9206	if (src)
9135	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);	9207	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9136	if (extra)	9208	if (extra)
9137	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);	9209	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
9138	int len = REAL(strsvis)(dst, src, flag, extra);	9210	int len = REAL(strsvis)(dst, src, flag, extra);
9139	if (dst)	9211	if (dst)
9140	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1);	9212	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, len + 1);
...	@@ -9145,9 +9217,9 @@ INTERCEPTOR(int, strsnvis, char dst, SIZE_T dlen, const char src, int flag,	...	@@ -9145,9 +9217,9 @@ INTERCEPTOR(int, strsnvis, char dst, SIZE_T dlen, const char src, int flag,
9145	void *ctx;	9217	void *ctx;
9146	COMMON_INTERCEPTOR_ENTER(ctx, strsnvis, dst, dlen, src, flag, extra);	9218	COMMON_INTERCEPTOR_ENTER(ctx, strsnvis, dst, dlen, src, flag, extra);
9147	if (src)	9219	if (src)
9148	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);	9220	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9149	if (extra)	9221	if (extra)
9150	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);	9222	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
9151	int len = REAL(strsnvis)(dst, dlen, src, flag, extra);	9223	int len = REAL(strsnvis)(dst, dlen, src, flag, extra);
9152	// The interface will be valid even if there is no space for NULL char	9224	// The interface will be valid even if there is no space for NULL char
9153	if (dst && len >= 0)	9225	if (dst && len >= 0)
...	@@ -9161,7 +9233,7 @@ INTERCEPTOR(int, strsvisx, char dst, const char src, SIZE_T len, int flag,	...	@@ -9161,7 +9233,7 @@ INTERCEPTOR(int, strsvisx, char dst, const char src, SIZE_T len, int flag,
9161	if (src)	9233	if (src)
9162	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);	9234	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);
9163	if (extra)	9235	if (extra)
9164	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);	9236	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
9165	int ret = REAL(strsvisx)(dst, src, len, flag, extra);	9237	int ret = REAL(strsvisx)(dst, src, len, flag, extra);
9166	if (dst)	9238	if (dst)
9167	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);	9239	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
...	@@ -9174,7 +9246,7 @@ INTERCEPTOR(int, strsnvisx, char dst, SIZE_T dlen, const char src, SIZE_T len,	...	@@ -9174,7 +9246,7 @@ INTERCEPTOR(int, strsnvisx, char dst, SIZE_T dlen, const char src, SIZE_T len,
9174	if (src)	9246	if (src)
9175	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);	9247	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);
9176	if (extra)	9248	if (extra)
9177	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);	9249	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
9178	int ret = REAL(strsnvisx)(dst, dlen, src, len, flag, extra);	9250	int ret = REAL(strsnvisx)(dst, dlen, src, len, flag, extra);
9179	if (dst && ret >= 0)	9251	if (dst && ret >= 0)
9180	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);	9252	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
...	@@ -9188,7 +9260,7 @@ INTERCEPTOR(int, strsenvisx, char dst, SIZE_T dlen, const char src,	...	@@ -9188,7 +9260,7 @@ INTERCEPTOR(int, strsenvisx, char dst, SIZE_T dlen, const char src,
9188	if (src)	9260	if (src)
9189	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);	9261	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, len);
9190	if (extra)	9262	if (extra)
9191	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, REAL(strlen)(extra) + 1);	9263	COMMON_INTERCEPTOR_READ_RANGE(ctx, extra, internal_strlen(extra) + 1);
9192	// FIXME: only need to be checked when "flag \| VIS_NOLOCALE" doesn't hold	9264	// FIXME: only need to be checked when "flag \| VIS_NOLOCALE" doesn't hold
9193	// according to the implementation	9265	// according to the implementation
9194	if (cerr_ptr)	9266	if (cerr_ptr)
...	@@ -9215,7 +9287,7 @@ INTERCEPTOR(int, strunvis, char dst, const char src) {	...	@@ -9215,7 +9287,7 @@ INTERCEPTOR(int, strunvis, char dst, const char src) {
9215	void *ctx;	9287	void *ctx;
9216	COMMON_INTERCEPTOR_ENTER(ctx, strunvis, dst, src);	9288	COMMON_INTERCEPTOR_ENTER(ctx, strunvis, dst, src);
9217	if (src)	9289	if (src)
9218	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);	9290	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9219	int ret = REAL(strunvis)(dst, src);	9291	int ret = REAL(strunvis)(dst, src);
9220	if (ret != -1)	9292	if (ret != -1)
9221	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);	9293	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
...	@@ -9225,7 +9297,7 @@ INTERCEPTOR(int, strnunvis, char dst, SIZE_T dlen, const char src) {	...	@@ -9225,7 +9297,7 @@ INTERCEPTOR(int, strnunvis, char dst, SIZE_T dlen, const char src) {
9225	void *ctx;	9297	void *ctx;
9226	COMMON_INTERCEPTOR_ENTER(ctx, strnunvis, dst, dlen, src);	9298	COMMON_INTERCEPTOR_ENTER(ctx, strnunvis, dst, dlen, src);
9227	if (src)	9299	if (src)
9228	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);	9300	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9229	int ret = REAL(strnunvis)(dst, dlen, src);	9301	int ret = REAL(strnunvis)(dst, dlen, src);
9230	if (ret != -1)	9302	if (ret != -1)
9231	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);	9303	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
...	@@ -9235,7 +9307,7 @@ INTERCEPTOR(int, strunvisx, char dst, const char src, int flag) {	...	@@ -9235,7 +9307,7 @@ INTERCEPTOR(int, strunvisx, char dst, const char src, int flag) {
9235	void *ctx;	9307	void *ctx;
9236	COMMON_INTERCEPTOR_ENTER(ctx, strunvisx, dst, src, flag);	9308	COMMON_INTERCEPTOR_ENTER(ctx, strunvisx, dst, src, flag);
9237	if (src)	9309	if (src)
9238	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);	9310	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9239	int ret = REAL(strunvisx)(dst, src, flag);	9311	int ret = REAL(strunvisx)(dst, src, flag);
9240	if (ret != -1)	9312	if (ret != -1)
9241	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);	9313	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
...	@@ -9246,7 +9318,7 @@ INTERCEPTOR(int, strnunvisx, char dst, SIZE_T dlen, const char src,	...	@@ -9246,7 +9318,7 @@ INTERCEPTOR(int, strnunvisx, char dst, SIZE_T dlen, const char src,
9246	void *ctx;	9318	void *ctx;
9247	COMMON_INTERCEPTOR_ENTER(ctx, strnunvisx, dst, dlen, src, flag);	9319	COMMON_INTERCEPTOR_ENTER(ctx, strnunvisx, dst, dlen, src, flag);
9248	if (src)	9320	if (src)
9249	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, REAL(strlen)(src) + 1);	9321	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, internal_strlen(src) + 1);
9250	int ret = REAL(strnunvisx)(dst, dlen, src, flag);	9322	int ret = REAL(strnunvisx)(dst, dlen, src, flag);
9251	if (ret != -1)	9323	if (ret != -1)
9252	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);	9324	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, ret + 1);
...	@@ -9282,7 +9354,7 @@ INTERCEPTOR(struct __sanitizer_cdbr , cdbr_open, const char path, int flags) {	...	@@ -9282,7 +9354,7 @@ INTERCEPTOR(struct __sanitizer_cdbr , cdbr_open, const char path, int flags) {
9282	void *ctx;	9354	void *ctx;
9283	COMMON_INTERCEPTOR_ENTER(ctx, cdbr_open, path, flags);	9355	COMMON_INTERCEPTOR_ENTER(ctx, cdbr_open, path, flags);
9284	if (path)	9356	if (path)
9285	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	9357	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
9286	struct __sanitizer_cdbr *cdbr = REAL(cdbr_open)(path, flags);	9358	struct __sanitizer_cdbr *cdbr = REAL(cdbr_open)(path, flags);
9287	if (cdbr)	9359	if (cdbr)
9288	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbr, sizeof(*cdbr));	9360	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cdbr, sizeof(*cdbr));
...	@@ -9474,7 +9546,7 @@ INTERCEPTOR(void , getfsspec, const char spec) {	...	@@ -9474,7 +9546,7 @@ INTERCEPTOR(void , getfsspec, const char spec) {
9474	void *ctx;	9546	void *ctx;
9475	COMMON_INTERCEPTOR_ENTER(ctx, getfsspec, spec);	9547	COMMON_INTERCEPTOR_ENTER(ctx, getfsspec, spec);
9476	if (spec)	9548	if (spec)
9477	COMMON_INTERCEPTOR_READ_RANGE(ctx, spec, REAL(strlen)(spec) + 1);	9549	COMMON_INTERCEPTOR_READ_RANGE(ctx, spec, internal_strlen(spec) + 1);
9478	void *ret = REAL(getfsspec)(spec);	9550	void *ret = REAL(getfsspec)(spec);
9479	if (ret)	9551	if (ret)
9480	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz);	9552	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz);
...	@@ -9485,7 +9557,7 @@ INTERCEPTOR(void , getfsfile, const char file) {	...	@@ -9485,7 +9557,7 @@ INTERCEPTOR(void , getfsfile, const char file) {
9485	void *ctx;	9557	void *ctx;
9486	COMMON_INTERCEPTOR_ENTER(ctx, getfsfile, file);	9558	COMMON_INTERCEPTOR_ENTER(ctx, getfsfile, file);
9487	if (file)	9559	if (file)
9488	COMMON_INTERCEPTOR_READ_RANGE(ctx, file, REAL(strlen)(file) + 1);	9560	COMMON_INTERCEPTOR_READ_RANGE(ctx, file, internal_strlen(file) + 1);
9489	void *ret = REAL(getfsfile)(file);	9561	void *ret = REAL(getfsfile)(file);
9490	if (ret)	9562	if (ret)
9491	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz);	9563	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ret, struct_fstab_sz);
...	@@ -9529,9 +9601,9 @@ INTERCEPTOR(__sanitizer_FILE , popen, const char command, const char *type) {	...	@@ -9529,9 +9601,9 @@ INTERCEPTOR(__sanitizer_FILE , popen, const char command, const char *type) {
9529	void *ctx;	9601	void *ctx;
9530	COMMON_INTERCEPTOR_ENTER(ctx, popen, command, type);	9602	COMMON_INTERCEPTOR_ENTER(ctx, popen, command, type);
9531	if (command)	9603	if (command)
9532	COMMON_INTERCEPTOR_READ_RANGE(ctx, command, REAL(strlen)(command) + 1);	9604	COMMON_INTERCEPTOR_READ_RANGE(ctx, command, internal_strlen(command) + 1);
9533	if (type)	9605	if (type)
9534	COMMON_INTERCEPTOR_READ_RANGE(ctx, type, REAL(strlen)(type) + 1);	9606	COMMON_INTERCEPTOR_READ_RANGE(ctx, type, internal_strlen(type) + 1);
9535	__sanitizer_FILE *res = REAL(popen)(command, type);	9607	__sanitizer_FILE *res = REAL(popen)(command, type);
9536	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, nullptr);	9608	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, nullptr);
9537	if (res) unpoison_file(res);	9609	if (res) unpoison_file(res);
...	@@ -9548,13 +9620,13 @@ INTERCEPTOR(__sanitizer_FILE , popenve, const char path,	...	@@ -9548,13 +9620,13 @@ INTERCEPTOR(__sanitizer_FILE , popenve, const char path,
9548	void *ctx;	9620	void *ctx;
9549	COMMON_INTERCEPTOR_ENTER(ctx, popenve, path, argv, envp, type);	9621	COMMON_INTERCEPTOR_ENTER(ctx, popenve, path, argv, envp, type);
9550	if (path)	9622	if (path)
9551	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	9623	COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
9552	if (argv) {	9624	if (argv) {
9553	for (char const pa = argv; ; ++pa) {	9625	for (char const pa = argv; ; ++pa) {
9554	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **));	9626	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **));
9555	if (!*pa)	9627	if (!*pa)
9556	break;	9628	break;
9557	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, REAL(strlen)(pa) + 1);	9629	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, internal_strlen(pa) + 1);
9558	}	9630	}
9559	}	9631	}
9560	if (envp) {	9632	if (envp) {
...	@@ -9562,11 +9634,11 @@ INTERCEPTOR(__sanitizer_FILE , popenve, const char path,	...	@@ -9562,11 +9634,11 @@ INTERCEPTOR(__sanitizer_FILE , popenve, const char path,
9562	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **));	9634	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, sizeof(char **));
9563	if (!*pa)	9635	if (!*pa)
9564	break;	9636	break;
9565	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, REAL(strlen)(pa) + 1);	9637	COMMON_INTERCEPTOR_READ_RANGE(ctx, pa, internal_strlen(pa) + 1);
9566	}	9638	}
9567	}	9639	}
9568	if (type)	9640	if (type)
9569	COMMON_INTERCEPTOR_READ_RANGE(ctx, type, REAL(strlen)(type) + 1);	9641	COMMON_INTERCEPTOR_READ_RANGE(ctx, type, internal_strlen(type) + 1);
9570	__sanitizer_FILE *res = REAL(popenve)(path, argv, envp, type);	9642	__sanitizer_FILE *res = REAL(popenve)(path, argv, envp, type);
9571	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, nullptr);	9643	COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, nullptr);
9572	if (res) unpoison_file(res);	9644	if (res) unpoison_file(res);
...	@@ -9762,7 +9834,7 @@ INTERCEPTOR(char *, fdevname, int fd) {	...	@@ -9762,7 +9834,7 @@ INTERCEPTOR(char *, fdevname, int fd) {
9762	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	9834	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
9763	char *name = REAL(fdevname)(fd);	9835	char *name = REAL(fdevname)(fd);
9764	if (name) {	9836	if (name) {
9765	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, REAL(strlen)(name) + 1);	9837	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
9766	if (fd > 0)	9838	if (fd > 0)
9767	COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);	9839	COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
9768	}	9840	}
...	@@ -9775,7 +9847,7 @@ INTERCEPTOR(char , fdevname_r, int fd, char buf, SIZE_T len) {	...	@@ -9775,7 +9847,7 @@ INTERCEPTOR(char , fdevname_r, int fd, char buf, SIZE_T len) {
9775	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);	9847	COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd);
9776	char *name = REAL(fdevname_r)(fd, buf, len);	9848	char *name = REAL(fdevname_r)(fd, buf, len);
9777	if (name && buf && len > 0) {	9849	if (name && buf && len > 0) {
9778	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, REAL(strlen)(buf) + 1);	9850	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
9779	if (fd > 0)	9851	if (fd > 0)
9780	COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);	9852	COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd);
9781	}	9853	}
...	@@ -9795,7 +9867,7 @@ INTERCEPTOR(char *, getusershell) {	...	@@ -9795,7 +9867,7 @@ INTERCEPTOR(char *, getusershell) {
9795	COMMON_INTERCEPTOR_ENTER(ctx, getusershell);	9867	COMMON_INTERCEPTOR_ENTER(ctx, getusershell);
9796	char *res = REAL(getusershell)();	9868	char *res = REAL(getusershell)();
9797	if (res)	9869	if (res)
9798	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);	9870	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
9799	return res;	9871	return res;
9800	}	9872	}
9801		9873
...	@@ -9820,7 +9892,7 @@ INTERCEPTOR(int, sl_add, void sl, char item) {	...	@@ -9820,7 +9892,7 @@ INTERCEPTOR(int, sl_add, void sl, char item) {
9820	if (sl)	9892	if (sl)
9821	COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);	9893	COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);
9822	if (item)	9894	if (item)
9823	COMMON_INTERCEPTOR_READ_RANGE(ctx, item, REAL(strlen)(item) + 1);	9895	COMMON_INTERCEPTOR_READ_RANGE(ctx, item, internal_strlen(item) + 1);
9824	int res = REAL(sl_add)(sl, item);	9896	int res = REAL(sl_add)(sl, item);
9825	if (!res)	9897	if (!res)
9826	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);	9898	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);
...	@@ -9833,10 +9905,10 @@ INTERCEPTOR(char , sl_find, void sl, const char *item) {	...	@@ -9833,10 +9905,10 @@ INTERCEPTOR(char , sl_find, void sl, const char *item) {
9833	if (sl)	9905	if (sl)
9834	COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);	9906	COMMON_INTERCEPTOR_READ_RANGE(ctx, sl, __sanitizer::struct_StringList_sz);
9835	if (item)	9907	if (item)
9836	COMMON_INTERCEPTOR_READ_RANGE(ctx, item, REAL(strlen)(item) + 1);	9908	COMMON_INTERCEPTOR_READ_RANGE(ctx, item, internal_strlen(item) + 1);
9837	char *res = REAL(sl_find)(sl, item);	9909	char *res = REAL(sl_find)(sl, item);
9838	if (res)	9910	if (res)
9839	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1);	9911	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, internal_strlen(res) + 1);
9840	return res;	9912	return res;
9841	}	9913	}
9842		9914
...	@@ -9872,41 +9944,6 @@ INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) {	...	@@ -9872,41 +9944,6 @@ INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) {
9872	#define INIT_GETRANDOM	9944	#define INIT_GETRANDOM
9873	#endif	9945	#endif
9874		9946
9875	#if SANITIZER_INTERCEPT_CRYPT
9876	INTERCEPTOR(char , crypt, char key, char *salt) {
9877	void *ctx;
9878	COMMON_INTERCEPTOR_ENTER(ctx, crypt, key, salt);
9879	COMMON_INTERCEPTOR_READ_RANGE(ctx, key, internal_strlen(key) + 1);
9880	COMMON_INTERCEPTOR_READ_RANGE(ctx, salt, internal_strlen(salt) + 1);
9881	char *res = REAL(crypt)(key, salt);
9882	if (res != nullptr)
9883	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
9884	return res;
9885	}
9886	#define INIT_CRYPT COMMON_INTERCEPT_FUNCTION(crypt);
9887	#else
9888	#define INIT_CRYPT
9889	#endif
9890
9891	#if SANITIZER_INTERCEPT_CRYPT_R
9892	INTERCEPTOR(char , crypt_r, char key, char salt, void data) {
9893	void *ctx;
9894	COMMON_INTERCEPTOR_ENTER(ctx, crypt_r, key, salt, data);
9895	COMMON_INTERCEPTOR_READ_RANGE(ctx, key, internal_strlen(key) + 1);
9896	COMMON_INTERCEPTOR_READ_RANGE(ctx, salt, internal_strlen(salt) + 1);
9897	char *res = REAL(crypt_r)(key, salt, data);
9898	if (res != nullptr) {
9899	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data,
9900	__sanitizer::struct_crypt_data_sz);
9901	COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
9902	}
9903	return res;
9904	}
9905	#define INIT_CRYPT_R COMMON_INTERCEPT_FUNCTION(crypt_r);
9906	#else
9907	#define INIT_CRYPT_R
9908	#endif
9909
9910	#if SANITIZER_INTERCEPT_GETENTROPY	9947	#if SANITIZER_INTERCEPT_GETENTROPY
9911	INTERCEPTOR(int, getentropy, void *buf, SIZE_T buflen) {	9948	INTERCEPTOR(int, getentropy, void *buf, SIZE_T buflen) {
9912	void *ctx;	9949	void *ctx;
...	@@ -9922,7 +9959,52 @@ INTERCEPTOR(int, getentropy, void *buf, SIZE_T buflen) {	...	@@ -9922,7 +9959,52 @@ INTERCEPTOR(int, getentropy, void *buf, SIZE_T buflen) {
9922	#define INIT_GETENTROPY	9959	#define INIT_GETENTROPY
9923	#endif	9960	#endif
9924		9961
9925	#if SANITIZER_INTERCEPT_QSORT	9962	#if SANITIZER_INTERCEPT_QSORT_R
		9963	typedef int (qsort_r_compar_f)(const void , const void , void );
		9964	struct qsort_r_compar_params {
		9965	SIZE_T size;
		9966	qsort_r_compar_f compar;
		9967	void *arg;
		9968	};
		9969	static int wrapped_qsort_r_compar(const void a, const void b, void *arg) {
		9970	qsort_r_compar_params params = (qsort_r_compar_params )arg;
		9971	COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
		9972	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, params->size);
		9973	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, params->size);
		9974	return params->compar(a, b, params->arg);
		9975	}
		9976
		9977	INTERCEPTOR(void, qsort_r, void *base, SIZE_T nmemb, SIZE_T size,
		9978	qsort_r_compar_f compar, void *arg) {
		9979	void *ctx;
		9980	COMMON_INTERCEPTOR_ENTER(ctx, qsort_r, base, nmemb, size, compar, arg);
		9981	// Run the comparator over all array elements to detect any memory issues.
		9982	if (nmemb > 1) {
		9983	for (SIZE_T i = 0; i < nmemb - 1; ++i) {
		9984	void p = (void )((char )base + i size);
		9985	void q = (void )((char )base + (i + 1) size);
		9986	COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
		9987	compar(p, q, arg);
		9988	}
		9989	}
		9990	qsort_r_compar_params params = {size, compar, arg};
		9991	REAL(qsort_r)(base, nmemb, size, wrapped_qsort_r_compar, &params);
		9992	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);
		9993	}
		9994	# define INIT_QSORT_R COMMON_INTERCEPT_FUNCTION(qsort_r)
		9995	#else
		9996	# define INIT_QSORT_R
		9997	#endif
		9998
		9999	#if SANITIZER_INTERCEPT_QSORT && SANITIZER_INTERCEPT_QSORT_R
		10000	INTERCEPTOR(void, qsort, void *base, SIZE_T nmemb, SIZE_T size,
		10001	qsort_r_compar_f compar) {
		10002	void *ctx;
		10003	COMMON_INTERCEPTOR_ENTER(ctx, qsort, base, nmemb, size, compar);
		10004	WRAP(qsort_r)(base, nmemb, size, compar, nullptr);
		10005	}
		10006	# define INIT_QSORT COMMON_INTERCEPT_FUNCTION(qsort)
		10007	#elif SANITIZER_INTERCEPT_QSORT && !SANITIZER_INTERCEPT_QSORT_R
9926	// Glibc qsort uses a temporary buffer allocated either on stack or on heap.	10008	// Glibc qsort uses a temporary buffer allocated either on stack or on heap.
9927	// Poisoned memory from there may get copied into the comparator arguments,	10009	// Poisoned memory from there may get copied into the comparator arguments,
9928	// where it needs to be dealt with. But even that is not enough - the results of	10010	// where it needs to be dealt with. But even that is not enough - the results of
...	@@ -9937,7 +10019,7 @@ INTERCEPTOR(int, getentropy, void *buf, SIZE_T buflen) {	...	@@ -9937,7 +10019,7 @@ INTERCEPTOR(int, getentropy, void *buf, SIZE_T buflen) {
9937	typedef int (qsort_compar_f)(const void , const void *);	10019	typedef int (qsort_compar_f)(const void , const void *);
9938	static THREADLOCAL qsort_compar_f qsort_compar;	10020	static THREADLOCAL qsort_compar_f qsort_compar;
9939	static THREADLOCAL SIZE_T qsort_size;	10021	static THREADLOCAL SIZE_T qsort_size;
9940	int wrapped_qsort_compar(const void a, const void b) {	10022	static int wrapped_qsort_compar(const void a, const void b) {
9941	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);	10023	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
9942	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, qsort_size);	10024	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, qsort_size);
9943	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, qsort_size);	10025	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, qsort_size);
...	@@ -9979,60 +10061,34 @@ INTERCEPTOR(void, qsort, void *base, SIZE_T nmemb, SIZE_T size,	...	@@ -9979,60 +10061,34 @@ INTERCEPTOR(void, qsort, void *base, SIZE_T nmemb, SIZE_T size,
9979	}	10061	}
9980	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);	10062	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);
9981	}	10063	}
9982	#define INIT_QSORT COMMON_INTERCEPT_FUNCTION(qsort)	10064	# define INIT_QSORT COMMON_INTERCEPT_FUNCTION(qsort)
9983	#else	10065	#else
9984	#define INIT_QSORT	10066	# define INIT_QSORT
9985	#endif	10067	#endif
9986		10068
9987	#if SANITIZER_INTERCEPT_QSORT_R	10069	#if SANITIZER_INTERCEPT_BSEARCH
9988	typedef int (qsort_r_compar_f)(const void , const void , void );	10070	typedef int (bsearch_compar_f)(const void , const void *);
9989	static THREADLOCAL qsort_r_compar_f qsort_r_compar;	10071	struct bsearch_compar_params {
9990	static THREADLOCAL SIZE_T qsort_r_size;	10072	const void *key;
9991	int wrapped_qsort_r_compar(const void a, const void b, void *arg) {	10073	bsearch_compar_f compar;
9992	COMMON_INTERCEPTOR_UNPOISON_PARAM(3);	10074	};
9993	COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, qsort_r_size);	10075
9994	COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, qsort_r_size);	10076	static int wrapped_bsearch_compar(const void key, const void b) {
9995	return qsort_r_compar(a, b, arg);	10077	const bsearch_compar_params params = (const bsearch_compar_params )key;
		10078	COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
		10079	return params->compar(params->key, b);
9996	}	10080	}
9997		10081
9998	INTERCEPTOR(void, qsort_r, void *base, SIZE_T nmemb, SIZE_T size,	10082	INTERCEPTOR(void , bsearch, const void key, const void *base, SIZE_T nmemb,
9999	qsort_r_compar_f compar, void *arg) {	10083	SIZE_T size, bsearch_compar_f compar) {
10000	void *ctx;	10084	void *ctx;
10001	COMMON_INTERCEPTOR_ENTER(ctx, qsort_r, base, nmemb, size, compar, arg);	10085	COMMON_INTERCEPTOR_ENTER(ctx, bsearch, key, base, nmemb, size, compar);
10002	// Run the comparator over all array elements to detect any memory issues.	10086	bsearch_compar_params params = {key, compar};
10003	if (nmemb > 1) {	10087	return REAL(bsearch)(&params, base, nmemb, size, wrapped_bsearch_compar);
10004	for (SIZE_T i = 0; i < nmemb - 1; ++i) {
10005	void p = (void )((char )base + i size);
10006	void q = (void )((char )base + (i + 1) size);
10007	COMMON_INTERCEPTOR_UNPOISON_PARAM(3);
10008	compar(p, q, arg);
10009	}
10010	}
10011	qsort_r_compar_f old_compar = qsort_r_compar;
10012	SIZE_T old_size = qsort_r_size;
10013	// Handle qsort_r() implementations that recurse using an
10014	// interposable function call:
10015	bool already_wrapped = compar == wrapped_qsort_r_compar;
10016	if (already_wrapped) {
10017	// This case should only happen if the qsort() implementation calls itself
10018	// using a preemptible function call (e.g. the FreeBSD libc version).
10019	// Check that the size and comparator arguments are as expected.
10020	CHECK_NE(compar, qsort_r_compar);
10021	CHECK_EQ(qsort_r_size, size);
10022	} else {
10023	qsort_r_compar = compar;
10024	qsort_r_size = size;
10025	}
10026	REAL(qsort_r)(base, nmemb, size, wrapped_qsort_r_compar, arg);
10027	if (!already_wrapped) {
10028	qsort_r_compar = old_compar;
10029	qsort_r_size = old_size;
10030	}
10031	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, base, nmemb * size);
10032	}	10088	}
10033	#define INIT_QSORT_R COMMON_INTERCEPT_FUNCTION(qsort_r)	10089	# define INIT_BSEARCH COMMON_INTERCEPT_FUNCTION(bsearch)
10034	#else	10090	#else
10035	#define INIT_QSORT_R	10091	# define INIT_BSEARCH
10036	#endif	10092	#endif
10037		10093
10038	#if SANITIZER_INTERCEPT_SIGALTSTACK	10094	#if SANITIZER_INTERCEPT_SIGALTSTACK
...	@@ -10050,6 +10106,42 @@ INTERCEPTOR(int, sigaltstack, void ss, void oss) {	...	@@ -10050,6 +10106,42 @@ INTERCEPTOR(int, sigaltstack, void ss, void oss) {
10050	#define INIT_SIGALTSTACK	10106	#define INIT_SIGALTSTACK
10051	#endif	10107	#endif
10052		10108
		10109	#if SANITIZER_INTERCEPT_PROCCTL
		10110	INTERCEPTOR(int, procctl, int idtype, u64 id, int cmd, uptr data) {
		10111	void *ctx;
		10112	COMMON_INTERCEPTOR_ENTER(ctx, procctl, idtype, id, cmd, data);
		10113	static const int PROC_REAP_ACQUIRE = 2;
		10114	static const int PROC_REAP_RELEASE = 3;
		10115	static const int PROC_REAP_STATUS = 4;
		10116	static const int PROC_REAP_GETPIDS = 5;
		10117	static const int PROC_REAP_KILL = 6;
		10118	if (cmd < PROC_REAP_ACQUIRE \|\| cmd > PROC_REAP_KILL) {
		10119	COMMON_INTERCEPTOR_READ_RANGE(ctx, (void *)data, sizeof(int));
		10120	} else {
		10121	// reap_acquire/reap_release bears no arguments.
		10122	if (cmd > PROC_REAP_RELEASE) {
		10123	unsigned int reapsz;
		10124	switch (cmd) {
		10125	case PROC_REAP_STATUS:
		10126	reapsz = struct_procctl_reaper_status_sz;
		10127	break;
		10128	case PROC_REAP_GETPIDS:
		10129	reapsz = struct_procctl_reaper_pids_sz;
		10130	break;
		10131	case PROC_REAP_KILL:
		10132	reapsz = struct_procctl_reaper_kill_sz;
		10133	break;
		10134	}
		10135	COMMON_INTERCEPTOR_READ_RANGE(ctx, (void *)data, reapsz);
		10136	}
		10137	}
		10138	return REAL(procctl)(idtype, id, cmd, data);
		10139	}
		10140	#define INIT_PROCCTL COMMON_INTERCEPT_FUNCTION(procctl)
		10141	#else
		10142	#define INIT_PROCCTL
		10143	#endif
		10144
10053	#if SANITIZER_INTERCEPT_UNAME	10145	#if SANITIZER_INTERCEPT_UNAME
10054	INTERCEPTOR(int, uname, struct utsname *utsname) {	10146	INTERCEPTOR(int, uname, struct utsname *utsname) {
10055	#if SANITIZER_LINUX	10147	#if SANITIZER_LINUX
...	@@ -10088,14 +10180,66 @@ INTERCEPTOR(int, __xuname, int size, void *utsname) {	...	@@ -10088,14 +10180,66 @@ INTERCEPTOR(int, __xuname, int size, void *utsname) {
10088	#define INIT___XUNAME	10180	#define INIT___XUNAME
10089	#endif	10181	#endif
10090		10182
		10183	#if SANITIZER_INTERCEPT_HEXDUMP
		10184	INTERCEPTOR(void, hexdump, const void ptr, int length, const char header, int flags) {
		10185	void *ctx;
		10186	COMMON_INTERCEPTOR_ENTER(ctx, hexdump, ptr, length, header, flags);
		10187	COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, length);
		10188	COMMON_INTERCEPTOR_READ_RANGE(ctx, header, internal_strlen(header) + 1);
		10189	REAL(hexdump)(ptr, length, header, flags);
		10190	}
		10191
		10192	#define INIT_HEXDUMP COMMON_INTERCEPT_FUNCTION(hexdump);
		10193	#else
		10194	#define INIT_HEXDUMP
		10195	#endif
		10196
		10197	#if SANITIZER_INTERCEPT_ARGP_PARSE
		10198	INTERCEPTOR(int, argp_parse, const struct argp argp, int argc, char *argv,
		10199	unsigned flags, int arg_index, void input) {
		10200	void *ctx;
		10201	COMMON_INTERCEPTOR_ENTER(ctx, argp_parse, argp, argc, argv, flags, arg_index,
		10202	input);
		10203	for (int i = 0; i < argc; i++)
		10204	COMMON_INTERCEPTOR_READ_RANGE(ctx, argv[i], internal_strlen(argv[i]) + 1);
		10205	int res = REAL(argp_parse)(argp, argc, argv, flags, arg_index, input);
		10206	if (!res && arg_index)
		10207	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, arg_index, sizeof(int));
		10208	return res;
		10209	}
		10210
		10211	#define INIT_ARGP_PARSE COMMON_INTERCEPT_FUNCTION(argp_parse);
		10212	#else
		10213	#define INIT_ARGP_PARSE
		10214	#endif
		10215
		10216	#if SANITIZER_INTERCEPT_CPUSET_GETAFFINITY
		10217	INTERCEPTOR(int, cpuset_getaffinity, int level, int which, __int64_t id, SIZE_T cpusetsize, __sanitizer_cpuset_t *mask) {
		10218	void *ctx;
		10219	COMMON_INTERCEPTOR_ENTER(ctx, cpuset_getaffinity, level, which, id, cpusetsize, mask);
		10220	int res = REAL(cpuset_getaffinity)(level, which, id, cpusetsize, mask);
		10221	if (mask && !res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mask, cpusetsize);
		10222	return res;
		10223	}
		10224	#define INIT_CPUSET_GETAFFINITY COMMON_INTERCEPT_FUNCTION(cpuset_getaffinity);
		10225	#else
		10226	#define INIT_CPUSET_GETAFFINITY
		10227	#endif
		10228
10091	#include "sanitizer_common_interceptors_netbsd_compat.inc"	10229	#include "sanitizer_common_interceptors_netbsd_compat.inc"
10092		10230
		10231	namespace __sanitizer {
		10232	void InitializeMemintrinsicInterceptors();
		10233	} // namespace __sanitizer
		10234
10093	static void InitializeCommonInterceptors() {	10235	static void InitializeCommonInterceptors() {
10094	#if SI_POSIX	10236	#if SI_POSIX
10095	static u64 metadata_mem[sizeof(MetadataHashMap) / sizeof(u64) + 1];	10237	static u64 metadata_mem[sizeof(MetadataHashMap) / sizeof(u64) + 1];
10096	interceptor_metadata_map = new ((void *)&metadata_mem) MetadataHashMap();	10238	interceptor_metadata_map = new ((void *)&metadata_mem) MetadataHashMap();
10097	#endif	10239	#endif
10098		10240
		10241	__sanitizer::InitializeMemintrinsicInterceptors();
		10242
10099	INIT_MMAP;	10243	INIT_MMAP;
10100	INIT_MMAP64;	10244	INIT_MMAP64;
10101	INIT_TEXTDOMAIN;	10245	INIT_TEXTDOMAIN;
...	@@ -10117,9 +10261,6 @@ static void InitializeCommonInterceptors() {	...	@@ -10117,9 +10261,6 @@ static void InitializeCommonInterceptors() {
10117	INIT_STRPBRK;	10261	INIT_STRPBRK;
10118	INIT_STRXFRM;	10262	INIT_STRXFRM;
10119	INIT___STRXFRM_L;	10263	INIT___STRXFRM_L;
10120	INIT_MEMSET;
10121	INIT_MEMMOVE;
10122	INIT_MEMCPY;
10123	INIT_MEMCHR;	10264	INIT_MEMCHR;
10124	INIT_MEMCMP;	10265	INIT_MEMCMP;
10125	INIT_BCMP;	10266	INIT_BCMP;
...	@@ -10166,6 +10307,9 @@ static void InitializeCommonInterceptors() {	...	@@ -10166,6 +10307,9 @@ static void InitializeCommonInterceptors() {
10166	INIT_TIME;	10307	INIT_TIME;
10167	INIT_GLOB;	10308	INIT_GLOB;
10168	INIT_GLOB64;	10309	INIT_GLOB64;
		10310	INIT___B64_TO;
		10311	INIT_DN_COMP_EXPAND;
		10312	INIT_POSIX_SPAWN;
10169	INIT_WAIT;	10313	INIT_WAIT;
10170	INIT_WAIT4;	10314	INIT_WAIT4;
10171	INIT_INET;	10315	INIT_INET;
...	@@ -10200,6 +10344,7 @@ static void InitializeCommonInterceptors() {	...	@@ -10200,6 +10344,7 @@ static void InitializeCommonInterceptors() {
10200	INIT_GETCWD;	10344	INIT_GETCWD;
10201	INIT_GET_CURRENT_DIR_NAME;	10345	INIT_GET_CURRENT_DIR_NAME;
10202	INIT_STRTOIMAX;	10346	INIT_STRTOIMAX;
		10347	INIT_STRTOIMAX_C23;
10203	INIT_MBSTOWCS;	10348	INIT_MBSTOWCS;
10204	INIT_MBSNRTOWCS;	10349	INIT_MBSNRTOWCS;
10205	INIT_WCSTOMBS;	10350	INIT_WCSTOMBS;
...	@@ -10231,12 +10376,6 @@ static void InitializeCommonInterceptors() {	...	@@ -10231,12 +10376,6 @@ static void InitializeCommonInterceptors() {
10231	INIT_PTHREAD_SIGMASK;	10376	INIT_PTHREAD_SIGMASK;
10232	INIT_BACKTRACE;	10377	INIT_BACKTRACE;
10233	INIT__EXIT;	10378	INIT__EXIT;
10234	INIT_PTHREAD_MUTEX_LOCK;
10235	INIT_PTHREAD_MUTEX_UNLOCK;
10236	INIT___PTHREAD_MUTEX_LOCK;
10237	INIT___PTHREAD_MUTEX_UNLOCK;
10238	INIT___LIBC_MUTEX_LOCK;
10239	INIT___LIBC_MUTEX_UNLOCK;
10240	INIT___LIBC_THR_SETCANCELSTATE;	10379	INIT___LIBC_THR_SETCANCELSTATE;
10241	INIT_GETMNTENT;	10380	INIT_GETMNTENT;
10242	INIT_GETMNTENT_R;	10381	INIT_GETMNTENT_R;
...	@@ -10254,6 +10393,7 @@ static void InitializeCommonInterceptors() {	...	@@ -10254,6 +10393,7 @@ static void InitializeCommonInterceptors() {
10254	INIT_PTHREAD_ATTR_GET_SCHED;	10393	INIT_PTHREAD_ATTR_GET_SCHED;
10255	INIT_PTHREAD_ATTR_GETINHERITSCHED;	10394	INIT_PTHREAD_ATTR_GETINHERITSCHED;
10256	INIT_PTHREAD_ATTR_GETAFFINITY_NP;	10395	INIT_PTHREAD_ATTR_GETAFFINITY_NP;
		10396	INIT_PTHREAD_GETAFFINITY_NP;
10257	INIT_PTHREAD_MUTEXATTR_GETPSHARED;	10397	INIT_PTHREAD_MUTEXATTR_GETPSHARED;
10258	INIT_PTHREAD_MUTEXATTR_GETTYPE;	10398	INIT_PTHREAD_MUTEXATTR_GETTYPE;
10259	INIT_PTHREAD_MUTEXATTR_GETPROTOCOL;	10399	INIT_PTHREAD_MUTEXATTR_GETPROTOCOL;
...	@@ -10293,9 +10433,6 @@ static void InitializeCommonInterceptors() {	...	@@ -10293,9 +10433,6 @@ static void InitializeCommonInterceptors() {
10293	INIT_GETIFADDRS;	10433	INIT_GETIFADDRS;
10294	INIT_IF_INDEXTONAME;	10434	INIT_IF_INDEXTONAME;
10295	INIT_CAPGET;	10435	INIT_CAPGET;
10296	INIT_AEABI_MEM;
10297	INIT___BZERO;
10298	INIT_BZERO;
10299	INIT_FTIME;	10436	INIT_FTIME;
10300	INIT_XDR;	10437	INIT_XDR;
10301	INIT_XDRREC_LINUX;	10438	INIT_XDRREC_LINUX;
...	@@ -10322,8 +10459,10 @@ static void InitializeCommonInterceptors() {	...	@@ -10322,8 +10459,10 @@ static void InitializeCommonInterceptors() {
10322	INIT_RECV_RECVFROM;	10459	INIT_RECV_RECVFROM;
10323	INIT_SEND_SENDTO;	10460	INIT_SEND_SENDTO;
10324	INIT_STAT;	10461	INIT_STAT;
		10462	INIT_STAT64;
10325	INIT_EVENTFD_READ_WRITE;	10463	INIT_EVENTFD_READ_WRITE;
10326	INIT_LSTAT;	10464	INIT_LSTAT;
		10465	INIT_LSTAT64;
10327	INIT___XSTAT;	10466	INIT___XSTAT;
10328	INIT___XSTAT64;	10467	INIT___XSTAT64;
10329	INIT___LXSTAT;	10468	INIT___LXSTAT;
...	@@ -10396,14 +10535,17 @@ static void InitializeCommonInterceptors() {	...	@@ -10396,14 +10535,17 @@ static void InitializeCommonInterceptors() {
10396	INIT_GETUSERSHELL;	10535	INIT_GETUSERSHELL;
10397	INIT_SL_INIT;	10536	INIT_SL_INIT;
10398	INIT_GETRANDOM;	10537	INIT_GETRANDOM;
10399	INIT_CRYPT;
10400	INIT_CRYPT_R;
10401	INIT_GETENTROPY;	10538	INIT_GETENTROPY;
10402	INIT_QSORT;	10539	INIT_QSORT;
10403	INIT_QSORT_R;	10540	INIT_QSORT_R;
		10541	INIT_BSEARCH;
10404	INIT_SIGALTSTACK;	10542	INIT_SIGALTSTACK;
		10543	INIT_PROCCTL
10405	INIT_UNAME;	10544	INIT_UNAME;
10406	INIT___XUNAME;	10545	INIT___XUNAME;
		10546	INIT_HEXDUMP;
		10547	INIT_ARGP_PARSE;
		10548	INIT_CPUSET_GETAFFINITY;
10407		10549
10408	INIT___PRINTF_CHK;	10550	INIT___PRINTF_CHK;
10409	}	10551	}

lib/tsan/sanitizer_common/sanitizer_common_interceptors_format.inc+17-9

 @@ -324,8 +324,8 @@ static void scanf_common(void *ctx, int n_inputs, bool allowGnuMalloc,
       continue;
     int size = scanf_get_value_size(&dir);
     if (size == FSS_INVALID) {
-      Report("%s: WARNING: unexpected format specifier in scanf interceptor: ",
+      Report("%s: WARNING: unexpected format specifier in scanf interceptor: %.*s\n",
-             SanitizerToolName, "%.*s\n", dir.end - dir.begin, dir.begin);
+             SanitizerToolName, static_cast<int>(dir.end - dir.begin), dir.begin);
       break;
     }
     void *argp = va_arg(aq, void *);
 @@ -340,11 +340,19 @@ static void scanf_common(void *ctx, int n_inputs, bool allowGnuMalloc,
       size = 0;
     }
     COMMON_INTERCEPTOR_WRITE_RANGE(ctx, argp, size);
-    // For %ms/%mc, write the allocated output buffer as well.
+    // For %mc/%mC/%ms/%m[/%mS, write the allocated output buffer as well.
     if (dir.allocate) {
-      char *buf = *(char **)argp;
+      if (char *buf = *(char **)argp) {
-      if (buf)
+        if (dir.convSpecifier == 'c')
-        COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
+          size = 1;
+        else if (dir.convSpecifier == 'C')
+          size = sizeof(wchar_t);
+        else if (dir.convSpecifier == 'S')
+          size = (internal_wcslen((wchar_t *)buf) + 1) * sizeof(wchar_t);
+        else  // 's' or '['
+          size = internal_strlen(buf) + 1;
+        COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, size);
+      }
     }
   }
 }
 @@ -469,7 +477,7 @@ static int printf_get_value_size(PrintfDirective *dir) {
         break;                                                     \
       default:                                                     \
         Report("WARNING: unexpected floating-point arg size"       \
-               " in printf interceptor: %d\n", size);              \
+               " in printf interceptor: %zu\n", static_cast<uptr>(size));             \
         return;                                                    \
       }                                                            \
     } else {                                                       \
 @@ -484,7 +492,7 @@ static int printf_get_value_size(PrintfDirective *dir) {
         break;                                                     \
       default:                                                     \
         Report("WARNING: unexpected arg size"                      \
-               " in printf interceptor: %d\n", size);              \
+               " in printf interceptor: %zu\n", static_cast<uptr>(size));             \
         return;                                                    \
       }                                                            \
     }                                                              \
 @@ -530,7 +538,7 @@ static void printf_common(void *ctx, const char *format, va_list aq) {
         Report(
             "%s: WARNING: unexpected format specifier in printf "
             "interceptor: %.*s (reported once per process)\n",
-            SanitizerToolName, dir.end - dir.begin, dir.begin);
+            SanitizerToolName, static_cast<int>(dir.end - dir.begin), dir.begin);
       break;
     }
     if (dir.convSpecifier == 'n') {

lib/tsan/sanitizer_common/sanitizer_common_interceptors_ioctl.inc+9-1

 @@ -115,11 +115,19 @@ static void ioctl_table_fill() {
   // _(SOUND_MIXER_WRITE_MUTE, WRITE, sizeof(int)); // same as ...WRITE_ENHANCE
   _(BLKFLSBUF, NONE, 0);
   _(BLKGETSIZE, WRITE, sizeof(uptr));
-  _(BLKRAGET, WRITE, sizeof(int));
+  _(BLKRAGET, WRITE, sizeof(uptr));
   _(BLKRASET, NONE, 0);
   _(BLKROGET, WRITE, sizeof(int));
   _(BLKROSET, READ, sizeof(int));
   _(BLKRRPART, NONE, 0);
+  _(BLKFRASET, NONE, 0);
+  _(BLKFRAGET, WRITE, sizeof(uptr));
+  _(BLKSECTSET, READ, sizeof(short));
+  _(BLKSECTGET, WRITE, sizeof(short));
+  _(BLKSSZGET, WRITE, sizeof(int));
+  _(BLKBSZGET, WRITE, sizeof(int));
+  _(BLKBSZSET, READ, sizeof(uptr));
+  _(BLKGETSIZE64, WRITE, sizeof(u64));
   _(CDROMEJECT, NONE, 0);
   _(CDROMEJECT_SW, NONE, 0);
   _(CDROMMULTISESSION, WRITE, struct_cdrom_multisession_sz);

lib/tsan/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc created+244

...	@@ -0,0 +1,244 @@
1	//===-- sanitizer_common_interceptors_memintrinsics.inc ---------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Memintrinsic function interceptors for tools like AddressSanitizer,
10	// ThreadSanitizer, MemorySanitizer, etc.
11	//
12	// These interceptors are part of the common interceptors, but separated out so
13	// that implementations may add them, if necessary, to a separate source file
14	// that should define SANITIZER_COMMON_NO_REDEFINE_BUILTINS at the top.
15	//
16	// This file should be included into the tool's memintrinsic interceptor file,
17	// which has to define its own macros:
18	// COMMON_INTERCEPTOR_ENTER
19	// COMMON_INTERCEPTOR_READ_RANGE
20	// COMMON_INTERCEPTOR_WRITE_RANGE
21	// COMMON_INTERCEPTOR_MEMSET_IMPL
22	// COMMON_INTERCEPTOR_MEMMOVE_IMPL
23	// COMMON_INTERCEPTOR_MEMCPY_IMPL
24	// COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED
25	//===----------------------------------------------------------------------===//
26
27	#ifdef SANITIZER_REDEFINE_BUILTINS_H
28	#error "Define SANITIZER_COMMON_NO_REDEFINE_BUILTINS in .cpp file"
29	#endif
30
31	#include "interception/interception.h"
32	#include "sanitizer_platform_interceptors.h"
33
34	// Platform-specific options.
35	#if SANITIZER_APPLE
36	#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
37	#elif SANITIZER_WINDOWS64
38	#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
39	#else
40	#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1
41	#endif // SANITIZER_APPLE
42
43	#ifndef COMMON_INTERCEPTOR_MEMSET_IMPL
44	#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \
45	{ \
46	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \
47	return internal_memset(dst, v, size); \
48	COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \
49	if (common_flags()->intercept_intrin) \
50	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
51	return REAL(memset)(dst, v, size); \
52	}
53	#endif
54
55	#ifndef COMMON_INTERCEPTOR_MEMMOVE_IMPL
56	#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size) \
57	{ \
58	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \
59	return internal_memmove(dst, src, size); \
60	COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size); \
61	if (common_flags()->intercept_intrin) { \
62	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
63	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \
64	} \
65	return REAL(memmove)(dst, src, size); \
66	}
67	#endif
68
69	#ifndef COMMON_INTERCEPTOR_MEMCPY_IMPL
70	#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size) \
71	{ \
72	if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) { \
73	return internal_memmove(dst, src, size); \
74	} \
75	COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size); \
76	if (common_flags()->intercept_intrin) { \
77	COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
78	COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \
79	} \
80	return REAL(memcpy)(dst, src, size); \
81	}
82	#endif
83
84	#if SANITIZER_INTERCEPT_MEMSET
85	INTERCEPTOR(void , memset, void dst, int v, uptr size) {
86	void *ctx;
87	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size);
88	}
89
90	#define INIT_MEMSET COMMON_INTERCEPT_FUNCTION(memset)
91	#else
92	#define INIT_MEMSET
93	#endif
94
95	#if SANITIZER_INTERCEPT_MEMMOVE
96	INTERCEPTOR(void , memmove, void dst, const void *src, uptr size) {
97	void *ctx;
98	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
99	}
100
101	#define INIT_MEMMOVE COMMON_INTERCEPT_FUNCTION(memmove)
102	#else
103	#define INIT_MEMMOVE
104	#endif
105
106	#if SANITIZER_INTERCEPT_MEMCPY
107	INTERCEPTOR(void , memcpy, void dst, const void *src, uptr size) {
108	// On OS X, calling internal_memcpy here will cause memory corruptions,
109	// because memcpy and memmove are actually aliases of the same
110	// implementation. We need to use internal_memmove here.
111	// N.B.: If we switch this to internal_ we'll have to use internal_memmove
112	// due to memcpy being an alias of memmove on OS X.
113	void *ctx;
114	#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE
115	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
116	#else
117	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
118	#endif
119	}
120
121	#define INIT_MEMCPY \
122	do { \
123	if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { \
124	COMMON_INTERCEPT_FUNCTION(memcpy); \
125	} else { \
126	ASSIGN_REAL(memcpy, memmove); \
127	} \
128	CHECK(REAL(memcpy)); \
129	} while (false)
130
131	#else
132	#define INIT_MEMCPY
133	#endif
134
135	#if SANITIZER_INTERCEPT_AEABI_MEM
136	INTERCEPTOR(void , __aeabi_memmove, void to, const void *from, uptr size) {
137	void *ctx;
138	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
139	}
140
141	INTERCEPTOR(void , __aeabi_memmove4, void to, const void *from, uptr size) {
142	void *ctx;
143	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
144	}
145
146	INTERCEPTOR(void , __aeabi_memmove8, void to, const void *from, uptr size) {
147	void *ctx;
148	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
149	}
150
151	INTERCEPTOR(void , __aeabi_memcpy, void to, const void *from, uptr size) {
152	void *ctx;
153	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
154	}
155
156	INTERCEPTOR(void , __aeabi_memcpy4, void to, const void *from, uptr size) {
157	void *ctx;
158	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
159	}
160
161	INTERCEPTOR(void , __aeabi_memcpy8, void to, const void *from, uptr size) {
162	void *ctx;
163	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
164	}
165
166	// Note the argument order.
167	INTERCEPTOR(void , __aeabi_memset, void block, uptr size, int c) {
168	void *ctx;
169	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
170	}
171
172	INTERCEPTOR(void , __aeabi_memset4, void block, uptr size, int c) {
173	void *ctx;
174	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
175	}
176
177	INTERCEPTOR(void , __aeabi_memset8, void block, uptr size, int c) {
178	void *ctx;
179	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
180	}
181
182	INTERCEPTOR(void , __aeabi_memclr, void block, uptr size) {
183	void *ctx;
184	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
185	}
186
187	INTERCEPTOR(void , __aeabi_memclr4, void block, uptr size) {
188	void *ctx;
189	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
190	}
191
192	INTERCEPTOR(void , __aeabi_memclr8, void block, uptr size) {
193	void *ctx;
194	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
195	}
196
197	#define INIT_AEABI_MEM \
198	COMMON_INTERCEPT_FUNCTION(__aeabi_memmove); \
199	COMMON_INTERCEPT_FUNCTION(__aeabi_memmove4); \
200	COMMON_INTERCEPT_FUNCTION(__aeabi_memmove8); \
201	COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy); \
202	COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy4); \
203	COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy8); \
204	COMMON_INTERCEPT_FUNCTION(__aeabi_memset); \
205	COMMON_INTERCEPT_FUNCTION(__aeabi_memset4); \
206	COMMON_INTERCEPT_FUNCTION(__aeabi_memset8); \
207	COMMON_INTERCEPT_FUNCTION(__aeabi_memclr); \
208	COMMON_INTERCEPT_FUNCTION(__aeabi_memclr4); \
209	COMMON_INTERCEPT_FUNCTION(__aeabi_memclr8);
210	#else
211	#define INIT_AEABI_MEM
212	#endif // SANITIZER_INTERCEPT_AEABI_MEM
213
214	#if SANITIZER_INTERCEPT___BZERO
215	INTERCEPTOR(void , __bzero, void block, uptr size) {
216	void *ctx;
217	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
218	}
219	#define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero);
220	#else
221	#define INIT___BZERO
222	#endif // SANITIZER_INTERCEPT___BZERO
223
224	#if SANITIZER_INTERCEPT_BZERO
225	INTERCEPTOR(void , bzero, void block, uptr size) {
226	void *ctx;
227	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
228	}
229	#define INIT_BZERO COMMON_INTERCEPT_FUNCTION(bzero);
230	#else
231	#define INIT_BZERO
232	#endif // SANITIZER_INTERCEPT_BZERO
233
234	namespace __sanitizer {
235	// This does not need to be called if InitializeCommonInterceptors() is called.
236	void InitializeMemintrinsicInterceptors() {
237	INIT_MEMSET;
238	INIT_MEMMOVE;
239	INIT_MEMCPY;
240	INIT_AEABI_MEM;
241	INIT___BZERO;
242	INIT_BZERO;
243	}
244	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_common_interceptors_netbsd_compat.inc+2-2

lib/tsan/sanitizer_common/sanitizer_common_interface.inc+10

 @@ -9,12 +9,16 @@
 //===----------------------------------------------------------------------===//
 INTERFACE_FUNCTION(__sanitizer_acquire_crash_state)
 INTERFACE_FUNCTION(__sanitizer_annotate_contiguous_container)
+INTERFACE_FUNCTION(__sanitizer_annotate_double_ended_contiguous_container)
 INTERFACE_FUNCTION(__sanitizer_contiguous_container_find_bad_address)
+INTERFACE_FUNCTION(
+    __sanitizer_double_ended_contiguous_container_find_bad_address)
 INTERFACE_FUNCTION(__sanitizer_set_death_callback)
 INTERFACE_FUNCTION(__sanitizer_set_report_path)
 INTERFACE_FUNCTION(__sanitizer_set_report_fd)
 INTERFACE_FUNCTION(__sanitizer_get_report_path)
 INTERFACE_FUNCTION(__sanitizer_verify_contiguous_container)
+INTERFACE_FUNCTION(__sanitizer_verify_double_ended_contiguous_container)
 INTERFACE_WEAK_FUNCTION(__sanitizer_on_print)
 INTERFACE_WEAK_FUNCTION(__sanitizer_report_error_summary)
 INTERFACE_WEAK_FUNCTION(__sanitizer_sandbox_on_notify)
 @@ -28,7 +32,9 @@ INTERFACE_FUNCTION(__sanitizer_get_module_and_offset_for_pc)
 INTERFACE_FUNCTION(__sanitizer_symbolize_global)
 INTERFACE_FUNCTION(__sanitizer_symbolize_pc)
 // Allocator interface.
+INTERFACE_FUNCTION(__sanitizer_get_allocated_begin)
 INTERFACE_FUNCTION(__sanitizer_get_allocated_size)
+INTERFACE_FUNCTION(__sanitizer_get_allocated_size_fast)
 INTERFACE_FUNCTION(__sanitizer_get_current_allocated_bytes)
 INTERFACE_FUNCTION(__sanitizer_get_estimated_allocated_size)
 INTERFACE_FUNCTION(__sanitizer_get_free_bytes)
 @@ -40,3 +46,7 @@ INTERFACE_FUNCTION(__sanitizer_purge_allocator)
 INTERFACE_FUNCTION(__sanitizer_print_memory_profile)
 INTERFACE_WEAK_FUNCTION(__sanitizer_free_hook)
 INTERFACE_WEAK_FUNCTION(__sanitizer_malloc_hook)
+// Memintrinsic functions.
+INTERFACE_FUNCTION(__sanitizer_internal_memcpy)
+INTERFACE_FUNCTION(__sanitizer_internal_memmove)
+INTERFACE_FUNCTION(__sanitizer_internal_memset)

lib/tsan/sanitizer_common/sanitizer_common_interface_posix.inc+2

...	@@ -11,3 +11,5 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_code)	...	@@ -11,3 +11,5 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_code)
11	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_data)	11	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_data)
12	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_demangle)	12	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_demangle)
13	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_flush)	13	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_flush)
		14	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_set_demangle)
		15	INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_set_inline_frames)

lib/tsan/sanitizer_common/sanitizer_common_libcdep.cpp+63-36

...	@@ -10,27 +10,22 @@	...	@@ -10,27 +10,22 @@
10	// run-time libraries.	10	// run-time libraries.
11	//===----------------------------------------------------------------------===//	11	//===----------------------------------------------------------------------===//
12		12
		13	#include "sanitizer_allocator.h"
13	#include "sanitizer_allocator_interface.h"	14	#include "sanitizer_allocator_interface.h"
14	#include "sanitizer_common.h"	15	#include "sanitizer_common.h"
15	#include "sanitizer_flags.h"	16	#include "sanitizer_flags.h"
		17	#include "sanitizer_interface_internal.h"
16	#include "sanitizer_procmaps.h"	18	#include "sanitizer_procmaps.h"
17		19	#include "sanitizer_stackdepot.h"
18		20
19	namespace __sanitizer {	21	namespace __sanitizer {
20		22
21	static void (*SoftRssLimitExceededCallback)(bool exceeded);
22	void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)) {
23	CHECK_EQ(SoftRssLimitExceededCallback, nullptr);
24	SoftRssLimitExceededCallback = Callback;
25	}
26
27	#if (SANITIZER_LINUX \|\| SANITIZER_NETBSD) && !SANITIZER_GO	23	#if (SANITIZER_LINUX \|\| SANITIZER_NETBSD) && !SANITIZER_GO
28	// Weak default implementation for when sanitizer_stackdepot is not linked in.	24	// Weak default implementation for when sanitizer_stackdepot is not linked in.
29	SANITIZER_WEAK_ATTRIBUTE StackDepotStats *StackDepotGetStats() {	25	SANITIZER_WEAK_ATTRIBUTE StackDepotStats StackDepotGetStats() { return {}; }
30	return nullptr;
31	}
32		26
33	void BackgroundThread(void arg) {	27	void BackgroundThread(void arg) {
		28	VPrintf(1, "%s: Started BackgroundThread\n", SanitizerToolName);
34	const uptr hard_rss_limit_mb = common_flags()->hard_rss_limit_mb;	29	const uptr hard_rss_limit_mb = common_flags()->hard_rss_limit_mb;
35	const uptr soft_rss_limit_mb = common_flags()->soft_rss_limit_mb;	30	const uptr soft_rss_limit_mb = common_flags()->soft_rss_limit_mb;
36	const bool heap_profile = common_flags()->heap_profile;	31	const bool heap_profile = common_flags()->heap_profile;
...	@@ -48,16 +43,12 @@ void BackgroundThread(void arg) {	...	@@ -48,16 +43,12 @@ void BackgroundThread(void arg) {
48	prev_reported_rss = current_rss_mb;	43	prev_reported_rss = current_rss_mb;
49	}	44	}
50	// If stack depot has grown 10% since last time, print it too.	45	// If stack depot has grown 10% since last time, print it too.
51	StackDepotStats *stack_depot_stats = StackDepotGetStats();	46	StackDepotStats stack_depot_stats = StackDepotGetStats();
52	if (stack_depot_stats) {	47	if (prev_reported_stack_depot_size * 11 / 10 <
53	if (prev_reported_stack_depot_size * 11 / 10 <	48	stack_depot_stats.allocated) {
54	stack_depot_stats->allocated) {	49	Printf("%s: StackDepot: %zd ids; %zdM allocated\n", SanitizerToolName,
55	Printf("%s: StackDepot: %zd ids; %zdM allocated\n",	50	stack_depot_stats.n_uniq_ids, stack_depot_stats.allocated >> 20);
56	SanitizerToolName,	51	prev_reported_stack_depot_size = stack_depot_stats.allocated;
57	stack_depot_stats->n_uniq_ids,
58	stack_depot_stats->allocated >> 20);
59	prev_reported_stack_depot_size = stack_depot_stats->allocated;
60	}
61	}	52	}
62	}	53	}
63	// Check RSS against the limit.	54	// Check RSS against the limit.
...	@@ -72,13 +63,13 @@ void BackgroundThread(void arg) {	...	@@ -72,13 +63,13 @@ void BackgroundThread(void arg) {
72	reached_soft_rss_limit = true;	63	reached_soft_rss_limit = true;
73	Report("%s: soft rss limit exhausted (%zdMb vs %zdMb)\n",	64	Report("%s: soft rss limit exhausted (%zdMb vs %zdMb)\n",
74	SanitizerToolName, soft_rss_limit_mb, current_rss_mb);	65	SanitizerToolName, soft_rss_limit_mb, current_rss_mb);
75	if (SoftRssLimitExceededCallback)	66	SetRssLimitExceeded(true);
76	SoftRssLimitExceededCallback(true);
77	} else if (soft_rss_limit_mb >= current_rss_mb &&	67	} else if (soft_rss_limit_mb >= current_rss_mb &&
78	reached_soft_rss_limit) {	68	reached_soft_rss_limit) {
79	reached_soft_rss_limit = false;	69	reached_soft_rss_limit = false;
80	if (SoftRssLimitExceededCallback)	70	Report("%s: soft rss limit unexhausted (%zdMb vs %zdMb)\n",
81	SoftRssLimitExceededCallback(false);	71	SanitizerToolName, soft_rss_limit_mb, current_rss_mb);
		72	SetRssLimitExceeded(false);
82	}	73	}
83	}	74	}
84	if (heap_profile &&	75	if (heap_profile &&
...	@@ -89,6 +80,42 @@ void BackgroundThread(void arg) {	...	@@ -89,6 +80,42 @@ void BackgroundThread(void arg) {
89	}	80	}
90	}	81	}
91	}	82	}
		83
		84	void MaybeStartBackgroudThread() {
		85	// Need to implement/test on other platforms.
		86	// Start the background thread if one of the rss limits is given.
		87	if (!common_flags()->hard_rss_limit_mb &&
		88	!common_flags()->soft_rss_limit_mb &&
		89	!common_flags()->heap_profile) return;
		90	if (!&real_pthread_create) {
		91	VPrintf(1, "%s: real_pthread_create undefined\n", SanitizerToolName);
		92	return; // Can't spawn the thread anyway.
		93	}
		94
		95	static bool started = false;
		96	if (!started) {
		97	started = true;
		98	internal_start_thread(BackgroundThread, nullptr);
		99	}
		100	}
		101
		102	# if !SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL
		103	# ifdef __clang__
		104	# pragma clang diagnostic push
		105	// We avoid global-constructors to be sure that globals are ready when
		106	// sanitizers need them. This can happend before global constructors executed.
		107	// Here we don't mind if thread is started on later stages.
		108	# pragma clang diagnostic ignored "-Wglobal-constructors"
		109	# endif
		110	static struct BackgroudThreadStarted {
		111	BackgroudThreadStarted() { MaybeStartBackgroudThread(); }
		112	} background_thread_strarter UNUSED;
		113	# ifdef __clang__
		114	# pragma clang diagnostic pop
		115	# endif
		116	# endif
		117	#else
		118	void MaybeStartBackgroudThread() {}
92	#endif	119	#endif
93		120
94	void WriteToSyslog(const char *msg) {	121	void WriteToSyslog(const char *msg) {
...	@@ -111,18 +138,6 @@ void WriteToSyslog(const char *msg) {	...	@@ -111,18 +138,6 @@ void WriteToSyslog(const char *msg) {
111	WriteOneLineToSyslog(p);	138	WriteOneLineToSyslog(p);
112	}	139	}
113		140
114	void MaybeStartBackgroudThread() {
115	#if (SANITIZER_LINUX \|\| SANITIZER_NETBSD) && \
116	!SANITIZER_GO // Need to implement/test on other platforms.
117	// Start the background thread if one of the rss limits is given.
118	if (!common_flags()->hard_rss_limit_mb &&
119	!common_flags()->soft_rss_limit_mb &&
120	!common_flags()->heap_profile) return;
121	if (!&real_pthread_create) return; // Can't spawn the thread anyway.
122	internal_start_thread(BackgroundThread, nullptr);
123	#endif
124	}
125
126	static void (*sandboxing_callback)();	141	static void (*sandboxing_callback)();
127	void SetSandboxingCallback(void (*f)()) {	142	void SetSandboxingCallback(void (*f)()) {
128	sandboxing_callback = f;	143	sandboxing_callback = f;
...	@@ -191,10 +206,22 @@ void ProtectGap(uptr addr, uptr size, uptr zero_base_shadow_start,	...	@@ -191,10 +206,22 @@ void ProtectGap(uptr addr, uptr size, uptr zero_base_shadow_start,
191		206
192	#endif // !SANITIZER_FUCHSIA	207	#endif // !SANITIZER_FUCHSIA
193		208
		209	#if !SANITIZER_WINDOWS && !SANITIZER_GO
		210	// Weak default implementation for when sanitizer_stackdepot is not linked in.
		211	SANITIZER_WEAK_ATTRIBUTE void StackDepotStopBackgroundThread() {}
		212	static void StopStackDepotBackgroundThread() {
		213	StackDepotStopBackgroundThread();
		214	}
		215	#else
		216	// SANITIZER_WEAK_ATTRIBUTE is unsupported.
		217	static void StopStackDepotBackgroundThread() {}
		218	#endif
		219
194	} // namespace __sanitizer	220	} // namespace __sanitizer
195		221
196	SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify,	222	SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_sandbox_on_notify,
197	__sanitizer_sandbox_arguments *args) {	223	__sanitizer_sandbox_arguments *args) {
		224	__sanitizer::StopStackDepotBackgroundThread();
198	__sanitizer::PlatformPrepareForSandboxing(args);	225	__sanitizer::PlatformPrepareForSandboxing(args);
199	if (__sanitizer::sandboxing_callback)	226	if (__sanitizer::sandboxing_callback)
200	__sanitizer::sandboxing_callback();	227	__sanitizer::sandboxing_callback();

lib/tsan/sanitizer_common/sanitizer_common_nolibc.cpp+2-1

 @@ -25,9 +25,10 @@ void LogMessageOnPrintf(const char *str) {}
 #endif
 void WriteToSyslog(const char *buffer) {}
 void Abort() { internal__exit(1); }
+bool CreateDir(const char *pathname) { return false; }
 #endif // !SANITIZER_WINDOWS
-#if !SANITIZER_WINDOWS && !SANITIZER_MAC
+#if !SANITIZER_WINDOWS && !SANITIZER_APPLE
 void ListOfModules::init() {}
 void InitializePlatformCommonFlags(CommonFlags *cf) {}
 #endif

lib/tsan/sanitizer_common/sanitizer_common_syscalls.inc+921-658

 @@ -43,45 +43,47 @@
 #include "sanitizer_platform.h"
 #if SANITIZER_LINUX
-#include "sanitizer_libc.h"
+#  include "sanitizer_libc.h"
-#define PRE_SYSCALL(name)                                                      \
+#  define PRE_SYSCALL(name) \
-  SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_pre_impl_##name
+    SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_pre_impl_##name
-#define PRE_READ(p, s) COMMON_SYSCALL_PRE_READ_RANGE(p, s)
+#  define PRE_READ(p, s) COMMON_SYSCALL_PRE_READ_RANGE(p, s)
-#define PRE_WRITE(p, s) COMMON_SYSCALL_PRE_WRITE_RANGE(p, s)
+#  define PRE_WRITE(p, s) COMMON_SYSCALL_PRE_WRITE_RANGE(p, s)
-#define POST_SYSCALL(name)                                                     \
+#  define POST_SYSCALL(name) \
-  SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_post_impl_##name
+    SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_syscall_post_impl_##name
-#define POST_READ(p, s) COMMON_SYSCALL_POST_READ_RANGE(p, s)
+#  define POST_READ(p, s) COMMON_SYSCALL_POST_READ_RANGE(p, s)
-#define POST_WRITE(p, s) COMMON_SYSCALL_POST_WRITE_RANGE(p, s)
+#  define POST_WRITE(p, s) COMMON_SYSCALL_POST_WRITE_RANGE(p, s)
-#ifndef COMMON_SYSCALL_ACQUIRE
+#  ifndef COMMON_SYSCALL_ACQUIRE
-# define COMMON_SYSCALL_ACQUIRE(addr) ((void)(addr))
+#    define COMMON_SYSCALL_ACQUIRE(addr) ((void)(addr))
-#endif
+#  endif
-#ifndef COMMON_SYSCALL_RELEASE
+#  ifndef COMMON_SYSCALL_RELEASE
-# define COMMON_SYSCALL_RELEASE(addr) ((void)(addr))
+#    define COMMON_SYSCALL_RELEASE(addr) ((void)(addr))
-#endif
+#  endif
-#ifndef COMMON_SYSCALL_FD_CLOSE
+#  ifndef COMMON_SYSCALL_FD_CLOSE
-# define COMMON_SYSCALL_FD_CLOSE(fd) ((void)(fd))
+#    define COMMON_SYSCALL_FD_CLOSE(fd) ((void)(fd))
-#endif
+#  endif
-#ifndef COMMON_SYSCALL_FD_ACQUIRE
+#  ifndef COMMON_SYSCALL_FD_ACQUIRE
-# define COMMON_SYSCALL_FD_ACQUIRE(fd) ((void)(fd))
+#    define COMMON_SYSCALL_FD_ACQUIRE(fd) ((void)(fd))
-#endif
+#  endif
-#ifndef COMMON_SYSCALL_FD_RELEASE
+#  ifndef COMMON_SYSCALL_FD_RELEASE
-# define COMMON_SYSCALL_FD_RELEASE(fd) ((void)(fd))
+#    define COMMON_SYSCALL_FD_RELEASE(fd) ((void)(fd))
-#endif
+#  endif
-#ifndef COMMON_SYSCALL_PRE_FORK
+#  ifndef COMMON_SYSCALL_PRE_FORK
-# define COMMON_SYSCALL_PRE_FORK() {}
+#    define COMMON_SYSCALL_PRE_FORK() \
-#endif
+      {}
+#  endif
-#ifndef COMMON_SYSCALL_POST_FORK
+#  ifndef COMMON_SYSCALL_POST_FORK
-# define COMMON_SYSCALL_POST_FORK(res) {}
+#    define COMMON_SYSCALL_POST_FORK(res) \
-#endif
+      {}
+#  endif
 // FIXME: do some kind of PRE_READ for all syscall arguments (int(s) and such).
 @@ -130,8 +132,8 @@ struct sanitizer_kernel_sockaddr {
 // Declare it "void" to catch sizeof(kernel_sigset_t).
 typedef void kernel_sigset_t;
-static void kernel_write_iovec(const __sanitizer_iovec *iovec,
+static void kernel_write_iovec(const __sanitizer_iovec *iovec, SIZE_T iovlen,
-                        SIZE_T iovlen, SIZE_T maxlen) {
+                               SIZE_T maxlen) {
   for (SIZE_T i = 0; i < iovlen && maxlen; ++i) {
     SSIZE_T sz = Min(iovec[i].iov_len, maxlen);
     POST_WRITE(iovec[i].iov_base, sz);
 @@ -141,8 +143,8 @@ static void kernel_write_iovec(const __sanitizer_iovec *iovec,
 // This functions uses POST_READ, because it needs to run after syscall to know
 // the real read range.
-static void kernel_read_iovec(const __sanitizer_iovec *iovec,
+static void kernel_read_iovec(const __sanitizer_iovec *iovec, SIZE_T iovlen,
-                       SIZE_T iovlen, SIZE_T maxlen) {
+                              SIZE_T maxlen) {
   POST_READ(iovec, sizeof(*iovec) * iovlen);
   for (SIZE_T i = 0; i < iovlen && maxlen; ++i) {
     SSIZE_T sz = Min(iovec[i].iov_len, maxlen);
 @@ -155,8 +157,8 @@ PRE_SYSCALL(recvmsg)(long sockfd, sanitizer_kernel_msghdr *msg, long flags) {
   PRE_READ(msg, sizeof(*msg));
 }
-POST_SYSCALL(recvmsg)(long res, long sockfd, sanitizer_kernel_msghdr *msg,
+POST_SYSCALL(recvmsg)
-                      long flags) {
+(long res, long sockfd, sanitizer_kernel_msghdr *msg, long flags) {
   if (res >= 0) {
     if (msg) {
       for (unsigned long i = 0; i < msg->msg_iovlen; ++i) {
 @@ -167,13 +169,14 @@ POST_SYSCALL(recvmsg)(long res, long sockfd, sanitizer_kernel_msghdr *msg,
   }
 }
-PRE_SYSCALL(recvmmsg)(long fd, sanitizer_kernel_mmsghdr *msg, long vlen,
+PRE_SYSCALL(recvmmsg)
-                      long flags, void *timeout) {
+(long fd, sanitizer_kernel_mmsghdr *msg, long vlen, long flags, void *timeout) {
   PRE_READ(msg, vlen * sizeof(*msg));
 }
-POST_SYSCALL(recvmmsg)(long res, long fd, sanitizer_kernel_mmsghdr *msg,
+POST_SYSCALL(recvmmsg)
-                       long vlen, long flags, void *timeout) {
+(long res, long fd, sanitizer_kernel_mmsghdr *msg, long vlen, long flags,
+ void *timeout) {
   if (res >= 0) {
     if (msg) {
       for (unsigned long i = 0; i < msg->msg_hdr.msg_iovlen; ++i) {
 @@ -183,7 +186,8 @@ POST_SYSCALL(recvmmsg)(long res, long fd, sanitizer_kernel_mmsghdr *msg,
       POST_WRITE(msg->msg_hdr.msg_control, msg->msg_hdr.msg_controllen);
       POST_WRITE(&msg->msg_len, sizeof(msg->msg_len));
     }
-    if (timeout) POST_WRITE(timeout, struct_timespec_sz);
+    if (timeout)
+      POST_WRITE(timeout, struct_timespec_sz);
   }
 }
 @@ -203,7 +207,8 @@ PRE_SYSCALL(time)(void *tloc) {}
 POST_SYSCALL(time)(long res, void *tloc) {
   if (res >= 0) {
-    if (tloc) POST_WRITE(tloc, sizeof(long));
+    if (tloc)
+      POST_WRITE(tloc, sizeof(long));
   }
 }
 @@ -211,7 +216,8 @@ PRE_SYSCALL(stime)(void *tptr) {}
 POST_SYSCALL(stime)(long res, void *tptr) {
   if (res >= 0) {
-    if (tptr) POST_WRITE(tptr, sizeof(long));
+    if (tptr)
+      POST_WRITE(tptr, sizeof(long));
   }
 }
 @@ -219,8 +225,10 @@ PRE_SYSCALL(gettimeofday)(void *tv, void *tz) {}
 POST_SYSCALL(gettimeofday)(long res, void *tv, void *tz) {
   if (res >= 0) {
-    if (tv) POST_WRITE(tv, timeval_sz);
+    if (tv)
-    if (tz) POST_WRITE(tz, struct_timezone_sz);
+      POST_WRITE(tv, timeval_sz);
+    if (tz)
+      POST_WRITE(tz, struct_timezone_sz);
   }
 }
 @@ -228,26 +236,30 @@ PRE_SYSCALL(settimeofday)(void *tv, void *tz) {}
 POST_SYSCALL(settimeofday)(long res, void *tv, void *tz) {
   if (res >= 0) {
-    if (tv) POST_WRITE(tv, timeval_sz);
+    if (tv)
-    if (tz) POST_WRITE(tz, struct_timezone_sz);
+      POST_WRITE(tv, timeval_sz);
+    if (tz)
+      POST_WRITE(tz, struct_timezone_sz);
   }
 }
-#if !SANITIZER_ANDROID
+#  if !SANITIZER_ANDROID
 PRE_SYSCALL(adjtimex)(void *txc_p) {}
 POST_SYSCALL(adjtimex)(long res, void *txc_p) {
   if (res >= 0) {
-    if (txc_p) POST_WRITE(txc_p, struct_timex_sz);
+    if (txc_p)
+      POST_WRITE(txc_p, struct_timex_sz);
   }
 }
-#endif
+#  endif
 PRE_SYSCALL(times)(void *tbuf) {}
 POST_SYSCALL(times)(long res, void *tbuf) {
   if (res >= 0) {
-    if (tbuf) POST_WRITE(tbuf, struct_tms_sz);
+    if (tbuf)
+      POST_WRITE(tbuf, struct_tms_sz);
   }
 }
 @@ -259,8 +271,10 @@ PRE_SYSCALL(nanosleep)(void *rqtp, void *rmtp) {}
 POST_SYSCALL(nanosleep)(long res, void *rqtp, void *rmtp) {
   if (res >= 0) {
-    if (rqtp) POST_WRITE(rqtp, struct_timespec_sz);
+    if (rqtp)
-    if (rmtp) POST_WRITE(rmtp, struct_timespec_sz);
+      POST_WRITE(rqtp, struct_timespec_sz);
+    if (rmtp)
+      POST_WRITE(rmtp, struct_timespec_sz);
   }
 }
 @@ -296,9 +310,12 @@ PRE_SYSCALL(getresuid)(void *ruid, void *euid, void *suid) {}
 POST_SYSCALL(getresuid)(long res, void *ruid, void *euid, void *suid) {
   if (res >= 0) {
-    if (ruid) POST_WRITE(ruid, sizeof(unsigned));
+    if (ruid)
-    if (euid) POST_WRITE(euid, sizeof(unsigned));
+      POST_WRITE(ruid, sizeof(unsigned));
-    if (suid) POST_WRITE(suid, sizeof(unsigned));
+    if (euid)
+      POST_WRITE(euid, sizeof(unsigned));
+    if (suid)
+      POST_WRITE(suid, sizeof(unsigned));
   }
 }
 @@ -306,9 +323,12 @@ PRE_SYSCALL(getresgid)(void *rgid, void *egid, void *sgid) {}
 POST_SYSCALL(getresgid)(long res, void *rgid, void *egid, void *sgid) {
   if (res >= 0) {
-    if (rgid) POST_WRITE(rgid, sizeof(unsigned));
+    if (rgid)
-    if (egid) POST_WRITE(egid, sizeof(unsigned));
+      POST_WRITE(rgid, sizeof(unsigned));
-    if (sgid) POST_WRITE(sgid, sizeof(unsigned));
+    if (egid)
+      POST_WRITE(egid, sizeof(unsigned));
+    if (sgid)
+      POST_WRITE(sgid, sizeof(unsigned));
   }
 }
 @@ -326,10 +346,11 @@ POST_SYSCALL(getsid)(long res, long pid) {}
 PRE_SYSCALL(getgroups)(long gidsetsize, void *grouplist) {}
-POST_SYSCALL(getgroups)(long res, long gidsetsize,
+POST_SYSCALL(getgroups)
-                        __sanitizer___kernel_gid_t *grouplist) {
+(long res, long gidsetsize, __sanitizer___kernel_gid_t *grouplist) {
   if (res >= 0) {
-    if (grouplist) POST_WRITE(grouplist, res * sizeof(*grouplist));
+    if (grouplist)
+      POST_WRITE(grouplist, res * sizeof(*grouplist));
   }
 }
 @@ -374,11 +395,12 @@ PRE_SYSCALL(setsid)() {}
 POST_SYSCALL(setsid)(long res) {}
 PRE_SYSCALL(setgroups)(long gidsetsize, __sanitizer___kernel_gid_t *grouplist) {
-  if (grouplist) POST_WRITE(grouplist, gidsetsize * sizeof(*grouplist));
+  if (grouplist)
+    POST_WRITE(grouplist, gidsetsize * sizeof(*grouplist));
 }
-POST_SYSCALL(setgroups)(long res, long gidsetsize,
+POST_SYSCALL(setgroups)
-                        __sanitizer___kernel_gid_t *grouplist) {}
+(long res, long gidsetsize, __sanitizer___kernel_gid_t *grouplist) {}
 PRE_SYSCALL(acct)(const void *name) {
   if (name)
 @@ -388,17 +410,21 @@ PRE_SYSCALL(acct)(const void *name) {
 POST_SYSCALL(acct)(long res, const void *name) {}
 PRE_SYSCALL(capget)(void *header, void *dataptr) {
-  if (header) PRE_READ(header, __user_cap_header_struct_sz);
+  if (header)
+    PRE_READ(header, __user_cap_header_struct_sz);
 }
 POST_SYSCALL(capget)(long res, void *header, void *dataptr) {
   if (res >= 0)
-    if (dataptr) POST_WRITE(dataptr, __user_cap_data_struct_sz);
+    if (dataptr)
+      POST_WRITE(dataptr, __user_cap_data_struct_sz(header));
 }
 PRE_SYSCALL(capset)(void *header, const void *data) {
-  if (header) PRE_READ(header, __user_cap_header_struct_sz);
+  if (header)
-  if (data) PRE_READ(data, __user_cap_data_struct_sz);
+    PRE_READ(header, __user_cap_header_struct_sz);
+  if (data)
+    PRE_READ(data, __user_cap_data_struct_sz(header));
 }
 POST_SYSCALL(capset)(long res, void *header, const void *data) {}
 @@ -411,7 +437,8 @@ PRE_SYSCALL(sigpending)(void *set) {}
 POST_SYSCALL(sigpending)(long res, void *set) {
   if (res >= 0) {
-    if (set) POST_WRITE(set, old_sigset_t_sz);
+    if (set)
+      POST_WRITE(set, old_sigset_t_sz);
   }
 }
 @@ -419,8 +446,10 @@ PRE_SYSCALL(sigprocmask)(long how, void *set, void *oset) {}
 POST_SYSCALL(sigprocmask)(long res, long how, void *set, void *oset) {
   if (res >= 0) {
-    if (set) POST_WRITE(set, old_sigset_t_sz);
+    if (set)
-    if (oset) POST_WRITE(oset, old_sigset_t_sz);
+      POST_WRITE(set, old_sigset_t_sz);
+    if (oset)
+      POST_WRITE(oset, old_sigset_t_sz);
   }
 }
 @@ -428,7 +457,8 @@ PRE_SYSCALL(getitimer)(long which, void *value) {}
 POST_SYSCALL(getitimer)(long res, long which, void *value) {
   if (res >= 0) {
-    if (value) POST_WRITE(value, struct_itimerval_sz);
+    if (value)
+      POST_WRITE(value, struct_itimerval_sz);
   }
 }
 @@ -436,19 +466,23 @@ PRE_SYSCALL(setitimer)(long which, void *value, void *ovalue) {}
 POST_SYSCALL(setitimer)(long res, long which, void *value, void *ovalue) {
   if (res >= 0) {
-    if (value) POST_WRITE(value, struct_itimerval_sz);
+    if (value)
-    if (ovalue) POST_WRITE(ovalue, struct_itimerval_sz);
+      POST_WRITE(value, struct_itimerval_sz);
+    if (ovalue)
+      POST_WRITE(ovalue, struct_itimerval_sz);
   }
 }
-PRE_SYSCALL(timer_create)(long which_clock, void *timer_event_spec,
+PRE_SYSCALL(timer_create)
-                          void *created_timer_id) {}
+(long which_clock, void *timer_event_spec, void *created_timer_id) {}
-POST_SYSCALL(timer_create)(long res, long which_clock, void *timer_event_spec,
+POST_SYSCALL(timer_create)
-                           void *created_timer_id) {
+(long res, long which_clock, void *timer_event_spec, void *created_timer_id) {
   if (res >= 0) {
-    if (timer_event_spec) POST_WRITE(timer_event_spec, struct_sigevent_sz);
+    if (timer_event_spec)
-    if (created_timer_id) POST_WRITE(created_timer_id, sizeof(long));
+      POST_WRITE(timer_event_spec, struct_sigevent_sz);
+    if (created_timer_id)
+      POST_WRITE(created_timer_id, sizeof(long));
   }
 }
 @@ -456,7 +490,8 @@ PRE_SYSCALL(timer_gettime)(long timer_id, void *setting) {}
 POST_SYSCALL(timer_gettime)(long res, long timer_id, void *setting) {
   if (res >= 0) {
-    if (setting) POST_WRITE(setting, struct_itimerspec_sz);
+    if (setting)
+      POST_WRITE(setting, struct_itimerspec_sz);
   }
 }
 @@ -464,15 +499,18 @@ PRE_SYSCALL(timer_getoverrun)(long timer_id) {}
 POST_SYSCALL(timer_getoverrun)(long res, long timer_id) {}
-PRE_SYSCALL(timer_settime)(long timer_id, long flags, const void *new_setting,
+PRE_SYSCALL(timer_settime)
-                           void *old_setting) {
+(long timer_id, long flags, const void *new_setting, void *old_setting) {
-  if (new_setting) PRE_READ(new_setting, struct_itimerspec_sz);
+  if (new_setting)
+    PRE_READ(new_setting, struct_itimerspec_sz);
 }
-POST_SYSCALL(timer_settime)(long res, long timer_id, long flags,
+POST_SYSCALL(timer_settime)
-                            const void *new_setting, void *old_setting) {
+(long res, long timer_id, long flags, const void *new_setting,
+ void *old_setting) {
   if (res >= 0) {
-    if (old_setting) POST_WRITE(old_setting, struct_itimerspec_sz);
+    if (old_setting)
+      POST_WRITE(old_setting, struct_itimerspec_sz);
   }
 }
 @@ -481,7 +519,8 @@ PRE_SYSCALL(timer_delete)(long timer_id) {}
 POST_SYSCALL(timer_delete)(long res, long timer_id) {}
 PRE_SYSCALL(clock_settime)(long which_clock, const void *tp) {
-  if (tp) PRE_READ(tp, struct_timespec_sz);
+  if (tp)
+    PRE_READ(tp, struct_timespec_sz);
 }
 POST_SYSCALL(clock_settime)(long res, long which_clock, const void *tp) {}
 @@ -490,37 +529,42 @@ PRE_SYSCALL(clock_gettime)(long which_clock, void *tp) {}
 POST_SYSCALL(clock_gettime)(long res, long which_clock, void *tp) {
   if (res >= 0) {
-    if (tp) POST_WRITE(tp, struct_timespec_sz);
+    if (tp)
+      POST_WRITE(tp, struct_timespec_sz);
   }
 }
-#if !SANITIZER_ANDROID
+#  if !SANITIZER_ANDROID
 PRE_SYSCALL(clock_adjtime)(long which_clock, void *tx) {}
 POST_SYSCALL(clock_adjtime)(long res, long which_clock, void *tx) {
   if (res >= 0) {
-    if (tx) POST_WRITE(tx, struct_timex_sz);
+    if (tx)
+      POST_WRITE(tx, struct_timex_sz);
   }
 }
-#endif
+#  endif
 PRE_SYSCALL(clock_getres)(long which_clock, void *tp) {}
 POST_SYSCALL(clock_getres)(long res, long which_clock, void *tp) {
   if (res >= 0) {
-    if (tp) POST_WRITE(tp, struct_timespec_sz);
+    if (tp)
+      POST_WRITE(tp, struct_timespec_sz);
   }
 }
-PRE_SYSCALL(clock_nanosleep)(long which_clock, long flags, const void *rqtp,
+PRE_SYSCALL(clock_nanosleep)
-                             void *rmtp) {
+(long which_clock, long flags, const void *rqtp, void *rmtp) {
-  if (rqtp) PRE_READ(rqtp, struct_timespec_sz);
+  if (rqtp)
+    PRE_READ(rqtp, struct_timespec_sz);
 }
-POST_SYSCALL(clock_nanosleep)(long res, long which_clock, long flags,
+POST_SYSCALL(clock_nanosleep)
-                              const void *rqtp, void *rmtp) {
+(long res, long which_clock, long flags, const void *rqtp, void *rmtp) {
   if (res >= 0) {
-    if (rmtp) POST_WRITE(rmtp, struct_timespec_sz);
+    if (rmtp)
+      POST_WRITE(rmtp, struct_timespec_sz);
   }
 }
 @@ -532,12 +576,14 @@ PRE_SYSCALL(sched_setscheduler)(long pid, long policy, void *param) {}
 POST_SYSCALL(sched_setscheduler)(long res, long pid, long policy, void *param) {
   if (res >= 0) {
-    if (param) POST_WRITE(param, struct_sched_param_sz);
+    if (param)
+      POST_WRITE(param, struct_sched_param_sz);
   }
 }
 PRE_SYSCALL(sched_setparam)(long pid, void *param) {
-  if (param) PRE_READ(param, struct_sched_param_sz);
+  if (param)
+    PRE_READ(param, struct_sched_param_sz);
 }
 POST_SYSCALL(sched_setparam)(long res, long pid, void *param) {}
 @@ -550,23 +596,26 @@ PRE_SYSCALL(sched_getparam)(long pid, void *param) {}
 POST_SYSCALL(sched_getparam)(long res, long pid, void *param) {
   if (res >= 0) {
-    if (param) POST_WRITE(param, struct_sched_param_sz);
+    if (param)
+      POST_WRITE(param, struct_sched_param_sz);
   }
 }
 PRE_SYSCALL(sched_setaffinity)(long pid, long len, void *user_mask_ptr) {
-  if (user_mask_ptr) PRE_READ(user_mask_ptr, len);
+  if (user_mask_ptr)
+    PRE_READ(user_mask_ptr, len);
 }
-POST_SYSCALL(sched_setaffinity)(long res, long pid, long len,
+POST_SYSCALL(sched_setaffinity)
-                                void *user_mask_ptr) {}
+(long res, long pid, long len, void *user_mask_ptr) {}
 PRE_SYSCALL(sched_getaffinity)(long pid, long len, void *user_mask_ptr) {}
-POST_SYSCALL(sched_getaffinity)(long res, long pid, long len,
+POST_SYSCALL(sched_getaffinity)
-                                void *user_mask_ptr) {
+(long res, long pid, long len, void *user_mask_ptr) {
   if (res >= 0) {
-    if (user_mask_ptr) POST_WRITE(user_mask_ptr, len);
+    if (user_mask_ptr)
+      POST_WRITE(user_mask_ptr, len);
   }
 }
 @@ -586,7 +635,8 @@ PRE_SYSCALL(sched_rr_get_interval)(long pid, void *interval) {}
 POST_SYSCALL(sched_rr_get_interval)(long res, long pid, void *interval) {
   if (res >= 0) {
-    if (interval) POST_WRITE(interval, struct_timespec_sz);
+    if (interval)
+      POST_WRITE(interval, struct_timespec_sz);
   }
 }
 @@ -610,13 +660,14 @@ PRE_SYSCALL(restart_syscall)() {}
 POST_SYSCALL(restart_syscall)(long res) {}
-PRE_SYSCALL(kexec_load)(long entry, long nr_segments, void *segments,
+PRE_SYSCALL(kexec_load)
-                        long flags) {}
+(long entry, long nr_segments, void *segments, long flags) {}
-POST_SYSCALL(kexec_load)(long res, long entry, long nr_segments, void *segments,
+POST_SYSCALL(kexec_load)
-                         long flags) {
+(long res, long entry, long nr_segments, void *segments, long flags) {
   if (res >= 0) {
-    if (segments) POST_WRITE(segments, struct_kexec_segment_sz);
+    if (segments)
+      POST_WRITE(segments, struct_kexec_segment_sz);
   }
 }
 @@ -630,22 +681,26 @@ POST_SYSCALL(exit_group)(long res, long error_code) {}
 PRE_SYSCALL(wait4)(long pid, void *stat_addr, long options, void *ru) {}
-POST_SYSCALL(wait4)(long res, long pid, void *stat_addr, long options,
+POST_SYSCALL(wait4)
-                    void *ru) {
+(long res, long pid, void *stat_addr, long options, void *ru) {
   if (res >= 0) {
-    if (stat_addr) POST_WRITE(stat_addr, sizeof(int));
+    if (stat_addr)
-    if (ru) POST_WRITE(ru, struct_rusage_sz);
+      POST_WRITE(stat_addr, sizeof(int));
+    if (ru)
+      POST_WRITE(ru, struct_rusage_sz);
   }
 }
-PRE_SYSCALL(waitid)(long which, long pid, void *infop, long options, void *ru) {
+PRE_SYSCALL(waitid)
-}
+(long which, long pid, void *infop, long options, void *ru) {}
-POST_SYSCALL(waitid)(long res, long which, long pid, void *infop, long options,
+POST_SYSCALL(waitid)
-                     void *ru) {
+(long res, long which, long pid, void *infop, long options, void *ru) {
   if (res >= 0) {
-    if (infop) POST_WRITE(infop, siginfo_t_sz);
+    if (infop)
-    if (ru) POST_WRITE(ru, struct_rusage_sz);
+      POST_WRITE(infop, siginfo_t_sz);
+    if (ru)
+      POST_WRITE(ru, struct_rusage_sz);
   }
 }
 @@ -653,7 +708,8 @@ PRE_SYSCALL(waitpid)(long pid, void *stat_addr, long options) {}
 POST_SYSCALL(waitpid)(long res, long pid, void *stat_addr, long options) {
   if (res >= 0) {
-    if (stat_addr) POST_WRITE(stat_addr, sizeof(int));
+    if (stat_addr)
+      POST_WRITE(stat_addr, sizeof(int));
   }
 }
 @@ -661,7 +717,8 @@ PRE_SYSCALL(set_tid_address)(void *tidptr) {}
 POST_SYSCALL(set_tid_address)(long res, void *tidptr) {
   if (res >= 0) {
-    if (tidptr) POST_WRITE(tidptr, sizeof(int));
+    if (tidptr)
+      POST_WRITE(tidptr, sizeof(int));
   }
 }
 @@ -682,11 +739,14 @@ POST_SYSCALL(delete_module)(long res, const void *name_user, long flags) {}
 PRE_SYSCALL(rt_sigprocmask)(long how, void *set, void *oset, long sigsetsize) {}
-POST_SYSCALL(rt_sigprocmask)(long res, long how, kernel_sigset_t *set,
+POST_SYSCALL(rt_sigprocmask)
-                             kernel_sigset_t *oset, long sigsetsize) {
+(long res, long how, kernel_sigset_t *set, kernel_sigset_t *oset,
+ long sigsetsize) {
   if (res >= 0) {
-    if (set) POST_WRITE(set, sigsetsize);
+    if (set)
-    if (oset) POST_WRITE(oset, sigsetsize);
+      POST_WRITE(set, sigsetsize);
+    if (oset)
+      POST_WRITE(oset, sigsetsize);
   }
 }
 @@ -694,29 +754,34 @@ PRE_SYSCALL(rt_sigpending)(void *set, long sigsetsize) {}
 POST_SYSCALL(rt_sigpending)(long res, kernel_sigset_t *set, long sigsetsize) {
   if (res >= 0) {
-    if (set) POST_WRITE(set, sigsetsize);
+    if (set)
+      POST_WRITE(set, sigsetsize);
   }
 }
-PRE_SYSCALL(rt_sigtimedwait)(const kernel_sigset_t *uthese, void *uinfo,
+PRE_SYSCALL(rt_sigtimedwait)
-                             const void *uts, long sigsetsize) {
+(const kernel_sigset_t *uthese, void *uinfo, const void *uts, long sigsetsize) {
-  if (uthese) PRE_READ(uthese, sigsetsize);
+  if (uthese)
-  if (uts) PRE_READ(uts, struct_timespec_sz);
+    PRE_READ(uthese, sigsetsize);
+  if (uts)
+    PRE_READ(uts, struct_timespec_sz);
 }
-POST_SYSCALL(rt_sigtimedwait)(long res, const void *uthese, void *uinfo,
+POST_SYSCALL(rt_sigtimedwait)
-                              const void *uts, long sigsetsize) {
+(long res, const void *uthese, void *uinfo, const void *uts, long sigsetsize) {
   if (res >= 0) {
-    if (uinfo) POST_WRITE(uinfo, siginfo_t_sz);
+    if (uinfo)
+      POST_WRITE(uinfo, siginfo_t_sz);
   }
 }
 PRE_SYSCALL(rt_tgsigqueueinfo)(long tgid, long pid, long sig, void *uinfo) {}
-POST_SYSCALL(rt_tgsigqueueinfo)(long res, long tgid, long pid, long sig,
+POST_SYSCALL(rt_tgsigqueueinfo)
-                                void *uinfo) {
+(long res, long tgid, long pid, long sig, void *uinfo) {
   if (res >= 0) {
-    if (uinfo) POST_WRITE(uinfo, siginfo_t_sz);
+    if (uinfo)
+      POST_WRITE(uinfo, siginfo_t_sz);
   }
 }
 @@ -736,7 +801,8 @@ PRE_SYSCALL(rt_sigqueueinfo)(long pid, long sig, void *uinfo) {}
 POST_SYSCALL(rt_sigqueueinfo)(long res, long pid, long sig, void *uinfo) {
   if (res >= 0) {
-    if (uinfo) POST_WRITE(uinfo, siginfo_t_sz);
+    if (uinfo)
+      POST_WRITE(uinfo, siginfo_t_sz);
   }
 }
 @@ -772,11 +838,11 @@ PRE_SYSCALL(bdflush)(long func, long data) {}
 POST_SYSCALL(bdflush)(long res, long func, long data) {}
-PRE_SYSCALL(mount)(void *dev_name, void *dir_name, void *type, long flags,
+PRE_SYSCALL(mount)
-                   void *data) {}
+(void *dev_name, void *dir_name, void *type, long flags, void *data) {}
-POST_SYSCALL(mount)(long res, void *dev_name, void *dir_name, void *type,
+POST_SYSCALL(mount)
-                    long flags, void *data) {
+(long res, void *dev_name, void *dir_name, void *type, long flags, void *data) {
   if (res >= 0) {
     if (dev_name)
       POST_WRITE(dev_name,
 @@ -826,11 +892,12 @@ PRE_SYSCALL(stat)(const void *filename, void *statbuf) {
 POST_SYSCALL(stat)(long res, const void *filename, void *statbuf) {
   if (res >= 0) {
-    if (statbuf) POST_WRITE(statbuf, struct___old_kernel_stat_sz);
+    if (statbuf)
+      POST_WRITE(statbuf, struct___old_kernel_stat_sz);
   }
 }
-#if !SANITIZER_ANDROID
+#  if !SANITIZER_ANDROID
 PRE_SYSCALL(statfs)(const void *path, void *buf) {
   if (path)
     PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
 @@ -838,26 +905,31 @@ PRE_SYSCALL(statfs)(const void *path, void *buf) {
 POST_SYSCALL(statfs)(long res, const void *path, void *buf) {
   if (res >= 0) {
-    if (buf) POST_WRITE(buf, struct_statfs_sz);
+    if (buf)
+      POST_WRITE(buf, struct_statfs_sz);
   }
 }
-PRE_SYSCALL(statfs64)(const void *path, long sz, void *buf) {
+PRE_SYSCALL(fstatfs)(long fd, void *buf) {}
-  if (path)
-    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
-}
-POST_SYSCALL(statfs64)(long res, const void *path, long sz, void *buf) {
+POST_SYSCALL(fstatfs)(long res, long fd, void *buf) {
   if (res >= 0) {
-    if (buf) POST_WRITE(buf, struct_statfs64_sz);
+    if (buf)
+      POST_WRITE(buf, struct_statfs_sz);
   }
 }
+#  endif  // !SANITIZER_ANDROID
-PRE_SYSCALL(fstatfs)(long fd, void *buf) {}
+#  if SANITIZER_GLIBC
+PRE_SYSCALL(statfs64)(const void *path, long sz, void *buf) {
+  if (path)
+    PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
+}
-POST_SYSCALL(fstatfs)(long res, long fd, void *buf) {
+POST_SYSCALL(statfs64)(long res, const void *path, long sz, void *buf) {
   if (res >= 0) {
-    if (buf) POST_WRITE(buf, struct_statfs_sz);
+    if (buf)
+      POST_WRITE(buf, struct_statfs64_sz);
   }
 }
 @@ -865,10 +937,11 @@ PRE_SYSCALL(fstatfs64)(long fd, long sz, void *buf) {}
 POST_SYSCALL(fstatfs64)(long res, long fd, long sz, void *buf) {
   if (res >= 0) {
-    if (buf) POST_WRITE(buf, struct_statfs64_sz);
+    if (buf)
+      POST_WRITE(buf, struct_statfs64_sz);
   }
 }
-#endif // !SANITIZER_ANDROID
+#  endif  // SANITIZER_GLIBC
 PRE_SYSCALL(lstat)(const void *filename, void *statbuf) {
   if (filename)
 @@ -878,7 +951,8 @@ PRE_SYSCALL(lstat)(const void *filename, void *statbuf) {
 POST_SYSCALL(lstat)(long res, const void *filename, void *statbuf) {
   if (res >= 0) {
-    if (statbuf) POST_WRITE(statbuf, struct___old_kernel_stat_sz);
+    if (statbuf)
+      POST_WRITE(statbuf, struct___old_kernel_stat_sz);
   }
 }
 @@ -886,7 +960,8 @@ PRE_SYSCALL(fstat)(long fd, void *statbuf) {}
 POST_SYSCALL(fstat)(long res, long fd, void *statbuf) {
   if (res >= 0) {
-    if (statbuf) POST_WRITE(statbuf, struct___old_kernel_stat_sz);
+    if (statbuf)
+      POST_WRITE(statbuf, struct___old_kernel_stat_sz);
   }
 }
 @@ -898,7 +973,8 @@ PRE_SYSCALL(newstat)(const void *filename, void *statbuf) {
 POST_SYSCALL(newstat)(long res, const void *filename, void *statbuf) {
   if (res >= 0) {
-    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
+    if (statbuf)
+      POST_WRITE(statbuf, struct_kernel_stat_sz);
   }
 }
 @@ -910,7 +986,8 @@ PRE_SYSCALL(newlstat)(const void *filename, void *statbuf) {
 POST_SYSCALL(newlstat)(long res, const void *filename, void *statbuf) {
   if (res >= 0) {
-    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
+    if (statbuf)
+      POST_WRITE(statbuf, struct_kernel_stat_sz);
   }
 }
 @@ -918,19 +995,21 @@ PRE_SYSCALL(newfstat)(long fd, void *statbuf) {}
 POST_SYSCALL(newfstat)(long res, long fd, void *statbuf) {
   if (res >= 0) {
-    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
+    if (statbuf)
+      POST_WRITE(statbuf, struct_kernel_stat_sz);
   }
 }
-#if !SANITIZER_ANDROID
+#  if SANITIZER_GLIBC
 PRE_SYSCALL(ustat)(long dev, void *ubuf) {}
 POST_SYSCALL(ustat)(long res, long dev, void *ubuf) {
   if (res >= 0) {
-    if (ubuf) POST_WRITE(ubuf, struct_ustat_sz);
+    if (ubuf)
+      POST_WRITE(ubuf, struct_ustat_sz);
   }
 }
-#endif  // !SANITIZER_ANDROID
+#  endif  // SANITIZER_GLIBC
 PRE_SYSCALL(stat64)(const void *filename, void *statbuf) {
   if (filename)
 @@ -940,7 +1019,8 @@ PRE_SYSCALL(stat64)(const void *filename, void *statbuf) {
 POST_SYSCALL(stat64)(long res, const void *filename, void *statbuf) {
   if (res >= 0) {
-    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
+    if (statbuf)
+      POST_WRITE(statbuf, struct_kernel_stat64_sz);
   }
 }
 @@ -948,7 +1028,8 @@ PRE_SYSCALL(fstat64)(long fd, void *statbuf) {}
 POST_SYSCALL(fstat64)(long res, long fd, void *statbuf) {
   if (res >= 0) {
-    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
+    if (statbuf)
+      POST_WRITE(statbuf, struct_kernel_stat64_sz);
   }
 }
 @@ -960,71 +1041,80 @@ PRE_SYSCALL(lstat64)(const void *filename, void *statbuf) {
 POST_SYSCALL(lstat64)(long res, const void *filename, void *statbuf) {
   if (res >= 0) {
-    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
+    if (statbuf)
+      POST_WRITE(statbuf, struct_kernel_stat64_sz);
   }
 }
-PRE_SYSCALL(setxattr)(const void *path, const void *name, const void *value,
+PRE_SYSCALL(setxattr)
-                      long size, long flags) {
+(const void *path, const void *name, const void *value, long size, long flags) {
   if (path)
     PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
   if (name)
     PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
-  if (value) PRE_READ(value, size);
+  if (value)
+    PRE_READ(value, size);
 }
-POST_SYSCALL(setxattr)(long res, const void *path, const void *name,
+POST_SYSCALL(setxattr)
-                       const void *value, long size, long flags) {}
+(long res, const void *path, const void *name, const void *value, long size,
+ long flags) {}
-PRE_SYSCALL(lsetxattr)(const void *path, const void *name, const void *value,
+PRE_SYSCALL(lsetxattr)
-                       long size, long flags) {
+(const void *path, const void *name, const void *value, long size, long flags) {
   if (path)
     PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
   if (name)
     PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
-  if (value) PRE_READ(value, size);
+  if (value)
+    PRE_READ(value, size);
 }
-POST_SYSCALL(lsetxattr)(long res, const void *path, const void *name,
+POST_SYSCALL(lsetxattr)
-                        const void *value, long size, long flags) {}
+(long res, const void *path, const void *name, const void *value, long size,
+ long flags) {}
-PRE_SYSCALL(fsetxattr)(long fd, const void *name, const void *value, long size,
+PRE_SYSCALL(fsetxattr)
-                       long flags) {
+(long fd, const void *name, const void *value, long size, long flags) {
   if (name)
     PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
-  if (value) PRE_READ(value, size);
+  if (value)
+    PRE_READ(value, size);
 }
-POST_SYSCALL(fsetxattr)(long res, long fd, const void *name, const void *value,
+POST_SYSCALL(fsetxattr)
-                        long size, long flags) {}
+(long res, long fd, const void *name, const void *value, long size,
+ long flags) {}
-PRE_SYSCALL(getxattr)(const void *path, const void *name, void *value,
+PRE_SYSCALL(getxattr)
-                      long size) {
+(const void *path, const void *name, void *value, long size) {
   if (path)
     PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
   if (name)
     PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
 }
-POST_SYSCALL(getxattr)(long res, const void *path, const void *name,
+POST_SYSCALL(getxattr)
-                       void *value, long size) {
+(long res, const void *path, const void *name, void *value, long size) {
   if (size && res > 0) {
-    if (value) POST_WRITE(value, res);
+    if (value)
+      POST_WRITE(value, res);
   }
 }
-PRE_SYSCALL(lgetxattr)(const void *path, const void *name, void *value,
+PRE_SYSCALL(lgetxattr)
-                       long size) {
+(const void *path, const void *name, void *value, long size) {
   if (path)
     PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
   if (name)
     PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
 }
-POST_SYSCALL(lgetxattr)(long res, const void *path, const void *name,
+POST_SYSCALL(lgetxattr)
-                        void *value, long size) {
+(long res, const void *path, const void *name, void *value, long size) {
   if (size && res > 0) {
-    if (value) POST_WRITE(value, res);
+    if (value)
+      POST_WRITE(value, res);
   }
 }
 @@ -1033,10 +1123,11 @@ PRE_SYSCALL(fgetxattr)(long fd, const void *name, void *value, long size) {
     PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
 }
-POST_SYSCALL(fgetxattr)(long res, long fd, const void *name, void *value,
+POST_SYSCALL(fgetxattr)
-                        long size) {
+(long res, long fd, const void *name, void *value, long size) {
   if (size && res > 0) {
-    if (value) POST_WRITE(value, res);
+    if (value)
+      POST_WRITE(value, res);
   }
 }
 @@ -1047,7 +1138,8 @@ PRE_SYSCALL(listxattr)(const void *path, void *list, long size) {
 POST_SYSCALL(listxattr)(long res, const void *path, void *list, long size) {
   if (size && res > 0) {
-    if (list) POST_WRITE(list, res);
+    if (list)
+      POST_WRITE(list, res);
   }
 }
 @@ -1058,7 +1150,8 @@ PRE_SYSCALL(llistxattr)(const void *path, void *list, long size) {
 POST_SYSCALL(llistxattr)(long res, const void *path, void *list, long size) {
   if (size && res > 0) {
-    if (list) POST_WRITE(list, res);
+    if (list)
+      POST_WRITE(list, res);
   }
 }
 @@ -1066,7 +1159,8 @@ PRE_SYSCALL(flistxattr)(long fd, void *list, long size) {}
 POST_SYSCALL(flistxattr)(long res, long fd, void *list, long size) {
   if (size && res > 0) {
-    if (list) POST_WRITE(list, res);
+    if (list)
+      POST_WRITE(list, res);
   }
 }
 @@ -1103,17 +1197,17 @@ PRE_SYSCALL(mprotect)(long start, long len, long prot) {}
 POST_SYSCALL(mprotect)(long res, long start, long len, long prot) {}
-PRE_SYSCALL(mremap)(long addr, long old_len, long new_len, long flags,
+PRE_SYSCALL(mremap)
-                    long new_addr) {}
+(long addr, long old_len, long new_len, long flags, long new_addr) {}
-POST_SYSCALL(mremap)(long res, long addr, long old_len, long new_len,
+POST_SYSCALL(mremap)
-                     long flags, long new_addr) {}
+(long res, long addr, long old_len, long new_len, long flags, long new_addr) {}
-PRE_SYSCALL(remap_file_pages)(long start, long size, long prot, long pgoff,
+PRE_SYSCALL(remap_file_pages)
-                              long flags) {}
+(long start, long size, long prot, long pgoff, long flags) {}
-POST_SYSCALL(remap_file_pages)(long res, long start, long size, long prot,
+POST_SYSCALL(remap_file_pages)
-                               long pgoff, long flags) {}
+(long res, long start, long size, long prot, long pgoff, long flags) {}
 PRE_SYSCALL(msync)(long start, long len, long flags) {}
 @@ -1189,7 +1283,8 @@ PRE_SYSCALL(link)(const void *oldname, const void *newname) {
 POST_SYSCALL(link)(long res, const void *oldname, const void *newname) {}
 PRE_SYSCALL(symlink)(const void *old, const void *new_) {
-  if (old) PRE_READ(old, __sanitizer::internal_strlen((const char *)old) + 1);
+  if (old)
+    PRE_READ(old, __sanitizer::internal_strlen((const char *)old) + 1);
   if (new_)
     PRE_READ(new_, __sanitizer::internal_strlen((const char *)new_) + 1);
 }
 @@ -1237,14 +1332,16 @@ PRE_SYSCALL(pipe)(void *fildes) {}
 POST_SYSCALL(pipe)(long res, void *fildes) {
   if (res >= 0)
-    if (fildes) POST_WRITE(fildes, sizeof(int) * 2);
+    if (fildes)
+      POST_WRITE(fildes, sizeof(int) * 2);
 }
 PRE_SYSCALL(pipe2)(void *fildes, long flags) {}
 POST_SYSCALL(pipe2)(long res, void *fildes, long flags) {
   if (res >= 0)
-    if (fildes) POST_WRITE(fildes, sizeof(int) * 2);
+    if (fildes)
+      POST_WRITE(fildes, sizeof(int) * 2);
 }
 PRE_SYSCALL(dup)(long fildes) {}
 @@ -1272,16 +1369,18 @@ PRE_SYSCALL(flock)(long fd, long cmd) {}
 POST_SYSCALL(flock)(long res, long fd, long cmd) {}
 PRE_SYSCALL(io_setup)(long nr_reqs, void **ctx) {
-  if (ctx) PRE_WRITE(ctx, sizeof(*ctx));
+  if (ctx)
+    PRE_WRITE(ctx, sizeof(*ctx));
 }
 POST_SYSCALL(io_setup)(long res, long nr_reqs, void **ctx) {
-  if (res >= 0) {
+  if (res >= 0 && ctx) {
-    if (ctx) POST_WRITE(ctx, sizeof(*ctx));
+    POST_WRITE(ctx, sizeof(*ctx));
     // (*ctx) is actually a pointer to a kernel mapped page, and there are
     // people out there who are crazy enough to peek into that page's 32-byte
     // header.
-    if (*ctx) POST_WRITE(*ctx, 32);
+    if (*ctx)
+      POST_WRITE(*ctx, 32);
   }
 }
 @@ -1289,16 +1388,21 @@ PRE_SYSCALL(io_destroy)(long ctx) {}
 POST_SYSCALL(io_destroy)(long res, long ctx) {}
-PRE_SYSCALL(io_getevents)(long ctx_id, long min_nr, long nr,
+PRE_SYSCALL(io_getevents)
-                          __sanitizer_io_event *ioevpp, void *timeout) {
+(long ctx_id, long min_nr, long nr, __sanitizer_io_event *ioevpp,
-  if (timeout) PRE_READ(timeout, struct_timespec_sz);
+ void *timeout) {
+  if (timeout)
+    PRE_READ(timeout, struct_timespec_sz);
 }
-POST_SYSCALL(io_getevents)(long res, long ctx_id, long min_nr, long nr,
+POST_SYSCALL(io_getevents)
-                           __sanitizer_io_event *ioevpp, void *timeout) {
+(long res, long ctx_id, long min_nr, long nr, __sanitizer_io_event *ioevpp,
+ void *timeout) {
   if (res >= 0) {
-    if (ioevpp) POST_WRITE(ioevpp, res * sizeof(*ioevpp));
+    if (ioevpp)
-    if (timeout) POST_WRITE(timeout, struct_timespec_sz);
+      POST_WRITE(ioevpp, res * sizeof(*ioevpp));
+    if (timeout)
+      POST_WRITE(timeout, struct_timespec_sz);
   }
   for (long i = 0; i < res; i++) {
     // We synchronize io_submit -> io_getevents/io_cancel using the
 @@ -1308,26 +1412,26 @@ POST_SYSCALL(io_getevents)(long res, long ctx_id, long min_nr, long nr,
     // synchronize on 0. But there does not seem to be a better solution
     // (except wrapping all operations in own context, which is unreliable).
     // We can not reliably extract fildes in io_getevents.
-    COMMON_SYSCALL_ACQUIRE((void*)ioevpp[i].data);
+    COMMON_SYSCALL_ACQUIRE((void *)ioevpp[i].data);
   }
 }
 PRE_SYSCALL(io_submit)(long ctx_id, long nr, __sanitizer_iocb **iocbpp) {
   for (long i = 0; i < nr; ++i) {
     uptr op = iocbpp[i]->aio_lio_opcode;
-    void *data = (void*)iocbpp[i]->aio_data;
+    void *data = (void *)iocbpp[i]->aio_data;
-    void *buf = (void*)iocbpp[i]->aio_buf;
+    void *buf = (void *)iocbpp[i]->aio_buf;
     uptr len = (uptr)iocbpp[i]->aio_nbytes;
     if (op == iocb_cmd_pwrite && buf && len) {
       PRE_READ(buf, len);
     } else if (op == iocb_cmd_pread && buf && len) {
       POST_WRITE(buf, len);
     } else if (op == iocb_cmd_pwritev) {
-      __sanitizer_iovec *iovec = (__sanitizer_iovec*)buf;
+      __sanitizer_iovec *iovec = (__sanitizer_iovec *)buf;
       for (uptr v = 0; v < len; v++)
         PRE_READ(iovec[v].iov_base, iovec[v].iov_len);
     } else if (op == iocb_cmd_preadv) {
-      __sanitizer_iovec *iovec = (__sanitizer_iovec*)buf;
+      __sanitizer_iovec *iovec = (__sanitizer_iovec *)buf;
       for (uptr v = 0; v < len; v++)
         POST_WRITE(iovec[v].iov_base, iovec[v].iov_len);
     }
 @@ -1336,19 +1440,18 @@ PRE_SYSCALL(io_submit)(long ctx_id, long nr, __sanitizer_iocb **iocbpp) {
   }
 }
-POST_SYSCALL(io_submit)(long res, long ctx_id, long nr,
+POST_SYSCALL(io_submit)
-    __sanitizer_iocb **iocbpp) {}
+(long res, long ctx_id, long nr, __sanitizer_iocb **iocbpp) {}
-PRE_SYSCALL(io_cancel)(long ctx_id, __sanitizer_iocb *iocb,
+PRE_SYSCALL(io_cancel)
-    __sanitizer_io_event *result) {
+(long ctx_id, __sanitizer_iocb *iocb, __sanitizer_io_event *result) {}
-}
-POST_SYSCALL(io_cancel)(long res, long ctx_id, __sanitizer_iocb *iocb,
+POST_SYSCALL(io_cancel)
-    __sanitizer_io_event *result) {
+(long res, long ctx_id, __sanitizer_iocb *iocb, __sanitizer_io_event *result) {
   if (res == 0) {
     if (result) {
       // See comment in io_getevents.
-      COMMON_SYSCALL_ACQUIRE((void*)result->data);
+      COMMON_SYSCALL_ACQUIRE((void *)result->data);
       POST_WRITE(result, sizeof(*result));
     }
     if (iocb)
 @@ -1358,19 +1461,23 @@ POST_SYSCALL(io_cancel)(long res, long ctx_id, __sanitizer_iocb *iocb,
 PRE_SYSCALL(sendfile)(long out_fd, long in_fd, void *offset, long count) {}
-POST_SYSCALL(sendfile)(long res, long out_fd, long in_fd,
+POST_SYSCALL(sendfile)
-                       __sanitizer___kernel_off_t *offset, long count) {
+(long res, long out_fd, long in_fd, __sanitizer___kernel_off_t *offset,
+ long count) {
   if (res >= 0) {
-    if (offset) POST_WRITE(offset, sizeof(*offset));
+    if (offset)
+      POST_WRITE(offset, sizeof(*offset));
   }
 }
 PRE_SYSCALL(sendfile64)(long out_fd, long in_fd, void *offset, long count) {}
-POST_SYSCALL(sendfile64)(long res, long out_fd, long in_fd,
+POST_SYSCALL(sendfile64)
-                         __sanitizer___kernel_loff_t *offset, long count) {
+(long res, long out_fd, long in_fd, __sanitizer___kernel_loff_t *offset,
+ long count) {
   if (res >= 0) {
-    if (offset) POST_WRITE(offset, sizeof(*offset));
+    if (offset)
+      POST_WRITE(offset, sizeof(*offset));
   }
 }
 @@ -1402,9 +1509,7 @@ PRE_SYSCALL(open)(const void *filename, long flags, long mode) {
 POST_SYSCALL(open)(long res, const void *filename, long flags, long mode) {}
-PRE_SYSCALL(close)(long fd) {
+PRE_SYSCALL(close)(long fd) { COMMON_SYSCALL_FD_CLOSE((int)fd); }
-  COMMON_SYSCALL_FD_CLOSE((int)fd);
-}
 POST_SYSCALL(close)(long res, long fd) {}
 @@ -1440,7 +1545,7 @@ PRE_SYSCALL(fchown)(long fd, long user, long group) {}
 POST_SYSCALL(fchown)(long res, long fd, long user, long group) {}
-#if SANITIZER_USES_UID16_SYSCALLS
+#  if SANITIZER_USES_UID16_SYSCALLS
 PRE_SYSCALL(chown16)(const void *filename, long user, long group) {
   if (filename)
     PRE_READ(filename,
 @@ -1483,13 +1588,16 @@ POST_SYSCALL(setresuid16)(long res, long ruid, long euid, long suid) {}
 PRE_SYSCALL(getresuid16)(void *ruid, void *euid, void *suid) {}
-POST_SYSCALL(getresuid16)(long res, __sanitizer___kernel_old_uid_t *ruid,
+POST_SYSCALL(getresuid16)
-                          __sanitizer___kernel_old_uid_t *euid,
+(long res, __sanitizer___kernel_old_uid_t *ruid,
-                          __sanitizer___kernel_old_uid_t *suid) {
+ __sanitizer___kernel_old_uid_t *euid, __sanitizer___kernel_old_uid_t *suid) {
   if (res >= 0) {
-    if (ruid) POST_WRITE(ruid, sizeof(*ruid));
+    if (ruid)
-    if (euid) POST_WRITE(euid, sizeof(*euid));
+      POST_WRITE(ruid, sizeof(*ruid));
-    if (suid) POST_WRITE(suid, sizeof(*suid));
+    if (euid)
+      POST_WRITE(euid, sizeof(*euid));
+    if (suid)
+      POST_WRITE(suid, sizeof(*suid));
   }
 }
 @@ -1499,13 +1607,16 @@ POST_SYSCALL(setresgid16)(long res, long rgid, long egid, long sgid) {}
 PRE_SYSCALL(getresgid16)(void *rgid, void *egid, void *sgid) {}
-POST_SYSCALL(getresgid16)(long res, __sanitizer___kernel_old_gid_t *rgid,
+POST_SYSCALL(getresgid16)
-                          __sanitizer___kernel_old_gid_t *egid,
+(long res, __sanitizer___kernel_old_gid_t *rgid,
-                          __sanitizer___kernel_old_gid_t *sgid) {
+ __sanitizer___kernel_old_gid_t *egid, __sanitizer___kernel_old_gid_t *sgid) {
   if (res >= 0) {
-    if (rgid) POST_WRITE(rgid, sizeof(*rgid));
+    if (rgid)
-    if (egid) POST_WRITE(egid, sizeof(*egid));
+      POST_WRITE(rgid, sizeof(*rgid));
-    if (sgid) POST_WRITE(sgid, sizeof(*sgid));
+    if (egid)
+      POST_WRITE(egid, sizeof(*egid));
+    if (sgid)
+      POST_WRITE(sgid, sizeof(*sgid));
   }
 }
 @@ -1517,23 +1628,25 @@ PRE_SYSCALL(setfsgid16)(long gid) {}
 POST_SYSCALL(setfsgid16)(long res, long gid) {}
-PRE_SYSCALL(getgroups16)(long gidsetsize,
+PRE_SYSCALL(getgroups16)
-                         __sanitizer___kernel_old_gid_t *grouplist) {}
+(long gidsetsize, __sanitizer___kernel_old_gid_t *grouplist) {}
-POST_SYSCALL(getgroups16)(long res, long gidsetsize,
+POST_SYSCALL(getgroups16)
-                          __sanitizer___kernel_old_gid_t *grouplist) {
+(long res, long gidsetsize, __sanitizer___kernel_old_gid_t *grouplist) {
   if (res >= 0) {
-    if (grouplist) POST_WRITE(grouplist, res * sizeof(*grouplist));
+    if (grouplist)
+      POST_WRITE(grouplist, res * sizeof(*grouplist));
   }
 }
-PRE_SYSCALL(setgroups16)(long gidsetsize,
+PRE_SYSCALL(setgroups16)
-                         __sanitizer___kernel_old_gid_t *grouplist) {
+(long gidsetsize, __sanitizer___kernel_old_gid_t *grouplist) {
-  if (grouplist) POST_WRITE(grouplist, gidsetsize * sizeof(*grouplist));
+  if (grouplist)
+    POST_WRITE(grouplist, gidsetsize * sizeof(*grouplist));
 }
-POST_SYSCALL(setgroups16)(long res, long gidsetsize,
+POST_SYSCALL(setgroups16)
-                          __sanitizer___kernel_old_gid_t *grouplist) {}
+(long res, long gidsetsize, __sanitizer___kernel_old_gid_t *grouplist) {}
 PRE_SYSCALL(getuid16)() {}
 @@ -1550,7 +1663,7 @@ POST_SYSCALL(getgid16)(long res) {}
 PRE_SYSCALL(getegid16)() {}
 POST_SYSCALL(getegid16)(long res) {}
-#endif // SANITIZER_USES_UID16_SYSCALLS
+#  endif  // SANITIZER_USES_UID16_SYSCALLS
 PRE_SYSCALL(utime)(void *filename, void *times) {}
 @@ -1559,7 +1672,8 @@ POST_SYSCALL(utime)(long res, void *filename, void *times) {
     if (filename)
       POST_WRITE(filename,
                  __sanitizer::internal_strlen((const char *)filename) + 1);
-    if (times) POST_WRITE(times, struct_utimbuf_sz);
+    if (times)
+      POST_WRITE(times, struct_utimbuf_sz);
   }
 }
 @@ -1570,7 +1684,8 @@ POST_SYSCALL(utimes)(long res, void *filename, void *utimes) {
     if (filename)
       POST_WRITE(filename,
                  __sanitizer::internal_strlen((const char *)filename) + 1);
-    if (utimes) POST_WRITE(utimes, timeval_sz);
+    if (utimes)
+      POST_WRITE(utimes, timeval_sz);
   }
 }
 @@ -1578,91 +1693,104 @@ PRE_SYSCALL(lseek)(long fd, long offset, long origin) {}
 POST_SYSCALL(lseek)(long res, long fd, long offset, long origin) {}
-PRE_SYSCALL(llseek)(long fd, long offset_high, long offset_low, void *result,
+PRE_SYSCALL(llseek)
-                    long origin) {}
+(long fd, long offset_high, long offset_low, void *result, long origin) {}
-POST_SYSCALL(llseek)(long res, long fd, long offset_high, long offset_low,
+POST_SYSCALL(llseek)
-                     void *result, long origin) {
+(long res, long fd, long offset_high, long offset_low, void *result,
+ long origin) {
   if (res >= 0) {
-    if (result) POST_WRITE(result, sizeof(long long));
+    if (result)
+      POST_WRITE(result, sizeof(long long));
   }
 }
 PRE_SYSCALL(readv)(long fd, const __sanitizer_iovec *vec, long vlen) {}
-POST_SYSCALL(readv)(long res, long fd, const __sanitizer_iovec *vec,
+POST_SYSCALL(readv)
-                    long vlen) {
+(long res, long fd, const __sanitizer_iovec *vec, long vlen) {
   if (res >= 0) {
-    if (vec) kernel_write_iovec(vec, vlen, res);
+    if (vec)
+      kernel_write_iovec(vec, vlen, res);
   }
 }
 PRE_SYSCALL(write)(long fd, const void *buf, long count) {
-  if (buf) PRE_READ(buf, count);
+  if (buf)
+    PRE_READ(buf, count);
 }
 POST_SYSCALL(write)(long res, long fd, const void *buf, long count) {}
 PRE_SYSCALL(writev)(long fd, const __sanitizer_iovec *vec, long vlen) {}
-POST_SYSCALL(writev)(long res, long fd, const __sanitizer_iovec *vec,
+POST_SYSCALL(writev)
-                     long vlen) {
+(long res, long fd, const __sanitizer_iovec *vec, long vlen) {
   if (res >= 0) {
-    if (vec) kernel_read_iovec(vec, vlen, res);
+    if (vec)
+      kernel_read_iovec(vec, vlen, res);
   }
 }
-#ifdef _LP64
+#  ifdef _LP64
 PRE_SYSCALL(pread64)(long fd, void *buf, long count, long pos) {}
 POST_SYSCALL(pread64)(long res, long fd, void *buf, long count, long pos) {
   if (res >= 0) {
-    if (buf) POST_WRITE(buf, res);
+    if (buf)
+      POST_WRITE(buf, res);
   }
 }
 PRE_SYSCALL(pwrite64)(long fd, const void *buf, long count, long pos) {
-  if (buf) PRE_READ(buf, count);
+  if (buf)
+    PRE_READ(buf, count);
 }
-POST_SYSCALL(pwrite64)(long res, long fd, const void *buf, long count,
+POST_SYSCALL(pwrite64)
-                       long pos) {}
+(long res, long fd, const void *buf, long count, long pos) {}
-#else
+#  else
 PRE_SYSCALL(pread64)(long fd, void *buf, long count, long pos0, long pos1) {}
-POST_SYSCALL(pread64)(long res, long fd, void *buf, long count, long pos0,
+POST_SYSCALL(pread64)
-                      long pos1) {
+(long res, long fd, void *buf, long count, long pos0, long pos1) {
   if (res >= 0) {
-    if (buf) POST_WRITE(buf, res);
+    if (buf)
+      POST_WRITE(buf, res);
   }
 }
-PRE_SYSCALL(pwrite64)(long fd, const void *buf, long count, long pos0,
+PRE_SYSCALL(pwrite64)
-                      long pos1) {
+(long fd, const void *buf, long count, long pos0, long pos1) {
-  if (buf) PRE_READ(buf, count);
+  if (buf)
+    PRE_READ(buf, count);
 }
-POST_SYSCALL(pwrite64)(long res, long fd, const void *buf, long count,
+POST_SYSCALL(pwrite64)
-                       long pos0, long pos1) {}
+(long res, long fd, const void *buf, long count, long pos0, long pos1) {}
-#endif
+#  endif
-PRE_SYSCALL(preadv)(long fd, const __sanitizer_iovec *vec, long vlen,
+PRE_SYSCALL(preadv)
-                    long pos_l, long pos_h) {}
+(long fd, const __sanitizer_iovec *vec, long vlen, long pos_l, long pos_h) {}
-POST_SYSCALL(preadv)(long res, long fd, const __sanitizer_iovec *vec, long vlen,
+POST_SYSCALL(preadv)
-                     long pos_l, long pos_h) {
+(long res, long fd, const __sanitizer_iovec *vec, long vlen, long pos_l,
+ long pos_h) {
   if (res >= 0) {
-    if (vec) kernel_write_iovec(vec, vlen, res);
+    if (vec)
+      kernel_write_iovec(vec, vlen, res);
   }
 }
-PRE_SYSCALL(pwritev)(long fd, const __sanitizer_iovec *vec, long vlen,
+PRE_SYSCALL(pwritev)
-                     long pos_l, long pos_h) {}
+(long fd, const __sanitizer_iovec *vec, long vlen, long pos_l, long pos_h) {}
-POST_SYSCALL(pwritev)(long res, long fd, const __sanitizer_iovec *vec,
+POST_SYSCALL(pwritev)
-                      long vlen, long pos_l, long pos_h) {
+(long res, long fd, const __sanitizer_iovec *vec, long vlen, long pos_l,
+ long pos_h) {
   if (res >= 0) {
-    if (vec) kernel_read_iovec(vec, vlen, res);
+    if (vec)
+      kernel_read_iovec(vec, vlen, res);
   }
 }
 @@ -1717,14 +1845,15 @@ PRE_SYSCALL(quotactl)(long cmd, const void *special, long id, void *addr) {
     PRE_READ(special, __sanitizer::internal_strlen((const char *)special) + 1);
 }
-POST_SYSCALL(quotactl)(long res, long cmd, const void *special, long id,
+POST_SYSCALL(quotactl)
-                       void *addr) {}
+(long res, long cmd, const void *special, long id, void *addr) {}
 PRE_SYSCALL(getdents)(long fd, void *dirent, long count) {}
 POST_SYSCALL(getdents)(long res, long fd, void *dirent, long count) {
   if (res >= 0) {
-    if (dirent) POST_WRITE(dirent, res);
+    if (dirent)
+      POST_WRITE(dirent, res);
   }
 }
 @@ -1732,15 +1861,16 @@ PRE_SYSCALL(getdents64)(long fd, void *dirent, long count) {}
 POST_SYSCALL(getdents64)(long res, long fd, void *dirent, long count) {
   if (res >= 0) {
-    if (dirent) POST_WRITE(dirent, res);
+    if (dirent)
+      POST_WRITE(dirent, res);
   }
 }
-PRE_SYSCALL(setsockopt)(long fd, long level, long optname, void *optval,
+PRE_SYSCALL(setsockopt)
-                        long optlen) {}
+(long fd, long level, long optname, void *optval, long optlen) {}
-POST_SYSCALL(setsockopt)(long res, long fd, long level, long optname,
+POST_SYSCALL(setsockopt)
-                         void *optval, long optlen) {
+(long res, long fd, long level, long optname, void *optval, long optlen) {
   if (res >= 0) {
     if (optval)
       POST_WRITE(optval,
 @@ -1748,77 +1878,88 @@ POST_SYSCALL(setsockopt)(long res, long fd, long level, long optname,
   }
 }
-PRE_SYSCALL(getsockopt)(long fd, long level, long optname, void *optval,
+PRE_SYSCALL(getsockopt)
-                        void *optlen) {}
+(long fd, long level, long optname, void *optval, void *optlen) {}
-POST_SYSCALL(getsockopt)(long res, long fd, long level, long optname,
+POST_SYSCALL(getsockopt)
-                         void *optval, void *optlen) {
+(long res, long fd, long level, long optname, void *optval, void *optlen) {
   if (res >= 0) {
     if (optval)
       POST_WRITE(optval,
                  __sanitizer::internal_strlen((const char *)optval) + 1);
-    if (optlen) POST_WRITE(optlen, sizeof(int));
+    if (optlen)
+      POST_WRITE(optlen, sizeof(int));
   }
 }
 PRE_SYSCALL(bind)(long arg0, sanitizer_kernel_sockaddr *arg1, long arg2) {}
-POST_SYSCALL(bind)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
+POST_SYSCALL(bind)
-                   long arg2) {
+(long res, long arg0, sanitizer_kernel_sockaddr *arg1, long arg2) {
   if (res >= 0) {
-    if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+    if (arg1)
+      POST_WRITE(arg1, sizeof(*arg1));
   }
 }
 PRE_SYSCALL(connect)(long arg0, sanitizer_kernel_sockaddr *arg1, long arg2) {}
-POST_SYSCALL(connect)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
+POST_SYSCALL(connect)
-                      long arg2) {
+(long res, long arg0, sanitizer_kernel_sockaddr *arg1, long arg2) {
   if (res >= 0) {
-    if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+    if (arg1)
+      POST_WRITE(arg1, sizeof(*arg1));
   }
 }
 PRE_SYSCALL(accept)(long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2) {}
-POST_SYSCALL(accept)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
+POST_SYSCALL(accept)
-                     void *arg2) {
+(long res, long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2) {
   if (res >= 0) {
-    if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+    if (arg1)
-    if (arg2) POST_WRITE(arg2, sizeof(unsigned));
+      POST_WRITE(arg1, sizeof(*arg1));
+    if (arg2)
+      POST_WRITE(arg2, sizeof(unsigned));
   }
 }
-PRE_SYSCALL(accept4)(long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2,
+PRE_SYSCALL(accept4)
-                     long arg3) {}
+(long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2, long arg3) {}
-POST_SYSCALL(accept4)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
+POST_SYSCALL(accept4)
-                      void *arg2, long arg3) {
+(long res, long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2, long arg3) {
   if (res >= 0) {
-    if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+    if (arg1)
-    if (arg2) POST_WRITE(arg2, sizeof(unsigned));
+      POST_WRITE(arg1, sizeof(*arg1));
+    if (arg2)
+      POST_WRITE(arg2, sizeof(unsigned));
   }
 }
-PRE_SYSCALL(getsockname)(long arg0, sanitizer_kernel_sockaddr *arg1,
+PRE_SYSCALL(getsockname)
-                         void *arg2) {}
+(long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2) {}
-POST_SYSCALL(getsockname)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
+POST_SYSCALL(getsockname)
-                          void *arg2) {
+(long res, long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2) {
   if (res >= 0) {
-    if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+    if (arg1)
-    if (arg2) POST_WRITE(arg2, sizeof(unsigned));
+      POST_WRITE(arg1, sizeof(*arg1));
+    if (arg2)
+      POST_WRITE(arg2, sizeof(unsigned));
   }
 }
-PRE_SYSCALL(getpeername)(long arg0, sanitizer_kernel_sockaddr *arg1,
+PRE_SYSCALL(getpeername)
-                         void *arg2) {}
+(long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2) {}
-POST_SYSCALL(getpeername)(long res, long arg0, sanitizer_kernel_sockaddr *arg1,
+POST_SYSCALL(getpeername)
-                          void *arg2) {
+(long res, long arg0, sanitizer_kernel_sockaddr *arg1, void *arg2) {
   if (res >= 0) {
-    if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+    if (arg1)
-    if (arg2) POST_WRITE(arg2, sizeof(unsigned));
+      POST_WRITE(arg1, sizeof(*arg1));
+    if (arg2)
+      POST_WRITE(arg2, sizeof(unsigned));
   }
 }
 @@ -1826,18 +1967,23 @@ PRE_SYSCALL(send)(long arg0, void *arg1, long arg2, long arg3) {}
 POST_SYSCALL(send)(long res, long arg0, void *arg1, long arg2, long arg3) {
   if (res) {
-    if (arg1) POST_READ(arg1, res);
+    if (arg1)
+      POST_READ(arg1, res);
   }
 }
-PRE_SYSCALL(sendto)(long arg0, void *arg1, long arg2, long arg3,
+PRE_SYSCALL(sendto)
-                    sanitizer_kernel_sockaddr *arg4, long arg5) {}
+(long arg0, void *arg1, long arg2, long arg3, sanitizer_kernel_sockaddr *arg4,
+ long arg5) {}
-POST_SYSCALL(sendto)(long res, long arg0, void *arg1, long arg2, long arg3,
+POST_SYSCALL(sendto)
-                     sanitizer_kernel_sockaddr *arg4, long arg5) {
+(long res, long arg0, void *arg1, long arg2, long arg3,
+ sanitizer_kernel_sockaddr *arg4, long arg5) {
   if (res >= 0) {
-    if (arg1) POST_READ(arg1, res);
+    if (arg1)
-    if (arg4) POST_WRITE(arg4, sizeof(*arg4));
+      POST_READ(arg1, res);
+    if (arg4)
+      POST_WRITE(arg4, sizeof(*arg4));
   }
 }
 @@ -1857,19 +2003,25 @@ PRE_SYSCALL(recv)(long arg0, void *buf, long len, long flags) {}
 POST_SYSCALL(recv)(long res, void *buf, long len, long flags) {
   if (res >= 0) {
-    if (buf) POST_WRITE(buf, res);
+    if (buf)
+      POST_WRITE(buf, res);
   }
 }
-PRE_SYSCALL(recvfrom)(long arg0, void *buf, long len, long flags,
+PRE_SYSCALL(recvfrom)
-                      sanitizer_kernel_sockaddr *arg4, void *arg5) {}
+(long arg0, void *buf, long len, long flags, sanitizer_kernel_sockaddr *arg4,
+ void *arg5) {}
-POST_SYSCALL(recvfrom)(long res, long arg0, void *buf, long len, long flags,
+POST_SYSCALL(recvfrom)
-                       sanitizer_kernel_sockaddr *arg4, void *arg5) {
+(long res, long arg0, void *buf, long len, long flags,
+ sanitizer_kernel_sockaddr *arg4, void *arg5) {
   if (res >= 0) {
-    if (buf) POST_WRITE(buf, res);
+    if (buf)
-    if (arg4) POST_WRITE(arg4, sizeof(*arg4));
+      POST_WRITE(buf, res);
-    if (arg5) POST_WRITE(arg5, sizeof(int));
+    if (arg4)
+      POST_WRITE(arg4, sizeof(*arg4));
+    if (arg5)
+      POST_WRITE(arg5, sizeof(int));
   }
 }
 @@ -1881,14 +2033,16 @@ PRE_SYSCALL(socketpair)(long arg0, long arg1, long arg2, int *sv) {}
 POST_SYSCALL(socketpair)(long res, long arg0, long arg1, long arg2, int *sv) {
   if (res >= 0)
-    if (sv) POST_WRITE(sv, sizeof(int) * 2);
+    if (sv)
+      POST_WRITE(sv, sizeof(int) * 2);
 }
 PRE_SYSCALL(socketcall)(long call, void *args) {}
 POST_SYSCALL(socketcall)(long res, long call, void *args) {
   if (res >= 0) {
-    if (args) POST_WRITE(args, sizeof(long));
+    if (args)
+      POST_WRITE(args, sizeof(long));
   }
 }
 @@ -1898,25 +2052,31 @@ POST_SYSCALL(listen)(long res, long arg0, long arg1) {}
 PRE_SYSCALL(poll)(void *ufds, long nfds, long timeout) {}
-POST_SYSCALL(poll)(long res, __sanitizer_pollfd *ufds, long nfds,
+POST_SYSCALL(poll)
-                   long timeout) {
+(long res, __sanitizer_pollfd *ufds, long nfds, long timeout) {
   if (res >= 0) {
-    if (ufds) POST_WRITE(ufds, nfds * sizeof(*ufds));
+    if (ufds)
+      POST_WRITE(ufds, nfds * sizeof(*ufds));
   }
 }
-PRE_SYSCALL(select)(long n, __sanitizer___kernel_fd_set *inp,
+PRE_SYSCALL(select)
-                    __sanitizer___kernel_fd_set *outp,
+(long n, __sanitizer___kernel_fd_set *inp, __sanitizer___kernel_fd_set *outp,
-                    __sanitizer___kernel_fd_set *exp, void *tvp) {}
+ __sanitizer___kernel_fd_set *exp, void *tvp) {}
-POST_SYSCALL(select)(long res, long n, __sanitizer___kernel_fd_set *inp,
+POST_SYSCALL(select)
-                     __sanitizer___kernel_fd_set *outp,
+(long res, long n, __sanitizer___kernel_fd_set *inp,
-                     __sanitizer___kernel_fd_set *exp, void *tvp) {
+ __sanitizer___kernel_fd_set *outp, __sanitizer___kernel_fd_set *exp,
+ void *tvp) {
   if (res >= 0) {
-    if (inp) POST_WRITE(inp, sizeof(*inp));
+    if (inp)
-    if (outp) POST_WRITE(outp, sizeof(*outp));
+      POST_WRITE(inp, sizeof(*inp));
-    if (exp) POST_WRITE(exp, sizeof(*exp));
+    if (outp)
-    if (tvp) POST_WRITE(tvp, timeval_sz);
+      POST_WRITE(outp, sizeof(*outp));
+    if (exp)
+      POST_WRITE(exp, sizeof(*exp));
+    if (tvp)
+      POST_WRITE(tvp, timeval_sz);
   }
 }
 @@ -1936,29 +2096,58 @@ PRE_SYSCALL(epoll_ctl)(long epfd, long op, long fd, void *event) {}
 POST_SYSCALL(epoll_ctl)(long res, long epfd, long op, long fd, void *event) {
   if (res >= 0) {
-    if (event) POST_WRITE(event, struct_epoll_event_sz);
+    if (event)
+      POST_WRITE(event, struct_epoll_event_sz);
+  }
+}
+PRE_SYSCALL(epoll_wait)
+(long epfd, void *events, long maxevents, long timeout) {}
+POST_SYSCALL(epoll_wait)
+(long res, long epfd, void *events, long maxevents, long timeout) {
+  if (res >= 0) {
+    COMMON_SYSCALL_FD_ACQUIRE(epfd);
+    if (events)
+      POST_WRITE(events, res * struct_epoll_event_sz);
   }
 }
-PRE_SYSCALL(epoll_wait)(long epfd, void *events, long maxevents, long timeout) {
+PRE_SYSCALL(epoll_pwait)
+(long epfd, void *events, long maxevents, long timeout,
+ const kernel_sigset_t *sigmask, long sigsetsize) {
+  if (sigmask)
+    PRE_READ(sigmask, sigsetsize);
 }
-POST_SYSCALL(epoll_wait)(long res, long epfd, void *events, long maxevents,
+POST_SYSCALL(epoll_pwait)
-                         long timeout) {
+(long res, long epfd, void *events, long maxevents, long timeout,
+ const void *sigmask, long sigsetsize) {
   if (res >= 0) {
-    if (events) POST_WRITE(events, struct_epoll_event_sz);
+    COMMON_SYSCALL_FD_ACQUIRE(epfd);
+    if (events)
+      POST_WRITE(events, res * struct_epoll_event_sz);
   }
 }
-PRE_SYSCALL(epoll_pwait)(long epfd, void *events, long maxevents, long timeout,
+PRE_SYSCALL(epoll_pwait2)
-                         const kernel_sigset_t *sigmask, long sigsetsize) {
+(long epfd, void *events, long maxevents,
-  if (sigmask) PRE_READ(sigmask, sigsetsize);
+ const sanitizer_kernel_timespec *timeout, const kernel_sigset_t *sigmask,
+ long sigsetsize) {
+  if (timeout)
+    PRE_READ(timeout, sizeof(*timeout));
+  if (sigmask)
+    PRE_READ(sigmask, sigsetsize);
 }
-POST_SYSCALL(epoll_pwait)(long res, long epfd, void *events, long maxevents,
+POST_SYSCALL(epoll_pwait2)
-                          long timeout, const void *sigmask, long sigsetsize) {
+(long res, long epfd, void *events, long maxevents,
+ const sanitizer_kernel_timespec *timeout, const void *sigmask,
+ long sigsetsize) {
   if (res >= 0) {
-    if (events) POST_WRITE(events, struct_epoll_event_sz);
+    COMMON_SYSCALL_FD_ACQUIRE(epfd);
+    if (events)
+      POST_WRITE(events, res * struct_epoll_event_sz);
   }
 }
 @@ -1993,7 +2182,8 @@ PRE_SYSCALL(newuname)(void *name) {}
 POST_SYSCALL(newuname)(long res, void *name) {
   if (res >= 0) {
-    if (name) POST_WRITE(name, struct_new_utsname_sz);
+    if (name)
+      POST_WRITE(name, struct_new_utsname_sz);
   }
 }
 @@ -2001,7 +2191,8 @@ PRE_SYSCALL(uname)(void *arg0) {}
 POST_SYSCALL(uname)(long res, void *arg0) {
   if (res >= 0) {
-    if (arg0) POST_WRITE(arg0, struct_old_utsname_sz);
+    if (arg0)
+      POST_WRITE(arg0, struct_old_utsname_sz);
   }
 }
 @@ -2009,7 +2200,8 @@ PRE_SYSCALL(olduname)(void *arg0) {}
 POST_SYSCALL(olduname)(long res, void *arg0) {
   if (res >= 0) {
-    if (arg0) POST_WRITE(arg0, struct_oldold_utsname_sz);
+    if (arg0)
+      POST_WRITE(arg0, struct_oldold_utsname_sz);
   }
 }
 @@ -2017,7 +2209,8 @@ PRE_SYSCALL(getrlimit)(long resource, void *rlim) {}
 POST_SYSCALL(getrlimit)(long res, long resource, void *rlim) {
   if (res >= 0) {
-    if (rlim) POST_WRITE(rlim, struct_rlimit_sz);
+    if (rlim)
+      POST_WRITE(rlim, struct_rlimit_sz);
   }
 }
 @@ -2025,7 +2218,8 @@ PRE_SYSCALL(old_getrlimit)(long resource, void *rlim) {}
 POST_SYSCALL(old_getrlimit)(long res, long resource, void *rlim) {
   if (res >= 0) {
-    if (rlim) POST_WRITE(rlim, struct_rlimit_sz);
+    if (rlim)
+      POST_WRITE(rlim, struct_rlimit_sz);
   }
 }
 @@ -2033,29 +2227,33 @@ PRE_SYSCALL(setrlimit)(long resource, void *rlim) {}
 POST_SYSCALL(setrlimit)(long res, long resource, void *rlim) {
   if (res >= 0) {
-    if (rlim) POST_WRITE(rlim, struct_rlimit_sz);
+    if (rlim)
+      POST_WRITE(rlim, struct_rlimit_sz);
   }
 }
-#if !SANITIZER_ANDROID
+#  if SANITIZER_GLIBC
-PRE_SYSCALL(prlimit64)(long pid, long resource, const void *new_rlim,
+PRE_SYSCALL(prlimit64)
-                       void *old_rlim) {
+(long pid, long resource, const void *new_rlim, void *old_rlim) {
-  if (new_rlim) PRE_READ(new_rlim, struct_rlimit64_sz);
+  if (new_rlim)
+    PRE_READ(new_rlim, struct_rlimit64_sz);
 }
-POST_SYSCALL(prlimit64)(long res, long pid, long resource, const void *new_rlim,
+POST_SYSCALL(prlimit64)
-                        void *old_rlim) {
+(long res, long pid, long resource, const void *new_rlim, void *old_rlim) {
   if (res >= 0) {
-    if (old_rlim) POST_WRITE(old_rlim, struct_rlimit64_sz);
+    if (old_rlim)
+      POST_WRITE(old_rlim, struct_rlimit64_sz);
   }
 }
-#endif
+#  endif
 PRE_SYSCALL(getrusage)(long who, void *ru) {}
 POST_SYSCALL(getrusage)(long res, long who, void *ru) {
   if (res >= 0) {
-    if (ru) POST_WRITE(ru, struct_rusage_sz);
+    if (ru)
+      POST_WRITE(ru, struct_rusage_sz);
   }
 }
 @@ -2068,31 +2266,34 @@ PRE_SYSCALL(msgget)(long key, long msgflg) {}
 POST_SYSCALL(msgget)(long res, long key, long msgflg) {}
 PRE_SYSCALL(msgsnd)(long msqid, void *msgp, long msgsz, long msgflg) {
-  if (msgp) PRE_READ(msgp, msgsz);
+  if (msgp)
+    PRE_READ(msgp, msgsz);
 }
-POST_SYSCALL(msgsnd)(long res, long msqid, void *msgp, long msgsz,
+POST_SYSCALL(msgsnd)
-                     long msgflg) {}
+(long res, long msqid, void *msgp, long msgsz, long msgflg) {}
-PRE_SYSCALL(msgrcv)(long msqid, void *msgp, long msgsz, long msgtyp,
+PRE_SYSCALL(msgrcv)
-                    long msgflg) {}
+(long msqid, void *msgp, long msgsz, long msgtyp, long msgflg) {}
-POST_SYSCALL(msgrcv)(long res, long msqid, void *msgp, long msgsz, long msgtyp,
+POST_SYSCALL(msgrcv)
-                     long msgflg) {
+(long res, long msqid, void *msgp, long msgsz, long msgtyp, long msgflg) {
   if (res >= 0) {
-    if (msgp) POST_WRITE(msgp, res);
+    if (msgp)
+      POST_WRITE(msgp, res);
   }
 }
-#if !SANITIZER_ANDROID
+#  if !SANITIZER_ANDROID
 PRE_SYSCALL(msgctl)(long msqid, long cmd, void *buf) {}
 POST_SYSCALL(msgctl)(long res, long msqid, long cmd, void *buf) {
   if (res >= 0) {
-    if (buf) POST_WRITE(buf, struct_msqid_ds_sz);
+    if (buf)
+      POST_WRITE(buf, struct_msqid_ds_sz);
   }
 }
-#endif
+#  endif
 PRE_SYSCALL(semget)(long key, long nsems, long semflg) {}
 @@ -2106,13 +2307,14 @@ PRE_SYSCALL(semctl)(long semid, long semnum, long cmd, void *arg) {}
 POST_SYSCALL(semctl)(long res, long semid, long semnum, long cmd, void *arg) {}
-PRE_SYSCALL(semtimedop)(long semid, void *sops, long nsops,
+PRE_SYSCALL(semtimedop)
-                        const void *timeout) {
+(long semid, void *sops, long nsops, const void *timeout) {
-  if (timeout) PRE_READ(timeout, struct_timespec_sz);
+  if (timeout)
+    PRE_READ(timeout, struct_timespec_sz);
 }
-POST_SYSCALL(semtimedop)(long res, long semid, void *sops, long nsops,
+POST_SYSCALL(semtimedop)
-                         const void *timeout) {}
+(long res, long semid, void *sops, long nsops, const void *timeout) {}
 PRE_SYSCALL(shmat)(long shmid, void *shmaddr, long shmflg) {}
 @@ -2138,18 +2340,20 @@ POST_SYSCALL(shmdt)(long res, void *shmaddr) {
   }
 }
-PRE_SYSCALL(ipc)(long call, long first, long second, long third, void *ptr,
+PRE_SYSCALL(ipc)
-                 long fifth) {}
+(long call, long first, long second, long third, void *ptr, long fifth) {}
-POST_SYSCALL(ipc)(long res, long call, long first, long second, long third,
+POST_SYSCALL(ipc)
-                  void *ptr, long fifth) {}
+(long res, long call, long first, long second, long third, void *ptr,
+ long fifth) {}
-#if !SANITIZER_ANDROID
+#  if !SANITIZER_ANDROID
 PRE_SYSCALL(shmctl)(long shmid, long cmd, void *buf) {}
 POST_SYSCALL(shmctl)(long res, long shmid, long cmd, void *buf) {
   if (res >= 0) {
-    if (buf) POST_WRITE(buf, sizeof(__sanitizer_shmid_ds));
+    if (buf)
+      POST_WRITE(buf, sizeof(__sanitizer_shmid_ds));
   }
 }
 @@ -2158,10 +2362,11 @@ PRE_SYSCALL(mq_open)(const void *name, long oflag, long mode, void *attr) {
     PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
 }
-POST_SYSCALL(mq_open)(long res, const void *name, long oflag, long mode,
+POST_SYSCALL(mq_open)
-                      void *attr) {
+(long res, const void *name, long oflag, long mode, void *attr) {
   if (res >= 0) {
-    if (attr) POST_WRITE(attr, struct_mq_attr_sz);
+    if (attr)
+      POST_WRITE(attr, struct_mq_attr_sz);
   }
 }
 @@ -2172,62 +2377,73 @@ PRE_SYSCALL(mq_unlink)(const void *name) {
 POST_SYSCALL(mq_unlink)(long res, const void *name) {}
-PRE_SYSCALL(mq_timedsend)(long mqdes, const void *msg_ptr, long msg_len,
+PRE_SYSCALL(mq_timedsend)
-                          long msg_prio, const void *abs_timeout) {
+(long mqdes, const void *msg_ptr, long msg_len, long msg_prio,
-  if (msg_ptr) PRE_READ(msg_ptr, msg_len);
+ const void *abs_timeout) {
-  if (abs_timeout) PRE_READ(abs_timeout, struct_timespec_sz);
+  if (msg_ptr)
+    PRE_READ(msg_ptr, msg_len);
+  if (abs_timeout)
+    PRE_READ(abs_timeout, struct_timespec_sz);
 }
-POST_SYSCALL(mq_timedsend)(long res, long mqdes, const void *msg_ptr,
+POST_SYSCALL(mq_timedsend)
-                           long msg_len, long msg_prio,
+(long res, long mqdes, const void *msg_ptr, long msg_len, long msg_prio,
-                           const void *abs_timeout) {}
+ const void *abs_timeout) {}
-PRE_SYSCALL(mq_timedreceive)(long mqdes, void *msg_ptr, long msg_len,
+PRE_SYSCALL(mq_timedreceive)
-                             void *msg_prio, const void *abs_timeout) {
+(long mqdes, void *msg_ptr, long msg_len, void *msg_prio,
-  if (abs_timeout) PRE_READ(abs_timeout, struct_timespec_sz);
+ const void *abs_timeout) {
+  if (abs_timeout)
+    PRE_READ(abs_timeout, struct_timespec_sz);
 }
-POST_SYSCALL(mq_timedreceive)(long res, long mqdes, void *msg_ptr, long msg_len,
+POST_SYSCALL(mq_timedreceive)
-                              int *msg_prio, const void *abs_timeout) {
+(long res, long mqdes, void *msg_ptr, long msg_len, int *msg_prio,
+ const void *abs_timeout) {
   if (res >= 0) {
-    if (msg_ptr) POST_WRITE(msg_ptr, res);
+    if (msg_ptr)
-    if (msg_prio) POST_WRITE(msg_prio, sizeof(*msg_prio));
+      POST_WRITE(msg_ptr, res);
+    if (msg_prio)
+      POST_WRITE(msg_prio, sizeof(*msg_prio));
   }
 }
 PRE_SYSCALL(mq_notify)(long mqdes, const void *notification) {
-  if (notification) PRE_READ(notification, struct_sigevent_sz);
+  if (notification)
+    PRE_READ(notification, struct_sigevent_sz);
 }
 POST_SYSCALL(mq_notify)(long res, long mqdes, const void *notification) {}
 PRE_SYSCALL(mq_getsetattr)(long mqdes, const void *mqstat, void *omqstat) {
-  if (mqstat) PRE_READ(mqstat, struct_mq_attr_sz);
+  if (mqstat)
+    PRE_READ(mqstat, struct_mq_attr_sz);
 }
-POST_SYSCALL(mq_getsetattr)(long res, long mqdes, const void *mqstat,
+POST_SYSCALL(mq_getsetattr)
-                            void *omqstat) {
+(long res, long mqdes, const void *mqstat, void *omqstat) {
   if (res >= 0) {
-    if (omqstat) POST_WRITE(omqstat, struct_mq_attr_sz);
+    if (omqstat)
+      POST_WRITE(omqstat, struct_mq_attr_sz);
   }
 }
-#endif  // SANITIZER_ANDROID
+#  endif  // SANITIZER_ANDROID
 PRE_SYSCALL(pciconfig_iobase)(long which, long bus, long devfn) {}
 POST_SYSCALL(pciconfig_iobase)(long res, long which, long bus, long devfn) {}
-PRE_SYSCALL(pciconfig_read)(long bus, long dfn, long off, long len, void *buf) {
+PRE_SYSCALL(pciconfig_read)
-}
+(long bus, long dfn, long off, long len, void *buf) {}
-POST_SYSCALL(pciconfig_read)(long res, long bus, long dfn, long off, long len,
+POST_SYSCALL(pciconfig_read)
-                             void *buf) {}
+(long res, long bus, long dfn, long off, long len, void *buf) {}
-PRE_SYSCALL(pciconfig_write)(long bus, long dfn, long off, long len,
+PRE_SYSCALL(pciconfig_write)
-                             void *buf) {}
+(long bus, long dfn, long off, long len, void *buf) {}
-POST_SYSCALL(pciconfig_write)(long res, long bus, long dfn, long off, long len,
+POST_SYSCALL(pciconfig_write)
-                              void *buf) {}
+(long res, long bus, long dfn, long off, long len, void *buf) {}
 PRE_SYSCALL(swapon)(const void *specialfile, long swap_flags) {
   if (specialfile)
 @@ -2247,8 +2463,10 @@ POST_SYSCALL(swapoff)(long res, const void *specialfile) {}
 PRE_SYSCALL(sysctl)(__sanitizer___sysctl_args *args) {
   if (args) {
-    if (args->name) PRE_READ(args->name, args->nlen * sizeof(*args->name));
+    if (args->name)
-    if (args->newval) PRE_READ(args->name, args->newlen);
+      PRE_READ(args->name, args->nlen * sizeof(*args->name));
+    if (args->newval)
+      PRE_READ(args->name, args->newlen);
   }
 }
 @@ -2265,7 +2483,8 @@ PRE_SYSCALL(sysinfo)(void *info) {}
 POST_SYSCALL(sysinfo)(long res, void *info) {
   if (res >= 0) {
-    if (info) POST_WRITE(info, struct_sysinfo_sz);
+    if (info)
+      POST_WRITE(info, struct_sysinfo_sz);
   }
 }
 @@ -2294,10 +2513,10 @@ PRE_SYSCALL(ni_syscall)() {}
 POST_SYSCALL(ni_syscall)(long res) {}
 PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) {
-#if !SANITIZER_ANDROID &&                                                   \
+#  if !SANITIZER_ANDROID &&                                                   \
-    (defined(__i386) || defined(__x86_64) || defined(__mips64) ||           \
+      (defined(__i386) || defined(__x86_64) || defined(__mips64) ||           \
-     defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \
+       defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \
-     SANITIZER_RISCV64)
+       defined(__loongarch__) || SANITIZER_RISCV64)
   if (data) {
     if (request == ptrace_setregs) {
       PRE_READ((void *)data, struct_user_regs_struct_sz);
 @@ -2312,14 +2531,14 @@ PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) {
       PRE_READ(iov->iov_base, iov->iov_len);
     }
   }
-#endif
+#  endif
 }
 POST_SYSCALL(ptrace)(long res, long request, long pid, long addr, long data) {
-#if !SANITIZER_ANDROID &&                                                   \
+#  if !SANITIZER_ANDROID &&                                                   \
-    (defined(__i386) || defined(__x86_64) || defined(__mips64) ||           \
+      (defined(__i386) || defined(__x86_64) || defined(__mips64) ||           \
-     defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \
+       defined(__powerpc64__) || defined(__aarch64__) || defined(__s390__) || \
-     SANITIZER_RISCV64)
+       defined(__loongarch__) || SANITIZER_RISCV64)
   if (res >= 0 && data) {
     // Note that this is different from the interceptor in
     // sanitizer_common_interceptors.inc.
 @@ -2340,11 +2559,12 @@ POST_SYSCALL(ptrace)(long res, long request, long pid, long addr, long data) {
       POST_WRITE((void *)data, sizeof(void *));
     }
   }
-#endif
+#  endif
 }
-PRE_SYSCALL(add_key)(const void *_type, const void *_description,
+PRE_SYSCALL(add_key)
-                     const void *_payload, long plen, long destringid) {
+(const void *_type, const void *_description, const void *_payload, long plen,
+ long destringid) {
   if (_type)
     PRE_READ(_type, __sanitizer::internal_strlen((const char *)_type) + 1);
   if (_description)
 @@ -2352,11 +2572,13 @@ PRE_SYSCALL(add_key)(const void *_type, const void *_description,
              __sanitizer::internal_strlen((const char *)_description) + 1);
 }
-POST_SYSCALL(add_key)(long res, const void *_type, const void *_description,
+POST_SYSCALL(add_key)
-                      const void *_payload, long plen, long destringid) {}
+(long res, const void *_type, const void *_description, const void *_payload,
+ long plen, long destringid) {}
-PRE_SYSCALL(request_key)(const void *_type, const void *_description,
+PRE_SYSCALL(request_key)
-                         const void *_callout_info, long destringid) {
+(const void *_type, const void *_description, const void *_callout_info,
+ long destringid) {
   if (_type)
     PRE_READ(_type, __sanitizer::internal_strlen((const char *)_type) + 1);
   if (_description)
 @@ -2367,13 +2589,14 @@ PRE_SYSCALL(request_key)(const void *_type, const void *_description,
              __sanitizer::internal_strlen((const char *)_callout_info) + 1);
 }
-POST_SYSCALL(request_key)(long res, const void *_type, const void *_description,
+POST_SYSCALL(request_key)
-                          const void *_callout_info, long destringid) {}
+(long res, const void *_type, const void *_description,
+ const void *_callout_info, long destringid) {}
 PRE_SYSCALL(keyctl)(long cmd, long arg2, long arg3, long arg4, long arg5) {}
-POST_SYSCALL(keyctl)(long res, long cmd, long arg2, long arg3, long arg4,
+POST_SYSCALL(keyctl)
-                     long arg5) {}
+(long res, long cmd, long arg2, long arg3, long arg4, long arg5) {}
 PRE_SYSCALL(ioprio_set)(long which, long who, long ioprio) {}
 @@ -2387,50 +2610,62 @@ PRE_SYSCALL(set_mempolicy)(long mode, void *nmask, long maxnode) {}
 POST_SYSCALL(set_mempolicy)(long res, long mode, void *nmask, long maxnode) {
   if (res >= 0) {
-    if (nmask) POST_WRITE(nmask, sizeof(long));
+    if (nmask)
+      POST_WRITE(nmask, sizeof(long));
   }
 }
-PRE_SYSCALL(migrate_pages)(long pid, long maxnode, const void *from,
+PRE_SYSCALL(migrate_pages)
-                           const void *to) {
+(long pid, long maxnode, const void *from, const void *to) {
-  if (from) PRE_READ(from, sizeof(long));
+  if (from)
-  if (to) PRE_READ(to, sizeof(long));
+    PRE_READ(from, sizeof(long));
+  if (to)
+    PRE_READ(to, sizeof(long));
 }
-POST_SYSCALL(migrate_pages)(long res, long pid, long maxnode, const void *from,
+POST_SYSCALL(migrate_pages)
-                            const void *to) {}
+(long res, long pid, long maxnode, const void *from, const void *to) {}
-PRE_SYSCALL(move_pages)(long pid, long nr_pages, const void **pages,
+PRE_SYSCALL(move_pages)
-                        const int *nodes, int *status, long flags) {
+(long pid, long nr_pages, const void **pages, const int *nodes, int *status,
-  if (pages) PRE_READ(pages, nr_pages * sizeof(*pages));
+ long flags) {
-  if (nodes) PRE_READ(nodes, nr_pages * sizeof(*nodes));
+  if (pages)
+    PRE_READ(pages, nr_pages * sizeof(*pages));
+  if (nodes)
+    PRE_READ(nodes, nr_pages * sizeof(*nodes));
 }
-POST_SYSCALL(move_pages)(long res, long pid, long nr_pages, const void **pages,
+POST_SYSCALL(move_pages)
-                         const int *nodes, int *status, long flags) {
+(long res, long pid, long nr_pages, const void **pages, const int *nodes,
+ int *status, long flags) {
   if (res >= 0) {
-    if (status) POST_WRITE(status, nr_pages * sizeof(*status));
+    if (status)
+      POST_WRITE(status, nr_pages * sizeof(*status));
   }
 }
-PRE_SYSCALL(mbind)(long start, long len, long mode, void *nmask, long maxnode,
+PRE_SYSCALL(mbind)
-                   long flags) {}
+(long start, long len, long mode, void *nmask, long maxnode, long flags) {}
-POST_SYSCALL(mbind)(long res, long start, long len, long mode, void *nmask,
+POST_SYSCALL(mbind)
-                    long maxnode, long flags) {
+(long res, long start, long len, long mode, void *nmask, long maxnode,
+ long flags) {
   if (res >= 0) {
-    if (nmask) POST_WRITE(nmask, sizeof(long));
+    if (nmask)
+      POST_WRITE(nmask, sizeof(long));
   }
 }
-PRE_SYSCALL(get_mempolicy)(void *policy, void *nmask, long maxnode, long addr,
+PRE_SYSCALL(get_mempolicy)
-                           long flags) {}
+(void *policy, void *nmask, long maxnode, long addr, long flags) {}
-POST_SYSCALL(get_mempolicy)(long res, void *policy, void *nmask, long maxnode,
+POST_SYSCALL(get_mempolicy)
-                            long addr, long flags) {
+(long res, void *policy, void *nmask, long maxnode, long addr, long flags) {
   if (res >= 0) {
-    if (policy) POST_WRITE(policy, sizeof(int));
+    if (policy)
-    if (nmask) POST_WRITE(nmask, sizeof(long));
+      POST_WRITE(policy, sizeof(int));
+    if (nmask)
+      POST_WRITE(nmask, sizeof(long));
   }
 }
 @@ -2447,8 +2682,8 @@ PRE_SYSCALL(inotify_add_watch)(long fd, const void *path, long mask) {
     PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
 }
-POST_SYSCALL(inotify_add_watch)(long res, long fd, const void *path,
+POST_SYSCALL(inotify_add_watch)
-                                long mask) {}
+(long res, long fd, const void *path, long mask) {}
 PRE_SYSCALL(inotify_rm_watch)(long fd, long wd) {}
 @@ -2458,8 +2693,10 @@ PRE_SYSCALL(spu_run)(long fd, void *unpc, void *ustatus) {}
 POST_SYSCALL(spu_run)(long res, long fd, unsigned *unpc, unsigned *ustatus) {
   if (res >= 0) {
-    if (unpc) POST_WRITE(unpc, sizeof(*unpc));
+    if (unpc)
-    if (ustatus) POST_WRITE(ustatus, sizeof(*ustatus));
+      POST_WRITE(unpc, sizeof(*unpc));
+    if (ustatus)
+      POST_WRITE(ustatus, sizeof(*ustatus));
   }
 }
 @@ -2468,8 +2705,8 @@ PRE_SYSCALL(spu_create)(const void *name, long flags, long mode, long fd) {
     PRE_READ(name, __sanitizer::internal_strlen((const char *)name) + 1);
 }
-POST_SYSCALL(spu_create)(long res, const void *name, long flags, long mode,
+POST_SYSCALL(spu_create)
-                         long fd) {}
+(long res, const void *name, long flags, long mode, long fd) {}
 PRE_SYSCALL(mknodat)(long dfd, const void *filename, long mode, long dev) {
   if (filename)
 @@ -2477,8 +2714,8 @@ PRE_SYSCALL(mknodat)(long dfd, const void *filename, long mode, long dev) {
              __sanitizer::internal_strlen((const char *)filename) + 1);
 }
-POST_SYSCALL(mknodat)(long res, long dfd, const void *filename, long mode,
+POST_SYSCALL(mknodat)
-                      long dev) {}
+(long res, long dfd, const void *filename, long mode, long dev) {}
 PRE_SYSCALL(mkdirat)(long dfd, const void *pathname, long mode) {
   if (pathname)
 @@ -2503,30 +2740,33 @@ PRE_SYSCALL(symlinkat)(const void *oldname, long newdfd, const void *newname) {
     PRE_READ(newname, __sanitizer::internal_strlen((const char *)newname) + 1);
 }
-POST_SYSCALL(symlinkat)(long res, const void *oldname, long newdfd,
+POST_SYSCALL(symlinkat)
-                        const void *newname) {}
+(long res, const void *oldname, long newdfd, const void *newname) {}
-PRE_SYSCALL(linkat)(long olddfd, const void *oldname, long newdfd,
+PRE_SYSCALL(linkat)
-                    const void *newname, long flags) {
+(long olddfd, const void *oldname, long newdfd, const void *newname,
+ long flags) {
   if (oldname)
     PRE_READ(oldname, __sanitizer::internal_strlen((const char *)oldname) + 1);
   if (newname)
     PRE_READ(newname, __sanitizer::internal_strlen((const char *)newname) + 1);
 }
-POST_SYSCALL(linkat)(long res, long olddfd, const void *oldname, long newdfd,
+POST_SYSCALL(linkat)
-                     const void *newname, long flags) {}
+(long res, long olddfd, const void *oldname, long newdfd, const void *newname,
+ long flags) {}
-PRE_SYSCALL(renameat)(long olddfd, const void *oldname, long newdfd,
+PRE_SYSCALL(renameat)
-                      const void *newname) {
+(long olddfd, const void *oldname, long newdfd, const void *newname) {
   if (oldname)
     PRE_READ(oldname, __sanitizer::internal_strlen((const char *)oldname) + 1);
   if (newname)
     PRE_READ(newname, __sanitizer::internal_strlen((const char *)newname) + 1);
 }
-POST_SYSCALL(renameat)(long res, long olddfd, const void *oldname, long newdfd,
+POST_SYSCALL(renameat)
-                       const void *newname) {}
+(long res, long olddfd, const void *oldname, long newdfd, const void *newname) {
+}
 PRE_SYSCALL(futimesat)(long dfd, const void *filename, void *utimes) {
   if (filename)
 @@ -2534,10 +2774,11 @@ PRE_SYSCALL(futimesat)(long dfd, const void *filename, void *utimes) {
              __sanitizer::internal_strlen((const char *)filename) + 1);
 }
-POST_SYSCALL(futimesat)(long res, long dfd, const void *filename,
+POST_SYSCALL(futimesat)
-                        void *utimes) {
+(long res, long dfd, const void *filename, void *utimes) {
   if (res >= 0) {
-    if (utimes) POST_WRITE(utimes, timeval_sz);
+    if (utimes)
+      POST_WRITE(utimes, timeval_sz);
   }
 }
 @@ -2557,15 +2798,15 @@ PRE_SYSCALL(fchmodat)(long dfd, const void *filename, long mode) {
 POST_SYSCALL(fchmodat)(long res, long dfd, const void *filename, long mode) {}
-PRE_SYSCALL(fchownat)(long dfd, const void *filename, long user, long group,
+PRE_SYSCALL(fchownat)
-                      long flag) {
+(long dfd, const void *filename, long user, long group, long flag) {
   if (filename)
     PRE_READ(filename,
              __sanitizer::internal_strlen((const char *)filename) + 1);
 }
-POST_SYSCALL(fchownat)(long res, long dfd, const void *filename, long user,
+POST_SYSCALL(fchownat)
-                       long group, long flag) {}
+(long res, long dfd, const void *filename, long user, long group, long flag) {}
 PRE_SYSCALL(openat)(long dfd, const void *filename, long flags, long mode) {
   if (filename)
 @@ -2573,34 +2814,36 @@ PRE_SYSCALL(openat)(long dfd, const void *filename, long flags, long mode) {
              __sanitizer::internal_strlen((const char *)filename) + 1);
 }
-POST_SYSCALL(openat)(long res, long dfd, const void *filename, long flags,
+POST_SYSCALL(openat)
-                     long mode) {}
+(long res, long dfd, const void *filename, long flags, long mode) {}
-PRE_SYSCALL(newfstatat)(long dfd, const void *filename, void *statbuf,
+PRE_SYSCALL(newfstatat)
-                        long flag) {
+(long dfd, const void *filename, void *statbuf, long flag) {
   if (filename)
     PRE_READ(filename,
              __sanitizer::internal_strlen((const char *)filename) + 1);
 }
-POST_SYSCALL(newfstatat)(long res, long dfd, const void *filename,
+POST_SYSCALL(newfstatat)
-                         void *statbuf, long flag) {
+(long res, long dfd, const void *filename, void *statbuf, long flag) {
   if (res >= 0) {
-    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat_sz);
+    if (statbuf)
+      POST_WRITE(statbuf, struct_kernel_stat_sz);
   }
 }
-PRE_SYSCALL(fstatat64)(long dfd, const void *filename, void *statbuf,
+PRE_SYSCALL(fstatat64)
-                       long flag) {
+(long dfd, const void *filename, void *statbuf, long flag) {
   if (filename)
     PRE_READ(filename,
              __sanitizer::internal_strlen((const char *)filename) + 1);
 }
-POST_SYSCALL(fstatat64)(long res, long dfd, const void *filename, void *statbuf,
+POST_SYSCALL(fstatat64)
-                        long flag) {
+(long res, long dfd, const void *filename, void *statbuf, long flag) {
   if (res >= 0) {
-    if (statbuf) POST_WRITE(statbuf, struct_kernel_stat64_sz);
+    if (statbuf)
+      POST_WRITE(statbuf, struct_kernel_stat64_sz);
   }
 }
 @@ -2609,25 +2852,26 @@ PRE_SYSCALL(readlinkat)(long dfd, const void *path, void *buf, long bufsiz) {
     PRE_READ(path, __sanitizer::internal_strlen((const char *)path) + 1);
 }
-POST_SYSCALL(readlinkat)(long res, long dfd, const void *path, void *buf,
+POST_SYSCALL(readlinkat)
-                         long bufsiz) {
+(long res, long dfd, const void *path, void *buf, long bufsiz) {
   if (res >= 0) {
     if (buf)
       POST_WRITE(buf, __sanitizer::internal_strlen((const char *)buf) + 1);
   }
 }
-PRE_SYSCALL(utimensat)(long dfd, const void *filename, void *utimes,
+PRE_SYSCALL(utimensat)
-                       long flags) {
+(long dfd, const void *filename, void *utimes, long flags) {
   if (filename)
     PRE_READ(filename,
              __sanitizer::internal_strlen((const char *)filename) + 1);
 }
-POST_SYSCALL(utimensat)(long res, long dfd, const void *filename, void *utimes,
+POST_SYSCALL(utimensat)
-                        long flags) {
+(long res, long dfd, const void *filename, void *utimes, long flags) {
   if (res >= 0) {
-    if (utimes) POST_WRITE(utimes, struct_timespec_sz);
+    if (utimes)
+      POST_WRITE(utimes, struct_timespec_sz);
   }
 }
 @@ -2635,24 +2879,28 @@ PRE_SYSCALL(unshare)(long unshare_flags) {}
 POST_SYSCALL(unshare)(long res, long unshare_flags) {}
-PRE_SYSCALL(splice)(long fd_in, void *off_in, long fd_out, void *off_out,
+PRE_SYSCALL(splice)
-                    long len, long flags) {}
+(long fd_in, void *off_in, long fd_out, void *off_out, long len, long flags) {}
-POST_SYSCALL(splice)(long res, long fd_in, void *off_in, long fd_out,
+POST_SYSCALL(splice)
-                     void *off_out, long len, long flags) {
+(long res, long fd_in, void *off_in, long fd_out, void *off_out, long len,
+ long flags) {
   if (res >= 0) {
-    if (off_in) POST_WRITE(off_in, sizeof(long long));
+    if (off_in)
-    if (off_out) POST_WRITE(off_out, sizeof(long long));
+      POST_WRITE(off_in, sizeof(long long));
+    if (off_out)
+      POST_WRITE(off_out, sizeof(long long));
   }
 }
-PRE_SYSCALL(vmsplice)(long fd, const __sanitizer_iovec *iov, long nr_segs,
+PRE_SYSCALL(vmsplice)
-                      long flags) {}
+(long fd, const __sanitizer_iovec *iov, long nr_segs, long flags) {}
-POST_SYSCALL(vmsplice)(long res, long fd, const __sanitizer_iovec *iov,
+POST_SYSCALL(vmsplice)
-                       long nr_segs, long flags) {
+(long res, long fd, const __sanitizer_iovec *iov, long nr_segs, long flags) {
   if (res >= 0) {
-    if (iov) kernel_read_iovec(iov, nr_segs, res);
+    if (iov)
+      kernel_read_iovec(iov, nr_segs, res);
   }
 }
 @@ -2662,8 +2910,8 @@ POST_SYSCALL(tee)(long res, long fdin, long fdout, long len, long flags) {}
 PRE_SYSCALL(get_robust_list)(long pid, void *head_ptr, void *len_ptr) {}
-POST_SYSCALL(get_robust_list)(long res, long pid, void *head_ptr,
+POST_SYSCALL(get_robust_list)
-                              void *len_ptr) {}
+(long res, long pid, void *head_ptr, void *len_ptr) {}
 PRE_SYSCALL(set_robust_list)(void *head, long len) {}
 @@ -2673,27 +2921,31 @@ PRE_SYSCALL(getcpu)(void *cpu, void *node, void *cache) {}
 POST_SYSCALL(getcpu)(long res, void *cpu, void *node, void *cache) {
   if (res >= 0) {
-    if (cpu) POST_WRITE(cpu, sizeof(unsigned));
+    if (cpu)
-    if (node) POST_WRITE(node, sizeof(unsigned));
+      POST_WRITE(cpu, sizeof(unsigned));
+    if (node)
+      POST_WRITE(node, sizeof(unsigned));
     // The third argument to this system call is nowadays unused.
   }
 }
 PRE_SYSCALL(signalfd)(long ufd, void *user_mask, long sizemask) {}
-POST_SYSCALL(signalfd)(long res, long ufd, kernel_sigset_t *user_mask,
+POST_SYSCALL(signalfd)
-                       long sizemask) {
+(long res, long ufd, kernel_sigset_t *user_mask, long sizemask) {
   if (res >= 0) {
-    if (user_mask) POST_WRITE(user_mask, sizemask);
+    if (user_mask)
+      POST_WRITE(user_mask, sizemask);
   }
 }
 PRE_SYSCALL(signalfd4)(long ufd, void *user_mask, long sizemask, long flags) {}
-POST_SYSCALL(signalfd4)(long res, long ufd, kernel_sigset_t *user_mask,
+POST_SYSCALL(signalfd4)
-                        long sizemask, long flags) {
+(long res, long ufd, kernel_sigset_t *user_mask, long sizemask, long flags) {
   if (res >= 0) {
-    if (user_mask) POST_WRITE(user_mask, sizemask);
+    if (user_mask)
+      POST_WRITE(user_mask, sizemask);
   }
 }
 @@ -2701,15 +2953,17 @@ PRE_SYSCALL(timerfd_create)(long clockid, long flags) {}
 POST_SYSCALL(timerfd_create)(long res, long clockid, long flags) {}
-PRE_SYSCALL(timerfd_settime)(long ufd, long flags, const void *utmr,
+PRE_SYSCALL(timerfd_settime)
-                             void *otmr) {
+(long ufd, long flags, const void *utmr, void *otmr) {
-  if (utmr) PRE_READ(utmr, struct_itimerspec_sz);
+  if (utmr)
+    PRE_READ(utmr, struct_itimerspec_sz);
 }
-POST_SYSCALL(timerfd_settime)(long res, long ufd, long flags, const void *utmr,
+POST_SYSCALL(timerfd_settime)
-                              void *otmr) {
+(long res, long ufd, long flags, const void *utmr, void *otmr) {
   if (res >= 0) {
-    if (otmr) POST_WRITE(otmr, struct_itimerspec_sz);
+    if (otmr)
+      POST_WRITE(otmr, struct_itimerspec_sz);
   }
 }
 @@ -2717,7 +2971,8 @@ PRE_SYSCALL(timerfd_gettime)(long ufd, void *otmr) {}
 POST_SYSCALL(timerfd_gettime)(long res, long ufd, void *otmr) {
   if (res >= 0) {
-    if (otmr) POST_WRITE(otmr, struct_itimerspec_sz);
+    if (otmr)
+      POST_WRITE(otmr, struct_itimerspec_sz);
   }
 }
 @@ -2735,33 +2990,42 @@ POST_SYSCALL(old_readdir)(long res, long arg0, void *arg1, long arg2) {
   // Missing definition of 'struct old_linux_dirent'.
 }
-PRE_SYSCALL(pselect6)(long arg0, __sanitizer___kernel_fd_set *arg1,
+PRE_SYSCALL(pselect6)
-                      __sanitizer___kernel_fd_set *arg2,
+(long arg0, __sanitizer___kernel_fd_set *arg1,
-                      __sanitizer___kernel_fd_set *arg3, void *arg4,
+ __sanitizer___kernel_fd_set *arg2, __sanitizer___kernel_fd_set *arg3,
-                      void *arg5) {}
+ void *arg4, void *arg5) {}
-POST_SYSCALL(pselect6)(long res, long arg0, __sanitizer___kernel_fd_set *arg1,
+POST_SYSCALL(pselect6)
-                       __sanitizer___kernel_fd_set *arg2,
+(long res, long arg0, __sanitizer___kernel_fd_set *arg1,
-                       __sanitizer___kernel_fd_set *arg3, void *arg4,
+ __sanitizer___kernel_fd_set *arg2, __sanitizer___kernel_fd_set *arg3,
-                       void *arg5) {
+ void *arg4, void *arg5) {
   if (res >= 0) {
-    if (arg1) POST_WRITE(arg1, sizeof(*arg1));
+    if (arg1)
-    if (arg2) POST_WRITE(arg2, sizeof(*arg2));
+      POST_WRITE(arg1, sizeof(*arg1));
-    if (arg3) POST_WRITE(arg3, sizeof(*arg3));
+    if (arg2)
-    if (arg4) POST_WRITE(arg4, struct_timespec_sz);
+      POST_WRITE(arg2, sizeof(*arg2));
+    if (arg3)
+      POST_WRITE(arg3, sizeof(*arg3));
+    if (arg4)
+      POST_WRITE(arg4, struct_timespec_sz);
   }
 }
-PRE_SYSCALL(ppoll)(__sanitizer_pollfd *arg0, long arg1, void *arg2,
+PRE_SYSCALL(ppoll)
-                   const kernel_sigset_t *arg3, long arg4) {
+(__sanitizer_pollfd *arg0, long arg1, void *arg2, const kernel_sigset_t *arg3,
-  if (arg3) PRE_READ(arg3, arg4);
+ long arg4) {
+  if (arg3)
+    PRE_READ(arg3, arg4);
 }
-POST_SYSCALL(ppoll)(long res, __sanitizer_pollfd *arg0, long arg1, void *arg2,
+POST_SYSCALL(ppoll)
-                    const void *arg3, long arg4) {
+(long res, __sanitizer_pollfd *arg0, long arg1, void *arg2, const void *arg3,
+ long arg4) {
   if (res >= 0) {
-    if (arg0) POST_WRITE(arg0, sizeof(*arg0));
+    if (arg0)
-    if (arg2) POST_WRITE(arg2, struct_timespec_sz);
+      POST_WRITE(arg0, sizeof(*arg0));
+    if (arg2)
+      POST_WRITE(arg2, struct_timespec_sz);
   }
 }
 @@ -2769,81 +3033,79 @@ PRE_SYSCALL(syncfs)(long fd) {}
 POST_SYSCALL(syncfs)(long res, long fd) {}
-PRE_SYSCALL(perf_event_open)(__sanitizer_perf_event_attr *attr_uptr, long pid,
+PRE_SYSCALL(perf_event_open)
-                             long cpu, long group_fd, long flags) {
+(__sanitizer_perf_event_attr *attr_uptr, long pid, long cpu, long group_fd,
-  if (attr_uptr) PRE_READ(attr_uptr, attr_uptr->size);
+ long flags) {
+  if (attr_uptr)
+    PRE_READ(attr_uptr, attr_uptr->size);
 }
-POST_SYSCALL(perf_event_open)(long res, __sanitizer_perf_event_attr *attr_uptr,
+POST_SYSCALL(perf_event_open)
-                              long pid, long cpu, long group_fd, long flags) {}
+(long res, __sanitizer_perf_event_attr *attr_uptr, long pid, long cpu,
+ long group_fd, long flags) {}
-PRE_SYSCALL(mmap_pgoff)(long addr, long len, long prot, long flags, long fd,
+PRE_SYSCALL(mmap_pgoff)
-                        long pgoff) {}
+(long addr, long len, long prot, long flags, long fd, long pgoff) {}
-POST_SYSCALL(mmap_pgoff)(long res, long addr, long len, long prot, long flags,
+POST_SYSCALL(mmap_pgoff)
-                         long fd, long pgoff) {}
+(long res, long addr, long len, long prot, long flags, long fd, long pgoff) {}
 PRE_SYSCALL(old_mmap)(void *arg) {}
 POST_SYSCALL(old_mmap)(long res, void *arg) {}
-PRE_SYSCALL(name_to_handle_at)(long dfd, const void *name, void *handle,
+PRE_SYSCALL(name_to_handle_at)
-                               void *mnt_id, long flag) {}
+(long dfd, const void *name, void *handle, void *mnt_id, long flag) {}
-POST_SYSCALL(name_to_handle_at)(long res, long dfd, const void *name,
+POST_SYSCALL(name_to_handle_at)
-                                void *handle, void *mnt_id, long flag) {}
+(long res, long dfd, const void *name, void *handle, void *mnt_id, long flag) {}
 PRE_SYSCALL(open_by_handle_at)(long mountdirfd, void *handle, long flags) {}
-POST_SYSCALL(open_by_handle_at)(long res, long mountdirfd, void *handle,
+POST_SYSCALL(open_by_handle_at)
-                                long flags) {}
+(long res, long mountdirfd, void *handle, long flags) {}
 PRE_SYSCALL(setns)(long fd, long nstype) {}
 POST_SYSCALL(setns)(long res, long fd, long nstype) {}
-PRE_SYSCALL(process_vm_readv)(long pid, const __sanitizer_iovec *lvec,
+PRE_SYSCALL(process_vm_readv)
-                              long liovcnt, const void *rvec, long riovcnt,
+(long pid, const __sanitizer_iovec *lvec, long liovcnt, const void *rvec,
-                              long flags) {}
+ long riovcnt, long flags) {}
-POST_SYSCALL(process_vm_readv)(long res, long pid,
+POST_SYSCALL(process_vm_readv)
-                               const __sanitizer_iovec *lvec, long liovcnt,
+(long res, long pid, const __sanitizer_iovec *lvec, long liovcnt,
-                               const void *rvec, long riovcnt, long flags) {
+ const void *rvec, long riovcnt, long flags) {
   if (res >= 0) {
-    if (lvec) kernel_write_iovec(lvec, liovcnt, res);
+    if (lvec)
+      kernel_write_iovec(lvec, liovcnt, res);
   }
 }
-PRE_SYSCALL(process_vm_writev)(long pid, const __sanitizer_iovec *lvec,
+PRE_SYSCALL(process_vm_writev)
-                               long liovcnt, const void *rvec, long riovcnt,
+(long pid, const __sanitizer_iovec *lvec, long liovcnt, const void *rvec,
-                               long flags) {}
+ long riovcnt, long flags) {}
-POST_SYSCALL(process_vm_writev)(long res, long pid,
+POST_SYSCALL(process_vm_writev)
-                                const __sanitizer_iovec *lvec, long liovcnt,
+(long res, long pid, const __sanitizer_iovec *lvec, long liovcnt,
-                                const void *rvec, long riovcnt, long flags) {
+ const void *rvec, long riovcnt, long flags) {
   if (res >= 0) {
-    if (lvec) kernel_read_iovec(lvec, liovcnt, res);
+    if (lvec)
+      kernel_read_iovec(lvec, liovcnt, res);
   }
 }
-PRE_SYSCALL(fork)() {
+PRE_SYSCALL(fork)() { COMMON_SYSCALL_PRE_FORK(); }
-  COMMON_SYSCALL_PRE_FORK();
-}
-POST_SYSCALL(fork)(long res) {
+POST_SYSCALL(fork)(long res) { COMMON_SYSCALL_POST_FORK(res); }
-  COMMON_SYSCALL_POST_FORK(res);
-}
-PRE_SYSCALL(vfork)() {
+PRE_SYSCALL(vfork)() { COMMON_SYSCALL_PRE_FORK(); }
-  COMMON_SYSCALL_PRE_FORK();
-}
-POST_SYSCALL(vfork)(long res) {
+POST_SYSCALL(vfork)(long res) { COMMON_SYSCALL_POST_FORK(res); }
-  COMMON_SYSCALL_POST_FORK(res);
-}
-PRE_SYSCALL(sigaction)(long signum, const __sanitizer_kernel_sigaction_t *act,
+PRE_SYSCALL(sigaction)
-                       __sanitizer_kernel_sigaction_t *oldact) {
+(long signum, const __sanitizer_kernel_sigaction_t *act,
+ __sanitizer_kernel_sigaction_t *oldact) {
   if (act) {
     PRE_READ(&act->sigaction, sizeof(act->sigaction));
     PRE_READ(&act->sa_flags, sizeof(act->sa_flags));
 @@ -2851,15 +3113,16 @@ PRE_SYSCALL(sigaction)(long signum, const __sanitizer_kernel_sigaction_t *act,
   }
 }
-POST_SYSCALL(sigaction)(long res, long signum,
+POST_SYSCALL(sigaction)
-                        const __sanitizer_kernel_sigaction_t *act,
+(long res, long signum, const __sanitizer_kernel_sigaction_t *act,
-                        __sanitizer_kernel_sigaction_t *oldact) {
+ __sanitizer_kernel_sigaction_t *oldact) {
-  if (res >= 0 && oldact) POST_WRITE(oldact, sizeof(*oldact));
+  if (res >= 0 && oldact)
+    POST_WRITE(oldact, sizeof(*oldact));
 }
-PRE_SYSCALL(rt_sigaction)(long signum,
+PRE_SYSCALL(rt_sigaction)
-                          const __sanitizer_kernel_sigaction_t *act,
+(long signum, const __sanitizer_kernel_sigaction_t *act,
-                          __sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) {
+ __sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) {
   if (act) {
     PRE_READ(&act->sigaction, sizeof(act->sigaction));
     PRE_READ(&act->sa_flags, sizeof(act->sa_flags));
 @@ -2867,9 +3130,9 @@ PRE_SYSCALL(rt_sigaction)(long signum,
   }
 }
-POST_SYSCALL(rt_sigaction)(long res, long signum,
+POST_SYSCALL(rt_sigaction)
-                           const __sanitizer_kernel_sigaction_t *act,
+(long res, long signum, const __sanitizer_kernel_sigaction_t *act,
-                           __sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) {
+ __sanitizer_kernel_sigaction_t *oldact, SIZE_T sz) {
   if (res >= 0 && oldact) {
     SIZE_T oldact_sz = ((char *)&oldact->sa_mask) - ((char *)oldact) + sz;
     POST_WRITE(oldact, oldact_sz);
 @@ -2906,11 +3169,11 @@ POST_SYSCALL(sigaltstack)(long res, void *ss, void *oss) {
 }
 }  // extern "C"
-#undef PRE_SYSCALL
+#  undef PRE_SYSCALL
-#undef PRE_READ
+#  undef PRE_READ
-#undef PRE_WRITE
+#  undef PRE_WRITE
-#undef POST_SYSCALL
+#  undef POST_SYSCALL
-#undef POST_READ
+#  undef POST_READ
-#undef POST_WRITE
+#  undef POST_WRITE
 #endif  // SANITIZER_LINUX

lib/tsan/sanitizer_common/sanitizer_coverage_interface.inc+10

...	@@ -27,6 +27,16 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_gep)	...	@@ -27,6 +27,16 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_gep)
27	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard)	27	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard)
28	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard_init)	28	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard_init)
29	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_indir)	29	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_indir)
		30	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load1)
		31	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load2)
		32	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load4)
		33	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load8)
		34	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load16)
		35	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store1)
		36	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store2)
		37	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store4)
		38	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store8)
		39	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store16)
30	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_switch)	40	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_switch)
31	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_8bit_counters_init)	41	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_8bit_counters_init)
32	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_bool_flag_init)	42	INTERFACE_WEAK_FUNCTION(__sanitizer_cov_bool_flag_init)

lib/tsan/sanitizer_common/sanitizer_coverage_win_dll_thunk.cpp created+20

...	@@ -0,0 +1,20 @@
1	//===-- sanitizer_coverage_win_dll_thunk.cpp ------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file defines a family of thunks that should be statically linked into
10	// the DLLs that have instrumentation in order to delegate the calls to the
11	// shared runtime that lives in the main binary.
12	// See https://github.com/google/sanitizers/issues/209 for the details.
13	//===----------------------------------------------------------------------===//
14	#ifdef SANITIZER_DLL_THUNK
15	#include "sanitizer_win_dll_thunk.h"
16	// Sanitizer Coverage interface functions.
17	#define INTERFACE_FUNCTION(Name) INTERCEPT_SANITIZER_FUNCTION(Name)
18	#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
19	#include "sanitizer_coverage_interface.inc"
20	#endif // SANITIZER_DLL_THUNK

lib/tsan/sanitizer_common/sanitizer_coverage_win_dynamic_runtime_thunk.cpp created+26

...	@@ -0,0 +1,26 @@
1	//===-- sanitizer_coverage_win_dynamic_runtime_thunk.cpp ------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file defines things that need to be present in the application modules
10	// to interact with Sanitizer Coverage, when it is included in a dll.
11	//
12	//===----------------------------------------------------------------------===//
13	#ifdef SANITIZER_DYNAMIC_RUNTIME_THUNK
14	#define SANITIZER_IMPORT_INTERFACE 1
15	#include "sanitizer_win_defs.h"
16	// Define weak alias for all weak functions imported from sanitizer coverage.
17	#define INTERFACE_FUNCTION(Name)
18	#define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name)
19	#include "sanitizer_coverage_interface.inc"
20	#endif // SANITIZER_DYNAMIC_RUNTIME_THUNK
21
22	namespace __sanitizer {
23	// Add one, otherwise unused, external symbol to this object file so that the
24	// Visual C++ linker includes it and reads the .drective section.
25	void ForceWholeArchiveIncludeForSanCov() {}
26	}

lib/tsan/sanitizer_common/sanitizer_coverage_win_weak_interception.cpp created+23

...	@@ -0,0 +1,23 @@
1	//===-- sanitizer_coverage_win_weak_interception.cpp ----------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	// This module should be included in Sanitizer Coverage when it implemented as a
9	// shared library on Windows (dll), in order to delegate the calls of weak
10	// functions to the implementation in the main executable when a strong
11	// definition is provided.
12	//===----------------------------------------------------------------------===//
13	#ifdef SANITIZER_DYNAMIC
14	#include "sanitizer_win_weak_interception.h"
15	#include "sanitizer_interface_internal.h"
16	#include "sancov_flags.h"
17	// Check if strong definitions for weak functions are present in the main
18	// executable. If that is the case, override dll functions to point to strong
19	// implementations.
20	#define INTERFACE_FUNCTION(Name)
21	#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
22	#include "sanitizer_coverage_interface.inc"
23	#endif // SANITIZER_DYNAMIC

lib/tsan/sanitizer_common/sanitizer_deadlock_detector.h+1-1

...	@@ -293,7 +293,7 @@ class DeadlockDetector {	...	@@ -293,7 +293,7 @@ class DeadlockDetector {
293	}	293	}
294		294
295	// Returns true iff dtls is empty (no locks are currently held) and we can	295	// Returns true iff dtls is empty (no locks are currently held) and we can
296	// add the node to the currently held locks w/o chanding the global state.	296	// add the node to the currently held locks w/o changing the global state.
297	// This operation is thread-safe as it only touches the dtls.	297	// This operation is thread-safe as it only touches the dtls.
298	bool onFirstLock(DeadlockDetectorTLS<BV> *dtls, uptr node, u32 stk = 0) {	298	bool onFirstLock(DeadlockDetectorTLS<BV> *dtls, uptr node, u32 stk = 0) {
299	if (!dtls->empty()) return false;	299	if (!dtls->empty()) return false;

lib/tsan/sanitizer_common/sanitizer_dense_map.h created+705

...	@@ -0,0 +1,705 @@
1	//===- sanitizer_dense_map.h - Dense probed hash table ----------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This is fork of llvm/ADT/DenseMap.h class with the following changes:
10	// * Use mmap to allocate.
11	// * No iterators.
12	// * Does not shrink.
13	//
14	//===----------------------------------------------------------------------===//
15
16	#ifndef SANITIZER_DENSE_MAP_H
17	#define SANITIZER_DENSE_MAP_H
18
19	#include "sanitizer_common.h"
20	#include "sanitizer_dense_map_info.h"
21	#include "sanitizer_internal_defs.h"
22	#include "sanitizer_type_traits.h"
23
24	namespace __sanitizer {
25
26	template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT,
27	typename BucketT>
28	class DenseMapBase {
29	public:
30	using size_type = unsigned;
31	using key_type = KeyT;
32	using mapped_type = ValueT;
33	using value_type = BucketT;
34
35	WARN_UNUSED_RESULT bool empty() const { return getNumEntries() == 0; }
36	unsigned size() const { return getNumEntries(); }
37
38	/// Grow the densemap so that it can contain at least \p NumEntries items
39	/// before resizing again.
40	void reserve(size_type NumEntries) {
41	auto NumBuckets = getMinBucketToReserveForEntries(NumEntries);
42	if (NumBuckets > getNumBuckets())
43	grow(NumBuckets);
44	}
45
46	void clear() {
47	if (getNumEntries() == 0 && getNumTombstones() == 0)
48	return;
49
50	const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey();
51	if (__sanitizer::is_trivially_destructible<ValueT>::value) {
52	// Use a simpler loop when values don't need destruction.
53	for (BucketT P = getBuckets(), E = getBucketsEnd(); P != E; ++P)
54	P->getFirst() = EmptyKey;
55	} else {
56	unsigned NumEntries = getNumEntries();
57	for (BucketT P = getBuckets(), E = getBucketsEnd(); P != E; ++P) {
58	if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey)) {
59	if (!KeyInfoT::isEqual(P->getFirst(), TombstoneKey)) {
60	P->getSecond().~ValueT();
61	--NumEntries;
62	}
63	P->getFirst() = EmptyKey;
64	}
65	}
66	CHECK_EQ(NumEntries, 0);
67	}
68	setNumEntries(0);
69	setNumTombstones(0);
70	}
71
72	/// Return 1 if the specified key is in the map, 0 otherwise.
73	size_type count(const KeyT &Key) const {
74	const BucketT *TheBucket;
75	return LookupBucketFor(Key, TheBucket) ? 1 : 0;
76	}
77
78	value_type *find(const KeyT &Key) {
79	BucketT *TheBucket;
80	if (LookupBucketFor(Key, TheBucket))
81	return TheBucket;
82	return nullptr;
83	}
84	const value_type *find(const KeyT &Key) const {
85	const BucketT *TheBucket;
86	if (LookupBucketFor(Key, TheBucket))
87	return TheBucket;
88	return nullptr;
89	}
90
91	/// Alternate version of find() which allows a different, and possibly
92	/// less expensive, key type.
93	/// The DenseMapInfo is responsible for supplying methods
94	/// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key
95	/// type used.
96	template <class LookupKeyT>
97	value_type *find_as(const LookupKeyT &Key) {
98	BucketT *TheBucket;
99	if (LookupBucketFor(Key, TheBucket))
100	return TheBucket;
101	return nullptr;
102	}
103	template <class LookupKeyT>
104	const value_type *find_as(const LookupKeyT &Key) const {
105	const BucketT *TheBucket;
106	if (LookupBucketFor(Key, TheBucket))
107	return TheBucket;
108	return nullptr;
109	}
110
111	/// lookup - Return the entry for the specified key, or a default
112	/// constructed value if no such entry exists.
113	ValueT lookup(const KeyT &Key) const {
114	const BucketT *TheBucket;
115	if (LookupBucketFor(Key, TheBucket))
116	return TheBucket->getSecond();
117	return ValueT();
118	}
119
120	// Inserts key,value pair into the map if the key isn't already in the map.
121	// If the key is already in the map, it returns false and doesn't update the
122	// value.
123	detail::DenseMapPair<value_type *, bool> insert(const value_type &KV) {
124	return try_emplace(KV.first, KV.second);
125	}
126
127	// Inserts key,value pair into the map if the key isn't already in the map.
128	// If the key is already in the map, it returns false and doesn't update the
129	// value.
130	detail::DenseMapPair<value_type *, bool> insert(value_type &&KV) {
131	return try_emplace(__sanitizer::move(KV.first),
132	__sanitizer::move(KV.second));
133	}
134
135	// Inserts key,value pair into the map if the key isn't already in the map.
136	// The value is constructed in-place if the key is not in the map, otherwise
137	// it is not moved.
138	template <typename... Ts>
139	detail::DenseMapPair<value_type *, bool> try_emplace(KeyT &&Key,
140	Ts &&...Args) {
141	BucketT *TheBucket;
142	if (LookupBucketFor(Key, TheBucket))
143	return {TheBucket, false}; // Already in map.
144
145	// Otherwise, insert the new element.
146	TheBucket = InsertIntoBucket(TheBucket, __sanitizer::move(Key),
147	__sanitizer::forward<Ts>(Args)...);
148	return {TheBucket, true};
149	}
150
151	// Inserts key,value pair into the map if the key isn't already in the map.
152	// The value is constructed in-place if the key is not in the map, otherwise
153	// it is not moved.
154	template <typename... Ts>
155	detail::DenseMapPair<value_type *, bool> try_emplace(const KeyT &Key,
156	Ts &&...Args) {
157	BucketT *TheBucket;
158	if (LookupBucketFor(Key, TheBucket))
159	return {TheBucket, false}; // Already in map.
160
161	// Otherwise, insert the new element.
162	TheBucket =
163	InsertIntoBucket(TheBucket, Key, __sanitizer::forward<Ts>(Args)...);
164	return {TheBucket, true};
165	}
166
167	/// Alternate version of insert() which allows a different, and possibly
168	/// less expensive, key type.
169	/// The DenseMapInfo is responsible for supplying methods
170	/// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key
171	/// type used.
172	template <typename LookupKeyT>
173	detail::DenseMapPair<value_type *, bool> insert_as(value_type &&KV,
174	const LookupKeyT &Val) {
175	BucketT *TheBucket;
176	if (LookupBucketFor(Val, TheBucket))
177	return {TheBucket, false}; // Already in map.
178
179	// Otherwise, insert the new element.
180	TheBucket =
181	InsertIntoBucketWithLookup(TheBucket, __sanitizer::move(KV.first),
182	__sanitizer::move(KV.second), Val);
183	return {TheBucket, true};
184	}
185
186	bool erase(const KeyT &Val) {
187	BucketT *TheBucket;
188	if (!LookupBucketFor(Val, TheBucket))
189	return false; // not in map.
190
191	TheBucket->getSecond().~ValueT();
192	TheBucket->getFirst() = getTombstoneKey();
193	decrementNumEntries();
194	incrementNumTombstones();
195	return true;
196	}
197
198	void erase(value_type *I) {
199	CHECK_NE(I, nullptr);
200	BucketT TheBucket = &I;
201	TheBucket->getSecond().~ValueT();
202	TheBucket->getFirst() = getTombstoneKey();
203	decrementNumEntries();
204	incrementNumTombstones();
205	}
206
207	value_type &FindAndConstruct(const KeyT &Key) {
208	BucketT *TheBucket;
209	if (LookupBucketFor(Key, TheBucket))
210	return *TheBucket;
211
212	return *InsertIntoBucket(TheBucket, Key);
213	}
214
215	ValueT &operator[](const KeyT &Key) { return FindAndConstruct(Key).second; }
216
217	value_type &FindAndConstruct(KeyT &&Key) {
218	BucketT *TheBucket;
219	if (LookupBucketFor(Key, TheBucket))
220	return *TheBucket;
221
222	return *InsertIntoBucket(TheBucket, __sanitizer::move(Key));
223	}
224
225	ValueT &operator[](KeyT &&Key) {
226	return FindAndConstruct(__sanitizer::move(Key)).second;
227	}
228
229	/// Iterate over active entries of the container.
230	///
231	/// Function can return fast to stop the process.
232	template <class Fn>
233	void forEach(Fn fn) {
234	const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey();
235	for (auto P = getBuckets(), E = getBucketsEnd(); P != E; ++P) {
236	const KeyT K = P->getFirst();
237	if (!KeyInfoT::isEqual(K, EmptyKey) &&
238	!KeyInfoT::isEqual(K, TombstoneKey)) {
239	if (!fn(*P))
240	return;
241	}
242	}
243	}
244
245	template <class Fn>
246	void forEach(Fn fn) const {
247	const_cast<DenseMapBase *>(this)->forEach(
248	[&](const value_type &KV) { return fn(KV); });
249	}
250
251	protected:
252	DenseMapBase() = default;
253
254	void destroyAll() {
255	if (getNumBuckets() == 0) // Nothing to do.
256	return;
257
258	const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey();
259	for (BucketT P = getBuckets(), E = getBucketsEnd(); P != E; ++P) {
260	if (!KeyInfoT::isEqual(P->getFirst(), EmptyKey) &&
261	!KeyInfoT::isEqual(P->getFirst(), TombstoneKey))
262	P->getSecond().~ValueT();
263	P->getFirst().~KeyT();
264	}
265	}
266
267	void initEmpty() {
268	setNumEntries(0);
269	setNumTombstones(0);
270
271	CHECK_EQ((getNumBuckets() & (getNumBuckets() - 1)), 0);
272	const KeyT EmptyKey = getEmptyKey();
273	for (BucketT B = getBuckets(), E = getBucketsEnd(); B != E; ++B)
274	::new (&B->getFirst()) KeyT(EmptyKey);
275	}
276
277	/// Returns the number of buckets to allocate to ensure that the DenseMap can
278	/// accommodate \p NumEntries without need to grow().
279	unsigned getMinBucketToReserveForEntries(unsigned NumEntries) {
280	// Ensure that "NumEntries * 4 < NumBuckets * 3"
281	if (NumEntries == 0)
282	return 0;
283	// +1 is required because of the strict equality.
284	// For example if NumEntries is 48, we need to return 401.
285	return RoundUpToPowerOfTwo((NumEntries * 4 / 3 + 1) + /* NextPowerOf2 */ 1);
286	}
287
288	void moveFromOldBuckets(BucketT OldBucketsBegin, BucketT OldBucketsEnd) {
289	initEmpty();
290
291	// Insert all the old elements.
292	const KeyT EmptyKey = getEmptyKey();
293	const KeyT TombstoneKey = getTombstoneKey();
294	for (BucketT B = OldBucketsBegin, E = OldBucketsEnd; B != E; ++B) {
295	if (!KeyInfoT::isEqual(B->getFirst(), EmptyKey) &&
296	!KeyInfoT::isEqual(B->getFirst(), TombstoneKey)) {
297	// Insert the key/value into the new table.
298	BucketT *DestBucket;
299	bool FoundVal = LookupBucketFor(B->getFirst(), DestBucket);
300	(void)FoundVal; // silence warning.
301	CHECK(!FoundVal);
302	DestBucket->getFirst() = __sanitizer::move(B->getFirst());
303	::new (&DestBucket->getSecond())
304	ValueT(__sanitizer::move(B->getSecond()));
305	incrementNumEntries();
306
307	// Free the value.
308	B->getSecond().~ValueT();
309	}
310	B->getFirst().~KeyT();
311	}
312	}
313
314	template <typename OtherBaseT>
315	void copyFrom(
316	const DenseMapBase<OtherBaseT, KeyT, ValueT, KeyInfoT, BucketT> &other) {
317	CHECK_NE(&other, this);
318	CHECK_EQ(getNumBuckets(), other.getNumBuckets());
319
320	setNumEntries(other.getNumEntries());
321	setNumTombstones(other.getNumTombstones());
322
323	if (__sanitizer::is_trivially_copyable<KeyT>::value &&
324	__sanitizer::is_trivially_copyable<ValueT>::value)
325	internal_memcpy(reinterpret_cast<void *>(getBuckets()),
326	other.getBuckets(), getNumBuckets() * sizeof(BucketT));
327	else
328	for (uptr i = 0; i < getNumBuckets(); ++i) {
329	::new (&getBuckets()[i].getFirst())
330	KeyT(other.getBuckets()[i].getFirst());
331	if (!KeyInfoT::isEqual(getBuckets()[i].getFirst(), getEmptyKey()) &&
332	!KeyInfoT::isEqual(getBuckets()[i].getFirst(), getTombstoneKey()))
333	::new (&getBuckets()[i].getSecond())
334	ValueT(other.getBuckets()[i].getSecond());
335	}
336	}
337
338	static unsigned getHashValue(const KeyT &Val) {
339	return KeyInfoT::getHashValue(Val);
340	}
341
342	template <typename LookupKeyT>
343	static unsigned getHashValue(const LookupKeyT &Val) {
344	return KeyInfoT::getHashValue(Val);
345	}
346
347	static const KeyT getEmptyKey() { return KeyInfoT::getEmptyKey(); }
348
349	static const KeyT getTombstoneKey() { return KeyInfoT::getTombstoneKey(); }
350
351	private:
352	unsigned getNumEntries() const {
353	return static_cast<const DerivedT *>(this)->getNumEntries();
354	}
355
356	void setNumEntries(unsigned Num) {
357	static_cast<DerivedT *>(this)->setNumEntries(Num);
358	}
359
360	void incrementNumEntries() { setNumEntries(getNumEntries() + 1); }
361
362	void decrementNumEntries() { setNumEntries(getNumEntries() - 1); }
363
364	unsigned getNumTombstones() const {
365	return static_cast<const DerivedT *>(this)->getNumTombstones();
366	}
367
368	void setNumTombstones(unsigned Num) {
369	static_cast<DerivedT *>(this)->setNumTombstones(Num);
370	}
371
372	void incrementNumTombstones() { setNumTombstones(getNumTombstones() + 1); }
373
374	void decrementNumTombstones() { setNumTombstones(getNumTombstones() - 1); }
375
376	const BucketT *getBuckets() const {
377	return static_cast<const DerivedT *>(this)->getBuckets();
378	}
379
380	BucketT getBuckets() { return static_cast<DerivedT >(this)->getBuckets(); }
381
382	unsigned getNumBuckets() const {
383	return static_cast<const DerivedT *>(this)->getNumBuckets();
384	}
385
386	BucketT *getBucketsEnd() { return getBuckets() + getNumBuckets(); }
387
388	const BucketT *getBucketsEnd() const {
389	return getBuckets() + getNumBuckets();
390	}
391
392	void grow(unsigned AtLeast) { static_cast<DerivedT *>(this)->grow(AtLeast); }
393
394	template <typename KeyArg, typename... ValueArgs>
395	BucketT InsertIntoBucket(BucketT TheBucket, KeyArg &&Key,
396	ValueArgs &&...Values) {
397	TheBucket = InsertIntoBucketImpl(Key, Key, TheBucket);
398
399	TheBucket->getFirst() = __sanitizer::forward<KeyArg>(Key);
400	::new (&TheBucket->getSecond())
401	ValueT(__sanitizer::forward<ValueArgs>(Values)...);
402	return TheBucket;
403	}
404
405	template <typename LookupKeyT>
406	BucketT InsertIntoBucketWithLookup(BucketT TheBucket, KeyT &&Key,
407	ValueT &&Value, LookupKeyT &Lookup) {
408	TheBucket = InsertIntoBucketImpl(Key, Lookup, TheBucket);
409
410	TheBucket->getFirst() = __sanitizer::move(Key);
411	::new (&TheBucket->getSecond()) ValueT(__sanitizer::move(Value));
412	return TheBucket;
413	}
414
415	template <typename LookupKeyT>
416	BucketT *InsertIntoBucketImpl(const KeyT &Key, const LookupKeyT &Lookup,
417	BucketT *TheBucket) {
418	// If the load of the hash table is more than 3/4, or if fewer than 1/8 of
419	// the buckets are empty (meaning that many are filled with tombstones),
420	// grow the table.
421	//
422	// The later case is tricky. For example, if we had one empty bucket with
423	// tons of tombstones, failing lookups (e.g. for insertion) would have to
424	// probe almost the entire table until it found the empty bucket. If the
425	// table completely filled with tombstones, no lookup would ever succeed,
426	// causing infinite loops in lookup.
427	unsigned NewNumEntries = getNumEntries() + 1;
428	unsigned NumBuckets = getNumBuckets();
429	if (UNLIKELY(NewNumEntries * 4 >= NumBuckets * 3)) {
430	this->grow(NumBuckets * 2);
431	LookupBucketFor(Lookup, TheBucket);
432	NumBuckets = getNumBuckets();
433	} else if (UNLIKELY(NumBuckets - (NewNumEntries + getNumTombstones()) <=
434	NumBuckets / 8)) {
435	this->grow(NumBuckets);
436	LookupBucketFor(Lookup, TheBucket);
437	}
438	CHECK(TheBucket);
439
440	// Only update the state after we've grown our bucket space appropriately
441	// so that when growing buckets we have self-consistent entry count.
442	incrementNumEntries();
443
444	// If we are writing over a tombstone, remember this.
445	const KeyT EmptyKey = getEmptyKey();
446	if (!KeyInfoT::isEqual(TheBucket->getFirst(), EmptyKey))
447	decrementNumTombstones();
448
449	return TheBucket;
450	}
451
452	/// LookupBucketFor - Lookup the appropriate bucket for Val, returning it in
453	/// FoundBucket. If the bucket contains the key and a value, this returns
454	/// true, otherwise it returns a bucket with an empty marker or tombstone and
455	/// returns false.
456	template <typename LookupKeyT>
457	bool LookupBucketFor(const LookupKeyT &Val,
458	const BucketT *&FoundBucket) const {
459	const BucketT *BucketsPtr = getBuckets();
460	const unsigned NumBuckets = getNumBuckets();
461
462	if (NumBuckets == 0) {
463	FoundBucket = nullptr;
464	return false;
465	}
466
467	// FoundTombstone - Keep track of whether we find a tombstone while probing.
468	const BucketT *FoundTombstone = nullptr;
469	const KeyT EmptyKey = getEmptyKey();
470	const KeyT TombstoneKey = getTombstoneKey();
471	CHECK(!KeyInfoT::isEqual(Val, EmptyKey));
472	CHECK(!KeyInfoT::isEqual(Val, TombstoneKey));
473
474	unsigned BucketNo = getHashValue(Val) & (NumBuckets - 1);
475	unsigned ProbeAmt = 1;
476	while (true) {
477	const BucketT *ThisBucket = BucketsPtr + BucketNo;
478	// Found Val's bucket? If so, return it.
479	if (LIKELY(KeyInfoT::isEqual(Val, ThisBucket->getFirst()))) {
480	FoundBucket = ThisBucket;
481	return true;
482	}
483
484	// If we found an empty bucket, the key doesn't exist in the set.
485	// Insert it and return the default value.
486	if (LIKELY(KeyInfoT::isEqual(ThisBucket->getFirst(), EmptyKey))) {
487	// If we've already seen a tombstone while probing, fill it in instead
488	// of the empty bucket we eventually probed to.
489	FoundBucket = FoundTombstone ? FoundTombstone : ThisBucket;
490	return false;
491	}
492
493	// If this is a tombstone, remember it. If Val ends up not in the map, we
494	// prefer to return it than something that would require more probing.
495	if (KeyInfoT::isEqual(ThisBucket->getFirst(), TombstoneKey) &&
496	!FoundTombstone)
497	FoundTombstone = ThisBucket; // Remember the first tombstone found.
498
499	// Otherwise, it's a hash collision or a tombstone, continue quadratic
500	// probing.
501	BucketNo += ProbeAmt++;
502	BucketNo &= (NumBuckets - 1);
503	}
504	}
505
506	template <typename LookupKeyT>
507	bool LookupBucketFor(const LookupKeyT &Val, BucketT *&FoundBucket) {
508	const BucketT *ConstFoundBucket;
509	bool Result = const_cast<const DenseMapBase *>(this)->LookupBucketFor(
510	Val, ConstFoundBucket);
511	FoundBucket = const_cast<BucketT *>(ConstFoundBucket);
512	return Result;
513	}
514
515	public:
516	/// Return the approximate size (in bytes) of the actual map.
517	/// This is just the raw memory used by DenseMap.
518	/// If entries are pointers to objects, the size of the referenced objects
519	/// are not included.
520	uptr getMemorySize() const {
521	return RoundUpTo(getNumBuckets() * sizeof(BucketT), GetPageSizeCached());
522	}
523	};
524
525	/// Equality comparison for DenseMap.
526	///
527	/// Iterates over elements of LHS confirming that each (key, value) pair in LHS
528	/// is also in RHS, and that no additional pairs are in RHS.
529	/// Equivalent to N calls to RHS.find and N value comparisons. Amortized
530	/// complexity is linear, worst case is O(N^2) (if every hash collides).
531	template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT,
532	typename BucketT>
533	bool operator==(
534	const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &LHS,
535	const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &RHS) {
536	if (LHS.size() != RHS.size())
537	return false;
538
539	bool R = true;
540	LHS.forEach(
541	[&](const typename DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT,
542	BucketT>::value_type &KV) -> bool {
543	const auto *I = RHS.find(KV.first);
544	if (!I \|\| I->second != KV.second) {
545	R = false;
546	return false;
547	}
548	return true;
549	});
550
551	return R;
552	}
553
554	/// Inequality comparison for DenseMap.
555	///
556	/// Equivalent to !(LHS == RHS). See operator== for performance notes.
557	template <typename DerivedT, typename KeyT, typename ValueT, typename KeyInfoT,
558	typename BucketT>
559	bool operator!=(
560	const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &LHS,
561	const DenseMapBase<DerivedT, KeyT, ValueT, KeyInfoT, BucketT> &RHS) {
562	return !(LHS == RHS);
563	}
564
565	template <typename KeyT, typename ValueT,
566	typename KeyInfoT = DenseMapInfo<KeyT>,
567	typename BucketT = detail::DenseMapPair<KeyT, ValueT>>
568	class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
569	KeyT, ValueT, KeyInfoT, BucketT> {
570	friend class DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
571
572	// Lift some types from the dependent base class into this class for
573	// simplicity of referring to them.
574	using BaseT = DenseMapBase<DenseMap, KeyT, ValueT, KeyInfoT, BucketT>;
575
576	BucketT *Buckets = nullptr;
577	unsigned NumEntries = 0;
578	unsigned NumTombstones = 0;
579	unsigned NumBuckets = 0;
580
581	public:
582	/// Create a DenseMap with an optional \p InitialReserve that guarantee that
583	/// this number of elements can be inserted in the map without grow()
584	explicit DenseMap(unsigned InitialReserve) { init(InitialReserve); }
585	constexpr DenseMap() = default;
586
587	DenseMap(const DenseMap &other) : BaseT() {
588	init(0);
589	copyFrom(other);
590	}
591
592	DenseMap(DenseMap &&other) : BaseT() {
593	init(0);
594	swap(other);
595	}
596
597	~DenseMap() {
598	this->destroyAll();
599	deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets);
600	}
601
602	void swap(DenseMap &RHS) {
603	Swap(Buckets, RHS.Buckets);
604	Swap(NumEntries, RHS.NumEntries);
605	Swap(NumTombstones, RHS.NumTombstones);
606	Swap(NumBuckets, RHS.NumBuckets);
607	}
608
609	DenseMap &operator=(const DenseMap &other) {
610	if (&other != this)
611	copyFrom(other);
612	return *this;
613	}
614
615	DenseMap &operator=(DenseMap &&other) {
616	this->destroyAll();
617	deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets, alignof(BucketT));
618	init(0);
619	swap(other);
620	return *this;
621	}
622
623	void copyFrom(const DenseMap &other) {
624	this->destroyAll();
625	deallocate_buffer(Buckets, sizeof(BucketT) * NumBuckets);
626	if (allocateBuckets(other.NumBuckets)) {
627	this->BaseT::copyFrom(other);
628	} else {
629	NumEntries = 0;
630	NumTombstones = 0;
631	}
632	}
633
634	void init(unsigned InitNumEntries) {
635	auto InitBuckets = BaseT::getMinBucketToReserveForEntries(InitNumEntries);
636	if (allocateBuckets(InitBuckets)) {
637	this->BaseT::initEmpty();
638	} else {
639	NumEntries = 0;
640	NumTombstones = 0;
641	}
642	}
643
644	void grow(unsigned AtLeast) {
645	unsigned OldNumBuckets = NumBuckets;
646	BucketT *OldBuckets = Buckets;
647
648	allocateBuckets(RoundUpToPowerOfTwo(Max<unsigned>(64, AtLeast)));
649	CHECK(Buckets);
650	if (!OldBuckets) {
651	this->BaseT::initEmpty();
652	return;
653	}
654
655	this->moveFromOldBuckets(OldBuckets, OldBuckets + OldNumBuckets);
656
657	// Free the old table.
658	deallocate_buffer(OldBuckets, sizeof(BucketT) * OldNumBuckets);
659	}
660
661	private:
662	unsigned getNumEntries() const { return NumEntries; }
663
664	void setNumEntries(unsigned Num) { NumEntries = Num; }
665
666	unsigned getNumTombstones() const { return NumTombstones; }
667
668	void setNumTombstones(unsigned Num) { NumTombstones = Num; }
669
670	BucketT *getBuckets() const { return Buckets; }
671
672	unsigned getNumBuckets() const { return NumBuckets; }
673
674	bool allocateBuckets(unsigned Num) {
675	NumBuckets = Num;
676	if (NumBuckets == 0) {
677	Buckets = nullptr;
678	return false;
679	}
680
681	uptr Size = sizeof(BucketT) * NumBuckets;
682	if (Size * 2 <= GetPageSizeCached()) {
683	// We always allocate at least a page, so use entire space.
684	unsigned Log2 = MostSignificantSetBitIndex(GetPageSizeCached() / Size);
685	Size <<= Log2;
686	NumBuckets <<= Log2;
687	CHECK_EQ(Size, sizeof(BucketT) * NumBuckets);
688	CHECK_GT(Size * 2, GetPageSizeCached());
689	}
690	Buckets = static_cast<BucketT *>(allocate_buffer(Size));
691	return true;
692	}
693
694	static void *allocate_buffer(uptr Size) {
695	return MmapOrDie(RoundUpTo(Size, GetPageSizeCached()), "DenseMap");
696	}
697
698	static void deallocate_buffer(void *Ptr, uptr Size) {
699	UnmapOrDie(Ptr, RoundUpTo(Size, GetPageSizeCached()));
700	}
701	};
702
703	} // namespace __sanitizer
704
705	#endif // SANITIZER_DENSE_MAP_H

lib/tsan/sanitizer_common/sanitizer_dense_map_info.h created+282

...	@@ -0,0 +1,282 @@
1	//===- sanitizer_dense_map_info.h - Type traits for DenseMap ----- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#ifndef SANITIZER_DENSE_MAP_INFO_H
10	#define SANITIZER_DENSE_MAP_INFO_H
11
12	#include "sanitizer_common.h"
13	#include "sanitizer_internal_defs.h"
14	#include "sanitizer_type_traits.h"
15
16	namespace __sanitizer {
17
18	namespace detail {
19
20	/// Simplistic combination of 32-bit hash values into 32-bit hash values.
21	static constexpr unsigned combineHashValue(unsigned a, unsigned b) {
22	u64 key = (u64)a << 32 \| (u64)b;
23	key += ~(key << 32);
24	key ^= (key >> 22);
25	key += ~(key << 13);
26	key ^= (key >> 8);
27	key += (key << 3);
28	key ^= (key >> 15);
29	key += ~(key << 27);
30	key ^= (key >> 31);
31	return (unsigned)key;
32	}
33
34	// We extend a pair to allow users to override the bucket type with their own
35	// implementation without requiring two members.
36	template <typename KeyT, typename ValueT>
37	struct DenseMapPair {
38	KeyT first = {};
39	ValueT second = {};
40	constexpr DenseMapPair() = default;
41	constexpr DenseMapPair(const KeyT &f, const ValueT &s)
42	: first(f), second(s) {}
43
44	template <typename KeyT2, typename ValueT2>
45	constexpr DenseMapPair(KeyT2 &&f, ValueT2 &&s)
46	: first(__sanitizer::forward<KeyT2>(f)),
47	second(__sanitizer::forward<ValueT2>(s)) {}
48
49	constexpr DenseMapPair(const DenseMapPair &other) = default;
50	constexpr DenseMapPair &operator=(const DenseMapPair &other) = default;
51	constexpr DenseMapPair(DenseMapPair &&other) = default;
52	constexpr DenseMapPair &operator=(DenseMapPair &&other) = default;
53
54	KeyT &getFirst() { return first; }
55	const KeyT &getFirst() const { return first; }
56	ValueT &getSecond() { return second; }
57	const ValueT &getSecond() const { return second; }
58	};
59
60	} // end namespace detail
61
62	template <typename T>
63	struct DenseMapInfo {
64	// static T getEmptyKey();
65	// static T getTombstoneKey();
66	// static unsigned getHashValue(const T &Val);
67	// static bool isEqual(const T &LHS, const T &RHS);
68	};
69
70	// Provide DenseMapInfo for all pointers. Come up with sentinel pointer values
71	// that are aligned to alignof(T) bytes, but try to avoid requiring T to be
72	// complete. This allows clients to instantiate DenseMap<T*, ...> with forward
73	// declared key types. Assume that no pointer key type requires more than 4096
74	// bytes of alignment.
75	template <typename T>
76	struct DenseMapInfo<T *> {
77	// The following should hold, but it would require T to be complete:
78	// static_assert(alignof(T) <= (1 << Log2MaxAlign),
79	// "DenseMap does not support pointer keys requiring more than "
80	// "Log2MaxAlign bits of alignment");
81	static constexpr uptr Log2MaxAlign = 12;
82
83	static constexpr T *getEmptyKey() {
84	uptr Val = static_cast<uptr>(-1);
85	Val <<= Log2MaxAlign;
86	return reinterpret_cast<T *>(Val);
87	}
88
89	static constexpr T *getTombstoneKey() {
90	uptr Val = static_cast<uptr>(-2);
91	Val <<= Log2MaxAlign;
92	return reinterpret_cast<T *>(Val);
93	}
94
95	static constexpr unsigned getHashValue(const T *PtrVal) {
96	return (unsigned((uptr)PtrVal) >> 4) ^ (unsigned((uptr)PtrVal) >> 9);
97	}
98
99	static constexpr bool isEqual(const T LHS, const T RHS) {
100	return LHS == RHS;
101	}
102	};
103
104	// Provide DenseMapInfo for chars.
105	template <>
106	struct DenseMapInfo<char> {
107	static constexpr char getEmptyKey() { return ~0; }
108	static constexpr char getTombstoneKey() { return ~0 - 1; }
109	static constexpr unsigned getHashValue(const char &Val) { return Val * 37U; }
110
111	static constexpr bool isEqual(const char &LHS, const char &RHS) {
112	return LHS == RHS;
113	}
114	};
115
116	// Provide DenseMapInfo for unsigned chars.
117	template <>
118	struct DenseMapInfo<unsigned char> {
119	static constexpr unsigned char getEmptyKey() { return ~0; }
120	static constexpr unsigned char getTombstoneKey() { return ~0 - 1; }
121	static constexpr unsigned getHashValue(const unsigned char &Val) {
122	return Val * 37U;
123	}
124
125	static constexpr bool isEqual(const unsigned char &LHS,
126	const unsigned char &RHS) {
127	return LHS == RHS;
128	}
129	};
130
131	// Provide DenseMapInfo for unsigned shorts.
132	template <>
133	struct DenseMapInfo<unsigned short> {
134	static constexpr unsigned short getEmptyKey() { return 0xFFFF; }
135	static constexpr unsigned short getTombstoneKey() { return 0xFFFF - 1; }
136	static constexpr unsigned getHashValue(const unsigned short &Val) {
137	return Val * 37U;
138	}
139
140	static constexpr bool isEqual(const unsigned short &LHS,
141	const unsigned short &RHS) {
142	return LHS == RHS;
143	}
144	};
145
146	// Provide DenseMapInfo for unsigned ints.
147	template <>
148	struct DenseMapInfo<unsigned> {
149	static constexpr unsigned getEmptyKey() { return ~0U; }
150	static constexpr unsigned getTombstoneKey() { return ~0U - 1; }
151	static constexpr unsigned getHashValue(const unsigned &Val) {
152	return Val * 37U;
153	}
154
155	static constexpr bool isEqual(const unsigned &LHS, const unsigned &RHS) {
156	return LHS == RHS;
157	}
158	};
159
160	// Provide DenseMapInfo for unsigned longs.
161	template <>
162	struct DenseMapInfo<unsigned long> {
163	static constexpr unsigned long getEmptyKey() { return ~0UL; }
164	static constexpr unsigned long getTombstoneKey() { return ~0UL - 1L; }
165
166	static constexpr unsigned getHashValue(const unsigned long &Val) {
167	return (unsigned)(Val * 37UL);
168	}
169
170	static constexpr bool isEqual(const unsigned long &LHS,
171	const unsigned long &RHS) {
172	return LHS == RHS;
173	}
174	};
175
176	// Provide DenseMapInfo for unsigned long longs.
177	template <>
178	struct DenseMapInfo<unsigned long long> {
179	static constexpr unsigned long long getEmptyKey() { return ~0ULL; }
180	static constexpr unsigned long long getTombstoneKey() { return ~0ULL - 1ULL; }
181
182	static constexpr unsigned getHashValue(const unsigned long long &Val) {
183	return (unsigned)(Val * 37ULL);
184	}
185
186	static constexpr bool isEqual(const unsigned long long &LHS,
187	const unsigned long long &RHS) {
188	return LHS == RHS;
189	}
190	};
191
192	// Provide DenseMapInfo for shorts.
193	template <>
194	struct DenseMapInfo<short> {
195	static constexpr short getEmptyKey() { return 0x7FFF; }
196	static constexpr short getTombstoneKey() { return -0x7FFF - 1; }
197	static constexpr unsigned getHashValue(const short &Val) { return Val * 37U; }
198	static constexpr bool isEqual(const short &LHS, const short &RHS) {
199	return LHS == RHS;
200	}
201	};
202
203	// Provide DenseMapInfo for ints.
204	template <>
205	struct DenseMapInfo<int> {
206	static constexpr int getEmptyKey() { return 0x7fffffff; }
207	static constexpr int getTombstoneKey() { return -0x7fffffff - 1; }
208	static constexpr unsigned getHashValue(const int &Val) {
209	return (unsigned)(Val * 37U);
210	}
211
212	static constexpr bool isEqual(const int &LHS, const int &RHS) {
213	return LHS == RHS;
214	}
215	};
216
217	// Provide DenseMapInfo for longs.
218	template <>
219	struct DenseMapInfo<long> {
220	static constexpr long getEmptyKey() {
221	return (1UL << (sizeof(long) * 8 - 1)) - 1UL;
222	}
223
224	static constexpr long getTombstoneKey() { return getEmptyKey() - 1L; }
225
226	static constexpr unsigned getHashValue(const long &Val) {
227	return (unsigned)(Val * 37UL);
228	}
229
230	static constexpr bool isEqual(const long &LHS, const long &RHS) {
231	return LHS == RHS;
232	}
233	};
234
235	// Provide DenseMapInfo for long longs.
236	template <>
237	struct DenseMapInfo<long long> {
238	static constexpr long long getEmptyKey() { return 0x7fffffffffffffffLL; }
239	static constexpr long long getTombstoneKey() {
240	return -0x7fffffffffffffffLL - 1;
241	}
242
243	static constexpr unsigned getHashValue(const long long &Val) {
244	return (unsigned)(Val * 37ULL);
245	}
246
247	static constexpr bool isEqual(const long long &LHS, const long long &RHS) {
248	return LHS == RHS;
249	}
250	};
251
252	// Provide DenseMapInfo for all pairs whose members have info.
253	template <typename T, typename U>
254	struct DenseMapInfo<detail::DenseMapPair<T, U>> {
255	using Pair = detail::DenseMapPair<T, U>;
256	using FirstInfo = DenseMapInfo<T>;
257	using SecondInfo = DenseMapInfo<U>;
258
259	static constexpr Pair getEmptyKey() {
260	return detail::DenseMapPair<T, U>(FirstInfo::getEmptyKey(),
261	SecondInfo::getEmptyKey());
262	}
263
264	static constexpr Pair getTombstoneKey() {
265	return detail::DenseMapPair<T, U>(FirstInfo::getTombstoneKey(),
266	SecondInfo::getTombstoneKey());
267	}
268
269	static constexpr unsigned getHashValue(const Pair &PairVal) {
270	return detail::combineHashValue(FirstInfo::getHashValue(PairVal.first),
271	SecondInfo::getHashValue(PairVal.second));
272	}
273
274	static constexpr bool isEqual(const Pair &LHS, const Pair &RHS) {
275	return FirstInfo::isEqual(LHS.first, RHS.first) &&
276	SecondInfo::isEqual(LHS.second, RHS.second);
277	}
278	};
279
280	} // namespace __sanitizer
281
282	#endif // SANITIZER_DENSE_MAP_INFO_H

lib/tsan/sanitizer_common/sanitizer_errno.h+1-1

...	@@ -21,7 +21,7 @@	...	@@ -21,7 +21,7 @@
21	#include "sanitizer_errno_codes.h"	21	#include "sanitizer_errno_codes.h"
22	#include "sanitizer_platform.h"	22	#include "sanitizer_platform.h"
23		23
24	#if SANITIZER_FREEBSD \|\| SANITIZER_MAC	24	#if SANITIZER_FREEBSD \|\| SANITIZER_APPLE
25	# define __errno_location __error	25	# define __errno_location __error
26	#elif SANITIZER_ANDROID \|\| SANITIZER_NETBSD	26	#elif SANITIZER_ANDROID \|\| SANITIZER_NETBSD
27	# define __errno_location __errno	27	# define __errno_location __errno

lib/tsan/sanitizer_common/sanitizer_errno_codes.h+1

...	@@ -25,6 +25,7 @@ namespace __sanitizer {	...	@@ -25,6 +25,7 @@ namespace __sanitizer {
25	#define errno_EBUSY 16	25	#define errno_EBUSY 16
26	#define errno_EINVAL 22	26	#define errno_EINVAL 22
27	#define errno_ENAMETOOLONG 36	27	#define errno_ENAMETOOLONG 36
		28	#define errno_ENOSYS 38
28		29
29	// Those might not present or their value differ on different platforms.	30	// Those might not present or their value differ on different platforms.
30	extern const int errno_EOWNERDEAD;	31	extern const int errno_EOWNERDEAD;

lib/tsan/sanitizer_common/sanitizer_file.cpp+20

...	@@ -19,6 +19,7 @@	...	@@ -19,6 +19,7 @@
19		19
20	#include "sanitizer_common.h"	20	#include "sanitizer_common.h"
21	#include "sanitizer_file.h"	21	#include "sanitizer_file.h"
		22	# include "sanitizer_interface_internal.h"
22		23
23	namespace __sanitizer {	24	namespace __sanitizer {
24		25
...	@@ -75,6 +76,24 @@ void ReportFile::ReopenIfNecessary() {	...	@@ -75,6 +76,24 @@ void ReportFile::ReopenIfNecessary() {
75	fd_pid = pid;	76	fd_pid = pid;
76	}	77	}
77		78
		79	static void RecursiveCreateParentDirs(char *path) {
		80	if (path[0] == '\0')
		81	return;
		82	for (int i = 1; path[i] != '\0'; ++i) {
		83	char save = path[i];
		84	if (!IsPathSeparator(path[i]))
		85	continue;
		86	path[i] = '\0';
		87	if (!DirExists(path) && !CreateDir(path)) {
		88	const char *ErrorMsgPrefix = "ERROR: Can't create directory: ";
		89	WriteToFile(kStderrFd, ErrorMsgPrefix, internal_strlen(ErrorMsgPrefix));
		90	WriteToFile(kStderrFd, path, internal_strlen(path));
		91	Die();
		92	}
		93	path[i] = save;
		94	}
		95	}
		96
78	void ReportFile::SetReportPath(const char *path) {	97	void ReportFile::SetReportPath(const char *path) {
79	if (path) {	98	if (path) {
80	uptr len = internal_strlen(path);	99	uptr len = internal_strlen(path);
...	@@ -95,6 +114,7 @@ void ReportFile::SetReportPath(const char *path) {	...	@@ -95,6 +114,7 @@ void ReportFile::SetReportPath(const char *path) {
95	fd = kStdoutFd;	114	fd = kStdoutFd;
96	} else {	115	} else {
97	internal_snprintf(path_prefix, kMaxPathLength, "%s", path);	116	internal_snprintf(path_prefix, kMaxPathLength, "%s", path);
		117	RecursiveCreateParentDirs(path_prefix);
98	}	118	}
99	}	119	}
100		120

lib/tsan/sanitizer_common/sanitizer_file.h+4-1

 @@ -15,7 +15,7 @@
 #ifndef SANITIZER_FILE_H
 #define SANITIZER_FILE_H
-#include "sanitizer_interface_internal.h"
+#include "sanitizer_common.h"
 #include "sanitizer_internal_defs.h"
 #include "sanitizer_libc.h"
 #include "sanitizer_mutex.h"
 @@ -78,9 +78,12 @@ bool SupportsColoredOutput(fd_t fd);
 // OS
 const char *GetPwd();
 bool FileExists(const char *filename);
+bool DirExists(const char *path);
 char *FindPathToBinary(const char *name);
 bool IsPathSeparator(const char c);
 bool IsAbsolutePath(const char *path);
+// Returns true on success, false on failure.
+bool CreateDir(const char *pathname);
 // Starts a subprocess and returs its pid.
 // If *_fd parameters are not kInvalidFd their corresponding input/output
 // streams will be redirect to the file. The files will always be closed

lib/tsan/sanitizer_common/sanitizer_flag_parser.cpp+2-2

 @@ -13,9 +13,9 @@
 #include "sanitizer_flag_parser.h"
 #include "sanitizer_common.h"
-#include "sanitizer_libc.h"
-#include "sanitizer_flags.h"
 #include "sanitizer_flag_parser.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_libc.h"
 namespace __sanitizer {

lib/tsan/sanitizer_common/sanitizer_flag_parser.h+2-2

...	@@ -13,9 +13,9 @@	...	@@ -13,9 +13,9 @@
13	#ifndef SANITIZER_FLAG_REGISTRY_H	13	#ifndef SANITIZER_FLAG_REGISTRY_H
14	#define SANITIZER_FLAG_REGISTRY_H	14	#define SANITIZER_FLAG_REGISTRY_H
15		15
		16	#include "sanitizer_common.h"
16	#include "sanitizer_internal_defs.h"	17	#include "sanitizer_internal_defs.h"
17	#include "sanitizer_libc.h"	18	#include "sanitizer_libc.h"
18	#include "sanitizer_common.h"
19		19
20	namespace __sanitizer {	20	namespace __sanitizer {
21		21
...	@@ -138,7 +138,7 @@ inline bool FlagHandler<uptr>::Parse(const char *value) {	...	@@ -138,7 +138,7 @@ inline bool FlagHandler<uptr>::Parse(const char *value) {
138		138
139	template <>	139	template <>
140	inline bool FlagHandler<uptr>::Format(char *buffer, uptr size) {	140	inline bool FlagHandler<uptr>::Format(char *buffer, uptr size) {
141	uptr num_symbols_should_write = internal_snprintf(buffer, size, "%p", *t_);	141	uptr num_symbols_should_write = internal_snprintf(buffer, size, "0x%zx", *t_);
142	return num_symbols_should_write < size;	142	return num_symbols_should_write < size;
143	}	143	}
144		144

lib/tsan/sanitizer_common/sanitizer_flags.inc+15-5

 @@ -62,16 +62,19 @@ COMMON_FLAG(
 COMMON_FLAG(const char *, log_suffix, nullptr,
             "String to append to log file name, e.g. \".txt\".")
 COMMON_FLAG(
-    bool, log_to_syslog, (bool)SANITIZER_ANDROID || (bool)SANITIZER_MAC,
+    bool, log_to_syslog, (bool)SANITIZER_ANDROID || (bool)SANITIZER_APPLE,
     "Write all sanitizer output to syslog in addition to other means of "
     "logging.")
 COMMON_FLAG(
     int, verbosity, 0,
     "Verbosity level (0 - silent, 1 - a bit of output, 2+ - more output).")
-COMMON_FLAG(bool, strip_env, 1,
+COMMON_FLAG(bool, strip_env, true,
             "Whether to remove the sanitizer from DYLD_INSERT_LIBRARIES to "
-            "avoid passing it to children. Default is true.")
+            "avoid passing it to children on Apple platforms. Default is true.")
-COMMON_FLAG(bool, detect_leaks, !SANITIZER_MAC, "Enable memory leak detection.")
+COMMON_FLAG(bool, verify_interceptors, true,
+            "Verify that interceptors are working on Apple platforms. Default "
+            "is true.")
+COMMON_FLAG(bool, detect_leaks, !SANITIZER_APPLE, "Enable memory leak detection.")
 COMMON_FLAG(
     bool, leak_check_at_exit, true,
     "Invoke leak checking in an atexit handler. Has no effect if "
 @@ -160,6 +163,10 @@ COMMON_FLAG(
 COMMON_FLAG(const char *, coverage_dir, ".",
             "Target directory for coverage dumps. Defaults to the current "
             "directory.")
+COMMON_FLAG(const char *, cov_8bit_counters_out, "",
+    "If non-empty, write 8bit counters to this file. ")
+COMMON_FLAG(const char *, cov_pcs_out, "",
+    "If non-empty, write the coverage pc table to this file. ")
 COMMON_FLAG(bool, full_address_space, false,
             "Sanitize complete address space; "
             "by default kernel area on 32-bit platforms will not be sanitized")
 @@ -175,6 +182,7 @@ COMMON_FLAG(bool, use_madv_dontdump, true,
           "in core file.")
 COMMON_FLAG(bool, symbolize_inline_frames, true,
             "Print inlined frames in stacktraces. Defaults to true.")
+COMMON_FLAG(bool, demangle, true, "Print demangled symbols.")
 COMMON_FLAG(bool, symbolize_vs_style, false,
             "Print file locations in Visual Studio style (e.g: "
             " file(10,42): ...")
 @@ -187,6 +195,8 @@ COMMON_FLAG(const char *, stack_trace_format, "DEFAULT",
             "Format string used to render stack frames. "
             "See sanitizer_stacktrace_printer.h for the format description. "
             "Use DEFAULT to get default format.")
+COMMON_FLAG(int, compress_stack_depot, 0,
+            "Compress stack depot to save memory.")
 COMMON_FLAG(bool, no_huge_pages_for_shadow, true,
             "If true, the shadow is not allowed to use huge pages. ")
 COMMON_FLAG(bool, strict_string_checks, false,
 @@ -238,7 +248,7 @@ COMMON_FLAG(bool, decorate_proc_maps, (bool)SANITIZER_ANDROID,
 COMMON_FLAG(int, exitcode, 1, "Override the program exit status if the tool "
                               "found an error")
 COMMON_FLAG(
-    bool, abort_on_error, (bool)SANITIZER_ANDROID || (bool)SANITIZER_MAC,
+    bool, abort_on_error, (bool)SANITIZER_ANDROID || (bool)SANITIZER_APPLE,
     "If set, the tool calls abort() instead of _exit() after printing the "
     "error report.")
 COMMON_FLAG(bool, suppress_equal_pcs, true,

lib/tsan/sanitizer_common/sanitizer_flat_map.h created+162

...	@@ -0,0 +1,162 @@
1	//===-- sanitizer_flat_map.h ------------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Part of the Sanitizer Allocator.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#ifndef SANITIZER_FLAT_MAP_H
14	#define SANITIZER_FLAT_MAP_H
15
16	#include "sanitizer_atomic.h"
17	#include "sanitizer_common.h"
18	#include "sanitizer_internal_defs.h"
19	#include "sanitizer_local_address_space_view.h"
20	#include "sanitizer_mutex.h"
21
22	namespace __sanitizer {
23
24	// Maps integers in rage [0, kSize) to values.
25	template <typename T, u64 kSize,
26	typename AddressSpaceViewTy = LocalAddressSpaceView>
27	class FlatMap {
28	public:
29	using AddressSpaceView = AddressSpaceViewTy;
30	void Init() { internal_memset(map_, 0, sizeof(map_)); }
31
32	constexpr uptr size() const { return kSize; }
33
34	bool contains(uptr idx) const {
35	CHECK_LT(idx, kSize);
36	return true;
37	}
38
39	T &operator[](uptr idx) {
40	DCHECK_LT(idx, kSize);
41	return map_[idx];
42	}
43
44	const T &operator[](uptr idx) const {
45	DCHECK_LT(idx, kSize);
46	return map_[idx];
47	}
48
49	private:
50	T map_[kSize];
51	};
52
53	// TwoLevelMap maps integers in range [0, kSize1*kSize2) to values.
54	// It is implemented as a two-dimensional array: array of kSize1 pointers
55	// to kSize2-byte arrays. The secondary arrays are mmaped on demand.
56	// Each value is initially zero and can be set to something else only once.
57	// Setting and getting values from multiple threads is safe w/o extra locking.
58	template <typename T, u64 kSize1, u64 kSize2,
59	typename AddressSpaceViewTy = LocalAddressSpaceView>
60	class TwoLevelMap {
61	static_assert(IsPowerOfTwo(kSize2), "Use a power of two for performance.");
62
63	public:
64	using AddressSpaceView = AddressSpaceViewTy;
65	void Init() {
66	mu_.Init();
67	internal_memset(map1_, 0, sizeof(map1_));
68	}
69
70	void TestOnlyUnmap() {
71	for (uptr i = 0; i < kSize1; i++) {
72	T *p = Get(i);
73	if (!p)
74	continue;
75	UnmapOrDie(p, kSize2);
76	}
77	Init();
78	}
79
80	uptr MemoryUsage() const {
81	uptr res = 0;
82	for (uptr i = 0; i < kSize1; i++) {
83	T *p = Get(i);
84	if (!p)
85	continue;
86	res += MmapSize();
87	}
88	return res;
89	}
90
91	constexpr uptr size() const { return kSize1 * kSize2; }
92	constexpr uptr size1() const { return kSize1; }
93	constexpr uptr size2() const { return kSize2; }
94
95	bool contains(uptr idx) const {
96	CHECK_LT(idx, kSize1 * kSize2);
97	return Get(idx / kSize2);
98	}
99
100	const T &operator[](uptr idx) const {
101	DCHECK_LT(idx, kSize1 * kSize2);
102	T *map2 = GetOrCreate(idx / kSize2);
103	return *AddressSpaceView::Load(&map2[idx % kSize2]);
104	}
105
106	T &operator[](uptr idx) {
107	DCHECK_LT(idx, kSize1 * kSize2);
108	T *map2 = GetOrCreate(idx / kSize2);
109	return *AddressSpaceView::LoadWritable(&map2[idx % kSize2]);
110	}
111
112	private:
113	constexpr uptr MmapSize() const {
114	return RoundUpTo(kSize2 * sizeof(T), GetPageSizeCached());
115	}
116
117	T *Get(uptr idx) const {
118	DCHECK_LT(idx, kSize1);
119	return reinterpret_cast<T *>(
120	atomic_load(&map1_[idx], memory_order_acquire));
121	}
122
123	T *GetOrCreate(uptr idx) const {
124	DCHECK_LT(idx, kSize1);
125	// This code needs to use memory_order_acquire/consume, but we use
126	// memory_order_relaxed for performance reasons (matters for arm64). We
127	// expect memory_order_relaxed to be effectively equivalent to
128	// memory_order_consume in this case for all relevant architectures: all
129	// dependent data is reachable only by dereferencing the resulting pointer.
130	// If relaxed load fails to see stored ptr, the code will fall back to
131	// Create() and reload the value again with locked mutex as a memory
132	// barrier.
133	T res = reinterpret_cast<T >(atomic_load_relaxed(&map1_[idx]));
134	if (LIKELY(res))
135	return res;
136	return Create(idx);
137	}
138
139	NOINLINE T *Create(uptr idx) const {
140	SpinMutexLock l(&mu_);
141	T *res = Get(idx);
142	if (!res) {
143	res = reinterpret_cast<T *>(MmapOrDie(MmapSize(), "TwoLevelMap"));
144	atomic_store(&map1_[idx], reinterpret_cast<uptr>(res),
145	memory_order_release);
146	}
147	return res;
148	}
149
150	mutable StaticSpinMutex mu_;
151	mutable atomic_uintptr_t map1_[kSize1];
152	};
153
154	template <u64 kSize, typename AddressSpaceViewTy = LocalAddressSpaceView>
155	using FlatByteMap = FlatMap<u8, kSize, AddressSpaceViewTy>;
156
157	template <u64 kSize1, u64 kSize2,
158	typename AddressSpaceViewTy = LocalAddressSpaceView>
159	using TwoLevelByteMap = TwoLevelMap<u8, kSize1, kSize2, AddressSpaceViewTy>;
160	} // namespace __sanitizer
161
162	#endif

lib/tsan/sanitizer_common/sanitizer_fuchsia.cpp+37-79

 @@ -14,24 +14,25 @@
 #include "sanitizer_fuchsia.h"
 #if SANITIZER_FUCHSIA
-#include <pthread.h>
+#  include <pthread.h>
-#include <stdlib.h>
+#  include <stdlib.h>
-#include <unistd.h>
+#  include <unistd.h>
-#include <zircon/errors.h>
+#  include <zircon/errors.h>
-#include <zircon/process.h>
+#  include <zircon/process.h>
-#include <zircon/syscalls.h>
+#  include <zircon/syscalls.h>
-#include <zircon/utc.h>
+#  include <zircon/utc.h>
-#include "sanitizer_common.h"
+#  include "sanitizer_common.h"
-#include "sanitizer_libc.h"
+#  include "sanitizer_interface_internal.h"
-#include "sanitizer_mutex.h"
+#  include "sanitizer_libc.h"
+#  include "sanitizer_mutex.h"
 namespace __sanitizer {
 void NORETURN internal__exit(int exitcode) { _zx_process_exit(exitcode); }
 uptr internal_sched_yield() {
-  zx_status_t status = _zx_nanosleep(0);
+  zx_status_t status = _zx_thread_legacy_yield(0u);
   CHECK_EQ(status, ZX_OK);
   return 0;  // Why doesn't this return void?
 }
 @@ -86,10 +87,9 @@ void GetThreadStackTopAndBottom(bool, uptr *stack_top, uptr *stack_bottom) {
 }
 void InitializePlatformEarly() {}
-void MaybeReexec() {}
 void CheckASLR() {}
 void CheckMPROTECT() {}
-void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {}
+void PlatformPrepareForSandboxing(void *args) {}
 void DisableCoreDumperIfNecessary() {}
 void InstallDeadlySignalHandlers(SignalHandlerType handler) {}
 void SetAlternateSignalStack() {}
 @@ -112,47 +112,6 @@ void FutexWake(atomic_uint32_t *p, u32 count) {
   CHECK_EQ(status, ZX_OK);
 }
-enum MutexState : int { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 };
-BlockingMutex::BlockingMutex() {
-  // NOTE!  It's important that this use internal_memset, because plain
-  // memset might be intercepted (e.g., actually be __asan_memset).
-  // Defining this so the compiler initializes each field, e.g.:
-  //   BlockingMutex::BlockingMutex() : BlockingMutex(LINKER_INITIALIZED) {}
-  // might result in the compiler generating a call to memset, which would
-  // have the same problem.
-  internal_memset(this, 0, sizeof(*this));
-}
-void BlockingMutex::Lock() {
-  CHECK_EQ(owner_, 0);
-  atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
-  if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked)
-    return;
-  while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) {
-    zx_status_t status =
-        _zx_futex_wait(reinterpret_cast<zx_futex_t *>(m), MtxSleeping,
-                       ZX_HANDLE_INVALID, ZX_TIME_INFINITE);
-    if (status != ZX_ERR_BAD_STATE)  // Normal race.
-      CHECK_EQ(status, ZX_OK);
-  }
-}
-void BlockingMutex::Unlock() {
-  atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_);
-  u32 v = atomic_exchange(m, MtxUnlocked, memory_order_release);
-  CHECK_NE(v, MtxUnlocked);
-  if (v == MtxSleeping) {
-    zx_status_t status = _zx_futex_wake(reinterpret_cast<zx_futex_t *>(m), 1);
-    CHECK_EQ(status, ZX_OK);
-  }
-}
-void BlockingMutex::CheckLocked() const {
-  auto m = reinterpret_cast<atomic_uint32_t const *>(&opaque_storage_);
-  CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
-}
 uptr GetPageSize() { return _zx_system_get_page_size(); }
 uptr GetMmapGranularity() { return _zx_system_get_page_size(); }
 @@ -168,6 +127,8 @@ uptr GetMaxUserVirtualAddress() {
 uptr GetMaxVirtualAddress() { return GetMaxUserVirtualAddress(); }
+bool ErrorIsOOM(error_t err) { return err == ZX_ERR_NO_MEMORY; }
 static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type,
                                   bool raw_report, bool die_for_nomem) {
   size = RoundUpTo(size, GetPageSize());
 @@ -315,6 +276,21 @@ void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) {
   UNIMPLEMENTED();
 }
+bool MprotectNoAccess(uptr addr, uptr size) {
+  return _zx_vmar_protect(_zx_vmar_root_self(), 0, addr, size) == ZX_OK;
+}
+bool MprotectReadOnly(uptr addr, uptr size) {
+  return _zx_vmar_protect(_zx_vmar_root_self(), ZX_VM_PERM_READ, addr, size) ==
+         ZX_OK;
+}
+bool MprotectReadWrite(uptr addr, uptr size) {
+  return _zx_vmar_protect(_zx_vmar_root_self(),
+                          ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, addr,
+                          size) == ZX_OK;
+}
 void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
                                    const char *mem_type) {
   CHECK_GE(size, GetPageSize());
 @@ -413,33 +389,12 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) {
 }
 // FIXME implement on this platform.
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {}
+void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
 bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
                       uptr *read_len, uptr max_len, error_t *errno_p) {
-  zx_handle_t vmo;
+  *errno_p = ZX_ERR_NOT_SUPPORTED;
-  zx_status_t status = __sanitizer_get_configuration(file_name, &vmo);
+  return false;
-  if (status == ZX_OK) {
-    uint64_t vmo_size;
-    status = _zx_vmo_get_size(vmo, &vmo_size);
-    if (status == ZX_OK) {
-      if (vmo_size < max_len)
-        max_len = vmo_size;
-      size_t map_size = RoundUpTo(max_len, GetPageSize());
-      uintptr_t addr;
-      status = _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ, 0, vmo, 0,
-                            map_size, &addr);
-      if (status == ZX_OK) {
-        *buff = reinterpret_cast<char *>(addr);
-        *buff_size = map_size;
-        *read_len = max_len;
-      }
-    }
-    _zx_handle_close(vmo);
-  }
-  if (status != ZX_OK && errno_p)
-    *errno_p = status;
-  return status == ZX_OK;
 }
 void RawWrite(const char *buffer) {
 @@ -516,6 +471,9 @@ u32 GetNumberOfCPUs() { return zx_system_get_num_cpus(); }
 uptr GetRSS() { UNIMPLEMENTED(); }
+void *internal_start_thread(void *(*func)(void *arg), void *arg) { return 0; }
+void internal_join_thread(void *th) {}
 void InitializePlatformCommonFlags(CommonFlags *cf) {}
 }  // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_hash.h+24

...	@@ -38,6 +38,30 @@ class MurMur2HashBuilder {	...	@@ -38,6 +38,30 @@ class MurMur2HashBuilder {
38	return x;	38	return x;
39	}	39	}
40	};	40	};
		41
		42	class MurMur2Hash64Builder {
		43	static const u64 m = 0xc6a4a7935bd1e995ull;
		44	static const u64 seed = 0x9747b28c9747b28cull;
		45	static const u64 r = 47;
		46	u64 h;
		47
		48	public:
		49	explicit MurMur2Hash64Builder(u64 init = 0) { h = seed ^ (init * m); }
		50	void add(u64 k) {
		51	k *= m;
		52	k ^= k >> r;
		53	k *= m;
		54	h ^= k;
		55	h *= m;
		56	}
		57	u64 get() {
		58	u64 x = h;
		59	x ^= x >> r;
		60	x *= m;
		61	x ^= x >> r;
		62	return x;
		63	}
		64	};
41	} //namespace __sanitizer	65	} //namespace __sanitizer
42		66
43	#endif // SANITIZER_HASH_H	67	#endif // SANITIZER_HASH_H

lib/tsan/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc+1-3

lib/tsan/sanitizer_common/sanitizer_interface_internal.h+121-90

...	@@ -20,103 +20,134 @@	...	@@ -20,103 +20,134 @@
20	#include "sanitizer_internal_defs.h"	20	#include "sanitizer_internal_defs.h"
21		21
22	extern "C" {	22	extern "C" {
23	// Tell the tools to write their reports to "path.<pid>" instead of stderr.	23	// Tell the tools to write their reports to "path.<pid>" instead of stderr.
24	// The special values are "stdout" and "stderr".	24	// The special values are "stdout" and "stderr".
25	SANITIZER_INTERFACE_ATTRIBUTE	25	SANITIZER_INTERFACE_ATTRIBUTE
26	void __sanitizer_set_report_path(const char *path);	26	void __sanitizer_set_report_path(const char *path);
27	// Tell the tools to write their reports to the provided file descriptor	27	// Tell the tools to write their reports to the provided file descriptor
28	// (casted to void *).	28	// (casted to void *).
29	SANITIZER_INTERFACE_ATTRIBUTE	29	SANITIZER_INTERFACE_ATTRIBUTE
30	void __sanitizer_set_report_fd(void *fd);	30	void __sanitizer_set_report_fd(void *fd);
31	// Get the current full report file path, if a path was specified by	31	// Get the current full report file path, if a path was specified by
32	// an earlier call to __sanitizer_set_report_path. Returns null otherwise.	32	// an earlier call to __sanitizer_set_report_path. Returns null otherwise.
33	SANITIZER_INTERFACE_ATTRIBUTE	33	SANITIZER_INTERFACE_ATTRIBUTE
34	const char *__sanitizer_get_report_path();	34	const char *__sanitizer_get_report_path();
35		35
36	typedef struct {	36	typedef struct {
37	int coverage_sandboxed;	37	int coverage_sandboxed;
38	__sanitizer::sptr coverage_fd;	38	__sanitizer::sptr coverage_fd;
39	unsigned int coverage_max_block_size;	39	unsigned int coverage_max_block_size;
40	} __sanitizer_sandbox_arguments;	40	} __sanitizer_sandbox_arguments;
41		41
42	// Notify the tools that the sandbox is going to be turned on.	42	// Notify the tools that the sandbox is going to be turned on.
43	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void	43	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
44	__sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args);	44	__sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args);
45		45
46	// This function is called by the tool when it has just finished reporting	46	// This function is called by the tool when it has just finished reporting
47	// an error. 'error_summary' is a one-line string that summarizes	47	// an error. 'error_summary' is a one-line string that summarizes
48	// the error message. This function can be overridden by the client.	48	// the error message. This function can be overridden by the client.
49	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	49	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
50	void __sanitizer_report_error_summary(const char *error_summary);	50	__sanitizer_report_error_summary(const char *error_summary);
51		51
52	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump();	52	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump();
53	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage(	53	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_coverage(
54	const __sanitizer::uptr *pcs, const __sanitizer::uptr len);	54	const __sanitizer::uptr *pcs, const __sanitizer::uptr len);
55	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage();	55	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_dump_trace_pc_guard_coverage();
56		56
57	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(__sanitizer::u32 *guard);	57	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(__sanitizer::u32 *guard);
58		58
59	// Returns 1 on the first call, then returns 0 thereafter. Called by the tool	59	// Returns 1 on the first call, then returns 0 thereafter. Called by the tool
60	// to ensure only one report is printed when multiple errors occur	60	// to ensure only one report is printed when multiple errors occur
61	// simultaneously.	61	// simultaneously.
62	SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_acquire_crash_state();	62	SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_acquire_crash_state();
63		63
64	SANITIZER_INTERFACE_ATTRIBUTE	64	SANITIZER_INTERFACE_ATTRIBUTE
65	void __sanitizer_annotate_contiguous_container(const void *beg,	65	void __sanitizer_annotate_contiguous_container(const void beg, const void end,
66	const void *end,	66	const void *old_mid,
67	const void *old_mid,	67	const void *new_mid);
68	const void *new_mid);	68	SANITIZER_INTERFACE_ATTRIBUTE
69	SANITIZER_INTERFACE_ATTRIBUTE	69	void __sanitizer_annotate_double_ended_contiguous_container(
70	int __sanitizer_verify_contiguous_container(const void beg, const void mid,	70	const void storage_beg, const void storage_end,
71	const void *end);	71	const void old_container_beg, const void old_container_end,
72	SANITIZER_INTERFACE_ATTRIBUTE	72	const void new_container_beg, const void new_container_end);
73	const void *__sanitizer_contiguous_container_find_bad_address(	73	SANITIZER_INTERFACE_ATTRIBUTE
74	const void beg, const void mid, const void *end);	74	int __sanitizer_verify_contiguous_container(const void beg, const void mid,
		75	const void *end);
		76	SANITIZER_INTERFACE_ATTRIBUTE
		77	int __sanitizer_verify_double_ended_contiguous_container(
		78	const void storage_beg, const void container_beg,
		79	const void container_end, const void storage_end);
		80	SANITIZER_INTERFACE_ATTRIBUTE
		81	const void __sanitizer_contiguous_container_find_bad_address(const void beg,
		82	const void *mid,
		83	const void *end);
		84	SANITIZER_INTERFACE_ATTRIBUTE
		85	const void *__sanitizer_double_ended_contiguous_container_find_bad_address(
		86	const void storage_beg, const void container_beg,
		87	const void container_end, const void storage_end);
75		88
76	SANITIZER_INTERFACE_ATTRIBUTE	89	SANITIZER_INTERFACE_ATTRIBUTE
77	int __sanitizer_get_module_and_offset_for_pc(	90	int __sanitizer_get_module_and_offset_for_pc(void pc, char module_path,
78	__sanitizer::uptr pc, char *module_path,	91	__sanitizer::uptr module_path_len,
79	__sanitizer::uptr module_path_len, __sanitizer::uptr *pc_offset);	92	void **pc_offset);
80		93	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
81	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	94	__sanitizer_cov_trace_cmp();
82	void __sanitizer_cov_trace_cmp();	95	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
83	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	96	__sanitizer_cov_trace_cmp1();
84	void __sanitizer_cov_trace_cmp1();	97	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
85	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	98	__sanitizer_cov_trace_cmp2();
86	void __sanitizer_cov_trace_cmp2();	99	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
87	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	100	__sanitizer_cov_trace_cmp4();
88	void __sanitizer_cov_trace_cmp4();	101	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
89	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	102	__sanitizer_cov_trace_cmp8();
90	void __sanitizer_cov_trace_cmp8();	103	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
91	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	104	__sanitizer_cov_trace_const_cmp1();
92	void __sanitizer_cov_trace_const_cmp1();	105	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
93	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	106	__sanitizer_cov_trace_const_cmp2();
94	void __sanitizer_cov_trace_const_cmp2();	107	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
95	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	108	__sanitizer_cov_trace_const_cmp4();
96	void __sanitizer_cov_trace_const_cmp4();	109	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
97	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	110	__sanitizer_cov_trace_const_cmp8();
98	void __sanitizer_cov_trace_const_cmp8();	111	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
99	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	112	__sanitizer_cov_trace_switch();
100	void __sanitizer_cov_trace_switch();	113	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
101	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	114	__sanitizer_cov_trace_div4();
102	void __sanitizer_cov_trace_div4();	115	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
103	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	116	__sanitizer_cov_trace_div8();
104	void __sanitizer_cov_trace_div8();	117	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
105	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	118	__sanitizer_cov_trace_gep();
106	void __sanitizer_cov_trace_gep();	119	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
107	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	120	__sanitizer_cov_trace_pc_indir();
108	void __sanitizer_cov_trace_pc_indir();	121	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
109	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	122	__sanitizer_cov_load1();
110	void __sanitizer_cov_trace_pc_guard(__sanitizer::u32*);	123	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
111	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	124	__sanitizer_cov_load2();
112	void __sanitizer_cov_trace_pc_guard_init(__sanitizer::u32*,	125	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
113	__sanitizer::u32*);	126	__sanitizer_cov_load4();
114	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	127	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
115	void __sanitizer_cov_8bit_counters_init();	128	__sanitizer_cov_load8();
116	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void	129	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
117	__sanitizer_cov_bool_flag_init();	130	__sanitizer_cov_load16();
118	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void	131	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
119	__sanitizer_cov_pcs_init();	132	__sanitizer_cov_store1();
120	} // extern "C"	133	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
		134	__sanitizer_cov_store2();
		135	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
		136	__sanitizer_cov_store4();
		137	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
		138	__sanitizer_cov_store8();
		139	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
		140	__sanitizer_cov_store16();
		141	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
		142	__sanitizer_cov_trace_pc_guard(__sanitizer::u32 *);
		143	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
		144	__sanitizer_cov_trace_pc_guard_init(__sanitizer::u32 , __sanitizer::u32 );
		145	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
		146	__sanitizer_cov_8bit_counters_init(char , char );
		147	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
		148	__sanitizer_cov_bool_flag_init();
		149	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
		150	__sanitizer_cov_pcs_init(const __sanitizer::uptr , const __sanitizer::uptr );
		151	} // extern "C"
121		152
122	#endif // SANITIZER_INTERFACE_INTERNAL_H	153	#endif // SANITIZER_INTERFACE_INTERNAL_H

lib/tsan/sanitizer_common/sanitizer_internal_defs.h+56-35

 @@ -13,6 +13,7 @@
 #define SANITIZER_DEFS_H
 #include "sanitizer_platform.h"
+#include "sanitizer_redefine_builtins.h"
 #ifndef SANITIZER_DEBUG
 # define SANITIZER_DEBUG 0
 @@ -37,15 +38,6 @@
 # define SANITIZER_WEAK_ATTRIBUTE  __attribute__((weak))
 #endif
-// TLS is handled differently on different platforms
-#if SANITIZER_LINUX || SANITIZER_NETBSD || \
-  SANITIZER_FREEBSD
-# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE \
-    __attribute__((tls_model("initial-exec"))) thread_local
-#else
-# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE
-#endif
 //--------------------------- WEAK FUNCTIONS ---------------------------------//
 // When working with weak functions, to simplify the code and make it more
 // portable, when possible define a default implementation using this macro:
 @@ -73,7 +65,7 @@
 // Before Xcode 4.5, the Darwin linker doesn't reliably support undefined
 // weak symbols.  Mac OS X 10.9/Darwin 13 is the first release only supported
 // by Xcode >= 4.5.
-#elif SANITIZER_MAC && \
+#elif SANITIZER_APPLE && \
     __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1090 && !SANITIZER_GO
 # define SANITIZER_SUPPORTS_WEAK_HOOKS 1
 #else
 @@ -125,6 +117,10 @@
 # define __has_attribute(x) 0
 #endif
+#if !defined(__has_cpp_attribute)
+#  define __has_cpp_attribute(x) 0
+#endif
 // For portability reasons we do not include stddef.h, stdint.h or any other
 // system header, but we do need some basic types that are not defined
 // in a portable way by the language itself.
 @@ -135,8 +131,13 @@ namespace __sanitizer {
 typedef unsigned long long uptr;
 typedef signed long long sptr;
 #else
+#  if (SANITIZER_WORDSIZE == 64) || SANITIZER_APPLE || SANITIZER_WINDOWS
 typedef unsigned long uptr;
 typedef signed long sptr;
+#  else
+typedef unsigned int uptr;
+typedef signed int sptr;
+#  endif
 #endif  // defined(_WIN64)
 #if defined(__x86_64__)
 // Since x32 uses ILP32 data model in 64-bit hardware mode, we must use
 @@ -168,17 +169,17 @@ typedef long pid_t;
 typedef int pid_t;
 #endif
-#if SANITIZER_FREEBSD || SANITIZER_NETBSD || \
+#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_APPLE ||             \
-    SANITIZER_MAC || \
     (SANITIZER_SOLARIS && (defined(_LP64) || _FILE_OFFSET_BITS == 64)) || \
-    (SANITIZER_LINUX && defined(__x86_64__))
+    (SANITIZER_LINUX && !SANITIZER_GLIBC && !SANITIZER_ANDROID) ||        \
+    (SANITIZER_LINUX && (defined(__x86_64__) || defined(__hexagon__)))
 typedef u64 OFF_T;
 #else
 typedef uptr OFF_T;
 #endif
 typedef u64  OFF64_T;
-#if (SANITIZER_WORDSIZE == 64) || SANITIZER_MAC
+#if (SANITIZER_WORDSIZE == 64) || SANITIZER_APPLE
 typedef uptr operator_new_size_type;
 #else
 # if defined(__s390__) && !defined(__s390x__)
 @@ -217,7 +218,7 @@ typedef u64 tid_t;
 # define WARN_UNUSED_RESULT
 #else  // _MSC_VER
 # define ALWAYS_INLINE inline __attribute__((always_inline))
-# define ALIAS(x) __attribute__((alias(x)))
+# define ALIAS(x) __attribute__((alias(SANITIZER_STRINGIFY(x))))
 // Please only use the ALIGNED macro before the type.
 // Using ALIGNED after the variable declaration is not portable!
 # define ALIGNED(x) __attribute__((aligned(x)))
 @@ -250,6 +251,20 @@ typedef u64 tid_t;
 # define NOEXCEPT throw()
 #endif
+#if __has_cpp_attribute(clang::fallthrough)
+#  define FALLTHROUGH [[clang::fallthrough]]
+#elif __has_cpp_attribute(fallthrough)
+#  define FALLTHROUGH [[fallthrough]]
+#else
+#  define FALLTHROUGH
+#endif
+#if __has_attribute(uninitialized)
+#  define UNINITIALIZED __attribute__((uninitialized))
+#else
+#  define UNINITIALIZED
+#endif
 // Unaligned versions of basic types.
 typedef ALIGNED(1) u16 uu16;
 typedef ALIGNED(1) u32 uu32;
 @@ -277,14 +292,17 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond,
                           u64 v1, u64 v2);
 // Check macro
-#define RAW_CHECK_MSG(expr, msg) do { \
+#define RAW_CHECK_MSG(expr, msg, ...)          \
-  if (UNLIKELY(!(expr))) { \
+  do {                                         \
-    RawWrite(msg); \
+    if (UNLIKELY(!(expr))) {                   \
-    Die(); \
+      const char* msgs[] = {msg, __VA_ARGS__}; \
-  } \
+      for (const char* m : msgs) RawWrite(m);  \
-} while (0)
+      Die();                                   \
+    }                                          \
+  } while (0)
-#define RAW_CHECK(expr) RAW_CHECK_MSG(expr, #expr)
+#define RAW_CHECK(expr) RAW_CHECK_MSG(expr, #expr "\n", )
+#define RAW_CHECK_VA(expr, ...) RAW_CHECK_MSG(expr, #expr "\n", __VA_ARGS__)
 #define CHECK_IMPL(c1, op, c2) \
   do { \
 @@ -366,13 +384,10 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond,
 enum LinkerInitialized { LINKER_INITIALIZED = 0 };
 #if !defined(_MSC_VER) || defined(__clang__)
-#if SANITIZER_S390_31
+#  define GET_CALLER_PC()                              \
-#define GET_CALLER_PC() \
+    ((__sanitizer::uptr)__builtin_extract_return_addr( \
-  (__sanitizer::uptr) __builtin_extract_return_addr(__builtin_return_address(0))
+        __builtin_return_address(0)))
-#else
+#  define GET_CURRENT_FRAME() ((__sanitizer::uptr)__builtin_frame_address(0))
-#define GET_CALLER_PC() (__sanitizer::uptr) __builtin_return_address(0)
-#endif
-#define GET_CURRENT_FRAME() (__sanitizer::uptr) __builtin_frame_address(0)
 inline void Trap() {
   __builtin_trap();
 }
 @@ -381,13 +396,13 @@ extern "C" void* _ReturnAddress(void);
 extern "C" void* _AddressOfReturnAddress(void);
 # pragma intrinsic(_ReturnAddress)
 # pragma intrinsic(_AddressOfReturnAddress)
-#define GET_CALLER_PC() (__sanitizer::uptr) _ReturnAddress()
+#  define GET_CALLER_PC() ((__sanitizer::uptr)_ReturnAddress())
 // CaptureStackBackTrace doesn't need to know BP on Windows.
-#define GET_CURRENT_FRAME() \
+#  define GET_CURRENT_FRAME() \
-  (((__sanitizer::uptr)_AddressOfReturnAddress()) + sizeof(__sanitizer::uptr))
+    (((__sanitizer::uptr)_AddressOfReturnAddress()) + sizeof(__sanitizer::uptr))
 extern "C" void __ud2(void);
-# pragma intrinsic(__ud2)
+#  pragma intrinsic(__ud2)
 inline void Trap() {
   __ud2();
 }
 @@ -409,8 +424,14 @@ inline void Trap() {
     (void)enable_fp;                      \
   } while (0)
-constexpr u32 kInvalidTid = -1;
+// Internal thread identifier allocated by ThreadRegistry.
-constexpr u32 kMainTid = 0;
+typedef u32 Tid;
+constexpr Tid kInvalidTid = -1;
+constexpr Tid kMainTid = 0;
+// Stack depot stack identifier.
+typedef u32 StackID;
+const StackID kInvalidStackID = 0;
 }  // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_leb128.h created+87

...	@@ -0,0 +1,87 @@
1	//===-- sanitizer_leb128.h --------------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#ifndef SANITIZER_LEB128_H
10	#define SANITIZER_LEB128_H
11
12	#include "sanitizer_common.h"
13	#include "sanitizer_internal_defs.h"
14
15	namespace __sanitizer {
16
17	template <typename T, typename It>
18	It EncodeSLEB128(T value, It begin, It end) {
19	bool more;
20	do {
21	u8 byte = value & 0x7f;
22	// NOTE: this assumes that this signed shift is an arithmetic right shift.
23	value >>= 7;
24	more = !((((value == 0) && ((byte & 0x40) == 0)) \|\|
25	((value == -1) && ((byte & 0x40) != 0))));
26	if (more)
27	byte \|= 0x80;
28	if (UNLIKELY(begin == end))
29	break;
30	*(begin++) = byte;
31	} while (more);
32	return begin;
33	}
34
35	template <typename T, typename It>
36	It DecodeSLEB128(It begin, It end, T* v) {
37	T value = 0;
38	unsigned shift = 0;
39	u8 byte;
40	do {
41	if (UNLIKELY(begin == end))
42	return begin;
43	byte = *(begin++);
44	T slice = byte & 0x7f;
45	value \|= slice << shift;
46	shift += 7;
47	} while (byte >= 128);
48	if (shift < 64 && (byte & 0x40))
49	value \|= (-1ULL) << shift;
50	*v = value;
51	return begin;
52	}
53
54	template <typename T, typename It>
55	It EncodeULEB128(T value, It begin, It end) {
56	do {
57	u8 byte = value & 0x7f;
58	value >>= 7;
59	if (value)
60	byte \|= 0x80;
61	if (UNLIKELY(begin == end))
62	break;
63	*(begin++) = byte;
64	} while (value);
65	return begin;
66	}
67
68	template <typename T, typename It>
69	It DecodeULEB128(It begin, It end, T* v) {
70	T value = 0;
71	unsigned shift = 0;
72	u8 byte;
73	do {
74	if (UNLIKELY(begin == end))
75	return begin;
76	byte = *(begin++);
77	T slice = byte & 0x7f;
78	value += slice << shift;
79	shift += 7;
80	} while (byte >= 128);
81	*v = value;
82	return begin;
83	}
84
85	} // namespace __sanitizer
86
87	#endif // SANITIZER_LEB128_H

lib/tsan/sanitizer_common/sanitizer_libc.cpp+24-3

 @@ -10,6 +10,9 @@
 // run-time libraries. See sanitizer_libc.h for details.
 //===----------------------------------------------------------------------===//
+// Do not redefine builtins; this file is defining the builtin replacements.
+#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS
 #include "sanitizer_allocator_internal.h"
 #include "sanitizer_common.h"
 #include "sanitizer_libc.h"
 @@ -46,7 +49,10 @@ int internal_memcmp(const void* s1, const void* s2, uptr n) {
   return 0;
 }
-void *internal_memcpy(void *dest, const void *src, uptr n) {
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memcpy(void *dest,
+                                                                const void *src,
+                                                                uptr n) {
   char *d = (char*)dest;
   const char *s = (const char *)src;
   for (uptr i = 0; i < n; ++i)
 @@ -54,7 +60,8 @@ void *internal_memcpy(void *dest, const void *src, uptr n) {
   return dest;
 }
-void *internal_memmove(void *dest, const void *src, uptr n) {
+SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memmove(
+    void *dest, const void *src, uptr n) {
   char *d = (char*)dest;
   const char *s = (const char *)src;
   sptr i, signed_n = (sptr)n;
 @@ -72,7 +79,8 @@ void *internal_memmove(void *dest, const void *src, uptr n) {
   return dest;
 }
-void *internal_memset(void* s, int c, uptr n) {
+SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memset(void *s, int c,
+                                                                uptr n) {
   // Optimize for the most performance-critical case:
   if ((reinterpret_cast<uptr>(s) % 16) == 0 && (n % 16) == 0) {
     u64 *p = reinterpret_cast<u64*>(s);
 @@ -95,6 +103,7 @@ void *internal_memset(void* s, int c, uptr n) {
   }
   return s;
 }
+}  // extern "C"
 uptr internal_strcspn(const char *s, const char *reject) {
   uptr i;
 @@ -258,6 +267,18 @@ s64 internal_simple_strtoll(const char *nptr, const char **endptr, int base) {
   }
 }
+uptr internal_wcslen(const wchar_t *s) {
+  uptr i = 0;
+  while (s[i]) i++;
+  return i;
+}
+uptr internal_wcsnlen(const wchar_t *s, uptr maxlen) {
+  uptr i = 0;
+  while (i < maxlen && s[i]) i++;
+  return i;
+}
 bool mem_is_zero(const char *beg, uptr size) {
   CHECK_LE(size, 1ULL << FIRST_32_SECOND_64(30, 40));  // Sanity check.
   const char *end = beg + size;

lib/tsan/sanitizer_common/sanitizer_libc.h+25-4

...	@@ -24,15 +24,33 @@ namespace __sanitizer {	...	@@ -24,15 +24,33 @@ namespace __sanitizer {
24		24
25	// internal_X() is a custom implementation of X() for use in RTL.	25	// internal_X() is a custom implementation of X() for use in RTL.
26		26
		27	extern "C" {
		28	// These are used as builtin replacements; see sanitizer_redefine_builtins.h.
		29	// In normal runtime code, use the __sanitizer::internal_X() aliases instead.
		30	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_internal_memcpy(void dest,
		31	const void *src,
		32	uptr n);
		33	SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memmove(
		34	void dest, const void src, uptr n);
		35	SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_internal_memset(void s, int c,
		36	uptr n);
		37	} // extern "C"
		38
27	// String functions	39	// String functions
28	s64 internal_atoll(const char *nptr);	40	s64 internal_atoll(const char *nptr);
29	void internal_memchr(const void s, int c, uptr n);	41	void internal_memchr(const void s, int c, uptr n);
30	void internal_memrchr(const void s, int c, uptr n);	42	void internal_memrchr(const void s, int c, uptr n);
31	int internal_memcmp(const void* s1, const void* s2, uptr n);	43	int internal_memcmp(const void* s1, const void* s2, uptr n);
32	void internal_memcpy(void dest, const void *src, uptr n);	44	ALWAYS_INLINE void internal_memcpy(void dest, const void *src, uptr n) {
33	void internal_memmove(void dest, const void *src, uptr n);	45	return __sanitizer_internal_memcpy(dest, src, n);
		46	}
		47	ALWAYS_INLINE void internal_memmove(void dest, const void *src, uptr n) {
		48	return __sanitizer_internal_memmove(dest, src, n);
		49	}
34	// Should not be used in performance-critical places.	50	// Should not be used in performance-critical places.
35	void internal_memset(void s, int c, uptr n);	51	ALWAYS_INLINE void internal_memset(void s, int c, uptr n) {
		52	return __sanitizer_internal_memset(s, c, n);
		53	}
36	char* internal_strchr(const char *s, int c);	54	char* internal_strchr(const char *s, int c);
37	char internal_strchrnul(const char s, int c);	55	char internal_strchrnul(const char s, int c);
38	int internal_strcmp(const char s1, const char s2);	56	int internal_strcmp(const char s1, const char s2);
...	@@ -49,7 +67,10 @@ char internal_strrchr(const char s, int c);	...	@@ -49,7 +67,10 @@ char internal_strrchr(const char s, int c);
49	char internal_strstr(const char haystack, const char *needle);	67	char internal_strstr(const char haystack, const char *needle);
50	// Works only for base=10 and doesn't set errno.	68	// Works only for base=10 and doesn't set errno.
51	s64 internal_simple_strtoll(const char nptr, const char *endptr, int base);	69	s64 internal_simple_strtoll(const char nptr, const char *endptr, int base);
52	int internal_snprintf(char buffer, uptr length, const char format, ...);	70	int internal_snprintf(char buffer, uptr length, const char format, ...)
		71	FORMAT(3, 4);
		72	uptr internal_wcslen(const wchar_t *s);
		73	uptr internal_wcsnlen(const wchar_t *s, uptr maxlen);
53		74
54	// Return true if all bytes in [mem, mem+size) are zero.	75	// Return true if all bytes in [mem, mem+size) are zero.
55	// Optimized for the case when the result is true.	76	// Optimized for the case when the result is true.

lib/tsan/sanitizer_common/sanitizer_libignore.cpp+6-6

...	@@ -8,7 +8,7 @@	...	@@ -8,7 +8,7 @@
8		8
9	#include "sanitizer_platform.h"	9	#include "sanitizer_platform.h"
10		10
11	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_MAC \|\| \	11	#if SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_APPLE \|\| \
12	SANITIZER_NETBSD	12	SANITIZER_NETBSD
13		13
14	#include "sanitizer_libignore.h"	14	#include "sanitizer_libignore.h"
...	@@ -22,9 +22,9 @@ LibIgnore::LibIgnore(LinkerInitialized) {	...	@@ -22,9 +22,9 @@ LibIgnore::LibIgnore(LinkerInitialized) {
22	}	22	}
23		23
24	void LibIgnore::AddIgnoredLibrary(const char *name_templ) {	24	void LibIgnore::AddIgnoredLibrary(const char *name_templ) {
25	BlockingMutexLock lock(&mutex_);	25	Lock lock(&mutex_);
26	if (count_ >= kMaxLibs) {	26	if (count_ >= kMaxLibs) {
27	Report("%s: too many ignored libraries (max: %d)\n", SanitizerToolName,	27	Report("%s: too many ignored libraries (max: %zu)\n", SanitizerToolName,
28	kMaxLibs);	28	kMaxLibs);
29	Die();	29	Die();
30	}	30	}
...	@@ -36,7 +36,7 @@ void LibIgnore::AddIgnoredLibrary(const char *name_templ) {	...	@@ -36,7 +36,7 @@ void LibIgnore::AddIgnoredLibrary(const char *name_templ) {
36	}	36	}
37		37
38	void LibIgnore::OnLibraryLoaded(const char *name) {	38	void LibIgnore::OnLibraryLoaded(const char *name) {
39	BlockingMutexLock lock(&mutex_);	39	Lock lock(&mutex_);
40	// Try to match suppressions with symlink target.	40	// Try to match suppressions with symlink target.
41	InternalMmapVector<char> buf(kMaxPathLength);	41	InternalMmapVector<char> buf(kMaxPathLength);
42	if (name && internal_readlink(name, buf.data(), buf.size() - 1) > 0 &&	42	if (name && internal_readlink(name, buf.data(), buf.size() - 1) > 0 &&
...	@@ -105,7 +105,7 @@ void LibIgnore::OnLibraryLoaded(const char *name) {	...	@@ -105,7 +105,7 @@ void LibIgnore::OnLibraryLoaded(const char *name) {
105	continue;	105	continue;
106	if (IsPcInstrumented(range.beg) && IsPcInstrumented(range.end - 1))	106	if (IsPcInstrumented(range.beg) && IsPcInstrumented(range.end - 1))
107	continue;	107	continue;
108	VReport(1, "Adding instrumented range %p-%p from library '%s'\n",	108	VReport(1, "Adding instrumented range 0x%zx-0x%zx from library '%s'\n",
109	range.beg, range.end, mod.full_name());	109	range.beg, range.end, mod.full_name());
110	const uptr idx =	110	const uptr idx =
111	atomic_load(&instrumented_ranges_count_, memory_order_relaxed);	111	atomic_load(&instrumented_ranges_count_, memory_order_relaxed);
...	@@ -125,5 +125,5 @@ void LibIgnore::OnLibraryUnloaded() {	...	@@ -125,5 +125,5 @@ void LibIgnore::OnLibraryUnloaded() {
125		125
126	} // namespace __sanitizer	126	} // namespace __sanitizer
127		127
128	#endif // SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_MAC \|\|	128	#endif // SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_APPLE \|\|
129	// SANITIZER_NETBSD	129	// SANITIZER_NETBSD

lib/tsan/sanitizer_common/sanitizer_libignore.h+1-1

...	@@ -77,7 +77,7 @@ class LibIgnore {	...	@@ -77,7 +77,7 @@ class LibIgnore {
77	LibCodeRange instrumented_code_ranges_[kMaxInstrumentedRanges];	77	LibCodeRange instrumented_code_ranges_[kMaxInstrumentedRanges];
78		78
79	// Cold part:	79	// Cold part:
80	BlockingMutex mutex_;	80	Mutex mutex_;
81	uptr count_;	81	uptr count_;
82	Lib libs_[kMaxLibs];	82	Lib libs_[kMaxLibs];
83	bool track_instrumented_libs_;	83	bool track_instrumented_libs_;

lib/tsan/sanitizer_common/sanitizer_linux.cpp+317-193

...	@@ -34,7 +34,7 @@	...	@@ -34,7 +34,7 @@
34	// format. Struct kernel_stat is defined as 'struct stat' in asm/stat.h. To	34	// format. Struct kernel_stat is defined as 'struct stat' in asm/stat.h. To
35	// access stat from asm/stat.h, without conflicting with definition in	35	// access stat from asm/stat.h, without conflicting with definition in
36	// sys/stat.h, we use this trick.	36	// sys/stat.h, we use this trick.
37	#if defined(__mips64)	37	#if SANITIZER_MIPS64
38	#include <asm/unistd.h>	38	#include <asm/unistd.h>
39	#include <sys/types.h>	39	#include <sys/types.h>
40	#define stat kernel_stat	40	#define stat kernel_stat
...	@@ -78,8 +78,13 @@	...	@@ -78,8 +78,13 @@
78	#include <sys/personality.h>	78	#include <sys/personality.h>
79	#endif	79	#endif
80		80
		81	#if SANITIZER_LINUX && defined(__loongarch__)
		82	# include <sys/sysmacros.h>
		83	#endif
		84
81	#if SANITIZER_FREEBSD	85	#if SANITIZER_FREEBSD
82	#include <sys/exec.h>	86	#include <sys/exec.h>
		87	#include <sys/procctl.h>
83	#include <sys/sysctl.h>	88	#include <sys/sysctl.h>
84	#include <machine/atomic.h>	89	#include <machine/atomic.h>
85	extern "C" {	90	extern "C" {
...	@@ -123,8 +128,9 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE \| FUTEX_PRIVATE_FLAG;	...	@@ -123,8 +128,9 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE \| FUTEX_PRIVATE_FLAG;
123	// Are we using 32-bit or 64-bit Linux syscalls?	128	// Are we using 32-bit or 64-bit Linux syscalls?
124	// x32 (which defines __x86_64__) has SANITIZER_WORDSIZE == 32	129	// x32 (which defines __x86_64__) has SANITIZER_WORDSIZE == 32
125	// but it still needs to use 64-bit syscalls.	130	// but it still needs to use 64-bit syscalls.
126	#if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__powerpc64__) \|\| \	131	#if SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__powerpc64__) \|\| \
127	SANITIZER_WORDSIZE == 64)	132	SANITIZER_WORDSIZE == 64 \|\| \
		133	(defined(__mips__) && _MIPS_SIM == _ABIN32))
128	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1	134	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1
129	#else	135	#else
130	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0	136	# define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0
...	@@ -150,17 +156,51 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE \| FUTEX_PRIVATE_FLAG;	...	@@ -150,17 +156,51 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE \| FUTEX_PRIVATE_FLAG;
150		156
151	namespace __sanitizer {	157	namespace __sanitizer {
152		158
153	#if SANITIZER_LINUX && defined(__x86_64__)	159	void SetSigProcMask(__sanitizer_sigset_t set, __sanitizer_sigset_t oldset) {
154	#include "sanitizer_syscall_linux_x86_64.inc"	160	CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, oldset));
155	#elif SANITIZER_LINUX && SANITIZER_RISCV64	161	}
156	#include "sanitizer_syscall_linux_riscv64.inc"	162
157	#elif SANITIZER_LINUX && defined(__aarch64__)	163	void BlockSignals(__sanitizer_sigset_t *oldset) {
158	#include "sanitizer_syscall_linux_aarch64.inc"	164	__sanitizer_sigset_t set;
159	#elif SANITIZER_LINUX && defined(__arm__)	165	internal_sigfillset(&set);
160	#include "sanitizer_syscall_linux_arm.inc"	166	# if SANITIZER_LINUX && !SANITIZER_ANDROID
161	#else	167	// Glibc uses SIGSETXID signal during setuid call. If this signal is blocked
162	#include "sanitizer_syscall_generic.inc"	168	// on any thread, setuid call hangs.
163	#endif	169	// See test/sanitizer_common/TestCases/Linux/setuid.c.
		170	internal_sigdelset(&set, 33);
		171	# endif
		172	# if SANITIZER_LINUX
		173	// Seccomp-BPF-sandboxed processes rely on SIGSYS to handle trapped syscalls.
		174	// If this signal is blocked, such calls cannot be handled and the process may
		175	// hang.
		176	internal_sigdelset(&set, 31);
		177	# endif
		178	SetSigProcMask(&set, oldset);
		179	}
		180
		181	ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) {
		182	BlockSignals(&saved_);
		183	if (copy)
		184	internal_memcpy(copy, &saved_, sizeof(saved_));
		185	}
		186
		187	ScopedBlockSignals::~ScopedBlockSignals() { SetSigProcMask(&saved_, nullptr); }
		188
		189	# if SANITIZER_LINUX && defined(__x86_64__)
		190	# include "sanitizer_syscall_linux_x86_64.inc"
		191	# elif SANITIZER_LINUX && SANITIZER_RISCV64
		192	# include "sanitizer_syscall_linux_riscv64.inc"
		193	# elif SANITIZER_LINUX && defined(__aarch64__)
		194	# include "sanitizer_syscall_linux_aarch64.inc"
		195	# elif SANITIZER_LINUX && defined(__arm__)
		196	# include "sanitizer_syscall_linux_arm.inc"
		197	# elif SANITIZER_LINUX && defined(__hexagon__)
		198	# include "sanitizer_syscall_linux_hexagon.inc"
		199	# elif SANITIZER_LINUX && SANITIZER_LOONGARCH64
		200	# include "sanitizer_syscall_linux_loongarch64.inc"
		201	# else
		202	# include "sanitizer_syscall_generic.inc"
		203	# endif
164		204
165	// --------------- sanitizer_libc.h	205	// --------------- sanitizer_libc.h
166	#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD	206	#if !SANITIZER_SOLARIS && !SANITIZER_NETBSD
...	@@ -204,7 +244,7 @@ uptr internal_close(fd_t fd) {	...	@@ -204,7 +244,7 @@ uptr internal_close(fd_t fd) {
204	}	244	}
205		245
206	uptr internal_open(const char *filename, int flags) {	246	uptr internal_open(const char *filename, int flags) {
207	#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS	247	# if SANITIZER_LINUX
208	return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags);	248	return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags);
209	#else	249	#else
210	return internal_syscall(SYSCALL(open), (uptr)filename, flags);	250	return internal_syscall(SYSCALL(open), (uptr)filename, flags);
...	@@ -212,7 +252,7 @@ uptr internal_open(const char *filename, int flags) {	...	@@ -212,7 +252,7 @@ uptr internal_open(const char *filename, int flags) {
212	}	252	}
213		253
214	uptr internal_open(const char *filename, int flags, u32 mode) {	254	uptr internal_open(const char *filename, int flags, u32 mode) {
215	#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS	255	# if SANITIZER_LINUX
216	return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags,	256	return internal_syscall(SYSCALL(openat), AT_FDCWD, (uptr)filename, flags,
217	mode);	257	mode);
218	#else	258	#else
...	@@ -241,7 +281,7 @@ uptr internal_ftruncate(fd_t fd, uptr size) {	...	@@ -241,7 +281,7 @@ uptr internal_ftruncate(fd_t fd, uptr size) {
241	return res;	281	return res;
242	}	282	}
243		283
244	#if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && SANITIZER_LINUX	284	#if (!SANITIZER_LINUX_USES_64BIT_SYSCALLS \|\| SANITIZER_SPARC) && SANITIZER_LINUX
245	static void stat64_to_stat(struct stat64 in, struct stat out) {	285	static void stat64_to_stat(struct stat64 in, struct stat out) {
246	internal_memset(out, 0, sizeof(*out));	286	internal_memset(out, 0, sizeof(*out));
247	out->st_dev = in->st_dev;	287	out->st_dev = in->st_dev;
...	@@ -260,7 +300,29 @@ static void stat64_to_stat(struct stat64 in, struct stat out) {	...	@@ -260,7 +300,29 @@ static void stat64_to_stat(struct stat64 in, struct stat out) {
260	}	300	}
261	#endif	301	#endif
262		302
263	#if defined(__mips64)	303	#if SANITIZER_LINUX && defined(__loongarch__)
		304	static void statx_to_stat(struct statx in, struct stat out) {
		305	internal_memset(out, 0, sizeof(*out));
		306	out->st_dev = makedev(in->stx_dev_major, in->stx_dev_minor);
		307	out->st_ino = in->stx_ino;
		308	out->st_mode = in->stx_mode;
		309	out->st_nlink = in->stx_nlink;
		310	out->st_uid = in->stx_uid;
		311	out->st_gid = in->stx_gid;
		312	out->st_rdev = makedev(in->stx_rdev_major, in->stx_rdev_minor);
		313	out->st_size = in->stx_size;
		314	out->st_blksize = in->stx_blksize;
		315	out->st_blocks = in->stx_blocks;
		316	out->st_atime = in->stx_atime.tv_sec;
		317	out->st_atim.tv_nsec = in->stx_atime.tv_nsec;
		318	out->st_mtime = in->stx_mtime.tv_sec;
		319	out->st_mtim.tv_nsec = in->stx_mtime.tv_nsec;
		320	out->st_ctime = in->stx_ctime.tv_sec;
		321	out->st_ctim.tv_nsec = in->stx_ctime.tv_nsec;
		322	}
		323	#endif
		324
		325	#if SANITIZER_MIPS64
264	// Undefine compatibility macros from <sys/stat.h>	326	// Undefine compatibility macros from <sys/stat.h>
265	// so that they would not clash with the kernel_stat	327	// so that they would not clash with the kernel_stat
266	// st_[a\|m\|c]time fields	328	// st_[a\|m\|c]time fields
...	@@ -311,52 +373,65 @@ static void kernel_stat_to_stat(struct kernel_stat in, struct stat out) {	...	@@ -311,52 +373,65 @@ static void kernel_stat_to_stat(struct kernel_stat in, struct stat out) {
311	#endif	373	#endif
312		374
313	uptr internal_stat(const char path, void buf) {	375	uptr internal_stat(const char path, void buf) {
314	#if SANITIZER_FREEBSD	376	# if SANITIZER_FREEBSD
315	return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0);	377	return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0);
316	#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS	378	# elif SANITIZER_LINUX
		379	# if defined(__loongarch__)
		380	struct statx bufx;
		381	int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path,
		382	AT_NO_AUTOMOUNT, STATX_BASIC_STATS, (uptr)&bufx);
		383	statx_to_stat(&bufx, (struct stat *)buf);
		384	return res;
		385	# elif (SANITIZER_WORDSIZE == 64 \|\| SANITIZER_X32 \|\| \
		386	(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \
		387	!SANITIZER_SPARC
317	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,	388	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
318	0);	389	0);
319	#elif SANITIZER_LINUX_USES_64BIT_SYSCALLS	390	# else
320	# if defined(__mips64)	391	struct stat64 buf64;
321	// For mips64, stat syscall fills buffer in the format of kernel_stat	392	int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path,
322	struct kernel_stat kbuf;	393	(uptr)&buf64, 0);
323	int res = internal_syscall(SYSCALL(stat), path, &kbuf);	394	stat64_to_stat(&buf64, (struct stat *)buf);
324	kernel_stat_to_stat(&kbuf, (struct stat *)buf);
325	return res;	395	return res;
326	# else	396	# endif
327	return internal_syscall(SYSCALL(stat), (uptr)path, (uptr)buf);	397	# else
328	# endif
329	#else
330	struct stat64 buf64;	398	struct stat64 buf64;
331	int res = internal_syscall(SYSCALL(stat64), path, &buf64);	399	int res = internal_syscall(SYSCALL(stat64), path, &buf64);
332	stat64_to_stat(&buf64, (struct stat *)buf);	400	stat64_to_stat(&buf64, (struct stat *)buf);
333	return res;	401	return res;
334	#endif	402	# endif
335	}	403	}
336		404
337	uptr internal_lstat(const char path, void buf) {	405	uptr internal_lstat(const char path, void buf) {
338	#if SANITIZER_FREEBSD	406	# if SANITIZER_FREEBSD
339	return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf,	407	return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf,
340	AT_SYMLINK_NOFOLLOW);	408	AT_SYMLINK_NOFOLLOW);
341	#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS	409	# elif SANITIZER_LINUX
		410	# if defined(__loongarch__)
		411	struct statx bufx;
		412	int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path,
		413	AT_SYMLINK_NOFOLLOW \| AT_NO_AUTOMOUNT,
		414	STATX_BASIC_STATS, (uptr)&bufx);
		415	statx_to_stat(&bufx, (struct stat *)buf);
		416	return res;
		417	# elif (defined(_LP64) \|\| SANITIZER_X32 \|\| \
		418	(defined(__mips__) && _MIPS_SIM == _ABIN32)) && \
		419	!SANITIZER_SPARC
342	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,	420	return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf,
343	AT_SYMLINK_NOFOLLOW);	421	AT_SYMLINK_NOFOLLOW);
344	#elif SANITIZER_LINUX_USES_64BIT_SYSCALLS	422	# else
345	# if SANITIZER_MIPS64	423	struct stat64 buf64;
346	// For mips64, lstat syscall fills buffer in the format of kernel_stat	424	int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path,
347	struct kernel_stat kbuf;	425	(uptr)&buf64, AT_SYMLINK_NOFOLLOW);
348	int res = internal_syscall(SYSCALL(lstat), path, &kbuf);	426	stat64_to_stat(&buf64, (struct stat *)buf);
349	kernel_stat_to_stat(&kbuf, (struct stat *)buf);
350	return res;	427	return res;
351	# else	428	# endif
352	return internal_syscall(SYSCALL(lstat), (uptr)path, (uptr)buf);	429	# else
353	# endif
354	#else
355	struct stat64 buf64;	430	struct stat64 buf64;
356	int res = internal_syscall(SYSCALL(lstat64), path, &buf64);	431	int res = internal_syscall(SYSCALL(lstat64), path, &buf64);
357	stat64_to_stat(&buf64, (struct stat *)buf);	432	stat64_to_stat(&buf64, (struct stat *)buf);
358	return res;	433	return res;
359	#endif	434	# endif
360	}	435	}
361		436
362	uptr internal_fstat(fd_t fd, void *buf) {	437	uptr internal_fstat(fd_t fd, void *buf) {
...	@@ -367,9 +442,15 @@ uptr internal_fstat(fd_t fd, void *buf) {	...	@@ -367,9 +442,15 @@ uptr internal_fstat(fd_t fd, void *buf) {
367	int res = internal_syscall(SYSCALL(fstat), fd, &kbuf);	442	int res = internal_syscall(SYSCALL(fstat), fd, &kbuf);
368	kernel_stat_to_stat(&kbuf, (struct stat *)buf);	443	kernel_stat_to_stat(&kbuf, (struct stat *)buf);
369	return res;	444	return res;
370	# else	445	# elif SANITIZER_LINUX && defined(__loongarch__)
		446	struct statx bufx;
		447	int res = internal_syscall(SYSCALL(statx), fd, "", AT_EMPTY_PATH,
		448	STATX_BASIC_STATS, (uptr)&bufx);
		449	statx_to_stat(&bufx, (struct stat *)buf);
		450	return res;
		451	# else
371	return internal_syscall(SYSCALL(fstat), fd, (uptr)buf);	452	return internal_syscall(SYSCALL(fstat), fd, (uptr)buf);
372	# endif	453	# endif
373	#else	454	#else
374	struct stat64 buf64;	455	struct stat64 buf64;
375	int res = internal_syscall(SYSCALL(fstat64), fd, &buf64);	456	int res = internal_syscall(SYSCALL(fstat64), fd, &buf64);
...	@@ -390,7 +471,7 @@ uptr internal_dup(int oldfd) {	...	@@ -390,7 +471,7 @@ uptr internal_dup(int oldfd) {
390	}	471	}
391		472
392	uptr internal_dup2(int oldfd, int newfd) {	473	uptr internal_dup2(int oldfd, int newfd) {
393	#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS	474	# if SANITIZER_LINUX
394	return internal_syscall(SYSCALL(dup3), oldfd, newfd, 0);	475	return internal_syscall(SYSCALL(dup3), oldfd, newfd, 0);
395	#else	476	#else
396	return internal_syscall(SYSCALL(dup2), oldfd, newfd);	477	return internal_syscall(SYSCALL(dup2), oldfd, newfd);
...	@@ -398,7 +479,7 @@ uptr internal_dup2(int oldfd, int newfd) {	...	@@ -398,7 +479,7 @@ uptr internal_dup2(int oldfd, int newfd) {
398	}	479	}
399		480
400	uptr internal_readlink(const char path, char buf, uptr bufsize) {	481	uptr internal_readlink(const char path, char buf, uptr bufsize) {
401	#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS	482	# if SANITIZER_LINUX
402	return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf,	483	return internal_syscall(SYSCALL(readlinkat), AT_FDCWD, (uptr)path, (uptr)buf,
403	bufsize);	484	bufsize);
404	#else	485	#else
...	@@ -407,7 +488,7 @@ uptr internal_readlink(const char path, char buf, uptr bufsize) {	...	@@ -407,7 +488,7 @@ uptr internal_readlink(const char path, char buf, uptr bufsize) {
407	}	488	}
408		489
409	uptr internal_unlink(const char *path) {	490	uptr internal_unlink(const char *path) {
410	#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS	491	# if SANITIZER_LINUX
411	return internal_syscall(SYSCALL(unlinkat), AT_FDCWD, (uptr)path, 0);	492	return internal_syscall(SYSCALL(unlinkat), AT_FDCWD, (uptr)path, 0);
412	#else	493	#else
413	return internal_syscall(SYSCALL(unlink), (uptr)path);	494	return internal_syscall(SYSCALL(unlink), (uptr)path);
...	@@ -415,15 +496,15 @@ uptr internal_unlink(const char *path) {	...	@@ -415,15 +496,15 @@ uptr internal_unlink(const char *path) {
415	}	496	}
416		497
417	uptr internal_rename(const char oldpath, const char newpath) {	498	uptr internal_rename(const char oldpath, const char newpath) {
418	#if defined(__riscv)	499	# if (defined(__riscv) \|\| defined(__loongarch__)) && defined(__linux__)
419	return internal_syscall(SYSCALL(renameat2), AT_FDCWD, (uptr)oldpath, AT_FDCWD,	500	return internal_syscall(SYSCALL(renameat2), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
420	(uptr)newpath, 0);	501	(uptr)newpath, 0);
421	#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS	502	# elif SANITIZER_LINUX
422	return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD,	503	return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD,
423	(uptr)newpath);	504	(uptr)newpath);
424	#else	505	# else
425	return internal_syscall(SYSCALL(rename), (uptr)oldpath, (uptr)newpath);	506	return internal_syscall(SYSCALL(rename), (uptr)oldpath, (uptr)newpath);
426	#endif	507	# endif
427	}	508	}
428		509
429	uptr internal_sched_yield() {	510	uptr internal_sched_yield() {
...	@@ -460,17 +541,20 @@ bool FileExists(const char *filename) {	...	@@ -460,17 +541,20 @@ bool FileExists(const char *filename) {
460	if (ShouldMockFailureToOpen(filename))	541	if (ShouldMockFailureToOpen(filename))
461	return false;	542	return false;
462	struct stat st;	543	struct stat st;
463	#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
464	if (internal_syscall(SYSCALL(newfstatat), AT_FDCWD, filename, &st, 0))
465	#else
466	if (internal_stat(filename, &st))	544	if (internal_stat(filename, &st))
467	#endif
468	return false;	545	return false;
469	// Sanity check: filename is a regular file.	546	// Sanity check: filename is a regular file.
470	return S_ISREG(st.st_mode);	547	return S_ISREG(st.st_mode);
471	}	548	}
472		549
473	#if !SANITIZER_NETBSD	550	bool DirExists(const char *path) {
		551	struct stat st;
		552	if (internal_stat(path, &st))
		553	return false;
		554	return S_ISDIR(st.st_mode);
		555	}
		556
		557	# if !SANITIZER_NETBSD
474	tid_t GetTid() {	558	tid_t GetTid() {
475	#if SANITIZER_FREEBSD	559	#if SANITIZER_FREEBSD
476	long Tid;	560	long Tid;
...	@@ -659,48 +743,6 @@ void FutexWake(atomic_uint32_t *p, u32 count) {	...	@@ -659,48 +743,6 @@ void FutexWake(atomic_uint32_t *p, u32 count) {
659	# endif	743	# endif
660	}	744	}
661		745
662	enum { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 };
663
664	BlockingMutex::BlockingMutex() {
665	internal_memset(this, 0, sizeof(*this));
666	}
667
668	void BlockingMutex::Lock() {
669	CHECK_EQ(owner_, 0);
670	atomic_uint32_t m = reinterpret_cast<atomic_uint32_t >(&opaque_storage_);
671	if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked)
672	return;
673	while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) {
674	#if SANITIZER_FREEBSD
675	_umtx_op(m, UMTX_OP_WAIT_UINT, MtxSleeping, 0, 0);
676	#elif SANITIZER_NETBSD
677	sched_yield(); /* No userspace futex-like synchronization */
678	#else
679	internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAIT_PRIVATE, MtxSleeping,
680	0, 0, 0);
681	#endif
682	}
683	}
684
685	void BlockingMutex::Unlock() {
686	atomic_uint32_t m = reinterpret_cast<atomic_uint32_t >(&opaque_storage_);
687	u32 v = atomic_exchange(m, MtxUnlocked, memory_order_release);
688	CHECK_NE(v, MtxUnlocked);
689	if (v == MtxSleeping) {
690	#if SANITIZER_FREEBSD
691	_umtx_op(m, UMTX_OP_WAKE, 1, 0, 0);
692	#elif SANITIZER_NETBSD
693	/* No userspace futex-like synchronization */
694	#else
695	internal_syscall(SYSCALL(futex), (uptr)m, FUTEX_WAKE_PRIVATE, 1, 0, 0, 0);
696	#endif
697	}
698	}
699
700	void BlockingMutex::CheckLocked() const {
701	auto m = reinterpret_cast<atomic_uint32_t const *>(&opaque_storage_);
702	CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed));
703	}
704	# endif // !SANITIZER_SOLARIS	746	# endif // !SANITIZER_SOLARIS
705		747
706	// ----------------- sanitizer_linux.h	748	// ----------------- sanitizer_linux.h
...	@@ -711,17 +753,17 @@ void BlockingMutex::CheckLocked() const {	...	@@ -711,17 +753,17 @@ void BlockingMutex::CheckLocked() const {
711	// Not used	753	// Not used
712	#else	754	#else
713	struct linux_dirent {	755	struct linux_dirent {
714	#if SANITIZER_X32 \|\| defined(__aarch64__) \|\| SANITIZER_RISCV64	756	# if SANITIZER_X32 \|\| SANITIZER_LINUX
715	u64 d_ino;	757	u64 d_ino;
716	u64 d_off;	758	u64 d_off;
717	#else	759	# else
718	unsigned long d_ino;	760	unsigned long d_ino;
719	unsigned long d_off;	761	unsigned long d_off;
720	#endif	762	# endif
721	unsigned short d_reclen;	763	unsigned short d_reclen;
722	#if defined(__aarch64__) \|\| SANITIZER_RISCV64	764	# if SANITIZER_LINUX
723	unsigned char d_type;	765	unsigned char d_type;
724	#endif	766	# endif
725	char d_name[256];	767	char d_name[256];
726	};	768	};
727	#endif	769	#endif
...	@@ -757,11 +799,11 @@ int internal_dlinfo(void handle, int request, void p) {	...	@@ -757,11 +799,11 @@ int internal_dlinfo(void handle, int request, void p) {
757	uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count) {	799	uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count) {
758	#if SANITIZER_FREEBSD	800	#if SANITIZER_FREEBSD
759	return internal_syscall(SYSCALL(getdirentries), fd, (uptr)dirp, count, NULL);	801	return internal_syscall(SYSCALL(getdirentries), fd, (uptr)dirp, count, NULL);
760	#elif SANITIZER_USES_CANONICAL_LINUX_SYSCALLS	802	# elif SANITIZER_LINUX
761	return internal_syscall(SYSCALL(getdents64), fd, (uptr)dirp, count);	803	return internal_syscall(SYSCALL(getdents64), fd, (uptr)dirp, count);
762	#else	804	# else
763	return internal_syscall(SYSCALL(getdents), fd, (uptr)dirp, count);	805	return internal_syscall(SYSCALL(getdents), fd, (uptr)dirp, count);
764	#endif	806	# endif
765	}	807	}
766		808
767	uptr internal_lseek(fd_t fd, OFF_T offset, int whence) {	809	uptr internal_lseek(fd_t fd, OFF_T offset, int whence) {
...	@@ -772,18 +814,29 @@ uptr internal_lseek(fd_t fd, OFF_T offset, int whence) {	...	@@ -772,18 +814,29 @@ uptr internal_lseek(fd_t fd, OFF_T offset, int whence) {
772	uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) {	814	uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) {
773	return internal_syscall(SYSCALL(prctl), option, arg2, arg3, arg4, arg5);	815	return internal_syscall(SYSCALL(prctl), option, arg2, arg3, arg4, arg5);
774	}	816	}
775	#endif	817	# if defined(__x86_64__)
		818	# include <asm/unistd_64.h>
		819	// Currently internal_arch_prctl() is only needed on x86_64.
		820	uptr internal_arch_prctl(int option, uptr arg2) {
		821	return internal_syscall(__NR_arch_prctl, option, arg2);
		822	}
		823	# endif
		824	# endif
776		825
777	uptr internal_sigaltstack(const void ss, void oss) {	826	uptr internal_sigaltstack(const void ss, void oss) {
778	return internal_syscall(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss);	827	return internal_syscall(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss);
779	}	828	}
780		829
781	int internal_fork() {	830	int internal_fork() {
782	#if SANITIZER_USES_CANONICAL_LINUX_SYSCALLS	831	# if SANITIZER_LINUX
		832	# if SANITIZER_S390
		833	return internal_syscall(SYSCALL(clone), 0, SIGCHLD);
		834	# else
783	return internal_syscall(SYSCALL(clone), SIGCHLD, 0);	835	return internal_syscall(SYSCALL(clone), SIGCHLD, 0);
784	#else	836	# endif
		837	# else
785	return internal_syscall(SYSCALL(fork));	838	return internal_syscall(SYSCALL(fork));
786	#endif	839	# endif
787	}	840	}
788		841
789	#if SANITIZER_FREEBSD	842	#if SANITIZER_FREEBSD
...	@@ -911,6 +964,10 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {	...	@@ -911,6 +964,10 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) {
911	return k_set->sig[idx] & ((uptr)1 << bit);	964	return k_set->sig[idx] & ((uptr)1 << bit);
912	}	965	}
913	#elif SANITIZER_FREEBSD	966	#elif SANITIZER_FREEBSD
		967	uptr internal_procctl(int type, int id, int cmd, void *data) {
		968	return internal_syscall(SYSCALL(procctl), type, id, cmd, data);
		969	}
		970
914	void internal_sigdelset(__sanitizer_sigset_t *set, int signum) {	971	void internal_sigdelset(__sanitizer_sigset_t *set, int signum) {
915	sigset_t rset = reinterpret_cast<sigset_t >(set);	972	sigset_t rset = reinterpret_cast<sigset_t >(set);
916	sigdelset(rset, signum);	973	sigdelset(rset, signum);
...	@@ -1052,7 +1109,7 @@ uptr GetMaxVirtualAddress() {	...	@@ -1052,7 +1109,7 @@ uptr GetMaxVirtualAddress() {
1052	#if SANITIZER_NETBSD && defined(__x86_64__)	1109	#if SANITIZER_NETBSD && defined(__x86_64__)
1053	return 0x7f7ffffff000ULL; // (0x00007f8000000000 - PAGE_SIZE)	1110	return 0x7f7ffffff000ULL; // (0x00007f8000000000 - PAGE_SIZE)
1054	#elif SANITIZER_WORDSIZE == 64	1111	#elif SANITIZER_WORDSIZE == 64
1055	# if defined(__powerpc64__) \|\| defined(__aarch64__)	1112	# if defined(__powerpc64__) \|\| defined(__aarch64__) \|\| defined(__loongarch__)
1056	// On PowerPC64 we have two different address space layouts: 44- and 46-bit.	1113	// On PowerPC64 we have two different address space layouts: 44- and 46-bit.
1057	// We somehow need to figure out which one we are using now and choose	1114	// We somehow need to figure out which one we are using now and choose
1058	// one of 0x00000fffffffffffUL and 0x00003fffffffffffUL.	1115	// one of 0x00000fffffffffffUL and 0x00003fffffffffffUL.
...	@@ -1060,10 +1117,11 @@ uptr GetMaxVirtualAddress() {	...	@@ -1060,10 +1117,11 @@ uptr GetMaxVirtualAddress() {
1060	// of the address space, so simply checking the stack address is not enough.	1117	// of the address space, so simply checking the stack address is not enough.
1061	// This should (does) work for both PowerPC64 Endian modes.	1118	// This should (does) work for both PowerPC64 Endian modes.
1062	// Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit.	1119	// Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit.
		1120	// loongarch64 also has multiple address space layouts: default is 47-bit.
1063	return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;	1121	return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;
1064	#elif SANITIZER_RISCV64	1122	#elif SANITIZER_RISCV64
1065	return (1ULL << 38) - 1;	1123	return (1ULL << 38) - 1;
1066	# elif defined(__mips64)	1124	# elif SANITIZER_MIPS64
1067	return (1ULL << 40) - 1; // 0x000000ffffffffffUL;	1125	return (1ULL << 40) - 1; // 0x000000ffffffffffUL;
1068	# elif defined(__s390x__)	1126	# elif defined(__s390x__)
1069	return (1ULL << 53) - 1; // 0x001fffffffffffffUL;	1127	return (1ULL << 53) - 1; // 0x001fffffffffffffUL;
...	@@ -1217,7 +1275,8 @@ void ForEachMappedRegion(link_map map, void (cb)(const void *, uptr)) {	...	@@ -1217,7 +1275,8 @@ void ForEachMappedRegion(link_map map, void (cb)(const void *, uptr)) {
1217	}	1275	}
1218	#endif	1276	#endif
1219		1277
1220	#if defined(__x86_64__) && SANITIZER_LINUX	1278	#if SANITIZER_LINUX
		1279	#if defined(__x86_64__)
1221	// We cannot use glibc's clone wrapper, because it messes with the child	1280	// We cannot use glibc's clone wrapper, because it messes with the child
1222	// task's TLS. It writes the PID and TID of the child task to its thread	1281	// task's TLS. It writes the PID and TID of the child task to its thread
1223	// descriptor, but in our case the child task shares the thread descriptor with	1282	// descriptor, but in our case the child task shares the thread descriptor with
...	@@ -1399,7 +1458,7 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	...	@@ -1399,7 +1458,7 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1399	#elif defined(__aarch64__)	1458	#elif defined(__aarch64__)
1400	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	1459	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1401	int parent_tidptr, void newtls, int *child_tidptr) {	1460	int parent_tidptr, void newtls, int *child_tidptr) {
1402	long long res;	1461	register long long res __asm__("x0");
1403	if (!fn \|\| !child_stack)	1462	if (!fn \|\| !child_stack)
1404	return -EINVAL;	1463	return -EINVAL;
1405	CHECK_EQ(0, (uptr)child_stack % 16);	1464	CHECK_EQ(0, (uptr)child_stack % 16);
...	@@ -1447,6 +1506,47 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	...	@@ -1447,6 +1506,47 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1447	: "x30", "memory");	1506	: "x30", "memory");
1448	return res;	1507	return res;
1449	}	1508	}
		1509	#elif SANITIZER_LOONGARCH64
		1510	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
		1511	int parent_tidptr, void newtls, int *child_tidptr) {
		1512	if (!fn \|\| !child_stack)
		1513	return -EINVAL;
		1514
		1515	CHECK_EQ(0, (uptr)child_stack % 16);
		1516
		1517	register int res __asm__("$a0");
		1518	register int __flags __asm__("$a0") = flags;
		1519	register void *__stack __asm__("$a1") = child_stack;
		1520	register int *__ptid __asm__("$a2") = parent_tidptr;
		1521	register int *__ctid __asm__("$a3") = child_tidptr;
		1522	register void *__tls __asm__("$a4") = newtls;
		1523	register int (__fn)(void ) __asm__("$a5") = fn;
		1524	register void *__arg __asm__("$a6") = arg;
		1525	register int nr_clone __asm__("$a7") = __NR_clone;
		1526
		1527	__asm__ __volatile__(
		1528	"syscall 0\n"
		1529
		1530	// if ($a0 != 0)
		1531	// return $a0;
		1532	"bnez $a0, 1f\n"
		1533
		1534	// In the child, now. Call "fn(arg)".
		1535	"move $a0, $a6\n"
		1536	"jirl $ra, $a5, 0\n"
		1537
		1538	// Call _exit($a0).
		1539	"addi.d $a7, $zero, %9\n"
		1540	"syscall 0\n"
		1541
		1542	"1:\n"
		1543
		1544	: "=r"(res)
		1545	: "0"(__flags), "r"(__stack), "r"(__ptid), "r"(__ctid), "r"(__tls),
		1546	"r"(__fn), "r"(__arg), "r"(nr_clone), "i"(__NR_exit)
		1547	: "memory", "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", "$t8");
		1548	return res;
		1549	}
1450	#elif defined(__powerpc64__)	1550	#elif defined(__powerpc64__)
1451	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	1551	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1452	int parent_tidptr, void newtls, int *child_tidptr) {	1552	int parent_tidptr, void newtls, int *child_tidptr) {
...	@@ -1556,7 +1656,7 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	...	@@ -1556,7 +1656,7 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1556	: "cr0", "cr1", "memory", "ctr", "r0", "r27", "r28", "r29");	1656	: "cr0", "cr1", "memory", "ctr", "r0", "r27", "r28", "r29");
1557	return res;	1657	return res;
1558	}	1658	}
1559	#elif defined(__i386__) && SANITIZER_LINUX	1659	#elif defined(__i386__)
1560	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	1660	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1561	int parent_tidptr, void newtls, int *child_tidptr) {	1661	int parent_tidptr, void newtls, int *child_tidptr) {
1562	int res;	1662	int res;
...	@@ -1621,7 +1721,7 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	...	@@ -1621,7 +1721,7 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1621	: "memory");	1721	: "memory");
1622	return res;	1722	return res;
1623	}	1723	}
1624	#elif defined(__arm__) && SANITIZER_LINUX	1724	#elif defined(__arm__)
1625	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	1725	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1626	int parent_tidptr, void newtls, int *child_tidptr) {	1726	int parent_tidptr, void newtls, int *child_tidptr) {
1627	unsigned int res;	1727	unsigned int res;
...	@@ -1687,7 +1787,8 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	...	@@ -1687,7 +1787,8 @@ uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
1687	: "memory");	1787	: "memory");
1688	return res;	1788	return res;
1689	}	1789	}
1690	#endif // defined(__x86_64__) && SANITIZER_LINUX	1790	#endif
		1791	#endif // SANITIZER_LINUX
1691		1792
1692	#if SANITIZER_LINUX	1793	#if SANITIZER_LINUX
1693	int internal_uname(struct utsname *buf) {	1794	int internal_uname(struct utsname *buf) {
...	@@ -1778,23 +1879,18 @@ HandleSignalMode GetHandleSignalMode(int signum) {	...	@@ -1778,23 +1879,18 @@ HandleSignalMode GetHandleSignalMode(int signum) {
1778		1879
1779	#if !SANITIZER_GO	1880	#if !SANITIZER_GO
1780	void internal_start_thread(void (func)(void arg), void *arg) {	1881	void internal_start_thread(void (func)(void arg), void *arg) {
		1882	if (&real_pthread_create == 0)
		1883	return nullptr;
1781	// Start the thread with signals blocked, otherwise it can steal user signals.	1884	// Start the thread with signals blocked, otherwise it can steal user signals.
1782	__sanitizer_sigset_t set, old;	1885	ScopedBlockSignals block(nullptr);
1783	internal_sigfillset(&set);
1784	#if SANITIZER_LINUX && !SANITIZER_ANDROID
1785	// Glibc uses SIGSETXID signal during setuid call. If this signal is blocked
1786	// on any thread, setuid call hangs (see test/tsan/setuid.c).
1787	internal_sigdelset(&set, 33);
1788	#endif
1789	internal_sigprocmask(SIG_SETMASK, &set, &old);
1790	void *th;	1886	void *th;
1791	real_pthread_create(&th, nullptr, func, arg);	1887	real_pthread_create(&th, nullptr, func, arg);
1792	internal_sigprocmask(SIG_SETMASK, &old, nullptr);
1793	return th;	1888	return th;
1794	}	1889	}
1795		1890
1796	void internal_join_thread(void *th) {	1891	void internal_join_thread(void *th) {
1797	real_pthread_join(th, nullptr);	1892	if (&real_pthread_join)
		1893	real_pthread_join(th, nullptr);
1798	}	1894	}
1799	#else	1895	#else
1800	void internal_start_thread(void (func)(void ), void *arg) { return 0; }	1896	void internal_start_thread(void (func)(void ), void *arg) { return 0; }
...	@@ -1802,7 +1898,7 @@ void internal_start_thread(void (func)(void ), void *arg) { return 0; }	...	@@ -1802,7 +1898,7 @@ void internal_start_thread(void (func)(void ), void *arg) { return 0; }
1802	void internal_join_thread(void *th) {}	1898	void internal_join_thread(void *th) {}
1803	#endif	1899	#endif
1804		1900
1805	#if defined(__aarch64__)	1901	#if SANITIZER_LINUX && defined(__aarch64__)
1806	// Android headers in the older NDK releases miss this definition.	1902	// Android headers in the older NDK releases miss this definition.
1807	struct __sanitizer_esr_context {	1903	struct __sanitizer_esr_context {
1808	struct _aarch64_ctx head;	1904	struct _aarch64_ctx head;
...	@@ -1811,7 +1907,7 @@ struct __sanitizer_esr_context {	...	@@ -1811,7 +1907,7 @@ struct __sanitizer_esr_context {
1811		1907
1812	static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {	1908	static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {
1813	static const u32 kEsrMagic = 0x45535201;	1909	static const u32 kEsrMagic = 0x45535201;
1814	u8 *aux = ucontext->uc_mcontext.__reserved;	1910	u8 aux = reinterpret_cast<u8 >(ucontext->uc_mcontext.__reserved);
1815	while (true) {	1911	while (true) {
1816	_aarch64_ctx ctx = (_aarch64_ctx )aux;	1912	_aarch64_ctx ctx = (_aarch64_ctx )aux;
1817	if (ctx->size == 0) break;	1913	if (ctx->size == 0) break;
...	@@ -1823,6 +1919,11 @@ static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {	...	@@ -1823,6 +1919,11 @@ static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {
1823	}	1919	}
1824	return false;	1920	return false;
1825	}	1921	}
		1922	#elif SANITIZER_FREEBSD && defined(__aarch64__)
		1923	// FreeBSD doesn't provide ESR in the ucontext.
		1924	static bool Aarch64GetESR(ucontext_t ucontext, u64 esr) {
		1925	return false;
		1926	}
1826	#endif	1927	#endif
1827		1928
1828	using Context = ucontext_t;	1929	using Context = ucontext_t;
...	@@ -1841,7 +1942,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -1841,7 +1942,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1841	#else	1942	#else
1842	uptr err = ucontext->uc_mcontext.gregs[REG_ERR];	1943	uptr err = ucontext->uc_mcontext.gregs[REG_ERR];
1843	#endif // SANITIZER_FREEBSD	1944	#endif // SANITIZER_FREEBSD
1844	return err & PF_WRITE ? WRITE : READ;	1945	return err & PF_WRITE ? Write : Read;
1845	#elif defined(__mips__)	1946	#elif defined(__mips__)
1846	uint32_t *exception_source;	1947	uint32_t *exception_source;
1847	uint32_t faulty_instruction;	1948	uint32_t faulty_instruction;
...	@@ -1864,7 +1965,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -1864,7 +1965,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1864	case 0x2a: // swl	1965	case 0x2a: // swl
1865	case 0x2e: // swr	1966	case 0x2e: // swr
1866	#endif	1967	#endif
1867	return SignalContext::WRITE;	1968	return SignalContext::Write;
1868		1969
1869	case 0x20: // lb	1970	case 0x20: // lb
1870	case 0x24: // lbu	1971	case 0x24: // lbu
...	@@ -1879,27 +1980,34 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -1879,27 +1980,34 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1879	case 0x22: // lwl	1980	case 0x22: // lwl
1880	case 0x26: // lwr	1981	case 0x26: // lwr
1881	#endif	1982	#endif
1882	return SignalContext::READ;	1983	return SignalContext::Read;
1883	#if __mips_isa_rev == 6	1984	#if __mips_isa_rev == 6
1884	case 0x3b: // pcrel	1985	case 0x3b: // pcrel
1885	op_code = (faulty_instruction >> 19) & 0x3;	1986	op_code = (faulty_instruction >> 19) & 0x3;
1886	switch (op_code) {	1987	switch (op_code) {
1887	case 0x1: // lwpc	1988	case 0x1: // lwpc
1888	case 0x2: // lwupc	1989	case 0x2: // lwupc
1889	return SignalContext::READ;	1990	return SignalContext::Read;
1890	}	1991	}
1891	#endif	1992	#endif
1892	}	1993	}
1893	return SignalContext::UNKNOWN;	1994	return SignalContext::Unknown;
1894	#elif defined(__arm__)	1995	#elif defined(__arm__)
1895	static const uptr FSR_WRITE = 1U << 11;	1996	static const uptr FSR_WRITE = 1U << 11;
1896	uptr fsr = ucontext->uc_mcontext.error_code;	1997	uptr fsr = ucontext->uc_mcontext.error_code;
1897	return fsr & FSR_WRITE ? WRITE : READ;	1998	return fsr & FSR_WRITE ? Write : Read;
1898	#elif defined(__aarch64__)	1999	#elif defined(__aarch64__)
1899	static const u64 ESR_ELx_WNR = 1U << 6;	2000	static const u64 ESR_ELx_WNR = 1U << 6;
1900	u64 esr;	2001	u64 esr;
1901	if (!Aarch64GetESR(ucontext, &esr)) return UNKNOWN;	2002	if (!Aarch64GetESR(ucontext, &esr)) return Unknown;
1902	return esr & ESR_ELx_WNR ? WRITE : READ;	2003	return esr & ESR_ELx_WNR ? Write : Read;
		2004	#elif defined(__loongarch__)
		2005	u32 flags = ucontext->uc_mcontext.__flags;
		2006	if (flags & SC_ADDRERR_RD)
		2007	return SignalContext::Read;
		2008	if (flags & SC_ADDRERR_WR)
		2009	return SignalContext::Write;
		2010	return SignalContext::Unknown;
1903	#elif defined(__sparc__)	2011	#elif defined(__sparc__)
1904	// Decode the instruction to determine the access type.	2012	// Decode the instruction to determine the access type.
1905	// From OpenSolaris $SRC/uts/sun4/os/trap.c (get_accesstype).	2013	// From OpenSolaris $SRC/uts/sun4/os/trap.c (get_accesstype).
...	@@ -1915,9 +2023,13 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -1915,9 +2023,13 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1915	#endif	2023	#endif
1916	#endif	2024	#endif
1917	u32 instr = (u32 )pc;	2025	u32 instr = (u32 )pc;
1918	return (instr >> 21) & 1 ? WRITE: READ;	2026	return (instr >> 21) & 1 ? Write: Read;
1919	#elif defined(__riscv)	2027	#elif defined(__riscv)
		2028	#if SANITIZER_FREEBSD
		2029	unsigned long pc = ucontext->uc_mcontext.mc_gpregs.gp_sepc;
		2030	#else
1920	unsigned long pc = ucontext->uc_mcontext.__gregs[REG_PC];	2031	unsigned long pc = ucontext->uc_mcontext.__gregs[REG_PC];
		2032	#endif
1921	unsigned faulty_instruction = (uint16_t )pc;	2033	unsigned faulty_instruction = (uint16_t )pc;
1922		2034
1923	#if defined(__riscv_compressed)	2035	#if defined(__riscv_compressed)
...	@@ -1931,7 +2043,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -1931,7 +2043,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1931	#if __riscv_xlen == 64	2043	#if __riscv_xlen == 64
1932	case 0b10'011: // c.ldsp (rd != x0)	2044	case 0b10'011: // c.ldsp (rd != x0)
1933	#endif	2045	#endif
1934	return rd ? SignalContext::READ : SignalContext::UNKNOWN;	2046	return rd ? SignalContext::Read : SignalContext::Unknown;
1935	case 0b00'010: // c.lw	2047	case 0b00'010: // c.lw
1936	#if __riscv_flen >= 32 && __riscv_xlen == 32	2048	#if __riscv_flen >= 32 && __riscv_xlen == 32
1937	case 0b10'011: // c.flwsp	2049	case 0b10'011: // c.flwsp
...	@@ -1943,7 +2055,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -1943,7 +2055,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1943	case 0b00'001: // c.fld	2055	case 0b00'001: // c.fld
1944	case 0b10'001: // c.fldsp	2056	case 0b10'001: // c.fldsp
1945	#endif	2057	#endif
1946	return SignalContext::READ;	2058	return SignalContext::Read;
1947	case 0b00'110: // c.sw	2059	case 0b00'110: // c.sw
1948	case 0b10'110: // c.swsp	2060	case 0b10'110: // c.swsp
1949	#if __riscv_flen >= 32 \|\| __riscv_xlen == 64	2061	#if __riscv_flen >= 32 \|\| __riscv_xlen == 64
...	@@ -1954,9 +2066,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -1954,9 +2066,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1954	case 0b00'101: // c.fsd	2066	case 0b00'101: // c.fsd
1955	case 0b10'101: // c.fsdsp	2067	case 0b10'101: // c.fsdsp
1956	#endif	2068	#endif
1957	return SignalContext::WRITE;	2069	return SignalContext::Write;
1958	default:	2070	default:
1959	return SignalContext::UNKNOWN;	2071	return SignalContext::Unknown;
1960	}	2072	}
1961	}	2073	}
1962	#endif	2074	#endif
...	@@ -1974,9 +2086,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -1974,9 +2086,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1974	#endif	2086	#endif
1975	case 0b100: // lbu	2087	case 0b100: // lbu
1976	case 0b101: // lhu	2088	case 0b101: // lhu
1977	return SignalContext::READ;	2089	return SignalContext::Read;
1978	default:	2090	default:
1979	return SignalContext::UNKNOWN;	2091	return SignalContext::Unknown;
1980	}	2092	}
1981	case 0b0100011: // stores	2093	case 0b0100011: // stores
1982	switch (funct3) {	2094	switch (funct3) {
...	@@ -1986,9 +2098,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -1986,9 +2098,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1986	#if __riscv_xlen == 64	2098	#if __riscv_xlen == 64
1987	case 0b011: // sd	2099	case 0b011: // sd
1988	#endif	2100	#endif
1989	return SignalContext::WRITE;	2101	return SignalContext::Write;
1990	default:	2102	default:
1991	return SignalContext::UNKNOWN;	2103	return SignalContext::Unknown;
1992	}	2104	}
1993	#if __riscv_flen >= 32	2105	#if __riscv_flen >= 32
1994	case 0b0000111: // floating-point loads	2106	case 0b0000111: // floating-point loads
...	@@ -1997,9 +2109,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -1997,9 +2109,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
1997	#if __riscv_flen == 64	2109	#if __riscv_flen == 64
1998	case 0b011: // fld	2110	case 0b011: // fld
1999	#endif	2111	#endif
2000	return SignalContext::READ;	2112	return SignalContext::Read;
2001	default:	2113	default:
2002	return SignalContext::UNKNOWN;	2114	return SignalContext::Unknown;
2003	}	2115	}
2004	case 0b0100111: // floating-point stores	2116	case 0b0100111: // floating-point stores
2005	switch (funct3) {	2117	switch (funct3) {
...	@@ -2007,17 +2119,17 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	...	@@ -2007,17 +2119,17 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
2007	#if __riscv_flen == 64	2119	#if __riscv_flen == 64
2008	case 0b011: // fsd	2120	case 0b011: // fsd
2009	#endif	2121	#endif
2010	return SignalContext::WRITE;	2122	return SignalContext::Write;
2011	default:	2123	default:
2012	return SignalContext::UNKNOWN;	2124	return SignalContext::Unknown;
2013	}	2125	}
2014	#endif	2126	#endif
2015	default:	2127	default:
2016	return SignalContext::UNKNOWN;	2128	return SignalContext::Unknown;
2017	}	2129	}
2018	#else	2130	#else
2019	(void)ucontext;	2131	(void)ucontext;
2020	return UNKNOWN; // FIXME: Implement.	2132	return Unknown; // FIXME: Implement.
2021	#endif	2133	#endif
2022	}	2134	}
2023		2135
...	@@ -2044,10 +2156,17 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {	...	@@ -2044,10 +2156,17 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2044	*bp = ucontext->uc_mcontext.arm_fp;	2156	*bp = ucontext->uc_mcontext.arm_fp;
2045	*sp = ucontext->uc_mcontext.arm_sp;	2157	*sp = ucontext->uc_mcontext.arm_sp;
2046	#elif defined(__aarch64__)	2158	#elif defined(__aarch64__)
		2159	# if SANITIZER_FREEBSD
		2160	ucontext_t ucontext = (ucontext_t)context;
		2161	*pc = ucontext->uc_mcontext.mc_gpregs.gp_elr;
		2162	*bp = ucontext->uc_mcontext.mc_gpregs.gp_x[29];
		2163	*sp = ucontext->uc_mcontext.mc_gpregs.gp_sp;
		2164	# else
2047	ucontext_t ucontext = (ucontext_t)context;	2165	ucontext_t ucontext = (ucontext_t)context;
2048	*pc = ucontext->uc_mcontext.pc;	2166	*pc = ucontext->uc_mcontext.pc;
2049	*bp = ucontext->uc_mcontext.regs[29];	2167	*bp = ucontext->uc_mcontext.regs[29];
2050	*sp = ucontext->uc_mcontext.sp;	2168	*sp = ucontext->uc_mcontext.sp;
		2169	# endif
2051	#elif defined(__hppa__)	2170	#elif defined(__hppa__)
2052	ucontext_t ucontext = (ucontext_t)context;	2171	ucontext_t ucontext = (ucontext_t)context;
2053	*pc = ucontext->uc_mcontext.sc_iaoq[0];	2172	*pc = ucontext->uc_mcontext.sc_iaoq[0];
...	@@ -2092,12 +2211,19 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {	...	@@ -2092,12 +2211,19 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2092	*sp = ucontext->uc_mcontext.gregs[REG_UESP];	2211	*sp = ucontext->uc_mcontext.gregs[REG_UESP];
2093	# endif	2212	# endif
2094	#elif defined(__powerpc__) \|\| defined(__powerpc64__)	2213	#elif defined(__powerpc__) \|\| defined(__powerpc64__)
		2214	# if SANITIZER_FREEBSD
		2215	ucontext_t ucontext = (ucontext_t )context;
		2216	*pc = ucontext->uc_mcontext.mc_srr0;
		2217	*sp = ucontext->uc_mcontext.mc_frame[1];
		2218	*bp = ucontext->uc_mcontext.mc_frame[31];
		2219	# else
2095	ucontext_t ucontext = (ucontext_t)context;	2220	ucontext_t ucontext = (ucontext_t)context;
2096	*pc = ucontext->uc_mcontext.regs->nip;	2221	*pc = ucontext->uc_mcontext.regs->nip;
2097	*sp = ucontext->uc_mcontext.regs->gpr[PT_R1];	2222	*sp = ucontext->uc_mcontext.regs->gpr[PT_R1];
2098	// The powerpc{,64}-linux ABIs do not specify r31 as the frame	2223	// The powerpc{,64}-linux ABIs do not specify r31 as the frame
2099	// pointer, but GCC always uses r31 when we need a frame pointer.	2224	// pointer, but GCC always uses r31 when we need a frame pointer.
2100	*bp = ucontext->uc_mcontext.regs->gpr[PT_R31];	2225	*bp = ucontext->uc_mcontext.regs->gpr[PT_R31];
		2226	# endif
2101	#elif defined(__sparc__)	2227	#elif defined(__sparc__)
2102	#if defined(__arch64__) \|\| defined(__sparcv9)	2228	#if defined(__arch64__) \|\| defined(__sparcv9)
2103	#define STACK_BIAS 2047	2229	#define STACK_BIAS 2047
...	@@ -2136,12 +2262,28 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {	...	@@ -2136,12 +2262,28 @@ static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
2136	*sp = ucontext->uc_mcontext.gregs[15];	2262	*sp = ucontext->uc_mcontext.gregs[15];
2137	#elif defined(__riscv)	2263	#elif defined(__riscv)
2138	ucontext_t ucontext = (ucontext_t)context;	2264	ucontext_t ucontext = (ucontext_t)context;
		2265	# if SANITIZER_FREEBSD
		2266	*pc = ucontext->uc_mcontext.mc_gpregs.gp_sepc;
		2267	*bp = ucontext->uc_mcontext.mc_gpregs.gp_s[0];
		2268	*sp = ucontext->uc_mcontext.mc_gpregs.gp_sp;
		2269	# else
2139	*pc = ucontext->uc_mcontext.__gregs[REG_PC];	2270	*pc = ucontext->uc_mcontext.__gregs[REG_PC];
2140	*bp = ucontext->uc_mcontext.__gregs[REG_S0];	2271	*bp = ucontext->uc_mcontext.__gregs[REG_S0];
2141	*sp = ucontext->uc_mcontext.__gregs[REG_SP];	2272	*sp = ucontext->uc_mcontext.__gregs[REG_SP];
2142	#else	2273	# endif
2143	# error "Unsupported arch"	2274	# elif defined(__hexagon__)
2144	#endif	2275	ucontext_t ucontext = (ucontext_t )context;
		2276	*pc = ucontext->uc_mcontext.pc;
		2277	*bp = ucontext->uc_mcontext.r30;
		2278	*sp = ucontext->uc_mcontext.r29;
		2279	# elif defined(__loongarch__)
		2280	ucontext_t ucontext = (ucontext_t )context;
		2281	*pc = ucontext->uc_mcontext.__pc;
		2282	*bp = ucontext->uc_mcontext.__gregs[22];
		2283	*sp = ucontext->uc_mcontext.__gregs[3];
		2284	# else
		2285	# error "Unsupported arch"
		2286	# endif
2145	}	2287	}
2146		2288
2147	void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); }	2289	void SignalContext::InitPcSpBp() { GetPcSpBp(context, &pc, &sp, &bp); }
...	@@ -2150,10 +2292,6 @@ void InitializePlatformEarly() {	...	@@ -2150,10 +2292,6 @@ void InitializePlatformEarly() {
2150	// Do nothing.	2292	// Do nothing.
2151	}	2293	}
2152		2294
2153	void MaybeReexec() {
2154	// No need to re-exec on Linux.
2155	}
2156
2157	void CheckASLR() {	2295	void CheckASLR() {
2158	#if SANITIZER_NETBSD	2296	#if SANITIZER_NETBSD
2159	int mib[3];	2297	int mib[3];
...	@@ -2175,49 +2313,35 @@ void CheckASLR() {	...	@@ -2175,49 +2313,35 @@ void CheckASLR() {
2175	GetArgv()[0]);	2313	GetArgv()[0]);
2176	Die();	2314	Die();
2177	}	2315	}
2178	#elif SANITIZER_PPC64V2
2179	// Disable ASLR for Linux PPC64LE.
2180	int old_personality = personality(0xffffffff);
2181	if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) {
2182	VReport(1, "WARNING: Program is being run with address space layout "
2183	"randomization (ASLR) enabled which prevents the thread and "
2184	"memory sanitizers from working on powerpc64le.\n"
2185	"ASLR will be disabled and the program re-executed.\n");
2186	CHECK_NE(personality(old_personality \| ADDR_NO_RANDOMIZE), -1);
2187	ReExec();
2188	}
2189	#elif SANITIZER_FREEBSD	2316	#elif SANITIZER_FREEBSD
2190	int aslr_pie;	2317	int aslr_status;
2191	uptr len = sizeof(aslr_pie);	2318	int r = internal_procctl(P_PID, 0, PROC_ASLR_STATUS, &aslr_status);
2192	#if SANITIZER_WORDSIZE == 64	2319	if (UNLIKELY(r == -1)) {
2193	if (UNLIKELY(internal_sysctlbyname("kern.elf64.aslr.pie_enable",
2194	&aslr_pie, &len, NULL, 0) == -1)) {
2195	// We're making things less 'dramatic' here since	2320	// We're making things less 'dramatic' here since
2196	// the OID is not necessarily guaranteed to be here	2321	// the cmd is not necessarily guaranteed to be here
2197	// just yet regarding FreeBSD release	2322	// just yet regarding FreeBSD release
2198	return;	2323	return;
2199	}	2324	}
2200		2325	if ((aslr_status & PROC_ASLR_ACTIVE) != 0) {
2201	if (aslr_pie > 0) {
2202	Printf("This sanitizer is not compatible with enabled ASLR "	2326	Printf("This sanitizer is not compatible with enabled ASLR "
2203	"and binaries compiled with PIE\n");	2327	"and binaries compiled with PIE\n");
2204	Die();	2328	Die();
2205	}	2329	}
2206	#endif	2330	# elif SANITIZER_PPC64V2
2207	// there might be 32 bits compat for 64 bits	2331	// Disable ASLR for Linux PPC64LE.
2208	if (UNLIKELY(internal_sysctlbyname("kern.elf32.aslr.pie_enable",	2332	int old_personality = personality(0xffffffff);
2209	&aslr_pie, &len, NULL, 0) == -1)) {	2333	if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) {
2210	return;	2334	VReport(1,
2211	}	2335	"WARNING: Program is being run with address space layout "
2212		2336	"randomization (ASLR) enabled which prevents the thread and "
2213	if (aslr_pie > 0) {	2337	"memory sanitizers from working on powerpc64le.\n"
2214	Printf("This sanitizer is not compatible with enabled ASLR "	2338	"ASLR will be disabled and the program re-executed.\n");
2215	"and binaries compiled with PIE\n");	2339	CHECK_NE(personality(old_personality \| ADDR_NO_RANDOMIZE), -1);
2216	Die();	2340	ReExec();
2217	}	2341	}
2218	#else	2342	# else
2219	// Do nothing	2343	// Do nothing
2220	#endif	2344	# endif
2221	}	2345	}
2222		2346
2223	void CheckMPROTECT() {	2347	void CheckMPROTECT() {

lib/tsan/sanitizer_common/sanitizer_linux.h+25-4

...	@@ -49,26 +49,44 @@ uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count);	...	@@ -49,26 +49,44 @@ uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count);
49	uptr internal_sigaltstack(const void* ss, void* oss);	49	uptr internal_sigaltstack(const void* ss, void* oss);
50	uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,	50	uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
51	__sanitizer_sigset_t *oldset);	51	__sanitizer_sigset_t *oldset);
52	#if SANITIZER_GLIBC	52
		53	void SetSigProcMask(__sanitizer_sigset_t set, __sanitizer_sigset_t oldset);
		54	void BlockSignals(__sanitizer_sigset_t *oldset = nullptr);
		55	struct ScopedBlockSignals {
		56	explicit ScopedBlockSignals(__sanitizer_sigset_t *copy);
		57	~ScopedBlockSignals();
		58
		59	ScopedBlockSignals &operator=(const ScopedBlockSignals &) = delete;
		60	ScopedBlockSignals(const ScopedBlockSignals &) = delete;
		61
		62	private:
		63	__sanitizer_sigset_t saved_;
		64	};
		65
		66	# if SANITIZER_GLIBC
53	uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp);	67	uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp);
54	#endif	68	#endif
55		69
56	// Linux-only syscalls.	70	// Linux-only syscalls.
57	#if SANITIZER_LINUX	71	#if SANITIZER_LINUX
58	uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5);	72	uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5);
		73	# if defined(__x86_64__)
		74	uptr internal_arch_prctl(int option, uptr arg2);
		75	# endif
59	// Used only by sanitizer_stoptheworld. Signal handlers that are actually used	76	// Used only by sanitizer_stoptheworld. Signal handlers that are actually used
60	// (like the process-wide error reporting SEGV handler) must use	77	// (like the process-wide error reporting SEGV handler) must use
61	// internal_sigaction instead.	78	// internal_sigaction instead.
62	int internal_sigaction_norestorer(int signum, const void act, void oldact);	79	int internal_sigaction_norestorer(int signum, const void act, void oldact);
63	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);	80	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
64	#if defined(__x86_64__) \|\| defined(__mips__) \|\| defined(__aarch64__) \|\| \	81	# if defined(__x86_64__) \|\| defined(__mips__) \|\| defined(__aarch64__) \|\| \
65	defined(__powerpc64__) \|\| defined(__s390__) \|\| defined(__i386__) \|\| \	82	defined(__powerpc64__) \|\| defined(__s390__) \|\| defined(__i386__) \|\| \
66	defined(__arm__) \|\| SANITIZER_RISCV64	83	defined(__arm__) \|\| SANITIZER_RISCV64 \|\| SANITIZER_LOONGARCH64
67	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,	84	uptr internal_clone(int (fn)(void ), void child_stack, int flags, void arg,
68	int parent_tidptr, void newtls, int *child_tidptr);	85	int parent_tidptr, void newtls, int *child_tidptr);
69	#endif	86	#endif
70	int internal_uname(struct utsname *buf);	87	int internal_uname(struct utsname *buf);
71	#elif SANITIZER_FREEBSD	88	#elif SANITIZER_FREEBSD
		89	uptr internal_procctl(int type, int id, int cmd, void *data);
72	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);	90	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
73	#elif SANITIZER_NETBSD	91	#elif SANITIZER_NETBSD
74	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);	92	void internal_sigdelset(__sanitizer_sigset_t *set, int signum);
...	@@ -135,6 +153,9 @@ inline void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) {	...	@@ -135,6 +153,9 @@ inline void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) {
135	"rdhwr %0,$29\n" \	153	"rdhwr %0,$29\n" \
136	".set pop\n" : "=r"(__v)); \	154	".set pop\n" : "=r"(__v)); \
137	__v; })	155	__v; })
		156	#elif defined (__riscv)
		157	# define __get_tls() \
		158	({ void** __v; __asm__("mv %0, tp" : "=r"(__v)); __v; })
138	#elif defined(__i386__)	159	#elif defined(__i386__)
139	# define __get_tls() \	160	# define __get_tls() \
140	({ void** __v; __asm__("movl %%gs:0, %0" : "=r"(__v)); __v; })	161	({ void** __v; __asm__("movl %%gs:0, %0" : "=r"(__v)); __v; })

lib/tsan/sanitizer_common/sanitizer_linux_libcdep.cpp+109-30

 @@ -27,6 +27,7 @@
 #include "sanitizer_linux.h"
 #include "sanitizer_placement_new.h"
 #include "sanitizer_procmaps.h"
+#include "sanitizer_solaris.h"
 #if SANITIZER_NETBSD
 #define _RTLD_SOURCE  // for __lwp_gettcb_fast() / __lwp_getprivate_fast()
 @@ -62,6 +63,7 @@
 #endif
 #if SANITIZER_SOLARIS
+#include <stddef.h>
 #include <stdlib.h>
 #include <thread.h>
 #endif
 @@ -146,7 +148,7 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
   pthread_attr_t attr;
   pthread_attr_init(&attr);
   CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
-  my_pthread_attr_getstack(&attr, &stackaddr, &stacksize);
+  internal_pthread_attr_getstack(&attr, &stackaddr, &stacksize);
   pthread_attr_destroy(&attr);
 #endif  // SANITIZER_SOLARIS
 @@ -203,7 +205,8 @@ void InitTlsSize() {
   g_use_dlpi_tls_data =
       GetLibcVersion(&major, &minor, &patch) && major == 2 && minor >= 25;
-#if defined(__aarch64__) || defined(__x86_64__) || defined(__powerpc64__)
+#if defined(__aarch64__) || defined(__x86_64__) || defined(__powerpc64__) || \
+    defined(__loongarch__)
   void *get_tls_static_info = dlsym(RTLD_NEXT, "_dl_get_tls_static_info");
   size_t tls_align;
   ((void (*)(size_t *, size_t *))get_tls_static_info)(&g_tls_size, &tls_align);
 @@ -216,14 +219,13 @@ void InitTlsSize() { }
 // On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage
 // of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan
 // to get the pointer to thread-specific data keys in the thread control block.
-#if (SANITIZER_FREEBSD || SANITIZER_LINUX) && !SANITIZER_ANDROID
+#if (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS) && \
+    !SANITIZER_ANDROID && !SANITIZER_GO
 // sizeof(struct pthread) from glibc.
 static atomic_uintptr_t thread_descriptor_size;
-uptr ThreadDescriptorSize() {
+static uptr ThreadDescriptorSizeFallback() {
-  uptr val = atomic_load_relaxed(&thread_descriptor_size);
+  uptr val = 0;
-  if (val)
-    return val;
 #if defined(__x86_64__) || defined(__i386__) || defined(__arm__)
   int major;
   int minor;
 @@ -264,6 +266,8 @@ uptr ThreadDescriptorSize() {
 #elif defined(__mips__)
   // TODO(sagarthakur): add more values as per different glibc versions.
   val = FIRST_32_SECOND_64(1152, 1776);
+#elif SANITIZER_LOONGARCH64
+  val = 1856; // from glibc 2.36
 #elif SANITIZER_RISCV64
   int major;
   int minor;
 @@ -285,12 +289,26 @@ uptr ThreadDescriptorSize() {
 #elif defined(__powerpc64__)
   val = 1776; // from glibc.ppc64le 2.20-8.fc21
 #endif
+  return val;
+}
+uptr ThreadDescriptorSize() {
+  uptr val = atomic_load_relaxed(&thread_descriptor_size);
   if (val)
-    atomic_store_relaxed(&thread_descriptor_size, val);
+    return val;
+  // _thread_db_sizeof_pthread is a GLIBC_PRIVATE symbol that is exported in
+  // glibc 2.34 and later.
+  if (unsigned *psizeof = static_cast<unsigned *>(
+          dlsym(RTLD_DEFAULT, "_thread_db_sizeof_pthread")))
+    val = *psizeof;
+  if (!val)
+    val = ThreadDescriptorSizeFallback();
+  atomic_store_relaxed(&thread_descriptor_size, val);
   return val;
 }
-#if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64
+#if defined(__mips__) || defined(__powerpc64__) || SANITIZER_RISCV64 || \
+    SANITIZER_LOONGARCH64
 // TlsPreTcbSize includes size of struct pthread_descr and size of tcb
 // head structure. It lies before the static tls blocks.
 static uptr TlsPreTcbSize() {
 @@ -300,6 +318,8 @@ static uptr TlsPreTcbSize() {
   const uptr kTcbHead = 88; // sizeof (tcbhead_t)
 #elif SANITIZER_RISCV64
   const uptr kTcbHead = 16;  // sizeof (tcbhead_t)
+#elif SANITIZER_LOONGARCH64
+  const uptr kTcbHead = 16;  // sizeof (tcbhead_t)
 #endif
   const uptr kTlsAlign = 16;
   const uptr kTlsPreTcbSize =
 @@ -308,7 +328,6 @@ static uptr TlsPreTcbSize() {
 }
 #endif
-#if !SANITIZER_GO
 namespace {
 struct TlsBlock {
   uptr begin, end, align;
 @@ -339,19 +358,43 @@ static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) {
 extern "C" void *__tls_get_addr(size_t *);
 #endif
+static size_t main_tls_modid;
 static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,
                                   void *data) {
-  if (!info->dlpi_tls_modid)
+  size_t tls_modid;
+#if SANITIZER_SOLARIS
+  // dlpi_tls_modid is only available since Solaris 11.4 SRU 10.  Use
+  // dlinfo(RTLD_DI_LINKMAP) instead which works on all of Solaris 11.3,
+  // 11.4, and Illumos.  The tlsmodid of the executable was changed to 1 in
+  // 11.4 to match other implementations.
+  if (size >= offsetof(dl_phdr_info_test, dlpi_tls_modid))
+    main_tls_modid = 1;
+  else
+    main_tls_modid = 0;
+  g_use_dlpi_tls_data = 0;
+  Rt_map *map;
+  dlinfo(RTLD_SELF, RTLD_DI_LINKMAP, &map);
+  tls_modid = map->rt_tlsmodid;
+#else
+  main_tls_modid = 1;
+  tls_modid = info->dlpi_tls_modid;
+#endif
+  if (tls_modid < main_tls_modid)
     return 0;
-  uptr begin = (uptr)info->dlpi_tls_data;
+  uptr begin;
+#if !SANITIZER_SOLARIS
+  begin = (uptr)info->dlpi_tls_data;
+#endif
   if (!g_use_dlpi_tls_data) {
     // Call __tls_get_addr as a fallback. This forces TLS allocation on glibc
     // and FreeBSD.
 #ifdef __s390__
     begin = (uptr)__builtin_thread_pointer() +
-            TlsGetOffset(info->dlpi_tls_modid, 0);
+            TlsGetOffset(tls_modid, 0);
 #else
-    size_t mod_and_off[2] = {info->dlpi_tls_modid, 0};
+    size_t mod_and_off[2] = {tls_modid, 0};
     begin = (uptr)__tls_get_addr(mod_and_off);
 #endif
   }
 @@ -359,7 +402,7 @@ static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size,
     if (info->dlpi_phdr[i].p_type == PT_TLS) {
       static_cast<InternalMmapVector<TlsBlock> *>(data)->push_back(
           TlsBlock{begin, begin + info->dlpi_phdr[i].p_memsz,
-                   info->dlpi_phdr[i].p_align, info->dlpi_tls_modid});
+                   info->dlpi_phdr[i].p_align, tls_modid});
       break;
     }
   return 0;
 @@ -371,11 +414,11 @@ __attribute__((unused)) static void GetStaticTlsBoundary(uptr *addr, uptr *size,
   dl_iterate_phdr(CollectStaticTlsBlocks, &ranges);
   uptr len = ranges.size();
   Sort(ranges.begin(), len);
-  // Find the range with tls_modid=1. For glibc, because libc.so uses PT_TLS,
+  // Find the range with tls_modid == main_tls_modid. For glibc, because
-  // this module is guaranteed to exist and is one of the initially loaded
+  // libc.so uses PT_TLS, this module is guaranteed to exist and is one of
-  // modules.
+  // the initially loaded modules.
   uptr one = 0;
-  while (one != len && ranges[one].tls_modid != 1) ++one;
+  while (one != len && ranges[one].tls_modid != main_tls_modid) ++one;
   if (one == len) {
     // This may happen with musl if no module uses PT_TLS.
     *addr = 0;
 @@ -384,21 +427,20 @@ __attribute__((unused)) static void GetStaticTlsBoundary(uptr *addr, uptr *size,
     return;
   }
   // Find the maximum consecutive ranges. We consider two modules consecutive if
-  // the gap is smaller than the alignment. The dynamic loader places static TLS
+  // the gap is smaller than the alignment of the latter range. The dynamic
-  // blocks this way not to waste space.
+  // loader places static TLS blocks this way not to waste space.
   uptr l = one;
   *align = ranges[l].align;
-  while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l - 1].align)
+  while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l].align)
     *align = Max(*align, ranges[--l].align);
   uptr r = one + 1;
-  while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r - 1].align)
+  while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r].align)
     *align = Max(*align, ranges[r++].align);
   *addr = ranges[l].begin;
   *size = ranges[r - 1].end - ranges[l].begin;
 }
-#endif  // !SANITIZER_GO
 #endif  // (x86_64 || i386 || mips || ...) && (SANITIZER_FREEBSD ||
-        // SANITIZER_LINUX) && !SANITIZER_ANDROID
+        // SANITIZER_LINUX) && !SANITIZER_ANDROID && !SANITIZER_GO
 #if SANITIZER_NETBSD
 static struct tls_tcb * ThreadSelfTlsTcb() {
 @@ -452,7 +494,11 @@ static void GetTls(uptr *addr, uptr *size) {
 #elif SANITIZER_GLIBC && defined(__x86_64__)
   // For aarch64 and x86-64, use an O(1) approach which requires relatively
   // precise ThreadDescriptorSize. g_tls_size was initialized in InitTlsSize.
+#  if SANITIZER_X32
+  asm("mov %%fs:8,%0" : "=r"(*addr));
+#  else
   asm("mov %%fs:16,%0" : "=r"(*addr));
+#  endif
   *size = g_tls_size;
   *addr -= *size;
   *addr += ThreadDescriptorSize();
 @@ -460,6 +506,15 @@ static void GetTls(uptr *addr, uptr *size) {
   *addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
           ThreadDescriptorSize();
   *size = g_tls_size + ThreadDescriptorSize();
+#elif SANITIZER_GLIBC && defined(__loongarch__)
+#  ifdef __clang__
+  *addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) -
+          ThreadDescriptorSize();
+#  else
+  asm("or %0,$tp,$zero" : "=r"(*addr));
+  *addr -= ThreadDescriptorSize();
+#  endif
+  *size = g_tls_size + ThreadDescriptorSize();
 #elif SANITIZER_GLIBC && defined(__powerpc64__)
   // Workaround for glibc<2.25(?). 2.27 is known to not need this.
   uptr tp;
 @@ -467,7 +522,7 @@ static void GetTls(uptr *addr, uptr *size) {
   const uptr pre_tcb_size = TlsPreTcbSize();
   *addr = tp - pre_tcb_size;
   *size = g_tls_size + pre_tcb_size;
-#elif SANITIZER_FREEBSD || SANITIZER_LINUX
+#elif SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_SOLARIS
   uptr align;
   GetStaticTlsBoundary(addr, size, &align);
 #if defined(__x86_64__) || defined(__i386__) || defined(__s390__) || \
 @@ -528,10 +583,6 @@ static void GetTls(uptr *addr, uptr *size) {
       *addr = (uptr)tcb->tcb_dtv[1];
     }
   }
-#elif SANITIZER_SOLARIS
-  // FIXME
-  *addr = 0;
-  *size = 0;
 #else
 #error "Unknown OS"
 #endif
 @@ -603,6 +654,34 @@ static int AddModuleSegments(const char *module_name, dl_phdr_info *info,
       bool writable = phdr->p_flags & PF_W;
       cur_module.addAddressRange(cur_beg, cur_end, executable,
                                  writable);
+    } else if (phdr->p_type == PT_NOTE) {
+#  ifdef NT_GNU_BUILD_ID
+      uptr off = 0;
+      while (off + sizeof(ElfW(Nhdr)) < phdr->p_memsz) {
+        auto *nhdr = reinterpret_cast<const ElfW(Nhdr) *>(info->dlpi_addr +
+                                                          phdr->p_vaddr + off);
+        constexpr auto kGnuNamesz = 4;  // "GNU" with NUL-byte.
+        static_assert(kGnuNamesz % 4 == 0, "kGnuNameSize is aligned to 4.");
+        if (nhdr->n_type == NT_GNU_BUILD_ID && nhdr->n_namesz == kGnuNamesz) {
+          if (off + sizeof(ElfW(Nhdr)) + nhdr->n_namesz + nhdr->n_descsz >
+              phdr->p_memsz) {
+            // Something is very wrong, bail out instead of reading potentially
+            // arbitrary memory.
+            break;
+          }
+          const char *name =
+              reinterpret_cast<const char *>(nhdr) + sizeof(*nhdr);
+          if (internal_memcmp(name, "GNU", 3) == 0) {
+            const char *value = reinterpret_cast<const char *>(nhdr) +
+                                sizeof(*nhdr) + kGnuNamesz;
+            cur_module.setUuid(value, nhdr->n_descsz);
+            break;
+          }
+        }
+        off += sizeof(*nhdr) + RoundUpTo(nhdr->n_namesz, 4) +
+               RoundUpTo(nhdr->n_descsz, 4);
+      }
+#  endif
     }
   }
   modules->push_back(cur_module);

lib/tsan/sanitizer_common/sanitizer_linux_s390.cpp+10-4

 @@ -57,8 +57,10 @@ uptr internal_mmap(void *addr, uptr length, int prot, int flags, int fd,
 uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
                     int *parent_tidptr, void *newtls, int *child_tidptr) {
-  if (!fn || !child_stack)
+  if (!fn || !child_stack) {
-    return -EINVAL;
+    errno = EINVAL;
+    return -1;
+  }
   CHECK_EQ(0, (uptr)child_stack % 16);
   // Minimum frame size.
 #ifdef __s390x__
 @@ -71,9 +73,9 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
   // And pass parameters.
   ((unsigned long *)child_stack)[1] = (uptr)fn;
   ((unsigned long *)child_stack)[2] = (uptr)arg;
-  register long res __asm__("r2");
+  register uptr res __asm__("r2");
   register void *__cstack      __asm__("r2") = child_stack;
-  register int __flags         __asm__("r3") = flags;
+  register long __flags        __asm__("r3") = flags;
   register int * __ptidptr     __asm__("r4") = parent_tidptr;
   register int * __ctidptr     __asm__("r5") = child_tidptr;
   register void * __newtls     __asm__("r6") = newtls;
 @@ -113,6 +115,10 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg,
                          "r"(__ctidptr),
                          "r"(__newtls)
                        : "memory", "cc");
+  if (res >= (uptr)-4095) {
+    errno = -res;
+    return -1;
+  }
   return res;
 }

lib/tsan/sanitizer_common/sanitizer_local_address_space_view.h+1-1

...	@@ -17,7 +17,7 @@	...	@@ -17,7 +17,7 @@
17	// instantiated with the `LocalAddressSpaceView` type. This type is used to	17	// instantiated with the `LocalAddressSpaceView` type. This type is used to
18	// load any pointers in instance methods. This implementation is effectively	18	// load any pointers in instance methods. This implementation is effectively
19	// a no-op. When an object is to be used in an out-of-process manner it is	19	// a no-op. When an object is to be used in an out-of-process manner it is
20	// instansiated with the `RemoteAddressSpaceView` type.	20	// instantiated with the `RemoteAddressSpaceView` type.
21	//	21	//
22	// By making `AddressSpaceView` a template parameter of an object, it can	22	// By making `AddressSpaceView` a template parameter of an object, it can
23	// change its implementation at compile time which has no run time overhead.	23	// change its implementation at compile time which has no run time overhead.

lib/tsan/sanitizer_common/sanitizer_lzw.h created+159

...	@@ -0,0 +1,159 @@
1	//===-- sanitizer_lzw.h ------------------------------------------ C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Lempel–Ziv–Welch encoding/decoding
10	//
11	//===----------------------------------------------------------------------===//
12
13	#ifndef SANITIZER_LZW_H
14	#define SANITIZER_LZW_H
15
16	#include "sanitizer_dense_map.h"
17
18	namespace __sanitizer {
19
20	using LzwCodeType = u32;
21
22	template <class T, class ItIn, class ItOut>
23	ItOut LzwEncode(ItIn begin, ItIn end, ItOut out) {
24	using Substring =
25	detail::DenseMapPair<LzwCodeType /* Prefix /, T / Next input */>;
26
27	// Sentinel value for substrings of len 1.
28	static constexpr LzwCodeType kNoPrefix =
29	Min(DenseMapInfo<Substring>::getEmptyKey().first,
30	DenseMapInfo<Substring>::getTombstoneKey().first) -
31	1;
32	DenseMap<Substring, LzwCodeType> prefix_to_code;
33	{
34	// Add all substring of len 1 as initial dictionary.
35	InternalMmapVector<T> dict_len1;
36	for (auto it = begin; it != end; ++it)
37	if (prefix_to_code.try_emplace({kNoPrefix, *it}, 0).second)
38	dict_len1.push_back(*it);
39
40	// Slightly helps with later delta encoding.
41	Sort(dict_len1.data(), dict_len1.size());
42
43	// For large sizeof(T) we have to store dict_len1. Smaller types like u8 can
44	// just generate them.
45	*out = dict_len1.size();
46	++out;
47
48	for (uptr i = 0; i != dict_len1.size(); ++i) {
49	// Remap after the Sort.
50	prefix_to_code[{kNoPrefix, dict_len1[i]}] = i;
51	*out = dict_len1[i];
52	++out;
53	}
54	CHECK_EQ(prefix_to_code.size(), dict_len1.size());
55	}
56
57	if (begin == end)
58	return out;
59
60	// Main LZW encoding loop.
61	LzwCodeType match = prefix_to_code.find({kNoPrefix, *begin})->second;
62	++begin;
63	for (auto it = begin; it != end; ++it) {
64	// Extend match with the new item.
65	auto ins = prefix_to_code.try_emplace({match, *it}, prefix_to_code.size());
66	if (ins.second) {
67	// This is a new substring, but emit the code for the current match
68	// (before extend). This allows LZW decoder to recover the dictionary.
69	*out = match;
70	++out;
71	// Reset the match to a single item, which must be already in the map.
72	match = prefix_to_code.find({kNoPrefix, *it})->second;
73	} else {
74	// Already known, use as the current match.
75	match = ins.first->second;
76	}
77	}
78
79	*out = match;
80	++out;
81
82	return out;
83	}
84
85	template <class T, class ItIn, class ItOut>
86	ItOut LzwDecode(ItIn begin, ItIn end, ItOut out) {
87	if (begin == end)
88	return out;
89
90	// Load dictionary of len 1 substrings. Theses correspont to lowest codes.
91	InternalMmapVector<T> dict_len1(*begin);
92	++begin;
93
94	if (begin == end)
95	return out;
96
97	for (auto& v : dict_len1) {
98	v = *begin;
99	++begin;
100	}
101
102	// Substrings of len 2 and up. Indexes are shifted because [0,
103	// dict_len1.size()) stored in dict_len1. Substings get here after being
104	// emitted to the output, so we can use output position.
105	InternalMmapVector<detail::DenseMapPair<ItOut /* begin. /, ItOut / end */>>
106	code_to_substr;
107
108	// Copies already emitted substrings into the output again.
109	auto copy = [&code_to_substr, &dict_len1](LzwCodeType code, ItOut out) {
110	if (code < dict_len1.size()) {
111	*out = dict_len1[code];
112	++out;
113	return out;
114	}
115	const auto& s = code_to_substr[code - dict_len1.size()];
116
117	for (ItOut it = s.first; it != s.second; ++it, ++out) out = it;
118	return out;
119	};
120
121	// Returns lens of the substring with the given code.
122	auto code_to_len = [&code_to_substr, &dict_len1](LzwCodeType code) -> uptr {
123	if (code < dict_len1.size())
124	return 1;
125	const auto& s = code_to_substr[code - dict_len1.size()];
126	return s.second - s.first;
127	};
128
129	// Main LZW decoding loop.
130	LzwCodeType prev_code = *begin;
131	++begin;
132	out = copy(prev_code, out);
133	for (auto it = begin; it != end; ++it) {
134	LzwCodeType code = *it;
135	auto start = out;
136	if (code == dict_len1.size() + code_to_substr.size()) {
137	// Special LZW case. The code is not in the dictionary yet. This is
138	// possible only when the new substring is the same as previous one plus
139	// the first item of the previous substring. We can emit that in two
140	// steps.
141	out = copy(prev_code, out);
142	out = start;
143	++out;
144	} else {
145	out = copy(code, out);
146	}
147
148	// Every time encoded emits the code, it also creates substing of len + 1
149	// including the first item of the just emmited substring. Do the same here.
150	uptr len = code_to_len(prev_code);
151	code_to_substr.push_back({start - len, start + 1});
152
153	prev_code = code;
154	}
155	return out;
156	}
157
158	} // namespace __sanitizer
159	#endif

lib/tsan/sanitizer_common/sanitizer_mac.cpp+229-210

...	@@ -11,80 +11,82 @@	...	@@ -11,80 +11,82 @@
11	//===----------------------------------------------------------------------===//	11	//===----------------------------------------------------------------------===//
12		12
13	#include "sanitizer_platform.h"	13	#include "sanitizer_platform.h"
14	#if SANITIZER_MAC	14	#if SANITIZER_APPLE
15	#include "sanitizer_mac.h"	15	# include "interception/interception.h"
16	#include "interception/interception.h"	16	# include "sanitizer_mac.h"
17		17
18	// Use 64-bit inodes in file operations. ASan does not support OS X 10.5, so	18	// Use 64-bit inodes in file operations. ASan does not support OS X 10.5, so
19	// the clients will most certainly use 64-bit ones as well.	19	// the clients will most certainly use 64-bit ones as well.
20	#ifndef _DARWIN_USE_64_BIT_INODE	20	# ifndef _DARWIN_USE_64_BIT_INODE
21	#define _DARWIN_USE_64_BIT_INODE 1	21	# define _DARWIN_USE_64_BIT_INODE 1
22	#endif	22	# endif
23	#include <stdio.h>	23	# include <stdio.h>
24		24
25	#include "sanitizer_common.h"	25	# include "sanitizer_common.h"
26	#include "sanitizer_file.h"	26	# include "sanitizer_file.h"
27	#include "sanitizer_flags.h"	27	# include "sanitizer_flags.h"
28	#include "sanitizer_internal_defs.h"	28	# include "sanitizer_interface_internal.h"
29	#include "sanitizer_libc.h"	29	# include "sanitizer_internal_defs.h"
30	#include "sanitizer_platform_limits_posix.h"	30	# include "sanitizer_libc.h"
31	#include "sanitizer_procmaps.h"	31	# include "sanitizer_platform_limits_posix.h"
32	#include "sanitizer_ptrauth.h"	32	# include "sanitizer_procmaps.h"
33		33	# include "sanitizer_ptrauth.h"
34	#if !SANITIZER_IOS	34
35	#include <crt_externs.h> // for _NSGetEnviron	35	# if !SANITIZER_IOS
36	#else	36	# include <crt_externs.h> // for _NSGetEnviron
		37	# else
37	extern char **environ;	38	extern char **environ;
38	#endif	39	# endif
39		40
40	#if defined(__has_include) && __has_include(<os/trace.h>)	41	# if defined(__has_include) && __has_include(<os/trace.h>)
41	#define SANITIZER_OS_TRACE 1	42	# define SANITIZER_OS_TRACE 1
42	#include <os/trace.h>	43	# include <os/trace.h>
43	#else	44	# else
44	#define SANITIZER_OS_TRACE 0	45	# define SANITIZER_OS_TRACE 0
45	#endif	46	# endif
46		47
47	// import new crash reporting api	48	// import new crash reporting api
48	#if defined(__has_include) && __has_include(<CrashReporterClient.h>)	49	# if defined(__has_include) && __has_include(<CrashReporterClient.h>)
49	#define HAVE_CRASHREPORTERCLIENT_H 1	50	# define HAVE_CRASHREPORTERCLIENT_H 1
50	#include <CrashReporterClient.h>	51	# include <CrashReporterClient.h>
51	#else	52	# else
52	#define HAVE_CRASHREPORTERCLIENT_H 0	53	# define HAVE_CRASHREPORTERCLIENT_H 0
53	#endif	54	# endif
54		55
55	#if !SANITIZER_IOS	56	# if !SANITIZER_IOS
56	#include <crt_externs.h> // for _NSGetArgv and _NSGetEnviron	57	# include <crt_externs.h> // for _NSGetArgv and _NSGetEnviron
57	#else	58	# else
58	extern "C" {	59	extern "C" {
59	extern char ***_NSGetArgv(void);	60	extern char ***_NSGetArgv(void);
60	}	61	}
61	#endif	62	# endif
62		63
63	#include <asl.h>	64	# include <asl.h>
64	#include <dlfcn.h> // for dladdr()	65	# include <dlfcn.h> // for dladdr()
65	#include <errno.h>	66	# include <errno.h>
66	#include <fcntl.h>	67	# include <fcntl.h>
67	#include <libkern/OSAtomic.h>	68	# include <libkern/OSAtomic.h>
68	#include <mach-o/dyld.h>	69	# include <mach-o/dyld.h>
69	#include <mach/mach.h>	70	# include <mach/mach.h>
70	#include <mach/mach_time.h>	71	# include <mach/mach_time.h>
71	#include <mach/vm_statistics.h>	72	# include <mach/vm_statistics.h>
72	#include <malloc/malloc.h>	73	# include <malloc/malloc.h>
73	#include <os/log.h>	74	# include <os/log.h>
74	#include <pthread.h>	75	# include <pthread.h>
75	#include <sched.h>	76	# include <pthread/introspection.h>
76	#include <signal.h>	77	# include <sched.h>
77	#include <spawn.h>	78	# include <signal.h>
78	#include <stdlib.h>	79	# include <spawn.h>
79	#include <sys/ioctl.h>	80	# include <stdlib.h>
80	#include <sys/mman.h>	81	# include <sys/ioctl.h>
81	#include <sys/resource.h>	82	# include <sys/mman.h>
82	#include <sys/stat.h>	83	# include <sys/resource.h>
83	#include <sys/sysctl.h>	84	# include <sys/stat.h>
84	#include <sys/types.h>	85	# include <sys/sysctl.h>
85	#include <sys/wait.h>	86	# include <sys/types.h>
86	#include <unistd.h>	87	# include <sys/wait.h>
87	#include <util.h>	88	# include <unistd.h>
		89	# include <util.h>
88		90
89	// From <crt_externs.h>, but we don't have that file on iOS.	91	// From <crt_externs.h>, but we don't have that file on iOS.
90	extern "C" {	92	extern "C" {
...	@@ -265,30 +267,32 @@ int internal_sysctlbyname(const char sname, void oldp, uptr *oldlenp,	...	@@ -265,30 +267,32 @@ int internal_sysctlbyname(const char sname, void oldp, uptr *oldlenp,
265		267
266	static fd_t internal_spawn_impl(const char argv[], const char envp[],	268	static fd_t internal_spawn_impl(const char argv[], const char envp[],
267	pid_t *pid) {	269	pid_t *pid) {
268	fd_t master_fd = kInvalidFd;	270	fd_t primary_fd = kInvalidFd;
269	fd_t slave_fd = kInvalidFd;	271	fd_t secondary_fd = kInvalidFd;
270		272
271	auto fd_closer = at_scope_exit([&] {	273	auto fd_closer = at_scope_exit([&] {
272	internal_close(master_fd);	274	internal_close(primary_fd);
273	internal_close(slave_fd);	275	internal_close(secondary_fd);
274	});	276	});
275		277
276	// We need a new pseudoterminal to avoid buffering problems. The 'atos' tool	278	// We need a new pseudoterminal to avoid buffering problems. The 'atos' tool
277	// in particular detects when it's talking to a pipe and forgets to flush the	279	// in particular detects when it's talking to a pipe and forgets to flush the
278	// output stream after sending a response.	280	// output stream after sending a response.
279	master_fd = posix_openpt(O_RDWR);	281	primary_fd = posix_openpt(O_RDWR);
280	if (master_fd == kInvalidFd) return kInvalidFd;	282	if (primary_fd == kInvalidFd)
		283	return kInvalidFd;
281		284
282	int res = grantpt(master_fd) \|\| unlockpt(master_fd);	285	int res = grantpt(primary_fd) \|\| unlockpt(primary_fd);
283	if (res != 0) return kInvalidFd;	286	if (res != 0) return kInvalidFd;
284		287
285	// Use TIOCPTYGNAME instead of ptsname() to avoid threading problems.	288	// Use TIOCPTYGNAME instead of ptsname() to avoid threading problems.
286	char slave_pty_name[128];	289	char secondary_pty_name[128];
287	res = ioctl(master_fd, TIOCPTYGNAME, slave_pty_name);	290	res = ioctl(primary_fd, TIOCPTYGNAME, secondary_pty_name);
288	if (res == -1) return kInvalidFd;	291	if (res == -1) return kInvalidFd;
289		292
290	slave_fd = internal_open(slave_pty_name, O_RDWR);	293	secondary_fd = internal_open(secondary_pty_name, O_RDWR);
291	if (slave_fd == kInvalidFd) return kInvalidFd;	294	if (secondary_fd == kInvalidFd)
		295	return kInvalidFd;
292		296
293	// File descriptor actions	297	// File descriptor actions
294	posix_spawn_file_actions_t acts;	298	posix_spawn_file_actions_t acts;
...	@@ -299,9 +303,9 @@ static fd_t internal_spawn_impl(const char argv[], const char envp[],	...	@@ -299,9 +303,9 @@ static fd_t internal_spawn_impl(const char argv[], const char envp[],
299	posix_spawn_file_actions_destroy(&acts);	303	posix_spawn_file_actions_destroy(&acts);
300	});	304	});
301		305
302	res = posix_spawn_file_actions_adddup2(&acts, slave_fd, STDIN_FILENO) \|\|	306	res = posix_spawn_file_actions_adddup2(&acts, secondary_fd, STDIN_FILENO) \|\|
303	posix_spawn_file_actions_adddup2(&acts, slave_fd, STDOUT_FILENO) \|\|	307	posix_spawn_file_actions_adddup2(&acts, secondary_fd, STDOUT_FILENO) \|\|
304	posix_spawn_file_actions_addclose(&acts, slave_fd);	308	posix_spawn_file_actions_addclose(&acts, secondary_fd);
305	if (res != 0) return kInvalidFd;	309	if (res != 0) return kInvalidFd;
306		310
307	// Spawn attributes	311	// Spawn attributes
...	@@ -326,14 +330,14 @@ static fd_t internal_spawn_impl(const char argv[], const char envp[],	...	@@ -326,14 +330,14 @@ static fd_t internal_spawn_impl(const char argv[], const char envp[],
326		330
327	// Disable echo in the new terminal, disable CR.	331	// Disable echo in the new terminal, disable CR.
328	struct termios termflags;	332	struct termios termflags;
329	tcgetattr(master_fd, &termflags);	333	tcgetattr(primary_fd, &termflags);
330	termflags.c_oflag &= ~ONLCR;	334	termflags.c_oflag &= ~ONLCR;
331	termflags.c_lflag &= ~ECHO;	335	termflags.c_lflag &= ~ECHO;
332	tcsetattr(master_fd, TCSANOW, &termflags);	336	tcsetattr(primary_fd, TCSANOW, &termflags);
333		337
334	// On success, do not close master_fd on scope exit.	338	// On success, do not close primary_fd on scope exit.
335	fd_t fd = master_fd;	339	fd_t fd = primary_fd;
336	master_fd = kInvalidFd;	340	primary_fd = kInvalidFd;
337		341
338	return fd;	342	return fd;
339	}	343	}
...	@@ -390,6 +394,13 @@ bool FileExists(const char *filename) {	...	@@ -390,6 +394,13 @@ bool FileExists(const char *filename) {
390	return S_ISREG(st.st_mode);	394	return S_ISREG(st.st_mode);
391	}	395	}
392		396
		397	bool DirExists(const char *path) {
		398	struct stat st;
		399	if (stat(path, &st))
		400	return false;
		401	return S_ISDIR(st.st_mode);
		402	}
		403
393	tid_t GetTid() {	404	tid_t GetTid() {
394	tid_t tid;	405	tid_t tid;
395	pthread_threadid_np(nullptr, &tid);	406	pthread_threadid_np(nullptr, &tid);
...	@@ -516,25 +527,6 @@ void FutexWait(atomic_uint32_t *p, u32 cmp) {	...	@@ -516,25 +527,6 @@ void FutexWait(atomic_uint32_t *p, u32 cmp) {
516		527
517	void FutexWake(atomic_uint32_t *p, u32 count) {}	528	void FutexWake(atomic_uint32_t *p, u32 count) {}
518		529
519	BlockingMutex::BlockingMutex() {
520	internal_memset(this, 0, sizeof(*this));
521	}
522
523	void BlockingMutex::Lock() {
524	CHECK(sizeof(OSSpinLock) <= sizeof(opaque_storage_));
525	CHECK_EQ(OS_SPINLOCK_INIT, 0);
526	CHECK_EQ(owner_, 0);
527	OSSpinLockLock((OSSpinLock*)&opaque_storage_);
528	}
529
530	void BlockingMutex::Unlock() {
531	OSSpinLockUnlock((OSSpinLock*)&opaque_storage_);
532	}
533
534	void BlockingMutex::CheckLocked() const {
535	CHECK_NE((const OSSpinLock)&opaque_storage_, 0);
536	}
537
538	u64 NanoTime() {	530	u64 NanoTime() {
539	timeval tv;	531	timeval tv;
540	internal_memset(&tv, 0, sizeof(tv));	532	internal_memset(&tv, 0, sizeof(tv));
...	@@ -562,6 +554,9 @@ uptr TlsBaseAddr() {	...	@@ -562,6 +554,9 @@ uptr TlsBaseAddr() {
562	asm("movq %%gs:0,%0" : "=r"(segbase));	554	asm("movq %%gs:0,%0" : "=r"(segbase));
563	#elif defined(__i386__)	555	#elif defined(__i386__)
564	asm("movl %%gs:0,%0" : "=r"(segbase));	556	asm("movl %%gs:0,%0" : "=r"(segbase));
		557	#elif defined(__aarch64__)
		558	asm("mrs %x0, tpidrro_el0" : "=r"(segbase));
		559	segbase &= 0x07ul; // clearing lower bits, cpu id stored there
565	#endif	560	#endif
566	return segbase;	561	return segbase;
567	}	562	}
...	@@ -784,8 +779,8 @@ void internal_start_thread(void (func)(void arg), void *arg) {	...	@@ -784,8 +779,8 @@ void internal_start_thread(void (func)(void arg), void *arg) {
784	void internal_join_thread(void *th) { pthread_join((pthread_t)th, 0); }	779	void internal_join_thread(void *th) { pthread_join((pthread_t)th, 0); }
785		780
786	#if !SANITIZER_GO	781	#if !SANITIZER_GO
787	static BlockingMutex syslog_lock(LINKER_INITIALIZED);	782	static Mutex syslog_lock;
788	#endif	783	# endif
789		784
790	void WriteOneLineToSyslog(const char *s) {	785	void WriteOneLineToSyslog(const char *s) {
791	#if !SANITIZER_GO	786	#if !SANITIZER_GO
...	@@ -800,7 +795,7 @@ void WriteOneLineToSyslog(const char *s) {	...	@@ -800,7 +795,7 @@ void WriteOneLineToSyslog(const char *s) {
800		795
801	// buffer to store crash report application information	796	// buffer to store crash report application information
802	static char crashreporter_info_buff[__sanitizer::kErrorMessageBufferSize] = {};	797	static char crashreporter_info_buff[__sanitizer::kErrorMessageBufferSize] = {};
803	static BlockingMutex crashreporter_info_mutex(LINKER_INITIALIZED);	798	static Mutex crashreporter_info_mutex;
804		799
805	extern "C" {	800	extern "C" {
806	// Integrate with crash reporter libraries.	801	// Integrate with crash reporter libraries.
...	@@ -830,7 +825,7 @@ asm(".desc ___crashreporter_info__, 0x10");	...	@@ -830,7 +825,7 @@ asm(".desc ___crashreporter_info__, 0x10");
830	} // extern "C"	825	} // extern "C"
831		826
832	static void CRAppendCrashLogMessage(const char *msg) {	827	static void CRAppendCrashLogMessage(const char *msg) {
833	BlockingMutexLock l(&crashreporter_info_mutex);	828	Lock l(&crashreporter_info_mutex);
834	internal_strlcat(crashreporter_info_buff, msg,	829	internal_strlcat(crashreporter_info_buff, msg,
835	sizeof(crashreporter_info_buff));	830	sizeof(crashreporter_info_buff));
836	#if HAVE_CRASHREPORTERCLIENT_H	831	#if HAVE_CRASHREPORTERCLIENT_H
...	@@ -874,7 +869,7 @@ void LogFullErrorReport(const char *buffer) {	...	@@ -874,7 +869,7 @@ void LogFullErrorReport(const char *buffer) {
874	// the reporting thread holds the thread registry mutex, and asl_log waits	869	// the reporting thread holds the thread registry mutex, and asl_log waits
875	// for GCD to dispatch a new thread, the process will deadlock, because the	870	// for GCD to dispatch a new thread, the process will deadlock, because the
876	// pthread_create wrapper needs to acquire the lock as well.	871	// pthread_create wrapper needs to acquire the lock as well.
877	BlockingMutexLock l(&syslog_lock);	872	Lock l(&syslog_lock);
878	if (common_flags()->log_to_syslog)	873	if (common_flags()->log_to_syslog)
879	WriteToSyslog(buffer);	874	WriteToSyslog(buffer);
880		875
...	@@ -885,9 +880,12 @@ void LogFullErrorReport(const char *buffer) {	...	@@ -885,9 +880,12 @@ void LogFullErrorReport(const char *buffer) {
885	SignalContext::WriteFlag SignalContext::GetWriteFlag() const {	880	SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
886	#if defined(__x86_64__) \|\| defined(__i386__)	881	#if defined(__x86_64__) \|\| defined(__i386__)
887	ucontext_t ucontext = static_cast<ucontext_t>(context);	882	ucontext_t ucontext = static_cast<ucontext_t>(context);
888	return ucontext->uc_mcontext->__es.__err & 2 /T_PF_WRITE/ ? WRITE : READ;	883	return ucontext->uc_mcontext->__es.__err & 2 /T_PF_WRITE/ ? Write : Read;
		884	#elif defined(__arm64__)
		885	ucontext_t ucontext = static_cast<ucontext_t>(context);
		886	return ucontext->uc_mcontext->__es.__esr & 0x40 /ISS_DA_WNR/ ? Write : Read;
889	#else	887	#else
890	return UNKNOWN;	888	return Unknown;
891	#endif	889	#endif
892	}	890	}
893		891
...	@@ -902,18 +900,14 @@ bool SignalContext::IsTrueFaultingAddress() const {	...	@@ -902,18 +900,14 @@ bool SignalContext::IsTrueFaultingAddress() const {
902	(uptr)ptrauth_strip( \	900	(uptr)ptrauth_strip( \
903	(void *)arm_thread_state64_get_##r(ucontext->uc_mcontext->__ss), 0)	901	(void *)arm_thread_state64_get_##r(ucontext->uc_mcontext->__ss), 0)
904	#else	902	#else
905	#define AARCH64_GET_REG(r) ucontext->uc_mcontext->__ss.__##r	903	#define AARCH64_GET_REG(r) (uptr)ucontext->uc_mcontext->__ss.__##r
906	#endif	904	#endif
907		905
908	static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {	906	static void GetPcSpBp(void context, uptr pc, uptr sp, uptr bp) {
909	ucontext_t ucontext = (ucontext_t)context;	907	ucontext_t ucontext = (ucontext_t)context;
910	# if defined(__aarch64__)	908	# if defined(__aarch64__)
911	*pc = AARCH64_GET_REG(pc);	909	*pc = AARCH64_GET_REG(pc);
912	# if defined(__IPHONE_8_0) && __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_8_0
913	*bp = AARCH64_GET_REG(fp);	910	*bp = AARCH64_GET_REG(fp);
914	# else
915	*bp = AARCH64_GET_REG(lr);
916	# endif
917	*sp = AARCH64_GET_REG(sp);	911	*sp = AARCH64_GET_REG(sp);
918	# elif defined(__x86_64__)	912	# elif defined(__x86_64__)
919	*pc = ucontext->uc_mcontext->__ss.__rip;	913	*pc = ucontext->uc_mcontext->__ss.__rip;
...	@@ -950,6 +944,9 @@ static void DisableMmapExcGuardExceptions() {	...	@@ -950,6 +944,9 @@ static void DisableMmapExcGuardExceptions() {
950	set_behavior(mach_task_self(), task_exc_guard_none);	944	set_behavior(mach_task_self(), task_exc_guard_none);
951	}	945	}
952		946
		947	static void VerifyInterceptorsWorking();
		948	static void StripEnv();
		949
953	void InitializePlatformEarly() {	950	void InitializePlatformEarly() {
954	// Only use xnu_fast_mmap when on x86_64 and the kernel supports it.	951	// Only use xnu_fast_mmap when on x86_64 and the kernel supports it.
955	use_xnu_fast_mmap =	952	use_xnu_fast_mmap =
...	@@ -960,17 +957,54 @@ void InitializePlatformEarly() {	...	@@ -960,17 +957,54 @@ void InitializePlatformEarly() {
960	#endif	957	#endif
961	if (GetDarwinKernelVersion() >= DarwinKernelVersion(19, 0))	958	if (GetDarwinKernelVersion() >= DarwinKernelVersion(19, 0))
962	DisableMmapExcGuardExceptions();	959	DisableMmapExcGuardExceptions();
		960
		961	# if !SANITIZER_GO
		962	MonotonicNanoTime(); // Call to initialize mach_timebase_info
		963	VerifyInterceptorsWorking();
		964	StripEnv();
		965	# endif
963	}	966	}
964		967
965	#if !SANITIZER_GO	968	#if !SANITIZER_GO
966	static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES";	969	static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES";
967	LowLevelAllocator allocator_for_env;	970	LowLevelAllocator allocator_for_env;
968		971
		972	static bool ShouldCheckInterceptors() {
		973	// Restrict "interceptors working?" check to ASan and TSan.
		974	const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer"};
		975	size_t count = sizeof(sanitizer_names) / sizeof(sanitizer_names[0]);
		976	for (size_t i = 0; i < count; i++) {
		977	if (internal_strcmp(sanitizer_names[i], SanitizerToolName) == 0)
		978	return true;
		979	}
		980	return false;
		981	}
		982
		983	static void VerifyInterceptorsWorking() {
		984	if (!common_flags()->verify_interceptors \|\| !ShouldCheckInterceptors())
		985	return;
		986
		987	// Verify that interceptors really work. We'll use dlsym to locate
		988	// "puts", if interceptors are working, it should really point to
		989	// "wrap_puts" within our own dylib.
		990	Dl_info info_puts, info_runtime;
		991	RAW_CHECK(dladdr(dlsym(RTLD_DEFAULT, "puts"), &info_puts));
		992	RAW_CHECK(dladdr((void *)&VerifyInterceptorsWorking, &info_runtime));
		993	if (internal_strcmp(info_puts.dli_fname, info_runtime.dli_fname) != 0) {
		994	Report(
		995	"ERROR: Interceptors are not working. This may be because %s is "
		996	"loaded too late (e.g. via dlopen). Please launch the executable "
		997	"with:\n%s=%s\n",
		998	SanitizerToolName, kDyldInsertLibraries, info_runtime.dli_fname);
		999	RAW_CHECK("interceptors not installed" && 0);
		1000	}
		1001	}
		1002
969	// Change the value of the env var \|name\|, leaking the original value.	1003	// Change the value of the env var \|name\|, leaking the original value.
970	// If \|name_value\| is NULL, the variable is deleted from the environment,	1004	// If \|name_value\| is NULL, the variable is deleted from the environment,
971	// otherwise the corresponding "NAME=value" string is replaced with	1005	// otherwise the corresponding "NAME=value" string is replaced with
972	// \|name_value\|.	1006	// \|name_value\|.
973	void LeakyResetEnv(const char name, const char name_value) {	1007	static void LeakyResetEnv(const char name, const char name_value) {
974	char **env = GetEnviron();	1008	char **env = GetEnviron();
975	uptr name_len = internal_strlen(name);	1009	uptr name_len = internal_strlen(name);
976	while (*env != 0) {	1010	while (*env != 0) {
...	@@ -995,100 +1029,28 @@ void LeakyResetEnv(const char name, const char name_value) {	...	@@ -995,100 +1029,28 @@ void LeakyResetEnv(const char name, const char name_value) {
995	}	1029	}
996	}	1030	}
997		1031
998	SANITIZER_WEAK_CXX_DEFAULT_IMPL	1032	static void StripEnv() {
999	bool ReexecDisabled() {	1033	if (!common_flags()->strip_env)
1000	return false;	1034	return;
1001	}
1002
1003	static bool DyldNeedsEnvVariable() {
1004	// If running on OS X 10.11+ or iOS 9.0+, dyld will interpose even if
1005	// DYLD_INSERT_LIBRARIES is not set.
1006	return GetMacosAlignedVersion() < MacosVersion(10, 11);
1007	}
1008
1009	void MaybeReexec() {
1010	// FIXME: This should really live in some "InitializePlatform" method.
1011	MonotonicNanoTime();
1012		1035
1013	if (ReexecDisabled()) return;	1036	char *dyld_insert_libraries =
		1037	const_cast<char *>(GetEnv(kDyldInsertLibraries));
		1038	if (!dyld_insert_libraries)
		1039	return;
1014		1040
1015	// Make sure the dynamic runtime library is preloaded so that the
1016	// wrappers work. If it is not, set DYLD_INSERT_LIBRARIES and re-exec
1017	// ourselves.
1018	Dl_info info;	1041	Dl_info info;
1019	RAW_CHECK(dladdr((void*)((uptr)&__sanitizer_report_error_summary), &info));	1042	RAW_CHECK(dladdr((void *)&StripEnv, &info));
1020	char *dyld_insert_libraries =
1021	const_cast<char*>(GetEnv(kDyldInsertLibraries));
1022	uptr old_env_len = dyld_insert_libraries ?
1023	internal_strlen(dyld_insert_libraries) : 0;
1024	uptr fname_len = internal_strlen(info.dli_fname);
1025	const char *dylib_name = StripModuleName(info.dli_fname);	1043	const char *dylib_name = StripModuleName(info.dli_fname);
1026	uptr dylib_name_len = internal_strlen(dylib_name);	1044	bool lib_is_in_env = internal_strstr(dyld_insert_libraries, dylib_name);
1027
1028	bool lib_is_in_env = dyld_insert_libraries &&
1029	internal_strstr(dyld_insert_libraries, dylib_name);
1030	if (DyldNeedsEnvVariable() && !lib_is_in_env) {
1031	// DYLD_INSERT_LIBRARIES is not set or does not contain the runtime
1032	// library.
1033	InternalMmapVector<char> program_name(1024);
1034	uint32_t buf_size = program_name.size();
1035	_NSGetExecutablePath(program_name.data(), &buf_size);
1036	char new_env = const_cast<char>(info.dli_fname);
1037	if (dyld_insert_libraries) {
1038	// Append the runtime dylib name to the existing value of
1039	// DYLD_INSERT_LIBRARIES.
1040	new_env = (char*)allocator_for_env.Allocate(old_env_len + fname_len + 2);
1041	internal_strncpy(new_env, dyld_insert_libraries, old_env_len);
1042	new_env[old_env_len] = ':';
1043	// Copy fname_len and add a trailing zero.
1044	internal_strncpy(new_env + old_env_len + 1, info.dli_fname,
1045	fname_len + 1);
1046	// Ok to use setenv() since the wrappers don't depend on the value of
1047	// asan_inited.
1048	setenv(kDyldInsertLibraries, new_env, /overwrite/1);
1049	} else {
1050	// Set DYLD_INSERT_LIBRARIES equal to the runtime dylib name.
1051	setenv(kDyldInsertLibraries, info.dli_fname, /overwrite/0);
1052	}
1053	VReport(1, "exec()-ing the program with\n");
1054	VReport(1, "%s=%s\n", kDyldInsertLibraries, new_env);
1055	VReport(1, "to enable wrappers.\n");
1056	execv(program_name.data(), *_NSGetArgv());
1057
1058	// We get here only if execv() failed.
1059	Report("ERROR: The process is launched without DYLD_INSERT_LIBRARIES, "
1060	"which is required for the sanitizer to work. We tried to set the "
1061	"environment variable and re-execute itself, but execv() failed, "
1062	"possibly because of sandbox restrictions. Make sure to launch the "
1063	"executable with:\n%s=%s\n", kDyldInsertLibraries, new_env);
1064	RAW_CHECK("execv failed" && 0);
1065	}
1066
1067	// Verify that interceptors really work. We'll use dlsym to locate
1068	// "pthread_create", if interceptors are working, it should really point to
1069	// "wrap_pthread_create" within our own dylib.
1070	Dl_info info_pthread_create;
1071	void *dlopen_addr = dlsym(RTLD_DEFAULT, "pthread_create");
1072	RAW_CHECK(dladdr(dlopen_addr, &info_pthread_create));
1073	if (internal_strcmp(info.dli_fname, info_pthread_create.dli_fname) != 0) {
1074	Report(
1075	"ERROR: Interceptors are not working. This may be because %s is "
1076	"loaded too late (e.g. via dlopen). Please launch the executable "
1077	"with:\n%s=%s\n",
1078	SanitizerToolName, kDyldInsertLibraries, info.dli_fname);
1079	RAW_CHECK("interceptors not installed" && 0);
1080	}
1081
1082	if (!lib_is_in_env)	1045	if (!lib_is_in_env)
1083	return;	1046	return;
1084		1047
1085	if (!common_flags()->strip_env)
1086	return;
1087
1088	// DYLD_INSERT_LIBRARIES is set and contains the runtime library. Let's remove	1048	// DYLD_INSERT_LIBRARIES is set and contains the runtime library. Let's remove
1089	// the dylib from the environment variable, because interceptors are installed	1049	// the dylib from the environment variable, because interceptors are installed
1090	// and we don't want our children to inherit the variable.	1050	// and we don't want our children to inherit the variable.
1091		1051
		1052	uptr old_env_len = internal_strlen(dyld_insert_libraries);
		1053	uptr dylib_name_len = internal_strlen(dylib_name);
1092	uptr env_name_len = internal_strlen(kDyldInsertLibraries);	1054	uptr env_name_len = internal_strlen(kDyldInsertLibraries);
1093	// Allocate memory to hold the previous env var name, its value, the '='	1055	// Allocate memory to hold the previous env var name, its value, the '='
1094	// sign and the '\0' char.	1056	// sign and the '\0' char.
...	@@ -1237,7 +1199,7 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,	...	@@ -1237,7 +1199,7 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
1237		1199
1238	uptr largest_gap_found = 0;	1200	uptr largest_gap_found = 0;
1239	uptr max_occupied_addr = 0;	1201	uptr max_occupied_addr = 0;
1240	VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);	1202	VReport(2, "FindDynamicShadowStart, space_size = %p\n", (void *)space_size);
1241	uptr shadow_start =	1203	uptr shadow_start =
1242	FindAvailableMemoryRange(space_size, alignment, granularity,	1204	FindAvailableMemoryRange(space_size, alignment, granularity,
1243	&largest_gap_found, &max_occupied_addr);	1205	&largest_gap_found, &max_occupied_addr);
...	@@ -1246,20 +1208,21 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,	...	@@ -1246,20 +1208,21 @@ uptr MapDynamicShadow(uptr shadow_size_bytes, uptr shadow_scale,
1246	VReport(	1208	VReport(
1247	2,	1209	2,
1248	"Shadow doesn't fit, largest_gap_found = %p, max_occupied_addr = %p\n",	1210	"Shadow doesn't fit, largest_gap_found = %p, max_occupied_addr = %p\n",
1249	largest_gap_found, max_occupied_addr);	1211	(void )largest_gap_found, (void )max_occupied_addr);
1250	uptr new_max_vm = RoundDownTo(largest_gap_found << shadow_scale, alignment);	1212	uptr new_max_vm = RoundDownTo(largest_gap_found << shadow_scale, alignment);
1251	if (new_max_vm < max_occupied_addr) {	1213	if (new_max_vm < max_occupied_addr) {
1252	Report("Unable to find a memory range for dynamic shadow.\n");	1214	Report("Unable to find a memory range for dynamic shadow.\n");
1253	Report(	1215	Report(
1254	"space_size = %p, largest_gap_found = %p, max_occupied_addr = %p, "	1216	"space_size = %p, largest_gap_found = %p, max_occupied_addr = %p, "
1255	"new_max_vm = %p\n",	1217	"new_max_vm = %p\n",
1256	space_size, largest_gap_found, max_occupied_addr, new_max_vm);	1218	(void )space_size, (void )largest_gap_found,
		1219	(void )max_occupied_addr, (void )new_max_vm);
1257	CHECK(0 && "cannot place shadow");	1220	CHECK(0 && "cannot place shadow");
1258	}	1221	}
1259	RestrictMemoryToMaxAddress(new_max_vm);	1222	RestrictMemoryToMaxAddress(new_max_vm);
1260	high_mem_end = new_max_vm - 1;	1223	high_mem_end = new_max_vm - 1;
1261	space_size = (high_mem_end >> shadow_scale) + left_padding;	1224	space_size = (high_mem_end >> shadow_scale) + left_padding;
1262	VReport(2, "FindDynamicShadowStart, space_size = %p\n", space_size);	1225	VReport(2, "FindDynamicShadowStart, space_size = %p\n", (void *)space_size);
1263	shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity,	1226	shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity,
1264	nullptr, nullptr);	1227	nullptr, nullptr);
1265	if (shadow_start == 0) {	1228	if (shadow_start == 0) {
...	@@ -1288,6 +1251,7 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,	...	@@ -1288,6 +1251,7 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
1288	mach_vm_address_t start_address =	1251	mach_vm_address_t start_address =
1289	(SANITIZER_WORDSIZE == 32) ? 0x000000001000 : 0x000100000000;	1252	(SANITIZER_WORDSIZE == 32) ? 0x000000001000 : 0x000100000000;
1290		1253
		1254	const mach_vm_address_t max_vm_address = GetMaxVirtualAddress() + 1;
1291	mach_vm_address_t address = start_address;	1255	mach_vm_address_t address = start_address;
1292	mach_vm_address_t free_begin = start_address;	1256	mach_vm_address_t free_begin = start_address;
1293	kern_return_t kr = KERN_SUCCESS;	1257	kern_return_t kr = KERN_SUCCESS;
...	@@ -1302,7 +1266,7 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,	...	@@ -1302,7 +1266,7 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
1302	(vm_region_info_t)&vminfo, &count);	1266	(vm_region_info_t)&vminfo, &count);
1303	if (kr == KERN_INVALID_ADDRESS) {	1267	if (kr == KERN_INVALID_ADDRESS) {
1304	// No more regions beyond "address", consider the gap at the end of VM.	1268	// No more regions beyond "address", consider the gap at the end of VM.
1305	address = GetMaxVirtualAddress() + 1;	1269	address = max_vm_address;
1306	vmsize = 0;	1270	vmsize = 0;
1307	} else {	1271	} else {
1308	if (max_occupied_addr) *max_occupied_addr = address + vmsize;	1272	if (max_occupied_addr) *max_occupied_addr = address + vmsize;
...	@@ -1310,7 +1274,7 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,	...	@@ -1310,7 +1274,7 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
1310	if (free_begin != address) {	1274	if (free_begin != address) {
1311	// We found a free region [free_begin..address-1].	1275	// We found a free region [free_begin..address-1].
1312	uptr gap_start = RoundUpTo((uptr)free_begin + left_padding, alignment);	1276	uptr gap_start = RoundUpTo((uptr)free_begin + left_padding, alignment);
1313	uptr gap_end = RoundDownTo((uptr)address, alignment);	1277	uptr gap_end = RoundDownTo((uptr)Min(address, max_vm_address), alignment);
1314	uptr gap_size = gap_end > gap_start ? gap_end - gap_start : 0;	1278	uptr gap_size = gap_end > gap_start ? gap_end - gap_start : 0;
1315	if (size < gap_size) {	1279	if (size < gap_size) {
1316	return gap_start;	1280	return gap_start;
...	@@ -1330,7 +1294,7 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,	...	@@ -1330,7 +1294,7 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
1330	}	1294	}
1331		1295
1332	// FIXME implement on this platform.	1296	// FIXME implement on this platform.
1333	void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { }	1297	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
1334		1298
1335	void SignalContext::DumpAllRegisters(void *context) {	1299	void SignalContext::DumpAllRegisters(void *context) {
1336	Report("Register values:\n");	1300	Report("Register values:\n");
...	@@ -1339,7 +1303,7 @@ void SignalContext::DumpAllRegisters(void *context) {	...	@@ -1339,7 +1303,7 @@ void SignalContext::DumpAllRegisters(void *context) {
1339	# define DUMPREG64(r) \	1303	# define DUMPREG64(r) \
1340	Printf("%s = 0x%016llx ", #r, ucontext->uc_mcontext->__ss.__ ## r);	1304	Printf("%s = 0x%016llx ", #r, ucontext->uc_mcontext->__ss.__ ## r);
1341	# define DUMPREGA64(r) \	1305	# define DUMPREGA64(r) \
1342	Printf(" %s = 0x%016llx ", #r, AARCH64_GET_REG(r));	1306	Printf(" %s = 0x%016lx ", #r, AARCH64_GET_REG(r));
1343	# define DUMPREG32(r) \	1307	# define DUMPREG32(r) \
1344	Printf("%s = 0x%08x ", #r, ucontext->uc_mcontext->__ss.__ ## r);	1308	Printf("%s = 0x%08x ", #r, ucontext->uc_mcontext->__ss.__ ## r);
1345	# define DUMPREG_(r) Printf(" "); DUMPREG(r);	1309	# define DUMPREG_(r) Printf(" "); DUMPREG(r);
...	@@ -1409,7 +1373,7 @@ void DumpProcessMap() {	...	@@ -1409,7 +1373,7 @@ void DumpProcessMap() {
1409	char uuid_str[128];	1373	char uuid_str[128];
1410	FormatUUID(uuid_str, sizeof(uuid_str), modules[i].uuid());	1374	FormatUUID(uuid_str, sizeof(uuid_str), modules[i].uuid());
1411	Printf("0x%zx-0x%zx %s (%s) %s\n", modules[i].base_address(),	1375	Printf("0x%zx-0x%zx %s (%s) %s\n", modules[i].base_address(),
1412	modules[i].max_executable_address(), modules[i].full_name(),	1376	modules[i].max_address(), modules[i].full_name(),
1413	ModuleArchToString(modules[i].arch()), uuid_str);	1377	ModuleArchToString(modules[i].arch()), uuid_str);
1414	}	1378	}
1415	Printf("End of module map.\n");	1379	Printf("End of module map.\n");
...	@@ -1433,6 +1397,61 @@ u32 GetNumberOfCPUs() {	...	@@ -1433,6 +1397,61 @@ u32 GetNumberOfCPUs() {
1433		1397
1434	void InitializePlatformCommonFlags(CommonFlags *cf) {}	1398	void InitializePlatformCommonFlags(CommonFlags *cf) {}
1435		1399
		1400	// Pthread introspection hook
		1401	//
		1402	// * GCD worker threads are created without a call to pthread_create(), but we
		1403	// still need to register these threads (with ThreadCreate/Start()).
		1404	// * We use the "pthread introspection hook" below to observe the creation of
		1405	// such threads.
		1406	// * GCD worker threads don't have parent threads and the CREATE event is
		1407	// delivered in the context of the thread itself. CREATE events for regular
		1408	// threads, are delivered on the parent. We use this to tell apart which
		1409	// threads are GCD workers with `thread == pthread_self()`.
		1410	//
		1411	static pthread_introspection_hook_t prev_pthread_introspection_hook;
		1412	static ThreadEventCallbacks thread_event_callbacks;
		1413
		1414	static void sanitizer_pthread_introspection_hook(unsigned int event,
		1415	pthread_t thread, void *addr,
		1416	size_t size) {
		1417	// create -> start -> terminate -> destroy
		1418	// * create/destroy are usually (not guaranteed) delivered on the parent and
		1419	// track resource allocation/reclamation
		1420	// * start/terminate are guaranteed to be delivered in the context of the
		1421	// thread and give hooks into "just after (before) thread starts (stops)
		1422	// executing"
		1423	DCHECK(event >= PTHREAD_INTROSPECTION_THREAD_CREATE &&
		1424	event <= PTHREAD_INTROSPECTION_THREAD_DESTROY);
		1425
		1426	if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) {
		1427	bool gcd_worker = (thread == pthread_self());
		1428	if (thread_event_callbacks.create)
		1429	thread_event_callbacks.create((uptr)thread, gcd_worker);
		1430	} else if (event == PTHREAD_INTROSPECTION_THREAD_START) {
		1431	CHECK_EQ(thread, pthread_self());
		1432	if (thread_event_callbacks.start)
		1433	thread_event_callbacks.start((uptr)thread);
		1434	}
		1435
		1436	if (prev_pthread_introspection_hook)
		1437	prev_pthread_introspection_hook(event, thread, addr, size);
		1438
		1439	if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) {
		1440	CHECK_EQ(thread, pthread_self());
		1441	if (thread_event_callbacks.terminate)
		1442	thread_event_callbacks.terminate((uptr)thread);
		1443	} else if (event == PTHREAD_INTROSPECTION_THREAD_DESTROY) {
		1444	if (thread_event_callbacks.destroy)
		1445	thread_event_callbacks.destroy((uptr)thread);
		1446	}
		1447	}
		1448
		1449	void InstallPthreadIntrospectionHook(const ThreadEventCallbacks &callbacks) {
		1450	thread_event_callbacks = callbacks;
		1451	prev_pthread_introspection_hook =
		1452	pthread_introspection_hook_install(&sanitizer_pthread_introspection_hook);
		1453	}
		1454
1436	} // namespace __sanitizer	1455	} // namespace __sanitizer
1437		1456
1438	#endif // SANITIZER_MAC	1457	#endif // SANITIZER_APPLE

lib/tsan/sanitizer_common/sanitizer_mac.h+16-5

 @@ -9,12 +9,12 @@
 // This file is shared between various sanitizers' runtime libraries and
 // provides definitions for OSX-specific functions.
 //===----------------------------------------------------------------------===//
-#ifndef SANITIZER_MAC_H
+#ifndef SANITIZER_APPLE_H
-#define SANITIZER_MAC_H
+#define SANITIZER_APPLE_H
 #include "sanitizer_common.h"
 #include "sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
 #include "sanitizer_posix.h"
 namespace __sanitizer {
 @@ -62,7 +62,18 @@ char **GetEnviron();
 void RestrictMemoryToMaxAddress(uptr max_address);
+using ThreadEventCallback = void (*)(uptr thread);
+using ThreadCreateEventCallback = void (*)(uptr thread, bool gcd_worker);
+struct ThreadEventCallbacks {
+  ThreadCreateEventCallback create;
+  ThreadEventCallback start;
+  ThreadEventCallback terminate;
+  ThreadEventCallback destroy;
+};
+void InstallPthreadIntrospectionHook(const ThreadEventCallbacks &callbacks);
 }  // namespace __sanitizer
-#endif  // SANITIZER_MAC
+#endif  // SANITIZER_APPLE
-#endif  // SANITIZER_MAC_H
+#endif  // SANITIZER_APPLE_H

lib/tsan/sanitizer_common/sanitizer_mac_libcdep.cpp+2-2

...	@@ -11,7 +11,7 @@	...	@@ -11,7 +11,7 @@
11	//===----------------------------------------------------------------------===//	11	//===----------------------------------------------------------------------===//
12		12
13	#include "sanitizer_platform.h"	13	#include "sanitizer_platform.h"
14	#if SANITIZER_MAC	14	#if SANITIZER_APPLE
15	#include "sanitizer_mac.h"	15	#include "sanitizer_mac.h"
16		16
17	#include <sys/mman.h>	17	#include <sys/mman.h>
...	@@ -26,4 +26,4 @@ void RestrictMemoryToMaxAddress(uptr max_address) {	...	@@ -26,4 +26,4 @@ void RestrictMemoryToMaxAddress(uptr max_address) {
26		26
27	} // namespace __sanitizer	27	} // namespace __sanitizer
28		28
29	#endif // SANITIZER_MAC	29	#endif // SANITIZER_APPLE

lib/tsan/sanitizer_common/sanitizer_mallinfo.h created+38

...	@@ -0,0 +1,38 @@
1	//===-- sanitizer_mallinfo.h ----------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of Sanitizer common code.
10	//
11	// Definition for mallinfo on different platforms.
12	//===----------------------------------------------------------------------===//
13
14	#ifndef SANITIZER_MALLINFO_H
15	#define SANITIZER_MALLINFO_H
16
17	#include "sanitizer_internal_defs.h"
18	#include "sanitizer_platform.h"
19
20	namespace __sanitizer {
21
22	#if SANITIZER_ANDROID
23
24	struct __sanitizer_struct_mallinfo {
25	uptr v[10];
26	};
27
28	#elif SANITIZER_LINUX \|\| SANITIZER_APPLE \|\| SANITIZER_FUCHSIA
29
30	struct __sanitizer_struct_mallinfo {
31	int v[10];
32	};
33
34	#endif
35
36	} // namespace __sanitizer
37
38	#endif // SANITIZER_MALLINFO_H

lib/tsan/sanitizer_common/sanitizer_malloc_mac.inc+10-12

 @@ -12,7 +12,7 @@
 //===----------------------------------------------------------------------===//
 #include "sanitizer_common/sanitizer_platform.h"
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
 #error "This file should only be compiled on Darwin."
 #endif
 @@ -23,6 +23,7 @@
 #include <sys/mman.h>
 #include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
 #include "sanitizer_common/sanitizer_mac.h"
 // Similar code is used in Google Perftools,
 @@ -192,20 +193,15 @@ void *__sanitizer_mz_malloc(malloc_zone_t *zone, uptr size) {
   return p;
 }
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+  static bool UseImpl() { return !COMMON_MALLOC_SANITIZER_INITIALIZED; }
+};
 extern "C"
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__sanitizer_mz_calloc(malloc_zone_t *zone, size_t nmemb, size_t size) {
-  if (UNLIKELY(!COMMON_MALLOC_SANITIZER_INITIALIZED)) {
+  if (DlsymAlloc::Use())
-    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
+    return DlsymAlloc::Callocate(nmemb, size);
-    const size_t kCallocPoolSize = 1024;
-    static uptr calloc_memory_for_dlsym[kCallocPoolSize];
-    static size_t allocated;
-    size_t size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize;
-    void *mem = (void*)&calloc_memory_for_dlsym[allocated];
-    allocated += size_in_words;
-    CHECK(allocated < kCallocPoolSize);
-    return mem;
-  }
   COMMON_MALLOC_CALLOC(nmemb, size);
   return p;
 }
 @@ -223,6 +219,8 @@ extern "C"
 SANITIZER_INTERFACE_ATTRIBUTE
 void __sanitizer_mz_free(malloc_zone_t *zone, void *ptr) {
   if (!ptr) return;
+  if (DlsymAlloc::PointerIsMine(ptr))
+    return DlsymAlloc::Free(ptr);
   COMMON_MALLOC_FREE(ptr);
 }

lib/tsan/sanitizer_common/sanitizer_mutex.cpp+2-2

lib/tsan/sanitizer_common/sanitizer_mutex.h+120-161

 @@ -20,25 +20,27 @@
 namespace __sanitizer {
-class MUTEX StaticSpinMutex {
+class SANITIZER_MUTEX StaticSpinMutex {
  public:
   void Init() {
     atomic_store(&state_, 0, memory_order_relaxed);
   }
-  void Lock() ACQUIRE() {
+  void Lock() SANITIZER_ACQUIRE() {
     if (LIKELY(TryLock()))
       return;
     LockSlow();
   }
-  bool TryLock() TRY_ACQUIRE(true) {
+  bool TryLock() SANITIZER_TRY_ACQUIRE(true) {
     return atomic_exchange(&state_, 1, memory_order_acquire) == 0;
   }
-  void Unlock() RELEASE() { atomic_store(&state_, 0, memory_order_release); }
+  void Unlock() SANITIZER_RELEASE() {
+    atomic_store(&state_, 0, memory_order_release);
+  }
-  void CheckLocked() const CHECK_LOCKED() {
+  void CheckLocked() const SANITIZER_CHECK_LOCKED() {
     CHECK_EQ(atomic_load(&state_, memory_order_relaxed), 1);
   }
 @@ -48,7 +50,7 @@ class MUTEX StaticSpinMutex {
   void LockSlow();
 };
-class MUTEX SpinMutex : public StaticSpinMutex {
+class SANITIZER_MUTEX SpinMutex : public StaticSpinMutex {
  public:
   SpinMutex() {
     Init();
 @@ -95,7 +97,11 @@ enum {
 // Go linker does not support THREADLOCAL variables,
 // so we can't use per-thread state.
-#define SANITIZER_CHECK_DEADLOCKS (SANITIZER_DEBUG && !SANITIZER_GO)
+// Disable checked locks on Darwin. Although Darwin platforms support
+// THREADLOCAL variables they are not usable early on during process init when
+// `__sanitizer::Mutex` is used.
+#define SANITIZER_CHECK_DEADLOCKS \
+  (SANITIZER_DEBUG && !SANITIZER_GO && SANITIZER_SUPPORTS_THREADLOCAL && !SANITIZER_APPLE)
 #if SANITIZER_CHECK_DEADLOCKS
 struct MutexMeta {
 @@ -111,7 +117,7 @@ struct MutexMeta {
 class CheckedMutex {
  public:
-  constexpr CheckedMutex(MutexType type)
+  explicit constexpr CheckedMutex(MutexType type)
 #if SANITIZER_CHECK_DEADLOCKS
       : type_(type)
 #endif
 @@ -152,15 +158,15 @@ class CheckedMutex {
 // Derive from CheckedMutex for the purposes of EBO.
 // We could make it a field marked with [[no_unique_address]],
 // but this attribute is not supported by some older compilers.
-class MUTEX Mutex : CheckedMutex {
+class SANITIZER_MUTEX Mutex : CheckedMutex {
  public:
-  constexpr Mutex(MutexType type = MutexUnchecked) : CheckedMutex(type) {}
+  explicit constexpr Mutex(MutexType type = MutexUnchecked)
+      : CheckedMutex(type) {}
-  void Lock() ACQUIRE() {
+  void Lock() SANITIZER_ACQUIRE() {
     CheckedMutex::Lock();
     u64 reset_mask = ~0ull;
     u64 state = atomic_load_relaxed(&state_);
-    const uptr kMaxSpinIters = 1500;
     for (uptr spin_iters = 0;; spin_iters++) {
       u64 new_state;
       bool locked = (state & (kWriterLock | kReaderLockMask)) != 0;
 @@ -189,8 +195,6 @@ class MUTEX Mutex : CheckedMutex {
         // We've incremented waiting writers, so now block.
         writers_.Wait();
         spin_iters = 0;
-        state = atomic_load(&state_, memory_order_relaxed);
-        DCHECK_NE(state & kWriterSpinWait, 0);
       } else {
         // We've set kWriterSpinWait, but we are still in active spinning.
       }
 @@ -199,10 +203,26 @@ class MUTEX Mutex : CheckedMutex {
       // Either way we need to reset kWriterSpinWait
       // next time we take the lock or block again.
       reset_mask = ~kWriterSpinWait;
+      state = atomic_load(&state_, memory_order_relaxed);
+      DCHECK_NE(state & kWriterSpinWait, 0);
+    }
+  }
+  bool TryLock() SANITIZER_TRY_ACQUIRE(true) {
+    u64 state = atomic_load_relaxed(&state_);
+    for (;;) {
+      if (UNLIKELY(state & (kWriterLock | kReaderLockMask)))
+        return false;
+      // The mutex is not read-/write-locked, try to lock.
+      if (LIKELY(atomic_compare_exchange_weak(
+              &state_, &state, state | kWriterLock, memory_order_acquire))) {
+        CheckedMutex::Lock();
+        return true;
+      }
     }
   }
-  void Unlock() RELEASE() {
+  void Unlock() SANITIZER_RELEASE() {
     CheckedMutex::Unlock();
     bool wake_writer;
     u64 wake_readers;
 @@ -212,17 +232,16 @@ class MUTEX Mutex : CheckedMutex {
       DCHECK_NE(state & kWriterLock, 0);
       DCHECK_EQ(state & kReaderLockMask, 0);
       new_state = state & ~kWriterLock;
-      wake_writer =
+      wake_writer = (state & (kWriterSpinWait | kReaderSpinWait)) == 0 &&
-          (state & kWriterSpinWait) == 0 && (state & kWaitingWriterMask) != 0;
+                    (state & kWaitingWriterMask) != 0;
       if (wake_writer)
         new_state = (new_state - kWaitingWriterInc) | kWriterSpinWait;
       wake_readers =
-          (state & (kWriterSpinWait | kWaitingWriterMask)) != 0
+          wake_writer || (state & kWriterSpinWait) != 0
               ? 0
               : ((state & kWaitingReaderMask) >> kWaitingReaderShift);
       if (wake_readers)
-        new_state = (new_state & ~kWaitingReaderMask) +
+        new_state = (new_state & ~kWaitingReaderMask) | kReaderSpinWait;
-                    (wake_readers << kReaderLockShift);
     } while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
                                                     memory_order_release)));
     if (UNLIKELY(wake_writer))
 @@ -231,37 +250,54 @@ class MUTEX Mutex : CheckedMutex {
       readers_.Post(wake_readers);
   }
-  void ReadLock() ACQUIRE_SHARED() {
+  void ReadLock() SANITIZER_ACQUIRE_SHARED() {
     CheckedMutex::Lock();
-    bool locked;
+    u64 reset_mask = ~0ull;
-    u64 new_state;
     u64 state = atomic_load_relaxed(&state_);
-    do {
+    for (uptr spin_iters = 0;; spin_iters++) {
-      locked =
+      bool locked = (state & kWriterLock) != 0;
-          (state & kReaderLockMask) == 0 &&
+      u64 new_state;
-          (state & (kWriterLock | kWriterSpinWait | kWaitingWriterMask)) != 0;
+      if (LIKELY(!locked)) {
+        new_state = (state + kReaderLockInc) & reset_mask;
+      } else if (spin_iters > kMaxSpinIters) {
+        new_state = (state + kWaitingReaderInc) & reset_mask;
+      } else if ((state & kReaderSpinWait) == 0) {
+        // Active spinning, but denote our presence so that unlocking
+        // thread does not wake up other threads.
+        new_state = state | kReaderSpinWait;
+      } else {
+        // Active spinning.
+        state = atomic_load(&state_, memory_order_relaxed);
+        continue;
+      }
+      if (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
+                                                 memory_order_acquire)))
+        continue;
       if (LIKELY(!locked))
-        new_state = state + kReaderLockInc;
+        return;  // We've locked the mutex.
-      else
+      if (spin_iters > kMaxSpinIters) {
-        new_state = state + kWaitingReaderInc;
+        // We've incremented waiting readers, so now block.
-    } while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state,
+        readers_.Wait();
-                                                    memory_order_acquire)));
+        spin_iters = 0;
-    if (UNLIKELY(locked))
+      } else {
-      readers_.Wait();
+        // We've set kReaderSpinWait, but we are still in active spinning.
-    DCHECK_EQ(atomic_load_relaxed(&state_) & kWriterLock, 0);
+      }
-    DCHECK_NE(atomic_load_relaxed(&state_) & kReaderLockMask, 0);
+      reset_mask = ~kReaderSpinWait;
+      state = atomic_load(&state_, memory_order_relaxed);
+    }
   }
-  void ReadUnlock() RELEASE_SHARED() {
+  void ReadUnlock() SANITIZER_RELEASE_SHARED() {
     CheckedMutex::Unlock();
     bool wake;
     u64 new_state;
     u64 state = atomic_load_relaxed(&state_);
     do {
       DCHECK_NE(state & kReaderLockMask, 0);
-      DCHECK_EQ(state & (kWaitingReaderMask | kWriterLock), 0);
+      DCHECK_EQ(state & kWriterLock, 0);
       new_state = state - kReaderLockInc;
-      wake = (new_state & (kReaderLockMask | kWriterSpinWait)) == 0 &&
+      wake = (new_state &
+              (kReaderLockMask | kWriterSpinWait | kReaderSpinWait)) == 0 &&
              (new_state & kWaitingWriterMask) != 0;
       if (wake)
         new_state = (new_state - kWaitingWriterInc) | kWriterSpinWait;
 @@ -277,13 +313,13 @@ class MUTEX Mutex : CheckedMutex {
   // owns the mutex but a child checks that it is locked. Rather than
   // maintaining complex state to work around those situations, the check only
   // checks that the mutex is owned.
-  void CheckWriteLocked() const CHECK_LOCKED() {
+  void CheckWriteLocked() const SANITIZER_CHECK_LOCKED() {
     CHECK(atomic_load(&state_, memory_order_relaxed) & kWriterLock);
   }
-  void CheckLocked() const CHECK_LOCKED() { CheckWriteLocked(); }
+  void CheckLocked() const SANITIZER_CHECK_LOCKED() { CheckWriteLocked(); }
-  void CheckReadLocked() const CHECK_LOCKED() {
+  void CheckReadLocked() const SANITIZER_CHECK_LOCKED() {
     CHECK(atomic_load(&state_, memory_order_relaxed) & kReaderLockMask);
   }
 @@ -305,16 +341,14 @@ class MUTEX Mutex : CheckedMutex {
   //  - a writer is awake and spin-waiting
   //    the flag is used to prevent thundering herd problem
   //    (new writers are not woken if this flag is set)
+  //  - a reader is awake and spin-waiting
   //
-  // Writer support active spinning, readers does not.
+  // Both writers and readers use active spinning before blocking.
   // But readers are more aggressive and always take the mutex
   // if there are any other readers.
-  // Writers hand off the mutex to readers: after wake up readers
+  // After wake up both writers and readers compete to lock the
-  // already assume ownership of the mutex (don't need to do any
+  // mutex again. This is needed to allow repeated locks even in presence
-  // state updates). But the mutex is not handed off to writers,
+  // of other blocked threads.
-  // after wake up writers compete to lock the mutex again.
-  // This is needed to allow repeated write locks even in presence
-  // of other blocked writers.
   static constexpr u64 kCounterWidth = 20;
   static constexpr u64 kReaderLockShift = 0;
   static constexpr u64 kReaderLockInc = 1ull << kReaderLockShift;
 @@ -330,7 +364,11 @@ class MUTEX Mutex : CheckedMutex {
                                             << kWaitingWriterShift;
   static constexpr u64 kWriterLock = 1ull << (3 * kCounterWidth);
   static constexpr u64 kWriterSpinWait = 1ull << (3 * kCounterWidth + 1);
+  static constexpr u64 kReaderSpinWait = 1ull << (3 * kCounterWidth + 2);
+  static constexpr uptr kMaxSpinIters = 1500;
+  Mutex(LinkerInitialized) = delete;
   Mutex(const Mutex &) = delete;
   void operator=(const Mutex &) = delete;
 };
 @@ -338,149 +376,70 @@ class MUTEX Mutex : CheckedMutex {
 void FutexWait(atomic_uint32_t *p, u32 cmp);
 void FutexWake(atomic_uint32_t *p, u32 count);
-class MUTEX BlockingMutex {
+template <typename MutexType>
- public:
+class SANITIZER_SCOPED_LOCK GenericScopedLock {
-  explicit constexpr BlockingMutex(LinkerInitialized)
-      : opaque_storage_ {0, }, owner_ {0} {}
-  BlockingMutex();
-  void Lock() ACQUIRE();
-  void Unlock() RELEASE();
-  // This function does not guarantee an explicit check that the calling thread
-  // is the thread which owns the mutex. This behavior, while more strictly
-  // correct, causes problems in cases like StopTheWorld, where a parent thread
-  // owns the mutex but a child checks that it is locked. Rather than
-  // maintaining complex state to work around those situations, the check only
-  // checks that the mutex is owned, and assumes callers to be generally
-  // well-behaved.
-  void CheckLocked() const CHECK_LOCKED();
- private:
-  // Solaris mutex_t has a member that requires 64-bit alignment.
-  ALIGNED(8) uptr opaque_storage_[10];
-  uptr owner_;  // for debugging
-};
-// Reader-writer spin mutex.
-class MUTEX RWMutex {
  public:
-  RWMutex() {
+  explicit GenericScopedLock(MutexType *mu) SANITIZER_ACQUIRE(mu) : mu_(mu) {
-    atomic_store(&state_, kUnlocked, memory_order_relaxed);
+    mu_->Lock();
-  }
-  ~RWMutex() {
-    CHECK_EQ(atomic_load(&state_, memory_order_relaxed), kUnlocked);
-  }
-  void Lock() ACQUIRE() {
-    u32 cmp = kUnlocked;
-    if (atomic_compare_exchange_strong(&state_, &cmp, kWriteLock,
-                                       memory_order_acquire))
-      return;
-    LockSlow();
-  }
-  void Unlock() RELEASE() {
-    u32 prev = atomic_fetch_sub(&state_, kWriteLock, memory_order_release);
-    DCHECK_NE(prev & kWriteLock, 0);
-    (void)prev;
-  }
-  void ReadLock() ACQUIRE_SHARED() {
-    u32 prev = atomic_fetch_add(&state_, kReadLock, memory_order_acquire);
-    if ((prev & kWriteLock) == 0)
-      return;
-    ReadLockSlow();
-  }
-  void ReadUnlock() RELEASE_SHARED() {
-    u32 prev = atomic_fetch_sub(&state_, kReadLock, memory_order_release);
-    DCHECK_EQ(prev & kWriteLock, 0);
-    DCHECK_GT(prev & ~kWriteLock, 0);
-    (void)prev;
   }
-  void CheckLocked() const CHECK_LOCKED() {
+  ~GenericScopedLock() SANITIZER_RELEASE() { mu_->Unlock(); }
-    CHECK_NE(atomic_load(&state_, memory_order_relaxed), kUnlocked);
-  }
  private:
-  atomic_uint32_t state_;
+  MutexType *mu_;
-  enum {
-    kUnlocked = 0,
-    kWriteLock = 1,
-    kReadLock = 2
-  };
-  void NOINLINE LockSlow() {
-    for (int i = 0;; i++) {
-      if (i < 10)
-        proc_yield(10);
-      else
-        internal_sched_yield();
-      u32 cmp = atomic_load(&state_, memory_order_relaxed);
-      if (cmp == kUnlocked &&
-          atomic_compare_exchange_weak(&state_, &cmp, kWriteLock,
-                                       memory_order_acquire))
-          return;
-    }
-  }
-  void NOINLINE ReadLockSlow() {
-    for (int i = 0;; i++) {
-      if (i < 10)
-        proc_yield(10);
-      else
-        internal_sched_yield();
-      u32 prev = atomic_load(&state_, memory_order_acquire);
-      if ((prev & kWriteLock) == 0)
-        return;
-    }
-  }
-  RWMutex(const RWMutex &) = delete;
+  GenericScopedLock(const GenericScopedLock &) = delete;
-  void operator=(const RWMutex &) = delete;
+  void operator=(const GenericScopedLock &) = delete;
 };
 template <typename MutexType>
-class SCOPED_LOCK GenericScopedLock {
+class SANITIZER_SCOPED_LOCK GenericScopedReadLock {
  public:
-  explicit GenericScopedLock(MutexType *mu) ACQUIRE(mu) : mu_(mu) {
+  explicit GenericScopedReadLock(MutexType *mu) SANITIZER_ACQUIRE(mu)
-    mu_->Lock();
+      : mu_(mu) {
+    mu_->ReadLock();
   }
-  ~GenericScopedLock() RELEASE() { mu_->Unlock(); }
+  ~GenericScopedReadLock() SANITIZER_RELEASE() { mu_->ReadUnlock(); }
  private:
   MutexType *mu_;
-  GenericScopedLock(const GenericScopedLock &) = delete;
+  GenericScopedReadLock(const GenericScopedReadLock &) = delete;
-  void operator=(const GenericScopedLock &) = delete;
+  void operator=(const GenericScopedReadLock &) = delete;
 };
 template <typename MutexType>
-class SCOPED_LOCK GenericScopedReadLock {
+class SANITIZER_SCOPED_LOCK GenericScopedRWLock {
  public:
-  explicit GenericScopedReadLock(MutexType *mu) ACQUIRE(mu) : mu_(mu) {
+  ALWAYS_INLINE explicit GenericScopedRWLock(MutexType *mu, bool write)
-    mu_->ReadLock();
+      SANITIZER_ACQUIRE(mu)
+      : mu_(mu), write_(write) {
+    if (write_)
+      mu_->Lock();
+    else
+      mu_->ReadLock();
   }
-  ~GenericScopedReadLock() RELEASE() { mu_->ReadUnlock(); }
+  ALWAYS_INLINE ~GenericScopedRWLock() SANITIZER_RELEASE() {
+    if (write_)
+      mu_->Unlock();
+    else
+      mu_->ReadUnlock();
+  }
  private:
   MutexType *mu_;
+  bool write_;
-  GenericScopedReadLock(const GenericScopedReadLock &) = delete;
+  GenericScopedRWLock(const GenericScopedRWLock &) = delete;
-  void operator=(const GenericScopedReadLock &) = delete;
+  void operator=(const GenericScopedRWLock &) = delete;
 };
 typedef GenericScopedLock<StaticSpinMutex> SpinMutexLock;
-typedef GenericScopedLock<BlockingMutex> BlockingMutexLock;
-typedef GenericScopedLock<RWMutex> RWMutexLock;
-typedef GenericScopedReadLock<RWMutex> RWMutexReadLock;
 typedef GenericScopedLock<Mutex> Lock;
 typedef GenericScopedReadLock<Mutex> ReadLock;
+typedef GenericScopedRWLock<Mutex> RWLock;
 }  // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_openbsd.cpp deleted

lib/tsan/sanitizer_common/sanitizer_persistent_allocator.cpp deleted-18

...	@@ -1,18 +0,0 @@
1	//===-- sanitizer_persistent_allocator.cpp ----------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between AddressSanitizer and ThreadSanitizer
10	// run-time libraries.
11	//===----------------------------------------------------------------------===//
12	#include "sanitizer_persistent_allocator.h"
13
14	namespace __sanitizer {
15
16	PersistentAllocator thePersistentAllocator;
17
18	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_persistent_allocator.h deleted-71

...	@@ -1,71 +0,0 @@
1	//===-- sanitizer_persistent_allocator.h ------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// A fast memory allocator that does not support free() nor realloc().
10	// All allocations are forever.
11	//===----------------------------------------------------------------------===//
12
13	#ifndef SANITIZER_PERSISTENT_ALLOCATOR_H
14	#define SANITIZER_PERSISTENT_ALLOCATOR_H
15
16	#include "sanitizer_internal_defs.h"
17	#include "sanitizer_mutex.h"
18	#include "sanitizer_atomic.h"
19	#include "sanitizer_common.h"
20
21	namespace __sanitizer {
22
23	class PersistentAllocator {
24	public:
25	void *alloc(uptr size);
26
27	private:
28	void *tryAlloc(uptr size);
29	StaticSpinMutex mtx; // Protects alloc of new blocks for region allocator.
30	atomic_uintptr_t region_pos; // Region allocator for Node's.
31	atomic_uintptr_t region_end;
32	};
33
34	inline void *PersistentAllocator::tryAlloc(uptr size) {
35	// Optimisic lock-free allocation, essentially try to bump the region ptr.
36	for (;;) {
37	uptr cmp = atomic_load(&region_pos, memory_order_acquire);
38	uptr end = atomic_load(&region_end, memory_order_acquire);
39	if (cmp == 0 \|\| cmp + size > end) return nullptr;
40	if (atomic_compare_exchange_weak(&region_pos, &cmp, cmp + size,
41	memory_order_acquire))
42	return (void *)cmp;
43	}
44	}
45
46	inline void *PersistentAllocator::alloc(uptr size) {
47	// First, try to allocate optimisitically.
48	void *s = tryAlloc(size);
49	if (s) return s;
50	// If failed, lock, retry and alloc new superblock.
51	SpinMutexLock l(&mtx);
52	for (;;) {
53	s = tryAlloc(size);
54	if (s) return s;
55	atomic_store(&region_pos, 0, memory_order_relaxed);
56	uptr allocsz = 64 * 1024;
57	if (allocsz < size) allocsz = size;
58	uptr mem = (uptr)MmapOrDie(allocsz, "stack depot");
59	atomic_store(&region_end, mem + allocsz, memory_order_release);
60	atomic_store(&region_pos, mem, memory_order_release);
61	}
62	}
63
64	extern PersistentAllocator thePersistentAllocator;
65	inline void *PersistentAlloc(uptr sz) {
66	return thePersistentAllocator.alloc(sz);
67	}
68
69	} // namespace __sanitizer
70
71	#endif // SANITIZER_PERSISTENT_ALLOCATOR_H

lib/tsan/sanitizer_common/sanitizer_platform.h+234-166

...	@@ -22,103 +22,123 @@	...	@@ -22,103 +22,123 @@
22	// function declarations into a .S file which doesn't compile.	22	// function declarations into a .S file which doesn't compile.
23	// https://crbug.com/1162741	23	// https://crbug.com/1162741
24	#if __has_include(<features.h>) && !defined(__ANDROID__)	24	#if __has_include(<features.h>) && !defined(__ANDROID__)
25	#include <features.h>	25	# include <features.h>
26	#endif	26	#endif
27		27
28	#if defined(__linux__)	28	#if defined(__linux__)
29	# define SANITIZER_LINUX 1	29	# define SANITIZER_LINUX 1
30	#else	30	#else
31	# define SANITIZER_LINUX 0	31	# define SANITIZER_LINUX 0
32	#endif	32	#endif
33		33
34	#if defined(__GLIBC__)	34	#if defined(__GLIBC__)
35	# define SANITIZER_GLIBC 1	35	# define SANITIZER_GLIBC 1
36	#else	36	#else
37	# define SANITIZER_GLIBC 0	37	# define SANITIZER_GLIBC 0
38	#endif	38	#endif
39		39
40	#if defined(__FreeBSD__)	40	#if defined(__FreeBSD__)
41	# define SANITIZER_FREEBSD 1	41	# define SANITIZER_FREEBSD 1
42	#else	42	#else
43	# define SANITIZER_FREEBSD 0	43	# define SANITIZER_FREEBSD 0
44	#endif	44	#endif
45		45
46	#if defined(__NetBSD__)	46	#if defined(__NetBSD__)
47	# define SANITIZER_NETBSD 1	47	# define SANITIZER_NETBSD 1
48	#else	48	#else
49	# define SANITIZER_NETBSD 0	49	# define SANITIZER_NETBSD 0
50	#endif	50	#endif
51		51
52	#if defined(__sun__) && defined(__svr4__)	52	#if defined(__sun__) && defined(__svr4__)
53	# define SANITIZER_SOLARIS 1	53	# define SANITIZER_SOLARIS 1
54	#else	54	#else
55	# define SANITIZER_SOLARIS 0	55	# define SANITIZER_SOLARIS 0
56	#endif	56	#endif
57		57
		58	// - SANITIZER_APPLE: all Apple code
		59	// - TARGET_OS_OSX: macOS
		60	// - SANITIZER_IOS: devices (iOS and iOS-like)
		61	// - SANITIZER_WATCHOS
		62	// - SANITIZER_TVOS
		63	// - SANITIZER_IOSSIM: simulators (iOS and iOS-like)
		64	// - SANITIZER_DRIVERKIT
58	#if defined(__APPLE__)	65	#if defined(__APPLE__)
59	# define SANITIZER_MAC 1	66	# define SANITIZER_APPLE 1
60	# include <TargetConditionals.h>	67	# include <TargetConditionals.h>
61	# if TARGET_OS_OSX	68	# if TARGET_OS_OSX
62	# define SANITIZER_OSX 1	69	# define SANITIZER_OSX 1
63	# else	70	# else
64	# define SANITIZER_OSX 0	71	# define SANITIZER_OSX 0
65	# endif	72	# endif
66	# if TARGET_OS_IPHONE	73	# if TARGET_OS_IPHONE
67	# define SANITIZER_IOS 1	74	# define SANITIZER_IOS 1
68	# else	75	# else
69	# define SANITIZER_IOS 0	76	# define SANITIZER_IOS 0
70	# endif	77	# endif
71	# if TARGET_OS_SIMULATOR	78	# if TARGET_OS_WATCH
72	# define SANITIZER_IOSSIM 1	79	# define SANITIZER_WATCHOS 1
73	# else	80	# else
74	# define SANITIZER_IOSSIM 0	81	# define SANITIZER_WATCHOS 0
75	# endif	82	# endif
76	#else	83	# if TARGET_OS_TV
77	# define SANITIZER_MAC 0	84	# define SANITIZER_TVOS 1
78	# define SANITIZER_IOS 0	85	# else
79	# define SANITIZER_IOSSIM 0	86	# define SANITIZER_TVOS 0
80	# define SANITIZER_OSX 0	87	# endif
81	#endif	88	# if TARGET_OS_SIMULATOR
82		89	# define SANITIZER_IOSSIM 1
83	#if defined(__APPLE__) && TARGET_OS_IPHONE && TARGET_OS_WATCH	90	# else
84	# define SANITIZER_WATCHOS 1	91	# define SANITIZER_IOSSIM 0
85	#else	92	# endif
86	# define SANITIZER_WATCHOS 0	93	# if defined(TARGET_OS_DRIVERKIT) && TARGET_OS_DRIVERKIT
87	#endif	94	# define SANITIZER_DRIVERKIT 1
88		95	# else
89	#if defined(__APPLE__) && TARGET_OS_IPHONE && TARGET_OS_TV	96	# define SANITIZER_DRIVERKIT 0
90	# define SANITIZER_TVOS 1	97	# endif
91	#else	98	#else
92	# define SANITIZER_TVOS 0	99	# define SANITIZER_APPLE 0
		100	# define SANITIZER_OSX 0
		101	# define SANITIZER_IOS 0
		102	# define SANITIZER_WATCHOS 0
		103	# define SANITIZER_TVOS 0
		104	# define SANITIZER_IOSSIM 0
		105	# define SANITIZER_DRIVERKIT 0
93	#endif	106	#endif
94		107
95	#if defined(_WIN32)	108	#if defined(_WIN32)
96	# define SANITIZER_WINDOWS 1	109	# define SANITIZER_WINDOWS 1
97	#else	110	#else
98	# define SANITIZER_WINDOWS 0	111	# define SANITIZER_WINDOWS 0
99	#endif	112	#endif
100		113
101	#if defined(_WIN64)	114	#if defined(_WIN64)
102	# define SANITIZER_WINDOWS64 1	115	# define SANITIZER_WINDOWS64 1
103	#else	116	#else
104	# define SANITIZER_WINDOWS64 0	117	# define SANITIZER_WINDOWS64 0
105	#endif	118	#endif
106		119
107	#if defined(__ANDROID__)	120	#if defined(__ANDROID__)
108	# define SANITIZER_ANDROID 1	121	# define SANITIZER_ANDROID 1
109	#else	122	#else
110	# define SANITIZER_ANDROID 0	123	# define SANITIZER_ANDROID 0
111	#endif	124	#endif
112		125
113	#if defined(__Fuchsia__)	126	#if defined(__Fuchsia__)
114	# define SANITIZER_FUCHSIA 1	127	# define SANITIZER_FUCHSIA 1
115	#else	128	#else
116	# define SANITIZER_FUCHSIA 0	129	# define SANITIZER_FUCHSIA 0
117	#endif	130	#endif
118		131
119	#define SANITIZER_POSIX \	132	// Assume linux that is not glibc or android is musl libc.
120	(SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_MAC \|\| \	133	#if SANITIZER_LINUX && !SANITIZER_GLIBC && !SANITIZER_ANDROID
121	SANITIZER_NETBSD \|\| SANITIZER_SOLARIS)	134	# define SANITIZER_MUSL 1
		135	#else
		136	# define SANITIZER_MUSL 0
		137	#endif
		138
		139	#define SANITIZER_POSIX \
		140	(SANITIZER_FREEBSD \|\| SANITIZER_LINUX \|\| SANITIZER_APPLE \|\| \
		141	SANITIZER_NETBSD \|\| SANITIZER_SOLARIS)
122		142
123	#if __LP64__ \|\| defined(_WIN64)	143	#if __LP64__ \|\| defined(_WIN64)
124	# define SANITIZER_WORDSIZE 64	144	# define SANITIZER_WORDSIZE 64
...	@@ -127,58 +147,79 @@	...	@@ -127,58 +147,79 @@
127	#endif	147	#endif
128		148
129	#if SANITIZER_WORDSIZE == 64	149	#if SANITIZER_WORDSIZE == 64
130	# define FIRST_32_SECOND_64(a, b) (b)	150	# define FIRST_32_SECOND_64(a, b) (b)
131	#else	151	#else
132	# define FIRST_32_SECOND_64(a, b) (a)	152	# define FIRST_32_SECOND_64(a, b) (a)
133	#endif	153	#endif
134		154
135	#if defined(__x86_64__) && !defined(_LP64)	155	#if defined(__x86_64__) && !defined(_LP64)
136	# define SANITIZER_X32 1	156	# define SANITIZER_X32 1
137	#else	157	#else
138	# define SANITIZER_X32 0	158	# define SANITIZER_X32 0
		159	#endif
		160
		161	#if defined(__x86_64__) \|\| defined(_M_X64)
		162	# define SANITIZER_X64 1
		163	#else
		164	# define SANITIZER_X64 0
139	#endif	165	#endif
140		166
141	#if defined(__i386__) \|\| defined(_M_IX86)	167	#if defined(__i386__) \|\| defined(_M_IX86)
142	# define SANITIZER_I386 1	168	# define SANITIZER_I386 1
143	#else	169	#else
144	# define SANITIZER_I386 0	170	# define SANITIZER_I386 0
145	#endif	171	#endif
146		172
147	#if defined(__mips__)	173	#if defined(__mips__)
148	# define SANITIZER_MIPS 1	174	# define SANITIZER_MIPS 1
149	# if defined(__mips64)	175	# if defined(__mips64) && _MIPS_SIM == _ABI64
		176	# define SANITIZER_MIPS32 0
		177	# define SANITIZER_MIPS64 1
		178	# else
		179	# define SANITIZER_MIPS32 1
		180	# define SANITIZER_MIPS64 0
		181	# endif
		182	#else
		183	# define SANITIZER_MIPS 0
150	# define SANITIZER_MIPS32 0	184	# define SANITIZER_MIPS32 0
151	# define SANITIZER_MIPS64 1
152	# else
153	# define SANITIZER_MIPS32 1
154	# define SANITIZER_MIPS64 0	185	# define SANITIZER_MIPS64 0
155	# endif
156	#else
157	# define SANITIZER_MIPS 0
158	# define SANITIZER_MIPS32 0
159	# define SANITIZER_MIPS64 0
160	#endif	186	#endif
161		187
162	#if defined(__s390__)	188	#if defined(__s390__)
163	# define SANITIZER_S390 1	189	# define SANITIZER_S390 1
164	# if defined(__s390x__)	190	# if defined(__s390x__)
		191	# define SANITIZER_S390_31 0
		192	# define SANITIZER_S390_64 1
		193	# else
		194	# define SANITIZER_S390_31 1
		195	# define SANITIZER_S390_64 0
		196	# endif
		197	#else
		198	# define SANITIZER_S390 0
165	# define SANITIZER_S390_31 0	199	# define SANITIZER_S390_31 0
166	# define SANITIZER_S390_64 1
167	# else
168	# define SANITIZER_S390_31 1
169	# define SANITIZER_S390_64 0	200	# define SANITIZER_S390_64 0
170	# endif	201	#endif
		202
		203	#if defined(__sparc__)
		204	# define SANITIZER_SPARC 1
		205	# if defined(__arch64__)
		206	# define SANITIZER_SPARC32 0
		207	# define SANITIZER_SPARC64 1
		208	# else
		209	# define SANITIZER_SPARC32 1
		210	# define SANITIZER_SPARC64 0
		211	# endif
171	#else	212	#else
172	# define SANITIZER_S390 0	213	# define SANITIZER_SPARC 0
173	# define SANITIZER_S390_31 0	214	# define SANITIZER_SPARC32 0
174	# define SANITIZER_S390_64 0	215	# define SANITIZER_SPARC64 0
175	#endif	216	#endif
176		217
177	#if defined(__powerpc__)	218	#if defined(__powerpc__)
178	# define SANITIZER_PPC 1	219	# define SANITIZER_PPC 1
179	# if defined(__powerpc64__)	220	# if defined(__powerpc64__)
180	# define SANITIZER_PPC32 0	221	# define SANITIZER_PPC32 0
181	# define SANITIZER_PPC64 1	222	# define SANITIZER_PPC64 1
182	// 64-bit PPC has two ABIs (v1 and v2). The old powerpc64 target is	223	// 64-bit PPC has two ABIs (v1 and v2). The old powerpc64 target is
183	// big-endian, and uses v1 ABI (known for its function descriptors),	224	// big-endian, and uses v1 ABI (known for its function descriptors),
184	// while the new powerpc64le target is little-endian and uses v2.	225	// while the new powerpc64le target is little-endian and uses v2.
...	@@ -186,106 +227,109 @@	...	@@ -186,106 +227,109 @@
186	// (eg. big-endian v2), but you won't find such combinations in the wild	227	// (eg. big-endian v2), but you won't find such combinations in the wild
187	// (it'd require bootstrapping a whole system, which would be quite painful	228	// (it'd require bootstrapping a whole system, which would be quite painful
188	// - there's no target triple for that). LLVM doesn't support them either.	229	// - there's no target triple for that). LLVM doesn't support them either.
189	# if _CALL_ELF == 2	230	# if _CALL_ELF == 2
190	# define SANITIZER_PPC64V1 0	231	# define SANITIZER_PPC64V1 0
191	# define SANITIZER_PPC64V2 1	232	# define SANITIZER_PPC64V2 1
		233	# else
		234	# define SANITIZER_PPC64V1 1
		235	# define SANITIZER_PPC64V2 0
		236	# endif
192	# else	237	# else
193	# define SANITIZER_PPC64V1 1	238	# define SANITIZER_PPC32 1
194	# define SANITIZER_PPC64V2 0	239	# define SANITIZER_PPC64 0
		240	# define SANITIZER_PPC64V1 0
		241	# define SANITIZER_PPC64V2 0
195	# endif	242	# endif
196	# else	243	#else
197	# define SANITIZER_PPC32 1	244	# define SANITIZER_PPC 0
		245	# define SANITIZER_PPC32 0
198	# define SANITIZER_PPC64 0	246	# define SANITIZER_PPC64 0
199	# define SANITIZER_PPC64V1 0	247	# define SANITIZER_PPC64V1 0
200	# define SANITIZER_PPC64V2 0	248	# define SANITIZER_PPC64V2 0
201	# endif	249	#endif
		250
		251	#if defined(__arm__) \|\| defined(_M_ARM)
		252	# define SANITIZER_ARM 1
202	#else	253	#else
203	# define SANITIZER_PPC 0	254	# define SANITIZER_ARM 0
204	# define SANITIZER_PPC32 0
205	# define SANITIZER_PPC64 0
206	# define SANITIZER_PPC64V1 0
207	# define SANITIZER_PPC64V2 0
208	#endif	255	#endif
209		256
210	#if defined(__arm__)	257	#if defined(__aarch64__) \|\| defined(_M_ARM64)
211	# define SANITIZER_ARM 1	258	# define SANITIZER_ARM64 1
212	#else	259	#else
213	# define SANITIZER_ARM 0	260	# define SANITIZER_ARM64 0
214	#endif	261	#endif
215		262
216	#if SANITIZER_SOLARIS && SANITIZER_WORDSIZE == 32	263	#if SANITIZER_SOLARIS && SANITIZER_WORDSIZE == 32
217	# define SANITIZER_SOLARIS32 1	264	# define SANITIZER_SOLARIS32 1
218	#else	265	#else
219	# define SANITIZER_SOLARIS32 0	266	# define SANITIZER_SOLARIS32 0
220	#endif	267	#endif
221		268
222	#if defined(__riscv) && (__riscv_xlen == 64)	269	#if defined(__riscv) && (__riscv_xlen == 64)
223	#define SANITIZER_RISCV64 1	270	# define SANITIZER_RISCV64 1
224	#else	271	#else
225	#define SANITIZER_RISCV64 0	272	# define SANITIZER_RISCV64 0
		273	#endif
		274
		275	#if defined(__loongarch_lp64)
		276	# define SANITIZER_LOONGARCH64 1
		277	#else
		278	# define SANITIZER_LOONGARCH64 0
226	#endif	279	#endif
227		280
228	// By default we allow to use SizeClassAllocator64 on 64-bit platform.	281	// By default we allow to use SizeClassAllocator64 on 64-bit platform.
229	// But in some cases (e.g. AArch64's 39-bit address space) SizeClassAllocator64	282	// But in some cases SizeClassAllocator64 does not work well and we need to
230	// does not work well and we need to fallback to SizeClassAllocator32.	283	// fallback to SizeClassAllocator32.
231	// For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or	284	// For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or
232	// change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here.	285	// change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here.
233	#ifndef SANITIZER_CAN_USE_ALLOCATOR64	286	#ifndef SANITIZER_CAN_USE_ALLOCATOR64
234	# if (SANITIZER_ANDROID && defined(__aarch64__)) \|\| SANITIZER_FUCHSIA	287	# if SANITIZER_RISCV64 \|\| SANITIZER_IOS
235	# define SANITIZER_CAN_USE_ALLOCATOR64 1	288	# define SANITIZER_CAN_USE_ALLOCATOR64 0
236	# elif defined(__mips64) \|\| defined(__aarch64__)	289	# elif defined(__mips64) \|\| defined(__hexagon__)
237	# define SANITIZER_CAN_USE_ALLOCATOR64 0	290	# define SANITIZER_CAN_USE_ALLOCATOR64 0
238	# else	291	# else
239	# define SANITIZER_CAN_USE_ALLOCATOR64 (SANITIZER_WORDSIZE == 64)	292	# define SANITIZER_CAN_USE_ALLOCATOR64 (SANITIZER_WORDSIZE == 64)
240	# endif	293	# endif
241	#endif	294	#endif
242		295
243	// The range of addresses which can be returned my mmap.	296	// The range of addresses which can be returned my mmap.
244	// FIXME: this value should be different on different platforms. Larger values	297	// FIXME: this value should be different on different platforms. Larger values
245	// will still work but will consume more memory for TwoLevelByteMap.	298	// will still work but will consume more memory for TwoLevelByteMap.
246	#if defined(__mips__)	299	#if defined(__mips__)
247	#if SANITIZER_GO && defined(__mips64)	300	# if SANITIZER_GO && defined(__mips64)
248	#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)	301	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
249	#else	302	# else
250	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40)	303	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40)
251	#endif	304	# endif
252	#elif SANITIZER_RISCV64	305	#elif SANITIZER_RISCV64
253	#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 38)	306	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 38)
254	#elif defined(__aarch64__)	307	#elif defined(__aarch64__)
255	# if SANITIZER_MAC	308	# if SANITIZER_APPLE
256	# if SANITIZER_OSX \|\| SANITIZER_IOSSIM	309	# if SANITIZER_OSX \|\| SANITIZER_IOSSIM
257	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)	310	# define SANITIZER_MMAP_RANGE_SIZE \
		311	FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
		312	# else
		313	// Darwin iOS/ARM64 has a 36-bit VMA, 64GiB VM
		314	# define SANITIZER_MMAP_RANGE_SIZE \
		315	FIRST_32_SECOND_64(1ULL << 32, 1ULL << 36)
		316	# endif
258	# else	317	# else
259	// Darwin iOS/ARM64 has a 36-bit VMA, 64GiB VM	318	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48)
260	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 36)
261	# endif	319	# endif
262	# else
263	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 48)
264	# endif
265	#elif defined(__sparc__)	320	#elif defined(__sparc__)
266	#define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 52)	321	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 52)
267	#else	322	#else
268	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)	323	# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47)
269	#endif	324	#endif
270		325
271	// Whether the addresses are sign-extended from the VMA range to the word.	326	// Whether the addresses are sign-extended from the VMA range to the word.
272	// The SPARC64 Linux port implements this to split the VMA space into two	327	// The SPARC64 Linux port implements this to split the VMA space into two
273	// non-contiguous halves with a huge hole in the middle.	328	// non-contiguous halves with a huge hole in the middle.
274	#if defined(__sparc__) && SANITIZER_WORDSIZE == 64	329	#if defined(__sparc__) && SANITIZER_WORDSIZE == 64
275	#define SANITIZER_SIGN_EXTENDED_ADDRESSES 1	330	# define SANITIZER_SIGN_EXTENDED_ADDRESSES 1
276	#else	331	#else
277	#define SANITIZER_SIGN_EXTENDED_ADDRESSES 0	332	# define SANITIZER_SIGN_EXTENDED_ADDRESSES 0
278	#endif
279
280	// The AArch64 and RISC-V linux ports use the canonical syscall set as
281	// mandated by the upstream linux community for all new ports. Other ports
282	// may still use legacy syscalls.
283	#ifndef SANITIZER_USES_CANONICAL_LINUX_SYSCALLS
284	# if (defined(__aarch64__) \|\| defined(__riscv)) && SANITIZER_LINUX
285	# define SANITIZER_USES_CANONICAL_LINUX_SYSCALLS 1
286	# else
287	# define SANITIZER_USES_CANONICAL_LINUX_SYSCALLS 0
288	# endif
289	#endif	333	#endif
290		334
291	// udi16 syscalls can only be used when the following conditions are	335	// udi16 syscalls can only be used when the following conditions are
...	@@ -296,15 +340,15 @@	...	@@ -296,15 +340,15 @@
296	// Since we don't want to include libc headers here, we check the	340	// Since we don't want to include libc headers here, we check the
297	// target only.	341	// target only.
298	#if defined(__arm__) \|\| SANITIZER_X32 \|\| defined(__sparc__)	342	#if defined(__arm__) \|\| SANITIZER_X32 \|\| defined(__sparc__)
299	#define SANITIZER_USES_UID16_SYSCALLS 1	343	# define SANITIZER_USES_UID16_SYSCALLS 1
300	#else	344	#else
301	#define SANITIZER_USES_UID16_SYSCALLS 0	345	# define SANITIZER_USES_UID16_SYSCALLS 0
302	#endif	346	#endif
303		347
304	#if defined(__mips__)	348	#if defined(__mips__)
305	# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 10)	349	# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 10)
306	#else	350	#else
307	# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 12)	351	# define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 12)
308	#endif	352	#endif
309		353
310	/// \macro MSC_PREREQ	354	/// \macro MSC_PREREQ
...	@@ -313,15 +357,15 @@	...	@@ -313,15 +357,15 @@
313	/// * 1800: Microsoft Visual Studio 2013 / 12.0	357	/// * 1800: Microsoft Visual Studio 2013 / 12.0
314	/// * 1900: Microsoft Visual Studio 2015 / 14.0	358	/// * 1900: Microsoft Visual Studio 2015 / 14.0
315	#ifdef _MSC_VER	359	#ifdef _MSC_VER
316	# define MSC_PREREQ(version) (_MSC_VER >= (version))	360	# define MSC_PREREQ(version) (_MSC_VER >= (version))
317	#else	361	#else
318	# define MSC_PREREQ(version) 0	362	# define MSC_PREREQ(version) 0
319	#endif	363	#endif
320		364
321	#if SANITIZER_MAC && !(defined(__arm64__) && SANITIZER_IOS)	365	#if SANITIZER_APPLE && defined(__x86_64__)
322	# define SANITIZER_NON_UNIQUE_TYPEINFO 0	366	# define SANITIZER_NON_UNIQUE_TYPEINFO 0
323	#else	367	#else
324	# define SANITIZER_NON_UNIQUE_TYPEINFO 1	368	# define SANITIZER_NON_UNIQUE_TYPEINFO 1
325	#endif	369	#endif
326		370
327	// On linux, some architectures had an ABI transition from 64-bit long double	371	// On linux, some architectures had an ABI transition from 64-bit long double
...	@@ -329,11 +373,11 @@	...	@@ -329,11 +373,11 @@
329	// involving long doubles come in two versions, and we need to pass the	373	// involving long doubles come in two versions, and we need to pass the
330	// correct one to dlvsym when intercepting them.	374	// correct one to dlvsym when intercepting them.
331	#if SANITIZER_LINUX && (SANITIZER_S390 \|\| SANITIZER_PPC32 \|\| SANITIZER_PPC64V1)	375	#if SANITIZER_LINUX && (SANITIZER_S390 \|\| SANITIZER_PPC32 \|\| SANITIZER_PPC64V1)
332	#define SANITIZER_NLDBL_VERSION "GLIBC_2.4"	376	# define SANITIZER_NLDBL_VERSION "GLIBC_2.4"
333	#endif	377	#endif
334		378
335	#if SANITIZER_GO == 0	379	#if SANITIZER_GO == 0
336	# define SANITIZER_GO 0	380	# define SANITIZER_GO 0
337	#endif	381	#endif
338		382
339	// On PowerPC and ARM Thumb, calling pthread_exit() causes LSan to detect leaks.	383	// On PowerPC and ARM Thumb, calling pthread_exit() causes LSan to detect leaks.
...	@@ -341,40 +385,64 @@	...	@@ -341,40 +385,64 @@
341	// dlopen mallocs "libgcc_s.so" string which confuses LSan, it fails to realize	385	// dlopen mallocs "libgcc_s.so" string which confuses LSan, it fails to realize
342	// that this allocation happens in dynamic linker and should be ignored.	386	// that this allocation happens in dynamic linker and should be ignored.
343	#if SANITIZER_PPC \|\| defined(__thumb__)	387	#if SANITIZER_PPC \|\| defined(__thumb__)
344	# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 1	388	# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 1
345	#else	389	#else
346	# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 0	390	# define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 0
347	#endif	391	#endif
348		392
349	#if SANITIZER_FREEBSD \|\| SANITIZER_MAC \|\| SANITIZER_NETBSD \|\| \	393	#if SANITIZER_FREEBSD \|\| SANITIZER_APPLE \|\| SANITIZER_NETBSD \|\| SANITIZER_SOLARIS
350	SANITIZER_SOLARIS	394	# define SANITIZER_MADVISE_DONTNEED MADV_FREE
351	# define SANITIZER_MADVISE_DONTNEED MADV_FREE
352	#else	395	#else
353	# define SANITIZER_MADVISE_DONTNEED MADV_DONTNEED	396	# define SANITIZER_MADVISE_DONTNEED MADV_DONTNEED
354	#endif	397	#endif
355		398
356	// Older gcc have issues aligning to a constexpr, and require an integer.	399	// Older gcc have issues aligning to a constexpr, and require an integer.
357	// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=56859 among others.	400	// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=56859 among others.
358	#if defined(__powerpc__) \|\| defined(__powerpc64__)	401	#if defined(__powerpc__) \|\| defined(__powerpc64__)
359	# define SANITIZER_CACHE_LINE_SIZE 128	402	# define SANITIZER_CACHE_LINE_SIZE 128
360	#else	403	#else
361	# define SANITIZER_CACHE_LINE_SIZE 64	404	# define SANITIZER_CACHE_LINE_SIZE 64
362	#endif	405	#endif
363		406
364	// Enable offline markup symbolizer for Fuchsia.	407	// Enable offline markup symbolizer for Fuchsia.
365	#if SANITIZER_FUCHSIA	408	#if SANITIZER_FUCHSIA
366	# define SANITIZER_SYMBOLIZER_MARKUP 1	409	# define SANITIZER_SYMBOLIZER_MARKUP 1
367	#else	410	#else
368	#define SANITIZER_SYMBOLIZER_MARKUP 0	411	# define SANITIZER_SYMBOLIZER_MARKUP 0
369	#endif	412	#endif
370		413
371	// Enable ability to support sanitizer initialization that is	414	// Enable ability to support sanitizer initialization that is
372	// compatible with the sanitizer library being loaded via	415	// compatible with the sanitizer library being loaded via
373	// `dlopen()`.	416	// `dlopen()`.
374	#if SANITIZER_MAC	417	#if SANITIZER_APPLE
375	#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 1	418	# define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 1
		419	#else
		420	# define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 0
		421	#endif
		422
		423	// SANITIZER_SUPPORTS_THREADLOCAL
		424	// 1 - THREADLOCAL macro is supported by target
		425	// 0 - THREADLOCAL macro is not supported by target
		426	#ifndef __has_feature
		427	// TODO: Support other compilers here
		428	# define SANITIZER_SUPPORTS_THREADLOCAL 1
		429	#else
		430	# if __has_feature(tls)
		431	# define SANITIZER_SUPPORTS_THREADLOCAL 1
		432	# else
		433	# define SANITIZER_SUPPORTS_THREADLOCAL 0
		434	# endif
		435	#endif
		436
		437	#if defined(__thumb__) && defined(__linux__)
		438	// Workaround for
		439	// https://lab.llvm.org/buildbot/#/builders/clang-thumbv7-full-2stage
		440	// or
		441	// https://lab.llvm.org/staging/#/builders/clang-thumbv7-full-2stage
		442	// It fails rss_limit_mb_test without meaningful errors.
		443	# define SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL 1
376	#else	444	#else
377	#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 0	445	# define SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL 0
378	#endif	446	#endif
379		447
380	#endif // SANITIZER_PLATFORM_H	448	#endif // SANITIZER_PLATFORM_H

lib/tsan/sanitizer_common/sanitizer_platform_interceptors.h+53-34

 @@ -76,7 +76,7 @@
 #define SI_LINUX 0
 #endif
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
 #define SI_MAC 1
 #define SI_NOT_MAC 0
 #else
 @@ -126,7 +126,7 @@
 #define SI_SOLARIS32 0
 #endif
-#if SANITIZER_POSIX && !SANITIZER_MAC
+#if SANITIZER_POSIX && !SANITIZER_APPLE
 #define SI_POSIX_NOT_MAC 1
 #else
 #define SI_POSIX_NOT_MAC 0
 @@ -229,11 +229,15 @@
   (SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_CLOCK_GETTIME \
   (SI_FREEBSD || SI_NETBSD || SI_LINUX || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID SI_LINUX
+#define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID \
+  (SI_LINUX || SI_FREEBSD || SI_NETBSD)
 #define SANITIZER_INTERCEPT_GETITIMER SI_POSIX
 #define SANITIZER_INTERCEPT_TIME SI_POSIX
 #define SANITIZER_INTERCEPT_GLOB (SI_GLIBC || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_GLOB64 SI_GLIBC
+#define SANITIZER_INTERCEPT___B64_TO SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_DN_COMP_EXPAND SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_POSIX_SPAWN SI_POSIX
 #define SANITIZER_INTERCEPT_WAIT SI_POSIX
 #define SANITIZER_INTERCEPT_INET SI_POSIX
 #define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM SI_POSIX
 @@ -251,7 +255,8 @@
 #define SANITIZER_INTERCEPT_GETHOSTENT_R (SI_FREEBSD || SI_GLIBC || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_GETSOCKOPT SI_POSIX
 #define SANITIZER_INTERCEPT_ACCEPT SI_POSIX
-#define SANITIZER_INTERCEPT_ACCEPT4 (SI_LINUX_NOT_ANDROID || SI_NETBSD)
+#define SANITIZER_INTERCEPT_ACCEPT4 \
+  (SI_LINUX_NOT_ANDROID || SI_NETBSD || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_PACCEPT SI_NETBSD
 #define SANITIZER_INTERCEPT_MODF SI_POSIX
 #define SANITIZER_INTERCEPT_RECVMSG SI_POSIX
 @@ -264,11 +269,11 @@
 #define SANITIZER_INTERCEPT_INET_ATON SI_POSIX
 #define SANITIZER_INTERCEPT_SYSINFO SI_LINUX
 #define SANITIZER_INTERCEPT_READDIR SI_POSIX
-#define SANITIZER_INTERCEPT_READDIR64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
+#define SANITIZER_INTERCEPT_READDIR64 SI_GLIBC || SI_SOLARIS32
 #if SI_LINUX_NOT_ANDROID &&                                                \
     (defined(__i386) || defined(__x86_64) || defined(__mips64) ||          \
      defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
-     defined(__s390__) || SANITIZER_RISCV64)
+     defined(__s390__) || defined(__loongarch__) || SANITIZER_RISCV64)
 #define SANITIZER_INTERCEPT_PTRACE 1
 #else
 #define SANITIZER_INTERCEPT_PTRACE 0
 @@ -303,13 +308,13 @@
 #define SANITIZER_INTERCEPT_XPG_STRERROR_R SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_SCANDIR \
   (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_SCANDIR64 SI_LINUX_NOT_ANDROID || SI_SOLARIS32
+#define SANITIZER_INTERCEPT_SCANDIR64 SI_GLIBC || SI_SOLARIS32
 #define SANITIZER_INTERCEPT_GETGROUPS SI_POSIX
 #define SANITIZER_INTERCEPT_POLL SI_POSIX
 #define SANITIZER_INTERCEPT_PPOLL SI_LINUX_NOT_ANDROID || SI_SOLARIS
 #define SANITIZER_INTERCEPT_WORDEXP                                          \
   (SI_FREEBSD || SI_NETBSD || (SI_MAC && !SI_IOS) || SI_LINUX_NOT_ANDROID || \
-   SI_SOLARIS)  // NOLINT
+   SI_SOLARIS)
 #define SANITIZER_INTERCEPT_SIGWAIT SI_POSIX
 #define SANITIZER_INTERCEPT_SIGWAITINFO SI_LINUX_NOT_ANDROID || SI_SOLARIS
 #define SANITIZER_INTERCEPT_SIGTIMEDWAIT SI_LINUX_NOT_ANDROID || SI_SOLARIS
 @@ -325,11 +330,10 @@
 #define SANITIZER_INTERCEPT_GETMNTENT_R SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT_STATFS \
   (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_STATFS64 \
+#define SANITIZER_INTERCEPT_STATFS64 SI_GLIBC && SANITIZER_HAS_STATFS64
-  (((SI_MAC && !TARGET_CPU_ARM64) && !SI_IOS) || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_STATVFS \
   (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
-#define SANITIZER_INTERCEPT_STATVFS64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_STATVFS64 SI_GLIBC
 #define SANITIZER_INTERCEPT_INITGROUPS SI_POSIX
 #define SANITIZER_INTERCEPT_ETHER_NTOA_ATON SI_POSIX
 #define SANITIZER_INTERCEPT_ETHER_HOST \
 @@ -337,12 +341,14 @@
 #define SANITIZER_INTERCEPT_ETHER_R (SI_FREEBSD || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_SHMCTL                                       \
   (((SI_FREEBSD || SI_LINUX_NOT_ANDROID) && SANITIZER_WORDSIZE == 64) || \
-   SI_NETBSD || SI_SOLARIS)  // NOLINT
+   SI_NETBSD || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_RANDOM_R SI_GLIBC
 #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET SI_POSIX
 #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \
   (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP SI_GLIBC
+#define SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP \
+  (SI_LINUX_NOT_ANDROID || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET_SCHED SI_POSIX
 #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED \
   (SI_POSIX && !SI_NETBSD)
 @@ -362,6 +368,8 @@
   (SI_LINUX_NOT_ANDROID || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED \
   (SI_LINUX_NOT_ANDROID && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_TRYJOIN SI_GLIBC
+#define SANITIZER_INTERCEPT_TIMEDJOIN SI_GLIBC
 #define SANITIZER_INTERCEPT_THR_EXIT SI_FREEBSD
 #define SANITIZER_INTERCEPT_TMPNAM SI_POSIX
 #define SANITIZER_INTERCEPT_TMPNAM_R (SI_GLIBC || SI_SOLARIS)
 @@ -391,8 +399,6 @@
 #define SANITIZER_INTERCEPT__EXIT \
   (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_MAC || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_PTHREAD_MUTEX SI_POSIX
-#define SANITIZER_INTERCEPT___PTHREAD_MUTEX SI_GLIBC
 #define SANITIZER_INTERCEPT___LIBC_MUTEX SI_NETBSD
 #define SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP \
   (SI_FREEBSD || SI_NETBSD || SI_GLIBC || SI_SOLARIS)
 @@ -400,7 +406,7 @@
   (SI_FREEBSD || SI_NETBSD || SI_GLIBC || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_TLS_GET_ADDR \
-  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
+  (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID)
 #define SANITIZER_INTERCEPT_LISTXATTR SI_LINUX
 #define SANITIZER_INTERCEPT_GETXATTR SI_LINUX
 @@ -445,7 +451,8 @@
 #define SANITIZER_INTERCEPT_SEM \
   (SI_LINUX || SI_FREEBSD || SI_NETBSD || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL SI_POSIX
-#define SANITIZER_INTERCEPT_MINCORE (SI_LINUX || SI_NETBSD || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_MINCORE \
+  (SI_LINUX || SI_NETBSD || SI_FREEBSD || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_PROCESS_VM_READV SI_LINUX
 #define SANITIZER_INTERCEPT_CTERMID \
   (SI_LINUX || SI_MAC || SI_FREEBSD || SI_NETBSD || SI_SOLARIS)
 @@ -457,13 +464,17 @@
 #define SANITIZER_INTERCEPT_SEND_SENDTO SI_POSIX
 #define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE SI_LINUX
-#define SANITIZER_INTERCEPT_STAT \
+#define SI_STAT_LINUX (SI_LINUX && __GLIBC_PREREQ(2, 33))
-  (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD || SI_SOLARIS)
+#define SANITIZER_INTERCEPT_STAT                                        \
-#define SANITIZER_INTERCEPT_LSTAT (SI_NETBSD || SI_FREEBSD)
+  (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD || SI_SOLARIS ||     \
-#define SANITIZER_INTERCEPT___XSTAT (!SANITIZER_INTERCEPT_STAT && SI_POSIX)
+   SI_STAT_LINUX)
-#define SANITIZER_INTERCEPT___XSTAT64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_STAT64 SI_STAT_LINUX && SANITIZER_HAS_STAT64
+#define SANITIZER_INTERCEPT_LSTAT (SI_NETBSD || SI_FREEBSD || SI_STAT_LINUX)
+#define SANITIZER_INTERCEPT___XSTAT \
+  ((!SANITIZER_INTERCEPT_STAT && SI_POSIX) || SI_STAT_LINUX)
+#define SANITIZER_INTERCEPT___XSTAT64 SI_GLIBC
 #define SANITIZER_INTERCEPT___LXSTAT SANITIZER_INTERCEPT___XSTAT
-#define SANITIZER_INTERCEPT___LXSTAT64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT___LXSTAT64 SI_GLIBC
 #define SANITIZER_INTERCEPT_UTMP \
   (SI_POSIX && !SI_MAC && !SI_FREEBSD && !SI_NETBSD)
 @@ -474,7 +485,7 @@
   (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD || SI_NETBSD)
 #define SANITIZER_INTERCEPT_MMAP SI_POSIX
-#define SANITIZER_INTERCEPT_MMAP64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_MMAP64 SI_GLIBC || SI_SOLARIS
 #define SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO (SI_GLIBC || SI_ANDROID)
 #define SANITIZER_INTERCEPT_MEMALIGN (!SI_FREEBSD && !SI_MAC && !SI_NETBSD)
 #define SANITIZER_INTERCEPT___LIBC_MEMALIGN SI_GLIBC
 @@ -484,6 +495,7 @@
 #define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC)
 #define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD)
 #define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_WCSLEN 1
 #define SANITIZER_INTERCEPT_WCSCAT SI_POSIX
 #define SANITIZER_INTERCEPT_WCSDUP SI_POSIX
 #define SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION (!SI_WINDOWS && SI_NOT_FUCHSIA)
 @@ -496,7 +508,8 @@
 #define SANITIZER_INTERCEPT_GID_FROM_GROUP SI_NETBSD
 #define SANITIZER_INTERCEPT_ACCESS (SI_NETBSD || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_FACCESSAT (SI_NETBSD || SI_FREEBSD)
-#define SANITIZER_INTERCEPT_GETGROUPLIST SI_NETBSD
+#define SANITIZER_INTERCEPT_GETGROUPLIST \
+  (SI_NETBSD || SI_FREEBSD || SI_LINUX)
 #define SANITIZER_INTERCEPT_STRLCPY \
   (SI_NETBSD || SI_FREEBSD || SI_MAC || SI_ANDROID)
 @@ -517,10 +530,11 @@
 #define SANITIZER_INTERCEPT_DEVNAME_R (SI_NETBSD || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_FGETLN (SI_NETBSD || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_STRMODE (SI_NETBSD || SI_FREEBSD)
-#define SANITIZER_INTERCEPT_TTYENT SI_NETBSD
+#define SANITIZER_INTERCEPT_TTYENT (SI_NETBSD || SI_FREEBSD)
-#define SANITIZER_INTERCEPT_PROTOENT (SI_NETBSD || SI_LINUX)
+#define SANITIZER_INTERCEPT_TTYENTPATH SI_NETBSD
+#define SANITIZER_INTERCEPT_PROTOENT (SI_LINUX || SI_NETBSD || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_PROTOENT_R SI_GLIBC
-#define SANITIZER_INTERCEPT_NETENT SI_NETBSD
+#define SANITIZER_INTERCEPT_NETENT (SI_LINUX || SI_NETBSD || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_SETVBUF \
   (SI_NETBSD || SI_FREEBSD || SI_LINUX || SI_MAC)
 #define SANITIZER_INTERCEPT_GETMNTINFO (SI_NETBSD || SI_FREEBSD || SI_MAC)
 @@ -536,17 +550,17 @@
 #define SANITIZER_INTERCEPT_MODCTL SI_NETBSD
 #define SANITIZER_INTERCEPT_CAPSICUM SI_FREEBSD
 #define SANITIZER_INTERCEPT_STRTONUM (SI_NETBSD || SI_FREEBSD)
-#define SANITIZER_INTERCEPT_FPARSELN SI_NETBSD
+#define SANITIZER_INTERCEPT_FPARSELN (SI_NETBSD || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_STATVFS1 SI_NETBSD
 #define SANITIZER_INTERCEPT_STRTOI SI_NETBSD
 #define SANITIZER_INTERCEPT_CAPSICUM SI_FREEBSD
 #define SANITIZER_INTERCEPT_SHA1 SI_NETBSD
 #define SANITIZER_INTERCEPT_MD4 SI_NETBSD
 #define SANITIZER_INTERCEPT_RMD160 SI_NETBSD
-#define SANITIZER_INTERCEPT_MD5 SI_NETBSD
+#define SANITIZER_INTERCEPT_MD5 (SI_NETBSD || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_FSEEK (SI_NETBSD || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_MD2 SI_NETBSD
-#define SANITIZER_INTERCEPT_SHA2 SI_NETBSD
+#define SANITIZER_INTERCEPT_SHA2 (SI_NETBSD || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_CDB SI_NETBSD
 #define SANITIZER_INTERCEPT_VIS (SI_NETBSD || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_POPEN SI_POSIX
 @@ -559,25 +573,30 @@
 #define SANITIZER_INTERCEPT_FDEVNAME SI_FREEBSD
 #define SANITIZER_INTERCEPT_GETUSERSHELL (SI_POSIX && !SI_ANDROID)
 #define SANITIZER_INTERCEPT_SL_INIT (SI_FREEBSD || SI_NETBSD)
-#define SANITIZER_INTERCEPT_CRYPT (SI_POSIX && !SI_ANDROID)
-#define SANITIZER_INTERCEPT_CRYPT_R (SI_LINUX && !SI_ANDROID)
 #define SANITIZER_INTERCEPT_GETRANDOM \
   ((SI_LINUX && __GLIBC_PREREQ(2, 25)) || SI_FREEBSD)
 #define SANITIZER_INTERCEPT___CXA_ATEXIT SI_NETBSD
 #define SANITIZER_INTERCEPT_ATEXIT SI_NETBSD
 #define SANITIZER_INTERCEPT_PTHREAD_ATFORK SI_NETBSD
-#define SANITIZER_INTERCEPT_GETENTROPY SI_FREEBSD
+#define SANITIZER_INTERCEPT_GETENTROPY \
+  ((SI_LINUX && __GLIBC_PREREQ(2, 25)) || SI_FREEBSD)
 #define SANITIZER_INTERCEPT_QSORT \
   (SI_POSIX && !SI_IOSSIM && !SI_WATCHOS && !SI_TVOS && !SI_ANDROID)
 #define SANITIZER_INTERCEPT_QSORT_R SI_GLIBC
+#define SANITIZER_INTERCEPT_BSEARCH \
+  (SI_POSIX && !SI_IOSSIM && !SI_WATCHOS && !SI_TVOS && !SI_ANDROID)
 // sigaltstack on i386 macOS cannot be intercepted due to setjmp()
 // calling it and assuming that it does not clobber registers.
 #define SANITIZER_INTERCEPT_SIGALTSTACK \
-  (SI_POSIX && !(SANITIZER_MAC && SANITIZER_I386))
+  (SI_POSIX && !(SANITIZER_APPLE && SANITIZER_I386))
 #define SANITIZER_INTERCEPT_UNAME (SI_POSIX && !SI_FREEBSD)
 #define SANITIZER_INTERCEPT___XUNAME SI_FREEBSD
 #define SANITIZER_INTERCEPT_FLOPEN SI_FREEBSD
+#define SANITIZER_INTERCEPT_PROCCTL SI_FREEBSD
+#define SANITIZER_INTERCEPT_HEXDUMP SI_FREEBSD
+#define SANITIZER_INTERCEPT_ARGP_PARSE SI_GLIBC
+#define SANITIZER_INTERCEPT_CPUSET_GETAFFINITY SI_FREEBSD
 // This macro gives a way for downstream users to override the above
 // interceptor macros irrespective of the platform they are on. They have

lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.cpp+39-1

 @@ -17,6 +17,7 @@
 #include <sys/capsicum.h>
 #include <sys/consio.h>
+#include <sys/cpuset.h>
 #include <sys/filio.h>
 #include <sys/ipc.h>
 #include <sys/kbio.h>
 @@ -69,11 +70,17 @@
 #include <semaphore.h>
 #include <signal.h>
 #include <stddef.h>
+#include <md5.h>
+#include <sha224.h>
+#include <sha256.h>
+#include <sha384.h>
+#include <sha512.h>
 #include <stdio.h>
 #include <stringlist.h>
 #include <term.h>
 #include <termios.h>
 #include <time.h>
+#include <ttyent.h>
 #include <utime.h>
 #include <utmpx.h>
 #include <vis.h>
 @@ -97,6 +104,7 @@ void *__sanitizer_get_link_map_by_dlopen_handle(void *handle) {
   return internal_dlinfo(handle, RTLD_DI_LINKMAP, &p) == 0 ? p : nullptr;
 }
+unsigned struct_cpuset_sz = sizeof(cpuset_t);
 unsigned struct_cap_rights_sz = sizeof(cap_rights_t);
 unsigned struct_utsname_sz = sizeof(struct utsname);
 unsigned struct_stat_sz = sizeof(struct stat);
 @@ -124,7 +132,7 @@ unsigned struct_sigevent_sz = sizeof(struct sigevent);
 unsigned struct_sched_param_sz = sizeof(struct sched_param);
 unsigned struct_statfs_sz = sizeof(struct statfs);
 unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
-unsigned ucontext_t_sz = sizeof(ucontext_t);
+unsigned ucontext_t_sz(void *ctx) { return sizeof(ucontext_t); }
 unsigned struct_rlimit_sz = sizeof(struct rlimit);
 unsigned struct_timespec_sz = sizeof(struct timespec);
 unsigned struct_utimbuf_sz = sizeof(struct utimbuf);
 @@ -167,12 +175,21 @@ uptr __sanitizer_in_addr_sz(int af) {
     return 0;
 }
+// For FreeBSD the actual size of a directory entry is not always in d_reclen.
+// Use the appropriate macro to get the correct size for all cases (e.g. NFS).
+u16 __sanitizer_dirsiz(const __sanitizer_dirent *dp) {
+  return _GENERIC_DIRSIZ(dp);
+}
 unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
 int glob_nomatch = GLOB_NOMATCH;
 int glob_altdirfunc = GLOB_ALTDIRFUNC;
+const int wordexp_wrde_dooffs = WRDE_DOOFFS;
 unsigned path_max = PATH_MAX;
+int struct_ttyent_sz = sizeof(struct ttyent);
 // ioctl arguments
 unsigned struct_ifreq_sz = sizeof(struct ifreq);
 unsigned struct_termios_sz = sizeof(struct termios);
 @@ -196,6 +213,10 @@ unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info);
 unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats);
 unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
 unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
+unsigned struct_procctl_reaper_status_sz = sizeof(struct __sanitizer_procctl_reaper_status);
+unsigned struct_procctl_reaper_pidinfo_sz = sizeof(struct __sanitizer_procctl_reaper_pidinfo);
+unsigned struct_procctl_reaper_pids_sz = sizeof(struct __sanitizer_procctl_reaper_pids);
+unsigned struct_procctl_reaper_kill_sz = sizeof(struct __sanitizer_procctl_reaper_kill);
 const unsigned long __sanitizer_bufsiz = BUFSIZ;
 const unsigned IOCTL_NOT_PRESENT = 0;
 @@ -357,6 +378,22 @@ const int si_SEGV_MAPERR = SEGV_MAPERR;
 const int si_SEGV_ACCERR = SEGV_ACCERR;
 const int unvis_valid = UNVIS_VALID;
 const int unvis_validpush = UNVIS_VALIDPUSH;
+const unsigned MD5_CTX_sz = sizeof(MD5_CTX);
+const unsigned MD5_return_length = MD5_DIGEST_STRING_LENGTH;
+#define SHA2_CONST(LEN)                                                      \
+  const unsigned SHA##LEN##_CTX_sz = sizeof(SHA##LEN##_CTX);                 \
+  const unsigned SHA##LEN##_return_length = SHA##LEN##_DIGEST_STRING_LENGTH; \
+  const unsigned SHA##LEN##_block_length = SHA##LEN##_BLOCK_LENGTH;          \
+  const unsigned SHA##LEN##_digest_length = SHA##LEN##_DIGEST_LENGTH
+SHA2_CONST(224);
+SHA2_CONST(256);
+SHA2_CONST(384);
+SHA2_CONST(512);
+#undef SHA2_CONST
 }  // namespace __sanitizer
 using namespace __sanitizer;
 @@ -529,4 +566,5 @@ COMPILER_CHECK(__sanitizer_XDR_FREE == XDR_FREE);
 CHECK_TYPE_SIZE(sem_t);
 COMPILER_CHECK(sizeof(__sanitizer_cap_rights_t) >= sizeof(cap_rights_t));
+COMPILER_CHECK(sizeof(__sanitizer_cpuset_t) >= sizeof(cpuset_t));
 #endif  // SANITIZER_FREEBSD

lib/tsan/sanitizer_common/sanitizer_platform_limits_freebsd.h+167-71

...	@@ -16,26 +16,26 @@	...	@@ -16,26 +16,26 @@
16		16
17	#if SANITIZER_FREEBSD	17	#if SANITIZER_FREEBSD
18		18
19	#include "sanitizer_internal_defs.h"	19	# include "sanitizer_internal_defs.h"
20	#include "sanitizer_platform.h"	20	# include "sanitizer_platform.h"
21	#include "sanitizer_platform_limits_posix.h"	21	# include "sanitizer_platform_limits_posix.h"
22		22
23	// Get sys/_types.h, because that tells us whether 64-bit inodes are	23	// Get sys/_types.h, because that tells us whether 64-bit inodes are
24	// used in struct dirent below.	24	// used in struct dirent below.
25	#include <sys/_types.h>	25	# include <sys/_types.h>
26		26
27	namespace __sanitizer {	27	namespace __sanitizer {
28	void __sanitizer_get_link_map_by_dlopen_handle(void handle);	28	void __sanitizer_get_link_map_by_dlopen_handle(void handle);
29	#define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \	29	# define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \
30	(link_map *)__sanitizer_get_link_map_by_dlopen_handle(handle)	30	(link_map *)__sanitizer_get_link_map_by_dlopen_handle(handle)
31		31
32	extern unsigned struct_utsname_sz;	32	extern unsigned struct_utsname_sz;
33	extern unsigned struct_stat_sz;	33	extern unsigned struct_stat_sz;
34	#if defined(__powerpc64__)	34	# if defined(__powerpc64__)
35	const unsigned struct___old_kernel_stat_sz = 0;	35	const unsigned struct___old_kernel_stat_sz = 0;
36	#else	36	# else
37	const unsigned struct___old_kernel_stat_sz = 32;	37	const unsigned struct___old_kernel_stat_sz = 32;
38	#endif	38	# endif
39	extern unsigned struct_rusage_sz;	39	extern unsigned struct_rusage_sz;
40	extern unsigned siginfo_t_sz;	40	extern unsigned siginfo_t_sz;
41	extern unsigned struct_itimerval_sz;	41	extern unsigned struct_itimerval_sz;
...	@@ -57,7 +57,7 @@ extern unsigned struct_sched_param_sz;	...	@@ -57,7 +57,7 @@ extern unsigned struct_sched_param_sz;
57	extern unsigned struct_statfs64_sz;	57	extern unsigned struct_statfs64_sz;
58	extern unsigned struct_statfs_sz;	58	extern unsigned struct_statfs_sz;
59	extern unsigned struct_sockaddr_sz;	59	extern unsigned struct_sockaddr_sz;
60	extern unsigned ucontext_t_sz;	60	unsigned ucontext_t_sz(void *ctx);
61	extern unsigned struct_rlimit_sz;	61	extern unsigned struct_rlimit_sz;
62	extern unsigned struct_utimbuf_sz;	62	extern unsigned struct_utimbuf_sz;
63	extern unsigned struct_timespec_sz;	63	extern unsigned struct_timespec_sz;
...	@@ -114,11 +114,24 @@ struct __sanitizer_ipc_perm {	...	@@ -114,11 +114,24 @@ struct __sanitizer_ipc_perm {
114	long key;	114	long key;
115	};	115	};
116		116
117	#if !defined(__i386__)	117	struct __sanitizer_protoent {
		118	char *p_name;
		119	char **p_aliases;
		120	int p_proto;
		121	};
		122
		123	struct __sanitizer_netent {
		124	char *n_name;
		125	char **n_aliases;
		126	int n_addrtype;
		127	u32 n_net;
		128	};
		129
		130	# if !defined(__i386__)
118	typedef long long __sanitizer_time_t;	131	typedef long long __sanitizer_time_t;
119	#else	132	# else
120	typedef long __sanitizer_time_t;	133	typedef long __sanitizer_time_t;
121	#endif	134	# endif
122		135
123	struct __sanitizer_shmid_ds {	136	struct __sanitizer_shmid_ds {
124	__sanitizer_ipc_perm shm_perm;	137	__sanitizer_ipc_perm shm_perm;
...	@@ -147,7 +160,7 @@ struct __sanitizer_ifaddrs {	...	@@ -147,7 +160,7 @@ struct __sanitizer_ifaddrs {
147	unsigned int ifa_flags;	160	unsigned int ifa_flags;
148	void ifa_addr; // (struct sockaddr )	161	void ifa_addr; // (struct sockaddr )
149	void ifa_netmask; // (struct sockaddr )	162	void ifa_netmask; // (struct sockaddr )
150	#undef ifa_dstaddr	163	# undef ifa_dstaddr
151	void ifa_dstaddr; // (struct sockaddr )	164	void ifa_dstaddr; // (struct sockaddr )
152	void *ifa_data;	165	void *ifa_data;
153	};	166	};
...	@@ -229,37 +242,43 @@ struct __sanitizer_cmsghdr {	...	@@ -229,37 +242,43 @@ struct __sanitizer_cmsghdr {
229	};	242	};
230		243
231	struct __sanitizer_dirent {	244	struct __sanitizer_dirent {
232	#if defined(__INO64)	245	# if defined(__INO64)
233	unsigned long long d_fileno;	246	unsigned long long d_fileno;
234	unsigned long long d_off;	247	unsigned long long d_off;
235	#else	248	# else
236	unsigned int d_fileno;	249	unsigned int d_fileno;
237	#endif	250	# endif
238	unsigned short d_reclen;	251	unsigned short d_reclen;
239	// more fields that we don't care about	252	u8 d_type;
		253	u8 d_pad0;
		254	u16 d_namlen;
		255	u16 d_pad1;
		256	char d_name[256];
240	};	257	};
241		258
		259	u16 __sanitizer_dirsiz(const __sanitizer_dirent *dp);
		260
242	// 'clock_t' is 32 bits wide on x64 FreeBSD	261	// 'clock_t' is 32 bits wide on x64 FreeBSD
243	typedef int __sanitizer_clock_t;	262	typedef int __sanitizer_clock_t;
244	typedef int __sanitizer_clockid_t;	263	typedef int __sanitizer_clockid_t;
245		264
246	#if defined(_LP64) \|\| defined(__x86_64__) \|\| defined(__powerpc__) \|\| \	265	# if defined(_LP64) \|\| defined(__x86_64__) \|\| defined(__powerpc__) \|\| \
247	defined(__mips__)	266	defined(__mips__)
248	typedef unsigned __sanitizer___kernel_uid_t;	267	typedef unsigned __sanitizer___kernel_uid_t;
249	typedef unsigned __sanitizer___kernel_gid_t;	268	typedef unsigned __sanitizer___kernel_gid_t;
250	#else	269	# else
251	typedef unsigned short __sanitizer___kernel_uid_t;	270	typedef unsigned short __sanitizer___kernel_uid_t;
252	typedef unsigned short __sanitizer___kernel_gid_t;	271	typedef unsigned short __sanitizer___kernel_gid_t;
253	#endif	272	# endif
254	typedef long long __sanitizer___kernel_off_t;	273	typedef long long __sanitizer___kernel_off_t;
255		274
256	#if defined(__powerpc__) \|\| defined(__mips__)	275	# if defined(__powerpc__) \|\| defined(__mips__)
257	typedef unsigned int __sanitizer___kernel_old_uid_t;	276	typedef unsigned int __sanitizer___kernel_old_uid_t;
258	typedef unsigned int __sanitizer___kernel_old_gid_t;	277	typedef unsigned int __sanitizer___kernel_old_gid_t;
259	#else	278	# else
260	typedef unsigned short __sanitizer___kernel_old_uid_t;	279	typedef unsigned short __sanitizer___kernel_old_uid_t;
261	typedef unsigned short __sanitizer___kernel_old_gid_t;	280	typedef unsigned short __sanitizer___kernel_old_gid_t;
262	#endif	281	# endif
263		282
264	typedef long long __sanitizer___kernel_loff_t;	283	typedef long long __sanitizer___kernel_loff_t;
265	typedef struct {	284	typedef struct {
...	@@ -366,9 +385,12 @@ struct __sanitizer_glob_t {	...	@@ -366,9 +385,12 @@ struct __sanitizer_glob_t {
366		385
367	extern int glob_nomatch;	386	extern int glob_nomatch;
368	extern int glob_altdirfunc;	387	extern int glob_altdirfunc;
		388	extern const int wordexp_wrde_dooffs;
369		389
370	extern unsigned path_max;	390	extern unsigned path_max;
371		391
		392	extern int struct_ttyent_sz;
		393
372	struct __sanitizer_wordexp_t {	394	struct __sanitizer_wordexp_t {
373	uptr we_wordc;	395	uptr we_wordc;
374	char **we_wordv;	396	char **we_wordv;
...	@@ -398,39 +420,81 @@ struct __sanitizer_ifconf {	...	@@ -398,39 +420,81 @@ struct __sanitizer_ifconf {
398	} ifc_ifcu;	420	} ifc_ifcu;
399	};	421	};
400		422
401	#define IOC_NRBITS 8	423	struct __sanitizer__ttyent {
402	#define IOC_TYPEBITS 8	424	char *ty_name;
403	#if defined(__powerpc__) \|\| defined(__powerpc64__) \|\| defined(__mips__)	425	char *ty_getty;
404	#define IOC_SIZEBITS 13	426	char *ty_type;
405	#define IOC_DIRBITS 3	427	int ty_status;
406	#define IOC_NONE 1U	428	char *ty_window;
407	#define IOC_WRITE 4U	429	char *ty_comment;
408	#define IOC_READ 2U	430	char *ty_group;
409	#else	431	};
410	#define IOC_SIZEBITS 14	432
411	#define IOC_DIRBITS 2	433	// procctl reaper data for PROCCTL_REAPER flags
412	#define IOC_NONE 0U	434	struct __sanitizer_procctl_reaper_status {
413	#define IOC_WRITE 1U	435	unsigned int rs_flags;
414	#define IOC_READ 2U	436	unsigned int rs_children;
415	#endif	437	unsigned int rs_descendants;
416	#define IOC_NRMASK ((1 << IOC_NRBITS) - 1)	438	pid_t rs_reaper;
417	#define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1)	439	pid_t rs_pid;
418	#define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1)	440	unsigned int rs_pad0[15];
419	#if defined(IOC_DIRMASK)	441	};
420	#undef IOC_DIRMASK	442
421	#endif	443	struct __sanitizer_procctl_reaper_pidinfo {
422	#define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1)	444	pid_t pi_pid;
423	#define IOC_NRSHIFT 0	445	pid_t pi_subtree;
424	#define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS)	446	unsigned int pi_flags;
425	#define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS)	447	unsigned int pi_pad0[15];
426	#define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS)	448	};
427	#define EVIOC_EV_MAX 0x1f	449
428	#define EVIOC_ABS_MAX 0x3f	450	struct __sanitizer_procctl_reaper_pids {
429		451	unsigned int rp_count;
430	#define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK)	452	unsigned int rp_pad0[15];
431	#define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK)	453	struct __sanitize_procctl_reapper_pidinfo *rp_pids;
432	#define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK)	454	};
433	#define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK)	455
		456	struct __sanitizer_procctl_reaper_kill {
		457	int rk_sig;
		458	unsigned int rk_flags;
		459	pid_t rk_subtree;
		460	unsigned int rk_killed;
		461	pid_t rk_fpid;
		462	unsigned int rk_pad[15];
		463	};
		464
		465	# define IOC_NRBITS 8
		466	# define IOC_TYPEBITS 8
		467	# if defined(__powerpc__) \|\| defined(__powerpc64__) \|\| defined(__mips__)
		468	# define IOC_SIZEBITS 13
		469	# define IOC_DIRBITS 3
		470	# define IOC_NONE 1U
		471	# define IOC_WRITE 4U
		472	# define IOC_READ 2U
		473	# else
		474	# define IOC_SIZEBITS 14
		475	# define IOC_DIRBITS 2
		476	# define IOC_NONE 0U
		477	# define IOC_WRITE 1U
		478	# define IOC_READ 2U
		479	# endif
		480	# define IOC_NRMASK ((1 << IOC_NRBITS) - 1)
		481	# define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1)
		482	# define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1)
		483	# if defined(IOC_DIRMASK)
		484	# undef IOC_DIRMASK
		485	# endif
		486	# define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1)
		487	# define IOC_NRSHIFT 0
		488	# define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS)
		489	# define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS)
		490	# define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS)
		491	# define EVIOC_EV_MAX 0x1f
		492	# define EVIOC_ABS_MAX 0x3f
		493
		494	# define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK)
		495	# define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK)
		496	# define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK)
		497	# define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK)
434		498
435	extern unsigned struct_ifreq_sz;	499	extern unsigned struct_ifreq_sz;
436	extern unsigned struct_termios_sz;	500	extern unsigned struct_termios_sz;
...	@@ -454,6 +518,11 @@ extern unsigned struct_ppp_stats_sz;	...	@@ -454,6 +518,11 @@ extern unsigned struct_ppp_stats_sz;
454	extern unsigned struct_sioc_sg_req_sz;	518	extern unsigned struct_sioc_sg_req_sz;
455	extern unsigned struct_sioc_vif_req_sz;	519	extern unsigned struct_sioc_vif_req_sz;
456		520
		521	extern unsigned struct_procctl_reaper_status_sz;
		522	extern unsigned struct_procctl_reaper_pidinfo_sz;
		523	extern unsigned struct_procctl_reaper_pids_sz;
		524	extern unsigned struct_procctl_reaper_kill_sz;
		525
457	// ioctl request identifiers	526	// ioctl request identifiers
458		527
459	// A special value to mark ioctls that are not present on the target platform,	528	// A special value to mark ioctls that are not present on the target platform,
...	@@ -621,6 +690,22 @@ extern unsigned IOCTL_KDSKBMODE;	...	@@ -621,6 +690,22 @@ extern unsigned IOCTL_KDSKBMODE;
621	extern const int si_SEGV_MAPERR;	690	extern const int si_SEGV_MAPERR;
622	extern const int si_SEGV_ACCERR;	691	extern const int si_SEGV_ACCERR;
623		692
		693	extern const unsigned MD5_CTX_sz;
		694	extern const unsigned MD5_return_length;
		695
		696	#define SHA2_EXTERN(LEN) \
		697	extern const unsigned SHA##LEN##_CTX_sz; \
		698	extern const unsigned SHA##LEN##_return_length; \
		699	extern const unsigned SHA##LEN##_block_length; \
		700	extern const unsigned SHA##LEN##_digest_length
		701
		702	SHA2_EXTERN(224);
		703	SHA2_EXTERN(256);
		704	SHA2_EXTERN(384);
		705	SHA2_EXTERN(512);
		706
		707	#undef SHA2_EXTERN
		708
624	struct __sanitizer_cap_rights {	709	struct __sanitizer_cap_rights {
625	u64 cr_rights[2];	710	u64 cr_rights[2];
626	};	711	};
...	@@ -630,26 +715,37 @@ extern unsigned struct_cap_rights_sz;	...	@@ -630,26 +715,37 @@ extern unsigned struct_cap_rights_sz;
630		715
631	extern unsigned struct_fstab_sz;	716	extern unsigned struct_fstab_sz;
632	extern unsigned struct_StringList_sz;	717	extern unsigned struct_StringList_sz;
		718
		719	struct __sanitizer_cpuset {
		720	#if __FreeBSD_version >= 1400090
		721	long __bits[(1024 + (sizeof(long) * 8) - 1) / (sizeof(long) * 8)];
		722	#else
		723	long __bits[(256 + (sizeof(long) * 8) - 1) / (sizeof(long) * 8)];
		724	#endif
		725	};
		726
		727	typedef struct __sanitizer_cpuset __sanitizer_cpuset_t;
		728	extern unsigned struct_cpuset_sz;
633	} // namespace __sanitizer	729	} // namespace __sanitizer
634		730
635	#define CHECK_TYPE_SIZE(TYPE) \	731	# define CHECK_TYPE_SIZE(TYPE) \
636	COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE))	732	COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE))
637		733
638	#define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \	734	# define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \
639	COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *)NULL)->MEMBER) == \	735	COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *)NULL)->MEMBER) == \
640	sizeof(((CLASS *)NULL)->MEMBER)); \	736	sizeof(((CLASS *)NULL)->MEMBER)); \
641	COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \	737	COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \
642	offsetof(CLASS, MEMBER))	738	offsetof(CLASS, MEMBER))
643		739
644	// For sigaction, which is a function and struct at the same time,	740	// For sigaction, which is a function and struct at the same time,
645	// and thus requires explicit "struct" in sizeof() expression.	741	// and thus requires explicit "struct" in sizeof() expression.
646	#define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \	742	# define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \
647	COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *)NULL)->MEMBER) == \	743	COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *)NULL)->MEMBER) == \
648	sizeof(((struct CLASS *)NULL)->MEMBER)); \	744	sizeof(((struct CLASS *)NULL)->MEMBER)); \
649	COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \	745	COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \
650	offsetof(struct CLASS, MEMBER))	746	offsetof(struct CLASS, MEMBER))
651		747
652	#define SIGACTION_SYMNAME sigaction	748	# define SIGACTION_SYMNAME sigaction
653		749
654	#endif	750	#endif
655		751

lib/tsan/sanitizer_common/sanitizer_platform_limits_linux.cpp+29-34

...	@@ -28,44 +28,39 @@	...	@@ -28,44 +28,39 @@
28	// are not defined anywhere in userspace headers. Fake them. This seems to work	28	// are not defined anywhere in userspace headers. Fake them. This seems to work
29	// fine with newer headers, too.	29	// fine with newer headers, too.
30	#include <linux/posix_types.h>	30	#include <linux/posix_types.h>
31	#if defined(__x86_64__) \|\| defined(__mips__)	31	# if defined(__x86_64__) \|\| defined(__mips__) \|\| defined(__hexagon__)
32	#include <sys/stat.h>	32	# include <sys/stat.h>
33	#else	33	# else
34	#define ino_t __kernel_ino_t	34	# define ino_t __kernel_ino_t
35	#define mode_t __kernel_mode_t	35	# define mode_t __kernel_mode_t
36	#define nlink_t __kernel_nlink_t	36	# define nlink_t __kernel_nlink_t
37	#define uid_t __kernel_uid_t	37	# define uid_t __kernel_uid_t
38	#define gid_t __kernel_gid_t	38	# define gid_t __kernel_gid_t
39	#define off_t __kernel_off_t	39	# define off_t __kernel_off_t
40	#define time_t __kernel_time_t	40	# define time_t __kernel_time_t
41	// This header seems to contain the definitions of _kernel_ stat* structs.	41	// This header seems to contain the definitions of _kernel_ stat* structs.
42	#include <asm/stat.h>	42	# include <asm/stat.h>
43	#undef ino_t	43	# undef ino_t
44	#undef mode_t	44	# undef mode_t
45	#undef nlink_t	45	# undef nlink_t
46	#undef uid_t	46	# undef uid_t
47	#undef gid_t	47	# undef gid_t
48	#undef off_t	48	# undef off_t
49	#endif	49	# endif
50		50
51	#include <linux/aio_abi.h>	51	# include <linux/aio_abi.h>
52		52
53	#if !SANITIZER_ANDROID	53	# if !SANITIZER_ANDROID
54	#include <sys/statfs.h>	54	# include <sys/statfs.h>
55	#include <linux/perf_event.h>	55	# include <linux/perf_event.h>
56	#endif	56	# endif
57		57
58	using namespace __sanitizer;	58	using namespace __sanitizer;
59		59
60	namespace __sanitizer {	60	# if !defined(__powerpc64__) && !defined(__x86_64__) && \
61	#if !SANITIZER_ANDROID	61	!defined(__aarch64__) && !defined(__mips__) && !defined(__s390__) && \
62	unsigned struct_statfs64_sz = sizeof(struct statfs64);	62	!defined(__sparc__) && !defined(__riscv) && !defined(__hexagon__) && \
63	#endif	63	!defined(__loongarch__)
64	} // namespace __sanitizer
65
66	#if !defined(__powerpc64__) && !defined(__x86_64__) && !defined(__aarch64__)\
67	&& !defined(__mips__) && !defined(__s390__)\
68	&& !defined(__sparc__) && !defined(__riscv)
69	COMPILER_CHECK(struct___old_kernel_stat_sz == sizeof(struct __old_kernel_stat));	64	COMPILER_CHECK(struct___old_kernel_stat_sz == sizeof(struct __old_kernel_stat));
70	#endif	65	#endif
71		66

lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.cpp+2-3

lib/tsan/sanitizer_common/sanitizer_platform_limits_netbsd.h+2-3

lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.cpp+110-44

 @@ -18,12 +18,13 @@
 // depends on _FILE_OFFSET_BITS setting.
 // To get this "true" dirent definition, we undefine _FILE_OFFSET_BITS below.
 #undef _FILE_OFFSET_BITS
+#undef _TIME_BITS
 #endif
 // Must go after undef _FILE_OFFSET_BITS.
 #include "sanitizer_platform.h"
-#if SANITIZER_LINUX || SANITIZER_MAC
+#if SANITIZER_LINUX || SANITIZER_APPLE
 // Must go after undef _FILE_OFFSET_BITS.
 #include "sanitizer_glibc_version.h"
 @@ -51,7 +52,7 @@
 #include <time.h>
 #include <wchar.h>
 #include <regex.h>
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
 #include <utmp.h>
 #endif
 @@ -73,7 +74,9 @@
 #include <sys/vt.h>
 #include <linux/cdrom.h>
 #include <linux/fd.h>
+#if SANITIZER_ANDROID
 #include <linux/fs.h>
+#endif
 #include <linux/hdreg.h>
 #include <linux/input.h>
 #include <linux/ioctl.h>
 @@ -91,10 +94,10 @@
 #if SANITIZER_LINUX
 # include <utime.h>
 # include <sys/ptrace.h>
-#if defined(__mips64) || defined(__aarch64__) || defined(__arm__) || \
+#    if defined(__mips64) || defined(__aarch64__) || defined(__arm__) || \
-    SANITIZER_RISCV64
+        defined(__hexagon__) || defined(__loongarch__) ||SANITIZER_RISCV64
-#  include <asm/ptrace.h>
+#      include <asm/ptrace.h>
-#  ifdef __arm__
+#      ifdef __arm__
 typedef struct user_fpregs elf_fpregset_t;
 #   define ARM_VFPREGS_SIZE_ASAN (32 * 8 /*fpregs*/ + 4 /*fpscr*/)
 #   if !defined(ARM_VFPREGS_SIZE)
 @@ -152,7 +155,6 @@ typedef struct user_fpregs elf_fpregset_t;
 #include <linux/serial.h>
 #include <sys/msg.h>
 #include <sys/ipc.h>
-#include <crypt.h>
 #endif  // SANITIZER_ANDROID
 #include <link.h>
 @@ -163,22 +165,24 @@ typedef struct user_fpregs elf_fpregset_t;
 #include <fstab.h>
 #endif // SANITIZER_LINUX
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
 #include <net/ethernet.h>
 #include <sys/filio.h>
 #include <sys/sockio.h>
 #endif
 // Include these after system headers to avoid name clashes and ambiguities.
-#include "sanitizer_internal_defs.h"
+#  include "sanitizer_common.h"
-#include "sanitizer_platform_limits_posix.h"
+#  include "sanitizer_internal_defs.h"
+#  include "sanitizer_platform_interceptors.h"
+#  include "sanitizer_platform_limits_posix.h"
 namespace __sanitizer {
   unsigned struct_utsname_sz = sizeof(struct utsname);
   unsigned struct_stat_sz = sizeof(struct stat);
-#if !SANITIZER_IOS && !(SANITIZER_MAC && TARGET_CPU_ARM64)
+#if SANITIZER_HAS_STAT64
   unsigned struct_stat64_sz = sizeof(struct stat64);
-#endif // !SANITIZER_IOS && !(SANITIZER_MAC && TARGET_CPU_ARM64)
+#endif // SANITIZER_HAS_STAT64
   unsigned struct_rusage_sz = sizeof(struct rusage);
   unsigned struct_tm_sz = sizeof(struct tm);
   unsigned struct_passwd_sz = sizeof(struct passwd);
 @@ -203,26 +207,60 @@ namespace __sanitizer {
   unsigned struct_regex_sz = sizeof(regex_t);
   unsigned struct_regmatch_sz = sizeof(regmatch_t);
-#if (SANITIZER_MAC && !TARGET_CPU_ARM64) && !SANITIZER_IOS
+#if SANITIZER_HAS_STATFS64
   unsigned struct_statfs64_sz = sizeof(struct statfs64);
-#endif // (SANITIZER_MAC && !TARGET_CPU_ARM64) && !SANITIZER_IOS
+#endif // SANITIZER_HAS_STATFS64
-#if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC
+#if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_APPLE
   unsigned struct_fstab_sz = sizeof(struct fstab);
 #endif  // SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD ||
-        // SANITIZER_MAC
+        // SANITIZER_APPLE
 #if !SANITIZER_ANDROID
   unsigned struct_statfs_sz = sizeof(struct statfs);
   unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
-  unsigned ucontext_t_sz = sizeof(ucontext_t);
-#endif // !SANITIZER_ANDROID
-#if SANITIZER_LINUX
+  unsigned ucontext_t_sz(void *ctx) {
+#    if SANITIZER_GLIBC && SANITIZER_X64
+    // Added in Linux kernel 3.4.0, merged to glibc in 2.16
+#      ifndef FP_XSTATE_MAGIC1
+#        define FP_XSTATE_MAGIC1 0x46505853U
+#      endif
+    // See kernel arch/x86/kernel/fpu/signal.c for details.
+    const auto *fpregs = static_cast<ucontext_t *>(ctx)->uc_mcontext.fpregs;
+    // The member names differ across header versions, but the actual layout
+    // is always the same.  So avoid using members, just use arithmetic.
+    const uint32_t *after_xmm =
+        reinterpret_cast<const uint32_t *>(fpregs + 1) - 24;
+    if (after_xmm[12] == FP_XSTATE_MAGIC1)
+      return reinterpret_cast<const char *>(fpregs) + after_xmm[13] -
+             static_cast<const char *>(ctx);
+#    endif
+    return sizeof(ucontext_t);
+  }
+#  endif  // !SANITIZER_ANDROID
+#  if SANITIZER_LINUX
   unsigned struct_epoll_event_sz = sizeof(struct epoll_event);
   unsigned struct_sysinfo_sz = sizeof(struct sysinfo);
   unsigned __user_cap_header_struct_sz =
       sizeof(struct __user_cap_header_struct);
-  unsigned __user_cap_data_struct_sz = sizeof(struct __user_cap_data_struct);
+  unsigned __user_cap_data_struct_sz(void *hdrp) {
+    int u32s = 0;
+    if (hdrp) {
+      switch (((struct __user_cap_header_struct *)hdrp)->version) {
+      case _LINUX_CAPABILITY_VERSION_1:
+        u32s = _LINUX_CAPABILITY_U32S_1;
+        break;
+      case _LINUX_CAPABILITY_VERSION_2:
+        u32s = _LINUX_CAPABILITY_U32S_2;
+        break;
+      case _LINUX_CAPABILITY_VERSION_3:
+        u32s = _LINUX_CAPABILITY_U32S_3;
+        break;
+      }
+    }
+    return sizeof(struct __user_cap_data_struct) * u32s;
+  }
   unsigned struct_new_utsname_sz = sizeof(struct new_utsname);
   unsigned struct_old_utsname_sz = sizeof(struct old_utsname);
   unsigned struct_oldold_utsname_sz = sizeof(struct oldold_utsname);
 @@ -235,24 +273,28 @@ namespace __sanitizer {
   unsigned struct_itimerspec_sz = sizeof(struct itimerspec);
 #endif // SANITIZER_LINUX
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
   // Use pre-computed size of struct ustat to avoid <sys/ustat.h> which
   // has been removed from glibc 2.28.
 #if defined(__aarch64__) || defined(__s390x__) || defined(__mips64) ||     \
     defined(__powerpc64__) || defined(__arch64__) || defined(__sparcv9) || \
     defined(__x86_64__) || SANITIZER_RISCV64
 #define SIZEOF_STRUCT_USTAT 32
-#elif defined(__arm__) || defined(__i386__) || defined(__mips__) \
+#    elif defined(__arm__) || defined(__i386__) || defined(__mips__) ||    \
-  || defined(__powerpc__) || defined(__s390__) || defined(__sparc__)
+        defined(__powerpc__) || defined(__s390__) || defined(__sparc__) || \
-#define SIZEOF_STRUCT_USTAT 20
+        defined(__hexagon__)
-#else
+#      define SIZEOF_STRUCT_USTAT 20
-#error Unknown size of struct ustat
+#    elif defined(__loongarch__)
-#endif
+  // Not used. The minimum Glibc version available for LoongArch is 2.36
+  // so ustat() wrapper is already gone.
+#      define SIZEOF_STRUCT_USTAT 0
+#    else
+#      error Unknown size of struct ustat
+#    endif
   unsigned struct_ustat_sz = SIZEOF_STRUCT_USTAT;
   unsigned struct_rlimit64_sz = sizeof(struct rlimit64);
   unsigned struct_statvfs64_sz = sizeof(struct statvfs64);
-  unsigned struct_crypt_data_sz = sizeof(struct crypt_data);
+#endif // SANITIZER_GLIBC
-#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
 #if SANITIZER_LINUX && !SANITIZER_ANDROID
   unsigned struct_timex_sz = sizeof(struct timex);
 @@ -280,7 +322,7 @@ namespace __sanitizer {
   int shmctl_shm_stat = (int)SHM_STAT;
 #endif
-#if !SANITIZER_MAC && !SANITIZER_FREEBSD
+#if !SANITIZER_APPLE && !SANITIZER_FREEBSD
   unsigned struct_utmp_sz = sizeof(struct utmp);
 #endif
 #if !SANITIZER_ANDROID
 @@ -312,10 +354,14 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
   int glob_altdirfunc = GLOB_ALTDIRFUNC;
 #endif
+#  if !SANITIZER_ANDROID
+  const int wordexp_wrde_dooffs = WRDE_DOOFFS;
+#  endif  // !SANITIZER_ANDROID
 #if SANITIZER_LINUX && !SANITIZER_ANDROID &&                               \
     (defined(__i386) || defined(__x86_64) || defined(__mips64) ||          \
      defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
-     defined(__s390__) || SANITIZER_RISCV64)
+     defined(__s390__) || defined(__loongarch__)|| SANITIZER_RISCV64)
 #if defined(__mips64) || defined(__powerpc64__) || defined(__arm__)
   unsigned struct_user_regs_struct_sz = sizeof(struct pt_regs);
   unsigned struct_user_fpregs_struct_sz = sizeof(elf_fpregset_t);
 @@ -325,21 +371,24 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
 #elif defined(__aarch64__)
   unsigned struct_user_regs_struct_sz = sizeof(struct user_pt_regs);
   unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpsimd_state);
+#elif defined(__loongarch__)
+  unsigned struct_user_regs_struct_sz = sizeof(struct user_pt_regs);
+  unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fp_state);
 #elif defined(__s390__)
   unsigned struct_user_regs_struct_sz = sizeof(struct _user_regs_struct);
   unsigned struct_user_fpregs_struct_sz = sizeof(struct _user_fpregs_struct);
 #else
   unsigned struct_user_regs_struct_sz = sizeof(struct user_regs_struct);
   unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpregs_struct);
-#endif // __mips64 || __powerpc64__ || __aarch64__
+#endif // __mips64 || __powerpc64__ || __aarch64__ || __loongarch__
 #if defined(__x86_64) || defined(__mips64) || defined(__powerpc64__) || \
     defined(__aarch64__) || defined(__arm__) || defined(__s390__) ||    \
-    SANITIZER_RISCV64
+    defined(__loongarch__) || SANITIZER_RISCV64
   unsigned struct_user_fpxregs_struct_sz = 0;
 #else
   unsigned struct_user_fpxregs_struct_sz = sizeof(struct user_fpxregs_struct);
 #endif // __x86_64 || __mips64 || __powerpc64__ || __aarch64__ || __arm__
-// || __s390__
+// || __s390__ || __loongarch__
 #ifdef __arm__
   unsigned struct_user_vfpregs_struct_sz = ARM_VFPREGS_SIZE;
 #else
 @@ -484,7 +533,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
   unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats);
 #endif  // SANITIZER_GLIBC
-#if !SANITIZER_ANDROID && !SANITIZER_MAC
+#if !SANITIZER_ANDROID && !SANITIZER_APPLE
   unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req);
   unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req);
 #endif
 @@ -570,6 +619,14 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
   unsigned IOCTL_BLKROGET = BLKROGET;
   unsigned IOCTL_BLKROSET = BLKROSET;
   unsigned IOCTL_BLKRRPART = BLKRRPART;
+  unsigned IOCTL_BLKFRASET = BLKFRASET;
+  unsigned IOCTL_BLKFRAGET = BLKFRAGET;
+  unsigned IOCTL_BLKSECTSET = BLKSECTSET;
+  unsigned IOCTL_BLKSECTGET = BLKSECTGET;
+  unsigned IOCTL_BLKSSZGET = BLKSSZGET;
+  unsigned IOCTL_BLKBSZGET = BLKBSZGET;
+  unsigned IOCTL_BLKBSZSET = BLKBSZSET;
+  unsigned IOCTL_BLKGETSIZE64 = BLKGETSIZE64;
   unsigned IOCTL_CDROMAUDIOBUFSIZ = CDROMAUDIOBUFSIZ;
   unsigned IOCTL_CDROMEJECT = CDROMEJECT;
   unsigned IOCTL_CDROMEJECT_SW = CDROMEJECT_SW;
 @@ -837,10 +894,10 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
   unsigned IOCTL_EVIOCGPROP = IOCTL_NOT_PRESENT;
   unsigned IOCTL_EVIOCSKEYCODE_V2 = IOCTL_NOT_PRESENT;
 #endif
-  unsigned IOCTL_FS_IOC_GETFLAGS = FS_IOC_GETFLAGS;
+  unsigned IOCTL_FS_IOC_GETFLAGS = _IOR('f', 1, long);
-  unsigned IOCTL_FS_IOC_GETVERSION = FS_IOC_GETVERSION;
+  unsigned IOCTL_FS_IOC_GETVERSION = _IOR('v', 1, long);
-  unsigned IOCTL_FS_IOC_SETFLAGS = FS_IOC_SETFLAGS;
+  unsigned IOCTL_FS_IOC_SETFLAGS = _IOW('f', 2, long);
-  unsigned IOCTL_FS_IOC_SETVERSION = FS_IOC_SETVERSION;
+  unsigned IOCTL_FS_IOC_SETVERSION = _IOW('v', 2, long);
   unsigned IOCTL_GIO_CMAP = GIO_CMAP;
   unsigned IOCTL_GIO_FONT = GIO_FONT;
   unsigned IOCTL_GIO_UNIMAP = GIO_UNIMAP;
 @@ -1035,7 +1092,7 @@ CHECK_SIZE_AND_OFFSET(mmsghdr, msg_len);
 COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent));
 CHECK_SIZE_AND_OFFSET(dirent, d_ino);
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
 CHECK_SIZE_AND_OFFSET(dirent, d_seekoff);
 #elif SANITIZER_FREEBSD
 // There is no 'd_off' field on FreeBSD.
 @@ -1044,7 +1101,7 @@ CHECK_SIZE_AND_OFFSET(dirent, d_off);
 #endif
 CHECK_SIZE_AND_OFFSET(dirent, d_reclen);
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#if SANITIZER_GLIBC
 COMPILER_CHECK(sizeof(__sanitizer_dirent64) <= sizeof(dirent64));
 CHECK_SIZE_AND_OFFSET(dirent64, d_ino);
 CHECK_SIZE_AND_OFFSET(dirent64, d_off);
 @@ -1077,6 +1134,15 @@ CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_flags);
 CHECK_STRUCT_SIZE_AND_OFFSET(sigaction, sa_restorer);
 #endif
+#if SANITIZER_HAS_SIGINFO
+COMPILER_CHECK(alignof(siginfo_t) == alignof(__sanitizer_siginfo));
+using __sanitizer_siginfo_t = __sanitizer_siginfo;
+CHECK_TYPE_SIZE(siginfo_t);
+CHECK_SIZE_AND_OFFSET(siginfo_t, si_signo);
+CHECK_SIZE_AND_OFFSET(siginfo_t, si_errno);
+CHECK_SIZE_AND_OFFSET(siginfo_t, si_code);
+#endif
 #if SANITIZER_LINUX
 CHECK_TYPE_SIZE(__sysctl_args);
 CHECK_SIZE_AND_OFFSET(__sysctl_args, name);
 @@ -1217,7 +1283,7 @@ CHECK_SIZE_AND_OFFSET(passwd, pw_shell);
 CHECK_SIZE_AND_OFFSET(passwd, pw_gecos);
 #endif
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
 CHECK_SIZE_AND_OFFSET(passwd, pw_change);
 CHECK_SIZE_AND_OFFSET(passwd, pw_expire);
 CHECK_SIZE_AND_OFFSET(passwd, pw_class);
 @@ -1230,7 +1296,7 @@ CHECK_SIZE_AND_OFFSET(group, gr_passwd);
 CHECK_SIZE_AND_OFFSET(group, gr_gid);
 CHECK_SIZE_AND_OFFSET(group, gr_mem);
-#if HAVE_RPC_XDR_H
+#if HAVE_RPC_XDR_H && !SANITIZER_APPLE
 CHECK_TYPE_SIZE(XDR);
 CHECK_SIZE_AND_OFFSET(XDR, x_op);
 CHECK_SIZE_AND_OFFSET(XDR, x_ops);
 @@ -1285,4 +1351,4 @@ CHECK_TYPE_SIZE(sem_t);
 COMPILER_CHECK(ARM_VFPREGS_SIZE == ARM_VFPREGS_SIZE_ASAN);
 #endif
-#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_MAC
+#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_APPLE

lib/tsan/sanitizer_common/sanitizer_platform_limits_posix.h+105-46

 @@ -14,10 +14,25 @@
 #ifndef SANITIZER_PLATFORM_LIMITS_POSIX_H
 #define SANITIZER_PLATFORM_LIMITS_POSIX_H
-#if SANITIZER_LINUX || SANITIZER_MAC
+#if SANITIZER_LINUX || SANITIZER_APPLE
 #include "sanitizer_internal_defs.h"
 #include "sanitizer_platform.h"
+#include "sanitizer_mallinfo.h"
+#if SANITIZER_APPLE
+#include <sys/cdefs.h>
+#if !__DARWIN_ONLY_64_BIT_INO_T
+#define SANITIZER_HAS_STAT64 1
+#define SANITIZER_HAS_STATFS64 1
+#else
+#define SANITIZER_HAS_STAT64 0
+#define SANITIZER_HAS_STATFS64 0
+#endif
+#elif SANITIZER_GLIBC || SANITIZER_ANDROID
+#define SANITIZER_HAS_STAT64 1
+#define SANITIZER_HAS_STATFS64 1
+#endif
 #if defined(__sparc__)
 // FIXME: This can't be included from tsan which does not support sparc yet.
 @@ -29,7 +44,7 @@
 namespace __sanitizer {
 extern unsigned struct_utsname_sz;
 extern unsigned struct_stat_sz;
-#if !SANITIZER_IOS
+#if SANITIZER_HAS_STAT64
 extern unsigned struct_stat64_sz;
 #endif
 extern unsigned struct_rusage_sz;
 @@ -49,7 +64,9 @@ extern unsigned struct_itimerspec_sz;
 extern unsigned struct_sigevent_sz;
 extern unsigned struct_stack_t_sz;
 extern unsigned struct_sched_param_sz;
+#if SANITIZER_HAS_STATFS64
 extern unsigned struct_statfs64_sz;
+#endif
 extern unsigned struct_regex_sz;
 extern unsigned struct_regmatch_sz;
 @@ -57,12 +74,12 @@ extern unsigned struct_regmatch_sz;
 extern unsigned struct_fstab_sz;
 extern unsigned struct_statfs_sz;
 extern unsigned struct_sockaddr_sz;
-extern unsigned ucontext_t_sz;
+unsigned ucontext_t_sz(void *uctx);
-#endif // !SANITIZER_ANDROID
+#  endif  // !SANITIZER_ANDROID
-#if SANITIZER_LINUX
+#  if SANITIZER_LINUX
-#if defined(__x86_64__)
+#    if defined(__x86_64__)
 const unsigned struct_kernel_stat_sz = 144;
 const unsigned struct_kernel_stat64_sz = 0;
 #elif defined(__i386__)
 @@ -81,9 +98,10 @@ const unsigned struct_kernel_stat64_sz = 104;
 const unsigned struct_kernel_stat_sz = 144;
 const unsigned struct_kernel_stat64_sz = 104;
 #elif defined(__mips__)
-const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID
+const unsigned struct_kernel_stat_sz =
-                                           ? FIRST_32_SECOND_64(104, 128)
+    SANITIZER_ANDROID
-                                           : FIRST_32_SECOND_64(160, 216);
+        ? FIRST_32_SECOND_64(104, 128)
+        : FIRST_32_SECOND_64((_MIPS_SIM == _ABIN32) ? 176 : 160, 216);
 const unsigned struct_kernel_stat64_sz = 104;
 #elif defined(__s390__) && !defined(__s390x__)
 const unsigned struct_kernel_stat_sz = 64;
 @@ -102,7 +120,13 @@ const unsigned struct_kernel_stat64_sz = 104;
 #elif SANITIZER_RISCV64
 const unsigned struct_kernel_stat_sz = 128;
 const unsigned struct_kernel_stat64_sz = 0;  // RISCV64 does not use stat64
-#endif
+#    elif defined(__hexagon__)
+const unsigned struct_kernel_stat_sz = 128;
+const unsigned struct_kernel_stat64_sz = 0;
+#    elif defined(__loongarch__)
+const unsigned struct_kernel_stat_sz = 128;
+const unsigned struct_kernel_stat64_sz = 0;
+#    endif
 struct __sanitizer_perf_event_attr {
   unsigned type;
   unsigned size;
 @@ -112,7 +136,7 @@ struct __sanitizer_perf_event_attr {
 extern unsigned struct_epoll_event_sz;
 extern unsigned struct_sysinfo_sz;
 extern unsigned __user_cap_header_struct_sz;
-extern unsigned __user_cap_data_struct_sz;
+extern unsigned __user_cap_data_struct_sz(void *hdrp);
 extern unsigned struct_new_utsname_sz;
 extern unsigned struct_old_utsname_sz;
 extern unsigned struct_oldold_utsname_sz;
 @@ -122,7 +146,7 @@ const unsigned struct_kexec_segment_sz = 4 * sizeof(unsigned long);
 #if SANITIZER_LINUX
-#if defined(__powerpc64__) || defined(__s390__)
+#if defined(__powerpc64__) || defined(__s390__) || defined(__loongarch__)
 const unsigned struct___old_kernel_stat_sz = 0;
 #elif !defined(__sparc__)
 const unsigned struct___old_kernel_stat_sz = 32;
 @@ -181,17 +205,7 @@ struct __sanitizer_sem_t {
 };
 #endif // SANITIZER_LINUX
-#if SANITIZER_ANDROID
-struct __sanitizer_struct_mallinfo {
-  uptr v[10];
-};
-#endif
 #if SANITIZER_LINUX && !SANITIZER_ANDROID
-struct __sanitizer_struct_mallinfo {
-  int v[10];
-};
 extern unsigned struct_ustat_sz;
 extern unsigned struct_rlimit64_sz;
 extern unsigned struct_statvfs64_sz;
 @@ -295,7 +309,6 @@ extern unsigned struct_msqid_ds_sz;
 extern unsigned struct_mq_attr_sz;
 extern unsigned struct_timex_sz;
 extern unsigned struct_statvfs_sz;
-extern unsigned struct_crypt_data_sz;
 #endif  // SANITIZER_LINUX && !SANITIZER_ANDROID
 struct __sanitizer_iovec {
 @@ -319,7 +332,7 @@ struct __sanitizer_ifaddrs {
 };
 #endif  // !SANITIZER_ANDROID
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
 typedef unsigned long __sanitizer_pthread_key_t;
 #else
 typedef unsigned __sanitizer_pthread_key_t;
 @@ -346,7 +359,7 @@ struct __sanitizer_passwd {
   char *pw_passwd;
   int pw_uid;
   int pw_gid;
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
   long pw_change;
   char *pw_class;
 #endif
 @@ -355,7 +368,7 @@ struct __sanitizer_passwd {
 #endif
   char *pw_dir;
   char *pw_shell;
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
   long pw_expire;
 #endif
 };
 @@ -367,7 +380,8 @@ struct __sanitizer_group {
   char **gr_mem;
 };
-#if defined(__x86_64__) && !defined(_LP64)
+#  if (SANITIZER_LINUX && !SANITIZER_GLIBC && !SANITIZER_ANDROID) || \
+      (defined(__x86_64__) && !defined(_LP64)) || defined(__hexagon__)
 typedef long long __sanitizer_time_t;
 #else
 typedef long __sanitizer_time_t;
 @@ -427,7 +441,7 @@ struct __sanitizer_file_handle {
 };
 #endif
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
 struct __sanitizer_msghdr {
   void *msg_name;
   unsigned msg_namelen;
 @@ -468,30 +482,31 @@ struct __sanitizer_mmsghdr {
 };
 #endif
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
 struct __sanitizer_dirent {
   unsigned long long d_ino;
   unsigned long long d_seekoff;
   unsigned short d_reclen;
   // more fields that we don't care about
 };
-#elif SANITIZER_ANDROID || defined(__x86_64__)
+#  elif (SANITIZER_LINUX && !SANITIZER_GLIBC) || defined(__x86_64__) || \
+      defined(__hexagon__)
 struct __sanitizer_dirent {
   unsigned long long d_ino;
   unsigned long long d_off;
   unsigned short d_reclen;
   // more fields that we don't care about
 };
-#else
+#  else
 struct __sanitizer_dirent {
   uptr d_ino;
   uptr d_off;
   unsigned short d_reclen;
   // more fields that we don't care about
 };
-#endif
+#  endif
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#  if SANITIZER_GLIBC
 struct __sanitizer_dirent64 {
   unsigned long long d_ino;
   unsigned long long d_off;
 @@ -511,8 +526,8 @@ typedef int __sanitizer_clockid_t;
 #endif
 #if SANITIZER_LINUX
-#if defined(_LP64) || defined(__x86_64__) || defined(__powerpc__) || \
+#    if defined(_LP64) || defined(__x86_64__) || defined(__powerpc__) || \
-    defined(__mips__)
+        defined(__mips__) || defined(__hexagon__)
 typedef unsigned __sanitizer___kernel_uid_t;
 typedef unsigned __sanitizer___kernel_gid_t;
 #else
 @@ -552,7 +567,7 @@ typedef unsigned long __sanitizer_sigset_t[16 / sizeof(unsigned long)];
 # else
 typedef unsigned long __sanitizer_sigset_t;
 # endif
-#elif SANITIZER_MAC
+#elif SANITIZER_APPLE
 typedef unsigned __sanitizer_sigset_t;
 #elif SANITIZER_LINUX
 struct __sanitizer_sigset_t {
 @@ -561,10 +576,35 @@ struct __sanitizer_sigset_t {
 };
 #endif
-struct __sanitizer_siginfo {
+struct __sanitizer_siginfo_pad {
-  // The size is determined by looking at sizeof of real siginfo_t on linux.
+#if SANITIZER_X32
-  u64 opaque[128 / sizeof(u64)];
+  // x32 siginfo_t is aligned to 8 bytes.
+  u64 pad[128 / sizeof(u64)];
+#else
+  // Require uptr, because siginfo_t is always pointer-size aligned on Linux.
+  uptr pad[128 / sizeof(uptr)];
+#endif
+};
+#if SANITIZER_LINUX
+# define SANITIZER_HAS_SIGINFO 1
+union __sanitizer_siginfo {
+  struct {
+    int si_signo;
+# if SANITIZER_MIPS
+    int si_code;
+    int si_errno;
+# else
+    int si_errno;
+    int si_code;
+# endif
+  };
+  __sanitizer_siginfo_pad pad;
 };
+#else
+# define SANITIZER_HAS_SIGINFO 0
+typedef __sanitizer_siginfo_pad __sanitizer_siginfo;
+#endif
 using __sanitizer_sighandler_ptr = void (*)(int sig);
 using __sanitizer_sigactionhandler_ptr = void (*)(int sig,
 @@ -712,12 +752,19 @@ struct __sanitizer_protoent {
   int p_proto;
 };
+struct __sanitizer_netent {
+  char *n_name;
+  char **n_aliases;
+  int n_addrtype;
+  u32 n_net;
+};
 struct __sanitizer_addrinfo {
   int ai_flags;
   int ai_family;
   int ai_socktype;
   int ai_protocol;
-#if SANITIZER_ANDROID || SANITIZER_MAC
+#if SANITIZER_ANDROID || SANITIZER_APPLE
   unsigned ai_addrlen;
   char *ai_canonname;
   void *ai_addr;
 @@ -743,7 +790,7 @@ struct __sanitizer_pollfd {
   short revents;
 };
-#if SANITIZER_ANDROID || SANITIZER_MAC
+#if SANITIZER_ANDROID || SANITIZER_APPLE
 typedef unsigned __sanitizer_nfds_t;
 #else
 typedef unsigned long __sanitizer_nfds_t;
 @@ -773,6 +820,10 @@ extern int glob_altdirfunc;
 extern unsigned path_max;
+#  if !SANITIZER_ANDROID
+extern const int wordexp_wrde_dooffs;
+#  endif  // !SANITIZER_ANDROID
 struct __sanitizer_wordexp_t {
   uptr we_wordc;
   char **we_wordv;
 @@ -806,7 +857,7 @@ typedef void __sanitizer_FILE;
 #if SANITIZER_LINUX && !SANITIZER_ANDROID &&                               \
     (defined(__i386) || defined(__x86_64) || defined(__mips64) ||          \
      defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \
-     defined(__s390__) || SANITIZER_RISCV64)
+     defined(__s390__) || defined(__loongarch__) || SANITIZER_RISCV64)
 extern unsigned struct_user_regs_struct_sz;
 extern unsigned struct_user_fpregs_struct_sz;
 extern unsigned struct_user_fpxregs_struct_sz;
 @@ -839,7 +890,7 @@ extern int shmctl_shm_info;
 extern int shmctl_shm_stat;
 #endif
-#if !SANITIZER_MAC && !SANITIZER_FREEBSD
+#if !SANITIZER_APPLE && !SANITIZER_FREEBSD
 extern unsigned struct_utmp_sz;
 #endif
 #if !SANITIZER_ANDROID
 @@ -854,7 +905,7 @@ struct __sanitizer_ifconf {
   union {
     void *ifcu_req;
   } ifc_ifcu;
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
 } __attribute__((packed));
 #else
 };
 @@ -1007,7 +1058,7 @@ extern unsigned struct_audio_buf_info_sz;
 extern unsigned struct_ppp_stats_sz;
 #endif  // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID
-#if !SANITIZER_ANDROID && !SANITIZER_MAC
+#if !SANITIZER_ANDROID && !SANITIZER_APPLE
 extern unsigned struct_sioc_sg_req_sz;
 extern unsigned struct_sioc_vif_req_sz;
 #endif
 @@ -1094,6 +1145,14 @@ extern unsigned IOCTL_BLKRASET;
 extern unsigned IOCTL_BLKROGET;
 extern unsigned IOCTL_BLKROSET;
 extern unsigned IOCTL_BLKRRPART;
+extern unsigned IOCTL_BLKFRASET;
+extern unsigned IOCTL_BLKFRAGET;
+extern unsigned IOCTL_BLKSECTSET;
+extern unsigned IOCTL_BLKSECTGET;
+extern unsigned IOCTL_BLKSSZGET;
+extern unsigned IOCTL_BLKBSZGET;
+extern unsigned IOCTL_BLKBSZSET;
+extern unsigned IOCTL_BLKGETSIZE64;
 extern unsigned IOCTL_CDROMAUDIOBUFSIZ;
 extern unsigned IOCTL_CDROMEJECT;
 extern unsigned IOCTL_CDROMEJECT_SW;
 @@ -1440,6 +1499,6 @@ extern const int si_SEGV_ACCERR;
 #define SIGACTION_SYMNAME sigaction
-#endif  // SANITIZER_LINUX || SANITIZER_MAC
+#endif  // SANITIZER_LINUX || SANITIZER_APPLE
 #endif

lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.cpp+2-1

...	@@ -89,7 +89,7 @@ namespace __sanitizer {	...	@@ -89,7 +89,7 @@ namespace __sanitizer {
89	unsigned struct_sched_param_sz = sizeof(struct sched_param);	89	unsigned struct_sched_param_sz = sizeof(struct sched_param);
90	unsigned struct_statfs_sz = sizeof(struct statfs);	90	unsigned struct_statfs_sz = sizeof(struct statfs);
91	unsigned struct_sockaddr_sz = sizeof(struct sockaddr);	91	unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
92	unsigned ucontext_t_sz = sizeof(ucontext_t);	92	unsigned ucontext_t_sz(void *ctx) { return sizeof(ucontext_t); }
93	unsigned struct_timespec_sz = sizeof(struct timespec);	93	unsigned struct_timespec_sz = sizeof(struct timespec);
94	#if SANITIZER_SOLARIS32	94	#if SANITIZER_SOLARIS32
95	unsigned struct_statvfs64_sz = sizeof(struct statvfs64);	95	unsigned struct_statvfs64_sz = sizeof(struct statvfs64);
...	@@ -123,6 +123,7 @@ namespace __sanitizer {	...	@@ -123,6 +123,7 @@ namespace __sanitizer {
123	unsigned struct_ElfW_Phdr_sz = sizeof(ElfW(Phdr));	123	unsigned struct_ElfW_Phdr_sz = sizeof(ElfW(Phdr));
124		124
125	int glob_nomatch = GLOB_NOMATCH;	125	int glob_nomatch = GLOB_NOMATCH;
		126	const int wordexp_wrde_dooffs = WRDE_DOOFFS;
126		127
127	unsigned path_max = PATH_MAX;	128	unsigned path_max = PATH_MAX;
128		129

lib/tsan/sanitizer_common/sanitizer_platform_limits_solaris.h+2-1

...	@@ -43,7 +43,7 @@ extern unsigned struct_sched_param_sz;	...	@@ -43,7 +43,7 @@ extern unsigned struct_sched_param_sz;
43	extern unsigned struct_statfs64_sz;	43	extern unsigned struct_statfs64_sz;
44	extern unsigned struct_statfs_sz;	44	extern unsigned struct_statfs_sz;
45	extern unsigned struct_sockaddr_sz;	45	extern unsigned struct_sockaddr_sz;
46	extern unsigned ucontext_t_sz;	46	unsigned ucontext_t_sz(void *ctx);
47		47
48	extern unsigned struct_timespec_sz;	48	extern unsigned struct_timespec_sz;
49	extern unsigned struct_rlimit_sz;	49	extern unsigned struct_rlimit_sz;
...	@@ -341,6 +341,7 @@ struct __sanitizer_glob_t {	...	@@ -341,6 +341,7 @@ struct __sanitizer_glob_t {
341		341
342	extern int glob_nomatch;	342	extern int glob_nomatch;
343	extern int glob_altdirfunc;	343	extern int glob_altdirfunc;
		344	extern const int wordexp_wrde_dooffs;
344		345
345	extern unsigned path_max;	346	extern unsigned path_max;
346		347

lib/tsan/sanitizer_common/sanitizer_posix.cpp+21-9

 @@ -41,6 +41,8 @@ uptr GetMmapGranularity() {
   return GetPageSize();
 }
+bool ErrorIsOOM(error_t err) { return err == ENOMEM; }
 void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
   size = RoundUpTo(size, GetPageSizeCached());
   uptr res = MmapNamed(nullptr, size, PROT_READ | PROT_WRITE,
 @@ -55,11 +57,9 @@ void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
 void UnmapOrDie(void *addr, uptr size) {
   if (!addr || !size) return;
   uptr res = internal_munmap(addr, size);
-  if (UNLIKELY(internal_iserror(res))) {
+  int reserrno;
-    Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n",
+  if (UNLIKELY(internal_iserror(res, &reserrno)))
-           SanitizerToolName, size, size, addr);
+    ReportMunmapFailureAndDie(addr, size, reserrno);
-    CHECK("unable to unmap" && 0);
-  }
   DecreaseTotalMmap(size);
 }
 @@ -85,18 +85,26 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
   CHECK(IsPowerOfTwo(size));
   CHECK(IsPowerOfTwo(alignment));
   uptr map_size = size + alignment;
+  // mmap maps entire pages and rounds up map_size needs to be a an integral
+  // number of pages.
+  // We need to be aware of this size for calculating end and for unmapping
+  // fragments before and after the alignment region.
+  map_size = RoundUpTo(map_size, GetPageSizeCached());
   uptr map_res = (uptr)MmapOrDieOnFatalError(map_size, mem_type);
   if (UNLIKELY(!map_res))
     return nullptr;
-  uptr map_end = map_res + map_size;
   uptr res = map_res;
   if (!IsAligned(res, alignment)) {
     res = (map_res + alignment - 1) & ~(alignment - 1);
     UnmapOrDie((void*)map_res, res - map_res);
   }
+  uptr map_end = map_res + map_size;
   uptr end = res + size;
-  if (end != map_end)
+  end = RoundUpTo(end, GetPageSizeCached());
+  if (end != map_end) {
+    CHECK_LT(end, map_end);
     UnmapOrDie((void*)end, map_end - end);
+  }
   return (void*)res;
 }
 @@ -146,7 +154,11 @@ bool MprotectReadOnly(uptr addr, uptr size) {
   return 0 == internal_mprotect((void *)addr, size, PROT_READ);
 }
-#if !SANITIZER_MAC
+bool MprotectReadWrite(uptr addr, uptr size) {
+  return 0 == internal_mprotect((void *)addr, size, PROT_READ | PROT_WRITE);
+}
+#if !SANITIZER_APPLE
 void MprotectMallocZones(void *addr, int prot) {}
 #endif
 @@ -239,7 +251,7 @@ bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
   return true;
 }
-#if !SANITIZER_MAC
+#if !SANITIZER_APPLE
 void DumpProcessMap() {
   MemoryMappingLayout proc_maps(/*cache_enabled*/true);
   const sptr kBufSize = 4095;

lib/tsan/sanitizer_common/sanitizer_posix.h+7-5

 @@ -20,10 +20,7 @@
 #include "sanitizer_platform_limits_posix.h"
 #include "sanitizer_platform_limits_solaris.h"
-#if !SANITIZER_POSIX
+#if SANITIZER_POSIX
-// Make it hard to accidentally use any of functions declared in this file:
-#error This file should only be included on POSIX
-#endif
 namespace __sanitizer {
 @@ -93,7 +90,7 @@ int real_pthread_join(void *th, void **ret);
   }                                                                            \
   }  // namespace __sanitizer
-int my_pthread_attr_getstack(void *attr, void **addr, uptr *size);
+int internal_pthread_attr_getstack(void *attr, void **addr, uptr *size);
 // A routine named real_sigaction() must be implemented by each sanitizer in
 // order for internal_sigaction() to bypass interceptors.
 @@ -123,7 +120,12 @@ int GetNamedMappingFd(const char *name, uptr size, int *flags);
 // alive at least as long as the mapping exists.
 void DecorateMapping(uptr addr, uptr size, const char *name);
+#  if !SANITIZER_FREEBSD
+#    define __sanitizer_dirsiz(dp) ((dp)->d_reclen)
+#  endif
 }  // namespace __sanitizer
+#endif  // SANITIZER_POSIX
 #endif  // SANITIZER_POSIX_H

lib/tsan/sanitizer_common/sanitizer_posix_libcdep.cpp+6-4

...	@@ -151,6 +151,8 @@ int Atexit(void (*function)(void)) {	...	@@ -151,6 +151,8 @@ int Atexit(void (*function)(void)) {
151	#endif	151	#endif
152	}	152	}
153		153
		154	bool CreateDir(const char *pathname) { return mkdir(pathname, 0755) == 0; }
		155
154	bool SupportsColoredOutput(fd_t fd) {	156	bool SupportsColoredOutput(fd_t fd) {
155	return isatty(fd) != 0;	157	return isatty(fd) != 0;
156	}	158	}
...	@@ -288,7 +290,7 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) {	...	@@ -288,7 +290,7 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) {
288	return result;	290	return result;
289	}	291	}
290		292
291	void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {	293	void PlatformPrepareForSandboxing(void *args) {
292	// Some kinds of sandboxes may forbid filesystem access, so we won't be able	294	// Some kinds of sandboxes may forbid filesystem access, so we won't be able
293	// to read the file mappings from /proc/self/maps. Luckily, neither the	295	// to read the file mappings from /proc/self/maps. Luckily, neither the
294	// process will be able to load additional libraries, so it's fine to use the	296	// process will be able to load additional libraries, so it's fine to use the
...	@@ -381,8 +383,8 @@ SANITIZER_WEAK_ATTRIBUTE int	...	@@ -381,8 +383,8 @@ SANITIZER_WEAK_ATTRIBUTE int
381	real_pthread_attr_getstack(void attr, void addr, size_t size);	383	real_pthread_attr_getstack(void attr, void addr, size_t size);
382	} // extern "C"	384	} // extern "C"
383		385
384	int my_pthread_attr_getstack(void attr, void addr, uptr size) {	386	int internal_pthread_attr_getstack(void attr, void addr, uptr size) {
385	#if !SANITIZER_GO && !SANITIZER_MAC	387	#if !SANITIZER_GO && !SANITIZER_APPLE
386	if (&real_pthread_attr_getstack)	388	if (&real_pthread_attr_getstack)
387	return real_pthread_attr_getstack((pthread_attr_t *)attr, addr,	389	return real_pthread_attr_getstack((pthread_attr_t *)attr, addr,
388	(size_t *)size);	390	(size_t *)size);
...	@@ -395,7 +397,7 @@ void AdjustStackSize(void *attr_) {	...	@@ -395,7 +397,7 @@ void AdjustStackSize(void *attr_) {
395	pthread_attr_t attr = (pthread_attr_t )attr_;	397	pthread_attr_t attr = (pthread_attr_t )attr_;
396	uptr stackaddr = 0;	398	uptr stackaddr = 0;
397	uptr stacksize = 0;	399	uptr stacksize = 0;
398	my_pthread_attr_getstack(attr, (void**)&stackaddr, &stacksize);	400	internal_pthread_attr_getstack(attr, (void **)&stackaddr, &stacksize);
399	// GLibC will return (0 - stacksize) as the stack address in the case when	401	// GLibC will return (0 - stacksize) as the stack address in the case when
400	// stacksize is set, but stackaddr is not.	402	// stacksize is set, but stackaddr is not.
401	bool stack_set = (stackaddr != 0) && (stackaddr + stacksize != 0);	403	bool stack_set = (stackaddr != 0) && (stackaddr + stacksize != 0);

lib/tsan/sanitizer_common/sanitizer_printf.cpp+13-24

 @@ -20,10 +20,6 @@
 #include <stdio.h>
 #include <stdarg.h>
-#if defined(__x86_64__)
-#  include <emmintrin.h>
-#endif
 #if SANITIZER_WINDOWS && defined(_MSC_VER) && _MSC_VER < 1800 &&               \
       !defined(va_copy)
 # define va_copy(dst, src) ((dst) = (src))
 @@ -132,8 +128,8 @@ static int AppendPointer(char **buff, const char *buff_end, u64 ptr_value) {
 int VSNPrintf(char *buff, int buff_length,
               const char *format, va_list args) {
   static const char *kPrintfFormatsHelp =
-      "Supported Printf formats: %([0-9]*)?(z|ll)?{d,u,x,X,V}; %p; "
+      "Supported Printf formats: %([0-9]*)?(z|l|ll)?{d,u,x,X}; %p; "
-      "%[-]([0-9]*)?(\\.\\*)?s; %c\n";
+      "%[-]([0-9]*)?(\\.\\*)?s; %c\nProvided format: ";
   RAW_CHECK(format);
   RAW_CHECK(buff_length > 0);
   const char *buff_end = &buff[buff_length - 1];
 @@ -164,9 +160,11 @@ int VSNPrintf(char *buff, int buff_length,
     }
     bool have_z = (*cur == 'z');
     cur += have_z;
-    bool have_ll = !have_z && (cur[0] == 'l' && cur[1] == 'l');
+    bool have_l = cur[0] == 'l' && cur[1] != 'l';
+    cur += have_l;
+    bool have_ll = cur[0] == 'l' && cur[1] == 'l';
     cur += have_ll * 2;
-    const bool have_length = have_z || have_ll;
+    const bool have_length = have_z || have_l || have_ll;
     const bool have_flags = have_width || have_length;
     // At the moment only %s supports precision and left-justification.
     CHECK(!((precision >= 0 || left_justified) && *cur != 's'));
 @@ -174,6 +172,7 @@ int VSNPrintf(char *buff, int buff_length,
       case 'd': {
         s64 dval = have_ll  ? va_arg(args, s64)
                    : have_z ? va_arg(args, sptr)
+                   : have_l ? va_arg(args, long)
                             : va_arg(args, int);
         result += AppendSignedDecimal(&buff, buff_end, dval, width,
                                       pad_with_zero);
 @@ -184,26 +183,20 @@ int VSNPrintf(char *buff, int buff_length,
       case 'X': {
         u64 uval = have_ll  ? va_arg(args, u64)
                    : have_z ? va_arg(args, uptr)
+                   : have_l ? va_arg(args, unsigned long)
                             : va_arg(args, unsigned);
         bool uppercase = (*cur == 'X');
         result += AppendUnsigned(&buff, buff_end, uval, (*cur == 'u') ? 10 : 16,
                                  width, pad_with_zero, uppercase);
         break;
       }
-      case 'V': {
-        for (uptr i = 0; i < 16; i++) {
-          unsigned x = va_arg(args, unsigned);
-          result += AppendUnsigned(&buff, buff_end, x, 16, 2, true, false);
-        }
-        break;
-      }
       case 'p': {
-        RAW_CHECK_MSG(!have_flags, kPrintfFormatsHelp);
+        RAW_CHECK_VA(!have_flags, kPrintfFormatsHelp, format);
         result += AppendPointer(&buff, buff_end, va_arg(args, uptr));
         break;
       }
       case 's': {
-        RAW_CHECK_MSG(!have_length, kPrintfFormatsHelp);
+        RAW_CHECK_VA(!have_length, kPrintfFormatsHelp, format);
         // Only left-justified width is supported.
         CHECK(!have_width || left_justified);
         result += AppendString(&buff, buff_end, left_justified ? -width : width,
 @@ -211,17 +204,17 @@ int VSNPrintf(char *buff, int buff_length,
         break;
       }
       case 'c': {
-        RAW_CHECK_MSG(!have_flags, kPrintfFormatsHelp);
+        RAW_CHECK_VA(!have_flags, kPrintfFormatsHelp, format);
         result += AppendChar(&buff, buff_end, va_arg(args, int));
         break;
       }
       case '%' : {
-        RAW_CHECK_MSG(!have_flags, kPrintfFormatsHelp);
+        RAW_CHECK_VA(!have_flags, kPrintfFormatsHelp, format);
         result += AppendChar(&buff, buff_end, '%');
         break;
       }
       default: {
-        RAW_CHECK_MSG(false, kPrintfFormatsHelp);
+        RAW_CHECK_VA(false, kPrintfFormatsHelp, format);
       }
     }
   }
 @@ -317,7 +310,6 @@ static void NOINLINE SharedPrintfCode(bool append_pid, const char *format,
                            format, args);
 }
-FORMAT(1, 2)
 void Printf(const char *format, ...) {
   va_list args;
   va_start(args, format);
 @@ -326,7 +318,6 @@ void Printf(const char *format, ...) {
 }
 // Like Printf, but prints the current PID before the output string.
-FORMAT(1, 2)
 void Report(const char *format, ...) {
   va_list args;
   va_start(args, format);
 @@ -338,7 +329,6 @@ void Report(const char *format, ...) {
 // Returns the number of symbols that should have been written to buffer
 // (not including trailing '\0'). Thus, the string is truncated
 // iff return value is not less than "length".
-FORMAT(3, 4)
 int internal_snprintf(char *buffer, uptr length, const char *format, ...) {
   va_list args;
   va_start(args, format);
 @@ -347,7 +337,6 @@ int internal_snprintf(char *buffer, uptr length, const char *format, ...) {
   return needed_length;
 }
-FORMAT(2, 3)
 void InternalScopedString::append(const char *format, ...) {
   uptr prev_len = length();

lib/tsan/sanitizer_common/sanitizer_procmaps.h+35-7

...	@@ -16,7 +16,7 @@	...	@@ -16,7 +16,7 @@
16	#include "sanitizer_platform.h"	16	#include "sanitizer_platform.h"
17		17
18	#if SANITIZER_LINUX \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| \	18	#if SANITIZER_LINUX \|\| SANITIZER_FREEBSD \|\| SANITIZER_NETBSD \|\| \
19	SANITIZER_MAC \|\| SANITIZER_SOLARIS \|\| \	19	SANITIZER_APPLE \|\| SANITIZER_SOLARIS \|\| \
20	SANITIZER_FUCHSIA	20	SANITIZER_FUCHSIA
21		21
22	#include "sanitizer_common.h"	22	#include "sanitizer_common.h"
...	@@ -65,13 +65,37 @@ class MemoryMappedSegment {	...	@@ -65,13 +65,37 @@ class MemoryMappedSegment {
65	MemoryMappedSegmentData *data_;	65	MemoryMappedSegmentData *data_;
66	};	66	};
67		67
68	class MemoryMappingLayout {	68	struct ImageHeader;
		69
		70	class MemoryMappingLayoutBase {
		71	public:
		72	virtual bool Next(MemoryMappedSegment *segment) { UNIMPLEMENTED(); }
		73	virtual bool Error() const { UNIMPLEMENTED(); };
		74	virtual void Reset() { UNIMPLEMENTED(); }
		75
		76	protected:
		77	~MemoryMappingLayoutBase() {}
		78	};
		79
		80	class MemoryMappingLayout : public MemoryMappingLayoutBase {
69	public:	81	public:
70	explicit MemoryMappingLayout(bool cache_enabled);	82	explicit MemoryMappingLayout(bool cache_enabled);
		83
		84	// This destructor cannot be virtual, as it would cause an operator new() linking
		85	// failures in hwasan test cases. However non-virtual destructors emit warnings
		86	// in macOS build, hence disabling those
		87	#ifdef __clang__
		88	#pragma clang diagnostic push
		89	#pragma clang diagnostic ignored "-Wnon-virtual-dtor"
		90	#endif
71	~MemoryMappingLayout();	91	~MemoryMappingLayout();
72	bool Next(MemoryMappedSegment *segment);	92	#ifdef __clang__
73	bool Error() const;	93	#pragma clang diagnostic pop
74	void Reset();	94	#endif
		95
		96	virtual bool Next(MemoryMappedSegment *segment) override;
		97	virtual bool Error() const override;
		98	virtual void Reset() override;
75	// In some cases, e.g. when running under a sandbox on Linux, ASan is unable	99	// In some cases, e.g. when running under a sandbox on Linux, ASan is unable
76	// to obtain the memory mappings. It should fall back to pre-cached data	100	// to obtain the memory mappings. It should fall back to pre-cached data
77	// instead of aborting.	101	// instead of aborting.
...	@@ -80,10 +104,14 @@ class MemoryMappingLayout {	...	@@ -80,10 +104,14 @@ class MemoryMappingLayout {
80	// Adds all mapped objects into a vector.	104	// Adds all mapped objects into a vector.
81	void DumpListOfModules(InternalMmapVectorNoCtor<LoadedModule> *modules);	105	void DumpListOfModules(InternalMmapVectorNoCtor<LoadedModule> *modules);
82		106
		107	protected:
		108	#if SANITIZER_APPLE
		109	virtual const ImageHeader *CurrentImageHeader();
		110	#endif
		111	MemoryMappingLayoutData data_;
		112
83	private:	113	private:
84	void LoadFromCache();	114	void LoadFromCache();
85
86	MemoryMappingLayoutData data_;
87	};	115	};
88		116
89	// Returns code range for the specified module.	117	// Returns code range for the specified module.

lib/tsan/sanitizer_common/sanitizer_procmaps_bsd.cpp+16

...	@@ -39,6 +39,22 @@	...	@@ -39,6 +39,22 @@
39		39
40	namespace __sanitizer {	40	namespace __sanitizer {
41		41
		42	#if SANITIZER_FREEBSD
		43	void GetMemoryProfile(fill_profile_f cb, uptr *stats) {
		44	const int Mib[] = {
		45	CTL_KERN,
		46	KERN_PROC,
		47	KERN_PROC_PID,
		48	getpid()
		49	};
		50
		51	struct kinfo_proc InfoProc;
		52	uptr Len = sizeof(InfoProc);
		53	CHECK_EQ(internal_sysctl(Mib, ARRAY_SIZE(Mib), nullptr, (uptr *)&InfoProc, &Len, 0), 0);
		54	cb(0, InfoProc.ki_rssize * GetPageSizeCached(), false, stats);
		55	}
		56	#endif
		57
42	void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {	58	void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
43	const int Mib[] = {	59	const int Mib[] = {
44	#if SANITIZER_FREEBSD	60	#if SANITIZER_FREEBSD

lib/tsan/sanitizer_common/sanitizer_procmaps_common.cpp+23-5

 @@ -145,29 +145,47 @@ void MemoryMappingLayout::DumpListOfModules(
   }
 }
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {
+#if SANITIZER_LINUX || SANITIZER_ANDROID || SANITIZER_SOLARIS || SANITIZER_NETBSD
+void GetMemoryProfile(fill_profile_f cb, uptr *stats) {
   char *smaps = nullptr;
   uptr smaps_cap = 0;
   uptr smaps_len = 0;
   if (!ReadFileToBuffer("/proc/self/smaps", &smaps, &smaps_cap, &smaps_len))
     return;
+  ParseUnixMemoryProfile(cb, stats, smaps, smaps_len);
+  UnmapOrDie(smaps, smaps_cap);
+}
+void ParseUnixMemoryProfile(fill_profile_f cb, uptr *stats, char *smaps,
+                            uptr smaps_len) {
   uptr start = 0;
   bool file = false;
   const char *pos = smaps;
-  while (pos < smaps + smaps_len) {
+  char *end = smaps + smaps_len;
+  if (smaps_len < 2)
+    return;
+  // The following parsing can crash on almost every line
+  // in the case of malformed/truncated input.
+  // Fixing that is hard b/c e.g. ParseDecimal does not
+  // even accept end of the buffer and assumes well-formed input.
+  // So instead we patch end of the input a bit,
+  // it does not affect well-formed complete inputs.
+  *--end = 0;
+  *--end = '\n';
+  while (pos < end) {
     if (IsHex(pos[0])) {
       start = ParseHex(&pos);
       for (; *pos != '/' && *pos > '\n'; pos++) {}
       file = *pos == '/';
     } else if (internal_strncmp(pos, "Rss:", 4) == 0) {
-      while (!IsDecimal(*pos)) pos++;
+      while (pos < end && !IsDecimal(*pos)) pos++;
       uptr rss = ParseDecimal(&pos) * 1024;
-      cb(start, rss, file, stats, stats_size);
+      cb(start, rss, file, stats);
     }
     while (*pos++ != '\n') {}
   }
-  UnmapOrDie(smaps, smaps_cap);
 }
+#endif
 } // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_procmaps_mac.cpp+91-17

...	@@ -10,7 +10,7 @@	...	@@ -10,7 +10,7 @@
10	//===----------------------------------------------------------------------===//	10	//===----------------------------------------------------------------------===//
11		11
12	#include "sanitizer_platform.h"	12	#include "sanitizer_platform.h"
13	#if SANITIZER_MAC	13	#if SANITIZER_APPLE
14	#include "sanitizer_common.h"	14	#include "sanitizer_common.h"
15	#include "sanitizer_placement_new.h"	15	#include "sanitizer_placement_new.h"
16	#include "sanitizer_procmaps.h"	16	#include "sanitizer_procmaps.h"
...	@@ -136,29 +136,34 @@ void MemoryMappingLayout::LoadFromCache() {	...	@@ -136,29 +136,34 @@ void MemoryMappingLayout::LoadFromCache() {
136	// No-op on Mac for now.	136	// No-op on Mac for now.
137	}	137	}
138		138
		139	static bool IsDyldHdr(const mach_header *hdr) {
		140	return (hdr->magic == MH_MAGIC \|\| hdr->magic == MH_MAGIC_64) &&
		141	hdr->filetype == MH_DYLINKER;
		142	}
		143
139	// _dyld_get_image_header() and related APIs don't report dyld itself.	144	// _dyld_get_image_header() and related APIs don't report dyld itself.
140	// We work around this by manually recursing through the memory map	145	// We work around this by manually recursing through the memory map
141	// until we hit a Mach header matching dyld instead. These recurse	146	// until we hit a Mach header matching dyld instead. These recurse
142	// calls are expensive, but the first memory map generation occurs	147	// calls are expensive, but the first memory map generation occurs
143	// early in the process, when dyld is one of the only images loaded,	148	// early in the process, when dyld is one of the only images loaded,
144	// so it will be hit after only a few iterations.	149	// so it will be hit after only a few iterations. These assumptions don't hold
145	static mach_header *get_dyld_image_header() {	150	// on macOS 13+ anymore (dyld itself has moved into the shared cache).
146	unsigned depth = 1;	151	static mach_header *GetDyldImageHeaderViaVMRegion() {
147	vm_size_t size = 0;
148	vm_address_t address = 0;	152	vm_address_t address = 0;
149	kern_return_t err = KERN_SUCCESS;
150	mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
151		153
152	while (true) {	154	while (true) {
		155	vm_size_t size = 0;
		156	unsigned depth = 1;
153	struct vm_region_submap_info_64 info;	157	struct vm_region_submap_info_64 info;
154	err = vm_region_recurse_64(mach_task_self(), &address, &size, &depth,	158	mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
155	(vm_region_info_t)&info, &count);	159	kern_return_t err =
		160	vm_region_recurse_64(mach_task_self(), &address, &size, &depth,
		161	(vm_region_info_t)&info, &count);
156	if (err != KERN_SUCCESS) return nullptr;	162	if (err != KERN_SUCCESS) return nullptr;
157		163
158	if (size >= sizeof(mach_header) && info.protection & kProtectionRead) {	164	if (size >= sizeof(mach_header) && info.protection & kProtectionRead) {
159	mach_header hdr = (mach_header )address;	165	mach_header hdr = (mach_header )address;
160	if ((hdr->magic == MH_MAGIC \|\| hdr->magic == MH_MAGIC_64) &&	166	if (IsDyldHdr(hdr)) {
161	hdr->filetype == MH_DYLINKER) {
162	return hdr;	167	return hdr;
163	}	168	}
164	}	169	}
...	@@ -166,8 +171,69 @@ static mach_header *get_dyld_image_header() {	...	@@ -166,8 +171,69 @@ static mach_header *get_dyld_image_header() {
166	}	171	}
167	}	172	}
168		173
		174	extern "C" {
		175	struct dyld_shared_cache_dylib_text_info {
		176	uint64_t version; // current version 2
		177	// following fields all exist in version 1
		178	uint64_t loadAddressUnslid;
		179	uint64_t textSegmentSize;
		180	uuid_t dylibUuid;
		181	const char *path; // pointer invalid at end of iterations
		182	// following fields all exist in version 2
		183	uint64_t textSegmentOffset; // offset from start of cache
		184	};
		185	typedef struct dyld_shared_cache_dylib_text_info
		186	dyld_shared_cache_dylib_text_info;
		187
		188	extern bool _dyld_get_shared_cache_uuid(uuid_t uuid);
		189	extern const void _dyld_get_shared_cache_range(size_t length);
		190	extern int dyld_shared_cache_iterate_text(
		191	const uuid_t cacheUuid,
		192	void (^callback)(const dyld_shared_cache_dylib_text_info *info));
		193	} // extern "C"
		194
		195	static mach_header *GetDyldImageHeaderViaSharedCache() {
		196	uuid_t uuid;
		197	bool hasCache = _dyld_get_shared_cache_uuid(uuid);
		198	if (!hasCache)
		199	return nullptr;
		200
		201	size_t cacheLength;
		202	__block uptr cacheStart = (uptr)_dyld_get_shared_cache_range(&cacheLength);
		203	CHECK(cacheStart && cacheLength);
		204
		205	__block mach_header *dyldHdr = nullptr;
		206	int res = dyld_shared_cache_iterate_text(
		207	uuid, ^(const dyld_shared_cache_dylib_text_info *info) {
		208	CHECK_GE(info->version, 2);
		209	mach_header *hdr =
		210	(mach_header *)(cacheStart + info->textSegmentOffset);
		211	if (IsDyldHdr(hdr))
		212	dyldHdr = hdr;
		213	});
		214	CHECK_EQ(res, 0);
		215
		216	return dyldHdr;
		217	}
		218
169	const mach_header *get_dyld_hdr() {	219	const mach_header *get_dyld_hdr() {
170	if (!dyld_hdr) dyld_hdr = get_dyld_image_header();	220	if (!dyld_hdr) {
		221	// On macOS 13+, dyld itself has moved into the shared cache. Looking it up
		222	// via vm_region_recurse_64() causes spins/hangs/crashes.
		223	if (GetMacosAlignedVersion() >= MacosVersion(13, 0)) {
		224	dyld_hdr = GetDyldImageHeaderViaSharedCache();
		225	if (!dyld_hdr) {
		226	VReport(1,
		227	"Failed to lookup the dyld image header in the shared cache on "
		228	"macOS 13+ (or no shared cache in use). Falling back to "
		229	"lookup via vm_region_recurse_64().\n");
		230	dyld_hdr = GetDyldImageHeaderViaVMRegion();
		231	}
		232	} else {
		233	dyld_hdr = GetDyldImageHeaderViaVMRegion();
		234	}
		235	CHECK(dyld_hdr);
		236	}
171		237
172	return dyld_hdr;	238	return dyld_hdr;
173	}	239	}
...	@@ -184,7 +250,9 @@ static bool NextSegmentLoad(MemoryMappedSegment *segment,	...	@@ -184,7 +250,9 @@ static bool NextSegmentLoad(MemoryMappedSegment *segment,
184	MemoryMappedSegmentData *seg_data,	250	MemoryMappedSegmentData *seg_data,
185	MemoryMappingLayoutData *layout_data) {	251	MemoryMappingLayoutData *layout_data) {
186	const char *lc = layout_data->current_load_cmd_addr;	252	const char *lc = layout_data->current_load_cmd_addr;
		253
187	layout_data->current_load_cmd_addr += ((const load_command *)lc)->cmdsize;	254	layout_data->current_load_cmd_addr += ((const load_command *)lc)->cmdsize;
		255	layout_data->current_load_cmd_count--;
188	if (((const load_command *)lc)->cmd == kLCSegment) {	256	if (((const load_command *)lc)->cmd == kLCSegment) {
189	const SegmentCommand* sc = (const SegmentCommand *)lc;	257	const SegmentCommand* sc = (const SegmentCommand *)lc;
190	uptr base_virt_addr, addr_mask;	258	uptr base_virt_addr, addr_mask;
...	@@ -292,11 +360,16 @@ static bool IsModuleInstrumented(const load_command *first_lc) {	...	@@ -292,11 +360,16 @@ static bool IsModuleInstrumented(const load_command *first_lc) {
292	return false;	360	return false;
293	}	361	}
294		362
		363	const ImageHeader *MemoryMappingLayout::CurrentImageHeader() {
		364	const mach_header *hdr = (data_.current_image == kDyldImageIdx)
		365	? get_dyld_hdr()
		366	: _dyld_get_image_header(data_.current_image);
		367	return (const ImageHeader *)hdr;
		368	}
		369
295	bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {	370	bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
296	for (; data_.current_image >= kDyldImageIdx; data_.current_image--) {	371	for (; data_.current_image >= kDyldImageIdx; data_.current_image--) {
297	const mach_header *hdr = (data_.current_image == kDyldImageIdx)	372	const mach_header hdr = (const mach_header )CurrentImageHeader();
298	? get_dyld_hdr()
299	: _dyld_get_image_header(data_.current_image);
300	if (!hdr) continue;	373	if (!hdr) continue;
301	if (data_.current_load_cmd_count < 0) {	374	if (data_.current_load_cmd_count < 0) {
302	// Set up for this image;	375	// Set up for this image;
...	@@ -326,7 +399,7 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {	...	@@ -326,7 +399,7 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
326	(const load_command *)data_.current_load_cmd_addr);	399	(const load_command *)data_.current_load_cmd_addr);
327	}	400	}
328		401
329	for (; data_.current_load_cmd_count >= 0; data_.current_load_cmd_count--) {	402	while (data_.current_load_cmd_count > 0) {
330	switch (data_.current_magic) {	403	switch (data_.current_magic) {
331	// data_.current_magic may be only one of MH_MAGIC, MH_MAGIC_64.	404	// data_.current_magic may be only one of MH_MAGIC, MH_MAGIC_64.
332	#ifdef MH_MAGIC_64	405	#ifdef MH_MAGIC_64
...	@@ -347,6 +420,7 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {	...	@@ -347,6 +420,7 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
347	}	420	}
348	// If we get here, no more load_cmd's in this image talk about	421	// If we get here, no more load_cmd's in this image talk about
349	// segments. Go on to the next image.	422	// segments. Go on to the next image.
		423	data_.current_load_cmd_count = -1; // This will trigger loading next image
350	}	424	}
351	return false;	425	return false;
352	}	426	}
...	@@ -376,4 +450,4 @@ void MemoryMappingLayout::DumpListOfModules(	...	@@ -376,4 +450,4 @@ void MemoryMappingLayout::DumpListOfModules(
376		450
377	} // namespace __sanitizer	451	} // namespace __sanitizer
378		452
379	#endif // SANITIZER_MAC	453	#endif // SANITIZER_APPLE

lib/tsan/sanitizer_common/sanitizer_procmaps_solaris.cpp+37-13

...	@@ -13,21 +13,30 @@	...	@@ -13,21 +13,30 @@
13	#undef _FILE_OFFSET_BITS	13	#undef _FILE_OFFSET_BITS
14	#include "sanitizer_platform.h"	14	#include "sanitizer_platform.h"
15	#if SANITIZER_SOLARIS	15	#if SANITIZER_SOLARIS
16	#include "sanitizer_common.h"	16	# include <fcntl.h>
17	#include "sanitizer_procmaps.h"	17	# include <limits.h>
		18	# include <procfs.h>
18		19
19	#include <procfs.h>	20	# include "sanitizer_common.h"
20	#include <limits.h>	21	# include "sanitizer_procmaps.h"
21		22
22	namespace __sanitizer {	23	namespace __sanitizer {
23		24
24	void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {	25	void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
25	if (!ReadFileToBuffer("/proc/self/xmap", &proc_maps->data,	26	uptr fd = internal_open("/proc/self/xmap", O_RDONLY);
26	&proc_maps->mmaped_size, &proc_maps->len)) {	27	CHECK_NE(fd, -1);
27	proc_maps->data = nullptr;	28	uptr Size = internal_filesize(fd);
28	proc_maps->mmaped_size = 0;	29	CHECK_GT(Size, 0);
29	proc_maps->len = 0;	30
30	}	31	// Allow for additional entries by following mmap.
		32	size_t MmapedSize = Size * 4 / 3;
		33	void *VmMap = MmapOrDie(MmapedSize, "ReadProcMaps()");
		34	Size = internal_read(fd, VmMap, MmapedSize);
		35	CHECK_NE(Size, -1);
		36	internal_close(fd);
		37	proc_maps->data = (char *)VmMap;
		38	proc_maps->mmaped_size = MmapedSize;
		39	proc_maps->len = Size;
31	}	40	}
32		41
33	bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {	42	bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
...	@@ -49,13 +58,28 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {	...	@@ -49,13 +58,28 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
49	segment->protection \|= kProtectionWrite;	58	segment->protection \|= kProtectionWrite;
50	if ((xmapentry->pr_mflags & MA_EXEC) != 0)	59	if ((xmapentry->pr_mflags & MA_EXEC) != 0)
51	segment->protection \|= kProtectionExecute;	60	segment->protection \|= kProtectionExecute;
		61	if ((xmapentry->pr_mflags & MA_SHARED) != 0)
		62	segment->protection \|= kProtectionShared;
52		63
53	if (segment->filename != NULL && segment->filename_size > 0) {	64	if (segment->filename != NULL && segment->filename_size > 0) {
54	char proc_path[PATH_MAX + 1];	65	char proc_path[PATH_MAX + 1];
55		66
56	internal_snprintf(proc_path, sizeof(proc_path), "/proc/self/path/%s",	67	// Avoid unnecessary readlink on unnamed entires.
57	xmapentry->pr_mapname);	68	if (xmapentry->pr_mapname[0] == '\0')
58	internal_readlink(proc_path, segment->filename, segment->filename_size);	69	segment->filename[0] = '\0';
		70	else {
		71	internal_snprintf(proc_path, sizeof(proc_path), "/proc/self/path/%s",
		72	xmapentry->pr_mapname);
		73	ssize_t sz = internal_readlink(proc_path, segment->filename,
		74	segment->filename_size - 1);
		75
		76	// If readlink failed, the map is anonymous.
		77	if (sz == -1)
		78	segment->filename[0] = '\0';
		79	else if ((size_t)sz < segment->filename_size)
		80	// readlink doesn't NUL-terminate.
		81	segment->filename[sz] = '\0';
		82	}
59	}	83	}
60		84
61	data_.current += sizeof(prxmap_t);	85	data_.current += sizeof(prxmap_t);

lib/tsan/sanitizer_common/sanitizer_quarantine.h+12-17

 @@ -68,10 +68,6 @@ struct QuarantineBatch {
 COMPILER_CHECK(sizeof(QuarantineBatch) <= (1 << 13));  // 8Kb.
-// The callback interface is:
-// void Callback::Recycle(Node *ptr);
-// void *cb.Allocate(uptr size);
-// void cb.Deallocate(void *ptr);
 template<typename Callback, typename Node>
 class Quarantine {
  public:
 @@ -94,21 +90,20 @@ class Quarantine {
     recycle_mutex_.Init();
   }
-  uptr GetSize() const { return atomic_load_relaxed(&max_size_); }
+  uptr GetMaxSize() const { return atomic_load_relaxed(&max_size_); }
-  uptr GetCacheSize() const {
+  uptr GetMaxCacheSize() const { return atomic_load_relaxed(&max_cache_size_); }
-    return atomic_load_relaxed(&max_cache_size_);
-  }
   void Put(Cache *c, Callback cb, Node *ptr, uptr size) {
-    uptr cache_size = GetCacheSize();
+    uptr max_cache_size = GetMaxCacheSize();
-    if (cache_size) {
+    if (max_cache_size && size <= GetMaxSize()) {
+      cb.PreQuarantine(ptr);
       c->Enqueue(cb, ptr, size);
     } else {
-      // GetCacheSize() == 0 only when GetSize() == 0 (see Init).
+      // GetMaxCacheSize() == 0 only when GetMaxSize() == 0 (see Init).
-      cb.Recycle(ptr);
+      cb.RecyclePassThrough(ptr);
     }
     // Check cache size anyway to accommodate for runtime cache_size change.
-    if (c->Size() > cache_size)
+    if (c->Size() > max_cache_size)
       Drain(c, cb);
   }
 @@ -117,7 +112,7 @@ class Quarantine {
       SpinMutexLock l(&cache_mutex_);
       cache_.Transfer(c);
     }
-    if (cache_.Size() > GetSize() && recycle_mutex_.TryLock())
+    if (cache_.Size() > GetMaxSize() && recycle_mutex_.TryLock())
       Recycle(atomic_load_relaxed(&min_size_), cb);
   }
 @@ -133,7 +128,7 @@ class Quarantine {
   void PrintStats() const {
     // It assumes that the world is stopped, just as the allocator's PrintStats.
     Printf("Quarantine limits: global: %zdMb; thread local: %zdKb\n",
-           GetSize() >> 20, GetCacheSize() >> 10);
+           GetMaxSize() >> 20, GetMaxCacheSize() >> 10);
     cache_.PrintStats();
   }
 @@ -149,8 +144,8 @@ class Quarantine {
   Cache cache_;
   char pad2_[kCacheLineSize];
-  void NOINLINE Recycle(uptr min_size, Callback cb) REQUIRES(recycle_mutex_)
+  void NOINLINE Recycle(uptr min_size, Callback cb)
-      RELEASE(recycle_mutex_) {
+      SANITIZER_REQUIRES(recycle_mutex_) SANITIZER_RELEASE(recycle_mutex_) {
     Cache tmp;
     {
       SpinMutexLock l(&cache_mutex_);

lib/tsan/sanitizer_common/sanitizer_range.cpp created+62

...	@@ -0,0 +1,62 @@
1	//===-- sanitizer_range.cpp -----------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "sanitizer_range.h"
10
11	#include "sanitizer_common/sanitizer_array_ref.h"
12
13	namespace __sanitizer {
14
15	void Intersect(ArrayRef<Range> a, ArrayRef<Range> b,
16	InternalMmapVectorNoCtor<Range> &output) {
17	output.clear();
18
19	struct Event {
20	uptr val;
21	s8 diff1;
22	s8 diff2;
23	};
24
25	InternalMmapVector<Event> events;
26	for (const Range &r : a) {
27	CHECK_LE(r.begin, r.end);
28	events.push_back({r.begin, 1, 0});
29	events.push_back({r.end, -1, 0});
30	}
31
32	for (const Range &r : b) {
33	CHECK_LE(r.begin, r.end);
34	events.push_back({r.begin, 0, 1});
35	events.push_back({r.end, 0, -1});
36	}
37
38	Sort(events.data(), events.size(),
39	[](const Event &lh, const Event &rh) { return lh.val < rh.val; });
40
41	uptr start = 0;
42	sptr state1 = 0;
43	sptr state2 = 0;
44	for (const auto &e : events) {
45	if (e.val != start) {
46	DCHECK_GE(state1, 0);
47	DCHECK_GE(state2, 0);
48	if (state1 && state2) {
49	if (!output.empty() && start == output.back().end)
50	output.back().end = e.val;
51	else
52	output.push_back({start, e.val});
53	}
54	start = e.val;
55	}
56
57	state1 += e.diff1;
58	state2 += e.diff2;
59	}
60	}
61
62	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_range.h created+40

...	@@ -0,0 +1,40 @@
1	//===-- sanitizer_range.h ---------------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Contais Range and related utilities.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#ifndef SANITIZER_RANGE_H
14	#define SANITIZER_RANGE_H
15
16	#include "sanitizer_common.h"
17	#include "sanitizer_common/sanitizer_array_ref.h"
18
19	namespace __sanitizer {
20
21	struct Range {
22	uptr begin;
23	uptr end;
24	};
25
26	inline bool operator==(const Range &lhs, const Range &rhs) {
27	return lhs.begin == rhs.begin && lhs.end == rhs.end;
28	}
29
30	inline bool operator!=(const Range &lhs, const Range &rhs) {
31	return !(lhs == rhs);
32	}
33
34	// Calculates intersection of two sets of regions in O(N log N) time.
35	void Intersect(ArrayRef<Range> a, ArrayRef<Range> b,
36	InternalMmapVectorNoCtor<Range> &output);
37
38	} // namespace __sanitizer
39
40	#endif // SANITIZER_RANGE_H

lib/tsan/sanitizer_common/sanitizer_redefine_builtins.h created+52

...	@@ -0,0 +1,52 @@
1	//===-- sanitizer_redefine_builtins.h ---------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Redefine builtin functions to use internal versions. This is needed where
10	// compiler optimizations end up producing unwanted libcalls!
11	//
12	//===----------------------------------------------------------------------===//
13	#ifndef SANITIZER_COMMON_NO_REDEFINE_BUILTINS
14	#ifndef SANITIZER_REDEFINE_BUILTINS_H
15	#define SANITIZER_REDEFINE_BUILTINS_H
16
17	// The asm hack only works with GCC and Clang.
18	#if !defined(_WIN32)
19
20	asm("memcpy = __sanitizer_internal_memcpy");
21	asm("memmove = __sanitizer_internal_memmove");
22	asm("memset = __sanitizer_internal_memset");
23
24	// The builtins should not be redefined in source files that make use of C++
25	// standard libraries, in particular where C++STL headers with inline functions
26	// are used. The redefinition in such cases would lead to ODR violations.
27	//
28	// Try to break the build in common cases where builtins shouldn't be redefined.
29	namespace std {
30	class Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file {
31	Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file(
32	const Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file&) = delete;
33	Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file& operator=(
34	const Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file&) = delete;
35	};
36	using array = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
37	using atomic = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
38	using function = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
39	using map = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
40	using set = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
41	using shared_ptr = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
42	using string = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
43	using unique_ptr = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
44	using unordered_map = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
45	using unordered_set = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
46	using vector = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
47	} // namespace std
48
49	#endif // !_WIN32
50
51	#endif // SANITIZER_REDEFINE_BUILTINS_H
52	#endif // SANITIZER_COMMON_NO_REDEFINE_BUILTINS

lib/tsan/sanitizer_common/sanitizer_ring_buffer.h+9-4

 @@ -86,10 +86,13 @@ class CompactRingBuffer {
   // Lower bytes store the address of the next buffer element.
   static constexpr int kPageSizeBits = 12;
   static constexpr int kSizeShift = 56;
+  static constexpr int kSizeBits = 64 - kSizeShift;
   static constexpr uptr kNextMask = (1ULL << kSizeShift) - 1;
   uptr GetStorageSize() const { return (long_ >> kSizeShift) << kPageSizeBits; }
+  static uptr SignExtend(uptr x) { return ((sptr)x) << kSizeBits >> kSizeBits; }
   void Init(void *storage, uptr size) {
     CHECK_EQ(sizeof(CompactRingBuffer<T>), sizeof(void *));
     CHECK(IsPowerOfTwo(size));
 @@ -97,12 +100,14 @@ class CompactRingBuffer {
     CHECK_LE(size, 128 << kPageSizeBits);
     CHECK_EQ(size % 4096, 0);
     CHECK_EQ(size % sizeof(T), 0);
-    CHECK_EQ((uptr)storage % (size * 2), 0);
+    uptr st = (uptr)storage;
-    long_ = (uptr)storage | ((size >> kPageSizeBits) << kSizeShift);
+    CHECK_EQ(st % (size * 2), 0);
+    CHECK_EQ(st, SignExtend(st & kNextMask));
+    long_ = (st & kNextMask) | ((size >> kPageSizeBits) << kSizeShift);
   }
   void SetNext(const T *next) {
-    long_ = (long_ & ~kNextMask) | (uptr)next;
+    long_ = (long_ & ~kNextMask) | ((uptr)next & kNextMask);
   }
  public:
 @@ -119,7 +124,7 @@ class CompactRingBuffer {
     SetNext((const T *)storage + Idx);
   }
-  T *Next() const { return (T *)(long_ & kNextMask); }
+  T *Next() const { return (T *)(SignExtend(long_ & kNextMask)); }
   void *StartOfStorage() const {
     return (void *)((uptr)Next() & ~(GetStorageSize() - 1));

lib/tsan/sanitizer_common/sanitizer_signal_interceptors.inc+13-2

 @@ -29,12 +29,21 @@ using namespace __sanitizer;
 #endif
 #ifndef SIGNAL_INTERCEPTOR_SIGACTION_IMPL
-#define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact) \
+#  define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signum, act, oldact)              \
-  { return REAL(sigaction_symname)(signum, act, oldact); }
+    {                                                                         \
+      if (!REAL(sigaction_symname)) {                                         \
+        Printf(                                                               \
+            "Warning: REAL(sigaction_symname) == nullptr. This may happen "   \
+            "if you link with ubsan statically. Sigaction will not work.\n"); \
+        return -1;                                                            \
+      }                                                                       \
+      return REAL(sigaction_symname)(signum, act, oldact);                    \
+    }
 #endif
 #if SANITIZER_INTERCEPT_BSD_SIGNAL
 INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) {
+  SIGNAL_INTERCEPTOR_ENTER();
   if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0;
   SIGNAL_INTERCEPTOR_SIGNAL_IMPL(bsd_signal, signum, handler);
 }
 @@ -45,6 +54,7 @@ INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) {
 #if SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION
 INTERCEPTOR(uptr, signal, int signum, uptr handler) {
+  SIGNAL_INTERCEPTOR_ENTER();
   if (GetHandleSignalMode(signum) == kHandleSignalExclusive)
     return (uptr) nullptr;
   SIGNAL_INTERCEPTOR_SIGNAL_IMPL(signal, signum, handler);
 @@ -53,6 +63,7 @@ INTERCEPTOR(uptr, signal, int signum, uptr handler) {
 INTERCEPTOR(int, sigaction_symname, int signum,
             const __sanitizer_sigaction *act, __sanitizer_sigaction *oldact) {
+  SIGNAL_INTERCEPTOR_ENTER();
   if (GetHandleSignalMode(signum) == kHandleSignalExclusive) {
     if (!oldact) return 0;
     act = nullptr;

lib/tsan/sanitizer_common/sanitizer_solaris.cpp-22

...	@@ -225,28 +225,6 @@ void FutexWait(atomic_uint32_t *p, u32 cmp) {	...	@@ -225,28 +225,6 @@ void FutexWait(atomic_uint32_t *p, u32 cmp) {
225		225
226	void FutexWake(atomic_uint32_t *p, u32 count) {}	226	void FutexWake(atomic_uint32_t *p, u32 count) {}
227		227
228	BlockingMutex::BlockingMutex() {
229	CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_));
230	internal_memset(this, 0, sizeof(*this));
231	CHECK_EQ(mutex_init((mutex_t *)&opaque_storage_, USYNC_THREAD, NULL), 0);
232	}
233
234	void BlockingMutex::Lock() {
235	CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_));
236	CHECK_NE(owner_, (uptr)thr_self());
237	CHECK_EQ(mutex_lock((mutex_t *)&opaque_storage_), 0);
238	CHECK(!owner_);
239	owner_ = (uptr)thr_self();
240	}
241
242	void BlockingMutex::Unlock() {
243	CHECK(owner_ == (uptr)thr_self());
244	owner_ = 0;
245	CHECK_EQ(mutex_unlock((mutex_t *)&opaque_storage_), 0);
246	}
247
248	void BlockingMutex::CheckLocked() const { CHECK_EQ((uptr)thr_self(), owner_); }
249
250	} // namespace __sanitizer	228	} // namespace __sanitizer
251		229
252	#endif // SANITIZER_SOLARIS	230	#endif // SANITIZER_SOLARIS

lib/tsan/sanitizer_common/sanitizer_solaris.h created+56

...	@@ -0,0 +1,56 @@
1	//===-- sanitizer_solaris.h -------------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of Sanitizer runtime. It contains Solaris-specific
10	// definitions.
11	//
12	//===----------------------------------------------------------------------===//
13
14	#ifndef SANITIZER_SOLARIS_H
15	#define SANITIZER_SOLARIS_H
16
17	#include "sanitizer_internal_defs.h"
18
19	#if SANITIZER_SOLARIS
20
21	#include <link.h>
22
23	namespace __sanitizer {
24
25	// Beginning of declaration from OpenSolaris/Illumos
26	// $SRC/cmd/sgs/include/rtld.h.
27	struct Rt_map {
28	Link_map rt_public;
29	const char *rt_pathname;
30	ulong_t rt_padstart;
31	ulong_t rt_padimlen;
32	ulong_t rt_msize;
33	uint_t rt_flags;
34	uint_t rt_flags1;
35	ulong_t rt_tlsmodid;
36	};
37
38	// Structure matching the Solaris 11.4 struct dl_phdr_info used to determine
39	// presence of dlpi_tls_modid field at runtime. Cf. Solaris 11.4
40	// dl_iterate_phdr(3C), Example 2.
41	struct dl_phdr_info_test {
42	ElfW(Addr) dlpi_addr;
43	const char *dlpi_name;
44	const ElfW(Phdr) * dlpi_phdr;
45	ElfW(Half) dlpi_phnum;
46	u_longlong_t dlpi_adds;
47	u_longlong_t dlpi_subs;
48	size_t dlpi_tls_modid;
49	void *dlpi_tls_data;
50	};
51
52	} // namespace __sanitizer
53
54	#endif // SANITIZER_SOLARIS
55
56	#endif // SANITIZER_SOLARIS_H

lib/tsan/sanitizer_common/sanitizer_stack_store.cpp created+379

...	@@ -0,0 +1,379 @@
1	//===-- sanitizer_stack_store.cpp -------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "sanitizer_stack_store.h"
10
11	#include "sanitizer_atomic.h"
12	#include "sanitizer_common.h"
13	#include "sanitizer_internal_defs.h"
14	#include "sanitizer_leb128.h"
15	#include "sanitizer_lzw.h"
16	#include "sanitizer_placement_new.h"
17	#include "sanitizer_stacktrace.h"
18
19	namespace __sanitizer {
20
21	namespace {
22	struct StackTraceHeader {
23	static constexpr u32 kStackSizeBits = 8;
24
25	u8 size;
26	u8 tag;
27	explicit StackTraceHeader(const StackTrace &trace)
28	: size(Min<uptr>(trace.size, (1u << 8) - 1)), tag(trace.tag) {
29	CHECK_EQ(trace.tag, static_cast<uptr>(tag));
30	}
31	explicit StackTraceHeader(uptr h)
32	: size(h & ((1 << kStackSizeBits) - 1)), tag(h >> kStackSizeBits) {}
33
34	uptr ToUptr() const {
35	return static_cast<uptr>(size) \| (static_cast<uptr>(tag) << kStackSizeBits);
36	}
37	};
38	} // namespace
39
40	StackStore::Id StackStore::Store(const StackTrace &trace, uptr *pack) {
41	if (!trace.size && !trace.tag)
42	return 0;
43	StackTraceHeader h(trace);
44	uptr idx = 0;
45	*pack = 0;
46	uptr *stack_trace = Alloc(h.size + 1, &idx, pack);
47	*stack_trace = h.ToUptr();
48	internal_memcpy(stack_trace + 1, trace.trace, h.size * sizeof(uptr));
49	*pack += blocks_[GetBlockIdx(idx)].Stored(h.size + 1);
50	return OffsetToId(idx);
51	}
52
53	StackTrace StackStore::Load(Id id) {
54	if (!id)
55	return {};
56	uptr idx = IdToOffset(id);
57	uptr block_idx = GetBlockIdx(idx);
58	CHECK_LT(block_idx, ARRAY_SIZE(blocks_));
59	const uptr *stack_trace = blocks_[block_idx].GetOrUnpack(this);
60	if (!stack_trace)
61	return {};
62	stack_trace += GetInBlockIdx(idx);
63	StackTraceHeader h(*stack_trace);
64	return StackTrace(stack_trace + 1, h.size, h.tag);
65	}
66
67	uptr StackStore::Allocated() const {
68	return atomic_load_relaxed(&allocated_) + sizeof(*this);
69	}
70
71	uptr StackStore::Alloc(uptr count, uptr idx, uptr *pack) {
72	for (;;) {
73	// Optimisic lock-free allocation, essentially try to bump the
74	// total_frames_.
75	uptr start = atomic_fetch_add(&total_frames_, count, memory_order_relaxed);
76	uptr block_idx = GetBlockIdx(start);
77	uptr last_idx = GetBlockIdx(start + count - 1);
78	if (LIKELY(block_idx == last_idx)) {
79	// Fits into the a single block.
80	CHECK_LT(block_idx, ARRAY_SIZE(blocks_));
81	*idx = start;
82	return blocks_[block_idx].GetOrCreate(this) + GetInBlockIdx(start);
83	}
84
85	// Retry. We can't use range allocated in two different blocks.
86	CHECK_LE(count, kBlockSizeFrames);
87	uptr in_first = kBlockSizeFrames - GetInBlockIdx(start);
88	// Mark tail/head of these blocks as "stored".to avoid waiting before we can
89	// Pack().
90	*pack += blocks_[block_idx].Stored(in_first);
91	*pack += blocks_[last_idx].Stored(count - in_first);
92	}
93	}
94
95	void StackStore::Map(uptr size, const char mem_type) {
96	atomic_fetch_add(&allocated_, size, memory_order_relaxed);
97	return MmapNoReserveOrDie(size, mem_type);
98	}
99
100	void StackStore::Unmap(void *addr, uptr size) {
101	atomic_fetch_sub(&allocated_, size, memory_order_relaxed);
102	UnmapOrDie(addr, size);
103	}
104
105	uptr StackStore::Pack(Compression type) {
106	uptr res = 0;
107	for (BlockInfo &b : blocks_) res += b.Pack(type, this);
108	return res;
109	}
110
111	void StackStore::LockAll() {
112	for (BlockInfo &b : blocks_) b.Lock();
113	}
114
115	void StackStore::UnlockAll() {
116	for (BlockInfo &b : blocks_) b.Unlock();
117	}
118
119	void StackStore::TestOnlyUnmap() {
120	for (BlockInfo &b : blocks_) b.TestOnlyUnmap(this);
121	internal_memset(this, 0, sizeof(*this));
122	}
123
124	uptr *StackStore::BlockInfo::Get() const {
125	// Idiomatic double-checked locking uses memory_order_acquire here. But
126	// relaxed is fine for us, justification is similar to
127	// TwoLevelMap::GetOrCreate.
128	return reinterpret_cast<uptr *>(atomic_load_relaxed(&data_));
129	}
130
131	uptr StackStore::BlockInfo::Create(StackStore store) {
132	SpinMutexLock l(&mtx_);
133	uptr *ptr = Get();
134	if (!ptr) {
135	ptr = reinterpret_cast<uptr *>(store->Map(kBlockSizeBytes, "StackStore"));
136	atomic_store(&data_, reinterpret_cast<uptr>(ptr), memory_order_release);
137	}
138	return ptr;
139	}
140
141	uptr StackStore::BlockInfo::GetOrCreate(StackStore store) {
142	uptr *ptr = Get();
143	if (LIKELY(ptr))
144	return ptr;
145	return Create(store);
146	}
147
148	class SLeb128Encoder {
149	public:
150	SLeb128Encoder(u8 begin, u8 end) : begin(begin), end(end) {}
151
152	bool operator==(const SLeb128Encoder &other) const {
153	return begin == other.begin;
154	}
155
156	bool operator!=(const SLeb128Encoder &other) const {
157	return begin != other.begin;
158	}
159
160	SLeb128Encoder &operator=(uptr v) {
161	sptr diff = v - previous;
162	begin = EncodeSLEB128(diff, begin, end);
163	previous = v;
164	return *this;
165	}
166	SLeb128Encoder &operator() { return this; }
167	SLeb128Encoder &operator++() { return *this; }
168
169	u8 *base() const { return begin; }
170
171	private:
172	u8 *begin;
173	u8 *end;
174	uptr previous = 0;
175	};
176
177	class SLeb128Decoder {
178	public:
179	SLeb128Decoder(const u8 begin, const u8 end) : begin(begin), end(end) {}
180
181	bool operator==(const SLeb128Decoder &other) const {
182	return begin == other.begin;
183	}
184
185	bool operator!=(const SLeb128Decoder &other) const {
186	return begin != other.begin;
187	}
188
189	uptr operator*() {
190	sptr diff;
191	begin = DecodeSLEB128(begin, end, &diff);
192	previous += diff;
193	return previous;
194	}
195	SLeb128Decoder &operator++() { return *this; }
196
197	SLeb128Decoder operator++(int) { return *this; }
198
199	private:
200	const u8 *begin;
201	const u8 *end;
202	uptr previous = 0;
203	};
204
205	static u8 CompressDelta(const uptr from, const uptr from_end, u8 to,
206	u8 *to_end) {
207	SLeb128Encoder encoder(to, to_end);
208	for (; from != from_end; ++from, ++encoder) encoder = from;
209	return encoder.base();
210	}
211
212	static uptr UncompressDelta(const u8 from, const u8 from_end, uptr to,
213	uptr *to_end) {
214	SLeb128Decoder decoder(from, from_end);
215	SLeb128Decoder end(from_end, from_end);
216	for (; decoder != end; ++to, ++decoder) to = decoder;
217	CHECK_EQ(to, to_end);
218	return to;
219	}
220
221	static u8 CompressLzw(const uptr from, const uptr from_end, u8 to,
222	u8 *to_end) {
223	SLeb128Encoder encoder(to, to_end);
224	encoder = LzwEncode<uptr>(from, from_end, encoder);
225	return encoder.base();
226	}
227
228	static uptr UncompressLzw(const u8 from, const u8 from_end, uptr to,
229	uptr *to_end) {
230	SLeb128Decoder decoder(from, from_end);
231	SLeb128Decoder end(from_end, from_end);
232	to = LzwDecode<uptr>(decoder, end, to);
233	CHECK_EQ(to, to_end);
234	return to;
235	}
236
237	#if defined(_MSC_VER) && !defined(__clang__)
238	# pragma warning(push)
239	// Disable 'nonstandard extension used: zero-sized array in struct/union'.
240	# pragma warning(disable : 4200)
241	#endif
242	namespace {
243	struct PackedHeader {
244	uptr size;
245	StackStore::Compression type;
246	u8 data[];
247	};
248	} // namespace
249	#if defined(_MSC_VER) && !defined(__clang__)
250	# pragma warning(pop)
251	#endif
252
253	uptr StackStore::BlockInfo::GetOrUnpack(StackStore store) {
254	SpinMutexLock l(&mtx_);
255	switch (state) {
256	case State::Storing:
257	state = State::Unpacked;
258	FALLTHROUGH;
259	case State::Unpacked:
260	return Get();
261	case State::Packed:
262	break;
263	}
264
265	u8 ptr = reinterpret_cast<u8 >(Get());
266	CHECK_NE(nullptr, ptr);
267	const PackedHeader header = reinterpret_cast<const PackedHeader >(ptr);
268	CHECK_LE(header->size, kBlockSizeBytes);
269	CHECK_GE(header->size, sizeof(PackedHeader));
270
271	uptr packed_size_aligned = RoundUpTo(header->size, GetPageSizeCached());
272
273	uptr *unpacked =
274	reinterpret_cast<uptr *>(store->Map(kBlockSizeBytes, "StackStoreUnpack"));
275
276	uptr *unpacked_end;
277	switch (header->type) {
278	case Compression::Delta:
279	unpacked_end = UncompressDelta(header->data, ptr + header->size, unpacked,
280	unpacked + kBlockSizeFrames);
281	break;
282	case Compression::LZW:
283	unpacked_end = UncompressLzw(header->data, ptr + header->size, unpacked,
284	unpacked + kBlockSizeFrames);
285	break;
286	default:
287	UNREACHABLE("Unexpected type");
288	break;
289	}
290
291	CHECK_EQ(kBlockSizeFrames, unpacked_end - unpacked);
292
293	MprotectReadOnly(reinterpret_cast<uptr>(unpacked), kBlockSizeBytes);
294	atomic_store(&data_, reinterpret_cast<uptr>(unpacked), memory_order_release);
295	store->Unmap(ptr, packed_size_aligned);
296
297	state = State::Unpacked;
298	return Get();
299	}
300
301	uptr StackStore::BlockInfo::Pack(Compression type, StackStore *store) {
302	if (type == Compression::None)
303	return 0;
304
305	SpinMutexLock l(&mtx_);
306	switch (state) {
307	case State::Unpacked:
308	case State::Packed:
309	return 0;
310	case State::Storing:
311	break;
312	}
313
314	uptr *ptr = Get();
315	if (!ptr \|\| !Stored(0))
316	return 0;
317
318	u8 *packed =
319	reinterpret_cast<u8 *>(store->Map(kBlockSizeBytes, "StackStorePack"));
320	PackedHeader header = reinterpret_cast<PackedHeader >(packed);
321	u8 *alloc_end = packed + kBlockSizeBytes;
322
323	u8 *packed_end = nullptr;
324	switch (type) {
325	case Compression::Delta:
326	packed_end =
327	CompressDelta(ptr, ptr + kBlockSizeFrames, header->data, alloc_end);
328	break;
329	case Compression::LZW:
330	packed_end =
331	CompressLzw(ptr, ptr + kBlockSizeFrames, header->data, alloc_end);
332	break;
333	default:
334	UNREACHABLE("Unexpected type");
335	break;
336	}
337
338	header->type = type;
339	header->size = packed_end - packed;
340
341	VPrintf(1, "Packed block of %zu KiB to %zu KiB\n", kBlockSizeBytes >> 10,
342	header->size >> 10);
343
344	if (kBlockSizeBytes - header->size < kBlockSizeBytes / 8) {
345	VPrintf(1, "Undo and keep block unpacked\n");
346	MprotectReadOnly(reinterpret_cast<uptr>(ptr), kBlockSizeBytes);
347	store->Unmap(packed, kBlockSizeBytes);
348	state = State::Unpacked;
349	return 0;
350	}
351
352	uptr packed_size_aligned = RoundUpTo(header->size, GetPageSizeCached());
353	store->Unmap(packed + packed_size_aligned,
354	kBlockSizeBytes - packed_size_aligned);
355	MprotectReadOnly(reinterpret_cast<uptr>(packed), packed_size_aligned);
356
357	atomic_store(&data_, reinterpret_cast<uptr>(packed), memory_order_release);
358	store->Unmap(ptr, kBlockSizeBytes);
359
360	state = State::Packed;
361	return kBlockSizeBytes - packed_size_aligned;
362	}
363
364	void StackStore::BlockInfo::TestOnlyUnmap(StackStore *store) {
365	if (uptr *ptr = Get())
366	store->Unmap(ptr, kBlockSizeBytes);
367	}
368
369	bool StackStore::BlockInfo::Stored(uptr n) {
370	return n + atomic_fetch_add(&stored_, n, memory_order_release) ==
371	kBlockSizeFrames;
372	}
373
374	bool StackStore::BlockInfo::IsPacked() const {
375	SpinMutexLock l(&mtx_);
376	return state == State::Packed;
377	}
378
379	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_stack_store.h created+121

...	@@ -0,0 +1,121 @@
1	//===-- sanitizer_stack_store.h ---------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#ifndef SANITIZER_STACK_STORE_H
10	#define SANITIZER_STACK_STORE_H
11
12	#include "sanitizer_atomic.h"
13	#include "sanitizer_common.h"
14	#include "sanitizer_internal_defs.h"
15	#include "sanitizer_mutex.h"
16	#include "sanitizer_stacktrace.h"
17
18	namespace __sanitizer {
19
20	class StackStore {
21	static constexpr uptr kBlockSizeFrames = 0x100000;
22	static constexpr uptr kBlockCount = 0x1000;
23	static constexpr uptr kBlockSizeBytes = kBlockSizeFrames * sizeof(uptr);
24
25	public:
26	enum class Compression : u8 {
27	None = 0,
28	Delta,
29	LZW,
30	};
31
32	constexpr StackStore() = default;
33
34	using Id = u32; // Enough for 2^32 * sizeof(uptr) bytes of traces.
35	static_assert(u64(kBlockCount) * kBlockSizeFrames == 1ull << (sizeof(Id) * 8),
36	"");
37
38	Id Store(const StackTrace &trace,
39	uptr pack / number of blocks completed by this call */);
40	StackTrace Load(Id id);
41	uptr Allocated() const;
42
43	// Packs all blocks which don't expect any more writes. A block is going to be
44	// packed once. As soon trace from that block was requested, it will unpack
45	// and stay unpacked after that.
46	// Returns the number of released bytes.
47	uptr Pack(Compression type);
48
49	void LockAll();
50	void UnlockAll();
51
52	void TestOnlyUnmap();
53
54	private:
55	friend class StackStoreTest;
56	static constexpr uptr GetBlockIdx(uptr frame_idx) {
57	return frame_idx / kBlockSizeFrames;
58	}
59
60	static constexpr uptr GetInBlockIdx(uptr frame_idx) {
61	return frame_idx % kBlockSizeFrames;
62	}
63
64	static constexpr uptr IdToOffset(Id id) {
65	CHECK_NE(id, 0);
66	return id - 1; // Avoid zero as id.
67	}
68
69	static constexpr uptr OffsetToId(Id id) {
70	// This makes UINT32_MAX to 0 and it will be retrived as and empty stack.
71	// But this is not a problem as we will not be able to store anything after
72	// that anyway.
73	return id + 1; // Avoid zero as id.
74	}
75
76	uptr Alloc(uptr count, uptr idx, uptr *pack);
77
78	void Map(uptr size, const char mem_type);
79	void Unmap(void *addr, uptr size);
80
81	// Total number of allocated frames.
82	atomic_uintptr_t total_frames_ = {};
83
84	// Tracks total allocated memory in bytes.
85	atomic_uintptr_t allocated_ = {};
86
87	// Each block will hold pointer to exactly kBlockSizeFrames.
88	class BlockInfo {
89	atomic_uintptr_t data_;
90	// Counter to track store progress to know when we can Pack() the block.
91	atomic_uint32_t stored_;
92	// Protects alloc of new blocks.
93	mutable StaticSpinMutex mtx_;
94
95	enum class State : u8 {
96	Storing = 0,
97	Packed,
98	Unpacked,
99	};
100	State state SANITIZER_GUARDED_BY(mtx_);
101
102	uptr Create(StackStore store);
103
104	public:
105	uptr *Get() const;
106	uptr GetOrCreate(StackStore store);
107	uptr GetOrUnpack(StackStore store);
108	uptr Pack(Compression type, StackStore *store);
109	void TestOnlyUnmap(StackStore *store);
110	bool Stored(uptr n);
111	bool IsPacked() const;
112	void Lock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { mtx_.Lock(); }
113	void Unlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { mtx_.Unlock(); }
114	};
115
116	BlockInfo blocks_[kBlockCount] = {};
117	};
118
119	} // namespace __sanitizer
120
121	#endif // SANITIZER_STACK_STORE_H

lib/tsan/sanitizer_common/sanitizer_stackdepot.cpp+174-81

...	@@ -12,95 +12,203 @@	...	@@ -12,95 +12,203 @@
12		12
13	#include "sanitizer_stackdepot.h"	13	#include "sanitizer_stackdepot.h"
14		14
		15	#include "sanitizer_atomic.h"
15	#include "sanitizer_common.h"	16	#include "sanitizer_common.h"
16	#include "sanitizer_hash.h"	17	#include "sanitizer_hash.h"
		18	#include "sanitizer_mutex.h"
		19	#include "sanitizer_stack_store.h"
17	#include "sanitizer_stackdepotbase.h"	20	#include "sanitizer_stackdepotbase.h"
18		21
19	namespace __sanitizer {	22	namespace __sanitizer {
20		23
21	struct StackDepotNode {	24	struct StackDepotNode {
22	StackDepotNode *link;	25	using hash_type = u64;
23	u32 id;	26	hash_type stack_hash;
24	atomic_uint32_t hash_and_use_count; // hash_bits : 12; use_count : 20;	27	u32 link;
25	u32 size;	28	StackStore::Id store_id;
26	u32 tag;
27	uptr stack[1]; // [size]
28		29
29	static const u32 kTabSizeLog = SANITIZER_ANDROID ? 16 : 20;	30	static const u32 kTabSizeLog = SANITIZER_ANDROID ? 16 : 20;
30	// Lower kTabSizeLog bits are equal for all items in one bucket.
31	// We use these bits to store the per-stack use counter.
32	static const u32 kUseCountBits = kTabSizeLog;
33	static const u32 kMaxUseCount = 1 << kUseCountBits;
34	static const u32 kUseCountMask = (1 << kUseCountBits) - 1;
35	static const u32 kHashMask = ~kUseCountMask;
36		31
37	typedef StackTrace args_type;	32	typedef StackTrace args_type;
38	bool eq(u32 hash, const args_type &args) const {	33	bool eq(hash_type hash, const args_type &args) const {
39	u32 hash_bits =	34	return hash == stack_hash;
40	atomic_load(&hash_and_use_count, memory_order_relaxed) & kHashMask;
41	if ((hash & kHashMask) != hash_bits \|\| args.size != size \|\| args.tag != tag)
42	return false;
43	uptr i = 0;
44	for (; i < size; i++) {
45	if (stack[i] != args.trace[i]) return false;
46	}
47	return true;
48	}
49	static uptr storage_size(const args_type &args) {
50	return sizeof(StackDepotNode) + (args.size - 1) * sizeof(uptr);
51	}	35	}
52	static u32 hash(const args_type &args) {	36	static uptr allocated();
53	MurMur2HashBuilder H(args.size * sizeof(uptr));	37	static hash_type hash(const args_type &args) {
		38	MurMur2Hash64Builder H(args.size * sizeof(uptr));
54	for (uptr i = 0; i < args.size; i++) H.add(args.trace[i]);	39	for (uptr i = 0; i < args.size; i++) H.add(args.trace[i]);
		40	H.add(args.tag);
55	return H.get();	41	return H.get();
56	}	42	}
57	static bool is_valid(const args_type &args) {	43	static bool is_valid(const args_type &args) {
58	return args.size > 0 && args.trace;	44	return args.size > 0 && args.trace;
59	}	45	}
60	void store(const args_type &args, u32 hash) {	46	void store(u32 id, const args_type &args, hash_type hash);
61	atomic_store(&hash_and_use_count, hash & kHashMask, memory_order_relaxed);	47	args_type load(u32 id) const;
62	size = args.size;	48	static StackDepotHandle get_handle(u32 id);
63	tag = args.tag;
64	internal_memcpy(stack, args.trace, size * sizeof(uptr));
65	}
66	args_type load() const {
67	return args_type(&stack[0], size, tag);
68	}
69	StackDepotHandle get_handle() { return StackDepotHandle(this); }
70		49
71	typedef StackDepotHandle handle_type;	50	typedef StackDepotHandle handle_type;
72	};	51	};
73		52
74	COMPILER_CHECK(StackDepotNode::kMaxUseCount == (u32)kStackDepotMaxUseCount);	53	static StackStore stackStore;
75
76	u32 StackDepotHandle::id() { return node_->id; }
77	int StackDepotHandle::use_count() {
78	return atomic_load(&node_->hash_and_use_count, memory_order_relaxed) &
79	StackDepotNode::kUseCountMask;
80	}
81	void StackDepotHandle::inc_use_count_unsafe() {
82	u32 prev =
83	atomic_fetch_add(&node_->hash_and_use_count, 1, memory_order_relaxed) &
84	StackDepotNode::kUseCountMask;
85	CHECK_LT(prev + 1, StackDepotNode::kMaxUseCount);
86	}
87		54
88	// FIXME(dvyukov): this single reserved bit is used in TSan.	55	// FIXME(dvyukov): this single reserved bit is used in TSan.
89	typedef StackDepotBase<StackDepotNode, 1, StackDepotNode::kTabSizeLog>	56	typedef StackDepotBase<StackDepotNode, 1, StackDepotNode::kTabSizeLog>
90	StackDepot;	57	StackDepot;
91	static StackDepot theDepot;	58	static StackDepot theDepot;
		59	// Keep mutable data out of frequently access nodes to improve caching
		60	// efficiency.
		61	static TwoLevelMap<atomic_uint32_t, StackDepot::kNodesSize1,
		62	StackDepot::kNodesSize2>
		63	useCounts;
		64
		65	int StackDepotHandle::use_count() const {
		66	return atomic_load_relaxed(&useCounts[id_]);
		67	}
		68
		69	void StackDepotHandle::inc_use_count_unsafe() {
		70	atomic_fetch_add(&useCounts[id_], 1, memory_order_relaxed);
		71	}
		72
		73	uptr StackDepotNode::allocated() {
		74	return stackStore.Allocated() + useCounts.MemoryUsage();
		75	}
		76
		77	static void CompressStackStore() {
		78	u64 start = Verbosity() >= 1 ? MonotonicNanoTime() : 0;
		79	uptr diff = stackStore.Pack(static_cast<StackStore::Compression>(
		80	Abs(common_flags()->compress_stack_depot)));
		81	if (!diff)
		82	return;
		83	if (Verbosity() >= 1) {
		84	u64 finish = MonotonicNanoTime();
		85	uptr total_before = theDepot.GetStats().allocated + diff;
		86	VPrintf(1, "%s: StackDepot released %zu KiB out of %zu KiB in %llu ms\n",
		87	SanitizerToolName, diff >> 10, total_before >> 10,
		88	(finish - start) / 1000000);
		89	}
		90	}
		91
		92	namespace {
		93
		94	class CompressThread {
		95	public:
		96	constexpr CompressThread() = default;
		97	void NewWorkNotify();
		98	void Stop();
		99	void LockAndStop() SANITIZER_NO_THREAD_SAFETY_ANALYSIS;
		100	void Unlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS;
		101
		102	private:
		103	enum class State {
		104	NotStarted = 0,
		105	Started,
		106	Failed,
		107	Stopped,
		108	};
		109
		110	void Run();
		111
		112	bool WaitForWork() {
		113	semaphore_.Wait();
		114	return atomic_load(&run_, memory_order_acquire);
		115	}
		116
		117	Semaphore semaphore_ = {};
		118	StaticSpinMutex mutex_ = {};
		119	State state_ SANITIZER_GUARDED_BY(mutex_) = State::NotStarted;
		120	void *thread_ SANITIZER_GUARDED_BY(mutex_) = nullptr;
		121	atomic_uint8_t run_ = {};
		122	};
		123
		124	static CompressThread compress_thread;
		125
		126	void CompressThread::NewWorkNotify() {
		127	int compress = common_flags()->compress_stack_depot;
		128	if (!compress)
		129	return;
		130	if (compress > 0 /* for testing or debugging */) {
		131	SpinMutexLock l(&mutex_);
		132	if (state_ == State::NotStarted) {
		133	atomic_store(&run_, 1, memory_order_release);
		134	CHECK_EQ(nullptr, thread_);
		135	thread_ = internal_start_thread(
		136	[](void arg) -> void {
		137	reinterpret_cast<CompressThread *>(arg)->Run();
		138	return nullptr;
		139	},
		140	this);
		141	state_ = thread_ ? State::Started : State::Failed;
		142	}
		143	if (state_ == State::Started) {
		144	semaphore_.Post();
		145	return;
		146	}
		147	}
		148	CompressStackStore();
		149	}
		150
		151	void CompressThread::Run() {
		152	VPrintf(1, "%s: StackDepot compression thread started\n", SanitizerToolName);
		153	while (WaitForWork()) CompressStackStore();
		154	VPrintf(1, "%s: StackDepot compression thread stopped\n", SanitizerToolName);
		155	}
		156
		157	void CompressThread::Stop() {
		158	void *t = nullptr;
		159	{
		160	SpinMutexLock l(&mutex_);
		161	if (state_ != State::Started)
		162	return;
		163	state_ = State::Stopped;
		164	CHECK_NE(nullptr, thread_);
		165	t = thread_;
		166	thread_ = nullptr;
		167	}
		168	atomic_store(&run_, 0, memory_order_release);
		169	semaphore_.Post();
		170	internal_join_thread(t);
		171	}
		172
		173	void CompressThread::LockAndStop() {
		174	mutex_.Lock();
		175	if (state_ != State::Started)
		176	return;
		177	CHECK_NE(nullptr, thread_);
		178
		179	atomic_store(&run_, 0, memory_order_release);
		180	semaphore_.Post();
		181	internal_join_thread(thread_);
		182	// Allow to restart after Unlock() if needed.
		183	state_ = State::NotStarted;
		184	thread_ = nullptr;
		185	}
		186
		187	void CompressThread::Unlock() { mutex_.Unlock(); }
		188
		189	} // namespace
92		190
93	StackDepotStats *StackDepotGetStats() {	191	void StackDepotNode::store(u32 id, const args_type &args, hash_type hash) {
94	return theDepot.GetStats();	192	stack_hash = hash;
		193	uptr pack = 0;
		194	store_id = stackStore.Store(args, &pack);
		195	if (LIKELY(!pack))
		196	return;
		197	compress_thread.NewWorkNotify();
95	}	198	}
96		199
97	u32 StackDepotPut(StackTrace stack) {	200	StackDepotNode::args_type StackDepotNode::load(u32 id) const {
98	StackDepotHandle h = theDepot.Put(stack);	201	if (!store_id)
99	return h.valid() ? h.id() : 0;	202	return {};
		203	return stackStore.Load(store_id);
100	}	204	}
101		205
		206	StackDepotStats StackDepotGetStats() { return theDepot.GetStats(); }
		207
		208	u32 StackDepotPut(StackTrace stack) { return theDepot.Put(stack); }
		209
102	StackDepotHandle StackDepotPut_WithHandle(StackTrace stack) {	210	StackDepotHandle StackDepotPut_WithHandle(StackTrace stack) {
103	return theDepot.Put(stack);	211	return StackDepotNode::get_handle(theDepot.Put(stack));
104	}	212	}
105		213
106	StackTrace StackDepotGet(u32 id) {	214	StackTrace StackDepotGet(u32 id) {
...	@@ -109,9 +217,13 @@ StackTrace StackDepotGet(u32 id) {	...	@@ -109,9 +217,13 @@ StackTrace StackDepotGet(u32 id) {
109		217
110	void StackDepotLockAll() {	218	void StackDepotLockAll() {
111	theDepot.LockAll();	219	theDepot.LockAll();
		220	compress_thread.LockAndStop();
		221	stackStore.LockAll();
112	}	222	}
113		223
114	void StackDepotUnlockAll() {	224	void StackDepotUnlockAll() {
		225	stackStore.UnlockAll();
		226	compress_thread.Unlock();
115	theDepot.UnlockAll();	227	theDepot.UnlockAll();
116	}	228	}
117		229
...	@@ -121,34 +233,15 @@ void StackDepotPrintAll() {	...	@@ -121,34 +233,15 @@ void StackDepotPrintAll() {
121	#endif	233	#endif
122	}	234	}
123		235
124	bool StackDepotReverseMap::IdDescPair::IdComparator(	236	void StackDepotStopBackgroundThread() { compress_thread.Stop(); }
125	const StackDepotReverseMap::IdDescPair &a,
126	const StackDepotReverseMap::IdDescPair &b) {
127	return a.id < b.id;
128	}
129		237
130	StackDepotReverseMap::StackDepotReverseMap() {	238	StackDepotHandle StackDepotNode::get_handle(u32 id) {
131	map_.reserve(StackDepotGetStats()->n_uniq_ids + 100);	239	return StackDepotHandle(&theDepot.nodes[id], id);
132	for (int idx = 0; idx < StackDepot::kTabSize; idx++) {
133	atomic_uintptr_t *p = &theDepot.tab[idx];
134	uptr v = atomic_load(p, memory_order_consume);
135	StackDepotNode s = (StackDepotNode)(v & ~1);
136	for (; s; s = s->link) {
137	IdDescPair pair = {s->id, s};
138	map_.push_back(pair);
139	}
140	}
141	Sort(map_.data(), map_.size(), &IdDescPair::IdComparator);
142	}	240	}
143		241
144	StackTrace StackDepotReverseMap::Get(u32 id) {	242	void StackDepotTestOnlyUnmap() {
145	if (!map_.size())	243	theDepot.TestOnlyUnmap();
146	return StackTrace();	244	stackStore.TestOnlyUnmap();
147	IdDescPair pair = {id, nullptr};
148	uptr idx = InternalLowerBound(map_, pair, IdDescPair::IdComparator);
149	if (idx > map_.size() \|\| map_[idx].id != id)
150	return StackTrace();
151	return map_[idx].desc->load();
152	}	245	}
153		246
154	} // namespace __sanitizer	247	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_stackdepot.h+9-30

 @@ -22,18 +22,18 @@ namespace __sanitizer {
 // StackDepot efficiently stores huge amounts of stack traces.
 struct StackDepotNode;
 struct StackDepotHandle {
-  StackDepotNode *node_;
+  StackDepotNode *node_ = nullptr;
-  StackDepotHandle() : node_(nullptr) {}
+  u32 id_ = 0;
-  explicit StackDepotHandle(StackDepotNode *node) : node_(node) {}
+  StackDepotHandle(StackDepotNode *node, u32 id) : node_(node), id_(id) {}
-  bool valid() { return node_; }
+  bool valid() const { return node_; }
-  u32 id();
+  u32 id() const { return id_; }
-  int use_count();
+  int use_count() const;
   void inc_use_count_unsafe();
 };
 const int kStackDepotMaxUseCount = 1U << (SANITIZER_ANDROID ? 16 : 20);
-StackDepotStats *StackDepotGetStats();
+StackDepotStats StackDepotGetStats();
 u32 StackDepotPut(StackTrace stack);
 StackDepotHandle StackDepotPut_WithHandle(StackTrace stack);
 // Retrieves a stored stack trace by the id.
 @@ -42,30 +42,9 @@ StackTrace StackDepotGet(u32 id);
 void StackDepotLockAll();
 void StackDepotUnlockAll();
 void StackDepotPrintAll();
+void StackDepotStopBackgroundThread();
-// Instantiating this class creates a snapshot of StackDepot which can be
+void StackDepotTestOnlyUnmap();
-// efficiently queried with StackDepotGet(). You can use it concurrently with
-// StackDepot, but the snapshot is only guaranteed to contain those stack traces
-// which were stored before it was instantiated.
-class StackDepotReverseMap {
- public:
-  StackDepotReverseMap();
-  StackTrace Get(u32 id);
- private:
-  struct IdDescPair {
-    u32 id;
-    StackDepotNode *desc;
-    static bool IdComparator(const IdDescPair &a, const IdDescPair &b);
-  };
-  InternalMmapVector<IdDescPair> map_;
-  // Disallow evil constructors.
-  StackDepotReverseMap(const StackDepotReverseMap&);
-  void operator=(const StackDepotReverseMap&);
-};
 } // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_stackdepotbase.h+86-87

 @@ -16,71 +16,87 @@
 #include <stdio.h>
 #include "sanitizer_atomic.h"
+#include "sanitizer_flat_map.h"
 #include "sanitizer_internal_defs.h"
 #include "sanitizer_mutex.h"
-#include "sanitizer_persistent_allocator.h"
 namespace __sanitizer {
 template <class Node, int kReservedBits, int kTabSizeLog>
 class StackDepotBase {
+  static constexpr u32 kIdSizeLog =
+      sizeof(u32) * 8 - Max(kReservedBits, 1 /* At least 1 bit for locking. */);
+  static constexpr u32 kNodesSize1Log = kIdSizeLog / 2;
+  static constexpr u32 kNodesSize2Log = kIdSizeLog - kNodesSize1Log;
+  static constexpr int kTabSize = 1 << kTabSizeLog;  // Hash table size.
+  static constexpr u32 kUnlockMask = (1ull << kIdSizeLog) - 1;
+  static constexpr u32 kLockMask = ~kUnlockMask;
  public:
   typedef typename Node::args_type args_type;
   typedef typename Node::handle_type handle_type;
+  typedef typename Node::hash_type hash_type;
+  static constexpr u64 kNodesSize1 = 1ull << kNodesSize1Log;
+  static constexpr u64 kNodesSize2 = 1ull << kNodesSize2Log;
   // Maps stack trace to an unique id.
-  handle_type Put(args_type args, bool *inserted = nullptr);
+  u32 Put(args_type args, bool *inserted = nullptr);
   // Retrieves a stored stack trace by the id.
   args_type Get(u32 id);
-  StackDepotStats *GetStats() { return &stats; }
+  StackDepotStats GetStats() const {
+    return {
+        atomic_load_relaxed(&n_uniq_ids),
+        nodes.MemoryUsage() + Node::allocated(),
+    };
+  }
   void LockAll();
   void UnlockAll();
   void PrintAll();
- private:
+  void TestOnlyUnmap() {
-  static Node *find(Node *s, args_type args, u32 hash);
+    nodes.TestOnlyUnmap();
-  static Node *lock(atomic_uintptr_t *p);
+    internal_memset(this, 0, sizeof(*this));
-  static void unlock(atomic_uintptr_t *p, Node *s);
+  }
-  static const int kTabSize = 1 << kTabSizeLog;  // Hash table size.
+ private:
-  static const int kPartBits = 8;
+  friend Node;
-  static const int kPartShift = sizeof(u32) * 8 - kPartBits - kReservedBits;
+  u32 find(u32 s, args_type args, hash_type hash) const;
-  static const int kPartCount =
+  static u32 lock(atomic_uint32_t *p);
-      1 << kPartBits;  // Number of subparts in the table.
+  static void unlock(atomic_uint32_t *p, u32 s);
-  static const int kPartSize = kTabSize / kPartCount;
+  atomic_uint32_t tab[kTabSize];  // Hash table of Node's.
-  static const int kMaxId = 1 << kPartShift;
-  atomic_uintptr_t tab[kTabSize];   // Hash table of Node's.
+  atomic_uint32_t n_uniq_ids;
-  atomic_uint32_t seq[kPartCount];  // Unique id generators.
-  StackDepotStats stats;
+  TwoLevelMap<Node, kNodesSize1, kNodesSize2> nodes;
   friend class StackDepotReverseMap;
 };
 template <class Node, int kReservedBits, int kTabSizeLog>
-Node *StackDepotBase<Node, kReservedBits, kTabSizeLog>::find(Node *s,
+u32 StackDepotBase<Node, kReservedBits, kTabSizeLog>::find(
-                                                             args_type args,
+    u32 s, args_type args, hash_type hash) const {
-                                                             u32 hash) {
   // Searches linked list s for the stack, returns its id.
-  for (; s; s = s->link) {
+  for (; s;) {
-    if (s->eq(hash, args)) {
+    const Node &node = nodes[s];
+    if (node.eq(hash, args))
       return s;
-    }
+    s = node.link;
   }
-  return nullptr;
+  return 0;
 }
 template <class Node, int kReservedBits, int kTabSizeLog>
-Node *StackDepotBase<Node, kReservedBits, kTabSizeLog>::lock(
+u32 StackDepotBase<Node, kReservedBits, kTabSizeLog>::lock(atomic_uint32_t *p) {
-    atomic_uintptr_t *p) {
   // Uses the pointer lsb as mutex.
   for (int i = 0;; i++) {
-    uptr cmp = atomic_load(p, memory_order_relaxed);
+    u32 cmp = atomic_load(p, memory_order_relaxed);
-    if ((cmp & 1) == 0 &&
+    if ((cmp & kLockMask) == 0 &&
-        atomic_compare_exchange_weak(p, &cmp, cmp | 1, memory_order_acquire))
+        atomic_compare_exchange_weak(p, &cmp, cmp | kLockMask,
-      return (Node *)cmp;
+                                     memory_order_acquire))
+      return cmp;
     if (i < 10)
       proc_yield(10);
     else
 @@ -90,73 +106,57 @@ Node *StackDepotBase<Node, kReservedBits, kTabSizeLog>::lock(
 template <class Node, int kReservedBits, int kTabSizeLog>
 void StackDepotBase<Node, kReservedBits, kTabSizeLog>::unlock(
-    atomic_uintptr_t *p, Node *s) {
+    atomic_uint32_t *p, u32 s) {
-  DCHECK_EQ((uptr)s & 1, 0);
+  DCHECK_EQ(s & kLockMask, 0);
-  atomic_store(p, (uptr)s, memory_order_release);
+  atomic_store(p, s, memory_order_release);
 }
 template <class Node, int kReservedBits, int kTabSizeLog>
-typename StackDepotBase<Node, kReservedBits, kTabSizeLog>::handle_type
+u32 StackDepotBase<Node, kReservedBits, kTabSizeLog>::Put(args_type args,
-StackDepotBase<Node, kReservedBits, kTabSizeLog>::Put(args_type args,
+                                                          bool *inserted) {
-                                                      bool *inserted) {
+  if (inserted)
-  if (inserted) *inserted = false;
+    *inserted = false;
-  if (!Node::is_valid(args)) return handle_type();
+  if (!LIKELY(Node::is_valid(args)))
-  uptr h = Node::hash(args);
+    return 0;
-  atomic_uintptr_t *p = &tab[h % kTabSize];
+  hash_type h = Node::hash(args);
-  uptr v = atomic_load(p, memory_order_consume);
+  atomic_uint32_t *p = &tab[h % kTabSize];
-  Node *s = (Node *)(v & ~1);
+  u32 v = atomic_load(p, memory_order_consume);
+  u32 s = v & kUnlockMask;
   // First, try to find the existing stack.
-  Node *node = find(s, args, h);
+  u32 node = find(s, args, h);
-  if (node) return node->get_handle();
+  if (LIKELY(node))
+    return node;
   // If failed, lock, retry and insert new.
-  Node *s2 = lock(p);
+  u32 s2 = lock(p);
   if (s2 != s) {
     node = find(s2, args, h);
     if (node) {
       unlock(p, s2);
-      return node->get_handle();
+      return node;
     }
   }
-  uptr part = (h % kTabSize) / kPartSize;
+  s = atomic_fetch_add(&n_uniq_ids, 1, memory_order_relaxed) + 1;
-  u32 id = atomic_fetch_add(&seq[part], 1, memory_order_relaxed) + 1;
+  CHECK_EQ(s & kUnlockMask, s);
-  stats.n_uniq_ids++;
+  CHECK_EQ(s & (((u32)-1) >> kReservedBits), s);
-  CHECK_LT(id, kMaxId);
+  Node &new_node = nodes[s];
-  id |= part << kPartShift;
+  new_node.store(s, args, h);
-  CHECK_NE(id, 0);
+  new_node.link = s2;
-  CHECK_EQ(id & (((u32)-1) >> kReservedBits), id);
-  uptr memsz = Node::storage_size(args);
-  s = (Node *)PersistentAlloc(memsz);
-  stats.allocated += memsz;
-  s->id = id;
-  s->store(args, h);
-  s->link = s2;
   unlock(p, s);
   if (inserted) *inserted = true;
-  return s->get_handle();
+  return s;
 }
 template <class Node, int kReservedBits, int kTabSizeLog>
 typename StackDepotBase<Node, kReservedBits, kTabSizeLog>::args_type
 StackDepotBase<Node, kReservedBits, kTabSizeLog>::Get(u32 id) {
-  if (id == 0) {
+  if (id == 0)
     return args_type();
-  }
   CHECK_EQ(id & (((u32)-1) >> kReservedBits), id);
-  // High kPartBits contain part id, so we need to scan at most kPartSize lists.
+  if (!nodes.contains(id))
-  uptr part = id >> kPartShift;
+    return args_type();
-  for (int i = 0; i != kPartSize; i++) {
+  const Node &node = nodes[id];
-    uptr idx = part * kPartSize + i;
+  return node.load(id);
-    CHECK_LT(idx, kTabSize);
-    atomic_uintptr_t *p = &tab[idx];
-    uptr v = atomic_load(p, memory_order_consume);
-    Node *s = (Node *)(v & ~1);
-    for (; s; s = s->link) {
-      if (s->id == id) {
-        return s->load();
-      }
-    }
-  }
-  return args_type();
 }
 template <class Node, int kReservedBits, int kTabSizeLog>
 @@ -169,24 +169,23 @@ void StackDepotBase<Node, kReservedBits, kTabSizeLog>::LockAll() {
 template <class Node, int kReservedBits, int kTabSizeLog>
 void StackDepotBase<Node, kReservedBits, kTabSizeLog>::UnlockAll() {
   for (int i = 0; i < kTabSize; ++i) {
-    atomic_uintptr_t *p = &tab[i];
+    atomic_uint32_t *p = &tab[i];
     uptr s = atomic_load(p, memory_order_relaxed);
-    unlock(p, (Node *)(s & ~1UL));
+    unlock(p, s & kUnlockMask);
   }
 }
 template <class Node, int kReservedBits, int kTabSizeLog>
 void StackDepotBase<Node, kReservedBits, kTabSizeLog>::PrintAll() {
   for (int i = 0; i < kTabSize; ++i) {
-    atomic_uintptr_t *p = &tab[i];
+    atomic_uint32_t *p = &tab[i];
-    lock(p);
+    u32 s = atomic_load(p, memory_order_consume) & kUnlockMask;
-    uptr v = atomic_load(p, memory_order_relaxed);
+    for (; s;) {
-    Node *s = (Node *)(v & ~1UL);
+      const Node &node = nodes[s];
-    for (; s; s = s->link) {
+      Printf("Stack for id %u:\n", s);
-      Printf("Stack for id %u:\n", s->id);
+      node.load(s).Print();
-      s->load().Print();
+      s = node.link;
     }
-    unlock(p, s);
   }
 }

lib/tsan/sanitizer_common/sanitizer_stacktrace.cpp+9-7

...	@@ -20,10 +20,10 @@	...	@@ -20,10 +20,10 @@
20	namespace __sanitizer {	20	namespace __sanitizer {
21		21
22	uptr StackTrace::GetNextInstructionPc(uptr pc) {	22	uptr StackTrace::GetNextInstructionPc(uptr pc) {
23	#if defined(__sparc__) \|\| defined(__mips__)	23	#if defined(__aarch64__)
		24	return STRIP_PAC_PC((void *)pc) + 4;
		25	#elif defined(__sparc__) \|\| defined(__mips__)
24	return pc + 8;	26	return pc + 8;
25	#elif defined(__powerpc__) \|\| defined(__arm__) \|\| defined(__aarch64__)
26	return pc + 4;
27	#elif SANITIZER_RISCV64	27	#elif SANITIZER_RISCV64
28	// Current check order is 4 -> 2 -> 6 -> 8	28	// Current check order is 4 -> 2 -> 6 -> 8
29	u8 InsnByte = (u8 )(pc);	29	u8 InsnByte = (u8 )(pc);
...	@@ -46,8 +46,10 @@ uptr StackTrace::GetNextInstructionPc(uptr pc) {	...	@@ -46,8 +46,10 @@ uptr StackTrace::GetNextInstructionPc(uptr pc) {
46	}	46	}
47	// bail-out if could not figure out the instruction size	47	// bail-out if could not figure out the instruction size
48	return 0;	48	return 0;
49	#else	49	#elif SANITIZER_S390 \|\| SANITIZER_I386 \|\| SANITIZER_X32 \|\| SANITIZER_X64
50	return pc + 1;	50	return pc + 1;
		51	#else
		52	return pc + 4;
51	#endif	53	#endif
52	}	54	}
53		55
...	@@ -64,7 +66,7 @@ void BufferedStackTrace::Init(const uptr *pcs, uptr cnt, uptr extra_top_pc) {	...	@@ -64,7 +66,7 @@ void BufferedStackTrace::Init(const uptr *pcs, uptr cnt, uptr extra_top_pc) {
64	top_frame_bp = 0;	66	top_frame_bp = 0;
65	}	67	}
66		68
67	// Sparc implemention is in its own file.	69	// Sparc implementation is in its own file.
68	#if !defined(__sparc__)	70	#if !defined(__sparc__)
69		71
70	// In GCC on ARM bp points to saved lr, not fp, so we should check the next	72	// In GCC on ARM bp points to saved lr, not fp, so we should check the next
...	@@ -119,7 +121,7 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,	...	@@ -119,7 +121,7 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
119	uhwptr pc1 = caller_frame[2];	121	uhwptr pc1 = caller_frame[2];
120	#elif defined(__s390__)	122	#elif defined(__s390__)
121	uhwptr pc1 = frame[14];	123	uhwptr pc1 = frame[14];
122	#elif defined(__riscv)	124	#elif defined(__loongarch__) \|\| defined(__riscv)
123	// frame[-1] contains the return address	125	// frame[-1] contains the return address
124	uhwptr pc1 = frame[-1];	126	uhwptr pc1 = frame[-1];
125	#else	127	#else
...	@@ -134,7 +136,7 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,	...	@@ -134,7 +136,7 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
134	trace_buffer[size++] = (uptr) pc1;	136	trace_buffer[size++] = (uptr) pc1;
135	}	137	}
136	bottom = (uptr)frame;	138	bottom = (uptr)frame;
137	#if defined(__riscv)	139	#if defined(__loongarch__) \|\| defined(__riscv)
138	// frame[-2] contain fp of the previous frame	140	// frame[-2] contain fp of the previous frame
139	uptr new_bp = (uptr)frame[-2];	141	uptr new_bp = (uptr)frame[-2];
140	#else	142	#else

lib/tsan/sanitizer_common/sanitizer_stacktrace.h+9-10

 @@ -20,7 +20,7 @@ namespace __sanitizer {
 struct BufferedStackTrace;
-static const u32 kStackTraceMax = 256;
+static const u32 kStackTraceMax = 255;
 #if SANITIZER_LINUX && defined(__mips__)
 # define SANITIZER_CAN_FAST_UNWIND 0
 @@ -33,7 +33,7 @@ static const u32 kStackTraceMax = 256;
 // Fast unwind is the only option on Mac for now; we will need to
 // revisit this macro when slow unwind works on Mac, see
 // https://github.com/google/sanitizers/issues/137
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
 #  define SANITIZER_CAN_SLOW_UNWIND 0
 #else
 # define SANITIZER_CAN_SLOW_UNWIND 1
 @@ -88,21 +88,20 @@ uptr StackTrace::GetPreviousInstructionPc(uptr pc) {
   // so we return (pc-2) in that case in order to be safe.
   // For A32 mode we return (pc-4) because all instructions are 32 bit long.
   return (pc - 3) & (~1);
-#elif defined(__powerpc__) || defined(__powerpc64__) || defined(__aarch64__)
-  // PCs are always 4 byte aligned.
-  return pc - 4;
 #elif defined(__sparc__) || defined(__mips__)
   return pc - 8;
 #elif SANITIZER_RISCV64
-  // RV-64 has variable instruciton length...
+  // RV-64 has variable instruction length...
   // C extentions gives us 2-byte instructoins
   // RV-64 has 4-byte instructions
-  // + RISCV architecture allows instructions up to 8 bytes
+  // + RISC-V architecture allows instructions up to 8 bytes
   // It seems difficult to figure out the exact instruction length -
   // pc - 2 seems like a safe option for the purposes of stack tracing
   return pc - 2;
-#else
+#elif SANITIZER_S390 || SANITIZER_I386 || SANITIZER_X32 || SANITIZER_X64
   return pc - 1;
+#else
+  return pc - 4;
 #endif
 }
 @@ -209,11 +208,11 @@ static inline bool IsValidFrame(uptr frame, uptr stack_top, uptr stack_bottom) {
 // StackTrace::GetCurrentPc() faster.
 #if defined(__x86_64__)
 #  define GET_CURRENT_PC()                \
-    ({                                    \
+    (__extension__({                      \
       uptr pc;                            \
       asm("lea 0(%%rip), %0" : "=r"(pc)); \
       pc;                                 \
-    })
+    }))
 #else
 #  define GET_CURRENT_PC() StackTrace::GetCurrentPc()
 #endif

lib/tsan/sanitizer_common/sanitizer_stacktrace_libcdep.cpp+8-7

...	@@ -64,7 +64,7 @@ class StackTraceTextPrinter {	...	@@ -64,7 +64,7 @@ class StackTraceTextPrinter {
64	if (dedup_token_->length())	64	if (dedup_token_->length())
65	dedup_token_->append("--");	65	dedup_token_->append("--");
66	if (stack->info.function != nullptr)	66	if (stack->info.function != nullptr)
67	dedup_token_->append(stack->info.function);	67	dedup_token_->append("%s", stack->info.function);
68	}	68	}
69	}	69	}
70		70
...	@@ -166,8 +166,8 @@ void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context,	...	@@ -166,8 +166,8 @@ void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context,
166	UnwindFast(pc, bp, stack_top, stack_bottom, max_depth);	166	UnwindFast(pc, bp, stack_top, stack_bottom, max_depth);
167	}	167	}
168		168
169	static int GetModuleAndOffsetForPc(uptr pc, char *module_name,	169	int GetModuleAndOffsetForPc(uptr pc, char *module_name, uptr module_name_len,
170	uptr module_name_len, uptr *pc_offset) {	170	uptr *pc_offset) {
171	const char *found_module_name = nullptr;	171	const char *found_module_name = nullptr;
172	bool ok = Symbolizer::GetOrInit()->GetModuleNameAndOffsetForPC(	172	bool ok = Symbolizer::GetOrInit()->GetModuleNameAndOffsetForPC(
173	pc, &found_module_name, pc_offset);	173	pc, &found_module_name, pc_offset);
...	@@ -216,10 +216,11 @@ void __sanitizer_symbolize_global(uptr data_addr, const char *fmt,	...	@@ -216,10 +216,11 @@ void __sanitizer_symbolize_global(uptr data_addr, const char *fmt,
216	}	216	}
217		217
218	SANITIZER_INTERFACE_ATTRIBUTE	218	SANITIZER_INTERFACE_ATTRIBUTE
219	int __sanitizer_get_module_and_offset_for_pc(uptr pc, char *module_name,	219	int __sanitizer_get_module_and_offset_for_pc(void pc, char module_name,
220	uptr module_name_len,	220	uptr module_name_len,
221	uptr *pc_offset) {	221	void **pc_offset) {
222	return __sanitizer::GetModuleAndOffsetForPc(pc, module_name, module_name_len,	222	return __sanitizer::GetModuleAndOffsetForPc(
223	pc_offset);	223	reinterpret_cast<uptr>(pc), module_name, module_name_len,
		224	reinterpret_cast<uptr *>(pc_offset));
224	}	225	}
225	} // extern "C"	226	} // extern "C"

lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.cpp+66-21

...	@@ -11,25 +11,47 @@	...	@@ -11,25 +11,47 @@
11	//===----------------------------------------------------------------------===//	11	//===----------------------------------------------------------------------===//
12		12
13	#include "sanitizer_stacktrace_printer.h"	13	#include "sanitizer_stacktrace_printer.h"
		14
14	#include "sanitizer_file.h"	15	#include "sanitizer_file.h"
		16	#include "sanitizer_flags.h"
15	#include "sanitizer_fuchsia.h"	17	#include "sanitizer_fuchsia.h"
16		18
17	namespace __sanitizer {	19	namespace __sanitizer {
18		20
19	// sanitizer_symbolizer_markup.cpp implements these differently.	21	const char StripFunctionName(const char function) {
20	#if !SANITIZER_SYMBOLIZER_MARKUP	22	if (!common_flags()->demangle)
21		23	return function;
22	static const char StripFunctionName(const char function, const char *prefix) {	24	if (!function)
23	if (!function) return nullptr;	25	return nullptr;
24	if (!prefix) return function;	26	auto try_strip = [function](const char prefix) -> const char {
25	uptr prefix_len = internal_strlen(prefix);	27	const uptr prefix_len = internal_strlen(prefix);
26	if (0 == internal_strncmp(function, prefix, prefix_len))	28	if (!internal_strncmp(function, prefix, prefix_len))
27	return function + prefix_len;	29	return function + prefix_len;
		30	return nullptr;
		31	};
		32	if (SANITIZER_APPLE) {
		33	if (const char *s = try_strip("wrap_"))
		34	return s;
		35	} else if (SANITIZER_WINDOWS) {
		36	if (const char *s = try_strip("__asan_wrap_"))
		37	return s;
		38	} else {
		39	if (const char *s = try_strip("___interceptor_"))
		40	return s;
		41	if (const char *s = try_strip("__interceptor_"))
		42	return s;
		43	}
28	return function;	44	return function;
29	}	45	}
30		46
		47	// sanitizer_symbolizer_markup.cpp implements these differently.
		48	#if !SANITIZER_SYMBOLIZER_MARKUP
		49
31	static const char DemangleFunctionName(const char function) {	50	static const char DemangleFunctionName(const char function) {
32	if (!function) return nullptr;	51	if (!common_flags()->demangle)
		52	return function;
		53	if (!function)
		54	return nullptr;
33		55
34	// NetBSD uses indirection for old threading functions for historical reasons	56	// NetBSD uses indirection for old threading functions for historical reasons
35	// The mangled names are internal implementation detail and should not be	57	// The mangled names are internal implementation detail and should not be
...	@@ -104,11 +126,24 @@ static const char DemangleFunctionName(const char function) {	...	@@ -104,11 +126,24 @@ static const char DemangleFunctionName(const char function) {
104	return function;	126	return function;
105	}	127	}
106		128
		129	static void MaybeBuildIdToBuffer(const AddressInfo &info, bool PrefixSpace,
		130	InternalScopedString *buffer) {
		131	if (info.uuid_size) {
		132	if (PrefixSpace)
		133	buffer->append(" ");
		134	buffer->append("(BuildId: ");
		135	for (uptr i = 0; i < info.uuid_size; ++i) {
		136	buffer->append("%02x", info.uuid[i]);
		137	}
		138	buffer->append(")");
		139	}
		140	}
		141
107	static const char kDefaultFormat[] = " #%n %p %F %L";	142	static const char kDefaultFormat[] = " #%n %p %F %L";
108		143
109	void RenderFrame(InternalScopedString buffer, const char format, int frame_no,	144	void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
110	uptr address, const AddressInfo *info, bool vs_style,	145	uptr address, const AddressInfo *info, bool vs_style,
111	const char strip_path_prefix, const char strip_func_prefix) {	146	const char *strip_path_prefix) {
112	// info will be null in the case where symbolization is not needed for the	147	// info will be null in the case where symbolization is not needed for the
113	// given format. This ensures that the code below will get a hard failure	148	// given format. This ensures that the code below will get a hard failure
114	// rather than print incorrect information in case RenderNeedsSymbolization	149	// rather than print incorrect information in case RenderNeedsSymbolization
...	@@ -129,7 +164,7 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,	...	@@ -129,7 +164,7 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
129	break;	164	break;
130	// Frame number and all fields of AddressInfo structure.	165	// Frame number and all fields of AddressInfo structure.
131	case 'n':	166	case 'n':
132	buffer->append("%zu", frame_no);	167	buffer->append("%u", frame_no);
133	break;	168	break;
134	case 'p':	169	case 'p':
135	buffer->append("0x%zx", address);	170	buffer->append("0x%zx", address);
...	@@ -140,9 +175,12 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,	...	@@ -140,9 +175,12 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
140	case 'o':	175	case 'o':
141	buffer->append("0x%zx", info->module_offset);	176	buffer->append("0x%zx", info->module_offset);
142	break;	177	break;
		178	case 'b':
		179	MaybeBuildIdToBuffer(info, /PrefixSpace=*/false, buffer);
		180	break;
143	case 'f':	181	case 'f':
144	buffer->append("%s", DemangleFunctionName(StripFunctionName(	182	buffer->append("%s",
145	info->function, strip_func_prefix)));	183	DemangleFunctionName(StripFunctionName(info->function)));
146	break;	184	break;
147	case 'q':	185	case 'q':
148	buffer->append("0x%zx", info->function_offset != AddressInfo::kUnknown	186	buffer->append("0x%zx", info->function_offset != AddressInfo::kUnknown
...	@@ -162,8 +200,8 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,	...	@@ -162,8 +200,8 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
162	case 'F':	200	case 'F':
163	// Function name and offset, if file is unknown.	201	// Function name and offset, if file is unknown.
164	if (info->function) {	202	if (info->function) {
165	buffer->append("in %s", DemangleFunctionName(StripFunctionName(	203	buffer->append("in %s",
166	info->function, strip_func_prefix)));	204	DemangleFunctionName(StripFunctionName(info->function)));
167	if (!info->file && info->function_offset != AddressInfo::kUnknown)	205	if (!info->file && info->function_offset != AddressInfo::kUnknown)
168	buffer->append("+0x%zx", info->function_offset);	206	buffer->append("+0x%zx", info->function_offset);
169	}	207	}
...	@@ -181,6 +219,10 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,	...	@@ -181,6 +219,10 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
181	} else if (info->module) {	219	} else if (info->module) {
182	RenderModuleLocation(buffer, info->module, info->module_offset,	220	RenderModuleLocation(buffer, info->module, info->module_offset,
183	info->module_arch, strip_path_prefix);	221	info->module_arch, strip_path_prefix);
		222
		223	#if !SANITIZER_APPLE
		224	MaybeBuildIdToBuffer(info, /PrefixSpace=*/true, buffer);
		225	#endif
184	} else {	226	} else {
185	buffer->append("(<unknown module>)");	227	buffer->append("(<unknown module>)");
186	}	228	}
...	@@ -193,13 +235,16 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,	...	@@ -193,13 +235,16 @@ void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
193	// Always strip the module name for %M.	235	// Always strip the module name for %M.
194	RenderModuleLocation(buffer, StripModuleName(info->module),	236	RenderModuleLocation(buffer, StripModuleName(info->module),
195	info->module_offset, info->module_arch, "");	237	info->module_offset, info->module_arch, "");
		238	#if !SANITIZER_APPLE
		239	MaybeBuildIdToBuffer(info, /PrefixSpace=*/true, buffer);
		240	#endif
196	} else {	241	} else {
197	buffer->append("(%p)", (void *)address);	242	buffer->append("(%p)", (void *)address);
198	}	243	}
199	break;	244	break;
200	default:	245	default:
201	Report("Unsupported specifier in stack frame format: %c (0x%zx)!\n", *p,	246	Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p,
202	*p);	247	(void *)p);
203	Die();	248	Die();
204	}	249	}
205	}	250	}
...	@@ -244,14 +289,14 @@ void RenderData(InternalScopedString buffer, const char format,	...	@@ -244,14 +289,14 @@ void RenderData(InternalScopedString buffer, const char format,
244	buffer->append("%s", StripPathPrefix(DI->file, strip_path_prefix));	289	buffer->append("%s", StripPathPrefix(DI->file, strip_path_prefix));
245	break;	290	break;
246	case 'l':	291	case 'l':
247	buffer->append("%d", DI->line);	292	buffer->append("%zu", DI->line);
248	break;	293	break;
249	case 'g':	294	case 'g':
250	buffer->append("%s", DI->name);	295	buffer->append("%s", DI->name);
251	break;	296	break;
252	default:	297	default:
253	Report("Unsupported specifier in stack frame format: %c (0x%zx)!\n", *p,	298	Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p,
254	*p);	299	(void *)p);
255	Die();	300	Die();
256	}	301	}
257	}	302	}

lib/tsan/sanitizer_common/sanitizer_stacktrace_printer.h+5-4

lib/tsan/sanitizer_common/sanitizer_stacktrace_sparc.cpp+1-7

 @@ -9,7 +9,7 @@
 // This file is shared between AddressSanitizer and ThreadSanitizer
 // run-time libraries.
 //
-// Implemention of fast stack unwinding for Sparc.
+// Implementation of fast stack unwinding for Sparc.
 //===----------------------------------------------------------------------===//
 #if defined(__sparc__)
 @@ -30,13 +30,7 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
   // TODO(yln): add arg sanity check for stack_top/stack_bottom
   CHECK_GE(max_depth, 2);
   const uptr kPageSize = GetPageSizeCached();
-#if defined(__GNUC__)
-  // __builtin_return_address returns the address of the call instruction
-  // on the SPARC and not the return address, so we need to compensate.
-  trace_buffer[0] = GetNextInstructionPc(pc);
-#else
   trace_buffer[0] = pc;
-#endif
   size = 1;
   if (stack_top < 4096) return;  // Sanity check for stack top.
   // Flush register windows to memory

lib/tsan/sanitizer_common/sanitizer_stoptheworld_fuchsia.h created+20

...	@@ -0,0 +1,20 @@
1	//===-- sanitizer_stoptheworld_fuchsia.h ------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#ifndef SANITIZER_STOPTHEWORLD_FUCHSIA_H
10	#define SANITIZER_STOPTHEWORLD_FUCHSIA_H
11
12	#include "sanitizer_stoptheworld.h"
13
14	namespace __sanitizer {
15
16	class SuspendedThreadsListFuchsia final : public SuspendedThreadsList {};
17
18	} // namespace __sanitizer
19
20	#endif // SANITIZER_STOPTHEWORLD_FUCHSIA_H

lib/tsan/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp+11-3

...	@@ -16,7 +16,7 @@	...	@@ -16,7 +16,7 @@
16	#if SANITIZER_LINUX && \	16	#if SANITIZER_LINUX && \
17	(defined(__x86_64__) \|\| defined(__mips__) \|\| defined(__aarch64__) \|\| \	17	(defined(__x86_64__) \|\| defined(__mips__) \|\| defined(__aarch64__) \|\| \
18	defined(__powerpc64__) \|\| defined(__s390__) \|\| defined(__i386__) \|\| \	18	defined(__powerpc64__) \|\| defined(__s390__) \|\| defined(__i386__) \|\| \
19	defined(__arm__) \|\| SANITIZER_RISCV64)	19	defined(__arm__) \|\| SANITIZER_RISCV64 \|\| SANITIZER_LOONGARCH64)
20		20
21	#include "sanitizer_stoptheworld.h"	21	#include "sanitizer_stoptheworld.h"
22		22
...	@@ -31,7 +31,8 @@	...	@@ -31,7 +31,8 @@
31	#include <sys/types.h> // for pid_t	31	#include <sys/types.h> // for pid_t
32	#include <sys/uio.h> // for iovec	32	#include <sys/uio.h> // for iovec
33	#include <elf.h> // for NT_PRSTATUS	33	#include <elf.h> // for NT_PRSTATUS
34	#if (defined(__aarch64__) \|\| SANITIZER_RISCV64) && !SANITIZER_ANDROID	34	#if (defined(__aarch64__) \|\| SANITIZER_RISCV64 \|\| SANITIZER_LOONGARCH64) && \
		35	!SANITIZER_ANDROID
35	// GLIBC 2.20+ sys/user does not include asm/ptrace.h	36	// GLIBC 2.20+ sys/user does not include asm/ptrace.h
36	# include <asm/ptrace.h>	37	# include <asm/ptrace.h>
37	#endif	38	#endif
...	@@ -108,7 +109,7 @@ struct TracerThreadArgument {	...	@@ -108,7 +109,7 @@ struct TracerThreadArgument {
108	void *callback_argument;	109	void *callback_argument;
109	// The tracer thread waits on this mutex while the parent finishes its	110	// The tracer thread waits on this mutex while the parent finishes its
110	// preparations.	111	// preparations.
111	BlockingMutex mutex;	112	Mutex mutex;
112	// Tracer thread signals its completion by setting done.	113	// Tracer thread signals its completion by setting done.
113	atomic_uintptr_t done;	114	atomic_uintptr_t done;
114	uptr parent_pid;	115	uptr parent_pid;
...	@@ -514,6 +515,12 @@ typedef struct user_pt_regs regs_struct;	...	@@ -514,6 +515,12 @@ typedef struct user_pt_regs regs_struct;
514	static constexpr uptr kExtraRegs[] = {0};	515	static constexpr uptr kExtraRegs[] = {0};
515	#define ARCH_IOVEC_FOR_GETREGSET	516	#define ARCH_IOVEC_FOR_GETREGSET
516		517
		518	#elif defined(__loongarch__)
		519	typedef struct user_pt_regs regs_struct;
		520	#define REG_SP regs[3]
		521	static constexpr uptr kExtraRegs[] = {0};
		522	#define ARCH_IOVEC_FOR_GETREGSET
		523
517	#elif SANITIZER_RISCV64	524	#elif SANITIZER_RISCV64
518	typedef struct user_regs_struct regs_struct;	525	typedef struct user_regs_struct regs_struct;
519	// sys/ucontext.h already defines REG_SP as 2. Undefine it first.	526	// sys/ucontext.h already defines REG_SP as 2. Undefine it first.
...	@@ -621,3 +628,4 @@ PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP(	...	@@ -621,3 +628,4 @@ PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP(
621	#endif // SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__mips__)	628	#endif // SANITIZER_LINUX && (defined(__x86_64__) \|\| defined(__mips__)
622	// \|\| defined(__aarch64__) \|\| defined(__powerpc64__)	629	// \|\| defined(__aarch64__) \|\| defined(__powerpc64__)
623	// \|\| defined(__s390__) \|\| defined(__i386__) \|\| defined(__arm__)	630	// \|\| defined(__s390__) \|\| defined(__i386__) \|\| defined(__arm__)
		631	// \|\| SANITIZER_LOONGARCH64

lib/tsan/sanitizer_common/sanitizer_stoptheworld_mac.cpp+10-9

 @@ -12,7 +12,7 @@
 #include "sanitizer_platform.h"
-#if SANITIZER_MAC && (defined(__x86_64__) || defined(__aarch64__) || \
+#if SANITIZER_APPLE && (defined(__x86_64__) || defined(__aarch64__) || \
                       defined(__i386))
 #include <mach/mach.h>
 @@ -29,7 +29,7 @@ typedef struct {
 class SuspendedThreadsListMac final : public SuspendedThreadsList {
  public:
-  SuspendedThreadsListMac() : threads_(1024) {}
+  SuspendedThreadsListMac() = default;
   tid_t GetThreadID(uptr index) const override;
   thread_t GetThread(uptr index) const;
 @@ -87,11 +87,13 @@ void StopTheWorld(StopTheWorldCallback callback, void *argument) {
 #if defined(__x86_64__)
 typedef x86_thread_state64_t regs_struct;
+#define regs_flavor x86_THREAD_STATE64
 #define SP_REG __rsp
 #elif defined(__aarch64__)
 typedef arm_thread_state64_t regs_struct;
+#define regs_flavor ARM_THREAD_STATE64
 # if __DARWIN_UNIX03
 #  define SP_REG __sp
 @@ -101,6 +103,7 @@ typedef arm_thread_state64_t regs_struct;
 #elif defined(__i386)
 typedef x86_thread_state32_t regs_struct;
+#define regs_flavor x86_THREAD_STATE32
 #define SP_REG __esp
 @@ -146,17 +149,15 @@ PtraceRegistersStatus SuspendedThreadsListMac::GetRegistersAndSP(
   thread_t thread = GetThread(index);
   regs_struct regs;
   int err;
-  mach_msg_type_number_t reg_count = MACHINE_THREAD_STATE_COUNT;
+  mach_msg_type_number_t reg_count = sizeof(regs) / sizeof(natural_t);
-  err = thread_get_state(thread, MACHINE_THREAD_STATE, (thread_state_t)&regs,
+  err = thread_get_state(thread, regs_flavor, (thread_state_t)&regs,
                          &reg_count);
   if (err != KERN_SUCCESS) {
     VReport(1, "Error - unable to get registers for a thread\n");
-    // KERN_INVALID_ARGUMENT indicates that either the flavor is invalid,
-    // or the thread does not exist. The other possible error case,
     // MIG_ARRAY_TOO_LARGE, means that the state is too large, but it's
     // still safe to proceed.
-    return err == KERN_INVALID_ARGUMENT ? REGISTERS_UNAVAILABLE_FATAL
+    return err == MIG_ARRAY_TOO_LARGE ? REGISTERS_UNAVAILABLE
-                                        : REGISTERS_UNAVAILABLE;
+                                      : REGISTERS_UNAVAILABLE_FATAL;
   }
   buffer->resize(RoundUpTo(sizeof(regs), sizeof(uptr)) / sizeof(uptr));
 @@ -176,5 +177,5 @@ PtraceRegistersStatus SuspendedThreadsListMac::GetRegistersAndSP(
 } // namespace __sanitizer
-#endif  // SANITIZER_MAC && (defined(__x86_64__) || defined(__aarch64__)) ||
+#endif  // SANITIZER_APPLE && (defined(__x86_64__) || defined(__aarch64__)) ||
         //                   defined(__i386))

lib/tsan/sanitizer_common/sanitizer_stoptheworld_netbsd_libcdep.cpp+1-1

...	@@ -68,7 +68,7 @@ class SuspendedThreadsListNetBSD final : public SuspendedThreadsList {	...	@@ -68,7 +68,7 @@ class SuspendedThreadsListNetBSD final : public SuspendedThreadsList {
68	struct TracerThreadArgument {	68	struct TracerThreadArgument {
69	StopTheWorldCallback callback;	69	StopTheWorldCallback callback;
70	void *callback_argument;	70	void *callback_argument;
71	BlockingMutex mutex;	71	Mutex mutex;
72	atomic_uintptr_t done;	72	atomic_uintptr_t done;
73	uptr parent_pid;	73	uptr parent_pid;
74	};	74	};

lib/tsan/sanitizer_common/sanitizer_stoptheworld_win.cpp created+175

...	@@ -0,0 +1,175 @@
1	//===-- sanitizer_stoptheworld_win.cpp ------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// See sanitizer_stoptheworld.h for details.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "sanitizer_platform.h"
14
15	#if SANITIZER_WINDOWS
16
17	# define WIN32_LEAN_AND_MEAN
18	# include <windows.h>
19	// windows.h needs to be included before tlhelp32.h
20	# include <tlhelp32.h>
21
22	# include "sanitizer_stoptheworld.h"
23
24	namespace __sanitizer {
25
26	namespace {
27
28	struct SuspendedThreadsListWindows final : public SuspendedThreadsList {
29	InternalMmapVector<HANDLE> threadHandles;
30	InternalMmapVector<DWORD> threadIds;
31
32	SuspendedThreadsListWindows() {
33	threadIds.reserve(1024);
34	threadHandles.reserve(1024);
35	}
36
37	PtraceRegistersStatus GetRegistersAndSP(uptr index,
38	InternalMmapVector<uptr> *buffer,
39	uptr *sp) const override;
40
41	tid_t GetThreadID(uptr index) const override;
42	uptr ThreadCount() const override;
43	};
44
45	// Stack Pointer register names on different architectures
46	# if SANITIZER_X64
47	# define SP_REG Rsp
48	# elif SANITIZER_I386
49	# define SP_REG Esp
50	# elif SANITIZER_ARM \| SANITIZER_ARM64
51	# define SP_REG Sp
52	# else
53	# error Architecture not supported!
54	# endif
55
56	PtraceRegistersStatus SuspendedThreadsListWindows::GetRegistersAndSP(
57	uptr index, InternalMmapVector<uptr> buffer, uptr sp) const {
58	CHECK_LT(index, threadHandles.size());
59
60	buffer->resize(RoundUpTo(sizeof(CONTEXT), sizeof(uptr)) / sizeof(uptr));
61	CONTEXT thread_context = reinterpret_cast<CONTEXT >(buffer->data());
62	thread_context->ContextFlags = CONTEXT_ALL;
63	CHECK(GetThreadContext(threadHandles[index], thread_context));
64	*sp = thread_context->SP_REG;
65
66	return REGISTERS_AVAILABLE;
67	}
68
69	tid_t SuspendedThreadsListWindows::GetThreadID(uptr index) const {
70	CHECK_LT(index, threadIds.size());
71	return threadIds[index];
72	}
73
74	uptr SuspendedThreadsListWindows::ThreadCount() const {
75	return threadIds.size();
76	}
77
78	struct RunThreadArgs {
79	StopTheWorldCallback callback;
80	void *argument;
81	};
82
83	DWORD WINAPI RunThread(void *argument) {
84	RunThreadArgs run_args = (RunThreadArgs )argument;
85
86	const DWORD this_thread = GetCurrentThreadId();
87	const DWORD this_process = GetCurrentProcessId();
88
89	SuspendedThreadsListWindows suspended_threads_list;
90	bool new_thread_found;
91
92	do {
93	// Take a snapshot of all Threads
94	const HANDLE threads = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0);
95	CHECK(threads != INVALID_HANDLE_VALUE);
96
97	THREADENTRY32 thread_entry;
98	thread_entry.dwSize = sizeof(thread_entry);
99	new_thread_found = false;
100
101	if (!Thread32First(threads, &thread_entry))
102	break;
103
104	do {
105	if (thread_entry.th32ThreadID == this_thread \|\|
106	thread_entry.th32OwnerProcessID != this_process)
107	continue;
108
109	bool suspended_thread = false;
110	for (const auto thread_id : suspended_threads_list.threadIds) {
111	if (thread_id == thread_entry.th32ThreadID) {
112	suspended_thread = true;
113	break;
114	}
115	}
116
117	// Skip the Thread if it was already suspended
118	if (suspended_thread)
119	continue;
120
121	const HANDLE thread =
122	OpenThread(THREAD_ALL_ACCESS, FALSE, thread_entry.th32ThreadID);
123	CHECK(thread);
124
125	if (SuspendThread(thread) == (DWORD)-1) {
126	DWORD last_error = GetLastError();
127
128	VPrintf(1, "Could not suspend thread %lu (error %lu)",
129	thread_entry.th32ThreadID, last_error);
130	continue;
131	}
132
133	suspended_threads_list.threadIds.push_back(thread_entry.th32ThreadID);
134	suspended_threads_list.threadHandles.push_back(thread);
135	new_thread_found = true;
136	} while (Thread32Next(threads, &thread_entry));
137
138	CloseHandle(threads);
139
140	// Between the call to `CreateToolhelp32Snapshot` and suspending the
141	// relevant Threads, new Threads could have potentially been created. So
142	// continue to find and suspend new Threads until we don't find any.
143	} while (new_thread_found);
144
145	// Now all Threads of this Process except of this Thread should be suspended.
146	// Execute the callback function.
147	run_args->callback(suspended_threads_list, run_args->argument);
148
149	// Resume all Threads
150	for (const auto suspended_thread_handle :
151	suspended_threads_list.threadHandles) {
152	CHECK_NE(ResumeThread(suspended_thread_handle), -1);
153	CloseHandle(suspended_thread_handle);
154	}
155
156	return 0;
157	}
158
159	} // namespace
160
161	void StopTheWorld(StopTheWorldCallback callback, void *argument) {
162	struct RunThreadArgs arg = {callback, argument};
163	DWORD trace_thread_id;
164
165	auto trace_thread =
166	CreateThread(nullptr, 0, RunThread, &arg, 0, &trace_thread_id);
167	CHECK(trace_thread);
168
169	WaitForSingleObject(trace_thread, INFINITE);
170	CloseHandle(trace_thread);
171	}
172
173	} // namespace __sanitizer
174
175	#endif // SANITIZER_WINDOWS

lib/tsan/sanitizer_common/sanitizer_suppressions.cpp+1

...	@@ -86,6 +86,7 @@ void SuppressionContext::ParseFromFile(const char *filename) {	...	@@ -86,6 +86,7 @@ void SuppressionContext::ParseFromFile(const char *filename) {
86	}	86	}
87		87
88	Parse(file_contents);	88	Parse(file_contents);
		89	UnmapOrDie(file_contents, contents_size);
89	}	90	}
90		91
91	bool SuppressionContext::Match(const char str, const char type,	92	bool SuppressionContext::Match(const char str, const char type,

lib/tsan/sanitizer_common/sanitizer_symbolizer.cpp+13-7

 @@ -11,10 +11,11 @@
 //===----------------------------------------------------------------------===//
 #include "sanitizer_allocator_internal.h"
-#include "sanitizer_platform.h"
+#include "sanitizer_common.h"
 #include "sanitizer_internal_defs.h"
 #include "sanitizer_libc.h"
 #include "sanitizer_placement_new.h"
+#include "sanitizer_platform.h"
 #include "sanitizer_symbolizer_internal.h"
 namespace __sanitizer {
 @@ -30,6 +31,7 @@ void AddressInfo::Clear() {
   InternalFree(file);
   internal_memset(this, 0, sizeof(AddressInfo));
   function_offset = kUnknown;
+  uuid_size = 0;
 }
 void AddressInfo::FillModuleInfo(const char *mod_name, uptr mod_offset,
 @@ -37,6 +39,16 @@ void AddressInfo::FillModuleInfo(const char *mod_name, uptr mod_offset,
   module = internal_strdup(mod_name);
   module_offset = mod_offset;
   module_arch = mod_arch;
+  uuid_size = 0;
+}
+void AddressInfo::FillModuleInfo(const LoadedModule &mod) {
+  module = internal_strdup(mod.full_name());
+  module_offset = address - mod.base_address();
+  module_arch = mod.arch();
+  if (mod.uuid_size())
+    internal_memcpy(uuid, mod.uuid(), mod.uuid_size());
+  uuid_size = mod.uuid_size();
 }
 SymbolizedStack::SymbolizedStack() : next(nullptr), info() {}
 @@ -126,10 +138,4 @@ Symbolizer::SymbolizerScope::~SymbolizerScope() {
     sym_->end_hook_();
 }
-void Symbolizer::LateInitializeTools() {
-  for (auto &tool : tools_) {
-    tool.LateInitialize();
-  }
-}
 }  // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_symbolizer.h+7-6

...	@@ -32,6 +32,8 @@ struct AddressInfo {	...	@@ -32,6 +32,8 @@ struct AddressInfo {
32	char *module;	32	char *module;
33	uptr module_offset;	33	uptr module_offset;
34	ModuleArch module_arch;	34	ModuleArch module_arch;
		35	u8 uuid[kModuleUUIDSize];
		36	uptr uuid_size;
35		37
36	static const uptr kUnknown = ~(uptr)0;	38	static const uptr kUnknown = ~(uptr)0;
37	char *function;	39	char *function;
...	@@ -45,6 +47,8 @@ struct AddressInfo {	...	@@ -45,6 +47,8 @@ struct AddressInfo {
45	// Deletes all strings and resets all fields.	47	// Deletes all strings and resets all fields.
46	void Clear();	48	void Clear();
47	void FillModuleInfo(const char *mod_name, uptr mod_offset, ModuleArch arch);	49	void FillModuleInfo(const char *mod_name, uptr mod_offset, ModuleArch arch);
		50	void FillModuleInfo(const LoadedModule &mod);
		51	uptr module_base() const { return address - module_offset; }
48	};	52	};
49		53
50	// Linked list of symbolized frames (each frame is described by AddressInfo).	54	// Linked list of symbolized frames (each frame is described by AddressInfo).
...	@@ -158,7 +162,7 @@ class Symbolizer final {	...	@@ -158,7 +162,7 @@ class Symbolizer final {
158	// its method should be protected by \|mu_\|.	162	// its method should be protected by \|mu_\|.
159	class ModuleNameOwner {	163	class ModuleNameOwner {
160	public:	164	public:
161	explicit ModuleNameOwner(BlockingMutex *synchronized_by)	165	explicit ModuleNameOwner(Mutex *synchronized_by)
162	: last_match_(nullptr), mu_(synchronized_by) {	166	: last_match_(nullptr), mu_(synchronized_by) {
163	storage_.reserve(kInitialCapacity);	167	storage_.reserve(kInitialCapacity);
164	}	168	}
...	@@ -169,7 +173,7 @@ class Symbolizer final {	...	@@ -169,7 +173,7 @@ class Symbolizer final {
169	InternalMmapVector<const char*> storage_;	173	InternalMmapVector<const char*> storage_;
170	const char *last_match_;	174	const char *last_match_;
171		175
172	BlockingMutex *mu_;	176	Mutex *mu_;
173	} module_names_;	177	} module_names_;
174		178
175	/// Platform-specific function for creating a Symbolizer object.	179	/// Platform-specific function for creating a Symbolizer object.
...	@@ -192,7 +196,7 @@ class Symbolizer final {	...	@@ -192,7 +196,7 @@ class Symbolizer final {
192	// Mutex locked from public methods of \|Symbolizer\|, so that the internals	196	// Mutex locked from public methods of \|Symbolizer\|, so that the internals
193	// (including individual symbolizer tools and platform-specific methods) are	197	// (including individual symbolizer tools and platform-specific methods) are
194	// always synchronized.	198	// always synchronized.
195	BlockingMutex mu_;	199	Mutex mu_;
196		200
197	IntrusiveList<SymbolizerTool> tools_;	201	IntrusiveList<SymbolizerTool> tools_;
198		202
...	@@ -209,9 +213,6 @@ class Symbolizer final {	...	@@ -209,9 +213,6 @@ class Symbolizer final {
209	private:	213	private:
210	const Symbolizer *sym_;	214	const Symbolizer *sym_;
211	};	215	};
212
213	// Calls `LateInitialize()` on all items in `tools_`.
214	void LateInitializeTools();
215	};	216	};
216		217
217	#ifdef SANITIZER_WINDOWS	218	#ifdef SANITIZER_WINDOWS

lib/tsan/sanitizer_common/sanitizer_symbolizer_internal.h+6-11

 @@ -13,15 +13,15 @@
 #ifndef SANITIZER_SYMBOLIZER_INTERNAL_H
 #define SANITIZER_SYMBOLIZER_INTERNAL_H
-#include "sanitizer_symbolizer.h"
 #include "sanitizer_file.h"
+#include "sanitizer_symbolizer.h"
 #include "sanitizer_vector.h"
 namespace __sanitizer {
 // Parsing helpers, 'str' is searched for delimiter(s) and a string or uptr
 // is extracted. When extracting a string, a newly allocated (using
-// InternalAlloc) and null-terminataed buffer is returned. They return a pointer
+// InternalAlloc) and null-terminated buffer is returned. They return a pointer
 // to the next characted after the found delimiter.
 const char *ExtractToken(const char *str, const char *delims, char **result);
 const char *ExtractInt(const char *str, const char *delims, int *result);
 @@ -70,11 +70,6 @@ class SymbolizerTool {
     return nullptr;
   }
-  // Called during the LateInitialize phase of Sanitizer initialization.
-  // Usually this is a safe place to call code that might need to use user
-  // memory allocators.
-  virtual void LateInitialize() {}
  protected:
   ~SymbolizerTool() {}
 };
 @@ -91,13 +86,14 @@ class SymbolizerProcess {
   ~SymbolizerProcess() {}
   /// The maximum number of arguments required to invoke a tool process.
-  static const unsigned kArgVMax = 6;
+  static const unsigned kArgVMax = 16;
   // Customizable by subclasses.
   virtual bool StartSymbolizerSubprocess();
-  virtual bool ReadFromSymbolizer(char *buffer, uptr max_length);
+  virtual bool ReadFromSymbolizer();
   // Return the environment to run the symbolizer in.
   virtual char **GetEnvP() { return GetEnviron(); }
+  InternalMmapVector<char> &GetBuff() { return buffer_; }
  private:
   virtual bool ReachedEndOfOutput(const char *buffer, uptr length) const {
 @@ -118,8 +114,7 @@ class SymbolizerProcess {
   fd_t input_fd_;
   fd_t output_fd_;
-  static const uptr kBufferSize = 16 * 1024;
+  InternalMmapVector<char> buffer_;
-  char buffer_[kBufferSize];
   static const uptr kMaxTimesRestarted = 5;
   static const int kSymbolizerStartupTimeMillis = 10;

lib/tsan/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp+2-2

 @@ -11,11 +11,11 @@
 // Libbacktrace implementation of symbolizer parts.
 //===----------------------------------------------------------------------===//
-#include "sanitizer_platform.h"
+#include "sanitizer_symbolizer_libbacktrace.h"
 #include "sanitizer_internal_defs.h"
+#include "sanitizer_platform.h"
 #include "sanitizer_symbolizer.h"
-#include "sanitizer_symbolizer_libbacktrace.h"
 #if SANITIZER_LIBBACKTRACE
 # include "backtrace-supported.h"

lib/tsan/sanitizer_common/sanitizer_symbolizer_libcdep.cpp+51-40

 @@ -83,16 +83,13 @@ const char *ExtractTokenUpToDelimiter(const char *str, const char *delimiter,
 }
 SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
-  BlockingMutexLock l(&mu_);
+  Lock l(&mu_);
-  const char *module_name = nullptr;
-  uptr module_offset;
-  ModuleArch arch;
   SymbolizedStack *res = SymbolizedStack::New(addr);
-  if (!FindModuleNameAndOffsetForAddress(addr, &module_name, &module_offset,
+  auto *mod = FindModuleForAddress(addr);
-                                         &arch))
+  if (!mod)
     return res;
   // Always fill data about module name and offset.
-  res->info.FillModuleInfo(module_name, module_offset, arch);
+  res->info.FillModuleInfo(*mod);
   for (auto &tool : tools_) {
     SymbolizerScope sym_scope(this);
     if (tool.SymbolizePC(addr, res)) {
 @@ -103,7 +100,7 @@ SymbolizedStack *Symbolizer::SymbolizePC(uptr addr) {
 }
 bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
-  BlockingMutexLock l(&mu_);
+  Lock l(&mu_);
   const char *module_name = nullptr;
   uptr module_offset;
   ModuleArch arch;
 @@ -124,7 +121,7 @@ bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) {
 }
 bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) {
-  BlockingMutexLock l(&mu_);
+  Lock l(&mu_);
   const char *module_name = nullptr;
   if (!FindModuleNameAndOffsetForAddress(
           addr, &module_name, &info->module_offset, &info->module_arch))
 @@ -141,7 +138,7 @@ bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) {
 bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
                                              uptr *module_address) {
-  BlockingMutexLock l(&mu_);
+  Lock l(&mu_);
   const char *internal_module_name = nullptr;
   ModuleArch arch;
   if (!FindModuleNameAndOffsetForAddress(pc, &internal_module_name,
 @@ -154,7 +151,7 @@ bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name,
 }
 void Symbolizer::Flush() {
-  BlockingMutexLock l(&mu_);
+  Lock l(&mu_);
   for (auto &tool : tools_) {
     SymbolizerScope sym_scope(this);
     tool.Flush();
 @@ -162,7 +159,7 @@ void Symbolizer::Flush() {
 }
 const char *Symbolizer::Demangle(const char *name) {
-  BlockingMutexLock l(&mu_);
+  Lock l(&mu_);
   for (auto &tool : tools_) {
     SymbolizerScope sym_scope(this);
     if (const char *demangled = tool.Demangle(name))
 @@ -240,7 +237,7 @@ const LoadedModule *Symbolizer::FindModuleForAddress(uptr address) {
 class LLVMSymbolizerProcess final : public SymbolizerProcess {
  public:
   explicit LLVMSymbolizerProcess(const char *path)
-      : SymbolizerProcess(path, /*use_posix_spawn=*/SANITIZER_MAC) {}
+      : SymbolizerProcess(path, /*use_posix_spawn=*/SANITIZER_APPLE) {}
  private:
   bool ReachedEndOfOutput(const char *buffer, uptr length) const override {
 @@ -259,6 +256,8 @@ class LLVMSymbolizerProcess final : public SymbolizerProcess {
     const char* const kSymbolizerArch = "--default-arch=x86_64";
 #elif defined(__i386__)
     const char* const kSymbolizerArch = "--default-arch=i386";
+#elif SANITIZER_LOONGARCH64
+    const char *const kSymbolizerArch = "--default-arch=loongarch64";
 #elif SANITIZER_RISCV64
     const char *const kSymbolizerArch = "--default-arch=riscv64";
 #elif defined(__aarch64__)
 @@ -277,14 +276,17 @@ class LLVMSymbolizerProcess final : public SymbolizerProcess {
     const char* const kSymbolizerArch = "--default-arch=unknown";
 #endif
-    const char *const inline_flag = common_flags()->symbolize_inline_frames
+    const char *const demangle_flag =
-                                        ? "--inlines"
+        common_flags()->demangle ? "--demangle" : "--no-demangle";
-                                        : "--no-inlines";
+    const char *const inline_flag =
+        common_flags()->symbolize_inline_frames ? "--inlines" : "--no-inlines";
     int i = 0;
     argv[i++] = path_to_binary;
+    argv[i++] = demangle_flag;
     argv[i++] = inline_flag;
     argv[i++] = kSymbolizerArch;
     argv[i++] = nullptr;
+    CHECK_LE(i, kArgVMax);
   }
 };
 @@ -363,14 +365,21 @@ void ParseSymbolizePCOutput(const char *str, SymbolizedStack *res) {
   }
 }
-// Parses a two-line string in the following format:
+// Parses a two- or three-line string in the following format:
 //   <symbol_name>
 //   <start_address> <size>
-// Used by LLVMSymbolizer and InternalSymbolizer.
+//   <filename>:<column>
+// Used by LLVMSymbolizer and InternalSymbolizer. LLVMSymbolizer added support
+// for symbolizing the third line in D123538, but we support the older two-line
+// information as well.
 void ParseSymbolizeDataOutput(const char *str, DataInfo *info) {
   str = ExtractToken(str, "\n", &info->name);
   str = ExtractUptr(str, " ", &info->start);
   str = ExtractUptr(str, "\n", &info->size);
+  // Note: If the third line isn't present, these calls will set info.{file,
+  // line} to empty strings.
+  str = ExtractToken(str, ":", &info->file);
+  str = ExtractUptr(str, "\n", &info->line);
 }
 static void ParseSymbolizeFrameOutput(const char *str,
 @@ -500,9 +509,9 @@ const char *SymbolizerProcess::SendCommandImpl(const char *command) {
       return nullptr;
   if (!WriteToSymbolizer(command, internal_strlen(command)))
       return nullptr;
-  if (!ReadFromSymbolizer(buffer_, kBufferSize))
+  if (!ReadFromSymbolizer())
-      return nullptr;
+    return nullptr;
-  return buffer_;
+  return buffer_.data();
 }
 bool SymbolizerProcess::Restart() {
 @@ -513,31 +522,33 @@ bool SymbolizerProcess::Restart() {
   return StartSymbolizerSubprocess();
 }
-bool SymbolizerProcess::ReadFromSymbolizer(char *buffer, uptr max_length) {
+bool SymbolizerProcess::ReadFromSymbolizer() {
-  if (max_length == 0)
+  buffer_.clear();
-    return true;
+  constexpr uptr max_length = 1024;
-  uptr read_len = 0;
+  bool ret = true;
-  while (true) {
+  do {
     uptr just_read = 0;
-    bool success = ReadFromFile(input_fd_, buffer + read_len,
+    uptr size_before = buffer_.size();
-                                max_length - read_len - 1, &just_read);
+    buffer_.resize(size_before + max_length);
+    buffer_.resize(buffer_.capacity());
+    bool ret = ReadFromFile(input_fd_, &buffer_[size_before],
+                            buffer_.size() - size_before, &just_read);
+    if (!ret)
+      just_read = 0;
+    buffer_.resize(size_before + just_read);
     // We can't read 0 bytes, as we don't expect external symbolizer to close
     // its stdout.
-    if (!success || just_read == 0) {
+    if (just_read == 0) {
       Report("WARNING: Can't read from symbolizer at fd %d\n", input_fd_);
-      return false;
+      ret = false;
-    }
-    read_len += just_read;
-    if (ReachedEndOfOutput(buffer, read_len))
-      break;
-    if (read_len + 1 == max_length) {
-      Report("WARNING: Symbolizer buffer too small\n");
-      read_len = 0;
       break;
     }
-  }
+  } while (!ReachedEndOfOutput(buffer_.data(), buffer_.size()));
-  buffer[read_len] = '\0';
+  buffer_.push_back('\0');
-  return true;
+  return ret;
 }
 bool SymbolizerProcess::WriteToSymbolizer(const char *buffer, uptr length) {

lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.cpp+15-66

...	@@ -12,19 +12,18 @@	...	@@ -12,19 +12,18 @@
12	//===----------------------------------------------------------------------===//	12	//===----------------------------------------------------------------------===//
13		13
14	#include "sanitizer_platform.h"	14	#include "sanitizer_platform.h"
15	#if SANITIZER_MAC	15	#if SANITIZER_APPLE
16		16
17	#include "sanitizer_allocator_internal.h"	17	# include <dlfcn.h>
18	#include "sanitizer_mac.h"	18	# include <errno.h>
19	#include "sanitizer_symbolizer_mac.h"	19	# include <stdlib.h>
		20	# include <sys/wait.h>
		21	# include <unistd.h>
		22	# include <util.h>
20		23
21	#include <dlfcn.h>	24	# include "sanitizer_allocator_internal.h"
22	#include <errno.h>	25	# include "sanitizer_mac.h"
23	#include <mach/mach.h>	26	# include "sanitizer_symbolizer_mac.h"
24	#include <stdlib.h>
25	#include <sys/wait.h>
26	#include <unistd.h>
27	#include <util.h>
28		27
29	namespace __sanitizer {	28	namespace __sanitizer {
30		29
...	@@ -58,13 +57,6 @@ bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) {	...	@@ -58,13 +57,6 @@ bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) {
58	return true;	57	return true;
59	}	58	}
60		59
61	#define K_ATOS_ENV_VAR "__check_mach_ports_lookup"
62
63	// This cannot live in `AtosSymbolizerProcess` because instances of that object
64	// are allocated by the internal allocator which under ASan is poisoned with
65	// kAsanInternalHeapMagic.
66	static char kAtosMachPortEnvEntry[] = K_ATOS_ENV_VAR "=000000000000000";
67
68	class AtosSymbolizerProcess final : public SymbolizerProcess {	60	class AtosSymbolizerProcess final : public SymbolizerProcess {
69	public:	61	public:
70	explicit AtosSymbolizerProcess(const char *path)	62	explicit AtosSymbolizerProcess(const char *path)
...	@@ -72,51 +64,13 @@ class AtosSymbolizerProcess final : public SymbolizerProcess {	...	@@ -72,51 +64,13 @@ class AtosSymbolizerProcess final : public SymbolizerProcess {
72	pid_str_[0] = '\0';	64	pid_str_[0] = '\0';
73	}	65	}
74		66
75	void LateInitialize() {
76	if (SANITIZER_IOSSIM) {
77	// `putenv()` may call malloc/realloc so it is only safe to do this
78	// during LateInitialize() or later (i.e. we can't do this in the
79	// constructor). We also can't do this in `StartSymbolizerSubprocess()`
80	// because in TSan we switch allocators when we're symbolizing.
81	// We use `putenv()` rather than `setenv()` so that we can later directly
82	// write into the storage without LibC getting involved to change what the
83	// variable is set to
84	int result = putenv(kAtosMachPortEnvEntry);
85	CHECK_EQ(result, 0);
86	}
87	}
88
89	private:	67	private:
90	bool StartSymbolizerSubprocess() override {	68	bool StartSymbolizerSubprocess() override {
91	// Configure sandbox before starting atos process.
92
93	// Put the string command line argument in the object so that it outlives	69	// Put the string command line argument in the object so that it outlives
94	// the call to GetArgV.	70	// the call to GetArgV.
95	internal_snprintf(pid_str_, sizeof(pid_str_), "%d", internal_getpid());	71	internal_snprintf(pid_str_, sizeof(pid_str_), "%d", (int)internal_getpid());
96
97	if (SANITIZER_IOSSIM) {
98	// `atos` in the simulator is restricted in its ability to retrieve the
99	// task port for the target process (us) so we need to do extra work
100	// to pass our task port to it.
101	mach_port_t ports[]{mach_task_self()};
102	kern_return_t ret =
103	mach_ports_register(mach_task_self(), ports, /count=/1);
104	CHECK_EQ(ret, KERN_SUCCESS);
105
106	// Set environment variable that signals to `atos` that it should look
107	// for our task port. We can't call `setenv()` here because it might call
108	// malloc/realloc. To avoid that we instead update the
109	// `mach_port_env_var_entry_` variable with our current PID.
110	uptr count = internal_snprintf(kAtosMachPortEnvEntry,
111	sizeof(kAtosMachPortEnvEntry),
112	K_ATOS_ENV_VAR "=%s", pid_str_);
113	CHECK_GE(count, sizeof(K_ATOS_ENV_VAR) + internal_strlen(pid_str_));
114	// Document our assumption but without calling `getenv()` in normal
115	// builds.
116	DCHECK(getenv(K_ATOS_ENV_VAR));
117	DCHECK_EQ(internal_strcmp(getenv(K_ATOS_ENV_VAR), pid_str_), 0);
118	}
119		72
		73	// Configure sandbox before starting atos process.
120	return SymbolizerProcess::StartSymbolizerSubprocess();	74	return SymbolizerProcess::StartSymbolizerSubprocess();
121	}	75	}
122		76
...	@@ -137,13 +91,10 @@ class AtosSymbolizerProcess final : public SymbolizerProcess {	...	@@ -137,13 +91,10 @@ class AtosSymbolizerProcess final : public SymbolizerProcess {
137	argv[i++] = "-d";	91	argv[i++] = "-d";
138	}	92	}
139	argv[i++] = nullptr;	93	argv[i++] = nullptr;
		94	CHECK_LE(i, kArgVMax);
140	}	95	}
141		96
142	char pid_str_[16];	97	char pid_str_[16];
143	// Space for `\0` in `K_ATOS_ENV_VAR` is reused for `=`.
144	static_assert(sizeof(kAtosMachPortEnvEntry) ==
145	(sizeof(K_ATOS_ENV_VAR) + sizeof(pid_str_)),
146	"sizes should match");
147	};	98	};
148		99
149	#undef K_ATOS_ENV_VAR	100	#undef K_ATOS_ENV_VAR
...	@@ -212,7 +163,7 @@ bool AtosSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {	...	@@ -212,7 +163,7 @@ bool AtosSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
212	uptr start_address = AddressInfo::kUnknown;	163	uptr start_address = AddressInfo::kUnknown;
213	if (!ParseCommandOutput(buf, addr, &stack->info.function, &stack->info.module,	164	if (!ParseCommandOutput(buf, addr, &stack->info.function, &stack->info.module,
214	&stack->info.file, &line, &start_address)) {	165	&stack->info.file, &line, &start_address)) {
215	process_ = nullptr;	166	Report("WARNING: atos failed to symbolize address \"0x%zx\"\n", addr);
216	return false;	167	return false;
217	}	168	}
218	stack->info.line = (int)line;	169	stack->info.line = (int)line;
...	@@ -249,8 +200,6 @@ bool AtosSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {	...	@@ -249,8 +200,6 @@ bool AtosSymbolizer::SymbolizeData(uptr addr, DataInfo *info) {
249	return true;	200	return true;
250	}	201	}
251		202
252	void AtosSymbolizer::LateInitialize() { process_->LateInitialize(); }
253
254	} // namespace __sanitizer	203	} // namespace __sanitizer
255		204
256	#endif // SANITIZER_MAC	205	#endif // SANITIZER_APPLE

lib/tsan/sanitizer_common/sanitizer_symbolizer_mac.h+2-3

lib/tsan/sanitizer_common/sanitizer_symbolizer_markup.cpp+2-4

...	@@ -91,7 +91,7 @@ bool RenderNeedsSymbolization(const char *format) { return false; }	...	@@ -91,7 +91,7 @@ bool RenderNeedsSymbolization(const char *format) { return false; }
91	// We don't support the stack_trace_format flag at all.	91	// We don't support the stack_trace_format flag at all.
92	void RenderFrame(InternalScopedString buffer, const char format, int frame_no,	92	void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
93	uptr address, const AddressInfo *info, bool vs_style,	93	uptr address, const AddressInfo *info, bool vs_style,
94	const char strip_path_prefix, const char strip_func_prefix) {	94	const char *strip_path_prefix) {
95	CHECK(!RenderNeedsSymbolization(format));	95	CHECK(!RenderNeedsSymbolization(format));
96	buffer->append(kFormatFrame, frame_no, address);	96	buffer->append(kFormatFrame, frame_no, address);
97	}	97	}
...	@@ -100,9 +100,7 @@ Symbolizer *Symbolizer::PlatformInit() {	...	@@ -100,9 +100,7 @@ Symbolizer *Symbolizer::PlatformInit() {
100	return new (symbolizer_allocator_) Symbolizer({});	100	return new (symbolizer_allocator_) Symbolizer({});
101	}	101	}
102		102
103	void Symbolizer::LateInitialize() {	103	void Symbolizer::LateInitialize() { Symbolizer::GetOrInit(); }
104	Symbolizer::GetOrInit()->LateInitializeTools();
105	}
106		104
107	void StartReportDeadlySignal() {}	105	void StartReportDeadlySignal() {}
108	void ReportDeadlySignal(const SignalContext &sig, u32 tid,	106	void ReportDeadlySignal(const SignalContext &sig, u32 tid,

lib/tsan/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp+71-62

...	@@ -13,25 +13,25 @@	...	@@ -13,25 +13,25 @@
13		13
14	#include "sanitizer_platform.h"	14	#include "sanitizer_platform.h"
15	#if SANITIZER_POSIX	15	#if SANITIZER_POSIX
16	#include "sanitizer_allocator_internal.h"	16	# include <dlfcn.h> // for dlsym()
17	#include "sanitizer_common.h"	17	# include <errno.h>
18	#include "sanitizer_file.h"	18	# include <stdint.h>
19	#include "sanitizer_flags.h"	19	# include <stdlib.h>
20	#include "sanitizer_internal_defs.h"	20	# include <sys/wait.h>
21	#include "sanitizer_linux.h"	21	# include <unistd.h>
22	#include "sanitizer_placement_new.h"	22
23	#include "sanitizer_posix.h"	23	# include "sanitizer_allocator_internal.h"
24	#include "sanitizer_procmaps.h"	24	# include "sanitizer_common.h"
25	#include "sanitizer_symbolizer_internal.h"	25	# include "sanitizer_file.h"
26	#include "sanitizer_symbolizer_libbacktrace.h"	26	# include "sanitizer_flags.h"
27	#include "sanitizer_symbolizer_mac.h"	27	# include "sanitizer_internal_defs.h"
28		28	# include "sanitizer_linux.h"
29	#include <dlfcn.h> // for dlsym()	29	# include "sanitizer_placement_new.h"
30	#include <errno.h>	30	# include "sanitizer_posix.h"
31	#include <stdint.h>	31	# include "sanitizer_procmaps.h"
32	#include <stdlib.h>	32	# include "sanitizer_symbolizer_internal.h"
33	#include <sys/wait.h>	33	# include "sanitizer_symbolizer_libbacktrace.h"
34	#include <unistd.h>	34	# include "sanitizer_symbolizer_mac.h"
35		35
36	// C++ demangling function, as required by Itanium C++ ABI. This is weak,	36	// C++ demangling function, as required by Itanium C++ ABI. This is weak,
37	// because we do not require a C++ ABI library to be linked to a program	37	// because we do not require a C++ ABI library to be linked to a program
...	@@ -72,7 +72,6 @@ static swift_demangle_ft swift_demangle_f;	...	@@ -72,7 +72,6 @@ static swift_demangle_ft swift_demangle_f;
72	// symbolication.	72	// symbolication.
73	static void InitializeSwiftDemangler() {	73	static void InitializeSwiftDemangler() {
74	swift_demangle_f = (swift_demangle_ft)dlsym(RTLD_DEFAULT, "swift_demangle");	74	swift_demangle_f = (swift_demangle_ft)dlsym(RTLD_DEFAULT, "swift_demangle");
75	(void)dlerror(); // Cleanup error message in case of failure
76	}	75	}
77		76
78	// Attempts to demangle a Swift name. The demangler will return nullptr if a	77	// Attempts to demangle a Swift name. The demangler will return nullptr if a
...	@@ -155,7 +154,7 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {	...	@@ -155,7 +154,7 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
155	}	154	}
156		155
157	if (use_posix_spawn_) {	156	if (use_posix_spawn_) {
158	#if SANITIZER_MAC	157	#if SANITIZER_APPLE
159	fd_t fd = internal_spawn(argv, const_cast<const char **>(GetEnvP()), &pid);	158	fd_t fd = internal_spawn(argv, const_cast<const char **>(GetEnvP()), &pid);
160	if (fd == kInvalidFd) {	159	if (fd == kInvalidFd) {
161	Report("WARNING: failed to spawn external symbolizer (errno: %d)\n",	160	Report("WARNING: failed to spawn external symbolizer (errno: %d)\n",
...	@@ -165,9 +164,9 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {	...	@@ -165,9 +164,9 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
165		164
166	input_fd_ = fd;	165	input_fd_ = fd;
167	output_fd_ = fd;	166	output_fd_ = fd;
168	#else // SANITIZER_MAC	167	#else // SANITIZER_APPLE
169	UNIMPLEMENTED();	168	UNIMPLEMENTED();
170	#endif // SANITIZER_MAC	169	#endif // SANITIZER_APPLE
171	} else {	170	} else {
172	fd_t infd[2] = {}, outfd[2] = {};	171	fd_t infd[2] = {}, outfd[2] = {};
173	if (!CreateTwoHighNumberedPipes(infd, outfd)) {	172	if (!CreateTwoHighNumberedPipes(infd, outfd)) {
...	@@ -213,31 +212,36 @@ class Addr2LineProcess final : public SymbolizerProcess {	...	@@ -213,31 +212,36 @@ class Addr2LineProcess final : public SymbolizerProcess {
213	const char *(&argv)[kArgVMax]) const override {	212	const char *(&argv)[kArgVMax]) const override {
214	int i = 0;	213	int i = 0;
215	argv[i++] = path_to_binary;	214	argv[i++] = path_to_binary;
216	argv[i++] = "-iCfe";	215	if (common_flags()->demangle)
		216	argv[i++] = "-C";
		217	if (common_flags()->symbolize_inline_frames)
		218	argv[i++] = "-i";
		219	argv[i++] = "-fe";
217	argv[i++] = module_name_;	220	argv[i++] = module_name_;
218	argv[i++] = nullptr;	221	argv[i++] = nullptr;
		222	CHECK_LE(i, kArgVMax);
219	}	223	}
220		224
221	bool ReachedEndOfOutput(const char *buffer, uptr length) const override;	225	bool ReachedEndOfOutput(const char *buffer, uptr length) const override;
222		226
223	bool ReadFromSymbolizer(char *buffer, uptr max_length) override {	227	bool ReadFromSymbolizer() override {
224	if (!SymbolizerProcess::ReadFromSymbolizer(buffer, max_length))	228	if (!SymbolizerProcess::ReadFromSymbolizer())
225	return false;	229	return false;
226	// The returned buffer is empty when output is valid, but exceeds	230	auto &buff = GetBuff();
227	// max_length.
228	if (*buffer == '\0')
229	return true;
230	// We should cut out output_terminator_ at the end of given buffer,	231	// We should cut out output_terminator_ at the end of given buffer,
231	// appended by addr2line to mark the end of its meaningful output.	232	// appended by addr2line to mark the end of its meaningful output.
232	// We cannot scan buffer from it's beginning, because it is legal for it	233	// We cannot scan buffer from it's beginning, because it is legal for it
233	// to start with output_terminator_ in case given offset is invalid. So,	234	// to start with output_terminator_ in case given offset is invalid. So,
234	// scanning from second character.	235	// scanning from second character.
235	char *garbage = internal_strstr(buffer + 1, output_terminator_);	236	char *garbage = internal_strstr(buff.data() + 1, output_terminator_);
236	// This should never be NULL since buffer must end up with	237	// This should never be NULL since buffer must end up with
237	// output_terminator_.	238	// output_terminator_.
238	CHECK(garbage);	239	CHECK(garbage);
		240
239	// Trim the buffer.	241	// Trim the buffer.
240	garbage[0] = '\0';	242	uintptr_t new_size = garbage - buff.data();
		243	GetBuff().resize(new_size);
		244	GetBuff().push_back('\0');
241	return true;	245	return true;
242	}	246	}
243		247
...	@@ -312,37 +316,42 @@ class Addr2LinePool final : public SymbolizerTool {	...	@@ -312,37 +316,42 @@ class Addr2LinePool final : public SymbolizerTool {
312	FIRST_32_SECOND_64(UINT32_MAX, UINT64_MAX);	316	FIRST_32_SECOND_64(UINT32_MAX, UINT64_MAX);
313	};	317	};
314		318
315	#if SANITIZER_SUPPORTS_WEAK_HOOKS	319	# if SANITIZER_SUPPORTS_WEAK_HOOKS
316	extern "C" {	320	extern "C" {
317	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool	321	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
318	__sanitizer_symbolize_code(const char *ModuleName, u64 ModuleOffset,	322	__sanitizer_symbolize_code(const char *ModuleName, u64 ModuleOffset,
319	char *Buffer, int MaxLength,	323	char *Buffer, int MaxLength);
320	bool SymbolizeInlineFrames);	324	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
321	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	325	__sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset,
322	bool __sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset,	326	char *Buffer, int MaxLength);
323	char *Buffer, int MaxLength);	327	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void
324	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	328	__sanitizer_symbolize_flush();
325	void __sanitizer_symbolize_flush();	329	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int
326	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE	330	__sanitizer_symbolize_demangle(const char Name, char Buffer, int MaxLength);
327	int __sanitizer_symbolize_demangle(const char Name, char Buffer,	331	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
328	int MaxLength);	332	__sanitizer_symbolize_set_demangle(bool Demangle);
		333	SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool
		334	__sanitizer_symbolize_set_inline_frames(bool InlineFrames);
329	} // extern "C"	335	} // extern "C"
330		336
331	class InternalSymbolizer final : public SymbolizerTool {	337	class InternalSymbolizer final : public SymbolizerTool {
332	public:	338	public:
333	static InternalSymbolizer get(LowLevelAllocator alloc) {	339	static InternalSymbolizer get(LowLevelAllocator alloc) {
334	if (__sanitizer_symbolize_code != 0 &&	340	if (__sanitizer_symbolize_set_demangle)
335	__sanitizer_symbolize_data != 0) {	341	CHECK(__sanitizer_symbolize_set_demangle(common_flags()->demangle));
336	return new(*alloc) InternalSymbolizer();	342	if (__sanitizer_symbolize_set_inline_frames)
337	}	343	CHECK(__sanitizer_symbolize_set_inline_frames(
		344	common_flags()->symbolize_inline_frames));
		345	if (__sanitizer_symbolize_code && __sanitizer_symbolize_data)
		346	return new (*alloc) InternalSymbolizer();
338	return 0;	347	return 0;
339	}	348	}
340		349
341	bool SymbolizePC(uptr addr, SymbolizedStack *stack) override {	350	bool SymbolizePC(uptr addr, SymbolizedStack *stack) override {
342	bool result = __sanitizer_symbolize_code(	351	bool result = __sanitizer_symbolize_code(
343	stack->info.module, stack->info.module_offset, buffer_, kBufferSize,	352	stack->info.module, stack->info.module_offset, buffer_, kBufferSize);
344	common_flags()->symbolize_inline_frames);	353	if (result)
345	if (result) ParseSymbolizePCOutput(buffer_, stack);	354	ParseSymbolizePCOutput(buffer_, stack);
346	return result;	355	return result;
347	}	356	}
348		357
...	@@ -365,7 +374,7 @@ class InternalSymbolizer final : public SymbolizerTool {	...	@@ -365,7 +374,7 @@ class InternalSymbolizer final : public SymbolizerTool {
365	if (__sanitizer_symbolize_demangle) {	374	if (__sanitizer_symbolize_demangle) {
366	for (uptr res_length = 1024;	375	for (uptr res_length = 1024;
367	res_length <= InternalSizeClassMap::kMaxSize;) {	376	res_length <= InternalSizeClassMap::kMaxSize;) {
368	char res_buff = static_cast<char>(InternalAlloc(res_length));	377	char res_buff = static_cast<char >(InternalAlloc(res_length));
369	uptr req_length =	378	uptr req_length =
370	__sanitizer_symbolize_demangle(name, res_buff, res_length);	379	__sanitizer_symbolize_demangle(name, res_buff, res_length);
371	if (req_length > res_length) {	380	if (req_length > res_length) {
...	@@ -380,19 +389,19 @@ class InternalSymbolizer final : public SymbolizerTool {	...	@@ -380,19 +389,19 @@ class InternalSymbolizer final : public SymbolizerTool {
380	}	389	}
381		390
382	private:	391	private:
383	InternalSymbolizer() { }	392	InternalSymbolizer() {}
384		393
385	static const int kBufferSize = 16 * 1024;	394	static const int kBufferSize = 16 * 1024;
386	char buffer_[kBufferSize];	395	char buffer_[kBufferSize];
387	};	396	};
388	#else // SANITIZER_SUPPORTS_WEAK_HOOKS	397	# else // SANITIZER_SUPPORTS_WEAK_HOOKS
389		398
390	class InternalSymbolizer final : public SymbolizerTool {	399	class InternalSymbolizer final : public SymbolizerTool {
391	public:	400	public:
392	static InternalSymbolizer get(LowLevelAllocator alloc) { return 0; }	401	static InternalSymbolizer get(LowLevelAllocator alloc) { return 0; }
393	};	402	};
394		403
395	#endif // SANITIZER_SUPPORTS_WEAK_HOOKS	404	# endif // SANITIZER_SUPPORTS_WEAK_HOOKS
396		405
397	const char Symbolizer::PlatformDemangle(const char name) {	406	const char Symbolizer::PlatformDemangle(const char name) {
398	return DemangleSwiftAndCXX(name);	407	return DemangleSwiftAndCXX(name);
...	@@ -417,13 +426,13 @@ static SymbolizerTool ChooseExternalSymbolizer(LowLevelAllocator allocator) {	...	@@ -417,13 +426,13 @@ static SymbolizerTool ChooseExternalSymbolizer(LowLevelAllocator allocator) {
417	VReport(2, "Using llvm-symbolizer at user-specified path: %s\n", path);	426	VReport(2, "Using llvm-symbolizer at user-specified path: %s\n", path);
418	return new(*allocator) LLVMSymbolizer(path, allocator);	427	return new(*allocator) LLVMSymbolizer(path, allocator);
419	} else if (!internal_strcmp(binary_name, "atos")) {	428	} else if (!internal_strcmp(binary_name, "atos")) {
420	#if SANITIZER_MAC	429	#if SANITIZER_APPLE
421	VReport(2, "Using atos at user-specified path: %s\n", path);	430	VReport(2, "Using atos at user-specified path: %s\n", path);
422	return new(*allocator) AtosSymbolizer(path, allocator);	431	return new(*allocator) AtosSymbolizer(path, allocator);
423	#else // SANITIZER_MAC	432	#else // SANITIZER_APPLE
424	Report("ERROR: Using `atos` is only supported on Darwin.\n");	433	Report("ERROR: Using `atos` is only supported on Darwin.\n");
425	Die();	434	Die();
426	#endif // SANITIZER_MAC	435	#endif // SANITIZER_APPLE
427	} else if (!internal_strcmp(binary_name, "addr2line")) {	436	} else if (!internal_strcmp(binary_name, "addr2line")) {
428	VReport(2, "Using addr2line at user-specified path: %s\n", path);	437	VReport(2, "Using addr2line at user-specified path: %s\n", path);
429	return new(*allocator) Addr2LinePool(path, allocator);	438	return new(*allocator) Addr2LinePool(path, allocator);
...	@@ -436,12 +445,12 @@ static SymbolizerTool ChooseExternalSymbolizer(LowLevelAllocator allocator) {	...	@@ -436,12 +445,12 @@ static SymbolizerTool ChooseExternalSymbolizer(LowLevelAllocator allocator) {
436		445
437	// Otherwise symbolizer program is unknown, let's search $PATH	446	// Otherwise symbolizer program is unknown, let's search $PATH
438	CHECK(path == nullptr);	447	CHECK(path == nullptr);
439	#if SANITIZER_MAC	448	#if SANITIZER_APPLE
440	if (const char *found_path = FindPathToBinary("atos")) {	449	if (const char *found_path = FindPathToBinary("atos")) {
441	VReport(2, "Using atos found at: %s\n", found_path);	450	VReport(2, "Using atos found at: %s\n", found_path);
442	return new(*allocator) AtosSymbolizer(found_path, allocator);	451	return new(*allocator) AtosSymbolizer(found_path, allocator);
443	}	452	}
444	#endif // SANITIZER_MAC	453	#endif // SANITIZER_APPLE
445	if (const char *found_path = FindPathToBinary("llvm-symbolizer")) {	454	if (const char *found_path = FindPathToBinary("llvm-symbolizer")) {
446	VReport(2, "Using llvm-symbolizer found at: %s\n", found_path);	455	VReport(2, "Using llvm-symbolizer found at: %s\n", found_path);
447	return new(*allocator) LLVMSymbolizer(found_path, allocator);	456	return new(*allocator) LLVMSymbolizer(found_path, allocator);
...	@@ -478,10 +487,10 @@ static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,	...	@@ -478,10 +487,10 @@ static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,
478	list->push_back(tool);	487	list->push_back(tool);
479	}	488	}
480		489
481	#if SANITIZER_MAC	490	#if SANITIZER_APPLE
482	VReport(2, "Using dladdr symbolizer.\n");	491	VReport(2, "Using dladdr symbolizer.\n");
483	list->push_back(new(*allocator) DlAddrSymbolizer());	492	list->push_back(new(*allocator) DlAddrSymbolizer());
484	#endif // SANITIZER_MAC	493	#endif // SANITIZER_APPLE
485	}	494	}
486		495
487	Symbolizer *Symbolizer::PlatformInit() {	496	Symbolizer *Symbolizer::PlatformInit() {
...	@@ -492,7 +501,7 @@ Symbolizer *Symbolizer::PlatformInit() {	...	@@ -492,7 +501,7 @@ Symbolizer *Symbolizer::PlatformInit() {
492	}	501	}
493		502
494	void Symbolizer::LateInitialize() {	503	void Symbolizer::LateInitialize() {
495	Symbolizer::GetOrInit()->LateInitializeTools();	504	Symbolizer::GetOrInit();
496	InitializeSwiftDemangler();	505	InitializeSwiftDemangler();
497	}	506	}
498		507

lib/tsan/sanitizer_common/sanitizer_symbolizer_report.cpp+11-6

 @@ -88,11 +88,17 @@ void ReportErrorSummary(const char *error_type, const StackTrace *stack,
 #endif
 }
-void ReportMmapWriteExec(int prot) {
+void ReportMmapWriteExec(int prot, int flags) {
 #if SANITIZER_POSIX && (!SANITIZER_GO && !SANITIZER_ANDROID)
-  if ((prot & (PROT_WRITE | PROT_EXEC)) != (PROT_WRITE | PROT_EXEC))
+  int pflags = (PROT_WRITE | PROT_EXEC);
+  if ((prot & pflags) != pflags)
     return;
+#  if SANITIZER_APPLE && defined(MAP_JIT)
+  if ((flags & MAP_JIT) == MAP_JIT)
+    return;
+#  endif
   ScopedErrorReportLock l;
   SanitizerCommonDecorator d;
 @@ -101,8 +107,7 @@ void ReportMmapWriteExec(int prot) {
   stack->Reset();
   uptr top = 0;
   uptr bottom = 0;
-  GET_CALLER_PC_BP_SP;
+  GET_CALLER_PC_BP;
-  (void)sp;
   bool fast = common_flags()->fast_unwind_on_fatal;
   if (StackTrace::WillUseFastUnwind(fast)) {
     GetThreadStackTopAndBottom(false, &top, &bottom);
 @@ -205,9 +210,9 @@ static void ReportDeadlySignalImpl(const SignalContext &sig, u32 tid,
     Report("Hint: pc points to the zero page.\n");
   if (sig.is_memory_access) {
     const char *access_type =
-        sig.write_flag == SignalContext::WRITE
+        sig.write_flag == SignalContext::Write
             ? "WRITE"
-            : (sig.write_flag == SignalContext::READ ? "READ" : "UNKNOWN");
+            : (sig.write_flag == SignalContext::Read ? "READ" : "UNKNOWN");
     Report("The signal is caused by a %s memory access.\n", access_type);
     if (!sig.is_true_faulting_addr)
       Report("Hint: this fault was caused by a dereference of a high value "

lib/tsan/sanitizer_common/sanitizer_symbolizer_win.cpp+8-10

 @@ -14,8 +14,8 @@
 #include "sanitizer_platform.h"
 #if SANITIZER_WINDOWS
-#include "sanitizer_dbghelp.h"
+#  include "sanitizer_dbghelp.h"
-#include "sanitizer_symbolizer_internal.h"
+#  include "sanitizer_symbolizer_internal.h"
 namespace __sanitizer {
 @@ -231,8 +231,6 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() {
     // Check that tool command lines are simple and that complete escaping is
     // unnecessary.
     CHECK(!internal_strchr(arg, '"') && "quotes in args unsupported");
-    CHECK(!internal_strstr(arg, "\\\\") &&
-          "double backslashes in args unsupported");
     CHECK(arglen > 0 && arg[arglen - 1] != '\\' &&
           "args ending in backslash and empty args unsupported");
     command_line.append("\"%s\" ", arg);
 @@ -294,15 +292,15 @@ static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list,
   const char *path =
       user_path ? user_path : FindPathToBinary("llvm-symbolizer.exe");
   if (path) {
-    VReport(2, "Using llvm-symbolizer at %spath: %s\n",
-            user_path ? "user-specified " : "", path);
-    list->push_back(new(*allocator) LLVMSymbolizer(path, allocator));
-  } else {
     if (user_path && user_path[0] == '\0') {
       VReport(2, "External symbolizer is explicitly disabled.\n");
     } else {
-      VReport(2, "External symbolizer is not present.\n");
+      VReport(2, "Using llvm-symbolizer at %spath: %s\n",
+              user_path ? "user-specified " : "", path);
+      list->push_back(new (*allocator) LLVMSymbolizer(path, allocator));
     }
+  } else {
+    VReport(2, "External symbolizer is not present.\n");
   }
   // Add the dbghelp based symbolizer.
 @@ -318,7 +316,7 @@ Symbolizer *Symbolizer::PlatformInit() {
 }
 void Symbolizer::LateInitialize() {
-  Symbolizer::GetOrInit()->LateInitializeTools();
+  Symbolizer::GetOrInit();
 }
 }  // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_syscall_generic.inc+3-2

 @@ -13,13 +13,14 @@
 // NetBSD uses libc calls directly
 #if !SANITIZER_NETBSD
-#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_SOLARIS
+#if SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_SOLARIS
 # define SYSCALL(name) SYS_ ## name
 #else
 # define SYSCALL(name) __NR_ ## name
 #endif
-#if defined(__x86_64__) && (SANITIZER_FREEBSD || SANITIZER_MAC)
+#if (defined(__x86_64__) && (SANITIZER_FREEBSD || SANITIZER_APPLE)) || \
+    (defined(__aarch64__) && SANITIZER_FREEBSD)
 # define internal_syscall __syscall
 # else
 # define internal_syscall syscall

lib/tsan/sanitizer_common/sanitizer_syscalls_netbsd.inc+2-2

...	@@ -2255,13 +2255,13 @@ PRE_SYSCALL(getcontext)(void ucp_) { / Nothing to do */ }	...	@@ -2255,13 +2255,13 @@ PRE_SYSCALL(getcontext)(void ucp_) { / Nothing to do */ }
2255	POST_SYSCALL(getcontext)(long long res, void ucp_) { / Nothing to do */ }	2255	POST_SYSCALL(getcontext)(long long res, void ucp_) { / Nothing to do */ }
2256	PRE_SYSCALL(setcontext)(void *ucp_) {	2256	PRE_SYSCALL(setcontext)(void *ucp_) {
2257	if (ucp_) {	2257	if (ucp_) {
2258	PRE_READ(ucp_, ucontext_t_sz);	2258	PRE_READ(ucp_, ucontext_t_sz(ucp_));
2259	}	2259	}
2260	}	2260	}
2261	POST_SYSCALL(setcontext)(long long res, void *ucp_) {}	2261	POST_SYSCALL(setcontext)(long long res, void *ucp_) {}
2262	PRE_SYSCALL(_lwp_create)(void ucp_, long long flags_, void new_lwp_) {	2262	PRE_SYSCALL(_lwp_create)(void ucp_, long long flags_, void new_lwp_) {
2263	if (ucp_) {	2263	if (ucp_) {
2264	PRE_READ(ucp_, ucontext_t_sz);	2264	PRE_READ(ucp_, ucontext_t_sz(ucp_));
2265	}	2265	}
2266	}	2266	}
2267	POST_SYSCALL(_lwp_create)	2267	POST_SYSCALL(_lwp_create)

lib/tsan/sanitizer_common/sanitizer_thread_arg_retval.cpp created+94

...	@@ -0,0 +1,94 @@
1	//===-- sanitizer_thread_arg_retval.cpp -------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between sanitizer tools.
10	//
11	// Tracks thread arguments and return value for leak checking.
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_thread_arg_retval.h"
15
16	#include "sanitizer_placement_new.h"
17
18	namespace __sanitizer {
19
20	void ThreadArgRetval::CreateLocked(uptr thread, bool detached,
21	const Args& args) {
22	CheckLocked();
23	Data& t = data_[thread];
24	t = {};
25	t.gen = gen_++;
26	t.detached = detached;
27	t.args = args;
28	}
29
30	ThreadArgRetval::Args ThreadArgRetval::GetArgs(uptr thread) const {
31	__sanitizer::Lock lock(&mtx_);
32	auto t = data_.find(thread);
33	CHECK(t);
34	if (t->second.done)
35	return {};
36	return t->second.args;
37	}
38
39	void ThreadArgRetval::Finish(uptr thread, void* retval) {
40	__sanitizer::Lock lock(&mtx_);
41	auto t = data_.find(thread);
42	if (!t)
43	return;
44	if (t->second.detached) {
45	// Retval of detached thread connot be retrieved.
46	data_.erase(t);
47	return;
48	}
49	t->second.done = true;
50	t->second.args.arg_retval = retval;
51	}
52
53	u32 ThreadArgRetval::BeforeJoin(uptr thread) const {
54	__sanitizer::Lock lock(&mtx_);
55	auto t = data_.find(thread);
56	CHECK(t);
57	CHECK(!t->second.detached);
58	return t->second.gen;
59	}
60
61	void ThreadArgRetval::AfterJoin(uptr thread, u32 gen) {
62	__sanitizer::Lock lock(&mtx_);
63	auto t = data_.find(thread);
64	if (!t \|\| gen != t->second.gen) {
65	// Thread was reused and erased by any other event.
66	return;
67	}
68	CHECK(!t->second.detached);
69	data_.erase(t);
70	}
71
72	void ThreadArgRetval::DetachLocked(uptr thread) {
73	CheckLocked();
74	auto t = data_.find(thread);
75	CHECK(t);
76	CHECK(!t->second.detached);
77	if (t->second.done) {
78	// We can't retrive retval after detached thread finished.
79	data_.erase(t);
80	return;
81	}
82	t->second.detached = true;
83	}
84
85	void ThreadArgRetval::GetAllPtrsLocked(InternalMmapVector<uptr>* ptrs) {
86	CheckLocked();
87	CHECK(ptrs);
88	data_.forEach([&](DenseMap<uptr, Data>::value_type& kv) -> bool {
89	ptrs->push_back((uptr)kv.second.args.arg_retval);
90	return true;
91	});
92	}
93
94	} // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_thread_arg_retval.h created+116

...	@@ -0,0 +1,116 @@
1	//===-- sanitizer_thread_arg_retval.h ---------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is shared between sanitizer tools.
10	//
11	// Tracks thread arguments and return value for leak checking.
12	//===----------------------------------------------------------------------===//
13
14	#ifndef SANITIZER_THREAD_ARG_RETVAL_H
15	#define SANITIZER_THREAD_ARG_RETVAL_H
16
17	#include "sanitizer_common.h"
18	#include "sanitizer_dense_map.h"
19	#include "sanitizer_list.h"
20	#include "sanitizer_mutex.h"
21
22	namespace __sanitizer {
23
24	// Primary goal of the class is to keep alive arg and retval pointer for leak
25	// checking. However it can be used to pass those pointer into wrappers used by
26	// interceptors. The difference from ThreadRegistry/ThreadList is that this
27	// class keeps data up to the detach or join, as exited thread still can be
28	// joined to retrive retval. ThreadRegistry/ThreadList can discard exited
29	// threads immediately.
30	class SANITIZER_MUTEX ThreadArgRetval {
31	public:
32	struct Args {
33	void* (routine)(void);
34	void* arg_retval; // Either arg or retval.
35	};
36	void Lock() SANITIZER_ACQUIRE() { mtx_.Lock(); }
37	void CheckLocked() const SANITIZER_CHECK_LOCKED() { mtx_.CheckLocked(); }
38	void Unlock() SANITIZER_RELEASE() { mtx_.Unlock(); }
39
40	// Wraps pthread_create or similar. We need to keep object locked, to
41	// prevent child thread from proceeding without thread handle.
42	template <typename CreateFn /* returns thread id on success, or 0 */>
43	void Create(bool detached, const Args& args, const CreateFn& fn) {
44	// No need to track detached threads with no args, but we will to do as it's
45	// not expensive and less edge-cases.
46	__sanitizer::Lock lock(&mtx_);
47	if (uptr thread = fn())
48	CreateLocked(thread, detached, args);
49	}
50
51	// Returns thread arg and routine.
52	Args GetArgs(uptr thread) const;
53
54	// Mark thread as done and stores retval or remove if detached. Should be
55	// called by the thread.
56	void Finish(uptr thread, void* retval);
57
58	// Mark thread as detached or remove if done.
59	template <typename DetachFn /* returns true on success */>
60	void Detach(uptr thread, const DetachFn& fn) {
61	// Lock to prevent re-use of the thread between fn() and DetachLocked()
62	// calls.
63	__sanitizer::Lock lock(&mtx_);
64	if (fn())
65	DetachLocked(thread);
66	}
67
68	// Joins the thread.
69	template <typename JoinFn /* returns true on success */>
70	void Join(uptr thread, const JoinFn& fn) {
71	// Remember internal id of the thread to prevent re-use of the thread
72	// between fn() and AfterJoin() calls. Locking JoinFn, like in
73	// Detach(), implementation can cause deadlock.
74	auto gen = BeforeJoin(thread);
75	if (fn())
76	AfterJoin(thread, gen);
77	}
78
79	// Returns all arg and retval which are considered alive.
80	void GetAllPtrsLocked(InternalMmapVector<uptr>* ptrs);
81
82	uptr size() const {
83	__sanitizer::Lock lock(&mtx_);
84	return data_.size();
85	}
86
87	// FIXME: Add fork support. Expected users of the class are sloppy with forks
88	// anyway. We likely should lock/unlock the object to avoid deadlocks, and
89	// erase all but the current threads, so we can detect leaked arg or retval in
90	// child process.
91
92	// FIXME: Add cancelation support. Now if a thread was canceled, the class
93	// will keep pointers alive forever, missing leaks caused by cancelation.
94
95	private:
96	struct Data {
97	Args args;
98	u32 gen; // Avoid collision if thread id re-used.
99	bool detached;
100	bool done;
101	};
102
103	void CreateLocked(uptr thread, bool detached, const Args& args);
104	u32 BeforeJoin(uptr thread) const;
105	void AfterJoin(uptr thread, u32 gen);
106	void DetachLocked(uptr thread);
107
108	mutable Mutex mtx_;
109
110	DenseMap<uptr, Data> data_;
111	u32 gen_ = 0;
112	};
113
114	} // namespace __sanitizer
115
116	#endif // SANITIZER_THREAD_ARG_RETVAL_H

lib/tsan/sanitizer_common/sanitizer_thread_registry.cpp+59-21

 @@ -13,6 +13,8 @@
 #include "sanitizer_thread_registry.h"
+#include "sanitizer_placement_new.h"
 namespace __sanitizer {
 ThreadContextBase::ThreadContextBase(u32 tid)
 @@ -108,7 +110,7 @@ ThreadRegistry::ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
       max_threads_(max_threads),
       thread_quarantine_size_(thread_quarantine_size),
       max_reuse_(max_reuse),
-      mtx_(),
+      mtx_(MutexThreadRegistry),
       total_threads_(0),
       alive_threads_(0),
       max_alive_threads_(0),
 @@ -119,7 +121,7 @@ ThreadRegistry::ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
 void ThreadRegistry::GetNumberOfThreads(uptr *total, uptr *running,
                                         uptr *alive) {
-  BlockingMutexLock l(&mtx_);
+  ThreadRegistryLock l(this);
   if (total)
     *total = threads_.size();
   if (running) *running = running_threads_;
 @@ -127,13 +129,13 @@ void ThreadRegistry::GetNumberOfThreads(uptr *total, uptr *running,
 }
 uptr ThreadRegistry::GetMaxAliveThreads() {
-  BlockingMutexLock l(&mtx_);
+  ThreadRegistryLock l(this);
   return max_alive_threads_;
 }
 u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
                                  void *arg) {
-  BlockingMutexLock l(&mtx_);
+  ThreadRegistryLock l(this);
   u32 tid = kInvalidTid;
   ThreadContextBase *tctx = QuarantinePop();
   if (tctx) {
 @@ -162,6 +164,12 @@ u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid,
     max_alive_threads_++;
     CHECK_EQ(alive_threads_, max_alive_threads_);
   }
+  if (user_id) {
+    // Ensure that user_id is unique. If it's not the case we are screwed.
+    // Ignoring this situation may lead to very hard to debug false
+    // positives later (e.g. if we join a wrong thread).
+    CHECK(live_.try_emplace(user_id, tid).second);
+  }
   tctx->SetCreated(user_id, total_threads_++, detached,
                    parent_tid, arg);
   return tid;
 @@ -179,7 +187,7 @@ void ThreadRegistry::RunCallbackForEachThreadLocked(ThreadCallback cb,
 }
 u32 ThreadRegistry::FindThread(FindThreadCallback cb, void *arg) {
-  BlockingMutexLock l(&mtx_);
+  ThreadRegistryLock l(this);
   for (u32 tid = 0; tid < threads_.size(); tid++) {
     ThreadContextBase *tctx = threads_[tid];
     if (tctx != 0 && cb(tctx, arg))
 @@ -211,7 +219,7 @@ ThreadContextBase *ThreadRegistry::FindThreadContextByOsIDLocked(tid_t os_id) {
 }
 void ThreadRegistry::SetThreadName(u32 tid, const char *name) {
-  BlockingMutexLock l(&mtx_);
+  ThreadRegistryLock l(this);
   ThreadContextBase *tctx = threads_[tid];
   CHECK_NE(tctx, 0);
   CHECK_EQ(SANITIZER_FUCHSIA ? ThreadStatusCreated : ThreadStatusRunning,
 @@ -220,19 +228,13 @@ void ThreadRegistry::SetThreadName(u32 tid, const char *name) {
 }
 void ThreadRegistry::SetThreadNameByUserId(uptr user_id, const char *name) {
-  BlockingMutexLock l(&mtx_);
+  ThreadRegistryLock l(this);
-  for (u32 tid = 0; tid < threads_.size(); tid++) {
+  if (const auto *tid = live_.find(user_id))
-    ThreadContextBase *tctx = threads_[tid];
+    threads_[tid->second]->SetName(name);
-    if (tctx != 0 && tctx->user_id == user_id &&
-        tctx->status != ThreadStatusInvalid) {
-      tctx->SetName(name);
-      return;
-    }
-  }
 }
 void ThreadRegistry::DetachThread(u32 tid, void *arg) {
-  BlockingMutexLock l(&mtx_);
+  ThreadRegistryLock l(this);
   ThreadContextBase *tctx = threads_[tid];
   CHECK_NE(tctx, 0);
   if (tctx->status == ThreadStatusInvalid) {
 @@ -241,6 +243,8 @@ void ThreadRegistry::DetachThread(u32 tid, void *arg) {
   }
   tctx->OnDetached(arg);
   if (tctx->status == ThreadStatusFinished) {
+    if (tctx->user_id)
+      live_.erase(tctx->user_id);
     tctx->SetDead();
     QuarantinePush(tctx);
   } else {
 @@ -252,7 +256,7 @@ void ThreadRegistry::JoinThread(u32 tid, void *arg) {
   bool destroyed = false;
   do {
     {
-      BlockingMutexLock l(&mtx_);
+      ThreadRegistryLock l(this);
       ThreadContextBase *tctx = threads_[tid];
       CHECK_NE(tctx, 0);
       if (tctx->status == ThreadStatusInvalid) {
 @@ -260,6 +264,8 @@ void ThreadRegistry::JoinThread(u32 tid, void *arg) {
         return;
       }
       if ((destroyed = tctx->GetDestroyed())) {
+        if (tctx->user_id)
+          live_.erase(tctx->user_id);
         tctx->SetJoined(arg);
         QuarantinePush(tctx);
       }
 @@ -275,7 +281,7 @@ void ThreadRegistry::JoinThread(u32 tid, void *arg) {
 // thread before trying to create it, and then failed to actually
 // create it, and so never called StartThread.
 ThreadStatus ThreadRegistry::FinishThread(u32 tid) {
-  BlockingMutexLock l(&mtx_);
+  ThreadRegistryLock l(this);
   CHECK_GT(alive_threads_, 0);
   alive_threads_--;
   ThreadContextBase *tctx = threads_[tid];
 @@ -292,6 +298,8 @@ ThreadStatus ThreadRegistry::FinishThread(u32 tid) {
   }
   tctx->SetFinished();
   if (dead) {
+    if (tctx->user_id)
+      live_.erase(tctx->user_id);
     tctx->SetDead();
     QuarantinePush(tctx);
   }
 @@ -301,7 +309,7 @@ ThreadStatus ThreadRegistry::FinishThread(u32 tid) {
 void ThreadRegistry::StartThread(u32 tid, tid_t os_id, ThreadType thread_type,
                                  void *arg) {
-  BlockingMutexLock l(&mtx_);
+  ThreadRegistryLock l(this);
   running_threads_++;
   ThreadContextBase *tctx = threads_[tid];
   CHECK_NE(tctx, 0);
 @@ -327,20 +335,50 @@ void ThreadRegistry::QuarantinePush(ThreadContextBase *tctx) {
 ThreadContextBase *ThreadRegistry::QuarantinePop() {
   if (invalid_threads_.size() == 0)
-    return 0;
+    return nullptr;
   ThreadContextBase *tctx = invalid_threads_.front();
   invalid_threads_.pop_front();
   return tctx;
 }
+u32 ThreadRegistry::ConsumeThreadUserId(uptr user_id) {
+  ThreadRegistryLock l(this);
+  u32 tid;
+  auto *t = live_.find(user_id);
+  CHECK(t);
+  tid = t->second;
+  live_.erase(t);
+  auto *tctx = threads_[tid];
+  CHECK_EQ(tctx->user_id, user_id);
+  tctx->user_id = 0;
+  return tid;
+}
 void ThreadRegistry::SetThreadUserId(u32 tid, uptr user_id) {
-  BlockingMutexLock l(&mtx_);
+  ThreadRegistryLock l(this);
   ThreadContextBase *tctx = threads_[tid];
   CHECK_NE(tctx, 0);
   CHECK_NE(tctx->status, ThreadStatusInvalid);
   CHECK_NE(tctx->status, ThreadStatusDead);
   CHECK_EQ(tctx->user_id, 0);
   tctx->user_id = user_id;
+  CHECK(live_.try_emplace(user_id, tctx->tid).second);
+}
+u32 ThreadRegistry::OnFork(u32 tid) {
+  ThreadRegistryLock l(this);
+  // We only purge user_id (pthread_t) of live threads because
+  // they cause CHECK failures if new threads with matching pthread_t
+  // created after fork.
+  // Potentially we could purge more info (ThreadContextBase themselves),
+  // but it's hard to test and easy to introduce new issues by doing this.
+  for (auto *tctx : threads_) {
+    if (tctx->tid == tid || !tctx->user_id)
+      continue;
+    CHECK(live_.erase(tctx->user_id));
+    tctx->user_id = 0;
+  }
+  return alive_threads_;
 }
 }  // namespace __sanitizer

lib/tsan/sanitizer_common/sanitizer_thread_registry.h+15-5

 @@ -15,6 +15,7 @@
 #define SANITIZER_THREAD_REGISTRY_H
 #include "sanitizer_common.h"
+#include "sanitizer_dense_map.h"
 #include "sanitizer_list.h"
 #include "sanitizer_mutex.h"
 @@ -85,7 +86,7 @@ class ThreadContextBase {
 typedef ThreadContextBase* (*ThreadContextFactory)(u32 tid);
-class MUTEX ThreadRegistry {
+class SANITIZER_MUTEX ThreadRegistry {
  public:
   ThreadRegistry(ThreadContextFactory factory);
   ThreadRegistry(ThreadContextFactory factory, u32 max_threads,
 @@ -94,15 +95,17 @@ class MUTEX ThreadRegistry {
                           uptr *alive = nullptr);
   uptr GetMaxAliveThreads();
-  void Lock() ACQUIRE() { mtx_.Lock(); }
+  void Lock() SANITIZER_ACQUIRE() { mtx_.Lock(); }
-  void CheckLocked() const CHECK_LOCKED() { mtx_.CheckLocked(); }
+  void CheckLocked() const SANITIZER_CHECK_LOCKED() { mtx_.CheckLocked(); }
-  void Unlock() RELEASE() { mtx_.Unlock(); }
+  void Unlock() SANITIZER_RELEASE() { mtx_.Unlock(); }
   // Should be guarded by ThreadRegistryLock.
   ThreadContextBase *GetThreadLocked(u32 tid) {
     return threads_.empty() ? nullptr : threads_[tid];
   }
+  u32 NumThreadsLocked() const { return threads_.size(); }
   u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg);
   typedef void (*ThreadCallback)(ThreadContextBase *tctx, void *arg);
 @@ -127,15 +130,21 @@ class MUTEX ThreadRegistry {
   // Finishes thread and returns previous status.
   ThreadStatus FinishThread(u32 tid);
   void StartThread(u32 tid, tid_t os_id, ThreadType thread_type, void *arg);
+  u32 ConsumeThreadUserId(uptr user_id);
   void SetThreadUserId(u32 tid, uptr user_id);
+  // OnFork must be called in the child process after fork to purge old
+  // threads that don't exist anymore (except for the current thread tid).
+  // Returns number of alive threads before fork.
+  u32 OnFork(u32 tid);
  private:
   const ThreadContextFactory context_factory_;
   const u32 max_threads_;
   const u32 thread_quarantine_size_;
   const u32 max_reuse_;
-  BlockingMutex mtx_;
+  Mutex mtx_;
   u64 total_threads_;   // Total number of created threads. May be greater than
                         // max_threads_ if contexts were reused.
 @@ -146,6 +155,7 @@ class MUTEX ThreadRegistry {
   InternalMmapVector<ThreadContextBase *> threads_;
   IntrusiveList<ThreadContextBase> dead_threads_;
   IntrusiveList<ThreadContextBase> invalid_threads_;
+  DenseMap<uptr, Tid> live_;
   void QuarantinePush(ThreadContextBase *tctx);
   ThreadContextBase *QuarantinePop();

lib/tsan/sanitizer_common/sanitizer_thread_safety.h+26-19

...	@@ -16,27 +16,34 @@	...	@@ -16,27 +16,34 @@
16	#define SANITIZER_THREAD_SAFETY_H	16	#define SANITIZER_THREAD_SAFETY_H
17		17
18	#if defined(__clang__)	18	#if defined(__clang__)
19	# define THREAD_ANNOTATION(x) __attribute__((x))	19	# define SANITIZER_THREAD_ANNOTATION(x) __attribute__((x))
20	#else	20	#else
21	# define THREAD_ANNOTATION(x)	21	# define SANITIZER_THREAD_ANNOTATION(x)
22	#endif	22	#endif
23		23
24	#define MUTEX THREAD_ANNOTATION(capability("mutex"))	24	#define SANITIZER_MUTEX SANITIZER_THREAD_ANNOTATION(capability("mutex"))
25	#define SCOPED_LOCK THREAD_ANNOTATION(scoped_lockable)	25	#define SANITIZER_SCOPED_LOCK SANITIZER_THREAD_ANNOTATION(scoped_lockable)
26	#define GUARDED_BY(x) THREAD_ANNOTATION(guarded_by(x))	26	#define SANITIZER_GUARDED_BY(x) SANITIZER_THREAD_ANNOTATION(guarded_by(x))
27	#define PT_GUARDED_BY(x) THREAD_ANNOTATION(pt_guarded_by(x))	27	#define SANITIZER_PT_GUARDED_BY(x) SANITIZER_THREAD_ANNOTATION(pt_guarded_by(x))
28	#define REQUIRES(...) THREAD_ANNOTATION(requires_capability(__VA_ARGS__))	28	#define SANITIZER_REQUIRES(...) \
29	#define REQUIRES_SHARED(...) \	29	SANITIZER_THREAD_ANNOTATION(requires_capability(__VA_ARGS__))
30	THREAD_ANNOTATION(requires_shared_capability(__VA_ARGS__))	30	#define SANITIZER_REQUIRES_SHARED(...) \
31	#define ACQUIRE(...) THREAD_ANNOTATION(acquire_capability(__VA_ARGS__))	31	SANITIZER_THREAD_ANNOTATION(requires_shared_capability(__VA_ARGS__))
32	#define ACQUIRE_SHARED(...) \	32	#define SANITIZER_ACQUIRE(...) \
33	THREAD_ANNOTATION(acquire_shared_capability(__VA_ARGS__))	33	SANITIZER_THREAD_ANNOTATION(acquire_capability(__VA_ARGS__))
34	#define TRY_ACQUIRE(...) THREAD_ANNOTATION(try_acquire_capability(__VA_ARGS__))	34	#define SANITIZER_ACQUIRE_SHARED(...) \
35	#define RELEASE(...) THREAD_ANNOTATION(release_capability(__VA_ARGS__))	35	SANITIZER_THREAD_ANNOTATION(acquire_shared_capability(__VA_ARGS__))
36	#define RELEASE_SHARED(...) \	36	#define SANITIZER_TRY_ACQUIRE(...) \
37	THREAD_ANNOTATION(release_shared_capability(__VA_ARGS__))	37	SANITIZER_THREAD_ANNOTATION(try_acquire_capability(__VA_ARGS__))
38	#define EXCLUDES(...) THREAD_ANNOTATION(locks_excluded(__VA_ARGS__))	38	#define SANITIZER_RELEASE(...) \
39	#define CHECK_LOCKED(...) THREAD_ANNOTATION(assert_capability(__VA_ARGS__))	39	SANITIZER_THREAD_ANNOTATION(release_capability(__VA_ARGS__))
40	#define NO_THREAD_SAFETY_ANALYSIS THREAD_ANNOTATION(no_thread_safety_analysis)	40	#define SANITIZER_RELEASE_SHARED(...) \
		41	SANITIZER_THREAD_ANNOTATION(release_shared_capability(__VA_ARGS__))
		42	#define SANITIZER_EXCLUDES(...) \
		43	SANITIZER_THREAD_ANNOTATION(locks_excluded(__VA_ARGS__))
		44	#define SANITIZER_CHECK_LOCKED(...) \
		45	SANITIZER_THREAD_ANNOTATION(assert_capability(__VA_ARGS__))
		46	#define SANITIZER_NO_THREAD_SAFETY_ANALYSIS \
		47	SANITIZER_THREAD_ANNOTATION(no_thread_safety_analysis)
41		48
42	#endif	49	#endif

lib/tsan/sanitizer_common/sanitizer_tls_get_addr.cpp+28-25

 @@ -12,6 +12,7 @@
 #include "sanitizer_tls_get_addr.h"
+#include "sanitizer_allocator_interface.h"
 #include "sanitizer_atomic.h"
 #include "sanitizer_flags.h"
 #include "sanitizer_platform_interceptors.h"
 @@ -26,13 +27,6 @@ struct TlsGetAddrParam {
   uptr offset;
 };
-// Glibc starting from 2.19 allocates tls using __signal_safe_memalign,
-// which has such header.
-struct Glibc_2_19_tls_header {
-  uptr size;
-  uptr start;
-};
 // This must be static TLS
 __attribute__((tls_model("initial-exec")))
 static __thread DTLS dtls;
 @@ -44,7 +38,7 @@ static atomic_uintptr_t number_of_live_dtls;
 static const uptr kDestroyedThread = -1;
 static void DTLS_Deallocate(DTLS::DTVBlock *block) {
-  VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", block);
+  VReport(2, "__tls_get_addr: DTLS_Deallocate %p\n", (void *)block);
   UnmapOrDie(block, sizeof(DTLS::DTVBlock));
   atomic_fetch_sub(&number_of_live_dtls, 1, memory_order_relaxed);
 }
 @@ -66,12 +60,13 @@ static DTLS::DTVBlock *DTLS_NextBlock(atomic_uintptr_t *cur) {
   }
   uptr num_live_dtls =
       atomic_fetch_add(&number_of_live_dtls, 1, memory_order_relaxed);
-  VReport(2, "__tls_get_addr: DTLS_NextBlock %p %zd\n", &dtls, num_live_dtls);
+  VReport(2, "__tls_get_addr: DTLS_NextBlock %p %zd\n", (void *)&dtls,
+          num_live_dtls);
   return new_dtv;
 }
 static DTLS::DTV *DTLS_Find(uptr id) {
-  VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", &dtls, id);
+  VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", (void *)&dtls, id);
   static constexpr uptr kPerBlock = ARRAY_SIZE(DTLS::DTVBlock::dtvs);
   DTLS::DTVBlock *cur = DTLS_NextBlock(&dtls.dtv_block);
   if (!cur)
 @@ -82,7 +77,7 @@ static DTLS::DTV *DTLS_Find(uptr id) {
 void DTLS_Destroy() {
   if (!common_flags()->intercept_tls_get_addr) return;
-  VReport(2, "__tls_get_addr: DTLS_Destroy %p\n", &dtls);
+  VReport(2, "__tls_get_addr: DTLS_Destroy %p\n", (void *)&dtls);
   DTLS::DTVBlock *block = (DTLS::DTVBlock *)atomic_exchange(
       &dtls.dtv_block, kDestroyedThread, memory_order_release);
   while (block) {
 @@ -107,6 +102,14 @@ static const uptr kDtvOffset = 0x800;
 static const uptr kDtvOffset = 0;
 #endif
+extern "C" {
+SANITIZER_WEAK_ATTRIBUTE
+uptr __sanitizer_get_allocated_size(const void *p);
+SANITIZER_WEAK_ATTRIBUTE
+const void *__sanitizer_get_allocated_begin(const void *p);
+}
 DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
                                 uptr static_tls_begin, uptr static_tls_end) {
   if (!common_flags()->intercept_tls_get_addr) return 0;
 @@ -117,26 +120,26 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
     return 0;
   uptr tls_size = 0;
   uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
-  VReport(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p "
+  VReport(2,
-             "num_live_dtls %zd\n",
+          "__tls_get_addr: %p {0x%zx,0x%zx} => %p; tls_beg: 0x%zx; sp: %p "
-          arg, arg->dso_id, arg->offset, res, tls_beg, &tls_beg,
+          "num_live_dtls %zd\n",
+          (void *)arg, arg->dso_id, arg->offset, res, tls_beg, (void *)&tls_beg,
           atomic_load(&number_of_live_dtls, memory_order_relaxed));
   if (dtls.last_memalign_ptr == tls_beg) {
     tls_size = dtls.last_memalign_size;
-    VReport(2, "__tls_get_addr: glibc <=2.18 suspected; tls={%p,%p}\n",
+    VReport(2, "__tls_get_addr: glibc <=2.24 suspected; tls={0x%zx,0x%zx}\n",
-        tls_beg, tls_size);
+            tls_beg, tls_size);
   } else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) {
     // This is the static TLS block which was initialized / unpoisoned at thread
     // creation.
-    VReport(2, "__tls_get_addr: static tls: %p\n", tls_beg);
+    VReport(2, "__tls_get_addr: static tls: 0x%zx\n", tls_beg);
     tls_size = 0;
-  } else if ((tls_beg % 4096) == sizeof(Glibc_2_19_tls_header)) {
+  } else if (const void *start =
-    // We may want to check gnu_get_libc_version().
+                 __sanitizer_get_allocated_begin((void *)tls_beg)) {
-    Glibc_2_19_tls_header *header = (Glibc_2_19_tls_header *)tls_beg - 1;
+    tls_beg = (uptr)start;
-    tls_size = header->size;
+    tls_size = __sanitizer_get_allocated_size(start);
-    tls_beg = header->start;
+    VReport(2, "__tls_get_addr: glibc >=2.25 suspected; tls={0x%zx,0x%zx}\n",
-    VReport(2, "__tls_get_addr: glibc >=2.19 suspected; tls={%p %p}\n",
+            tls_beg, tls_size);
-        tls_beg, tls_size);
   } else {
     VReport(2, "__tls_get_addr: Can't guess glibc version\n");
     // This may happen inside the DTOR of main thread, so just ignore it.
 @@ -149,7 +152,7 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
 void DTLS_on_libc_memalign(void *ptr, uptr size) {
   if (!common_flags()->intercept_tls_get_addr) return;
-  VReport(2, "DTLS_on_libc_memalign: %p %p\n", ptr, size);
+  VReport(2, "DTLS_on_libc_memalign: %p 0x%zx\n", ptr, size);
   dtls.last_memalign_ptr = reinterpret_cast<uptr>(ptr);
   dtls.last_memalign_size = size;
 }

lib/tsan/sanitizer_common/sanitizer_tls_get_addr.h+17-9

 @@ -12,16 +12,24 @@
 // the lack of interface that would tell us about the Dynamic TLS (DTLS).
 // https://sourceware.org/bugzilla/show_bug.cgi?id=16291
 //
-// The matters get worse because the glibc implementation changed between
+// Before 2.25: every DTLS chunk is allocated with __libc_memalign,
-// 2.18 and 2.19:
-// https://groups.google.com/forum/#!topic/address-sanitizer/BfwYD8HMxTM
-//
-// Before 2.19, every DTLS chunk is allocated with __libc_memalign,
 // which we intercept and thus know where is the DTLS.
-// Since 2.19, DTLS chunks are allocated with __signal_safe_memalign,
+//
-// which is an internal function that wraps a mmap call, neither of which
+// Since 2.25: DTLS chunks are allocated with malloc. We could co-opt
-// we can intercept. Luckily, __signal_safe_memalign has a simple parseable
+// the malloc interceptor to keep track of the last allocation, similar
-// header which we can use.
+// to how we handle __libc_memalign; however, this adds some overhead
+// (since malloc, unlike __libc_memalign, is commonly called), and
+// requires care to avoid false negatives for LeakSanitizer.
+// Instead, we rely on our internal allocators - which keep track of all
+// its allocations - to determine if an address points to a malloc
+// allocation.
+//
+// There exists a since-deprecated version of Google's internal glibc fork
+// that used __signal_safe_memalign. DTLS_on_tls_get_addr relied on a
+// heuristic check (is the allocation 16 bytes from the start of a page
+// boundary?), which was sometimes erroneous:
+//     https://bugs.chromium.org/p/chromium/issues/detail?id=1275223#c15
+// Since that check has no practical use anymore, we have removed it.
 //
 //===----------------------------------------------------------------------===//

lib/tsan/sanitizer_common/sanitizer_type_traits.h+79

...	@@ -13,6 +13,8 @@	...	@@ -13,6 +13,8 @@
13	#ifndef SANITIZER_TYPE_TRAITS_H	13	#ifndef SANITIZER_TYPE_TRAITS_H
14	#define SANITIZER_TYPE_TRAITS_H	14	#define SANITIZER_TYPE_TRAITS_H
15		15
		16	#include "sanitizer_common/sanitizer_internal_defs.h"
		17
16	namespace __sanitizer {	18	namespace __sanitizer {
17		19
18	struct true_type {	20	struct true_type {
...	@@ -57,6 +59,83 @@ struct conditional<false, T, F> {	...	@@ -57,6 +59,83 @@ struct conditional<false, T, F> {
57	using type = F;	59	using type = F;
58	};	60	};
59		61
		62	template <class T>
		63	struct remove_reference {
		64	using type = T;
		65	};
		66	template <class T>
		67	struct remove_reference<T&> {
		68	using type = T;
		69	};
		70	template <class T>
		71	struct remove_reference<T&&> {
		72	using type = T;
		73	};
		74
		75	template <class T>
		76	WARN_UNUSED_RESULT inline typename remove_reference<T>::type&& move(T&& t) {
		77	return static_cast<typename remove_reference<T>::type&&>(t);
		78	}
		79
		80	template <class T>
		81	WARN_UNUSED_RESULT inline constexpr T&& forward(
		82	typename remove_reference<T>::type& t) {
		83	return static_cast<T&&>(t);
		84	}
		85
		86	template <class T>
		87	WARN_UNUSED_RESULT inline constexpr T&& forward(
		88	typename remove_reference<T>::type&& t) {
		89	return static_cast<T&&>(t);
		90	}
		91
		92	template <class T, T v>
		93	struct integral_constant {
		94	static constexpr const T value = v;
		95	typedef T value_type;
		96	typedef integral_constant type;
		97	constexpr operator value_type() const { return value; }
		98	constexpr value_type operator()() const { return value; }
		99	};
		100
		101	#ifndef __has_builtin
		102	# define __has_builtin(x) 0
		103	#endif
		104
		105	#if __has_builtin(__is_trivially_destructible)
		106
		107	template <class T>
		108	struct is_trivially_destructible
		109	: public integral_constant<bool, __is_trivially_destructible(T)> {};
		110
		111	#elif __has_builtin(__has_trivial_destructor)
		112
		113	template <class T>
		114	struct is_trivially_destructible
		115	: public integral_constant<bool, __has_trivial_destructor(T)> {};
		116
		117	#else
		118
		119	template <class T>
		120	struct is_trivially_destructible
		121	: public integral_constant<bool, /* less efficient fallback */ false> {};
		122
		123	#endif
		124
		125	#if __has_builtin(__is_trivially_copyable)
		126
		127	template <class T>
		128	struct is_trivially_copyable
		129	: public integral_constant<bool, __is_trivially_copyable(T)> {};
		130
		131	#else
		132
		133	template <class T>
		134	struct is_trivially_copyable
		135	: public integral_constant<bool, /* less efficient fallback */ false> {};
		136
		137	#endif
		138
60	} // namespace __sanitizer	139	} // namespace __sanitizer
61		140
62	#endif	141	#endif

lib/tsan/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp+1-7

 @@ -58,7 +58,7 @@ unwind_backtrace_signal_arch_func unwind_backtrace_signal_arch;
 #endif
 uptr Unwind_GetIP(struct _Unwind_Context *ctx) {
-#if defined(__arm__) && !SANITIZER_MAC
+#if defined(__arm__) && !SANITIZER_APPLE
   uptr val;
   _Unwind_VRS_Result res = _Unwind_VRS_Get(ctx, _UVRSC_CORE,
       15 /* r15 = PC */, _UVRSD_UINT32, &val);
 @@ -139,13 +139,7 @@ void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
   if (to_pop == 0 && size > 1)
     to_pop = 1;
   PopStackFrames(to_pop);
-#if defined(__GNUC__) && defined(__sparc__)
-  // __builtin_return_address returns the address of the call instruction
-  // on the SPARC and not the return address, so we need to compensate.
-  trace_buffer[0] = GetNextInstructionPc(pc);
-#else
   trace_buffer[0] = pc;
-#endif
 }
 void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {

lib/tsan/sanitizer_common/sanitizer_unwind_win.cpp+17-10

 @@ -57,30 +57,37 @@ void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
   InitializeDbgHelpIfNeeded();
   size = 0;
-#if defined(_WIN64)
+#    if SANITIZER_WINDOWS64
+#      if SANITIZER_ARM64
+  int machine_type = IMAGE_FILE_MACHINE_ARM64;
+  stack_frame.AddrPC.Offset = ctx.Pc;
+  stack_frame.AddrFrame.Offset = ctx.Fp;
+  stack_frame.AddrStack.Offset = ctx.Sp;
+#      else
   int machine_type = IMAGE_FILE_MACHINE_AMD64;
   stack_frame.AddrPC.Offset = ctx.Rip;
   stack_frame.AddrFrame.Offset = ctx.Rbp;
   stack_frame.AddrStack.Offset = ctx.Rsp;
-#else
+#      endif
+#    else
   int machine_type = IMAGE_FILE_MACHINE_I386;
   stack_frame.AddrPC.Offset = ctx.Eip;
   stack_frame.AddrFrame.Offset = ctx.Ebp;
   stack_frame.AddrStack.Offset = ctx.Esp;
-#endif
+#    endif
   stack_frame.AddrPC.Mode = AddrModeFlat;
   stack_frame.AddrFrame.Mode = AddrModeFlat;
   stack_frame.AddrStack.Mode = AddrModeFlat;
   while (StackWalk64(machine_type, GetCurrentProcess(), GetCurrentThread(),
-    &stack_frame, &ctx, NULL, SymFunctionTableAccess64,
+                     &stack_frame, &ctx, NULL, SymFunctionTableAccess64,
-    SymGetModuleBase64, NULL) &&
+                     SymGetModuleBase64, NULL) &&
-    size < Min(max_depth, kStackTraceMax)) {
+         size < Min(max_depth, kStackTraceMax)) {
     trace_buffer[size++] = (uptr)stack_frame.AddrPC.Offset;
   }
 }
-#ifdef __clang__
+#    ifdef __clang__
-#pragma clang diagnostic pop
+#      pragma clang diagnostic pop
-#endif
+#    endif
-#endif  // #if !SANITIZER_GO
+#  endif  // #if !SANITIZER_GO
 #endif  // SANITIZER_WINDOWS

lib/tsan/sanitizer_common/sanitizer_vector.h+2-2

...	@@ -83,8 +83,8 @@ class Vector {	...	@@ -83,8 +83,8 @@ class Vector {
83	}	83	}
84	EnsureSize(size);	84	EnsureSize(size);
85	if (old_size < size) {	85	if (old_size < size) {
86	for (uptr i = old_size; i < size; i++)	86	internal_memset(&begin_[old_size], 0,
87	internal_memset(&begin_[i], 0, sizeof(begin_[i]));	87	sizeof(begin_[old_size]) * (size - old_size));
88	}	88	}
89	}	89	}
90		90

lib/tsan/sanitizer_common/sanitizer_win.cpp+62-36

 @@ -93,6 +93,11 @@ bool FileExists(const char *filename) {
   return ::GetFileAttributesA(filename) != INVALID_FILE_ATTRIBUTES;
 }
+bool DirExists(const char *path) {
+  auto attr = ::GetFileAttributesA(path);
+  return (attr != INVALID_FILE_ATTRIBUTES) && (attr & FILE_ATTRIBUTE_DIRECTORY);
+}
 uptr internal_getpid() {
   return GetProcessId(GetCurrentProcess());
 }
 @@ -126,6 +131,11 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
 }
 #endif  // #if !SANITIZER_GO
+bool ErrorIsOOM(error_t err) {
+  // TODO: This should check which `err`s correspond to OOM.
+  return false;
+}
 void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
   void *rv = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
   if (rv == 0)
 @@ -224,6 +234,17 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
   return (void *)mapped_addr;
 }
+// ZeroMmapFixedRegion zero's out a region of memory previously returned from a
+// call to one of the MmapFixed* helpers. On non-windows systems this would be
+// done with another mmap, but on windows remapping is not an option.
+// VirtualFree(DECOMMIT)+VirtualAlloc(RECOMMIT) would also be a way to zero the
+// memory, but we can't do this atomically, so instead we fall back to using
+// internal_memset.
+bool ZeroMmapFixedRegion(uptr fixed_addr, uptr size) {
+  internal_memset((void*) fixed_addr, 0, size);
+  return true;
+}
 bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) {
   // FIXME: is this really "NoReserve"? On Win32 this does not matter much,
   // but on Win64 it does.
 @@ -336,6 +357,16 @@ bool MprotectNoAccess(uptr addr, uptr size) {
   return VirtualProtect((LPVOID)addr, size, PAGE_NOACCESS, &old_protection);
 }
+bool MprotectReadOnly(uptr addr, uptr size) {
+  DWORD old_protection;
+  return VirtualProtect((LPVOID)addr, size, PAGE_READONLY, &old_protection);
+}
+bool MprotectReadWrite(uptr addr, uptr size) {
+  DWORD old_protection;
+  return VirtualProtect((LPVOID)addr, size, PAGE_READWRITE, &old_protection);
+}
 void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
   uptr beg_aligned = RoundDownTo(beg, GetPageSizeCached()),
        end_aligned = RoundDownTo(end, GetPageSizeCached());
 @@ -512,7 +543,7 @@ void ReExec() {
   UNIMPLEMENTED();
 }
-void PlatformPrepareForSandboxing(__sanitizer_sandbox_arguments *args) {}
+void PlatformPrepareForSandboxing(void *args) {}
 bool StackSizeIsUnlimited() {
   UNIMPLEMENTED();
 @@ -565,6 +596,10 @@ void Abort() {
   internal__exit(3);
 }
+bool CreateDir(const char *pathname) {
+  return CreateDirectoryA(pathname, nullptr) != 0;
+}
 #if !SANITIZER_GO
 // Read the file to extract the ImageBase field from the PE header. If ASLR is
 // disabled and this virtual address is available, the loader will typically
 @@ -688,13 +723,24 @@ void ListOfModules::fallbackInit() { clear(); }
 // atexit() as soon as it is ready for use (i.e. after .CRT$XIC initializers).
 InternalMmapVectorNoCtor<void (*)(void)> atexit_functions;
-int Atexit(void (*function)(void)) {
+static int queueAtexit(void (*function)(void)) {
   atexit_functions.push_back(function);
   return 0;
 }
+// If Atexit() is being called after RunAtexit() has already been run, it needs
+// to be able to call atexit() directly. Here we use a function ponter to
+// switch out its behaviour.
+// An example of where this is needed is the asan_dynamic runtime on MinGW-w64.
+// On this environment, __asan_init is called during global constructor phase,
+// way after calling the .CRT$XID initializer.
+static int (*volatile queueOrCallAtExit)(void (*)(void)) = &queueAtexit;
+int Atexit(void (*function)(void)) { return queueOrCallAtExit(function); }
 static int RunAtexit() {
   TraceLoggingUnregister(g_asan_provider);
+  queueOrCallAtExit = &atexit;
   int ret = 0;
   for (uptr i = 0; i < atexit_functions.size(); ++i) {
     ret |= atexit(atexit_functions[i]);
 @@ -827,27 +873,6 @@ void FutexWake(atomic_uint32_t *p, u32 count) {
     WakeByAddressAll(p);
 }
-// ---------------------- BlockingMutex ---------------- {{{1
-BlockingMutex::BlockingMutex() {
-  CHECK(sizeof(SRWLOCK) <= sizeof(opaque_storage_));
-  internal_memset(this, 0, sizeof(*this));
-}
-void BlockingMutex::Lock() {
-  AcquireSRWLockExclusive((PSRWLOCK)opaque_storage_);
-  CHECK_EQ(owner_, 0);
-  owner_ = GetThreadSelf();
-}
-void BlockingMutex::Unlock() {
-  CheckLocked();
-  owner_ = 0;
-  ReleaseSRWLockExclusive((PSRWLOCK)opaque_storage_);
-}
-void BlockingMutex::CheckLocked() const { CHECK_EQ(owner_, GetThreadSelf()); }
 uptr GetTlsSize() {
   return 0;
 }
 @@ -962,13 +987,18 @@ void SignalContext::InitPcSpBp() {
   CONTEXT *context_record = (CONTEXT *)context;
   pc = (uptr)exception_record->ExceptionAddress;
-#ifdef _WIN64
+#  if SANITIZER_WINDOWS64
+#    if SANITIZER_ARM64
+  bp = (uptr)context_record->Fp;
+  sp = (uptr)context_record->Sp;
+#    else
   bp = (uptr)context_record->Rbp;
   sp = (uptr)context_record->Rsp;
-#else
+#    endif
+#  else
   bp = (uptr)context_record->Ebp;
   sp = (uptr)context_record->Esp;
-#endif
+#  endif
 }
 uptr SignalContext::GetAddress() const {
 @@ -990,7 +1020,7 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
   // The write flag is only available for access violation exceptions.
   if (exception_record->ExceptionCode != EXCEPTION_ACCESS_VIOLATION)
-    return SignalContext::UNKNOWN;
+    return SignalContext::Unknown;
   // The contents of this array are documented at
   // https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record
 @@ -998,13 +1028,13 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
   // second element is the faulting address.
   switch (exception_record->ExceptionInformation[0]) {
     case 0:
-      return SignalContext::READ;
+      return SignalContext::Read;
     case 1:
-      return SignalContext::WRITE;
+      return SignalContext::Write;
     case 8:
-      return SignalContext::UNKNOWN;
+      return SignalContext::Unknown;
   }
-  return SignalContext::UNKNOWN;
+  return SignalContext::Unknown;
 }
 void SignalContext::DumpAllRegisters(void *context) {
 @@ -1091,10 +1121,6 @@ void InitializePlatformEarly() {
   // Do nothing.
 }
-void MaybeReexec() {
-  // No need to re-exec on Windows.
-}
 void CheckASLR() {
   // Do nothing
 }
 @@ -1131,7 +1157,7 @@ bool IsProcessRunning(pid_t pid) {
 int WaitForProcess(pid_t pid) { return -1; }
 // FIXME implement on this platform.
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { }
+void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
 void CheckNoDeepBind(const char *filename, int flag) {
   // Do nothing.

lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.cpp created+101

...	@@ -0,0 +1,101 @@
1	//===-- sanitizer_win_dll_thunk.cpp ---------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	// This file defines a family of thunks that should be statically linked into
9	// the DLLs that have instrumentation in order to delegate the calls to the
10	// shared runtime that lives in the main binary.
11	// See https://github.com/google/sanitizers/issues/209 for the details.
12	//===----------------------------------------------------------------------===//
13
14	#ifdef SANITIZER_DLL_THUNK
15	#include "sanitizer_win_defs.h"
16	#include "sanitizer_win_dll_thunk.h"
17	#include "interception/interception.h"
18
19	extern "C" {
20	void WINAPI GetModuleHandleA(const char module_name);
21	void abort();
22	}
23
24	namespace __sanitizer {
25	uptr dllThunkGetRealAddrOrDie(const char *name) {
26	uptr ret =
27	__interception::InternalGetProcAddress((void *)GetModuleHandleA(0), name);
28	if (!ret)
29	abort();
30	return ret;
31	}
32
33	int dllThunkIntercept(const char* main_function, uptr dll_function) {
34	uptr wrapper = dllThunkGetRealAddrOrDie(main_function);
35	if (!__interception::OverrideFunction(dll_function, wrapper, 0))
36	abort();
37	return 0;
38	}
39
40	int dllThunkInterceptWhenPossible(const char* main_function,
41	const char* default_function, uptr dll_function) {
42	uptr wrapper = __interception::InternalGetProcAddress(
43	(void *)GetModuleHandleA(0), main_function);
44	if (!wrapper)
45	wrapper = dllThunkGetRealAddrOrDie(default_function);
46	if (!__interception::OverrideFunction(dll_function, wrapper, 0))
47	abort();
48	return 0;
49	}
50	} // namespace __sanitizer
51
52	// Include Sanitizer Common interface.
53	#define INTERFACE_FUNCTION(Name) INTERCEPT_SANITIZER_FUNCTION(Name)
54	#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
55	#include "sanitizer_common_interface.inc"
56
57	#pragma section(".DLLTH$A", read)
58	#pragma section(".DLLTH$Z", read)
59
60	typedef void (*DllThunkCB)();
61	extern "C" {
62	__declspec(allocate(".DLLTH$A")) DllThunkCB __start_dll_thunk;
63	__declspec(allocate(".DLLTH$Z")) DllThunkCB __stop_dll_thunk;
64	}
65
66	// Disable compiler warnings that show up if we declare our own version
67	// of a compiler intrinsic (e.g. strlen).
68	#pragma warning(disable: 4391)
69	#pragma warning(disable: 4392)
70
71	extern "C" int __dll_thunk_init() {
72	static bool flag = false;
73	// __dll_thunk_init is expected to be called by only one thread.
74	if (flag) return 0;
75	flag = true;
76
77	for (DllThunkCB *it = &__start_dll_thunk; it < &__stop_dll_thunk; ++it)
78	if (*it)
79	(*it)();
80
81	// In DLLs, the callbacks are expected to return 0,
82	// otherwise CRT initialization fails.
83	return 0;
84	}
85
86	// We want to call dll_thunk_init before C/C++ initializers / constructors are
87	// executed, otherwise functions like memset might be invoked.
88	#pragma section(".CRT$XIB", long, read)
89	__declspec(allocate(".CRT$XIB")) int (*__dll_thunk_preinit)() =
90	__dll_thunk_init;
91
92	static void WINAPI dll_thunk_thread_init(void *mod, unsigned long reason,
93	void *reserved) {
94	if (reason == /DLL_PROCESS_ATTACH=/1) __dll_thunk_init();
95	}
96
97	#pragma section(".CRT$XLAB", long, read)
98	__declspec(allocate(".CRT$XLAB")) void (WINAPI __dll_thunk_tls_init)(void ,
99	unsigned long, void *) = dll_thunk_thread_init;
100
101	#endif // SANITIZER_DLL_THUNK

lib/tsan/sanitizer_common/sanitizer_win_dll_thunk.h+1-1

...	@@ -84,7 +84,7 @@ extern "C" int __dll_thunk_init();	...	@@ -84,7 +84,7 @@ extern "C" int __dll_thunk_init();
84	// which isn't a big deal.	84	// which isn't a big deal.
85	#define INTERCEPT_LIBRARY_FUNCTION(name) \	85	#define INTERCEPT_LIBRARY_FUNCTION(name) \
86	extern "C" void name(); \	86	extern "C" void name(); \
87	INTERCEPT_OR_DIE(WRAPPER_NAME(name), name)	87	INTERCEPT_OR_DIE(STRINGIFY(WRAP(name)), name)
88		88
89	// Use these macros for functions that could be called before __dll_thunk_init()	89	// Use these macros for functions that could be called before __dll_thunk_init()
90	// is executed and don't lead to errors if defined (free, malloc, etc).	90	// is executed and don't lead to errors if defined (free, malloc, etc).

lib/tsan/sanitizer_common/sanitizer_win_dynamic_runtime_thunk.cpp created+26

...	@@ -0,0 +1,26 @@
1	//===-- santizer_win_dynamic_runtime_thunk.cpp ----------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file defines things that need to be present in the application modules
10	// to interact with Sanitizer Common, when it is included in a dll.
11	//
12	//===----------------------------------------------------------------------===//
13	#ifdef SANITIZER_DYNAMIC_RUNTIME_THUNK
14	#define SANITIZER_IMPORT_INTERFACE 1
15	#include "sanitizer_win_defs.h"
16	// Define weak alias for all weak functions imported from sanitizer common.
17	#define INTERFACE_FUNCTION(Name)
18	#define INTERFACE_WEAK_FUNCTION(Name) WIN_WEAK_IMPORT_DEF(Name)
19	#include "sanitizer_common_interface.inc"
20	#endif // SANITIZER_DYNAMIC_RUNTIME_THUNK
21
22	namespace __sanitizer {
23	// Add one, otherwise unused, external symbol to this object file so that the
24	// Visual C++ linker includes it and reads the .drective section.
25	void ForceWholeArchiveIncludeForSanitizerCommon() {}
26	}

lib/tsan/sanitizer_common/sanitizer_win_weak_interception.cpp created+94

...	@@ -0,0 +1,94 @@
1	//===-- sanitizer_win_weak_interception.cpp -------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	// This module should be included in the sanitizer when it is implemented as a
9	// shared library on Windows (dll), in order to delegate the calls of weak
10	// functions to the implementation in the main executable when a strong
11	// definition is provided.
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_platform.h"
15	#if SANITIZER_WINDOWS && SANITIZER_DYNAMIC
16	#include "sanitizer_win_weak_interception.h"
17	#include "sanitizer_allocator_interface.h"
18	#include "sanitizer_interface_internal.h"
19	#include "sanitizer_win_defs.h"
20	#include "interception/interception.h"
21
22	extern "C" {
23	void WINAPI GetModuleHandleA(const char module_name);
24	void abort();
25	}
26
27	namespace __sanitizer {
28	// Try to get a pointer to real_function in the main module and override
29	// dll_function with that pointer. If the function isn't found, nothing changes.
30	int interceptWhenPossible(uptr dll_function, const char *real_function) {
31	uptr real = __interception::InternalGetProcAddress(
32	(void *)GetModuleHandleA(0), real_function);
33	if (real && !__interception::OverrideFunction((uptr)dll_function, real, 0))
34	abort();
35	return 0;
36	}
37	} // namespace __sanitizer
38
39	// Declare weak hooks.
40	extern "C" {
41	void __sanitizer_on_print(const char *str);
42	void __sanitizer_weak_hook_memcmp(uptr called_pc, const void *s1,
43	const void *s2, uptr n, int result);
44	void __sanitizer_weak_hook_strcmp(uptr called_pc, const char *s1,
45	const char *s2, int result);
46	void __sanitizer_weak_hook_strncmp(uptr called_pc, const char *s1,
47	const char *s2, uptr n, int result);
48	void __sanitizer_weak_hook_strstr(uptr called_pc, const char *s1,
49	const char s2, char result);
50	}
51
52	// Include Sanitizer Common interface.
53	#define INTERFACE_FUNCTION(Name)
54	#define INTERFACE_WEAK_FUNCTION(Name) INTERCEPT_SANITIZER_WEAK_FUNCTION(Name)
55	#include "sanitizer_common_interface.inc"
56
57	#pragma section(".WEAK$A", read)
58	#pragma section(".WEAK$Z", read)
59
60	typedef void (*InterceptCB)();
61	extern "C" {
62	__declspec(allocate(".WEAK$A")) InterceptCB __start_weak_list;
63	__declspec(allocate(".WEAK$Z")) InterceptCB __stop_weak_list;
64	}
65
66	static int weak_intercept_init() {
67	static bool flag = false;
68	// weak_interception_init is expected to be called by only one thread.
69	if (flag) return 0;
70	flag = true;
71
72	for (InterceptCB *it = &__start_weak_list; it < &__stop_weak_list; ++it)
73	if (*it)
74	(*it)();
75
76	// In DLLs, the callbacks are expected to return 0,
77	// otherwise CRT initialization fails.
78	return 0;
79	}
80
81	#pragma section(".CRT$XIB", long, read)
82	__declspec(allocate(".CRT$XIB")) int (*__weak_intercept_preinit)() =
83	weak_intercept_init;
84
85	static void WINAPI weak_intercept_thread_init(void *mod, unsigned long reason,
86	void *reserved) {
87	if (reason == /DLL_PROCESS_ATTACH=/1) weak_intercept_init();
88	}
89
90	#pragma section(".CRT$XLAB", long, read)
91	__declspec(allocate(".CRT$XLAB")) void(WINAPI *__weak_intercept_tls_init)(
92	void , unsigned long, void ) = weak_intercept_thread_init;
93
94	#endif // SANITIZER_WINDOWS && SANITIZER_DYNAMIC

lib/tsan/tsan_clock.cpp deleted-625

...	@@ -1,625 +0,0 @@
1	//===-- tsan_clock.cpp ----------------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	//===----------------------------------------------------------------------===//
12	#include "tsan_clock.h"
13	#include "tsan_rtl.h"
14	#include "sanitizer_common/sanitizer_placement_new.h"
15
16	// SyncClock and ThreadClock implement vector clocks for sync variables
17	// (mutexes, atomic variables, file descriptors, etc) and threads, respectively.
18	// ThreadClock contains fixed-size vector clock for maximum number of threads.
19	// SyncClock contains growable vector clock for currently necessary number of
20	// threads.
21	// Together they implement very simple model of operations, namely:
22	//
23	// void ThreadClock::acquire(const SyncClock *src) {
24	// for (int i = 0; i < kMaxThreads; i++)
25	// clock[i] = max(clock[i], src->clock[i]);
26	// }
27	//
28	// void ThreadClock::release(SyncClock *dst) const {
29	// for (int i = 0; i < kMaxThreads; i++)
30	// dst->clock[i] = max(dst->clock[i], clock[i]);
31	// }
32	//
33	// void ThreadClock::releaseStoreAcquire(SyncClock *sc) const {
34	// for (int i = 0; i < kMaxThreads; i++) {
35	// tmp = clock[i];
36	// clock[i] = max(clock[i], sc->clock[i]);
37	// sc->clock[i] = tmp;
38	// }
39	// }
40	//
41	// void ThreadClock::ReleaseStore(SyncClock *dst) const {
42	// for (int i = 0; i < kMaxThreads; i++)
43	// dst->clock[i] = clock[i];
44	// }
45	//
46	// void ThreadClock::acq_rel(SyncClock *dst) {
47	// acquire(dst);
48	// release(dst);
49	// }
50	//
51	// Conformance to this model is extensively verified in tsan_clock_test.cpp.
52	// However, the implementation is significantly more complex. The complexity
53	// allows to implement important classes of use cases in O(1) instead of O(N).
54	//
55	// The use cases are:
56	// 1. Singleton/once atomic that has a single release-store operation followed
57	// by zillions of acquire-loads (the acquire-load is O(1)).
58	// 2. Thread-local mutex (both lock and unlock can be O(1)).
59	// 3. Leaf mutex (unlock is O(1)).
60	// 4. A mutex shared by 2 threads (both lock and unlock can be O(1)).
61	// 5. An atomic with a single writer (writes can be O(1)).
62	// The implementation dynamically adopts to workload. So if an atomic is in
63	// read-only phase, these reads will be O(1); if it later switches to read/write
64	// phase, the implementation will correctly handle that by switching to O(N).
65	//
66	// Thread-safety note: all const operations on SyncClock's are conducted under
67	// a shared lock; all non-const operations on SyncClock's are conducted under
68	// an exclusive lock; ThreadClock's are private to respective threads and so
69	// do not need any protection.
70	//
71	// Description of SyncClock state:
72	// clk_ - variable size vector clock, low kClkBits hold timestamp,
73	// the remaining bits hold "acquired" flag (the actual value is thread's
74	// reused counter);
75	// if acquried == thr->reused_, then the respective thread has already
76	// acquired this clock (except possibly for dirty elements).
77	// dirty_ - holds up to two indeces in the vector clock that other threads
78	// need to acquire regardless of "acquired" flag value;
79	// release_store_tid_ - denotes that the clock state is a result of
80	// release-store operation by the thread with release_store_tid_ index.
81	// release_store_reused_ - reuse count of release_store_tid_.
82
83	namespace __tsan {
84
85	static atomic_uint32_t ref_ptr(ClockBlock cb) {
86	return reinterpret_cast<atomic_uint32_t *>(&cb->table[ClockBlock::kRefIdx]);
87	}
88
89	// Drop reference to the first level block idx.
90	static void UnrefClockBlock(ClockCache *c, u32 idx, uptr blocks) {
91	ClockBlock *cb = ctx->clock_alloc.Map(idx);
92	atomic_uint32_t *ref = ref_ptr(cb);
93	u32 v = atomic_load(ref, memory_order_acquire);
94	for (;;) {
95	CHECK_GT(v, 0);
96	if (v == 1)
97	break;
98	if (atomic_compare_exchange_strong(ref, &v, v - 1, memory_order_acq_rel))
99	return;
100	}
101	// First level block owns second level blocks, so them as well.
102	for (uptr i = 0; i < blocks; i++)
103	ctx->clock_alloc.Free(c, cb->table[ClockBlock::kBlockIdx - i]);
104	ctx->clock_alloc.Free(c, idx);
105	}
106
107	ThreadClock::ThreadClock(unsigned tid, unsigned reused)
108	: tid_(tid)
109	, reused_(reused + 1) // 0 has special meaning
110	, last_acquire_()
111	, global_acquire_()
112	, cached_idx_()
113	, cached_size_()
114	, cached_blocks_() {
115	CHECK_LT(tid, kMaxTidInClock);
116	CHECK_EQ(reused_, ((u64)reused_ << kClkBits) >> kClkBits);
117	nclk_ = tid_ + 1;
118	internal_memset(clk_, 0, sizeof(clk_));
119	}
120
121	void ThreadClock::ResetCached(ClockCache *c) {
122	if (cached_idx_) {
123	UnrefClockBlock(c, cached_idx_, cached_blocks_);
124	cached_idx_ = 0;
125	cached_size_ = 0;
126	cached_blocks_ = 0;
127	}
128	}
129
130	void ThreadClock::acquire(ClockCache c, SyncClock src) {
131	DCHECK_LE(nclk_, kMaxTid);
132	DCHECK_LE(src->size_, kMaxTid);
133
134	// Check if it's empty -> no need to do anything.
135	const uptr nclk = src->size_;
136	if (nclk == 0)
137	return;
138
139	bool acquired = false;
140	for (unsigned i = 0; i < kDirtyTids; i++) {
141	SyncClock::Dirty dirty = src->dirty_[i];
142	unsigned tid = dirty.tid();
143	if (tid != kInvalidTid) {
144	if (clk_[tid] < dirty.epoch) {
145	clk_[tid] = dirty.epoch;
146	acquired = true;
147	}
148	}
149	}
150
151	// Check if we've already acquired src after the last release operation on src
152	if (tid_ >= nclk \|\| src->elem(tid_).reused != reused_) {
153	// O(N) acquire.
154	nclk_ = max(nclk_, nclk);
155	u64 *dst_pos = &clk_[0];
156	for (ClockElem &src_elem : *src) {
157	u64 epoch = src_elem.epoch;
158	if (*dst_pos < epoch) {
159	*dst_pos = epoch;
160	acquired = true;
161	}
162	dst_pos++;
163	}
164
165	// Remember that this thread has acquired this clock.
166	if (nclk > tid_)
167	src->elem(tid_).reused = reused_;
168	}
169
170	if (acquired) {
171	last_acquire_ = clk_[tid_];
172	ResetCached(c);
173	}
174	}
175
176	void ThreadClock::releaseStoreAcquire(ClockCache c, SyncClock sc) {
177	DCHECK_LE(nclk_, kMaxTid);
178	DCHECK_LE(sc->size_, kMaxTid);
179
180	if (sc->size_ == 0) {
181	// ReleaseStore will correctly set release_store_tid_,
182	// which can be important for future operations.
183	ReleaseStore(c, sc);
184	return;
185	}
186
187	nclk_ = max(nclk_, (uptr) sc->size_);
188
189	// Check if we need to resize sc.
190	if (sc->size_ < nclk_)
191	sc->Resize(c, nclk_);
192
193	bool acquired = false;
194
195	sc->Unshare(c);
196	// Update sc->clk_.
197	sc->FlushDirty();
198	uptr i = 0;
199	for (ClockElem &ce : *sc) {
200	u64 tmp = clk_[i];
201	if (clk_[i] < ce.epoch) {
202	clk_[i] = ce.epoch;
203	acquired = true;
204	}
205	ce.epoch = tmp;
206	ce.reused = 0;
207	i++;
208	}
209	sc->release_store_tid_ = kInvalidTid;
210	sc->release_store_reused_ = 0;
211
212	if (acquired) {
213	last_acquire_ = clk_[tid_];
214	ResetCached(c);
215	}
216	}
217
218	void ThreadClock::release(ClockCache c, SyncClock dst) {
219	DCHECK_LE(nclk_, kMaxTid);
220	DCHECK_LE(dst->size_, kMaxTid);
221
222	if (dst->size_ == 0) {
223	// ReleaseStore will correctly set release_store_tid_,
224	// which can be important for future operations.
225	ReleaseStore(c, dst);
226	return;
227	}
228
229	// Check if we need to resize dst.
230	if (dst->size_ < nclk_)
231	dst->Resize(c, nclk_);
232
233	// Check if we had not acquired anything from other threads
234	// since the last release on dst. If so, we need to update
235	// only dst->elem(tid_).
236	if (!HasAcquiredAfterRelease(dst)) {
237	UpdateCurrentThread(c, dst);
238	if (dst->release_store_tid_ != tid_ \|\|
239	dst->release_store_reused_ != reused_)
240	dst->release_store_tid_ = kInvalidTid;
241	return;
242	}
243
244	// O(N) release.
245	dst->Unshare(c);
246	// First, remember whether we've acquired dst.
247	bool acquired = IsAlreadyAcquired(dst);
248	// Update dst->clk_.
249	dst->FlushDirty();
250	uptr i = 0;
251	for (ClockElem &ce : *dst) {
252	ce.epoch = max(ce.epoch, clk_[i]);
253	ce.reused = 0;
254	i++;
255	}
256	// Clear 'acquired' flag in the remaining elements.
257	dst->release_store_tid_ = kInvalidTid;
258	dst->release_store_reused_ = 0;
259	// If we've acquired dst, remember this fact,
260	// so that we don't need to acquire it on next acquire.
261	if (acquired)
262	dst->elem(tid_).reused = reused_;
263	}
264
265	void ThreadClock::ReleaseStore(ClockCache c, SyncClock dst) {
266	DCHECK_LE(nclk_, kMaxTid);
267	DCHECK_LE(dst->size_, kMaxTid);
268
269	if (dst->size_ == 0 && cached_idx_ != 0) {
270	// Reuse the cached clock.
271	// Note: we could reuse/cache the cached clock in more cases:
272	// we could update the existing clock and cache it, or replace it with the
273	// currently cached clock and release the old one. And for a shared
274	// existing clock, we could replace it with the currently cached;
275	// or unshare, update and cache. But, for simplicity, we currnetly reuse
276	// cached clock only when the target clock is empty.
277	dst->tab_ = ctx->clock_alloc.Map(cached_idx_);
278	dst->tab_idx_ = cached_idx_;
279	dst->size_ = cached_size_;
280	dst->blocks_ = cached_blocks_;
281	CHECK_EQ(dst->dirty_[0].tid(), kInvalidTid);
282	// The cached clock is shared (immutable),
283	// so this is where we store the current clock.
284	dst->dirty_[0].set_tid(tid_);
285	dst->dirty_[0].epoch = clk_[tid_];
286	dst->release_store_tid_ = tid_;
287	dst->release_store_reused_ = reused_;
288	// Rememeber that we don't need to acquire it in future.
289	dst->elem(tid_).reused = reused_;
290	// Grab a reference.
291	atomic_fetch_add(ref_ptr(dst->tab_), 1, memory_order_relaxed);
292	return;
293	}
294
295	// Check if we need to resize dst.
296	if (dst->size_ < nclk_)
297	dst->Resize(c, nclk_);
298
299	if (dst->release_store_tid_ == tid_ &&
300	dst->release_store_reused_ == reused_ &&
301	!HasAcquiredAfterRelease(dst)) {
302	UpdateCurrentThread(c, dst);
303	return;
304	}
305
306	// O(N) release-store.
307	dst->Unshare(c);
308	// Note: dst can be larger than this ThreadClock.
309	// This is fine since clk_ beyond size is all zeros.
310	uptr i = 0;
311	for (ClockElem &ce : *dst) {
312	ce.epoch = clk_[i];
313	ce.reused = 0;
314	i++;
315	}
316	for (uptr i = 0; i < kDirtyTids; i++) dst->dirty_[i].set_tid(kInvalidTid);
317	dst->release_store_tid_ = tid_;
318	dst->release_store_reused_ = reused_;
319	// Rememeber that we don't need to acquire it in future.
320	dst->elem(tid_).reused = reused_;
321
322	// If the resulting clock is cachable, cache it for future release operations.
323	// The clock is always cachable if we released to an empty sync object.
324	if (cached_idx_ == 0 && dst->Cachable()) {
325	// Grab a reference to the ClockBlock.
326	atomic_uint32_t *ref = ref_ptr(dst->tab_);
327	if (atomic_load(ref, memory_order_acquire) == 1)
328	atomic_store_relaxed(ref, 2);
329	else
330	atomic_fetch_add(ref_ptr(dst->tab_), 1, memory_order_relaxed);
331	cached_idx_ = dst->tab_idx_;
332	cached_size_ = dst->size_;
333	cached_blocks_ = dst->blocks_;
334	}
335	}
336
337	void ThreadClock::acq_rel(ClockCache c, SyncClock dst) {
338	acquire(c, dst);
339	ReleaseStore(c, dst);
340	}
341
342	// Updates only single element related to the current thread in dst->clk_.
343	void ThreadClock::UpdateCurrentThread(ClockCache c, SyncClock dst) const {
344	// Update the threads time, but preserve 'acquired' flag.
345	for (unsigned i = 0; i < kDirtyTids; i++) {
346	SyncClock::Dirty *dirty = &dst->dirty_[i];
347	const unsigned tid = dirty->tid();
348	if (tid == tid_ \|\| tid == kInvalidTid) {
349	dirty->set_tid(tid_);
350	dirty->epoch = clk_[tid_];
351	return;
352	}
353	}
354	// Reset all 'acquired' flags, O(N).
355	// We are going to touch dst elements, so we need to unshare it.
356	dst->Unshare(c);
357	dst->elem(tid_).epoch = clk_[tid_];
358	for (uptr i = 0; i < dst->size_; i++)
359	dst->elem(i).reused = 0;
360	dst->FlushDirty();
361	}
362
363	// Checks whether the current thread has already acquired src.
364	bool ThreadClock::IsAlreadyAcquired(const SyncClock *src) const {
365	if (src->elem(tid_).reused != reused_)
366	return false;
367	for (unsigned i = 0; i < kDirtyTids; i++) {
368	SyncClock::Dirty dirty = src->dirty_[i];
369	if (dirty.tid() != kInvalidTid) {
370	if (clk_[dirty.tid()] < dirty.epoch)
371	return false;
372	}
373	}
374	return true;
375	}
376
377	// Checks whether the current thread has acquired anything
378	// from other clocks after releasing to dst (directly or indirectly).
379	bool ThreadClock::HasAcquiredAfterRelease(const SyncClock *dst) const {
380	const u64 my_epoch = dst->elem(tid_).epoch;
381	return my_epoch <= last_acquire_ \|\|
382	my_epoch <= atomic_load_relaxed(&global_acquire_);
383	}
384
385	// Sets a single element in the vector clock.
386	// This function is called only from weird places like AcquireGlobal.
387	void ThreadClock::set(ClockCache *c, unsigned tid, u64 v) {
388	DCHECK_LT(tid, kMaxTid);
389	DCHECK_GE(v, clk_[tid]);
390	clk_[tid] = v;
391	if (nclk_ <= tid)
392	nclk_ = tid + 1;
393	last_acquire_ = clk_[tid_];
394	ResetCached(c);
395	}
396
397	void ThreadClock::DebugDump(int(printf)(const char s, ...)) {
398	printf("clock=[");
399	for (uptr i = 0; i < nclk_; i++)
400	printf("%s%llu", i == 0 ? "" : ",", clk_[i]);
401	printf("] tid=%u/%u last_acq=%llu", tid_, reused_, last_acquire_);
402	}
403
404	SyncClock::SyncClock() {
405	ResetImpl();
406	}
407
408	SyncClock::~SyncClock() {
409	// Reset must be called before dtor.
410	CHECK_EQ(size_, 0);
411	CHECK_EQ(blocks_, 0);
412	CHECK_EQ(tab_, 0);
413	CHECK_EQ(tab_idx_, 0);
414	}
415
416	void SyncClock::Reset(ClockCache *c) {
417	if (size_)
418	UnrefClockBlock(c, tab_idx_, blocks_);
419	ResetImpl();
420	}
421
422	void SyncClock::ResetImpl() {
423	tab_ = 0;
424	tab_idx_ = 0;
425	size_ = 0;
426	blocks_ = 0;
427	release_store_tid_ = kInvalidTid;
428	release_store_reused_ = 0;
429	for (uptr i = 0; i < kDirtyTids; i++) dirty_[i].set_tid(kInvalidTid);
430	}
431
432	void SyncClock::Resize(ClockCache *c, uptr nclk) {
433	Unshare(c);
434	if (nclk <= capacity()) {
435	// Memory is already allocated, just increase the size.
436	size_ = nclk;
437	return;
438	}
439	if (size_ == 0) {
440	// Grow from 0 to one-level table.
441	CHECK_EQ(size_, 0);
442	CHECK_EQ(blocks_, 0);
443	CHECK_EQ(tab_, 0);
444	CHECK_EQ(tab_idx_, 0);
445	tab_idx_ = ctx->clock_alloc.Alloc(c);
446	tab_ = ctx->clock_alloc.Map(tab_idx_);
447	internal_memset(tab_, 0, sizeof(*tab_));
448	atomic_store_relaxed(ref_ptr(tab_), 1);
449	size_ = 1;
450	} else if (size_ > blocks_ * ClockBlock::kClockCount) {
451	u32 idx = ctx->clock_alloc.Alloc(c);
452	ClockBlock *new_cb = ctx->clock_alloc.Map(idx);
453	uptr top = size_ - blocks_ * ClockBlock::kClockCount;
454	CHECK_LT(top, ClockBlock::kClockCount);
455	const uptr move = top * sizeof(tab_->clock[0]);
456	internal_memcpy(&new_cb->clock[0], tab_->clock, move);
457	internal_memset(&new_cb->clock[top], 0, sizeof(*new_cb) - move);
458	internal_memset(tab_->clock, 0, move);
459	append_block(idx);
460	}
461	// At this point we have first level table allocated and all clock elements
462	// are evacuated from it to a second level block.
463	// Add second level tables as necessary.
464	while (nclk > capacity()) {
465	u32 idx = ctx->clock_alloc.Alloc(c);
466	ClockBlock *cb = ctx->clock_alloc.Map(idx);
467	internal_memset(cb, 0, sizeof(*cb));
468	append_block(idx);
469	}
470	size_ = nclk;
471	}
472
473	// Flushes all dirty elements into the main clock array.
474	void SyncClock::FlushDirty() {
475	for (unsigned i = 0; i < kDirtyTids; i++) {
476	Dirty *dirty = &dirty_[i];
477	if (dirty->tid() != kInvalidTid) {
478	CHECK_LT(dirty->tid(), size_);
479	elem(dirty->tid()).epoch = dirty->epoch;
480	dirty->set_tid(kInvalidTid);
481	}
482	}
483	}
484
485	bool SyncClock::IsShared() const {
486	if (size_ == 0)
487	return false;
488	atomic_uint32_t *ref = ref_ptr(tab_);
489	u32 v = atomic_load(ref, memory_order_acquire);
490	CHECK_GT(v, 0);
491	return v > 1;
492	}
493
494	// Unshares the current clock if it's shared.
495	// Shared clocks are immutable, so they need to be unshared before any updates.
496	// Note: this does not apply to dirty entries as they are not shared.
497	void SyncClock::Unshare(ClockCache *c) {
498	if (!IsShared())
499	return;
500	// First, copy current state into old.
501	SyncClock old;
502	old.tab_ = tab_;
503	old.tab_idx_ = tab_idx_;
504	old.size_ = size_;
505	old.blocks_ = blocks_;
506	old.release_store_tid_ = release_store_tid_;
507	old.release_store_reused_ = release_store_reused_;
508	for (unsigned i = 0; i < kDirtyTids; i++)
509	old.dirty_[i] = dirty_[i];
510	// Then, clear current object.
511	ResetImpl();
512	// Allocate brand new clock in the current object.
513	Resize(c, old.size_);
514	// Now copy state back into this object.
515	Iter old_iter(&old);
516	for (ClockElem &ce : *this) {
517	ce = *old_iter;
518	++old_iter;
519	}
520	release_store_tid_ = old.release_store_tid_;
521	release_store_reused_ = old.release_store_reused_;
522	for (unsigned i = 0; i < kDirtyTids; i++)
523	dirty_[i] = old.dirty_[i];
524	// Drop reference to old and delete if necessary.
525	old.Reset(c);
526	}
527
528	// Can we cache this clock for future release operations?
529	ALWAYS_INLINE bool SyncClock::Cachable() const {
530	if (size_ == 0)
531	return false;
532	for (unsigned i = 0; i < kDirtyTids; i++) {
533	if (dirty_[i].tid() != kInvalidTid)
534	return false;
535	}
536	return atomic_load_relaxed(ref_ptr(tab_)) == 1;
537	}
538
539	// elem linearizes the two-level structure into linear array.
540	// Note: this is used only for one time accesses, vector operations use
541	// the iterator as it is much faster.
542	ALWAYS_INLINE ClockElem &SyncClock::elem(unsigned tid) const {
543	DCHECK_LT(tid, size_);
544	const uptr block = tid / ClockBlock::kClockCount;
545	DCHECK_LE(block, blocks_);
546	tid %= ClockBlock::kClockCount;
547	if (block == blocks_)
548	return tab_->clock[tid];
549	u32 idx = get_block(block);
550	ClockBlock *cb = ctx->clock_alloc.Map(idx);
551	return cb->clock[tid];
552	}
553
554	ALWAYS_INLINE uptr SyncClock::capacity() const {
555	if (size_ == 0)
556	return 0;
557	uptr ratio = sizeof(ClockBlock::clock[0]) / sizeof(ClockBlock::table[0]);
558	// How many clock elements we can fit into the first level block.
559	// +1 for ref counter.
560	uptr top = ClockBlock::kClockCount - RoundUpTo(blocks_ + 1, ratio) / ratio;
561	return blocks_ * ClockBlock::kClockCount + top;
562	}
563
564	ALWAYS_INLINE u32 SyncClock::get_block(uptr bi) const {
565	DCHECK(size_);
566	DCHECK_LT(bi, blocks_);
567	return tab_->table[ClockBlock::kBlockIdx - bi];
568	}
569
570	ALWAYS_INLINE void SyncClock::append_block(u32 idx) {
571	uptr bi = blocks_++;
572	CHECK_EQ(get_block(bi), 0);
573	tab_->table[ClockBlock::kBlockIdx - bi] = idx;
574	}
575
576	// Used only by tests.
577	u64 SyncClock::get(unsigned tid) const {
578	for (unsigned i = 0; i < kDirtyTids; i++) {
579	Dirty dirty = dirty_[i];
580	if (dirty.tid() == tid)
581	return dirty.epoch;
582	}
583	return elem(tid).epoch;
584	}
585
586	// Used only by Iter test.
587	u64 SyncClock::get_clean(unsigned tid) const {
588	return elem(tid).epoch;
589	}
590
591	void SyncClock::DebugDump(int(printf)(const char s, ...)) {
592	printf("clock=[");
593	for (uptr i = 0; i < size_; i++)
594	printf("%s%llu", i == 0 ? "" : ",", elem(i).epoch);
595	printf("] reused=[");
596	for (uptr i = 0; i < size_; i++)
597	printf("%s%llu", i == 0 ? "" : ",", elem(i).reused);
598	printf("] release_store_tid=%d/%d dirty_tids=%d[%llu]/%d[%llu]",
599	release_store_tid_, release_store_reused_, dirty_[0].tid(),
600	dirty_[0].epoch, dirty_[1].tid(), dirty_[1].epoch);
601	}
602
603	void SyncClock::Iter::Next() {
604	// Finished with the current block, move on to the next one.
605	block_++;
606	if (block_ < parent_->blocks_) {
607	// Iterate over the next second level block.
608	u32 idx = parent_->get_block(block_);
609	ClockBlock *cb = ctx->clock_alloc.Map(idx);
610	pos_ = &cb->clock[0];
611	end_ = pos_ + min(parent_->size_ - block_ * ClockBlock::kClockCount,
612	ClockBlock::kClockCount);
613	return;
614	}
615	if (block_ == parent_->blocks_ &&
616	parent_->size_ > parent_->blocks_ * ClockBlock::kClockCount) {
617	// Iterate over elements in the first level block.
618	pos_ = &parent_->tab_->clock[0];
619	end_ = pos_ + min(parent_->size_ - block_ * ClockBlock::kClockCount,
620	ClockBlock::kClockCount);
621	return;
622	}
623	parent_ = nullptr; // denotes end
624	}
625	} // namespace __tsan

lib/tsan/tsan_clock.h deleted-293

...	@@ -1,293 +0,0 @@
1	//===-- tsan_clock.h --------------------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	//===----------------------------------------------------------------------===//
12	#ifndef TSAN_CLOCK_H
13	#define TSAN_CLOCK_H
14
15	#include "tsan_defs.h"
16	#include "tsan_dense_alloc.h"
17
18	namespace __tsan {
19
20	typedef DenseSlabAlloc<ClockBlock, 1 << 22, 1 << 10> ClockAlloc;
21	typedef DenseSlabAllocCache ClockCache;
22
23	// The clock that lives in sync variables (mutexes, atomics, etc).
24	class SyncClock {
25	public:
26	SyncClock();
27	~SyncClock();
28
29	uptr size() const;
30
31	// These are used only in tests.
32	u64 get(unsigned tid) const;
33	u64 get_clean(unsigned tid) const;
34
35	void Resize(ClockCache *c, uptr nclk);
36	void Reset(ClockCache *c);
37
38	void DebugDump(int(printf)(const char s, ...));
39
40	// Clock element iterator.
41	// Note: it iterates only over the table without regard to dirty entries.
42	class Iter {
43	public:
44	explicit Iter(SyncClock* parent);
45	Iter& operator++();
46	bool operator!=(const Iter& other);
47	ClockElem &operator*();
48
49	private:
50	SyncClock *parent_;
51	// [pos_, end_) is the current continuous range of clock elements.
52	ClockElem *pos_;
53	ClockElem *end_;
54	int block_; // Current number of second level block.
55
56	NOINLINE void Next();
57	};
58
59	Iter begin();
60	Iter end();
61
62	private:
63	friend class ThreadClock;
64	friend class Iter;
65	static const uptr kDirtyTids = 2;
66
67	struct Dirty {
68	u32 tid() const { return tid_ == kShortInvalidTid ? kInvalidTid : tid_; }
69	void set_tid(u32 tid) {
70	tid_ = tid == kInvalidTid ? kShortInvalidTid : tid;
71	}
72	u64 epoch : kClkBits;
73
74	private:
75	// Full kInvalidTid won't fit into Dirty::tid.
76	static const u64 kShortInvalidTid = (1ull << (64 - kClkBits)) - 1;
77	u64 tid_ : 64 - kClkBits; // kInvalidId if not active
78	};
79
80	static_assert(sizeof(Dirty) == 8, "Dirty is not 64bit");
81
82	unsigned release_store_tid_;
83	unsigned release_store_reused_;
84	Dirty dirty_[kDirtyTids];
85	// If size_ is 0, tab_ is nullptr.
86	// If size <= 64 (kClockCount), tab_ contains pointer to an array with
87	// 64 ClockElem's (ClockBlock::clock).
88	// Otherwise, tab_ points to an array with up to 127 u32 elements,
89	// each pointing to the second-level 512b block with 64 ClockElem's.
90	// Unused space in the first level ClockBlock is used to store additional
91	// clock elements.
92	// The last u32 element in the first level ClockBlock is always used as
93	// reference counter.
94	//
95	// See the following scheme for details.
96	// All memory blocks are 512 bytes (allocated from ClockAlloc).
97	// Clock (clk) elements are 64 bits.
98	// Idx and ref are 32 bits.
99	//
100	// tab_
101	// \|
102	// \/
103	// +----------------------------------------------------+
104	// \| clk128 \| clk129 \| ...unused... \| idx1 \| idx0 \| ref \|
105	// +----------------------------------------------------+
106	// \| \|
107	// \| \/
108	// \| +----------------+
109	// \| \| clk0 ... clk63 \|
110	// \| +----------------+
111	// \/
112	// +------------------+
113	// \| clk64 ... clk127 \|
114	// +------------------+
115	//
116	// Note: dirty entries, if active, always override what's stored in the clock.
117	ClockBlock *tab_;
118	u32 tab_idx_;
119	u16 size_;
120	u16 blocks_; // Number of second level blocks.
121
122	void Unshare(ClockCache *c);
123	bool IsShared() const;
124	bool Cachable() const;
125	void ResetImpl();
126	void FlushDirty();
127	uptr capacity() const;
128	u32 get_block(uptr bi) const;
129	void append_block(u32 idx);
130	ClockElem &elem(unsigned tid) const;
131	};
132
133	// The clock that lives in threads.
134	class ThreadClock {
135	public:
136	typedef DenseSlabAllocCache Cache;
137
138	explicit ThreadClock(unsigned tid, unsigned reused = 0);
139
140	u64 get(unsigned tid) const;
141	void set(ClockCache *c, unsigned tid, u64 v);
142	void set(u64 v);
143	void tick();
144	uptr size() const;
145
146	void acquire(ClockCache c, SyncClock src);
147	void releaseStoreAcquire(ClockCache c, SyncClock src);
148	void release(ClockCache c, SyncClock dst);
149	void acq_rel(ClockCache c, SyncClock dst);
150	void ReleaseStore(ClockCache c, SyncClock dst);
151	void ResetCached(ClockCache *c);
152	void NoteGlobalAcquire(u64 v);
153
154	void DebugReset();
155	void DebugDump(int(printf)(const char s, ...));
156
157	private:
158	static const uptr kDirtyTids = SyncClock::kDirtyTids;
159	// Index of the thread associated with he clock ("current thread").
160	const unsigned tid_;
161	const unsigned reused_; // tid_ reuse count.
162	// Current thread time when it acquired something from other threads.
163	u64 last_acquire_;
164
165	// Last time another thread has done a global acquire of this thread's clock.
166	// It helps to avoid problem described in:
167	// https://github.com/golang/go/issues/39186
168	// See test/tsan/java_finalizer2.cpp for a regression test.
169	// Note the failuire is _extremely_ hard to hit, so if you are trying
170	// to reproduce it, you may want to run something like:
171	// $ go get golang.org/x/tools/cmd/stress
172	// $ stress -p=64 ./a.out
173	//
174	// The crux of the problem is roughly as follows.
175	// A number of O(1) optimizations in the clocks algorithm assume proper
176	// transitive cumulative propagation of clock values. The AcquireGlobal
177	// operation may produce an inconsistent non-linearazable view of
178	// thread clocks. Namely, it may acquire a later value from a thread
179	// with a higher ID, but fail to acquire an earlier value from a thread
180	// with a lower ID. If a thread that executed AcquireGlobal then releases
181	// to a sync clock, it will spoil the sync clock with the inconsistent
182	// values. If another thread later releases to the sync clock, the optimized
183	// algorithm may break.
184	//
185	// The exact sequence of events that leads to the failure.
186	// - thread 1 executes AcquireGlobal
187	// - thread 1 acquires value 1 for thread 2
188	// - thread 2 increments clock to 2
189	// - thread 2 releases to sync object 1
190	// - thread 3 at time 1
191	// - thread 3 acquires from sync object 1
192	// - thread 3 increments clock to 2
193	// - thread 1 acquires value 2 for thread 3
194	// - thread 1 releases to sync object 2
195	// - sync object 2 clock has 1 for thread 2 and 2 for thread 3
196	// - thread 3 releases to sync object 2
197	// - thread 3 sees value 2 in the clock for itself
198	// and decides that it has already released to the clock
199	// and did not acquire anything from other threads after that
200	// (the last_acquire_ check in release operation)
201	// - thread 3 does not update the value for thread 2 in the clock from 1 to 2
202	// - thread 4 acquires from sync object 2
203	// - thread 4 detects a false race with thread 2
204	// as it should have been synchronized with thread 2 up to time 2,
205	// but because of the broken clock it is now synchronized only up to time 1
206	//
207	// The global_acquire_ value helps to prevent this scenario.
208	// Namely, thread 3 will not trust any own clock values up to global_acquire_
209	// for the purposes of the last_acquire_ optimization.
210	atomic_uint64_t global_acquire_;
211
212	// Cached SyncClock (without dirty entries and release_store_tid_).
213	// We reuse it for subsequent store-release operations without intervening
214	// acquire operations. Since it is shared (and thus constant), clock value
215	// for the current thread is then stored in dirty entries in the SyncClock.
216	// We host a refernece to the table while it is cached here.
217	u32 cached_idx_;
218	u16 cached_size_;
219	u16 cached_blocks_;
220
221	// Number of active elements in the clk_ table (the rest is zeros).
222	uptr nclk_;
223	u64 clk_[kMaxTidInClock]; // Fixed size vector clock.
224
225	bool IsAlreadyAcquired(const SyncClock *src) const;
226	bool HasAcquiredAfterRelease(const SyncClock *dst) const;
227	void UpdateCurrentThread(ClockCache c, SyncClock dst) const;
228	};
229
230	ALWAYS_INLINE u64 ThreadClock::get(unsigned tid) const {
231	DCHECK_LT(tid, kMaxTidInClock);
232	return clk_[tid];
233	}
234
235	ALWAYS_INLINE void ThreadClock::set(u64 v) {
236	DCHECK_GE(v, clk_[tid_]);
237	clk_[tid_] = v;
238	}
239
240	ALWAYS_INLINE void ThreadClock::tick() {
241	clk_[tid_]++;
242	}
243
244	ALWAYS_INLINE uptr ThreadClock::size() const {
245	return nclk_;
246	}
247
248	ALWAYS_INLINE void ThreadClock::NoteGlobalAcquire(u64 v) {
249	// Here we rely on the fact that AcquireGlobal is protected by
250	// ThreadRegistryLock, thus only one thread at a time executes it
251	// and values passed to this function should not go backwards.
252	CHECK_LE(atomic_load_relaxed(&global_acquire_), v);
253	atomic_store_relaxed(&global_acquire_, v);
254	}
255
256	ALWAYS_INLINE SyncClock::Iter SyncClock::begin() {
257	return Iter(this);
258	}
259
260	ALWAYS_INLINE SyncClock::Iter SyncClock::end() {
261	return Iter(nullptr);
262	}
263
264	ALWAYS_INLINE uptr SyncClock::size() const {
265	return size_;
266	}
267
268	ALWAYS_INLINE SyncClock::Iter::Iter(SyncClock* parent)
269	: parent_(parent)
270	, pos_(nullptr)
271	, end_(nullptr)
272	, block_(-1) {
273	if (parent)
274	Next();
275	}
276
277	ALWAYS_INLINE SyncClock::Iter& SyncClock::Iter::operator++() {
278	pos_++;
279	if (UNLIKELY(pos_ >= end_))
280	Next();
281	return *this;
282	}
283
284	ALWAYS_INLINE bool SyncClock::Iter::operator!=(const SyncClock::Iter& other) {
285	return parent_ != other.parent_;
286	}
287
288	ALWAYS_INLINE ClockElem &SyncClock::Iter::operator*() {
289	return *pos_;
290	}
291	} // namespace __tsan
292
293	#endif // TSAN_CLOCK_H

lib/tsan/tsan_debugging.cpp+6-6

...	@@ -157,7 +157,7 @@ int __tsan_get_report_mutex(void report, uptr idx, uptr mutex_id, void **addr,	...	@@ -157,7 +157,7 @@ int __tsan_get_report_mutex(void report, uptr idx, uptr mutex_id, void **addr,
157	ReportMutex *mutex = rep->mutexes[idx];	157	ReportMutex *mutex = rep->mutexes[idx];
158	*mutex_id = mutex->id;	158	*mutex_id = mutex->id;
159	addr = (void )mutex->addr;	159	addr = (void )mutex->addr;
160	*destroyed = mutex->destroyed;	160	*destroyed = false;
161	if (mutex->stack) CopyTrace(mutex->stack->frames, trace, trace_size);	161	if (mutex->stack) CopyTrace(mutex->stack->frames, trace, trace_size);
162	return 1;	162	return 1;
163	}	163	}
...	@@ -195,9 +195,9 @@ const char __tsan_locate_address(uptr addr, char name, uptr name_size,	...	@@ -195,9 +195,9 @@ const char __tsan_locate_address(uptr addr, char name, uptr name_size,
195	const char *region_kind = nullptr;	195	const char *region_kind = nullptr;
196	if (name && name_size > 0) name[0] = 0;	196	if (name && name_size > 0) name[0] = 0;
197		197
198	if (IsMetaMem(addr)) {	198	if (IsMetaMem(reinterpret_cast<u32 *>(addr))) {
199	region_kind = "meta shadow";	199	region_kind = "meta shadow";
200	} else if (IsShadowMem(addr)) {	200	} else if (IsShadowMem(reinterpret_cast<RawShadow *>(addr))) {
201	region_kind = "shadow";	201	region_kind = "shadow";
202	} else {	202	} else {
203	bool is_stack = false;	203	bool is_stack = false;
...	@@ -215,9 +215,9 @@ const char __tsan_locate_address(uptr addr, char name, uptr name_size,	...	@@ -215,9 +215,9 @@ const char __tsan_locate_address(uptr addr, char name, uptr name_size,
215	} else {	215	} else {
216	// TODO(kuba.brecka): We should not lock. This is supposed to be called	216	// TODO(kuba.brecka): We should not lock. This is supposed to be called
217	// from within the debugger when other threads are stopped.	217	// from within the debugger when other threads are stopped.
218	ctx->thread_registry->Lock();	218	ctx->thread_registry.Lock();
219	ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack);	219	ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack);
220	ctx->thread_registry->Unlock();	220	ctx->thread_registry.Unlock();
221	if (tctx) {	221	if (tctx) {
222	region_kind = is_stack ? "stack" : "tls";	222	region_kind = is_stack ? "stack" : "tls";
223	} else {	223	} else {
...	@@ -252,7 +252,7 @@ int __tsan_get_alloc_stack(uptr addr, uptr trace, uptr size, int thread_id,	...	@@ -252,7 +252,7 @@ int __tsan_get_alloc_stack(uptr addr, uptr trace, uptr size, int thread_id,
252	*thread_id = b->tid;	252	*thread_id = b->tid;
253	// No locking. This is supposed to be called from within the debugger when	253	// No locking. This is supposed to be called from within the debugger when
254	// other threads are stopped.	254	// other threads are stopped.
255	ThreadContextBase *tctx = ctx->thread_registry->GetThreadLocked(b->tid);	255	ThreadContextBase *tctx = ctx->thread_registry.GetThreadLocked(b->tid);
256	*os_id = tctx->os_id;	256	*os_id = tctx->os_id;
257		257
258	StackTrace stack = StackDepotGet(b->stk);	258	StackTrace stack = StackDepotGet(b->stk);

lib/tsan/tsan_defs.h+64-36

 @@ -18,6 +18,24 @@
 #include "sanitizer_common/sanitizer_mutex.h"
 #include "ubsan/ubsan_platform.h"
+#ifndef TSAN_VECTORIZE
+#  define TSAN_VECTORIZE __SSE4_2__
+#endif
+#if TSAN_VECTORIZE
+// <emmintrin.h> transitively includes <stdlib.h>,
+// and it's prohibited to include std headers into tsan runtime.
+// So we do this dirty trick.
+#  define _MM_MALLOC_H_INCLUDED
+#  define __MM_MALLOC_H
+#  include <emmintrin.h>
+#  include <smmintrin.h>
+#  define VECTOR_ALIGNED ALIGNED(16)
+typedef __m128i m128;
+#else
+#  define VECTOR_ALIGNED
+#endif
 // Setup defaults for compile definitions.
 #ifndef TSAN_NO_HISTORY
 # define TSAN_NO_HISTORY 0
 @@ -33,40 +51,26 @@
 namespace __tsan {
-const int kClkBits = 42;
+constexpr uptr kByteBits = 8;
-const unsigned kMaxTidReuse = (1 << (64 - kClkBits)) - 1;
-struct ClockElem {
+// Thread slot ID.
-  u64 epoch  : kClkBits;
+enum class Sid : u8 {};
-  u64 reused : 64 - kClkBits;  // tid reuse count
+constexpr uptr kThreadSlotCount = 256;
-};
+constexpr Sid kFreeSid = static_cast<Sid>(255);
-struct ClockBlock {
+// Abstract time unit, vector clock element.
-  static const uptr kSize = 512;
+enum class Epoch : u16 {};
-  static const uptr kTableSize = kSize / sizeof(u32);
+constexpr uptr kEpochBits = 14;
-  static const uptr kClockCount = kSize / sizeof(ClockElem);
+constexpr Epoch kEpochZero = static_cast<Epoch>(0);
-  static const uptr kRefIdx = kTableSize - 1;
+constexpr Epoch kEpochOver = static_cast<Epoch>(1 << kEpochBits);
-  static const uptr kBlockIdx = kTableSize - 2;
+constexpr Epoch kEpochLast = static_cast<Epoch>((1 << kEpochBits) - 1);
-  union {
+inline Epoch EpochInc(Epoch epoch) {
-    u32       table[kTableSize];
+  return static_cast<Epoch>(static_cast<u16>(epoch) + 1);
-    ClockElem clock[kClockCount];
+}
-  };
-  ClockBlock() {
+inline bool EpochOverflow(Epoch epoch) { return epoch == kEpochOver; }
-  }
-};
-const int kTidBits = 13;
-// Reduce kMaxTid by kClockCount because one slot in ClockBlock table is
-// occupied by reference counter, so total number of elements we can store
-// in SyncClock is kClockCount * (kTableSize - 1).
-const unsigned kMaxTid = (1 << kTidBits) - ClockBlock::kClockCount;
-#if !SANITIZER_GO
-const unsigned kMaxTidInClock = kMaxTid * 2;  // This includes msb 'freed' bit.
-#else
-const unsigned kMaxTidInClock = kMaxTid;  // Go does not track freed memory.
-#endif
 const uptr kShadowStackSize = 64 * 1024;
 // Count of shadow values in a shadow cell.
 @@ -75,8 +79,9 @@ const uptr kShadowCnt = 4;
 // That many user bytes are mapped onto a single shadow cell.
 const uptr kShadowCell = 8;
-// Size of a single shadow value (u64).
+// Single shadow value.
-const uptr kShadowSize = 8;
+enum class RawShadow : u32 {};
+const uptr kShadowSize = sizeof(RawShadow);
 // Shadow memory is kShadowMultiplier times larger than user memory.
 const uptr kShadowMultiplier = kShadowSize * kShadowCnt / kShadowCell;
 @@ -88,6 +93,9 @@ const uptr kMetaShadowCell = 8;
 // Size of a single meta shadow value (u32).
 const uptr kMetaShadowSize = 4;
+// All addresses and PCs are assumed to be compressable to that many bits.
+const uptr kCompressedAddrBits = 44;
 #if TSAN_NO_HISTORY
 const bool kCollectHistory = false;
 #else
 @@ -149,17 +157,34 @@ MD5Hash md5_hash(const void *data, uptr size);
 struct Processor;
 struct ThreadState;
 class ThreadContext;
+struct TidSlot;
 struct Context;
 struct ReportStack;
 class ReportDesc;
 class RegionAlloc;
+struct Trace;
+struct TracePart;
+typedef uptr AccessType;
+enum : AccessType {
+  kAccessWrite = 0,
+  kAccessRead = 1 << 0,
+  kAccessAtomic = 1 << 1,
+  kAccessVptr = 1 << 2,  // read or write of an object virtual table pointer
+  kAccessFree = 1 << 3,  // synthetic memory access during memory freeing
+  kAccessExternalPC = 1 << 4,  // access PC can have kExternalPCBit set
+  kAccessCheckOnly = 1 << 5,   // check for races, but don't store
+  kAccessNoRodata = 1 << 6,    // don't check for .rodata marker
+  kAccessSlotLocked = 1 << 7,  // memory access with TidSlot locked
+};
 // Descriptor of user's memory block.
 struct MBlock {
   u64  siz : 48;
   u64  tag : 16;
-  u32  stk;
+  StackID stk;
-  u16  tid;
+  Tid tid;
 };
 COMPILER_CHECK(sizeof(MBlock) == 16);
 @@ -173,15 +198,18 @@ enum ExternalTag : uptr {
   // as 16-bit values, see tsan_defs.h.
 };
-enum MutexType {
+enum {
-  MutexTypeTrace = MutexLastCommon,
+  MutexTypeReport = MutexLastCommon,
-  MutexTypeReport,
   MutexTypeSyncVar,
   MutexTypeAnnotations,
   MutexTypeAtExit,
   MutexTypeFired,
   MutexTypeRacy,
   MutexTypeGlobalProc,
+  MutexTypeInternalAlloc,
+  MutexTypeTrace,
+  MutexTypeSlot,
+  MutexTypeSlots,
 };
 }  // namespace __tsan

lib/tsan/tsan_dense_alloc.h+89-45

...	@@ -49,11 +49,7 @@ class DenseSlabAlloc {	...	@@ -49,11 +49,7 @@ class DenseSlabAlloc {
49	static_assert(sizeof(T) > sizeof(IndexT),	49	static_assert(sizeof(T) > sizeof(IndexT),
50	"it doesn't make sense to use dense alloc");	50	"it doesn't make sense to use dense alloc");
51		51
52	explicit DenseSlabAlloc(LinkerInitialized, const char *name) {	52	DenseSlabAlloc(LinkerInitialized, const char *name) : name_(name) {}
53	freelist_ = 0;
54	fillpos_ = 0;
55	name_ = name;
56	}
57		53
58	explicit DenseSlabAlloc(const char *name)	54	explicit DenseSlabAlloc(const char *name)
59	: DenseSlabAlloc(LINKER_INITIALIZED, name) {	55	: DenseSlabAlloc(LINKER_INITIALIZED, name) {
...	@@ -89,12 +85,7 @@ class DenseSlabAlloc {	...	@@ -89,12 +85,7 @@ class DenseSlabAlloc {
89	}	85	}
90		86
91	void FlushCache(Cache *c) {	87	void FlushCache(Cache *c) {
92	SpinMutexLock lock(&mtx_);	88	while (c->pos) Drain(c);
93	while (c->pos) {
94	IndexT idx = c->cache[--c->pos];
95	(IndexT)Map(idx) = freelist_;
96	freelist_ = idx;
97	}
98	}	89	}
99		90
100	void InitCache(Cache *c) {	91	void InitCache(Cache *c) {
...	@@ -102,48 +93,101 @@ class DenseSlabAlloc {	...	@@ -102,48 +93,101 @@ class DenseSlabAlloc {
102	internal_memset(c->cache, 0, sizeof(c->cache));	93	internal_memset(c->cache, 0, sizeof(c->cache));
103	}	94	}
104		95
		96	uptr AllocatedMemory() const {
		97	return atomic_load_relaxed(&fillpos_) * kL2Size * sizeof(T);
		98	}
		99
		100	template <typename Func>
		101	void ForEach(Func func) {
		102	Lock lock(&mtx_);
		103	uptr fillpos = atomic_load_relaxed(&fillpos_);
		104	for (uptr l1 = 0; l1 < fillpos; l1++) {
		105	for (IndexT l2 = l1 == 0 ? 1 : 0; l2 < kL2Size; l2++) func(&map_[l1][l2]);
		106	}
		107	}
		108
105	private:	109	private:
106	T *map_[kL1Size];	110	T *map_[kL1Size];
107	SpinMutex mtx_;	111	Mutex mtx_;
108	IndexT freelist_;	112	// The freelist is organized as a lock-free stack of batches of nodes.
109	uptr fillpos_;	113	// The stack itself uses Block::next links, while the batch within each
110	const char *name_;	114	// stack node uses Block::batch links.
111		115	// Low 32-bits of freelist_ is the node index, top 32-bits is ABA-counter.
112	void Refill(Cache *c) {	116	atomic_uint64_t freelist_ = {0};
113	SpinMutexLock lock(&mtx_);	117	atomic_uintptr_t fillpos_ = {0};
114	if (freelist_ == 0) {	118	const char *const name_;
115	if (fillpos_ == kL1Size) {	119
116	Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n",	120	struct Block {
117	name_, kL1Size, kL2Size);	121	IndexT next;
118	Die();	122	IndexT batch;
119	}	123	};
120	VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n",	124
121	name_, fillpos_, kL1Size, kL2Size);	125	Block MapBlock(IndexT idx) { return reinterpret_cast<Block >(Map(idx)); }
122	T batch = (T)MmapOrDie(kL2Size * sizeof(T), name_);	126
123	// Reserve 0 as invalid index.	127	static constexpr u64 kCounterInc = 1ull << 32;
124	IndexT start = fillpos_ == 0 ? 1 : 0;	128	static constexpr u64 kCounterMask = ~(kCounterInc - 1);
125	for (IndexT i = start; i < kL2Size; i++) {	129
126	new(batch + i) T;	130	NOINLINE void Refill(Cache *c) {
127	(IndexT)(batch + i) = i + 1 + fillpos_ * kL2Size;	131	// Pop 1 batch of nodes from the freelist.
128	}	132	IndexT idx;
129	(IndexT)(batch + kL2Size - 1) = 0;	133	u64 xchg;
130	freelist_ = fillpos_ * kL2Size + start;	134	u64 cmp = atomic_load(&freelist_, memory_order_acquire);
131	map_[fillpos_++] = batch;	135	do {
132	}	136	idx = static_cast<IndexT>(cmp);
133	for (uptr i = 0; i < Cache::kSize / 2 && freelist_ != 0; i++) {	137	if (!idx)
134	IndexT idx = freelist_;	138	return AllocSuperBlock(c);
		139	Block *ptr = MapBlock(idx);
		140	xchg = ptr->next \| (cmp & kCounterMask);
		141	} while (!atomic_compare_exchange_weak(&freelist_, &cmp, xchg,
		142	memory_order_acq_rel));
		143	// Unpack it into c->cache.
		144	while (idx) {
135	c->cache[c->pos++] = idx;	145	c->cache[c->pos++] = idx;
136	freelist_ = (IndexT)Map(idx);	146	idx = MapBlock(idx)->batch;
137	}	147	}
138	}	148	}
139		149
140	void Drain(Cache *c) {	150	NOINLINE void Drain(Cache *c) {
141	SpinMutexLock lock(&mtx_);	151	// Build a batch of at most Cache::kSize / 2 nodes linked by Block::batch.
142	for (uptr i = 0; i < Cache::kSize / 2; i++) {	152	IndexT head_idx = 0;
		153	for (uptr i = 0; i < Cache::kSize / 2 && c->pos; i++) {
143	IndexT idx = c->cache[--c->pos];	154	IndexT idx = c->cache[--c->pos];
144	(IndexT)Map(idx) = freelist_;	155	Block *ptr = MapBlock(idx);
145	freelist_ = idx;	156	ptr->batch = head_idx;
		157	head_idx = idx;
		158	}
		159	// Push it onto the freelist stack.
		160	Block *head = MapBlock(head_idx);
		161	u64 xchg;
		162	u64 cmp = atomic_load(&freelist_, memory_order_acquire);
		163	do {
		164	head->next = static_cast<IndexT>(cmp);
		165	xchg = head_idx \| (cmp & kCounterMask) + kCounterInc;
		166	} while (!atomic_compare_exchange_weak(&freelist_, &cmp, xchg,
		167	memory_order_acq_rel));
		168	}
		169
		170	NOINLINE void AllocSuperBlock(Cache *c) {
		171	Lock lock(&mtx_);
		172	uptr fillpos = atomic_load_relaxed(&fillpos_);
		173	if (fillpos == kL1Size) {
		174	Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n", name_, kL1Size,
		175	kL2Size);
		176	Die();
		177	}
		178	VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n", name_,
		179	fillpos, kL1Size, kL2Size);
		180	T batch = (T )MmapOrDie(kL2Size * sizeof(T), name_);
		181	map_[fillpos] = batch;
		182	// Reserve 0 as invalid index.
		183	for (IndexT i = fillpos ? 0 : 1; i < kL2Size; i++) {
		184	new (batch + i) T;
		185	c->cache[c->pos++] = i + fillpos * kL2Size;
		186	if (c->pos == Cache::kSize)
		187	Drain(c);
146	}	188	}
		189	atomic_store_relaxed(&fillpos_, fillpos + 1);
		190	CHECK(c->pos);
147	}	191	}
148	};	192	};
149		193

lib/tsan/tsan_dispatch_defs.h created+73

...	@@ -0,0 +1,73 @@
1	//===-- tsan_dispatch_defs.h ------------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	//===----------------------------------------------------------------------===//
12	#ifndef TSAN_DISPATCH_DEFS_H
13	#define TSAN_DISPATCH_DEFS_H
14
15	#include "sanitizer_common/sanitizer_internal_defs.h"
16
17	typedef struct dispatch_object_s {} *dispatch_object_t;
18
19	#define DISPATCH_DECL(name) \
20	typedef struct name##_s : public dispatch_object_s {} *name##_t
21
22	DISPATCH_DECL(dispatch_queue);
23	DISPATCH_DECL(dispatch_source);
24	DISPATCH_DECL(dispatch_group);
25	DISPATCH_DECL(dispatch_data);
26	DISPATCH_DECL(dispatch_semaphore);
27	DISPATCH_DECL(dispatch_io);
28
29	typedef void (dispatch_function_t)(void arg);
30	typedef void (^dispatch_block_t)(void);
31	typedef void (^dispatch_io_handler_t)(bool done, dispatch_data_t data,
32	int error);
33
34	typedef long dispatch_once_t;
35	typedef __sanitizer::u64 dispatch_time_t;
36	typedef int dispatch_fd_t;
37	typedef unsigned long dispatch_io_type_t;
38	typedef unsigned long dispatch_io_close_flags_t;
39
40	extern "C" {
41	void *dispatch_get_context(dispatch_object_t object);
42	void dispatch_retain(dispatch_object_t object);
43	void dispatch_release(dispatch_object_t object);
44
45	extern const dispatch_block_t _dispatch_data_destructor_free;
46	extern const dispatch_block_t _dispatch_data_destructor_munmap;
47	} // extern "C"
48
49	#define DISPATCH_DATA_DESTRUCTOR_DEFAULT nullptr
50	#define DISPATCH_DATA_DESTRUCTOR_FREE _dispatch_data_destructor_free
51	#define DISPATCH_DATA_DESTRUCTOR_MUNMAP _dispatch_data_destructor_munmap
52
53	#if __has_attribute(noescape)
54	# define DISPATCH_NOESCAPE __attribute__((__noescape__))
55	#else
56	# define DISPATCH_NOESCAPE
57	#endif
58
59	#if SANITIZER_APPLE
60	# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak_import))
61	#else
62	# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak))
63	#endif
64
65
66	// Data types used in dispatch APIs
67	typedef unsigned long size_t;
68	typedef unsigned long uintptr_t;
69	typedef __sanitizer::s64 off_t;
70	typedef __sanitizer::u16 mode_t;
71	typedef long long_t;
72
73	#endif // TSAN_DISPATCH_DEFS_H

lib/tsan/tsan_external.cpp+24-16

 @@ -10,9 +10,12 @@
 //
 //===----------------------------------------------------------------------===//
 #include "tsan_rtl.h"
-#include "tsan_interceptors.h"
 #include "sanitizer_common/sanitizer_ptrauth.h"
+#if !SANITIZER_GO
+#  include "tsan_interceptors.h"
+#endif
 namespace __tsan {
 #define CALLERPC ((uptr)__builtin_return_address(0))
 @@ -43,10 +46,6 @@ const char *GetReportHeaderFromTag(uptr tag) {
   return tag_data ? tag_data->header : nullptr;
 }
-void InsertShadowStackFrameForTag(ThreadState *thr, uptr tag) {
-  FuncEntry(thr, (uptr)&registered_tags[tag]);
-}
 uptr TagFromShadowStackFrame(uptr pc) {
   uptr tag_count = atomic_load(&used_tags, memory_order_relaxed);
   void *pc_ptr = (void *)pc;
 @@ -57,17 +56,26 @@ uptr TagFromShadowStackFrame(uptr pc) {
 #if !SANITIZER_GO
-typedef void(*AccessFunc)(ThreadState *, uptr, uptr, int);
+// We need to track tags for individual memory accesses, but there is no space
-void ExternalAccess(void *addr, uptr caller_pc, void *tag, AccessFunc access) {
+// in the shadow cells for them.  Instead we push/pop them onto the thread
+// traces and ignore the extra tag frames when printing reports.
+static void PushTag(ThreadState *thr, uptr tag) {
+  FuncEntry(thr, (uptr)&registered_tags[tag]);
+}
+static void PopTag(ThreadState *thr) { FuncExit(thr); }
+static void ExternalAccess(void *addr, uptr caller_pc, uptr tsan_caller_pc,
+                           void *tag, AccessType typ) {
   CHECK_LT(tag, atomic_load(&used_tags, memory_order_relaxed));
+  bool in_ignored_lib;
+  if (caller_pc && libignore()->IsIgnored(caller_pc, &in_ignored_lib))
+    return;
   ThreadState *thr = cur_thread();
   if (caller_pc) FuncEntry(thr, caller_pc);
-  InsertShadowStackFrameForTag(thr, (uptr)tag);
+  PushTag(thr, (uptr)tag);
-  bool in_ignored_lib;
+  MemoryAccess(thr, tsan_caller_pc, (uptr)addr, 1, typ);
-  if (!caller_pc || !libignore()->IsIgnored(caller_pc, &in_ignored_lib)) {
+  PopTag(thr);
-    access(thr, CALLERPC, (uptr)addr, kSizeLog1);
-  }
-  FuncExit(thr);
   if (caller_pc) FuncExit(thr);
 }
 @@ -92,7 +100,7 @@ void __tsan_external_register_header(void *tag, const char *header) {
   header = internal_strdup(header);
   char *old_header =
       (char *)atomic_exchange(header_ptr, (uptr)header, memory_order_seq_cst);
-  if (old_header) internal_free(old_header);
+  Free(old_header);
 }
 SANITIZER_INTERFACE_ATTRIBUTE
 @@ -111,12 +119,12 @@ void __tsan_external_assign_tag(void *addr, void *tag) {
 SANITIZER_INTERFACE_ATTRIBUTE
 void __tsan_external_read(void *addr, void *caller_pc, void *tag) {
-  ExternalAccess(addr, STRIP_PAC_PC(caller_pc), tag, MemoryRead);
+  ExternalAccess(addr, STRIP_PAC_PC(caller_pc), CALLERPC, tag, kAccessRead);
 }
 SANITIZER_INTERFACE_ATTRIBUTE
 void __tsan_external_write(void *addr, void *caller_pc, void *tag) {
-  ExternalAccess(addr, STRIP_PAC_PC(caller_pc), tag, MemoryWrite);
+  ExternalAccess(addr, STRIP_PAC_PC(caller_pc), CALLERPC, tag, kAccessWrite);
 }
 }  // extern "C"

lib/tsan/tsan_fd.cpp+86-27

...	@@ -11,9 +11,12 @@	...	@@ -11,9 +11,12 @@
11	//===----------------------------------------------------------------------===//	11	//===----------------------------------------------------------------------===//
12		12
13	#include "tsan_fd.h"	13	#include "tsan_fd.h"
14	#include "tsan_rtl.h"	14
15	#include <sanitizer_common/sanitizer_atomic.h>	15	#include <sanitizer_common/sanitizer_atomic.h>
16		16
		17	#include "tsan_interceptors.h"
		18	#include "tsan_rtl.h"
		19
17	namespace __tsan {	20	namespace __tsan {
18		21
19	const int kTableSizeL1 = 1024;	22	const int kTableSizeL1 = 1024;
...	@@ -26,8 +29,12 @@ struct FdSync {	...	@@ -26,8 +29,12 @@ struct FdSync {
26		29
27	struct FdDesc {	30	struct FdDesc {
28	FdSync *sync;	31	FdSync *sync;
29	int creation_tid;	32	// This is used to establish write -> epoll_wait synchronization
30	u32 creation_stack;	33	// where epoll_wait receives notification about the write.
		34	atomic_uintptr_t aux_sync; // FdSync*
		35	Tid creation_tid;
		36	StackID creation_stack;
		37	bool closed;
31	};	38	};
32		39
33	struct FdContext {	40	struct FdContext {
...	@@ -100,6 +107,10 @@ static void init(ThreadState thr, uptr pc, int fd, FdSync s,	...	@@ -100,6 +107,10 @@ static void init(ThreadState thr, uptr pc, int fd, FdSync s,
100	unref(thr, pc, d->sync);	107	unref(thr, pc, d->sync);
101	d->sync = 0;	108	d->sync = 0;
102	}	109	}
		110	unref(thr, pc,
		111	reinterpret_cast<FdSync *>(
		112	atomic_load(&d->aux_sync, memory_order_relaxed)));
		113	atomic_store(&d->aux_sync, 0, memory_order_relaxed);
103	if (flags()->io_sync == 0) {	114	if (flags()->io_sync == 0) {
104	unref(thr, pc, s);	115	unref(thr, pc, s);
105	} else if (flags()->io_sync == 1) {	116	} else if (flags()->io_sync == 1) {
...	@@ -110,12 +121,18 @@ static void init(ThreadState thr, uptr pc, int fd, FdSync s,	...	@@ -110,12 +121,18 @@ static void init(ThreadState thr, uptr pc, int fd, FdSync s,
110	}	121	}
111	d->creation_tid = thr->tid;	122	d->creation_tid = thr->tid;
112	d->creation_stack = CurrentStackId(thr, pc);	123	d->creation_stack = CurrentStackId(thr, pc);
		124	d->closed = false;
		125	// This prevents false positives on fd_close_norace3.cpp test.
		126	// The mechanics of the false positive are not completely clear,
		127	// but it happens only if global reset is enabled (flush_memory_ms=1)
		128	// and may be related to lost writes during asynchronous MADV_DONTNEED.
		129	SlotLocker locker(thr);
113	if (write) {	130	if (write) {
114	// To catch races between fd usage and open.	131	// To catch races between fd usage and open.
115	MemoryRangeImitateWrite(thr, pc, (uptr)d, 8);	132	MemoryRangeImitateWrite(thr, pc, (uptr)d, 8);
116	} else {	133	} else {
117	// See the dup-related comment in FdClose.	134	// See the dup-related comment in FdClose.
118	MemoryRead(thr, pc, (uptr)d, kSizeLog8);	135	MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead \| kAccessSlotLocked);
119	}	136	}
120	}	137	}
121		138
...	@@ -140,7 +157,7 @@ void FdOnFork(ThreadState *thr, uptr pc) {	...	@@ -140,7 +157,7 @@ void FdOnFork(ThreadState *thr, uptr pc) {
140	}	157	}
141	}	158	}
142		159
143	bool FdLocation(uptr addr, int fd, int tid, u32 *stack) {	160	bool FdLocation(uptr addr, int fd, Tid tid, StackID stack, bool closed) {
144	for (int l1 = 0; l1 < kTableSizeL1; l1++) {	161	for (int l1 = 0; l1 < kTableSizeL1; l1++) {
145	FdDesc tab = (FdDesc)atomic_load(&fdctx.tab[l1], memory_order_relaxed);	162	FdDesc tab = (FdDesc)atomic_load(&fdctx.tab[l1], memory_order_relaxed);
146	if (tab == 0)	163	if (tab == 0)
...	@@ -151,6 +168,7 @@ bool FdLocation(uptr addr, int fd, int tid, u32 *stack) {	...	@@ -151,6 +168,7 @@ bool FdLocation(uptr addr, int fd, int tid, u32 *stack) {
151	fd = l1 kTableSizeL1 + l2;	168	fd = l1 kTableSizeL1 + l2;
152	*tid = d->creation_tid;	169	*tid = d->creation_tid;
153	*stack = d->creation_stack;	170	*stack = d->creation_stack;
		171	*closed = d->closed;
154	return true;	172	return true;
155	}	173	}
156	}	174	}
...	@@ -163,7 +181,7 @@ void FdAcquire(ThreadState *thr, uptr pc, int fd) {	...	@@ -163,7 +181,7 @@ void FdAcquire(ThreadState *thr, uptr pc, int fd) {
163	FdDesc *d = fddesc(thr, pc, fd);	181	FdDesc *d = fddesc(thr, pc, fd);
164	FdSync *s = d->sync;	182	FdSync *s = d->sync;
165	DPrintf("#%d: FdAcquire(%d) -> %p\n", thr->tid, fd, s);	183	DPrintf("#%d: FdAcquire(%d) -> %p\n", thr->tid, fd, s);
166	MemoryRead(thr, pc, (uptr)d, kSizeLog8);	184	MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
167	if (s)	185	if (s)
168	Acquire(thr, pc, (uptr)s);	186	Acquire(thr, pc, (uptr)s);
169	}	187	}
...	@@ -174,9 +192,11 @@ void FdRelease(ThreadState *thr, uptr pc, int fd) {	...	@@ -174,9 +192,11 @@ void FdRelease(ThreadState *thr, uptr pc, int fd) {
174	FdDesc *d = fddesc(thr, pc, fd);	192	FdDesc *d = fddesc(thr, pc, fd);
175	FdSync *s = d->sync;	193	FdSync *s = d->sync;
176	DPrintf("#%d: FdRelease(%d) -> %p\n", thr->tid, fd, s);	194	DPrintf("#%d: FdRelease(%d) -> %p\n", thr->tid, fd, s);
177	MemoryRead(thr, pc, (uptr)d, kSizeLog8);	195	MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
178	if (s)	196	if (s)
179	Release(thr, pc, (uptr)s);	197	Release(thr, pc, (uptr)s);
		198	if (uptr aux_sync = atomic_load(&d->aux_sync, memory_order_acquire))
		199	Release(thr, pc, aux_sync);
180	}	200	}
181		201
182	void FdAccess(ThreadState *thr, uptr pc, int fd) {	202	void FdAccess(ThreadState *thr, uptr pc, int fd) {
...	@@ -184,7 +204,7 @@ void FdAccess(ThreadState *thr, uptr pc, int fd) {	...	@@ -184,7 +204,7 @@ void FdAccess(ThreadState *thr, uptr pc, int fd) {
184	if (bogusfd(fd))	204	if (bogusfd(fd))
185	return;	205	return;
186	FdDesc *d = fddesc(thr, pc, fd);	206	FdDesc *d = fddesc(thr, pc, fd);
187	MemoryRead(thr, pc, (uptr)d, kSizeLog8);	207	MemoryAccess(thr, pc, (uptr)d, 8, kAccessRead);
188	}	208	}
189		209
190	void FdClose(ThreadState *thr, uptr pc, int fd, bool write) {	210	void FdClose(ThreadState *thr, uptr pc, int fd, bool write) {
...	@@ -192,27 +212,42 @@ void FdClose(ThreadState *thr, uptr pc, int fd, bool write) {	...	@@ -192,27 +212,42 @@ void FdClose(ThreadState *thr, uptr pc, int fd, bool write) {
192	if (bogusfd(fd))	212	if (bogusfd(fd))
193	return;	213	return;
194	FdDesc *d = fddesc(thr, pc, fd);	214	FdDesc *d = fddesc(thr, pc, fd);
195	if (write) {	215	{
196	// To catch races between fd usage and close.	216	// Need to lock the slot to make MemoryAccess and MemoryResetRange atomic
197	MemoryWrite(thr, pc, (uptr)d, kSizeLog8);	217	// with respect to global reset. See the comment in MemoryRangeFreed.
198	} else {	218	SlotLocker locker(thr);
199	// This path is used only by dup2/dup3 calls.	219	if (!MustIgnoreInterceptor(thr)) {
200	// We do read instead of write because there is a number of legitimate	220	if (write) {
201	// cases where write would lead to false positives:	221	// To catch races between fd usage and close.
202	// 1. Some software dups a closed pipe in place of a socket before closing	222	MemoryAccess(thr, pc, (uptr)d, 8,
203	// the socket (to prevent races actually).	223	kAccessWrite \| kAccessCheckOnly \| kAccessSlotLocked);
204	// 2. Some daemons dup /dev/null in place of stdin/stdout.	224	} else {
205	// On the other hand we have not seen cases when write here catches real	225	// This path is used only by dup2/dup3 calls.
206	// bugs.	226	// We do read instead of write because there is a number of legitimate
207	MemoryRead(thr, pc, (uptr)d, kSizeLog8);	227	// cases where write would lead to false positives:
		228	// 1. Some software dups a closed pipe in place of a socket before
		229	// closing
		230	// the socket (to prevent races actually).
		231	// 2. Some daemons dup /dev/null in place of stdin/stdout.
		232	// On the other hand we have not seen cases when write here catches real
		233	// bugs.
		234	MemoryAccess(thr, pc, (uptr)d, 8,
		235	kAccessRead \| kAccessCheckOnly \| kAccessSlotLocked);
		236	}
		237	}
		238	// We need to clear it, because if we do not intercept any call out there
		239	// that creates fd, we will hit false postives.
		240	MemoryResetRange(thr, pc, (uptr)d, 8);
208	}	241	}
209	// We need to clear it, because if we do not intercept any call out there
210	// that creates fd, we will hit false postives.
211	MemoryResetRange(thr, pc, (uptr)d, 8);
212	unref(thr, pc, d->sync);	242	unref(thr, pc, d->sync);
213	d->sync = 0;	243	d->sync = 0;
214	d->creation_tid = 0;	244	unref(thr, pc,
215	d->creation_stack = 0;	245	reinterpret_cast<FdSync *>(
		246	atomic_load(&d->aux_sync, memory_order_relaxed)));
		247	atomic_store(&d->aux_sync, 0, memory_order_relaxed);
		248	d->closed = true;
		249	d->creation_tid = thr->tid;
		250	d->creation_stack = CurrentStackId(thr, pc);
216	}	251	}
217		252
218	void FdFileCreate(ThreadState *thr, uptr pc, int fd) {	253	void FdFileCreate(ThreadState *thr, uptr pc, int fd) {
...	@@ -228,7 +263,7 @@ void FdDup(ThreadState *thr, uptr pc, int oldfd, int newfd, bool write) {	...	@@ -228,7 +263,7 @@ void FdDup(ThreadState *thr, uptr pc, int oldfd, int newfd, bool write) {
228	return;	263	return;
229	// Ignore the case when user dups not yet connected socket.	264	// Ignore the case when user dups not yet connected socket.
230	FdDesc *od = fddesc(thr, pc, oldfd);	265	FdDesc *od = fddesc(thr, pc, oldfd);
231	MemoryRead(thr, pc, (uptr)od, kSizeLog8);	266	MemoryAccess(thr, pc, (uptr)od, 8, kAccessRead);
232	FdClose(thr, pc, newfd, write);	267	FdClose(thr, pc, newfd, write);
233	init(thr, pc, newfd, ref(od->sync), write);	268	init(thr, pc, newfd, ref(od->sync), write);
234	}	269	}
...	@@ -269,6 +304,30 @@ void FdPollCreate(ThreadState *thr, uptr pc, int fd) {	...	@@ -269,6 +304,30 @@ void FdPollCreate(ThreadState *thr, uptr pc, int fd) {
269	init(thr, pc, fd, allocsync(thr, pc));	304	init(thr, pc, fd, allocsync(thr, pc));
270	}	305	}
271		306
		307	void FdPollAdd(ThreadState *thr, uptr pc, int epfd, int fd) {
		308	DPrintf("#%d: FdPollAdd(%d, %d)\n", thr->tid, epfd, fd);
		309	if (bogusfd(epfd) \|\| bogusfd(fd))
		310	return;
		311	FdDesc *d = fddesc(thr, pc, fd);
		312	// Associate fd with epoll fd only once.
		313	// While an fd can be associated with multiple epolls at the same time,
		314	// or with different epolls during different phases of lifetime,
		315	// synchronization semantics (and examples) of this are unclear.
		316	// So we don't support this for now.
		317	// If we change the association, it will also create lifetime management
		318	// problem for FdRelease which accesses the aux_sync.
		319	if (atomic_load(&d->aux_sync, memory_order_relaxed))
		320	return;
		321	FdDesc *epd = fddesc(thr, pc, epfd);
		322	FdSync *s = epd->sync;
		323	if (!s)
		324	return;
		325	uptr cmp = 0;
		326	if (atomic_compare_exchange_strong(
		327	&d->aux_sync, &cmp, reinterpret_cast<uptr>(s), memory_order_release))
		328	ref(s);
		329	}
		330
272	void FdSocketCreate(ThreadState *thr, uptr pc, int fd) {	331	void FdSocketCreate(ThreadState *thr, uptr pc, int fd) {
273	DPrintf("#%d: FdSocketCreate(%d)\n", thr->tid, fd);	332	DPrintf("#%d: FdSocketCreate(%d)\n", thr->tid, fd);
274	if (bogusfd(fd))	333	if (bogusfd(fd))

lib/tsan/tsan_fd.h+2-1

 @@ -49,11 +49,12 @@ void FdEventCreate(ThreadState *thr, uptr pc, int fd);
 void FdSignalCreate(ThreadState *thr, uptr pc, int fd);
 void FdInotifyCreate(ThreadState *thr, uptr pc, int fd);
 void FdPollCreate(ThreadState *thr, uptr pc, int fd);
+void FdPollAdd(ThreadState *thr, uptr pc, int epfd, int fd);
 void FdSocketCreate(ThreadState *thr, uptr pc, int fd);
 void FdSocketAccept(ThreadState *thr, uptr pc, int fd, int newfd);
 void FdSocketConnecting(ThreadState *thr, uptr pc, int fd);
 void FdSocketConnect(ThreadState *thr, uptr pc, int fd);
-bool FdLocation(uptr addr, int *fd, int *tid, u32 *stack);
+bool FdLocation(uptr addr, int *fd, Tid *tid, StackID *stack, bool *closed);
 void FdOnFork(ThreadState *thr, uptr pc);
 uptr File2addr(const char *path);

lib/tsan/tsan_flags.cpp+8-11

 @@ -10,19 +10,21 @@
 //
 //===----------------------------------------------------------------------===//
-#include "sanitizer_common/sanitizer_flags.h"
+#include "tsan_flags.h"
 #include "sanitizer_common/sanitizer_flag_parser.h"
+#include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_libc.h"
-#include "tsan_flags.h"
+#include "tsan_interface.h"
-#include "tsan_rtl.h"
 #include "tsan_mman.h"
+#include "tsan_rtl.h"
 #include "ubsan/ubsan_flags.h"
 namespace __tsan {
 // Can be overriden in frontend.
 #ifdef TSAN_EXTERNAL_HOOKS
-extern "C" const char* __tsan_default_options();
+extern "C" const char *__tsan_default_options();
 #else
 SANITIZER_WEAK_DEFAULT_IMPL
 const char *__tsan_default_options() {
 @@ -55,6 +57,7 @@ void InitializeFlags(Flags *f, const char *env, const char *env_option_name) {
     // Override some common flags defaults.
     CommonFlags cf;
     cf.CopyFrom(*common_flags());
+    cf.external_symbolizer_path = GetEnv("TSAN_SYMBOLIZER_PATH");
     cf.allow_addr2line = true;
     if (SANITIZER_GO) {
       // Does not work as expected for Go: runtime handles SIGABRT and crashes.
 @@ -96,7 +99,7 @@ void InitializeFlags(Flags *f, const char *env, const char *env_option_name) {
   ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");
 #endif
-  // Sanity check.
+  // Check flags.
   if (!f->report_bugs) {
     f->report_thread_leaks = false;
     f->report_destroy_locked = false;
 @@ -109,12 +112,6 @@ void InitializeFlags(Flags *f, const char *env, const char *env_option_name) {
   if (common_flags()->help) parser.PrintFlagDescriptions();
-  if (f->history_size < 0 || f->history_size > 7) {
-    Printf("ThreadSanitizer: incorrect value for history_size"
-           " (must be [0..7])\n");
-    Die();
-  }
   if (f->io_sync < 0 || f->io_sync > 2) {
     Printf("ThreadSanitizer: incorrect value for io_sync"
            " (must be [0..2])\n");

lib/tsan/tsan_flags.inc+11-14

 @@ -23,10 +23,6 @@ TSAN_FLAG(bool, enable_annotations, true,
 TSAN_FLAG(bool, suppress_equal_stacks, true,
           "Suppress a race report if we've already output another race report "
           "with the same stack.")
-TSAN_FLAG(bool, suppress_equal_addresses, true,
-          "Suppress a race report if we've already output another race report "
-          "on the same address.")
 TSAN_FLAG(bool, report_bugs, true,
           "Turns off bug reporting entirely (useful for benchmarking).")
 TSAN_FLAG(bool, report_thread_leaks, true, "Report thread leaks at exit?")
 @@ -43,7 +39,9 @@ TSAN_FLAG(
     bool, force_seq_cst_atomics, false,
     "If set, all atomics are effectively sequentially consistent (seq_cst), "
     "regardless of what user actually specified.")
-TSAN_FLAG(bool, print_benign, false, "Print matched \"benign\" races at exit.")
+TSAN_FLAG(bool, force_background_thread, false,
+          "If set, eagerly launch a background thread for memory reclamation "
+          "instead of waiting for a user call to pthread_create.")
 TSAN_FLAG(bool, halt_on_error, false, "Exit after first reported error.")
 TSAN_FLAG(int, atexit_sleep_ms, 1000,
           "Sleep in main thread before exiting for that many ms "
 @@ -60,14 +58,10 @@ TSAN_FLAG(bool, stop_on_start, false,
           "Stops on start until __tsan_resume() is called (for debugging).")
 TSAN_FLAG(bool, running_on_valgrind, false,
           "Controls whether RunningOnValgrind() returns true or false.")
-// There are a lot of goroutines in Go, so we use smaller history.
 TSAN_FLAG(
-    int, history_size, SANITIZER_GO ? 1 : 3,
+    uptr, history_size, 0,
-    "Per-thread history size, controls how many previous memory accesses "
+    "Per-thread history size,"
-    "are remembered per thread.  Possible values are [0..7]. "
+    " controls how many extra previous memory accesses are remembered per thread.")
-    "history_size=0 amounts to 32K memory accesses.  Each next value doubles "
-    "the amount of memory accesses, up to history_size=7 that amounts to "
-    "4M memory accesses.  The default value is 2 (128K memory accesses).")
 TSAN_FLAG(int, io_sync, 1,
           "Controls level of synchronization implied by IO operations. "
           "0 - no synchronization "
 @@ -76,10 +70,13 @@ TSAN_FLAG(int, io_sync, 1,
 TSAN_FLAG(bool, die_after_fork, true,
           "Die after multi-threaded fork if the child creates new threads.")
 TSAN_FLAG(const char *, suppressions, "", "Suppressions file name.")
-TSAN_FLAG(bool, ignore_interceptors_accesses, SANITIZER_MAC ? true : false,
+TSAN_FLAG(bool, ignore_interceptors_accesses, SANITIZER_APPLE ? true : false,
           "Ignore reads and writes from all interceptors.")
-TSAN_FLAG(bool, ignore_noninstrumented_modules, SANITIZER_MAC ? true : false,
+TSAN_FLAG(bool, ignore_noninstrumented_modules, SANITIZER_APPLE ? true : false,
           "Interceptors should only detect races when called from instrumented "
           "modules.")
 TSAN_FLAG(bool, shared_ptr_interceptor, true,
           "Track atomic reference counting in libc++ shared_ptr and weak_ptr.")
+TSAN_FLAG(bool, print_full_thread_history, false,
+          "If set, prints thread creation stacks for the threads involved in "
+          "the report and their ancestors up to the main thread.")

lib/tsan/tsan_ignoreset.cpp+2-10

 @@ -19,7 +19,7 @@ IgnoreSet::IgnoreSet()
     : size_() {
 }
-void IgnoreSet::Add(u32 stack_id) {
+void IgnoreSet::Add(StackID stack_id) {
   if (size_ == kMaxSize)
     return;
   for (uptr i = 0; i < size_; i++) {
 @@ -29,15 +29,7 @@ void IgnoreSet::Add(u32 stack_id) {
   stacks_[size_++] = stack_id;
 }
-void IgnoreSet::Reset() {
+StackID IgnoreSet::At(uptr i) const {
-  size_ = 0;
-}
-uptr IgnoreSet::Size() const {
-  return size_;
-}
-u32 IgnoreSet::At(uptr i) const {
   CHECK_LT(i, size_);
   CHECK_LE(size_, kMaxSize);
   return stacks_[i];

lib/tsan/tsan_ignoreset.h+6-7

 @@ -19,17 +19,16 @@ namespace __tsan {
 class IgnoreSet {
  public:
-  static const uptr kMaxSize = 16;
   IgnoreSet();
-  void Add(u32 stack_id);
+  void Add(StackID stack_id);
-  void Reset();
+  void Reset() { size_ = 0; }
-  uptr Size() const;
+  uptr Size() const { return size_; }
-  u32 At(uptr i) const;
+  StackID At(uptr i) const;
  private:
+  static constexpr uptr kMaxSize = 16;
   uptr size_;
-  u32 stacks_[kMaxSize];
+  StackID stacks_[kMaxSize];
 };
 }  // namespace __tsan

lib/tsan/tsan_ilist.h created+189

...	@@ -0,0 +1,189 @@
1	//===-- tsan_ilist.h --------------------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	//===----------------------------------------------------------------------===//
12	#ifndef TSAN_ILIST_H
13	#define TSAN_ILIST_H
14
15	#include "sanitizer_common/sanitizer_internal_defs.h"
16
17	namespace __tsan {
18
19	class INode {
20	public:
21	INode() = default;
22
23	private:
24	INode* next_ = nullptr;
25	INode* prev_ = nullptr;
26
27	template <typename Base, INode Base::*Node, typename Elem>
28	friend class IList;
29	INode(const INode&) = delete;
30	void operator=(const INode&) = delete;
31	};
32
33	// Intrusive doubly-linked list.
34	//
35	// The node class (MyNode) needs to include "INode foo" field,
36	// then the list can be declared as IList<MyNode, &MyNode::foo>.
37	// This design allows to link MyNode into multiple lists using
38	// different INode fields.
39	// The optional Elem template argument allows to specify node MDT
40	// (most derived type) if it's different from MyNode.
41	template <typename Base, INode Base::*Node, typename Elem = Base>
42	class IList {
43	public:
44	IList();
45
46	void PushFront(Elem* e);
47	void PushBack(Elem* e);
48	void Remove(Elem* e);
49
50	Elem* PopFront();
51	Elem* PopBack();
52	Elem* Front();
53	Elem* Back();
54
55	// Prev links point towards front of the queue.
56	Elem* Prev(Elem* e);
57	// Next links point towards back of the queue.
58	Elem* Next(Elem* e);
59
60	uptr Size() const;
61	bool Empty() const;
62	bool Queued(Elem* e) const;
63
64	private:
65	INode node_;
66	uptr size_ = 0;
67
68	void Push(Elem* e, INode* after);
69	static INode* ToNode(Elem* e);
70	static Elem* ToElem(INode* n);
71
72	IList(const IList&) = delete;
73	void operator=(const IList&) = delete;
74	};
75
76	template <typename Base, INode Base::*Node, typename Elem>
77	IList<Base, Node, Elem>::IList() {
78	node_.next_ = node_.prev_ = &node_;
79	}
80
81	template <typename Base, INode Base::*Node, typename Elem>
82	void IList<Base, Node, Elem>::PushFront(Elem* e) {
83	Push(e, &node_);
84	}
85
86	template <typename Base, INode Base::*Node, typename Elem>
87	void IList<Base, Node, Elem>::PushBack(Elem* e) {
88	Push(e, node_.prev_);
89	}
90
91	template <typename Base, INode Base::*Node, typename Elem>
92	void IList<Base, Node, Elem>::Push(Elem* e, INode* after) {
93	INode* n = ToNode(e);
94	DCHECK_EQ(n->next_, nullptr);
95	DCHECK_EQ(n->prev_, nullptr);
96	INode* next = after->next_;
97	n->next_ = next;
98	n->prev_ = after;
99	next->prev_ = n;
100	after->next_ = n;
101	size_++;
102	}
103
104	template <typename Base, INode Base::*Node, typename Elem>
105	void IList<Base, Node, Elem>::Remove(Elem* e) {
106	INode* n = ToNode(e);
107	INode* next = n->next_;
108	INode* prev = n->prev_;
109	DCHECK(next);
110	DCHECK(prev);
111	DCHECK(size_);
112	next->prev_ = prev;
113	prev->next_ = next;
114	n->prev_ = n->next_ = nullptr;
115	size_--;
116	}
117
118	template <typename Base, INode Base::*Node, typename Elem>
119	Elem* IList<Base, Node, Elem>::PopFront() {
120	Elem* e = Front();
121	if (e)
122	Remove(e);
123	return e;
124	}
125
126	template <typename Base, INode Base::*Node, typename Elem>
127	Elem* IList<Base, Node, Elem>::PopBack() {
128	Elem* e = Back();
129	if (e)
130	Remove(e);
131	return e;
132	}
133
134	template <typename Base, INode Base::*Node, typename Elem>
135	Elem* IList<Base, Node, Elem>::Front() {
136	return size_ ? ToElem(node_.next_) : nullptr;
137	}
138
139	template <typename Base, INode Base::*Node, typename Elem>
140	Elem* IList<Base, Node, Elem>::Back() {
141	return size_ ? ToElem(node_.prev_) : nullptr;
142	}
143
144	template <typename Base, INode Base::*Node, typename Elem>
145	Elem* IList<Base, Node, Elem>::Prev(Elem* e) {
146	INode* n = ToNode(e);
147	DCHECK(n->prev_);
148	return n->prev_ != &node_ ? ToElem(n->prev_) : nullptr;
149	}
150
151	template <typename Base, INode Base::*Node, typename Elem>
152	Elem* IList<Base, Node, Elem>::Next(Elem* e) {
153	INode* n = ToNode(e);
154	DCHECK(n->next_);
155	return n->next_ != &node_ ? ToElem(n->next_) : nullptr;
156	}
157
158	template <typename Base, INode Base::*Node, typename Elem>
159	uptr IList<Base, Node, Elem>::Size() const {
160	return size_;
161	}
162
163	template <typename Base, INode Base::*Node, typename Elem>
164	bool IList<Base, Node, Elem>::Empty() const {
165	return size_ == 0;
166	}
167
168	template <typename Base, INode Base::*Node, typename Elem>
169	bool IList<Base, Node, Elem>::Queued(Elem* e) const {
170	INode* n = ToNode(e);
171	DCHECK_EQ(!n->next_, !n->prev_);
172	return n->next_;
173	}
174
175	template <typename Base, INode Base::*Node, typename Elem>
176	INode* IList<Base, Node, Elem>::ToNode(Elem* e) {
177	return &(e->*Node);
178	}
179
180	template <typename Base, INode Base::*Node, typename Elem>
181	Elem* IList<Base, Node, Elem>::ToElem(INode* n) {
182	return static_cast<Elem>(reinterpret_cast<Base>(
183	reinterpret_cast<uptr>(n) -
184	reinterpret_cast<uptr>(&(reinterpret_cast<Elem>(0)->Node))));
185	}
186
187	} // namespace __tsan
188
189	#endif

lib/tsan/tsan_interceptors.h+83-27

...	@@ -10,44 +10,71 @@ class ScopedInterceptor {	...	@@ -10,44 +10,71 @@ class ScopedInterceptor {
10	public:	10	public:
11	ScopedInterceptor(ThreadState thr, const char fname, uptr pc);	11	ScopedInterceptor(ThreadState thr, const char fname, uptr pc);
12	~ScopedInterceptor();	12	~ScopedInterceptor();
13	void DisableIgnores();	13	void DisableIgnores() {
14	void EnableIgnores();	14	if (UNLIKELY(ignoring_))
		15	DisableIgnoresImpl();
		16	}
		17	void EnableIgnores() {
		18	if (UNLIKELY(ignoring_))
		19	EnableIgnoresImpl();
		20	}
		21
15	private:	22	private:
16	ThreadState *const thr_;	23	ThreadState *const thr_;
17	const uptr pc_;	24	bool in_ignored_lib_ = false;
18	bool in_ignored_lib_;	25	bool in_blocking_func_ = false;
19	bool ignoring_;	26	bool ignoring_ = false;
		27
		28	void DisableIgnoresImpl();
		29	void EnableIgnoresImpl();
		30	};
		31
		32	struct TsanInterceptorContext {
		33	ThreadState *thr;
		34	const uptr pc;
20	};	35	};
21		36
22	LibIgnore *libignore();	37	LibIgnore *libignore();
23		38
24	#if !SANITIZER_GO	39	#if !SANITIZER_GO
25	inline bool in_symbolizer() {	40	inline bool in_symbolizer() {
26	cur_thread_init();	41	return UNLIKELY(cur_thread_init()->in_symbolizer);
27	return UNLIKELY(cur_thread()->in_symbolizer);
28	}	42	}
29	#endif	43	#endif
30		44
		45	inline bool MustIgnoreInterceptor(ThreadState *thr) {
		46	return !thr->is_inited \|\| thr->ignore_interceptors \|\| thr->in_ignored_lib;
		47	}
		48
31	} // namespace __tsan	49	} // namespace __tsan
32		50
33	#define SCOPED_INTERCEPTOR_RAW(func, ...) \	51	#define SCOPED_INTERCEPTOR_RAW(func, ...) \
34	cur_thread_init(); \	52	ThreadState *thr = cur_thread_init(); \
35	ThreadState *thr = cur_thread(); \	53	ScopedInterceptor si(thr, #func, GET_CALLER_PC()); \
36	const uptr caller_pc = GET_CALLER_PC(); \	54	UNUSED const uptr pc = GET_CURRENT_PC();
37	ScopedInterceptor si(thr, #func, caller_pc); \	55
38	const uptr pc = GET_CURRENT_PC(); \	56	#ifdef __powerpc64__
39	(void)pc; \	57	// Debugging of crashes on powerpc after commit:
40	/**/	58	// c80604f7a3 ("tsan: remove real func check from interceptors")
41		59	// Somehow replacing if with DCHECK leads to strange failures in:
42	#define SCOPED_TSAN_INTERCEPTOR(func, ...) \	60	// SanitizerCommon-tsan-powerpc64le-Linux :: Linux/ptrace.cpp
43	SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \	61	// https://lab.llvm.org/buildbot/#/builders/105
44	if (REAL(func) == 0) { \	62	// https://lab.llvm.org/buildbot/#/builders/121
		63	// https://lab.llvm.org/buildbot/#/builders/57
		64	# define CHECK_REAL_FUNC(func) \
		65	if (REAL(func) == 0) { \
45	Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \	66	Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \
46	Die(); \	67	Die(); \
47	} \	68	}
48	if (!thr->is_inited \|\| thr->ignore_interceptors \|\| thr->in_ignored_lib) \	69	#else
49	return REAL(func)(__VA_ARGS__); \	70	# define CHECK_REAL_FUNC(func) DCHECK(REAL(func))
50	/**/	71	#endif
		72
		73	#define SCOPED_TSAN_INTERCEPTOR(func, ...) \
		74	SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
		75	CHECK_REAL_FUNC(func); \
		76	if (MustIgnoreInterceptor(thr)) \
		77	return REAL(func)(__VA_ARGS__);
51		78
52	#define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() \	79	#define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() \
53	si.DisableIgnores();	80	si.DisableIgnores();
...	@@ -57,20 +84,49 @@ inline bool in_symbolizer() {	...	@@ -57,20 +84,49 @@ inline bool in_symbolizer() {
57		84
58	#define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)	85	#define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)
59		86
		87	#if SANITIZER_FREEBSD
		88	# define TSAN_INTERCEPTOR_FREEBSD_ALIAS(ret, func, ...) \
		89	TSAN_INTERCEPTOR(ret, _pthread_##func, __VA_ARGS__) \
		90	ALIAS(WRAP(pthread_##func));
		91	#else
		92	# define TSAN_INTERCEPTOR_FREEBSD_ALIAS(ret, func, ...)
		93	#endif
		94
60	#if SANITIZER_NETBSD	95	#if SANITIZER_NETBSD
61	# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) \	96	# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) \
62	TSAN_INTERCEPTOR(ret, __libc_##func, __VA_ARGS__) \	97	TSAN_INTERCEPTOR(ret, __libc_##func, __VA_ARGS__) \
63	ALIAS(WRAPPER_NAME(pthread_##func));	98	ALIAS(WRAP(pthread_##func));
64	# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) \	99	# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) \
65	TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \	100	TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \
66	ALIAS(WRAPPER_NAME(pthread_##func));	101	ALIAS(WRAP(pthread_##func));
67	# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...) \	102	# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...) \
68	TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \	103	TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \
69	ALIAS(WRAPPER_NAME(pthread_##func2));	104	ALIAS(WRAP(pthread_##func2));
70	#else	105	#else
71	# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...)	106	# define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...)
72	# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...)	107	# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...)
73	# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...)	108	# define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...)
74	#endif	109	#endif
75		110
		111	#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
		112
		113	#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \
		114	(!cur_thread_init()->is_inited)
		115
		116	#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
		117	MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \
		118	((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
		119	true)
		120
		121	#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
		122	MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \
		123	((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
		124	false)
		125
		126	#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
		127	SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \
		128	TsanInterceptorContext _ctx = {thr, pc}; \
		129	ctx = (void *)&_ctx; \
		130	(void)ctx;
		131
76	#endif // TSAN_INTERCEPTORS_H	132	#endif // TSAN_INTERCEPTORS_H

lib/tsan/tsan_interceptors_mac.cpp+6-5

...	@@ -12,12 +12,13 @@	...	@@ -12,12 +12,13 @@
12	//===----------------------------------------------------------------------===//	12	//===----------------------------------------------------------------------===//
13		13
14	#include "sanitizer_common/sanitizer_platform.h"	14	#include "sanitizer_common/sanitizer_platform.h"
15	#if SANITIZER_MAC	15	#if SANITIZER_APPLE
16		16
17	#include "interception/interception.h"	17	#include "interception/interception.h"
18	#include "tsan_interceptors.h"	18	#include "tsan_interceptors.h"
19	#include "tsan_interface.h"	19	#include "tsan_interface.h"
20	#include "tsan_interface_ann.h"	20	#include "tsan_interface_ann.h"
		21	#include "tsan_spinlock_defs_mac.h"
21	#include "sanitizer_common/sanitizer_addrhashmap.h"	22	#include "sanitizer_common/sanitizer_addrhashmap.h"
22		23
23	#include <errno.h>	24	#include <errno.h>
...	@@ -365,7 +366,7 @@ static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) {	...	@@ -365,7 +366,7 @@ static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) {
365	if (h.created()) {	366	if (h.created()) {
366	ThreadIgnoreBegin(thr, pc);	367	ThreadIgnoreBegin(thr, pc);
367	h = (uptr) user_alloc(thr, pc, /size=*/1);	368	h = (uptr) user_alloc(thr, pc, /size=*/1);
368	ThreadIgnoreEnd(thr, pc);	369	ThreadIgnoreEnd(thr);
369	}	370	}
370	return *h;	371	return *h;
371	}	372	}
...	@@ -405,8 +406,8 @@ TSAN_INTERCEPTOR(int, swapcontext, ucontext_t oucp, const ucontext_t ucp) {	...	@@ -405,8 +406,8 @@ TSAN_INTERCEPTOR(int, swapcontext, ucontext_t oucp, const ucontext_t ucp) {
405	{	406	{
406	SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp);	407	SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp);
407	}	408	}
408	// Bacause of swapcontext() semantics we have no option but to copy its	409	// Because of swapcontext() semantics we have no option but to copy its
409	// impementation here	410	// implementation here
410	if (!oucp \|\| !ucp) {	411	if (!oucp \|\| !ucp) {
411	errno = EINVAL;	412	errno = EINVAL;
412	return -1;	413	return -1;
...	@@ -518,4 +519,4 @@ STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag,	...	@@ -518,4 +519,4 @@ STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag,
518		519
519	} // namespace __tsan	520	} // namespace __tsan
520		521
521	#endif // SANITIZER_MAC	522	#endif // SANITIZER_APPLE

lib/tsan/tsan_interceptors_memintrinsics.cpp created+43

...	@@ -0,0 +1,43 @@
1	//===-- tsan_interceptors_posix.cpp ---------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS
14
15	#include "tsan_interceptors.h"
16	#include "tsan_interface.h"
17
18	using namespace __tsan;
19
20	#include "sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc"
21
22	extern "C" {
23
24	void __tsan_memcpy(void dst, const void *src, uptr size) {
25	void *ctx;
26	#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE
27	COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
28	#else
29	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
30	#endif
31	}
32
33	void __tsan_memset(void dst, int c, uptr size) {
34	void *ctx;
35	COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, c, size);
36	}
37
38	void __tsan_memmove(void dst, const void *src, uptr size) {
39	void *ctx;
40	COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
41	}
42
43	} // extern "C"

lib/tsan/tsan_interceptors_posix.cpp+616-440

...	@@ -35,7 +35,10 @@	...	@@ -35,7 +35,10 @@
35		35
36	using namespace __tsan;	36	using namespace __tsan;
37		37
38	#if SANITIZER_FREEBSD \|\| SANITIZER_MAC	38	DECLARE_REAL(void , memcpy, void to, const void *from, SIZE_T size)
		39	DECLARE_REAL(void , memset, void block, int c, SIZE_T size)
		40
		41	#if SANITIZER_FREEBSD \|\| SANITIZER_APPLE
39	#define stdout __stdoutp	42	#define stdout __stdoutp
40	#define stderr __stderrp	43	#define stderr __stderrp
41	#endif	44	#endif
...	@@ -76,6 +79,8 @@ struct ucontext_t {	...	@@ -76,6 +79,8 @@ struct ucontext_t {
76	#define PTHREAD_ABI_BASE "GLIBC_2.3.2"	79	#define PTHREAD_ABI_BASE "GLIBC_2.3.2"
77	#elif defined(__aarch64__) \|\| SANITIZER_PPC64V2	80	#elif defined(__aarch64__) \|\| SANITIZER_PPC64V2
78	#define PTHREAD_ABI_BASE "GLIBC_2.17"	81	#define PTHREAD_ABI_BASE "GLIBC_2.17"
		82	#elif SANITIZER_LOONGARCH64
		83	#define PTHREAD_ABI_BASE "GLIBC_2.36"
79	#endif	84	#endif
80		85
81	extern "C" int pthread_attr_init(void *attr);	86	extern "C" int pthread_attr_init(void *attr);
...	@@ -90,28 +95,26 @@ DECLARE_REAL(int, pthread_mutexattr_gettype, void , void )	...	@@ -90,28 +95,26 @@ DECLARE_REAL(int, pthread_mutexattr_gettype, void , void )
90	DECLARE_REAL(int, fflush, __sanitizer_FILE *fp)	95	DECLARE_REAL(int, fflush, __sanitizer_FILE *fp)
91	DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size)	96	DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size)
92	DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)	97	DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
		98	extern "C" int pthread_equal(void t1, void t2);
93	extern "C" void *pthread_self();	99	extern "C" void *pthread_self();
94	extern "C" void _exit(int status);	100	extern "C" void _exit(int status);
95	#if !SANITIZER_NETBSD	101	#if !SANITIZER_NETBSD
96	extern "C" int fileno_unlocked(void *stream);	102	extern "C" int fileno_unlocked(void *stream);
97	extern "C" int dirfd(void *dirp);	103	extern "C" int dirfd(void *dirp);
98	#endif	104	#endif
99	#if SANITIZER_GLIBC
100	extern "C" int mallopt(int param, int value);
101	#endif
102	#if SANITIZER_NETBSD	105	#if SANITIZER_NETBSD
103	extern __sanitizer_FILE __sF[];	106	extern __sanitizer_FILE __sF[];
104	#else	107	#else
105	extern __sanitizer_FILE stdout, stderr;	108	extern __sanitizer_FILE stdout, stderr;
106	#endif	109	#endif
107	#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD	110	#if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD
108	const int PTHREAD_MUTEX_RECURSIVE = 1;	111	const int PTHREAD_MUTEX_RECURSIVE = 1;
109	const int PTHREAD_MUTEX_RECURSIVE_NP = 1;	112	const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
110	#else	113	#else
111	const int PTHREAD_MUTEX_RECURSIVE = 2;	114	const int PTHREAD_MUTEX_RECURSIVE = 2;
112	const int PTHREAD_MUTEX_RECURSIVE_NP = 2;	115	const int PTHREAD_MUTEX_RECURSIVE_NP = 2;
113	#endif	116	#endif
114	#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD	117	#if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD
115	const int EPOLL_CTL_ADD = 1;	118	const int EPOLL_CTL_ADD = 1;
116	#endif	119	#endif
117	const int SIGILL = 4;	120	const int SIGILL = 4;
...	@@ -121,17 +124,20 @@ const int SIGFPE = 8;	...	@@ -121,17 +124,20 @@ const int SIGFPE = 8;
121	const int SIGSEGV = 11;	124	const int SIGSEGV = 11;
122	const int SIGPIPE = 13;	125	const int SIGPIPE = 13;
123	const int SIGTERM = 15;	126	const int SIGTERM = 15;
124	#if defined(__mips__) \|\| SANITIZER_FREEBSD \|\| SANITIZER_MAC \|\| SANITIZER_NETBSD	127	#if defined(__mips__) \|\| SANITIZER_FREEBSD \|\| SANITIZER_APPLE \|\| SANITIZER_NETBSD
125	const int SIGBUS = 10;	128	const int SIGBUS = 10;
126	const int SIGSYS = 12;	129	const int SIGSYS = 12;
127	#else	130	#else
128	const int SIGBUS = 7;	131	const int SIGBUS = 7;
129	const int SIGSYS = 31;	132	const int SIGSYS = 31;
130	#endif	133	#endif
		134	#if SANITIZER_HAS_SIGINFO
		135	const int SI_TIMER = -2;
		136	#endif
131	void const MAP_FAILED = (void)-1;	137	void const MAP_FAILED = (void)-1;
132	#if SANITIZER_NETBSD	138	#if SANITIZER_NETBSD
133	const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567;	139	const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567;
134	#elif !SANITIZER_MAC	140	#elif !SANITIZER_APPLE
135	const int PTHREAD_BARRIER_SERIAL_THREAD = -1;	141	const int PTHREAD_BARRIER_SERIAL_THREAD = -1;
136	#endif	142	#endif
137	const int MAP_FIXED = 0x10;	143	const int MAP_FIXED = 0x10;
...	@@ -144,7 +150,7 @@ typedef __sanitizer::u16 mode_t;	...	@@ -144,7 +150,7 @@ typedef __sanitizer::u16 mode_t;
144	# define F_TLOCK 2 /* Test and lock a region for exclusive use. */	150	# define F_TLOCK 2 /* Test and lock a region for exclusive use. */
145	# define F_TEST 3 /* Test a region for other processes locks. */	151	# define F_TEST 3 /* Test a region for other processes locks. */
146		152
147	#if SANITIZER_FREEBSD \|\| SANITIZER_MAC \|\| SANITIZER_NETBSD	153	#if SANITIZER_FREEBSD \|\| SANITIZER_APPLE \|\| SANITIZER_NETBSD
148	const int SA_SIGINFO = 0x40;	154	const int SA_SIGINFO = 0x40;
149	const int SIG_SETMASK = 3;	155	const int SIG_SETMASK = 3;
150	#elif defined(__mips__)	156	#elif defined(__mips__)
...	@@ -155,32 +161,41 @@ const int SA_SIGINFO = 4;	...	@@ -155,32 +161,41 @@ const int SA_SIGINFO = 4;
155	const int SIG_SETMASK = 2;	161	const int SIG_SETMASK = 2;
156	#endif	162	#endif
157		163
158	#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \
159	(cur_thread_init(), !cur_thread()->is_inited)
160
161	namespace __tsan {	164	namespace __tsan {
162	struct SignalDesc {	165	struct SignalDesc {
163	bool armed;	166	bool armed;
164	bool sigaction;
165	__sanitizer_siginfo siginfo;	167	__sanitizer_siginfo siginfo;
166	ucontext_t ctx;	168	ucontext_t ctx;
167	};	169	};
168		170
169	struct ThreadSignalContext {	171	struct ThreadSignalContext {
170	int int_signal_send;	172	int int_signal_send;
171	atomic_uintptr_t in_blocking_func;
172	atomic_uintptr_t have_pending_signals;
173	SignalDesc pending_signals[kSigCount];	173	SignalDesc pending_signals[kSigCount];
174	// emptyset and oldset are too big for stack.	174	// emptyset and oldset are too big for stack.
175	__sanitizer_sigset_t emptyset;	175	__sanitizer_sigset_t emptyset;
176	__sanitizer_sigset_t oldset;	176	__sanitizer_sigset_t oldset;
177	};	177	};
178		178
		179	void EnterBlockingFunc(ThreadState *thr) {
		180	for (;;) {
		181	// The order is important to not delay a signal infinitely if it's
		182	// delivered right before we set in_blocking_func. Note: we can't call
		183	// ProcessPendingSignals when in_blocking_func is set, or we can handle
		184	// a signal synchronously when we are already handling a signal.
		185	atomic_store(&thr->in_blocking_func, 1, memory_order_relaxed);
		186	if (atomic_load(&thr->pending_signals, memory_order_relaxed) == 0)
		187	break;
		188	atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
		189	ProcessPendingSignals(thr);
		190	}
		191	}
		192
179	// The sole reason tsan wraps atexit callbacks is to establish synchronization	193	// The sole reason tsan wraps atexit callbacks is to establish synchronization
180	// between callback setup and callback execution.	194	// between callback setup and callback execution.
181	struct AtExitCtx {	195	struct AtExitCtx {
182	void (*f)();	196	void (*f)();
183	void *arg;	197	void *arg;
		198	uptr pc;
184	};	199	};
185		200
186	// InterceptorContext holds all global data required for interceptors.	201	// InterceptorContext holds all global data required for interceptors.
...	@@ -192,7 +207,7 @@ struct InterceptorContext {	...	@@ -192,7 +207,7 @@ struct InterceptorContext {
192	// in a single cache line if possible (it's accessed in every interceptor).	207	// in a single cache line if possible (it's accessed in every interceptor).
193	ALIGNED(64) LibIgnore libignore;	208	ALIGNED(64) LibIgnore libignore;
194	__sanitizer_sigaction sigactions[kSigCount];	209	__sanitizer_sigaction sigactions[kSigCount];
195	#if !SANITIZER_MAC && !SANITIZER_NETBSD	210	#if !SANITIZER_APPLE && !SANITIZER_NETBSD
196	unsigned finalize_key;	211	unsigned finalize_key;
197	#endif	212	#endif
198		213
...	@@ -237,19 +252,37 @@ SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {}	...	@@ -237,19 +252,37 @@ SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {}
237	} // namespace __tsan	252	} // namespace __tsan
238		253
239	static ThreadSignalContext SigCtx(ThreadState thr) {	254	static ThreadSignalContext SigCtx(ThreadState thr) {
240	ThreadSignalContext ctx = (ThreadSignalContext)thr->signal_ctx;	255	// This function may be called reentrantly if it is interrupted by a signal
		256	// handler. Use CAS to handle the race.
		257	uptr ctx = atomic_load(&thr->signal_ctx, memory_order_relaxed);
241	if (ctx == 0 && !thr->is_dead) {	258	if (ctx == 0 && !thr->is_dead) {
242	ctx = (ThreadSignalContext)MmapOrDie(sizeof(ctx), "ThreadSignalContext");	259	uptr pctx =
243	MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx));	260	(uptr)MmapOrDie(sizeof(ThreadSignalContext), "ThreadSignalContext");
244	thr->signal_ctx = ctx;	261	MemoryResetRange(thr, (uptr)&SigCtx, pctx, sizeof(ThreadSignalContext));
		262	if (atomic_compare_exchange_strong(&thr->signal_ctx, &ctx, pctx,
		263	memory_order_relaxed)) {
		264	ctx = pctx;
		265	} else {
		266	UnmapOrDie((ThreadSignalContext *)pctx, sizeof(ThreadSignalContext));
		267	}
245	}	268	}
246	return ctx;	269	return (ThreadSignalContext *)ctx;
247	}	270	}
248		271
249	ScopedInterceptor::ScopedInterceptor(ThreadState thr, const char fname,	272	ScopedInterceptor::ScopedInterceptor(ThreadState thr, const char fname,
250	uptr pc)	273	uptr pc)
251	: thr_(thr), pc_(pc), in_ignored_lib_(false), ignoring_(false) {	274	: thr_(thr) {
252	Initialize(thr);	275	LazyInitialize(thr);
		276	if (UNLIKELY(atomic_load(&thr->in_blocking_func, memory_order_relaxed))) {
		277	// pthread_join is marked as blocking, but it's also known to call other
		278	// intercepted functions (mmap, free). If we don't reset in_blocking_func
		279	// we can get deadlocks and memory corruptions if we deliver a synchronous
		280	// signal inside of an mmap/free interceptor.
		281	// So reset it and restore it back in the destructor.
		282	// See https://github.com/google/sanitizers/issues/1540
		283	atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
		284	in_blocking_func_ = true;
		285	}
253	if (!thr_->is_inited) return;	286	if (!thr_->is_inited) return;
254	if (!thr_->ignore_interceptors) FuncEntry(thr, pc);	287	if (!thr_->ignore_interceptors) FuncEntry(thr, pc);
255	DPrintf("#%d: intercept %s()\n", thr_->tid, fname);	288	DPrintf("#%d: intercept %s()\n", thr_->tid, fname);
...	@@ -262,6 +295,8 @@ ScopedInterceptor::ScopedInterceptor(ThreadState thr, const char fname,	...	@@ -262,6 +295,8 @@ ScopedInterceptor::ScopedInterceptor(ThreadState thr, const char fname,
262	ScopedInterceptor::~ScopedInterceptor() {	295	ScopedInterceptor::~ScopedInterceptor() {
263	if (!thr_->is_inited) return;	296	if (!thr_->is_inited) return;
264	DisableIgnores();	297	DisableIgnores();
		298	if (UNLIKELY(in_blocking_func_))
		299	EnterBlockingFunc(thr_);
265	if (!thr_->ignore_interceptors) {	300	if (!thr_->ignore_interceptors) {
266	ProcessPendingSignals(thr_);	301	ProcessPendingSignals(thr_);
267	FuncExit(thr_);	302	FuncExit(thr_);
...	@@ -269,43 +304,48 @@ ScopedInterceptor::~ScopedInterceptor() {	...	@@ -269,43 +304,48 @@ ScopedInterceptor::~ScopedInterceptor() {
269	}	304	}
270	}	305	}
271		306
272	void ScopedInterceptor::EnableIgnores() {	307	NOINLINE
273	if (ignoring_) {	308	void ScopedInterceptor::EnableIgnoresImpl() {
274	ThreadIgnoreBegin(thr_, pc_, /save_stack=/false);	309	ThreadIgnoreBegin(thr_, 0);
275	if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports++;	310	if (flags()->ignore_noninstrumented_modules)
276	if (in_ignored_lib_) {	311	thr_->suppress_reports++;
277	DCHECK(!thr_->in_ignored_lib);	312	if (in_ignored_lib_) {
278	thr_->in_ignored_lib = true;	313	DCHECK(!thr_->in_ignored_lib);
279	}	314	thr_->in_ignored_lib = true;
280	}	315	}
281	}	316	}
282		317
283	void ScopedInterceptor::DisableIgnores() {	318	NOINLINE
284	if (ignoring_) {	319	void ScopedInterceptor::DisableIgnoresImpl() {
285	ThreadIgnoreEnd(thr_, pc_);	320	ThreadIgnoreEnd(thr_);
286	if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports--;	321	if (flags()->ignore_noninstrumented_modules)
287	if (in_ignored_lib_) {	322	thr_->suppress_reports--;
288	DCHECK(thr_->in_ignored_lib);	323	if (in_ignored_lib_) {
289	thr_->in_ignored_lib = false;	324	DCHECK(thr_->in_ignored_lib);
290	}	325	thr_->in_ignored_lib = false;
291	}	326	}
292	}	327	}
293		328
294	#define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)	329	#define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
		330	#if SANITIZER_FREEBSD \|\| SANITIZER_NETBSD
		331	# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
		332	#else
		333	# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
		334	#endif
295	#if SANITIZER_FREEBSD	335	#if SANITIZER_FREEBSD
296	# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)	336	# define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func) \
297	# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)	337	INTERCEPT_FUNCTION(_pthread_##func)
298	# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
299	#elif SANITIZER_NETBSD
300	# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
301	# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \
302	INTERCEPT_FUNCTION(__libc_##func)
303	# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \
304	INTERCEPT_FUNCTION(__libc_thr_##func)
305	#else	338	#else
306	# define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)	339	# define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func)
307	# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)	340	#endif
308	# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)	341	#if SANITIZER_NETBSD
		342	# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \
		343	INTERCEPT_FUNCTION(__libc_##func)
		344	# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \
		345	INTERCEPT_FUNCTION(__libc_thr_##func)
		346	#else
		347	# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
		348	# define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
309	#endif	349	#endif
310		350
311	#define READ_STRING_OF_LEN(thr, pc, s, len, n) \	351	#define READ_STRING_OF_LEN(thr, pc, s, len, n) \
...	@@ -319,15 +359,8 @@ void ScopedInterceptor::DisableIgnores() {	...	@@ -319,15 +359,8 @@ void ScopedInterceptor::DisableIgnores() {
319		359
320	struct BlockingCall {	360	struct BlockingCall {
321	explicit BlockingCall(ThreadState *thr)	361	explicit BlockingCall(ThreadState *thr)
322	: thr(thr)	362	: thr(thr) {
323	, ctx(SigCtx(thr)) {	363	EnterBlockingFunc(thr);
324	for (;;) {
325	atomic_store(&ctx->in_blocking_func, 1, memory_order_relaxed);
326	if (atomic_load(&ctx->have_pending_signals, memory_order_relaxed) == 0)
327	break;
328	atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
329	ProcessPendingSignals(thr);
330	}
331	// When we are in a "blocking call", we process signals asynchronously	364	// When we are in a "blocking call", we process signals asynchronously
332	// (right when they arrive). In this context we do not expect to be	365	// (right when they arrive). In this context we do not expect to be
333	// executing any user/runtime code. The known interceptor sequence when	366	// executing any user/runtime code. The known interceptor sequence when
...	@@ -338,11 +371,10 @@ struct BlockingCall {	...	@@ -338,11 +371,10 @@ struct BlockingCall {
338		371
339	~BlockingCall() {	372	~BlockingCall() {
340	thr->ignore_interceptors--;	373	thr->ignore_interceptors--;
341	atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);	374	atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
342	}	375	}
343		376
344	ThreadState *thr;	377	ThreadState *thr;
345	ThreadSignalContext *ctx;
346	};	378	};
347		379
348	TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) {	380	TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) {
...	@@ -371,7 +403,10 @@ TSAN_INTERCEPTOR(int, pause, int fake) {	...	@@ -371,7 +403,10 @@ TSAN_INTERCEPTOR(int, pause, int fake) {
371	return BLOCK_REAL(pause)(fake);	403	return BLOCK_REAL(pause)(fake);
372	}	404	}
373		405
374	static void at_exit_wrapper() {	406	// Note: we specifically call the function in such strange way
		407	// with "installed_at" because in reports it will appear between
		408	// callback frames and the frame that installed the callback.
		409	static void at_exit_callback_installed_at() {
375	AtExitCtx *ctx;	410	AtExitCtx *ctx;
376	{	411	{
377	// Ensure thread-safety.	412	// Ensure thread-safety.
...	@@ -383,16 +418,22 @@ static void at_exit_wrapper() {	...	@@ -383,16 +418,22 @@ static void at_exit_wrapper() {
383	interceptor_ctx()->AtExitStack.PopBack();	418	interceptor_ctx()->AtExitStack.PopBack();
384	}	419	}
385		420
386	Acquire(cur_thread(), (uptr)0, (uptr)ctx);	421	ThreadState *thr = cur_thread();
		422	Acquire(thr, ctx->pc, (uptr)ctx);
		423	FuncEntry(thr, ctx->pc);
387	((void(*)())ctx->f)();	424	((void(*)())ctx->f)();
388	InternalFree(ctx);	425	FuncExit(thr);
		426	Free(ctx);
389	}	427	}
390		428
391	static void cxa_at_exit_wrapper(void *arg) {	429	static void cxa_at_exit_callback_installed_at(void *arg) {
392	Acquire(cur_thread(), 0, (uptr)arg);	430	ThreadState *thr = cur_thread();
393	AtExitCtx ctx = (AtExitCtx)arg;	431	AtExitCtx ctx = (AtExitCtx)arg;
		432	Acquire(thr, ctx->pc, (uptr)arg);
		433	FuncEntry(thr, ctx->pc);
394	((void()(void arg))ctx->f)(ctx->arg);	434	((void()(void arg))ctx->f)(ctx->arg);
395	InternalFree(ctx);	435	FuncExit(thr);
		436	Free(ctx);
396	}	437	}
397		438
398	static int setup_at_exit_wrapper(ThreadState thr, uptr pc, void(f)(),	439	static int setup_at_exit_wrapper(ThreadState thr, uptr pc, void(f)(),
...	@@ -405,7 +446,7 @@ TSAN_INTERCEPTOR(int, atexit, void (*f)()) {	...	@@ -405,7 +446,7 @@ TSAN_INTERCEPTOR(int, atexit, void (*f)()) {
405	// We want to setup the atexit callback even if we are in ignored lib	446	// We want to setup the atexit callback even if we are in ignored lib
406	// or after fork.	447	// or after fork.
407	SCOPED_INTERCEPTOR_RAW(atexit, f);	448	SCOPED_INTERCEPTOR_RAW(atexit, f);
408	return setup_at_exit_wrapper(thr, pc, (void(*)())f, 0, 0);	449	return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, 0, 0);
409	}	450	}
410	#endif	451	#endif
411		452
...	@@ -413,14 +454,15 @@ TSAN_INTERCEPTOR(int, __cxa_atexit, void (f)(void a), void arg, void dso) {	...	@@ -413,14 +454,15 @@ TSAN_INTERCEPTOR(int, __cxa_atexit, void (f)(void a), void arg, void dso) {
413	if (in_symbolizer())	454	if (in_symbolizer())
414	return 0;	455	return 0;
415	SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso);	456	SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso);
416	return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso);	457	return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, arg, dso);
417	}	458	}
418		459
419	static int setup_at_exit_wrapper(ThreadState thr, uptr pc, void(f)(),	460	static int setup_at_exit_wrapper(ThreadState thr, uptr pc, void(f)(),
420	void arg, void dso) {	461	void arg, void dso) {
421	AtExitCtx ctx = (AtExitCtx)InternalAlloc(sizeof(AtExitCtx));	462	auto *ctx = New<AtExitCtx>();
422	ctx->f = f;	463	ctx->f = f;
423	ctx->arg = arg;	464	ctx->arg = arg;
		465	ctx->pc = pc;
424	Release(thr, pc, (uptr)ctx);	466	Release(thr, pc, (uptr)ctx);
425	// Memory allocation in __cxa_atexit will race with free during exit,	467	// Memory allocation in __cxa_atexit will race with free during exit,
426	// because we do not see synchronization around atexit callback list.	468	// because we do not see synchronization around atexit callback list.
...	@@ -436,41 +478,44 @@ static int setup_at_exit_wrapper(ThreadState thr, uptr pc, void(f)(),	...	@@ -436,41 +478,44 @@ static int setup_at_exit_wrapper(ThreadState thr, uptr pc, void(f)(),
436	// due to atexit_mu held on exit from the calloc interceptor.	478	// due to atexit_mu held on exit from the calloc interceptor.
437	ScopedIgnoreInterceptors ignore;	479	ScopedIgnoreInterceptors ignore;
438		480
439	res = REAL(__cxa_atexit)((void ()(void a))at_exit_wrapper, 0, 0);	481	res = REAL(__cxa_atexit)((void ()(void a))at_exit_callback_installed_at,
		482	0, 0);
440	// Push AtExitCtx on the top of the stack of callback functions	483	// Push AtExitCtx on the top of the stack of callback functions
441	if (!res) {	484	if (!res) {
442	interceptor_ctx()->AtExitStack.PushBack(ctx);	485	interceptor_ctx()->AtExitStack.PushBack(ctx);
443	}	486	}
444	} else {	487	} else {
445	res = REAL(__cxa_atexit)(cxa_at_exit_wrapper, ctx, dso);	488	res = REAL(__cxa_atexit)(cxa_at_exit_callback_installed_at, ctx, dso);
446	}	489	}
447	ThreadIgnoreEnd(thr, pc);	490	ThreadIgnoreEnd(thr);
448	return res;	491	return res;
449	}	492	}
450		493
451	#if !SANITIZER_MAC && !SANITIZER_NETBSD	494	#if !SANITIZER_APPLE && !SANITIZER_NETBSD
452	static void on_exit_wrapper(int status, void *arg) {	495	static void on_exit_callback_installed_at(int status, void *arg) {
453	ThreadState *thr = cur_thread();	496	ThreadState *thr = cur_thread();
454	uptr pc = 0;
455	Acquire(thr, pc, (uptr)arg);
456	AtExitCtx ctx = (AtExitCtx)arg;	497	AtExitCtx ctx = (AtExitCtx)arg;
		498	Acquire(thr, ctx->pc, (uptr)arg);
		499	FuncEntry(thr, ctx->pc);
457	((void()(int status, void arg))ctx->f)(status, ctx->arg);	500	((void()(int status, void arg))ctx->f)(status, ctx->arg);
458	InternalFree(ctx);	501	FuncExit(thr);
		502	Free(ctx);
459	}	503	}
460		504
461	TSAN_INTERCEPTOR(int, on_exit, void(f)(int, void), void *arg) {	505	TSAN_INTERCEPTOR(int, on_exit, void(f)(int, void), void *arg) {
462	if (in_symbolizer())	506	if (in_symbolizer())
463	return 0;	507	return 0;
464	SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg);	508	SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg);
465	AtExitCtx ctx = (AtExitCtx)InternalAlloc(sizeof(AtExitCtx));	509	auto *ctx = New<AtExitCtx>();
466	ctx->f = (void(*)())f;	510	ctx->f = (void(*)())f;
467	ctx->arg = arg;	511	ctx->arg = arg;
		512	ctx->pc = GET_CALLER_PC();
468	Release(thr, pc, (uptr)ctx);	513	Release(thr, pc, (uptr)ctx);
469	// Memory allocation in __cxa_atexit will race with free during exit,	514	// Memory allocation in __cxa_atexit will race with free during exit,
470	// because we do not see synchronization around atexit callback list.	515	// because we do not see synchronization around atexit callback list.
471	ThreadIgnoreBegin(thr, pc);	516	ThreadIgnoreBegin(thr, pc);
472	int res = REAL(on_exit)(on_exit_wrapper, ctx);	517	int res = REAL(on_exit)(on_exit_callback_installed_at, ctx);
473	ThreadIgnoreEnd(thr, pc);	518	ThreadIgnoreEnd(thr);
474	return res;	519	return res;
475	}	520	}
476	#define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit)	521	#define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit)
...	@@ -502,9 +547,7 @@ static void SetJmp(ThreadState *thr, uptr sp) {	...	@@ -502,9 +547,7 @@ static void SetJmp(ThreadState *thr, uptr sp) {
502	buf->shadow_stack_pos = thr->shadow_stack_pos;	547	buf->shadow_stack_pos = thr->shadow_stack_pos;
503	ThreadSignalContext *sctx = SigCtx(thr);	548	ThreadSignalContext *sctx = SigCtx(thr);
504	buf->int_signal_send = sctx ? sctx->int_signal_send : 0;	549	buf->int_signal_send = sctx ? sctx->int_signal_send : 0;
505	buf->in_blocking_func = sctx ?	550	buf->in_blocking_func = atomic_load(&thr->in_blocking_func, memory_order_relaxed);
506	atomic_load(&sctx->in_blocking_func, memory_order_relaxed) :
507	false;
508	buf->in_signal_handler = atomic_load(&thr->in_signal_handler,	551	buf->in_signal_handler = atomic_load(&thr->in_signal_handler,
509	memory_order_relaxed);	552	memory_order_relaxed);
510	}	553	}
...	@@ -520,11 +563,10 @@ static void LongJmp(ThreadState thr, uptr env) {	...	@@ -520,11 +563,10 @@ static void LongJmp(ThreadState thr, uptr env) {
520	while (thr->shadow_stack_pos > buf->shadow_stack_pos)	563	while (thr->shadow_stack_pos > buf->shadow_stack_pos)
521	FuncExit(thr);	564	FuncExit(thr);
522	ThreadSignalContext *sctx = SigCtx(thr);	565	ThreadSignalContext *sctx = SigCtx(thr);
523	if (sctx) {	566	if (sctx)
524	sctx->int_signal_send = buf->int_signal_send;	567	sctx->int_signal_send = buf->int_signal_send;
525	atomic_store(&sctx->in_blocking_func, buf->in_blocking_func,	568	atomic_store(&thr->in_blocking_func, buf->in_blocking_func,
526	memory_order_relaxed);	569	memory_order_relaxed);
527	}
528	atomic_store(&thr->in_signal_handler, buf->in_signal_handler,	570	atomic_store(&thr->in_signal_handler, buf->in_signal_handler,
529	memory_order_relaxed);	571	memory_order_relaxed);
530	JmpBufGarbageCollect(thr, buf->sp - 1); // do not collect buf->sp	572	JmpBufGarbageCollect(thr, buf->sp - 1); // do not collect buf->sp
...	@@ -536,16 +578,13 @@ static void LongJmp(ThreadState thr, uptr env) {	...	@@ -536,16 +578,13 @@ static void LongJmp(ThreadState thr, uptr env) {
536	}	578	}
537		579
538	// FIXME: put everything below into a common extern "C" block?	580	// FIXME: put everything below into a common extern "C" block?
539	extern "C" void __tsan_setjmp(uptr sp) {	581	extern "C" void __tsan_setjmp(uptr sp) { SetJmp(cur_thread_init(), sp); }
540	cur_thread_init();
541	SetJmp(cur_thread(), sp);
542	}
543		582
544	#if SANITIZER_MAC	583	#if SANITIZER_APPLE
545	TSAN_INTERCEPTOR(int, setjmp, void *env);	584	TSAN_INTERCEPTOR(int, setjmp, void *env);
546	TSAN_INTERCEPTOR(int, _setjmp, void *env);	585	TSAN_INTERCEPTOR(int, _setjmp, void *env);
547	TSAN_INTERCEPTOR(int, sigsetjmp, void *env);	586	TSAN_INTERCEPTOR(int, sigsetjmp, void *env);
548	#else // SANITIZER_MAC	587	#else // SANITIZER_APPLE
549		588
550	#if SANITIZER_NETBSD	589	#if SANITIZER_NETBSD
551	#define setjmp_symname __setjmp14	590	#define setjmp_symname __setjmp14
...	@@ -555,59 +594,28 @@ TSAN_INTERCEPTOR(int, sigsetjmp, void *env);	...	@@ -555,59 +594,28 @@ TSAN_INTERCEPTOR(int, sigsetjmp, void *env);
555	#define sigsetjmp_symname sigsetjmp	594	#define sigsetjmp_symname sigsetjmp
556	#endif	595	#endif
557		596
558	#define TSAN_INTERCEPTOR_SETJMP_(x) __interceptor_ ## x
559	#define TSAN_INTERCEPTOR_SETJMP__(x) TSAN_INTERCEPTOR_SETJMP_(x)
560	#define TSAN_INTERCEPTOR_SETJMP TSAN_INTERCEPTOR_SETJMP__(setjmp_symname)
561	#define TSAN_INTERCEPTOR_SIGSETJMP TSAN_INTERCEPTOR_SETJMP__(sigsetjmp_symname)
562
563	#define TSAN_STRING_SETJMP SANITIZER_STRINGIFY(setjmp_symname)
564	#define TSAN_STRING_SIGSETJMP SANITIZER_STRINGIFY(sigsetjmp_symname)
565
566	// Not called. Merely to satisfy TSAN_INTERCEPT().
567	extern "C" SANITIZER_INTERFACE_ATTRIBUTE
568	int TSAN_INTERCEPTOR_SETJMP(void *env);
569	extern "C" int TSAN_INTERCEPTOR_SETJMP(void *env) {
570	CHECK(0);
571	return 0;
572	}
573
574	// FIXME: any reason to have a separate declaration?
575	extern "C" SANITIZER_INTERFACE_ATTRIBUTE
576	int __interceptor__setjmp(void *env);
577	extern "C" int __interceptor__setjmp(void *env) {
578	CHECK(0);
579	return 0;
580	}
581
582	extern "C" SANITIZER_INTERFACE_ATTRIBUTE
583	int TSAN_INTERCEPTOR_SIGSETJMP(void *env);
584	extern "C" int TSAN_INTERCEPTOR_SIGSETJMP(void *env) {
585	CHECK(0);
586	return 0;
587	}
588
589	#if !SANITIZER_NETBSD
590	extern "C" SANITIZER_INTERFACE_ATTRIBUTE
591	int __interceptor___sigsetjmp(void *env);
592	extern "C" int __interceptor___sigsetjmp(void *env) {
593	CHECK(0);
594	return 0;
595	}
596	#endif
597
598	extern "C" int setjmp_symname(void *env);
599	extern "C" int _setjmp(void *env);
600	extern "C" int sigsetjmp_symname(void *env);
601	#if !SANITIZER_NETBSD
602	extern "C" int __sigsetjmp(void *env);
603	#endif
604	DEFINE_REAL(int, setjmp_symname, void *env)	597	DEFINE_REAL(int, setjmp_symname, void *env)
605	DEFINE_REAL(int, _setjmp, void *env)	598	DEFINE_REAL(int, _setjmp, void *env)
606	DEFINE_REAL(int, sigsetjmp_symname, void *env)	599	DEFINE_REAL(int, sigsetjmp_symname, void *env)
607	#if !SANITIZER_NETBSD	600	#if !SANITIZER_NETBSD
608	DEFINE_REAL(int, __sigsetjmp, void *env)	601	DEFINE_REAL(int, __sigsetjmp, void *env)
609	#endif	602	#endif
610	#endif // SANITIZER_MAC	603
		604	// The real interceptor for setjmp is special, and implemented in pure asm. We
		605	// just need to initialize the REAL functions so that they can be used in asm.
		606	static void InitializeSetjmpInterceptors() {
		607	// We can not use TSAN_INTERCEPT to get setjmp addr, because it does &setjmp and
		608	// setjmp is not present in some versions of libc.
		609	using __interception::InterceptFunction;
		610	InterceptFunction(SANITIZER_STRINGIFY(setjmp_symname), (uptr*)&REAL(setjmp_symname), 0, 0);
		611	InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0);
		612	InterceptFunction(SANITIZER_STRINGIFY(sigsetjmp_symname), (uptr*)&REAL(sigsetjmp_symname), 0,
		613	0);
		614	#if !SANITIZER_NETBSD
		615	InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0);
		616	#endif
		617	}
		618	#endif // SANITIZER_APPLE
611		619
612	#if SANITIZER_NETBSD	620	#if SANITIZER_NETBSD
613	#define longjmp_symname __longjmp14	621	#define longjmp_symname __longjmp14
...	@@ -646,7 +654,7 @@ TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) {	...	@@ -646,7 +654,7 @@ TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) {
646	}	654	}
647	#endif	655	#endif
648		656
649	#if !SANITIZER_MAC	657	#if !SANITIZER_APPLE
650	TSAN_INTERCEPTOR(void*, malloc, uptr size) {	658	TSAN_INTERCEPTOR(void*, malloc, uptr size) {
651	if (in_symbolizer())	659	if (in_symbolizer())
652	return InternalAlloc(size);	660	return InternalAlloc(size);
...	@@ -787,10 +795,11 @@ static void mmap_interceptor(ThreadState thr, uptr pc, Mmap real_mmap,	...	@@ -787,10 +795,11 @@ static void mmap_interceptor(ThreadState thr, uptr pc, Mmap real_mmap,
787	return res;	795	return res;
788	}	796	}
789		797
790	TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) {	798	template <class Munmap>
791	SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz);	799	static int munmap_interceptor(ThreadState *thr, uptr pc, Munmap real_munmap,
		800	void *addr, SIZE_T sz) {
792	UnmapShadow(thr, (uptr)addr, sz);	801	UnmapShadow(thr, (uptr)addr, sz);
793	int res = REAL(munmap)(addr, sz);	802	int res = real_munmap(addr, sz);
794	return res;	803	return res;
795	}	804	}
796		805
...	@@ -804,7 +813,7 @@ TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {	...	@@ -804,7 +813,7 @@ TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
804	#define TSAN_MAYBE_INTERCEPT_MEMALIGN	813	#define TSAN_MAYBE_INTERCEPT_MEMALIGN
805	#endif	814	#endif
806		815
807	#if !SANITIZER_MAC	816	#if !SANITIZER_APPLE
808	TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) {	817	TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) {
809	if (in_symbolizer())	818	if (in_symbolizer())
810	return InternalAlloc(sz, nullptr, align);	819	return InternalAlloc(sz, nullptr, align);
...	@@ -835,7 +844,7 @@ TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {	...	@@ -835,7 +844,7 @@ TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
835	#define TSAN_MAYBE_INTERCEPT_PVALLOC	844	#define TSAN_MAYBE_INTERCEPT_PVALLOC
836	#endif	845	#endif
837		846
838	#if !SANITIZER_MAC	847	#if !SANITIZER_APPLE
839	TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {	848	TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
840	if (in_symbolizer()) {	849	if (in_symbolizer()) {
841	void *p = InternalAlloc(sz, nullptr, align);	850	void *p = InternalAlloc(sz, nullptr, align);
...	@@ -849,6 +858,54 @@ TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {	...	@@ -849,6 +858,54 @@ TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
849	}	858	}
850	#endif	859	#endif
851		860
		861	// Both __cxa_guard_acquire and pthread_once 0-initialize
		862	// the object initially. pthread_once does not have any
		863	// other ABI requirements. __cxa_guard_acquire assumes
		864	// that any non-0 value in the first byte means that
		865	// initialization is completed. Contents of the remaining
		866	// bytes are up to us.
		867	constexpr u32 kGuardInit = 0;
		868	constexpr u32 kGuardDone = 1;
		869	constexpr u32 kGuardRunning = 1 << 16;
		870	constexpr u32 kGuardWaiter = 1 << 17;
		871
		872	static int guard_acquire(ThreadState thr, uptr pc, atomic_uint32_t g,
		873	bool blocking_hooks = true) {
		874	if (blocking_hooks)
		875	OnPotentiallyBlockingRegionBegin();
		876	auto on_exit = at_scope_exit([blocking_hooks] {
		877	if (blocking_hooks)
		878	OnPotentiallyBlockingRegionEnd();
		879	});
		880
		881	for (;;) {
		882	u32 cmp = atomic_load(g, memory_order_acquire);
		883	if (cmp == kGuardInit) {
		884	if (atomic_compare_exchange_strong(g, &cmp, kGuardRunning,
		885	memory_order_relaxed))
		886	return 1;
		887	} else if (cmp == kGuardDone) {
		888	if (!thr->in_ignored_lib)
		889	Acquire(thr, pc, (uptr)g);
		890	return 0;
		891	} else {
		892	if ((cmp & kGuardWaiter) \|\|
		893	atomic_compare_exchange_strong(g, &cmp, cmp \| kGuardWaiter,
		894	memory_order_relaxed))
		895	FutexWait(g, cmp \| kGuardWaiter);
		896	}
		897	}
		898	}
		899
		900	static void guard_release(ThreadState thr, uptr pc, atomic_uint32_t g,
		901	u32 v) {
		902	if (!thr->in_ignored_lib)
		903	Release(thr, pc, (uptr)g);
		904	u32 old = atomic_exchange(g, v, memory_order_release);
		905	if (old & kGuardWaiter)
		906	FutexWake(g, 1 << 30);
		907	}
		908
852	// __cxa_guard_acquire and friends need to be intercepted in a special way -	909	// __cxa_guard_acquire and friends need to be intercepted in a special way -
853	// regular interceptors will break statically-linked libstdc++. Linux	910	// regular interceptors will break statically-linked libstdc++. Linux
854	// interceptors are especially defined as weak functions (so that they don't	911	// interceptors are especially defined as weak functions (so that they don't
...	@@ -859,7 +916,7 @@ TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {	...	@@ -859,7 +916,7 @@ TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
859	// these interceptors with INTERFACE_ATTRIBUTE.	916	// these interceptors with INTERFACE_ATTRIBUTE.
860	// On OS X, we don't support statically linking, so we just use a regular	917	// On OS X, we don't support statically linking, so we just use a regular
861	// interceptor.	918	// interceptor.
862	#if SANITIZER_MAC	919	#if SANITIZER_APPLE
863	#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR	920	#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
864	#else	921	#else
865	#define STDCXX_INTERCEPTOR(rettype, name, ...) \	922	#define STDCXX_INTERCEPTOR(rettype, name, ...) \
...	@@ -869,31 +926,17 @@ TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {	...	@@ -869,31 +926,17 @@ TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
869	// Used in thread-safe function static initialization.	926	// Used in thread-safe function static initialization.
870	STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) {	927	STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) {
871	SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g);	928	SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g);
872	OnPotentiallyBlockingRegionBegin();	929	return guard_acquire(thr, pc, g);
873	auto on_exit = at_scope_exit(&OnPotentiallyBlockingRegionEnd);
874	for (;;) {
875	u32 cmp = atomic_load(g, memory_order_acquire);
876	if (cmp == 0) {
877	if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed))
878	return 1;
879	} else if (cmp == 1) {
880	Acquire(thr, pc, (uptr)g);
881	return 0;
882	} else {
883	internal_sched_yield();
884	}
885	}
886	}	930	}
887		931
888	STDCXX_INTERCEPTOR(void, __cxa_guard_release, atomic_uint32_t *g) {	932	STDCXX_INTERCEPTOR(void, __cxa_guard_release, atomic_uint32_t *g) {
889	SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g);	933	SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g);
890	Release(thr, pc, (uptr)g);	934	guard_release(thr, pc, g, kGuardDone);
891	atomic_store(g, 1, memory_order_release);
892	}	935	}
893		936
894	STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) {	937	STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) {
895	SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g);	938	SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g);
896	atomic_store(g, 0, memory_order_relaxed);	939	guard_release(thr, pc, g, kGuardInit);
897	}	940	}
898		941
899	namespace __tsan {	942	namespace __tsan {
...	@@ -908,15 +951,16 @@ void DestroyThreadState() {	...	@@ -908,15 +951,16 @@ void DestroyThreadState() {
908	}	951	}
909		952
910	void PlatformCleanUpThreadState(ThreadState *thr) {	953	void PlatformCleanUpThreadState(ThreadState *thr) {
911	ThreadSignalContext *sctx = thr->signal_ctx;	954	ThreadSignalContext sctx = (ThreadSignalContext )atomic_load(
		955	&thr->signal_ctx, memory_order_relaxed);
912	if (sctx) {	956	if (sctx) {
913	thr->signal_ctx = 0;	957	atomic_store(&thr->signal_ctx, 0, memory_order_relaxed);
914	UnmapOrDie(sctx, sizeof(*sctx));	958	UnmapOrDie(sctx, sizeof(*sctx));
915	}	959	}
916	}	960	}
917	} // namespace __tsan	961	} // namespace __tsan
918		962
919	#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD	963	#if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
920	static void thread_finalize(void *v) {	964	static void thread_finalize(void *v) {
921	uptr iter = (uptr)v;	965	uptr iter = (uptr)v;
922	if (iter > 1) {	966	if (iter > 1) {
...	@@ -935,34 +979,33 @@ static void thread_finalize(void *v) {	...	@@ -935,34 +979,33 @@ static void thread_finalize(void *v) {
935	struct ThreadParam {	979	struct ThreadParam {
936	void* (callback)(void arg);	980	void* (callback)(void arg);
937	void *param;	981	void *param;
938	atomic_uintptr_t tid;	982	Tid tid;
		983	Semaphore created;
		984	Semaphore started;
939	};	985	};
940		986
941	extern "C" void __tsan_thread_start_func(void arg) {	987	extern "C" void __tsan_thread_start_func(void arg) {
942	ThreadParam p = (ThreadParam)arg;	988	ThreadParam p = (ThreadParam)arg;
943	void* (callback)(void arg) = p->callback;	989	void* (callback)(void arg) = p->callback;
944	void *param = p->param;	990	void *param = p->param;
945	int tid = 0;
946	{	991	{
947	cur_thread_init();	992	ThreadState *thr = cur_thread_init();
948	ThreadState *thr = cur_thread();
949	// Thread-local state is not initialized yet.	993	// Thread-local state is not initialized yet.
950	ScopedIgnoreInterceptors ignore;	994	ScopedIgnoreInterceptors ignore;
951	#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD	995	#if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
952	ThreadIgnoreBegin(thr, 0);	996	ThreadIgnoreBegin(thr, 0);
953	if (pthread_setspecific(interceptor_ctx()->finalize_key,	997	if (pthread_setspecific(interceptor_ctx()->finalize_key,
954	(void *)GetPthreadDestructorIterations())) {	998	(void *)GetPthreadDestructorIterations())) {
955	Printf("ThreadSanitizer: failed to set thread key\n");	999	Printf("ThreadSanitizer: failed to set thread key\n");
956	Die();	1000	Die();
957	}	1001	}
958	ThreadIgnoreEnd(thr, 0);	1002	ThreadIgnoreEnd(thr);
959	#endif	1003	#endif
960	while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0)	1004	p->created.Wait();
961	internal_sched_yield();
962	Processor *proc = ProcCreate();	1005	Processor *proc = ProcCreate();
963	ProcWire(proc, thr);	1006	ProcWire(proc, thr);
964	ThreadStart(thr, tid, GetTid(), ThreadType::Regular);	1007	ThreadStart(thr, p->tid, GetTid(), ThreadType::Regular);
965	atomic_store(&p->tid, 0, memory_order_release);	1008	p->started.Post();
966	}	1009	}
967	void *res = callback(param);	1010	void *res = callback(param);
968	// Prevent the callback from being tail called,	1011	// Prevent the callback from being tail called,
...	@@ -984,9 +1027,11 @@ TSAN_INTERCEPTOR(int, pthread_create,	...	@@ -984,9 +1027,11 @@ TSAN_INTERCEPTOR(int, pthread_create,
984	"fork is not supported. Dying (set die_after_fork=0 to override)\n");	1027	"fork is not supported. Dying (set die_after_fork=0 to override)\n");
985	Die();	1028	Die();
986	} else {	1029	} else {
987	VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded "	1030	VPrintf(1,
988	"fork is not supported (pid %d). Continuing because of "	1031	"ThreadSanitizer: starting new threads after multi-threaded "
989	"die_after_fork=0, but you are on your own\n", internal_getpid());	1032	"fork is not supported (pid %lu). Continuing because of "
		1033	"die_after_fork=0, but you are on your own\n",
		1034	internal_getpid());
990	}	1035	}
991	}	1036	}
992	__sanitizer_pthread_attr_t myattr;	1037	__sanitizer_pthread_attr_t myattr;
...	@@ -1001,18 +1046,18 @@ TSAN_INTERCEPTOR(int, pthread_create,	...	@@ -1001,18 +1046,18 @@ TSAN_INTERCEPTOR(int, pthread_create,
1001	ThreadParam p;	1046	ThreadParam p;
1002	p.callback = callback;	1047	p.callback = callback;
1003	p.param = param;	1048	p.param = param;
1004	atomic_store(&p.tid, 0, memory_order_relaxed);	1049	p.tid = kMainTid;
1005	int res = -1;	1050	int res = -1;
1006	{	1051	{
1007	// Otherwise we see false positives in pthread stack manipulation.	1052	// Otherwise we see false positives in pthread stack manipulation.
1008	ScopedIgnoreInterceptors ignore;	1053	ScopedIgnoreInterceptors ignore;
1009	ThreadIgnoreBegin(thr, pc);	1054	ThreadIgnoreBegin(thr, pc);
1010	res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p);	1055	res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p);
1011	ThreadIgnoreEnd(thr, pc);	1056	ThreadIgnoreEnd(thr);
1012	}	1057	}
1013	if (res == 0) {	1058	if (res == 0) {
1014	int tid = ThreadCreate(thr, pc, (uptr)th, IsStateDetached(detached));	1059	p.tid = ThreadCreate(thr, pc, (uptr )th, IsStateDetached(detached));
1015	CHECK_NE(tid, 0);	1060	CHECK_NE(p.tid, kMainTid);
1016	// Synchronization on p.tid serves two purposes:	1061	// Synchronization on p.tid serves two purposes:
1017	// 1. ThreadCreate must finish before the new thread starts.	1062	// 1. ThreadCreate must finish before the new thread starts.
1018	// Otherwise the new thread can call pthread_detach, but the pthread_t	1063	// Otherwise the new thread can call pthread_detach, but the pthread_t
...	@@ -1020,9 +1065,8 @@ TSAN_INTERCEPTOR(int, pthread_create,	...	@@ -1020,9 +1065,8 @@ TSAN_INTERCEPTOR(int, pthread_create,
1020	// 2. ThreadStart must finish before this thread continues.	1065	// 2. ThreadStart must finish before this thread continues.
1021	// Otherwise, this thread can call pthread_detach and reset thr->sync	1066	// Otherwise, this thread can call pthread_detach and reset thr->sync
1022	// before the new thread got a chance to acquire from it in ThreadStart.	1067	// before the new thread got a chance to acquire from it in ThreadStart.
1023	atomic_store(&p.tid, tid, memory_order_release);	1068	p.created.Post();
1024	while (atomic_load(&p.tid, memory_order_acquire) != 0)	1069	p.started.Wait();
1025	internal_sched_yield();
1026	}	1070	}
1027	if (attr == &myattr)	1071	if (attr == &myattr)
1028	pthread_attr_destroy(&myattr);	1072	pthread_attr_destroy(&myattr);
...	@@ -1031,10 +1075,10 @@ TSAN_INTERCEPTOR(int, pthread_create,	...	@@ -1031,10 +1075,10 @@ TSAN_INTERCEPTOR(int, pthread_create,
1031		1075
1032	TSAN_INTERCEPTOR(int, pthread_join, void th, void *ret) {	1076	TSAN_INTERCEPTOR(int, pthread_join, void th, void *ret) {
1033	SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret);	1077	SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret);
1034	int tid = ThreadConsumeTid(thr, pc, (uptr)th);	1078	Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
1035	ThreadIgnoreBegin(thr, pc);	1079	ThreadIgnoreBegin(thr, pc);
1036	int res = BLOCK_REAL(pthread_join)(th, ret);	1080	int res = BLOCK_REAL(pthread_join)(th, ret);
1037	ThreadIgnoreEnd(thr, pc);	1081	ThreadIgnoreEnd(thr);
1038	if (res == 0) {	1082	if (res == 0) {
1039	ThreadJoin(thr, pc, tid);	1083	ThreadJoin(thr, pc, tid);
1040	}	1084	}
...	@@ -1045,7 +1089,7 @@ DEFINE_REAL_PTHREAD_FUNCTIONS	...	@@ -1045,7 +1089,7 @@ DEFINE_REAL_PTHREAD_FUNCTIONS
1045		1089
1046	TSAN_INTERCEPTOR(int, pthread_detach, void *th) {	1090	TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
1047	SCOPED_INTERCEPTOR_RAW(pthread_detach, th);	1091	SCOPED_INTERCEPTOR_RAW(pthread_detach, th);
1048	int tid = ThreadConsumeTid(thr, pc, (uptr)th);	1092	Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
1049	int res = REAL(pthread_detach)(th);	1093	int res = REAL(pthread_detach)(th);
1050	if (res == 0) {	1094	if (res == 0) {
1051	ThreadDetach(thr, pc, tid);	1095	ThreadDetach(thr, pc, tid);
...	@@ -1056,7 +1100,7 @@ TSAN_INTERCEPTOR(int, pthread_detach, void *th) {	...	@@ -1056,7 +1100,7 @@ TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
1056	TSAN_INTERCEPTOR(void, pthread_exit, void *retval) {	1100	TSAN_INTERCEPTOR(void, pthread_exit, void *retval) {
1057	{	1101	{
1058	SCOPED_INTERCEPTOR_RAW(pthread_exit, retval);	1102	SCOPED_INTERCEPTOR_RAW(pthread_exit, retval);
1059	#if !SANITIZER_MAC && !SANITIZER_ANDROID	1103	#if !SANITIZER_APPLE && !SANITIZER_ANDROID
1060	CHECK_EQ(thr, &cur_thread_placeholder);	1104	CHECK_EQ(thr, &cur_thread_placeholder);
1061	#endif	1105	#endif
1062	}	1106	}
...	@@ -1066,10 +1110,10 @@ TSAN_INTERCEPTOR(void, pthread_exit, void *retval) {	...	@@ -1066,10 +1110,10 @@ TSAN_INTERCEPTOR(void, pthread_exit, void *retval) {
1066	#if SANITIZER_LINUX	1110	#if SANITIZER_LINUX
1067	TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void th, void *ret) {	1111	TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void th, void *ret) {
1068	SCOPED_INTERCEPTOR_RAW(pthread_tryjoin_np, th, ret);	1112	SCOPED_INTERCEPTOR_RAW(pthread_tryjoin_np, th, ret);
1069	int tid = ThreadConsumeTid(thr, pc, (uptr)th);	1113	Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
1070	ThreadIgnoreBegin(thr, pc);	1114	ThreadIgnoreBegin(thr, pc);
1071	int res = REAL(pthread_tryjoin_np)(th, ret);	1115	int res = REAL(pthread_tryjoin_np)(th, ret);
1072	ThreadIgnoreEnd(thr, pc);	1116	ThreadIgnoreEnd(thr);
1073	if (res == 0)	1117	if (res == 0)
1074	ThreadJoin(thr, pc, tid);	1118	ThreadJoin(thr, pc, tid);
1075	else	1119	else
...	@@ -1080,10 +1124,10 @@ TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void th, void *ret) {	...	@@ -1080,10 +1124,10 @@ TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void th, void *ret) {
1080	TSAN_INTERCEPTOR(int, pthread_timedjoin_np, void th, void *ret,	1124	TSAN_INTERCEPTOR(int, pthread_timedjoin_np, void th, void *ret,
1081	const struct timespec *abstime) {	1125	const struct timespec *abstime) {
1082	SCOPED_INTERCEPTOR_RAW(pthread_timedjoin_np, th, ret, abstime);	1126	SCOPED_INTERCEPTOR_RAW(pthread_timedjoin_np, th, ret, abstime);
1083	int tid = ThreadConsumeTid(thr, pc, (uptr)th);	1127	Tid tid = ThreadConsumeTid(thr, pc, (uptr)th);
1084	ThreadIgnoreBegin(thr, pc);	1128	ThreadIgnoreBegin(thr, pc);
1085	int res = BLOCK_REAL(pthread_timedjoin_np)(th, ret, abstime);	1129	int res = BLOCK_REAL(pthread_timedjoin_np)(th, ret, abstime);
1086	ThreadIgnoreEnd(thr, pc);	1130	ThreadIgnoreEnd(thr);
1087	if (res == 0)	1131	if (res == 0)
1088	ThreadJoin(thr, pc, tid);	1132	ThreadJoin(thr, pc, tid);
1089	else	1133	else
...	@@ -1152,9 +1196,8 @@ void CondMutexUnlockCtx<Fn>::Unlock() const {	...	@@ -1152,9 +1196,8 @@ void CondMutexUnlockCtx<Fn>::Unlock() const {
1152	// tsan code. Also ScopedInterceptor and BlockingCall destructors won't run	1196	// tsan code. Also ScopedInterceptor and BlockingCall destructors won't run
1153	// since the thread is cancelled, so we have to manually execute them	1197	// since the thread is cancelled, so we have to manually execute them
1154	// (the thread still can run some user code due to pthread_cleanup_push).	1198	// (the thread still can run some user code due to pthread_cleanup_push).
1155	ThreadSignalContext *ctx = SigCtx(thr);	1199	CHECK_EQ(atomic_load(&thr->in_blocking_func, memory_order_relaxed), 1);
1156	CHECK_EQ(atomic_load(&ctx->in_blocking_func, memory_order_relaxed), 1);	1200	atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
1157	atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed);
1158	MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock);	1201	MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock);
1159	// Undo BlockingCall ctor effects.	1202	// Undo BlockingCall ctor effects.
1160	thr->ignore_interceptors--;	1203	thr->ignore_interceptors--;
...	@@ -1225,7 +1268,7 @@ INTERCEPTOR(int, pthread_cond_clockwait, void c, void m,	...	@@ -1225,7 +1268,7 @@ INTERCEPTOR(int, pthread_cond_clockwait, void c, void m,
1225	#define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT	1268	#define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT
1226	#endif	1269	#endif
1227		1270
1228	#if SANITIZER_MAC	1271	#if SANITIZER_APPLE
1229	INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void c, void m,	1272	INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void c, void m,
1230	void *reltime) {	1273	void *reltime) {
1231	void *cond = init_cond(c);	1274	void *cond = init_cond(c);
...	@@ -1292,6 +1335,19 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {	...	@@ -1292,6 +1335,19 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
1292	return res;	1335	return res;
1293	}	1336	}
1294		1337
		1338	TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) {
		1339	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock, m);
		1340	MutexPreLock(thr, pc, (uptr)m);
		1341	int res = REAL(pthread_mutex_lock)(m);
		1342	if (res == errno_EOWNERDEAD)
		1343	MutexRepair(thr, pc, (uptr)m);
		1344	if (res == 0 \|\| res == errno_EOWNERDEAD)
		1345	MutexPostLock(thr, pc, (uptr)m);
		1346	if (res == errno_EINVAL)
		1347	MutexInvalidAccess(thr, pc, (uptr)m);
		1348	return res;
		1349	}
		1350
1295	TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {	1351	TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
1296	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);	1352	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
1297	int res = REAL(pthread_mutex_trylock)(m);	1353	int res = REAL(pthread_mutex_trylock)(m);
...	@@ -1302,7 +1358,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {	...	@@ -1302,7 +1358,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
1302	return res;	1358	return res;
1303	}	1359	}
1304		1360
1305	#if !SANITIZER_MAC	1361	#if !SANITIZER_APPLE
1306	TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void m, void abstime) {	1362	TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void m, void abstime) {
1307	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime);	1363	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime);
1308	int res = REAL(pthread_mutex_timedlock)(m, abstime);	1364	int res = REAL(pthread_mutex_timedlock)(m, abstime);
...	@@ -1313,7 +1369,44 @@ TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void m, void abstime) {	...	@@ -1313,7 +1369,44 @@ TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void m, void abstime) {
1313	}	1369	}
1314	#endif	1370	#endif
1315		1371
1316	#if !SANITIZER_MAC	1372	TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) {
		1373	SCOPED_TSAN_INTERCEPTOR(pthread_mutex_unlock, m);
		1374	MutexUnlock(thr, pc, (uptr)m);
		1375	int res = REAL(pthread_mutex_unlock)(m);
		1376	if (res == errno_EINVAL)
		1377	MutexInvalidAccess(thr, pc, (uptr)m);
		1378	return res;
		1379	}
		1380
		1381	#if SANITIZER_GLIBC
		1382	# if !__GLIBC_PREREQ(2, 34)
		1383	// glibc 2.34 applies a non-default version for the two functions. They are no
		1384	// longer expected to be intercepted by programs.
		1385	TSAN_INTERCEPTOR(int, __pthread_mutex_lock, void *m) {
		1386	SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_lock, m);
		1387	MutexPreLock(thr, pc, (uptr)m);
		1388	int res = REAL(__pthread_mutex_lock)(m);
		1389	if (res == errno_EOWNERDEAD)
		1390	MutexRepair(thr, pc, (uptr)m);
		1391	if (res == 0 \|\| res == errno_EOWNERDEAD)
		1392	MutexPostLock(thr, pc, (uptr)m);
		1393	if (res == errno_EINVAL)
		1394	MutexInvalidAccess(thr, pc, (uptr)m);
		1395	return res;
		1396	}
		1397
		1398	TSAN_INTERCEPTOR(int, __pthread_mutex_unlock, void *m) {
		1399	SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_unlock, m);
		1400	MutexUnlock(thr, pc, (uptr)m);
		1401	int res = REAL(__pthread_mutex_unlock)(m);
		1402	if (res == errno_EINVAL)
		1403	MutexInvalidAccess(thr, pc, (uptr)m);
		1404	return res;
		1405	}
		1406	# endif
		1407	#endif
		1408
		1409	#if !SANITIZER_APPLE
1317	TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) {	1410	TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) {
1318	SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared);	1411	SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared);
1319	int res = REAL(pthread_spin_init)(m, pshared);	1412	int res = REAL(pthread_spin_init)(m, pshared);
...	@@ -1396,7 +1489,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) {	...	@@ -1396,7 +1489,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) {
1396	return res;	1489	return res;
1397	}	1490	}
1398		1491
1399	#if !SANITIZER_MAC	1492	#if !SANITIZER_APPLE
1400	TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void m, void abstime) {	1493	TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void m, void abstime) {
1401	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime);	1494	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime);
1402	int res = REAL(pthread_rwlock_timedrdlock)(m, abstime);	1495	int res = REAL(pthread_rwlock_timedrdlock)(m, abstime);
...	@@ -1426,7 +1519,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) {	...	@@ -1426,7 +1519,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) {
1426	return res;	1519	return res;
1427	}	1520	}
1428		1521
1429	#if !SANITIZER_MAC	1522	#if !SANITIZER_APPLE
1430	TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void m, void abstime) {	1523	TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void m, void abstime) {
1431	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime);	1524	SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime);
1432	int res = REAL(pthread_rwlock_timedwrlock)(m, abstime);	1525	int res = REAL(pthread_rwlock_timedwrlock)(m, abstime);
...	@@ -1444,17 +1537,17 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) {	...	@@ -1444,17 +1537,17 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) {
1444	return res;	1537	return res;
1445	}	1538	}
1446		1539
1447	#if !SANITIZER_MAC	1540	#if !SANITIZER_APPLE
1448	TSAN_INTERCEPTOR(int, pthread_barrier_init, void b, void a, unsigned count) {	1541	TSAN_INTERCEPTOR(int, pthread_barrier_init, void b, void a, unsigned count) {
1449	SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count);	1542	SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count);
1450	MemoryWrite(thr, pc, (uptr)b, kSizeLog1);	1543	MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite);
1451	int res = REAL(pthread_barrier_init)(b, a, count);	1544	int res = REAL(pthread_barrier_init)(b, a, count);
1452	return res;	1545	return res;
1453	}	1546	}
1454		1547
1455	TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) {	1548	TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) {
1456	SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b);	1549	SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b);
1457	MemoryWrite(thr, pc, (uptr)b, kSizeLog1);	1550	MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite);
1458	int res = REAL(pthread_barrier_destroy)(b);	1551	int res = REAL(pthread_barrier_destroy)(b);
1459	return res;	1552	return res;
1460	}	1553	}
...	@@ -1462,9 +1555,9 @@ TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) {	...	@@ -1462,9 +1555,9 @@ TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) {
1462	TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {	1555	TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
1463	SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b);	1556	SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b);
1464	Release(thr, pc, (uptr)b);	1557	Release(thr, pc, (uptr)b);
1465	MemoryRead(thr, pc, (uptr)b, kSizeLog1);	1558	MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead);
1466	int res = REAL(pthread_barrier_wait)(b);	1559	int res = REAL(pthread_barrier_wait)(b);
1467	MemoryRead(thr, pc, (uptr)b, kSizeLog1);	1560	MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead);
1468	if (res == 0 \|\| res == PTHREAD_BARRIER_SERIAL_THREAD) {	1561	if (res == 0 \|\| res == PTHREAD_BARRIER_SERIAL_THREAD) {
1469	Acquire(thr, pc, (uptr)b);	1562	Acquire(thr, pc, (uptr)b);
1470	}	1563	}
...	@@ -1478,7 +1571,7 @@ TSAN_INTERCEPTOR(int, pthread_once, void o, void (f)()) {	...	@@ -1478,7 +1571,7 @@ TSAN_INTERCEPTOR(int, pthread_once, void o, void (f)()) {
1478	return errno_EINVAL;	1571	return errno_EINVAL;
1479	atomic_uint32_t *a;	1572	atomic_uint32_t *a;
1480		1573
1481	if (SANITIZER_MAC)	1574	if (SANITIZER_APPLE)
1482	a = static_cast<atomic_uint32_t>((void )((char *)o + sizeof(long_t)));	1575	a = static_cast<atomic_uint32_t>((void )((char *)o + sizeof(long_t)));
1483	else if (SANITIZER_NETBSD)	1576	else if (SANITIZER_NETBSD)
1484	a = static_cast<atomic_uint32_t*>	1577	a = static_cast<atomic_uint32_t*>
...	@@ -1486,25 +1579,16 @@ TSAN_INTERCEPTOR(int, pthread_once, void o, void (f)()) {	...	@@ -1486,25 +1579,16 @@ TSAN_INTERCEPTOR(int, pthread_once, void o, void (f)()) {
1486	else	1579	else
1487	a = static_cast<atomic_uint32_t*>(o);	1580	a = static_cast<atomic_uint32_t*>(o);
1488		1581
1489	u32 v = atomic_load(a, memory_order_acquire);	1582	// Mac OS X appears to use pthread_once() where calling BlockingRegion hooks
1490	if (v == 0 && atomic_compare_exchange_strong(a, &v, 1,	1583	// result in crashes due to too little stack space.
1491	memory_order_relaxed)) {	1584	if (guard_acquire(thr, pc, a, !SANITIZER_APPLE)) {
1492	(*f)();	1585	(*f)();
1493	if (!thr->in_ignored_lib)	1586	guard_release(thr, pc, a, kGuardDone);
1494	Release(thr, pc, (uptr)o);
1495	atomic_store(a, 2, memory_order_release);
1496	} else {
1497	while (v != 2) {
1498	internal_sched_yield();
1499	v = atomic_load(a, memory_order_acquire);
1500	}
1501	if (!thr->in_ignored_lib)
1502	Acquire(thr, pc, (uptr)o);
1503	}	1587	}
1504	return 0;	1588	return 0;
1505	}	1589	}
1506		1590
1507	#if SANITIZER_LINUX && !SANITIZER_ANDROID	1591	#if SANITIZER_GLIBC
1508	TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {	1592	TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
1509	SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf);	1593	SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf);
1510	if (fd > 0)	1594	if (fd > 0)
...	@@ -1517,20 +1601,20 @@ TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {	...	@@ -1517,20 +1601,20 @@ TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
1517	#endif	1601	#endif
1518		1602
1519	TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {	1603	TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {
1520	#if SANITIZER_FREEBSD \|\| SANITIZER_MAC \|\| SANITIZER_ANDROID \|\| SANITIZER_NETBSD	1604	#if SANITIZER_GLIBC
1521	SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf);	1605	SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);
1522	if (fd > 0)	1606	if (fd > 0)
1523	FdAccess(thr, pc, fd);	1607	FdAccess(thr, pc, fd);
1524	return REAL(fstat)(fd, buf);	1608	return REAL(__fxstat)(0, fd, buf);
1525	#else	1609	#else
1526	SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);	1610	SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf);
1527	if (fd > 0)	1611	if (fd > 0)
1528	FdAccess(thr, pc, fd);	1612	FdAccess(thr, pc, fd);
1529	return REAL(__fxstat)(0, fd, buf);	1613	return REAL(fstat)(fd, buf);
1530	#endif	1614	#endif
1531	}	1615	}
1532		1616
1533	#if SANITIZER_LINUX && !SANITIZER_ANDROID	1617	#if SANITIZER_GLIBC
1534	TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {	1618	TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
1535	SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);	1619	SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
1536	if (fd > 0)	1620	if (fd > 0)
...	@@ -1542,7 +1626,7 @@ TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {	...	@@ -1542,7 +1626,7 @@ TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
1542	#define TSAN_MAYBE_INTERCEPT___FXSTAT64	1626	#define TSAN_MAYBE_INTERCEPT___FXSTAT64
1543	#endif	1627	#endif
1544		1628
1545	#if SANITIZER_LINUX && !SANITIZER_ANDROID	1629	#if SANITIZER_GLIBC
1546	TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {	1630	TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
1547	SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf);	1631	SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf);
1548	if (fd > 0)	1632	if (fd > 0)
...	@@ -1624,7 +1708,7 @@ TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) {	...	@@ -1624,7 +1708,7 @@ TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) {
1624	return newfd2;	1708	return newfd2;
1625	}	1709	}
1626		1710
1627	#if !SANITIZER_MAC	1711	#if !SANITIZER_APPLE
1628	TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) {	1712	TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) {
1629	SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags);	1713	SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags);
1630	int newfd2 = REAL(dup3)(oldfd, newfd, flags);	1714	int newfd2 = REAL(dup3)(oldfd, newfd, flags);
...	@@ -1649,11 +1733,10 @@ TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) {	...	@@ -1649,11 +1733,10 @@ TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) {
1649		1733
1650	#if SANITIZER_LINUX	1734	#if SANITIZER_LINUX
1651	TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) {	1735	TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) {
1652	SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags);	1736	SCOPED_INTERCEPTOR_RAW(signalfd, fd, mask, flags);
1653	if (fd >= 0)	1737	FdClose(thr, pc, fd);
1654	FdClose(thr, pc, fd);
1655	fd = REAL(signalfd)(fd, mask, flags);	1738	fd = REAL(signalfd)(fd, mask, flags);
1656	if (fd >= 0)	1739	if (!MustIgnoreInterceptor(thr))
1657	FdSignalCreate(thr, pc, fd);	1740	FdSignalCreate(thr, pc, fd);
1658	return fd;	1741	return fd;
1659	}	1742	}
...	@@ -1730,17 +1813,16 @@ TSAN_INTERCEPTOR(int, listen, int fd, int backlog) {	...	@@ -1730,17 +1813,16 @@ TSAN_INTERCEPTOR(int, listen, int fd, int backlog) {
1730	}	1813	}
1731		1814
1732	TSAN_INTERCEPTOR(int, close, int fd) {	1815	TSAN_INTERCEPTOR(int, close, int fd) {
1733	SCOPED_TSAN_INTERCEPTOR(close, fd);	1816	SCOPED_INTERCEPTOR_RAW(close, fd);
1734	if (fd >= 0)	1817	if (!in_symbolizer())
1735	FdClose(thr, pc, fd);	1818	FdClose(thr, pc, fd);
1736	return REAL(close)(fd);	1819	return REAL(close)(fd);
1737	}	1820	}
1738		1821
1739	#if SANITIZER_LINUX	1822	#if SANITIZER_LINUX
1740	TSAN_INTERCEPTOR(int, __close, int fd) {	1823	TSAN_INTERCEPTOR(int, __close, int fd) {
1741	SCOPED_TSAN_INTERCEPTOR(__close, fd);	1824	SCOPED_INTERCEPTOR_RAW(__close, fd);
1742	if (fd >= 0)	1825	FdClose(thr, pc, fd);
1743	FdClose(thr, pc, fd);
1744	return REAL(__close)(fd);	1826	return REAL(__close)(fd);
1745	}	1827	}
1746	#define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close)	1828	#define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close)
...	@@ -1751,13 +1833,10 @@ TSAN_INTERCEPTOR(int, __close, int fd) {	...	@@ -1751,13 +1833,10 @@ TSAN_INTERCEPTOR(int, __close, int fd) {
1751	// glibc guts	1833	// glibc guts
1752	#if SANITIZER_LINUX && !SANITIZER_ANDROID	1834	#if SANITIZER_LINUX && !SANITIZER_ANDROID
1753	TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) {	1835	TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) {
1754	SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr);	1836	SCOPED_INTERCEPTOR_RAW(__res_iclose, state, free_addr);
1755	int fds[64];	1837	int fds[64];
1756	int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds));	1838	int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds));
1757	for (int i = 0; i < cnt; i++) {	1839	for (int i = 0; i < cnt; i++) FdClose(thr, pc, fds[i]);
1758	if (fds[i] > 0)
1759	FdClose(thr, pc, fds[i]);
1760	}
1761	REAL(__res_iclose)(state, free_addr);	1840	REAL(__res_iclose)(state, free_addr);
1762	}	1841	}
1763	#define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose)	1842	#define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose)
...	@@ -1773,7 +1852,7 @@ TSAN_INTERCEPTOR(int, pipe, int *pipefd) {	...	@@ -1773,7 +1852,7 @@ TSAN_INTERCEPTOR(int, pipe, int *pipefd) {
1773	return res;	1852	return res;
1774	}	1853	}
1775		1854
1776	#if !SANITIZER_MAC	1855	#if !SANITIZER_APPLE
1777	TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) {	1856	TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) {
1778	SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags);	1857	SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags);
1779	int res = REAL(pipe2)(pipefd, flags);	1858	int res = REAL(pipe2)(pipefd, flags);
...	@@ -1838,7 +1917,7 @@ TSAN_INTERCEPTOR(int, rmdir, char *path) {	...	@@ -1838,7 +1917,7 @@ TSAN_INTERCEPTOR(int, rmdir, char *path) {
1838	}	1917	}
1839		1918
1840	TSAN_INTERCEPTOR(int, closedir, void *dirp) {	1919	TSAN_INTERCEPTOR(int, closedir, void *dirp) {
1841	SCOPED_TSAN_INTERCEPTOR(closedir, dirp);	1920	SCOPED_INTERCEPTOR_RAW(closedir, dirp);
1842	if (dirp) {	1921	if (dirp) {
1843	int fd = dirfd(dirp);	1922	int fd = dirfd(dirp);
1844	FdClose(thr, pc, fd);	1923	FdClose(thr, pc, fd);
...	@@ -1869,8 +1948,10 @@ TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) {	...	@@ -1869,8 +1948,10 @@ TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) {
1869	FdAccess(thr, pc, epfd);	1948	FdAccess(thr, pc, epfd);
1870	if (epfd >= 0 && fd >= 0)	1949	if (epfd >= 0 && fd >= 0)
1871	FdAccess(thr, pc, fd);	1950	FdAccess(thr, pc, fd);
1872	if (op == EPOLL_CTL_ADD && epfd >= 0)	1951	if (op == EPOLL_CTL_ADD && epfd >= 0) {
		1952	FdPollAdd(thr, pc, epfd, fd);
1873	FdRelease(thr, pc, epfd);	1953	FdRelease(thr, pc, epfd);
		1954	}
1874	int res = REAL(epoll_ctl)(epfd, op, fd, ev);	1955	int res = REAL(epoll_ctl)(epfd, op, fd, ev);
1875	return res;	1956	return res;
1876	}	1957	}
...	@@ -1896,12 +1977,34 @@ TSAN_INTERCEPTOR(int, epoll_pwait, int epfd, void *ev, int cnt, int timeout,	...	@@ -1896,12 +1977,34 @@ TSAN_INTERCEPTOR(int, epoll_pwait, int epfd, void *ev, int cnt, int timeout,
1896	return res;	1977	return res;
1897	}	1978	}
1898		1979
1899	#define TSAN_MAYBE_INTERCEPT_EPOLL \	1980	TSAN_INTERCEPTOR(int, epoll_pwait2, int epfd, void ev, int cnt, void timeout,
1900	TSAN_INTERCEPT(epoll_create); \	1981	void *sigmask) {
1901	TSAN_INTERCEPT(epoll_create1); \	1982	SCOPED_INTERCEPTOR_RAW(epoll_pwait2, epfd, ev, cnt, timeout, sigmask);
1902	TSAN_INTERCEPT(epoll_ctl); \	1983	// This function is new and may not be present in libc and/or kernel.
1903	TSAN_INTERCEPT(epoll_wait); \	1984	// Since we effectively add it to libc (as will be probed by the program
1904	TSAN_INTERCEPT(epoll_pwait)	1985	// using dlsym or a weak function pointer) we need to handle the case
		1986	// when it's not present in the actual libc.
		1987	if (!REAL(epoll_pwait2)) {
		1988	errno = errno_ENOSYS;
		1989	return -1;
		1990	}
		1991	if (MustIgnoreInterceptor(thr))
		1992	REAL(epoll_pwait2)(epfd, ev, cnt, timeout, sigmask);
		1993	if (epfd >= 0)
		1994	FdAccess(thr, pc, epfd);
		1995	int res = BLOCK_REAL(epoll_pwait2)(epfd, ev, cnt, timeout, sigmask);
		1996	if (res > 0 && epfd >= 0)
		1997	FdAcquire(thr, pc, epfd);
		1998	return res;
		1999	}
		2000
		2001	# define TSAN_MAYBE_INTERCEPT_EPOLL \
		2002	TSAN_INTERCEPT(epoll_create); \
		2003	TSAN_INTERCEPT(epoll_create1); \
		2004	TSAN_INTERCEPT(epoll_ctl); \
		2005	TSAN_INTERCEPT(epoll_wait); \
		2006	TSAN_INTERCEPT(epoll_pwait); \
		2007	TSAN_INTERCEPT(epoll_pwait2)
1905	#else	2008	#else
1906	#define TSAN_MAYBE_INTERCEPT_EPOLL	2009	#define TSAN_MAYBE_INTERCEPT_EPOLL
1907	#endif	2010	#endif
...	@@ -1933,24 +2036,47 @@ TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set,	...	@@ -1933,24 +2036,47 @@ TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set,
1933		2036
1934	namespace __tsan {	2037	namespace __tsan {
1935		2038
		2039	static void ReportErrnoSpoiling(ThreadState *thr, uptr pc, int sig) {
		2040	VarSizeStackTrace stack;
		2041	// StackTrace::GetNestInstructionPc(pc) is used because return address is
		2042	// expected, OutputReport() will undo this.
		2043	ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack);
		2044	ThreadRegistryLock l(&ctx->thread_registry);
		2045	ScopedReport rep(ReportTypeErrnoInSignal);
		2046	rep.SetSigNum(sig);
		2047	if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) {
		2048	rep.AddStack(stack, true);
		2049	OutputReport(thr, rep);
		2050	}
		2051	}
		2052
1936	static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,	2053	static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
1937	bool sigact, int sig,	2054	int sig, __sanitizer_siginfo *info,
1938	__sanitizer_siginfo info, void uctx) {	2055	void *uctx) {
		2056	CHECK(thr->slot);
1939	__sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions;	2057	__sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions;
1940	if (acquire)	2058	if (acquire)
1941	Acquire(thr, 0, (uptr)&sigactions[sig]);	2059	Acquire(thr, 0, (uptr)&sigactions[sig]);
1942	// Signals are generally asynchronous, so if we receive a signals when	2060	// Signals are generally asynchronous, so if we receive a signals when
1943	// ignores are enabled we should disable ignores. This is critical for sync	2061	// ignores are enabled we should disable ignores. This is critical for sync
1944	// and interceptors, because otherwise we can miss syncronization and report	2062	// and interceptors, because otherwise we can miss synchronization and report
1945	// false races.	2063	// false races.
1946	int ignore_reads_and_writes = thr->ignore_reads_and_writes;	2064	int ignore_reads_and_writes = thr->ignore_reads_and_writes;
1947	int ignore_interceptors = thr->ignore_interceptors;	2065	int ignore_interceptors = thr->ignore_interceptors;
1948	int ignore_sync = thr->ignore_sync;	2066	int ignore_sync = thr->ignore_sync;
		2067	// For symbolizer we only process SIGSEGVs synchronously
		2068	// (bug in symbolizer or in tsan). But we want to reset
		2069	// in_symbolizer to fail gracefully. Symbolizer and user code
		2070	// use different memory allocators, so if we don't reset
		2071	// in_symbolizer we can get memory allocated with one being
		2072	// feed with another, which can cause more crashes.
		2073	int in_symbolizer = thr->in_symbolizer;
1949	if (!ctx->after_multithreaded_fork) {	2074	if (!ctx->after_multithreaded_fork) {
1950	thr->ignore_reads_and_writes = 0;	2075	thr->ignore_reads_and_writes = 0;
1951	thr->fast_state.ClearIgnoreBit();	2076	thr->fast_state.ClearIgnoreBit();
1952	thr->ignore_interceptors = 0;	2077	thr->ignore_interceptors = 0;
1953	thr->ignore_sync = 0;	2078	thr->ignore_sync = 0;
		2079	thr->in_symbolizer = 0;
1954	}	2080	}
1955	// Ensure that the handler does not spoil errno.	2081	// Ensure that the handler does not spoil errno.
1956	const int saved_errno = errno;	2082	const int saved_errno = errno;
...	@@ -1958,13 +2084,14 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,	...	@@ -1958,13 +2084,14 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
1958	// This code races with sigaction. Be careful to not read sa_sigaction twice.	2084	// This code races with sigaction. Be careful to not read sa_sigaction twice.
1959	// Also need to remember pc for reporting before the call,	2085	// Also need to remember pc for reporting before the call,
1960	// because the handler can reset it.	2086	// because the handler can reset it.
1961	volatile uptr pc =	2087	volatile uptr pc = (sigactions[sig].sa_flags & SA_SIGINFO)
1962	sigact ? (uptr)sigactions[sig].sigaction : (uptr)sigactions[sig].handler;	2088	? (uptr)sigactions[sig].sigaction
		2089	: (uptr)sigactions[sig].handler;
1963	if (pc != sig_dfl && pc != sig_ign) {	2090	if (pc != sig_dfl && pc != sig_ign) {
1964	if (sigact)	2091	// The callback can be either sa_handler or sa_sigaction.
1965	((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx);	2092	// They have different signatures, but we assume that passing
1966	else	2093	// additional arguments to sa_handler works and is harmless.
1967	((__sanitizer_sighandler_ptr)pc)(sig);	2094	((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx);
1968	}	2095	}
1969	if (!ctx->after_multithreaded_fork) {	2096	if (!ctx->after_multithreaded_fork) {
1970	thr->ignore_reads_and_writes = ignore_reads_and_writes;	2097	thr->ignore_reads_and_writes = ignore_reads_and_writes;
...	@@ -1972,6 +2099,7 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,	...	@@ -1972,6 +2099,7 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
1972	thr->fast_state.SetIgnoreBit();	2099	thr->fast_state.SetIgnoreBit();
1973	thr->ignore_interceptors = ignore_interceptors;	2100	thr->ignore_interceptors = ignore_interceptors;
1974	thr->ignore_sync = ignore_sync;	2101	thr->ignore_sync = ignore_sync;
		2102	thr->in_symbolizer = in_symbolizer;
1975	}	2103	}
1976	// We do not detect errno spoiling for SIGTERM,	2104	// We do not detect errno spoiling for SIGTERM,
1977	// because some SIGTERM handlers do spoil errno but reraise SIGTERM,	2105	// because some SIGTERM handlers do spoil errno but reraise SIGTERM,
...	@@ -1981,27 +2109,16 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,	...	@@ -1981,27 +2109,16 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
1981	// from rtl_generic_sighandler) we have not yet received the reraised	2109	// from rtl_generic_sighandler) we have not yet received the reraised
1982	// signal; and it looks too fragile to intercept all ways to reraise a signal.	2110	// signal; and it looks too fragile to intercept all ways to reraise a signal.
1983	if (ShouldReport(thr, ReportTypeErrnoInSignal) && !sync && sig != SIGTERM &&	2111	if (ShouldReport(thr, ReportTypeErrnoInSignal) && !sync && sig != SIGTERM &&
1984	errno != 99) {	2112	errno != 99)
1985	VarSizeStackTrace stack;	2113	ReportErrnoSpoiling(thr, pc, sig);
1986	// StackTrace::GetNestInstructionPc(pc) is used because return address is
1987	// expected, OutputReport() will undo this.
1988	ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack);
1989	ThreadRegistryLock l(ctx->thread_registry);
1990	ScopedReport rep(ReportTypeErrnoInSignal);
1991	if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) {
1992	rep.AddStack(stack, true);
1993	OutputReport(thr, rep);
1994	}
1995	}
1996	errno = saved_errno;	2114	errno = saved_errno;
1997	}	2115	}
1998		2116
1999	void ProcessPendingSignals(ThreadState *thr) {	2117	void ProcessPendingSignalsImpl(ThreadState *thr) {
		2118	atomic_store(&thr->pending_signals, 0, memory_order_relaxed);
2000	ThreadSignalContext *sctx = SigCtx(thr);	2119	ThreadSignalContext *sctx = SigCtx(thr);
2001	if (sctx == 0 \|\|	2120	if (sctx == 0)
2002	atomic_load(&sctx->have_pending_signals, memory_order_relaxed) == 0)
2003	return;	2121	return;
2004	atomic_store(&sctx->have_pending_signals, 0, memory_order_relaxed);
2005	atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed);	2122	atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed);
2006	internal_sigfillset(&sctx->emptyset);	2123	internal_sigfillset(&sctx->emptyset);
2007	int res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->emptyset, &sctx->oldset);	2124	int res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->emptyset, &sctx->oldset);
...	@@ -2010,8 +2127,8 @@ void ProcessPendingSignals(ThreadState *thr) {	...	@@ -2010,8 +2127,8 @@ void ProcessPendingSignals(ThreadState *thr) {
2010	SignalDesc *signal = &sctx->pending_signals[sig];	2127	SignalDesc *signal = &sctx->pending_signals[sig];
2011	if (signal->armed) {	2128	if (signal->armed) {
2012	signal->armed = false;	2129	signal->armed = false;
2013	CallUserSignalHandler(thr, false, true, signal->sigaction, sig,	2130	CallUserSignalHandler(thr, false, true, sig, &signal->siginfo,
2014	&signal->siginfo, &signal->ctx);	2131	&signal->ctx);
2015	}	2132	}
2016	}	2133	}
2017	res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->oldset, 0);	2134	res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->oldset, 0);
...	@@ -2021,35 +2138,40 @@ void ProcessPendingSignals(ThreadState *thr) {	...	@@ -2021,35 +2138,40 @@ void ProcessPendingSignals(ThreadState *thr) {
2021		2138
2022	} // namespace __tsan	2139	} // namespace __tsan
2023		2140
2024	static bool is_sync_signal(ThreadSignalContext *sctx, int sig) {	2141	static bool is_sync_signal(ThreadSignalContext *sctx, int sig,
		2142	__sanitizer_siginfo *info) {
		2143	// If we are sending signal to ourselves, we must process it now.
		2144	if (sctx && sig == sctx->int_signal_send)
		2145	return true;
		2146	#if SANITIZER_HAS_SIGINFO
		2147	// POSIX timers can be configured to send any kind of signal; however, it
		2148	// doesn't make any sense to consider a timer signal as synchronous!
		2149	if (info->si_code == SI_TIMER)
		2150	return false;
		2151	#endif
2025	return sig == SIGSEGV \|\| sig == SIGBUS \|\| sig == SIGILL \|\| sig == SIGTRAP \|\|	2152	return sig == SIGSEGV \|\| sig == SIGBUS \|\| sig == SIGILL \|\| sig == SIGTRAP \|\|
2026	sig == SIGABRT \|\| sig == SIGFPE \|\| sig == SIGPIPE \|\| sig == SIGSYS \|\|	2153	sig == SIGABRT \|\| sig == SIGFPE \|\| sig == SIGPIPE \|\| sig == SIGSYS;
2027	// If we are sending signal to ourselves, we must process it now.
2028	(sctx && sig == sctx->int_signal_send);
2029	}	2154	}
2030		2155
2031	void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,	2156	void sighandler(int sig, __sanitizer_siginfo info, void ctx) {
2032	__sanitizer_siginfo *info,	2157	ThreadState *thr = cur_thread_init();
2033	void *ctx) {
2034	cur_thread_init();
2035	ThreadState *thr = cur_thread();
2036	ThreadSignalContext *sctx = SigCtx(thr);	2158	ThreadSignalContext *sctx = SigCtx(thr);
2037	if (sig < 0 \|\| sig >= kSigCount) {	2159	if (sig < 0 \|\| sig >= kSigCount) {
2038	VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig);	2160	VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig);
2039	return;	2161	return;
2040	}	2162	}
2041	// Don't mess with synchronous signals.	2163	// Don't mess with synchronous signals.
2042	const bool sync = is_sync_signal(sctx, sig);	2164	const bool sync = is_sync_signal(sctx, sig, info);
2043	if (sync \|\|	2165	if (sync \|\|
2044	// If we are in blocking function, we can safely process it now	2166	// If we are in blocking function, we can safely process it now
2045	// (but check if we are in a recursive interceptor,	2167	// (but check if we are in a recursive interceptor,
2046	// i.e. pthread_join()->munmap()).	2168	// i.e. pthread_join()->munmap()).
2047	(sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed))) {	2169	atomic_load(&thr->in_blocking_func, memory_order_relaxed)) {
2048	atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed);	2170	atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed);
2049	if (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed)) {	2171	if (atomic_load(&thr->in_blocking_func, memory_order_relaxed)) {
2050	atomic_store(&sctx->in_blocking_func, 0, memory_order_relaxed);	2172	atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed);
2051	CallUserSignalHandler(thr, sync, true, sigact, sig, info, ctx);	2173	CallUserSignalHandler(thr, sync, true, sig, info, ctx);
2052	atomic_store(&sctx->in_blocking_func, 1, memory_order_relaxed);	2174	atomic_store(&thr->in_blocking_func, 1, memory_order_relaxed);
2053	} else {	2175	} else {
2054	// Be very conservative with when we do acquire in this case.	2176	// Be very conservative with when we do acquire in this case.
2055	// It's unsafe to do acquire in async handlers, because ThreadState	2177	// It's unsafe to do acquire in async handlers, because ThreadState
...	@@ -2057,7 +2179,7 @@ void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,	...	@@ -2057,7 +2179,7 @@ void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,
2057	// SIGSYS looks relatively safe -- it's synchronous and can actually	2179	// SIGSYS looks relatively safe -- it's synchronous and can actually
2058	// need some global state.	2180	// need some global state.
2059	bool acq = (sig == SIGSYS);	2181	bool acq = (sig == SIGSYS);
2060	CallUserSignalHandler(thr, sync, acq, sigact, sig, info, ctx);	2182	CallUserSignalHandler(thr, sync, acq, sig, info, ctx);
2061	}	2183	}
2062	atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed);	2184	atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed);
2063	return;	2185	return;
...	@@ -2068,23 +2190,12 @@ void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,	...	@@ -2068,23 +2190,12 @@ void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,
2068	SignalDesc *signal = &sctx->pending_signals[sig];	2190	SignalDesc *signal = &sctx->pending_signals[sig];
2069	if (signal->armed == false) {	2191	if (signal->armed == false) {
2070	signal->armed = true;	2192	signal->armed = true;
2071	signal->sigaction = sigact;	2193	internal_memcpy(&signal->siginfo, info, sizeof(*info));
2072	if (info)	2194	internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx));
2073	internal_memcpy(&signal->siginfo, info, sizeof(*info));	2195	atomic_store(&thr->pending_signals, 1, memory_order_relaxed);
2074	if (ctx)
2075	internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx));
2076	atomic_store(&sctx->have_pending_signals, 1, memory_order_relaxed);
2077	}	2196	}
2078	}	2197	}
2079		2198
2080	static void rtl_sighandler(int sig) {
2081	rtl_generic_sighandler(false, sig, 0, 0);
2082	}
2083
2084	static void rtl_sigaction(int sig, __sanitizer_siginfo info, void ctx) {
2085	rtl_generic_sighandler(true, sig, info, ctx);
2086	}
2087
2088	TSAN_INTERCEPTOR(int, raise, int sig) {	2199	TSAN_INTERCEPTOR(int, raise, int sig) {
2089	SCOPED_TSAN_INTERCEPTOR(raise, sig);	2200	SCOPED_TSAN_INTERCEPTOR(raise, sig);
2090	ThreadSignalContext *sctx = SigCtx(thr);	2201	ThreadSignalContext *sctx = SigCtx(thr);
...	@@ -2118,11 +2229,11 @@ TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) {	...	@@ -2118,11 +2229,11 @@ TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) {
2118	ThreadSignalContext *sctx = SigCtx(thr);	2229	ThreadSignalContext *sctx = SigCtx(thr);
2119	CHECK_NE(sctx, 0);	2230	CHECK_NE(sctx, 0);
2120	int prev = sctx->int_signal_send;	2231	int prev = sctx->int_signal_send;
2121	if (tid == pthread_self()) {	2232	bool self = pthread_equal(tid, pthread_self());
		2233	if (self)
2122	sctx->int_signal_send = sig;	2234	sctx->int_signal_send = sig;
2123	}
2124	int res = REAL(pthread_kill)(tid, sig);	2235	int res = REAL(pthread_kill)(tid, sig);
2125	if (tid == pthread_self()) {	2236	if (self) {
2126	CHECK_EQ(sctx->int_signal_send, sig);	2237	CHECK_EQ(sctx->int_signal_send, sig);
2127	sctx->int_signal_send = prev;	2238	sctx->int_signal_send = prev;
2128	}	2239	}
...	@@ -2143,7 +2254,7 @@ TSAN_INTERCEPTOR(int, getaddrinfo, void node, void service,	...	@@ -2143,7 +2254,7 @@ TSAN_INTERCEPTOR(int, getaddrinfo, void node, void service,
2143	// inside of getaddrinfo. So ignore memory accesses.	2254	// inside of getaddrinfo. So ignore memory accesses.
2144	ThreadIgnoreBegin(thr, pc);	2255	ThreadIgnoreBegin(thr, pc);
2145	int res = REAL(getaddrinfo)(node, service, hints, rv);	2256	int res = REAL(getaddrinfo)(node, service, hints, rv);
2146	ThreadIgnoreEnd(thr, pc);	2257	ThreadIgnoreEnd(thr);
2147	return res;	2258	return res;
2148	}	2259	}
2149		2260
...	@@ -2175,10 +2286,11 @@ void atfork_child() {	...	@@ -2175,10 +2286,11 @@ void atfork_child() {
2175	return;	2286	return;
2176	ThreadState *thr = cur_thread();	2287	ThreadState *thr = cur_thread();
2177	const uptr pc = StackTrace::GetCurrentPc();	2288	const uptr pc = StackTrace::GetCurrentPc();
2178	ForkChildAfter(thr, pc);	2289	ForkChildAfter(thr, pc, true);
2179	FdOnFork(thr, pc);	2290	FdOnFork(thr, pc);
2180	}	2291	}
2181		2292
		2293	#if !SANITIZER_IOS
2182	TSAN_INTERCEPTOR(int, vfork, int fake) {	2294	TSAN_INTERCEPTOR(int, vfork, int fake) {
2183	// Some programs (e.g. openjdk) call close for all file descriptors	2295	// Some programs (e.g. openjdk) call close for all file descriptors
2184	// in the child process. Under tsan it leads to false positives, because	2296	// in the child process. Under tsan it leads to false positives, because
...	@@ -2195,8 +2307,40 @@ TSAN_INTERCEPTOR(int, vfork, int fake) {	...	@@ -2195,8 +2307,40 @@ TSAN_INTERCEPTOR(int, vfork, int fake) {
2195	// Instead we simply turn vfork into fork.	2307	// Instead we simply turn vfork into fork.
2196	return WRAP(fork)(fake);	2308	return WRAP(fork)(fake);
2197	}	2309	}
		2310	#endif
		2311
		2312	#if SANITIZER_LINUX
		2313	TSAN_INTERCEPTOR(int, clone, int (fn)(void ), void *stack, int flags,
		2314	void arg, int parent_tid, void tls, pid_t child_tid) {
		2315	SCOPED_INTERCEPTOR_RAW(clone, fn, stack, flags, arg, parent_tid, tls,
		2316	child_tid);
		2317	struct Arg {
		2318	int (fn)(void );
		2319	void *arg;
		2320	};
		2321	auto wrapper = +[](void *p) -> int {
		2322	auto *thr = cur_thread();
		2323	uptr pc = GET_CURRENT_PC();
		2324	// Start the background thread for fork, but not for clone.
		2325	// For fork we did this always and it's known to work (or user code has
		2326	// adopted). But if we do this for the new clone interceptor some code
		2327	// (sandbox2) fails. So model we used to do for years and don't start the
		2328	// background thread after clone.
		2329	ForkChildAfter(thr, pc, false);
		2330	FdOnFork(thr, pc);
		2331	auto arg = static_cast<Arg >(p);
		2332	return arg->fn(arg->arg);
		2333	};
		2334	ForkBefore(thr, pc);
		2335	Arg arg_wrapper = {fn, arg};
		2336	int pid = REAL(clone)(wrapper, stack, flags, &arg_wrapper, parent_tid, tls,
		2337	child_tid);
		2338	ForkParentAfter(thr, pc);
		2339	return pid;
		2340	}
		2341	#endif
2198		2342
2199	#if !SANITIZER_MAC && !SANITIZER_ANDROID	2343	#if !SANITIZER_APPLE && !SANITIZER_ANDROID
2200	typedef int (dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info info, SIZE_T size,	2344	typedef int (dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info info, SIZE_T size,
2201	void *data);	2345	void *data);
2202	struct dl_iterate_phdr_data {	2346	struct dl_iterate_phdr_data {
...	@@ -2207,7 +2351,7 @@ struct dl_iterate_phdr_data {	...	@@ -2207,7 +2351,7 @@ struct dl_iterate_phdr_data {
2207	};	2351	};
2208		2352
2209	static bool IsAppNotRodata(uptr addr) {	2353	static bool IsAppNotRodata(uptr addr) {
2210	return IsAppMem(addr) && (u64)MemToShadow(addr) != kShadowRodata;	2354	return IsAppMem(addr) && *MemToShadow(addr) != Shadow::kRodata;
2211	}	2355	}
2212		2356
2213	static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size,	2357	static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size,
...	@@ -2248,13 +2392,7 @@ static int OnExit(ThreadState *thr) {	...	@@ -2248,13 +2392,7 @@ static int OnExit(ThreadState *thr) {
2248	return status;	2392	return status;
2249	}	2393	}
2250		2394
2251	struct TsanInterceptorContext {	2395	#if !SANITIZER_APPLE
2252	ThreadState *thr;
2253	const uptr caller_pc;
2254	const uptr pc;
2255	};
2256
2257	#if !SANITIZER_MAC
2258	static void HandleRecvmsg(ThreadState *thr, uptr pc,	2396	static void HandleRecvmsg(ThreadState *thr, uptr pc,
2259	__sanitizer_msghdr *msg) {	2397	__sanitizer_msghdr *msg) {
2260	int fds[64];	2398	int fds[64];
...	@@ -2275,33 +2413,16 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,	...	@@ -2275,33 +2413,16 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
2275	#define SANITIZER_INTERCEPT_TLS_GET_OFFSET 1	2413	#define SANITIZER_INTERCEPT_TLS_GET_OFFSET 1
2276	#undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK	2414	#undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK
2277		2415
2278	#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
2279	#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \	2416	#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
2280	INTERCEPT_FUNCTION_VER(name, ver)	2417	INTERCEPT_FUNCTION_VER(name, ver)
2281	#define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \	2418	#define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \
2282	(INTERCEPT_FUNCTION_VER(name, ver) \|\| INTERCEPT_FUNCTION(name))	2419	(INTERCEPT_FUNCTION_VER(name, ver) \|\| INTERCEPT_FUNCTION(name))
2283		2420
2284	#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
2285	MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \
2286	((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
2287	true)
2288
2289	#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
2290	MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \
2291	((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
2292	false)
2293
2294	#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
2295	SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \
2296	TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \
2297	ctx = (void *)&_ctx; \
2298	(void) ctx;
2299
2300	#define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \	2421	#define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \
2301	SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \	2422	SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
2302	TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \	2423	TsanInterceptorContext _ctx = {thr, pc}; \
2303	ctx = (void *)&_ctx; \	2424	ctx = (void *)&_ctx; \
2304	(void) ctx;	2425	(void)ctx;
2305		2426
2306	#define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \	2427	#define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \
2307	if (path) \	2428	if (path) \
...	@@ -2314,14 +2435,33 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,	...	@@ -2314,14 +2435,33 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
2314	#define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \	2435	#define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \
2315	if (file) { \	2436	if (file) { \
2316	int fd = fileno_unlocked(file); \	2437	int fd = fileno_unlocked(file); \
2317	if (fd >= 0) FdClose(thr, pc, fd); \	2438	FdClose(thr, pc, fd); \
2318	}	2439	}
2319		2440
		2441	#define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \
		2442	({ \
		2443	CheckNoDeepBind(filename, flag); \
		2444	ThreadIgnoreBegin(thr, 0); \
		2445	void *res = REAL(dlopen)(filename, flag); \
		2446	ThreadIgnoreEnd(thr); \
		2447	res; \
		2448	})
		2449
		2450	// Ignore interceptors in OnLibraryLoaded()/Unloaded(). These hooks use code
		2451	// (ListOfModules::init, MemoryMappingLayout::DumpListOfModules) that make
		2452	// intercepted calls, which can cause deadlockes with ReportRace() which also
		2453	// uses this code.
2320	#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \	2454	#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \
2321	libignore()->OnLibraryLoaded(filename)	2455	({ \
		2456	ScopedIgnoreInterceptors ignore_interceptors; \
		2457	libignore()->OnLibraryLoaded(filename); \
		2458	})
2322		2459
2323	#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \	2460	#define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \
2324	libignore()->OnLibraryUnloaded()	2461	({ \
		2462	ScopedIgnoreInterceptors ignore_interceptors; \
		2463	libignore()->OnLibraryUnloaded(); \
		2464	})
2325		2465
2326	#define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \	2466	#define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \
2327	Acquire(((TsanInterceptorContext *) ctx)->thr, pc, u)	2467	Acquire(((TsanInterceptorContext *) ctx)->thr, pc, u)
...	@@ -2347,34 +2487,17 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,	...	@@ -2347,34 +2487,17 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
2347	#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \	2487	#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
2348	ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name)	2488	ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name)
2349		2489
2350	#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \	2490	#define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
2351	__tsan::ctx->thread_registry->SetThreadNameByUserId(thread, name)	2491	if (pthread_equal(pthread_self(), reinterpret_cast<void *>(thread))) \
		2492	COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name); \
		2493	else \
		2494	__tsan::ctx->thread_registry.SetThreadNameByUserId(thread, name)
2352		2495
2353	#define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name)	2496	#define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name)
2354		2497
2355	#define COMMON_INTERCEPTOR_ON_EXIT(ctx) \	2498	#define COMMON_INTERCEPTOR_ON_EXIT(ctx) \
2356	OnExit(((TsanInterceptorContext *) ctx)->thr)	2499	OnExit(((TsanInterceptorContext *) ctx)->thr)
2357		2500
2358	#define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) \
2359	MutexPreLock(((TsanInterceptorContext *)ctx)->thr, \
2360	((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2361
2362	#define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) \
2363	MutexPostLock(((TsanInterceptorContext *)ctx)->thr, \
2364	((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2365
2366	#define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \
2367	MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \
2368	((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2369
2370	#define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \
2371	MutexRepair(((TsanInterceptorContext *)ctx)->thr, \
2372	((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2373
2374	#define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) \
2375	MutexInvalidAccess(((TsanInterceptorContext *)ctx)->thr, \
2376	((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2377
2378	#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \	2501	#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \
2379	off) \	2502	off) \
2380	do { \	2503	do { \
...	@@ -2382,7 +2505,12 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,	...	@@ -2382,7 +2505,12 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
2382	off); \	2505	off); \
2383	} while (false)	2506	} while (false)
2384		2507
2385	#if !SANITIZER_MAC	2508	#define COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz) \
		2509	do { \
		2510	return munmap_interceptor(thr, pc, REAL(munmap), addr, sz); \
		2511	} while (false)
		2512
		2513	#if !SANITIZER_APPLE
2386	#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \	2514	#define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \
2387	HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \	2515	HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \
2388	((TsanInterceptorContext *)ctx)->pc, msg)	2516	((TsanInterceptorContext *)ctx)->pc, msg)
...	@@ -2415,12 +2543,14 @@ static __sanitizer_sighandler_ptr signal_impl(int sig,	...	@@ -2415,12 +2543,14 @@ static __sanitizer_sighandler_ptr signal_impl(int sig,
2415	#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \	2543	#define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \
2416	{ return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); }	2544	{ return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); }
2417		2545
		2546	#define SIGNAL_INTERCEPTOR_ENTER() LazyInitialize(cur_thread_init())
		2547
2418	#include "sanitizer_common/sanitizer_signal_interceptors.inc"	2548	#include "sanitizer_common/sanitizer_signal_interceptors.inc"
2419		2549
2420	int sigaction_impl(int sig, const __sanitizer_sigaction *act,	2550	int sigaction_impl(int sig, const __sanitizer_sigaction *act,
2421	__sanitizer_sigaction *old) {	2551	__sanitizer_sigaction *old) {
2422	// Note: if we call REAL(sigaction) directly for any reason without proxying	2552	// Note: if we call REAL(sigaction) directly for any reason without proxying
2423	// the signal handler through rtl_sigaction, very bad things will happen.	2553	// the signal handler through sighandler, very bad things will happen.
2424	// The handler will run synchronously and corrupt tsan per-thread state.	2554	// The handler will run synchronously and corrupt tsan per-thread state.
2425	SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old);	2555	SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old);
2426	if (sig <= 0 \|\| sig >= kSigCount) {	2556	if (sig <= 0 \|\| sig >= kSigCount) {
...	@@ -2443,27 +2573,22 @@ int sigaction_impl(int sig, const __sanitizer_sigaction *act,	...	@@ -2443,27 +2573,22 @@ int sigaction_impl(int sig, const __sanitizer_sigaction *act,
2443	sigactions[sig].sa_flags = (volatile int const )&act->sa_flags;	2573	sigactions[sig].sa_flags = (volatile int const )&act->sa_flags;
2444	internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask,	2574	internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask,
2445	sizeof(sigactions[sig].sa_mask));	2575	sizeof(sigactions[sig].sa_mask));
2446	#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD	2576	#if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD
2447	sigactions[sig].sa_restorer = act->sa_restorer;	2577	sigactions[sig].sa_restorer = act->sa_restorer;
2448	#endif	2578	#endif
2449	internal_memcpy(&newact, act, sizeof(newact));	2579	internal_memcpy(&newact, act, sizeof(newact));
2450	internal_sigfillset(&newact.sa_mask);	2580	internal_sigfillset(&newact.sa_mask);
2451	if ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl) {	2581	if ((act->sa_flags & SA_SIGINFO) \|\|
2452	if (newact.sa_flags & SA_SIGINFO)	2582	((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl)) {
2453	newact.sigaction = rtl_sigaction;	2583	newact.sa_flags \|= SA_SIGINFO;
2454	else	2584	newact.sigaction = sighandler;
2455	newact.handler = rtl_sighandler;
2456	}	2585	}
2457	ReleaseStore(thr, pc, (uptr)&sigactions[sig]);	2586	ReleaseStore(thr, pc, (uptr)&sigactions[sig]);
2458	act = &newact;	2587	act = &newact;
2459	}	2588	}
2460	int res = REAL(sigaction)(sig, act, old);	2589	int res = REAL(sigaction)(sig, act, old);
2461	if (res == 0 && old) {	2590	if (res == 0 && old && old->sigaction == sighandler)
2462	uptr cb = (uptr)old->sigaction;	2591	internal_memcpy(old, &old_stored, sizeof(*old));
2463	if (cb == (uptr)rtl_sigaction \|\| cb == (uptr)rtl_sighandler) {
2464	internal_memcpy(old, &old_stored, sizeof(*old));
2465	}
2466	}
2467	return res;	2592	return res;
2468	}	2593	}
2469		2594
...	@@ -2479,27 +2604,23 @@ static __sanitizer_sighandler_ptr signal_impl(int sig,	...	@@ -2479,27 +2604,23 @@ static __sanitizer_sighandler_ptr signal_impl(int sig,
2479	return old.handler;	2604	return old.handler;
2480	}	2605	}
2481		2606
2482	#define TSAN_SYSCALL() \	2607	#define TSAN_SYSCALL() \
2483	ThreadState *thr = cur_thread(); \	2608	ThreadState *thr = cur_thread(); \
2484	if (thr->ignore_interceptors) \	2609	if (thr->ignore_interceptors) \
2485	return; \	2610	return; \
2486	ScopedSyscall scoped_syscall(thr) \	2611	ScopedSyscall scoped_syscall(thr)
2487	/**/
2488		2612
2489	struct ScopedSyscall {	2613	struct ScopedSyscall {
2490	ThreadState *thr;	2614	ThreadState *thr;
2491		2615
2492	explicit ScopedSyscall(ThreadState *thr)	2616	explicit ScopedSyscall(ThreadState *thr) : thr(thr) { LazyInitialize(thr); }
2493	: thr(thr) {
2494	Initialize(thr);
2495	}
2496		2617
2497	~ScopedSyscall() {	2618	~ScopedSyscall() {
2498	ProcessPendingSignals(thr);	2619	ProcessPendingSignals(thr);
2499	}	2620	}
2500	};	2621	};
2501		2622
2502	#if !SANITIZER_FREEBSD && !SANITIZER_MAC	2623	#if !SANITIZER_FREEBSD && !SANITIZER_APPLE
2503	static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {	2624	static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {
2504	TSAN_SYSCALL();	2625	TSAN_SYSCALL();
2505	MemoryAccessRange(thr, pc, p, s, write);	2626	MemoryAccessRange(thr, pc, p, s, write);
...	@@ -2508,29 +2629,29 @@ static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {	...	@@ -2508,29 +2629,29 @@ static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {
2508	static USED void syscall_acquire(uptr pc, uptr addr) {	2629	static USED void syscall_acquire(uptr pc, uptr addr) {
2509	TSAN_SYSCALL();	2630	TSAN_SYSCALL();
2510	Acquire(thr, pc, addr);	2631	Acquire(thr, pc, addr);
2511	DPrintf("syscall_acquire(%p)\n", addr);	2632	DPrintf("syscall_acquire(0x%zx))\n", addr);
2512	}	2633	}
2513		2634
2514	static USED void syscall_release(uptr pc, uptr addr) {	2635	static USED void syscall_release(uptr pc, uptr addr) {
2515	TSAN_SYSCALL();	2636	TSAN_SYSCALL();
2516	DPrintf("syscall_release(%p)\n", addr);	2637	DPrintf("syscall_release(0x%zx)\n", addr);
2517	Release(thr, pc, addr);	2638	Release(thr, pc, addr);
2518	}	2639	}
2519		2640
2520	static void syscall_fd_close(uptr pc, int fd) {	2641	static void syscall_fd_close(uptr pc, int fd) {
2521	TSAN_SYSCALL();	2642	auto *thr = cur_thread();
2522	FdClose(thr, pc, fd);	2643	FdClose(thr, pc, fd);
2523	}	2644	}
2524		2645
2525	static USED void syscall_fd_acquire(uptr pc, int fd) {	2646	static USED void syscall_fd_acquire(uptr pc, int fd) {
2526	TSAN_SYSCALL();	2647	TSAN_SYSCALL();
2527	FdAcquire(thr, pc, fd);	2648	FdAcquire(thr, pc, fd);
2528	DPrintf("syscall_fd_acquire(%p)\n", fd);	2649	DPrintf("syscall_fd_acquire(%d)\n", fd);
2529	}	2650	}
2530		2651
2531	static USED void syscall_fd_release(uptr pc, int fd) {	2652	static USED void syscall_fd_release(uptr pc, int fd) {
2532	TSAN_SYSCALL();	2653	TSAN_SYSCALL();
2533	DPrintf("syscall_fd_release(%p)\n", fd);	2654	DPrintf("syscall_fd_release(%d)\n", fd);
2534	FdRelease(thr, pc, fd);	2655	FdRelease(thr, pc, fd);
2535	}	2656	}
2536		2657
...	@@ -2540,7 +2661,7 @@ static void syscall_post_fork(uptr pc, int pid) {	...	@@ -2540,7 +2661,7 @@ static void syscall_post_fork(uptr pc, int pid) {
2540	ThreadState *thr = cur_thread();	2661	ThreadState *thr = cur_thread();
2541	if (pid == 0) {	2662	if (pid == 0) {
2542	// child	2663	// child
2543	ForkChildAfter(thr, pc);	2664	ForkChildAfter(thr, pc, true);
2544	FdOnFork(thr, pc);	2665	FdOnFork(thr, pc);
2545	} else if (pid > 0) {	2666	} else if (pid > 0) {
2546	// parent	2667	// parent
...	@@ -2653,6 +2774,26 @@ TSAN_INTERCEPTOR(void, thr_exit, tid_t *state) {	...	@@ -2653,6 +2774,26 @@ TSAN_INTERCEPTOR(void, thr_exit, tid_t *state) {
2653	#define TSAN_MAYBE_INTERCEPT_THR_EXIT	2774	#define TSAN_MAYBE_INTERCEPT_THR_EXIT
2654	#endif	2775	#endif
2655		2776
		2777	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_init, void c, void a)
		2778	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_destroy, void *c)
		2779	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_signal, void *c)
		2780	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_broadcast, void *c)
		2781	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_wait, void c, void m)
		2782	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_init, void m, void a)
		2783	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_destroy, void *m)
		2784	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_lock, void *m)
		2785	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_trylock, void *m)
		2786	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_unlock, void *m)
		2787	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_init, void l, void a)
		2788	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_destroy, void *l)
		2789	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_rdlock, void *l)
		2790	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_tryrdlock, void *l)
		2791	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_wrlock, void *l)
		2792	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_trywrlock, void *l)
		2793	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_unlock, void *l)
		2794	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, once, void o, void (i)())
		2795	TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, sigmask, int f, void n, void o)
		2796
2656	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init, void c, void a)	2797	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init, void c, void a)
2657	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal, void *c)	2798	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal, void *c)
2658	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast, void *c)	2799	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast, void *c)
...	@@ -2660,7 +2801,9 @@ TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_wait, void c, void m)	...	@@ -2660,7 +2801,9 @@ TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_wait, void c, void m)
2660	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy, void *c)	2801	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy, void *c)
2661	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init, void m, void a)	2802	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init, void m, void a)
2662	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy, void *m)	2803	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy, void *m)
		2804	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_lock, void *m)
2663	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock, void *m)	2805	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock, void *m)
		2806	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_unlock, void *m)
2664	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init, void m, void a)	2807	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init, void m, void a)
2665	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy, void *m)	2808	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy, void *m)
2666	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock, void *m)	2809	TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock, void *m)
...	@@ -2683,7 +2826,7 @@ static void finalize(void *arg) {	...	@@ -2683,7 +2826,7 @@ static void finalize(void *arg) {
2683	Die();	2826	Die();
2684	}	2827	}
2685		2828
2686	#if !SANITIZER_MAC && !SANITIZER_ANDROID	2829	#if !SANITIZER_APPLE && !SANITIZER_ANDROID
2687	static void unreachable() {	2830	static void unreachable() {
2688	Report("FATAL: ThreadSanitizer: unreachable called\n");	2831	Report("FATAL: ThreadSanitizer: unreachable called\n");
2689	Die();	2832	Die();
...	@@ -2694,35 +2837,20 @@ static void unreachable() {	...	@@ -2694,35 +2837,20 @@ static void unreachable() {
2694	SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {}	2837	SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {}
2695		2838
2696	void InitializeInterceptors() {	2839	void InitializeInterceptors() {
2697	#if !SANITIZER_MAC	2840	#if !SANITIZER_APPLE
2698	// We need to setup it early, because functions like dlsym() can call it.	2841	// We need to setup it early, because functions like dlsym() can call it.
2699	REAL(memset) = internal_memset;	2842	REAL(memset) = internal_memset;
2700	REAL(memcpy) = internal_memcpy;	2843	REAL(memcpy) = internal_memcpy;
2701	#endif	2844	#endif
2702		2845
2703	// Instruct libc malloc to consume less memory.
2704	#if SANITIZER_GLIBC
2705	mallopt(1, 0); // M_MXFAST
2706	mallopt(-3, 32*1024); // M_MMAP_THRESHOLD
2707	#endif
2708
2709	new(interceptor_ctx()) InterceptorContext();	2846	new(interceptor_ctx()) InterceptorContext();
2710		2847
2711	InitializeCommonInterceptors();	2848	InitializeCommonInterceptors();
2712	InitializeSignalInterceptors();	2849	InitializeSignalInterceptors();
2713	InitializeLibdispatchInterceptors();	2850	InitializeLibdispatchInterceptors();
2714		2851
2715	#if !SANITIZER_MAC	2852	#if !SANITIZER_APPLE
2716	// We can not use TSAN_INTERCEPT to get setjmp addr,	2853	InitializeSetjmpInterceptors();
2717	// because it does &setjmp and setjmp is not present in some versions of libc.
2718	using __interception::InterceptFunction;
2719	InterceptFunction(TSAN_STRING_SETJMP, (uptr*)&REAL(setjmp_symname), 0, 0);
2720	InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0);
2721	InterceptFunction(TSAN_STRING_SIGSETJMP, (uptr*)&REAL(sigsetjmp_symname), 0,
2722	0);
2723	#if !SANITIZER_NETBSD
2724	InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0);
2725	#endif
2726	#endif	2854	#endif
2727		2855
2728	TSAN_INTERCEPT(longjmp_symname);	2856	TSAN_INTERCEPT(longjmp_symname);
...	@@ -2768,8 +2896,16 @@ void InitializeInterceptors() {	...	@@ -2768,8 +2896,16 @@ void InitializeInterceptors() {
2768		2896
2769	TSAN_INTERCEPT(pthread_mutex_init);	2897	TSAN_INTERCEPT(pthread_mutex_init);
2770	TSAN_INTERCEPT(pthread_mutex_destroy);	2898	TSAN_INTERCEPT(pthread_mutex_destroy);
		2899	TSAN_INTERCEPT(pthread_mutex_lock);
2771	TSAN_INTERCEPT(pthread_mutex_trylock);	2900	TSAN_INTERCEPT(pthread_mutex_trylock);
2772	TSAN_INTERCEPT(pthread_mutex_timedlock);	2901	TSAN_INTERCEPT(pthread_mutex_timedlock);
		2902	TSAN_INTERCEPT(pthread_mutex_unlock);
		2903	#if SANITIZER_GLIBC
		2904	# if !__GLIBC_PREREQ(2, 34)
		2905	TSAN_INTERCEPT(__pthread_mutex_lock);
		2906	TSAN_INTERCEPT(__pthread_mutex_unlock);
		2907	# endif
		2908	#endif
2773		2909
2774	TSAN_INTERCEPT(pthread_spin_init);	2910	TSAN_INTERCEPT(pthread_spin_init);
2775	TSAN_INTERCEPT(pthread_spin_destroy);	2911	TSAN_INTERCEPT(pthread_spin_destroy);
...	@@ -2843,6 +2979,9 @@ void InitializeInterceptors() {	...	@@ -2843,6 +2979,9 @@ void InitializeInterceptors() {
2843		2979
2844	TSAN_INTERCEPT(fork);	2980	TSAN_INTERCEPT(fork);
2845	TSAN_INTERCEPT(vfork);	2981	TSAN_INTERCEPT(vfork);
		2982	#if SANITIZER_LINUX
		2983	TSAN_INTERCEPT(clone);
		2984	#endif
2846	#if !SANITIZER_ANDROID	2985	#if !SANITIZER_ANDROID
2847	TSAN_INTERCEPT(dl_iterate_phdr);	2986	TSAN_INTERCEPT(dl_iterate_phdr);
2848	#endif	2987	#endif
...	@@ -2862,7 +3001,7 @@ void InitializeInterceptors() {	...	@@ -2862,7 +3001,7 @@ void InitializeInterceptors() {
2862	TSAN_MAYBE_INTERCEPT__LWP_EXIT;	3001	TSAN_MAYBE_INTERCEPT__LWP_EXIT;
2863	TSAN_MAYBE_INTERCEPT_THR_EXIT;	3002	TSAN_MAYBE_INTERCEPT_THR_EXIT;
2864		3003
2865	#if !SANITIZER_MAC && !SANITIZER_ANDROID	3004	#if !SANITIZER_APPLE && !SANITIZER_ANDROID
2866	// Need to setup it, because interceptors check that the function is resolved.	3005	// Need to setup it, because interceptors check that the function is resolved.
2867	// But atexit is emitted directly into the module, so can't be resolved.	3006	// But atexit is emitted directly into the module, so can't be resolved.
2868	REAL(atexit) = (int()(void()()))unreachable;	3007	REAL(atexit) = (int()(void()()))unreachable;
...	@@ -2877,13 +3016,33 @@ void InitializeInterceptors() {	...	@@ -2877,13 +3016,33 @@ void InitializeInterceptors() {
2877	Die();	3016	Die();
2878	}	3017	}
2879		3018
2880	#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD	3019	#if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
2881	if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) {	3020	if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) {
2882	Printf("ThreadSanitizer: failed to create thread key\n");	3021	Printf("ThreadSanitizer: failed to create thread key\n");
2883	Die();	3022	Die();
2884	}	3023	}
2885	#endif	3024	#endif
2886		3025
		3026	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_init);
		3027	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_destroy);
		3028	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_signal);
		3029	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_broadcast);
		3030	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_wait);
		3031	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_init);
		3032	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_destroy);
		3033	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_lock);
		3034	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_trylock);
		3035	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_unlock);
		3036	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_init);
		3037	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_destroy);
		3038	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_rdlock);
		3039	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_tryrdlock);
		3040	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_wrlock);
		3041	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_trywrlock);
		3042	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_unlock);
		3043	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(once);
		3044	TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(sigmask);
		3045
2887	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init);	3046	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init);
2888	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal);	3047	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal);
2889	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast);	3048	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast);
...	@@ -2891,7 +3050,9 @@ void InitializeInterceptors() {	...	@@ -2891,7 +3050,9 @@ void InitializeInterceptors() {
2891	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy);	3050	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy);
2892	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init);	3051	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init);
2893	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy);	3052	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy);
		3053	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_lock);
2894	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock);	3054	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock);
		3055	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_unlock);
2895	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init);	3056	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init);
2896	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy);	3057	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy);
2897	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock);	3058	TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock);
...	@@ -2920,25 +3081,40 @@ void InitializeInterceptors() {	...	@@ -2920,25 +3081,40 @@ void InitializeInterceptors() {
2920	// Note that no_sanitize_thread attribute does not turn off atomic interception	3081	// Note that no_sanitize_thread attribute does not turn off atomic interception
2921	// so attaching it to the function defined in user code does not help.	3082	// so attaching it to the function defined in user code does not help.
2922	// That's why we now have what we have.	3083	// That's why we now have what we have.
2923	extern "C" SANITIZER_INTERFACE_ATTRIBUTE	3084	constexpr u32 kBarrierThreadBits = 10;
2924	void __tsan_testonly_barrier_init(u64 *barrier, u32 count) {	3085	constexpr u32 kBarrierThreads = 1 << kBarrierThreadBits;
2925	if (count >= (1 << 8)) {	3086
2926	Printf("barrier_init: count is too large (%d)\n", count);	3087	extern "C" {
2927	Die();	3088
		3089	SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_init(
		3090	atomic_uint32_t *barrier, u32 num_threads) {
		3091	if (num_threads >= kBarrierThreads) {
		3092	Printf("barrier_init: count is too large (%d)\n", num_threads);
		3093	Die();
2928	}	3094	}
2929	// 8 lsb is thread count, the remaining are count of entered threads.	3095	// kBarrierThreadBits lsb is thread count,
2930	*barrier = count;	3096	// the remaining are count of entered threads.
		3097	atomic_store(barrier, num_threads, memory_order_relaxed);
		3098	}
		3099
		3100	static u32 barrier_epoch(u32 value) {
		3101	return (value >> kBarrierThreadBits) / (value & (kBarrierThreads - 1));
2931	}	3102	}
2932		3103
2933	extern "C" SANITIZER_INTERFACE_ATTRIBUTE	3104	SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait(
2934	void __tsan_testonly_barrier_wait(u64 *barrier) {	3105	atomic_uint32_t *barrier) {
2935	unsigned old = __atomic_fetch_add(barrier, 1 << 8, __ATOMIC_RELAXED);	3106	u32 old = atomic_fetch_add(barrier, kBarrierThreads, memory_order_relaxed);
2936	unsigned old_epoch = (old >> 8) / (old & 0xff);	3107	u32 old_epoch = barrier_epoch(old);
		3108	if (barrier_epoch(old + kBarrierThreads) != old_epoch) {
		3109	FutexWake(barrier, (1 << 30));
		3110	return;
		3111	}
2937	for (;;) {	3112	for (;;) {
2938	unsigned cur = __atomic_load_n(barrier, __ATOMIC_RELAXED);	3113	u32 cur = atomic_load(barrier, memory_order_relaxed);
2939	unsigned cur_epoch = (cur >> 8) / (cur & 0xff);	3114	if (barrier_epoch(cur) != old_epoch)
2940	if (cur_epoch != old_epoch)
2941	return;	3115	return;
2942	internal_sched_yield();	3116	FutexWait(barrier, cur);
2943	}	3117	}
2944	}	3118	}
		3119
		3120	} // extern "C"

lib/tsan/tsan_interface.cpp+17-82

...	@@ -20,108 +20,43 @@	...	@@ -20,108 +20,43 @@
20		20
21	using namespace __tsan;	21	using namespace __tsan;
22		22
23	void __tsan_init() {	23	void __tsan_init() { Initialize(cur_thread_init()); }
24	cur_thread_init();
25	Initialize(cur_thread());
26	}
27		24
28	void __tsan_flush_memory() {	25	void __tsan_flush_memory() {
29	FlushShadowMemory();	26	FlushShadowMemory();
30	}	27	}
31		28
32	void __tsan_read16(void *addr) {
33	MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
34	MemoryRead(cur_thread(), CALLERPC, (uptr)addr + 8, kSizeLog8);
35	}
36
37	void __tsan_write16(void *addr) {
38	MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
39	MemoryWrite(cur_thread(), CALLERPC, (uptr)addr + 8, kSizeLog8);
40	}
41
42	void __tsan_read16_pc(void addr, void pc) {	29	void __tsan_read16_pc(void addr, void pc) {
43	MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog8);	30	uptr pc_no_pac = STRIP_PAC_PC(pc);
44	MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr + 8, kSizeLog8);	31	ThreadState *thr = cur_thread();
		32	MemoryAccess(thr, pc_no_pac, (uptr)addr, 8, kAccessRead);
		33	MemoryAccess(thr, pc_no_pac, (uptr)addr + 8, 8, kAccessRead);
45	}	34	}
46		35
47	void __tsan_write16_pc(void addr, void pc) {	36	void __tsan_write16_pc(void addr, void pc) {
48	MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog8);	37	uptr pc_no_pac = STRIP_PAC_PC(pc);
49	MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr + 8, kSizeLog8);	38	ThreadState *thr = cur_thread();
		39	MemoryAccess(thr, pc_no_pac, (uptr)addr, 8, kAccessWrite);
		40	MemoryAccess(thr, pc_no_pac, (uptr)addr + 8, 8, kAccessWrite);
50	}	41	}
51		42
52	// __tsan_unaligned_read/write calls are emitted by compiler.	43	// __tsan_unaligned_read/write calls are emitted by compiler.
53		44
54	void __tsan_unaligned_read2(const void *addr) {
55	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, false, false);
56	}
57
58	void __tsan_unaligned_read4(const void *addr) {
59	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, false, false);
60	}
61
62	void __tsan_unaligned_read8(const void *addr) {
63	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, false, false);
64	}
65
66	void __tsan_unaligned_read16(const void *addr) {	45	void __tsan_unaligned_read16(const void *addr) {
67	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 16, false, false);	46	uptr pc = CALLERPC;
68	}	47	ThreadState *thr = cur_thread();
69		48	UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessRead);
70	void __tsan_unaligned_write2(void *addr) {	49	UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessRead);
71	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, true, false);
72	}
73
74	void __tsan_unaligned_write4(void *addr) {
75	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, true, false);
76	}
77
78	void __tsan_unaligned_write8(void *addr) {
79	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, true, false);
80	}	50	}
81		51
82	void __tsan_unaligned_write16(void *addr) {	52	void __tsan_unaligned_write16(void *addr) {
83	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 16, true, false);	53	uptr pc = CALLERPC;
		54	ThreadState *thr = cur_thread();
		55	UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessWrite);
		56	UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessWrite);
84	}	57	}
85		58
86	// __sanitizer_unaligned_load/store are for user instrumentation.
87
88	extern "C" {	59	extern "C" {
89	SANITIZER_INTERFACE_ATTRIBUTE
90	u16 __sanitizer_unaligned_load16(const uu16 *addr) {
91	__tsan_unaligned_read2(addr);
92	return *addr;
93	}
94
95	SANITIZER_INTERFACE_ATTRIBUTE
96	u32 __sanitizer_unaligned_load32(const uu32 *addr) {
97	__tsan_unaligned_read4(addr);
98	return *addr;
99	}
100
101	SANITIZER_INTERFACE_ATTRIBUTE
102	u64 __sanitizer_unaligned_load64(const uu64 *addr) {
103	__tsan_unaligned_read8(addr);
104	return *addr;
105	}
106
107	SANITIZER_INTERFACE_ATTRIBUTE
108	void __sanitizer_unaligned_store16(uu16 *addr, u16 v) {
109	__tsan_unaligned_write2(addr);
110	*addr = v;
111	}
112
113	SANITIZER_INTERFACE_ATTRIBUTE
114	void __sanitizer_unaligned_store32(uu32 *addr, u32 v) {
115	__tsan_unaligned_write4(addr);
116	*addr = v;
117	}
118
119	SANITIZER_INTERFACE_ATTRIBUTE
120	void __sanitizer_unaligned_store64(uu64 *addr, u64 v) {
121	__tsan_unaligned_write8(addr);
122	*addr = v;
123	}
124
125	SANITIZER_INTERFACE_ATTRIBUTE	60	SANITIZER_INTERFACE_ATTRIBUTE
126	void *__tsan_get_current_fiber() {	61	void *__tsan_get_current_fiber() {
127	return cur_thread();	62	return cur_thread();

lib/tsan/tsan_interface.h+14-8

 @@ -32,6 +32,9 @@ extern "C" {
 // before any instrumented code is executed and before any call to malloc.
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_init();
+SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char *
+__tsan_default_options();
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_flush_memory();
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read1(void *addr);
 @@ -72,12 +75,21 @@ SANITIZER_INTERFACE_ATTRIBUTE void __tsan_vptr_read(void **vptr_p);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __tsan_vptr_update(void **vptr_p, void *new_val);
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_memcpy(void *dest, const void *src, uptr count);
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_memset(void *dest, int ch, uptr count);
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_memmove(void *dest, const void *src, uptr count);
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_entry(void *call_pc);
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_func_exit();
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_ignore_thread_begin();
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_ignore_thread_end();
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_on_thread_idle();
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__tsan_external_register_tag(const char *object_type);
 SANITIZER_INTERFACE_ATTRIBUTE
 @@ -95,9 +107,9 @@ SANITIZER_INTERFACE_ATTRIBUTE
 void __tsan_write_range(void *addr, unsigned long size);
 SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_read_range_pc(void *addr, unsigned long size, void *pc);  // NOLINT
+void __tsan_read_range_pc(void *addr, unsigned long size, void *pc);
 SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_write_range_pc(void *addr, unsigned long size, void *pc);  // NOLINT
+void __tsan_write_range_pc(void *addr, unsigned long size, void *pc);
 // User may provide function that would be called right when TSan detects
 // an error. The argument 'report' is an opaque pointer that can be used to
 @@ -417,12 +429,6 @@ SANITIZER_INTERFACE_ATTRIBUTE
 void __tsan_go_atomic64_compare_exchange(ThreadState *thr, uptr cpc, uptr pc,
                                          u8 *a);
-SANITIZER_INTERFACE_ATTRIBUTE
-void __tsan_on_initialize();
-SANITIZER_INTERFACE_ATTRIBUTE
-int __tsan_on_finalize(int failed);
 }  // extern "C"
 }  // namespace __tsan

lib/tsan/tsan_interface.inc created+190

...	@@ -0,0 +1,190 @@
1	//===-- tsan_interface.inc --------------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "sanitizer_common/sanitizer_ptrauth.h"
14	#include "tsan_interface.h"
15	#include "tsan_rtl.h"
16
17	#define CALLERPC ((uptr)__builtin_return_address(0))
18
19	using namespace __tsan;
20
21	void __tsan_read1(void *addr) {
22	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessRead);
23	}
24
25	void __tsan_read2(void *addr) {
26	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessRead);
27	}
28
29	void __tsan_read4(void *addr) {
30	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessRead);
31	}
32
33	void __tsan_read8(void *addr) {
34	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead);
35	}
36
37	void __tsan_read16(void *addr) {
38	MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessRead);
39	}
40
41	void __tsan_write1(void *addr) {
42	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessWrite);
43	}
44
45	void __tsan_write2(void *addr) {
46	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessWrite);
47	}
48
49	void __tsan_write4(void *addr) {
50	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessWrite);
51	}
52
53	void __tsan_write8(void *addr) {
54	MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite);
55	}
56
57	void __tsan_write16(void *addr) {
58	MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessWrite);
59	}
60
61	void __tsan_read1_pc(void addr, void pc) {
62	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessRead \| kAccessExternalPC);
63	}
64
65	void __tsan_read2_pc(void addr, void pc) {
66	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 2, kAccessRead \| kAccessExternalPC);
67	}
68
69	void __tsan_read4_pc(void addr, void pc) {
70	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 4, kAccessRead \| kAccessExternalPC);
71	}
72
73	void __tsan_read8_pc(void addr, void pc) {
74	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 8, kAccessRead \| kAccessExternalPC);
75	}
76
77	void __tsan_write1_pc(void addr, void pc) {
78	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessWrite \| kAccessExternalPC);
79	}
80
81	void __tsan_write2_pc(void addr, void pc) {
82	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 2, kAccessWrite \| kAccessExternalPC);
83	}
84
85	void __tsan_write4_pc(void addr, void pc) {
86	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 4, kAccessWrite \| kAccessExternalPC);
87	}
88
89	void __tsan_write8_pc(void addr, void pc) {
90	MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 8, kAccessWrite \| kAccessExternalPC);
91	}
92
93	ALWAYS_INLINE USED void __tsan_unaligned_read2(const void *addr) {
94	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessRead);
95	}
96
97	ALWAYS_INLINE USED void __tsan_unaligned_read4(const void *addr) {
98	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessRead);
99	}
100
101	ALWAYS_INLINE USED void __tsan_unaligned_read8(const void *addr) {
102	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessRead);
103	}
104
105	ALWAYS_INLINE USED void __tsan_unaligned_write2(void *addr) {
106	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, kAccessWrite);
107	}
108
109	ALWAYS_INLINE USED void __tsan_unaligned_write4(void *addr) {
110	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, kAccessWrite);
111	}
112
113	ALWAYS_INLINE USED void __tsan_unaligned_write8(void *addr) {
114	UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, kAccessWrite);
115	}
116
117	extern "C" {
118	// __sanitizer_unaligned_load/store are for user instrumentation.
119	SANITIZER_INTERFACE_ATTRIBUTE
120	u16 __sanitizer_unaligned_load16(const uu16 *addr) {
121	__tsan_unaligned_read2(addr);
122	return *addr;
123	}
124
125	SANITIZER_INTERFACE_ATTRIBUTE
126	u32 __sanitizer_unaligned_load32(const uu32 *addr) {
127	__tsan_unaligned_read4(addr);
128	return *addr;
129	}
130
131	SANITIZER_INTERFACE_ATTRIBUTE
132	u64 __sanitizer_unaligned_load64(const uu64 *addr) {
133	__tsan_unaligned_read8(addr);
134	return *addr;
135	}
136
137	SANITIZER_INTERFACE_ATTRIBUTE
138	void __sanitizer_unaligned_store16(uu16 *addr, u16 v) {
139	*addr = v;
140	__tsan_unaligned_write2(addr);
141	}
142
143	SANITIZER_INTERFACE_ATTRIBUTE
144	void __sanitizer_unaligned_store32(uu32 *addr, u32 v) {
145	*addr = v;
146	__tsan_unaligned_write4(addr);
147	}
148
149	SANITIZER_INTERFACE_ATTRIBUTE
150	void __sanitizer_unaligned_store64(uu64 *addr, u64 v) {
151	*addr = v;
152	__tsan_unaligned_write8(addr);
153	}
154	}
155
156	void __tsan_vptr_update(void *vptr_p, void new_val) {
157	if (*vptr_p == new_val)
158	return;
159	MemoryAccess(cur_thread(), CALLERPC, (uptr)vptr_p, sizeof(*vptr_p),
160	kAccessWrite \| kAccessVptr);
161	}
162
163	void __tsan_vptr_read(void **vptr_p) {
164	MemoryAccess(cur_thread(), CALLERPC, (uptr)vptr_p, sizeof(*vptr_p),
165	kAccessRead \| kAccessVptr);
166	}
167
168	void __tsan_func_entry(void *pc) { FuncEntry(cur_thread(), STRIP_PAC_PC(pc)); }
169
170	void __tsan_func_exit() { FuncExit(cur_thread()); }
171
172	void __tsan_ignore_thread_begin() { ThreadIgnoreBegin(cur_thread(), CALLERPC); }
173
174	void __tsan_ignore_thread_end() { ThreadIgnoreEnd(cur_thread()); }
175
176	void __tsan_read_range(void *addr, uptr size) {
177	MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, false);
178	}
179
180	void __tsan_write_range(void *addr, uptr size) {
181	MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, true);
182	}
183
184	void __tsan_read_range_pc(void addr, uptr size, void pc) {
185	MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, false);
186	}
187
188	void __tsan_write_range_pc(void addr, uptr size, void pc) {
189	MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, true);
190	}

lib/tsan/tsan_interface_ann.cpp+30-139

 @@ -43,15 +43,14 @@ class ScopedAnnotation {
   ThreadState *const thr_;
 };
-#define SCOPED_ANNOTATION_RET(typ, ret) \
+#define SCOPED_ANNOTATION_RET(typ, ret)                     \
-    if (!flags()->enable_annotations) \
+  if (!flags()->enable_annotations)                         \
-      return ret; \
+    return ret;                                             \
-    ThreadState *thr = cur_thread(); \
+  ThreadState *thr = cur_thread();                          \
-    const uptr caller_pc = (uptr)__builtin_return_address(0); \
+  const uptr caller_pc = (uptr)__builtin_return_address(0); \
-    ScopedAnnotation sa(thr, __func__, caller_pc); \
+  ScopedAnnotation sa(thr, __func__, caller_pc);            \
-    const uptr pc = StackTrace::GetCurrentPc(); \
+  const uptr pc = StackTrace::GetCurrentPc();               \
-    (void)pc; \
+  (void)pc;
-/**/
 #define SCOPED_ANNOTATION(typ) SCOPED_ANNOTATION_RET(typ, )
 @@ -71,7 +70,6 @@ struct ExpectRace {
 struct DynamicAnnContext {
   Mutex mtx;
-  ExpectRace expect;
   ExpectRace benign;
   DynamicAnnContext() : mtx(MutexTypeAnnotations) {}
 @@ -90,7 +88,7 @@ static void AddExpectRace(ExpectRace *list,
       return;
     }
   }
-  race = (ExpectRace*)internal_alloc(MBlockExpectRace, sizeof(ExpectRace));
+  race = static_cast<ExpectRace *>(Alloc(sizeof(ExpectRace)));
   race->addr = addr;
   race->size = size;
   race->file = f;
 @@ -137,81 +135,12 @@ static void InitList(ExpectRace *list) {
 void InitializeDynamicAnnotations() {
   dyn_ann_ctx = new(dyn_ann_ctx_placeholder) DynamicAnnContext;
-  InitList(&dyn_ann_ctx->expect);
   InitList(&dyn_ann_ctx->benign);
 }
 bool IsExpectedReport(uptr addr, uptr size) {
   ReadLock lock(&dyn_ann_ctx->mtx);
-  if (CheckContains(&dyn_ann_ctx->expect, addr, size))
+  return CheckContains(&dyn_ann_ctx->benign, addr, size);
-    return true;
-  if (CheckContains(&dyn_ann_ctx->benign, addr, size))
-    return true;
-  return false;
-}
-static void CollectMatchedBenignRaces(Vector<ExpectRace> *matched,
-    int *unique_count, int *hit_count, atomic_uintptr_t ExpectRace::*counter) {
-  ExpectRace *list = &dyn_ann_ctx->benign;
-  for (ExpectRace *race = list->next; race != list; race = race->next) {
-    (*unique_count)++;
-    const uptr cnt = atomic_load_relaxed(&(race->*counter));
-    if (cnt == 0)
-      continue;
-    *hit_count += cnt;
-    uptr i = 0;
-    for (; i < matched->Size(); i++) {
-      ExpectRace *race0 = &(*matched)[i];
-      if (race->line == race0->line
-          && internal_strcmp(race->file, race0->file) == 0
-          && internal_strcmp(race->desc, race0->desc) == 0) {
-        atomic_fetch_add(&(race0->*counter), cnt, memory_order_relaxed);
-        break;
-      }
-    }
-    if (i == matched->Size())
-      matched->PushBack(*race);
-  }
-}
-void PrintMatchedBenignRaces() {
-  Lock lock(&dyn_ann_ctx->mtx);
-  int unique_count = 0;
-  int hit_count = 0;
-  int add_count = 0;
-  Vector<ExpectRace> hit_matched;
-  CollectMatchedBenignRaces(&hit_matched, &unique_count, &hit_count,
-      &ExpectRace::hitcount);
-  Vector<ExpectRace> add_matched;
-  CollectMatchedBenignRaces(&add_matched, &unique_count, &add_count,
-      &ExpectRace::addcount);
-  if (hit_matched.Size()) {
-    Printf("ThreadSanitizer: Matched %d \"benign\" races (pid=%d):\n",
-        hit_count, (int)internal_getpid());
-    for (uptr i = 0; i < hit_matched.Size(); i++) {
-      Printf("%d %s:%d %s\n",
-          atomic_load_relaxed(&hit_matched[i].hitcount),
-          hit_matched[i].file, hit_matched[i].line, hit_matched[i].desc);
-    }
-  }
-  if (hit_matched.Size()) {
-    Printf("ThreadSanitizer: Annotated %d \"benign\" races, %d unique"
-           " (pid=%d):\n",
-        add_count, unique_count, (int)internal_getpid());
-    for (uptr i = 0; i < add_matched.Size(); i++) {
-      Printf("%d %s:%d %s\n",
-          atomic_load_relaxed(&add_matched[i].addcount),
-          add_matched[i].file, add_matched[i].line, add_matched[i].desc);
-    }
-  }
-}
-static void ReportMissedExpectedRace(ExpectRace *race) {
-  Printf("==================\n");
-  Printf("WARNING: ThreadSanitizer: missed expected data race\n");
-  Printf("  %s addr=%zx %s:%d\n",
-      race->desc, race->addr, race->file, race->line);
-  Printf("==================\n");
 }
 }  // namespace __tsan
 @@ -229,20 +158,16 @@ void INTERFACE_ATTRIBUTE AnnotateHappensAfter(char *f, int l, uptr addr) {
 }
 void INTERFACE_ATTRIBUTE AnnotateCondVarSignal(char *f, int l, uptr cv) {
-  SCOPED_ANNOTATION(AnnotateCondVarSignal);
 }
 void INTERFACE_ATTRIBUTE AnnotateCondVarSignalAll(char *f, int l, uptr cv) {
-  SCOPED_ANNOTATION(AnnotateCondVarSignalAll);
 }
 void INTERFACE_ATTRIBUTE AnnotateMutexIsNotPHB(char *f, int l, uptr mu) {
-  SCOPED_ANNOTATION(AnnotateMutexIsNotPHB);
 }
 void INTERFACE_ATTRIBUTE AnnotateCondVarWait(char *f, int l, uptr cv,
                                              uptr lock) {
-  SCOPED_ANNOTATION(AnnotateCondVarWait);
 }
 void INTERFACE_ATTRIBUTE AnnotateRWLockCreate(char *f, int l, uptr m) {
 @@ -279,86 +204,56 @@ void INTERFACE_ATTRIBUTE AnnotateRWLockReleased(char *f, int l, uptr m,
 }
 void INTERFACE_ATTRIBUTE AnnotateTraceMemory(char *f, int l, uptr mem) {
-  SCOPED_ANNOTATION(AnnotateTraceMemory);
 }
 void INTERFACE_ATTRIBUTE AnnotateFlushState(char *f, int l) {
-  SCOPED_ANNOTATION(AnnotateFlushState);
 }
 void INTERFACE_ATTRIBUTE AnnotateNewMemory(char *f, int l, uptr mem,
                                            uptr size) {
-  SCOPED_ANNOTATION(AnnotateNewMemory);
 }
 void INTERFACE_ATTRIBUTE AnnotateNoOp(char *f, int l, uptr mem) {
-  SCOPED_ANNOTATION(AnnotateNoOp);
 }
 void INTERFACE_ATTRIBUTE AnnotateFlushExpectedRaces(char *f, int l) {
-  SCOPED_ANNOTATION(AnnotateFlushExpectedRaces);
-  Lock lock(&dyn_ann_ctx->mtx);
-  while (dyn_ann_ctx->expect.next != &dyn_ann_ctx->expect) {
-    ExpectRace *race = dyn_ann_ctx->expect.next;
-    if (atomic_load_relaxed(&race->hitcount) == 0) {
-      ctx->nmissed_expected++;
-      ReportMissedExpectedRace(race);
-    }
-    race->prev->next = race->next;
-    race->next->prev = race->prev;
-    internal_free(race);
-  }
 }
 void INTERFACE_ATTRIBUTE AnnotateEnableRaceDetection(
     char *f, int l, int enable) {
-  SCOPED_ANNOTATION(AnnotateEnableRaceDetection);
-  // FIXME: Reconsider this functionality later. It may be irrelevant.
 }
 void INTERFACE_ATTRIBUTE AnnotateMutexIsUsedAsCondVar(
     char *f, int l, uptr mu) {
-  SCOPED_ANNOTATION(AnnotateMutexIsUsedAsCondVar);
 }
 void INTERFACE_ATTRIBUTE AnnotatePCQGet(
     char *f, int l, uptr pcq) {
-  SCOPED_ANNOTATION(AnnotatePCQGet);
 }
 void INTERFACE_ATTRIBUTE AnnotatePCQPut(
     char *f, int l, uptr pcq) {
-  SCOPED_ANNOTATION(AnnotatePCQPut);
 }
 void INTERFACE_ATTRIBUTE AnnotatePCQDestroy(
     char *f, int l, uptr pcq) {
-  SCOPED_ANNOTATION(AnnotatePCQDestroy);
 }
 void INTERFACE_ATTRIBUTE AnnotatePCQCreate(
     char *f, int l, uptr pcq) {
-  SCOPED_ANNOTATION(AnnotatePCQCreate);
 }
 void INTERFACE_ATTRIBUTE AnnotateExpectRace(
     char *f, int l, uptr mem, char *desc) {
-  SCOPED_ANNOTATION(AnnotateExpectRace);
-  Lock lock(&dyn_ann_ctx->mtx);
-  AddExpectRace(&dyn_ann_ctx->expect,
-                f, l, mem, 1, desc);
-  DPrintf("Add expected race: %s addr=%zx %s:%d\n", desc, mem, f, l);
 }
-static void BenignRaceImpl(
+static void BenignRaceImpl(char *f, int l, uptr mem, uptr size, char *desc) {
-    char *f, int l, uptr mem, uptr size, char *desc) {
   Lock lock(&dyn_ann_ctx->mtx);
   AddExpectRace(&dyn_ann_ctx->benign,
                 f, l, mem, size, desc);
   DPrintf("Add benign race: %s addr=%zx %s:%d\n", desc, mem, f, l);
 }
-// FIXME: Turn it off later. WTF is benign race?1?? Go talk to Hans Boehm.
 void INTERFACE_ATTRIBUTE AnnotateBenignRaceSized(
     char *f, int l, uptr mem, uptr size, char *desc) {
   SCOPED_ANNOTATION(AnnotateBenignRaceSized);
 @@ -378,7 +273,7 @@ void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsBegin(char *f, int l) {
 void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsEnd(char *f, int l) {
   SCOPED_ANNOTATION(AnnotateIgnoreReadsEnd);
-  ThreadIgnoreEnd(thr, pc);
+  ThreadIgnoreEnd(thr);
 }
 void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesBegin(char *f, int l) {
 @@ -388,7 +283,7 @@ void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesBegin(char *f, int l) {
 void INTERFACE_ATTRIBUTE AnnotateIgnoreWritesEnd(char *f, int l) {
   SCOPED_ANNOTATION(AnnotateIgnoreWritesEnd);
-  ThreadIgnoreEnd(thr, pc);
+  ThreadIgnoreEnd(thr);
 }
 void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncBegin(char *f, int l) {
 @@ -398,17 +293,15 @@ void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncBegin(char *f, int l) {
 void INTERFACE_ATTRIBUTE AnnotateIgnoreSyncEnd(char *f, int l) {
   SCOPED_ANNOTATION(AnnotateIgnoreSyncEnd);
-  ThreadIgnoreSyncEnd(thr, pc);
+  ThreadIgnoreSyncEnd(thr);
 }
 void INTERFACE_ATTRIBUTE AnnotatePublishMemoryRange(
     char *f, int l, uptr addr, uptr size) {
-  SCOPED_ANNOTATION(AnnotatePublishMemoryRange);
 }
 void INTERFACE_ATTRIBUTE AnnotateUnpublishMemoryRange(
     char *f, int l, uptr addr, uptr size) {
-  SCOPED_ANNOTATION(AnnotateUnpublishMemoryRange);
 }
 void INTERFACE_ATTRIBUTE AnnotateThreadName(
 @@ -421,11 +314,9 @@ void INTERFACE_ATTRIBUTE AnnotateThreadName(
 // WTFAnnotateHappensAfter(). Those are being used by Webkit to annotate
 // atomic operations, which should be handled by ThreadSanitizer correctly.
 void INTERFACE_ATTRIBUTE WTFAnnotateHappensBefore(char *f, int l, uptr addr) {
-  SCOPED_ANNOTATION(AnnotateHappensBefore);
 }
 void INTERFACE_ATTRIBUTE WTFAnnotateHappensAfter(char *f, int l, uptr addr) {
-  SCOPED_ANNOTATION(AnnotateHappensAfter);
 }
 void INTERFACE_ATTRIBUTE WTFAnnotateBenignRaceSized(
 @@ -477,15 +368,15 @@ void __tsan_mutex_pre_lock(void *m, unsigned flagz) {
     else
       MutexPreLock(thr, pc, (uptr)m);
   }
-  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreBegin(thr, 0);
-  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, 0);
 }
 INTERFACE_ATTRIBUTE
 void __tsan_mutex_post_lock(void *m, unsigned flagz, int rec) {
   SCOPED_ANNOTATION(__tsan_mutex_post_lock);
-  ThreadIgnoreSyncEnd(thr, pc);
+  ThreadIgnoreSyncEnd(thr);
-  ThreadIgnoreEnd(thr, pc);
+  ThreadIgnoreEnd(thr);
   if (!(flagz & MutexFlagTryLockFailed)) {
     if (flagz & MutexFlagReadLock)
       MutexPostReadLock(thr, pc, (uptr)m, flagz);
 @@ -504,44 +395,44 @@ int __tsan_mutex_pre_unlock(void *m, unsigned flagz) {
   } else {
     ret = MutexUnlock(thr, pc, (uptr)m, flagz);
   }
-  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreBegin(thr, 0);
-  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, 0);
   return ret;
 }
 INTERFACE_ATTRIBUTE
 void __tsan_mutex_post_unlock(void *m, unsigned flagz) {
   SCOPED_ANNOTATION(__tsan_mutex_post_unlock);
-  ThreadIgnoreSyncEnd(thr, pc);
+  ThreadIgnoreSyncEnd(thr);
-  ThreadIgnoreEnd(thr, pc);
+  ThreadIgnoreEnd(thr);
 }
 INTERFACE_ATTRIBUTE
 void __tsan_mutex_pre_signal(void *addr, unsigned flagz) {
   SCOPED_ANNOTATION(__tsan_mutex_pre_signal);
-  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreBegin(thr, 0);
-  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, 0);
 }
 INTERFACE_ATTRIBUTE
 void __tsan_mutex_post_signal(void *addr, unsigned flagz) {
   SCOPED_ANNOTATION(__tsan_mutex_post_signal);
-  ThreadIgnoreSyncEnd(thr, pc);
+  ThreadIgnoreSyncEnd(thr);
-  ThreadIgnoreEnd(thr, pc);
+  ThreadIgnoreEnd(thr);
 }
 INTERFACE_ATTRIBUTE
 void __tsan_mutex_pre_divert(void *addr, unsigned flagz) {
   SCOPED_ANNOTATION(__tsan_mutex_pre_divert);
   // Exit from ignore region started in __tsan_mutex_pre_lock/unlock/signal.
-  ThreadIgnoreSyncEnd(thr, pc);
+  ThreadIgnoreSyncEnd(thr);
-  ThreadIgnoreEnd(thr, pc);
+  ThreadIgnoreEnd(thr);
 }
 INTERFACE_ATTRIBUTE
 void __tsan_mutex_post_divert(void *addr, unsigned flagz) {
   SCOPED_ANNOTATION(__tsan_mutex_post_divert);
-  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreBegin(thr, 0);
-  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, 0);
 }
 }  // extern "C"

lib/tsan/tsan_interface_atomic.cpp+155-181

...	@@ -32,6 +32,7 @@ using namespace __tsan;	...	@@ -32,6 +32,7 @@ using namespace __tsan;
32	static StaticSpinMutex mutex128;	32	static StaticSpinMutex mutex128;
33	#endif	33	#endif
34		34
		35	#if SANITIZER_DEBUG
35	static bool IsLoadOrder(morder mo) {	36	static bool IsLoadOrder(morder mo) {
36	return mo == mo_relaxed \|\| mo == mo_consume	37	return mo == mo_relaxed \|\| mo == mo_consume
37	\|\| mo == mo_acquire \|\| mo == mo_seq_cst;	38	\|\| mo == mo_acquire \|\| mo == mo_seq_cst;
...	@@ -40,6 +41,7 @@ static bool IsLoadOrder(morder mo) {	...	@@ -40,6 +41,7 @@ static bool IsLoadOrder(morder mo) {
40	static bool IsStoreOrder(morder mo) {	41	static bool IsStoreOrder(morder mo) {
41	return mo == mo_relaxed \|\| mo == mo_release \|\| mo == mo_seq_cst;	42	return mo == mo_relaxed \|\| mo == mo_release \|\| mo == mo_seq_cst;
42	}	43	}
		44	#endif
43		45
44	static bool IsReleaseOrder(morder mo) {	46	static bool IsReleaseOrder(morder mo) {
45	return mo == mo_release \|\| mo == mo_acq_rel \|\| mo == mo_seq_cst;	47	return mo == mo_release \|\| mo == mo_acq_rel \|\| mo == mo_seq_cst;
...	@@ -161,16 +163,16 @@ a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {	...	@@ -161,16 +163,16 @@ a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {
161	}	163	}
162	#endif	164	#endif
163		165
164	template<typename T>	166	template <typename T>
165	static int SizeLog() {	167	static int AccessSize() {
166	if (sizeof(T) <= 1)	168	if (sizeof(T) <= 1)
167	return kSizeLog1;	169	return 1;
168	else if (sizeof(T) <= 2)	170	else if (sizeof(T) <= 2)
169	return kSizeLog2;	171	return 2;
170	else if (sizeof(T) <= 4)	172	else if (sizeof(T) <= 4)
171	return kSizeLog4;	173	return 4;
172	else	174	else
173	return kSizeLog8;	175	return 8;
174	// For 16-byte atomics we also use 8-byte memory access,	176	// For 16-byte atomics we also use 8-byte memory access,
175	// this leads to false negatives only in very obscure cases.	177	// this leads to false negatives only in very obscure cases.
176	}	178	}
...	@@ -202,7 +204,7 @@ static memory_order to_mo(morder mo) {	...	@@ -202,7 +204,7 @@ static memory_order to_mo(morder mo) {
202	case mo_acq_rel: return memory_order_acq_rel;	204	case mo_acq_rel: return memory_order_acq_rel;
203	case mo_seq_cst: return memory_order_seq_cst;	205	case mo_seq_cst: return memory_order_seq_cst;
204	}	206	}
205	CHECK(0);	207	DCHECK(0);
206	return memory_order_seq_cst;	208	return memory_order_seq_cst;
207	}	209	}
208		210
...	@@ -219,27 +221,28 @@ static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {	...	@@ -219,27 +221,28 @@ static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
219	#endif	221	#endif
220		222
221	template <typename T>	223	template <typename T>
222	static T AtomicLoad(ThreadState thr, uptr pc, const volatile T a,	224	static T AtomicLoad(ThreadState thr, uptr pc, const volatile T a, morder mo) {
223	morder mo) NO_THREAD_SAFETY_ANALYSIS {	225	DCHECK(IsLoadOrder(mo));
224	CHECK(IsLoadOrder(mo));
225	// This fast-path is critical for performance.	226	// This fast-path is critical for performance.
226	// Assume the access is atomic.	227	// Assume the access is atomic.
227	if (!IsAcquireOrder(mo)) {	228	if (!IsAcquireOrder(mo)) {
228	MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());	229	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(),
		230	kAccessRead \| kAccessAtomic);
229	return NoTsanAtomicLoad(a, mo);	231	return NoTsanAtomicLoad(a, mo);
230	}	232	}
231	// Don't create sync object if it does not exist yet. For example, an atomic	233	// Don't create sync object if it does not exist yet. For example, an atomic
232	// pointer is initialized to nullptr and then periodically acquire-loaded.	234	// pointer is initialized to nullptr and then periodically acquire-loaded.
233	T v = NoTsanAtomicLoad(a, mo);	235	T v = NoTsanAtomicLoad(a, mo);
234	SyncVar *s = ctx->metamap.GetIfExistsAndLock((uptr)a, false);	236	SyncVar *s = ctx->metamap.GetSyncIfExists((uptr)a);
235	if (s) {	237	if (s) {
236	AcquireImpl(thr, pc, &s->clock);	238	SlotLocker locker(thr);
		239	ReadLock lock(&s->mtx);
		240	thr->clock.Acquire(s->clock);
237	// Re-read under sync mutex because we need a consistent snapshot	241	// Re-read under sync mutex because we need a consistent snapshot
238	// of the value and the clock we acquire.	242	// of the value and the clock we acquire.
239	v = NoTsanAtomicLoad(a, mo);	243	v = NoTsanAtomicLoad(a, mo);
240	s->mtx.ReadUnlock();
241	}	244	}
242	MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());	245	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessRead \| kAccessAtomic);
243	return v;	246	return v;
244	}	247	}
245		248
...	@@ -257,9 +260,9 @@ static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) {	...	@@ -257,9 +260,9 @@ static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) {
257		260
258	template <typename T>	261	template <typename T>
259	static void AtomicStore(ThreadState thr, uptr pc, volatile T a, T v,	262	static void AtomicStore(ThreadState thr, uptr pc, volatile T a, T v,
260	morder mo) NO_THREAD_SAFETY_ANALYSIS {	263	morder mo) {
261	CHECK(IsStoreOrder(mo));	264	DCHECK(IsStoreOrder(mo));
262	MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());	265	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite \| kAccessAtomic);
263	// This fast-path is critical for performance.	266	// This fast-path is critical for performance.
264	// Assume the access is atomic.	267	// Assume the access is atomic.
265	// Strictly saying even relaxed store cuts off release sequence,	268	// Strictly saying even relaxed store cuts off release sequence,
...	@@ -268,36 +271,35 @@ static void AtomicStore(ThreadState thr, uptr pc, volatile T a, T v,	...	@@ -268,36 +271,35 @@ static void AtomicStore(ThreadState thr, uptr pc, volatile T a, T v,
268	NoTsanAtomicStore(a, v, mo);	271	NoTsanAtomicStore(a, v, mo);
269	return;	272	return;
270	}	273	}
271	__sync_synchronize();	274	SlotLocker locker(thr);
272	SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);	275	{
273	thr->fast_state.IncrementEpoch();	276	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
274	// Can't increment epoch w/o writing to the trace as well.	277	Lock lock(&s->mtx);
275	TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);	278	thr->clock.ReleaseStore(&s->clock);
276	ReleaseStoreImpl(thr, pc, &s->clock);	279	NoTsanAtomicStore(a, v, mo);
277	NoTsanAtomicStore(a, v, mo);	280	}
278	s->mtx.Unlock();	281	IncrementEpoch(thr);
279	}	282	}
280		283
281	template <typename T, T (F)(volatile T v, T op)>	284	template <typename T, T (F)(volatile T v, T op)>
282	static T AtomicRMW(ThreadState thr, uptr pc, volatile T a, T v,	285	static T AtomicRMW(ThreadState thr, uptr pc, volatile T a, T v, morder mo) {
283	morder mo) NO_THREAD_SAFETY_ANALYSIS {	286	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite \| kAccessAtomic);
284	MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());	287	if (LIKELY(mo == mo_relaxed))
285	SyncVar *s = 0;	288	return F(a, v);
286	if (mo != mo_relaxed) {	289	SlotLocker locker(thr);
287	s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);	290	{
288	thr->fast_state.IncrementEpoch();	291	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
289	// Can't increment epoch w/o writing to the trace as well.	292	RWLock lock(&s->mtx, IsReleaseOrder(mo));
290	TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
291	if (IsAcqRelOrder(mo))	293	if (IsAcqRelOrder(mo))
292	AcquireReleaseImpl(thr, pc, &s->clock);	294	thr->clock.ReleaseAcquire(&s->clock);
293	else if (IsReleaseOrder(mo))	295	else if (IsReleaseOrder(mo))
294	ReleaseImpl(thr, pc, &s->clock);	296	thr->clock.Release(&s->clock);
295	else if (IsAcquireOrder(mo))	297	else if (IsAcquireOrder(mo))
296	AcquireImpl(thr, pc, &s->clock);	298	thr->clock.Acquire(s->clock);
		299	v = F(a, v);
297	}	300	}
298	v = F(a, v);	301	if (IsReleaseOrder(mo))
299	if (s)	302	IncrementEpoch(thr);
300	s->mtx.Unlock();
301	return v;	303	return v;
302	}	304	}
303		305
...	@@ -402,46 +404,44 @@ static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {	...	@@ -402,46 +404,44 @@ static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {
402	}	404	}
403		405
404	template <typename T>	406	template <typename T>
405	static bool AtomicCAS(ThreadState thr, uptr pc, volatile T a, T *c, T v, morder mo,	407	static bool AtomicCAS(ThreadState thr, uptr pc, volatile T a, T *c, T v,
406	morder fmo) NO_THREAD_SAFETY_ANALYSIS {	408	morder mo, morder fmo) {
407	// 31.7.2.18: "The failure argument shall not be memory_order_release	409	// 31.7.2.18: "The failure argument shall not be memory_order_release
408	// nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic	410	// nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic
409	// (mo_relaxed) when those are used.	411	// (mo_relaxed) when those are used.
410	CHECK(IsLoadOrder(fmo));	412	DCHECK(IsLoadOrder(fmo));
411		413
412	MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());	414	MemoryAccess(thr, pc, (uptr)a, AccessSize<T>(), kAccessWrite \| kAccessAtomic);
413	SyncVar *s = 0;	415	if (LIKELY(mo == mo_relaxed && fmo == mo_relaxed)) {
414	bool write_lock = IsReleaseOrder(mo);	416	T cc = *c;
415		417	T pr = func_cas(a, cc, v);
416	if (mo != mo_relaxed \|\| fmo != mo_relaxed)	418	if (pr == cc)
417	s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, write_lock);	419	return true;
418
419	T cc = *c;
420	T pr = func_cas(a, cc, v);
421	bool success = pr == cc;
422	if (!success) {
423	*c = pr;	420	*c = pr;
424	mo = fmo;	421	return false;
425	}	422	}
426		423	SlotLocker locker(thr);
427	if (s) {	424	bool release = IsReleaseOrder(mo);
428	thr->fast_state.IncrementEpoch();	425	bool success;
429	// Can't increment epoch w/o writing to the trace as well.	426	{
430	TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);	427	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, (uptr)a, false);
431		428	RWLock lock(&s->mtx, release);
		429	T cc = *c;
		430	T pr = func_cas(a, cc, v);
		431	success = pr == cc;
		432	if (!success) {
		433	*c = pr;
		434	mo = fmo;
		435	}
432	if (success && IsAcqRelOrder(mo))	436	if (success && IsAcqRelOrder(mo))
433	AcquireReleaseImpl(thr, pc, &s->clock);	437	thr->clock.ReleaseAcquire(&s->clock);
434	else if (success && IsReleaseOrder(mo))	438	else if (success && IsReleaseOrder(mo))
435	ReleaseImpl(thr, pc, &s->clock);	439	thr->clock.Release(&s->clock);
436	else if (IsAcquireOrder(mo))	440	else if (IsAcquireOrder(mo))
437	AcquireImpl(thr, pc, &s->clock);	441	thr->clock.Acquire(s->clock);
438
439	if (write_lock)
440	s->mtx.Unlock();
441	else
442	s->mtx.ReadUnlock();
443	}	442	}
444		443	if (success && release)
		444	IncrementEpoch(thr);
445	return success;	445	return success;
446	}	446	}
447		447
...	@@ -485,380 +485,356 @@ static morder convert_morder(morder mo) {	...	@@ -485,380 +485,356 @@ static morder convert_morder(morder mo) {
485	return (morder)(mo & 0x7fff);	485	return (morder)(mo & 0x7fff);
486	}	486	}
487		487
488	#define SCOPED_ATOMIC(func, ...) \	488	# define ATOMIC_IMPL(func, ...) \
489	ThreadState *const thr = cur_thread(); \	489	ThreadState *const thr = cur_thread(); \
490	if (UNLIKELY(thr->ignore_sync \|\| thr->ignore_interceptors)) { \	490	ProcessPendingSignals(thr); \
491	ProcessPendingSignals(thr); \	491	if (UNLIKELY(thr->ignore_sync \|\| thr->ignore_interceptors)) \
492	return NoTsanAtomic##func(__VA_ARGS__); \	492	return NoTsanAtomic##func(__VA_ARGS__); \
493	} \	493	mo = convert_morder(mo); \
494	const uptr callpc = (uptr)__builtin_return_address(0); \	494	return Atomic##func(thr, GET_CALLER_PC(), __VA_ARGS__);
495	uptr pc = StackTrace::GetCurrentPc(); \
496	mo = convert_morder(mo); \
497	ScopedAtomic sa(thr, callpc, a, mo, __func__); \
498	return Atomic##func(thr, pc, __VA_ARGS__); \
499	/**/
500
501	class ScopedAtomic {
502	public:
503	ScopedAtomic(ThreadState thr, uptr pc, const volatile void a,
504	morder mo, const char *func)
505	: thr_(thr) {
506	FuncEntry(thr_, pc);
507	DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo);
508	}
509	~ScopedAtomic() {
510	ProcessPendingSignals(thr_);
511	FuncExit(thr_);
512	}
513	private:
514	ThreadState *thr_;
515	};
516		495
517	extern "C" {	496	extern "C" {
518	SANITIZER_INTERFACE_ATTRIBUTE	497	SANITIZER_INTERFACE_ATTRIBUTE
519	a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {	498	a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
520	SCOPED_ATOMIC(Load, a, mo);	499	ATOMIC_IMPL(Load, a, mo);
521	}	500	}
522		501
523	SANITIZER_INTERFACE_ATTRIBUTE	502	SANITIZER_INTERFACE_ATTRIBUTE
524	a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {	503	a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {
525	SCOPED_ATOMIC(Load, a, mo);	504	ATOMIC_IMPL(Load, a, mo);
526	}	505	}
527		506
528	SANITIZER_INTERFACE_ATTRIBUTE	507	SANITIZER_INTERFACE_ATTRIBUTE
529	a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {	508	a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {
530	SCOPED_ATOMIC(Load, a, mo);	509	ATOMIC_IMPL(Load, a, mo);
531	}	510	}
532		511
533	SANITIZER_INTERFACE_ATTRIBUTE	512	SANITIZER_INTERFACE_ATTRIBUTE
534	a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {	513	a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {
535	SCOPED_ATOMIC(Load, a, mo);	514	ATOMIC_IMPL(Load, a, mo);
536	}	515	}
537		516
538	#if __TSAN_HAS_INT128	517	#if __TSAN_HAS_INT128
539	SANITIZER_INTERFACE_ATTRIBUTE	518	SANITIZER_INTERFACE_ATTRIBUTE
540	a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {	519	a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {
541	SCOPED_ATOMIC(Load, a, mo);	520	ATOMIC_IMPL(Load, a, mo);
542	}	521	}
543	#endif	522	#endif
544		523
545	SANITIZER_INTERFACE_ATTRIBUTE	524	SANITIZER_INTERFACE_ATTRIBUTE
546	void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {	525	void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {
547	SCOPED_ATOMIC(Store, a, v, mo);	526	ATOMIC_IMPL(Store, a, v, mo);
548	}	527	}
549		528
550	SANITIZER_INTERFACE_ATTRIBUTE	529	SANITIZER_INTERFACE_ATTRIBUTE
551	void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {	530	void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {
552	SCOPED_ATOMIC(Store, a, v, mo);	531	ATOMIC_IMPL(Store, a, v, mo);
553	}	532	}
554		533
555	SANITIZER_INTERFACE_ATTRIBUTE	534	SANITIZER_INTERFACE_ATTRIBUTE
556	void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {	535	void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {
557	SCOPED_ATOMIC(Store, a, v, mo);	536	ATOMIC_IMPL(Store, a, v, mo);
558	}	537	}
559		538
560	SANITIZER_INTERFACE_ATTRIBUTE	539	SANITIZER_INTERFACE_ATTRIBUTE
561	void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {	540	void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {
562	SCOPED_ATOMIC(Store, a, v, mo);	541	ATOMIC_IMPL(Store, a, v, mo);
563	}	542	}
564		543
565	#if __TSAN_HAS_INT128	544	#if __TSAN_HAS_INT128
566	SANITIZER_INTERFACE_ATTRIBUTE	545	SANITIZER_INTERFACE_ATTRIBUTE
567	void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {	546	void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {
568	SCOPED_ATOMIC(Store, a, v, mo);	547	ATOMIC_IMPL(Store, a, v, mo);
569	}	548	}
570	#endif	549	#endif
571		550
572	SANITIZER_INTERFACE_ATTRIBUTE	551	SANITIZER_INTERFACE_ATTRIBUTE
573	a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {	552	a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {
574	SCOPED_ATOMIC(Exchange, a, v, mo);	553	ATOMIC_IMPL(Exchange, a, v, mo);
575	}	554	}
576		555
577	SANITIZER_INTERFACE_ATTRIBUTE	556	SANITIZER_INTERFACE_ATTRIBUTE
578	a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {	557	a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {
579	SCOPED_ATOMIC(Exchange, a, v, mo);	558	ATOMIC_IMPL(Exchange, a, v, mo);
580	}	559	}
581		560
582	SANITIZER_INTERFACE_ATTRIBUTE	561	SANITIZER_INTERFACE_ATTRIBUTE
583	a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {	562	a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {
584	SCOPED_ATOMIC(Exchange, a, v, mo);	563	ATOMIC_IMPL(Exchange, a, v, mo);
585	}	564	}
586		565
587	SANITIZER_INTERFACE_ATTRIBUTE	566	SANITIZER_INTERFACE_ATTRIBUTE
588	a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {	567	a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {
589	SCOPED_ATOMIC(Exchange, a, v, mo);	568	ATOMIC_IMPL(Exchange, a, v, mo);
590	}	569	}
591		570
592	#if __TSAN_HAS_INT128	571	#if __TSAN_HAS_INT128
593	SANITIZER_INTERFACE_ATTRIBUTE	572	SANITIZER_INTERFACE_ATTRIBUTE
594	a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {	573	a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {
595	SCOPED_ATOMIC(Exchange, a, v, mo);	574	ATOMIC_IMPL(Exchange, a, v, mo);
596	}	575	}
597	#endif	576	#endif
598		577
599	SANITIZER_INTERFACE_ATTRIBUTE	578	SANITIZER_INTERFACE_ATTRIBUTE
600	a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {	579	a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {
601	SCOPED_ATOMIC(FetchAdd, a, v, mo);	580	ATOMIC_IMPL(FetchAdd, a, v, mo);
602	}	581	}
603		582
604	SANITIZER_INTERFACE_ATTRIBUTE	583	SANITIZER_INTERFACE_ATTRIBUTE
605	a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {	584	a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {
606	SCOPED_ATOMIC(FetchAdd, a, v, mo);	585	ATOMIC_IMPL(FetchAdd, a, v, mo);
607	}	586	}
608		587
609	SANITIZER_INTERFACE_ATTRIBUTE	588	SANITIZER_INTERFACE_ATTRIBUTE
610	a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {	589	a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {
611	SCOPED_ATOMIC(FetchAdd, a, v, mo);	590	ATOMIC_IMPL(FetchAdd, a, v, mo);
612	}	591	}
613		592
614	SANITIZER_INTERFACE_ATTRIBUTE	593	SANITIZER_INTERFACE_ATTRIBUTE
615	a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {	594	a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {
616	SCOPED_ATOMIC(FetchAdd, a, v, mo);	595	ATOMIC_IMPL(FetchAdd, a, v, mo);
617	}	596	}
618		597
619	#if __TSAN_HAS_INT128	598	#if __TSAN_HAS_INT128
620	SANITIZER_INTERFACE_ATTRIBUTE	599	SANITIZER_INTERFACE_ATTRIBUTE
621	a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {	600	a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {
622	SCOPED_ATOMIC(FetchAdd, a, v, mo);	601	ATOMIC_IMPL(FetchAdd, a, v, mo);
623	}	602	}
624	#endif	603	#endif
625		604
626	SANITIZER_INTERFACE_ATTRIBUTE	605	SANITIZER_INTERFACE_ATTRIBUTE
627	a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {	606	a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {
628	SCOPED_ATOMIC(FetchSub, a, v, mo);	607	ATOMIC_IMPL(FetchSub, a, v, mo);
629	}	608	}
630		609
631	SANITIZER_INTERFACE_ATTRIBUTE	610	SANITIZER_INTERFACE_ATTRIBUTE
632	a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {	611	a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {
633	SCOPED_ATOMIC(FetchSub, a, v, mo);	612	ATOMIC_IMPL(FetchSub, a, v, mo);
634	}	613	}
635		614
636	SANITIZER_INTERFACE_ATTRIBUTE	615	SANITIZER_INTERFACE_ATTRIBUTE
637	a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {	616	a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {
638	SCOPED_ATOMIC(FetchSub, a, v, mo);	617	ATOMIC_IMPL(FetchSub, a, v, mo);
639	}	618	}
640		619
641	SANITIZER_INTERFACE_ATTRIBUTE	620	SANITIZER_INTERFACE_ATTRIBUTE
642	a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {	621	a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {
643	SCOPED_ATOMIC(FetchSub, a, v, mo);	622	ATOMIC_IMPL(FetchSub, a, v, mo);
644	}	623	}
645		624
646	#if __TSAN_HAS_INT128	625	#if __TSAN_HAS_INT128
647	SANITIZER_INTERFACE_ATTRIBUTE	626	SANITIZER_INTERFACE_ATTRIBUTE
648	a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {	627	a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {
649	SCOPED_ATOMIC(FetchSub, a, v, mo);	628	ATOMIC_IMPL(FetchSub, a, v, mo);
650	}	629	}
651	#endif	630	#endif
652		631
653	SANITIZER_INTERFACE_ATTRIBUTE	632	SANITIZER_INTERFACE_ATTRIBUTE
654	a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {	633	a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {
655	SCOPED_ATOMIC(FetchAnd, a, v, mo);	634	ATOMIC_IMPL(FetchAnd, a, v, mo);
656	}	635	}
657		636
658	SANITIZER_INTERFACE_ATTRIBUTE	637	SANITIZER_INTERFACE_ATTRIBUTE
659	a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {	638	a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {
660	SCOPED_ATOMIC(FetchAnd, a, v, mo);	639	ATOMIC_IMPL(FetchAnd, a, v, mo);
661	}	640	}
662		641
663	SANITIZER_INTERFACE_ATTRIBUTE	642	SANITIZER_INTERFACE_ATTRIBUTE
664	a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {	643	a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {
665	SCOPED_ATOMIC(FetchAnd, a, v, mo);	644	ATOMIC_IMPL(FetchAnd, a, v, mo);
666	}	645	}
667		646
668	SANITIZER_INTERFACE_ATTRIBUTE	647	SANITIZER_INTERFACE_ATTRIBUTE
669	a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {	648	a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {
670	SCOPED_ATOMIC(FetchAnd, a, v, mo);	649	ATOMIC_IMPL(FetchAnd, a, v, mo);
671	}	650	}
672		651
673	#if __TSAN_HAS_INT128	652	#if __TSAN_HAS_INT128
674	SANITIZER_INTERFACE_ATTRIBUTE	653	SANITIZER_INTERFACE_ATTRIBUTE
675	a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {	654	a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {
676	SCOPED_ATOMIC(FetchAnd, a, v, mo);	655	ATOMIC_IMPL(FetchAnd, a, v, mo);
677	}	656	}
678	#endif	657	#endif
679		658
680	SANITIZER_INTERFACE_ATTRIBUTE	659	SANITIZER_INTERFACE_ATTRIBUTE
681	a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {	660	a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {
682	SCOPED_ATOMIC(FetchOr, a, v, mo);	661	ATOMIC_IMPL(FetchOr, a, v, mo);
683	}	662	}
684		663
685	SANITIZER_INTERFACE_ATTRIBUTE	664	SANITIZER_INTERFACE_ATTRIBUTE
686	a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {	665	a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {
687	SCOPED_ATOMIC(FetchOr, a, v, mo);	666	ATOMIC_IMPL(FetchOr, a, v, mo);
688	}	667	}
689		668
690	SANITIZER_INTERFACE_ATTRIBUTE	669	SANITIZER_INTERFACE_ATTRIBUTE
691	a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {	670	a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {
692	SCOPED_ATOMIC(FetchOr, a, v, mo);	671	ATOMIC_IMPL(FetchOr, a, v, mo);
693	}	672	}
694		673
695	SANITIZER_INTERFACE_ATTRIBUTE	674	SANITIZER_INTERFACE_ATTRIBUTE
696	a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {	675	a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {
697	SCOPED_ATOMIC(FetchOr, a, v, mo);	676	ATOMIC_IMPL(FetchOr, a, v, mo);
698	}	677	}
699		678
700	#if __TSAN_HAS_INT128	679	#if __TSAN_HAS_INT128
701	SANITIZER_INTERFACE_ATTRIBUTE	680	SANITIZER_INTERFACE_ATTRIBUTE
702	a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {	681	a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {
703	SCOPED_ATOMIC(FetchOr, a, v, mo);	682	ATOMIC_IMPL(FetchOr, a, v, mo);
704	}	683	}
705	#endif	684	#endif
706		685
707	SANITIZER_INTERFACE_ATTRIBUTE	686	SANITIZER_INTERFACE_ATTRIBUTE
708	a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {	687	a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {
709	SCOPED_ATOMIC(FetchXor, a, v, mo);	688	ATOMIC_IMPL(FetchXor, a, v, mo);
710	}	689	}
711		690
712	SANITIZER_INTERFACE_ATTRIBUTE	691	SANITIZER_INTERFACE_ATTRIBUTE
713	a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {	692	a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {
714	SCOPED_ATOMIC(FetchXor, a, v, mo);	693	ATOMIC_IMPL(FetchXor, a, v, mo);
715	}	694	}
716		695
717	SANITIZER_INTERFACE_ATTRIBUTE	696	SANITIZER_INTERFACE_ATTRIBUTE
718	a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {	697	a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {
719	SCOPED_ATOMIC(FetchXor, a, v, mo);	698	ATOMIC_IMPL(FetchXor, a, v, mo);
720	}	699	}
721		700
722	SANITIZER_INTERFACE_ATTRIBUTE	701	SANITIZER_INTERFACE_ATTRIBUTE
723	a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {	702	a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {
724	SCOPED_ATOMIC(FetchXor, a, v, mo);	703	ATOMIC_IMPL(FetchXor, a, v, mo);
725	}	704	}
726		705
727	#if __TSAN_HAS_INT128	706	#if __TSAN_HAS_INT128
728	SANITIZER_INTERFACE_ATTRIBUTE	707	SANITIZER_INTERFACE_ATTRIBUTE
729	a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {	708	a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {
730	SCOPED_ATOMIC(FetchXor, a, v, mo);	709	ATOMIC_IMPL(FetchXor, a, v, mo);
731	}	710	}
732	#endif	711	#endif
733		712
734	SANITIZER_INTERFACE_ATTRIBUTE	713	SANITIZER_INTERFACE_ATTRIBUTE
735	a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {	714	a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {
736	SCOPED_ATOMIC(FetchNand, a, v, mo);	715	ATOMIC_IMPL(FetchNand, a, v, mo);
737	}	716	}
738		717
739	SANITIZER_INTERFACE_ATTRIBUTE	718	SANITIZER_INTERFACE_ATTRIBUTE
740	a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {	719	a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {
741	SCOPED_ATOMIC(FetchNand, a, v, mo);	720	ATOMIC_IMPL(FetchNand, a, v, mo);
742	}	721	}
743		722
744	SANITIZER_INTERFACE_ATTRIBUTE	723	SANITIZER_INTERFACE_ATTRIBUTE
745	a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {	724	a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {
746	SCOPED_ATOMIC(FetchNand, a, v, mo);	725	ATOMIC_IMPL(FetchNand, a, v, mo);
747	}	726	}
748		727
749	SANITIZER_INTERFACE_ATTRIBUTE	728	SANITIZER_INTERFACE_ATTRIBUTE
750	a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {	729	a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {
751	SCOPED_ATOMIC(FetchNand, a, v, mo);	730	ATOMIC_IMPL(FetchNand, a, v, mo);
752	}	731	}
753		732
754	#if __TSAN_HAS_INT128	733	#if __TSAN_HAS_INT128
755	SANITIZER_INTERFACE_ATTRIBUTE	734	SANITIZER_INTERFACE_ATTRIBUTE
756	a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {	735	a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {
757	SCOPED_ATOMIC(FetchNand, a, v, mo);	736	ATOMIC_IMPL(FetchNand, a, v, mo);
758	}	737	}
759	#endif	738	#endif
760		739
761	SANITIZER_INTERFACE_ATTRIBUTE	740	SANITIZER_INTERFACE_ATTRIBUTE
762	int __tsan_atomic8_compare_exchange_strong(volatile a8 a, a8 c, a8 v,	741	int __tsan_atomic8_compare_exchange_strong(volatile a8 a, a8 c, a8 v,
763	morder mo, morder fmo) {	742	morder mo, morder fmo) {
764	SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);	743	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
765	}	744	}
766		745
767	SANITIZER_INTERFACE_ATTRIBUTE	746	SANITIZER_INTERFACE_ATTRIBUTE
768	int __tsan_atomic16_compare_exchange_strong(volatile a16 a, a16 c, a16 v,	747	int __tsan_atomic16_compare_exchange_strong(volatile a16 a, a16 c, a16 v,
769	morder mo, morder fmo) {	748	morder mo, morder fmo) {
770	SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);	749	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
771	}	750	}
772		751
773	SANITIZER_INTERFACE_ATTRIBUTE	752	SANITIZER_INTERFACE_ATTRIBUTE
774	int __tsan_atomic32_compare_exchange_strong(volatile a32 a, a32 c, a32 v,	753	int __tsan_atomic32_compare_exchange_strong(volatile a32 a, a32 c, a32 v,
775	morder mo, morder fmo) {	754	morder mo, morder fmo) {
776	SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);	755	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
777	}	756	}
778		757
779	SANITIZER_INTERFACE_ATTRIBUTE	758	SANITIZER_INTERFACE_ATTRIBUTE
780	int __tsan_atomic64_compare_exchange_strong(volatile a64 a, a64 c, a64 v,	759	int __tsan_atomic64_compare_exchange_strong(volatile a64 a, a64 c, a64 v,
781	morder mo, morder fmo) {	760	morder mo, morder fmo) {
782	SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);	761	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
783	}	762	}
784		763
785	#if __TSAN_HAS_INT128	764	#if __TSAN_HAS_INT128
786	SANITIZER_INTERFACE_ATTRIBUTE	765	SANITIZER_INTERFACE_ATTRIBUTE
787	int __tsan_atomic128_compare_exchange_strong(volatile a128 a, a128 c, a128 v,	766	int __tsan_atomic128_compare_exchange_strong(volatile a128 a, a128 c, a128 v,
788	morder mo, morder fmo) {	767	morder mo, morder fmo) {
789	SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);	768	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
790	}	769	}
791	#endif	770	#endif
792		771
793	SANITIZER_INTERFACE_ATTRIBUTE	772	SANITIZER_INTERFACE_ATTRIBUTE
794	int __tsan_atomic8_compare_exchange_weak(volatile a8 a, a8 c, a8 v,	773	int __tsan_atomic8_compare_exchange_weak(volatile a8 a, a8 c, a8 v,
795	morder mo, morder fmo) {	774	morder mo, morder fmo) {
796	SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);	775	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
797	}	776	}
798		777
799	SANITIZER_INTERFACE_ATTRIBUTE	778	SANITIZER_INTERFACE_ATTRIBUTE
800	int __tsan_atomic16_compare_exchange_weak(volatile a16 a, a16 c, a16 v,	779	int __tsan_atomic16_compare_exchange_weak(volatile a16 a, a16 c, a16 v,
801	morder mo, morder fmo) {	780	morder mo, morder fmo) {
802	SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);	781	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
803	}	782	}
804		783
805	SANITIZER_INTERFACE_ATTRIBUTE	784	SANITIZER_INTERFACE_ATTRIBUTE
806	int __tsan_atomic32_compare_exchange_weak(volatile a32 a, a32 c, a32 v,	785	int __tsan_atomic32_compare_exchange_weak(volatile a32 a, a32 c, a32 v,
807	morder mo, morder fmo) {	786	morder mo, morder fmo) {
808	SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);	787	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
809	}	788	}
810		789
811	SANITIZER_INTERFACE_ATTRIBUTE	790	SANITIZER_INTERFACE_ATTRIBUTE
812	int __tsan_atomic64_compare_exchange_weak(volatile a64 a, a64 c, a64 v,	791	int __tsan_atomic64_compare_exchange_weak(volatile a64 a, a64 c, a64 v,
813	morder mo, morder fmo) {	792	morder mo, morder fmo) {
814	SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);	793	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
815	}	794	}
816		795
817	#if __TSAN_HAS_INT128	796	#if __TSAN_HAS_INT128
818	SANITIZER_INTERFACE_ATTRIBUTE	797	SANITIZER_INTERFACE_ATTRIBUTE
819	int __tsan_atomic128_compare_exchange_weak(volatile a128 a, a128 c, a128 v,	798	int __tsan_atomic128_compare_exchange_weak(volatile a128 a, a128 c, a128 v,
820	morder mo, morder fmo) {	799	morder mo, morder fmo) {
821	SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);	800	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
822	}	801	}
823	#endif	802	#endif
824		803
825	SANITIZER_INTERFACE_ATTRIBUTE	804	SANITIZER_INTERFACE_ATTRIBUTE
826	a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,	805	a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,
827	morder mo, morder fmo) {	806	morder mo, morder fmo) {
828	SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);	807	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
829	}	808	}
830		809
831	SANITIZER_INTERFACE_ATTRIBUTE	810	SANITIZER_INTERFACE_ATTRIBUTE
832	a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,	811	a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
833	morder mo, morder fmo) {	812	morder mo, morder fmo) {
834	SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);	813	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
835	}	814	}
836		815
837	SANITIZER_INTERFACE_ATTRIBUTE	816	SANITIZER_INTERFACE_ATTRIBUTE
838	a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,	817	a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
839	morder mo, morder fmo) {	818	morder mo, morder fmo) {
840	SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);	819	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
841	}	820	}
842		821
843	SANITIZER_INTERFACE_ATTRIBUTE	822	SANITIZER_INTERFACE_ATTRIBUTE
844	a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,	823	a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
845	morder mo, morder fmo) {	824	morder mo, morder fmo) {
846	SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);	825	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
847	}	826	}
848		827
849	#if __TSAN_HAS_INT128	828	#if __TSAN_HAS_INT128
850	SANITIZER_INTERFACE_ATTRIBUTE	829	SANITIZER_INTERFACE_ATTRIBUTE
851	a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,	830	a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
852	morder mo, morder fmo) {	831	morder mo, morder fmo) {
853	SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);	832	ATOMIC_IMPL(CAS, a, c, v, mo, fmo);
854	}	833	}
855	#endif	834	#endif
856		835
857	SANITIZER_INTERFACE_ATTRIBUTE	836	SANITIZER_INTERFACE_ATTRIBUTE
858	void __tsan_atomic_thread_fence(morder mo) {	837	void __tsan_atomic_thread_fence(morder mo) { ATOMIC_IMPL(Fence, mo); }
859	char* a = 0;
860	SCOPED_ATOMIC(Fence, mo);
861	}
862		838
863	SANITIZER_INTERFACE_ATTRIBUTE	839	SANITIZER_INTERFACE_ATTRIBUTE
864	void __tsan_atomic_signal_fence(morder mo) {	840	void __tsan_atomic_signal_fence(morder mo) {
...	@@ -869,25 +845,23 @@ void __tsan_atomic_signal_fence(morder mo) {	...	@@ -869,25 +845,23 @@ void __tsan_atomic_signal_fence(morder mo) {
869		845
870	// Go	846	// Go
871		847
872	#define ATOMIC(func, ...) \	848	# define ATOMIC(func, ...) \
873	if (thr->ignore_sync) { \	849	if (thr->ignore_sync) { \
874	NoTsanAtomic##func(__VA_ARGS__); \	850	NoTsanAtomic##func(__VA_ARGS__); \
875	} else { \	851	} else { \
876	FuncEntry(thr, cpc); \	852	FuncEntry(thr, cpc); \
877	Atomic##func(thr, pc, __VA_ARGS__); \	853	Atomic##func(thr, pc, __VA_ARGS__); \
878	FuncExit(thr); \	854	FuncExit(thr); \
879	} \	855	}
880	/**/	856
881		857	# define ATOMIC_RET(func, ret, ...) \
882	#define ATOMIC_RET(func, ret, ...) \	858	if (thr->ignore_sync) { \
883	if (thr->ignore_sync) { \	859	(ret) = NoTsanAtomic##func(__VA_ARGS__); \
884	(ret) = NoTsanAtomic##func(__VA_ARGS__); \	860	} else { \
885	} else { \	861	FuncEntry(thr, cpc); \
886	FuncEntry(thr, cpc); \
887	(ret) = Atomic##func(thr, pc, __VA_ARGS__); \	862	(ret) = Atomic##func(thr, pc, __VA_ARGS__); \
888	FuncExit(thr); \	863	FuncExit(thr); \
889	} \	864	}
890	/**/
891		865
892	extern "C" {	866	extern "C" {
893	SANITIZER_INTERFACE_ATTRIBUTE	867	SANITIZER_INTERFACE_ATTRIBUTE

lib/tsan/tsan_interface_inl.h deleted-133

...	@@ -1,133 +0,0 @@
1	//===-- tsan_interface_inl.h ------------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "tsan_interface.h"
14	#include "tsan_rtl.h"
15	#include "sanitizer_common/sanitizer_ptrauth.h"
16
17	#define CALLERPC ((uptr)__builtin_return_address(0))
18
19	using namespace __tsan;
20
21	void __tsan_read1(void *addr) {
22	MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog1);
23	}
24
25	void __tsan_read2(void *addr) {
26	MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog2);
27	}
28
29	void __tsan_read4(void *addr) {
30	MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog4);
31	}
32
33	void __tsan_read8(void *addr) {
34	MemoryRead(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
35	}
36
37	void __tsan_write1(void *addr) {
38	MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog1);
39	}
40
41	void __tsan_write2(void *addr) {
42	MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog2);
43	}
44
45	void __tsan_write4(void *addr) {
46	MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog4);
47	}
48
49	void __tsan_write8(void *addr) {
50	MemoryWrite(cur_thread(), CALLERPC, (uptr)addr, kSizeLog8);
51	}
52
53	void __tsan_read1_pc(void addr, void pc) {
54	MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog1);
55	}
56
57	void __tsan_read2_pc(void addr, void pc) {
58	MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog2);
59	}
60
61	void __tsan_read4_pc(void addr, void pc) {
62	MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog4);
63	}
64
65	void __tsan_read8_pc(void addr, void pc) {
66	MemoryRead(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog8);
67	}
68
69	void __tsan_write1_pc(void addr, void pc) {
70	MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog1);
71	}
72
73	void __tsan_write2_pc(void addr, void pc) {
74	MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog2);
75	}
76
77	void __tsan_write4_pc(void addr, void pc) {
78	MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog4);
79	}
80
81	void __tsan_write8_pc(void addr, void pc) {
82	MemoryWrite(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, kSizeLog8);
83	}
84
85	void __tsan_vptr_update(void *vptr_p, void new_val) {
86	CHECK_EQ(sizeof(vptr_p), 8);
87	if (*vptr_p != new_val) {
88	ThreadState *thr = cur_thread();
89	thr->is_vptr_access = true;
90	MemoryWrite(thr, CALLERPC, (uptr)vptr_p, kSizeLog8);
91	thr->is_vptr_access = false;
92	}
93	}
94
95	void __tsan_vptr_read(void **vptr_p) {
96	CHECK_EQ(sizeof(vptr_p), 8);
97	ThreadState *thr = cur_thread();
98	thr->is_vptr_access = true;
99	MemoryRead(thr, CALLERPC, (uptr)vptr_p, kSizeLog8);
100	thr->is_vptr_access = false;
101	}
102
103	void __tsan_func_entry(void *pc) {
104	FuncEntry(cur_thread(), STRIP_PAC_PC(pc));
105	}
106
107	void __tsan_func_exit() {
108	FuncExit(cur_thread());
109	}
110
111	void __tsan_ignore_thread_begin() {
112	ThreadIgnoreBegin(cur_thread(), CALLERPC);
113	}
114
115	void __tsan_ignore_thread_end() {
116	ThreadIgnoreEnd(cur_thread(), CALLERPC);
117	}
118
119	void __tsan_read_range(void *addr, uptr size) {
120	MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, false);
121	}
122
123	void __tsan_write_range(void *addr, uptr size) {
124	MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, true);
125	}
126
127	void __tsan_read_range_pc(void addr, uptr size, void pc) {
128	MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, false);
129	}
130
131	void __tsan_write_range_pc(void addr, uptr size, void pc) {
132	MemoryAccessRange(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, size, true);
133	}

lib/tsan/tsan_interface_java.cpp+143-151

...	@@ -34,52 +34,49 @@ struct JavaContext {	...	@@ -34,52 +34,49 @@ struct JavaContext {
34	}	34	}
35	};	35	};
36		36
37	class ScopedJavaFunc {
38	public:
39	ScopedJavaFunc(ThreadState *thr, uptr pc)
40	: thr_(thr) {
41	Initialize(thr_);
42	FuncEntry(thr, pc);
43	}
44
45	~ScopedJavaFunc() {
46	FuncExit(thr_);
47	// FIXME(dvyukov): process pending signals.
48	}
49
50	private:
51	ThreadState *thr_;
52	};
53
54	static u64 jctx_buf[sizeof(JavaContext) / sizeof(u64) + 1];	37	static u64 jctx_buf[sizeof(JavaContext) / sizeof(u64) + 1];
55	static JavaContext *jctx;	38	static JavaContext *jctx;
56		39
		40	MBlock JavaHeapBlock(uptr addr, uptr start) {
		41	if (!jctx \|\| addr < jctx->heap_begin \|\|
		42	addr >= jctx->heap_begin + jctx->heap_size)
		43	return nullptr;
		44	for (uptr p = RoundDown(addr, kMetaShadowCell); p >= jctx->heap_begin;
		45	p -= kMetaShadowCell) {
		46	MBlock *b = ctx->metamap.GetBlock(p);
		47	if (!b)
		48	continue;
		49	if (p + b->siz <= addr)
		50	return nullptr;
		51	*start = p;
		52	return b;
		53	}
		54	return nullptr;
		55	}
		56
57	} // namespace __tsan	57	} // namespace __tsan
58		58
59	#define SCOPED_JAVA_FUNC(func) \	59	#define JAVA_FUNC_ENTER(func) \
60	ThreadState *thr = cur_thread(); \	60	ThreadState *thr = cur_thread(); \
61	const uptr caller_pc = GET_CALLER_PC(); \	61	(void)thr;
62	const uptr pc = StackTrace::GetCurrentPc(); \
63	(void)pc; \
64	ScopedJavaFunc scoped(thr, caller_pc); \
65	/**/
66		62
67	void __tsan_java_init(jptr heap_begin, jptr heap_size) {	63	void __tsan_java_init(jptr heap_begin, jptr heap_size) {
68	SCOPED_JAVA_FUNC(__tsan_java_init);	64	JAVA_FUNC_ENTER(__tsan_java_init);
69	DPrintf("#%d: java_init(%p, %p)\n", thr->tid, heap_begin, heap_size);	65	Initialize(thr);
70	CHECK_EQ(jctx, 0);	66	DPrintf("#%d: java_init(0x%zx, 0x%zx)\n", thr->tid, heap_begin, heap_size);
71	CHECK_GT(heap_begin, 0);	67	DCHECK_EQ(jctx, 0);
72	CHECK_GT(heap_size, 0);	68	DCHECK_GT(heap_begin, 0);
73	CHECK_EQ(heap_begin % kHeapAlignment, 0);	69	DCHECK_GT(heap_size, 0);
74	CHECK_EQ(heap_size % kHeapAlignment, 0);	70	DCHECK_EQ(heap_begin % kHeapAlignment, 0);
75	CHECK_LT(heap_begin, heap_begin + heap_size);	71	DCHECK_EQ(heap_size % kHeapAlignment, 0);
		72	DCHECK_LT(heap_begin, heap_begin + heap_size);
76	jctx = new(jctx_buf) JavaContext(heap_begin, heap_size);	73	jctx = new(jctx_buf) JavaContext(heap_begin, heap_size);
77	}	74	}
78		75
79	int __tsan_java_fini() {	76	int __tsan_java_fini() {
80	SCOPED_JAVA_FUNC(__tsan_java_fini);	77	JAVA_FUNC_ENTER(__tsan_java_fini);
81	DPrintf("#%d: java_fini()\n", thr->tid);	78	DPrintf("#%d: java_fini()\n", thr->tid);
82	CHECK_NE(jctx, 0);	79	DCHECK_NE(jctx, 0);
83	// FIXME(dvyukov): this does not call atexit() callbacks.	80	// FIXME(dvyukov): this does not call atexit() callbacks.
84	int status = Finalize(thr);	81	int status = Finalize(thr);
85	DPrintf("#%d: java_fini() = %d\n", thr->tid, status);	82	DPrintf("#%d: java_fini() = %d\n", thr->tid, status);
...	@@ -87,74 +84,65 @@ int __tsan_java_fini() {	...	@@ -87,74 +84,65 @@ int __tsan_java_fini() {
87	}	84	}
88		85
89	void __tsan_java_alloc(jptr ptr, jptr size) {	86	void __tsan_java_alloc(jptr ptr, jptr size) {
90	SCOPED_JAVA_FUNC(__tsan_java_alloc);	87	JAVA_FUNC_ENTER(__tsan_java_alloc);
91	DPrintf("#%d: java_alloc(%p, %p)\n", thr->tid, ptr, size);	88	DPrintf("#%d: java_alloc(0x%zx, 0x%zx)\n", thr->tid, ptr, size);
92	CHECK_NE(jctx, 0);	89	DCHECK_NE(jctx, 0);
93	CHECK_NE(size, 0);	90	DCHECK_NE(size, 0);
94	CHECK_EQ(ptr % kHeapAlignment, 0);	91	DCHECK_EQ(ptr % kHeapAlignment, 0);
95	CHECK_EQ(size % kHeapAlignment, 0);	92	DCHECK_EQ(size % kHeapAlignment, 0);
96	CHECK_GE(ptr, jctx->heap_begin);	93	DCHECK_GE(ptr, jctx->heap_begin);
97	CHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);	94	DCHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
98		95
99	OnUserAlloc(thr, pc, ptr, size, false);	96	OnUserAlloc(thr, 0, ptr, size, false);
100	}	97	}
101		98
102	void __tsan_java_free(jptr ptr, jptr size) {	99	void __tsan_java_free(jptr ptr, jptr size) {
103	SCOPED_JAVA_FUNC(__tsan_java_free);	100	JAVA_FUNC_ENTER(__tsan_java_free);
104	DPrintf("#%d: java_free(%p, %p)\n", thr->tid, ptr, size);	101	DPrintf("#%d: java_free(0x%zx, 0x%zx)\n", thr->tid, ptr, size);
105	CHECK_NE(jctx, 0);	102	DCHECK_NE(jctx, 0);
106	CHECK_NE(size, 0);	103	DCHECK_NE(size, 0);
107	CHECK_EQ(ptr % kHeapAlignment, 0);	104	DCHECK_EQ(ptr % kHeapAlignment, 0);
108	CHECK_EQ(size % kHeapAlignment, 0);	105	DCHECK_EQ(size % kHeapAlignment, 0);
109	CHECK_GE(ptr, jctx->heap_begin);	106	DCHECK_GE(ptr, jctx->heap_begin);
110	CHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);	107	DCHECK_LE(ptr + size, jctx->heap_begin + jctx->heap_size);
111		108
112	ctx->metamap.FreeRange(thr->proc(), ptr, size);	109	ctx->metamap.FreeRange(thr->proc(), ptr, size, false);
113	}	110	}
114		111
115	void __tsan_java_move(jptr src, jptr dst, jptr size) {	112	void __tsan_java_move(jptr src, jptr dst, jptr size) {
116	SCOPED_JAVA_FUNC(__tsan_java_move);	113	JAVA_FUNC_ENTER(__tsan_java_move);
117	DPrintf("#%d: java_move(%p, %p, %p)\n", thr->tid, src, dst, size);	114	DPrintf("#%d: java_move(0x%zx, 0x%zx, 0x%zx)\n", thr->tid, src, dst, size);
118	CHECK_NE(jctx, 0);	115	DCHECK_NE(jctx, 0);
119	CHECK_NE(size, 0);	116	DCHECK_NE(size, 0);
120	CHECK_EQ(src % kHeapAlignment, 0);	117	DCHECK_EQ(src % kHeapAlignment, 0);
121	CHECK_EQ(dst % kHeapAlignment, 0);	118	DCHECK_EQ(dst % kHeapAlignment, 0);
122	CHECK_EQ(size % kHeapAlignment, 0);	119	DCHECK_EQ(size % kHeapAlignment, 0);
123	CHECK_GE(src, jctx->heap_begin);	120	DCHECK_GE(src, jctx->heap_begin);
124	CHECK_LE(src + size, jctx->heap_begin + jctx->heap_size);	121	DCHECK_LE(src + size, jctx->heap_begin + jctx->heap_size);
125	CHECK_GE(dst, jctx->heap_begin);	122	DCHECK_GE(dst, jctx->heap_begin);
126	CHECK_LE(dst + size, jctx->heap_begin + jctx->heap_size);	123	DCHECK_LE(dst + size, jctx->heap_begin + jctx->heap_size);
127	CHECK_NE(dst, src);	124	DCHECK_NE(dst, src);
128	CHECK_NE(size, 0);	125	DCHECK_NE(size, 0);
129		126
130	// Assuming it's not running concurrently with threads that do	127	// Assuming it's not running concurrently with threads that do
131	// memory accesses and mutex operations (stop-the-world phase).	128	// memory accesses and mutex operations (stop-the-world phase).
132	ctx->metamap.MoveMemory(src, dst, size);	129	ctx->metamap.MoveMemory(src, dst, size);
133		130
134	// Move shadow.	131	// Clear the destination shadow range.
135	u64 s = (u64)MemToShadow(src);	132	// We used to move shadow from src to dst, but the trace format does not
136	u64 d = (u64)MemToShadow(dst);	133	// support that anymore as it contains addresses of accesses.
137	u64 send = (u64)MemToShadow(src + size);	134	RawShadow *d = MemToShadow(dst);
138	uptr inc = 1;	135	RawShadow *dend = MemToShadow(dst + size);
139	if (dst > src) {	136	ShadowSet(d, dend, Shadow::kEmpty);
140	s = (u64*)MemToShadow(src + size) - 1;
141	d = (u64*)MemToShadow(dst + size) - 1;
142	send = (u64*)MemToShadow(src) - 1;
143	inc = -1;
144	}
145	for (; s != send; s += inc, d += inc) {
146	d = s;
147	*s = 0;
148	}
149	}	137	}
150		138
151	jptr __tsan_java_find(jptr *from_ptr, jptr to) {	139	jptr __tsan_java_find(jptr *from_ptr, jptr to) {
152	SCOPED_JAVA_FUNC(__tsan_java_find);	140	JAVA_FUNC_ENTER(__tsan_java_find);
153	DPrintf("#%d: java_find(&%p, %p)\n", *from_ptr, to);	141	DPrintf("#%d: java_find(&0x%zx, 0x%zx)\n", thr->tid, *from_ptr, to);
154	CHECK_EQ((*from_ptr) % kHeapAlignment, 0);	142	DCHECK_EQ((*from_ptr) % kHeapAlignment, 0);
155	CHECK_EQ(to % kHeapAlignment, 0);	143	DCHECK_EQ(to % kHeapAlignment, 0);
156	CHECK_GE(*from_ptr, jctx->heap_begin);	144	DCHECK_GE(*from_ptr, jctx->heap_begin);
157	CHECK_LE(to, jctx->heap_begin + jctx->heap_size);	145	DCHECK_LE(to, jctx->heap_begin + jctx->heap_size);
158	for (uptr from = *from_ptr; from < to; from += kHeapAlignment) {	146	for (uptr from = *from_ptr; from < to; from += kHeapAlignment) {
159	MBlock *b = ctx->metamap.GetBlock(from);	147	MBlock *b = ctx->metamap.GetBlock(from);
160	if (b) {	148	if (b) {
...	@@ -166,101 +154,105 @@ jptr __tsan_java_find(jptr *from_ptr, jptr to) {	...	@@ -166,101 +154,105 @@ jptr __tsan_java_find(jptr *from_ptr, jptr to) {
166	}	154	}
167		155
168	void __tsan_java_finalize() {	156	void __tsan_java_finalize() {
169	SCOPED_JAVA_FUNC(__tsan_java_finalize);	157	JAVA_FUNC_ENTER(__tsan_java_finalize);
170	DPrintf("#%d: java_mutex_finalize()\n", thr->tid);	158	DPrintf("#%d: java_finalize()\n", thr->tid);
171	AcquireGlobal(thr, 0);	159	AcquireGlobal(thr);
172	}	160	}
173		161
174	void __tsan_java_mutex_lock(jptr addr) {	162	void __tsan_java_mutex_lock(jptr addr) {
175	SCOPED_JAVA_FUNC(__tsan_java_mutex_lock);	163	JAVA_FUNC_ENTER(__tsan_java_mutex_lock);
176	DPrintf("#%d: java_mutex_lock(%p)\n", thr->tid, addr);	164	DPrintf("#%d: java_mutex_lock(0x%zx)\n", thr->tid, addr);
177	CHECK_NE(jctx, 0);	165	DCHECK_NE(jctx, 0);
178	CHECK_GE(addr, jctx->heap_begin);	166	DCHECK_GE(addr, jctx->heap_begin);
179	CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);	167	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
180		168
181	MutexPostLock(thr, pc, addr, MutexFlagLinkerInit \| MutexFlagWriteReentrant \|	169	MutexPostLock(thr, 0, addr,
182	MutexFlagDoPreLockOnPostLock);	170	MutexFlagLinkerInit \| MutexFlagWriteReentrant \|
		171	MutexFlagDoPreLockOnPostLock);
183	}	172	}
184		173
185	void __tsan_java_mutex_unlock(jptr addr) {	174	void __tsan_java_mutex_unlock(jptr addr) {
186	SCOPED_JAVA_FUNC(__tsan_java_mutex_unlock);	175	JAVA_FUNC_ENTER(__tsan_java_mutex_unlock);
187	DPrintf("#%d: java_mutex_unlock(%p)\n", thr->tid, addr);	176	DPrintf("#%d: java_mutex_unlock(0x%zx)\n", thr->tid, addr);
188	CHECK_NE(jctx, 0);	177	DCHECK_NE(jctx, 0);
189	CHECK_GE(addr, jctx->heap_begin);	178	DCHECK_GE(addr, jctx->heap_begin);
190	CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);	179	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
191		180
192	MutexUnlock(thr, pc, addr);	181	MutexUnlock(thr, 0, addr);
193	}	182	}
194		183
195	void __tsan_java_mutex_read_lock(jptr addr) {	184	void __tsan_java_mutex_read_lock(jptr addr) {
196	SCOPED_JAVA_FUNC(__tsan_java_mutex_read_lock);	185	JAVA_FUNC_ENTER(__tsan_java_mutex_read_lock);
197	DPrintf("#%d: java_mutex_read_lock(%p)\n", thr->tid, addr);	186	DPrintf("#%d: java_mutex_read_lock(0x%zx)\n", thr->tid, addr);
198	CHECK_NE(jctx, 0);	187	DCHECK_NE(jctx, 0);
199	CHECK_GE(addr, jctx->heap_begin);	188	DCHECK_GE(addr, jctx->heap_begin);
200	CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);	189	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
201		190
202	MutexPostReadLock(thr, pc, addr, MutexFlagLinkerInit \|	191	MutexPostReadLock(thr, 0, addr,
203	MutexFlagWriteReentrant \| MutexFlagDoPreLockOnPostLock);	192	MutexFlagLinkerInit \| MutexFlagWriteReentrant \|
		193	MutexFlagDoPreLockOnPostLock);
204	}	194	}
205		195
206	void __tsan_java_mutex_read_unlock(jptr addr) {	196	void __tsan_java_mutex_read_unlock(jptr addr) {
207	SCOPED_JAVA_FUNC(__tsan_java_mutex_read_unlock);	197	JAVA_FUNC_ENTER(__tsan_java_mutex_read_unlock);
208	DPrintf("#%d: java_mutex_read_unlock(%p)\n", thr->tid, addr);	198	DPrintf("#%d: java_mutex_read_unlock(0x%zx)\n", thr->tid, addr);
209	CHECK_NE(jctx, 0);	199	DCHECK_NE(jctx, 0);
210	CHECK_GE(addr, jctx->heap_begin);	200	DCHECK_GE(addr, jctx->heap_begin);
211	CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);	201	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
212		202
213	MutexReadUnlock(thr, pc, addr);	203	MutexReadUnlock(thr, 0, addr);
214	}	204	}
215		205
216	void __tsan_java_mutex_lock_rec(jptr addr, int rec) {	206	void __tsan_java_mutex_lock_rec(jptr addr, int rec) {
217	SCOPED_JAVA_FUNC(__tsan_java_mutex_lock_rec);	207	JAVA_FUNC_ENTER(__tsan_java_mutex_lock_rec);
218	DPrintf("#%d: java_mutex_lock_rec(%p, %d)\n", thr->tid, addr, rec);	208	DPrintf("#%d: java_mutex_lock_rec(0x%zx, %d)\n", thr->tid, addr, rec);
219	CHECK_NE(jctx, 0);	209	DCHECK_NE(jctx, 0);
220	CHECK_GE(addr, jctx->heap_begin);	210	DCHECK_GE(addr, jctx->heap_begin);
221	CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);	211	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
222	CHECK_GT(rec, 0);	212	DCHECK_GT(rec, 0);
223		213
224	MutexPostLock(thr, pc, addr, MutexFlagLinkerInit \| MutexFlagWriteReentrant \|	214	MutexPostLock(thr, 0, addr,
225	MutexFlagDoPreLockOnPostLock \| MutexFlagRecursiveLock, rec);	215	MutexFlagLinkerInit \| MutexFlagWriteReentrant \|
		216	MutexFlagDoPreLockOnPostLock \| MutexFlagRecursiveLock,
		217	rec);
226	}	218	}
227		219
228	int __tsan_java_mutex_unlock_rec(jptr addr) {	220	int __tsan_java_mutex_unlock_rec(jptr addr) {
229	SCOPED_JAVA_FUNC(__tsan_java_mutex_unlock_rec);	221	JAVA_FUNC_ENTER(__tsan_java_mutex_unlock_rec);
230	DPrintf("#%d: java_mutex_unlock_rec(%p)\n", thr->tid, addr);	222	DPrintf("#%d: java_mutex_unlock_rec(0x%zx)\n", thr->tid, addr);
231	CHECK_NE(jctx, 0);	223	DCHECK_NE(jctx, 0);
232	CHECK_GE(addr, jctx->heap_begin);	224	DCHECK_GE(addr, jctx->heap_begin);
233	CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);	225	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
234		226
235	return MutexUnlock(thr, pc, addr, MutexFlagRecursiveUnlock);	227	return MutexUnlock(thr, 0, addr, MutexFlagRecursiveUnlock);
236	}	228	}
237		229
238	void __tsan_java_acquire(jptr addr) {	230	void __tsan_java_acquire(jptr addr) {
239	SCOPED_JAVA_FUNC(__tsan_java_acquire);	231	JAVA_FUNC_ENTER(__tsan_java_acquire);
240	DPrintf("#%d: java_acquire(%p)\n", thr->tid, addr);	232	DPrintf("#%d: java_acquire(0x%zx)\n", thr->tid, addr);
241	CHECK_NE(jctx, 0);	233	DCHECK_NE(jctx, 0);
242	CHECK_GE(addr, jctx->heap_begin);	234	DCHECK_GE(addr, jctx->heap_begin);
243	CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);	235	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
244		236
245	Acquire(thr, caller_pc, addr);	237	Acquire(thr, 0, addr);
246	}	238	}
247		239
248	void __tsan_java_release(jptr addr) {	240	void __tsan_java_release(jptr addr) {
249	SCOPED_JAVA_FUNC(__tsan_java_release);	241	JAVA_FUNC_ENTER(__tsan_java_release);
250	DPrintf("#%d: java_release(%p)\n", thr->tid, addr);	242	DPrintf("#%d: java_release(0x%zx)\n", thr->tid, addr);
251	CHECK_NE(jctx, 0);	243	DCHECK_NE(jctx, 0);
252	CHECK_GE(addr, jctx->heap_begin);	244	DCHECK_GE(addr, jctx->heap_begin);
253	CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);	245	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
254		246
255	Release(thr, caller_pc, addr);	247	Release(thr, 0, addr);
256	}	248	}
257		249
258	void __tsan_java_release_store(jptr addr) {	250	void __tsan_java_release_store(jptr addr) {
259	SCOPED_JAVA_FUNC(__tsan_java_release);	251	JAVA_FUNC_ENTER(__tsan_java_release);
260	DPrintf("#%d: java_release_store(%p)\n", thr->tid, addr);	252	DPrintf("#%d: java_release_store(0x%zx)\n", thr->tid, addr);
261	CHECK_NE(jctx, 0);	253	DCHECK_NE(jctx, 0);
262	CHECK_GE(addr, jctx->heap_begin);	254	DCHECK_GE(addr, jctx->heap_begin);
263	CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);	255	DCHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
264		256
265	ReleaseStore(thr, caller_pc, addr);	257	ReleaseStore(thr, 0, addr);
266	}	258	}

lib/tsan/tsan_malloc_mac.cpp+23-7

 @@ -12,11 +12,12 @@
 //===----------------------------------------------------------------------===//
 #include "sanitizer_common/sanitizer_platform.h"
-#if SANITIZER_MAC
+#if SANITIZER_APPLE
 #include "sanitizer_common/sanitizer_errno.h"
 #include "tsan_interceptors.h"
 #include "tsan_stack_trace.h"
+#include "tsan_mman.h"
 using namespace __tsan;
 #define COMMON_MALLOC_ZONE_NAME "tsan"
 @@ -29,16 +30,30 @@ using namespace __tsan;
       user_memalign(cur_thread(), StackTrace::GetCurrentPc(), alignment, size)
 #define COMMON_MALLOC_MALLOC(size)                             \
   if (in_symbolizer()) return InternalAlloc(size);             \
-  SCOPED_INTERCEPTOR_RAW(malloc, size);                        \
+  void *p = 0;                                                 \
-  void *p = user_alloc(thr, pc, size)
+  {                                                            \
+    SCOPED_INTERCEPTOR_RAW(malloc, size);                      \
+    p = user_alloc(thr, pc, size);                             \
+  }                                                            \
+  invoke_malloc_hook(p, size)
 #define COMMON_MALLOC_REALLOC(ptr, size)                              \
   if (in_symbolizer()) return InternalRealloc(ptr, size);             \
-  SCOPED_INTERCEPTOR_RAW(realloc, ptr, size);                         \
+  if (ptr)                                                            \
-  void *p = user_realloc(thr, pc, ptr, size)
+    invoke_free_hook(ptr);                                            \
+  void *p = 0;                                                        \
+  {                                                                   \
+    SCOPED_INTERCEPTOR_RAW(realloc, ptr, size);                       \
+    p = user_realloc(thr, pc, ptr, size);                             \
+  }                                                                   \
+  invoke_malloc_hook(p, size)
 #define COMMON_MALLOC_CALLOC(count, size)                              \
   if (in_symbolizer()) return InternalCalloc(count, size);             \
-  SCOPED_INTERCEPTOR_RAW(calloc, size, count);                         \
+  void *p = 0;                                                         \
-  void *p = user_calloc(thr, pc, size, count)
+  {                                                                    \
+    SCOPED_INTERCEPTOR_RAW(calloc, size, count);                       \
+    p = user_calloc(thr, pc, size, count);                             \
+  }                                                                    \
+  invoke_malloc_hook(p, size * count)
 #define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size)      \
   if (in_symbolizer()) {                                           \
     void *p = InternalAlloc(size, nullptr, alignment);             \
 @@ -55,6 +70,7 @@ using namespace __tsan;
   void *p = user_valloc(thr, pc, size)
 #define COMMON_MALLOC_FREE(ptr)                              \
   if (in_symbolizer()) return InternalFree(ptr);             \
+  invoke_free_hook(ptr);                                     \
   SCOPED_INTERCEPTOR_RAW(free, ptr);                         \
   user_free(thr, pc, ptr)
 #define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr);

lib/tsan/tsan_mman.cpp+109-26

...	@@ -15,27 +15,18 @@	...	@@ -15,27 +15,18 @@
15	#include "sanitizer_common/sanitizer_common.h"	15	#include "sanitizer_common/sanitizer_common.h"
16	#include "sanitizer_common/sanitizer_errno.h"	16	#include "sanitizer_common/sanitizer_errno.h"
17	#include "sanitizer_common/sanitizer_placement_new.h"	17	#include "sanitizer_common/sanitizer_placement_new.h"
		18	#include "tsan_interface.h"
18	#include "tsan_mman.h"	19	#include "tsan_mman.h"
19	#include "tsan_rtl.h"	20	#include "tsan_rtl.h"
20	#include "tsan_report.h"	21	#include "tsan_report.h"
21	#include "tsan_flags.h"	22	#include "tsan_flags.h"
22		23
23	// May be overriden by front-end.
24	SANITIZER_WEAK_DEFAULT_IMPL
25	void __sanitizer_malloc_hook(void *ptr, uptr size) {
26	(void)ptr;
27	(void)size;
28	}
29
30	SANITIZER_WEAK_DEFAULT_IMPL
31	void __sanitizer_free_hook(void *ptr) {
32	(void)ptr;
33	}
34
35	namespace __tsan {	24	namespace __tsan {
36		25
37	struct MapUnmapCallback {	26	struct MapUnmapCallback {
38	void OnMap(uptr p, uptr size) const { }	27	void OnMap(uptr p, uptr size) const { }
		28	void OnMapSecondary(uptr p, uptr size, uptr user_begin,
		29	uptr user_size) const {};
39	void OnUnmap(uptr p, uptr size) const {	30	void OnUnmap(uptr p, uptr size) const {
40	// We are about to unmap a chunk of user memory.	31	// We are about to unmap a chunk of user memory.
41	// Mark the corresponding shadow memory as not needed.	32	// Mark the corresponding shadow memory as not needed.
...	@@ -69,8 +60,17 @@ Allocator *allocator() {	...	@@ -69,8 +60,17 @@ Allocator *allocator() {
69	struct GlobalProc {	60	struct GlobalProc {
70	Mutex mtx;	61	Mutex mtx;
71	Processor *proc;	62	Processor *proc;
72		63	// This mutex represents the internal allocator combined for
73	GlobalProc() : mtx(MutexTypeGlobalProc), proc(ProcCreate()) {}	64	// the purposes of deadlock detection. The internal allocator
		65	// uses multiple mutexes, moreover they are locked only occasionally
		66	// and they are spin mutexes which don't support deadlock detection.
		67	// So we use this fake mutex to serve as a substitute for these mutexes.
		68	CheckedMutex internal_alloc_mtx;
		69
		70	GlobalProc()
		71	: mtx(MutexTypeGlobalProc),
		72	proc(ProcCreate()),
		73	internal_alloc_mtx(MutexTypeInternalAlloc) {}
74	};	74	};
75		75
76	static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64);	76	static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64);
...	@@ -78,6 +78,11 @@ GlobalProc *global_proc() {	...	@@ -78,6 +78,11 @@ GlobalProc *global_proc() {
78	return reinterpret_cast<GlobalProc*>(&global_proc_placeholder);	78	return reinterpret_cast<GlobalProc*>(&global_proc_placeholder);
79	}	79	}
80		80
		81	static void InternalAllocAccess() {
		82	global_proc()->internal_alloc_mtx.Lock();
		83	global_proc()->internal_alloc_mtx.Unlock();
		84	}
		85
81	ScopedGlobalProcessor::ScopedGlobalProcessor() {	86	ScopedGlobalProcessor::ScopedGlobalProcessor() {
82	GlobalProc *gp = global_proc();	87	GlobalProc *gp = global_proc();
83	ThreadState *thr = cur_thread();	88	ThreadState *thr = cur_thread();
...	@@ -110,6 +115,24 @@ ScopedGlobalProcessor::~ScopedGlobalProcessor() {	...	@@ -110,6 +115,24 @@ ScopedGlobalProcessor::~ScopedGlobalProcessor() {
110	gp->mtx.Unlock();	115	gp->mtx.Unlock();
111	}	116	}
112		117
		118	void AllocatorLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
		119	global_proc()->internal_alloc_mtx.Lock();
		120	InternalAllocatorLock();
		121	}
		122
		123	void AllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
		124	InternalAllocatorUnlock();
		125	global_proc()->internal_alloc_mtx.Unlock();
		126	}
		127
		128	void GlobalProcessorLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
		129	global_proc()->mtx.Lock();
		130	}
		131
		132	void GlobalProcessorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
		133	global_proc()->mtx.Unlock();
		134	}
		135
113	static constexpr uptr kMaxAllowedMallocSize = 1ull << 40;	136	static constexpr uptr kMaxAllowedMallocSize = 1ull << 40;
114	static uptr max_user_defined_malloc_size;	137	static uptr max_user_defined_malloc_size;
115		138
...	@@ -148,7 +171,7 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) {	...	@@ -148,7 +171,7 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
148	ObtainCurrentStack(thr, pc, &stack);	171	ObtainCurrentStack(thr, pc, &stack);
149	if (IsFiredSuppression(ctx, ReportTypeSignalUnsafe, stack))	172	if (IsFiredSuppression(ctx, ReportTypeSignalUnsafe, stack))
150	return;	173	return;
151	ThreadRegistryLock l(ctx->thread_registry);	174	ThreadRegistryLock l(&ctx->thread_registry);
152	ScopedReport rep(ReportTypeSignalUnsafe);	175	ScopedReport rep(ReportTypeSignalUnsafe);
153	rep.AddStack(stack, true);	176	rep.AddStack(stack, true);
154	OutputReport(thr, rep);	177	OutputReport(thr, rep);
...	@@ -166,6 +189,12 @@ void user_alloc_internal(ThreadState thr, uptr pc, uptr sz, uptr align,	...	@@ -166,6 +189,12 @@ void user_alloc_internal(ThreadState thr, uptr pc, uptr sz, uptr align,
166	GET_STACK_TRACE_FATAL(thr, pc);	189	GET_STACK_TRACE_FATAL(thr, pc);
167	ReportAllocationSizeTooBig(sz, malloc_limit, &stack);	190	ReportAllocationSizeTooBig(sz, malloc_limit, &stack);
168	}	191	}
		192	if (UNLIKELY(IsRssLimitExceeded())) {
		193	if (AllocatorMayReturnNull())
		194	return nullptr;
		195	GET_STACK_TRACE_FATAL(thr, pc);
		196	ReportRssLimitExceeded(&stack);
		197	}
169	void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align);	198	void *p = allocator()->Allocate(&thr->proc()->alloc_cache, sz, align);
170	if (UNLIKELY(!p)) {	199	if (UNLIKELY(!p)) {
171	SetAllocatorOutOfMemory();	200	SetAllocatorOutOfMemory();
...	@@ -218,9 +247,18 @@ void user_reallocarray(ThreadState thr, uptr pc, void *p, uptr size, uptr n) {	...	@@ -218,9 +247,18 @@ void user_reallocarray(ThreadState thr, uptr pc, void *p, uptr size, uptr n) {
218	}	247	}
219		248
220	void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {	249	void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {
221	DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p);	250	DPrintf("#%d: alloc(%zu) = 0x%zx\n", thr->tid, sz, p);
		251	// Note: this can run before thread initialization/after finalization.
		252	// As a result this is not necessarily synchronized with DoReset,
		253	// which iterates over and resets all sync objects,
		254	// but it is fine to create new MBlocks in this context.
222	ctx->metamap.AllocBlock(thr, pc, p, sz);	255	ctx->metamap.AllocBlock(thr, pc, p, sz);
223	if (write && thr->ignore_reads_and_writes == 0)	256	// If this runs before thread initialization/after finalization
		257	// and we don't have trace initialized, we can't imitate writes.
		258	// In such case just reset the shadow range, it is fine since
		259	// it affects only a small fraction of special objects.
		260	if (write && thr->ignore_reads_and_writes == 0 &&
		261	atomic_load_relaxed(&thr->trace_pos))
224	MemoryRangeImitateWrite(thr, pc, (uptr)p, sz);	262	MemoryRangeImitateWrite(thr, pc, (uptr)p, sz);
225	else	263	else
226	MemoryResetRange(thr, pc, (uptr)p, sz);	264	MemoryResetRange(thr, pc, (uptr)p, sz);
...	@@ -228,8 +266,15 @@ void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {	...	@@ -228,8 +266,15 @@ void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) {
228		266
229	void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) {	267	void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) {
230	CHECK_NE(p, (void*)0);	268	CHECK_NE(p, (void*)0);
231	uptr sz = ctx->metamap.FreeBlock(thr->proc(), p);	269	if (!thr->slot) {
232	DPrintf("#%d: free(%p, %zu)\n", thr->tid, p, sz);	270	// Very early/late in thread lifetime, or during fork.
		271	UNUSED uptr sz = ctx->metamap.FreeBlock(thr->proc(), p, false);
		272	DPrintf("#%d: free(0x%zx, %zu) (no slot)\n", thr->tid, p, sz);
		273	return;
		274	}
		275	SlotLocker locker(thr);
		276	uptr sz = ctx->metamap.FreeBlock(thr->proc(), p, true);
		277	DPrintf("#%d: free(0x%zx, %zu)\n", thr->tid, p, sz);
233	if (write && thr->ignore_reads_and_writes == 0)	278	if (write && thr->ignore_reads_and_writes == 0)
234	MemoryRangeFreed(thr, pc, (uptr)p, sz);	279	MemoryRangeFreed(thr, pc, (uptr)p, sz);
235	}	280	}
...	@@ -309,8 +354,22 @@ void user_pvalloc(ThreadState thr, uptr pc, uptr sz) {	...	@@ -309,8 +354,22 @@ void user_pvalloc(ThreadState thr, uptr pc, uptr sz) {
309	return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, PageSize));	354	return SetErrnoOnNull(user_alloc_internal(thr, pc, sz, PageSize));
310	}	355	}
311		356
		357	static const void user_alloc_begin(const void p) {
		358	if (p == nullptr \|\| !IsAppMem((uptr)p))
		359	return nullptr;
		360	void *beg = allocator()->GetBlockBegin(p);
		361	if (!beg)
		362	return nullptr;
		363
		364	MBlock *b = ctx->metamap.GetBlock((uptr)beg);
		365	if (!b)
		366	return nullptr; // Not a valid pointer.
		367
		368	return (const void *)beg;
		369	}
		370
312	uptr user_alloc_usable_size(const void *p) {	371	uptr user_alloc_usable_size(const void *p) {
313	if (p == 0)	372	if (p == 0 \|\| !IsAppMem((uptr)p))
314	return 0;	373	return 0;
315	MBlock *b = ctx->metamap.GetBlock((uptr)p);	374	MBlock *b = ctx->metamap.GetBlock((uptr)p);
316	if (!b)	375	if (!b)
...	@@ -320,11 +379,21 @@ uptr user_alloc_usable_size(const void *p) {	...	@@ -320,11 +379,21 @@ uptr user_alloc_usable_size(const void *p) {
320	return b->siz;	379	return b->siz;
321	}	380	}
322		381
		382	uptr user_alloc_usable_size_fast(const void *p) {
		383	MBlock *b = ctx->metamap.GetBlock((uptr)p);
		384	// Static objects may have malloc'd before tsan completes
		385	// initialization, and may believe returned ptrs to be valid.
		386	if (!b)
		387	return 0; // Not a valid pointer.
		388	if (b->siz == 0)
		389	return 1; // Zero-sized allocations are actually 1 byte.
		390	return b->siz;
		391	}
		392
323	void invoke_malloc_hook(void *ptr, uptr size) {	393	void invoke_malloc_hook(void *ptr, uptr size) {
324	ThreadState *thr = cur_thread();	394	ThreadState *thr = cur_thread();
325	if (ctx == 0 \|\| !ctx->initialized \|\| thr->ignore_interceptors)	395	if (ctx == 0 \|\| !ctx->initialized \|\| thr->ignore_interceptors)
326	return;	396	return;
327	__sanitizer_malloc_hook(ptr, size);
328	RunMallocHooks(ptr, size);	397	RunMallocHooks(ptr, size);
329	}	398	}
330		399
...	@@ -332,25 +401,26 @@ void invoke_free_hook(void *ptr) {	...	@@ -332,25 +401,26 @@ void invoke_free_hook(void *ptr) {
332	ThreadState *thr = cur_thread();	401	ThreadState *thr = cur_thread();
333	if (ctx == 0 \|\| !ctx->initialized \|\| thr->ignore_interceptors)	402	if (ctx == 0 \|\| !ctx->initialized \|\| thr->ignore_interceptors)
334	return;	403	return;
335	__sanitizer_free_hook(ptr);
336	RunFreeHooks(ptr);	404	RunFreeHooks(ptr);
337	}	405	}
338		406
339	void *internal_alloc(MBlockType typ, uptr sz) {	407	void *Alloc(uptr sz) {
340	ThreadState *thr = cur_thread();	408	ThreadState *thr = cur_thread();
341	if (thr->nomalloc) {	409	if (thr->nomalloc) {
342	thr->nomalloc = 0; // CHECK calls internal_malloc().	410	thr->nomalloc = 0; // CHECK calls internal_malloc().
343	CHECK(0);	411	CHECK(0);
344	}	412	}
		413	InternalAllocAccess();
345	return InternalAlloc(sz, &thr->proc()->internal_alloc_cache);	414	return InternalAlloc(sz, &thr->proc()->internal_alloc_cache);
346	}	415	}
347		416
348	void internal_free(void *p) {	417	void FreeImpl(void *p) {
349	ThreadState *thr = cur_thread();	418	ThreadState *thr = cur_thread();
350	if (thr->nomalloc) {	419	if (thr->nomalloc) {
351	thr->nomalloc = 0; // CHECK calls internal_malloc().	420	thr->nomalloc = 0; // CHECK calls internal_malloc().
352	CHECK(0);	421	CHECK(0);
353	}	422	}
		423	InternalAllocAccess();
354	InternalFree(p, &thr->proc()->internal_alloc_cache);	424	InternalFree(p, &thr->proc()->internal_alloc_cache);
355	}	425	}
356		426
...	@@ -387,14 +457,27 @@ int __sanitizer_get_ownership(const void *p) {	...	@@ -387,14 +457,27 @@ int __sanitizer_get_ownership(const void *p) {
387	return allocator()->GetBlockBegin(p) != 0;	457	return allocator()->GetBlockBegin(p) != 0;
388	}	458	}
389		459
		460	const void __sanitizer_get_allocated_begin(const void p) {
		461	return user_alloc_begin(p);
		462	}
		463
390	uptr __sanitizer_get_allocated_size(const void *p) {	464	uptr __sanitizer_get_allocated_size(const void *p) {
391	return user_alloc_usable_size(p);	465	return user_alloc_usable_size(p);
392	}	466	}
393		467
		468	uptr __sanitizer_get_allocated_size_fast(const void *p) {
		469	DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
		470	uptr ret = user_alloc_usable_size_fast(p);
		471	DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
		472	return ret;
		473	}
		474
		475	void __sanitizer_purge_allocator() {
		476	allocator()->ForceReleaseToOS();
		477	}
		478
394	void __tsan_on_thread_idle() {	479	void __tsan_on_thread_idle() {
395	ThreadState *thr = cur_thread();	480	ThreadState *thr = cur_thread();
396	thr->clock.ResetCached(&thr->proc()->clock_cache);
397	thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
398	allocator()->SwallowCache(&thr->proc()->alloc_cache);	481	allocator()->SwallowCache(&thr->proc()->alloc_cache);
399	internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache);	482	internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache);
400	ctx->metamap.OnProcIdle(thr->proc());	483	ctx->metamap.OnProcIdle(thr->proc());

lib/tsan/tsan_mman.h+22-31

 @@ -24,6 +24,10 @@ void ReplaceSystemMalloc();
 void AllocatorProcStart(Processor *proc);
 void AllocatorProcFinish(Processor *proc);
 void AllocatorPrintStats();
+void AllocatorLock();
+void AllocatorUnlock();
+void GlobalProcessorLock();
+void GlobalProcessorUnlock();
 // For user allocations.
 void *user_alloc_internal(ThreadState *thr, uptr pc, uptr sz,
 @@ -47,42 +51,29 @@ uptr user_alloc_usable_size(const void *p);
 void invoke_malloc_hook(void *ptr, uptr size);
 void invoke_free_hook(void *ptr);
-enum MBlockType {
+// For internal data structures.
-  MBlockScopedBuf,
+void *Alloc(uptr sz);
-  MBlockString,
+void FreeImpl(void *p);
-  MBlockStackTrace,
-  MBlockShadowStack,
-  MBlockSync,
-  MBlockClock,
-  MBlockThreadContex,
-  MBlockDeadInfo,
-  MBlockRacyStacks,
-  MBlockRacyAddresses,
-  MBlockAtExit,
-  MBlockFlag,
-  MBlockReport,
-  MBlockReportMop,
-  MBlockReportThread,
-  MBlockReportMutex,
-  MBlockReportLoc,
-  MBlockReportStack,
-  MBlockSuppression,
-  MBlockExpectRace,
-  MBlockSignal,
-  MBlockJmpBuf,
-  // This must be the last.
+template <typename T, typename... Args>
-  MBlockTypeCount
+T *New(Args &&...args) {
-};
+  return new (Alloc(sizeof(T))) T(static_cast<Args &&>(args)...);
+}
-// For internal data structures.
+template <typename T>
-void *internal_alloc(MBlockType typ, uptr sz);
+void Free(T *&p) {
-void internal_free(void *p);
+  if (p == nullptr)
+    return;
+  FreeImpl(p);
+  p = nullptr;
+}
 template <typename T>
-void DestroyAndFree(T *p) {
+void DestroyAndFree(T *&p) {
+  if (p == nullptr)
+    return;
   p->~T();
-  internal_free(p);
+  Free(p);
 }
 }  // namespace __tsan

lib/tsan/tsan_mutexset.cpp+20-28

 @@ -10,66 +10,55 @@
 //
 //===----------------------------------------------------------------------===//
 #include "tsan_mutexset.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
 #include "tsan_rtl.h"
 namespace __tsan {
-const uptr MutexSet::kMaxSize;
 MutexSet::MutexSet() {
-  size_ = 0;
-  internal_memset(&descs_, 0, sizeof(descs_));
 }
-void MutexSet::Add(u64 id, bool write, u64 epoch) {
+void MutexSet::Reset() { internal_memset(this, 0, sizeof(*this)); }
+void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) {
   // Look up existing mutex with the same id.
   for (uptr i = 0; i < size_; i++) {
-    if (descs_[i].id == id) {
+    if (descs_[i].addr == addr) {
       descs_[i].count++;
-      descs_[i].epoch = epoch;
+      descs_[i].seq = seq_++;
       return;
     }
   }
   // On overflow, find the oldest mutex and drop it.
   if (size_ == kMaxSize) {
-    u64 minepoch = (u64)-1;
+    uptr min = 0;
-    u64 mini = (u64)-1;
     for (uptr i = 0; i < size_; i++) {
-      if (descs_[i].epoch < minepoch) {
+      if (descs_[i].seq < descs_[min].seq)
-        minepoch = descs_[i].epoch;
+        min = i;
-        mini = i;
-      }
     }
-    RemovePos(mini);
+    RemovePos(min);
     CHECK_EQ(size_, kMaxSize - 1);
   }
   // Add new mutex descriptor.
-  descs_[size_].id = id;
+  descs_[size_].addr = addr;
+  descs_[size_].stack_id = stack_id;
   descs_[size_].write = write;
-  descs_[size_].epoch = epoch;
+  descs_[size_].seq = seq_++;
   descs_[size_].count = 1;
   size_++;
 }
-void MutexSet::Del(u64 id, bool write) {
+void MutexSet::DelAddr(uptr addr, bool destroy) {
   for (uptr i = 0; i < size_; i++) {
-    if (descs_[i].id == id) {
+    if (descs_[i].addr == addr) {
-      if (--descs_[i].count == 0)
+      if (destroy || --descs_[i].count == 0)
         RemovePos(i);
       return;
     }
   }
 }
-void MutexSet::Remove(u64 id) {
-  for (uptr i = 0; i < size_; i++) {
-    if (descs_[i].id == id) {
-      RemovePos(i);
-      return;
-    }
-  }
-}
 void MutexSet::RemovePos(uptr i) {
   CHECK_LT(i, size_);
   descs_[i] = descs_[size_ - 1];
 @@ -85,4 +74,7 @@ MutexSet::Desc MutexSet::Get(uptr i) const {
   return descs_[i];
 }
+DynamicMutexSet::DynamicMutexSet() : ptr_(New<MutexSet>()) {}
+DynamicMutexSet::~DynamicMutexSet() { DestroyAndFree(ptr_); }
 }  // namespace __tsan

lib/tsan/tsan_mutexset.h+41-19

 @@ -21,34 +21,55 @@ class MutexSet {
  public:
   // Holds limited number of mutexes.
   // The oldest mutexes are discarded on overflow.
-  static const uptr kMaxSize = 16;
+  static constexpr uptr kMaxSize = 16;
   struct Desc {
-    u64 id;
+    uptr addr;
-    u64 epoch;
+    StackID stack_id;
-    int count;
+    u32 seq;
+    u32 count;
     bool write;
+    Desc() { internal_memset(this, 0, sizeof(*this)); }
+    Desc(const Desc& other) { *this = other; }
+    Desc& operator=(const MutexSet::Desc& other) {
+      internal_memcpy(this, &other, sizeof(*this));
+      return *this;
+    }
   };
   MutexSet();
-  // The 'id' is obtained from SyncVar::GetId().
+  void Reset();
-  void Add(u64 id, bool write, u64 epoch);
+  void AddAddr(uptr addr, StackID stack_id, bool write);
-  void Del(u64 id, bool write);
+  void DelAddr(uptr addr, bool destroy = false);
-  void Remove(u64 id);  // Removes the mutex completely (if it's destroyed).
   uptr Size() const;
   Desc Get(uptr i) const;
-  void operator=(const MutexSet &other) {
-    internal_memcpy(this, &other, sizeof(*this));
-  }
  private:
 #if !SANITIZER_GO
-  uptr size_;
+  u32 seq_ = 0;
+  uptr size_ = 0;
   Desc descs_[kMaxSize];
-#endif
   void RemovePos(uptr i);
-  MutexSet(const MutexSet&);
+#endif
+};
+// MutexSet is too large to live on stack.
+// DynamicMutexSet can be use used to create local MutexSet's.
+class DynamicMutexSet {
+ public:
+  DynamicMutexSet();
+  ~DynamicMutexSet();
+  MutexSet* operator->() { return ptr_; }
+  operator MutexSet*() { return ptr_; }
+  DynamicMutexSet(const DynamicMutexSet&) = delete;
+  DynamicMutexSet& operator=(const DynamicMutexSet&) = delete;
+ private:
+  MutexSet* ptr_;
+#if SANITIZER_GO
+  MutexSet set_;
+#endif
 };
 // Go does not have mutexes, so do not spend memory and time.
 @@ -56,12 +77,13 @@ class MutexSet {
 // in different goroutine).
 #if SANITIZER_GO
 MutexSet::MutexSet() {}
-void MutexSet::Add(u64 id, bool write, u64 epoch) {}
+void MutexSet::Reset() {}
-void MutexSet::Del(u64 id, bool write) {}
+void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) {}
-void MutexSet::Remove(u64 id) {}
+void MutexSet::DelAddr(uptr addr, bool destroy) {}
-void MutexSet::RemovePos(uptr i) {}
 uptr MutexSet::Size() const { return 0; }
 MutexSet::Desc MutexSet::Get(uptr i) const { return Desc(); }
+DynamicMutexSet::DynamicMutexSet() : ptr_(&set_) {}
+DynamicMutexSet::~DynamicMutexSet() {}
 #endif
 }  // namespace __tsan

lib/tsan/tsan_new_delete.cpp created+199

...	@@ -0,0 +1,199 @@
1	//===-- tsan_new_delete.cpp ---------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	// Interceptors for operators new and delete.
12	//===----------------------------------------------------------------------===//
13	#include "interception/interception.h"
14	#include "sanitizer_common/sanitizer_allocator.h"
15	#include "sanitizer_common/sanitizer_allocator_report.h"
16	#include "sanitizer_common/sanitizer_internal_defs.h"
17	#include "tsan_interceptors.h"
18	#include "tsan_rtl.h"
19
20	using namespace __tsan;
21
22	namespace std {
23	struct nothrow_t {};
24	enum class align_val_t: __sanitizer::uptr {};
25	} // namespace std
26
27	DECLARE_REAL(void *, malloc, uptr size)
28	DECLARE_REAL(void, free, void *ptr)
29
30	// TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
31	#define OPERATOR_NEW_BODY(mangled_name, nothrow) \
32	if (in_symbolizer()) \
33	return InternalAlloc(size); \
34	void *p = 0; \
35	{ \
36	SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
37	p = user_alloc(thr, pc, size); \
38	if (!nothrow && UNLIKELY(!p)) { \
39	GET_STACK_TRACE_FATAL(thr, pc); \
40	ReportOutOfMemory(size, &stack); \
41	} \
42	} \
43	invoke_malloc_hook(p, size); \
44	return p;
45
46	#define OPERATOR_NEW_BODY_ALIGN(mangled_name, nothrow) \
47	if (in_symbolizer()) \
48	return InternalAlloc(size, nullptr, (uptr)align); \
49	void *p = 0; \
50	{ \
51	SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
52	p = user_memalign(thr, pc, (uptr)align, size); \
53	if (!nothrow && UNLIKELY(!p)) { \
54	GET_STACK_TRACE_FATAL(thr, pc); \
55	ReportOutOfMemory(size, &stack); \
56	} \
57	} \
58	invoke_malloc_hook(p, size); \
59	return p;
60
61	SANITIZER_INTERFACE_ATTRIBUTE
62	void *operator new(__sanitizer::uptr size);
63	void *operator new(__sanitizer::uptr size) {
64	OPERATOR_NEW_BODY(_Znwm, false /nothrow/);
65	}
66
67	SANITIZER_INTERFACE_ATTRIBUTE
68	void *operator new[](__sanitizer::uptr size);
69	void *operator new[](__sanitizer::uptr size) {
70	OPERATOR_NEW_BODY(_Znam, false /nothrow/);
71	}
72
73	SANITIZER_INTERFACE_ATTRIBUTE
74	void *operator new(__sanitizer::uptr size, std::nothrow_t const&);
75	void *operator new(__sanitizer::uptr size, std::nothrow_t const&) {
76	OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t, true /nothrow/);
77	}
78
79	SANITIZER_INTERFACE_ATTRIBUTE
80	void *operator new[](__sanitizer::uptr size, std::nothrow_t const&);
81	void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) {
82	OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t, true /nothrow/);
83	}
84
85	SANITIZER_INTERFACE_ATTRIBUTE
86	void *operator new(__sanitizer::uptr size, std::align_val_t align);
87	void *operator new(__sanitizer::uptr size, std::align_val_t align) {
88	OPERATOR_NEW_BODY_ALIGN(_ZnwmSt11align_val_t, false /nothrow/);
89	}
90
91	SANITIZER_INTERFACE_ATTRIBUTE
92	void *operator new[](__sanitizer::uptr size, std::align_val_t align);
93	void *operator new[](__sanitizer::uptr size, std::align_val_t align) {
94	OPERATOR_NEW_BODY_ALIGN(_ZnamSt11align_val_t, false /nothrow/);
95	}
96
97	SANITIZER_INTERFACE_ATTRIBUTE
98	void *operator new(__sanitizer::uptr size, std::align_val_t align,
99	std::nothrow_t const&);
100	void *operator new(__sanitizer::uptr size, std::align_val_t align,
101	std::nothrow_t const&) {
102	OPERATOR_NEW_BODY_ALIGN(_ZnwmSt11align_val_tRKSt9nothrow_t,
103	true /nothrow/);
104	}
105
106	SANITIZER_INTERFACE_ATTRIBUTE
107	void *operator new[](__sanitizer::uptr size, std::align_val_t align,
108	std::nothrow_t const&);
109	void *operator new[](__sanitizer::uptr size, std::align_val_t align,
110	std::nothrow_t const&) {
111	OPERATOR_NEW_BODY_ALIGN(_ZnamSt11align_val_tRKSt9nothrow_t,
112	true /nothrow/);
113	}
114
115	#define OPERATOR_DELETE_BODY(mangled_name) \
116	if (ptr == 0) return; \
117	if (in_symbolizer()) \
118	return InternalFree(ptr); \
119	invoke_free_hook(ptr); \
120	SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \
121	user_free(thr, pc, ptr);
122
123	SANITIZER_INTERFACE_ATTRIBUTE
124	void operator delete(void *ptr) NOEXCEPT;
125	void operator delete(void *ptr) NOEXCEPT {
126	OPERATOR_DELETE_BODY(_ZdlPv);
127	}
128
129	SANITIZER_INTERFACE_ATTRIBUTE
130	void operator delete[](void *ptr) NOEXCEPT;
131	void operator delete[](void *ptr) NOEXCEPT {
132	OPERATOR_DELETE_BODY(_ZdaPv);
133	}
134
135	SANITIZER_INTERFACE_ATTRIBUTE
136	void operator delete(void *ptr, std::nothrow_t const&);
137	void operator delete(void *ptr, std::nothrow_t const&) {
138	OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t);
139	}
140
141	SANITIZER_INTERFACE_ATTRIBUTE
142	void operator delete[](void *ptr, std::nothrow_t const&);
143	void operator delete[](void *ptr, std::nothrow_t const&) {
144	OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t);
145	}
146
147	SANITIZER_INTERFACE_ATTRIBUTE
148	void operator delete(void *ptr, __sanitizer::uptr size) NOEXCEPT;
149	void operator delete(void *ptr, __sanitizer::uptr size) NOEXCEPT {
150	OPERATOR_DELETE_BODY(_ZdlPvm);
151	}
152
153	SANITIZER_INTERFACE_ATTRIBUTE
154	void operator delete[](void *ptr, __sanitizer::uptr size) NOEXCEPT;
155	void operator delete[](void *ptr, __sanitizer::uptr size) NOEXCEPT {
156	OPERATOR_DELETE_BODY(_ZdaPvm);
157	}
158
159	SANITIZER_INTERFACE_ATTRIBUTE
160	void operator delete(void *ptr, std::align_val_t align) NOEXCEPT;
161	void operator delete(void *ptr, std::align_val_t align) NOEXCEPT {
162	OPERATOR_DELETE_BODY(_ZdlPvSt11align_val_t);
163	}
164
165	SANITIZER_INTERFACE_ATTRIBUTE
166	void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT;
167	void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT {
168	OPERATOR_DELETE_BODY(_ZdaPvSt11align_val_t);
169	}
170
171	SANITIZER_INTERFACE_ATTRIBUTE
172	void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&);
173	void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&) {
174	OPERATOR_DELETE_BODY(_ZdlPvSt11align_val_tRKSt9nothrow_t);
175	}
176
177	SANITIZER_INTERFACE_ATTRIBUTE
178	void operator delete[](void *ptr, std::align_val_t align,
179	std::nothrow_t const&);
180	void operator delete[](void *ptr, std::align_val_t align,
181	std::nothrow_t const&) {
182	OPERATOR_DELETE_BODY(_ZdaPvSt11align_val_tRKSt9nothrow_t);
183	}
184
185	SANITIZER_INTERFACE_ATTRIBUTE
186	void operator delete(void *ptr, __sanitizer::uptr size,
187	std::align_val_t align) NOEXCEPT;
188	void operator delete(void *ptr, __sanitizer::uptr size,
189	std::align_val_t align) NOEXCEPT {
190	OPERATOR_DELETE_BODY(_ZdlPvmSt11align_val_t);
191	}
192
193	SANITIZER_INTERFACE_ATTRIBUTE
194	void operator delete[](void *ptr, __sanitizer::uptr size,
195	std::align_val_t align) NOEXCEPT;
196	void operator delete[](void *ptr, __sanitizer::uptr size,
197	std::align_val_t align) NOEXCEPT {
198	OPERATOR_DELETE_BODY(_ZdaPvmSt11align_val_t);
199	}

lib/tsan/tsan_platform.h+568-841

...	@@ -18,38 +18,42 @@	...	@@ -18,38 +18,42 @@
18	# error "Only 64-bit is supported"	18	# error "Only 64-bit is supported"
19	#endif	19	#endif
20		20
		21	#include "sanitizer_common/sanitizer_common.h"
21	#include "tsan_defs.h"	22	#include "tsan_defs.h"
22	#include "tsan_trace.h"
23		23
24	namespace __tsan {	24	namespace __tsan {
25		25
26	#if defined(__x86_64__)	26	enum {
27	#define HAS_48_BIT_ADDRESS_SPACE 1	27	// App memory is not mapped onto shadow memory range.
28	#elif SANITIZER_IOSSIM // arm64 iOS simulators (order of #if matters)	28	kBrokenMapping = 1 << 0,
29	#define HAS_48_BIT_ADDRESS_SPACE 1	29	// Mapping app memory and back does not produce the same address,
30	#elif SANITIZER_IOS // arm64 iOS devices (order of #if matters)	30	// this can lead to wrong addresses in reports and potentially
31	#define HAS_48_BIT_ADDRESS_SPACE 0	31	// other bad consequences.
32	#elif SANITIZER_MAC // arm64 macOS (order of #if matters)	32	kBrokenReverseMapping = 1 << 1,
33	#define HAS_48_BIT_ADDRESS_SPACE 1	33	// Mapping is non-linear for linear user range.
34	#else	34	// This is bad and can lead to unpredictable memory corruptions, etc
35	#define HAS_48_BIT_ADDRESS_SPACE 0	35	// because range access functions assume linearity.
36	#endif	36	kBrokenLinearity = 1 << 2,
37		37	// Meta for an app region overlaps with the meta of another app region.
38	#if !SANITIZER_GO	38	// This is determined by recomputing the individual meta regions for
		39	// each app region.
		40	//
		41	// N.B. There is no "kBrokenReverseMetaMapping" constant because there
		42	// is no MetaToMem function. However, note that (!kBrokenLinearity
		43	// && !kBrokenAliasedMetas) implies that MemToMeta is invertible.
		44	kBrokenAliasedMetas = 1 << 3,
		45	};
39		46
40	#if HAS_48_BIT_ADDRESS_SPACE
41	/*	47	/*
42	C/C++ on linux/x86_64 and freebsd/x86_64	48	C/C++ on linux/x86_64 and freebsd/x86_64
43	0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)	49	0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
44	0040 0000 0000 - 0100 0000 0000: -	50	0040 0000 0000 - 0100 0000 0000: -
45	0100 0000 0000 - 2000 0000 0000: shadow	51	0100 0000 0000 - 1000 0000 0000: shadow
46	2000 0000 0000 - 3000 0000 0000: -	52	1000 0000 0000 - 3000 0000 0000: -
47	3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)	53	3000 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects)
48	4000 0000 0000 - 5500 0000 0000: -	54	3400 0000 0000 - 5500 0000 0000: -
49	5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels	55	5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels
50	5680 0000 0000 - 6000 0000 0000: -	56	5680 0000 0000 - 7d00 0000 0000: -
51	6000 0000 0000 - 6200 0000 0000: traces
52	6200 0000 0000 - 7d00 0000 0000: -
53	7b00 0000 0000 - 7c00 0000 0000: heap	57	7b00 0000 0000 - 7c00 0000 0000: heap
54	7c00 0000 0000 - 7e80 0000 0000: -	58	7c00 0000 0000 - 7e80 0000 0000: -
55	7e80 0000 0000 - 8000 0000 0000: modules and main thread stack	59	7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
...	@@ -65,15 +69,12 @@ C/C++ on netbsd/amd64 can reuse the same mapping:	...	@@ -65,15 +69,12 @@ C/C++ on netbsd/amd64 can reuse the same mapping:
65	* Stack on NetBSD/amd64 has prereserved 128MB.	69	* Stack on NetBSD/amd64 has prereserved 128MB.
66	* Heap grows downwards (top-down).	70	* Heap grows downwards (top-down).
67	* ASLR must be disabled per-process or globally.	71	* ASLR must be disabled per-process or globally.
68
69	*/	72	*/
70	struct Mapping {	73	struct Mapping48AddressSpace {
71	static const uptr kMetaShadowBeg = 0x300000000000ull;	74	static const uptr kMetaShadowBeg = 0x300000000000ull;
72	static const uptr kMetaShadowEnd = 0x340000000000ull;	75	static const uptr kMetaShadowEnd = 0x340000000000ull;
73	static const uptr kTraceMemBeg = 0x600000000000ull;
74	static const uptr kTraceMemEnd = 0x620000000000ull;
75	static const uptr kShadowBeg = 0x010000000000ull;	76	static const uptr kShadowBeg = 0x010000000000ull;
76	static const uptr kShadowEnd = 0x200000000000ull;	77	static const uptr kShadowEnd = 0x100000000000ull;
77	static const uptr kHeapMemBeg = 0x7b0000000000ull;	78	static const uptr kHeapMemBeg = 0x7b0000000000ull;
78	static const uptr kHeapMemEnd = 0x7c0000000000ull;	79	static const uptr kHeapMemEnd = 0x7c0000000000ull;
79	static const uptr kLoAppMemBeg = 0x000000001000ull;	80	static const uptr kLoAppMemBeg = 0x000000001000ull;
...	@@ -82,36 +83,32 @@ struct Mapping {	...	@@ -82,36 +83,32 @@ struct Mapping {
82	static const uptr kMidAppMemEnd = 0x568000000000ull;	83	static const uptr kMidAppMemEnd = 0x568000000000ull;
83	static const uptr kHiAppMemBeg = 0x7e8000000000ull;	84	static const uptr kHiAppMemBeg = 0x7e8000000000ull;
84	static const uptr kHiAppMemEnd = 0x800000000000ull;	85	static const uptr kHiAppMemEnd = 0x800000000000ull;
85	static const uptr kAppMemMsk = 0x780000000000ull;	86	static const uptr kShadowMsk = 0x780000000000ull;
86	static const uptr kAppMemXor = 0x040000000000ull;	87	static const uptr kShadowXor = 0x040000000000ull;
		88	static const uptr kShadowAdd = 0x000000000000ull;
87	static const uptr kVdsoBeg = 0xf000000000000000ull;	89	static const uptr kVdsoBeg = 0xf000000000000000ull;
88	};	90	};
89		91
90	#define TSAN_MID_APP_RANGE 1
91	#elif defined(__mips64)
92	/*	92	/*
93	C/C++ on linux/mips64 (40-bit VMA)	93	C/C++ on linux/mips64 (40-bit VMA)
94	0000 0000 00 - 0100 0000 00: - (4 GB)	94	0000 0000 00 - 0100 0000 00: - (4 GB)
95	0100 0000 00 - 0200 0000 00: main binary (4 GB)	95	0100 0000 00 - 0200 0000 00: main binary (4 GB)
96	0200 0000 00 - 2000 0000 00: - (120 GB)	96	0200 0000 00 - 1200 0000 00: - (64 GB)
97	2000 0000 00 - 4000 0000 00: shadow (128 GB)	97	1200 0000 00 - 2200 0000 00: shadow (64 GB)
		98	2200 0000 00 - 4000 0000 00: - (120 GB)
98	4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects) (64 GB)	99	4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects) (64 GB)
99	5000 0000 00 - aa00 0000 00: - (360 GB)	100	5000 0000 00 - aa00 0000 00: - (360 GB)
100	aa00 0000 00 - ab00 0000 00: main binary (PIE) (4 GB)	101	aa00 0000 00 - ab00 0000 00: main binary (PIE) (4 GB)
101	ab00 0000 00 - b000 0000 00: - (20 GB)	102	ab00 0000 00 - fe00 0000 00: - (332 GB)
102	b000 0000 00 - b200 0000 00: traces (8 GB)
103	b200 0000 00 - fe00 0000 00: - (304 GB)
104	fe00 0000 00 - ff00 0000 00: heap (4 GB)	103	fe00 0000 00 - ff00 0000 00: heap (4 GB)
105	ff00 0000 00 - ff80 0000 00: - (2 GB)	104	ff00 0000 00 - ff80 0000 00: - (2 GB)
106	ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB)	105	ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB)
107	*/	106	*/
108	struct Mapping40 {	107	struct MappingMips64_40 {
109	static const uptr kMetaShadowBeg = 0x4000000000ull;	108	static const uptr kMetaShadowBeg = 0x4000000000ull;
110	static const uptr kMetaShadowEnd = 0x5000000000ull;	109	static const uptr kMetaShadowEnd = 0x5000000000ull;
111	static const uptr kTraceMemBeg = 0xb000000000ull;	110	static const uptr kShadowBeg = 0x1200000000ull;
112	static const uptr kTraceMemEnd = 0xb200000000ull;	111	static const uptr kShadowEnd = 0x2200000000ull;
113	static const uptr kShadowBeg = 0x2000000000ull;
114	static const uptr kShadowEnd = 0x4000000000ull;
115	static const uptr kHeapMemBeg = 0xfe00000000ull;	112	static const uptr kHeapMemBeg = 0xfe00000000ull;
116	static const uptr kHeapMemEnd = 0xff00000000ull;	113	static const uptr kHeapMemEnd = 0xff00000000ull;
117	static const uptr kLoAppMemBeg = 0x0100000000ull;	114	static const uptr kLoAppMemBeg = 0x0100000000ull;
...	@@ -120,152 +117,170 @@ struct Mapping40 {	...	@@ -120,152 +117,170 @@ struct Mapping40 {
120	static const uptr kMidAppMemEnd = 0xab00000000ull;	117	static const uptr kMidAppMemEnd = 0xab00000000ull;
121	static const uptr kHiAppMemBeg = 0xff80000000ull;	118	static const uptr kHiAppMemBeg = 0xff80000000ull;
122	static const uptr kHiAppMemEnd = 0xffffffffffull;	119	static const uptr kHiAppMemEnd = 0xffffffffffull;
123	static const uptr kAppMemMsk = 0xf800000000ull;	120	static const uptr kShadowMsk = 0xf800000000ull;
124	static const uptr kAppMemXor = 0x0800000000ull;	121	static const uptr kShadowXor = 0x0800000000ull;
		122	static const uptr kShadowAdd = 0x0000000000ull;
125	static const uptr kVdsoBeg = 0xfffff00000ull;	123	static const uptr kVdsoBeg = 0xfffff00000ull;
126	};	124	};
127		125
128	#define TSAN_MID_APP_RANGE 1
129	#define TSAN_RUNTIME_VMA 1
130	#elif defined(__aarch64__) && defined(__APPLE__)
131	/*	126	/*
132	C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)	127	C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
133	0000 0000 00 - 0100 0000 00: - (4 GB)	128	0000 0000 00 - 0100 0000 00: - (4 GB)
134	0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB)	129	0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB)
135	0200 0000 00 - 0300 0000 00: heap (4 GB)	130	0200 0000 00 - 0300 0000 00: heap (4 GB)
136	0300 0000 00 - 0400 0000 00: - (4 GB)	131	0300 0000 00 - 0400 0000 00: - (4 GB)
137	0400 0000 00 - 0c00 0000 00: shadow memory (32 GB)	132	0400 0000 00 - 0800 0000 00: shadow memory (16 GB)
138	0c00 0000 00 - 0d00 0000 00: - (4 GB)	133	0800 0000 00 - 0d00 0000 00: - (20 GB)
139	0d00 0000 00 - 0e00 0000 00: metainfo (4 GB)	134	0d00 0000 00 - 0e00 0000 00: metainfo (4 GB)
140	0e00 0000 00 - 0f00 0000 00: - (4 GB)	135	0e00 0000 00 - 1000 0000 00: -
141	0f00 0000 00 - 0fc0 0000 00: traces (3 GB)
142	0fc0 0000 00 - 1000 0000 00: -
143	*/	136	*/
144	struct Mapping {	137	struct MappingAppleAarch64 {
145	static const uptr kLoAppMemBeg = 0x0100000000ull;	138	static const uptr kLoAppMemBeg = 0x0100000000ull;
146	static const uptr kLoAppMemEnd = 0x0200000000ull;	139	static const uptr kLoAppMemEnd = 0x0200000000ull;
147	static const uptr kHeapMemBeg = 0x0200000000ull;	140	static const uptr kHeapMemBeg = 0x0200000000ull;
148	static const uptr kHeapMemEnd = 0x0300000000ull;	141	static const uptr kHeapMemEnd = 0x0300000000ull;
149	static const uptr kShadowBeg = 0x0400000000ull;	142	static const uptr kShadowBeg = 0x0400000000ull;
150	static const uptr kShadowEnd = 0x0c00000000ull;	143	static const uptr kShadowEnd = 0x0800000000ull;
151	static const uptr kMetaShadowBeg = 0x0d00000000ull;	144	static const uptr kMetaShadowBeg = 0x0d00000000ull;
152	static const uptr kMetaShadowEnd = 0x0e00000000ull;	145	static const uptr kMetaShadowEnd = 0x0e00000000ull;
153	static const uptr kTraceMemBeg = 0x0f00000000ull;
154	static const uptr kTraceMemEnd = 0x0fc0000000ull;
155	static const uptr kHiAppMemBeg = 0x0fc0000000ull;	146	static const uptr kHiAppMemBeg = 0x0fc0000000ull;
156	static const uptr kHiAppMemEnd = 0x0fc0000000ull;	147	static const uptr kHiAppMemEnd = 0x0fc0000000ull;
157	static const uptr kAppMemMsk = 0x0ull;	148	static const uptr kShadowMsk = 0x0ull;
158	static const uptr kAppMemXor = 0x0ull;	149	static const uptr kShadowXor = 0x0ull;
		150	static const uptr kShadowAdd = 0x0200000000ull;
159	static const uptr kVdsoBeg = 0x7000000000000000ull;	151	static const uptr kVdsoBeg = 0x7000000000000000ull;
		152	static const uptr kMidAppMemBeg = 0;
		153	static const uptr kMidAppMemEnd = 0;
160	};	154	};
161		155
162	#elif defined(__aarch64__) && !defined(__APPLE__)
163	// AArch64 supports multiple VMA which leads to multiple address transformation
164	// functions. To support these multiple VMAS transformations and mappings TSAN
165	// runtime for AArch64 uses an external memory read (vmaSize) to select which
166	// mapping to use. Although slower, it make a same instrumented binary run on
167	// multiple kernels.
168
169	/*	156	/*
170	C/C++ on linux/aarch64 (39-bit VMA)	157	C/C++ on linux/aarch64 (39-bit VMA)
171	0000 0010 00 - 0100 0000 00: main binary	158	0000 0010 00 - 0500 0000 00: main binary (20 GB)
172	0100 0000 00 - 0800 0000 00: -	159	0100 0000 00 - 2000 0000 00: -
173	0800 0000 00 - 2000 0000 00: shadow memory	160	2000 0000 00 - 4000 0000 00: shadow memory (128 GB)
174	2000 0000 00 - 3100 0000 00: -	161	4000 0000 00 - 4800 0000 00: metainfo (32 GB)
175	3100 0000 00 - 3400 0000 00: metainfo	162	4800 0000 00 - 5500 0000 00: -
176	3400 0000 00 - 5500 0000 00: -	163	5500 0000 00 - 5a00 0000 00: main binary (PIE) (20 GB)
177	5500 0000 00 - 5600 0000 00: main binary (PIE)	164	5600 0000 00 - 7c00 0000 00: -
178	5600 0000 00 - 6000 0000 00: -	165	7a00 0000 00 - 7d00 0000 00: heap (12 GB)
179	6000 0000 00 - 6200 0000 00: traces	166	7d00 0000 00 - 7fff ffff ff: modules and main thread stack (12 GB)
180	6200 0000 00 - 7d00 0000 00: -
181	7c00 0000 00 - 7d00 0000 00: heap
182	7d00 0000 00 - 7fff ffff ff: modules and main thread stack
183	*/	167	*/
184	struct Mapping39 {	168	struct MappingAarch64_39 {
185	static const uptr kLoAppMemBeg = 0x0000001000ull;	169	static const uptr kLoAppMemBeg = 0x0000001000ull;
186	static const uptr kLoAppMemEnd = 0x0100000000ull;	170	static const uptr kLoAppMemEnd = 0x0500000000ull;
187	static const uptr kShadowBeg = 0x0800000000ull;	171	static const uptr kShadowBeg = 0x2000000000ull;
188	static const uptr kShadowEnd = 0x2000000000ull;	172	static const uptr kShadowEnd = 0x4000000000ull;
189	static const uptr kMetaShadowBeg = 0x3100000000ull;	173	static const uptr kMetaShadowBeg = 0x4000000000ull;
190	static const uptr kMetaShadowEnd = 0x3400000000ull;	174	static const uptr kMetaShadowEnd = 0x4800000000ull;
191	static const uptr kMidAppMemBeg = 0x5500000000ull;	175	static const uptr kMidAppMemBeg = 0x5500000000ull;
192	static const uptr kMidAppMemEnd = 0x5600000000ull;	176	static const uptr kMidAppMemEnd = 0x5a00000000ull;
193	static const uptr kTraceMemBeg = 0x6000000000ull;	177	static const uptr kHeapMemBeg = 0x7a00000000ull;
194	static const uptr kTraceMemEnd = 0x6200000000ull;
195	static const uptr kHeapMemBeg = 0x7c00000000ull;
196	static const uptr kHeapMemEnd = 0x7d00000000ull;	178	static const uptr kHeapMemEnd = 0x7d00000000ull;
197	static const uptr kHiAppMemBeg = 0x7e00000000ull;	179	static const uptr kHiAppMemBeg = 0x7d00000000ull;
198	static const uptr kHiAppMemEnd = 0x7fffffffffull;	180	static const uptr kHiAppMemEnd = 0x7fffffffffull;
199	static const uptr kAppMemMsk = 0x7800000000ull;	181	static const uptr kShadowMsk = 0x7000000000ull;
200	static const uptr kAppMemXor = 0x0200000000ull;	182	static const uptr kShadowXor = 0x1000000000ull;
		183	static const uptr kShadowAdd = 0x0000000000ull;
201	static const uptr kVdsoBeg = 0x7f00000000ull;	184	static const uptr kVdsoBeg = 0x7f00000000ull;
202	};	185	};
203		186
204	/*	187	/*
205	C/C++ on linux/aarch64 (42-bit VMA)	188	C/C++ on linux/aarch64 (42-bit VMA)
206	00000 0010 00 - 01000 0000 00: main binary	189	00000 0010 00 - 02000 0000 00: main binary (128 GB)
207	01000 0000 00 - 10000 0000 00: -	190	02000 0000 00 - 08000 0000 00: -
208	10000 0000 00 - 20000 0000 00: shadow memory	191	10000 0000 00 - 20000 0000 00: shadow memory (1024 GB)
209	20000 0000 00 - 26000 0000 00: -	192	20000 0000 00 - 24000 0000 00: metainfo (256 GB)
210	26000 0000 00 - 28000 0000 00: metainfo	193	24000 0000 00 - 2aa00 0000 00: -
211	28000 0000 00 - 2aa00 0000 00: -	194	2aa00 0000 00 - 2c000 0000 00: main binary (PIE) (88 GB)
212	2aa00 0000 00 - 2ab00 0000 00: main binary (PIE)	195	2c000 0000 00 - 3c000 0000 00: -
213	2ab00 0000 00 - 36200 0000 00: -	196	3c000 0000 00 - 3f000 0000 00: heap (192 GB)
214	36200 0000 00 - 36240 0000 00: traces	197	3f000 0000 00 - 3ffff ffff ff: modules and main thread stack (64 GB)
215	36240 0000 00 - 3e000 0000 00: -
216	3e000 0000 00 - 3f000 0000 00: heap
217	3f000 0000 00 - 3ffff ffff ff: modules and main thread stack
218	*/	198	*/
219	struct Mapping42 {	199	struct MappingAarch64_42 {
220	static const uptr kLoAppMemBeg = 0x00000001000ull;	200	static const uptr kLoAppMemBeg = 0x00000001000ull;
221	static const uptr kLoAppMemEnd = 0x01000000000ull;	201	static const uptr kLoAppMemEnd = 0x02000000000ull;
222	static const uptr kShadowBeg = 0x10000000000ull;	202	static const uptr kShadowBeg = 0x10000000000ull;
223	static const uptr kShadowEnd = 0x20000000000ull;	203	static const uptr kShadowEnd = 0x20000000000ull;
224	static const uptr kMetaShadowBeg = 0x26000000000ull;	204	static const uptr kMetaShadowBeg = 0x20000000000ull;
225	static const uptr kMetaShadowEnd = 0x28000000000ull;	205	static const uptr kMetaShadowEnd = 0x24000000000ull;
226	static const uptr kMidAppMemBeg = 0x2aa00000000ull;	206	static const uptr kMidAppMemBeg = 0x2aa00000000ull;
227	static const uptr kMidAppMemEnd = 0x2ab00000000ull;	207	static const uptr kMidAppMemEnd = 0x2c000000000ull;
228	static const uptr kTraceMemBeg = 0x36200000000ull;	208	static const uptr kHeapMemBeg = 0x3c000000000ull;
229	static const uptr kTraceMemEnd = 0x36400000000ull;
230	static const uptr kHeapMemBeg = 0x3e000000000ull;
231	static const uptr kHeapMemEnd = 0x3f000000000ull;	209	static const uptr kHeapMemEnd = 0x3f000000000ull;
232	static const uptr kHiAppMemBeg = 0x3f000000000ull;	210	static const uptr kHiAppMemBeg = 0x3f000000000ull;
233	static const uptr kHiAppMemEnd = 0x3ffffffffffull;	211	static const uptr kHiAppMemEnd = 0x3ffffffffffull;
234	static const uptr kAppMemMsk = 0x3c000000000ull;	212	static const uptr kShadowMsk = 0x38000000000ull;
235	static const uptr kAppMemXor = 0x04000000000ull;	213	static const uptr kShadowXor = 0x08000000000ull;
		214	static const uptr kShadowAdd = 0x00000000000ull;
236	static const uptr kVdsoBeg = 0x37f00000000ull;	215	static const uptr kVdsoBeg = 0x37f00000000ull;
237	};	216	};
238		217
239	struct Mapping48 {	218	/*
		219	C/C++ on linux/aarch64 (48-bit VMA)
		220	0000 0000 1000 - 0a00 0000 0000: main binary (10240 GB)
		221	0a00 0000 1000 - 1554 0000 0000: -
		222	1554 0000 1000 - 5400 0000 0000: shadow memory (64176 GB)
		223	5400 0000 1000 - 8000 0000 0000: -
		224	8000 0000 1000 - 0a00 0000 0000: metainfo (32768 GB)
		225	a000 0000 1000 - aaaa 0000 0000: -
		226	aaaa 0000 1000 - ac00 0000 0000: main binary (PIE) (1368 GB)
		227	ac00 0000 1000 - fc00 0000 0000: -
		228	fc00 0000 1000 - ffff ffff ffff: modules and main thread stack (4096 GB)
		229
		230	N.B. the shadow memory region has a strange start address, because it
		231	contains the shadows for the mid, high and low app regions (in this
		232	unusual order).
		233	*/
		234	struct MappingAarch64_48 {
240	static const uptr kLoAppMemBeg = 0x0000000001000ull;	235	static const uptr kLoAppMemBeg = 0x0000000001000ull;
241	static const uptr kLoAppMemEnd = 0x0000200000000ull;	236	static const uptr kLoAppMemEnd = 0x00a0000000000ull;
242	static const uptr kShadowBeg = 0x0002000000000ull;	237	static const uptr kShadowBeg = 0x0155400000000ull;
243	static const uptr kShadowEnd = 0x0004000000000ull;	238	static const uptr kShadowEnd = 0x0540000000000ull;
244	static const uptr kMetaShadowBeg = 0x0005000000000ull;	239	static const uptr kMetaShadowBeg = 0x0800000000000ull;
245	static const uptr kMetaShadowEnd = 0x0006000000000ull;	240	static const uptr kMetaShadowEnd = 0x0a00000000000ull;
246	static const uptr kMidAppMemBeg = 0x0aaaa00000000ull;	241	static const uptr kMidAppMemBeg = 0x0aaaa00000000ull;
247	static const uptr kMidAppMemEnd = 0x0aaaf00000000ull;	242	static const uptr kMidAppMemEnd = 0x0ac0000000000ull;
248	static const uptr kTraceMemBeg = 0x0f06000000000ull;	243	static const uptr kHiAppMemBeg = 0x0fc0000000000ull;
249	static const uptr kTraceMemEnd = 0x0f06200000000ull;
250	static const uptr kHeapMemBeg = 0x0ffff00000000ull;
251	static const uptr kHeapMemEnd = 0x0ffff00000000ull;
252	static const uptr kHiAppMemBeg = 0x0ffff00000000ull;
253	static const uptr kHiAppMemEnd = 0x1000000000000ull;	244	static const uptr kHiAppMemEnd = 0x1000000000000ull;
254	static const uptr kAppMemMsk = 0x0fff800000000ull;	245	static const uptr kHeapMemBeg = 0x0fc0000000000ull;
255	static const uptr kAppMemXor = 0x0000800000000ull;	246	static const uptr kHeapMemEnd = 0x0fc0000000000ull;
		247	static const uptr kShadowMsk = 0x0c00000000000ull;
		248	static const uptr kShadowXor = 0x0200000000000ull;
		249	static const uptr kShadowAdd = 0x0000000000000ull;
256	static const uptr kVdsoBeg = 0xffff000000000ull;	250	static const uptr kVdsoBeg = 0xffff000000000ull;
257	};	251	};
258		252
259	// Indicates the runtime will define the memory regions at runtime.	253	/* C/C++ on linux/loongarch64 (47-bit VMA)
260	#define TSAN_RUNTIME_VMA 1	254	0000 0000 4000 - 0080 0000 0000: main binary
261	// Indicates that mapping defines a mid range memory segment.	255	0080 0000 0000 - 0100 0000 0000: -
262	#define TSAN_MID_APP_RANGE 1	256	0100 0000 0000 - 1000 0000 0000: shadow memory
263	#elif defined(__powerpc64__)	257	1000 0000 0000 - 3000 0000 0000: -
264	// PPC64 supports multiple VMA which leads to multiple address transformation	258	3000 0000 0000 - 3400 0000 0000: metainfo
265	// functions. To support these multiple VMAS transformations and mappings TSAN	259	3400 0000 0000 - 5555 0000 0000: -
266	// runtime for PPC64 uses an external memory read (vmaSize) to select which	260	5555 0000 0000 - 5556 0000 0000: main binary (PIE)
267	// mapping to use. Although slower, it make a same instrumented binary run on	261	5556 0000 0000 - 7ffe 0000 0000: -
268	// multiple kernels.	262	7ffe 0000 0000 - 7fff 0000 0000: heap
		263	7fff 0000 0000 - 7fff 8000 0000: -
		264	7fff 8000 0000 - 8000 0000 0000: modules and main thread stack
		265	*/
		266	struct MappingLoongArch64_47 {
		267	static const uptr kMetaShadowBeg = 0x300000000000ull;
		268	static const uptr kMetaShadowEnd = 0x340000000000ull;
		269	static const uptr kShadowBeg = 0x010000000000ull;
		270	static const uptr kShadowEnd = 0x100000000000ull;
		271	static const uptr kHeapMemBeg = 0x7ffe00000000ull;
		272	static const uptr kHeapMemEnd = 0x7fff00000000ull;
		273	static const uptr kLoAppMemBeg = 0x000000004000ull;
		274	static const uptr kLoAppMemEnd = 0x008000000000ull;
		275	static const uptr kMidAppMemBeg = 0x555500000000ull;
		276	static const uptr kMidAppMemEnd = 0x555600000000ull;
		277	static const uptr kHiAppMemBeg = 0x7fff80000000ull;
		278	static const uptr kHiAppMemEnd = 0x800000000000ull;
		279	static const uptr kShadowMsk = 0x780000000000ull;
		280	static const uptr kShadowXor = 0x040000000000ull;
		281	static const uptr kShadowAdd = 0x000000000000ull;
		282	static const uptr kVdsoBeg = 0x7fffffffc000ull;
		283	};
269		284
270	/*	285	/*
271	C/C++ on linux/powerpc64 (44-bit VMA)	286	C/C++ on linux/powerpc64 (44-bit VMA)
...	@@ -274,18 +289,16 @@ C/C++ on linux/powerpc64 (44-bit VMA)	...	@@ -274,18 +289,16 @@ C/C++ on linux/powerpc64 (44-bit VMA)
274	0001 0000 0000 - 0b00 0000 0000: shadow	289	0001 0000 0000 - 0b00 0000 0000: shadow
275	0b00 0000 0000 - 0b00 0000 0000: -	290	0b00 0000 0000 - 0b00 0000 0000: -
276	0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)	291	0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
277	0d00 0000 0000 - 0d00 0000 0000: -	292	0d00 0000 0000 - 0f00 0000 0000: -
278	0d00 0000 0000 - 0f00 0000 0000: traces
279	0f00 0000 0000 - 0f00 0000 0000: -
280	0f00 0000 0000 - 0f50 0000 0000: heap	293	0f00 0000 0000 - 0f50 0000 0000: heap
281	0f50 0000 0000 - 0f60 0000 0000: -	294	0f50 0000 0000 - 0f60 0000 0000: -
282	0f60 0000 0000 - 1000 0000 0000: modules and main thread stack	295	0f60 0000 0000 - 1000 0000 0000: modules and main thread stack
283	*/	296	*/
284	struct Mapping44 {	297	struct MappingPPC64_44 {
		298	static const uptr kBroken = kBrokenMapping \| kBrokenReverseMapping \|
		299	kBrokenLinearity \| kBrokenAliasedMetas;
285	static const uptr kMetaShadowBeg = 0x0b0000000000ull;	300	static const uptr kMetaShadowBeg = 0x0b0000000000ull;
286	static const uptr kMetaShadowEnd = 0x0d0000000000ull;	301	static const uptr kMetaShadowEnd = 0x0d0000000000ull;
287	static const uptr kTraceMemBeg = 0x0d0000000000ull;
288	static const uptr kTraceMemEnd = 0x0f0000000000ull;
289	static const uptr kShadowBeg = 0x000100000000ull;	302	static const uptr kShadowBeg = 0x000100000000ull;
290	static const uptr kShadowEnd = 0x0b0000000000ull;	303	static const uptr kShadowEnd = 0x0b0000000000ull;
291	static const uptr kLoAppMemBeg = 0x000000000100ull;	304	static const uptr kLoAppMemBeg = 0x000000000100ull;
...	@@ -294,188 +307,196 @@ struct Mapping44 {	...	@@ -294,188 +307,196 @@ struct Mapping44 {
294	static const uptr kHeapMemEnd = 0x0f5000000000ull;	307	static const uptr kHeapMemEnd = 0x0f5000000000ull;
295	static const uptr kHiAppMemBeg = 0x0f6000000000ull;	308	static const uptr kHiAppMemBeg = 0x0f6000000000ull;
296	static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits	309	static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits
297	static const uptr kAppMemMsk = 0x0f0000000000ull;	310	static const uptr kShadowMsk = 0x0f0000000000ull;
298	static const uptr kAppMemXor = 0x002100000000ull;	311	static const uptr kShadowXor = 0x002100000000ull;
		312	static const uptr kShadowAdd = 0x000000000000ull;
299	static const uptr kVdsoBeg = 0x3c0000000000000ull;	313	static const uptr kVdsoBeg = 0x3c0000000000000ull;
		314	static const uptr kMidAppMemBeg = 0;
		315	static const uptr kMidAppMemEnd = 0;
300	};	316	};
301		317
302	/*	318	/*
303	C/C++ on linux/powerpc64 (46-bit VMA)	319	C/C++ on linux/powerpc64 (46-bit VMA)
304	0000 0000 1000 - 0100 0000 0000: main binary	320	0000 0000 1000 - 0100 0000 0000: main binary
305	0100 0000 0000 - 0200 0000 0000: -	321	0100 0000 0000 - 0200 0000 0000: -
306	0100 0000 0000 - 1000 0000 0000: shadow	322	0100 0000 0000 - 0800 0000 0000: shadow
307	1000 0000 0000 - 1000 0000 0000: -	323	0800 0000 0000 - 1000 0000 0000: -
308	1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)	324	1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
309	2000 0000 0000 - 2000 0000 0000: -	325	1200 0000 0000 - 3d00 0000 0000: -
310	2000 0000 0000 - 2200 0000 0000: traces
311	2200 0000 0000 - 3d00 0000 0000: -
312	3d00 0000 0000 - 3e00 0000 0000: heap	326	3d00 0000 0000 - 3e00 0000 0000: heap
313	3e00 0000 0000 - 3e80 0000 0000: -	327	3e00 0000 0000 - 3e80 0000 0000: -
314	3e80 0000 0000 - 4000 0000 0000: modules and main thread stack	328	3e80 0000 0000 - 4000 0000 0000: modules and main thread stack
315	*/	329	*/
316	struct Mapping46 {	330	struct MappingPPC64_46 {
317	static const uptr kMetaShadowBeg = 0x100000000000ull;	331	static const uptr kMetaShadowBeg = 0x100000000000ull;
318	static const uptr kMetaShadowEnd = 0x200000000000ull;	332	static const uptr kMetaShadowEnd = 0x120000000000ull;
319	static const uptr kTraceMemBeg = 0x200000000000ull;
320	static const uptr kTraceMemEnd = 0x220000000000ull;
321	static const uptr kShadowBeg = 0x010000000000ull;	333	static const uptr kShadowBeg = 0x010000000000ull;
322	static const uptr kShadowEnd = 0x100000000000ull;	334	static const uptr kShadowEnd = 0x080000000000ull;
323	static const uptr kHeapMemBeg = 0x3d0000000000ull;	335	static const uptr kHeapMemBeg = 0x3d0000000000ull;
324	static const uptr kHeapMemEnd = 0x3e0000000000ull;	336	static const uptr kHeapMemEnd = 0x3e0000000000ull;
325	static const uptr kLoAppMemBeg = 0x000000001000ull;	337	static const uptr kLoAppMemBeg = 0x000000001000ull;
326	static const uptr kLoAppMemEnd = 0x010000000000ull;	338	static const uptr kLoAppMemEnd = 0x010000000000ull;
327	static const uptr kHiAppMemBeg = 0x3e8000000000ull;	339	static const uptr kHiAppMemBeg = 0x3e8000000000ull;
328	static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits	340	static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits
329	static const uptr kAppMemMsk = 0x3c0000000000ull;	341	static const uptr kShadowMsk = 0x3c0000000000ull;
330	static const uptr kAppMemXor = 0x020000000000ull;	342	static const uptr kShadowXor = 0x020000000000ull;
		343	static const uptr kShadowAdd = 0x000000000000ull;
331	static const uptr kVdsoBeg = 0x7800000000000000ull;	344	static const uptr kVdsoBeg = 0x7800000000000000ull;
		345	static const uptr kMidAppMemBeg = 0;
		346	static const uptr kMidAppMemEnd = 0;
332	};	347	};
333		348
334	/*	349	/*
335	C/C++ on linux/powerpc64 (47-bit VMA)	350	C/C++ on linux/powerpc64 (47-bit VMA)
336	0000 0000 1000 - 0100 0000 0000: main binary	351	0000 0000 1000 - 0100 0000 0000: main binary
337	0100 0000 0000 - 0200 0000 0000: -	352	0100 0000 0000 - 0200 0000 0000: -
338	0100 0000 0000 - 1000 0000 0000: shadow	353	0100 0000 0000 - 0800 0000 0000: shadow
339	1000 0000 0000 - 1000 0000 0000: -	354	0800 0000 0000 - 1000 0000 0000: -
340	1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)	355	1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
341	2000 0000 0000 - 2000 0000 0000: -	356	1200 0000 0000 - 7d00 0000 0000: -
342	2000 0000 0000 - 2200 0000 0000: traces
343	2200 0000 0000 - 7d00 0000 0000: -
344	7d00 0000 0000 - 7e00 0000 0000: heap	357	7d00 0000 0000 - 7e00 0000 0000: heap
345	7e00 0000 0000 - 7e80 0000 0000: -	358	7e00 0000 0000 - 7e80 0000 0000: -
346	7e80 0000 0000 - 8000 0000 0000: modules and main thread stack	359	7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
347	*/	360	*/
348	struct Mapping47 {	361	struct MappingPPC64_47 {
349	static const uptr kMetaShadowBeg = 0x100000000000ull;	362	static const uptr kMetaShadowBeg = 0x100000000000ull;
350	static const uptr kMetaShadowEnd = 0x200000000000ull;	363	static const uptr kMetaShadowEnd = 0x120000000000ull;
351	static const uptr kTraceMemBeg = 0x200000000000ull;
352	static const uptr kTraceMemEnd = 0x220000000000ull;
353	static const uptr kShadowBeg = 0x010000000000ull;	364	static const uptr kShadowBeg = 0x010000000000ull;
354	static const uptr kShadowEnd = 0x100000000000ull;	365	static const uptr kShadowEnd = 0x080000000000ull;
355	static const uptr kHeapMemBeg = 0x7d0000000000ull;	366	static const uptr kHeapMemBeg = 0x7d0000000000ull;
356	static const uptr kHeapMemEnd = 0x7e0000000000ull;	367	static const uptr kHeapMemEnd = 0x7e0000000000ull;
357	static const uptr kLoAppMemBeg = 0x000000001000ull;	368	static const uptr kLoAppMemBeg = 0x000000001000ull;
358	static const uptr kLoAppMemEnd = 0x010000000000ull;	369	static const uptr kLoAppMemEnd = 0x010000000000ull;
359	static const uptr kHiAppMemBeg = 0x7e8000000000ull;	370	static const uptr kHiAppMemBeg = 0x7e8000000000ull;
360	static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits	371	static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits
361	static const uptr kAppMemMsk = 0x7c0000000000ull;	372	static const uptr kShadowMsk = 0x7c0000000000ull;
362	static const uptr kAppMemXor = 0x020000000000ull;	373	static const uptr kShadowXor = 0x020000000000ull;
		374	static const uptr kShadowAdd = 0x000000000000ull;
363	static const uptr kVdsoBeg = 0x7800000000000000ull;	375	static const uptr kVdsoBeg = 0x7800000000000000ull;
		376	static const uptr kMidAppMemBeg = 0;
		377	static const uptr kMidAppMemEnd = 0;
364	};	378	};
365		379
366	// Indicates the runtime will define the memory regions at runtime.
367	#define TSAN_RUNTIME_VMA 1
368	#elif defined(__s390x__)
369	/*	380	/*
370	C/C++ on linux/s390x	381	C/C++ on linux/s390x
371	While the kernel provides a 64-bit address space, we have to restrict ourselves	382	While the kernel provides a 64-bit address space, we have to restrict ourselves
372	to 48 bits due to how e.g. SyncVar::GetId() works.	383	to 48 bits due to how e.g. SyncVar::GetId() works.
373	0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB	384	0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
374	0e00 0000 0000 - 4000 0000 0000: -	385	0e00 0000 0000 - 2000 0000 0000: -
375	4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)	386	2000 0000 0000 - 4000 0000 0000: shadow - 32TiB (2 * app)
376	8000 0000 0000 - 9000 0000 0000: -	387	4000 0000 0000 - 9000 0000 0000: -
377	9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)	388	9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
378	9800 0000 0000 - a000 0000 0000: -	389	9800 0000 0000 - be00 0000 0000: -
379	a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
380	b000 0000 0000 - be00 0000 0000: -
381	be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)	390	be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
382	*/	391	*/
383	struct Mapping {	392	struct MappingS390x {
384	static const uptr kMetaShadowBeg = 0x900000000000ull;	393	static const uptr kMetaShadowBeg = 0x900000000000ull;
385	static const uptr kMetaShadowEnd = 0x980000000000ull;	394	static const uptr kMetaShadowEnd = 0x980000000000ull;
386	static const uptr kTraceMemBeg = 0xa00000000000ull;	395	static const uptr kShadowBeg = 0x200000000000ull;
387	static const uptr kTraceMemEnd = 0xb00000000000ull;	396	static const uptr kShadowEnd = 0x400000000000ull;
388	static const uptr kShadowBeg = 0x400000000000ull;
389	static const uptr kShadowEnd = 0x800000000000ull;
390	static const uptr kHeapMemBeg = 0xbe0000000000ull;	397	static const uptr kHeapMemBeg = 0xbe0000000000ull;
391	static const uptr kHeapMemEnd = 0xc00000000000ull;	398	static const uptr kHeapMemEnd = 0xc00000000000ull;
392	static const uptr kLoAppMemBeg = 0x000000001000ull;	399	static const uptr kLoAppMemBeg = 0x000000001000ull;
393	static const uptr kLoAppMemEnd = 0x0e0000000000ull;	400	static const uptr kLoAppMemEnd = 0x0e0000000000ull;
394	static const uptr kHiAppMemBeg = 0xc00000004000ull;	401	static const uptr kHiAppMemBeg = 0xc00000004000ull;
395	static const uptr kHiAppMemEnd = 0xc00000004000ull;	402	static const uptr kHiAppMemEnd = 0xc00000004000ull;
396	static const uptr kAppMemMsk = 0xb00000000000ull;	403	static const uptr kShadowMsk = 0xb00000000000ull;
397	static const uptr kAppMemXor = 0x100000000000ull;	404	static const uptr kShadowXor = 0x100000000000ull;
		405	static const uptr kShadowAdd = 0x000000000000ull;
398	static const uptr kVdsoBeg = 0xfffffffff000ull;	406	static const uptr kVdsoBeg = 0xfffffffff000ull;
		407	static const uptr kMidAppMemBeg = 0;
		408	static const uptr kMidAppMemEnd = 0;
399	};	409	};
400	#endif
401
402	#elif SANITIZER_GO && !SANITIZER_WINDOWS && HAS_48_BIT_ADDRESS_SPACE
403		410
404	/* Go on linux, darwin and freebsd on x86_64	411	/* Go on linux, darwin and freebsd on x86_64
405	0000 0000 1000 - 0000 1000 0000: executable	412	0000 0000 1000 - 0000 1000 0000: executable
406	0000 1000 0000 - 00c0 0000 0000: -	413	0000 1000 0000 - 00c0 0000 0000: -
407	00c0 0000 0000 - 00e0 0000 0000: heap	414	00c0 0000 0000 - 00e0 0000 0000: heap
408	00e0 0000 0000 - 2000 0000 0000: -	415	00e0 0000 0000 - 2000 0000 0000: -
409	2000 0000 0000 - 2380 0000 0000: shadow	416	2000 0000 0000 - 21c0 0000 0000: shadow
410	2380 0000 0000 - 3000 0000 0000: -	417	21c0 0000 0000 - 3000 0000 0000: -
411	3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)	418	3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
412	4000 0000 0000 - 6000 0000 0000: -	419	4000 0000 0000 - 8000 0000 0000: -
413	6000 0000 0000 - 6200 0000 0000: traces
414	6200 0000 0000 - 8000 0000 0000: -
415	*/	420	*/
416		421
417	struct Mapping {	422	struct MappingGo48 {
418	static const uptr kMetaShadowBeg = 0x300000000000ull;	423	static const uptr kMetaShadowBeg = 0x300000000000ull;
419	static const uptr kMetaShadowEnd = 0x400000000000ull;	424	static const uptr kMetaShadowEnd = 0x400000000000ull;
420	static const uptr kTraceMemBeg = 0x600000000000ull;
421	static const uptr kTraceMemEnd = 0x620000000000ull;
422	static const uptr kShadowBeg = 0x200000000000ull;	425	static const uptr kShadowBeg = 0x200000000000ull;
423	static const uptr kShadowEnd = 0x238000000000ull;	426	static const uptr kShadowEnd = 0x21c000000000ull;
424	static const uptr kAppMemBeg = 0x000000001000ull;	427	static const uptr kLoAppMemBeg = 0x000000001000ull;
425	static const uptr kAppMemEnd = 0x00e000000000ull;	428	static const uptr kLoAppMemEnd = 0x00e000000000ull;
		429	static const uptr kMidAppMemBeg = 0;
		430	static const uptr kMidAppMemEnd = 0;
		431	static const uptr kHiAppMemBeg = 0;
		432	static const uptr kHiAppMemEnd = 0;
		433	static const uptr kHeapMemBeg = 0;
		434	static const uptr kHeapMemEnd = 0;
		435	static const uptr kVdsoBeg = 0;
		436	static const uptr kShadowMsk = 0;
		437	static const uptr kShadowXor = 0;
		438	static const uptr kShadowAdd = 0x200000000000ull;
426	};	439	};
427		440
428	#elif SANITIZER_GO && SANITIZER_WINDOWS
429
430	/* Go on windows	441	/* Go on windows
431	0000 0000 1000 - 0000 1000 0000: executable	442	0000 0000 1000 - 0000 1000 0000: executable
432	0000 1000 0000 - 00f8 0000 0000: -	443	0000 1000 0000 - 00f8 0000 0000: -
433	00c0 0000 0000 - 00e0 0000 0000: heap	444	00c0 0000 0000 - 00e0 0000 0000: heap
434	00e0 0000 0000 - 0100 0000 0000: -	445	00e0 0000 0000 - 0100 0000 0000: -
435	0100 0000 0000 - 0500 0000 0000: shadow	446	0100 0000 0000 - 0300 0000 0000: shadow
436	0500 0000 0000 - 0560 0000 0000: -	447	0300 0000 0000 - 0700 0000 0000: -
437	0560 0000 0000 - 0760 0000 0000: traces	448	0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects)
438	0760 0000 0000 - 07d0 0000 0000: metainfo (memory blocks and sync objects)
439	07d0 0000 0000 - 8000 0000 0000: -	449	07d0 0000 0000 - 8000 0000 0000: -
		450	PIE binaries currently not supported, but it should be theoretically possible.
440	*/	451	*/
441		452
442	struct Mapping {	453	struct MappingGoWindows {
443	static const uptr kMetaShadowBeg = 0x076000000000ull;	454	static const uptr kMetaShadowBeg = 0x070000000000ull;
444	static const uptr kMetaShadowEnd = 0x07d000000000ull;	455	static const uptr kMetaShadowEnd = 0x077000000000ull;
445	static const uptr kTraceMemBeg = 0x056000000000ull;
446	static const uptr kTraceMemEnd = 0x076000000000ull;
447	static const uptr kShadowBeg = 0x010000000000ull;	456	static const uptr kShadowBeg = 0x010000000000ull;
448	static const uptr kShadowEnd = 0x050000000000ull;	457	static const uptr kShadowEnd = 0x030000000000ull;
449	static const uptr kAppMemBeg = 0x000000001000ull;	458	static const uptr kLoAppMemBeg = 0x000000001000ull;
450	static const uptr kAppMemEnd = 0x00e000000000ull;	459	static const uptr kLoAppMemEnd = 0x00e000000000ull;
		460	static const uptr kMidAppMemBeg = 0;
		461	static const uptr kMidAppMemEnd = 0;
		462	static const uptr kHiAppMemBeg = 0;
		463	static const uptr kHiAppMemEnd = 0;
		464	static const uptr kHeapMemBeg = 0;
		465	static const uptr kHeapMemEnd = 0;
		466	static const uptr kVdsoBeg = 0;
		467	static const uptr kShadowMsk = 0;
		468	static const uptr kShadowXor = 0;
		469	static const uptr kShadowAdd = 0x010000000000ull;
451	};	470	};
452		471
453	#elif SANITIZER_GO && defined(__powerpc64__)
454
455	/* Only Mapping46 and Mapping47 are currently supported for powercp64 on Go. */
456
457	/* Go on linux/powerpc64 (46-bit VMA)	472	/* Go on linux/powerpc64 (46-bit VMA)
458	0000 0000 1000 - 0000 1000 0000: executable	473	0000 0000 1000 - 0000 1000 0000: executable
459	0000 1000 0000 - 00c0 0000 0000: -	474	0000 1000 0000 - 00c0 0000 0000: -
460	00c0 0000 0000 - 00e0 0000 0000: heap	475	00c0 0000 0000 - 00e0 0000 0000: heap
461	00e0 0000 0000 - 2000 0000 0000: -	476	00e0 0000 0000 - 2000 0000 0000: -
462	2000 0000 0000 - 2380 0000 0000: shadow	477	2000 0000 0000 - 21c0 0000 0000: shadow
463	2380 0000 0000 - 2400 0000 0000: -	478	21c0 0000 0000 - 2400 0000 0000: -
464	2400 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects)	479	2400 0000 0000 - 2470 0000 0000: metainfo (memory blocks and sync objects)
465	3400 0000 0000 - 3600 0000 0000: -	480	2470 0000 0000 - 4000 0000 0000: -
466	3600 0000 0000 - 3800 0000 0000: traces
467	3800 0000 0000 - 4000 0000 0000: -
468	*/	481	*/
469		482
470	struct Mapping46 {	483	struct MappingGoPPC64_46 {
471	static const uptr kMetaShadowBeg = 0x240000000000ull;	484	static const uptr kMetaShadowBeg = 0x240000000000ull;
472	static const uptr kMetaShadowEnd = 0x340000000000ull;	485	static const uptr kMetaShadowEnd = 0x247000000000ull;
473	static const uptr kTraceMemBeg = 0x360000000000ull;
474	static const uptr kTraceMemEnd = 0x380000000000ull;
475	static const uptr kShadowBeg = 0x200000000000ull;	486	static const uptr kShadowBeg = 0x200000000000ull;
476	static const uptr kShadowEnd = 0x238000000000ull;	487	static const uptr kShadowEnd = 0x21c000000000ull;
477	static const uptr kAppMemBeg = 0x000000001000ull;	488	static const uptr kLoAppMemBeg = 0x000000001000ull;
478	static const uptr kAppMemEnd = 0x00e000000000ull;	489	static const uptr kLoAppMemEnd = 0x00e000000000ull;
		490	static const uptr kMidAppMemBeg = 0;
		491	static const uptr kMidAppMemEnd = 0;
		492	static const uptr kHiAppMemBeg = 0;
		493	static const uptr kHiAppMemEnd = 0;
		494	static const uptr kHeapMemBeg = 0;
		495	static const uptr kHeapMemEnd = 0;
		496	static const uptr kVdsoBeg = 0;
		497	static const uptr kShadowMsk = 0;
		498	static const uptr kShadowXor = 0;
		499	static const uptr kShadowAdd = 0x200000000000ull;
479	};	500	};
480		501
481	/* Go on linux/powerpc64 (47-bit VMA)	502	/* Go on linux/powerpc64 (47-bit VMA)
...	@@ -483,718 +504,424 @@ struct Mapping46 {	...	@@ -483,718 +504,424 @@ struct Mapping46 {
483	0000 1000 0000 - 00c0 0000 0000: -	504	0000 1000 0000 - 00c0 0000 0000: -
484	00c0 0000 0000 - 00e0 0000 0000: heap	505	00c0 0000 0000 - 00e0 0000 0000: heap
485	00e0 0000 0000 - 2000 0000 0000: -	506	00e0 0000 0000 - 2000 0000 0000: -
486	2000 0000 0000 - 3000 0000 0000: shadow	507	2000 0000 0000 - 2800 0000 0000: shadow
487	3000 0000 0000 - 3000 0000 0000: -	508	2800 0000 0000 - 3000 0000 0000: -
488	3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)	509	3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
489	4000 0000 0000 - 6000 0000 0000: -	510	3200 0000 0000 - 8000 0000 0000: -
490	6000 0000 0000 - 6200 0000 0000: traces
491	6200 0000 0000 - 8000 0000 0000: -
492	*/	511	*/
493		512
494	struct Mapping47 {	513	struct MappingGoPPC64_47 {
495	static const uptr kMetaShadowBeg = 0x300000000000ull;	514	static const uptr kMetaShadowBeg = 0x300000000000ull;
496	static const uptr kMetaShadowEnd = 0x400000000000ull;	515	static const uptr kMetaShadowEnd = 0x320000000000ull;
497	static const uptr kTraceMemBeg = 0x600000000000ull;
498	static const uptr kTraceMemEnd = 0x620000000000ull;
499	static const uptr kShadowBeg = 0x200000000000ull;	516	static const uptr kShadowBeg = 0x200000000000ull;
500	static const uptr kShadowEnd = 0x300000000000ull;	517	static const uptr kShadowEnd = 0x280000000000ull;
501	static const uptr kAppMemBeg = 0x000000001000ull;	518	static const uptr kLoAppMemBeg = 0x000000001000ull;
502	static const uptr kAppMemEnd = 0x00e000000000ull;	519	static const uptr kLoAppMemEnd = 0x00e000000000ull;
		520	static const uptr kMidAppMemBeg = 0;
		521	static const uptr kMidAppMemEnd = 0;
		522	static const uptr kHiAppMemBeg = 0;
		523	static const uptr kHiAppMemEnd = 0;
		524	static const uptr kHeapMemBeg = 0;
		525	static const uptr kHeapMemEnd = 0;
		526	static const uptr kVdsoBeg = 0;
		527	static const uptr kShadowMsk = 0;
		528	static const uptr kShadowXor = 0;
		529	static const uptr kShadowAdd = 0x200000000000ull;
503	};	530	};
504		531
505	#define TSAN_RUNTIME_VMA 1
506
507	#elif SANITIZER_GO && defined(__aarch64__)
508
509	/* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA)	532	/* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA)
510	0000 0000 1000 - 0000 1000 0000: executable	533	0000 0000 1000 - 0000 1000 0000: executable
511	0000 1000 0000 - 00c0 0000 0000: -	534	0000 1000 0000 - 00c0 0000 0000: -
512	00c0 0000 0000 - 00e0 0000 0000: heap	535	00c0 0000 0000 - 00e0 0000 0000: heap
513	00e0 0000 0000 - 2000 0000 0000: -	536	00e0 0000 0000 - 2000 0000 0000: -
514	2000 0000 0000 - 3000 0000 0000: shadow	537	2000 0000 0000 - 2800 0000 0000: shadow
515	3000 0000 0000 - 3000 0000 0000: -	538	2800 0000 0000 - 3000 0000 0000: -
516	3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)	539	3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
517	4000 0000 0000 - 6000 0000 0000: -	540	3200 0000 0000 - 8000 0000 0000: -
518	6000 0000 0000 - 6200 0000 0000: traces
519	6200 0000 0000 - 8000 0000 0000: -
520	*/	541	*/
521		542	struct MappingGoAarch64 {
522	struct Mapping {
523	static const uptr kMetaShadowBeg = 0x300000000000ull;	543	static const uptr kMetaShadowBeg = 0x300000000000ull;
524	static const uptr kMetaShadowEnd = 0x400000000000ull;	544	static const uptr kMetaShadowEnd = 0x320000000000ull;
525	static const uptr kTraceMemBeg = 0x600000000000ull;
526	static const uptr kTraceMemEnd = 0x620000000000ull;
527	static const uptr kShadowBeg = 0x200000000000ull;	545	static const uptr kShadowBeg = 0x200000000000ull;
528	static const uptr kShadowEnd = 0x300000000000ull;	546	static const uptr kShadowEnd = 0x280000000000ull;
529	static const uptr kAppMemBeg = 0x000000001000ull;	547	static const uptr kLoAppMemBeg = 0x000000001000ull;
530	static const uptr kAppMemEnd = 0x00e000000000ull;	548	static const uptr kLoAppMemEnd = 0x00e000000000ull;
		549	static const uptr kMidAppMemBeg = 0;
		550	static const uptr kMidAppMemEnd = 0;
		551	static const uptr kHiAppMemBeg = 0;
		552	static const uptr kHiAppMemEnd = 0;
		553	static const uptr kHeapMemBeg = 0;
		554	static const uptr kHeapMemEnd = 0;
		555	static const uptr kVdsoBeg = 0;
		556	static const uptr kShadowMsk = 0;
		557	static const uptr kShadowXor = 0;
		558	static const uptr kShadowAdd = 0x200000000000ull;
531	};	559	};
532		560
533	// Indicates the runtime will define the memory regions at runtime.
534	#define TSAN_RUNTIME_VMA 1
535
536	#elif SANITIZER_GO && defined(__mips64)
537	/*	561	/*
538	Go on linux/mips64 (47-bit VMA)	562	Go on linux/mips64 (47-bit VMA)
539	0000 0000 1000 - 0000 1000 0000: executable	563	0000 0000 1000 - 0000 1000 0000: executable
540	0000 1000 0000 - 00c0 0000 0000: -	564	0000 1000 0000 - 00c0 0000 0000: -
541	00c0 0000 0000 - 00e0 0000 0000: heap	565	00c0 0000 0000 - 00e0 0000 0000: heap
542	00e0 0000 0000 - 2000 0000 0000: -	566	00e0 0000 0000 - 2000 0000 0000: -
543	2000 0000 0000 - 3000 0000 0000: shadow	567	2000 0000 0000 - 2800 0000 0000: shadow
544	3000 0000 0000 - 3000 0000 0000: -	568	2800 0000 0000 - 3000 0000 0000: -
545	3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)	569	3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
546	4000 0000 0000 - 6000 0000 0000: -	570	3200 0000 0000 - 8000 0000 0000: -
547	6000 0000 0000 - 6200 0000 0000: traces
548	6200 0000 0000 - 8000 0000 0000: -
549	*/	571	*/
550	struct Mapping47 {	572	struct MappingGoMips64_47 {
551	static const uptr kMetaShadowBeg = 0x300000000000ull;	573	static const uptr kMetaShadowBeg = 0x300000000000ull;
552	static const uptr kMetaShadowEnd = 0x400000000000ull;	574	static const uptr kMetaShadowEnd = 0x320000000000ull;
553	static const uptr kTraceMemBeg = 0x600000000000ull;
554	static const uptr kTraceMemEnd = 0x620000000000ull;
555	static const uptr kShadowBeg = 0x200000000000ull;	575	static const uptr kShadowBeg = 0x200000000000ull;
556	static const uptr kShadowEnd = 0x300000000000ull;	576	static const uptr kShadowEnd = 0x280000000000ull;
557	static const uptr kAppMemBeg = 0x000000001000ull;	577	static const uptr kLoAppMemBeg = 0x000000001000ull;
558	static const uptr kAppMemEnd = 0x00e000000000ull;	578	static const uptr kLoAppMemEnd = 0x00e000000000ull;
		579	static const uptr kMidAppMemBeg = 0;
		580	static const uptr kMidAppMemEnd = 0;
		581	static const uptr kHiAppMemBeg = 0;
		582	static const uptr kHiAppMemEnd = 0;
		583	static const uptr kHeapMemBeg = 0;
		584	static const uptr kHeapMemEnd = 0;
		585	static const uptr kVdsoBeg = 0;
		586	static const uptr kShadowMsk = 0;
		587	static const uptr kShadowXor = 0;
		588	static const uptr kShadowAdd = 0x200000000000ull;
559	};	589	};
560		590
561	#define TSAN_RUNTIME_VMA 1
562
563	#elif SANITIZER_GO && defined(__s390x__)
564	/*	591	/*
565	Go on linux/s390x	592	Go on linux/s390x
566	0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB	593	0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
567	1000 0000 0000 - 4000 0000 0000: -	594	1000 0000 0000 - 4000 0000 0000: -
568	4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)	595	4000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app)
569	8000 0000 0000 - 9000 0000 0000: -	596	6000 0000 0000 - 9000 0000 0000: -
570	9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)	597	9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
571	9800 0000 0000 - a000 0000 0000: -
572	a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
573	*/	598	*/
574	struct Mapping {	599	struct MappingGoS390x {
575	static const uptr kMetaShadowBeg = 0x900000000000ull;	600	static const uptr kMetaShadowBeg = 0x900000000000ull;
576	static const uptr kMetaShadowEnd = 0x980000000000ull;	601	static const uptr kMetaShadowEnd = 0x980000000000ull;
577	static const uptr kTraceMemBeg = 0xa00000000000ull;
578	static const uptr kTraceMemEnd = 0xb00000000000ull;
579	static const uptr kShadowBeg = 0x400000000000ull;	602	static const uptr kShadowBeg = 0x400000000000ull;
580	static const uptr kShadowEnd = 0x800000000000ull;	603	static const uptr kShadowEnd = 0x600000000000ull;
581	static const uptr kAppMemBeg = 0x000000001000ull;	604	static const uptr kLoAppMemBeg = 0x000000001000ull;
582	static const uptr kAppMemEnd = 0x100000000000ull;	605	static const uptr kLoAppMemEnd = 0x100000000000ull;
		606	static const uptr kMidAppMemBeg = 0;
		607	static const uptr kMidAppMemEnd = 0;
		608	static const uptr kHiAppMemBeg = 0;
		609	static const uptr kHiAppMemEnd = 0;
		610	static const uptr kHeapMemBeg = 0;
		611	static const uptr kHeapMemEnd = 0;
		612	static const uptr kVdsoBeg = 0;
		613	static const uptr kShadowMsk = 0;
		614	static const uptr kShadowXor = 0;
		615	static const uptr kShadowAdd = 0x400000000000ull;
583	};	616	};
584		617
585	#else
586	# error "Unknown platform"
587	#endif
588
589
590	#ifdef TSAN_RUNTIME_VMA
591	extern uptr vmaSize;	618	extern uptr vmaSize;
592	#endif
593
594
595	enum MappingType {
596	MAPPING_LO_APP_BEG,
597	MAPPING_LO_APP_END,
598	MAPPING_HI_APP_BEG,
599	MAPPING_HI_APP_END,
600	#ifdef TSAN_MID_APP_RANGE
601	MAPPING_MID_APP_BEG,
602	MAPPING_MID_APP_END,
603	#endif
604	MAPPING_HEAP_BEG,
605	MAPPING_HEAP_END,
606	MAPPING_APP_BEG,
607	MAPPING_APP_END,
608	MAPPING_SHADOW_BEG,
609	MAPPING_SHADOW_END,
610	MAPPING_META_SHADOW_BEG,
611	MAPPING_META_SHADOW_END,
612	MAPPING_TRACE_BEG,
613	MAPPING_TRACE_END,
614	MAPPING_VDSO_BEG,
615	};
616		619
617	template<typename Mapping, int Type>	620	template <typename Func, typename Arg>
618	uptr MappingImpl(void) {	621	ALWAYS_INLINE auto SelectMapping(Arg arg) {
619	switch (Type) {	622	#if SANITIZER_GO
620	#if !SANITIZER_GO	623	# if defined(__powerpc64__)
621	case MAPPING_LO_APP_BEG: return Mapping::kLoAppMemBeg;
622	case MAPPING_LO_APP_END: return Mapping::kLoAppMemEnd;
623	# ifdef TSAN_MID_APP_RANGE
624	case MAPPING_MID_APP_BEG: return Mapping::kMidAppMemBeg;
625	case MAPPING_MID_APP_END: return Mapping::kMidAppMemEnd;
626	# endif
627	case MAPPING_HI_APP_BEG: return Mapping::kHiAppMemBeg;
628	case MAPPING_HI_APP_END: return Mapping::kHiAppMemEnd;
629	case MAPPING_HEAP_BEG: return Mapping::kHeapMemBeg;
630	case MAPPING_HEAP_END: return Mapping::kHeapMemEnd;
631	case MAPPING_VDSO_BEG: return Mapping::kVdsoBeg;
632	#else
633	case MAPPING_APP_BEG: return Mapping::kAppMemBeg;
634	case MAPPING_APP_END: return Mapping::kAppMemEnd;
635	#endif
636	case MAPPING_SHADOW_BEG: return Mapping::kShadowBeg;
637	case MAPPING_SHADOW_END: return Mapping::kShadowEnd;
638	case MAPPING_META_SHADOW_BEG: return Mapping::kMetaShadowBeg;
639	case MAPPING_META_SHADOW_END: return Mapping::kMetaShadowEnd;
640	case MAPPING_TRACE_BEG: return Mapping::kTraceMemBeg;
641	case MAPPING_TRACE_END: return Mapping::kTraceMemEnd;
642	}
643	}
644
645	template<int Type>
646	uptr MappingArchImpl(void) {
647	#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
648	switch (vmaSize) {	624	switch (vmaSize) {
649	case 39: return MappingImpl<Mapping39, Type>();	625	case 46:
650	case 42: return MappingImpl<Mapping42, Type>();	626	return Func::template Apply<MappingGoPPC64_46>(arg);
651	case 48: return MappingImpl<Mapping48, Type>();	627	case 47:
		628	return Func::template Apply<MappingGoPPC64_47>(arg);
652	}	629	}
653	DCHECK(0);	630	# elif defined(__mips64)
654	return 0;	631	return Func::template Apply<MappingGoMips64_47>(arg);
655	#elif defined(__powerpc64__)	632	# elif defined(__s390x__)
		633	return Func::template Apply<MappingGoS390x>(arg);
		634	# elif defined(__aarch64__)
		635	return Func::template Apply<MappingGoAarch64>(arg);
		636	# elif SANITIZER_WINDOWS
		637	return Func::template Apply<MappingGoWindows>(arg);
		638	# else
		639	return Func::template Apply<MappingGo48>(arg);
		640	# endif
		641	#else // SANITIZER_GO
		642	# if SANITIZER_IOS && !SANITIZER_IOSSIM
		643	return Func::template Apply<MappingAppleAarch64>(arg);
		644	# elif defined(__x86_64__) \|\| SANITIZER_APPLE
		645	return Func::template Apply<Mapping48AddressSpace>(arg);
		646	# elif defined(__aarch64__)
656	switch (vmaSize) {	647	switch (vmaSize) {
657	#if !SANITIZER_GO	648	case 39:
658	case 44: return MappingImpl<Mapping44, Type>();	649	return Func::template Apply<MappingAarch64_39>(arg);
659	#endif	650	case 42:
660	case 46: return MappingImpl<Mapping46, Type>();	651	return Func::template Apply<MappingAarch64_42>(arg);
661	case 47: return MappingImpl<Mapping47, Type>();	652	case 48:
		653	return Func::template Apply<MappingAarch64_48>(arg);
662	}	654	}
663	DCHECK(0);	655	# elif SANITIZER_LOONGARCH64
664	return 0;	656	return Func::template Apply<MappingLoongArch64_47>(arg);
665	#elif defined(__mips64)	657	# elif defined(__powerpc64__)
666	switch (vmaSize) {	658	switch (vmaSize) {
667	#if !SANITIZER_GO	659	case 44:
668	case 40: return MappingImpl<Mapping40, Type>();	660	return Func::template Apply<MappingPPC64_44>(arg);
669	#else	661	case 46:
670	case 47: return MappingImpl<Mapping47, Type>();	662	return Func::template Apply<MappingPPC64_46>(arg);
671	#endif	663	case 47:
		664	return Func::template Apply<MappingPPC64_47>(arg);
672	}	665	}
673	DCHECK(0);	666	# elif defined(__mips64)
674	return 0;	667	return Func::template Apply<MappingMips64_40>(arg);
675	#else	668	# elif defined(__s390x__)
676	return MappingImpl<Mapping, Type>();	669	return Func::template Apply<MappingS390x>(arg);
677	#endif	670	# else
		671	# error "unsupported platform"
		672	# endif
		673	#endif
		674	Die();
		675	}
		676
		677	template <typename Func>
		678	void ForEachMapping() {
		679	Func::template Apply<Mapping48AddressSpace>();
		680	Func::template Apply<MappingMips64_40>();
		681	Func::template Apply<MappingAppleAarch64>();
		682	Func::template Apply<MappingAarch64_39>();
		683	Func::template Apply<MappingAarch64_42>();
		684	Func::template Apply<MappingAarch64_48>();
		685	Func::template Apply<MappingLoongArch64_47>();
		686	Func::template Apply<MappingPPC64_44>();
		687	Func::template Apply<MappingPPC64_46>();
		688	Func::template Apply<MappingPPC64_47>();
		689	Func::template Apply<MappingS390x>();
		690	Func::template Apply<MappingGo48>();
		691	Func::template Apply<MappingGoWindows>();
		692	Func::template Apply<MappingGoPPC64_46>();
		693	Func::template Apply<MappingGoPPC64_47>();
		694	Func::template Apply<MappingGoAarch64>();
		695	Func::template Apply<MappingGoMips64_47>();
		696	Func::template Apply<MappingGoS390x>();
678	}	697	}
679		698
680	#if !SANITIZER_GO	699	enum MappingType {
681	ALWAYS_INLINE	700	kLoAppMemBeg,
682	uptr LoAppMemBeg(void) {	701	kLoAppMemEnd,
683	return MappingArchImpl<MAPPING_LO_APP_BEG>();	702	kHiAppMemBeg,
684	}	703	kHiAppMemEnd,
685	ALWAYS_INLINE	704	kMidAppMemBeg,
686	uptr LoAppMemEnd(void) {	705	kMidAppMemEnd,
687	return MappingArchImpl<MAPPING_LO_APP_END>();	706	kHeapMemBeg,
688	}	707	kHeapMemEnd,
		708	kShadowBeg,
		709	kShadowEnd,
		710	kMetaShadowBeg,
		711	kMetaShadowEnd,
		712	kVdsoBeg,
		713	};
689		714
690	#ifdef TSAN_MID_APP_RANGE	715	struct MappingField {
691	ALWAYS_INLINE	716	template <typename Mapping>
692	uptr MidAppMemBeg(void) {	717	static uptr Apply(MappingType type) {
693	return MappingArchImpl<MAPPING_MID_APP_BEG>();	718	switch (type) {
694	}	719	case kLoAppMemBeg:
695	ALWAYS_INLINE	720	return Mapping::kLoAppMemBeg;
696	uptr MidAppMemEnd(void) {	721	case kLoAppMemEnd:
697	return MappingArchImpl<MAPPING_MID_APP_END>();	722	return Mapping::kLoAppMemEnd;
698	}	723	case kMidAppMemBeg:
699	#endif	724	return Mapping::kMidAppMemBeg;
		725	case kMidAppMemEnd:
		726	return Mapping::kMidAppMemEnd;
		727	case kHiAppMemBeg:
		728	return Mapping::kHiAppMemBeg;
		729	case kHiAppMemEnd:
		730	return Mapping::kHiAppMemEnd;
		731	case kHeapMemBeg:
		732	return Mapping::kHeapMemBeg;
		733	case kHeapMemEnd:
		734	return Mapping::kHeapMemEnd;
		735	case kVdsoBeg:
		736	return Mapping::kVdsoBeg;
		737	case kShadowBeg:
		738	return Mapping::kShadowBeg;
		739	case kShadowEnd:
		740	return Mapping::kShadowEnd;
		741	case kMetaShadowBeg:
		742	return Mapping::kMetaShadowBeg;
		743	case kMetaShadowEnd:
		744	return Mapping::kMetaShadowEnd;
		745	}
		746	Die();
		747	}
		748	};
700		749
701	ALWAYS_INLINE	750	ALWAYS_INLINE
702	uptr HeapMemBeg(void) {	751	uptr LoAppMemBeg(void) { return SelectMapping<MappingField>(kLoAppMemBeg); }
703	return MappingArchImpl<MAPPING_HEAP_BEG>();
704	}
705	ALWAYS_INLINE	752	ALWAYS_INLINE
706	uptr HeapMemEnd(void) {	753	uptr LoAppMemEnd(void) { return SelectMapping<MappingField>(kLoAppMemEnd); }
707	return MappingArchImpl<MAPPING_HEAP_END>();
708	}
709		754
710	ALWAYS_INLINE	755	ALWAYS_INLINE
711	uptr HiAppMemBeg(void) {	756	uptr MidAppMemBeg(void) { return SelectMapping<MappingField>(kMidAppMemBeg); }
712	return MappingArchImpl<MAPPING_HI_APP_BEG>();
713	}
714	ALWAYS_INLINE	757	ALWAYS_INLINE
715	uptr HiAppMemEnd(void) {	758	uptr MidAppMemEnd(void) { return SelectMapping<MappingField>(kMidAppMemEnd); }
716	return MappingArchImpl<MAPPING_HI_APP_END>();
717	}
718		759
719	ALWAYS_INLINE	760	ALWAYS_INLINE
720	uptr VdsoBeg(void) {	761	uptr HeapMemBeg(void) { return SelectMapping<MappingField>(kHeapMemBeg); }
721	return MappingArchImpl<MAPPING_VDSO_BEG>();	762	ALWAYS_INLINE
722	}	763	uptr HeapMemEnd(void) { return SelectMapping<MappingField>(kHeapMemEnd); }
723
724	#else
725		764
726	ALWAYS_INLINE	765	ALWAYS_INLINE
727	uptr AppMemBeg(void) {	766	uptr HiAppMemBeg(void) { return SelectMapping<MappingField>(kHiAppMemBeg); }
728	return MappingArchImpl<MAPPING_APP_BEG>();
729	}
730	ALWAYS_INLINE	767	ALWAYS_INLINE
731	uptr AppMemEnd(void) {	768	uptr HiAppMemEnd(void) { return SelectMapping<MappingField>(kHiAppMemEnd); }
732	return MappingArchImpl<MAPPING_APP_END>();
733	}
734
735	#endif
736		769
737	static inline
738	bool GetUserRegion(int i, uptr start, uptr end) {
739	switch (i) {
740	default:
741	return false;
742	#if !SANITIZER_GO
743	case 0:
744	*start = LoAppMemBeg();
745	*end = LoAppMemEnd();
746	return true;
747	case 1:
748	*start = HiAppMemBeg();
749	*end = HiAppMemEnd();
750	return true;
751	case 2:
752	*start = HeapMemBeg();
753	*end = HeapMemEnd();
754	return true;
755	# ifdef TSAN_MID_APP_RANGE
756	case 3:
757	*start = MidAppMemBeg();
758	*end = MidAppMemEnd();
759	return true;
760	# endif
761	#else
762	case 0:
763	*start = AppMemBeg();
764	*end = AppMemEnd();
765	return true;
766	#endif
767	}
768	}
769
770	ALWAYS_INLINE
771	uptr ShadowBeg(void) {
772	return MappingArchImpl<MAPPING_SHADOW_BEG>();
773	}
774	ALWAYS_INLINE	770	ALWAYS_INLINE
775	uptr ShadowEnd(void) {	771	uptr VdsoBeg(void) { return SelectMapping<MappingField>(kVdsoBeg); }
776	return MappingArchImpl<MAPPING_SHADOW_END>();
777	}
778		772
779	ALWAYS_INLINE	773	ALWAYS_INLINE
780	uptr MetaShadowBeg(void) {	774	uptr ShadowBeg(void) { return SelectMapping<MappingField>(kShadowBeg); }
781	return MappingArchImpl<MAPPING_META_SHADOW_BEG>();
782	}
783	ALWAYS_INLINE	775	ALWAYS_INLINE
784	uptr MetaShadowEnd(void) {	776	uptr ShadowEnd(void) { return SelectMapping<MappingField>(kShadowEnd); }
785	return MappingArchImpl<MAPPING_META_SHADOW_END>();
786	}
787		777
788	ALWAYS_INLINE	778	ALWAYS_INLINE
789	uptr TraceMemBeg(void) {	779	uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(kMetaShadowBeg); }
790	return MappingArchImpl<MAPPING_TRACE_BEG>();
791	}
792	ALWAYS_INLINE	780	ALWAYS_INLINE
793	uptr TraceMemEnd(void) {	781	uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(kMetaShadowEnd); }
794	return MappingArchImpl<MAPPING_TRACE_END>();
795	}
796
797		782
798	template<typename Mapping>	783	struct IsAppMemImpl {
799	bool IsAppMemImpl(uptr mem) {	784	template <typename Mapping>
800	#if !SANITIZER_GO	785	static bool Apply(uptr mem) {
801	return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) \|\|	786	return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) \|\|
802	# ifdef TSAN_MID_APP_RANGE
803	(mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) \|\|	787	(mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) \|\|
804	# endif
805	(mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) \|\|	788	(mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) \|\|
806	(mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);	789	(mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);
807	#else
808	return mem >= Mapping::kAppMemBeg && mem < Mapping::kAppMemEnd;
809	#endif
810	}
811
812	ALWAYS_INLINE
813	bool IsAppMem(uptr mem) {
814	#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
815	switch (vmaSize) {
816	case 39: return IsAppMemImpl<Mapping39>(mem);
817	case 42: return IsAppMemImpl<Mapping42>(mem);
818	case 48: return IsAppMemImpl<Mapping48>(mem);
819	}
820	DCHECK(0);
821	return false;
822	#elif defined(__powerpc64__)
823	switch (vmaSize) {
824	#if !SANITIZER_GO
825	case 44: return IsAppMemImpl<Mapping44>(mem);
826	#endif
827	case 46: return IsAppMemImpl<Mapping46>(mem);
828	case 47: return IsAppMemImpl<Mapping47>(mem);
829	}
830	DCHECK(0);
831	return false;
832	#elif defined(__mips64)
833	switch (vmaSize) {
834	#if !SANITIZER_GO
835	case 40: return IsAppMemImpl<Mapping40>(mem);
836	#else
837	case 47: return IsAppMemImpl<Mapping47>(mem);
838	#endif
839	}	790	}
840	DCHECK(0);	791	};
841	return false;
842	#else
843	return IsAppMemImpl<Mapping>(mem);
844	#endif
845	}
846
847
848	template<typename Mapping>
849	bool IsShadowMemImpl(uptr mem) {
850	return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
851	}
852		792
853	ALWAYS_INLINE	793	ALWAYS_INLINE
854	bool IsShadowMem(uptr mem) {	794	bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(mem); }
855	#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
856	switch (vmaSize) {
857	case 39: return IsShadowMemImpl<Mapping39>(mem);
858	case 42: return IsShadowMemImpl<Mapping42>(mem);
859	case 48: return IsShadowMemImpl<Mapping48>(mem);
860	}
861	DCHECK(0);
862	return false;
863	#elif defined(__powerpc64__)
864	switch (vmaSize) {
865	#if !SANITIZER_GO
866	case 44: return IsShadowMemImpl<Mapping44>(mem);
867	#endif
868	case 46: return IsShadowMemImpl<Mapping46>(mem);
869	case 47: return IsShadowMemImpl<Mapping47>(mem);
870	}
871	DCHECK(0);
872	return false;
873	#elif defined(__mips64)
874	switch (vmaSize) {
875	#if !SANITIZER_GO
876	case 40: return IsShadowMemImpl<Mapping40>(mem);
877	#else
878	case 47: return IsShadowMemImpl<Mapping47>(mem);
879	#endif
880	}
881	DCHECK(0);
882	return false;
883	#else
884	return IsShadowMemImpl<Mapping>(mem);
885	#endif
886	}
887		795
888		796	struct IsShadowMemImpl {
889	template<typename Mapping>	797	template <typename Mapping>
890	bool IsMetaMemImpl(uptr mem) {	798	static bool Apply(uptr mem) {
891	return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;	799	return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
892	}
893
894	ALWAYS_INLINE
895	bool IsMetaMem(uptr mem) {
896	#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO
897	switch (vmaSize) {
898	case 39: return IsMetaMemImpl<Mapping39>(mem);
899	case 42: return IsMetaMemImpl<Mapping42>(mem);
900	case 48: return IsMetaMemImpl<Mapping48>(mem);
901	}
902	DCHECK(0);
903	return false;
904	#elif defined(__powerpc64__)
905	switch (vmaSize) {
906	#if !SANITIZER_GO
907	case 44: return IsMetaMemImpl<Mapping44>(mem);
908	#endif
909	case 46: return IsMetaMemImpl<Mapping46>(mem);
910	case 47: return IsMetaMemImpl<Mapping47>(mem);
911	}	800	}
912	DCHECK(0);	801	};
913	return false;
914	#elif defined(__mips64)
915	switch (vmaSize) {
916	#if !SANITIZER_GO
917	case 40: return IsMetaMemImpl<Mapping40>(mem);
918	#else
919	case 47: return IsMetaMemImpl<Mapping47>(mem);
920	#endif
921	}
922	DCHECK(0);
923	return false;
924	#else
925	return IsMetaMemImpl<Mapping>(mem);
926	#endif
927	}
928
929
930	template<typename Mapping>
931	uptr MemToShadowImpl(uptr x) {
932	DCHECK(IsAppMem(x));
933	#if !SANITIZER_GO
934	return (((x) & ~(Mapping::kAppMemMsk \| (kShadowCell - 1)))
935	^ Mapping::kAppMemXor) * kShadowCnt;
936	#else
937	# ifndef SANITIZER_WINDOWS
938	return ((x & ~(kShadowCell - 1)) * kShadowCnt) \| Mapping::kShadowBeg;
939	# else
940	return ((x & ~(kShadowCell - 1)) * kShadowCnt) + Mapping::kShadowBeg;
941	# endif
942	#endif
943	}
944		802
945	ALWAYS_INLINE	803	ALWAYS_INLINE
946	uptr MemToShadow(uptr x) {	804	bool IsShadowMem(RawShadow *p) {
947	#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO	805	return SelectMapping<IsShadowMemImpl>(reinterpret_cast<uptr>(p));
948	switch (vmaSize) {
949	case 39: return MemToShadowImpl<Mapping39>(x);
950	case 42: return MemToShadowImpl<Mapping42>(x);
951	case 48: return MemToShadowImpl<Mapping48>(x);
952	}
953	DCHECK(0);
954	return 0;
955	#elif defined(__powerpc64__)
956	switch (vmaSize) {
957	#if !SANITIZER_GO
958	case 44: return MemToShadowImpl<Mapping44>(x);
959	#endif
960	case 46: return MemToShadowImpl<Mapping46>(x);
961	case 47: return MemToShadowImpl<Mapping47>(x);
962	}
963	DCHECK(0);
964	return 0;
965	#elif defined(__mips64)
966	switch (vmaSize) {
967	#if !SANITIZER_GO
968	case 40: return MemToShadowImpl<Mapping40>(x);
969	#else
970	case 47: return MemToShadowImpl<Mapping47>(x);
971	#endif
972	}
973	DCHECK(0);
974	return 0;
975	#else
976	return MemToShadowImpl<Mapping>(x);
977	#endif
978	}	806	}
979		807
980		808	struct IsMetaMemImpl {
981	template<typename Mapping>	809	template <typename Mapping>
982	u32 *MemToMetaImpl(uptr x) {	810	static bool Apply(uptr mem) {
983	DCHECK(IsAppMem(x));	811	return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
984	#if !SANITIZER_GO	812	}
985	return (u32*)(((((x) & ~(Mapping::kAppMemMsk \| (kMetaShadowCell - 1)))) /	813	};
986	kMetaShadowCell * kMetaShadowSize) \| Mapping::kMetaShadowBeg);
987	#else
988	# ifndef SANITIZER_WINDOWS
989	return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
990	kMetaShadowCell * kMetaShadowSize) \| Mapping::kMetaShadowBeg);
991	# else
992	return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
993	kMetaShadowCell * kMetaShadowSize) + Mapping::kMetaShadowBeg);
994	# endif
995	#endif
996	}
997		814
998	ALWAYS_INLINE	815	ALWAYS_INLINE
999	u32 *MemToMeta(uptr x) {	816	bool IsMetaMem(const u32 *p) {
1000	#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO	817	return SelectMapping<IsMetaMemImpl>(reinterpret_cast<uptr>(p));
1001	switch (vmaSize) {	818	}
1002	case 39: return MemToMetaImpl<Mapping39>(x);	819
1003	case 42: return MemToMetaImpl<Mapping42>(x);	820	struct MemToShadowImpl {
1004	case 48: return MemToMetaImpl<Mapping48>(x);	821	template <typename Mapping>
1005	}	822	static uptr Apply(uptr x) {
1006	DCHECK(0);	823	DCHECK(IsAppMemImpl::Apply<Mapping>(x));
1007	return 0;	824	return (((x) & ~(Mapping::kShadowMsk \| (kShadowCell - 1))) ^
1008	#elif defined(__powerpc64__)	825	Mapping::kShadowXor) *
1009	switch (vmaSize) {	826	kShadowMultiplier +
1010	#if !SANITIZER_GO	827	Mapping::kShadowAdd;
1011	case 44: return MemToMetaImpl<Mapping44>(x);
1012	#endif
1013	case 46: return MemToMetaImpl<Mapping46>(x);
1014	case 47: return MemToMetaImpl<Mapping47>(x);
1015	}
1016	DCHECK(0);
1017	return 0;
1018	#elif defined(__mips64)
1019	switch (vmaSize) {
1020	#if !SANITIZER_GO
1021	case 40: return MemToMetaImpl<Mapping40>(x);
1022	#else
1023	case 47: return MemToMetaImpl<Mapping47>(x);
1024	#endif
1025	}	828	}
1026	DCHECK(0);	829	};
1027	return 0;
1028	#else
1029	return MemToMetaImpl<Mapping>(x);
1030	#endif
1031	}
1032
1033
1034	template<typename Mapping>
1035	uptr ShadowToMemImpl(uptr s) {
1036	DCHECK(IsShadowMem(s));
1037	#if !SANITIZER_GO
1038	// The shadow mapping is non-linear and we've lost some bits, so we don't have
1039	// an easy way to restore the original app address. But the mapping is a
1040	// bijection, so we try to restore the address as belonging to low/mid/high
1041	// range consecutively and see if shadow->app->shadow mapping gives us the
1042	// same address.
1043	uptr p = (s / kShadowCnt) ^ Mapping::kAppMemXor;
1044	if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
1045	MemToShadow(p) == s)
1046	return p;
1047	# ifdef TSAN_MID_APP_RANGE
1048	p = ((s / kShadowCnt) ^ Mapping::kAppMemXor) +
1049	(Mapping::kMidAppMemBeg & Mapping::kAppMemMsk);
1050	if (p >= Mapping::kMidAppMemBeg && p < Mapping::kMidAppMemEnd &&
1051	MemToShadow(p) == s)
1052	return p;
1053	# endif
1054	return ((s / kShadowCnt) ^ Mapping::kAppMemXor) \| Mapping::kAppMemMsk;
1055	#else // #if !SANITIZER_GO
1056	# ifndef SANITIZER_WINDOWS
1057	return (s & ~Mapping::kShadowBeg) / kShadowCnt;
1058	# else
1059	return (s - Mapping::kShadowBeg) / kShadowCnt;
1060	# endif // SANITIZER_WINDOWS
1061	#endif
1062	}
1063		830
1064	ALWAYS_INLINE	831	ALWAYS_INLINE
1065	uptr ShadowToMem(uptr s) {	832	RawShadow *MemToShadow(uptr x) {
1066	#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO	833	return reinterpret_cast<RawShadow *>(SelectMapping<MemToShadowImpl>(x));
1067	switch (vmaSize) {
1068	case 39: return ShadowToMemImpl<Mapping39>(s);
1069	case 42: return ShadowToMemImpl<Mapping42>(s);
1070	case 48: return ShadowToMemImpl<Mapping48>(s);
1071	}
1072	DCHECK(0);
1073	return 0;
1074	#elif defined(__powerpc64__)
1075	switch (vmaSize) {
1076	#if !SANITIZER_GO
1077	case 44: return ShadowToMemImpl<Mapping44>(s);
1078	#endif
1079	case 46: return ShadowToMemImpl<Mapping46>(s);
1080	case 47: return ShadowToMemImpl<Mapping47>(s);
1081	}
1082	DCHECK(0);
1083	return 0;
1084	#elif defined(__mips64)
1085	switch (vmaSize) {
1086	#if !SANITIZER_GO
1087	case 40: return ShadowToMemImpl<Mapping40>(s);
1088	#else
1089	case 47: return ShadowToMemImpl<Mapping47>(s);
1090	#endif
1091	}
1092	DCHECK(0);
1093	return 0;
1094	#else
1095	return ShadowToMemImpl<Mapping>(s);
1096	#endif
1097	}	834	}
1098		835
1099		836	struct MemToMetaImpl {
1100		837	template <typename Mapping>
1101	// The additional page is to catch shadow stack overflow as paging fault.	838	static u32 *Apply(uptr x) {
1102	// Windows wants 64K alignment for mmaps.	839	DCHECK(IsAppMemImpl::Apply<Mapping>(x));
1103	const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace)	840	return (u32 *)(((((x) & ~(Mapping::kShadowMsk \| (kMetaShadowCell - 1)))) /
1104	+ (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1);	841	kMetaShadowCell * kMetaShadowSize) \|
1105		842	Mapping::kMetaShadowBeg);
1106	template<typename Mapping>	843	}
1107	uptr GetThreadTraceImpl(int tid) {	844	};
1108	uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize;
1109	DCHECK_LT(p, Mapping::kTraceMemEnd);
1110	return p;
1111	}
1112		845
1113	ALWAYS_INLINE	846	ALWAYS_INLINE
1114	uptr GetThreadTrace(int tid) {	847	u32 *MemToMeta(uptr x) { return SelectMapping<MemToMetaImpl>(x); }
1115	#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO	848
1116	switch (vmaSize) {	849	struct ShadowToMemImpl {
1117	case 39: return GetThreadTraceImpl<Mapping39>(tid);	850	template <typename Mapping>
1118	case 42: return GetThreadTraceImpl<Mapping42>(tid);	851	static uptr Apply(uptr sp) {
1119	case 48: return GetThreadTraceImpl<Mapping48>(tid);	852	if (!IsShadowMemImpl::Apply<Mapping>(sp))
1120	}	853	return 0;
1121	DCHECK(0);	854	// The shadow mapping is non-linear and we've lost some bits, so we don't
1122	return 0;	855	// have an easy way to restore the original app address. But the mapping is
1123	#elif defined(__powerpc64__)	856	// a bijection, so we try to restore the address as belonging to
1124	switch (vmaSize) {	857	// low/mid/high range consecutively and see if shadow->app->shadow mapping
1125	#if !SANITIZER_GO	858	// gives us the same address.
1126	case 44: return GetThreadTraceImpl<Mapping44>(tid);	859	uptr p =
1127	#endif	860	((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor;
1128	case 46: return GetThreadTraceImpl<Mapping46>(tid);	861	if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
1129	case 47: return GetThreadTraceImpl<Mapping47>(tid);	862	MemToShadowImpl::Apply<Mapping>(p) == sp)
		863	return p;
		864	if (Mapping::kMidAppMemBeg) {
		865	uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk);
		866	if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd &&
		867	MemToShadowImpl::Apply<Mapping>(p_mid) == sp)
		868	return p_mid;
		869	}
		870	return p \| Mapping::kShadowMsk;
1130	}	871	}
1131	DCHECK(0);	872	};
1132	return 0;
1133	#elif defined(__mips64)
1134	switch (vmaSize) {
1135	#if !SANITIZER_GO
1136	case 40: return GetThreadTraceImpl<Mapping40>(tid);
1137	#else
1138	case 47: return GetThreadTraceImpl<Mapping47>(tid);
1139	#endif
1140	}
1141	DCHECK(0);
1142	return 0;
1143	#else
1144	return GetThreadTraceImpl<Mapping>(tid);
1145	#endif
1146	}
1147
1148
1149	template<typename Mapping>
1150	uptr GetThreadTraceHeaderImpl(int tid) {
1151	uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize
1152	+ kTraceSize * sizeof(Event);
1153	DCHECK_LT(p, Mapping::kTraceMemEnd);
1154	return p;
1155	}
1156		873
1157	ALWAYS_INLINE	874	ALWAYS_INLINE
1158	uptr GetThreadTraceHeader(int tid) {	875	uptr ShadowToMem(RawShadow *s) {
1159	#if defined(__aarch64__) && !defined(__APPLE__) && !SANITIZER_GO	876	return SelectMapping<ShadowToMemImpl>(reinterpret_cast<uptr>(s));
1160	switch (vmaSize) {	877	}
1161	case 39: return GetThreadTraceHeaderImpl<Mapping39>(tid);	878
1162	case 42: return GetThreadTraceHeaderImpl<Mapping42>(tid);	879	// Compresses addr to kCompressedAddrBits stored in least significant bits.
1163	case 48: return GetThreadTraceHeaderImpl<Mapping48>(tid);	880	ALWAYS_INLINE uptr CompressAddr(uptr addr) {
1164	}	881	return addr & ((1ull << kCompressedAddrBits) - 1);
1165	DCHECK(0);	882	}
1166	return 0;	883
1167	#elif defined(__powerpc64__)	884	struct RestoreAddrImpl {
1168	switch (vmaSize) {	885	typedef uptr Result;
1169	#if !SANITIZER_GO	886	template <typename Mapping>
1170	case 44: return GetThreadTraceHeaderImpl<Mapping44>(tid);	887	static Result Apply(uptr addr) {
1171	#endif	888	// To restore the address we go over all app memory ranges and check if top
1172	case 46: return GetThreadTraceHeaderImpl<Mapping46>(tid);	889	// 3 bits of the compressed addr match that of the app range. If yes, we
1173	case 47: return GetThreadTraceHeaderImpl<Mapping47>(tid);	890	// assume that the compressed address come from that range and restore the
		891	// missing top bits to match the app range address.
		892	const uptr ranges[] = {
		893	Mapping::kLoAppMemBeg, Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg,
		894	Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd,
		895	Mapping::kHeapMemBeg, Mapping::kHeapMemEnd,
		896	};
		897	const uptr indicator = 0x0e0000000000ull;
		898	const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator);
		899	for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) {
		900	uptr beg = ranges[i];
		901	uptr end = ranges[i + 1];
		902	if (beg == end)
		903	continue;
		904	for (uptr p = beg; p < end; p = RoundDown(p + ind_lsb, ind_lsb)) {
		905	if ((addr & indicator) == (p & indicator))
		906	return addr \| (p & ~(ind_lsb - 1));
		907	}
		908	}
		909	Printf("ThreadSanitizer: failed to restore address 0x%zx\n", addr);
		910	Die();
1174	}	911	}
1175	DCHECK(0);	912	};
1176	return 0;	913
1177	#elif defined(__mips64)	914	// Restores compressed addr from kCompressedAddrBits to full representation.
1178	switch (vmaSize) {	915	// This is called only during reporting and is not performance-critical.
1179	#if !SANITIZER_GO	916	inline uptr RestoreAddr(uptr addr) {
1180	case 40: return GetThreadTraceHeaderImpl<Mapping40>(tid);	917	return SelectMapping<RestoreAddrImpl>(addr);
1181	#else
1182	case 47: return GetThreadTraceHeaderImpl<Mapping47>(tid);
1183	#endif
1184	}
1185	DCHECK(0);
1186	return 0;
1187	#else
1188	return GetThreadTraceHeaderImpl<Mapping>(tid);
1189	#endif
1190	}	918	}
1191		919
1192	void InitializePlatform();	920	void InitializePlatform();
1193	void InitializePlatformEarly();	921	void InitializePlatformEarly();
1194	void CheckAndProtect();	922	void CheckAndProtect();
1195	void InitializeShadowMemoryPlatform();	923	void InitializeShadowMemoryPlatform();
1196	void FlushShadowMemory();	924	void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);
1197	void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive);
1198	int ExtractResolvFDs(void state, int fds, int nfd);	925	int ExtractResolvFDs(void state, int fds, int nfd);
1199	int ExtractRecvmsgFDs(void msg, int fds, int nfd);	926	int ExtractRecvmsgFDs(void msg, int fds, int nfd);
1200	uptr ExtractLongJmpSp(uptr *env);	927	uptr ExtractLongJmpSp(uptr *env);

lib/tsan/tsan_platform_linux.cpp+89-68

 @@ -66,7 +66,8 @@ extern "C" void *__libc_stack_end;
 void *__libc_stack_end = 0;
 #endif
-#if SANITIZER_LINUX && defined(__aarch64__) && !SANITIZER_GO
+#if SANITIZER_LINUX && (defined(__aarch64__) || defined(__loongarch_lp64)) && \
+    !SANITIZER_GO
 # define INIT_LONGJMP_XOR_KEY 1
 #else
 # define INIT_LONGJMP_XOR_KEY 0
 @@ -85,78 +86,71 @@ static void InitializeLongjmpXorKey();
 static uptr longjmp_xor_key;
 #endif
-#ifdef TSAN_RUNTIME_VMA
 // Runtime detected VMA size.
 uptr vmaSize;
-#endif
 enum {
-  MemTotal  = 0,
+  MemTotal,
-  MemShadow = 1,
+  MemShadow,
-  MemMeta   = 2,
+  MemMeta,
-  MemFile   = 3,
+  MemFile,
-  MemMmap   = 4,
+  MemMmap,
-  MemTrace  = 5,
+  MemHeap,
-  MemHeap   = 6,
+  MemOther,
-  MemOther  = 7,
+  MemCount,
-  MemCount  = 8,
 };
-void FillProfileCallback(uptr p, uptr rss, bool file,
+void FillProfileCallback(uptr p, uptr rss, bool file, uptr *mem) {
-                         uptr *mem, uptr stats_size) {
   mem[MemTotal] += rss;
   if (p >= ShadowBeg() && p < ShadowEnd())
     mem[MemShadow] += rss;
   else if (p >= MetaShadowBeg() && p < MetaShadowEnd())
     mem[MemMeta] += rss;
-#if !SANITIZER_GO
+  else if ((p >= LoAppMemBeg() && p < LoAppMemEnd()) ||
+           (p >= MidAppMemBeg() && p < MidAppMemEnd()) ||
+           (p >= HiAppMemBeg() && p < HiAppMemEnd()))
+    mem[file ? MemFile : MemMmap] += rss;
   else if (p >= HeapMemBeg() && p < HeapMemEnd())
     mem[MemHeap] += rss;
-  else if (p >= LoAppMemBeg() && p < LoAppMemEnd())
-    mem[file ? MemFile : MemMmap] += rss;
-  else if (p >= HiAppMemBeg() && p < HiAppMemEnd())
-    mem[file ? MemFile : MemMmap] += rss;
-#else
-  else if (p >= AppMemBeg() && p < AppMemEnd())
-    mem[file ? MemFile : MemMmap] += rss;
-#endif
-  else if (p >= TraceMemBeg() && p < TraceMemEnd())
-    mem[MemTrace] += rss;
   else
     mem[MemOther] += rss;
 }
-void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
+void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
   uptr mem[MemCount];
-  internal_memset(mem, 0, sizeof(mem[0]) * MemCount);
+  internal_memset(mem, 0, sizeof(mem));
-  __sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7);
+  GetMemoryProfile(FillProfileCallback, mem);
-  StackDepotStats *stacks = StackDepotGetStats();
+  auto meta = ctx->metamap.GetMemoryStats();
-  internal_snprintf(buf, buf_size,
+  StackDepotStats stacks = StackDepotGetStats();
-      "RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd"
+  uptr nthread, nlive;
-      " trace:%zd heap:%zd other:%zd stacks=%zd[%zd] nthr=%zd/%zd\n",
+  ctx->thread_registry.GetNumberOfThreads(&nthread, &nlive);
+  uptr trace_mem;
+  {
+    Lock l(&ctx->slot_mtx);
+    trace_mem = ctx->trace_part_total_allocated * sizeof(TracePart);
+  }
+  uptr internal_stats[AllocatorStatCount];
+  internal_allocator()->GetStats(internal_stats);
+  // All these are allocated from the common mmap region.
+  mem[MemMmap] -= meta.mem_block + meta.sync_obj + trace_mem +
+                  stacks.allocated + internal_stats[AllocatorStatMapped];
+  if (s64(mem[MemMmap]) < 0)
+    mem[MemMmap] = 0;
+  internal_snprintf(
+      buf, buf_size,
+      "==%zu== %llus [%zu]: RSS %zd MB: shadow:%zd meta:%zd file:%zd"
+      " mmap:%zd heap:%zd other:%zd intalloc:%zd memblocks:%zd syncobj:%zu"
+      " trace:%zu stacks=%zd threads=%zu/%zu\n",
+      internal_getpid(), uptime_ns / (1000 * 1000 * 1000), ctx->global_epoch,
       mem[MemTotal] >> 20, mem[MemShadow] >> 20, mem[MemMeta] >> 20,
-      mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemTrace] >> 20,
+      mem[MemFile] >> 20, mem[MemMmap] >> 20, mem[MemHeap] >> 20,
-      mem[MemHeap] >> 20, mem[MemOther] >> 20,
+      mem[MemOther] >> 20, internal_stats[AllocatorStatMapped] >> 20,
-      stacks->allocated >> 20, stacks->n_uniq_ids,
+      meta.mem_block >> 20, meta.sync_obj >> 20, trace_mem >> 20,
-      nlive, nthread);
+      stacks.allocated >> 20, nlive, nthread);
-}
-#if SANITIZER_LINUX
-void FlushShadowMemoryCallback(
-    const SuspendedThreadsList &suspended_threads_list,
-    void *argument) {
-  ReleaseMemoryPagesToOS(ShadowBeg(), ShadowEnd());
-}
-#endif
-void FlushShadowMemory() {
-#if SANITIZER_LINUX
-  StopTheWorld(FlushShadowMemoryCallback, 0);
-#endif
 }
 #if !SANITIZER_GO
-// Mark shadow for .rodata sections with the special kShadowRodata marker.
+// Mark shadow for .rodata sections with the special Shadow::kRodata marker.
 // Accesses to .rodata can't race, so this saves time, memory and trace space.
 static void MapRodata() {
   // First create temp file.
 @@ -177,13 +171,14 @@ static void MapRodata() {
     return;
   internal_unlink(name);  // Unlink it now, so that we can reuse the buffer.
   fd_t fd = openrv;
-  // Fill the file with kShadowRodata.
+  // Fill the file with Shadow::kRodata.
-  const uptr kMarkerSize = 512 * 1024 / sizeof(u64);
+  const uptr kMarkerSize = 512 * 1024 / sizeof(RawShadow);
-  InternalMmapVector<u64> marker(kMarkerSize);
+  InternalMmapVector<RawShadow> marker(kMarkerSize);
   // volatile to prevent insertion of memset
-  for (volatile u64 *p = marker.data(); p < marker.data() + kMarkerSize; p++)
+  for (volatile RawShadow *p = marker.data(); p < marker.data() + kMarkerSize;
-    *p = kShadowRodata;
+       p++)
-  internal_write(fd, marker.data(), marker.size() * sizeof(u64));
+    *p = Shadow::kRodata;
+  internal_write(fd, marker.data(), marker.size() * sizeof(RawShadow));
   // Map the file into memory.
   uptr page = internal_mmap(0, GetPageSizeCached(), PROT_READ | PROT_WRITE,
                             MAP_PRIVATE | MAP_ANONYMOUS, fd, 0);
 @@ -203,9 +198,10 @@ static void MapRodata() {
       char *shadow_start = (char *)MemToShadow(segment.start);
       char *shadow_end = (char *)MemToShadow(segment.end);
       for (char *p = shadow_start; p < shadow_end;
-           p += marker.size() * sizeof(u64)) {
+           p += marker.size() * sizeof(RawShadow)) {
-        internal_mmap(p, Min<uptr>(marker.size() * sizeof(u64), shadow_end - p),
+        internal_mmap(
-                      PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0);
+            p, Min<uptr>(marker.size() * sizeof(RawShadow), shadow_end - p),
+            PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0);
       }
     }
   }
 @@ -219,7 +215,6 @@ void InitializeShadowMemoryPlatform() {
 #endif  // #if !SANITIZER_GO
 void InitializePlatformEarly() {
-#ifdef TSAN_RUNTIME_VMA
   vmaSize =
     (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
 #if defined(__aarch64__)
 @@ -236,6 +231,14 @@ void InitializePlatformEarly() {
     Die();
   }
 #endif
+#elif SANITIZER_LOONGARCH64
+# if !SANITIZER_GO
+  if (vmaSize != 47) {
+    Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
+    Printf("FATAL: Found %zd - Supported 47\n", vmaSize);
+    Die();
+  }
+# endif
 #elif defined(__powerpc64__)
 # if !SANITIZER_GO
   if (vmaSize != 44 && vmaSize != 46 && vmaSize != 47) {
 @@ -265,7 +268,6 @@ void InitializePlatformEarly() {
   }
 # endif
 #endif
-#endif
 }
 void InitializePlatform() {
 @@ -297,11 +299,12 @@ void InitializePlatform() {
       SetAddressSpaceUnlimited();
       reexec = true;
     }
-#if SANITIZER_LINUX && defined(__aarch64__)
+#if SANITIZER_ANDROID && (defined(__aarch64__) || defined(__x86_64__))
     // After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in
     // linux kernel, the random gap between stack and mapped area is increased
     // from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover
     // this big range, we should disable randomized virtual space on aarch64.
+    // ASLR personality check.
     int old_personality = personality(0xffffffff);
     if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) {
       VReport(1, "WARNING: Program is run with randomized virtual address "
 @@ -310,6 +313,9 @@ void InitializePlatform() {
       CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1);
       reexec = true;
     }
+#endif
+#if SANITIZER_LINUX && (defined(__aarch64__) || defined(__loongarch_lp64))
     // Initialize the xor key used in {sig}{set,long}jump.
     InitializeLongjmpXorKey();
 #endif
 @@ -341,7 +347,7 @@ int ExtractResolvFDs(void *state, int *fds, int nfd) {
 }
 // Extract file descriptors passed via UNIX domain sockets.
-// This is requried to properly handle "open" of these fds.
+// This is required to properly handle "open" of these fds.
 // see 'man recvmsg' and 'man 3 cmsg'.
 int ExtractRecvmsgFDs(void *msgp, int *fds, int nfd) {
   int res = 0;
 @@ -382,6 +388,8 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {
 # else
   return mangled_sp;
 # endif
+#elif defined(__loongarch_lp64)
+  return mangled_sp ^ longjmp_xor_key;
 #elif defined(__powerpc64__)
   // Reverse of:
   //   ld   r4, -28696(r13)
 @@ -409,10 +417,16 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) {
 #elif defined(__powerpc__)
 # define LONG_JMP_SP_ENV_SLOT 0
 #elif SANITIZER_FREEBSD
-# define LONG_JMP_SP_ENV_SLOT 2
+# ifdef __aarch64__
+#  define LONG_JMP_SP_ENV_SLOT 1
+# else
+#  define LONG_JMP_SP_ENV_SLOT 2
+# endif
 #elif SANITIZER_LINUX
 # ifdef __aarch64__
 #  define LONG_JMP_SP_ENV_SLOT 13
+# elif defined(__loongarch__)
+#  define LONG_JMP_SP_ENV_SLOT 1
 # elif defined(__mips64)
 #  define LONG_JMP_SP_ENV_SLOT 1
 # elif defined(__s390x__)
 @@ -439,7 +453,11 @@ static void InitializeLongjmpXorKey() {
   // 2. Retrieve vanilla/mangled SP.
   uptr sp;
+#ifdef __loongarch__
+  asm("move  %0, $sp" : "=r" (sp));
+#else
   asm("mov  %0, sp" : "=r" (sp));
+#endif
   uptr mangled_sp = ((uptr *)&env)[LONG_JMP_SP_ENV_SLOT];
   // 3. xor SPs to obtain key.
 @@ -447,6 +465,8 @@ static void InitializeLongjmpXorKey() {
 }
 #endif
+extern "C" void __tsan_tls_initialization() {}
 void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
   // Check that the thr object is in tls;
   const uptr thr_beg = (uptr)thr;
 @@ -456,9 +476,10 @@ void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
   CHECK_GE(thr_end, tls_addr);
   CHECK_LE(thr_end, tls_addr + tls_size);
   // Since the thr object is huge, skip it.
-  MemoryRangeImitateWrite(thr, /*pc=*/2, tls_addr, thr_beg - tls_addr);
+  const uptr pc = StackTrace::GetNextInstructionPc(
-  MemoryRangeImitateWrite(thr, /*pc=*/2, thr_end,
+      reinterpret_cast<uptr>(__tsan_tls_initialization));
-                          tls_addr + tls_size - thr_end);
+  MemoryRangeImitateWrite(thr, pc, tls_addr, thr_beg - tls_addr);
+  MemoryRangeImitateWrite(thr, pc, thr_end, tls_addr + tls_size - thr_end);
 }
 // Note: this function runs with async signals enabled,

lib/tsan/tsan_platform_mac.cpp+130-149

...	@@ -12,7 +12,7 @@	...	@@ -12,7 +12,7 @@
12	//===----------------------------------------------------------------------===//	12	//===----------------------------------------------------------------------===//
13		13
14	#include "sanitizer_common/sanitizer_platform.h"	14	#include "sanitizer_common/sanitizer_platform.h"
15	#if SANITIZER_MAC	15	#if SANITIZER_APPLE
16		16
17	#include "sanitizer_common/sanitizer_atomic.h"	17	#include "sanitizer_common/sanitizer_atomic.h"
18	#include "sanitizer_common/sanitizer_common.h"	18	#include "sanitizer_common/sanitizer_common.h"
...	@@ -25,6 +25,7 @@	...	@@ -25,6 +25,7 @@
25	#include "tsan_rtl.h"	25	#include "tsan_rtl.h"
26	#include "tsan_flags.h"	26	#include "tsan_flags.h"
27		27
		28	#include <limits.h>
28	#include <mach/mach.h>	29	#include <mach/mach.h>
29	#include <pthread.h>	30	#include <pthread.h>
30	#include <signal.h>	31	#include <signal.h>
...	@@ -45,76 +46,86 @@	...	@@ -45,76 +46,86 @@
45	namespace __tsan {	46	namespace __tsan {
46		47
47	#if !SANITIZER_GO	48	#if !SANITIZER_GO
48	static void SignalSafeGetOrAllocate(uptr dst, uptr size) {	49	static char main_thread_state[sizeof(ThreadState)] ALIGNED(
49	atomic_uintptr_t a = (atomic_uintptr_t )dst;	50	SANITIZER_CACHE_LINE_SIZE);
50	void val = (void )atomic_load_relaxed(a);	51	static ThreadState *dead_thread_state;
51	atomic_signal_fence(memory_order_acquire); // Turns the previous load into	52	static pthread_key_t thread_state_key;
52	// acquire wrt signals.	53
53	if (UNLIKELY(val == nullptr)) {	54	// We rely on the following documented, but Darwin-specific behavior to keep the
54	val = (void *)internal_mmap(nullptr, size, PROT_READ \| PROT_WRITE,	55	// reference to the ThreadState object alive in TLS:
55	MAP_PRIVATE \| MAP_ANON, -1, 0);	56	// pthread_key_create man page:
56	CHECK(val);	57	// If, after all the destructors have been called for all non-NULL values with
57	void *cmp = nullptr;	58	// associated destructors, there are still some non-NULL values with
58	if (!atomic_compare_exchange_strong(a, (uintptr_t *)&cmp, (uintptr_t)val,	59	// associated destructors, then the process is repeated. If, after at least
59	memory_order_acq_rel)) {	60	// [PTHREAD_DESTRUCTOR_ITERATIONS] iterations of destructor calls for
60	internal_munmap(val, size);	61	// outstanding non-NULL values, there are still some non-NULL values with
61	val = cmp;	62	// associated destructors, the implementation stops calling destructors.
62	}	63	static_assert(PTHREAD_DESTRUCTOR_ITERATIONS == 4, "Small number of iterations");
63	}	64	static void ThreadStateDestructor(void *thr) {
64	return val;	65	int res = pthread_setspecific(thread_state_key, thr);
		66	CHECK_EQ(res, 0);
65	}	67	}
66		68
67	// On OS X, accessing TLVs via __thread or manually by using pthread_key_* is	69	static void InitializeThreadStateStorage() {
68	// problematic, because there are several places where interceptors are called	70	int res;
69	// when TLVs are not accessible (early process startup, thread cleanup, ...).	71	CHECK_EQ(thread_state_key, 0);
70	// The following provides a "poor man's TLV" implementation, where we use the	72	res = pthread_key_create(&thread_state_key, ThreadStateDestructor);
71	// shadow memory of the pointer returned by pthread_self() to store a pointer to	73	CHECK_EQ(res, 0);
72	// the ThreadState object. The main thread's ThreadState is stored separately	74	res = pthread_setspecific(thread_state_key, main_thread_state);
73	// in a static variable, because we need to access it even before the	75	CHECK_EQ(res, 0);
74	// shadow memory is set up.	76
75	static uptr main_thread_identity = 0;	77	auto dts = (ThreadState *)MmapOrDie(sizeof(ThreadState), "ThreadState");
76	ALIGNED(64) static char main_thread_state[sizeof(ThreadState)];	78	dts->fast_state.SetIgnoreBit();
77	static ThreadState main_thread_state_loc = (ThreadState )main_thread_state;	79	dts->ignore_interceptors = 1;
78		80	dts->is_dead = true;
79	// We cannot use pthread_self() before libpthread has been initialized. Our	81	const_cast<Tid &>(dts->tid) = kInvalidTid;
80	// current heuristic for guarding this is checking `main_thread_identity` which	82	res = internal_mprotect(dts, sizeof(ThreadState), PROT_READ); // immutable
81	// is only assigned in `__tsan::InitializePlatform`.	83	CHECK_EQ(res, 0);
82	static ThreadState **cur_thread_location() {	84	dead_thread_state = dts;
83	if (main_thread_identity == 0)
84	return &main_thread_state_loc;
85	uptr thread_identity = (uptr)pthread_self();
86	if (thread_identity == main_thread_identity)
87	return &main_thread_state_loc;
88	return (ThreadState **)MemToShadow(thread_identity);
89	}	85	}
90		86
91	ThreadState *cur_thread() {	87	ThreadState *cur_thread() {
92	return (ThreadState *)SignalSafeGetOrAllocate(	88	// Some interceptors get called before libpthread has been initialized and in
93	(uptr *)cur_thread_location(), sizeof(ThreadState));	89	// these cases we must avoid calling any pthread APIs.
		90	if (UNLIKELY(!thread_state_key)) {
		91	return (ThreadState *)main_thread_state;
		92	}
		93
		94	// We only reach this line after InitializeThreadStateStorage() ran, i.e,
		95	// after TSan (and therefore libpthread) have been initialized.
		96	ThreadState thr = (ThreadState )pthread_getspecific(thread_state_key);
		97	if (UNLIKELY(!thr)) {
		98	thr = (ThreadState *)MmapOrDie(sizeof(ThreadState), "ThreadState");
		99	int res = pthread_setspecific(thread_state_key, thr);
		100	CHECK_EQ(res, 0);
		101	}
		102	return thr;
94	}	103	}
95		104
96	void set_cur_thread(ThreadState *thr) {	105	void set_cur_thread(ThreadState *thr) {
97	*cur_thread_location() = thr;	106	int res = pthread_setspecific(thread_state_key, thr);
		107	CHECK_EQ(res, 0);
98	}	108	}
99		109
100	// TODO(kuba.brecka): This is not async-signal-safe. In particular, we call
101	// munmap first and then clear `fake_tls`; if we receive a signal in between,
102	// handler will try to access the unmapped ThreadState.
103	void cur_thread_finalize() {	110	void cur_thread_finalize() {
104	ThreadState **thr_state_loc = cur_thread_location();	111	ThreadState thr = (ThreadState )pthread_getspecific(thread_state_key);
105	if (thr_state_loc == &main_thread_state_loc) {	112	CHECK(thr);
		113	if (thr == (ThreadState *)main_thread_state) {
106	// Calling dispatch_main() or xpc_main() actually invokes pthread_exit to	114	// Calling dispatch_main() or xpc_main() actually invokes pthread_exit to
107	// exit the main thread. Let's keep the main thread's ThreadState.	115	// exit the main thread. Let's keep the main thread's ThreadState.
108	return;	116	return;
109	}	117	}
110	internal_munmap(*thr_state_loc, sizeof(ThreadState));	118	// Intercepted functions can still get called after cur_thread_finalize()
111	*thr_state_loc = nullptr;	119	// (called from DestroyThreadState()), so put a fake thread state for "dead"
		120	// threads. An alternative solution would be to release the ThreadState
		121	// object from THREAD_DESTROY (which is delivered later and on the parent
		122	// thread) instead of THREAD_TERMINATE.
		123	int res = pthread_setspecific(thread_state_key, dead_thread_state);
		124	CHECK_EQ(res, 0);
		125	UnmapOrDie(thr, sizeof(ThreadState));
112	}	126	}
113	#endif	127	#endif
114		128
115	void FlushShadowMemory() {
116	}
117
118	static void RegionMemUsage(uptr start, uptr end, uptr res, uptr dirty) {	129	static void RegionMemUsage(uptr start, uptr end, uptr res, uptr dirty) {
119	vm_address_t address = start;	130	vm_address_t address = start;
120	vm_address_t end_address = end;	131	vm_address_t end_address = end;
...	@@ -139,15 +150,13 @@ static void RegionMemUsage(uptr start, uptr end, uptr res, uptr dirty) {	...	@@ -139,15 +150,13 @@ static void RegionMemUsage(uptr start, uptr end, uptr res, uptr dirty) {
139	dirty = dirty_pages GetPageSizeCached();	150	dirty = dirty_pages GetPageSizeCached();
140	}	151	}
141		152
142	void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {	153	void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
143	uptr shadow_res, shadow_dirty;	154	uptr shadow_res, shadow_dirty;
144	uptr meta_res, meta_dirty;	155	uptr meta_res, meta_dirty;
145	uptr trace_res, trace_dirty;
146	RegionMemUsage(ShadowBeg(), ShadowEnd(), &shadow_res, &shadow_dirty);	156	RegionMemUsage(ShadowBeg(), ShadowEnd(), &shadow_res, &shadow_dirty);
147	RegionMemUsage(MetaShadowBeg(), MetaShadowEnd(), &meta_res, &meta_dirty);	157	RegionMemUsage(MetaShadowBeg(), MetaShadowEnd(), &meta_res, &meta_dirty);
148	RegionMemUsage(TraceMemBeg(), TraceMemEnd(), &trace_res, &trace_dirty);
149		158
150	#if !SANITIZER_GO	159	# if !SANITIZER_GO
151	uptr low_res, low_dirty;	160	uptr low_res, low_dirty;
152	uptr high_res, high_dirty;	161	uptr high_res, high_dirty;
153	uptr heap_res, heap_dirty;	162	uptr heap_res, heap_dirty;
...	@@ -156,89 +165,70 @@ void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {	...	@@ -156,89 +165,70 @@ void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) {
156	RegionMemUsage(HeapMemBeg(), HeapMemEnd(), &heap_res, &heap_dirty);	165	RegionMemUsage(HeapMemBeg(), HeapMemEnd(), &heap_res, &heap_dirty);
157	#else // !SANITIZER_GO	166	#else // !SANITIZER_GO
158	uptr app_res, app_dirty;	167	uptr app_res, app_dirty;
159	RegionMemUsage(AppMemBeg(), AppMemEnd(), &app_res, &app_dirty);	168	RegionMemUsage(LoAppMemBeg(), LoAppMemEnd(), &app_res, &app_dirty);
160	#endif	169	#endif
161		170
162	StackDepotStats *stacks = StackDepotGetStats();	171	StackDepotStats stacks = StackDepotGetStats();
163	internal_snprintf(buf, buf_size,	172	uptr nthread, nlive;
164	"shadow (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"	173	ctx->thread_registry.GetNumberOfThreads(&nthread, &nlive);
165	"meta (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"	174	internal_snprintf(
166	"traces (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"	175	buf, buf_size,
167	#if !SANITIZER_GO	176	"shadow (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
168	"low app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"	177	"meta (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
169	"high app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"	178	# if !SANITIZER_GO
170	"heap (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"	179	"low app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
171	#else // !SANITIZER_GO	180	"high app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
172	"app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"	181	"heap (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
173	#endif	182	# else // !SANITIZER_GO
174	"stacks: %zd unique IDs, %zd kB allocated\n"	183	"app (0x%016zx-0x%016zx): resident %zd kB, dirty %zd kB\n"
175	"threads: %zd total, %zd live\n"	184	# endif
176	"------------------------------\n",	185	"stacks: %zd unique IDs, %zd kB allocated\n"
177	ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024,	186	"threads: %zd total, %zd live\n"
178	MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024,	187	"------------------------------\n",
179	TraceMemBeg(), TraceMemEnd(), trace_res / 1024, trace_dirty / 1024,	188	ShadowBeg(), ShadowEnd(), shadow_res / 1024, shadow_dirty / 1024,
180	#if !SANITIZER_GO	189	MetaShadowBeg(), MetaShadowEnd(), meta_res / 1024, meta_dirty / 1024,
181	LoAppMemBeg(), LoAppMemEnd(), low_res / 1024, low_dirty / 1024,	190	# if !SANITIZER_GO
182	HiAppMemBeg(), HiAppMemEnd(), high_res / 1024, high_dirty / 1024,	191	LoAppMemBeg(), LoAppMemEnd(), low_res / 1024, low_dirty / 1024,
183	HeapMemBeg(), HeapMemEnd(), heap_res / 1024, heap_dirty / 1024,	192	HiAppMemBeg(), HiAppMemEnd(), high_res / 1024, high_dirty / 1024,
184	#else // !SANITIZER_GO	193	HeapMemBeg(), HeapMemEnd(), heap_res / 1024, heap_dirty / 1024,
185	AppMemBeg(), AppMemEnd(), app_res / 1024, app_dirty / 1024,	194	# else // !SANITIZER_GO
186	#endif	195	LoAppMemBeg(), LoAppMemEnd(), app_res / 1024, app_dirty / 1024,
187	stacks->n_uniq_ids, stacks->allocated / 1024,	196	# endif
188	nthread, nlive);	197	stacks.n_uniq_ids, stacks.allocated / 1024, nthread, nlive);
189	}	198	}
190		199
191	#if !SANITIZER_GO	200	# if !SANITIZER_GO
192	void InitializeShadowMemoryPlatform() { }	201	void InitializeShadowMemoryPlatform() { }
193		202
194	// On OS X, GCD worker threads are created without a call to pthread_create. We	203	// Register GCD worker threads, which are created without an observable call to
195	// need to properly register these threads with ThreadCreate and ThreadStart.	204	// pthread_create().
196	// These threads don't have a parent thread, as they are created "spuriously".	205	static void ThreadCreateCallback(uptr thread, bool gcd_worker) {
197	// We're using a libpthread API that notifies us about a newly created thread.	206	if (gcd_worker) {
198	// The `thread == pthread_self()` check indicates this is actually a worker	207	ThreadState *thr = cur_thread();
199	// thread. If it's just a regular thread, this hook is called on the parent	208	Processor *proc = ProcCreate();
200	// thread.	209	ProcWire(proc, thr);
201	typedef void (*pthread_introspection_hook_t)(unsigned int event,	210	ThreadState *parent_thread_state = nullptr; // No parent.
202	pthread_t thread, void *addr,	211	Tid tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true);
203	size_t size);	212	CHECK_NE(tid, kMainTid);
204	extern "C" pthread_introspection_hook_t pthread_introspection_hook_install(	213	ThreadStart(thr, tid, GetTid(), ThreadType::Worker);
205	pthread_introspection_hook_t hook);
206	static const uptr PTHREAD_INTROSPECTION_THREAD_CREATE = 1;
207	static const uptr PTHREAD_INTROSPECTION_THREAD_TERMINATE = 3;
208	static pthread_introspection_hook_t prev_pthread_introspection_hook;
209	static void my_pthread_introspection_hook(unsigned int event, pthread_t thread,
210	void *addr, size_t size) {
211	if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) {
212	if (thread == pthread_self()) {
213	// The current thread is a newly created GCD worker thread.
214	ThreadState *thr = cur_thread();
215	Processor *proc = ProcCreate();
216	ProcWire(proc, thr);
217	ThreadState *parent_thread_state = nullptr; // No parent.
218	int tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true);
219	CHECK_NE(tid, 0);
220	ThreadStart(thr, tid, GetTid(), ThreadType::Worker);
221	}
222	} else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) {
223	if (thread == pthread_self()) {
224	ThreadState *thr = cur_thread();
225	if (thr->tctx) {
226	DestroyThreadState();
227	}
228	}
229	}	214	}
		215	}
230		216
231	if (prev_pthread_introspection_hook != nullptr)	217	// Destroy thread state for all threads.
232	prev_pthread_introspection_hook(event, thread, addr, size);	218	static void ThreadTerminateCallback(uptr thread) {
		219	ThreadState *thr = cur_thread();
		220	if (thr->tctx) {
		221	DestroyThreadState();
		222	}
233	}	223	}
234	#endif	224	#endif
235		225
236	void InitializePlatformEarly() {	226	void InitializePlatformEarly() {
237	#if !SANITIZER_GO && !HAS_48_BIT_ADDRESS_SPACE	227	# if !SANITIZER_GO && SANITIZER_IOS
238	uptr max_vm = GetMaxUserVirtualAddress() + 1;	228	uptr max_vm = GetMaxUserVirtualAddress() + 1;
239	if (max_vm != Mapping::kHiAppMemEnd) {	229	if (max_vm != HiAppMemEnd()) {
240	Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",	230	Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
241	max_vm, Mapping::kHiAppMemEnd);	231	(void )max_vm, (void )HiAppMemEnd());
242	Die();	232	Die();
243	}	233	}
244	#endif	234	#endif
...	@@ -251,11 +241,13 @@ void InitializePlatform() {	...	@@ -251,11 +241,13 @@ void InitializePlatform() {
251	#if !SANITIZER_GO	241	#if !SANITIZER_GO
252	CheckAndProtect();	242	CheckAndProtect();
253		243
254	CHECK_EQ(main_thread_identity, 0);	244	InitializeThreadStateStorage();
255	main_thread_identity = (uptr)pthread_self();
256		245
257	prev_pthread_introspection_hook =	246	ThreadEventCallbacks callbacks = {
258	pthread_introspection_hook_install(&my_pthread_introspection_hook);	247	.create = ThreadCreateCallback,
		248	.terminate = ThreadTerminateCallback,
		249	};
		250	InstallPthreadIntrospectionHook(callbacks);
259	#endif	251	#endif
260		252
261	if (GetMacosAlignedVersion() >= MacosVersion(10, 14)) {	253	if (GetMacosAlignedVersion() >= MacosVersion(10, 14)) {
...	@@ -281,25 +273,14 @@ uptr ExtractLongJmpSp(uptr *env) {	...	@@ -281,25 +273,14 @@ uptr ExtractLongJmpSp(uptr *env) {
281	}	273	}
282		274
283	#if !SANITIZER_GO	275	#if !SANITIZER_GO
		276	extern "C" void __tsan_tls_initialization() {}
		277
284	void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {	278	void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size) {
285	// The pointer to the ThreadState object is stored in the shadow memory	279	const uptr pc = StackTrace::GetNextInstructionPc(
286	// of the tls.	280	reinterpret_cast<uptr>(__tsan_tls_initialization));
287	uptr tls_end = tls_addr + tls_size;	281	// Unlike Linux, we only store a pointer to the ThreadState object in TLS;
288	uptr thread_identity = (uptr)pthread_self();	282	// just mark the entire range as written to.
289	if (thread_identity == main_thread_identity) {	283	MemoryRangeImitateWrite(thr, pc, tls_addr, tls_size);
290	MemoryRangeImitateWrite(thr, /pc=/2, tls_addr, tls_size);
291	} else {
292	uptr thr_state_start = thread_identity;
293	uptr thr_state_end = thr_state_start + sizeof(uptr);
294	CHECK_GE(thr_state_start, tls_addr);
295	CHECK_LE(thr_state_start, tls_addr + tls_size);
296	CHECK_GE(thr_state_end, tls_addr);
297	CHECK_LE(thr_state_end, tls_addr + tls_size);
298	MemoryRangeImitateWrite(thr, /pc=/2, tls_addr,
299	thr_state_start - tls_addr);
300	MemoryRangeImitateWrite(thr, /pc=/2, thr_state_end,
301	tls_end - thr_state_end);
302	}
303	}	284	}
304	#endif	285	#endif
305		286
...	@@ -320,4 +301,4 @@ int call_pthread_cancel_with_cleanup(int (fn)(void arg),	...	@@ -320,4 +301,4 @@ int call_pthread_cancel_with_cleanup(int (fn)(void arg),
320		301
321	} // namespace __tsan	302	} // namespace __tsan
322		303
323	#endif // SANITIZER_MAC	304	#endif // SANITIZER_APPLE

lib/tsan/tsan_platform_posix.cpp+27-24

...	@@ -14,12 +14,14 @@	...	@@ -14,12 +14,14 @@
14	#include "sanitizer_common/sanitizer_platform.h"	14	#include "sanitizer_common/sanitizer_platform.h"
15	#if SANITIZER_POSIX	15	#if SANITIZER_POSIX
16		16
17	#include "sanitizer_common/sanitizer_common.h"	17	# include <dlfcn.h>
18	#include "sanitizer_common/sanitizer_errno.h"	18
19	#include "sanitizer_common/sanitizer_libc.h"	19	# include "sanitizer_common/sanitizer_common.h"
20	#include "sanitizer_common/sanitizer_procmaps.h"	20	# include "sanitizer_common/sanitizer_errno.h"
21	#include "tsan_platform.h"	21	# include "sanitizer_common/sanitizer_libc.h"
22	#include "tsan_rtl.h"	22	# include "sanitizer_common/sanitizer_procmaps.h"
		23	# include "tsan_platform.h"
		24	# include "tsan_rtl.h"
23		25
24	namespace __tsan {	26	namespace __tsan {
25		27
...	@@ -29,7 +31,8 @@ static const char kShadowMemoryMappingHint[] =	...	@@ -29,7 +31,8 @@ static const char kShadowMemoryMappingHint[] =
29	"HINT: if %s is not supported in your environment, you may set "	31	"HINT: if %s is not supported in your environment, you may set "
30	"TSAN_OPTIONS=%s=0\n";	32	"TSAN_OPTIONS=%s=0\n";
31		33
32	static void DontDumpShadow(uptr addr, uptr size) {	34	# if !SANITIZER_GO
		35	void DontDumpShadow(uptr addr, uptr size) {
33	if (common_flags()->use_madv_dontdump)	36	if (common_flags()->use_madv_dontdump)
34	if (!DontDumpShadowMemory(addr, size)) {	37	if (!DontDumpShadowMemory(addr, size)) {
35	Printf(kShadowMemoryMappingWarning, SanitizerToolName, addr, addr + size,	38	Printf(kShadowMemoryMappingWarning, SanitizerToolName, addr, addr + size,
...	@@ -39,7 +42,6 @@ static void DontDumpShadow(uptr addr, uptr size) {	...	@@ -39,7 +42,6 @@ static void DontDumpShadow(uptr addr, uptr size) {
39	}	42	}
40	}	43	}
41		44
42	#if !SANITIZER_GO
43	void InitializeShadowMemory() {	45	void InitializeShadowMemory() {
44	// Map memory shadow.	46	// Map memory shadow.
45	if (!MmapFixedSuperNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(),	47	if (!MmapFixedSuperNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(),
...	@@ -70,6 +72,11 @@ void InitializeShadowMemory() {	...	@@ -70,6 +72,11 @@ void InitializeShadowMemory() {
70	meta, meta + meta_size, meta_size >> 30);	72	meta, meta + meta_size, meta_size >> 30);
71		73
72	InitializeShadowMemoryPlatform();	74	InitializeShadowMemoryPlatform();
		75
		76	on_initialize = reinterpret_cast<void (*)(void)>(
		77	dlsym(RTLD_DEFAULT, "__tsan_on_initialize"));
		78	on_finalize =
		79	reinterpret_cast<int (*)(int)>(dlsym(RTLD_DEFAULT, "__tsan_on_finalize"));
73	}	80	}
74		81
75	static bool TryProtectRange(uptr beg, uptr end) {	82	static bool TryProtectRange(uptr beg, uptr end) {
...	@@ -98,32 +105,28 @@ void CheckAndProtect() {	...	@@ -98,32 +105,28 @@ void CheckAndProtect() {
98	continue;	105	continue;
99	if (segment.start >= VdsoBeg()) // vdso	106	if (segment.start >= VdsoBeg()) // vdso
100	break;	107	break;
101	Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n",	108	Printf("FATAL: ThreadSanitizer: unexpected memory mapping 0x%zx-0x%zx\n",
102	segment.start, segment.end);	109	segment.start, segment.end);
103	Die();	110	Die();
104	}	111	}
105		112
106	#if defined(__aarch64__) && defined(__APPLE__) && !HAS_48_BIT_ADDRESS_SPACE	113	# if SANITIZER_IOS && !SANITIZER_IOSSIM
107	ProtectRange(HeapMemEnd(), ShadowBeg());	114	ProtectRange(HeapMemEnd(), ShadowBeg());
108	ProtectRange(ShadowEnd(), MetaShadowBeg());	115	ProtectRange(ShadowEnd(), MetaShadowBeg());
109	ProtectRange(MetaShadowEnd(), TraceMemBeg());	116	ProtectRange(MetaShadowEnd(), HiAppMemBeg());
110	#else	117	# else
111	ProtectRange(LoAppMemEnd(), ShadowBeg());	118	ProtectRange(LoAppMemEnd(), ShadowBeg());
112	ProtectRange(ShadowEnd(), MetaShadowBeg());	119	ProtectRange(ShadowEnd(), MetaShadowBeg());
113	#ifdef TSAN_MID_APP_RANGE	120	if (MidAppMemBeg()) {
114	ProtectRange(MetaShadowEnd(), MidAppMemBeg());	121	ProtectRange(MetaShadowEnd(), MidAppMemBeg());
115	ProtectRange(MidAppMemEnd(), TraceMemBeg());	122	ProtectRange(MidAppMemEnd(), HeapMemBeg());
116	#else	123	} else {
117	ProtectRange(MetaShadowEnd(), TraceMemBeg());	124	ProtectRange(MetaShadowEnd(), HeapMemBeg());
118	#endif	125	}
119	// Memory for traces is mapped lazily in MapThreadTrace.
120	// Protect the whole range for now, so that user does not map something here.
121	ProtectRange(TraceMemBeg(), TraceMemEnd());
122	ProtectRange(TraceMemEnd(), HeapMemBeg());
123	ProtectRange(HeapEnd(), HiAppMemBeg());	126	ProtectRange(HeapEnd(), HiAppMemBeg());
124	#endif	127	# endif
125		128
126	#if defined(__s390x__)	129	# if defined(__s390x__)
127	// Protect the rest of the address space.	130	// Protect the rest of the address space.
128	const uptr user_addr_max_l4 = 0x0020000000000000ull;	131	const uptr user_addr_max_l4 = 0x0020000000000000ull;
129	const uptr user_addr_max_l5 = 0xfffffffffffff000ull;	132	const uptr user_addr_max_l5 = 0xfffffffffffff000ull;

lib/tsan/tsan_platform_windows.cpp created+33

...	@@ -0,0 +1,33 @@
1	//===-- tsan_platform_windows.cpp -----------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	// Windows-specific code.
12	//===----------------------------------------------------------------------===//
13
14	#include "sanitizer_common/sanitizer_platform.h"
15	#if SANITIZER_WINDOWS
16
17	#include "tsan_platform.h"
18
19	#include <stdlib.h>
20
21	namespace __tsan {
22
23	void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {}
24
25	void InitializePlatformEarly() {
26	}
27
28	void InitializePlatform() {
29	}
30
31	} // namespace __tsan
32
33	#endif // SANITIZER_WINDOWS

lib/tsan/tsan_report.cpp+46-62

 @@ -19,22 +19,6 @@
 namespace __tsan {
-ReportStack::ReportStack() : frames(nullptr), suppressable(false) {}
-ReportStack *ReportStack::New() {
-  void *mem = internal_alloc(MBlockReportStack, sizeof(ReportStack));
-  return new(mem) ReportStack();
-}
-ReportLocation::ReportLocation(ReportLocationType type)
-    : type(type), global(), heap_chunk_start(0), heap_chunk_size(0), tid(0),
-      fd(0), suppressable(false), stack(nullptr) {}
-ReportLocation *ReportLocation::New(ReportLocationType type) {
-  void *mem = internal_alloc(MBlockReportStack, sizeof(ReportLocation));
-  return new(mem) ReportLocation(type);
-}
 class Decorator: public __sanitizer::SanitizerCommonDecorator {
  public:
   Decorator() : SanitizerCommonDecorator() { }
 @@ -68,7 +52,7 @@ ReportDesc::~ReportDesc() {
 #if !SANITIZER_GO
 const int kThreadBufSize = 32;
-const char *thread_name(char *buf, int tid) {
+const char *thread_name(char *buf, Tid tid) {
   if (tid == kMainTid)
     return "main thread";
   internal_snprintf(buf, kThreadBufSize, "thread T%d", tid);
 @@ -114,12 +98,6 @@ static const char *ReportTypeString(ReportType typ, uptr tag) {
   UNREACHABLE("missing case");
 }
-#if SANITIZER_MAC
-static const char *const kInterposedFunctionPrefix = "wrap_";
-#else
-static const char *const kInterposedFunctionPrefix = "__interceptor_";
-#endif
 void PrintStack(const ReportStack *ent) {
   if (ent == 0 || ent->frames == 0) {
     Printf("    [failed to restore the stack]\n\n");
 @@ -131,7 +109,7 @@ void PrintStack(const ReportStack *ent) {
     RenderFrame(&res, common_flags()->stack_trace_format, i,
                 frame->info.address, &frame->info,
                 common_flags()->symbolize_vs_style,
-                common_flags()->strip_path_prefix, kInterposedFunctionPrefix);
+                common_flags()->strip_path_prefix);
     Printf("%s\n", res.data());
   }
   Printf("\n");
 @@ -142,7 +120,7 @@ static void PrintMutexSet(Vector<ReportMopMutex> const& mset) {
     if (i == 0)
       Printf(" (mutexes:");
     const ReportMopMutex m = mset[i];
-    Printf(" %s M%llu", m.write ? "write" : "read", m.id);
+    Printf(" %s M%u", m.write ? "write" : "read", m.id);
     Printf(i == mset.Size() - 1 ? ")" : ",");
   }
 }
 @@ -189,23 +167,25 @@ static void PrintLocation(const ReportLocation *loc) {
   if (loc->type == ReportLocationGlobal) {
     const DataInfo &global = loc->global;
     if (global.size != 0)
-      Printf("  Location is global '%s' of size %zu at %p (%s+%p)\n\n",
+      Printf("  Location is global '%s' of size %zu at %p (%s+0x%zx)\n\n",
-             global.name, global.size, global.start,
+             global.name, global.size, reinterpret_cast<void *>(global.start),
              StripModuleName(global.module), global.module_offset);
     else
-      Printf("  Location is global '%s' at %p (%s+%p)\n\n", global.name,
+      Printf("  Location is global '%s' at %p (%s+0x%zx)\n\n", global.name,
-             global.start, StripModuleName(global.module),
+             reinterpret_cast<void *>(global.start),
-             global.module_offset);
+             StripModuleName(global.module), global.module_offset);
   } else if (loc->type == ReportLocationHeap) {
     char thrbuf[kThreadBufSize];
     const char *object_type = GetObjectTypeFromTag(loc->external_tag);
     if (!object_type) {
       Printf("  Location is heap block of size %zu at %p allocated by %s:\n",
-             loc->heap_chunk_size, loc->heap_chunk_start,
+             loc->heap_chunk_size,
+             reinterpret_cast<void *>(loc->heap_chunk_start),
              thread_name(thrbuf, loc->tid));
     } else {
       Printf("  Location is %s of size %zu at %p allocated by %s:\n",
-             object_type, loc->heap_chunk_size, loc->heap_chunk_start,
+             object_type, loc->heap_chunk_size,
+             reinterpret_cast<void *>(loc->heap_chunk_start),
              thread_name(thrbuf, loc->tid));
     }
     print_stack = true;
 @@ -214,8 +194,9 @@ static void PrintLocation(const ReportLocation *loc) {
   } else if (loc->type == ReportLocationTLS) {
     Printf("  Location is TLS of %s.\n\n", thread_name(thrbuf, loc->tid));
   } else if (loc->type == ReportLocationFD) {
-    Printf("  Location is file descriptor %d created by %s at:\n",
+    Printf("  Location is file descriptor %d %s by %s at:\n", loc->fd,
-        loc->fd, thread_name(thrbuf, loc->tid));
+           loc->fd_closed ? "destroyed" : "created",
+           thread_name(thrbuf, loc->tid));
     print_stack = true;
   }
   Printf("%s", d.Default());
 @@ -225,27 +206,23 @@ static void PrintLocation(const ReportLocation *loc) {
 static void PrintMutexShort(const ReportMutex *rm, const char *after) {
   Decorator d;
-  Printf("%sM%zd%s%s", d.Mutex(), rm->id, d.Default(), after);
+  Printf("%sM%d%s%s", d.Mutex(), rm->id, d.Default(), after);
 }
 static void PrintMutexShortWithAddress(const ReportMutex *rm,
                                        const char *after) {
   Decorator d;
-  Printf("%sM%zd (%p)%s%s", d.Mutex(), rm->id, rm->addr, d.Default(), after);
+  Printf("%sM%d (%p)%s%s", d.Mutex(), rm->id,
+         reinterpret_cast<void *>(rm->addr), d.Default(), after);
 }
 static void PrintMutex(const ReportMutex *rm) {
   Decorator d;
-  if (rm->destroyed) {
+  Printf("%s", d.Mutex());
-    Printf("%s", d.Mutex());
+  Printf("  Mutex M%u (%p) created at:\n", rm->id,
-    Printf("  Mutex M%llu is already destroyed.\n\n", rm->id);
+         reinterpret_cast<void *>(rm->addr));
-    Printf("%s", d.Default());
+  Printf("%s", d.Default());
-  } else {
+  PrintStack(rm->stack);
-    Printf("%s", d.Mutex());
-    Printf("  Mutex M%llu (%p) created at:\n", rm->id, rm->addr);
-    Printf("%s", d.Default());
-    PrintStack(rm->stack);
-  }
 }
 static void PrintThread(const ReportThread *rt) {
 @@ -259,12 +236,13 @@ static void PrintThread(const ReportThread *rt) {
   char thrbuf[kThreadBufSize];
   const char *thread_status = rt->running ? "running" : "finished";
   if (rt->thread_type == ThreadType::Worker) {
-    Printf(" (tid=%zu, %s) is a GCD worker thread\n", rt->os_id, thread_status);
+    Printf(" (tid=%llu, %s) is a GCD worker thread\n", rt->os_id,
+           thread_status);
     Printf("\n");
     Printf("%s", d.Default());
     return;
   }
-  Printf(" (tid=%zu, %s) created by %s", rt->os_id, thread_status,
+  Printf(" (tid=%llu, %s) created by %s", rt->os_id, thread_status,
          thread_name(thrbuf, rt->parent_tid));
   if (rt->stack)
     Printf(" at:");
 @@ -300,6 +278,7 @@ static bool FrameIsInternal(const SymbolizedStack *frame) {
   const char *module = frame->info.module;
   if (file != 0 &&
       (internal_strstr(file, "tsan_interceptors_posix.cpp") ||
+       internal_strstr(file, "tsan_interceptors_memintrinsics.cpp") ||
        internal_strstr(file, "sanitizer_common_interceptors.inc") ||
        internal_strstr(file, "tsan_interface_")))
     return true;
 @@ -323,6 +302,9 @@ void PrintReport(const ReportDesc *rep) {
          (int)internal_getpid());
   Printf("%s", d.Default());
+  if (rep->typ == ReportTypeErrnoInSignal)
+    Printf("  Signal %u handler invoked at:\n", rep->signum);
   if (rep->typ == ReportTypeDeadlock) {
     char thrbuf[kThreadBufSize];
     Printf("  Cycle in lock order graph: ");
 @@ -394,7 +376,7 @@ void PrintReport(const ReportDesc *rep) {
 #else  // #if !SANITIZER_GO
-const u32 kMainGoroutineId = 1;
+const Tid kMainGoroutineId = 1;
 void PrintStack(const ReportStack *ent) {
   if (ent == 0 || ent->frames == 0) {
 @@ -405,16 +387,17 @@ void PrintStack(const ReportStack *ent) {
   for (int i = 0; frame; frame = frame->next, i++) {
     const AddressInfo &info = frame->info;
     Printf("  %s()\n      %s:%d +0x%zx\n", info.function,
-        StripPathPrefix(info.file, common_flags()->strip_path_prefix),
+           StripPathPrefix(info.file, common_flags()->strip_path_prefix),
-        info.line, (void *)info.module_offset);
+           info.line, info.module_offset);
   }
 }
 static void PrintMop(const ReportMop *mop, bool first) {
   Printf("\n");
   Printf("%s at %p by ",
-      (first ? (mop->write ? "Write" : "Read")
+         (first ? (mop->write ? "Write" : "Read")
-             : (mop->write ? "Previous write" : "Previous read")), mop->addr);
+                : (mop->write ? "Previous write" : "Previous read")),
+         reinterpret_cast<void *>(mop->addr));
   if (mop->tid == kMainGoroutineId)
     Printf("main goroutine:\n");
   else
 @@ -426,8 +409,8 @@ static void PrintLocation(const ReportLocation *loc) {
   switch (loc->type) {
   case ReportLocationHeap: {
     Printf("\n");
-    Printf("Heap block of size %zu at %p allocated by ",
+    Printf("Heap block of size %zu at %p allocated by ", loc->heap_chunk_size,
-        loc->heap_chunk_size, loc->heap_chunk_start);
+           reinterpret_cast<void *>(loc->heap_chunk_start));
     if (loc->tid == kMainGoroutineId)
       Printf("main goroutine:\n");
     else
 @@ -438,8 +421,9 @@ static void PrintLocation(const ReportLocation *loc) {
   case ReportLocationGlobal: {
     Printf("\n");
     Printf("Global var %s of size %zu at %p declared at %s:%zu\n",
-        loc->global.name, loc->global.size, loc->global.start,
+           loc->global.name, loc->global.size,
-        loc->global.file, loc->global.line);
+           reinterpret_cast<void *>(loc->global.start), loc->global.file,
+           loc->global.line);
     break;
   }
   default:
 @@ -469,13 +453,13 @@ void PrintReport(const ReportDesc *rep) {
   } else if (rep->typ == ReportTypeDeadlock) {
     Printf("WARNING: DEADLOCK\n");
     for (uptr i = 0; i < rep->mutexes.Size(); i++) {
-      Printf("Goroutine %d lock mutex %d while holding mutex %d:\n",
+      Printf("Goroutine %d lock mutex %u while holding mutex %u:\n", 999,
-          999, rep->mutexes[i]->id,
+             rep->mutexes[i]->id,
-          rep->mutexes[(i+1) % rep->mutexes.Size()]->id);
+             rep->mutexes[(i + 1) % rep->mutexes.Size()]->id);
       PrintStack(rep->stacks[2*i]);
       Printf("\n");
-      Printf("Mutex %d was previously locked here:\n",
+      Printf("Mutex %u was previously locked here:\n",
-          rep->mutexes[(i+1) % rep->mutexes.Size()]->id);
+             rep->mutexes[(i + 1) % rep->mutexes.Size()]->id);
       PrintStack(rep->stacks[2*i + 1]);
       Printf("\n");
     }

lib/tsan/tsan_report.h+18-25

...	@@ -38,16 +38,12 @@ enum ReportType {	...	@@ -38,16 +38,12 @@ enum ReportType {
38	};	38	};
39		39
40	struct ReportStack {	40	struct ReportStack {
41	SymbolizedStack *frames;	41	SymbolizedStack *frames = nullptr;
42	bool suppressable;	42	bool suppressable = false;
43	static ReportStack *New();
44
45	private:
46	ReportStack();
47	};	43	};
48		44
49	struct ReportMopMutex {	45	struct ReportMopMutex {
50	u64 id;	46	int id;
51	bool write;	47	bool write;
52	};	48	};
53		49
...	@@ -73,35 +69,31 @@ enum ReportLocationType {	...	@@ -73,35 +69,31 @@ enum ReportLocationType {
73	};	69	};
74		70
75	struct ReportLocation {	71	struct ReportLocation {
76	ReportLocationType type;	72	ReportLocationType type = ReportLocationGlobal;
77	DataInfo global;	73	DataInfo global = {};
78	uptr heap_chunk_start;	74	uptr heap_chunk_start = 0;
79	uptr heap_chunk_size;	75	uptr heap_chunk_size = 0;
80	uptr external_tag;	76	uptr external_tag = 0;
81	int tid;	77	Tid tid = kInvalidTid;
82	int fd;	78	int fd = 0;
83	bool suppressable;	79	bool fd_closed = false;
84	ReportStack *stack;	80	bool suppressable = false;
85		81	ReportStack *stack = nullptr;
86	static ReportLocation *New(ReportLocationType type);
87	private:
88	explicit ReportLocation(ReportLocationType type);
89	};	82	};
90		83
91	struct ReportThread {	84	struct ReportThread {
92	int id;	85	Tid id;
93	tid_t os_id;	86	tid_t os_id;
94	bool running;	87	bool running;
95	ThreadType thread_type;	88	ThreadType thread_type;
96	char *name;	89	char *name;
97	u32 parent_tid;	90	Tid parent_tid;
98	ReportStack *stack;	91	ReportStack *stack;
99	};	92	};
100		93
101	struct ReportMutex {	94	struct ReportMutex {
102	u64 id;	95	int id;
103	uptr addr;	96	uptr addr;
104	bool destroyed;
105	ReportStack *stack;	97	ReportStack *stack;
106	};	98	};
107		99
...	@@ -114,9 +106,10 @@ class ReportDesc {	...	@@ -114,9 +106,10 @@ class ReportDesc {
114	Vector<ReportLocation*> locs;	106	Vector<ReportLocation*> locs;
115	Vector<ReportMutex*> mutexes;	107	Vector<ReportMutex*> mutexes;
116	Vector<ReportThread*> threads;	108	Vector<ReportThread*> threads;
117	Vector<int> unique_tids;	109	Vector<Tid> unique_tids;
118	ReportStack *sleep;	110	ReportStack *sleep;
119	int count;	111	int count;
		112	int signum = 0;
120		113
121	ReportDesc();	114	ReportDesc();
122	~ReportDesc();	115	~ReportDesc();

lib/tsan/tsan_rtl.cpp+657-693

...	@@ -16,6 +16,7 @@	...	@@ -16,6 +16,7 @@
16	#include "sanitizer_common/sanitizer_atomic.h"	16	#include "sanitizer_common/sanitizer_atomic.h"
17	#include "sanitizer_common/sanitizer_common.h"	17	#include "sanitizer_common/sanitizer_common.h"
18	#include "sanitizer_common/sanitizer_file.h"	18	#include "sanitizer_common/sanitizer_file.h"
		19	#include "sanitizer_common/sanitizer_interface_internal.h"
19	#include "sanitizer_common/sanitizer_libc.h"	20	#include "sanitizer_common/sanitizer_libc.h"
20	#include "sanitizer_common/sanitizer_placement_new.h"	21	#include "sanitizer_common/sanitizer_placement_new.h"
21	#include "sanitizer_common/sanitizer_stackdepot.h"	22	#include "sanitizer_common/sanitizer_stackdepot.h"
...	@@ -28,29 +29,28 @@	...	@@ -28,29 +29,28 @@
28	#include "tsan_symbolize.h"	29	#include "tsan_symbolize.h"
29	#include "ubsan/ubsan_init.h"	30	#include "ubsan/ubsan_init.h"
30		31
31	#ifdef __SSE3__
32	// <emmintrin.h> transitively includes <stdlib.h>,
33	// and it's prohibited to include std headers into tsan runtime.
34	// So we do this dirty trick.
35	#define _MM_MALLOC_H_INCLUDED
36	#define __MM_MALLOC_H
37	#include <emmintrin.h>
38	typedef __m128i m128;
39	#endif
40
41	volatile int __tsan_resumed = 0;	32	volatile int __tsan_resumed = 0;
42		33
43	extern "C" void __tsan_resume() {	34	extern "C" void __tsan_resume() {
44	__tsan_resumed = 1;	35	__tsan_resumed = 1;
45	}	36	}
46		37
		38	SANITIZER_WEAK_DEFAULT_IMPL
		39	void __tsan_test_only_on_fork() {}
		40
47	namespace __tsan {	41	namespace __tsan {
48		42
49	#if !SANITIZER_GO && !SANITIZER_MAC	43	#if !SANITIZER_GO
		44	void (*on_initialize)(void);
		45	int (*on_finalize)(int);
		46	#endif
		47
		48	#if !SANITIZER_GO && !SANITIZER_APPLE
50	__attribute__((tls_model("initial-exec")))	49	__attribute__((tls_model("initial-exec")))
51	THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64);	50	THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(
		51	SANITIZER_CACHE_LINE_SIZE);
52	#endif	52	#endif
53	static char ctx_placeholder[sizeof(Context)] ALIGNED(64);	53	static char ctx_placeholder[sizeof(Context)] ALIGNED(SANITIZER_CACHE_LINE_SIZE);
54	Context *ctx;	54	Context *ctx;
55		55
56	// Can be overriden by a front-end.	56	// Can be overriden by a front-end.
...	@@ -58,113 +58,404 @@ Context *ctx;	...	@@ -58,113 +58,404 @@ Context *ctx;
58	bool OnFinalize(bool failed);	58	bool OnFinalize(bool failed);
59	void OnInitialize();	59	void OnInitialize();
60	#else	60	#else
61	#include <dlfcn.h>
62	SANITIZER_WEAK_CXX_DEFAULT_IMPL	61	SANITIZER_WEAK_CXX_DEFAULT_IMPL
63	bool OnFinalize(bool failed) {	62	bool OnFinalize(bool failed) {
64	#if !SANITIZER_GO	63	# if !SANITIZER_GO
65	if (auto *ptr = dlsym(RTLD_DEFAULT, "__tsan_on_finalize"))	64	if (on_finalize)
66	return reinterpret_cast<decltype(&__tsan_on_finalize)>(ptr)(failed);	65	return on_finalize(failed);
67	#endif	66	# endif
68	return failed;	67	return failed;
69	}	68	}
		69
70	SANITIZER_WEAK_CXX_DEFAULT_IMPL	70	SANITIZER_WEAK_CXX_DEFAULT_IMPL
71	void OnInitialize() {	71	void OnInitialize() {
72	#if !SANITIZER_GO	72	# if !SANITIZER_GO
73	if (auto *ptr = dlsym(RTLD_DEFAULT, "__tsan_on_initialize")) {	73	if (on_initialize)
74	return reinterpret_cast<decltype(&__tsan_on_initialize)>(ptr)();	74	on_initialize();
75	}	75	# endif
		76	}
76	#endif	77	#endif
		78
		79	static TracePart* TracePartAlloc(ThreadState* thr) {
		80	TracePart* part = nullptr;
		81	{
		82	Lock lock(&ctx->slot_mtx);
		83	uptr max_parts = Trace::kMinParts + flags()->history_size;
		84	Trace* trace = &thr->tctx->trace;
		85	if (trace->parts_allocated == max_parts \|\|
		86	ctx->trace_part_finished_excess) {
		87	part = ctx->trace_part_recycle.PopFront();
		88	DPrintf("#%d: TracePartAlloc: part=%p\n", thr->tid, part);
		89	if (part && part->trace) {
		90	Trace* trace1 = part->trace;
		91	Lock trace_lock(&trace1->mtx);
		92	part->trace = nullptr;
		93	TracePart* part1 = trace1->parts.PopFront();
		94	CHECK_EQ(part, part1);
		95	if (trace1->parts_allocated > trace1->parts.Size()) {
		96	ctx->trace_part_finished_excess +=
		97	trace1->parts_allocated - trace1->parts.Size();
		98	trace1->parts_allocated = trace1->parts.Size();
		99	}
		100	}
		101	}
		102	if (trace->parts_allocated < max_parts) {
		103	trace->parts_allocated++;
		104	if (ctx->trace_part_finished_excess)
		105	ctx->trace_part_finished_excess--;
		106	}
		107	if (!part)
		108	ctx->trace_part_total_allocated++;
		109	else if (ctx->trace_part_recycle_finished)
		110	ctx->trace_part_recycle_finished--;
		111	}
		112	if (!part)
		113	part = new (MmapOrDie(sizeof(*part), "TracePart")) TracePart();
		114	return part;
		115	}
		116
		117	static void TracePartFree(TracePart* part) SANITIZER_REQUIRES(ctx->slot_mtx) {
		118	DCHECK(part->trace);
		119	part->trace = nullptr;
		120	ctx->trace_part_recycle.PushFront(part);
		121	}
		122
		123	void TraceResetForTesting() {
		124	Lock lock(&ctx->slot_mtx);
		125	while (auto* part = ctx->trace_part_recycle.PopFront()) {
		126	if (auto trace = part->trace)
		127	CHECK_EQ(trace->parts.PopFront(), part);
		128	UnmapOrDie(part, sizeof(*part));
		129	}
		130	ctx->trace_part_total_allocated = 0;
		131	ctx->trace_part_recycle_finished = 0;
		132	ctx->trace_part_finished_excess = 0;
77	}	133	}
		134
		135	static void DoResetImpl(uptr epoch) {
		136	ThreadRegistryLock lock0(&ctx->thread_registry);
		137	Lock lock1(&ctx->slot_mtx);
		138	CHECK_EQ(ctx->global_epoch, epoch);
		139	ctx->global_epoch++;
		140	CHECK(!ctx->resetting);
		141	ctx->resetting = true;
		142	for (u32 i = ctx->thread_registry.NumThreadsLocked(); i--;) {
		143	ThreadContext* tctx = (ThreadContext*)ctx->thread_registry.GetThreadLocked(
		144	static_cast<Tid>(i));
		145	// Potentially we could purge all ThreadStatusDead threads from the
		146	// registry. Since we reset all shadow, they can't race with anything
		147	// anymore. However, their tid's can still be stored in some aux places
		148	// (e.g. tid of thread that created something).
		149	auto trace = &tctx->trace;
		150	Lock lock(&trace->mtx);
		151	bool attached = tctx->thr && tctx->thr->slot;
		152	auto parts = &trace->parts;
		153	bool local = false;
		154	while (!parts->Empty()) {
		155	auto part = parts->Front();
		156	local = local \|\| part == trace->local_head;
		157	if (local)
		158	CHECK(!ctx->trace_part_recycle.Queued(part));
		159	else
		160	ctx->trace_part_recycle.Remove(part);
		161	if (attached && parts->Size() == 1) {
		162	// The thread is running and this is the last/current part.
		163	// Set the trace position to the end of the current part
		164	// to force the thread to call SwitchTracePart and re-attach
		165	// to a new slot and allocate a new trace part.
		166	// Note: the thread is concurrently modifying the position as well,
		167	// so this is only best-effort. The thread can only modify position
		168	// within this part, because switching parts is protected by
		169	// slot/trace mutexes that we hold here.
		170	atomic_store_relaxed(
		171	&tctx->thr->trace_pos,
		172	reinterpret_cast<uptr>(&part->events[TracePart::kSize]));
		173	break;
		174	}
		175	parts->Remove(part);
		176	TracePartFree(part);
		177	}
		178	CHECK_LE(parts->Size(), 1);
		179	trace->local_head = parts->Front();
		180	if (tctx->thr && !tctx->thr->slot) {
		181	atomic_store_relaxed(&tctx->thr->trace_pos, 0);
		182	tctx->thr->trace_prev_pc = 0;
		183	}
		184	if (trace->parts_allocated > trace->parts.Size()) {
		185	ctx->trace_part_finished_excess +=
		186	trace->parts_allocated - trace->parts.Size();
		187	trace->parts_allocated = trace->parts.Size();
		188	}
		189	}
		190	while (ctx->slot_queue.PopFront()) {
		191	}
		192	for (auto& slot : ctx->slots) {
		193	slot.SetEpoch(kEpochZero);
		194	slot.journal.Reset();
		195	slot.thr = nullptr;
		196	ctx->slot_queue.PushBack(&slot);
		197	}
		198
		199	DPrintf("Resetting shadow...\n");
		200	auto shadow_begin = ShadowBeg();
		201	auto shadow_end = ShadowEnd();
		202	#if SANITIZER_GO
		203	CHECK_NE(0, ctx->mapped_shadow_begin);
		204	shadow_begin = ctx->mapped_shadow_begin;
		205	shadow_end = ctx->mapped_shadow_end;
		206	VPrintf(2, "shadow_begin-shadow_end: (0x%zx-0x%zx)\n",
		207	shadow_begin, shadow_end);
78	#endif	208	#endif
79		209
80	static ALIGNED(64) char thread_registry_placeholder[sizeof(ThreadRegistry)];	210	#if SANITIZER_WINDOWS
81		211	auto resetFailed =
82	static ThreadContextBase *CreateThreadContext(u32 tid) {	212	!ZeroMmapFixedRegion(shadow_begin, shadow_end - shadow_begin);
83	// Map thread trace when context is created.	213	#else
84	char name[50];	214	auto resetFailed =
85	internal_snprintf(name, sizeof(name), "trace %u", tid);	215	!MmapFixedSuperNoReserve(shadow_begin, shadow_end-shadow_begin, "shadow");
86	MapThreadTrace(GetThreadTrace(tid), TraceSize() * sizeof(Event), name);	216	# if !SANITIZER_GO
87	const uptr hdr = GetThreadTraceHeader(tid);	217	DontDumpShadow(shadow_begin, shadow_end - shadow_begin);
88	internal_snprintf(name, sizeof(name), "trace header %u", tid);	218	# endif
89	MapThreadTrace(hdr, sizeof(Trace), name);	219	#endif
90	new((void*)hdr) Trace();	220	if (resetFailed) {
91	// We are going to use only a small part of the trace with the default	221	Printf("failed to reset shadow memory\n");
92	// value of history_size. However, the constructor writes to the whole trace.	222	Die();
93	// Release the unused part.	223	}
94	uptr hdr_end = hdr + sizeof(Trace);	224	DPrintf("Resetting meta shadow...\n");
95	hdr_end -= sizeof(TraceHeader) * (kTraceParts - TraceParts());	225	ctx->metamap.ResetClocks();
96	hdr_end = RoundUp(hdr_end, GetPageSizeCached());	226	StoreShadow(&ctx->last_spurious_race, Shadow::kEmpty);
97	if (hdr_end < hdr + sizeof(Trace)) {	227	ctx->resetting = false;
98	ReleaseMemoryPagesToOS(hdr_end, hdr + sizeof(Trace));	228	}
99	uptr unused = hdr + sizeof(Trace) - hdr_end;	229
100	if (hdr_end != (uptr)MmapFixedNoAccess(hdr_end, unused)) {	230	// Clang does not understand locking all slots in the loop:
101	Report("ThreadSanitizer: failed to mprotect(%p, %p)\n",	231	// error: expecting mutex 'slot.mtx' to be held at start of each loop
102	hdr_end, unused);	232	void DoReset(ThreadState* thr, uptr epoch) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
103	CHECK("unable to mprotect" && 0);	233	for (auto& slot : ctx->slots) {
		234	slot.mtx.Lock();
		235	if (UNLIKELY(epoch == 0))
		236	epoch = ctx->global_epoch;
		237	if (UNLIKELY(epoch != ctx->global_epoch)) {
		238	// Epoch can't change once we've locked the first slot.
		239	CHECK_EQ(slot.sid, 0);
		240	slot.mtx.Unlock();
		241	return;
		242	}
		243	}
		244	DPrintf("#%d: DoReset epoch=%lu\n", thr ? thr->tid : -1, epoch);
		245	DoResetImpl(epoch);
		246	for (auto& slot : ctx->slots) slot.mtx.Unlock();
		247	}
		248
		249	void FlushShadowMemory() { DoReset(nullptr, 0); }
		250
		251	static TidSlot* FindSlotAndLock(ThreadState* thr)
		252	SANITIZER_ACQUIRE(thr->slot->mtx) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
		253	CHECK(!thr->slot);
		254	TidSlot* slot = nullptr;
		255	for (;;) {
		256	uptr epoch;
		257	{
		258	Lock lock(&ctx->slot_mtx);
		259	epoch = ctx->global_epoch;
		260	if (slot) {
		261	// This is an exhausted slot from the previous iteration.
		262	if (ctx->slot_queue.Queued(slot))
		263	ctx->slot_queue.Remove(slot);
		264	thr->slot_locked = false;
		265	slot->mtx.Unlock();
		266	}
		267	for (;;) {
		268	slot = ctx->slot_queue.PopFront();
		269	if (!slot)
		270	break;
		271	if (slot->epoch() != kEpochLast) {
		272	ctx->slot_queue.PushBack(slot);
		273	break;
		274	}
		275	}
		276	}
		277	if (!slot) {
		278	DoReset(thr, epoch);
		279	continue;
104	}	280	}
		281	slot->mtx.Lock();
		282	CHECK(!thr->slot_locked);
		283	thr->slot_locked = true;
		284	if (slot->thr) {
		285	DPrintf("#%d: preempting sid=%d tid=%d\n", thr->tid, (u32)slot->sid,
		286	slot->thr->tid);
		287	slot->SetEpoch(slot->thr->fast_state.epoch());
		288	slot->thr = nullptr;
		289	}
		290	if (slot->epoch() != kEpochLast)
		291	return slot;
105	}	292	}
106	void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadContext));
107	return new(mem) ThreadContext(tid);
108	}	293	}
109		294
		295	void SlotAttachAndLock(ThreadState* thr) {
		296	TidSlot* slot = FindSlotAndLock(thr);
		297	DPrintf("#%d: SlotAttach: slot=%u\n", thr->tid, static_cast<int>(slot->sid));
		298	CHECK(!slot->thr);
		299	CHECK(!thr->slot);
		300	slot->thr = thr;
		301	thr->slot = slot;
		302	Epoch epoch = EpochInc(slot->epoch());
		303	CHECK(!EpochOverflow(epoch));
		304	slot->SetEpoch(epoch);
		305	thr->fast_state.SetSid(slot->sid);
		306	thr->fast_state.SetEpoch(epoch);
		307	if (thr->slot_epoch != ctx->global_epoch) {
		308	thr->slot_epoch = ctx->global_epoch;
		309	thr->clock.Reset();
110	#if !SANITIZER_GO	310	#if !SANITIZER_GO
111	static const u32 kThreadQuarantineSize = 16;	311	thr->last_sleep_stack_id = kInvalidStackID;
112	#else	312	thr->last_sleep_clock.Reset();
113	static const u32 kThreadQuarantineSize = 64;	313	#endif
		314	}
		315	thr->clock.Set(slot->sid, epoch);
		316	slot->journal.PushBack({thr->tid, epoch});
		317	}
		318
		319	static void SlotDetachImpl(ThreadState* thr, bool exiting) {
		320	TidSlot* slot = thr->slot;
		321	thr->slot = nullptr;
		322	if (thr != slot->thr) {
		323	slot = nullptr; // we don't own the slot anymore
		324	if (thr->slot_epoch != ctx->global_epoch) {
		325	TracePart* part = nullptr;
		326	auto* trace = &thr->tctx->trace;
		327	{
		328	Lock l(&trace->mtx);
		329	auto* parts = &trace->parts;
		330	// The trace can be completely empty in an unlikely event
		331	// the thread is preempted right after it acquired the slot
		332	// in ThreadStart and did not trace any events yet.
		333	CHECK_LE(parts->Size(), 1);
		334	part = parts->PopFront();
		335	thr->tctx->trace.local_head = nullptr;
		336	atomic_store_relaxed(&thr->trace_pos, 0);
		337	thr->trace_prev_pc = 0;
		338	}
		339	if (part) {
		340	Lock l(&ctx->slot_mtx);
		341	TracePartFree(part);
		342	}
		343	}
		344	return;
		345	}
		346	CHECK(exiting \|\| thr->fast_state.epoch() == kEpochLast);
		347	slot->SetEpoch(thr->fast_state.epoch());
		348	slot->thr = nullptr;
		349	}
		350
		351	void SlotDetach(ThreadState* thr) {
		352	Lock lock(&thr->slot->mtx);
		353	SlotDetachImpl(thr, true);
		354	}
		355
		356	void SlotLock(ThreadState* thr) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
		357	DCHECK(!thr->slot_locked);
		358	#if SANITIZER_DEBUG
		359	// Check these mutexes are not locked.
		360	// We can call DoReset from SlotAttachAndLock, which will lock
		361	// these mutexes, but it happens only every once in a while.
		362	{ ThreadRegistryLock lock(&ctx->thread_registry); }
		363	{ Lock lock(&ctx->slot_mtx); }
114	#endif	364	#endif
		365	TidSlot* slot = thr->slot;
		366	slot->mtx.Lock();
		367	thr->slot_locked = true;
		368	if (LIKELY(thr == slot->thr && thr->fast_state.epoch() != kEpochLast))
		369	return;
		370	SlotDetachImpl(thr, false);
		371	thr->slot_locked = false;
		372	slot->mtx.Unlock();
		373	SlotAttachAndLock(thr);
		374	}
		375
		376	void SlotUnlock(ThreadState* thr) {
		377	DCHECK(thr->slot_locked);
		378	thr->slot_locked = false;
		379	thr->slot->mtx.Unlock();
		380	}
115		381
116	Context::Context()	382	Context::Context()
117	: initialized(),	383	: initialized(),
118	report_mtx(MutexTypeReport),	384	report_mtx(MutexTypeReport),
119	nreported(),	385	nreported(),
120	nmissed_expected(),	386	thread_registry([](Tid tid) -> ThreadContextBase* {
121	thread_registry(new (thread_registry_placeholder) ThreadRegistry(	387	return new (Alloc(sizeof(ThreadContext))) ThreadContext(tid);
122	CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse)),	388	}),
123	racy_mtx(MutexTypeRacy),	389	racy_mtx(MutexTypeRacy),
124	racy_stacks(),	390	racy_stacks(),
125	racy_addresses(),
126	fired_suppressions_mtx(MutexTypeFired),	391	fired_suppressions_mtx(MutexTypeFired),
127	clock_alloc(LINKER_INITIALIZED, "clock allocator") {	392	slot_mtx(MutexTypeSlots),
		393	resetting() {
128	fired_suppressions.reserve(8);	394	fired_suppressions.reserve(8);
		395	for (uptr i = 0; i < ARRAY_SIZE(slots); i++) {
		396	TidSlot* slot = &slots[i];
		397	slot->sid = static_cast<Sid>(i);
		398	slot_queue.PushBack(slot);
		399	}
		400	global_epoch = 1;
129	}	401	}
130		402
		403	TidSlot::TidSlot() : mtx(MutexTypeSlot) {}
		404
131	// The objects are allocated in TLS, so one may rely on zero-initialization.	405	// The objects are allocated in TLS, so one may rely on zero-initialization.
132	ThreadState::ThreadState(Context *ctx, u32 tid, int unique_id, u64 epoch,	406	ThreadState::ThreadState(Tid tid)
133	unsigned reuse_count, uptr stk_addr, uptr stk_size,	407	// Do not touch these, rely on zero initialization,
134	uptr tls_addr, uptr tls_size)	408	// they may be accessed before the ctor.
135	: fast_state(tid, epoch)	409	// ignore_reads_and_writes()
136	// Do not touch these, rely on zero initialization,	410	// ignore_interceptors()
137	// they may be accessed before the ctor.	411	: tid(tid) {
138	// , ignore_reads_and_writes()	412	CHECK_EQ(reinterpret_cast<uptr>(this) % SANITIZER_CACHE_LINE_SIZE, 0);
139	// , ignore_interceptors()
140	,
141	clock(tid, reuse_count)
142	#if !SANITIZER_GO
143	,
144	jmp_bufs()
145	#endif
146	,
147	tid(tid),
148	unique_id(unique_id),
149	stk_addr(stk_addr),
150	stk_size(stk_size),
151	tls_addr(tls_addr),
152	tls_size(tls_size)
153	#if !SANITIZER_GO	413	#if !SANITIZER_GO
154	,	414	// C/C++ uses fixed size shadow stack.
155	last_sleep_clock(tid)	415	const int kInitStackSize = kShadowStackSize;
		416	shadow_stack = static_cast<uptr*>(
		417	MmapNoReserveOrDie(kInitStackSize * sizeof(uptr), "shadow stack"));
		418	SetShadowRegionHugePageMode(reinterpret_cast<uptr>(shadow_stack),
		419	kInitStackSize * sizeof(uptr));
		420	#else
		421	// Go uses malloc-allocated shadow stack with dynamic size.
		422	const int kInitStackSize = 8;
		423	shadow_stack = static_cast<uptr>(Alloc(kInitStackSize sizeof(uptr)));
156	#endif	424	#endif
157	{	425	shadow_stack_pos = shadow_stack;
		426	shadow_stack_end = shadow_stack + kInitStackSize;
158	}	427	}
159		428
160	#if !SANITIZER_GO	429	#if !SANITIZER_GO
161	static void MemoryProfiler(Context *ctx, fd_t fd, int i) {	430	void MemoryProfiler(u64 uptime) {
162	uptr n_threads;	431	if (ctx->memprof_fd == kInvalidFd)
163	uptr n_running_threads;	432	return;
164	ctx->thread_registry->GetNumberOfThreads(&n_threads, &n_running_threads);
165	InternalMmapVector<char> buf(4096);	433	InternalMmapVector<char> buf(4096);
166	WriteMemoryProfile(buf.data(), buf.size(), n_threads, n_running_threads);	434	WriteMemoryProfile(buf.data(), buf.size(), uptime);
167	WriteToFile(fd, buf.data(), internal_strlen(buf.data()));	435	WriteToFile(ctx->memprof_fd, buf.data(), internal_strlen(buf.data()));
		436	}
		437
		438	static bool InitializeMemoryProfiler() {
		439	ctx->memprof_fd = kInvalidFd;
		440	const char *fname = flags()->profile_memory;
		441	if (!fname \|\| !fname[0])
		442	return false;
		443	if (internal_strcmp(fname, "stdout") == 0) {
		444	ctx->memprof_fd = 1;
		445	} else if (internal_strcmp(fname, "stderr") == 0) {
		446	ctx->memprof_fd = 2;
		447	} else {
		448	InternalScopedString filename;
		449	filename.append("%s.%d", fname, (int)internal_getpid());
		450	ctx->memprof_fd = OpenFile(filename.data(), WrOnly);
		451	if (ctx->memprof_fd == kInvalidFd) {
		452	Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
		453	filename.data());
		454	return false;
		455	}
		456	}
		457	MemoryProfiler(0);
		458	return true;
168	}	459	}
169		460
170	static void BackgroundThread(void arg) {	461	static void BackgroundThread(void arg) {
...	@@ -172,64 +463,43 @@ static void BackgroundThread(void arg) {	...	@@ -172,64 +463,43 @@ static void BackgroundThread(void arg) {
172	// We don't use ScopedIgnoreInterceptors, because we want ignores to be	463	// We don't use ScopedIgnoreInterceptors, because we want ignores to be
173	// enabled even when the thread function exits (e.g. during pthread thread	464	// enabled even when the thread function exits (e.g. during pthread thread
174	// shutdown code).	465	// shutdown code).
175	cur_thread_init();	466	cur_thread_init()->ignore_interceptors++;
176	cur_thread()->ignore_interceptors++;
177	const u64 kMs2Ns = 1000 * 1000;	467	const u64 kMs2Ns = 1000 * 1000;
		468	const u64 start = NanoTime();
178		469
179	fd_t mprof_fd = kInvalidFd;	470	u64 last_flush = start;
180	if (flags()->profile_memory && flags()->profile_memory[0]) {
181	if (internal_strcmp(flags()->profile_memory, "stdout") == 0) {
182	mprof_fd = 1;
183	} else if (internal_strcmp(flags()->profile_memory, "stderr") == 0) {
184	mprof_fd = 2;
185	} else {
186	InternalScopedString filename;
187	filename.append("%s.%d", flags()->profile_memory, (int)internal_getpid());
188	fd_t fd = OpenFile(filename.data(), WrOnly);
189	if (fd == kInvalidFd) {
190	Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
191	filename.data());
192	} else {
193	mprof_fd = fd;
194	}
195	}
196	}
197
198	u64 last_flush = NanoTime();
199	uptr last_rss = 0;	471	uptr last_rss = 0;
200	for (int i = 0;	472	while (!atomic_load_relaxed(&ctx->stop_background_thread)) {
201	atomic_load(&ctx->stop_background_thread, memory_order_relaxed) == 0;
202	i++) {
203	SleepForMillis(100);	473	SleepForMillis(100);
204	u64 now = NanoTime();	474	u64 now = NanoTime();
205		475
206	// Flush memory if requested.	476	// Flush memory if requested.
207	if (flags()->flush_memory_ms > 0) {	477	if (flags()->flush_memory_ms > 0) {
208	if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) {	478	if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) {
209	VPrintf(1, "ThreadSanitizer: periodic memory flush\n");	479	VReport(1, "ThreadSanitizer: periodic memory flush\n");
210	FlushShadowMemory();	480	FlushShadowMemory();
211	last_flush = NanoTime();	481	now = last_flush = NanoTime();
212	}	482	}
213	}	483	}
214	// GetRSS can be expensive on huge programs, so don't do it every 100ms.
215	if (flags()->memory_limit_mb > 0) {	484	if (flags()->memory_limit_mb > 0) {
216	uptr rss = GetRSS();	485	uptr rss = GetRSS();
217	uptr limit = uptr(flags()->memory_limit_mb) << 20;	486	uptr limit = uptr(flags()->memory_limit_mb) << 20;
218	VPrintf(1, "ThreadSanitizer: memory flush check"	487	VReport(1,
219	" RSS=%llu LAST=%llu LIMIT=%llu\n",	488	"ThreadSanitizer: memory flush check"
		489	" RSS=%llu LAST=%llu LIMIT=%llu\n",
220	(u64)rss >> 20, (u64)last_rss >> 20, (u64)limit >> 20);	490	(u64)rss >> 20, (u64)last_rss >> 20, (u64)limit >> 20);
221	if (2 * rss > limit + last_rss) {	491	if (2 * rss > limit + last_rss) {
222	VPrintf(1, "ThreadSanitizer: flushing memory due to RSS\n");	492	VReport(1, "ThreadSanitizer: flushing memory due to RSS\n");
223	FlushShadowMemory();	493	FlushShadowMemory();
224	rss = GetRSS();	494	rss = GetRSS();
225	VPrintf(1, "ThreadSanitizer: memory flushed RSS=%llu\n", (u64)rss>>20);	495	now = NanoTime();
		496	VReport(1, "ThreadSanitizer: memory flushed RSS=%llu\n",
		497	(u64)rss >> 20);
226	}	498	}
227	last_rss = rss;	499	last_rss = rss;
228	}	500	}
229		501
230	// Write memory profile if requested.	502	MemoryProfiler(now - start);
231	if (mprof_fd != kInvalidFd)
232	MemoryProfiler(ctx, mprof_fd, i);
233		503
234	// Flush symbolizer cache if requested.	504	// Flush symbolizer cache if requested.
235	if (flags()->flush_symbolizer_ms > 0) {	505	if (flags()->flush_symbolizer_ms > 0) {
...	@@ -260,31 +530,96 @@ static void StopBackgroundThread() {	...	@@ -260,31 +530,96 @@ static void StopBackgroundThread() {
260	#endif	530	#endif
261		531
262	void DontNeedShadowFor(uptr addr, uptr size) {	532	void DontNeedShadowFor(uptr addr, uptr size) {
263	ReleaseMemoryPagesToOS(MemToShadow(addr), MemToShadow(addr + size));	533	ReleaseMemoryPagesToOS(reinterpret_cast<uptr>(MemToShadow(addr)),
		534	reinterpret_cast<uptr>(MemToShadow(addr + size)));
264	}	535	}
265		536
266	#if !SANITIZER_GO	537	#if !SANITIZER_GO
		538	// We call UnmapShadow before the actual munmap, at that point we don't yet
		539	// know if the provided address/size are sane. We can't call UnmapShadow
		540	// after the actual munmap becuase at that point the memory range can
		541	// already be reused for something else, so we can't rely on the munmap
		542	// return value to understand is the values are sane.
		543	// While calling munmap with insane values (non-canonical address, negative
		544	// size, etc) is an error, the kernel won't crash. We must also try to not
		545	// crash as the failure mode is very confusing (paging fault inside of the
		546	// runtime on some derived shadow address).
		547	static bool IsValidMmapRange(uptr addr, uptr size) {
		548	if (size == 0)
		549	return true;
		550	if (static_cast<sptr>(size) < 0)
		551	return false;
		552	if (!IsAppMem(addr) \|\| !IsAppMem(addr + size - 1))
		553	return false;
		554	// Check that if the start of the region belongs to one of app ranges,
		555	// end of the region belongs to the same region.
		556	const uptr ranges[][2] = {
		557	{LoAppMemBeg(), LoAppMemEnd()},
		558	{MidAppMemBeg(), MidAppMemEnd()},
		559	{HiAppMemBeg(), HiAppMemEnd()},
		560	};
		561	for (auto range : ranges) {
		562	if (addr >= range[0] && addr < range[1])
		563	return addr + size <= range[1];
		564	}
		565	return false;
		566	}
		567
267	void UnmapShadow(ThreadState *thr, uptr addr, uptr size) {	568	void UnmapShadow(ThreadState *thr, uptr addr, uptr size) {
268	if (size == 0) return;	569	if (size == 0 \|\| !IsValidMmapRange(addr, size))
		570	return;
269	DontNeedShadowFor(addr, size);	571	DontNeedShadowFor(addr, size);
270	ScopedGlobalProcessor sgp;	572	ScopedGlobalProcessor sgp;
271	ctx->metamap.ResetRange(thr->proc(), addr, size);	573	SlotLocker locker(thr, true);
		574	ctx->metamap.ResetRange(thr->proc(), addr, size, true);
272	}	575	}
273	#endif	576	#endif
274		577
275	void MapShadow(uptr addr, uptr size) {	578	void MapShadow(uptr addr, uptr size) {
		579	// Ensure thead registry lock held, so as to synchronize
		580	// with DoReset, which also access the mapped_shadow_* ctxt fields.
		581	ThreadRegistryLock lock0(&ctx->thread_registry);
		582	static bool data_mapped = false;
		583
		584	#if !SANITIZER_GO
276	// Global data is not 64K aligned, but there are no adjacent mappings,	585	// Global data is not 64K aligned, but there are no adjacent mappings,
277	// so we can get away with unaligned mapping.	586	// so we can get away with unaligned mapping.
278	// CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment	587	// CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment
279	const uptr kPageSize = GetPageSizeCached();	588	const uptr kPageSize = GetPageSizeCached();
280	uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize);	589	uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize);
281	uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize);	590	uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize);
282	if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin,	591	if (!MmapFixedNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow"))
283	"shadow"))
284	Die();	592	Die();
		593	#else
		594	uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), (64 << 10));
		595	uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), (64 << 10));
		596	VPrintf(2, "MapShadow for (0x%zx-0x%zx), begin/end: (0x%zx-0x%zx)\n",
		597	addr, addr + size, shadow_begin, shadow_end);
		598
		599	if (!data_mapped) {
		600	// First call maps data+bss.
		601	if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow"))
		602	Die();
		603	} else {
		604	VPrintf(2, "ctx->mapped_shadow_{begin,end} = (0x%zx-0x%zx)\n",
		605	ctx->mapped_shadow_begin, ctx->mapped_shadow_end);
		606	// Second and subsequent calls map heap.
		607	if (shadow_end <= ctx->mapped_shadow_end)
		608	return;
		609	if (!ctx->mapped_shadow_begin \|\| ctx->mapped_shadow_begin > shadow_begin)
		610	ctx->mapped_shadow_begin = shadow_begin;
		611	if (shadow_begin < ctx->mapped_shadow_end)
		612	shadow_begin = ctx->mapped_shadow_end;
		613	VPrintf(2, "MapShadow begin/end = (0x%zx-0x%zx)\n",
		614	shadow_begin, shadow_end);
		615	if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin,
		616	"shadow"))
		617	Die();
		618	ctx->mapped_shadow_end = shadow_end;
		619	}
		620	#endif
285		621
286	// Meta shadow is 2:1, so tread carefully.	622	// Meta shadow is 2:1, so tread carefully.
287	static bool data_mapped = false;
288	static uptr mapped_meta_end = 0;	623	static uptr mapped_meta_end = 0;
289	uptr meta_begin = (uptr)MemToMeta(addr);	624	uptr meta_begin = (uptr)MemToMeta(addr);
290	uptr meta_end = (uptr)MemToMeta(addr + size);	625	uptr meta_end = (uptr)MemToMeta(addr + size);
...	@@ -297,12 +632,11 @@ void MapShadow(uptr addr, uptr size) {	...	@@ -297,12 +632,11 @@ void MapShadow(uptr addr, uptr size) {
297	"meta shadow"))	632	"meta shadow"))
298	Die();	633	Die();
299	} else {	634	} else {
300	// Mapping continous heap.	635	// Mapping continuous heap.
301	// Windows wants 64K alignment.	636	// Windows wants 64K alignment.
302	meta_begin = RoundDownTo(meta_begin, 64 << 10);	637	meta_begin = RoundDownTo(meta_begin, 64 << 10);
303	meta_end = RoundUpTo(meta_end, 64 << 10);	638	meta_end = RoundUpTo(meta_end, 64 << 10);
304	if (meta_end <= mapped_meta_end)	639	CHECK_GT(meta_end, mapped_meta_end);
305	return;
306	if (meta_begin < mapped_meta_end)	640	if (meta_begin < mapped_meta_end)
307	meta_begin = mapped_meta_end;	641	meta_begin = mapped_meta_end;
308	if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin,	642	if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin,
...	@@ -310,56 +644,8 @@ void MapShadow(uptr addr, uptr size) {	...	@@ -310,56 +644,8 @@ void MapShadow(uptr addr, uptr size) {
310	Die();	644	Die();
311	mapped_meta_end = meta_end;	645	mapped_meta_end = meta_end;
312	}	646	}
313	VPrintf(2, "mapped meta shadow for (%p-%p) at (%p-%p)\n",	647	VPrintf(2, "mapped meta shadow for (0x%zx-0x%zx) at (0x%zx-0x%zx)\n", addr,
314	addr, addr+size, meta_begin, meta_end);	648	addr + size, meta_begin, meta_end);
315	}
316
317	void MapThreadTrace(uptr addr, uptr size, const char *name) {
318	DPrintf("#0: Mapping trace at %p-%p(0x%zx)\n", addr, addr + size, size);
319	CHECK_GE(addr, TraceMemBeg());
320	CHECK_LE(addr + size, TraceMemEnd());
321	CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment
322	if (!MmapFixedSuperNoReserve(addr, size, name)) {
323	Printf("FATAL: ThreadSanitizer can not mmap thread trace (%p/%p)\n",
324	addr, size);
325	Die();
326	}
327	}
328
329	static void CheckShadowMapping() {
330	uptr beg, end;
331	for (int i = 0; GetUserRegion(i, &beg, &end); i++) {
332	// Skip cases for empty regions (heap definition for architectures that
333	// do not use 64-bit allocator).
334	if (beg == end)
335	continue;
336	VPrintf(3, "checking shadow region %p-%p\n", beg, end);
337	uptr prev = 0;
338	for (uptr p0 = beg; p0 <= end; p0 += (end - beg) / 4) {
339	for (int x = -(int)kShadowCell; x <= (int)kShadowCell; x += kShadowCell) {
340	const uptr p = RoundDown(p0 + x, kShadowCell);
341	if (p < beg \|\| p >= end)
342	continue;
343	const uptr s = MemToShadow(p);
344	const uptr m = (uptr)MemToMeta(p);
345	VPrintf(3, " checking pointer %p: shadow=%p meta=%p\n", p, s, m);
346	CHECK(IsAppMem(p));
347	CHECK(IsShadowMem(s));
348	CHECK_EQ(p, ShadowToMem(s));
349	CHECK(IsMetaMem(m));
350	if (prev) {
351	// Ensure that shadow and meta mappings are linear within a single
352	// user range. Lots of code that processes memory ranges assumes it.
353	const uptr prev_s = MemToShadow(prev);
354	const uptr prev_m = (uptr)MemToMeta(prev);
355	CHECK_EQ(s - prev_s, (p - prev) * kShadowMultiplier);
356	CHECK_EQ((m - prev_m) / kMetaShadowSize,
357	(p - prev) / kMetaShadowCell);
358	}
359	prev = p;
360	}
361	}
362	}
363	}	649	}
364		650
365	#if !SANITIZER_GO	651	#if !SANITIZER_GO
...	@@ -380,15 +666,19 @@ void CheckUnwind() {	...	@@ -380,15 +666,19 @@ void CheckUnwind() {
380	// since we are going to die soon.	666	// since we are going to die soon.
381	ScopedIgnoreInterceptors ignore;	667	ScopedIgnoreInterceptors ignore;
382	#if !SANITIZER_GO	668	#if !SANITIZER_GO
383	cur_thread()->ignore_sync++;	669	ThreadState* thr = cur_thread();
384	cur_thread()->ignore_reads_and_writes++;	670	thr->nomalloc = false;
		671	thr->ignore_sync++;
		672	thr->ignore_reads_and_writes++;
		673	atomic_store_relaxed(&thr->in_signal_handler, 0);
385	#endif	674	#endif
386	PrintCurrentStackSlow(StackTrace::GetCurrentPc());	675	PrintCurrentStackSlow(StackTrace::GetCurrentPc());
387	}	676	}
388		677
		678	bool is_initialized;
		679
389	void Initialize(ThreadState *thr) {	680	void Initialize(ThreadState *thr) {
390	// Thread safe because done before all threads exist.	681	// Thread safe because done before all threads exist.
391	static bool is_initialized = false;
392	if (is_initialized)	682	if (is_initialized)
393	return;	683	return;
394	is_initialized = true;	684	is_initialized = true;
...	@@ -409,9 +699,6 @@ void Initialize(ThreadState *thr) {	...	@@ -409,9 +699,6 @@ void Initialize(ThreadState *thr) {
409	__tsan::InitializePlatformEarly();	699	__tsan::InitializePlatformEarly();
410		700
411	#if !SANITIZER_GO	701	#if !SANITIZER_GO
412	// Re-exec ourselves if we need to set additional env or command line args.
413	MaybeReexec();
414
415	InitializeAllocator();	702	InitializeAllocator();
416	ReplaceSystemMalloc();	703	ReplaceSystemMalloc();
417	#endif	704	#endif
...	@@ -420,7 +707,6 @@ void Initialize(ThreadState *thr) {	...	@@ -420,7 +707,6 @@ void Initialize(ThreadState *thr) {
420	Processor *proc = ProcCreate();	707	Processor *proc = ProcCreate();
421	ProcWire(proc, thr);	708	ProcWire(proc, thr);
422	InitializeInterceptors();	709	InitializeInterceptors();
423	CheckShadowMapping();
424	InitializePlatform();	710	InitializePlatform();
425	InitializeDynamicAnnotations();	711	InitializeDynamicAnnotations();
426	#if !SANITIZER_GO	712	#if !SANITIZER_GO
...	@@ -436,21 +722,23 @@ void Initialize(ThreadState *thr) {	...	@@ -436,21 +722,23 @@ void Initialize(ThreadState *thr) {
436	Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);	722	Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer, ExitSymbolizer);
437	#endif	723	#endif
438		724
439	VPrintf(1, "*** Running under ThreadSanitizer v2 (pid %d) ***\n",	725	VPrintf(1, "*** Running under ThreadSanitizer v3 (pid %d) ***\n",
440	(int)internal_getpid());	726	(int)internal_getpid());
441		727
442	// Initialize thread 0.	728	// Initialize thread 0.
443	int tid = ThreadCreate(thr, 0, 0, true);	729	Tid tid = ThreadCreate(nullptr, 0, 0, true);
444	CHECK_EQ(tid, 0);	730	CHECK_EQ(tid, kMainTid);
445	ThreadStart(thr, tid, GetTid(), ThreadType::Regular);	731	ThreadStart(thr, tid, GetTid(), ThreadType::Regular);
446	#if TSAN_CONTAINS_UBSAN	732	#if TSAN_CONTAINS_UBSAN
447	__ubsan::InitAsPlugin();	733	__ubsan::InitAsPlugin();
448	#endif	734	#endif
449	ctx->initialized = true;
450		735
451	#if !SANITIZER_GO	736	#if !SANITIZER_GO
452	Symbolizer::LateInitialize();	737	Symbolizer::LateInitialize();
		738	if (InitializeMemoryProfiler() \|\| flags()->force_background_thread)
		739	MaybeSpawnBackgroundThread();
453	#endif	740	#endif
		741	ctx->initialized = true;
454		742
455	if (flags()->stop_on_start) {	743	if (flags()->stop_on_start) {
456	Printf("ThreadSanitizer is suspended at startup (pid %d)."	744	Printf("ThreadSanitizer is suspended at startup (pid %d)."
...	@@ -476,20 +764,21 @@ void MaybeSpawnBackgroundThread() {	...	@@ -476,20 +764,21 @@ void MaybeSpawnBackgroundThread() {
476	#endif	764	#endif
477	}	765	}
478		766
479
480	int Finalize(ThreadState *thr) {	767	int Finalize(ThreadState *thr) {
481	bool failed = false;	768	bool failed = false;
482		769
		770	#if !SANITIZER_GO
483	if (common_flags()->print_module_map == 1)	771	if (common_flags()->print_module_map == 1)
484	DumpProcessMap();	772	DumpProcessMap();
		773	#endif
485		774
486	if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1)	775	if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1)
487	SleepForMillis(flags()->atexit_sleep_ms);	776	internal_usleep(u64(flags()->atexit_sleep_ms) * 1000);
488		777
489	// Wait for pending reports.	778	{
490	ctx->report_mtx.Lock();	779	// Wait for pending reports.
491	{ ScopedErrorReportLock l; }	780	ScopedErrorReportLock lock;
492	ctx->report_mtx.Unlock();	781	}
493		782
494	#if !SANITIZER_GO	783	#if !SANITIZER_GO
495	if (Verbosity()) AllocatorPrintStats();	784	if (Verbosity()) AllocatorPrintStats();
...	@@ -506,18 +795,8 @@ int Finalize(ThreadState *thr) {	...	@@ -506,18 +795,8 @@ int Finalize(ThreadState *thr) {
506	#endif	795	#endif
507	}	796	}
508		797
509	if (ctx->nmissed_expected) {
510	failed = true;
511	Printf("ThreadSanitizer: missed %d expected races\n",
512	ctx->nmissed_expected);
513	}
514
515	if (common_flags()->print_suppressions)	798	if (common_flags()->print_suppressions)
516	PrintMatchedSuppressions();	799	PrintMatchedSuppressions();
517	#if !SANITIZER_GO
518	if (flags()->print_benign)
519	PrintMatchedBenignRaces();
520	#endif
521		800
522	failed = OnFinalize(failed);	801	failed = OnFinalize(failed);
523		802
...	@@ -525,10 +804,16 @@ int Finalize(ThreadState *thr) {	...	@@ -525,10 +804,16 @@ int Finalize(ThreadState *thr) {
525	}	804	}
526		805
527	#if !SANITIZER_GO	806	#if !SANITIZER_GO
528	void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {	807	void ForkBefore(ThreadState* thr, uptr pc) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
529	ctx->thread_registry->Lock();	808	GlobalProcessorLock();
530	ctx->report_mtx.Lock();	809	// Detaching from the slot makes OnUserFree skip writing to the shadow.
		810	// The slot will be locked so any attempts to use it will deadlock anyway.
		811	SlotDetach(thr);
		812	for (auto& slot : ctx->slots) slot.mtx.Lock();
		813	ctx->thread_registry.Lock();
		814	ctx->slot_mtx.Lock();
531	ScopedErrorReportLock::Lock();	815	ScopedErrorReportLock::Lock();
		816	AllocatorLock();
532	// Suppress all reports in the pthread_atfork callbacks.	817	// Suppress all reports in the pthread_atfork callbacks.
533	// Reports will deadlock on the report_mtx.	818	// Reports will deadlock on the report_mtx.
534	// We could ignore sync operations as well,	819	// We could ignore sync operations as well,
...	@@ -537,36 +822,48 @@ void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {	...	@@ -537,36 +822,48 @@ void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
537	thr->suppress_reports++;	822	thr->suppress_reports++;
538	// On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and	823	// On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and
539	// we'll assert in CheckNoLocks() unless we ignore interceptors.	824	// we'll assert in CheckNoLocks() unless we ignore interceptors.
		825	// On OS X libSystem_atfork_prepare/parent/child callbacks are called
		826	// after/before our callbacks and they call free.
540	thr->ignore_interceptors++;	827	thr->ignore_interceptors++;
		828	// Disables memory write in OnUserAlloc/Free.
		829	thr->ignore_reads_and_writes++;
		830
		831	__tsan_test_only_on_fork();
541	}	832	}
542		833
543	void ForkParentAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {	834	static void ForkAfter(ThreadState* thr) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
544	thr->suppress_reports--; // Enabled in ForkBefore.	835	thr->suppress_reports--; // Enabled in ForkBefore.
545	thr->ignore_interceptors--;	836	thr->ignore_interceptors--;
		837	thr->ignore_reads_and_writes--;
		838	AllocatorUnlock();
546	ScopedErrorReportLock::Unlock();	839	ScopedErrorReportLock::Unlock();
547	ctx->report_mtx.Unlock();	840	ctx->slot_mtx.Unlock();
548	ctx->thread_registry->Unlock();	841	ctx->thread_registry.Unlock();
		842	for (auto& slot : ctx->slots) slot.mtx.Unlock();
		843	SlotAttachAndLock(thr);
		844	SlotUnlock(thr);
		845	GlobalProcessorUnlock();
549	}	846	}
550		847
551	void ForkChildAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {	848	void ForkParentAfter(ThreadState* thr, uptr pc) { ForkAfter(thr); }
552	thr->suppress_reports--; // Enabled in ForkBefore.
553	thr->ignore_interceptors--;
554	ScopedErrorReportLock::Unlock();
555	ctx->report_mtx.Unlock();
556	ctx->thread_registry->Unlock();
557		849
558	uptr nthread = 0;	850	void ForkChildAfter(ThreadState* thr, uptr pc, bool start_thread) {
559	ctx->thread_registry->GetNumberOfThreads(0, 0, &nthread /* alive threads */);	851	ForkAfter(thr);
560	VPrintf(1, "ThreadSanitizer: forked new process with pid %d,"	852	u32 nthread = ctx->thread_registry.OnFork(thr->tid);
561	" parent had %d threads\n", (int)internal_getpid(), (int)nthread);	853	VPrintf(1,
		854	"ThreadSanitizer: forked new process with pid %d,"
		855	" parent had %d threads\n",
		856	(int)internal_getpid(), (int)nthread);
562	if (nthread == 1) {	857	if (nthread == 1) {
563	StartBackgroundThread();	858	if (start_thread)
		859	StartBackgroundThread();
564	} else {	860	} else {
565	// We've just forked a multi-threaded process. We cannot reasonably function	861	// We've just forked a multi-threaded process. We cannot reasonably function
566	// after that (some mutexes may be locked before fork). So just enable	862	// after that (some mutexes may be locked before fork). So just enable
567	// ignores for everything in the hope that we will exec soon.	863	// ignores for everything in the hope that we will exec soon.
568	ctx->after_multithreaded_fork = true;	864	ctx->after_multithreaded_fork = true;
569	thr->ignore_interceptors++;	865	thr->ignore_interceptors++;
		866	thr->suppress_reports++;
570	ThreadIgnoreBegin(thr, pc);	867	ThreadIgnoreBegin(thr, pc);
571	ThreadIgnoreSyncBegin(thr, pc);	868	ThreadIgnoreSyncBegin(thr, pc);
572	}	869	}
...	@@ -578,19 +875,20 @@ NOINLINE	...	@@ -578,19 +875,20 @@ NOINLINE
578	void GrowShadowStack(ThreadState *thr) {	875	void GrowShadowStack(ThreadState *thr) {
579	const int sz = thr->shadow_stack_end - thr->shadow_stack;	876	const int sz = thr->shadow_stack_end - thr->shadow_stack;
580	const int newsz = 2 * sz;	877	const int newsz = 2 * sz;
581	uptr newstack = (uptr)internal_alloc(MBlockShadowStack,	878	auto newstack = (uptr )Alloc(newsz * sizeof(uptr));
582	newsz * sizeof(uptr));
583	internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr));	879	internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr));
584	internal_free(thr->shadow_stack);	880	Free(thr->shadow_stack);
585	thr->shadow_stack = newstack;	881	thr->shadow_stack = newstack;
586	thr->shadow_stack_pos = newstack + sz;	882	thr->shadow_stack_pos = newstack + sz;
587	thr->shadow_stack_end = newstack + newsz;	883	thr->shadow_stack_end = newstack + newsz;
588	}	884	}
589	#endif	885	#endif
590		886
591	u32 CurrentStackId(ThreadState *thr, uptr pc) {	887	StackID CurrentStackId(ThreadState *thr, uptr pc) {
		888	#if !SANITIZER_GO
592	if (!thr->is_inited) // May happen during bootstrap.	889	if (!thr->is_inited) // May happen during bootstrap.
593	return 0;	890	return kInvalidStackID;
		891	#endif
594	if (pc != 0) {	892	if (pc != 0) {
595	#if !SANITIZER_GO	893	#if !SANITIZER_GO
596	DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);	894	DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
...	@@ -601,486 +899,149 @@ u32 CurrentStackId(ThreadState *thr, uptr pc) {	...	@@ -601,486 +899,149 @@ u32 CurrentStackId(ThreadState *thr, uptr pc) {
601	thr->shadow_stack_pos[0] = pc;	899	thr->shadow_stack_pos[0] = pc;
602	thr->shadow_stack_pos++;	900	thr->shadow_stack_pos++;
603	}	901	}
604	u32 id = StackDepotPut(	902	StackID id = StackDepotPut(
605	StackTrace(thr->shadow_stack, thr->shadow_stack_pos - thr->shadow_stack));	903	StackTrace(thr->shadow_stack, thr->shadow_stack_pos - thr->shadow_stack));
606	if (pc != 0)	904	if (pc != 0)
607	thr->shadow_stack_pos--;	905	thr->shadow_stack_pos--;
608	return id;	906	return id;
609	}	907	}
610		908
611	void TraceSwitch(ThreadState *thr) {	909	static bool TraceSkipGap(ThreadState* thr) {
612	#if !SANITIZER_GO	910	Trace *trace = &thr->tctx->trace;
613	if (ctx->after_multithreaded_fork)	911	Event pos = reinterpret_cast<Event >(atomic_load_relaxed(&thr->trace_pos));
614	return;	912	DCHECK_EQ(reinterpret_cast<uptr>(pos + 1) & TracePart::kAlignment, 0);
615	#endif	913	auto *part = trace->parts.Back();
616	thr->nomalloc++;	914	DPrintf("#%d: TraceSwitchPart enter trace=%p parts=%p-%p pos=%p\n", thr->tid,
617	Trace *thr_trace = ThreadTrace(thr->tid);	915	trace, trace->parts.Front(), part, pos);
618	Lock l(&thr_trace->mtx);	916	if (!part)
619	unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % TraceParts();	917	return false;
620	TraceHeader *hdr = &thr_trace->headers[trace];	918	// We can get here when we still have space in the current trace part.
621	hdr->epoch0 = thr->fast_state.epoch();	919	// The fast-path check in TraceAcquire has false positives in the middle of
622	ObtainCurrentStack(thr, 0, &hdr->stack0);	920	// the part. Check if we are indeed at the end of the current part or not,
623	hdr->mset0 = thr->mset;	921	// and fill any gaps with NopEvent's.
624	thr->nomalloc--;	922	Event* end = &part->events[TracePart::kSize];
625	}	923	DCHECK_GE(pos, &part->events[0]);
626		924	DCHECK_LE(pos, end);
627	Trace *ThreadTrace(int tid) {	925	if (pos + 1 < end) {
628	return (Trace*)GetThreadTraceHeader(tid);	926	if ((reinterpret_cast<uptr>(pos) & TracePart::kAlignment) ==
629	}	927	TracePart::kAlignment)
630		928	*pos++ = NopEvent;
631	uptr TraceTopPC(ThreadState *thr) {	929	*pos++ = NopEvent;
632	Event events = (Event)GetThreadTrace(thr->tid);	930	DCHECK_LE(pos + 2, end);
633	uptr pc = events[thr->fast_state.GetTracePos()];	931	atomic_store_relaxed(&thr->trace_pos, reinterpret_cast<uptr>(pos));
634	return pc;	932	return true;
635	}
636
637	uptr TraceSize() {
638	return (uptr)(1ull << (kTracePartSizeBits + flags()->history_size + 1));
639	}
640
641	uptr TraceParts() {
642	return TraceSize() / kTracePartSize;
643	}
644
645	#if !SANITIZER_GO
646	extern "C" void __tsan_trace_switch() {
647	TraceSwitch(cur_thread());
648	}
649
650	extern "C" void __tsan_report_race() {
651	ReportRace(cur_thread());
652	}
653	#endif
654
655	ALWAYS_INLINE
656	Shadow LoadShadow(u64 *p) {
657	u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed);
658	return Shadow(raw);
659	}
660
661	ALWAYS_INLINE
662	void StoreShadow(u64 *sp, u64 s) {
663	atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed);
664	}
665
666	ALWAYS_INLINE
667	void StoreIfNotYetStored(u64 sp, u64 s) {
668	StoreShadow(sp, *s);
669	*s = 0;
670	}
671
672	ALWAYS_INLINE
673	void HandleRace(ThreadState thr, u64 shadow_mem,
674	Shadow cur, Shadow old) {
675	thr->racy_state[0] = cur.raw();
676	thr->racy_state[1] = old.raw();
677	thr->racy_shadow_addr = shadow_mem;
678	#if !SANITIZER_GO
679	HACKY_CALL(__tsan_report_race);
680	#else
681	ReportRace(thr);
682	#endif
683	}
684
685	static inline bool HappensBefore(Shadow old, ThreadState *thr) {
686	return thr->clock.get(old.TidWithIgnore()) >= old.epoch();
687	}
688
689	ALWAYS_INLINE
690	void MemoryAccessImpl1(ThreadState *thr, uptr addr,
691	int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
692	u64 *shadow_mem, Shadow cur) {
693
694	// This potentially can live in an MMX/SSE scratch register.
695	// The required intrinsics are:
696	// __m128i _mm_move_epi64(__m128i*);
697	// _mm_storel_epi64(u64*, __m128i);
698	u64 store_word = cur.raw();
699	bool stored = false;
700
701	// scan all the shadow values and dispatch to 4 categories:
702	// same, replace, candidate and race (see comments below).
703	// we consider only 3 cases regarding access sizes:
704	// equal, intersect and not intersect. initially I considered
705	// larger and smaller as well, it allowed to replace some
706	// 'candidates' with 'same' or 'replace', but I think
707	// it's just not worth it (performance- and complexity-wise).
708
709	Shadow old(0);
710
711	// It release mode we manually unroll the loop,
712	// because empirically gcc generates better code this way.
713	// However, we can't afford unrolling in debug mode, because the function
714	// consumes almost 4K of stack. Gtest gives only 4K of stack to death test
715	// threads, which is not enough for the unrolled loop.
716	#if SANITIZER_DEBUG
717	for (int idx = 0; idx < 4; idx++) {
718	#include "tsan_update_shadow_word_inl.h"
719	}
720	#else
721	int idx = 0;
722	#include "tsan_update_shadow_word_inl.h"
723	idx = 1;
724	if (stored) {
725	#include "tsan_update_shadow_word_inl.h"
726	} else {
727	#include "tsan_update_shadow_word_inl.h"
728	}
729	idx = 2;
730	if (stored) {
731	#include "tsan_update_shadow_word_inl.h"
732	} else {
733	#include "tsan_update_shadow_word_inl.h"
734	}
735	idx = 3;
736	if (stored) {
737	#include "tsan_update_shadow_word_inl.h"
738	} else {
739	#include "tsan_update_shadow_word_inl.h"
740	}
741	#endif
742
743	// we did not find any races and had already stored
744	// the current access info, so we are done
745	if (LIKELY(stored))
746	return;
747	// choose a random candidate slot and replace it
748	StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word);
749	return;
750	RACE:
751	HandleRace(thr, shadow_mem, cur, old);
752	return;
753	}
754
755	void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr,
756	int size, bool kAccessIsWrite, bool kIsAtomic) {
757	while (size) {
758	int size1 = 1;
759	int kAccessSizeLog = kSizeLog1;
760	if (size >= 8 && (addr & ~7) == ((addr + 7) & ~7)) {
761	size1 = 8;
762	kAccessSizeLog = kSizeLog8;
763	} else if (size >= 4 && (addr & ~7) == ((addr + 3) & ~7)) {
764	size1 = 4;
765	kAccessSizeLog = kSizeLog4;
766	} else if (size >= 2 && (addr & ~7) == ((addr + 1) & ~7)) {
767	size1 = 2;
768	kAccessSizeLog = kSizeLog2;
769	}
770	MemoryAccess(thr, pc, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic);
771	addr += size1;
772	size -= size1;
773	}
774	}
775
776	ALWAYS_INLINE
777	bool ContainsSameAccessSlow(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
778	Shadow cur(a);
779	for (uptr i = 0; i < kShadowCnt; i++) {
780	Shadow old(LoadShadow(&s[i]));
781	if (Shadow::Addr0AndSizeAreEqual(cur, old) &&
782	old.TidWithIgnore() == cur.TidWithIgnore() &&
783	old.epoch() > sync_epoch &&
784	old.IsAtomic() == cur.IsAtomic() &&
785	old.IsRead() <= cur.IsRead())
786	return true;
787	}	933	}
		934	// We are indeed at the end.
		935	for (; pos < end; pos++) *pos = NopEvent;
788	return false;	936	return false;
789	}	937	}
790		938
791	#if defined(__SSE3__)	939	NOINLINE
792	#define SHUF(v0, v1, i0, i1, i2, i3) _mm_castps_si128(_mm_shuffle_ps( \	940	void TraceSwitchPart(ThreadState* thr) {
793	_mm_castsi128_ps(v0), _mm_castsi128_ps(v1), \	941	if (TraceSkipGap(thr))
794	(i0)1 + (i1)4 + (i2)16 + (i3)64))
795	ALWAYS_INLINE
796	bool ContainsSameAccessFast(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
797	// This is an optimized version of ContainsSameAccessSlow.
798	// load current access into access[0:63]
799	const m128 access = _mm_cvtsi64_si128(a);
800	// duplicate high part of access in addr0:
801	// addr0[0:31] = access[32:63]
802	// addr0[32:63] = access[32:63]
803	// addr0[64:95] = access[32:63]
804	// addr0[96:127] = access[32:63]
805	const m128 addr0 = SHUF(access, access, 1, 1, 1, 1);
806	// load 4 shadow slots
807	const m128 shadow0 = _mm_load_si128((__m128i*)s);
808	const m128 shadow1 = _mm_load_si128((__m128i*)s + 1);
809	// load high parts of 4 shadow slots into addr_vect:
810	// addr_vect[0:31] = shadow0[32:63]
811	// addr_vect[32:63] = shadow0[96:127]
812	// addr_vect[64:95] = shadow1[32:63]
813	// addr_vect[96:127] = shadow1[96:127]
814	m128 addr_vect = SHUF(shadow0, shadow1, 1, 3, 1, 3);
815	if (!is_write) {
816	// set IsRead bit in addr_vect
817	const m128 rw_mask1 = _mm_cvtsi64_si128(1<<15);
818	const m128 rw_mask = SHUF(rw_mask1, rw_mask1, 0, 0, 0, 0);
819	addr_vect = _mm_or_si128(addr_vect, rw_mask);
820	}
821	// addr0 == addr_vect?
822	const m128 addr_res = _mm_cmpeq_epi32(addr0, addr_vect);
823	// epoch1[0:63] = sync_epoch
824	const m128 epoch1 = _mm_cvtsi64_si128(sync_epoch);
825	// epoch[0:31] = sync_epoch[0:31]
826	// epoch[32:63] = sync_epoch[0:31]
827	// epoch[64:95] = sync_epoch[0:31]
828	// epoch[96:127] = sync_epoch[0:31]
829	const m128 epoch = SHUF(epoch1, epoch1, 0, 0, 0, 0);
830	// load low parts of shadow cell epochs into epoch_vect:
831	// epoch_vect[0:31] = shadow0[0:31]
832	// epoch_vect[32:63] = shadow0[64:95]
833	// epoch_vect[64:95] = shadow1[0:31]
834	// epoch_vect[96:127] = shadow1[64:95]
835	const m128 epoch_vect = SHUF(shadow0, shadow1, 0, 2, 0, 2);
836	// epoch_vect >= sync_epoch?
837	const m128 epoch_res = _mm_cmpgt_epi32(epoch_vect, epoch);
838	// addr_res & epoch_res
839	const m128 res = _mm_and_si128(addr_res, epoch_res);
840	// mask[0] = res[7]
841	// mask[1] = res[15]
842	// ...
843	// mask[15] = res[127]
844	const int mask = _mm_movemask_epi8(res);
845	return mask != 0;
846	}
847	#endif
848
849	ALWAYS_INLINE
850	bool ContainsSameAccess(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
851	#if defined(__SSE3__)
852	bool res = ContainsSameAccessFast(s, a, sync_epoch, is_write);
853	// NOTE: this check can fail if the shadow is concurrently mutated
854	// by other threads. But it still can be useful if you modify
855	// ContainsSameAccessFast and want to ensure that it's not completely broken.
856	// DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write));
857	return res;
858	#else
859	return ContainsSameAccessSlow(s, a, sync_epoch, is_write);
860	#endif
861	}
862
863	ALWAYS_INLINE USED
864	void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
865	int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic) {
866	u64 shadow_mem = (u64)MemToShadow(addr);
867	DPrintf2("#%d: MemoryAccess: @%p %p size=%d"
868	" is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n",
869	(int)thr->fast_state.tid(), (void)pc, (void)addr,
870	(int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem,
871	(uptr)shadow_mem[0], (uptr)shadow_mem[1],
872	(uptr)shadow_mem[2], (uptr)shadow_mem[3]);
873	#if SANITIZER_DEBUG
874	if (!IsAppMem(addr)) {
875	Printf("Access to non app mem %zx\n", addr);
876	DCHECK(IsAppMem(addr));
877	}
878	if (!IsShadowMem((uptr)shadow_mem)) {
879	Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
880	DCHECK(IsShadowMem((uptr)shadow_mem));
881	}
882	#endif
883
884	if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) {
885	// Access to .rodata section, no races here.
886	// Measurements show that it can be 10-20% of all memory accesses.
887	return;
888	}
889
890	FastState fast_state = thr->fast_state;
891	if (UNLIKELY(fast_state.GetIgnoreBit())) {
892	return;
893	}
894
895	Shadow cur(fast_state);
896	cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog);
897	cur.SetWrite(kAccessIsWrite);
898	cur.SetAtomic(kIsAtomic);
899
900	if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
901	thr->fast_synch_epoch, kAccessIsWrite))) {
902	return;
903	}
904
905	if (kCollectHistory) {
906	fast_state.IncrementEpoch();
907	thr->fast_state = fast_state;
908	TraceAddEvent(thr, fast_state, EventTypeMop, pc);
909	cur.IncrementEpoch();
910	}
911
912	MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
913	shadow_mem, cur);
914	}
915
916	// Called by MemoryAccessRange in tsan_rtl_thread.cpp
917	ALWAYS_INLINE USED
918	void MemoryAccessImpl(ThreadState *thr, uptr addr,
919	int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
920	u64 *shadow_mem, Shadow cur) {
921	if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
922	thr->fast_synch_epoch, kAccessIsWrite))) {
923	return;
924	}
925
926	MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
927	shadow_mem, cur);
928	}
929
930	static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
931	u64 val) {
932	(void)thr;
933	(void)pc;
934	if (size == 0)
935	return;	942	return;
936	// FIXME: fix me.	943	#if !SANITIZER_GO
937	uptr offset = addr % kShadowCell;	944	if (ctx->after_multithreaded_fork) {
938	if (offset) {	945	// We just need to survive till exec.
939	offset = kShadowCell - offset;	946	TracePart* part = thr->tctx->trace.parts.Back();
940	if (size <= offset)	947	if (part) {
		948	atomic_store_relaxed(&thr->trace_pos,
		949	reinterpret_cast<uptr>(&part->events[0]));
941	return;	950	return;
942	addr += offset;
943	size -= offset;
944	}
945	DCHECK_EQ(addr % 8, 0);
946	// If a user passes some insane arguments (memset(0)),
947	// let it just crash as usual.
948	if (!IsAppMem(addr) \|\| !IsAppMem(addr + size - 1))
949	return;
950	// Don't want to touch lots of shadow memory.
951	// If a program maps 10MB stack, there is no need reset the whole range.
952	size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1);
953	// UnmapOrDie/MmapFixedNoReserve does not work on Windows.
954	if (SANITIZER_WINDOWS \|\| size < common_flags()->clear_shadow_mmap_threshold) {
955	u64 p = (u64)MemToShadow(addr);
956	CHECK(IsShadowMem((uptr)p));
957	CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1)));
958	// FIXME: may overwrite a part outside the region
959	for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) {
960	p[i++] = val;
961	for (uptr j = 1; j < kShadowCnt; j++)
962	p[i++] = 0;
963	}
964	} else {
965	// The region is big, reset only beginning and end.
966	const uptr kPageSize = GetPageSizeCached();
967	u64 begin = (u64)MemToShadow(addr);
968	u64 end = begin + size / kShadowCell kShadowCnt;
969	u64 *p = begin;
970	// Set at least first kPageSize/2 to page boundary.
971	while ((p < begin + kPageSize / kShadowSize / 2) \|\| ((uptr)p % kPageSize)) {
972	*p++ = val;
973	for (uptr j = 1; j < kShadowCnt; j++)
974	*p++ = 0;
975	}
976	// Reset middle part.
977	u64 *p1 = p;
978	p = RoundDown(end, kPageSize);
979	if (!MmapFixedSuperNoReserve((uptr)p1, (uptr)p - (uptr)p1))
980	Die();
981	// Set the ending.
982	while (p < end) {
983	*p++ = val;
984	for (uptr j = 1; j < kShadowCnt; j++)
985	*p++ = 0;
986	}	951	}
987	}	952	}
		953	#endif
		954	TraceSwitchPartImpl(thr);
988	}	955	}
989		956
990	void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) {	957	void TraceSwitchPartImpl(ThreadState* thr) {
991	MemoryRangeSet(thr, pc, addr, size, 0);	958	SlotLocker locker(thr, true);
992	}	959	Trace* trace = &thr->tctx->trace;
993		960	TracePart* part = TracePartAlloc(thr);
994	void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) {	961	part->trace = trace;
995	// Processing more than 1k (4k of shadow) is expensive,	962	thr->trace_prev_pc = 0;
996	// can cause excessive memory consumption (user does not necessary touch	963	TracePart* recycle = nullptr;
997	// the whole range) and most likely unnecessary.	964	// Keep roughly half of parts local to the thread
998	if (size > 1024)	965	// (not queued into the recycle queue).
999	size = 1024;	966	uptr local_parts = (Trace::kMinParts + flags()->history_size + 1) / 2;
1000	CHECK_EQ(thr->is_freeing, false);	967	{
1001	thr->is_freeing = true;	968	Lock lock(&trace->mtx);
1002	MemoryAccessRange(thr, pc, addr, size, true);	969	if (trace->parts.Empty())
1003	thr->is_freeing = false;	970	trace->local_head = part;
1004	if (kCollectHistory) {	971	if (trace->parts.Size() >= local_parts) {
1005	thr->fast_state.IncrementEpoch();	972	recycle = trace->local_head;
1006	TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);	973	trace->local_head = trace->parts.Next(recycle);
		974	}
		975	trace->parts.PushBack(part);
		976	atomic_store_relaxed(&thr->trace_pos,
		977	reinterpret_cast<uptr>(&part->events[0]));
1007	}	978	}
1008	Shadow s(thr->fast_state);	979	// Make this part self-sufficient by restoring the current stack
1009	s.ClearIgnoreBit();	980	// and mutex set in the beginning of the trace.
1010	s.MarkAsFreed();	981	TraceTime(thr);
1011	s.SetWrite(true);	982	{
1012	s.SetAddr0AndSizeLog(0, 3);	983	// Pathologically large stacks may not fit into the part.
1013	MemoryRangeSet(thr, pc, addr, size, s.raw());	984	// In these cases we log only fixed number of top frames.
1014	}	985	const uptr kMaxFrames = 1000;
1015		986	// Check that kMaxFrames won't consume the whole part.
1016	void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) {	987	static_assert(kMaxFrames < TracePart::kSize / 2, "kMaxFrames is too big");
1017	if (kCollectHistory) {	988	uptr* pos = Max(&thr->shadow_stack[0], thr->shadow_stack_pos - kMaxFrames);
1018	thr->fast_state.IncrementEpoch();	989	for (; pos < thr->shadow_stack_pos; pos++) {
1019	TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);	990	if (TryTraceFunc(thr, *pos))
		991	continue;
		992	CHECK(TraceSkipGap(thr));
		993	CHECK(TryTraceFunc(thr, *pos));
		994	}
1020	}	995	}
1021	Shadow s(thr->fast_state);	996	for (uptr i = 0; i < thr->mset.Size(); i++) {
1022	s.ClearIgnoreBit();	997	MutexSet::Desc d = thr->mset.Get(i);
1023	s.SetWrite(true);	998	for (uptr i = 0; i < d.count; i++)
1024	s.SetAddr0AndSizeLog(0, 3);	999	TraceMutexLock(thr, d.write ? EventType::kLock : EventType::kRLock, 0,
1025	MemoryRangeSet(thr, pc, addr, size, s.raw());	1000	d.addr, d.stack_id);
1026	}
1027
1028	void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
1029	uptr size) {
1030	if (thr->ignore_reads_and_writes == 0)
1031	MemoryRangeImitateWrite(thr, pc, addr, size);
1032	else
1033	MemoryResetRange(thr, pc, addr, size);
1034	}
1035
1036	ALWAYS_INLINE USED
1037	void FuncEntry(ThreadState *thr, uptr pc) {
1038	DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc);
1039	if (kCollectHistory) {
1040	thr->fast_state.IncrementEpoch();
1041	TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc);
1042	}	1001	}
1043		1002	// Callers of TraceSwitchPart expect that TraceAcquire will always succeed
1044	// Shadow stack maintenance can be replaced with	1003	// after the call. It's possible that TryTraceFunc/TraceMutexLock above
1045	// stack unwinding during trace switch (which presumably must be faster).	1004	// filled the trace part exactly up to the TracePart::kAlignment gap
1046	DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);	1005	// and the next TraceAcquire won't succeed. Skip the gap to avoid that.
1047	#if !SANITIZER_GO	1006	EventFunc *ev;
1048	DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);	1007	if (!TraceAcquire(thr, &ev)) {
1049	#else	1008	CHECK(TraceSkipGap(thr));
1050	if (thr->shadow_stack_pos == thr->shadow_stack_end)	1009	CHECK(TraceAcquire(thr, &ev));
1051	GrowShadowStack(thr);
1052	#endif
1053	thr->shadow_stack_pos[0] = pc;
1054	thr->shadow_stack_pos++;
1055	}
1056
1057	ALWAYS_INLINE USED
1058	void FuncExit(ThreadState *thr) {
1059	DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid());
1060	if (kCollectHistory) {
1061	thr->fast_state.IncrementEpoch();
1062	TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0);
1063	}	1010	}
1064		1011	{
1065	DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);	1012	Lock lock(&ctx->slot_mtx);
1066	#if !SANITIZER_GO	1013	// There is a small chance that the slot may be not queued at this point.
1067	DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);	1014	// This can happen if the slot has kEpochLast epoch and another thread
1068	#endif	1015	// in FindSlotAndLock discovered that it's exhausted and removed it from
1069	thr->shadow_stack_pos--;	1016	// the slot queue. kEpochLast can happen in 2 cases: (1) if TraceSwitchPart
		1017	// was called with the slot locked and epoch already at kEpochLast,
		1018	// or (2) if we've acquired a new slot in SlotLock in the beginning
		1019	// of the function and the slot was at kEpochLast - 1, so after increment
		1020	// in SlotAttachAndLock it become kEpochLast.
		1021	if (ctx->slot_queue.Queued(thr->slot)) {
		1022	ctx->slot_queue.Remove(thr->slot);
		1023	ctx->slot_queue.PushBack(thr->slot);
		1024	}
		1025	if (recycle)
		1026	ctx->trace_part_recycle.PushBack(recycle);
		1027	}
		1028	DPrintf("#%d: TraceSwitchPart exit parts=%p-%p pos=0x%zx\n", thr->tid,
		1029	trace->parts.Front(), trace->parts.Back(),
		1030	atomic_load_relaxed(&thr->trace_pos));
1070	}	1031	}
1071		1032
1072	void ThreadIgnoreBegin(ThreadState *thr, uptr pc, bool save_stack) {	1033	void ThreadIgnoreBegin(ThreadState* thr, uptr pc) {
1073	DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid);	1034	DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid);
1074	thr->ignore_reads_and_writes++;	1035	thr->ignore_reads_and_writes++;
1075	CHECK_GT(thr->ignore_reads_and_writes, 0);	1036	CHECK_GT(thr->ignore_reads_and_writes, 0);
1076	thr->fast_state.SetIgnoreBit();	1037	thr->fast_state.SetIgnoreBit();
1077	#if !SANITIZER_GO	1038	#if !SANITIZER_GO
1078	if (save_stack && !ctx->after_multithreaded_fork)	1039	if (pc && !ctx->after_multithreaded_fork)
1079	thr->mop_ignore_set.Add(CurrentStackId(thr, pc));	1040	thr->mop_ignore_set.Add(CurrentStackId(thr, pc));
1080	#endif	1041	#endif
1081	}	1042	}
1082		1043
1083	void ThreadIgnoreEnd(ThreadState *thr, uptr pc) {	1044	void ThreadIgnoreEnd(ThreadState *thr) {
1084	DPrintf("#%d: ThreadIgnoreEnd\n", thr->tid);	1045	DPrintf("#%d: ThreadIgnoreEnd\n", thr->tid);
1085	CHECK_GT(thr->ignore_reads_and_writes, 0);	1046	CHECK_GT(thr->ignore_reads_and_writes, 0);
1086	thr->ignore_reads_and_writes--;	1047	thr->ignore_reads_and_writes--;
...	@@ -1100,17 +1061,17 @@ uptr __tsan_testonly_shadow_stack_current_size() {	...	@@ -1100,17 +1061,17 @@ uptr __tsan_testonly_shadow_stack_current_size() {
1100	}	1061	}
1101	#endif	1062	#endif
1102		1063
1103	void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc, bool save_stack) {	1064	void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc) {
1104	DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr->tid);	1065	DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr->tid);
1105	thr->ignore_sync++;	1066	thr->ignore_sync++;
1106	CHECK_GT(thr->ignore_sync, 0);	1067	CHECK_GT(thr->ignore_sync, 0);
1107	#if !SANITIZER_GO	1068	#if !SANITIZER_GO
1108	if (save_stack && !ctx->after_multithreaded_fork)	1069	if (pc && !ctx->after_multithreaded_fork)
1109	thr->sync_ignore_set.Add(CurrentStackId(thr, pc));	1070	thr->sync_ignore_set.Add(CurrentStackId(thr, pc));
1110	#endif	1071	#endif
1111	}	1072	}
1112		1073
1113	void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc) {	1074	void ThreadIgnoreSyncEnd(ThreadState *thr) {
1114	DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr->tid);	1075	DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr->tid);
1115	CHECK_GT(thr->ignore_sync, 0);	1076	CHECK_GT(thr->ignore_sync, 0);
1116	thr->ignore_sync--;	1077	thr->ignore_sync--;
...	@@ -1129,7 +1090,6 @@ void build_consistency_debug() {}	...	@@ -1129,7 +1090,6 @@ void build_consistency_debug() {}
1129	#else	1090	#else
1130	void build_consistency_release() {}	1091	void build_consistency_release() {}
1131	#endif	1092	#endif
1132
1133	} // namespace __tsan	1093	} // namespace __tsan
1134		1094
1135	#if SANITIZER_CHECK_DEADLOCKS	1095	#if SANITIZER_CHECK_DEADLOCKS
...	@@ -1137,23 +1097,27 @@ namespace __sanitizer {	...	@@ -1137,23 +1097,27 @@ namespace __sanitizer {
1137	using namespace __tsan;	1097	using namespace __tsan;
1138	MutexMeta mutex_meta[] = {	1098	MutexMeta mutex_meta[] = {
1139	{MutexInvalid, "Invalid", {}},	1099	{MutexInvalid, "Invalid", {}},
1140	{MutexThreadRegistry, "ThreadRegistry", {}},	1100	{MutexThreadRegistry,
1141	{MutexTypeTrace, "Trace", {MutexLeaf}},	1101	"ThreadRegistry",
1142	{MutexTypeReport, "Report", {MutexTypeSyncVar}},	1102	{MutexTypeSlots, MutexTypeTrace, MutexTypeReport}},
1143	{MutexTypeSyncVar, "SyncVar", {}},	1103	{MutexTypeReport, "Report", {MutexTypeTrace}},
		1104	{MutexTypeSyncVar, "SyncVar", {MutexTypeReport, MutexTypeTrace}},
1144	{MutexTypeAnnotations, "Annotations", {}},	1105	{MutexTypeAnnotations, "Annotations", {}},
1145	{MutexTypeAtExit, "AtExit", {MutexTypeSyncVar}},	1106	{MutexTypeAtExit, "AtExit", {}},
1146	{MutexTypeFired, "Fired", {MutexLeaf}},	1107	{MutexTypeFired, "Fired", {MutexLeaf}},
1147	{MutexTypeRacy, "Racy", {MutexLeaf}},	1108	{MutexTypeRacy, "Racy", {MutexLeaf}},
1148	{MutexTypeGlobalProc, "GlobalProc", {}},	1109	{MutexTypeGlobalProc, "GlobalProc", {MutexTypeSlot, MutexTypeSlots}},
		1110	{MutexTypeInternalAlloc, "InternalAlloc", {MutexLeaf}},
		1111	{MutexTypeTrace, "Trace", {}},
		1112	{MutexTypeSlot,
		1113	"Slot",
		1114	{MutexMulti, MutexTypeTrace, MutexTypeSyncVar, MutexThreadRegistry,
		1115	MutexTypeSlots}},
		1116	{MutexTypeSlots, "Slots", {MutexTypeTrace, MutexTypeReport}},
1149	{},	1117	{},
1150	};	1118	};
1151		1119
1152	void PrintMutexPC(uptr pc) { StackTrace(&pc, 1).Print(); }	1120	void PrintMutexPC(uptr pc) { StackTrace(&pc, 1).Print(); }
1153	} // namespace __sanitizer
1154	#endif
1155		1121
1156	#if !SANITIZER_GO	1122	} // namespace __sanitizer
1157	// Must be included in this file to make sure everything is inlined.
1158	# include "tsan_interface_inl.h"
1159	#endif	1123	#endif

lib/tsan/tsan_rtl.h+376-429

...	@@ -34,17 +34,19 @@	...	@@ -34,17 +34,19 @@
34	#include "sanitizer_common/sanitizer_suppressions.h"	34	#include "sanitizer_common/sanitizer_suppressions.h"
35	#include "sanitizer_common/sanitizer_thread_registry.h"	35	#include "sanitizer_common/sanitizer_thread_registry.h"
36	#include "sanitizer_common/sanitizer_vector.h"	36	#include "sanitizer_common/sanitizer_vector.h"
37	#include "tsan_clock.h"
38	#include "tsan_defs.h"	37	#include "tsan_defs.h"
39	#include "tsan_flags.h"	38	#include "tsan_flags.h"
		39	#include "tsan_ignoreset.h"
		40	#include "tsan_ilist.h"
40	#include "tsan_mman.h"	41	#include "tsan_mman.h"
41	#include "tsan_sync.h"
42	#include "tsan_trace.h"
43	#include "tsan_report.h"
44	#include "tsan_platform.h"
45	#include "tsan_mutexset.h"	42	#include "tsan_mutexset.h"
46	#include "tsan_ignoreset.h"	43	#include "tsan_platform.h"
		44	#include "tsan_report.h"
		45	#include "tsan_shadow.h"
47	#include "tsan_stack_trace.h"	46	#include "tsan_stack_trace.h"
		47	#include "tsan_sync.h"
		48	#include "tsan_trace.h"
		49	#include "tsan_vector_clock.h"
48		50
49	#if SANITIZER_WORDSIZE != 64	51	#if SANITIZER_WORDSIZE != 64
50	# error "ThreadSanitizer is supported only on 64-bit platforms"	52	# error "ThreadSanitizer is supported only on 64-bit platforms"
...	@@ -54,7 +56,8 @@ namespace __tsan {	...	@@ -54,7 +56,8 @@ namespace __tsan {
54		56
55	#if !SANITIZER_GO	57	#if !SANITIZER_GO
56	struct MapUnmapCallback;	58	struct MapUnmapCallback;
57	#if defined(__mips64) \|\| defined(__aarch64__) \|\| defined(__powerpc__)	59	#if defined(__mips64) \|\| defined(__aarch64__) \|\| defined(__loongarch__) \|\| \
		60	defined(__powerpc__)
58		61
59	struct AP32 {	62	struct AP32 {
60	static const uptr kSpaceBeg = 0;	63	static const uptr kSpaceBeg = 0;
...	@@ -69,6 +72,11 @@ struct AP32 {	...	@@ -69,6 +72,11 @@ struct AP32 {
69	typedef SizeClassAllocator32<AP32> PrimaryAllocator;	72	typedef SizeClassAllocator32<AP32> PrimaryAllocator;
70	#else	73	#else
71	struct AP64 { // Allocator64 parameters. Deliberately using a short name.	74	struct AP64 { // Allocator64 parameters. Deliberately using a short name.
		75	# if defined(__s390x__)
		76	typedef MappingS390x Mapping;
		77	# else
		78	typedef Mapping48AddressSpace Mapping;
		79	# endif
72	static const uptr kSpaceBeg = Mapping::kHeapMemBeg;	80	static const uptr kSpaceBeg = Mapping::kHeapMemBeg;
73	static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg;	81	static const uptr kSpaceSize = Mapping::kHeapMemEnd - Mapping::kHeapMemBeg;
74	static const uptr kMetadataSize = 0;	82	static const uptr kMetadataSize = 0;
...	@@ -84,240 +92,6 @@ typedef Allocator::AllocatorCache AllocatorCache;	...	@@ -84,240 +92,6 @@ typedef Allocator::AllocatorCache AllocatorCache;
84	Allocator *allocator();	92	Allocator *allocator();
85	#endif	93	#endif
86		94
87	const u64 kShadowRodata = (u64)-1; // .rodata shadow marker
88
89	// FastState (from most significant bit):
90	// ignore : 1
91	// tid : kTidBits
92	// unused : -
93	// history_size : 3
94	// epoch : kClkBits
95	class FastState {
96	public:
97	FastState(u64 tid, u64 epoch) {
98	x_ = tid << kTidShift;
99	x_ \|= epoch;
100	DCHECK_EQ(tid, this->tid());
101	DCHECK_EQ(epoch, this->epoch());
102	DCHECK_EQ(GetIgnoreBit(), false);
103	}
104
105	explicit FastState(u64 x)
106	: x_(x) {
107	}
108
109	u64 raw() const {
110	return x_;
111	}
112
113	u64 tid() const {
114	u64 res = (x_ & ~kIgnoreBit) >> kTidShift;
115	return res;
116	}
117
118	u64 TidWithIgnore() const {
119	u64 res = x_ >> kTidShift;
120	return res;
121	}
122
123	u64 epoch() const {
124	u64 res = x_ & ((1ull << kClkBits) - 1);
125	return res;
126	}
127
128	void IncrementEpoch() {
129	u64 old_epoch = epoch();
130	x_ += 1;
131	DCHECK_EQ(old_epoch + 1, epoch());
132	(void)old_epoch;
133	}
134
135	void SetIgnoreBit() { x_ \|= kIgnoreBit; }
136	void ClearIgnoreBit() { x_ &= ~kIgnoreBit; }
137	bool GetIgnoreBit() const { return (s64)x_ < 0; }
138
139	void SetHistorySize(int hs) {
140	CHECK_GE(hs, 0);
141	CHECK_LE(hs, 7);
142	x_ = (x_ & ~(kHistoryMask << kHistoryShift)) \| (u64(hs) << kHistoryShift);
143	}
144
145	ALWAYS_INLINE
146	int GetHistorySize() const {
147	return (int)((x_ >> kHistoryShift) & kHistoryMask);
148	}
149
150	void ClearHistorySize() {
151	SetHistorySize(0);
152	}
153
154	ALWAYS_INLINE
155	u64 GetTracePos() const {
156	const int hs = GetHistorySize();
157	// When hs == 0, the trace consists of 2 parts.
158	const u64 mask = (1ull << (kTracePartSizeBits + hs + 1)) - 1;
159	return epoch() & mask;
160	}
161
162	private:
163	friend class Shadow;
164	static const int kTidShift = 64 - kTidBits - 1;
165	static const u64 kIgnoreBit = 1ull << 63;
166	static const u64 kFreedBit = 1ull << 63;
167	static const u64 kHistoryShift = kClkBits;
168	static const u64 kHistoryMask = 7;
169	u64 x_;
170	};
171
172	// Shadow (from most significant bit):
173	// freed : 1
174	// tid : kTidBits
175	// is_atomic : 1
176	// is_read : 1
177	// size_log : 2
178	// addr0 : 3
179	// epoch : kClkBits
180	class Shadow : public FastState {
181	public:
182	explicit Shadow(u64 x)
183	: FastState(x) {
184	}
185
186	explicit Shadow(const FastState &s)
187	: FastState(s.x_) {
188	ClearHistorySize();
189	}
190
191	void SetAddr0AndSizeLog(u64 addr0, unsigned kAccessSizeLog) {
192	DCHECK_EQ((x_ >> kClkBits) & 31, 0);
193	DCHECK_LE(addr0, 7);
194	DCHECK_LE(kAccessSizeLog, 3);
195	x_ \|= ((kAccessSizeLog << 3) \| addr0) << kClkBits;
196	DCHECK_EQ(kAccessSizeLog, size_log());
197	DCHECK_EQ(addr0, this->addr0());
198	}
199
200	void SetWrite(unsigned kAccessIsWrite) {
201	DCHECK_EQ(x_ & kReadBit, 0);
202	if (!kAccessIsWrite)
203	x_ \|= kReadBit;
204	DCHECK_EQ(kAccessIsWrite, IsWrite());
205	}
206
207	void SetAtomic(bool kIsAtomic) {
208	DCHECK(!IsAtomic());
209	if (kIsAtomic)
210	x_ \|= kAtomicBit;
211	DCHECK_EQ(IsAtomic(), kIsAtomic);
212	}
213
214	bool IsAtomic() const {
215	return x_ & kAtomicBit;
216	}
217
218	bool IsZero() const {
219	return x_ == 0;
220	}
221
222	static inline bool TidsAreEqual(const Shadow s1, const Shadow s2) {
223	u64 shifted_xor = (s1.x_ ^ s2.x_) >> kTidShift;
224	DCHECK_EQ(shifted_xor == 0, s1.TidWithIgnore() == s2.TidWithIgnore());
225	return shifted_xor == 0;
226	}
227
228	static ALWAYS_INLINE
229	bool Addr0AndSizeAreEqual(const Shadow s1, const Shadow s2) {
230	u64 masked_xor = ((s1.x_ ^ s2.x_) >> kClkBits) & 31;
231	return masked_xor == 0;
232	}
233
234	static ALWAYS_INLINE bool TwoRangesIntersect(Shadow s1, Shadow s2,
235	unsigned kS2AccessSize) {
236	bool res = false;
237	u64 diff = s1.addr0() - s2.addr0();
238	if ((s64)diff < 0) { // s1.addr0 < s2.addr0
239	// if (s1.addr0() + size1) > s2.addr0()) return true;
240	if (s1.size() > -diff)
241	res = true;
242	} else {
243	// if (s2.addr0() + kS2AccessSize > s1.addr0()) return true;
244	if (kS2AccessSize > diff)
245	res = true;
246	}
247	DCHECK_EQ(res, TwoRangesIntersectSlow(s1, s2));
248	DCHECK_EQ(res, TwoRangesIntersectSlow(s2, s1));
249	return res;
250	}
251
252	u64 ALWAYS_INLINE addr0() const { return (x_ >> kClkBits) & 7; }
253	u64 ALWAYS_INLINE size() const { return 1ull << size_log(); }
254	bool ALWAYS_INLINE IsWrite() const { return !IsRead(); }
255	bool ALWAYS_INLINE IsRead() const { return x_ & kReadBit; }
256
257	// The idea behind the freed bit is as follows.
258	// When the memory is freed (or otherwise unaccessible) we write to the shadow
259	// values with tid/epoch related to the free and the freed bit set.
260	// During memory accesses processing the freed bit is considered
261	// as msb of tid. So any access races with shadow with freed bit set
262	// (it is as if write from a thread with which we never synchronized before).
263	// This allows us to detect accesses to freed memory w/o additional
264	// overheads in memory access processing and at the same time restore
265	// tid/epoch of free.
266	void MarkAsFreed() {
267	x_ \|= kFreedBit;
268	}
269
270	bool IsFreed() const {
271	return x_ & kFreedBit;
272	}
273
274	bool GetFreedAndReset() {
275	bool res = x_ & kFreedBit;
276	x_ &= ~kFreedBit;
277	return res;
278	}
279
280	bool ALWAYS_INLINE IsBothReadsOrAtomic(bool kIsWrite, bool kIsAtomic) const {
281	bool v = x_ & ((u64(kIsWrite ^ 1) << kReadShift)
282	\| (u64(kIsAtomic) << kAtomicShift));
283	DCHECK_EQ(v, (!IsWrite() && !kIsWrite) \|\| (IsAtomic() && kIsAtomic));
284	return v;
285	}
286
287	bool ALWAYS_INLINE IsRWNotWeaker(bool kIsWrite, bool kIsAtomic) const {
288	bool v = ((x_ >> kReadShift) & 3)
289	<= u64((kIsWrite ^ 1) \| (kIsAtomic << 1));
290	DCHECK_EQ(v, (IsAtomic() < kIsAtomic) \|\|
291	(IsAtomic() == kIsAtomic && !IsWrite() <= !kIsWrite));
292	return v;
293	}
294
295	bool ALWAYS_INLINE IsRWWeakerOrEqual(bool kIsWrite, bool kIsAtomic) const {
296	bool v = ((x_ >> kReadShift) & 3)
297	>= u64((kIsWrite ^ 1) \| (kIsAtomic << 1));
298	DCHECK_EQ(v, (IsAtomic() > kIsAtomic) \|\|
299	(IsAtomic() == kIsAtomic && !IsWrite() >= !kIsWrite));
300	return v;
301	}
302
303	private:
304	static const u64 kReadShift = 5 + kClkBits;
305	static const u64 kReadBit = 1ull << kReadShift;
306	static const u64 kAtomicShift = 6 + kClkBits;
307	static const u64 kAtomicBit = 1ull << kAtomicShift;
308
309	u64 size_log() const { return (x_ >> (3 + kClkBits)) & 3; }
310
311	static bool TwoRangesIntersectSlow(const Shadow s1, const Shadow s2) {
312	if (s1.addr0() == s2.addr0()) return true;
313	if (s1.addr0() < s2.addr0() && s1.addr0() + s1.size() > s2.addr0())
314	return true;
315	if (s2.addr0() < s1.addr0() && s2.addr0() + s2.size() > s1.addr0())
316	return true;
317	return false;
318	}
319	};
320
321	struct ThreadSignalContext;	95	struct ThreadSignalContext;
322		96
323	struct JmpBuf {	97	struct JmpBuf {
...	@@ -344,7 +118,6 @@ struct Processor {	...	@@ -344,7 +118,6 @@ struct Processor {
344	#endif	118	#endif
345	DenseSlabAllocCache block_cache;	119	DenseSlabAllocCache block_cache;
346	DenseSlabAllocCache sync_cache;	120	DenseSlabAllocCache sync_cache;
347	DenseSlabAllocCache clock_cache;
348	DDPhysicalThread *dd_pt;	121	DDPhysicalThread *dd_pt;
349	};	122	};
350		123
...	@@ -358,64 +131,86 @@ struct ScopedGlobalProcessor {	...	@@ -358,64 +131,86 @@ struct ScopedGlobalProcessor {
358	};	131	};
359	#endif	132	#endif
360		133
		134	struct TidEpoch {
		135	Tid tid;
		136	Epoch epoch;
		137	};
		138
		139	struct TidSlot {
		140	Mutex mtx;
		141	Sid sid;
		142	atomic_uint32_t raw_epoch;
		143	ThreadState *thr;
		144	Vector<TidEpoch> journal;
		145	INode node;
		146
		147	Epoch epoch() const {
		148	return static_cast<Epoch>(atomic_load(&raw_epoch, memory_order_relaxed));
		149	}
		150
		151	void SetEpoch(Epoch v) {
		152	atomic_store(&raw_epoch, static_cast<u32>(v), memory_order_relaxed);
		153	}
		154
		155	TidSlot();
		156	} ALIGNED(SANITIZER_CACHE_LINE_SIZE);
		157
361	// This struct is stored in TLS.	158	// This struct is stored in TLS.
362	struct ThreadState {	159	struct ThreadState {
363	FastState fast_state;	160	FastState fast_state;
364	// Synch epoch represents the threads's epoch before the last synchronization	161	int ignore_sync;
365	// action. It allows to reduce number of shadow state updates.	162	#if !SANITIZER_GO
366	// For example, fast_synch_epoch=100, last write to addr X was at epoch=150,	163	int ignore_interceptors;
367	// if we are processing write to X from the same thread at epoch=200,	164	#endif
368	// we do nothing, because both writes happen in the same 'synch epoch'.	165	uptr *shadow_stack_pos;
369	// That is, if another memory access does not race with the former write,	166
370	// it does not race with the latter as well.	167	// Current position in tctx->trace.Back()->events (Event*).
371	// QUESTION: can we can squeeze this into ThreadState::Fast?	168	atomic_uintptr_t trace_pos;
372	// E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are	169	// PC of the last memory access, used to compute PC deltas in the trace.
373	// taken by epoch between synchs.	170	uptr trace_prev_pc;
374	// This way we can save one load from tls.	171
375	u64 fast_synch_epoch;
376	// Technically `current` should be a separate THREADLOCAL variable;	172	// Technically `current` should be a separate THREADLOCAL variable;
377	// but it is placed here in order to share cache line with previous fields.	173	// but it is placed here in order to share cache line with previous fields.
378	ThreadState* current;	174	ThreadState* current;
		175
		176	atomic_sint32_t pending_signals;
		177
		178	VectorClock clock;
		179
379	// This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.	180	// This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
380	// We do not distinguish beteween ignoring reads and writes	181	// We do not distinguish beteween ignoring reads and writes
381	// for better performance.	182	// for better performance.
382	int ignore_reads_and_writes;	183	int ignore_reads_and_writes;
383	int ignore_sync;
384	int suppress_reports;	184	int suppress_reports;
385	// Go does not support ignores.	185	// Go does not support ignores.
386	#if !SANITIZER_GO	186	#if !SANITIZER_GO
387	IgnoreSet mop_ignore_set;	187	IgnoreSet mop_ignore_set;
388	IgnoreSet sync_ignore_set;	188	IgnoreSet sync_ignore_set;
389	#endif	189	#endif
390	// C/C++ uses fixed size shadow stack embed into Trace.
391	// Go uses malloc-allocated shadow stack with dynamic size.
392	uptr *shadow_stack;	190	uptr *shadow_stack;
393	uptr *shadow_stack_end;	191	uptr *shadow_stack_end;
394	uptr *shadow_stack_pos;
395	u64 *racy_shadow_addr;
396	u64 racy_state[2];
397	MutexSet mset;
398	ThreadClock clock;
399	#if !SANITIZER_GO	192	#if !SANITIZER_GO
400	Vector<JmpBuf> jmp_bufs;	193	Vector<JmpBuf> jmp_bufs;
401	int ignore_interceptors;	194	int in_symbolizer;
402	#endif	195	atomic_uintptr_t in_blocking_func;
403	const u32 tid;
404	const int unique_id;
405	bool in_symbolizer;
406	bool in_ignored_lib;	196	bool in_ignored_lib;
407	bool is_inited;	197	bool is_inited;
		198	#endif
		199	MutexSet mset;
408	bool is_dead;	200	bool is_dead;
409	bool is_freeing;	201	const Tid tid;
410	bool is_vptr_access;	202	uptr stk_addr;
411	const uptr stk_addr;	203	uptr stk_size;
412	const uptr stk_size;	204	uptr tls_addr;
413	const uptr tls_addr;	205	uptr tls_size;
414	const uptr tls_size;
415	ThreadContext *tctx;	206	ThreadContext *tctx;
416		207
417	DDLogicalThread *dd_lt;	208	DDLogicalThread *dd_lt;
418		209
		210	TidSlot *slot;
		211	uptr slot_epoch;
		212	bool slot_locked;
		213
419	// Current wired Processor, or nullptr. Required to handle any events.	214	// Current wired Processor, or nullptr. Required to handle any events.
420	Processor *proc1;	215	Processor *proc1;
421	#if !SANITIZER_GO	216	#if !SANITIZER_GO
...	@@ -425,11 +220,11 @@ struct ThreadState {	...	@@ -425,11 +220,11 @@ struct ThreadState {
425	#endif	220	#endif
426		221
427	atomic_uintptr_t in_signal_handler;	222	atomic_uintptr_t in_signal_handler;
428	ThreadSignalContext *signal_ctx;	223	atomic_uintptr_t signal_ctx;
429		224
430	#if !SANITIZER_GO	225	#if !SANITIZER_GO
431	u32 last_sleep_stack_id;	226	StackID last_sleep_stack_id;
432	ThreadClock last_sleep_clock;	227	VectorClock last_sleep_clock;
433	#endif	228	#endif
434		229
435	// Set in regions of runtime that must be signal-safe and fork-safe.	230	// Set in regions of runtime that must be signal-safe and fork-safe.
...	@@ -438,47 +233,43 @@ struct ThreadState {	...	@@ -438,47 +233,43 @@ struct ThreadState {
438		233
439	const ReportDesc *current_report;	234	const ReportDesc *current_report;
440		235
441	explicit ThreadState(Context *ctx, u32 tid, int unique_id, u64 epoch,	236	explicit ThreadState(Tid tid);
442	unsigned reuse_count, uptr stk_addr, uptr stk_size,	237	} ALIGNED(SANITIZER_CACHE_LINE_SIZE);
443	uptr tls_addr, uptr tls_size);
444	};
445		238
446	#if !SANITIZER_GO	239	#if !SANITIZER_GO
447	#if SANITIZER_MAC \|\| SANITIZER_ANDROID	240	#if SANITIZER_APPLE \|\| SANITIZER_ANDROID
448	ThreadState *cur_thread();	241	ThreadState *cur_thread();
449	void set_cur_thread(ThreadState *thr);	242	void set_cur_thread(ThreadState *thr);
450	void cur_thread_finalize();	243	void cur_thread_finalize();
451	inline void cur_thread_init() { }	244	inline ThreadState *cur_thread_init() { return cur_thread(); }
452	#else	245	# else
453	__attribute__((tls_model("initial-exec")))	246	__attribute__((tls_model("initial-exec")))
454	extern THREADLOCAL char cur_thread_placeholder[];	247	extern THREADLOCAL char cur_thread_placeholder[];
455	inline ThreadState *cur_thread() {	248	inline ThreadState *cur_thread() {
456	return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current;	249	return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current;
457	}	250	}
458	inline void cur_thread_init() {	251	inline ThreadState *cur_thread_init() {
459	ThreadState thr = reinterpret_cast<ThreadState >(cur_thread_placeholder);	252	ThreadState thr = reinterpret_cast<ThreadState >(cur_thread_placeholder);
460	if (UNLIKELY(!thr->current))	253	if (UNLIKELY(!thr->current))
461	thr->current = thr;	254	thr->current = thr;
		255	return thr->current;
462	}	256	}
463	inline void set_cur_thread(ThreadState *thr) {	257	inline void set_cur_thread(ThreadState *thr) {
464	reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr;	258	reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr;
465	}	259	}
466	inline void cur_thread_finalize() { }	260	inline void cur_thread_finalize() { }
467	#endif // SANITIZER_MAC \|\| SANITIZER_ANDROID	261	# endif // SANITIZER_APPLE \|\| SANITIZER_ANDROID
468	#endif // SANITIZER_GO	262	#endif // SANITIZER_GO
469		263
470	class ThreadContext final : public ThreadContextBase {	264	class ThreadContext final : public ThreadContextBase {
471	public:	265	public:
472	explicit ThreadContext(int tid);	266	explicit ThreadContext(Tid tid);
473	~ThreadContext();	267	~ThreadContext();
474	ThreadState *thr;	268	ThreadState *thr;
475	u32 creation_stack_id;	269	StackID creation_stack_id;
476	SyncClock sync;	270	VectorClock *sync;
477	// Epoch at which the thread had started.	271	uptr sync_epoch;
478	// If we see an event from the thread stamped by an older epoch,	272	Trace trace;
479	// the event is from a dead thread that shared tid with this thread.
480	u64 epoch0;
481	u64 epoch1;
482		273
483	// Override superclass callbacks.	274	// Override superclass callbacks.
484	void OnDead() override;	275	void OnDead() override;
...	@@ -492,13 +283,7 @@ class ThreadContext final : public ThreadContextBase {	...	@@ -492,13 +283,7 @@ class ThreadContext final : public ThreadContextBase {
492		283
493	struct RacyStacks {	284	struct RacyStacks {
494	MD5Hash hash[2];	285	MD5Hash hash[2];
495	bool operator==(const RacyStacks &other) const {	286	bool operator==(const RacyStacks &other) const;
496	if (hash[0] == other.hash[0] && hash[1] == other.hash[1])
497	return true;
498	if (hash[0] == other.hash[1] && hash[1] == other.hash[0])
499	return true;
500	return false;
501	}
502	};	287	};
503		288
504	struct RacyAddress {	289	struct RacyAddress {
...	@@ -524,28 +309,75 @@ struct Context {	...	@@ -524,28 +309,75 @@ struct Context {
524		309
525	Mutex report_mtx;	310	Mutex report_mtx;
526	int nreported;	311	int nreported;
527	int nmissed_expected;
528	atomic_uint64_t last_symbolize_time_ns;	312	atomic_uint64_t last_symbolize_time_ns;
529		313
530	void *background_thread;	314	void *background_thread;
531	atomic_uint32_t stop_background_thread;	315	atomic_uint32_t stop_background_thread;
532		316
533	ThreadRegistry *thread_registry;	317	ThreadRegistry thread_registry;
		318
		319	// This is used to prevent a very unlikely but very pathological behavior.
		320	// Since memory access handling is not synchronized with DoReset,
		321	// a thread running concurrently with DoReset can leave a bogus shadow value
		322	// that will be later falsely detected as a race. For such false races
		323	// RestoreStack will return false and we will not report it.
		324	// However, consider that a thread leaves a whole lot of such bogus values
		325	// and these values are later read by a whole lot of threads.
		326	// This will cause massive amounts of ReportRace calls and lots of
		327	// serialization. In very pathological cases the resulting slowdown
		328	// can be >100x. This is very unlikely, but it was presumably observed
		329	// in practice: https://github.com/google/sanitizers/issues/1552
		330	// If this happens, previous access sid+epoch will be the same for all of
		331	// these false races b/c if the thread will try to increment epoch, it will
		332	// notice that DoReset has happened and will stop producing bogus shadow
		333	// values. So, last_spurious_race is used to remember the last sid+epoch
		334	// for which RestoreStack returned false. Then it is used to filter out
		335	// races with the same sid+epoch very early and quickly.
		336	// It is of course possible that multiple threads left multiple bogus shadow
		337	// values and all of them are read by lots of threads at the same time.
		338	// In such case last_spurious_race will only be able to deduplicate a few
		339	// races from one thread, then few from another and so on. An alternative
		340	// would be to hold an array of such sid+epoch, but we consider such scenario
		341	// as even less likely.
		342	// Note: this can lead to some rare false negatives as well:
		343	// 1. When a legit access with the same sid+epoch participates in a race
		344	// as the "previous" memory access, it will be wrongly filtered out.
		345	// 2. When RestoreStack returns false for a legit memory access because it
		346	// was already evicted from the thread trace, we will still remember it in
		347	// last_spurious_race. Then if there is another racing memory access from
		348	// the same thread that happened in the same epoch, but was stored in the
		349	// next thread trace part (which is still preserved in the thread trace),
		350	// we will also wrongly filter it out while RestoreStack would actually
		351	// succeed for that second memory access.
		352	RawShadow last_spurious_race;
534		353
535	Mutex racy_mtx;	354	Mutex racy_mtx;
536	Vector<RacyStacks> racy_stacks;	355	Vector<RacyStacks> racy_stacks;
537	Vector<RacyAddress> racy_addresses;
538	// Number of fired suppressions may be large enough.	356	// Number of fired suppressions may be large enough.
539	Mutex fired_suppressions_mtx;	357	Mutex fired_suppressions_mtx;
540	InternalMmapVector<FiredSuppression> fired_suppressions;	358	InternalMmapVector<FiredSuppression> fired_suppressions;
541	DDetector *dd;	359	DDetector *dd;
542		360
543	ClockAlloc clock_alloc;
544
545	Flags flags;	361	Flags flags;
546		362	fd_t memprof_fd;
547	u64 int_alloc_cnt[MBlockTypeCount];	363
548	u64 int_alloc_siz[MBlockTypeCount];	364	// The last slot index (kFreeSid) is used to denote freed memory.
		365	TidSlot slots[kThreadSlotCount - 1];
		366
		367	// Protects global_epoch, slot_queue, trace_part_recycle.
		368	Mutex slot_mtx;
		369	uptr global_epoch; // guarded by slot_mtx and by all slot mutexes
		370	bool resetting; // global reset is in progress
		371	IList<TidSlot, &TidSlot::node> slot_queue SANITIZER_GUARDED_BY(slot_mtx);
		372	IList<TraceHeader, &TraceHeader::global, TracePart> trace_part_recycle
		373	SANITIZER_GUARDED_BY(slot_mtx);
		374	uptr trace_part_total_allocated SANITIZER_GUARDED_BY(slot_mtx);
		375	uptr trace_part_recycle_finished SANITIZER_GUARDED_BY(slot_mtx);
		376	uptr trace_part_finished_excess SANITIZER_GUARDED_BY(slot_mtx);
		377	#if SANITIZER_GO
		378	uptr mapped_shadow_begin;
		379	uptr mapped_shadow_end;
		380	#endif
549	};	381	};
550		382
551	extern Context *ctx; // The one and the only global runtime context.	383	extern Context *ctx; // The one and the only global runtime context.
...	@@ -574,17 +406,17 @@ uptr TagFromShadowStackFrame(uptr pc);	...	@@ -574,17 +406,17 @@ uptr TagFromShadowStackFrame(uptr pc);
574		406
575	class ScopedReportBase {	407	class ScopedReportBase {
576	public:	408	public:
577	void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, StackTrace stack,	409	void AddMemoryAccess(uptr addr, uptr external_tag, Shadow s, Tid tid,
578	const MutexSet *mset);	410	StackTrace stack, const MutexSet *mset);
579	void AddStack(StackTrace stack, bool suppressable = false);	411	void AddStack(StackTrace stack, bool suppressable = false);
580	void AddThread(const ThreadContext *tctx, bool suppressable = false);	412	void AddThread(const ThreadContext *tctx, bool suppressable = false);
581	void AddThread(int unique_tid, bool suppressable = false);	413	void AddThread(Tid tid, bool suppressable = false);
582	void AddUniqueTid(int unique_tid);	414	void AddUniqueTid(Tid unique_tid);
583	void AddMutex(const SyncVar *s);	415	int AddMutex(uptr addr, StackID creation_stack_id);
584	u64 AddMutex(u64 id);
585	void AddLocation(uptr addr, uptr size);	416	void AddLocation(uptr addr, uptr size);
586	void AddSleep(u32 stack_id);	417	void AddSleep(StackID stack_id);
587	void SetCount(int count);	418	void SetCount(int count);
		419	void SetSigNum(int sig);
588		420
589	const ReportDesc *GetReport() const;	421	const ReportDesc *GetReport() const;
590		422
...	@@ -598,8 +430,6 @@ class ScopedReportBase {	...	@@ -598,8 +430,6 @@ class ScopedReportBase {
598	// at best it will cause deadlocks on internal mutexes.	430	// at best it will cause deadlocks on internal mutexes.
599	ScopedIgnoreInterceptors ignore_interceptors_;	431	ScopedIgnoreInterceptors ignore_interceptors_;
600		432
601	void AddDeadMutex(u64 id);
602
603	ScopedReportBase(const ScopedReportBase &) = delete;	433	ScopedReportBase(const ScopedReportBase &) = delete;
604	void operator=(const ScopedReportBase &) = delete;	434	void operator=(const ScopedReportBase &) = delete;
605	};	435	};
...	@@ -615,8 +445,6 @@ class ScopedReport : public ScopedReportBase {	...	@@ -615,8 +445,6 @@ class ScopedReport : public ScopedReportBase {
615		445
616	bool ShouldReport(ThreadState *thr, ReportType typ);	446	bool ShouldReport(ThreadState *thr, ReportType typ);
617	ThreadContext IsThreadStackOrTls(uptr addr, bool is_stack);	447	ThreadContext IsThreadStackOrTls(uptr addr, bool is_stack);
618	void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
619	MutexSet mset, uptr tag = nullptr);
620		448
621	// The stack could look like:	449	// The stack could look like:
622	// <start> \| <main> \| <foo> \| tag \| <bar>	450	// <start> \| <main> \| <foo> \| tag \| <bar>
...	@@ -656,19 +484,20 @@ void MapThreadTrace(uptr addr, uptr size, const char *name);	...	@@ -656,19 +484,20 @@ void MapThreadTrace(uptr addr, uptr size, const char *name);
656	void DontNeedShadowFor(uptr addr, uptr size);	484	void DontNeedShadowFor(uptr addr, uptr size);
657	void UnmapShadow(ThreadState *thr, uptr addr, uptr size);	485	void UnmapShadow(ThreadState *thr, uptr addr, uptr size);
658	void InitializeShadowMemory();	486	void InitializeShadowMemory();
		487	void DontDumpShadow(uptr addr, uptr size);
659	void InitializeInterceptors();	488	void InitializeInterceptors();
660	void InitializeLibIgnore();	489	void InitializeLibIgnore();
661	void InitializeDynamicAnnotations();	490	void InitializeDynamicAnnotations();
662		491
663	void ForkBefore(ThreadState *thr, uptr pc);	492	void ForkBefore(ThreadState *thr, uptr pc);
664	void ForkParentAfter(ThreadState *thr, uptr pc);	493	void ForkParentAfter(ThreadState *thr, uptr pc);
665	void ForkChildAfter(ThreadState *thr, uptr pc);	494	void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread);
666		495
667	void ReportRace(ThreadState *thr);	496	void ReportRace(ThreadState thr, RawShadow shadow_mem, Shadow cur, Shadow old,
		497	AccessType typ);
668	bool OutputReport(ThreadState *thr, const ScopedReport &srep);	498	bool OutputReport(ThreadState *thr, const ScopedReport &srep);
669	bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);	499	bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace);
670	bool IsExpectedReport(uptr addr, uptr size);	500	bool IsExpectedReport(uptr addr, uptr size);
671	void PrintMatchedBenignRaces();
672		501
673	#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1	502	#if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
674	# define DPrintf Printf	503	# define DPrintf Printf
...	@@ -682,10 +511,11 @@ void PrintMatchedBenignRaces();	...	@@ -682,10 +511,11 @@ void PrintMatchedBenignRaces();
682	# define DPrintf2(...)	511	# define DPrintf2(...)
683	#endif	512	#endif
684		513
685	u32 CurrentStackId(ThreadState *thr, uptr pc);	514	StackID CurrentStackId(ThreadState *thr, uptr pc);
686	ReportStack *SymbolizeStackId(u32 stack_id);	515	ReportStack *SymbolizeStackId(StackID stack_id);
687	void PrintCurrentStack(ThreadState *thr, uptr pc);	516	void PrintCurrentStack(ThreadState *thr, uptr pc);
688	void PrintCurrentStackSlow(uptr pc); // uses libunwind	517	void PrintCurrentStackSlow(uptr pc); // uses libunwind
		518	MBlock JavaHeapBlock(uptr addr, uptr start);
689		519
690	void Initialize(ThreadState *thr);	520	void Initialize(ThreadState *thr);
691	void MaybeSpawnBackgroundThread();	521	void MaybeSpawnBackgroundThread();
...	@@ -694,69 +524,49 @@ int Finalize(ThreadState *thr);	...	@@ -694,69 +524,49 @@ int Finalize(ThreadState *thr);
694	void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);	524	void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write);
695	void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);	525	void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write);
696		526
697	void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,	527	void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
698	int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic);	528	AccessType typ);
699	void MemoryAccessImpl(ThreadState *thr, uptr addr,	529	void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
700	int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,	530	AccessType typ);
701	u64 *shadow_mem, Shadow cur);	531	// This creates 2 non-inlined specialized versions of MemoryAccessRange.
702	void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,	532	template <bool is_read>
703	uptr size, bool is_write);	533	void MemoryAccessRangeT(ThreadState *thr, uptr pc, uptr addr, uptr size);
704	void MemoryAccessRangeStep(ThreadState *thr, uptr pc, uptr addr,	534
705	uptr size, uptr step, bool is_write);	535	ALWAYS_INLINE
706	void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr,	536	void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
707	int size, bool kAccessIsWrite, bool kIsAtomic);	537	bool is_write) {
708		538	if (size == 0)
709	const int kSizeLog1 = 0;	539	return;
710	const int kSizeLog2 = 1;	540	if (is_write)
711	const int kSizeLog4 = 2;	541	MemoryAccessRangeT<false>(thr, pc, addr, size);
712	const int kSizeLog8 = 3;	542	else
713		543	MemoryAccessRangeT<true>(thr, pc, addr, size);
714	void ALWAYS_INLINE MemoryRead(ThreadState *thr, uptr pc,
715	uptr addr, int kAccessSizeLog) {
716	MemoryAccess(thr, pc, addr, kAccessSizeLog, false, false);
717	}
718
719	void ALWAYS_INLINE MemoryWrite(ThreadState *thr, uptr pc,
720	uptr addr, int kAccessSizeLog) {
721	MemoryAccess(thr, pc, addr, kAccessSizeLog, true, false);
722	}
723
724	void ALWAYS_INLINE MemoryReadAtomic(ThreadState *thr, uptr pc,
725	uptr addr, int kAccessSizeLog) {
726	MemoryAccess(thr, pc, addr, kAccessSizeLog, false, true);
727	}
728
729	void ALWAYS_INLINE MemoryWriteAtomic(ThreadState *thr, uptr pc,
730	uptr addr, int kAccessSizeLog) {
731	MemoryAccess(thr, pc, addr, kAccessSizeLog, true, true);
732	}	544	}
733		545
734	void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);	546	void ShadowSet(RawShadow p, RawShadow end, RawShadow v);
735	void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);	547	void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size);
		548	void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size);
736	void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);	549	void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size);
737	void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,	550	void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
738	uptr size);	551	uptr size);
739		552
740	void ThreadIgnoreBegin(ThreadState *thr, uptr pc, bool save_stack = true);	553	void ThreadIgnoreBegin(ThreadState *thr, uptr pc);
741	void ThreadIgnoreEnd(ThreadState *thr, uptr pc);	554	void ThreadIgnoreEnd(ThreadState *thr);
742	void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc, bool save_stack = true);	555	void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc);
743	void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc);	556	void ThreadIgnoreSyncEnd(ThreadState *thr);
744
745	void FuncEntry(ThreadState *thr, uptr pc);
746	void FuncExit(ThreadState *thr);
747		557
748	int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);	558	Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached);
749	void ThreadStart(ThreadState *thr, int tid, tid_t os_id,	559	void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
750	ThreadType thread_type);	560	ThreadType thread_type);
751	void ThreadFinish(ThreadState *thr);	561	void ThreadFinish(ThreadState *thr);
752	int ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid);	562	Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid);
753	void ThreadJoin(ThreadState *thr, uptr pc, int tid);	563	void ThreadJoin(ThreadState *thr, uptr pc, Tid tid);
754	void ThreadDetach(ThreadState *thr, uptr pc, int tid);	564	void ThreadDetach(ThreadState *thr, uptr pc, Tid tid);
755	void ThreadFinalize(ThreadState *thr);	565	void ThreadFinalize(ThreadState *thr);
756	void ThreadSetName(ThreadState thr, const char name);	566	void ThreadSetName(ThreadState thr, const char name);
757	int ThreadCount(ThreadState *thr);	567	int ThreadCount(ThreadState *thr);
758	void ProcessPendingSignals(ThreadState *thr);	568	void ProcessPendingSignalsImpl(ThreadState *thr);
759	void ThreadNotJoined(ThreadState *thr, uptr pc, int tid, uptr uid);	569	void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid);
760		570
761	Processor *ProcCreate();	571	Processor *ProcCreate();
762	void ProcDestroy(Processor *proc);	572	void ProcDestroy(Processor *proc);
...	@@ -785,65 +595,12 @@ void Acquire(ThreadState *thr, uptr pc, uptr addr);	...	@@ -785,65 +595,12 @@ void Acquire(ThreadState *thr, uptr pc, uptr addr);
785	// handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal	595	// handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
786	// right before executing finalizers. This provides a coarse, but simple	596	// right before executing finalizers. This provides a coarse, but simple
787	// approximation of the actual required synchronization.	597	// approximation of the actual required synchronization.
788	void AcquireGlobal(ThreadState *thr, uptr pc);	598	void AcquireGlobal(ThreadState *thr);
789	void Release(ThreadState *thr, uptr pc, uptr addr);	599	void Release(ThreadState *thr, uptr pc, uptr addr);
790	void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr);	600	void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr);
791	void ReleaseStore(ThreadState *thr, uptr pc, uptr addr);	601	void ReleaseStore(ThreadState *thr, uptr pc, uptr addr);
792	void AfterSleep(ThreadState *thr, uptr pc);	602	void AfterSleep(ThreadState *thr, uptr pc);
793	void AcquireImpl(ThreadState thr, uptr pc, SyncClock c);	603	void IncrementEpoch(ThreadState *thr);
794	void ReleaseImpl(ThreadState thr, uptr pc, SyncClock c);
795	void ReleaseStoreAcquireImpl(ThreadState thr, uptr pc, SyncClock c);
796	void ReleaseStoreImpl(ThreadState thr, uptr pc, SyncClock c);
797	void AcquireReleaseImpl(ThreadState thr, uptr pc, SyncClock c);
798
799	// The hacky call uses custom calling convention and an assembly thunk.
800	// It is considerably faster that a normal call for the caller
801	// if it is not executed (it is intended for slow paths from hot functions).
802	// The trick is that the call preserves all registers and the compiler
803	// does not treat it as a call.
804	// If it does not work for you, use normal call.
805	#if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC
806	// The caller may not create the stack frame for itself at all,
807	// so we create a reserve stack frame for it (1024b must be enough).
808	#define HACKY_CALL(f) \
809	__asm__ __volatile__("sub $1024, %%rsp;" \
810	CFI_INL_ADJUST_CFA_OFFSET(1024) \
811	".hidden " #f "_thunk;" \
812	"call " #f "_thunk;" \
813	"add $1024, %%rsp;" \
814	CFI_INL_ADJUST_CFA_OFFSET(-1024) \
815	::: "memory", "cc");
816	#else
817	#define HACKY_CALL(f) f()
818	#endif
819
820	void TraceSwitch(ThreadState *thr);
821	uptr TraceTopPC(ThreadState *thr);
822	uptr TraceSize();
823	uptr TraceParts();
824	Trace *ThreadTrace(int tid);
825
826	extern "C" void __tsan_trace_switch();
827	void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs,
828	EventType typ, u64 addr) {
829	if (!kCollectHistory)
830	return;
831	DCHECK_GE((int)typ, 0);
832	DCHECK_LE((int)typ, 7);
833	DCHECK_EQ(GetLsb(addr, kEventPCBits), addr);
834	u64 pos = fs.GetTracePos();
835	if (UNLIKELY((pos % kTracePartSize) == 0)) {
836	#if !SANITIZER_GO
837	HACKY_CALL(__tsan_trace_switch);
838	#else
839	TraceSwitch(thr);
840	#endif
841	}
842	Event trace = (Event)GetThreadTrace(fs.tid());
843	Event *evp = &trace[pos];
844	Event ev = (u64)addr \| ((u64)typ << kEventPCBits);
845	*evp = ev;
846	}
847		604
848	#if !SANITIZER_GO	605	#if !SANITIZER_GO
849	uptr ALWAYS_INLINE HeapEnd() {	606	uptr ALWAYS_INLINE HeapEnd() {
...	@@ -851,6 +608,13 @@ uptr ALWAYS_INLINE HeapEnd() {	...	@@ -851,6 +608,13 @@ uptr ALWAYS_INLINE HeapEnd() {
851	}	608	}
852	#endif	609	#endif
853		610
		611	void SlotAttachAndLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx);
		612	void SlotDetach(ThreadState *thr);
		613	void SlotLock(ThreadState *thr) SANITIZER_ACQUIRE(thr->slot->mtx);
		614	void SlotUnlock(ThreadState *thr) SANITIZER_RELEASE(thr->slot->mtx);
		615	void DoReset(ThreadState *thr, uptr epoch);
		616	void FlushShadowMemory();
		617
854	ThreadState FiberCreate(ThreadState thr, uptr pc, unsigned flags);	618	ThreadState FiberCreate(ThreadState thr, uptr pc, unsigned flags);
855	void FiberDestroy(ThreadState thr, uptr pc, ThreadState fiber);	619	void FiberDestroy(ThreadState thr, uptr pc, ThreadState fiber);
856	void FiberSwitch(ThreadState thr, uptr pc, ThreadState fiber, unsigned flags);	620	void FiberSwitch(ThreadState thr, uptr pc, ThreadState fiber, unsigned flags);
...	@@ -861,6 +625,189 @@ enum FiberSwitchFlags {	...	@@ -861,6 +625,189 @@ enum FiberSwitchFlags {
861	FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync	625	FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync
862	};	626	};
863		627
		628	class SlotLocker {
		629	public:
		630	ALWAYS_INLINE
		631	SlotLocker(ThreadState *thr, bool recursive = false)
		632	: thr_(thr), locked_(recursive ? thr->slot_locked : false) {
		633	#if !SANITIZER_GO
		634	// We are in trouble if we are here with in_blocking_func set.
		635	// If in_blocking_func is set, all signals will be delivered synchronously,
		636	// which means we can't lock slots since the signal handler will try
		637	// to lock it recursively and deadlock.
		638	DCHECK(!atomic_load(&thr->in_blocking_func, memory_order_relaxed));
		639	#endif
		640	if (!locked_)
		641	SlotLock(thr_);
		642	}
		643
		644	ALWAYS_INLINE
		645	~SlotLocker() {
		646	if (!locked_)
		647	SlotUnlock(thr_);
		648	}
		649
		650	private:
		651	ThreadState *thr_;
		652	bool locked_;
		653	};
		654
		655	class SlotUnlocker {
		656	public:
		657	SlotUnlocker(ThreadState *thr) : thr_(thr), locked_(thr->slot_locked) {
		658	if (locked_)
		659	SlotUnlock(thr_);
		660	}
		661
		662	~SlotUnlocker() {
		663	if (locked_)
		664	SlotLock(thr_);
		665	}
		666
		667	private:
		668	ThreadState *thr_;
		669	bool locked_;
		670	};
		671
		672	ALWAYS_INLINE void ProcessPendingSignals(ThreadState *thr) {
		673	if (UNLIKELY(atomic_load_relaxed(&thr->pending_signals)))
		674	ProcessPendingSignalsImpl(thr);
		675	}
		676
		677	extern bool is_initialized;
		678
		679	ALWAYS_INLINE
		680	void LazyInitialize(ThreadState *thr) {
		681	// If we can use .preinit_array, assume that __tsan_init
		682	// called from .preinit_array initializes runtime before
		683	// any instrumented code except when tsan is used as a
		684	// shared library.
		685	#if (!SANITIZER_CAN_USE_PREINIT_ARRAY \|\| defined(SANITIZER_SHARED))
		686	if (UNLIKELY(!is_initialized))
		687	Initialize(thr);
		688	#endif
		689	}
		690
		691	void TraceResetForTesting();
		692	void TraceSwitchPart(ThreadState *thr);
		693	void TraceSwitchPartImpl(ThreadState *thr);
		694	bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size,
		695	AccessType typ, Tid ptid, VarSizeStackTrace pstk,
		696	MutexSet pmset, uptr ptag);
		697
		698	template <typename EventT>
		699	ALWAYS_INLINE WARN_UNUSED_RESULT bool TraceAcquire(ThreadState *thr,
		700	EventT **ev) {
		701	// TraceSwitchPart accesses shadow_stack, but it's called infrequently,
		702	// so we check it here proactively.
		703	DCHECK(thr->shadow_stack);
		704	Event pos = reinterpret_cast<Event >(atomic_load_relaxed(&thr->trace_pos));
		705	#if SANITIZER_DEBUG
		706	// TraceSwitch acquires these mutexes,
		707	// so we lock them here to detect deadlocks more reliably.
		708	{ Lock lock(&ctx->slot_mtx); }
		709	{ Lock lock(&thr->tctx->trace.mtx); }
		710	TracePart *current = thr->tctx->trace.parts.Back();
		711	if (current) {
		712	DCHECK_GE(pos, &current->events[0]);
		713	DCHECK_LE(pos, &current->events[TracePart::kSize]);
		714	} else {
		715	DCHECK_EQ(pos, nullptr);
		716	}
		717	#endif
		718	// TracePart is allocated with mmap and is at least 4K aligned.
		719	// So the following check is a faster way to check for part end.
		720	// It may have false positives in the middle of the trace,
		721	// they are filtered out in TraceSwitch.
		722	if (UNLIKELY(((uptr)(pos + 1) & TracePart::kAlignment) == 0))
		723	return false;
		724	ev = reinterpret_cast<EventT >(pos);
		725	return true;
		726	}
		727
		728	template <typename EventT>
		729	ALWAYS_INLINE void TraceRelease(ThreadState thr, EventT evp) {
		730	DCHECK_LE(evp + 1, &thr->tctx->trace.parts.Back()->events[TracePart::kSize]);
		731	atomic_store_relaxed(&thr->trace_pos, (uptr)(evp + 1));
		732	}
		733
		734	template <typename EventT>
		735	void TraceEvent(ThreadState *thr, EventT ev) {
		736	EventT *evp;
		737	if (!TraceAcquire(thr, &evp)) {
		738	TraceSwitchPart(thr);
		739	UNUSED bool res = TraceAcquire(thr, &evp);
		740	DCHECK(res);
		741	}
		742	*evp = ev;
		743	TraceRelease(thr, evp);
		744	}
		745
		746	ALWAYS_INLINE WARN_UNUSED_RESULT bool TryTraceFunc(ThreadState *thr,
		747	uptr pc = 0) {
		748	if (!kCollectHistory)
		749	return true;
		750	EventFunc *ev;
		751	if (UNLIKELY(!TraceAcquire(thr, &ev)))
		752	return false;
		753	ev->is_access = 0;
		754	ev->is_func = 1;
		755	ev->pc = pc;
		756	TraceRelease(thr, ev);
		757	return true;
		758	}
		759
		760	WARN_UNUSED_RESULT
		761	bool TryTraceMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
		762	AccessType typ);
		763	WARN_UNUSED_RESULT
		764	bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
		765	AccessType typ);
		766	void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
		767	AccessType typ);
		768	void TraceFunc(ThreadState *thr, uptr pc = 0);
		769	void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr,
		770	StackID stk);
		771	void TraceMutexUnlock(ThreadState *thr, uptr addr);
		772	void TraceTime(ThreadState *thr);
		773
		774	void TraceRestartFuncExit(ThreadState *thr);
		775	void TraceRestartFuncEntry(ThreadState *thr, uptr pc);
		776
		777	void GrowShadowStack(ThreadState *thr);
		778
		779	ALWAYS_INLINE
		780	void FuncEntry(ThreadState *thr, uptr pc) {
		781	DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.sid(), (void *)pc);
		782	if (UNLIKELY(!TryTraceFunc(thr, pc)))
		783	return TraceRestartFuncEntry(thr, pc);
		784	DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
		785	#if !SANITIZER_GO
		786	DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
		787	#else
		788	if (thr->shadow_stack_pos == thr->shadow_stack_end)
		789	GrowShadowStack(thr);
		790	#endif
		791	thr->shadow_stack_pos[0] = pc;
		792	thr->shadow_stack_pos++;
		793	}
		794
		795	ALWAYS_INLINE
		796	void FuncExit(ThreadState *thr) {
		797	DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.sid());
		798	if (UNLIKELY(!TryTraceFunc(thr, 0)))
		799	return TraceRestartFuncExit(thr);
		800	DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
		801	#if !SANITIZER_GO
		802	DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
		803	#endif
		804	thr->shadow_stack_pos--;
		805	}
		806
		807	#if !SANITIZER_GO
		808	extern void (*on_initialize)(void);
		809	extern int (*on_finalize)(int);
		810	#endif
864	} // namespace __tsan	811	} // namespace __tsan
865		812
866	#endif // TSAN_RTL_H	813	#endif // TSAN_RTL_H

lib/tsan/tsan_rtl_aarch64.S+6-31

 @@ -3,28 +3,6 @@
 #include "sanitizer_common/sanitizer_asm.h"
-#if defined(__APPLE__)
-.align  2
-.section  __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
-.long _setjmp$non_lazy_ptr
-_setjmp$non_lazy_ptr:
-.indirect_symbol _setjmp
-.long 0
-.section  __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
-.long __setjmp$non_lazy_ptr
-__setjmp$non_lazy_ptr:
-.indirect_symbol __setjmp
-.long 0
-.section  __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
-.long _sigsetjmp$non_lazy_ptr
-_sigsetjmp$non_lazy_ptr:
-.indirect_symbol _sigsetjmp
-.long 0
-#endif
 #if !defined(__APPLE__)
 .section .text
 #else
 @@ -75,9 +53,8 @@ ASM_SYMBOL_INTERCEPTOR(setjmp):
   ldr     x1, [x1, #:got_lo12:_ZN14__interception11real_setjmpE]
   ldr     x1, [x1]
 #else
-  adrp    x1, _setjmp$non_lazy_ptr@page
+  adrp    x1, _setjmp@GOTPAGE
-  add     x1, x1, _setjmp$non_lazy_ptr@pageoff
+  ldr     x1, [x1, _setjmp@GOTPAGEOFF]
-  ldr     x1, [x1]
 #endif
   br      x1
 @@ -126,9 +103,8 @@ ASM_SYMBOL_INTERCEPTOR(_setjmp):
   ldr     x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE]
   ldr     x1, [x1]
 #else
-  adrp    x1, __setjmp$non_lazy_ptr@page
+  adrp    x1, __setjmp@GOTPAGE
-  add     x1, x1, __setjmp$non_lazy_ptr@pageoff
+  ldr     x1, [x1, __setjmp@GOTPAGEOFF]
-  ldr     x1, [x1]
 #endif
   br      x1
 @@ -179,9 +155,8 @@ ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
   ldr     x2, [x2, #:got_lo12:_ZN14__interception14real_sigsetjmpE]
   ldr     x2, [x2]
 #else
-  adrp    x2, _sigsetjmp$non_lazy_ptr@page
+  adrp    x2, _sigsetjmp@GOTPAGE
-  add     x2, x2, _sigsetjmp$non_lazy_ptr@pageoff
+  ldr     x2, [x2, _sigsetjmp@GOTPAGEOFF]
-  ldr     x2, [x2]
 #endif
   br      x2
   CFI_ENDPROC

lib/tsan/tsan_rtl_access.cpp created+744

...	@@ -0,0 +1,744 @@
1	//===-- tsan_rtl_access.cpp -----------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	// Definitions of memory access and function entry/exit entry points.
12	//===----------------------------------------------------------------------===//
13
14	#include "tsan_rtl.h"
15
16	namespace __tsan {
17
18	ALWAYS_INLINE USED bool TryTraceMemoryAccess(ThreadState* thr, uptr pc,
19	uptr addr, uptr size,
20	AccessType typ) {
21	DCHECK(size == 1 \|\| size == 2 \|\| size == 4 \|\| size == 8);
22	if (!kCollectHistory)
23	return true;
24	EventAccess* ev;
25	if (UNLIKELY(!TraceAcquire(thr, &ev)))
26	return false;
27	u64 size_log = size == 1 ? 0 : size == 2 ? 1 : size == 4 ? 2 : 3;
28	uptr pc_delta = pc - thr->trace_prev_pc + (1 << (EventAccess::kPCBits - 1));
29	thr->trace_prev_pc = pc;
30	if (LIKELY(pc_delta < (1 << EventAccess::kPCBits))) {
31	ev->is_access = 1;
32	ev->is_read = !!(typ & kAccessRead);
33	ev->is_atomic = !!(typ & kAccessAtomic);
34	ev->size_log = size_log;
35	ev->pc_delta = pc_delta;
36	DCHECK_EQ(ev->pc_delta, pc_delta);
37	ev->addr = CompressAddr(addr);
38	TraceRelease(thr, ev);
39	return true;
40	}
41	auto* evex = reinterpret_cast<EventAccessExt*>(ev);
42	evex->is_access = 0;
43	evex->is_func = 0;
44	evex->type = EventType::kAccessExt;
45	evex->is_read = !!(typ & kAccessRead);
46	evex->is_atomic = !!(typ & kAccessAtomic);
47	evex->size_log = size_log;
48	// Note: this is important, see comment in EventAccessExt.
49	evex->_ = 0;
50	evex->addr = CompressAddr(addr);
51	evex->pc = pc;
52	TraceRelease(thr, evex);
53	return true;
54	}
55
56	ALWAYS_INLINE
57	bool TryTraceMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr, uptr size,
58	AccessType typ) {
59	if (!kCollectHistory)
60	return true;
61	EventAccessRange* ev;
62	if (UNLIKELY(!TraceAcquire(thr, &ev)))
63	return false;
64	thr->trace_prev_pc = pc;
65	ev->is_access = 0;
66	ev->is_func = 0;
67	ev->type = EventType::kAccessRange;
68	ev->is_read = !!(typ & kAccessRead);
69	ev->is_free = !!(typ & kAccessFree);
70	ev->size_lo = size;
71	ev->pc = CompressAddr(pc);
72	ev->addr = CompressAddr(addr);
73	ev->size_hi = size >> EventAccessRange::kSizeLoBits;
74	TraceRelease(thr, ev);
75	return true;
76	}
77
78	void TraceMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr, uptr size,
79	AccessType typ) {
80	if (LIKELY(TryTraceMemoryAccessRange(thr, pc, addr, size, typ)))
81	return;
82	TraceSwitchPart(thr);
83	UNUSED bool res = TryTraceMemoryAccessRange(thr, pc, addr, size, typ);
84	DCHECK(res);
85	}
86
87	void TraceFunc(ThreadState* thr, uptr pc) {
88	if (LIKELY(TryTraceFunc(thr, pc)))
89	return;
90	TraceSwitchPart(thr);
91	UNUSED bool res = TryTraceFunc(thr, pc);
92	DCHECK(res);
93	}
94
95	NOINLINE void TraceRestartFuncEntry(ThreadState* thr, uptr pc) {
96	TraceSwitchPart(thr);
97	FuncEntry(thr, pc);
98	}
99
100	NOINLINE void TraceRestartFuncExit(ThreadState* thr) {
101	TraceSwitchPart(thr);
102	FuncExit(thr);
103	}
104
105	void TraceMutexLock(ThreadState* thr, EventType type, uptr pc, uptr addr,
106	StackID stk) {
107	DCHECK(type == EventType::kLock \|\| type == EventType::kRLock);
108	if (!kCollectHistory)
109	return;
110	EventLock ev;
111	ev.is_access = 0;
112	ev.is_func = 0;
113	ev.type = type;
114	ev.pc = CompressAddr(pc);
115	ev.stack_lo = stk;
116	ev.stack_hi = stk >> EventLock::kStackIDLoBits;
117	ev._ = 0;
118	ev.addr = CompressAddr(addr);
119	TraceEvent(thr, ev);
120	}
121
122	void TraceMutexUnlock(ThreadState* thr, uptr addr) {
123	if (!kCollectHistory)
124	return;
125	EventUnlock ev;
126	ev.is_access = 0;
127	ev.is_func = 0;
128	ev.type = EventType::kUnlock;
129	ev._ = 0;
130	ev.addr = CompressAddr(addr);
131	TraceEvent(thr, ev);
132	}
133
134	void TraceTime(ThreadState* thr) {
135	if (!kCollectHistory)
136	return;
137	FastState fast_state = thr->fast_state;
138	EventTime ev;
139	ev.is_access = 0;
140	ev.is_func = 0;
141	ev.type = EventType::kTime;
142	ev.sid = static_cast<u64>(fast_state.sid());
143	ev.epoch = static_cast<u64>(fast_state.epoch());
144	ev._ = 0;
145	TraceEvent(thr, ev);
146	}
147
148	NOINLINE void DoReportRace(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
149	Shadow old,
150	AccessType typ) SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
151	// For the free shadow markers the first element (that contains kFreeSid)
152	// triggers the race, but the second element contains info about the freeing
153	// thread, take it.
154	if (old.sid() == kFreeSid)
155	old = Shadow(LoadShadow(&shadow_mem[1]));
156	// This prevents trapping on this address in future.
157	for (uptr i = 0; i < kShadowCnt; i++)
158	StoreShadow(&shadow_mem[i], i == 0 ? Shadow::kRodata : Shadow::kEmpty);
159	// See the comment in MemoryRangeFreed as to why the slot is locked
160	// for free memory accesses. ReportRace must not be called with
161	// the slot locked because of the fork. But MemoryRangeFreed is not
162	// called during fork because fork sets ignore_reads_and_writes,
163	// so simply unlocking the slot should be fine.
164	if (typ & kAccessSlotLocked)
165	SlotUnlock(thr);
166	ReportRace(thr, shadow_mem, cur, Shadow(old), typ);
167	if (typ & kAccessSlotLocked)
168	SlotLock(thr);
169	}
170
171	#if !TSAN_VECTORIZE
172	ALWAYS_INLINE
173	bool ContainsSameAccess(RawShadow* s, Shadow cur, int unused0, int unused1,
174	AccessType typ) {
175	for (uptr i = 0; i < kShadowCnt; i++) {
176	auto old = LoadShadow(&s[i]);
177	if (!(typ & kAccessRead)) {
178	if (old == cur.raw())
179	return true;
180	continue;
181	}
182	auto masked = static_cast<RawShadow>(static_cast<u32>(old) \|
183	static_cast<u32>(Shadow::kRodata));
184	if (masked == cur.raw())
185	return true;
186	if (!(typ & kAccessNoRodata) && !SANITIZER_GO) {
187	if (old == Shadow::kRodata)
188	return true;
189	}
190	}
191	return false;
192	}
193
194	ALWAYS_INLINE
195	bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
196	int unused0, int unused1, AccessType typ) {
197	bool stored = false;
198	for (uptr idx = 0; idx < kShadowCnt; idx++) {
199	RawShadow* sp = &shadow_mem[idx];
200	Shadow old(LoadShadow(sp));
201	if (LIKELY(old.raw() == Shadow::kEmpty)) {
202	if (!(typ & kAccessCheckOnly) && !stored)
203	StoreShadow(sp, cur.raw());
204	return false;
205	}
206	if (LIKELY(!(cur.access() & old.access())))
207	continue;
208	if (LIKELY(cur.sid() == old.sid())) {
209	if (!(typ & kAccessCheckOnly) &&
210	LIKELY(cur.access() == old.access() && old.IsRWWeakerOrEqual(typ))) {
211	StoreShadow(sp, cur.raw());
212	stored = true;
213	}
214	continue;
215	}
216	if (LIKELY(old.IsBothReadsOrAtomic(typ)))
217	continue;
218	if (LIKELY(thr->clock.Get(old.sid()) >= old.epoch()))
219	continue;
220	DoReportRace(thr, shadow_mem, cur, old, typ);
221	return true;
222	}
223	// We did not find any races and had already stored
224	// the current access info, so we are done.
225	if (LIKELY(stored))
226	return false;
227	// Choose a random candidate slot and replace it.
228	uptr index =
229	atomic_load_relaxed(&thr->trace_pos) / sizeof(Event) % kShadowCnt;
230	StoreShadow(&shadow_mem[index], cur.raw());
231	return false;
232	}
233
234	# define LOAD_CURRENT_SHADOW(cur, shadow_mem) UNUSED int access = 0, shadow = 0
235
236	#else /* !TSAN_VECTORIZE */
237
238	ALWAYS_INLINE
239	bool ContainsSameAccess(RawShadow* unused0, Shadow unused1, m128 shadow,
240	m128 access, AccessType typ) {
241	// Note: we could check if there is a larger access of the same type,
242	// e.g. we just allocated/memset-ed a block (so it contains 8 byte writes)
243	// and now do smaller reads/writes, these can also be considered as "same
244	// access". However, it will make the check more expensive, so it's unclear
245	// if it's worth it. But this would conserve trace space, so it's useful
246	// besides potential speed up.
247	if (!(typ & kAccessRead)) {
248	const m128 same = _mm_cmpeq_epi32(shadow, access);
249	return _mm_movemask_epi8(same);
250	}
251	// For reads we need to reset read bit in the shadow,
252	// because we need to match read with both reads and writes.
253	// Shadow::kRodata has only read bit set, so it does what we want.
254	// We also abuse it for rodata check to save few cycles
255	// since we already loaded Shadow::kRodata into a register.
256	// Reads from rodata can't race.
257	// Measurements show that they can be 10-20% of all memory accesses.
258	// Shadow::kRodata has epoch 0 which cannot appear in shadow normally
259	// (thread epochs start from 1). So the same read bit mask
260	// serves as rodata indicator.
261	const m128 read_mask = _mm_set1_epi32(static_cast<u32>(Shadow::kRodata));
262	const m128 masked_shadow = _mm_or_si128(shadow, read_mask);
263	m128 same = _mm_cmpeq_epi32(masked_shadow, access);
264	// Range memory accesses check Shadow::kRodata before calling this,
265	// Shadow::kRodatas is not possible for free memory access
266	// and Go does not use Shadow::kRodata.
267	if (!(typ & kAccessNoRodata) && !SANITIZER_GO) {
268	const m128 ro = _mm_cmpeq_epi32(shadow, read_mask);
269	same = _mm_or_si128(ro, same);
270	}
271	return _mm_movemask_epi8(same);
272	}
273
274	NOINLINE void DoReportRaceV(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
275	u32 race_mask, m128 shadow, AccessType typ) {
276	// race_mask points which of the shadow elements raced with the current
277	// access. Extract that element.
278	CHECK_NE(race_mask, 0);
279	u32 old;
280	// Note: _mm_extract_epi32 index must be a constant value.
281	switch (__builtin_ffs(race_mask) / 4) {
282	case 0:
283	old = _mm_extract_epi32(shadow, 0);
284	break;
285	case 1:
286	old = _mm_extract_epi32(shadow, 1);
287	break;
288	case 2:
289	old = _mm_extract_epi32(shadow, 2);
290	break;
291	case 3:
292	old = _mm_extract_epi32(shadow, 3);
293	break;
294	}
295	Shadow prev(static_cast<RawShadow>(old));
296	// For the free shadow markers the first element (that contains kFreeSid)
297	// triggers the race, but the second element contains info about the freeing
298	// thread, take it.
299	if (prev.sid() == kFreeSid)
300	prev = Shadow(static_cast<RawShadow>(_mm_extract_epi32(shadow, 1)));
301	DoReportRace(thr, shadow_mem, cur, prev, typ);
302	}
303
304	ALWAYS_INLINE
305	bool CheckRaces(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
306	m128 shadow, m128 access, AccessType typ) {
307	// Note: empty/zero slots don't intersect with any access.
308	const m128 zero = _mm_setzero_si128();
309	const m128 mask_access = _mm_set1_epi32(0x000000ff);
310	const m128 mask_sid = _mm_set1_epi32(0x0000ff00);
311	const m128 mask_read_atomic = _mm_set1_epi32(0xc0000000);
312	const m128 access_and = _mm_and_si128(access, shadow);
313	const m128 access_xor = _mm_xor_si128(access, shadow);
314	const m128 intersect = _mm_and_si128(access_and, mask_access);
315	const m128 not_intersect = _mm_cmpeq_epi32(intersect, zero);
316	const m128 not_same_sid = _mm_and_si128(access_xor, mask_sid);
317	const m128 same_sid = _mm_cmpeq_epi32(not_same_sid, zero);
318	const m128 both_read_or_atomic = _mm_and_si128(access_and, mask_read_atomic);
319	const m128 no_race =
320	_mm_or_si128(_mm_or_si128(not_intersect, same_sid), both_read_or_atomic);
321	const int race_mask = _mm_movemask_epi8(_mm_cmpeq_epi32(no_race, zero));
322	if (UNLIKELY(race_mask))
323	goto SHARED;
324
325	STORE : {
326	if (typ & kAccessCheckOnly)
327	return false;
328	// We could also replace different sid's if access is the same,
329	// rw weaker and happens before. However, just checking access below
330	// is not enough because we also need to check that !both_read_or_atomic
331	// (reads from different sids can be concurrent).
332	// Theoretically we could replace smaller accesses with larger accesses,
333	// but it's unclear if it's worth doing.
334	const m128 mask_access_sid = _mm_set1_epi32(0x0000ffff);
335	const m128 not_same_sid_access = _mm_and_si128(access_xor, mask_access_sid);
336	const m128 same_sid_access = _mm_cmpeq_epi32(not_same_sid_access, zero);
337	const m128 access_read_atomic =
338	_mm_set1_epi32((typ & (kAccessRead \| kAccessAtomic)) << 30);
339	const m128 rw_weaker =
340	_mm_cmpeq_epi32(_mm_max_epu32(shadow, access_read_atomic), shadow);
341	const m128 rewrite = _mm_and_si128(same_sid_access, rw_weaker);
342	const int rewrite_mask = _mm_movemask_epi8(rewrite);
343	int index = __builtin_ffs(rewrite_mask);
344	if (UNLIKELY(index == 0)) {
345	const m128 empty = _mm_cmpeq_epi32(shadow, zero);
346	const int empty_mask = _mm_movemask_epi8(empty);
347	index = __builtin_ffs(empty_mask);
348	if (UNLIKELY(index == 0))
349	index = (atomic_load_relaxed(&thr->trace_pos) / 2) % 16;
350	}
351	StoreShadow(&shadow_mem[index / 4], cur.raw());
352	// We could zero other slots determined by rewrite_mask.
353	// That would help other threads to evict better slots,
354	// but it's unclear if it's worth it.
355	return false;
356	}
357
358	SHARED:
359	m128 thread_epochs = _mm_set1_epi32(0x7fffffff);
360	// Need to unwind this because _mm_extract_epi8/_mm_insert_epi32
361	// indexes must be constants.
362	# define LOAD_EPOCH(idx) \
363	if (LIKELY(race_mask & (1 << (idx * 4)))) { \
364	u8 sid = _mm_extract_epi8(shadow, idx * 4 + 1); \
365	u16 epoch = static_cast<u16>(thr->clock.Get(static_cast<Sid>(sid))); \
366	thread_epochs = _mm_insert_epi32(thread_epochs, u32(epoch) << 16, idx); \
367	}
368	LOAD_EPOCH(0);
369	LOAD_EPOCH(1);
370	LOAD_EPOCH(2);
371	LOAD_EPOCH(3);
372	# undef LOAD_EPOCH
373	const m128 mask_epoch = _mm_set1_epi32(0x3fff0000);
374	const m128 shadow_epochs = _mm_and_si128(shadow, mask_epoch);
375	const m128 concurrent = _mm_cmplt_epi32(thread_epochs, shadow_epochs);
376	const int concurrent_mask = _mm_movemask_epi8(concurrent);
377	if (LIKELY(concurrent_mask == 0))
378	goto STORE;
379
380	DoReportRaceV(thr, shadow_mem, cur, concurrent_mask, shadow, typ);
381	return true;
382	}
383
384	# define LOAD_CURRENT_SHADOW(cur, shadow_mem) \
385	const m128 access = _mm_set1_epi32(static_cast<u32>((cur).raw())); \
386	const m128 shadow = _mm_load_si128(reinterpret_cast<m128*>(shadow_mem))
387	#endif
388
389	char* DumpShadow(char* buf, RawShadow raw) {
390	if (raw == Shadow::kEmpty) {
391	internal_snprintf(buf, 64, "0");
392	return buf;
393	}
394	Shadow s(raw);
395	AccessType typ;
396	s.GetAccess(nullptr, nullptr, &typ);
397	internal_snprintf(buf, 64, "{tid=%u@%u access=0x%x typ=%x}",
398	static_cast<u32>(s.sid()), static_cast<u32>(s.epoch()),
399	s.access(), static_cast<u32>(typ));
400	return buf;
401	}
402
403	// TryTrace* and TraceRestart* functions allow to turn memory access and func
404	// entry/exit callbacks into leaf functions with all associated performance
405	// benefits. These hottest callbacks do only 2 slow path calls: report a race
406	// and trace part switching. Race reporting is easy to turn into a tail call, we
407	// just always return from the runtime after reporting a race. But trace part
408	// switching is harder because it needs to be in the middle of callbacks. To
409	// turn it into a tail call we immidiately return after TraceRestart* functions,
410	// but TraceRestart* functions themselves recurse into the callback after
411	// switching trace part. As the result the hottest callbacks contain only tail
412	// calls, which effectively makes them leaf functions (can use all registers,
413	// no frame setup, etc).
414	NOINLINE void TraceRestartMemoryAccess(ThreadState* thr, uptr pc, uptr addr,
415	uptr size, AccessType typ) {
416	TraceSwitchPart(thr);
417	MemoryAccess(thr, pc, addr, size, typ);
418	}
419
420	ALWAYS_INLINE USED void MemoryAccess(ThreadState* thr, uptr pc, uptr addr,
421	uptr size, AccessType typ) {
422	RawShadow* shadow_mem = MemToShadow(addr);
423	UNUSED char memBuf[4][64];
424	DPrintf2("#%d: Access: %d@%d %p/%zd typ=0x%x {%s, %s, %s, %s}\n", thr->tid,
425	static_cast<int>(thr->fast_state.sid()),
426	static_cast<int>(thr->fast_state.epoch()), (void*)addr, size,
427	static_cast<int>(typ), DumpShadow(memBuf[0], shadow_mem[0]),
428	DumpShadow(memBuf[1], shadow_mem[1]),
429	DumpShadow(memBuf[2], shadow_mem[2]),
430	DumpShadow(memBuf[3], shadow_mem[3]));
431
432	FastState fast_state = thr->fast_state;
433	Shadow cur(fast_state, addr, size, typ);
434
435	LOAD_CURRENT_SHADOW(cur, shadow_mem);
436	if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
437	return;
438	if (UNLIKELY(fast_state.GetIgnoreBit()))
439	return;
440	if (!TryTraceMemoryAccess(thr, pc, addr, size, typ))
441	return TraceRestartMemoryAccess(thr, pc, addr, size, typ);
442	CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
443	}
444
445	void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr, AccessType typ);
446
447	NOINLINE
448	void RestartMemoryAccess16(ThreadState* thr, uptr pc, uptr addr,
449	AccessType typ) {
450	TraceSwitchPart(thr);
451	MemoryAccess16(thr, pc, addr, typ);
452	}
453
454	ALWAYS_INLINE USED void MemoryAccess16(ThreadState* thr, uptr pc, uptr addr,
455	AccessType typ) {
456	const uptr size = 16;
457	FastState fast_state = thr->fast_state;
458	if (UNLIKELY(fast_state.GetIgnoreBit()))
459	return;
460	Shadow cur(fast_state, 0, 8, typ);
461	RawShadow* shadow_mem = MemToShadow(addr);
462	bool traced = false;
463	{
464	LOAD_CURRENT_SHADOW(cur, shadow_mem);
465	if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
466	goto SECOND;
467	if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
468	return RestartMemoryAccess16(thr, pc, addr, typ);
469	traced = true;
470	if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
471	return;
472	}
473	SECOND:
474	shadow_mem += kShadowCnt;
475	LOAD_CURRENT_SHADOW(cur, shadow_mem);
476	if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
477	return;
478	if (!traced && !TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
479	return RestartMemoryAccess16(thr, pc, addr, typ);
480	CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
481	}
482
483	NOINLINE
484	void RestartUnalignedMemoryAccess(ThreadState* thr, uptr pc, uptr addr,
485	uptr size, AccessType typ) {
486	TraceSwitchPart(thr);
487	UnalignedMemoryAccess(thr, pc, addr, size, typ);
488	}
489
490	ALWAYS_INLINE USED void UnalignedMemoryAccess(ThreadState* thr, uptr pc,
491	uptr addr, uptr size,
492	AccessType typ) {
493	DCHECK_LE(size, 8);
494	FastState fast_state = thr->fast_state;
495	if (UNLIKELY(fast_state.GetIgnoreBit()))
496	return;
497	RawShadow* shadow_mem = MemToShadow(addr);
498	bool traced = false;
499	uptr size1 = Min<uptr>(size, RoundUp(addr + 1, kShadowCell) - addr);
500	{
501	Shadow cur(fast_state, addr, size1, typ);
502	LOAD_CURRENT_SHADOW(cur, shadow_mem);
503	if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
504	goto SECOND;
505	if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
506	return RestartUnalignedMemoryAccess(thr, pc, addr, size, typ);
507	traced = true;
508	if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
509	return;
510	}
511	SECOND:
512	uptr size2 = size - size1;
513	if (LIKELY(size2 == 0))
514	return;
515	shadow_mem += kShadowCnt;
516	Shadow cur(fast_state, 0, size2, typ);
517	LOAD_CURRENT_SHADOW(cur, shadow_mem);
518	if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
519	return;
520	if (!traced && !TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
521	return RestartUnalignedMemoryAccess(thr, pc, addr, size, typ);
522	CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
523	}
524
525	void ShadowSet(RawShadow* p, RawShadow* end, RawShadow v) {
526	DCHECK_LE(p, end);
527	DCHECK(IsShadowMem(p));
528	DCHECK(IsShadowMem(end));
529	UNUSED const uptr kAlign = kShadowCnt * kShadowSize;
530	DCHECK_EQ(reinterpret_cast<uptr>(p) % kAlign, 0);
531	DCHECK_EQ(reinterpret_cast<uptr>(end) % kAlign, 0);
532	#if !TSAN_VECTORIZE
533	for (; p < end; p += kShadowCnt) {
534	p[0] = v;
535	for (uptr i = 1; i < kShadowCnt; i++) p[i] = Shadow::kEmpty;
536	}
537	#else
538	m128 vv = _mm_setr_epi32(
539	static_cast<u32>(v), static_cast<u32>(Shadow::kEmpty),
540	static_cast<u32>(Shadow::kEmpty), static_cast<u32>(Shadow::kEmpty));
541	m128* vp = reinterpret_cast<m128*>(p);
542	m128* vend = reinterpret_cast<m128*>(end);
543	for (; vp < vend; vp++) _mm_store_si128(vp, vv);
544	#endif
545	}
546
547	static void MemoryRangeSet(uptr addr, uptr size, RawShadow val) {
548	if (size == 0)
549	return;
550	DCHECK_EQ(addr % kShadowCell, 0);
551	DCHECK_EQ(size % kShadowCell, 0);
552	// If a user passes some insane arguments (memset(0)),
553	// let it just crash as usual.
554	if (!IsAppMem(addr) \|\| !IsAppMem(addr + size - 1))
555	return;
556	RawShadow* begin = MemToShadow(addr);
557	RawShadow* end = begin + size / kShadowCell * kShadowCnt;
558	// Don't want to touch lots of shadow memory.
559	// If a program maps 10MB stack, there is no need reset the whole range.
560	// UnmapOrDie/MmapFixedNoReserve does not work on Windows.
561	if (SANITIZER_WINDOWS \|\|
562	size <= common_flags()->clear_shadow_mmap_threshold) {
563	ShadowSet(begin, end, val);
564	return;
565	}
566	// The region is big, reset only beginning and end.
567	const uptr kPageSize = GetPageSizeCached();
568	// Set at least first kPageSize/2 to page boundary.
569	RawShadow* mid1 =
570	Min(end, reinterpret_cast<RawShadow*>(RoundUp(
571	reinterpret_cast<uptr>(begin) + kPageSize / 2, kPageSize)));
572	ShadowSet(begin, mid1, val);
573	// Reset middle part.
574	RawShadow* mid2 = RoundDown(end, kPageSize);
575	if (mid2 > mid1) {
576	if (!MmapFixedSuperNoReserve((uptr)mid1, (uptr)mid2 - (uptr)mid1))
577	Die();
578	}
579	// Set the ending.
580	ShadowSet(mid2, end, val);
581	}
582
583	void MemoryResetRange(ThreadState* thr, uptr pc, uptr addr, uptr size) {
584	uptr addr1 = RoundDown(addr, kShadowCell);
585	uptr size1 = RoundUp(size + addr - addr1, kShadowCell);
586	MemoryRangeSet(addr1, size1, Shadow::kEmpty);
587	}
588
589	void MemoryRangeFreed(ThreadState* thr, uptr pc, uptr addr, uptr size) {
590	// Callers must lock the slot to ensure synchronization with the reset.
591	// The problem with "freed" memory is that it's not "monotonic"
592	// with respect to bug detection: freed memory is bad to access,
593	// but then if the heap block is reallocated later, it's good to access.
594	// As the result a garbage "freed" shadow can lead to a false positive
595	// if it happens to match a real free in the thread trace,
596	// but the heap block was reallocated before the current memory access,
597	// so it's still good to access. It's not the case with data races.
598	DCHECK(thr->slot_locked);
599	DCHECK_EQ(addr % kShadowCell, 0);
600	size = RoundUp(size, kShadowCell);
601	// Processing more than 1k (2k of shadow) is expensive,
602	// can cause excessive memory consumption (user does not necessary touch
603	// the whole range) and most likely unnecessary.
604	size = Min<uptr>(size, 1024);
605	const AccessType typ = kAccessWrite \| kAccessFree \| kAccessSlotLocked \|
606	kAccessCheckOnly \| kAccessNoRodata;
607	TraceMemoryAccessRange(thr, pc, addr, size, typ);
608	RawShadow* shadow_mem = MemToShadow(addr);
609	Shadow cur(thr->fast_state, 0, kShadowCell, typ);
610	#if TSAN_VECTORIZE
611	const m128 access = _mm_set1_epi32(static_cast<u32>(cur.raw()));
612	const m128 freed = _mm_setr_epi32(
613	static_cast<u32>(Shadow::FreedMarker()),
614	static_cast<u32>(Shadow::FreedInfo(cur.sid(), cur.epoch())), 0, 0);
615	for (; size; size -= kShadowCell, shadow_mem += kShadowCnt) {
616	const m128 shadow = _mm_load_si128((m128*)shadow_mem);
617	if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, shadow, access, typ)))
618	return;
619	_mm_store_si128((m128*)shadow_mem, freed);
620	}
621	#else
622	for (; size; size -= kShadowCell, shadow_mem += kShadowCnt) {
623	if (UNLIKELY(CheckRaces(thr, shadow_mem, cur, 0, 0, typ)))
624	return;
625	StoreShadow(&shadow_mem[0], Shadow::FreedMarker());
626	StoreShadow(&shadow_mem[1], Shadow::FreedInfo(cur.sid(), cur.epoch()));
627	StoreShadow(&shadow_mem[2], Shadow::kEmpty);
628	StoreShadow(&shadow_mem[3], Shadow::kEmpty);
629	}
630	#endif
631	}
632
633	void MemoryRangeImitateWrite(ThreadState* thr, uptr pc, uptr addr, uptr size) {
634	DCHECK_EQ(addr % kShadowCell, 0);
635	size = RoundUp(size, kShadowCell);
636	TraceMemoryAccessRange(thr, pc, addr, size, kAccessWrite);
637	Shadow cur(thr->fast_state, 0, 8, kAccessWrite);
638	MemoryRangeSet(addr, size, cur.raw());
639	}
640
641	void MemoryRangeImitateWriteOrResetRange(ThreadState* thr, uptr pc, uptr addr,
642	uptr size) {
643	if (thr->ignore_reads_and_writes == 0)
644	MemoryRangeImitateWrite(thr, pc, addr, size);
645	else
646	MemoryResetRange(thr, pc, addr, size);
647	}
648
649	ALWAYS_INLINE
650	bool MemoryAccessRangeOne(ThreadState* thr, RawShadow* shadow_mem, Shadow cur,
651	AccessType typ) {
652	LOAD_CURRENT_SHADOW(cur, shadow_mem);
653	if (LIKELY(ContainsSameAccess(shadow_mem, cur, shadow, access, typ)))
654	return false;
655	return CheckRaces(thr, shadow_mem, cur, shadow, access, typ);
656	}
657
658	template <bool is_read>
659	NOINLINE void RestartMemoryAccessRange(ThreadState* thr, uptr pc, uptr addr,
660	uptr size) {
661	TraceSwitchPart(thr);
662	MemoryAccessRangeT<is_read>(thr, pc, addr, size);
663	}
664
665	template <bool is_read>
666	void MemoryAccessRangeT(ThreadState* thr, uptr pc, uptr addr, uptr size) {
667	const AccessType typ =
668	(is_read ? kAccessRead : kAccessWrite) \| kAccessNoRodata;
669	RawShadow* shadow_mem = MemToShadow(addr);
670	DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_read=%d\n", thr->tid,
671	(void)pc, (void)addr, (int)size, is_read);
672
673	#if SANITIZER_DEBUG
674	if (!IsAppMem(addr)) {
675	Printf("Access to non app mem %zx\n", addr);
676	DCHECK(IsAppMem(addr));
677	}
678	if (!IsAppMem(addr + size - 1)) {
679	Printf("Access to non app mem %zx\n", addr + size - 1);
680	DCHECK(IsAppMem(addr + size - 1));
681	}
682	if (!IsShadowMem(shadow_mem)) {
683	Printf("Bad shadow addr %p (%zx)\n", static_cast<void*>(shadow_mem), addr);
684	DCHECK(IsShadowMem(shadow_mem));
685	}
686	if (!IsShadowMem(shadow_mem + size * kShadowCnt - 1)) {
687	Printf("Bad shadow addr %p (%zx)\n",
688	static_cast<void>(shadow_mem + size kShadowCnt - 1),
689	addr + size - 1);
690	DCHECK(IsShadowMem(shadow_mem + size * kShadowCnt - 1));
691	}
692	#endif
693
694	// Access to .rodata section, no races here.
695	// Measurements show that it can be 10-20% of all memory accesses.
696	// Check here once to not check for every access separately.
697	// Note: we could (and should) do this only for the is_read case
698	// (writes shouldn't go to .rodata). But it happens in Chromium tests:
699	// https://bugs.chromium.org/p/chromium/issues/detail?id=1275581#c19
700	// Details are unknown since it happens only on CI machines.
701	if (*shadow_mem == Shadow::kRodata)
702	return;
703
704	FastState fast_state = thr->fast_state;
705	if (UNLIKELY(fast_state.GetIgnoreBit()))
706	return;
707
708	if (!TryTraceMemoryAccessRange(thr, pc, addr, size, typ))
709	return RestartMemoryAccessRange<is_read>(thr, pc, addr, size);
710
711	if (UNLIKELY(addr % kShadowCell)) {
712	// Handle unaligned beginning, if any.
713	uptr size1 = Min(size, RoundUp(addr, kShadowCell) - addr);
714	size -= size1;
715	Shadow cur(fast_state, addr, size1, typ);
716	if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ)))
717	return;
718	shadow_mem += kShadowCnt;
719	}
720	// Handle middle part, if any.
721	Shadow cur(fast_state, 0, kShadowCell, typ);
722	for (; size >= kShadowCell; size -= kShadowCell, shadow_mem += kShadowCnt) {
723	if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ)))
724	return;
725	}
726	// Handle ending, if any.
727	if (UNLIKELY(size)) {
728	Shadow cur(fast_state, 0, size, typ);
729	if (UNLIKELY(MemoryAccessRangeOne(thr, shadow_mem, cur, typ)))
730	return;
731	}
732	}
733
734	template void MemoryAccessRangeT<true>(ThreadState* thr, uptr pc, uptr addr,
735	uptr size);
736	template void MemoryAccessRangeT<false>(ThreadState* thr, uptr pc, uptr addr,
737	uptr size);
738
739	} // namespace __tsan
740
741	#if !SANITIZER_GO
742	// Must be included in this file to make sure everything is inlined.
743	# include "tsan_interface.inc"
744	#endif

lib/tsan/tsan_rtl_amd64.S+4-160

...	@@ -9,166 +9,6 @@	...	@@ -9,166 +9,6 @@
9	.section __TEXT,__text	9	.section __TEXT,__text
10	#endif	10	#endif
11		11
12	ASM_HIDDEN(__tsan_trace_switch)
13	.globl ASM_SYMBOL(__tsan_trace_switch_thunk)
14	ASM_SYMBOL(__tsan_trace_switch_thunk):
15	CFI_STARTPROC
16	# Save scratch registers.
17	push %rax
18	CFI_ADJUST_CFA_OFFSET(8)
19	CFI_REL_OFFSET(%rax, 0)
20	push %rcx
21	CFI_ADJUST_CFA_OFFSET(8)
22	CFI_REL_OFFSET(%rcx, 0)
23	push %rdx
24	CFI_ADJUST_CFA_OFFSET(8)
25	CFI_REL_OFFSET(%rdx, 0)
26	push %rsi
27	CFI_ADJUST_CFA_OFFSET(8)
28	CFI_REL_OFFSET(%rsi, 0)
29	push %rdi
30	CFI_ADJUST_CFA_OFFSET(8)
31	CFI_REL_OFFSET(%rdi, 0)
32	push %r8
33	CFI_ADJUST_CFA_OFFSET(8)
34	CFI_REL_OFFSET(%r8, 0)
35	push %r9
36	CFI_ADJUST_CFA_OFFSET(8)
37	CFI_REL_OFFSET(%r9, 0)
38	push %r10
39	CFI_ADJUST_CFA_OFFSET(8)
40	CFI_REL_OFFSET(%r10, 0)
41	push %r11
42	CFI_ADJUST_CFA_OFFSET(8)
43	CFI_REL_OFFSET(%r11, 0)
44	# Align stack frame.
45	push %rbx # non-scratch
46	CFI_ADJUST_CFA_OFFSET(8)
47	CFI_REL_OFFSET(%rbx, 0)
48	mov %rsp, %rbx # save current rsp
49	CFI_DEF_CFA_REGISTER(%rbx)
50	shr $4, %rsp # clear 4 lsb, align to 16
51	shl $4, %rsp
52
53	call ASM_SYMBOL(__tsan_trace_switch)
54
55	# Unalign stack frame back.
56	mov %rbx, %rsp # restore the original rsp
57	CFI_DEF_CFA_REGISTER(%rsp)
58	pop %rbx
59	CFI_ADJUST_CFA_OFFSET(-8)
60	# Restore scratch registers.
61	pop %r11
62	CFI_ADJUST_CFA_OFFSET(-8)
63	pop %r10
64	CFI_ADJUST_CFA_OFFSET(-8)
65	pop %r9
66	CFI_ADJUST_CFA_OFFSET(-8)
67	pop %r8
68	CFI_ADJUST_CFA_OFFSET(-8)
69	pop %rdi
70	CFI_ADJUST_CFA_OFFSET(-8)
71	pop %rsi
72	CFI_ADJUST_CFA_OFFSET(-8)
73	pop %rdx
74	CFI_ADJUST_CFA_OFFSET(-8)
75	pop %rcx
76	CFI_ADJUST_CFA_OFFSET(-8)
77	pop %rax
78	CFI_ADJUST_CFA_OFFSET(-8)
79	CFI_RESTORE(%rax)
80	CFI_RESTORE(%rbx)
81	CFI_RESTORE(%rcx)
82	CFI_RESTORE(%rdx)
83	CFI_RESTORE(%rsi)
84	CFI_RESTORE(%rdi)
85	CFI_RESTORE(%r8)
86	CFI_RESTORE(%r9)
87	CFI_RESTORE(%r10)
88	CFI_RESTORE(%r11)
89	ret
90	CFI_ENDPROC
91
92	ASM_HIDDEN(__tsan_report_race)
93	.globl ASM_SYMBOL(__tsan_report_race_thunk)
94	ASM_SYMBOL(__tsan_report_race_thunk):
95	CFI_STARTPROC
96	# Save scratch registers.
97	push %rax
98	CFI_ADJUST_CFA_OFFSET(8)
99	CFI_REL_OFFSET(%rax, 0)
100	push %rcx
101	CFI_ADJUST_CFA_OFFSET(8)
102	CFI_REL_OFFSET(%rcx, 0)
103	push %rdx
104	CFI_ADJUST_CFA_OFFSET(8)
105	CFI_REL_OFFSET(%rdx, 0)
106	push %rsi
107	CFI_ADJUST_CFA_OFFSET(8)
108	CFI_REL_OFFSET(%rsi, 0)
109	push %rdi
110	CFI_ADJUST_CFA_OFFSET(8)
111	CFI_REL_OFFSET(%rdi, 0)
112	push %r8
113	CFI_ADJUST_CFA_OFFSET(8)
114	CFI_REL_OFFSET(%r8, 0)
115	push %r9
116	CFI_ADJUST_CFA_OFFSET(8)
117	CFI_REL_OFFSET(%r9, 0)
118	push %r10
119	CFI_ADJUST_CFA_OFFSET(8)
120	CFI_REL_OFFSET(%r10, 0)
121	push %r11
122	CFI_ADJUST_CFA_OFFSET(8)
123	CFI_REL_OFFSET(%r11, 0)
124	# Align stack frame.
125	push %rbx # non-scratch
126	CFI_ADJUST_CFA_OFFSET(8)
127	CFI_REL_OFFSET(%rbx, 0)
128	mov %rsp, %rbx # save current rsp
129	CFI_DEF_CFA_REGISTER(%rbx)
130	shr $4, %rsp # clear 4 lsb, align to 16
131	shl $4, %rsp
132
133	call ASM_SYMBOL(__tsan_report_race)
134
135	# Unalign stack frame back.
136	mov %rbx, %rsp # restore the original rsp
137	CFI_DEF_CFA_REGISTER(%rsp)
138	pop %rbx
139	CFI_ADJUST_CFA_OFFSET(-8)
140	# Restore scratch registers.
141	pop %r11
142	CFI_ADJUST_CFA_OFFSET(-8)
143	pop %r10
144	CFI_ADJUST_CFA_OFFSET(-8)
145	pop %r9
146	CFI_ADJUST_CFA_OFFSET(-8)
147	pop %r8
148	CFI_ADJUST_CFA_OFFSET(-8)
149	pop %rdi
150	CFI_ADJUST_CFA_OFFSET(-8)
151	pop %rsi
152	CFI_ADJUST_CFA_OFFSET(-8)
153	pop %rdx
154	CFI_ADJUST_CFA_OFFSET(-8)
155	pop %rcx
156	CFI_ADJUST_CFA_OFFSET(-8)
157	pop %rax
158	CFI_ADJUST_CFA_OFFSET(-8)
159	CFI_RESTORE(%rax)
160	CFI_RESTORE(%rbx)
161	CFI_RESTORE(%rcx)
162	CFI_RESTORE(%rdx)
163	CFI_RESTORE(%rsi)
164	CFI_RESTORE(%rdi)
165	CFI_RESTORE(%r8)
166	CFI_RESTORE(%r9)
167	CFI_RESTORE(%r10)
168	CFI_RESTORE(%r11)
169	ret
170	CFI_ENDPROC
171
172	ASM_HIDDEN(__tsan_setjmp)	12	ASM_HIDDEN(__tsan_setjmp)
173	#if defined(__NetBSD__)	13	#if defined(__NetBSD__)
174	.comm _ZN14__interception15real___setjmp14E,8,8	14	.comm _ZN14__interception15real___setjmp14E,8,8
...	@@ -185,6 +25,7 @@ ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))	...	@@ -185,6 +25,7 @@ ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
185	ASM_SYMBOL_INTERCEPTOR(setjmp):	25	ASM_SYMBOL_INTERCEPTOR(setjmp):
186	#endif	26	#endif
187	CFI_STARTPROC	27	CFI_STARTPROC
		28	_CET_ENDBR
188	// save env parameter	29	// save env parameter
189	push %rdi	30	push %rdi
190	CFI_ADJUST_CFA_OFFSET(8)	31	CFI_ADJUST_CFA_OFFSET(8)
...	@@ -226,6 +67,7 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))	...	@@ -226,6 +67,7 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
226	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))	67	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
227	ASM_SYMBOL_INTERCEPTOR(_setjmp):	68	ASM_SYMBOL_INTERCEPTOR(_setjmp):
228	CFI_STARTPROC	69	CFI_STARTPROC
		70	_CET_ENDBR
229	// save env parameter	71	// save env parameter
230	push %rdi	72	push %rdi
231	CFI_ADJUST_CFA_OFFSET(8)	73	CFI_ADJUST_CFA_OFFSET(8)
...	@@ -267,6 +109,7 @@ ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))	...	@@ -267,6 +109,7 @@ ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
267	ASM_SYMBOL_INTERCEPTOR(sigsetjmp):	109	ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
268	#endif	110	#endif
269	CFI_STARTPROC	111	CFI_STARTPROC
		112	_CET_ENDBR
270	// save env parameter	113	// save env parameter
271	push %rdi	114	push %rdi
272	CFI_ADJUST_CFA_OFFSET(8)	115	CFI_ADJUST_CFA_OFFSET(8)
...	@@ -323,6 +166,7 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))	...	@@ -323,6 +166,7 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
323	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))	166	ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
324	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):	167	ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
325	CFI_STARTPROC	168	CFI_STARTPROC
		169	_CET_ENDBR
326	// save env parameter	170	// save env parameter
327	push %rdi	171	push %rdi
328	CFI_ADJUST_CFA_OFFSET(8)	172	CFI_ADJUST_CFA_OFFSET(8)

lib/tsan/tsan_rtl_mutex.cpp+369-338

...	@@ -23,6 +23,8 @@	...	@@ -23,6 +23,8 @@
23	namespace __tsan {	23	namespace __tsan {
24		24
25	void ReportDeadlock(ThreadState thr, uptr pc, DDReport r);	25	void ReportDeadlock(ThreadState thr, uptr pc, DDReport r);
		26	void ReportDestroyLocked(ThreadState *thr, uptr pc, uptr addr,
		27	FastState last_lock, StackID creation_stack_id);
26		28
27	struct Callback final : public DDCallback {	29	struct Callback final : public DDCallback {
28	ThreadState *thr;	30	ThreadState *thr;
...	@@ -35,27 +37,27 @@ struct Callback final : public DDCallback {	...	@@ -35,27 +37,27 @@ struct Callback final : public DDCallback {
35	DDCallback::lt = thr->dd_lt;	37	DDCallback::lt = thr->dd_lt;
36	}	38	}
37		39
38	u32 Unwind() override { return CurrentStackId(thr, pc); }	40	StackID Unwind() override { return CurrentStackId(thr, pc); }
39	int UniqueTid() override { return thr->unique_id; }	41	int UniqueTid() override { return thr->tid; }
40	};	42	};
41		43
42	void DDMutexInit(ThreadState thr, uptr pc, SyncVar s) {	44	void DDMutexInit(ThreadState thr, uptr pc, SyncVar s) {
43	Callback cb(thr, pc);	45	Callback cb(thr, pc);
44	ctx->dd->MutexInit(&cb, &s->dd);	46	ctx->dd->MutexInit(&cb, &s->dd);
45	s->dd.ctx = s->GetId();	47	s->dd.ctx = s->addr;
46	}	48	}
47		49
48	static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,	50	static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,
49	uptr addr, u64 mid) {	51	uptr addr, StackID creation_stack_id) {
50	// In Go, these misuses are either impossible, or detected by std lib,	52	// In Go, these misuses are either impossible, or detected by std lib,
51	// or false positives (e.g. unlock in a different thread).	53	// or false positives (e.g. unlock in a different thread).
52	if (SANITIZER_GO)	54	if (SANITIZER_GO)
53	return;	55	return;
54	if (!ShouldReport(thr, typ))	56	if (!ShouldReport(thr, typ))
55	return;	57	return;
56	ThreadRegistryLock l(ctx->thread_registry);	58	ThreadRegistryLock l(&ctx->thread_registry);
57	ScopedReport rep(typ);	59	ScopedReport rep(typ);
58	rep.AddMutex(mid);	60	rep.AddMutex(addr, creation_stack_id);
59	VarSizeStackTrace trace;	61	VarSizeStackTrace trace;
60	ObtainCurrentStack(thr, pc, &trace);	62	ObtainCurrentStack(thr, pc, &trace);
61	rep.AddStack(trace, true);	63	rep.AddStack(trace, true);
...	@@ -63,185 +65,197 @@ static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,	...	@@ -63,185 +65,197 @@ static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,
63	OutputReport(thr, rep);	65	OutputReport(thr, rep);
64	}	66	}
65		67
66	void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {	68	static void RecordMutexLock(ThreadState *thr, uptr pc, uptr addr,
		69	StackID stack_id, bool write) {
		70	auto typ = write ? EventType::kLock : EventType::kRLock;
		71	// Note: it's important to trace before modifying mutex set
		72	// because tracing can switch trace part and we write the current
		73	// mutex set in the beginning of each part.
		74	// If we do it in the opposite order, we will write already reduced
		75	// mutex set in the beginning of the part and then trace unlock again.
		76	TraceMutexLock(thr, typ, pc, addr, stack_id);
		77	thr->mset.AddAddr(addr, stack_id, write);
		78	}
		79
		80	static void RecordMutexUnlock(ThreadState *thr, uptr addr) {
		81	// See the comment in RecordMutexLock re order of operations.
		82	TraceMutexUnlock(thr, addr);
		83	thr->mset.DelAddr(addr);
		84	}
		85
		86	void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
67	DPrintf("#%d: MutexCreate %zx flagz=0x%x\n", thr->tid, addr, flagz);	87	DPrintf("#%d: MutexCreate %zx flagz=0x%x\n", thr->tid, addr, flagz);
68	if (!(flagz & MutexFlagLinkerInit) && IsAppMem(addr)) {	88	if (!(flagz & MutexFlagLinkerInit) && pc && IsAppMem(addr))
69	CHECK(!thr->is_freeing);	89	MemoryAccess(thr, pc, addr, 1, kAccessWrite);
70	thr->is_freeing = true;	90	SlotLocker locker(thr);
71	MemoryWrite(thr, pc, addr, kSizeLog1);	91	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
72	thr->is_freeing = false;
73	}
74	SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
75	s->SetFlags(flagz & MutexCreationFlagMask);	92	s->SetFlags(flagz & MutexCreationFlagMask);
76	if (!SANITIZER_GO && s->creation_stack_id == 0)	93	// Save stack in the case the sync object was created before as atomic.
		94	if (!SANITIZER_GO && s->creation_stack_id == kInvalidStackID)
77	s->creation_stack_id = CurrentStackId(thr, pc);	95	s->creation_stack_id = CurrentStackId(thr, pc);
78	s->mtx.Unlock();
79	}	96	}
80		97
81	void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {	98	void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
82	DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr);	99	DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr);
83	SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
84	if (s == 0)
85	return;
86	if ((flagz & MutexFlagLinkerInit)
87	\|\| s->IsFlagSet(MutexFlagLinkerInit)
88	\|\| ((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) {
89	// Destroy is no-op for linker-initialized mutexes.
90	s->mtx.Unlock();
91	return;
92	}
93	if (common_flags()->detect_deadlocks) {
94	Callback cb(thr, pc);
95	ctx->dd->MutexDestroy(&cb, &s->dd);
96	ctx->dd->MutexInit(&cb, &s->dd);
97	}
98	bool unlock_locked = false;	100	bool unlock_locked = false;
99	if (flags()->report_destroy_locked && s->owner_tid != kInvalidTid &&	101	StackID creation_stack_id;
100	!s->IsFlagSet(MutexFlagBroken)) {	102	FastState last_lock;
101	s->SetFlags(MutexFlagBroken);	103	{
102	unlock_locked = true;	104	auto s = ctx->metamap.GetSyncIfExists(addr);
103	}	105	if (!s)
104	u64 mid = s->GetId();	106	return;
105	u64 last_lock = s->last_lock;	107	SlotLocker locker(thr);
106	if (!unlock_locked)	108	{
107	s->Reset(thr->proc()); // must not reset it before the report is printed	109	Lock lock(&s->mtx);
108	s->mtx.Unlock();	110	creation_stack_id = s->creation_stack_id;
109	if (unlock_locked && ShouldReport(thr, ReportTypeMutexDestroyLocked)) {	111	last_lock = s->last_lock;
110	ThreadRegistryLock l(ctx->thread_registry);	112	if ((flagz & MutexFlagLinkerInit) \|\| s->IsFlagSet(MutexFlagLinkerInit) \|\|
111	ScopedReport rep(ReportTypeMutexDestroyLocked);	113	((flagz & MutexFlagNotStatic) && !s->IsFlagSet(MutexFlagNotStatic))) {
112	rep.AddMutex(mid);	114	// Destroy is no-op for linker-initialized mutexes.
113	VarSizeStackTrace trace;	115	return;
114	ObtainCurrentStack(thr, pc, &trace);	116	}
115	rep.AddStack(trace, true);	117	if (common_flags()->detect_deadlocks) {
116	FastState last(last_lock);	118	Callback cb(thr, pc);
117	RestoreStack(last.tid(), last.epoch(), &trace, 0);	119	ctx->dd->MutexDestroy(&cb, &s->dd);
118	rep.AddStack(trace, true);	120	ctx->dd->MutexInit(&cb, &s->dd);
119	rep.AddLocation(addr, 1);	121	}
120	OutputReport(thr, rep);	122	if (flags()->report_destroy_locked && s->owner_tid != kInvalidTid &&
121		123	!s->IsFlagSet(MutexFlagBroken)) {
122	SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);	124	s->SetFlags(MutexFlagBroken);
123	if (s != 0) {	125	unlock_locked = true;
124	s->Reset(thr->proc());	126	}
125	s->mtx.Unlock();	127	s->Reset();
126	}	128	}
		129	// Imitate a memory write to catch unlock-destroy races.
		130	if (pc && IsAppMem(addr))
		131	MemoryAccess(thr, pc, addr, 1,
		132	kAccessWrite \| kAccessFree \| kAccessSlotLocked);
127	}	133	}
128	thr->mset.Remove(mid);	134	if (unlock_locked && ShouldReport(thr, ReportTypeMutexDestroyLocked))
129	// Imitate a memory write to catch unlock-destroy races.	135	ReportDestroyLocked(thr, pc, addr, last_lock, creation_stack_id);
130	// Do this outside of sync mutex, because it can report a race which locks	136	thr->mset.DelAddr(addr, true);
131	// sync mutexes.
132	if (IsAppMem(addr)) {
133	CHECK(!thr->is_freeing);
134	thr->is_freeing = true;
135	MemoryWrite(thr, pc, addr, kSizeLog1);
136	thr->is_freeing = false;
137	}
138	// s will be destroyed and freed in MetaMap::FreeBlock.	137	// s will be destroyed and freed in MetaMap::FreeBlock.
139	}	138	}
140		139
141	void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {	140	void MutexPreLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
142	DPrintf("#%d: MutexPreLock %zx flagz=0x%x\n", thr->tid, addr, flagz);	141	DPrintf("#%d: MutexPreLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
143	if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {	142	if (flagz & MutexFlagTryLock)
144	SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);	143	return;
		144	if (!common_flags()->detect_deadlocks)
		145	return;
		146	Callback cb(thr, pc);
		147	{
		148	SlotLocker locker(thr);
		149	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
		150	ReadLock lock(&s->mtx);
145	s->UpdateFlags(flagz);	151	s->UpdateFlags(flagz);
146	if (s->owner_tid != thr->tid) {	152	if (s->owner_tid != thr->tid)
147	Callback cb(thr, pc);
148	ctx->dd->MutexBeforeLock(&cb, &s->dd, true);	153	ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
149	s->mtx.ReadUnlock();
150	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
151	} else {
152	s->mtx.ReadUnlock();
153	}
154	}	154	}
		155	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
155	}	156	}
156		157
157	void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz,	158	void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) {
158	int rec) NO_THREAD_SAFETY_ANALYSIS {
159	DPrintf("#%d: MutexPostLock %zx flag=0x%x rec=%d\n",	159	DPrintf("#%d: MutexPostLock %zx flag=0x%x rec=%d\n",
160	thr->tid, addr, flagz, rec);	160	thr->tid, addr, flagz, rec);
161	if (flagz & MutexFlagRecursiveLock)	161	if (flagz & MutexFlagRecursiveLock)
162	CHECK_GT(rec, 0);	162	CHECK_GT(rec, 0);
163	else	163	else
164	rec = 1;	164	rec = 1;
165	if (IsAppMem(addr))	165	if (pc && IsAppMem(addr))
166	MemoryReadAtomic(thr, pc, addr, kSizeLog1);	166	MemoryAccess(thr, pc, addr, 1, kAccessRead \| kAccessAtomic);
167	SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);
168	s->UpdateFlags(flagz);
169	thr->fast_state.IncrementEpoch();
170	TraceAddEvent(thr, thr->fast_state, EventTypeLock, s->GetId());
171	bool report_double_lock = false;	167	bool report_double_lock = false;
172	if (s->owner_tid == kInvalidTid) {
173	CHECK_EQ(s->recursion, 0);
174	s->owner_tid = thr->tid;
175	s->last_lock = thr->fast_state.raw();
176	} else if (s->owner_tid == thr->tid) {
177	CHECK_GT(s->recursion, 0);
178	} else if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
179	s->SetFlags(MutexFlagBroken);
180	report_double_lock = true;
181	}
182	const bool first = s->recursion == 0;
183	s->recursion += rec;
184	if (first) {
185	AcquireImpl(thr, pc, &s->clock);
186	AcquireImpl(thr, pc, &s->read_clock);
187	} else if (!s->IsFlagSet(MutexFlagWriteReentrant)) {
188	}
189	thr->mset.Add(s->GetId(), true, thr->fast_state.epoch());
190	bool pre_lock = false;	168	bool pre_lock = false;
191	if (first && common_flags()->detect_deadlocks) {	169	bool first = false;
192	pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&	170	StackID creation_stack_id = kInvalidStackID;
193	!(flagz & MutexFlagTryLock);	171	{
194	Callback cb(thr, pc);	172	SlotLocker locker(thr);
195	if (pre_lock)	173	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
196	ctx->dd->MutexBeforeLock(&cb, &s->dd, true);	174	creation_stack_id = s->creation_stack_id;
197	ctx->dd->MutexAfterLock(&cb, &s->dd, true, flagz & MutexFlagTryLock);	175	RecordMutexLock(thr, pc, addr, creation_stack_id, true);
		176	{
		177	Lock lock(&s->mtx);
		178	first = s->recursion == 0;
		179	s->UpdateFlags(flagz);
		180	if (s->owner_tid == kInvalidTid) {
		181	CHECK_EQ(s->recursion, 0);
		182	s->owner_tid = thr->tid;
		183	s->last_lock = thr->fast_state;
		184	} else if (s->owner_tid == thr->tid) {
		185	CHECK_GT(s->recursion, 0);
		186	} else if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
		187	s->SetFlags(MutexFlagBroken);
		188	report_double_lock = true;
		189	}
		190	s->recursion += rec;
		191	if (first) {
		192	if (!thr->ignore_sync) {
		193	thr->clock.Acquire(s->clock);
		194	thr->clock.Acquire(s->read_clock);
		195	}
		196	}
		197	if (first && common_flags()->detect_deadlocks) {
		198	pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
		199	!(flagz & MutexFlagTryLock);
		200	Callback cb(thr, pc);
		201	if (pre_lock)
		202	ctx->dd->MutexBeforeLock(&cb, &s->dd, true);
		203	ctx->dd->MutexAfterLock(&cb, &s->dd, true, flagz & MutexFlagTryLock);
		204	}
		205	}
198	}	206	}
199	u64 mid = s->GetId();
200	s->mtx.Unlock();
201	// Can't touch s after this point.
202	s = 0;
203	if (report_double_lock)	207	if (report_double_lock)
204	ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr, mid);	208	ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr,
		209	creation_stack_id);
205	if (first && pre_lock && common_flags()->detect_deadlocks) {	210	if (first && pre_lock && common_flags()->detect_deadlocks) {
206	Callback cb(thr, pc);	211	Callback cb(thr, pc);
207	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));	212	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
208	}	213	}
209	}	214	}
210		215
211	int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {	216	int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
212	DPrintf("#%d: MutexUnlock %zx flagz=0x%x\n", thr->tid, addr, flagz);	217	DPrintf("#%d: MutexUnlock %zx flagz=0x%x\n", thr->tid, addr, flagz);
213	if (IsAppMem(addr))	218	if (pc && IsAppMem(addr))
214	MemoryReadAtomic(thr, pc, addr, kSizeLog1);	219	MemoryAccess(thr, pc, addr, 1, kAccessRead \| kAccessAtomic);
215	SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);	220	StackID creation_stack_id;
216	thr->fast_state.IncrementEpoch();	221	RecordMutexUnlock(thr, addr);
217	TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId());
218	int rec = 0;
219	bool report_bad_unlock = false;	222	bool report_bad_unlock = false;
220	if (!SANITIZER_GO && (s->recursion == 0 \|\| s->owner_tid != thr->tid)) {	223	int rec = 0;
221	if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {	224	{
222	s->SetFlags(MutexFlagBroken);	225	SlotLocker locker(thr);
223	report_bad_unlock = true;	226	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
224	}	227	bool released = false;
225	} else {	228	{
226	rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1;	229	Lock lock(&s->mtx);
227	s->recursion -= rec;	230	creation_stack_id = s->creation_stack_id;
228	if (s->recursion == 0) {	231	if (!SANITIZER_GO && (s->recursion == 0 \|\| s->owner_tid != thr->tid)) {
229	s->owner_tid = kInvalidTid;	232	if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
230	ReleaseStoreImpl(thr, pc, &s->clock);	233	s->SetFlags(MutexFlagBroken);
231	} else {	234	report_bad_unlock = true;
		235	}
		236	} else {
		237	rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1;
		238	s->recursion -= rec;
		239	if (s->recursion == 0) {
		240	s->owner_tid = kInvalidTid;
		241	if (!thr->ignore_sync) {
		242	thr->clock.ReleaseStore(&s->clock);
		243	released = true;
		244	}
		245	}
		246	}
		247	if (common_flags()->detect_deadlocks && s->recursion == 0 &&
		248	!report_bad_unlock) {
		249	Callback cb(thr, pc);
		250	ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true);
		251	}
232	}	252	}
		253	if (released)
		254	IncrementEpoch(thr);
233	}	255	}
234	thr->mset.Del(s->GetId(), true);
235	if (common_flags()->detect_deadlocks && s->recursion == 0 &&
236	!report_bad_unlock) {
237	Callback cb(thr, pc);
238	ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true);
239	}
240	u64 mid = s->GetId();
241	s->mtx.Unlock();
242	// Can't touch s after this point.
243	if (report_bad_unlock)	256	if (report_bad_unlock)
244	ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid);	257	ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr,
		258	creation_stack_id);
245	if (common_flags()->detect_deadlocks && !report_bad_unlock) {	259	if (common_flags()->detect_deadlocks && !report_bad_unlock) {
246	Callback cb(thr, pc);	260	Callback cb(thr, pc);
247	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));	261	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
...	@@ -249,282 +263,275 @@ int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFET	...	@@ -249,282 +263,275 @@ int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFET
249	return rec;	263	return rec;
250	}	264	}
251		265
252	void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {	266	void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
253	DPrintf("#%d: MutexPreReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);	267	DPrintf("#%d: MutexPreReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
254	if (!(flagz & MutexFlagTryLock) && common_flags()->detect_deadlocks) {	268	if ((flagz & MutexFlagTryLock) \|\| !common_flags()->detect_deadlocks)
255	SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);	269	return;
		270	Callback cb(thr, pc);
		271	{
		272	SlotLocker locker(thr);
		273	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
		274	ReadLock lock(&s->mtx);
256	s->UpdateFlags(flagz);	275	s->UpdateFlags(flagz);
257	Callback cb(thr, pc);
258	ctx->dd->MutexBeforeLock(&cb, &s->dd, false);	276	ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
259	s->mtx.ReadUnlock();
260	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
261	}	277	}
		278	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
262	}	279	}
263		280
264	void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) NO_THREAD_SAFETY_ANALYSIS {	281	void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) {
265	DPrintf("#%d: MutexPostReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);	282	DPrintf("#%d: MutexPostReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz);
266	if (IsAppMem(addr))	283	if (pc && IsAppMem(addr))
267	MemoryReadAtomic(thr, pc, addr, kSizeLog1);	284	MemoryAccess(thr, pc, addr, 1, kAccessRead \| kAccessAtomic);
268	SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false);
269	s->UpdateFlags(flagz);
270	thr->fast_state.IncrementEpoch();
271	TraceAddEvent(thr, thr->fast_state, EventTypeRLock, s->GetId());
272	bool report_bad_lock = false;	285	bool report_bad_lock = false;
273	if (s->owner_tid != kInvalidTid) {
274	if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
275	s->SetFlags(MutexFlagBroken);
276	report_bad_lock = true;
277	}
278	}
279	AcquireImpl(thr, pc, &s->clock);
280	s->last_lock = thr->fast_state.raw();
281	thr->mset.Add(s->GetId(), false, thr->fast_state.epoch());
282	bool pre_lock = false;	286	bool pre_lock = false;
283	if (common_flags()->detect_deadlocks) {	287	StackID creation_stack_id = kInvalidStackID;
284	pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&	288	{
285	!(flagz & MutexFlagTryLock);	289	SlotLocker locker(thr);
286	Callback cb(thr, pc);	290	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
287	if (pre_lock)	291	creation_stack_id = s->creation_stack_id;
288	ctx->dd->MutexBeforeLock(&cb, &s->dd, false);	292	RecordMutexLock(thr, pc, addr, creation_stack_id, false);
289	ctx->dd->MutexAfterLock(&cb, &s->dd, false, flagz & MutexFlagTryLock);	293	{
		294	ReadLock lock(&s->mtx);
		295	s->UpdateFlags(flagz);
		296	if (s->owner_tid != kInvalidTid) {
		297	if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
		298	s->SetFlags(MutexFlagBroken);
		299	report_bad_lock = true;
		300	}
		301	}
		302	if (!thr->ignore_sync)
		303	thr->clock.Acquire(s->clock);
		304	s->last_lock = thr->fast_state;
		305	if (common_flags()->detect_deadlocks) {
		306	pre_lock = (flagz & MutexFlagDoPreLockOnPostLock) &&
		307	!(flagz & MutexFlagTryLock);
		308	Callback cb(thr, pc);
		309	if (pre_lock)
		310	ctx->dd->MutexBeforeLock(&cb, &s->dd, false);
		311	ctx->dd->MutexAfterLock(&cb, &s->dd, false, flagz & MutexFlagTryLock);
		312	}
		313	}
290	}	314	}
291	u64 mid = s->GetId();
292	s->mtx.ReadUnlock();
293	// Can't touch s after this point.
294	s = 0;
295	if (report_bad_lock)	315	if (report_bad_lock)
296	ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr, mid);	316	ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr,
		317	creation_stack_id);
297	if (pre_lock && common_flags()->detect_deadlocks) {	318	if (pre_lock && common_flags()->detect_deadlocks) {
298	Callback cb(thr, pc);	319	Callback cb(thr, pc);
299	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));	320	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
300	}	321	}
301	}	322	}
302		323
303	void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {	324	void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) {
304	DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr);	325	DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr);
305	if (IsAppMem(addr))	326	if (pc && IsAppMem(addr))
306	MemoryReadAtomic(thr, pc, addr, kSizeLog1);	327	MemoryAccess(thr, pc, addr, 1, kAccessRead \| kAccessAtomic);
307	SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);	328	RecordMutexUnlock(thr, addr);
308	thr->fast_state.IncrementEpoch();	329	StackID creation_stack_id;
309	TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());
310	bool report_bad_unlock = false;	330	bool report_bad_unlock = false;
311	if (s->owner_tid != kInvalidTid) {	331	{
312	if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {	332	SlotLocker locker(thr);
313	s->SetFlags(MutexFlagBroken);	333	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
314	report_bad_unlock = true;	334	bool released = false;
		335	{
		336	Lock lock(&s->mtx);
		337	creation_stack_id = s->creation_stack_id;
		338	if (s->owner_tid != kInvalidTid) {
		339	if (flags()->report_mutex_bugs && !s->IsFlagSet(MutexFlagBroken)) {
		340	s->SetFlags(MutexFlagBroken);
		341	report_bad_unlock = true;
		342	}
		343	}
		344	if (!thr->ignore_sync) {
		345	thr->clock.Release(&s->read_clock);
		346	released = true;
		347	}
		348	if (common_flags()->detect_deadlocks && s->recursion == 0) {
		349	Callback cb(thr, pc);
		350	ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false);
		351	}
315	}	352	}
		353	if (released)
		354	IncrementEpoch(thr);
316	}	355	}
317	ReleaseImpl(thr, pc, &s->read_clock);
318	if (common_flags()->detect_deadlocks && s->recursion == 0) {
319	Callback cb(thr, pc);
320	ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false);
321	}
322	u64 mid = s->GetId();
323	s->mtx.Unlock();
324	// Can't touch s after this point.
325	thr->mset.Del(mid, false);
326	if (report_bad_unlock)	356	if (report_bad_unlock)
327	ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr, mid);	357	ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr,
		358	creation_stack_id);
328	if (common_flags()->detect_deadlocks) {	359	if (common_flags()->detect_deadlocks) {
329	Callback cb(thr, pc);	360	Callback cb(thr, pc);
330	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));	361	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
331	}	362	}
332	}	363	}
333		364
334	void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {	365	void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) {
335	DPrintf("#%d: MutexReadOrWriteUnlock %zx\n", thr->tid, addr);	366	DPrintf("#%d: MutexReadOrWriteUnlock %zx\n", thr->tid, addr);
336	if (IsAppMem(addr))	367	if (pc && IsAppMem(addr))
337	MemoryReadAtomic(thr, pc, addr, kSizeLog1);	368	MemoryAccess(thr, pc, addr, 1, kAccessRead \| kAccessAtomic);
338	SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);	369	RecordMutexUnlock(thr, addr);
339	bool write = true;	370	StackID creation_stack_id;
340	bool report_bad_unlock = false;	371	bool report_bad_unlock = false;
341	if (s->owner_tid == kInvalidTid) {	372	bool write = true;
342	// Seems to be read unlock.	373	{
343	write = false;	374	SlotLocker locker(thr);
344	thr->fast_state.IncrementEpoch();	375	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
345	TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId());	376	bool released = false;
346	ReleaseImpl(thr, pc, &s->read_clock);	377	{
347	} else if (s->owner_tid == thr->tid) {	378	Lock lock(&s->mtx);
348	// Seems to be write unlock.	379	creation_stack_id = s->creation_stack_id;
349	thr->fast_state.IncrementEpoch();	380	if (s->owner_tid == kInvalidTid) {
350	TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId());	381	// Seems to be read unlock.
351	CHECK_GT(s->recursion, 0);	382	write = false;
352	s->recursion--;	383	if (!thr->ignore_sync) {
353	if (s->recursion == 0) {	384	thr->clock.Release(&s->read_clock);
354	s->owner_tid = kInvalidTid;	385	released = true;
355	ReleaseStoreImpl(thr, pc, &s->clock);	386	}
356	} else {	387	} else if (s->owner_tid == thr->tid) {
		388	// Seems to be write unlock.
		389	CHECK_GT(s->recursion, 0);
		390	s->recursion--;
		391	if (s->recursion == 0) {
		392	s->owner_tid = kInvalidTid;
		393	if (!thr->ignore_sync) {
		394	thr->clock.ReleaseStore(&s->clock);
		395	released = true;
		396	}
		397	}
		398	} else if (!s->IsFlagSet(MutexFlagBroken)) {
		399	s->SetFlags(MutexFlagBroken);
		400	report_bad_unlock = true;
		401	}
		402	if (common_flags()->detect_deadlocks && s->recursion == 0) {
		403	Callback cb(thr, pc);
		404	ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write);
		405	}
357	}	406	}
358	} else if (!s->IsFlagSet(MutexFlagBroken)) {	407	if (released)
359	s->SetFlags(MutexFlagBroken);	408	IncrementEpoch(thr);
360	report_bad_unlock = true;
361	}	409	}
362	thr->mset.Del(s->GetId(), write);
363	if (common_flags()->detect_deadlocks && s->recursion == 0) {
364	Callback cb(thr, pc);
365	ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write);
366	}
367	u64 mid = s->GetId();
368	s->mtx.Unlock();
369	// Can't touch s after this point.
370	if (report_bad_unlock)	410	if (report_bad_unlock)
371	ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid);	411	ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr,
		412	creation_stack_id);
372	if (common_flags()->detect_deadlocks) {	413	if (common_flags()->detect_deadlocks) {
373	Callback cb(thr, pc);	414	Callback cb(thr, pc);
374	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));	415	ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb));
375	}	416	}
376	}	417	}
377		418
378	void MutexRepair(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {	419	void MutexRepair(ThreadState *thr, uptr pc, uptr addr) {
379	DPrintf("#%d: MutexRepair %zx\n", thr->tid, addr);	420	DPrintf("#%d: MutexRepair %zx\n", thr->tid, addr);
380	SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);	421	SlotLocker locker(thr);
		422	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
		423	Lock lock(&s->mtx);
381	s->owner_tid = kInvalidTid;	424	s->owner_tid = kInvalidTid;
382	s->recursion = 0;	425	s->recursion = 0;
383	s->mtx.Unlock();
384	}	426	}
385		427
386	void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {	428	void MutexInvalidAccess(ThreadState *thr, uptr pc, uptr addr) {
387	DPrintf("#%d: MutexInvalidAccess %zx\n", thr->tid, addr);	429	DPrintf("#%d: MutexInvalidAccess %zx\n", thr->tid, addr);
388	SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);	430	StackID creation_stack_id = kInvalidStackID;
389	u64 mid = s->GetId();	431	{
390	s->mtx.Unlock();	432	SlotLocker locker(thr);
391	ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr, mid);	433	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, true);
		434	if (s)
		435	creation_stack_id = s->creation_stack_id;
		436	}
		437	ReportMutexMisuse(thr, pc, ReportTypeMutexInvalidAccess, addr,
		438	creation_stack_id);
392	}	439	}
393		440
394	void Acquire(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {	441	void Acquire(ThreadState *thr, uptr pc, uptr addr) {
395	DPrintf("#%d: Acquire %zx\n", thr->tid, addr);	442	DPrintf("#%d: Acquire %zx\n", thr->tid, addr);
396	if (thr->ignore_sync)	443	if (thr->ignore_sync)
397	return;	444	return;
398	SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, false);	445	auto s = ctx->metamap.GetSyncIfExists(addr);
399	if (!s)	446	if (!s)
400	return;	447	return;
401	AcquireImpl(thr, pc, &s->clock);	448	SlotLocker locker(thr);
402	s->mtx.ReadUnlock();	449	if (!s->clock)
403	}	450	return;
404		451	ReadLock lock(&s->mtx);
405	static void UpdateClockCallback(ThreadContextBase tctx_base, void arg) {	452	thr->clock.Acquire(s->clock);
406	ThreadState thr = reinterpret_cast<ThreadState>(arg);
407	ThreadContext tctx = static_cast<ThreadContext>(tctx_base);
408	u64 epoch = tctx->epoch1;
409	if (tctx->status == ThreadStatusRunning) {
410	epoch = tctx->thr->fast_state.epoch();
411	tctx->thr->clock.NoteGlobalAcquire(epoch);
412	}
413	thr->clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
414	}	453	}
415		454
416	void AcquireGlobal(ThreadState *thr, uptr pc) {	455	void AcquireGlobal(ThreadState *thr) {
417	DPrintf("#%d: AcquireGlobal\n", thr->tid);	456	DPrintf("#%d: AcquireGlobal\n", thr->tid);
418	if (thr->ignore_sync)	457	if (thr->ignore_sync)
419	return;	458	return;
420	ThreadRegistryLock l(ctx->thread_registry);	459	SlotLocker locker(thr);
421	ctx->thread_registry->RunCallbackForEachThreadLocked(	460	for (auto &slot : ctx->slots) thr->clock.Set(slot.sid, slot.epoch());
422	UpdateClockCallback, thr);
423	}	461	}
424		462
425	void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {	463	void Release(ThreadState *thr, uptr pc, uptr addr) {
426	DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr);	464	DPrintf("#%d: Release %zx\n", thr->tid, addr);
427	if (thr->ignore_sync)	465	if (thr->ignore_sync)
428	return;	466	return;
429	SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);	467	SlotLocker locker(thr);
430	thr->fast_state.IncrementEpoch();	468	{
431	// Can't increment epoch w/o writing to the trace as well.	469	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
432	TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);	470	Lock lock(&s->mtx);
433	ReleaseStoreAcquireImpl(thr, pc, &s->clock);	471	thr->clock.Release(&s->clock);
434	s->mtx.Unlock();	472	}
		473	IncrementEpoch(thr);
435	}	474	}
436		475
437	void Release(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {	476	void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) {
438	DPrintf("#%d: Release %zx\n", thr->tid, addr);	477	DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr);
439	if (thr->ignore_sync)	478	if (thr->ignore_sync)
440	return;	479	return;
441	SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);	480	SlotLocker locker(thr);
442	thr->fast_state.IncrementEpoch();	481	{
443	// Can't increment epoch w/o writing to the trace as well.	482	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
444	TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);	483	Lock lock(&s->mtx);
445	ReleaseImpl(thr, pc, &s->clock);	484	thr->clock.ReleaseStore(&s->clock);
446	s->mtx.Unlock();	485	}
		486	IncrementEpoch(thr);
447	}	487	}
448		488
449	void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) NO_THREAD_SAFETY_ANALYSIS {	489	void ReleaseStoreAcquire(ThreadState *thr, uptr pc, uptr addr) {
450	DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr);	490	DPrintf("#%d: ReleaseStoreAcquire %zx\n", thr->tid, addr);
451	if (thr->ignore_sync)	491	if (thr->ignore_sync)
452	return;	492	return;
453	SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true);	493	SlotLocker locker(thr);
454	thr->fast_state.IncrementEpoch();	494	{
455	// Can't increment epoch w/o writing to the trace as well.	495	auto s = ctx->metamap.GetSyncOrCreate(thr, pc, addr, false);
456	TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);	496	Lock lock(&s->mtx);
457	ReleaseStoreImpl(thr, pc, &s->clock);	497	thr->clock.ReleaseStoreAcquire(&s->clock);
458	s->mtx.Unlock();	498	}
		499	IncrementEpoch(thr);
459	}	500	}
460		501
461	#if !SANITIZER_GO	502	void IncrementEpoch(ThreadState *thr) {
462	static void UpdateSleepClockCallback(ThreadContextBase tctx_base, void arg) {	503	DCHECK(!thr->ignore_sync);
463	ThreadState thr = reinterpret_cast<ThreadState>(arg);	504	DCHECK(thr->slot_locked);
464	ThreadContext tctx = static_cast<ThreadContext>(tctx_base);	505	Epoch epoch = EpochInc(thr->fast_state.epoch());
465	u64 epoch = tctx->epoch1;	506	if (!EpochOverflow(epoch)) {
466	if (tctx->status == ThreadStatusRunning)	507	Sid sid = thr->fast_state.sid();
467	epoch = tctx->thr->fast_state.epoch();	508	thr->clock.Set(sid, epoch);
468	thr->last_sleep_clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);	509	thr->fast_state.SetEpoch(epoch);
		510	thr->slot->SetEpoch(epoch);
		511	TraceTime(thr);
		512	}
469	}	513	}
470		514
		515	#if !SANITIZER_GO
471	void AfterSleep(ThreadState *thr, uptr pc) {	516	void AfterSleep(ThreadState *thr, uptr pc) {
472	DPrintf("#%d: AfterSleep %zx\n", thr->tid);	517	DPrintf("#%d: AfterSleep\n", thr->tid);
473	if (thr->ignore_sync)	518	if (thr->ignore_sync)
474	return;	519	return;
475	thr->last_sleep_stack_id = CurrentStackId(thr, pc);	520	thr->last_sleep_stack_id = CurrentStackId(thr, pc);
476	ThreadRegistryLock l(ctx->thread_registry);	521	thr->last_sleep_clock.Reset();
477	ctx->thread_registry->RunCallbackForEachThreadLocked(	522	SlotLocker locker(thr);
478	UpdateSleepClockCallback, thr);	523	for (auto &slot : ctx->slots)
		524	thr->last_sleep_clock.Set(slot.sid, slot.epoch());
479	}	525	}
480	#endif	526	#endif
481		527
482	void AcquireImpl(ThreadState thr, uptr pc, SyncClock c) {
483	if (thr->ignore_sync)
484	return;
485	thr->clock.set(thr->fast_state.epoch());
486	thr->clock.acquire(&thr->proc()->clock_cache, c);
487	}
488
489	void ReleaseStoreAcquireImpl(ThreadState thr, uptr pc, SyncClock c) {
490	if (thr->ignore_sync)
491	return;
492	thr->clock.set(thr->fast_state.epoch());
493	thr->fast_synch_epoch = thr->fast_state.epoch();
494	thr->clock.releaseStoreAcquire(&thr->proc()->clock_cache, c);
495	}
496
497	void ReleaseImpl(ThreadState thr, uptr pc, SyncClock c) {
498	if (thr->ignore_sync)
499	return;
500	thr->clock.set(thr->fast_state.epoch());
501	thr->fast_synch_epoch = thr->fast_state.epoch();
502	thr->clock.release(&thr->proc()->clock_cache, c);
503	}
504
505	void ReleaseStoreImpl(ThreadState thr, uptr pc, SyncClock c) {
506	if (thr->ignore_sync)
507	return;
508	thr->clock.set(thr->fast_state.epoch());
509	thr->fast_synch_epoch = thr->fast_state.epoch();
510	thr->clock.ReleaseStore(&thr->proc()->clock_cache, c);
511	}
512
513	void AcquireReleaseImpl(ThreadState thr, uptr pc, SyncClock c) {
514	if (thr->ignore_sync)
515	return;
516	thr->clock.set(thr->fast_state.epoch());
517	thr->fast_synch_epoch = thr->fast_state.epoch();
518	thr->clock.acq_rel(&thr->proc()->clock_cache, c);
519	}
520
521	void ReportDeadlock(ThreadState thr, uptr pc, DDReport r) {	528	void ReportDeadlock(ThreadState thr, uptr pc, DDReport r) {
522	if (r == 0 \|\| !ShouldReport(thr, ReportTypeDeadlock))	529	if (r == 0 \|\| !ShouldReport(thr, ReportTypeDeadlock))
523	return;	530	return;
524	ThreadRegistryLock l(ctx->thread_registry);	531	ThreadRegistryLock l(&ctx->thread_registry);
525	ScopedReport rep(ReportTypeDeadlock);	532	ScopedReport rep(ReportTypeDeadlock);
526	for (int i = 0; i < r->n; i++) {	533	for (int i = 0; i < r->n; i++) {
527	rep.AddMutex(r->loop[i].mtx_ctx0);	534	rep.AddMutex(r->loop[i].mtx_ctx0, r->loop[i].stk[0]);
528	rep.AddUniqueTid((int)r->loop[i].thr_ctx);	535	rep.AddUniqueTid((int)r->loop[i].thr_ctx);
529	rep.AddThread((int)r->loop[i].thr_ctx);	536	rep.AddThread((int)r->loop[i].thr_ctx);
530	}	537	}
...	@@ -532,7 +539,7 @@ void ReportDeadlock(ThreadState thr, uptr pc, DDReport r) {	...	@@ -532,7 +539,7 @@ void ReportDeadlock(ThreadState thr, uptr pc, DDReport r) {
532	for (int i = 0; i < r->n; i++) {	539	for (int i = 0; i < r->n; i++) {
533	for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) {	540	for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) {
534	u32 stk = r->loop[i].stk[j];	541	u32 stk = r->loop[i].stk[j];
535	if (stk && stk != 0xffffffff) {	542	if (stk && stk != kInvalidStackID) {
536	rep.AddStack(StackDepotGet(stk), true);	543	rep.AddStack(StackDepotGet(stk), true);
537	} else {	544	} else {
538	// Sometimes we fail to extract the stack trace (FIXME: investigate),	545	// Sometimes we fail to extract the stack trace (FIXME: investigate),
...	@@ -544,4 +551,28 @@ void ReportDeadlock(ThreadState thr, uptr pc, DDReport r) {	...	@@ -544,4 +551,28 @@ void ReportDeadlock(ThreadState thr, uptr pc, DDReport r) {
544	OutputReport(thr, rep);	551	OutputReport(thr, rep);
545	}	552	}
546		553
		554	void ReportDestroyLocked(ThreadState *thr, uptr pc, uptr addr,
		555	FastState last_lock, StackID creation_stack_id) {
		556	// We need to lock the slot during RestoreStack because it protects
		557	// the slot journal.
		558	Lock slot_lock(&ctx->slots[static_cast<uptr>(last_lock.sid())].mtx);
		559	ThreadRegistryLock l0(&ctx->thread_registry);
		560	Lock slots_lock(&ctx->slot_mtx);
		561	ScopedReport rep(ReportTypeMutexDestroyLocked);
		562	rep.AddMutex(addr, creation_stack_id);
		563	VarSizeStackTrace trace;
		564	ObtainCurrentStack(thr, pc, &trace);
		565	rep.AddStack(trace, true);
		566
		567	Tid tid;
		568	DynamicMutexSet mset;
		569	uptr tag;
		570	if (!RestoreStack(EventType::kLock, last_lock.sid(), last_lock.epoch(), addr,
		571	0, kAccessWrite, &tid, &trace, mset, &tag))
		572	return;
		573	rep.AddStack(trace, true);
		574	rep.AddLocation(addr, 1);
		575	OutputReport(thr, rep);
		576	}
		577
547	} // namespace __tsan	578	} // namespace __tsan

lib/tsan/tsan_rtl_proc.cpp-1

...	@@ -35,7 +35,6 @@ void ProcDestroy(Processor *proc) {	...	@@ -35,7 +35,6 @@ void ProcDestroy(Processor *proc) {
35	#if !SANITIZER_GO	35	#if !SANITIZER_GO
36	AllocatorProcFinish(proc);	36	AllocatorProcFinish(proc);
37	#endif	37	#endif
38	ctx->clock_alloc.FlushCache(&proc->clock_cache);
39	ctx->metamap.OnProcIdle(proc);	38	ctx->metamap.OnProcIdle(proc);
40	if (common_flags()->detect_deadlocks)	39	if (common_flags()->detect_deadlocks)
41	ctx->dd->DestroyPhysicalThread(proc->dd_pt);	40	ctx->dd->DestroyPhysicalThread(proc->dd_pt);

lib/tsan/tsan_rtl_report.cpp+388-290

...	@@ -10,20 +10,20 @@	...	@@ -10,20 +10,20 @@
10	//	10	//
11	//===----------------------------------------------------------------------===//	11	//===----------------------------------------------------------------------===//
12		12
		13	#include "sanitizer_common/sanitizer_common.h"
13	#include "sanitizer_common/sanitizer_libc.h"	14	#include "sanitizer_common/sanitizer_libc.h"
14	#include "sanitizer_common/sanitizer_placement_new.h"	15	#include "sanitizer_common/sanitizer_placement_new.h"
15	#include "sanitizer_common/sanitizer_stackdepot.h"	16	#include "sanitizer_common/sanitizer_stackdepot.h"
16	#include "sanitizer_common/sanitizer_common.h"
17	#include "sanitizer_common/sanitizer_stacktrace.h"	17	#include "sanitizer_common/sanitizer_stacktrace.h"
		18	#include "tsan_fd.h"
		19	#include "tsan_flags.h"
		20	#include "tsan_mman.h"
18	#include "tsan_platform.h"	21	#include "tsan_platform.h"
		22	#include "tsan_report.h"
19	#include "tsan_rtl.h"	23	#include "tsan_rtl.h"
20	#include "tsan_suppressions.h"	24	#include "tsan_suppressions.h"
21	#include "tsan_symbolize.h"	25	#include "tsan_symbolize.h"
22	#include "tsan_report.h"
23	#include "tsan_sync.h"	26	#include "tsan_sync.h"
24	#include "tsan_mman.h"
25	#include "tsan_flags.h"
26	#include "tsan_fd.h"
27		27
28	namespace __tsan {	28	namespace __tsan {
29		29
...	@@ -68,8 +68,10 @@ static void StackStripMain(SymbolizedStack *frames) {	...	@@ -68,8 +68,10 @@ static void StackStripMain(SymbolizedStack *frames) {
68	} else if (last && 0 == internal_strcmp(last, "__tsan_thread_start_func")) {	68	} else if (last && 0 == internal_strcmp(last, "__tsan_thread_start_func")) {
69	last_frame->ClearAll();	69	last_frame->ClearAll();
70	last_frame2->next = nullptr;	70	last_frame2->next = nullptr;
71	// Strip global ctors init.	71	// Strip global ctors init, .preinit_array and main caller.
72	} else if (last && 0 == internal_strcmp(last, "__do_global_ctors_aux")) {	72	} else if (last && (0 == internal_strcmp(last, "__do_global_ctors_aux") \|\|
		73	0 == internal_strcmp(last, "__libc_csu_init") \|\|
		74	0 == internal_strcmp(last, "__libc_start_main"))) {
73	last_frame->ClearAll();	75	last_frame->ClearAll();
74	last_frame2->next = nullptr;	76	last_frame2->next = nullptr;
75	// If both are 0, then we probably just failed to symbolize.	77	// If both are 0, then we probably just failed to symbolize.
...	@@ -120,7 +122,7 @@ static ReportStack *SymbolizeStack(StackTrace trace) {	...	@@ -120,7 +122,7 @@ static ReportStack *SymbolizeStack(StackTrace trace) {
120	}	122	}
121	StackStripMain(top);	123	StackStripMain(top);
122		124
123	ReportStack *stack = ReportStack::New();	125	auto *stack = New<ReportStack>();
124	stack->frames = top;	126	stack->frames = top;
125	return stack;	127	return stack;
126	}	128	}
...	@@ -132,7 +134,7 @@ bool ShouldReport(ThreadState *thr, ReportType typ) {	...	@@ -132,7 +134,7 @@ bool ShouldReport(ThreadState *thr, ReportType typ) {
132	CheckedMutex::CheckNoLocks();	134	CheckedMutex::CheckNoLocks();
133	// For the same reason check we didn't lock thread_registry yet.	135	// For the same reason check we didn't lock thread_registry yet.
134	if (SANITIZER_DEBUG)	136	if (SANITIZER_DEBUG)
135	ThreadRegistryLock l(ctx->thread_registry);	137	ThreadRegistryLock l(&ctx->thread_registry);
136	if (!flags()->report_bugs \|\| thr->suppress_reports)	138	if (!flags()->report_bugs \|\| thr->suppress_reports)
137	return false;	139	return false;
138	switch (typ) {	140	switch (typ) {
...	@@ -154,9 +156,8 @@ bool ShouldReport(ThreadState *thr, ReportType typ) {	...	@@ -154,9 +156,8 @@ bool ShouldReport(ThreadState *thr, ReportType typ) {
154	}	156	}
155		157
156	ScopedReportBase::ScopedReportBase(ReportType typ, uptr tag) {	158	ScopedReportBase::ScopedReportBase(ReportType typ, uptr tag) {
157	ctx->thread_registry->CheckLocked();	159	ctx->thread_registry.CheckLocked();
158	void *mem = internal_alloc(MBlockReport, sizeof(ReportDesc));	160	rep_ = New<ReportDesc>();
159	rep_ = new(mem) ReportDesc;
160	rep_->typ = typ;	161	rep_->typ = typ;
161	rep_->tag = tag;	162	rep_->tag = tag;
162	ctx->report_mtx.Lock();	163	ctx->report_mtx.Lock();
...	@@ -165,7 +166,6 @@ ScopedReportBase::ScopedReportBase(ReportType typ, uptr tag) {	...	@@ -165,7 +166,6 @@ ScopedReportBase::ScopedReportBase(ReportType typ, uptr tag) {
165	ScopedReportBase::~ScopedReportBase() {	166	ScopedReportBase::~ScopedReportBase() {
166	ctx->report_mtx.Unlock();	167	ctx->report_mtx.Unlock();
167	DestroyAndFree(rep_);	168	DestroyAndFree(rep_);
168	rep_ = nullptr;
169	}	169	}
170		170
171	void ScopedReportBase::AddStack(StackTrace stack, bool suppressable) {	171	void ScopedReportBase::AddStack(StackTrace stack, bool suppressable) {
...	@@ -175,28 +175,31 @@ void ScopedReportBase::AddStack(StackTrace stack, bool suppressable) {	...	@@ -175,28 +175,31 @@ void ScopedReportBase::AddStack(StackTrace stack, bool suppressable) {
175	}	175	}
176		176
177	void ScopedReportBase::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s,	177	void ScopedReportBase::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s,
178	StackTrace stack, const MutexSet *mset) {	178	Tid tid, StackTrace stack,
179	void *mem = internal_alloc(MBlockReportMop, sizeof(ReportMop));	179	const MutexSet *mset) {
180	ReportMop *mop = new(mem) ReportMop;	180	uptr addr0, size;
		181	AccessType typ;
		182	s.GetAccess(&addr0, &size, &typ);
		183	auto *mop = New<ReportMop>();
181	rep_->mops.PushBack(mop);	184	rep_->mops.PushBack(mop);
182	mop->tid = s.tid();	185	mop->tid = tid;
183	mop->addr = addr + s.addr0();	186	mop->addr = addr + addr0;
184	mop->size = s.size();	187	mop->size = size;
185	mop->write = s.IsWrite();	188	mop->write = !(typ & kAccessRead);
186	mop->atomic = s.IsAtomic();	189	mop->atomic = typ & kAccessAtomic;
187	mop->stack = SymbolizeStack(stack);	190	mop->stack = SymbolizeStack(stack);
188	mop->external_tag = external_tag;	191	mop->external_tag = external_tag;
189	if (mop->stack)	192	if (mop->stack)
190	mop->stack->suppressable = true;	193	mop->stack->suppressable = true;
191	for (uptr i = 0; i < mset->Size(); i++) {	194	for (uptr i = 0; i < mset->Size(); i++) {
192	MutexSet::Desc d = mset->Get(i);	195	MutexSet::Desc d = mset->Get(i);
193	u64 mid = this->AddMutex(d.id);	196	int id = this->AddMutex(d.addr, d.stack_id);
194	ReportMopMutex mtx = {mid, d.write};	197	ReportMopMutex mtx = {id, d.write};
195	mop->mset.PushBack(mtx);	198	mop->mset.PushBack(mtx);
196	}	199	}
197	}	200	}
198		201
199	void ScopedReportBase::AddUniqueTid(int unique_tid) {	202	void ScopedReportBase::AddUniqueTid(Tid unique_tid) {
200	rep_->unique_tids.PushBack(unique_tid);	203	rep_->unique_tids.PushBack(unique_tid);
201	}	204	}
202		205
...	@@ -205,8 +208,7 @@ void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) {	...	@@ -205,8 +208,7 @@ void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) {
205	if ((u32)rep_->threads[i]->id == tctx->tid)	208	if ((u32)rep_->threads[i]->id == tctx->tid)
206	return;	209	return;
207	}	210	}
208	void *mem = internal_alloc(MBlockReportThread, sizeof(ReportThread));	211	auto *rt = New<ReportThread>();
209	ReportThread *rt = new(mem) ReportThread;
210	rep_->threads.PushBack(rt);	212	rep_->threads.PushBack(rt);
211	rt->id = tctx->tid;	213	rt->id = tctx->tid;
212	rt->os_id = tctx->os_id;	214	rt->os_id = tctx->os_id;
...	@@ -221,22 +223,10 @@ void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) {	...	@@ -221,22 +223,10 @@ void ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) {
221	}	223	}
222		224
223	#if !SANITIZER_GO	225	#if !SANITIZER_GO
224	static bool FindThreadByUidLockedCallback(ThreadContextBase tctx, void arg) {	226	static ThreadContext *FindThreadByTidLocked(Tid tid) {
225	int unique_id = (int )arg;	227	ctx->thread_registry.CheckLocked();
226	return tctx->unique_id == (u32)unique_id;
227	}
228
229	static ThreadContext *FindThreadByUidLocked(int unique_id) {
230	ctx->thread_registry->CheckLocked();
231	return static_cast<ThreadContext *>(	228	return static_cast<ThreadContext *>(
232	ctx->thread_registry->FindThreadContextLocked(	229	ctx->thread_registry.GetThreadLocked(tid));
233	FindThreadByUidLockedCallback, &unique_id));
234	}
235
236	static ThreadContext *FindThreadByTidLocked(int tid) {
237	ctx->thread_registry->CheckLocked();
238	return static_cast<ThreadContext*>(
239	ctx->thread_registry->GetThreadLocked(tid));
240	}	230	}
241		231
242	static bool IsInStackOrTls(ThreadContextBase tctx_base, void arg) {	232	static bool IsInStackOrTls(ThreadContextBase tctx_base, void arg) {
...	@@ -251,10 +241,10 @@ static bool IsInStackOrTls(ThreadContextBase tctx_base, void arg) {	...	@@ -251,10 +241,10 @@ static bool IsInStackOrTls(ThreadContextBase tctx_base, void arg) {
251	}	241	}
252		242
253	ThreadContext IsThreadStackOrTls(uptr addr, bool is_stack) {	243	ThreadContext IsThreadStackOrTls(uptr addr, bool is_stack) {
254	ctx->thread_registry->CheckLocked();	244	ctx->thread_registry.CheckLocked();
255	ThreadContext tctx = static_cast<ThreadContext>(	245	ThreadContext *tctx =
256	ctx->thread_registry->FindThreadContextLocked(IsInStackOrTls,	246	static_cast<ThreadContext *>(ctx->thread_registry.FindThreadContextLocked(
257	(void*)addr));	247	IsInStackOrTls, (void *)addr));
258	if (!tctx)	248	if (!tctx)
259	return 0;	249	return 0;
260	ThreadState *thr = tctx->thr;	250	ThreadState *thr = tctx->thr;
...	@@ -264,58 +254,24 @@ ThreadContext IsThreadStackOrTls(uptr addr, bool is_stack) {	...	@@ -264,58 +254,24 @@ ThreadContext IsThreadStackOrTls(uptr addr, bool is_stack) {
264	}	254	}
265	#endif	255	#endif
266		256
267	void ScopedReportBase::AddThread(int unique_tid, bool suppressable) {	257	void ScopedReportBase::AddThread(Tid tid, bool suppressable) {
268	#if !SANITIZER_GO	258	#if !SANITIZER_GO
269	if (const ThreadContext *tctx = FindThreadByUidLocked(unique_tid))	259	if (const ThreadContext *tctx = FindThreadByTidLocked(tid))
270	AddThread(tctx, suppressable);	260	AddThread(tctx, suppressable);
271	#endif	261	#endif
272	}	262	}
273		263
274	void ScopedReportBase::AddMutex(const SyncVar *s) {	264	int ScopedReportBase::AddMutex(uptr addr, StackID creation_stack_id) {
275	for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
276	if (rep_->mutexes[i]->id == s->uid)
277	return;
278	}
279	void *mem = internal_alloc(MBlockReportMutex, sizeof(ReportMutex));
280	ReportMutex *rm = new(mem) ReportMutex;
281	rep_->mutexes.PushBack(rm);
282	rm->id = s->uid;
283	rm->addr = s->addr;
284	rm->destroyed = false;
285	rm->stack = SymbolizeStackId(s->creation_stack_id);
286	}
287
288	u64 ScopedReportBase::AddMutex(u64 id) NO_THREAD_SAFETY_ANALYSIS {
289	u64 uid = 0;
290	u64 mid = id;
291	uptr addr = SyncVar::SplitId(id, &uid);
292	SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
293	// Check that the mutex is still alive.
294	// Another mutex can be created at the same address,
295	// so check uid as well.
296	if (s && s->CheckId(uid)) {
297	mid = s->uid;
298	AddMutex(s);
299	} else {
300	AddDeadMutex(id);
301	}
302	if (s)
303	s->mtx.Unlock();
304	return mid;
305	}
306
307	void ScopedReportBase::AddDeadMutex(u64 id) {
308	for (uptr i = 0; i < rep_->mutexes.Size(); i++) {	265	for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
309	if (rep_->mutexes[i]->id == id)	266	if (rep_->mutexes[i]->addr == addr)
310	return;	267	return rep_->mutexes[i]->id;
311	}	268	}
312	void *mem = internal_alloc(MBlockReportMutex, sizeof(ReportMutex));	269	auto *rm = New<ReportMutex>();
313	ReportMutex *rm = new(mem) ReportMutex;
314	rep_->mutexes.PushBack(rm);	270	rep_->mutexes.PushBack(rm);
315	rm->id = id;	271	rm->id = rep_->mutexes.Size() - 1;
316	rm->addr = 0;	272	rm->addr = addr;
317	rm->destroyed = true;	273	rm->stack = SymbolizeStackId(creation_stack_id);
318	rm->stack = 0;	274	return rm->id;
319	}	275	}
320		276
321	void ScopedReportBase::AddLocation(uptr addr, uptr size) {	277	void ScopedReportBase::AddLocation(uptr addr, uptr size) {
...	@@ -323,43 +279,46 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) {	...	@@ -323,43 +279,46 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) {
323	return;	279	return;
324	#if !SANITIZER_GO	280	#if !SANITIZER_GO
325	int fd = -1;	281	int fd = -1;
326	int creat_tid = kInvalidTid;	282	Tid creat_tid = kInvalidTid;
327	u32 creat_stack = 0;	283	StackID creat_stack = 0;
328	if (FdLocation(addr, &fd, &creat_tid, &creat_stack)) {	284	bool closed = false;
329	ReportLocation *loc = ReportLocation::New(ReportLocationFD);	285	if (FdLocation(addr, &fd, &creat_tid, &creat_stack, &closed)) {
		286	auto *loc = New<ReportLocation>();
		287	loc->type = ReportLocationFD;
		288	loc->fd_closed = closed;
330	loc->fd = fd;	289	loc->fd = fd;
331	loc->tid = creat_tid;	290	loc->tid = creat_tid;
332	loc->stack = SymbolizeStackId(creat_stack);	291	loc->stack = SymbolizeStackId(creat_stack);
333	rep_->locs.PushBack(loc);	292	rep_->locs.PushBack(loc);
334	ThreadContext *tctx = FindThreadByUidLocked(creat_tid);	293	AddThread(creat_tid);
335	if (tctx)
336	AddThread(tctx);
337	return;	294	return;
338	}	295	}
339	MBlock *b = 0;	296	MBlock *b = 0;
		297	uptr block_begin = 0;
340	Allocator *a = allocator();	298	Allocator *a = allocator();
341	if (a->PointerIsMine((void*)addr)) {	299	if (a->PointerIsMine((void*)addr)) {
342	void block_begin = a->GetBlockBegin((void)addr);	300	block_begin = (uptr)a->GetBlockBegin((void *)addr);
343	if (block_begin)	301	if (block_begin)
344	b = ctx->metamap.GetBlock((uptr)block_begin);	302	b = ctx->metamap.GetBlock(block_begin);
345	}	303	}
		304	if (!b)
		305	b = JavaHeapBlock(addr, &block_begin);
346	if (b != 0) {	306	if (b != 0) {
347	ThreadContext *tctx = FindThreadByTidLocked(b->tid);	307	auto *loc = New<ReportLocation>();
348	ReportLocation *loc = ReportLocation::New(ReportLocationHeap);	308	loc->type = ReportLocationHeap;
349	loc->heap_chunk_start = (uptr)allocator()->GetBlockBegin((void *)addr);	309	loc->heap_chunk_start = block_begin;
350	loc->heap_chunk_size = b->siz;	310	loc->heap_chunk_size = b->siz;
351	loc->external_tag = b->tag;	311	loc->external_tag = b->tag;
352	loc->tid = tctx ? tctx->tid : b->tid;	312	loc->tid = b->tid;
353	loc->stack = SymbolizeStackId(b->stk);	313	loc->stack = SymbolizeStackId(b->stk);
354	rep_->locs.PushBack(loc);	314	rep_->locs.PushBack(loc);
355	if (tctx)	315	AddThread(b->tid);
356	AddThread(tctx);
357	return;	316	return;
358	}	317	}
359	bool is_stack = false;	318	bool is_stack = false;
360	if (ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack)) {	319	if (ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack)) {
361	ReportLocation *loc =	320	auto *loc = New<ReportLocation>();
362	ReportLocation::New(is_stack ? ReportLocationStack : ReportLocationTLS);	321	loc->type = is_stack ? ReportLocationStack : ReportLocationTLS;
363	loc->tid = tctx->tid;	322	loc->tid = tctx->tid;
364	rep_->locs.PushBack(loc);	323	rep_->locs.PushBack(loc);
365	AddThread(tctx);	324	AddThread(tctx);
...	@@ -373,13 +332,15 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) {	...	@@ -373,13 +332,15 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) {
373	}	332	}
374		333
375	#if !SANITIZER_GO	334	#if !SANITIZER_GO
376	void ScopedReportBase::AddSleep(u32 stack_id) {	335	void ScopedReportBase::AddSleep(StackID stack_id) {
377	rep_->sleep = SymbolizeStackId(stack_id);	336	rep_->sleep = SymbolizeStackId(stack_id);
378	}	337	}
379	#endif	338	#endif
380		339
381	void ScopedReportBase::SetCount(int count) { rep_->count = count; }	340	void ScopedReportBase::SetCount(int count) { rep_->count = count; }
382		341
		342	void ScopedReportBase::SetSigNum(int sig) { rep_->signum = sig; }
		343
383	const ReportDesc *ScopedReportBase::GetReport() const { return rep_; }	344	const ReportDesc *ScopedReportBase::GetReport() const { return rep_; }
384		345
385	ScopedReport::ScopedReport(ReportType typ, uptr tag)	346	ScopedReport::ScopedReport(ReportType typ, uptr tag)
...	@@ -387,67 +348,256 @@ ScopedReport::ScopedReport(ReportType typ, uptr tag)	...	@@ -387,67 +348,256 @@ ScopedReport::ScopedReport(ReportType typ, uptr tag)
387		348
388	ScopedReport::~ScopedReport() {}	349	ScopedReport::~ScopedReport() {}
389		350
390	void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,	351	// Replays the trace up to last_pos position in the last part
391	MutexSet mset, uptr tag) {	352	// or up to the provided epoch/sid (whichever is earlier)
		353	// and calls the provided function f for each event.
		354	template <typename Func>
		355	void TraceReplay(Trace trace, TracePart last, Event *last_pos, Sid sid,
		356	Epoch epoch, Func f) {
		357	TracePart *part = trace->parts.Front();
		358	Sid ev_sid = kFreeSid;
		359	Epoch ev_epoch = kEpochOver;
		360	for (;;) {
		361	DCHECK_EQ(part->trace, trace);
		362	// Note: an event can't start in the last element.
		363	// Since an event can take up to 2 elements,
		364	// we ensure we have at least 2 before adding an event.
		365	Event *end = &part->events[TracePart::kSize - 1];
		366	if (part == last)
		367	end = last_pos;
		368	f(kFreeSid, kEpochOver, nullptr); // notify about part start
		369	for (Event *evp = &part->events[0]; evp < end; evp++) {
		370	Event *evp0 = evp;
		371	if (!evp->is_access && !evp->is_func) {
		372	switch (evp->type) {
		373	case EventType::kTime: {
		374	auto ev = reinterpret_cast<EventTime >(evp);
		375	ev_sid = static_cast<Sid>(ev->sid);
		376	ev_epoch = static_cast<Epoch>(ev->epoch);
		377	if (ev_sid == sid && ev_epoch > epoch)
		378	return;
		379	break;
		380	}
		381	case EventType::kAccessExt:
		382	FALLTHROUGH;
		383	case EventType::kAccessRange:
		384	FALLTHROUGH;
		385	case EventType::kLock:
		386	FALLTHROUGH;
		387	case EventType::kRLock:
		388	// These take 2 Event elements.
		389	evp++;
		390	break;
		391	case EventType::kUnlock:
		392	// This takes 1 Event element.
		393	break;
		394	}
		395	}
		396	CHECK_NE(ev_sid, kFreeSid);
		397	CHECK_NE(ev_epoch, kEpochOver);
		398	f(ev_sid, ev_epoch, evp0);
		399	}
		400	if (part == last)
		401	return;
		402	part = trace->parts.Next(part);
		403	CHECK(part);
		404	}
		405	CHECK(0);
		406	}
		407
		408	static void RestoreStackMatch(VarSizeStackTrace pstk, MutexSet pmset,
		409	Vector<uptr> stack, MutexSet mset, uptr pc,
		410	bool *found) {
		411	DPrintf2(" MATCHED\n");
		412	pmset = mset;
		413	stack->PushBack(pc);
		414	pstk->Init(&(*stack)[0], stack->Size());
		415	stack->PopBack();
		416	*found = true;
		417	}
		418
		419	// Checks if addr1\|size1 is fully contained in addr2\|size2.
		420	// We check for fully contained instread of just overlapping
		421	// because a memory access is always traced once, but can be
		422	// split into multiple accesses in the shadow.
		423	static constexpr bool IsWithinAccess(uptr addr1, uptr size1, uptr addr2,
		424	uptr size2) {
		425	return addr1 >= addr2 && addr1 + size1 <= addr2 + size2;
		426	}
		427
		428	// Replays the trace of slot sid up to the target event identified
		429	// by epoch/addr/size/typ and restores and returns tid, stack, mutex set
		430	// and tag for that event. If there are multiple such events, it returns
		431	// the last one. Returns false if the event is not present in the trace.
		432	bool RestoreStack(EventType type, Sid sid, Epoch epoch, uptr addr, uptr size,
		433	AccessType typ, Tid ptid, VarSizeStackTrace pstk,
		434	MutexSet pmset, uptr ptag) {
392	// This function restores stack trace and mutex set for the thread/epoch.	435	// This function restores stack trace and mutex set for the thread/epoch.
393	// It does so by getting stack trace and mutex set at the beginning of	436	// It does so by getting stack trace and mutex set at the beginning of
394	// trace part, and then replaying the trace till the given epoch.	437	// trace part, and then replaying the trace till the given epoch.
395	Trace* trace = ThreadTrace(tid);	438	DPrintf2("RestoreStack: sid=%u@%u addr=0x%zx/%zu typ=%x\n",
396	ReadLock l(&trace->mtx);	439	static_cast<int>(sid), static_cast<int>(epoch), addr, size,
397	const int partidx = (epoch / kTracePartSize) % TraceParts();	440	static_cast<int>(typ));
398	TraceHeader* hdr = &trace->headers[partidx];	441	ctx->slot_mtx.CheckLocked(); // needed to prevent trace part recycling
399	if (epoch < hdr->epoch0 \|\| epoch >= hdr->epoch0 + kTracePartSize)	442	ctx->thread_registry.CheckLocked();
400	return;	443	TidSlot *slot = &ctx->slots[static_cast<uptr>(sid)];
401	CHECK_EQ(RoundDown(epoch, kTracePartSize), hdr->epoch0);	444	Tid tid = kInvalidTid;
402	const u64 epoch0 = RoundDown(epoch, TraceSize());	445	// Need to lock the slot mutex as it protects slot->journal.
403	const u64 eend = epoch % TraceSize();	446	slot->mtx.CheckLocked();
404	const u64 ebegin = RoundDown(eend, kTracePartSize);	447	for (uptr i = 0; i < slot->journal.Size(); i++) {
405	DPrintf("#%d: RestoreStack epoch=%zu ebegin=%zu eend=%zu partidx=%d\n",	448	DPrintf2(" journal: epoch=%d tid=%d\n",
406	tid, (uptr)epoch, (uptr)ebegin, (uptr)eend, partidx);	449	static_cast<int>(slot->journal[i].epoch), slot->journal[i].tid);
407	Vector<uptr> stack;	450	if (i == slot->journal.Size() - 1 \|\| slot->journal[i + 1].epoch > epoch) {
408	stack.Resize(hdr->stack0.size + 64);	451	tid = slot->journal[i].tid;
409	for (uptr i = 0; i < hdr->stack0.size; i++) {	452	break;
410	stack[i] = hdr->stack0.trace[i];
411	DPrintf2(" #%02zu: pc=%zx\n", i, stack[i]);
412	}
413	if (mset)
414	*mset = hdr->mset0;
415	uptr pos = hdr->stack0.size;
416	Event events = (Event)GetThreadTrace(tid);
417	for (uptr i = ebegin; i <= eend; i++) {
418	Event ev = events[i];
419	EventType typ = (EventType)(ev >> kEventPCBits);
420	uptr pc = (uptr)(ev & ((1ull << kEventPCBits) - 1));
421	DPrintf2(" %zu typ=%d pc=%zx\n", i, typ, pc);
422	if (typ == EventTypeMop) {
423	stack[pos] = pc;
424	} else if (typ == EventTypeFuncEnter) {
425	if (stack.Size() < pos + 2)
426	stack.Resize(pos + 2);
427	stack[pos++] = pc;
428	} else if (typ == EventTypeFuncExit) {
429	if (pos > 0)
430	pos--;
431	}	453	}
432	if (mset) {	454	}
433	if (typ == EventTypeLock) {	455	if (tid == kInvalidTid)
434	mset->Add(pc, true, epoch0 + i);	456	return false;
435	} else if (typ == EventTypeUnlock) {	457	*ptid = tid;
436	mset->Del(pc, true);	458	ThreadContext *tctx =
437	} else if (typ == EventTypeRLock) {	459	static_cast<ThreadContext *>(ctx->thread_registry.GetThreadLocked(tid));
438	mset->Add(pc, false, epoch0 + i);	460	Trace *trace = &tctx->trace;
439	} else if (typ == EventTypeRUnlock) {	461	// Snapshot first/last parts and the current position in the last part.
440	mset->Del(pc, false);	462	TracePart *first_part;
441	}	463	TracePart *last_part;
		464	Event *last_pos;
		465	{
		466	Lock lock(&trace->mtx);
		467	first_part = trace->parts.Front();
		468	if (!first_part) {
		469	DPrintf2("RestoreStack: tid=%d trace=%p no trace parts\n", tid, trace);
		470	return false;
442	}	471	}
443	for (uptr j = 0; j <= pos; j++)	472	last_part = trace->parts.Back();
444	DPrintf2(" #%zu: %zx\n", j, stack[j]);	473	last_pos = trace->final_pos;
		474	if (tctx->thr)
		475	last_pos = (Event *)atomic_load_relaxed(&tctx->thr->trace_pos);
445	}	476	}
446	if (pos == 0 && stack[0] == 0)	477	DynamicMutexSet mset;
447	return;	478	Vector<uptr> stack;
448	pos++;	479	uptr prev_pc = 0;
449	stk->Init(&stack[0], pos);	480	bool found = false;
450	ExtractTagFromStack(stk, tag);	481	bool is_read = typ & kAccessRead;
		482	bool is_atomic = typ & kAccessAtomic;
		483	bool is_free = typ & kAccessFree;
		484	DPrintf2("RestoreStack: tid=%d parts=[%p-%p] last_pos=%p\n", tid,
		485	trace->parts.Front(), last_part, last_pos);
		486	TraceReplay(
		487	trace, last_part, last_pos, sid, epoch,
		488	[&](Sid ev_sid, Epoch ev_epoch, Event *evp) {
		489	if (evp == nullptr) {
		490	// Each trace part is self-consistent, so we reset state.
		491	stack.Resize(0);
		492	mset->Reset();
		493	prev_pc = 0;
		494	return;
		495	}
		496	bool match = ev_sid == sid && ev_epoch == epoch;
		497	if (evp->is_access) {
		498	if (evp->is_func == 0 && evp->type == EventType::kAccessExt &&
		499	evp->_ == 0) // NopEvent
		500	return;
		501	auto ev = reinterpret_cast<EventAccess >(evp);
		502	uptr ev_addr = RestoreAddr(ev->addr);
		503	uptr ev_size = 1 << ev->size_log;
		504	uptr ev_pc =
		505	prev_pc + ev->pc_delta - (1 << (EventAccess::kPCBits - 1));
		506	prev_pc = ev_pc;
		507	DPrintf2(" Access: pc=0x%zx addr=0x%zx/%zu type=%u/%u\n", ev_pc,
		508	ev_addr, ev_size, ev->is_read, ev->is_atomic);
		509	if (match && type == EventType::kAccessExt &&
		510	IsWithinAccess(addr, size, ev_addr, ev_size) &&
		511	is_read == ev->is_read && is_atomic == ev->is_atomic && !is_free)
		512	RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found);
		513	return;
		514	}
		515	if (evp->is_func) {
		516	auto ev = reinterpret_cast<EventFunc >(evp);
		517	if (ev->pc) {
		518	DPrintf2(" FuncEnter: pc=0x%llx\n", ev->pc);
		519	stack.PushBack(ev->pc);
		520	} else {
		521	DPrintf2(" FuncExit\n");
		522	// We don't log pathologically large stacks in each part,
		523	// if the stack was truncated we can have more func exits than
		524	// entries.
		525	if (stack.Size())
		526	stack.PopBack();
		527	}
		528	return;
		529	}
		530	switch (evp->type) {
		531	case EventType::kAccessExt: {
		532	auto ev = reinterpret_cast<EventAccessExt >(evp);
		533	uptr ev_addr = RestoreAddr(ev->addr);
		534	uptr ev_size = 1 << ev->size_log;
		535	prev_pc = ev->pc;
		536	DPrintf2(" AccessExt: pc=0x%llx addr=0x%zx/%zu type=%u/%u\n",
		537	ev->pc, ev_addr, ev_size, ev->is_read, ev->is_atomic);
		538	if (match && type == EventType::kAccessExt &&
		539	IsWithinAccess(addr, size, ev_addr, ev_size) &&
		540	is_read == ev->is_read && is_atomic == ev->is_atomic &&
		541	!is_free)
		542	RestoreStackMatch(pstk, pmset, &stack, mset, ev->pc, &found);
		543	break;
		544	}
		545	case EventType::kAccessRange: {
		546	auto ev = reinterpret_cast<EventAccessRange >(evp);
		547	uptr ev_addr = RestoreAddr(ev->addr);
		548	uptr ev_size =
		549	(ev->size_hi << EventAccessRange::kSizeLoBits) + ev->size_lo;
		550	uptr ev_pc = RestoreAddr(ev->pc);
		551	prev_pc = ev_pc;
		552	DPrintf2(" Range: pc=0x%zx addr=0x%zx/%zu type=%u/%u\n", ev_pc,
		553	ev_addr, ev_size, ev->is_read, ev->is_free);
		554	if (match && type == EventType::kAccessExt &&
		555	IsWithinAccess(addr, size, ev_addr, ev_size) &&
		556	is_read == ev->is_read && !is_atomic && is_free == ev->is_free)
		557	RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found);
		558	break;
		559	}
		560	case EventType::kLock:
		561	FALLTHROUGH;
		562	case EventType::kRLock: {
		563	auto ev = reinterpret_cast<EventLock >(evp);
		564	bool is_write = ev->type == EventType::kLock;
		565	uptr ev_addr = RestoreAddr(ev->addr);
		566	uptr ev_pc = RestoreAddr(ev->pc);
		567	StackID stack_id =
		568	(ev->stack_hi << EventLock::kStackIDLoBits) + ev->stack_lo;
		569	DPrintf2(" Lock: pc=0x%zx addr=0x%zx stack=%u write=%d\n", ev_pc,
		570	ev_addr, stack_id, is_write);
		571	mset->AddAddr(ev_addr, stack_id, is_write);
		572	// Events with ev_pc == 0 are written to the beginning of trace
		573	// part as initial mutex set (are not real).
		574	if (match && type == EventType::kLock && addr == ev_addr && ev_pc)
		575	RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found);
		576	break;
		577	}
		578	case EventType::kUnlock: {
		579	auto ev = reinterpret_cast<EventUnlock >(evp);
		580	uptr ev_addr = RestoreAddr(ev->addr);
		581	DPrintf2(" Unlock: addr=0x%zx\n", ev_addr);
		582	mset->DelAddr(ev_addr);
		583	break;
		584	}
		585	case EventType::kTime:
		586	// TraceReplay already extracted sid/epoch from it,
		587	// nothing else to do here.
		588	break;
		589	}
		590	});
		591	ExtractTagFromStack(pstk, ptag);
		592	return found;
		593	}
		594
		595	bool RacyStacks::operator==(const RacyStacks &other) const {
		596	if (hash[0] == other.hash[0] && hash[1] == other.hash[1])
		597	return true;
		598	if (hash[0] == other.hash[1] && hash[1] == other.hash[0])
		599	return true;
		600	return false;
451	}	601	}
452		602
453	static bool FindRacyStacks(const RacyStacks &hash) {	603	static bool FindRacyStacks(const RacyStacks &hash) {
...	@@ -478,35 +628,6 @@ static bool HandleRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2]) {	...	@@ -478,35 +628,6 @@ static bool HandleRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2]) {
478	return false;	628	return false;
479	}	629	}
480		630
481	static bool FindRacyAddress(const RacyAddress &ra0) {
482	for (uptr i = 0; i < ctx->racy_addresses.Size(); i++) {
483	RacyAddress ra2 = ctx->racy_addresses[i];
484	uptr maxbeg = max(ra0.addr_min, ra2.addr_min);
485	uptr minend = min(ra0.addr_max, ra2.addr_max);
486	if (maxbeg < minend) {
487	VPrintf(2, "ThreadSanitizer: suppressing report as doubled (addr)\n");
488	return true;
489	}
490	}
491	return false;
492	}
493
494	static bool HandleRacyAddress(ThreadState *thr, uptr addr_min, uptr addr_max) {
495	if (!flags()->suppress_equal_addresses)
496	return false;
497	RacyAddress ra0 = {addr_min, addr_max};
498	{
499	ReadLock lock(&ctx->racy_mtx);
500	if (FindRacyAddress(ra0))
501	return true;
502	}
503	Lock lock(&ctx->racy_mtx);
504	if (FindRacyAddress(ra0))
505	return true;
506	ctx->racy_addresses.PushBack(ra0);
507	return false;
508	}
509
510	bool OutputReport(ThreadState *thr, const ScopedReport &srep) {	631	bool OutputReport(ThreadState *thr, const ScopedReport &srep) {
511	// These should have been checked in ShouldReport.	632	// These should have been checked in ShouldReport.
512	// It's too late to check them here, we have already taken locks.	633	// It's too late to check them here, we have already taken locks.
...	@@ -532,10 +653,7 @@ bool OutputReport(ThreadState *thr, const ScopedReport &srep) {	...	@@ -532,10 +653,7 @@ bool OutputReport(ThreadState *thr, const ScopedReport &srep) {
532	ctx->fired_suppressions.push_back(s);	653	ctx->fired_suppressions.push_back(s);
533	}	654	}
534	{	655	{
535	bool old_is_freeing = thr->is_freeing;
536	thr->is_freeing = false;
537	bool suppressed = OnReport(rep, pc_or_addr != 0);	656	bool suppressed = OnReport(rep, pc_or_addr != 0);
538	thr->is_freeing = old_is_freeing;
539	if (suppressed) {	657	if (suppressed) {
540	thr->current_report = nullptr;	658	thr->current_report = nullptr;
541	return false;	659	return false;
...	@@ -582,101 +700,81 @@ static bool IsFiredSuppression(Context *ctx, ReportType type, uptr addr) {	...	@@ -582,101 +700,81 @@ static bool IsFiredSuppression(Context *ctx, ReportType type, uptr addr) {
582	return false;	700	return false;
583	}	701	}
584		702
585	static bool RaceBetweenAtomicAndFree(ThreadState *thr) {	703	static bool SpuriousRace(Shadow old) {
586	Shadow s0(thr->racy_state[0]);	704	Shadow last(LoadShadow(&ctx->last_spurious_race));
587	Shadow s1(thr->racy_state[1]);	705	return last.sid() == old.sid() && last.epoch() == old.epoch();
588	CHECK(!(s0.IsAtomic() && s1.IsAtomic()));
589	if (!s0.IsAtomic() && !s1.IsAtomic())
590	return true;
591	if (s0.IsAtomic() && s1.IsFreed())
592	return true;
593	if (s1.IsAtomic() && thr->is_freeing)
594	return true;
595	return false;
596	}	706	}
597		707
598	void ReportRace(ThreadState *thr) {	708	void ReportRace(ThreadState thr, RawShadow shadow_mem, Shadow cur, Shadow old,
		709	AccessType typ0) {
599	CheckedMutex::CheckNoLocks();	710	CheckedMutex::CheckNoLocks();
600		711
601	// Symbolizer makes lots of intercepted calls. If we try to process them,	712	// Symbolizer makes lots of intercepted calls. If we try to process them,
602	// at best it will cause deadlocks on internal mutexes.	713	// at best it will cause deadlocks on internal mutexes.
603	ScopedIgnoreInterceptors ignore;	714	ScopedIgnoreInterceptors ignore;
604		715
		716	uptr addr = ShadowToMem(shadow_mem);
		717	DPrintf("#%d: ReportRace %p\n", thr->tid, (void *)addr);
605	if (!ShouldReport(thr, ReportTypeRace))	718	if (!ShouldReport(thr, ReportTypeRace))
606	return;	719	return;
607	if (!flags()->report_atomic_races && !RaceBetweenAtomicAndFree(thr))	720	uptr addr_off0, size0;
		721	cur.GetAccess(&addr_off0, &size0, nullptr);
		722	uptr addr_off1, size1, typ1;
		723	old.GetAccess(&addr_off1, &size1, &typ1);
		724	if (!flags()->report_atomic_races &&
		725	((typ0 & kAccessAtomic) \|\| (typ1 & kAccessAtomic)) &&
		726	!(typ0 & kAccessFree) && !(typ1 & kAccessFree))
		727	return;
		728	if (SpuriousRace(old))
608	return;	729	return;
609		730
610	bool freed = false;	731	const uptr kMop = 2;
611	{	732	Shadow s[kMop] = {cur, old};
612	Shadow s(thr->racy_state[1]);	733	uptr addr0 = addr + addr_off0;
613	freed = s.GetFreedAndReset();	734	uptr addr1 = addr + addr_off1;
614	thr->racy_state[1] = s.raw();	735	uptr end0 = addr0 + size0;
615	}	736	uptr end1 = addr1 + size1;
616		737	uptr addr_min = min(addr0, addr1);
617	uptr addr = ShadowToMem((uptr)thr->racy_shadow_addr);	738	uptr addr_max = max(end0, end1);
618	uptr addr_min = 0;	739	if (IsExpectedReport(addr_min, addr_max - addr_min))
619	uptr addr_max = 0;
620	{
621	uptr a0 = addr + Shadow(thr->racy_state[0]).addr0();
622	uptr a1 = addr + Shadow(thr->racy_state[1]).addr0();
623	uptr e0 = a0 + Shadow(thr->racy_state[0]).size();
624	uptr e1 = a1 + Shadow(thr->racy_state[1]).size();
625	addr_min = min(a0, a1);
626	addr_max = max(e0, e1);
627	if (IsExpectedReport(addr_min, addr_max - addr_min))
628	return;
629	}
630	if (HandleRacyAddress(thr, addr_min, addr_max))
631	return;	740	return;
632		741
633	ReportType typ = ReportTypeRace;	742	ReportType rep_typ = ReportTypeRace;
634	if (thr->is_vptr_access && freed)	743	if ((typ0 & kAccessVptr) && (typ1 & kAccessFree))
635	typ = ReportTypeVptrUseAfterFree;	744	rep_typ = ReportTypeVptrUseAfterFree;
636	else if (thr->is_vptr_access)	745	else if (typ0 & kAccessVptr)
637	typ = ReportTypeVptrRace;	746	rep_typ = ReportTypeVptrRace;
638	else if (freed)	747	else if (typ1 & kAccessFree)
639	typ = ReportTypeUseAfterFree;	748	rep_typ = ReportTypeUseAfterFree;
640		749
641	if (IsFiredSuppression(ctx, typ, addr))	750	if (IsFiredSuppression(ctx, rep_typ, addr))
642	return;	751	return;
643		752
644	const uptr kMop = 2;
645	VarSizeStackTrace traces[kMop];	753	VarSizeStackTrace traces[kMop];
646	uptr tags[kMop] = {kExternalTagNone};	754	Tid tids[kMop] = {thr->tid, kInvalidTid};
647	uptr toppc = TraceTopPC(thr);	755	uptr tags[kMop] = {kExternalTagNone, kExternalTagNone};
648	if (toppc >> kEventPCBits) {	756
649	// This is a work-around for a known issue.	757	ObtainCurrentStack(thr, thr->trace_prev_pc, &traces[0], &tags[0]);
650	// The scenario where this happens is rather elaborate and requires	758	if (IsFiredSuppression(ctx, rep_typ, traces[0]))
651	// an instrumented __sanitizer_report_error_summary callback and
652	// a __tsan_symbolize_external callback and a race during a range memory
653	// access larger than 8 bytes. MemoryAccessRange adds the current PC to
654	// the trace and starts processing memory accesses. A first memory access
655	// triggers a race, we report it and call the instrumented
656	// __sanitizer_report_error_summary, which adds more stuff to the trace
657	// since it is intrumented. Then a second memory access in MemoryAccessRange
658	// also triggers a race and we get here and call TraceTopPC to get the
659	// current PC, however now it contains some unrelated events from the
660	// callback. Most likely, TraceTopPC will now return a EventTypeFuncExit
661	// event. Later we subtract -1 from it (in GetPreviousInstructionPc)
662	// and the resulting PC has kExternalPCBit set, so we pass it to
663	// __tsan_symbolize_external_ex. __tsan_symbolize_external_ex is within its
664	// rights to crash since the PC is completely bogus.
665	// test/tsan/double_race.cpp contains a test case for this.
666	toppc = 0;
667	}
668	ObtainCurrentStack(thr, toppc, &traces[0], &tags[0]);
669	if (IsFiredSuppression(ctx, typ, traces[0]))
670	return;	759	return;
671		760
672	// MutexSet is too large to live on stack.	761	DynamicMutexSet mset1;
673	Vector<u64> mset_buffer;	762	MutexSet *mset[kMop] = {&thr->mset, mset1};
674	mset_buffer.Resize(sizeof(MutexSet) / sizeof(u64) + 1);
675	MutexSet *mset2 = new(&mset_buffer[0]) MutexSet();
676		763
677	Shadow s2(thr->racy_state[1]);	764	// We need to lock the slot during RestoreStack because it protects
678	RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2, &tags[1]);	765	// the slot journal.
679	if (IsFiredSuppression(ctx, typ, traces[1]))	766	Lock slot_lock(&ctx->slots[static_cast<uptr>(s[1].sid())].mtx);
		767	ThreadRegistryLock l0(&ctx->thread_registry);
		768	Lock slots_lock(&ctx->slot_mtx);
		769	if (SpuriousRace(old))
		770	return;
		771	if (!RestoreStack(EventType::kAccessExt, s[1].sid(), s[1].epoch(), addr1,
		772	size1, typ1, &tids[1], &traces[1], mset[1], &tags[1])) {
		773	StoreShadow(&ctx->last_spurious_race, old.raw());
		774	return;
		775	}
		776
		777	if (IsFiredSuppression(ctx, rep_typ, traces[1]))
680	return;	778	return;
681		779
682	if (HandleRacyStacks(thr, traces))	780	if (HandleRacyStacks(thr, traces))
...	@@ -686,39 +784,41 @@ void ReportRace(ThreadState *thr) {	...	@@ -686,39 +784,41 @@ void ReportRace(ThreadState *thr) {
686	uptr tag = kExternalTagNone;	784	uptr tag = kExternalTagNone;
687	for (uptr i = 0; i < kMop; i++) {	785	for (uptr i = 0; i < kMop; i++) {
688	if (tags[i] != kExternalTagNone) {	786	if (tags[i] != kExternalTagNone) {
689	typ = ReportTypeExternalRace;	787	rep_typ = ReportTypeExternalRace;
690	tag = tags[i];	788	tag = tags[i];
691	break;	789	break;
692	}	790	}
693	}	791	}
694		792
695	ThreadRegistryLock l0(ctx->thread_registry);	793	ScopedReport rep(rep_typ, tag);
696	ScopedReport rep(typ, tag);	794	for (uptr i = 0; i < kMop; i++)
697	for (uptr i = 0; i < kMop; i++) {	795	rep.AddMemoryAccess(addr, tags[i], s[i], tids[i], traces[i], mset[i]);
698	Shadow s(thr->racy_state[i]);
699	rep.AddMemoryAccess(addr, tags[i], s, traces[i],
700	i == 0 ? &thr->mset : mset2);
701	}
702		796
703	for (uptr i = 0; i < kMop; i++) {	797	for (uptr i = 0; i < kMop; i++) {
704	FastState s(thr->racy_state[i]);	798	ThreadContext tctx = static_cast<ThreadContext >(
705	ThreadContext tctx = static_cast<ThreadContext>(	799	ctx->thread_registry.GetThreadLocked(tids[i]));
706	ctx->thread_registry->GetThreadLocked(s.tid()));
707	if (s.epoch() < tctx->epoch0 \|\| s.epoch() > tctx->epoch1)
708	continue;
709	rep.AddThread(tctx);	800	rep.AddThread(tctx);
710	}	801	}
711		802
712	rep.AddLocation(addr_min, addr_max - addr_min);	803	rep.AddLocation(addr_min, addr_max - addr_min);
713		804
714	#if !SANITIZER_GO	805	if (flags()->print_full_thread_history) {
715	{	806	const ReportDesc *rep_desc = rep.GetReport();
716	Shadow s(thr->racy_state[1]);	807	for (uptr i = 0; i < rep_desc->threads.Size(); i++) {
717	if (s.epoch() <= thr->last_sleep_clock.get(s.tid()))	808	Tid parent_tid = rep_desc->threads[i]->parent_tid;
718	rep.AddSleep(thr->last_sleep_stack_id);	809	if (parent_tid == kMainTid \|\| parent_tid == kInvalidTid)
		810	continue;
		811	ThreadContext parent_tctx = static_cast<ThreadContext >(
		812	ctx->thread_registry.GetThreadLocked(parent_tid));
		813	rep.AddThread(parent_tctx);
		814	}
719	}	815	}
720	#endif
721		816
		817	#if !SANITIZER_GO
		818	if (!((typ0 \| typ1) & kAccessFree) &&
		819	s[1].epoch() <= thr->last_sleep_clock.Get(s[1].sid()))
		820	rep.AddSleep(thr->last_sleep_stack_id);
		821	#endif
722	OutputReport(thr, rep);	822	OutputReport(thr, rep);
723	}	823	}
724		824
...	@@ -738,9 +838,7 @@ void PrintCurrentStack(ThreadState *thr, uptr pc) {	...	@@ -738,9 +838,7 @@ void PrintCurrentStack(ThreadState *thr, uptr pc) {
738	ALWAYS_INLINE USED void PrintCurrentStackSlow(uptr pc) {	838	ALWAYS_INLINE USED void PrintCurrentStackSlow(uptr pc) {
739	#if !SANITIZER_GO	839	#if !SANITIZER_GO
740	uptr bp = GET_CURRENT_FRAME();	840	uptr bp = GET_CURRENT_FRAME();
741	BufferedStackTrace *ptrace =	841	auto *ptrace = New<BufferedStackTrace>();
742	new(internal_alloc(MBlockStackTrace, sizeof(BufferedStackTrace)))
743	BufferedStackTrace();
744	ptrace->Unwind(pc, bp, nullptr, false);	842	ptrace->Unwind(pc, bp, nullptr, false);
745		843
746	for (uptr i = 0; i < ptrace->size / 2; i++) {	844	for (uptr i = 0; i < ptrace->size / 2; i++) {

lib/tsan/tsan_rtl_thread.cpp+185-268

...	@@ -21,133 +21,14 @@ namespace __tsan {	...	@@ -21,133 +21,14 @@ namespace __tsan {
21		21
22	// ThreadContext implementation.	22	// ThreadContext implementation.
23		23
24	ThreadContext::ThreadContext(int tid)	24	ThreadContext::ThreadContext(Tid tid) : ThreadContextBase(tid), thr(), sync() {}
25	: ThreadContextBase(tid)
26	, thr()
27	, sync()
28	, epoch0()
29	, epoch1() {
30	}
31		25
32	#if !SANITIZER_GO	26	#if !SANITIZER_GO
33	ThreadContext::~ThreadContext() {	27	ThreadContext::~ThreadContext() {
34	}	28	}
35	#endif	29	#endif
36		30
37	void ThreadContext::OnDead() {	31	void ThreadContext::OnReset() { CHECK(!sync); }
38	CHECK_EQ(sync.size(), 0);
39	}
40
41	void ThreadContext::OnJoined(void *arg) {
42	ThreadState caller_thr = static_cast<ThreadState >(arg);
43	AcquireImpl(caller_thr, 0, &sync);
44	sync.Reset(&caller_thr->proc()->clock_cache);
45	}
46
47	struct OnCreatedArgs {
48	ThreadState *thr;
49	uptr pc;
50	};
51
52	void ThreadContext::OnCreated(void *arg) {
53	thr = 0;
54	if (tid == kMainTid)
55	return;
56	OnCreatedArgs args = static_cast<OnCreatedArgs >(arg);
57	if (!args->thr) // GCD workers don't have a parent thread.
58	return;
59	args->thr->fast_state.IncrementEpoch();
60	// Can't increment epoch w/o writing to the trace as well.
61	TraceAddEvent(args->thr, args->thr->fast_state, EventTypeMop, 0);
62	ReleaseImpl(args->thr, 0, &sync);
63	creation_stack_id = CurrentStackId(args->thr, args->pc);
64	}
65
66	void ThreadContext::OnReset() {
67	CHECK_EQ(sync.size(), 0);
68	uptr trace_p = GetThreadTrace(tid);
69	ReleaseMemoryPagesToOS(trace_p, trace_p + TraceSize() * sizeof(Event));
70	//!!! ReleaseMemoryToOS(GetThreadTraceHeader(tid), sizeof(Trace));
71	}
72
73	void ThreadContext::OnDetached(void *arg) {
74	ThreadState thr1 = static_cast<ThreadState>(arg);
75	sync.Reset(&thr1->proc()->clock_cache);
76	}
77
78	struct OnStartedArgs {
79	ThreadState *thr;
80	uptr stk_addr;
81	uptr stk_size;
82	uptr tls_addr;
83	uptr tls_size;
84	};
85
86	void ThreadContext::OnStarted(void *arg) {
87	OnStartedArgs args = static_cast<OnStartedArgs>(arg);
88	thr = args->thr;
89	// RoundUp so that one trace part does not contain events
90	// from different threads.
91	epoch0 = RoundUp(epoch1 + 1, kTracePartSize);
92	epoch1 = (u64)-1;
93	new(thr) ThreadState(ctx, tid, unique_id, epoch0, reuse_count,
94	args->stk_addr, args->stk_size, args->tls_addr, args->tls_size);
95	#if !SANITIZER_GO
96	thr->shadow_stack = &ThreadTrace(thr->tid)->shadow_stack[0];
97	thr->shadow_stack_pos = thr->shadow_stack;
98	thr->shadow_stack_end = thr->shadow_stack + kShadowStackSize;
99	#else
100	// Setup dynamic shadow stack.
101	const int kInitStackSize = 8;
102	thr->shadow_stack = (uptr*)internal_alloc(MBlockShadowStack,
103	kInitStackSize * sizeof(uptr));
104	thr->shadow_stack_pos = thr->shadow_stack;
105	thr->shadow_stack_end = thr->shadow_stack + kInitStackSize;
106	#endif
107	if (common_flags()->detect_deadlocks)
108	thr->dd_lt = ctx->dd->CreateLogicalThread(unique_id);
109	thr->fast_state.SetHistorySize(flags()->history_size);
110	// Commit switch to the new part of the trace.
111	// TraceAddEvent will reset stack0/mset0 in the new part for us.
112	TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
113
114	thr->fast_synch_epoch = epoch0;
115	AcquireImpl(thr, 0, &sync);
116	sync.Reset(&thr->proc()->clock_cache);
117	thr->is_inited = true;
118	DPrintf("#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx "
119	"tls_addr=%zx tls_size=%zx\n",
120	tid, (uptr)epoch0, args->stk_addr, args->stk_size,
121	args->tls_addr, args->tls_size);
122	}
123
124	void ThreadContext::OnFinished() {
125	#if SANITIZER_GO
126	internal_free(thr->shadow_stack);
127	thr->shadow_stack = nullptr;
128	thr->shadow_stack_pos = nullptr;
129	thr->shadow_stack_end = nullptr;
130	#endif
131	if (!detached) {
132	thr->fast_state.IncrementEpoch();
133	// Can't increment epoch w/o writing to the trace as well.
134	TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
135	ReleaseImpl(thr, 0, &sync);
136	}
137	epoch1 = thr->fast_state.epoch();
138
139	if (common_flags()->detect_deadlocks)
140	ctx->dd->DestroyLogicalThread(thr->dd_lt);
141	thr->clock.ResetCached(&thr->proc()->clock_cache);
142	#if !SANITIZER_GO
143	thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
144	#endif
145	#if !SANITIZER_GO
146	PlatformCleanUpThreadState(thr);
147	#endif
148	thr->~ThreadState();
149	thr = 0;
150	}
151		32
152	#if !SANITIZER_GO	33	#if !SANITIZER_GO
153	struct ThreadLeak {	34	struct ThreadLeak {
...	@@ -155,9 +36,9 @@ struct ThreadLeak {	...	@@ -155,9 +36,9 @@ struct ThreadLeak {
155	int count;	36	int count;
156	};	37	};
157		38
158	static void MaybeReportThreadLeak(ThreadContextBase tctx_base, void arg) {	39	static void CollectThreadLeaks(ThreadContextBase tctx_base, void arg) {
159	Vector<ThreadLeak> &leaks = (Vector<ThreadLeak>)arg;	40	auto &leaks = static_cast<Vector<ThreadLeak> >(arg);
160	ThreadContext tctx = static_cast<ThreadContext>(tctx_base);	41	auto tctx = static_cast<ThreadContext >(tctx_base);
161	if (tctx->detached \|\| tctx->status != ThreadStatusFinished)	42	if (tctx->detached \|\| tctx->status != ThreadStatusFinished)
162	return;	43	return;
163	for (uptr i = 0; i < leaks.Size(); i++) {	44	for (uptr i = 0; i < leaks.Size(); i++) {
...	@@ -166,12 +47,13 @@ static void MaybeReportThreadLeak(ThreadContextBase tctx_base, void arg) {	...	@@ -166,12 +47,13 @@ static void MaybeReportThreadLeak(ThreadContextBase tctx_base, void arg) {
166	return;	47	return;
167	}	48	}
168	}	49	}
169	ThreadLeak leak = {tctx, 1};	50	leaks.PushBack({tctx, 1});
170	leaks.PushBack(leak);
171	}	51	}
172	#endif	52	#endif
173		53
174	#if !SANITIZER_GO	54	// Disabled on Mac because lldb test TestTsanBasic fails:
		55	// https://reviews.llvm.org/D112603#3163158
		56	#if !SANITIZER_GO && !SANITIZER_APPLE
175	static void ReportIgnoresEnabled(ThreadContext tctx, IgnoreSet set) {	57	static void ReportIgnoresEnabled(ThreadContext tctx, IgnoreSet set) {
176	if (tctx->tid == kMainTid) {	58	if (tctx->tid == kMainTid) {
177	Printf("ThreadSanitizer: main thread finished with ignores enabled\n");	59	Printf("ThreadSanitizer: main thread finished with ignores enabled\n");
...	@@ -206,10 +88,10 @@ void ThreadFinalize(ThreadState *thr) {	...	@@ -206,10 +88,10 @@ void ThreadFinalize(ThreadState *thr) {
206	#if !SANITIZER_GO	88	#if !SANITIZER_GO
207	if (!ShouldReport(thr, ReportTypeThreadLeak))	89	if (!ShouldReport(thr, ReportTypeThreadLeak))
208	return;	90	return;
209	ThreadRegistryLock l(ctx->thread_registry);	91	ThreadRegistryLock l(&ctx->thread_registry);
210	Vector<ThreadLeak> leaks;	92	Vector<ThreadLeak> leaks;
211	ctx->thread_registry->RunCallbackForEachThreadLocked(	93	ctx->thread_registry.RunCallbackForEachThreadLocked(CollectThreadLeaks,
212	MaybeReportThreadLeak, &leaks);	94	&leaks);
213	for (uptr i = 0; i < leaks.Size(); i++) {	95	for (uptr i = 0; i < leaks.Size(); i++) {
214	ScopedReport rep(ReportTypeThreadLeak);	96	ScopedReport rep(ReportTypeThreadLeak);
215	rep.AddThread(leaks[i].tctx, true);	97	rep.AddThread(leaks[i].tctx, true);
...	@@ -221,21 +103,63 @@ void ThreadFinalize(ThreadState *thr) {	...	@@ -221,21 +103,63 @@ void ThreadFinalize(ThreadState *thr) {
221		103
222	int ThreadCount(ThreadState *thr) {	104	int ThreadCount(ThreadState *thr) {
223	uptr result;	105	uptr result;
224	ctx->thread_registry->GetNumberOfThreads(0, 0, &result);	106	ctx->thread_registry.GetNumberOfThreads(0, 0, &result);
225	return (int)result;	107	return (int)result;
226	}	108	}
227		109
228	int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) {	110	struct OnCreatedArgs {
229	OnCreatedArgs args = { thr, pc };	111	VectorClock *sync;
230	u32 parent_tid = thr ? thr->tid : kInvalidTid; // No parent for GCD workers.	112	uptr sync_epoch;
231	int tid =	113	StackID stack;
232	ctx->thread_registry->CreateThread(uid, detached, parent_tid, &args);	114	};
233	DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent_tid, tid, uid);	115
		116	Tid ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) {
		117	// The main thread and GCD workers don't have a parent thread.
		118	Tid parent = kInvalidTid;
		119	OnCreatedArgs arg = {nullptr, 0, kInvalidStackID};
		120	if (thr) {
		121	parent = thr->tid;
		122	arg.stack = CurrentStackId(thr, pc);
		123	if (!thr->ignore_sync) {
		124	SlotLocker locker(thr);
		125	thr->clock.ReleaseStore(&arg.sync);
		126	arg.sync_epoch = ctx->global_epoch;
		127	IncrementEpoch(thr);
		128	}
		129	}
		130	Tid tid = ctx->thread_registry.CreateThread(uid, detached, parent, &arg);
		131	DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent, tid, uid);
234	return tid;	132	return tid;
235	}	133	}
236		134
237	void ThreadStart(ThreadState *thr, int tid, tid_t os_id,	135	void ThreadContext::OnCreated(void *arg) {
		136	OnCreatedArgs args = static_cast<OnCreatedArgs >(arg);
		137	sync = args->sync;
		138	sync_epoch = args->sync_epoch;
		139	creation_stack_id = args->stack;
		140	}
		141
		142	extern "C" void __tsan_stack_initialization() {}
		143
		144	struct OnStartedArgs {
		145	ThreadState *thr;
		146	uptr stk_addr;
		147	uptr stk_size;
		148	uptr tls_addr;
		149	uptr tls_size;
		150	};
		151
		152	void ThreadStart(ThreadState *thr, Tid tid, tid_t os_id,
238	ThreadType thread_type) {	153	ThreadType thread_type) {
		154	ctx->thread_registry.StartThread(tid, os_id, thread_type, thr);
		155	if (!thr->ignore_sync) {
		156	SlotAttachAndLock(thr);
		157	if (thr->tctx->sync_epoch == ctx->global_epoch)
		158	thr->clock.Acquire(thr->tctx->sync);
		159	SlotUnlock(thr);
		160	}
		161	Free(thr->tctx->sync);
		162
239	uptr stk_addr = 0;	163	uptr stk_addr = 0;
240	uptr stk_size = 0;	164	uptr stk_size = 0;
241	uptr tls_addr = 0;	165	uptr tls_addr = 0;
...	@@ -244,22 +168,11 @@ void ThreadStart(ThreadState *thr, int tid, tid_t os_id,	...	@@ -244,22 +168,11 @@ void ThreadStart(ThreadState *thr, int tid, tid_t os_id,
244	if (thread_type != ThreadType::Fiber)	168	if (thread_type != ThreadType::Fiber)
245	GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_size, &tls_addr,	169	GetThreadStackAndTls(tid == kMainTid, &stk_addr, &stk_size, &tls_addr,
246	&tls_size);	170	&tls_size);
247
248	if (tid != kMainTid) {
249	if (stk_addr && stk_size)
250	MemoryRangeImitateWrite(thr, /pc=/ 1, stk_addr, stk_size);
251
252	if (tls_addr && tls_size) ImitateTlsWrite(thr, tls_addr, tls_size);
253	}
254	#endif	171	#endif
255		172	thr->stk_addr = stk_addr;
256	ThreadRegistry *tr = ctx->thread_registry;	173	thr->stk_size = stk_size;
257	OnStartedArgs args = { thr, stk_addr, stk_size, tls_addr, tls_size };	174	thr->tls_addr = tls_addr;
258	tr->StartThread(tid, os_id, thread_type, &args);	175	thr->tls_size = tls_size;
259
260	tr->Lock();
261	thr->tctx = (ThreadContext*)tr->GetThreadLocked(tid);
262	tr->Unlock();
263		176
264	#if !SANITIZER_GO	177	#if !SANITIZER_GO
265	if (ctx->after_multithreaded_fork) {	178	if (ctx->after_multithreaded_fork) {
...	@@ -268,16 +181,99 @@ void ThreadStart(ThreadState *thr, int tid, tid_t os_id,	...	@@ -268,16 +181,99 @@ void ThreadStart(ThreadState *thr, int tid, tid_t os_id,
268	ThreadIgnoreSyncBegin(thr, 0);	181	ThreadIgnoreSyncBegin(thr, 0);
269	}	182	}
270	#endif	183	#endif
		184
		185	#if !SANITIZER_GO
		186	// Don't imitate stack/TLS writes for the main thread,
		187	// because its initialization is synchronized with all
		188	// subsequent threads anyway.
		189	if (tid != kMainTid) {
		190	if (stk_addr && stk_size) {
		191	const uptr pc = StackTrace::GetNextInstructionPc(
		192	reinterpret_cast<uptr>(__tsan_stack_initialization));
		193	MemoryRangeImitateWrite(thr, pc, stk_addr, stk_size);
		194	}
		195
		196	if (tls_addr && tls_size)
		197	ImitateTlsWrite(thr, tls_addr, tls_size);
		198	}
		199	#endif
		200	}
		201
		202	void ThreadContext::OnStarted(void *arg) {
		203	thr = static_cast<ThreadState *>(arg);
		204	DPrintf("#%d: ThreadStart\n", tid);
		205	new (thr) ThreadState(tid);
		206	if (common_flags()->detect_deadlocks)
		207	thr->dd_lt = ctx->dd->CreateLogicalThread(tid);
		208	thr->tctx = this;
		209	#if !SANITIZER_GO
		210	thr->is_inited = true;
		211	#endif
271	}	212	}
272		213
273	void ThreadFinish(ThreadState *thr) {	214	void ThreadFinish(ThreadState *thr) {
		215	DPrintf("#%d: ThreadFinish\n", thr->tid);
274	ThreadCheckIgnore(thr);	216	ThreadCheckIgnore(thr);
275	if (thr->stk_addr && thr->stk_size)	217	if (thr->stk_addr && thr->stk_size)
276	DontNeedShadowFor(thr->stk_addr, thr->stk_size);	218	DontNeedShadowFor(thr->stk_addr, thr->stk_size);
277	if (thr->tls_addr && thr->tls_size)	219	if (thr->tls_addr && thr->tls_size)
278	DontNeedShadowFor(thr->tls_addr, thr->tls_size);	220	DontNeedShadowFor(thr->tls_addr, thr->tls_size);
279	thr->is_dead = true;	221	thr->is_dead = true;
280	ctx->thread_registry->FinishThread(thr->tid);	222	#if !SANITIZER_GO
		223	thr->is_inited = false;
		224	thr->ignore_interceptors++;
		225	PlatformCleanUpThreadState(thr);
		226	#endif
		227	if (!thr->ignore_sync) {
		228	SlotLocker locker(thr);
		229	ThreadRegistryLock lock(&ctx->thread_registry);
		230	// Note: detached is protected by the thread registry mutex,
		231	// the thread may be detaching concurrently in another thread.
		232	if (!thr->tctx->detached) {
		233	thr->clock.ReleaseStore(&thr->tctx->sync);
		234	thr->tctx->sync_epoch = ctx->global_epoch;
		235	IncrementEpoch(thr);
		236	}
		237	}
		238	#if !SANITIZER_GO
		239	UnmapOrDie(thr->shadow_stack, kShadowStackSize * sizeof(uptr));
		240	#else
		241	Free(thr->shadow_stack);
		242	#endif
		243	thr->shadow_stack = nullptr;
		244	thr->shadow_stack_pos = nullptr;
		245	thr->shadow_stack_end = nullptr;
		246	if (common_flags()->detect_deadlocks)
		247	ctx->dd->DestroyLogicalThread(thr->dd_lt);
		248	SlotDetach(thr);
		249	ctx->thread_registry.FinishThread(thr->tid);
		250	thr->~ThreadState();
		251	}
		252
		253	void ThreadContext::OnFinished() {
		254	Lock lock(&ctx->slot_mtx);
		255	Lock lock1(&trace.mtx);
		256	// Queue all trace parts into the global recycle queue.
		257	auto parts = &trace.parts;
		258	while (trace.local_head) {
		259	CHECK(parts->Queued(trace.local_head));
		260	ctx->trace_part_recycle.PushBack(trace.local_head);
		261	trace.local_head = parts->Next(trace.local_head);
		262	}
		263	ctx->trace_part_recycle_finished += parts->Size();
		264	if (ctx->trace_part_recycle_finished > Trace::kFinishedThreadHi) {
		265	ctx->trace_part_finished_excess += parts->Size();
		266	trace.parts_allocated = 0;
		267	} else if (ctx->trace_part_recycle_finished > Trace::kFinishedThreadLo &&
		268	parts->Size() > 1) {
		269	ctx->trace_part_finished_excess += parts->Size() - 1;
		270	trace.parts_allocated = 1;
		271	}
		272	// From now on replay will use trace->final_pos.
		273	trace.final_pos = (Event *)atomic_load_relaxed(&thr->trace_pos);
		274	atomic_store_relaxed(&thr->trace_pos, 0);
		275	thr->tctx = nullptr;
		276	thr = nullptr;
281	}	277	}
282		278
283	struct ConsumeThreadContext {	279	struct ConsumeThreadContext {
...	@@ -285,131 +281,52 @@ struct ConsumeThreadContext {	...	@@ -285,131 +281,52 @@ struct ConsumeThreadContext {
285	ThreadContextBase *tctx;	281	ThreadContextBase *tctx;
286	};	282	};
287		283
288	static bool ConsumeThreadByUid(ThreadContextBase tctx, void arg) {	284	Tid ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid) {
289	ConsumeThreadContext findCtx = (ConsumeThreadContext )arg;	285	return ctx->thread_registry.ConsumeThreadUserId(uid);
290	if (tctx->user_id == findCtx->uid && tctx->status != ThreadStatusInvalid) {
291	if (findCtx->tctx) {
292	// Ensure that user_id is unique. If it's not the case we are screwed.
293	// Something went wrong before, but now there is no way to recover.
294	// Returning a wrong thread is not an option, it may lead to very hard
295	// to debug false positives (e.g. if we join a wrong thread).
296	Report("ThreadSanitizer: dup thread with used id 0x%zx\n", findCtx->uid);
297	Die();
298	}
299	findCtx->tctx = tctx;
300	tctx->user_id = 0;
301	}
302	return false;
303	}	286	}
304		287
305	int ThreadConsumeTid(ThreadState *thr, uptr pc, uptr uid) {	288	struct JoinArg {
306	ConsumeThreadContext findCtx = {uid, nullptr};	289	VectorClock *sync;
307	ctx->thread_registry->FindThread(ConsumeThreadByUid, &findCtx);	290	uptr sync_epoch;
308	int tid = findCtx.tctx ? findCtx.tctx->tid : kInvalidTid;	291	};
309	DPrintf("#%d: ThreadTid uid=%zu tid=%d\n", thr->tid, uid, tid);
310	return tid;
311	}
312		292
313	void ThreadJoin(ThreadState *thr, uptr pc, int tid) {	293	void ThreadJoin(ThreadState *thr, uptr pc, Tid tid) {
314	CHECK_GT(tid, 0);	294	CHECK_GT(tid, 0);
315	CHECK_LT(tid, kMaxTid);
316	DPrintf("#%d: ThreadJoin tid=%d\n", thr->tid, tid);	295	DPrintf("#%d: ThreadJoin tid=%d\n", thr->tid, tid);
317	ctx->thread_registry->JoinThread(tid, thr);	296	JoinArg arg = {};
		297	ctx->thread_registry.JoinThread(tid, &arg);
		298	if (!thr->ignore_sync) {
		299	SlotLocker locker(thr);
		300	if (arg.sync_epoch == ctx->global_epoch)
		301	thr->clock.Acquire(arg.sync);
		302	}
		303	Free(arg.sync);
318	}	304	}
319		305
320	void ThreadDetach(ThreadState *thr, uptr pc, int tid) {	306	void ThreadContext::OnJoined(void *ptr) {
321	CHECK_GT(tid, 0);	307	auto arg = static_cast<JoinArg *>(ptr);
322	CHECK_LT(tid, kMaxTid);	308	arg->sync = sync;
323	ctx->thread_registry->DetachThread(tid, thr);	309	arg->sync_epoch = sync_epoch;
		310	sync = nullptr;
		311	sync_epoch = 0;
324	}	312	}
325		313
326	void ThreadNotJoined(ThreadState *thr, uptr pc, int tid, uptr uid) {	314	void ThreadContext::OnDead() { CHECK_EQ(sync, nullptr); }
327	CHECK_GT(tid, 0);
328	CHECK_LT(tid, kMaxTid);
329	ctx->thread_registry->SetThreadUserId(tid, uid);
330	}
331		315
332	void ThreadSetName(ThreadState thr, const char name) {	316	void ThreadDetach(ThreadState *thr, uptr pc, Tid tid) {
333	ctx->thread_registry->SetThreadName(thr->tid, name);	317	CHECK_GT(tid, 0);
		318	ctx->thread_registry.DetachThread(tid, thr);
334	}	319	}
335		320
336	void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,	321	void ThreadContext::OnDetached(void *arg) { Free(sync); }
337	uptr size, bool is_write) {
338	if (size == 0)
339	return;
340
341	u64 shadow_mem = (u64)MemToShadow(addr);
342	DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_write=%d\n",
343	thr->tid, (void)pc, (void)addr,
344	(int)size, is_write);
345		322
346	#if SANITIZER_DEBUG	323	void ThreadNotJoined(ThreadState *thr, uptr pc, Tid tid, uptr uid) {
347	if (!IsAppMem(addr)) {	324	CHECK_GT(tid, 0);
348	Printf("Access to non app mem %zx\n", addr);	325	ctx->thread_registry.SetThreadUserId(tid, uid);
349	DCHECK(IsAppMem(addr));	326	}
350	}
351	if (!IsAppMem(addr + size - 1)) {
352	Printf("Access to non app mem %zx\n", addr + size - 1);
353	DCHECK(IsAppMem(addr + size - 1));
354	}
355	if (!IsShadowMem((uptr)shadow_mem)) {
356	Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
357	DCHECK(IsShadowMem((uptr)shadow_mem));
358	}
359	if (!IsShadowMem((uptr)(shadow_mem + size * kShadowCnt / 8 - 1))) {
360	Printf("Bad shadow addr %p (%zx)\n",
361	shadow_mem + size * kShadowCnt / 8 - 1, addr + size - 1);
362	DCHECK(IsShadowMem((uptr)(shadow_mem + size * kShadowCnt / 8 - 1)));
363	}
364	#endif
365
366	if (*shadow_mem == kShadowRodata) {
367	DCHECK(!is_write);
368	// Access to .rodata section, no races here.
369	// Measurements show that it can be 10-20% of all memory accesses.
370	return;
371	}
372
373	FastState fast_state = thr->fast_state;
374	if (fast_state.GetIgnoreBit())
375	return;
376
377	fast_state.IncrementEpoch();
378	thr->fast_state = fast_state;
379	TraceAddEvent(thr, fast_state, EventTypeMop, pc);
380
381	bool unaligned = (addr % kShadowCell) != 0;
382		327
383	// Handle unaligned beginning, if any.	328	void ThreadSetName(ThreadState thr, const char name) {
384	for (; addr % kShadowCell && size; addr++, size--) {	329	ctx->thread_registry.SetThreadName(thr->tid, name);
385	int const kAccessSizeLog = 0;
386	Shadow cur(fast_state);
387	cur.SetWrite(is_write);
388	cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
389	MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false,
390	shadow_mem, cur);
391	}
392	if (unaligned)
393	shadow_mem += kShadowCnt;
394	// Handle middle part, if any.
395	for (; size >= kShadowCell; addr += kShadowCell, size -= kShadowCell) {
396	int const kAccessSizeLog = 3;
397	Shadow cur(fast_state);
398	cur.SetWrite(is_write);
399	cur.SetAddr0AndSizeLog(0, kAccessSizeLog);
400	MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false,
401	shadow_mem, cur);
402	shadow_mem += kShadowCnt;
403	}
404	// Handle ending, if any.
405	for (; size; addr++, size--) {
406	int const kAccessSizeLog = 0;
407	Shadow cur(fast_state);
408	cur.SetWrite(is_write);
409	cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
410	MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false,
411	shadow_mem, cur);
412	}
413	}	330	}
414		331
415	#if !SANITIZER_GO	332	#if !SANITIZER_GO
...	@@ -421,10 +338,10 @@ void FiberSwitchImpl(ThreadState from, ThreadState to) {	...	@@ -421,10 +338,10 @@ void FiberSwitchImpl(ThreadState from, ThreadState to) {
421	}	338	}
422		339
423	ThreadState FiberCreate(ThreadState thr, uptr pc, unsigned flags) {	340	ThreadState FiberCreate(ThreadState thr, uptr pc, unsigned flags) {
424	void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadState));	341	void *mem = Alloc(sizeof(ThreadState));
425	ThreadState fiber = static_cast<ThreadState >(mem);	342	ThreadState fiber = static_cast<ThreadState >(mem);
426	internal_memset(fiber, 0, sizeof(*fiber));	343	internal_memset(fiber, 0, sizeof(*fiber));
427	int tid = ThreadCreate(thr, pc, 0, true);	344	Tid tid = ThreadCreate(thr, pc, 0, true);
428	FiberSwitchImpl(thr, fiber);	345	FiberSwitchImpl(thr, fiber);
429	ThreadStart(fiber, tid, 0, ThreadType::Fiber);	346	ThreadStart(fiber, tid, 0, ThreadType::Fiber);
430	FiberSwitchImpl(fiber, thr);	347	FiberSwitchImpl(fiber, thr);
...	@@ -435,7 +352,7 @@ void FiberDestroy(ThreadState thr, uptr pc, ThreadState fiber) {	...	@@ -435,7 +352,7 @@ void FiberDestroy(ThreadState thr, uptr pc, ThreadState fiber) {
435	FiberSwitchImpl(thr, fiber);	352	FiberSwitchImpl(thr, fiber);
436	ThreadFinish(fiber);	353	ThreadFinish(fiber);
437	FiberSwitchImpl(fiber, thr);	354	FiberSwitchImpl(fiber, thr);
438	internal_free(fiber);	355	Free(fiber);
439	}	356	}
440		357
441	void FiberSwitch(ThreadState *thr, uptr pc,	358	void FiberSwitch(ThreadState *thr, uptr pc,

lib/tsan/tsan_shadow.h created+193

...	@@ -0,0 +1,193 @@
1	//===-- tsan_shadow.h -------------------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#ifndef TSAN_SHADOW_H
10	#define TSAN_SHADOW_H
11
12	#include "tsan_defs.h"
13
14	namespace __tsan {
15
16	class FastState {
17	public:
18	FastState() { Reset(); }
19
20	void Reset() {
21	part_.unused0_ = 0;
22	part_.sid_ = static_cast<u8>(kFreeSid);
23	part_.epoch_ = static_cast<u16>(kEpochLast);
24	part_.unused1_ = 0;
25	part_.ignore_accesses_ = false;
26	}
27
28	void SetSid(Sid sid) { part_.sid_ = static_cast<u8>(sid); }
29
30	Sid sid() const { return static_cast<Sid>(part_.sid_); }
31
32	Epoch epoch() const { return static_cast<Epoch>(part_.epoch_); }
33
34	void SetEpoch(Epoch epoch) { part_.epoch_ = static_cast<u16>(epoch); }
35
36	void SetIgnoreBit() { part_.ignore_accesses_ = 1; }
37	void ClearIgnoreBit() { part_.ignore_accesses_ = 0; }
38	bool GetIgnoreBit() const { return part_.ignore_accesses_; }
39
40	private:
41	friend class Shadow;
42	struct Parts {
43	u32 unused0_ : 8;
44	u32 sid_ : 8;
45	u32 epoch_ : kEpochBits;
46	u32 unused1_ : 1;
47	u32 ignore_accesses_ : 1;
48	};
49	union {
50	Parts part_;
51	u32 raw_;
52	};
53	};
54
55	static_assert(sizeof(FastState) == kShadowSize, "bad FastState size");
56
57	class Shadow {
58	public:
59	static constexpr RawShadow kEmpty = static_cast<RawShadow>(0);
60
61	Shadow(FastState state, u32 addr, u32 size, AccessType typ) {
62	raw_ = state.raw_;
63	DCHECK_GT(size, 0);
64	DCHECK_LE(size, 8);
65	UNUSED Sid sid0 = part_.sid_;
66	UNUSED u16 epoch0 = part_.epoch_;
67	raw_ \|= (!!(typ & kAccessAtomic) << kIsAtomicShift) \|
68	(!!(typ & kAccessRead) << kIsReadShift) \|
69	(((((1u << size) - 1) << (addr & 0x7)) & 0xff) << kAccessShift);
70	// Note: we don't check kAccessAtomic because it overlaps with
71	// FastState::ignore_accesses_ and it may be set spuriously.
72	DCHECK_EQ(part_.is_read_, !!(typ & kAccessRead));
73	DCHECK_EQ(sid(), sid0);
74	DCHECK_EQ(epoch(), epoch0);
75	}
76
77	explicit Shadow(RawShadow x = Shadow::kEmpty) { raw_ = static_cast<u32>(x); }
78
79	RawShadow raw() const { return static_cast<RawShadow>(raw_); }
80	Sid sid() const { return part_.sid_; }
81	Epoch epoch() const { return static_cast<Epoch>(part_.epoch_); }
82	u8 access() const { return part_.access_; }
83
84	void GetAccess(uptr addr, uptr size, AccessType *typ) const {
85	DCHECK(part_.access_ != 0 \|\| raw_ == static_cast<u32>(Shadow::kRodata));
86	if (addr)
87	*addr = part_.access_ ? __builtin_ffs(part_.access_) - 1 : 0;
88	if (size)
89	*size = part_.access_ == kFreeAccess ? kShadowCell
90	: __builtin_popcount(part_.access_);
91	if (typ) {
92	*typ = part_.is_read_ ? kAccessRead : kAccessWrite;
93	if (part_.is_atomic_)
94	*typ \|= kAccessAtomic;
95	if (part_.access_ == kFreeAccess)
96	*typ \|= kAccessFree;
97	}
98	}
99
100	ALWAYS_INLINE
101	bool IsBothReadsOrAtomic(AccessType typ) const {
102	u32 is_read = !!(typ & kAccessRead);
103	u32 is_atomic = !!(typ & kAccessAtomic);
104	bool res =
105	raw_ & ((is_atomic << kIsAtomicShift) \| (is_read << kIsReadShift));
106	DCHECK_EQ(res,
107	(part_.is_read_ && is_read) \|\| (part_.is_atomic_ && is_atomic));
108	return res;
109	}
110
111	ALWAYS_INLINE
112	bool IsRWWeakerOrEqual(AccessType typ) const {
113	u32 is_read = !!(typ & kAccessRead);
114	u32 is_atomic = !!(typ & kAccessAtomic);
115	UNUSED u32 res0 =
116	(part_.is_atomic_ > is_atomic) \|\|
117	(part_.is_atomic_ == is_atomic && part_.is_read_ >= is_read);
118	#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
119	const u32 kAtomicReadMask = (1 << kIsAtomicShift) \| (1 << kIsReadShift);
120	bool res = (raw_ & kAtomicReadMask) >=
121	((is_atomic << kIsAtomicShift) \| (is_read << kIsReadShift));
122
123	DCHECK_EQ(res, res0);
124	return res;
125	#else
126	return res0;
127	#endif
128	}
129
130	// The FreedMarker must not pass "the same access check" so that we don't
131	// return from the race detection algorithm early.
132	static RawShadow FreedMarker() {
133	FastState fs;
134	fs.SetSid(kFreeSid);
135	fs.SetEpoch(kEpochLast);
136	Shadow s(fs, 0, 8, kAccessWrite);
137	return s.raw();
138	}
139
140	static RawShadow FreedInfo(Sid sid, Epoch epoch) {
141	Shadow s;
142	s.part_.sid_ = sid;
143	s.part_.epoch_ = static_cast<u16>(epoch);
144	s.part_.access_ = kFreeAccess;
145	return s.raw();
146	}
147
148	private:
149	struct Parts {
150	u8 access_;
151	Sid sid_;
152	u16 epoch_ : kEpochBits;
153	u16 is_read_ : 1;
154	u16 is_atomic_ : 1;
155	};
156	union {
157	Parts part_;
158	u32 raw_;
159	};
160
161	static constexpr u8 kFreeAccess = 0x81;
162
163	#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
164	static constexpr uptr kAccessShift = 0;
165	static constexpr uptr kIsReadShift = 30;
166	static constexpr uptr kIsAtomicShift = 31;
167	#else
168	static constexpr uptr kAccessShift = 24;
169	static constexpr uptr kIsReadShift = 1;
170	static constexpr uptr kIsAtomicShift = 0;
171	#endif
172
173	public:
174	// .rodata shadow marker, see MapRodata and ContainsSameAccessFast.
175	static constexpr RawShadow kRodata =
176	static_cast<RawShadow>(1 << kIsReadShift);
177	};
178
179	static_assert(sizeof(Shadow) == kShadowSize, "bad Shadow size");
180
181	ALWAYS_INLINE RawShadow LoadShadow(RawShadow *p) {
182	return static_cast<RawShadow>(
183	atomic_load((atomic_uint32_t *)p, memory_order_relaxed));
184	}
185
186	ALWAYS_INLINE void StoreShadow(RawShadow *sp, RawShadow s) {
187	atomic_store((atomic_uint32_t *)sp, static_cast<u32>(s),
188	memory_order_relaxed);
189	}
190
191	} // namespace __tsan
192
193	#endif

lib/tsan/tsan_spinlock_defs_mac.h created+45

...	@@ -0,0 +1,45 @@
1	//===-- tsan_spinlock_defs_mac.h -------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	// Mac-specific forward-declared function defintions that may be
12	// deprecated in later versions of the OS.
13	// These are needed for interceptors.
14	//
15	//===----------------------------------------------------------------------===//
16
17	#if SANITIZER_APPLE
18
19	#ifndef TSAN_SPINLOCK_DEFS_MAC_H
20	#define TSAN_SPINLOCK_DEFS_MAC_H
21
22	#include <stdint.h>
23
24	extern "C" {
25
26	/*
27	Provides forward declarations related to OSSpinLocks on Darwin. These functions are
28	deprecated on macOS version 10.12 and later,
29	and are no longer included in the system headers.
30
31	However, the symbols are still available on the system, so we provide these forward
32	declarations to prevent compilation errors in tsan_interceptors_mac.cpp, which
33	references these functions when defining TSAN interceptor functions.
34	*/
35
36	typedef int32_t OSSpinLock;
37
38	void OSSpinLockLock(volatile OSSpinLock *__lock);
39	void OSSpinLockUnlock(volatile OSSpinLock *__lock);
40	bool OSSpinLockTry(volatile OSSpinLock *__lock);
41
42	}
43
44	#endif //TSAN_SPINLOCK_DEFS_MAC_H
45	#endif // SANITIZER_APPLE

lib/tsan/tsan_stack_trace.cpp+4-8

 @@ -23,14 +23,10 @@ VarSizeStackTrace::~VarSizeStackTrace() {
 }
 void VarSizeStackTrace::ResizeBuffer(uptr new_size) {
-  if (trace_buffer) {
+  Free(trace_buffer);
-    internal_free(trace_buffer);
+  trace_buffer = (new_size > 0)
-  }
+                     ? (uptr *)Alloc(new_size * sizeof(trace_buffer[0]))
-  trace_buffer =
+                     : nullptr;
-      (new_size > 0)
-          ? (uptr *)internal_alloc(MBlockStackTrace,
-                                   new_size * sizeof(trace_buffer[0]))
-          : nullptr;
   trace = trace_buffer;
   size = new_size;
 }

lib/tsan/tsan_suppressions.cpp+3-2

 @@ -10,15 +10,16 @@
 //
 //===----------------------------------------------------------------------===//
+#include "tsan_suppressions.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_placement_new.h"
 #include "sanitizer_common/sanitizer_suppressions.h"
-#include "tsan_suppressions.h"
-#include "tsan_rtl.h"
 #include "tsan_flags.h"
 #include "tsan_mman.h"
 #include "tsan_platform.h"
+#include "tsan_rtl.h"
 #if !SANITIZER_GO
 // Suppressions for true/false positives in standard libraries.

lib/tsan/tsan_symbolize.cpp+2-1

...	@@ -110,7 +110,8 @@ ReportLocation *SymbolizeData(uptr addr) {	...	@@ -110,7 +110,8 @@ ReportLocation *SymbolizeData(uptr addr) {
110	DataInfo info;	110	DataInfo info;
111	if (!Symbolizer::GetOrInit()->SymbolizeData(addr, &info))	111	if (!Symbolizer::GetOrInit()->SymbolizeData(addr, &info))
112	return 0;	112	return 0;
113	ReportLocation *ent = ReportLocation::New(ReportLocationGlobal);	113	auto *ent = New<ReportLocation>();
		114	ent->type = ReportLocationGlobal;
114	internal_memcpy(&ent->global, &info, sizeof(info));	115	internal_memcpy(&ent->global, &info, sizeof(info));
115	return ent;	116	return ent;
116	}	117	}

lib/tsan/tsan_sync.cpp+64-68

...	@@ -18,42 +18,31 @@ namespace __tsan {	...	@@ -18,42 +18,31 @@ namespace __tsan {
18		18
19	void DDMutexInit(ThreadState thr, uptr pc, SyncVar s);	19	void DDMutexInit(ThreadState thr, uptr pc, SyncVar s);
20		20
21	SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(0); }	21	SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(); }
22		22
23	void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, u64 uid) {	23	void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, bool save_stack) {
		24	Reset();
24	this->addr = addr;	25	this->addr = addr;
25	this->uid = uid;	26	next = 0;
26	this->next = 0;	27	if (save_stack && !SANITIZER_GO) // Go does not use them
27
28	creation_stack_id = 0;
29	if (!SANITIZER_GO) // Go does not use them
30	creation_stack_id = CurrentStackId(thr, pc);	28	creation_stack_id = CurrentStackId(thr, pc);
31	if (common_flags()->detect_deadlocks)	29	if (common_flags()->detect_deadlocks)
32	DDMutexInit(thr, pc, this);	30	DDMutexInit(thr, pc, this);
33	}	31	}
34		32
35	void SyncVar::Reset(Processor *proc) {	33	void SyncVar::Reset() {
36	uid = 0;	34	CHECK(!ctx->resetting);
37	creation_stack_id = 0;	35	creation_stack_id = kInvalidStackID;
38	owner_tid = kInvalidTid;	36	owner_tid = kInvalidTid;
39	last_lock = 0;	37	last_lock.Reset();
40	recursion = 0;	38	recursion = 0;
41	atomic_store_relaxed(&flags, 0);	39	atomic_store_relaxed(&flags, 0);
42		40	Free(clock);
43	if (proc == 0) {	41	Free(read_clock);
44	CHECK_EQ(clock.size(), 0);
45	CHECK_EQ(read_clock.size(), 0);
46	} else {
47	clock.Reset(&proc->clock_cache);
48	read_clock.Reset(&proc->clock_cache);
49	}
50	}	42	}
51		43
52	MetaMap::MetaMap()	44	MetaMap::MetaMap()
53	: block_alloc_(LINKER_INITIALIZED, "heap block allocator"),	45	: block_alloc_("heap block allocator"), sync_alloc_("sync allocator") {}
54	sync_alloc_(LINKER_INITIALIZED, "sync allocator") {
55	atomic_store(&uid_gen_, 0, memory_order_relaxed);
56	}
57		46
58	void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) {	47	void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) {
59	u32 idx = block_alloc_.Alloc(&thr->proc()->block_cache);	48	u32 idx = block_alloc_.Alloc(&thr->proc()->block_cache);
...	@@ -67,16 +56,16 @@ void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) {	...	@@ -67,16 +56,16 @@ void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) {
67	*meta = idx \| kFlagBlock;	56	*meta = idx \| kFlagBlock;
68	}	57	}
69		58
70	uptr MetaMap::FreeBlock(Processor *proc, uptr p) {	59	uptr MetaMap::FreeBlock(Processor *proc, uptr p, bool reset) {
71	MBlock* b = GetBlock(p);	60	MBlock* b = GetBlock(p);
72	if (b == 0)	61	if (b == 0)
73	return 0;	62	return 0;
74	uptr sz = RoundUpTo(b->siz, kMetaShadowCell);	63	uptr sz = RoundUpTo(b->siz, kMetaShadowCell);
75	FreeRange(proc, p, sz);	64	FreeRange(proc, p, sz, reset);
76	return sz;	65	return sz;
77	}	66	}
78		67
79	bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) {	68	bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz, bool reset) {
80	bool has_something = false;	69	bool has_something = false;
81	u32 *meta = MemToMeta(p);	70	u32 *meta = MemToMeta(p);
82	u32 *end = MemToMeta(p + sz);	71	u32 *end = MemToMeta(p + sz);
...	@@ -98,7 +87,8 @@ bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) {	...	@@ -98,7 +87,8 @@ bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) {
98	DCHECK(idx & kFlagSync);	87	DCHECK(idx & kFlagSync);
99	SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask);	88	SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask);
100	u32 next = s->next;	89	u32 next = s->next;
101	s->Reset(proc);	90	if (reset)
		91	s->Reset();
102	sync_alloc_.Free(&proc->sync_cache, idx & ~kFlagMask);	92	sync_alloc_.Free(&proc->sync_cache, idx & ~kFlagMask);
103	idx = next;	93	idx = next;
104	} else {	94	} else {
...	@@ -115,30 +105,30 @@ bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) {	...	@@ -115,30 +105,30 @@ bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz) {
115	// which can be huge. The function probes pages one-by-one until it finds a page	105	// which can be huge. The function probes pages one-by-one until it finds a page
116	// without meta objects, at this point it stops freeing meta objects. Because	106	// without meta objects, at this point it stops freeing meta objects. Because
117	// thread stacks grow top-down, we do the same starting from end as well.	107	// thread stacks grow top-down, we do the same starting from end as well.
118	void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {	108	void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz, bool reset) {
119	if (SANITIZER_GO) {	109	if (SANITIZER_GO) {
120	// UnmapOrDie/MmapFixedNoReserve does not work on Windows,	110	// UnmapOrDie/MmapFixedNoReserve does not work on Windows,
121	// so we do the optimization only for C/C++.	111	// so we do the optimization only for C/C++.
122	FreeRange(proc, p, sz);	112	FreeRange(proc, p, sz, reset);
123	return;	113	return;
124	}	114	}
125	const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize;	115	const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize;
126	const uptr kPageSize = GetPageSizeCached() * kMetaRatio;	116	const uptr kPageSize = GetPageSizeCached() * kMetaRatio;
127	if (sz <= 4 * kPageSize) {	117	if (sz <= 4 * kPageSize) {
128	// If the range is small, just do the normal free procedure.	118	// If the range is small, just do the normal free procedure.
129	FreeRange(proc, p, sz);	119	FreeRange(proc, p, sz, reset);
130	return;	120	return;
131	}	121	}
132	// First, round both ends of the range to page size.	122	// First, round both ends of the range to page size.
133	uptr diff = RoundUp(p, kPageSize) - p;	123	uptr diff = RoundUp(p, kPageSize) - p;
134	if (diff != 0) {	124	if (diff != 0) {
135	FreeRange(proc, p, diff);	125	FreeRange(proc, p, diff, reset);
136	p += diff;	126	p += diff;
137	sz -= diff;	127	sz -= diff;
138	}	128	}
139	diff = p + sz - RoundDown(p + sz, kPageSize);	129	diff = p + sz - RoundDown(p + sz, kPageSize);
140	if (diff != 0) {	130	if (diff != 0) {
141	FreeRange(proc, p + sz - diff, diff);	131	FreeRange(proc, p + sz - diff, diff, reset);
142	sz -= diff;	132	sz -= diff;
143	}	133	}
144	// Now we must have a non-empty page-aligned range.	134	// Now we must have a non-empty page-aligned range.
...	@@ -149,7 +139,7 @@ void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {	...	@@ -149,7 +139,7 @@ void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
149	const uptr sz0 = sz;	139	const uptr sz0 = sz;
150	// Probe start of the range.	140	// Probe start of the range.
151	for (uptr checked = 0; sz > 0; checked += kPageSize) {	141	for (uptr checked = 0; sz > 0; checked += kPageSize) {
152	bool has_something = FreeRange(proc, p, kPageSize);	142	bool has_something = FreeRange(proc, p, kPageSize, reset);
153	p += kPageSize;	143	p += kPageSize;
154	sz -= kPageSize;	144	sz -= kPageSize;
155	if (!has_something && checked > (128 << 10))	145	if (!has_something && checked > (128 << 10))
...	@@ -157,7 +147,7 @@ void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {	...	@@ -157,7 +147,7 @@ void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
157	}	147	}
158	// Probe end of the range.	148	// Probe end of the range.
159	for (uptr checked = 0; sz > 0; checked += kPageSize) {	149	for (uptr checked = 0; sz > 0; checked += kPageSize) {
160	bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize);	150	bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize, reset);
161	sz -= kPageSize;	151	sz -= kPageSize;
162	// Stacks grow down, so sync object are most likely at the end of the region	152	// Stacks grow down, so sync object are most likely at the end of the region
163	// (if it is a stack). The very end of the stack is TLS and tsan increases	153	// (if it is a stack). The very end of the stack is TLS and tsan increases
...	@@ -176,6 +166,27 @@ void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {	...	@@ -176,6 +166,27 @@ void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz) {
176	Die();	166	Die();
177	}	167	}
178		168
		169	void MetaMap::ResetClocks() {
		170	// This can be called from the background thread
		171	// which does not have proc/cache.
		172	// The cache is too large for stack.
		173	static InternalAllocatorCache cache;
		174	internal_memset(&cache, 0, sizeof(cache));
		175	internal_allocator()->InitCache(&cache);
		176	sync_alloc_.ForEach([&](SyncVar *s) {
		177	if (s->clock) {
		178	InternalFree(s->clock, &cache);
		179	s->clock = nullptr;
		180	}
		181	if (s->read_clock) {
		182	InternalFree(s->read_clock, &cache);
		183	s->read_clock = nullptr;
		184	}
		185	s->last_lock.Reset();
		186	});
		187	internal_allocator()->DestroyCache(&cache);
		188	}
		189
179	MBlock* MetaMap::GetBlock(uptr p) {	190	MBlock* MetaMap::GetBlock(uptr p) {
180	u32 *meta = MemToMeta(p);	191	u32 *meta = MemToMeta(p);
181	u32 idx = *meta;	192	u32 idx = *meta;
...	@@ -190,63 +201,41 @@ MBlock* MetaMap::GetBlock(uptr p) {	...	@@ -190,63 +201,41 @@ MBlock* MetaMap::GetBlock(uptr p) {
190	}	201	}
191	}	202	}
192		203
193	SyncVar* MetaMap::GetOrCreateAndLock(ThreadState *thr, uptr pc,	204	SyncVar MetaMap::GetSync(ThreadState thr, uptr pc, uptr addr, bool create,
194	uptr addr, bool write_lock) {	205	bool save_stack) {
195	return GetAndLock(thr, pc, addr, write_lock, true);	206	DCHECK(!create \|\| thr->slot_locked);
196	}
197
198	SyncVar* MetaMap::GetIfExistsAndLock(uptr addr, bool write_lock) {
199	return GetAndLock(0, 0, addr, write_lock, false);
200	}
201
202	SyncVar MetaMap::GetAndLock(ThreadState thr, uptr pc, uptr addr, bool write_lock,
203	bool create) NO_THREAD_SAFETY_ANALYSIS {
204	u32 *meta = MemToMeta(addr);	207	u32 *meta = MemToMeta(addr);
205	u32 idx0 = *meta;	208	u32 idx0 = *meta;
206	u32 myidx = 0;	209	u32 myidx = 0;
207	SyncVar *mys = 0;	210	SyncVar *mys = nullptr;
208	for (;;) {	211	for (;;) {
209	u32 idx = idx0;	212	for (u32 idx = idx0; idx && !(idx & kFlagBlock);) {
210	for (;;) {
211	if (idx == 0)
212	break;
213	if (idx & kFlagBlock)
214	break;
215	DCHECK(idx & kFlagSync);	213	DCHECK(idx & kFlagSync);
216	SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask);	214	SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask);
217	if (s->addr == addr) {	215	if (LIKELY(s->addr == addr)) {
218	if (myidx != 0) {	216	if (UNLIKELY(myidx != 0)) {
219	mys->Reset(thr->proc());	217	mys->Reset();
220	sync_alloc_.Free(&thr->proc()->sync_cache, myidx);	218	sync_alloc_.Free(&thr->proc()->sync_cache, myidx);
221	}	219	}
222	if (write_lock)
223	s->mtx.Lock();
224	else
225	s->mtx.ReadLock();
226	return s;	220	return s;
227	}	221	}
228	idx = s->next;	222	idx = s->next;
229	}	223	}
230	if (!create)	224	if (!create)
231	return 0;	225	return nullptr;
232	if (*meta != idx0) {	226	if (UNLIKELY(*meta != idx0)) {
233	idx0 = *meta;	227	idx0 = *meta;
234	continue;	228	continue;
235	}	229	}
236		230
237	if (myidx == 0) {	231	if (LIKELY(myidx == 0)) {
238	const u64 uid = atomic_fetch_add(&uid_gen_, 1, memory_order_relaxed);
239	myidx = sync_alloc_.Alloc(&thr->proc()->sync_cache);	232	myidx = sync_alloc_.Alloc(&thr->proc()->sync_cache);
240	mys = sync_alloc_.Map(myidx);	233	mys = sync_alloc_.Map(myidx);
241	mys->Init(thr, pc, addr, uid);	234	mys->Init(thr, pc, addr, save_stack);
242	}	235	}
243	mys->next = idx0;	236	mys->next = idx0;
244	if (atomic_compare_exchange_strong((atomic_uint32_t*)meta, &idx0,	237	if (atomic_compare_exchange_strong((atomic_uint32_t*)meta, &idx0,
245	myidx \| kFlagSync, memory_order_release)) {	238	myidx \| kFlagSync, memory_order_release)) {
246	if (write_lock)
247	mys->mtx.Lock();
248	else
249	mys->mtx.ReadLock();
250	return mys;	239	return mys;
251	}	240	}
252	}	241	}
...	@@ -290,4 +279,11 @@ void MetaMap::OnProcIdle(Processor *proc) {	...	@@ -290,4 +279,11 @@ void MetaMap::OnProcIdle(Processor *proc) {
290	sync_alloc_.FlushCache(&proc->sync_cache);	279	sync_alloc_.FlushCache(&proc->sync_cache);
291	}	280	}
292		281
		282	MetaMap::MemoryStats MetaMap::GetMemoryStats() const {
		283	MemoryStats stats;
		284	stats.mem_block = block_alloc_.AllocatedMemory();
		285	stats.sync_obj = sync_alloc_.AllocatedMemory();
		286	return stats;
		287	}
		288
293	} // namespace __tsan	289	} // namespace __tsan

lib/tsan/tsan_sync.h+43-37

...	@@ -16,8 +16,9 @@	...	@@ -16,8 +16,9 @@
16	#include "sanitizer_common/sanitizer_common.h"	16	#include "sanitizer_common/sanitizer_common.h"
17	#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"	17	#include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
18	#include "tsan_defs.h"	18	#include "tsan_defs.h"
19	#include "tsan_clock.h"
20	#include "tsan_dense_alloc.h"	19	#include "tsan_dense_alloc.h"
		20	#include "tsan_shadow.h"
		21	#include "tsan_vector_clock.h"
21		22
22	namespace __tsan {	23	namespace __tsan {
23		24
...	@@ -46,39 +47,25 @@ enum MutexFlags {	...	@@ -46,39 +47,25 @@ enum MutexFlags {
46	MutexFlagNotStatic,	47	MutexFlagNotStatic,
47	};	48	};
48		49
		50	// SyncVar is a descriptor of a user synchronization object
		51	// (mutex or an atomic variable).
49	struct SyncVar {	52	struct SyncVar {
50	SyncVar();	53	SyncVar();
51		54
52	uptr addr; // overwritten by DenseSlabAlloc freelist	55	uptr addr; // overwritten by DenseSlabAlloc freelist
53	Mutex mtx;	56	Mutex mtx;
54	u64 uid; // Globally unique id.	57	StackID creation_stack_id;
55	u32 creation_stack_id;	58	Tid owner_tid; // Set only by exclusive owners.
56	u32 owner_tid; // Set only by exclusive owners.	59	FastState last_lock;
57	u64 last_lock;
58	int recursion;	60	int recursion;
59	atomic_uint32_t flags;	61	atomic_uint32_t flags;
60	u32 next; // in MetaMap	62	u32 next; // in MetaMap
61	DDMutex dd;	63	DDMutex dd;
62	SyncClock read_clock; // Used for rw mutexes only.	64	VectorClock *read_clock; // Used for rw mutexes only.
63	// The clock is placed last, so that it is situated on a different cache line	65	VectorClock *clock;
64	// with the mtx. This reduces contention for hot sync objects.
65	SyncClock clock;
66		66
67	void Init(ThreadState *thr, uptr pc, uptr addr, u64 uid);	67	void Init(ThreadState *thr, uptr pc, uptr addr, bool save_stack);
68	void Reset(Processor *proc);	68	void Reset();
69
70	u64 GetId() const {
71	// 48 lsb is addr, then 14 bits is low part of uid, then 2 zero bits.
72	return GetLsb((u64)addr \| (uid << 48), 60);
73	}
74	bool CheckId(u64 uid) const {
75	CHECK_EQ(uid, GetLsb(uid, 14));
76	return GetLsb(this->uid, 14) == uid;
77	}
78	static uptr SplitId(u64 id, u64 *uid) {
79	*uid = id >> 48;
80	return (uptr)GetLsb(id, 48);
81	}
82		69
83	bool IsFlagSet(u32 f) const {	70	bool IsFlagSet(u32 f) const {
84	return atomic_load_relaxed(&flags) & f;	71	return atomic_load_relaxed(&flags) & f;
...	@@ -101,28 +88,48 @@ struct SyncVar {	...	@@ -101,28 +88,48 @@ struct SyncVar {
101	}	88	}
102	};	89	};
103		90
104	/* MetaMap allows to map arbitrary user pointers onto various descriptors.	91	// MetaMap maps app addresses to heap block (MBlock) and sync var (SyncVar)
105	Currently it maps pointers to heap block descriptors and sync var descs.	92	// descriptors. It uses 1/2 direct shadow, see tsan_platform.h for the mapping.
106	It uses 1/2 direct shadow, see tsan_platform.h.
107	*/
108	class MetaMap {	93	class MetaMap {
109	public:	94	public:
110	MetaMap();	95	MetaMap();
111		96
112	void AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz);	97	void AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz);
113	uptr FreeBlock(Processor *proc, uptr p);	98
114	bool FreeRange(Processor *proc, uptr p, uptr sz);	99	// FreeBlock resets all sync objects in the range if reset=true and must not
115	void ResetRange(Processor *proc, uptr p, uptr sz);	100	// run concurrently with ResetClocks which resets all sync objects
		101	// w/o any synchronization (as part of DoReset).
		102	// If we don't have a thread slot (very early/late in thread lifetime or
		103	// Go/Java callbacks) or the slot is not locked, then reset must be set to
		104	// false. In such case sync object clocks will be reset later (when it's
		105	// reused or during the next ResetClocks).
		106	uptr FreeBlock(Processor *proc, uptr p, bool reset);
		107	bool FreeRange(Processor *proc, uptr p, uptr sz, bool reset);
		108	void ResetRange(Processor *proc, uptr p, uptr sz, bool reset);
		109	// Reset vector clocks of all sync objects.
		110	// Must be called when no other threads access sync objects.
		111	void ResetClocks();
116	MBlock* GetBlock(uptr p);	112	MBlock* GetBlock(uptr p);
117		113
118	SyncVar* GetOrCreateAndLock(ThreadState *thr, uptr pc,	114	SyncVar GetSyncOrCreate(ThreadState thr, uptr pc, uptr addr,
119	uptr addr, bool write_lock);	115	bool save_stack) {
120	SyncVar* GetIfExistsAndLock(uptr addr, bool write_lock);	116	return GetSync(thr, pc, addr, true, save_stack);
		117	}
		118	SyncVar *GetSyncIfExists(uptr addr) {
		119	return GetSync(nullptr, 0, addr, false, false);
		120	}
121		121
122	void MoveMemory(uptr src, uptr dst, uptr sz);	122	void MoveMemory(uptr src, uptr dst, uptr sz);
123		123
124	void OnProcIdle(Processor *proc);	124	void OnProcIdle(Processor *proc);
125		125
		126	struct MemoryStats {
		127	uptr mem_block;
		128	uptr sync_obj;
		129	};
		130
		131	MemoryStats GetMemoryStats() const;
		132
126	private:	133	private:
127	static const u32 kFlagMask = 3u << 30;	134	static const u32 kFlagMask = 3u << 30;
128	static const u32 kFlagBlock = 1u << 30;	135	static const u32 kFlagBlock = 1u << 30;
...	@@ -131,10 +138,9 @@ class MetaMap {	...	@@ -131,10 +138,9 @@ class MetaMap {
131	typedef DenseSlabAlloc<SyncVar, 1 << 20, 1 << 10, kFlagMask> SyncAlloc;	138	typedef DenseSlabAlloc<SyncVar, 1 << 20, 1 << 10, kFlagMask> SyncAlloc;
132	BlockAlloc block_alloc_;	139	BlockAlloc block_alloc_;
133	SyncAlloc sync_alloc_;	140	SyncAlloc sync_alloc_;
134	atomic_uint64_t uid_gen_;
135		141
136	SyncVar* GetAndLock(ThreadState *thr, uptr pc, uptr addr, bool write_lock,	142	SyncVar GetSync(ThreadState thr, uptr pc, uptr addr, bool create,
137	bool create);	143	bool save_stack);
138	};	144	};
139		145
140	} // namespace __tsan	146	} // namespace __tsan

lib/tsan/tsan_trace.h+180-37

...	@@ -13,58 +13,201 @@	...	@@ -13,58 +13,201 @@
13	#define TSAN_TRACE_H	13	#define TSAN_TRACE_H
14		14
15	#include "tsan_defs.h"	15	#include "tsan_defs.h"
16	#include "tsan_stack_trace.h"	16	#include "tsan_ilist.h"
17	#include "tsan_mutexset.h"	17	#include "tsan_mutexset.h"
		18	#include "tsan_stack_trace.h"
18		19
19	namespace __tsan {	20	namespace __tsan {
20		21
21	const int kTracePartSizeBits = 13;	22	enum class EventType : u64 {
22	const int kTracePartSize = 1 << kTracePartSizeBits;	23	kAccessExt,
23	const int kTraceParts = 2 * 1024 * 1024 / kTracePartSize;	24	kAccessRange,
24	const int kTraceSize = kTracePartSize * kTraceParts;	25	kLock,
25		26	kRLock,
26	// Must fit into 3 bits.	27	kUnlock,
27	enum EventType {	28	kTime,
28	EventTypeMop,	29	};
29	EventTypeFuncEnter,	30
30	EventTypeFuncExit,	31	// "Base" type for all events for type dispatch.
31	EventTypeLock,	32	struct Event {
32	EventTypeUnlock,	33	// We use variable-length type encoding to give more bits to some event
33	EventTypeRLock,	34	// types that need them. If is_access is set, this is EventAccess.
34	EventTypeRUnlock	35	// Otherwise, if is_func is set, this is EventFunc.
		36	// Otherwise type denotes the type.
		37	u64 is_access : 1;
		38	u64 is_func : 1;
		39	EventType type : 3;
		40	u64 _ : 59;
		41	};
		42	static_assert(sizeof(Event) == 8, "bad Event size");
		43
		44	// Nop event used as padding and does not affect state during replay.
		45	static constexpr Event NopEvent = {1, 0, EventType::kAccessExt, 0};
		46
		47	// Compressed memory access can represent only some events with PCs
		48	// close enough to each other. Otherwise we fall back to EventAccessExt.
		49	struct EventAccess {
		50	static constexpr uptr kPCBits = 15;
		51	static_assert(kPCBits + kCompressedAddrBits + 5 == 64,
		52	"unused bits in EventAccess");
		53
		54	u64 is_access : 1; // = 1
		55	u64 is_read : 1;
		56	u64 is_atomic : 1;
		57	u64 size_log : 2;
		58	u64 pc_delta : kPCBits; // signed delta from the previous memory access PC
		59	u64 addr : kCompressedAddrBits;
35	};	60	};
		61	static_assert(sizeof(EventAccess) == 8, "bad EventAccess size");
36		62
37	// Represents a thread event (from most significant bit):	63	// Function entry (pc != 0) or exit (pc == 0).
38	// u64 typ : 3; // EventType.	64	struct EventFunc {
39	// u64 addr : 61; // Associated pc.	65	u64 is_access : 1; // = 0
40	typedef u64 Event;	66	u64 is_func : 1; // = 1
		67	u64 pc : 62;
		68	};
		69	static_assert(sizeof(EventFunc) == 8, "bad EventFunc size");
		70
		71	// Extended memory access with full PC.
		72	struct EventAccessExt {
		73	// Note: precisely specifying the unused parts of the bitfield is critical for
		74	// performance. If we don't specify them, compiler will generate code to load
		75	// the old value and shuffle it to extract the unused bits to apply to the new
		76	// value. If we specify the unused part and store 0 in there, all that
		77	// unnecessary code goes away (store of the 0 const is combined with other
		78	// constant parts).
		79	static constexpr uptr kUnusedBits = 11;
		80	static_assert(kCompressedAddrBits + kUnusedBits + 9 == 64,
		81	"unused bits in EventAccessExt");
		82
		83	u64 is_access : 1; // = 0
		84	u64 is_func : 1; // = 0
		85	EventType type : 3; // = EventType::kAccessExt
		86	u64 is_read : 1;
		87	u64 is_atomic : 1;
		88	u64 size_log : 2;
		89	u64 _ : kUnusedBits;
		90	u64 addr : kCompressedAddrBits;
		91	u64 pc;
		92	};
		93	static_assert(sizeof(EventAccessExt) == 16, "bad EventAccessExt size");
		94
		95	// Access to a memory range.
		96	struct EventAccessRange {
		97	static constexpr uptr kSizeLoBits = 13;
		98	static_assert(kCompressedAddrBits + kSizeLoBits + 7 == 64,
		99	"unused bits in EventAccessRange");
		100
		101	u64 is_access : 1; // = 0
		102	u64 is_func : 1; // = 0
		103	EventType type : 3; // = EventType::kAccessRange
		104	u64 is_read : 1;
		105	u64 is_free : 1;
		106	u64 size_lo : kSizeLoBits;
		107	u64 pc : kCompressedAddrBits;
		108	u64 addr : kCompressedAddrBits;
		109	u64 size_hi : 64 - kCompressedAddrBits;
		110	};
		111	static_assert(sizeof(EventAccessRange) == 16, "bad EventAccessRange size");
41		112
42	const uptr kEventPCBits = 61;	113	// Mutex lock.
		114	struct EventLock {
		115	static constexpr uptr kStackIDLoBits = 15;
		116	static constexpr uptr kStackIDHiBits =
		117	sizeof(StackID) * kByteBits - kStackIDLoBits;
		118	static constexpr uptr kUnusedBits = 3;
		119	static_assert(kCompressedAddrBits + kStackIDLoBits + 5 == 64,
		120	"unused bits in EventLock");
		121	static_assert(kCompressedAddrBits + kStackIDHiBits + kUnusedBits == 64,
		122	"unused bits in EventLock");
		123
		124	u64 is_access : 1; // = 0
		125	u64 is_func : 1; // = 0
		126	EventType type : 3; // = EventType::kLock or EventType::kRLock
		127	u64 pc : kCompressedAddrBits;
		128	u64 stack_lo : kStackIDLoBits;
		129	u64 stack_hi : sizeof(StackID) * kByteBits - kStackIDLoBits;
		130	u64 _ : kUnusedBits;
		131	u64 addr : kCompressedAddrBits;
		132	};
		133	static_assert(sizeof(EventLock) == 16, "bad EventLock size");
		134
		135	// Mutex unlock.
		136	struct EventUnlock {
		137	static constexpr uptr kUnusedBits = 15;
		138	static_assert(kCompressedAddrBits + kUnusedBits + 5 == 64,
		139	"unused bits in EventUnlock");
		140
		141	u64 is_access : 1; // = 0
		142	u64 is_func : 1; // = 0
		143	EventType type : 3; // = EventType::kUnlock
		144	u64 _ : kUnusedBits;
		145	u64 addr : kCompressedAddrBits;
		146	};
		147	static_assert(sizeof(EventUnlock) == 8, "bad EventUnlock size");
		148
		149	// Time change event.
		150	struct EventTime {
		151	static constexpr uptr kUnusedBits = 37;
		152	static_assert(kUnusedBits + sizeof(Sid) * kByteBits + kEpochBits + 5 == 64,
		153	"unused bits in EventTime");
		154
		155	u64 is_access : 1; // = 0
		156	u64 is_func : 1; // = 0
		157	EventType type : 3; // = EventType::kTime
		158	u64 sid : sizeof(Sid) * kByteBits;
		159	u64 epoch : kEpochBits;
		160	u64 _ : kUnusedBits;
		161	};
		162	static_assert(sizeof(EventTime) == 8, "bad EventTime size");
		163
		164	struct Trace;
43		165
44	struct TraceHeader {	166	struct TraceHeader {
45	#if !SANITIZER_GO	167	Trace* trace = nullptr; // back-pointer to Trace containing this part
46	BufferedStackTrace stack0; // Start stack for the trace.	168	INode trace_parts; // in Trace::parts
47	#else	169	INode global; // in Contex::trace_part_recycle
48	VarSizeStackTrace stack0;
49	#endif
50	u64 epoch0; // Start epoch for the trace.
51	MutexSet mset0;
52
53	TraceHeader() : stack0(), epoch0() {}
54	};	170	};
55		171
		172	struct TracePart : TraceHeader {
		173	// There are a lot of goroutines in Go, so we use smaller parts.
		174	static constexpr uptr kByteSize = (SANITIZER_GO ? 128 : 256) << 10;
		175	static constexpr uptr kSize =
		176	(kByteSize - sizeof(TraceHeader)) / sizeof(Event);
		177	// TraceAcquire does a fast event pointer overflow check by comparing
		178	// pointer into TracePart::events with kAlignment mask. Since TracePart's
		179	// are allocated page-aligned, this check detects end of the array
		180	// (it also have false positives in the middle that are filtered separately).
		181	// This also requires events to be the last field.
		182	static constexpr uptr kAlignment = 0xff0;
		183	Event events[kSize];
		184
		185	TracePart() {}
		186	};
		187	static_assert(sizeof(TracePart) == TracePart::kByteSize, "bad TracePart size");
		188
56	struct Trace {	189	struct Trace {
57	Mutex mtx;	190	Mutex mtx;
58	#if !SANITIZER_GO	191	IList<TraceHeader, &TraceHeader::trace_parts, TracePart> parts;
59	// Must be last to catch overflow as paging fault.	192	// First node non-queued into ctx->trace_part_recycle.
60	// Go shadow stack is dynamically allocated.	193	TracePart* local_head;
61	uptr shadow_stack[kShadowStackSize];	194	// Final position in the last part for finished threads.
62	#endif	195	Event* final_pos = nullptr;
63	// Must be the last field, because we unmap the unused part in	196	// Number of trace parts allocated on behalf of this trace specifically.
64	// CreateThreadContext.	197	// Total number of parts in this trace can be larger if we retake some
65	TraceHeader headers[kTraceParts];	198	// parts from other traces.
		199	uptr parts_allocated = 0;
66		200
67	Trace() : mtx(MutexTypeTrace) {}	201	Trace() : mtx(MutexTypeTrace) {}
		202
		203	// We need at least 3 parts per thread, because we want to keep at last
		204	// 2 parts per thread that are not queued into ctx->trace_part_recycle
		205	// (the current one being filled and one full part that ensures that
		206	// we always have at least one part worth of previous memory accesses).
		207	static constexpr uptr kMinParts = 3;
		208
		209	static constexpr uptr kFinishedThreadLo = 16;
		210	static constexpr uptr kFinishedThreadHi = 64;
68	};	211	};
69		212
70	} // namespace __tsan	213	} // namespace __tsan

lib/tsan/tsan_update_shadow_word_inl.h deleted-59

...	@@ -1,59 +0,0 @@
1	//===-- tsan_update_shadow_word_inl.h ---------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	// Body of the hottest inner loop.
12	// If we wrap this body into a function, compilers (both gcc and clang)
13	// produce sligtly less efficient code.
14	//===----------------------------------------------------------------------===//
15	do {
16	const unsigned kAccessSize = 1 << kAccessSizeLog;
17	u64 *sp = &shadow_mem[idx];
18	old = LoadShadow(sp);
19	if (LIKELY(old.IsZero())) {
20	if (!stored) {
21	StoreIfNotYetStored(sp, &store_word);
22	stored = true;
23	}
24	break;
25	}
26	// is the memory access equal to the previous?
27	if (LIKELY(Shadow::Addr0AndSizeAreEqual(cur, old))) {
28	// same thread?
29	if (LIKELY(Shadow::TidsAreEqual(old, cur))) {
30	if (LIKELY(old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))) {
31	StoreIfNotYetStored(sp, &store_word);
32	stored = true;
33	}
34	break;
35	}
36	if (HappensBefore(old, thr)) {
37	if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) {
38	StoreIfNotYetStored(sp, &store_word);
39	stored = true;
40	}
41	break;
42	}
43	if (LIKELY(old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)))
44	break;
45	goto RACE;
46	}
47	// Do the memory access intersect?
48	if (Shadow::TwoRangesIntersect(old, cur, kAccessSize)) {
49	if (Shadow::TidsAreEqual(old, cur))
50	break;
51	if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
52	break;
53	if (LIKELY(HappensBefore(old, thr)))
54	break;
55	goto RACE;
56	}
57	// The accesses do not intersect.
58	break;
59	} while (0);

lib/tsan/tsan_vector_clock.cpp created+126

...	@@ -0,0 +1,126 @@
1	//===-- tsan_vector_clock.cpp ---------------------------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	//===----------------------------------------------------------------------===//
12	#include "tsan_vector_clock.h"
13
14	#include "sanitizer_common/sanitizer_placement_new.h"
15	#include "tsan_mman.h"
16
17	namespace __tsan {
18
19	#if TSAN_VECTORIZE
20	const uptr kVectorClockSize = kThreadSlotCount * sizeof(Epoch) / sizeof(m128);
21	#endif
22
23	VectorClock::VectorClock() { Reset(); }
24
25	void VectorClock::Reset() {
26	#if !TSAN_VECTORIZE
27	for (uptr i = 0; i < kThreadSlotCount; i++)
28	clk_[i] = kEpochZero;
29	#else
30	m128 z = _mm_setzero_si128();
31	m128* vclk = reinterpret_cast<m128*>(clk_);
32	for (uptr i = 0; i < kVectorClockSize; i++) _mm_store_si128(&vclk[i], z);
33	#endif
34	}
35
36	void VectorClock::Acquire(const VectorClock* src) {
37	if (!src)
38	return;
39	#if !TSAN_VECTORIZE
40	for (uptr i = 0; i < kThreadSlotCount; i++)
41	clk_[i] = max(clk_[i], src->clk_[i]);
42	#else
43	m128* __restrict vdst = reinterpret_cast<m128*>(clk_);
44	m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(src->clk_);
45	for (uptr i = 0; i < kVectorClockSize; i++) {
46	m128 s = _mm_load_si128(&vsrc[i]);
47	m128 d = _mm_load_si128(&vdst[i]);
48	m128 m = _mm_max_epu16(s, d);
49	_mm_store_si128(&vdst[i], m);
50	}
51	#endif
52	}
53
54	static VectorClock* AllocClock(VectorClock** dstp) {
55	if (UNLIKELY(!*dstp))
56	*dstp = New<VectorClock>();
57	return *dstp;
58	}
59
60	void VectorClock::Release(VectorClock** dstp) const {
61	VectorClock* dst = AllocClock(dstp);
62	dst->Acquire(this);
63	}
64
65	void VectorClock::ReleaseStore(VectorClock** dstp) const {
66	VectorClock* dst = AllocClock(dstp);
67	dst = this;
68	}
69
70	VectorClock& VectorClock::operator=(const VectorClock& other) {
71	#if !TSAN_VECTORIZE
72	for (uptr i = 0; i < kThreadSlotCount; i++)
73	clk_[i] = other.clk_[i];
74	#else
75	m128* __restrict vdst = reinterpret_cast<m128*>(clk_);
76	m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(other.clk_);
77	for (uptr i = 0; i < kVectorClockSize; i++) {
78	m128 s = _mm_load_si128(&vsrc[i]);
79	_mm_store_si128(&vdst[i], s);
80	}
81	#endif
82	return *this;
83	}
84
85	void VectorClock::ReleaseStoreAcquire(VectorClock** dstp) {
86	VectorClock* dst = AllocClock(dstp);
87	#if !TSAN_VECTORIZE
88	for (uptr i = 0; i < kThreadSlotCount; i++) {
89	Epoch tmp = dst->clk_[i];
90	dst->clk_[i] = clk_[i];
91	clk_[i] = max(clk_[i], tmp);
92	}
93	#else
94	m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);
95	m128* __restrict vclk = reinterpret_cast<m128*>(clk_);
96	for (uptr i = 0; i < kVectorClockSize; i++) {
97	m128 t = _mm_load_si128(&vdst[i]);
98	m128 c = _mm_load_si128(&vclk[i]);
99	m128 m = _mm_max_epu16(c, t);
100	_mm_store_si128(&vdst[i], c);
101	_mm_store_si128(&vclk[i], m);
102	}
103	#endif
104	}
105
106	void VectorClock::ReleaseAcquire(VectorClock** dstp) {
107	VectorClock* dst = AllocClock(dstp);
108	#if !TSAN_VECTORIZE
109	for (uptr i = 0; i < kThreadSlotCount; i++) {
110	dst->clk_[i] = max(dst->clk_[i], clk_[i]);
111	clk_[i] = dst->clk_[i];
112	}
113	#else
114	m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);
115	m128* __restrict vclk = reinterpret_cast<m128*>(clk_);
116	for (uptr i = 0; i < kVectorClockSize; i++) {
117	m128 c = _mm_load_si128(&vclk[i]);
118	m128 d = _mm_load_si128(&vdst[i]);
119	m128 m = _mm_max_epu16(c, d);
120	_mm_store_si128(&vdst[i], m);
121	_mm_store_si128(&vclk[i], m);
122	}
123	#endif
124	}
125
126	} // namespace __tsan

lib/tsan/tsan_vector_clock.h created+51

...	@@ -0,0 +1,51 @@
1	//===-- tsan_vector_clock.h -------------------------------------- C++ --===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file is a part of ThreadSanitizer (TSan), a race detector.
10	//
11	//===----------------------------------------------------------------------===//
12	#ifndef TSAN_VECTOR_CLOCK_H
13	#define TSAN_VECTOR_CLOCK_H
14
15	#include "tsan_defs.h"
16
17	namespace __tsan {
18
19	// Fixed-size vector clock, used both for threads and sync objects.
20	class VectorClock {
21	public:
22	VectorClock();
23
24	Epoch Get(Sid sid) const;
25	void Set(Sid sid, Epoch v);
26
27	void Reset();
28	void Acquire(const VectorClock* src);
29	void Release(VectorClock** dstp) const;
30	void ReleaseStore(VectorClock** dstp) const;
31	void ReleaseStoreAcquire(VectorClock** dstp);
32	void ReleaseAcquire(VectorClock** dstp);
33
34	VectorClock& operator=(const VectorClock& other);
35
36	private:
37	Epoch clk_[kThreadSlotCount] VECTOR_ALIGNED;
38	};
39
40	ALWAYS_INLINE Epoch VectorClock::Get(Sid sid) const {
41	return clk_[static_cast<u8>(sid)];
42	}
43
44	ALWAYS_INLINE void VectorClock::Set(Sid sid, Epoch v) {
45	DCHECK_GE(v, clk_[static_cast<u8>(sid)]);
46	clk_[static_cast<u8>(sid)] = v;
47	}
48
49	} // namespace __tsan
50
51	#endif // TSAN_VECTOR_CLOCK_H

src/Compilation.zig+5-4

 @@ -3759,13 +3759,14 @@ fn processOneJob(comp: *Compilation, job: Job, prog_node: *std.Progress.Node) !v
             const named_frame = tracy.namedFrame("libtsan");
             defer named_frame.end();
-            libtsan.buildTsan(comp, prog_node) catch |err| {
+            libtsan.buildTsan(comp, prog_node) catch |err| switch (err) {
-                // TODO Surface more error details.
+                error.OutOfMemory => return error.OutOfMemory,
-                comp.lockAndSetMiscFailure(
+                error.SubCompilationFailed => return, // error reported already
+                else => comp.lockAndSetMiscFailure(
                     .libtsan,
                     "unable to build TSAN library: {s}",
                     .{@errorName(err)},
-                );
+                ),
             };
         },
         .wasi_libc_crt_file => |crt_file| {

src/libcxx.zig+2

...	@@ -153,6 +153,7 @@ pub fn buildLibCXX(comp: Compilation, prog_node: std.Progress.Node) !void {	...	@@ -153,6 +153,7 @@ pub fn buildLibCXX(comp: Compilation, prog_node: std.Progress.Node) !void {
153	.root_strip = strip,	153	.root_strip = strip,
154	.link_libc = true,	154	.link_libc = true,
155	.lto = comp.config.lto,	155	.lto = comp.config.lto,
		156	.any_sanitize_thread = comp.config.any_sanitize_thread,
156	});	157	});
157		158
158	const root_mod = try Module.create(arena, .{	159	const root_mod = try Module.create(arena, .{
...	@@ -358,6 +359,7 @@ pub fn buildLibCXXABI(comp: Compilation, prog_node: std.Progress.Node) !void {	...	@@ -358,6 +359,7 @@ pub fn buildLibCXXABI(comp: Compilation, prog_node: std.Progress.Node) !void {
358	.link_libc = true,	359	.link_libc = true,
359	.any_unwind_tables = unwind_tables,	360	.any_unwind_tables = unwind_tables,
360	.lto = comp.config.lto,	361	.lto = comp.config.lto,
		362	.any_sanitize_thread = comp.config.any_sanitize_thread,
361	});	363	});
362		364
363	const root_mod = try Module.create(arena, .{	365	const root_mod = try Module.create(arena, .{

src/libtsan.zig+78-21

 @@ -6,7 +6,14 @@ const build_options = @import("build_options");
 const trace = @import("tracy.zig").trace;
 const Module = @import("Package/Module.zig");
-pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void {
+pub const BuildError = error{
+    OutOfMemory,
+    SubCompilationFailed,
+    ZigCompilerNotBuiltWithLLVMExtensions,
+    TSANUnsupportedCPUArchitecture,
+};
+pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) BuildError!void {
     if (!build_options.have_llvm) {
         return error.ZigCompilerNotBuiltWithLLVMExtensions;
     }
 @@ -37,7 +44,7 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void {
     const optimize_mode = comp.compilerRtOptMode();
     const strip = comp.compilerRtStrip();
-    const config = try Compilation.Config.resolve(.{
+    const config = Compilation.Config.resolve(.{
         .output_mode = output_mode,
         .link_mode = link_mode,
         .resolved_target = comp.root_mod.resolved_target,
 @@ -47,13 +54,20 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void {
         .root_optimize_mode = optimize_mode,
         .root_strip = strip,
         .link_libc = true,
-    });
+    }) catch |err| {
+        comp.setMiscFailure(
+            .libtsan,
+            "unable to build thread sanitizer runtime: resolving configuration failed: {s}",
+            .{@errorName(err)},
+        );
+        return error.SubCompilationFailed;
+    };
     const common_flags = [_][]const u8{
         "-DTSAN_CONTAINS_UBSAN=0",
     };
-    const root_mod = try Module.create(arena, .{
+    const root_mod = Module.create(arena, .{
         .global_cache_directory = comp.global_cache_directory,
         .paths = .{
             .root = .{ .root_dir = comp.zig_lib_directory },
 @@ -78,7 +92,14 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void {
         .cc_argv = &common_flags,
         .parent = null,
         .builtin_mod = null,
-    });
+    }) catch |err| {
+        comp.setMiscFailure(
+            .libtsan,
+            "unable to build thread sanitizer runtime: creating module failed: {s}",
+            .{@errorName(err)},
+        );
+        return error.SubCompilationFailed;
+    };
     var c_source_files = std.ArrayList(Compilation.CSourceFile).init(arena);
     try c_source_files.ensureUnusedCapacity(tsan_sources.len);
 @@ -92,7 +113,7 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void {
         try cflags.append("-nostdinc++");
         try cflags.append("-fvisibility-inlines-hidden");
-        try cflags.append("-std=c++14");
+        try cflags.append("-std=c++17");
         try cflags.append("-fno-rtti");
         c_source_files.appendAssumeCapacity(.{
 @@ -102,10 +123,11 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void {
         });
     }
-    const platform_tsan_sources = if (target.isDarwin())
+    const platform_tsan_sources = switch (target.os.tag) {
-        &darwin_tsan_sources
+        .ios, .macos, .watchos, .tvos => &darwin_tsan_sources,
-    else
+        .windows => &windows_tsan_sources,
-        &unix_tsan_sources;
+        else => &unix_tsan_sources,
+    };
     try c_source_files.ensureUnusedCapacity(platform_tsan_sources.len);
     for (platform_tsan_sources) |tsan_src| {
         var cflags = std.ArrayList([]const u8).init(arena);
 @@ -115,7 +137,7 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void {
         try cflags.append("-nostdinc++");
         try cflags.append("-fvisibility-inlines-hidden");
-        try cflags.append("-std=c++14");
+        try cflags.append("-std=c++17");
         try cflags.append("-fno-rtti");
         c_source_files.appendAssumeCapacity(.{
 @@ -155,10 +177,12 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void {
         try cflags.append("-I");
         try cflags.append(sanitizer_common_include_path);
+        try cflags.append("-I");
+        try cflags.append(tsan_include_path);
         try cflags.append("-nostdinc++");
         try cflags.append("-fvisibility-inlines-hidden");
-        try cflags.append("-std=c++14");
+        try cflags.append("-std=c++17");
         try cflags.append("-fno-rtti");
         c_source_files.appendAssumeCapacity(.{
 @@ -180,10 +204,12 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void {
         try cflags.append("-I");
         try cflags.append(sanitizer_common_include_path);
+        try cflags.append("-I");
+        try cflags.append(tsan_include_path);
         try cflags.append("-nostdinc++");
         try cflags.append("-fvisibility-inlines-hidden");
-        try cflags.append("-std=c++14");
+        try cflags.append("-std=c++17");
         try cflags.append("-fno-rtti");
         c_source_files.appendAssumeCapacity(.{
 @@ -204,7 +230,7 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void {
         try cflags.append("-nostdinc++");
         try cflags.append("-fvisibility-inlines-hidden");
-        try cflags.append("-std=c++14");
+        try cflags.append("-std=c++17");
         try cflags.append("-fno-rtti");
         c_source_files.appendAssumeCapacity(.{
 @@ -233,7 +259,7 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void {
         try cflags.append("-nostdinc++");
         try cflags.append("-fvisibility-inlines-hidden");
-        try cflags.append("-std=c++14");
+        try cflags.append("-std=c++17");
         try cflags.append("-fno-rtti");
         c_source_files.appendAssumeCapacity(.{
 @@ -245,7 +271,7 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void {
         });
     }
-    const sub_compilation = try Compilation.create(comp.gpa, arena, .{
+    const sub_compilation = Compilation.create(comp.gpa, arena, .{
         .local_cache_directory = comp.global_cache_directory,
         .global_cache_directory = comp.global_cache_directory,
         .zig_lib_directory = comp.zig_lib_directory,
 @@ -268,22 +294,36 @@ pub fn buildTsan(comp: *Compilation, prog_node: *std.Progress.Node) !void {
         .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
         .clang_passthrough_mode = comp.clang_passthrough_mode,
         .skip_linker_dependencies = true,
-    });
+    }) catch |err| {
+        comp.setMiscFailure(
+            .libtsan,
+            "unable to build thread sanitizer runtime: create compilation failed: {s}",
+            .{@errorName(err)},
+        );
+        return error.SubCompilationFailed;
+    };
     defer sub_compilation.destroy();
-    try comp.updateSubCompilation(sub_compilation, .libtsan, prog_node);
+    comp.updateSubCompilation(sub_compilation, .libtsan, prog_node) catch |err| {
+        comp.setMiscFailure(
+            .libtsan,
+            "unable to build thread sanitizer runtime: compilation failed: {s}",
+            .{@errorName(err)},
+        );
+        return error.SubCompilationFailed;
+    };
     assert(comp.tsan_static_lib == null);
     comp.tsan_static_lib = try sub_compilation.toCrtFile();
 }
 const tsan_sources = [_][]const u8{
-    "tsan_clock.cpp",
     "tsan_debugging.cpp",
     "tsan_external.cpp",
     "tsan_fd.cpp",
     "tsan_flags.cpp",
     "tsan_ignoreset.cpp",
+    "tsan_interceptors_memintrinsics.cpp",
     "tsan_interceptors_posix.cpp",
     "tsan_interface.cpp",
     "tsan_interface_ann.cpp",
 @@ -293,9 +333,12 @@ const tsan_sources = [_][]const u8{
     "tsan_md5.cpp",
     "tsan_mman.cpp",
     "tsan_mutexset.cpp",
+    "tsan_new_delete.cpp",
+    "tsan_platform_windows.cpp",
     "tsan_preinit.cpp",
     "tsan_report.cpp",
     "tsan_rtl.cpp",
+    "tsan_rtl_access.cpp",
     "tsan_rtl_mutex.cpp",
     "tsan_rtl_proc.cpp",
     "tsan_rtl_report.cpp",
 @@ -304,6 +347,7 @@ const tsan_sources = [_][]const u8{
     "tsan_suppressions.cpp",
     "tsan_symbolize.cpp",
     "tsan_sync.cpp",
+    "tsan_vector_clock.cpp",
 };
 const darwin_tsan_sources = [_][]const u8{
 @@ -318,9 +362,17 @@ const unix_tsan_sources = [_][]const u8{
     "tsan_platform_posix.cpp",
 };
+const windows_tsan_sources = [_][]const u8{
+    "tsan_platform_windows.cpp",
+};
 const sanitizer_common_sources = [_][]const u8{
     "sanitizer_allocator.cpp",
+    "sanitizer_chained_origin_depot.cpp",
     "sanitizer_common.cpp",
+    "sanitizer_coverage_win_dll_thunk.cpp",
+    "sanitizer_coverage_win_dynamic_runtime_thunk.cpp",
+    "sanitizer_coverage_win_weak_interception.cpp",
     "sanitizer_deadlock_detector1.cpp",
     "sanitizer_deadlock_detector2.cpp",
     "sanitizer_errno.cpp",
 @@ -335,8 +387,6 @@ const sanitizer_common_sources = [_][]const u8{
     "sanitizer_mac.cpp",
     "sanitizer_mutex.cpp",
     "sanitizer_netbsd.cpp",
-    "sanitizer_openbsd.cpp",
-    "sanitizer_persistent_allocator.cpp",
     "sanitizer_platform_limits_freebsd.cpp",
     "sanitizer_platform_limits_linux.cpp",
     "sanitizer_platform_limits_netbsd.cpp",
 @@ -351,15 +401,22 @@ const sanitizer_common_sources = [_][]const u8{
     "sanitizer_procmaps_linux.cpp",
     "sanitizer_procmaps_mac.cpp",
     "sanitizer_procmaps_solaris.cpp",
+    "sanitizer_range.cpp",
     "sanitizer_solaris.cpp",
+    "sanitizer_stack_store.cpp",
     "sanitizer_stoptheworld_fuchsia.cpp",
     "sanitizer_stoptheworld_mac.cpp",
+    "sanitizer_stoptheworld_win.cpp",
     "sanitizer_suppressions.cpp",
     "sanitizer_termination.cpp",
+    "sanitizer_thread_arg_retval.cpp",
     "sanitizer_thread_registry.cpp",
     "sanitizer_tls_get_addr.cpp",
     "sanitizer_type_traits.cpp",
     "sanitizer_win.cpp",
+    "sanitizer_win_dll_thunk.cpp",
+    "sanitizer_win_dynamic_runtime_thunk.cpp",
+    "sanitizer_win_weak_interception.cpp",
 };
 const sanitizer_nolibc_sources = [_][]const u8{

author	Andrew Kelley <andrew@ziglang.org> 2024-01-10 15:08:49-08:00
committer	GitHub <noreply@github.com> 2024-01-10 15:08:49-08:00
log	df6aed0fc3a85285ad9bf15c942a66255c2b0b31
tree	6b57572f4956e89c7c36b618187038fb6346521a
parent	aafff258973fb58f1a964eb050d5602ed8b9f60c
parent	71e7bef2a2731dbffc6a3673538bb4c678be4757
signature	Signed by PGP key 4AEE18F83AFDEB23

...	@@ -189,7 +189,7 @@ class SizeClassAllocator32 {	...	@@ -189,7 +189,7 @@ class SizeClassAllocator32 {
189	sci->free_list.push_front(b);	189	sci->free_list.push_front(b);
190	}	190	}
191		191
192	bool PointerIsMine(const void *p) {	192	bool PointerIsMine(const void *p) const {
193	uptr mem = reinterpret_cast<uptr>(p);	193	uptr mem = reinterpret_cast<uptr>(p);
194	if (SANITIZER_SIGN_EXTENDED_ADDRESSES)	194	if (SANITIZER_SIGN_EXTENDED_ADDRESSES)
195	mem &= (kSpaceSize - 1);	195	mem &= (kSpaceSize - 1);
...	@@ -198,8 +198,9 @@ class SizeClassAllocator32 {	...	@@ -198,8 +198,9 @@ class SizeClassAllocator32 {
198	return GetSizeClass(p) != 0;	198	return GetSizeClass(p) != 0;
199	}	199	}
200		200
201	uptr GetSizeClass(const void *p) {	201	uptr GetSizeClass(const void *p) const {
202	return possible_regions[ComputeRegionId(reinterpret_cast<uptr>(p))];	202	uptr id = ComputeRegionId(reinterpret_cast<uptr>(p));
		203	return possible_regions.contains(id) ? possible_regions[id] : 0;
203	}	204	}
204		205
205	void GetBlockBegin(const void p) {	206	void GetBlockBegin(const void p) {
...	@@ -237,13 +238,13 @@ class SizeClassAllocator32 {	...	@@ -237,13 +238,13 @@ class SizeClassAllocator32 {
237		238
238	// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone	239	// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
239	// introspection API.	240	// introspection API.
240	void ForceLock() NO_THREAD_SAFETY_ANALYSIS {	241	void ForceLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
241	for (uptr i = 0; i < kNumClasses; i++) {	242	for (uptr i = 0; i < kNumClasses; i++) {
242	GetSizeClassInfo(i)->mutex.Lock();	243	GetSizeClassInfo(i)->mutex.Lock();
243	}	244	}
244	}	245	}
245		246
246	void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {	247	void ForceUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
247	for (int i = kNumClasses - 1; i >= 0; i--) {	248	for (int i = kNumClasses - 1; i >= 0; i--) {
248	GetSizeClassInfo(i)->mutex.Unlock();	249	GetSizeClassInfo(i)->mutex.Unlock();
249	}	250	}
...	@@ -251,9 +252,9 @@ class SizeClassAllocator32 {	...	@@ -251,9 +252,9 @@ class SizeClassAllocator32 {
251		252
252	// Iterate over all existing chunks.	253	// Iterate over all existing chunks.
253	// The allocator must be locked when calling this function.	254	// The allocator must be locked when calling this function.
254	void ForEachChunk(ForEachChunkCallback callback, void *arg) {	255	void ForEachChunk(ForEachChunkCallback callback, void *arg) const {
255	for (uptr region = 0; region < kNumPossibleRegions; region++)	256	for (uptr region = 0; region < kNumPossibleRegions; region++)
256	if (possible_regions[region]) {	257	if (possible_regions.contains(region) && possible_regions[region]) {
257	uptr chunk_size = ClassIdToSize(possible_regions[region]);	258	uptr chunk_size = ClassIdToSize(possible_regions[region]);
258	uptr max_chunks_in_region = kRegionSize / (chunk_size + kMetadataSize);	259	uptr max_chunks_in_region = kRegionSize / (chunk_size + kMetadataSize);
259	uptr region_beg = region * kRegionSize;	260	uptr region_beg = region * kRegionSize;
...	@@ -292,9 +293,7 @@ class SizeClassAllocator32 {	...	@@ -292,9 +293,7 @@ class SizeClassAllocator32 {
292	return res;	293	return res;
293	}	294	}
294		295
295	uptr ComputeRegionBeg(uptr mem) {	296	uptr ComputeRegionBeg(uptr mem) const { return mem & ~(kRegionSize - 1); }
296	return mem & ~(kRegionSize - 1);
297	}
298		297
299	uptr AllocateRegion(AllocatorStats *stat, uptr class_id) {	298	uptr AllocateRegion(AllocatorStats *stat, uptr class_id) {
300	DCHECK_LT(class_id, kNumClasses);	299	DCHECK_LT(class_id, kNumClasses);
...	@@ -305,7 +304,7 @@ class SizeClassAllocator32 {	...	@@ -305,7 +304,7 @@ class SizeClassAllocator32 {
305	MapUnmapCallback().OnMap(res, kRegionSize);	304	MapUnmapCallback().OnMap(res, kRegionSize);
306	stat->Add(AllocatorStatMapped, kRegionSize);	305	stat->Add(AllocatorStatMapped, kRegionSize);
307	CHECK(IsAligned(res, kRegionSize));	306	CHECK(IsAligned(res, kRegionSize));
308	possible_regions.set(ComputeRegionId(res), static_cast<u8>(class_id));	307	possible_regions[ComputeRegionId(res)] = class_id;
309	return res;	308	return res;
310	}	309	}
311		310
...	@@ -354,7 +353,7 @@ class SizeClassAllocator32 {	...	@@ -354,7 +353,7 @@ class SizeClassAllocator32 {
354	DCHECK_GT(max_count, 0);	353	DCHECK_GT(max_count, 0);
355	TransferBatch *b = nullptr;	354	TransferBatch *b = nullptr;
356	constexpr uptr kShuffleArraySize = 48;	355	constexpr uptr kShuffleArraySize = 48;
357	uptr shuffle_array[kShuffleArraySize];	356	UNINITIALIZED uptr shuffle_array[kShuffleArraySize];
358	uptr count = 0;	357	uptr count = 0;
359	for (uptr i = region; i < region + n_chunks * size; i += size) {	358	for (uptr i = region; i < region + n_chunks * size; i += size) {
360	shuffle_array[count++] = i;	359	shuffle_array[count++] = i;

...	@@ -161,7 +161,7 @@ class SizeClassAllocator64 {	...	@@ -161,7 +161,7 @@ class SizeClassAllocator64 {
161	void ForceReleaseToOS() {	161	void ForceReleaseToOS() {
162	MemoryMapperT memory_mapper(*this);	162	MemoryMapperT memory_mapper(*this);
163	for (uptr class_id = 1; class_id < kNumClasses; class_id++) {	163	for (uptr class_id = 1; class_id < kNumClasses; class_id++) {
164	BlockingMutexLock l(&GetRegionInfo(class_id)->mutex);	164	Lock l(&GetRegionInfo(class_id)->mutex);
165	MaybeReleaseToOS(&memory_mapper, class_id, true /force/);	165	MaybeReleaseToOS(&memory_mapper, class_id, true /force/);
166	}	166	}
167	}	167	}
...	@@ -178,7 +178,7 @@ class SizeClassAllocator64 {	...	@@ -178,7 +178,7 @@ class SizeClassAllocator64 {
178	uptr region_beg = GetRegionBeginBySizeClass(class_id);	178	uptr region_beg = GetRegionBeginBySizeClass(class_id);
179	CompactPtrT *free_array = GetFreeArray(region_beg);	179	CompactPtrT *free_array = GetFreeArray(region_beg);
180		180
181	BlockingMutexLock l(&region->mutex);	181	Lock l(&region->mutex);
182	uptr old_num_chunks = region->num_freed_chunks;	182	uptr old_num_chunks = region->num_freed_chunks;
183	uptr new_num_freed_chunks = old_num_chunks + n_chunks;	183	uptr new_num_freed_chunks = old_num_chunks + n_chunks;
184	// Failure to allocate free array space while releasing memory is non	184	// Failure to allocate free array space while releasing memory is non
...	@@ -204,7 +204,7 @@ class SizeClassAllocator64 {	...	@@ -204,7 +204,7 @@ class SizeClassAllocator64 {
204	uptr region_beg = GetRegionBeginBySizeClass(class_id);	204	uptr region_beg = GetRegionBeginBySizeClass(class_id);
205	CompactPtrT *free_array = GetFreeArray(region_beg);	205	CompactPtrT *free_array = GetFreeArray(region_beg);
206		206
207	BlockingMutexLock l(&region->mutex);	207	Lock l(&region->mutex);
208	#if SANITIZER_WINDOWS	208	#if SANITIZER_WINDOWS
209	/* On Windows unmapping of memory during __sanitizer_purge_allocator is	209	/* On Windows unmapping of memory during __sanitizer_purge_allocator is
210	explicit and immediate, so unmapped regions must be explicitly mapped back	210	explicit and immediate, so unmapped regions must be explicitly mapped back
...	@@ -282,6 +282,8 @@ class SizeClassAllocator64 {	...	@@ -282,6 +282,8 @@ class SizeClassAllocator64 {
282	CHECK(kMetadataSize);	282	CHECK(kMetadataSize);
283	uptr class_id = GetSizeClass(p);	283	uptr class_id = GetSizeClass(p);
284	uptr size = ClassIdToSize(class_id);	284	uptr size = ClassIdToSize(class_id);
		285	if (!size)
		286	return nullptr;
285	uptr chunk_idx = GetChunkIdx(reinterpret_cast<uptr>(p), size);	287	uptr chunk_idx = GetChunkIdx(reinterpret_cast<uptr>(p), size);
286	uptr region_beg = GetRegionBeginBySizeClass(class_id);	288	uptr region_beg = GetRegionBeginBySizeClass(class_id);
287	return reinterpret_cast<void *>(GetMetadataEnd(region_beg) -	289	return reinterpret_cast<void *>(GetMetadataEnd(region_beg) -
...	@@ -300,9 +302,8 @@ class SizeClassAllocator64 {	...	@@ -300,9 +302,8 @@ class SizeClassAllocator64 {
300	UnmapWithCallbackOrDie((uptr)address_range.base(), address_range.size());	302	UnmapWithCallbackOrDie((uptr)address_range.base(), address_range.size());
301	}	303	}
302		304
303	static void FillMemoryProfile(uptr start, uptr rss, bool file, uptr *stats,	305	static void FillMemoryProfile(uptr start, uptr rss, bool file, uptr *stats) {
304	uptr stats_size) {	306	for (uptr class_id = 0; class_id < kNumClasses; class_id++)
305	for (uptr class_id = 0; class_id < stats_size; class_id++)
306	if (stats[class_id] == start)	307	if (stats[class_id] == start)
307	stats[class_id] = rss;	308	stats[class_id] = rss;
308	}	309	}
...	@@ -315,7 +316,7 @@ class SizeClassAllocator64 {	...	@@ -315,7 +316,7 @@ class SizeClassAllocator64 {
315	Printf(	316	Printf(
316	"%s %02zd (%6zd): mapped: %6zdK allocs: %7zd frees: %7zd inuse: %6zd "	317	"%s %02zd (%6zd): mapped: %6zdK allocs: %7zd frees: %7zd inuse: %6zd "
317	"num_freed_chunks %7zd avail: %6zd rss: %6zdK releases: %6zd "	318	"num_freed_chunks %7zd avail: %6zd rss: %6zdK releases: %6zd "
318	"last released: %6zdK region: 0x%zx\n",	319	"last released: %6lldK region: 0x%zx\n",
319	region->exhausted ? "F" : " ", class_id, ClassIdToSize(class_id),	320	region->exhausted ? "F" : " ", class_id, ClassIdToSize(class_id),
320	region->mapped_user >> 10, region->stats.n_allocated,	321	region->mapped_user >> 10, region->stats.n_allocated,
321	region->stats.n_freed, in_use, region->num_freed_chunks, avail_chunks,	322	region->stats.n_freed, in_use, region->num_freed_chunks, avail_chunks,
...	@@ -328,7 +329,7 @@ class SizeClassAllocator64 {	...	@@ -328,7 +329,7 @@ class SizeClassAllocator64 {
328	uptr rss_stats[kNumClasses];	329	uptr rss_stats[kNumClasses];
329	for (uptr class_id = 0; class_id < kNumClasses; class_id++)	330	for (uptr class_id = 0; class_id < kNumClasses; class_id++)
330	rss_stats[class_id] = SpaceBeg() + kRegionSize * class_id;	331	rss_stats[class_id] = SpaceBeg() + kRegionSize * class_id;
331	GetMemoryProfile(FillMemoryProfile, rss_stats, kNumClasses);	332	GetMemoryProfile(FillMemoryProfile, rss_stats);
332		333
333	uptr total_mapped = 0;	334	uptr total_mapped = 0;
334	uptr total_rss = 0;	335	uptr total_rss = 0;
...	@@ -353,13 +354,13 @@ class SizeClassAllocator64 {	...	@@ -353,13 +354,13 @@ class SizeClassAllocator64 {
353		354
354	// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone	355	// ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone
355	// introspection API.	356	// introspection API.
356	void ForceLock() NO_THREAD_SAFETY_ANALYSIS {	357	void ForceLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
357	for (uptr i = 0; i < kNumClasses; i++) {	358	for (uptr i = 0; i < kNumClasses; i++) {
358	GetRegionInfo(i)->mutex.Lock();	359	GetRegionInfo(i)->mutex.Lock();
359	}	360	}
360	}	361	}
361		362
362	void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS {	363	void ForceUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
363	for (int i = (int)kNumClasses - 1; i >= 0; i--) {	364	for (int i = (int)kNumClasses - 1; i >= 0; i--) {
364	GetRegionInfo(i)->mutex.Unlock();	365	GetRegionInfo(i)->mutex.Unlock();
365	}	366	}
...	@@ -623,7 +624,7 @@ class SizeClassAllocator64 {	...	@@ -623,7 +624,7 @@ class SizeClassAllocator64 {
623		624
624	static const uptr kRegionSize = kSpaceSize / kNumClassesRounded;	625	static const uptr kRegionSize = kSpaceSize / kNumClassesRounded;
625	// FreeArray is the array of free-d chunks (stored as 4-byte offsets).	626	// FreeArray is the array of free-d chunks (stored as 4-byte offsets).
626	// In the worst case it may reguire kRegionSize/SizeClassMap::kMinSize	627	// In the worst case it may require kRegionSize/SizeClassMap::kMinSize
627	// elements, but in reality this will not happen. For simplicity we	628	// elements, but in reality this will not happen. For simplicity we
628	// dedicate 1/8 of the region's virtual space to FreeArray.	629	// dedicate 1/8 of the region's virtual space to FreeArray.
629	static const uptr kFreeArraySize = kRegionSize / 8;	630	static const uptr kFreeArraySize = kRegionSize / 8;
...	@@ -634,8 +635,8 @@ class SizeClassAllocator64 {	...	@@ -634,8 +635,8 @@ class SizeClassAllocator64 {
634	return kUsingConstantSpaceBeg ? kSpaceBeg : NonConstSpaceBeg;	635	return kUsingConstantSpaceBeg ? kSpaceBeg : NonConstSpaceBeg;
635	}	636	}
636	uptr SpaceEnd() const { return SpaceBeg() + kSpaceSize; }	637	uptr SpaceEnd() const { return SpaceBeg() + kSpaceSize; }
637	// kRegionSize must be >= 2^32.	638	// kRegionSize should be able to satisfy the largest size class.
638	COMPILER_CHECK((kRegionSize) >= (1ULL << (SANITIZER_WORDSIZE / 2)));	639	static_assert(kRegionSize >= SizeClassMap::kMaxSize);
639	// kRegionSize must be <= 2^36, see CompactPtrT.	640	// kRegionSize must be <= 2^36, see CompactPtrT.
640	COMPILER_CHECK((kRegionSize) <= (1ULL << (SANITIZER_WORDSIZE / 2 + 4)));	641	COMPILER_CHECK((kRegionSize) <= (1ULL << (SANITIZER_WORDSIZE / 2 + 4)));
641	// Call mmap for user memory with at least this size.	642	// Call mmap for user memory with at least this size.
...	@@ -665,7 +666,7 @@ class SizeClassAllocator64 {	...	@@ -665,7 +666,7 @@ class SizeClassAllocator64 {
665	};	666	};
666		667
667	struct ALIGNED(SANITIZER_CACHE_LINE_SIZE) RegionInfo {	668	struct ALIGNED(SANITIZER_CACHE_LINE_SIZE) RegionInfo {
668	BlockingMutex mutex;	669	Mutex mutex;
669	uptr num_freed_chunks; // Number of elements in the freearray.	670	uptr num_freed_chunks; // Number of elements in the freearray.
670	uptr mapped_free_array; // Bytes mapped for freearray.	671	uptr mapped_free_array; // Bytes mapped for freearray.
671	uptr allocated_user; // Bytes allocated for user memory.	672	uptr allocated_user; // Bytes allocated for user memory.

...	@@ -33,7 +33,7 @@	...	@@ -33,7 +33,7 @@
33	INTERCEPTOR(int, statvfs, char path, void buf) {	33	INTERCEPTOR(int, statvfs, char path, void buf) {
34	void *ctx;	34	void *ctx;
35	COMMON_INTERCEPTOR_ENTER(ctx, statvfs, path, buf);	35	COMMON_INTERCEPTOR_ENTER(ctx, statvfs, path, buf);
36	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	36	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
37	// FIXME: under ASan the call below may write to freed memory and corrupt	37	// FIXME: under ASan the call below may write to freed memory and corrupt
38	// its metadata. See	38	// its metadata. See
39	// https://github.com/google/sanitizers/issues/321.	39	// https://github.com/google/sanitizers/issues/321.
...	@@ -99,7 +99,7 @@ INTERCEPTOR(int, getvfsstat, void *buf, SIZE_T bufsize, int flags) {	...	@@ -99,7 +99,7 @@ INTERCEPTOR(int, getvfsstat, void *buf, SIZE_T bufsize, int flags) {
99	INTERCEPTOR(int, statvfs1, const char path, void buf, int flags) {	99	INTERCEPTOR(int, statvfs1, const char path, void buf, int flags) {
100	void *ctx;	100	void *ctx;
101	COMMON_INTERCEPTOR_ENTER(ctx, statvfs1, path, buf, flags);	101	COMMON_INTERCEPTOR_ENTER(ctx, statvfs1, path, buf, flags);
102	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1);	102	if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, internal_strlen(path) + 1);
103	int res = REAL(statvfs1)(path, buf, flags);	103	int res = REAL(statvfs1)(path, buf, flags);
104	if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs90_sz);	104	if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs90_sz);
105	return res;	105	return res;

...	@@ -1267,8 +1267,6 @@ static void ioctl_table_fill() {	...	@@ -1267,8 +1267,6 @@ static void ioctl_table_fill() {
1267	_(TIOCGFLAGS, WRITE, sizeof(int));	1267	_(TIOCGFLAGS, WRITE, sizeof(int));
1268	_(TIOCSFLAGS, READ, sizeof(int));	1268	_(TIOCSFLAGS, READ, sizeof(int));
1269	_(TIOCDCDTIMESTAMP, WRITE, struct_timeval_sz);	1269	_(TIOCDCDTIMESTAMP, WRITE, struct_timeval_sz);
1270	_(TIOCRCVFRAME, READ, sizeof(uptr));
1271	_(TIOCXMTFRAME, READ, sizeof(uptr));
1272	_(TIOCPTMGET, WRITE, struct_ptmget_sz);	1270	_(TIOCPTMGET, WRITE, struct_ptmget_sz);
1273	_(TIOCGRANTPT, NONE, 0);	1271	_(TIOCGRANTPT, NONE, 0);
1274	_(TIOCPTSNAME, WRITE, struct_ptmget_sz);	1272	_(TIOCPTSNAME, WRITE, struct_ptmget_sz);
...	@@ -1406,7 +1404,7 @@ static void ioctl_table_fill() {	...	@@ -1406,7 +1404,7 @@ static void ioctl_table_fill() {
1406	_(URIO_SEND_COMMAND, READWRITE, struct_urio_command_sz);	1404	_(URIO_SEND_COMMAND, READWRITE, struct_urio_command_sz);
1407	_(URIO_RECV_COMMAND, READWRITE, struct_urio_command_sz);	1405	_(URIO_RECV_COMMAND, READWRITE, struct_urio_command_sz);
1408	#undef _	1406	#undef _
1409	} // NOLINT	1407	}
1410		1408
1411	static bool ioctl_initialized = false;	1409	static bool ioctl_initialized = false;
1412		1410

...	@@ -73,7 +73,7 @@ void DebugMutexInit() {	...	@@ -73,7 +73,7 @@ void DebugMutexInit() {
73	// Build adjacency matrix.	73	// Build adjacency matrix.
74	bool leaf[kMutexTypeMax];	74	bool leaf[kMutexTypeMax];
75	internal_memset(&leaf, 0, sizeof(leaf));	75	internal_memset(&leaf, 0, sizeof(leaf));
76	int cnt[kMutexTypeMax] = {};	76	int cnt[kMutexTypeMax];
77	internal_memset(&cnt, 0, sizeof(cnt));	77	internal_memset(&cnt, 0, sizeof(cnt));
78	for (int t = 0; t < kMutexTypeMax; t++) {	78	for (int t = 0; t < kMutexTypeMax; t++) {
79	mutex_type_count = t;	79	mutex_type_count = t;
...	@@ -174,7 +174,7 @@ struct InternalDeadlockDetector {	...	@@ -174,7 +174,7 @@ struct InternalDeadlockDetector {
174	if (max_idx != MutexInvalid && !mutex_can_lock[max_idx][type]) {	174	if (max_idx != MutexInvalid && !mutex_can_lock[max_idx][type]) {
175	Printf("%s: internal deadlock: can't lock %s under %s mutex\n", SanitizerToolName,	175	Printf("%s: internal deadlock: can't lock %s under %s mutex\n", SanitizerToolName,
176	mutex_meta[type].name, mutex_meta[max_idx].name);	176	mutex_meta[type].name, mutex_meta[max_idx].name);
177	PrintMutexPC(pc);	177	PrintMutexPC(locked[max_idx].pc);
178	CHECK(0);	178	CHECK(0);
179	}	179	}
180	locked[type].seq = ++sequence;	180	locked[type].seq = ++sequence;

...	@@ -554,7 +554,7 @@ unsigned struct_tms_sz = sizeof(struct tms);	...	@@ -554,7 +554,7 @@ unsigned struct_tms_sz = sizeof(struct tms);
554	unsigned struct_sigevent_sz = sizeof(struct sigevent);	554	unsigned struct_sigevent_sz = sizeof(struct sigevent);
555	unsigned struct_sched_param_sz = sizeof(struct sched_param);	555	unsigned struct_sched_param_sz = sizeof(struct sched_param);
556	unsigned struct_sockaddr_sz = sizeof(struct sockaddr);	556	unsigned struct_sockaddr_sz = sizeof(struct sockaddr);
557	unsigned ucontext_t_sz = sizeof(ucontext_t);	557	unsigned ucontext_t_sz(void *ctx) { return sizeof(ucontext_t); }
558	unsigned struct_rlimit_sz = sizeof(struct rlimit);	558	unsigned struct_rlimit_sz = sizeof(struct rlimit);
559	unsigned struct_timespec_sz = sizeof(struct timespec);	559	unsigned struct_timespec_sz = sizeof(struct timespec);
560	unsigned struct_sembuf_sz = sizeof(struct sembuf);	560	unsigned struct_sembuf_sz = sizeof(struct sembuf);
...	@@ -666,6 +666,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);	...	@@ -666,6 +666,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
666		666
667	int glob_nomatch = GLOB_NOMATCH;	667	int glob_nomatch = GLOB_NOMATCH;
668	int glob_altdirfunc = GLOB_ALTDIRFUNC;	668	int glob_altdirfunc = GLOB_ALTDIRFUNC;
		669	const int wordexp_wrde_dooffs = WRDE_DOOFFS;
669		670
670	unsigned path_max = PATH_MAX;	671	unsigned path_max = PATH_MAX;
671		672
...	@@ -2341,8 +2342,6 @@ unsigned IOCTL_TIOCDRAIN = TIOCDRAIN;	...	@@ -2341,8 +2342,6 @@ unsigned IOCTL_TIOCDRAIN = TIOCDRAIN;
2341	unsigned IOCTL_TIOCGFLAGS = TIOCGFLAGS;	2342	unsigned IOCTL_TIOCGFLAGS = TIOCGFLAGS;
2342	unsigned IOCTL_TIOCSFLAGS = TIOCSFLAGS;	2343	unsigned IOCTL_TIOCSFLAGS = TIOCSFLAGS;
2343	unsigned IOCTL_TIOCDCDTIMESTAMP = TIOCDCDTIMESTAMP;	2344	unsigned IOCTL_TIOCDCDTIMESTAMP = TIOCDCDTIMESTAMP;
2344	unsigned IOCTL_TIOCRCVFRAME = TIOCRCVFRAME;
2345	unsigned IOCTL_TIOCXMTFRAME = TIOCXMTFRAME;
2346	unsigned IOCTL_TIOCPTMGET = TIOCPTMGET;	2345	unsigned IOCTL_TIOCPTMGET = TIOCPTMGET;
2347	unsigned IOCTL_TIOCGRANTPT = TIOCGRANTPT;	2346	unsigned IOCTL_TIOCGRANTPT = TIOCGRANTPT;
2348	unsigned IOCTL_TIOCPTSNAME = TIOCPTSNAME;	2347	unsigned IOCTL_TIOCPTSNAME = TIOCPTSNAME;

...	@@ -45,7 +45,7 @@ extern unsigned struct_stack_t_sz;	...	@@ -45,7 +45,7 @@ extern unsigned struct_stack_t_sz;
45	extern unsigned struct_sched_param_sz;	45	extern unsigned struct_sched_param_sz;
46	extern unsigned struct_statfs_sz;	46	extern unsigned struct_statfs_sz;
47	extern unsigned struct_sockaddr_sz;	47	extern unsigned struct_sockaddr_sz;
48	extern unsigned ucontext_t_sz;	48	unsigned ucontext_t_sz(void *ctx);
49		49
50	extern unsigned struct_rlimit_sz;	50	extern unsigned struct_rlimit_sz;
51	extern unsigned struct_utimbuf_sz;	51	extern unsigned struct_utimbuf_sz;
...	@@ -394,6 +394,7 @@ struct __sanitizer_glob_t {	...	@@ -394,6 +394,7 @@ struct __sanitizer_glob_t {
394		394
395	extern int glob_nomatch;	395	extern int glob_nomatch;
396	extern int glob_altdirfunc;	396	extern int glob_altdirfunc;
		397	extern const int wordexp_wrde_dooffs;
397		398
398	extern unsigned path_max;	399	extern unsigned path_max;
399		400
...	@@ -2194,8 +2195,6 @@ extern unsigned IOCTL_TIOCDRAIN;	...	@@ -2194,8 +2195,6 @@ extern unsigned IOCTL_TIOCDRAIN;
2194	extern unsigned IOCTL_TIOCGFLAGS;	2195	extern unsigned IOCTL_TIOCGFLAGS;
2195	extern unsigned IOCTL_TIOCSFLAGS;	2196	extern unsigned IOCTL_TIOCSFLAGS;
2196	extern unsigned IOCTL_TIOCDCDTIMESTAMP;	2197	extern unsigned IOCTL_TIOCDCDTIMESTAMP;
2197	extern unsigned IOCTL_TIOCRCVFRAME;
2198	extern unsigned IOCTL_TIOCXMTFRAME;
2199	extern unsigned IOCTL_TIOCPTMGET;	2198	extern unsigned IOCTL_TIOCPTMGET;
2200	extern unsigned IOCTL_TIOCGRANTPT;	2199	extern unsigned IOCTL_TIOCGRANTPT;
2201	extern unsigned IOCTL_TIOCPTSNAME;	2200	extern unsigned IOCTL_TIOCPTSNAME;

...	@@ -17,6 +17,9 @@	...	@@ -17,6 +17,9 @@
17		17
18	namespace __sanitizer {	18	namespace __sanitizer {
19		19
		20	// Strip interceptor prefixes from function name.
		21	const char StripFunctionName(const char function);
		22
20	// Render the contents of "info" structure, which represents the contents of	23	// Render the contents of "info" structure, which represents the contents of
21	// stack frame "frame_no" and appends it to the "buffer". "format" is a	24	// stack frame "frame_no" and appends it to the "buffer". "format" is a
22	// string with placeholders, which is copied to the output with	25	// string with placeholders, which is copied to the output with
...	@@ -26,8 +29,7 @@ namespace __sanitizer {	...	@@ -26,8 +29,7 @@ namespace __sanitizer {
26	// will be turned into	29	// will be turned into
27	// " frame 10: function foo::bar() at my/file.cc:10"	30	// " frame 10: function foo::bar() at my/file.cc:10"
28	// You may additionally pass "strip_path_prefix" to strip prefixes of paths to	31	// You may additionally pass "strip_path_prefix" to strip prefixes of paths to
29	// source files and modules, and "strip_func_prefix" to strip prefixes of	32	// source files and modules.
30	// function names.
31	// Here's the full list of available placeholders:	33	// Here's the full list of available placeholders:
32	// %% - represents a '%' character;	34	// %% - represents a '%' character;
33	// %n - frame number (copy of frame_no);	35	// %n - frame number (copy of frame_no);
...	@@ -48,8 +50,7 @@ namespace __sanitizer {	...	@@ -48,8 +50,7 @@ namespace __sanitizer {
48	// %M - prints module basename and offset, if it is known, or PC.	50	// %M - prints module basename and offset, if it is known, or PC.
49	void RenderFrame(InternalScopedString buffer, const char format, int frame_no,	51	void RenderFrame(InternalScopedString buffer, const char format, int frame_no,
50	uptr address, const AddressInfo *info, bool vs_style,	52	uptr address, const AddressInfo *info, bool vs_style,
51	const char *strip_path_prefix = "",	53	const char *strip_path_prefix = "");
52	const char *strip_func_prefix = "");
53		54
54	bool RenderNeedsSymbolization(const char *format);	55	bool RenderNeedsSymbolization(const char *format);
55		56

...	@@ -15,7 +15,7 @@	...	@@ -15,7 +15,7 @@
15	#define SANITIZER_SYMBOLIZER_MAC_H	15	#define SANITIZER_SYMBOLIZER_MAC_H
16		16
17	#include "sanitizer_platform.h"	17	#include "sanitizer_platform.h"
18	#if SANITIZER_MAC	18	#if SANITIZER_APPLE
19		19
20	#include "sanitizer_symbolizer_internal.h"	20	#include "sanitizer_symbolizer_internal.h"
21		21
...	@@ -35,7 +35,6 @@ class AtosSymbolizer final : public SymbolizerTool {	...	@@ -35,7 +35,6 @@ class AtosSymbolizer final : public SymbolizerTool {
35		35
36	bool SymbolizePC(uptr addr, SymbolizedStack *stack) override;	36	bool SymbolizePC(uptr addr, SymbolizedStack *stack) override;
37	bool SymbolizeData(uptr addr, DataInfo *info) override;	37	bool SymbolizeData(uptr addr, DataInfo *info) override;
38	void LateInitialize() override;
39		38
40	private:	39	private:
41	AtosSymbolizerProcess *process_;	40	AtosSymbolizerProcess *process_;
...	@@ -43,6 +42,6 @@ class AtosSymbolizer final : public SymbolizerTool {	...	@@ -43,6 +42,6 @@ class AtosSymbolizer final : public SymbolizerTool {
43		42
44	} // namespace __sanitizer	43	} // namespace __sanitizer
45		44
46	#endif // SANITIZER_MAC	45	#endif // SANITIZER_APPLE
47		46
48	#endif // SANITIZER_SYMBOLIZER_MAC_H	47	#endif // SANITIZER_SYMBOLIZER_MAC_H

Merge pull request #18505 from ziglang/tsan